Python - Einfaches Multithread-Beispiel

import threading

urlTaskList = [
# your tasks here
]

parallelism = 4

# here is the task you want to run
def subTask(urlList):
  import urllib2

  for url in urlList:
    data = urllib2.urlopen(url).read()
  
    # process the data here
    # ...


# this method is used to seperate task chunk
def splitChunk(tasks):
  """ Yield successive n-sized chunks from l.
  """
  for i in xrange(0, len(l), n):
    yield l[i:i+n]


threadList = list()
chunkList = list(splitChunk(urlTaskList))


for subChunk in chunkList:
  worker = threading.Thread(target=subTask, args=(subChunk))
  worker.start()
  threadList.append(worker)
  

# wait for every thread finish its work
for worker in threadList:
  worker.join()

Recommended Posts