python多进程执行任务

from tqdm import tqdm
from multiprocessing import Pool
import functools
from pymongo import MongoClient
mdb = MongoClient('120.xx.26.xx:20002', username='xx', password='xxxxx')# 三种main的写法只写一种即可def create_data(image): # TODO 具体处理逻辑
print(image)
return str(image)def main_deal(): num_processor = 20
p = Pool(num_processor)
images = mdb['db_name']['image'].find(no_cursor_timeout=True).batch_size(200) fw = open('result.txt', 'w+')
for result in tqdm(p.imap(create_data, images), total=images.count()):
fw.write(result + '\n')
fw.close() for _ in tqdm(p.imap_unordered(create_data, images)):
pass
p.close()
p.join()def main_deal(): num_processor = 20
p = Pool(num_processor) images = mdb['goodlook']['image_generated_data'].find(no_cursor_timeout=True).batch_size(200)
fw = open('result.txt', 'w+')
for result in tqdm(p.imap_unordered(create_data, images)):
fw.write(result + '\n')
fw.close() p.close()
p.join()def main_deal():
num_processor = 20
p = Pool(num_processor) images = mdb['goodlook']['image_generated_data'].find(no_cursor_timeout=True).batch_size(200)
fw = open('result.txt', 'w+') pt = functools.partial(create_data)
for result in tqdm(p.imap_unordered(pt, images)):
fw.write(result + '\n')
fw.close() p.close()
p.join()if __name__ == '__main__':
main_deal()