Python 多进程数据共享及异步调用 multiprocessing Manager ThreadPoolExecutor

用 Manager.dict 的例子（多进程数据共享）：

import multiprocessing, time

sum = 0


def worker(d, key):
    d[key] = key * 2
    print("Individual d : ", d[key])


if __name__ == '__main__':
    mgr = multiprocessing.Manager()
    d = mgr.dict()
    jobs = [multiprocessing.Process(target=worker, args=(d, i)) for i in range(1, 11)]
    for j in jobs:
        j.start()
    print("Start: ", sum)
    time.sleep(2)
    for i in dict(d):
        sum = sum + d[i]

    print("End: ", sum)

 # Start: 0
 # Individual d : 2
 # Individual d : 4
 # Individual d : 6
 # Individual d : 8
 # Individual d : 10
 # Individual d : 12
 # Individual d : 14
 # Individual d : 16
 # Individual d : 18
 # Individual d : 20
 # End: 110

 

用 Manager.list 的例子（多进程数据共享）：

import multiprocessing, time

sum = 0


def worker(l, v):
    l.append(v)
    print("Individual l : ", l)


if __name__ == '__main__':
    mgr = multiprocessing.Manager()
    l = mgr.list()
    jobs = [multiprocessing.Process(target=worker, args=(l, i)) for i in range(1, 11)]
    for j in jobs:
        j.start()
    print("Start: ", sum)
    time.sleep(2)
    for i in list(l):
        sum = sum + i

    print("End: ", sum)

 # Start: 0
 # Individual l : [2]
 # Individual l : [2, 1]
 # Individual l : [2, 1, 5]
 # Individual l : [2, 1, 5, 6]
 # Individual l : [2, 1, 5, 6, 3]
 # Individual l : [2, 1, 5, 6, 3, 4]
 # Individual l : [2, 1, 5, 6, 3, 4, 9]
 # Individual l : [2, 1, 5, 6, 3, 4, 9, 7]
 # Individual l : [2, 1, 5, 6, 3, 4, 9, 7, 8]
 # Individual l : [2, 1, 5, 6, 3, 4, 9, 7, 8, 10]
 # End: 55

 

用 ThreadPoolExecutor 的例子（异步调用，及回调函数）：

import time
from concurrent.futures import ThreadPoolExecutor

name_list = ['A', 'B', 'C', 'D', 'E', 'F', 'G']
enhanced_list = []


def say_hello(i):
    print("Hello ", i)
    time.sleep(1)
    return i + "-" + i


def call_back(res):
    global enhanced_list
    res = res.result()
    enhanced_list = enhanced_list + [res]
    print(res, enhanced_list)


def main():
    executor = ThreadPoolExecutor(max_workers=2)

    for i in name_list:
        executor.submit(say_hello, i).add_done_callback(call_back)
    executor.shutdown(wait=False)


if __name__ == '__main__':
    print("Start...")
    main()
    print("End...")


 # Start...
 # Hello  A
 # Hello  B
 # End...
 # B-B ['B-B']
 # A-A ['B-B', 'A-A']
 # Hello  C
 # Hello  D
 # C-C ['B-B', 'A-A', 'C-C']
 # D-D ['B-B', 'A-A', 'C-C', 'D-D']
 # Hello  E
 # Hello  F
 # F-F ['B-B', 'A-A', 'C-C', 'D-D', 'F-F']
 # Hello  G
 # E-E ['B-B', 'A-A', 'C-C', 'D-D', 'F-F', 'E-E']
 # G-G ['B-B', 'A-A', 'C-C', 'D-D', 'F-F', 'E-E', 'G-G']

 

（多进程数据共享）