python-进程四(共享与通信)
前言:进程之间内存资源都是相互独立的,互不影响和干涉。
那么如何让进程能共享资源或通信呢?python中提供了一些方法,代码如下
1)共享队列
# 进程同步1:使用Queue实现 # 消费者与生产者模式 from multiprocessing import Process, Queue import time, random def write(q): for value in ['A', 'B', 'C']: print('Put %s to queue...'% value) q.put(value) time.sleep(random.random()) def read(q): time.sleep(1) while not q.empty(): print('Get %s from queue.' % q.get(True)) time.sleep(1) if __name__=='__main__': q = Queue() pw = Process(target=write, args=(q,)) pr = Process(target=write, args=(q,)) pw.start() pr.start() pw.join() pr.join() print('Done!')
2)JoinableQueue
import multiprocessing import time # 继承了Process,必须要实现run方法,因为这是start调用的方法 # multiprocessing.JoinableQueue(),这也是进程里面的队列, # 先进先出,取出来的对象是个任务对象(函数或者是一个对象), # 在执行这个队列的时候,需要告诉这个队列当前任务执行完了,需要调用一下task_done() # 如果队列里面的所有任务都被执行完了,且每一个任务都调用了task_done()方法,认为此任务的所有任务被执行完了 # 最后有几个进程,需要给这个队列里面加入几个None,作为任务结束的标志 class Consumer(multiprocessing.Process): # 消费者的类:参数1是任务队列,参数2为结果队列 # 派生进程 def __init__(self, task_queue, result_queue): super().__init__() # multiprocessing.Process.__init__(self) self.task_queue = task_queue self.result_queue = result_queue # 重写原进程的run方法 def run(self): proc_name = self.name # 获取进程名字 while True: next_task = self.task_queue.get() # 获取任务队列里面的任务对象 if next_task is None: # Poison pill means shutdown print(('%s: Exiting' % proc_name)) self.task_queue.task_done() break print(('%s: %s' % (proc_name, next_task))) answer = next_task() # __call__() self.task_queue.task_done() self.result_queue.put(answer) return class Task(object): # 进程执行的任务,返回了一个表达式的字符串 def __init__(self, a, b): self.a = a self.b = b def __call__(self): time.sleep(0.1) # pretend to take some time to do the work return '%s * %s = %s' % (self.a, self.b, self.a * self.b) def __str__(self): return '%s * %s' % (self.a, self.b) if __name__ == '__main__': # 新建的一个任务队列 tasks = multiprocessing.JoinableQueue() # 普通的跨进程队列 results = multiprocessing.Queue() # 计算机器cpu的核数 num_consumers = multiprocessing.cpu_count() print(('Creating %d consumers' % num_consumers)) # 创建cup核数量个的子进程 consumers = [Consumer(tasks, results) for i in range(num_consumers)] # 依次启动子进程 for w in consumers: w.start() num_jobs = 10 for i in range(num_jobs): tasks.put(Task(i, i)) # 毒丸:有几个进程,放几个None # 进程执行的是run方法, 里面有死循环, 只有遇到None才会跳出死循环,结束任务 for i in range(num_consumers): tasks.put(None) # 等待所有的任务执行完 tasks.join() # 从结果队列中打印任务的执行结果 while num_jobs: result = results.get() print('Result: %s' % result) num_jobs -= 1
3)信号传递-Event
import multiprocessing import time def wait_for_event(e): print('wait_for_event: starting') e.wait() # 等待收到能执行信号,如果一直未收到将一直阻塞 print('wait_for_event: e.is_set()->', e.is_set()) def wait_for_event_timeout(e, t): print('wait_for_event_timeout: starting') e.wait(t) # 等待t秒超时,此时Event的状态仍未未设置,继续执行 print('wait_for_event_timeout: e.is_set()->', e.is_set()) e.set() # 初始内部标志为真 if __name__ == '__main__': e = multiprocessing.Event() # 创建信号 print("begin,e.is_set()", e.is_set()) w1 = multiprocessing.Process(name='block', target=wait_for_event, args=(e,)) w1.start() w2 = multiprocessing.Process(name='nonblock', target=wait_for_event_timeout, args=(e, 2)) # 可将2改为5,看看执行结果 w2.start() print('main: waiting before calling Event.set()') time.sleep(3) # e.set() #可注释此句话看效果 print('main: event is set') 结果: begin,e.is_set() False main: waiting before calling Event.set() wait_for_event_timeout: starting wait_for_event: starting wait_for_event_timeout: e.is_set()-> False wait_for_event: e.is_set()-> True main: event is set
4)管道-Pipe
# 进程间信息传递(使用管道-Pipe,只能两个进程间的一对一,不能一对多,多对一,而且send和recv也是只能一对一) import multiprocessing as mp def proc_1(pipe): pipe.send('hello') print('proc_1 received: %s' %pipe.recv()) pipe.send("what is your name?") print('proc_1 received: %s' %pipe.recv()) def proc_2(pipe): print('proc_2 received: %s' %pipe.recv()) pipe.send('hello, too') print('proc_2 received: %s' %pipe.recv()) pipe.send("I don't tell you!") if __name__ == '__main__': # 创建一个管道对象pipe pipe = mp.Pipe() print(len(pipe)) print(type(pipe)) # 将第一个pipe对象传给进程1 p1 = mp.Process(target=proc_1, args=(pipe[0], )) # 将第二个pipe对象传给进程2 p2 = mp.Process(target=proc_2, args=(pipe[1], )) p2.start() p1.start() p2.join() p1.join() 结果: 2 <class 'tuple'> proc_2 received: hello proc_1 received: hello, too proc_2 received: what is your name? proc_1 received: I don't tell you!
5)Condition
# 进程间的通信(使用Condition:notify_all与wait,这个可以一对多,就是可以通知和多个wait状态的进程) import multiprocessing as mp import time def consumer(cond): # 消费者函数 with cond: # 获得一个底层锁,获得了锁才能调用wait和notify_all() 或 notify() print("consumer before wait") cond.wait() # 等待消费 print("consumer after wait") def producer(cond): # 生产者函数 with cond: print("producer before notifyAll") cond.notify_all() # 通知消费者可以消费了 print("producer after notifyAll") if __name__ == '__main__': condition = mp.Condition() p1 = mp.Process(name="p1", target=consumer, args=(condition,)) p2 = mp.Process(name="p2", target=consumer, args=(condition,)) p3 = mp.Process(name="p3", target=producer, args=(condition,)) p1.start() time.sleep(2) p2.start() time.sleep(2) p3.start()
6)共享数字、队列
from multiprocessing import Process, Value, Array def f(n, a): n.value = 3.1415927 print('子进程改变的n', n) for i in range(len(a)): a[i] = -a[i] print('子进程改变的元素: ', a[i]) if __name__ == '__main__': num = Value('d', 0.0) # 创建一个进程间共享的数字类型,默认值为0,d 的类型为双精度小数 arr = Array('i', range(10)) # 创建一个进程间共享的数组类型,初始值为range[10],i是指整型 p = Process(target=f, args=(num, arr)) p.start() p.join() print(num.value) # 获取共享变量num的值 print(arr[:]) 结果: 子进程改变的n 3.1415927 子进程改变的元素: 0 子进程改变的元素: -1 子进程改变的元素: -2 子进程改变的元素: -3 子进程改变的元素: -4 子进程改变的元素: -5 子进程改变的元素: -6 子进程改变的元素: -7 子进程改变的元素: -8 子进程改变的元素: -9 3.1415927 [0, -1, -2, -3, -4, -5, -6, -7, -8, -9] # 结论:Value, Array 使 数字和序列可以共享
6)Manager共享模块
# 共享字符串 from multiprocessing import Process, Manager from ctypes import c_char_p def greet(shareStr): shareStr.value = shareStr.value + ", World!" if __name__ == '__main__': manager = Manager() shareStr = manager.Value(c_char_p, "Hello") # 字符串共享 process = Process(target=greet, args=(shareStr,)) process.start() process.join() print(shareStr.value) 结果: Hello, World! # 用Manager共享列表和字典的操作 from multiprocessing import Process, Manager def f( shareDict, shareList ): shareDict[1] = '1' shareDict['2'] = 2 shareDict[0.25] = None shareList.reverse() # 翻转列表 if __name__ == '__main__': manager = Manager() shareDict = manager.dict() # 创建共享的字典类型 shareList = manager.list(range(10)) # 创建共享的列表类型 p = Process(target=f, args=(shareDict, shareList)) p.start() p.join() print(shareDict) print(shareList) 结果: {1: '1', '2': 2, 0.25: None} [9, 8, 7, 6, 5, 4, 3, 2, 1, 0] # Manager 实现的进程池的队列, 请注意:from multiprocessing import Queue无法在进程池中实现共享,只能在Process类生成的进程间可以共享。 from multiprocessing import Pool,Manager def func(q): print("*"*10) q.put("12346") if __name__ == "__main__": manager = Manager() q = manager.Queue() pool = Pool(processes=4) for i in range(5): pool.apply_async(func,(q,)) pool.close() pool.join() print(q.qsize()) 结果: ********** ********** ********** ********** ********** 5
6)共享实例对象
# 类实例在不同进程中的共享 import time, os from multiprocessing import Pool, Lock from multiprocessing.managers import BaseManager class MyManager(BaseManager): # 生产了一个BaseManager的子类 pass def Manager(): # 生成一个MyManager的实例 m = MyManager() m.start() return m class Counter: # 在不同进程中共享的实例的类 def __init__(self, value): self.val = value self.lock = Lock() def increment(self): # 使用锁来累加1 with self.lock: self.val += 1 def value(self): # 使用锁来读取数据 with self.lock: return self.val MyManager.register('Counter', Counter) # 将Counter类注册到MyManager的类里面 def long_time_task(name, counter): # 进程的任务函数 time.sleep(0.2) print('Run task %s (%s)...\n' % (name, os.getpid())) start = time.time() for i in range(50): time.sleep(0.01) counter.increment() end = time.time() print('Task %s runs %0.2f seconds.' % (name, (end - start))) if __name__ == '__main__': manager = Manager() counter = manager.Counter(0) # 创建全局共享的Counter类实例,Counter类的初始值0 print('Parent process %s.' % os.getpid()) p = Pool() for i in range(5): p.apply_async(long_time_task, args=(str(i), counter)) print('Waiting for all subprocesses done...') p.close() p.join() print('All subprocesses done.') print(counter.value()) 结果: Parent process 93864. Waiting for all subprocesses done... Run task 0 (94144)... Run task 1 (50392)... Run task 3 (87032)... Run task 2 (91040)... Run task 4 (88856)... Task 0 runs 0.54 seconds. Task 3 runs 0.54 seconds. Task 1 runs 0.55 seconds. Task 4 runs 0.54 seconds. Task 2 runs 0.54 seconds. All subprocesses done. 250