进程线程__池

(1)Pool

1. p.apply(func ，args):在一个池工作进程中执行func,然后返回结果。

2. p.apply_async(func,args,callback):在一个池工作进程中执行func,然后返回结果。当func的结果变为可用时，将结果传递给callback。

3. p.close():关闭进程池，防止进一步操作。

4. p.join():等待所有工作进程退出。此方法只能在close（）之后调用

(2) concurrent.futures

1. ProcessPoolExecutor:进程池，提供异步调用；    ThreadPoolExecutor:线程池，提供异步调用

2. submit:异步提交任务

3. shutdown(wait=True)相当于pool.close()+pool.join()

4. result:取得结果

5. add_done_callback(f):回调函数

#进程池,异步提交
from multiprocessing import Pool
import time
import os

def work(i):
    print("target %s is running" %os.getpid())
    time.sleep(2)
    return i*2
    

if __name__=="__main__":
    p=Pool(5)
    
    res_list=[]
    for i in range(10):#10个任务
        res=p.apply_async(work,args=(i,))
        res_list.append(res)
        
    p.close()#不允许再往进程池里提交新得任务
    p.join()
    
    for res in res_list:
        print(res.get())
'''
target 17047 is running
target 17049 is running
target 17048 is running
target 17050 is running
target 17051 is running
target 17049 is running
target 17047 is running
target 17050 is running
target 17048 is running
target 17051 is running
0
2
4
6
8
10
12
14
16
18
'''

from concurrent.futures import ProcessPoolExecutor,ThreadPoolExecutor
from threading import current_thread
import os,time,random
def work(i):
    print("target %s is running" %os.getpid())
    time.sleep(2)
    return i*2
    

if __name__=="__main__":
    executor=ProcessPoolExecutor(5)
    futures=[]
    for i in range(10):
        future=executor.submit(work,i)
        futures.append(future)
    executor.shutdown()
    for future in futures:        
        print(future.result())
'''
target 24421 is running
target 24422 is running
target 24425 is running
target 24423 is running
target 24424 is running
target 24421 is running
target 24425 is running
target 24422 is running
target 24423 is running
target 24424 is running
0
2
4
6
8
10
12
14
16
18
'''

 

 

 

回调函数

#进程池：把时间长的程序放入进程池，把时间短的程序放入回调函数
import os,requests
from multiprocessing import Pool
def get_page(url):
    print("%s get %s" %(os.getpid(),url))
    respone=requests.get(url)
    if respone.status_code==200:
        return {"url":url,"text":respone.text}
def parse_page(dic):
    parse_res="%s size: %s \n" %(dic["url"],len(dic["text"]))
    print(parse_res)
        
if __name__=="__main__":
    urls=[
        'https://www.baidu.com',
        'https://bbs.csdn.net/topics/396220247',
        'https://bbs.csdn.net/topics/396210757',
        'https://bbs.csdn.net/topics/396205902',
        'https://bbs.csdn.net/topics/396196538'
    ]
    p=Pool(4)
    for url in urls:
        #get_page函数的返回值作为parse_page函数的参数
        p.apply_async(get_page,args=(url,),callback=parse_page)  #get_page返回值为字典
    p.close()
    p.join()
    
'''
165874 get https://www.baidu.com
165875 get https://bbs.csdn.net/topics/396220247
165876 get https://bbs.csdn.net/topics/396210757
165877 get https://bbs.csdn.net/topics/396205902
165874 get https://bbs.csdn.net/topics/396196538
https://www.baidu.com size: 2443 

https://bbs.csdn.net/topics/396210757 size: 175310 

https://bbs.csdn.net/topics/396205902 size: 144902 

https://bbs.csdn.net/topics/396196538 size: 177225 

https://bbs.csdn.net/topics/396220247 size: 238247

'''

 

#concurrent.futures
import os,requests
from concurrent.futures import ProcessPoolExecutor,ThreadPoolExecutor
def get_page(url):
    print("%s get %s" %(os.getpid(),url))
    respone=requests.get(url)
    if respone.status_code==200:
        return {"url":url,"text":respone.text}
def parse_page(obj):
    dic=obj.result()
    parse_res="%s size: %s \n" %(dic["url"],len(dic["text"]))
    print(parse_res)
        
if __name__=="__main__":
    urls=[
        'https://www.baidu.com',
        'https://bbs.csdn.net/topics/396220247',
        'https://bbs.csdn.net/topics/396210757',
        'https://bbs.csdn.net/topics/396205902',
        'https://bbs.csdn.net/topics/396196538'
    ]
    p=ProcessPoolExecutor(3)
    for url in urls:
        #把get_page的返回值直接作为parse_page的参数
        p.submit(get_page,url).add_done_callback(parse_page) #返回值是对象
    p.shutdown()
'''
126617 get https://www.baidu.com
126618 get https://bbs.csdn.net/topics/396220247
126619 get https://bbs.csdn.net/topics/396210757
126617 get https://bbs.csdn.net/topics/396205902
https://www.baidu.com size: 2443 

126618 get https://bbs.csdn.net/topics/396196538
https://bbs.csdn.net/topics/396220247 size: 183020 

https://bbs.csdn.net/topics/396210757 size: 87029 

https://bbs.csdn.net/topics/396205902 size: 145358 

https://bbs.csdn.net/topics/396196538 size: 211907

'''