python 之进程池与线程池

1、什么进程池、线程池

池指的一个容器，该容器用来存放进程或线程，存放的数目是一定的

 

2、为什么要用池

用池是为了将并发的进程或线程数目控制在计算机可承受的范围内

 

为何要用进程进池？

当任务是计算密集型的情况下应该用进程来利用多核优势

为何要用线程进池？

当任务是IO密集型的情况下应该用线程减少开销

 

3、如何用？

 

1、同步调用 vs 异步调用

异步调用与同步调用指的是提交任务的两种方式

 

同步调用：提交完任务后，就在原地等待任务执行完毕，拿到运行结果/返回值后再执行下一行代码

同步调用下任务的执行是串行执行

 

异步调用：提交完任务后，不会原地等待任务执行完毕，结果？？？ 直接执行下一行代码

异步调用下任务的执行是并发执行

 

案例代码：

from concurrent.futures import ProcessPoolExecutor,ThreadPoolExecutor
import os
import time
import random

def task(x):
    print('%s is running' %os.getpid())
    time.sleep(random.randint(1,3))
    return x**2

if __name__ == '__main__':
    异步调用
    p=ProcessPoolExecutor() #不指定参数默认池的大写等于cpu的核数
    futrues=[]
    for i in range(10):
        futrue=p.submit(task,i)
        futrues.append(futrue)

    # pool.close()
    # pool.join()
    p.shutdown(wait=True)

    for futrue in futrues:
        print(futrue.result())
    print('主')


    #同步调用
    p=ProcessPoolExecutor() #不指定参数默认池的大写等于cpu的核数
    for i in range(10):
        res=p.submit(task,i).result()
        print(res)

    print('主')






from concurrent.futures import ProcessPoolExecutor,ThreadPoolExecutor
from threading import current_thread
import os
import time
import random
import requests


def get(url):
    print('%s GET %s' %(os.getpid(),url))
    time.sleep(2)
    response=requests.get(url)
    if response.status_code == 200:
        res=response.text
        return res

def parse(res):
    print('%s 解析[url]结果是 %s' % (os.getpid(), len(res)))

if __name__ == '__main__':
    p=ProcessPoolExecutor(10)

    urls=[
        'https://www.baidu.com',
        'https://www.python.org',
        'https://www.openstack.org',
        'https://www.taobao.com',
        'https://www.jd.com',
    ]
    obj_l=[]
    for url in urls:
        obj=p.submit(get,url)
        obj_l.append(obj)
    # p.shutdown(wait=True)
    # for obj in obj_l:
    #     res=obj.result()
    #     print('%s 解析[url]结果是 %s' %(os.getpid(),len(res)))

    print('主')



# 回调函数
from concurrent.futures import ProcessPoolExecutor,ThreadPoolExecutor
import os
import time
import random
import requests


def get(url):
    print('%s GET %s' %(os.getpid(),url))
    time.sleep(2)
    response=requests.get(url)
    if response.status_code == 200:
        res=response.text
        return res

def parse(obj):
    res=obj.result()
    print('%s 解析[url]结果是 %s' % (os.getpid(), len(res)))

if __name__ == '__main__':
    p=ProcessPoolExecutor(10)

    urls=[
        'https://www.baidu.com',
        'https://www.python.org',
        'https://www.openstack.org',
        'https://www.taobao.com',
        'https://www.jd.com',
    ]

    for url in urls:
        p.submit(get,url).add_done_callback(parse) # 回调函数会在任务运行完毕后自动触发，并且接收该任务对象

    print('主',os.getpid())



from concurrent.futures import ProcessPoolExecutor,ThreadPoolExecutor
import os
import time
import random
import requests


def get(url):
    print('%s GET %s' %(os.getpid(),url))
    time.sleep(2)
    response=requests.get(url)
    if response.status_code == 200:
        res=response.text
        return res

def parse(obj):
    res=obj.result()
    print('%s 解析[url]结果是 %s' % (os.getpid(), len(res)))

if __name__ == '__main__':
    p=ProcessPoolExecutor(10)

    urls=[
        'https://www.baidu.com',
        'https://www.python.org',
        'https://www.openstack.org',
        'https://www.taobao.com',
        'https://www.jd.com',
    ]

    for url in urls:
        p.submit(get,url).add_done_callback(parse) # 回调函数会在任务运行完毕后自动触发，并且接收该任务对象

    print('主',os.getpid())


# 线程池

from concurrent.futures import ThreadPoolExecutor
from threading import current_thread
import os
import time
import random

def task(x):
    print('%s is running' %current_thread().getName())
    time.sleep(random.randint(1,3))
    return x**2

def parse(obj):
    res=obj.result()
    print('%s 解析的结果为%s' %(current_thread().getName(),res))

if __name__ == '__main__':
    t=ThreadPoolExecutor(3)

    for i in range(10):
        t.submit(task,i).add_done_callback(parse)