concurrent.future模块 - 进程池和线程池

concurrent.future模块 - 进程池和线程池

from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor

开进程池
    executor = ProcessPoolExecutor(max_workers=4)
        max_works可以不写,写的话直接写参数就行

开线程池
    executor = ThreadPoolExecutor(max_workers=4)
        max_works可以不写,写的话直接写参数就行

提交任务
    for i in range(10):
        future = executor.submit(work, i) # 可以拿到结果
    future = executor.map(work, range(10) # 没有返回值，不能拿到结果,只能传递可迭代对象
拿结果
    future.result()

 

开进程池

from concurrent.futures import ProcessPoolExecutor,ThreadPoolExecutor
import time,random,os

def work(n):
    print("%s is running"%os.getpid())
    time.sleep((random.randint(1,3)))
    return n**2


if __name__ == '__main__':
    executor = ProcessPoolExecutor(4)
    futures = []
    for i in range(10):
        future = executor.submit(work,i) #异步提交
        futures.append(future)

    executor.shutdown(wait=True) # 相当于close和join
    print("主")
    for obj in futures:
        print(obj.result())

from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor
import time, random, os

def work(n):
    print("%s is running" % os.getpid())
    time.sleep((random.randint(1, 3)))
    return n ** 2


if __name__ == '__main__':
    executor = ProcessPoolExecutor(4)
    for i in range(10):
        future = executor.submit(work, i)
        print(future.result())  # 相当于同步提交

    executor.shutdown(wait=True)  # 相当于close和join
    print("主")

 

开线程池

from concurrent.futures import ProcessPoolExecutor,ThreadPoolExecutor
import time,random,os

def work(n):
    print("%s is running"%os.getpid())
    time.sleep((random.randint(1,3)))
    return n**2


if __name__ == '__main__':
    executor = ThreadPoolExecutor()
    futures = []
    for i in range(10):
        future = executor.submit(work,i) #异步提交
        futures.append(future)

    executor.shutdown(wait=True) # 相当于close和join
    print("主")
    for obj in futures:
        print(obj.result())

from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor
import time, random, os

def work(n):
    print("%s is running" % os.getpid())
    time.sleep((random.randint(1, 3)))
    return n ** 2


if __name__ == '__main__':
    executor = ThreadPoolExecutor()
    for i in range(10):
        future = executor.submit(work, i)
        print(future.result())  # 相当于同步提交

    executor.shutdown(wait=True)  # 相当于close和join
    print("主")

from concurrent.futures import ProcessPoolExecutor,ThreadPoolExecutor
import time,random,os

def work(n):
    print('%s is running' % os.getpid())
    time.sleep(random.randint(1,3))
    return n**2

if __name__ == '__main__':
    executor=ProcessPoolExecutor()
    # futures=[]
    # for i in range(10):
    #     # future=executor.submit(work,i)
    #     # futures.append(future)
    #     futures.append(executor.submit(work,i))

    futures=[executor.submit(work,i) for i in range(10)]

    executor.shutdown(wait=True)
    print('主')
    for i in futures:
        print(i.result())

 

map方法

map方法--不能拿到结果
from concurrent.futures import ProcessPoolExecutor,ThreadPoolExecutor
import time,random,os

def work(n):
    print("%s is running"%os.getpid())
    time.sleep((random.randint(1,3)))
    return n**2


if __name__ == '__main__':
    executor = ThreadPoolExecutor()
    executor.map(work,range(10))
    executor.shutdown(wait=True) # 相当于close和join
    print("主")

回调函数
submit没有callback参数，直接future.submit(函数,参数).add_done_callback(函数)
    参数就是future.submit(回调函数,参数)的对象
    取值要传到回调函数里面后，future.result()

from concurrent.futures import ThreadPoolExecutor,ProcessPoolExecutor
import requests
import os

def get_page(url):
    print('<进程%s> get %s' %(os.getpid(),url))
    respone=requests.get(url)
    if respone.status_code == 200:
        return {'url':url,'text':respone.text}

def parse_page(res):
    res=res.result()
    print('<进程%s> parse %s' %(os.getpid(),res['url']))
    parse_res='url:<%s> size:[%s]\n' %(res['url'],len(res['text']))
    with open('db.txt','a') as f:
        f.write(parse_res)


if __name__ == '__main__':
    urls=[
        'https://www.baidu.com',
        'https://www.python.org',
        'https://www.openstack.org',
        'https://help.github.com/',
        'http://www.sina.com.cn/'
    ]

    p=ProcessPoolExecutor()
    for url in urls:
        p.submit(get_page,url).add_done_callback(parse_page) #parse_page拿到的是一个future对象obj，需要用obj.result()拿到结果

 

抛异常--只能识别concurrent内部的异常
    future.exception(时间)  # 如果超出指定时间还没执行完任务，就抛出concurrent的内部异常
    不写时间的时候，相当于join
future.cancel() -- 取消

from concurrent.futures import ProcessPoolExecutor,ThreadPoolExecutor
import time,random,os

def work(n):
    # try:
    #     time.sleep(random.randint(1,3))
    #     raise TypeError
    #     return n**2
    # except TypeError as e:
    #     #logger.log(e)
    #     pass
    print('%s is running' %os.getpid())
    time.sleep(3)
    # raise NotImplementedError
if __name__ == '__main__':
    executor=ProcessPoolExecutor()
    for i in range(10):
        future=executor.submit(work,i)
        # future.cancel()
        # future.exception()

    executor.shutdown(wait=True)
    print('主')

 

死锁现象和递归锁
    死锁：互相锁死
    递归锁Rlock
        特点：可以多次 acquire
            每 acquire 一次，计数加1，只要计数不为0，其他的就不能抢到
    互斥锁的特点：
        只能 acquire 一次

from threading import RLock,Thread
import time

mutexA=mutexB=RLock()
class MyThread(Thread):
    def run(self):
        self.f1()
        self.f2()

    def f1(self):
        mutexA.acquire()
        print('%s 抢 A锁' %self.name)
        mutexB.acquire()
        print('%s 抢 B锁' %self.name)
        mutexB.release()
        mutexA.release()

    def f2(self):
        mutexB.acquire()
        print('%s 抢 B锁' %self.name)

        time.sleep(0.1)
        mutexA.acquire()
        print('%s 抢 A锁' %self.name)
        mutexA.release()

        mutexB.release()


if __name__ == '__main__':
    for i in range(10):
        t=MyThread()
        t.start()

 

信号量 -- Semaphore 模块
    本质是一把锁

from threading import Semaphore,Thread,current_thread
import time,random
def task():
    with sm:
        print('%s 正在上厕所' %current_thread().getName())
        time.sleep(random.randint(1,3))

if __name__ == '__main__':
    sm=Semaphore(5)
    for i in range(11):
        t=Thread(target=task)
        # --同时有11个线程开启，但是一次执行的个数只有5个
        # 这点是和线程池不同的点，线程池是自始至终都是开启指定个数的线程
        t.start()

 

事件Event
    e = Event()
    e.set() #发信号
    e.wait(timeout = 时间) #等信号，指定时间后，如果超时就直接执行
    e.is_set() #查看是否set

from threading import Event,current_thread,Thread
import time

e=Event()

def check():
    print('%s 正在检测' %current_thread().getName())
    time.sleep(5)
    e.set() # 执行到这里发送信号

def conn():
    count=1
    while not e.is_set():
        if count > 3:
            raise TimeoutError('连接超时')
        print('%s 正在等待%s连接' % (current_thread().getName(),count))
        e.wait(timeout=1)   # 如果到了指定时间上个任务还没完成，会抛出异常
        count+=1
    print('%s 开始连接' % current_thread().getName())

if __name__ == '__main__':
    t1=Thread(target=check)
    t2=Thread(target=conn)
    t3=Thread(target=conn)
    t4=Thread(target=conn)

    t1.start()
    t2.start()
    t3.start()
    t4.start()

 

定时器 Timer--模块
    t = Timer(时间,函数,args=(参数,))
    t.start() # 启动

from threading import Timer

def hello(n):
    print("hello, world",n)

t = Timer(1, hello,args=(1,))
t.start()  # after 1 seconds, "hello, world" will be printed

 

线程queue
queue.Queue() # 队列
queue.PriorityQueue() # 优先级队列
    q.put(优先级,"数据")   # 优先级--数字越小越高，如果优先级相同，就看后面数据大小
queue.LifoQueue() # 堆栈，后进先出

import queue

q=queue.Queue(3) #队列
q.put(1)
q.put(2)
q.put(3)
q.put(4)

print(q.get())
print(q.get())
print(q.get())


# 优先级队列
q=queue.PriorityQueue(4) #数字越小优先级越高
q.put((10,'aaa'))
q.put((7,'bbb'))
q.put((30,'ddd'))
q.put((30,'ccccccccc'))

print(q.get())
print(q.get())
print(q.get())
print(q.get())

# 后进先出，堆栈
q=queue.LifoQueue(3)
q.put(1)
q.put(2)
q.put(3)
print(q.get())
print(q.get())
print(q.get())

 

序列大小比较--从第一开始比较大小

# 序列判断大小
s1='abc'
s2='b'

print(s1 > s2)
l1=[1,'a',3]
l2=[1,'b']
print(l2 > l1)