day38网络编程——concurrent.futures进程池线程池

进程池 concurrent.futures模块的 ProcessPoolExecutor 的用法

from concurrent.futures import ProcessPoolExecutor
import time,random,os
def work(i):
    print("%s is running %s"%(os.getpid(),i))
    return i**2
if __name__ == '__main__':
    executor = ProcessPoolExecutor(4)
    futures = []
    for i in range(10):
        future = executor.submit(work,i)
        futures.append(future)
    executor.shutdown(wait=True)
    print("主")
    for e in futures:
        print(e.result())

线程池 concurrent.futures模块的 ThreadPoolExecutor 的用法

from concurrent.futures import ThreadPoolExecutor
import time,random,os,threading
def work(i):
    time.sleep(random.randint(1, 2))
    print("%s is running"%threading.current_thread().getName())

    return i**2
if __name__ == '__main__':
    executor = ThreadPoolExecutor(6)
    futrues = []
    for i in range(10):
        t = executor.submit(work,i)
        futrues.append(t)
    executor.shutdown(wait=True)
    print("主")
    for future in futrues:
        print(future.result())

map的用法

from concurrent.futures import ThreadPoolExecutor
import time,random,os,threading
def work(i):
    print("%s is running "%threading.current_thread().getName())
if __name__ == '__main__':
    executor = ThreadPoolExecutor()
    executor.map(work,range(10))
    executor.shutdown(wait=True)
    print("主")

线程的回调函数的爬虫练习

from concurrent.futures import ThreadPoolExecutor,ProcessPoolExecutor
import requests
import os
import time,random
def get(url):
    print('%s GET %s' %(os.getpid(),url))
    response=requests.get(url)
    time.sleep(random.randint(1,3))
    if response.status_code == 200:
        print('%s DONE %s' % (os.getpid(), url))
        return {'url':url,'text':response.text}

def parse(future):
    dic=future.result()
    print('%s PARSE %s' %(os.getpid(),dic['url']))
    time.sleep(1)
    res='%s:%s\n' %(dic['url'],len(dic['text']))
    with open('db.txt','a') as f:
        f.write(res)

if __name__ == '__main__':
    urls=[
        'https://www.baidu.com',
        'https://www.python.org',
        'https://www.openstack.org',
        'https://help.github.com/',
        'http://www.sina.com.cn/'
    ]
    p=ProcessPoolExecutor()
    start_time=time.time()
    objs=[]
    for url in urls:
        p.submit(get,url).add_done_callback(parse)
    print('主',(time.time()-start_time))

死锁和递归锁

 

from threading import Thread,RLock
import time
mutexA = mutexB = RLock()
class MyThread(Thread):
    def run(self):
        self.f1()
        self.f2()
    def f1(self):
        mutexA.acquire()
        print("%s 抢到A锁"%self.name)
        mutexB.acquire()
        print("%s 抢到B锁"%self.name)
        mutexB.release()
        mutexA.release()
    def f2(self):
        mutexB.acquire()
        print("%s 抢到B锁"%self.name)
        time.sleep(0.1)
        mutexA.acquire()
        print("%s 抢到A锁"%self.name)
        mutexA.release()
        mutexB.release()
if __name__ == '__main__':
    for i in range(10):
        t = MyThread()
        t.start()

 

#递归锁的特点：
#1：可以acquire多次
#2：每acquire一次，计数加1，只要计数不为0，就不能被其他线程抢到
#互斥锁的特点：
#1：只能acquire一次

 

 

信号量

 

from threading import Semaphore,Thread,current_thread
import time,random
def task():
    with sm:      #线程量锁
        print("%s 正在上运行"%current_thread().getName())
        time.sleep(random.randint(1,3))
if __name__ == '__main__':
    sm = Semaphore(5)
    for i in range(11):
        t=Thread(target=task)
        t.start()

# 11 个线程一起运行 默认开启的信号量是主机核数的5倍

 

 

Event事件

from threading import Event,current_thread,Thread
import time

e=Event()

def check():
    print('%s 正在检测' %current_thread().getName())
    time.sleep(5)
    e.set()

def conn():
    count=1
    while not e.is_set():
        if count > 3:
            raise TimeoutError('连接超时')
        print('%s 正在等待%s连接' % (current_thread().getName(),count))
        e.wait(timeout=1)
        count+=1
    print('%s 开始连接' % current_thread().getName())

if __name__ == '__main__':
    t1=Thread(target=check)
    t2=Thread(target=conn)
    t3=Thread(target=conn)
    t4=Thread(target=conn)

    t1.start()
    t2.start()
    t3.start()
    t4.start()

e =Event()  相当于在全局定义一个变量  FLAG =False

e.set() 相当于给变量FLAG= True 

e.is_set() 是判断e的性质 是True / False

 

 线程queue

q = queue.Queue(3)    相当于进程的列队

q.put()

q.get()

用法和列队相同

 

q=queue.PriorityQueue(4) #数字越小优先级越高
q.put((10,'aaa'))
q.put((7,'bbb'))
q.put((30,'ddd'))
q.put((30,'ccccccccc'))

 

q=queue.LifoQueue(3) #后进先出，堆栈

q.put(1)
q.put(2)
q.put(3)
print(q.get())
print(q.get())
print(q.get())