生产者消费者模型爬虫小例子

from multiprocessing import Process,Queue
import requests
def consumer(q):
    while True:
        tup = q.get()
        if tup:
            name,res =tup
            with open('%s.html'%name,mode='w',encoding='utf-8') as f:
                f.write(res)
        else:break

def producter(name,url,q):
    res  = requests.get(url).text
    q.put((name,res))


if __name__ == '__main__':
    q = Queue()
    url_dic = {
        '临汾58':'https://linfen.58.com/?utm_source=market&spm=u-2d2yxv86y3v43nkddh1.BDPCPZ_BT',
        '北京赶集':'http://bj.ganji.com/',
        'e_val博客':'https://www.cnblogs.com/eva-j/'
    }
    #同时开启三个进程
    url_obj =[]
    for name,value in url_dic.items():
        p = Process(target=producter,args=(name,value,q)).start()
        url_obj.append(p)
    p1 = Process(target=consumer,args=(q,))
    p1.start()
    for p in url_obj:
        p.join()
    q.put(None)