并发实现方式

多进程

多线程

import threading
t = threading.Thread(target=task,args=(url,))
t.start()

协程+IO操作

import gevent
from gevent import monkey
monkey.patch_all()

def func(url):
　　print(url)

urls = [
    'http://www.baidu.com/', 
]
spawn_list = []
for url in urls:
    spawn_list.append(gevent.spawn(func, url))
gevent.joinall(spawn_list)

 

twisted 基于事件循环的异步非阻塞模块

from twisted.web.client import getPage, defer
from twisted.internet import reactor


def stop_loop(arg):
    reactor.stop()


def get_response(contents):
    print(contents)


deferred_list = []

url_list = [
    'http://www.baidu.com/',
    'https://www.cnblogs.com/',
]

for url in url_list:
    deferred = getPage(bytes(url, encoding='utf8'))
deferred.addCallback(get_response)
deferred_list.append(deferred)

dlist = defer.DeferredList(deferred_list)
dlist.addBoth(stop_loop)

reactor.run()

 

twisted实现原理：
　　用的select监测变换

 

自定义twisted

import socket
import select

class ChunSheng(object):

    def __init__(self):
        self.socket_list = []
        self.conn_list = []
        self.conn_func_dict = {}

    def add_request(self, url_func):
        conn = socket.socket()
        conn.setblocking(False)
        try:
            conn.connect((url_func[0], 80))
        except BlockingIOError as e:
            pass
        self.conn_func_dict[conn] = url_func[1]

        self.socket_list.append(conn)
        self.conn_list.append(conn)

    def run(self):
        """
        检测self.socket_list中的5个socket对象是否连接成功
        :return:
        """
        while True:
            #   select.select
            #   第一个参数： 用于检测其中socket是否已经获取到响应内容
            #   第二个参数： 用于检测其中socket是否已经连接成功

            # 第一个返回值 r：具体是那一个socket获取到结果
            # 第二个返回值 r：具体是那一个socket连接成功
            r, w, e = select.select(self.socket_list, self.conn_list, [], 0.05)
            for sock in w:  # [socket1,socket2]
                sock.send(b'GET / http1.1\r\nhost:xxxx.com\r\n\r\n')
                self.conn_list.remove(sock)

            for sock in r:
                data = sock.recv(8096)
                func = self.conn_func_dict[sock]
                func(data)
                sock.close()
                self.socket_list.remove(sock)

            if not self.socket_list:
                break

from chun import ChunSheng


def callback1(data):
    print('下载完成', data)


def callback2(data):
    print('下载完成', data)


chun = ChunSheng()
urls = [
    ('www.baidu.com', callback1),
    ('www.cnblogs.com', callback1),
    ('www.pythonav.com', callback2),
    ('www.bing.com', callback2),
    ('www.stackoverflow.com', callback2),
]
for url in urls:
    chun.add_request(url)

chun.run()