python之路——42

学习内容

协程
    1.本质是一个线程
    2.能够在多个任务之间切换节省io时间
    3.协程中任务之间的切换也要消耗时间，但是开销远远小于进程线程之间的切换
    4.在任务的执行过程中，检测到IO就切换至他任务
    5.爬虫和socket相关的操作，可使用协程
IO模型
    1.阻塞IO
    2.非阻塞IO
    3.多路复用IO
        1.select 机制 (window,linux) 操作系统循环列表中每一个被监听的项，看是否有读操作
        2.poll 机制  (linux)  可以监听对象比select多
                select 和 poll 机制 都会随着监听项的增多，效率降低
        3.epoll 机制 linux 更高级的算法，不是循环监听

代码区

1.消费者模型 函数间切换

def consumer():
    while True:
        x = yield
        print('消费了%s数据'%x)

def producer():
    c = consumer()
    next(c)
    for i in range(10):
        print('制造了%s数据'%i)
        c.send(i)
producer()

2.greenlet 跳转

from greenlet import greenlet
def eat():
    print('eating start')
    g2.switch()
    print('eating end')
    g2.switch()

def play():
    print('playing start')
    g1.switch()
    print('playing end')
g1 = greenlet(eat)
g2 = greenlet(play)
g1.switch()

3.gevent 异步

from gevent import monkey;monkey.patch_all()
import time
import gevent

def task():
    print('******')
    time.sleep(1)


def sync():
    for i in range(5):
        print('sync:')
        task()

def async():
    c_list = []
    for i in range(5):
        print('async:')
        c = gevent.spawn(task)
        c_list.append(c)
    gevent.joinall(c_list)

sync()
async()

4.gevent 爬虫

from gevent import monkey; monkey.patch_all()
from urllib.request import urlopen
import gevent
def func(url):
    ret = urlopen(url)
    ret = ret.read().decode('utf-8')
    return ret

url = 'http://www.baidu.com'
url1 = 'http://www.taobao.com'
url2 = 'http://www.qq.com'
url3 = 'http://www.hao123.com'

g = gevent.spawn(func,url)
g1 = gevent.spawn(func,url1)
g2 = gevent.spawn(func,url2)
g3 = gevent.spawn(func,url3)
gevent.joinall([g,g1,g2,g3])
print(g.value,g1.value,g2.value,g3.value)

5.gevent 实现socketserver 

# server
import socket
from gevent import monkey;monkey.patch_all()
import gevent
def func(conn):
    conn.send(b'hello')
    ret = conn.recv(1024).decode('utf-8')
    print(ret)
sk = socket.socket()
sk.bind(('127.0.0.1',8080))
sk.listen()

while 1 :
    conn, addr = sk.accept()
    gevent.spawn(func,conn)
conn.close()
sk.close()
#client
import socket

sk = socket.socket()
sk.connect(('127.0.0.1',8080))

ret = sk.recv(1024).decode('utf-8')
print(ret)
info = input('>>> ')
sk.send(info.encode('utf-8'))
sk.close()

6.阻塞IO实现socketserver 

#server
import socket
sk = socket.socket()
sk.bind(('127.0.0.1',8080))
sk.setblocking(False)
sk.listen()
conn_list = []
del_conn = []
while True:
    try:
        conn, addr = sk.accept()
        print('一个连接被建立')
        conn_list.append(conn)
    except BlockingIOError:
        for conn in conn_list:
            try:
                msg = conn.recv(1024)
                if msg == b'':
                    del_conn.append(conn)
                    continue
                print(msg)
                conn.send(b'bye bye')
            except BlockingIOError:
                pass
        for conn in del_conn:
            conn.close()
            conn_list.remove(conn)
        del_conn.clear()
#client
import socket,time
import threading
def func():
    sk = socket.socket()
    sk.connect(('127.0.0.1',8080))
    sk.send(b'hello')
    time.sleep(1)
    print(sk.recv(1024))
    sk.close()
for i in range(2):
    threading.Thread(target=func).start()

7. 多路复用 IO实现socketserver 

#server
import select
import socket

sk = socket.socket()
sk.bind(('127.0.0.1',8080))
sk.setblocking(False)
sk.listen()

read_list = [sk]
while 1:
    r_list, w_list, x_list = select.select(read_list,[],[])
    for i in r_list:
        if i is sk:
            conn, addr = i.accept()
            read_list.append(conn)
        else:
            ret = i.recv(1024)
            if ret == b'':
                i.close()
                read_list.remove(i)
                continue
            print(ret)
            i.send(b'go')
#client
import socket,time
import threading
def func():
    sk = socket.socket()
    sk.connect(('127.0.0.1',8080))
    sk.send(b'hello')
    time.sleep(4)
    print(sk.recv(1024))
    sk.close()
for i in range(20):
    threading.Thread(target=func).start()