Python学习day10

一、进程队列queue.py:

不同进程间内存是不共享的，要想实现两个进程间的数据交换，可以用以下方法：

#Author:Fred
from multiprocessing import Process, Queue
def f(q):
    q.put([42, None, 'hello'])

if __name__ == '__main__':
    q = Queue()
    p = Process(target=f, args=(q,))
    p.start()
    print(q.get())  # prints "[42, None, 'hello']"
    p.join()

 

二、Pipes

The Pipe() function returns a pair of connection objects connected by a pipe which by default is duplex (two-way). For example:

#Author:Fred
from multiprocessing import Process, Pipe
def f(conn):
    conn.send([42, None, 'hello from child'])
    print("from parents:",conn.recv())
    conn.close()

if __name__ == '__main__':
    parent_conn, child_conn = Pipe()
    p = Process(target=f, args=(child_conn,))
    p.start()
    print(parent_conn.recv())  # prints "[42, None, 'hello']"
    parent_conn.send("zy")
    p.join()

 

三、Managers

A manager object returned by Manager() controls a server process which holds Python objects and allows other processes to manipulate them using proxies.

A manager returned by Manager() will support types list, dict, Namespace, Lock, RLock, Semaphore, BoundedSemaphore, Condition, Event, Barrier, Queue, Value and Array. For example,

#Author:Fred
from multiprocessing import Process, Manager
import os
def f(d, l):
    d[os.getpid()] = os.getpid()
    l.append(os.getpid())
    print(l)

if __name__ == '__main__':
    with Manager() as manager:
        d = manager.dict() #{}生成一个字典，可在过个进程共享和传递
        l = manager.list(range(5)) #生成一个列表，可在过个进程共享和传递
        p_list = []
        for i in range(10):
            p = Process(target=f, args=(d, l))
            p.start()
            p_list.append(p)
        for res in p_list: #等结果
            res.join()

        print(d)
        print(l)

 

四、进程同步：

#Author:Fred
from multiprocessing import Process, Lock
def f(l, i):
    l.acquire()
    print('hello world', i)
    l.release()

if __name__ == '__main__':
    lock = Lock()

    for num in range(10):
        Process(target=f, args=(lock, num)).start()

 

五、进程池　　

进程池内部维护一个进程序列，当使用时，则去进程池中获取一个进程，如果进程池序列中没有可供使用的进进程，那么程序就会等待，直到进程池中有可用进程为止。

进程池中有两个方法：

apply

apply_async

#Author:Fred
from multiprocessing import Process, Pool,freeze_support
import time,os
def Foo(i):
    time.sleep(2)
    print("in process",os.getpid())
    return i + 100

def Bar(arg):
    print('-->exec done:', arg,os.getpid())

if __name__ == '__main__':
        freeze_support()
        pool = Pool(processes=5)
        print("主进程：",os.getpid())
        for i in range(10):
            pool.apply_async(func=Foo, args=(i,),callback=Bar) #callback回调
            #pool.apply(func=Foo, args=(i,)) #串行
            #pool.apply_async(func=Foo, args=(i,)) #并行

        print('end')
        pool.close()
        pool.join()  # 进程池中进程执行完毕后再关闭，如果注释，那么程序直接关闭。

 

六、协程

协程，又称微线程，纤程。英文名Coroutine。一句话说明什么是线程：协程是一种用户态的轻量级线程。

协程拥有自己的寄存器上下文和栈。协程调度切换时，将寄存器上下文和栈保存到其他地方，在切回来的时候，恢复先前保存的寄存器上下文和栈。因此：

协程能保留上一次调用时的状态（即所有局部状态的一个特定组合），每次过程重入时，就相当于进入上一次调用的状态，换种说法：进入上一次离开时所处逻辑流的位置。

协程的好处：

无需线程上下文切换的开销

无需原子操作锁定及同步的开销

　　"原子操作(atomic operation)是不需要synchronized"，所谓原子操作是指不会被线程调度机制打断的操作；这种操作一旦开始，就一直运行到结束，中间不会有任何 context switch （切换到另一个线程）。原子操作可以是一个步骤，也可以是多个操作步骤，但是其顺序是不可以被打乱，或者切割掉只执行部分。视作整体是原子性的核心。

方便切换控制流，简化编程模型

高并发+高扩展性+低成本：一个CPU支持上万的协程都不是问题。所以很适合用于高并发处理。

缺点：

无法利用多核资源：协程的本质是个单线程,它不能同时将 单个CPU 的多个核用上,协程需要和进程配合才能运行在多CPU上.当然我们日常所编写的绝大部分应用都没有这个必要，除非是cpu密集型应用。

进行阻塞（Blocking）操作（如IO时）会阻塞掉整个程序

1. 使用yield实现协程操作例子：

#Author:Fred
import time
import queue
def consumer(name):
    print("--->starting eating baozi...")
    while True:
        new_baozi = yield
        print("[%s] is eating baozi %s" % (name, new_baozi))
        # time.sleep(1)

def producer():
    r = con.__next__()
    r = con2.__next__()
    n = 0
    while n < 5:
        n += 1
        con.send(n)
        con2.send(n)
        print("\033[32;1m[producer]\033[0m is making baozi %s" % n)

if __name__ == '__main__':
    con = consumer("c1")
    con2 = consumer("c2")
    p = producer()

 

1. Greenlet

greenlet是一个用C实现的协程模块，相比与python自带的yield，它可以使你在任意函数之间随意切换，而不需把这个函数先声明为generator:

from greenlet import greenlet

def test1():

    print(12)

    gr2.switch()

    print(34)

    gr2.switch()

 

def test2():

    print(56)

    gr1.switch()

    print(78)

 

gr1 = greenlet(test1)

gr2 = greenlet(test2)

gr1.switch()

 

1. Gevent

一个第三方库，可以轻松通过gevent实现并发同步或异步编程，在gevent中用到的主要模式是Greenlet, 它是以C扩展模块形式接入Python的轻量级协程。 Greenlet全部运行在主程序操作系统进程的内部，但它们被协作式地调度。

import gevent

def func1():

    print('\033[31;1m李闯在跟海涛搞...\033[0m')

    gevent.sleep(2)

    print('\033[31;1m李闯又回去跟继续跟海涛搞...\033[0m')

 

def func2():

    print('\033[32;1m李闯切换到了跟海龙搞...\033[0m')

    gevent.sleep(1)

    print('\033[32;1m李闯搞完了海涛，回来继续跟海龙搞...\033[0m')

 

gevent.joinall([

    gevent.spawn(func1),

    gevent.spawn(func2),

    #gevent.spawn(func3),

])

1. 爬虫

#Author:Fred
from urllib import request
import gevent
from gevent import monkey

monkey.patch_all() #吧当前程序的所有io操作单独做标记
def f(url):
    print('GET: %s' % url)
    resp =request.urlopen(url)
    data = resp.read()
    print('%d bytes received from %s.' % (len(data), url))

gevent.joinall([
        gevent.spawn(f, 'https://www.python.org/'),
        gevent.spawn(f, 'https://www.yahoo.com/'),
        gevent.spawn(f, 'https://github.com/'),
])

 

5.通过gevent实现单线程下的多socket并发

server side:

#Author:Fred
import sys,socket,time
import gevent
from gevent import socket, monkey
monkey.patch_all()

def server(port):
    s = socket.socket()
    s.bind(('0.0.0.0', port))
    s.listen(500)
    while True:
        cli, addr = s.accept()
        gevent.spawn(handle_request, cli)

def handle_request(conn):
    try:
        while True:
            data = conn.recv(1024)
            print("recv:", data)
            conn.send(data)
            if not data:
                conn.shutdown(socket.SHUT_WR)

    except Exception as  ex:
        print(ex)
    finally:
        conn.close()

if __name__ == '__main__':
    server(8001)

 

client side :

#Author:Fred
import socket

HOST = 'localhost'  # The remote host
PORT = 9999  # The same port as used by the server
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((HOST, PORT))
while True:
    msg = bytes(input(">>:"), encoding="utf8")
    s.sendall(msg)
    data = s.recv(1024)
    print('Received', data) #repr格式化输出
s.close()

 

 

七、select模块

1.select socket server.py：

.#Author:Fred
import select,socket,queue

server = socket.socket()
server.bind(('localhost',9000))
server.listen(1000)

server.setblocking(False) #不阻塞

msg_dic = {}
inputs = [server,]
outputs = []
while True:
    readable,writeable,exceptional = select.select(inputs,outputs,inputs)
    print(readable,writeable,exceptional)
    for r in readable:
        if r is server: #代表来了一个新链接
            coon,addr = server.accept()
            print("来了个新链接",addr)
            inputs.append(coon) #因为这个新建立的连接还没发数据过来，
            # 现在就接收的话会报错，所以想实现这个客户端发数据来时能知道，
            # 就需要让select再监测这个coon
            msg_dic[coon] = queue.Queue() #初始化一个队列，后面存要返回给这个客户端的数据
        else:
            data = coon.recv(1024)
            print("收到数据",data)
            msg_dic[r].put(data)
            outputs.append(r) #放入返回的连接队列
            # r.send(data)
            # print("send done..")

    for w in writeable:#要返回给客户端的连接列表
        data_to_client = msg_dic[w].get()
        w.send(data_to_client) #返回给客户端源数据

        outputs.remove(w) #确保下次循环时候writeable不返回这个已经处理完的连接

    for e in exceptional:
        if e in outputs:
            outputs.remove(e)
        inputs.remove(e)
        del msg_dic[e]

 

2.mutlti conn socket client.py:

#Author:Fred
import socket
import sys

messages = [ b'This is the message. ',
             b'It will be sent ',
             b'in parts.',
             ]
server_address = ('localhost', 9999)

# Create a TCP/IP socket
socks = [ socket.socket(socket.AF_INET, socket.SOCK_STREAM) for i in range(10)]

# Connect the socket to the port where the server is listening
print('connecting to %s port %s' % server_address)
for s in socks:
    s.connect(server_address)

for message in messages:

    # Send messages on both sockets
    for s in socks:
        print('%s: sending "%s"' % (s.getsockname(), message) )
        s.send(message)

    # Read responses on both sockets
    for s in socks:
        data = s.recv(1024)
        print( '%s: received "%s"' % (s.getsockname(), data) )
        if not data:
            print( 'closing socket', s.getsockname() )

 

3.selectors模块

This module allows high-level and efficient I/O multiplexing, built upon the select module primitives. Users are encouraged to use this module instead, unless they want precise control over the OS-level primitives used.

#Author:Fred
import selectors
import socket

sel = selectors.DefaultSelector()

def accept(sock, mask):
    conn, addr = sock.accept()  # Should be ready
    print('accepted', conn, 'from', addr)
    conn.setblocking(False)
    sel.register(conn, selectors.EVENT_READ, read)

def read(conn, mask):
    data = conn.recv(1024)  # Should be ready
    if data:
        print('echoing', repr(data), 'to', conn)
        conn.send(data)  # Hope it won't block
    else:
        print('closing', conn)
        sel.unregister(conn)
        conn.close()

sock = socket.socket()
sock.bind(('localhost', 9999))
sock.listen(100)
sock.setblocking(False)
sel.register(sock, selectors.EVENT_READ, accept)

while True:
    events = sel.select() #默认阻塞，有活动就返回活动的连接列表
    for key, mask in events:
        callback = key.data #accept
        callback(key.fileobj, mask) #key.fileobj = 文件句柄