python 3.x asyncio模拟http请求

asyncio模拟http请求

• asyncio没有提供http协议的接口.提供了更底层的接口TCP,UDP协议接口.
• 想使用http协议去请求url的话,使用aiohttp,aiohttp是专门用作http服务的,首先aiohttp是可以搭建一个http服务器.aiohttp也可以做爬虫使用,就是把aiohttp当作异步的requests使用.
• 使用原身的asyncio来完成aiohttp,aio是基于asyncio来完成的.

asyncio模拟http请求

import asyncio
from urllib.parse import urlparse
import time


# 通过协程的方式,也就是以同步代码的形式,来编写高效的http请求.
async def get_url(url):
    new_url = urlparse(url)
    host = new_url.netloc
    path = new_url.path
    if path == "":
        path = "/"

    # client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    # client.setblocking(False)
    # client.connect((host, 80))  # 阻塞不会消耗cpu
    # client.send("GET {} HTTP/1.1\r\nHost:{}\r\nConnection:close\r\n\r\n".format(path, host).encode("utf8"))

    # 等待连接建立好的,希望await可以返回一个可以直接发送的socket.
    # asyncio提供了一个方法open_connection(),open_connection是一个协程.
    # open_connection()可以跟服务端建立一个连接.
    # 如果使用client.setblocking(False),需要使用selector.register(self.client.fileno(), EVENT_READ, self.readable),所以说这一步是省略不了的
    # 这里就出现疑问了open_connection()是怎样完成register注册的呢???
    # open_connection()方法,返回reader, writer
    # 建立socket连接是耗时的操作,所以需要await,跳转到另外的一个协程.
    # open_connection是一个协程.open_connection()执行完后会自动调用这个地方.会直接驱动get_url协程
    reader, writer = await asyncio.open_connection(host, port=80)
    # 直接写数据,write()方法内部的实现还是调用send()方法
    # write()方法实际上帮助我们完成unregister以及之前说过send()之后要重新register一个可读的事件,write()方法的内部已经帮我们实现重新Register的逻辑
    writer.write("GET {} HTTP/1.1\r\nHost:{}\r\nConnection:close\r\n\r\n".format(path, host).encode("utf8"))

    all_lines = []
    # async for语法
    # 在对某一阻塞的对象进行for循环,将读取数据异步化
    # async 背后实现了魔法函数__anext__
    async for raw_line in reader:
        data = raw_line.decode("utf8")
        all_lines.append(data)
    html = "\n".join(all_lines)
    print(html)
    return html


if __name__ == '__main__':
    start_time = time.time()
    loop = asyncio.get_event_loop()
    tasks = []
    for i in range(1, 21):
        url = "http://shop.projectsedu.com/goods/{}/".format(i)
        tasks.append(get_url(url))
    loop.run_until_complete(asyncio.gather(*tasks))
    print("code run finished use time: {}".format(time.time() - start_time))

"""
输出结果:
    HTTP/1.1 200 OK
    Server: nginx/1.16.1
    Date: Fri, 27 Dec 2019 05:06:14 GMT
    Content-Type: application/json
    Content-Length: 1639
    Connection: close
    Vary: Accept, Cookie
    Allow: GET, HEAD, OPTIONS
    X-Frame-Options: SAMEORIGIN

    {"id":13,"category":{"id":121,"sub_cat":[{"id":122,"sub_cat":[],"name":"松花蛋/咸鸭蛋","code":"xhd_xyd","desc":"",
    "category_type":3,"is_tab":false,"add_time":"2017-07-29T18:56:34","parent_category":121},{"id":123,"sub_cat":[],
    "name":"鸡蛋","code":"jd","desc":"","category_type":3,"is_tab":false,"add_time":"2017-07-29T18:56:34",
    "parent_category":121}],"name":"蛋制品","code":"dzp","desc":"","category_type":2,"is_tab":false,
    "add_time":"2017-07-29T18:56:34","parent_category":110},"images":[{
    "image":"http://shop.projectsedu.com/media/goods/images/5_P_1448945270390.jpg"},{
    "image":"http://shop.projectsedu.com/media/goods/images/5_P_1448945270067.jpg"},{
    "image":"http://shop.projectsedu.com/media/goods/images/5_P_1448945270432.jpg"}],"goods_sn":"",
    "name":"澳洲进口安格斯牛切片上脑牛排1000g","click_num":1650,"sold_num":0,"fav_num":0,"goods_num":-1,"market_price":144.0,
    "shop_price":120.0,"goods_brief":"澳大利亚是国际公认的没有疯牛病和口蹄疫的国家。为了保持澳大利亚产品的高标准，
    澳大利亚牛肉业和各级政府共同努力简历了严格的标准和体系，以保证生产的整体化和产品的可追溯性",
    "goods_desc":"<p><img src=\"/media/goods/images/2_20170719161405_249.jpg\" title=\"\" alt=\"2.jpg\"/></p>
    <p><img src=\"/media/goods/images/2_20170719161414_628.jpg\" title=\"\" alt=\"2.jpg\"/></p>
    <p><img src=\"/media/goods/images/2_20170719161435_381.jpg\" title=\"\" alt=\"2.jpg\"/></p>","ship_free":true,
    "goods_front_image":"http://shop.projectsedu.com/media/goods/images/5_P_1448945270390.jpg","is_new":false,
    "is_hot":false,"add_time":"2017-07-31T23:53:54"}
    
    ...省略其他的结果...
    
    code run finished use time: 2.284130573272705
"""

 

 

asyncio模拟http请求(去除注释的代码)

"""
asyncio模拟http请求.
"""

import asyncio
from urllib.parse import urlparse
import time


async def get_url(url):
    new_url = urlparse(url)
    host = new_url.netloc
    path = new_url.path
    if path == "":
        path = "/"
    reader, writer = await asyncio.open_connection(host, port=80)
    writer.write("GET {} HTTP/1.1\r\nHost:{}\r\nConnection:close\r\n\r\n".format(path, host).encode("utf8"))

    all_lines = []
    async for raw_line in reader:
        data = raw_line.decode("utf8")
        all_lines.append(data)
    html = "\n".join(all_lines)
    # 提问: 怎样才能不在协程get_url中打印html的结果,而是在外部直接获取html的结果.
    print(html)
    return html


if __name__ == '__main__':
    start_time = time.time()
    loop = asyncio.get_event_loop()
    tasks = []
    for i in range(1, 21):
        url = "http://shop.projectsedu.com/goods/{}/".format(i)
        tasks.append(get_url(url))
    loop.run_until_complete(asyncio.gather(*tasks))
    print("code run finished use time: {}".format(time.time() - start_time))

"""
输出结果:
    code run finished use time: 2.324403762817383
"""

 

 

open_connection()是怎样完成register注册的呢???

源码open_connection():

        @coroutine
        def open_connection(host=None, port=None, *,loop=None, limit=_DEFAULT_LIMIT, **kwds):
            if loop is None:
                loop = events.get_event_loop()
            reader = StreamReader(limit=limit, loop=loop)
            protocol = StreamReaderProtocol(reader, loop=loop)

            # 此处查看create_connection()源码
            transport, _ = yield from loop.create_connection(lambda: protocol, host, port, **kwds)
            writer = StreamWriter(transport, protocol, reader, loop)
            return reader, writer

 

源码StreamReader:

        class StreamReader:
            # 省略...

            @coroutine
            def readexactly(self, n):
                # 省略...
                if compat.PY35:
                    @coroutine
                    def __aiter__(self):
                        return self

                    # 迭代协议__anext___(),并且是一个协程.
                    @coroutine
                    def __anext__(self):
                        # readline()
                        val = yield from self.readline()
                        if val == b'':
                            raise StopAsyncIteration
                        return val

 

源码readline():

        @coroutine
        def readline(self):
            sep = b'\n'
            seplen = len(sep)
            try:
                # 直接调用readuntil()方法
                line = yield from self.readuntil(sep)
            except IncompleteReadError as e:
                return e.partial
            except LimitOverrunError as e:
                if self._buffer.startswith(sep, e.consumed):
                    del self._buffer[:e.consumed + seplen]
                else:
                    self._buffer.clear()
                self._maybe_resume_transport()
                raise ValueError(e.args[0])
            return line

 

源码readuntil():

        @coroutine
        def readuntil(self, separator=b'\n'):
            # 省略...

            while True:
                # 省略...
                yield from self._wait_for_data('readuntil')

            # 省略...

 

源码_wait_for_data():

        @coroutine
        def _wait_for_data(self, func_name):
            # 省略...

            if self._paused:
                self._paused = False
                # 继续读取数据
                self._transport.resume_reading()

            self._waiter = self._loop.create_future()
            try:
                yield from self._waiter
            finally:
                self._waiter = None

 

源码resume_reading():

        def resume_reading(self):
            if not self._paused:
                raise RuntimeError('Not paused')
            self._paused = False
            if self._closing:
                return
            # 向loop里面添加_add_reader
            self._loop._add_reader(self._sock_fd, self._read_ready)
            if self._loop.get_debug():
                logger.debug("%r resumes reading", self)

 

源码_add_reader():

        def _add_reader(self, fd, callback, *args):
            self._check_closed()
            # 当有数据的时候会调用callback回调函数,callback就是resume_reading()源码中的_read_ready
            handle = events.Handle(callback, args, self)
            try:
                key = self._selector.get_key(fd)
            except KeyError:
                # 注册一个读的事件
                self._selector.register(fd, selectors.EVENT_READ,
                                        (handle, None))
            else:
                mask, (reader, writer) = key.events, key.data
                self._selector.modify(fd, mask | selectors.EVENT_READ,
                                      (handle, writer))
                if reader is not None:
                    reader.cancel()

 

源码_read_ready():

        def _read_ready(self):
            if self._conn_lost:
                return
            try:
                # 调用socket的recv的方法,这就是之前自己介绍的逻辑
                data = self._sock.recv(self.max_size)
            except (BlockingIOError, InterruptedError):
                pass
            except Exception as exc:
                self._fatal_error(exc, 'Fatal read error on socket transport')
            else:
                if data:
                    self._protocol.data_received(data)
                else:
                    if self._loop.get_debug():
                        logger.debug("%r received EOF", self)
                    keep_open = self._protocol.eof_received()
                    if keep_open:
                        self._loop._remove_reader(self._sock_fd)
                    else:
                        self.close()

 

源码create_connection():

        @coroutine
        def create_connection(self, protocol_factory, host=None, port=None, *,
                              ssl=None, family=0, proto=0, flags=0, sock=None,
                              local_addr=None, server_hostname=None):

            # 参数检查代码省略...

            if host is not None or port is not None:
                if sock is not None:
                    raise ValueError(
                        'host/port and sock can not be specified at the same time')

                # 查看_ensure_resolved()源码,_ensure_resolved()方法执行完毕,立马返回一个future对象,并没有对IO进行操作
                f1 = _ensure_resolved((host, port), family=family, type=socket.SOCK_STREAM, proto=proto,flags=flags, loop=self)

                # 返回的future对象f1立马添加到fs列表中
                fs = [f1]
                if local_addr is not None:
                    f2 = _ensure_resolved(local_addr, family=family,type=socket.SOCK_STREAM, proto=proto,flags=flags, loop=self)
                    fs.append(f2)
                else:
                    f2 = None

                # wait()会等待future的完成,此处会阻塞,因为第一步会根据传递进来的host,port进行DNS解析.会等到future完成之后,在向下进一步执行.
                # 通过url完成地址的解析.实际上是需要时间的,所以使用了yield from
                yield from tasks.wait(fs, loop=self)

                # 解析完后会获取到解析的结果
                infos = f1.result()

                # 省略...
                for family, type, proto, cname, address in infos:
                    try:
                        # 获取解析结果之后,此处才是主体逻辑.
                        # 建立socket
                        sock = socket.socket(family=family, type=type, proto=proto)
                        # 设置成非阻塞模式.
                        sock.setblocking(False)
                        if f2 is not None:
                            for _, _, _, _, laddr in laddr_infos:
                                try:
                                    # 绑定地址
                                    sock.bind(laddr)
                                    break
                                # 省略...
                else:
                    # 省略...
            else:
                # 省略...
            return transport, protocol

 

源码_ensure_resolved():

        def _ensure_resolved(address, *, family=0, type=socket.SOCK_STREAM, proto=0, flags=0, loop):
            host, port = address[:2]
            # 对传递进来的url进行解析.
            info = _ipaddr_info(host, port, family, type, proto)
            if info is not None:
                # 创建一个future对象,为了不使异步IO阻塞,所以在调用的时候,立马返回一个future
                # 当future里面的任务执行完后,会直接发送一个结果给future
                fut = loop.create_future()
                fut.set_result([info])
                return fut
            else:
                return loop.getaddrinfo(host, port, family=family, type=type, proto=proto, flags=flags)

 

源码getaddrinfo():

        def getaddrinfo(self, host, port, *, family=0, type=0, proto=0, flags=0):
            if self._debug:
                return self.run_in_executor(None, self._getaddrinfo_debug,
                                            host, port, family, type, proto, flags)
            else:
                return self.run_in_executor(None, socket.getaddrinfo,
                                            host, port, family, type, proto, flags)

 

源码socket.getaddrinfo()

• getaddrinfo()是一个阻塞的方法,所以将getaddrinfo()放到线程池来运行.运行完后将结果放到future的result中.

        def getaddrinfo(host, port, family=0, type=0, proto=0, flags=0):
            addrlist = []
            for res in _socket.getaddrinfo(host, port, family, type, proto, flags):
                af, socktype, proto, canonname, sa = res
                addrlist.append((_intenum_converter(af, AddressFamily),
                                 _intenum_converter(socktype, SocketKind),
                                 proto, canonname, sa))
            return addrlist

 

源码sock_connect():selector_events.py模块下

        @coroutine
        def sock_connect(self, sock, address):
            if self._debug and sock.gettimeout() != 0:
                raise ValueError("the socket must be non-blocking")

            if not hasattr(socket, 'AF_UNIX') or sock.family != socket.AF_UNIX:
                resolved = base_events._ensure_resolved(
                    address, family=sock.family, proto=sock.proto, loop=self)
                if not resolved.done():
                    yield from resolved
                _, _, _, _, address = resolved.result()[0]

            fut = self.create_future()
            self._sock_connect(fut, sock, address)
            # 直接返回create_future(fut = self.create_future())
            # 在asyncio里面,凡是异步的地方都会创建一个future,
            # 在使用同步IO库的时候都没有fut = self.create_future()这行语句.
            # 凡是去完成异步IO的库主要是阻塞的地方,都需要自己去创建一个future
            return (yield from fut)

 

源码_sock_connect():

        def _sock_connect(self, fut, sock, address):
            fd = sock.fileno()
            try:
                # 建立socket连接
                sock.connect(address)
            except (BlockingIOError, InterruptedError):
                # 如果抛出BlockingIOError错误.会添加一个_sock_connect_done()方法
                fut.add_done_callback(functools.partial(self._sock_connect_done, fd))
                self.add_writer(fd, self._sock_connect_cb, fut, sock, address)
            except Exception as exc:
                fut.set_exception(exc)
            else:
                fut.set_result(None)

 

源码_sock_connect_done():

        def _sock_connect_done(self, fd, fut):
            # 与add_write()关联,主要是用来注册与注销connect
            self.remove_writer(fd)

 

源码add_writer():

        def add_writer(self, fd, callback, *args):
            self._ensure_fd_no_transport(fd)
            return self._add_writer(fd, callback, *args)

 

源码_add_writer():

        def _add_writer(self, fd, callback, *args):
            self._check_closed()
            handle = events.Handle(callback, args, self)
            try:
                key = self._selector.get_key(fd)
            except KeyError:
                # 在此处完成了selector的注册.
                self._selector.register(fd, selectors.EVENT_WRITE,
                                        (None, handle))
            else:
                mask, (reader, writer) = key.events, key.data
                self._selector.modify(fd, mask | selectors.EVENT_WRITE,
                                      (reader, handle))
                if writer is not None:
                    writer.cancel()

 

源码remove_writer():

        def remove_writer(self, fd):
            self._ensure_fd_no_transport(fd)
            return self._remove_writer(fd)

 

源码_remove_writer():

        def _remove_writer(self, fd):
            if self.is_closed():
                return False
            try:
                key = self._selector.get_key(fd)
            except KeyError:
                return False
            else:
                mask, (reader, writer) = key.events, key.data
                # Remove both writer and connector.
                mask &= ~selectors.EVENT_WRITE
                if not mask:
                    # 此处注销selector.
                    self._selector.unregister(fd)
                else:
                    # 变更selector
                    self._selector.modify(fd, mask, (reader, None))

                if writer is not None:
                    writer.cancel()
                    return True
                else:
                    return False

 

怎样才能不在协程get_url中打印html的结果,而是在外部直接获取html的结果.

通过future对象的result()方法可以在外部直接获取协程get_url的返回结果.

import asyncio
from urllib.parse import urlparse
import time


async def get_url(url):
    new_url = urlparse(url)
    host = new_url.netloc
    path = new_url.path
    if path == "":
        path = "/"
    reader, writer = await asyncio.open_connection(host, port=80)
    writer.write("GET {} HTTP/1.1\r\nHost:{}\r\nConnection:close\r\n\r\n".format(path, host).encode("utf8"))

    all_lines = []
    async for raw_line in reader:
        data = raw_line.decode("utf8")
        all_lines.append(data)
    html = "\n".join(all_lines)
    # 提问: 怎样才能不在协程get_url中打印html的结果,而是在外部直接获取html的结果.
    # 通过future对象的result()方法可以在外部直接获取协程get_url的返回结果.
    # print(html)
    return html


if __name__ == '__main__':
    start_time = time.time()
    loop = asyncio.get_event_loop()
    tasks = []
    for i in range(1, 21):
        url = "http://shop.projectsedu.com/goods/{}/".format(i)
        # tasks.append(get_url(url))语句:直接将协程get_url放到tasks列表中
        # 将协程get_url转变成future对象后,将future对象放到tasks列表中
        tasks.append(asyncio.ensure_future(get_url(url)))
    # gather()方法接收的参数既可以是协程对象也可以是future对象.
    loop.run_until_complete(asyncio.gather(*tasks))
    # 通过future对象的result()方法可以在外部直接获取协程get_url的返回结果
    for future_task in tasks:
        print(future_task.result())

    print("code run finished use time: {}".format(time.time() - start_time))

"""
输出结果:
    code run finished use time: 2.324403762817383
"""

 

 

怎样做到像as_complete()这个函数里面做到的执行完一个打印一个结果,执行完一个打印一个结果.

• asyncio也提供了一个函数as_completed()与线程池as_complete()方法的效果是一样的,某一个协程一但完成就会立马返回.
• as_completed()方法,返回值为协程的迭代器
• 源码as_completed():

•      def as_completed(fs, *, loop=None, timeout=None):
          # Return an iterator whose values are coroutines.返回值为协程的迭代器
          # ...省略代码
  
          # _wait_for_one()是一个协程
          @coroutine
          def _wait_for_one():
              f = yield from done.get()
              if f is None:
                  # Dummy value from _on_timeout().
                  raise futures.TimeoutError
              return f.result()  # May raise f.exception().
  
          # ...省略代码
          for _ in range(len(todo)):
              # 返回协程对象.
              yield _wait_for_one()

import asyncio
from urllib.parse import urlparse
import time


async def get_url(url):
    new_url = urlparse(url)
    host = new_url.netloc
    path = new_url.path
    if path == "":
        path = "/"
    reader, writer = await asyncio.open_connection(host, port=80)
    writer.write("GET {} HTTP/1.1\r\nHost:{}\r\nConnection:close\r\n\r\n".format(path, host).encode("utf8"))

    all_lines = []
    async for raw_line in reader:
        data = raw_line.decode("utf8")
        all_lines.append(data)
    html = "\n".join(all_lines)
    return html


async def main(loop):
    tasks = []
    for i in range(1, 21):
        url = "http://shop.projectsedu.com/goods/{}/".format(i)
        tasks.append(asyncio.ensure_future(get_url(url)))
    # as_completed()方法与线程池as_complete()方法的效果是一样的,某一个协程一但完成就会立马返回.
    # as_completed()方法,返回值为协程的迭代器
    for task in asyncio.as_completed(tasks):
        # task就是一个协程对象,既然是协程就要使用await
        result = await task
        print(result)


if __name__ == '__main__':
    start_time = time.time()
    loop = asyncio.get_event_loop()
    # 此处对协程main进行一个等待,等待协程main执行完成
    loop.run_until_complete(main(loop))
    print("code run finished use time: {}".format(time.time() - start_time))

"""
输出结果:
    code run finished use time: 2.291131019592285

总结:
    通过源码的大概解读,可以看到协程的内部原理都是实现事件循环以及socket进行注册和注销这些核心原理.
    但是协程关键的地方做了future,凡是await的地方协程里面都返回了future对象,这个future对象是不用等待的.
    future对于协程来说是非常的重要.协程不像传统的阻塞IO只有执行成功之后才会返回一个结果.
    但是现在是一个协程,协程需要尽快的返回一个future.
    所以说传统的阻塞驱动实际上已经在协程里面,如果要使用协程的并发特性的话,阻塞驱动是用不上的,
    需要将传统阻塞IO里面的一些函数里面的返回,给我们返回成future(也就是将返回值包装成future).
"""

 

 

 

 

 

 

 

 

 

 

 

 

 

 

 

 

 

 

 

 

 

 

 

 

 

 

 

********