Python: asyncio实战

 

Server端设置3s才返回数据

 

 

单线程

import requests, random, string, datetime


def download(url):
    print('start')
    response = requests.get(url)
    print('complete')
    lst = url.rsplit(sep = '/', maxsplit = 1)
    file = 'images/{}-{}'.format(random.choice(string.ascii_lowercase), lst[-1])
    with open(file, mode = 'w+b') as f:
        f.write(response.content)


if __name__ == '__main__':
    urls = ['http://localhost:555/fake.txt'] * 3
    commence = datetime.datetime.now()
    for url in urls:
        download(url)
    closure = datetime.datetime.now()
    print(f'elapsed: {(closure - commence).total_seconds()}')

  

aiohttp & asyncio.create_task

import requests, random, string, datetime, asyncio, aiohttp


async def fetch(session: aiohttp.ClientSession, url):
    print(f'fetch {url}')
    async with session.get(url, verify_ssl = False) as response:
        content = await response.content.read()
        file_name = 'images/{}-{}'.format(random.choice(string.ascii_lowercase), url.rsplit(sep = '/', maxsplit = 1)[-1])
        with open(file = file_name, mode = 'w+b') as f:
            f.write(content)
    print(f'fetch done')
    return 55


async def main():
    async with aiohttp.ClientSession() as session:
        urls = ['http://localhost:555/fake.txt'] * 3
        tasks = [asyncio.create_task(fetch(session, url)) for url in urls]
        await asyncio.wait(tasks)
        res = [task.result() for task in tasks]
        print(res)


if __name__ == '__main__':
    commence = datetime.datetime.now()
    asyncio.run(main())
    closure = datetime.datetime.now()
    print(f'elapsed: {(closure - commence).total_seconds()}')

 

asyncio+concurrent.futures.ThreadPoolExecutor+concurrent.futures.ProcessPoolExecutor 解决不支持异步的函数

DeprecationWarning: 底层方法

import asyncio, concurrent.futures, requests, random, string


async def download(url: str):
    print('commence', url)
    loop = asyncio.get_event_loop()
    future = loop.run_in_executor(None, requests.get, url)
    response = await future
    print('closure')
    file_name = 'images/{1}-{0}'.format(url.rsplit(sep = '/', maxsplit = 1)[-1], random.choice(string.ascii_lowercase))
    with open(file_name, mode = 'wb') as f:
        f.write(response.content)


if __name__ == '__main__':
    urls = ['http://localhost:555/fake.txt'] * 3
    tasks = [download(url) for url in urls]
    loop = asyncio.get_event_loop()
    loop.run_until_complete(asyncio.wait(tasks))

NEW API

import asyncio, concurrent.futures, requests, random, string


async def download(url: str):
    print('commence', url)
    loop = asyncio.get_event_loop()
    future = loop.run_in_executor(None, requests.get, url)
    response = await future
    print('closure')
    file_name = 'images/{1}-{0}'.format(url.rsplit(sep = '/', maxsplit = 1)[-1], random.choice(string.ascii_lowercase))
    with open(file_name, mode = 'wb') as f:
        f.write(response.content)
    return 55


if __name__ == '__main__':
    urls = ['http://localhost:555/fake.txt'] * 3
    tasks = [download(url) for url in urls]
    result = asyncio.run(asyncio.wait(tasks))
    print(tasks)
    print(result)