Python多进程、多线程与协程

多进程: 适合处理CPU密集型任务，利用多核处理器并行计算。
多线程: 适合IO密集型任务，因为线程可以在等待IO完成时释放GIL(全局解释器锁)，允许其他线程运行。
协程: 适合处理大量并发的IO密集型任务，相比于传统的线程切换，协程切换的开销更小。

多进程

不使用多进程，统计耗时

import time

# CPU密集型任务，例如计算斐波那契数列
def fibonacci(n):
    if n <= 1:
        return n
    else:
        return fibonacci(n - 1) + fibonacci(n - 2)

if __name__ == '__main__':
    numbers = [30, 31, 32, 33]
    # 统计耗时
    start_time = time.time()
    results_single = [fibonacci(num) for num in numbers]
    end_time = time.time()

    print("Results:", results_single)
    print("Time Cost:", end_time - start_time, "seconds")
# 耗时约2.6s

使用multiprocessing库

import multiprocessing
import time

# CPU密集型任务，例如计算斐波那契数列
def fibonacci(n):
    if n <= 1:
        return n
    else:
        return fibonacci(n - 1) + fibonacci(n - 2)

if __name__ == '__main__':
    # 创建一个进程池，这里使用默认的CPU核心数
    pool = multiprocessing.Pool()
    numbers = [30, 31, 32, 33]
    start_time = time.time()
    # 使用进程池并行计算每个数的斐波那契数
    results = pool.map(fibonacci, numbers)
    end_time = time.time()

    # 关闭进程池
    pool.close()
    pool.join()

    # 打印结果和耗时
    print("Results:", results)
    print("Time Cost with multiprocessing:", end_time - start_time, "seconds")
# 耗时约1.3s

多线程

不使用多线程，统计耗时

import requests
import time

# 免费可用于测试的API
# https://jsonplaceholder.typicode.com/
# Free fake and reliable API for testing and prototyping.
urls = [
    'https://jsonplaceholder.typicode.com/posts/1',
    'https://jsonplaceholder.typicode.com/posts/2',
    'https://jsonplaceholder.typicode.com/posts/3',
    'https://jsonplaceholder.typicode.com/posts/4',
    'https://jsonplaceholder.typicode.com/posts/5'
]

# IO密集型任务，例如网络请求
def fetch_data(url):
    response = requests.get(url)
    return response.json()

if __name__ == '__main__':
    start_time = time.time()
    results = [fetch_data(url) for url in urls]
    end_time = time.time()

    print("Time Cost:", end_time - start_time, "seconds")
# 耗时约3.0s

使用threading模块创建线程

import requests
import threading
import time

urls = [
    'https://jsonplaceholder.typicode.com/posts/1',
    'https://jsonplaceholder.typicode.com/posts/2',
    'https://jsonplaceholder.typicode.com/posts/3',
    'https://jsonplaceholder.typicode.com/posts/4',
    'https://jsonplaceholder.typicode.com/posts/5'
]

def fetch_data(url):
    response = requests.get(url)
    return response.json()

if __name__ == '__main__':
    start_time = time.time()
    # 为每个URL创建一个线程
    threads = [threading.Thread(target=fetch_data, args=(url,)) for url in urls]
    # 启动所有线程
    for thread in threads:
        thread.start()
    # 确保所有线程结束
    for thread in threads:
        thread.join()

    end_time = time.time()
    print("Time taken with threading:", end_time - start_time, "seconds")
# 耗时约0.68s

使用线程池方式实现

import requests
import concurrent.futures
import time

urls = [
    'https://jsonplaceholder.typicode.com/posts/1',
    'https://jsonplaceholder.typicode.com/posts/2',
    'https://jsonplaceholder.typicode.com/posts/3',
    'https://jsonplaceholder.typicode.com/posts/4',
    'https://jsonplaceholder.typicode.com/posts/5'
]


def fetch_data(url):
    response = requests.get(url)
    return response.json()


if __name__ == '__main__':
    start_time = time.time()
    # Create a thread pool with a maximum of 5 threads (one for each URL)
    with concurrent.futures.ThreadPoolExecutor(max_workers=len(urls)) as executor:
        # Submit each URL fetching task to the thread pool
        futures = [executor.submit(fetch_data, url) for url in urls]
        # Wait for all futures (tasks) to complete
        results = [future.result() for future in concurrent.futures.as_completed(futures)]

    end_time = time.time()
    print("Results with thread pool:", results)
    print("Time taken with thread pool:", end_time - start_time, "seconds")
# 耗时约0.84s

协程

使用协程方式实现

import asyncio
import aiohttp
import time

urls = [
    'https://jsonplaceholder.typicode.com/posts/1',
    'https://jsonplaceholder.typicode.com/posts/2',
    'https://jsonplaceholder.typicode.com/posts/3',
    'https://jsonplaceholder.typicode.com/posts/4',
    'https://jsonplaceholder.typicode.com/posts/5'
]


# 协程方式获取数据
async def fetch_data(url):
    async with aiohttp.ClientSession() as session:
        async with session.get(url) as response:
            return await response.json()


async def main():
    start_time = time.time()

    # Gather all coroutine tasks for fetching data
    tasks = [fetch_data(url) for url in urls]
    results = await asyncio.gather(*tasks)

    end_time = time.time()

    print("Results with coroutines:", results)
    print("Time taken with coroutines:", end_time - start_time, "seconds")


if __name__ == '__main__':
    asyncio.run(main())
# 耗时约0.70s