多线程爬虫

# 多线程爬虫
# map函数的使用
# from multiprocessing.dummy import Pool
# pool=Pool(4)
# results = pool.map(爬取函数,网址列表)
# 实例演示:
from multiprocessing.dummy import Pool as ThreadPool
import requests
import time

def getsource(url):
    html = requests.get(url)

urls = []

for i in range(1,21):
    newpage = 'http://tieba.baidu.com/p/3522395718?pn=' + str(i)
    urls.append(newpage)

time1 = time.time()

for i in urls:
    print(i)
    getsource(i)
time2= time.time()
print('单线程耗时:' + str(time2-time1))

# 启用多线程
pool = ThreadPool(4)
time3=time.time()
results = pool.map(getsource,urls)
pool.close()
pool.join()
time4 = time.time()
print('并行耗时:' + str(time4-time3))

# 输出结果:
# 单线程耗时:20.18715476989746
# 并行耗时:5.100291728973389