python多线程编程示例

抄下这个代码，需要的时候直接改一下。

操作过程是把rfc官网的目录复制下来，保存在data.txt文件中，然后将data.txt文件读取进来，只要文件中.txt扩展名的名字，拼接rfc官网地址，然后进行下载，由于rfc文件特别多，于是我开了200个线程。下面是干货。

# encoding=utf-8
#
import requests
import urllib3
from threading import Thread as thread
import os

urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
url = 'https://www.rfc-editor.org/rfc/'
header = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:6.0) Gecko/20100101 Firefox/6.0'}
data = open('D:/rfc/data.txt').read()
data = eval(data)
data = [i.strip() for i in data if i.strip().endswith('txt')]


def f1(x):
    while data:
        obj = data.pop()
        myURL = url + obj
        if not os.path.exists(f'D:/rfc/{obj}'):
            print(f'Thread {x} downloading {obj}')
            try:
                mydata = requests.get(url=myURL, headers=header, verify=False).content
                with open(f'D:/rfc/{obj}', 'wb') as f:
                    f.write(mydata)
            except Exception as e:
                print(e)


T = []
num = 200
for i in range(num):
    T.append(thread(target=f1, args=(i,)))
for i in range(num):
    T[i].start()
for i in range(num):
    T[i].join()