python 线程池和进程池例子

python 线程池和进程池例子

Tips: 风扇可能会呼呼响

import difflib
import random
import os
from concurrent.futures import ThreadPoolExecutor, wait, ProcessPoolExecutor
import time
import traceback

def spend_time(f):
    def wrapper(*args):
        start = time.time()
        f(*args)
        end = time.time()
        print("%s spend %ss" %(f.__name__, end - start))
    return wrapper


def fn():
    s1 = '''
    比较 fromlines 和 tolines (字符串列表) 并返回一个字符串，表示一个完整 HTML 文件，其中包含各行差异的表格，行间与行外的更改将突出显示。'''.split('\n')
    s2 = '''比较 fromlines 和 tolines (字符串列表) 并返回一个字符串，表示一个完整 HTML 文件，其中包含各行差异的表格，行间与行外的更改将突出显'''.split('\n')
    d = difflib.HtmlDiff()
    filename = os.path.join(os.path.dirname(__file__),'diff%s.html' % random.randint(1,1000000)) 
    # print(filename)
    with open(filename, 'w') as f:
        f.write(d.make_file(s1, s2))
    os.remove(filename)

@spend_time
def ex1(n):
    tasks = []
    ex = ThreadPoolExecutor()
    for i in range(n):
        tasks.append(ex.submit(fn))
    wait(tasks)

@spend_time
def ex0(n):
    for i in range(n):
        fn()

@spend_time
def ex2(n):
    tasks = []
    ex = ProcessPoolExecutor()
    for i in range(n):
        tasks.append(ex.submit(fn))
    wait(tasks)

if __name__ == "__main__":
    try:
        # s
        n = 50000
        ex0(n)
        ex1(n)
        ex2(n)
    except:
        print(traceback.format_exc())

方便获取入参的例子

批量读取 yaml 文件数据到字典中，as_completed 传入的参数只要是可迭代对象就可以。传入的是字典，迭代的是 key，之后可以通过 key 来获取值。

def read_all_yaml(self):
    all_data = {}
    yaml_files = os.listdir(self.directory)
    yaml_file_paths = [os.path.join(self.directory, f) for f in yaml_files if f.endswith('.yaml')]
    with ProcessPoolExecutor() as executor:
        futures = {executor.submit(self._load_yaml, f): f for f in yaml_file_paths}
        for future in as_completed(futures):
            name = futures[future].rsplit('.yaml')[0]
            content = future.result()
            if content is not None:
                all_data[name] = content
    return all_data