joblib多进程总结

基本用法

通过这个用例可以观察到启动了5个进程。

from joblib import Parallel, delayed

def get_pid(p_num):
    import os
    import time
    time.sleep(10)
    return "%d_%d" % (p_num, os.getpid())
# function delayed to warp (proxy) a parallel function
tasks = [delayed(get_pid)(_) for _ in range(100)]

# use 5 progress to run tasks
ret = Parallel(n_jobs=5)(tasks)

print(ret)

多任务进度条tqdm

tqdm 可以展示进度，并且预估时间。

from tqdm.auto import tqdm
from joblib import Parallel

class ProgressParallel(Parallel):
    def __init__(self, use_tqdm=True, total=None, *args, **kwargs):
        self._use_tqdm = use_tqdm
        self._total = total
        super().__init__(*args, **kwargs)

    def __call__(self, *args, **kwargs):
        with tqdm(disable=not self._use_tqdm, total=self._total) as self._pbar:
            return super(ProgressParallel, self).__call__(*args, **kwargs)

    def print_progress(self):
        if self._total is None:
            self._pbar.total = self.n_dispatched_tasks
        self._pbar.n = self.n_completed_tasks
        self._pbar.refresh()

使用的时候：

ret = ProgressParallel(n_jobs=5)(tasks, total=5)