python 多进程+多线程实例

multiprocessing是一个与threading模块类似API的多进程库。multiprocessing库提供了一致的本地和远程的方法，使用子程序替代线程来处理全局解释锁。因此，multiprocessing库允许编程者在给定机器上使用多个进程。在Unix和Windows上都有效果。

from multiprocessing import Pool, Manager
import threading
import os


class ProcessAndThread:
    def __init__(self, input_path, processes_num=2, threads_num=3):
        self.input_path = input_path
        self.processes_num = processes_num                          # 进程数
        self.threads_num = threads_num                              # 线程数
        self.queue = Manager().list()                               # 共享管道队列

        # 将要处理的文件放入共享队列中
        for root, dirs, files in os.walk(self.input_path):
            for filename in files:
                if filename.endswith(".jpg") or filename.endswith(".png"):
                    file_path = os.path.join(root, filename)
                    self.queue.append(file_path)

    def process_file(self, file_path):
        # 处理文件的函数，这里只是简单输出文件路径
        print("Processing file: {}".format(file_path))

    def process(self):
        while True:
            # 从共享队列中获取一个待处理的文件
            try:
                file_path = self.queue.pop(0)
            except IndexError:
                break
            self.process_file(file_path)

    def start_threading(self):                                              # 开启多线程
        for i in range(self.threads_num):
            thread = threading.Thread(target=self.process)
            thread.start()
        for thread in threading.enumerate():
            if thread != threading.current_thread():
                thread.join()

    def start_processes(self):
        with Pool(processes=self.processes_num) as pool:                    # 创建进程池
            pool.apply(func=self.start_threading)                           # 同步执行
        pool.close()                                                        # 关闭进程池
        pool.join()                                                         # 回收


if __name__ == '__main__':
    ProcessAndThread("E:/Xinke_Download_Files/SAC_IMG_D",6, 4).start_processes()