python 六re正则表达式、multiprocessing

一、import re

两种使用re.findall、re.compile(pattern[,flags])

 二、multiprocessing 参见(1)，参见(2) 

• 进程的意义 mul
• 进程间不共享全局变量
• 临界资源互斥访问锁 Lock
• 两进程间管道通信（默认全双工） Pipe
• 多进程间通信 Queue
• Pool进程池
• 进程间的共享内存
• Manager以共享进程的方式实现共享数据

① test.py主程序由固有主进程运行，可以调用子进程。好处是运行时间大大减少

　用法：类 Process([group [, target [, name [, args [, kwargs]]]]])  args是被调用函数target的参数。

　　类方法start(),is_alive()判断是否进程是否还活着，join([timeout])等待子进程运行结束的时间 默认无限，terminate()强制终止。

　　类属性：pid os.getpid()当前进程号，os.getppid()父进程号。

# ---- by xu, multiprocessing 实现多进程 
from multiprocessing import Process
import time
import os

def coding(language):
    # 子进程要执行的代码
    for i in range(3):
        print("{} coding({})".format(language, os.getpid()), end=' | ')
        time.sleep(1)

if __name__ == '__main__':
    # 单进程
    start = time.time()
    coding('python')
    for i in range(3):
        print("main program", end=' | ')
        time.sleep(1)
    end = time.time()
    print('\n\tOne process cost time:', end - start)

    # 多进程
    multi_start = time.time()
    p = Process(target=coding, args=('python',))
    p.start()
    for i in range(3):
        print("main program", end=' | ')
        time.sleep(1)
    multi_end = time.time()
    p.join()
    print('\nMulti process cost time:', multi_end - multi_start)

 ② 进程间不共享全局变量，需要管道Pipe Queue的方式

# --- 子进程间不能共享全局变量
from multiprocessing import Process
import time
import os

dig_list = [7, 8, 9]
n_list = len(dig_list)

def add_str1():
    # 子进程1
    for i in range(2):
        for k in range(n_list):
            dig_list[k] += 100
        time.sleep(1)
        print('In process one {}:'.format(os.getpid()), dig_list)

def add_str2():
    # 子进程2
    for i in range(2):
        for k in range(n_list):
            dig_list[k] += 1
        time.sleep(1)
        print('In process two {}:'.format(os.getpid()), dig_list)

if __name__ == '__main__':
    p1 = Process(target=add_str1)
    p1.start()
    p2 = Process(target=add_str2)
    p2.start()

 

 ③ 临界资源互斥访问锁multiprocessing.Lock()   # Main这一句很有意思

# --- 临界资源互斥访问锁
import os
import multiprocessing

# Main
print('Main:', os.getpid())

# worker function
def worker(sign, lock):
    lock.acquire()
    print(sign, os.getpid())
    lock.release()

lock = multiprocessing.Lock()

if __name__ == '__main__':
    record = []
    for i in range(5):
        process = multiprocessing.Process(target=worker, args=('process', lock))
        process.start()
        record.append(process)

    for process in record:
        process.join()

④ 双进程间管道通信， Pipe只用于双进程间 且默认双工通信

# --- by xu, 双进程间管道通信
import multiprocessing as mul


def proc1(pipe):
    pipe.send('hello, my name is proc1.')
    print('proc1 receive: ', pipe.recv())


def proc2(pipe):
    print('proc2 receive: ', pipe.recv())
    pipe.send('shine, i’am proc2.')


# Build a pipe
pipe = mul.Pipe()
if __name__ == '__main__':
    # Pass an end of the pipe to process 1
    p1 = mul.Process(target=proc1, args=(pipe[0],))
    # Pass the other end of the pipe to process 2
    p2 = mul.Process(target=proc2, args=(pipe[1],))

    p1.start()
    p2.start()
    p1.join()
    p2.join()

⑤ 多进程间的管道通信，用Queue实现

# --- by xu, 多进程间管道通信
import multiprocessing as mul

# input worker
def inputQ(queue):
    info = str(os.getpid()) + '(put):' + str(time.time())
    queue.put(info)

# output worker
def outputQ(queue, lock):
    info = queue.get()
    lock.acquire()
    print(str(os.getpid()) + ' get: ' + info)
    lock.release()

lock = mul.Lock()  # To prevent messy print
queue = mul.Queue(3)

if __name__ == '__main__':
    record1 = []  # store input processes
    record2 = []  # store output processes
    # input processes
    for i in range(8):
        process = mul.Process(target=inputQ, args=(queue,))
        process.start()
        record1.append(process)

    # output processes
    for i in range(8):
        process = mul.Process(target=outputQ, args=(queue, lock))
        process.start()
        record2.append(process)

    for p, q in zip(record1, record2):
        p.join()
        q.join()

    queue.close()  # No more object will come, close the queue

 ⑥ Pool进程池

　　示例5个进程的进程池，对Pool.map函数对list中每个元素都作用于f函数。 Pool还有其它函数apply_async(func,args), close(), join()

import multiprocessing as mul

def f(x):
    return x ** 2

if __name__ == '__main__':
    pool = mul.Pool(5)
    rel = pool.map(f, [1, 2, 3, 4, 5, 6, 7, 8, 9, 10])
    print(rel)
# [1, 4, 9, 16, 25, 36, 49, 64, 81, 100]

⑦进程之间共享内存(另一种方式实现)，可略过 用以下⑧中的Manager实现

# --- by xu, 主程序和子进程共享内存
import multiprocessing as mul

# Value/Array
def func1(a, arr):
    a.value = a.value ** 2 * 3.14
    for i in range(len(arr)):
        arr[i] *= 2

if __name__ == '__main__':
    num = mul.Value('d', 2.0)  # num=0
    arr = mul.Array('i', range(10))  # arr=range(10)
    p = mul.Process(target=func1, args=(num, arr))
    p.start()
    p.join()
    print(num.value)
    print(arr[:])

⑧Manager以共享进程的方式 实现共享数据。以下在主/子进程间共享

from multiprocessing import Process, Manager

def func1(shareList, shareDict, shareArray, shareValue, lock):
    with lock:
        for i in range(len(shareList)):
            shareList[i] *= 2
        shareDict['hi'] = 'shine!'
        shareDict['hey'] = '@20034'
        shareArray[0] = 36
        shareValue.value += 1

if __name__ == '__main__':
    manager = Manager()
    lock = manager.Lock()

    list1 = manager.list([1, 2, 3, 4, 5])
    dict1 = manager.dict({'hello': 'ustc.edu'})
    array1 = manager.Array('i', range(10))
    value1 = manager.Value('i', 1)

    proc = [Process(target=func1, args=(list1, dict1, array1, value1, lock)) for i in range(6)]
    for p in proc:
        p.start()
    for p in proc:
        p.join()
    print(list1, dict1, array1, value1, sep='\n')