Using python contral bash commmands and run them in parallel

Goal:

1. Read commands from pipe line by line, each command occpuying one line.

2. Run these commands in parallel, and output their outputs(stdout and stderr) simutaliously. 

Technical nutshell:

1. Run bash commands by subprocess

2. Using threads moniterring outputs.

CODE:

#!/usr/bin/env python

'''
    Read commands from stdin and process it line by line.
    
    @Author: 
    @Date: 2013 - 07 - 27
    @Version: 1.0
    
    @Ref.:
    http://code.activestate.com/recipes/577376-simple-way-to-execute-multiple-process-in-parallel/
    http://sharats.me/the-ever-useful-and-neat-subprocess-module.html
'''

import sys,time
from subprocess import Popen,PIPE,STDOUT
from threading import Thread
from Queue import Empty,Queue

def help():
    sys.stderr.write('''
    Read commands from stdin, and run them in parallel line by line
    
    @Author: 
    @Version: 1.0
    
    @Usages:
    para1: How many cpus do you want to use.(eg. 10)
    
    @Note:
    1. Read from stdin and output messages to stdout.
    2. This program can only run on Linux system, invoke '/bin/bash' to interpretate,
        this could make the substitution feature works. 
    \n''')
    sys.stderr.flush()
    sys.stderr.close()
    sys.exit(-1)

class P(object):
    cpu_num = 1 #the number of cpus want to use.

def executeCommands(cmdList):
    '''Execute commands from cmd list in parallel.'''
    totalTask = len(cmdList)
    doneTask = 0

    def checkDone(p):
        '''check whether process is done!'''
        return p.poll() is not None
    def checkSuccess(p):
        '''check whether process exited successful'''
        return p.returncode == 0
    def closeWriter():
        '''flush all writer out'''
        sys.stdout.flush()
        sys.stdout.close()
        sys.stderr.flush()
        sys.stderr.close()    
    
    stdout_Queue = Queue()
    stderr_Queue = Queue()
    #process starts from here!
    processing = []
    
    
    def streamWatcher(queue, stream):
        #print('streamWatcher-stdout\n')
        for line in stream: #this will be blocked if no data in buffer.
            queue.put(line)
        if not stream.closed:
            stream.close()
        
        #queue.task_done()
    
    #def streamPrinter(stdout_Queue,stderr_Queue):
    def streamPrinter(queue,stream):
        '''printing messages in queue'''
        while(1):
            try:
                line = queue.get(True,0.1)
            except Empty:
                pass
            else:
                queue.task_done()
                stream.write(line)
        
    #Using a thread to do printing.
    threadPool = []
    
    t = Thread(target=streamPrinter,name='stdout_printer',args=(stdout_Queue,sys.stdout))
    t.daemon = True
    t.start()
    
    t = Thread(target=streamPrinter,name='stderr_printer',args=(stderr_Queue,sys.stderr))
    t.daemon = True
    t.start()
    #threadPool.append(t)
    
    while(1):
        while cmdList and len(processing) < P.cpu_num:
            #rediect stderr to STDOUT
            #print(cmdList)
            cmd=cmdList.pop(0)
            p = Popen(cmd,stdout=PIPE,shell=True,stderr=PIPE,bufsize=1
                      ,close_fds=True
                      ,executable='/bin/bash'
                      )
            p.cmd = cmd
            processing.append(p)
            #Using threading to watch output stream
            t =Thread(target=streamWatcher, name='stdoutWatcher',args=(stdout_Queue,p.stdout))
            t.daemon = True
            t.start()

            
            t = Thread(target=streamWatcher, name='stderrWatcher',args=(stderr_Queue,p.stderr))
            t.daemon = True
            t.start()

            
        #check output and run states.
        for p in processing:
            #checkOutput(p)
            if checkDone(p):
                if checkSuccess(p):
                    doneTask += 1
                    processing.remove(p)
                else:
                    sys.stderr.write('***Caution: there are commands exited unexpectedly , System exited!***\n')
                    sys.stderr.write('Error Invoking Command:\n%s\n'%(p.cmd))
                    
                    #kill all the running process
                    for x in processing:
                        try:
                            x.kill()
                        except OSError:
                            pass
                        
                    closeWriter()
                    sys.exit(-1)
        
        if not processing and not cmdList:
            break
        else:
            time.sleep(0.1)
            
    # waiting queue done
    stdout_Queue.join()
    stderr_Queue.join()
    

if __name__ == '__main__':
    if(len(sys.argv) != 2):
        help()
    
    P.cpu_num = int(sys.argv[1])
    
    cmds = []
    for line in sys.stdin:
        line = line.strip()
        if len(line) > 0:
            cmds.append(line)
    
    executeCommands(cmds)
    
    

 

 Points:

　　1.  t.daemon = True # set thread dies with the program. 

　　2. stdout_Queue.join() # wait on the Queue until everything has been processed.

　　3. stdout_Queue.task_done() # singals to Queue job is done.

　　4. Queue.get([block[, timeout]])¶ Remove and return an item from the queue. If optional args block is true and timeout is None (the default), block if necessary until an item is available. If timeout is a positive number, it blocks at most timeout seconds and raises the Empty exception if no item was available within that time. Otherwise (block is false), return an item if one is immediately available, else raise the Empty exception (timeout is ignored in that case).

　　5. Queue.put(item[, block[, timeout]])¶ Put item into the queue. If optional args block is true and timeout is None (the default), block if necessary until a free slot is available. If timeout is a positive number, it blocks at mosttimeout seconds and raises the Full exception if no free slot was available within that time. Otherwise (block is false), put an item on the queue if a free slot is immediately available, else raise the Full exception (timeout is ignored in that case).