search_url_get_msg.py

 

 

#!/usr/bin/env python3
import os
import logging
import glob
import sys
import re
from logging.handlers import RotatingFileHandler
from multiprocessing import Process
from multiprocessing import cpu_count
import multiprocessing
def search_url_message(i,url,path,result):
    logger.info("child_process {} cmd grep... ".format(i))
    comand = 'grep "{}" {}'.format(url,path)
    result_cmd = os.popen(comand)
    res = result_cmd.read()
    #print(comand)
    if res:
        result["0"] = res
def start_process(url):
    process_list = []
    result = multiprocessing.Manager().dict()
    path = '/var/log/yisa_get_msg_from_jd_kafka_process.log*'
    paths_list = []
    for i in range(321):
        if i != 0:
            paths_list.append('/var/log/yisa_get_msg_from_jd_kafka_process.log' + '.' +str(321-i))
        else:
            paths_list.append('/var/log/yisa_get_msg_from_jd_kafka_process.log')
    #for p in glob.glob(path):
    for p in paths_list:
        if '0' in result.keys():
            print(result["0"])
            break
            #sys.exit(0)
        if len(process_list) == cpu_count():
            while True:
                flag = 0
                for pro in process_list:
                    if not pro.is_alive():
                        process_list.remove(pro)
                        flag = 1
                if flag == 1:
                    break
        Pro = Process(target=search_url_message,args=(i,url,p,result))
        logger.info("child_process {} started... ".format(i))
        Pro.start()
        process_list.append(Pro)
        i = i + 1
        #if i == 60:
        #    break
    for p in process_list:
         p.kill()
if __name__ == '__main__':
    logger = logging.getLogger()
    logger.setLevel(logging.INFO)
    handler = RotatingFileHandler(os.path.dirname(os.path.abspath(__file__)) + "/dl_images.log")
    formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
    handler.setFormatter(formatter)
    logger.addHandler(handler)
    if len(sys.argv) == 2:
        url = sys.argv[1]
        start_process(url)
    else:
        print("Unknown command")