python多线程拉数据库数据

#!/usr/bin/python
#coding:utf-8
''' The main function '''

import sys
import time
import traceback
import json
import MySQLdb
import httplib
import urllib
import hashlib
from multiprocessing import Pool,Manager
import random

COPY_DB = {
    'xxxxxxxxxx_0':{'host':'xxxxxxx.12', 'port':0000},
    'xxxxxxxxxx_1':{'host':'xxxxxxx.12', 'port':0000},
    'xxxxxxxxxx_2':{'host':'xxxxxxx.12', 'port':0000},
    'xxxxxxxxxx_3':{'host':'xxxxxxx.12', 'port':0000},
    'xxxxxxxxxx_4':{'host':'xxxxxxx.12', 'port':0000},
    'xxxxxxxxxx_5':{'host':'xxxxxxx.12', 'port':0000},
    'xxxxxxxxxx_6':{'host':'xxxxxxx.12', 'port':0000},
    'xxxxxxxxxx_7':{'host':'xxxxxxx.12', 'port':0000},
    'xxxxxxxxxx_8':{'host':'xxxxxxx.16', 'port':0000},
    'xxxxxxxxxx_9':{'host':'xxxxxxx.16', 'port':0000},
    'xxxxxxxxxx_a':{'host':'xxxxxxx.16', 'port':0000},
    'xxxxxxxxxx_b':{'host':'xxxxxxx.16', 'port':0000},
    'xxxxxxxxxx_c':{'host':'xxxxxxx.16', 'port':0000},
    'xxxxxxxxxx_d':{'host':'xxxxxxx.16', 'port':0000},
    'xxxxxxxxxx_e':{'host':'xxxxxxx.16', 'port':0000},
    'xxxxxxxxxx_f':{'host':'xxxxxxx.16', 'port':0000},
}

def get_md5(raw):
    hs = hashlib.md5()
    hs.update((raw).encode(encoding='utf-8'))
    return hs.hexdigest()

def get_vfilename(bucketname,filename):
    return xxxxxxxxx

def get_contentkey(filename):
    vfilename = get_vfilename('xxxxx',filename)
    database = 'xxxxx'+ vfilename[0:1]
    try:
        conn = MySQLdb.connect(host = COPY_DB[database]['host'], port = COPY_DB[database]['port'], user="xxxxxxx", passwd="xxxxxx", db=database, charset="utf8", connect_timeout=5)

        cursor = conn.cursor()
        table_name = "bss_file_" + vfilename[1:3]
        sql = "SELECT xxxx FROM " + table_name + " where xxxx =\'"+vfilename+"\'"
        if cursor.execute(sql) <= 0:
            fp = open('err.log', 'a')
            fp.write('err, ' + database  + ',' + table_name +',' + filename+'\n')
            fp.close()

        fp = open('/data1/upload/xxxxxxxxxx.log', 'a')
        for row in cursor.fetchall():
            fp.write(filename + ',' + str(row[0]) + '\n')
        fp.close()

        return str(row[0])


    except Exception as e:
        print(e)    

def get_path(filename, contentkey):
    database = 'xxxxxxxxxx_'+ contentkey[0:1]
    try:
        conn = MySQLdb.connect(host = COPY_DB[database]['host'], port = COPY_DB[database]['port'], user="xxxxx", passwd="xxxx", db=database, charset="utf8", connect_timeout=5)

        cursor = conn.cursor()
        table_name = "xxxx_" + contentkey[1:3]
        sql = "SELECT xxxx FROM " + table_name + " where xxxx =\'"+contentkey+"\'"
        if cursor.execute(sql) <= 0:
            fp = open('err.log', 'a')
            fp.write('err, ' + database  + ',' + table_name +',' + filename+'\n')
            fp.close()

        fp = open('/data1/upload/xxxxxxxx.log', 'a')
        for row in cursor.fetchall():
            fp.write(filename + ',' + str(row[0]) + '\n')
        fp.close()

        return str(row[0])


    except Exception as e:
        print(e)

def get_filename(filepath):
    data = []
    with open(filepath, "r") as f:  # 打开文件
        for line in f.readlines():
            line = line.strip('\n')  #去掉列表中每一个元素的换行符
            data.append(line)

        return data

def worker(filename,count_que):
    count = count_que.get()
    contentkey = get_contentkey(filename)
    filepath = get_path(filename, contentkey)
    count = count+1
    count_que.put(count)
    print(str(count)+', '+filename)
    time.sleep(random.random())

if __name__ == "__main__":
    filenames = get_filename("xxxxxxxxx.txt")
    pool = Pool(256)
    count_que = Manager().Queue()
    count_que.put(0)
    for filename in filenames:
        pool.apply_async(worker, (filename,count_que))

    pool.close()
    pool.join()
    print("Total solve:", count_que.get())