Python 多线程是多鸡肋
http://blog.csdn.net/temanm/article/details/53389963
摘要：
Python 对并行化支持的名声就不是很好，如果你用过 Python 自带的线程库 thread 和 threading，你应该知道 Python 线程其实并不好用。例如：没有自带的获取线程返回值的方法，需要自己重写自己的threading。
目录：
- Python 多线程 基础
- Python 多线程 阻塞
- Python 多线程 获取返回值
- Python 多线程 数据对比测试
正文：
一. Python 多线程 基础
1.1 thread 模块
# -*- coding: UTF-8 -*-
import thread
import time
# 为线程定义一个函数
def print_time( threadName, delay):
count = 0
while count < 5:
time.sleep(delay)
count += 1
print "%s: %s" % ( threadName, time.ctime(time.time()) )
# 创建两个线程
try:
thread.start_new_thread( print_time, ("Thread-1", 2, ) )
thread.start_new_thread( print_time, ("Thread-2", 4, ) )
except:
print "Error: unable to start thread"
while 1:
pass- 1
- 2
- 3
- 4
- 5
- 6
- 7
- 8
- 9
- 10
- 11
- 12
- 13
- 14
- 15
- 16
- 17
- 18
- 19
- 20
- 21
执行结果：
Thread-1: Thu Jan 22 15:42:17 2009
Thread-1: Thu Jan 22 15:42:19 2009
Thread-2: Thu Jan 22 15:42:19 2009
Thread-1: Thu Jan 22 15:42:21 2009
Thread-2: Thu Jan 22 15:42:23 2009
Thread-1: Thu Jan 22 15:42:23 2009
Thread-1: Thu Jan 22 15:42:25 2009
Thread-2: Thu Jan 22 15:42:27 2009
Thread-2: Thu Jan 22 15:42:31 2009
Thread-2: Thu Jan 22 15:42:35 2009- 1
- 2
- 3
- 4
- 5
- 6
- 7
- 8
- 9
- 10
1.2 threading模块
# -*- coding: UTF-8 -*-
import threading
def func(a):
print "input is :" + a
threads = []
for i in range(10):
threads.append(threading.Thread(target=func, args=str(i)))
for i in threads:
i.start()- 1
- 2
- 3
- 4
- 5
- 6
- 7
- 8
- 9
- 10
- 11
- 12
- 13
执行结果：
D:\Python\python.exe C:/Users/timen.xu/Desktop/CsdnSpider/Test.py
input is :0
input is :1
input is :2
input is :3
input is :4
input is :5
input is :6
input is :7
input is :8
input is :9
Process finished with exit code 0- 1
- 2
- 3
- 4
- 5
- 6
- 7
- 8
- 9
- 10
- 11
- 12
- 13
二. Python 多线程 阻塞
2.1 阻塞的例子
import threading
import time
def foo():
time.sleep(100)
threads = []
for i in range(5):
threads.append(threading.Thread(target=foo))
for t in threads:
t.start()- 1
- 2
- 3
- 4
- 5
- 6
- 7
- 8
- 9
- 10
- 11
- 12
- 13
- 14
你会发现你的 python 脚本不能使用 ctrl+c 来取消了，那么问题来了，为什么会出现之前说的子线程“阻塞”主线程的问题呢？其实这并不是阻塞，默认情况不加任何参数的话：主线程完成后退出，而子线程并不会退出(daemon 属性为 False)，此时主线程已经完成退出，没有线程接收 ctrl+c 的信号，子线程相当于没人托管，所以不能停止子线程的输出。
2.2 解决方法
import threading
import time
def foo():
time.sleep(100)
threads = []
for i in range(5):
threads.append(threading.Thread(target=foo)
for t in threads:
t.daemon = True
t.start()
while threading.activeCount() > 0:
time.sleep(1)- 1
- 2
- 3
- 4
- 5
- 6
- 7
- 8
- 9
- 10
- 11
- 12
- 13
- 14
- 15
- 16
- 17
- 18
首先配置 daemon 为 True，随后每一秒检查一次活跃的子线程数目，直至所有子线程结束。功能相当于手动实现了一个不会 block 信号的对所有线程调用的 join 方法。
三. Python 多线程 获取返回值
3.1 简介
其实这才是 Python 线程使用中最常遇到的问题，起了很多个线程调用了很多函数，然而直接用 threading 库基本没有办法获取返回值。
3.2 采用队列
这是目前最主流的获取线程数据的方法。使用 Queue 库创建队列实例，用来储存和传递线程间的数据。Python 的队列是线程安全的，也就是说多个线程同时访问一个队列也不会有冲突。Python 队列有三种 FIFO 先进先出，FILO 先进后出（类似栈），优先级队列（由单独的优先级参数决定顺序）。使用队列可以实现简单 生产者 – 消费者 模型
# -*- coding:utf-8 -*-
import threading
import Queue
import random
import time
class Producer(threading.Thread):
def __init__(self, queue):
threading.Thread.__init__(self)
self.data = queue
def run(self):
# 生成随机数字并放入队列
for i in range(10):
num = random.randint(1,20)
print "original num", num
self.data.put(num)
time.sleep(1)
# 主线程
def main():
queue = Queue.Queue()
producer = Producer(queue)
producer.daemon = True
producer.start()
while threading.activeCount() > 0:
time.sleep(1)
if __name__ == '__main__':
main()- 1
- 2
- 3
- 4
- 5
- 6
- 7
- 8
- 9
- 10
- 11
- 12
- 13
- 14
- 15
- 16
- 17
- 18
- 19
- 20
- 21
- 22
- 23
- 24
- 25
- 26
- 27
- 28
- 29
- 30
- 31
- 32
四. Python 多线程 数据对比测试
4.1 数据对比需求
对有鱼股票、新浪财经、富途牛牛中个股的今开、昨收、最高、最低、成交额、成交量、最新价进行数值对比，判断有鱼股票提供的数据的正确性，对比的结果误差不查过1%
需要对比的股票：hs_code_list.xls
4.2 代码编写
4.2.1 Python xlrd 获取excel需要对比数据的股票
# -*- coding:utf-8 -*-
import xlrd
# 从excel中读取数据
def read_text(excel_name, excel_table):
data_excel = []
xls = xlrd.open_workbook(excel_name)
table = xls.sheet_by_name(excel_table)
num_row = table.nrows
for i in range(num_row):
if i is not 0:
stock_market = table.row(i)[1].value
stock_code = table.row(i)[2].value
data = (stock_market, stock_code)
data_excel.append(data)
return data_excel
if __name__ == '__main__':
print len(read_text("hs_code_list.xls", "Sheet1"))- 1
- 2
- 3
- 4
- 5
- 6
- 7
- 8
- 9
- 10
- 11
- 12
- 13
- 14
- 15
- 16
- 17
- 18
- 19
- 20
4.2.2 Python Request获取有鱼股票、新浪财经、富途牛牛个股数据
# -*- coding:utf-8 -*-
import requests
import time
import json
# 有鱼
def request_yff(market_code, stock_code, graph_tab_index, k_not_refresh, stock_type):
url = "https://market-qa.ruifusoft.com/app/v1/quote/user/query/stockdetail"
payload = {
"marketcode": market_code,
"stockcode": stock_code,
"graph_tab_index": graph_tab_index,
"k_not_refresh": k_not_refresh,
"stock_type": stock_type
}
try:
r = requests.get(url, payload)
return r.json()
except Exception, e:
return "http_error"
# 新浪
def request_sina(stock_market, stock_code):
url = "http://hq.sinajs.cn/?list=" + stock_market + stock_code
try:
r = requests.get(url)
return r.text
except Exception, e:
return "http_error"
# 获取富途security_id
def request_futu_security_id(stock_code):
url = "http://www.futunn.com/trade/search?k=" + stock_code
try:
r = requests.get(url)
result = r.json()
security_id = result.get("data")[0].get("security_id")
return security_id
except Exception, e:
return "http_error"
# 富途
def request_futu(stock_code):
date = str(time.time()).split(".")
if request_futu_security_id(stock_code) == "http_error":
return "http_error"
else:
url = "http://www.futunn.com/trade/quote-basic?security_id=" + request_futu_security_id(stock_code) + "&_=" + date[0] + date[1]
r = requests.get(url)
return r.json()
if __name__ == '__main__':
print json.dumps(request_yff("sz", "118850", "0", "0", "010101"), sort_keys=True, indent=2)
print json.dumps(request_futu("150292"), sort_keys=True, indent=2)
sina = request_sina("sh", "000001")
print sina- 1
- 2
- 3
- 4
- 5
- 6
- 7
- 8
- 9
- 10
- 11
- 12
- 13
- 14
- 15
- 16
- 17
- 18
- 19
- 20
- 21
- 22
- 23
- 24
- 25
- 26
- 27
- 28
- 29
- 30
- 31
- 32
- 33
- 34
- 35
- 36
- 37
- 38
- 39
- 40
- 41
- 42
- 43
- 44
- 45
- 46
- 47
- 48
- 49
- 50
- 51
- 52
- 53
- 54
- 55
- 56
- 57
- 58
- 59
- 60
- 61
执行结果：
D:\Python\python.exe C:/Users/timen.xu/Desktop/mw-test-feature/AT-Mobile-IOS-V1.6.0-5db9aa896b7e819c36134436b7244ff07f63b998/automation/data-compare/DealRequest.py
{
"code": 0,
"data": {
"107": "010101",
"205": 1,
"207": 0,
"208": 0,
"ask": [
......
],
"marketcode": "sz",
"price": {
"1": "9.65",
"16": "",
"6": "+0.02",
"7": "+0.21%",
"ct": 1
},
"stockcode": "000001",
"subtitle": {
"content": "\u5348\u95f4\u4f11\u5e02 11-29 11:30:03 ",
"ct": 2
},
"title": "\u5e73\u5b89\u94f6\u884c(000001)",
"trade_detail": [
......
]
},
"msg": ""
}
{
"code": "0",
"data": {
"comm_rate": null,
"look_over": {
"view_num": 4773,
"watch_num": 190130
},
"marketStatus": 4,
"one_queue": null,
"price_range": [
2949.3,
3604.7
],
"quote": {
......
},
"quoteState": 1,
"securityStatic": {
......
}
},
"message": ""
}
var hq_str_sh000001="上证指数,3269.2339,3276.9996,3291.6566,3293.5930,3263.3973,0,0,156706614,173350038164,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2016-11-29,11:33:03,00";
Process finished with exit code 0
- 1
- 2
- 3
- 4
- 5
- 6
- 7
- 8
- 9
- 10
- 11
- 12
- 13
- 14
- 15
- 16
- 17
- 18
- 19
- 20
- 21
- 22
- 23
- 24
- 25
- 26
- 27
- 28
- 29
- 30
- 31
- 32
- 33
- 34
- 35
- 36
- 37
- 38
- 39
- 40
- 41
- 42
- 43
- 44
- 45
- 46
- 47
- 48
- 49
- 50
- 51
- 52
- 53
- 54
- 55
- 56
- 57
- 58
- 59
- 60
4.2.3 设计单个Test Case
# -*- coding:utf-8 -*-
import DealRequest
import CommonMethod
from decimal import Decimal
# 将数据(eg: 366.3612)显示2位小数
def deal_number(result):
if result == "http_error":
return "http_error"
else:
return CommonMethod.round_number(result)
# return str(Decimal(float(result)).quantize(Decimal('0.00')))
# 处理 成交量(eg: 2.10亿手 转换成 210000000)
def deal_deal_number(result):
if result == "http_error":
return "http_error"
else:
if result == 0 or result == "0":
return str(result)
else:
if isinstance(result, float):
return str(result)
elif isinstance(result, int):
return str(result)
else:
message = result[-2]
if message == u"亿":
return str(int(float(result[0: -2]) * 100000000))
elif message == u"万":
return str(int(float(result[0: -2]) * 10000))
else:
return str(result[0: -1])
# 处理 成交额(eg: 2.10亿 转换成 210000000)
def deal_deal_value(result):
if result == "http_error":
return "http_error"
else:
if result == 0 or result == "0":
return str(result)
else:
if isinstance(result, float):
return str(result)
elif isinstance(result, int):
return str(result)
else:
message = result[-1]
if message == u"亿":
return str(float(result[0: -1]) * 100000000)
elif message == u"万":
return str(float(result[0: -1]) * 10000)
else:
return str(result)
# 选择股票的区号类型
def stock_code_type(stock_market):
if stock_market == "sz":
return "010101"
if stock_market == "hk":
return "010104"
if stock_market == "sh":
return "010101"
if stock_market == "us":
return "010105"
# 对比数据： 今开、昨收、最高、最低、最新价
def compare(stock_market, stock_code, module, yff, sina, futu, yff_original, sina_original, futu_original):
if yff == "None":
return CommonMethod.time_now() + u" 【有鱼】" + stock_market + stock_code + " " + module + u" 未找到该个股数据"
else:
if sina == "http_error" or yff == "http_error":
compare_result_sina = "http_error"
else:
compare_result_sina = CommonMethod.percentage(float(yff), float(sina))
if futu == "http_error" or yff == "http_error":
compare_result_futu = "http_error"
else:
compare_result_futu = CommonMethod.percentage(float(yff), float(futu))
if compare_result_futu == "http_error":
if compare_result_sina == "fail":
return "\n" + CommonMethod.time_now() + u" 【有鱼】" + stock_market + stock_code + " " + module + u"数据不一致，差额超过1%。" \
u"有鱼" + module + u"为: " + yff_original +u"，" \
u"新浪" + module + u"为: " + sina_original + u"，" \
u"富途" + module + u"为: 未找到该个股数据"
else:
return "pass"
elif compare_result_sina == "http_error":
if compare_result_futu == "fail":
return "\n" + CommonMethod.time_now() + u" 【有鱼】" + stock_market + stock_code + " " + module + u"数据不一致，差额超过1%。" \
u"有鱼" + module + u"为: " + yff_original + u"，" \
u"新浪" + module + u"为: 未找到该个股数据， " \
u"富途" + module + u"为: " + futu_original
else:
return "pass"
else:
if compare_result_sina == "fail" or compare_result_futu == "fail":
return "\n" + CommonMethod.time_now() + u" 【有鱼】" + stock_market + stock_code + " " + module + u"数据不一致，差额超过1%。" \
u"有鱼" + module + u"为: " + yff_original + u"，" \
u"新浪" + module + u"为: " + sina_original + u"，" \
u"富途" + module + u"为: " + futu_original
else:
return "pass"
# 数据对比
def compare_data(stock_market, stock_code):
CommonMethod.debug("stock_market: " + stock_market, " stock_code: " + stock_code)
yff = DealRequest.request_yff(stock_market, stock_code, "0", "0", stock_code_type(stock_market))
yff_result = yff.get("data").get("datagrid")
if yff_result is not None:
yff_today_price = yff_result[0].get("v")
yff_yesterday_price = yff_result[1].get("v")
yff_top_price = yff_result[4].get("v")
yff_low_price = yff_result[5].get("v")
yff_deal_number = yff_result[2].get("v")
yff_deal_value = yff_result[15].get("v")
yff_values = yff_result[16].get("v")
yff_price_ratio = yff_result[6].get("v")
yff_new_price = yff.get("data").get("price").get("1")
else:
yff_today_price = "None"
yff_yesterday_price = "None"
yff_top_price = "None"
yff_low_price = "None"
yff_deal_number = "None"
yff_deal_value = "None"
yff_values = "None"
yff_price_ratio = "None"
yff_new_price = "None"
CommonMethod.debug(u"------------- you yu ------------- ", "")
CommonMethod.debug(u"今开：", yff_today_price)
CommonMethod.debug(u"昨收：", yff_yesterday_price)
CommonMethod.debug(