爬取沪深a股数据
首先从东方财富网获取股票代码
再从网易财经下载股票历史数据
import requests
import random
from bs4 import BeautifulSoup as bs
import time
#import redis
import re
import json
def get_stock_names():
"""
通过东方财富网上爬取股票的名称代码,并存入redis数据库和本地txt文档
"""
rds = redis.from_url('redis://:666666@192.168.3.98:6379', db=1, decode_responses=True) # 连接redis db1
url = "http://quote.eastmoney.com/stocklist.html"
headers = {
'Referer': 'http://quote.eastmoney.com/center/gridlist.html',
'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36'
}
response = requests.get(url, headers=headers).content.decode('utf-8') # 网站编码为gbk 需要解码
soup = bs(response, 'lxml')
all_ul = soup.find('div', id='table_wrapper-table').find_all('ul') # 获取两个ul 标签数据
with open('stock_names.txt', 'w+', encoding='utf-8') as f:
for ul in all_ul:
all_a = ul.find_all('a') # 获取ul 下的所有的a 标签
for a in all_a:
rds.rpush('stock_names', a.text) # a.text 为a标签中的text数据 rpush将数据右侧插入数据库
f.write(a.text + '\n')
def get_data(stocklist, outfile=r'D:\PycharmProjects\web_scraping\stockdata'):
headers = {
'Referer': 'http://quotes.money.163.com/',
'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36'
}
#filelist = [os.path.splitext(file)[0] for file in os.listdir(r'D:\PycharmProjects\web_scraping\stockdata')]
for stock_code, stock_name in stocklist:
#if stock_code in filelist: continue
try:
#stock_code = stock_name.split('(')[1].split(')')[0]
# 由于东方财富网上获取的代码一部分为基金，无法获取数据，故将基金剔除掉。
# 沪市股票以6,9开头，深市以0,2,3开头，但是部分基金也是2开头，201/202/203/204这些也是基金
# 另外获取data的网址股票代码 沪市前加0， 深市前加1
if int(stock_code[0]) in [0, 2, 3, 6, 9]:
if int(stock_code[0]) in [6, 9]:
stock_code_new = '0' + stock_code
elif int(stock_code[0]) in [0, 2, 3]:
if not int(stock_code[:3]) in [201, 202, 203, 204]:
stock_code_new = '1' + stock_code
else: continue
else: continue
else: continue
stock_url = 'http://quotes.money.163.com/trade/lsjysj_{}.html'.format(stock_code)
respones = requests.get(stock_url, headers=headers).text
soup = bs(respones, 'lxml')
start_time = soup.find('input', {'name': 'date_start_type'}).get('value').replace('-', '') #获取起始时间
end_time = soup.find('input', {'name': 'date_end_type'}).get('value').replace('-', '') #获取结束时间
time.sleep(random.choice([1, 2])) #两次访问之间休息1-2秒
download_url = "http://quotes.money.163.com/service/chddata.html?code={}&start={}&end={}&fields=TCLOSE;HIGH;LOW;TOPEN;LCLOSE;CHG;PCHG;TURNOVER;VOTURNOVER;VATURNOVER;TCAP;MCAP".format(stock_code_new, start_time, end_time)
data = requests.get(download_url, headers=headers)
file_name = outfile + '\\{}.csv'.format(stock_code)
with open(file_name, 'wb') as f:
for chunk in data.iter_content(chunk_size=10000): #批量写入数据
if chunk:
f.write(chunk)
print("{}数据已下载".format(stock_code))
except Exception as e:
print("{}({})数据下载报错".format(stock_name, stock_code))
print(e)
import os
# 获取目录下所有文件，绝对路径
# 方法一
def file_name(file_dir):
L=[]
for root, dirs, files in os.walk(file_dir):
for file in files:
if os.path.splitext(file)[1] == '.jpeg':
L.append(os.path.join(root, file))
return L
# 方法二
def listdir(path, list_name):
for file in os.listdir(path): #不包括子目录文件 -> 递归
file_path = os.path.join(path, file)
if os.path.isdir(file_path):
listdir(file_path, list_name)
elif os.path.splitext(file_path)[1]=='.jpeg':
list_name.append(file_path)
stocklist = [] #3770支，只有'0','3','6'开头的
max_page = 189
for i in range(max_page):
url = '''http://1.push2.eastmoney.com/api/qt/clist/get?cb=jQuery112405721872315676919_1566176986516&pn={}
&pz=20&po=1&np=1&ut=bd1d9ddb04089700cf9c27f6f7426281&fltt=2&invt=2&fid=f3&fs=m:0+t:6,m:0+t:13,m:0+t:80,m:1+t:2&
fields=f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f12,f13,f14,f15,f16,f17,f18,f20,f21,f23,f24,f25,f22,f11,f62,f128,f136,f115,f152
&_=1566176986517'''.format(i+1)
response = requests.get(url).content.decode('utf-8')
json_text = re.sub(r'jQuery112405721872315676919_1566176986516\(', '', response)[:-2]
#json_str = re.sub(r'\)', '', response)
json_text = json.loads(json_text)
for fi in json_text['data']['diff']:
stocklist.append([fi['f12'], fi['f14']])
# 下载数据
get_data(stocklist, outfile=r'D:\PycharmProjects\web_scraping\stockdata')
参考资料：
