python 【爬取百度股票】 

import requests
from bs4 import BeautifulSoup
import traceback
import re


def getHtmlText(url):

    try:
        r = requests.get(url)
        r.raise_for_status
        r.encoding = r.apparent_encoding
        html = r.text
        return html
    except:
        return ""

        

def getStockList(lst,stockURL):

    html = getHtmlText(stockURL)
    soup = BeautifulSoup(html,"html.parser")
    a = soup.find_all('a')
    for i in a:
        try:
            href = i.attrs['href']
            lst.append(re.findall(r"[s][hz]\d{6}",href)[0])
        except:
            continue
    

def getStockInfo(lst,stockURL,fpath):

    for stock in lst:

        url = stockURL + stock + ".html"
        html = getHtmlText(url)
        try:
            if html == "":
                continue
            infoDict = {}
            soup = BeautifulSoup(html,"html.parser")
            stockInfo = soup.find('div',attrs={'class':'stock-bets'})

            name = stockInfo.find_all(attrs={'class':'bets-name'})[0]

            infoDict.update({"股票名称":name.text.split()[0]})

            keyList = stockInfo.find_all('dt')
            valueList = stockInfo.find_all('dd')
            for i in range(len(keyList)):
                key = keyList[i].text
                val = valueList[i].text
                infoDict[key] = val
            with open(fpath,'a',encoding="utf-8") as f:
                f.write(str(infoDict) + '\n')
        except:
            traceback.print_exc()
            continue


def main():
    stock_list_url = "http://quote.eastmoney.com/stocklist.html"
    stock_info_url = "https://gupiao.baidu.com/stock/"
    output_file = "D://stock.txt"
    slist = []
    getStockList(slist,stock_info_url)
    getStockInfo(slist,stock_info_url,output_file)
if __name__ == "__main__":

    main()

 

 1 import requests
 2 from bs4 import BeautifulSoup
 3 import traceback
 4 import re
 5 
 6 
 7 def getHtmlText(url):
 8 
 9     try:
10         r = requests.get(url)
11         r.raise_for_status
12         r.encoding = r.apparent_encoding
13         html = r.text
14         return html
15     except:
16         return ""
17 
18         
19 
20 def getStockList(lst,stockURL):
21 
22     html = getHtmlText(stockURL)
23     soup = BeautifulSoup(html,"html.parser")
24     a = soup.find_all('a')
25     for i in a:
26         try:
27             href = i.attrs['href']
28             lst.append(re.findall(r"[s][hz]\d{6}",href)[0])
29         except:
30             continue
31     
32 
33 def getStockInfo(lst,stockURL,fpath):
34     count = 0
35     for stock in lst:
36 
37         url = stockURL + stock + ".html"
38         html = getHtmlText(url)
39         try:
40             if html == "":
41                 continue
42             infoDict = {}
43             soup = BeautifulSoup(html,"html.parser")
44             stockInfo = soup.find('div',attrs={'class':'stock-bets'})
45 
46             name = stockInfo.find_all(attrs={'class':'bets-name'})[0]
47 
48             infoDict.update({"股票名称":name.text.split()[0]})
49 
50             keyList = stockInfo.find_all('dt')
51             valueList = stockInfo.find_all('dd')
52             for i in range(len(keyList)):
53                 key = keyList[i].text
54                 val = valueList[i].text
55                 infoDict[key] = val
56             with open(fpath,'a',encoding="utf-8") as f:
57                 f.write(str(infoDict) + '\n')
58                 count = count + 1
59                 print('\r当前速度:{:.2f}%'.format(count*100/len(lst)),end="")
60         except:
61             
62             count = count + 1
63             print('\r当前速度:{:.2f}%'.format(count*100/len(lst)),end="")
64             traceback.print_exc()
65             continue
66 
67 
68 def main():
69     stock_list_url = "http://quote.eastmoney.com/stocklist.html"
70     stock_info_url = "https://gupiao.baidu.com/stock/"
71     output_file = "D://stock.txt"
72     slist = []
73     getStockList(slist,stock_info_url)
74     getStockInfo(slist,stock_info_url,output_file)
75 if __name__ == "__main__":
76 
77     main()
78
优化

 

posted @ 2018-08-15 11:39  Justice-V  阅读(260)  评论(0)    收藏  举报