第二次作业
作业①
1)、爬取与储存天气预报数据实验
from bs4 import BeautifulSoup
from bs4 import UnicodeDammit
import urllib.request
import sqlite3
import prettytable as pt
x = pt.PrettyTable() # 制表
x.field_names = ["序号", "地区", "日期", "天气信息", "温度"] # 设置表头
class WeatherDB:
def openDB(self):
self.con = sqlite3.connect("weathers.db")
self.cursor = self.con.cursor()
try:
self.cursor.execute(
"create table weathers (wCity varchar(16),wDate varchar(16),wWeather varchar(64),wTemp varchar(32),constraint pk_weather primary key (wCity,wDate))")
except:
self.cursor.execute("delete from weathers")
def closeDB(self):
self.con.commit()
self.con.close()
def insert(self, city, date, weather, temp):
try:
self.cursor.execute("insert into weathers (wCity,wDate,wWeather,wTemp) values (?,?,?,?)",
(city, date, weather, temp))
except Exception as err:
print(err)
def show(self):
num = 1 # 设置序号
self.cursor.execute("select * from weathers")
rows = self.cursor.fetchall()
for row in rows:
x.add_row([num, row[0], row[1], row[2], row[3]])
num += 1
print(x)
class Weatherforecast():
def __init__(self):
self.headers = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) "
"Chrome/70.0.3538.25 Safari/537.36 Core/1.70.3775.400 QQBrowser/10.6.4209.400"}
self.citycode = {"北京": "101010100", "上海": "101020100", "广州": "101280101", "深圳": "101280601"}
def forecastcity(self, city):
if city not in self.citycode.keys():
print(city + "code not found")
return
url = "http://www.weather.com.cn/weather/" + self.citycode[city] + ".shtml"
try:
req = urllib.request.Request(url, headers=self.headers)
data = urllib.request.urlopen(req)
data = data.read()
dammit = UnicodeDammit(data, ["utf-8", "gbk"])
data = dammit.unicode_markup
soup = BeautifulSoup(data, 'html.parser')
lis = soup.select("ul[class='t clearfix'] li")
for li in lis:
try:
date_ = li.select('h1')[0].text
weather_ = li.select('p[class="wea"]')[0].text
temp_ = li.select('p[class="tem"] span')[0].text + '℃/' + li.select("p[class='tem'] i")[0].text
self.db.insert(city, date_, weather_, temp_)
except Exception as err:
print(err)
except Exception as err:
print(err)
def process(self, cities):
self.db = WeatherDB()
self.db.openDB()
for city in cities:
self.forecastcity(city)
self.db.show()
self.db.closeDB()
ws = Weatherforecast()
ws.process(["北京", '上海', '广州', '深圳'])
2)、心得体会
这次实验初步学习了在Python中对于数据库的一些用法，和上学期学的大数据基础实践学习的知识有一部分重叠，所以几乎都能不费力地理解代码。之前不理解self的用法，在这次实验中我通过上网查找以及询问同学，弄懂了self的用法。
作业②
1)、爬取股票相关信息实验
import urllib
import urllib.request
import re
from bs4 import UnicodeDammit, BeautifulSoup
import prettytable as pt
x = pt.PrettyTable() # 制表
x.field_names = ["序号", "代码", "名称", "最新价", "涨跌幅", "跌涨额", "成交量", "成交额", "振幅","最高","最低","今开","昨收"] # 设置表头
def getHtml(fs, fields, page, pz):
headers = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) ""Chrome/70.0.3538.25 Safari/537.36 Core/1.70.3775.400 QQBrowser/10.6.4209.400"}
# 需要把page和pz转换成字符，不然会出现can only concatenate str (not "int") to str的报错
url = "http://58.push2.eastmoney.com/api/qt/clist/get?cb=jQuery112409968248217612661_1601548126340&pn=" + str(
page) + "&pz=" + str(pz) + "&po=1&np=1&ut=bd1d9ddb04089700cf9c27f6f7426281&fltt=2&invt=2&fid=f3&fs=" + \
fs + "&fields=" + fields + "&_=1601548126345"
req = urllib.request.Request(url, headers=headers)
data = urllib.request.urlopen(req)
data = data.read()
dammit = UnicodeDammit(data, ["utf-8", "gbk"])
data = dammit.unicode_markup
soup = BeautifulSoup(data, 'html.parser')
data = re.findall(r'"diff":\[(.*?)]', soup.text)
return data
# 获取股票数据
def getOnePageStock(num, fields, fs, page, pz):
data = getHtml(fs, fields, page, pz)
datas = data[0].strip("{").strip("}").split('},{') # 去掉头尾的"{"和"}"，再通过"},{"切片
for i in range(len(datas)):
stock = datas[i].replace('"', "").split(",") # 去掉双引号并通过","切片
x.add_row(
[num, stock[6][4:], stock[7][4:], stock[0][3:], stock[1][3:], stock[2][3:], stock[3][3:], stock[4][3:],
stock[5][3:], stock[8][4:], stock[9][4:], stock[10][4:], stock[11][4:]]) # 将数据按行存入表格
num = num + 1
return num # 更新序号
def main():
num = 1 # 序号
page = 1 # 设置所爬取的页数
pz = 20 # 设置一页有多少条信息
fields = "f12,f14,f2,f3,f4,f5,f6,f7,f15,f16,f17,f18" # 设置爬取信息（f12:代码，f14：名称，f2：最新价，f3：涨跌幅，f4：涨跌额，f5：成交量，f6：成交额，f7：涨幅, f15: 最高, f16: 最低, f17: 今开, f18: 昨收）
fs = {
"沪深A股": "m:0+t:6,m:0+t:13,m:0+t:80,m:1+t:2,m:1+t:23",
"上证A股": "m:1+t:2,m:1+t:23"
} # 设置爬取哪些市场
for i in fs.keys():
num = getOnePageStock(num, fields, fs[i], page, pz)
print(x) # 输出表格
main()
2)、心得体会
这次实验我爬取了沪深A股和上证A股第一页的数据（一页20条数据，共40条数据）。因为表格的表头是中文，字节数不同，所以在Pycharm中表格没办法对齐，但用IDLE运行可以做到对齐的效果。在这次实验中，我学会了浏览器F12中的json用法以及通过抓包获取数据集URL。对于得到"fxx:xxxxxx"数据后的处理方法，我只想到了两种方法，一种是通过循环来切片得到后半部分的数据，另一种是只能针对当前题目的自定义字符串来截取后半部分的数据，我用的是第二种方法。
作业③
1)、爬取特定代码的股票的相关信息实验
import urllib
import urllib.request
import re
from bs4 import UnicodeDammit, BeautifulSoup
import prettytable as pt
x = pt.PrettyTable() # 制表
x.field_names = ["股票代码号", "名称", "今日开", "今日最高", "今日最低"] # 设置表头
def getHtml(number, fields):
headers = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) ""Chrome/70.0.3538.25 Safari/537.36 Core/1.70.3775.400 QQBrowser/10.6.4209.400"}
url = "http://push2.eastmoney.com/api/qt/stock/get?ut=fa5fd1943c7b386f172d6893dbfba10b&invt=2&fltt=2&fields=" + fields + "&secid=1." + number + "&cb=jQuery1124028850151626570875_1601633354589&_=1601633354607"
req = urllib.request.Request(url, headers=headers)
data = urllib.request.urlopen(req)
data = data.read()
dammit = UnicodeDammit(data, ["utf-8", "gbk"])
data = dammit.unicode_markup
soup = BeautifulSoup(data, 'html.parser')
data = re.findall('{"f.*?}', soup.text)
return data
def getOnePageStock(number, fields):
data = getHtml(number, fields)
datas = data[0].strip("{").strip("}").split('},{') # 去掉头尾的"{"和"}"，再通过"},{"切片
for i in range(len(datas)):
stock = datas[i].replace('"', "").split(",") # 去掉双引号并通过","切片
# print(stock)
x.add_row([stock[3][4:], stock[4][4:], stock[2][4:], stock[0][4:], stock[1][4:]]) # 将数据按行存入表格
def main():
number = "600" + "121" # 设置股票代码
fields = "f44,f45,f46,f57,f58" # 设置爬取信息（f44:今日最高，f45：今日最低，f46：今日开，f57：股票代码号，f58：名称）
try:
getOnePageStock(number, fields)
print(x)
except:
print("目标股票不存在") # 不是所有代码都存在股票信息，所以需要异常处理
main()
2)、心得体会
实验三和实验二差不多，就不赘述了。在这个实验中，"secid="后面的数字是1或者0需要注意一下。还有一点我没有理解，那就是有的股票代码在网站上搜得到，但是程序运行结果却是股票不存在，例如300121，只能猜测是自身的代码还存在一些的问题。
