网络爬虫实例代码优化 面试重点

我在“python3开发桌面软件之程序主体”一文中已经分享了919网监助手1.0程序的主体源代码（即events.py中的代码），后用迭代器、生成器表达式等对代码进行优化，使程序运行更快、更稳定。优化后的代码如下：

#coding:utf-8
import wx
import configparser
import requests
import re
import os
import string
from bs4 import BeautifulSoup
from openpyxl import load_workbook

class Getlike():
def __init__(self, filePath,row1,erow1,bcol1,obj1):
self.filePath=filePath
self.row1=row1
self.erow1=erow1
self.bcol1=bcol1
self.obj1=obj1
def main1(self):
list1 = self.rexcel()
hr1 = []
st1 = []
for site in list1:
if site == None:
break

if site[0:4] == 'http':
demo = self.get_html0(site)

if demo == '':
hr1.append(site)
st1.append(u'获取主页数据失败')
continue

soup = BeautifulSoup(demo, "html.parser")
self.obj1.SetLabel(site)
if site[-1] == '/':
site = site[0:-1]
i = 0
for a in soup.find_all('a'):
i += 1
if a.has_attr('href'):
if (a['href'][0:4] != 'http') and ('@' not in a['href']) and (
'javascript:' not in a['href']) and ('javasrcipt:' not in a['href']) and (
'tel:' not in a['href']): # 非站外链接且非正常链接
urlg = a['href']
if len(urlg) > 0:
if urlg[0] != '/':
urlg = '/' + urlg
urlhb = site + urlg
if urlhb in hr1: # 过虑重复链接
continue
hr1.append(urlhb)
st1.append(a.string)
if i == 0:
hr1.append(site)
st1.append(u'该网页无链接')

hr2=iter(hr1)# 创建迭代器对象
st2=iter(st1)# 创建迭代器对象
self.wexcel0(hr2, st2)
def get_html0(self,url):
headers = {
'User-Agent': 'Mozilla/5.0(Windows NT 10.0; WOW64; Trident/7.0; rv:11.0)\
like Gecko'
} # 模拟浏览器访问
try:
r = requests.get(url, timeout=30, headers=headers) # 请求访问网站
r.raise_for_status()
r.encoding = r.apparent_encoding

if r.status_code == 200:
return r.text
else:
return ''
except:
return ''

def rexcel(self): # 读excel文件
list1 = []
if not os.path.exists(self.filePath):
wx.MessageBox(u'没找到excel文件', u'提示')
return list1
wb = load_workbook(self.filePath,read_only=True)
#wb = load_workbook(self.filePath)
ws = wb.active
j =self.row1
if self.erow1<j :
erow2=ws.max_row
else:
erow2 =self.erow1
list1=(ws.cell(row=r, column=self.bcol1).value for r in range(j,erow2)) #生成器表达式
return list1

def wexcel0(self, hr1, st1): # 写excel文件，站内链接
if not os.path.exists(self.filePath):
return 0
wb = load_workbook(self.filePath)
ws = wb.active
nrows = ws.max_row+2
if ws.column_dimensions['B'].width<10:
upper_string = string.ascii_uppercase[:5]
for co in upper_string:
ws.column_dimensions[co].width = 50 # 设置列宽

for r in range(1, nrows): #避免覆盖原数据
if ws.cell(row=r, column=self.bcol1+1).value is None:
break

for (x,y) in zip(hr1,st1):
ws.cell(row=r, column=self.bcol1 + 1).value = x
ws.cell(row=r, column=self.bcol1 + 2).value =y
r = r + 1
if r> 1048575:
break
try:
wb.save(self.filePath)
wb.close()
except:
wx.MessageBox(u'权限被拒绝，请关闭打开的excel文件', u'提示')
return 0
return 1

class Mgzfind(Getlike):
def __init__(self, filePath, row1, erow1, bcol1, obj1,mgz):
Getlike.__init__(self, filePath, row1, erow1, bcol1, obj1)
self.mgz=mgz

def main2(self):
list1= Getlike.rexcel(self)
regex1 = self.mgz # 敏感词之间用|分隔
pattern = \
r'([\u4e00-\u9fa5]{1,}(' + regex1 + '))'
re_compile = re.compile(pattern)
list11=[]
list2 = []
for m in list1:
list11.append(m)
content = ''
#s1 = list1[m]
if m == None:
break

if m[0:4] == 'http':
webcontent = self.get_html(m)
else:
continue

if webcontent == '':
list2.append(u'获取页面数据失败')
continue
else:
for n in re_compile.finditer(webcontent):
content = content + n.group() + ';'
if content=='':
content ='nothing'
list2.append(content)
self.obj1.SetLabel(m)
list1s=iter(list11) # 创建迭代器对象
list2s = iter(list2) # 创建迭代器对象
self.wexcel(list1s,list2s)

def get_html(self,url):
headers = {
'User-Agent': 'Mozilla/5.0(Windows NT 10.0; WOW64; Trident/7.0; rv:11.0)\
like Gecko'
} # 模拟浏览器访问
requests.adapters.DEFAULT_RETRIES = 3
# s = requests.session()
# s.keep_alive = False
try:
r = requests.get(url, timeout=10, headers=headers) # 请求访问网站
r.raise_for_status()
r.encoding = r.apparent_encoding

if r.status_code == 200:
return r.text
else:
return ''
except:
return ''

def wexcel(self,list1, list2): # 写excel文件,敏感词
if not os.path.exists(self.filePath):
return 0
wb = load_workbook(self.filePath)
ws = wb.active
col = 2 + self.bcol1
nrows = ws.max_row+2

for r in range(1, nrows): #避免覆盖原数据
if ws.cell(row=r, column=col+1).value is None:
break

for (x,y) in zip(list1,list2): #在空白处追加数据
ws.cell(row=r, column=col).hyperlink = x
ws.cell(row=r, column=col + 1).value =y
r = r + 1
if r > 1048575:
break
try:
wb.save(self.filePath)
wb.close()
except:
wx.MessageBox(u'权限被拒绝，请关闭打开的excel文件', u'提示')
return 0
return 1

#主窗体事件
class MyFrame1_events():
def __init__(self, frame):
self.frame = frame

#设置图标
self.frame.SetIcon(wx.Icon(u't1.ico', wx.BITMAP_TYPE_ICO))
self.readcon()

#绑定事件
self.frame.Bind(wx.EVT_CLOSE, self.MyFrame1_OnClose)
self.frame.m_radioBtn15.Bind(wx.EVT_RADIOBUTTON, self.radio5)
self.frame.m_radioBtn6.Bind(wx.EVT_RADIOBUTTON, self.radio6)
self.frame.m_button14.Bind(wx.EVT_BUTTON, self.openf)
self.frame.m_button12.Bind(wx.EVT_BUTTON, self.mymain)

# 自定义函数
def readcon(self):
cf = configparser.ConfigParser()
cf.read("config.ini",encoding="utf-8")
secs = cf.sections()
filePath = cf.get("excel", "path")
if self.frame.m_radioBtn15.GetValue():
brow1 = cf.get("excel", "brow1")
erow1 = cf.get("excel", "erow1")
bcol1 = cf.get("excel", "bcol1")
else:
brow1 = cf.get("excel", "brow2")
erow1 = cf.get("excel", "erow2")
bcol1 = cf.get("excel", "bcol2")

word = cf.get("keyword", "word")
word = word.replace("，", "|")
self.frame.m_textCtrl25.Clear()
#filePath=filePath.decode('gbk', 'ignore').encode('utf-8')
self.frame.m_textCtrl25.WriteText(filePath)
self.frame.m_spinCtrl1.SetValue(brow1)
self.frame.m_spinCtrl2.SetValue(erow1)
self.frame.m_spinCtrl3.SetValue(bcol1)
#word1=word.decode('gbk', 'ignore').encode('utf-8')
self.frame.m_textCtrl32.Clear()
self.frame.m_textCtrl32.WriteText(word)
return
def savecon(self):
filePath = self.frame.m_textCtrl25.GetValue()
#brow1 = self.frame.m_spinCtrl1.GetValue()
erow1 = self.frame.m_spinCtrl2.GetValue()
bcol1 = self.frame.m_spinCtrl3.GetValue()
mgz = self.frame.m_textCtrl32.GetValue()
mgz = mgz.replace("，", "|")
cf = configparser.ConfigParser()
cf.read("config.ini", encoding="utf-8")
secs = cf.sections()
cf.set("keyword", "word", mgz)
cf.set("excel", "path", filePath)
if self.frame.m_radioBtn15.GetValue():
cf.set("excel", "brow1",str(erow1))
cf.set("excel", "erow1", str(erow1+10))
self.frame.m_spinCtrl2.SetValue(erow1 + 10)
cf.set("excel", "bcol1", str(bcol1))
else:
cf.set("excel", "brow2", str(erow1))
cf.set("excel", "erow2", str(erow1+30))
self.frame.m_spinCtrl2.SetValue(erow1 + 30)
cf.set("excel", "bcol2", str(bcol1))
cf.write(open('config.ini', "r+",encoding='UTF-8'))
self.frame.m_spinCtrl1.SetValue(erow1)
return

#关闭窗口事件
def MyFrame1_OnClose(self, event):
"""窗体关闭"""
if self.frame.m_button12.GetLabel()==u'确定':
frame = event.GetEventObject()
frame.Destroy()
else:
wx.MessageBox(u'程序进在运行，不能关闭窗口', u'提示')
frame = event.GetEventObject()
frame.Destroy()
event.Skip()

# 浏览excel文件事件
def openf(self, event):
openFileDialog = wx.FileDialog(self.frame, "请选择要打开的Excel文件", "", "",
"Excel格式 (*.xlsx)|*.xlsx",
wx.FD_OPEN | wx.FD_FILE_MUST_EXIST)

if openFileDialog.ShowModal() == wx.ID_OK:
filePath = openFileDialog.GetPath()
self.frame.m_textCtrl25.Clear()
self.frame.m_textCtrl25.WriteText(filePath)
openFileDialog.Destroy()

event.Skip()

def radio5(self, event):
self.readcon()
event.Skip()
def radio6(self, event):
self.readcon()
event.Skip()

def mymain(self, event):
#self.frame.m_button12.Enable(False)
self.frame.m_button12.SetLabel('正在运行...')
filePath = self.frame.m_textCtrl25.GetValue()
row1 = self.frame.m_spinCtrl1.GetValue()
erow1=self.frame.m_spinCtrl2.GetValue()
bcol1=self.frame.m_spinCtrl3.GetValue()
obj1=self.frame.m_staticText11
obj1.SetLabel('正在读取网址...')
mgz=self.frame.m_textCtrl32.GetValue()
mgz =mgz.replace("，","|")
if self.frame.m_radioBtn15.GetValue():
emp1 =Getlike(filePath,row1,erow1,bcol1,obj1)
emp1.main1()
else:
emp2 = Mgzfind(filePath, row1, erow1, bcol1, obj1,mgz)
emp2.main2()
self.savecon()
obj1.SetLabel('完成')
#self.frame.m_button12.Enable(True)
wx.MessageBox(u'完成', u'提示')
self.frame.m_button12.SetLabel('确定')
event.Skip()
注意：很多人学Python过程中会遇到各种烦恼问题解决不了。为此小编建了个Python全栈免费答疑交流.裙 ：留而思思思零期思无（数字的谐音）转换下可以找到了，不懂的问题有老司机解决里面还有最新Python教程项目可拿,，一起相互监督共同进步！
本文的文字及图片来源于网络加上自己的想法,仅供学习、交流使用,不具有任何商业用途,版权归原作者所有,如有问题请及时联系我们以作处理。