爬虫学习(十四)——xpath项目实践

import os
import time
import urllib.request
import urllib.parse
from lxml import etree


# 构建面向对象的代码方式
class ZhanZhang(object):

# 实例化参数,让参数能够全局调用
def __init__(self,url,headers):
self.headers=headers
self.url=url
self.opener=urllib.request.build_opener(urllib.request.HTTPHandler())

# 构建请求对象
def request(self):
request=urllib.request.Request(self.url,headers=self.headers)
opener = self.opener
response=opener.open(request)
return response
# 解析网页标签,获取图片地址
def paserHtml(self):
response=self.request()
html=response.read().decode("utf8")
htmlInfo=etree.HTML(html)
# xpath解析网页标签
parserXpath =htmlInfo.xpath('//div[@id="container"]/div[@class]/div/a')
# 获取标签的href属性
imgUrl=[iHref.get("href") for iHref in parserXpath]
# 获取图片的名字
imgName=[iName.get("alt") for iName in parserXpath]
num = len(imgUrl)
imgData={"imgUrl":imgUrl,"imgName":imgName,"num":num}
return imgData


# 下载请求图片
def download(self):
imgData=self.paserHtml()
if not os.path.exists("金毛"):
os.mkdir("金毛")
try:
for index in range(imgData["num"]):
print("正在下载第%d张图片"%index)
lastUrl =imgData["imgUrl"][index]
filename=imgData["imgName"][index]+".png"
filepath=os.path.join("金毛",filename)
newresponse_html= urllib.request.urlopen(lastUrl).read().decode("utf8")
newTree=etree.HTML(newresponse_html)
newUrl=newTree.xpath("//div[@class='img_text']/span/a")[0].get("href")
urllib.request.urlretrieve(newUrl,filepath)
time.sleep(1)
except Exception as e:
print("数据请求完成")

# 主函数,调用类方法
def main():
url = "http://sc.chinaz.com/tupian/jinmaoquantupian.html"
headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.96 Safari/537.36'}
zhanzhang_photo=ZhanZhang(url,headers)
zhanzhang_photo.download()

if __name__ == '__main__':
main()
posted @ 2019-02-18 20:14 石桥浪子 阅读(...) 评论(...) 编辑 收藏