爬取糗事百科

# -*- coding: utf-8 -*-
import urllib2
from lxml import etree
import json

url = "http://www.qiushibaike.com/8hr/page/1/"
headers = {"User-Agent" : "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.186 Safari/537.36"}
request = urllib2.Request(url,headers = headers)

html = urllib2.urlopen(request).read()

#相应返回的是字符串。解析 为HTML DOM模式
text = etree.HTML(html)

#返回所有段子的节点位置，contains()模糊查询方法，第一个参数是匹配的标签，第二个参数是标签名部分内容
node_list = text.xpath('//div[contains(@id,"qiushi_tag")]')

item = {}
for node in node_list:
    #xpath返回的列表，这个列表就这一个参数，用索引方式取出来，用户名
    username = node.xpath('./div/a/@title')[0]
    #图片链接
    image = node.xpath('.//div[@class = "thumb"]//@src')[0]
    #取出标签下的内容，段子内容
    content = node.xpath('.//div[@calss = "content"]/span')[0].text
    #取出标签里包含的内荣，点赞
    zan = node.xpath('.//i')[0].text
    #评论
    comments = node.xpath('.//i')[1].text

    items = {
        "username" : username,
        "image" : image,
        "content" : content,
        "zan" : zan,
        "comments" : comments
    }
    #a是追加的方式
    with open("qiushi.json","a") as f:
        f.write(json.dumps(items,ensure_ascii=False).encode("utf-8") + '\n')