xpath之段子网内容及标题抓取

import requests
from lxml import etree

#指定url
url = "https://ishuo.cn/joke"

headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36'
    }
#发起请求并获取页面数据
page_text=requests.get(url=url,headers=headers).text

#数据解析
tree=etree.HTML(page_text)
#获取所有的li标签（段子内容和标题都被包含在li标签中）
li_list = tree.xpath('//div[@id="list"]/ul/li')
#注意：Element类型的对象可以继续调用xpath函数，对该对象表示的局部内容进行解析
fp = open('./duanzi.txt','w',encoding='utf-8')
for li in li_list:
    content = li.xpath('./div[@class="content"]/text()')[0]
    title = li.xpath('./div[@class="info"]/a/text()')[0]
    #持久化存储
    fp.write(title+":"+content+"\n\n")
print("所有数据写入成功")
fp.close()

注：此代码仅供学习参阅