爬虫
爬虫：向网站发起请求，获取资源后分析并提取有用数据的程序
流程
1、发起请求：使用http库向目标站点发起请求，即发送一个Request，Request包含：请求头、请求体等
2、获取响应内容：如果服务器能正常响应，则会得到一个Response，Response包含：html，json，图片，视频等
3、解析内容：解析html数据：正则表达式，第三方解析库如Beautifulsoup，pyquery等。解析json数据：json模块。解析二进制数据:以b的方式写入文件
4、保存数据：数据库。文件
工具
请求库：requests,selenium
解析库：正则，beautifulsoup,pyquery
存储库：文件，MySQL,Mongodb,Redis
框架：scrapy
安装：pip3 install requests
基本请求
import requests res = requests.get('https://www.baidu.com') res.encoding = 'utf-8' print(res.text) with open('a.html', 'w') as f: f.write(res.text)
带参数的GET请求->params
import requests res = requests.get('https://www.baidu.com/s', params={'wd':'图片'}, headers={'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Safari/537.36', 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3', 'Accept-Encoding': 'gzip, deflate, br', 'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8', 'Cache-Control': 'no-cache', 'Connection': 'keep-alive', 'Cookie': 'BD_UPN=12314753; PSTM=1572350125; BAIDUID=79D0925D8720B930D1F1E5BFF612720F:FG=1; BIDUPSID=AA6E74403EED680B571512C161DCBEA9; BDUSS=EyeXBkQXJNZ1Q0QXk0dzhoTlh1ODFzUzNwa0lySWJwMFBrOVJHMS1SNn5ILTFkRVFBQUFBJCQAAAAAAAAAAAEAAACxNoeFsM3A6GZlbGzIyAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAL-SxV2~ksVdRE; BDORZ=B490B5EBF6F3CD402E515D22BCDA1598; ispeed_lsm=2; BD_HOME=1; H_PS_PSSID=1449_21086_18560_20698_29567_29220_26350; delPer=0; BD_CK_SAM=1; PSINO=3; H_PS_645EC=2d24IwpbvK2eVobcmeLgWHGcv8LmvTpWTYgrzRwRetwbEpdCPi08ahOlrNs; COOKIE_SESSION=15438_1_7_5_14_10_0_1_3_5_39_3_72210_0_0_0_1574650244_1574491787_1574665633%7C9%233409_3_1574491763%7C2', 'Host': 'www.baidu.com', 'Pragma': 'no-cache', 'Sec-Fetch-Mode': 'navigate', 'Sec-Fetch-Site': 'none', 'Sec-Fetch-User': '?1', 'Upgrade-Insecure-Requests': '1' }) res.encoding = 'gbk' print(res.text) with open('a.html', 'w') as f: f.write(res.text)
带参数的GET请求->headers 华华手机商城
import requests headers = {'Referer': 'http://www.aa7a.cn/user.php?&ref=http%3A%2F%2Fwww.aa7a.cn%2Fuser.php%3Fact%3Dlogout', 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Safari/537.36'} # 登录时发送的请求 res = requests.post('http://www.aa7a.cn/user.php', headers=headers, data={ 'username': '2960113637@qq.com', 'password':'zrh960906*', 'captcha': 'GC3T', 'remember': 1, 'ref': 'http://www.aa7a.cn/', 'act': 'act_login' }) cookie=res.cookies.get_dict() # 登录成功，获取cookie res=requests.get('http://www.aa7a.cn/',headers=headers, cookies=cookie, ) if '2960113637@qq.com' in res.text: print("登录成功") else: print("没有登录")
梨视频
import requests import re res = requests.get('https://www.pearvideo.com/category_loading.jsp?reqType=5&categoryId=1&start=0') # 刷新页面得到的数据 reg_text = '<a href="(.*?)" class="vervideo-lilink actplay">' # html页面 obj = re.findall(reg_text,res.text) # print(obj) for url in obj: url = 'https://www.pearvideo.com/'+ url # 拼接路径 res1 = requests.get(url) obj1 = re.findall('srcUrl="(.*?)"',res1.text) print(obj1[0]) name = obj1[0].rsplit('/',1)[1] res2 = requests.get(obj1[0]) with open (name,'wb') as f: for line in res2.iter_content(): f.write(line)
ssl https=http+ssl
import requests respone=requests.get('https://www.12306.cn', cert=('/path/server.crt', '/path/key')) print(respone.status_code)
ip 代理收费（通过代理访问自己的服务，在服务端取出客户端ip查看一下）
import requests proxies={ 'http':'http://egon:123@localhost:9743',#带用户名密码的代理,@符号前是用户名与密码 'http':'http://localhost:9743', 'https':'https://localhost:9743', 'http':'http://124.205.155.148:9090' } respone=requests.get('https://www.12306.cn', proxies=proxies) print(respone.status_code)
超时设置
import requests respone=requests.get('https://www.baidu.com', timeout=0.0001)
上传文件
import requests files={'file':open('a.jpg','rb')} respone=requests.post('http://httpbin.org/post',files=files) print(respone.status_code)
安装：easy_install lxml
爬取建材 import requests
for i in range(5): response = requests.get('http://shop.jc001.cn/1373528/goods/?p=%s'%(i)) response.encoding = 'gbk' # # print(response.text) from bs4 import BeautifulSoup soup = BeautifulSoup(response.text,'lxml') # # res = soup.prettify() # # print(res)
# copy selector # body > div:nth-child(3) > div > div.col-md-9.col-md-push-3 > div > div.cnt.clearfix.line > ul > li:nth-child(1) > a res = soup.select('body > div:nth-child(3) > div > div.col-md-9.col-md-push-3 > div > div.cnt.clearfix.line > ul > li > a') # print(res) for i,v in enumerate(res): print(v.attrs['href'])
获取手机
import requests, json url = 'http://10.23.255.15/v1/api/provider' # 请求头 headers = {"Cookie":"csrftoken=pAJN4t4EcLs9UH0nCzoevqn7dd2HzYIxLKA873Hm1p6EZd7PPAgukvM9UKM9N7qu; sessionid=7kabh663t34qt4he03ittndf48ikjdni", "User-Agent" : "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.87 Safari/537.36"} data = requests.request('get', url, headers=headers) data = json.loads(data.text) #得到网页的所有数据 print(data) list = [] list2 = [] for i in range(len(data)): for j in range(len(data[i]['device_details'])): # 将所有手机添加到空列表中 list.append(data[i]["device_details"][j]["manufacturer"]) # 去重 list1 = set(list) for i in list1: list2.append(i) # 将手机设备以字典方式存储 dict = {} for i in range(len(list2)): dict[list2[i]] = [] for i in range(len(data)): for j in range(len(data[i]['device_details'])): dict[data[i]["device_details"][j]["manufacturer"]].append('Android://' + data[i]["device_details"][j]["provider_ip"] + ':5039/' +data[i]["device_details"][j]["serialno"]) print(dict) # 将手机存入excel表中 l = 0 import xlwt # 创建一个workbook 设置编码 workbook = xlwt.Workbook(encoding = 'utf-8') # 创建一个worksheet worksheet = workbook.add_sheet('My Worksheet') # 写入excel worksheet.write(0, 0, label='品牌') worksheet.write(0, 1, label='数量') worksheet.write(0, 2, label='设备信息') for i in range(len(list2)): for j in range(len(dict[list2[i]])): worksheet.write(2+j+l, 2, label=dict[list2[i]][j]) worksheet.write(1+l, 0, label=list2[i]) worksheet.write(1+l, 1, label=len(dict[list2[i]])) worksheet.write(1+l, 2, label=','.join(dict[list2[i]])) l = l+len(dict[list2[i]])+1 workbook.save('test.xls')
find: -name="标签名"
-id，class_,="" 把这个标签拿出来
-标签.text 取标签的内容
-标签.get(属性名) 取标签属性的内容
find_all
import requests from bs4 import BeautifulSoup url='https://www.autohome.com.cn/news/1/#liststart' res=requests.get(url) #生成一个bs4对象 soup=BeautifulSoup(res.text,'lxml') div=soup.find(id='auto-channel-lazyload-article') #div 是个对象 # print(type(div)) ul=div.find(name='ul') #只找第一个ul标签 # ul_list=div.find_all(class_="article") #找出下面所有类名为article的标签 # print(len(ul_list)) li_list=ul.find_all(name='li') # print(len(li_list)) for li in li_list: h3=li.find(name='h3') if h3: title=h3.text #把h3标签的text取出来 print(title) a=li.find(name='a') if a: article_url=a.get('href') #取出a标签的href属性 print(article_url) img=li.find(name='img') if img: img_url=img.get('src') print(img_url) p=li.find(name='p') if p: content=p.text print(content)
查找文档 五种过滤器 :字符串，正则，布尔，方法，列表
from bs4 import BeautifulSoup html_doc = """ <html><head><title>The Dormouse's story</title></head> <body> <p class="title" id="bbaa"><b name="xx" age="18">The Dormouse's story</b><b>xxxx</b></p> <p class="xxx" a="xxx">asdfasdf</p> <p class="story">Once upon a time there were three little sisters; and their names were <a href="http://example.com/elsie" class="sister" id="link1">Elsie</a>, <a href="http://example.com/lacie" class="sister" id="link2">Lacie</a> and <a href="http://example.com/tillie" class="sister" id="link3">Tillie</a>; and they lived at the bottom of a well.</p> <p class="story">...</p> """ # soup=BeautifulSoup(html_doc,'lxml') # ress=soup.prettify() #美化一下 # soup=BeautifulSoup(ress,'lxml') # print(ress) #遍历文档树 # print(soup.p.name) # print(soup.p.attrs) # print(soup.p.string) # print(list(soup.p.strings)) # print(soup.p.text) import re print(soup.find_all(name='b')) # 字符串 print(soup.find_all(name=re.compile('^b'))) # 正则 print(soup.find_all(id=re.compile('^b'))) print(soup.find_all(name=['a','b'])) # 列表 print(soup.find_all(name=True)) # 布尔 def has_class_but_no_id(tag): # 方法 return tag.has_attr('class') and not tag.has_attr('id') print(soup.find_all(name=has_class_but_no_id))
css 选择
sibling_soup = BeautifulSoup("<a><b>text1</b><c>text2</c></b></a>",'lxml') print(sibling_soup.b.next_sibling) print(sibling_soup.c.previous_sibling )
自动打开百度
from selenium import webdriver from selenium.webdriver.common.keys import Keys #键盘按键操作 import time bro=webdriver.Chrome() bro.get('https://www.baidu.com') #取到输入框 inp=bro.find_element_by_id('kw') #往框里写字 inp.send_keys("图片") inp.send_keys(Keys.ENTER) #输入回车 time.sleep(3) bro.close()
百度自动登录
from selenium import webdriver import time bro = webdriver.Chrome() bro.get("https://www.baidu.com") bro.implicitly_wait(10) dl_button=bro.find_element_by_link_text("登录") dl_button.click() user_login=bro.find_element_by_id('TANGRAM__PSP_10__footerULoginBtn') user_login.click() time.sleep(1) input_name=bro.find_element_by_name('userName') input_name.send_keys("2960113637@qq.com") input_password=bro.find_element_by_id("TANGRAM__PSP_10__password") input_password.send_keys("zrh960906") submit_button=bro.find_element_by_id('TANGRAM__PSP_10__submit') time.sleep(1) submit_button.click() time.sleep(10) print(bro.get_cookies()) bro.close()
京东购物
from selenium import webdriver from selenium.webdriver.common.keys import Keys import time bro = webdriver.Chrome() bro.get("https://www.jd.com") bro.implicitly_wait(5) def get_goods(bro): print("------------------------------------") goods_li = bro.find_elements_by_class_name('gl-item') for good in goods_li: img_url = good.find_element_by_css_selector('.p-img a img').get_attribute('src') if not img_url: img_url = 'https:' + good.find_element_by_css_selector('.p-img a img').get_attribute('data-lazy-img') url = good.find_element_by_css_selector('.p-img a').get_attribute('href') price = good.find_element_by_css_selector('.p-price i').text name = good.find_element_by_css_selector('.p-name em').text.replace('\n', '') commit = good.find_element_by_css_selector('.p-commit a').text print(''' 商品链接：%s 商品图片：%s 商品名字：%s 商品价格：%s 商品评论数：%s ''' % (url, img_url, name, price, commit)) next_page = bro.find_element_by_partial_link_text("下一页") time.sleep(1) next_page.click() time.sleep(1) get_goods(bro) input_search=bro.find_element_by_id('key') input_search.send_keys("大裤衩") input_search.send_keys(Keys.ENTER) try: get_goods(bro) except Exception as e: print('结束') finally: bro.close()
selenium拿cookie
#获取属性： # tag.get_attribute('src') #获取文本内容 # tag.text #获取标签ID，位置，名称，大小（了解） # print(tag.id) # print(tag.location) # print(tag.tag_name) # print(tag.size) #模拟浏览器前进后退 # browser.back() # time.sleep(10) # browser.forward() #cookies管理 # print(browser.get_cookies()) 获取cookie # browser.add_cookie({'k1':'xxx','k2':'yyy'}) 设置cookie # print(browser.get_cookies()) #运行js # from selenium import webdriver # import time # # bro=webdriver.Chrome() # bro.get("http://www.baidu.com") # bro.execute_script('alert("hello world")') #打印警告 # time.sleep(5) #选项卡管理 # import time # from selenium import webdriver # # browser=webdriver.Chrome() # browser.get('https://www.baidu.com') # browser.execute_script('window.open()') # # print(browser.window_handles) #获取所有的选项卡 # browser.switch_to_window(browser.window_handles[1]) # browser.get('https://www.taobao.com') # time.sleep(3) # browser.switch_to_window(browser.window_handles[0]) # browser.get('https://www.sina.com.cn') # browser.close() #动作链 # from selenium import webdriver # from selenium.webdriver import ActionChains # # from selenium.webdriver.support.wait import WebDriverWait # 等待页面加载某些元素 # import time # # driver = webdriver.Chrome() # driver.get('http://www.runoob.com/try/try.php?filename=jqueryui-api-droppable') # wait=WebDriverWait(driver,3) # # driver.implicitly_wait(3) # 使用隐式等待 # # try: # driver.switch_to.frame('iframeResult') ##切换到iframeResult # sourse=driver.find_element_by_id('draggable') # target=driver.find_element_by_id('droppable') # # # #方式一：基于同一个动作链串行执行 # # actions=ActionChains(driver) #拿到动作链对象 # # actions.drag_and_drop(sourse,target) #把动作放到动作链中，准备串行执行 # # actions.perform() # # #方式二：不同的动作链，每次移动的位移都不同 # # # ActionChains(driver).click_and_hold(sourse).perform() # distance=target.location['x']-sourse.location['x'] # # # track=0 # while track < distance: # ActionChains(driver).move_by_offset(xoffset=2,yoffset=0).perform() # track+=2 # # ActionChains(driver).release().perform() # # time.sleep(10) # # # finally: # driver.close()
获取cookie
# import time # from selenium import webdriver # import json # browser = webdriver.Chrome() # browser.get('https://account.cnblogs.com/signin?returnUrl=https%3A%2F%2Fwww.cnblogs.com%2F') # # time.sleep(50) # cookie=browser.get_cookies() # print(cookie) # with open('cookie.json','w')as f: # json.dump(cookie,f) import requests import json with open('cookie.json','r')as f: di=json.load(f) cookies = {} # 获取cookie中的name和value,转化成requests可以使用的形式 for cookie in di: print(cookie) for key in cookie.keys(): cookies[key] = cookie[key] print(cookies) res=requests.get('https://i-beta.cnblogs.com/api/user', cookies=cookies) print(res.text)
破解验证码
import requests from selenium import webdriver import time import json url = 'https://account.cnblogs.com/signin?returnUrl=https%3A%2F%2Fwww.cnblogs.com%2F' driver = webdriver.Chrome() driver.get(url=url) time.sleep(50) driver.refresh() c = driver.get_cookies() print(c) with open('xxx.txt','w') as f: json.dump(c,f) time.sleep(3) with open('xxx.txt', 'r') as f: di = json.load(f) cookies = {} for cookie in di: cookies[cookie['name']] = cookie['value'] print(cookies) headers = { # 'authority': 'www.jd.com', # 'method': 'GET', # 'path': '/', # 'scheme': 'https', # 'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3', # 'accept-encoding': 'gzip, deflate, br', # 'accept-language': 'zh-CN,zh;q=0.9', # 'cache-control': 'max-age=0', # 'upgrade-insecure-requests': '1', 'authority': 'i-beta.cnblogs.com', 'method': 'GET', 'path': '/', 'scheme': 'https', 'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3', 'accept-encoding': 'gzip, deflate, br', 'accept-language': 'zh-CN,zh;q=0.9', 'cache-control': 'max-age=0', 'if-modified-since': 'Sun, 24 Nov 2019 06:14:53 GMT', # 'if-modified-since': 'Sun, 24 Nov 2019 06:14:53 GMT, 'sec-fetch-mode': 'navigate', 'sec-fetch-site': 'none', 'sec-fetch-user': '?1', 'upgrade-insecure-requests': '1', 'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36' } # 使用该cookie完成请求 response = requests.get(url='https://i-beta.cnblogs.com/api/user', headers=headers, cookies=cookies) print('xxx') response.encoding = response.apparent_encoding print(response.text)
from urllib.parse import unquote_plus #字符转中文 from urllib.parse import urlencode #中文转字符 msg = ''' "client_id=c3cef7c66a1843f8b3a9e6a1e3160e20&grant_type=password×tamp=1574838172749&source=com.zhihu.web&signature=d9ca5ecd24ebcfd42360eabd392d860e837005d8&username=%2B8618953675221&password=lqz12345&captcha=&lang=cn&utm_source=&ref_source=other_https%3A%2F%2Fwww.zhihu.com%2Fsignin%3Fnext%3D%252F" ''' print(unquote_plus(msg))
登录知乎
from requests_html import HTMLSession #请求解析库 pip install requests-html import base64 #base64解密加密库 from PIL import Image #图片处理库 import hmac #加密库 from hashlib import sha1 #加密库 import time from urllib.parse import urlencode #url编码库 import execjs #python调用node.js pip install PyExecJS from http import cookiejar class Spider(): def __init__(self): self.session = HTMLSession() self.session.cookies = cookiejar.LWPCookieJar() # 使cookie可以调用save和load方法 self.login_page_url = 'https://www.zhihu.com/signin?next=%2F' self.login_api = 'https://www.zhihu.com/api/v3/oauth/sign_in' self.captcha_api = 'https://www.zhihu.com/api/v3/oauth/captcha?lang=en' self.headers = { 'user-agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.98 Safari/537.36 LBBROWSER', } self.captcha ='' #存验证码 self.signature = '' #存签名 # 首次请求获取cookie def get_base_cookie(self): self.session.get(url=self.login_page_url, headers=self.headers) # 处理验证码 def deal_captcha(self): r = self.session.get(url=self.captcha_api, headers=self.headers) r = r.json() if r.get('show_captcha'): while True: r = self.session.put(url=self.captcha_api, headers=self.headers) img_base64 = r.json().get('img_base64') with open('captcha.png', 'wb') as f: f.write(base64.b64decode(img_base64)) captcha_img = Image.open('captcha.png') captcha_img.show() self.captcha = input('输入验证码:') r = self.session.post(url=self.captcha_api, data={'input_text': self.captcha}, headers=self.headers) if r.json().get('success'): break def get_signature(self): # 生成加密签名 a = hmac.new(b'd1b964811afb40118a12068ff74a12f4', digestmod=sha1) a.update(b'password') a.update(b'c3cef7c66a1843f8b3a9e6a1e3160e20') a.update(b'com.zhihu.web') a.update(str(int(time.time() * 1000)).encode('utf-8')) self.signature = a.hexdigest() def post_login_data(self): data = { 'client_id': 'c3cef7c66a1843f8b3a9e6a1e3160e20', 'grant_type': 'password', 'timestamp': str(int(time.time() * 1000)), 'source': 'com.zhihu.web', 'signature': self.signature, 'username': '+8618217210664', 'password': 'zrh960906*', 'captcha': self.captcha, 'lang': 'en', 'utm_source': '', 'ref_source': 'other_https://www.zhihu.com/signin?next=%2F', } headers = { 'x-zse-83': '3_2.0', 'content-type': 'application/x-www-form-urlencoded', 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.98 Safari/537.36 LBBROWSER', } data = urlencode(data) with open('ttt.js', 'rt', encoding='utf-8') as f: js = execjs.compile(f.read()) data = js.call('b', data) print(data) r = self.session.post(url=self.login_api, headers=headers, data=data) if r.status_code == 201: self.session.cookies.save('mycookie') print('登录成功') else: print('登录失败') def login(self): self.get_base_cookie() self.deal_captcha() self.get_signature() self.post_login_data() if __name__ == '__main__': zhihu_spider = Spider() zhihu_spider.login()
xpath选择
doc=''' <html> <head> <base href='http://example.com/' /> <title>Example website</title> </head> <body> <div id='images'> <a href='image1.html' a="xxx">Name: My image 1 <br /><img src='image1_thumb.jpg' /></a> <a href='image2.html'>Name: My image 2 <br /><img src='image2_thumb.jpg' /></a> <a href='image3.html'>Name: My image 3 <br /><img src='image3_thumb.jpg' /></a> <a href='image4.html' class='li'>Name: My image 4 <br /><img src='image4_thumb.jpg' /></a> <a href='image5.html' class='li li-item' name='items'>Name: My image 5 <br /><img src='image5_thumb.jpg' /></a> <a href='image6.html' name='items'><span><h5>test</h5></span>Name: My image 6 <br /><img src='image6_thumb.jpg' /></a> </div> </body> </html> ''' from lxml import etree html=etree.HTML(doc) # html=etree.parse('search.html',etree.HTMLParser()) # 1 所有节点 a=html.xpath('//*') #匹配所有标签 # 2 指定节点（结果为列表） # a=html.xpath('//head') # 3 子节点，子孙节点 a=html.xpath('//div/a') a=html.xpath('//body/a') #无数据 a=html.xpath('//body//a') # 4 父节点 # a=html.xpath('//body//a[@href="image1.html"]/..') a=html.xpath('//body//a[1]/..') #从1开始 # 也可以这样 a=html.xpath('//body//a[1]/parent::*') # 5 属性匹配 a=html.xpath('//body//a[@href="image1.html"]') # 6 文本获取 a=html.xpath('//body//a[@href="image1.html"]/text()') a=html.xpath('//body//a/text()') # 7 属性获取 # a=html.xpath('//body//a/@href') # # 注意从1 开始取（不是从0） a=html.xpath('//body//a[2]/@href') # 8 属性多值匹配 # a 标签有多个class类，直接匹配就不可以了，需要用contains # a=html.xpath('//body//a[@class="li"]') a=html.xpath('//body//a[contains(@class,"li")]/text()') # a=html.xpath('//body//a[contains(@class,"li")]/text()') # 9 多属性匹配 a=html.xpath('//body//a[contains(@class,"li") or @name="items"]') a=html.xpath('//body//a[contains(@class,"li") and @name="items"]/text()') a=html.xpath('//body//a[contains(@class,"li")]/text()') # 10 按序选择 a=html.xpath('//a[2]/text()') a=html.xpath('//a[2]/@href') # 取最后一个 a=html.xpath('//a[last()]/@href') # 位置小于3的 a=html.xpath('//a[position()<3]/@href') # 倒数第二个 a=html.xpath('//a[last()-2]/@href') # 11 节点轴选择 # ancestor：祖先节点 # 使用了* 获取所有祖先节点 a=html.xpath('//a/ancestor::*') # # 获取祖先节点中的div a=html.xpath('//a/ancestor::div') # attribute：属性值 a=html.xpath('//a[1]/attribute::*') # child：直接子节点 a=html.xpath('//a[1]/child::*') # descendant：所有子孙节点 a=html.xpath('//a[6]/descendant::*') # following:当前节点之后所有节点 a=html.xpath('//a[1]/following::*') a=html.xpath('//a[1]/following::*[1]/@href') # following-sibling:当前节点之后同级节点 a=html.xpath('//a[1]/following-sibling::*') a=html.xpath('//a[1]/following-sibling::a') a=html.xpath('//a[1]/following-sibling::*[2]/text()') a=html.xpath('//a[1]/following-sibling::*[2]/@href') print(a)
dayehui
