Python爬虫Spider 二

来自北京图灵学院刘英

ajax

运行有问题：urllib.error.HTTPError: HTTP Error 418

from urllib import request
import json

url='https://movie.douban.com/j/chart/top_list?type=11&interval_id=100%3A90&action=&start=40&limit=20'
rsp=request.urlopen(url)
data=rsp.read().decode()
data=json.loads(data)
print(data)

 

---

requests库：

Requests - 献给人类

HTTP for Humans，更简洁更友好

继承了urllib的所有特征

底层使用的是urllib3

开源地址：https://github.com/requests/requests

中文文档：http://docs.python-requests.org/zh_CN/latest/index.html

import requests

url='http://sogo.com'

# 方式一：
rsp=requests.get(url)
print(rsp.text)

#方式二：
rsp=requests.request('get',url)
print(rsp.text)

 requests带参数的get

import requests

url='http://sogo.com/web?'
kw={'query':'天龙八部'}
headers={'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:71.0) Gecko/20100101 Firefox/71.0'}
rsp=requests.get(url,params=kw,headers=headers)
print(rsp.text)
print(rsp.content)
print(rsp.url)
print(rsp.encoding)
print(rsp.status_code)

 requests的Post请求：

import requests

url = 'https://fanyi.baidu.com/sug'
data = {'kw': 'girl'}
headers = {
            'Content-Length': str(len(data)),
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:71.0) Gecko/20100101 Firefox/71.0'
           }
rsp = requests.post(url, data=data, headers=headers)
print(rsp.text)
print(rsp.json())

---

request  Proxy

proxies={'http':'address of proxy','https':'address of proxy'}

rsp=requests.request('get','http:xxx',proxies=proxies)

 

proxy={'http':'China:123456@192.168.1.2:1234}

rsp=requests.get('http://baidu.com',proxies=proxy)

 

auth={'username','password'}

rsp=requests.get('http://sogo.com',auth=auth)

---

 https请求验证SSL证书，参数verify负责表示是否需要验证SSL证书，默认是True，如果不需要验证SSL证书，则设置成False表示关闭：

rsp=requests.get('https://sogo.com',verify=False)

---

正则：

match、pattern、group等用法：

# 正则match使用案例
import re

s = r'([a-z]+) ([a-z])+'
pattern = re.compile(s, re.I)  # re.I表示忽略大小写
m = pattern.match('Hello world wide web')

print(type(m))  # <class '_sre.SRE_Match'>
print(m)  # <_sre.SRE_Match object; span=(0, 11), match='Hello world'>

print(m.group())  # 即group(0)表示匹配成功的整个子串。结果：Hello world
print(m.span())  # 即 span(0) , (0, 11) 返回匹配成功整个子串的跨度

print(m.group(1))  # Hello
print(m.span(1))  # (0, 5)

print(m.group(2))  # d 这个没搞懂？！
print(m.span(2))  # (10, 11)

print(m.groups())  # ('Hello', 'd')

# print(m.group(3)) # IndexError: no such group
# print(m.span(3))

print('##############以下同上#############')

26　　 """Try to apply the pattern at the start of the string, returning
　　　　　　a match object, or None if no match was found."""
result = re.match(r'([a-z]+) ([a-z])+', 'Hello world wide web', re.I) #从开头开始匹配，好比加了^

print(type(result))  # <class '_sre.SRE_Match'>
print(result)  # <_sre.SRE_Match object; span=(0, 11), match='Hello world'>
print(result.group())  # Hello world
print(result.span())  # (0, 11)
print(result.group(1))  # Hello
print(result.span(1))  # (0, 5)
print(result.group(2))  # d
print(result.span(2))  # (10, 11)
print(result.groups())  # ('Hello', 'd')
# print(result.group(3)) #IndexError: no such group
# print(result.span(3))

 search使用：

# 正则search使用案例
import re

s = r'\d+'
pattern = re.compile(s)
m = pattern.search('one12two34three56')
print(m)  # <_sre.SRE_Match object; span=(3, 5), match='12'>
print(m.group())  # 12
print(m.groups())  # ()
print(m.group(0))  # 12
# print(m.group(1))# IndexError: no such group
print(m.span())  # (3, 5)

m = pattern.search('one12two34three56', 10, 40)
print(m.group())  # 56

print('############################')

result = re.search(r'\d+', 'one12two34three56')
print(result)  # <_sre.SRE_Match object; span=(3, 5), match='12'>
print(result.group())  # 12
print(result.groups())  # ()
print(result.group(0))  # 12
print(result.span())  # (3, 5)

 findall

# findall使用案例
import re

pattern = re.compile(r'\d+')
s = pattern.findall('i am 18 years old and 170 height')
print(s)  # ['18', '170']

# 以下同上

print(re.findall(r'\d+', 'i am 18 years old and 170 height'))  # ['18', '170']

finditer

# finditer案例
import re

pattern = re.compile(r'\d+')
s = pattern.finditer('i am 19 years old and 170 heigth')
print(s)  # <callable_iterator object at 0x0000021B4904FA90>
for i in s:
    print(i.group())
    '''19
170'''

print('##########以下同上##########')

s = re.finditer(r'\d+', 'i am 19 years old and 170 heigth')
print(s)  # <callable_iterator object at 0x0000021B48FC6C18>
for i in s:
    print(i.group())
    '''19
170'''

 匹配中文：

# 匹配中文unicode案例
import re

hello = '世界，你好！'
pattern = re.compile(r'[\u4e00-\u9fa5]+')
m = pattern.findall(hello)
print(m)  # ['世界', '你好']

print(re.findall(r'[\u4e00-\u9fa5]+', '世界，你好！'))  # ['世界', '你好']

 贪婪与非贪婪模式

• 贪婪模式：在整个表达式匹配成功的前提下，尽可能多的匹配
• 非贪婪模式：在整个表达式匹配成功的前提下，尽可以少的匹配
• python里面数量词默认是贪婪模式

例：查找文本：abbbc

　　正则表达式：ab*

　　贪婪：结果abbb

　　非贪婪：结果a

# 贪婪非贪婪默认值案例
import re

result = re.match('ab*', 'abbb')
print(result.group())  # abbb

 

xpath定位：/bookstore/book[1]，/bookstore/book[last()]，/bookstore/book[last()-1]

---

beautifulSoup4

http://beautifulsoup.readthedocs.io/zh_CN/v4.4.0/

运行有问题：bs4.FeatureNotFound: Couldn't find a tree builder with the features you requested: lxml. Do you need to install a parser library?

from urllib import request
from bs4 import BeautifulSoup

url='http://www.baidu.com'
rsp=request.urlopen(url)
content=rsp.read()
soup=BeautifulSoup(content,'lxml') #'lxml'改成'html.parser'就可以

#bs自动转码
content=soup.prettify()
print(content)

 四大对象：

• Tag

1. 对应HTML中的标签
2. 可以通过soup.tag_name
3. tag两个重要属性：name、attrs

• NavigableString :对应内容值
• BeautifulSoup

1. 表示的是一个文档的内容，大部分可以把他当做tag对象
2. 一般我们可用soup表示

• Comment

1. 特殊类型的NavigableString对象
2. 对其输出，则内容不包括注释符号

from urllib import request
from bs4 import BeautifulSoup

url='http://www.baidu.com'
rsp=request.urlopen(url)
content=rsp.read()
soup=BeautifulSoup(content,'html.parser')

#bs自动转码
content=soup.prettify()
print(content)
print('-'*50)
print(soup.head)
print('-'*50)
print(soup.meta)
print('-'*50)
print(soup.link)
print(soup.link.name)
print(soup.link.attrs)
print(soup.link.attrs['type'])
soup.link.attrs['type']='hello world'
print(soup.link)
print('-'*50)
print(soup.title)
print(soup.title.name)
print(soup.title.attrs)
print(soup.title.string)
print('-'*50)
print(soup.name)
print(soup.attrs)

 find_all：

from urllib import request
from bs4 import BeautifulSoup

url='http://www.baidu.com'
rsp=request.urlopen(url)
content=rsp.read()
soup=BeautifulSoup(content,'html.parser')

print(soup.name)
print('='*50)
for node in soup.head.contents:
    if node.name=='meta':
        print(node)
    if node.name=='title':
        print(node) #没有打印内容，好奇怪！
print('='*50)
tags=soup.find_all(name='meta')
print(tags)
print('='*50)
import re
tags=soup.find_all(re.compile('^me'))
for tag in tags:
    print(tag)
print('='*50)
tags=soup.find_all(re.compile('^me'),content='always')
for tag in tags:
    print(tag)

 CSS选择器：

　　- 使用soup.select，返回一个列表

　　- 通过标签名称：soup.select('title')

　　- 通过类名：soup.select('.content')

　　- id查找：soup.select('#name_id')

　　- 组合查找：soup.select('title .content #name_id')

　　- 属性查找：soup.select('img[class='photo'])

　　- 获取tag内容：tag.get_text

from urllib import request
from bs4 import BeautifulSoup

url='http://www.baidu.com'
rsp=request.urlopen(url)
content=rsp.read()
soup=BeautifulSoup(content,'html.parser')

print(soup.prettify())
print('*'*50)
titles=soup.select('title')
print(titles[0])

print('*'*20)
metas=soup.select('meta[content="always"]')
print(metas[0])

 

---

 

DHTML

Selenium: pip install selenium==2.48.0

官网：http://selenium-python.readthedocs.io/index.html

无界面浏览器：PhantomJS（幽灵）

from selenium import webdriver

driver=webdriver.Firefox()
driver.get('http://www.baidu.com')
print(driver.title)

 from selenium.webdriver.common.keys import Keys

driver.find_element_by_id('kw').send_keys(Key.CONTROL,'a')

driver.find_element_by_id('kw').send_keys(Key.CONTROL,'x')

driver.find_element_by_id('kw').send_keys(u'黄山')

driver.save_screenshot('huangshang.png')

driver.find_element_by_id('su').send_keys(Keys.RETURN)

---

 

获取图片验证码：运行有问题：pytesseract.pytesseract.TesseractNotFoundError: tesseract is not installed or it's not in your path

import pytesseract
from PIL import Image

image=Image.open('a.png')
text=pytesseract.image_to_string(image)
print(text)

 

……