爬虫基础

正则:

import re

# re库采用raw string类型表示正则表达式，表示为:r'text' 原生字符串是不包含转义字符的
# 邮政编码 r'[1-9]\d{5}'

match = re.search(r'[1-9]\d{5}','BIT 100081')
if match:
	print(match.group(0))

# match方法从开头匹配
match = re.match(r'[1-9]\d{5}','100082 BIT ')
if match:
	print(match.group(0))

# findall 返回list
match = re.findall(r'[1-9]\d{5}','100082 100081 BIT ')
print(match)

# split 函数 按匹配的分割
match = re.split(r'[1-9]\d{5}','A100082B100081C100083',maxsplit=3)
print(match)

#re.finditer

#sub替换
match = re.sub(r'[1-9]\d{5}','zipcode','A100082B100081C100083',count=3)
print(match)

#编译模式
regex = re.compile(r'[1-9]\d{5}')
match = regex.search('A100082B100081C100083')
print(match.group(0))

使用request库，get获取url网页代码

import requests
url = "https://www.amazon.cn/dp/B0785D5L1H"
try:
	kv = {'user-agent':'Mozilla/5.0'}
	r = requests.get(url,headers=kv)
	# 如果返回code不是200则抛出异常
	r.raise_for_status()
	r.encoding = r.apparent_encoding
	print(r.text[1000:2000])
except:
	print("抓取失败")

带get参数的

import requests
url = "https://www.baidu.com/s"
try:
	params = {'wd':'Python'}
	r = requests.get(url,params=params)
	print(r.request.url)
	r.raise_for_status()
	print(len(r.text))
except:
	print("抓取失败")

占位符

import requests
url = "http://www.882667.com/ip_%s.html"
try:
    # 占位符
    r = requests.get(url % '202.204.80.112')
    print(r.request.url)
    r.raise_for_status()
    r.encoding = r.apparent_encoding
    print(r.text[-500:])
except:
    print("抓取失败")

请求图片，并保存图片

import requests
import os
url = "http://image.nationalgeographic.com.cn/2017/0211/20170211061910157.jpg"
root = "D://爬虫//requests//pics//"
path = root + url.split('/')[-1]
try:
    if not os.path.exists(root):
        os.mkdir(root)
    if not os.path.exists(path):
        r = requests.get(url)
        with open(path, 'wb') as f:
            f.write(r.content)
            f.close()
            print("save success")
    else:
        print("file is exists")
except:
    print("爬取失败")

爬虫使用beautifulsoup解析html

import requests
from bs4 import BeautifulSoup
url = "https://python123.io/ws/demo.html"
try:
    r = requests.get(url)
    r.raise_for_status()
    # r.encoding = r.apparent_encoding
    demo =r.text
    soup = BeautifulSoup(demo,'html.parser')
    print(soup.prettify)

except:
    print("抓取失败")

import requests
from bs4 import BeautifulSoup
import bs4

def getHTMLText(url):
    try:
        r = requests.get(url)
        r.raise_for_status()
        # print(r.encoding) ISO-8859-1
        r.encoding = r.apparent_encoding
        # print(r.apparent_encoding) utf-8
        html =r.text
        return html
    except:
        print("抓取失败")

def fillUnivList(ulist,html):
	soup = BeautifulSoup(html,'html.parser')
	for tr in soup.find('tbody').children:
		if isinstance(tr,bs4.element.Tag):
			tds = tr.find_all('td')
			ulist.append([tds[0].string,tds[1].string,tds[2].string])


def printUnivList(ulist,num):
	temple = '{0:^10}\t{1:{3}^10}\t{2:^10}'
	print(temple.format('排名','大学名称','地区',chr(12288)))
	for i in range(num):
		item = ulist[i]
		
		print(temple.format(item[0],item[1],item[2],chr(12288)))


def main():
	uinfo = []
	url = 'http://www.zuihaodaxue.cn/zuihaodaxuepaiming2016.html'
	html = getHTMLText(url)
	fillUnivList(uinfo,html)
	printUnivList(uinfo,20)

if __name__ == '__main__':
	main()

import requests
# import pypinyin
from bs4 import BeautifulSoup
def get(city,cityname_zh):
    url = "https://www.tianqi.com/"+ cityname_zh +"/"
    headers = {
        'user-agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.122 Safari/537.36'

        }
    print(url)
    print(cityname_zh,"市天气预报\n---------------------")
    res = requests.get(url,headers=headers).text
    print(res)
    soup = BeautifulSoup(res,'lxml')
    du = soup.find("p",class_="now")
    shidu_atart = soup.find("dd",class_="shidu")
    shidu = shidu_atart.find_all("b")
    print("温度：",du.b.text,"摄氏度")
    for each in shidu:
        print(each.text)
city = ""
# city_start = input("请输入城市：")
# for each in city_start:
#     city = city + pypinyin.pinyin(each,style=pypinyin.NORMAL)[0][0]
get(city,'tianjin')