python 爬虫启航

1. 使用excel（简单使用）

数据- 自网站-导入

2.you-get

 

python爬虫入门

1.环境配置

python，request，lxml

 

2.原理

爬虫的框架如下：

1.挑选种子URL；
2.将这些URL放入待抓取的URL队列；
3.取出待抓取的URL，下载并存储进已下载网页库中。此外，将这些URL放入待抓取URL队列，进入下一循环；
4.分析已抓取队列中的URL，并且将URL放入待抓取URL队列，从而进入下一循环。

爬虫的基本流程：

简单的说，我们向服务器发送请求后，会得到返回的页面，通过解析页面之后，我们可以抽取我们想要的那部分信息，并存储在指定的文档或数据库中。这样，我们想要的信息就被我们“爬”下来啦~

Requests+Xpath爬去豆瓣电影

import requests
from lxml import etree
from bs4 import BeautifulSoup
def movie_spider(url):
    data = requests.get(url).text
    s = etree.HTML(data)

    # 根据页面的xpath解析数据
    film = s.xpath('/html/body/div[3]/div[1]/h1/span[1]/text()')
    director = s.xpath('/html/body/div[3]/div[1]/div[3]/div[1]/div[1]/div[1]/div[1]/div[2]/span[1]/span[2]/a/text()')

    # 浏览器检查的Xpath和实际不对应？为什么？
    actor1= s.xpath('//*[@id="info"]/span[3]/span[2]/a[1]/text()')
    time = s.xpath('/html/body/div[3]/div[1]/div[3]/div[1]/div[1]/div[1]/div[1]/div[2]/span[13]/text()')
    print(film,director,actor1,time)


静态网页抓取

def top250_movie_spider():
    """
    1.定制请求头，network>user-agent
    :return:
    """
    movie_list = []
    headers = {
        'user-agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:63.0) Gecko/20100101 Firefox/63.0',
        'Host':'movie.douban.com'
    }
    for i in range(0,10):
        link = 'https://movie.douban.com/top250?start='+str(i*25)+'&filter='
        r = requests.get(link,headers=headers,timeout=10)
        # 使用BeautifulSoup解析网页
        soup = BeautifulSoup(r.text,'lxml')
        div_list = soup.find_all('div',class_='hd')
        for each in div_list:
            movie = each.a.span.text.strip()
            movie_list.append(movie)
    return movie_list
if __name__ == "__main__":
    url = "https://movie.douban.com/subject/1292052/"
    # movie_spider(url)
    print(top250_movie_spider())

动态网页抓取

解析真实地址抓取：

network（刷新网页抓包）>XHR>确定真实地址

json库解析数据

import requests
import json
link ="https://api-zero.livere.com/v1/comments/list?callback=jQuery1124047657659644175476_1543235317407&limit=10&repSeq=4272904&requestPath=%2Fv1%2Fcomments%2Flist&consumerSeq=1020&livereSeq=28583&smartloginSeq=5154&_=1543235317409"
def single_page_comment(link):
    # 定制请求头
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.89 Safari/537.36'}
    r = requests.get(link, headers=headers)

    json_string = r.text
    json_string = json_string[json_string.find('{'):-2]

    json_data = json.loads(json_string)

    common_list = json_data['results']['parents']
    for eachone in common_list:
        message = eachone['content']
        print(message)

# 对比不同的地址，找出改变的量
for page in range(1,4):
    link1 = "https://api-zero.livere.com/v1/comments/list?callback=jQuery112403473268296510956_1531502963311&limit=10&offset="
    link2 = "&repSeq=4272904&requestPath=%2Fv1%2Fcomments%2Flist&consumerSeq=1020&livereSeq=28583&smartloginSeq=5154&_=1531502963316"
    page_str = str(page)
    link = link1 + page_str + link2
    print (link)
    single_page_comment(link)

使用Selenium模拟浏览器抓取

from selenium  import webdriver
from selenium.webdriver.firefox.firefox_binary import FirefoxBinary
import time
# # 需要geckodriver,并放入环境变量中 --错误
# driver = webdriver.Firefox()
# driver.get("https://www.baidu.com")

#动态爬取网页
caps = webdriver.DesiredCapabilities().FIREFOX
caps["marionette"] = True
# 计算机中firefox的地址
binary = FirefoxBinary(r"C:\Program Files (x86)\Mozilla Firefox\firefox.exe")
driver = webdriver.Firefox(firefox_binary=binary,capabilities=caps)
for i in range(0,20):
    driver.get("https://zh.airbnb.com/s/Shenzhen--China/homes?refinement_paths%5B%5D=%2Fhomes&click_referer=t%3ASEE_ALL%7Csid%3Ad66de168-dbd5-42e7-a122-bd4a001f781e%7Cst%3AMAGAZINE_HOMES&title_type=MAGAZINE_HOMES&query=Shenzhen%2C%20China&allow_override%5B%5D=&s_tag=ckaduMCI&section_offset=4&items_offset="+str(i*18))
    rent_list = driver.find_elements_by_css_selector('div._gig1e7')
    for eachhouse in rent_list:
        comment = eachhouse.find_element_by_css_selector('span._1cy09umr')
        comment = comment.text

        price_detail = eachhouse.find_elements_by_css_selector("span._1sfeueqe")
        price_list = [i.text for i in price_detail]
        price = price_list[1]

        name = eachhouse.find_element_by_css_selector("div._190019zr")
        name = name.text

        details = eachhouse.find_element_by_css_selector("div._1etkxf1")
        details = details.text

        print(comment,price,name,details)