Python 3爬虫、数据清洗与可视化实战 一书中第四章代码

# -*- coding: UTF-8 -*-
import requests
import time
from urllib.parse import quote
import pymongo
#获取出发地
def from_point():
    # client=pymongo.MongoClient('localhost',27017)
    # book_qunar=client["qunar"]
    # sheet_qunar_departures=book_qunar["departures"]
    departures=[]
    url='https://touch.dujia.qunar.com/depCities.qunar'
    headers = {
        "Cookie": "QN1=00005d8007582ec65570f753; QN300=organic; QN99=9507; QunarGlobal=10.86.213.148_72f0b4ea_17785db2378_-4a7a|1612858558381; _i=DFiEZP-9LIfILrQL-klnUJleKPew; QN269=6C15D1106A2711EB9457FA163E9C4675; QN48=tc_31bb3cf43baffa60_17785dc914e_93c7; QN601=61c00804ff4e6cf5c62399d60c8137cd; fid=e4b08ee9-8d88-4b22-a716-722c31ff8a82; QN205=organic; QN277=organic; csrfToken=MAB3oQhEU1LjIVfrXFxFMChDKPvfBZDm; QN163=0; QN234=home_free_t; _vi=dR2_jE6Ci6sJNrjrV8pEY8lhK48eDy5eyqScBAFkqcaeg2C8_UeFQKf1pwguae2mV8RCYoFmA-_fz1b0bxT1nJEfetrt5Sa1gGfSOixM5kijBKwZgwDnT1CRckqF1WbSdvjmFx4wiPuXNuFmLBfpaoWzBU525Qei6dqRCJPhA1h7; QN271=4751e44a-7756-4e80-bbb0-67671a9fefb6; QN267=02030666817faad142f; QN233=freetrip_list; QN243=92",
        "Referer": "https://touch.dujia.qunar.com/p/public/dep",
        "User-Agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 13_2_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.3 Mobile/15E148 Safari/604.1",
    }
    dep=requests.get(url,headers=headers)
    # print(dep.text)
    dep_dict=dep.json()
    for dep_item in dep_dict['data']:
        for city_name in dep_dict['data'][dep_item]:
            print(city_name)
            # results={
            #     'departure':city_name
            # }
            # sheet_qunar_departures.insert_one(results)
            departures.append(city_name)
    return departures
#获取目的地
def to_point():
    client=pymongo.MongoClient('localhost',27017)
    book_qunar=client['qunar']
    sheet_qunar_destination=book_qunar['destinations']

    destinations=[]
    headers = {
        "Cookie": "QN1=00005d8007582ec65570f753; QN300=organic; QN99=9507; QunarGlobal=10.86.213.148_72f0b4ea_17785db2378_-4a7a|1612858558381; _i=DFiEZP-9LIfILrQL-klnUJleKPew; QN269=6C15D1106A2711EB9457FA163E9C4675; QN48=tc_31bb3cf43baffa60_17785dc914e_93c7; QN601=61c00804ff4e6cf5c62399d60c8137cd; fid=e4b08ee9-8d88-4b22-a716-722c31ff8a82; QN205=organic; QN277=organic; csrfToken=MAB3oQhEU1LjIVfrXFxFMChDKPvfBZDm; QN163=0; QN234=home_free_t; _vi=dR2_jE6Ci6sJNrjrV8pEY8lhK48eDy5eyqScBAFkqcaeg2C8_UeFQKf1pwguae2mV8RCYoFmA-_fz1b0bxT1nJEfetrt5Sa1gGfSOixM5kijBKwZgwDnT1CRckqF1WbSdvjmFx4wiPuXNuFmLBfpaoWzBU525Qei6dqRCJPhA1h7; QN271=4751e44a-7756-4e80-bbb0-67671a9fefb6; QN267=02030666817faad142f; QN233=freetrip_list; QN243=92",
        "Referer": "https://touch.dujia.qunar.com/p/public/dep",
        "User-Agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 13_2_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.3 Mobile/15E148 Safari/604.1",
    }
    collection=book_qunar['departures']
    data = collection.find()

    # deps=from_point()
    # url1="https://touch.dujia.qunar.com/cf/api/destination/list.json?parentId=374810364&depName=%E6%9C%AC%E6%BA%AA&configType=1&callback=jsonp_1613781368272_26960"
    for dep in data:
    #     print(item['departure'])
    # for dep in deps:
        print("#########--##########")
        print(dep['departure'])
        url = "https://touch.dujia.qunar.com/golfz/sight/arriveRecommend?dep={}&needBlackList=true&exclude=&extensionImg=255,175".format(quote(dep['departure']))
        to_dep = requests.get(url,headers=headers)
        to_dice=to_dep.json()
        for to_itmes in to_dice['data']:
            time.sleep(2)
            # print(to_itmes['subModules'])
            for items in to_itmes['subModules']:
                # print(items['items'])
                for item in items['items']:
                    print(item['query'])
                    if item['query'] not in destinations:
                        destinations.append(item['query'])
                        results={
                            'destination':item['query']
                        }
                        sheet_qunar_destination.insert_one(results)

    return destinations


def products():
    client=pymongo.MongoClient('localhost',27017)
    db=client['qunar']
    collections_pro=db['products'].find()
    collections_dep=db['departures'].find()
    collections_que=db['destinations'].find()

    # dep='长春'





    # 读取数据
    # for its in collections_pro:  # 遍历查询结果
    #     # print(its['result']['data'])
    #     l = len(its['result']['data']['list']['results'])  # 获取['result']>['data']>['list']>['results']下的子项目数量
    #     for i in range(0, l):
    #         print("No:", i)
    #         print(its['result']['data']['list']['results'][i]['title'])
    #     break

    # que='泰国'
    for dep in collections_dep:
        for que in collections_que:
            url="https://touch.dujia.qunar.com/list?modules=list,bookingInfo,activityDetail&dep={dep}&query={que}&dappDealTrace=true&mobFunction=%E6%89%A9%E5%B1%95%E8%87%AA%E7%94%B1%E8%A1%8C&page=home&date=&it=dujia_hy_destination&needNoResult=true&originalquery={que}&width=480&height=320&quality=90&limit=0,24&includeAD=true&qsact=search&filterTagPlatform=mobile_touch".format(dep=quote(dep['departure']),que=quote(que['destination']))
            htmlstr=requests.get(url)
            arrive_products=htmlstr.json()
            result={
                'date':time.strftime('%Y-%m-%d', time.localtime(time.time())),
                'departure':dep,
                'destination':que,
                'result':arrive_products
            }
            collections_pro.insert_one(result)

if __name__ == '__main__':
    # print(to_point())
    products()