selenium捕获ajax请求内容

 

import json
import os, sys
import platform
import time

from seleniumwire import webdriver
from selenium.webdriver.firefox.firefox_binary import FirefoxBinary

system_ver = platform.system().lower()


ff = None
with open('cookies.json', 'r+', encoding='UTF-8') as f:
    cookies = list(json.loads(f.read()))
    url = "https://www.douyin.com/search/%E5%B0%8F%E7%8C%AB?publish_time=0&sort_type=0&source=tab_search&type=video"

    if system_ver == 'windows':
        ff = FirefoxBinary("C:/Program Files/Mozilla Firefox/firefox.exe")
    # if system_ver == 'linux':
    #     ff = None
    gecko = os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])), "geckodriver.exe")
    opts = webdriver.FirefoxOptions()
    opts.add_argument("--headless")
    driver = webdriver.Firefox(firefox_binary=ff, executable_path=gecko, options=opts)
    driver.get(url)
    time.sleep(2)
    for val in cookies:
        driver.add_cookie(val)
    driver.get(url)
    time.sleep(5)
    data_list = []
    for request in driver.requests:
        if request.response:
            data = str(str(request.body).encode(),encoding='utf-8')
            print(data)
            if request.url.find('https://www.douyin.com/aweme/v1/web/search/item/') >= 0:
                data_list.append({
                    'url':request.url,
                    'data':data
                })
    with open('./results/data.json', 'w+', encoding='UTF-8') as f:
        f.write(json.dumps(data_list,indent=4))
        print('文件写入成功')
    driver.quit()