使用requests直接访问页面url返回的信息与页面信息不匹配
import requests
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.97 Safari/537.36 Edg/83.0.478.50',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9'
}
response = requests.get('https://dynamic1.scrape.cuiqingcai.com/', headers=headers, verify=False) # 该网站证书是自行签名的,忽略校验
print(response.text) # 没有获取到页面信息,因为网站数据是ajax获取的
分析请求获取ajax
import requests
import logging
import json
from os import makedirs
from os.path import exists
# import urllib3
# urllib3.disable_warnings() # 禁用警告
# import ssl
# ssl._create_default_https_context = ssl._create_unverified_context # 全局忽略
logging.basicConfig(level=logging.INFO,
format='%(asctime)s - %(levelname)s: %(message)s')
INDEX_URL = 'https://dynamic1.scrape.cuiqingcai.com/api/movie/?limit={limit}&offset={offset}'
DETAIL_URL = 'https://dynamic1.scrape.cuiqingcai.com/api/movie/{id}/'
LIMIT = 10
TOTAL_PAGE = 10
RESULTS_DIR = 'results'
exists(RESULTS_DIR) or makedirs(RESULTS_DIR)
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.97 Safari/537.36 Edg/83.0.478.50',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9'
}
def scrape_api(url):
logging.info('scraping %s...', url)
try:
response = requests.get(url, headers=headers, verify=False) # 该网站证书是自行签名的,忽略校验
if response.status_code == 200:
return response.json()
logging.error('get invalid status code %s while scraping %s', response.status_code, url)
except requests.RequestException:
logging.error('error occurred while scraping %s', url, exc_info=True) # 传递移除信息
def scrape_index(page):
url = INDEX_URL.format(limit=LIMIT, offset=LIMIT*(page-1))
return scrape_api(url)
def scrape_detail(id):
url = DETAIL_URL.format(id=id)
return scrape_api(url)
def save_data(data):
name = data.get('name')
data_path = f'{RESULTS_DIR}/{name}.json'
json.dump(data, open(data_path, 'w', encoding='utf-8'), ensure_ascii=False, indent=2)
def main():
for page in range(1, TOTAL_PAGE + 1):
index_data = scrape_index(page)
for item in index_data.get('results'):
id = item.get('id')
detail_data = scrape_detail(id)
# logging.info('detail data %s', detail_data)
save_data(detail_data)
if __name__ == '__main__':
main()