python Selenium中如何抓取网络请求响应及WebSocket信息
我们在使用Selenium测试Web或Electronjs/Cef框架应用时，有时候操作一个元素需要判断是否发送了请求以及请求的参数是否正确
我们可以通过，开启Chrome的性能日志来然后配合driver.get_log("performance")来查看请求，然后对Network相关的日子进行过滤，
实现如下：
获取Chrome性能日志
import json from pprint import pprint from selenium import webdriver caps = { 'browserName': 'chrome', 'version': '', 'platform': 'ANY', 'goog:loggingPrefs': {'performance': 'ALL'}, # 记录性能日志 'goog:chromeOptions': {'extensions': [], 'args': ['--headless']} # 无界面模式 } driver = webdriver.Chrome(desired_capabilities=caps) driver.get('https://httpbin.org/get') logs = driver.get_log("performance") for item in logs: log = json.loads(item["message"])["message"] pprint(log) if "Network.response" in log["method"] or "Network.request" in log["method"] or "Network.webSocket" in log["method"]: pprint(log)
运行结果如下:
{'method': 'Network.responseReceived',
'params': {'frameId': '2445B94E9E1DB51A1B1F4F3B0A3F03F5',
'loaderId': 'D0DE1754D5C5F1E54DC3B0DB2A09ADD6',
'requestId': 'D0DE1754D5C5F1E54DC3B0DB2A09ADD6',
'response': {'connectionId': 0,
'connectionReused': False,
'encodedDataLength': -1,
'fromDiskCache': False,
'fromPrefetchCache': False,
'fromServiceWorker': False,
'headers': {'Content-Type': 'text/plain;charset=US-ASCII'},
'mimeType': 'text/plain',
'protocol': 'data',
'remoteIPAddress': '',
'remotePort': 0,
'securityState': 'secure',
'status': 200,
'statusText': 'OK',
'url': 'data:,'},
'timestamp': 57524.763168,
'type': 'Document'}}
{'method': 'Network.requestWillBeSent',
'params': {'documentURL': 'https://httpbin.org/get',
'frameId': '2445B94E9E1DB51A1B1F4F3B0A3F03F5',
'hasUserGesture': False,
'initiator': {'type': 'other'},
'loaderId': '8BB61F3D2448E8BC91A4A5AD7E690673',
'request': {'headers': {'Upgrade-Insecure-Requests': '1',
'User-Agent': 'Mozilla/5.0 (Macintosh; '
'Intel Mac OS X 10_15_7) '
'AppleWebKit/537.36 (KHTML, '
'like Gecko) '
'HeadlessChrome/91.0.4472.114 '
'Safari/537.36'},
'initialPriority': 'VeryHigh',
'method': 'GET',
'mixedContentType': 'none',
'referrerPolicy': 'strict-origin-when-cross-origin',
'url': 'https://httpbin.org/get'},
'requestId': '8BB61F3D2448E8BC91A4A5AD7E690673',
'timestamp': 57524.961438,
'type': 'Document',
'wallTime': 1626501610.512192}}
{'method': 'Network.requestWillBeSentExtraInfo',
'params': {'associatedCookies': [],
'headers': {':authority': 'httpbin.org',
':method': 'GET',
':path': '/get',
':scheme': 'https',
'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
'accept-encoding': 'gzip, deflate, br',
'accept-language': 'en-US',
'sec-fetch-dest': 'document',
'sec-fetch-mode': 'navigate',
'sec-fetch-site': 'none',
'sec-fetch-user': '?1',
'upgrade-insecure-requests': '1',
'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X '
'10_15_7) AppleWebKit/537.36 (KHTML, '
'like Gecko) '
'HeadlessChrome/91.0.4472.114 '
'Safari/537.36'},
'requestId': '8BB61F3D2448E8BC91A4A5AD7E690673'}}
{'method': 'Network.responseReceivedExtraInfo',
'params': {'blockedCookies': [],
'headers': {'access-control-allow-credentials': 'true',
'access-control-allow-origin': '*',
'content-length': '754',
'content-type': 'application/json',
'date': 'Sat, 17 Jul 2021 06:00:11 GMT',
'server': 'gunicorn/19.9.0'},
'requestId': '8BB61F3D2448E8BC91A4A5AD7E690673',
'resourceIPAddressSpace': 'Public'}}
{'method': 'Network.responseReceived',
'params': {'frameId': '2445B94E9E1DB51A1B1F4F3B0A3F03F5',
'loaderId': '8BB61F3D2448E8BC91A4A5AD7E690673',
'requestId': '8BB61F3D2448E8BC91A4A5AD7E690673',
'response': {'connectionId': 12,
'connectionReused': False,
'encodedDataLength': 123,
'fromDiskCache': False,
'fromPrefetchCache': False,
'fromServiceWorker': False,
'headers': {'access-control-allow-credentials': 'true',
'access-control-allow-origin': '*',
'content-length': '754',
'content-type': 'application/json',
'date': 'Sat, 17 Jul 2021 06:00:11 GMT',
'server': 'gunicorn/19.9.0'},
'mimeType': 'application/json',
'protocol': 'h2',
'remoteIPAddress': '52.201.75.114',
'remotePort': 443,
'requestHeaders': {':authority': 'httpbin.org',
':method': 'GET',
':path': '/get',
':scheme': 'https',
'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
'accept-encoding': 'gzip, deflate, '
'br',
'accept-language': 'en-US',
'sec-fetch-dest': 'document',
'sec-fetch-mode': 'navigate',
'sec-fetch-site': 'none',
'sec-fetch-user': '?1',
'upgrade-insecure-requests': '1',
'user-agent': 'Mozilla/5.0 '
'(Macintosh; Intel '
'Mac OS X 10_15_7) '
'AppleWebKit/537.36 '
'(KHTML, like Gecko) '
'HeadlessChrome/91.0.4472.114 '
'Safari/537.36'},
'responseTime': 1626501611316.694,
'securityDetails': {'certificateId': 0,
'certificateTransparencyCompliance': 'unknown',
'cipher': 'AES_128_GCM',
'issuer': 'Amazon',
'keyExchange': 'ECDHE_RSA',
'keyExchangeGroup': 'P-256',
'protocol': 'TLS 1.2',
'sanList': ['httpbin.org',
'*.httpbin.org'],
'signedCertificateTimestampList': [],
'subjectName': 'httpbin.org',
'validFrom': 1608508800,
'validTo': 1642636799},
'securityState': 'secure',
'status': 200,
'statusText': '',
'timing': {'connectEnd': 548.386,
'connectStart': 26.524,
'dnsEnd': 26.524,
'dnsStart': 14.11,
'proxyEnd': -1,
'proxyStart': -1,
'pushEnd': 0,
'pushStart': 0,
'receiveHeadersEnd': 803.146,
'requestTime': 57524.962922,
'sendEnd': 548.745,
'sendStart': 548.611,
'sslEnd': 548.36,
'sslStart': 277.934,
'workerFetchStart': -1,
'workerReady': -1,
'workerRespondWithSettled': -1,
'workerStart': -1},
'url': 'https://httpbin.org/get'},
'timestamp': 57525.76746,
'type': 'Document'}}
获取请求及响应信息
由于日志中没有接口后台数据和响应数据，我们可以通过执行cdp名利获取，修改后代码如下
import json from pprint import pprint from selenium import webdriver from selenium.common.exceptions import WebDriverException caps = { 'browserName': 'chrome', 'version': '', 'platform': 'ANY', 'goog:loggingPrefs': {'performance': 'ALL'}, 'goog:chromeOptions': {'extensions': [], 'args': ['--headless']} } driver = webdriver.Chrome(desired_capabilities=caps) driver.get('https://httpbin.org/get') logs = driver.get_log("performance") for item in logs: log = json.loads(item["message"])["message"] # if "Network.response" in log["method"] or "Network.request" in log["method"] or "Network.webSocket" in log["method"]: # pprint(log) if log["method"] == 'Network.responseReceived': url = log['params']['response']['url'] if url == 'data:,': # 过滤掉初始data页面，后续可以根据 log['params']['response']['type']过滤请求类型 continue print('请求', url) request_id = log['params']['requestId'] request_headers = log['params']['response']['requestHeaders'] response_headers = log['params']['response']['headers'] response_time = log['params']['response']['responseTime'] status_code = log['params']['response']['status'] try: request_data = driver.execute_cdp_cmd('Network.getRequestPostData', {'requestId': request_id}) except WebDriverException: # 没有后台数据获取时会有异常 request_data = None response_body = driver.execute_cdp_cmd('Network.getResponseBody', {'requestId': request_id})['body'] print('响应', response_body)
执行后显示如下：
请求 https://httpbin.org/get
响应 {
"args": {},
"headers": {
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
"Accept-Encoding": "gzip, deflate, br",
"Accept-Language": "en-US",
"Cache-Control": "max-age=0",
"Host": "httpbin.org",
"Sec-Fetch-Dest": "document",
"Sec-Fetch-Mode": "navigate",
"Sec-Fetch-Site": "none",
"Sec-Fetch-User": "?1",
"Upgrade-Insecure-Requests": "1",
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/91.0.4472.114 Safari/537.36",
"X-Amzn-Trace-Id": "Root=1-60f2dd9d-6533f9526707f25f7d6c38de"
},
"origin": "123.118.150.190",
"url": "https://httpbin.org/get"
}
Network.webSocketFrameSent和Network.webSocketFrameReceived类似如下：
[{ 'level': 'INFO', 'message': '{"message":{"method":"Network.webSocketFrameSent","params":{"requestId":"66372.184","response":{"mask":true,"opcode":1,"payloadData":"{\\"msgId\\":\\"88E804E973C979F50D414879D6BF59D0432147608438\\",\\"from\\":\\"wrtest015\\",\\"fromName\\":\\"AA海洋\\",\\"to\\":\\"wrtest016\\",\\"cmd\\":\\"11\\",\\"extras\\":{\\"createTime\\":\\"1697101208435\\"},\\"msgType\\":0,\\"chatType\\":2,\\"content\\":\\"哈哈\\"}"},"timestamp":13214530.443031}},"webview":"A71F46A4493C33C8690BD989CEFB4D2C"}', 'timestamp': 1697101208436 }, { 'level': 'INFO', 'message': '{"message":{"method":"Network.webSocketFrameReceived","params":{"requestId":"66372.184","response":{"mask":false,"opcode":1,"payloadData":"{\\"code\\":10001,\\"command\\":12,\\"extras\\":{\\"chatRoomId\\":\\"4e4134f9f90743ddb881b31dd73062a1\\",\\"msgId\\":\\"f9dbfe3ba9f24787a469ae49da21f43d\\"},\\"msg\\":\\"offline 用户不在线\\"}"},"timestamp":13214530.511594}},"webview":"A71F46A4493C33C8690BD989CEFB4D2C"}', 'timestamp': 1697101208505 }, { 'level': 'INFO', 'message': '{"message":{"method":"Network.webSocketFrameSent","params":{"requestId":"66372.184","response":{"mask":true,"opcode":1,"payloadData":"{\\"cmd\\":13,\\"hbbyte\\":\\"-127\\"}"},"timestamp":13214540.982827}},"webview":"A71F46A4493C33C8690BD989CEFB4D2C"}', 'timestamp': 1697101218976 }, { 'level': 'INFO', 'message': '{"message":{"method":"Network.webSocketFrameReceived","params":{"requestId":"66372.184","response":{"mask":false,"opcode":1,"payloadData":"{\\"command\\":13,\\"data\\":{\\"hbbyte\\":-128}}"},"timestamp":13214541.031321}},"webview":"A71F46A4493C33C8690BD989CEFB4D2C"}', 'timestamp': 1697101219025 }]
————————————————
转：https://blog.csdn.net/MXB_1220/article/details/131775148
浙公网安备 33010602011771号