python selenium 获取 devtools 请求标头数据

Question

我尝试了很多搜索和尝试，但都没有达到我想要的结果。

我需要开发工具>>网络>>标头>> cookie数据

帮助！！！

1. 请让我知道如何在 Python 中使用 Selenium 从

   devtools >> network >> headers

   检索信息。
   或

2. 如果有比Selenium更好的方法，请随时告诉我（python）

    url = "http://url.com"
    driver.get(url)
    
    time.sleep(0.3)
    logs = driver.get_log("performance")

搜索谷歌，youtube .....

Answer 1

如果您的浏览器和执行脚本的计算机是相同的，那么您可以利用基于 chromium 的 Web 浏览器的 debugUrl 并使用 Chrome DevTools 协议来拦截所需的 url 并捕获标头。

这是在 python 中进行的相同尝试。它发送带有 urlPattern 的

Fetch.enable

消息，让浏览器知道要拦截的请求。从拦截的请求中捕获标头并按预期继续拦截的请求。

from selenium import webdriver
from selenium.webdriver.edge.options import Options
from selenium.webdriver.edge.service import Service as EdgeService
from webdriver_manager.microsoft import EdgeChromiumDriverManager

import time
import requests
import websockets
import asyncio
import json

options = Options()
options.add_argument('--ignore-ssl-errors=yes')
options.add_argument("--remote-debugging-port=9222")
options.add_argument("--remote-allow-origins=*")
options.add_argument('--ignore-certificate-errors')
driver = webdriver.Edge(service=EdgeService(EdgeChromiumDriverManager().install()), options=options)

def get_debugger_url():
    time.sleep(2)
    response = requests.get('http://localhost:9222/json')
    response_data = response.json()
    print(response_data)
    debugger_url = response_data[0]['webSocketDebuggerUrl']
    return debugger_url

async def listen_to_chrome_debugger():

    debugger_url = get_debugger_url()

    async with websockets.connect(uri=debugger_url, origin="*") as websocket:
        print("Connected to Debugger")

        # Enable interception of request based on the urlPattern provided
        await websocket.send(json.dumps({"id": 1, "method": "Fetch.enable", "params": {"patterns" : [{"urlPattern" : "*"}]}}))

        async for message in websocket:
            event = json.loads(message)
            # print("Received event:", event)

            # When request is captured, the requestPaused event is raised
            if event.get("method") == "Fetch.requestPaused":
                # Capture the cookies, headers, requests, etc..
                print(event['params']['request']['headers'])
                request_id = event["params"]["requestId"]

                # Continue the requests as it is or make changes to the intercepted request
                await websocket.send(json.dumps({
                    "id": 2,
                    "method": "Fetch.continueRequest",
                    "params": {"requestId": request_id}
                }))

driver.get("http://www.example.com")
print("Page title:", driver.title)

asyncio.run(listen_to_chrome_debugger())

time.sleep(10)

driver.quit()
print("Driver quit")

资源：https://chromedevtools.github.io/devtools-protocol/tot/Fetch/