利用selenium获取接口数据

起因：页面展示的内容被隐藏了部分，无法获取完整内容

处理方案：
1.利用request模拟接口获取返参，模拟了请求头，但操作时一直无法获得数据，报错：org.apache.catalina.connector.ClientAbortException。未深究，大概率是服务器安全问题
selenium获取请求头可参考：https://blog.csdn.net/qq_31042199/article/details/119278315
但是跟selenium库有冲突好像，不知道是否需要把selenium卸载了再安装seleniumwire，待后续有时间验证一下。

2.通过webdriver提供的API查询，使用的函数是Network.getResponseBody
代码：

import json
from selenium import webdriver
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
import time
 
caps ={'browserName':'chrome','loggingPrefs':{'browser':'ALL','driver':'ALL','performance':'ALL',},'goog:chromeOptions':{'perfLoggingPrefs':{'enableNetwork':True,},'w3c':False,},}
driver = webdriver.Chrome(desired_capabilities=caps)
 
driver.get('https://partner.oceanengine.com/union/media/login/')# 必须等待一定的时间，不然会报错提示获取不到日志信息，因为絮叨等所有请求结束才能获取日志信息
time.sleep(3)
 
request_log = driver.get_log('performance')print(request_log)for i inrange(len(request_log)):
    message = json.loads(request_log[i]['message'])
    message = message['message']['params']# .get() 方式获取是了避免字段不存在时报错
    request = message.get('request')if(request isNone):continue
 
    url = request.get('url')if(url =="https://s3.pstatp.com/bytecom/resource/union_web2/media/manifest.json"):# 得到requestIdprint(message['requestId'])# 通过requestId获取接口内容
        content = driver.execute_cdp_cmd('Network.getResponseBody',{'requestId': message['requestId']})print(content)break

3.以上方法在操作中一直报错“no resource with given identifier found”，经排查，requestId是可以获取的，后来发现该接口是Ajax，selenium抓取Ajax接口的数据用如下demo（这个demo中包含了动态url获取过程）：

import os, time, json
from selenium.webdriver.common.by import By
from selenium import webdriver
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities

if __name__ =='__main__':# 引入chromedriver
    chrome_options = webdriver.ChromeOptions()# chrome_options.add_argument('--headless')# chrome_options.add_argument('--no-sandbox')# chrome_options.add_argument("--disable-extensions")# chrome_options.add_argument("--disable-gpu")
    prefs ={'download.default_directory': os.getenv('OS_LOG_PATH')}
    chrome_options.add_experimental_option('prefs', prefs)# make chrome log requests
    capabilities = DesiredCapabilities.CHROME
    # caps['goog:loggingPrefs']
    capabilities["goog:loggingPrefs"]={"performance":"ALL"}# newer: goog:loggingPrefs# capabilities['acceptSslCerts'] = True#browser = webdriver.Chrome(executable_path=r'/opt/google/chrome/chromedriver', options=chrome_options)
    browser = webdriver.Chrome(executable_path=r'C:\Program Files (x86)\Google\Chrome\Application\chromedriver.exe', options=chrome_options,desired_capabilities=capabilities)
    browser.implicitly_wait(3)
    browser.get("https://#########")# 输入账号
    browser.implicitly_wait(3)
    username_inputbox = browser.find_elements(By.XPATH,'//*[@id="app"]/section/div[1]/div/div[1]/div/div[2]/form/div[1]/div/div/div/input')[0]
    username_inputbox.click()
    browser.implicitly_wait(3)
    username_inputbox.clear()
    username_inputbox.send_keys("#######")# 输入密码
    browser.implicitly_wait(3)
    password_inputbox = browser.find_elements(By.XPATH,'//*[@id="app"]/section/div[1]/div/div[1]/div/div[2]/form/div[2]/div/div/div/input')[0]
    password_inputbox.click()
    browser.implicitly_wait(3)
    password_inputbox.clear()
    password_inputbox.send_keys("#######")# 点击登录
    browser.implicitly_wait(3)
    submit_btn = browser.find_elements(By.XPATH,'//*[@id="app"]/section/div[1]/div/div[1]/div/div[2]/form/div[3]/div/button')[0]
    submit_btn.click()# 等待5秒
    time.sleep(5)#-------------------------------------------------
    base_url ="https://#########"
    browser.get(base_url)

    time.sleep(3)# extract requests from logs
    logs_raw = browser.get_log("performance")
    logs =[json.loads(lr["message"])["message"]for lr in logs_raw]deflog_filter(log_):return(# is an actual response
                log_["method"]=="Network.responseReceived"# and jsonand"json"in log_["params"]["response"]["mimeType"])for log infilter(log_filter, logs):
        request_id = log["params"]["requestId"]
        resp_url = log["params"]["response"]["url"]print(request_id)print(f"Caught {resp_url}")
        json_str = json.dumps(browser.execute_cdp_cmd("Network.getResponseBody",{"requestId": request_id}), indent=4)# 创建一个params.json文件withopen(f'{request_id}.json','w')as f:
            f.write(json_str)# 将json_str写到文件中

    browser.quit()

后续如遇见更多的获取接口数据的情况，再做补充