2025年进行网页爬取时最佳的AWS WAF解决方案是什么？

pip install requests

CAPSOLVER_API_KEY = "YOUR_CAPSOLVER_API_KEY"

site_key = "6Le-wvkSAAAAAPBMRTvw0Q4Muexq9bi0DJwx_mJ-"  # 替换为您的网站AWS密钥
site_url = "https://efw47fpad9.execute-api.us-east-1.amazonaws.com/latest"  # 替换为您的网站URL

import requests
import re
import time

# 您的CapSolver API密钥
CAPSOLVER_API_KEY = "YOUR_CAPSOLVER_API_KEY"
CAPSOLVER_CREATE_TASK_ENDPOINT = "https://api.capsolver.com/createTask"
CAPSOLVER_GET_TASK_RESULT_ENDPOINT = "https://api.capsolver.com/getTaskResult"

# 受AWS WAF保护的网站URL
WEBSITE_URL = "https://efw47fpad9.execute-api.us-east-1.amazonaws.com/latest" # 示例URL

def solve_aws_waf_captcha(website_url, capsolver_api_key):
    client = requests.Session()
    response = client.get(website_url)
    script_content = response.text

    key_match = re.search(r'"key":"([^"]+)"', script_content)
    iv_match = re.search(r'"iv":"([^"]+)"', script_content)
    context_match = re.search(r'"context":"([^"]+)"', script_content)
    jschallenge_match = re.search(r'<script.*?src="(.*?)".*?></script>', script_content)

    key = key_match.group(1) if key_match else None
    iv = iv_match.group(1) if iv_match else None
    context = context_match.group(1) if context_match else None
    jschallenge = jschallenge_match.group(1) if jschallenge_match else None

    if not all([key, iv, context, jschallenge]):
        print("错误：在页面内容中未找到AWS WAF参数。")
        return None

    task_payload = {
        "clientKey": capsolver_api_key,
        "task": {
            "type": "AntiAwsWafTaskProxyLess",
            "websiteURL": website_url,
            "awsKey": key,
            "awsIv": iv,
            "awsContext": context,
            "awsChallengeJS": jschallenge
        }
    }

    create_task_response = client.post(CAPSOLVER_CREATE_TASK_ENDPOINT, json=task_payload).json()
    task_id = create_task_response.get('taskId')

    if not task_id:
        print(f"创建CapSolver任务时出错：{create_task_response.get('errorId')}, {create_task_response.get('errorCode')}")
        return None

    print(f"CapSolver任务已创建，任务ID为：{task_id}")

    # 轮询任务结果
    for _ in range(10): # 最多尝试10次，每次间隔5秒
        time.sleep(5)
        get_result_payload = {"clientKey": capsolver_api_key, "taskId": task_id}
        get_result_response = client.post(CAPSOLVER_GET_TASK_RESULT_ENDPOINT, json=get_result_payload).json()

        if get_result_response.get('status') == 'ready':
            aws_waf_token_cookie = get_result_response['solution']['cookie']
            print("CapSolver成功解决CAPTCHA。")
            return aws_waf_token_cookie
        elif get_result_response.get('status') == 'failed':
            print(f"CapSolver任务失败：{get_result_response.get('errorId')}, {get_result_response.get('errorCode')}")
            return None

    print("CapSolver任务超时。")
    return None

# 示例用法：
# aws_waf_token = solve_aws_waf_captcha(WEBSITE_URL, CAPSOLVER_API_KEY)
# if aws_waf_token:
#     print(f"收到AWS WAF令牌：{aws_waf_token}")
#     # 在后续请求中使用该令牌
#     final_response = requests.get(WEBSITE_URL, cookies={"aws-waf-token": aws_waf_token})
#     print(final_response.text)