如何在Crawl4AI中通过CapSolver集成解决AWS WAF问题

import asyncio
import capsolver
from crawl4ai import *


# TODO: 设置您的配置
api_key = "CAP-xxxxxxxxxxxxxxxxxxxxx"              # 您的 CapSolver API 密钥
site_url = "https://nft.porsche.com/onboarding@6"  # 您目标网站的页面 URL
cookie_domain = ".nft.porsche.com"                 # 您希望应用 Cookie 的域名
captcha_type = "AntiAwsWafTaskProxyLess"           # 您的目标验证码类型
capsolver.api_key = api_key


async def main():
    browser_config = BrowserConfig(
        verbose=True,
        headless=False,
        use_persistent_context=True,
    )

    async with AsyncWebCrawler(config=browser_config) as crawler:
        await crawler.arun(
            url=site_url,
            cache_mode=CacheMode.BYPASS,
            session_id="session_captcha_test"
        )

        # 使用 CapSolver SDK 获取 AWS WAF Cookie
        solution = capsolver.solve({
            "type": captcha_type,
            "websiteURL": site_url,
        })
        cookie = solution["cookie"]
        print("aws waf cookie:", cookie)

        js_code = """
            document.cookie = \'aws-waf-token=""" + cookie + """;domain=""" + cookie_domain + """;path=/\
\';
            location.reload();
        """

        wait_condition = """() => {
            return document.title === \'Join Porsche’s journey into Web3\';
        }"""

        run_config = CrawlerRunConfig(
            cache_mode=CacheMode.BYPASS,
            session_id="session_captcha_test",
            js_code=js_code,
            js_only=True,
            wait_for=f"js:{wait_condition}"
        )

        result_next = await crawler.arun(
            url=site_url,
            config=run_config,
        )
        print(result_next.markdown)


if __name__ == "__main__":
    asyncio.run(main())

import asyncio
import time

from crawl4ai import *


# TODO: 设置您的配置
user_data_dir = "/browser-profile/Default1" # 请确保此路径正确并包含您的配置扩展

browser_config = BrowserConfig(
    verbose=True,
    headless=False,
    user_data_dir=user_data_dir,
    use_persistent_context=True,
    proxy="http://127.0.0.1:13120", # 可选：如需配置代理
)

async def main():
    async with AsyncWebCrawler(config=browser_config) as crawler:
        result_initial = await crawler.arun(
            url="https://nft.porsche.com/onboarding@6",
            cache_mode=CacheMode.BYPASS,
            session_id="session_captcha_test"
        )

        # 扩展将在页面加载时自动解决 AWS WAF。
        # 您可能需要添加等待条件或 time.sleep 以在继续进一步操作前让 WAF 解决。
        time.sleep(30) # 示例等待，根据扩展操作调整

        # 在 AWS WAF 解决后继续其他 Crawl4AI 操作
        # 例如，检查在成功验证后出现的元素或内容
        # print(result_initial.markdown) # 您可以在等待后检查页面内容


if __name__ == "__main__":
    asyncio.run(main())

import asyncio
import time

from crawl4ai import *


# TODO: 设置您的配置
user_data_dir = "/browser-profile/Default1" # 请确保此路径正确并包含您的配置扩展

browser_config = BrowserConfig(
    verbose=True,
    headless=False,
    user_data_dir=user_data_dir,
    use_persistent_context=True,
    proxy="http://127.0.0.1:13120", # 可选：如需配置代理
)

async def main():
    async with AsyncWebCrawler(config=browser_config) as crawler:
        result_initial = await crawler.arun(
            url="https://nft.porsche.com/onboarding@6",
            cache_mode=CacheMode.BYPASS,
            session_id="session_captcha_test"
        )

        # 等待片刻以让页面加载并让扩展准备好
        time.sleep(6)

        # 使用 js_code 触发 CapSolver 扩展提供的手动解决按钮
        js_code = """
            let solverButton = document.querySelector(\'#capsolver-solver-tip-button\');
            if (solverButton) {
            // 点击事件
                const clickEvent = new MouseEvent(\'click\', {
                    bubbles: true,
                    cancelable: true,
                    view: window
                });
                
                solverButton.dispatchEvent(clickEvent);
            }
        """
        print(js_code)
        run_config = CrawlerRunConfig(
            cache_mode=CacheMode.BYPASS,
            session_id="session_captcha_test",
            js_code=js_code,
            js_only=True,
        )
        result_next = await crawler.arun(
            url="https://nft.porsche.com/onboarding@6",
            config=run_config
        )
        print("JS 执行结果:", result_next.js_execution_result)

        # 在手动触发后允许时间让 AWS WAF 解决
        time.sleep(30) # 示例等待，根据需要调整

        # 继续其他 Crawl4AI 操作


if __name__ == "__main__":
    asyncio.run(main())