如何为人工智能搜索代理构建搜索结果页面功能监控

pip install git+https://github.com/capsolver-ai/capsolver-core.git
pip install aiohttp beautifulsoup4 pandas schedule

export CAPSOLVER_API_KEY="your-capsolver-api-key"

from dataclasses import dataclass
from typing import List, Optional
from enum import Enum

class SERPFeature(Enum):
    FEATURED_SNIPPET = "featured_snippet"
    PEOPLE_ALSO_ASK = "people_also_ask"
    AI_OVERVIEW = "ai_overview"
    LOCAL_PACK = "local_pack"
    KNOWLEDGE_PANEL = "knowledge_panel"
    VIDEO_CAROUSEL = "video_carousel"
    IMAGE_PACK = "image_pack"
    TOP_STORIES = "top_stories"
    SHOPPING_RESULTS = "shopping_results"

@dataclass
class SERPResult:
    keyword: str
    features_detected: List[SERPFeature]
    featured_snippet_url: Optional[str]
    featured_snippet_text: Optional[str]
    paa_questions: List[str]
    ai_overview_present: bool
    ai_overview_sources: List[str]
    organic_positions: List[dict]  # [{url, title, position}]
    timestamp: float

class SERPFeatureDetector:
    """从解析的搜索结果页面中检测SERP功能。"""
    
    def detect_features(self, html_content: str, keyword: str) -> SERPResult:
        from bs4 import BeautifulSoup
        soup = BeautifulSoup(html_content, 'html.parser')
        
        features = []
        
        # 精选摘要检测
        snippet_div = soup.select_one('[data-attrid="wa:/description"], .xpdopen')
        featured_url = None
        featured_text = None
        if snippet_div:
            features.append(SERPFeature.FEATURED_SNIPPET)
            featured_text = snippet_div.get_text(strip=True)[:500]
            link = snippet_div.find('a')
            featured_url = link['href'] if link else None
        
        # 人们也问检测
        paa_questions = []
        paa_section = soup.select('[data-q]')
        if paa_section:
            features.append(SERPFeature.PEOPLE_ALSO_ASK)
            paa_questions = [q.get('data-q', '') for q in paa_section[:8]]
        
        # AI摘要检测
        ai_overview = soup.select_one('[data-attrid*="ai"], .ai-overview-container')
        ai_sources = []
        if ai_overview:
            features.append(SERPFeature.AI_OVERVIEW)
            source_links = ai_overview.select('a[href]')
            ai_sources = [a['href'] for a in source_links[:5]]
        
        # 本地包检测
        if soup.select_one('.VkpGBb, [data-attrid*="local"]'):
            features.append(SERPFeature.LOCAL_PACK)
        
        return SERPResult(
            keyword=keyword,
            features_detected=features,
            featured_snippet_url=featured_url,
            featured_snippet_text=featured_text,
            paa_questions=paa_questions,
            ai_overview_present=SERPFeature.AI_OVERVIEW in features,
            ai_overview_sources=ai_sources,
            organic_positions=self._extract_organic(soup),
            timestamp=time.time()
        )
    
    def _extract_organic(self, soup) -> list:
        results = []
        for i, item in enumerate(soup.select('.g, [data-sokoban-container]')[:10], 1):
            link = item.select_one('a[href^="http"]')
            title = item.select_one('h3')
            if link and title:
                results.append({
                    "position": i,
                    "url": link['href'],
                    "title": title.get_text(strip=True)
                })
        return results

import asyncio
import aiohttp
import time
import random
from capsolver_core import create_capsolver, CaptchaType, CaptchaInfo

class SERPCollector:
    """具有自动验证码处理的SERP数据收集器。"""
    
    def __init__(self, api_key: str, proxies: list):
        self.cap = create_capsolver(api_key=api_key)
        self.proxies = proxies
        self.proxy_index = 0
        self.stats = {"queries": 0, "captchas_solved": 0, "failures": 0}
    
    def _get_proxy(self) -> str:
        proxy = self.proxies[self.proxy_index % len(self.proxies)]
        self.proxy_index += 1
        return proxy
    
    async def search(self, keyword: str, location: str = "us") -> str:
        """执行Google搜索并处理验证码。"""
        proxy = self._get_proxy()
        url = f"https://www.google.com/search?q={keyword}&gl={location}&hl=en"
        
        headers = {
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
            "Accept-Language": "en-US,en;q=0.9",
            "Accept": "text/html,application/xhtml+xml"
        }
        
        async with aiohttp.ClientSession() as session:
            async with session.get(url, headers=headers, proxy=proxy, timeout=30) as resp:
                html = await resp.text()
                
                # 检查是否为验证码页面
                if self._is_captcha_page(html):
                    html = await self._solve_and_retry(keyword, location, proxy, session)
                
                self.stats["queries"] += 1
                return html
    
    def _is_captcha_page(self, html: str) -> bool:
        """检测响应是否为验证码挑战页面。"""
        captcha_indicators = [
            "unusual traffic from your computer",
            "g-recaptcha",
            "recaptcha/api",
            "sorry/index",
            "captcha"
        ]
        return any(indicator in html.lower() for indicator in captcha_indicators)
    
    async def _solve_and_retry(self, keyword: str, location: str, proxy: str, session) -> str:
        """解决验证码并重试搜索。"""
        # Google在其挑战页面上使用reCAPTCHA v2
        info = CaptchaInfo(
            type=CaptchaType.RECAPTCHA_V2,
            website_url="https://www.google.com/sorry/index",
            website_key="6LfwuyUTAAAAAOAmoS0fdqijC2PbbdH4kjq62Y1b"
        )
        
        solution = await self.cap.solve(info)
        self.stats["captchas_solved"] += 1
        
        # 使用解决的令牌重试搜索
        retry_url = f"https://www.google.com/search?q={keyword}&gl={location}&hl=en"
        headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"}
        
        # 在重试请求中包含验证码令牌
        async with session.get(retry_url, headers=headers, proxy=proxy) as resp:
            return await resp.text()
    
    async def close():
        await self.cap.aclose()

class SERPMonitoringPipeline:
    """完整的SERP功能监控管道。"""
    
    def __init__(self, api_key: str, proxies: list):
        self.collector = SERPCollector(api_key, proxies)
        self.detector = SERPFeatureDetector()
        self.results_history = []
    
    async def monitor_keywords(self, keywords: list, location: str = "us") -> list:
        """监控一批关键词的SERP功能变化。"""
        results = []
        
        for keyword in keywords:
            try:
                # 收集SERP数据
                html = await self.collector.search(keyword, location)
                
                # 检测功能
                serp_result = self.detector.detect_features(html, keyword)
                results.append(serp_result)
                
                # 速率限制——5-10秒随机延迟
                await asyncio.sleep(random.uniform(5, 10))
                
            except Exception as e:
                results.append(SERPResult(
                    keyword=keyword, features_detected=[], 
                    featured_snippet_url=None, featured_snippet_text=None,
                    paa_questions=[], ai_overview_present=False,
                    ai_overview_sources=[], organic_positions=[],
                    timestamp=time.time()
                ))
        
        self.results_history.extend(results)
        return results
    
    def detect_changes(self, current: list, previous: list) -> list:
        """比较当前和之前的結果以检测变化。"""
        changes = []
        prev_map = {r.keyword: r for r in previous}
        
        for result in current:
            prev = prev_map.get(result.keyword)
            if not prev:
                continue
            
            # 精选摘要获得/丢失
            had_snippet = SERPFeature.FEATURED_SNIPPET in prev.features_detected
            has_snippet = SERPFeature.FEATURED_SNIPPET in result.features_detected
            if has_snippet and not had_snippet:
                changes.append(f"[获得] '{result.keyword}' 的精选摘要：{result.featured_snippet_url}")
            elif had_snippet and not has_snippet:
                changes.append(f"[丢失] '{result.keyword}' 的精选摘要")
            
            # AI摘要出现/消失
            if result.ai_overview_present and not prev.ai_overview_present:
                changes.append(f"[新增] '{result.keyword}' 的AI摘要出现")
            elif prev.ai_overview_present and not result.ai_overview_present:
                changes.append(f"[移除] '{result.keyword}' 的AI摘要移除")
        
        return changes