Convert Claude Code or Codex into the most intelligent Deep Research Agent. Collect, search, and synthesize web research into a persistent, searchable wiki that builds on itself. Hosted API + MCP: hyperresearch.ai
Python agentsagentskillscitationsclaude-codecodexcodex-plugincodex-skilldeep-researchdeep-research-agentdeepresearchmcp-serverresearchresearch-agentresearch-tools
fix(web): engage patchright stealth by passing UndetectedAdapter to the crawler (#36)
Sean Youngbergeb4a4a84af1d
diff --git a/pyproject.toml b/pyproject.tomlindex 871e22c..eea22b9 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -28,7 +28,7 @@ dependencies = ["pydantic>=2.0","jinja2>=3.1","platformdirs>=4.0",- "Crawl4AI>=0.4",+ "Crawl4AI>=0.7.3","pymupdf>=1.24","httpx>=0.27",]diff --git a/src/hyperresearch/web/crawl4ai_provider.py b/src/hyperresearch/web/crawl4ai_provider.pyindex 3a9a74b..bfe253f 100644--- a/src/hyperresearch/web/crawl4ai_provider.py+++ b/src/hyperresearch/web/crawl4ai_provider.py@@ -15,6 +15,8 @@ import sysfrom datetime import UTC, datetimefrom crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, DefaultMarkdownGenerator+from crawl4ai.async_crawler_strategy import AsyncPlaywrightCrawlerStrategy+from crawl4ai.browser_adapter import UndetectedAdapterfrom crawl4ai.content_filter_strategy import PruningContentFilterfrom hyperresearch.web.base import WebResult, binary_garbage_ratio@@ -190,6 +192,23 @@ class Crawl4AIProvider:markdown_generator=self._md_generator,)+ def _make_crawler(self) -> AsyncWebCrawler:+ """Build an AsyncWebCrawler wired with the stealth (patchright) adapter.++ crawl4ai's AsyncWebCrawler defaults to PlaywrightAdapter (plain+ playwright); patchright/stealth only engages when an UndetectedAdapter is+ passed via an explicit AsyncPlaywrightCrawlerStrategy. Without this the+ provider's anti-bot behavior is only simulate_user + smart-wait. The+ adapter's use_undetected flag is threaded through every BrowserManager+ launch branch (default, persistent-context, and managed-browser/CDP), so+ this is compatible with the authenticated-profile (user_data_dir) path.+ """+ strategy = AsyncPlaywrightCrawlerStrategy(+ browser_config=self._browser_config,+ browser_adapter=UndetectedAdapter(),+ )+ return AsyncWebCrawler(crawler_strategy=strategy, config=self._browser_config)+def fetch(self, url: str) -> WebResult:# PDF detection: fetch directly with httpx, extract text with pymupdfif _is_pdf_url(url):@@ -275,7 +294,7 @@ class Crawl4AIProvider:)async def _fetch_async(self, url: str) -> WebResult:- async with AsyncWebCrawler(config=self._browser_config) as crawler:+ async with self._make_crawler() as crawler:result = await crawler.arun(url=url, config=self._run_config)metadata = result.metadata or {}@@ -346,7 +365,7 @@ class Crawl4AIProvider:# Fetch HTML pages with browserif html_urls:- async with AsyncWebCrawler(config=self._browser_config) as crawler:+ async with self._make_crawler() as crawler:results = await crawler.arun_many(urls=html_urls, config=self._run_config)for cr, url in zip(results, html_urls, strict=False):if not cr.success:diff --git a/tests/test_web/test_crawl4ai_stealth.py b/tests/test_web/test_crawl4ai_stealth.pynew file mode 100644index 0000000..1b5cdd0--- /dev/null+++ b/tests/test_web/test_crawl4ai_stealth.py@@ -0,0 +1,49 @@+"""Tests for the crawl4ai provider's stealth (patchright) wiring.++Deterministic and offline: asserts the crawler is built with an explicit+UndetectedAdapter so patchright engages. crawl4ai's AsyncWebCrawler otherwise+defaults to PlaywrightAdapter (plain Playwright) and stealth never runs. No+browser is launched — only in-memory object construction is exercised.+"""++from __future__ import annotations++import pytest++# crawl4ai is a core dependency, but guard so the suite degrades gracefully if a+# minimal install lacks the browser_adapter API (introduced in crawl4ai 0.7.3).+pytest.importorskip("crawl4ai.browser_adapter")++from crawl4ai.async_crawler_strategy import AsyncPlaywrightCrawlerStrategy+from crawl4ai.browser_adapter import UndetectedAdapter++from hyperresearch.web.crawl4ai_provider import Crawl4AIProvider+++def test_make_crawler_uses_undetected_adapter() -> None:+ """_make_crawler() must wire an UndetectedAdapter via AsyncPlaywrightCrawlerStrategy."""+ crawler = Crawl4AIProvider(headless=True)._make_crawler()+ strategy = crawler.crawler_strategy++ assert isinstance(strategy, AsyncPlaywrightCrawlerStrategy)+ assert isinstance(strategy.adapter, UndetectedAdapter)+++def test_browser_manager_marked_undetected() -> None:+ """The adapter choice must propagate to BrowserManager.use_undetected.++ This is the flag crawl4ai checks to import patchright instead of plain+ playwright at launch — the actual engagement point.+ """+ crawler = Crawl4AIProvider(headless=True)._make_crawler()++ assert crawler.crawler_strategy.browser_manager.use_undetected is True+++def test_undetected_wiring_survives_profile_path() -> None:+ """The stealth adapter must also be wired when an authenticated profile+ (user_data_dir / managed browser) is configured — the two are compatible."""+ crawler = Crawl4AIProvider(headless=True, user_data_dir="/tmp/does-not-matter")._make_crawler()++ assert isinstance(crawler.crawler_strategy.adapter, UndetectedAdapter)+ assert crawler.crawler_strategy.browser_manager.use_undetected is True