Skip to content

Convert Claude Code or Codex into the most intelligent Deep Research Agent. Collect, search, and synthesize web research into a persistent, searchable wiki that builds on itself. Hosted API + MCP: hyperresearch.ai

fix(web): engage patchright stealth by passing UndetectedAdapter to the crawler (#36)

Sean Youngbergeb4a4a84af1d
diff --git a/pyproject.toml b/pyproject.toml
index 871e22c..eea22b9 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -28,7 +28,7 @@ dependencies = [
"pydantic>=2.0",
"jinja2>=3.1",
"platformdirs>=4.0",
- "Crawl4AI>=0.4",
+ "Crawl4AI>=0.7.3",
"pymupdf>=1.24",
"httpx>=0.27",
]
diff --git a/src/hyperresearch/web/crawl4ai_provider.py b/src/hyperresearch/web/crawl4ai_provider.py
index 3a9a74b..bfe253f 100644
--- a/src/hyperresearch/web/crawl4ai_provider.py
+++ b/src/hyperresearch/web/crawl4ai_provider.py
@@ -15,6 +15,8 @@ import sys
from datetime import UTC, datetime
from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, DefaultMarkdownGenerator
+from crawl4ai.async_crawler_strategy import AsyncPlaywrightCrawlerStrategy
+from crawl4ai.browser_adapter import UndetectedAdapter
from crawl4ai.content_filter_strategy import PruningContentFilter
from hyperresearch.web.base import WebResult, binary_garbage_ratio
@@ -190,6 +192,23 @@ class Crawl4AIProvider:
markdown_generator=self._md_generator,
)
+ def _make_crawler(self) -> AsyncWebCrawler:
+ """Build an AsyncWebCrawler wired with the stealth (patchright) adapter.
+
+ crawl4ai's AsyncWebCrawler defaults to PlaywrightAdapter (plain
+ playwright); patchright/stealth only engages when an UndetectedAdapter is
+ passed via an explicit AsyncPlaywrightCrawlerStrategy. Without this the
+ provider's anti-bot behavior is only simulate_user + smart-wait. The
+ adapter's use_undetected flag is threaded through every BrowserManager
+ launch branch (default, persistent-context, and managed-browser/CDP), so
+ this is compatible with the authenticated-profile (user_data_dir) path.
+ """
+ strategy = AsyncPlaywrightCrawlerStrategy(
+ browser_config=self._browser_config,
+ browser_adapter=UndetectedAdapter(),
+ )
+ return AsyncWebCrawler(crawler_strategy=strategy, config=self._browser_config)
+
def fetch(self, url: str) -> WebResult:
# PDF detection: fetch directly with httpx, extract text with pymupdf
if _is_pdf_url(url):
@@ -275,7 +294,7 @@ class Crawl4AIProvider:
)
async def _fetch_async(self, url: str) -> WebResult:
- async with AsyncWebCrawler(config=self._browser_config) as crawler:
+ async with self._make_crawler() as crawler:
result = await crawler.arun(url=url, config=self._run_config)
metadata = result.metadata or {}
@@ -346,7 +365,7 @@ class Crawl4AIProvider:
# Fetch HTML pages with browser
if html_urls:
- async with AsyncWebCrawler(config=self._browser_config) as crawler:
+ async with self._make_crawler() as crawler:
results = await crawler.arun_many(urls=html_urls, config=self._run_config)
for cr, url in zip(results, html_urls, strict=False):
if not cr.success:
diff --git a/tests/test_web/test_crawl4ai_stealth.py b/tests/test_web/test_crawl4ai_stealth.py
new file mode 100644
index 0000000..1b5cdd0
--- /dev/null
+++ b/tests/test_web/test_crawl4ai_stealth.py
@@ -0,0 +1,49 @@
+"""Tests for the crawl4ai provider's stealth (patchright) wiring.
+
+Deterministic and offline: asserts the crawler is built with an explicit
+UndetectedAdapter so patchright engages. crawl4ai's AsyncWebCrawler otherwise
+defaults to PlaywrightAdapter (plain Playwright) and stealth never runs. No
+browser is launched — only in-memory object construction is exercised.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+# crawl4ai is a core dependency, but guard so the suite degrades gracefully if a
+# minimal install lacks the browser_adapter API (introduced in crawl4ai 0.7.3).
+pytest.importorskip("crawl4ai.browser_adapter")
+
+from crawl4ai.async_crawler_strategy import AsyncPlaywrightCrawlerStrategy
+from crawl4ai.browser_adapter import UndetectedAdapter
+
+from hyperresearch.web.crawl4ai_provider import Crawl4AIProvider
+
+
+def test_make_crawler_uses_undetected_adapter() -> None:
+ """_make_crawler() must wire an UndetectedAdapter via AsyncPlaywrightCrawlerStrategy."""
+ crawler = Crawl4AIProvider(headless=True)._make_crawler()
+ strategy = crawler.crawler_strategy
+
+ assert isinstance(strategy, AsyncPlaywrightCrawlerStrategy)
+ assert isinstance(strategy.adapter, UndetectedAdapter)
+
+
+def test_browser_manager_marked_undetected() -> None:
+ """The adapter choice must propagate to BrowserManager.use_undetected.
+
+ This is the flag crawl4ai checks to import patchright instead of plain
+ playwright at launch — the actual engagement point.
+ """
+ crawler = Crawl4AIProvider(headless=True)._make_crawler()
+
+ assert crawler.crawler_strategy.browser_manager.use_undetected is True
+
+
+def test_undetected_wiring_survives_profile_path() -> None:
+ """The stealth adapter must also be wired when an authenticated profile
+ (user_data_dir / managed browser) is configured — the two are compatible."""
+ crawler = Crawl4AIProvider(headless=True, user_data_dir="/tmp/does-not-matter")._make_crawler()
+
+ assert isinstance(crawler.crawler_strategy.adapter, UndetectedAdapter)
+ assert crawler.crawler_strategy.browser_manager.use_undetected is True