Convert Claude Code or Codex into the most intelligent Deep Research Agent. Collect, search, and synthesize web research into a persistent, searchable wiki that builds on itself. Hosted API + MCP: hyperresearch.ai
Python agentsagentskillscitationsclaude-codecodexcodex-plugincodex-skilldeep-researchdeep-research-agentdeepresearchmcp-serverresearchresearch-agentresearch-tools
"""fetch() must work when called on a thread that already has a running event loop.
The MCP server's fastmcp tool dispatch calls synchronous tool functions directly on
its own running event-loop thread (it does not offload them to a worker thread), so
fetch_url's call chain into Crawl4AIProvider.fetch() lands there too. A plain
asyncio.run() inside fetch() would raise "cannot be called from a running event
loop" in that case; the CLI's plain synchronous invocation has no such loop and must
keep working exactly as before.
"""
from __future__ import annotations
import asyncio
import threading
import pytest
pytest.importorskip("crawl4ai.browser_adapter")
from hyperresearch.web.base import WebResult
from hyperresearch.web.crawl4ai_provider import Crawl4AIProvider
def _run_on_a_fresh_thread(coro):
"""Run ``coro`` to completion on a brand-new thread, guaranteeing a real,
isolated running event loop for the duration of the call.
See tests/test_web/test_fetch_many_fallback.py's ``_run`` for why: a prior
test in the full suite can leave the main thread's asyncio state marked
"running" (observed once crawl4ai's sync Playwright bridge has been
exercised), which makes a plain ``asyncio.run()`` here raise regardless of
what this test is trying to isolate. A fresh thread has no such leftover
state.
"""
box: dict = {}
def target() -> None:
try:
box["result"] = asyncio.run(coro)
except BaseException as exc:
box["error"] = exc
thread = threading.Thread(target=target)
thread.start()
thread.join()
if "error" in box:
raise box["error"]
return box["result"]
def test_fetch_succeeds_when_called_from_a_running_event_loop(monkeypatch) -> None:
async def fake_fetch_async(self, url):
return WebResult(url=url, title="ok", content="fetched content")
monkeypatch.setattr(Crawl4AIProvider, "_fetch_async", fake_fetch_async)
provider = Crawl4AIProvider(headless=True)
async def call_fetch_on_the_running_loop():
return provider.fetch("https://example.com")
result = _run_on_a_fresh_thread(call_fetch_on_the_running_loop())
assert result.title == "ok"
assert result.content == "fetched content"
def test_fetch_still_succeeds_with_no_running_loop(monkeypatch) -> None:
"""The CLI's plain synchronous call path (no event loop on the thread)."""
async def fake_fetch_async(self, url):
return WebResult(url=url, title="ok", content="fetched content")
monkeypatch.setattr(Crawl4AIProvider, "_fetch_async", fake_fetch_async)
provider = Crawl4AIProvider(headless=True)
result = provider.fetch("https://example.com")
assert result.title == "ok"
assert result.content == "fetched content"
def test_fetch_many_succeeds_when_called_from_a_running_event_loop(monkeypatch) -> None:
"""fetch_many() shares fetch()'s wrapper; a bare asyncio.run() here would
fail the same way on the MCP server's loop thread."""
async def fake_fetch_many_async(self, urls):
return [WebResult(url=u, title="ok", content="fetched content") for u in urls]
monkeypatch.setattr(Crawl4AIProvider, "_fetch_many_async", fake_fetch_many_async)
provider = Crawl4AIProvider(headless=True)
urls = ["https://example.com/a", "https://example.com/b"]
async def call_fetch_many_on_the_running_loop():
return provider.fetch_many(urls)
results = _run_on_a_fresh_thread(call_fetch_many_on_the_running_loop())
assert [r.url for r in results] == urls
assert all(r.content == "fetched content" for r in results)