Convert Claude Code or Codex into the most intelligent Deep Research Agent. Collect, search, and synthesize web research into a persistent, searchable wiki that builds on itself. Hosted API + MCP: hyperresearch.ai
Python agentsagentskillscitationsclaude-codecodexcodex-plugincodex-skilldeep-researchdeep-research-agentdeepresearchmcp-serverresearchresearch-agentresearch-tools
"""fetch_many must apply the same SSRF gate as fetch().
It is the main fetch path (every wave-1 fetcher batch goes through it), so a
gate that only covered fetch() would leave the bulk of outbound traffic
unprotected. Offline: only .pdf URLs are used and _fetch_pdf is stubbed to
SUCCEED, so the browser lane is never touched — since v0.9.1 a *failed* PDF
falls back to the browser (that path is covered in test_fetch_many_fallback
and test_final_url_recheck).
"""
from __future__ import annotations
import asyncio
import logging
import hyperresearch.web.crawl4ai_provider as provider
from hyperresearch.web.base import WebResult
def _run(coro):
"""Run ``coro`` to completion in an isolated thread with its own loop.
A prior test in the full suite can leave this thread's event-loop state as
"running" (seen intermittently on Python 3.11 once crawl4ai's async
machinery has been exercised), which makes a plain ``asyncio.run()`` here
raise "cannot be called from a running event loop". A fresh thread has no
running loop, so running the coroutine there is immune to that leaked state.
Logging still lands in caplog: handlers are process-global, and the thread
is joined inside the caplog context below.
"""
import threading
box: dict = {}
def target() -> None:
try:
box["result"] = asyncio.run(coro)
except Exception as exc:
box["error"] = exc
thread = threading.Thread(target=target)
thread.start()
thread.join()
if "error" in box:
raise box["error"]
return box["result"]
def test_fetch_many_skips_malformed_url_and_fetches_the_rest(monkeypatch, caplog):
"""A URL check_url cannot parse must be skipped like a refused one. urlparse
raises ValueError for unbalanced IPv6 brackets, and httpx.URL raises
UnicodeEncodeError for a lone surrogate (a non-UTF-8 byte in a
command-line argument becomes one); if either escapes check_url as
anything but SafeHTTPError, the per-URL handler in _fetch_many_async
does not catch it and every other URL in the batch is lost with it."""
fetched: list[str] = []
def fake_fetch_pdf(url, settings=None):
fetched.append(url)
return WebResult(url=url, title="ok", content="pdf text")
monkeypatch.setattr(provider, "_fetch_pdf", fake_fetch_pdf)
inst = provider.Crawl4AIProvider.__new__(provider.Crawl4AIProvider)
inst._settings = provider.FetchSettings()
urls = [
"http://[::1/broken.pdf",
"http://8.8.8.8/pa\udcffper.pdf",
"http://8.8.8.8/paper.pdf",
]
with caplog.at_level(logging.WARNING, logger="hyperresearch.web"):
results = _run(inst._fetch_many_async(urls))
assert [r.url for r in results] == ["http://8.8.8.8/paper.pdf"]
assert fetched == ["http://8.8.8.8/paper.pdf"]
assert "refused batch fetch" in caplog.text
def test_fetch_many_skips_refused_urls_and_fetches_the_rest(monkeypatch, caplog):
fetched: list[str] = []
def fake_fetch_pdf(url, settings=None):
fetched.append(url)
return WebResult(url=url, title="ok", content="pdf text")
monkeypatch.setattr(provider, "_fetch_pdf", fake_fetch_pdf)
inst = provider.Crawl4AIProvider.__new__(provider.Crawl4AIProvider)
inst._settings = provider.FetchSettings()
urls = [
"http://127.0.0.1/internal.pdf", # loopback — refused
"http://169.254.169.254/meta.pdf", # cloud metadata — refused
"http://8.8.8.8/paper.pdf", # public literal — allowed
]
with caplog.at_level(logging.WARNING, logger="hyperresearch.web"):
results = _run(inst._fetch_many_async(urls))
assert [r.url for r in results] == ["http://8.8.8.8/paper.pdf"]
assert fetched == ["http://8.8.8.8/paper.pdf"], (
"refused URLs must be skipped, allowed ones still fetched"
)
assert "refused batch fetch" in caplog.text
assert "127.0.0.1" in caplog.text