Skip to content

Convert Claude Code or Codex into the most intelligent Deep Research Agent. Collect, search, and synthesize web research into a persistent, searchable wiki that builds on itself. Hosted API + MCP: hyperresearch.ai

3.9 KB
Raw Blame
"""fetch_many must apply the same SSRF gate as fetch().

It is the main fetch path (every wave-1 fetcher batch goes through it), so a
gate that only covered fetch() would leave the bulk of outbound traffic
unprotected. Offline: only .pdf URLs are used and _fetch_pdf is stubbed to
SUCCEED, so the browser lane is never touched — since v0.9.1 a *failed* PDF
falls back to the browser (that path is covered in test_fetch_many_fallback
and test_final_url_recheck).
"""

from __future__ import annotations

import asyncio
import logging

import hyperresearch.web.crawl4ai_provider as provider
from hyperresearch.web.base import WebResult


def _run(coro):
    """Run ``coro`` to completion in an isolated thread with its own loop.

    A prior test in the full suite can leave this thread's event-loop state as
    "running" (seen intermittently on Python 3.11 once crawl4ai's async
    machinery has been exercised), which makes a plain ``asyncio.run()`` here
    raise "cannot be called from a running event loop". A fresh thread has no
    running loop, so running the coroutine there is immune to that leaked state.
    Logging still lands in caplog: handlers are process-global, and the thread
    is joined inside the caplog context below.
    """
    import threading

    box: dict = {}

    def target() -> None:
        try:
            box["result"] = asyncio.run(coro)
        except Exception as exc:
            box["error"] = exc

    thread = threading.Thread(target=target)
    thread.start()
    thread.join()
    if "error" in box:
        raise box["error"]
    return box["result"]


def test_fetch_many_skips_malformed_url_and_fetches_the_rest(monkeypatch, caplog):
    """A URL check_url cannot parse must be skipped like a refused one. urlparse
    raises ValueError for unbalanced IPv6 brackets, and httpx.URL raises
    UnicodeEncodeError for a lone surrogate (a non-UTF-8 byte in a
    command-line argument becomes one); if either escapes check_url as
    anything but SafeHTTPError, the per-URL handler in _fetch_many_async
    does not catch it and every other URL in the batch is lost with it."""
    fetched: list[str] = []

    def fake_fetch_pdf(url, settings=None):
        fetched.append(url)
        return WebResult(url=url, title="ok", content="pdf text")

    monkeypatch.setattr(provider, "_fetch_pdf", fake_fetch_pdf)

    inst = provider.Crawl4AIProvider.__new__(provider.Crawl4AIProvider)
    inst._settings = provider.FetchSettings()

    urls = [
        "http://[::1/broken.pdf",
        "http://8.8.8.8/pa\udcffper.pdf",
        "http://8.8.8.8/paper.pdf",
    ]
    with caplog.at_level(logging.WARNING, logger="hyperresearch.web"):
        results = _run(inst._fetch_many_async(urls))

    assert [r.url for r in results] == ["http://8.8.8.8/paper.pdf"]
    assert fetched == ["http://8.8.8.8/paper.pdf"]
    assert "refused batch fetch" in caplog.text


def test_fetch_many_skips_refused_urls_and_fetches_the_rest(monkeypatch, caplog):
    fetched: list[str] = []

    def fake_fetch_pdf(url, settings=None):
        fetched.append(url)
        return WebResult(url=url, title="ok", content="pdf text")

    monkeypatch.setattr(provider, "_fetch_pdf", fake_fetch_pdf)

    inst = provider.Crawl4AIProvider.__new__(provider.Crawl4AIProvider)
    inst._settings = provider.FetchSettings()

    urls = [
        "http://127.0.0.1/internal.pdf",  # loopback — refused
        "http://169.254.169.254/meta.pdf",  # cloud metadata — refused
        "http://8.8.8.8/paper.pdf",  # public literal — allowed
    ]
    with caplog.at_level(logging.WARNING, logger="hyperresearch.web"):
        results = _run(inst._fetch_many_async(urls))

    assert [r.url for r in results] == ["http://8.8.8.8/paper.pdf"]
    assert fetched == ["http://8.8.8.8/paper.pdf"], (
        "refused URLs must be skipped, allowed ones still fetched"
    )
    assert "refused batch fetch" in caplog.text
    assert "127.0.0.1" in caplog.text