Skip to content

Convert Claude Code or Codex into the most intelligent Deep Research Agent. Collect, search, and synthesize web research into a persistent, searchable wiki that builds on itself. Hosted API + MCP: hyperresearch.ai

4.7 KB
Raw Blame
"""Tests for FetchSettings threading into the crawl4ai provider internals."""

from __future__ import annotations

from unittest.mock import patch

import pytest

from hyperresearch.core.config import FetchSettings, JunkGates
from hyperresearch.web.safe_http import SafeResponse

crawl4ai_provider = pytest.importorskip(
    "hyperresearch.web.crawl4ai_provider",
    reason="crawl4ai extra not installed",
)


class TestSmartWaitJs:
    def test_defaults_match_pre_20_literals(self):
        js = crawl4ai_provider._smart_wait_js(FetchSettings())
        assert "}, 2000);" in js  # initial delay
        assert "}, 500);" in js  # poll interval
        assert "stable >= 2" in js
        assert "checks > 16" in js

    def test_custom_settings_rendered(self):
        js = crawl4ai_provider._smart_wait_js(
            FetchSettings(wait_initial_ms=100, poll_interval_ms=50, stable_checks=5, max_checks=99)
        )
        assert "}, 100);" in js
        assert "}, 50);" in js
        assert "stable >= 5" in js
        assert "checks > 99" in js

    def test_js_is_well_formed(self):
        js = crawl4ai_provider._smart_wait_js(FetchSettings())
        assert js.startswith("() => new Promise")
        assert js.count("{") == js.count("}")
        assert js.count("(") == js.count(")")


class TestLooksLikeBinary:
    def test_pdf_markers_always_binary(self):
        assert crawl4ai_provider._looks_like_binary("endstream endobj blah") is True

    def test_ratio_uses_gates(self):
        text = ("real words all over the place here " * 20) + "\x01\x02" * 60
        assert crawl4ai_provider._looks_like_binary(text) is True
        assert crawl4ai_provider._looks_like_binary(text, JunkGates(binary_garbage_ratio=0.9)) is False

    def test_empty_is_not_binary(self):
        assert crawl4ai_provider._looks_like_binary("") is False


class TestProviderConstruction:
    def test_settings_reach_run_config(self):
        prov = crawl4ai_provider.Crawl4AIProvider(
            settings=FetchSettings(page_timeout_ms=77777, stable_checks=7)
        )
        assert prov._run_config.page_timeout == 77777
        assert "stable >= 7" in prov._wait_js
        assert prov._wait_js.startswith("js:")

    def test_default_construction_matches_pre_20(self):
        prov = crawl4ai_provider.Crawl4AIProvider()
        assert prov._run_config.page_timeout == 30000
        assert "stable >= 2" in prov._wait_js
        assert "checks > 16" in prov._wait_js


class TestGetProviderThreading:
    def test_get_provider_passes_settings(self):
        from hyperresearch.web.base import get_provider

        prov = get_provider(
            "crawl4ai",
            settings=FetchSettings(page_timeout_ms=12345),
            gates=JunkGates(binary_garbage_ratio=0.9),
        )
        assert prov._run_config.page_timeout == 12345
        assert prov._gates.binary_garbage_ratio == 0.9


class TestByteCapThreading:
    """The configurable size caps only protect anything if the FetchSettings
    field actually reaches safe_get's max_bytes. safe_get's own enforcement is
    tested in test_safe_http; these pin the wiring from config to gate, which
    is what a refactor to the hardcoded MAX_BYTES_* constants would silently
    break."""

    def test_html_cap_reaches_safe_get(self):
        from hyperresearch.web.builtin import BuiltinProvider

        seen = {}

        def spy(url, *, max_bytes, **kwargs):
            seen["max_bytes"] = max_bytes
            return SafeResponse(url=url, status_code=200, headers={}, content=b"<html></html>")

        prov = BuiltinProvider(settings=FetchSettings(max_html_bytes=1234))
        with patch("hyperresearch.web.safe_http.safe_get", spy):
            prov._download("http://8.8.8.8/page")
        assert seen["max_bytes"] == 1234

    def test_pdf_cap_reaches_safe_get(self):
        seen = {}

        def spy(url, *, max_bytes, **kwargs):
            seen["max_bytes"] = max_bytes
            return SafeResponse(url=url, status_code=200, headers={}, content=b"%PDF-")

        with patch("hyperresearch.web.safe_http.safe_get", spy):
            crawl4ai_provider._safe_get_pdf("http://8.8.8.8/paper.pdf", FetchSettings(max_pdf_bytes=777))
        assert seen["max_bytes"] == 777


class TestAllowPrivateHostsConfig:
    def test_default_is_empty(self):
        """Fresh installs get the full gate; admitting a private host is an
        explicit config act."""
        assert FetchSettings().allow_private_hosts == ()

    def test_toml_list_parses_to_tuple(self):
        from hyperresearch.core.config import _build_section

        s = _build_section(
            FetchSettings,
            {"allow_private_hosts": ["10.8.0.0/16", "mirror.internal"]},
        )
        assert s.allow_private_hosts == ("10.8.0.0/16", "mirror.internal")