Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

19.3 KB
Raw Blame
"""Availability transitions use a fake clock, never network requests or sleeps."""

from concurrent.futures import ThreadPoolExecutor
from datetime import UTC, datetime
from email.utils import format_datetime
from types import SimpleNamespace
from unittest.mock import Mock, patch

import pytest
import requests

from local_deep_research.web_search_engines.engine_availability import (
    COOLDOWN_WAIT_BUDGET_SECONDS,
    MAX_TEMPORARY_COOLDOWN_SECONDS,
    SearchEngineAvailability,
    SearchFailure,
    available_tool_names,
    engine_failure,
    failure_from_exception,
    retry_after_seconds,
)
from local_deep_research.web_search_engines.rate_limiting import RateLimitError
from local_deep_research.web_search_engines.search_engine_base import (
    BaseSearchEngine,
)


@pytest.fixture
def clock():
    return SimpleNamespace(monotonic=100.0, wall=1_790_000_000.0)


@pytest.fixture
def tracker(clock):
    return SearchEngineAvailability(
        clock=lambda: clock.monotonic, wall_clock=lambda: clock.wall
    )


def test_cooldown_logs_one_transition_and_expires_on_monotonic_time(clock):
    events = Mock()
    tracker = SearchEngineAvailability(
        events, clock=lambda: clock.monotonic, wall_clock=lambda: clock.wall
    )
    first = tracker.disable("pubmed", SearchFailure("Rate limit reached", 300))
    assert tracker.disable("pubmed", SearchFailure("duplicate", 900)) == first
    assert events.call_count == 1
    message, _, meta = events.call_args.args
    assert "for 300s" in message and "retry available at" in message
    assert meta["disabled_at"] == clock.wall
    assert meta["retry_at"] == clock.wall + 300
    clock.wall += 86_400  # Wall-clock correction cannot prematurely reopen it.
    clock.monotonic += 299
    assert "pubmed" in tracker.disabled()
    clock.monotonic += 1
    assert tracker.disabled() == {}
    assert events.call_args.args[2]["status"] == "retry_available"
    assert events.call_count == 2
    assert tracker.disabled() == {}
    assert events.call_count == 2


def test_configuration_failure_is_local_to_research(tracker, clock):
    state = tracker.disable("searxng", SearchFailure("URL blocked", None))
    clock.monotonic += 1_000_000
    assert tracker.disabled()["searxng"] == state
    assert "rest of this research" in state.describe()
    assert state.retry_at is None
    assert SearchEngineAvailability().disabled() == {}


def test_concurrent_disablement_is_reported_once(clock):
    events = Mock()
    tracker = SearchEngineAvailability(events)
    with ThreadPoolExecutor(max_workers=4) as pool:
        states = list(
            pool.map(
                lambda _: tracker.disable("searxng", SearchFailure("offline")),
                range(20),
            )
        )
    assert len(set(states)) == 1
    assert events.call_count == 1


@pytest.mark.parametrize(
    "value", [None, "invalid", "nan", "inf", -1, 0, True, "1e100", 10**500]
)
def test_invalid_retry_after_ignored(value):
    assert retry_after_seconds(value) is None


def test_retry_after_seconds_and_http_date(clock):
    assert retry_after_seconds("120") == 120
    date = format_datetime(
        datetime.fromtimestamp(clock.wall + 180, UTC), usegmt=True
    )
    assert retry_after_seconds(date, now=clock.wall) == 180


@pytest.mark.parametrize(
    "error,reason,delay",
    [
        (RateLimitError("secret", retry_after=90), "Rate limit", 90),
        (requests.Timeout("secret"), "timed out", 60),
        (requests.ConnectionError("secret"), "connect", 60),
        (requests.exceptions.ChunkedEncodingError("secret"), "read", 60),
        (requests.exceptions.ContentDecodingError("secret"), "read", 60),
        # The validator also refuses unresolvable hosts, so a transient DNS
        # failure must not disable the engine for the whole research.
        (ValueError("URL failed security validation: secret"), "blocked", 300),
    ],
)
def test_classification_uses_safe_reasons(error, reason, delay):
    failure = failure_from_exception(error)
    assert reason in failure.reason
    assert "secret" not in failure.reason
    assert failure.cooldown_seconds == delay


@pytest.mark.parametrize(
    "status,delay",
    [(401, None), (402, None), (403, 480), (429, 480), (503, 480), (500, 480)],
)
def test_http_failure_classification_preserves_retry_after(status, delay):
    response = requests.Response()
    response.status_code = status
    response.headers["Retry-After"] = "480"
    failure = failure_from_exception(
        requests.HTTPError(
            "https://user:secret@provider/?api_key=secret", response=response
        )
    )
    assert "secret" not in failure.reason
    assert failure.cooldown_seconds == delay


@pytest.mark.parametrize(
    "body,headers",
    [
        (b'{"error": "Invalid API key"}', {}),
        (b"Unauthorized: subscription expired", {}),
        (b"", {"WWW-Authenticate": "Bearer"}),
    ],
)
def test_403_that_is_clearly_about_credentials_lasts_the_run(body, headers):
    response = requests.Response()
    response.status_code = 403
    response._content = body
    response.headers.update(headers)
    failure = failure_from_exception(requests.HTTPError(response=response))
    assert failure.cooldown_seconds is None
    assert "HTTP 403" in failure.reason


def test_403_block_page_is_a_cooldown_not_a_run_long_disable():
    response = requests.Response()
    response.status_code = 403
    response._content = b"<html>Access denied: automated traffic</html>"
    # The request URL may name api_key; only the response body counts.
    error = requests.HTTPError("https://p/?api_key=x", response=response)
    assert failure_from_exception(error).cooldown_seconds == 300


def test_one_hour_cooldown_is_kept(tracker, clock):
    state = tracker.disable(
        "pubmed", SearchFailure("Rate limit", COOLDOWN_WAIT_BUDGET_SECONDS)
    )
    assert state.retry_at - state.disabled_at == COOLDOWN_WAIT_BUDGET_SECONDS
    clock.monotonic += COOLDOWN_WAIT_BUDGET_SECONDS
    assert tracker.disabled() == {}


def test_provider_retry_after_beyond_the_wait_budget_stays_temporary(
    tracker, clock
):
    state = tracker.disable("pubmed", SearchFailure("Rate limit", 7200))
    assert state.retry_at == clock.wall + 7200
    assert "Rate limit" in state.failure.reason
    clock.monotonic += 7200
    assert tracker.disabled() == {}


def test_absurd_retry_after_disables_for_the_run_with_a_clear_reason(
    tracker, clock
):
    """An HTTP date years away is no cooldown to show or plan around."""
    date = format_datetime(
        datetime(9998, 12, 31, 23, 59, 59, tzinfo=UTC), usegmt=True
    )
    delay = retry_after_seconds(date, now=clock.wall)
    assert delay > 10**11  # parseable, so it reaches the tracker as-is
    state = tracker.disable(
        "pubmed", SearchFailure("Rate limit reached", delay)
    )
    assert state.retry_at is None
    assert "Rate limit reached" in state.failure.reason
    assert (
        f"asked to wait longer than {MAX_TEMPORARY_COOLDOWN_SECONDS}s"
        in state.failure.reason
    )
    described = state.describe()
    assert "for the rest of this research" in described
    assert "9998" not in described
    clock.monotonic += 10 * 365 * 86_400
    assert "pubmed" in tracker.disabled()


def test_provider_error_with_an_absurd_retry_after_disables_for_the_run(
    tracker,
):
    failure = failure_from_exception(RateLimitError("x", retry_after=10**9))
    assert failure.cooldown_seconds == 10**9
    assert tracker.disable("pubmed", failure).retry_at is None


@pytest.mark.parametrize(
    "budget,delay,temporary",
    [
        (COOLDOWN_WAIT_BUDGET_SECONDS, MAX_TEMPORARY_COOLDOWN_SECONDS, True),
        (
            COOLDOWN_WAIT_BUDGET_SECONDS,
            MAX_TEMPORARY_COOLDOWN_SECONDS + 1,
            False,
        ),
        # A zero budget means "never wait", not "every cooldown is run-long".
        (0, 7200, True),
        # A budget above the limit raises it: whatever a waiter could wait
        # for must stay a cooldown.
        (3 * 86_400, 2 * 86_400, True),
        (3 * 86_400, 3 * 86_400, True),
        (3 * 86_400, 3 * 86_400 + 1, False),
    ],
)
def test_longest_temporary_cooldown_follows_the_wait_budget(
    clock, budget, delay, temporary
):
    tracker = SearchEngineAvailability(
        clock=lambda: clock.monotonic,
        wall_clock=lambda: clock.wall,
        wait_budget_seconds=budget,
    )
    state = tracker.disable("pubmed", SearchFailure("Rate limit", delay))
    assert (state.retry_at is not None) is temporary
    if temporary:
        assert state.retry_at - state.disabled_at == delay
    else:
        assert "asked to wait longer than" in state.failure.reason


def test_default_does_not_disable_after_three_temporary_failures(clock):
    tracker = SearchEngineAvailability(
        clock=lambda: clock.monotonic, wall_clock=lambda: clock.wall
    )
    for _ in range(4):
        state = tracker.disable("searxng", SearchFailure("Rate limit", 60))
        assert state.retry_at is not None
        clock.monotonic += 60
        assert tracker.disabled() == {}


def test_opt_in_consecutive_cooldown_limit(clock):
    tracker = SearchEngineAvailability(
        clock=lambda: clock.monotonic,
        wall_clock=lambda: clock.wall,
        max_consecutive_cooldowns=3,
    )
    for _ in range(3):
        state = tracker.disable("searxng", SearchFailure("Rate limit", 300))
        assert state.retry_at is not None
        clock.monotonic += 300
        assert tracker.disabled() == {}
    state = tracker.disable("searxng", SearchFailure("Rate limit", 300))
    assert state.retry_at is None
    assert "still failing after 3 cooldowns in a row" in state.failure.reason


def test_successful_search_resets_the_cooldown_streak(clock):
    tracker = SearchEngineAvailability(
        clock=lambda: clock.monotonic,
        wall_clock=lambda: clock.wall,
        max_consecutive_cooldowns=2,
    )
    for _ in range(5):
        state = tracker.disable("searxng", SearchFailure("offline", 60))
        assert state.retry_at is not None
        clock.monotonic += 60
        tracker.disabled()
        tracker.record_success("searxng")
    # Streaks are per engine.
    tracker.disable("pubmed", SearchFailure("offline", 60))
    clock.monotonic += 60
    tracker.disabled()
    tracker.disable("pubmed", SearchFailure("offline", 60))
    clock.monotonic += 60
    tracker.disabled()
    assert (
        tracker.disable("pubmed", SearchFailure("offline", 60)).retry_at is None
    )
    assert (
        tracker.disable("searxng", SearchFailure("offline", 60)).retry_at
        is not None
    )


def test_wait_budget_counts_overlapping_waiters_once(clock):
    tracker = SearchEngineAvailability(
        clock=lambda: clock.monotonic,
        wall_clock=lambda: clock.wall,
        wait_budget_seconds=900,
    )
    with tracker.waiting():
        with tracker.waiting():
            clock.monotonic += 100
        assert tracker.wait_budget_remaining() == 800
        clock.monotonic += 100
    clock.monotonic += 1000  # time not spent waiting is free
    assert tracker.wait_budget_remaining() == 700


def test_end_cooldowns_disables_only_cooling_engines(tracker):
    tracker.disable("searxng", SearchFailure("Rate limit", 300))
    tracker.disable("pubmed", SearchFailure("Bad key", None))
    tracker.end_cooldowns({"searxng", "pubmed", "arxiv"})
    disabled = tracker.disabled()
    assert disabled["searxng"].retry_at is None
    assert "Rate limit" in disabled["searxng"].failure.reason
    assert "wait budget" in disabled["searxng"].failure.reason
    assert disabled["pubmed"].failure.reason == "Bad key"
    assert "arxiv" not in disabled


def test_unknown_error_and_empty_engine_are_not_assumed_rate_limited():
    assert failure_from_exception(ValueError("bad query")) is None
    assert engine_failure(SimpleNamespace()) is None


class FailingEngine(BaseSearchEngine):
    def _get_previews(self, query):
        raise RateLimitError("test", retry_after=120)


@pytest.mark.parametrize("retry_enabled,attempts", [(True, 3), (False, 1)])
def test_swallowed_rate_limit_survives_base_retry_exhaustion(
    retry_enabled, attempts
):
    engine = FailingEngine(settings_snapshot={}, programmatic_mode=True)
    engine.rate_tracker = Mock(enabled=retry_enabled)
    engine.rate_tracker.get_wait_time.return_value = 0
    engine._get_previews = Mock(
        side_effect=RateLimitError("test", retry_after=120)
    )
    assert engine.run("query") == []
    assert engine._get_previews.call_count == attempts
    assert engine.last_search_failure.cooldown_seconds == 120


def test_retry_success_clears_failure_signal():
    engine = FailingEngine(settings_snapshot={}, programmatic_mode=True)
    engine.rate_tracker = Mock(enabled=True)
    engine.rate_tracker.get_wait_time.return_value = 0
    engine.last_search_failure = SearchFailure("old failure")
    engine._get_previews = Mock(
        side_effect=[RateLimitError("test"), [{"title": "result"}]]
    )
    assert engine.run("query") == [{"title": "result"}]
    assert engine.last_search_failure is None


def test_new_failure_replaces_expired_disablement_without_a_prior_poll(
    tracker, clock
):
    first = tracker.disable("pubmed", SearchFailure("old failure", 60))
    clock.monotonic += 60
    clock.wall += 60
    second = tracker.disable("pubmed", SearchFailure("new failure", 120))
    assert second != first
    assert second.failure.reason == "new failure"
    assert second.retry_at == clock.wall + 120
    assert tracker.disabled()["pubmed"] == second


@pytest.mark.parametrize("engine", [None, ""])
def test_missing_engine_name_is_never_recorded(tracker, engine):
    events = Mock()
    tracker._on_change = events
    state = tracker.disable(engine, SearchFailure("blocked", None))
    assert state.describe() == "Search failed: blocked"
    assert tracker.disabled() == {}
    events.assert_not_called()
    names = ["fetch_content", "research_subtopic", "search_pubmed"]
    # Even a stray None key must not hide non-search tools, which map to
    # None in engine_for_tool.
    assert available_tool_names(names, None, {None: object()}) == names


def _exhausted(status):
    """Drive a real HTTPAdapter + Retry until urllib3 gives up on ``status``.

    Only the pool's per-attempt request is stubbed, so urllib3's own retry
    loop (status_forcelist, increment, MaxRetryError) runs unmodified.
    """
    import urllib3
    from requests.adapters import HTTPAdapter
    from urllib3.connectionpool import HTTPConnectionPool
    from urllib3.util.retry import Retry

    session = requests.Session()
    session.mount(
        "http://",
        HTTPAdapter(
            max_retries=Retry(
                total=2,
                backoff_factor=0,
                status_forcelist=[429, 500, 502, 503, 504],
                allowed_methods={"GET"},
            )
        ),
    )

    def respond(self, conn, method, url, **kwargs):
        return urllib3.HTTPResponse(
            body=b"", status=status, headers={}, preload_content=False
        )

    with patch.object(HTTPConnectionPool, "_make_request", respond):
        with pytest.raises(requests.exceptions.RetryError) as raised:
            session.get("http://provider.invalid/search", timeout=1)
    session.close()
    return raised.value


def test_exhausted_429_retries_are_a_rate_limit():
    failure = failure_from_exception(_exhausted(429))
    assert failure.reason == "Rate limit reached"
    assert failure.cooldown_seconds == 300


def test_exhausted_5xx_retries_stay_generic():
    assert failure_from_exception(_exhausted(503)) is None


def test_seconds_until_retry_reports_soonest_finite_cooldown(tracker, clock):
    tracker.disable("pubmed", SearchFailure("offline", 90))
    tracker.disable("arxiv", SearchFailure("offline", 30))
    tracker.disable("searxng", SearchFailure("blocked", None))
    assert tracker.seconds_until_retry({"pubmed", "arxiv"}) == 30
    assert tracker.seconds_until_retry({"searxng"}) is None
    assert tracker.seconds_until_retry({"wikipedia"}) is None
    clock.monotonic += 40
    assert tracker.seconds_until_retry({"pubmed", "arxiv"}) == 0


def _http_error(status):
    response = requests.Response()
    response.status_code = status
    return requests.HTTPError(response=response)


@pytest.mark.parametrize("status", [400, 414, 422])
def test_query_specific_http_error_is_not_an_engine_failure(status):
    engine = FailingEngine(settings_snapshot={}, programmatic_mode=True)
    engine._record_search_failure(_http_error(status))
    assert engine.last_search_failure is None
    engine._record_search_failure(_http_error(500))
    assert engine.last_search_failure is not None


class _PreviewEngine(BaseSearchEngine):
    def _get_previews(self, query):
        # One follow-up request failed; the search itself answered.
        self._record_search_failure(requests.Timeout())
        return [{"title": "a"}, {"title": "b"}]

    def _get_full_content(self, items):
        return items


def test_results_dropped_by_the_relevance_filter_are_not_a_failure():
    engine = _PreviewEngine(settings_snapshot={}, programmatic_mode=True)
    engine.rate_tracker = Mock(enabled=False)
    engine.enable_llm_relevance_filter = True
    engine.llm = Mock()
    with patch.object(engine, "_filter_for_relevance", return_value=[]):
        assert engine.run("query") == []
    assert engine.last_search_failure is None


def test_error_after_previews_arrived_is_not_an_engine_failure():
    engine = _PreviewEngine(settings_snapshot={}, programmatic_mode=True)
    engine.rate_tracker = Mock(enabled=False)
    engine._content_filters = [Mock()]
    engine._content_filters[0].filter_results.side_effect = KeyError("x")
    assert engine.run("query") == []
    assert engine.last_search_failure is None


def test_error_before_previews_is_still_an_engine_failure():
    engine = _PreviewEngine(settings_snapshot={}, programmatic_mode=True)
    engine.rate_tracker = Mock(enabled=False)
    engine._get_previews = Mock(side_effect=requests.ConnectionError())
    assert engine.run("query") == []
    assert "connect" in engine.last_search_failure.reason


@pytest.mark.parametrize("fetch_name", ["fetch_content", "fetch_archive"])
def test_delegation_is_not_offered_when_only_fetch_remains(fetch_name):
    names = [
        "web_search",
        "search_pubmed",
        fetch_name,
        "research_subtopic",
    ]
    disabled = {"searxng": object(), "pubmed": object()}
    assert available_tool_names(names, "searxng", disabled) == [fetch_name]
    # While any search tool works, delegation stays.
    assert available_tool_names(names, "searxng", {"pubmed": object()}) == [
        "web_search",
        fetch_name,
        "research_subtopic",
    ]
    # A toolbox configured without search is not availability's business.
    assert available_tool_names(
        [fetch_name, "research_subtopic"], "searxng", disabled
    ) == [fetch_name, "research_subtopic"]


def test_run_long_failure_upgrades_an_active_cooldown(tracker, clock):
    events = Mock()
    tracker._on_change = events
    tracker.disable("pubmed", SearchFailure("offline", 60))
    # A search already in flight comes back with HTTP 401.
    upgraded = tracker.disable("pubmed", SearchFailure("Denied", None))
    assert upgraded.retry_at is None
    assert upgraded.failure.reason == "Denied"
    assert events.call_count == 2
    clock.monotonic += 3600
    assert tracker.disabled()["pubmed"] == upgraded
    # A cooldown never downgrades a run-long disable.
    assert tracker.disable("pubmed", SearchFailure("offline", 60)) == upgraded
    assert events.call_count == 2