~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""Availability transitions use a fake clock, never network requests or sleeps."""
from concurrent.futures import ThreadPoolExecutor
from datetime import UTC, datetime
from email.utils import format_datetime
from types import SimpleNamespace
from unittest.mock import Mock, patch
import pytest
import requests
from local_deep_research.web_search_engines.engine_availability import (
COOLDOWN_WAIT_BUDGET_SECONDS,
MAX_TEMPORARY_COOLDOWN_SECONDS,
SearchEngineAvailability,
SearchFailure,
available_tool_names,
engine_failure,
failure_from_exception,
retry_after_seconds,
)
from local_deep_research.web_search_engines.rate_limiting import RateLimitError
from local_deep_research.web_search_engines.search_engine_base import (
BaseSearchEngine,
)
@pytest.fixture
def clock():
return SimpleNamespace(monotonic=100.0, wall=1_790_000_000.0)
@pytest.fixture
def tracker(clock):
return SearchEngineAvailability(
clock=lambda: clock.monotonic, wall_clock=lambda: clock.wall
)
def test_cooldown_logs_one_transition_and_expires_on_monotonic_time(clock):
events = Mock()
tracker = SearchEngineAvailability(
events, clock=lambda: clock.monotonic, wall_clock=lambda: clock.wall
)
first = tracker.disable("pubmed", SearchFailure("Rate limit reached", 300))
assert tracker.disable("pubmed", SearchFailure("duplicate", 900)) == first
assert events.call_count == 1
message, _, meta = events.call_args.args
assert "for 300s" in message and "retry available at" in message
assert meta["disabled_at"] == clock.wall
assert meta["retry_at"] == clock.wall + 300
clock.wall += 86_400 # Wall-clock correction cannot prematurely reopen it.
clock.monotonic += 299
assert "pubmed" in tracker.disabled()
clock.monotonic += 1
assert tracker.disabled() == {}
assert events.call_args.args[2]["status"] == "retry_available"
assert events.call_count == 2
assert tracker.disabled() == {}
assert events.call_count == 2
def test_configuration_failure_is_local_to_research(tracker, clock):
state = tracker.disable("searxng", SearchFailure("URL blocked", None))
clock.monotonic += 1_000_000
assert tracker.disabled()["searxng"] == state
assert "rest of this research" in state.describe()
assert state.retry_at is None
assert SearchEngineAvailability().disabled() == {}
def test_concurrent_disablement_is_reported_once(clock):
events = Mock()
tracker = SearchEngineAvailability(events)
with ThreadPoolExecutor(max_workers=4) as pool:
states = list(
pool.map(
lambda _: tracker.disable("searxng", SearchFailure("offline")),
range(20),
)
)
assert len(set(states)) == 1
assert events.call_count == 1
@pytest.mark.parametrize(
"value", [None, "invalid", "nan", "inf", -1, 0, True, "1e100", 10**500]
)
def test_invalid_retry_after_ignored(value):
assert retry_after_seconds(value) is None
def test_retry_after_seconds_and_http_date(clock):
assert retry_after_seconds("120") == 120
date = format_datetime(
datetime.fromtimestamp(clock.wall + 180, UTC), usegmt=True
)
assert retry_after_seconds(date, now=clock.wall) == 180
@pytest.mark.parametrize(
"error,reason,delay",
[
(RateLimitError("secret", retry_after=90), "Rate limit", 90),
(requests.Timeout("secret"), "timed out", 60),
(requests.ConnectionError("secret"), "connect", 60),
(requests.exceptions.ChunkedEncodingError("secret"), "read", 60),
(requests.exceptions.ContentDecodingError("secret"), "read", 60),
# The validator also refuses unresolvable hosts, so a transient DNS
# failure must not disable the engine for the whole research.
(ValueError("URL failed security validation: secret"), "blocked", 300),
],
)
def test_classification_uses_safe_reasons(error, reason, delay):
failure = failure_from_exception(error)
assert reason in failure.reason
assert "secret" not in failure.reason
assert failure.cooldown_seconds == delay
@pytest.mark.parametrize(
"status,delay",
[(401, None), (402, None), (403, 480), (429, 480), (503, 480), (500, 480)],
)
def test_http_failure_classification_preserves_retry_after(status, delay):
response = requests.Response()
response.status_code = status
response.headers["Retry-After"] = "480"
failure = failure_from_exception(
requests.HTTPError(
"https://user:secret@provider/?api_key=secret", response=response
)
)
assert "secret" not in failure.reason
assert failure.cooldown_seconds == delay
@pytest.mark.parametrize(
"body,headers",
[
(b'{"error": "Invalid API key"}', {}),
(b"Unauthorized: subscription expired", {}),
(b"", {"WWW-Authenticate": "Bearer"}),
],
)
def test_403_that_is_clearly_about_credentials_lasts_the_run(body, headers):
response = requests.Response()
response.status_code = 403
response._content = body
response.headers.update(headers)
failure = failure_from_exception(requests.HTTPError(response=response))
assert failure.cooldown_seconds is None
assert "HTTP 403" in failure.reason
def test_403_block_page_is_a_cooldown_not_a_run_long_disable():
response = requests.Response()
response.status_code = 403
response._content = b"<html>Access denied: automated traffic</html>"
# The request URL may name api_key; only the response body counts.
error = requests.HTTPError("https://p/?api_key=x", response=response)
assert failure_from_exception(error).cooldown_seconds == 300
def test_one_hour_cooldown_is_kept(tracker, clock):
state = tracker.disable(
"pubmed", SearchFailure("Rate limit", COOLDOWN_WAIT_BUDGET_SECONDS)
)
assert state.retry_at - state.disabled_at == COOLDOWN_WAIT_BUDGET_SECONDS
clock.monotonic += COOLDOWN_WAIT_BUDGET_SECONDS
assert tracker.disabled() == {}
def test_provider_retry_after_beyond_the_wait_budget_stays_temporary(
tracker, clock
):
state = tracker.disable("pubmed", SearchFailure("Rate limit", 7200))
assert state.retry_at == clock.wall + 7200
assert "Rate limit" in state.failure.reason
clock.monotonic += 7200
assert tracker.disabled() == {}
def test_absurd_retry_after_disables_for_the_run_with_a_clear_reason(
tracker, clock
):
"""An HTTP date years away is no cooldown to show or plan around."""
date = format_datetime(
datetime(9998, 12, 31, 23, 59, 59, tzinfo=UTC), usegmt=True
)
delay = retry_after_seconds(date, now=clock.wall)
assert delay > 10**11 # parseable, so it reaches the tracker as-is
state = tracker.disable(
"pubmed", SearchFailure("Rate limit reached", delay)
)
assert state.retry_at is None
assert "Rate limit reached" in state.failure.reason
assert (
f"asked to wait longer than {MAX_TEMPORARY_COOLDOWN_SECONDS}s"
in state.failure.reason
)
described = state.describe()
assert "for the rest of this research" in described
assert "9998" not in described
clock.monotonic += 10 * 365 * 86_400
assert "pubmed" in tracker.disabled()
def test_provider_error_with_an_absurd_retry_after_disables_for_the_run(
tracker,
):
failure = failure_from_exception(RateLimitError("x", retry_after=10**9))
assert failure.cooldown_seconds == 10**9
assert tracker.disable("pubmed", failure).retry_at is None
@pytest.mark.parametrize(
"budget,delay,temporary",
[
(COOLDOWN_WAIT_BUDGET_SECONDS, MAX_TEMPORARY_COOLDOWN_SECONDS, True),
(
COOLDOWN_WAIT_BUDGET_SECONDS,
MAX_TEMPORARY_COOLDOWN_SECONDS + 1,
False,
),
# A zero budget means "never wait", not "every cooldown is run-long".
(0, 7200, True),
# A budget above the limit raises it: whatever a waiter could wait
# for must stay a cooldown.
(3 * 86_400, 2 * 86_400, True),
(3 * 86_400, 3 * 86_400, True),
(3 * 86_400, 3 * 86_400 + 1, False),
],
)
def test_longest_temporary_cooldown_follows_the_wait_budget(
clock, budget, delay, temporary
):
tracker = SearchEngineAvailability(
clock=lambda: clock.monotonic,
wall_clock=lambda: clock.wall,
wait_budget_seconds=budget,
)
state = tracker.disable("pubmed", SearchFailure("Rate limit", delay))
assert (state.retry_at is not None) is temporary
if temporary:
assert state.retry_at - state.disabled_at == delay
else:
assert "asked to wait longer than" in state.failure.reason
def test_default_does_not_disable_after_three_temporary_failures(clock):
tracker = SearchEngineAvailability(
clock=lambda: clock.monotonic, wall_clock=lambda: clock.wall
)
for _ in range(4):
state = tracker.disable("searxng", SearchFailure("Rate limit", 60))
assert state.retry_at is not None
clock.monotonic += 60
assert tracker.disabled() == {}
def test_opt_in_consecutive_cooldown_limit(clock):
tracker = SearchEngineAvailability(
clock=lambda: clock.monotonic,
wall_clock=lambda: clock.wall,
max_consecutive_cooldowns=3,
)
for _ in range(3):
state = tracker.disable("searxng", SearchFailure("Rate limit", 300))
assert state.retry_at is not None
clock.monotonic += 300
assert tracker.disabled() == {}
state = tracker.disable("searxng", SearchFailure("Rate limit", 300))
assert state.retry_at is None
assert "still failing after 3 cooldowns in a row" in state.failure.reason
def test_successful_search_resets_the_cooldown_streak(clock):
tracker = SearchEngineAvailability(
clock=lambda: clock.monotonic,
wall_clock=lambda: clock.wall,
max_consecutive_cooldowns=2,
)
for _ in range(5):
state = tracker.disable("searxng", SearchFailure("offline", 60))
assert state.retry_at is not None
clock.monotonic += 60
tracker.disabled()
tracker.record_success("searxng")
# Streaks are per engine.
tracker.disable("pubmed", SearchFailure("offline", 60))
clock.monotonic += 60
tracker.disabled()
tracker.disable("pubmed", SearchFailure("offline", 60))
clock.monotonic += 60
tracker.disabled()
assert (
tracker.disable("pubmed", SearchFailure("offline", 60)).retry_at is None
)
assert (
tracker.disable("searxng", SearchFailure("offline", 60)).retry_at
is not None
)
def test_wait_budget_counts_overlapping_waiters_once(clock):
tracker = SearchEngineAvailability(
clock=lambda: clock.monotonic,
wall_clock=lambda: clock.wall,
wait_budget_seconds=900,
)
with tracker.waiting():
with tracker.waiting():
clock.monotonic += 100
assert tracker.wait_budget_remaining() == 800
clock.monotonic += 100
clock.monotonic += 1000 # time not spent waiting is free
assert tracker.wait_budget_remaining() == 700
def test_end_cooldowns_disables_only_cooling_engines(tracker):
tracker.disable("searxng", SearchFailure("Rate limit", 300))
tracker.disable("pubmed", SearchFailure("Bad key", None))
tracker.end_cooldowns({"searxng", "pubmed", "arxiv"})
disabled = tracker.disabled()
assert disabled["searxng"].retry_at is None
assert "Rate limit" in disabled["searxng"].failure.reason
assert "wait budget" in disabled["searxng"].failure.reason
assert disabled["pubmed"].failure.reason == "Bad key"
assert "arxiv" not in disabled
def test_unknown_error_and_empty_engine_are_not_assumed_rate_limited():
assert failure_from_exception(ValueError("bad query")) is None
assert engine_failure(SimpleNamespace()) is None
class FailingEngine(BaseSearchEngine):
def _get_previews(self, query):
raise RateLimitError("test", retry_after=120)
@pytest.mark.parametrize("retry_enabled,attempts", [(True, 3), (False, 1)])
def test_swallowed_rate_limit_survives_base_retry_exhaustion(
retry_enabled, attempts
):
engine = FailingEngine(settings_snapshot={}, programmatic_mode=True)
engine.rate_tracker = Mock(enabled=retry_enabled)
engine.rate_tracker.get_wait_time.return_value = 0
engine._get_previews = Mock(
side_effect=RateLimitError("test", retry_after=120)
)
assert engine.run("query") == []
assert engine._get_previews.call_count == attempts
assert engine.last_search_failure.cooldown_seconds == 120
def test_retry_success_clears_failure_signal():
engine = FailingEngine(settings_snapshot={}, programmatic_mode=True)
engine.rate_tracker = Mock(enabled=True)
engine.rate_tracker.get_wait_time.return_value = 0
engine.last_search_failure = SearchFailure("old failure")
engine._get_previews = Mock(
side_effect=[RateLimitError("test"), [{"title": "result"}]]
)
assert engine.run("query") == [{"title": "result"}]
assert engine.last_search_failure is None
def test_new_failure_replaces_expired_disablement_without_a_prior_poll(
tracker, clock
):
first = tracker.disable("pubmed", SearchFailure("old failure", 60))
clock.monotonic += 60
clock.wall += 60
second = tracker.disable("pubmed", SearchFailure("new failure", 120))
assert second != first
assert second.failure.reason == "new failure"
assert second.retry_at == clock.wall + 120
assert tracker.disabled()["pubmed"] == second
@pytest.mark.parametrize("engine", [None, ""])
def test_missing_engine_name_is_never_recorded(tracker, engine):
events = Mock()
tracker._on_change = events
state = tracker.disable(engine, SearchFailure("blocked", None))
assert state.describe() == "Search failed: blocked"
assert tracker.disabled() == {}
events.assert_not_called()
names = ["fetch_content", "research_subtopic", "search_pubmed"]
# Even a stray None key must not hide non-search tools, which map to
# None in engine_for_tool.
assert available_tool_names(names, None, {None: object()}) == names
def _exhausted(status):
"""Drive a real HTTPAdapter + Retry until urllib3 gives up on ``status``.
Only the pool's per-attempt request is stubbed, so urllib3's own retry
loop (status_forcelist, increment, MaxRetryError) runs unmodified.
"""
import urllib3
from requests.adapters import HTTPAdapter
from urllib3.connectionpool import HTTPConnectionPool
from urllib3.util.retry import Retry
session = requests.Session()
session.mount(
"http://",
HTTPAdapter(
max_retries=Retry(
total=2,
backoff_factor=0,
status_forcelist=[429, 500, 502, 503, 504],
allowed_methods={"GET"},
)
),
)
def respond(self, conn, method, url, **kwargs):
return urllib3.HTTPResponse(
body=b"", status=status, headers={}, preload_content=False
)
with patch.object(HTTPConnectionPool, "_make_request", respond):
with pytest.raises(requests.exceptions.RetryError) as raised:
session.get("http://provider.invalid/search", timeout=1)
session.close()
return raised.value
def test_exhausted_429_retries_are_a_rate_limit():
failure = failure_from_exception(_exhausted(429))
assert failure.reason == "Rate limit reached"
assert failure.cooldown_seconds == 300
def test_exhausted_5xx_retries_stay_generic():
assert failure_from_exception(_exhausted(503)) is None
def test_seconds_until_retry_reports_soonest_finite_cooldown(tracker, clock):
tracker.disable("pubmed", SearchFailure("offline", 90))
tracker.disable("arxiv", SearchFailure("offline", 30))
tracker.disable("searxng", SearchFailure("blocked", None))
assert tracker.seconds_until_retry({"pubmed", "arxiv"}) == 30
assert tracker.seconds_until_retry({"searxng"}) is None
assert tracker.seconds_until_retry({"wikipedia"}) is None
clock.monotonic += 40
assert tracker.seconds_until_retry({"pubmed", "arxiv"}) == 0
def _http_error(status):
response = requests.Response()
response.status_code = status
return requests.HTTPError(response=response)
@pytest.mark.parametrize("status", [400, 414, 422])
def test_query_specific_http_error_is_not_an_engine_failure(status):
engine = FailingEngine(settings_snapshot={}, programmatic_mode=True)
engine._record_search_failure(_http_error(status))
assert engine.last_search_failure is None
engine._record_search_failure(_http_error(500))
assert engine.last_search_failure is not None
class _PreviewEngine(BaseSearchEngine):
def _get_previews(self, query):
# One follow-up request failed; the search itself answered.
self._record_search_failure(requests.Timeout())
return [{"title": "a"}, {"title": "b"}]
def _get_full_content(self, items):
return items
def test_results_dropped_by_the_relevance_filter_are_not_a_failure():
engine = _PreviewEngine(settings_snapshot={}, programmatic_mode=True)
engine.rate_tracker = Mock(enabled=False)
engine.enable_llm_relevance_filter = True
engine.llm = Mock()
with patch.object(engine, "_filter_for_relevance", return_value=[]):
assert engine.run("query") == []
assert engine.last_search_failure is None
def test_error_after_previews_arrived_is_not_an_engine_failure():
engine = _PreviewEngine(settings_snapshot={}, programmatic_mode=True)
engine.rate_tracker = Mock(enabled=False)
engine._content_filters = [Mock()]
engine._content_filters[0].filter_results.side_effect = KeyError("x")
assert engine.run("query") == []
assert engine.last_search_failure is None
def test_error_before_previews_is_still_an_engine_failure():
engine = _PreviewEngine(settings_snapshot={}, programmatic_mode=True)
engine.rate_tracker = Mock(enabled=False)
engine._get_previews = Mock(side_effect=requests.ConnectionError())
assert engine.run("query") == []
assert "connect" in engine.last_search_failure.reason
@pytest.mark.parametrize("fetch_name", ["fetch_content", "fetch_archive"])
def test_delegation_is_not_offered_when_only_fetch_remains(fetch_name):
names = [
"web_search",
"search_pubmed",
fetch_name,
"research_subtopic",
]
disabled = {"searxng": object(), "pubmed": object()}
assert available_tool_names(names, "searxng", disabled) == [fetch_name]
# While any search tool works, delegation stays.
assert available_tool_names(names, "searxng", {"pubmed": object()}) == [
"web_search",
fetch_name,
"research_subtopic",
]
# A toolbox configured without search is not availability's business.
assert available_tool_names(
[fetch_name, "research_subtopic"], "searxng", disabled
) == [fetch_name, "research_subtopic"]
def test_run_long_failure_upgrades_an_active_cooldown(tracker, clock):
events = Mock()
tracker._on_change = events
tracker.disable("pubmed", SearchFailure("offline", 60))
# A search already in flight comes back with HTTP 401.
upgraded = tracker.disable("pubmed", SearchFailure("Denied", None))
assert upgraded.retry_at is None
assert upgraded.failure.reason == "Denied"
assert events.call_count == 2
clock.monotonic += 3600
assert tracker.disabled()["pubmed"] == upgraded
# A cooldown never downgrades a run-long disable.
assert tracker.disable("pubmed", SearchFailure("offline", 60)) == upgraded
assert events.call_count == 2