Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

23.5 KB
Raw Blame
"""Research-scoped search engine availability, independent of agent frameworks.

Empty results are not failures. Engines report an explicit ``SearchFailure``
when a request cannot work; callers decide when to apply it (after retries and
fallbacks have finished). One tracker is shared by a research run's workers,
never by unrelated users or runs. No settings or credentials are persisted.
"""

from __future__ import annotations

import math
import re
import threading
import time
from contextlib import contextmanager
from dataclasses import dataclass
from datetime import UTC, datetime
from email.utils import parsedate_to_datetime
from typing import Callable

import requests
from urllib3.exceptions import ResponseError

from ..security.secure_logging import logger
from .rate_limiting.exceptions import RateLimitError

# Total time one research run (one availability tracker) spends waiting for
# cooldowns. Once it is used up, cooling engines are disabled for the rest
# of the run instead of being waited for again, so a persistently failing
# sole engine cannot keep a run waiting indefinitely. Users can adjust this
# per research run with langgraph_agent.cooldown_wait_budget_seconds.
COOLDOWN_WAIT_BUDGET_SECONDS = 3600

# Longest provider Retry-After kept as a temporary cooldown; a larger wait
# budget raises this limit. A longer delay is not one this research can plan
# around (an HTTP date years away parses to centuries of seconds), so the
# engine is disabled for the rest of the run with that reason, instead of
# being shown as "temporary" with an absurd retry time.
MAX_TEMPORARY_COOLDOWN_SECONDS = 24 * 60 * 60

# HTTP 403 is often a temporary bot/WAF block rather than bad credentials;
# unless the reply says it is about authentication, cool down this long.
FORBIDDEN_COOLDOWN_SECONDS = 300

# Words in a 403 reply that mark it as an authentication or billing refusal.
_AUTH_MARKERS = re.compile(
    r"api[ _-]?key|access[ _-]?token|invalid[ _-]?token|unauthori[sz]ed|"
    r"authenticat|credential|subscription|billing|payment",
    re.IGNORECASE,
)


@dataclass(frozen=True)
class SearchFailure:
    """Safe, actionable reason and cooldown; None means the rest of this run."""

    reason: str
    cooldown_seconds: float | None = 60


class SearchEngineError(RuntimeError):
    """A provider-specific failure normalized before its original data is lost."""

    def __init__(self, failure: SearchFailure):
        self.failure = failure
        super().__init__(failure.reason)


class SearchEngineInitializationError(SearchEngineError):
    """A factory failure with a safe reason and its original retry policy."""


class SearchQueryError(ValueError):
    """A rejected query that leaves the provider available for other queries."""


def retry_after_seconds(value, *, now: float | None = None) -> float | None:
    """Parse HTTP Retry-After (seconds or HTTP date), ignoring invalid values."""
    if not isinstance(value, (str, int, float)) or isinstance(value, bool):
        return None
    try:
        seconds = float(value)
    except OverflowError:
        return None
    except (ValueError, TypeError):
        if not isinstance(value, str):
            return None
        try:
            seconds = parsedate_to_datetime(value).timestamp() - (
                time.time() if now is None else now
            )
        except (ValueError, TypeError, OverflowError):
            return None
    if not math.isfinite(seconds) or seconds <= 0:
        return None
    # Provider headers are untrusted; an absurd finite delta must not break
    # timestamp formatting in every subsequent progress/model update.
    try:
        datetime.fromtimestamp(
            (time.time() if now is None else now) + seconds, UTC
        )
    except (OverflowError, OSError, ValueError):
        return None
    return seconds


def retry_after_from_error(error: object) -> float | None:
    """Read retry timing from responses, translated errors and SDK metadata."""
    delay = retry_after_seconds(getattr(error, "retry_after", None))
    if delay is not None:
        return delay
    for source in (
        getattr(error, "response", None),
        error,
        getattr(error, "meta", None),
    ):
        headers = getattr(source, "headers", None) or {}
        delay = retry_after_seconds(headers.get("Retry-After"))
        if delay is not None:
            return delay
    return None


# HTTP statuses caused by the request's query (malformed syntax, too long,
# unprocessable), not by the provider being down or refusing this client.
QUERY_ERROR_STATUSES = frozenset({400, 414, 422})


def http_status(error: object) -> int | None:
    """Read the status exposed by HTTP responses and common client SDKs."""
    status = getattr(getattr(error, "response", None), "status_code", None)
    if isinstance(status, int):
        return status
    for attr in ("status_code", "status", "code"):
        status = getattr(error, attr, None)
        if isinstance(status, int):
            return status
    # Some HTTP wrappers discard the response and raise this exact message.
    # Restrict the fallback to that complete shape, never a query or URL.
    if isinstance(error, Exception):
        match = re.fullmatch(
            r"HTTP error ([1-5][0-9]{2})", str(error), re.IGNORECASE
        )
        if match:
            return int(match.group(1))
    return None


def is_query_error(error: BaseException) -> bool:
    """True when the failure is specific to this query, not to the engine.

    Such an error must not cool the engine down for every other search in
    the run; a different query can still succeed right away.
    """
    return (
        isinstance(error, SearchQueryError)
        or http_status(error) in QUERY_ERROR_STATUSES
    )


def failure_from_exception(
    error: BaseException, *, bare_403_is_bad_key: bool = False
) -> SearchFailure | None:
    """Classify explicit failures without putting provider text/secrets in UI.

    ``bare_403_is_bad_key``: the failed request carried an API key to a
    provider known to reject an invalid key with a plain HTTP 403 (no
    ``WWW-Authenticate``, nothing telling in the body), so any 403 from it
    is treated as denied credentials. Other 403s stay a temporary block
    unless the reply itself says it is about credentials.
    """
    if isinstance(error, SearchEngineError):
        return error.failure
    # A provider adapter may recreate an exception to redact its message.
    # Preserve classified health without retaining its request/response data.
    recorded_failure = getattr(error, "_ldr_search_failure", None)
    if isinstance(recorded_failure, SearchFailure):
        return recorded_failure
    retry_delay = retry_after_from_error(error)
    status = http_status(error)
    if isinstance(error, RateLimitError) or status == 429:
        return SearchFailure("Rate limit reached", retry_delay or 300)
    if status in (401, 402) or (
        status == 403 and (bare_403_is_bad_key or _is_auth_refusal(error))
    ):
        return SearchFailure(
            f"Provider denied access (HTTP {status}); check credentials, permissions or quota",
            None,
        )
    if status == 403:
        return SearchFailure(
            "Provider refused the request (HTTP 403); this is often a "
            "temporary block",
            retry_delay or FORBIDDEN_COOLDOWN_SECONDS,
        )
    if isinstance(status, int) and status >= 500:
        return SearchFailure(
            f"Provider unavailable (HTTP {status})",
            retry_delay or 60,
        )
    if _retries_exhausted_on_429(error):
        # urllib3 already retried (honoring Retry-After) and discarded the
        # response, so no provider delay is left to preserve.
        return SearchFailure("Rate limit reached", retry_delay or 300)
    if isinstance(error, (requests.Timeout, TimeoutError)):
        return SearchFailure("Search request timed out", retry_delay or 60)
    if isinstance(error, (requests.ConnectionError, ConnectionError)):
        return SearchFailure(
            "Could not connect to the search provider", retry_delay or 60
        )
    if isinstance(
        error,
        (
            requests.exceptions.ChunkedEncodingError,
            requests.exceptions.ContentDecodingError,
        ),
    ):
        return SearchFailure(
            "Could not read the provider response", retry_delay or 60
        )
    if isinstance(error, ValueError) and "security validation" in str(error):
        # The SSRF validator also refuses a host it cannot resolve, and
        # its ValueError does not say which. A transient DNS failure must
        # not take an engine out for the whole run, so use a finite
        # cooldown. A retry is not free: validation resolves the host, and
        # the redirect-hop form of this error is raised only after the
        # provider request was sent. Engines that can identify a policy
        # block (SearXNG's unapproved private instance URL) report a
        # run-long SearchFailure themselves.
        return SearchFailure(
            "Request URL blocked by security validation or its host could "
            "not be resolved; check server configuration and DNS",
            retry_delay or 300,
        )
    if retry_delay is not None:
        return SearchFailure(
            "Search request failed; provider supplied a retry delay",
            retry_delay,
        )
    return None


def _is_auth_refusal(error: BaseException) -> bool:
    """Whether a 403 reply says it is about credentials, not a block.

    Reads only the response (never the exception text, whose request URL
    can carry ``api_key=``) and never surfaces what it reads.
    """
    response = getattr(error, "response", None)
    headers = getattr(response, "headers", None) or {}
    try:
        if headers.get("WWW-Authenticate"):
            return True
        text = getattr(response, "text", "")
    except Exception:
        return False
    return isinstance(text, str) and bool(_AUTH_MARKERS.search(text[:4000]))


def _retries_exhausted_on_429(error: BaseException) -> bool:
    """A mounted ``Retry(status_forcelist=[429, ...])`` gave up on HTTP 429.

    requests raises ``RetryError(MaxRetryError(reason=ResponseError(...)))``
    without a response; only the reason text names the status. Exhausted
    retries on other statuses stay generic transient failures.
    """
    if not isinstance(error, requests.exceptions.RetryError):
        return False
    cause = error.args[0] if error.args else None
    reason = getattr(cause, "reason", None)
    return isinstance(reason, ResponseError) and str(
        reason
    ) == ResponseError.SPECIFIC_ERROR.format(status_code=429)


@dataclass(frozen=True)
class UnrecordedFailure:
    """A failure that could not be attributed to an engine; not tracked."""

    failure: SearchFailure

    def describe(self, name: str | None = None) -> str:
        return f"{name or 'Search'} failed: {self.failure.reason}"


@dataclass(frozen=True)
class EngineDisablement:
    engine: str
    failure: SearchFailure
    disabled_at: float
    retry_at: float | None
    deadline: float | None

    def describe(self, name: str | None = None) -> str:
        start = datetime.fromtimestamp(self.disabled_at, UTC).strftime(
            "%Y-%m-%d %H:%M:%S UTC"
        )
        if self.retry_at is None:
            timing = "for the rest of this research; fix configuration before the next run"
        else:
            end = datetime.fromtimestamp(self.retry_at, UTC).strftime(
                "%Y-%m-%d %H:%M:%S UTC"
            )
            duration = math.ceil(self.retry_at - self.disabled_at)
            timing = f"for {duration}s; retry available at {end}"
        return (
            f"{name or self.engine} disabled since {start} {timing}: "
            f"{self.failure.reason}"
        )


class SearchEngineAvailability:
    """Thread-safe circuit breaker with lazy cooldown expiry and change events.

    No lock is held while a search runs: parallel searches, including the
    first calls to an engine and retries after a cooldown, run concurrently
    and record their outcome under the tracker's lock. The first failure
    wins; later failures of the same outage return the existing state. A
    failure blocks new calls; work already in flight may still return
    usable results. Expiry permits a real retry, not a claim that the
    provider has recovered.
    """

    def __init__(
        self,
        on_change: Callable | None = None,
        *,
        clock: Callable[[], float] = time.monotonic,
        wall_clock: Callable[[], float] = time.time,
        check_cancelled: Callable[[], None] | None = None,
        wait_budget_seconds: float = COOLDOWN_WAIT_BUDGET_SECONDS,
        max_consecutive_cooldowns: int | None = None,
    ):
        self._clock = clock
        self._wall_clock = wall_clock
        self._on_change = on_change
        # Raises (e.g. ResearchTerminatedException) when the run is
        # cancelled; cooldown waits call it before every sleep. In a research
        # run the call that sees the cancel first also clears the run's
        # single-use termination flag while cleaning up; see note_cancelled.
        self.check_cancelled = check_cancelled
        # Set once any thread sharing this tracker has seen the run cancelled.
        self._cancelled = threading.Event()
        self._lock = threading.RLock()
        self._disabled: dict[str, EngineDisablement] = {}
        self._metrics_names: dict[str, str] = {}
        self._wait_budget = wait_budget_seconds
        self._max_consecutive_cooldowns = max_consecutive_cooldowns
        self._consecutive_cooldowns: dict[str, int] = {}
        # Wall time with at least one waiter, so parallel workers waiting
        # for the same cooldown spend the budget once, not once each.
        self._waited = 0.0
        self._waiters = 0
        self._waiting_since = 0.0

    def _emit(self, state: EngineDisablement, retry: bool = False):
        message = (
            f"{state.engine}: cooldown ended; available for retry"
            if retry
            else state.describe()
        )
        if retry:
            logger.info(message)
        else:
            logger.warning(message)
        if self._on_change:
            try:
                self._on_change(
                    message,
                    None,
                    {
                        "phase": "engine_availability",
                        "type": "info" if retry else "warning",
                        "engine": state.engine,
                        "status": "retry_available" if retry else "disabled",
                        "reason": state.failure.reason,
                        "disabled_at": state.disabled_at,
                        "retry_at": state.retry_at,
                    },
                )
            except Exception:
                logger.exception("Could not report search engine availability")

    def disable(
        self, engine: str, failure: SearchFailure
    ) -> EngineDisablement | UnrecordedFailure:
        if not engine:
            # A missing engine name maps to every non-search tool in
            # engine_for_tool; recording it would hide fetch/delegation.
            logger.warning("Ignoring availability failure without an engine")
            return UnrecordedFailure(failure)
        with self._lock:
            current = self._disabled.get(engine)
            if (
                current is not None
                and (
                    current.deadline is None or self._clock() < current.deadline
                )
                # A run-long failure (e.g. HTTP 401 from a search already
                # in flight) upgrades a cooldown instead of being dropped.
                and not (
                    failure.cooldown_seconds is None
                    and current.deadline is not None
                )
            ):
                return current
            now = self._wall_clock()
            delay = failure.cooldown_seconds
            if delay is not None:
                delay = retry_after_seconds(delay, now=now) or 60
                count = self._consecutive_cooldowns.get(engine, 0) + 1
                longest = max(self._wait_budget, MAX_TEMPORARY_COOLDOWN_SECONDS)
                if delay > longest:
                    failure = SearchFailure(
                        f"{failure.reason}; the provider asked to wait "
                        f"longer than {math.ceil(longest)}s",
                        None,
                    )
                    delay = None
                elif (
                    self._max_consecutive_cooldowns is not None
                    and count > self._max_consecutive_cooldowns
                ):
                    failure = SearchFailure(
                        f"{failure.reason}; still failing after "
                        f"{count - 1} cooldowns in a row",
                        None,
                    )
                    delay = None
                else:
                    self._consecutive_cooldowns[engine] = count
            state = EngineDisablement(
                engine,
                failure,
                now,
                None if delay is None else now + delay,
                None if delay is None else self._clock() + delay,
            )
            self._disabled[engine] = state
        self._emit(state)
        return state

    def disabled(self) -> dict[str, EngineDisablement]:
        expired = []
        with self._lock:
            now = self._clock()
            for engine, state in list(self._disabled.items()):
                if state.deadline is not None and now >= state.deadline:
                    expired.append(self._disabled.pop(engine))
            states = dict(self._disabled)
        for state in expired:
            self._emit(state, retry=True)
        return states

    def note_cancelled(self) -> None:
        """Record that this run was cancelled, for every thread sharing it.

        The run's termination flag is single-use: the first check to see it
        clears it while cleaning up, and so does the cleanup after the lead
        agent stops. In the LangGraph strategy that check is the research
        progress callback, which records every cancel it raises, on any
        thread; drain loops, subagent workers and cooldown waits also record
        the cancels they see. Parallel drain loops, cooldown waits and
        running subagents read this record, so they stop as well.
        """
        self._cancelled.set()

    def cancelled(self) -> bool:
        """Whether a cancel has been recorded (``note_cancelled``)."""
        return self._cancelled.is_set()

    def now(self) -> float:
        """Current time on the clock that cooldown deadlines use."""
        return self._clock()

    def record_success(self, engine: str) -> None:
        """The engine answered a search; its cooldown streak starts over."""
        with self._lock:
            self._consecutive_cooldowns.pop(engine, None)

    @contextmanager
    def waiting(self):
        """Charge the time spent inside to the run's cooldown-wait budget."""
        with self._lock:
            if self._waiters == 0:
                self._waiting_since = self._clock()
            self._waiters += 1
        try:
            yield
        finally:
            with self._lock:
                self._waiters -= 1
                if self._waiters == 0:
                    self._waited += self._clock() - self._waiting_since

    def wait_budget_remaining(self) -> float:
        """Seconds of cooldown waiting this run may still spend."""
        with self._lock:
            waited = self._waited
            if self._waiters:
                waited += self._clock() - self._waiting_since
            return max(0.0, self._wait_budget - waited)

    def end_cooldowns(self, engines) -> None:
        """Disable ``engines`` still cooling down for the rest of the run.

        Used once the run's cooldown-wait budget is spent: waiting again
        is no longer an option, so the usual "no research tools" handling
        applies instead of an endless wait-and-retry cycle.
        """
        for engine in engines:
            state = self.state(engine)
            if state is not None and state.deadline is not None:
                self.disable(
                    engine,
                    SearchFailure(
                        f"{state.failure.reason}; cooldown exceeds this "
                        "research's remaining search wait budget",
                        None,
                    ),
                )

    def seconds_until_retry(self, engines) -> float | None:
        """Seconds until the soonest of ``engines`` leaves its cooldown.

        Only finite cooldowns count; None when none of them has one.
        """
        with self._lock:
            now = self._clock()
            remaining = [
                state.deadline - now
                for engine, state in self._disabled.items()
                if engine in engines and state.deadline is not None
            ]
        return max(0.0, min(remaining)) if remaining else None

    def note_metrics_name(self, engine: str, name: str) -> None:
        """Remember the name search metrics record for ``engine``.

        Metrics name engines after their class (``semanticscholar``), not
        their configuration key (``semantic_scholar``). Skipped searches are
        recorded under the same name so they group with real ones.
        """
        with self._lock:
            self._metrics_names[engine] = name

    def metrics_name(self, engine: str) -> str:
        with self._lock:
            return self._metrics_names.get(engine, engine)

    def state(self, engine: str) -> EngineDisablement | None:
        """The engine's current disablement, or None when it may be searched."""
        return self.disabled().get(engine)


def engine_failure(engine) -> SearchFailure | None:
    """Read explicit engine health, without mistaking empty results for failure."""
    failure = getattr(engine, "last_search_failure", None)
    if isinstance(failure, SearchFailure):
        return failure
    if getattr(engine, "_is_available", None) is False:
        return SearchFailure("Search engine failed its availability check")
    return None


def engine_for_tool(name: str, primary_engine: str) -> str | None:
    if name == "fetch_archive":
        return "wayback"
    if name == "web_search":
        return primary_engine
    if name.startswith("search_"):
        return name.removeprefix("search_")
    return None


def search_engines_for_tools(names, primary_engine) -> set[str]:
    """Providers that discover URLs; archive fetching only reads known URLs."""
    engines = {
        engine_for_tool(name, primary_engine)
        for name in names
        if name == "web_search" or name.startswith("search_")
    }
    return {engine for engine in engines if engine is not None}


def search_exhausted(names, primary_engine, disabled) -> bool:
    """The toolbox had query search tools, and every one is disabled."""
    engines = search_engines_for_tools(names, primary_engine)
    return bool(engines) and engines <= disabled.keys()


def available_tool_names(names, primary_engine, disabled) -> list[str]:
    """Filter an already policy-approved toolbox, including empty delegation.

    Delegation is dropped once availability has removed every search tool:
    its subagents would be left with fetch tools alone and no URLs
    to read. (A toolbox configured without search is left as it is.)
    """
    available = [
        name
        for name in names
        if (engine := engine_for_tool(name, primary_engine)) is None
        or engine not in disabled
    ]
    if search_exhausted(names, primary_engine, disabled):
        available = [name for name in available if name != "research_subtopic"]
    if available == ["research_subtopic"]:
        return []
    return available