Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

41.0 KB
Raw Blame
"""Cross-field egress-policy validators run at settings-save time.

Each validator takes the about-to-be-saved ``form_data`` plus the current
``all_db_settings`` (key -> setting row with a ``.value``) and returns a
validation-error dict ``{"key", "error"}`` to surface in the save response,
or ``None`` when the combination is fine. Most are pure; the engine-URL
guard machinery here (allowlist, operator-approval resolver, descriptor
sweep) additionally reads operator environment state — never the DB — and
the allowlist parser logs dropped entries once per distinct env value.

The settings write routes (``web/routers/settings.py``) orchestrate these
— they stay in the route layer because they also enforce non-egress concerns —
but the egress rules themselves live here next to the policy they encode.
"""

import functools
import ipaddress
from dataclasses import dataclass
from typing import Optional, Tuple
from urllib.parse import unquote, urlsplit

from ...utilities.type_utils import unwrap_setting
from .policy import (
    EgressContext,
    EgressScope,
    PolicyDeniedError,
    _canonical_host,
    _classify_host,
    _is_metadata_host,
    _resolve_with_timeout,
    parse_user_egress_scope,
)

# The SearXNG instance URL is the canonical exploitable case and is always
# guarded even if dynamic engine-class enumeration fails below.
_SEARXNG_INSTANCE_URL_KEY = (
    "search.engine.web.searxng.default_params.instance_url"
)


def validate_egress_scope(form_data, _all_db_settings):
    """Validate the scope and enforce the operator-controlled escape hatch."""
    key = "policy.egress_scope"
    if key not in form_data:
        return None
    value = form_data[key]
    if not isinstance(value, str):
        return {"key": key, "error": "Egress scope must be a string"}
    try:
        parse_user_egress_scope(value)
    except PolicyDeniedError as exc:
        if exc.decision.reason == "unprotected_egress_disabled":
            return {
                "key": key,
                "error": "Unprotected egress is disabled by the server operator",
            }
        return {
            "key": key,
            "error": "Unknown egress scope",
        }
    return None


def validate_allowed_local_hostnames(form_data, all_db_settings):
    """Reject entries in llm.allowed_local_hostnames that would not, or must
    not, make a host local.

    The default-settings description for this key claims "Public hostnames
    added here are rejected at save time", but until now no code actually
    did the rejection. Each entry is judged exactly as host classification
    reads it (``policy._canonical_host``: decoded once, case folded, one
    trailing dot ignored), and refused when it

    * contains ``%`` (percent-encoding or an IPv6 zone ID): no hostname
      needs it, and refusing it keeps the saved text and the name it
      matches the same (classification reads a stored entry decoded once);
    * is not a plain DNS name or a canonical IP address (non-ASCII
      characters, whitespace or control characters, an integer / hex /
      octal / short IPv4 notation) -- classification ignores such an entry,
      so accepting it would only hide that it has no effect;
    * names a cloud-metadata endpoint (``policy._is_metadata_host``), which
      can never be local; or
    * resolves to public addresses, via the same host classifier the policy
      uses. A hostname that fails to resolve (DNS down) is accepted --
      fail-open on transient lookup errors so the user can recover.

    Only a changed value is validated: one equal to the stored value,
    entry for entry, is accepted unchecked
    (``_allowed_local_hostnames_unchanged``). Never raises: a missing
    ``form_data`` (None) has nothing to validate, and an unexpected error is
    reported as a validation error (fail closed) rather than escaping into
    the save route.
    """
    key = "llm.allowed_local_hostnames"
    try:
        if not form_data or key not in form_data:
            return None
        value = form_data[key]
        # Validate-on-change, as for the engine URLs: the All-Settings tab
        # resubmits every setting, so an unchanged list (say, one holding an
        # entry saved before this validator became stricter) must not block
        # an unrelated save. Runtime classification ignores such an entry
        # anyway. A changed list is validated in full.
        if _allowed_local_hostnames_unchanged(value, all_db_settings, key):
            return None
        return _check_allowed_local_hostnames(key, value)
    except Exception:
        return {
            "key": key,
            "error": (
                "allowed_local_hostnames could not be validated; check that "
                "it is a JSON array of plain hostnames or IP addresses"
            ),
        }


def _allowed_local_hostnames_entries(value):
    """The entries of an ``llm.allowed_local_hostnames`` value (a list, or
    its JSON text), or None when it is neither."""
    import json as _json

    value = unwrap_setting(value)
    if isinstance(value, str):
        try:
            value = _json.loads(value) if value.strip() else []
        except ValueError:
            return None
    return value if isinstance(value, list) else None


def _allowed_local_hostnames_unchanged(submitted, all_db_settings, key):
    """True when the submitted ``llm.allowed_local_hostnames`` value holds
    exactly the stored entries, in the same order (a list and its JSON text
    compare equal). Fails toward validating: no stored value, or a value on
    either side that is not a list, counts as changed."""
    stored = _stored_engine_url_value(all_db_settings, key)
    if stored is _NO_STORED_VALUE:
        return False
    entries = _allowed_local_hostnames_entries(submitted)
    return entries is not None and entries == _allowed_local_hostnames_entries(
        stored
    )


def _display_entry(entry: str) -> str:
    """An entry as quoted in an error message: ``repr`` shows whitespace and
    control characters a plain rendering would hide, bounded in length."""
    text = repr(entry)
    return text if len(text) <= 80 else text[:77] + "..."


def _check_allowed_local_hostnames(key, value):
    # Setting is JSON-typed; the save pipeline may hand us a list or a
    # JSON string. Decode defensively.
    if isinstance(value, str):
        try:
            import json as _json

            decoded = _json.loads(value) if value.strip() else []
        except Exception:
            return {
                "key": key,
                "error": "allowed_local_hostnames must be a JSON array of hostnames",
            }
        value = decoded
    if not isinstance(value, list):
        return {
            "key": key,
            "error": "allowed_local_hostnames must be a list",
        }

    # Build a minimal real context just for the resolver. Use the
    # dataclass constructor — NOT EgressContext.__new__ + setattr, which
    # raised FrozenInstanceError on this frozen dataclass and (separately)
    # set a non-existent ``allowed_local_hostnames`` field instead of the
    # real ``local_hostnames`` the classifier reads. The constructor
    # initializes the init=False internals (_dns_cache, _lock as RLock)
    # correctly. Empty local_hostnames => classify purely on IP class.
    probe_ctx = EgressContext(
        scope=EgressScope.BOTH,
        primary_engine="searxng",
        require_local_llm=False,
        require_local_embeddings=False,
        local_hostnames=(),
    )

    rejected = []
    metadata = []
    invalid = []
    for entry in value:
        if not isinstance(entry, str) or not entry.strip():
            continue
        # Judged exactly as classification reads the entry, as written:
        # _canonical_host decodes it once and folds case itself. An entry
        # with "%" is refused before that, so a saved entry is always the
        # text it matches.
        hostname = None if "%" in entry else _canonical_host(entry)
        if hostname is None:
            invalid.append(_display_entry(entry))
            continue
        # A cloud-metadata name or address can never be declared local
        # (classification ignores such an entry), whatever it resolves to.
        # Report it as what it is rather than as a public address: IBM
        # Cloud classic's metadata name resolves to an RFC1918 address.
        if _is_metadata_host(hostname):
            metadata.append(hostname)
            continue
        try:
            # Distinguish "could not resolve" from "resolved to a public IP".
            # _classify_host collapses BOTH to False (its documented fail-safe
            # treats an unresolvable host as public), so relying on it here
            # would reject a legitimate intranet/VPN host on any DNS hiccup or
            # split-horizon DNS — the exact use case this setting exists for.
            # Only reject names that actually resolve to a public address;
            # accept unresolvable ones (fail-open on save, as documented).
            # _resolve_with_timeout returns the addrinfo for literal IPs too,
            # so literal public/private IPs still flow through _classify_host.
            if _resolve_with_timeout(hostname) is None:
                continue
            classification = _classify_host(hostname, probe_ctx)
        except Exception:
            # DNS or unknown error — allow (fail open) so the user can
            # save when networking is flaky. Runtime classification will
            # still gate egress.
            continue
        if classification is False:
            rejected.append(hostname)
    errors = []
    if invalid:
        errors.append(
            "These entries are not valid hostnames for this list: "
            f"{', '.join(invalid)}. Only plain DNS names (ASCII letters, "
            "digits, hyphens, underscores and dots, written without "
            "percent-encoding) and standard IP addresses (dotted-quad IPv4, "
            "or IPv6 without a zone ID) are accepted; any other spelling "
            "would be ignored. Write the name in that form (an "
            "internationalised name in its xn-- form) or remove it."
        )
    if metadata:
        errors.append(
            "These entries are cloud-metadata endpoints, which are blocked "
            "under every scope and can never be treated as local: "
            f"{', '.join(metadata)}. Remove them."
        )
    if rejected:
        errors.append(
            "These hostnames resolve to PUBLIC addresses and would "
            "let the policy treat external hosts as local: "
            f"{', '.join(rejected)}. Remove them, or use the SSRF "
            "allowlist instead."
        )
    if errors:
        return {"key": key, "error": " ".join(errors)}
    return None


def validate_trusted_search_engines(form_data, all_db_settings):
    """Reject inherently-public engines from ``policy.trusted_search_engines``.

    Trusting a search engine relaxes its Exposure to CONTAINED — meaningful only
    for a local-nature store you host (Elasticsearch/Paperless). An inherently
    public engine (google, searxng, brave, …) genuinely queries the internet, so
    trusting it would launder a public sink past the two-axis rule. Reject those;
    local / unknown names are accepted (runtime classification still gates them,
    and ``engine_label`` also refuses to relax an ``is_public`` engine).
    """
    key = "policy.trusted_search_engines"
    if key not in form_data:
        return None
    value = form_data[key]
    if isinstance(value, str):
        import json

        try:
            value = json.loads(value) if value.strip() else []
        except Exception:
            return {
                "key": key,
                "error": "trusted_search_engines must be a JSON array",
            }
    if not isinstance(value, list):
        return {"key": key, "error": "trusted_search_engines must be a list"}

    from .policy import _get_engine_class

    rejected = []
    for entry in value:
        if not isinstance(entry, str) or not entry.strip():
            continue
        name = entry.strip().lower()
        cls = _get_engine_class(name)
        if cls is not None and getattr(cls, "is_public", False) is True:
            rejected.append(name)
    if rejected:
        return {
            "key": key,
            "error": (
                "These are inherently-public search engines and cannot be "
                "trusted as contained: " + ", ".join(rejected) + "."
            ),
        }
    return None


@dataclass(frozen=True)
class GuardedEngineUrl:
    """A search engine whose configurable URL participates in the egress
    URL guards, derived from the engine class's own declarations.

    ``is_public`` True → the engine proxies the public internet, so a
    PRIVATE URL is the anomaly (save-time rejection + runtime refusal +
    "private URL not approved" banner). ``is_public`` False → a LOCAL
    document engine, where a PUBLIC-looking URL is the anomaly (advisory
    banner only — a hosted endpoint can be legitimate).

    Per-engine UI conventions derived from ``engine_name``:
    display name setting ``search.engine.web.{name}.display_name``,
    activation flags ``.enabled`` / ``.use_in_auto_search`` /
    ``.agent_enabled``, and dismiss keys
    ``app.warnings.dismiss_{name}_private_url`` (public) /
    ``app.warnings.dismiss_{name}_public_url`` (local) — each dismiss key
    must be registered in ``defaults/default_settings.json`` (pinned by a
    registry-contract test). A future public engine that declares
    ``url_setting`` must also call ``resolve_engine_allow_private_ips``
    on its fetch path, as SearXNG does.
    """

    engine_name: str
    url_setting: str
    is_public: bool


@functools.lru_cache(maxsize=1)
def guarded_engine_url_descriptors() -> Tuple[GuardedEngineUrl, ...]:
    """All engines participating in the URL guards, from the registry.

    Derived once from ``ENGINE_REGISTRY`` + engine-class declarations
    (``url_setting`` plus ``is_public``/``is_local``); seeded with SearXNG
    so the canonical guard survives an engine-class import failure.

    COST NOTE: the first call imports every engine module in the registry
    (~1-3 s cold) to read the class declarations; results are cached for
    the process lifetime (``.cache_clear()`` resets, for tests). Callers on
    render paths pay this once per process.
    """
    found = {
        "searxng": GuardedEngineUrl(
            engine_name="searxng",
            url_setting=_SEARXNG_INSTANCE_URL_KEY,
            is_public=True,
        )
    }
    try:
        from ...web_search_engines.engine_registry import ENGINE_REGISTRY
        from .policy import _get_engine_class

        for name in ENGINE_REGISTRY:
            cls = _get_engine_class(name)
            if cls is None:
                continue
            url_setting = getattr(cls, "url_setting", None)
            if not (isinstance(url_setting, str) and url_setting):
                continue
            if getattr(cls, "is_public", None) is True:
                found[name] = GuardedEngineUrl(
                    engine_name=name, url_setting=url_setting, is_public=True
                )
            elif getattr(cls, "is_local", None) is True:
                found[name] = GuardedEngineUrl(
                    engine_name=name, url_setting=url_setting, is_public=False
                )
    except Exception:  # noqa: silent-exception - seed still guards SearXNG
        pass
    return tuple(found[name] for name in sorted(found))


@functools.lru_cache(maxsize=1)
def _public_engine_url_settings() -> frozenset:
    """URL-setting keys belonging to PUBLIC-nature search engines.

    These are the engine URLs a save-time SSRF check must guard: a public
    engine (SearXNG) proxies the public web, so a private/loopback
    ``instance_url`` is only ever an internal-network probe target, never a
    legitimate data source. LOCAL engines (Elasticsearch, Paperless) are
    deliberately EXCLUDED — a private URL is their whole purpose, and the PDP's
    fail-up URL override already reclassifies them public when pointed at an
    external host.

    Delegates to ``guarded_engine_url_descriptors`` so the save-time guard
    and the banner/orchestrator machinery can never disagree about which
    engines are covered (single registry sweep, single seed).
    """
    return frozenset(
        d.url_setting for d in guarded_engine_url_descriptors() if d.is_public
    )


# Env-only operator allowlist of exact private engine-URL origins — the
# finer-grained alternative to the blanket search.allow_private_engine_urls
# gate (see env_definitions/security.py for the operator-facing contract).
_ALLOWLIST_SETTING_KEY = "search.private_engine_url_allowlist"

_DEFAULT_PORTS = {"http": 80, "https": 443}


def _engine_url_origin(value):
    """Parse ``value`` into a comparable ``(scheme, host, port)`` origin.

    Normalization mirrors what actually denotes the same network
    destination and nothing more: scheme and host are lowercased, the host
    is stripped of a trailing dot, an IP-literal host is canonicalized
    through :mod:`ipaddress` (so ``[::1]`` and its long form compare
    equal), and a missing port takes the scheme default (80/443). Path,
    query, fragment, and userinfo are ignored — they don't change the
    destination host.

    Two deliberate refusals keep this parser from ever disagreeing with
    the HTTP client (urllib3) about which host a hostile URL names:
    URLs containing RFC-forbidden characters (backslash, whitespace,
    control chars — the parser-differential class) return ``None``, and
    the host is NOT percent-decoded here (urllib3 before 2.8 did not
    decode http(s) hosts and 2.8+ does, so this parser stays decode-free:
    an escaped host matches only an allowlist entry written with the same
    escapes, never a plain entry by decoding, and ``validate_url`` refuses
    percent-encoded hosts at fetch time). Returns ``None`` for anything
    that isn't a cleanly parseable
    http(s) URL with a hostname; every failure mode therefore fails
    CLOSED (no allowlist match), never open.
    """
    try:
        text = str(value).strip()
        from ..ssrf_validator import RFC_FORBIDDEN_URL_CHARS_RE

        if RFC_FORBIDDEN_URL_CHARS_RE.search(text):
            return None
        parsed = urlsplit(text)
        scheme = (parsed.scheme or "").lower()
        if scheme not in _DEFAULT_PORTS:
            return None
        host = parsed.hostname
        if not host:
            return None
        host = host.rstrip(".").lower()
        if not host:
            return None
        try:
            host = str(ipaddress.ip_address(host))
        except ValueError:
            pass  # not an IP literal — keep the hostname text
        port = parsed.port
        if port is None:
            port = _DEFAULT_PORTS[scheme]
        return (scheme, host, port)
    except Exception:
        return None


def engine_url_origin_text(origin) -> str:
    """Render an ``_engine_url_origin`` tuple back to allowlist-entry text.

    The inverse of ``_engine_url_origin`` for display: IPv6 hosts are
    re-bracketed and a scheme-default port is omitted, so the returned
    string, fed into ``LDR_SEARCH_PRIVATE_ENGINE_URL_ALLOWLIST``, is
    guaranteed to match the origin it was rendered from (round-trip by
    construction — pinned by the banner remedy tests). Lives next to the
    parser so the pair can't drift apart.
    """
    scheme, host, port = origin
    host_disp = f"[{host}]" if ":" in host else host
    suffix = "" if port == _DEFAULT_PORTS[scheme] else f":{port}"
    return f"{scheme}://{host_disp}{suffix}"


# Raw allowlist value we last warned about — the dropped-entry WARNING is
# emitted once per distinct env value, not on every consult (the matcher
# runs on every settings save, engine init, and /api/warnings recompute).
_last_warned_allowlist_raw: Optional[str] = None


def _private_engine_url_allowlist() -> frozenset:
    """Operator-approved private engine-URL origins from the environment.

    Reads the env-only ``search.private_engine_url_allowlist`` setting and
    parses each comma-separated entry with ``_engine_url_origin``.
    Unparseable entries are dropped (fail-closed) and warned about once per
    distinct env value. Any lookup error yields the empty set — the safe
    posture.
    """
    try:
        from ...settings.env_registry import get_env_setting

        raw = get_env_setting(_ALLOWLIST_SETTING_KEY, None)
    except Exception:  # noqa: silent-exception - default to the safe posture
        return frozenset()
    if not raw or not isinstance(raw, str):
        return frozenset()
    origins = set()
    dropped = []
    for entry in raw.split(","):
        entry = entry.strip()
        if not entry:
            continue
        origin = _engine_url_origin(entry)
        if origin is not None:
            origins.add(origin)
        else:
            dropped.append(entry)
    if dropped:
        # An unparseable entry grants nothing (fail-closed), but silently
        # dropping it leaves the operator staring at the same rejection
        # message telling them to add the origin they think they already
        # added — so name what was ignored (once per distinct env value,
        # to keep a permanently-malformed entry from spamming the logs).
        global _last_warned_allowlist_raw
        if raw != _last_warned_allowlist_raw:
            _last_warned_allowlist_raw = raw
            try:
                from ..secure_logging import logger

                logger.warning(
                    "Ignoring unparseable entries in "
                    "LDR_SEARCH_PRIVATE_ENGINE_URL_ALLOWLIST (each entry "
                    "must be a full origin like http://localhost:8080): "
                    f"{dropped}"
                )
            except Exception:  # noqa: silent-exception - best-effort logging
                pass
    return frozenset(origins)


def engine_url_in_private_allowlist(url) -> bool:
    """True when ``url``'s origin is in the operator's env-only allowlist.

    Grants ONLY the private/loopback/link-local exemption a matching origin
    was listed for — callers consult it strictly in place of the blanket
    ``search.allow_private_engine_urls`` gate, so the cloud-metadata block
    and the http(s)-scheme requirement are unaffected by a listing.
    """
    origin = _engine_url_origin(url)
    if origin is None:
        return False
    return origin in _private_engine_url_allowlist()


def resolve_engine_allow_private_ips(
    instance_url: str, url_setting: str
) -> bool:
    """Whether a PUBLIC engine may target a private/loopback instance URL.

    A public-nature engine proxies the public web, so a private instance
    URL is only legitimate when an OPERATOR chose it, never when an
    arbitrary authenticated user smuggled an internal host in as an SSRF
    probe. Private egress to the configured instance is therefore
    permitted only when:

      1. the operator opt-in ``search.allow_private_engine_urls`` is set
         (env-only; the blanket LAN / localhost self-host case), OR
      2. ``instance_url``'s exact origin is listed in the env-only operator
         allowlist ``search.private_engine_url_allowlist`` (the
         finer-grained alternative to the blanket opt-in), OR
      3. the engine's own ``url_setting`` is env-locked (its derived
         ``LDR_…`` variable is set — Docker / operator-provisioned,
         trusted).

    All three are OPERATOR-controlled. The run's egress scope
    (``policy.egress_scope``) deliberately does NOT grant it: that setting is a
    user-editable dropdown (STRICT included), so relaxing on it would be a
    self-service SSRF bypass. By default private IPs are refused, so a
    tampered instance URL cannot reach an internal host. Cloud-metadata /
    link-local metadata IPs stay blocked regardless
    (ALWAYS_BLOCKED_METADATA_IPS in the SSRF validator's probe path). Fails
    CLOSED (False) on any error: a genuinely-public instance is unaffected —
    only private egress is withheld.
    """
    # 1. Operator opt-in (LAN self-host).
    try:
        from ...settings.env_registry import get_env_setting

        if bool(get_env_setting("search.allow_private_engine_urls", False)):
            return True
    except Exception:  # noqa: silent-exception - fall through to next check
        pass

    # 2. Operator allowlist: this exact origin was individually approved.
    try:
        if engine_url_in_private_allowlist(instance_url):
            return True
    except Exception:  # noqa: silent-exception - fall through to next check
        pass

    # 3. Env-locked instance URL (Docker / operator-provisioned).
    try:
        from ...settings.manager import check_env_setting

        if check_env_setting(url_setting) is not None:
            return True
    except Exception:  # noqa: silent-exception - fall through to next check
        pass

    # Otherwise: withhold private egress. We deliberately DO NOT relax this
    # based on the run's resolved egress scope. ``policy.egress_scope`` is a
    # user-editable, self-service setting (STRICT is a plain dropdown option,
    # not operator-gated), and a public engine always queries the internet.
    # Letting a user-chosen scope grant private-IP access would be a
    # self-service SSRF bypass (set scope=strict, then point the instance
    # URL at an internal host). Private egress for a public engine is an
    # OPERATOR decision only — conditions 1–3 above.
    return False


def resolve_searxng_allow_private_ips(instance_url: str) -> bool:
    """SearXNG-keyed convenience wrapper over
    ``resolve_engine_allow_private_ips`` (kept for the engine and tests)."""
    return resolve_engine_allow_private_ips(
        instance_url, _SEARXNG_INSTANCE_URL_KEY
    )


def resolve_engine_allow_private_result_fetch(
    instance_url: str, url_setting: str
) -> bool:
    """Whether full-content fetches of a PUBLIC engine's RESULT URLs may
    reach private / loopback hosts.

    Distinct from, and strictly narrower than,
    ``resolve_engine_allow_private_ips``: that grant answers "may this
    engine reach its configured *instance* URL", and its env-locked-URL
    condition is met by every Docker (compose) deployment. A public engine
    proxies the public web, so extending the instance grant to the URLs
    the engine *returns* would let a poisoned index turn result fetching
    into SSRF against the operator's LAN (loopback included). This
    returns True (the engine holds the grant) only when BOTH hold:

      1. the operator set the env-only opt-in
         ``search.allow_private_result_fetch``
         (``LDR_SEARCH_ALLOW_PRIVATE_RESULT_FETCH=true``), which the
         bundled ``docker-compose.yml`` does not set, AND
      2. the engine is approved for its own instance URL
         (``resolve_engine_allow_private_ips``).

    Holding the grant is necessary but not sufficient: the full-content
    wrapper applies it only when the run's egress scope is
    ``unprotected`` (``FullSearchResults._effective_allow_private_ips``).
    Under every other scope, the default adaptive scope included, private
    result URLs stay blocked even when this returns True.

    Link-local (``169.254.0.0/16``, ``fe80::/10``) and the cloud-metadata
    literals in ``ALWAYS_BLOCKED_METADATA_IPS`` stay blocked even when
    this returns True: the full-content fetch path validates with
    ``block_link_local=True``. Fails CLOSED (False) on any error.
    """
    try:
        from ...settings.env_registry import get_env_setting

        if not bool(
            get_env_setting("search.allow_private_result_fetch", False)
        ):
            return False
    except Exception:  # noqa: silent-exception - fail closed
        return False
    return resolve_engine_allow_private_ips(instance_url, url_setting)


def resolve_searxng_allow_private_result_fetch(instance_url: str) -> bool:
    """SearXNG-keyed convenience wrapper over
    ``resolve_engine_allow_private_result_fetch``."""
    return resolve_engine_allow_private_result_fetch(
        instance_url, _SEARXNG_INSTANCE_URL_KEY
    )


def _engine_url_ssrf_error(key, value, *, allow_private: bool):
    """Return an error dict for a public-engine URL that must be refused.

    Rejects:
      - a non-http(s) scheme (never a valid engine URL — refused even when the
        operator opt-in is set),
      - a host that IS or RESOLVES TO a cloud-metadata / always-blocked
        address (refused regardless of the operator opt-in), and
      - unless ``allow_private`` is set, a host that IS or RESOLVES TO a
        private / loopback / link-local address.

    Reuses ``ssrf_validator.is_ip_blocked`` for the classification: the
    ``allow_localhost=True, allow_private_ips=True`` form isolates the
    always-blocked cloud-metadata set, so metadata stays refused even under the
    operator opt-in. Returns ``None`` when the value is not a URL string (type
    validation handles that elsewhere).

    Fail-open on transient resolution failure (DNS down / split-horizon): only
    names that actually resolve to a blocked address are rejected, so a
    legitimate public host stays saveable during a DNS hiccup — mirroring
    ``validate_allowed_local_hostnames``.
    """
    if not isinstance(value, str) or not value.strip():
        return None
    url = value.strip()

    from ..ssrf_validator import (
        RFC_FORBIDDEN_URL_CHARS_RE,
        is_ip_blocked,
        is_metadata_hostname,
    )

    if RFC_FORBIDDEN_URL_CHARS_RE.search(url):
        return {
            "key": key,
            "error": f"{key} contains illegal URL characters",
        }
    try:
        parsed = urlsplit(url)
    except Exception:
        return {"key": key, "error": f"{key} is not a valid URL"}

    if (parsed.scheme or "").lower() not in ("http", "https"):
        return {
            "key": key,
            "error": f"{key} must be an http:// or https:// URL",
        }

    host = parsed.hostname
    if not host:
        return {"key": key, "error": f"{key} has no hostname"}
    host = unquote(host)
    if host.startswith("[") and host.endswith("]"):
        host = host[1:-1]
    host = host.rstrip(".")

    # Cloud-metadata hostname (GCP, IBM Cloud classic): refused regardless of
    # opt-in. IBM's name resolves to RFC1918, so the IP checks below would
    # pass it when allow_private is set. is_metadata_hostname also fails
    # closed on a non-ASCII name no IDNA encoder (IDNA 2003, IDNA 2008 or
    # UTS #46) can encode; that name is refused too, but reported as what it
    # is rather than as metadata.
    if is_metadata_hostname(host):
        if not _idna_encodable(host):
            return _engine_url_error(key, "invalid_host")
        return _engine_url_error(key, "metadata")

    def _block_reason(ip_str) -> Optional[str]:
        # Cloud-metadata / always-blocked set: refused regardless of opt-in.
        if is_ip_blocked(ip_str, allow_localhost=True, allow_private_ips=True):
            return "metadata"
        # Private / loopback / link-local: refused unless operator opted in.
        if not allow_private and is_ip_blocked(
            ip_str, allow_localhost=False, allow_private_ips=False
        ):
            return "private"
        return None

    try:
        ipaddress.ip_address(host)
        is_literal_ip = True
    except ValueError:
        is_literal_ip = False

    if is_literal_ip:
        reason = _block_reason(host)
        return _engine_url_error(key, reason) if reason else None

    addr_info = _resolve_with_timeout(host)
    if not addr_info:
        return None  # fail-open on unresolvable host
    for entry in addr_info:
        try:
            ip_str = entry[4][0]
        except (IndexError, TypeError):
            continue
        reason = _block_reason(ip_str)
        if reason:
            return _engine_url_error(key, reason)
    return None


def _idna_encodable(host: str) -> bool:
    """Whether any IDNA encoder a resolver or HTTP client uses can encode
    ``host`` as a DNS name.

    ``ssrf_validator._idna_folded_names`` tries the stdlib ``idna`` codec
    (IDNA 2003, what ``socket.getaddrinfo`` uses), the ``idna`` package's
    IDNA 2008 encoding (what urllib3 and ``httpx`` use) and its UTS #46
    mapping. A name none of them accepts (for example one with a label
    over 63 octets) cannot be looked up through any of them;
    ``is_metadata_hostname`` refuses such a name (fails closed), and this
    lets the error message say why. A name one of them accepts is
    encodable.
    """
    from ..ssrf_validator import _idna_folded_names

    name = host.lower()
    return name.isascii() or _idna_folded_names(name) is not None


def _engine_url_error(key, reason):
    if reason == "metadata":
        message = (
            f"{key} points at a cloud-metadata / always-blocked address, "
            "which is never a permitted destination."
        )
    elif reason == "invalid_host":
        message = (
            f"{key} has an invalid or unsupported hostname: no IDNA "
            "encoding (IDNA 2003, IDNA 2008 or UTS #46) turns it into a DNS "
            "name."
        )
    else:
        message = (
            f"{key} points at a private, loopback, or link-local address. A "
            "public search engine proxies the internet, so an internal URL is "
            "refused to prevent it being used to reach your private network. "
            "Self-hosted instances on localhost/LAN are still supported, but "
            "the server operator must approve them in the server "
            "environment: add this exact URL origin to "
            "LDR_SEARCH_PRIVATE_ENGINE_URL_ALLOWLIST (comma-separated "
            "scheme://host:port entries), or pin the URL via its LDR_ "
            "environment variable (as the bundled docker-compose.yml does), "
            "or set LDR_SEARCH_ALLOW_PRIVATE_ENGINE_URLS=true to allow all "
            "private addresses. Only one of these is needed; restart after "
            "changing. See docs/SearXNG-Setup.md."
        )
    return {"key": key, "error": message}


# Sentinel marking a guarded key that has no stored value yet — treated as a
# genuine change so a brand-new setting is still validated.
_NO_STORED_VALUE = object()


def _stored_engine_url_value(all_db_settings, key):
    """Return the currently-stored value for ``key`` in ``all_db_settings``.

    Entries are ``Setting`` ORM rows exposing the value via ``.value`` in the
    live save routes (``_filter_editable_settings`` builds ``{key: Setting}``),
    but some callers hand plain ``{"value": ...}`` dicts. Normalize both to the
    scalar with ``unwrap_setting`` — the dict form directly, an ORM/attribute
    row via its ``.value``. Returns ``_NO_STORED_VALUE`` when the key is absent.
    """
    if not all_db_settings or key not in all_db_settings:
        return _NO_STORED_VALUE
    entry = all_db_settings[key]
    if not isinstance(entry, dict):
        entry = getattr(entry, "value", _NO_STORED_VALUE)
        if entry is _NO_STORED_VALUE:
            return _NO_STORED_VALUE
    return unwrap_setting(entry)


def _normalize_engine_url_for_comparison(value):
    """Parse ``value`` into a tuple of URL parts for cosmetic-variant
    comparison, or ``None`` if it cannot be parsed as a URL.

    Collapses only variation that denotes the exact SAME url:
      - scheme and host case (``HTTP://LOCALHOST`` == ``http://localhost``),
      - any trailing slash(es) on the path (``rstrip("/")`` strips all of
        them, so ``"/path//"`` and ``"/path"`` compare equal, and an
        empty path and ``"/"`` compare equal too).

    Everything that could denote a DIFFERENT destination is preserved
    verbatim and NOT normalized away: the host itself, the port, any path
    beyond a bare trailing slash, the query, the fragment, and any userinfo
    (unusual in an ``instance_url``, but kept rather than risk folding two
    distinct credentials together). In particular this never strips the
    host, resolves DNS, or drops the port — so a genuine change to a
    different host/port/scheme still fails to compare equal here.

    Returns ``None`` on any parse failure (e.g. a non-numeric port, a
    malformed IPv6 literal) so the caller falls back to the exact
    string comparison — fail toward validating, never toward skipping.
    """
    try:
        parsed = urlsplit(str(value).strip())
        return (
            (parsed.scheme or "").lower(),
            parsed.username,
            parsed.password,
            (parsed.hostname or "").lower(),
            parsed.port,
            (parsed.path or "").rstrip("/"),
            parsed.query,
            parsed.fragment,
        )
    except Exception:
        return None


def _engine_url_is_unchanged(submitted_value, all_db_settings, key):
    """True when the submitted engine URL equals the currently-stored one.

    The frontend "All Settings" tab resubmits every rendered input, including a
    shipped default the user never touched (e.g. the SearXNG
    ``instance_url = http://localhost:8080`` loopback default). Validating an
    unchanged value would 400 an otherwise-unrelated save, so only a genuine
    change is validated. A brand-new key with no stored value is treated as
    changed.

    Both sides are first string-normalized so a metadata-wrapped vs scalar
    representation of the same value is not misread as a change. If that
    exact match fails, both sides are also compared via
    ``_normalize_engine_url_for_comparison`` so a purely cosmetic resubmit —
    different scheme/host case, or a trailing slash — of the SAME url is
    still recognized as unchanged (e.g. resubmitting the stored
    ``http://localhost:8080`` as ``HTTP://LOCALHOST:8080`` or
    ``http://localhost:8080/``). This only ever widens what counts as
    UNCHANGED (and is therefore skipped from validation); it never narrows
    it, so it cannot make two genuinely different hosts compare equal. If
    either side fails to parse, that comparison is skipped and the exact
    match above is the only thing that can call it unchanged.
    """
    stored = _stored_engine_url_value(all_db_settings, key)
    if stored is _NO_STORED_VALUE:
        return False
    submitted_str = str(submitted_value).strip()
    stored_str = str(stored).strip()
    if submitted_str == stored_str:
        return True
    submitted_norm = _normalize_engine_url_for_comparison(submitted_value)
    stored_norm = _normalize_engine_url_for_comparison(stored)
    if submitted_norm is None or stored_norm is None:
        return False
    return submitted_norm == stored_norm


def validate_engine_instance_urls(form_data, all_db_settings):
    """Reject a PUBLIC search-engine URL that targets a private address.

    SearXNG (and any future public-nature engine with a configurable URL) is
    user-editable through the settings API. Because such an engine proxies the
    public web, a private / loopback / link-local ``instance_url`` is only ever
    an internal-network probe target — so at save time we refuse it under the
    default posture, closing the SSRF where any authenticated user turns a
    research run into an internal port scan. Non-http(s) schemes are refused
    outright.

    The operator opt-in ``search.allow_private_engine_urls`` (env-only) lifts
    the private-address rejection for genuine LAN self-hosting, and the
    env-only allowlist ``search.private_engine_url_allowlist`` lifts it for
    specific listed origins only; Docker deployments instead env-lock the
    URL, which the route layer honors via ``check_env_setting`` before these
    validators run. Neither mechanism lifts the cloud-metadata or
    non-http(s)-scheme rejections.
    """
    guarded = _public_engine_url_settings()
    present = [k for k in guarded if k in form_data]
    if not present:
        return None

    allow_private = False
    try:
        from ...settings.env_registry import get_env_setting

        allow_private = bool(
            get_env_setting("search.allow_private_engine_urls", False)
        )
    except Exception:  # noqa: silent-exception - default to the safe posture
        allow_private = False

    for key in present:
        value = unwrap_setting(form_data[key])
        # Validate-on-change: skip an UNCHANGED value (e.g. the shipped default
        # loopback ``instance_url`` resubmitted untouched by the All-Settings
        # tab) so it can't block an unrelated save. The runtime
        # allow_private_ips=False backstop still blocks the actual fetch.
        if _engine_url_is_unchanged(value, all_db_settings, key):
            continue
        err = _engine_url_ssrf_error(
            key,
            value,
            allow_private=allow_private
            or engine_url_in_private_allowlist(value),
        )
        if err is not None:
            return err
    return None


# The cross-field egress validators every settings-write route must run. Keep
# the *list* here (single source of truth) so adding one doesn't require a
# lockstep edit at each save route; each route only supplies its own response
# shaping around ``first_egress_validation_error``.
EGRESS_SETTINGS_VALIDATORS = (
    validate_egress_scope,
    validate_allowed_local_hostnames,
    validate_trusted_search_engines,
    validate_engine_instance_urls,
)


def first_egress_validation_error(form_data, all_db_settings):
    """Run the egress validators; return the first error dict, or ``None``."""
    for _validator in EGRESS_SETTINGS_VALIDATORS:
        err = _validator(form_data, all_db_settings)
        if err is not None:
            return err
    return None