~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""Cross-field egress-policy validators run at settings-save time.
Each validator takes the about-to-be-saved ``form_data`` plus the current
``all_db_settings`` (key -> setting row with a ``.value``) and returns a
validation-error dict ``{"key", "error"}`` to surface in the save response,
or ``None`` when the combination is fine. Most are pure; the engine-URL
guard machinery here (allowlist, operator-approval resolver, descriptor
sweep) additionally reads operator environment state — never the DB — and
the allowlist parser logs dropped entries once per distinct env value.
The settings write routes (``web/routers/settings.py``) orchestrate these
— they stay in the route layer because they also enforce non-egress concerns —
but the egress rules themselves live here next to the policy they encode.
"""
import functools
import ipaddress
from dataclasses import dataclass
from typing import Optional, Tuple
from urllib.parse import unquote, urlsplit
from ...utilities.type_utils import unwrap_setting
from .policy import (
EgressContext,
EgressScope,
PolicyDeniedError,
_canonical_host,
_classify_host,
_is_metadata_host,
_resolve_with_timeout,
parse_user_egress_scope,
)
# The SearXNG instance URL is the canonical exploitable case and is always
# guarded even if dynamic engine-class enumeration fails below.
_SEARXNG_INSTANCE_URL_KEY = (
"search.engine.web.searxng.default_params.instance_url"
)
def validate_egress_scope(form_data, _all_db_settings):
"""Validate the scope and enforce the operator-controlled escape hatch."""
key = "policy.egress_scope"
if key not in form_data:
return None
value = form_data[key]
if not isinstance(value, str):
return {"key": key, "error": "Egress scope must be a string"}
try:
parse_user_egress_scope(value)
except PolicyDeniedError as exc:
if exc.decision.reason == "unprotected_egress_disabled":
return {
"key": key,
"error": "Unprotected egress is disabled by the server operator",
}
return {
"key": key,
"error": "Unknown egress scope",
}
return None
def validate_allowed_local_hostnames(form_data, all_db_settings):
"""Reject entries in llm.allowed_local_hostnames that would not, or must
not, make a host local.
The default-settings description for this key claims "Public hostnames
added here are rejected at save time", but until now no code actually
did the rejection. Each entry is judged exactly as host classification
reads it (``policy._canonical_host``: decoded once, case folded, one
trailing dot ignored), and refused when it
* contains ``%`` (percent-encoding or an IPv6 zone ID): no hostname
needs it, and refusing it keeps the saved text and the name it
matches the same (classification reads a stored entry decoded once);
* is not a plain DNS name or a canonical IP address (non-ASCII
characters, whitespace or control characters, an integer / hex /
octal / short IPv4 notation) -- classification ignores such an entry,
so accepting it would only hide that it has no effect;
* names a cloud-metadata endpoint (``policy._is_metadata_host``), which
can never be local; or
* resolves to public addresses, via the same host classifier the policy
uses. A hostname that fails to resolve (DNS down) is accepted --
fail-open on transient lookup errors so the user can recover.
Only a changed value is validated: one equal to the stored value,
entry for entry, is accepted unchecked
(``_allowed_local_hostnames_unchanged``). Never raises: a missing
``form_data`` (None) has nothing to validate, and an unexpected error is
reported as a validation error (fail closed) rather than escaping into
the save route.
"""
key = "llm.allowed_local_hostnames"
try:
if not form_data or key not in form_data:
return None
value = form_data[key]
# Validate-on-change, as for the engine URLs: the All-Settings tab
# resubmits every setting, so an unchanged list (say, one holding an
# entry saved before this validator became stricter) must not block
# an unrelated save. Runtime classification ignores such an entry
# anyway. A changed list is validated in full.
if _allowed_local_hostnames_unchanged(value, all_db_settings, key):
return None
return _check_allowed_local_hostnames(key, value)
except Exception:
return {
"key": key,
"error": (
"allowed_local_hostnames could not be validated; check that "
"it is a JSON array of plain hostnames or IP addresses"
),
}
def _allowed_local_hostnames_entries(value):
"""The entries of an ``llm.allowed_local_hostnames`` value (a list, or
its JSON text), or None when it is neither."""
import json as _json
value = unwrap_setting(value)
if isinstance(value, str):
try:
value = _json.loads(value) if value.strip() else []
except ValueError:
return None
return value if isinstance(value, list) else None
def _allowed_local_hostnames_unchanged(submitted, all_db_settings, key):
"""True when the submitted ``llm.allowed_local_hostnames`` value holds
exactly the stored entries, in the same order (a list and its JSON text
compare equal). Fails toward validating: no stored value, or a value on
either side that is not a list, counts as changed."""
stored = _stored_engine_url_value(all_db_settings, key)
if stored is _NO_STORED_VALUE:
return False
entries = _allowed_local_hostnames_entries(submitted)
return entries is not None and entries == _allowed_local_hostnames_entries(
stored
)
def _display_entry(entry: str) -> str:
"""An entry as quoted in an error message: ``repr`` shows whitespace and
control characters a plain rendering would hide, bounded in length."""
text = repr(entry)
return text if len(text) <= 80 else text[:77] + "..."
def _check_allowed_local_hostnames(key, value):
# Setting is JSON-typed; the save pipeline may hand us a list or a
# JSON string. Decode defensively.
if isinstance(value, str):
try:
import json as _json
decoded = _json.loads(value) if value.strip() else []
except Exception:
return {
"key": key,
"error": "allowed_local_hostnames must be a JSON array of hostnames",
}
value = decoded
if not isinstance(value, list):
return {
"key": key,
"error": "allowed_local_hostnames must be a list",
}
# Build a minimal real context just for the resolver. Use the
# dataclass constructor — NOT EgressContext.__new__ + setattr, which
# raised FrozenInstanceError on this frozen dataclass and (separately)
# set a non-existent ``allowed_local_hostnames`` field instead of the
# real ``local_hostnames`` the classifier reads. The constructor
# initializes the init=False internals (_dns_cache, _lock as RLock)
# correctly. Empty local_hostnames => classify purely on IP class.
probe_ctx = EgressContext(
scope=EgressScope.BOTH,
primary_engine="searxng",
require_local_llm=False,
require_local_embeddings=False,
local_hostnames=(),
)
rejected = []
metadata = []
invalid = []
for entry in value:
if not isinstance(entry, str) or not entry.strip():
continue
# Judged exactly as classification reads the entry, as written:
# _canonical_host decodes it once and folds case itself. An entry
# with "%" is refused before that, so a saved entry is always the
# text it matches.
hostname = None if "%" in entry else _canonical_host(entry)
if hostname is None:
invalid.append(_display_entry(entry))
continue
# A cloud-metadata name or address can never be declared local
# (classification ignores such an entry), whatever it resolves to.
# Report it as what it is rather than as a public address: IBM
# Cloud classic's metadata name resolves to an RFC1918 address.
if _is_metadata_host(hostname):
metadata.append(hostname)
continue
try:
# Distinguish "could not resolve" from "resolved to a public IP".
# _classify_host collapses BOTH to False (its documented fail-safe
# treats an unresolvable host as public), so relying on it here
# would reject a legitimate intranet/VPN host on any DNS hiccup or
# split-horizon DNS — the exact use case this setting exists for.
# Only reject names that actually resolve to a public address;
# accept unresolvable ones (fail-open on save, as documented).
# _resolve_with_timeout returns the addrinfo for literal IPs too,
# so literal public/private IPs still flow through _classify_host.
if _resolve_with_timeout(hostname) is None:
continue
classification = _classify_host(hostname, probe_ctx)
except Exception:
# DNS or unknown error — allow (fail open) so the user can
# save when networking is flaky. Runtime classification will
# still gate egress.
continue
if classification is False:
rejected.append(hostname)
errors = []
if invalid:
errors.append(
"These entries are not valid hostnames for this list: "
f"{', '.join(invalid)}. Only plain DNS names (ASCII letters, "
"digits, hyphens, underscores and dots, written without "
"percent-encoding) and standard IP addresses (dotted-quad IPv4, "
"or IPv6 without a zone ID) are accepted; any other spelling "
"would be ignored. Write the name in that form (an "
"internationalised name in its xn-- form) or remove it."
)
if metadata:
errors.append(
"These entries are cloud-metadata endpoints, which are blocked "
"under every scope and can never be treated as local: "
f"{', '.join(metadata)}. Remove them."
)
if rejected:
errors.append(
"These hostnames resolve to PUBLIC addresses and would "
"let the policy treat external hosts as local: "
f"{', '.join(rejected)}. Remove them, or use the SSRF "
"allowlist instead."
)
if errors:
return {"key": key, "error": " ".join(errors)}
return None
def validate_trusted_search_engines(form_data, all_db_settings):
"""Reject inherently-public engines from ``policy.trusted_search_engines``.
Trusting a search engine relaxes its Exposure to CONTAINED — meaningful only
for a local-nature store you host (Elasticsearch/Paperless). An inherently
public engine (google, searxng, brave, …) genuinely queries the internet, so
trusting it would launder a public sink past the two-axis rule. Reject those;
local / unknown names are accepted (runtime classification still gates them,
and ``engine_label`` also refuses to relax an ``is_public`` engine).
"""
key = "policy.trusted_search_engines"
if key not in form_data:
return None
value = form_data[key]
if isinstance(value, str):
import json
try:
value = json.loads(value) if value.strip() else []
except Exception:
return {
"key": key,
"error": "trusted_search_engines must be a JSON array",
}
if not isinstance(value, list):
return {"key": key, "error": "trusted_search_engines must be a list"}
from .policy import _get_engine_class
rejected = []
for entry in value:
if not isinstance(entry, str) or not entry.strip():
continue
name = entry.strip().lower()
cls = _get_engine_class(name)
if cls is not None and getattr(cls, "is_public", False) is True:
rejected.append(name)
if rejected:
return {
"key": key,
"error": (
"These are inherently-public search engines and cannot be "
"trusted as contained: " + ", ".join(rejected) + "."
),
}
return None
@dataclass(frozen=True)
class GuardedEngineUrl:
"""A search engine whose configurable URL participates in the egress
URL guards, derived from the engine class's own declarations.
``is_public`` True → the engine proxies the public internet, so a
PRIVATE URL is the anomaly (save-time rejection + runtime refusal +
"private URL not approved" banner). ``is_public`` False → a LOCAL
document engine, where a PUBLIC-looking URL is the anomaly (advisory
banner only — a hosted endpoint can be legitimate).
Per-engine UI conventions derived from ``engine_name``:
display name setting ``search.engine.web.{name}.display_name``,
activation flags ``.enabled`` / ``.use_in_auto_search`` /
``.agent_enabled``, and dismiss keys
``app.warnings.dismiss_{name}_private_url`` (public) /
``app.warnings.dismiss_{name}_public_url`` (local) — each dismiss key
must be registered in ``defaults/default_settings.json`` (pinned by a
registry-contract test). A future public engine that declares
``url_setting`` must also call ``resolve_engine_allow_private_ips``
on its fetch path, as SearXNG does.
"""
engine_name: str
url_setting: str
is_public: bool
@functools.lru_cache(maxsize=1)
def guarded_engine_url_descriptors() -> Tuple[GuardedEngineUrl, ...]:
"""All engines participating in the URL guards, from the registry.
Derived once from ``ENGINE_REGISTRY`` + engine-class declarations
(``url_setting`` plus ``is_public``/``is_local``); seeded with SearXNG
so the canonical guard survives an engine-class import failure.
COST NOTE: the first call imports every engine module in the registry
(~1-3 s cold) to read the class declarations; results are cached for
the process lifetime (``.cache_clear()`` resets, for tests). Callers on
render paths pay this once per process.
"""
found = {
"searxng": GuardedEngineUrl(
engine_name="searxng",
url_setting=_SEARXNG_INSTANCE_URL_KEY,
is_public=True,
)
}
try:
from ...web_search_engines.engine_registry import ENGINE_REGISTRY
from .policy import _get_engine_class
for name in ENGINE_REGISTRY:
cls = _get_engine_class(name)
if cls is None:
continue
url_setting = getattr(cls, "url_setting", None)
if not (isinstance(url_setting, str) and url_setting):
continue
if getattr(cls, "is_public", None) is True:
found[name] = GuardedEngineUrl(
engine_name=name, url_setting=url_setting, is_public=True
)
elif getattr(cls, "is_local", None) is True:
found[name] = GuardedEngineUrl(
engine_name=name, url_setting=url_setting, is_public=False
)
except Exception: # noqa: silent-exception - seed still guards SearXNG
pass
return tuple(found[name] for name in sorted(found))
@functools.lru_cache(maxsize=1)
def _public_engine_url_settings() -> frozenset:
"""URL-setting keys belonging to PUBLIC-nature search engines.
These are the engine URLs a save-time SSRF check must guard: a public
engine (SearXNG) proxies the public web, so a private/loopback
``instance_url`` is only ever an internal-network probe target, never a
legitimate data source. LOCAL engines (Elasticsearch, Paperless) are
deliberately EXCLUDED — a private URL is their whole purpose, and the PDP's
fail-up URL override already reclassifies them public when pointed at an
external host.
Delegates to ``guarded_engine_url_descriptors`` so the save-time guard
and the banner/orchestrator machinery can never disagree about which
engines are covered (single registry sweep, single seed).
"""
return frozenset(
d.url_setting for d in guarded_engine_url_descriptors() if d.is_public
)
# Env-only operator allowlist of exact private engine-URL origins — the
# finer-grained alternative to the blanket search.allow_private_engine_urls
# gate (see env_definitions/security.py for the operator-facing contract).
_ALLOWLIST_SETTING_KEY = "search.private_engine_url_allowlist"
_DEFAULT_PORTS = {"http": 80, "https": 443}
def _engine_url_origin(value):
"""Parse ``value`` into a comparable ``(scheme, host, port)`` origin.
Normalization mirrors what actually denotes the same network
destination and nothing more: scheme and host are lowercased, the host
is stripped of a trailing dot, an IP-literal host is canonicalized
through :mod:`ipaddress` (so ``[::1]`` and its long form compare
equal), and a missing port takes the scheme default (80/443). Path,
query, fragment, and userinfo are ignored — they don't change the
destination host.
Two deliberate refusals keep this parser from ever disagreeing with
the HTTP client (urllib3) about which host a hostile URL names:
URLs containing RFC-forbidden characters (backslash, whitespace,
control chars — the parser-differential class) return ``None``, and
the host is NOT percent-decoded here (urllib3 before 2.8 did not
decode http(s) hosts and 2.8+ does, so this parser stays decode-free:
an escaped host matches only an allowlist entry written with the same
escapes, never a plain entry by decoding, and ``validate_url`` refuses
percent-encoded hosts at fetch time). Returns ``None`` for anything
that isn't a cleanly parseable
http(s) URL with a hostname; every failure mode therefore fails
CLOSED (no allowlist match), never open.
"""
try:
text = str(value).strip()
from ..ssrf_validator import RFC_FORBIDDEN_URL_CHARS_RE
if RFC_FORBIDDEN_URL_CHARS_RE.search(text):
return None
parsed = urlsplit(text)
scheme = (parsed.scheme or "").lower()
if scheme not in _DEFAULT_PORTS:
return None
host = parsed.hostname
if not host:
return None
host = host.rstrip(".").lower()
if not host:
return None
try:
host = str(ipaddress.ip_address(host))
except ValueError:
pass # not an IP literal — keep the hostname text
port = parsed.port
if port is None:
port = _DEFAULT_PORTS[scheme]
return (scheme, host, port)
except Exception:
return None
def engine_url_origin_text(origin) -> str:
"""Render an ``_engine_url_origin`` tuple back to allowlist-entry text.
The inverse of ``_engine_url_origin`` for display: IPv6 hosts are
re-bracketed and a scheme-default port is omitted, so the returned
string, fed into ``LDR_SEARCH_PRIVATE_ENGINE_URL_ALLOWLIST``, is
guaranteed to match the origin it was rendered from (round-trip by
construction — pinned by the banner remedy tests). Lives next to the
parser so the pair can't drift apart.
"""
scheme, host, port = origin
host_disp = f"[{host}]" if ":" in host else host
suffix = "" if port == _DEFAULT_PORTS[scheme] else f":{port}"
return f"{scheme}://{host_disp}{suffix}"
# Raw allowlist value we last warned about — the dropped-entry WARNING is
# emitted once per distinct env value, not on every consult (the matcher
# runs on every settings save, engine init, and /api/warnings recompute).
_last_warned_allowlist_raw: Optional[str] = None
def _private_engine_url_allowlist() -> frozenset:
"""Operator-approved private engine-URL origins from the environment.
Reads the env-only ``search.private_engine_url_allowlist`` setting and
parses each comma-separated entry with ``_engine_url_origin``.
Unparseable entries are dropped (fail-closed) and warned about once per
distinct env value. Any lookup error yields the empty set — the safe
posture.
"""
try:
from ...settings.env_registry import get_env_setting
raw = get_env_setting(_ALLOWLIST_SETTING_KEY, None)
except Exception: # noqa: silent-exception - default to the safe posture
return frozenset()
if not raw or not isinstance(raw, str):
return frozenset()
origins = set()
dropped = []
for entry in raw.split(","):
entry = entry.strip()
if not entry:
continue
origin = _engine_url_origin(entry)
if origin is not None:
origins.add(origin)
else:
dropped.append(entry)
if dropped:
# An unparseable entry grants nothing (fail-closed), but silently
# dropping it leaves the operator staring at the same rejection
# message telling them to add the origin they think they already
# added — so name what was ignored (once per distinct env value,
# to keep a permanently-malformed entry from spamming the logs).
global _last_warned_allowlist_raw
if raw != _last_warned_allowlist_raw:
_last_warned_allowlist_raw = raw
try:
from ..secure_logging import logger
logger.warning(
"Ignoring unparseable entries in "
"LDR_SEARCH_PRIVATE_ENGINE_URL_ALLOWLIST (each entry "
"must be a full origin like http://localhost:8080): "
f"{dropped}"
)
except Exception: # noqa: silent-exception - best-effort logging
pass
return frozenset(origins)
def engine_url_in_private_allowlist(url) -> bool:
"""True when ``url``'s origin is in the operator's env-only allowlist.
Grants ONLY the private/loopback/link-local exemption a matching origin
was listed for — callers consult it strictly in place of the blanket
``search.allow_private_engine_urls`` gate, so the cloud-metadata block
and the http(s)-scheme requirement are unaffected by a listing.
"""
origin = _engine_url_origin(url)
if origin is None:
return False
return origin in _private_engine_url_allowlist()
def resolve_engine_allow_private_ips(
instance_url: str, url_setting: str
) -> bool:
"""Whether a PUBLIC engine may target a private/loopback instance URL.
A public-nature engine proxies the public web, so a private instance
URL is only legitimate when an OPERATOR chose it, never when an
arbitrary authenticated user smuggled an internal host in as an SSRF
probe. Private egress to the configured instance is therefore
permitted only when:
1. the operator opt-in ``search.allow_private_engine_urls`` is set
(env-only; the blanket LAN / localhost self-host case), OR
2. ``instance_url``'s exact origin is listed in the env-only operator
allowlist ``search.private_engine_url_allowlist`` (the
finer-grained alternative to the blanket opt-in), OR
3. the engine's own ``url_setting`` is env-locked (its derived
``LDR_…`` variable is set — Docker / operator-provisioned,
trusted).
All three are OPERATOR-controlled. The run's egress scope
(``policy.egress_scope``) deliberately does NOT grant it: that setting is a
user-editable dropdown (STRICT included), so relaxing on it would be a
self-service SSRF bypass. By default private IPs are refused, so a
tampered instance URL cannot reach an internal host. Cloud-metadata /
link-local metadata IPs stay blocked regardless
(ALWAYS_BLOCKED_METADATA_IPS in the SSRF validator's probe path). Fails
CLOSED (False) on any error: a genuinely-public instance is unaffected —
only private egress is withheld.
"""
# 1. Operator opt-in (LAN self-host).
try:
from ...settings.env_registry import get_env_setting
if bool(get_env_setting("search.allow_private_engine_urls", False)):
return True
except Exception: # noqa: silent-exception - fall through to next check
pass
# 2. Operator allowlist: this exact origin was individually approved.
try:
if engine_url_in_private_allowlist(instance_url):
return True
except Exception: # noqa: silent-exception - fall through to next check
pass
# 3. Env-locked instance URL (Docker / operator-provisioned).
try:
from ...settings.manager import check_env_setting
if check_env_setting(url_setting) is not None:
return True
except Exception: # noqa: silent-exception - fall through to next check
pass
# Otherwise: withhold private egress. We deliberately DO NOT relax this
# based on the run's resolved egress scope. ``policy.egress_scope`` is a
# user-editable, self-service setting (STRICT is a plain dropdown option,
# not operator-gated), and a public engine always queries the internet.
# Letting a user-chosen scope grant private-IP access would be a
# self-service SSRF bypass (set scope=strict, then point the instance
# URL at an internal host). Private egress for a public engine is an
# OPERATOR decision only — conditions 1–3 above.
return False
def resolve_searxng_allow_private_ips(instance_url: str) -> bool:
"""SearXNG-keyed convenience wrapper over
``resolve_engine_allow_private_ips`` (kept for the engine and tests)."""
return resolve_engine_allow_private_ips(
instance_url, _SEARXNG_INSTANCE_URL_KEY
)
def resolve_engine_allow_private_result_fetch(
instance_url: str, url_setting: str
) -> bool:
"""Whether full-content fetches of a PUBLIC engine's RESULT URLs may
reach private / loopback hosts.
Distinct from, and strictly narrower than,
``resolve_engine_allow_private_ips``: that grant answers "may this
engine reach its configured *instance* URL", and its env-locked-URL
condition is met by every Docker (compose) deployment. A public engine
proxies the public web, so extending the instance grant to the URLs
the engine *returns* would let a poisoned index turn result fetching
into SSRF against the operator's LAN (loopback included). This
returns True (the engine holds the grant) only when BOTH hold:
1. the operator set the env-only opt-in
``search.allow_private_result_fetch``
(``LDR_SEARCH_ALLOW_PRIVATE_RESULT_FETCH=true``), which the
bundled ``docker-compose.yml`` does not set, AND
2. the engine is approved for its own instance URL
(``resolve_engine_allow_private_ips``).
Holding the grant is necessary but not sufficient: the full-content
wrapper applies it only when the run's egress scope is
``unprotected`` (``FullSearchResults._effective_allow_private_ips``).
Under every other scope, the default adaptive scope included, private
result URLs stay blocked even when this returns True.
Link-local (``169.254.0.0/16``, ``fe80::/10``) and the cloud-metadata
literals in ``ALWAYS_BLOCKED_METADATA_IPS`` stay blocked even when
this returns True: the full-content fetch path validates with
``block_link_local=True``. Fails CLOSED (False) on any error.
"""
try:
from ...settings.env_registry import get_env_setting
if not bool(
get_env_setting("search.allow_private_result_fetch", False)
):
return False
except Exception: # noqa: silent-exception - fail closed
return False
return resolve_engine_allow_private_ips(instance_url, url_setting)
def resolve_searxng_allow_private_result_fetch(instance_url: str) -> bool:
"""SearXNG-keyed convenience wrapper over
``resolve_engine_allow_private_result_fetch``."""
return resolve_engine_allow_private_result_fetch(
instance_url, _SEARXNG_INSTANCE_URL_KEY
)
def _engine_url_ssrf_error(key, value, *, allow_private: bool):
"""Return an error dict for a public-engine URL that must be refused.
Rejects:
- a non-http(s) scheme (never a valid engine URL — refused even when the
operator opt-in is set),
- a host that IS or RESOLVES TO a cloud-metadata / always-blocked
address (refused regardless of the operator opt-in), and
- unless ``allow_private`` is set, a host that IS or RESOLVES TO a
private / loopback / link-local address.
Reuses ``ssrf_validator.is_ip_blocked`` for the classification: the
``allow_localhost=True, allow_private_ips=True`` form isolates the
always-blocked cloud-metadata set, so metadata stays refused even under the
operator opt-in. Returns ``None`` when the value is not a URL string (type
validation handles that elsewhere).
Fail-open on transient resolution failure (DNS down / split-horizon): only
names that actually resolve to a blocked address are rejected, so a
legitimate public host stays saveable during a DNS hiccup — mirroring
``validate_allowed_local_hostnames``.
"""
if not isinstance(value, str) or not value.strip():
return None
url = value.strip()
from ..ssrf_validator import (
RFC_FORBIDDEN_URL_CHARS_RE,
is_ip_blocked,
is_metadata_hostname,
)
if RFC_FORBIDDEN_URL_CHARS_RE.search(url):
return {
"key": key,
"error": f"{key} contains illegal URL characters",
}
try:
parsed = urlsplit(url)
except Exception:
return {"key": key, "error": f"{key} is not a valid URL"}
if (parsed.scheme or "").lower() not in ("http", "https"):
return {
"key": key,
"error": f"{key} must be an http:// or https:// URL",
}
host = parsed.hostname
if not host:
return {"key": key, "error": f"{key} has no hostname"}
host = unquote(host)
if host.startswith("[") and host.endswith("]"):
host = host[1:-1]
host = host.rstrip(".")
# Cloud-metadata hostname (GCP, IBM Cloud classic): refused regardless of
# opt-in. IBM's name resolves to RFC1918, so the IP checks below would
# pass it when allow_private is set. is_metadata_hostname also fails
# closed on a non-ASCII name no IDNA encoder (IDNA 2003, IDNA 2008 or
# UTS #46) can encode; that name is refused too, but reported as what it
# is rather than as metadata.
if is_metadata_hostname(host):
if not _idna_encodable(host):
return _engine_url_error(key, "invalid_host")
return _engine_url_error(key, "metadata")
def _block_reason(ip_str) -> Optional[str]:
# Cloud-metadata / always-blocked set: refused regardless of opt-in.
if is_ip_blocked(ip_str, allow_localhost=True, allow_private_ips=True):
return "metadata"
# Private / loopback / link-local: refused unless operator opted in.
if not allow_private and is_ip_blocked(
ip_str, allow_localhost=False, allow_private_ips=False
):
return "private"
return None
try:
ipaddress.ip_address(host)
is_literal_ip = True
except ValueError:
is_literal_ip = False
if is_literal_ip:
reason = _block_reason(host)
return _engine_url_error(key, reason) if reason else None
addr_info = _resolve_with_timeout(host)
if not addr_info:
return None # fail-open on unresolvable host
for entry in addr_info:
try:
ip_str = entry[4][0]
except (IndexError, TypeError):
continue
reason = _block_reason(ip_str)
if reason:
return _engine_url_error(key, reason)
return None
def _idna_encodable(host: str) -> bool:
"""Whether any IDNA encoder a resolver or HTTP client uses can encode
``host`` as a DNS name.
``ssrf_validator._idna_folded_names`` tries the stdlib ``idna`` codec
(IDNA 2003, what ``socket.getaddrinfo`` uses), the ``idna`` package's
IDNA 2008 encoding (what urllib3 and ``httpx`` use) and its UTS #46
mapping. A name none of them accepts (for example one with a label
over 63 octets) cannot be looked up through any of them;
``is_metadata_hostname`` refuses such a name (fails closed), and this
lets the error message say why. A name one of them accepts is
encodable.
"""
from ..ssrf_validator import _idna_folded_names
name = host.lower()
return name.isascii() or _idna_folded_names(name) is not None
def _engine_url_error(key, reason):
if reason == "metadata":
message = (
f"{key} points at a cloud-metadata / always-blocked address, "
"which is never a permitted destination."
)
elif reason == "invalid_host":
message = (
f"{key} has an invalid or unsupported hostname: no IDNA "
"encoding (IDNA 2003, IDNA 2008 or UTS #46) turns it into a DNS "
"name."
)
else:
message = (
f"{key} points at a private, loopback, or link-local address. A "
"public search engine proxies the internet, so an internal URL is "
"refused to prevent it being used to reach your private network. "
"Self-hosted instances on localhost/LAN are still supported, but "
"the server operator must approve them in the server "
"environment: add this exact URL origin to "
"LDR_SEARCH_PRIVATE_ENGINE_URL_ALLOWLIST (comma-separated "
"scheme://host:port entries), or pin the URL via its LDR_ "
"environment variable (as the bundled docker-compose.yml does), "
"or set LDR_SEARCH_ALLOW_PRIVATE_ENGINE_URLS=true to allow all "
"private addresses. Only one of these is needed; restart after "
"changing. See docs/SearXNG-Setup.md."
)
return {"key": key, "error": message}
# Sentinel marking a guarded key that has no stored value yet — treated as a
# genuine change so a brand-new setting is still validated.
_NO_STORED_VALUE = object()
def _stored_engine_url_value(all_db_settings, key):
"""Return the currently-stored value for ``key`` in ``all_db_settings``.
Entries are ``Setting`` ORM rows exposing the value via ``.value`` in the
live save routes (``_filter_editable_settings`` builds ``{key: Setting}``),
but some callers hand plain ``{"value": ...}`` dicts. Normalize both to the
scalar with ``unwrap_setting`` — the dict form directly, an ORM/attribute
row via its ``.value``. Returns ``_NO_STORED_VALUE`` when the key is absent.
"""
if not all_db_settings or key not in all_db_settings:
return _NO_STORED_VALUE
entry = all_db_settings[key]
if not isinstance(entry, dict):
entry = getattr(entry, "value", _NO_STORED_VALUE)
if entry is _NO_STORED_VALUE:
return _NO_STORED_VALUE
return unwrap_setting(entry)
def _normalize_engine_url_for_comparison(value):
"""Parse ``value`` into a tuple of URL parts for cosmetic-variant
comparison, or ``None`` if it cannot be parsed as a URL.
Collapses only variation that denotes the exact SAME url:
- scheme and host case (``HTTP://LOCALHOST`` == ``http://localhost``),
- any trailing slash(es) on the path (``rstrip("/")`` strips all of
them, so ``"/path//"`` and ``"/path"`` compare equal, and an
empty path and ``"/"`` compare equal too).
Everything that could denote a DIFFERENT destination is preserved
verbatim and NOT normalized away: the host itself, the port, any path
beyond a bare trailing slash, the query, the fragment, and any userinfo
(unusual in an ``instance_url``, but kept rather than risk folding two
distinct credentials together). In particular this never strips the
host, resolves DNS, or drops the port — so a genuine change to a
different host/port/scheme still fails to compare equal here.
Returns ``None`` on any parse failure (e.g. a non-numeric port, a
malformed IPv6 literal) so the caller falls back to the exact
string comparison — fail toward validating, never toward skipping.
"""
try:
parsed = urlsplit(str(value).strip())
return (
(parsed.scheme or "").lower(),
parsed.username,
parsed.password,
(parsed.hostname or "").lower(),
parsed.port,
(parsed.path or "").rstrip("/"),
parsed.query,
parsed.fragment,
)
except Exception:
return None
def _engine_url_is_unchanged(submitted_value, all_db_settings, key):
"""True when the submitted engine URL equals the currently-stored one.
The frontend "All Settings" tab resubmits every rendered input, including a
shipped default the user never touched (e.g. the SearXNG
``instance_url = http://localhost:8080`` loopback default). Validating an
unchanged value would 400 an otherwise-unrelated save, so only a genuine
change is validated. A brand-new key with no stored value is treated as
changed.
Both sides are first string-normalized so a metadata-wrapped vs scalar
representation of the same value is not misread as a change. If that
exact match fails, both sides are also compared via
``_normalize_engine_url_for_comparison`` so a purely cosmetic resubmit —
different scheme/host case, or a trailing slash — of the SAME url is
still recognized as unchanged (e.g. resubmitting the stored
``http://localhost:8080`` as ``HTTP://LOCALHOST:8080`` or
``http://localhost:8080/``). This only ever widens what counts as
UNCHANGED (and is therefore skipped from validation); it never narrows
it, so it cannot make two genuinely different hosts compare equal. If
either side fails to parse, that comparison is skipped and the exact
match above is the only thing that can call it unchanged.
"""
stored = _stored_engine_url_value(all_db_settings, key)
if stored is _NO_STORED_VALUE:
return False
submitted_str = str(submitted_value).strip()
stored_str = str(stored).strip()
if submitted_str == stored_str:
return True
submitted_norm = _normalize_engine_url_for_comparison(submitted_value)
stored_norm = _normalize_engine_url_for_comparison(stored)
if submitted_norm is None or stored_norm is None:
return False
return submitted_norm == stored_norm
def validate_engine_instance_urls(form_data, all_db_settings):
"""Reject a PUBLIC search-engine URL that targets a private address.
SearXNG (and any future public-nature engine with a configurable URL) is
user-editable through the settings API. Because such an engine proxies the
public web, a private / loopback / link-local ``instance_url`` is only ever
an internal-network probe target — so at save time we refuse it under the
default posture, closing the SSRF where any authenticated user turns a
research run into an internal port scan. Non-http(s) schemes are refused
outright.
The operator opt-in ``search.allow_private_engine_urls`` (env-only) lifts
the private-address rejection for genuine LAN self-hosting, and the
env-only allowlist ``search.private_engine_url_allowlist`` lifts it for
specific listed origins only; Docker deployments instead env-lock the
URL, which the route layer honors via ``check_env_setting`` before these
validators run. Neither mechanism lifts the cloud-metadata or
non-http(s)-scheme rejections.
"""
guarded = _public_engine_url_settings()
present = [k for k in guarded if k in form_data]
if not present:
return None
allow_private = False
try:
from ...settings.env_registry import get_env_setting
allow_private = bool(
get_env_setting("search.allow_private_engine_urls", False)
)
except Exception: # noqa: silent-exception - default to the safe posture
allow_private = False
for key in present:
value = unwrap_setting(form_data[key])
# Validate-on-change: skip an UNCHANGED value (e.g. the shipped default
# loopback ``instance_url`` resubmitted untouched by the All-Settings
# tab) so it can't block an unrelated save. The runtime
# allow_private_ips=False backstop still blocks the actual fetch.
if _engine_url_is_unchanged(value, all_db_settings, key):
continue
err = _engine_url_ssrf_error(
key,
value,
allow_private=allow_private
or engine_url_in_private_allowlist(value),
)
if err is not None:
return err
return None
# The cross-field egress validators every settings-write route must run. Keep
# the *list* here (single source of truth) so adding one doesn't require a
# lockstep edit at each save route; each route only supplies its own response
# shaping around ``first_egress_validation_error``.
EGRESS_SETTINGS_VALIDATORS = (
validate_egress_scope,
validate_allowed_local_hostnames,
validate_trusted_search_engines,
validate_engine_instance_urls,
)
def first_egress_validation_error(form_data, all_db_settings):
"""Run the egress validators; return the first error dict, or ``None``."""
for _validator in EGRESS_SETTINGS_VALIDATORS:
err = _validator(form_data, all_db_settings)
if err is not None:
return err
return None