~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""Cloud-metadata HOSTNAME block (issue #6869).
IBM Cloud classic infrastructure serves its instance metadata
(``SoftLayer_Resource_Metadata``) from ``api.service.softlayer.com``, which
resolves to ``10.0.80.88`` -- an RFC1918 address. Every IP-based check
therefore passes it whenever private addresses are allowed
(``allow_private_ips=True``, the LLM base-URL path, the notification
plugin partition, STRICT / PRIVATE_ONLY egress). The name is blocked instead,
from the single list ``ssrf_validator.ALWAYS_BLOCKED_METADATA_HOSTNAMES``,
at every entry point that classifies a destination host.
Each test resolves the name to the REAL IBM address (10.0.80.88) and uses the
most permissive flags, so only the name block can refuse it: removing the
IBM entry from the list, or dropping the ``is_metadata_hostname`` call at the
entry point under test, makes the test fail.
The controls pin the deliberate trade-off: 10.0.80.88 itself stays allowed
under ``allow_private_ips=True`` (a LAN service may legitimately live there),
and neighbouring SoftLayer names are not over-blocked. The name block does
NOT stop a different DNS name that resolves to 10.0.80.88; that limit is
documented next to the list.
"""
import ipaddress
import socket
from unittest.mock import patch
import pytest
from local_deep_research.security.egress import validators
from local_deep_research.security.egress.policy import (
EgressContext,
EgressScope,
_classify_host,
_is_metadata_destination,
evaluate_engine,
evaluate_llm_endpoint,
evaluate_url,
)
from local_deep_research.security.notification_validator import (
NotificationURLValidator,
)
from local_deep_research.security.ssrf_validator import (
ALWAYS_BLOCKED_METADATA_HOSTNAMES,
ALWAYS_BLOCKED_METADATA_IPS,
assert_base_url_safe,
is_metadata_hostname,
validate_url,
)
from local_deep_research.web_search_engines.engines import (
search_engine_elasticsearch as es_engine,
)
IBM_HOST = "api.service.softlayer.com"
IBM_IP = "10.0.80.88"
IBM_ADDRINFO = [(socket.AF_INET, socket.SOCK_STREAM, 6, "", (IBM_IP, 0))]
IBM_RESOLVED = [ipaddress.IPv4Address(IBM_IP)]
# Spellings the resolver / HTTP client treat as the same host.
IBM_SPELLINGS = [
IBM_HOST,
"API.SERVICE.SOFTLAYER.COM",
"Api.Service.SoftLayer.Com",
f"{IBM_HOST}.",
f"{IBM_HOST}..",
"API.Service.SoftLayer.com.",
"api%2eservice%2esoftlayer%2ecom",
]
# Nearby names that are NOT the metadata endpoint. Resolved to the same
# RFC1918 address in the controls, so only the exact-name match separates
# them from IBM_HOST.
NEARBY_NON_METADATA = [
"api.softlayer.com",
"service.softlayer.com",
"softlayer.com",
"xapi.service.softlayer.com",
f"{IBM_HOST}.attacker.example",
"api-service.softlayer.com",
]
def _ssrf_dns(addrinfo):
return patch(
"local_deep_research.security.ssrf_validator.socket.getaddrinfo",
return_value=addrinfo,
)
# ---------------------------------------------------------------------------
# Single source of truth
# ---------------------------------------------------------------------------
def test_ibm_classic_metadata_hostname_is_listed():
assert IBM_HOST in ALWAYS_BLOCKED_METADATA_HOSTNAMES
def test_ibm_rfc1918_address_is_not_always_blocked():
"""The trade-off: 10.0.80.88 is RFC1918 and must NOT be an absolute
block, or a user's own LAN service at that address would break.
(161.26.0.6 is IBM's package mirror / NTP, not metadata, and must not be
listed either.)"""
assert IBM_IP not in ALWAYS_BLOCKED_METADATA_IPS
assert "161.26.0.6" not in ALWAYS_BLOCKED_METADATA_IPS
@pytest.mark.parametrize("host", IBM_SPELLINGS)
def test_is_metadata_hostname_normalizes_case_and_trailing_dot(host):
assert is_metadata_hostname(host) is True
@pytest.mark.parametrize("host", NEARBY_NON_METADATA)
def test_is_metadata_hostname_is_exact_match(host):
assert is_metadata_hostname(host) is False
# ---------------------------------------------------------------------------
# validate_url / assert_base_url_safe
# ---------------------------------------------------------------------------
@pytest.mark.parametrize("host", IBM_SPELLINGS)
@pytest.mark.parametrize(
"flags",
[
{},
{"allow_localhost": True},
{"allow_private_ips": True},
{"allow_localhost": True, "allow_private_ips": True},
],
)
def test_validate_url_blocks_ibm_metadata_hostname_under_all_flags(host, flags):
with _ssrf_dns(IBM_ADDRINFO):
assert (
validate_url(
f"https://{host}/rest/v3.1/SoftLayer_Resource_Metadata/getId",
**flags,
)
is False
)
def test_assert_base_url_safe_refuses_ibm_metadata_hostname():
"""The LLM base-URL path uses allow_private_ips=True, so the RFC1918
answer alone would pass; only the name block refuses it."""
with _ssrf_dns(IBM_ADDRINFO):
with pytest.raises(ValueError):
assert_base_url_safe(
f"https://{IBM_HOST}./v1", setting_key="llm.openai.url"
)
@pytest.mark.parametrize("host", NEARBY_NON_METADATA)
def test_validate_url_allows_nearby_hostname_on_same_address(host):
"""CONTROL: same RFC1918 answer, different name -> allowed under
allow_private_ips=True. Fails if the block is widened to a suffix /
substring match or to the IP itself."""
with _ssrf_dns(IBM_ADDRINFO):
assert validate_url(f"https://{host}/", allow_private_ips=True) is True
def test_validate_url_allows_lan_service_at_ibm_address():
"""CONTROL: a LAN service on 10.0.80.88 keeps working under
allow_private_ips=True (the documented reason the IP is not blocked)."""
assert validate_url(f"http://{IBM_IP}:11434/", allow_private_ips=True)
# ---------------------------------------------------------------------------
# Egress policy
# ---------------------------------------------------------------------------
def _ctx(scope, local_hostnames=()):
return EgressContext(
scope=scope,
primary_engine="test",
require_local_llm=True,
require_local_embeddings=True,
local_hostnames=tuple(local_hostnames),
)
@pytest.mark.parametrize("scope", list(EgressScope))
@pytest.mark.parametrize("host", IBM_SPELLINGS)
def test_evaluate_url_blocks_ibm_metadata_hostname_in_every_scope(scope, host):
with patch(
"local_deep_research.security.egress.policy._resolve_with_timeout",
return_value=IBM_ADDRINFO,
):
d = evaluate_url(f"https://{host}/", _ctx(scope))
assert not d.allowed
assert d.reason == "blocked_metadata_ip"
def test_evaluate_url_allows_nearby_hostname_under_private_only():
"""CONTROL: a nearby name on the same RFC1918 answer is local and allowed
under PRIVATE_ONLY."""
with patch(
"local_deep_research.security.egress.policy._resolve_with_timeout",
return_value=IBM_ADDRINFO,
):
d = evaluate_url(
"https://api.softlayer.com/", _ctx(EgressScope.PRIVATE_ONLY)
)
assert d.allowed
def test_classify_host_never_calls_ibm_metadata_hostname_local():
"""Reached by evaluate_llm_endpoint / evaluate_embeddings / engines,
which do not go through evaluate_url. The RFC1918 answer would classify
it local; it must not -- even if the user listed it as a local host."""
with patch(
"local_deep_research.security.egress.policy._resolve_with_timeout",
return_value=IBM_ADDRINFO,
):
ctx = _ctx(EgressScope.PRIVATE_ONLY, local_hostnames=[IBM_HOST])
assert _classify_host(IBM_HOST, ctx) is False
assert _classify_host("API.Service.SoftLayer.com.", ctx) is False
# CONTROL: a nearby name on the same answer is still local.
assert _classify_host("api.softlayer.com", ctx) is True
# ---------------------------------------------------------------------------
# Notification validator
# ---------------------------------------------------------------------------
@pytest.mark.parametrize(
"url",
[
f"https://{IBM_HOST}/hook",
"https://API.Service.SoftLayer.com./hook",
f"json://{IBM_HOST}/hook",
f"ntfy://{IBM_HOST}./topic",
],
)
def test_notification_validator_blocks_ibm_metadata_hostname(url):
with patch.object(
NotificationURLValidator,
"_resolve_hostname_ips",
return_value=IBM_RESOLVED,
):
ok, err = NotificationURLValidator.validate_service_url(
url, allow_private_ips=True
)
assert ok is False
assert err
@pytest.mark.parametrize(
"url", ["https://api.softlayer.com/hook", "json://api.softlayer.com/hook"]
)
def test_notification_validator_allows_nearby_hostname(url):
"""CONTROL: same RFC1918 answer, different name, private IPs allowed."""
with patch.object(
NotificationURLValidator,
"_resolve_hostname_ips",
return_value=IBM_RESOLVED,
):
ok, err = NotificationURLValidator.validate_service_url(
url, allow_private_ips=True
)
assert ok is True, err
# ---------------------------------------------------------------------------
# Search-engine URL settings validator
# ---------------------------------------------------------------------------
def test_engine_url_validator_refuses_ibm_metadata_hostname_with_opt_in():
with patch.object(
validators, "_resolve_with_timeout", return_value=IBM_ADDRINFO
):
err = validators._engine_url_ssrf_error(
"search.engine.web.searxng.default_params.instance_url",
f"https://{IBM_HOST}./",
allow_private=True,
)
assert err is not None
assert "cloud-metadata" in err["error"]
def test_engine_url_validator_allows_nearby_hostname_with_opt_in():
"""CONTROL."""
with patch.object(
validators, "_resolve_with_timeout", return_value=IBM_ADDRINFO
):
err = validators._engine_url_ssrf_error(
"search.engine.web.searxng.default_params.instance_url",
"https://api.softlayer.com/",
allow_private=True,
)
assert err is None
def test_policy_keeps_no_private_copy_of_the_hostname_list():
"""Single source of truth: the egress policy must consult
ssrf_validator.is_metadata_hostname, not a private list that can drift
(the pre-#6869 ``_METADATA_HOSTNAMES`` held only the GCP names)."""
from local_deep_research.security.egress import policy
assert not hasattr(policy, "_METADATA_HOSTNAMES")
# ---------------------------------------------------------------------------
# IDNA look-alike spellings
#
# getaddrinfo IDNA-encodes a non-ASCII str host (stdlib nameprep) and httpx /
# requests use the idna package (UTS #46). Both fold the ideographic /
# fullwidth / halfwidth full stops to "." and fullwidth letters to ASCII, so
# these spellings connect to the IBM metadata host.
# ---------------------------------------------------------------------------
IDNA_SPELLINGS = [
"api。service。softlayer。com", # U+3002 ideographic stop
"api.service.softlayer.com", # U+FF0E fullwidth stop
"api。service。softlayer。com", # U+FF61 halfwidth stop
"api.service.softlayer.com", # fullwidth "api"
# fullwidth upper-case letters and stops throughout
"API.SERVICE.SOFTLAYER.COM",
"api.service.softlayer.com。", # trailing ideographic stop
"api.service.softlayer.com", # soft hyphen is dropped
]
# Non-ASCII names that must stay allowed: a genuine IDN, and a fullwidth
# spelling that folds to a NEARBY (non-metadata) name.
NON_ASCII_CONTROLS = [
"bücher.example",
"api.softlayer.com", # folds to api.softlayer.com
"api。softlayer。com",
]
@pytest.mark.parametrize("host", IDNA_SPELLINGS)
def test_is_metadata_hostname_folds_idna_spellings(host):
assert is_metadata_hostname(host) is True
@pytest.mark.parametrize("host", NON_ASCII_CONTROLS)
def test_is_metadata_hostname_allows_non_ascii_controls(host):
"""CONTROL: IDNA folding must not turn into a blanket non-ASCII block."""
assert is_metadata_hostname(host) is False
def test_is_metadata_hostname_fails_closed_on_unencodable_name():
"""A non-ASCII name the stdlib IDNA codec cannot encode (label > 63
octets) is treated as a match: the check must not fail open."""
assert is_metadata_hostname("a" * 64 + "ü.example") is True
@pytest.mark.parametrize("scope", list(EgressScope))
@pytest.mark.parametrize("host", IDNA_SPELLINGS)
def test_evaluate_url_blocks_idna_spellings_in_every_scope(scope, host):
with patch(
"local_deep_research.security.egress.policy._resolve_with_timeout",
return_value=IBM_ADDRINFO,
):
d = evaluate_url(f"https://{host}/", _ctx(scope))
assert not d.allowed
assert d.reason == "blocked_metadata_ip"
@pytest.mark.parametrize("host", NON_ASCII_CONTROLS)
def test_evaluate_url_allows_non_ascii_controls_under_private_only(host):
"""CONTROL: same RFC1918 answer, non-metadata non-ASCII name."""
with patch(
"local_deep_research.security.egress.policy._resolve_with_timeout",
return_value=IBM_ADDRINFO,
):
d = evaluate_url(f"https://{host}/", _ctx(EgressScope.PRIVATE_ONLY))
assert d.allowed, d.reason
@pytest.mark.parametrize("host", IDNA_SPELLINGS)
def test_classify_host_never_calls_idna_spelling_local(host):
with patch(
"local_deep_research.security.egress.policy._resolve_with_timeout",
return_value=IBM_ADDRINFO,
):
assert _classify_host(host, _ctx(EgressScope.PRIVATE_ONLY)) is False
def test_classify_host_does_not_call_a_non_ascii_name_local():
"""Changed by the canonical-host rule: host classification accepts only
plain ASCII DNS names and canonical IP literals, so a non-ASCII name is
not local even when its IDNA folding is an ordinary name on a private
address (it used to be classified local here). Writing the name in its
ASCII / ``xn--`` form keeps it classifiable, as the control shows. The
fetch-time check keeps classifying such a name by DNS
(``test_evaluate_url_allows_non_ascii_controls_under_private_only``)."""
with patch(
"local_deep_research.security.egress.policy._resolve_with_timeout",
return_value=IBM_ADDRINFO,
):
ctx = _ctx(EgressScope.PRIVATE_ONLY)
assert _classify_host("api.softlayer.com", ctx) is False
# CONTROL: the same name written in ASCII is still local.
assert _classify_host("api.softlayer.com", ctx) is True
@pytest.mark.parametrize("host", IDNA_SPELLINGS)
def test_engine_url_validator_refuses_idna_spellings_with_opt_in(host):
with patch.object(
validators, "_resolve_with_timeout", return_value=IBM_ADDRINFO
):
err = validators._engine_url_ssrf_error(
"search.engine.web.searxng.default_params.instance_url",
f"https://{host}/",
allow_private=True,
)
assert err is not None
assert "cloud-metadata" in err["error"]
def test_engine_url_validator_allows_non_ascii_control_with_opt_in():
"""CONTROL."""
with patch.object(
validators, "_resolve_with_timeout", return_value=IBM_ADDRINFO
):
err = validators._engine_url_ssrf_error(
"search.engine.web.searxng.default_params.instance_url",
"https://api.softlayer.com/",
allow_private=True,
)
assert err is None
UNENCODABLE_IDN = "a" * 64 + "ü.example"
def test_engine_url_validator_reports_unencodable_idn_as_invalid():
"""A non-ASCII name the IDNA codec cannot encode is refused (the
metadata-name check fails closed on it) but reported as an invalid
hostname, not as a cloud-metadata address."""
err = validators._engine_url_ssrf_error(
"search.engine.web.searxng.default_params.instance_url",
f"https://{UNENCODABLE_IDN}/",
allow_private=True,
)
assert err is not None
assert "invalid or unsupported hostname" in err["error"]
assert "cloud-metadata" not in err["error"]
def test_local_hostnames_validator_reports_unencodable_idn_as_invalid():
"""Any non-ASCII entry is refused as an invalid spelling for the list
(classification would ignore it), not as a cloud-metadata endpoint."""
result = validators.validate_allowed_local_hostnames(
{"llm.allowed_local_hostnames": [UNENCODABLE_IDN]}, {}
)
assert result is not None
assert "not valid hostnames" in result["error"]
assert "cloud-metadata" not in result["error"]
# A public name the stdlib IDNA 2003 codec rejects but IDNA 2008 accepts: a
# right-to-left label ending in a digit. urllib3 and httpx encode it with the
# idna package (IDNA 2008) and connect to ``xn--1-9hcuf1d.co.il``, so it is
# judged by that form instead of being refused as an unencodable name.
IDNA2008_ONLY_NAME = "\u05e9\u05dc\u05d5\u05dd1.co.il"
# A metadata name with a character UTS #46 ignores but the IDNA 2003 codec
# rejects (U+2063 INVISIBLE SEPARATOR): only the UTS #46 form is the listed
# name, so it is matched there rather than through the fail-closed path.
UTS46_ONLY_METADATA_SPELLING = "metadata.goo\u2063g"
def test_idna2008_only_name_is_not_a_metadata_name():
assert is_metadata_hostname(IDNA2008_ONLY_NAME) is False
@pytest.mark.parametrize(
"scope",
[EgressScope.PUBLIC_ONLY, EgressScope.BOTH, EgressScope.UNPROTECTED],
)
def test_evaluate_url_admits_idna2008_only_public_name(scope):
"""evaluate_url used to refuse such a name as blocked_metadata_ip in
every scope while validate_url (which sees urllib3's IDNA 2008 form)
admitted it. With a public DNS answer it is now an ordinary public
host."""
with patch(
"local_deep_research.security.egress.policy._resolve_with_timeout",
return_value=PUBLIC_ADDRINFO,
):
d = evaluate_url(f"https://{IDNA2008_ONLY_NAME}/", _ctx(scope))
assert d.allowed, d.reason
with _ssrf_dns(PUBLIC_ADDRINFO):
assert validate_url(f"https://{IDNA2008_ONLY_NAME}/") is True
def test_engine_url_validator_accepts_idna2008_only_public_name():
with patch.object(
validators, "_resolve_with_timeout", return_value=PUBLIC_ADDRINFO
):
err = validators._engine_url_ssrf_error(
"search.engine.web.searxng.default_params.instance_url",
f"https://{IDNA2008_ONLY_NAME}/",
allow_private=False,
)
assert err is None
@pytest.mark.parametrize("scope", list(EgressScope))
def test_metadata_spelling_only_uts46_folds_is_still_refused(scope):
"""A name is still refused when only one encoder's form is a listed
metadata name, and the engine URL validator reports it as a metadata
endpoint (an encoder does accept it), not as an unencodable name."""
assert is_metadata_hostname(UTS46_ONLY_METADATA_SPELLING) is True
with patch(
"local_deep_research.security.egress.policy._resolve_with_timeout",
return_value=PUBLIC_ADDRINFO,
):
d = evaluate_url(
f"https://{UTS46_ONLY_METADATA_SPELLING}/", _ctx(scope)
)
assert d.reason == "blocked_metadata_ip"
err = validators._engine_url_ssrf_error(
"search.engine.web.searxng.default_params.instance_url",
f"https://{UTS46_ONLY_METADATA_SPELLING}/",
allow_private=True,
)
assert err is not None
assert "cloud-metadata" in err["error"]
@pytest.mark.parametrize("host", IDNA_SPELLINGS)
def test_validate_url_refuses_idna_spellings(host):
"""validate_url / assert_base_url_safe refuse these through urllib3's
host parsing and the ASCII-only authority guard; pinned here so a
loosening of either is caught at this entry point too."""
with _ssrf_dns(IBM_ADDRINFO):
assert validate_url(f"https://{host}/", allow_private_ips=True) is False
@pytest.mark.parametrize("host", IDNA_SPELLINGS)
def test_notification_validator_refuses_idna_spellings(host):
"""Held by the ASCII-only authority guard; pinned for the same reason."""
for url in (f"https://{host}/hook", f"json://{host}/hook"):
with patch.object(
NotificationURLValidator,
"_resolve_hostname_ips",
return_value=IBM_RESOLVED,
):
ok, _ = NotificationURLValidator.validate_service_url(
url, allow_private_ips=True
)
assert ok is False, url
def test_evaluate_url_name_denial_survives_ip_helper_exception():
"""The metadata-NAME denial sits outside the try/except that guards the
IP helpers, so an exception there cannot swallow it. Under PUBLIC_ONLY
the name otherwise classifies public and would be ALLOWED."""
with (
patch(
"local_deep_research.security.ssrf_validator.is_ip_blocked",
side_effect=RuntimeError("boom"),
),
patch(
"local_deep_research.security.egress.policy._resolve_with_timeout",
return_value=IBM_ADDRINFO,
),
):
d = evaluate_url(f"https://{IBM_HOST}/", _ctx(EgressScope.PUBLIC_ONLY))
assert not d.allowed
assert d.reason == "blocked_metadata_ip"
# ---------------------------------------------------------------------------
# Elasticsearch engine: scope gate and constructor
# ---------------------------------------------------------------------------
#
# _classify_host reports a metadata host as NOT local (False). The engine URL
# fail-up used to read that False as "public" and turn the local-nature
# Elasticsearch engine public, so PUBLIC_ONLY (and BOTH) admitted it. The
# fail-up now leaves unclassified every host _classifiable_host refuses (a
# metadata name or address, or any non-canonical spelling), and the
# constructor validates every node before the client connects.
ES_HOSTS_SETTING = "search.engine.web.elasticsearch.default_params.hosts"
PUBLIC_IP = "93.184.216.34"
PUBLIC_ADDRINFO = [(socket.AF_INET, socket.SOCK_STREAM, 6, "", (PUBLIC_IP, 0))]
# Every shape the hosts setting takes: a list of URLs, a comma-separated
# string (split into nodes by the engine's _ensure_list), and a host
# dictionary. A private node sits next to the metadata one in the string
# form so the metadata node is not the only entry.
IBM_ES_HOSTS = [
[f"http://{IBM_HOST}:9200"],
f"http://10.0.0.42:9200,http://{IBM_HOST}:9200",
[{"host": IBM_HOST, "port": 9200, "scheme": "http"}],
]
METADATA_IP_ES_HOSTS = [
["http://169.254.169.254:9200"],
"http://10.0.0.42:9200,http://169.254.169.254:9200",
["http://[fd00:ec2::254]:9200"],
[{"host": "169.254.169.254", "port": 9200, "scheme": "http"}],
[{"host": "::ffff:169.254.169.254", "port": 9200, "scheme": "http"}],
]
# PUBLIC_ONLY and BOTH admitted the engine before the fix (it failed up to
# public); PRIVATE_ONLY and STRICT would admit it if the name block in
# _classify_host were lost (the IBM name resolves to RFC1918).
ES_PROTECTED_SCOPES = [
EgressScope.PUBLIC_ONLY,
EgressScope.BOTH,
EgressScope.PRIVATE_ONLY,
EgressScope.STRICT,
]
def _es_ctx(scope):
return EgressContext(
scope=scope,
primary_engine="elasticsearch",
require_local_llm=False,
require_local_embeddings=False,
)
def _evaluate_es(hosts, scope, addrinfo):
with patch(
"local_deep_research.security.egress.policy._resolve_with_timeout",
return_value=addrinfo,
):
return evaluate_engine(
"elasticsearch",
_es_ctx(scope),
settings_snapshot={ES_HOSTS_SETTING: hosts},
)
@pytest.mark.parametrize("scope", ES_PROTECTED_SCOPES)
@pytest.mark.parametrize("hosts", IBM_ES_HOSTS)
def test_evaluate_engine_denies_elasticsearch_on_ibm_metadata_hostname(
scope, hosts
):
"""The IBM name resolves to RFC1918; neither the local nor the public
classification may admit it."""
d = _evaluate_es(hosts, scope, IBM_ADDRINFO)
assert not d.allowed
assert d.reason == "unclassified"
@pytest.mark.parametrize("scope", ES_PROTECTED_SCOPES)
@pytest.mark.parametrize("hosts", METADATA_IP_ES_HOSTS)
def test_evaluate_engine_denies_elasticsearch_on_literal_metadata_ip(
scope, hosts
):
"""A literal metadata address is classified without DNS. The resolver
is stubbed with a PUBLIC answer so that, should a literal ever reach it,
the test still fails rather than passing on a lookup error."""
d = _evaluate_es(hosts, scope, PUBLIC_ADDRINFO)
assert not d.allowed
assert d.reason == "unclassified"
@pytest.mark.parametrize(
"hosts",
[
[f"http://{PUBLIC_IP}:9200"],
["https://search.example.com:9243"],
[{"host": "search.example.com", "port": 9243, "scheme": "https"}],
],
)
def test_evaluate_engine_still_allows_public_elasticsearch_under_public_only(
hosts,
):
"""CONTROL: an ordinary public cluster still fails up to public and is
allowed under PUBLIC_ONLY."""
d = _evaluate_es(hosts, EgressScope.PUBLIC_ONLY, PUBLIC_ADDRINFO)
assert d.allowed, d.reason
def test_evaluate_engine_still_allows_nearby_hostname_under_private_only():
"""CONTROL: a non-metadata name on the same RFC1918 answer stays local."""
d = _evaluate_es(
["http://api.softlayer.com:9200"],
EgressScope.PRIVATE_ONLY,
IBM_ADDRINFO,
)
assert d.allowed, d.reason
# A backslash before "@" is a parser differential: urlparse (and so the
# probe-target check) reads the host after "@" (localhost), while urllib3 and
# elastic_transport, which the client uses, read the metadata host before the
# backslash. Only the constructor's second validate_url, on the entry as
# written, refuses these; with it removed the client is built for them.
BACKSLASH_USERINFO_ES_HOSTS = [
["http://169.254.169.254:80\\@localhost:9200"],
["http://metadata.google.internal:80\\@localhost:9200"],
[f"http://{IBM_HOST}:80\\@localhost:9200"],
]
@pytest.mark.parametrize(
"hosts,addrinfo",
[(h, IBM_ADDRINFO) for h in IBM_ES_HOSTS]
+ [(h, PUBLIC_ADDRINFO) for h in METADATA_IP_ES_HOSTS]
+ [(h, PUBLIC_ADDRINFO) for h in BACKSLASH_USERINFO_ES_HOSTS],
)
def test_elasticsearch_constructor_refuses_metadata_host(hosts, addrinfo):
"""Defence in depth: the client is never built for a metadata node, even
when no scope gate ran (unprotected scope, direct construction). The
resolver answers PUBLIC for every name, so a backslash-userinfo entry
passes the probe-target check (host "localhost") and is refused only by
validating the entry as written."""
with (
patch.object(es_engine, "Elasticsearch") as client_cls,
_ssrf_dns(addrinfo),
):
with pytest.raises(ValueError, match="SSRF validation"):
es_engine.ElasticsearchSearchEngine(hosts=hosts)
client_cls.assert_not_called()
@pytest.mark.parametrize(
"hosts,addrinfo",
[
([f"http://{PUBLIC_IP}:9200"], PUBLIC_ADDRINFO),
("https://search.example.com:9243", PUBLIC_ADDRINFO),
(
[{"host": "search.example.com", "port": 9243, "scheme": "https"}],
PUBLIC_ADDRINFO,
),
(["http://api.softlayer.com:9200"], IBM_ADDRINFO),
],
)
def test_elasticsearch_constructor_allows_ordinary_hosts(hosts, addrinfo):
"""CONTROL: a public cluster, and a LAN name on the IBM address, still
connect."""
with (
patch.object(es_engine, "Elasticsearch") as client_cls,
_ssrf_dns(addrinfo),
):
client_cls.return_value.info.return_value = {"cluster_name": "c"}
es_engine.ElasticsearchSearchEngine(hosts=hosts)
client_cls.assert_called_once()
# A host dictionary is not a URL. The client hands its ``host`` value to the
# connection pool unparsed and picks the pool by ``scheme`` alone: an HTTPS
# pool for "https", plain HTTP for any other value. The check therefore
# judges that (scheme, host, port) itself: a scheme other than http(s) is
# refused whatever it contains, and the host is never re-read out of a URL
# rebuilt from the dictionary. Every name resolves to the IBM address
# (RFC1918, allowed) here, so only that rule can refuse these entries.
NON_HTTP_SCHEME_ES_HOSTS = [
{"host": "169.254.169.254", "port": 80, "scheme": "http://localhost/#"},
{"host": "169.254.169.254", "port": 80, "scheme": "http://localhost?"},
{"host": IBM_HOST, "port": 80, "scheme": "http://127.0.0.1/#"},
{
"host": "metadata.google.internal",
"port": 80,
"scheme": "http://127.0.0.1#",
},
{"host": "fd00:ec2::254", "port": 80, "scheme": "http://localhost/#"},
{"host": "10.0.0.42", "port": 9200, "scheme": "ftp"},
]
@pytest.mark.parametrize("entry", NON_HTTP_SCHEME_ES_HOSTS)
def test_elasticsearch_constructor_refuses_dict_host_without_http_scheme(
entry,
):
with (
patch.object(es_engine, "Elasticsearch") as client_cls,
_ssrf_dns(IBM_ADDRINFO),
):
with pytest.raises(ValueError, match="SSRF validation"):
es_engine.ElasticsearchSearchEngine(hosts=[entry])
client_cls.assert_not_called()
@pytest.mark.parametrize("entry", NON_HTTP_SCHEME_ES_HOSTS)
def test_elasticsearch_probe_skips_dict_host_without_http_scheme(entry):
with (
patch("socket.create_connection") as connect,
_ssrf_dns(IBM_ADDRINFO),
):
assert (
es_engine.ElasticsearchSearchEngine._probe_hosts_available([entry])
is False
)
connect.assert_not_called()
@pytest.mark.parametrize(
"entry",
[
# A port the client would not connect to as given.
{"host": "10.0.0.42", "port": "9200", "scheme": "http"},
{"host": "10.0.0.42", "port": True, "scheme": "http"},
{"host": "10.0.0.42", "port": 9200.0, "scheme": "http"},
{"host": "10.0.0.42", "port": 0, "scheme": "http"},
{"host": "10.0.0.42", "port": 70000, "scheme": "http"},
# A host the connection pool would rewrite (it percent-decodes and
# IDNA-encodes) or the client refuses (brackets).
{"host": "10%2e0%2e0%2e42", "port": 9200, "scheme": "http"},
{"host": "metadata%2egoogle%2einternal", "port": 80, "scheme": "http"},
{"host": "fd12::1%eth0", "port": 9200, "scheme": "http"},
{"host": "[fd12::1]", "port": 9200, "scheme": "http"},
{
"host": "\uff41pi.service.softlayer.com",
"port": 80,
"scheme": "http",
},
# Not a host name at all.
{"host": "localhost#@169.254.169.254", "port": 80, "scheme": "http"},
{"host": "2852039166", "port": 80, "scheme": "http"},
# An address with a trailing dot, which a resolver may look up as a
# name.
{"host": "10.0.0.42.", "port": 9200, "scheme": "http"},
{"host": None, "port": 9200, "scheme": "http"},
],
)
def test_elasticsearch_check_refuses_unusable_dict_entry(entry):
with _ssrf_dns(IBM_ADDRINFO):
assert (
es_engine.ElasticsearchSearchEngine._host_passes_ssrf_check(entry)
is False
)
@pytest.mark.parametrize(
"entry,target",
[
(
{"host": "es.internal", "port": 9201, "scheme": "https"},
("https", "es.internal", 9201),
),
(
{"host": "es.internal", "port": 9201, "scheme": "HTTPS"},
("https", "es.internal", 9201),
),
({"host": "es.internal"}, ("http", "es.internal", 9200)),
(
{"host": "fd12::1", "port": 9200, "scheme": "http"},
("http", "fd12::1", 9200),
),
("https://es.internal", ("https", "es.internal", 443)),
("es.internal:9201", ("http", "es.internal", 9201)),
("http://[fd12::1]:9200", ("http", "fd12::1", 9200)),
# Parsed as the client parses it (urllib3), not as urlsplit does.
(
"http://169.254.169.254:80\\@localhost:9200",
("http", "169.254.169.254", 80),
),
({"host": "es.internal", "port": 9200, "scheme": "gopher"}, None),
({"host": "es.internal", "port": "9200", "scheme": "http"}, None),
],
)
def test_elasticsearch_probe_target_is_the_client_destination(entry, target):
assert es_engine.ElasticsearchSearchEngine._host_probe_target(entry) == (
target
)
@pytest.mark.parametrize(
"entry,destination",
[
(
{"host": "es.internal", "port": 9201, "scheme": "https"},
("es.internal", 9201),
),
("http://es.internal:9202", ("es.internal", 9202)),
("http://[fd12::1]:9200", ("fd12::1", 9200)),
],
)
def test_elasticsearch_probe_connects_to_the_checked_destination(
entry, destination
):
"""The probe parses each entry once: the destination the SSRF check
judged (and the only name it resolved) is the one it connects to."""
engine_cls = es_engine.ElasticsearchSearchEngine
check = engine_cls._target_passes_ssrf_check
checked = []
resolved = []
def record_check(host, target):
checked.append(target)
return check(host, target)
def resolve(host, *args, **kwargs):
resolved.append(host)
return IBM_ADDRINFO
with (
patch.object(
engine_cls, "_target_passes_ssrf_check", side_effect=record_check
),
patch(
"local_deep_research.security.ssrf_validator.socket.getaddrinfo",
side_effect=resolve,
),
patch("socket.create_connection") as connect,
):
assert engine_cls._probe_hosts_available([entry]) is True
connect.assert_called_once_with(destination, timeout=1.0)
assert [(t[1], t[2]) for t in checked] == [destination]
assert set(resolved) == {destination[0]}
# ---------------------------------------------------------------------------
# Non-canonical spellings of a metadata address
# ---------------------------------------------------------------------------
#
# Each entry reaches a listed metadata address: the resolver ignores trailing
# dots and accepts the inet_aton notations (integer, hex, octal, short
# forms), and an IPv4-mapped or NAT64 address carries the metadata IPv4 as
# its payload. The engine checks no longer need to recognise these: host
# classification accepts only canonical spellings (_canonical_host), so an
# alternate notation is unclassified whatever address it names, and a
# canonical metadata literal or wrap is refused by _is_metadata_host. The
# fetch-time check is broader on purpose: evaluate_url refuses every one of
# these as a metadata destination (_is_metadata_destination, over the forms
# _policy_host_forms produces). The resolver is stubbed with a PUBLIC answer:
# a spelling that slipped through would classify public and fail the local
# engine UP, which PUBLIC_ONLY and BOTH admit (and STRICT admits for the
# primary engine). The generated corpus in
# test_egress_host_canonical_classification.py covers far more spellings.
METADATA_IP_SPELLINGS = [
# trailing dots
"169.254.169.254.",
"169.254.169.254..",
"168.63.129.16.",
# inet_aton notations, bare and with a trailing dot
"2852039166",
"2852039166.",
"0xa9fea9fe",
"0XA9FEA9FE.",
"0251.0376.0251.0376",
"0251.0376.0251.0376.",
"0xa9.0xfe.0xa9.0xfe",
"169.254.43518",
"169.254.43518.",
"169.16689662",
"0xa83f8110", # 168.63.129.16 (Azure WireServer)
# non-ASCII spellings the resolver's IDNA encoding folds to the address
"\uff11\uff16\uff19.\uff12\uff15\uff14.\uff11\uff16\uff19.\uff12\uff15\uff14",
"169\u3002254\u3002169\u3002254",
"\uff12\uff18\uff15\uff12\uff10\uff13\uff19\uff11\uff16\uff16",
# IPv4-mapped, NAT64 (both well-known prefixes) and non-canonical IPv6
"[::ffff:169.254.169.254]",
"[::ffff:a9fe:a9fe]",
"[64:ff9b::169.254.169.254]",
"[64:ff9b::a9fe:a9fe]",
"[64:ff9b:1:a9fe:a9:fe00::]",
"[FD00:EC2:0:0:0:0:0:254]",
]
# Same notations, but not a metadata address.
NON_METADATA_SPELLINGS = [
"10.0.0.42.",
"0x0a00002a", # 10.0.0.42
"2852039234", # 169.254.170.66
"169.254.1.2",
"search.example.com.",
"\uff11\uff10.\uff10.\uff10.\uff14\uff12", # fullwidth 10.0.0.42
]
URL_ENGINE_SETTINGS = {
"elasticsearch": ES_HOSTS_SETTING,
"paperless": "search.engine.web.paperless.default_params.api_url",
}
def _evaluate_url_engine(engine, value, scope):
ctx = EgressContext(
scope=scope,
primary_engine=engine,
require_local_llm=False,
require_local_embeddings=False,
)
with patch(
"local_deep_research.security.egress.policy._resolve_with_timeout",
return_value=PUBLIC_ADDRINFO,
):
return evaluate_engine(
engine, ctx, settings_snapshot={URL_ENGINE_SETTINGS[engine]: value}
)
@pytest.mark.parametrize("scope", ES_PROTECTED_SCOPES)
@pytest.mark.parametrize("engine", list(URL_ENGINE_SETTINGS))
@pytest.mark.parametrize("host", METADATA_IP_SPELLINGS)
def test_evaluate_engine_denies_non_canonical_metadata_spelling(
scope, engine, host
):
d = _evaluate_url_engine(engine, f"http://{host}:9200", scope)
assert not d.allowed
assert d.reason == "unclassified"
@pytest.mark.parametrize("scope", ES_PROTECTED_SCOPES)
@pytest.mark.parametrize("host", METADATA_IP_SPELLINGS)
def test_evaluate_engine_denies_metadata_spelling_in_es_host_list(scope, host):
"""A metadata spelling next to a public node in a comma-separated hosts
string still withholds the engine."""
d = _evaluate_url_engine(
"elasticsearch",
f"http://{PUBLIC_IP}:9200,http://{host}:9200",
scope,
)
assert not d.allowed
assert d.reason == "unclassified"
@pytest.mark.parametrize("engine", list(URL_ENGINE_SETTINGS))
def test_evaluate_engine_trailing_dot_public_host_still_fails_up(engine):
"""CONTROL: stripping trailing dots does not withhold an ordinary host;
a public name with a trailing dot still fails up to public."""
d = _evaluate_url_engine(
engine, "https://search.example.com.:9243", EgressScope.PUBLIC_ONLY
)
assert d.allowed, d.reason
@pytest.mark.parametrize("engine", list(URL_ENGINE_SETTINGS))
def test_evaluate_engine_trailing_dot_private_literal_stays_local(engine):
"""CONTROL for the string branch's trailing-dot strip: a private literal
written with a trailing dot is classified as that literal (local), not
sent to the resolver, whose stub here answers PUBLIC."""
d = _evaluate_url_engine(
engine, "http://10.0.0.42.:9200", EgressScope.PRIVATE_ONLY
)
assert d.allowed, d.reason
@pytest.mark.parametrize("scope", list(EgressScope))
@pytest.mark.parametrize("host", METADATA_IP_SPELLINGS)
def test_evaluate_url_blocks_non_canonical_metadata_spelling(scope, host):
with patch(
"local_deep_research.security.egress.policy._resolve_with_timeout",
return_value=PUBLIC_ADDRINFO,
):
d = evaluate_url(f"http://{host}/", _ctx(scope))
assert not d.allowed
assert d.reason == "blocked_metadata_ip"
@pytest.mark.parametrize("host", METADATA_IP_SPELLINGS)
def test_is_metadata_destination_recognises_spelling(host):
assert _is_metadata_destination(host) is True
assert _is_metadata_destination(host.strip("[]")) is True
@pytest.mark.parametrize("host", NON_METADATA_SPELLINGS)
def test_is_metadata_destination_control(host):
"""CONTROL: the normaliser does not turn other addresses into
metadata."""
assert _is_metadata_destination(host) is False
@pytest.mark.parametrize("host", METADATA_IP_SPELLINGS + NON_METADATA_SPELLINGS)
def test_evaluate_url_and_is_metadata_destination_agree(host):
"""Both decide from the same normalised host, so they agree on every
spelling in the corpus."""
with patch(
"local_deep_research.security.egress.policy._resolve_with_timeout",
return_value=PUBLIC_ADDRINFO,
):
d = evaluate_url(f"http://{host}/", _ctx(EgressScope.PUBLIC_ONLY))
assert (d.reason == "blocked_metadata_ip") is _is_metadata_destination(host)
# NAT64 and zone-scope branches of _is_metadata_destination.
NAT64_METADATA = [
"64:ff9b::a9fe:a9fe",
"64:ff9b::169.254.169.254",
"64:ff9b::a9fe:aa02", # 169.254.170.2 (ECS)
"64:ff9b:1:a9fe:a9:fe00::", # /48 local-use prefix, RFC 6052 layout
]
ZONE_SCOPED_METADATA = [
"fd00:ec2::254%eth0",
"fe80::a9fe:a9fe%eth0",
"[fe80::a9fe:a9fe%25eth0]", # as urlsplit returns a zone in a URL
"::ffff:169.254.169.254%1",
]
@pytest.mark.parametrize("host", NAT64_METADATA)
def test_is_metadata_destination_nat64_branch(host):
assert _is_metadata_destination(host) is True
@pytest.mark.parametrize("host", ZONE_SCOPED_METADATA)
def test_is_metadata_destination_zone_scope_branch(host):
assert _is_metadata_destination(host) is True
@pytest.mark.parametrize(
"host",
["64:ff9b::808:808", "64:ff9b::a00:2a", "fe80::1%eth0", "fd00:ec2::253%1"],
)
def test_is_metadata_destination_nat64_and_zone_controls(host):
"""CONTROL: a NAT64 wrap of a non-metadata IPv4, and a zone-scoped
non-metadata address."""
assert _is_metadata_destination(host) is False
# ---------------------------------------------------------------------------
# A metadata address listed in llm.allowed_local_hostnames is still not local
# ---------------------------------------------------------------------------
LOCAL_HOSTNAME_METADATA_ENTRIES = [
"169.254.169.254",
"169\u3002254\u3002169\u3002254",
"fd00:ec2::254",
"169.254.169.254.",
"2852039166",
"0xa9fea9fe",
"::ffff:169.254.169.254",
"64:ff9b::a9fe:a9fe",
]
@pytest.mark.parametrize("entry", LOCAL_HOSTNAME_METADATA_ENTRIES)
def test_classify_host_ignores_local_hostnames_for_metadata_address(entry):
"""An entry that names a metadata endpoint (a canonical literal or wrap)
or has no canonical spelling (the other entries) is ignored by the
user-declared local_hostnames override, and the host itself is never
local. The resolver answers RFC1918, so only that rule keeps the entry
from classifying local."""
with patch(
"local_deep_research.security.egress.policy._resolve_with_timeout",
return_value=IBM_ADDRINFO,
):
ctx = _ctx(EgressScope.PRIVATE_ONLY, local_hostnames=[entry])
assert _classify_host(entry, ctx) is False
@pytest.mark.parametrize("entry", LOCAL_HOSTNAME_METADATA_ENTRIES)
def test_require_local_llm_refuses_metadata_address_in_local_hostnames(entry):
host = f"[{entry}]" if ":" in entry else entry
with patch(
"local_deep_research.security.egress.policy._resolve_with_timeout",
return_value=IBM_ADDRINFO,
):
d = evaluate_llm_endpoint(
"ollama",
_ctx(EgressScope.PRIVATE_ONLY, local_hostnames=[entry]),
settings_snapshot={"llm.ollama.url": f"http://{host}:11434"},
)
assert not d.allowed
assert d.reason == "provider_remote"
def test_require_local_llm_still_honours_local_hostnames_control():
"""CONTROL: an ordinary declared local name is still local."""
with patch(
"local_deep_research.security.egress.policy._resolve_with_timeout",
return_value=PUBLIC_ADDRINFO,
):
d = evaluate_llm_endpoint(
"ollama",
_ctx(EgressScope.PRIVATE_ONLY, local_hostnames=["gpu-box.lan"]),
settings_snapshot={"llm.ollama.url": "http://gpu-box.lan:11434"},
)
assert d.allowed, d.reason
# ---------------------------------------------------------------------------
# Error wording names a metadata endpoint as such
# ---------------------------------------------------------------------------
@pytest.mark.parametrize("entry", [IBM_HOST, "169.254.169.254"])
def test_local_hostnames_validator_names_metadata_entry(entry):
"""The IBM name resolves to RFC1918, so "resolves to PUBLIC addresses"
would be wrong; the entry is refused as a metadata endpoint."""
with patch.object(
validators, "_resolve_with_timeout", return_value=IBM_ADDRINFO
):
result = validators.validate_allowed_local_hostnames(
{"llm.allowed_local_hostnames": [entry]}, {}
)
assert result is not None
assert "cloud-metadata" in result["error"]
assert "PUBLIC" not in result["error"]
def test_notification_validator_names_metadata_hostname():
"""A name block must not be reported as an IP-address block."""
with patch.object(
NotificationURLValidator,
"_resolve_hostname_ips",
return_value=IBM_RESOLVED,
):
ok, err = NotificationURLValidator.validate_service_url(
f"https://{IBM_HOST}/hook", allow_private_ips=True
)
assert ok is False
assert err.startswith("Blocked cloud-metadata hostname:")