Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

44.0 KB
Raw Blame
"""Cloud-metadata HOSTNAME block (issue #6869).

IBM Cloud classic infrastructure serves its instance metadata
(``SoftLayer_Resource_Metadata``) from ``api.service.softlayer.com``, which
resolves to ``10.0.80.88`` -- an RFC1918 address. Every IP-based check
therefore passes it whenever private addresses are allowed
(``allow_private_ips=True``, the LLM base-URL path, the notification
plugin partition, STRICT / PRIVATE_ONLY egress). The name is blocked instead,
from the single list ``ssrf_validator.ALWAYS_BLOCKED_METADATA_HOSTNAMES``,
at every entry point that classifies a destination host.

Each test resolves the name to the REAL IBM address (10.0.80.88) and uses the
most permissive flags, so only the name block can refuse it: removing the
IBM entry from the list, or dropping the ``is_metadata_hostname`` call at the
entry point under test, makes the test fail.

The controls pin the deliberate trade-off: 10.0.80.88 itself stays allowed
under ``allow_private_ips=True`` (a LAN service may legitimately live there),
and neighbouring SoftLayer names are not over-blocked. The name block does
NOT stop a different DNS name that resolves to 10.0.80.88; that limit is
documented next to the list.
"""

import ipaddress
import socket
from unittest.mock import patch

import pytest

from local_deep_research.security.egress import validators
from local_deep_research.security.egress.policy import (
    EgressContext,
    EgressScope,
    _classify_host,
    _is_metadata_destination,
    evaluate_engine,
    evaluate_llm_endpoint,
    evaluate_url,
)
from local_deep_research.security.notification_validator import (
    NotificationURLValidator,
)
from local_deep_research.security.ssrf_validator import (
    ALWAYS_BLOCKED_METADATA_HOSTNAMES,
    ALWAYS_BLOCKED_METADATA_IPS,
    assert_base_url_safe,
    is_metadata_hostname,
    validate_url,
)
from local_deep_research.web_search_engines.engines import (
    search_engine_elasticsearch as es_engine,
)

IBM_HOST = "api.service.softlayer.com"
IBM_IP = "10.0.80.88"
IBM_ADDRINFO = [(socket.AF_INET, socket.SOCK_STREAM, 6, "", (IBM_IP, 0))]
IBM_RESOLVED = [ipaddress.IPv4Address(IBM_IP)]

# Spellings the resolver / HTTP client treat as the same host.
IBM_SPELLINGS = [
    IBM_HOST,
    "API.SERVICE.SOFTLAYER.COM",
    "Api.Service.SoftLayer.Com",
    f"{IBM_HOST}.",
    f"{IBM_HOST}..",
    "API.Service.SoftLayer.com.",
    "api%2eservice%2esoftlayer%2ecom",
]

# Nearby names that are NOT the metadata endpoint. Resolved to the same
# RFC1918 address in the controls, so only the exact-name match separates
# them from IBM_HOST.
NEARBY_NON_METADATA = [
    "api.softlayer.com",
    "service.softlayer.com",
    "softlayer.com",
    "xapi.service.softlayer.com",
    f"{IBM_HOST}.attacker.example",
    "api-service.softlayer.com",
]


def _ssrf_dns(addrinfo):
    return patch(
        "local_deep_research.security.ssrf_validator.socket.getaddrinfo",
        return_value=addrinfo,
    )


# ---------------------------------------------------------------------------
# Single source of truth
# ---------------------------------------------------------------------------


def test_ibm_classic_metadata_hostname_is_listed():
    assert IBM_HOST in ALWAYS_BLOCKED_METADATA_HOSTNAMES


def test_ibm_rfc1918_address_is_not_always_blocked():
    """The trade-off: 10.0.80.88 is RFC1918 and must NOT be an absolute
    block, or a user's own LAN service at that address would break.
    (161.26.0.6 is IBM's package mirror / NTP, not metadata, and must not be
    listed either.)"""
    assert IBM_IP not in ALWAYS_BLOCKED_METADATA_IPS
    assert "161.26.0.6" not in ALWAYS_BLOCKED_METADATA_IPS


@pytest.mark.parametrize("host", IBM_SPELLINGS)
def test_is_metadata_hostname_normalizes_case_and_trailing_dot(host):
    assert is_metadata_hostname(host) is True


@pytest.mark.parametrize("host", NEARBY_NON_METADATA)
def test_is_metadata_hostname_is_exact_match(host):
    assert is_metadata_hostname(host) is False


# ---------------------------------------------------------------------------
# validate_url / assert_base_url_safe
# ---------------------------------------------------------------------------


@pytest.mark.parametrize("host", IBM_SPELLINGS)
@pytest.mark.parametrize(
    "flags",
    [
        {},
        {"allow_localhost": True},
        {"allow_private_ips": True},
        {"allow_localhost": True, "allow_private_ips": True},
    ],
)
def test_validate_url_blocks_ibm_metadata_hostname_under_all_flags(host, flags):
    with _ssrf_dns(IBM_ADDRINFO):
        assert (
            validate_url(
                f"https://{host}/rest/v3.1/SoftLayer_Resource_Metadata/getId",
                **flags,
            )
            is False
        )


def test_assert_base_url_safe_refuses_ibm_metadata_hostname():
    """The LLM base-URL path uses allow_private_ips=True, so the RFC1918
    answer alone would pass; only the name block refuses it."""
    with _ssrf_dns(IBM_ADDRINFO):
        with pytest.raises(ValueError):
            assert_base_url_safe(
                f"https://{IBM_HOST}./v1", setting_key="llm.openai.url"
            )


@pytest.mark.parametrize("host", NEARBY_NON_METADATA)
def test_validate_url_allows_nearby_hostname_on_same_address(host):
    """CONTROL: same RFC1918 answer, different name -> allowed under
    allow_private_ips=True. Fails if the block is widened to a suffix /
    substring match or to the IP itself."""
    with _ssrf_dns(IBM_ADDRINFO):
        assert validate_url(f"https://{host}/", allow_private_ips=True) is True


def test_validate_url_allows_lan_service_at_ibm_address():
    """CONTROL: a LAN service on 10.0.80.88 keeps working under
    allow_private_ips=True (the documented reason the IP is not blocked)."""
    assert validate_url(f"http://{IBM_IP}:11434/", allow_private_ips=True)


# ---------------------------------------------------------------------------
# Egress policy
# ---------------------------------------------------------------------------


def _ctx(scope, local_hostnames=()):
    return EgressContext(
        scope=scope,
        primary_engine="test",
        require_local_llm=True,
        require_local_embeddings=True,
        local_hostnames=tuple(local_hostnames),
    )


@pytest.mark.parametrize("scope", list(EgressScope))
@pytest.mark.parametrize("host", IBM_SPELLINGS)
def test_evaluate_url_blocks_ibm_metadata_hostname_in_every_scope(scope, host):
    with patch(
        "local_deep_research.security.egress.policy._resolve_with_timeout",
        return_value=IBM_ADDRINFO,
    ):
        d = evaluate_url(f"https://{host}/", _ctx(scope))
    assert not d.allowed
    assert d.reason == "blocked_metadata_ip"


def test_evaluate_url_allows_nearby_hostname_under_private_only():
    """CONTROL: a nearby name on the same RFC1918 answer is local and allowed
    under PRIVATE_ONLY."""
    with patch(
        "local_deep_research.security.egress.policy._resolve_with_timeout",
        return_value=IBM_ADDRINFO,
    ):
        d = evaluate_url(
            "https://api.softlayer.com/", _ctx(EgressScope.PRIVATE_ONLY)
        )
    assert d.allowed


def test_classify_host_never_calls_ibm_metadata_hostname_local():
    """Reached by evaluate_llm_endpoint / evaluate_embeddings / engines,
    which do not go through evaluate_url. The RFC1918 answer would classify
    it local; it must not -- even if the user listed it as a local host."""
    with patch(
        "local_deep_research.security.egress.policy._resolve_with_timeout",
        return_value=IBM_ADDRINFO,
    ):
        ctx = _ctx(EgressScope.PRIVATE_ONLY, local_hostnames=[IBM_HOST])
        assert _classify_host(IBM_HOST, ctx) is False
        assert _classify_host("API.Service.SoftLayer.com.", ctx) is False
        # CONTROL: a nearby name on the same answer is still local.
        assert _classify_host("api.softlayer.com", ctx) is True


# ---------------------------------------------------------------------------
# Notification validator
# ---------------------------------------------------------------------------


@pytest.mark.parametrize(
    "url",
    [
        f"https://{IBM_HOST}/hook",
        "https://API.Service.SoftLayer.com./hook",
        f"json://{IBM_HOST}/hook",
        f"ntfy://{IBM_HOST}./topic",
    ],
)
def test_notification_validator_blocks_ibm_metadata_hostname(url):
    with patch.object(
        NotificationURLValidator,
        "_resolve_hostname_ips",
        return_value=IBM_RESOLVED,
    ):
        ok, err = NotificationURLValidator.validate_service_url(
            url, allow_private_ips=True
        )
    assert ok is False
    assert err


@pytest.mark.parametrize(
    "url", ["https://api.softlayer.com/hook", "json://api.softlayer.com/hook"]
)
def test_notification_validator_allows_nearby_hostname(url):
    """CONTROL: same RFC1918 answer, different name, private IPs allowed."""
    with patch.object(
        NotificationURLValidator,
        "_resolve_hostname_ips",
        return_value=IBM_RESOLVED,
    ):
        ok, err = NotificationURLValidator.validate_service_url(
            url, allow_private_ips=True
        )
    assert ok is True, err


# ---------------------------------------------------------------------------
# Search-engine URL settings validator
# ---------------------------------------------------------------------------


def test_engine_url_validator_refuses_ibm_metadata_hostname_with_opt_in():
    with patch.object(
        validators, "_resolve_with_timeout", return_value=IBM_ADDRINFO
    ):
        err = validators._engine_url_ssrf_error(
            "search.engine.web.searxng.default_params.instance_url",
            f"https://{IBM_HOST}./",
            allow_private=True,
        )
    assert err is not None
    assert "cloud-metadata" in err["error"]


def test_engine_url_validator_allows_nearby_hostname_with_opt_in():
    """CONTROL."""
    with patch.object(
        validators, "_resolve_with_timeout", return_value=IBM_ADDRINFO
    ):
        err = validators._engine_url_ssrf_error(
            "search.engine.web.searxng.default_params.instance_url",
            "https://api.softlayer.com/",
            allow_private=True,
        )
    assert err is None


def test_policy_keeps_no_private_copy_of_the_hostname_list():
    """Single source of truth: the egress policy must consult
    ssrf_validator.is_metadata_hostname, not a private list that can drift
    (the pre-#6869 ``_METADATA_HOSTNAMES`` held only the GCP names)."""
    from local_deep_research.security.egress import policy

    assert not hasattr(policy, "_METADATA_HOSTNAMES")


# ---------------------------------------------------------------------------
# IDNA look-alike spellings
#
# getaddrinfo IDNA-encodes a non-ASCII str host (stdlib nameprep) and httpx /
# requests use the idna package (UTS #46). Both fold the ideographic /
# fullwidth / halfwidth full stops to "." and fullwidth letters to ASCII, so
# these spellings connect to the IBM metadata host.
# ---------------------------------------------------------------------------

IDNA_SPELLINGS = [
    "api。service。softlayer。com",  # U+3002 ideographic stop
    "api.service.softlayer.com",  # U+FF0E fullwidth stop
    "api。service。softlayer。com",  # U+FF61 halfwidth stop
    "api.service.softlayer.com",  # fullwidth "api"
    # fullwidth upper-case letters and stops throughout
    "API.SERVICE.SOFTLAYER.COM",
    "api.service.softlayer.com。",  # trailing ideographic stop
    "api.servi­ce.softlayer.com",  # soft hyphen is dropped
]

# Non-ASCII names that must stay allowed: a genuine IDN, and a fullwidth
# spelling that folds to a NEARBY (non-metadata) name.
NON_ASCII_CONTROLS = [
    "bücher.example",
    "api.softlayer.com",  # folds to api.softlayer.com
    "api。softlayer。com",
]


@pytest.mark.parametrize("host", IDNA_SPELLINGS)
def test_is_metadata_hostname_folds_idna_spellings(host):
    assert is_metadata_hostname(host) is True


@pytest.mark.parametrize("host", NON_ASCII_CONTROLS)
def test_is_metadata_hostname_allows_non_ascii_controls(host):
    """CONTROL: IDNA folding must not turn into a blanket non-ASCII block."""
    assert is_metadata_hostname(host) is False


def test_is_metadata_hostname_fails_closed_on_unencodable_name():
    """A non-ASCII name the stdlib IDNA codec cannot encode (label > 63
    octets) is treated as a match: the check must not fail open."""
    assert is_metadata_hostname("a" * 64 + "ü.example") is True


@pytest.mark.parametrize("scope", list(EgressScope))
@pytest.mark.parametrize("host", IDNA_SPELLINGS)
def test_evaluate_url_blocks_idna_spellings_in_every_scope(scope, host):
    with patch(
        "local_deep_research.security.egress.policy._resolve_with_timeout",
        return_value=IBM_ADDRINFO,
    ):
        d = evaluate_url(f"https://{host}/", _ctx(scope))
    assert not d.allowed
    assert d.reason == "blocked_metadata_ip"


@pytest.mark.parametrize("host", NON_ASCII_CONTROLS)
def test_evaluate_url_allows_non_ascii_controls_under_private_only(host):
    """CONTROL: same RFC1918 answer, non-metadata non-ASCII name."""
    with patch(
        "local_deep_research.security.egress.policy._resolve_with_timeout",
        return_value=IBM_ADDRINFO,
    ):
        d = evaluate_url(f"https://{host}/", _ctx(EgressScope.PRIVATE_ONLY))
    assert d.allowed, d.reason


@pytest.mark.parametrize("host", IDNA_SPELLINGS)
def test_classify_host_never_calls_idna_spelling_local(host):
    with patch(
        "local_deep_research.security.egress.policy._resolve_with_timeout",
        return_value=IBM_ADDRINFO,
    ):
        assert _classify_host(host, _ctx(EgressScope.PRIVATE_ONLY)) is False


def test_classify_host_does_not_call_a_non_ascii_name_local():
    """Changed by the canonical-host rule: host classification accepts only
    plain ASCII DNS names and canonical IP literals, so a non-ASCII name is
    not local even when its IDNA folding is an ordinary name on a private
    address (it used to be classified local here). Writing the name in its
    ASCII / ``xn--`` form keeps it classifiable, as the control shows. The
    fetch-time check keeps classifying such a name by DNS
    (``test_evaluate_url_allows_non_ascii_controls_under_private_only``)."""
    with patch(
        "local_deep_research.security.egress.policy._resolve_with_timeout",
        return_value=IBM_ADDRINFO,
    ):
        ctx = _ctx(EgressScope.PRIVATE_ONLY)
        assert _classify_host("api.softlayer.com", ctx) is False
        # CONTROL: the same name written in ASCII is still local.
        assert _classify_host("api.softlayer.com", ctx) is True


@pytest.mark.parametrize("host", IDNA_SPELLINGS)
def test_engine_url_validator_refuses_idna_spellings_with_opt_in(host):
    with patch.object(
        validators, "_resolve_with_timeout", return_value=IBM_ADDRINFO
    ):
        err = validators._engine_url_ssrf_error(
            "search.engine.web.searxng.default_params.instance_url",
            f"https://{host}/",
            allow_private=True,
        )
    assert err is not None
    assert "cloud-metadata" in err["error"]


def test_engine_url_validator_allows_non_ascii_control_with_opt_in():
    """CONTROL."""
    with patch.object(
        validators, "_resolve_with_timeout", return_value=IBM_ADDRINFO
    ):
        err = validators._engine_url_ssrf_error(
            "search.engine.web.searxng.default_params.instance_url",
            "https://api.softlayer.com/",
            allow_private=True,
        )
    assert err is None


UNENCODABLE_IDN = "a" * 64 + "ü.example"


def test_engine_url_validator_reports_unencodable_idn_as_invalid():
    """A non-ASCII name the IDNA codec cannot encode is refused (the
    metadata-name check fails closed on it) but reported as an invalid
    hostname, not as a cloud-metadata address."""
    err = validators._engine_url_ssrf_error(
        "search.engine.web.searxng.default_params.instance_url",
        f"https://{UNENCODABLE_IDN}/",
        allow_private=True,
    )
    assert err is not None
    assert "invalid or unsupported hostname" in err["error"]
    assert "cloud-metadata" not in err["error"]


def test_local_hostnames_validator_reports_unencodable_idn_as_invalid():
    """Any non-ASCII entry is refused as an invalid spelling for the list
    (classification would ignore it), not as a cloud-metadata endpoint."""
    result = validators.validate_allowed_local_hostnames(
        {"llm.allowed_local_hostnames": [UNENCODABLE_IDN]}, {}
    )
    assert result is not None
    assert "not valid hostnames" in result["error"]
    assert "cloud-metadata" not in result["error"]


# A public name the stdlib IDNA 2003 codec rejects but IDNA 2008 accepts: a
# right-to-left label ending in a digit. urllib3 and httpx encode it with the
# idna package (IDNA 2008) and connect to ``xn--1-9hcuf1d.co.il``, so it is
# judged by that form instead of being refused as an unencodable name.
IDNA2008_ONLY_NAME = "\u05e9\u05dc\u05d5\u05dd1.co.il"

# A metadata name with a character UTS #46 ignores but the IDNA 2003 codec
# rejects (U+2063 INVISIBLE SEPARATOR): only the UTS #46 form is the listed
# name, so it is matched there rather than through the fail-closed path.
UTS46_ONLY_METADATA_SPELLING = "metadata.goo\u2063g"


def test_idna2008_only_name_is_not_a_metadata_name():
    assert is_metadata_hostname(IDNA2008_ONLY_NAME) is False


@pytest.mark.parametrize(
    "scope",
    [EgressScope.PUBLIC_ONLY, EgressScope.BOTH, EgressScope.UNPROTECTED],
)
def test_evaluate_url_admits_idna2008_only_public_name(scope):
    """evaluate_url used to refuse such a name as blocked_metadata_ip in
    every scope while validate_url (which sees urllib3's IDNA 2008 form)
    admitted it. With a public DNS answer it is now an ordinary public
    host."""
    with patch(
        "local_deep_research.security.egress.policy._resolve_with_timeout",
        return_value=PUBLIC_ADDRINFO,
    ):
        d = evaluate_url(f"https://{IDNA2008_ONLY_NAME}/", _ctx(scope))
    assert d.allowed, d.reason
    with _ssrf_dns(PUBLIC_ADDRINFO):
        assert validate_url(f"https://{IDNA2008_ONLY_NAME}/") is True


def test_engine_url_validator_accepts_idna2008_only_public_name():
    with patch.object(
        validators, "_resolve_with_timeout", return_value=PUBLIC_ADDRINFO
    ):
        err = validators._engine_url_ssrf_error(
            "search.engine.web.searxng.default_params.instance_url",
            f"https://{IDNA2008_ONLY_NAME}/",
            allow_private=False,
        )
    assert err is None


@pytest.mark.parametrize("scope", list(EgressScope))
def test_metadata_spelling_only_uts46_folds_is_still_refused(scope):
    """A name is still refused when only one encoder's form is a listed
    metadata name, and the engine URL validator reports it as a metadata
    endpoint (an encoder does accept it), not as an unencodable name."""
    assert is_metadata_hostname(UTS46_ONLY_METADATA_SPELLING) is True
    with patch(
        "local_deep_research.security.egress.policy._resolve_with_timeout",
        return_value=PUBLIC_ADDRINFO,
    ):
        d = evaluate_url(
            f"https://{UTS46_ONLY_METADATA_SPELLING}/", _ctx(scope)
        )
    assert d.reason == "blocked_metadata_ip"
    err = validators._engine_url_ssrf_error(
        "search.engine.web.searxng.default_params.instance_url",
        f"https://{UTS46_ONLY_METADATA_SPELLING}/",
        allow_private=True,
    )
    assert err is not None
    assert "cloud-metadata" in err["error"]


@pytest.mark.parametrize("host", IDNA_SPELLINGS)
def test_validate_url_refuses_idna_spellings(host):
    """validate_url / assert_base_url_safe refuse these through urllib3's
    host parsing and the ASCII-only authority guard; pinned here so a
    loosening of either is caught at this entry point too."""
    with _ssrf_dns(IBM_ADDRINFO):
        assert validate_url(f"https://{host}/", allow_private_ips=True) is False


@pytest.mark.parametrize("host", IDNA_SPELLINGS)
def test_notification_validator_refuses_idna_spellings(host):
    """Held by the ASCII-only authority guard; pinned for the same reason."""
    for url in (f"https://{host}/hook", f"json://{host}/hook"):
        with patch.object(
            NotificationURLValidator,
            "_resolve_hostname_ips",
            return_value=IBM_RESOLVED,
        ):
            ok, _ = NotificationURLValidator.validate_service_url(
                url, allow_private_ips=True
            )
        assert ok is False, url


def test_evaluate_url_name_denial_survives_ip_helper_exception():
    """The metadata-NAME denial sits outside the try/except that guards the
    IP helpers, so an exception there cannot swallow it. Under PUBLIC_ONLY
    the name otherwise classifies public and would be ALLOWED."""
    with (
        patch(
            "local_deep_research.security.ssrf_validator.is_ip_blocked",
            side_effect=RuntimeError("boom"),
        ),
        patch(
            "local_deep_research.security.egress.policy._resolve_with_timeout",
            return_value=IBM_ADDRINFO,
        ),
    ):
        d = evaluate_url(f"https://{IBM_HOST}/", _ctx(EgressScope.PUBLIC_ONLY))
    assert not d.allowed
    assert d.reason == "blocked_metadata_ip"


# ---------------------------------------------------------------------------
# Elasticsearch engine: scope gate and constructor
# ---------------------------------------------------------------------------
#
# _classify_host reports a metadata host as NOT local (False). The engine URL
# fail-up used to read that False as "public" and turn the local-nature
# Elasticsearch engine public, so PUBLIC_ONLY (and BOTH) admitted it. The
# fail-up now leaves unclassified every host _classifiable_host refuses (a
# metadata name or address, or any non-canonical spelling), and the
# constructor validates every node before the client connects.

ES_HOSTS_SETTING = "search.engine.web.elasticsearch.default_params.hosts"
PUBLIC_IP = "93.184.216.34"
PUBLIC_ADDRINFO = [(socket.AF_INET, socket.SOCK_STREAM, 6, "", (PUBLIC_IP, 0))]

# Every shape the hosts setting takes: a list of URLs, a comma-separated
# string (split into nodes by the engine's _ensure_list), and a host
# dictionary. A private node sits next to the metadata one in the string
# form so the metadata node is not the only entry.
IBM_ES_HOSTS = [
    [f"http://{IBM_HOST}:9200"],
    f"http://10.0.0.42:9200,http://{IBM_HOST}:9200",
    [{"host": IBM_HOST, "port": 9200, "scheme": "http"}],
]

METADATA_IP_ES_HOSTS = [
    ["http://169.254.169.254:9200"],
    "http://10.0.0.42:9200,http://169.254.169.254:9200",
    ["http://[fd00:ec2::254]:9200"],
    [{"host": "169.254.169.254", "port": 9200, "scheme": "http"}],
    [{"host": "::ffff:169.254.169.254", "port": 9200, "scheme": "http"}],
]

# PUBLIC_ONLY and BOTH admitted the engine before the fix (it failed up to
# public); PRIVATE_ONLY and STRICT would admit it if the name block in
# _classify_host were lost (the IBM name resolves to RFC1918).
ES_PROTECTED_SCOPES = [
    EgressScope.PUBLIC_ONLY,
    EgressScope.BOTH,
    EgressScope.PRIVATE_ONLY,
    EgressScope.STRICT,
]


def _es_ctx(scope):
    return EgressContext(
        scope=scope,
        primary_engine="elasticsearch",
        require_local_llm=False,
        require_local_embeddings=False,
    )


def _evaluate_es(hosts, scope, addrinfo):
    with patch(
        "local_deep_research.security.egress.policy._resolve_with_timeout",
        return_value=addrinfo,
    ):
        return evaluate_engine(
            "elasticsearch",
            _es_ctx(scope),
            settings_snapshot={ES_HOSTS_SETTING: hosts},
        )


@pytest.mark.parametrize("scope", ES_PROTECTED_SCOPES)
@pytest.mark.parametrize("hosts", IBM_ES_HOSTS)
def test_evaluate_engine_denies_elasticsearch_on_ibm_metadata_hostname(
    scope, hosts
):
    """The IBM name resolves to RFC1918; neither the local nor the public
    classification may admit it."""
    d = _evaluate_es(hosts, scope, IBM_ADDRINFO)
    assert not d.allowed
    assert d.reason == "unclassified"


@pytest.mark.parametrize("scope", ES_PROTECTED_SCOPES)
@pytest.mark.parametrize("hosts", METADATA_IP_ES_HOSTS)
def test_evaluate_engine_denies_elasticsearch_on_literal_metadata_ip(
    scope, hosts
):
    """A literal metadata address is classified without DNS. The resolver
    is stubbed with a PUBLIC answer so that, should a literal ever reach it,
    the test still fails rather than passing on a lookup error."""
    d = _evaluate_es(hosts, scope, PUBLIC_ADDRINFO)
    assert not d.allowed
    assert d.reason == "unclassified"


@pytest.mark.parametrize(
    "hosts",
    [
        [f"http://{PUBLIC_IP}:9200"],
        ["https://search.example.com:9243"],
        [{"host": "search.example.com", "port": 9243, "scheme": "https"}],
    ],
)
def test_evaluate_engine_still_allows_public_elasticsearch_under_public_only(
    hosts,
):
    """CONTROL: an ordinary public cluster still fails up to public and is
    allowed under PUBLIC_ONLY."""
    d = _evaluate_es(hosts, EgressScope.PUBLIC_ONLY, PUBLIC_ADDRINFO)
    assert d.allowed, d.reason


def test_evaluate_engine_still_allows_nearby_hostname_under_private_only():
    """CONTROL: a non-metadata name on the same RFC1918 answer stays local."""
    d = _evaluate_es(
        ["http://api.softlayer.com:9200"],
        EgressScope.PRIVATE_ONLY,
        IBM_ADDRINFO,
    )
    assert d.allowed, d.reason


# A backslash before "@" is a parser differential: urlparse (and so the
# probe-target check) reads the host after "@" (localhost), while urllib3 and
# elastic_transport, which the client uses, read the metadata host before the
# backslash. Only the constructor's second validate_url, on the entry as
# written, refuses these; with it removed the client is built for them.
BACKSLASH_USERINFO_ES_HOSTS = [
    ["http://169.254.169.254:80\\@localhost:9200"],
    ["http://metadata.google.internal:80\\@localhost:9200"],
    [f"http://{IBM_HOST}:80\\@localhost:9200"],
]


@pytest.mark.parametrize(
    "hosts,addrinfo",
    [(h, IBM_ADDRINFO) for h in IBM_ES_HOSTS]
    + [(h, PUBLIC_ADDRINFO) for h in METADATA_IP_ES_HOSTS]
    + [(h, PUBLIC_ADDRINFO) for h in BACKSLASH_USERINFO_ES_HOSTS],
)
def test_elasticsearch_constructor_refuses_metadata_host(hosts, addrinfo):
    """Defence in depth: the client is never built for a metadata node, even
    when no scope gate ran (unprotected scope, direct construction). The
    resolver answers PUBLIC for every name, so a backslash-userinfo entry
    passes the probe-target check (host "localhost") and is refused only by
    validating the entry as written."""
    with (
        patch.object(es_engine, "Elasticsearch") as client_cls,
        _ssrf_dns(addrinfo),
    ):
        with pytest.raises(ValueError, match="SSRF validation"):
            es_engine.ElasticsearchSearchEngine(hosts=hosts)
    client_cls.assert_not_called()


@pytest.mark.parametrize(
    "hosts,addrinfo",
    [
        ([f"http://{PUBLIC_IP}:9200"], PUBLIC_ADDRINFO),
        ("https://search.example.com:9243", PUBLIC_ADDRINFO),
        (
            [{"host": "search.example.com", "port": 9243, "scheme": "https"}],
            PUBLIC_ADDRINFO,
        ),
        (["http://api.softlayer.com:9200"], IBM_ADDRINFO),
    ],
)
def test_elasticsearch_constructor_allows_ordinary_hosts(hosts, addrinfo):
    """CONTROL: a public cluster, and a LAN name on the IBM address, still
    connect."""
    with (
        patch.object(es_engine, "Elasticsearch") as client_cls,
        _ssrf_dns(addrinfo),
    ):
        client_cls.return_value.info.return_value = {"cluster_name": "c"}
        es_engine.ElasticsearchSearchEngine(hosts=hosts)
    client_cls.assert_called_once()


# A host dictionary is not a URL. The client hands its ``host`` value to the
# connection pool unparsed and picks the pool by ``scheme`` alone: an HTTPS
# pool for "https", plain HTTP for any other value. The check therefore
# judges that (scheme, host, port) itself: a scheme other than http(s) is
# refused whatever it contains, and the host is never re-read out of a URL
# rebuilt from the dictionary. Every name resolves to the IBM address
# (RFC1918, allowed) here, so only that rule can refuse these entries.
NON_HTTP_SCHEME_ES_HOSTS = [
    {"host": "169.254.169.254", "port": 80, "scheme": "http://localhost/#"},
    {"host": "169.254.169.254", "port": 80, "scheme": "http://localhost?"},
    {"host": IBM_HOST, "port": 80, "scheme": "http://127.0.0.1/#"},
    {
        "host": "metadata.google.internal",
        "port": 80,
        "scheme": "http://127.0.0.1#",
    },
    {"host": "fd00:ec2::254", "port": 80, "scheme": "http://localhost/#"},
    {"host": "10.0.0.42", "port": 9200, "scheme": "ftp"},
]


@pytest.mark.parametrize("entry", NON_HTTP_SCHEME_ES_HOSTS)
def test_elasticsearch_constructor_refuses_dict_host_without_http_scheme(
    entry,
):
    with (
        patch.object(es_engine, "Elasticsearch") as client_cls,
        _ssrf_dns(IBM_ADDRINFO),
    ):
        with pytest.raises(ValueError, match="SSRF validation"):
            es_engine.ElasticsearchSearchEngine(hosts=[entry])
    client_cls.assert_not_called()


@pytest.mark.parametrize("entry", NON_HTTP_SCHEME_ES_HOSTS)
def test_elasticsearch_probe_skips_dict_host_without_http_scheme(entry):
    with (
        patch("socket.create_connection") as connect,
        _ssrf_dns(IBM_ADDRINFO),
    ):
        assert (
            es_engine.ElasticsearchSearchEngine._probe_hosts_available([entry])
            is False
        )
    connect.assert_not_called()


@pytest.mark.parametrize(
    "entry",
    [
        # A port the client would not connect to as given.
        {"host": "10.0.0.42", "port": "9200", "scheme": "http"},
        {"host": "10.0.0.42", "port": True, "scheme": "http"},
        {"host": "10.0.0.42", "port": 9200.0, "scheme": "http"},
        {"host": "10.0.0.42", "port": 0, "scheme": "http"},
        {"host": "10.0.0.42", "port": 70000, "scheme": "http"},
        # A host the connection pool would rewrite (it percent-decodes and
        # IDNA-encodes) or the client refuses (brackets).
        {"host": "10%2e0%2e0%2e42", "port": 9200, "scheme": "http"},
        {"host": "metadata%2egoogle%2einternal", "port": 80, "scheme": "http"},
        {"host": "fd12::1%eth0", "port": 9200, "scheme": "http"},
        {"host": "[fd12::1]", "port": 9200, "scheme": "http"},
        {
            "host": "\uff41pi.service.softlayer.com",
            "port": 80,
            "scheme": "http",
        },
        # Not a host name at all.
        {"host": "localhost#@169.254.169.254", "port": 80, "scheme": "http"},
        {"host": "2852039166", "port": 80, "scheme": "http"},
        # An address with a trailing dot, which a resolver may look up as a
        # name.
        {"host": "10.0.0.42.", "port": 9200, "scheme": "http"},
        {"host": None, "port": 9200, "scheme": "http"},
    ],
)
def test_elasticsearch_check_refuses_unusable_dict_entry(entry):
    with _ssrf_dns(IBM_ADDRINFO):
        assert (
            es_engine.ElasticsearchSearchEngine._host_passes_ssrf_check(entry)
            is False
        )


@pytest.mark.parametrize(
    "entry,target",
    [
        (
            {"host": "es.internal", "port": 9201, "scheme": "https"},
            ("https", "es.internal", 9201),
        ),
        (
            {"host": "es.internal", "port": 9201, "scheme": "HTTPS"},
            ("https", "es.internal", 9201),
        ),
        ({"host": "es.internal"}, ("http", "es.internal", 9200)),
        (
            {"host": "fd12::1", "port": 9200, "scheme": "http"},
            ("http", "fd12::1", 9200),
        ),
        ("https://es.internal", ("https", "es.internal", 443)),
        ("es.internal:9201", ("http", "es.internal", 9201)),
        ("http://[fd12::1]:9200", ("http", "fd12::1", 9200)),
        # Parsed as the client parses it (urllib3), not as urlsplit does.
        (
            "http://169.254.169.254:80\\@localhost:9200",
            ("http", "169.254.169.254", 80),
        ),
        ({"host": "es.internal", "port": 9200, "scheme": "gopher"}, None),
        ({"host": "es.internal", "port": "9200", "scheme": "http"}, None),
    ],
)
def test_elasticsearch_probe_target_is_the_client_destination(entry, target):
    assert es_engine.ElasticsearchSearchEngine._host_probe_target(entry) == (
        target
    )


@pytest.mark.parametrize(
    "entry,destination",
    [
        (
            {"host": "es.internal", "port": 9201, "scheme": "https"},
            ("es.internal", 9201),
        ),
        ("http://es.internal:9202", ("es.internal", 9202)),
        ("http://[fd12::1]:9200", ("fd12::1", 9200)),
    ],
)
def test_elasticsearch_probe_connects_to_the_checked_destination(
    entry, destination
):
    """The probe parses each entry once: the destination the SSRF check
    judged (and the only name it resolved) is the one it connects to."""
    engine_cls = es_engine.ElasticsearchSearchEngine
    check = engine_cls._target_passes_ssrf_check
    checked = []
    resolved = []

    def record_check(host, target):
        checked.append(target)
        return check(host, target)

    def resolve(host, *args, **kwargs):
        resolved.append(host)
        return IBM_ADDRINFO

    with (
        patch.object(
            engine_cls, "_target_passes_ssrf_check", side_effect=record_check
        ),
        patch(
            "local_deep_research.security.ssrf_validator.socket.getaddrinfo",
            side_effect=resolve,
        ),
        patch("socket.create_connection") as connect,
    ):
        assert engine_cls._probe_hosts_available([entry]) is True
    connect.assert_called_once_with(destination, timeout=1.0)
    assert [(t[1], t[2]) for t in checked] == [destination]
    assert set(resolved) == {destination[0]}


# ---------------------------------------------------------------------------
# Non-canonical spellings of a metadata address
# ---------------------------------------------------------------------------
#
# Each entry reaches a listed metadata address: the resolver ignores trailing
# dots and accepts the inet_aton notations (integer, hex, octal, short
# forms), and an IPv4-mapped or NAT64 address carries the metadata IPv4 as
# its payload. The engine checks no longer need to recognise these: host
# classification accepts only canonical spellings (_canonical_host), so an
# alternate notation is unclassified whatever address it names, and a
# canonical metadata literal or wrap is refused by _is_metadata_host. The
# fetch-time check is broader on purpose: evaluate_url refuses every one of
# these as a metadata destination (_is_metadata_destination, over the forms
# _policy_host_forms produces). The resolver is stubbed with a PUBLIC answer:
# a spelling that slipped through would classify public and fail the local
# engine UP, which PUBLIC_ONLY and BOTH admit (and STRICT admits for the
# primary engine). The generated corpus in
# test_egress_host_canonical_classification.py covers far more spellings.

METADATA_IP_SPELLINGS = [
    # trailing dots
    "169.254.169.254.",
    "169.254.169.254..",
    "168.63.129.16.",
    # inet_aton notations, bare and with a trailing dot
    "2852039166",
    "2852039166.",
    "0xa9fea9fe",
    "0XA9FEA9FE.",
    "0251.0376.0251.0376",
    "0251.0376.0251.0376.",
    "0xa9.0xfe.0xa9.0xfe",
    "169.254.43518",
    "169.254.43518.",
    "169.16689662",
    "0xa83f8110",  # 168.63.129.16 (Azure WireServer)
    # non-ASCII spellings the resolver's IDNA encoding folds to the address
    "\uff11\uff16\uff19.\uff12\uff15\uff14.\uff11\uff16\uff19.\uff12\uff15\uff14",
    "169\u3002254\u3002169\u3002254",
    "\uff12\uff18\uff15\uff12\uff10\uff13\uff19\uff11\uff16\uff16",
    # IPv4-mapped, NAT64 (both well-known prefixes) and non-canonical IPv6
    "[::ffff:169.254.169.254]",
    "[::ffff:a9fe:a9fe]",
    "[64:ff9b::169.254.169.254]",
    "[64:ff9b::a9fe:a9fe]",
    "[64:ff9b:1:a9fe:a9:fe00::]",
    "[FD00:EC2:0:0:0:0:0:254]",
]

# Same notations, but not a metadata address.
NON_METADATA_SPELLINGS = [
    "10.0.0.42.",
    "0x0a00002a",  # 10.0.0.42
    "2852039234",  # 169.254.170.66
    "169.254.1.2",
    "search.example.com.",
    "\uff11\uff10.\uff10.\uff10.\uff14\uff12",  # fullwidth 10.0.0.42
]

URL_ENGINE_SETTINGS = {
    "elasticsearch": ES_HOSTS_SETTING,
    "paperless": "search.engine.web.paperless.default_params.api_url",
}


def _evaluate_url_engine(engine, value, scope):
    ctx = EgressContext(
        scope=scope,
        primary_engine=engine,
        require_local_llm=False,
        require_local_embeddings=False,
    )
    with patch(
        "local_deep_research.security.egress.policy._resolve_with_timeout",
        return_value=PUBLIC_ADDRINFO,
    ):
        return evaluate_engine(
            engine, ctx, settings_snapshot={URL_ENGINE_SETTINGS[engine]: value}
        )


@pytest.mark.parametrize("scope", ES_PROTECTED_SCOPES)
@pytest.mark.parametrize("engine", list(URL_ENGINE_SETTINGS))
@pytest.mark.parametrize("host", METADATA_IP_SPELLINGS)
def test_evaluate_engine_denies_non_canonical_metadata_spelling(
    scope, engine, host
):
    d = _evaluate_url_engine(engine, f"http://{host}:9200", scope)
    assert not d.allowed
    assert d.reason == "unclassified"


@pytest.mark.parametrize("scope", ES_PROTECTED_SCOPES)
@pytest.mark.parametrize("host", METADATA_IP_SPELLINGS)
def test_evaluate_engine_denies_metadata_spelling_in_es_host_list(scope, host):
    """A metadata spelling next to a public node in a comma-separated hosts
    string still withholds the engine."""
    d = _evaluate_url_engine(
        "elasticsearch",
        f"http://{PUBLIC_IP}:9200,http://{host}:9200",
        scope,
    )
    assert not d.allowed
    assert d.reason == "unclassified"


@pytest.mark.parametrize("engine", list(URL_ENGINE_SETTINGS))
def test_evaluate_engine_trailing_dot_public_host_still_fails_up(engine):
    """CONTROL: stripping trailing dots does not withhold an ordinary host;
    a public name with a trailing dot still fails up to public."""
    d = _evaluate_url_engine(
        engine, "https://search.example.com.:9243", EgressScope.PUBLIC_ONLY
    )
    assert d.allowed, d.reason


@pytest.mark.parametrize("engine", list(URL_ENGINE_SETTINGS))
def test_evaluate_engine_trailing_dot_private_literal_stays_local(engine):
    """CONTROL for the string branch's trailing-dot strip: a private literal
    written with a trailing dot is classified as that literal (local), not
    sent to the resolver, whose stub here answers PUBLIC."""
    d = _evaluate_url_engine(
        engine, "http://10.0.0.42.:9200", EgressScope.PRIVATE_ONLY
    )
    assert d.allowed, d.reason


@pytest.mark.parametrize("scope", list(EgressScope))
@pytest.mark.parametrize("host", METADATA_IP_SPELLINGS)
def test_evaluate_url_blocks_non_canonical_metadata_spelling(scope, host):
    with patch(
        "local_deep_research.security.egress.policy._resolve_with_timeout",
        return_value=PUBLIC_ADDRINFO,
    ):
        d = evaluate_url(f"http://{host}/", _ctx(scope))
    assert not d.allowed
    assert d.reason == "blocked_metadata_ip"


@pytest.mark.parametrize("host", METADATA_IP_SPELLINGS)
def test_is_metadata_destination_recognises_spelling(host):
    assert _is_metadata_destination(host) is True
    assert _is_metadata_destination(host.strip("[]")) is True


@pytest.mark.parametrize("host", NON_METADATA_SPELLINGS)
def test_is_metadata_destination_control(host):
    """CONTROL: the normaliser does not turn other addresses into
    metadata."""
    assert _is_metadata_destination(host) is False


@pytest.mark.parametrize("host", METADATA_IP_SPELLINGS + NON_METADATA_SPELLINGS)
def test_evaluate_url_and_is_metadata_destination_agree(host):
    """Both decide from the same normalised host, so they agree on every
    spelling in the corpus."""
    with patch(
        "local_deep_research.security.egress.policy._resolve_with_timeout",
        return_value=PUBLIC_ADDRINFO,
    ):
        d = evaluate_url(f"http://{host}/", _ctx(EgressScope.PUBLIC_ONLY))
    assert (d.reason == "blocked_metadata_ip") is _is_metadata_destination(host)


# NAT64 and zone-scope branches of _is_metadata_destination.
NAT64_METADATA = [
    "64:ff9b::a9fe:a9fe",
    "64:ff9b::169.254.169.254",
    "64:ff9b::a9fe:aa02",  # 169.254.170.2 (ECS)
    "64:ff9b:1:a9fe:a9:fe00::",  # /48 local-use prefix, RFC 6052 layout
]
ZONE_SCOPED_METADATA = [
    "fd00:ec2::254%eth0",
    "fe80::a9fe:a9fe%eth0",
    "[fe80::a9fe:a9fe%25eth0]",  # as urlsplit returns a zone in a URL
    "::ffff:169.254.169.254%1",
]


@pytest.mark.parametrize("host", NAT64_METADATA)
def test_is_metadata_destination_nat64_branch(host):
    assert _is_metadata_destination(host) is True


@pytest.mark.parametrize("host", ZONE_SCOPED_METADATA)
def test_is_metadata_destination_zone_scope_branch(host):
    assert _is_metadata_destination(host) is True


@pytest.mark.parametrize(
    "host",
    ["64:ff9b::808:808", "64:ff9b::a00:2a", "fe80::1%eth0", "fd00:ec2::253%1"],
)
def test_is_metadata_destination_nat64_and_zone_controls(host):
    """CONTROL: a NAT64 wrap of a non-metadata IPv4, and a zone-scoped
    non-metadata address."""
    assert _is_metadata_destination(host) is False


# ---------------------------------------------------------------------------
# A metadata address listed in llm.allowed_local_hostnames is still not local
# ---------------------------------------------------------------------------

LOCAL_HOSTNAME_METADATA_ENTRIES = [
    "169.254.169.254",
    "169\u3002254\u3002169\u3002254",
    "fd00:ec2::254",
    "169.254.169.254.",
    "2852039166",
    "0xa9fea9fe",
    "::ffff:169.254.169.254",
    "64:ff9b::a9fe:a9fe",
]


@pytest.mark.parametrize("entry", LOCAL_HOSTNAME_METADATA_ENTRIES)
def test_classify_host_ignores_local_hostnames_for_metadata_address(entry):
    """An entry that names a metadata endpoint (a canonical literal or wrap)
    or has no canonical spelling (the other entries) is ignored by the
    user-declared local_hostnames override, and the host itself is never
    local. The resolver answers RFC1918, so only that rule keeps the entry
    from classifying local."""
    with patch(
        "local_deep_research.security.egress.policy._resolve_with_timeout",
        return_value=IBM_ADDRINFO,
    ):
        ctx = _ctx(EgressScope.PRIVATE_ONLY, local_hostnames=[entry])
        assert _classify_host(entry, ctx) is False


@pytest.mark.parametrize("entry", LOCAL_HOSTNAME_METADATA_ENTRIES)
def test_require_local_llm_refuses_metadata_address_in_local_hostnames(entry):
    host = f"[{entry}]" if ":" in entry else entry
    with patch(
        "local_deep_research.security.egress.policy._resolve_with_timeout",
        return_value=IBM_ADDRINFO,
    ):
        d = evaluate_llm_endpoint(
            "ollama",
            _ctx(EgressScope.PRIVATE_ONLY, local_hostnames=[entry]),
            settings_snapshot={"llm.ollama.url": f"http://{host}:11434"},
        )
    assert not d.allowed
    assert d.reason == "provider_remote"


def test_require_local_llm_still_honours_local_hostnames_control():
    """CONTROL: an ordinary declared local name is still local."""
    with patch(
        "local_deep_research.security.egress.policy._resolve_with_timeout",
        return_value=PUBLIC_ADDRINFO,
    ):
        d = evaluate_llm_endpoint(
            "ollama",
            _ctx(EgressScope.PRIVATE_ONLY, local_hostnames=["gpu-box.lan"]),
            settings_snapshot={"llm.ollama.url": "http://gpu-box.lan:11434"},
        )
    assert d.allowed, d.reason


# ---------------------------------------------------------------------------
# Error wording names a metadata endpoint as such
# ---------------------------------------------------------------------------


@pytest.mark.parametrize("entry", [IBM_HOST, "169.254.169.254"])
def test_local_hostnames_validator_names_metadata_entry(entry):
    """The IBM name resolves to RFC1918, so "resolves to PUBLIC addresses"
    would be wrong; the entry is refused as a metadata endpoint."""
    with patch.object(
        validators, "_resolve_with_timeout", return_value=IBM_ADDRINFO
    ):
        result = validators.validate_allowed_local_hostnames(
            {"llm.allowed_local_hostnames": [entry]}, {}
        )
    assert result is not None
    assert "cloud-metadata" in result["error"]
    assert "PUBLIC" not in result["error"]


def test_notification_validator_names_metadata_hostname():
    """A name block must not be reported as an IP-address block."""
    with patch.object(
        NotificationURLValidator,
        "_resolve_hostname_ips",
        return_value=IBM_RESOLVED,
    ):
        ok, err = NotificationURLValidator.validate_service_url(
            f"https://{IBM_HOST}/hook", allow_private_ips=True
        )
    assert ok is False
    assert err.startswith("Blocked cloud-metadata hostname:")