Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

20.2 KB
Raw Blame
"""Regression: the bulk settings GET redacts every password-typed setting we
ship, including ones whose key is flat snake_case.

``GET /settings/api/bulk`` redacts via ``redact_value(key, None, value)`` — it
passes no ``ui_element``, so a password-typed setting is masked only if its key
name matches ``DEFAULT_SENSITIVE_KEYS``. That name check was originally an
exact match on the key's last *dotted* segment, which a flat snake_case key
such as ``local_search_milvus_token`` can never satisfy: with no dots in it,
its "leaf" is the entire key. Such settings shipped in the clear (#5762).

The heart of this module is ``test_every_shipped_password_setting_is_redacted``:
rather than hardcoding leaf names (the shape this file had after #5028, which
is why #5762 slipped through), it ENUMERATES the shipped
``defaults/**/*.json`` and asserts the suffix-only predicate covers every
setting declared ``ui_element: "password"``. Add a new secret setting whose key
the predicate does not recognize and this test fails before the leak ships.

IMPORTANT CAVEAT verified against the merge commit that added the broadened
underscore-suffix arm (#5771, 0a75d2df5): every one of the 34 settings this
test currently enumerates is a *dotted* key with an exact ``api_key`` or
``password`` leaf (e.g. ``llm.openai.api_key``), which arm 1 (exact match)
already redacted before #5771. No shipped setting has the flat snake_case
shape (``local_search_milvus_token``-style, no dots) that the broadened arm
exists to catch — that key is used only in this file's own docstrings and
synthetic test cases below. So this test currently passes identically with
the broadened arm reverted; it guards against a *regression* in the exact-
match arm and against a future password setting shaped so the qualifier
carve-out wrongly exempts it (see the comment above
``_NON_SECRET_LEAF_PREFIXES`` in ``data_sanitizer.py``), but it does not
exercise, and cannot fail on, the flat-key broadening itself. That coverage
lives in ``test_bulk_get_redacts_secret_key_shapes`` and
``test_bulk_get_redacts_named_secret`` below, which use synthetic
flat-key names precisely because no shipped setting takes that shape yet —
today, the broadened arm protects only USER-CREATED settings (e.g. a
self-hosted search engine's user-added ``*_token``/``*_api_key`` config),
not anything this project ships.

Its mirror, ``test_no_shipped_non_secret_setting_is_redacted``, pins the other
direction: widening the predicate must not start masking ordinary settings.
``search.engine.web.*.requires_api_key`` is a checkbox whose leaf ends in
``_api_key``; masking it would ship "[REDACTED]" to the settings UI in place of
true/false and make the write-back no-op guards refuse legitimate writes.
"""

import json
from pathlib import Path

import pytest
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker

from local_deep_research.database.models import Base
from local_deep_research.database.models.settings import Setting
from local_deep_research.security.data_sanitizer import DataSanitizer
from local_deep_research.settings.manager import SettingsManager

DEFAULTS_DIR = (
    Path(__file__).resolve().parents[2]
    / "src"
    / "local_deep_research"
    / "defaults"
)


def _iter_shipped_settings():
    """Yield ``(source, key, ui_element)`` for every setting in the defaults.

    The defaults are a mix of shapes — a flat ``{key: {...}}`` mapping, nested
    groups, and files that wrap the mapping — so walk generically and treat any
    dict carrying setting metadata as an entry rather than assuming a depth.
    """
    for path in sorted(DEFAULTS_DIR.rglob("*.json")):
        data = json.loads(path.read_text(encoding="utf-8"))
        stack = [data]
        while stack:
            node = stack.pop()
            if isinstance(node, dict):
                for key, value in node.items():
                    if isinstance(value, dict) and (
                        "ui_element" in value or "value" in value
                    ):
                        yield path.name, key, value.get("ui_element")
                    stack.append(value)
            elif isinstance(node, list):
                stack.extend(node)


SHIPPED_SETTINGS = list(_iter_shipped_settings())
SHIPPED_PASSWORD_SETTINGS = [
    (source, key) for source, key, ui in SHIPPED_SETTINGS if ui == "password"
]

# Settings that are deliberately redacted despite NOT being declared
# ui_element == "password" -- DEFAULT_SENSITIVE_KEYS masks these on purpose,
# so they are not a bug for test_no_shipped_non_secret_setting_is_redacted to
# catch. notifications.service_url is a "textarea" field, but it embeds
# credentials (e.g. discord://webhook_id/token, mailto://user:pass@host) --
# see the "service_url" entry in DataSanitizer.DEFAULT_SENSITIVE_KEYS.
_INTENTIONALLY_SENSITIVE_NON_PASSWORD_KEYS = {"notifications.service_url"}

SHIPPED_NON_SECRET_SETTINGS = [
    (source, key)
    for source, key, ui in SHIPPED_SETTINGS
    if ui != "password"
    and key not in _INTENTIONALLY_SENSITIVE_NON_PASSWORD_KEYS
]


def _bulk_shipped(key, value="topsecret-value"):
    """``redact_value`` applied the way the bulk GET applies it: with NO
    ``ui_element``, which is the asymmetry these tests exist to cover.

    NOT a stand-in for the endpoint (post-#6201 review, R2). An earlier
    docstring claimed this was "what ``GET /settings/api/bulk`` would put
    on the wire", which it is not: the route reaches ``redact_value``
    through the TYPED getter, so a container on a str-typed row arrives
    here as a Python repr rather than a container and ships in the clear
    (#6222). This helper answers only the NAME question -- does the
    sanitizer recognize ``key`` as a secret when it is given no
    ``ui_element`` to help it -- which is exactly the property the
    enumeration tests below assert over every shipped setting. The
    endpoint itself is pinned end-to-end in
    ``tests/security/test_settings_egress_and_secrets_fastapi.py::
    TestBulkSecretWriteBackAndEcho::
    test_bulk_settings_get_leaks_a_container_row_xfail_6222``.
    """
    return DataSanitizer.redact_value(key, None, value)


def test_defaults_enumeration_is_not_vacuously_empty():
    """Guard the guard: a defaults refactor that breaks the walk must not
    silently turn the enumerating tests below into no-ops."""
    assert len(SHIPPED_SETTINGS) > 100
    assert len(SHIPPED_PASSWORD_SETTINGS) > 15


@pytest.mark.parametrize(
    "source,key",
    SHIPPED_PASSWORD_SETTINGS,
    ids=[f"{source}:{key}" for source, key in SHIPPED_PASSWORD_SETTINGS],
)
def test_every_shipped_password_setting_is_redacted(source, key):
    """Every ``ui_element: "password"`` setting we ship must survive the
    bulk GET's suffix-only predicate, which sees no ui_element."""
    assert _bulk_shipped(key) == DataSanitizer.REDACTION_TEXT, (
        f"{key} (from {source}) is declared ui_element='password' but the "
        "bulk settings GET would ship its value in the clear. Either name it "
        "so DataSanitizer's key heuristic recognizes it (a *.api_key / "
        "*_token / *_password suffix) or add its name to DEFAULT_SENSITIVE_KEYS."
    )


@pytest.mark.parametrize(
    "source,key",
    SHIPPED_NON_SECRET_SETTINGS,
    ids=[f"{source}:{key}" for source, key in SHIPPED_NON_SECRET_SETTINGS],
)
def test_no_shipped_non_secret_setting_is_redacted(source, key):
    """The mirror: no ordinary setting may be masked by accident."""
    assert _bulk_shipped(key, "plain-value") == "plain-value", (
        f"{key} (from {source}) is not a password field but the bulk settings "
        "GET would mask its value, breaking the settings UI and the "
        "write-back no-op guards."
    )


@pytest.mark.parametrize(
    "key",
    [
        # The #5762 reproduction: flat snake_case, no dots, so the dotted
        # "leaf" is the whole key and an exact-match check finds nothing.
        "local_search_milvus_token",
        # Same shape, other secret names — the miss was never token-specific.
        "local_search_milvus_api_key",
        "some_service_password",
        "some_service_client_secret",
        # Classic dotted keys.
        "integrations.oauth.client_secret",
        "integrations.oauth.secret_key",
        "integrations.oauth.bearer_token",
        "integrations.oauth.api_secret",
        "integrations.oauth.app_secret",
        "integrations.oauth.api_token",
    ],
)
def test_bulk_get_redacts_secret_key_shapes(key):
    assert _bulk_shipped(key) == DataSanitizer.REDACTION_TEXT


@pytest.mark.parametrize(
    "key",
    [
        # Token COUNTS and capability flags are not secrets. These pin the two
        # carve-outs that keep the suffix rule from over-redacting.
        "llm.max_tokens",
        "llm.supports_max_tokens",
        "search.engine.web.brave.requires_api_key",
        "local_search_chunk_size",
        "llm.provider",
    ],
)
def test_bulk_get_leaves_non_secrets_readable(key):
    assert _bulk_shipped(key, "plain-value") == "plain-value"


def test_bulk_get_does_not_corrupt_non_string_lexical_match():
    """Type-guard regression: the broadened snake_case suffix match is a
    lexical heuristic backed by a hand-maintained non-secret-prefix
    carve-out (``_NON_SECRET_LEAF_PREFIXES``). A future boolean flag whose
    name lexically ends in a sensitive suffix but isn't covered by that
    carve-out (e.g. a hypothetical ``verify_token`` checkbox — "verify_" is
    not in the prefix list) must NOT be masked: replacing a real ``bool``
    with the "[REDACTED]" string corrupts it in the settings UI (a checkbox
    only renders ``checked`` for a literal ``True``) and can silently write
    back the wrong value on the next save. A same-shape STRING setting must
    still be redacted -- the type guard narrows by value type only, not by
    loosening the name match itself.
    """
    # Boolean lexically matching a sensitive suffix, uncovered by the
    # carve-out prefixes: must survive as a real bool, not "[REDACTED]".
    assert _bulk_shipped("verify_token", True) is True
    assert _bulk_shipped("verify_token", False) is False

    # Same hazard for a plain number.
    assert _bulk_shipped("retry_token", 3) == 3

    # The corresponding STRING setting is still redacted (#5762 stays fixed).
    assert (
        _bulk_shipped("milvus_api_token", "sk-fake-not-a-real-secret")
        == DataSanitizer.REDACTION_TEXT
    )


def test_bulk_get_masks_string_leaves_inside_a_broadened_match_container():
    """Container regression: a key that matches ONLY the broadened
    underscore-suffix arm (not the exact/password match) with a dict/list
    VALUE must still have its string leaves masked.

    Round 2's type guard stops a bool/int from being corrupted into the
    "[REDACTED]" string, but ``redact_value``'s normal recursion masks by
    SUB-KEY name -- so a secret nested under a non-sensitive sub-key such as
    ``{"value": "s3cr3t"}`` under a ``milvus_token`` setting would otherwise
    survive unredacted (sub-key ``value`` isn't itself a sensitive name).
    """
    assert _bulk_shipped("local_search_milvus_token", {"value": "s3cr3t"}) == {
        "value": DataSanitizer.REDACTION_TEXT
    }

    # Bool/int leaves inside the same container stay untouched -- the
    # string-only guard applies leaf-by-leaf, not to the whole container.
    assert _bulk_shipped(
        "local_search_milvus_token",
        {"value": "s3cr3t", "enabled": True, "retries": 3},
    ) == {
        "value": DataSanitizer.REDACTION_TEXT,
        "enabled": True,
        "retries": 3,
    }

    # List containers are covered too.
    assert _bulk_shipped(
        "local_search_milvus_token", [{"value": "s3cr3t"}, "plain-scalar"]
    ) == [{"value": DataSanitizer.REDACTION_TEXT}, DataSanitizer.REDACTION_TEXT]

    # Empty/whitespace-only string leaves stay readable, matching the
    # top-level empty-value rule (an unset nested field shouldn't look
    # configured).
    assert _bulk_shipped("local_search_milvus_token", {"value": "  "}) == {
        "value": "  "
    }


def test_bulk_get_leaves_qualifier_named_flag_readable_at_leaf_start():
    """The genuine, load-bearing carve-out: a leaf that STARTS WITH a
    ``_NON_SECRET_LEAF_PREFIXES`` qualifier stays readable.

    ``requires_api_key`` is a real, shipped shape --
    ``search.engine.web.*.requires_api_key`` for every search engine that
    needs one (``defaults/settings_*.json``) -- and the identical string is
    used verbatim as a config dict key throughout the search-engine
    factories (``tests/security/test_factory_secret_redaction.py``,
    ``tests/security/test_noncollection_agent_enabled.py``). Both the exact
    flat form and a dotted key's last segment must stay readable, for a
    bool and for a string.

    SCOPE NOTE: this is a CONTROL, not a pin on this round's narrowing. It
    passes identically against every prior version of
    ``_has_non_secret_qualifier`` -- the unanchored substring check, the
    ``startswith``-anchored prefix list, and the single-leaf carve-out all
    leave ``requires_api_key`` readable. What actually falsifies a revert
    of the narrowing is its two siblings below
    (``test_bulk_get_qualifier_mid_key_no_longer_exempts_a_real_secret``
    and the ``rag.skip_password`` / ``llm.x.use_auth_token`` case). This
    test exists to catch the opposite mistake: a narrowing that goes too
    far and starts redacting the one shape that must stay readable.
    """
    for key in ("requires_api_key", "search.engine.web.brave.requires_api_key"):
        assert _bulk_shipped(key, True) is True
        assert _bulk_shipped(key, False) is False
        assert _bulk_shipped(key, "plain-value") == "plain-value"


def test_bulk_get_qualifier_mid_key_no_longer_exempts_a_real_secret():
    """Anchoring regression for the post-#5771 review (defect 2):
    ``_has_non_secret_qualifier`` used to match a qualifier at ANY
    underscore boundary, not just the start of the leaf -- so a qualifier
    word appearing anywhere before an otherwise-unrelated sensitive suffix
    silently exempted a real secret from redaction.

    Each of these ships in the clear before the fix and must be redacted
    after it. None is the ``requires_api_key`` shape pinned above --
    a namespace segment leads every one of them, not a qualifier:

      * ``db_skip_password`` / ``vault_show_secret`` -- qualifier
        immediately precedes the noun, but is not the leaf's first segment.
      * ``x_needs_auth_token`` -- ``needs_`` precedes the 2-word compound
        ``auth_token``, still not the first segment.
      * ``local_search_use_milvus_token`` -- ``use_`` is two segments
        before the actual credential noun (``milvus_token``'s ``token``),
        the clearest case of the qualifier being unrelated to the secret it
        happened to exempt.

    A synthetic flat key that WOULD have been protected by the old
    mid-string rule -- ``llm_requires_api_key`` -- is included too: nothing
    in its shape distinguishes it from ``db_skip_password``, so it is no
    longer treated as safe for a string value (a bool/int stays protected
    regardless, by the type guard in ``redact_value``, independent of this
    predicate).
    """
    now_sensitive = [
        "db_skip_password",
        "x_needs_auth_token",
        "local_search_use_milvus_token",
        "vault_show_secret",
        "llm_requires_api_key",
    ]
    for key in now_sensitive:
        assert DataSanitizer.is_sensitive_setting(key), (
            f"{key!r} must be recognized as sensitive after anchoring the "
            "qualifier to the start of the leaf"
        )
        assert (
            _bulk_shipped(key, "a-real-secret-value")
            == DataSanitizer.REDACTION_TEXT
        )
        # The type guard still protects a bool/int leaf under a name-only
        # (non-exact) sensitive match, independent of the qualifier fix.
        assert _bulk_shipped(key, True) is True
        assert _bulk_shipped(key, False) is False

    # A real secret with an unrelated prefix segment is still redacted --
    # the carve-out must not become a blanket exemption for any multi-segment
    # flat key.
    assert (
        _bulk_shipped("llm_openai_api_key", "sk-fake-not-a-real-secret")
        == DataSanitizer.REDACTION_TEXT
    )


def test_qualifier_carve_out_does_not_extend_past_requires_api_key():
    """Post-#6201 review: ``_has_non_secret_qualifier`` used to exempt ANY
    leaf STARTING WITH ANY ``_NON_SECRET_LEAF_PREFIXES`` qualifier, combined
    with ANY sensitive suffix -- not just the one confirmed, load-bearing
    shape (``requires_api_key``) its own docstring claimed. A DOTTED key's
    last segment genuinely starting with a qualifier immediately followed
    by an unrelated sensitive suffix -- ``rag.skip_password``,
    ``llm.x.use_auth_token`` -- was exempted the same way, even though
    nothing confirms either is a boolean flag rather than a real credential
    stored under a badly-named key.

    ``requires_api_key`` itself, both as a flat key and as a dotted key's
    last segment, must stay the one exemption -- pinned already by
    ``test_bulk_get_leaves_qualifier_named_flag_readable_at_leaf_start``.
    """
    now_sensitive = [
        "rag.skip_password",
        "llm.x.use_auth_token",
        "skip_password",
        "use_auth_token",
    ]
    for key in now_sensitive:
        assert DataSanitizer.is_sensitive_setting(key), (
            f"{key!r} must be recognized as sensitive: only the exact "
            "requires_api_key shape is a carve-out"
        )
        assert (
            _bulk_shipped(key, "a-real-secret-value")
            == DataSanitizer.REDACTION_TEXT
        )
        # The type guard still protects a bool leaf under a name-only match.
        assert _bulk_shipped(key, True) is True
        assert _bulk_shipped(key, False) is False

    # requires_api_key is unaffected by the narrowing.
    for key in ("requires_api_key", "search.engine.web.brave.requires_api_key"):
        assert not DataSanitizer.is_sensitive_setting(key)
        assert _bulk_shipped(key, "plain-value") == "plain-value"


@pytest.fixture
def manager_factory():
    """Create managers and release their sessions and engines after a test."""
    resources = []

    def _make(settings):
        engine = create_engine("sqlite:///:memory:")
        Base.metadata.create_all(engine)
        session = sessionmaker(bind=engine)()
        session.add_all(settings)
        session.commit()
        manager = SettingsManager(db_session=session)
        resources.append((manager, session, engine))
        return manager

    yield _make

    for manager, session, engine in reversed(resources):
        manager.close()
        session.close()
        engine.dispose()


@pytest.mark.parametrize(
    "key",
    ["integrations.oauth.client_secret", "local_search_milvus_token"],
)
def test_bulk_get_redacts_named_secret(key, manager_factory):
    """End-to-end through a real SettingsManager, mirroring the endpoint."""
    manager = manager_factory(
        [
            Setting(
                key=key,
                value="topsecret-value",
                type="app",
                name=key,
                ui_element="password",
            )
        ]
    )

    # mirror GET /settings/api/bulk: get_setting(key) then redact_value(key, None, value)
    shipped = DataSanitizer.redact_value(key, None, manager.get_setting(key))
    assert shipped == DataSanitizer.REDACTION_TEXT


def test_bulk_get_redacts_named_secret_in_subtree(manager_factory):
    """A subtree request (keys[]=integrations.oauth) must redact a nested
    non-classic secret while keeping non-secret siblings — pins the added
    names composing with redact_value's subtree recursion (#5028)."""
    manager = manager_factory(
        [
            Setting(
                key="integrations.oauth.client_secret",
                value="topsecret-value",
                type="app",
                name="cs",
                ui_element="password",
            ),
            Setting(
                key="integrations.oauth.client_id",
                value="public-id-123",
                type="app",
                name="cid",
                ui_element="text",
            ),
        ]
    )

    shipped = DataSanitizer.redact_value(
        "integrations.oauth", None, manager.get_setting("integrations.oauth")
    )
    assert "topsecret-value" not in str(shipped)
    assert shipped["client_secret"] == DataSanitizer.REDACTION_TEXT
    assert shipped["client_id"] == "public-id-123"