~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
fix(security): redact flat snake_case secret settings from /settings/api/bulk (#5771)
LearningCircuit0a75d2df50aa
diff --git a/changelog.d/5771.security.md b/changelog.d/5771.security.mdnew file mode 100644index 0000000000..8041dc5098--- /dev/null+++ b/changelog.d/5771.security.md@@ -0,0 +1 @@+Redact password-typed settings whose key is flat snake_case (for example `local_search_milvus_token`) from `GET /settings/api/bulk`. That endpoint has no `ui_element` metadata, so it falls back to matching the key's last dotted segment against a set of secret names — a key with no dots in it has the whole key as its "leaf" and matched nothing, shipping the value in the clear. The name check now also matches a secret name at an underscore boundary, while leaving capability flags (`requires_api_key`) and token counts (`max_tokens`) readable.diff --git a/src/local_deep_research/security/data_sanitizer.py b/src/local_deep_research/security/data_sanitizer.pyindex 5347a75532..c526fbbf30 100644--- a/src/local_deep_research/security/data_sanitizer.py+++ b/src/local_deep_research/security/data_sanitizer.py@@ -85,6 +85,150 @@ def _is_empty_value(value: Any) -> bool:return value in (None, "", [], {})+# Leading qualifiers that mark a leaf as a boolean FLAG *about* a secret rather+# than the secret itself. ``search.engine.web.brave.requires_api_key`` is a+# checkbox saying an engine needs a key; masking it would ship "[REDACTED]" to+# the settings UI in place of true/false and make the write-back no-op guards+# refuse legitimate writes. Consulted only by the underscore-suffix arm below —+# a leaf that *equals* a sensitive name has no qualifier to strip.+# This is a name-shape heuristic, so it is accepted as under-redaction risk+# for a future setting that reuses one of these qualifier prefixes on a key+# that actually stores a real string credential (rather than a boolean flag)+# -- test_every_shipped_password_setting_is_redacted+# (tests/security/test_bulk_secret_name_coverage.py) audits every shipped+# password-typed setting's key shape and fails the moment such a setting+# ships, so the gap does not survive silently.+_NON_SECRET_LEAF_PREFIXES = (+ "allow_",+ "allows_",+ "disable_",+ "enable_",+ "enabled_",+ "has_",+ "is_",+ "need_",+ "needs_",+ "require_",+ "requires_",+ "show_",+ "skip_",+ "support_",+ "supports_",+ "use_",+ "uses_",+)+++def _has_non_secret_qualifier(leaf: str) -> bool:+ """True when ``leaf`` carries a ``_NON_SECRET_LEAF_PREFIXES`` qualifier+ at ANY underscore segment boundary, not only at the very start of the+ string.++ A dotted key's leaf is a single segment (``requires_api_key``), so a+ start-of-string check is enough on its own. But a flat snake_case key+ has no dots at all, so its "leaf" (see ``_matches_sensitive_name``) is+ the ENTIRE key — a qualifier that isn't the first segment, e.g.+ ``llm_requires_api_key``, would slip past a ``startswith`` check even+ though it is the exact same "flag about a secret" shape as+ ``requires_api_key``. Checking every underscore boundary (the start of+ the leaf, or immediately after an ``_``) catches the qualifier wherever+ it falls in the key, at the cost of also matching it as a middle+ segment (e.g. ``foo_use_bar``) — intentional, since the prefix list is+ a name-shape heuristic, not a position rule.+ """+ return any(+ leaf.startswith(prefix) or f"_{prefix}" in leaf+ for prefix in _NON_SECRET_LEAF_PREFIXES+ )+++def _matches_sensitive_name(leaf: str, sensitive_names: Set[str]) -> bool:+ """True when a normalized leaf names a secret.++ Two arms:++ 1. Exact match (``llm.openai.api_key`` -> ``api_key``).+ 2. Underscore-boundary suffix match. Settings keys use two separator+ conventions: dotted (``llm.openai.api_key``) and flat snake_case+ (``local_search_milvus_token``). A flat key has no dots, so its dotted+ "leaf" is the entire key and arm 1 can never match it — which is how a+ password-typed setting shipped in the clear from the bulk settings GET+ (#5762). Note the miss was never specific to ``token``:+ ``local_search_milvus_api_key`` or ``some_password`` would have evaded+ arm 1 just as completely.++ Arm 2 skips leaves carrying a ``_NON_SECRET_LEAF_PREFIXES`` qualifier+ anywhere at an underscore boundary (see ``_has_non_secret_qualifier``),+ so both ``requires_api_key`` (dotted leaf) and ``llm_requires_api_key``+ (flat leaf, qualifier not in the first segment) stay readable while+ ``milvus_api_key`` does not. Plural token counts (``max_tokens``,+ ``supports_max_tokens``) end in ``_tokens``, a different suffix than+ ``_token``, and stay readable too.+ """+ if leaf in sensitive_names:+ return True+ if _has_non_secret_qualifier(leaf):+ return False+ return any(leaf.endswith(f"_{name}") for name in sensitive_names)+++def _is_exact_sensitive_match(+ key: str, ui_element: str | None, sensitive_names: Set[str]+) -> bool:+ """True under the ORIGINAL (pre-#5771) sensitivity rule: ``ui_element ==+ "password"`` or an exact dotted-leaf match — i.e. everything+ ``is_sensitive_setting`` matched before the broadened underscore-suffix+ arm (arm 2 of ``_matches_sensitive_name``) was added.++ ``redact_value`` uses this to scope its non-string type guard to only+ the NEW broadened match. The exact/password arms have masked non-string+ values (e.g. a boolean under a key literally named ``api_key``) since+ before this PR; that established behavior is intentionally left as-is.+ The broadened arm is a lexical suffix heuristic backed by a+ hand-maintained non-secret-prefix carve-out (``_NON_SECRET_LEAF_PREFIXES``),+ so a boolean/number setting whose name happens to lexically match a+ sensitive suffix (a future ``verify_token``-style flag the carve-out+ list doesn't yet know about) must NOT be redacted: replacing a bool/int+ with the "[REDACTED]" string corrupts it in the settings UI, which+ only special-cases real booleans when rendering a checkbox+ (``web/static/js/components/settings.js``), and silently writes back+ ``false``/empty on the next save.+ """+ if ui_element == "password":+ return True+ raw_leaf = key.rsplit(".", 1)[-1].lower()+ return raw_leaf in sensitive_names or _visible_leaf(key) in sensitive_names+++def _force_redact_strings(value: Any, redaction_text: str) -> Any:+ """Mask every string leaf inside a container, regardless of its own+ sub-key name, while leaving bool/int/None leaves untouched.++ Used when a KEY matches the broadened suffix-only sensitive arm (arm 2+ of ``_matches_sensitive_name``) and its VALUE is a dict/list rather than+ a plain string. ``redact_value``'s normal recursion masks by SUB-KEY+ name, so a secret nested under a non-sensitive sub-key -- e.g.+ ``{"value": "s3cr3t"}`` under a ``milvus_token`` setting -- would+ otherwise survive unredacted: ``value`` is not itself a sensitive leaf+ name. This walks the container and replaces every non-empty string leaf+ with the sentinel unconditionally, so the secret cannot hide behind an+ innocuous sub-key. It preserves round 2's guarantee that a bool/int+ leaf is never corrupted into a string, and leaves empty/whitespace-only+ string leaves readable so an unset nested field doesn't look configured+ (matching ``_is_empty_value``'s rule for the top-level value).+ """+ if isinstance(value, str):+ return redaction_text if value.strip() else value+ if isinstance(value, dict):+ return {+ sub_key: _force_redact_strings(sub_val, redaction_text)+ for sub_key, sub_val in value.items()+ }+ if isinstance(value, list):+ return [_force_redact_strings(item, redaction_text) for item in value]+ return value++class DataSanitizer:"""Utility class for removing sensitive information from data structures."""@@ -120,6 +264,16 @@ class DataSanitizer:# notifications.service_url has this leaf, so no other setting is# affected. The empty-value rule keeps an unconfigured URL readable."service_url",+ # A bare "token" leaf. The qualified spellings above (access_token,+ # auth_token, bearer_token, ...) left the unqualified name uncovered,+ # so a setting keyed "<prefix>_token" with no dots in it (its leaf is+ # the whole key) matched nothing and shipped in the clear from the+ # bulk settings GET (#5762). "token" and "api_token" name a secret in+ # every credential convention we ship; token COUNTS are spelled+ # max_tokens / context_tokens / supports_max_tokens, which are+ # different leaves and stay readable.+ "token",+ "api_token",}@staticmethod@@ -129,8 +283,11 @@ class DataSanitizer:sensitive_keys: Set[str] | None = None,) -> bool:"""True when a setting holds a secret: it is ``ui_element ==- "password"`` OR the last dotted segment of its key is a sensitive- name (``llm.openai.api_key`` -> ``api_key``).+ "password"`` OR the last dotted segment of its key names a secret+ (``llm.openai.api_key`` -> ``api_key``), either exactly or as an+ underscore-delimited suffix (``local_search_milvus_token`` ->+ ``_token``). See ``_matches_sensitive_name`` for both arms and for+ the qualifier carve-out that keeps ``requires_api_key`` readable.Single source of truth for "is this a secret" so the GET redactorand the write-back no-op guards apply the SAME predicate — a value@@ -150,7 +307,9 @@ class DataSanitizer:# custom sensitive name containing one of those characters, and an# exact match must remain sensitive for backward compatibility.raw_leaf = key.rsplit(".", 1)[-1].lower()- return raw_leaf in sens or _visible_leaf(key) in sens+ return _matches_sensitive_name(+ raw_leaf, sens+ ) or _matches_sensitive_name(_visible_leaf(key), sens)@staticmethoddef redact_value(@@ -181,11 +340,42 @@ class DataSanitizer:those nested secrets in the clear. A sensitive OUTER key still masks itswhole value wholesale (the check below runs first), so plain lists undera non-sensitive key pass through untouched.++ Non-string values are only redacted through the original exact-leaf+ / ``password`` match (see ``_is_exact_sensitive_match``). The+ broadened snake_case suffix match added for #5771 is a lexical+ heuristic and must not fire on a bool/int/dict value: a checkbox or+ number setting whose name happens to lexically match a sensitive+ suffix cannot hold a real credential, and replacing its typed value+ with the "[REDACTED]" string corrupts it in the settings UI (a+ checkbox only renders ``checked`` for a literal ``True``) and can+ silently write back the wrong value on the next save.++ A dict/list value under a key that matches ONLY the broadened arm+ (not the exact/password match) is neither a plain string nor caught+ by the type guard above, so it falls through this first check+ untouched. Left alone, the general recursion below would then mask+ it by SUB-KEY name only -- missing a secret nested under a+ non-sensitive sub-key such as ``{"value": "s3cr3t"}``. So that case+ is handled separately: every string leaf inside the container is+ force-masked (``_force_redact_strings``), while bool/int/None+ leaves stay untouched, same as the type guard above."""if DataSanitizer.is_sensitive_setting(key, ui_element, sensitive_keys) and not _is_empty_value(value):- return redaction_text+ sens = {+ k.lower()+ for k in (+ sensitive_keys or DataSanitizer.DEFAULT_SENSITIVE_KEYS+ )+ }+ if isinstance(value, str) or _is_exact_sensitive_match(+ key, ui_element, sens+ ):+ return redaction_text+ if isinstance(value, (dict, list)):+ return _force_redact_strings(value, redaction_text)if isinstance(value, dict):redacted: dict = {}for sub_key, sub_val in value.items():diff --git a/src/local_deep_research/web/routers/settings.py b/src/local_deep_research/web/routers/settings.pyindex fdc939240b..79054261cf 100644--- a/src/local_deep_research/web/routers/settings.py+++ b/src/local_deep_research/web/routers/settings.py@@ -557,8 +557,120 @@ BLOCKED_SETTING_PREFIXES = frozenset()+def _container_embeds_sentinel(value: object, redaction_text: str) -> bool:+ """True when a dict/list contains the redaction sentinel as a substring+ of ANY string leaf, exact-leaf-match or partially spliced into one.++ Recursion counterpart of the ``str`` guards' ``redaction_text in value``+ check, used for a container value under a setting that only matches the+ broadened suffix arm (see ``_force_redact_strings``, which is what+ produces such a container on the read side: every non-empty string leaf+ becomes the bare sentinel, regardless of its own sub-key name). bool/int/+ None leaves are never masked and so can never embed it.+ """+ if isinstance(value, str):+ return redaction_text in value+ if isinstance(value, dict):+ return any(+ _container_embeds_sentinel(sub_val, redaction_text)+ for sub_val in value.values()+ )+ if isinstance(value, list):+ return any(+ _container_embeds_sentinel(item, redaction_text) for item in value+ )+ return False+++def _container_all_leaves_are_sentinel(+ value: object, redaction_text: str+) -> bool:+ """True when every non-empty string leaf inside a dict/list equals the+ redaction sentinel EXACTLY -- i.e. the container is indistinguishable+ from what ``_force_redact_strings`` would emit for it, so it can only be+ an untouched GET round-trip rather than an edit.++ An empty/whitespace-only string leaf is not something+ ``_force_redact_strings`` would have masked (it mirrors+ ``_is_empty_value``'s carve-out), so it is not considered here either+ and does not break purity. A single non-sentinel, non-empty string leaf+ (a legitimately edited sibling field, e.g. a hostname retyped next to an+ untouched ``"[REDACTED]"`` token) fails this and the container is then+ handled as an edit by ``_container_embeds_sentinel`` instead, exactly+ like a partially edited ``notifications.service_url`` string does.+ """+ if isinstance(value, str):+ return not value.strip() or value == redaction_text+ if isinstance(value, dict):+ return all(+ _container_all_leaves_are_sentinel(sub_val, redaction_text)+ for sub_val in value.values()+ )+ if isinstance(value, list):+ return all(+ _container_all_leaves_are_sentinel(item, redaction_text)+ for item in value+ )+ return True+++def _container_matches_stored_shape(+ value: object, stored_value: object, redaction_text: str+) -> bool:+ """True when every NON-maskable leaf of *value* -- anything that is not+ a non-empty string, plus any empty/whitespace-only string leaf -- is+ identical to the corresponding leaf of *stored_value*, with matching+ dict keys / list lengths at every level of nesting.++ Maskable (non-empty string) leaves are exempt from the equality check:+ those are expected to hold the redaction sentinel rather than the real+ secret, and ``_container_all_leaves_are_sentinel`` already verifies+ they are exactly that. What this function catches is the case that+ slipped past round 4: a container round-trip where every string leaf+ is untouched (still the sentinel) but a NON-string sibling was edited+ -- e.g. ``{"token": "[REDACTED]", "port": 19530}`` submitted back as+ ``{"token": "[REDACTED]", "port": 19531}``. Comparing only string+ leaves against the sentinel can never see that edit; comparing the+ non-maskable leaves against the stored row can.++ A maskable leaf whose stored counterpart was NOT itself a non-empty+ string (or a structural mismatch -- different dict keys, different+ list length, a dict submitted where the stored value is a list, etc.)+ also fails this, since that cannot be an untouched round-trip either.+ """+ if isinstance(value, dict):+ if not isinstance(stored_value, dict) or set(value.keys()) != set(+ stored_value.keys()+ ):+ return False+ return all(+ _container_matches_stored_shape(+ value[k], stored_value[k], redaction_text+ )+ for k in value+ )+ if isinstance(value, list):+ if not isinstance(stored_value, list) or len(value) != len(+ stored_value+ ):+ return False+ return all(+ _container_matches_stored_shape(v, s, redaction_text)+ for v, s in zip(value, stored_value)+ )+ if isinstance(value, str) and value.strip():+ # Maskable leaf: its own exactness is checked elsewhere. Just+ # confirm the stored counterpart was maskable too, so a non-string+ # leaf can't silently "become" a string one under this exemption.+ return isinstance(stored_value, str) and bool(stored_value.strip())+ return value == stored_value++def _is_secret_empty_noop(- key: str, ui_element: str | None, value: object+ key: str,+ ui_element: str | None,+ value: object,+ stored_value: object = None,) -> bool:"""True when a secret write must be ignored: the redaction sentinelfor any sensitive setting, or an empty string for password inputs@@ -568,15 +680,54 @@ def _is_secret_empty_noop(``notifications.service_url`` is a sensitive setting on a ``textarea``whose control renders its real value, so an empty write there is adeliberate "clear it" gesture and must reach the database (#5960).++ A dict/list value goes through the container arm: ``redact_value`` can+ mask a container's string leaves via ``_force_redact_strings`` rather+ than replacing the whole value with the bare sentinel (see its+ docstring), so the exact-string check above never sees a container's+ round-tripped sentinel. Without this arm, a GET-then-save-untouched+ container -- e.g. ``{"uri": "[REDACTED]", "token": "[REDACTED]", "port":+ 19530}`` -- would sail past every guard here (all are ``isinstance(value,+ str)``-gated) and persist the sentinel over the real credential. Every+ maskable (non-empty string) leaf must be exactly the sentinel for this+ to count as untouched; a container that merely embeds it somewhere while+ another leaf was edited is a corrupted edit, handled by+ ``_embeds_redaction_sentinel`` instead.++ Checking string leaves alone is not enough, though: a container edit+ that touches only a NON-string leaf (a port number, a bool toggle)+ while every string leaf stays exactly the sentinel would still look+ like a pure round-trip by that check alone, and the edit would be+ silently discarded. ``stored_value`` -- the setting's current value in+ the DB, threaded in by every call site that has a prior row to compare+ against -- lets ``_container_matches_stored_shape`` catch that: the+ container is only a genuine no-op if its non-maskable leaves are+ byte-for-byte identical to what's already stored. If the caller has no+ stored value to compare against (``stored_value is None``, e.g. no+ prior row), a sentinel-bearing container can never be verified as an+ untouched round-trip and is therefore never treated as this kind of+ no-op -- it falls through to ``_embeds_redaction_sentinel``, which+ rejects it with a 400 instead of risking a silent drop or a silent+ sentinel write."""- return (- isinstance(value, str)- and DataSanitizer.is_sensitive_setting(key, ui_element)- and (- value == DataSanitizer.REDACTION_TEXT- or (value == "" and ui_element == "password")+ if not DataSanitizer.is_sensitive_setting(key, ui_element):+ return False+ if isinstance(value, str):+ return value == DataSanitizer.REDACTION_TEXT or (+ value == "" and ui_element == "password")- )+ if isinstance(value, (dict, list)):+ return (+ stored_value is not None+ and _container_embeds_sentinel(value, DataSanitizer.REDACTION_TEXT)+ and _container_all_leaves_are_sentinel(+ value, DataSanitizer.REDACTION_TEXT+ )+ and _container_matches_stored_shape(+ value, stored_value, DataSanitizer.REDACTION_TEXT+ )+ )+ return Falsedef _redaction_sentinel_error(ui_element: str | None) -> str:@@ -608,7 +759,10 @@ def _redaction_sentinel_error(ui_element: str | None) -> str:def _embeds_redaction_sentinel(- key: str, ui_element: str | None, value: object+ key: str,+ ui_element: str | None,+ value: object,+ stored_value: object = None,) -> bool:"""True when a submitted sensitive value *embeds* the redaction sentinel.@@ -624,13 +778,51 @@ def _embeds_redaction_sentinel(every notification. No legitimate secret contains the sentinel, so thisis a hard 400 rather than a no-op: unlike the exact-match case the usermade an edit, and silently dropping it would look like a successful save.++ A dict/list value takes the container arm. There are two ways a+ container can embed the sentinel without being the pure round-trip+ ``_is_secret_empty_noop`` claims:++ 1. Not every maskable (non-empty string) leaf is exactly the+ sentinel (``_container_all_leaves_are_sentinel`` is False) --+ at least one string leaf was edited while another still carries+ "[REDACTED]" verbatim, e.g. a hostname retyped next to an+ untouched token field.+ 2. Every maskable leaf IS exactly the sentinel, but a non-maskable+ leaf (a port number, a bool toggle) does not match what's+ currently stored -- ``_container_matches_stored_shape`` is False,+ or there is no ``stored_value`` to compare against at all. This+ is the case round 4 missed: a container that looks like a pure+ string round-trip but actually carries an edit to a non-string+ sibling. Silently persisting it would splice the sentinel into+ the stored credential's string leaves; silently treating it as a+ no-op (what ``_is_secret_empty_noop`` used to do) would discard+ the non-string edit instead. Neither is acceptable, so both+ shapes get the same 400 as the plain string case.++ Without a ``stored_value`` to check case 2 against, a sentinel-bearing+ container can never be proven to be an untouched round-trip, so it is+ conservatively treated as case 2 (400) rather than risking a silent+ drop or a silent sentinel write."""- return (- isinstance(value, str)- and DataSanitizer.is_sensitive_setting(key, ui_element)- and DataSanitizer.REDACTION_TEXT in value- and value != DataSanitizer.REDACTION_TEXT- )+ if not DataSanitizer.is_sensitive_setting(key, ui_element):+ return False+ if isinstance(value, str):+ return (+ DataSanitizer.REDACTION_TEXT in value+ and value != DataSanitizer.REDACTION_TEXT+ )+ if isinstance(value, (dict, list)):+ if not _container_embeds_sentinel(value, DataSanitizer.REDACTION_TEXT):+ return False+ if not _container_all_leaves_are_sentinel(+ value, DataSanitizer.REDACTION_TEXT+ ):+ return True+ return stored_value is None or not _container_matches_stored_shape(+ value, stored_value, DataSanitizer.REDACTION_TEXT+ )+ return Falsedef _embeds_sentinel_on_create(@@ -641,15 +833,21 @@ def _embeds_sentinel_on_create(Creation has no prior value, so the sentinel cannot mean "keep thestored secret" the way it does on the update path — every occurrenceof it, exact match included, is a corrupted client value that would be- stored verbatim as the credential.+ stored verbatim as the credential. A dict/list value gets the same+ treatment via ``_container_embeds_sentinel``: there is no "pure+ round-trip" exemption here (unlike ``_is_secret_empty_noop`` on the+ update path) because creation has no prior stored value for an untouched+ field to round-trip from."""- return (- isinstance(value, str)- and DataSanitizer.is_sensitive_setting(- key, ui_element if isinstance(ui_element, str) else None- )- and DataSanitizer.REDACTION_TEXT in value- )+ if not DataSanitizer.is_sensitive_setting(+ key, ui_element if isinstance(ui_element, str) else None+ ):+ return False+ if isinstance(value, str):+ return DataSanitizer.REDACTION_TEXT in value+ if isinstance(value, (dict, list)):+ return _container_embeds_sentinel(value, DataSanitizer.REDACTION_TEXT)+ return Falsedef _is_allowed_new_setting_key(key: str) -> bool:@@ -775,7 +973,10 @@ def _save_all_settings_sync(form_data: dict, username: str):# and must stay clearable (#5960). To unset a password# setting, clear the source env var or use settings import.if current_setting and _is_secret_empty_noop(- key, current_setting.ui_element, value+ key,+ current_setting.ui_element,+ value,+ current_setting.value,):logger.debug(f"Skipping empty secret write for {key} (no-op)"@@ -787,7 +988,10 @@ def _save_all_settings_sync(form_data: dict, username: str):# of persisting a secret with "[REDACTED]" spliced into it# (#5947).if current_setting and _embeds_redaction_sentinel(- key, current_setting.ui_element, value+ key,+ current_setting.ui_element,+ value,+ current_setting.value,):logger.warning("Rejected redaction-sentinel value for {!r} via "@@ -1380,7 +1584,7 @@ def _save_settings_sync(form_data: dict, username: str) -> dict:# Matches the guards in save_all_settings +# api_update_setting.if db_setting and _is_secret_empty_noop(- key, db_setting.ui_element, value+ key, db_setting.ui_element, value, db_setting.value):logger.debug(f"Skipping empty secret write for {key} via "@@ -1393,7 +1597,7 @@ def _save_settings_sync(form_data: dict, username: str) -> dict:# so collect it rather than saving a secret with "[REDACTED]"# spliced into it (#5947).if db_setting and _embeds_redaction_sentinel(- key, db_setting.ui_element, value+ key, db_setting.ui_element, value, db_setting.value):logger.warning("Rejected redaction-sentinel value for {!r} via "@@ -3650,7 +3854,9 @@ def _api_update_setting_sync(data, key, username):# wipe the secret). Companion to the same guard in# save_all_settings/save_settings. The idempotent 200 keeps# client-side save indicators from erroring (#5960).- if _is_secret_empty_noop(key, db_setting.ui_element, value):+ if _is_secret_empty_noop(+ key, db_setting.ui_element, value, db_setting.value+ ):logger.debug(f"Skipping sensitive value write for {key} via ""api_update_setting (no-op)"@@ -3667,7 +3873,7 @@ def _api_update_setting_sync(data, key, username):# while the exact-match case above stays an idempotent 200# no-op (#5947).if _embeds_redaction_sentinel(- key, db_setting.ui_element, value+ key, db_setting.ui_element, value, db_setting.value):logger.warning("Rejected redaction-sentinel value for {!r} via "diff --git a/tests/security/test_bulk_secret_name_coverage.py b/tests/security/test_bulk_secret_name_coverage.pyindex 2eff731142..036b3c6d8e 100644--- a/tests/security/test_bulk_secret_name_coverage.py+++ b/tests/security/test_bulk_secret_name_coverage.py@@ -1,13 +1,30 @@-"""Regression: the bulk settings GET redacts secrets whose leaf name isn't a-classic secret token (client_secret, bearer_token, …).+"""Regression: the bulk settings GET redacts every password-typed setting we+ship, including ones whose key is flat snake_case.``GET /settings/api/bulk`` redacts via ``redact_value(key, None, value)`` — it-passes no ``ui_element``, so a password-typed setting is masked only if its leaf-name is in ``DEFAULT_SENSITIVE_KEYS``. These names were previously missing, so-such settings shipped in the clear. Uses a real in-memory ``SettingsManager`` and-mirrors the endpoint's redaction call; fails if a name is dropped from the set.+passes no ``ui_element``, so a password-typed setting is masked only if its key+name matches ``DEFAULT_SENSITIVE_KEYS``. That name check was originally an+exact match on the key's last *dotted* segment, which a flat snake_case key+such as ``local_search_milvus_token`` can never satisfy: with no dots in it,+its "leaf" is the entire key. Such settings shipped in the clear (#5762).++The heart of this module is ``test_every_shipped_password_setting_is_redacted``:+rather than hardcoding leaf names (the shape this file had after #5028, which+is why #5762 slipped through), it ENUMERATES the shipped+``defaults/**/*.json`` and asserts the suffix-only predicate covers every+setting declared ``ui_element: "password"``. Add a new secret setting whose key+the predicate does not recognize and this test fails before the leak ships.++Its mirror, ``test_no_shipped_non_secret_setting_is_redacted``, pins the other+direction: widening the predicate must not start masking ordinary settings.+``search.engine.web.*.requires_api_key`` is a checkbox whose leaf ends in+``_api_key``; masking it would ship "[REDACTED]" to the settings UI in place of+true/false and make the write-back no-op guards refuse legitimate writes."""+import json+from pathlib import Path+import pytestfrom sqlalchemy import create_enginefrom sqlalchemy.orm import sessionmaker@@ -17,6 +34,233 @@ from local_deep_research.database.models.settings import Settingfrom local_deep_research.security.data_sanitizer import DataSanitizerfrom local_deep_research.settings.manager import SettingsManager+DEFAULTS_DIR = (+ Path(__file__).resolve().parents[2]+ / "src"+ / "local_deep_research"+ / "defaults"+)+++def _iter_shipped_settings():+ """Yield ``(source, key, ui_element)`` for every setting in the defaults.++ The defaults are a mix of shapes — a flat ``{key: {...}}`` mapping, nested+ groups, and files that wrap the mapping — so walk generically and treat any+ dict carrying setting metadata as an entry rather than assuming a depth.+ """+ for path in sorted(DEFAULTS_DIR.rglob("*.json")):+ data = json.loads(path.read_text())+ stack = [data]+ while stack:+ node = stack.pop()+ if isinstance(node, dict):+ for key, value in node.items():+ if isinstance(value, dict) and (+ "ui_element" in value or "value" in value+ ):+ yield path.name, key, value.get("ui_element")+ stack.append(value)+ elif isinstance(node, list):+ stack.extend(node)+++SHIPPED_SETTINGS = list(_iter_shipped_settings())+SHIPPED_PASSWORD_SETTINGS = [+ (source, key) for source, key, ui in SHIPPED_SETTINGS if ui == "password"+]++# Settings that are deliberately redacted despite NOT being declared+# ui_element == "password" -- DEFAULT_SENSITIVE_KEYS masks these on purpose,+# so they are not a bug for test_no_shipped_non_secret_setting_is_redacted to+# catch. notifications.service_url is a "textarea" field, but it embeds+# credentials (e.g. discord://webhook_id/token, mailto://user:pass@host) --+# see the "service_url" entry in DataSanitizer.DEFAULT_SENSITIVE_KEYS.+_INTENTIONALLY_SENSITIVE_NON_PASSWORD_KEYS = {"notifications.service_url"}++SHIPPED_NON_SECRET_SETTINGS = [+ (source, key)+ for source, key, ui in SHIPPED_SETTINGS+ if ui != "password"+ and key not in _INTENTIONALLY_SENSITIVE_NON_PASSWORD_KEYS+]+++def _bulk_shipped(key, value="topsecret-value"):+ """What ``GET /settings/api/bulk`` would put on the wire for ``key``."""+ return DataSanitizer.redact_value(key, None, value)+++def test_defaults_enumeration_is_not_vacuously_empty():+ """Guard the guard: a defaults refactor that breaks the walk must not+ silently turn the enumerating tests below into no-ops."""+ assert len(SHIPPED_SETTINGS) > 100+ assert len(SHIPPED_PASSWORD_SETTINGS) > 15+++ "source,key",+ SHIPPED_PASSWORD_SETTINGS,+ ids=[f"{source}:{key}" for source, key in SHIPPED_PASSWORD_SETTINGS],+)+def test_every_shipped_password_setting_is_redacted(source, key):+ """Every ``ui_element: "password"`` setting we ship must survive the+ bulk GET's suffix-only predicate, which sees no ui_element."""+ assert _bulk_shipped(key) == DataSanitizer.REDACTION_TEXT, (+ f"{key} (from {source}) is declared ui_element='password' but the "+ "bulk settings GET would ship its value in the clear. Either name it "+ "so DataSanitizer's key heuristic recognizes it (a *.api_key / "+ "*_token / *_password suffix) or add its name to DEFAULT_SENSITIVE_KEYS."+ )+++ "source,key",+ SHIPPED_NON_SECRET_SETTINGS,+ ids=[f"{source}:{key}" for source, key in SHIPPED_NON_SECRET_SETTINGS],+)+def test_no_shipped_non_secret_setting_is_redacted(source, key):+ """The mirror: no ordinary setting may be masked by accident."""+ assert _bulk_shipped(key, "plain-value") == "plain-value", (+ f"{key} (from {source}) is not a password field but the bulk settings "+ "GET would mask its value, breaking the settings UI and the "+ "write-back no-op guards."+ )+++ "key",+ [+ # The #5762 reproduction: flat snake_case, no dots, so the dotted+ # "leaf" is the whole key and an exact-match check finds nothing.+ "local_search_milvus_token",+ # Same shape, other secret names — the miss was never token-specific.+ "local_search_milvus_api_key",+ "some_service_password",+ "some_service_client_secret",+ # Classic dotted keys.+ "integrations.oauth.client_secret",+ "integrations.oauth.secret_key",+ "integrations.oauth.bearer_token",+ "integrations.oauth.api_secret",+ "integrations.oauth.app_secret",+ "integrations.oauth.api_token",+ ],+)+def test_bulk_get_redacts_secret_key_shapes(key):+ assert _bulk_shipped(key) == DataSanitizer.REDACTION_TEXT+++ "key",+ [+ # Token COUNTS and capability flags are not secrets. These pin the two+ # carve-outs that keep the suffix rule from over-redacting.+ "llm.max_tokens",+ "llm.supports_max_tokens",+ "search.engine.web.brave.requires_api_key",+ "local_search_chunk_size",+ "llm.provider",+ ],+)+def test_bulk_get_leaves_non_secrets_readable(key):+ assert _bulk_shipped(key, "plain-value") == "plain-value"+++def test_bulk_get_does_not_corrupt_non_string_lexical_match():+ """Type-guard regression: the broadened snake_case suffix match is a+ lexical heuristic backed by a hand-maintained non-secret-prefix+ carve-out (``_NON_SECRET_LEAF_PREFIXES``). A future boolean flag whose+ name lexically ends in a sensitive suffix but isn't covered by that+ carve-out (e.g. a hypothetical ``verify_token`` checkbox — "verify_" is+ not in the prefix list) must NOT be masked: replacing a real ``bool``+ with the "[REDACTED]" string corrupts it in the settings UI (a checkbox+ only renders ``checked`` for a literal ``True``) and can silently write+ back the wrong value on the next save. A same-shape STRING setting must+ still be redacted -- the type guard narrows by value type only, not by+ loosening the name match itself.+ """+ # Boolean lexically matching a sensitive suffix, uncovered by the+ # carve-out prefixes: must survive as a real bool, not "[REDACTED]".+ assert _bulk_shipped("verify_token", True) is True+ assert _bulk_shipped("verify_token", False) is False++ # Same hazard for a plain number.+ assert _bulk_shipped("retry_token", 3) == 3++ # The corresponding STRING setting is still redacted (#5762 stays fixed).+ assert (+ _bulk_shipped("milvus_api_token", "sk-fake-not-a-real-secret")+ == DataSanitizer.REDACTION_TEXT+ )+++def test_bulk_get_masks_string_leaves_inside_a_broadened_match_container():+ """Container regression: a key that matches ONLY the broadened+ underscore-suffix arm (not the exact/password match) with a dict/list+ VALUE must still have its string leaves masked.++ Round 2's type guard stops a bool/int from being corrupted into the+ "[REDACTED]" string, but ``redact_value``'s normal recursion masks by+ SUB-KEY name -- so a secret nested under a non-sensitive sub-key such as+ ``{"value": "s3cr3t"}`` under a ``milvus_token`` setting would otherwise+ survive unredacted (sub-key ``value`` isn't itself a sensitive name).+ """+ assert _bulk_shipped("local_search_milvus_token", {"value": "s3cr3t"}) == {+ "value": DataSanitizer.REDACTION_TEXT+ }++ # Bool/int leaves inside the same container stay untouched -- the+ # string-only guard applies leaf-by-leaf, not to the whole container.+ assert _bulk_shipped(+ "local_search_milvus_token",+ {"value": "s3cr3t", "enabled": True, "retries": 3},+ ) == {+ "value": DataSanitizer.REDACTION_TEXT,+ "enabled": True,+ "retries": 3,+ }++ # List containers are covered too.+ assert _bulk_shipped(+ "local_search_milvus_token", [{"value": "s3cr3t"}, "plain-scalar"]+ ) == [{"value": DataSanitizer.REDACTION_TEXT}, DataSanitizer.REDACTION_TEXT]++ # Empty/whitespace-only string leaves stay readable, matching the+ # top-level empty-value rule (an unset nested field shouldn't look+ # configured).+ assert _bulk_shipped("local_search_milvus_token", {"value": " "}) == {+ "value": " "+ }+++def test_bulk_get_leaves_qualifier_named_flag_readable_mid_key():+ """Carve-out semantics regression: ``_NON_SECRET_LEAF_PREFIXES`` must+ apply wherever the qualifier falls in a FLAT snake_case key, not only at+ its very first segment.++ A flat key has no dots, so its "leaf" (see ``_matches_sensitive_name``)+ is the entire key. ``llm_requires_api_key`` is the same "flag ABOUT a+ secret" shape as the dotted ``search.engine.web.brave.requires_api_key``+ (already pinned above), but the qualifier ``requires_`` is not the first+ segment. It must be recognized and left readable either way -- for a+ bool (round 2's type guard already prevents corruption there) and for a+ same-shape string (the residual hazard this test pins: without the+ fix, the carve-out silently misses the qualifier and over-redacts the+ string).+ """+ assert _bulk_shipped("llm_requires_api_key", True) is True+ assert _bulk_shipped("llm_requires_api_key", False) is False+ assert _bulk_shipped("llm_requires_api_key", "plain-value") == "plain-value"++ # A real secret with an unrelated prefix segment is still redacted --+ # the carve-out must not become a blanket exemption for any multi-segment+ # flat key.+ assert (+ _bulk_shipped("llm_openai_api_key", "sk-fake-not-a-real-secret")+ == DataSanitizer.REDACTION_TEXT+ )+@pytest.fixturedef manager_factory():@@ -42,11 +286,11 @@ def manager_factory():@pytest.mark.parametrize(- "leaf",- ["client_secret", "secret_key", "bearer_token", "api_secret", "app_secret"],+ "key",+ ["integrations.oauth.client_secret", "local_search_milvus_token"],)-def test_bulk_get_redacts_named_secret(leaf, manager_factory):- key = f"integrations.oauth.{leaf}"+def test_bulk_get_redacts_named_secret(key, manager_factory):+ """End-to-end through a real SettingsManager, mirroring the endpoint."""manager = manager_factory([Setting(diff --git a/tests/security/test_data_sanitizer_coverage.py b/tests/security/test_data_sanitizer_coverage.pyindex 2d52c140f7..2caa953984 100644--- a/tests/security/test_data_sanitizer_coverage.py+++ b/tests/security/test_data_sanitizer_coverage.py@@ -558,11 +558,13 @@ class TestDefaultSensitiveKeysComplete:"app_secret",# Notification webhook URL (apprise-style) embeds credentials."service_url",+ "token",+ "api_token",}assert DataSanitizer.DEFAULT_SENSITIVE_KEYS == expecteddef test_count(self):- assert len(DataSanitizer.DEFAULT_SENSITIVE_KEYS) == 16+ assert len(DataSanitizer.DEFAULT_SENSITIVE_KEYS) == 18def test_all_lowercase(self):"""All default keys are stored in lowercase."""diff --git a/tests/security/test_settings_secret_redaction_isolation.py b/tests/security/test_settings_secret_redaction_isolation.pyindex 8a38ed8323..ea7c4c8317 100644--- a/tests/security/test_settings_secret_redaction_isolation.py+++ b/tests/security/test_settings_secret_redaction_isolation.py@@ -374,6 +374,186 @@ def test_embedded_sentinel_predicate():assert not _embeds_redaction_sentinel(WEBHOOK_KEY, "textarea", 17)+# --------------------------------------------------------------------------- #+# Container guard: round 3 taught ``redact_value`` to mask a dict/list value+# leaf-by-leaf (``_force_redact_strings``) for a key that matches ONLY the+# broadened suffix arm, e.g. ``local_search_milvus_token`` -> {"uri": ...,+# "token": ...}. Round 3 left the three write-back guards (a)+# ``isinstance(value, str)``-gated, so a masked container sailed past every+# one of them: GET, save untouched, and the sentinel is persisted over the+# real credential and its non-secret siblings. These pin the fix -- the+# container arm added to ``_is_secret_empty_noop`` /+# ``_embeds_redaction_sentinel`` / ``_embeds_sentinel_on_create``.+# --------------------------------------------------------------------------- #+BROADENED_ARM_KEY = "local_search_milvus_token"+++def test_container_sentinel_roundtrip_is_noop():+ """(a) A container GET'd and saved back UNTOUCHED -- every string leaf+ is exactly "[REDACTED]", matching what ``_force_redact_strings`` would+ have produced, AND every non-string leaf still matches the stored row+ -- must be an idempotent no-op, not a write. The stored credential+ must not be overwritten with the sentinel.++ Round 5 (#6089-follow-up): the no-op arm now takes the setting's+ current stored value as a fourth argument and requires non-string+ leaves to match it too -- see ``test_container_non_string_leaf_edit_+ is_not_a_silent_noop`` below for the bug this closes.+ """+ from local_deep_research.web.routers.settings import (+ _embeds_redaction_sentinel,+ _is_secret_empty_noop,+ )++ stored = {"uri": "mysql://real", "token": "s3cr3t-real", "port": 19530}+ roundtrip = {"uri": REDACTED, "token": REDACTED, "port": 19530}+ assert _is_secret_empty_noop(BROADENED_ARM_KEY, None, roundtrip, stored)+ assert not _embeds_redaction_sentinel(+ BROADENED_ARM_KEY, None, roundtrip, stored+ )++ # List containers, and empty/whitespace leaves (never masked by+ # _force_redact_strings), are covered the same way.+ stored_list = [{"value": "s3cr3t-real"}, "s3cr3t-real", 42, True]+ assert _is_secret_empty_noop(+ BROADENED_ARM_KEY,+ None,+ [{"value": REDACTED}, REDACTED, 42, True],+ stored_list,+ )+ stored_with_blank = {+ "uri": "mysql://real",+ "token": "s3cr3t-real",+ "note": " ",+ }+ assert _is_secret_empty_noop(+ BROADENED_ARM_KEY,+ None,+ {"uri": REDACTED, "token": REDACTED, "note": " "},+ stored_with_blank,+ )++ # A non-sensitive key's identical-looking container is not special-cased.+ assert not _is_secret_empty_noop(+ "llm.some_plain_setting", None, roundtrip, stored+ )++ # No stored value to verify against (e.g. no prior row): a+ # sentinel-bearing container can never be proven an untouched+ # round-trip, so it must NOT be treated as a no-op.+ assert not _is_secret_empty_noop(BROADENED_ARM_KEY, None, roundtrip, None)+ assert _embeds_redaction_sentinel(BROADENED_ARM_KEY, None, roundtrip, None)+++def test_container_partial_edit_still_embeds_sentinel():+ """(b) A container that still carries "[REDACTED]" in some leaf but is+ NOT the pure untouched round-trip -- a leaf with the sentinel spliced+ into it, or a legitimately edited sibling next to an untouched+ sentinel leaf -- is a corrupted/partial edit and must be rejected with+ the same 400 semantics as the string case, not silently persisted with+ "[REDACTED]" overwriting the real secret.+ """+ from local_deep_research.web.routers.settings import (+ _embeds_redaction_sentinel,+ _is_secret_empty_noop,+ )++ stored = {"uri": "mysql://real", "token": "s3cr3t-real", "port": 19530}++ # A leaf with the sentinel spliced into it rather than exactly equal.+ spliced = {+ "uri": REDACTED,+ "token": f"pre{REDACTED}post",+ "port": 19530,+ }+ assert _embeds_redaction_sentinel(BROADENED_ARM_KEY, None, spliced, stored)+ assert not _is_secret_empty_noop(BROADENED_ARM_KEY, None, spliced, stored)++ # A non-secret sibling legitimately edited while the secret leaves are+ # still verbatim "[REDACTED]" -- must still be rejected, not silently+ # persisted (which would overwrite uri/token with the sentinel) or+ # silently dropped (which would lose the host edit).+ mixed = {+ "uri": REDACTED,+ "token": REDACTED,+ "host": "new-host.example.com",+ "port": 19531,+ }+ assert _embeds_redaction_sentinel(BROADENED_ARM_KEY, None, mixed, stored)+ assert not _is_secret_empty_noop(BROADENED_ARM_KEY, None, mixed, stored)++ # A genuinely fresh value with no sentinel anywhere trips neither guard.+ fresh = {"uri": "mysql://real", "token": "s3cr3t-real", "port": 19530}+ assert not _embeds_redaction_sentinel(+ BROADENED_ARM_KEY, None, fresh, stored+ )+ assert not _is_secret_empty_noop(BROADENED_ARM_KEY, None, fresh, stored)+++def test_container_non_string_leaf_edit_is_not_a_silent_noop():+ """R1d / round 5 (#6089-follow-up): editing ONLY a non-string leaf+ inside a sensitive container -- e.g. a port number -- while every+ string leaf stays exactly "[REDACTED]" must NOT be silently discarded+ as a no-op, and must NOT be silently persisted with the sentinel+ spliced into the string leaves either. It must be rejected with the+ same 400 "retype the whole value" semantics as any other corrupted+ container edit.++ Round 4's ``_container_all_leaves_are_sentinel`` only ever inspected+ STRING leaves, so a stored ``{"token": "s", "port": 19530}`` submitted+ back as ``{"token": "[REDACTED]", "port": 19531}`` looked identical to+ a pure round-trip and the port edit vanished silently. This pins the+ fix: the no-op arm now also requires non-string leaves to match the+ stored row.+ """+ from local_deep_research.web.routers.settings import (+ _embeds_redaction_sentinel,+ _is_secret_empty_noop,+ )++ stored = {"token": "s", "port": 19530}+ port_edit = {"token": REDACTED, "port": 19531}++ assert not _is_secret_empty_noop(BROADENED_ARM_KEY, None, port_edit, stored)+ assert _embeds_redaction_sentinel(+ BROADENED_ARM_KEY, None, port_edit, stored+ )++ # An untouched round-trip against the same stored row stays a no-op.+ untouched = {"token": REDACTED, "port": 19530}+ assert _is_secret_empty_noop(BROADENED_ARM_KEY, None, untouched, stored)+ assert not _embeds_redaction_sentinel(+ BROADENED_ARM_KEY, None, untouched, stored+ )++ # A bool-toggle sibling edit is caught the same way.+ stored_with_flag = {"token": "s", "enabled": False}+ flag_edit = {"token": REDACTED, "enabled": True}+ assert not _is_secret_empty_noop(+ BROADENED_ARM_KEY, None, flag_edit, stored_with_flag+ )+ assert _embeds_redaction_sentinel(+ BROADENED_ARM_KEY, None, flag_edit, stored_with_flag+ )+++def test_container_sentinel_on_create_has_no_noop_exemption():+ """Creation has no prior stored value, so unlike the update path, even+ the pure round-trip shape (every leaf exactly the sentinel) cannot mean+ "keep what's there" -- it is rejected too."""+ from local_deep_research.web.routers.settings import (+ _embeds_sentinel_on_create,+ )++ roundtrip = {"uri": REDACTED, "token": REDACTED, "port": 19530}+ spliced = {"uri": REDACTED, "token": f"pre{REDACTED}post"}+ fresh = {"uri": "mysql://real", "token": "s3cr3t-real"}++ assert _embeds_sentinel_on_create(BROADENED_ARM_KEY, None, roundtrip)+ assert _embeds_sentinel_on_create(BROADENED_ARM_KEY, None, spliced)+ assert not _embeds_sentinel_on_create(BROADENED_ARM_KEY, None, fresh)++def test_sentinel_error_matches_the_clear_semantics_per_ui_element():"""The 400 message must not advertise an escape that cannot work.