Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

security: WebAPIException to_dict() backstop + extract shared sanitize_error_details (DRY) (#4937)

LearningCircuit004c32ccc049
diff --git a/changelog.d/4937.security.md b/changelog.d/4937.security.md
new file mode 100644
index 0000000000..39429fee57
--- /dev/null
+++ b/changelog.d/4937.security.md
@@ -0,0 +1 @@
+`WebAPIException.to_dict()` now redacts credential shapes from the fields it serializes to the client (`message` via `sanitize_error_for_client()`, `details` string leaves via `sanitize_error_message()`), mirroring the `NewsAPIException` backstop. This is a defence-in-depth measure at the response boundary so that a future raise site, subclass, or external caller that lets a credential-bearing string into `message` or `details` has it redacted before it ships. Legitimate text is unchanged, and `status` / `error_code` / `status_code` are left untouched.
diff --git a/src/local_deep_research/news/exceptions.py b/src/local_deep_research/news/exceptions.py
index e943e871df..4f7afdbf8e 100644
--- a/src/local_deep_research/news/exceptions.py
+++ b/src/local_deep_research/news/exceptions.py
@@ -8,7 +8,7 @@ appropriate JSON responses.
from typing import Optional, Dict, Any
-from ..security import sanitize_error_for_client, sanitize_error_message
+from ..security import sanitize_error_details, sanitize_error_for_client
class NewsAPIException(Exception):
@@ -36,40 +36,6 @@ class NewsAPIException(Exception):
self.error_code = error_code or self.__class__.__name__
self.details = details or {}
- @staticmethod
- def _sanitize_details(value: Any) -> Any:
- """Recursively redact credential shapes from string leaves of ``details``.
-
- ``details`` can carry caller-supplied free text — e.g.
- ``SubscriptionCreationException`` forwards ``{"query": <user query>}``
- (``news/api.py``), and the base constructor accepts an arbitrary
- ``details`` dict from any caller. So the same "don't trust a future
- caller" premise that scrubs ``message`` applies here. This walks
- ``dict`` / ``list`` / ``tuple`` containers and runs each ``str`` leaf
- through ``sanitize_error_message`` (credential-shape redaction only —
- no length cap or control-char strip, so structured values such as IDs
- are preserved). Non-string leaves (ints, bools, ``None``) pass through
- untouched. Redaction is in place and shape-based, so it never removes a
- value by key name — a benign ``{"query": "reset my password"}`` is
- unchanged.
- """
- if isinstance(value, dict):
- return {
- k: NewsAPIException._sanitize_details(v)
- for k, v in value.items()
- }
- if isinstance(value, (list, tuple)):
- # Always rebuild as a plain list. ``details`` is about to be
- # jsonify-ed (a tuple already serializes to a JSON array), so tuple
- # identity is irrelevant downstream — and ``type(value)(<gen>)``
- # would raise on a namedtuple / tuple subclass whose constructor is
- # not ``(iterable) -> instance``, turning a clean structured error
- # response into a bare 500 from inside the error handler.
- return [NewsAPIException._sanitize_details(v) for v in value]
- if isinstance(value, str):
- return sanitize_error_message(value)
- return value
-
def to_dict(self) -> Dict[str, Any]:
"""Convert exception to dictionary for JSON response.
@@ -85,8 +51,8 @@ class NewsAPIException(Exception):
* ``message`` — through ``sanitize_error_for_client`` (credential
redaction + control-char strip + 200-char cap; it is prose meant for
display).
- * ``details`` string leaves — through ``sanitize_error_message`` (see
- ``_sanitize_details``).
+ * ``details`` string leaves — through ``sanitize_error_details`` (the
+ shared helper, also used by ``WebAPIException``).
Redaction is credential-*shape* based (``Bearer``/``Authorization``,
``?api_key=``-style query params, known token prefixes, ``http(s)``
@@ -108,7 +74,7 @@ class NewsAPIException(Exception):
"status_code": self.status_code,
}
if self.details:
- result["details"] = self._sanitize_details(self.details)
+ result["details"] = sanitize_error_details(self.details)
return result
diff --git a/src/local_deep_research/security/__init__.py b/src/local_deep_research/security/__init__.py
index 5e1f384403..1abbcd7b15 100644
--- a/src/local_deep_research/security/__init__.py
+++ b/src/local_deep_research/security/__init__.py
@@ -35,6 +35,7 @@ from .account_lockout import AccountLockoutManager, get_account_lockout_manager
from .password_validator import PasswordValidator
from .log_sanitizer import (
redact_secrets,
+ sanitize_error_details,
sanitize_error_for_client,
sanitize_error_message,
sanitize_for_log,
@@ -101,6 +102,7 @@ __all__ = [
"get_account_lockout_manager",
"PasswordValidator",
"redact_secrets",
+ "sanitize_error_details",
"sanitize_error_for_client",
"sanitize_error_message",
"sanitize_for_log",
diff --git a/src/local_deep_research/security/log_sanitizer.py b/src/local_deep_research/security/log_sanitizer.py
index 920ec58fa6..1b68ff3310 100644
--- a/src/local_deep_research/security/log_sanitizer.py
+++ b/src/local_deep_research/security/log_sanitizer.py
@@ -10,10 +10,15 @@ concerns:
* :func:`redact_secrets` \u2014 scrub known sensitive *values* (API keys,
passwords, session tokens) from an arbitrary string before it is
logged, returned in an error message, or persisted.
+* :func:`sanitize_error_details` \u2014 recurse a structured ``details`` value
+ and redact credential *shapes* from its string leaves before it is
+ serialized to a client. (Value-shape based; contrast
+ ``data_sanitizer.DataSanitizer``, which redacts by key *name*.)
"""
+import dataclasses
import re
-from typing import Optional
+from typing import Any, Optional
# Strip C0/C1 control characters and dangerous Unicode format characters,
@@ -304,3 +309,61 @@ def sanitize_error_for_client(message: str, max_length: int = 200) -> str:
return sanitize_for_log(
sanitize_error_message(message), max_length=max_length
)
+
+
+def sanitize_error_details(value: Any) -> Any:
+ """Recursively redact credential *shapes* from the string leaves of a
+ structured ``details`` value (``dict`` / ``list`` / ``tuple`` / dataclass).
+
+ Intended for the ``details`` payload of an exception ``to_dict()`` that is
+ serialized to a client (e.g. ``NewsAPIException`` / ``WebAPIException``).
+ This is the *value-shape* counterpart to :class:`data_sanitizer.DataSanitizer`,
+ which redacts by *key name*; use this one when the concern is a credential
+ embedded anywhere in the text, regardless of the key it sits under.
+
+ Behaviour by node type:
+
+ * ``dict`` — recurse into values; ``str`` keys are also run through
+ :func:`sanitize_error_message` (a credential used *as* a key would
+ otherwise ship verbatim as a JSON key). Two distinct keys that both
+ redact to the same token collapse to one entry — an acceptable
+ fidelity loss for the pathological case of credential-shaped keys.
+ * ``list`` / ``tuple`` — both rebuilt as a plain ``list``: the payload is
+ about to be JSON-serialized (a tuple already becomes a JSON array), and
+ rebuilding via ``type(value)(<generator>)`` would raise on a namedtuple /
+ tuple subclass whose constructor is not ``(iterable) -> instance`` —
+ inside a Flask error handler that degrades a structured error into a bare
+ 500.
+ * **dataclass instance** — converted via ``dataclasses.asdict`` and recursed.
+ Flask's default JSON provider serializes a dataclass (via ``asdict``), so
+ without this a credential in a dataclass field would ship un-redacted.
+ * ``str`` leaf — redacted via :func:`sanitize_error_message` (credential-shape
+ redaction only — no length cap or control-char strip, so structured values
+ such as IDs survive).
+ * anything else (ints, bools, ``None``, and any other object) — passed
+ through untouched. Note some passthrough types (``set``/``frozenset``,
+ ``bytes``, a non-``str``-mixin ``Enum``) are not JSON-serializable, so a
+ credential inside them fails *closed* at ``jsonify`` (generic 500) rather
+ than leaking.
+
+ Redaction is shape-based, so it never removes a value by key name — a benign
+ ``{"query": "reset my password"}`` is unchanged. Containers are rebuilt fresh
+ (the input is not mutated); passthrough leaf objects are returned by
+ reference.
+ """
+ if isinstance(value, dict):
+ return {
+ (sanitize_error_message(k) if isinstance(k, str) else k): (
+ sanitize_error_details(v)
+ )
+ for k, v in value.items()
+ }
+ if isinstance(value, (list, tuple)):
+ return [sanitize_error_details(v) for v in value]
+ # dataclass *instance* (not the class itself) — Flask serializes it via
+ # asdict(), so recurse into its fields to redact any credential-shaped one.
+ if dataclasses.is_dataclass(value) and not isinstance(value, type):
+ return sanitize_error_details(dataclasses.asdict(value))
+ if isinstance(value, str):
+ return sanitize_error_message(value)
+ return value
diff --git a/src/local_deep_research/web/exceptions.py b/src/local_deep_research/web/exceptions.py
index 1de4d1091d..ca8a593a97 100644
--- a/src/local_deep_research/web/exceptions.py
+++ b/src/local_deep_research/web/exceptions.py
@@ -8,6 +8,8 @@ appropriate JSON responses.
from typing import Any, Optional
+from ..security import sanitize_error_details, sanitize_error_for_client
+
class WebAPIException(Exception):
"""Base exception for all web API related errors."""
@@ -35,14 +37,37 @@ class WebAPIException(Exception):
self.details = details or {}
def to_dict(self) -> dict[str, Any]:
- """Convert exception to dictionary for JSON response."""
- result = {
+ """Convert exception to dictionary for JSON response.
+
+ This dict is ``jsonify``-ed straight to the client by the two handlers
+ that catch ``WebAPIException`` (``web/app_factory.py`` and
+ ``research_library/routes/library_routes.py``). As a boundary backstop,
+ the outgoing fields are scrubbed for credential shapes so that a future
+ raise site, subclass, or external caller that lets a credential-bearing
+ string in has it redacted before it ships:
+
+ * ``message`` — through ``sanitize_error_for_client`` (credential
+ redaction + control-char strip + 200-char cap; it is prose meant for
+ display).
+ * ``details`` string leaves — through ``sanitize_error_details`` (the
+ shared helper, also used by ``NewsAPIException``).
+
+ Redaction is credential-*shape* based (``Bearer``/``Authorization``,
+ ``?api_key=``-style query params, known token prefixes, URL userinfo),
+ so legitimate text is unchanged. It deliberately does NOT try to catch
+ DSN userinfo beyond URL form, SQL, schema, or filesystem paths. The
+ ``status`` literal and ``error_code`` are left untouched (``status_code``
+ is not part of this body — it is applied by the handler as the HTTP
+ status). ``self.message`` / ``self.details`` are not mutated, so the raw
+ text is retained on the exception object for server-side diagnosis.
+ """
+ result: dict[str, Any] = {
"status": "error",
- "message": self.message,
+ "message": sanitize_error_for_client(self.message),
"error_code": self.error_code,
}
if self.details:
- result["details"] = self.details # type: ignore[assignment]
+ result["details"] = sanitize_error_details(self.details)
return result
diff --git a/tests/security/test_sanitize_error_message.py b/tests/security/test_sanitize_error_message.py
index a1751bc6d6..1dee2e04e0 100644
--- a/tests/security/test_sanitize_error_message.py
+++ b/tests/security/test_sanitize_error_message.py
@@ -234,3 +234,136 @@ class TestDatabaseDsnCredentials:
engine = _MinimalEngine.__new__(_MinimalEngine)
actual = engine._sanitize_error_message(msg)
assert actual == expected
+
+
+class TestSanitizeErrorDetails:
+ """Unit tests for the shared ``sanitize_error_details`` helper used by the
+ exception ``to_dict()`` backstops (NewsAPIException / WebAPIException)."""
+
+ def test_str_leaf_credential_redacted(self):
+ from local_deep_research.security.log_sanitizer import (
+ sanitize_error_details,
+ )
+
+ out = sanitize_error_details(
+ {"note": "Authorization: Bearer sk-abc123DEF456ghi789"}
+ )
+ assert "sk-abc123DEF456ghi789" not in out["note"]
+ assert "REDACTED" in out["note"]
+
+ def test_benign_and_non_string_leaves_pass_through(self):
+ from local_deep_research.security.log_sanitizer import (
+ sanitize_error_details,
+ )
+
+ value = {
+ "query": "reset my password",
+ "count": 3,
+ "ok": True,
+ "x": None,
+ }
+ assert sanitize_error_details(value) == value
+
+ def test_nested_dict_list_recursion(self):
+ from local_deep_research.security.log_sanitizer import (
+ sanitize_error_details,
+ )
+
+ out = sanitize_error_details(
+ {"outer": {"inner": ["?api_key=AIzaSyD-EXAMPLE_1234567890", 7]}}
+ )
+ leaf = out["outer"]["inner"][0]
+ assert "AIzaSyD-EXAMPLE_1234567890" not in leaf
+ assert "REDACTED" in leaf
+ assert out["outer"]["inner"][1] == 7
+
+ def test_tuple_normalizes_to_list(self):
+ from local_deep_research.security.log_sanitizer import (
+ sanitize_error_details,
+ )
+
+ assert sanitize_error_details({"pair": ("a", "b")})["pair"] == [
+ "a",
+ "b",
+ ]
+
+ def test_namedtuple_does_not_crash(self):
+ from collections import namedtuple
+
+ from local_deep_research.security.log_sanitizer import (
+ sanitize_error_details,
+ )
+
+ Pair = namedtuple("Pair", ["a", "b"])
+ out = sanitize_error_details(
+ {"pair": Pair("Bearer sk-abc123DEF456ghi789", "x")}
+ )["pair"]
+ assert isinstance(out, list)
+ assert "sk-abc123DEF456ghi789" not in out[0]
+ assert "REDACTED" in out[0]
+ assert out[1] == "x"
+
+ def test_input_is_not_mutated(self):
+ from local_deep_research.security.log_sanitizer import (
+ sanitize_error_details,
+ )
+
+ original = {"note": "Bearer sk-secret999value000", "keep": [1, 2]}
+ out = sanitize_error_details(original)
+ assert original["note"] == "Bearer sk-secret999value000"
+ assert out["note"] != original["note"]
+ assert out["keep"] is not original["keep"]
+
+ def test_non_container_scalar_passthrough(self):
+ from local_deep_research.security.log_sanitizer import (
+ sanitize_error_details,
+ )
+
+ assert sanitize_error_details(42) == 42
+ assert sanitize_error_details(None) is None
+
+ def test_dataclass_leaf_is_recursed_and_redacted(self):
+ # Flask's default JSON provider serializes a dataclass via asdict(), so
+ # a credential in a dataclass field would ship to the client unless the
+ # helper recurses into it.
+ import dataclasses
+
+ from local_deep_research.security.log_sanitizer import (
+ sanitize_error_details,
+ )
+
+ @dataclasses.dataclass
+ class Ctx:
+ note: str
+ count: int
+
+ out = sanitize_error_details(
+ {"ctx": Ctx("Authorization: Bearer sk-abc123DEF456ghi789", 5)}
+ )
+ assert "sk-abc123DEF456ghi789" not in out["ctx"]["note"]
+ assert "REDACTED" in out["ctx"]["note"]
+ assert out["ctx"]["count"] == 5
+
+ def test_credential_shaped_dict_key_redacted(self):
+ from local_deep_research.security.log_sanitizer import (
+ sanitize_error_details,
+ )
+
+ out = sanitize_error_details(
+ {"Authorization: Bearer sk-abc123DEF456ghi789": "v"}
+ )
+ assert not any("sk-abc123DEF456ghi789" in k for k in out)
+ # benign field-name keys are untouched
+ assert sanitize_error_details({"username": "bob"}) == {
+ "username": "bob"
+ }
+
+ def test_set_leaf_passes_through(self):
+ # A set is not JSON-serializable, so it is left as-is (fails closed at
+ # jsonify rather than leaking) — pin the passthrough behavior.
+ from local_deep_research.security.log_sanitizer import (
+ sanitize_error_details,
+ )
+
+ s = {"a", "b"}
+ assert sanitize_error_details({"tags": s})["tags"] == s
diff --git a/tests/web/test_exceptions_behavior.py b/tests/web/test_exceptions_behavior.py
index 6fc0e0bc1c..a4817422cc 100644
--- a/tests/web/test_exceptions_behavior.py
+++ b/tests/web/test_exceptions_behavior.py
@@ -217,3 +217,123 @@ class TestAuthenticationRequiredError:
with pytest.raises(WebAPIException):
raise AuthenticationRequiredError()
+
+
+class TestWebToDictSanitization:
+ """to_dict() runs message + details string leaves through the central
+ sanitizers before the two WebAPIException handlers jsonify them.
+
+ Mirrors the NewsAPIException backstop: credential-shape redaction that
+ leaves legitimate content (the class a denylist would false-positive on)
+ untouched.
+ """
+
+ def test_benign_message_passes_through_unchanged(self):
+ from local_deep_research.web.exceptions import WebAPIException
+
+ msg = (
+ "Authentication required: Please refresh the page and log in again."
+ )
+ assert WebAPIException(msg).to_dict()["message"] == msg
+
+ def test_credential_keyword_in_prose_not_scrubbed(self):
+ from local_deep_research.web.exceptions import WebAPIException
+
+ msg = "how do I reset my password?"
+ assert WebAPIException(msg).to_dict()["message"] == msg
+
+ def test_credential_in_message_redacted(self):
+ from local_deep_research.web.exceptions import WebAPIException
+
+ exc = WebAPIException(
+ "auth failed: Authorization: Bearer sk-abc123DEF456ghi789"
+ )
+ message = exc.to_dict()["message"]
+ assert "sk-abc123DEF456ghi789" not in message
+ assert "REDACTED" in message
+
+ def test_long_message_truncated_to_client_cap(self):
+ from local_deep_research.web.exceptions import WebAPIException
+
+ message = WebAPIException("A" * 300).to_dict()["message"]
+ assert len(message) <= 200
+ assert message.endswith("...")
+
+ def test_stored_message_attribute_is_not_mutated(self):
+ from local_deep_research.web.exceptions import WebAPIException
+
+ raw = "boom: Authorization: Bearer sk-secret999value000"
+ exc = WebAPIException(raw)
+ assert exc.to_dict()["message"] != raw
+ assert exc.message == raw
+
+ def test_status_and_error_code_untouched(self):
+ from local_deep_research.web.exceptions import WebAPIException
+
+ d = WebAPIException(
+ "Bearer sk-abc123DEF456ghi789",
+ status_code=502,
+ error_code="UPSTREAM",
+ ).to_dict()
+ assert d["status"] == "error"
+ assert d["error_code"] == "UPSTREAM"
+
+ def test_benign_details_pass_through_unchanged(self):
+ from local_deep_research.web.exceptions import WebAPIException
+
+ exc = WebAPIException(
+ "err", details={"username": "testuser", "attempts": 3}
+ )
+ assert exc.to_dict()["details"] == {
+ "username": "testuser",
+ "attempts": 3,
+ }
+
+ def test_credential_in_details_string_leaf_redacted(self):
+ from local_deep_research.web.exceptions import WebAPIException
+
+ exc = WebAPIException(
+ "err",
+ details={"note": "Authorization: Bearer sk-abc123DEF456ghi789"},
+ )
+ note = exc.to_dict()["details"]["note"]
+ assert "sk-abc123DEF456ghi789" not in note
+ assert "REDACTED" in note
+
+ def test_namedtuple_detail_does_not_crash(self):
+ # Regression guard: a namedtuple in details must not crash to_dict()
+ # inside the error handler (the shared helper rebuilds it as a list).
+ from collections import namedtuple
+
+ from local_deep_research.web.exceptions import WebAPIException
+
+ Pair = namedtuple("Pair", ["a", "b"])
+ d = WebAPIException(
+ "err", details={"pair": Pair("Bearer sk-abc123DEF456ghi789", "x")}
+ ).to_dict()["details"]["pair"]
+ assert isinstance(d, list)
+ assert "sk-abc123DEF456ghi789" not in d[0]
+ assert "REDACTED" in d[0]
+ assert d[1] == "x"
+
+ def test_stored_details_attribute_is_not_mutated(self):
+ from local_deep_research.web.exceptions import WebAPIException
+
+ raw = {"note": "Bearer sk-secret999value000"}
+ exc = WebAPIException("err", details=raw)
+ assert "REDACTED" in exc.to_dict()["details"]["note"]
+ assert exc.details["note"] == "Bearer sk-secret999value000"
+
+ def test_subclass_details_sanitized_via_inherited_to_dict(self):
+ # The PR's rationale: a subclass that forwards caller data into details
+ # still gets the backstop, because to_dict() is inherited.
+ from local_deep_research.web.exceptions import (
+ AuthenticationRequiredError,
+ )
+
+ exc = AuthenticationRequiredError(
+ username="Bearer sk-abc123DEF456ghi789"
+ )
+ username = exc.to_dict()["details"]["username"]
+ assert "sk-abc123DEF456ghi789" not in username
+ assert "REDACTED" in username