~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
security: WebAPIException to_dict() backstop + extract shared sanitize_error_details (DRY) (#4937)
LearningCircuit004c32ccc049
diff --git a/changelog.d/4937.security.md b/changelog.d/4937.security.mdnew file mode 100644index 0000000000..39429fee57--- /dev/null+++ b/changelog.d/4937.security.md@@ -0,0 +1 @@+`WebAPIException.to_dict()` now redacts credential shapes from the fields it serializes to the client (`message` via `sanitize_error_for_client()`, `details` string leaves via `sanitize_error_message()`), mirroring the `NewsAPIException` backstop. This is a defence-in-depth measure at the response boundary so that a future raise site, subclass, or external caller that lets a credential-bearing string into `message` or `details` has it redacted before it ships. Legitimate text is unchanged, and `status` / `error_code` / `status_code` are left untouched.diff --git a/src/local_deep_research/news/exceptions.py b/src/local_deep_research/news/exceptions.pyindex e943e871df..4f7afdbf8e 100644--- a/src/local_deep_research/news/exceptions.py+++ b/src/local_deep_research/news/exceptions.py@@ -8,7 +8,7 @@ appropriate JSON responses.from typing import Optional, Dict, Any-from ..security import sanitize_error_for_client, sanitize_error_message+from ..security import sanitize_error_details, sanitize_error_for_clientclass NewsAPIException(Exception):@@ -36,40 +36,6 @@ class NewsAPIException(Exception):self.error_code = error_code or self.__class__.__name__self.details = details or {}- @staticmethod- def _sanitize_details(value: Any) -> Any:- """Recursively redact credential shapes from string leaves of ``details``.-- ``details`` can carry caller-supplied free text — e.g.- ``SubscriptionCreationException`` forwards ``{"query": <user query>}``- (``news/api.py``), and the base constructor accepts an arbitrary- ``details`` dict from any caller. So the same "don't trust a future- caller" premise that scrubs ``message`` applies here. This walks- ``dict`` / ``list`` / ``tuple`` containers and runs each ``str`` leaf- through ``sanitize_error_message`` (credential-shape redaction only —- no length cap or control-char strip, so structured values such as IDs- are preserved). Non-string leaves (ints, bools, ``None``) pass through- untouched. Redaction is in place and shape-based, so it never removes a- value by key name — a benign ``{"query": "reset my password"}`` is- unchanged.- """- if isinstance(value, dict):- return {- k: NewsAPIException._sanitize_details(v)- for k, v in value.items()- }- if isinstance(value, (list, tuple)):- # Always rebuild as a plain list. ``details`` is about to be- # jsonify-ed (a tuple already serializes to a JSON array), so tuple- # identity is irrelevant downstream — and ``type(value)(<gen>)``- # would raise on a namedtuple / tuple subclass whose constructor is- # not ``(iterable) -> instance``, turning a clean structured error- # response into a bare 500 from inside the error handler.- return [NewsAPIException._sanitize_details(v) for v in value]- if isinstance(value, str):- return sanitize_error_message(value)- return value-def to_dict(self) -> Dict[str, Any]:"""Convert exception to dictionary for JSON response.@@ -85,8 +51,8 @@ class NewsAPIException(Exception):* ``message`` — through ``sanitize_error_for_client`` (credentialredaction + control-char strip + 200-char cap; it is prose meant fordisplay).- * ``details`` string leaves — through ``sanitize_error_message`` (see- ``_sanitize_details``).+ * ``details`` string leaves — through ``sanitize_error_details`` (the+ shared helper, also used by ``WebAPIException``).Redaction is credential-*shape* based (``Bearer``/``Authorization``,``?api_key=``-style query params, known token prefixes, ``http(s)``@@ -108,7 +74,7 @@ class NewsAPIException(Exception):"status_code": self.status_code,}if self.details:- result["details"] = self._sanitize_details(self.details)+ result["details"] = sanitize_error_details(self.details)return resultdiff --git a/src/local_deep_research/security/__init__.py b/src/local_deep_research/security/__init__.pyindex 5e1f384403..1abbcd7b15 100644--- a/src/local_deep_research/security/__init__.py+++ b/src/local_deep_research/security/__init__.py@@ -35,6 +35,7 @@ from .account_lockout import AccountLockoutManager, get_account_lockout_managerfrom .password_validator import PasswordValidatorfrom .log_sanitizer import (redact_secrets,+ sanitize_error_details,sanitize_error_for_client,sanitize_error_message,sanitize_for_log,@@ -101,6 +102,7 @@ __all__ = ["get_account_lockout_manager","PasswordValidator","redact_secrets",+ "sanitize_error_details","sanitize_error_for_client","sanitize_error_message","sanitize_for_log",diff --git a/src/local_deep_research/security/log_sanitizer.py b/src/local_deep_research/security/log_sanitizer.pyindex 920ec58fa6..1b68ff3310 100644--- a/src/local_deep_research/security/log_sanitizer.py+++ b/src/local_deep_research/security/log_sanitizer.py@@ -10,10 +10,15 @@ concerns:* :func:`redact_secrets` \u2014 scrub known sensitive *values* (API keys,passwords, session tokens) from an arbitrary string before it islogged, returned in an error message, or persisted.+* :func:`sanitize_error_details` \u2014 recurse a structured ``details`` value+ and redact credential *shapes* from its string leaves before it is+ serialized to a client. (Value-shape based; contrast+ ``data_sanitizer.DataSanitizer``, which redacts by key *name*.)"""+import dataclassesimport re-from typing import Optional+from typing import Any, Optional# Strip C0/C1 control characters and dangerous Unicode format characters,@@ -304,3 +309,61 @@ def sanitize_error_for_client(message: str, max_length: int = 200) -> str:return sanitize_for_log(sanitize_error_message(message), max_length=max_length)+++def sanitize_error_details(value: Any) -> Any:+ """Recursively redact credential *shapes* from the string leaves of a+ structured ``details`` value (``dict`` / ``list`` / ``tuple`` / dataclass).++ Intended for the ``details`` payload of an exception ``to_dict()`` that is+ serialized to a client (e.g. ``NewsAPIException`` / ``WebAPIException``).+ This is the *value-shape* counterpart to :class:`data_sanitizer.DataSanitizer`,+ which redacts by *key name*; use this one when the concern is a credential+ embedded anywhere in the text, regardless of the key it sits under.++ Behaviour by node type:++ * ``dict`` — recurse into values; ``str`` keys are also run through+ :func:`sanitize_error_message` (a credential used *as* a key would+ otherwise ship verbatim as a JSON key). Two distinct keys that both+ redact to the same token collapse to one entry — an acceptable+ fidelity loss for the pathological case of credential-shaped keys.+ * ``list`` / ``tuple`` — both rebuilt as a plain ``list``: the payload is+ about to be JSON-serialized (a tuple already becomes a JSON array), and+ rebuilding via ``type(value)(<generator>)`` would raise on a namedtuple /+ tuple subclass whose constructor is not ``(iterable) -> instance`` —+ inside a Flask error handler that degrades a structured error into a bare+ 500.+ * **dataclass instance** — converted via ``dataclasses.asdict`` and recursed.+ Flask's default JSON provider serializes a dataclass (via ``asdict``), so+ without this a credential in a dataclass field would ship un-redacted.+ * ``str`` leaf — redacted via :func:`sanitize_error_message` (credential-shape+ redaction only — no length cap or control-char strip, so structured values+ such as IDs survive).+ * anything else (ints, bools, ``None``, and any other object) — passed+ through untouched. Note some passthrough types (``set``/``frozenset``,+ ``bytes``, a non-``str``-mixin ``Enum``) are not JSON-serializable, so a+ credential inside them fails *closed* at ``jsonify`` (generic 500) rather+ than leaking.++ Redaction is shape-based, so it never removes a value by key name — a benign+ ``{"query": "reset my password"}`` is unchanged. Containers are rebuilt fresh+ (the input is not mutated); passthrough leaf objects are returned by+ reference.+ """+ if isinstance(value, dict):+ return {+ (sanitize_error_message(k) if isinstance(k, str) else k): (+ sanitize_error_details(v)+ )+ for k, v in value.items()+ }+ if isinstance(value, (list, tuple)):+ return [sanitize_error_details(v) for v in value]+ # dataclass *instance* (not the class itself) — Flask serializes it via+ # asdict(), so recurse into its fields to redact any credential-shaped one.+ if dataclasses.is_dataclass(value) and not isinstance(value, type):+ return sanitize_error_details(dataclasses.asdict(value))+ if isinstance(value, str):+ return sanitize_error_message(value)+ return valuediff --git a/src/local_deep_research/web/exceptions.py b/src/local_deep_research/web/exceptions.pyindex 1de4d1091d..ca8a593a97 100644--- a/src/local_deep_research/web/exceptions.py+++ b/src/local_deep_research/web/exceptions.py@@ -8,6 +8,8 @@ appropriate JSON responses.from typing import Any, Optional+from ..security import sanitize_error_details, sanitize_error_for_client+class WebAPIException(Exception):"""Base exception for all web API related errors."""@@ -35,14 +37,37 @@ class WebAPIException(Exception):self.details = details or {}def to_dict(self) -> dict[str, Any]:- """Convert exception to dictionary for JSON response."""- result = {+ """Convert exception to dictionary for JSON response.++ This dict is ``jsonify``-ed straight to the client by the two handlers+ that catch ``WebAPIException`` (``web/app_factory.py`` and+ ``research_library/routes/library_routes.py``). As a boundary backstop,+ the outgoing fields are scrubbed for credential shapes so that a future+ raise site, subclass, or external caller that lets a credential-bearing+ string in has it redacted before it ships:++ * ``message`` — through ``sanitize_error_for_client`` (credential+ redaction + control-char strip + 200-char cap; it is prose meant for+ display).+ * ``details`` string leaves — through ``sanitize_error_details`` (the+ shared helper, also used by ``NewsAPIException``).++ Redaction is credential-*shape* based (``Bearer``/``Authorization``,+ ``?api_key=``-style query params, known token prefixes, URL userinfo),+ so legitimate text is unchanged. It deliberately does NOT try to catch+ DSN userinfo beyond URL form, SQL, schema, or filesystem paths. The+ ``status`` literal and ``error_code`` are left untouched (``status_code``+ is not part of this body — it is applied by the handler as the HTTP+ status). ``self.message`` / ``self.details`` are not mutated, so the raw+ text is retained on the exception object for server-side diagnosis.+ """+ result: dict[str, Any] = {"status": "error",- "message": self.message,+ "message": sanitize_error_for_client(self.message),"error_code": self.error_code,}if self.details:- result["details"] = self.details # type: ignore[assignment]+ result["details"] = sanitize_error_details(self.details)return resultdiff --git a/tests/security/test_sanitize_error_message.py b/tests/security/test_sanitize_error_message.pyindex a1751bc6d6..1dee2e04e0 100644--- a/tests/security/test_sanitize_error_message.py+++ b/tests/security/test_sanitize_error_message.py@@ -234,3 +234,136 @@ class TestDatabaseDsnCredentials:engine = _MinimalEngine.__new__(_MinimalEngine)actual = engine._sanitize_error_message(msg)assert actual == expected+++class TestSanitizeErrorDetails:+ """Unit tests for the shared ``sanitize_error_details`` helper used by the+ exception ``to_dict()`` backstops (NewsAPIException / WebAPIException)."""++ def test_str_leaf_credential_redacted(self):+ from local_deep_research.security.log_sanitizer import (+ sanitize_error_details,+ )++ out = sanitize_error_details(+ {"note": "Authorization: Bearer sk-abc123DEF456ghi789"}+ )+ assert "sk-abc123DEF456ghi789" not in out["note"]+ assert "REDACTED" in out["note"]++ def test_benign_and_non_string_leaves_pass_through(self):+ from local_deep_research.security.log_sanitizer import (+ sanitize_error_details,+ )++ value = {+ "query": "reset my password",+ "count": 3,+ "ok": True,+ "x": None,+ }+ assert sanitize_error_details(value) == value++ def test_nested_dict_list_recursion(self):+ from local_deep_research.security.log_sanitizer import (+ sanitize_error_details,+ )++ out = sanitize_error_details(+ {"outer": {"inner": ["?api_key=AIzaSyD-EXAMPLE_1234567890", 7]}}+ )+ leaf = out["outer"]["inner"][0]+ assert "AIzaSyD-EXAMPLE_1234567890" not in leaf+ assert "REDACTED" in leaf+ assert out["outer"]["inner"][1] == 7++ def test_tuple_normalizes_to_list(self):+ from local_deep_research.security.log_sanitizer import (+ sanitize_error_details,+ )++ assert sanitize_error_details({"pair": ("a", "b")})["pair"] == [+ "a",+ "b",+ ]++ def test_namedtuple_does_not_crash(self):+ from collections import namedtuple++ from local_deep_research.security.log_sanitizer import (+ sanitize_error_details,+ )++ Pair = namedtuple("Pair", ["a", "b"])+ out = sanitize_error_details(+ {"pair": Pair("Bearer sk-abc123DEF456ghi789", "x")}+ )["pair"]+ assert isinstance(out, list)+ assert "sk-abc123DEF456ghi789" not in out[0]+ assert "REDACTED" in out[0]+ assert out[1] == "x"++ def test_input_is_not_mutated(self):+ from local_deep_research.security.log_sanitizer import (+ sanitize_error_details,+ )++ original = {"note": "Bearer sk-secret999value000", "keep": [1, 2]}+ out = sanitize_error_details(original)+ assert original["note"] == "Bearer sk-secret999value000"+ assert out["note"] != original["note"]+ assert out["keep"] is not original["keep"]++ def test_non_container_scalar_passthrough(self):+ from local_deep_research.security.log_sanitizer import (+ sanitize_error_details,+ )++ assert sanitize_error_details(42) == 42+ assert sanitize_error_details(None) is None++ def test_dataclass_leaf_is_recursed_and_redacted(self):+ # Flask's default JSON provider serializes a dataclass via asdict(), so+ # a credential in a dataclass field would ship to the client unless the+ # helper recurses into it.+ import dataclasses++ from local_deep_research.security.log_sanitizer import (+ sanitize_error_details,+ )++ @dataclasses.dataclass+ class Ctx:+ note: str+ count: int++ out = sanitize_error_details(+ {"ctx": Ctx("Authorization: Bearer sk-abc123DEF456ghi789", 5)}+ )+ assert "sk-abc123DEF456ghi789" not in out["ctx"]["note"]+ assert "REDACTED" in out["ctx"]["note"]+ assert out["ctx"]["count"] == 5++ def test_credential_shaped_dict_key_redacted(self):+ from local_deep_research.security.log_sanitizer import (+ sanitize_error_details,+ )++ out = sanitize_error_details(+ {"Authorization: Bearer sk-abc123DEF456ghi789": "v"}+ )+ assert not any("sk-abc123DEF456ghi789" in k for k in out)+ # benign field-name keys are untouched+ assert sanitize_error_details({"username": "bob"}) == {+ "username": "bob"+ }++ def test_set_leaf_passes_through(self):+ # A set is not JSON-serializable, so it is left as-is (fails closed at+ # jsonify rather than leaking) — pin the passthrough behavior.+ from local_deep_research.security.log_sanitizer import (+ sanitize_error_details,+ )++ s = {"a", "b"}+ assert sanitize_error_details({"tags": s})["tags"] == sdiff --git a/tests/web/test_exceptions_behavior.py b/tests/web/test_exceptions_behavior.pyindex 6fc0e0bc1c..a4817422cc 100644--- a/tests/web/test_exceptions_behavior.py+++ b/tests/web/test_exceptions_behavior.py@@ -217,3 +217,123 @@ class TestAuthenticationRequiredError:with pytest.raises(WebAPIException):raise AuthenticationRequiredError()+++class TestWebToDictSanitization:+ """to_dict() runs message + details string leaves through the central+ sanitizers before the two WebAPIException handlers jsonify them.++ Mirrors the NewsAPIException backstop: credential-shape redaction that+ leaves legitimate content (the class a denylist would false-positive on)+ untouched.+ """++ def test_benign_message_passes_through_unchanged(self):+ from local_deep_research.web.exceptions import WebAPIException++ msg = (+ "Authentication required: Please refresh the page and log in again."+ )+ assert WebAPIException(msg).to_dict()["message"] == msg++ def test_credential_keyword_in_prose_not_scrubbed(self):+ from local_deep_research.web.exceptions import WebAPIException++ msg = "how do I reset my password?"+ assert WebAPIException(msg).to_dict()["message"] == msg++ def test_credential_in_message_redacted(self):+ from local_deep_research.web.exceptions import WebAPIException++ exc = WebAPIException(+ "auth failed: Authorization: Bearer sk-abc123DEF456ghi789"+ )+ message = exc.to_dict()["message"]+ assert "sk-abc123DEF456ghi789" not in message+ assert "REDACTED" in message++ def test_long_message_truncated_to_client_cap(self):+ from local_deep_research.web.exceptions import WebAPIException++ message = WebAPIException("A" * 300).to_dict()["message"]+ assert len(message) <= 200+ assert message.endswith("...")++ def test_stored_message_attribute_is_not_mutated(self):+ from local_deep_research.web.exceptions import WebAPIException++ raw = "boom: Authorization: Bearer sk-secret999value000"+ exc = WebAPIException(raw)+ assert exc.to_dict()["message"] != raw+ assert exc.message == raw++ def test_status_and_error_code_untouched(self):+ from local_deep_research.web.exceptions import WebAPIException++ d = WebAPIException(+ "Bearer sk-abc123DEF456ghi789",+ status_code=502,+ error_code="UPSTREAM",+ ).to_dict()+ assert d["status"] == "error"+ assert d["error_code"] == "UPSTREAM"++ def test_benign_details_pass_through_unchanged(self):+ from local_deep_research.web.exceptions import WebAPIException++ exc = WebAPIException(+ "err", details={"username": "testuser", "attempts": 3}+ )+ assert exc.to_dict()["details"] == {+ "username": "testuser",+ "attempts": 3,+ }++ def test_credential_in_details_string_leaf_redacted(self):+ from local_deep_research.web.exceptions import WebAPIException++ exc = WebAPIException(+ "err",+ details={"note": "Authorization: Bearer sk-abc123DEF456ghi789"},+ )+ note = exc.to_dict()["details"]["note"]+ assert "sk-abc123DEF456ghi789" not in note+ assert "REDACTED" in note++ def test_namedtuple_detail_does_not_crash(self):+ # Regression guard: a namedtuple in details must not crash to_dict()+ # inside the error handler (the shared helper rebuilds it as a list).+ from collections import namedtuple++ from local_deep_research.web.exceptions import WebAPIException++ Pair = namedtuple("Pair", ["a", "b"])+ d = WebAPIException(+ "err", details={"pair": Pair("Bearer sk-abc123DEF456ghi789", "x")}+ ).to_dict()["details"]["pair"]+ assert isinstance(d, list)+ assert "sk-abc123DEF456ghi789" not in d[0]+ assert "REDACTED" in d[0]+ assert d[1] == "x"++ def test_stored_details_attribute_is_not_mutated(self):+ from local_deep_research.web.exceptions import WebAPIException++ raw = {"note": "Bearer sk-secret999value000"}+ exc = WebAPIException("err", details=raw)+ assert "REDACTED" in exc.to_dict()["details"]["note"]+ assert exc.details["note"] == "Bearer sk-secret999value000"++ def test_subclass_details_sanitized_via_inherited_to_dict(self):+ # The PR's rationale: a subclass that forwards caller data into details+ # still gets the backstop, because to_dict() is inherited.+ from local_deep_research.web.exceptions import (+ AuthenticationRequiredError,+ )++ exc = AuthenticationRequiredError(+ username="Bearer sk-abc123DEF456ghi789"+ )+ username = exc.to_dict()["details"]["username"]+ assert "sk-abc123DEF456ghi789" not in username+ assert "REDACTED" in username