~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
fix(security): close password frame-locals leak class at the sink + targeted sweep (#4182) (#4555)
LearningCircuit1b81e80293fa
diff --git a/changelog.d/+full-traceback-redaction-sweep.security.md b/changelog.d/+full-traceback-redaction-sweep.security.mdnew file mode 100644index 0000000000..747b0e9e30--- /dev/null+++ b/changelog.d/+full-traceback-redaction-sweep.security.md@@ -0,0 +1 @@+Hardened the SQLCipher-master-password frame-locals leak class (#4182) at two levels. (1) Sink level: `log_utils` now forces `diagnose=False` on the database sink (which persists into the user's own encrypted DB) and the frontend sink (which ships to the browser), so loguru can never render frame-local credentials into a durable or remote sink even when an operator enables `LDR_LOGURU_DIAGNOSE` for local debugging — a single chokepoint covering every credential-bearing exception handler app-wide. (2) Targeted sweep: the credential-centric session helpers `metrics/search_tracker`, `metrics/token_counter`, `database/library_init`, `database/backup/backup_executor`, `web_search_engines/rate_limiting/tracker`, and `research_library/.../research_history_indexer` now log failures with `logger.warning` instead of `logger.exception`, and are added to the `test_password_redaction_invariant` allow-list so CI permanently blocks regressions. Big multi-purpose route/service handlers are intentionally left logging full tracebacks for unrelated errors — they are covered by the sink-level guard. See #4182.diff --git a/src/local_deep_research/database/backup/backup_executor.py b/src/local_deep_research/database/backup/backup_executor.pyindex 200c3a1e0f..8c7afe0609 100644--- a/src/local_deep_research/database/backup/backup_executor.py+++ b/src/local_deep_research/database/backup/backup_executor.py@@ -132,7 +132,7 @@ class BackupExecutor:return resultexcept Exception as e:- logger.exception("Background backup error")+ logger.warning("Background backup error")return BackupResult(success=False, error=str(e))def _backup_completed(self, username: str, future) -> None:diff --git a/src/local_deep_research/database/library_init.py b/src/local_deep_research/database/library_init.pyindex 290c0fdd0e..2cbcab8e12 100644--- a/src/local_deep_research/database/library_init.py+++ b/src/local_deep_research/database/library_init.py@@ -118,7 +118,7 @@ def seed_source_types(username: str, password: str = None) -> None:except IntegrityError:logger.warning("Source types may already exist")except Exception:- logger.exception("Error seeding source types")+ logger.warning("Error seeding source types")raise@@ -166,7 +166,7 @@ def ensure_default_library_collection(return library_idexcept Exception:- logger.exception("Error ensuring default Library collection")+ logger.warning("Error ensuring default Library collection")raise@@ -219,7 +219,7 @@ def ensure_research_history_collection(return collection_idexcept Exception:- logger.exception("Error ensuring Research History collection")+ logger.warning("Error ensuring Research History collection")raise@@ -261,7 +261,7 @@ def initialize_library_for_user(username: str, password: str = None) -> dict:logger.info(f"Library initialization complete for user: {username}")except Exception as e:- logger.exception(f"Library initialization failed for {username}")+ logger.warning(f"Library initialization failed for {username}")results["error"] = str(e)return results@@ -311,5 +311,5 @@ def get_source_type_id(return source_type.idexcept Exception:- logger.exception("Error getting source type ID")+ logger.warning("Error getting source type ID")raisediff --git a/src/local_deep_research/metrics/search_tracker.py b/src/local_deep_research/metrics/search_tracker.pyindex d479c26929..b0d98aa449 100644--- a/src/local_deep_research/metrics/search_tracker.py+++ b/src/local_deep_research/metrics/search_tracker.py@@ -114,10 +114,10 @@ class SearchTracker:f"{results_count} results in {response_time_ms}ms")except Exception:- logger.exception("Failed to write search metrics")+ logger.warning("Failed to write search metrics")except Exception:- logger.exception("Failed to record search call")+ logger.warning("Failed to record search call")def get_search_metrics(self,@@ -236,7 +236,7 @@ class SearchTracker:}except Exception:- logger.exception("Error getting search metrics")+ logger.warning("Error getting search metrics")return {"search_engine_stats": [], "recent_calls": []}def get_research_search_metrics(@@ -337,7 +337,7 @@ class SearchTracker:}except Exception:- logger.exception("Error getting research search metrics")+ logger.warning("Error getting research search metrics")return {"total_searches": 0,"total_results": 0,@@ -415,7 +415,7 @@ class SearchTracker:return time_seriesexcept Exception:- logger.exception("Error getting search time series")+ logger.warning("Error getting search time series")return []diff --git a/src/local_deep_research/metrics/token_counter.py b/src/local_deep_research/metrics/token_counter.pyindex d52686ee68..763a438425 100644--- a/src/local_deep_research/metrics/token_counter.py+++ b/src/local_deep_research/metrics/token_counter.py@@ -662,7 +662,7 @@ class TokenCountingCallback(BaseCallbackHandler):username, self.research_id, token_data)except Exception:- logger.exception("Failed to write metrics from thread")+ logger.warning("Failed to write metrics from thread")return# In MainThread, save directly@@ -769,7 +769,7 @@ class TokenCountingCallback(BaseCallbackHandler):session.commit()except Exception:- logger.exception("Error saving token usage to database")+ logger.warning("Error saving token usage to database")def get_counts(self) -> Dict[str, Any]:"""Get the current token counts."""diff --git a/src/local_deep_research/research_library/search/services/research_history_indexer.py b/src/local_deep_research/research_library/search/services/research_history_indexer.pyindex 1d434d29ca..3dc8be7f15 100644--- a/src/local_deep_research/research_library/search/services/research_history_indexer.py+++ b/src/local_deep_research/research_library/search/services/research_history_indexer.py@@ -365,6 +365,6 @@ def auto_convert_research(f"{result.get('status')}")except Exception:- logger.exception(+ logger.warning(f"Failed to auto-convert research {research_id} for user {username}")diff --git a/src/local_deep_research/utilities/log_utils.py b/src/local_deep_research/utilities/log_utils.pyindex d9e5481f02..76519ee03e 100644--- a/src/local_deep_research/utilities/log_utils.py+++ b/src/local_deep_research/utilities/log_utils.py@@ -553,9 +553,19 @@ def config_logger(name: str, debug: bool = False) -> None:"LDR_LOGURU_DIAGNOSE", "").strip().lower() in ("1", "true", "yes")+ # ``diagnose`` renders the repr() of every frame-local — which can+ # include the SQLCipher master password and other credentials — into+ # the rendered exception block. Allow it ONLY on the ephemeral, local+ # stderr sink the operator explicitly opted into. The database sink+ # PERSISTS into the user's own encrypted DB and the frontend sink+ # SHIPS to the browser, so they must NEVER render frame locals, even+ # under LDR_LOGURU_DIAGNOSE. This single chokepoint protects every+ # credential-bearing exception handler app-wide against the+ # frame-locals leak, independent of per-site logging discipline+ # (#4182).logger.add(sys.stderr, level=stderr_level, diagnose=diagnose)- logger.add(database_sink, level="DEBUG", diagnose=diagnose)- logger.add(frontend_progress_sink, diagnose=diagnose)+ logger.add(database_sink, level="DEBUG", diagnose=False)+ logger.add(frontend_progress_sink, diagnose=False)if debug:logger.warning(diff --git a/src/local_deep_research/web_search_engines/rate_limiting/tracker.py b/src/local_deep_research/web_search_engines/rate_limiting/tracker.pyindex d06e20b747..b919b9ec3a 100644--- a/src/local_deep_research/web_search_engines/rate_limiting/tracker.py+++ b/src/local_deep_research/web_search_engines/rate_limiting/tracker.py@@ -517,7 +517,7 @@ class AdaptiveRateLimitTracker:session_inner.add(estimate)except Exception:- logger.exception("Failed to persist rate limit estimate")+ logger.warning("Failed to persist rate limit estimate")else:logger.debug("Skipping rate limit estimate save - no user context"diff --git a/tests/security/test_password_redaction_invariant.py b/tests/security/test_password_redaction_invariant.pyindex f01db94fd5..a73d92ee9f 100644--- a/tests/security/test_password_redaction_invariant.py+++ b/tests/security/test_password_redaction_invariant.py@@ -58,6 +58,12 @@ import local_deep_research.web.queue.processor_v2 as processor_v2_moduleimport local_deep_research.database.thread_local_session as thread_local_session_moduleimport local_deep_research.library.download_management.status_tracker as status_tracker_moduleimport local_deep_research.database.thread_metrics as thread_metrics_module+import local_deep_research.metrics.search_tracker as search_tracker_module+import local_deep_research.metrics.token_counter as token_counter_module+import local_deep_research.database.library_init as library_init_module+import local_deep_research.database.backup.backup_executor as backup_executor_module+import local_deep_research.web_search_engines.rate_limiting.tracker as rate_limit_tracker_module+import local_deep_research.research_library.search.services.research_history_indexer as research_history_indexer_moduleSWEPT_MODULES = [encrypted_db_module,@@ -71,6 +77,18 @@ SWEPT_MODULES = [# Holds the master password in scope while opening a per-thread# metrics session and logging the failure (#4182 follow-up).thread_metrics_module,+ # Credential-centric helpers that open an encrypted session with the+ # master password and log failures (#4182 targeted sweep). The big+ # multi-purpose route/service modules are NOT listed: they are+ # protected at the sink level (log_utils forces diagnose=False on the+ # persisted DB / frontend sinks) so their unrelated-error tracebacks+ # stay useful.+ search_tracker_module,+ token_counter_module,+ library_init_module,+ backup_executor_module,+ rate_limit_tracker_module,+ research_history_indexer_module,]# Names that match the password heuristic but are not credential values.diff --git a/tests/test_context_overflow_detection.py b/tests/test_context_overflow_detection.pyindex a35532b68b..1ff272aa99 100644--- a/tests/test_context_overflow_detection.py+++ b/tests/test_context_overflow_detection.py@@ -12,6 +12,22 @@ from local_deep_research.database.models import Basefrom local_deep_research.metrics.token_counter import TokenCountingCallback+def _overflow_warning(mock_logger) -> str:+ """Return the concatenated text of the "Context overflow detected"+ warning among all ``logger.warning`` calls (``""`` if none).++ The callback also persists metrics, and that path's failure handler now+ logs via ``logger.warning`` too (#4182 sink/redaction sweep), so the+ overflow warning is no longer guaranteed to be the *last* warning call.+ Match it by its stable anchor instead of by call position.+ """+ for call in mock_logger.warning.call_args_list:+ text = " ".join(str(a) for a in call.args)+ if "Context overflow detected" in text:+ return text+ return ""++class TestContextOverflowDetection:"""Test suite for context overflow detection."""@@ -197,9 +213,7 @@ class TestContextOverflowDetection:mock_logger.warning.assert_called()# The new logger.warning is called with multiple string args that get# concatenated by f-strings; inspect each positional arg for the fields.- warning_call = " ".join(- str(a) for a in mock_logger.warning.call_args[0]- )+ warning_call = _overflow_warning(mock_logger)assert "Context overflow detected" in warning_callassert "[provider-confirmed]" in warning_callassert "prompt_tokens=2000" in warning_call@@ -236,9 +250,7 @@ class TestContextOverflowDetection:assert token_callback.context_truncated is Trueassert token_callback.tokens_truncated > 0mock_logger.warning.assert_called()- warning_call = " ".join(- str(a) for a in mock_logger.warning.call_args[0]- )+ warning_call = _overflow_warning(mock_logger)assert "[estimated]" in warning_callassert "provider=openai" in warning_callassert "context_limit=2048" in warning_call@@ -272,9 +284,7 @@ class TestContextOverflowDetection:assert token_callback.context_truncated is Truemock_logger.warning.assert_called()- warning_call = " ".join(- str(a) for a in mock_logger.warning.call_args[0]- )+ warning_call = _overflow_warning(mock_logger)assert "[total-context]" in warning_callassert "prompt_tokens=1500" in warning_callassert "completion_tokens=500" in warning_call@@ -314,9 +324,7 @@ class TestContextOverflowDetection:assert token_callback.context_truncated is Trueassert token_callback.tokens_truncated > 0mock_logger.warning.assert_called()- warning_call = " ".join(- str(a) for a in mock_logger.warning.call_args[0]- )+ warning_call = _overflow_warning(mock_logger)assert "[total-context]" in warning_callassert "prompt_tokens=1500" in warning_callassert "completion_tokens=600" in warning_call@@ -367,9 +375,7 @@ class TestContextOverflowDetection:assert token_callback.context_truncated is Trueassert token_callback.tokens_truncated > 0mock_logger.warning.assert_called()- warning_call = " ".join(- str(a) for a in mock_logger.warning.call_args[0]- )+ warning_call = _overflow_warning(mock_logger)assert "[estimated-total-context]" in warning_callassert "prompt_tokens=1500" in warning_callassert "completion_tokens=600" in warning_calldiff --git a/tests/utilities/test_log_utils_coverage.py b/tests/utilities/test_log_utils_coverage.pyindex a7d7092804..9bdea7e9be 100644--- a/tests/utilities/test_log_utils_coverage.py+++ b/tests/utilities/test_log_utils_coverage.py@@ -272,3 +272,68 @@ class TestConfigLoggerDebugAndFileLogging:first_add = mock_logger.add.call_args_list[0]assert first_add[1].get("level") == "DEBUG"+++# ---------------------------------------------------------------------------+# Sink-level diagnose policy (#4182): even when the operator opts into+# LDR_LOGURU_DIAGNOSE, the persisted DB sink and the browser-facing+# frontend sink must NEVER render frame locals (which can hold the+# SQLCipher master password). Only the local stderr sink may.+# ---------------------------------------------------------------------------+++class TestDiagnoseSinkPolicy:+ def test_diagnose_never_enabled_on_db_or_frontend_sink(self):+ from local_deep_research.utilities.log_utils import (+ config_logger,+ database_sink,+ frontend_progress_sink,+ )+ import os++ env = {+ k: v+ for k, v in os.environ.items()+ if k != "LDR_ENABLE_FILE_LOGGING"+ }+ # Opt into diagnose the way an operator debugging locally would.+ env["LDR_LOGURU_DIAGNOSE"] = "true"++ with patch.dict("os.environ", env, clear=True):+ with patch(f"{MODULE}.logger") as mock_logger:+ config_logger("app", debug=True)++ # Map each add() call to its sink (first positional arg).+ by_sink = {+ call.args[0]: call.kwargs.get("diagnose")+ for call in mock_logger.add.call_args_list+ }++ # stderr opted in -> diagnose may be True; the persisted and+ # shipped sinks must be hard False regardless.+ assert by_sink[database_sink] is False+ assert by_sink[frontend_progress_sink] is False++ def test_stderr_gets_diagnose_when_opted_in(self):+ import sys+ from local_deep_research.utilities.log_utils import config_logger+ import os++ env = {+ k: v+ for k, v in os.environ.items()+ if k != "LDR_ENABLE_FILE_LOGGING"+ }+ env["LDR_LOGURU_DIAGNOSE"] = "true"++ with patch.dict("os.environ", env, clear=True):+ with patch(f"{MODULE}.logger") as mock_logger:+ config_logger("app", debug=True)++ stderr_calls = [+ c+ for c in mock_logger.add.call_args_list+ if c.args and c.args[0] is sys.stderr+ ]+ assert stderr_calls, "stderr sink was not configured"+ assert stderr_calls[0].kwargs.get("diagnose") is Truediff --git a/tests/utilities/test_log_utils_extended.py b/tests/utilities/test_log_utils_extended.pyindex d01e2c7385..f0737ef7d0 100644--- a/tests/utilities/test_log_utils_extended.py+++ b/tests/utilities/test_log_utils_extended.py@@ -592,7 +592,13 @@ class TestConfigLoggerExtended:assert mock_logger.add.call_count == 4def test_diagnose_true_requires_both_debug_and_optin(self):- """diagnose=True requires BOTH debug=True AND LDR_LOGURU_DIAGNOSE opt-in."""+ """diagnose=True requires BOTH debug=True AND LDR_LOGURU_DIAGNOSE opt-in.++ With both set, the local stderr sink renders frame locals, but the+ persisted DB sink and the browser-facing frontend sink are forced+ diagnose=False regardless (#4182).+ """+ import sysfrom local_deep_research.utilities.log_utils import config_loggerimport local_deep_research.utilities.log_utils as module@@ -603,17 +609,22 @@ class TestConfigLoggerExtended:with patch.object(module, "logger") as mock_logger:config_logger("test_app", debug=True)- # Check that all three add calls used diagnose=True- for add_call in mock_logger.add.call_args_list:- assert add_call[1].get("diagnose") is True+ by_sink = {+ c.args[0]: c.kwargs.get("diagnose")+ for c in mock_logger.add.call_args_list+ }+ assert by_sink[sys.stderr] is True+ assert by_sink[module.database_sink] is False+ assert by_sink[module.frontend_progress_sink] is Falsedef test_diagnose_propagates_to_file_sink(self):- """File sink must receive the same diagnose flag as the other sinks.+ """File sink follows the diagnose opt-in like the local stderr sink.Regression guard: with LDR_ENABLE_FILE_LOGGING=true the file sink is the- 4th logger.add() call. If that branch ever loses the diagnose argument,- the file (the most persistent and credential-risky sink) would silently- leak frame locals despite the opt-in being off.+ 4th logger.add() call. When the opt-in is off it must be diagnose=False;+ when on it tracks the opt-in (it is a local operator-enabled sink). The+ persisted DB sink and the browser-facing frontend sink are forced+ diagnose=False in both cases (#4182)."""from local_deep_research.utilities.log_utils import config_loggerimport local_deep_research.utilities.log_utils as module@@ -637,8 +648,23 @@ class TestConfigLoggerExtended:config_logger("test_app", debug=True)assert mock_logger.add.call_count == 4- for add_call in mock_logger.add.call_args_list:- assert add_call[1].get("diagnose") is True+ import sys+ from pathlib import Path++ by_sink = {+ c.args[0]: c.kwargs.get("diagnose")+ for c in mock_logger.add.call_args_list+ }+ # stderr and the file sink honor the opt-in; the persisted+ # DB sink and browser-facing frontend sink never do.+ # Keyed by sink identity (not call order) for robustness.+ assert by_sink[sys.stderr] is True+ assert by_sink[module.database_sink] is False+ assert by_sink[module.frontend_progress_sink] is False+ file_diagnose = [+ d for s, d in by_sink.items() if isinstance(s, Path)+ ]+ assert file_diagnose == [True]def test_diagnose_off_when_debug_without_optin(self):"""debug=True alone must NOT enable diagnose (localvar leak, issue #4185).@@ -697,9 +723,17 @@ class TestConfigLoggerExtended:with patch.object(module, "logger") as mock_logger:config_logger("test_app", debug=True)- assert mock_logger.add.call_args_list- for add_call in mock_logger.add.call_args_list:- assert add_call[1].get("diagnose") is True+ import sys++ by_sink = {+ c.args[0]: c.kwargs.get("diagnose")+ for c in mock_logger.add.call_args_list+ }+ # Whitespace-padded opt-in still enables diagnose on the+ # local stderr sink; DB + frontend stay forced off (#4182).+ assert by_sink[sys.stderr] is True+ assert by_sink[module.database_sink] is False+ assert by_sink[module.frontend_progress_sink] is Falsedef test_creates_milestone_level(self):"""Should create MILESTONE log level with level no=26."""