~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
refactor(security): post-#4976 polish - shared SECURE_LOGGING_DIRS source, EXTENDING.md scope, vacuous-pass guards (#5000)
LearningCircuitda4f1226767a
diff --git a/.pre-commit-hooks/_hook_common.py b/.pre-commit-hooks/_hook_common.pynew file mode 100644index 0000000000..37eecaee20--- /dev/null+++ b/.pre-commit-hooks/_hook_common.py@@ -0,0 +1,24 @@+"""Constants shared across pre-commit hooks in this directory.++Hooks in ``.pre-commit-hooks/`` are standalone scripts — they don't form+a package — so constants they must agree on can drift silently when+duplicated across files. This module is the single source of truth for+such constants; hooks load it with the same ``sys.path`` + ``from+_hook_common import ...`` idiom that the ``_commit_analysis`` consumers+in this directory use (see e.g. ``recommend-docs.py``).++Add sparingly: when a new constant cluster emerges, prefer splitting it+into a domain-named module over growing this file into a catch-all.+"""++# Directories where ``logger.exception()`` routes through the diagnose-gated+# ``security.secure_logging`` wrapper: the message is production-visible at+# ERROR level (only the traceback is gated), so interpolating the raw+# exception variable leaks error details there just like any other level+# (#4183). Outside these directories, ordinary loguru exception output+# follows the configured sinks; this tuple does not make it dev-only.+SECURE_LOGGING_DIRS = (+ "src/local_deep_research/llm/providers/",+ "src/local_deep_research/embeddings/providers/",+ "src/local_deep_research/web_search_engines/",+)diff --git a/.pre-commit-hooks/check-deprecated-db.py b/.pre-commit-hooks/check-deprecated-db.pyindex 33c55b77ec..fbfa4baa48 100755--- a/.pre-commit-hooks/check-deprecated-db.py+++ b/.pre-commit-hooks/check-deprecated-db.py@@ -8,9 +8,6 @@ import sysimport reimport os-# Set environment variable for pre-commit hooks to allow unencrypted databases-os.environ["LDR_ALLOW_UNENCRYPTED"] = "true"-def check_file(filepath):"""Check a single file for deprecated database usage."""@@ -88,6 +85,12 @@ def check_file(filepath):def main():"""Main function to check all provided files."""+ # Allow unencrypted databases when the hook runs standalone (dev+ # machines may still have a legacy unencrypted DB). Kept out of module+ # scope so importing this module — as tests/ does — cannot mutate the+ # process environment.+ os.environ["LDR_ALLOW_UNENCRYPTED"] = "true"+if len(sys.argv) < 2:print("No files to check")return 0diff --git a/.pre-commit-hooks/check-ldr-db.py b/.pre-commit-hooks/check-ldr-db.pyindex 4385f16341..038adcae7b 100755--- a/.pre-commit-hooks/check-ldr-db.py+++ b/.pre-commit-hooks/check-ldr-db.py@@ -9,9 +9,6 @@ import reimport osfrom pathlib import Path-# Set environment variable for pre-commit hooks to allow unencrypted databases-os.environ["LDR_ALLOW_UNENCRYPTED"] = "true"-def check_file_for_ldr_db(file_path):"""Check if a file contains references to ldr.db."""@@ -44,6 +41,12 @@ def check_file_for_ldr_db(file_path):def main():"""Main function to check all Python files for ldr.db usage."""+ # Allow unencrypted databases when the hook runs standalone (dev+ # machines may still have a legacy unencrypted DB). Kept out of module+ # scope so importing this module — as tests/ does — cannot mutate the+ # process environment.+ os.environ["LDR_ALLOW_UNENCRYPTED"] = "true"+# Get all Python files from command line argumentsfiles_to_check = sys.argv[1:] if len(sys.argv) > 1 else []diff --git a/.pre-commit-hooks/check-research-id-type.py b/.pre-commit-hooks/check-research-id-type.pyindex 2e0b84daf6..843d3a8b6d 100755--- a/.pre-commit-hooks/check-research-id-type.py+++ b/.pre-commit-hooks/check-research-id-type.py@@ -9,9 +9,6 @@ import reimport osfrom pathlib import Path-# Set environment variable for pre-commit hooks to allow unencrypted databases-os.environ["LDR_ALLOW_UNENCRYPTED"] = "true"-_REPO_ROOT = Path(__file__).resolve().parent.parent# Patterns to check for, as (id, regex, message). The id is what a file may be@@ -103,6 +100,12 @@ def check_file(filepath):def main():"""Main entry point."""+ # Allow unencrypted databases when the hook runs standalone (dev+ # machines may still have a legacy unencrypted DB). Kept out of module+ # scope so importing this module — as tests/ does — cannot mutate the+ # process environment.+ os.environ["LDR_ALLOW_UNENCRYPTED"] = "true"+# Get files to check from command line argumentsfiles_to_check = sys.argv[1:]diff --git a/.pre-commit-hooks/check-sensitive-logging.py b/.pre-commit-hooks/check-sensitive-logging.pyindex 5c12112ec2..a194d73557 100755--- a/.pre-commit-hooks/check-sensitive-logging.py+++ b/.pre-commit-hooks/check-sensitive-logging.py@@ -14,6 +14,13 @@ import sysfrom pathlib import Pathfrom typing import List, Optional+# Hooks in this directory are standalone scripts, not a package, so we+# add the directory to sys.path and import shared constants. Same idiom+# the _commit_analysis consumers use; see _hook_common.py for the rationale.+sys.path.insert(0, str(Path(__file__).resolve().parent))++from _hook_common import SECURE_LOGGING_DIRS # noqa: E402+SENSITIVE_VARS = {"password",@@ -118,17 +125,6 @@ ALLOWED_LOGGING = []-# Directories where ``logger.exception()`` routes through the diagnose-gated-# ``security.secure_logging`` wrapper: the message is production-visible at-# ERROR level (only the traceback is gated), so interpolating the raw-# exception variable leaks error details there just like any other level-# (#4183). Elsewhere, plain loguru ``.exception()`` output remains dev-only.-SECURE_LOGGING_DIRS = (- "src/local_deep_research/llm/providers/",- "src/local_deep_research/embeddings/providers/",- "src/local_deep_research/web_search_engines/",-)-# The only sanctioned logger import inside SECURE_LOGGING_DIRS. Matched# exactly (module + level), never by suffix, so lookalike modules such as# ``notsecurity.secure_logging`` cannot spoof it.@@ -1209,8 +1205,9 @@ class SensitiveLoggingChecker(ast.NodeVisitor):"""Flag exc_info=True on warning/error/critical level logs.exc_info dumps full tracebacks which are visible in production.- Use logger.exception() for dev-only traceback logging, or- logger.debug(..., exc_info=True) which is off in production.+ Use logger.exception() instead: its traceback still reaches the+ configured sinks (see _hook_common.py) — it is not dev-only — or+ use logger.debug(..., exc_info=True), which is off in production."""level = self._get_log_level(node)if level not in {"warning", "error", "critical"}:@@ -1231,16 +1228,24 @@ class SensitiveLoggingChecker(ast.NodeVisitor):"""Flag logging calls that interpolate the exception variable.Patterns like logger.warning(f"...{e}") or logger.warning("...%s", e)- leak error details. Use logger.exception() instead, which is- configured for dev-only output.+ leak error details. Prefer logger.exception() — its output still+ reaches the configured sinks, it is not dev-only (see+ _hook_common.py); see below for why plain logger.exception() calls+ are exempted here outside the secure-logging dirs."""level = self._get_log_level(node)# logger.debug() is fine — not visible in productionif level == "debug":return- # Plain loguru logger.exception() output is dev-only — but in the- # secure_logging dirs the wrapper makes the message production-visible- # at ERROR, so the exception variable must not be interpolated there.+ # Outside the secure_logging dirs, plain logger.exception() always+ # attaches the active exception to the record (loguru's default,+ # not dev-only — see _hook_common.py); the traceback already+ # carries the exception's text on every configured sink, so+ # interpolating the same variable into the message adds nothing+ # and this check skips it below. Inside the secure_logging dirs,+ # the wrapper attaches that exception only in diagnose mode+ # (security/secure_logging.py) — by default nothing is attached,+ # so the message is the one channel that can leak it there.if level == "exception" and not self.in_secure_dir:return@@ -1547,7 +1552,9 @@ def main():"self._scrub_error()) and log the safe_msg")print(- " - Elsewhere: use logger.exception() (traceback is dev-only) or remove the variable"+ " - Elsewhere: use logger.exception() (its traceback still "+ "reaches the configured sinks — not dev-only) or remove the "+ "variable")print(" - Using sanitized versions of sensitive data structures before logging"diff --git a/.pre-commit-hooks/custom-checks.py b/.pre-commit-hooks/custom-checks.pyindex 52cd6c2c80..f73133c0d2 100755--- a/.pre-commit-hooks/custom-checks.py+++ b/.pre-commit-hooks/custom-checks.py@@ -16,18 +16,13 @@ import osfrom pathlib import Pathfrom typing import List, Tuple-# Set environment variable for pre-commit hooks to allow unencrypted databases-os.environ["LDR_ALLOW_UNENCRYPTED"] = "true"-# Dirs where logger.exception() routes through the diagnose-gated-# security.secure_logging wrapper (#4183): the message is production-visible-# at ERROR there, so the "automatically includes exception details" advice-# is wrong — keep in sync with check-sensitive-logging.py.-SECURE_LOGGING_DIRS = (- "src/local_deep_research/llm/providers/",- "src/local_deep_research/embeddings/providers/",- "src/local_deep_research/web_search_engines/",-)+# Hooks in this directory are standalone scripts, not a package, so we+# add the directory to sys.path and import shared constants. Same idiom+# the _commit_analysis consumers use; see _hook_common.py for the rationale.+sys.path.insert(0, str(Path(__file__).resolve().parent))++from _hook_common import SECURE_LOGGING_DIRS # noqa: E402class CustomCodeChecker(ast.NodeVisitor):@@ -577,6 +572,12 @@ def check_file(filename: str) -> bool:def main():"""Main function to check all staged Python files."""+ # Allow unencrypted databases when the hook runs standalone (dev+ # machines may still have a legacy unencrypted DB). Kept out of module+ # scope so importing this module — as tests/ does — cannot mutate the+ # process environment.+ os.environ["LDR_ALLOW_UNENCRYPTED"] = "true"+if len(sys.argv) < 2:print("Usage: custom-checks.py <file1> <file2> ...")sys.exit(1)diff --git a/docs/developing/EXTENDING.md b/docs/developing/EXTENDING.mdindex 0ec40cf875..f1bd7e7619 100644--- a/docs/developing/EXTENDING.md+++ b/docs/developing/EXTENDING.md@@ -247,6 +247,7 @@ Create a new file in `src/local_deep_research/advanced_search_system/strategies/from typing import Dict, List, Optionalfrom loguru import logger+from ...security.log_sanitizer import sanitize_error_for_clientfrom .base_strategy import BaseSearchStrategy@@ -345,14 +346,14 @@ class MyCustomStrategy(BaseSearchStrategy):}except Exception as e:- logger.error(f"Strategy error: {e}")+ logger.exception(f"Strategy error: {sanitize_error_for_client(str(e))}")return {"findings": findings,"iterations": 0,"questions": self.questions_by_iteration,"formatted_findings": "","current_knowledge": current_knowledge,- "error": str(e),+ "error": sanitize_error_for_client(str(e)),}def _generate_questions(self, query: str, knowledge: Dict) -> List[str]:@@ -431,6 +432,27 @@ def create_strategy(strategy_name: str, ...) -> BaseSearchStrategy:5. **Handle errors gracefully** - return partial results with error message+6. **Scrub untrusted input before logging.** Strategies live outside the+ secure-logging dirs, so plain `from loguru import logger` is correct+ here (engines and providers must use the wrapper; see "Search Engine+ Best Practices" #4). In `except` blocks, log with `logger.exception()`+ — this repo's pre-commit hooks reject `logger.error()` wrapping the+ exception variable — and scrub the exception with+ `sanitize_error_for_client` (from `security.log_sanitizer`) before+ interpolating or returning it. It applies both required passes in one+ call — credential-shape redaction, then `sanitize_for_log`'s+ control-character strip and length cap, redacting first so truncation+ cannot split and hide a secret. The web and MCP logging entry points+ also install control-character filtering at their sinks, but that does+ not redact credentials or protect returned error strings. Keep the+ explicit boundary sanitizer so both outputs receive the same treatment,+ including when the strategy runs through another entry point.+ `sanitize_error_for_client` only redacts credential shapes — it does+ not remove server filesystem paths, SQL text, provider endpoints, or+ dependency internals — so unless the exception text is already+ known-safe, return a fixed client-facing message and keep the+ sanitized text for the log only.+---## Using LangChain Retrieversdiff --git a/tests/hooks/test_hook_common.py b/tests/hooks/test_hook_common.pynew file mode 100644index 0000000000..e38121a07a--- /dev/null+++ b/tests/hooks/test_hook_common.py@@ -0,0 +1,134 @@+"""+Tests for the _hook_common pre-commit-hook shared-constants module.++Covers the invariants introduced when SECURE_LOGGING_DIRS was extracted+from check-sensitive-logging.py and custom-checks.py into _hook_common.py:++ * the constant is well-formed and its paths are substring-match-safe+ * both hook modules bind the SAME object (identity, not just equality),+ i.e. neither hook re-introduces a local literal that shadows the+ shared one+ * importing custom-checks, check-deprecated-db, check-ldr-db or+ check-research-id-type does not mutate the importer's environment+ (LDR_ALLOW_UNENCRYPTED is set inside main(), not at module scope)++The co-located-test pattern mirrors tests/hooks/test_commit_analysis.py+for the other shared module in .pre-commit-hooks/.+"""++import os+import subprocess+import sys+from importlib import import_module+from pathlib import Path++import pytest++HOOKS_DIR = Path(__file__).parent.parent.parent / ".pre-commit-hooks"+sys.path.insert(0, str(HOOKS_DIR))++from _hook_common import SECURE_LOGGING_DIRS # noqa: E402++# Hook filenames are hyphenated, so they must be imported by string via+# import_module (the same idiom tests/security/test_sensitive_logging_hook.py+# uses). _hook_common itself is a valid identifier and is imported directly+# above, which is the contract this module exists to verify.+_check_sensitive_logging = import_module("check-sensitive-logging")+_custom_checks = import_module("custom-checks")+++class TestSecureLoggingDirsShape:+ """SECURE_LOGGING_DIRS is well-formed for the substring match the hooks do."""++ def test_is_non_empty_tuple_of_str(self):+ assert isinstance(SECURE_LOGGING_DIRS, tuple)+ assert len(SECURE_LOGGING_DIRS) >= 1+ assert all(isinstance(d, str) for d in SECURE_LOGGING_DIRS)++ def test_entries_end_with_slash(self):+ # The hooks match via ``d in filename``; a missing trailing slash+ # would let "src/.../web_search_engines" match "web_search_engines_v2/".+ bad = [d for d in SECURE_LOGGING_DIRS if not d.endswith("/")]+ assert not bad, (+ "SECURE_LOGGING_DIRS entries must end with '/': " + ", ".join(bad)+ )+++class TestHooksBindToHookCommon:+ """Both hook modules bind the shared constant by identity.++ Equality alone is not enough: if either hook re-introduced a local+ ``SECURE_LOGGING_DIRS = (...)`` literal that happened to match, the+ constant could drift again silently. Identity (``is``) proves the hook+ references the single source of truth in _hook_common rather than a+ private copy.+ """++ def test_check_sensitive_logging_binds_to_hook_common(self):+ assert (+ _check_sensitive_logging.SECURE_LOGGING_DIRS is SECURE_LOGGING_DIRS+ )++ def test_custom_checks_binds_to_hook_common(self):+ assert _custom_checks.SECURE_LOGGING_DIRS is SECURE_LOGGING_DIRS++ def test_both_hooks_bind_the_same_object(self):+ assert (+ _check_sensitive_logging.SECURE_LOGGING_DIRS+ is _custom_checks.SECURE_LOGGING_DIRS+ )+++class TestHookImportPurity:+ """Importing these hooks must not mutate the process environment.++ ``LDR_ALLOW_UNENCRYPTED=true`` lets a hook tolerate legacy+ unencrypted dev DBs while it *runs*; setting it at module scope+ instead made every importer — including this test suite — silently+ mutate its own process env. Sibling hook tests still import modules+ that set it at module scope, so the shared pytest process env cannot+ be trusted for this assertion; it is checked in a pristine+ subprocess instead.++ Covers every hook that sets ``LDR_ALLOW_UNENCRYPTED``: custom-checks+ was the first to move the assignment into ``main()``;+ check-deprecated-db, check-ldr-db and check-research-id-type followed+ the same move.+ """++ @pytest.mark.parametrize(+ "hook_module_name",+ [+ "custom-checks",+ "check-deprecated-db",+ "check-ldr-db",+ "check-research-id-type",+ ],+ )+ def test_import_does_not_set_ldr_allow_unencrypted(self, hook_module_name):+ """A clean child that imports the hook must not gain the variable.++ Fails if the assignment is reverted to (or left at) module scope:+ the child would then print "true" instead of "None".+ """+ child_code = (+ "import os, sys\n"+ "from importlib import import_module\n"+ "sys.path.insert(0, sys.argv[1])\n"+ f"import_module({hook_module_name!r})\n"+ "print(os.environ.get('LDR_ALLOW_UNENCRYPTED'))\n"+ )+ env = {+ k: v for k, v in os.environ.items() if k != "LDR_ALLOW_UNENCRYPTED"+ }+ result = subprocess.run(+ [sys.executable, "-c", child_code, str(HOOKS_DIR)],+ capture_output=True,+ text=True,+ env=env,+ )+ assert result.returncode == 0, result.stderr+ assert result.stdout.strip() == "None", (+ f"importing {hook_module_name} must not set "+ "LDR_ALLOW_UNENCRYPTED; keep the assignment inside main()"+ )diff --git a/tests/security/test_sensitive_logging_hook.py b/tests/security/test_sensitive_logging_hook.pyindex 2183eb3c70..e7f2cece32 100644--- a/tests/security/test_sensitive_logging_hook.py+++ b/tests/security/test_sensitive_logging_hook.py@@ -1550,12 +1550,72 @@ class TestSourceTreeInvariant:)def test_secure_logging_dirs_constant_in_sync_with_custom_checks(self):- """custom-checks.py duplicates SECURE_LOGGING_DIRS for its own- exception-message advice; the two must stay identical so the- wrapper-gated exception rule covers the same files in both hooks."""+ """Both hooks consume SECURE_LOGGING_DIRS via _hook_common.py, so+ they must stay identical; this catches either hook re-introducing+ a local literal with drifted content (the identity checks in+ tests/hooks/test_hook_common.py catch same-content copies)."""custom_checks = import_module("custom-checks")assert custom_checks.SECURE_LOGGING_DIRS == SECURE_LOGGING_DIRS, ("SECURE_LOGGING_DIRS drifted between hooks:\n"f" custom-checks.py: {custom_checks.SECURE_LOGGING_DIRS}\n"f" check-sensitive-logging: {SECURE_LOGGING_DIRS}")++ def test_secure_logging_dirs_nonempty_and_dirs_exist(self):+ """Vacuous-pass guard for the tree scans above.++ _secure_dir_files() iterates SECURE_LOGGING_DIRS; if the constant+ were ever emptied or a path typo'd, the scans would yield nothing+ (or skip the bad entry) and ``assert not failures`` would pass+ over an empty list. Pinning non-emptiness and on-disk existence+ here makes that case fail loudly. Structural + identity checks+ live in tests/hooks/test_hook_common.py.+ """+ assert SECURE_LOGGING_DIRS, "SECURE_LOGGING_DIRS must not be empty"+ repo_root = self._src_root().parent+ missing = [+ d for d in SECURE_LOGGING_DIRS if not (repo_root / d).is_dir()+ ]+ assert not missing, (+ "SECURE_LOGGING_DIRS lists dirs absent from the tree: "+ + ", ".join(missing)+ )+ # Existence is necessary but not sufficient: a dir that survived but+ # was emptied of *.py would still let the scans pass vacuously over+ # its (empty) contribution. Pin that every listed dir actually feeds+ # the generator the scans consume, so a dir going dark fails here.+ empty = [+ d+ for d in SECURE_LOGGING_DIRS+ if not any((repo_root / d).rglob("*.py"))+ ]+ assert not empty, (+ "SECURE_LOGGING_DIRS lists dirs with no .py files to scan: "+ + ", ".join(empty)+ )+ # And the aggregate generator the tree scans iterate must be non-empty.+ assert any(self._secure_dir_files()), (+ "_secure_dir_files() yielded nothing; the tree-scan invariants "+ "above would pass vacuously"+ )++ def test_secure_logging_dirs_membership_pinned(self):+ """Partial-loss guard: deleting ANY member must fail loudly.++ The checks above cannot catch a tuple that shrank but still+ lists valid, non-empty dirs — every remaining scan would pass+ while the removed security domain silently lost coverage.+ Pin the exact expected membership so any deletion (or other+ membership change) must consciously update this list too.+ """+ expected = (+ "src/local_deep_research/llm/providers/",+ "src/local_deep_research/embeddings/providers/",+ "src/local_deep_research/web_search_engines/",+ )+ assert SECURE_LOGGING_DIRS == expected, (+ "SECURE_LOGGING_DIRS membership changed; if intentional, update "+ "the pinned expected list with it:\n"+ f" actual: {SECURE_LOGGING_DIRS}\n"+ f" expected: {expected}"+ )