Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

refactor(security): post-#4976 polish - shared SECURE_LOGGING_DIRS source, EXTENDING.md scope, vacuous-pass guards (#5000)

LearningCircuitda4f1226767a
diff --git a/.pre-commit-hooks/_hook_common.py b/.pre-commit-hooks/_hook_common.py
new file mode 100644
index 0000000000..37eecaee20
--- /dev/null
+++ b/.pre-commit-hooks/_hook_common.py
@@ -0,0 +1,24 @@
+"""Constants shared across pre-commit hooks in this directory.
+
+Hooks in ``.pre-commit-hooks/`` are standalone scripts — they don't form
+a package — so constants they must agree on can drift silently when
+duplicated across files. This module is the single source of truth for
+such constants; hooks load it with the same ``sys.path`` + ``from
+_hook_common import ...`` idiom that the ``_commit_analysis`` consumers
+in this directory use (see e.g. ``recommend-docs.py``).
+
+Add sparingly: when a new constant cluster emerges, prefer splitting it
+into a domain-named module over growing this file into a catch-all.
+"""
+
+# Directories where ``logger.exception()`` routes through the diagnose-gated
+# ``security.secure_logging`` wrapper: the message is production-visible at
+# ERROR level (only the traceback is gated), so interpolating the raw
+# exception variable leaks error details there just like any other level
+# (#4183). Outside these directories, ordinary loguru exception output
+# follows the configured sinks; this tuple does not make it dev-only.
+SECURE_LOGGING_DIRS = (
+ "src/local_deep_research/llm/providers/",
+ "src/local_deep_research/embeddings/providers/",
+ "src/local_deep_research/web_search_engines/",
+)
diff --git a/.pre-commit-hooks/check-deprecated-db.py b/.pre-commit-hooks/check-deprecated-db.py
index 33c55b77ec..fbfa4baa48 100755
--- a/.pre-commit-hooks/check-deprecated-db.py
+++ b/.pre-commit-hooks/check-deprecated-db.py
@@ -8,9 +8,6 @@ import sys
import re
import os
-# Set environment variable for pre-commit hooks to allow unencrypted databases
-os.environ["LDR_ALLOW_UNENCRYPTED"] = "true"
-
def check_file(filepath):
"""Check a single file for deprecated database usage."""
@@ -88,6 +85,12 @@ def check_file(filepath):
def main():
"""Main function to check all provided files."""
+ # Allow unencrypted databases when the hook runs standalone (dev
+ # machines may still have a legacy unencrypted DB). Kept out of module
+ # scope so importing this module — as tests/ does — cannot mutate the
+ # process environment.
+ os.environ["LDR_ALLOW_UNENCRYPTED"] = "true"
+
if len(sys.argv) < 2:
print("No files to check")
return 0
diff --git a/.pre-commit-hooks/check-ldr-db.py b/.pre-commit-hooks/check-ldr-db.py
index 4385f16341..038adcae7b 100755
--- a/.pre-commit-hooks/check-ldr-db.py
+++ b/.pre-commit-hooks/check-ldr-db.py
@@ -9,9 +9,6 @@ import re
import os
from pathlib import Path
-# Set environment variable for pre-commit hooks to allow unencrypted databases
-os.environ["LDR_ALLOW_UNENCRYPTED"] = "true"
-
def check_file_for_ldr_db(file_path):
"""Check if a file contains references to ldr.db."""
@@ -44,6 +41,12 @@ def check_file_for_ldr_db(file_path):
def main():
"""Main function to check all Python files for ldr.db usage."""
+ # Allow unencrypted databases when the hook runs standalone (dev
+ # machines may still have a legacy unencrypted DB). Kept out of module
+ # scope so importing this module — as tests/ does — cannot mutate the
+ # process environment.
+ os.environ["LDR_ALLOW_UNENCRYPTED"] = "true"
+
# Get all Python files from command line arguments
files_to_check = sys.argv[1:] if len(sys.argv) > 1 else []
diff --git a/.pre-commit-hooks/check-research-id-type.py b/.pre-commit-hooks/check-research-id-type.py
index 2e0b84daf6..843d3a8b6d 100755
--- a/.pre-commit-hooks/check-research-id-type.py
+++ b/.pre-commit-hooks/check-research-id-type.py
@@ -9,9 +9,6 @@ import re
import os
from pathlib import Path
-# Set environment variable for pre-commit hooks to allow unencrypted databases
-os.environ["LDR_ALLOW_UNENCRYPTED"] = "true"
-
_REPO_ROOT = Path(__file__).resolve().parent.parent
# Patterns to check for, as (id, regex, message). The id is what a file may be
@@ -103,6 +100,12 @@ def check_file(filepath):
def main():
"""Main entry point."""
+ # Allow unencrypted databases when the hook runs standalone (dev
+ # machines may still have a legacy unencrypted DB). Kept out of module
+ # scope so importing this module — as tests/ does — cannot mutate the
+ # process environment.
+ os.environ["LDR_ALLOW_UNENCRYPTED"] = "true"
+
# Get files to check from command line arguments
files_to_check = sys.argv[1:]
diff --git a/.pre-commit-hooks/check-sensitive-logging.py b/.pre-commit-hooks/check-sensitive-logging.py
index 5c12112ec2..a194d73557 100755
--- a/.pre-commit-hooks/check-sensitive-logging.py
+++ b/.pre-commit-hooks/check-sensitive-logging.py
@@ -14,6 +14,13 @@ import sys
from pathlib import Path
from typing import List, Optional
+# Hooks in this directory are standalone scripts, not a package, so we
+# add the directory to sys.path and import shared constants. Same idiom
+# the _commit_analysis consumers use; see _hook_common.py for the rationale.
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+
+from _hook_common import SECURE_LOGGING_DIRS # noqa: E402
+
SENSITIVE_VARS = {
"password",
@@ -118,17 +125,6 @@ ALLOWED_LOGGING = [
]
-# Directories where ``logger.exception()`` routes through the diagnose-gated
-# ``security.secure_logging`` wrapper: the message is production-visible at
-# ERROR level (only the traceback is gated), so interpolating the raw
-# exception variable leaks error details there just like any other level
-# (#4183). Elsewhere, plain loguru ``.exception()`` output remains dev-only.
-SECURE_LOGGING_DIRS = (
- "src/local_deep_research/llm/providers/",
- "src/local_deep_research/embeddings/providers/",
- "src/local_deep_research/web_search_engines/",
-)
-
# The only sanctioned logger import inside SECURE_LOGGING_DIRS. Matched
# exactly (module + level), never by suffix, so lookalike modules such as
# ``notsecurity.secure_logging`` cannot spoof it.
@@ -1209,8 +1205,9 @@ class SensitiveLoggingChecker(ast.NodeVisitor):
"""Flag exc_info=True on warning/error/critical level logs.
exc_info dumps full tracebacks which are visible in production.
- Use logger.exception() for dev-only traceback logging, or
- logger.debug(..., exc_info=True) which is off in production.
+ Use logger.exception() instead: its traceback still reaches the
+ configured sinks (see _hook_common.py) — it is not dev-only — or
+ use logger.debug(..., exc_info=True), which is off in production.
"""
level = self._get_log_level(node)
if level not in {"warning", "error", "critical"}:
@@ -1231,16 +1228,24 @@ class SensitiveLoggingChecker(ast.NodeVisitor):
"""Flag logging calls that interpolate the exception variable.
Patterns like logger.warning(f"...{e}") or logger.warning("...%s", e)
- leak error details. Use logger.exception() instead, which is
- configured for dev-only output.
+ leak error details. Prefer logger.exception() — its output still
+ reaches the configured sinks, it is not dev-only (see
+ _hook_common.py); see below for why plain logger.exception() calls
+ are exempted here outside the secure-logging dirs.
"""
level = self._get_log_level(node)
# logger.debug() is fine — not visible in production
if level == "debug":
return
- # Plain loguru logger.exception() output is dev-only — but in the
- # secure_logging dirs the wrapper makes the message production-visible
- # at ERROR, so the exception variable must not be interpolated there.
+ # Outside the secure_logging dirs, plain logger.exception() always
+ # attaches the active exception to the record (loguru's default,
+ # not dev-only — see _hook_common.py); the traceback already
+ # carries the exception's text on every configured sink, so
+ # interpolating the same variable into the message adds nothing
+ # and this check skips it below. Inside the secure_logging dirs,
+ # the wrapper attaches that exception only in diagnose mode
+ # (security/secure_logging.py) — by default nothing is attached,
+ # so the message is the one channel that can leak it there.
if level == "exception" and not self.in_secure_dir:
return
@@ -1547,7 +1552,9 @@ def main():
"self._scrub_error()) and log the safe_msg"
)
print(
- " - Elsewhere: use logger.exception() (traceback is dev-only) or remove the variable"
+ " - Elsewhere: use logger.exception() (its traceback still "
+ "reaches the configured sinks — not dev-only) or remove the "
+ "variable"
)
print(
" - Using sanitized versions of sensitive data structures before logging"
diff --git a/.pre-commit-hooks/custom-checks.py b/.pre-commit-hooks/custom-checks.py
index 52cd6c2c80..f73133c0d2 100755
--- a/.pre-commit-hooks/custom-checks.py
+++ b/.pre-commit-hooks/custom-checks.py
@@ -16,18 +16,13 @@ import os
from pathlib import Path
from typing import List, Tuple
-# Set environment variable for pre-commit hooks to allow unencrypted databases
-os.environ["LDR_ALLOW_UNENCRYPTED"] = "true"
-# Dirs where logger.exception() routes through the diagnose-gated
-# security.secure_logging wrapper (#4183): the message is production-visible
-# at ERROR there, so the "automatically includes exception details" advice
-# is wrong — keep in sync with check-sensitive-logging.py.
-SECURE_LOGGING_DIRS = (
- "src/local_deep_research/llm/providers/",
- "src/local_deep_research/embeddings/providers/",
- "src/local_deep_research/web_search_engines/",
-)
+# Hooks in this directory are standalone scripts, not a package, so we
+# add the directory to sys.path and import shared constants. Same idiom
+# the _commit_analysis consumers use; see _hook_common.py for the rationale.
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+
+from _hook_common import SECURE_LOGGING_DIRS # noqa: E402
class CustomCodeChecker(ast.NodeVisitor):
@@ -577,6 +572,12 @@ def check_file(filename: str) -> bool:
def main():
"""Main function to check all staged Python files."""
+ # Allow unencrypted databases when the hook runs standalone (dev
+ # machines may still have a legacy unencrypted DB). Kept out of module
+ # scope so importing this module — as tests/ does — cannot mutate the
+ # process environment.
+ os.environ["LDR_ALLOW_UNENCRYPTED"] = "true"
+
if len(sys.argv) < 2:
print("Usage: custom-checks.py <file1> <file2> ...")
sys.exit(1)
diff --git a/docs/developing/EXTENDING.md b/docs/developing/EXTENDING.md
index 0ec40cf875..f1bd7e7619 100644
--- a/docs/developing/EXTENDING.md
+++ b/docs/developing/EXTENDING.md
@@ -247,6 +247,7 @@ Create a new file in `src/local_deep_research/advanced_search_system/strategies/
from typing import Dict, List, Optional
from loguru import logger
+from ...security.log_sanitizer import sanitize_error_for_client
from .base_strategy import BaseSearchStrategy
@@ -345,14 +346,14 @@ class MyCustomStrategy(BaseSearchStrategy):
}
except Exception as e:
- logger.error(f"Strategy error: {e}")
+ logger.exception(f"Strategy error: {sanitize_error_for_client(str(e))}")
return {
"findings": findings,
"iterations": 0,
"questions": self.questions_by_iteration,
"formatted_findings": "",
"current_knowledge": current_knowledge,
- "error": str(e),
+ "error": sanitize_error_for_client(str(e)),
}
def _generate_questions(self, query: str, knowledge: Dict) -> List[str]:
@@ -431,6 +432,27 @@ def create_strategy(strategy_name: str, ...) -> BaseSearchStrategy:
5. **Handle errors gracefully** - return partial results with error message
+6. **Scrub untrusted input before logging.** Strategies live outside the
+ secure-logging dirs, so plain `from loguru import logger` is correct
+ here (engines and providers must use the wrapper; see "Search Engine
+ Best Practices" #4). In `except` blocks, log with `logger.exception()`
+ — this repo's pre-commit hooks reject `logger.error()` wrapping the
+ exception variable — and scrub the exception with
+ `sanitize_error_for_client` (from `security.log_sanitizer`) before
+ interpolating or returning it. It applies both required passes in one
+ call — credential-shape redaction, then `sanitize_for_log`'s
+ control-character strip and length cap, redacting first so truncation
+ cannot split and hide a secret. The web and MCP logging entry points
+ also install control-character filtering at their sinks, but that does
+ not redact credentials or protect returned error strings. Keep the
+ explicit boundary sanitizer so both outputs receive the same treatment,
+ including when the strategy runs through another entry point.
+ `sanitize_error_for_client` only redacts credential shapes — it does
+ not remove server filesystem paths, SQL text, provider endpoints, or
+ dependency internals — so unless the exception text is already
+ known-safe, return a fixed client-facing message and keep the
+ sanitized text for the log only.
+
---
## Using LangChain Retrievers
diff --git a/tests/hooks/test_hook_common.py b/tests/hooks/test_hook_common.py
new file mode 100644
index 0000000000..e38121a07a
--- /dev/null
+++ b/tests/hooks/test_hook_common.py
@@ -0,0 +1,134 @@
+"""
+Tests for the _hook_common pre-commit-hook shared-constants module.
+
+Covers the invariants introduced when SECURE_LOGGING_DIRS was extracted
+from check-sensitive-logging.py and custom-checks.py into _hook_common.py:
+
+ * the constant is well-formed and its paths are substring-match-safe
+ * both hook modules bind the SAME object (identity, not just equality),
+ i.e. neither hook re-introduces a local literal that shadows the
+ shared one
+ * importing custom-checks, check-deprecated-db, check-ldr-db or
+ check-research-id-type does not mutate the importer's environment
+ (LDR_ALLOW_UNENCRYPTED is set inside main(), not at module scope)
+
+The co-located-test pattern mirrors tests/hooks/test_commit_analysis.py
+for the other shared module in .pre-commit-hooks/.
+"""
+
+import os
+import subprocess
+import sys
+from importlib import import_module
+from pathlib import Path
+
+import pytest
+
+HOOKS_DIR = Path(__file__).parent.parent.parent / ".pre-commit-hooks"
+sys.path.insert(0, str(HOOKS_DIR))
+
+from _hook_common import SECURE_LOGGING_DIRS # noqa: E402
+
+# Hook filenames are hyphenated, so they must be imported by string via
+# import_module (the same idiom tests/security/test_sensitive_logging_hook.py
+# uses). _hook_common itself is a valid identifier and is imported directly
+# above, which is the contract this module exists to verify.
+_check_sensitive_logging = import_module("check-sensitive-logging")
+_custom_checks = import_module("custom-checks")
+
+
+class TestSecureLoggingDirsShape:
+ """SECURE_LOGGING_DIRS is well-formed for the substring match the hooks do."""
+
+ def test_is_non_empty_tuple_of_str(self):
+ assert isinstance(SECURE_LOGGING_DIRS, tuple)
+ assert len(SECURE_LOGGING_DIRS) >= 1
+ assert all(isinstance(d, str) for d in SECURE_LOGGING_DIRS)
+
+ def test_entries_end_with_slash(self):
+ # The hooks match via ``d in filename``; a missing trailing slash
+ # would let "src/.../web_search_engines" match "web_search_engines_v2/".
+ bad = [d for d in SECURE_LOGGING_DIRS if not d.endswith("/")]
+ assert not bad, (
+ "SECURE_LOGGING_DIRS entries must end with '/': " + ", ".join(bad)
+ )
+
+
+class TestHooksBindToHookCommon:
+ """Both hook modules bind the shared constant by identity.
+
+ Equality alone is not enough: if either hook re-introduced a local
+ ``SECURE_LOGGING_DIRS = (...)`` literal that happened to match, the
+ constant could drift again silently. Identity (``is``) proves the hook
+ references the single source of truth in _hook_common rather than a
+ private copy.
+ """
+
+ def test_check_sensitive_logging_binds_to_hook_common(self):
+ assert (
+ _check_sensitive_logging.SECURE_LOGGING_DIRS is SECURE_LOGGING_DIRS
+ )
+
+ def test_custom_checks_binds_to_hook_common(self):
+ assert _custom_checks.SECURE_LOGGING_DIRS is SECURE_LOGGING_DIRS
+
+ def test_both_hooks_bind_the_same_object(self):
+ assert (
+ _check_sensitive_logging.SECURE_LOGGING_DIRS
+ is _custom_checks.SECURE_LOGGING_DIRS
+ )
+
+
+class TestHookImportPurity:
+ """Importing these hooks must not mutate the process environment.
+
+ ``LDR_ALLOW_UNENCRYPTED=true`` lets a hook tolerate legacy
+ unencrypted dev DBs while it *runs*; setting it at module scope
+ instead made every importer — including this test suite — silently
+ mutate its own process env. Sibling hook tests still import modules
+ that set it at module scope, so the shared pytest process env cannot
+ be trusted for this assertion; it is checked in a pristine
+ subprocess instead.
+
+ Covers every hook that sets ``LDR_ALLOW_UNENCRYPTED``: custom-checks
+ was the first to move the assignment into ``main()``;
+ check-deprecated-db, check-ldr-db and check-research-id-type followed
+ the same move.
+ """
+
+ @pytest.mark.parametrize(
+ "hook_module_name",
+ [
+ "custom-checks",
+ "check-deprecated-db",
+ "check-ldr-db",
+ "check-research-id-type",
+ ],
+ )
+ def test_import_does_not_set_ldr_allow_unencrypted(self, hook_module_name):
+ """A clean child that imports the hook must not gain the variable.
+
+ Fails if the assignment is reverted to (or left at) module scope:
+ the child would then print "true" instead of "None".
+ """
+ child_code = (
+ "import os, sys\n"
+ "from importlib import import_module\n"
+ "sys.path.insert(0, sys.argv[1])\n"
+ f"import_module({hook_module_name!r})\n"
+ "print(os.environ.get('LDR_ALLOW_UNENCRYPTED'))\n"
+ )
+ env = {
+ k: v for k, v in os.environ.items() if k != "LDR_ALLOW_UNENCRYPTED"
+ }
+ result = subprocess.run(
+ [sys.executable, "-c", child_code, str(HOOKS_DIR)],
+ capture_output=True,
+ text=True,
+ env=env,
+ )
+ assert result.returncode == 0, result.stderr
+ assert result.stdout.strip() == "None", (
+ f"importing {hook_module_name} must not set "
+ "LDR_ALLOW_UNENCRYPTED; keep the assignment inside main()"
+ )
diff --git a/tests/security/test_sensitive_logging_hook.py b/tests/security/test_sensitive_logging_hook.py
index 2183eb3c70..e7f2cece32 100644
--- a/tests/security/test_sensitive_logging_hook.py
+++ b/tests/security/test_sensitive_logging_hook.py
@@ -1550,12 +1550,72 @@ class TestSourceTreeInvariant:
)
def test_secure_logging_dirs_constant_in_sync_with_custom_checks(self):
- """custom-checks.py duplicates SECURE_LOGGING_DIRS for its own
- exception-message advice; the two must stay identical so the
- wrapper-gated exception rule covers the same files in both hooks."""
+ """Both hooks consume SECURE_LOGGING_DIRS via _hook_common.py, so
+ they must stay identical; this catches either hook re-introducing
+ a local literal with drifted content (the identity checks in
+ tests/hooks/test_hook_common.py catch same-content copies)."""
custom_checks = import_module("custom-checks")
assert custom_checks.SECURE_LOGGING_DIRS == SECURE_LOGGING_DIRS, (
"SECURE_LOGGING_DIRS drifted between hooks:\n"
f" custom-checks.py: {custom_checks.SECURE_LOGGING_DIRS}\n"
f" check-sensitive-logging: {SECURE_LOGGING_DIRS}"
)
+
+ def test_secure_logging_dirs_nonempty_and_dirs_exist(self):
+ """Vacuous-pass guard for the tree scans above.
+
+ _secure_dir_files() iterates SECURE_LOGGING_DIRS; if the constant
+ were ever emptied or a path typo'd, the scans would yield nothing
+ (or skip the bad entry) and ``assert not failures`` would pass
+ over an empty list. Pinning non-emptiness and on-disk existence
+ here makes that case fail loudly. Structural + identity checks
+ live in tests/hooks/test_hook_common.py.
+ """
+ assert SECURE_LOGGING_DIRS, "SECURE_LOGGING_DIRS must not be empty"
+ repo_root = self._src_root().parent
+ missing = [
+ d for d in SECURE_LOGGING_DIRS if not (repo_root / d).is_dir()
+ ]
+ assert not missing, (
+ "SECURE_LOGGING_DIRS lists dirs absent from the tree: "
+ + ", ".join(missing)
+ )
+ # Existence is necessary but not sufficient: a dir that survived but
+ # was emptied of *.py would still let the scans pass vacuously over
+ # its (empty) contribution. Pin that every listed dir actually feeds
+ # the generator the scans consume, so a dir going dark fails here.
+ empty = [
+ d
+ for d in SECURE_LOGGING_DIRS
+ if not any((repo_root / d).rglob("*.py"))
+ ]
+ assert not empty, (
+ "SECURE_LOGGING_DIRS lists dirs with no .py files to scan: "
+ + ", ".join(empty)
+ )
+ # And the aggregate generator the tree scans iterate must be non-empty.
+ assert any(self._secure_dir_files()), (
+ "_secure_dir_files() yielded nothing; the tree-scan invariants "
+ "above would pass vacuously"
+ )
+
+ def test_secure_logging_dirs_membership_pinned(self):
+ """Partial-loss guard: deleting ANY member must fail loudly.
+
+ The checks above cannot catch a tuple that shrank but still
+ lists valid, non-empty dirs — every remaining scan would pass
+ while the removed security domain silently lost coverage.
+ Pin the exact expected membership so any deletion (or other
+ membership change) must consciously update this list too.
+ """
+ expected = (
+ "src/local_deep_research/llm/providers/",
+ "src/local_deep_research/embeddings/providers/",
+ "src/local_deep_research/web_search_engines/",
+ )
+ assert SECURE_LOGGING_DIRS == expected, (
+ "SECURE_LOGGING_DIRS membership changed; if intentional, update "
+ "the pinned expected list with it:\n"
+ f" actual: {SECURE_LOGGING_DIRS}\n"
+ f" expected: {expected}"
+ )