~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
# allow: no-sut-import — drives real log_utils config_logger() file sinks inside isolated subprocesses (LDR_TEST_LOG_PAYLOAD); production code runs in the child, not imported at module level here.
"""Security battery for the secure-logging FILE write path.
Complements ``test_secure_logging.py`` (record-level wrapper gates verified
via caplog) and ``tests/utilities/test_log_utils*.py`` (mocked sink wiring):
this battery drives REAL ``config_logger()`` file-sink writes in isolated
subprocesses and inspects what actually lands on disk.
Surfaces pinned:
* ``utilities/log_utils.py`` — the ``LDR_ENABLE_FILE_LOGGING`` gate (~:761),
import-time ``logs`` dir creation (~:27-28), the control-char patcher
(~:689-692), and the file sink's ``diagnose=False`` (~:781).
* ``config/paths.py`` — ``get_logs_directory()`` derives strictly from
``LDR_DATA_DIR``. Neither a hostile ``LDR_LOG_DIR`` env var nor
``LDR_BOOTSTRAP_LOG_DIR`` (the variable of a ``bootstrap.log_dir`` setting
that nothing ever read and that has been removed from the registry;
operators may still export it) can move the sink: the same containment
contract as ``test_settings_path_escape.py`` (battery B).
* ``security/log_sanitizer.py`` — the sanctioned dual-scrub keeps known
secrets off disk. The file sink also redacts raw credential fields;
``test_raw_sensitive_fields_are_redacted_before_disk_write`` pins that
protection independently of explicit caller-side scrubbing.
Each test spawns a fresh interpreter so ``log_utils`` resolves ``_LOG_DIR``
under the controlled environment (the real import-time path) and no global
loguru state leaks into the pytest session.
"""
import json
import os
import subprocess
import sys
from pathlib import Path
import pytest
_REPO_ROOT = Path(__file__).resolve().parents[2]
# Env vars that must never leak from the outer session into the child.
_SCRUBBED_ENV_VARS = (
"LDR_APP_DEBUG",
"LDR_LOGURU_DIAGNOSE",
"LDR_LOG_DIR",
"LDR_BOOTSTRAP_LOG_DIR",
"LDR_ENABLE_FILE_LOGGING",
)
# The child: configure logging exactly as the application does, emit one
# record selected by LDR_TEST_EMIT_MODE, then report every *.log file that
# exists under LDR_DATA_DIR.
_EMIT_SCRIPT = """
import json
import os
from pathlib import Path
from local_deep_research.utilities.log_utils import config_logger
from local_deep_research.security.secure_logging import logger
config_logger("write-battery")
mode = os.environ.get("LDR_TEST_EMIT_MODE", "info")
if mode == "info":
logger.info(os.environ["LDR_TEST_LOG_PAYLOAD"])
elif mode == "error":
logger.error(os.environ["LDR_TEST_LOG_PAYLOAD"])
elif mode == "scrubbed":
from local_deep_research.security.log_sanitizer import scrub_error
logger.error(
scrub_error(
os.environ["LDR_TEST_LOG_PAYLOAD"],
os.environ["LDR_TEST_LOG_SECRET"],
)
)
elif mode == "control-chars":
logger.error("ctrl-marker\\x00\\x1b[31m\\u202efin")
elif mode == "exception":
try:
raise ValueError(os.environ["LDR_TEST_LOG_PAYLOAD"])
except ValueError:
logger.exception("emit exception marker")
data_dir = Path(os.environ["LDR_DATA_DIR"])
print(json.dumps(
{"logs": [str(path.resolve()) for path in data_dir.rglob("*.log")]},
))
"""
def _files_below(directory: Path) -> dict[Path, bytes]:
"""Capture files beneath a controlled test directory, including canaries."""
return {
path.relative_to(directory): path.read_bytes()
for path in directory.rglob("*")
if path.is_file()
}
def _run_emit(
tmp_path: Path,
data_dir: str | None = None,
expect_success: bool = True,
**environment: str | None,
) -> tuple[subprocess.CompletedProcess[str], Path]:
"""Run the emit script in a fresh interpreter under a controlled env.
The child runs with its own workspace directory as cwd (so a relative
``../escape`` resolves inside ``tmp_path``) and its own ``$HOME`` (so
``$HOME``-based escape attempts stay observable). Returns the completed
process and the effective data directory.
"""
env = os.environ.copy()
for var in _SCRUBBED_ENV_VARS:
_ = env.pop(var, None)
env["LDR_DATA_DIR"] = data_dir or str(tmp_path / "data")
env["LDR_TEST_LOG_PAYLOAD"] = "write-battery marker"
for name, value in environment.items():
if value is None:
_ = env.pop(name, None)
else:
env[name] = value
workspace = tmp_path / "workspace"
workspace.mkdir(exist_ok=True)
home = tmp_path / "home"
home.mkdir(exist_ok=True)
env["HOME"] = str(home)
result = subprocess.run(
[sys.executable, "-c", _EMIT_SCRIPT],
cwd=workspace,
env=env,
capture_output=True,
text=True,
# A fresh interpreter pays the full LDR import cost (see the
# module docstring); 20s is tight on a loaded CI runner.
timeout=60,
check=False,
)
if expect_success:
assert result.returncode == 0, result.stderr
return result, Path(env["LDR_DATA_DIR"])
def _log_paths(result: subprocess.CompletedProcess[str]) -> list[Path]:
"""Parse the child's JSON report of *.log files under its data dir."""
payload: dict[str, list[str]] = json.loads(result.stdout)
return [Path(entry) for entry in payload["logs"]]
def _log_contents(result: subprocess.CompletedProcess[str]) -> str:
return "\n".join(path.read_text() for path in _log_paths(result))
class TestFileLoggingGate:
"""LDR_ENABLE_FILE_LOGGING must gate every file write."""
@pytest.mark.parametrize("enabled", [None, "false"])
def test_gate_off_creates_no_log_file(self, tmp_path, enabled):
"""Unset or explicitly false: no *.log anywhere under the data root."""
result, _ = _run_emit(tmp_path, LDR_ENABLE_FILE_LOGGING=enabled)
assert not _log_paths(result)
assert not list(tmp_path.rglob("*.log"))
@pytest.mark.parametrize("enabled", ["true", "TRUE"])
def test_gate_on_writes_inside_data_logs(self, tmp_path, enabled):
"""Enabled: exactly <LDR_DATA_DIR>/logs/write-battery.log, with content."""
result, data_dir = _run_emit(tmp_path, LDR_ENABLE_FILE_LOGGING=enabled)
logs = _log_paths(result)
assert logs, "file logging was enabled but no log file exists"
assert all(path.parent == data_dir / "logs" for path in logs), logs
assert "write-battery marker" in _log_contents(result)
class TestLogDirContainment:
"""The log file must stay inside LDR_DATA_DIR under hostile inputs."""
@pytest.mark.parametrize(
"attack",
["relative", "expandvars", "absolute"],
)
def test_hostile_log_dir_env_cannot_move_log_file(self, tmp_path, attack):
"""LDR_LOG_DIR / LDR_BOOTSTRAP_LOG_DIR cannot relocate the sink.
``LDR_BOOTSTRAP_LOG_DIR`` was the variable of a ``bootstrap.log_dir``
setting that nothing ever read and that has been removed from
``settings/env_definitions/bootstrap.py``; operators may still
export it. No code reads it or ``LDR_LOG_DIR`` — the sink location
is derived only from ``LDR_DATA_DIR``
(``config/paths.py::get_logs_directory``).
"""
outside = tmp_path / "outside"
outside.mkdir()
(outside / "CANARY").write_text("do not alter")
hostile_values = {
"relative": "../escape",
"expandvars": "$HOME/escape",
"absolute": str(outside / "absolute-logs"),
}
before = _files_below(tmp_path)
result, data_dir = _run_emit(
tmp_path,
LDR_ENABLE_FILE_LOGGING="true",
LDR_LOG_DIR=hostile_values[attack],
LDR_BOOTSTRAP_LOG_DIR=hostile_values[attack],
)
logs = _log_paths(result)
data_dir = data_dir.resolve()
assert logs, "file logging was enabled but no log file exists"
assert all(path.is_relative_to(data_dir) for path in logs), logs
new_files = set(_files_below(tmp_path)) - set(before)
assert new_files, "the child wrote nothing at all"
assert all(
(tmp_path / rel).is_relative_to(data_dir) for rel in new_files
), sorted(new_files)
assert not (tmp_path / "escape").exists()
assert not (tmp_path / "home" / "escape").exists()
assert _files_below(outside) == {Path("CANARY"): b"do not alter"}
def test_relative_data_dir_fails_closed_without_writing(self, tmp_path):
"""A relative LDR_DATA_DIR must abort loudly, never write logs."""
result, _ = _run_emit(
tmp_path,
data_dir="relative/ldr-data",
expect_success=False,
)
assert result.returncode != 0
assert "absolute" in result.stderr
assert not list(tmp_path.rglob("*.log"))
def test_import_creates_logs_dir_under_data_dir_when_gate_off(
self, tmp_path
):
"""Import-time mkdir (log_utils.py:27-28) stays inside LDR_DATA_DIR.
The gate suppresses the FILE, not the directory: ``logs/`` is
created eagerly, which is safe only because it is clamped under the
(validated) data directory.
"""
result, data_dir = _run_emit(tmp_path) # gate unset
assert (data_dir / "logs").is_dir()
assert not _log_paths(result)
class TestDiskScrubbing:
"""What reaches the persistent, unencrypted log file."""
def test_scrubbed_secret_never_reaches_disk(self, tmp_path):
"""Dual-scrub output survives the write path without the secrets.
The sanctioned catch-site pattern is
``logger.error(scrub_error(msg, known_literal))`` — credential
shapes (URL userinfo, query params, key prefixes) plus the caller's
literal secret must all be absent from the file.
"""
message = (
"scrubbed-marker provider failed "
"https://user:[email protected]/v1"
"?api_key=AKIAABCDEFGHIJKLMNOP&token=tok-abcdefghijklmnop1234 "
"sk-live-abcdefghijklmnopqrst and known SuperSecretLiteral99"
)
result, _ = _run_emit(
tmp_path,
LDR_ENABLE_FILE_LOGGING="true",
LDR_TEST_EMIT_MODE="scrubbed",
LDR_TEST_LOG_PAYLOAD=message,
LDR_TEST_LOG_SECRET="SuperSecretLiteral99",
)
contents = _log_contents(result)
assert "scrubbed-marker" in contents
assert "[REDACTED" in contents
for leak in (
"hunter2secret",
"AKIAABCDEFGHIJKLMNOP",
"tok-abcdefghijklmnop1234",
"sk-live-abcdefghijklmnopqrst",
"SuperSecretLiteral99",
):
assert leak not in contents, leak
def test_control_characters_stripped_before_disk_write(self, tmp_path):
"""The config_logger patcher strips control/format chars on disk."""
result, _ = _run_emit(
tmp_path,
LDR_ENABLE_FILE_LOGGING="true",
LDR_TEST_EMIT_MODE="control-chars",
)
contents = _log_contents(result)
assert "ctrl-marker" in contents
assert "\x00" not in contents
assert "\x1b" not in contents
assert "\u202e" not in contents
def test_secure_logger_exception_writes_error_without_traceback(
self, tmp_path
):
"""SecureLogger.exception() on disk: ERROR line, no traceback text.
The on-disk counterpart of ``test_secure_logging.py``'s caplog
assertions: with both diagnose flags unset, the record carries no
exception, so the unencrypted file cannot capture the
secret-bearing traceback.
"""
result, _ = _run_emit(
tmp_path,
LDR_ENABLE_FILE_LOGGING="true",
LDR_TEST_EMIT_MODE="exception",
LDR_TEST_LOG_PAYLOAD="api_key=sk-LEAK-0123456789abcdef",
)
contents = _log_contents(result)
assert "emit exception marker" in contents
assert "ERROR" in contents
assert "Traceback" not in contents
assert "sk-LEAK" not in contents
def test_raw_sensitive_fields_are_redacted_before_disk_write(
self, tmp_path
):
"""Raw ``field=value`` secrets must not reach the persistent file.
All four field shapes ride one log line so this pays for a single
interpreter spawn instead of four -- each ``_run_emit`` starts a
fresh ``sys.executable`` and pays the full LDR import cost.
"""
fields = {
"api_key": "api-key-plaintext-12345678",
"password": "password-plaintext-12345678",
"secret": "secret-plaintext-12345678",
"token": "token-plaintext-12345678",
}
payload = "write-battery raw " + " ".join(
f"{field}={secret}" for field, secret in fields.items()
)
result, _ = _run_emit(
tmp_path,
LDR_ENABLE_FILE_LOGGING="true",
LDR_TEST_EMIT_MODE="error",
LDR_TEST_LOG_PAYLOAD=payload,
)
contents = _log_contents(result)
assert "write-battery raw " in contents # the line itself arrived
for field, secret in fields.items():
assert f"{field}=" in contents # the field survived to disk
assert secret not in contents, (
f"plaintext {field} value reached the unencrypted log file"
)