~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""
Fixtures for security module tests.
"""
from __future__ import annotations
import uuid
from pathlib import Path
import platformdirs
import pytest
from fastapi.testclient import TestClient
from loguru import logger
from .auth_fixtures import AuthenticatedUser
# ---------------------------------------------------------------------------
# Guard against unconfigured platform-default-directory writes
#
# security/test-file-write-risk-battery (PR #5638) audited these files to
# confirm every real filesystem write happens under tmp_path, pinning
# LDR_DATA_DIR (via monkeypatch.setenv) wherever a call chain resolves a
# settings-derived path. The audit found several holes caused by an
# EAGERLY-EVALUATED DEFAULT ARGUMENT: e.g.
# ``settings.get_setting("research_library.storage_path",
# str(get_library_directory()))`` in ``research_library/utils/__init__.py``
# -- Python evaluates ``str(get_library_directory())`` (which
# ``mkdir(parents=True)``s ``<data_dir>/library`` via ``config.paths.
# _ensure_dir``) BEFORE ``get_setting`` even runs, regardless of whether a
# stub/mock settings manager ever looks at that default. A test that forgot
# to pin ``LDR_DATA_DIR`` therefore created a real directory under
# ``platformdirs.user_data_dir("local-deep-research")`` on the machine
# running the suite -- exactly what this battery exists to prove doesn't
# happen. See ``test_document_path_deletion.py``'s ``_settings_root`` and
# ``test_settings_path_escape.py``'s
# ``test_relative_storage_path_still_creates_a_library`` for the fixed
# instances.
#
# Security tests use a worker-private platform default. Guarded tests get a
# fresh default of their own, so writes still fail rather than disappearing
# into an already populated worker directory. Workers otherwise observe the
# same user directory and can attribute another worker's write to the wrong
# test. Only the application's default is redirected; LDR_DATA_DIR overrides
# and other applications' platformdirs calls retain their usual behavior.
# The write assertion covers exactly the audited battery below (by filename).
# The rest of tests/security/ has not been swept for this same hazard, so a
# blanket assertion there could flag files nobody has vetted. Guarded tests
# must pin LDR_DATA_DIR or patch the path
# provider they intend to exercise; a write through the unconfigured default
# still fails even though its destination is now safely under pytest's base
# temporary directory.
_GUARDED_BATTERY_FILES = {
"test_benchmark_and_temp_lifecycle.py",
"test_disk_write_guards.py",
"test_disk_write_guards_extended.py",
"test_document_path_deletion.py",
"test_downloader_filename_pipeline.py",
"test_identity_and_index_pins.py",
"test_identity_paths_and_host_effects.py",
"test_journal_dataset_writes.py",
"test_mcp_and_secure_logging.py",
"test_notes_and_upload_paths.py",
"test_path_property_fuzz.py",
"test_poisoned_path_reads.py",
"test_queue_processor_writes.py",
"test_raw_config_toml_smuggling.py",
"test_remote_filename_writes.py",
"test_report_path_deletion.py",
"test_secure_logging_writes.py",
"test_settings_bulk_import.py",
"test_settings_path_escape.py",
"test_zotero_ingest_writes.py",
}
def _is_guarded_battery_test(request: pytest.FixtureRequest) -> bool:
return request.node.path.name in _GUARDED_BATTERY_FILES
def _isolated_user_data_dir(default_data_dir: Path, original_user_data_dir):
def user_data_dir(appname=None, *args, **kwargs):
if appname == "local-deep-research":
return str(default_data_dir)
return original_user_data_dir(appname, *args, **kwargs)
return user_data_dir
@pytest.fixture(scope="session")
def _worker_platform_default_dir(tmp_path_factory):
return (
tmp_path_factory.mktemp("platform-default-worker")
/ "local-deep-research"
)
@pytest.fixture(autouse=True)
def _isolate_default_data_dir(monkeypatch, _worker_platform_default_dir):
"""Keep security-test fallbacks off the machine's actual data directory.
The path belongs to this worker; the provider patch is function-scoped
so it cannot leak into tests outside this conftest's directory.
"""
monkeypatch.setattr(
platformdirs,
"user_data_dir",
_isolated_user_data_dir(
_worker_platform_default_dir, platformdirs.user_data_dir
),
)
@pytest.fixture(autouse=True)
def _guard_against_real_data_dir_writes(
request, monkeypatch, tmp_path_factory, _isolate_default_data_dir
):
"""Reject default-data writes without sharing another worker's baseline.
Use a separate temporary path for the platform default, outside the
test's explicit data root. Application path helpers still resolve and
create real directories, so the same before/after check catches an
unintended fallback. A peer worker's activity cannot change this path.
"""
if not _is_guarded_battery_test(request):
yield
return
default_data_dir = (
tmp_path_factory.mktemp("platform-default-guard")
/ "local-deep-research"
)
monkeypatch.setattr(
platformdirs,
"user_data_dir",
_isolated_user_data_dir(default_data_dir, platformdirs.user_data_dir),
)
before = (
set(default_data_dir.iterdir()) if default_data_dir.is_dir() else set()
)
yield
after = (
set(default_data_dir.iterdir()) if default_data_dir.is_dir() else set()
)
leaked = after - before
assert not leaked, (
"File-write-risk battery test leaked a write to the unconfigured "
f"platformdirs data directory ({default_data_dir}), instead of an "
"explicit temporary data root: "
f"{sorted(p.name for p in leaked)}. "
"This almost always means a call chain resolved a "
"settings-derived path (get_library_directory / "
"get_absolute_path_from_settings / _ensure_dir / BackupService / "
"DownloadService / get_library_storage_path / etc.) without the "
"test pinning LDR_DATA_DIR via monkeypatch.setenv -- see this "
"fixture's module docstring for the eager-default-argument "
"mechanism."
)
def _loguru_global_state() -> dict[str, object]:
"""Snapshot the process-wide loguru configuration a test can leak.
``config_logger()`` and ``configure_mcp_logging()`` (which
``run_server()`` calls) replace the handler table, install the
credential-redaction patcher and enable the ``local_deep_research``
namespace. Loguru's per-module ``enabled`` cache is left out on purpose:
loguru fills it lazily for every module that logs, so it also grows in
tests that never touch the configuration.
"""
core = logger._core
return {
"patcher": core.patcher,
"activation_list": list(core.activation_list),
"activation_none": core.activation_none,
"handlers": dict(core.handlers),
"min_level": core.min_level,
}
@pytest.fixture(autouse=True)
def _guard_against_loguru_state_leaks(request):
"""Fail the battery test that leaves loguru's global configuration changed.
Loguru keeps one handler table and one patcher per process, and
monkeypatch restores neither. A leaked redaction patcher rewrites every
later record in the same worker, so the failure surfaces in an unrelated
file instead of the test that caused it: the ``***REDACTED***`` marker
asserted by ``test_settings_logger_non_dict_redaction.py`` arrives as
``[REDACTED]``. Run in-process logging configuration inside
``tests.test_utils.restored_loguru_state()``.
"""
if not _is_guarded_battery_test(request):
yield
return
before = _loguru_global_state()
yield
after = _loguru_global_state()
changed = sorted(
name for name, value in before.items() if after[name] != value
)
assert not changed, (
"File-write-risk battery test leaked process-wide loguru state "
f"({', '.join(changed)}). Run in-process config_logger() and "
"run_server() calls inside tests.test_utils.restored_loguru_state()."
)
@pytest.fixture
def mock_pdf_content():
"""Create minimal valid PDF content for testing."""
return b"""%PDF-1.4
1 0 obj
<< /Type /Catalog /Pages 2 0 R >>
endobj
2 0 obj
<< /Type /Pages /Kids [3 0 R] /Count 1 >>
endobj
3 0 obj
<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] >>
endobj
xref
0 4
0000000000 65535 f
0000000009 00000 n
0000000058 00000 n
0000000115 00000 n
trailer
<< /Size 4 /Root 1 0 R >>
startxref
193
%%EOF"""
@pytest.fixture
def sample_data_with_secrets():
"""Sample data containing sensitive keys."""
return {
"username": "testuser",
"email": "[email protected]",
"api_key": "sk-secret-12345",
"password": "supersecretpassword",
"settings": {
"theme": "dark",
"access_token": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9",
},
"items": [
{"name": "item1", "secret": "hidden_value"},
{"name": "item2", "value": "visible"},
],
}
@pytest.fixture
def sample_urls():
"""Collection of sample URLs for testing."""
return {
"valid_http": "http://example.com",
"valid_https": "https://example.com/page",
"valid_arxiv": "https://arxiv.org/abs/2301.12345",
"valid_pubmed": "https://pubmed.ncbi.nlm.nih.gov/12345678",
"javascript": "javascript:alert('xss')",
"data": "data:text/html,<script>alert('xss')</script>",
"vbscript": "vbscript:msgbox('xss')",
"file": "file:///etc/passwd",
"mailto": "mailto:[email protected]",
"fragment": "#section-1",
"empty": "",
"none": None,
}
# ---------------------------------------------------------------------------
# Shared library-PDF fixtures.
#
# Hoisted out of the four security batteries that each carried a verbatim
# copy (test_remote_filename_writes, test_zotero_ingest_writes,
# test_downloader_filename_pipeline, test_path_property_fuzz) so the set is
# defined once. ``test_poisoned_path_reads`` keeps its own file-local
# ``library_session`` on purpose: it yields ``(session, source_type)`` rather
# than a bare session, and a file-local fixture shadows this one.
#
# The heavy imports stay inside the fixture bodies so importing this conftest
# does not drag the library stack into every unrelated security test module.
# ---------------------------------------------------------------------------
@pytest.fixture
def library_session():
"""In-memory SQLite session with the library models loaded.
``StaticPool`` + ``check_same_thread=False``: some callers (e.g.
``test_path_property_fuzz.py``'s ``TestPositiveControls``) patch this
session into a real FastAPI ``TestClient`` request for a sync route,
which Starlette dispatches through ``run_in_threadpool`` — a
different OS thread than the one that created/seeded this session.
Without a shared, thread-agnostic connection, SQLAlchemy either hands
that worker thread a fresh, empty ``:memory:`` database or reuses the
literal ``sqlite3.Connection`` object across threads and trips
Python's ``check_same_thread`` guard (``sqlite3.ProgrammingError:
SQLite objects created in a thread can only be used in that same
thread``) on close/rollback. Same fix as
``test_socket_ownership_edges_fastapi.py`` and
``test_pagination_bounds.py``.
"""
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from sqlalchemy.pool import StaticPool
from local_deep_research.database.models import Base
engine = create_engine(
"sqlite:///:memory:",
poolclass=StaticPool,
connect_args={"check_same_thread": False},
)
Base.metadata.create_all(engine)
with sessionmaker(bind=engine)() as session:
yield session
engine.dispose()
@pytest.fixture
def fresh_document(library_session):
"""A new completed Document per test, so file_path/storage_mode reset.
The ``SourceType`` row is created inline rather than in its own
``source_type`` fixture: no test in this directory requests it
directly, and ``source_type`` is already used as a parametrize
argname in test_journal_dataset_writes.py.
"""
import hashlib
import uuid
from local_deep_research.database.models.library import (
Document,
DocumentStatus,
SourceType,
)
source_type = SourceType(
id=str(uuid.uuid4()),
name="research_download",
display_name="Research Download",
description="x",
)
library_session.add(source_type)
library_session.commit()
value = Document(
id=str(uuid.uuid4()),
source_type_id=source_type.id,
document_hash=hashlib.sha256(b"placeholder").hexdigest(),
file_size=0,
file_type="pdf",
status=DocumentStatus.COMPLETED,
title="t",
)
library_session.add(value)
library_session.commit()
return value
@pytest.fixture
def fs_storage(tmp_path):
"""Real PDFStorageManager in filesystem mode rooted under tmp_path."""
from local_deep_research.research_library.services.pdf_storage_manager import (
PDFStorageManager,
)
library_root = tmp_path / "library"
(library_root / "pdfs").mkdir(parents=True)
return PDFStorageManager(
library_root=library_root,
storage_mode="filesystem",
username="testuser",
)
@pytest.fixture
def canary_paths(tmp_path):
"""Files planted OUTSIDE library_root that must remain untouched."""
outside = tmp_path / "outside_canary_zone"
outside.mkdir()
canaries = {
"outer_passwd": outside / "passwd.txt",
"outer_home": outside / "home.txt",
"outer_etc": outside / "etc_marker.txt",
}
for path in canaries.values():
path.write_text("UNTOUCHED-CANARY\n", encoding="utf-8")
return canaries
# =============================================================================
# Cross-user (two-principal) fixtures
# =============================================================================
#
# FastAPI-native replacement for the Flask-era fixture of the same name (the
# production code it originally targeted, news/flask_api.py, was removed by
# the FastAPI migration, #3299). These pin BOLA/per-user routing end-to-end:
# require_auth -> session cookie -> the caller's own per-user encrypted
# database (get_user_db_session(<authenticated username>)), not a shared
# table filtered by a foreign key. The warm-cache wrong-password regression
# from #5596 has separate focused tests.
#
# Modeled on the ``_new_client``/``_register_and_login`` helpers in
# ``test_two_user_attack_simulation.py`` -- kept local (not imported) so this
# fixture has no import-order dependency on that test module.
def _new_test_client(app) -> TestClient:
"""A client with its own cookie jar and its own rate-limit bucket.
``/auth/register`` is capped per-IP; the TestClient peer address is
treated as private so ``X-Forwarded-For`` is honoured for the slowapi
key. Without a unique one, user B's registration lands in user A's
bucket.
"""
client = TestClient(app, raise_server_exceptions=False)
octet_a = uuid.uuid4().int % 254 + 1
octet_b = uuid.uuid4().int % 254 + 1
client.headers.update({"X-Forwarded-For": f"10.{octet_a}.{octet_b}.1"})
return client
def _csrf(client: TestClient) -> str:
client.get("/auth/login")
resp = client.get("/auth/csrf-token")
return resp.json().get("csrf_token", "") if resp.status_code == 200 else ""
def _drop_stale_csrf_header(client: TestClient) -> None:
"""CSRFMiddleware prefers the header over the form field.
``_register_and_login`` attaches a persistent ``X-CSRFToken`` default
header; once the session rotates its token that stale header shadows the
correct ``csrf_token`` form field and every form POST 403s.
"""
for name in ("X-CSRFToken", "X-CSRF-Token"):
client.headers.pop(name, None)
def _register_and_login(
client: TestClient, username: str, password: str
) -> None:
_drop_stale_csrf_header(client)
resp = client.post(
"/auth/register",
data={
"username": username,
"password": password,
"confirm_password": password,
"acknowledge": "true",
"csrf_token": _csrf(client),
},
follow_redirects=False,
)
assert resp.status_code == 302, (
f"Registration failed for {username}: "
f"{resp.status_code} {resp.text[:300]}"
)
_drop_stale_csrf_header(client)
resp = client.post(
"/auth/login",
data={
"username": username,
"password": password,
"csrf_token": _csrf(client),
},
follow_redirects=False,
)
assert resp.status_code == 302, (
f"Login failed for {username}: {resp.status_code} {resp.text[:300]}"
)
# Attach the fresh (post-login) CSRF token as a default header so later
# PUT/POST/DELETE calls through this client pass CSRFMiddleware without
# each test having to fetch and pass it explicitly.
token = client.get("/auth/csrf-token")
if token.status_code == 200:
client.headers.update({"X-CSRFToken": token.json()["csrf_token"]})
@pytest.fixture
def two_authenticated_users(
app,
) -> tuple[AuthenticatedUser, AuthenticatedUser]:
"""Register and log in two independent users against one FastAPI app.
Returns ``(user_a, user_b)`` as :class:`AuthenticatedUser` tuples. The
``app`` fixture (tests/conftest.py) points ``LDR_DATA_DIR`` at a fresh
per-test temp directory before the app is created, so each test gets its
own encrypted-database directory -- no manual cleanup needed between
user A and user B.
"""
username_a = f"sec2u_{uuid.uuid4().hex[:12]}"
username_b = f"sec2u_{uuid.uuid4().hex[:12]}"
password_a = "TestPass123!" # noqa: S105
password_b = "OtherPass456!" # noqa: S105
client_a = _new_test_client(app)
client_b = _new_test_client(app)
_register_and_login(client_a, username_a, password_a)
_register_and_login(client_b, username_b, password_b)
return (
AuthenticatedUser(username_a, password_a, client_a),
AuthenticatedUser(username_b, password_b, client_b),
)