~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""End-to-end security battery: hostile remote-derived filenames -> disk writes.
Complements:
* tests/security/test_pdf_path_traversal.py (covers only the
_get_safe_file_path READ-path validator method, no actual writes)
* tests/security/test_filename_sanitizer.py (covers only the bare
sanitize_filename unit)
This file exercises the FULL save pipeline the way the production download
service feeds it: PDFStorageManager.save_pdf(pdf_content, document, session,
filename=...) in filesystem mode, with filenames derived the way
DownloadService._generate_filename derives them from a URL.
For each hostile input the test asserts:
* The written file lands STRICTLY inside the configured library root
(a tmp_path subdirectory; canary files planted OUTSIDE the root must
remain untouched and unmodified).
* No symlink escape: a pre-planted symlink inside the root pointing
outside must NOT be followed by the write path.
* Either the write is rejected cleanly (rejection is a GOOD security
outcome) OR it succeeds with the file strictly inside the root.
FAILURE CLASSIFICATION:
* Red because of vulnerability: any file lands outside library_root, any
canary outside library_root is touched, or a symlink is followed.
* Broken-test: setup/import/expectation bugs.
Run with: .venv/bin/python -m pytest tests/security/test_remote_filename_writes.py -v --no-header
"""
from __future__ import annotations
from pathlib import Path
from typing import Optional
import pytest
from local_deep_research.research_library.services.pdf_storage_manager import (
PDFStorageManager,
)
from local_deep_research.security.filename_sanitizer import (
UnsafeFilenameError,
sanitize_filename,
)
# Minimal bytes that pass "%PDF" sniffers — content is irrelevant, the
# write path is what we care about.
_MINIMAL_PDF = b"%PDF-1.4\n1 0 obj <<>>endobj\ntrailer<<>>\n%%EOF\n"
# ---------------------------------------------------------------------------
# Fixtures and helpers
# ---------------------------------------------------------------------------
# ``library_session``, ``fresh_document``, ``fs_storage`` and
# ``canary_paths`` now live in tests/security/conftest.py -- they were
# duplicated verbatim across four batteries in this directory.
def _assert_files_inside(
library_root: Path,
written_relative: Optional[str],
canaries: dict,
):
"""Strict safety assertion: any file written MUST be inside library_root
AND must not have modified any canary outside it."""
library_root = library_root.resolve()
if written_relative is not None:
written = (library_root / written_relative).resolve()
try:
written.relative_to(library_root)
except ValueError as exc:
pytest.fail(
f"VULNERABILITY: save_pdf wrote {written} OUTSIDE library_root "
f"{library_root} (relative_to raised: {exc})"
)
for name, canary in canaries.items():
assert canary.exists(), (
f"VULNERABILITY: canary {name} at {canary} was DELETED"
)
assert canary.read_text(encoding="utf-8") == "UNTOUCHED-CANARY\n", (
f"VULNERABILITY: canary {name} at {canary} was MODIFIED "
f"(content: {canary.read_text(encoding='utf-8')!r})"
)
# ---------------------------------------------------------------------------
# Hostile filename battery — full save_pdf() end-to-end
# ---------------------------------------------------------------------------
# (id, filename, url, resource_id). url=None means the filename is passed
# straight through to _save_to_filesystem (mirrors how DownloadService hands
# a derived filename in). url=<str> means _generate_filename will run first.
# Covers every hostile shape specified in the task brief (13 in the brief + 1 nested variant).
HOSTILE_FILENAMES = [
pytest.param("../escape.pdf", None, 1, id="dotdot-traversal-single"),
pytest.param("a/../../b.pdf", None, 1, id="nested-traversal"),
pytest.param("..%2Fescape.pdf", None, 1, id="encoded-slash-single"),
pytest.param("%2e%2e%2fescape.pdf", None, 1, id="encoded-dotdot-slash"),
pytest.param("evil\x00.pdf", None, 1, id="null-byte"),
pytest.param("/tmp/x.pdf", None, 1, id="absolute-path-as-filename"),
pytest.param("..\\escape.pdf", None, 1, id="backslash-traversal"),
pytest.param("....//escape.pdf", None, 1, id="quad-dot-double-slash"),
pytest.param(
"\uff0e\uff0e\uff0fescape.pdf", # fullwidth ../escape.pdf
None,
1,
id="fullwidth-unicode-traversal",
),
pytest.param("-rf.pdf", None, 1, id="leading-dash"),
pytest.param("a" * 5000 + ".pdf", None, 1, id="extremely-long-name"),
pytest.param(".pdf", None, 1, id="extension-only-hidden"),
pytest.param("x.pdf.exe", None, 1, id="double-extension"),
]
class TestHostileFilenameEndToEnd:
"""Drive the REAL PDFStorageManager.save_pdf() with hostile filenames.
Failure mode this guards against: any file landing OUTSIDE
library_root, or any canary outside the root being touched.
Exceptions raised by PathValidator / write_file_verified are ACCEPTABLE
rejections; the test passes when the rejection occurs cleanly. The
test only FAILS when a file actually lands in the wrong place.
"""
@pytest.mark.parametrize("filename,url,resource_id", HOSTILE_FILENAMES)
def test_hostile_filename_does_not_escape_library_root(
self,
fs_storage,
fresh_document,
library_session,
canary_paths,
filename,
url,
resource_id,
):
# Reset document between parametrize iterations.
fresh_document.storage_mode = None
fresh_document.file_path = None
try:
relative, _size = fs_storage.save_pdf(
pdf_content=_MINIMAL_PDF,
document=fresh_document,
session=library_session,
filename=filename,
url=url,
resource_id=resource_id,
)
except (ValueError, FileNotFoundError, OSError):
# PathValidator rejecting traversal, OS rejecting a non-existent
# intermediate dir, etc. — all valid "rejection" outcomes.
_assert_files_inside(fs_storage.library_root, None, canary_paths)
return
# save_pdf accepted the filename — assert it stayed strictly inside.
_assert_files_inside(fs_storage.library_root, relative, canary_paths)
# ---------------------------------------------------------------------------
# URL-derivation fidelity — feed _generate_filename then save_pdf
# ---------------------------------------------------------------------------
class TestURLDerivedFilename:
"""Calls DownloadService._generate_filename with hostile URLs and feeds
its OUTPUT into the real save_pdf. For non-arxiv/non-pmc URLs
_generate_filename returns the supplied fallback_filename unchanged
(no URL-path traversal can sneak through the regex extractor) — so
the hostile bit really is what _save_to_filesystem sees.
"""
URL_CASES = [
pytest.param(
"http://arxiv.org/abs/2301.12345/../../../etc/passwd.pdf",
"fallback.pdf",
id="arxiv-url-with-traversal-in-path",
),
pytest.param(
"http://attacker.example/%2e%2e/escape.pdf",
"hostile-fallback.pdf",
id="attacker-url-with-encoded-traversal",
),
]
@pytest.mark.parametrize("url,fallback", URL_CASES)
def test_url_derived_filename_does_not_escape(
self,
fs_storage,
fresh_document,
library_session,
canary_paths,
url,
fallback,
):
fresh_document.storage_mode = None
fresh_document.file_path = None
derived = PDFStorageManager._generate_filename(
fs_storage, url, 1, fallback
)
try:
relative, _size = fs_storage.save_pdf(
pdf_content=_MINIMAL_PDF,
document=fresh_document,
session=library_session,
filename=derived,
url=url,
resource_id=1,
)
except (ValueError, FileNotFoundError, OSError):
_assert_files_inside(fs_storage.library_root, None, canary_paths)
return
_assert_files_inside(fs_storage.library_root, relative, canary_paths)
# ---------------------------------------------------------------------------
# Symlink-write escape — pre-plant symlink at the EXACT write path
# ---------------------------------------------------------------------------
class TestSymlinkWriteEscape:
"""A symlink planted at the intended PDF filename must not redirect writes.
``_save_to_filesystem`` writes the bytes to a staged file and refuses a
symlinked destination present when the write begins with ELOOP (its
``is_symlink()`` check) before moving that file into place, on every
platform; the verified writer's O_NOFOLLOW never sees the destination.
This protects against a link present at the check, not against one
planted after it: the move then replaces the link itself and the
download records success, but it never writes through a link, so the
link's target is never touched. The outside canary must survive, and
legitimate in-root saves remain covered by the positive controls."""
def test_symlink_inside_pdf_subdir_does_not_let_write_escape(
self,
fs_storage,
fresh_document,
library_session,
tmp_path,
canary_paths,
):
library_root = fs_storage.library_root
target_name = "evil_link_target.pdf"
# Real canary file outside the library root.
outside_target = tmp_path / "outside_canary_zone" / target_name
outside_target.write_text("UNTOUCHED-CANARY\n", encoding="utf-8")
canary_paths["symlink_target"] = outside_target
# Discover the real destination with a successful save. New writes
# append a content hash, so planting the unhashed display name would
# leave the canary untouched without exercising the symlink guard.
saved_relative, _ = fs_storage.save_pdf(
_MINIMAL_PDF, fresh_document, library_session, target_name
)
link = library_root / saved_relative
assert link.read_bytes() == _MINIMAL_PDF
link.unlink()
try:
link.symlink_to(outside_target)
except (OSError, NotImplementedError):
pytest.skip("Cannot create symlinks in this environment")
assert link.resolve() == outside_target.resolve()
fresh_document.storage_mode = None
fresh_document.file_path = None
try:
relative, _size = fs_storage.save_pdf(
pdf_content=_MINIMAL_PDF,
document=fresh_document,
session=library_session,
filename=target_name,
)
except (ValueError, OSError):
_assert_files_inside(library_root, None, canary_paths)
return
assert relative == saved_relative
assert (
outside_target.read_text(encoding="utf-8") == "UNTOUCHED-CANARY\n"
), (
f"VULNERABILITY: save_pdf wrote through symlink and overwrote "
f"outside-root canary at {outside_target}; relative returned "
f"was {relative!r}"
)
_assert_files_inside(library_root, relative, canary_paths)
# ---------------------------------------------------------------------------
# Happy path — clean name saves inside pdfs/ — guards against over-blocking
# ---------------------------------------------------------------------------
class TestHappyPath:
def test_clean_name_saves_inside_pdfs(
self,
fs_storage,
fresh_document,
library_session,
canary_paths,
):
relative, _size = fs_storage.save_pdf(
pdf_content=_MINIMAL_PDF,
document=fresh_document,
session=library_session,
filename="clean_paper.pdf",
url="http://example.com/clean_paper.pdf",
resource_id=1,
)
written = fs_storage.library_root / relative
assert written.is_file()
# Must be inside library_root/pdfs/ — the intended location.
assert written.resolve().is_relative_to(
(fs_storage.library_root / "pdfs").resolve()
), f"clean file landed at unexpected path: {written}"
assert written.read_bytes() == _MINIMAL_PDF
assert fresh_document.file_path == relative
assert fresh_document.storage_mode == "filesystem"
_assert_files_inside(fs_storage.library_root, relative, canary_paths)
# ---------------------------------------------------------------------------
# Zotero sanitize_filename battery — remote-controlled hostile names
# ---------------------------------------------------------------------------
class TestZoteroFilenameSanitizer:
"""The Zotero sync_service (_filename_for in sync_service.py) builds a
candidate display filename from the remote item's title and feeds it
through sanitize_filename before storing on Document.filename. Drives
sanitize_filename with the same shapes of remote payloads an
attacker-controlled Zotero response could carry, and asserts the
sanitized output:
* never contains a path separator
* never begins with '..'
* raises UnsafeFilenameError when nothing safe survives
"""
CASES = [
pytest.param("../../etc/passwd", None, id="dotdot-passes"),
pytest.param("evil\x00name", None, id="null-byte"),
pytest.param("", None, id="empty-raises"),
]
@pytest.mark.parametrize("raw,exts", CASES)
def test_sanitize_blocks_traversal_in_zotero_titles(self, raw, exts):
try:
safe = sanitize_filename(raw, allowed_extensions=exts)
except UnsafeFilenameError:
return
assert "/" not in safe, (
f"sanitize_filename returned a path-containing value: {safe!r}"
)
assert "\\" not in safe, (
f"sanitize_filename returned a backslash-containing value: {safe!r}"
)
assert not safe.startswith(".."), (
f"sanitize_filename returned a traversal-leading value: {safe!r}"
)
assert "\x00" not in safe, (
f"sanitize_filename did not strip null bytes: {safe!r}"
)