Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

14.3 KB
Raw Blame
"""End-to-end security battery: hostile remote-derived filenames -> disk writes.

Complements:
  * tests/security/test_pdf_path_traversal.py  (covers only the
    _get_safe_file_path READ-path validator method, no actual writes)
  * tests/security/test_filename_sanitizer.py  (covers only the bare
    sanitize_filename unit)

This file exercises the FULL save pipeline the way the production download
service feeds it: PDFStorageManager.save_pdf(pdf_content, document, session,
filename=...) in filesystem mode, with filenames derived the way
DownloadService._generate_filename derives them from a URL.

For each hostile input the test asserts:
  * The written file lands STRICTLY inside the configured library root
    (a tmp_path subdirectory; canary files planted OUTSIDE the root must
    remain untouched and unmodified).
  * No symlink escape: a pre-planted symlink inside the root pointing
    outside must NOT be followed by the write path.
  * Either the write is rejected cleanly (rejection is a GOOD security
    outcome) OR it succeeds with the file strictly inside the root.

FAILURE CLASSIFICATION:
  * Red because of vulnerability: any file lands outside library_root, any
    canary outside library_root is touched, or a symlink is followed.
  * Broken-test: setup/import/expectation bugs.

Run with: .venv/bin/python -m pytest tests/security/test_remote_filename_writes.py -v --no-header
"""

from __future__ import annotations

from pathlib import Path
from typing import Optional

import pytest

from local_deep_research.research_library.services.pdf_storage_manager import (
    PDFStorageManager,
)
from local_deep_research.security.filename_sanitizer import (
    UnsafeFilenameError,
    sanitize_filename,
)


# Minimal bytes that pass "%PDF" sniffers — content is irrelevant, the
# write path is what we care about.
_MINIMAL_PDF = b"%PDF-1.4\n1 0 obj <<>>endobj\ntrailer<<>>\n%%EOF\n"


# ---------------------------------------------------------------------------
# Fixtures and helpers
# ---------------------------------------------------------------------------


# ``library_session``, ``fresh_document``, ``fs_storage`` and
# ``canary_paths`` now live in tests/security/conftest.py -- they were
# duplicated verbatim across four batteries in this directory.


def _assert_files_inside(
    library_root: Path,
    written_relative: Optional[str],
    canaries: dict,
):
    """Strict safety assertion: any file written MUST be inside library_root
    AND must not have modified any canary outside it."""
    library_root = library_root.resolve()

    if written_relative is not None:
        written = (library_root / written_relative).resolve()
        try:
            written.relative_to(library_root)
        except ValueError as exc:
            pytest.fail(
                f"VULNERABILITY: save_pdf wrote {written} OUTSIDE library_root "
                f"{library_root} (relative_to raised: {exc})"
            )

    for name, canary in canaries.items():
        assert canary.exists(), (
            f"VULNERABILITY: canary {name} at {canary} was DELETED"
        )
        assert canary.read_text(encoding="utf-8") == "UNTOUCHED-CANARY\n", (
            f"VULNERABILITY: canary {name} at {canary} was MODIFIED "
            f"(content: {canary.read_text(encoding='utf-8')!r})"
        )


# ---------------------------------------------------------------------------
# Hostile filename battery — full save_pdf() end-to-end
# ---------------------------------------------------------------------------


# (id, filename, url, resource_id).  url=None means the filename is passed
# straight through to _save_to_filesystem (mirrors how DownloadService hands
# a derived filename in).  url=<str> means _generate_filename will run first.
# Covers every hostile shape specified in the task brief (13 in the brief + 1 nested variant).
HOSTILE_FILENAMES = [
    pytest.param("../escape.pdf", None, 1, id="dotdot-traversal-single"),
    pytest.param("a/../../b.pdf", None, 1, id="nested-traversal"),
    pytest.param("..%2Fescape.pdf", None, 1, id="encoded-slash-single"),
    pytest.param("%2e%2e%2fescape.pdf", None, 1, id="encoded-dotdot-slash"),
    pytest.param("evil\x00.pdf", None, 1, id="null-byte"),
    pytest.param("/tmp/x.pdf", None, 1, id="absolute-path-as-filename"),
    pytest.param("..\\escape.pdf", None, 1, id="backslash-traversal"),
    pytest.param("....//escape.pdf", None, 1, id="quad-dot-double-slash"),
    pytest.param(
        "\uff0e\uff0e\uff0fescape.pdf",  # fullwidth ../escape.pdf
        None,
        1,
        id="fullwidth-unicode-traversal",
    ),
    pytest.param("-rf.pdf", None, 1, id="leading-dash"),
    pytest.param("a" * 5000 + ".pdf", None, 1, id="extremely-long-name"),
    pytest.param(".pdf", None, 1, id="extension-only-hidden"),
    pytest.param("x.pdf.exe", None, 1, id="double-extension"),
]


class TestHostileFilenameEndToEnd:
    """Drive the REAL PDFStorageManager.save_pdf() with hostile filenames.

    Failure mode this guards against: any file landing OUTSIDE
    library_root, or any canary outside the root being touched.

    Exceptions raised by PathValidator / write_file_verified are ACCEPTABLE
    rejections; the test passes when the rejection occurs cleanly.  The
    test only FAILS when a file actually lands in the wrong place.
    """

    @pytest.mark.parametrize("filename,url,resource_id", HOSTILE_FILENAMES)
    def test_hostile_filename_does_not_escape_library_root(
        self,
        fs_storage,
        fresh_document,
        library_session,
        canary_paths,
        filename,
        url,
        resource_id,
    ):
        # Reset document between parametrize iterations.
        fresh_document.storage_mode = None
        fresh_document.file_path = None

        try:
            relative, _size = fs_storage.save_pdf(
                pdf_content=_MINIMAL_PDF,
                document=fresh_document,
                session=library_session,
                filename=filename,
                url=url,
                resource_id=resource_id,
            )
        except (ValueError, FileNotFoundError, OSError):
            # PathValidator rejecting traversal, OS rejecting a non-existent
            # intermediate dir, etc. — all valid "rejection" outcomes.
            _assert_files_inside(fs_storage.library_root, None, canary_paths)
            return

        # save_pdf accepted the filename — assert it stayed strictly inside.
        _assert_files_inside(fs_storage.library_root, relative, canary_paths)


# ---------------------------------------------------------------------------
# URL-derivation fidelity — feed _generate_filename then save_pdf
# ---------------------------------------------------------------------------


class TestURLDerivedFilename:
    """Calls DownloadService._generate_filename with hostile URLs and feeds
    its OUTPUT into the real save_pdf.  For non-arxiv/non-pmc URLs
    _generate_filename returns the supplied fallback_filename unchanged
    (no URL-path traversal can sneak through the regex extractor) — so
    the hostile bit really is what _save_to_filesystem sees.
    """

    URL_CASES = [
        pytest.param(
            "http://arxiv.org/abs/2301.12345/../../../etc/passwd.pdf",
            "fallback.pdf",
            id="arxiv-url-with-traversal-in-path",
        ),
        pytest.param(
            "http://attacker.example/%2e%2e/escape.pdf",
            "hostile-fallback.pdf",
            id="attacker-url-with-encoded-traversal",
        ),
    ]

    @pytest.mark.parametrize("url,fallback", URL_CASES)
    def test_url_derived_filename_does_not_escape(
        self,
        fs_storage,
        fresh_document,
        library_session,
        canary_paths,
        url,
        fallback,
    ):
        fresh_document.storage_mode = None
        fresh_document.file_path = None

        derived = PDFStorageManager._generate_filename(
            fs_storage, url, 1, fallback
        )

        try:
            relative, _size = fs_storage.save_pdf(
                pdf_content=_MINIMAL_PDF,
                document=fresh_document,
                session=library_session,
                filename=derived,
                url=url,
                resource_id=1,
            )
        except (ValueError, FileNotFoundError, OSError):
            _assert_files_inside(fs_storage.library_root, None, canary_paths)
            return

        _assert_files_inside(fs_storage.library_root, relative, canary_paths)


# ---------------------------------------------------------------------------
# Symlink-write escape — pre-plant symlink at the EXACT write path
# ---------------------------------------------------------------------------


class TestSymlinkWriteEscape:
    """A symlink planted at the intended PDF filename must not redirect writes.

    ``_save_to_filesystem`` writes the bytes to a staged file and refuses a
    symlinked destination present when the write begins with ELOOP (its
    ``is_symlink()`` check) before moving that file into place, on every
    platform; the verified writer's O_NOFOLLOW never sees the destination.
    This protects against a link present at the check, not against one
    planted after it: the move then replaces the link itself and the
    download records success, but it never writes through a link, so the
    link's target is never touched. The outside canary must survive, and
    legitimate in-root saves remain covered by the positive controls."""

    def test_symlink_inside_pdf_subdir_does_not_let_write_escape(
        self,
        fs_storage,
        fresh_document,
        library_session,
        tmp_path,
        canary_paths,
    ):
        library_root = fs_storage.library_root
        target_name = "evil_link_target.pdf"

        # Real canary file outside the library root.
        outside_target = tmp_path / "outside_canary_zone" / target_name
        outside_target.write_text("UNTOUCHED-CANARY\n", encoding="utf-8")
        canary_paths["symlink_target"] = outside_target

        # Discover the real destination with a successful save. New writes
        # append a content hash, so planting the unhashed display name would
        # leave the canary untouched without exercising the symlink guard.
        saved_relative, _ = fs_storage.save_pdf(
            _MINIMAL_PDF, fresh_document, library_session, target_name
        )
        link = library_root / saved_relative
        assert link.read_bytes() == _MINIMAL_PDF
        link.unlink()
        try:
            link.symlink_to(outside_target)
        except (OSError, NotImplementedError):
            pytest.skip("Cannot create symlinks in this environment")
        assert link.resolve() == outside_target.resolve()

        fresh_document.storage_mode = None
        fresh_document.file_path = None

        try:
            relative, _size = fs_storage.save_pdf(
                pdf_content=_MINIMAL_PDF,
                document=fresh_document,
                session=library_session,
                filename=target_name,
            )
        except (ValueError, OSError):
            _assert_files_inside(library_root, None, canary_paths)
            return

        assert relative == saved_relative
        assert (
            outside_target.read_text(encoding="utf-8") == "UNTOUCHED-CANARY\n"
        ), (
            f"VULNERABILITY: save_pdf wrote through symlink and overwrote "
            f"outside-root canary at {outside_target}; relative returned "
            f"was {relative!r}"
        )
        _assert_files_inside(library_root, relative, canary_paths)


# ---------------------------------------------------------------------------
# Happy path — clean name saves inside pdfs/ — guards against over-blocking
# ---------------------------------------------------------------------------


class TestHappyPath:
    def test_clean_name_saves_inside_pdfs(
        self,
        fs_storage,
        fresh_document,
        library_session,
        canary_paths,
    ):
        relative, _size = fs_storage.save_pdf(
            pdf_content=_MINIMAL_PDF,
            document=fresh_document,
            session=library_session,
            filename="clean_paper.pdf",
            url="http://example.com/clean_paper.pdf",
            resource_id=1,
        )

        written = fs_storage.library_root / relative
        assert written.is_file()
        # Must be inside library_root/pdfs/ — the intended location.
        assert written.resolve().is_relative_to(
            (fs_storage.library_root / "pdfs").resolve()
        ), f"clean file landed at unexpected path: {written}"
        assert written.read_bytes() == _MINIMAL_PDF
        assert fresh_document.file_path == relative
        assert fresh_document.storage_mode == "filesystem"
        _assert_files_inside(fs_storage.library_root, relative, canary_paths)


# ---------------------------------------------------------------------------
# Zotero sanitize_filename battery — remote-controlled hostile names
# ---------------------------------------------------------------------------


class TestZoteroFilenameSanitizer:
    """The Zotero sync_service (_filename_for in sync_service.py) builds a
    candidate display filename from the remote item's title and feeds it
    through sanitize_filename before storing on Document.filename.  Drives
    sanitize_filename with the same shapes of remote payloads an
    attacker-controlled Zotero response could carry, and asserts the
    sanitized output:
      * never contains a path separator
      * never begins with '..'
      * raises UnsafeFilenameError when nothing safe survives
    """

    CASES = [
        pytest.param("../../etc/passwd", None, id="dotdot-passes"),
        pytest.param("evil\x00name", None, id="null-byte"),
        pytest.param("", None, id="empty-raises"),
    ]

    @pytest.mark.parametrize("raw,exts", CASES)
    def test_sanitize_blocks_traversal_in_zotero_titles(self, raw, exts):
        try:
            safe = sanitize_filename(raw, allowed_extensions=exts)
        except UnsafeFilenameError:
            return

        assert "/" not in safe, (
            f"sanitize_filename returned a path-containing value: {safe!r}"
        )
        assert "\\" not in safe, (
            f"sanitize_filename returned a backslash-containing value: {safe!r}"
        )
        assert not safe.startswith(".."), (
            f"sanitize_filename returned a traversal-leading value: {safe!r}"
        )
        assert "\x00" not in safe, (
            f"sanitize_filename did not strip null bytes: {safe!r}"
        )