Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

7.7 KB
Raw Blame
from __future__ import annotations

from io import BytesIO
from pathlib import Path

import pytest

from local_deep_research.research_library.downloaders.arxiv import (
    ArxivDownloader,
)
from local_deep_research.research_library.downloaders.biorxiv import (
    BioRxivDownloader,
)
from local_deep_research.research_library.downloaders.direct_pdf import (
    DirectPDFDownloader,
)
from local_deep_research.research_library.downloaders.generic import (
    GenericDownloader,
)
from local_deep_research.research_library.downloaders.html import HTMLDownloader
from local_deep_research.research_library.downloaders.openalex import (
    OpenAlexDownloader,
)
from local_deep_research.research_library.downloaders.playwright_html import (
    PlaywrightHTMLDownloader,
)
from local_deep_research.research_library.downloaders.pubmed import (
    PubMedDownloader,
)
from local_deep_research.research_library.downloaders.semantic_scholar import (
    SemanticScholarDownloader,
)
from local_deep_research.research_library.services.pdf_storage_manager import (
    PDFStorageManager,
)
from local_deep_research.security.filename_sanitizer import sanitize_filename
from local_deep_research.security.path_validator import PathValidator


_PDF_BYTES = b"%PDF-1.4\n1 0 obj <<>>endobj\n%%EOF\n"
_HOSTILE_CONTENT_DISPOSITION = 'attachment; filename="../../outside.pdf"'


class _OfflineResponse:
    def __init__(self, *, content=_PDF_BYTES, payload=None, text=""):
        self.status_code = 200
        self.content = content
        self.text = text
        self.headers = {
            "content-type": "application/pdf",
            "content-disposition": _HOSTILE_CONTENT_DISPOSITION,
        }
        self._payload = payload or {}

    def json(self):
        return self._payload


class _OfflineSession:
    def __init__(self):
        self.headers = {}

    def get(self, url, **_kwargs):
        if "api.semanticscholar.org" in url:
            return _OfflineResponse(
                payload={
                    "openAccessPdf": {"url": "https://files.test/paper.pdf"}
                }
            )
        if "api.openalex.org" in url:
            return _OfflineResponse(
                payload={
                    "open_access": {"is_oa": True},
                    "best_oa_location": {
                        "pdf_url": "https://files.test/paper.pdf"
                    },
                }
            )
        return _OfflineResponse()

    def close(self):
        pass


def _snapshot(root: Path) -> set[Path]:
    return {path.relative_to(root) for path in root.rglob("*")}


def test_notes_create_with_hostile_title_creates_no_attachment_file(
    authenticated_client, temp_data_dir, tmp_path
):
    outside_canary = tmp_path / "notes-attachment-canary.pdf"
    outside_canary.write_bytes(b"unchanged")

    response = authenticated_client.post(
        "/notes/api/notes",
        json={
            "title": "../../notes-attachment-canary.pdf",
            "content": "note body",
        },
    )

    assert response.status_code == 201
    assert response.get_json()["success"] is True
    assert outside_canary.read_bytes() == b"unchanged"
    assert not list(temp_data_dir.rglob("*.pdf"))


def test_authenticated_pdf_upload_sanitizes_filename_before_extraction(
    authenticated_client, monkeypatch, tmp_path
):
    # ``research_routes`` (Flask) was deleted by the FastAPI migration
    # (#3299); ``/api/upload/pdf`` and both symbols patched below now live
    # in ``web.routers.research``.
    from local_deep_research.web.routers import research as research_routes

    captured_filenames = []

    class _ExtractionService:
        def extract_text_and_metadata(self, content, filename):
            captured_filenames.append(filename)
            assert content == _PDF_BYTES
            return {
                "success": True,
                "filename": filename,
                "text": "offline text",
                "size": len(content),
                "pages": 1,
            }

    outside_canary = tmp_path / "upload-canary.pdf"
    outside_canary.write_bytes(b"unchanged")
    monkeypatch.setattr(
        research_routes,
        "get_pdf_extraction_service",
        lambda: _ExtractionService(),
    )
    monkeypatch.setattr(
        research_routes.FileUploadValidator,
        "validate_upload",
        lambda **_kwargs: (True, ""),
    )

    response = authenticated_client.post(
        "/api/upload/pdf",
        data={"files": (BytesIO(_PDF_BYTES), "../../upload-canary.pdf")},
        content_type="multipart/form-data",
    )

    assert response.status_code == 200
    assert captured_filenames == ["upload-canary.pdf"]
    assert outside_canary.read_bytes() == b"unchanged"


@pytest.mark.parametrize(
    ("url", "expected_filename"),
    [
        (
            "https://arxiv.org/abs/2401.12345/../../outside",
            "42.pdf",
        ),
        (
            "https://ncbi.nlm.nih.gov/pmc/articles/PMC12345/../../outside",
            "pmc_PMC12345.pdf",
        ),
        ("https://hostile.example/../../outside.pdf", "42.pdf"),
    ],
)
def test_download_service_filename_resolution_stays_validated_under_library_root(
    tmp_path, url, expected_filename
):
    manager = PDFStorageManager(tmp_path / "library", storage_mode="filesystem")

    filename = manager._generate_filename(
        url, resource_id=42, fallback_filename="42.pdf"
    )
    validated = PathValidator.validate_safe_path(
        f"pdfs/{filename}",
        base_dir=str(manager.library_root),
        required_extensions=(".pdf",),
    )

    assert filename == expected_filename
    assert sanitize_filename(filename, allowed_extensions={".pdf"}) == filename
    assert (
        Path(validated).resolve().is_relative_to(manager.library_root.resolve())
    )


@pytest.mark.parametrize(
    ("downloader_class", "url"),
    [
        (ArxivDownloader, "https://arxiv.org/abs/2401.12345"),
        (PubMedDownloader, "https://pubmed.ncbi.nlm.nih.gov/12345/"),
        (
            BioRxivDownloader,
            "https://www.biorxiv.org/content/10.1101/2024.01.01",
        ),
        (
            SemanticScholarDownloader,
            "https://www.semanticscholar.org/paper/"
            "0123456789abcdef0123456789abcdef01234567",
        ),
        (OpenAlexDownloader, "https://openalex.org/W123456789"),
        (DirectPDFDownloader, "https://files.test/../../outside.pdf"),
        (GenericDownloader, "https://files.test/../../outside.pdf"),
    ],
)
def test_pdf_downloaders_return_bytes_without_writing_remote_filename(
    tmp_path, monkeypatch, downloader_class, url
):
    monkeypatch.chdir(tmp_path)
    downloader = downloader_class()
    downloader.session = _OfflineSession()
    if downloader_class is PubMedDownloader:
        monkeypatch.setattr(
            downloader, "_download_pdf_content", lambda _url: _PDF_BYTES
        )
    before = _snapshot(tmp_path)

    result = downloader.download(url)

    assert result == _PDF_BYTES
    assert _snapshot(tmp_path) == before
    downloader.close()


@pytest.mark.parametrize(
    "downloader_class", [HTMLDownloader, PlaywrightHTMLDownloader]
)
def test_html_downloaders_extract_in_memory_without_file_writes(
    tmp_path, monkeypatch, downloader_class
):
    monkeypatch.chdir(tmp_path)
    downloader = downloader_class()
    monkeypatch.setattr(downloader, "_fetch_html", lambda _url: "<html></html>")
    monkeypatch.setattr(
        downloader,
        "_extract_content",
        lambda _html, url: {
            "title": "t",
            "description": None,
            "content": "body",
            "url": url,
        },
    )
    before = _snapshot(tmp_path)

    result = downloader.download("https://hostile.example/../../outside.html")

    assert result is not None
    assert _snapshot(tmp_path) == before
    downloader.close()