~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
from __future__ import annotations
from io import BytesIO
from pathlib import Path
import pytest
from local_deep_research.research_library.downloaders.arxiv import (
ArxivDownloader,
)
from local_deep_research.research_library.downloaders.biorxiv import (
BioRxivDownloader,
)
from local_deep_research.research_library.downloaders.direct_pdf import (
DirectPDFDownloader,
)
from local_deep_research.research_library.downloaders.generic import (
GenericDownloader,
)
from local_deep_research.research_library.downloaders.html import HTMLDownloader
from local_deep_research.research_library.downloaders.openalex import (
OpenAlexDownloader,
)
from local_deep_research.research_library.downloaders.playwright_html import (
PlaywrightHTMLDownloader,
)
from local_deep_research.research_library.downloaders.pubmed import (
PubMedDownloader,
)
from local_deep_research.research_library.downloaders.semantic_scholar import (
SemanticScholarDownloader,
)
from local_deep_research.research_library.services.pdf_storage_manager import (
PDFStorageManager,
)
from local_deep_research.security.filename_sanitizer import sanitize_filename
from local_deep_research.security.path_validator import PathValidator
_PDF_BYTES = b"%PDF-1.4\n1 0 obj <<>>endobj\n%%EOF\n"
_HOSTILE_CONTENT_DISPOSITION = 'attachment; filename="../../outside.pdf"'
class _OfflineResponse:
def __init__(self, *, content=_PDF_BYTES, payload=None, text=""):
self.status_code = 200
self.content = content
self.text = text
self.headers = {
"content-type": "application/pdf",
"content-disposition": _HOSTILE_CONTENT_DISPOSITION,
}
self._payload = payload or {}
def json(self):
return self._payload
class _OfflineSession:
def __init__(self):
self.headers = {}
def get(self, url, **_kwargs):
if "api.semanticscholar.org" in url:
return _OfflineResponse(
payload={
"openAccessPdf": {"url": "https://files.test/paper.pdf"}
}
)
if "api.openalex.org" in url:
return _OfflineResponse(
payload={
"open_access": {"is_oa": True},
"best_oa_location": {
"pdf_url": "https://files.test/paper.pdf"
},
}
)
return _OfflineResponse()
def close(self):
pass
def _snapshot(root: Path) -> set[Path]:
return {path.relative_to(root) for path in root.rglob("*")}
def test_notes_create_with_hostile_title_creates_no_attachment_file(
authenticated_client, temp_data_dir, tmp_path
):
outside_canary = tmp_path / "notes-attachment-canary.pdf"
outside_canary.write_bytes(b"unchanged")
response = authenticated_client.post(
"/notes/api/notes",
json={
"title": "../../notes-attachment-canary.pdf",
"content": "note body",
},
)
assert response.status_code == 201
assert response.get_json()["success"] is True
assert outside_canary.read_bytes() == b"unchanged"
assert not list(temp_data_dir.rglob("*.pdf"))
def test_authenticated_pdf_upload_sanitizes_filename_before_extraction(
authenticated_client, monkeypatch, tmp_path
):
# ``research_routes`` (Flask) was deleted by the FastAPI migration
# (#3299); ``/api/upload/pdf`` and both symbols patched below now live
# in ``web.routers.research``.
from local_deep_research.web.routers import research as research_routes
captured_filenames = []
class _ExtractionService:
def extract_text_and_metadata(self, content, filename):
captured_filenames.append(filename)
assert content == _PDF_BYTES
return {
"success": True,
"filename": filename,
"text": "offline text",
"size": len(content),
"pages": 1,
}
outside_canary = tmp_path / "upload-canary.pdf"
outside_canary.write_bytes(b"unchanged")
monkeypatch.setattr(
research_routes,
"get_pdf_extraction_service",
lambda: _ExtractionService(),
)
monkeypatch.setattr(
research_routes.FileUploadValidator,
"validate_upload",
lambda **_kwargs: (True, ""),
)
response = authenticated_client.post(
"/api/upload/pdf",
data={"files": (BytesIO(_PDF_BYTES), "../../upload-canary.pdf")},
content_type="multipart/form-data",
)
assert response.status_code == 200
assert captured_filenames == ["upload-canary.pdf"]
assert outside_canary.read_bytes() == b"unchanged"
@pytest.mark.parametrize(
("url", "expected_filename"),
[
(
"https://arxiv.org/abs/2401.12345/../../outside",
"42.pdf",
),
(
"https://ncbi.nlm.nih.gov/pmc/articles/PMC12345/../../outside",
"pmc_PMC12345.pdf",
),
("https://hostile.example/../../outside.pdf", "42.pdf"),
],
)
def test_download_service_filename_resolution_stays_validated_under_library_root(
tmp_path, url, expected_filename
):
manager = PDFStorageManager(tmp_path / "library", storage_mode="filesystem")
filename = manager._generate_filename(
url, resource_id=42, fallback_filename="42.pdf"
)
validated = PathValidator.validate_safe_path(
f"pdfs/{filename}",
base_dir=str(manager.library_root),
required_extensions=(".pdf",),
)
assert filename == expected_filename
assert sanitize_filename(filename, allowed_extensions={".pdf"}) == filename
assert (
Path(validated).resolve().is_relative_to(manager.library_root.resolve())
)
@pytest.mark.parametrize(
("downloader_class", "url"),
[
(ArxivDownloader, "https://arxiv.org/abs/2401.12345"),
(PubMedDownloader, "https://pubmed.ncbi.nlm.nih.gov/12345/"),
(
BioRxivDownloader,
"https://www.biorxiv.org/content/10.1101/2024.01.01",
),
(
SemanticScholarDownloader,
"https://www.semanticscholar.org/paper/"
"0123456789abcdef0123456789abcdef01234567",
),
(OpenAlexDownloader, "https://openalex.org/W123456789"),
(DirectPDFDownloader, "https://files.test/../../outside.pdf"),
(GenericDownloader, "https://files.test/../../outside.pdf"),
],
)
def test_pdf_downloaders_return_bytes_without_writing_remote_filename(
tmp_path, monkeypatch, downloader_class, url
):
monkeypatch.chdir(tmp_path)
downloader = downloader_class()
downloader.session = _OfflineSession()
if downloader_class is PubMedDownloader:
monkeypatch.setattr(
downloader, "_download_pdf_content", lambda _url: _PDF_BYTES
)
before = _snapshot(tmp_path)
result = downloader.download(url)
assert result == _PDF_BYTES
assert _snapshot(tmp_path) == before
downloader.close()
@pytest.mark.parametrize(
"downloader_class", [HTMLDownloader, PlaywrightHTMLDownloader]
)
def test_html_downloaders_extract_in_memory_without_file_writes(
tmp_path, monkeypatch, downloader_class
):
monkeypatch.chdir(tmp_path)
downloader = downloader_class()
monkeypatch.setattr(downloader, "_fetch_html", lambda _url: "<html></html>")
monkeypatch.setattr(
downloader,
"_extract_content",
lambda _html, url: {
"title": "t",
"description": None,
"content": "body",
"url": url,
},
)
before = _snapshot(tmp_path)
result = downloader.download("https://hostile.example/../../outside.html")
assert result is not None
assert _snapshot(tmp_path) == before
downloader.close()