~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
from __future__ import annotations
import gzip
import json
from pathlib import Path
from types import SimpleNamespace
import pytest
from local_deep_research.config.paths import get_journal_data_directory
from local_deep_research.journal_quality import downloader
from local_deep_research.journal_quality.data_sources import _openalex_common
from local_deep_research.journal_quality.data_sources.doaj import DOAJSource
from local_deep_research.journal_quality.data_sources.institutions import (
InstitutionSource,
)
from local_deep_research.journal_quality.data_sources.jabref import JabRefSource
from local_deep_research.journal_quality.data_sources.openalex import (
OpenAlexSource,
)
from local_deep_research.journal_quality.data_sources.predatory import (
PredatorySource,
)
from local_deep_research.journal_quality.db import (
DB_FILENAME,
JournalQualityDB,
build_db,
)
CANARY = b"OUTSIDE-CANARY-MUST-SURVIVE"
class _Response:
def __init__(
self, content: bytes, payload: dict[str, object] | None = None
):
self.content = content
self.text = content.decode("utf-8", errors="replace")
self._payload = payload
def raise_for_status(self) -> None:
return None
def json(self) -> dict[str, object]:
assert self._payload is not None
return self._payload
@pytest.fixture
def journal_dir(tmp_path, monkeypatch) -> Path:
root = tmp_path / "ldr-root"
monkeypatch.setenv("LDR_DATA_DIR", str(root))
return get_journal_data_directory()
@pytest.fixture
def outside_canary(tmp_path) -> Path:
path = tmp_path / "outside-canary"
path.write_bytes(CANARY)
return path
def _assert_contained(data_dir: Path, outside_canary: Path) -> None:
root = data_dir.resolve()
for path in data_dir.rglob("*"):
assert path.resolve().is_relative_to(root), (
f"VULNERABILITY: escaped write {path}"
)
assert outside_canary.read_bytes() == CANARY, (
"VULNERABILITY: outside canary changed"
)
def test_ldr_data_dir_resolves_journal_writes_under_override(
journal_dir, outside_canary
):
assert journal_dir == journal_dir.parent / "journal_data"
assert journal_dir.resolve().is_relative_to(journal_dir.parent.resolve())
_assert_contained(journal_dir, outside_canary)
def test_openalex_partition_tmp_is_contained_and_cleaned(
journal_dir, outside_canary
):
compressed = gzip.compress(b'{"id":"https://openalex.org/S1"}\n')
parts = _openalex_common.iter_partitions(
[{"url": "s3://openalex/data/part.gz"}],
journal_dir,
file_prefix="openalex_sources",
label="test",
safe_get=lambda *_args, **_kwargs: _Response(compressed),
max_retries=0,
backoff_times=(),
)
_, _, records = next(parts)
assert list(records) == [{"id": "https://openalex.org/S1"}]
with pytest.raises(StopIteration):
next(parts)
assert not list(journal_dir.glob(".openalex_sources_part_*.gz"))
_assert_contained(journal_dir, outside_canary)
def test_openalex_partition_failure_preserves_existing_dataset_and_cleans_tmp(
journal_dir, outside_canary, monkeypatch
):
final = journal_dir / "openalex_sources.json.gz"
final.write_bytes(b"OLD-DATASET")
original_write_bytes = Path.write_bytes
def partial_write_then_fail(path: Path, data: bytes) -> int:
if path.name == ".openalex_sources_part_0.gz":
original_write_bytes(path, data)
raise OSError("mid-download write failure")
return original_write_bytes(path, data)
monkeypatch.setattr(Path, "write_bytes", partial_write_then_fail)
parts = _openalex_common.iter_partitions(
[{"url": "s3://openalex/data/part.gz"}],
journal_dir,
file_prefix="openalex_sources",
label="test",
safe_get=lambda *_args, **_kwargs: _Response(
gzip.compress(b'{"id":"https://openalex.org/S1"}\n')
),
max_retries=0,
backoff_times=(),
)
with pytest.raises(OSError, match="mid-download write failure"):
next(parts)
assert final.read_bytes() == b"OLD-DATASET"
assert not list(journal_dir.glob(".openalex_sources_part_*.gz"))
_assert_contained(journal_dir, outside_canary)
def test_db_atomic_build_replaces_only_the_contained_target(
journal_dir, outside_canary
):
(journal_dir / "openalex_sources.json.gz").write_bytes(
gzip.compress(b'{"s": {}}')
)
target = journal_dir / DB_FILENAME
target.write_bytes(b"OLD-DB")
build_db(data_dir=journal_dir, output_path=target)
assert target.read_bytes().startswith(b"SQLite format 3")
assert not list(journal_dir.glob(f"{DB_FILENAME}.tmp-*"))
_assert_contained(journal_dir, outside_canary)
def test_doaj_atomic_replace_stays_contained(
journal_dir, outside_canary, monkeypatch
):
source = DOAJSource()
target = journal_dir / source.filename
target.write_bytes(b"OLD-DOAJ")
monkeypatch.setattr(
"local_deep_research.journal_quality.data_sources.doaj._MIN_DOAJ_JOURNALS",
1,
)
csv_bytes = b"Journal title,Journal ISSN (print version),Journal EISSN (online version),Publisher\nJ,1234-5678,,P\n"
monkeypatch.setattr(
"local_deep_research.security.safe_requests.safe_get_with_retries",
lambda *_args, **_kwargs: _Response(csv_bytes),
)
assert source.fetch(journal_dir) == 1
assert (
json.loads(target.read_text(encoding="utf-8"))["journals"]["12345678"][
"name"
]
== "J"
)
assert not (journal_dir / f"{source.filename}.tmp").exists()
_assert_contained(journal_dir, outside_canary)
def test_predatory_atomic_replace_stays_contained(
journal_dir, outside_canary, monkeypatch
):
source = PredatorySource()
target = journal_dir / source.filename
target.write_bytes(b"OLD-PREDATORY")
monkeypatch.setattr(
"local_deep_research.journal_quality.data_sources.predatory._MIN_PREDATORY_TOTAL",
1,
)
payloads = iter(
(
b"name,url\nPublisher,https://p\n",
b"name,url\nJournal,https://j\n",
b"hijacked,authentic,hijackedurl,authenticurl\n",
)
)
monkeypatch.setattr(
"local_deep_research.security.safe_requests.safe_get_with_retries",
lambda *_args, **_kwargs: _Response(next(payloads)),
)
assert source.fetch(journal_dir) == 2
assert (
json.loads(target.read_text(encoding="utf-8"))["publishers"][0]["name"]
== "Publisher"
)
assert not (journal_dir / f"{source.filename}.tmp").exists()
_assert_contained(journal_dir, outside_canary)
def test_jabref_atomic_replace_stays_contained(
journal_dir, outside_canary, monkeypatch
):
source = JabRefSource()
target = journal_dir / source.filename
target.write_bytes(b"OLD-JABREF")
monkeypatch.setattr(
"local_deep_research.journal_quality.data_sources.jabref._JABREF_FILES",
["fixture.csv"],
)
monkeypatch.setattr(
"local_deep_research.security.safe_requests.safe_get_with_retries",
lambda *_args, **_kwargs: _Response(
b"\n".join(
f"Journal {index},J{index}".encode() for index in range(100)
)
+ b"\n"
),
)
assert source.fetch(journal_dir) == 100
with gzip.open(target, "rt", encoding="utf-8") as handle:
assert handle.read()
assert not (journal_dir / f"{source.filename}.tmp").exists()
_assert_contained(journal_dir, outside_canary)
@pytest.mark.parametrize(
("source_type", "module_path", "filename"),
[
(OpenAlexSource, "openalex", "openalex_sources.json.gz"),
(InstitutionSource, "institutions", "openalex_institutions.json.gz"),
],
)
def test_openalex_final_dataset_atomic_replace_stays_contained(
journal_dir,
outside_canary,
monkeypatch,
source_type,
module_path,
filename,
):
target = journal_dir / filename
target.write_bytes(b"OLD-OPENALEX")
module = __import__(
f"local_deep_research.journal_quality.data_sources.{module_path}",
fromlist=["x"],
)
records = (
iter(
{
"id": f"https://openalex.org/S{index}",
"display_name": f"Source {index}",
}
for index in range(10_000)
)
if source_type is OpenAlexSource
else iter(
[{"id": "https://openalex.org/I1", "display_name": "Institution"}]
)
)
if source_type is InstitutionSource:
monkeypatch.setattr(module, "_MIN_INSTITUTIONS", 1)
monkeypatch.setattr(
module,
"iter_partitions",
lambda *_args, **_kwargs: iter([(0, 1, records)]),
)
monkeypatch.setattr(
"local_deep_research.security.safe_requests.safe_get_with_retries",
lambda *_args, **_kwargs: _Response(b"{}", {"files": []}),
)
assert source_type().fetch(journal_dir) == (
10_000 if source_type is OpenAlexSource else 1
)
assert target.read_bytes() != b"OLD-OPENALEX"
assert not (journal_dir / f"{filename}.tmp").exists()
_assert_contained(journal_dir, outside_canary)
def test_sentinel_uses_exclusive_creation_and_is_removed_on_success(
journal_dir, outside_canary, monkeypatch
):
calls: list[tuple[Path, str]] = []
original_open = Path.open
def tracking_open(path: Path, mode="r", *args, **kwargs):
calls.append((path, mode))
return original_open(path, mode, *args, **kwargs)
monkeypatch.setattr(Path, "open", tracking_open)
monkeypatch.setattr(downloader, "ALL_SOURCES", [])
monkeypatch.setattr(
"shutil.disk_usage", lambda _path: SimpleNamespace(free=10**15)
)
monkeypatch.setattr(
"local_deep_research.journal_quality.db.build_db",
lambda **_kwargs: None,
)
success, _ = downloader.download_journal_data(force=True)
assert success
assert (journal_dir / "version.json").is_file()
assert (journal_dir / ".downloading", "x") in calls
assert not (journal_dir / ".downloading").exists()
_assert_contained(journal_dir, outside_canary)
def test_sentinel_is_removed_after_required_source_failure(
journal_dir, outside_canary, monkeypatch
):
class FailingSource:
key = "required"
name = "required"
count_label = "items"
required = True
def fetch(self, _data_dir, progress_cb=None):
raise RuntimeError("offline canary failure")
monkeypatch.setattr(downloader, "ALL_SOURCES", [FailingSource()])
monkeypatch.setattr(
"shutil.disk_usage", lambda _path: SimpleNamespace(free=10**15)
)
success, message = downloader.download_journal_data(force=True)
assert not success
assert "Failed to fetch required" in message
assert not (journal_dir / ".downloading").exists()
_assert_contained(journal_dir, outside_canary)
def test_stale_sentinel_contents_cannot_redirect_download_writes(
journal_dir, outside_canary, monkeypatch
):
(journal_dir / ".downloading").write_text(
str(outside_canary), encoding="utf-8"
)
class WritingSource:
key = "required"
name = "required"
count_label = "items"
required = True
def fetch(self, data_dir, progress_cb=None):
(data_dir / "fixed_dataset.json").write_bytes(b"INSIDE")
return 1
monkeypatch.setattr(downloader, "ALL_SOURCES", [WritingSource()])
monkeypatch.setattr(
"shutil.disk_usage", lambda _path: SimpleNamespace(free=10**15)
)
monkeypatch.setattr(
"local_deep_research.journal_quality.db.build_db",
lambda **_kwargs: None,
)
success, _ = downloader.download_journal_data(force=True)
assert success
assert (journal_dir / "fixed_dataset.json").read_bytes() == b"INSIDE"
assert not (journal_dir / ".downloading").exists()
_assert_contained(journal_dir, outside_canary)
def test_unusable_db_cleanup_does_not_unlink_outside_canary(
journal_dir, outside_canary
):
corrupt_db = journal_dir / "journal_quality.db"
corrupt_db.write_bytes(b"NOT-A-DB")
JournalQualityDB._unlink_unusable_db(corrupt_db)
assert not corrupt_db.exists()
_assert_contained(journal_dir, outside_canary)