Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

12.1 KB
Raw Blame
from __future__ import annotations

import gzip
import json
from pathlib import Path
from types import SimpleNamespace

import pytest

from local_deep_research.config.paths import get_journal_data_directory
from local_deep_research.journal_quality import downloader
from local_deep_research.journal_quality.data_sources import _openalex_common
from local_deep_research.journal_quality.data_sources.doaj import DOAJSource
from local_deep_research.journal_quality.data_sources.institutions import (
    InstitutionSource,
)
from local_deep_research.journal_quality.data_sources.jabref import JabRefSource
from local_deep_research.journal_quality.data_sources.openalex import (
    OpenAlexSource,
)
from local_deep_research.journal_quality.data_sources.predatory import (
    PredatorySource,
)
from local_deep_research.journal_quality.db import (
    DB_FILENAME,
    JournalQualityDB,
    build_db,
)


CANARY = b"OUTSIDE-CANARY-MUST-SURVIVE"


class _Response:
    def __init__(
        self, content: bytes, payload: dict[str, object] | None = None
    ):
        self.content = content
        self.text = content.decode("utf-8", errors="replace")
        self._payload = payload

    def raise_for_status(self) -> None:
        return None

    def json(self) -> dict[str, object]:
        assert self._payload is not None
        return self._payload


@pytest.fixture
def journal_dir(tmp_path, monkeypatch) -> Path:
    root = tmp_path / "ldr-root"
    monkeypatch.setenv("LDR_DATA_DIR", str(root))
    return get_journal_data_directory()


@pytest.fixture
def outside_canary(tmp_path) -> Path:
    path = tmp_path / "outside-canary"
    path.write_bytes(CANARY)
    return path


def _assert_contained(data_dir: Path, outside_canary: Path) -> None:
    root = data_dir.resolve()
    for path in data_dir.rglob("*"):
        assert path.resolve().is_relative_to(root), (
            f"VULNERABILITY: escaped write {path}"
        )
    assert outside_canary.read_bytes() == CANARY, (
        "VULNERABILITY: outside canary changed"
    )


def test_ldr_data_dir_resolves_journal_writes_under_override(
    journal_dir, outside_canary
):
    assert journal_dir == journal_dir.parent / "journal_data"
    assert journal_dir.resolve().is_relative_to(journal_dir.parent.resolve())
    _assert_contained(journal_dir, outside_canary)


def test_openalex_partition_tmp_is_contained_and_cleaned(
    journal_dir, outside_canary
):
    compressed = gzip.compress(b'{"id":"https://openalex.org/S1"}\n')

    parts = _openalex_common.iter_partitions(
        [{"url": "s3://openalex/data/part.gz"}],
        journal_dir,
        file_prefix="openalex_sources",
        label="test",
        safe_get=lambda *_args, **_kwargs: _Response(compressed),
        max_retries=0,
        backoff_times=(),
    )
    _, _, records = next(parts)
    assert list(records) == [{"id": "https://openalex.org/S1"}]
    with pytest.raises(StopIteration):
        next(parts)

    assert not list(journal_dir.glob(".openalex_sources_part_*.gz"))
    _assert_contained(journal_dir, outside_canary)


def test_openalex_partition_failure_preserves_existing_dataset_and_cleans_tmp(
    journal_dir, outside_canary, monkeypatch
):
    final = journal_dir / "openalex_sources.json.gz"
    final.write_bytes(b"OLD-DATASET")

    original_write_bytes = Path.write_bytes

    def partial_write_then_fail(path: Path, data: bytes) -> int:
        if path.name == ".openalex_sources_part_0.gz":
            original_write_bytes(path, data)
            raise OSError("mid-download write failure")
        return original_write_bytes(path, data)

    monkeypatch.setattr(Path, "write_bytes", partial_write_then_fail)

    parts = _openalex_common.iter_partitions(
        [{"url": "s3://openalex/data/part.gz"}],
        journal_dir,
        file_prefix="openalex_sources",
        label="test",
        safe_get=lambda *_args, **_kwargs: _Response(
            gzip.compress(b'{"id":"https://openalex.org/S1"}\n')
        ),
        max_retries=0,
        backoff_times=(),
    )
    with pytest.raises(OSError, match="mid-download write failure"):
        next(parts)

    assert final.read_bytes() == b"OLD-DATASET"
    assert not list(journal_dir.glob(".openalex_sources_part_*.gz"))
    _assert_contained(journal_dir, outside_canary)


def test_db_atomic_build_replaces_only_the_contained_target(
    journal_dir, outside_canary
):
    (journal_dir / "openalex_sources.json.gz").write_bytes(
        gzip.compress(b'{"s": {}}')
    )
    target = journal_dir / DB_FILENAME
    target.write_bytes(b"OLD-DB")

    build_db(data_dir=journal_dir, output_path=target)

    assert target.read_bytes().startswith(b"SQLite format 3")
    assert not list(journal_dir.glob(f"{DB_FILENAME}.tmp-*"))
    _assert_contained(journal_dir, outside_canary)


def test_doaj_atomic_replace_stays_contained(
    journal_dir, outside_canary, monkeypatch
):
    source = DOAJSource()
    target = journal_dir / source.filename
    target.write_bytes(b"OLD-DOAJ")
    monkeypatch.setattr(
        "local_deep_research.journal_quality.data_sources.doaj._MIN_DOAJ_JOURNALS",
        1,
    )
    csv_bytes = b"Journal title,Journal ISSN (print version),Journal EISSN (online version),Publisher\nJ,1234-5678,,P\n"
    monkeypatch.setattr(
        "local_deep_research.security.safe_requests.safe_get_with_retries",
        lambda *_args, **_kwargs: _Response(csv_bytes),
    )

    assert source.fetch(journal_dir) == 1
    assert (
        json.loads(target.read_text(encoding="utf-8"))["journals"]["12345678"][
            "name"
        ]
        == "J"
    )
    assert not (journal_dir / f"{source.filename}.tmp").exists()
    _assert_contained(journal_dir, outside_canary)


def test_predatory_atomic_replace_stays_contained(
    journal_dir, outside_canary, monkeypatch
):
    source = PredatorySource()
    target = journal_dir / source.filename
    target.write_bytes(b"OLD-PREDATORY")
    monkeypatch.setattr(
        "local_deep_research.journal_quality.data_sources.predatory._MIN_PREDATORY_TOTAL",
        1,
    )
    payloads = iter(
        (
            b"name,url\nPublisher,https://p\n",
            b"name,url\nJournal,https://j\n",
            b"hijacked,authentic,hijackedurl,authenticurl\n",
        )
    )
    monkeypatch.setattr(
        "local_deep_research.security.safe_requests.safe_get_with_retries",
        lambda *_args, **_kwargs: _Response(next(payloads)),
    )

    assert source.fetch(journal_dir) == 2
    assert (
        json.loads(target.read_text(encoding="utf-8"))["publishers"][0]["name"]
        == "Publisher"
    )
    assert not (journal_dir / f"{source.filename}.tmp").exists()
    _assert_contained(journal_dir, outside_canary)


def test_jabref_atomic_replace_stays_contained(
    journal_dir, outside_canary, monkeypatch
):
    source = JabRefSource()
    target = journal_dir / source.filename
    target.write_bytes(b"OLD-JABREF")
    monkeypatch.setattr(
        "local_deep_research.journal_quality.data_sources.jabref._JABREF_FILES",
        ["fixture.csv"],
    )
    monkeypatch.setattr(
        "local_deep_research.security.safe_requests.safe_get_with_retries",
        lambda *_args, **_kwargs: _Response(
            b"\n".join(
                f"Journal {index},J{index}".encode() for index in range(100)
            )
            + b"\n"
        ),
    )

    assert source.fetch(journal_dir) == 100
    with gzip.open(target, "rt", encoding="utf-8") as handle:
        assert handle.read()
    assert not (journal_dir / f"{source.filename}.tmp").exists()
    _assert_contained(journal_dir, outside_canary)


@pytest.mark.parametrize(
    ("source_type", "module_path", "filename"),
    [
        (OpenAlexSource, "openalex", "openalex_sources.json.gz"),
        (InstitutionSource, "institutions", "openalex_institutions.json.gz"),
    ],
)
def test_openalex_final_dataset_atomic_replace_stays_contained(
    journal_dir,
    outside_canary,
    monkeypatch,
    source_type,
    module_path,
    filename,
):
    target = journal_dir / filename
    target.write_bytes(b"OLD-OPENALEX")
    module = __import__(
        f"local_deep_research.journal_quality.data_sources.{module_path}",
        fromlist=["x"],
    )
    records = (
        iter(
            {
                "id": f"https://openalex.org/S{index}",
                "display_name": f"Source {index}",
            }
            for index in range(10_000)
        )
        if source_type is OpenAlexSource
        else iter(
            [{"id": "https://openalex.org/I1", "display_name": "Institution"}]
        )
    )
    if source_type is InstitutionSource:
        monkeypatch.setattr(module, "_MIN_INSTITUTIONS", 1)
    monkeypatch.setattr(
        module,
        "iter_partitions",
        lambda *_args, **_kwargs: iter([(0, 1, records)]),
    )
    monkeypatch.setattr(
        "local_deep_research.security.safe_requests.safe_get_with_retries",
        lambda *_args, **_kwargs: _Response(b"{}", {"files": []}),
    )

    assert source_type().fetch(journal_dir) == (
        10_000 if source_type is OpenAlexSource else 1
    )
    assert target.read_bytes() != b"OLD-OPENALEX"
    assert not (journal_dir / f"{filename}.tmp").exists()
    _assert_contained(journal_dir, outside_canary)


def test_sentinel_uses_exclusive_creation_and_is_removed_on_success(
    journal_dir, outside_canary, monkeypatch
):
    calls: list[tuple[Path, str]] = []
    original_open = Path.open

    def tracking_open(path: Path, mode="r", *args, **kwargs):
        calls.append((path, mode))
        return original_open(path, mode, *args, **kwargs)

    monkeypatch.setattr(Path, "open", tracking_open)
    monkeypatch.setattr(downloader, "ALL_SOURCES", [])
    monkeypatch.setattr(
        "shutil.disk_usage", lambda _path: SimpleNamespace(free=10**15)
    )
    monkeypatch.setattr(
        "local_deep_research.journal_quality.db.build_db",
        lambda **_kwargs: None,
    )

    success, _ = downloader.download_journal_data(force=True)
    assert success
    assert (journal_dir / "version.json").is_file()
    assert (journal_dir / ".downloading", "x") in calls
    assert not (journal_dir / ".downloading").exists()
    _assert_contained(journal_dir, outside_canary)


def test_sentinel_is_removed_after_required_source_failure(
    journal_dir, outside_canary, monkeypatch
):
    class FailingSource:
        key = "required"
        name = "required"
        count_label = "items"
        required = True

        def fetch(self, _data_dir, progress_cb=None):
            raise RuntimeError("offline canary failure")

    monkeypatch.setattr(downloader, "ALL_SOURCES", [FailingSource()])
    monkeypatch.setattr(
        "shutil.disk_usage", lambda _path: SimpleNamespace(free=10**15)
    )
    success, message = downloader.download_journal_data(force=True)

    assert not success
    assert "Failed to fetch required" in message
    assert not (journal_dir / ".downloading").exists()
    _assert_contained(journal_dir, outside_canary)


def test_stale_sentinel_contents_cannot_redirect_download_writes(
    journal_dir, outside_canary, monkeypatch
):
    (journal_dir / ".downloading").write_text(
        str(outside_canary), encoding="utf-8"
    )

    class WritingSource:
        key = "required"
        name = "required"
        count_label = "items"
        required = True

        def fetch(self, data_dir, progress_cb=None):
            (data_dir / "fixed_dataset.json").write_bytes(b"INSIDE")
            return 1

    monkeypatch.setattr(downloader, "ALL_SOURCES", [WritingSource()])
    monkeypatch.setattr(
        "shutil.disk_usage", lambda _path: SimpleNamespace(free=10**15)
    )
    monkeypatch.setattr(
        "local_deep_research.journal_quality.db.build_db",
        lambda **_kwargs: None,
    )

    success, _ = downloader.download_journal_data(force=True)
    assert success
    assert (journal_dir / "fixed_dataset.json").read_bytes() == b"INSIDE"
    assert not (journal_dir / ".downloading").exists()
    _assert_contained(journal_dir, outside_canary)


def test_unusable_db_cleanup_does_not_unlink_outside_canary(
    journal_dir, outside_canary
):
    corrupt_db = journal_dir / "journal_quality.db"
    corrupt_db.write_bytes(b"NOT-A-DB")

    JournalQualityDB._unlink_unusable_db(corrupt_db)

    assert not corrupt_db.exists()
    _assert_contained(journal_dir, outside_canary)