Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

9.3 KB
Raw Blame
"""Library sync composes with canonical PDF reuse and text-index invalidation."""

import hashlib
from types import SimpleNamespace

import pytest

from local_deep_research.database.models.library import (
    Collection,
    Document,
    DocumentCollection,
    DocumentStatus,
    DownloadQueue,
    RagDocumentStatus,
)
from local_deep_research.database.models.research import ResearchResource
from local_deep_research.research_library.services import library_service
from local_deep_research.research_library.services.library_rag_service import (
    LibraryRAGService,
)
from local_deep_research.research_library.services.pdf_storage_manager import (
    PDFStorageManager,
)
from local_deep_research.research_library.utils import document_awaits_pdf
from tests.research_library.services.test_download_text_quality import (
    LONGER,
    extraction_case as extraction_case,
)
from tests.research_library.services.test_text_index_invalidation import (
    assert_pending,
    indexer_for,
    persist,
)
from tests.research_library.test_library_sync_contracts import (
    EXTRACTED_TEXT,
    REGENERATED_PDF_BYTES,
    REGENERATED_TEXT,
    TEXT_PDF_BYTES,
    StubDownloader,
    db as db,
    download_all_new,
    filesystem_mode as filesystem_mode,
    make_library_service,
    make_service,
    one_download_session as one_download_session,
    store as store,
    world as world,
)


@pytest.mark.parametrize("changed", [False, True])
@pytest.mark.parametrize(
    "storage,limit", [("database", 3072), ("none", 3072), ("database", 0)]
)
def test_linked_refill_settles_storage_and_requested_collection(
    db,
    store,
    world,
    one_download_session,
    filesystem_mode,
    monkeypatch,
    changed,
    storage,
    limit,
):
    """A refill through a linked resource settles storage and the requested
    collection.

    ``("database", 0)``: the size limit refuses every PDF. Before #6913 a
    refused refill settled the document as a text-only download and
    reported success. #6913 changed that premise on purpose: a document
    awaiting its PDF whose recorded ``file_size`` the current limit refuses
    keeps awaiting it, and ``download_resource`` answers
    ``pdf_storage_size_limit`` without fetching anything (see
    ``DownloadService._pdf_refused_at_current_limit``), so that case now
    asserts the refusal and the untouched document, queue and collection.
    """
    owner = world.add_resource("https://example.com/refill-owner.pdf")
    downloader = StubDownloader(TEXT_PDF_BYTES)
    service = make_service(store, downloader, storage_mode="filesystem")
    assert service.download_resource(owner) == (True, None)
    with db.fresh() as session:
        document = session.query(Document).filter_by(resource_id=owner).one()
        document_id = document.id
        (store.root / document.file_path).unlink()
    library = make_library_service(db, store, monkeypatch)
    assert library.sync_library_with_filesystem()["files_missing"] == 1

    linked = world.add_resource("https://example.com/refill-mirror.pdf")
    db.session.get(ResearchResource, linked).document_id = document_id
    db.session.add(Collection(id="requested", name="Requested"))
    db.session.flush()
    db.session.add(
        DownloadQueue(
            resource_id=linked,
            research_id=world.research_id,
            collection_id="requested",
            status=DocumentStatus.PENDING,
        )
    )
    db.session.commit()
    content = REGENERATED_PDF_BYTES if changed else TEXT_PDF_BYTES
    downloader.content = content
    service.settings._values.update(
        {
            "research_library.pdf_storage_mode": storage,
            "research_library.max_pdf_size_mb": limit,
        }
    )

    if limit == 0:
        assert service.download_resource(linked) == (
            False,
            "pdf_storage_size_limit",
        )
        with db.fresh() as session:
            assert session.query(Document).count() == 1
            document = session.get(Document, document_id)
            assert (
                document.document_hash
                == hashlib.sha256(TEXT_PDF_BYTES).hexdigest()
            )
            assert document.text_content == EXTRACTED_TEXT
            assert document_awaits_pdf(document)
            assert (
                session.query(DocumentCollection)
                .filter_by(document_id=document_id, collection_id="requested")
                .count()
                == 0
            )
            assert (
                session.query(DownloadQueue)
                .filter_by(resource_id=linked)
                .one()
                .status
                is DocumentStatus.PENDING
            )
        assert downloader.urls == ["https://example.com/refill-owner.pdf"]
        return

    assert service.download_resource(linked) == (True, None)
    with db.fresh() as session:
        assert session.query(Document).count() == 1
        document = session.get(Document, document_id)
        assert document.document_hash == hashlib.sha256(content).hexdigest()
        assert document.text_content == (
            REGENERATED_TEXT if changed else EXTRACTED_TEXT
        )
        assert not document_awaits_pdf(document)
        assert (
            session.query(DocumentCollection)
            .filter_by(
                document_id=document_id,
                collection_id="requested",
            )
            .count()
            == 1
        )
        assert (
            session.query(DownloadQueue)
            .filter_by(
                resource_id=linked,
            )
            .one()
            .status
            is DocumentStatus.COMPLETED
        )
        manager = PDFStorageManager(store.root, storage)
        if storage == "database" and limit:
            assert manager.load_pdf(document, session) == content
        else:
            assert document.storage_mode == "none"
            assert document.file_path is None
            assert not manager.has_pdf(document, session)
    assert downloader.urls == [
        "https://example.com/refill-owner.pdf",
        "https://example.com/refill-mirror.pdf",
    ]
    assert download_all_new(db, monkeypatch)["queued"] == 0
    assert service.queue_research_downloads(world.research_id, "requested") == 0


@pytest.mark.parametrize("mode", ["serial", "prepared"])
def test_regenerated_refill_keeps_old_vectors_invalid_until_reindexed(
    extraction_case,
    monkeypatch,
    tmp_path,
    mode,
):
    """Refilled docs keep old vectors invalidated until a reindex replaces them.

    Premise note (#7557, PR 7759): mode="prepared" used to snapshot status
    existence against the coordinator's ``rag_index_record``; preparation
    now resolves the index through its own worker-owned session (the
    coordinator's ORM record may expire mid-run and must not be refreshed
    by a worker), so the ``indexer_for`` double provides that seam. The
    premise tested here — a regenerated refill never resurrects old
    vectors before the reindex — is unchanged.
    """
    case = extraction_case(quality="low", incoming=LONGER, storage="database")
    base = tmp_path / "library"
    case.service.library_root = base / case.service.username
    with case.sessions() as session:
        document = session.get(Document, "canonical")
        document.original_url = "https://example.org/old-paper.pdf"
        document.document_hash = hashlib.sha256(
            b"previous PDF bytes"
        ).hexdigest()
        document.storage_mode = "filesystem"
        document.file_path = "pdfs/missing.pdf"
        session.get(
            ResearchResource, case.resource_id
        ).document_id = document.id
        session.commit()
    monkeypatch.setattr(
        library_service, "get_user_db_session", lambda *a, **k: case.sessions()
    )
    monkeypatch.setattr(
        library_service,
        "get_settings_manager",
        lambda **kw: SimpleNamespace(
            get_setting=lambda key, default=None: str(base),
            close=lambda: None,
        ),
    )
    library = library_service.LibraryService.__new__(
        library_service.LibraryService
    )
    library.username = case.service.username
    assert library.sync_library_with_filesystem()["files_missing"] == 1
    with case.sessions() as session:
        assert document_awaits_pdf(session.get(Document, "canonical"))

    indexer, backend = indexer_for(case)
    old_ids = backend._store.ids.copy()
    assert case.service.download_resource(case.resource_id) == (True, None)
    with case.sessions() as session:
        assert session.query(Document).count() == 1
        document = session.get(Document, "canonical")
        assert document.document_hash == case.pdf_hash
        assert not document_awaits_pdf(document)
    assert_pending(case)
    assert_pending(case, "selected")
    assert (
        LibraryRAGService.reconcile_collection_index(indexer, "library")[
            "indexed_documents"
        ]
        == 0
    )
    assert_pending(case)
    assert persist(indexer, mode)["status"] == "success"
    assert backend._store.removed == old_ids
    assert old_ids.isdisjoint(backend._store.ids)
    assert (
        LibraryRAGService.reconcile_collection_index(indexer, "library")[
            "indexed_documents"
        ]
        == 1
    )
    with case.sessions() as session:
        assert (
            session.get(RagDocumentStatus, ("canonical", "library")) is not None
        )
    assert_pending(case, "selected")