~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""Library sync composes with canonical PDF reuse and text-index invalidation."""
import hashlib
from types import SimpleNamespace
import pytest
from local_deep_research.database.models.library import (
Collection,
Document,
DocumentCollection,
DocumentStatus,
DownloadQueue,
RagDocumentStatus,
)
from local_deep_research.database.models.research import ResearchResource
from local_deep_research.research_library.services import library_service
from local_deep_research.research_library.services.library_rag_service import (
LibraryRAGService,
)
from local_deep_research.research_library.services.pdf_storage_manager import (
PDFStorageManager,
)
from local_deep_research.research_library.utils import document_awaits_pdf
from tests.research_library.services.test_download_text_quality import (
LONGER,
extraction_case as extraction_case,
)
from tests.research_library.services.test_text_index_invalidation import (
assert_pending,
indexer_for,
persist,
)
from tests.research_library.test_library_sync_contracts import (
EXTRACTED_TEXT,
REGENERATED_PDF_BYTES,
REGENERATED_TEXT,
TEXT_PDF_BYTES,
StubDownloader,
db as db,
download_all_new,
filesystem_mode as filesystem_mode,
make_library_service,
make_service,
one_download_session as one_download_session,
store as store,
world as world,
)
@pytest.mark.parametrize("changed", [False, True])
@pytest.mark.parametrize(
"storage,limit", [("database", 3072), ("none", 3072), ("database", 0)]
)
def test_linked_refill_settles_storage_and_requested_collection(
db,
store,
world,
one_download_session,
filesystem_mode,
monkeypatch,
changed,
storage,
limit,
):
"""A refill through a linked resource settles storage and the requested
collection.
``("database", 0)``: the size limit refuses every PDF. Before #6913 a
refused refill settled the document as a text-only download and
reported success. #6913 changed that premise on purpose: a document
awaiting its PDF whose recorded ``file_size`` the current limit refuses
keeps awaiting it, and ``download_resource`` answers
``pdf_storage_size_limit`` without fetching anything (see
``DownloadService._pdf_refused_at_current_limit``), so that case now
asserts the refusal and the untouched document, queue and collection.
"""
owner = world.add_resource("https://example.com/refill-owner.pdf")
downloader = StubDownloader(TEXT_PDF_BYTES)
service = make_service(store, downloader, storage_mode="filesystem")
assert service.download_resource(owner) == (True, None)
with db.fresh() as session:
document = session.query(Document).filter_by(resource_id=owner).one()
document_id = document.id
(store.root / document.file_path).unlink()
library = make_library_service(db, store, monkeypatch)
assert library.sync_library_with_filesystem()["files_missing"] == 1
linked = world.add_resource("https://example.com/refill-mirror.pdf")
db.session.get(ResearchResource, linked).document_id = document_id
db.session.add(Collection(id="requested", name="Requested"))
db.session.flush()
db.session.add(
DownloadQueue(
resource_id=linked,
research_id=world.research_id,
collection_id="requested",
status=DocumentStatus.PENDING,
)
)
db.session.commit()
content = REGENERATED_PDF_BYTES if changed else TEXT_PDF_BYTES
downloader.content = content
service.settings._values.update(
{
"research_library.pdf_storage_mode": storage,
"research_library.max_pdf_size_mb": limit,
}
)
if limit == 0:
assert service.download_resource(linked) == (
False,
"pdf_storage_size_limit",
)
with db.fresh() as session:
assert session.query(Document).count() == 1
document = session.get(Document, document_id)
assert (
document.document_hash
== hashlib.sha256(TEXT_PDF_BYTES).hexdigest()
)
assert document.text_content == EXTRACTED_TEXT
assert document_awaits_pdf(document)
assert (
session.query(DocumentCollection)
.filter_by(document_id=document_id, collection_id="requested")
.count()
== 0
)
assert (
session.query(DownloadQueue)
.filter_by(resource_id=linked)
.one()
.status
is DocumentStatus.PENDING
)
assert downloader.urls == ["https://example.com/refill-owner.pdf"]
return
assert service.download_resource(linked) == (True, None)
with db.fresh() as session:
assert session.query(Document).count() == 1
document = session.get(Document, document_id)
assert document.document_hash == hashlib.sha256(content).hexdigest()
assert document.text_content == (
REGENERATED_TEXT if changed else EXTRACTED_TEXT
)
assert not document_awaits_pdf(document)
assert (
session.query(DocumentCollection)
.filter_by(
document_id=document_id,
collection_id="requested",
)
.count()
== 1
)
assert (
session.query(DownloadQueue)
.filter_by(
resource_id=linked,
)
.one()
.status
is DocumentStatus.COMPLETED
)
manager = PDFStorageManager(store.root, storage)
if storage == "database" and limit:
assert manager.load_pdf(document, session) == content
else:
assert document.storage_mode == "none"
assert document.file_path is None
assert not manager.has_pdf(document, session)
assert downloader.urls == [
"https://example.com/refill-owner.pdf",
"https://example.com/refill-mirror.pdf",
]
assert download_all_new(db, monkeypatch)["queued"] == 0
assert service.queue_research_downloads(world.research_id, "requested") == 0
@pytest.mark.parametrize("mode", ["serial", "prepared"])
def test_regenerated_refill_keeps_old_vectors_invalid_until_reindexed(
extraction_case,
monkeypatch,
tmp_path,
mode,
):
"""Refilled docs keep old vectors invalidated until a reindex replaces them.
Premise note (#7557, PR 7759): mode="prepared" used to snapshot status
existence against the coordinator's ``rag_index_record``; preparation
now resolves the index through its own worker-owned session (the
coordinator's ORM record may expire mid-run and must not be refreshed
by a worker), so the ``indexer_for`` double provides that seam. The
premise tested here — a regenerated refill never resurrects old
vectors before the reindex — is unchanged.
"""
case = extraction_case(quality="low", incoming=LONGER, storage="database")
base = tmp_path / "library"
case.service.library_root = base / case.service.username
with case.sessions() as session:
document = session.get(Document, "canonical")
document.original_url = "https://example.org/old-paper.pdf"
document.document_hash = hashlib.sha256(
b"previous PDF bytes"
).hexdigest()
document.storage_mode = "filesystem"
document.file_path = "pdfs/missing.pdf"
session.get(
ResearchResource, case.resource_id
).document_id = document.id
session.commit()
monkeypatch.setattr(
library_service, "get_user_db_session", lambda *a, **k: case.sessions()
)
monkeypatch.setattr(
library_service,
"get_settings_manager",
lambda **kw: SimpleNamespace(
get_setting=lambda key, default=None: str(base),
close=lambda: None,
),
)
library = library_service.LibraryService.__new__(
library_service.LibraryService
)
library.username = case.service.username
assert library.sync_library_with_filesystem()["files_missing"] == 1
with case.sessions() as session:
assert document_awaits_pdf(session.get(Document, "canonical"))
indexer, backend = indexer_for(case)
old_ids = backend._store.ids.copy()
assert case.service.download_resource(case.resource_id) == (True, None)
with case.sessions() as session:
assert session.query(Document).count() == 1
document = session.get(Document, "canonical")
assert document.document_hash == case.pdf_hash
assert not document_awaits_pdf(document)
assert_pending(case)
assert_pending(case, "selected")
assert (
LibraryRAGService.reconcile_collection_index(indexer, "library")[
"indexed_documents"
]
== 0
)
assert_pending(case)
assert persist(indexer, mode)["status"] == "success"
assert backend._store.removed == old_ids
assert old_ids.isdisjoint(backend._store.ids)
assert (
LibraryRAGService.reconcile_collection_index(indexer, "library")[
"indexed_documents"
]
== 1
)
with case.sessions() as session:
assert (
session.get(RagDocumentStatus, ("canonical", "library")) is not None
)
assert_pending(case, "selected")