~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""Sync distinguishes usable content from empty text and empty PDF storage."""
import hashlib
import pytest
from sqlalchemy import event, inspect
from local_deep_research.constants import FILE_PATH_TEXT_ONLY
from local_deep_research.database.models.library import (
Document,
DocumentBlob,
DocumentChunk,
DocumentCollection,
DocumentStatus,
)
from local_deep_research.database.models.research import ResearchResource
from local_deep_research.research_library.services.pdf_storage_manager import (
PDFStorageManager,
)
from local_deep_research.research_library.utils import (
document_awaits_pdf,
find_tracker_by_url,
)
from tests.research_library.test_library_sync_contracts import (
PDF_BYTES,
REGENERATED_PDF_BYTES,
SAVED_TEXT,
StubDownloader,
_add_chunk,
_cite_in_another_research,
_add_database_row_without_pdf,
db as db,
filesystem_mode as filesystem_mode,
make_library_service,
make_service,
one_download_session as one_download_session,
store as store,
world as world,
)
def add_document(db, world, *, text=None, content=PDF_BYTES, url=None):
url = url or "https://example.com/empty-content.pdf"
resource = world.add_resource(url, downloaded=True)
document = _add_database_row_without_pdf(
db, world, resource, world.research_id, url, content
)
row = db.session.get(Document, document)
row.text_content = text
db.session.get(ResearchResource, resource).document_id = document
tracker = find_tracker_by_url(db.session, url)
tracker.file_path = "database"
tracker.file_hash = hashlib.sha256(content).hexdigest()
db.session.commit()
return document, resource
@pytest.mark.parametrize(
"text",
[
"\t\n\r ",
"\u00a0\u2003\u202f",
"\v\f\x1c\x1d\x1e\x1f\x85\u1680\u2000\u2001\u2002\u2004\u2005\u2006\u2007\u2008\u2009\u200a\u2028\u2029\u205f\u3000",
],
)
def test_whitespace_only_is_removed(db, store, world, monkeypatch, text):
document, _ = add_document(db, world, text=text)
stats = make_library_service(
db, store, monkeypatch
).sync_library_with_filesystem()
with db.fresh() as session:
assert session.get(Document, document) is None, (
"whitespace is not usable extracted text"
)
assert stats["files_missing"] == 1
@pytest.mark.parametrize("storage", ["filesystem", "database"])
@pytest.mark.parametrize("tracked", [True, False])
def test_empty_pdf_is_refilled_in_same_document(
db,
store,
world,
one_download_session,
filesystem_mode,
monkeypatch,
storage,
tracked,
):
document, resource = add_document(db, world, text=SAVED_TEXT)
chunk = _add_chunk(db, document)
row = db.session.get(Document, document)
row.storage_mode = storage
if storage == "filesystem":
row.file_path = "pdfs/empty.pdf"
(store.root / row.file_path).write_bytes(b"")
else:
db.session.add(DocumentBlob(document_id=document, pdf_binary=b""))
tracker = find_tracker_by_url(db.session, row.original_url)
tracker.file_path = (row.file_path or "database") if tracked else None
db.session.commit()
library = make_library_service(db, store, monkeypatch)
stats = library.sync_library_with_filesystem()
assert stats["files_missing"] == 1, (
"a confirmed empty PDF must be eligible for refill"
)
with db.fresh() as session:
kept = session.get(Document, document)
assert kept.text_content == SAVED_TEXT
assert kept.file_path == FILE_PATH_TEXT_ONLY
assert document_awaits_pdf(kept)
assert not PDFStorageManager.pdf_exists(store.root, kept, session)
if storage == "database":
assert session.get(DocumentBlob, document) is None
assert session.get(DocumentChunk, chunk) is not None
assert (
session.query(DocumentCollection)
.filter_by(document_id=document)
.count()
== 1
)
assert session.get(ResearchResource, resource).document_id == document
downloader = StubDownloader(REGENERATED_PDF_BYTES)
service = make_service(store, downloader, storage_mode=storage)
assert service.download_resource(resource) == (True, None)
with db.fresh() as session:
kept = session.get(Document, document)
assert session.query(Document).count() == 1
assert (
PDFStorageManager(store.root, storage).load_pdf(kept, session)
== REGENERATED_PDF_BYTES
)
assert session.get(ResearchResource, resource).document_id == document
assert (
service.queue_research_downloads(
world.research_id, "library-sync-contracts"
)
== 0
)
assert downloader.urls == ["https://example.com/empty-content.pdf"]
assert library.sync_library_with_filesystem()["files_missing"] == 0
def test_clear_missing_pdf_keeps_a_concurrently_refilled_blob(
db, store, world, monkeypatch
):
"""A blob that gained real bytes before the delete must survive.
``_clear_missing_pdf`` removes the zero-length blob row, but only while
it is still empty. If a concurrent download writes real bytes into the
row between the earlier size read and this delete, those bytes must be
kept. The delete is constrained to empty blobs, so the refilled row is
preserved.
"""
document, _ = add_document(db, world, text=SAVED_TEXT)
row = db.session.get(Document, document)
row.storage_mode = "database"
db.session.add(DocumentBlob(document_id=document, pdf_binary=PDF_BYTES))
tracker = find_tracker_by_url(db.session, row.original_url)
db.session.commit()
service = make_library_service(db, store, monkeypatch)
service._clear_missing_pdf(row, tracker, {"trackers_updated": 0})
db.session.commit()
with db.fresh() as session:
blob = session.get(DocumentBlob, document)
assert blob is not None, "a refilled (nonempty) blob must be kept"
assert blob.pdf_binary == PDF_BYTES
@pytest.mark.parametrize(
"status",
[DocumentStatus.FAILED, DocumentStatus.PENDING, DocumentStatus.PROCESSING],
)
@pytest.mark.parametrize("holder_first", [True, False])
@pytest.mark.parametrize(
"blob", [PDF_BYTES, b"", None], ids=["positive", "zero", "absent"]
)
def test_database_tracker_census_includes_noncompleted_holders(
db, store, world, monkeypatch, status, holder_first, blob
):
url = "https://example.com/empty-content.pdf"
empty_resource = world.add_resource(url, downloaded=True)
holder_research, holder_resource = _cite_in_another_research(db, url)
tracker = find_tracker_by_url(db.session, url)
tracker.file_path = "database"
db.session.commit()
def holder():
document = _add_database_row_without_pdf(
db,
world,
holder_resource,
holder_research,
url,
REGENERATED_PDF_BYTES,
)
row = db.session.get(Document, document)
row.status = status
row.text_content = "retained holder text " * 100_000
db.session.get(ResearchResource, holder_resource).document_id = document
if blob is not None:
db.session.add(
DocumentBlob(
document_id=document,
pdf_binary=REGENERATED_PDF_BYTES if blob else b"",
)
)
db.session.commit()
return document
if holder_first:
holder_id = holder()
empty = _add_database_row_without_pdf(
db, world, empty_resource, world.research_id, url, PDF_BYTES
)
if not holder_first:
holder_id = holder()
library = make_library_service(db, store, monkeypatch)
stats = sync_without_loading_bodies(db, library)
with db.fresh() as session:
assert session.get(Document, empty) is None
assert session.get(Document, holder_id).status == status
tracker = find_tracker_by_url(
session, "https://example.com/empty-content.pdf"
)
assert (tracker.file_path, tracker.is_downloaded) == (
("database", True) if blob else (None, False)
), (
"a noncompleted holder's positive PDF must protect the shared tracker"
)
assert stats["trackers_updated"] == (0 if blob else 1)
def sync_without_loading_bodies(db, library):
statements = []
def capture(
connection, cursor, statement, parameters, context, executemany
):
statements.append(statement)
def check_deferred(session, instance):
if isinstance(instance, Document):
assert "text_content" in inspect(instance).unloaded
event.listen(db.engine, "before_cursor_execute", capture)
event.listen(type(db.session), "loaded_as_persistent", check_deferred)
try:
stats = library.sync_library_with_filesystem()
finally:
event.remove(db.engine, "before_cursor_execute", capture)
event.remove(type(db.session), "loaded_as_persistent", check_deferred)
# SQL may inspect text or blob length, but must not project either body.
for statement in statements:
projection = statement.lower().split("from", 1)[0]
assert "documents.text_content" not in projection, statement
assert "document_blobs.pdf_binary" not in projection.replace(
"length(document_blobs.pdf_binary)", ""
), statement
return stats
@pytest.mark.parametrize(
"text",
["普通の本文", "\t\u00a0 useful text \u2003\n", "x" * 2_000_000, "\u200b"],
ids=["unicode", "surrounded", "large", "non-whitespace-separator"],
)
def test_text_stays_deferred_and_nonblank_is_retained(
db, store, world, monkeypatch, text
):
document, _ = add_document(db, world, text=text)
stats = sync_without_loading_bodies(
db, make_library_service(db, store, monkeypatch)
)
assert stats["files_missing"] == 0
with db.fresh() as session:
assert session.get(Document, document).text_content == text
@pytest.mark.parametrize("storage", ["filesystem", "database"])
@pytest.mark.parametrize("tracked", [True, False])
def test_empty_pdf_without_text_is_deleted(
db, store, world, monkeypatch, storage, tracked
):
document, _ = add_document(db, world)
row = db.session.get(Document, document)
row.storage_mode = storage
if storage == "filesystem":
row.file_path = "pdfs/zero.pdf"
(store.root / row.file_path).write_bytes(b"")
else:
db.session.add(DocumentBlob(document_id=document, pdf_binary=b""))
tracker = find_tracker_by_url(db.session, row.original_url)
tracker.file_path = (row.file_path or "database") if tracked else None
db.session.commit()
stats = make_library_service(
db, store, monkeypatch
).sync_library_with_filesystem()
with db.fresh() as session:
assert session.get(Document, document) is None, (
"zero bytes cannot preserve an empty document"
)
assert stats["files_missing"] == 1
@pytest.mark.parametrize(
"identity", ["hash", "link", "conflicting-link", "conflicting-hash"]
)
def test_legacy_tracker_identity_is_not_overridden_by_raw_url(
db, store, world, monkeypatch, identity
):
from local_deep_research.database.models.download_tracker import (
DownloadTracker,
)
from local_deep_research.research_library.utils import get_legacy_url_hashes
url = "https://example.com/identity.pdf?utm_source=legacy"
document, _ = add_document(db, world, url=url)
canonical = find_tracker_by_url(db.session, url)
canonical.file_hash = hashlib.sha256(b"newer version").hexdigest()
canonical_path = canonical.file_path
legacy = DownloadTracker(
url=url,
url_hash=get_legacy_url_hashes(url)[0],
first_resource_id=canonical.first_resource_id,
is_downloaded=True,
file_path="database",
file_hash=hashlib.sha256(PDF_BYTES).hexdigest()
if identity != "link"
else None,
library_document_id=document
if identity in ("link", "conflicting-hash")
else None,
)
if identity == "conflicting-link":
# Point the explicit link at a real, unrelated document.
other, _ = add_document(
db,
world,
url="https://example.com/unrelated.pdf",
content=REGENERATED_PDF_BYTES,
)
legacy.library_document_id = other
if identity == "conflicting-hash":
legacy.file_hash = canonical.file_hash
db.session.add(legacy)
db.session.commit()
canonical_id, legacy_id = canonical.id, legacy.id
make_library_service(db, store, monkeypatch).sync_library_with_filesystem()
with db.fresh() as session:
assert session.get(Document, document) is None
current = session.get(DownloadTracker, canonical_id)
assert (current.file_path, current.is_downloaded) == (
canonical_path,
True,
)
old = session.get(DownloadTracker, legacy_id)
identified = identity in ("hash", "link")
assert (old.file_path, old.is_downloaded) == (
(None, False) if identified else ("database", True)
)