~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
fix(library): handle empty content during sync (#7427) (#7763)
Chanhyo Jung0ec4d2998cfc
diff --git a/changelog.d/7427.bugfix.md b/changelog.d/7427.bugfix.mdnew file mode 100644index 0000000000..b634ec2656--- /dev/null+++ b/changelog.d/7427.bugfix.md@@ -0,0 +1 @@+Fix library sync to discard whitespace-only content, recover empty PDFs without losing retained text, and preserve shared database trackers while any document still stores PDF bytes.diff --git a/src/local_deep_research/research_library/services/library_service.py b/src/local_deep_research/research_library/services/library_service.pyindex b9c3511249..01fe88c5a6 100644--- a/src/local_deep_research/research_library/services/library_service.py+++ b/src/local_deep_research/research_library/services/library_service.py@@ -59,6 +59,14 @@ _DROPDOWN_RESEARCH_LIMIT = 1000# distinct-domain set it accumulates is small regardless of library size._DOMAIN_SCAN_BATCH_SIZE = 1000+# Python str.strip() whitespace, including Unicode separators and the four+# ASCII information separators. SQLite's default trim() removes only spaces.+_STRIP_WHITESPACE = (+ "\t\n\v\f\r\x1c\x1d\x1e\x1f \x85\u00a0\u1680"+ "\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007"+ "\u2008\u2009\u200a\u2028\u2029\u202f\u205f\u3000"+)+def _names_a_file(file_path: Optional[str]) -> bool:"""Whether a ``file_path`` value names a file on disk.@@ -196,13 +204,24 @@ class LibraryService:is not None)+ def _sync_pdf_blob_size(self, session, document_id: str) -> Optional[int]:+ """Read size only: None is absent, zero is a confirmed empty PDF.++ Keep _has_blob_in_db's existence contract unchanged for other callers.+ """+ return (+ session.query(func.length(DocumentBlob.pdf_binary))+ .filter_by(document_id=document_id)+ .scalar()+ )+def _remaining_content(self, session, doc: Document, settings_manager) -> Optional[str]:"""What a download stores itself, whatever its URL's tracker names.- Returns ``"pdf"`` when its PDF is still stored (a blob in the- database, or a file at the document's own ``file_path``),+ Returns ``"pdf"`` when nonempty PDF bytes are still stored (a blob+ in the database, or a file at the document's own ``file_path``),``"text"`` when only its extracted text is left, and ``None`` whennothing is.@@ -215,20 +234,20 @@ class LibraryService:Deleting a document that holds any of it would also delete thatcontent, its chunks and its vectors."""- if self._has_blob_in_db(session, doc.id):+ if (self._sync_pdf_blob_size(session, doc.id) or 0) > 0:return "pdf"if _names_a_file(doc.file_path):resolved = get_absolute_path_from_settings(doc.file_path, self.username, settings_manager=settings_manager)- if resolved and resolved.is_file():+ if resolved and resolved.is_file() and resolved.stat().st_size:return "pdf"has_text = (session.query(Document.id).filter(Document.id == doc.id,Document.text_content.isnot(None),- Document.text_content != "",+ func.trim(Document.text_content, _STRIP_WHITESPACE) != "",).first()is not None@@ -1737,13 +1756,9 @@ class LibraryService:tracker.file_path == "database"or doc.storage_mode == "database"):- has_blob = (- session.query(DocumentBlob.document_id)- .filter_by(document_id=doc.id)- .first()- is not None- )- if has_blob:+ if (+ self._sync_pdf_blob_size(session, doc.id) or 0+ ) > 0:stats["files_found"] += 1continue# Check if the file exists in the per-user root.@@ -1762,6 +1777,7 @@ class LibraryService:_names_a_file(tracker.file_path)and file_pathand file_path.is_file()+ and file_path.stat().st_size):stats["files_found"] += 1continue@@ -1770,9 +1786,17 @@ class LibraryService:remaining = self._remaining_content(session, doc, settings_manager)- if remaining == "text" and _names_a_file(- tracker.file_path+ if remaining == "text" and (+ _names_a_file(tracker.file_path)+ or self._sync_pdf_blob_size(session, doc.id) == 0):+ # An absent blob can be an intentional text-only+ # placeholder. A present but empty PDF is damaged+ # storage and must permit the native refill writer.+ text_tracker = tracker+ if tracker.file_path == "database":+ emptied_database_trackers[tracker.id] = tracker+ text_tracker = None# Only the text is left of a PDF that was a# file. Keep the text and clear the dangling# PDF reference.@@ -1782,7 +1806,7 @@ class LibraryService:# path that named the missing file.self._keep_text_report_missing_pdf(doc,- tracker,+ text_tracker,tracker.file_path,file_path,stats,@@ -1791,7 +1815,9 @@ class LibraryService:# No download path asks for its PDF, but a# later download of the same bytes stores# the PDF in it (see _clear_missing_pdf).- self._clear_missing_pdf(doc, tracker, stats)+ self._clear_missing_pdf(+ doc, text_tracker, stats+ )stats["files_found"] += 1elif remaining is not None:# Its PDF is still stored as a blob, or its text@@ -1856,7 +1882,10 @@ class LibraryService:remaining = self._remaining_content(session, doc, settings_manager)- if remaining == "text" and _names_a_file(doc.file_path):+ if remaining == "text" and (+ _names_a_file(doc.file_path)+ or self._sync_pdf_blob_size(session, doc.id) == 0+ ):# Its own PDF file is gone and only the text is# left. Keep the text and clear the dangling# PDF reference.@@ -1866,11 +1895,15 @@ class LibraryService:self._keep_text_report_missing_pdf(doc,tracker,- doc.file_path,- get_absolute_path_from_settings(- doc.file_path,- self.username,- settings_manager=settings_manager,+ doc.file_path or "database",+ (+ get_absolute_path_from_settings(+ doc.file_path,+ self.username,+ settings_manager=settings_manager,+ )+ if _names_a_file(doc.file_path)+ else None),stats,)@@ -1937,6 +1970,29 @@ class LibraryService:# blob still keeps the record true. Every document under the# tracker has been checked by now, so the order in which the# sync read them does not change whether it is reset.+ if emptied_database_trackers:+ holders = (+ session.query(Document)+ .join(DocumentBlob, DocumentBlob.document_id == Document.id)+ .filter(+ Document.original_url.isnot(None),+ func.length(DocumentBlob.pdf_binary) > 0,+ )+ .options(defer(Document.text_content))+ .all()+ )+ for holder in holders:+ # Use the same canonical/legacy lookups as the completed+ # loop. A URL's shared record can remain true for another+ # version, but only an identifying tracker can be reset.+ for held_tracker in (+ find_tracker_by_url(session, holder.original_url),+ find_tracker_for_document(session, holder),+ ):+ if held_tracker is not None:+ database_tracker_documents.setdefault(+ held_tracker.id, []+ ).append(holder.id)removed = set(removed_doc_ids)for tracker_id, tracker in emptied_database_trackers.items():others = [@@ -1946,7 +2002,10 @@ class LibraryService:]if others and (session.query(DocumentBlob.document_id)- .filter(DocumentBlob.document_id.in_(others))+ .filter(+ DocumentBlob.document_id.in_(others),+ func.length(DocumentBlob.pdf_binary) > 0,+ ).first()is not None):diff --git a/tests/research_library/services/test_library_sync_empty_content.py b/tests/research_library/services/test_library_sync_empty_content.pynew file mode 100644index 0000000000..b3578a9668--- /dev/null+++ b/tests/research_library/services/test_library_sync_empty_content.py@@ -0,0 +1,332 @@+"""Sync distinguishes usable content from empty text and empty PDF storage."""++import hashlib++import pytest+from sqlalchemy import event, inspect++from local_deep_research.constants import FILE_PATH_TEXT_ONLY+from local_deep_research.database.models.library import (+ Document,+ DocumentBlob,+ DocumentChunk,+ DocumentCollection,+ DocumentStatus,+)+from local_deep_research.database.models.research import ResearchResource+from local_deep_research.research_library.services.pdf_storage_manager import (+ PDFStorageManager,+)+from local_deep_research.research_library.utils import (+ document_awaits_pdf,+ find_tracker_by_url,+)+from tests.research_library.test_library_sync_contracts import (+ PDF_BYTES,+ REGENERATED_PDF_BYTES,+ SAVED_TEXT,+ StubDownloader,+ _add_chunk,+ _cite_in_another_research,+ _add_database_row_without_pdf,+ db as db,+ filesystem_mode as filesystem_mode,+ make_library_service,+ make_service,+ one_download_session as one_download_session,+ store as store,+ world as world,+)+++def add_document(db, world, *, text=None, content=PDF_BYTES, url=None):+ url = url or "https://example.com/empty-content.pdf"+ resource = world.add_resource(url, downloaded=True)+ document = _add_database_row_without_pdf(+ db, world, resource, world.research_id, url, content+ )+ row = db.session.get(Document, document)+ row.text_content = text+ db.session.get(ResearchResource, resource).document_id = document+ tracker = find_tracker_by_url(db.session, url)+ tracker.file_path = "database"+ tracker.file_hash = hashlib.sha256(content).hexdigest()+ db.session.commit()+ return document, resource+++ "text",+ [+ "\t\n\r ",+ "\u00a0\u2003\u202f",+ "\v\f\x1c\x1d\x1e\x1f\x85\u1680\u2000\u2001\u2002\u2004\u2005\u2006\u2007\u2008\u2009\u200a\u2028\u2029\u205f\u3000",+ ],+)+def test_whitespace_only_is_removed(db, store, world, monkeypatch, text):+ document, _ = add_document(db, world, text=text)+ stats = make_library_service(+ db, store, monkeypatch+ ).sync_library_with_filesystem()+ with db.fresh() as session:+ assert session.get(Document, document) is None, (+ "whitespace is not usable extracted text"+ )+ assert stats["files_missing"] == 1++[email protected]("storage", ["filesystem", "database"])[email protected]("tracked", [True, False])+def test_empty_pdf_is_refilled_in_same_document(+ db,+ store,+ world,+ one_download_session,+ filesystem_mode,+ monkeypatch,+ storage,+ tracked,+):+ document, resource = add_document(db, world, text=SAVED_TEXT)+ chunk = _add_chunk(db, document)+ row = db.session.get(Document, document)+ row.storage_mode = storage+ if storage == "filesystem":+ row.file_path = "pdfs/empty.pdf"+ (store.root / row.file_path).write_bytes(b"")+ else:+ db.session.add(DocumentBlob(document_id=document, pdf_binary=b""))+ tracker = find_tracker_by_url(db.session, row.original_url)+ tracker.file_path = (row.file_path or "database") if tracked else None+ db.session.commit()+ library = make_library_service(db, store, monkeypatch)+ stats = library.sync_library_with_filesystem()+ assert stats["files_missing"] == 1, (+ "a confirmed empty PDF must be eligible for refill"+ )+ with db.fresh() as session:+ kept = session.get(Document, document)+ assert kept.text_content == SAVED_TEXT+ assert kept.file_path == FILE_PATH_TEXT_ONLY+ assert document_awaits_pdf(kept)+ assert session.get(DocumentChunk, chunk) is not None+ assert (+ session.query(DocumentCollection)+ .filter_by(document_id=document)+ .count()+ == 1+ )+ assert session.get(ResearchResource, resource).document_id == document+ downloader = StubDownloader(REGENERATED_PDF_BYTES)+ service = make_service(store, downloader, storage_mode=storage)+ assert service.download_resource(resource) == (True, None)+ with db.fresh() as session:+ kept = session.get(Document, document)+ assert session.query(Document).count() == 1+ assert (+ PDFStorageManager(store.root, storage).load_pdf(kept, session)+ == REGENERATED_PDF_BYTES+ )+ assert session.get(ResearchResource, resource).document_id == document+ assert (+ service.queue_research_downloads(+ world.research_id, "library-sync-contracts"+ )+ == 0+ )+ assert downloader.urls == ["https://example.com/empty-content.pdf"]+ assert library.sync_library_with_filesystem()["files_missing"] == 0+++ "status",+ [DocumentStatus.FAILED, DocumentStatus.PENDING, DocumentStatus.PROCESSING],+)[email protected]("holder_first", [True, False])+ "blob", [PDF_BYTES, b"", None], ids=["positive", "zero", "absent"]+)+def test_database_tracker_census_includes_noncompleted_holders(+ db, store, world, monkeypatch, status, holder_first, blob+):+ url = "https://example.com/empty-content.pdf"+ empty_resource = world.add_resource(url, downloaded=True)+ holder_research, holder_resource = _cite_in_another_research(db, url)+ tracker = find_tracker_by_url(db.session, url)+ tracker.file_path = "database"+ db.session.commit()++ def holder():+ document = _add_database_row_without_pdf(+ db,+ world,+ holder_resource,+ holder_research,+ url,+ REGENERATED_PDF_BYTES,+ )+ row = db.session.get(Document, document)+ row.status = status+ row.text_content = "retained holder text " * 100_000+ db.session.get(ResearchResource, holder_resource).document_id = document+ if blob is not None:+ db.session.add(+ DocumentBlob(+ document_id=document,+ pdf_binary=REGENERATED_PDF_BYTES if blob else b"",+ )+ )+ db.session.commit()+ return document++ if holder_first:+ holder_id = holder()+ empty = _add_database_row_without_pdf(+ db, world, empty_resource, world.research_id, url, PDF_BYTES+ )+ if not holder_first:+ holder_id = holder()+ library = make_library_service(db, store, monkeypatch)+ stats = sync_without_loading_bodies(db, library)+ with db.fresh() as session:+ assert session.get(Document, empty) is None+ assert session.get(Document, holder_id).status == status+ tracker = find_tracker_by_url(+ session, "https://example.com/empty-content.pdf"+ )+ assert (tracker.file_path, tracker.is_downloaded) == (+ ("database", True) if blob else (None, False)+ ), (+ "a noncompleted holder's positive PDF must protect the shared tracker"+ )+ assert stats["trackers_updated"] == (0 if blob else 1)+++def sync_without_loading_bodies(db, library):+ statements = []++ def capture(+ connection, cursor, statement, parameters, context, executemany+ ):+ statements.append(statement)++ def check_deferred(session, instance):+ if isinstance(instance, Document):+ assert "text_content" in inspect(instance).unloaded++ event.listen(db.engine, "before_cursor_execute", capture)+ event.listen(type(db.session), "loaded_as_persistent", check_deferred)+ try:+ stats = library.sync_library_with_filesystem()+ finally:+ event.remove(db.engine, "before_cursor_execute", capture)+ event.remove(type(db.session), "loaded_as_persistent", check_deferred)+ # SQL may inspect text or blob length, but must not project either body.+ for statement in statements:+ projection = statement.lower().split("from", 1)[0]+ assert "documents.text_content" not in projection, statement+ assert "document_blobs.pdf_binary" not in projection.replace(+ "length(document_blobs.pdf_binary)", ""+ ), statement+ return stats+++ "text",+ ["普通の本文", "\t\u00a0 useful text \u2003\n", "x" * 2_000_000, "\u200b"],+ ids=["unicode", "surrounded", "large", "non-whitespace-separator"],+)+def test_text_stays_deferred_and_nonblank_is_retained(+ db, store, world, monkeypatch, text+):+ document, _ = add_document(db, world, text=text)+ stats = sync_without_loading_bodies(+ db, make_library_service(db, store, monkeypatch)+ )+ assert stats["files_missing"] == 0+ with db.fresh() as session:+ assert session.get(Document, document).text_content == text++[email protected]("storage", ["filesystem", "database"])[email protected]("tracked", [True, False])+def test_empty_pdf_without_text_is_deleted(+ db, store, world, monkeypatch, storage, tracked+):+ document, _ = add_document(db, world)+ row = db.session.get(Document, document)+ row.storage_mode = storage+ if storage == "filesystem":+ row.file_path = "pdfs/zero.pdf"+ (store.root / row.file_path).write_bytes(b"")+ else:+ db.session.add(DocumentBlob(document_id=document, pdf_binary=b""))+ tracker = find_tracker_by_url(db.session, row.original_url)+ tracker.file_path = (row.file_path or "database") if tracked else None+ db.session.commit()+ stats = make_library_service(+ db, store, monkeypatch+ ).sync_library_with_filesystem()+ with db.fresh() as session:+ assert session.get(Document, document) is None, (+ "zero bytes cannot preserve an empty document"+ )+ assert stats["files_missing"] == 1+++ "identity", ["hash", "link", "conflicting-link", "conflicting-hash"]+)+def test_legacy_tracker_identity_is_not_overridden_by_raw_url(+ db, store, world, monkeypatch, identity+):+ from local_deep_research.database.models.download_tracker import (+ DownloadTracker,+ )+ from local_deep_research.research_library.utils import get_legacy_url_hashes++ url = "https://example.com/identity.pdf?utm_source=legacy"+ document, _ = add_document(db, world, url=url)+ canonical = find_tracker_by_url(db.session, url)+ canonical.file_hash = hashlib.sha256(b"newer version").hexdigest()+ canonical_path = canonical.file_path+ legacy = DownloadTracker(+ url=url,+ url_hash=get_legacy_url_hashes(url)[0],+ first_resource_id=canonical.first_resource_id,+ is_downloaded=True,+ file_path="database",+ file_hash=hashlib.sha256(PDF_BYTES).hexdigest()+ if identity != "link"+ else None,+ library_document_id=document+ if identity in ("link", "conflicting-hash")+ else None,+ )+ if identity == "conflicting-link":+ # Point the explicit link at a real, unrelated document.+ other, _ = add_document(+ db,+ world,+ url="https://example.com/unrelated.pdf",+ content=REGENERATED_PDF_BYTES,+ )+ legacy.library_document_id = other+ if identity == "conflicting-hash":+ legacy.file_hash = canonical.file_hash+ db.session.add(legacy)+ db.session.commit()+ canonical_id, legacy_id = canonical.id, legacy.id+ make_library_service(db, store, monkeypatch).sync_library_with_filesystem()+ with db.fresh() as session:+ assert session.get(Document, document) is None+ current = session.get(DownloadTracker, canonical_id)+ assert (current.file_path, current.is_downloaded) == (+ canonical_path,+ True,+ )+ old = session.get(DownloadTracker, legacy_id)+ identified = identity in ("hash", "link")+ assert (old.file_path, old.is_downloaded) == (+ (None, False) if identified else ("database", True)+ )