Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

fix(library): handle empty content during sync (#7427) (#7763)

Chanhyo Jung0ec4d2998cfc
diff --git a/changelog.d/7427.bugfix.md b/changelog.d/7427.bugfix.md
new file mode 100644
index 0000000000..b634ec2656
--- /dev/null
+++ b/changelog.d/7427.bugfix.md
@@ -0,0 +1 @@
+Fix library sync to discard whitespace-only content, recover empty PDFs without losing retained text, and preserve shared database trackers while any document still stores PDF bytes.
diff --git a/src/local_deep_research/research_library/services/library_service.py b/src/local_deep_research/research_library/services/library_service.py
index b9c3511249..01fe88c5a6 100644
--- a/src/local_deep_research/research_library/services/library_service.py
+++ b/src/local_deep_research/research_library/services/library_service.py
@@ -59,6 +59,14 @@ _DROPDOWN_RESEARCH_LIMIT = 1000
# distinct-domain set it accumulates is small regardless of library size.
_DOMAIN_SCAN_BATCH_SIZE = 1000
+# Python str.strip() whitespace, including Unicode separators and the four
+# ASCII information separators. SQLite's default trim() removes only spaces.
+_STRIP_WHITESPACE = (
+ "\t\n\v\f\r\x1c\x1d\x1e\x1f \x85\u00a0\u1680"
+ "\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007"
+ "\u2008\u2009\u200a\u2028\u2029\u202f\u205f\u3000"
+)
+
def _names_a_file(file_path: Optional[str]) -> bool:
"""Whether a ``file_path`` value names a file on disk.
@@ -196,13 +204,24 @@ class LibraryService:
is not None
)
+ def _sync_pdf_blob_size(self, session, document_id: str) -> Optional[int]:
+ """Read size only: None is absent, zero is a confirmed empty PDF.
+
+ Keep _has_blob_in_db's existence contract unchanged for other callers.
+ """
+ return (
+ session.query(func.length(DocumentBlob.pdf_binary))
+ .filter_by(document_id=document_id)
+ .scalar()
+ )
+
def _remaining_content(
self, session, doc: Document, settings_manager
) -> Optional[str]:
"""What a download stores itself, whatever its URL's tracker names.
- Returns ``"pdf"`` when its PDF is still stored (a blob in the
- database, or a file at the document's own ``file_path``),
+ Returns ``"pdf"`` when nonempty PDF bytes are still stored (a blob
+ in the database, or a file at the document's own ``file_path``),
``"text"`` when only its extracted text is left, and ``None`` when
nothing is.
@@ -215,20 +234,20 @@ class LibraryService:
Deleting a document that holds any of it would also delete that
content, its chunks and its vectors.
"""
- if self._has_blob_in_db(session, doc.id):
+ if (self._sync_pdf_blob_size(session, doc.id) or 0) > 0:
return "pdf"
if _names_a_file(doc.file_path):
resolved = get_absolute_path_from_settings(
doc.file_path, self.username, settings_manager=settings_manager
)
- if resolved and resolved.is_file():
+ if resolved and resolved.is_file() and resolved.stat().st_size:
return "pdf"
has_text = (
session.query(Document.id)
.filter(
Document.id == doc.id,
Document.text_content.isnot(None),
- Document.text_content != "",
+ func.trim(Document.text_content, _STRIP_WHITESPACE) != "",
)
.first()
is not None
@@ -1737,13 +1756,9 @@ class LibraryService:
tracker.file_path == "database"
or doc.storage_mode == "database"
):
- has_blob = (
- session.query(DocumentBlob.document_id)
- .filter_by(document_id=doc.id)
- .first()
- is not None
- )
- if has_blob:
+ if (
+ self._sync_pdf_blob_size(session, doc.id) or 0
+ ) > 0:
stats["files_found"] += 1
continue
# Check if the file exists in the per-user root.
@@ -1762,6 +1777,7 @@ class LibraryService:
_names_a_file(tracker.file_path)
and file_path
and file_path.is_file()
+ and file_path.stat().st_size
):
stats["files_found"] += 1
continue
@@ -1770,9 +1786,17 @@ class LibraryService:
remaining = self._remaining_content(
session, doc, settings_manager
)
- if remaining == "text" and _names_a_file(
- tracker.file_path
+ if remaining == "text" and (
+ _names_a_file(tracker.file_path)
+ or self._sync_pdf_blob_size(session, doc.id) == 0
):
+ # An absent blob can be an intentional text-only
+ # placeholder. A present but empty PDF is damaged
+ # storage and must permit the native refill writer.
+ text_tracker = tracker
+ if tracker.file_path == "database":
+ emptied_database_trackers[tracker.id] = tracker
+ text_tracker = None
# Only the text is left of a PDF that was a
# file. Keep the text and clear the dangling
# PDF reference.
@@ -1782,7 +1806,7 @@ class LibraryService:
# path that named the missing file.
self._keep_text_report_missing_pdf(
doc,
- tracker,
+ text_tracker,
tracker.file_path,
file_path,
stats,
@@ -1791,7 +1815,9 @@ class LibraryService:
# No download path asks for its PDF, but a
# later download of the same bytes stores
# the PDF in it (see _clear_missing_pdf).
- self._clear_missing_pdf(doc, tracker, stats)
+ self._clear_missing_pdf(
+ doc, text_tracker, stats
+ )
stats["files_found"] += 1
elif remaining is not None:
# Its PDF is still stored as a blob, or its text
@@ -1856,7 +1882,10 @@ class LibraryService:
remaining = self._remaining_content(
session, doc, settings_manager
)
- if remaining == "text" and _names_a_file(doc.file_path):
+ if remaining == "text" and (
+ _names_a_file(doc.file_path)
+ or self._sync_pdf_blob_size(session, doc.id) == 0
+ ):
# Its own PDF file is gone and only the text is
# left. Keep the text and clear the dangling
# PDF reference.
@@ -1866,11 +1895,15 @@ class LibraryService:
self._keep_text_report_missing_pdf(
doc,
tracker,
- doc.file_path,
- get_absolute_path_from_settings(
- doc.file_path,
- self.username,
- settings_manager=settings_manager,
+ doc.file_path or "database",
+ (
+ get_absolute_path_from_settings(
+ doc.file_path,
+ self.username,
+ settings_manager=settings_manager,
+ )
+ if _names_a_file(doc.file_path)
+ else None
),
stats,
)
@@ -1937,6 +1970,29 @@ class LibraryService:
# blob still keeps the record true. Every document under the
# tracker has been checked by now, so the order in which the
# sync read them does not change whether it is reset.
+ if emptied_database_trackers:
+ holders = (
+ session.query(Document)
+ .join(DocumentBlob, DocumentBlob.document_id == Document.id)
+ .filter(
+ Document.original_url.isnot(None),
+ func.length(DocumentBlob.pdf_binary) > 0,
+ )
+ .options(defer(Document.text_content))
+ .all()
+ )
+ for holder in holders:
+ # Use the same canonical/legacy lookups as the completed
+ # loop. A URL's shared record can remain true for another
+ # version, but only an identifying tracker can be reset.
+ for held_tracker in (
+ find_tracker_by_url(session, holder.original_url),
+ find_tracker_for_document(session, holder),
+ ):
+ if held_tracker is not None:
+ database_tracker_documents.setdefault(
+ held_tracker.id, []
+ ).append(holder.id)
removed = set(removed_doc_ids)
for tracker_id, tracker in emptied_database_trackers.items():
others = [
@@ -1946,7 +2002,10 @@ class LibraryService:
]
if others and (
session.query(DocumentBlob.document_id)
- .filter(DocumentBlob.document_id.in_(others))
+ .filter(
+ DocumentBlob.document_id.in_(others),
+ func.length(DocumentBlob.pdf_binary) > 0,
+ )
.first()
is not None
):
diff --git a/tests/research_library/services/test_library_sync_empty_content.py b/tests/research_library/services/test_library_sync_empty_content.py
new file mode 100644
index 0000000000..b3578a9668
--- /dev/null
+++ b/tests/research_library/services/test_library_sync_empty_content.py
@@ -0,0 +1,332 @@
+"""Sync distinguishes usable content from empty text and empty PDF storage."""
+
+import hashlib
+
+import pytest
+from sqlalchemy import event, inspect
+
+from local_deep_research.constants import FILE_PATH_TEXT_ONLY
+from local_deep_research.database.models.library import (
+ Document,
+ DocumentBlob,
+ DocumentChunk,
+ DocumentCollection,
+ DocumentStatus,
+)
+from local_deep_research.database.models.research import ResearchResource
+from local_deep_research.research_library.services.pdf_storage_manager import (
+ PDFStorageManager,
+)
+from local_deep_research.research_library.utils import (
+ document_awaits_pdf,
+ find_tracker_by_url,
+)
+from tests.research_library.test_library_sync_contracts import (
+ PDF_BYTES,
+ REGENERATED_PDF_BYTES,
+ SAVED_TEXT,
+ StubDownloader,
+ _add_chunk,
+ _cite_in_another_research,
+ _add_database_row_without_pdf,
+ db as db,
+ filesystem_mode as filesystem_mode,
+ make_library_service,
+ make_service,
+ one_download_session as one_download_session,
+ store as store,
+ world as world,
+)
+
+
+def add_document(db, world, *, text=None, content=PDF_BYTES, url=None):
+ url = url or "https://example.com/empty-content.pdf"
+ resource = world.add_resource(url, downloaded=True)
+ document = _add_database_row_without_pdf(
+ db, world, resource, world.research_id, url, content
+ )
+ row = db.session.get(Document, document)
+ row.text_content = text
+ db.session.get(ResearchResource, resource).document_id = document
+ tracker = find_tracker_by_url(db.session, url)
+ tracker.file_path = "database"
+ tracker.file_hash = hashlib.sha256(content).hexdigest()
+ db.session.commit()
+ return document, resource
+
+
+ "text",
+ [
+ "\t\n\r ",
+ "\u00a0\u2003\u202f",
+ "\v\f\x1c\x1d\x1e\x1f\x85\u1680\u2000\u2001\u2002\u2004\u2005\u2006\u2007\u2008\u2009\u200a\u2028\u2029\u205f\u3000",
+ ],
+)
+def test_whitespace_only_is_removed(db, store, world, monkeypatch, text):
+ document, _ = add_document(db, world, text=text)
+ stats = make_library_service(
+ db, store, monkeypatch
+ ).sync_library_with_filesystem()
+ with db.fresh() as session:
+ assert session.get(Document, document) is None, (
+ "whitespace is not usable extracted text"
+ )
+ assert stats["files_missing"] == 1
+
+
[email protected]("storage", ["filesystem", "database"])
[email protected]("tracked", [True, False])
+def test_empty_pdf_is_refilled_in_same_document(
+ db,
+ store,
+ world,
+ one_download_session,
+ filesystem_mode,
+ monkeypatch,
+ storage,
+ tracked,
+):
+ document, resource = add_document(db, world, text=SAVED_TEXT)
+ chunk = _add_chunk(db, document)
+ row = db.session.get(Document, document)
+ row.storage_mode = storage
+ if storage == "filesystem":
+ row.file_path = "pdfs/empty.pdf"
+ (store.root / row.file_path).write_bytes(b"")
+ else:
+ db.session.add(DocumentBlob(document_id=document, pdf_binary=b""))
+ tracker = find_tracker_by_url(db.session, row.original_url)
+ tracker.file_path = (row.file_path or "database") if tracked else None
+ db.session.commit()
+ library = make_library_service(db, store, monkeypatch)
+ stats = library.sync_library_with_filesystem()
+ assert stats["files_missing"] == 1, (
+ "a confirmed empty PDF must be eligible for refill"
+ )
+ with db.fresh() as session:
+ kept = session.get(Document, document)
+ assert kept.text_content == SAVED_TEXT
+ assert kept.file_path == FILE_PATH_TEXT_ONLY
+ assert document_awaits_pdf(kept)
+ assert session.get(DocumentChunk, chunk) is not None
+ assert (
+ session.query(DocumentCollection)
+ .filter_by(document_id=document)
+ .count()
+ == 1
+ )
+ assert session.get(ResearchResource, resource).document_id == document
+ downloader = StubDownloader(REGENERATED_PDF_BYTES)
+ service = make_service(store, downloader, storage_mode=storage)
+ assert service.download_resource(resource) == (True, None)
+ with db.fresh() as session:
+ kept = session.get(Document, document)
+ assert session.query(Document).count() == 1
+ assert (
+ PDFStorageManager(store.root, storage).load_pdf(kept, session)
+ == REGENERATED_PDF_BYTES
+ )
+ assert session.get(ResearchResource, resource).document_id == document
+ assert (
+ service.queue_research_downloads(
+ world.research_id, "library-sync-contracts"
+ )
+ == 0
+ )
+ assert downloader.urls == ["https://example.com/empty-content.pdf"]
+ assert library.sync_library_with_filesystem()["files_missing"] == 0
+
+
+ "status",
+ [DocumentStatus.FAILED, DocumentStatus.PENDING, DocumentStatus.PROCESSING],
+)
[email protected]("holder_first", [True, False])
+ "blob", [PDF_BYTES, b"", None], ids=["positive", "zero", "absent"]
+)
+def test_database_tracker_census_includes_noncompleted_holders(
+ db, store, world, monkeypatch, status, holder_first, blob
+):
+ url = "https://example.com/empty-content.pdf"
+ empty_resource = world.add_resource(url, downloaded=True)
+ holder_research, holder_resource = _cite_in_another_research(db, url)
+ tracker = find_tracker_by_url(db.session, url)
+ tracker.file_path = "database"
+ db.session.commit()
+
+ def holder():
+ document = _add_database_row_without_pdf(
+ db,
+ world,
+ holder_resource,
+ holder_research,
+ url,
+ REGENERATED_PDF_BYTES,
+ )
+ row = db.session.get(Document, document)
+ row.status = status
+ row.text_content = "retained holder text " * 100_000
+ db.session.get(ResearchResource, holder_resource).document_id = document
+ if blob is not None:
+ db.session.add(
+ DocumentBlob(
+ document_id=document,
+ pdf_binary=REGENERATED_PDF_BYTES if blob else b"",
+ )
+ )
+ db.session.commit()
+ return document
+
+ if holder_first:
+ holder_id = holder()
+ empty = _add_database_row_without_pdf(
+ db, world, empty_resource, world.research_id, url, PDF_BYTES
+ )
+ if not holder_first:
+ holder_id = holder()
+ library = make_library_service(db, store, monkeypatch)
+ stats = sync_without_loading_bodies(db, library)
+ with db.fresh() as session:
+ assert session.get(Document, empty) is None
+ assert session.get(Document, holder_id).status == status
+ tracker = find_tracker_by_url(
+ session, "https://example.com/empty-content.pdf"
+ )
+ assert (tracker.file_path, tracker.is_downloaded) == (
+ ("database", True) if blob else (None, False)
+ ), (
+ "a noncompleted holder's positive PDF must protect the shared tracker"
+ )
+ assert stats["trackers_updated"] == (0 if blob else 1)
+
+
+def sync_without_loading_bodies(db, library):
+ statements = []
+
+ def capture(
+ connection, cursor, statement, parameters, context, executemany
+ ):
+ statements.append(statement)
+
+ def check_deferred(session, instance):
+ if isinstance(instance, Document):
+ assert "text_content" in inspect(instance).unloaded
+
+ event.listen(db.engine, "before_cursor_execute", capture)
+ event.listen(type(db.session), "loaded_as_persistent", check_deferred)
+ try:
+ stats = library.sync_library_with_filesystem()
+ finally:
+ event.remove(db.engine, "before_cursor_execute", capture)
+ event.remove(type(db.session), "loaded_as_persistent", check_deferred)
+ # SQL may inspect text or blob length, but must not project either body.
+ for statement in statements:
+ projection = statement.lower().split("from", 1)[0]
+ assert "documents.text_content" not in projection, statement
+ assert "document_blobs.pdf_binary" not in projection.replace(
+ "length(document_blobs.pdf_binary)", ""
+ ), statement
+ return stats
+
+
+ "text",
+ ["普通の本文", "\t\u00a0 useful text \u2003\n", "x" * 2_000_000, "\u200b"],
+ ids=["unicode", "surrounded", "large", "non-whitespace-separator"],
+)
+def test_text_stays_deferred_and_nonblank_is_retained(
+ db, store, world, monkeypatch, text
+):
+ document, _ = add_document(db, world, text=text)
+ stats = sync_without_loading_bodies(
+ db, make_library_service(db, store, monkeypatch)
+ )
+ assert stats["files_missing"] == 0
+ with db.fresh() as session:
+ assert session.get(Document, document).text_content == text
+
+
[email protected]("storage", ["filesystem", "database"])
[email protected]("tracked", [True, False])
+def test_empty_pdf_without_text_is_deleted(
+ db, store, world, monkeypatch, storage, tracked
+):
+ document, _ = add_document(db, world)
+ row = db.session.get(Document, document)
+ row.storage_mode = storage
+ if storage == "filesystem":
+ row.file_path = "pdfs/zero.pdf"
+ (store.root / row.file_path).write_bytes(b"")
+ else:
+ db.session.add(DocumentBlob(document_id=document, pdf_binary=b""))
+ tracker = find_tracker_by_url(db.session, row.original_url)
+ tracker.file_path = (row.file_path or "database") if tracked else None
+ db.session.commit()
+ stats = make_library_service(
+ db, store, monkeypatch
+ ).sync_library_with_filesystem()
+ with db.fresh() as session:
+ assert session.get(Document, document) is None, (
+ "zero bytes cannot preserve an empty document"
+ )
+ assert stats["files_missing"] == 1
+
+
+ "identity", ["hash", "link", "conflicting-link", "conflicting-hash"]
+)
+def test_legacy_tracker_identity_is_not_overridden_by_raw_url(
+ db, store, world, monkeypatch, identity
+):
+ from local_deep_research.database.models.download_tracker import (
+ DownloadTracker,
+ )
+ from local_deep_research.research_library.utils import get_legacy_url_hashes
+
+ url = "https://example.com/identity.pdf?utm_source=legacy"
+ document, _ = add_document(db, world, url=url)
+ canonical = find_tracker_by_url(db.session, url)
+ canonical.file_hash = hashlib.sha256(b"newer version").hexdigest()
+ canonical_path = canonical.file_path
+ legacy = DownloadTracker(
+ url=url,
+ url_hash=get_legacy_url_hashes(url)[0],
+ first_resource_id=canonical.first_resource_id,
+ is_downloaded=True,
+ file_path="database",
+ file_hash=hashlib.sha256(PDF_BYTES).hexdigest()
+ if identity != "link"
+ else None,
+ library_document_id=document
+ if identity in ("link", "conflicting-hash")
+ else None,
+ )
+ if identity == "conflicting-link":
+ # Point the explicit link at a real, unrelated document.
+ other, _ = add_document(
+ db,
+ world,
+ url="https://example.com/unrelated.pdf",
+ content=REGENERATED_PDF_BYTES,
+ )
+ legacy.library_document_id = other
+ if identity == "conflicting-hash":
+ legacy.file_hash = canonical.file_hash
+ db.session.add(legacy)
+ db.session.commit()
+ canonical_id, legacy_id = canonical.id, legacy.id
+ make_library_service(db, store, monkeypatch).sync_library_with_filesystem()
+ with db.fresh() as session:
+ assert session.get(Document, document) is None
+ current = session.get(DownloadTracker, canonical_id)
+ assert (current.file_path, current.is_downloaded) == (
+ canonical_path,
+ True,
+ )
+ old = session.get(DownloadTracker, legacy_id)
+ identified = identity in ("hash", "link")
+ assert (old.file_path, old.is_downloaded) == (
+ (None, False) if identified else ("database", True)
+ )