Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

fix(library): guard sync stat() calls against file-vanish races (#7924) (#8065)

Ziyang Guo81233fdf490e
diff --git a/changelog.d/7924.bugfix.md b/changelog.d/7924.bugfix.md
new file mode 100644
index 0000000000..7358eddf0b
--- /dev/null
+++ b/changelog.d/7924.bugfix.md
@@ -0,0 +1 @@
+Library sync no longer aborts when a PDF file is deleted between its existence and size checks: the vanishing-file race is read as a missing file, while other filesystem errors (unreadable file, I/O failure) still stop the sync and roll back instead of treating existing files as gone.
diff --git a/src/local_deep_research/research_library/services/library_service.py b/src/local_deep_research/research_library/services/library_service.py
index f408782e76..10468426be 100644
--- a/src/local_deep_research/research_library/services/library_service.py
+++ b/src/local_deep_research/research_library/services/library_service.py
@@ -81,6 +81,25 @@ def _names_a_file(file_path: Optional[str]) -> bool:
)
+def _nonempty_file_size(path: Path) -> int:
+ """Size of ``path`` in bytes, or 0 when the file vanishes mid-race.
+
+ The sync checks ``is_file()`` and ``stat()`` in separate calls, so a
+ file deleted in between must not abort the whole run. That race
+ surfaces as ``FileNotFoundError`` (the file is gone) or
+ ``NotADirectoryError`` (its parent is), and both read as 0 — exactly
+ the pre-race outcome for a file that is already gone. Other
+ ``OSError`` s are NOT converted: ``is_file()`` raises for EACCES,
+ EIO and ESTALE on Python 3.12/3.13, and an unreadable-but-present
+ file is not a missing file — the sync should still abort and roll
+ back rather than treat existing PDFs as vanished.
+ """
+ try:
+ return path.stat().st_size if path.is_file() else 0
+ except (FileNotFoundError, NotADirectoryError):
+ return 0
+
+
def _tracker_describes_storage(
tracker_path: Optional[str], doc: Document
) -> bool:
@@ -240,7 +259,7 @@ class LibraryService:
resolved = get_absolute_path_from_settings(
doc.file_path, self.username, settings_manager=settings_manager
)
- if resolved and resolved.is_file() and resolved.stat().st_size:
+ if resolved and _nonempty_file_size(resolved):
return "pdf"
has_text = (
session.query(Document.id)
@@ -1785,8 +1804,7 @@ class LibraryService:
if (
_names_a_file(tracker.file_path)
and file_path
- and file_path.is_file()
- and file_path.stat().st_size
+ and _nonempty_file_size(file_path)
):
stats["files_found"] += 1
continue
diff --git a/tests/research_library/services/test_library_sync_stat_race.py b/tests/research_library/services/test_library_sync_stat_race.py
new file mode 100644
index 0000000000..326ab8fe0e
--- /dev/null
+++ b/tests/research_library/services/test_library_sync_stat_race.py
@@ -0,0 +1,206 @@
+"""A file vanishing between ``is_file()`` and ``stat()`` must not abort the sync (#7924).
+
+The census and the content probe both used the unguarded chain
+``path.is_file() and path.stat().st_size``; a deletion racing the two
+calls raised ``FileNotFoundError`` out of the sync loop. They now share
+``_nonempty_file_size``, which reads the race as "no usable file".
+"""
+
+import hashlib
+from pathlib import Path
+
+import pytest
+
+from local_deep_research.constants import FILE_PATH_TEXT_ONLY
+from local_deep_research.database.models.library import Document
+from local_deep_research.database.models.research import ResearchResource
+from local_deep_research.research_library.services.library_service import (
+ _nonempty_file_size,
+)
+from local_deep_research.research_library.utils import (
+ document_awaits_pdf,
+ find_tracker_by_url,
+)
+from tests.research_library.test_library_sync_contracts import (
+ PDF_BYTES,
+ REGENERATED_PDF_BYTES,
+ SAVED_TEXT,
+ _add_database_row_without_pdf,
+ db as db,
+ filesystem_mode as filesystem_mode,
+ make_library_service,
+ store as store,
+ world as world,
+)
+
+
+def test_size_of_present_file(tmp_path):
+ target = tmp_path / "doc.pdf"
+ target.write_bytes(b"pdf-bytes")
+ assert _nonempty_file_size(target) == len(b"pdf-bytes")
+
+
+def test_size_of_missing_file(tmp_path):
+ assert _nonempty_file_size(tmp_path / "gone.pdf") == 0
+
+
+def test_size_of_directory(tmp_path):
+ assert _nonempty_file_size(tmp_path) == 0
+
+
+def test_size_when_file_vanishes_between_isfile_and_stat(tmp_path, monkeypatch):
+ """TOCTOU regression pin: ``is_file()`` wins, ``stat()`` finds it gone."""
+ target = tmp_path / "vanishing.pdf"
+ target.write_bytes(b"pdf-bytes")
+
+ monkeypatch.setattr(Path, "is_file", lambda self: True)
+
+ def vanished(self, *args, **kwargs):
+ raise FileNotFoundError(target)
+
+ monkeypatch.setattr(Path, "stat", vanished)
+
+ assert _nonempty_file_size(target) == 0
+
+
+def test_size_when_parent_directory_vanishes_between_isfile_and_stat(
+ tmp_path, monkeypatch
+):
+ """The race also fires through a deleted parent directory.
+
+ ``stat()`` against a path whose parent no longer exists raises
+ ``NotADirectoryError`` (ENOTDIR on traversal), not
+ ``FileNotFoundError``; the sync treats both as "no usable file".
+ """
+ target = tmp_path / "doc.pdf"
+ target.write_bytes(b"pdf-bytes")
+
+ monkeypatch.setattr(Path, "is_file", lambda self: True)
+
+ def vanished(self, *args, **kwargs):
+ raise NotADirectoryError(target)
+
+ monkeypatch.setattr(Path, "stat", vanished)
+
+ assert _nonempty_file_size(target) == 0
+
+
+def test_unreadable_file_still_raises(tmp_path, monkeypatch):
+ """Only the vanish race is swallowed — other ``OSError`` s propagate.
+
+ A file whose lookup fails with EACCES/EIO is present but
+ unreadable, not missing: converting that to 0 would make the sync
+ treat existing PDFs as vanished (text-only conversion or deletion).
+ The sync must keep aborting and rolling back on those.
+ """
+ target = tmp_path / "locked.pdf"
+ target.write_bytes(b"pdf-bytes")
+
+ monkeypatch.setattr(Path, "is_file", lambda self: True)
+
+ def io_error(self, *args, **kwargs):
+ raise PermissionError(target)
+
+ monkeypatch.setattr(Path, "stat", io_error)
+
+ with pytest.raises(PermissionError):
+ _nonempty_file_size(target)
+
+
+# --------------------------------------------------------------------------
+# Sync-level regression: drive the real sync, not just the helper.
+#
+# The unit tests above pin _nonempty_file_size's behaviour directly; a
+# revert of either call site (the census probe or _remaining_content)
+# would still leave them green. The test below runs the production
+# ``sync_library_with_filesystem`` over a library whose document vanishes
+# mid-race and asserts the run completes and the other document is still
+# processed.
+# --------------------------------------------------------------------------
+
+
+def _add_filesystem_document(db, world, *, url, filename, content, text):
+ """A completed filesystem-mode download whose PDF lives at pdfs/<filename>."""
+ resource = world.add_resource(url, downloaded=True)
+ document = _add_database_row_without_pdf(
+ db, world, resource, world.research_id, url, content
+ )
+ row = db.session.get(Document, document)
+ row.storage_mode = "filesystem"
+ row.file_path = f"pdfs/{filename}"
+ row.text_content = text
+ db.session.get(ResearchResource, resource).document_id = document
+ tracker = find_tracker_by_url(db.session, url)
+ tracker.file_path = row.file_path
+ tracker.file_hash = hashlib.sha256(content).hexdigest()
+ db.session.commit()
+ return document
+
+
+def test_sync_survives_vanish_race_and_still_serves_neighbours(
+ db, store, world, filesystem_mode, monkeypatch
+):
+ """The census's ``is_file()``/``stat()`` race, end to end.
+
+ The raced document's PDF answers ``is_file()`` with True and is gone
+ by the time ``stat()`` runs. The unguarded chain raised
+ ``FileNotFoundError`` out of the sync loop, aborting the whole run;
+ the sync must instead read the race as "no usable file" — keep the
+ raced document's retained text as text-only — and go on to find its
+ healthy neighbour's PDF.
+ """
+ raced = _add_filesystem_document(
+ db,
+ world,
+ url="https://example.com/vanish-race.pdf",
+ filename="race.pdf",
+ content=PDF_BYTES,
+ text=SAVED_TEXT,
+ )
+ healthy = _add_filesystem_document(
+ db,
+ world,
+ url="https://example.com/vanish-healthy.pdf",
+ filename="healthy.pdf",
+ # Other bytes than the raced document: document_hash is unique.
+ content=REGENERATED_PDF_BYTES,
+ text=SAVED_TEXT,
+ )
+ (store.root / "pdfs" / "race.pdf").write_bytes(PDF_BYTES)
+ (store.root / "pdfs" / "healthy.pdf").write_bytes(REGENERATED_PDF_BYTES)
+
+ # Reproduce the mid-race state without patching ``stat`` itself: the
+ # raced file is deleted for real, while ``is_file()`` still answers
+ # from the stale pre-deletion entry (True). Real ``stat()`` then
+ # raises the real ``FileNotFoundError`` the unguarded chain died on,
+ # and every other path in the sync keeps its honest filesystem view.
+ (store.root / "pdfs" / "race.pdf").unlink()
+
+ real_is_file = Path.is_file
+
+ def racing_is_file(self):
+ if self.name == "race.pdf":
+ return True
+ return real_is_file(self)
+
+ monkeypatch.setattr(Path, "is_file", racing_is_file)
+
+ stats = make_library_service(
+ db, store, monkeypatch
+ ).sync_library_with_filesystem()
+
+ assert stats["files_found"] == 1, (
+ "the healthy neighbour must still be counted; an aborted run "
+ "would report neither document"
+ )
+ assert stats["files_missing"] == 1
+ with db.fresh() as session:
+ kept = session.get(Document, raced)
+ assert kept is not None, (
+ "a document with retained text must survive the race as text-only"
+ )
+ assert kept.file_path == FILE_PATH_TEXT_ONLY
+ assert kept.text_content == SAVED_TEXT
+ assert document_awaits_pdf(kept)
+ intact = session.get(Document, healthy)
+ assert intact.file_path == "pdfs/healthy.pdf"