~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
fix(library): guard sync stat() calls against file-vanish races (#7924) (#8065)
Ziyang Guo81233fdf490e
diff --git a/changelog.d/7924.bugfix.md b/changelog.d/7924.bugfix.mdnew file mode 100644index 0000000000..7358eddf0b--- /dev/null+++ b/changelog.d/7924.bugfix.md@@ -0,0 +1 @@+Library sync no longer aborts when a PDF file is deleted between its existence and size checks: the vanishing-file race is read as a missing file, while other filesystem errors (unreadable file, I/O failure) still stop the sync and roll back instead of treating existing files as gone.diff --git a/src/local_deep_research/research_library/services/library_service.py b/src/local_deep_research/research_library/services/library_service.pyindex f408782e76..10468426be 100644--- a/src/local_deep_research/research_library/services/library_service.py+++ b/src/local_deep_research/research_library/services/library_service.py@@ -81,6 +81,25 @@ def _names_a_file(file_path: Optional[str]) -> bool:)+def _nonempty_file_size(path: Path) -> int:+ """Size of ``path`` in bytes, or 0 when the file vanishes mid-race.++ The sync checks ``is_file()`` and ``stat()`` in separate calls, so a+ file deleted in between must not abort the whole run. That race+ surfaces as ``FileNotFoundError`` (the file is gone) or+ ``NotADirectoryError`` (its parent is), and both read as 0 — exactly+ the pre-race outcome for a file that is already gone. Other+ ``OSError`` s are NOT converted: ``is_file()`` raises for EACCES,+ EIO and ESTALE on Python 3.12/3.13, and an unreadable-but-present+ file is not a missing file — the sync should still abort and roll+ back rather than treat existing PDFs as vanished.+ """+ try:+ return path.stat().st_size if path.is_file() else 0+ except (FileNotFoundError, NotADirectoryError):+ return 0++def _tracker_describes_storage(tracker_path: Optional[str], doc: Document) -> bool:@@ -240,7 +259,7 @@ class LibraryService:resolved = get_absolute_path_from_settings(doc.file_path, self.username, settings_manager=settings_manager)- if resolved and resolved.is_file() and resolved.stat().st_size:+ if resolved and _nonempty_file_size(resolved):return "pdf"has_text = (session.query(Document.id)@@ -1785,8 +1804,7 @@ class LibraryService:if (_names_a_file(tracker.file_path)and file_path- and file_path.is_file()- and file_path.stat().st_size+ and _nonempty_file_size(file_path)):stats["files_found"] += 1continuediff --git a/tests/research_library/services/test_library_sync_stat_race.py b/tests/research_library/services/test_library_sync_stat_race.pynew file mode 100644index 0000000000..326ab8fe0e--- /dev/null+++ b/tests/research_library/services/test_library_sync_stat_race.py@@ -0,0 +1,206 @@+"""A file vanishing between ``is_file()`` and ``stat()`` must not abort the sync (#7924).++The census and the content probe both used the unguarded chain+``path.is_file() and path.stat().st_size``; a deletion racing the two+calls raised ``FileNotFoundError`` out of the sync loop. They now share+``_nonempty_file_size``, which reads the race as "no usable file".+"""++import hashlib+from pathlib import Path++import pytest++from local_deep_research.constants import FILE_PATH_TEXT_ONLY+from local_deep_research.database.models.library import Document+from local_deep_research.database.models.research import ResearchResource+from local_deep_research.research_library.services.library_service import (+ _nonempty_file_size,+)+from local_deep_research.research_library.utils import (+ document_awaits_pdf,+ find_tracker_by_url,+)+from tests.research_library.test_library_sync_contracts import (+ PDF_BYTES,+ REGENERATED_PDF_BYTES,+ SAVED_TEXT,+ _add_database_row_without_pdf,+ db as db,+ filesystem_mode as filesystem_mode,+ make_library_service,+ store as store,+ world as world,+)+++def test_size_of_present_file(tmp_path):+ target = tmp_path / "doc.pdf"+ target.write_bytes(b"pdf-bytes")+ assert _nonempty_file_size(target) == len(b"pdf-bytes")+++def test_size_of_missing_file(tmp_path):+ assert _nonempty_file_size(tmp_path / "gone.pdf") == 0+++def test_size_of_directory(tmp_path):+ assert _nonempty_file_size(tmp_path) == 0+++def test_size_when_file_vanishes_between_isfile_and_stat(tmp_path, monkeypatch):+ """TOCTOU regression pin: ``is_file()`` wins, ``stat()`` finds it gone."""+ target = tmp_path / "vanishing.pdf"+ target.write_bytes(b"pdf-bytes")++ monkeypatch.setattr(Path, "is_file", lambda self: True)++ def vanished(self, *args, **kwargs):+ raise FileNotFoundError(target)++ monkeypatch.setattr(Path, "stat", vanished)++ assert _nonempty_file_size(target) == 0+++def test_size_when_parent_directory_vanishes_between_isfile_and_stat(+ tmp_path, monkeypatch+):+ """The race also fires through a deleted parent directory.++ ``stat()`` against a path whose parent no longer exists raises+ ``NotADirectoryError`` (ENOTDIR on traversal), not+ ``FileNotFoundError``; the sync treats both as "no usable file".+ """+ target = tmp_path / "doc.pdf"+ target.write_bytes(b"pdf-bytes")++ monkeypatch.setattr(Path, "is_file", lambda self: True)++ def vanished(self, *args, **kwargs):+ raise NotADirectoryError(target)++ monkeypatch.setattr(Path, "stat", vanished)++ assert _nonempty_file_size(target) == 0+++def test_unreadable_file_still_raises(tmp_path, monkeypatch):+ """Only the vanish race is swallowed — other ``OSError`` s propagate.++ A file whose lookup fails with EACCES/EIO is present but+ unreadable, not missing: converting that to 0 would make the sync+ treat existing PDFs as vanished (text-only conversion or deletion).+ The sync must keep aborting and rolling back on those.+ """+ target = tmp_path / "locked.pdf"+ target.write_bytes(b"pdf-bytes")++ monkeypatch.setattr(Path, "is_file", lambda self: True)++ def io_error(self, *args, **kwargs):+ raise PermissionError(target)++ monkeypatch.setattr(Path, "stat", io_error)++ with pytest.raises(PermissionError):+ _nonempty_file_size(target)+++# --------------------------------------------------------------------------+# Sync-level regression: drive the real sync, not just the helper.+#+# The unit tests above pin _nonempty_file_size's behaviour directly; a+# revert of either call site (the census probe or _remaining_content)+# would still leave them green. The test below runs the production+# ``sync_library_with_filesystem`` over a library whose document vanishes+# mid-race and asserts the run completes and the other document is still+# processed.+# --------------------------------------------------------------------------+++def _add_filesystem_document(db, world, *, url, filename, content, text):+ """A completed filesystem-mode download whose PDF lives at pdfs/<filename>."""+ resource = world.add_resource(url, downloaded=True)+ document = _add_database_row_without_pdf(+ db, world, resource, world.research_id, url, content+ )+ row = db.session.get(Document, document)+ row.storage_mode = "filesystem"+ row.file_path = f"pdfs/{filename}"+ row.text_content = text+ db.session.get(ResearchResource, resource).document_id = document+ tracker = find_tracker_by_url(db.session, url)+ tracker.file_path = row.file_path+ tracker.file_hash = hashlib.sha256(content).hexdigest()+ db.session.commit()+ return document+++def test_sync_survives_vanish_race_and_still_serves_neighbours(+ db, store, world, filesystem_mode, monkeypatch+):+ """The census's ``is_file()``/``stat()`` race, end to end.++ The raced document's PDF answers ``is_file()`` with True and is gone+ by the time ``stat()`` runs. The unguarded chain raised+ ``FileNotFoundError`` out of the sync loop, aborting the whole run;+ the sync must instead read the race as "no usable file" — keep the+ raced document's retained text as text-only — and go on to find its+ healthy neighbour's PDF.+ """+ raced = _add_filesystem_document(+ db,+ world,+ url="https://example.com/vanish-race.pdf",+ filename="race.pdf",+ content=PDF_BYTES,+ text=SAVED_TEXT,+ )+ healthy = _add_filesystem_document(+ db,+ world,+ url="https://example.com/vanish-healthy.pdf",+ filename="healthy.pdf",+ # Other bytes than the raced document: document_hash is unique.+ content=REGENERATED_PDF_BYTES,+ text=SAVED_TEXT,+ )+ (store.root / "pdfs" / "race.pdf").write_bytes(PDF_BYTES)+ (store.root / "pdfs" / "healthy.pdf").write_bytes(REGENERATED_PDF_BYTES)++ # Reproduce the mid-race state without patching ``stat`` itself: the+ # raced file is deleted for real, while ``is_file()`` still answers+ # from the stale pre-deletion entry (True). Real ``stat()`` then+ # raises the real ``FileNotFoundError`` the unguarded chain died on,+ # and every other path in the sync keeps its honest filesystem view.+ (store.root / "pdfs" / "race.pdf").unlink()++ real_is_file = Path.is_file++ def racing_is_file(self):+ if self.name == "race.pdf":+ return True+ return real_is_file(self)++ monkeypatch.setattr(Path, "is_file", racing_is_file)++ stats = make_library_service(+ db, store, monkeypatch+ ).sync_library_with_filesystem()++ assert stats["files_found"] == 1, (+ "the healthy neighbour must still be counted; an aborted run "+ "would report neither document"+ )+ assert stats["files_missing"] == 1+ with db.fresh() as session:+ kept = session.get(Document, raced)+ assert kept is not None, (+ "a document with retained text must survive the race as text-only"+ )+ assert kept.file_path == FILE_PATH_TEXT_ONLY+ assert kept.text_content == SAVED_TEXT+ assert document_awaits_pdf(kept)+ intact = session.get(Document, healthy)+ assert intact.file_path == "pdfs/healthy.pdf"