Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

fix(library): do not mark a document indexed when no embeddings were stored (#5063)

Ehsan3bc3373292c7
diff --git a/changelog.d/4855.bugfix.md b/changelog.d/4855.bugfix.md
new file mode 100644
index 0000000000..8862900d73
--- /dev/null
+++ b/changelog.d/4855.bugfix.md
@@ -0,0 +1 @@
+A document whose embeddings failed to store is no longer marked as indexed. Previously a chunk-store error left the document flagged indexed but with zero vectors persisted, so it was silently unsearchable and the background reconciler (which only retries un-indexed documents) never picked it up again. The failed index now leaves the document un-indexed so it is retried.
diff --git a/src/local_deep_research/research_library/services/library_rag_service.py b/src/local_deep_research/research_library/services/library_rag_service.py
index e7bde612d9..3b3b5a3c34 100644
--- a/src/local_deep_research/research_library/services/library_rag_service.py
+++ b/src/local_deep_research/research_library/services/library_rag_service.py
@@ -1033,6 +1033,28 @@ class LibraryRAGService:
source_id=document_id,
)
+ # Guard the "chunks produced but nothing stored" case.
+ # _store_chunks_to_db appends one id per chunk on success
+ # (a reused id for a pre-existing chunk, a fresh id for a new
+ # one), so len(embedding_ids) == len(chunks); it returns []
+ # ONLY when it caught an exception. A non-empty chunk set with
+ # no embedding ids therefore means the store failed: no vectors
+ # were persisted. Do NOT fall through to mark the document
+ # indexed=True: a document flagged indexed must be searchable
+ # (its embeddings must exist), and the background reconciler
+ # only retries rows with indexed=False. Marking it indexed here
+ # would leave it silently unsearchable and never retried.
+ if chunks and not embedding_ids:
+ logger.error(
+ f"Chunk store returned no embeddings for document "
+ f"{document_id} in collection {collection_id} "
+ f"({len(chunks)} chunks); leaving it unindexed for retry"
+ )
+ return {
+ "status": "error",
+ "error": "Failed to store chunks: no embeddings were persisted",
+ }
+
# Load or create FAISS index (lazy; the merge step
# below will reload from disk under the lock anyway).
if self.faiss_index is None:
diff --git a/tests/research_library/services/test_library_rag_service_index_failure.py b/tests/research_library/services/test_library_rag_service_index_failure.py
new file mode 100644
index 0000000000..6de4f7da92
--- /dev/null
+++ b/tests/research_library/services/test_library_rag_service_index_failure.py
@@ -0,0 +1,117 @@
+"""Regression test for #4855: a document whose chunk store persisted zero
+embeddings must not be marked ``indexed=True``.
+
+``LocalEmbeddingManager._store_chunks_to_db`` appends exactly one id per chunk
+on success (a reused id for a pre-existing chunk, a fresh id for a new one), so
+``len(embedding_ids) == len(chunks)``; it returns ``[]`` only when nothing was
+persisted (it caught an exception, or there was no user context). Before this
+fix ``index_document`` ignored that empty return: it ran the FAISS merge (which
+adds zero vectors), then still set ``DocumentCollection.indexed=True`` and
+returned ``status="success"``. The document was silently unsearchable, and the
+background reconciler (which retries only ``indexed.is_(False)`` rows) never
+picked it up again.
+
+Invariant: a document flagged ``indexed=True`` has had its embeddings
+persisted. A failed store must leave it ``indexed=False`` so the reconciler
+retries it.
+"""
+
+from unittest.mock import MagicMock, patch
+
+_MOD = "local_deep_research.research_library.services.library_rag_service"
+
+
+def _make_service(**overrides):
+ """Instantiate LibraryRAGService with all heavy deps mocked out."""
+ with (
+ patch(f"{_MOD}.LocalEmbeddingManager") as _lem,
+ patch(f"{_MOD}.get_user_db_session"),
+ patch(f"{_MOD}.FileIntegrityManager"),
+ patch(f"{_MOD}.get_text_splitter"),
+ ):
+ _lem.return_value.embeddings = MagicMock()
+ from local_deep_research.research_library.services.library_rag_service import (
+ LibraryRAGService,
+ )
+
+ defaults = dict(username="testuser", db_password="pw")
+ defaults.update(overrides)
+ return LibraryRAGService(**defaults)
+
+
+def _ctx_yielding(session):
+ """Context-manager mock that yields *session*."""
+ ctx = MagicMock()
+ ctx.__enter__ = MagicMock(return_value=session)
+ ctx.__exit__ = MagicMock(return_value=False)
+ return ctx
+
+
+def _make_doc(text="Long text content for indexing test."):
+ doc = MagicMock()
+ doc.id = "doc-1"
+ doc.text_content = text
+ doc.title = "Test"
+ doc.filename = None
+ doc.original_url = "http://example.com/d"
+ doc.authors = None
+ doc.published_date = None
+ doc.doi = None
+ doc.arxiv_id = None
+ doc.pmid = None
+ doc.pmcid = None
+ doc.extraction_method = None
+ doc.word_count = None
+ return doc
+
+
+@patch(f"{_MOD}.ensure_in_collection")
+@patch(f"{_MOD}.get_user_db_session")
+def test_index_document_not_marked_indexed_when_store_persists_no_embeddings(
+ mock_get_session, mock_ensure
+):
+ """The chunk store yields a real chunk set but returns no embedding ids
+ (its exception path). index_document must report an error, must not
+ commit, and must never flip DocumentCollection.indexed to True."""
+ svc = _make_service()
+
+ session = MagicMock()
+ mock_get_session.return_value = _ctx_yielding(session)
+ session.query.return_value.filter_by.return_value.first.return_value = (
+ _make_doc()
+ )
+ mock_ensure.return_value = MagicMock(indexed=False, chunk_count=0)
+
+ # The splitter produces a real (non-empty) chunk set...
+ svc.text_splitter = MagicMock()
+ svc.text_splitter.split_documents.return_value = [MagicMock()]
+
+ # ...but the DB store persisted nothing and returned []. The remaining
+ # mocks let the *unfixed* code path run all the way to the indexed=True
+ # update and a "success" return, so the test proves the guard, not a
+ # crash somewhere downstream.
+ svc.embedding_manager = MagicMock()
+ svc.embedding_manager._store_chunks_to_db.return_value = []
+ svc.faiss_index = MagicMock()
+ svc.rag_index_record = MagicMock()
+ svc.rag_index_record.index_path = "/tmp/idx-4855"
+ svc.rag_index_record.id = 1
+ svc._merge_and_persist_locked = MagicMock(return_value={"added": 0})
+
+ result = svc.index_document("doc-1", "coll-1")
+
+ assert result["status"] == "error"
+ session.commit.assert_not_called()
+
+ indexed_true_updates = [
+ c
+ for c in session.mock_calls
+ if c[0].endswith(".update")
+ and c.args
+ and isinstance(c.args[0], dict)
+ and c.args[0].get("indexed") is True
+ ]
+ assert indexed_true_updates == [], (
+ "index_document set DocumentCollection.indexed=True despite the chunk "
+ "store persisting no embeddings"
+ )