~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
fix(library): do not mark a document indexed when no embeddings were stored (#5063)
Ehsan3bc3373292c7
diff --git a/changelog.d/4855.bugfix.md b/changelog.d/4855.bugfix.mdnew file mode 100644index 0000000000..8862900d73--- /dev/null+++ b/changelog.d/4855.bugfix.md@@ -0,0 +1 @@+A document whose embeddings failed to store is no longer marked as indexed. Previously a chunk-store error left the document flagged indexed but with zero vectors persisted, so it was silently unsearchable and the background reconciler (which only retries un-indexed documents) never picked it up again. The failed index now leaves the document un-indexed so it is retried.diff --git a/src/local_deep_research/research_library/services/library_rag_service.py b/src/local_deep_research/research_library/services/library_rag_service.pyindex e7bde612d9..3b3b5a3c34 100644--- a/src/local_deep_research/research_library/services/library_rag_service.py+++ b/src/local_deep_research/research_library/services/library_rag_service.py@@ -1033,6 +1033,28 @@ class LibraryRAGService:source_id=document_id,)+ # Guard the "chunks produced but nothing stored" case.+ # _store_chunks_to_db appends one id per chunk on success+ # (a reused id for a pre-existing chunk, a fresh id for a new+ # one), so len(embedding_ids) == len(chunks); it returns []+ # ONLY when it caught an exception. A non-empty chunk set with+ # no embedding ids therefore means the store failed: no vectors+ # were persisted. Do NOT fall through to mark the document+ # indexed=True: a document flagged indexed must be searchable+ # (its embeddings must exist), and the background reconciler+ # only retries rows with indexed=False. Marking it indexed here+ # would leave it silently unsearchable and never retried.+ if chunks and not embedding_ids:+ logger.error(+ f"Chunk store returned no embeddings for document "+ f"{document_id} in collection {collection_id} "+ f"({len(chunks)} chunks); leaving it unindexed for retry"+ )+ return {+ "status": "error",+ "error": "Failed to store chunks: no embeddings were persisted",+ }+# Load or create FAISS index (lazy; the merge step# below will reload from disk under the lock anyway).if self.faiss_index is None:diff --git a/tests/research_library/services/test_library_rag_service_index_failure.py b/tests/research_library/services/test_library_rag_service_index_failure.pynew file mode 100644index 0000000000..6de4f7da92--- /dev/null+++ b/tests/research_library/services/test_library_rag_service_index_failure.py@@ -0,0 +1,117 @@+"""Regression test for #4855: a document whose chunk store persisted zero+embeddings must not be marked ``indexed=True``.++``LocalEmbeddingManager._store_chunks_to_db`` appends exactly one id per chunk+on success (a reused id for a pre-existing chunk, a fresh id for a new one), so+``len(embedding_ids) == len(chunks)``; it returns ``[]`` only when nothing was+persisted (it caught an exception, or there was no user context). Before this+fix ``index_document`` ignored that empty return: it ran the FAISS merge (which+adds zero vectors), then still set ``DocumentCollection.indexed=True`` and+returned ``status="success"``. The document was silently unsearchable, and the+background reconciler (which retries only ``indexed.is_(False)`` rows) never+picked it up again.++Invariant: a document flagged ``indexed=True`` has had its embeddings+persisted. A failed store must leave it ``indexed=False`` so the reconciler+retries it.+"""++from unittest.mock import MagicMock, patch++_MOD = "local_deep_research.research_library.services.library_rag_service"+++def _make_service(**overrides):+ """Instantiate LibraryRAGService with all heavy deps mocked out."""+ with (+ patch(f"{_MOD}.LocalEmbeddingManager") as _lem,+ patch(f"{_MOD}.get_user_db_session"),+ patch(f"{_MOD}.FileIntegrityManager"),+ patch(f"{_MOD}.get_text_splitter"),+ ):+ _lem.return_value.embeddings = MagicMock()+ from local_deep_research.research_library.services.library_rag_service import (+ LibraryRAGService,+ )++ defaults = dict(username="testuser", db_password="pw")+ defaults.update(overrides)+ return LibraryRAGService(**defaults)+++def _ctx_yielding(session):+ """Context-manager mock that yields *session*."""+ ctx = MagicMock()+ ctx.__enter__ = MagicMock(return_value=session)+ ctx.__exit__ = MagicMock(return_value=False)+ return ctx+++def _make_doc(text="Long text content for indexing test."):+ doc = MagicMock()+ doc.id = "doc-1"+ doc.text_content = text+ doc.title = "Test"+ doc.filename = None+ doc.original_url = "http://example.com/d"+ doc.authors = None+ doc.published_date = None+ doc.doi = None+ doc.arxiv_id = None+ doc.pmid = None+ doc.pmcid = None+ doc.extraction_method = None+ doc.word_count = None+ return doc+++@patch(f"{_MOD}.ensure_in_collection")+@patch(f"{_MOD}.get_user_db_session")+def test_index_document_not_marked_indexed_when_store_persists_no_embeddings(+ mock_get_session, mock_ensure+):+ """The chunk store yields a real chunk set but returns no embedding ids+ (its exception path). index_document must report an error, must not+ commit, and must never flip DocumentCollection.indexed to True."""+ svc = _make_service()++ session = MagicMock()+ mock_get_session.return_value = _ctx_yielding(session)+ session.query.return_value.filter_by.return_value.first.return_value = (+ _make_doc()+ )+ mock_ensure.return_value = MagicMock(indexed=False, chunk_count=0)++ # The splitter produces a real (non-empty) chunk set...+ svc.text_splitter = MagicMock()+ svc.text_splitter.split_documents.return_value = [MagicMock()]++ # ...but the DB store persisted nothing and returned []. The remaining+ # mocks let the *unfixed* code path run all the way to the indexed=True+ # update and a "success" return, so the test proves the guard, not a+ # crash somewhere downstream.+ svc.embedding_manager = MagicMock()+ svc.embedding_manager._store_chunks_to_db.return_value = []+ svc.faiss_index = MagicMock()+ svc.rag_index_record = MagicMock()+ svc.rag_index_record.index_path = "/tmp/idx-4855"+ svc.rag_index_record.id = 1+ svc._merge_and_persist_locked = MagicMock(return_value={"added": 0})++ result = svc.index_document("doc-1", "coll-1")++ assert result["status"] == "error"+ session.commit.assert_not_called()++ indexed_true_updates = [+ c+ for c in session.mock_calls+ if c[0].endswith(".update")+ and c.args+ and isinstance(c.args[0], dict)+ and c.args[0].get("indexed") is True+ ]+ assert indexed_true_updates == [], (+ "index_document set DocumentCollection.indexed=True despite the chunk "+ "store persisting no embeddings"+ )