~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
fix(rag): preserve text invalidation through indexing and reconciliation (#7307)
LearningCircuit503f6f76ec0f
diff --git a/changelog.d/+library-text-index-invalidation.bugfix.md b/changelog.d/+library-text-index-invalidation.bugfix.mdnew file mode 100644index 0000000000..d40c6c15b2--- /dev/null+++ b/changelog.d/+library-text-index-invalidation.bugfix.md@@ -0,0 +1 @@+Keep improved document text pending for reindexing even when older vectors still exist. Index persistence now serializes with text replacement, and reconciliation preserves invalidated chunk records until a successful reindex replaces their vectors.diff --git a/src/local_deep_research/research_library/services/download_service.py b/src/local_deep_research/research_library/services/download_service.pyindex 703e1fec9c..e4e24240e7 100644--- a/src/local_deep_research/research_library/services/download_service.py+++ b/src/local_deep_research/research_library/services/download_service.py@@ -23,7 +23,7 @@ import requestsfrom loguru import loggerfrom ...security import redact_url_for_logfrom sqlalchemy import func-from sqlalchemy.orm import Session+from sqlalchemy.orm import Session, load_onlyimport pdfplumberfrom pypdf import PdfReader@@ -56,6 +56,7 @@ from ...security.secure_logging import log_exception_typefrom ...database.models.library import (Collection,Document as Document,+ DocumentChunk,DocumentCollection,DocumentStatus,DownloadQueue as LibraryDownloadQueue,@@ -3143,6 +3144,19 @@ class DownloadService:return Noneif old_text != text:+ # Keep the vector IDs for replacement, but persist why+ # these chunks are pending. Live vectors alone must not+ # let reconciliation mark an older extraction current.+ for chunk in (+ session.query(DocumentChunk)+ .options(load_only(DocumentChunk.document_metadata))+ .filter_by(source_type="document", source_id=doc.id)+ .all()+ ):+ chunk.document_metadata = {+ **(chunk.document_metadata or {}),+ "_ldr_text_invalidated": True,+ }# The legacy flag and canonical per-index marker must both# move with the text, in this transaction. Leave chunk rows# available for replace-on-reindex to remove their vectors.diff --git a/src/local_deep_research/research_library/services/library_rag_service.py b/src/local_deep_research/research_library/services/library_rag_service.pyindex 7f26532d50..89d5e25ef1 100644--- a/src/local_deep_research/research_library/services/library_rag_service.py+++ b/src/local_deep_research/research_library/services/library_rag_service.py@@ -65,6 +65,25 @@ from langchain_core.embeddings import Embeddingsimport hashlib+def _claim_document_snapshot(session, document_id, text_content, document_hash):+ """Keep the validated SQLite snapshot stable until the index commit.++ SELECT FOR UPDATE is ignored by SQLite. A conditional no-op write either+ detects a writer that already won or holds the write lock through chunk+ and status persistence. Call after vector initialization, which may commit+ index setup, and never before embedding.+ """+ return (+ session.query(Document)+ .filter(+ Document.id == document_id,+ Document.text_content == text_content,+ Document.document_hash == document_hash,+ )+ .update({Document.id: document_id}, synchronize_session=False)+ )++@dataclassclass _PreparedDocument:document_id: str@@ -1998,9 +2017,24 @@ class LibraryRAGService:}collection_name = f"collection_{collection_id}"+ observed_text = current_doc.text_content+ observed_hash = current_doc.document_hashvindex = self._get_vector_index(collection_id, collection_name, reset_stale_state=True)+ if (+ _claim_document_snapshot(+ session, document_id, observed_text, observed_hash+ )+ == 0+ ):+ self._flag_document_for_reindex(+ session, document_id, collection_id+ )+ return {+ "status": "stale",+ "message": "Document modified during indexing, flagged for reindex",+ }stats = vindex.index_prepared(source_type="document",source_id=document_id,@@ -2531,10 +2565,26 @@ class LibraryRAGService:"message": "Document indexed state changed during indexing, flagged for reindex",}+ observed_text = current_doc.text_content+ observed_hash = current_doc.document_hashvindex = self._get_vector_index(collection_id, collection_name, reset_stale_state=True)+ if (+ _claim_document_snapshot(+ session, document_id, observed_text, observed_hash+ )+ == 0+ ):+ self._flag_document_for_reindex(+ session, document_id, collection_id+ )+ return {+ "status": "stale",+ "message": "Document modified during indexing, flagged for reindex",+ }+if hasattr(vindex, "index_prepared"):index_stats = vindex.index_prepared(source_type="document",@@ -3575,8 +3625,9 @@ class LibraryRAGService:"""Reconcile indexed flags and stats against durable FAISS membership.A document is indexed only when it has at least one current-model chunk- row and every such chunk id exists in the live FAISS index. Stale status- rows are removed and compatibility flags/stats are rebuilt atomically.+ row, none were invalidated by text replacement, and every such chunk+ id exists in the live FAISS index. Stale status rows are removed and+ compatibility flags/stats are rebuilt atomically.Safety invariant: if the live FAISS index reports zero ids (transientload error, missing file, on-disk corruption the verifier just@@ -3612,8 +3663,18 @@ class LibraryRAGService:with get_user_db_session(self.username, self.db_password) as session:+ # Serialize this SQLite repair with text invalidation. The+ # vector store was initialized above; no nested setup commit+ # may release this write lock before flags/status are saved.+ session.query(RAGIndex).filter_by(id=rag_index_id).update(+ {RAGIndex.id: rag_index_id}, synchronize_session=False+ )rows = (- session.query(DocumentChunk.id, DocumentChunk.source_id)+ session.query(+ DocumentChunk.id,+ DocumentChunk.source_id,+ DocumentChunk.document_metadata,+ ).filter_by(source_type="document",collection_name=collection_name,@@ -3633,11 +3694,14 @@ class LibraryRAGService:.all())ids_by_document: Dict[str, Set[int]] = {}- for chunk_id, source_id in rows:+ invalidated_documents: Set[str] = set()+ for chunk_id, source_id, metadata in rows:if source_id is not None:ids_by_document.setdefault(str(source_id), set()).add(int(chunk_id))+ if (metadata or {}).get("_ldr_text_invalidated"):+ invalidated_documents.add(str(source_id))# Refuse-to-shrink guard: a transient empty live_ids while# chunk rows exist would otherwise mass-clear every indexed@@ -3666,7 +3730,9 @@ class LibraryRAGService:durable = {doc_id: chunk_idsfor doc_id, chunk_ids in ids_by_document.items()- if chunk_ids and chunk_ids.issubset(live_ids)+ if chunk_ids+ and chunk_ids.issubset(live_ids)+ and doc_id not in invalidated_documents}# Snapshot existing indexed_at BEFORE the bulk delete sodiff --git a/tests/research_library/services/test_index_documents_parallel.py b/tests/research_library/services/test_index_documents_parallel.pyindex 4b140b438e..699f2b35ca 100644--- a/tests/research_library/services/test_index_documents_parallel.py+++ b/tests/research_library/services/test_index_documents_parallel.py@@ -989,12 +989,16 @@ class TestWritePreparedDocumentReindexDelta:# 1. DocumentCollection link (gate)# 2. Document existence (gate)# 3. RagDocumentStatus existence (gate)- # 4. RagDocumentStatus (existing, for delta)- # 5. DocumentCollection update (TOCTOU rowcount; MagicMock -> no abort)- # 6. RAGIndex+ # 4. Conditional document snapshot claim+ # 5. RagDocumentStatus (existing, for delta)+ # 6. DocumentCollection update (TOCTOU rowcount; MagicMock -> no abort)+ # 7. RAGIndexq_link = make_q(first_val=mock_link)q_doc = make_q(first_val=mock_doc)q_gate_status = make_q(first_val=mock_existing)+ q_claim = make_q()+ q_claim.filter.return_value = q_claim+ q_claim.update.return_value = 1q_existing = make_q(first_val=mock_existing)q_doc_coll = make_q()q_rag_index = make_q(first_val=mock_rag_index)@@ -1002,6 +1006,7 @@ class TestWritePreparedDocumentReindexDelta:q_link,q_doc,q_gate_status,+ q_claim,q_existing,q_doc_coll,q_rag_index,diff --git a/tests/research_library/services/test_issue_6559_rag_indexing_lock.py b/tests/research_library/services/test_issue_6559_rag_indexing_lock.pyindex 9cc1f592ef..88731555f4 100644--- a/tests/research_library/services/test_issue_6559_rag_indexing_lock.py+++ b/tests/research_library/services/test_issue_6559_rag_indexing_lock.py@@ -1455,6 +1455,7 @@ class TestRAGIndexingLockDecoupling:q_link,q_doc,q_gate_status,+ q_doc, # Conditional document snapshot claimq_existing,q_update,]diff --git a/tests/research_library/services/test_reconcile_collection_index.py b/tests/research_library/services/test_reconcile_collection_index.pyindex 01450365bb..822b718f5b 100644--- a/tests/research_library/services/test_reconcile_collection_index.py+++ b/tests/research_library/services/test_reconcile_collection_index.py@@ -95,7 +95,7 @@ def _make_chunk_row(chunk_id, source_id):row = MagicMock()row.id = chunk_idrow.source_id = source_id- row.__iter__ = lambda self: iter((chunk_id, source_id))+ row.__iter__ = lambda self: iter((chunk_id, source_id, None))return row@@ -218,12 +218,14 @@ class TestReconcilePreservesIndexedAt:rag_index_q = _build_query_mock()session = MagicMock()# Order of session.query() calls in production:+ # 0. RAGIndex no-op write claim (rag_index_q)# 1. DocumentChunk rows (chunk_q)# 2. RagDocumentStatus prior indexed_at (prior_status_q)# 3. DocumentCollection links (empty_q)# 4. RagDocumentStatus bulk delete (empty_q — same shape)# 5. RAGIndex row (rag_index_q)session.query.side_effect = [+ rag_index_q, # SQLite write claim before reading chunkschunk_q,prior_status_q,empty_q,@@ -281,6 +283,7 @@ class TestReconcilePartialDurabilityAndOrphans:rag_index_q = _build_query_mock()session = MagicMock()session.query.side_effect = [+ rag_index_q, # SQLite write claim before reading chunkschunk_q,empty_q,empty_q,@@ -315,6 +318,7 @@ class TestReconcilePartialDurabilityAndOrphans:rag_index_q = _build_query_mock()session = MagicMock()session.query.side_effect = [+ rag_index_q, # SQLite write claim before reading chunkschunk_q,empty_q,empty_q,@@ -401,7 +405,7 @@ class TestReconcileDimensionDrift:q.first.return_value = db_row# The chunk query also keeps only rows whose document still exists.q.filter.return_value = q- q.all.return_value = [(101, "doc-1")]+ q.all.return_value = [(101, "doc-1", None)]svc._reset_index_state_for_rebuild = MagicMock()# After a drift-triggered rebuild, the facade's live_ids() comesdiff --git a/tests/research_library/services/test_text_index_invalidation.py b/tests/research_library/services/test_text_index_invalidation.pynew file mode 100644index 0000000000..0bc4ff4c1f--- /dev/null+++ b/tests/research_library/services/test_text_index_invalidation.py@@ -0,0 +1,340 @@+"""Text replacement and index persistence serialize across real SQLite sessions."""++from contextlib import nullcontext+from types import MethodType, SimpleNamespace++import pytest+from sqlalchemy import event+from sqlalchemy.exc import OperationalError++from local_deep_research.database.models.library import (+ Document,+ DocumentChunk,+ DocumentCollection,+ EmbeddingProvider,+ RAGIndex,+ RagDocumentStatus,+)+from local_deep_research.database.models.research import ResearchResource+from local_deep_research.research_library.services.library_rag_service import (+ LibraryRAGService,+)+from local_deep_research.vector_stores.facade import VectorIndex+from tests.research_library.services.test_download_text_quality import (+ LONGER,+ ORIGINAL,+ extraction_case as extraction_case,+)+++class MemoryVectors:+ """Control vector membership while retaining the real chunk SQL facade."""++ dimension = 2++ def __init__(self, ids):+ self.ids = set(ids)+ self.removed = set()++ def live_ids(self):+ return list(self.ids)++ def apply(self, *, add_ids, add_vectors, remove_ids):+ self.ids.difference_update(remove_ids)+ self.ids.update(add_ids)+ self.removed.update(remove_ids)+ return {"added": len(add_ids), "removed": len(remove_ids)}+++def indexer_for(case, collection="library"):+ with case.sessions() as session:+ index_id = (+ session.query(RAGIndex.id)+ .filter_by(collection_name=f"collection_{collection}")+ .scalar()+ )+ chunk_ids = [+ row[0]+ for row in session.query(DocumentChunk.id).filter_by(+ collection_name=f"collection_{collection}"+ )+ ]+ backend = VectorIndex.__new__(VectorIndex)+ backend.collection_name = f"collection_{collection}"+ backend.embedding_model = "test"+ backend.embedding_model_type = EmbeddingProvider.SENTENCE_TRANSFORMERS+ backend._store = MemoryVectors(chunk_ids)+ obj = SimpleNamespace(+ username=case.service.username,+ db_password=case.service.password,+ rag_index_record=SimpleNamespace(id=index_id),+ embedding_model="test",+ embedding_provider="sentence_transformers",+ _collection_transaction_lock=lambda coll: nullcontext(),+ _get_vector_index=lambda *a, **kw: backend,+ _get_index_hash=lambda coll, *args: coll,+ _get_index_path=lambda h: case.service.library_root / h,+ text_splitter=SimpleNamespace(split_documents=lambda docs: docs),+ _embed_texts_cancellable=lambda texts, cancelled: [+ [1.0, 0.0] for _ in texts+ ],+ embedding_manager=SimpleNamespace(+ embeddings=SimpleNamespace(+ embed_documents=lambda texts: [[1.0, 0.0] for _ in texts]+ )+ ),+ )+ obj._flag_document_for_reindex = MethodType(+ LibraryRAGService._flag_document_for_reindex, obj+ )+ return obj, backend+++def replace_text(case, *, text=LONGER, immediate=False):+ with case.sessions() as writer:+ if immediate:+ writer.connection().exec_driver_sql("PRAGMA busy_timeout=0")+ resource = writer.get(ResearchResource, case.resource_id)+ case.service._save_text_with_db(+ resource, text, writer, "html_extraction", "arxiv_html", "canonical"+ )+ writer.commit()+++def persist(indexer, mode):+ if mode == "prepared":+ prep = LibraryRAGService._prepare_document(+ indexer, "canonical", "library", True+ )+ assert prep["status"] == "prepared", prep+ return LibraryRAGService._write_prepared_document(+ indexer, prep["prepared"]+ )+ return LibraryRAGService._index_document_locked(+ indexer, "canonical", "library", True+ )+++def assert_pending(case, collection="library"):+ with case.sessions() as session:+ assert session.get(Document, "canonical").text_content == LONGER+ assert (+ not session.query(DocumentCollection)+ .filter_by(document_id="canonical", collection_id=collection)+ .one()+ .indexed+ )+ assert session.get(RagDocumentStatus, ("canonical", collection)) is None++[email protected]("mode", ["prepared", "serial"])+def test_reconciliation_preserves_invalidation_until_replacement(+ extraction_case, mode+):+ case = extraction_case(quality="low", incoming=LONGER)+ indexer, backend = indexer_for(case)+ old_ids = backend._store.ids.copy()+ assert case.service.download_resource(case.resource_id) == (True, None)+ assert_pending(case)+ result = LibraryRAGService.reconcile_collection_index(indexer, "library")+ assert result["indexed_documents"] == 0+ assert_pending(case)+ with case.sessions() as session:+ chunk = (+ session.query(DocumentChunk)+ .filter_by(collection_name="collection_library")+ .one()+ )+ assert chunk.chunk_text == ORIGINAL+ assert chunk.id in old_ids+ assert persist(indexer, mode)["status"] == "success"+ assert backend._store.removed == old_ids+ assert old_ids.isdisjoint(backend._store.ids)+ result = LibraryRAGService.reconcile_collection_index(indexer, "library")+ assert result["indexed_documents"] == 1+ with case.sessions() as session:+ assert (+ session.query(DocumentChunk)+ .filter_by(collection_name="collection_library")+ .one()+ .chunk_text+ == LONGER+ )+ assert (+ session.query(DocumentCollection)+ .filter_by(document_id="canonical", collection_id="library")+ .one()+ .indexed+ )+ status = session.get(RagDocumentStatus, ("canonical", "library"))+ index = session.get(RAGIndex, status.rag_index_id)+ assert (+ index.total_documents+ == index.chunk_count+ == status.chunk_count+ == 1+ )++[email protected]("mode", ["prepared", "serial"])+def test_text_change_after_validation_is_detected_before_vector_writes(+ extraction_case, mode+):+ case = extraction_case(quality="low")+ indexer, backend = indexer_for(case)+ before = backend._store.ids.copy()++ def initialize(*args, **kwargs):+ # Real index initialization can commit the shared session. A claim+ # taken before this call would not protect subsequent persistence.+ case.shared.commit()+ replace_text(case)+ return backend++ indexer._get_vector_index = initialize+ result = persist(indexer, mode)+ assert result["status"] == "stale", result+ assert_pending(case)+ assert backend._store.ids == before+ assert not backend._store.removed++[email protected]("mode", ["prepared", "serial"])+def test_text_writer_cannot_commit_between_claim_and_index_commit(+ extraction_case, mode+):+ case = extraction_case(quality="low")+ indexer, backend = indexer_for(case)+ real_persist = backend.index_prepared+ checked = []++ def write_with_competitor(**kwargs):+ with pytest.raises(OperationalError, match="locked"):+ replace_text(case, immediate=True)+ checked.append(True)+ return real_persist(**kwargs)++ backend.index_prepared = write_with_competitor+ result = persist(indexer, mode)+ assert result["status"] == "success", result+ assert checked == [True]+ # Once indexing commits, the same text update succeeds and invalidates+ # even the newly committed chunks. The claim is transaction-scoped.+ replace_text(case)+ LibraryRAGService.reconcile_collection_index(indexer, "library")+ assert_pending(case)+++def test_reconcile_serializes_its_snapshot_with_text_replacement(+ extraction_case,+):+ case = extraction_case(quality="low")+ indexer, _ = indexer_for(case)+ checked = []++ def replace_after_chunks_read(_conn, _cursor, statement, *_args):+ if not checked and statement.startswith(+ "SELECT rag_document_status.document_id"+ ):+ checked.append(True)+ with pytest.raises(OperationalError, match="locked"):+ replace_text(case, immediate=True)++ event.listen(+ case.shared.bind, "before_cursor_execute", replace_after_chunks_read+ )+ try:+ result = LibraryRAGService.reconcile_collection_index(+ indexer, "library"+ )+ finally:+ event.remove(+ case.shared.bind, "before_cursor_execute", replace_after_chunks_read+ )+ assert checked == [True]+ assert result["indexed_documents"] == 1+ replace_text(case)+ LibraryRAGService.reconcile_collection_index(indexer, "library")+ assert_pending(case)+++def test_metadata_recovery_without_text_replacement_still_works(+ extraction_case,+):+ case = extraction_case()+ indexer, _ = indexer_for(case)+ with case.sessions() as session:+ session.query(DocumentCollection).update({"indexed": False})+ session.query(RagDocumentStatus).delete()+ session.commit()+ result = LibraryRAGService.reconcile_collection_index(indexer, "library")+ assert result["indexed_documents"] == 1+ with case.sessions() as session:+ assert (+ session.get(RagDocumentStatus, ("canonical", "library")) is not None+ )+++def test_chunk_invalidation_rolls_back_with_text_and_preserves_metadata(+ extraction_case,+):+ case = extraction_case(quality="low")+ with case.sessions() as session:+ for chunk in session.query(DocumentChunk):+ chunk.document_metadata = {"title": "Existing citation context"}+ session.commit()+ resource = session.get(ResearchResource, case.resource_id)+ case.service._save_text_with_db(+ resource,+ LONGER,+ session,+ "html_extraction",+ "arxiv_html",+ "canonical",+ )+ session.flush()+ assert all(+ chunk.document_metadata+ == {+ "title": "Existing citation context",+ "_ldr_text_invalidated": True,+ }+ for chunk in session.query(DocumentChunk)+ )+ session.rollback()+ with case.sessions() as session:+ assert session.get(Document, "canonical").text_content == ORIGINAL+ assert all(+ chunk.document_metadata == {"title": "Existing citation context"}+ for chunk in session.query(DocumentChunk)+ )+ replace_text(case)+ for collection in ["library", "selected"]:+ # Every collection, not just the resource's default one, stays pending.+ indexer, _ = indexer_for(case, collection)+ LibraryRAGService.reconcile_collection_index(indexer, collection)+ assert_pending(case, collection)++[email protected]("mode", ["prepared", "serial"])+def test_failed_reindex_keeps_old_chunks_invalidated(extraction_case, mode):+ case = extraction_case(quality="low")+ indexer, backend = indexer_for(case)+ replace_text(case)++ def fail_apply(**kwargs):+ raise RuntimeError("controlled vector failure")++ backend._store.apply = fail_apply+ result = persist(indexer, mode)+ assert result["status"] == "error", result+ LibraryRAGService.reconcile_collection_index(indexer, "library")+ assert_pending(case)+ with case.sessions() as session:+ assert (+ session.query(DocumentChunk)+ .filter_by(collection_name="collection_library")+ .one()+ .chunk_text+ == ORIGINAL+ )