Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

fix(rag): preserve text invalidation through indexing and reconciliation (#7307)

LearningCircuit503f6f76ec0f
diff --git a/changelog.d/+library-text-index-invalidation.bugfix.md b/changelog.d/+library-text-index-invalidation.bugfix.md
new file mode 100644
index 0000000000..d40c6c15b2
--- /dev/null
+++ b/changelog.d/+library-text-index-invalidation.bugfix.md
@@ -0,0 +1 @@
+Keep improved document text pending for reindexing even when older vectors still exist. Index persistence now serializes with text replacement, and reconciliation preserves invalidated chunk records until a successful reindex replaces their vectors.
diff --git a/src/local_deep_research/research_library/services/download_service.py b/src/local_deep_research/research_library/services/download_service.py
index 703e1fec9c..e4e24240e7 100644
--- a/src/local_deep_research/research_library/services/download_service.py
+++ b/src/local_deep_research/research_library/services/download_service.py
@@ -23,7 +23,7 @@ import requests
from loguru import logger
from ...security import redact_url_for_log
from sqlalchemy import func
-from sqlalchemy.orm import Session
+from sqlalchemy.orm import Session, load_only
import pdfplumber
from pypdf import PdfReader
@@ -56,6 +56,7 @@ from ...security.secure_logging import log_exception_type
from ...database.models.library import (
Collection,
Document as Document,
+ DocumentChunk,
DocumentCollection,
DocumentStatus,
DownloadQueue as LibraryDownloadQueue,
@@ -3143,6 +3144,19 @@ class DownloadService:
return None
if old_text != text:
+ # Keep the vector IDs for replacement, but persist why
+ # these chunks are pending. Live vectors alone must not
+ # let reconciliation mark an older extraction current.
+ for chunk in (
+ session.query(DocumentChunk)
+ .options(load_only(DocumentChunk.document_metadata))
+ .filter_by(source_type="document", source_id=doc.id)
+ .all()
+ ):
+ chunk.document_metadata = {
+ **(chunk.document_metadata or {}),
+ "_ldr_text_invalidated": True,
+ }
# The legacy flag and canonical per-index marker must both
# move with the text, in this transaction. Leave chunk rows
# available for replace-on-reindex to remove their vectors.
diff --git a/src/local_deep_research/research_library/services/library_rag_service.py b/src/local_deep_research/research_library/services/library_rag_service.py
index 7f26532d50..89d5e25ef1 100644
--- a/src/local_deep_research/research_library/services/library_rag_service.py
+++ b/src/local_deep_research/research_library/services/library_rag_service.py
@@ -65,6 +65,25 @@ from langchain_core.embeddings import Embeddings
import hashlib
+def _claim_document_snapshot(session, document_id, text_content, document_hash):
+ """Keep the validated SQLite snapshot stable until the index commit.
+
+ SELECT FOR UPDATE is ignored by SQLite. A conditional no-op write either
+ detects a writer that already won or holds the write lock through chunk
+ and status persistence. Call after vector initialization, which may commit
+ index setup, and never before embedding.
+ """
+ return (
+ session.query(Document)
+ .filter(
+ Document.id == document_id,
+ Document.text_content == text_content,
+ Document.document_hash == document_hash,
+ )
+ .update({Document.id: document_id}, synchronize_session=False)
+ )
+
+
@dataclass
class _PreparedDocument:
document_id: str
@@ -1998,9 +2017,24 @@ class LibraryRAGService:
}
collection_name = f"collection_{collection_id}"
+ observed_text = current_doc.text_content
+ observed_hash = current_doc.document_hash
vindex = self._get_vector_index(
collection_id, collection_name, reset_stale_state=True
)
+ if (
+ _claim_document_snapshot(
+ session, document_id, observed_text, observed_hash
+ )
+ == 0
+ ):
+ self._flag_document_for_reindex(
+ session, document_id, collection_id
+ )
+ return {
+ "status": "stale",
+ "message": "Document modified during indexing, flagged for reindex",
+ }
stats = vindex.index_prepared(
source_type="document",
source_id=document_id,
@@ -2531,10 +2565,26 @@ class LibraryRAGService:
"message": "Document indexed state changed during indexing, flagged for reindex",
}
+ observed_text = current_doc.text_content
+ observed_hash = current_doc.document_hash
vindex = self._get_vector_index(
collection_id, collection_name, reset_stale_state=True
)
+ if (
+ _claim_document_snapshot(
+ session, document_id, observed_text, observed_hash
+ )
+ == 0
+ ):
+ self._flag_document_for_reindex(
+ session, document_id, collection_id
+ )
+ return {
+ "status": "stale",
+ "message": "Document modified during indexing, flagged for reindex",
+ }
+
if hasattr(vindex, "index_prepared"):
index_stats = vindex.index_prepared(
source_type="document",
@@ -3575,8 +3625,9 @@ class LibraryRAGService:
"""Reconcile indexed flags and stats against durable FAISS membership.
A document is indexed only when it has at least one current-model chunk
- row and every such chunk id exists in the live FAISS index. Stale status
- rows are removed and compatibility flags/stats are rebuilt atomically.
+ row, none were invalidated by text replacement, and every such chunk
+ id exists in the live FAISS index. Stale status rows are removed and
+ compatibility flags/stats are rebuilt atomically.
Safety invariant: if the live FAISS index reports zero ids (transient
load error, missing file, on-disk corruption the verifier just
@@ -3612,8 +3663,18 @@ class LibraryRAGService:
with get_user_db_session(
self.username, self.db_password
) as session:
+ # Serialize this SQLite repair with text invalidation. The
+ # vector store was initialized above; no nested setup commit
+ # may release this write lock before flags/status are saved.
+ session.query(RAGIndex).filter_by(id=rag_index_id).update(
+ {RAGIndex.id: rag_index_id}, synchronize_session=False
+ )
rows = (
- session.query(DocumentChunk.id, DocumentChunk.source_id)
+ session.query(
+ DocumentChunk.id,
+ DocumentChunk.source_id,
+ DocumentChunk.document_metadata,
+ )
.filter_by(
source_type="document",
collection_name=collection_name,
@@ -3633,11 +3694,14 @@ class LibraryRAGService:
.all()
)
ids_by_document: Dict[str, Set[int]] = {}
- for chunk_id, source_id in rows:
+ invalidated_documents: Set[str] = set()
+ for chunk_id, source_id, metadata in rows:
if source_id is not None:
ids_by_document.setdefault(str(source_id), set()).add(
int(chunk_id)
)
+ if (metadata or {}).get("_ldr_text_invalidated"):
+ invalidated_documents.add(str(source_id))
# Refuse-to-shrink guard: a transient empty live_ids while
# chunk rows exist would otherwise mass-clear every indexed
@@ -3666,7 +3730,9 @@ class LibraryRAGService:
durable = {
doc_id: chunk_ids
for doc_id, chunk_ids in ids_by_document.items()
- if chunk_ids and chunk_ids.issubset(live_ids)
+ if chunk_ids
+ and chunk_ids.issubset(live_ids)
+ and doc_id not in invalidated_documents
}
# Snapshot existing indexed_at BEFORE the bulk delete so
diff --git a/tests/research_library/services/test_index_documents_parallel.py b/tests/research_library/services/test_index_documents_parallel.py
index 4b140b438e..699f2b35ca 100644
--- a/tests/research_library/services/test_index_documents_parallel.py
+++ b/tests/research_library/services/test_index_documents_parallel.py
@@ -989,12 +989,16 @@ class TestWritePreparedDocumentReindexDelta:
# 1. DocumentCollection link (gate)
# 2. Document existence (gate)
# 3. RagDocumentStatus existence (gate)
- # 4. RagDocumentStatus (existing, for delta)
- # 5. DocumentCollection update (TOCTOU rowcount; MagicMock -> no abort)
- # 6. RAGIndex
+ # 4. Conditional document snapshot claim
+ # 5. RagDocumentStatus (existing, for delta)
+ # 6. DocumentCollection update (TOCTOU rowcount; MagicMock -> no abort)
+ # 7. RAGIndex
q_link = make_q(first_val=mock_link)
q_doc = make_q(first_val=mock_doc)
q_gate_status = make_q(first_val=mock_existing)
+ q_claim = make_q()
+ q_claim.filter.return_value = q_claim
+ q_claim.update.return_value = 1
q_existing = make_q(first_val=mock_existing)
q_doc_coll = make_q()
q_rag_index = make_q(first_val=mock_rag_index)
@@ -1002,6 +1006,7 @@ class TestWritePreparedDocumentReindexDelta:
q_link,
q_doc,
q_gate_status,
+ q_claim,
q_existing,
q_doc_coll,
q_rag_index,
diff --git a/tests/research_library/services/test_issue_6559_rag_indexing_lock.py b/tests/research_library/services/test_issue_6559_rag_indexing_lock.py
index 9cc1f592ef..88731555f4 100644
--- a/tests/research_library/services/test_issue_6559_rag_indexing_lock.py
+++ b/tests/research_library/services/test_issue_6559_rag_indexing_lock.py
@@ -1455,6 +1455,7 @@ class TestRAGIndexingLockDecoupling:
q_link,
q_doc,
q_gate_status,
+ q_doc, # Conditional document snapshot claim
q_existing,
q_update,
]
diff --git a/tests/research_library/services/test_reconcile_collection_index.py b/tests/research_library/services/test_reconcile_collection_index.py
index 01450365bb..822b718f5b 100644
--- a/tests/research_library/services/test_reconcile_collection_index.py
+++ b/tests/research_library/services/test_reconcile_collection_index.py
@@ -95,7 +95,7 @@ def _make_chunk_row(chunk_id, source_id):
row = MagicMock()
row.id = chunk_id
row.source_id = source_id
- row.__iter__ = lambda self: iter((chunk_id, source_id))
+ row.__iter__ = lambda self: iter((chunk_id, source_id, None))
return row
@@ -218,12 +218,14 @@ class TestReconcilePreservesIndexedAt:
rag_index_q = _build_query_mock()
session = MagicMock()
# Order of session.query() calls in production:
+ # 0. RAGIndex no-op write claim (rag_index_q)
# 1. DocumentChunk rows (chunk_q)
# 2. RagDocumentStatus prior indexed_at (prior_status_q)
# 3. DocumentCollection links (empty_q)
# 4. RagDocumentStatus bulk delete (empty_q — same shape)
# 5. RAGIndex row (rag_index_q)
session.query.side_effect = [
+ rag_index_q, # SQLite write claim before reading chunks
chunk_q,
prior_status_q,
empty_q,
@@ -281,6 +283,7 @@ class TestReconcilePartialDurabilityAndOrphans:
rag_index_q = _build_query_mock()
session = MagicMock()
session.query.side_effect = [
+ rag_index_q, # SQLite write claim before reading chunks
chunk_q,
empty_q,
empty_q,
@@ -315,6 +318,7 @@ class TestReconcilePartialDurabilityAndOrphans:
rag_index_q = _build_query_mock()
session = MagicMock()
session.query.side_effect = [
+ rag_index_q, # SQLite write claim before reading chunks
chunk_q,
empty_q,
empty_q,
@@ -401,7 +405,7 @@ class TestReconcileDimensionDrift:
q.first.return_value = db_row
# The chunk query also keeps only rows whose document still exists.
q.filter.return_value = q
- q.all.return_value = [(101, "doc-1")]
+ q.all.return_value = [(101, "doc-1", None)]
svc._reset_index_state_for_rebuild = MagicMock()
# After a drift-triggered rebuild, the facade's live_ids() comes
diff --git a/tests/research_library/services/test_text_index_invalidation.py b/tests/research_library/services/test_text_index_invalidation.py
new file mode 100644
index 0000000000..0bc4ff4c1f
--- /dev/null
+++ b/tests/research_library/services/test_text_index_invalidation.py
@@ -0,0 +1,340 @@
+"""Text replacement and index persistence serialize across real SQLite sessions."""
+
+from contextlib import nullcontext
+from types import MethodType, SimpleNamespace
+
+import pytest
+from sqlalchemy import event
+from sqlalchemy.exc import OperationalError
+
+from local_deep_research.database.models.library import (
+ Document,
+ DocumentChunk,
+ DocumentCollection,
+ EmbeddingProvider,
+ RAGIndex,
+ RagDocumentStatus,
+)
+from local_deep_research.database.models.research import ResearchResource
+from local_deep_research.research_library.services.library_rag_service import (
+ LibraryRAGService,
+)
+from local_deep_research.vector_stores.facade import VectorIndex
+from tests.research_library.services.test_download_text_quality import (
+ LONGER,
+ ORIGINAL,
+ extraction_case as extraction_case,
+)
+
+
+class MemoryVectors:
+ """Control vector membership while retaining the real chunk SQL facade."""
+
+ dimension = 2
+
+ def __init__(self, ids):
+ self.ids = set(ids)
+ self.removed = set()
+
+ def live_ids(self):
+ return list(self.ids)
+
+ def apply(self, *, add_ids, add_vectors, remove_ids):
+ self.ids.difference_update(remove_ids)
+ self.ids.update(add_ids)
+ self.removed.update(remove_ids)
+ return {"added": len(add_ids), "removed": len(remove_ids)}
+
+
+def indexer_for(case, collection="library"):
+ with case.sessions() as session:
+ index_id = (
+ session.query(RAGIndex.id)
+ .filter_by(collection_name=f"collection_{collection}")
+ .scalar()
+ )
+ chunk_ids = [
+ row[0]
+ for row in session.query(DocumentChunk.id).filter_by(
+ collection_name=f"collection_{collection}"
+ )
+ ]
+ backend = VectorIndex.__new__(VectorIndex)
+ backend.collection_name = f"collection_{collection}"
+ backend.embedding_model = "test"
+ backend.embedding_model_type = EmbeddingProvider.SENTENCE_TRANSFORMERS
+ backend._store = MemoryVectors(chunk_ids)
+ obj = SimpleNamespace(
+ username=case.service.username,
+ db_password=case.service.password,
+ rag_index_record=SimpleNamespace(id=index_id),
+ embedding_model="test",
+ embedding_provider="sentence_transformers",
+ _collection_transaction_lock=lambda coll: nullcontext(),
+ _get_vector_index=lambda *a, **kw: backend,
+ _get_index_hash=lambda coll, *args: coll,
+ _get_index_path=lambda h: case.service.library_root / h,
+ text_splitter=SimpleNamespace(split_documents=lambda docs: docs),
+ _embed_texts_cancellable=lambda texts, cancelled: [
+ [1.0, 0.0] for _ in texts
+ ],
+ embedding_manager=SimpleNamespace(
+ embeddings=SimpleNamespace(
+ embed_documents=lambda texts: [[1.0, 0.0] for _ in texts]
+ )
+ ),
+ )
+ obj._flag_document_for_reindex = MethodType(
+ LibraryRAGService._flag_document_for_reindex, obj
+ )
+ return obj, backend
+
+
+def replace_text(case, *, text=LONGER, immediate=False):
+ with case.sessions() as writer:
+ if immediate:
+ writer.connection().exec_driver_sql("PRAGMA busy_timeout=0")
+ resource = writer.get(ResearchResource, case.resource_id)
+ case.service._save_text_with_db(
+ resource, text, writer, "html_extraction", "arxiv_html", "canonical"
+ )
+ writer.commit()
+
+
+def persist(indexer, mode):
+ if mode == "prepared":
+ prep = LibraryRAGService._prepare_document(
+ indexer, "canonical", "library", True
+ )
+ assert prep["status"] == "prepared", prep
+ return LibraryRAGService._write_prepared_document(
+ indexer, prep["prepared"]
+ )
+ return LibraryRAGService._index_document_locked(
+ indexer, "canonical", "library", True
+ )
+
+
+def assert_pending(case, collection="library"):
+ with case.sessions() as session:
+ assert session.get(Document, "canonical").text_content == LONGER
+ assert (
+ not session.query(DocumentCollection)
+ .filter_by(document_id="canonical", collection_id=collection)
+ .one()
+ .indexed
+ )
+ assert session.get(RagDocumentStatus, ("canonical", collection)) is None
+
+
[email protected]("mode", ["prepared", "serial"])
+def test_reconciliation_preserves_invalidation_until_replacement(
+ extraction_case, mode
+):
+ case = extraction_case(quality="low", incoming=LONGER)
+ indexer, backend = indexer_for(case)
+ old_ids = backend._store.ids.copy()
+ assert case.service.download_resource(case.resource_id) == (True, None)
+ assert_pending(case)
+ result = LibraryRAGService.reconcile_collection_index(indexer, "library")
+ assert result["indexed_documents"] == 0
+ assert_pending(case)
+ with case.sessions() as session:
+ chunk = (
+ session.query(DocumentChunk)
+ .filter_by(collection_name="collection_library")
+ .one()
+ )
+ assert chunk.chunk_text == ORIGINAL
+ assert chunk.id in old_ids
+ assert persist(indexer, mode)["status"] == "success"
+ assert backend._store.removed == old_ids
+ assert old_ids.isdisjoint(backend._store.ids)
+ result = LibraryRAGService.reconcile_collection_index(indexer, "library")
+ assert result["indexed_documents"] == 1
+ with case.sessions() as session:
+ assert (
+ session.query(DocumentChunk)
+ .filter_by(collection_name="collection_library")
+ .one()
+ .chunk_text
+ == LONGER
+ )
+ assert (
+ session.query(DocumentCollection)
+ .filter_by(document_id="canonical", collection_id="library")
+ .one()
+ .indexed
+ )
+ status = session.get(RagDocumentStatus, ("canonical", "library"))
+ index = session.get(RAGIndex, status.rag_index_id)
+ assert (
+ index.total_documents
+ == index.chunk_count
+ == status.chunk_count
+ == 1
+ )
+
+
[email protected]("mode", ["prepared", "serial"])
+def test_text_change_after_validation_is_detected_before_vector_writes(
+ extraction_case, mode
+):
+ case = extraction_case(quality="low")
+ indexer, backend = indexer_for(case)
+ before = backend._store.ids.copy()
+
+ def initialize(*args, **kwargs):
+ # Real index initialization can commit the shared session. A claim
+ # taken before this call would not protect subsequent persistence.
+ case.shared.commit()
+ replace_text(case)
+ return backend
+
+ indexer._get_vector_index = initialize
+ result = persist(indexer, mode)
+ assert result["status"] == "stale", result
+ assert_pending(case)
+ assert backend._store.ids == before
+ assert not backend._store.removed
+
+
[email protected]("mode", ["prepared", "serial"])
+def test_text_writer_cannot_commit_between_claim_and_index_commit(
+ extraction_case, mode
+):
+ case = extraction_case(quality="low")
+ indexer, backend = indexer_for(case)
+ real_persist = backend.index_prepared
+ checked = []
+
+ def write_with_competitor(**kwargs):
+ with pytest.raises(OperationalError, match="locked"):
+ replace_text(case, immediate=True)
+ checked.append(True)
+ return real_persist(**kwargs)
+
+ backend.index_prepared = write_with_competitor
+ result = persist(indexer, mode)
+ assert result["status"] == "success", result
+ assert checked == [True]
+ # Once indexing commits, the same text update succeeds and invalidates
+ # even the newly committed chunks. The claim is transaction-scoped.
+ replace_text(case)
+ LibraryRAGService.reconcile_collection_index(indexer, "library")
+ assert_pending(case)
+
+
+def test_reconcile_serializes_its_snapshot_with_text_replacement(
+ extraction_case,
+):
+ case = extraction_case(quality="low")
+ indexer, _ = indexer_for(case)
+ checked = []
+
+ def replace_after_chunks_read(_conn, _cursor, statement, *_args):
+ if not checked and statement.startswith(
+ "SELECT rag_document_status.document_id"
+ ):
+ checked.append(True)
+ with pytest.raises(OperationalError, match="locked"):
+ replace_text(case, immediate=True)
+
+ event.listen(
+ case.shared.bind, "before_cursor_execute", replace_after_chunks_read
+ )
+ try:
+ result = LibraryRAGService.reconcile_collection_index(
+ indexer, "library"
+ )
+ finally:
+ event.remove(
+ case.shared.bind, "before_cursor_execute", replace_after_chunks_read
+ )
+ assert checked == [True]
+ assert result["indexed_documents"] == 1
+ replace_text(case)
+ LibraryRAGService.reconcile_collection_index(indexer, "library")
+ assert_pending(case)
+
+
+def test_metadata_recovery_without_text_replacement_still_works(
+ extraction_case,
+):
+ case = extraction_case()
+ indexer, _ = indexer_for(case)
+ with case.sessions() as session:
+ session.query(DocumentCollection).update({"indexed": False})
+ session.query(RagDocumentStatus).delete()
+ session.commit()
+ result = LibraryRAGService.reconcile_collection_index(indexer, "library")
+ assert result["indexed_documents"] == 1
+ with case.sessions() as session:
+ assert (
+ session.get(RagDocumentStatus, ("canonical", "library")) is not None
+ )
+
+
+def test_chunk_invalidation_rolls_back_with_text_and_preserves_metadata(
+ extraction_case,
+):
+ case = extraction_case(quality="low")
+ with case.sessions() as session:
+ for chunk in session.query(DocumentChunk):
+ chunk.document_metadata = {"title": "Existing citation context"}
+ session.commit()
+ resource = session.get(ResearchResource, case.resource_id)
+ case.service._save_text_with_db(
+ resource,
+ LONGER,
+ session,
+ "html_extraction",
+ "arxiv_html",
+ "canonical",
+ )
+ session.flush()
+ assert all(
+ chunk.document_metadata
+ == {
+ "title": "Existing citation context",
+ "_ldr_text_invalidated": True,
+ }
+ for chunk in session.query(DocumentChunk)
+ )
+ session.rollback()
+ with case.sessions() as session:
+ assert session.get(Document, "canonical").text_content == ORIGINAL
+ assert all(
+ chunk.document_metadata == {"title": "Existing citation context"}
+ for chunk in session.query(DocumentChunk)
+ )
+ replace_text(case)
+ for collection in ["library", "selected"]:
+ # Every collection, not just the resource's default one, stays pending.
+ indexer, _ = indexer_for(case, collection)
+ LibraryRAGService.reconcile_collection_index(indexer, collection)
+ assert_pending(case, collection)
+
+
[email protected]("mode", ["prepared", "serial"])
+def test_failed_reindex_keeps_old_chunks_invalidated(extraction_case, mode):
+ case = extraction_case(quality="low")
+ indexer, backend = indexer_for(case)
+ replace_text(case)
+
+ def fail_apply(**kwargs):
+ raise RuntimeError("controlled vector failure")
+
+ backend._store.apply = fail_apply
+ result = persist(indexer, mode)
+ assert result["status"] == "error", result
+ LibraryRAGService.reconcile_collection_index(indexer, "library")
+ assert_pending(case)
+ with case.sessions() as session:
+ assert (
+ session.query(DocumentChunk)
+ .filter_by(collection_name="collection_library")
+ .one()
+ .chunk_text
+ == ORIGINAL
+ )