~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
fix(rag): decouple indexing into 3 phases to avoid SQLite lock during embedding (#6559) (#6563)
Sdee2dcb8d65b
diff --git a/changelog.d/6559.bugfix.md b/changelog.d/6559.bugfix.mdnew file mode 100644index 0000000000..a085467ff4--- /dev/null+++ b/changelog.d/6559.bugfix.md@@ -0,0 +1,14 @@+Decouple RAG document indexing into three phases so the SQLite write lock is+not held across the embedding round-trip. Previously, `_index_document_locked`+held an open database session with uncommitted staging across embedding model+calls, starving concurrent readers and writers of the SQLite database. Indexing+now splits into: Phase 1 (short DB session to snapshot document metadata and+collection link, releasing locks prior to embedding), Phase 2 (text chunking+and vector embedding executed strictly outside any DB session), and Phase 3+(short DB transaction to persist chunk records, write vectors, and update RAG+status). Phase 3 includes a revalidation gate verifying collection membership,+document content hash, and indexed status before persisting to prevent+resurrecting unlinked documents or overwriting concurrent note edits. First-time+ingest (research orphans with no `DocumentCollection` row at prepare) still+creates membership in Phase 3; only a link that existed at prepare and vanished+during embedding is treated as a concurrent removal.diff --git a/src/local_deep_research/research_library/services/library_rag_service.py b/src/local_deep_research/research_library/services/library_rag_service.pyindex c929c4b4c8..325c89d0f7 100644--- a/src/local_deep_research/research_library/services/library_rag_service.py+++ b/src/local_deep_research/research_library/services/library_rag_service.py@@ -71,6 +71,13 @@ class _PreparedDocument:collection_id: strchunk_inputs: List[ChunkInput]vectors: np.ndarray+ content_hash: Optional[str] = None+ initial_indexed: Optional[bool] = None+ status_exists: Optional[bool] = None+ # True iff a DocumentCollection row existed at prepare time. False+ # means first-time ingest: Phase 3 may create the link. Distinguishes+ # that from a concurrent unlink of a row that was present then.+ had_collection_link: Optional[bool] = Noneclass _LazyEmbeddings(Embeddings):@@ -1764,6 +1771,23 @@ class LibraryRAGService:source_doc = LangchainDocument(page_content=document.text_content, metadata=metadata)+ prep_content_hash = hashlib.sha256(+ (document.text_content or "").encode("utf-8")+ ).hexdigest()+ prep_initial_indexed = link.indexed if link is not None else False+ prep_had_collection_link = link is not None+ prep_status_exists = (+ session.query(RagDocumentStatus)+ .filter_by(+ document_id=document_id,+ collection_id=collection_id,+ rag_index_id=self.rag_index_record.id,+ )+ .first()+ is not None+ if self.rag_index_record+ else False+ )chunks = self.text_splitter.split_documents([source_doc])chunk_inputs = [@@ -1789,6 +1813,10 @@ class LibraryRAGService:collection_id=collection_id,chunk_inputs=chunk_inputs,vectors=vectors,+ content_hash=prep_content_hash,+ initial_indexed=prep_initial_indexed,+ status_exists=prep_status_exists,+ had_collection_link=prep_had_collection_link,),}@@ -1810,7 +1838,110 @@ class LibraryRAGService:self.username, self.db_password) as session:try:- ensure_in_collection(session, document_id, collection_id)+ # Phase 3 Revalidation Gate: verify collection link, content, and indexed status.+ # The membership check runs unconditionally so legacy+ # prepared objects without a content_hash snapshot still+ # get first-time vs concurrent-unlink handling instead of+ # skipping the gate entirely.+ link = (+ session.query(DocumentCollection)+ .filter_by(+ document_id=document_id,+ collection_id=collection_id,+ )+ .first()+ )+ if link is None:+ if not getattr(prepared, "had_collection_link", None):+ # Reconciler case (b): research orphans have+ # no membership at prepare. Create it now+ # rather than treating absence as a concurrent+ # unlink. A missing snapshot (None) preserves+ # main's unconditional-create behavior.+ link = ensure_in_collection(+ session, document_id, collection_id+ )+ session.flush()+ else:+ logger.info(+ f"Document {document_id} was removed from collection {collection_id} "+ "during embedding; aborting Phase 3 persist."+ )+ self._flag_document_for_reindex(+ session, document_id, collection_id+ )+ return {+ "status": "skipped",+ "message": "Document removed from collection during embedding",+ }++ current_doc = (+ session.query(Document)+ .filter_by(id=document_id)+ .first()+ )+ if not current_doc:+ logger.info(+ f"Document {document_id} was deleted during embedding; "+ "aborting Phase 3 persist."+ )+ self._flag_document_for_reindex(+ session, document_id, collection_id+ )+ return {+ "status": "error",+ "error": "Document deleted during indexing",+ }++ if isinstance(getattr(prepared, "content_hash", None), str):+ current_hash = hashlib.sha256(+ (current_doc.text_content or "").encode("utf-8")+ ).hexdigest()+ if current_hash != prepared.content_hash:+ logger.info(+ f"Document {document_id} content changed during embedding; "+ "aborting persist and flagging for reindex."+ )+ self._flag_document_for_reindex(+ session, document_id, collection_id+ )+ return {+ "status": "stale",+ "message": "Document modified during indexing, flagged for reindex",+ }++ current_status_exists = (+ session.query(RagDocumentStatus)+ .filter_by(+ document_id=document_id,+ collection_id=collection_id,+ rag_index_id=self.rag_index_record.id,+ )+ .first()+ is not None+ if self.rag_index_record+ else False+ )+ if (+ isinstance(prepared.initial_indexed, bool)+ and isinstance(link.indexed, bool)+ and link.indexed != prepared.initial_indexed+ ) or (+ isinstance(prepared.status_exists, bool)+ and current_status_exists != prepared.status_exists+ ):+ logger.info(+ f"Document {document_id} indexing status modified during embedding; "+ "aborting persist and flagging for reindex."+ )+ self._flag_document_for_reindex(+ session, document_id, collection_id+ )+ return {+ "status": "stale",+ "message": "Document indexed state changed during indexing, flagged for reindex",+ }+collection_name = f"collection_{collection_id}"vindex = self._get_vector_index(collection_id, collection_name, reset_stale_state=True@@ -1844,15 +1975,41 @@ class LibraryRAGService:indexed_at=timestamp,))- session.query(DocumentCollection).filter_by(- document_id=document_id, collection_id=collection_id- ).update(- {- "indexed": True,- "chunk_count": len(chunks),- "last_indexed_at": timestamp,- }+ updated_rows = (+ session.query(DocumentCollection)+ .filter_by(+ document_id=document_id,+ collection_id=collection_id,+ )+ .update(+ {+ "indexed": True,+ "chunk_count": len(chunks),+ "last_indexed_at": timestamp,+ }+ ))+ if isinstance(updated_rows, int) and updated_rows == 0:+ # An unlink committed between the Phase-3 gate query+ # and these persist writes (the unlink DB phase does+ # not hold the FAISS lock). Rolling back avoids+ # re-creating status/aggregate rows for a non-member+ # and leaves phantom vectors to the unlink's purge.+ logger.info(+ f"Document {document_id} was removed from collection {collection_id} "+ "during persist; aborting Phase 3 commit."+ )+ safe_rollback(+ session,+ "library_rag_service._write_prepared_document link vanished",+ )+ self._flag_document_for_reindex(+ session, document_id, collection_id+ )+ return {+ "status": "skipped",+ "message": "Document removed from collection during embedding",+ }rag_index = (session.query(RAGIndex).filter_by(id=self.rag_index_record.id)@@ -1932,6 +2089,7 @@ class LibraryRAGService:Returns:Dict with status, chunk_count, and any errors"""+ # Phase 1: Read & chunk docwith get_user_db_session(self.username, self.db_password) as session:# Get the documentdocument = session.query(Document).filter_by(id=document_id).first()@@ -1943,6 +2101,25 @@ class LibraryRAGService:doc_collection = ensure_in_collection(session, document_id, collection_id)+ # If doc_collection was newly created in this session, commit it so the link+ # exists durably before Phase 2 without holding a write lock during embedding.+ # No test-session import is needed here: `in session.new` is+ # False for unittest.mock sessions (MagicMock __contains__+ # defaults to False), so mocks safely skip the pre-commit.+ try:+ _is_new_link = (+ getattr(session, "new", None) is not None+ and doc_collection in session.new+ )+ except Exception:+ _is_new_link = False+ if _is_new_link:+ try:+ session.commit()+ except Exception: # allow: silent-exception+ logger.debug(+ f"Could not pre-commit newly created collection link for {document_id} in {collection_id}"+ )# Check if already indexed for this collectionif doc_collection.indexed and not force_reindex:@@ -2050,6 +2227,24 @@ class LibraryRAGService:# orphaning rows or poisoning the shared session).try:+ phase1_content_hash = hashlib.sha256(+ (document.text_content or "").encode("utf-8")+ ).hexdigest()+ phase1_doc_hash = document.document_hash+ phase1_indexed = doc_collection.indexed+ phase1_status_exists = (+ session.query(RagDocumentStatus)+ .filter_by(+ document_id=document_id,+ collection_id=collection_id,+ rag_index_id=self.rag_index_record.id,+ )+ .first()+ is not None+ if self.rag_index_record+ else False+ )+# Create LangChain Document from textdoc = LangchainDocument(page_content=document.text_content,@@ -2130,34 +2325,179 @@ class LibraryRAGService:for i, chunk in enumerate(chunks)]- # Embed + persist via the new int-id-keyed vector store —- # this writes DocumentChunk rows (text only in the encrypted- # DB) and the vector-store's ids/vectors (never text; see- # the SECURITY INVARIANT in vector_stores/base.py). Passing- # ``session`` means the facade does not commit/rollback —- # this transaction still owns that (below).- #- # replace=True ALWAYS: a full-document (re)index owns all of a- # source's chunks, so it must drop the source's prior- # chunks/vectors before adding the new ones. force_reindex is- # NOT a safe proxy: a note edit clears the indexed STATUS- # (_mark_note_stale_for_reindex_in_session) but leaves the- # pre-edit chunk rows/vectors in place, so the subsequent- # non-forced reindex would reach here with force_reindex=False- # AND prior chunks present — appending would leave stale/deleted- # note text permanently searchable and duplicate every chunk.- # For a genuinely never-indexed source, prior_ids is empty and- # replace is a no-op, so this is always safe.+ # Roll back uncommitted changes (such as ensure_in_collection staging+ # and autoflush from query(Collection)) so no SQLite transaction+ # or write lock is held during Phase 2 embedding.+ safe_rollback(+ session, "library_rag_service.index_document phase 1"+ )++ except Exception as e:+ # The session is shared (thread-local) with the caller.+ # If session.flush() or session.commit() raised, the session+ # is in PendingRollbackError state until rolled back —+ # leaving subsequent operations to cascade. Roll back BEFORE+ # returning the error dict so the caller sees a clean+ # session. (Same pattern as the #3827 fix.)+ safe_rollback(session, "library_rag_service.index_document")+ # Backstop for the store-durable-before-DB-commit window: a+ # REPLACE reindex ran store.apply() (DURABLY removing this+ # source's prior vectors) before the DB commit, so the rollback+ # above resurrects rows whose vectors are already gone while+ # reverting indexed→its pre-attempt True. Flag for re-index so+ # the reconciler re-embeds and store+DB reconverge.+ self._flag_document_for_reindex(+ session, document_id, collection_id+ )+ logger.exception(+ f"Error indexing document {document_id} for collection {collection_id}"+ )+ return {+ "status": "error",+ "error": f"Operation failed: {type(e).__name__}",+ }++ # Phase 2: Embed vectors outside any DB session+ try:+ texts = [c.text for c in chunk_inputs]+ vectors = (+ np.asarray(+ self.embedding_manager.embeddings.embed_documents(texts),+ dtype="float32",+ )+ if texts+ else None+ )+ except Exception as e:+ logger.exception(+ f"Error embedding document {document_id} for collection {collection_id}"+ )+ with get_user_db_session(+ self.username, self.db_password+ ) as session:+ self._flag_document_for_reindex(+ session, document_id, collection_id+ )+ return {+ "status": "error",+ "error": f"Operation failed: {type(e).__name__}",+ }++ # Phase 3: Open short session to persist chunks, apply vectors, and commit+ with get_user_db_session(self.username, self.db_password) as session:+ try:+ # Phase 3 Revalidation Gate: verify collection link, content, and indexed status+ link = (+ session.query(DocumentCollection)+ .filter_by(+ document_id=document_id,+ collection_id=collection_id,+ )+ .first()+ )+ if link is None:+ logger.info(+ f"Document {document_id} was removed from collection {collection_id} "+ "during embedding; aborting Phase 3 persist."+ )+ self._flag_document_for_reindex(+ session, document_id, collection_id+ )+ return {+ "status": "skipped",+ "message": "Document removed from collection during embedding",+ }++ current_doc = (+ session.query(Document).filter_by(id=document_id).first()+ )+ if not current_doc:+ logger.info(+ f"Document {document_id} was deleted during embedding; "+ "aborting Phase 3 persist."+ )+ self._flag_document_for_reindex(+ session, document_id, collection_id+ )+ return {+ "status": "error",+ "error": "Document deleted during indexing",+ }++ current_content_hash = hashlib.sha256(+ (current_doc.text_content or "").encode("utf-8")+ ).hexdigest()+ if (+ current_content_hash != phase1_content_hash+ or current_doc.document_hash != phase1_doc_hash+ ):+ logger.info(+ f"Document {document_id} content changed during embedding; "+ "aborting persist and flagging for reindex."+ )+ self._flag_document_for_reindex(+ session, document_id, collection_id+ )+ return {+ "status": "stale",+ "message": "Document modified during indexing, flagged for reindex",+ }++ current_status_exists = (+ session.query(RagDocumentStatus)+ .filter_by(+ document_id=document_id,+ collection_id=collection_id,+ rag_index_id=self.rag_index_record.id,+ )+ .first()+ is not None+ if self.rag_index_record+ else False+ )+ if (+ isinstance(link.indexed, bool)+ and isinstance(phase1_indexed, bool)+ and link.indexed != phase1_indexed+ ) or (+ isinstance(current_status_exists, bool)+ and isinstance(phase1_status_exists, bool)+ and current_status_exists != phase1_status_exists+ ):+ logger.info(+ f"Document {document_id} indexing status modified during embedding; "+ "aborting persist and flagging for reindex."+ )+ self._flag_document_for_reindex(+ session, document_id, collection_id+ )+ return {+ "status": "stale",+ "message": "Document indexed state changed during indexing, flagged for reindex",+ }+vindex = self._get_vector_index(collection_id, collection_name, reset_stale_state=True)- index_stats = vindex.index(- source_type="document",- source_id=document_id,- chunks=chunk_inputs,- replace=True,- session=session,- )++ if hasattr(vindex, "index_prepared"):+ index_stats = vindex.index_prepared(+ source_type="document",+ source_id=document_id,+ chunks=chunk_inputs,+ vectors=vectors,+ replace=True,+ session=session,+ )+ else:+ index_stats = vindex.index(+ source_type="document",+ source_id=document_id,+ chunks=chunk_inputs,+ replace=True,+ session=session,+ )+logger.info(f"Vector index update for document {document_id}: "f"{index_stats.added} added, {index_stats.removed} "@@ -2198,15 +2538,39 @@ class LibraryRAGService:)# Also update DocumentCollection table for backward compatibility- session.query(DocumentCollection).filter_by(- document_id=document_id, collection_id=collection_id- ).update(- {- "indexed": True,- "chunk_count": len(chunks),- "last_indexed_at": timestamp,- }+ _serial_updated = (+ session.query(DocumentCollection)+ .filter_by(+ document_id=document_id,+ collection_id=collection_id,+ )+ .update(+ {+ "indexed": True,+ "chunk_count": len(chunks),+ "last_indexed_at": timestamp,+ }+ ))+ if isinstance(_serial_updated, int) and _serial_updated == 0:+ # An unlink committed between the Phase-3 gate query and+ # these persist writes. Roll back the staged status merge+ # above so no non-member row/aggregate is resurrected.+ logger.info(+ f"Document {document_id} was removed from collection {collection_id} "+ "during persist; aborting Phase 3 commit."+ )+ safe_rollback(+ session,+ "library_rag_service.index_document link vanished",+ )+ self._flag_document_for_reindex(+ session, document_id, collection_id+ )+ return {+ "status": "skipped",+ "message": "Document removed from collection during embedding",+ }logger.info("Also updated DocumentCollection.indexed for backward compatibility"@@ -2233,12 +2597,6 @@ class LibraryRAGService:f"total_documents {'+1' if not was_already_indexed else '+0'}")- # Replace-on-reindex DB prune: no longer needed here —- # ``VectorIndex.index(replace=True)`` above already deletes- # this source's prior DocumentChunk rows (and their- # vectors) as part of the same transaction/lock when- # ``force_reindex`` is set.-# Flush ORM changes to database before commitsession.flush()logger.info(f"Flushed ORM changes for document {document_id}")diff --git a/tests/research_library/services/test_index_documents_parallel.py b/tests/research_library/services/test_index_documents_parallel.pyindex 94cb09a6a0..b23d56db2c 100644--- a/tests/research_library/services/test_index_documents_parallel.py+++ b/tests/research_library/services/test_index_documents_parallel.py@@ -956,6 +956,8 @@ class TestWritePreparedDocumentReindexDelta:"""``_write_prepared_document`` accurately updates ``rag_index.chunk_count`` on re-index."""def test_chunk_count_delta_captures_old_chunks_before_merge(self):+ import hashlib+svc = _make_service()svc.rag_index_record = MagicMock()svc.rag_index_record.id = 1@@ -975,15 +977,35 @@ class TestWritePreparedDocumentReindexDelta:q.first.return_value = first_valreturn q+ body = "delta body"+ body_hash = hashlib.sha256(body.encode()).hexdigest()+ mock_link = MagicMock()+ mock_link.indexed = False+ mock_doc = MagicMock()+ mock_doc.text_content = body+session = MagicMock()- # session.query calls:- # 1. RagDocumentStatus (existing)- # 2. DocumentCollection update- # 3. RAGIndex+ # session.query calls (Phase-3 gate runs unconditionally):+ # 1. DocumentCollection link (gate)+ # 2. Document existence (gate)+ # 3. RagDocumentStatus existence (gate)+ # 4. RagDocumentStatus (existing, for delta)+ # 5. DocumentCollection update (TOCTOU rowcount; MagicMock -> no abort)+ # 6. RAGIndex+ q_link = make_q(first_val=mock_link)+ q_doc = make_q(first_val=mock_doc)+ q_gate_status = make_q(first_val=mock_existing)q_existing = make_q(first_val=mock_existing)q_doc_coll = make_q()q_rag_index = make_q(first_val=mock_rag_index)- session.query.side_effect = [q_existing, q_doc_coll, q_rag_index]+ session.query.side_effect = [+ q_link,+ q_doc,+ q_gate_status,+ q_existing,+ q_doc_coll,+ q_rag_index,+ ]prepared = MagicMock()prepared.document_id = "doc-1"@@ -993,6 +1015,10 @@ class TestWritePreparedDocumentReindexDelta:MagicMock(),MagicMock(),] # 2 new chunks (delta = 2 - 5 = -3)+ prepared.content_hash = body_hash+ prepared.initial_indexed = False+ prepared.status_exists = True+ prepared.had_collection_link = Truesession_ctx = MagicMock()session_ctx.__enter__ = MagicMock(return_value=session)diff --git a/tests/research_library/services/test_issue_6559_rag_indexing_lock.py b/tests/research_library/services/test_issue_6559_rag_indexing_lock.pynew file mode 100644index 0000000000..397cd7faf5--- /dev/null+++ b/tests/research_library/services/test_issue_6559_rag_indexing_lock.py@@ -0,0 +1,1506 @@+"""Exhaustive tests for Issue #6559: RAG Indexing Lock Across Embedding Round-Trip.++Verifies that RAG document indexing is decoupled into 3 phases:+1. Phase 1: Read & chunk document in a short session (closing transaction before embedding).+2. Phase 2: Embed vectors strictly outside of any DB session / write lock.+3. Phase 3: Open short session with revalidation gate to persist chunks, apply vectors, and commit.+"""++from contextlib import contextmanager+import hashlib+import threading+from unittest.mock import MagicMock, patch+import pytest+from sqlalchemy import create_engine, text+from sqlalchemy.orm import sessionmaker++from local_deep_research.database.models import Base+from local_deep_research.database.models.library import (+ Collection,+ Document,+ DocumentChunk,+ DocumentCollection,+ DocumentStatus,+ EmbeddingProvider,+ RAGIndex,+ RagDocumentStatus,+ SourceType,+)+from local_deep_research.research_library.services.library_rag_service import (+ LibraryRAGService,+ _PreparedDocument,+)+from local_deep_research.vector_stores.facade import ChunkInput+++class TestRAGIndexingLockDecoupling:+ """Test suite verifying lock decoupling across embedding round-trip."""++ @pytest.fixture+ def mock_service(self):+ """Create a LibraryRAGService with mocked dependencies."""+ svc = LibraryRAGService.__new__(LibraryRAGService)+ svc.username = "test_user"+ svc._db_password = "test_password"+ svc.embedding_model = "test-model"+ svc.embedding_provider = "test-provider"+ svc.chunk_size = 500+ svc.chunk_overlap = 50+ svc.text_splitter = MagicMock()+ svc.embedding_manager = MagicMock()++ rag_rec = MagicMock(spec=RAGIndex)+ rag_rec.id = 1+ rag_rec.index_path = "/tmp/test_index"+ rag_rec.chunk_count = 0+ rag_rec.total_documents = 0+ svc.rag_index_record = rag_rec++ svc._get_index_hash = MagicMock(return_value="test_hash")+ svc._get_index_path = MagicMock(return_value="/tmp/test_index_path")+ return svc++ def test_embed_documents_called_outside_db_session(self, mock_service):+ """Verify embed_documents executes while NO database session is held."""+ session_active = False+ embed_executed_outside_session = False++ doc = MagicMock(spec=Document)+ doc.id = "doc-1"+ doc.text_content = "Test content for chunking and embedding."+ doc.title = "Test Doc"+ doc.filename = "test.pdf"+ doc.original_url = "http://example.com/test"+ doc.authors = "Author"+ doc.published_date = None+ doc.doi = None+ doc.arxiv_id = None+ doc.pmid = None+ doc.pmcid = None+ doc.extraction_method = "pdf"+ doc.word_count = 10+ doc.document_hash = "doc_hash_1"++ coll = MagicMock(spec=Collection)+ coll.id = "coll-1"++ doc_coll = MagicMock(spec=DocumentCollection)+ doc_coll.indexed = False+ doc_coll.chunk_count = 0++ rag_index = MagicMock(spec=RAGIndex)+ rag_index.id = 1+ rag_index.chunk_count = 0+ rag_index.total_documents = 0++ # Create session mock tracking active context state+ class TrackedSessionContext:+ def __init__(self, session_obj):+ self.session = session_obj++ def __enter__(self):+ nonlocal session_active+ session_active = True+ return self.session++ def __exit__(self, exc_type, exc_val, exc_tb):+ nonlocal session_active+ session_active = False+ return False++ mock_session = MagicMock()++ def query_side_effect(model):+ q = MagicMock()+ if model == Document:+ q.filter_by.return_value.first.return_value = doc+ elif model == Collection:+ q.filter_by.return_value.first.return_value = coll+ elif model == DocumentCollection:+ q.filter_by.return_value.first.return_value = doc_coll+ elif model == RagDocumentStatus:+ q.filter_by.return_value.first.return_value = None+ elif model == RAGIndex:+ q.filter_by.return_value.first.return_value = rag_index+ return q++ mock_session.query.side_effect = query_side_effect++ def mock_embed(texts):+ nonlocal embed_executed_outside_session, session_active+ # Assert that session is NOT active when embed_documents is called+ assert not session_active, (+ "embed_documents was called while DB session was ACTIVE!"+ )+ embed_executed_outside_session = True+ return [[0.1, 0.2, 0.3]]++ mock_service.embedding_manager.embeddings.embed_documents.side_effect = mock_embed++ chunk_mock = MagicMock()+ chunk_mock.page_content = "Test content chunk"+ mock_service.text_splitter.split_documents.return_value = [chunk_mock]++ vindex = MagicMock()+ vindex.index_prepared.return_value = MagicMock(+ added=1, removed=0, chunks=1+ )+ mock_service._get_vector_index = MagicMock(return_value=vindex)++ with (+ patch(+ "local_deep_research.research_library.services.library_rag_service.get_user_db_session",+ side_effect=lambda *args, **kwargs: TrackedSessionContext(+ mock_session+ ),+ ),+ patch(+ "local_deep_research.research_library.services.library_rag_service.ensure_in_collection",+ return_value=doc_coll,+ ),+ ):+ result = mock_service._index_document_locked("doc-1", "coll-1")++ assert result["status"] == "success"+ assert result["chunk_count"] == 1+ assert embed_executed_outside_session, (+ "embed_documents was never executed"+ )+ vindex.index_prepared.assert_called_once()++ def test_embedding_failure_flags_document_for_reindex_without_dirtying_main_session(+ self, mock_service+ ):+ """If embedding in Phase 2 fails, document is flagged indexed=False."""+ doc = MagicMock(spec=Document)+ doc.id = "doc-1"+ doc.text_content = "Valid content"+ doc.title = "Test Doc"+ doc.filename = "test.pdf"+ doc.original_url = None+ doc.authors = None+ doc.published_date = None+ doc.doi = None+ doc.arxiv_id = None+ doc.pmid = None+ doc.pmcid = None+ doc.extraction_method = None+ doc.word_count = 2+ doc.document_hash = "h1"++ doc_coll = MagicMock(spec=DocumentCollection)+ doc_coll.indexed = False+ doc_coll.chunk_count = 0++ mock_session = MagicMock()+ mock_session.query.return_value.filter_by.return_value.first.return_value = doc++ # Fail during Phase 2 embedding+ mock_service.embedding_manager.embeddings.embed_documents.side_effect = RuntimeError(+ "Embedding network timeout"+ )+ chunk_mock = MagicMock()+ chunk_mock.page_content = "Chunk"+ mock_service.text_splitter.split_documents.return_value = [chunk_mock]++ mock_service._flag_document_for_reindex = MagicMock()++ class DummyCtx:+ def __enter__(self):+ return mock_session++ def __exit__(self, *args):+ return False++ with (+ patch(+ "local_deep_research.research_library.services.library_rag_service.get_user_db_session",+ return_value=DummyCtx(),+ ),+ patch(+ "local_deep_research.research_library.services.library_rag_service.ensure_in_collection",+ return_value=doc_coll,+ ),+ ):+ result = mock_service._index_document_locked("doc-1", "coll-1")++ assert result["status"] == "error"+ assert "RuntimeError" in result["error"]+ mock_service._flag_document_for_reindex.assert_called_once_with(+ mock_session, "doc-1", "coll-1"+ )++ def test_vindex_index_prepared_failure_triggers_rollback_and_reindex_flag(+ self, mock_service+ ):+ """When vindex.index_prepared raises an error, session rolls back and document is flagged."""+ doc = MagicMock(spec=Document)+ doc.id = "doc-1"+ doc.text_content = "Valid content"+ doc.title = "Test Doc"+ doc.filename = "test.pdf"+ doc.original_url = None+ doc.authors = None+ doc.published_date = None+ doc.doi = None+ doc.arxiv_id = None+ doc.pmid = None+ doc.pmcid = None+ doc.extraction_method = None+ doc.word_count = 2+ doc.document_hash = "h1"++ doc_coll = MagicMock(spec=DocumentCollection)+ doc_coll.indexed = False+ doc_coll.chunk_count = 0++ mock_session = MagicMock()++ def query_side_effect(model):+ q = MagicMock()+ if model == Document:+ q.filter_by.return_value.first.return_value = doc+ elif model == DocumentCollection:+ q.filter_by.return_value.first.return_value = doc_coll+ elif model == RagDocumentStatus:+ q.filter_by.return_value.first.return_value = None+ return q++ mock_session.query.side_effect = query_side_effect++ chunk_mock = MagicMock()+ chunk_mock.page_content = "Chunk"+ mock_service.text_splitter.split_documents.return_value = [chunk_mock]+ mock_service.embedding_manager.embeddings.embed_documents.return_value = [+ [0.1, 0.2]+ ]++ # vindex with index_prepared raising RuntimeError+ vindex = MagicMock()+ vindex.index_prepared.side_effect = RuntimeError(+ "store persisted no embeddings"+ )+ mock_service._get_vector_index = MagicMock(return_value=vindex)+ mock_service._flag_document_for_reindex = MagicMock()++ class DummyCtx:+ def __enter__(self):+ return mock_session++ def __exit__(self, *args):+ return False++ with (+ patch(+ "local_deep_research.research_library.services.library_rag_service.get_user_db_session",+ return_value=DummyCtx(),+ ),+ patch(+ "local_deep_research.research_library.services.library_rag_service.ensure_in_collection",+ return_value=doc_coll,+ ),+ ):+ result = mock_service._index_document_locked("doc-1", "coll-1")++ assert result["status"] == "error"+ assert "RuntimeError" in result["error"]+ mock_service._flag_document_for_reindex.assert_called_once_with(+ mock_session, "doc-1", "coll-1"+ )++ def test_vindex_fallback_to_index_when_index_prepared_not_implemented(+ self, mock_service+ ):+ """Falls back to calling vindex.index when index_prepared is absent on vindex."""+ doc = MagicMock(spec=Document)+ doc.id = "doc-1"+ doc.text_content = "Valid content"+ doc.title = "Test Doc"+ doc.filename = "test.pdf"+ doc.original_url = None+ doc.authors = None+ doc.published_date = None+ doc.doi = None+ doc.arxiv_id = None+ doc.pmid = None+ doc.pmcid = None+ doc.extraction_method = None+ doc.word_count = 2+ doc.document_hash = "h1"++ doc_coll = MagicMock(spec=DocumentCollection)+ doc_coll.indexed = False+ doc_coll.chunk_count = 0++ mock_session = MagicMock()++ def query_side_effect(model):+ q = MagicMock()+ if model == Document:+ q.filter_by.return_value.first.return_value = doc+ elif model == DocumentCollection:+ q.filter_by.return_value.first.return_value = doc_coll+ elif model == RagDocumentStatus:+ q.filter_by.return_value.first.return_value = None+ elif model == RAGIndex:+ rag_rec = MagicMock(spec=RAGIndex)+ rag_rec.chunk_count = 0+ rag_rec.total_documents = 0+ q.filter_by.return_value.first.return_value = rag_rec+ return q++ mock_session.query.side_effect = query_side_effect++ chunk_mock = MagicMock()+ chunk_mock.page_content = "Chunk"+ mock_service.text_splitter.split_documents.return_value = [chunk_mock]+ mock_service.embedding_manager.embeddings.embed_documents.return_value = [+ [0.1, 0.2]+ ]++ # vindex with only index, no index_prepared+ class LegacyVectorIndex:+ def __init__(self):+ self.index = MagicMock(+ return_value=MagicMock(added=1, removed=0, chunks=1)+ )++ vindex = LegacyVectorIndex()+ mock_service._get_vector_index = MagicMock(return_value=vindex)++ class DummyCtx:+ def __enter__(self):+ return mock_session++ def __exit__(self, *args):+ return False++ with (+ patch(+ "local_deep_research.research_library.services.library_rag_service.get_user_db_session",+ return_value=DummyCtx(),+ ),+ patch(+ "local_deep_research.research_library.services.library_rag_service.ensure_in_collection",+ return_value=doc_coll,+ ),+ ):+ result = mock_service._index_document_locked("doc-1", "coll-1")++ assert result["status"] == "success"+ vindex.index.assert_called_once()++ def test_phase3_aborts_when_document_removed_from_collection(+ self, mock_service+ ):+ """Phase 3 revalidation gate: if document was unlinked during embedding, abort without writing."""+ doc = MagicMock(spec=Document)+ doc.id = "doc-1"+ doc.text_content = "Valid content"+ doc.title = "Test Doc"+ doc.filename = "test.pdf"+ doc.original_url = None+ doc.authors = None+ doc.published_date = None+ doc.doi = None+ doc.arxiv_id = None+ doc.pmid = None+ doc.pmcid = None+ doc.extraction_method = None+ doc.word_count = 2+ doc.document_hash = "h1"++ doc_coll = MagicMock(spec=DocumentCollection)+ doc_coll.indexed = False+ doc_coll.chunk_count = 0++ # Phase 1 sees link; Phase 3 sees link deleted (None)+ p1_session = MagicMock()+ p3_session = MagicMock()++ def p1_query(model):+ q = MagicMock()+ if model == Document:+ q.filter_by.return_value.first.return_value = doc+ elif model == DocumentCollection:+ q.filter_by.return_value.first.return_value = doc_coll+ elif model == RagDocumentStatus:+ q.filter_by.return_value.first.return_value = None+ return q++ p1_session.query.side_effect = p1_query++ def p3_query(model):+ q = MagicMock()+ if model == DocumentCollection:+ # Document was unlinked from collection during Phase 2 embedding!+ q.filter_by.return_value.first.return_value = None+ elif model == Document:+ q.filter_by.return_value.first.return_value = doc+ return q++ p3_session.query.side_effect = p3_query++ chunk_mock = MagicMock()+ chunk_mock.page_content = "Chunk"+ mock_service.text_splitter.split_documents.return_value = [chunk_mock]+ mock_service.embedding_manager.embeddings.embed_documents.return_value = [+ [0.1, 0.2]+ ]++ vindex = MagicMock()+ mock_service._get_vector_index = MagicMock(return_value=vindex)+ mock_service._flag_document_for_reindex = MagicMock()++ session_sequence = [p1_session, p3_session]++ class SequenceCtx:+ def __enter__(self):+ return session_sequence.pop(0)++ def __exit__(self, *args):+ return False++ with (+ patch(+ "local_deep_research.research_library.services.library_rag_service.get_user_db_session",+ side_effect=lambda *a, **kw: SequenceCtx(),+ ),+ patch(+ "local_deep_research.research_library.services.library_rag_service.ensure_in_collection",+ return_value=doc_coll,+ ) as mock_ensure,+ ):+ result = mock_service._index_document_locked("doc-1", "coll-1")++ # Must abort with skipped+ assert result["status"] == "skipped"+ assert "removed from collection" in result["message"]+ # Must flag for reindex+ mock_service._flag_document_for_reindex.assert_called_once_with(+ p3_session, "doc-1", "coll-1"+ )+ # Vector index must NOT be written+ vindex.index_prepared.assert_not_called()+ # ensure_in_collection must NOT be called in Phase 3 (called only once in Phase 1)+ assert mock_ensure.call_count == 1++ def test_phase3_aborts_when_note_edited_during_embedding(+ self, mock_service+ ):+ """Phase 3 revalidation gate: if note content changed during embedding, abort to avoid stomping."""+ p1_doc = MagicMock(spec=Document)+ p1_doc.id = "doc-1"+ p1_doc.text_content = "Pre-edit note content"+ p1_doc.document_hash = "pre_edit_hash"+ p1_doc.title = "Note"+ p1_doc.filename = "note.txt"+ p1_doc.original_url = None+ p1_doc.authors = None+ p1_doc.published_date = None+ p1_doc.doi = None+ p1_doc.arxiv_id = None+ p1_doc.pmid = None+ p1_doc.pmcid = None+ p1_doc.extraction_method = None+ p1_doc.word_count = 4++ # In Phase 3, document has new edited text+ p3_doc = MagicMock(spec=Document)+ p3_doc.id = "doc-1"+ p3_doc.text_content = "Post-edit updated note content"+ p3_doc.document_hash = "post_edit_hash"++ doc_coll = MagicMock(spec=DocumentCollection)+ doc_coll.indexed = False+ doc_coll.chunk_count = 0++ p1_session = MagicMock()+ p3_session = MagicMock()++ def p1_query(model):+ q = MagicMock()+ if model == Document:+ q.filter_by.return_value.first.return_value = p1_doc+ elif model == DocumentCollection:+ q.filter_by.return_value.first.return_value = doc_coll+ elif model == RagDocumentStatus:+ q.filter_by.return_value.first.return_value = None+ return q++ p1_session.query.side_effect = p1_query++ def p3_query(model):+ q = MagicMock()+ if model == Document:+ q.filter_by.return_value.first.return_value = p3_doc+ elif model == DocumentCollection:+ q.filter_by.return_value.first.return_value = doc_coll+ elif model == RagDocumentStatus:+ q.filter_by.return_value.first.return_value = None+ return q++ p3_session.query.side_effect = p3_query++ chunk_mock = MagicMock()+ chunk_mock.page_content = "Pre-edit note content"+ mock_service.text_splitter.split_documents.return_value = [chunk_mock]+ mock_service.embedding_manager.embeddings.embed_documents.return_value = [+ [0.1, 0.2]+ ]++ vindex = MagicMock()+ mock_service._get_vector_index = MagicMock(return_value=vindex)+ mock_service._flag_document_for_reindex = MagicMock()++ session_sequence = [p1_session, p3_session]++ class SequenceCtx:+ def __enter__(self):+ return session_sequence.pop(0)++ def __exit__(self, *args):+ return False++ with (+ patch(+ "local_deep_research.research_library.services.library_rag_service.get_user_db_session",+ side_effect=lambda *a, **kw: SequenceCtx(),+ ),+ patch(+ "local_deep_research.research_library.services.library_rag_service.ensure_in_collection",+ return_value=doc_coll,+ ),+ ):+ result = mock_service._index_document_locked("doc-1", "coll-1")++ # Aborts with stale status+ assert result["status"] == "stale"+ assert "modified during indexing" in result["message"]+ mock_service._flag_document_for_reindex.assert_called_once_with(+ p3_session, "doc-1", "coll-1"+ )+ # Pre-edit chunks must NOT be written to vector index+ vindex.index_prepared.assert_not_called()++ def test_phase3_aborts_when_document_deleted(self, mock_service):+ """Phase 3 revalidation gate: if document row was deleted during embedding, abort."""+ doc = MagicMock(spec=Document)+ doc.id = "doc-1"+ doc.text_content = "Some content"+ doc.document_hash = "h1"+ doc.title = "Doc"+ doc.filename = "doc.txt"+ doc.original_url = None+ doc.authors = None+ doc.published_date = None+ doc.doi = None+ doc.arxiv_id = None+ doc.pmid = None+ doc.pmcid = None+ doc.extraction_method = None+ doc.word_count = 2++ doc_coll = MagicMock(spec=DocumentCollection)+ doc_coll.indexed = False++ p1_session = MagicMock()+ p3_session = MagicMock()++ p1_session.query.return_value.filter_by.return_value.first.return_value = doc++ def p3_query(model):+ q = MagicMock()+ if model == DocumentCollection:+ q.filter_by.return_value.first.return_value = doc_coll+ elif model == Document:+ # Document was deleted!+ q.filter_by.return_value.first.return_value = None+ return q++ p3_session.query.side_effect = p3_query++ chunk_mock = MagicMock()+ chunk_mock.page_content = "Chunk"+ mock_service.text_splitter.split_documents.return_value = [chunk_mock]+ mock_service.embedding_manager.embeddings.embed_documents.return_value = [+ [0.1]+ ]++ vindex = MagicMock()+ mock_service._get_vector_index = MagicMock(return_value=vindex)+ mock_service._flag_document_for_reindex = MagicMock()++ session_sequence = [p1_session, p3_session]++ class SequenceCtx:+ def __enter__(self):+ return session_sequence.pop(0)++ def __exit__(self, *args):+ return False++ with (+ patch(+ "local_deep_research.research_library.services.library_rag_service.get_user_db_session",+ side_effect=lambda *a, **kw: SequenceCtx(),+ ),+ patch(+ "local_deep_research.research_library.services.library_rag_service.ensure_in_collection",+ return_value=doc_coll,+ ),+ ):+ result = mock_service._index_document_locked("doc-1", "coll-1")++ assert result["status"] == "error"+ assert "deleted" in result["error"].lower()+ mock_service._flag_document_for_reindex.assert_called_once_with(+ p3_session, "doc-1", "coll-1"+ )+ vindex.index_prepared.assert_not_called()++ def test_write_prepared_document_revalidation_gate(self, mock_service):+ """_write_prepared_document enforces revalidation gate before persisting."""+ mock_session = MagicMock()+ current_doc = MagicMock(spec=Document)+ current_doc.text_content = "Post-edit content"+ link = MagicMock(spec=DocumentCollection)+ link.indexed = False++ def query_side_effect(model):+ q = MagicMock()+ if model == Document:+ q.filter_by.return_value.first.return_value = current_doc+ elif model == DocumentCollection:+ q.filter_by.return_value.first.return_value = link+ elif model == RagDocumentStatus:+ q.filter_by.return_value.first.return_value = None+ return q++ mock_session.query.side_effect = query_side_effect+ mock_service._flag_document_for_reindex = MagicMock()++ class DummyCtx:+ def __enter__(self):+ return mock_session++ def __exit__(self, *args):+ return False++ prepared = _PreparedDocument(+ document_id="doc-1",+ collection_id="coll-1",+ chunk_inputs=[ChunkInput(text="Pre-edit content", metadata={})],+ vectors=MagicMock(),+ content_hash=hashlib.sha256(b"Pre-edit content").hexdigest(),+ initial_indexed=False,+ status_exists=False,+ )++ with (+ patch(+ "local_deep_research.research_library.services.library_rag_service.get_user_db_session",+ return_value=DummyCtx(),+ ),+ patch(+ "local_deep_research.research_library.services.library_rag_service._hold_faiss_write_lock",+ ),+ ):+ result = mock_service._write_prepared_document(prepared)++ assert result["status"] == "stale"+ assert "modified during indexing" in result["message"]+ mock_service._flag_document_for_reindex.assert_called_once_with(+ mock_session, "doc-1", "coll-1"+ )++ def test_write_prepared_initially_unlinked_creates_membership(+ self, mock_service+ ):+ """First-time ingest: no DocumentCollection at prepare is not a concurrent unlink.++ Reconciler case (b) research orphans have no membership when+ ``_prepare_document`` runs. Phase 3 must create the link and persist+ vectors instead of returning skipped.+ """+ mock_session = MagicMock()+ current_doc = MagicMock(spec=Document)+ current_doc.text_content = "Orphan research download"+ created_link = MagicMock(spec=DocumentCollection)+ created_link.indexed = False++ def query_side_effect(model):+ q = MagicMock()+ if model == Document:+ q.filter_by.return_value.first.return_value = current_doc+ elif model == DocumentCollection:+ q.filter_by.return_value.first.return_value = None+ elif model == RagDocumentStatus:+ q.filter_by.return_value.first.return_value = None+ elif model == RAGIndex:+ q.filter_by.return_value.first.return_value = (+ mock_service.rag_index_record+ )+ return q++ mock_session.query.side_effect = query_side_effect+ mock_service._flag_document_for_reindex = MagicMock()++ vindex = MagicMock()+ vindex.index_prepared.return_value = MagicMock(+ added=1, removed=0, chunks=1+ )+ mock_service._get_vector_index = MagicMock(return_value=vindex)++ class DummyCtx:+ def __enter__(self):+ return mock_session++ def __exit__(self, *args):+ return False++ prepared = _PreparedDocument(+ document_id="doc-orphan",+ collection_id="default-lib",+ chunk_inputs=[+ ChunkInput(text="Orphan research download", metadata={})+ ],+ vectors=MagicMock(),+ content_hash=hashlib.sha256(+ b"Orphan research download"+ ).hexdigest(),+ initial_indexed=False,+ status_exists=False,+ had_collection_link=False,+ )++ with (+ patch(+ "local_deep_research.research_library.services.library_rag_service.get_user_db_session",+ return_value=DummyCtx(),+ ),+ patch(+ "local_deep_research.research_library.services.library_rag_service._hold_faiss_write_lock",+ ),+ patch(+ "local_deep_research.research_library.services.library_rag_service.ensure_in_collection",+ return_value=created_link,+ ) as mock_ensure,+ ):+ result = mock_service._write_prepared_document(prepared)++ assert result["status"] == "success"+ assert result["chunk_count"] == 1+ mock_ensure.assert_called_once_with(+ mock_session, "doc-orphan", "default-lib"+ )+ mock_session.flush.assert_called()+ vindex.index_prepared.assert_called_once()+ mock_service._flag_document_for_reindex.assert_not_called()++ def test_write_prepared_concurrent_unlink_still_skips(self, mock_service):+ """A link present at prepare and gone at write is still a concurrent removal."""+ mock_session = MagicMock()+ current_doc = MagicMock(spec=Document)+ current_doc.text_content = "Linked note"++ def query_side_effect(model):+ q = MagicMock()+ if model == Document:+ q.filter_by.return_value.first.return_value = current_doc+ elif model == DocumentCollection:+ q.filter_by.return_value.first.return_value = None+ elif model == RagDocumentStatus:+ q.filter_by.return_value.first.return_value = None+ return q++ mock_session.query.side_effect = query_side_effect+ mock_service._flag_document_for_reindex = MagicMock()+ vindex = MagicMock()+ mock_service._get_vector_index = MagicMock(return_value=vindex)++ class DummyCtx:+ def __enter__(self):+ return mock_session++ def __exit__(self, *args):+ return False++ prepared = _PreparedDocument(+ document_id="doc-1",+ collection_id="coll-1",+ chunk_inputs=[ChunkInput(text="Linked note", metadata={})],+ vectors=MagicMock(),+ content_hash=hashlib.sha256(b"Linked note").hexdigest(),+ initial_indexed=False,+ status_exists=False,+ had_collection_link=True,+ )++ with (+ patch(+ "local_deep_research.research_library.services.library_rag_service.get_user_db_session",+ return_value=DummyCtx(),+ ),+ patch(+ "local_deep_research.research_library.services.library_rag_service._hold_faiss_write_lock",+ ),+ patch(+ "local_deep_research.research_library.services.library_rag_service.ensure_in_collection",+ ) as mock_ensure,+ ):+ result = mock_service._write_prepared_document(prepared)++ assert result["status"] == "skipped"+ assert "removed from collection" in result["message"]+ mock_ensure.assert_not_called()+ vindex.index_prepared.assert_not_called()+ mock_service._flag_document_for_reindex.assert_called_once_with(+ mock_session, "doc-1", "coll-1"+ )++ def test_prepare_snapshots_missing_collection_link(self, mock_service):+ """``_prepare_document`` records that no DocumentCollection existed."""+ doc = MagicMock(spec=Document)+ doc.id = "doc-orphan"+ doc.text_content = "Orphan body"+ doc.title = "Orphan"+ doc.filename = "orphan.txt"+ doc.original_url = None+ doc.authors = None+ doc.published_date = None+ doc.doi = None+ doc.arxiv_id = None+ doc.pmid = None+ doc.pmcid = None+ doc.extraction_method = None+ doc.word_count = 2++ mock_session = MagicMock()++ def query_side_effect(model):+ q = MagicMock()+ if model == Document:+ q.filter_by.return_value.first.return_value = doc+ elif model == DocumentCollection:+ q.filter_by.return_value.first.return_value = None+ elif model == RagDocumentStatus:+ q.filter_by.return_value.first.return_value = None+ return q++ mock_session.query.side_effect = query_side_effect+ chunk_mock = MagicMock()+ chunk_mock.page_content = "Orphan body"+ mock_service.text_splitter.split_documents.return_value = [chunk_mock]+ mock_service.embedding_manager.embeddings.embed_documents.return_value = [+ [0.1, 0.2]+ ]++ class DummyCtx:+ def __enter__(self):+ return mock_session++ def __exit__(self, *args):+ return False++ with patch(+ "local_deep_research.research_library.services.library_rag_service.get_user_db_session",+ return_value=DummyCtx(),+ ):+ result = mock_service._prepare_document(+ "doc-orphan", "default-lib", force_reindex=False+ )++ assert result["status"] == "prepared"+ prepared = result["prepared"]+ assert prepared.had_collection_link is False+ assert prepared.initial_indexed is False++ def test_prepared_pipeline_indexes_research_orphan_with_real_sqlite(+ self, tmp_path+ ):+ """Reconciler case (b): unlinked document becomes searchable via prepared path.++ Mirrors the background sweep: Document exists, no DocumentCollection,+ ``_prepare_document`` then ``_write_prepared_document`` with real SQLite+ and stubbed embedding/vector collaborators.+ """+ db_path = tmp_path / "test_rag_orphan.db"+ engine = create_engine(+ f"sqlite:///{db_path}",+ connect_args={"check_same_thread": False},+ )+ Base.metadata.create_all(engine)+ SessionLocal = sessionmaker(bind=engine)++ body = "Research orphan text for first-time ingest"+ body_hash = hashlib.sha256(body.encode()).hexdigest()++ with SessionLocal() as init_session:+ st = SourceType(+ id="st-orphan", name="document", display_name="Document"+ )+ coll = Collection(+ id="default-lib",+ name="Library",+ collection_type="default_library",+ is_default=True,+ )+ doc = Document(+ id="doc-orphan",+ source_type_id="st-orphan",+ document_hash=body_hash,+ title="Orphan Paper",+ text_content=body,+ file_size=len(body),+ file_type="txt",+ status=DocumentStatus.COMPLETED,+ )+ rag_idx = RAGIndex(+ id=1,+ collection_name="default-lib",+ embedding_model="test-model",+ embedding_model_type=EmbeddingProvider.SENTENCE_TRANSFORMERS,+ embedding_dimension=3,+ index_path=str(tmp_path / "index"),+ index_hash="test_hash",+ chunk_size=500,+ chunk_overlap=50,+ )+ init_session.add_all([st, coll, doc, rag_idx])+ init_session.commit()++ svc = LibraryRAGService.__new__(LibraryRAGService)+ svc.username = "test_user"+ svc._db_password = "test_password"+ svc.embedding_model = "test-model"+ svc.embedding_provider = "test-provider"+ svc.chunk_size = 500+ svc.chunk_overlap = 50+ svc.text_splitter = MagicMock()+ chunk_mock = MagicMock()+ chunk_mock.page_content = body+ svc.text_splitter.split_documents.return_value = [chunk_mock]+ svc.embedding_manager = MagicMock()+ svc.embedding_manager.embeddings.embed_documents.return_value = [+ [0.1, 0.2, 0.3]+ ]+ rag_rec = MagicMock(spec=RAGIndex)+ rag_rec.id = 1+ rag_rec.chunk_count = 0+ rag_rec.total_documents = 0+ svc.rag_index_record = rag_rec+ svc._get_index_hash = MagicMock(return_value="test_hash")+ svc._get_index_path = MagicMock(return_value=str(tmp_path / "index"))++ vindex = MagicMock()+ vindex.index_prepared.return_value = MagicMock(+ added=1, removed=0, chunks=1+ )+ svc._get_vector_index = MagicMock(return_value=vindex)++ @contextmanager+ def real_session_ctx(*args, **kwargs):+ session = SessionLocal()+ try:+ yield session+ finally:+ session.close()++ with patch(+ "local_deep_research.research_library.services.library_rag_service.get_user_db_session",+ side_effect=real_session_ctx,+ ):+ prepared_result = svc._prepare_document(+ "doc-orphan", "default-lib", force_reindex=False+ )+ assert prepared_result["status"] == "prepared"+ prepared = prepared_result["prepared"]+ assert prepared.had_collection_link is False++ result = svc._write_prepared_document(prepared)++ assert result["status"] == "success"+ assert result["chunk_count"] == 1+ vindex.index_prepared.assert_called_once()++ with SessionLocal() as check_session:+ membership = (+ check_session.query(DocumentCollection)+ .filter_by(+ document_id="doc-orphan", collection_id="default-lib"+ )+ .first()+ )+ assert membership is not None+ assert membership.indexed is True+ assert membership.chunk_count == 1++ def test_already_indexed_document_skips_without_embedding(+ self, mock_service+ ):+ """If document is already indexed and force_reindex=False, return skipped without embedding."""+ doc = MagicMock(spec=Document)+ doc.id = "doc-1"+ doc.text_content = "Some text"++ doc_coll = MagicMock(spec=DocumentCollection)+ doc_coll.indexed = True+ doc_coll.chunk_count = 5++ mock_session = MagicMock()+ mock_session.query.return_value.filter_by.return_value.first.return_value = doc++ class DummyCtx:+ def __enter__(self):+ return mock_session++ def __exit__(self, *args):+ return False++ with (+ patch(+ "local_deep_research.research_library.services.library_rag_service.get_user_db_session",+ return_value=DummyCtx(),+ ),+ patch(+ "local_deep_research.research_library.services.library_rag_service.ensure_in_collection",+ return_value=doc_coll,+ ),+ ):+ result = mock_service._index_document_locked(+ "doc-1", "coll-1", force_reindex=False+ )++ assert result["status"] == "skipped"+ assert result["chunk_count"] == 5+ mock_service.embedding_manager.embeddings.embed_documents.assert_not_called()++ def test_empty_text_with_prior_chunks_purges_without_embedding(+ self, mock_service+ ):+ """Document with empty text and prior chunks purges chunks/vectors without embedding."""+ doc = MagicMock(spec=Document)+ doc.id = "doc-1"+ doc.text_content = ""++ doc_coll = MagicMock(spec=DocumentCollection)+ doc_coll.indexed = False++ mock_session = MagicMock()+ # 1st .first() = Document lookup, 2nd .first() = DocumentChunk prior lookup+ mock_session.query.return_value.filter_by.return_value.first.side_effect = [+ doc,+ MagicMock(spec=DocumentChunk), # Prior chunk exists+ ]++ vindex = MagicMock()+ mock_service._get_vector_index = MagicMock(return_value=vindex)++ class DummyCtx:+ def __enter__(self):+ return mock_session++ def __exit__(self, *args):+ return False++ with (+ patch(+ "local_deep_research.research_library.services.library_rag_service.get_user_db_session",+ return_value=DummyCtx(),+ ),+ patch(+ "local_deep_research.research_library.services.library_rag_service.ensure_in_collection",+ return_value=doc_coll,+ ),+ ):+ result = mock_service._index_document_locked("doc-1", "coll-1")++ assert result["status"] == "cleared"+ assert result["chunk_count"] == 0+ mock_service.embedding_manager.embeddings.embed_documents.assert_not_called()+ vindex.index.assert_called_once_with(+ source_type="document",+ source_id="doc-1",+ chunks=[],+ replace=True,+ session=mock_session,+ )+ assert doc_coll.indexed is False+ mock_session.commit.assert_called_once()++ def test_concurrent_db_write_during_embedding_phase_with_real_sqlite(+ self, tmp_path+ ):+ """Exhaustive test with real SQLite: honest discriminator demonstrating lock fix.++ Part 1 (Discriminator on main flow):+ Proves that if an uncommitted write transaction is held during embedding (the single-phase bug),+ a concurrent writer fails with SQLite 'database is locked'.++ Part 2 (Three-phase fix):+ Proves that because Phase 1 releases its lock, the concurrent writer succeeds during Phase 2+ embedding, and both transactions commit cleanly.+ """+ db_path = tmp_path / "test_rag_concurrent.db"+ engine = create_engine(+ f"sqlite:///{db_path}?timeout=0.2",+ connect_args={"check_same_thread": False},+ )+ Base.metadata.create_all(engine)+ SessionLocal = sessionmaker(bind=engine)++ body = "Content that takes time to embed..."+ body_hash = hashlib.sha256(body.encode()).hexdigest()++ # Seed database with collection and document+ with SessionLocal() as init_session:+ st = SourceType(+ id="st-conc", name="document", display_name="Document"+ )+ coll = Collection(id="coll-conc", name="Test Collection")+ doc = Document(+ id="doc-conc",+ source_type_id="st-conc",+ document_hash=body_hash,+ title="Concurrent Test Doc",+ text_content=body,+ file_size=len(body),+ file_type="txt",+ status=DocumentStatus.COMPLETED,+ )+ rag_idx = RAGIndex(+ id=1,+ collection_name="coll-conc",+ embedding_model="test-model",+ embedding_model_type=EmbeddingProvider.SENTENCE_TRANSFORMERS,+ embedding_dimension=3,+ index_path=str(tmp_path / "index"),+ index_hash="test_hash",+ chunk_size=500,+ chunk_overlap=50,+ )+ init_session.add_all([st, coll, doc, rag_idx])+ init_session.commit()++ # ------------------------------------------------------------------+ # PART 1: Honest Discriminator - Single-phase flow holds write lock+ # ------------------------------------------------------------------+ # When an active write transaction is held across embedding, a concurrent+ # writer trying to commit hits SQLite busy timeout ("database is locked").+ discriminator_failed_with_lock = False+ with SessionLocal() as lock_session:+ # Stage a write that acquires an EXCLUSIVE lock on SQLite+ lock_session.execute(+ text(+ "UPDATE documents SET word_count = 999 WHERE id = 'doc-conc'"+ )+ )++ # While holding this open transaction, attempt a concurrent write on another thread+ def failing_writer():+ nonlocal discriminator_failed_with_lock+ try:+ with SessionLocal() as writer_sess:+ writer_sess.add(+ Document(+ id="doc-should-fail",+ source_type_id="st-conc",+ document_hash="hfail",+ title="Blocked Doc",+ text_content="Blocked",+ file_size=7,+ file_type="txt",+ status=DocumentStatus.COMPLETED,+ )+ )+ writer_sess.commit()+ except Exception as e:+ # SQLite lock error observed+ if "locked" in str(e).lower():+ discriminator_failed_with_lock = True++ t_fail = threading.Thread(target=failing_writer)+ t_fail.start()+ t_fail.join()+ lock_session.rollback()++ assert discriminator_failed_with_lock, (+ "Discriminator check failed: holding a write transaction should lock SQLite"+ )++ # ------------------------------------------------------------------+ # PART 2: Three-phase decoupled flow succeeds without lock+ # ------------------------------------------------------------------+ svc = LibraryRAGService.__new__(LibraryRAGService)+ svc.username = "test_user"+ svc._db_password = "test_password"+ svc.embedding_model = "test-model"+ svc.embedding_provider = "test-provider"+ svc.chunk_size = 500+ svc.chunk_overlap = 50+ svc.text_splitter = MagicMock()+ chunk_mock = MagicMock()+ chunk_mock.page_content = body+ svc.text_splitter.split_documents.return_value = [chunk_mock]+ svc.embedding_manager = MagicMock()++ rag_rec = MagicMock(spec=RAGIndex)+ rag_rec.id = 1+ rag_rec.chunk_count = 0+ rag_rec.total_documents = 0+ svc.rag_index_record = rag_rec+ svc._get_index_hash = MagicMock(return_value="test_hash")+ svc._get_index_path = MagicMock(return_value=str(tmp_path / "index"))++ concurrent_write_succeeded = False+ concurrent_error = None++ writer_body = "concurrent write succeeded"+ writer_body_hash = hashlib.sha256(writer_body.encode()).hexdigest()++ def concurrent_writer():+ nonlocal concurrent_write_succeeded, concurrent_error+ try:+ # Open an independent session on a separate thread+ with SessionLocal() as concurrent_session:+ new_doc = Document(+ id="doc-concurrent-writer",+ source_type_id="st-conc",+ document_hash=writer_body_hash,+ title="Inserted while embedding",+ text_content=writer_body,+ file_size=len(writer_body),+ file_type="txt",+ status=DocumentStatus.COMPLETED,+ )+ concurrent_session.add(new_doc)+ concurrent_session.commit()+ concurrent_write_succeeded = True+ except Exception as e:+ concurrent_error = e++ def mock_embed_documents(texts):+ # This is Phase 2 (embedding round-trip outside DB session).+ # Start the concurrent writer thread now while embedding is running!+ t = threading.Thread(target=concurrent_writer)+ t.start()+ t.join()+ return [[0.1, 0.2, 0.3]]++ svc.embedding_manager.embeddings.embed_documents.side_effect = (+ mock_embed_documents+ )++ vindex = MagicMock()+ vindex.index_prepared.return_value = MagicMock(+ added=1, removed=0, chunks=1+ )+ svc._get_vector_index = MagicMock(return_value=vindex)++ @contextmanager+ def real_session_ctx(*args, **kwargs):+ session = SessionLocal()+ try:+ yield session+ finally:+ session.close()++ with patch(+ "local_deep_research.research_library.services.library_rag_service.get_user_db_session",+ side_effect=real_session_ctx,+ ):+ result = svc._index_document_locked("doc-conc", "coll-conc")++ assert result["status"] == "success"+ assert concurrent_write_succeeded, (+ f"Concurrent write failed with: {concurrent_error}"+ )+ assert concurrent_error is None++ # Verify both the indexed document and the concurrently inserted document exist in SQLite+ with SessionLocal() as check_session:+ doc_coll = (+ check_session.query(DocumentCollection)+ .filter_by(document_id="doc-conc", collection_id="coll-conc")+ .first()+ )+ assert doc_coll is not None+ assert doc_coll.indexed is True++ conc_doc = (+ check_session.query(Document)+ .filter_by(id="doc-concurrent-writer")+ .first()+ )+ assert conc_doc is not None+ assert conc_doc.title == "Inserted while embedding"++ def test_write_prepared_legacy_snapshot_still_ingests_orphan(+ self, mock_service+ ):+ """Legacy prepared objects without snapshots still create first-time links.++ The membership check runs unconditionally (not gated on+ ``content_hash`` being a str) and a missing ``had_collection_link``+ snapshot (None) preserves main's unconditional-create behavior+ instead of misreading absence as a concurrent unlink.+ """+ mock_session = MagicMock()+ body = "Legacy orphan without snapshots"+ current_doc = MagicMock(spec=Document)+ current_doc.text_content = body+ created_link = MagicMock(spec=DocumentCollection)+ created_link.indexed = False++ def query_side_effect(model):+ q = MagicMock()+ if model == Document:+ q.filter_by.return_value.first.return_value = current_doc+ elif model == DocumentCollection:+ q.filter_by.return_value.first.return_value = None+ elif model == RagDocumentStatus:+ q.filter_by.return_value.first.return_value = None+ elif model == RAGIndex:+ q.filter_by.return_value.first.return_value = (+ mock_service.rag_index_record+ )+ return q++ mock_session.query.side_effect = query_side_effect+ mock_service._flag_document_for_reindex = MagicMock()++ vindex = MagicMock()+ vindex.index_prepared.return_value = MagicMock(+ added=1, removed=0, chunks=1+ )+ mock_service._get_vector_index = MagicMock(return_value=vindex)++ class DummyCtx:+ def __enter__(self):+ return mock_session++ def __exit__(self, *args):+ return False++ prepared = _PreparedDocument(+ document_id="doc-legacy-orphan",+ collection_id="default-lib",+ chunk_inputs=[ChunkInput(text=body, metadata={})],+ vectors=MagicMock(),+ content_hash=None,+ initial_indexed=None,+ status_exists=None,+ had_collection_link=None,+ )++ with (+ patch(+ "local_deep_research.research_library.services.library_rag_service.get_user_db_session",+ return_value=DummyCtx(),+ ),+ patch(+ "local_deep_research.research_library.services.library_rag_service._hold_faiss_write_lock",+ ),+ patch(+ "local_deep_research.research_library.services.library_rag_service.ensure_in_collection",+ return_value=created_link,+ ) as mock_ensure,+ ):+ result = mock_service._write_prepared_document(prepared)++ assert result["status"] == "success"+ assert result["chunk_count"] == 1+ mock_ensure.assert_called_once_with(+ mock_session, "doc-legacy-orphan", "default-lib"+ )+ vindex.index_prepared.assert_called_once()+ mock_service._flag_document_for_reindex.assert_not_called()++ def test_write_prepared_aborts_when_link_vanishes_after_gate(+ self, mock_service+ ):+ """An unlink racing the gate (UPDATE matches 0 rows) aborts the commit.++ The gate query saw the link, but the membership was deleted before+ the persist writes. The zero-row UPDATE is the backstop: roll back+ the staged status merge instead of resurrecting a non-member.+ """+ import hashlib as _hashlib++ body = "Raced unlink body"+ body_hash = _hashlib.sha256(body.encode()).hexdigest()+ mock_link = MagicMock(spec=DocumentCollection)+ mock_link.indexed = False+ mock_doc = MagicMock(spec=Document)+ mock_doc.text_content = body+ mock_existing = MagicMock()+ mock_existing.chunk_count = 0++ def make_q(first_val=None):+ q = MagicMock()+ q.filter_by.return_value = q+ q.first.return_value = first_val+ return q++ q_link = make_q(first_val=mock_link)+ q_doc = make_q(first_val=mock_doc)+ q_gate_status = make_q(first_val=None)+ q_existing = make_q(first_val=None)+ q_update = make_q()+ q_update.update.return_value = 0++ mock_session = MagicMock()+ mock_session.query.side_effect = [+ q_link,+ q_doc,+ q_gate_status,+ q_existing,+ q_update,+ ]+ mock_service._flag_document_for_reindex = MagicMock()++ vindex = MagicMock()+ vindex.index_prepared.return_value = MagicMock(+ added=1, removed=0, chunks=1+ )+ mock_service._get_vector_index = MagicMock(return_value=vindex)++ class DummyCtx:+ def __enter__(self):+ return mock_session++ def __exit__(self, *args):+ return False++ prepared = _PreparedDocument(+ document_id="doc-raced",+ collection_id="coll-1",+ chunk_inputs=[ChunkInput(text=body, metadata={})],+ vectors=MagicMock(),+ content_hash=body_hash,+ initial_indexed=False,+ status_exists=False,+ had_collection_link=True,+ )++ with (+ patch(+ "local_deep_research.research_library.services.library_rag_service.get_user_db_session",+ return_value=DummyCtx(),+ ),+ patch(+ "local_deep_research.research_library.services.library_rag_service._hold_faiss_write_lock",+ ),+ ):+ result = mock_service._write_prepared_document(prepared)++ assert result["status"] == "skipped"+ assert "removed from collection" in result["message"]+ # Vectors were already staged before the backstop fired; the DB+ # commit must not follow them.+ vindex.index_prepared.assert_called_once()+ mock_session.commit.assert_not_called()+ mock_service._flag_document_for_reindex.assert_called_once_with(+ mock_session, "doc-raced", "coll-1"+ )diff --git a/tests/research_library/services/test_library_rag_service_index_failure.py b/tests/research_library/services/test_library_rag_service_index_failure.pyindex 3ab715fea6..5043fdc944 100644--- a/tests/research_library/services/test_library_rag_service_index_failure.py+++ b/tests/research_library/services/test_library_rag_service_index_failure.py@@ -88,7 +88,7 @@ def test_index_document_not_marked_indexed_when_store_persists_no_embeddings(svc.rag_index_record.index_path = "/tmp/idx-4855"svc.rag_index_record.id = 1- # ...but the vector store fails to persist it: VectorIndex.index() raises+ # ...but the vector store fails to persist it: VectorIndex.index_prepared() raises# (its store.apply / embed failure path). index_document must catch that,# roll back, and flag the document indexed=False for retry — NOT mark it# indexed=True. Because the int-id store mutates the FAISS file durably@@ -96,6 +96,9 @@ def test_index_document_not_marked_indexed_when_store_persists_no_embeddings(# (see _flag_document_for_reindex).vindex = MagicMock()vindex.index.side_effect = RuntimeError("store persisted no embeddings")+ vindex.index_prepared.side_effect = RuntimeError(+ "store persisted no embeddings"+ )svc._get_vector_index = MagicMock(return_value=vindex)result = svc.index_document("doc-1", "coll-1")