Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

fix(rag): decouple indexing into 3 phases to avoid SQLite lock during embedding (#6559) (#6563)

Sdee2dcb8d65b
diff --git a/changelog.d/6559.bugfix.md b/changelog.d/6559.bugfix.md
new file mode 100644
index 0000000000..a085467ff4
--- /dev/null
+++ b/changelog.d/6559.bugfix.md
@@ -0,0 +1,14 @@
+Decouple RAG document indexing into three phases so the SQLite write lock is
+not held across the embedding round-trip. Previously, `_index_document_locked`
+held an open database session with uncommitted staging across embedding model
+calls, starving concurrent readers and writers of the SQLite database. Indexing
+now splits into: Phase 1 (short DB session to snapshot document metadata and
+collection link, releasing locks prior to embedding), Phase 2 (text chunking
+and vector embedding executed strictly outside any DB session), and Phase 3
+(short DB transaction to persist chunk records, write vectors, and update RAG
+status). Phase 3 includes a revalidation gate verifying collection membership,
+document content hash, and indexed status before persisting to prevent
+resurrecting unlinked documents or overwriting concurrent note edits. First-time
+ingest (research orphans with no `DocumentCollection` row at prepare) still
+creates membership in Phase 3; only a link that existed at prepare and vanished
+during embedding is treated as a concurrent removal.
diff --git a/src/local_deep_research/research_library/services/library_rag_service.py b/src/local_deep_research/research_library/services/library_rag_service.py
index c929c4b4c8..325c89d0f7 100644
--- a/src/local_deep_research/research_library/services/library_rag_service.py
+++ b/src/local_deep_research/research_library/services/library_rag_service.py
@@ -71,6 +71,13 @@ class _PreparedDocument:
collection_id: str
chunk_inputs: List[ChunkInput]
vectors: np.ndarray
+ content_hash: Optional[str] = None
+ initial_indexed: Optional[bool] = None
+ status_exists: Optional[bool] = None
+ # True iff a DocumentCollection row existed at prepare time. False
+ # means first-time ingest: Phase 3 may create the link. Distinguishes
+ # that from a concurrent unlink of a row that was present then.
+ had_collection_link: Optional[bool] = None
class _LazyEmbeddings(Embeddings):
@@ -1764,6 +1771,23 @@ class LibraryRAGService:
source_doc = LangchainDocument(
page_content=document.text_content, metadata=metadata
)
+ prep_content_hash = hashlib.sha256(
+ (document.text_content or "").encode("utf-8")
+ ).hexdigest()
+ prep_initial_indexed = link.indexed if link is not None else False
+ prep_had_collection_link = link is not None
+ prep_status_exists = (
+ session.query(RagDocumentStatus)
+ .filter_by(
+ document_id=document_id,
+ collection_id=collection_id,
+ rag_index_id=self.rag_index_record.id,
+ )
+ .first()
+ is not None
+ if self.rag_index_record
+ else False
+ )
chunks = self.text_splitter.split_documents([source_doc])
chunk_inputs = [
@@ -1789,6 +1813,10 @@ class LibraryRAGService:
collection_id=collection_id,
chunk_inputs=chunk_inputs,
vectors=vectors,
+ content_hash=prep_content_hash,
+ initial_indexed=prep_initial_indexed,
+ status_exists=prep_status_exists,
+ had_collection_link=prep_had_collection_link,
),
}
@@ -1810,7 +1838,110 @@ class LibraryRAGService:
self.username, self.db_password
) as session:
try:
- ensure_in_collection(session, document_id, collection_id)
+ # Phase 3 Revalidation Gate: verify collection link, content, and indexed status.
+ # The membership check runs unconditionally so legacy
+ # prepared objects without a content_hash snapshot still
+ # get first-time vs concurrent-unlink handling instead of
+ # skipping the gate entirely.
+ link = (
+ session.query(DocumentCollection)
+ .filter_by(
+ document_id=document_id,
+ collection_id=collection_id,
+ )
+ .first()
+ )
+ if link is None:
+ if not getattr(prepared, "had_collection_link", None):
+ # Reconciler case (b): research orphans have
+ # no membership at prepare. Create it now
+ # rather than treating absence as a concurrent
+ # unlink. A missing snapshot (None) preserves
+ # main's unconditional-create behavior.
+ link = ensure_in_collection(
+ session, document_id, collection_id
+ )
+ session.flush()
+ else:
+ logger.info(
+ f"Document {document_id} was removed from collection {collection_id} "
+ "during embedding; aborting Phase 3 persist."
+ )
+ self._flag_document_for_reindex(
+ session, document_id, collection_id
+ )
+ return {
+ "status": "skipped",
+ "message": "Document removed from collection during embedding",
+ }
+
+ current_doc = (
+ session.query(Document)
+ .filter_by(id=document_id)
+ .first()
+ )
+ if not current_doc:
+ logger.info(
+ f"Document {document_id} was deleted during embedding; "
+ "aborting Phase 3 persist."
+ )
+ self._flag_document_for_reindex(
+ session, document_id, collection_id
+ )
+ return {
+ "status": "error",
+ "error": "Document deleted during indexing",
+ }
+
+ if isinstance(getattr(prepared, "content_hash", None), str):
+ current_hash = hashlib.sha256(
+ (current_doc.text_content or "").encode("utf-8")
+ ).hexdigest()
+ if current_hash != prepared.content_hash:
+ logger.info(
+ f"Document {document_id} content changed during embedding; "
+ "aborting persist and flagging for reindex."
+ )
+ self._flag_document_for_reindex(
+ session, document_id, collection_id
+ )
+ return {
+ "status": "stale",
+ "message": "Document modified during indexing, flagged for reindex",
+ }
+
+ current_status_exists = (
+ session.query(RagDocumentStatus)
+ .filter_by(
+ document_id=document_id,
+ collection_id=collection_id,
+ rag_index_id=self.rag_index_record.id,
+ )
+ .first()
+ is not None
+ if self.rag_index_record
+ else False
+ )
+ if (
+ isinstance(prepared.initial_indexed, bool)
+ and isinstance(link.indexed, bool)
+ and link.indexed != prepared.initial_indexed
+ ) or (
+ isinstance(prepared.status_exists, bool)
+ and current_status_exists != prepared.status_exists
+ ):
+ logger.info(
+ f"Document {document_id} indexing status modified during embedding; "
+ "aborting persist and flagging for reindex."
+ )
+ self._flag_document_for_reindex(
+ session, document_id, collection_id
+ )
+ return {
+ "status": "stale",
+ "message": "Document indexed state changed during indexing, flagged for reindex",
+ }
+
collection_name = f"collection_{collection_id}"
vindex = self._get_vector_index(
collection_id, collection_name, reset_stale_state=True
@@ -1844,15 +1975,41 @@ class LibraryRAGService:
indexed_at=timestamp,
)
)
- session.query(DocumentCollection).filter_by(
- document_id=document_id, collection_id=collection_id
- ).update(
- {
- "indexed": True,
- "chunk_count": len(chunks),
- "last_indexed_at": timestamp,
- }
+ updated_rows = (
+ session.query(DocumentCollection)
+ .filter_by(
+ document_id=document_id,
+ collection_id=collection_id,
+ )
+ .update(
+ {
+ "indexed": True,
+ "chunk_count": len(chunks),
+ "last_indexed_at": timestamp,
+ }
+ )
)
+ if isinstance(updated_rows, int) and updated_rows == 0:
+ # An unlink committed between the Phase-3 gate query
+ # and these persist writes (the unlink DB phase does
+ # not hold the FAISS lock). Rolling back avoids
+ # re-creating status/aggregate rows for a non-member
+ # and leaves phantom vectors to the unlink's purge.
+ logger.info(
+ f"Document {document_id} was removed from collection {collection_id} "
+ "during persist; aborting Phase 3 commit."
+ )
+ safe_rollback(
+ session,
+ "library_rag_service._write_prepared_document link vanished",
+ )
+ self._flag_document_for_reindex(
+ session, document_id, collection_id
+ )
+ return {
+ "status": "skipped",
+ "message": "Document removed from collection during embedding",
+ }
rag_index = (
session.query(RAGIndex)
.filter_by(id=self.rag_index_record.id)
@@ -1932,6 +2089,7 @@ class LibraryRAGService:
Returns:
Dict with status, chunk_count, and any errors
"""
+ # Phase 1: Read & chunk doc
with get_user_db_session(self.username, self.db_password) as session:
# Get the document
document = session.query(Document).filter_by(id=document_id).first()
@@ -1943,6 +2101,25 @@ class LibraryRAGService:
doc_collection = ensure_in_collection(
session, document_id, collection_id
)
+ # If doc_collection was newly created in this session, commit it so the link
+ # exists durably before Phase 2 without holding a write lock during embedding.
+ # No test-session import is needed here: `in session.new` is
+ # False for unittest.mock sessions (MagicMock __contains__
+ # defaults to False), so mocks safely skip the pre-commit.
+ try:
+ _is_new_link = (
+ getattr(session, "new", None) is not None
+ and doc_collection in session.new
+ )
+ except Exception:
+ _is_new_link = False
+ if _is_new_link:
+ try:
+ session.commit()
+ except Exception: # allow: silent-exception
+ logger.debug(
+ f"Could not pre-commit newly created collection link for {document_id} in {collection_id}"
+ )
# Check if already indexed for this collection
if doc_collection.indexed and not force_reindex:
@@ -2050,6 +2227,24 @@ class LibraryRAGService:
# orphaning rows or poisoning the shared session).
try:
+ phase1_content_hash = hashlib.sha256(
+ (document.text_content or "").encode("utf-8")
+ ).hexdigest()
+ phase1_doc_hash = document.document_hash
+ phase1_indexed = doc_collection.indexed
+ phase1_status_exists = (
+ session.query(RagDocumentStatus)
+ .filter_by(
+ document_id=document_id,
+ collection_id=collection_id,
+ rag_index_id=self.rag_index_record.id,
+ )
+ .first()
+ is not None
+ if self.rag_index_record
+ else False
+ )
+
# Create LangChain Document from text
doc = LangchainDocument(
page_content=document.text_content,
@@ -2130,34 +2325,179 @@ class LibraryRAGService:
for i, chunk in enumerate(chunks)
]
- # Embed + persist via the new int-id-keyed vector store —
- # this writes DocumentChunk rows (text only in the encrypted
- # DB) and the vector-store's ids/vectors (never text; see
- # the SECURITY INVARIANT in vector_stores/base.py). Passing
- # ``session`` means the facade does not commit/rollback —
- # this transaction still owns that (below).
- #
- # replace=True ALWAYS: a full-document (re)index owns all of a
- # source's chunks, so it must drop the source's prior
- # chunks/vectors before adding the new ones. force_reindex is
- # NOT a safe proxy: a note edit clears the indexed STATUS
- # (_mark_note_stale_for_reindex_in_session) but leaves the
- # pre-edit chunk rows/vectors in place, so the subsequent
- # non-forced reindex would reach here with force_reindex=False
- # AND prior chunks present — appending would leave stale/deleted
- # note text permanently searchable and duplicate every chunk.
- # For a genuinely never-indexed source, prior_ids is empty and
- # replace is a no-op, so this is always safe.
+ # Roll back uncommitted changes (such as ensure_in_collection staging
+ # and autoflush from query(Collection)) so no SQLite transaction
+ # or write lock is held during Phase 2 embedding.
+ safe_rollback(
+ session, "library_rag_service.index_document phase 1"
+ )
+
+ except Exception as e:
+ # The session is shared (thread-local) with the caller.
+ # If session.flush() or session.commit() raised, the session
+ # is in PendingRollbackError state until rolled back —
+ # leaving subsequent operations to cascade. Roll back BEFORE
+ # returning the error dict so the caller sees a clean
+ # session. (Same pattern as the #3827 fix.)
+ safe_rollback(session, "library_rag_service.index_document")
+ # Backstop for the store-durable-before-DB-commit window: a
+ # REPLACE reindex ran store.apply() (DURABLY removing this
+ # source's prior vectors) before the DB commit, so the rollback
+ # above resurrects rows whose vectors are already gone while
+ # reverting indexed→its pre-attempt True. Flag for re-index so
+ # the reconciler re-embeds and store+DB reconverge.
+ self._flag_document_for_reindex(
+ session, document_id, collection_id
+ )
+ logger.exception(
+ f"Error indexing document {document_id} for collection {collection_id}"
+ )
+ return {
+ "status": "error",
+ "error": f"Operation failed: {type(e).__name__}",
+ }
+
+ # Phase 2: Embed vectors outside any DB session
+ try:
+ texts = [c.text for c in chunk_inputs]
+ vectors = (
+ np.asarray(
+ self.embedding_manager.embeddings.embed_documents(texts),
+ dtype="float32",
+ )
+ if texts
+ else None
+ )
+ except Exception as e:
+ logger.exception(
+ f"Error embedding document {document_id} for collection {collection_id}"
+ )
+ with get_user_db_session(
+ self.username, self.db_password
+ ) as session:
+ self._flag_document_for_reindex(
+ session, document_id, collection_id
+ )
+ return {
+ "status": "error",
+ "error": f"Operation failed: {type(e).__name__}",
+ }
+
+ # Phase 3: Open short session to persist chunks, apply vectors, and commit
+ with get_user_db_session(self.username, self.db_password) as session:
+ try:
+ # Phase 3 Revalidation Gate: verify collection link, content, and indexed status
+ link = (
+ session.query(DocumentCollection)
+ .filter_by(
+ document_id=document_id,
+ collection_id=collection_id,
+ )
+ .first()
+ )
+ if link is None:
+ logger.info(
+ f"Document {document_id} was removed from collection {collection_id} "
+ "during embedding; aborting Phase 3 persist."
+ )
+ self._flag_document_for_reindex(
+ session, document_id, collection_id
+ )
+ return {
+ "status": "skipped",
+ "message": "Document removed from collection during embedding",
+ }
+
+ current_doc = (
+ session.query(Document).filter_by(id=document_id).first()
+ )
+ if not current_doc:
+ logger.info(
+ f"Document {document_id} was deleted during embedding; "
+ "aborting Phase 3 persist."
+ )
+ self._flag_document_for_reindex(
+ session, document_id, collection_id
+ )
+ return {
+ "status": "error",
+ "error": "Document deleted during indexing",
+ }
+
+ current_content_hash = hashlib.sha256(
+ (current_doc.text_content or "").encode("utf-8")
+ ).hexdigest()
+ if (
+ current_content_hash != phase1_content_hash
+ or current_doc.document_hash != phase1_doc_hash
+ ):
+ logger.info(
+ f"Document {document_id} content changed during embedding; "
+ "aborting persist and flagging for reindex."
+ )
+ self._flag_document_for_reindex(
+ session, document_id, collection_id
+ )
+ return {
+ "status": "stale",
+ "message": "Document modified during indexing, flagged for reindex",
+ }
+
+ current_status_exists = (
+ session.query(RagDocumentStatus)
+ .filter_by(
+ document_id=document_id,
+ collection_id=collection_id,
+ rag_index_id=self.rag_index_record.id,
+ )
+ .first()
+ is not None
+ if self.rag_index_record
+ else False
+ )
+ if (
+ isinstance(link.indexed, bool)
+ and isinstance(phase1_indexed, bool)
+ and link.indexed != phase1_indexed
+ ) or (
+ isinstance(current_status_exists, bool)
+ and isinstance(phase1_status_exists, bool)
+ and current_status_exists != phase1_status_exists
+ ):
+ logger.info(
+ f"Document {document_id} indexing status modified during embedding; "
+ "aborting persist and flagging for reindex."
+ )
+ self._flag_document_for_reindex(
+ session, document_id, collection_id
+ )
+ return {
+ "status": "stale",
+ "message": "Document indexed state changed during indexing, flagged for reindex",
+ }
+
vindex = self._get_vector_index(
collection_id, collection_name, reset_stale_state=True
)
- index_stats = vindex.index(
- source_type="document",
- source_id=document_id,
- chunks=chunk_inputs,
- replace=True,
- session=session,
- )
+
+ if hasattr(vindex, "index_prepared"):
+ index_stats = vindex.index_prepared(
+ source_type="document",
+ source_id=document_id,
+ chunks=chunk_inputs,
+ vectors=vectors,
+ replace=True,
+ session=session,
+ )
+ else:
+ index_stats = vindex.index(
+ source_type="document",
+ source_id=document_id,
+ chunks=chunk_inputs,
+ replace=True,
+ session=session,
+ )
+
logger.info(
f"Vector index update for document {document_id}: "
f"{index_stats.added} added, {index_stats.removed} "
@@ -2198,15 +2538,39 @@ class LibraryRAGService:
)
# Also update DocumentCollection table for backward compatibility
- session.query(DocumentCollection).filter_by(
- document_id=document_id, collection_id=collection_id
- ).update(
- {
- "indexed": True,
- "chunk_count": len(chunks),
- "last_indexed_at": timestamp,
- }
+ _serial_updated = (
+ session.query(DocumentCollection)
+ .filter_by(
+ document_id=document_id,
+ collection_id=collection_id,
+ )
+ .update(
+ {
+ "indexed": True,
+ "chunk_count": len(chunks),
+ "last_indexed_at": timestamp,
+ }
+ )
)
+ if isinstance(_serial_updated, int) and _serial_updated == 0:
+ # An unlink committed between the Phase-3 gate query and
+ # these persist writes. Roll back the staged status merge
+ # above so no non-member row/aggregate is resurrected.
+ logger.info(
+ f"Document {document_id} was removed from collection {collection_id} "
+ "during persist; aborting Phase 3 commit."
+ )
+ safe_rollback(
+ session,
+ "library_rag_service.index_document link vanished",
+ )
+ self._flag_document_for_reindex(
+ session, document_id, collection_id
+ )
+ return {
+ "status": "skipped",
+ "message": "Document removed from collection during embedding",
+ }
logger.info(
"Also updated DocumentCollection.indexed for backward compatibility"
@@ -2233,12 +2597,6 @@ class LibraryRAGService:
f"total_documents {'+1' if not was_already_indexed else '+0'}"
)
- # Replace-on-reindex DB prune: no longer needed here —
- # ``VectorIndex.index(replace=True)`` above already deletes
- # this source's prior DocumentChunk rows (and their
- # vectors) as part of the same transaction/lock when
- # ``force_reindex`` is set.
-
# Flush ORM changes to database before commit
session.flush()
logger.info(f"Flushed ORM changes for document {document_id}")
diff --git a/tests/research_library/services/test_index_documents_parallel.py b/tests/research_library/services/test_index_documents_parallel.py
index 94cb09a6a0..b23d56db2c 100644
--- a/tests/research_library/services/test_index_documents_parallel.py
+++ b/tests/research_library/services/test_index_documents_parallel.py
@@ -956,6 +956,8 @@ class TestWritePreparedDocumentReindexDelta:
"""``_write_prepared_document`` accurately updates ``rag_index.chunk_count`` on re-index."""
def test_chunk_count_delta_captures_old_chunks_before_merge(self):
+ import hashlib
+
svc = _make_service()
svc.rag_index_record = MagicMock()
svc.rag_index_record.id = 1
@@ -975,15 +977,35 @@ class TestWritePreparedDocumentReindexDelta:
q.first.return_value = first_val
return q
+ body = "delta body"
+ body_hash = hashlib.sha256(body.encode()).hexdigest()
+ mock_link = MagicMock()
+ mock_link.indexed = False
+ mock_doc = MagicMock()
+ mock_doc.text_content = body
+
session = MagicMock()
- # session.query calls:
- # 1. RagDocumentStatus (existing)
- # 2. DocumentCollection update
- # 3. RAGIndex
+ # session.query calls (Phase-3 gate runs unconditionally):
+ # 1. DocumentCollection link (gate)
+ # 2. Document existence (gate)
+ # 3. RagDocumentStatus existence (gate)
+ # 4. RagDocumentStatus (existing, for delta)
+ # 5. DocumentCollection update (TOCTOU rowcount; MagicMock -> no abort)
+ # 6. RAGIndex
+ q_link = make_q(first_val=mock_link)
+ q_doc = make_q(first_val=mock_doc)
+ q_gate_status = make_q(first_val=mock_existing)
q_existing = make_q(first_val=mock_existing)
q_doc_coll = make_q()
q_rag_index = make_q(first_val=mock_rag_index)
- session.query.side_effect = [q_existing, q_doc_coll, q_rag_index]
+ session.query.side_effect = [
+ q_link,
+ q_doc,
+ q_gate_status,
+ q_existing,
+ q_doc_coll,
+ q_rag_index,
+ ]
prepared = MagicMock()
prepared.document_id = "doc-1"
@@ -993,6 +1015,10 @@ class TestWritePreparedDocumentReindexDelta:
MagicMock(),
MagicMock(),
] # 2 new chunks (delta = 2 - 5 = -3)
+ prepared.content_hash = body_hash
+ prepared.initial_indexed = False
+ prepared.status_exists = True
+ prepared.had_collection_link = True
session_ctx = MagicMock()
session_ctx.__enter__ = MagicMock(return_value=session)
diff --git a/tests/research_library/services/test_issue_6559_rag_indexing_lock.py b/tests/research_library/services/test_issue_6559_rag_indexing_lock.py
new file mode 100644
index 0000000000..397cd7faf5
--- /dev/null
+++ b/tests/research_library/services/test_issue_6559_rag_indexing_lock.py
@@ -0,0 +1,1506 @@
+"""Exhaustive tests for Issue #6559: RAG Indexing Lock Across Embedding Round-Trip.
+
+Verifies that RAG document indexing is decoupled into 3 phases:
+1. Phase 1: Read & chunk document in a short session (closing transaction before embedding).
+2. Phase 2: Embed vectors strictly outside of any DB session / write lock.
+3. Phase 3: Open short session with revalidation gate to persist chunks, apply vectors, and commit.
+"""
+
+from contextlib import contextmanager
+import hashlib
+import threading
+from unittest.mock import MagicMock, patch
+import pytest
+from sqlalchemy import create_engine, text
+from sqlalchemy.orm import sessionmaker
+
+from local_deep_research.database.models import Base
+from local_deep_research.database.models.library import (
+ Collection,
+ Document,
+ DocumentChunk,
+ DocumentCollection,
+ DocumentStatus,
+ EmbeddingProvider,
+ RAGIndex,
+ RagDocumentStatus,
+ SourceType,
+)
+from local_deep_research.research_library.services.library_rag_service import (
+ LibraryRAGService,
+ _PreparedDocument,
+)
+from local_deep_research.vector_stores.facade import ChunkInput
+
+
+class TestRAGIndexingLockDecoupling:
+ """Test suite verifying lock decoupling across embedding round-trip."""
+
+ @pytest.fixture
+ def mock_service(self):
+ """Create a LibraryRAGService with mocked dependencies."""
+ svc = LibraryRAGService.__new__(LibraryRAGService)
+ svc.username = "test_user"
+ svc._db_password = "test_password"
+ svc.embedding_model = "test-model"
+ svc.embedding_provider = "test-provider"
+ svc.chunk_size = 500
+ svc.chunk_overlap = 50
+ svc.text_splitter = MagicMock()
+ svc.embedding_manager = MagicMock()
+
+ rag_rec = MagicMock(spec=RAGIndex)
+ rag_rec.id = 1
+ rag_rec.index_path = "/tmp/test_index"
+ rag_rec.chunk_count = 0
+ rag_rec.total_documents = 0
+ svc.rag_index_record = rag_rec
+
+ svc._get_index_hash = MagicMock(return_value="test_hash")
+ svc._get_index_path = MagicMock(return_value="/tmp/test_index_path")
+ return svc
+
+ def test_embed_documents_called_outside_db_session(self, mock_service):
+ """Verify embed_documents executes while NO database session is held."""
+ session_active = False
+ embed_executed_outside_session = False
+
+ doc = MagicMock(spec=Document)
+ doc.id = "doc-1"
+ doc.text_content = "Test content for chunking and embedding."
+ doc.title = "Test Doc"
+ doc.filename = "test.pdf"
+ doc.original_url = "http://example.com/test"
+ doc.authors = "Author"
+ doc.published_date = None
+ doc.doi = None
+ doc.arxiv_id = None
+ doc.pmid = None
+ doc.pmcid = None
+ doc.extraction_method = "pdf"
+ doc.word_count = 10
+ doc.document_hash = "doc_hash_1"
+
+ coll = MagicMock(spec=Collection)
+ coll.id = "coll-1"
+
+ doc_coll = MagicMock(spec=DocumentCollection)
+ doc_coll.indexed = False
+ doc_coll.chunk_count = 0
+
+ rag_index = MagicMock(spec=RAGIndex)
+ rag_index.id = 1
+ rag_index.chunk_count = 0
+ rag_index.total_documents = 0
+
+ # Create session mock tracking active context state
+ class TrackedSessionContext:
+ def __init__(self, session_obj):
+ self.session = session_obj
+
+ def __enter__(self):
+ nonlocal session_active
+ session_active = True
+ return self.session
+
+ def __exit__(self, exc_type, exc_val, exc_tb):
+ nonlocal session_active
+ session_active = False
+ return False
+
+ mock_session = MagicMock()
+
+ def query_side_effect(model):
+ q = MagicMock()
+ if model == Document:
+ q.filter_by.return_value.first.return_value = doc
+ elif model == Collection:
+ q.filter_by.return_value.first.return_value = coll
+ elif model == DocumentCollection:
+ q.filter_by.return_value.first.return_value = doc_coll
+ elif model == RagDocumentStatus:
+ q.filter_by.return_value.first.return_value = None
+ elif model == RAGIndex:
+ q.filter_by.return_value.first.return_value = rag_index
+ return q
+
+ mock_session.query.side_effect = query_side_effect
+
+ def mock_embed(texts):
+ nonlocal embed_executed_outside_session, session_active
+ # Assert that session is NOT active when embed_documents is called
+ assert not session_active, (
+ "embed_documents was called while DB session was ACTIVE!"
+ )
+ embed_executed_outside_session = True
+ return [[0.1, 0.2, 0.3]]
+
+ mock_service.embedding_manager.embeddings.embed_documents.side_effect = mock_embed
+
+ chunk_mock = MagicMock()
+ chunk_mock.page_content = "Test content chunk"
+ mock_service.text_splitter.split_documents.return_value = [chunk_mock]
+
+ vindex = MagicMock()
+ vindex.index_prepared.return_value = MagicMock(
+ added=1, removed=0, chunks=1
+ )
+ mock_service._get_vector_index = MagicMock(return_value=vindex)
+
+ with (
+ patch(
+ "local_deep_research.research_library.services.library_rag_service.get_user_db_session",
+ side_effect=lambda *args, **kwargs: TrackedSessionContext(
+ mock_session
+ ),
+ ),
+ patch(
+ "local_deep_research.research_library.services.library_rag_service.ensure_in_collection",
+ return_value=doc_coll,
+ ),
+ ):
+ result = mock_service._index_document_locked("doc-1", "coll-1")
+
+ assert result["status"] == "success"
+ assert result["chunk_count"] == 1
+ assert embed_executed_outside_session, (
+ "embed_documents was never executed"
+ )
+ vindex.index_prepared.assert_called_once()
+
+ def test_embedding_failure_flags_document_for_reindex_without_dirtying_main_session(
+ self, mock_service
+ ):
+ """If embedding in Phase 2 fails, document is flagged indexed=False."""
+ doc = MagicMock(spec=Document)
+ doc.id = "doc-1"
+ doc.text_content = "Valid content"
+ doc.title = "Test Doc"
+ doc.filename = "test.pdf"
+ doc.original_url = None
+ doc.authors = None
+ doc.published_date = None
+ doc.doi = None
+ doc.arxiv_id = None
+ doc.pmid = None
+ doc.pmcid = None
+ doc.extraction_method = None
+ doc.word_count = 2
+ doc.document_hash = "h1"
+
+ doc_coll = MagicMock(spec=DocumentCollection)
+ doc_coll.indexed = False
+ doc_coll.chunk_count = 0
+
+ mock_session = MagicMock()
+ mock_session.query.return_value.filter_by.return_value.first.return_value = doc
+
+ # Fail during Phase 2 embedding
+ mock_service.embedding_manager.embeddings.embed_documents.side_effect = RuntimeError(
+ "Embedding network timeout"
+ )
+ chunk_mock = MagicMock()
+ chunk_mock.page_content = "Chunk"
+ mock_service.text_splitter.split_documents.return_value = [chunk_mock]
+
+ mock_service._flag_document_for_reindex = MagicMock()
+
+ class DummyCtx:
+ def __enter__(self):
+ return mock_session
+
+ def __exit__(self, *args):
+ return False
+
+ with (
+ patch(
+ "local_deep_research.research_library.services.library_rag_service.get_user_db_session",
+ return_value=DummyCtx(),
+ ),
+ patch(
+ "local_deep_research.research_library.services.library_rag_service.ensure_in_collection",
+ return_value=doc_coll,
+ ),
+ ):
+ result = mock_service._index_document_locked("doc-1", "coll-1")
+
+ assert result["status"] == "error"
+ assert "RuntimeError" in result["error"]
+ mock_service._flag_document_for_reindex.assert_called_once_with(
+ mock_session, "doc-1", "coll-1"
+ )
+
+ def test_vindex_index_prepared_failure_triggers_rollback_and_reindex_flag(
+ self, mock_service
+ ):
+ """When vindex.index_prepared raises an error, session rolls back and document is flagged."""
+ doc = MagicMock(spec=Document)
+ doc.id = "doc-1"
+ doc.text_content = "Valid content"
+ doc.title = "Test Doc"
+ doc.filename = "test.pdf"
+ doc.original_url = None
+ doc.authors = None
+ doc.published_date = None
+ doc.doi = None
+ doc.arxiv_id = None
+ doc.pmid = None
+ doc.pmcid = None
+ doc.extraction_method = None
+ doc.word_count = 2
+ doc.document_hash = "h1"
+
+ doc_coll = MagicMock(spec=DocumentCollection)
+ doc_coll.indexed = False
+ doc_coll.chunk_count = 0
+
+ mock_session = MagicMock()
+
+ def query_side_effect(model):
+ q = MagicMock()
+ if model == Document:
+ q.filter_by.return_value.first.return_value = doc
+ elif model == DocumentCollection:
+ q.filter_by.return_value.first.return_value = doc_coll
+ elif model == RagDocumentStatus:
+ q.filter_by.return_value.first.return_value = None
+ return q
+
+ mock_session.query.side_effect = query_side_effect
+
+ chunk_mock = MagicMock()
+ chunk_mock.page_content = "Chunk"
+ mock_service.text_splitter.split_documents.return_value = [chunk_mock]
+ mock_service.embedding_manager.embeddings.embed_documents.return_value = [
+ [0.1, 0.2]
+ ]
+
+ # vindex with index_prepared raising RuntimeError
+ vindex = MagicMock()
+ vindex.index_prepared.side_effect = RuntimeError(
+ "store persisted no embeddings"
+ )
+ mock_service._get_vector_index = MagicMock(return_value=vindex)
+ mock_service._flag_document_for_reindex = MagicMock()
+
+ class DummyCtx:
+ def __enter__(self):
+ return mock_session
+
+ def __exit__(self, *args):
+ return False
+
+ with (
+ patch(
+ "local_deep_research.research_library.services.library_rag_service.get_user_db_session",
+ return_value=DummyCtx(),
+ ),
+ patch(
+ "local_deep_research.research_library.services.library_rag_service.ensure_in_collection",
+ return_value=doc_coll,
+ ),
+ ):
+ result = mock_service._index_document_locked("doc-1", "coll-1")
+
+ assert result["status"] == "error"
+ assert "RuntimeError" in result["error"]
+ mock_service._flag_document_for_reindex.assert_called_once_with(
+ mock_session, "doc-1", "coll-1"
+ )
+
+ def test_vindex_fallback_to_index_when_index_prepared_not_implemented(
+ self, mock_service
+ ):
+ """Falls back to calling vindex.index when index_prepared is absent on vindex."""
+ doc = MagicMock(spec=Document)
+ doc.id = "doc-1"
+ doc.text_content = "Valid content"
+ doc.title = "Test Doc"
+ doc.filename = "test.pdf"
+ doc.original_url = None
+ doc.authors = None
+ doc.published_date = None
+ doc.doi = None
+ doc.arxiv_id = None
+ doc.pmid = None
+ doc.pmcid = None
+ doc.extraction_method = None
+ doc.word_count = 2
+ doc.document_hash = "h1"
+
+ doc_coll = MagicMock(spec=DocumentCollection)
+ doc_coll.indexed = False
+ doc_coll.chunk_count = 0
+
+ mock_session = MagicMock()
+
+ def query_side_effect(model):
+ q = MagicMock()
+ if model == Document:
+ q.filter_by.return_value.first.return_value = doc
+ elif model == DocumentCollection:
+ q.filter_by.return_value.first.return_value = doc_coll
+ elif model == RagDocumentStatus:
+ q.filter_by.return_value.first.return_value = None
+ elif model == RAGIndex:
+ rag_rec = MagicMock(spec=RAGIndex)
+ rag_rec.chunk_count = 0
+ rag_rec.total_documents = 0
+ q.filter_by.return_value.first.return_value = rag_rec
+ return q
+
+ mock_session.query.side_effect = query_side_effect
+
+ chunk_mock = MagicMock()
+ chunk_mock.page_content = "Chunk"
+ mock_service.text_splitter.split_documents.return_value = [chunk_mock]
+ mock_service.embedding_manager.embeddings.embed_documents.return_value = [
+ [0.1, 0.2]
+ ]
+
+ # vindex with only index, no index_prepared
+ class LegacyVectorIndex:
+ def __init__(self):
+ self.index = MagicMock(
+ return_value=MagicMock(added=1, removed=0, chunks=1)
+ )
+
+ vindex = LegacyVectorIndex()
+ mock_service._get_vector_index = MagicMock(return_value=vindex)
+
+ class DummyCtx:
+ def __enter__(self):
+ return mock_session
+
+ def __exit__(self, *args):
+ return False
+
+ with (
+ patch(
+ "local_deep_research.research_library.services.library_rag_service.get_user_db_session",
+ return_value=DummyCtx(),
+ ),
+ patch(
+ "local_deep_research.research_library.services.library_rag_service.ensure_in_collection",
+ return_value=doc_coll,
+ ),
+ ):
+ result = mock_service._index_document_locked("doc-1", "coll-1")
+
+ assert result["status"] == "success"
+ vindex.index.assert_called_once()
+
+ def test_phase3_aborts_when_document_removed_from_collection(
+ self, mock_service
+ ):
+ """Phase 3 revalidation gate: if document was unlinked during embedding, abort without writing."""
+ doc = MagicMock(spec=Document)
+ doc.id = "doc-1"
+ doc.text_content = "Valid content"
+ doc.title = "Test Doc"
+ doc.filename = "test.pdf"
+ doc.original_url = None
+ doc.authors = None
+ doc.published_date = None
+ doc.doi = None
+ doc.arxiv_id = None
+ doc.pmid = None
+ doc.pmcid = None
+ doc.extraction_method = None
+ doc.word_count = 2
+ doc.document_hash = "h1"
+
+ doc_coll = MagicMock(spec=DocumentCollection)
+ doc_coll.indexed = False
+ doc_coll.chunk_count = 0
+
+ # Phase 1 sees link; Phase 3 sees link deleted (None)
+ p1_session = MagicMock()
+ p3_session = MagicMock()
+
+ def p1_query(model):
+ q = MagicMock()
+ if model == Document:
+ q.filter_by.return_value.first.return_value = doc
+ elif model == DocumentCollection:
+ q.filter_by.return_value.first.return_value = doc_coll
+ elif model == RagDocumentStatus:
+ q.filter_by.return_value.first.return_value = None
+ return q
+
+ p1_session.query.side_effect = p1_query
+
+ def p3_query(model):
+ q = MagicMock()
+ if model == DocumentCollection:
+ # Document was unlinked from collection during Phase 2 embedding!
+ q.filter_by.return_value.first.return_value = None
+ elif model == Document:
+ q.filter_by.return_value.first.return_value = doc
+ return q
+
+ p3_session.query.side_effect = p3_query
+
+ chunk_mock = MagicMock()
+ chunk_mock.page_content = "Chunk"
+ mock_service.text_splitter.split_documents.return_value = [chunk_mock]
+ mock_service.embedding_manager.embeddings.embed_documents.return_value = [
+ [0.1, 0.2]
+ ]
+
+ vindex = MagicMock()
+ mock_service._get_vector_index = MagicMock(return_value=vindex)
+ mock_service._flag_document_for_reindex = MagicMock()
+
+ session_sequence = [p1_session, p3_session]
+
+ class SequenceCtx:
+ def __enter__(self):
+ return session_sequence.pop(0)
+
+ def __exit__(self, *args):
+ return False
+
+ with (
+ patch(
+ "local_deep_research.research_library.services.library_rag_service.get_user_db_session",
+ side_effect=lambda *a, **kw: SequenceCtx(),
+ ),
+ patch(
+ "local_deep_research.research_library.services.library_rag_service.ensure_in_collection",
+ return_value=doc_coll,
+ ) as mock_ensure,
+ ):
+ result = mock_service._index_document_locked("doc-1", "coll-1")
+
+ # Must abort with skipped
+ assert result["status"] == "skipped"
+ assert "removed from collection" in result["message"]
+ # Must flag for reindex
+ mock_service._flag_document_for_reindex.assert_called_once_with(
+ p3_session, "doc-1", "coll-1"
+ )
+ # Vector index must NOT be written
+ vindex.index_prepared.assert_not_called()
+ # ensure_in_collection must NOT be called in Phase 3 (called only once in Phase 1)
+ assert mock_ensure.call_count == 1
+
+ def test_phase3_aborts_when_note_edited_during_embedding(
+ self, mock_service
+ ):
+ """Phase 3 revalidation gate: if note content changed during embedding, abort to avoid stomping."""
+ p1_doc = MagicMock(spec=Document)
+ p1_doc.id = "doc-1"
+ p1_doc.text_content = "Pre-edit note content"
+ p1_doc.document_hash = "pre_edit_hash"
+ p1_doc.title = "Note"
+ p1_doc.filename = "note.txt"
+ p1_doc.original_url = None
+ p1_doc.authors = None
+ p1_doc.published_date = None
+ p1_doc.doi = None
+ p1_doc.arxiv_id = None
+ p1_doc.pmid = None
+ p1_doc.pmcid = None
+ p1_doc.extraction_method = None
+ p1_doc.word_count = 4
+
+ # In Phase 3, document has new edited text
+ p3_doc = MagicMock(spec=Document)
+ p3_doc.id = "doc-1"
+ p3_doc.text_content = "Post-edit updated note content"
+ p3_doc.document_hash = "post_edit_hash"
+
+ doc_coll = MagicMock(spec=DocumentCollection)
+ doc_coll.indexed = False
+ doc_coll.chunk_count = 0
+
+ p1_session = MagicMock()
+ p3_session = MagicMock()
+
+ def p1_query(model):
+ q = MagicMock()
+ if model == Document:
+ q.filter_by.return_value.first.return_value = p1_doc
+ elif model == DocumentCollection:
+ q.filter_by.return_value.first.return_value = doc_coll
+ elif model == RagDocumentStatus:
+ q.filter_by.return_value.first.return_value = None
+ return q
+
+ p1_session.query.side_effect = p1_query
+
+ def p3_query(model):
+ q = MagicMock()
+ if model == Document:
+ q.filter_by.return_value.first.return_value = p3_doc
+ elif model == DocumentCollection:
+ q.filter_by.return_value.first.return_value = doc_coll
+ elif model == RagDocumentStatus:
+ q.filter_by.return_value.first.return_value = None
+ return q
+
+ p3_session.query.side_effect = p3_query
+
+ chunk_mock = MagicMock()
+ chunk_mock.page_content = "Pre-edit note content"
+ mock_service.text_splitter.split_documents.return_value = [chunk_mock]
+ mock_service.embedding_manager.embeddings.embed_documents.return_value = [
+ [0.1, 0.2]
+ ]
+
+ vindex = MagicMock()
+ mock_service._get_vector_index = MagicMock(return_value=vindex)
+ mock_service._flag_document_for_reindex = MagicMock()
+
+ session_sequence = [p1_session, p3_session]
+
+ class SequenceCtx:
+ def __enter__(self):
+ return session_sequence.pop(0)
+
+ def __exit__(self, *args):
+ return False
+
+ with (
+ patch(
+ "local_deep_research.research_library.services.library_rag_service.get_user_db_session",
+ side_effect=lambda *a, **kw: SequenceCtx(),
+ ),
+ patch(
+ "local_deep_research.research_library.services.library_rag_service.ensure_in_collection",
+ return_value=doc_coll,
+ ),
+ ):
+ result = mock_service._index_document_locked("doc-1", "coll-1")
+
+ # Aborts with stale status
+ assert result["status"] == "stale"
+ assert "modified during indexing" in result["message"]
+ mock_service._flag_document_for_reindex.assert_called_once_with(
+ p3_session, "doc-1", "coll-1"
+ )
+ # Pre-edit chunks must NOT be written to vector index
+ vindex.index_prepared.assert_not_called()
+
+ def test_phase3_aborts_when_document_deleted(self, mock_service):
+ """Phase 3 revalidation gate: if document row was deleted during embedding, abort."""
+ doc = MagicMock(spec=Document)
+ doc.id = "doc-1"
+ doc.text_content = "Some content"
+ doc.document_hash = "h1"
+ doc.title = "Doc"
+ doc.filename = "doc.txt"
+ doc.original_url = None
+ doc.authors = None
+ doc.published_date = None
+ doc.doi = None
+ doc.arxiv_id = None
+ doc.pmid = None
+ doc.pmcid = None
+ doc.extraction_method = None
+ doc.word_count = 2
+
+ doc_coll = MagicMock(spec=DocumentCollection)
+ doc_coll.indexed = False
+
+ p1_session = MagicMock()
+ p3_session = MagicMock()
+
+ p1_session.query.return_value.filter_by.return_value.first.return_value = doc
+
+ def p3_query(model):
+ q = MagicMock()
+ if model == DocumentCollection:
+ q.filter_by.return_value.first.return_value = doc_coll
+ elif model == Document:
+ # Document was deleted!
+ q.filter_by.return_value.first.return_value = None
+ return q
+
+ p3_session.query.side_effect = p3_query
+
+ chunk_mock = MagicMock()
+ chunk_mock.page_content = "Chunk"
+ mock_service.text_splitter.split_documents.return_value = [chunk_mock]
+ mock_service.embedding_manager.embeddings.embed_documents.return_value = [
+ [0.1]
+ ]
+
+ vindex = MagicMock()
+ mock_service._get_vector_index = MagicMock(return_value=vindex)
+ mock_service._flag_document_for_reindex = MagicMock()
+
+ session_sequence = [p1_session, p3_session]
+
+ class SequenceCtx:
+ def __enter__(self):
+ return session_sequence.pop(0)
+
+ def __exit__(self, *args):
+ return False
+
+ with (
+ patch(
+ "local_deep_research.research_library.services.library_rag_service.get_user_db_session",
+ side_effect=lambda *a, **kw: SequenceCtx(),
+ ),
+ patch(
+ "local_deep_research.research_library.services.library_rag_service.ensure_in_collection",
+ return_value=doc_coll,
+ ),
+ ):
+ result = mock_service._index_document_locked("doc-1", "coll-1")
+
+ assert result["status"] == "error"
+ assert "deleted" in result["error"].lower()
+ mock_service._flag_document_for_reindex.assert_called_once_with(
+ p3_session, "doc-1", "coll-1"
+ )
+ vindex.index_prepared.assert_not_called()
+
+ def test_write_prepared_document_revalidation_gate(self, mock_service):
+ """_write_prepared_document enforces revalidation gate before persisting."""
+ mock_session = MagicMock()
+ current_doc = MagicMock(spec=Document)
+ current_doc.text_content = "Post-edit content"
+ link = MagicMock(spec=DocumentCollection)
+ link.indexed = False
+
+ def query_side_effect(model):
+ q = MagicMock()
+ if model == Document:
+ q.filter_by.return_value.first.return_value = current_doc
+ elif model == DocumentCollection:
+ q.filter_by.return_value.first.return_value = link
+ elif model == RagDocumentStatus:
+ q.filter_by.return_value.first.return_value = None
+ return q
+
+ mock_session.query.side_effect = query_side_effect
+ mock_service._flag_document_for_reindex = MagicMock()
+
+ class DummyCtx:
+ def __enter__(self):
+ return mock_session
+
+ def __exit__(self, *args):
+ return False
+
+ prepared = _PreparedDocument(
+ document_id="doc-1",
+ collection_id="coll-1",
+ chunk_inputs=[ChunkInput(text="Pre-edit content", metadata={})],
+ vectors=MagicMock(),
+ content_hash=hashlib.sha256(b"Pre-edit content").hexdigest(),
+ initial_indexed=False,
+ status_exists=False,
+ )
+
+ with (
+ patch(
+ "local_deep_research.research_library.services.library_rag_service.get_user_db_session",
+ return_value=DummyCtx(),
+ ),
+ patch(
+ "local_deep_research.research_library.services.library_rag_service._hold_faiss_write_lock",
+ ),
+ ):
+ result = mock_service._write_prepared_document(prepared)
+
+ assert result["status"] == "stale"
+ assert "modified during indexing" in result["message"]
+ mock_service._flag_document_for_reindex.assert_called_once_with(
+ mock_session, "doc-1", "coll-1"
+ )
+
+ def test_write_prepared_initially_unlinked_creates_membership(
+ self, mock_service
+ ):
+ """First-time ingest: no DocumentCollection at prepare is not a concurrent unlink.
+
+ Reconciler case (b) research orphans have no membership when
+ ``_prepare_document`` runs. Phase 3 must create the link and persist
+ vectors instead of returning skipped.
+ """
+ mock_session = MagicMock()
+ current_doc = MagicMock(spec=Document)
+ current_doc.text_content = "Orphan research download"
+ created_link = MagicMock(spec=DocumentCollection)
+ created_link.indexed = False
+
+ def query_side_effect(model):
+ q = MagicMock()
+ if model == Document:
+ q.filter_by.return_value.first.return_value = current_doc
+ elif model == DocumentCollection:
+ q.filter_by.return_value.first.return_value = None
+ elif model == RagDocumentStatus:
+ q.filter_by.return_value.first.return_value = None
+ elif model == RAGIndex:
+ q.filter_by.return_value.first.return_value = (
+ mock_service.rag_index_record
+ )
+ return q
+
+ mock_session.query.side_effect = query_side_effect
+ mock_service._flag_document_for_reindex = MagicMock()
+
+ vindex = MagicMock()
+ vindex.index_prepared.return_value = MagicMock(
+ added=1, removed=0, chunks=1
+ )
+ mock_service._get_vector_index = MagicMock(return_value=vindex)
+
+ class DummyCtx:
+ def __enter__(self):
+ return mock_session
+
+ def __exit__(self, *args):
+ return False
+
+ prepared = _PreparedDocument(
+ document_id="doc-orphan",
+ collection_id="default-lib",
+ chunk_inputs=[
+ ChunkInput(text="Orphan research download", metadata={})
+ ],
+ vectors=MagicMock(),
+ content_hash=hashlib.sha256(
+ b"Orphan research download"
+ ).hexdigest(),
+ initial_indexed=False,
+ status_exists=False,
+ had_collection_link=False,
+ )
+
+ with (
+ patch(
+ "local_deep_research.research_library.services.library_rag_service.get_user_db_session",
+ return_value=DummyCtx(),
+ ),
+ patch(
+ "local_deep_research.research_library.services.library_rag_service._hold_faiss_write_lock",
+ ),
+ patch(
+ "local_deep_research.research_library.services.library_rag_service.ensure_in_collection",
+ return_value=created_link,
+ ) as mock_ensure,
+ ):
+ result = mock_service._write_prepared_document(prepared)
+
+ assert result["status"] == "success"
+ assert result["chunk_count"] == 1
+ mock_ensure.assert_called_once_with(
+ mock_session, "doc-orphan", "default-lib"
+ )
+ mock_session.flush.assert_called()
+ vindex.index_prepared.assert_called_once()
+ mock_service._flag_document_for_reindex.assert_not_called()
+
+ def test_write_prepared_concurrent_unlink_still_skips(self, mock_service):
+ """A link present at prepare and gone at write is still a concurrent removal."""
+ mock_session = MagicMock()
+ current_doc = MagicMock(spec=Document)
+ current_doc.text_content = "Linked note"
+
+ def query_side_effect(model):
+ q = MagicMock()
+ if model == Document:
+ q.filter_by.return_value.first.return_value = current_doc
+ elif model == DocumentCollection:
+ q.filter_by.return_value.first.return_value = None
+ elif model == RagDocumentStatus:
+ q.filter_by.return_value.first.return_value = None
+ return q
+
+ mock_session.query.side_effect = query_side_effect
+ mock_service._flag_document_for_reindex = MagicMock()
+ vindex = MagicMock()
+ mock_service._get_vector_index = MagicMock(return_value=vindex)
+
+ class DummyCtx:
+ def __enter__(self):
+ return mock_session
+
+ def __exit__(self, *args):
+ return False
+
+ prepared = _PreparedDocument(
+ document_id="doc-1",
+ collection_id="coll-1",
+ chunk_inputs=[ChunkInput(text="Linked note", metadata={})],
+ vectors=MagicMock(),
+ content_hash=hashlib.sha256(b"Linked note").hexdigest(),
+ initial_indexed=False,
+ status_exists=False,
+ had_collection_link=True,
+ )
+
+ with (
+ patch(
+ "local_deep_research.research_library.services.library_rag_service.get_user_db_session",
+ return_value=DummyCtx(),
+ ),
+ patch(
+ "local_deep_research.research_library.services.library_rag_service._hold_faiss_write_lock",
+ ),
+ patch(
+ "local_deep_research.research_library.services.library_rag_service.ensure_in_collection",
+ ) as mock_ensure,
+ ):
+ result = mock_service._write_prepared_document(prepared)
+
+ assert result["status"] == "skipped"
+ assert "removed from collection" in result["message"]
+ mock_ensure.assert_not_called()
+ vindex.index_prepared.assert_not_called()
+ mock_service._flag_document_for_reindex.assert_called_once_with(
+ mock_session, "doc-1", "coll-1"
+ )
+
+ def test_prepare_snapshots_missing_collection_link(self, mock_service):
+ """``_prepare_document`` records that no DocumentCollection existed."""
+ doc = MagicMock(spec=Document)
+ doc.id = "doc-orphan"
+ doc.text_content = "Orphan body"
+ doc.title = "Orphan"
+ doc.filename = "orphan.txt"
+ doc.original_url = None
+ doc.authors = None
+ doc.published_date = None
+ doc.doi = None
+ doc.arxiv_id = None
+ doc.pmid = None
+ doc.pmcid = None
+ doc.extraction_method = None
+ doc.word_count = 2
+
+ mock_session = MagicMock()
+
+ def query_side_effect(model):
+ q = MagicMock()
+ if model == Document:
+ q.filter_by.return_value.first.return_value = doc
+ elif model == DocumentCollection:
+ q.filter_by.return_value.first.return_value = None
+ elif model == RagDocumentStatus:
+ q.filter_by.return_value.first.return_value = None
+ return q
+
+ mock_session.query.side_effect = query_side_effect
+ chunk_mock = MagicMock()
+ chunk_mock.page_content = "Orphan body"
+ mock_service.text_splitter.split_documents.return_value = [chunk_mock]
+ mock_service.embedding_manager.embeddings.embed_documents.return_value = [
+ [0.1, 0.2]
+ ]
+
+ class DummyCtx:
+ def __enter__(self):
+ return mock_session
+
+ def __exit__(self, *args):
+ return False
+
+ with patch(
+ "local_deep_research.research_library.services.library_rag_service.get_user_db_session",
+ return_value=DummyCtx(),
+ ):
+ result = mock_service._prepare_document(
+ "doc-orphan", "default-lib", force_reindex=False
+ )
+
+ assert result["status"] == "prepared"
+ prepared = result["prepared"]
+ assert prepared.had_collection_link is False
+ assert prepared.initial_indexed is False
+
+ def test_prepared_pipeline_indexes_research_orphan_with_real_sqlite(
+ self, tmp_path
+ ):
+ """Reconciler case (b): unlinked document becomes searchable via prepared path.
+
+ Mirrors the background sweep: Document exists, no DocumentCollection,
+ ``_prepare_document`` then ``_write_prepared_document`` with real SQLite
+ and stubbed embedding/vector collaborators.
+ """
+ db_path = tmp_path / "test_rag_orphan.db"
+ engine = create_engine(
+ f"sqlite:///{db_path}",
+ connect_args={"check_same_thread": False},
+ )
+ Base.metadata.create_all(engine)
+ SessionLocal = sessionmaker(bind=engine)
+
+ body = "Research orphan text for first-time ingest"
+ body_hash = hashlib.sha256(body.encode()).hexdigest()
+
+ with SessionLocal() as init_session:
+ st = SourceType(
+ id="st-orphan", name="document", display_name="Document"
+ )
+ coll = Collection(
+ id="default-lib",
+ name="Library",
+ collection_type="default_library",
+ is_default=True,
+ )
+ doc = Document(
+ id="doc-orphan",
+ source_type_id="st-orphan",
+ document_hash=body_hash,
+ title="Orphan Paper",
+ text_content=body,
+ file_size=len(body),
+ file_type="txt",
+ status=DocumentStatus.COMPLETED,
+ )
+ rag_idx = RAGIndex(
+ id=1,
+ collection_name="default-lib",
+ embedding_model="test-model",
+ embedding_model_type=EmbeddingProvider.SENTENCE_TRANSFORMERS,
+ embedding_dimension=3,
+ index_path=str(tmp_path / "index"),
+ index_hash="test_hash",
+ chunk_size=500,
+ chunk_overlap=50,
+ )
+ init_session.add_all([st, coll, doc, rag_idx])
+ init_session.commit()
+
+ svc = LibraryRAGService.__new__(LibraryRAGService)
+ svc.username = "test_user"
+ svc._db_password = "test_password"
+ svc.embedding_model = "test-model"
+ svc.embedding_provider = "test-provider"
+ svc.chunk_size = 500
+ svc.chunk_overlap = 50
+ svc.text_splitter = MagicMock()
+ chunk_mock = MagicMock()
+ chunk_mock.page_content = body
+ svc.text_splitter.split_documents.return_value = [chunk_mock]
+ svc.embedding_manager = MagicMock()
+ svc.embedding_manager.embeddings.embed_documents.return_value = [
+ [0.1, 0.2, 0.3]
+ ]
+ rag_rec = MagicMock(spec=RAGIndex)
+ rag_rec.id = 1
+ rag_rec.chunk_count = 0
+ rag_rec.total_documents = 0
+ svc.rag_index_record = rag_rec
+ svc._get_index_hash = MagicMock(return_value="test_hash")
+ svc._get_index_path = MagicMock(return_value=str(tmp_path / "index"))
+
+ vindex = MagicMock()
+ vindex.index_prepared.return_value = MagicMock(
+ added=1, removed=0, chunks=1
+ )
+ svc._get_vector_index = MagicMock(return_value=vindex)
+
+ @contextmanager
+ def real_session_ctx(*args, **kwargs):
+ session = SessionLocal()
+ try:
+ yield session
+ finally:
+ session.close()
+
+ with patch(
+ "local_deep_research.research_library.services.library_rag_service.get_user_db_session",
+ side_effect=real_session_ctx,
+ ):
+ prepared_result = svc._prepare_document(
+ "doc-orphan", "default-lib", force_reindex=False
+ )
+ assert prepared_result["status"] == "prepared"
+ prepared = prepared_result["prepared"]
+ assert prepared.had_collection_link is False
+
+ result = svc._write_prepared_document(prepared)
+
+ assert result["status"] == "success"
+ assert result["chunk_count"] == 1
+ vindex.index_prepared.assert_called_once()
+
+ with SessionLocal() as check_session:
+ membership = (
+ check_session.query(DocumentCollection)
+ .filter_by(
+ document_id="doc-orphan", collection_id="default-lib"
+ )
+ .first()
+ )
+ assert membership is not None
+ assert membership.indexed is True
+ assert membership.chunk_count == 1
+
+ def test_already_indexed_document_skips_without_embedding(
+ self, mock_service
+ ):
+ """If document is already indexed and force_reindex=False, return skipped without embedding."""
+ doc = MagicMock(spec=Document)
+ doc.id = "doc-1"
+ doc.text_content = "Some text"
+
+ doc_coll = MagicMock(spec=DocumentCollection)
+ doc_coll.indexed = True
+ doc_coll.chunk_count = 5
+
+ mock_session = MagicMock()
+ mock_session.query.return_value.filter_by.return_value.first.return_value = doc
+
+ class DummyCtx:
+ def __enter__(self):
+ return mock_session
+
+ def __exit__(self, *args):
+ return False
+
+ with (
+ patch(
+ "local_deep_research.research_library.services.library_rag_service.get_user_db_session",
+ return_value=DummyCtx(),
+ ),
+ patch(
+ "local_deep_research.research_library.services.library_rag_service.ensure_in_collection",
+ return_value=doc_coll,
+ ),
+ ):
+ result = mock_service._index_document_locked(
+ "doc-1", "coll-1", force_reindex=False
+ )
+
+ assert result["status"] == "skipped"
+ assert result["chunk_count"] == 5
+ mock_service.embedding_manager.embeddings.embed_documents.assert_not_called()
+
+ def test_empty_text_with_prior_chunks_purges_without_embedding(
+ self, mock_service
+ ):
+ """Document with empty text and prior chunks purges chunks/vectors without embedding."""
+ doc = MagicMock(spec=Document)
+ doc.id = "doc-1"
+ doc.text_content = ""
+
+ doc_coll = MagicMock(spec=DocumentCollection)
+ doc_coll.indexed = False
+
+ mock_session = MagicMock()
+ # 1st .first() = Document lookup, 2nd .first() = DocumentChunk prior lookup
+ mock_session.query.return_value.filter_by.return_value.first.side_effect = [
+ doc,
+ MagicMock(spec=DocumentChunk), # Prior chunk exists
+ ]
+
+ vindex = MagicMock()
+ mock_service._get_vector_index = MagicMock(return_value=vindex)
+
+ class DummyCtx:
+ def __enter__(self):
+ return mock_session
+
+ def __exit__(self, *args):
+ return False
+
+ with (
+ patch(
+ "local_deep_research.research_library.services.library_rag_service.get_user_db_session",
+ return_value=DummyCtx(),
+ ),
+ patch(
+ "local_deep_research.research_library.services.library_rag_service.ensure_in_collection",
+ return_value=doc_coll,
+ ),
+ ):
+ result = mock_service._index_document_locked("doc-1", "coll-1")
+
+ assert result["status"] == "cleared"
+ assert result["chunk_count"] == 0
+ mock_service.embedding_manager.embeddings.embed_documents.assert_not_called()
+ vindex.index.assert_called_once_with(
+ source_type="document",
+ source_id="doc-1",
+ chunks=[],
+ replace=True,
+ session=mock_session,
+ )
+ assert doc_coll.indexed is False
+ mock_session.commit.assert_called_once()
+
+ def test_concurrent_db_write_during_embedding_phase_with_real_sqlite(
+ self, tmp_path
+ ):
+ """Exhaustive test with real SQLite: honest discriminator demonstrating lock fix.
+
+ Part 1 (Discriminator on main flow):
+ Proves that if an uncommitted write transaction is held during embedding (the single-phase bug),
+ a concurrent writer fails with SQLite 'database is locked'.
+
+ Part 2 (Three-phase fix):
+ Proves that because Phase 1 releases its lock, the concurrent writer succeeds during Phase 2
+ embedding, and both transactions commit cleanly.
+ """
+ db_path = tmp_path / "test_rag_concurrent.db"
+ engine = create_engine(
+ f"sqlite:///{db_path}?timeout=0.2",
+ connect_args={"check_same_thread": False},
+ )
+ Base.metadata.create_all(engine)
+ SessionLocal = sessionmaker(bind=engine)
+
+ body = "Content that takes time to embed..."
+ body_hash = hashlib.sha256(body.encode()).hexdigest()
+
+ # Seed database with collection and document
+ with SessionLocal() as init_session:
+ st = SourceType(
+ id="st-conc", name="document", display_name="Document"
+ )
+ coll = Collection(id="coll-conc", name="Test Collection")
+ doc = Document(
+ id="doc-conc",
+ source_type_id="st-conc",
+ document_hash=body_hash,
+ title="Concurrent Test Doc",
+ text_content=body,
+ file_size=len(body),
+ file_type="txt",
+ status=DocumentStatus.COMPLETED,
+ )
+ rag_idx = RAGIndex(
+ id=1,
+ collection_name="coll-conc",
+ embedding_model="test-model",
+ embedding_model_type=EmbeddingProvider.SENTENCE_TRANSFORMERS,
+ embedding_dimension=3,
+ index_path=str(tmp_path / "index"),
+ index_hash="test_hash",
+ chunk_size=500,
+ chunk_overlap=50,
+ )
+ init_session.add_all([st, coll, doc, rag_idx])
+ init_session.commit()
+
+ # ------------------------------------------------------------------
+ # PART 1: Honest Discriminator - Single-phase flow holds write lock
+ # ------------------------------------------------------------------
+ # When an active write transaction is held across embedding, a concurrent
+ # writer trying to commit hits SQLite busy timeout ("database is locked").
+ discriminator_failed_with_lock = False
+ with SessionLocal() as lock_session:
+ # Stage a write that acquires an EXCLUSIVE lock on SQLite
+ lock_session.execute(
+ text(
+ "UPDATE documents SET word_count = 999 WHERE id = 'doc-conc'"
+ )
+ )
+
+ # While holding this open transaction, attempt a concurrent write on another thread
+ def failing_writer():
+ nonlocal discriminator_failed_with_lock
+ try:
+ with SessionLocal() as writer_sess:
+ writer_sess.add(
+ Document(
+ id="doc-should-fail",
+ source_type_id="st-conc",
+ document_hash="hfail",
+ title="Blocked Doc",
+ text_content="Blocked",
+ file_size=7,
+ file_type="txt",
+ status=DocumentStatus.COMPLETED,
+ )
+ )
+ writer_sess.commit()
+ except Exception as e:
+ # SQLite lock error observed
+ if "locked" in str(e).lower():
+ discriminator_failed_with_lock = True
+
+ t_fail = threading.Thread(target=failing_writer)
+ t_fail.start()
+ t_fail.join()
+ lock_session.rollback()
+
+ assert discriminator_failed_with_lock, (
+ "Discriminator check failed: holding a write transaction should lock SQLite"
+ )
+
+ # ------------------------------------------------------------------
+ # PART 2: Three-phase decoupled flow succeeds without lock
+ # ------------------------------------------------------------------
+ svc = LibraryRAGService.__new__(LibraryRAGService)
+ svc.username = "test_user"
+ svc._db_password = "test_password"
+ svc.embedding_model = "test-model"
+ svc.embedding_provider = "test-provider"
+ svc.chunk_size = 500
+ svc.chunk_overlap = 50
+ svc.text_splitter = MagicMock()
+ chunk_mock = MagicMock()
+ chunk_mock.page_content = body
+ svc.text_splitter.split_documents.return_value = [chunk_mock]
+ svc.embedding_manager = MagicMock()
+
+ rag_rec = MagicMock(spec=RAGIndex)
+ rag_rec.id = 1
+ rag_rec.chunk_count = 0
+ rag_rec.total_documents = 0
+ svc.rag_index_record = rag_rec
+ svc._get_index_hash = MagicMock(return_value="test_hash")
+ svc._get_index_path = MagicMock(return_value=str(tmp_path / "index"))
+
+ concurrent_write_succeeded = False
+ concurrent_error = None
+
+ writer_body = "concurrent write succeeded"
+ writer_body_hash = hashlib.sha256(writer_body.encode()).hexdigest()
+
+ def concurrent_writer():
+ nonlocal concurrent_write_succeeded, concurrent_error
+ try:
+ # Open an independent session on a separate thread
+ with SessionLocal() as concurrent_session:
+ new_doc = Document(
+ id="doc-concurrent-writer",
+ source_type_id="st-conc",
+ document_hash=writer_body_hash,
+ title="Inserted while embedding",
+ text_content=writer_body,
+ file_size=len(writer_body),
+ file_type="txt",
+ status=DocumentStatus.COMPLETED,
+ )
+ concurrent_session.add(new_doc)
+ concurrent_session.commit()
+ concurrent_write_succeeded = True
+ except Exception as e:
+ concurrent_error = e
+
+ def mock_embed_documents(texts):
+ # This is Phase 2 (embedding round-trip outside DB session).
+ # Start the concurrent writer thread now while embedding is running!
+ t = threading.Thread(target=concurrent_writer)
+ t.start()
+ t.join()
+ return [[0.1, 0.2, 0.3]]
+
+ svc.embedding_manager.embeddings.embed_documents.side_effect = (
+ mock_embed_documents
+ )
+
+ vindex = MagicMock()
+ vindex.index_prepared.return_value = MagicMock(
+ added=1, removed=0, chunks=1
+ )
+ svc._get_vector_index = MagicMock(return_value=vindex)
+
+ @contextmanager
+ def real_session_ctx(*args, **kwargs):
+ session = SessionLocal()
+ try:
+ yield session
+ finally:
+ session.close()
+
+ with patch(
+ "local_deep_research.research_library.services.library_rag_service.get_user_db_session",
+ side_effect=real_session_ctx,
+ ):
+ result = svc._index_document_locked("doc-conc", "coll-conc")
+
+ assert result["status"] == "success"
+ assert concurrent_write_succeeded, (
+ f"Concurrent write failed with: {concurrent_error}"
+ )
+ assert concurrent_error is None
+
+ # Verify both the indexed document and the concurrently inserted document exist in SQLite
+ with SessionLocal() as check_session:
+ doc_coll = (
+ check_session.query(DocumentCollection)
+ .filter_by(document_id="doc-conc", collection_id="coll-conc")
+ .first()
+ )
+ assert doc_coll is not None
+ assert doc_coll.indexed is True
+
+ conc_doc = (
+ check_session.query(Document)
+ .filter_by(id="doc-concurrent-writer")
+ .first()
+ )
+ assert conc_doc is not None
+ assert conc_doc.title == "Inserted while embedding"
+
+ def test_write_prepared_legacy_snapshot_still_ingests_orphan(
+ self, mock_service
+ ):
+ """Legacy prepared objects without snapshots still create first-time links.
+
+ The membership check runs unconditionally (not gated on
+ ``content_hash`` being a str) and a missing ``had_collection_link``
+ snapshot (None) preserves main's unconditional-create behavior
+ instead of misreading absence as a concurrent unlink.
+ """
+ mock_session = MagicMock()
+ body = "Legacy orphan without snapshots"
+ current_doc = MagicMock(spec=Document)
+ current_doc.text_content = body
+ created_link = MagicMock(spec=DocumentCollection)
+ created_link.indexed = False
+
+ def query_side_effect(model):
+ q = MagicMock()
+ if model == Document:
+ q.filter_by.return_value.first.return_value = current_doc
+ elif model == DocumentCollection:
+ q.filter_by.return_value.first.return_value = None
+ elif model == RagDocumentStatus:
+ q.filter_by.return_value.first.return_value = None
+ elif model == RAGIndex:
+ q.filter_by.return_value.first.return_value = (
+ mock_service.rag_index_record
+ )
+ return q
+
+ mock_session.query.side_effect = query_side_effect
+ mock_service._flag_document_for_reindex = MagicMock()
+
+ vindex = MagicMock()
+ vindex.index_prepared.return_value = MagicMock(
+ added=1, removed=0, chunks=1
+ )
+ mock_service._get_vector_index = MagicMock(return_value=vindex)
+
+ class DummyCtx:
+ def __enter__(self):
+ return mock_session
+
+ def __exit__(self, *args):
+ return False
+
+ prepared = _PreparedDocument(
+ document_id="doc-legacy-orphan",
+ collection_id="default-lib",
+ chunk_inputs=[ChunkInput(text=body, metadata={})],
+ vectors=MagicMock(),
+ content_hash=None,
+ initial_indexed=None,
+ status_exists=None,
+ had_collection_link=None,
+ )
+
+ with (
+ patch(
+ "local_deep_research.research_library.services.library_rag_service.get_user_db_session",
+ return_value=DummyCtx(),
+ ),
+ patch(
+ "local_deep_research.research_library.services.library_rag_service._hold_faiss_write_lock",
+ ),
+ patch(
+ "local_deep_research.research_library.services.library_rag_service.ensure_in_collection",
+ return_value=created_link,
+ ) as mock_ensure,
+ ):
+ result = mock_service._write_prepared_document(prepared)
+
+ assert result["status"] == "success"
+ assert result["chunk_count"] == 1
+ mock_ensure.assert_called_once_with(
+ mock_session, "doc-legacy-orphan", "default-lib"
+ )
+ vindex.index_prepared.assert_called_once()
+ mock_service._flag_document_for_reindex.assert_not_called()
+
+ def test_write_prepared_aborts_when_link_vanishes_after_gate(
+ self, mock_service
+ ):
+ """An unlink racing the gate (UPDATE matches 0 rows) aborts the commit.
+
+ The gate query saw the link, but the membership was deleted before
+ the persist writes. The zero-row UPDATE is the backstop: roll back
+ the staged status merge instead of resurrecting a non-member.
+ """
+ import hashlib as _hashlib
+
+ body = "Raced unlink body"
+ body_hash = _hashlib.sha256(body.encode()).hexdigest()
+ mock_link = MagicMock(spec=DocumentCollection)
+ mock_link.indexed = False
+ mock_doc = MagicMock(spec=Document)
+ mock_doc.text_content = body
+ mock_existing = MagicMock()
+ mock_existing.chunk_count = 0
+
+ def make_q(first_val=None):
+ q = MagicMock()
+ q.filter_by.return_value = q
+ q.first.return_value = first_val
+ return q
+
+ q_link = make_q(first_val=mock_link)
+ q_doc = make_q(first_val=mock_doc)
+ q_gate_status = make_q(first_val=None)
+ q_existing = make_q(first_val=None)
+ q_update = make_q()
+ q_update.update.return_value = 0
+
+ mock_session = MagicMock()
+ mock_session.query.side_effect = [
+ q_link,
+ q_doc,
+ q_gate_status,
+ q_existing,
+ q_update,
+ ]
+ mock_service._flag_document_for_reindex = MagicMock()
+
+ vindex = MagicMock()
+ vindex.index_prepared.return_value = MagicMock(
+ added=1, removed=0, chunks=1
+ )
+ mock_service._get_vector_index = MagicMock(return_value=vindex)
+
+ class DummyCtx:
+ def __enter__(self):
+ return mock_session
+
+ def __exit__(self, *args):
+ return False
+
+ prepared = _PreparedDocument(
+ document_id="doc-raced",
+ collection_id="coll-1",
+ chunk_inputs=[ChunkInput(text=body, metadata={})],
+ vectors=MagicMock(),
+ content_hash=body_hash,
+ initial_indexed=False,
+ status_exists=False,
+ had_collection_link=True,
+ )
+
+ with (
+ patch(
+ "local_deep_research.research_library.services.library_rag_service.get_user_db_session",
+ return_value=DummyCtx(),
+ ),
+ patch(
+ "local_deep_research.research_library.services.library_rag_service._hold_faiss_write_lock",
+ ),
+ ):
+ result = mock_service._write_prepared_document(prepared)
+
+ assert result["status"] == "skipped"
+ assert "removed from collection" in result["message"]
+ # Vectors were already staged before the backstop fired; the DB
+ # commit must not follow them.
+ vindex.index_prepared.assert_called_once()
+ mock_session.commit.assert_not_called()
+ mock_service._flag_document_for_reindex.assert_called_once_with(
+ mock_session, "doc-raced", "coll-1"
+ )
diff --git a/tests/research_library/services/test_library_rag_service_index_failure.py b/tests/research_library/services/test_library_rag_service_index_failure.py
index 3ab715fea6..5043fdc944 100644
--- a/tests/research_library/services/test_library_rag_service_index_failure.py
+++ b/tests/research_library/services/test_library_rag_service_index_failure.py
@@ -88,7 +88,7 @@ def test_index_document_not_marked_indexed_when_store_persists_no_embeddings(
svc.rag_index_record.index_path = "/tmp/idx-4855"
svc.rag_index_record.id = 1
- # ...but the vector store fails to persist it: VectorIndex.index() raises
+ # ...but the vector store fails to persist it: VectorIndex.index_prepared() raises
# (its store.apply / embed failure path). index_document must catch that,
# roll back, and flag the document indexed=False for retry — NOT mark it
# indexed=True. Because the int-id store mutates the FAISS file durably
@@ -96,6 +96,9 @@ def test_index_document_not_marked_indexed_when_store_persists_no_embeddings(
# (see _flag_document_for_reindex).
vindex = MagicMock()
vindex.index.side_effect = RuntimeError("store persisted no embeddings")
+ vindex.index_prepared.side_effect = RuntimeError(
+ "store persisted no embeddings"
+ )
svc._get_vector_index = MagicMock(return_value=vindex)
result = svc.index_document("doc-1", "coll-1")