Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

fix(library): stop deleting user FAISS index on integrity mismatch; serialise concurrent indexers (#4197) (#4200)

LearningCircuit014bdf441b2a
diff --git a/changelog.d/+faiss-merge-write-race.bugfix.md b/changelog.d/+faiss-merge-write-race.bugfix.md
new file mode 100644
index 0000000000..a6232d247f
--- /dev/null
+++ b/changelog.d/+faiss-merge-write-race.bugfix.md
@@ -0,0 +1 @@
+Closed a follow-up race condition in the library RAG indexer where two concurrent workers indexing different documents into the same collection could lose each other's embeddings from the FAISS file — last writer's `save_local` overwrote the earlier writer's chunks (the chunks survived in the DB, but the index file was missing them until a force-reindex rebuilt it). The save path now reloads from disk under the per-`(user, index_path)` lock before adding, so concurrent writers' chunks are merged instead of overwriting each other. Follow-up to #4197/#4200.
diff --git a/changelog.d/+faiss-quarantine-and-lock.bugfix.md b/changelog.d/+faiss-quarantine-and-lock.bugfix.md
new file mode 100644
index 0000000000..37eab39631
--- /dev/null
+++ b/changelog.d/+faiss-quarantine-and-lock.bugfix.md
@@ -0,0 +1 @@
+Fixed a data-loss bug where the library RAG index file (`.faiss`) was silently deleted when integrity verification failed during concurrent auto-indexing — destroying hundreds of previously-indexed documents in one go. Corrupted index files are now quarantined to `<path>.corrupt-<ns>` so they remain recoverable, and concurrent indexers are serialised by a per-`(user, index_path)` lock so the race that produced the checksum mismatch no longer occurs. Also drops the per-document `PRAGMA wal_checkpoint(FULL)` that contributed to `database is locked` errors under bulk-download concurrency. Fixes #4197.
diff --git a/src/local_deep_research/research_library/services/library_rag_service.py b/src/local_deep_research/research_library/services/library_rag_service.py
index 40169873f5..1e691fb5d2 100644
--- a/src/local_deep_research/research_library/services/library_rag_service.py
+++ b/src/local_deep_research/research_library/services/library_rag_service.py
@@ -9,6 +9,8 @@ Handles indexing and searching library documents using RAG:
- Track RAG status in library
"""
+import threading
+import time
from pathlib import Path
from typing import Any, Dict, List, Optional, Tuple
@@ -40,6 +42,55 @@ from langchain_community.vectorstores import FAISS
from langchain_community.docstore.in_memory import InMemoryDocstore
+# Module-level locks serialise the FAISS save+record critical section and the
+# load_or_create verify→quarantine→build sequence per (username, index_path).
+# MUST be module-level: each auto-index / scheduler / search worker constructs
+# its own LibraryRAGService, so an instance-scoped lock would coordinate
+# nothing. Pattern is adapted from web/queue/processor_v2._user_critical_locks
+# — instance-scoped→module-scoped, username-only-key→(username, path)-key.
+# See #4197 for the race this guards (concurrent save_local interleaves bytes,
+# producing checksum_mismatch → destructive unlink, lost data).
+_faiss_write_locks: Dict[Tuple[str, str], threading.Lock] = {}
+_faiss_write_locks_lock = threading.Lock()
+
+# Hard cap on suffix-increment retries when generating the .corrupt-<ns> path.
+# Normal case is one attempt — same-ns collisions only happen if the user
+# manually created such files. 32 is a safety bound that converts a deadlock
+# into a loud OSError.
+_QUARANTINE_SUFFIX_RETRY_CAP = 32
+
+
+def _get_faiss_write_lock(username: str, index_path: str) -> threading.Lock:
+ """Return the lock for ``(username, index_path)``, creating it on first
+ access. Key is normalised via ``Path.resolve()`` to match
+ ``FileIntegrityManager._normalize_path`` so writers/readers/quarantine
+ all agree on identity.
+ """
+ key = (username, str(Path(index_path).resolve()))
+ with _faiss_write_locks_lock:
+ lock = _faiss_write_locks.get(key)
+ if lock is None:
+ lock = threading.Lock()
+ _faiss_write_locks[key] = lock
+ return lock
+
+
+def pop_faiss_locks_for_user(username: str) -> None:
+ """Remove all FAISS-write locks belonging to ``username``.
+
+ Called from the user-close paths (connection_cleanup) so the dict
+ doesn't grow one entry per (user × collection) across the process
+ lifetime. Safe to call while a lock is held: the holder keeps using
+ its local reference, and the next access lazily creates a fresh
+ lock. Same semantics as ``pop_user_critical_lock`` in
+ web/queue/processor_v2.py.
+ """
+ with _faiss_write_locks_lock:
+ stale = [k for k in _faiss_write_locks if k[0] == username]
+ for k in stale:
+ _faiss_write_locks.pop(k, None)
+
+
class LibraryRAGService:
"""Service for managing RAG indexing of library documents."""
@@ -297,6 +348,206 @@ class LibraryRAGService:
return rag_index
+ def _quarantine_corrupt_index(self, index_path: Path, reason: str) -> None:
+ """Rename a corrupted FAISS index and its ``.pkl`` companion to
+ ``<path>.corrupt-<ns>`` instead of deleting them.
+
+ Preserves user data for inspection/recovery. The
+ dimension-mismatch branch elsewhere in this method intentionally
+ *deletes* — that case rebuilds from scratch and the old bytes
+ are unreadable with the new model. This helper is for the two
+ "transient or unknown failure" branches (verify_file said no,
+ load_local raised) where the on-disk bytes may still be
+ usable by a human.
+
+ Raises ``OSError`` on rename failure (disk full, read-only fs,
+ permission denied). Re-raising prevents silent data loss: if we
+ swallowed the error, the next ``save_local`` would truncate the
+ corrupt bytes anyway. Caller paths log the exception via the
+ broader try/except around their indexer call.
+
+ # TODO(#4197-followup): FileIntegrityRecord.consecutive_failures
+ # is not reset by the next record_file call, leaking failure
+ # counts across recovery cycles. Orthogonal to this fix.
+ """
+ ns = time.time_ns()
+ pkl_path = index_path.with_suffix(".pkl")
+
+ # Same-nanosecond collisions essentially can't happen between
+ # concurrent threads (we hold the per-path lock), but a user
+ # could manually create such files. Loop with a hard cap so a
+ # weird state surfaces as a clean OSError rather than a hang.
+ suffix_n = 0
+ faiss_target = Path(f"{index_path}.corrupt-{ns}")
+ pkl_target = Path(f"{pkl_path}.corrupt-{ns}")
+ while faiss_target.exists() or pkl_target.exists():
+ suffix_n += 1
+ if suffix_n > _QUARANTINE_SUFFIX_RETRY_CAP:
+ raise OSError(
+ f"Quarantine path collisions exceeded "
+ f"{_QUARANTINE_SUFFIX_RETRY_CAP} retries for "
+ f"{index_path}"
+ )
+ faiss_target = Path(f"{index_path}.corrupt-{ns}-{suffix_n}")
+ pkl_target = Path(f"{pkl_path}.corrupt-{ns}-{suffix_n}")
+
+ try:
+ index_path.rename(faiss_target)
+ logger.warning(
+ f"Quarantined corrupted FAISS index to {faiss_target} "
+ f"(reason: {reason})"
+ )
+ if pkl_path.exists():
+ pkl_path.rename(pkl_target)
+ logger.info(f"Quarantined companion PKL to {pkl_target}")
+ else:
+ # Missing .pkl is recoverable (FAISS may have crashed
+ # mid-write between .faiss and .pkl). Not re-raised:
+ # the .faiss is already preserved, fresh build will
+ # write a fresh .pkl.
+ logger.warning(
+ f"PKL companion missing at {pkl_path}; only "
+ f".faiss quarantined."
+ )
+ except OSError:
+ # Disk-full, read-only fs, or permission error. Re-raise
+ # so the caller surfaces a real failure instead of letting
+ # the next save_local truncate the corrupt bytes.
+ logger.exception(
+ f"Failed to quarantine corrupted index at {index_path}"
+ )
+ raise
+
+ def _save_and_record_locked(self, index_path: Path) -> None:
+ """Atomically save the in-memory FAISS index to ``index_path``
+ and record its checksum in the integrity manager.
+
+ The save_local + record_file pair must be serialised across
+ concurrent indexers for the same ``(username, index_path)`` so
+ a second writer doesn't overwrite the file between the first
+ writer's save and checksum recording — that race produces the
+ checksum_mismatch / silent-deletion bug in #4197.
+ """
+ with _get_faiss_write_lock(self.username, str(index_path)):
+ self.faiss_index.save_local(
+ str(index_path.parent), index_name=index_path.stem
+ )
+ self.integrity_manager.record_file(
+ index_path,
+ related_entity_type="rag_index",
+ related_entity_id=self.rag_index_record.id,
+ )
+
+ def _merge_and_persist_locked(
+ self,
+ index_path: Path,
+ chunks_to_add: list,
+ embedding_ids: list,
+ force_reindex: bool = False,
+ ) -> Dict[str, int]:
+ """Read-modify-write the on-disk FAISS index under the
+ ``(username, index_path)`` write lock so concurrent workers
+ don't lose each other's embeddings.
+
+ Without this, two workers indexing different documents into
+ the same collection both load on-disk state X into memory,
+ each calls ``add_documents`` on their own private FAISS object
+ (worker A: ``X+docA``, worker B: ``X+docB``), then they save
+ in sequence — last writer wins, the loser's chunks are gone
+ from the FAISS file. The chunks survive in the per-document
+ DB rows so a force-reindex rebuilds, but the index file is
+ wrong until then. See AI review of #4200.
+
+ The slow embedding + DB-chunk-insert path stays OUTSIDE this
+ lock (handled by the caller). The lock only covers the fast
+ FAISS reload→merge→save sequence (~10-100ms even for large
+ indices), so cross-document parallelism is preserved up to
+ the point where the in-memory states need reconciliation.
+
+ Args:
+ index_path: Path of the on-disk ``.faiss`` file.
+ chunks_to_add: Chunks the caller wants to add.
+ embedding_ids: Matching embedding/chunk IDs (same length).
+ force_reindex: If True, delete any IDs from
+ ``embedding_ids`` that already exist in the index
+ (so the caller's metadata wins). If False, skip IDs
+ that already exist (idempotent re-indexing).
+
+ Returns:
+ ``{"added": n, "skipped": m}`` for caller logging.
+ """
+ with _get_faiss_write_lock(self.username, str(index_path)):
+ # Reload from disk to absorb other writers' saves. If the
+ # file doesn't exist or fails verification, keep whatever
+ # the caller already had in ``self.faiss_index`` (likely
+ # a fresh in-memory index from load_or_create_faiss_index).
+ if index_path.exists():
+ verified, _reason = self.integrity_manager.verify_file(
+ index_path
+ )
+ if verified:
+ try:
+ self.faiss_index = FAISS.load_local(
+ str(index_path.parent),
+ self.embedding_manager.embeddings,
+ index_name=index_path.stem,
+ allow_dangerous_deserialization=True,
+ normalize_L2=True,
+ )
+ except Exception:
+ # Reload failed (torn write, etc.). Keep
+ # in-memory state — it's stale but valid;
+ # better than losing this write entirely.
+ logger.warning(
+ "Failed to reload FAISS for merge; "
+ "proceeding with in-memory state."
+ )
+
+ # Force-reindex: remove old copies of IDs we're about to
+ # re-add, so updated metadata replaces stale. Dedup via
+ # set→list because ``embedding_ids`` can contain repeats
+ # (same chunk hash appearing twice in the document) —
+ # FAISS.delete with duplicate IDs raises.
+ if force_reindex and hasattr(self.faiss_index, "docstore"):
+ fresh_ids = set(self.faiss_index.docstore._dict.keys())
+ old_chunk_ids = list(
+ {eid for eid in embedding_ids if eid in fresh_ids}
+ )
+ if old_chunk_ids:
+ logger.info(
+ f"Force re-index: removing {len(old_chunk_ids)} "
+ f"existing chunks from FAISS"
+ )
+ self.faiss_index.delete(old_chunk_ids)
+
+ # Dedup against the freshly-loaded state.
+ if not force_reindex and hasattr(self.faiss_index, "docstore"):
+ fresh_ids = set(self.faiss_index.docstore._dict.keys())
+ else:
+ fresh_ids = None
+
+ new_chunks, new_ids = self._deduplicate_chunks(
+ chunks_to_add, embedding_ids, fresh_ids
+ )
+
+ if new_chunks:
+ self.faiss_index.add_documents(new_chunks, ids=new_ids)
+
+ self.faiss_index.save_local(
+ str(index_path.parent), index_name=index_path.stem
+ )
+ self.integrity_manager.record_file(
+ index_path,
+ related_entity_type="rag_index",
+ related_entity_id=self.rag_index_record.id,
+ )
+
+ return {
+ "added": len(new_chunks),
+ "skipped": len(chunks_to_add) - len(new_chunks),
+ "added_ids": new_ids,
+ }
+
def load_or_create_faiss_index(self, collection_id: str) -> FAISS:
"""
Load existing FAISS index or create new one.
@@ -312,97 +563,117 @@ class LibraryRAGService:
index_path = Path(rag_index.index_path)
+ # Hold the per-(username, index_path) write lock across the
+ # entire verify → quarantine → load sequence. A narrower scope
+ # leaves room for a concurrent save_local to race the
+ # verification (verify sees bytes A, save overwrites with bytes
+ # B, load_local reads bytes B which no longer match verified
+ # checksum). See #4197. The fresh-build path below this block
+ # is in-memory only and doesn't need the lock.
if index_path.exists():
- # Verify integrity before loading
- verified, reason = self.integrity_manager.verify_file(index_path)
- if not verified:
- logger.error(
- f"Integrity verification failed for {index_path}: {reason}. "
- f"Refusing to load. Creating new index."
+ load_lock = _get_faiss_write_lock(self.username, str(index_path))
+ with load_lock:
+ # Verify integrity before loading
+ verified, reason = self.integrity_manager.verify_file(
+ index_path
)
- # Remove corrupted index
- try:
- index_path.unlink()
- logger.info(f"Removed corrupted index file: {index_path}")
- except Exception:
- logger.exception("Failed to remove corrupted index")
- else:
- try:
- # Check for embedding dimension mismatch before loading
- current_dim = len(
- self.embedding_manager.embeddings.embed_query(
- "dimension_check"
- )
+ if not verified:
+ logger.error(
+ f"Integrity verification failed for {index_path}: "
+ f"{reason}. Quarantining for recovery; creating "
+ f"new index."
)
- stored_dim = rag_index.embedding_dimension
-
- if stored_dim and current_dim != stored_dim:
- logger.warning(
- f"Embedding dimension mismatch detected! "
- f"Index created with dim={stored_dim}, "
- f"current model returns dim={current_dim}. "
- f"Deleting old index and rebuilding."
- )
- # Delete old index files
- try:
- index_path.unlink()
- pkl_path = index_path.with_suffix(".pkl")
- if pkl_path.exists():
- pkl_path.unlink()
- logger.info(
- f"Deleted old FAISS index files at {index_path}"
+ self._quarantine_corrupt_index(index_path, reason)
+ else:
+ try:
+ # Check for embedding dimension mismatch before loading
+ current_dim = len(
+ self.embedding_manager.embeddings.embed_query(
+ "dimension_check"
)
- except Exception:
- logger.exception("Failed to delete old index files")
-
- # Update RAGIndex with new dimension and reset counts
- with get_user_db_session(
- self.username, self.db_password
- ) as session:
- idx = (
- session.query(RAGIndex)
- .filter_by(id=rag_index.id)
- .first()
+ )
+ stored_dim = rag_index.embedding_dimension
+
+ if stored_dim and current_dim != stored_dim:
+ logger.warning(
+ f"Embedding dimension mismatch detected! "
+ f"Index created with dim={stored_dim}, "
+ f"current model returns dim={current_dim}. "
+ f"Deleting old index and rebuilding."
)
- if idx:
- idx.embedding_dimension = current_dim
- idx.chunk_count = 0
- idx.total_documents = 0
+ # Delete old index files (legitimate deletion:
+ # the bytes are unreadable with the new model).
+ try:
+ index_path.unlink()
+ pkl_path = index_path.with_suffix(".pkl")
+ if pkl_path.exists():
+ pkl_path.unlink()
+ logger.info(
+ f"Deleted old FAISS index files at {index_path}"
+ )
+ except Exception:
+ logger.exception(
+ "Failed to delete old index files"
+ )
+
+ # Update RAGIndex with new dimension and reset counts
+ with get_user_db_session(
+ self.username, self.db_password
+ ) as session:
+ idx = (
+ session.query(RAGIndex)
+ .filter_by(id=rag_index.id)
+ .first()
+ )
+ if idx:
+ idx.embedding_dimension = current_dim
+ idx.chunk_count = 0
+ idx.total_documents = 0
+ session.commit()
+ logger.info(
+ f"Updated RAGIndex dimension to {current_dim}"
+ )
+
+ # Clear rag_document_status for this index
+ session.query(RagDocumentStatus).filter_by(
+ rag_index_id=rag_index.id
+ ).delete()
session.commit()
logger.info(
- f"Updated RAGIndex dimension to {current_dim}"
+ "Cleared indexed status for documents in this "
+ "collection"
)
- # Clear rag_document_status for this index
- session.query(RagDocumentStatus).filter_by(
- rag_index_id=rag_index.id
- ).delete()
- session.commit()
+ # Update local reference for index creation below
+ rag_index.embedding_dimension = current_dim
+ # Fall through to create new index below
+ else:
+ # Dimensions match (or no stored dimension), load index
+ faiss_index = FAISS.load_local(
+ str(index_path.parent),
+ self.embedding_manager.embeddings,
+ index_name=index_path.stem,
+ allow_dangerous_deserialization=True,
+ normalize_L2=True,
+ )
logger.info(
- "Cleared indexed status for documents in this "
- "collection"
+ f"Loaded existing FAISS index from {index_path}"
)
-
- # Update local reference for index creation below
- rag_index.embedding_dimension = current_dim
- # Fall through to create new index below
- else:
- # Dimensions match (or no stored dimension), load index
- faiss_index = FAISS.load_local(
- str(index_path.parent),
- self.embedding_manager.embeddings,
- index_name=index_path.stem,
- allow_dangerous_deserialization=True,
- normalize_L2=True,
- )
- logger.info(
- f"Loaded existing FAISS index from {index_path}"
+ return faiss_index
+ except Exception:
+ # load_local raised (torn .pkl, malformed pickle,
+ # FAISS read failure). The .faiss bytes may still
+ # be recoverable, so quarantine before falling
+ # through to fresh build — don't just leave
+ # broken state on disk.
+ logger.warning(
+ "Failed to load FAISS index, quarantining "
+ "and creating new one"
)
- return faiss_index
- except Exception:
- logger.warning(
- "Failed to load FAISS index, creating new one"
- )
+ if index_path.exists():
+ self._quarantine_corrupt_index(
+ index_path, "load_local_raised"
+ )
# Create new FAISS index with configurable type and distance metric
logger.info(
@@ -615,80 +886,50 @@ class LibraryRAGService:
source_id=document_id,
)
- # Load or create FAISS index
+ # Load or create FAISS index (lazy; the merge step
+ # below will reload from disk under the lock anyway).
if self.faiss_index is None:
self.faiss_index = self.load_or_create_faiss_index(
collection_id
)
- # If force_reindex, remove old chunks from FAISS before adding new ones
- if force_reindex:
- existing_ids = (
- set(self.faiss_index.docstore._dict.keys())
- if hasattr(self.faiss_index, "docstore")
- else set()
- )
- old_chunk_ids = list(
- {eid for eid in embedding_ids if eid in existing_ids}
- )
- if old_chunk_ids:
- logger.info(
- f"Force re-index: removing {len(old_chunk_ids)} existing chunks from FAISS"
- )
- self.faiss_index.delete(old_chunk_ids)
-
- # Filter out chunks that already exist in FAISS (unless force_reindex)
- if not force_reindex:
- existing_ids = (
- set(self.faiss_index.docstore._dict.keys())
- if hasattr(self.faiss_index, "docstore")
- else set()
- )
- else:
- existing_ids = None
-
+ # Read-modify-write the on-disk FAISS index under
+ # the per-(user, index_path) lock. The lock spans
+ # reload + dedup + add + save so concurrent indexers
+ # of different documents into the same collection
+ # don't lose each other's embeddings (see AI review
+ # of #4200).
+ index_path = Path(self.rag_index_record.index_path)
unique_count = len(set(embedding_ids))
batch_dups = len(chunks) - unique_count
-
- new_chunks, new_ids = self._deduplicate_chunks(
- chunks, embedding_ids, existing_ids
+ merge_stats = self._merge_and_persist_locked(
+ index_path,
+ chunks,
+ embedding_ids,
+ force_reindex=force_reindex,
)
-
- # Add embeddings to FAISS index
- if new_chunks:
+ if merge_stats["added"]:
if force_reindex:
logger.info(
- f"Force re-index: adding {len(new_chunks)} chunks with updated metadata to FAISS index"
+ f"Force re-index: added {merge_stats['added']} "
+ f"chunks with updated metadata to FAISS index"
)
else:
- already_exist = unique_count - len(new_chunks)
+ already_exist = unique_count - merge_stats["added"]
logger.info(
- f"Adding {len(new_chunks)} new embeddings to FAISS index "
- f"({already_exist} already exist, {batch_dups} batch duplicates removed)"
+ f"Added {merge_stats['added']} new embeddings to FAISS "
+ f"({already_exist} already exist, "
+ f"{batch_dups} batch duplicates removed)"
)
- self.faiss_index.add_documents(new_chunks, ids=new_ids)
else:
logger.info(
f"All {len(chunks)} chunks already exist in FAISS index, skipping"
)
-
- # Save FAISS index
- index_path = Path(self.rag_index_record.index_path)
- self.faiss_index.save_local(
- str(index_path.parent), index_name=index_path.stem
- )
- # Record file integrity
- self.integrity_manager.record_file(
- index_path,
- related_entity_type="rag_index",
- related_entity_id=self.rag_index_record.id,
- )
logger.info(
f"Saved FAISS index to {index_path} with integrity tracking"
)
from datetime import datetime, UTC
- from sqlalchemy import text
# Check if document was already indexed (for stats update)
existing_status = (
@@ -751,12 +992,14 @@ class LibraryRAGService:
session.flush()
logger.info(f"Flushed ORM changes for document {document_id}")
- # Commit the transaction
+ # Commit the transaction. Durability is provided by
+ # synchronous=NORMAL (sqlcipher_utils.py); SQLite
+ # auto-checkpoints WAL at wal_autocheckpoint=250 frames.
+ # An explicit PRAGMA wal_checkpoint(FULL) here used to
+ # block other writers long enough to exhaust busy_timeout
+ # under bulk-download concurrency (#4197).
session.commit()
- # WAL checkpoint after commit to ensure persistence
- session.execute(text("PRAGMA wal_checkpoint(FULL)"))
-
logger.info(
f"Successfully indexed document {document_id} for collection {collection_id} "
f"with {len(chunks)} chunks"
@@ -1198,6 +1441,7 @@ class LibraryRAGService:
)
# Load or create FAISS index using default library collection
+ # (lazy; merge step below reloads under the lock anyway).
if self.faiss_index is None:
from ...database.library_init import get_default_library_id
@@ -1208,52 +1452,50 @@ class LibraryRAGService:
default_collection_id
)
- # Filter out chunks that already exist in FAISS and deduplicate
- if self.faiss_index is not None:
- existing_ids = (
- set(self.faiss_index.docstore._dict.keys())
- if hasattr(self.faiss_index, "docstore")
- else set()
- )
- else:
- existing_ids = None
- new_chunks, new_ids = self._deduplicate_chunks(
- chunks, embedding_ids, existing_ids
- )
-
- # Add embeddings to FAISS index
- if new_chunks:
- self.faiss_index.add_documents(new_chunks, ids=new_ids)
-
- # Save FAISS index
+ # Read-modify-write the on-disk FAISS under the lock.
index_path = (
Path(self.rag_index_record.index_path)
if self.rag_index_record
else None
)
if index_path:
- self.faiss_index.save_local(
- str(index_path.parent), index_name=index_path.stem
- )
- # Record file integrity
- self.integrity_manager.record_file(
- index_path,
- related_entity_type="rag_index",
- related_entity_id=self.rag_index_record.id,
+ merge_stats = self._merge_and_persist_locked(
+ index_path, chunks, embedding_ids, force_reindex=False
)
logger.info(
f"Saved FAISS index to {index_path} with integrity tracking"
)
+ else:
+ # No persistent index path — in-memory only path.
+ # Preserve old behavior: dedup against in-memory state
+ # and add directly without saving.
+ if self.faiss_index is not None and hasattr(
+ self.faiss_index, "docstore"
+ ):
+ existing_ids = set(self.faiss_index.docstore._dict.keys())
+ else:
+ existing_ids = None
+ new_chunks, new_ids = self._deduplicate_chunks(
+ chunks, embedding_ids, existing_ids
+ )
+ if new_chunks:
+ self.faiss_index.add_documents(new_chunks, ids=new_ids)
+ merge_stats = {
+ "added": len(new_chunks),
+ "skipped": len(chunks) - len(new_chunks),
+ "added_ids": new_ids,
+ }
logger.info(
- f"Successfully indexed local file {file_path} with {len(new_chunks)} new chunks "
- f"({len(chunks) - len(new_chunks)} skipped)"
+ f"Successfully indexed local file {file_path} with "
+ f"{merge_stats['added']} new chunks "
+ f"({merge_stats['skipped']} skipped)"
)
return {
"status": "success",
- "chunk_count": len(new_chunks),
- "embedding_ids": new_ids,
+ "chunk_count": merge_stats["added"],
+ "embedding_ids": merge_stats["added_ids"],
}
except Exception as e:
@@ -1316,7 +1558,8 @@ class LibraryRAGService:
source_id=user_doc.id,
)
- # Load or create FAISS index for this collection
+ # Load or create FAISS index for this collection (lazy;
+ # merge step below reloads under the lock anyway).
if self.faiss_index is None:
# Extract collection_id from collection_name (format: "collection_<uuid>")
collection_id = collection_name.removeprefix("collection_")
@@ -1324,74 +1567,71 @@ class LibraryRAGService:
collection_id
)
- # If force_reindex, remove old chunks from FAISS before adding new ones
- if force_reindex:
- existing_ids = (
- set(self.faiss_index.docstore._dict.keys())
- if hasattr(self.faiss_index, "docstore")
- else set()
- )
- old_chunk_ids = list(
- {eid for eid in embedding_ids if eid in existing_ids}
- )
- if old_chunk_ids:
- logger.info(
- f"Force re-index: removing {len(old_chunk_ids)} existing chunks from FAISS"
- )
- self.faiss_index.delete(old_chunk_ids)
-
- # Filter out chunks that already exist in FAISS (unless force_reindex)
- if not force_reindex:
- existing_ids = (
- set(self.faiss_index.docstore._dict.keys())
- if hasattr(self.faiss_index, "docstore")
- else set()
- )
- else:
- existing_ids = None
-
unique_count = len(set(embedding_ids))
batch_dups = len(chunks) - unique_count
- new_chunks, new_ids = self._deduplicate_chunks(
- chunks, embedding_ids, existing_ids
+ # Read-modify-write the on-disk FAISS under the lock so
+ # concurrent uploads to the same collection don't lose
+ # each other's chunks.
+ index_path = (
+ Path(self.rag_index_record.index_path)
+ if self.rag_index_record
+ else None
)
-
- # Add embeddings to FAISS index
- if new_chunks:
+ if index_path:
+ merge_stats = self._merge_and_persist_locked(
+ index_path,
+ chunks,
+ embedding_ids,
+ force_reindex=force_reindex,
+ )
+ else:
+ # No persistent index path — in-memory only path.
+ # Preserve old behavior: handle force_reindex deletion
+ # and dedup add against in-memory state without saving.
+ if force_reindex and hasattr(self.faiss_index, "docstore"):
+ existing_ids = set(self.faiss_index.docstore._dict.keys())
+ old_chunk_ids = list(
+ {eid for eid in embedding_ids if eid in existing_ids}
+ )
+ if old_chunk_ids:
+ logger.info(
+ f"Force re-index: removing {len(old_chunk_ids)} "
+ f"existing chunks from FAISS"
+ )
+ self.faiss_index.delete(old_chunk_ids)
+ if not force_reindex and hasattr(self.faiss_index, "docstore"):
+ existing_ids = set(self.faiss_index.docstore._dict.keys())
+ else:
+ existing_ids = None
+ new_chunks, new_ids = self._deduplicate_chunks(
+ chunks, embedding_ids, existing_ids
+ )
+ if new_chunks:
+ self.faiss_index.add_documents(new_chunks, ids=new_ids)
+ merge_stats = {
+ "added": len(new_chunks),
+ "skipped": len(chunks) - len(new_chunks),
+ "added_ids": new_ids,
+ }
+ if merge_stats["added"]:
if force_reindex:
logger.info(
- f"Force re-index: adding {len(new_chunks)} chunks with updated metadata to FAISS index"
+ f"Force re-index: added {merge_stats['added']} "
+ f"chunks with updated metadata to FAISS index"
)
else:
- already_exist = unique_count - len(new_chunks)
+ already_exist = unique_count - merge_stats["added"]
logger.info(
- f"Adding {len(new_chunks)} new chunks to FAISS index "
- f"({already_exist} already exist, {batch_dups} batch duplicates removed)"
+ f"Added {merge_stats['added']} new chunks to FAISS "
+ f"({already_exist} already exist, "
+ f"{batch_dups} batch duplicates removed)"
)
- self.faiss_index.add_documents(new_chunks, ids=new_ids)
else:
logger.info(
f"All {len(chunks)} chunks already exist in FAISS index, skipping"
)
- # Save FAISS index
- index_path = (
- Path(self.rag_index_record.index_path)
- if self.rag_index_record
- else None
- )
- if index_path:
- self.faiss_index.save_local(
- str(index_path.parent), index_name=index_path.stem
- )
- # Record file integrity
- self.integrity_manager.record_file(
- index_path,
- related_entity_type="rag_index",
- related_entity_id=self.rag_index_record.id,
- )
-
logger.info(
f"Successfully indexed user document {user_doc.filename} with {len(chunks)} chunks"
)
@@ -1452,28 +1692,34 @@ class LibraryRAGService:
collection_id
)
- # Remove from FAISS index
+ # Remove from FAISS index. delete + save + record must all
+ # be inside the same lock — otherwise a concurrent writer
+ # could sandwich a save_local between our delete and our
+ # save, leaving stale chunks back on disk (#4197).
if hasattr(self.faiss_index, "delete"):
try:
- self.faiss_index.delete(chunk_ids)
-
- # Save updated index
index_path = (
Path(self.rag_index_record.index_path)
if self.rag_index_record
else None
)
if index_path:
- self.faiss_index.save_local(
- str(index_path.parent),
- index_name=index_path.stem,
- )
- # Record file integrity
- self.integrity_manager.record_file(
- index_path,
- related_entity_type="rag_index",
- related_entity_id=self.rag_index_record.id,
- )
+ with _get_faiss_write_lock(
+ self.username, str(index_path)
+ ):
+ self.faiss_index.delete(chunk_ids)
+ self.faiss_index.save_local(
+ str(index_path.parent),
+ index_name=index_path.stem,
+ )
+ self.integrity_manager.record_file(
+ index_path,
+ related_entity_type="rag_index",
+ related_entity_id=self.rag_index_record.id,
+ )
+ else:
+ # No index path → in-memory-only delete
+ self.faiss_index.delete(chunk_ids)
except Exception:
logger.warning("Could not delete chunks from FAISS")
diff --git a/src/local_deep_research/web/auth/connection_cleanup.py b/src/local_deep_research/web/auth/connection_cleanup.py
index 77203caef0..2c8f95926f 100644
--- a/src/local_deep_research/web/auth/connection_cleanup.py
+++ b/src/local_deep_research/web/auth/connection_cleanup.py
@@ -48,21 +48,22 @@ _last_dispose_time = 0.0
def _pop_per_user_locks(username: str) -> None:
- """Pop ``username`` from the three module-level per-user lock dicts.
-
- The library-init, backup, and queue-processor modules each maintain
- a ``dict[username, threading.Lock]`` for serialising per-user
- critical sections. None of them had a removal hook, so without
- this the dicts accumulated one entry per username across the
- process lifetime — bounded by total users (~296 bytes/entry × 3 =
- ~900 B/user/dict) but real on long-lived self-hosted instances
- with churn. Called from the user-close paths so the next login
- starts with fresh locks (the locks hold no state worth preserving
- across login/logout).
+ """Pop ``username`` from the four module-level per-user lock dicts.
+
+ The library-init, backup, queue-processor, and library-RAG modules
+ each maintain a ``dict[..., threading.Lock]`` keyed by username (or
+ by ``(username, ...)``) for serialising per-user critical sections.
+ None of them had a removal hook, so without this the dicts
+ accumulated one entry per username across the process lifetime —
+ bounded by total users (~296 bytes/entry × 4 = ~1.2 KB/user/dict)
+ but real on long-lived self-hosted instances with churn. Called
+ from the user-close paths so the next login starts with fresh
+ locks (the locks hold no state worth preserving across
+ login/logout).
Lazy-imported here to keep this module's import graph shallow:
``connection_cleanup`` runs at startup and shouldn't pull in the
- queue / backup / library-init modules eagerly.
+ queue / backup / library-init / library-RAG modules eagerly.
"""
try:
from ...database.library_init import pop_user_init_lock
@@ -89,6 +90,15 @@ def _pop_per_user_locks(username: str) -> None:
except Exception:
logger.warning(f"Failed to pop _user_critical_locks for {username}")
+ try:
+ from ...research_library.services.library_rag_service import (
+ pop_faiss_locks_for_user,
+ )
+
+ pop_faiss_locks_for_user(username)
+ except Exception:
+ logger.warning(f"Failed to pop _faiss_write_locks for {username}")
+
def _count_open_fds() -> int:
"""Count open file descriptors for the current process."""
diff --git a/tests/research_library/services/test_library_rag_service_coverage.py b/tests/research_library/services/test_library_rag_service_coverage.py
index b83e19740b..6e52c215aa 100644
--- a/tests/research_library/services/test_library_rag_service_coverage.py
+++ b/tests/research_library/services/test_library_rag_service_coverage.py
@@ -13,9 +13,11 @@ Focuses on logic paths not exercised by the existing test_library_rag_service.py
"""
import hashlib
+import threading
from pathlib import Path
from unittest.mock import MagicMock, patch
+import pytest
from langchain_core.documents import Document as LangchainDocument
# ---------------------------------------------------------------------------
@@ -393,9 +395,13 @@ class TestLoadOrCreateFaissIndex:
@patch(f"{_MOD}.FAISS")
@patch(f"{_MOD}.InMemoryDocstore")
@patch(f"{_MOD}.IndexFlatIP")
- def test_integrity_failure_creates_new_index(
- self, mock_flat_ip, mock_docstore, mock_faiss_cls
+ def test_integrity_failure_quarantines_and_creates_new_index(
+ self, mock_flat_ip, mock_docstore, mock_faiss_cls, tmp_path
):
+ """Verify-failure path: corrupt .faiss + .pkl are RENAMED to
+ .corrupt-<ns>, NOT unlinked. Then fresh index is created.
+ Regression for #4197 data-loss bug.
+ """
svc = _make_service()
svc.embedding_manager = MagicMock()
svc.integrity_manager = MagicMock()
@@ -404,18 +410,32 @@ class TestLoadOrCreateFaissIndex:
"hash mismatch",
)
+ # Use real on-disk files so .exists() in the quarantine
+ # collision check returns False naturally for the .corrupt-*
+ # paths, not True-for-everything.
+ idx_path = tmp_path / "corrupt.faiss"
+ pkl_path = tmp_path / "corrupt.pkl"
+ idx_path.write_bytes(b"faiss-bytes")
+ pkl_path.write_bytes(b"pkl-bytes")
+
rag_idx = self._make_rag_index()
- rag_idx.index_path = "/tmp/corrupt.faiss"
+ rag_idx.index_path = str(idx_path)
self._patch_get_or_create(svc, rag_idx)
- with (
- patch.object(Path, "exists", return_value=True),
- patch.object(Path, "unlink") as mock_unlink,
- ):
+ with patch.object(Path, "unlink") as mock_unlink:
svc.load_or_create_faiss_index("coll-1")
- # Should have tried to unlink the corrupted file
- mock_unlink.assert_called_once()
+ # Should NOT have unlinked anything in the verify-failure path
+ mock_unlink.assert_not_called()
+ # Originals quarantined (renamed away from their paths)
+ assert not idx_path.exists()
+ assert not pkl_path.exists()
+ # Both files preserved under .corrupt-<ns> names
+ corrupt_faiss = list(tmp_path.glob("corrupt.faiss.corrupt-*"))
+ corrupt_pkl = list(tmp_path.glob("corrupt.pkl.corrupt-*"))
+ assert len(corrupt_faiss) == 1
+ assert len(corrupt_pkl) == 1
+ assert corrupt_faiss[0].read_bytes() == b"faiss-bytes"
# Should return a new FAISS instance, not load_local
mock_faiss_cls.load_local.assert_not_called()
mock_faiss_cls.assert_called_once()
@@ -423,26 +443,34 @@ class TestLoadOrCreateFaissIndex:
@patch(f"{_MOD}.FAISS")
@patch(f"{_MOD}.InMemoryDocstore")
@patch(f"{_MOD}.IndexFlatIP")
- def test_corrupted_unlink_failure_still_creates_new_index(
- self, mock_flat_ip, mock_docstore, mock_faiss_cls
+ def test_quarantine_rename_oserror_re_raises(
+ self, mock_flat_ip, mock_docstore, mock_faiss_cls, tmp_path
):
+ """Disk-full / read-only fs during quarantine MUST propagate.
+ Silently falling through would let the next save_local truncate
+ the corrupt bytes, recreating the very data loss #4197 fixes.
+ """
svc = _make_service()
svc.embedding_manager = MagicMock()
svc.integrity_manager = MagicMock()
svc.integrity_manager.verify_file.return_value = (False, "bad hash")
+ idx_path = tmp_path / "corrupt.faiss"
+ idx_path.write_bytes(b"faiss-bytes")
+ (tmp_path / "corrupt.pkl").write_bytes(b"pkl-bytes")
+
rag_idx = self._make_rag_index()
- rag_idx.index_path = "/tmp/corrupt.faiss"
+ rag_idx.index_path = str(idx_path)
self._patch_get_or_create(svc, rag_idx)
- with (
- patch.object(Path, "exists", return_value=True),
- patch.object(Path, "unlink", side_effect=PermissionError("denied")),
- ):
- svc.load_or_create_faiss_index("coll-1")
+ with patch.object(Path, "rename", side_effect=OSError("ENOSPC")):
+ with pytest.raises(OSError, match="ENOSPC"):
+ svc.load_or_create_faiss_index("coll-1")
- # Should still create a new FAISS, not crash
- mock_faiss_cls.assert_called_once()
+ # Fresh index must NOT have been created — corrupt bytes are
+ # still on disk and creating one would let the next save
+ # overwrite them.
+ mock_faiss_cls.assert_not_called()
@patch(f"{_MOD}.get_user_db_session")
@patch(f"{_MOD}.FAISS")
@@ -487,24 +515,38 @@ class TestLoadOrCreateFaissIndex:
@patch(f"{_MOD}.FAISS")
@patch(f"{_MOD}.InMemoryDocstore")
@patch(f"{_MOD}.IndexFlatIP")
- def test_load_failure_creates_new_index(
- self, mock_flat_ip, mock_docstore, mock_faiss_cls
+ def test_load_failure_quarantines_and_creates_new_index(
+ self, mock_flat_ip, mock_docstore, mock_faiss_cls, tmp_path
):
+ """When load_local raises (torn .pkl, malformed pickle), the
+ .faiss and .pkl are quarantined before falling through to a
+ fresh index. Previously the old code silently discarded the
+ broken-state files without preserving evidence.
+ """
svc = _make_service()
svc.embedding_manager = MagicMock()
svc.embedding_manager.embeddings.embed_query.return_value = [0.0] * 384
svc.integrity_manager = MagicMock()
svc.integrity_manager.verify_file.return_value = (True, None)
+ idx_path = tmp_path / "broken.faiss"
+ pkl_path = tmp_path / "broken.pkl"
+ idx_path.write_bytes(b"faiss-bytes")
+ pkl_path.write_bytes(b"pkl-bytes")
+
rag_idx = self._make_rag_index(dim=384)
- rag_idx.index_path = "/tmp/broken.faiss"
+ rag_idx.index_path = str(idx_path)
self._patch_get_or_create(svc, rag_idx)
mock_faiss_cls.load_local.side_effect = RuntimeError("corrupted file")
- with patch.object(Path, "exists", return_value=True):
- svc.load_or_create_faiss_index("coll-1")
+ svc.load_or_create_faiss_index("coll-1")
+ # Both files quarantined (renamed away)
+ assert not idx_path.exists()
+ assert not pkl_path.exists()
+ assert len(list(tmp_path.glob("broken.faiss.corrupt-*"))) == 1
+ assert len(list(tmp_path.glob("broken.pkl.corrupt-*"))) == 1
# Should fall through and create new index
mock_faiss_cls.assert_called_once()
@@ -753,3 +795,528 @@ class TestContextManager:
svc = _make_service()
result = svc.__exit__(None, None, None)
assert result is False
+
+
+# =========================================================================
+# Quarantine helper + concurrent-write locks (#4197)
+# =========================================================================
+
+
+def _import_module():
+ import local_deep_research.research_library.services.library_rag_service as m
+
+ return m
+
+
+class TestCorruptionQuarantine:
+ """Verify the verify-failure and load-failure paths preserve
+ corrupted bytes instead of deleting them.
+ """
+
+ def _make_idx(self, tmp_path, name="idx"):
+ idx = tmp_path / f"{name}.faiss"
+ pkl = tmp_path / f"{name}.pkl"
+ idx.write_bytes(b"faiss-bytes")
+ pkl.write_bytes(b"pkl-bytes")
+ return idx, pkl
+
+ def test_renames_both_faiss_and_pkl(self, tmp_path):
+ svc = _make_service()
+ idx, pkl = self._make_idx(tmp_path)
+ svc._quarantine_corrupt_index(idx, "test")
+ # Original files are gone from their paths
+ assert not idx.exists()
+ assert not pkl.exists()
+ # Both quarantined as .corrupt-*
+ corrupted_faiss = list(tmp_path.glob("idx.faiss.corrupt-*"))
+ corrupted_pkl = list(tmp_path.glob("idx.pkl.corrupt-*"))
+ assert len(corrupted_faiss) == 1
+ assert len(corrupted_pkl) == 1
+ assert corrupted_faiss[0].read_bytes() == b"faiss-bytes"
+ assert corrupted_pkl[0].read_bytes() == b"pkl-bytes"
+
+ def test_skips_pkl_when_missing(self, tmp_path):
+ svc = _make_service()
+ idx = tmp_path / "idx.faiss"
+ idx.write_bytes(b"faiss-bytes")
+ # No .pkl present
+ svc._quarantine_corrupt_index(idx, "test")
+ assert not idx.exists()
+ corrupted = list(tmp_path.glob("idx.faiss.corrupt-*"))
+ assert len(corrupted) == 1
+ # No orphan pkl created
+ assert not any(tmp_path.glob("idx.pkl.corrupt-*"))
+
+ def test_rename_oserror_re_raises(self, tmp_path):
+ """Disk-full / permission failure must propagate, not be swallowed."""
+ svc = _make_service()
+ idx, _pkl = self._make_idx(tmp_path)
+ with patch.object(Path, "rename", side_effect=OSError("EROFS")):
+ with pytest.raises(OSError, match="EROFS"):
+ svc._quarantine_corrupt_index(idx, "test")
+
+ def test_collision_increments_suffix(self, tmp_path):
+ """Pre-existing quarantine path → loop increments to -1, -2, ..."""
+ svc = _make_service()
+ idx, pkl = self._make_idx(tmp_path)
+ # Force a specific timestamp so we can predict collision targets
+ with patch.object(
+ _import_module().time,
+ "time_ns",
+ return_value=12345,
+ ):
+ # Pre-create the base collision target
+ (tmp_path / "idx.faiss.corrupt-12345").write_bytes(b"old1")
+ svc._quarantine_corrupt_index(idx, "test")
+ # Original moved out
+ assert not idx.exists()
+ # New corrupt-12345-1 created (the -1 increment)
+ moved = tmp_path / "idx.faiss.corrupt-12345-1"
+ assert moved.exists()
+ assert moved.read_bytes() == b"faiss-bytes"
+
+ def test_collision_cap_raises(self, tmp_path):
+ """If 32 collisions in a row, surface an OSError, don't hang."""
+ svc = _make_service()
+ mod = _import_module()
+ idx, pkl = self._make_idx(tmp_path)
+ with patch.object(mod.time, "time_ns", return_value=999):
+ # Pre-create base + 32 collisions
+ (tmp_path / "idx.faiss.corrupt-999").write_bytes(b"")
+ for n in range(1, mod._QUARANTINE_SUFFIX_RETRY_CAP + 1):
+ (tmp_path / f"idx.faiss.corrupt-999-{n}").write_bytes(b"")
+ with pytest.raises(OSError, match="collisions exceeded"):
+ svc._quarantine_corrupt_index(idx, "test")
+
+
+class TestFaissWriteLock:
+ """Lock infrastructure: identity, parallelism across keys,
+ cleanup on user-close.
+ """
+
+ def _reset_locks(self):
+ mod = _import_module()
+ with mod._faiss_write_locks_lock:
+ mod._faiss_write_locks.clear()
+
+ def test_same_key_returns_same_lock(self, tmp_path):
+ self._reset_locks()
+ mod = _import_module()
+ p = str(tmp_path / "shared.faiss")
+ a = mod._get_faiss_write_lock("u1", p)
+ b = mod._get_faiss_write_lock("u1", p)
+ assert a is b
+
+ def test_different_key_returns_different_lock(self, tmp_path):
+ self._reset_locks()
+ mod = _import_module()
+ a = mod._get_faiss_write_lock("u1", str(tmp_path / "a.faiss"))
+ b = mod._get_faiss_write_lock("u1", str(tmp_path / "b.faiss"))
+ c = mod._get_faiss_write_lock("u2", str(tmp_path / "a.faiss"))
+ assert a is not b
+ assert a is not c
+
+ def test_pop_removes_only_target_user(self, tmp_path):
+ self._reset_locks()
+ mod = _import_module()
+ u1_lock = mod._get_faiss_write_lock("u1", str(tmp_path / "a.faiss"))
+ mod._get_faiss_write_lock("u1", str(tmp_path / "b.faiss"))
+ u2_lock = mod._get_faiss_write_lock("u2", str(tmp_path / "a.faiss"))
+
+ mod.pop_faiss_locks_for_user("u1")
+
+ # u1 locks gone (new lookup yields a fresh lock object)
+ new_u1 = mod._get_faiss_write_lock("u1", str(tmp_path / "a.faiss"))
+ assert new_u1 is not u1_lock
+ # u2 lock untouched
+ same_u2 = mod._get_faiss_write_lock("u2", str(tmp_path / "a.faiss"))
+ assert same_u2 is u2_lock
+
+ def test_concurrent_save_record_serialised_for_same_key(self, tmp_path):
+ """Two threads calling _save_and_record_locked on the same
+ (username, index_path) MUST observe non-overlapping critical
+ sections. Regression for the #4197 race.
+ """
+ self._reset_locks()
+ index_path = tmp_path / "idx.faiss"
+ index_path.touch()
+
+ in_flight = 0
+ max_in_flight = 0
+ observed_overlap = False
+ lock_for_counters = threading.Lock()
+ save_started = threading.Event()
+ save_can_finish = threading.Event()
+
+ def fake_save_local(folder, index_name):
+ nonlocal in_flight, max_in_flight, observed_overlap
+ with lock_for_counters:
+ in_flight += 1
+ if in_flight > max_in_flight:
+ max_in_flight = in_flight
+ if in_flight > 1:
+ observed_overlap = True
+ save_started.set()
+ # Hold the critical section open for the second thread
+ save_can_finish.wait(timeout=2.0)
+ with lock_for_counters:
+ in_flight -= 1
+
+ def fake_record_file(path, **kwargs):
+ return None
+
+ def worker():
+ svc = _make_service()
+ svc.faiss_index = MagicMock()
+ svc.faiss_index.save_local.side_effect = fake_save_local
+ svc.integrity_manager = MagicMock()
+ svc.integrity_manager.record_file.side_effect = fake_record_file
+ svc.rag_index_record = MagicMock(id=42)
+ svc._save_and_record_locked(index_path)
+
+ t1 = threading.Thread(target=worker)
+ t2 = threading.Thread(target=worker)
+ t1.start()
+ # Make sure t1 is inside save_local before t2 starts trying
+ assert save_started.wait(timeout=2.0)
+ t2.start()
+ # Brief moment for t2 to attempt and (hopefully) block on the lock
+ save_can_finish.set()
+ t1.join(timeout=3.0)
+ t2.join(timeout=3.0)
+
+ assert not observed_overlap, (
+ "Two threads ran save_local on the same index concurrently; "
+ "the lock did not serialise them."
+ )
+ assert max_in_flight == 1
+
+ def test_different_keys_run_in_parallel(self, tmp_path):
+ """Two threads with different (username, index_path) keys must
+ be able to run save+record simultaneously.
+ """
+ self._reset_locks()
+ barrier = threading.Barrier(2, timeout=2.0)
+ idx1 = tmp_path / "a.faiss"
+ idx2 = tmp_path / "b.faiss"
+ idx1.touch()
+ idx2.touch()
+
+ def fake_save_local(folder, index_name):
+ # If the lock blocked us, barrier.wait would time out
+ barrier.wait()
+
+ def worker(index_path):
+ svc = _make_service()
+ svc.faiss_index = MagicMock()
+ svc.faiss_index.save_local.side_effect = fake_save_local
+ svc.integrity_manager = MagicMock()
+ svc.rag_index_record = MagicMock(id=99)
+ svc._save_and_record_locked(index_path)
+
+ t1 = threading.Thread(target=worker, args=(idx1,))
+ t2 = threading.Thread(target=worker, args=(idx2,))
+ t1.start()
+ t2.start()
+ t1.join(timeout=3.0)
+ t2.join(timeout=3.0)
+ # If barrier broke (timeout), one of the threads would be alive
+ assert not t1.is_alive()
+ assert not t2.is_alive()
+
+
+class TestNoWalCheckpointInIndexingPath:
+ """Regression guard: the per-document PRAGMA wal_checkpoint(FULL)
+ has been removed (#4197 secondary fix from PR #3539). Re-adding it
+ would re-introduce 'database is locked' errors under concurrent
+ bulk indexing.
+ """
+
+ def test_module_source_has_no_executable_wal_checkpoint(self):
+ import inspect
+
+ mod = _import_module()
+ src = inspect.getsource(mod)
+ # Strip out comment / docstring lines so the regression test
+ # tolerates explanatory references like 'PRAGMA wal_checkpoint'
+ # in comments, but fails if any code line re-adds the call.
+ for lineno, line in enumerate(src.splitlines(), 1):
+ stripped = line.strip()
+ if not stripped:
+ continue
+ if stripped.startswith("#"):
+ continue
+ assert "wal_checkpoint(" not in stripped, (
+ f"library_rag_service.py:{lineno} re-introduced "
+ f"a wal_checkpoint() call: {line!r}"
+ )
+
+
+class TestMergeAndPersistLocked:
+ """Read-modify-write under the FAISS write lock.
+
+ Regression for the AI-review concern on #4200: two workers
+ indexing different documents into the same collection used to
+ both load on-disk state X, each add their own chunk in memory,
+ then save in sequence — last writer wins, the loser's chunks
+ were lost from the FAISS file (chunks survived in the DB so a
+ rebuild recovered them, but the index file was wrong until then).
+
+ The helper reloads from disk under the lock before adding, so
+ the second writer sees the first writer's save and merges
+ instead of overwriting.
+ """
+
+ def _reset_locks(self):
+ mod = _import_module()
+ with mod._faiss_write_locks_lock:
+ mod._faiss_write_locks.clear()
+
+ def test_reload_picks_up_concurrent_writers_save(self, tmp_path):
+ """The merge helper must reload from disk before adding so
+ chunks committed by a concurrent writer between the caller's
+ in-memory load and this lock acquisition are preserved.
+ """
+ self._reset_locks()
+ mod = _import_module()
+ index_path = tmp_path / "idx.faiss"
+ # The on-disk file must exist + verify so the helper actually
+ # takes the reload branch (not "no on-disk version, keep
+ # stale in-memory").
+ index_path.touch()
+
+ svc = _make_service()
+ svc.embedding_manager = MagicMock()
+ svc.integrity_manager = MagicMock()
+ svc.integrity_manager.verify_file.return_value = (True, None)
+ svc.rag_index_record = MagicMock(id=42)
+
+ # Caller's stale in-memory state knows about no chunks.
+ stale_index = MagicMock()
+ stale_index.docstore._dict = {}
+ svc.faiss_index = stale_index
+
+ # Disk state (loaded by the reload step) already has another
+ # writer's chunk "concurrent-id".
+ fresh_index = MagicMock()
+ fresh_index.docstore._dict = {"concurrent-id": MagicMock()}
+
+ new_chunk = LangchainDocument(page_content="our new content")
+
+ with patch.object(mod.FAISS, "load_local", return_value=fresh_index):
+ stats = svc._merge_and_persist_locked(
+ index_path,
+ [new_chunk],
+ ["our-id"],
+ force_reindex=False,
+ )
+
+ # Helper reloaded → svc.faiss_index is now the fresh state
+ assert svc.faiss_index is fresh_index
+ # Our chunk was added to the fresh state (not the stale one)
+ fresh_index.add_documents.assert_called_once()
+ added_ids = fresh_index.add_documents.call_args[1]["ids"]
+ assert added_ids == ["our-id"]
+ # The stale state was NOT mutated (otherwise we'd be saving it)
+ stale_index.add_documents.assert_not_called()
+ # save_local was called on the fresh (merged) state
+ fresh_index.save_local.assert_called_once()
+ assert stats["added"] == 1
+ assert stats["added_ids"] == ["our-id"]
+
+ def test_reload_skips_when_existing_id_now_on_disk(self, tmp_path):
+ """If the caller wanted to add id X but a concurrent writer
+ already saved X to disk, dedup against the fresh state must
+ drop X (idempotent re-index).
+ """
+ self._reset_locks()
+ mod = _import_module()
+ index_path = tmp_path / "idx.faiss"
+ index_path.touch()
+
+ svc = _make_service()
+ svc.embedding_manager = MagicMock()
+ svc.integrity_manager = MagicMock()
+ svc.integrity_manager.verify_file.return_value = (True, None)
+ svc.rag_index_record = MagicMock(id=42)
+ svc.faiss_index = MagicMock() # stale, irrelevant
+
+ # Disk state already has the same id we're about to add.
+ fresh_index = MagicMock()
+ fresh_index.docstore._dict = {"shared-id": MagicMock()}
+
+ with patch.object(mod.FAISS, "load_local", return_value=fresh_index):
+ stats = svc._merge_and_persist_locked(
+ index_path,
+ [LangchainDocument(page_content="duplicate")],
+ ["shared-id"],
+ force_reindex=False,
+ )
+
+ # No add (already on disk); save still happens (idempotent +
+ # touches mtime so the integrity record updates).
+ fresh_index.add_documents.assert_not_called()
+ fresh_index.save_local.assert_called_once()
+ assert stats["added"] == 0
+ assert stats["skipped"] == 1
+
+ def test_force_reindex_deletes_after_reload(self, tmp_path):
+ """force_reindex deletes the matching IDs from the FRESH
+ on-disk state (not the caller's stale in-memory state)
+ before re-adding.
+ """
+ self._reset_locks()
+ mod = _import_module()
+ index_path = tmp_path / "idx.faiss"
+ index_path.touch()
+
+ svc = _make_service()
+ svc.embedding_manager = MagicMock()
+ svc.integrity_manager = MagicMock()
+ svc.integrity_manager.verify_file.return_value = (True, None)
+ svc.rag_index_record = MagicMock(id=42)
+ svc.faiss_index = MagicMock() # stale
+
+ fresh_index = MagicMock()
+ fresh_index.docstore._dict = {"to-update": MagicMock()}
+
+ with patch.object(mod.FAISS, "load_local", return_value=fresh_index):
+ svc._merge_and_persist_locked(
+ index_path,
+ [LangchainDocument(page_content="updated")],
+ ["to-update"],
+ force_reindex=True,
+ )
+
+ # Old copy removed from FRESH state
+ fresh_index.delete.assert_called_once_with(["to-update"])
+ # New copy added (no dedup under force_reindex)
+ fresh_index.add_documents.assert_called_once()
+ added_ids = fresh_index.add_documents.call_args[1]["ids"]
+ assert added_ids == ["to-update"]
+
+ def test_reload_failure_falls_back_to_in_memory(self, tmp_path):
+ """If FAISS.load_local raises (torn write, partial pickle),
+ the merge helper keeps the caller's in-memory state instead
+ of losing the write entirely.
+ """
+ self._reset_locks()
+ mod = _import_module()
+ index_path = tmp_path / "idx.faiss"
+ index_path.touch()
+
+ svc = _make_service()
+ svc.embedding_manager = MagicMock()
+ svc.integrity_manager = MagicMock()
+ svc.integrity_manager.verify_file.return_value = (True, None)
+ svc.rag_index_record = MagicMock(id=42)
+
+ in_memory = MagicMock()
+ in_memory.docstore._dict = {}
+ svc.faiss_index = in_memory
+
+ with patch.object(
+ mod.FAISS, "load_local", side_effect=RuntimeError("torn pickle")
+ ):
+ svc._merge_and_persist_locked(
+ index_path,
+ [LangchainDocument(page_content="kept")],
+ ["kept-id"],
+ force_reindex=False,
+ )
+
+ # Reload failed → kept the in-memory object, added to it,
+ # saved it. The write is preserved.
+ assert svc.faiss_index is in_memory
+ in_memory.add_documents.assert_called_once()
+ in_memory.save_local.assert_called_once()
+
+ def test_concurrent_writers_both_chunks_survive(self, tmp_path):
+ """End-to-end regression: two workers each call
+ ``_merge_and_persist_locked`` for the same on-disk index.
+ Each starts from a stale in-memory snapshot. After both
+ complete, the on-disk state must contain BOTH chunks.
+
+ Before the read-modify-write fix, this was last-writer-wins:
+ worker A's chunk was overwritten by worker B's save because
+ B never reloaded A's write.
+ """
+ self._reset_locks()
+ mod = _import_module()
+ index_path = tmp_path / "shared.faiss"
+ index_path.touch()
+
+ # Single shared "disk state" as a dict — proxies for what
+ # would be on disk via FAISS.load_local/save_local.
+ disk_state: dict = {}
+ disk_lock = threading.Lock()
+
+ def make_index_proxy():
+ """Each call returns a FAISS-like proxy reading/writing
+ the shared disk_state. Mimics a fresh load_local()."""
+ ix = MagicMock()
+ # Snapshot disk_state at load time (proxy for what
+ # FAISS.load_local would return).
+ snapshot = dict(disk_state)
+ ix.docstore = MagicMock()
+ ix.docstore._dict = snapshot
+
+ def add(chunks, ids):
+ for cid in ids:
+ snapshot[cid] = "doc"
+
+ def save(_folder, index_name=None):
+ # Atomically replace disk_state with this index's
+ # current snapshot — mirrors save_local's truncating
+ # semantics for the test.
+ with disk_lock:
+ disk_state.clear()
+ disk_state.update(snapshot)
+
+ ix.add_documents.side_effect = add
+ ix.save_local.side_effect = save
+ return ix
+
+ def worker(cid: str, start_event: threading.Event):
+ svc = _make_service()
+ svc.embedding_manager = MagicMock()
+ svc.integrity_manager = MagicMock()
+ svc.integrity_manager.verify_file.return_value = (True, None)
+ svc.rag_index_record = MagicMock(id=99)
+ # Pre-merge in-memory state is stale (empty)
+ svc.faiss_index = MagicMock()
+ svc.faiss_index.docstore = MagicMock()
+ svc.faiss_index.docstore._dict = {}
+ start_event.wait() # Race start
+ with patch.object(
+ mod.FAISS,
+ "load_local",
+ side_effect=lambda *a, **kw: make_index_proxy(),
+ ):
+ svc._merge_and_persist_locked(
+ index_path,
+ [LangchainDocument(page_content=f"chunk-{cid}")],
+ [cid],
+ force_reindex=False,
+ )
+
+ go = threading.Event()
+ t1 = threading.Thread(target=worker, args=("A", go))
+ t2 = threading.Thread(target=worker, args=("B", go))
+ t1.start()
+ t2.start()
+ go.set()
+ t1.join(timeout=5.0)
+ t2.join(timeout=5.0)
+
+ # Both writers' chunks present — read-modify-write under the
+ # lock did its job. Without the fix, only one of {A, B} would
+ # remain.
+ assert "A" in disk_state, (
+ f"Worker A's chunk lost; disk_state={disk_state}. "
+ f"Read-modify-write race regressed."
+ )
+ assert "B" in disk_state, (
+ f"Worker B's chunk lost; disk_state={disk_state}. "
+ f"Read-modify-write race regressed."
+ )
diff --git a/tests/web/auth/test_connection_cleanup.py b/tests/web/auth/test_connection_cleanup.py
index 4cca97bb60..199d91ea46 100644
--- a/tests/web/auth/test_connection_cleanup.py
+++ b/tests/web/auth/test_connection_cleanup.py
@@ -283,13 +283,14 @@ class TestCleanupIdleConnections:
class TestPopPerUserLocks:
- """Tests for ``_pop_per_user_locks``: drops the three module-level
+ """Tests for ``_pop_per_user_locks``: drops the four module-level
per-user lock-dict entries (library-init, backup, queue-processor
- critical sections) on user-close. Without this each dict grew one
- entry per username over process lifetime.
+ critical sections, library-RAG FAISS-write locks) on user-close.
+ Without this each dict grew one entry per username over process
+ lifetime.
"""
- def test_pops_all_three_lock_dicts(self):
+ def test_pops_all_four_lock_dicts(self):
from local_deep_research.web.auth.connection_cleanup import (
_pop_per_user_locks,
)
@@ -304,6 +305,10 @@ class TestPopPerUserLocks:
from local_deep_research.web.queue.processor_v2 import (
queue_processor,
)
+ from local_deep_research.research_library.services.library_rag_service import (
+ _faiss_write_locks,
+ _get_faiss_write_lock,
+ )
# Populate each dict with a unique test username so we don't
# collide with state any other test might have left behind.
@@ -311,16 +316,19 @@ class TestPopPerUserLocks:
_get_user_init_lock(u)
_get_user_lock(u)
queue_processor._get_user_critical_lock(u)
+ _get_faiss_write_lock(u, "/tmp/test-pop-locks/idx.faiss")
assert u in _user_init_locks
assert u in _user_locks
assert u in queue_processor._user_critical_locks
+ assert any(k[0] == u for k in _faiss_write_locks)
_pop_per_user_locks(u)
assert u not in _user_init_locks
assert u not in _user_locks
assert u not in queue_processor._user_critical_locks
+ assert not any(k[0] == u for k in _faiss_write_locks)
def test_idempotent_on_missing_user(self):
"""Pop on a username that was never inserted must not raise."""
@@ -334,8 +342,8 @@ class TestPopPerUserLocks:
def test_pop_called_from_idle_close_path(self, sm, db):
"""Integration: ``cleanup_idle_connections`` calls
``_pop_per_user_locks`` for each user it closes — verified
- by asserting all three lock-dict entries are gone, so a
- regression that drops the call (or breaks one of the three
+ by asserting all four lock-dict entries are gone, so a
+ regression that drops the call (or breaks one of the four
pop hops) would be caught.
"""
from local_deep_research.database.library_init import (
@@ -349,6 +357,10 @@ class TestPopPerUserLocks:
from local_deep_research.web.queue.processor_v2 import (
queue_processor,
)
+ from local_deep_research.research_library.services.library_rag_service import (
+ _faiss_write_locks,
+ _get_faiss_write_lock,
+ )
# Use a dedicated test username (not "alice") to avoid colliding
# with other tests that may also touch these module-level dicts.
@@ -356,10 +368,12 @@ class TestPopPerUserLocks:
_get_user_init_lock(u)
_get_user_lock(u)
queue_processor._get_user_critical_lock(u)
+ _get_faiss_write_lock(u, "/tmp/test-idle-close/idx.faiss")
assert u in _user_init_locks
assert u in _user_locks
assert u in queue_processor._user_critical_locks
+ assert any(k[0] == u for k in _faiss_write_locks)
db.get_connected_usernames.return_value = {u}
@@ -374,10 +388,11 @@ class TestPopPerUserLocks:
):
cleanup_idle_connections(sm, db)
- # All three lock-dict entries removed by the idle-close path.
+ # All four lock-dict entries removed by the idle-close path.
assert u not in _user_init_locks
assert u not in _user_locks
assert u not in queue_processor._user_critical_locks
+ assert not any(k[0] == u for k in _faiss_write_locks)
def test_pop_runs_even_when_close_user_database_fails(self, sm, db):
"""Regression for the original PR: ``_pop_per_user_locks`` was