~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
fix(library): stop deleting user FAISS index on integrity mismatch; serialise concurrent indexers (#4197) (#4200)
LearningCircuit014bdf441b2a
diff --git a/changelog.d/+faiss-merge-write-race.bugfix.md b/changelog.d/+faiss-merge-write-race.bugfix.mdnew file mode 100644index 0000000000..a6232d247f--- /dev/null+++ b/changelog.d/+faiss-merge-write-race.bugfix.md@@ -0,0 +1 @@+Closed a follow-up race condition in the library RAG indexer where two concurrent workers indexing different documents into the same collection could lose each other's embeddings from the FAISS file — last writer's `save_local` overwrote the earlier writer's chunks (the chunks survived in the DB, but the index file was missing them until a force-reindex rebuilt it). The save path now reloads from disk under the per-`(user, index_path)` lock before adding, so concurrent writers' chunks are merged instead of overwriting each other. Follow-up to #4197/#4200.diff --git a/changelog.d/+faiss-quarantine-and-lock.bugfix.md b/changelog.d/+faiss-quarantine-and-lock.bugfix.mdnew file mode 100644index 0000000000..37eab39631--- /dev/null+++ b/changelog.d/+faiss-quarantine-and-lock.bugfix.md@@ -0,0 +1 @@+Fixed a data-loss bug where the library RAG index file (`.faiss`) was silently deleted when integrity verification failed during concurrent auto-indexing — destroying hundreds of previously-indexed documents in one go. Corrupted index files are now quarantined to `<path>.corrupt-<ns>` so they remain recoverable, and concurrent indexers are serialised by a per-`(user, index_path)` lock so the race that produced the checksum mismatch no longer occurs. Also drops the per-document `PRAGMA wal_checkpoint(FULL)` that contributed to `database is locked` errors under bulk-download concurrency. Fixes #4197.diff --git a/src/local_deep_research/research_library/services/library_rag_service.py b/src/local_deep_research/research_library/services/library_rag_service.pyindex 40169873f5..1e691fb5d2 100644--- a/src/local_deep_research/research_library/services/library_rag_service.py+++ b/src/local_deep_research/research_library/services/library_rag_service.py@@ -9,6 +9,8 @@ Handles indexing and searching library documents using RAG:- Track RAG status in library"""+import threading+import timefrom pathlib import Pathfrom typing import Any, Dict, List, Optional, Tuple@@ -40,6 +42,55 @@ from langchain_community.vectorstores import FAISSfrom langchain_community.docstore.in_memory import InMemoryDocstore+# Module-level locks serialise the FAISS save+record critical section and the+# load_or_create verify→quarantine→build sequence per (username, index_path).+# MUST be module-level: each auto-index / scheduler / search worker constructs+# its own LibraryRAGService, so an instance-scoped lock would coordinate+# nothing. Pattern is adapted from web/queue/processor_v2._user_critical_locks+# — instance-scoped→module-scoped, username-only-key→(username, path)-key.+# See #4197 for the race this guards (concurrent save_local interleaves bytes,+# producing checksum_mismatch → destructive unlink, lost data).+_faiss_write_locks: Dict[Tuple[str, str], threading.Lock] = {}+_faiss_write_locks_lock = threading.Lock()++# Hard cap on suffix-increment retries when generating the .corrupt-<ns> path.+# Normal case is one attempt — same-ns collisions only happen if the user+# manually created such files. 32 is a safety bound that converts a deadlock+# into a loud OSError.+_QUARANTINE_SUFFIX_RETRY_CAP = 32+++def _get_faiss_write_lock(username: str, index_path: str) -> threading.Lock:+ """Return the lock for ``(username, index_path)``, creating it on first+ access. Key is normalised via ``Path.resolve()`` to match+ ``FileIntegrityManager._normalize_path`` so writers/readers/quarantine+ all agree on identity.+ """+ key = (username, str(Path(index_path).resolve()))+ with _faiss_write_locks_lock:+ lock = _faiss_write_locks.get(key)+ if lock is None:+ lock = threading.Lock()+ _faiss_write_locks[key] = lock+ return lock+++def pop_faiss_locks_for_user(username: str) -> None:+ """Remove all FAISS-write locks belonging to ``username``.++ Called from the user-close paths (connection_cleanup) so the dict+ doesn't grow one entry per (user × collection) across the process+ lifetime. Safe to call while a lock is held: the holder keeps using+ its local reference, and the next access lazily creates a fresh+ lock. Same semantics as ``pop_user_critical_lock`` in+ web/queue/processor_v2.py.+ """+ with _faiss_write_locks_lock:+ stale = [k for k in _faiss_write_locks if k[0] == username]+ for k in stale:+ _faiss_write_locks.pop(k, None)++class LibraryRAGService:"""Service for managing RAG indexing of library documents."""@@ -297,6 +348,206 @@ class LibraryRAGService:return rag_index+ def _quarantine_corrupt_index(self, index_path: Path, reason: str) -> None:+ """Rename a corrupted FAISS index and its ``.pkl`` companion to+ ``<path>.corrupt-<ns>`` instead of deleting them.++ Preserves user data for inspection/recovery. The+ dimension-mismatch branch elsewhere in this method intentionally+ *deletes* — that case rebuilds from scratch and the old bytes+ are unreadable with the new model. This helper is for the two+ "transient or unknown failure" branches (verify_file said no,+ load_local raised) where the on-disk bytes may still be+ usable by a human.++ Raises ``OSError`` on rename failure (disk full, read-only fs,+ permission denied). Re-raising prevents silent data loss: if we+ swallowed the error, the next ``save_local`` would truncate the+ corrupt bytes anyway. Caller paths log the exception via the+ broader try/except around their indexer call.++ # TODO(#4197-followup): FileIntegrityRecord.consecutive_failures+ # is not reset by the next record_file call, leaking failure+ # counts across recovery cycles. Orthogonal to this fix.+ """+ ns = time.time_ns()+ pkl_path = index_path.with_suffix(".pkl")++ # Same-nanosecond collisions essentially can't happen between+ # concurrent threads (we hold the per-path lock), but a user+ # could manually create such files. Loop with a hard cap so a+ # weird state surfaces as a clean OSError rather than a hang.+ suffix_n = 0+ faiss_target = Path(f"{index_path}.corrupt-{ns}")+ pkl_target = Path(f"{pkl_path}.corrupt-{ns}")+ while faiss_target.exists() or pkl_target.exists():+ suffix_n += 1+ if suffix_n > _QUARANTINE_SUFFIX_RETRY_CAP:+ raise OSError(+ f"Quarantine path collisions exceeded "+ f"{_QUARANTINE_SUFFIX_RETRY_CAP} retries for "+ f"{index_path}"+ )+ faiss_target = Path(f"{index_path}.corrupt-{ns}-{suffix_n}")+ pkl_target = Path(f"{pkl_path}.corrupt-{ns}-{suffix_n}")++ try:+ index_path.rename(faiss_target)+ logger.warning(+ f"Quarantined corrupted FAISS index to {faiss_target} "+ f"(reason: {reason})"+ )+ if pkl_path.exists():+ pkl_path.rename(pkl_target)+ logger.info(f"Quarantined companion PKL to {pkl_target}")+ else:+ # Missing .pkl is recoverable (FAISS may have crashed+ # mid-write between .faiss and .pkl). Not re-raised:+ # the .faiss is already preserved, fresh build will+ # write a fresh .pkl.+ logger.warning(+ f"PKL companion missing at {pkl_path}; only "+ f".faiss quarantined."+ )+ except OSError:+ # Disk-full, read-only fs, or permission error. Re-raise+ # so the caller surfaces a real failure instead of letting+ # the next save_local truncate the corrupt bytes.+ logger.exception(+ f"Failed to quarantine corrupted index at {index_path}"+ )+ raise++ def _save_and_record_locked(self, index_path: Path) -> None:+ """Atomically save the in-memory FAISS index to ``index_path``+ and record its checksum in the integrity manager.++ The save_local + record_file pair must be serialised across+ concurrent indexers for the same ``(username, index_path)`` so+ a second writer doesn't overwrite the file between the first+ writer's save and checksum recording — that race produces the+ checksum_mismatch / silent-deletion bug in #4197.+ """+ with _get_faiss_write_lock(self.username, str(index_path)):+ self.faiss_index.save_local(+ str(index_path.parent), index_name=index_path.stem+ )+ self.integrity_manager.record_file(+ index_path,+ related_entity_type="rag_index",+ related_entity_id=self.rag_index_record.id,+ )++ def _merge_and_persist_locked(+ self,+ index_path: Path,+ chunks_to_add: list,+ embedding_ids: list,+ force_reindex: bool = False,+ ) -> Dict[str, int]:+ """Read-modify-write the on-disk FAISS index under the+ ``(username, index_path)`` write lock so concurrent workers+ don't lose each other's embeddings.++ Without this, two workers indexing different documents into+ the same collection both load on-disk state X into memory,+ each calls ``add_documents`` on their own private FAISS object+ (worker A: ``X+docA``, worker B: ``X+docB``), then they save+ in sequence — last writer wins, the loser's chunks are gone+ from the FAISS file. The chunks survive in the per-document+ DB rows so a force-reindex rebuilds, but the index file is+ wrong until then. See AI review of #4200.++ The slow embedding + DB-chunk-insert path stays OUTSIDE this+ lock (handled by the caller). The lock only covers the fast+ FAISS reload→merge→save sequence (~10-100ms even for large+ indices), so cross-document parallelism is preserved up to+ the point where the in-memory states need reconciliation.++ Args:+ index_path: Path of the on-disk ``.faiss`` file.+ chunks_to_add: Chunks the caller wants to add.+ embedding_ids: Matching embedding/chunk IDs (same length).+ force_reindex: If True, delete any IDs from+ ``embedding_ids`` that already exist in the index+ (so the caller's metadata wins). If False, skip IDs+ that already exist (idempotent re-indexing).++ Returns:+ ``{"added": n, "skipped": m}`` for caller logging.+ """+ with _get_faiss_write_lock(self.username, str(index_path)):+ # Reload from disk to absorb other writers' saves. If the+ # file doesn't exist or fails verification, keep whatever+ # the caller already had in ``self.faiss_index`` (likely+ # a fresh in-memory index from load_or_create_faiss_index).+ if index_path.exists():+ verified, _reason = self.integrity_manager.verify_file(+ index_path+ )+ if verified:+ try:+ self.faiss_index = FAISS.load_local(+ str(index_path.parent),+ self.embedding_manager.embeddings,+ index_name=index_path.stem,+ allow_dangerous_deserialization=True,+ normalize_L2=True,+ )+ except Exception:+ # Reload failed (torn write, etc.). Keep+ # in-memory state — it's stale but valid;+ # better than losing this write entirely.+ logger.warning(+ "Failed to reload FAISS for merge; "+ "proceeding with in-memory state."+ )++ # Force-reindex: remove old copies of IDs we're about to+ # re-add, so updated metadata replaces stale. Dedup via+ # set→list because ``embedding_ids`` can contain repeats+ # (same chunk hash appearing twice in the document) —+ # FAISS.delete with duplicate IDs raises.+ if force_reindex and hasattr(self.faiss_index, "docstore"):+ fresh_ids = set(self.faiss_index.docstore._dict.keys())+ old_chunk_ids = list(+ {eid for eid in embedding_ids if eid in fresh_ids}+ )+ if old_chunk_ids:+ logger.info(+ f"Force re-index: removing {len(old_chunk_ids)} "+ f"existing chunks from FAISS"+ )+ self.faiss_index.delete(old_chunk_ids)++ # Dedup against the freshly-loaded state.+ if not force_reindex and hasattr(self.faiss_index, "docstore"):+ fresh_ids = set(self.faiss_index.docstore._dict.keys())+ else:+ fresh_ids = None++ new_chunks, new_ids = self._deduplicate_chunks(+ chunks_to_add, embedding_ids, fresh_ids+ )++ if new_chunks:+ self.faiss_index.add_documents(new_chunks, ids=new_ids)++ self.faiss_index.save_local(+ str(index_path.parent), index_name=index_path.stem+ )+ self.integrity_manager.record_file(+ index_path,+ related_entity_type="rag_index",+ related_entity_id=self.rag_index_record.id,+ )++ return {+ "added": len(new_chunks),+ "skipped": len(chunks_to_add) - len(new_chunks),+ "added_ids": new_ids,+ }+def load_or_create_faiss_index(self, collection_id: str) -> FAISS:"""Load existing FAISS index or create new one.@@ -312,97 +563,117 @@ class LibraryRAGService:index_path = Path(rag_index.index_path)+ # Hold the per-(username, index_path) write lock across the+ # entire verify → quarantine → load sequence. A narrower scope+ # leaves room for a concurrent save_local to race the+ # verification (verify sees bytes A, save overwrites with bytes+ # B, load_local reads bytes B which no longer match verified+ # checksum). See #4197. The fresh-build path below this block+ # is in-memory only and doesn't need the lock.if index_path.exists():- # Verify integrity before loading- verified, reason = self.integrity_manager.verify_file(index_path)- if not verified:- logger.error(- f"Integrity verification failed for {index_path}: {reason}. "- f"Refusing to load. Creating new index."+ load_lock = _get_faiss_write_lock(self.username, str(index_path))+ with load_lock:+ # Verify integrity before loading+ verified, reason = self.integrity_manager.verify_file(+ index_path)- # Remove corrupted index- try:- index_path.unlink()- logger.info(f"Removed corrupted index file: {index_path}")- except Exception:- logger.exception("Failed to remove corrupted index")- else:- try:- # Check for embedding dimension mismatch before loading- current_dim = len(- self.embedding_manager.embeddings.embed_query(- "dimension_check"- )+ if not verified:+ logger.error(+ f"Integrity verification failed for {index_path}: "+ f"{reason}. Quarantining for recovery; creating "+ f"new index.")- stored_dim = rag_index.embedding_dimension-- if stored_dim and current_dim != stored_dim:- logger.warning(- f"Embedding dimension mismatch detected! "- f"Index created with dim={stored_dim}, "- f"current model returns dim={current_dim}. "- f"Deleting old index and rebuilding."- )- # Delete old index files- try:- index_path.unlink()- pkl_path = index_path.with_suffix(".pkl")- if pkl_path.exists():- pkl_path.unlink()- logger.info(- f"Deleted old FAISS index files at {index_path}"+ self._quarantine_corrupt_index(index_path, reason)+ else:+ try:+ # Check for embedding dimension mismatch before loading+ current_dim = len(+ self.embedding_manager.embeddings.embed_query(+ "dimension_check")- except Exception:- logger.exception("Failed to delete old index files")-- # Update RAGIndex with new dimension and reset counts- with get_user_db_session(- self.username, self.db_password- ) as session:- idx = (- session.query(RAGIndex)- .filter_by(id=rag_index.id)- .first()+ )+ stored_dim = rag_index.embedding_dimension++ if stored_dim and current_dim != stored_dim:+ logger.warning(+ f"Embedding dimension mismatch detected! "+ f"Index created with dim={stored_dim}, "+ f"current model returns dim={current_dim}. "+ f"Deleting old index and rebuilding.")- if idx:- idx.embedding_dimension = current_dim- idx.chunk_count = 0- idx.total_documents = 0+ # Delete old index files (legitimate deletion:+ # the bytes are unreadable with the new model).+ try:+ index_path.unlink()+ pkl_path = index_path.with_suffix(".pkl")+ if pkl_path.exists():+ pkl_path.unlink()+ logger.info(+ f"Deleted old FAISS index files at {index_path}"+ )+ except Exception:+ logger.exception(+ "Failed to delete old index files"+ )++ # Update RAGIndex with new dimension and reset counts+ with get_user_db_session(+ self.username, self.db_password+ ) as session:+ idx = (+ session.query(RAGIndex)+ .filter_by(id=rag_index.id)+ .first()+ )+ if idx:+ idx.embedding_dimension = current_dim+ idx.chunk_count = 0+ idx.total_documents = 0+ session.commit()+ logger.info(+ f"Updated RAGIndex dimension to {current_dim}"+ )++ # Clear rag_document_status for this index+ session.query(RagDocumentStatus).filter_by(+ rag_index_id=rag_index.id+ ).delete()session.commit()logger.info(- f"Updated RAGIndex dimension to {current_dim}"+ "Cleared indexed status for documents in this "+ "collection")- # Clear rag_document_status for this index- session.query(RagDocumentStatus).filter_by(- rag_index_id=rag_index.id- ).delete()- session.commit()+ # Update local reference for index creation below+ rag_index.embedding_dimension = current_dim+ # Fall through to create new index below+ else:+ # Dimensions match (or no stored dimension), load index+ faiss_index = FAISS.load_local(+ str(index_path.parent),+ self.embedding_manager.embeddings,+ index_name=index_path.stem,+ allow_dangerous_deserialization=True,+ normalize_L2=True,+ )logger.info(- "Cleared indexed status for documents in this "- "collection"+ f"Loaded existing FAISS index from {index_path}")-- # Update local reference for index creation below- rag_index.embedding_dimension = current_dim- # Fall through to create new index below- else:- # Dimensions match (or no stored dimension), load index- faiss_index = FAISS.load_local(- str(index_path.parent),- self.embedding_manager.embeddings,- index_name=index_path.stem,- allow_dangerous_deserialization=True,- normalize_L2=True,- )- logger.info(- f"Loaded existing FAISS index from {index_path}"+ return faiss_index+ except Exception:+ # load_local raised (torn .pkl, malformed pickle,+ # FAISS read failure). The .faiss bytes may still+ # be recoverable, so quarantine before falling+ # through to fresh build — don't just leave+ # broken state on disk.+ logger.warning(+ "Failed to load FAISS index, quarantining "+ "and creating new one")- return faiss_index- except Exception:- logger.warning(- "Failed to load FAISS index, creating new one"- )+ if index_path.exists():+ self._quarantine_corrupt_index(+ index_path, "load_local_raised"+ )# Create new FAISS index with configurable type and distance metriclogger.info(@@ -615,80 +886,50 @@ class LibraryRAGService:source_id=document_id,)- # Load or create FAISS index+ # Load or create FAISS index (lazy; the merge step+ # below will reload from disk under the lock anyway).if self.faiss_index is None:self.faiss_index = self.load_or_create_faiss_index(collection_id)- # If force_reindex, remove old chunks from FAISS before adding new ones- if force_reindex:- existing_ids = (- set(self.faiss_index.docstore._dict.keys())- if hasattr(self.faiss_index, "docstore")- else set()- )- old_chunk_ids = list(- {eid for eid in embedding_ids if eid in existing_ids}- )- if old_chunk_ids:- logger.info(- f"Force re-index: removing {len(old_chunk_ids)} existing chunks from FAISS"- )- self.faiss_index.delete(old_chunk_ids)-- # Filter out chunks that already exist in FAISS (unless force_reindex)- if not force_reindex:- existing_ids = (- set(self.faiss_index.docstore._dict.keys())- if hasattr(self.faiss_index, "docstore")- else set()- )- else:- existing_ids = None-+ # Read-modify-write the on-disk FAISS index under+ # the per-(user, index_path) lock. The lock spans+ # reload + dedup + add + save so concurrent indexers+ # of different documents into the same collection+ # don't lose each other's embeddings (see AI review+ # of #4200).+ index_path = Path(self.rag_index_record.index_path)unique_count = len(set(embedding_ids))batch_dups = len(chunks) - unique_count-- new_chunks, new_ids = self._deduplicate_chunks(- chunks, embedding_ids, existing_ids+ merge_stats = self._merge_and_persist_locked(+ index_path,+ chunks,+ embedding_ids,+ force_reindex=force_reindex,)-- # Add embeddings to FAISS index- if new_chunks:+ if merge_stats["added"]:if force_reindex:logger.info(- f"Force re-index: adding {len(new_chunks)} chunks with updated metadata to FAISS index"+ f"Force re-index: added {merge_stats['added']} "+ f"chunks with updated metadata to FAISS index")else:- already_exist = unique_count - len(new_chunks)+ already_exist = unique_count - merge_stats["added"]logger.info(- f"Adding {len(new_chunks)} new embeddings to FAISS index "- f"({already_exist} already exist, {batch_dups} batch duplicates removed)"+ f"Added {merge_stats['added']} new embeddings to FAISS "+ f"({already_exist} already exist, "+ f"{batch_dups} batch duplicates removed)")- self.faiss_index.add_documents(new_chunks, ids=new_ids)else:logger.info(f"All {len(chunks)} chunks already exist in FAISS index, skipping")-- # Save FAISS index- index_path = Path(self.rag_index_record.index_path)- self.faiss_index.save_local(- str(index_path.parent), index_name=index_path.stem- )- # Record file integrity- self.integrity_manager.record_file(- index_path,- related_entity_type="rag_index",- related_entity_id=self.rag_index_record.id,- )logger.info(f"Saved FAISS index to {index_path} with integrity tracking")from datetime import datetime, UTC- from sqlalchemy import text# Check if document was already indexed (for stats update)existing_status = (@@ -751,12 +992,14 @@ class LibraryRAGService:session.flush()logger.info(f"Flushed ORM changes for document {document_id}")- # Commit the transaction+ # Commit the transaction. Durability is provided by+ # synchronous=NORMAL (sqlcipher_utils.py); SQLite+ # auto-checkpoints WAL at wal_autocheckpoint=250 frames.+ # An explicit PRAGMA wal_checkpoint(FULL) here used to+ # block other writers long enough to exhaust busy_timeout+ # under bulk-download concurrency (#4197).session.commit()- # WAL checkpoint after commit to ensure persistence- session.execute(text("PRAGMA wal_checkpoint(FULL)"))-logger.info(f"Successfully indexed document {document_id} for collection {collection_id} "f"with {len(chunks)} chunks"@@ -1198,6 +1441,7 @@ class LibraryRAGService:)# Load or create FAISS index using default library collection+ # (lazy; merge step below reloads under the lock anyway).if self.faiss_index is None:from ...database.library_init import get_default_library_id@@ -1208,52 +1452,50 @@ class LibraryRAGService:default_collection_id)- # Filter out chunks that already exist in FAISS and deduplicate- if self.faiss_index is not None:- existing_ids = (- set(self.faiss_index.docstore._dict.keys())- if hasattr(self.faiss_index, "docstore")- else set()- )- else:- existing_ids = None- new_chunks, new_ids = self._deduplicate_chunks(- chunks, embedding_ids, existing_ids- )-- # Add embeddings to FAISS index- if new_chunks:- self.faiss_index.add_documents(new_chunks, ids=new_ids)-- # Save FAISS index+ # Read-modify-write the on-disk FAISS under the lock.index_path = (Path(self.rag_index_record.index_path)if self.rag_index_recordelse None)if index_path:- self.faiss_index.save_local(- str(index_path.parent), index_name=index_path.stem- )- # Record file integrity- self.integrity_manager.record_file(- index_path,- related_entity_type="rag_index",- related_entity_id=self.rag_index_record.id,+ merge_stats = self._merge_and_persist_locked(+ index_path, chunks, embedding_ids, force_reindex=False)logger.info(f"Saved FAISS index to {index_path} with integrity tracking")+ else:+ # No persistent index path — in-memory only path.+ # Preserve old behavior: dedup against in-memory state+ # and add directly without saving.+ if self.faiss_index is not None and hasattr(+ self.faiss_index, "docstore"+ ):+ existing_ids = set(self.faiss_index.docstore._dict.keys())+ else:+ existing_ids = None+ new_chunks, new_ids = self._deduplicate_chunks(+ chunks, embedding_ids, existing_ids+ )+ if new_chunks:+ self.faiss_index.add_documents(new_chunks, ids=new_ids)+ merge_stats = {+ "added": len(new_chunks),+ "skipped": len(chunks) - len(new_chunks),+ "added_ids": new_ids,+ }logger.info(- f"Successfully indexed local file {file_path} with {len(new_chunks)} new chunks "- f"({len(chunks) - len(new_chunks)} skipped)"+ f"Successfully indexed local file {file_path} with "+ f"{merge_stats['added']} new chunks "+ f"({merge_stats['skipped']} skipped)")return {"status": "success",- "chunk_count": len(new_chunks),- "embedding_ids": new_ids,+ "chunk_count": merge_stats["added"],+ "embedding_ids": merge_stats["added_ids"],}except Exception as e:@@ -1316,7 +1558,8 @@ class LibraryRAGService:source_id=user_doc.id,)- # Load or create FAISS index for this collection+ # Load or create FAISS index for this collection (lazy;+ # merge step below reloads under the lock anyway).if self.faiss_index is None:# Extract collection_id from collection_name (format: "collection_<uuid>")collection_id = collection_name.removeprefix("collection_")@@ -1324,74 +1567,71 @@ class LibraryRAGService:collection_id)- # If force_reindex, remove old chunks from FAISS before adding new ones- if force_reindex:- existing_ids = (- set(self.faiss_index.docstore._dict.keys())- if hasattr(self.faiss_index, "docstore")- else set()- )- old_chunk_ids = list(- {eid for eid in embedding_ids if eid in existing_ids}- )- if old_chunk_ids:- logger.info(- f"Force re-index: removing {len(old_chunk_ids)} existing chunks from FAISS"- )- self.faiss_index.delete(old_chunk_ids)-- # Filter out chunks that already exist in FAISS (unless force_reindex)- if not force_reindex:- existing_ids = (- set(self.faiss_index.docstore._dict.keys())- if hasattr(self.faiss_index, "docstore")- else set()- )- else:- existing_ids = None-unique_count = len(set(embedding_ids))batch_dups = len(chunks) - unique_count- new_chunks, new_ids = self._deduplicate_chunks(- chunks, embedding_ids, existing_ids+ # Read-modify-write the on-disk FAISS under the lock so+ # concurrent uploads to the same collection don't lose+ # each other's chunks.+ index_path = (+ Path(self.rag_index_record.index_path)+ if self.rag_index_record+ else None)-- # Add embeddings to FAISS index- if new_chunks:+ if index_path:+ merge_stats = self._merge_and_persist_locked(+ index_path,+ chunks,+ embedding_ids,+ force_reindex=force_reindex,+ )+ else:+ # No persistent index path — in-memory only path.+ # Preserve old behavior: handle force_reindex deletion+ # and dedup add against in-memory state without saving.+ if force_reindex and hasattr(self.faiss_index, "docstore"):+ existing_ids = set(self.faiss_index.docstore._dict.keys())+ old_chunk_ids = list(+ {eid for eid in embedding_ids if eid in existing_ids}+ )+ if old_chunk_ids:+ logger.info(+ f"Force re-index: removing {len(old_chunk_ids)} "+ f"existing chunks from FAISS"+ )+ self.faiss_index.delete(old_chunk_ids)+ if not force_reindex and hasattr(self.faiss_index, "docstore"):+ existing_ids = set(self.faiss_index.docstore._dict.keys())+ else:+ existing_ids = None+ new_chunks, new_ids = self._deduplicate_chunks(+ chunks, embedding_ids, existing_ids+ )+ if new_chunks:+ self.faiss_index.add_documents(new_chunks, ids=new_ids)+ merge_stats = {+ "added": len(new_chunks),+ "skipped": len(chunks) - len(new_chunks),+ "added_ids": new_ids,+ }+ if merge_stats["added"]:if force_reindex:logger.info(- f"Force re-index: adding {len(new_chunks)} chunks with updated metadata to FAISS index"+ f"Force re-index: added {merge_stats['added']} "+ f"chunks with updated metadata to FAISS index")else:- already_exist = unique_count - len(new_chunks)+ already_exist = unique_count - merge_stats["added"]logger.info(- f"Adding {len(new_chunks)} new chunks to FAISS index "- f"({already_exist} already exist, {batch_dups} batch duplicates removed)"+ f"Added {merge_stats['added']} new chunks to FAISS "+ f"({already_exist} already exist, "+ f"{batch_dups} batch duplicates removed)")- self.faiss_index.add_documents(new_chunks, ids=new_ids)else:logger.info(f"All {len(chunks)} chunks already exist in FAISS index, skipping")- # Save FAISS index- index_path = (- Path(self.rag_index_record.index_path)- if self.rag_index_record- else None- )- if index_path:- self.faiss_index.save_local(- str(index_path.parent), index_name=index_path.stem- )- # Record file integrity- self.integrity_manager.record_file(- index_path,- related_entity_type="rag_index",- related_entity_id=self.rag_index_record.id,- )-logger.info(f"Successfully indexed user document {user_doc.filename} with {len(chunks)} chunks")@@ -1452,28 +1692,34 @@ class LibraryRAGService:collection_id)- # Remove from FAISS index+ # Remove from FAISS index. delete + save + record must all+ # be inside the same lock — otherwise a concurrent writer+ # could sandwich a save_local between our delete and our+ # save, leaving stale chunks back on disk (#4197).if hasattr(self.faiss_index, "delete"):try:- self.faiss_index.delete(chunk_ids)-- # Save updated indexindex_path = (Path(self.rag_index_record.index_path)if self.rag_index_recordelse None)if index_path:- self.faiss_index.save_local(- str(index_path.parent),- index_name=index_path.stem,- )- # Record file integrity- self.integrity_manager.record_file(- index_path,- related_entity_type="rag_index",- related_entity_id=self.rag_index_record.id,- )+ with _get_faiss_write_lock(+ self.username, str(index_path)+ ):+ self.faiss_index.delete(chunk_ids)+ self.faiss_index.save_local(+ str(index_path.parent),+ index_name=index_path.stem,+ )+ self.integrity_manager.record_file(+ index_path,+ related_entity_type="rag_index",+ related_entity_id=self.rag_index_record.id,+ )+ else:+ # No index path → in-memory-only delete+ self.faiss_index.delete(chunk_ids)except Exception:logger.warning("Could not delete chunks from FAISS")diff --git a/src/local_deep_research/web/auth/connection_cleanup.py b/src/local_deep_research/web/auth/connection_cleanup.pyindex 77203caef0..2c8f95926f 100644--- a/src/local_deep_research/web/auth/connection_cleanup.py+++ b/src/local_deep_research/web/auth/connection_cleanup.py@@ -48,21 +48,22 @@ _last_dispose_time = 0.0def _pop_per_user_locks(username: str) -> None:- """Pop ``username`` from the three module-level per-user lock dicts.-- The library-init, backup, and queue-processor modules each maintain- a ``dict[username, threading.Lock]`` for serialising per-user- critical sections. None of them had a removal hook, so without- this the dicts accumulated one entry per username across the- process lifetime — bounded by total users (~296 bytes/entry × 3 =- ~900 B/user/dict) but real on long-lived self-hosted instances- with churn. Called from the user-close paths so the next login- starts with fresh locks (the locks hold no state worth preserving- across login/logout).+ """Pop ``username`` from the four module-level per-user lock dicts.++ The library-init, backup, queue-processor, and library-RAG modules+ each maintain a ``dict[..., threading.Lock]`` keyed by username (or+ by ``(username, ...)``) for serialising per-user critical sections.+ None of them had a removal hook, so without this the dicts+ accumulated one entry per username across the process lifetime —+ bounded by total users (~296 bytes/entry × 4 = ~1.2 KB/user/dict)+ but real on long-lived self-hosted instances with churn. Called+ from the user-close paths so the next login starts with fresh+ locks (the locks hold no state worth preserving across+ login/logout).Lazy-imported here to keep this module's import graph shallow:``connection_cleanup`` runs at startup and shouldn't pull in the- queue / backup / library-init modules eagerly.+ queue / backup / library-init / library-RAG modules eagerly."""try:from ...database.library_init import pop_user_init_lock@@ -89,6 +90,15 @@ def _pop_per_user_locks(username: str) -> None:except Exception:logger.warning(f"Failed to pop _user_critical_locks for {username}")+ try:+ from ...research_library.services.library_rag_service import (+ pop_faiss_locks_for_user,+ )++ pop_faiss_locks_for_user(username)+ except Exception:+ logger.warning(f"Failed to pop _faiss_write_locks for {username}")+def _count_open_fds() -> int:"""Count open file descriptors for the current process."""diff --git a/tests/research_library/services/test_library_rag_service_coverage.py b/tests/research_library/services/test_library_rag_service_coverage.pyindex b83e19740b..6e52c215aa 100644--- a/tests/research_library/services/test_library_rag_service_coverage.py+++ b/tests/research_library/services/test_library_rag_service_coverage.py@@ -13,9 +13,11 @@ Focuses on logic paths not exercised by the existing test_library_rag_service.py"""import hashlib+import threadingfrom pathlib import Pathfrom unittest.mock import MagicMock, patch+import pytestfrom langchain_core.documents import Document as LangchainDocument# ---------------------------------------------------------------------------@@ -393,9 +395,13 @@ class TestLoadOrCreateFaissIndex:@patch(f"{_MOD}.FAISS")@patch(f"{_MOD}.InMemoryDocstore")@patch(f"{_MOD}.IndexFlatIP")- def test_integrity_failure_creates_new_index(- self, mock_flat_ip, mock_docstore, mock_faiss_cls+ def test_integrity_failure_quarantines_and_creates_new_index(+ self, mock_flat_ip, mock_docstore, mock_faiss_cls, tmp_path):+ """Verify-failure path: corrupt .faiss + .pkl are RENAMED to+ .corrupt-<ns>, NOT unlinked. Then fresh index is created.+ Regression for #4197 data-loss bug.+ """svc = _make_service()svc.embedding_manager = MagicMock()svc.integrity_manager = MagicMock()@@ -404,18 +410,32 @@ class TestLoadOrCreateFaissIndex:"hash mismatch",)+ # Use real on-disk files so .exists() in the quarantine+ # collision check returns False naturally for the .corrupt-*+ # paths, not True-for-everything.+ idx_path = tmp_path / "corrupt.faiss"+ pkl_path = tmp_path / "corrupt.pkl"+ idx_path.write_bytes(b"faiss-bytes")+ pkl_path.write_bytes(b"pkl-bytes")+rag_idx = self._make_rag_index()- rag_idx.index_path = "/tmp/corrupt.faiss"+ rag_idx.index_path = str(idx_path)self._patch_get_or_create(svc, rag_idx)- with (- patch.object(Path, "exists", return_value=True),- patch.object(Path, "unlink") as mock_unlink,- ):+ with patch.object(Path, "unlink") as mock_unlink:svc.load_or_create_faiss_index("coll-1")- # Should have tried to unlink the corrupted file- mock_unlink.assert_called_once()+ # Should NOT have unlinked anything in the verify-failure path+ mock_unlink.assert_not_called()+ # Originals quarantined (renamed away from their paths)+ assert not idx_path.exists()+ assert not pkl_path.exists()+ # Both files preserved under .corrupt-<ns> names+ corrupt_faiss = list(tmp_path.glob("corrupt.faiss.corrupt-*"))+ corrupt_pkl = list(tmp_path.glob("corrupt.pkl.corrupt-*"))+ assert len(corrupt_faiss) == 1+ assert len(corrupt_pkl) == 1+ assert corrupt_faiss[0].read_bytes() == b"faiss-bytes"# Should return a new FAISS instance, not load_localmock_faiss_cls.load_local.assert_not_called()mock_faiss_cls.assert_called_once()@@ -423,26 +443,34 @@ class TestLoadOrCreateFaissIndex:@patch(f"{_MOD}.FAISS")@patch(f"{_MOD}.InMemoryDocstore")@patch(f"{_MOD}.IndexFlatIP")- def test_corrupted_unlink_failure_still_creates_new_index(- self, mock_flat_ip, mock_docstore, mock_faiss_cls+ def test_quarantine_rename_oserror_re_raises(+ self, mock_flat_ip, mock_docstore, mock_faiss_cls, tmp_path):+ """Disk-full / read-only fs during quarantine MUST propagate.+ Silently falling through would let the next save_local truncate+ the corrupt bytes, recreating the very data loss #4197 fixes.+ """svc = _make_service()svc.embedding_manager = MagicMock()svc.integrity_manager = MagicMock()svc.integrity_manager.verify_file.return_value = (False, "bad hash")+ idx_path = tmp_path / "corrupt.faiss"+ idx_path.write_bytes(b"faiss-bytes")+ (tmp_path / "corrupt.pkl").write_bytes(b"pkl-bytes")+rag_idx = self._make_rag_index()- rag_idx.index_path = "/tmp/corrupt.faiss"+ rag_idx.index_path = str(idx_path)self._patch_get_or_create(svc, rag_idx)- with (- patch.object(Path, "exists", return_value=True),- patch.object(Path, "unlink", side_effect=PermissionError("denied")),- ):- svc.load_or_create_faiss_index("coll-1")+ with patch.object(Path, "rename", side_effect=OSError("ENOSPC")):+ with pytest.raises(OSError, match="ENOSPC"):+ svc.load_or_create_faiss_index("coll-1")- # Should still create a new FAISS, not crash- mock_faiss_cls.assert_called_once()+ # Fresh index must NOT have been created — corrupt bytes are+ # still on disk and creating one would let the next save+ # overwrite them.+ mock_faiss_cls.assert_not_called()@patch(f"{_MOD}.get_user_db_session")@patch(f"{_MOD}.FAISS")@@ -487,24 +515,38 @@ class TestLoadOrCreateFaissIndex:@patch(f"{_MOD}.FAISS")@patch(f"{_MOD}.InMemoryDocstore")@patch(f"{_MOD}.IndexFlatIP")- def test_load_failure_creates_new_index(- self, mock_flat_ip, mock_docstore, mock_faiss_cls+ def test_load_failure_quarantines_and_creates_new_index(+ self, mock_flat_ip, mock_docstore, mock_faiss_cls, tmp_path):+ """When load_local raises (torn .pkl, malformed pickle), the+ .faiss and .pkl are quarantined before falling through to a+ fresh index. Previously the old code silently discarded the+ broken-state files without preserving evidence.+ """svc = _make_service()svc.embedding_manager = MagicMock()svc.embedding_manager.embeddings.embed_query.return_value = [0.0] * 384svc.integrity_manager = MagicMock()svc.integrity_manager.verify_file.return_value = (True, None)+ idx_path = tmp_path / "broken.faiss"+ pkl_path = tmp_path / "broken.pkl"+ idx_path.write_bytes(b"faiss-bytes")+ pkl_path.write_bytes(b"pkl-bytes")+rag_idx = self._make_rag_index(dim=384)- rag_idx.index_path = "/tmp/broken.faiss"+ rag_idx.index_path = str(idx_path)self._patch_get_or_create(svc, rag_idx)mock_faiss_cls.load_local.side_effect = RuntimeError("corrupted file")- with patch.object(Path, "exists", return_value=True):- svc.load_or_create_faiss_index("coll-1")+ svc.load_or_create_faiss_index("coll-1")+ # Both files quarantined (renamed away)+ assert not idx_path.exists()+ assert not pkl_path.exists()+ assert len(list(tmp_path.glob("broken.faiss.corrupt-*"))) == 1+ assert len(list(tmp_path.glob("broken.pkl.corrupt-*"))) == 1# Should fall through and create new indexmock_faiss_cls.assert_called_once()@@ -753,3 +795,528 @@ class TestContextManager:svc = _make_service()result = svc.__exit__(None, None, None)assert result is False+++# =========================================================================+# Quarantine helper + concurrent-write locks (#4197)+# =========================================================================+++def _import_module():+ import local_deep_research.research_library.services.library_rag_service as m++ return m+++class TestCorruptionQuarantine:+ """Verify the verify-failure and load-failure paths preserve+ corrupted bytes instead of deleting them.+ """++ def _make_idx(self, tmp_path, name="idx"):+ idx = tmp_path / f"{name}.faiss"+ pkl = tmp_path / f"{name}.pkl"+ idx.write_bytes(b"faiss-bytes")+ pkl.write_bytes(b"pkl-bytes")+ return idx, pkl++ def test_renames_both_faiss_and_pkl(self, tmp_path):+ svc = _make_service()+ idx, pkl = self._make_idx(tmp_path)+ svc._quarantine_corrupt_index(idx, "test")+ # Original files are gone from their paths+ assert not idx.exists()+ assert not pkl.exists()+ # Both quarantined as .corrupt-*+ corrupted_faiss = list(tmp_path.glob("idx.faiss.corrupt-*"))+ corrupted_pkl = list(tmp_path.glob("idx.pkl.corrupt-*"))+ assert len(corrupted_faiss) == 1+ assert len(corrupted_pkl) == 1+ assert corrupted_faiss[0].read_bytes() == b"faiss-bytes"+ assert corrupted_pkl[0].read_bytes() == b"pkl-bytes"++ def test_skips_pkl_when_missing(self, tmp_path):+ svc = _make_service()+ idx = tmp_path / "idx.faiss"+ idx.write_bytes(b"faiss-bytes")+ # No .pkl present+ svc._quarantine_corrupt_index(idx, "test")+ assert not idx.exists()+ corrupted = list(tmp_path.glob("idx.faiss.corrupt-*"))+ assert len(corrupted) == 1+ # No orphan pkl created+ assert not any(tmp_path.glob("idx.pkl.corrupt-*"))++ def test_rename_oserror_re_raises(self, tmp_path):+ """Disk-full / permission failure must propagate, not be swallowed."""+ svc = _make_service()+ idx, _pkl = self._make_idx(tmp_path)+ with patch.object(Path, "rename", side_effect=OSError("EROFS")):+ with pytest.raises(OSError, match="EROFS"):+ svc._quarantine_corrupt_index(idx, "test")++ def test_collision_increments_suffix(self, tmp_path):+ """Pre-existing quarantine path → loop increments to -1, -2, ..."""+ svc = _make_service()+ idx, pkl = self._make_idx(tmp_path)+ # Force a specific timestamp so we can predict collision targets+ with patch.object(+ _import_module().time,+ "time_ns",+ return_value=12345,+ ):+ # Pre-create the base collision target+ (tmp_path / "idx.faiss.corrupt-12345").write_bytes(b"old1")+ svc._quarantine_corrupt_index(idx, "test")+ # Original moved out+ assert not idx.exists()+ # New corrupt-12345-1 created (the -1 increment)+ moved = tmp_path / "idx.faiss.corrupt-12345-1"+ assert moved.exists()+ assert moved.read_bytes() == b"faiss-bytes"++ def test_collision_cap_raises(self, tmp_path):+ """If 32 collisions in a row, surface an OSError, don't hang."""+ svc = _make_service()+ mod = _import_module()+ idx, pkl = self._make_idx(tmp_path)+ with patch.object(mod.time, "time_ns", return_value=999):+ # Pre-create base + 32 collisions+ (tmp_path / "idx.faiss.corrupt-999").write_bytes(b"")+ for n in range(1, mod._QUARANTINE_SUFFIX_RETRY_CAP + 1):+ (tmp_path / f"idx.faiss.corrupt-999-{n}").write_bytes(b"")+ with pytest.raises(OSError, match="collisions exceeded"):+ svc._quarantine_corrupt_index(idx, "test")+++class TestFaissWriteLock:+ """Lock infrastructure: identity, parallelism across keys,+ cleanup on user-close.+ """++ def _reset_locks(self):+ mod = _import_module()+ with mod._faiss_write_locks_lock:+ mod._faiss_write_locks.clear()++ def test_same_key_returns_same_lock(self, tmp_path):+ self._reset_locks()+ mod = _import_module()+ p = str(tmp_path / "shared.faiss")+ a = mod._get_faiss_write_lock("u1", p)+ b = mod._get_faiss_write_lock("u1", p)+ assert a is b++ def test_different_key_returns_different_lock(self, tmp_path):+ self._reset_locks()+ mod = _import_module()+ a = mod._get_faiss_write_lock("u1", str(tmp_path / "a.faiss"))+ b = mod._get_faiss_write_lock("u1", str(tmp_path / "b.faiss"))+ c = mod._get_faiss_write_lock("u2", str(tmp_path / "a.faiss"))+ assert a is not b+ assert a is not c++ def test_pop_removes_only_target_user(self, tmp_path):+ self._reset_locks()+ mod = _import_module()+ u1_lock = mod._get_faiss_write_lock("u1", str(tmp_path / "a.faiss"))+ mod._get_faiss_write_lock("u1", str(tmp_path / "b.faiss"))+ u2_lock = mod._get_faiss_write_lock("u2", str(tmp_path / "a.faiss"))++ mod.pop_faiss_locks_for_user("u1")++ # u1 locks gone (new lookup yields a fresh lock object)+ new_u1 = mod._get_faiss_write_lock("u1", str(tmp_path / "a.faiss"))+ assert new_u1 is not u1_lock+ # u2 lock untouched+ same_u2 = mod._get_faiss_write_lock("u2", str(tmp_path / "a.faiss"))+ assert same_u2 is u2_lock++ def test_concurrent_save_record_serialised_for_same_key(self, tmp_path):+ """Two threads calling _save_and_record_locked on the same+ (username, index_path) MUST observe non-overlapping critical+ sections. Regression for the #4197 race.+ """+ self._reset_locks()+ index_path = tmp_path / "idx.faiss"+ index_path.touch()++ in_flight = 0+ max_in_flight = 0+ observed_overlap = False+ lock_for_counters = threading.Lock()+ save_started = threading.Event()+ save_can_finish = threading.Event()++ def fake_save_local(folder, index_name):+ nonlocal in_flight, max_in_flight, observed_overlap+ with lock_for_counters:+ in_flight += 1+ if in_flight > max_in_flight:+ max_in_flight = in_flight+ if in_flight > 1:+ observed_overlap = True+ save_started.set()+ # Hold the critical section open for the second thread+ save_can_finish.wait(timeout=2.0)+ with lock_for_counters:+ in_flight -= 1++ def fake_record_file(path, **kwargs):+ return None++ def worker():+ svc = _make_service()+ svc.faiss_index = MagicMock()+ svc.faiss_index.save_local.side_effect = fake_save_local+ svc.integrity_manager = MagicMock()+ svc.integrity_manager.record_file.side_effect = fake_record_file+ svc.rag_index_record = MagicMock(id=42)+ svc._save_and_record_locked(index_path)++ t1 = threading.Thread(target=worker)+ t2 = threading.Thread(target=worker)+ t1.start()+ # Make sure t1 is inside save_local before t2 starts trying+ assert save_started.wait(timeout=2.0)+ t2.start()+ # Brief moment for t2 to attempt and (hopefully) block on the lock+ save_can_finish.set()+ t1.join(timeout=3.0)+ t2.join(timeout=3.0)++ assert not observed_overlap, (+ "Two threads ran save_local on the same index concurrently; "+ "the lock did not serialise them."+ )+ assert max_in_flight == 1++ def test_different_keys_run_in_parallel(self, tmp_path):+ """Two threads with different (username, index_path) keys must+ be able to run save+record simultaneously.+ """+ self._reset_locks()+ barrier = threading.Barrier(2, timeout=2.0)+ idx1 = tmp_path / "a.faiss"+ idx2 = tmp_path / "b.faiss"+ idx1.touch()+ idx2.touch()++ def fake_save_local(folder, index_name):+ # If the lock blocked us, barrier.wait would time out+ barrier.wait()++ def worker(index_path):+ svc = _make_service()+ svc.faiss_index = MagicMock()+ svc.faiss_index.save_local.side_effect = fake_save_local+ svc.integrity_manager = MagicMock()+ svc.rag_index_record = MagicMock(id=99)+ svc._save_and_record_locked(index_path)++ t1 = threading.Thread(target=worker, args=(idx1,))+ t2 = threading.Thread(target=worker, args=(idx2,))+ t1.start()+ t2.start()+ t1.join(timeout=3.0)+ t2.join(timeout=3.0)+ # If barrier broke (timeout), one of the threads would be alive+ assert not t1.is_alive()+ assert not t2.is_alive()+++class TestNoWalCheckpointInIndexingPath:+ """Regression guard: the per-document PRAGMA wal_checkpoint(FULL)+ has been removed (#4197 secondary fix from PR #3539). Re-adding it+ would re-introduce 'database is locked' errors under concurrent+ bulk indexing.+ """++ def test_module_source_has_no_executable_wal_checkpoint(self):+ import inspect++ mod = _import_module()+ src = inspect.getsource(mod)+ # Strip out comment / docstring lines so the regression test+ # tolerates explanatory references like 'PRAGMA wal_checkpoint'+ # in comments, but fails if any code line re-adds the call.+ for lineno, line in enumerate(src.splitlines(), 1):+ stripped = line.strip()+ if not stripped:+ continue+ if stripped.startswith("#"):+ continue+ assert "wal_checkpoint(" not in stripped, (+ f"library_rag_service.py:{lineno} re-introduced "+ f"a wal_checkpoint() call: {line!r}"+ )+++class TestMergeAndPersistLocked:+ """Read-modify-write under the FAISS write lock.++ Regression for the AI-review concern on #4200: two workers+ indexing different documents into the same collection used to+ both load on-disk state X, each add their own chunk in memory,+ then save in sequence — last writer wins, the loser's chunks+ were lost from the FAISS file (chunks survived in the DB so a+ rebuild recovered them, but the index file was wrong until then).++ The helper reloads from disk under the lock before adding, so+ the second writer sees the first writer's save and merges+ instead of overwriting.+ """++ def _reset_locks(self):+ mod = _import_module()+ with mod._faiss_write_locks_lock:+ mod._faiss_write_locks.clear()++ def test_reload_picks_up_concurrent_writers_save(self, tmp_path):+ """The merge helper must reload from disk before adding so+ chunks committed by a concurrent writer between the caller's+ in-memory load and this lock acquisition are preserved.+ """+ self._reset_locks()+ mod = _import_module()+ index_path = tmp_path / "idx.faiss"+ # The on-disk file must exist + verify so the helper actually+ # takes the reload branch (not "no on-disk version, keep+ # stale in-memory").+ index_path.touch()++ svc = _make_service()+ svc.embedding_manager = MagicMock()+ svc.integrity_manager = MagicMock()+ svc.integrity_manager.verify_file.return_value = (True, None)+ svc.rag_index_record = MagicMock(id=42)++ # Caller's stale in-memory state knows about no chunks.+ stale_index = MagicMock()+ stale_index.docstore._dict = {}+ svc.faiss_index = stale_index++ # Disk state (loaded by the reload step) already has another+ # writer's chunk "concurrent-id".+ fresh_index = MagicMock()+ fresh_index.docstore._dict = {"concurrent-id": MagicMock()}++ new_chunk = LangchainDocument(page_content="our new content")++ with patch.object(mod.FAISS, "load_local", return_value=fresh_index):+ stats = svc._merge_and_persist_locked(+ index_path,+ [new_chunk],+ ["our-id"],+ force_reindex=False,+ )++ # Helper reloaded → svc.faiss_index is now the fresh state+ assert svc.faiss_index is fresh_index+ # Our chunk was added to the fresh state (not the stale one)+ fresh_index.add_documents.assert_called_once()+ added_ids = fresh_index.add_documents.call_args[1]["ids"]+ assert added_ids == ["our-id"]+ # The stale state was NOT mutated (otherwise we'd be saving it)+ stale_index.add_documents.assert_not_called()+ # save_local was called on the fresh (merged) state+ fresh_index.save_local.assert_called_once()+ assert stats["added"] == 1+ assert stats["added_ids"] == ["our-id"]++ def test_reload_skips_when_existing_id_now_on_disk(self, tmp_path):+ """If the caller wanted to add id X but a concurrent writer+ already saved X to disk, dedup against the fresh state must+ drop X (idempotent re-index).+ """+ self._reset_locks()+ mod = _import_module()+ index_path = tmp_path / "idx.faiss"+ index_path.touch()++ svc = _make_service()+ svc.embedding_manager = MagicMock()+ svc.integrity_manager = MagicMock()+ svc.integrity_manager.verify_file.return_value = (True, None)+ svc.rag_index_record = MagicMock(id=42)+ svc.faiss_index = MagicMock() # stale, irrelevant++ # Disk state already has the same id we're about to add.+ fresh_index = MagicMock()+ fresh_index.docstore._dict = {"shared-id": MagicMock()}++ with patch.object(mod.FAISS, "load_local", return_value=fresh_index):+ stats = svc._merge_and_persist_locked(+ index_path,+ [LangchainDocument(page_content="duplicate")],+ ["shared-id"],+ force_reindex=False,+ )++ # No add (already on disk); save still happens (idempotent ++ # touches mtime so the integrity record updates).+ fresh_index.add_documents.assert_not_called()+ fresh_index.save_local.assert_called_once()+ assert stats["added"] == 0+ assert stats["skipped"] == 1++ def test_force_reindex_deletes_after_reload(self, tmp_path):+ """force_reindex deletes the matching IDs from the FRESH+ on-disk state (not the caller's stale in-memory state)+ before re-adding.+ """+ self._reset_locks()+ mod = _import_module()+ index_path = tmp_path / "idx.faiss"+ index_path.touch()++ svc = _make_service()+ svc.embedding_manager = MagicMock()+ svc.integrity_manager = MagicMock()+ svc.integrity_manager.verify_file.return_value = (True, None)+ svc.rag_index_record = MagicMock(id=42)+ svc.faiss_index = MagicMock() # stale++ fresh_index = MagicMock()+ fresh_index.docstore._dict = {"to-update": MagicMock()}++ with patch.object(mod.FAISS, "load_local", return_value=fresh_index):+ svc._merge_and_persist_locked(+ index_path,+ [LangchainDocument(page_content="updated")],+ ["to-update"],+ force_reindex=True,+ )++ # Old copy removed from FRESH state+ fresh_index.delete.assert_called_once_with(["to-update"])+ # New copy added (no dedup under force_reindex)+ fresh_index.add_documents.assert_called_once()+ added_ids = fresh_index.add_documents.call_args[1]["ids"]+ assert added_ids == ["to-update"]++ def test_reload_failure_falls_back_to_in_memory(self, tmp_path):+ """If FAISS.load_local raises (torn write, partial pickle),+ the merge helper keeps the caller's in-memory state instead+ of losing the write entirely.+ """+ self._reset_locks()+ mod = _import_module()+ index_path = tmp_path / "idx.faiss"+ index_path.touch()++ svc = _make_service()+ svc.embedding_manager = MagicMock()+ svc.integrity_manager = MagicMock()+ svc.integrity_manager.verify_file.return_value = (True, None)+ svc.rag_index_record = MagicMock(id=42)++ in_memory = MagicMock()+ in_memory.docstore._dict = {}+ svc.faiss_index = in_memory++ with patch.object(+ mod.FAISS, "load_local", side_effect=RuntimeError("torn pickle")+ ):+ svc._merge_and_persist_locked(+ index_path,+ [LangchainDocument(page_content="kept")],+ ["kept-id"],+ force_reindex=False,+ )++ # Reload failed → kept the in-memory object, added to it,+ # saved it. The write is preserved.+ assert svc.faiss_index is in_memory+ in_memory.add_documents.assert_called_once()+ in_memory.save_local.assert_called_once()++ def test_concurrent_writers_both_chunks_survive(self, tmp_path):+ """End-to-end regression: two workers each call+ ``_merge_and_persist_locked`` for the same on-disk index.+ Each starts from a stale in-memory snapshot. After both+ complete, the on-disk state must contain BOTH chunks.++ Before the read-modify-write fix, this was last-writer-wins:+ worker A's chunk was overwritten by worker B's save because+ B never reloaded A's write.+ """+ self._reset_locks()+ mod = _import_module()+ index_path = tmp_path / "shared.faiss"+ index_path.touch()++ # Single shared "disk state" as a dict — proxies for what+ # would be on disk via FAISS.load_local/save_local.+ disk_state: dict = {}+ disk_lock = threading.Lock()++ def make_index_proxy():+ """Each call returns a FAISS-like proxy reading/writing+ the shared disk_state. Mimics a fresh load_local()."""+ ix = MagicMock()+ # Snapshot disk_state at load time (proxy for what+ # FAISS.load_local would return).+ snapshot = dict(disk_state)+ ix.docstore = MagicMock()+ ix.docstore._dict = snapshot++ def add(chunks, ids):+ for cid in ids:+ snapshot[cid] = "doc"++ def save(_folder, index_name=None):+ # Atomically replace disk_state with this index's+ # current snapshot — mirrors save_local's truncating+ # semantics for the test.+ with disk_lock:+ disk_state.clear()+ disk_state.update(snapshot)++ ix.add_documents.side_effect = add+ ix.save_local.side_effect = save+ return ix++ def worker(cid: str, start_event: threading.Event):+ svc = _make_service()+ svc.embedding_manager = MagicMock()+ svc.integrity_manager = MagicMock()+ svc.integrity_manager.verify_file.return_value = (True, None)+ svc.rag_index_record = MagicMock(id=99)+ # Pre-merge in-memory state is stale (empty)+ svc.faiss_index = MagicMock()+ svc.faiss_index.docstore = MagicMock()+ svc.faiss_index.docstore._dict = {}+ start_event.wait() # Race start+ with patch.object(+ mod.FAISS,+ "load_local",+ side_effect=lambda *a, **kw: make_index_proxy(),+ ):+ svc._merge_and_persist_locked(+ index_path,+ [LangchainDocument(page_content=f"chunk-{cid}")],+ [cid],+ force_reindex=False,+ )++ go = threading.Event()+ t1 = threading.Thread(target=worker, args=("A", go))+ t2 = threading.Thread(target=worker, args=("B", go))+ t1.start()+ t2.start()+ go.set()+ t1.join(timeout=5.0)+ t2.join(timeout=5.0)++ # Both writers' chunks present — read-modify-write under the+ # lock did its job. Without the fix, only one of {A, B} would+ # remain.+ assert "A" in disk_state, (+ f"Worker A's chunk lost; disk_state={disk_state}. "+ f"Read-modify-write race regressed."+ )+ assert "B" in disk_state, (+ f"Worker B's chunk lost; disk_state={disk_state}. "+ f"Read-modify-write race regressed."+ )diff --git a/tests/web/auth/test_connection_cleanup.py b/tests/web/auth/test_connection_cleanup.pyindex 4cca97bb60..199d91ea46 100644--- a/tests/web/auth/test_connection_cleanup.py+++ b/tests/web/auth/test_connection_cleanup.py@@ -283,13 +283,14 @@ class TestCleanupIdleConnections:class TestPopPerUserLocks:- """Tests for ``_pop_per_user_locks``: drops the three module-level+ """Tests for ``_pop_per_user_locks``: drops the four module-levelper-user lock-dict entries (library-init, backup, queue-processor- critical sections) on user-close. Without this each dict grew one- entry per username over process lifetime.+ critical sections, library-RAG FAISS-write locks) on user-close.+ Without this each dict grew one entry per username over process+ lifetime."""- def test_pops_all_three_lock_dicts(self):+ def test_pops_all_four_lock_dicts(self):from local_deep_research.web.auth.connection_cleanup import (_pop_per_user_locks,)@@ -304,6 +305,10 @@ class TestPopPerUserLocks:from local_deep_research.web.queue.processor_v2 import (queue_processor,)+ from local_deep_research.research_library.services.library_rag_service import (+ _faiss_write_locks,+ _get_faiss_write_lock,+ )# Populate each dict with a unique test username so we don't# collide with state any other test might have left behind.@@ -311,16 +316,19 @@ class TestPopPerUserLocks:_get_user_init_lock(u)_get_user_lock(u)queue_processor._get_user_critical_lock(u)+ _get_faiss_write_lock(u, "/tmp/test-pop-locks/idx.faiss")assert u in _user_init_locksassert u in _user_locksassert u in queue_processor._user_critical_locks+ assert any(k[0] == u for k in _faiss_write_locks)_pop_per_user_locks(u)assert u not in _user_init_locksassert u not in _user_locksassert u not in queue_processor._user_critical_locks+ assert not any(k[0] == u for k in _faiss_write_locks)def test_idempotent_on_missing_user(self):"""Pop on a username that was never inserted must not raise."""@@ -334,8 +342,8 @@ class TestPopPerUserLocks:def test_pop_called_from_idle_close_path(self, sm, db):"""Integration: ``cleanup_idle_connections`` calls``_pop_per_user_locks`` for each user it closes — verified- by asserting all three lock-dict entries are gone, so a- regression that drops the call (or breaks one of the three+ by asserting all four lock-dict entries are gone, so a+ regression that drops the call (or breaks one of the fourpop hops) would be caught."""from local_deep_research.database.library_init import (@@ -349,6 +357,10 @@ class TestPopPerUserLocks:from local_deep_research.web.queue.processor_v2 import (queue_processor,)+ from local_deep_research.research_library.services.library_rag_service import (+ _faiss_write_locks,+ _get_faiss_write_lock,+ )# Use a dedicated test username (not "alice") to avoid colliding# with other tests that may also touch these module-level dicts.@@ -356,10 +368,12 @@ class TestPopPerUserLocks:_get_user_init_lock(u)_get_user_lock(u)queue_processor._get_user_critical_lock(u)+ _get_faiss_write_lock(u, "/tmp/test-idle-close/idx.faiss")assert u in _user_init_locksassert u in _user_locksassert u in queue_processor._user_critical_locks+ assert any(k[0] == u for k in _faiss_write_locks)db.get_connected_usernames.return_value = {u}@@ -374,10 +388,11 @@ class TestPopPerUserLocks:):cleanup_idle_connections(sm, db)- # All three lock-dict entries removed by the idle-close path.+ # All four lock-dict entries removed by the idle-close path.assert u not in _user_init_locksassert u not in _user_locksassert u not in queue_processor._user_critical_locks+ assert not any(k[0] == u for k in _faiss_write_locks)def test_pop_runs_even_when_close_user_database_fails(self, sm, db):"""Regression for the original PR: ``_pop_per_user_locks`` was