~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
feat(library): prune old quarantined FAISS indexes to cap disk growth (#4203)
LearningCircuitda5317f413a8
diff --git a/changelog.d/+faiss-quarantine-retention.feature.md b/changelog.d/+faiss-quarantine-retention.feature.mdnew file mode 100644index 0000000000..023e522728--- /dev/null+++ b/changelog.d/+faiss-quarantine-retention.feature.md@@ -0,0 +1 @@+The library RAG indexer now prunes old quarantined FAISS index files (`<hash>.faiss.corrupt-*` and matching `.pkl.corrupt-*`) at quarantine time, keeping only the 5 most recent per base path. Prevents the `rag_indices/` cache directory from filling up on systems that experience recurring corruption, while preserving recent diagnostic artefacts. Follow-up to #4197 / #4200.diff --git a/src/local_deep_research/research_library/services/library_rag_service.py b/src/local_deep_research/research_library/services/library_rag_service.pyindex 55479f0f1e..d747aef5b0 100644--- a/src/local_deep_research/research_library/services/library_rag_service.py+++ b/src/local_deep_research/research_library/services/library_rag_service.py@@ -59,6 +59,14 @@ _faiss_write_locks_lock = threading.Lock()# into a loud OSError._QUARANTINE_SUFFIX_RETRY_CAP = 32+# After a successful quarantine, keep at most this many older .corrupt-*+# files per base path (per side: .faiss.corrupt-* and .pkl.corrupt-* are+# counted independently). Prevents unbounded disk growth on systems that+# experience recurring corruption while preserving recent diagnostic+# artefacts. Keeping 5 means the user has the last 5 corruption events+# to inspect / submit with a bug report; anything older is dropped.+_QUARANTINE_KEEP_RECENT = 5+def _get_faiss_write_lock(username: str, index_path: str) -> threading.Lock:"""Return the lock for ``(username, index_path)``, creating it on first@@ -418,6 +426,91 @@ class LibraryRAGService:)raise+ # Best-effort retention sweep. The quarantine itself succeeded+ # above; failing to prune older files is not a correctness+ # issue, just a disk-usage one — log and move on.+ self._prune_old_quarantined_files(index_path)++ @staticmethod+ def _corrupt_sort_key(path: Path) -> Tuple[int, int]:+ """Extract ``(ns, suffix_n)`` from a ``.corrupt-<ns>[-<n>]``+ filename so retention can sort by the monotonic nanosecond+ suffix the quarantine path embeds — *not* by ``st_mtime``.++ Filesystem timestamp granularity is sometimes 1s or 2s+ (FAT32/ext3/SMB shares), making mtime ordering non-deterministic+ when multiple quarantines happen within one tick. The+ ``.corrupt-<ns>`` suffix carries the original ``time.time_ns()``+ from the quarantine and is reliable across all filesystems.++ Returns ``(-1, -1)`` for malformed names (manually-placed+ files) so they sort below any real entry under ``reverse=True``+ — i.e., they get pruned first.+ """+ name = path.name+ marker = ".corrupt-"+ idx = name.rfind(marker)+ if idx == -1:+ return (-1, -1)+ tail = name[idx + len(marker) :]+ parts = tail.split("-")+ try:+ ns = int(parts[0])+ except ValueError:+ return (-1, -1)+ suffix_n = 0+ if len(parts) > 1:+ try:+ suffix_n = int(parts[-1])+ except ValueError:+ # Unknown trailing component — keep the file but at the+ # base ns ordering.+ pass+ return (ns, suffix_n)++ def _prune_old_quarantined_files(self, index_path: Path) -> None:+ """Keep only the ``_QUARANTINE_KEEP_RECENT`` most-recent+ ``.corrupt-*`` files for ``index_path`` and its ``.pkl``+ companion. Sweeps the two sides independently — pairs share+ the same ``-<ns>`` suffix so they're ordered identically.++ Ordering uses the embedded ``-<ns>`` from the quarantine+ filename, not ``st_mtime``: file systems with 1-2s timestamp+ granularity can otherwise produce non-deterministic retention+ on bursts.++ Best-effort: logs and swallows any error so a sweep failure+ never propagates back into the indexing path.+ """+ parent = index_path.parent+ pkl_path = index_path.with_suffix(".pkl")++ for base in (index_path, pkl_path):+ pattern = f"{base.name}.corrupt-*"+ try:+ # Sort newest-first by the embedded -<ns>; everything+ # past the keep window is stale.+ candidates = sorted(+ parent.glob(pattern),+ key=self._corrupt_sort_key,+ reverse=True,+ )+ except OSError:+ logger.warning(+ f"Failed to enumerate {pattern} in {parent} for "+ f"quarantine retention sweep"+ )+ continue++ for stale in candidates[_QUARANTINE_KEEP_RECENT:]:+ try:+ stale.unlink()+ logger.info(f"Pruned old quarantined file: {stale}")+ except OSError:+ logger.warning(+ f"Failed to prune quarantined file {stale}; continuing"+ )+def _merge_and_persist_locked(self,index_path: Path,diff --git a/tests/research_library/services/test_library_rag_service_coverage.py b/tests/research_library/services/test_library_rag_service_coverage.pyindex 55af1fc8db..f5bfcd81c3 100644--- a/tests/research_library/services/test_library_rag_service_coverage.py+++ b/tests/research_library/services/test_library_rag_service_coverage.py@@ -889,6 +889,202 @@ class TestCorruptionQuarantine:svc._quarantine_corrupt_index(idx, "test")+class TestQuarantineRetention:+ """Retention sweep keeps the rag_indices/ directory from filling+ with old .corrupt-* files on systems that experience recurring+ corruption.+ """++ def _make_idx(self, tmp_path, name="idx"):+ idx = tmp_path / f"{name}.faiss"+ pkl = tmp_path / f"{name}.pkl"+ idx.write_bytes(b"faiss-bytes")+ pkl.write_bytes(b"pkl-bytes")+ return idx, pkl++ def _make_quarantined(self, tmp_path, name, ns, mtime):+ """Create a paired (.faiss, .pkl) quarantined-style file at+ the given ns suffix and stamp it with ``mtime`` so the+ retention sweep's age sort is deterministic.+ """+ faiss_f = tmp_path / f"{name}.faiss.corrupt-{ns}"+ pkl_f = tmp_path / f"{name}.pkl.corrupt-{ns}"+ faiss_f.write_bytes(b"")+ pkl_f.write_bytes(b"")+ import os++ os.utime(faiss_f, (mtime, mtime))+ os.utime(pkl_f, (mtime, mtime))+ return faiss_f, pkl_f++ def test_prune_keeps_most_recent_drops_older(self, tmp_path):+ """After N+2 simulated past corruption events, pruning leaves+ exactly N (most recent by mtime) and removes the oldest 2.+ """+ svc = _make_service()+ mod = _import_module()+ keep = mod._QUARANTINE_KEEP_RECENT++ # Pre-seed keep+2 quarantined pairs with increasing mtimes+ # (later ns → newer mtime).+ for i in range(keep + 2):+ self._make_quarantined(+ tmp_path, "idx", ns=1000 + i, mtime=1000.0 + i+ )++ # Trigger pruning via the public quarantine call (which now+ # invokes the sweep at the end).+ idx, _pkl = self._make_idx(tmp_path)+ svc._quarantine_corrupt_index(idx, "test")++ faiss_corrupts = sorted(tmp_path.glob("idx.faiss.corrupt-*"))+ pkl_corrupts = sorted(tmp_path.glob("idx.pkl.corrupt-*"))+ # keep pre-existing + 1 fresh = keep total on each side+ assert len(faiss_corrupts) == keep+ assert len(pkl_corrupts) == keep++ # Oldest two pre-existing pairs (ns 1000, 1001) must be gone.+ for stale_ns in (1000, 1001):+ assert not (tmp_path / f"idx.faiss.corrupt-{stale_ns}").exists()+ assert not (tmp_path / f"idx.pkl.corrupt-{stale_ns}").exists()++ def test_prune_skipped_when_under_threshold(self, tmp_path):+ """With fewer than the retention cap, no pruning happens."""+ svc = _make_service()+ mod = _import_module()+ keep = mod._QUARANTINE_KEEP_RECENT++ # Pre-seed (keep - 2) quarantined pairs. After this call we'll+ # add 1 more via the quarantine, ending at keep - 1 — still+ # under the cap.+ for i in range(keep - 2):+ self._make_quarantined(+ tmp_path, "idx", ns=2000 + i, mtime=2000.0 + i+ )++ idx, _pkl = self._make_idx(tmp_path)+ svc._quarantine_corrupt_index(idx, "test")++ # All pre-existing + the new one survive.+ assert len(list(tmp_path.glob("idx.faiss.corrupt-*"))) == keep - 1++ def test_prune_only_targets_same_base(self, tmp_path):+ """Pruning of ``idx.faiss.corrupt-*`` must NOT touch+ ``other.faiss.corrupt-*`` — different base paths are+ independent.+ """+ svc = _make_service()+ mod = _import_module()+ keep = mod._QUARANTINE_KEEP_RECENT++ # Many old corrupts for the other base — these MUST be untouched.+ for i in range(keep + 3):+ self._make_quarantined(+ tmp_path, "other", ns=3000 + i, mtime=3000.0 + i+ )+ # Trigger many corrupts for our base so the sweep runs.+ for i in range(keep + 2):+ self._make_quarantined(+ tmp_path, "idx", ns=4000 + i, mtime=4000.0 + i+ )++ idx, _pkl = self._make_idx(tmp_path)+ svc._quarantine_corrupt_index(idx, "test")++ # Other base untouched+ other_corrupts = list(tmp_path.glob("other.faiss.corrupt-*"))+ assert len(other_corrupts) == keep + 3+ # Our base pruned+ idx_corrupts = list(tmp_path.glob("idx.faiss.corrupt-*"))+ assert len(idx_corrupts) == keep++ def test_prune_failure_does_not_propagate(self, tmp_path):+ """If unlink raises during the sweep, the quarantine call+ still succeeds — retention is best-effort.+ """+ svc = _make_service()+ mod = _import_module()+ keep = mod._QUARANTINE_KEEP_RECENT++ # Seed enough corrupts to trigger pruning.+ for i in range(keep + 2):+ self._make_quarantined(+ tmp_path, "idx", ns=5000 + i, mtime=5000.0 + i+ )++ before_count = len(list(tmp_path.glob("idx.faiss.corrupt-*")))+ idx, _pkl = self._make_idx(tmp_path)++ # Allow the quarantine rename to succeed, but make unlink raise+ # for the prune sweep. The quarantine itself uses rename, not+ # unlink, so this only affects the retention path.+ real_unlink = Path.unlink++ def selective_unlink(self):+ if ".corrupt-" in self.name:+ raise PermissionError(f"denied: {self}")+ return real_unlink(self)++ with patch.object(Path, "unlink", selective_unlink):+ # Must not raise — sweep is best-effort.+ svc._quarantine_corrupt_index(idx, "test")++ # Quarantine still produced a new pair even though the sweep+ # was unable to delete anything (count went up, not down).+ after_count = len(list(tmp_path.glob("idx.faiss.corrupt-*")))+ assert after_count == before_count + 1, (+ f"expected one new corrupt file; before={before_count}, "+ f"after={after_count}"+ )++ def test_retention_sort_uses_filename_ns_not_mtime(self, tmp_path):+ """On low-resolution filesystems (FAT32, SMB, ext3) mtime can+ round to whole seconds, making same-second quarantines sort+ non-deterministically by ``st_mtime``. The embedded+ ``-<ns>`` suffix is the authoritative ordering — confirm+ retention drops the file with the *lowest* ns even when+ its mtime would mark it as the *newest*.+ """+ svc = _make_service()+ mod = _import_module()+ keep = mod._QUARANTINE_KEEP_RECENT+ import os++ # Seed exactly `keep` corrupts. The OLDEST ns gets the NEWEST+ # mtime; any retention that sorts by mtime would keep it. A+ # sort by ns must prune it once one more quarantine pushes us+ # over the cap.+ ns_values = list(range(6000, 6000 + keep))+ mtimes = list(reversed([7000.0 + i for i in range(keep)]))+ for ns, mtime in zip(ns_values, mtimes):+ self._make_quarantined(tmp_path, "idx", ns=ns, mtime=mtime)++ # Add one more quarantine via the real helper. Its ns is+ # `time.time_ns()` which is far larger than 6000+, so by ns+ # it's the freshest. Force its mtime to ancient so any+ # mtime-based sort would prune *it* instead of the seeded+ # lowest-ns file — that's the regression this test guards.+ idx, _pkl = self._make_idx(tmp_path)+ original_rename = Path.rename++ def rename_then_age(self, target):+ result = original_rename(self, target)+ os.utime(target, (1.0, 1.0))+ return result++ with patch.object(Path, "rename", rename_then_age):+ svc._quarantine_corrupt_index(idx, "test")++ # The lowest-ns seeded file (ns=6000) must be the one pruned.+ # The other seeded files (ns=6001..6004) plus the brand-new+ # one (highest ns) survive.+ assert not (tmp_path / f"idx.faiss.corrupt-{ns_values[0]}").exists()+ for ns in ns_values[1:]:+ assert (tmp_path / f"idx.faiss.corrupt-{ns}").exists(), (+ f"ns={ns} should still exist after retention sort"+ )++class TestFaissWriteLock:"""Lock infrastructure: identity, parallelism across keys,cleanup on user-close.