Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

feat(library): prune old quarantined FAISS indexes to cap disk growth (#4203)

LearningCircuitda5317f413a8
diff --git a/changelog.d/+faiss-quarantine-retention.feature.md b/changelog.d/+faiss-quarantine-retention.feature.md
new file mode 100644
index 0000000000..023e522728
--- /dev/null
+++ b/changelog.d/+faiss-quarantine-retention.feature.md
@@ -0,0 +1 @@
+The library RAG indexer now prunes old quarantined FAISS index files (`<hash>.faiss.corrupt-*` and matching `.pkl.corrupt-*`) at quarantine time, keeping only the 5 most recent per base path. Prevents the `rag_indices/` cache directory from filling up on systems that experience recurring corruption, while preserving recent diagnostic artefacts. Follow-up to #4197 / #4200.
diff --git a/src/local_deep_research/research_library/services/library_rag_service.py b/src/local_deep_research/research_library/services/library_rag_service.py
index 55479f0f1e..d747aef5b0 100644
--- a/src/local_deep_research/research_library/services/library_rag_service.py
+++ b/src/local_deep_research/research_library/services/library_rag_service.py
@@ -59,6 +59,14 @@ _faiss_write_locks_lock = threading.Lock()
# into a loud OSError.
_QUARANTINE_SUFFIX_RETRY_CAP = 32
+# After a successful quarantine, keep at most this many older .corrupt-*
+# files per base path (per side: .faiss.corrupt-* and .pkl.corrupt-* are
+# counted independently). Prevents unbounded disk growth on systems that
+# experience recurring corruption while preserving recent diagnostic
+# artefacts. Keeping 5 means the user has the last 5 corruption events
+# to inspect / submit with a bug report; anything older is dropped.
+_QUARANTINE_KEEP_RECENT = 5
+
def _get_faiss_write_lock(username: str, index_path: str) -> threading.Lock:
"""Return the lock for ``(username, index_path)``, creating it on first
@@ -418,6 +426,91 @@ class LibraryRAGService:
)
raise
+ # Best-effort retention sweep. The quarantine itself succeeded
+ # above; failing to prune older files is not a correctness
+ # issue, just a disk-usage one — log and move on.
+ self._prune_old_quarantined_files(index_path)
+
+ @staticmethod
+ def _corrupt_sort_key(path: Path) -> Tuple[int, int]:
+ """Extract ``(ns, suffix_n)`` from a ``.corrupt-<ns>[-<n>]``
+ filename so retention can sort by the monotonic nanosecond
+ suffix the quarantine path embeds — *not* by ``st_mtime``.
+
+ Filesystem timestamp granularity is sometimes 1s or 2s
+ (FAT32/ext3/SMB shares), making mtime ordering non-deterministic
+ when multiple quarantines happen within one tick. The
+ ``.corrupt-<ns>`` suffix carries the original ``time.time_ns()``
+ from the quarantine and is reliable across all filesystems.
+
+ Returns ``(-1, -1)`` for malformed names (manually-placed
+ files) so they sort below any real entry under ``reverse=True``
+ — i.e., they get pruned first.
+ """
+ name = path.name
+ marker = ".corrupt-"
+ idx = name.rfind(marker)
+ if idx == -1:
+ return (-1, -1)
+ tail = name[idx + len(marker) :]
+ parts = tail.split("-")
+ try:
+ ns = int(parts[0])
+ except ValueError:
+ return (-1, -1)
+ suffix_n = 0
+ if len(parts) > 1:
+ try:
+ suffix_n = int(parts[-1])
+ except ValueError:
+ # Unknown trailing component — keep the file but at the
+ # base ns ordering.
+ pass
+ return (ns, suffix_n)
+
+ def _prune_old_quarantined_files(self, index_path: Path) -> None:
+ """Keep only the ``_QUARANTINE_KEEP_RECENT`` most-recent
+ ``.corrupt-*`` files for ``index_path`` and its ``.pkl``
+ companion. Sweeps the two sides independently — pairs share
+ the same ``-<ns>`` suffix so they're ordered identically.
+
+ Ordering uses the embedded ``-<ns>`` from the quarantine
+ filename, not ``st_mtime``: file systems with 1-2s timestamp
+ granularity can otherwise produce non-deterministic retention
+ on bursts.
+
+ Best-effort: logs and swallows any error so a sweep failure
+ never propagates back into the indexing path.
+ """
+ parent = index_path.parent
+ pkl_path = index_path.with_suffix(".pkl")
+
+ for base in (index_path, pkl_path):
+ pattern = f"{base.name}.corrupt-*"
+ try:
+ # Sort newest-first by the embedded -<ns>; everything
+ # past the keep window is stale.
+ candidates = sorted(
+ parent.glob(pattern),
+ key=self._corrupt_sort_key,
+ reverse=True,
+ )
+ except OSError:
+ logger.warning(
+ f"Failed to enumerate {pattern} in {parent} for "
+ f"quarantine retention sweep"
+ )
+ continue
+
+ for stale in candidates[_QUARANTINE_KEEP_RECENT:]:
+ try:
+ stale.unlink()
+ logger.info(f"Pruned old quarantined file: {stale}")
+ except OSError:
+ logger.warning(
+ f"Failed to prune quarantined file {stale}; continuing"
+ )
+
def _merge_and_persist_locked(
self,
index_path: Path,
diff --git a/tests/research_library/services/test_library_rag_service_coverage.py b/tests/research_library/services/test_library_rag_service_coverage.py
index 55af1fc8db..f5bfcd81c3 100644
--- a/tests/research_library/services/test_library_rag_service_coverage.py
+++ b/tests/research_library/services/test_library_rag_service_coverage.py
@@ -889,6 +889,202 @@ class TestCorruptionQuarantine:
svc._quarantine_corrupt_index(idx, "test")
+class TestQuarantineRetention:
+ """Retention sweep keeps the rag_indices/ directory from filling
+ with old .corrupt-* files on systems that experience recurring
+ corruption.
+ """
+
+ def _make_idx(self, tmp_path, name="idx"):
+ idx = tmp_path / f"{name}.faiss"
+ pkl = tmp_path / f"{name}.pkl"
+ idx.write_bytes(b"faiss-bytes")
+ pkl.write_bytes(b"pkl-bytes")
+ return idx, pkl
+
+ def _make_quarantined(self, tmp_path, name, ns, mtime):
+ """Create a paired (.faiss, .pkl) quarantined-style file at
+ the given ns suffix and stamp it with ``mtime`` so the
+ retention sweep's age sort is deterministic.
+ """
+ faiss_f = tmp_path / f"{name}.faiss.corrupt-{ns}"
+ pkl_f = tmp_path / f"{name}.pkl.corrupt-{ns}"
+ faiss_f.write_bytes(b"")
+ pkl_f.write_bytes(b"")
+ import os
+
+ os.utime(faiss_f, (mtime, mtime))
+ os.utime(pkl_f, (mtime, mtime))
+ return faiss_f, pkl_f
+
+ def test_prune_keeps_most_recent_drops_older(self, tmp_path):
+ """After N+2 simulated past corruption events, pruning leaves
+ exactly N (most recent by mtime) and removes the oldest 2.
+ """
+ svc = _make_service()
+ mod = _import_module()
+ keep = mod._QUARANTINE_KEEP_RECENT
+
+ # Pre-seed keep+2 quarantined pairs with increasing mtimes
+ # (later ns → newer mtime).
+ for i in range(keep + 2):
+ self._make_quarantined(
+ tmp_path, "idx", ns=1000 + i, mtime=1000.0 + i
+ )
+
+ # Trigger pruning via the public quarantine call (which now
+ # invokes the sweep at the end).
+ idx, _pkl = self._make_idx(tmp_path)
+ svc._quarantine_corrupt_index(idx, "test")
+
+ faiss_corrupts = sorted(tmp_path.glob("idx.faiss.corrupt-*"))
+ pkl_corrupts = sorted(tmp_path.glob("idx.pkl.corrupt-*"))
+ # keep pre-existing + 1 fresh = keep total on each side
+ assert len(faiss_corrupts) == keep
+ assert len(pkl_corrupts) == keep
+
+ # Oldest two pre-existing pairs (ns 1000, 1001) must be gone.
+ for stale_ns in (1000, 1001):
+ assert not (tmp_path / f"idx.faiss.corrupt-{stale_ns}").exists()
+ assert not (tmp_path / f"idx.pkl.corrupt-{stale_ns}").exists()
+
+ def test_prune_skipped_when_under_threshold(self, tmp_path):
+ """With fewer than the retention cap, no pruning happens."""
+ svc = _make_service()
+ mod = _import_module()
+ keep = mod._QUARANTINE_KEEP_RECENT
+
+ # Pre-seed (keep - 2) quarantined pairs. After this call we'll
+ # add 1 more via the quarantine, ending at keep - 1 — still
+ # under the cap.
+ for i in range(keep - 2):
+ self._make_quarantined(
+ tmp_path, "idx", ns=2000 + i, mtime=2000.0 + i
+ )
+
+ idx, _pkl = self._make_idx(tmp_path)
+ svc._quarantine_corrupt_index(idx, "test")
+
+ # All pre-existing + the new one survive.
+ assert len(list(tmp_path.glob("idx.faiss.corrupt-*"))) == keep - 1
+
+ def test_prune_only_targets_same_base(self, tmp_path):
+ """Pruning of ``idx.faiss.corrupt-*`` must NOT touch
+ ``other.faiss.corrupt-*`` — different base paths are
+ independent.
+ """
+ svc = _make_service()
+ mod = _import_module()
+ keep = mod._QUARANTINE_KEEP_RECENT
+
+ # Many old corrupts for the other base — these MUST be untouched.
+ for i in range(keep + 3):
+ self._make_quarantined(
+ tmp_path, "other", ns=3000 + i, mtime=3000.0 + i
+ )
+ # Trigger many corrupts for our base so the sweep runs.
+ for i in range(keep + 2):
+ self._make_quarantined(
+ tmp_path, "idx", ns=4000 + i, mtime=4000.0 + i
+ )
+
+ idx, _pkl = self._make_idx(tmp_path)
+ svc._quarantine_corrupt_index(idx, "test")
+
+ # Other base untouched
+ other_corrupts = list(tmp_path.glob("other.faiss.corrupt-*"))
+ assert len(other_corrupts) == keep + 3
+ # Our base pruned
+ idx_corrupts = list(tmp_path.glob("idx.faiss.corrupt-*"))
+ assert len(idx_corrupts) == keep
+
+ def test_prune_failure_does_not_propagate(self, tmp_path):
+ """If unlink raises during the sweep, the quarantine call
+ still succeeds — retention is best-effort.
+ """
+ svc = _make_service()
+ mod = _import_module()
+ keep = mod._QUARANTINE_KEEP_RECENT
+
+ # Seed enough corrupts to trigger pruning.
+ for i in range(keep + 2):
+ self._make_quarantined(
+ tmp_path, "idx", ns=5000 + i, mtime=5000.0 + i
+ )
+
+ before_count = len(list(tmp_path.glob("idx.faiss.corrupt-*")))
+ idx, _pkl = self._make_idx(tmp_path)
+
+ # Allow the quarantine rename to succeed, but make unlink raise
+ # for the prune sweep. The quarantine itself uses rename, not
+ # unlink, so this only affects the retention path.
+ real_unlink = Path.unlink
+
+ def selective_unlink(self):
+ if ".corrupt-" in self.name:
+ raise PermissionError(f"denied: {self}")
+ return real_unlink(self)
+
+ with patch.object(Path, "unlink", selective_unlink):
+ # Must not raise — sweep is best-effort.
+ svc._quarantine_corrupt_index(idx, "test")
+
+ # Quarantine still produced a new pair even though the sweep
+ # was unable to delete anything (count went up, not down).
+ after_count = len(list(tmp_path.glob("idx.faiss.corrupt-*")))
+ assert after_count == before_count + 1, (
+ f"expected one new corrupt file; before={before_count}, "
+ f"after={after_count}"
+ )
+
+ def test_retention_sort_uses_filename_ns_not_mtime(self, tmp_path):
+ """On low-resolution filesystems (FAT32, SMB, ext3) mtime can
+ round to whole seconds, making same-second quarantines sort
+ non-deterministically by ``st_mtime``. The embedded
+ ``-<ns>`` suffix is the authoritative ordering — confirm
+ retention drops the file with the *lowest* ns even when
+ its mtime would mark it as the *newest*.
+ """
+ svc = _make_service()
+ mod = _import_module()
+ keep = mod._QUARANTINE_KEEP_RECENT
+ import os
+
+ # Seed exactly `keep` corrupts. The OLDEST ns gets the NEWEST
+ # mtime; any retention that sorts by mtime would keep it. A
+ # sort by ns must prune it once one more quarantine pushes us
+ # over the cap.
+ ns_values = list(range(6000, 6000 + keep))
+ mtimes = list(reversed([7000.0 + i for i in range(keep)]))
+ for ns, mtime in zip(ns_values, mtimes):
+ self._make_quarantined(tmp_path, "idx", ns=ns, mtime=mtime)
+
+ # Add one more quarantine via the real helper. Its ns is
+ # `time.time_ns()` which is far larger than 6000+, so by ns
+ # it's the freshest. Force its mtime to ancient so any
+ # mtime-based sort would prune *it* instead of the seeded
+ # lowest-ns file — that's the regression this test guards.
+ idx, _pkl = self._make_idx(tmp_path)
+ original_rename = Path.rename
+
+ def rename_then_age(self, target):
+ result = original_rename(self, target)
+ os.utime(target, (1.0, 1.0))
+ return result
+
+ with patch.object(Path, "rename", rename_then_age):
+ svc._quarantine_corrupt_index(idx, "test")
+
+ # The lowest-ns seeded file (ns=6000) must be the one pruned.
+ # The other seeded files (ns=6001..6004) plus the brand-new
+ # one (highest ns) survive.
+ assert not (tmp_path / f"idx.faiss.corrupt-{ns_values[0]}").exists()
+ for ns in ns_values[1:]:
+ assert (tmp_path / f"idx.faiss.corrupt-{ns}").exists(), (
+ f"ns={ns} should still exist after retention sort"
+ )
+
+
class TestFaissWriteLock:
"""Lock infrastructure: identity, parallelism across keys,
cleanup on user-close.