[MLsys2026 Best Paper]: https://arxiv.org/abs/2506.08276. RAG on Everything with LEANN. Enjoy 97% storage savings while running a fast, accurate, and 100% private RAG application on your personal device.
Python aifaissgpt-osslangchainllama-indexllmlocalstorageoffline-firstollamaprivacypythonragretrieval-augmented-generationvector-databasevector-searchvectors
fix: IVF incremental update duplicate chunks from stale passages.jsonl (#277)
Yichuan Wangb30f382c29c2
diff --git a/.gitignore b/.gitignoreindex 46ce36c..1c55812 100755--- a/.gitignore+++ b/.gitignore@@ -117,4 +117,6 @@ apps/multimodal/vision-based-pdf-multi-vector/multi-vector-colpali-native-weaviaparu-bin/merkle-tree-test/test-code/+localtestmcp/+*.csv*.picklediff --git a/packages/leann-backend-ivf/leann_backend_ivf/ivf_backend.py b/packages/leann-backend-ivf/leann_backend_ivf/ivf_backend.pyindex ca893da..23c1411 100644--- a/packages/leann-backend-ivf/leann_backend_ivf/ivf_backend.py+++ b/packages/leann-backend-ivf/leann_backend_ivf/ivf_backend.py@@ -263,6 +263,7 @@ def remove_ids(index_path: str, passage_ids: list[str]) -> int:return 0index = faiss.read_index(str(index_file))+ ntotal_before = index.ntotalsel = np.array(to_remove_int, dtype=np.int64)nremoved = index.remove_ids(sel)faiss.write_index(index, str(index_file))@@ -273,5 +274,18 @@ def remove_ids(index_path: str, passage_ids: list[str]) -> int:id_to_passage.pop(i, None)# passage_to_id is rebuilt from id_to_passage when we save; we don't decrease next_id so new adds get new ids_save_id_map(index_dir, index_prefix, id_to_passage, next_id=next_id)- logger.info("IVF remove_ids: removed %d vectors (requested %d)", nremoved, len(passage_ids))+ logger.info(+ "IVF remove_ids: ntotal %d -> %d, removed %d vectors (requested %d, found %d in id_map)",+ ntotal_before,+ index.ntotal,+ nremoved,+ len(passage_ids),+ len(to_remove_int),+ )+ if nremoved != len(to_remove_int):+ logger.warning(+ "IVF remove_ids: FAISS removed %d but expected %d. Possible index inconsistency.",+ nremoved,+ len(to_remove_int),+ )return nremoveddiff --git a/packages/leann-core/src/leann/api.py b/packages/leann-core/src/leann/api.pyindex 743adc0..9924f52 100644--- a/packages/leann-core/src/leann/api.py+++ b/packages/leann-core/src/leann/api.py@@ -718,6 +718,31 @@ class LeannBuilder:logger.info(f"Index built successfully from precomputed embeddings: {index_path}")+ @staticmethod+ def _compact_passages(+ passages_file: Path, offset_file: Path, offset_map: dict[str, int]+ ) -> None:+ """Rewrite passages.jsonl keeping only entries referenced by offset_map."""+ live_entries: list[str] = []+ for _pid, offset in sorted(offset_map.items(), key=lambda x: x[1]):+ with open(passages_file, encoding="utf-8") as f:+ f.seek(offset)+ live_entries.append(f.readline())++ tmp_file = passages_file.with_suffix(".jsonl.tmp")+ new_offset_map: dict[str, int] = {}+ with open(tmp_file, "w", encoding="utf-8") as f:+ for line in live_entries:+ data = json.loads(line)+ new_offset_map[data["id"]] = f.tell()+ f.write(line if line.endswith("\n") else line + "\n")++ tmp_file.replace(passages_file)+ offset_map.clear()+ offset_map.update(new_offset_map)+ with open(offset_file, "wb") as f:+ pickle.dump(offset_map, f)+def update_index(self, index_path: str, remove_passage_ids: Optional[list[str]] = None) -> None:"""Append new passages and vectors to an existing index (HNSW or IVF).For IVF, optional remove_passage_ids removes those ids first (e.g. from file-change API).@@ -757,7 +782,14 @@ class LeannBuilder:try:from leann_backend_ivf import remove_ids as ivf_remove_ids- ivf_remove_ids(str(path), remove_passage_ids)+ nremoved = ivf_remove_ids(str(path), remove_passage_ids)+ if nremoved < len(remove_passage_ids):+ logger.warning(+ "IVF update_index: removed %d of %d requested passage IDs "+ "(some may have been stale).",+ nremoved,+ len(remove_passage_ids),+ )except ImportError:raise RuntimeError("IVF backend required for remove_ids. Install leann-backend-ivf."@@ -765,8 +797,9 @@ class LeannBuilder:for pid in remove_passage_ids:offset_map.pop(pid, None)existing_ids -= set(remove_passage_ids)- with open(offset_file, "wb") as f:- pickle.dump(offset_map, f)++ # Compact passages.jsonl: rewrite keeping only entries in offset_map+ self._compact_passages(passages_file, offset_file, offset_map)if not self.chunks:meta["total_passages"] = len(offset_map)diff --git a/packages/leann-core/src/leann/cli.py b/packages/leann-core/src/leann/cli.pyindex 01e6c0b..4afb0c3 100644--- a/packages/leann-core/src/leann/cli.py+++ b/packages/leann-core/src/leann/cli.py@@ -5,8 +5,10 @@ import hashlibimport ioimport jsonimport os+import pickleimport sysimport time+import uuidfrom pathlib import Pathfrom typing import Any, Optional, Union@@ -1735,6 +1737,14 @@ Examples:c.setdefault("metadata", {})["id"] = sidc["id"] = sid+ @staticmethod+ def _assign_unique_chunk_ids(chunks: list[dict]) -> None:+ """Assign unique IDs for incremental (avoids collision when path lookup misses some old ids)."""+ for c in chunks:+ sid = uuid.uuid4().hex[:16]+ c.setdefault("metadata", {})["id"] = sid+ c["id"] = sid+def _chunks_for_paths(self, all_texts: list[dict], paths: set[str]) -> list[dict]:"""Filter chunks belonging to the given file paths."""return [@@ -1788,10 +1798,21 @@ Examples:passages_file = index_dir / "documents.leann.passages.jsonl"if not passages_file.exists():return False- chunk_ids_by_file = self._load_chunk_ids_by_file(passages_file)+ offset_file = index_dir / "documents.leann.passages.idx"+ live_ids: set[str] | None = None+ if offset_file.exists():+ with open(offset_file, "rb") as f:+ live_ids = set(pickle.load(f).keys())+ chunk_ids_by_file = self._load_chunk_ids_by_file(passages_file, live_ids=live_ids)+ roots = self._load_sync_roots(index_dir)ids_to_remove: list[str] = []+ seen_ids: set[str] = set()for p in removed_paths:- ids_to_remove.extend(chunk_ids_by_file.get(p, []))+ for key in self._path_lookup_keys(p, roots):+ for pid in chunk_ids_by_file.get(key, []):+ if pid not in seen_ids:+ seen_ids.add(pid)+ ids_to_remove.append(pid)if not ids_to_remove:return Falseprint(@@ -1802,6 +1823,15 @@ Examples:print(f"Index updated at {index_path}")return True+ def _path_lookup_keys(self, path: str, roots: list[str]) -> list[str]:+ """Return path variations for lookup (sync paths may be relative to roots)."""+ keys = [path, _normalize_path(path)]+ for root in roots:+ candidate = str((Path(root) / path).resolve())+ if candidate not in keys:+ keys.append(candidate)+ return keys+def _incremental_ivf_update(self,index_path: str,@@ -1811,22 +1841,41 @@ Examples:new_paths: set[str],removed_paths: set[str],modified_paths: set[str],+ sync_roots: list[str],) -> bool:"""IVF incremental update: remove old chunks for modified/removed files, add new chunks."""passages_file = index_dir / "documents.leann.passages.jsonl"+ offset_file = index_dir / "documents.leann.passages.idx"+ live_ids: set[str] | None = None+ if offset_file.exists():+ with open(offset_file, "rb") as f:+ live_ids = set(pickle.load(f).keys())chunk_ids_by_file = (- self._load_chunk_ids_by_file(passages_file) if passages_file.exists() else {}+ self._load_chunk_ids_by_file(passages_file, live_ids=live_ids)+ if passages_file.exists()+ else {})# Collect old chunk IDs to remove (modified + removed files)+ # Try all path variations: same file can have different path formats in passagesids_to_remove: list[str] = []+ seen_ids: set[str] = set()for p in modified_paths | removed_paths:- ids_to_remove.extend(chunk_ids_by_file.get(p, []))+ for key in self._path_lookup_keys(p, sync_roots):+ for pid in chunk_ids_by_file.get(key, []):+ if pid not in seen_ids:+ seen_ids.add(pid)+ ids_to_remove.append(pid)# Collect new chunks to add (modified + new files)+ # Build path set for matching: chunks may have paths in different formatschanged_paths = new_paths | modified_paths- new_chunks = self._chunks_for_paths(all_texts, changed_paths)- self._assign_chunk_ids(new_chunks)+ path_set: set[str] = set()+ for p in changed_paths:+ path_set.update(self._path_lookup_keys(p, sync_roots))+ new_chunks = self._chunks_for_paths(all_texts, path_set)+ # Use unique IDs: passages can have mixed path formats so we may miss some ids_to_remove+ self._assign_unique_chunk_ids(new_chunks)if not ids_to_remove and not new_chunks:return False@@ -1913,6 +1962,18 @@ Examples:with open(sync_config_path, "w", encoding="utf-8") as f:json.dump(config, f, indent=2)+ def _load_sync_roots(self, index_dir: Path) -> list[str]:+ """Load sync roots from index dir (for path resolution in incremental updates)."""+ sync_config_path = index_dir / "sync_roots.json"+ if not sync_config_path.exists():+ return []+ try:+ with open(sync_config_path, encoding="utf-8") as f:+ config = json.load(f)+ return config.get("roots") or []+ except (json.JSONDecodeError, OSError):+ return []+def _resolve_index_for_watch(self, index_name: str) -> Optional[dict[str, Path]]:if self.index_exists(index_name):index_dir = self.indexes_dir / index_name@@ -1950,7 +2011,14 @@ Examples:return {"index_dir": index_dir, "passages_file": passages_file}- def _load_chunk_ids_by_file(self, passages_file: Path) -> dict[str, list[str]]:+ def _load_chunk_ids_by_file(+ self, passages_file: Path, live_ids: set[str] | None = None+ ) -> dict[str, list[str]]:+ """Load chunk IDs grouped by file path from passages.jsonl.++ If *live_ids* is provided, skip entries whose ID is not in the set+ (filters out stale entries left by prior incremental updates).+ """chunk_ids_by_file: dict[str, list[str]] = {}with open(passages_file, encoding="utf-8") as f:for line in f:@@ -1968,6 +2036,8 @@ Examples:chunk_id = data.get("id")if chunk_id is None:continue+ if live_ids is not None and str(chunk_id) not in live_ids:+ continuenormalized_path = str(Path(file_path).resolve())chunk_ids_by_file.setdefault(normalized_path, []).append(str(chunk_id))if file_path != normalized_path:@@ -2097,9 +2167,24 @@ Examples:return# Load only changed files (no need to load/chunk the entire corpus)+ # Resolve paths relative to sync roots (sync returns paths relative to each root)+ roots = self._resolve_sync_roots(docs_paths)paths_to_load = new_paths | modified_paths+ resolved_paths: list[str] = []+ for p in paths_to_load:+ path_obj = Path(p)+ if path_obj.is_absolute() and path_obj.exists():+ resolved_paths.append(p)+ else:+ for root in roots:+ candidate = Path(root) / p+ if candidate.exists():+ resolved_paths.append(str(candidate.resolve()))+ break+ else:+ resolved_paths.append(p) # fallback: pass as-isall_texts = self.load_documents(- list(paths_to_load),+ resolved_paths,args.file_types,include_hidden=args.include_hidden,args=args,@@ -2118,6 +2203,7 @@ Examples:new_paths,removed_paths,modified_paths,+ roots,)if result:self._commit_synchronizers(synchronizers)diff --git a/tests/test_incremental_build.py b/tests/test_incremental_build.pyindex a7922ed..63ec0ea 100644--- a/tests/test_incremental_build.py+++ b/tests/test_incremental_build.py@@ -240,3 +240,134 @@ def test_ivf_incremental_add_then_remove_searchable(tmp_path):assert all(unique_phrase not in r.text for r in results), ("Deleted content should not appear in search results")+++ os.environ.get("CI") == "true",+ reason="Skip in CI to avoid embedding/model load",+)+def test_ivf_multiple_incremental_no_duplicates(tmp_path):+ """IVF: modifying the same file across multiple incremental builds must not create duplicate chunks."""+ import asyncio+ import json+ import pickle++ from leann.api import LeannSearcher+ from leann.cli import LeannCLI++ docs_dir = tmp_path / "docs"+ docs_dir.mkdir()++ target_phrase_v1 = "UNIQUE_TARGET_V1_ALPHA_BRAVO"+ target_phrase_v2 = "UNIQUE_TARGET_V2_CHARLIE_DELTA"+ target_phrase_v3 = "UNIQUE_TARGET_V3_ECHO_FOXTROT"++ (docs_dir / "target.txt").write_text(+ f"Version one content.\n\n{target_phrase_v1}\n\nEnd of version one.",+ encoding="utf-8",+ )+ # Filler files for IVF training (nlist=100)+ for i in range(110):+ (docs_dir / f"filler_{i:03d}.txt").write_text(+ f"Filler document {i} with unique content for padding the IVF index.",+ encoding="utf-8",+ )++ cli = LeannCLI()+ cli.indexes_dir = tmp_path / ".leann" / "indexes"+ cli.indexes_dir.mkdir(parents=True, exist_ok=True)+ index_name = "ivf_dup_test"+ index_path = cli.get_index_path(index_name)+ index_dir = cli.indexes_dir / index_name++ parser = cli.create_parser()+ build_args = [+ "build",+ index_name,+ "--docs",+ str(docs_dir),+ "--backend-name",+ "ivf",+ "--embedding-model",+ "all-MiniLM-L6-v2",+ "--embedding-mode",+ "sentence-transformers",+ ]++ # --- Initial build (--force) ---+ asyncio.run(cli.build_index(parser.parse_args([*build_args, "--force"])))++ searcher = LeannSearcher(index_path)+ results = searcher.search(target_phrase_v1, top_k=10)+ searcher.cleanup()+ v1_hits = [r for r in results if target_phrase_v1 in r.text]+ assert len(v1_hits) >= 1, "V1 content should be searchable after initial build"++ # --- Modify target file to V2, incremental build ---+ (docs_dir / "target.txt").write_text(+ f"Version two content.\n\n{target_phrase_v2}\n\nEnd of version two.",+ encoding="utf-8",+ )+ asyncio.run(cli.build_index(parser.parse_args(build_args)))++ searcher = LeannSearcher(index_path)+ results_v2 = searcher.search(target_phrase_v1, top_k=10)+ searcher.cleanup()+ assert all(target_phrase_v1 not in r.text for r in results_v2), (+ "V1 content should be gone after first incremental update"+ )++ searcher = LeannSearcher(index_path)+ results_v2b = searcher.search(target_phrase_v2, top_k=10)+ searcher.cleanup()+ v2_hits = [r for r in results_v2b if target_phrase_v2 in r.text]+ assert len(v2_hits) >= 1, "V2 content should be searchable"++ # --- Modify target file to V3, second incremental build ---+ (docs_dir / "target.txt").write_text(+ f"Version three content.\n\n{target_phrase_v3}\n\nEnd of version three.",+ encoding="utf-8",+ )+ asyncio.run(cli.build_index(parser.parse_args(build_args)))++ # V1 and V2 should be gone, only V3 present+ searcher = LeannSearcher(index_path)+ results_v3_check_v1 = searcher.search(target_phrase_v1, top_k=10)+ searcher.cleanup()+ assert all(target_phrase_v1 not in r.text for r in results_v3_check_v1), (+ "V1 content should NOT appear after two incremental updates"+ )++ searcher = LeannSearcher(index_path)+ results_v3_check_v2 = searcher.search(target_phrase_v2, top_k=10)+ searcher.cleanup()+ assert all(target_phrase_v2 not in r.text for r in results_v3_check_v2), (+ "V2 content should NOT appear after second incremental update"+ )++ searcher = LeannSearcher(index_path)+ results_v3 = searcher.search(target_phrase_v3, top_k=10)+ searcher.cleanup()+ v3_hits = [r for r in results_v3 if target_phrase_v3 in r.text]+ assert len(v3_hits) >= 1, "V3 content should be searchable"++ # --- Verify passages.jsonl has no stale entries ---+ passages_file = index_dir / "documents.leann.passages.jsonl"+ offset_file = index_dir / "documents.leann.passages.idx"+ with open(offset_file, "rb") as f:+ offset_map = pickle.load(f)+ live_ids = set(offset_map.keys())++ jsonl_ids = []+ with open(passages_file, encoding="utf-8") as f:+ for line in f:+ line = line.strip()+ if not line:+ continue+ data = json.loads(line)+ jsonl_ids.append(data["id"])++ stale_ids = [pid for pid in jsonl_ids if pid not in live_ids]+ assert len(stale_ids) == 0, (+ f"passages.jsonl has {len(stale_ids)} stale entries not in offset_map: {stale_ids[:5]}"+ )