Skip to content

armgit / LEANN

Public Imported from GitHub
0

[MLsys2026 Best Paper]: https://arxiv.org/abs/2506.08276. RAG on Everything with LEANN. Enjoy 97% storage savings while running a fast, accurate, and 100% private RAG application on your personal device.

fix: IVF incremental update duplicate chunks from stale passages.jsonl (#277)

Yichuan Wangb30f382c29c2
diff --git a/.gitignore b/.gitignore
index 46ce36c..1c55812 100755
--- a/.gitignore
+++ b/.gitignore
@@ -117,4 +117,6 @@ apps/multimodal/vision-based-pdf-multi-vector/multi-vector-colpali-native-weavia
paru-bin/
merkle-tree-test/
test-code/
+localtestmcp/
+*.csv
*.pickle
diff --git a/packages/leann-backend-ivf/leann_backend_ivf/ivf_backend.py b/packages/leann-backend-ivf/leann_backend_ivf/ivf_backend.py
index ca893da..23c1411 100644
--- a/packages/leann-backend-ivf/leann_backend_ivf/ivf_backend.py
+++ b/packages/leann-backend-ivf/leann_backend_ivf/ivf_backend.py
@@ -263,6 +263,7 @@ def remove_ids(index_path: str, passage_ids: list[str]) -> int:
return 0
index = faiss.read_index(str(index_file))
+ ntotal_before = index.ntotal
sel = np.array(to_remove_int, dtype=np.int64)
nremoved = index.remove_ids(sel)
faiss.write_index(index, str(index_file))
@@ -273,5 +274,18 @@ def remove_ids(index_path: str, passage_ids: list[str]) -> int:
id_to_passage.pop(i, None)
# passage_to_id is rebuilt from id_to_passage when we save; we don't decrease next_id so new adds get new ids
_save_id_map(index_dir, index_prefix, id_to_passage, next_id=next_id)
- logger.info("IVF remove_ids: removed %d vectors (requested %d)", nremoved, len(passage_ids))
+ logger.info(
+ "IVF remove_ids: ntotal %d -> %d, removed %d vectors (requested %d, found %d in id_map)",
+ ntotal_before,
+ index.ntotal,
+ nremoved,
+ len(passage_ids),
+ len(to_remove_int),
+ )
+ if nremoved != len(to_remove_int):
+ logger.warning(
+ "IVF remove_ids: FAISS removed %d but expected %d. Possible index inconsistency.",
+ nremoved,
+ len(to_remove_int),
+ )
return nremoved
diff --git a/packages/leann-core/src/leann/api.py b/packages/leann-core/src/leann/api.py
index 743adc0..9924f52 100644
--- a/packages/leann-core/src/leann/api.py
+++ b/packages/leann-core/src/leann/api.py
@@ -718,6 +718,31 @@ class LeannBuilder:
logger.info(f"Index built successfully from precomputed embeddings: {index_path}")
+ @staticmethod
+ def _compact_passages(
+ passages_file: Path, offset_file: Path, offset_map: dict[str, int]
+ ) -> None:
+ """Rewrite passages.jsonl keeping only entries referenced by offset_map."""
+ live_entries: list[str] = []
+ for _pid, offset in sorted(offset_map.items(), key=lambda x: x[1]):
+ with open(passages_file, encoding="utf-8") as f:
+ f.seek(offset)
+ live_entries.append(f.readline())
+
+ tmp_file = passages_file.with_suffix(".jsonl.tmp")
+ new_offset_map: dict[str, int] = {}
+ with open(tmp_file, "w", encoding="utf-8") as f:
+ for line in live_entries:
+ data = json.loads(line)
+ new_offset_map[data["id"]] = f.tell()
+ f.write(line if line.endswith("\n") else line + "\n")
+
+ tmp_file.replace(passages_file)
+ offset_map.clear()
+ offset_map.update(new_offset_map)
+ with open(offset_file, "wb") as f:
+ pickle.dump(offset_map, f)
+
def update_index(self, index_path: str, remove_passage_ids: Optional[list[str]] = None) -> None:
"""Append new passages and vectors to an existing index (HNSW or IVF).
For IVF, optional remove_passage_ids removes those ids first (e.g. from file-change API).
@@ -757,7 +782,14 @@ class LeannBuilder:
try:
from leann_backend_ivf import remove_ids as ivf_remove_ids
- ivf_remove_ids(str(path), remove_passage_ids)
+ nremoved = ivf_remove_ids(str(path), remove_passage_ids)
+ if nremoved < len(remove_passage_ids):
+ logger.warning(
+ "IVF update_index: removed %d of %d requested passage IDs "
+ "(some may have been stale).",
+ nremoved,
+ len(remove_passage_ids),
+ )
except ImportError:
raise RuntimeError(
"IVF backend required for remove_ids. Install leann-backend-ivf."
@@ -765,8 +797,9 @@ class LeannBuilder:
for pid in remove_passage_ids:
offset_map.pop(pid, None)
existing_ids -= set(remove_passage_ids)
- with open(offset_file, "wb") as f:
- pickle.dump(offset_map, f)
+
+ # Compact passages.jsonl: rewrite keeping only entries in offset_map
+ self._compact_passages(passages_file, offset_file, offset_map)
if not self.chunks:
meta["total_passages"] = len(offset_map)
diff --git a/packages/leann-core/src/leann/cli.py b/packages/leann-core/src/leann/cli.py
index 01e6c0b..4afb0c3 100644
--- a/packages/leann-core/src/leann/cli.py
+++ b/packages/leann-core/src/leann/cli.py
@@ -5,8 +5,10 @@ import hashlib
import io
import json
import os
+import pickle
import sys
import time
+import uuid
from pathlib import Path
from typing import Any, Optional, Union
@@ -1735,6 +1737,14 @@ Examples:
c.setdefault("metadata", {})["id"] = sid
c["id"] = sid
+ @staticmethod
+ def _assign_unique_chunk_ids(chunks: list[dict]) -> None:
+ """Assign unique IDs for incremental (avoids collision when path lookup misses some old ids)."""
+ for c in chunks:
+ sid = uuid.uuid4().hex[:16]
+ c.setdefault("metadata", {})["id"] = sid
+ c["id"] = sid
+
def _chunks_for_paths(self, all_texts: list[dict], paths: set[str]) -> list[dict]:
"""Filter chunks belonging to the given file paths."""
return [
@@ -1788,10 +1798,21 @@ Examples:
passages_file = index_dir / "documents.leann.passages.jsonl"
if not passages_file.exists():
return False
- chunk_ids_by_file = self._load_chunk_ids_by_file(passages_file)
+ offset_file = index_dir / "documents.leann.passages.idx"
+ live_ids: set[str] | None = None
+ if offset_file.exists():
+ with open(offset_file, "rb") as f:
+ live_ids = set(pickle.load(f).keys())
+ chunk_ids_by_file = self._load_chunk_ids_by_file(passages_file, live_ids=live_ids)
+ roots = self._load_sync_roots(index_dir)
ids_to_remove: list[str] = []
+ seen_ids: set[str] = set()
for p in removed_paths:
- ids_to_remove.extend(chunk_ids_by_file.get(p, []))
+ for key in self._path_lookup_keys(p, roots):
+ for pid in chunk_ids_by_file.get(key, []):
+ if pid not in seen_ids:
+ seen_ids.add(pid)
+ ids_to_remove.append(pid)
if not ids_to_remove:
return False
print(
@@ -1802,6 +1823,15 @@ Examples:
print(f"Index updated at {index_path}")
return True
+ def _path_lookup_keys(self, path: str, roots: list[str]) -> list[str]:
+ """Return path variations for lookup (sync paths may be relative to roots)."""
+ keys = [path, _normalize_path(path)]
+ for root in roots:
+ candidate = str((Path(root) / path).resolve())
+ if candidate not in keys:
+ keys.append(candidate)
+ return keys
+
def _incremental_ivf_update(
self,
index_path: str,
@@ -1811,22 +1841,41 @@ Examples:
new_paths: set[str],
removed_paths: set[str],
modified_paths: set[str],
+ sync_roots: list[str],
) -> bool:
"""IVF incremental update: remove old chunks for modified/removed files, add new chunks."""
passages_file = index_dir / "documents.leann.passages.jsonl"
+ offset_file = index_dir / "documents.leann.passages.idx"
+ live_ids: set[str] | None = None
+ if offset_file.exists():
+ with open(offset_file, "rb") as f:
+ live_ids = set(pickle.load(f).keys())
chunk_ids_by_file = (
- self._load_chunk_ids_by_file(passages_file) if passages_file.exists() else {}
+ self._load_chunk_ids_by_file(passages_file, live_ids=live_ids)
+ if passages_file.exists()
+ else {}
)
# Collect old chunk IDs to remove (modified + removed files)
+ # Try all path variations: same file can have different path formats in passages
ids_to_remove: list[str] = []
+ seen_ids: set[str] = set()
for p in modified_paths | removed_paths:
- ids_to_remove.extend(chunk_ids_by_file.get(p, []))
+ for key in self._path_lookup_keys(p, sync_roots):
+ for pid in chunk_ids_by_file.get(key, []):
+ if pid not in seen_ids:
+ seen_ids.add(pid)
+ ids_to_remove.append(pid)
# Collect new chunks to add (modified + new files)
+ # Build path set for matching: chunks may have paths in different formats
changed_paths = new_paths | modified_paths
- new_chunks = self._chunks_for_paths(all_texts, changed_paths)
- self._assign_chunk_ids(new_chunks)
+ path_set: set[str] = set()
+ for p in changed_paths:
+ path_set.update(self._path_lookup_keys(p, sync_roots))
+ new_chunks = self._chunks_for_paths(all_texts, path_set)
+ # Use unique IDs: passages can have mixed path formats so we may miss some ids_to_remove
+ self._assign_unique_chunk_ids(new_chunks)
if not ids_to_remove and not new_chunks:
return False
@@ -1913,6 +1962,18 @@ Examples:
with open(sync_config_path, "w", encoding="utf-8") as f:
json.dump(config, f, indent=2)
+ def _load_sync_roots(self, index_dir: Path) -> list[str]:
+ """Load sync roots from index dir (for path resolution in incremental updates)."""
+ sync_config_path = index_dir / "sync_roots.json"
+ if not sync_config_path.exists():
+ return []
+ try:
+ with open(sync_config_path, encoding="utf-8") as f:
+ config = json.load(f)
+ return config.get("roots") or []
+ except (json.JSONDecodeError, OSError):
+ return []
+
def _resolve_index_for_watch(self, index_name: str) -> Optional[dict[str, Path]]:
if self.index_exists(index_name):
index_dir = self.indexes_dir / index_name
@@ -1950,7 +2011,14 @@ Examples:
return {"index_dir": index_dir, "passages_file": passages_file}
- def _load_chunk_ids_by_file(self, passages_file: Path) -> dict[str, list[str]]:
+ def _load_chunk_ids_by_file(
+ self, passages_file: Path, live_ids: set[str] | None = None
+ ) -> dict[str, list[str]]:
+ """Load chunk IDs grouped by file path from passages.jsonl.
+
+ If *live_ids* is provided, skip entries whose ID is not in the set
+ (filters out stale entries left by prior incremental updates).
+ """
chunk_ids_by_file: dict[str, list[str]] = {}
with open(passages_file, encoding="utf-8") as f:
for line in f:
@@ -1968,6 +2036,8 @@ Examples:
chunk_id = data.get("id")
if chunk_id is None:
continue
+ if live_ids is not None and str(chunk_id) not in live_ids:
+ continue
normalized_path = str(Path(file_path).resolve())
chunk_ids_by_file.setdefault(normalized_path, []).append(str(chunk_id))
if file_path != normalized_path:
@@ -2097,9 +2167,24 @@ Examples:
return
# Load only changed files (no need to load/chunk the entire corpus)
+ # Resolve paths relative to sync roots (sync returns paths relative to each root)
+ roots = self._resolve_sync_roots(docs_paths)
paths_to_load = new_paths | modified_paths
+ resolved_paths: list[str] = []
+ for p in paths_to_load:
+ path_obj = Path(p)
+ if path_obj.is_absolute() and path_obj.exists():
+ resolved_paths.append(p)
+ else:
+ for root in roots:
+ candidate = Path(root) / p
+ if candidate.exists():
+ resolved_paths.append(str(candidate.resolve()))
+ break
+ else:
+ resolved_paths.append(p) # fallback: pass as-is
all_texts = self.load_documents(
- list(paths_to_load),
+ resolved_paths,
args.file_types,
include_hidden=args.include_hidden,
args=args,
@@ -2118,6 +2203,7 @@ Examples:
new_paths,
removed_paths,
modified_paths,
+ roots,
)
if result:
self._commit_synchronizers(synchronizers)
diff --git a/tests/test_incremental_build.py b/tests/test_incremental_build.py
index a7922ed..63ec0ea 100644
--- a/tests/test_incremental_build.py
+++ b/tests/test_incremental_build.py
@@ -240,3 +240,134 @@ def test_ivf_incremental_add_then_remove_searchable(tmp_path):
assert all(unique_phrase not in r.text for r in results), (
"Deleted content should not appear in search results"
)
+
+
+ os.environ.get("CI") == "true",
+ reason="Skip in CI to avoid embedding/model load",
+)
+def test_ivf_multiple_incremental_no_duplicates(tmp_path):
+ """IVF: modifying the same file across multiple incremental builds must not create duplicate chunks."""
+ import asyncio
+ import json
+ import pickle
+
+ from leann.api import LeannSearcher
+ from leann.cli import LeannCLI
+
+ docs_dir = tmp_path / "docs"
+ docs_dir.mkdir()
+
+ target_phrase_v1 = "UNIQUE_TARGET_V1_ALPHA_BRAVO"
+ target_phrase_v2 = "UNIQUE_TARGET_V2_CHARLIE_DELTA"
+ target_phrase_v3 = "UNIQUE_TARGET_V3_ECHO_FOXTROT"
+
+ (docs_dir / "target.txt").write_text(
+ f"Version one content.\n\n{target_phrase_v1}\n\nEnd of version one.",
+ encoding="utf-8",
+ )
+ # Filler files for IVF training (nlist=100)
+ for i in range(110):
+ (docs_dir / f"filler_{i:03d}.txt").write_text(
+ f"Filler document {i} with unique content for padding the IVF index.",
+ encoding="utf-8",
+ )
+
+ cli = LeannCLI()
+ cli.indexes_dir = tmp_path / ".leann" / "indexes"
+ cli.indexes_dir.mkdir(parents=True, exist_ok=True)
+ index_name = "ivf_dup_test"
+ index_path = cli.get_index_path(index_name)
+ index_dir = cli.indexes_dir / index_name
+
+ parser = cli.create_parser()
+ build_args = [
+ "build",
+ index_name,
+ "--docs",
+ str(docs_dir),
+ "--backend-name",
+ "ivf",
+ "--embedding-model",
+ "all-MiniLM-L6-v2",
+ "--embedding-mode",
+ "sentence-transformers",
+ ]
+
+ # --- Initial build (--force) ---
+ asyncio.run(cli.build_index(parser.parse_args([*build_args, "--force"])))
+
+ searcher = LeannSearcher(index_path)
+ results = searcher.search(target_phrase_v1, top_k=10)
+ searcher.cleanup()
+ v1_hits = [r for r in results if target_phrase_v1 in r.text]
+ assert len(v1_hits) >= 1, "V1 content should be searchable after initial build"
+
+ # --- Modify target file to V2, incremental build ---
+ (docs_dir / "target.txt").write_text(
+ f"Version two content.\n\n{target_phrase_v2}\n\nEnd of version two.",
+ encoding="utf-8",
+ )
+ asyncio.run(cli.build_index(parser.parse_args(build_args)))
+
+ searcher = LeannSearcher(index_path)
+ results_v2 = searcher.search(target_phrase_v1, top_k=10)
+ searcher.cleanup()
+ assert all(target_phrase_v1 not in r.text for r in results_v2), (
+ "V1 content should be gone after first incremental update"
+ )
+
+ searcher = LeannSearcher(index_path)
+ results_v2b = searcher.search(target_phrase_v2, top_k=10)
+ searcher.cleanup()
+ v2_hits = [r for r in results_v2b if target_phrase_v2 in r.text]
+ assert len(v2_hits) >= 1, "V2 content should be searchable"
+
+ # --- Modify target file to V3, second incremental build ---
+ (docs_dir / "target.txt").write_text(
+ f"Version three content.\n\n{target_phrase_v3}\n\nEnd of version three.",
+ encoding="utf-8",
+ )
+ asyncio.run(cli.build_index(parser.parse_args(build_args)))
+
+ # V1 and V2 should be gone, only V3 present
+ searcher = LeannSearcher(index_path)
+ results_v3_check_v1 = searcher.search(target_phrase_v1, top_k=10)
+ searcher.cleanup()
+ assert all(target_phrase_v1 not in r.text for r in results_v3_check_v1), (
+ "V1 content should NOT appear after two incremental updates"
+ )
+
+ searcher = LeannSearcher(index_path)
+ results_v3_check_v2 = searcher.search(target_phrase_v2, top_k=10)
+ searcher.cleanup()
+ assert all(target_phrase_v2 not in r.text for r in results_v3_check_v2), (
+ "V2 content should NOT appear after second incremental update"
+ )
+
+ searcher = LeannSearcher(index_path)
+ results_v3 = searcher.search(target_phrase_v3, top_k=10)
+ searcher.cleanup()
+ v3_hits = [r for r in results_v3 if target_phrase_v3 in r.text]
+ assert len(v3_hits) >= 1, "V3 content should be searchable"
+
+ # --- Verify passages.jsonl has no stale entries ---
+ passages_file = index_dir / "documents.leann.passages.jsonl"
+ offset_file = index_dir / "documents.leann.passages.idx"
+ with open(offset_file, "rb") as f:
+ offset_map = pickle.load(f)
+ live_ids = set(offset_map.keys())
+
+ jsonl_ids = []
+ with open(passages_file, encoding="utf-8") as f:
+ for line in f:
+ line = line.strip()
+ if not line:
+ continue
+ data = json.loads(line)
+ jsonl_ids.append(data["id"])
+
+ stale_ids = [pid for pid in jsonl_ids if pid not in live_ids]
+ assert len(stale_ids) == 0, (
+ f"passages.jsonl has {len(stale_ids)} stale entries not in offset_map: {stale_ids[:5]}"
+ )