[MLsys2026 Best Paper]: https://arxiv.org/abs/2506.08276. RAG on Everything with LEANN. Enjoy 97% storage savings while running a fast, accurate, and 100% private RAG application on your personal device.
Python aifaissgpt-osslangchainllama-indexllmlocalstorageoffline-firstollamaprivacypythonragretrieval-augmented-generationvector-databasevector-searchvectors
feat: support content-hash passage IDs
Abi2d430a445d16
diff --git a/packages/leann-core/src/leann/api.py b/packages/leann-core/src/leann/api.pyindex e22633a..b5ee65d 100644--- a/packages/leann-core/src/leann/api.py+++ b/packages/leann-core/src/leann/api.py@@ -30,6 +30,15 @@ from .registry import BACKEND_REGISTRYlogger = logging.getLogger(__name__)+# Passage ID schemes recorded in <index>.meta.json["passage_id_scheme"].+# - "sequential": today's default; IDs are str(insertion_index) (api.py:add_text).+# - "content-hash": planned in #329; IDs are sha256(text)[:16], stable across+# file moves and reorderings.+# Older indexes have no passage_id_scheme field — readers must default to+# "sequential" when the key is absent. See #329 for the rollout plan.+PASSAGE_ID_SCHEME_SEQUENTIAL = "sequential"+PASSAGE_ID_SCHEME_CONTENT_HASH = "content-hash"+def get_registered_backends() -> list[str]:"""Get list of registered backend names."""@@ -376,6 +385,7 @@ class LeannBuilder:embedding_options: Optional[dict[str, Any]] = None,prebuild_bm25: bool = False,bm25_backend: str = "fts5",+ passage_id_scheme: str = PASSAGE_ID_SCHEME_SEQUENTIAL,**backend_kwargs,):if bm25_backend != "fts5":@@ -383,6 +393,16 @@ class LeannBuilder:bm25_backend = "fts5"self.bm25_backend = bm25_backendself.prebuild_bm25 = prebuild_bm25 or bm25_backend == "fts5"+ if passage_id_scheme not in (+ PASSAGE_ID_SCHEME_SEQUENTIAL,+ PASSAGE_ID_SCHEME_CONTENT_HASH,+ ):+ raise ValueError(+ f"Unknown passage_id_scheme: {passage_id_scheme!r}. "+ f"Expected one of: {PASSAGE_ID_SCHEME_SEQUENTIAL!r}, "+ f"{PASSAGE_ID_SCHEME_CONTENT_HASH!r}."+ )+ self.passage_id_scheme = passage_id_schemeself.backend_name = backend_name# Normalize incompatible combinations early (for consistent metadata)if backend_name == "hnsw":@@ -477,10 +497,23 @@ class LeannBuilder:self.backend_kwargs = backend_kwargsself.chunks: list[dict[str, Any]] = []+ def _generate_passage_id(self, text: str) -> str:+ """Generate a passage ID per the configured scheme.++ sequential: str(insertion index) — fast, position-dependent, current default.+ content-hash: sha256(text)[:16] — content-stable, dedup-friendly across+ file moves and reorderings. See #329 for the design.+ """+ if self.passage_id_scheme == PASSAGE_ID_SCHEME_CONTENT_HASH:+ import hashlib++ return hashlib.sha256(text.encode("utf-8")).hexdigest()[:16]+ return str(len(self.chunks))+def add_text(self, text: str, metadata: Optional[dict[str, Any]] = None):if metadata is None:metadata = {}- passage_id = metadata.get("id", str(len(self.chunks)))+ passage_id = metadata.get("id") or self._generate_passage_id(text)chunk_data = {"id": passage_id, "text": text, "metadata": metadata}self.chunks.append(chunk_data)@@ -570,12 +603,13 @@ class LeannBuilder:builder_instance.build(embeddings, string_ids, index_path, **current_backend_kwargs)leann_meta_path = index_dir / f"{index_name}.meta.json"meta_data = {- "version": "1.0",+ "version": "1.1","backend_name": self.backend_name,"embedding_model": self.embedding_model,"dimensions": self.dimensions,"backend_kwargs": self.backend_kwargs,"embedding_mode": self.embedding_mode,+ "passage_id_scheme": self.passage_id_scheme,"passage_sources": [{"type": "jsonl",@@ -714,12 +748,13 @@ class LeannBuilder:# Create metadata fileleann_meta_path = index_dir / f"{index_name}.meta.json"meta_data = {- "version": "1.0",+ "version": "1.1","backend_name": self.backend_name,"embedding_model": self.embedding_model,"dimensions": self.dimensions,"backend_kwargs": self.backend_kwargs,"embedding_mode": self.embedding_mode,+ "passage_id_scheme": self.passage_id_scheme,"passage_sources": [{"type": "jsonl",@@ -1177,6 +1212,12 @@ class LeannSearcher:)self.bm25_scorer: Optional[BM25Index] = None+ # Surface the index's passage ID scheme so callers can introspect.+ # Older indexes (pre-#330) don't record this field — they're sequential.+ self.passage_id_scheme: str = self.meta_data.get(+ "passage_id_scheme", PASSAGE_ID_SCHEME_SEQUENTIAL+ )+# Optional one-shot warmup at construction time to hide cold-start latency.if self._warmup:self.warmup()diff --git a/packages/leann-core/src/leann/cli.py b/packages/leann-core/src/leann/cli.pyindex 0d50cf3..0b56acc 100644--- a/packages/leann-core/src/leann/cli.py+++ b/packages/leann-core/src/leann/cli.py@@ -16,7 +16,7 @@ from llama_index.core import SimpleDirectoryReaderfrom llama_index.core.node_parser import SentenceSplitterfrom tqdm import tqdm-from .api import LeannBuilder, LeannChat, LeannSearcher+from .api import Fts5BM25Index, LeannBuilder, LeannChat, LeannSearcherfrom .embedding_server_manager import EmbeddingServerManagerfrom .interactive_utils import create_cli_sessionfrom .registry import register_project_directory@@ -347,6 +347,16 @@ Examples:default=True,help="Fall back to traditional chunking if AST chunking fails (default: True)",)+ build_parser.add_argument(+ "--id-scheme",+ choices=["sequential", "content-hash"],+ default="sequential",+ help=(+ "How passage IDs are assigned. 'sequential' (default) keys by insertion "+ "order; 'content-hash' uses sha256(text)[:16], stable across file moves "+ "and reorderings. See #329."+ ),+ )# Watch commandwatch_parser = subparsers.add_parser(@@ -371,6 +381,26 @@ Examples:help="Report changes without rebuilding (original watch behavior)",)+ migrate_parser = subparsers.add_parser(+ "migrate-ids",+ help=(+ "Rewrite an existing index's passage IDs to content-hash form. "+ "Irreversible; back up the index first."+ ),+ )+ migrate_parser.add_argument("index_name", help="Index name")+ migrate_parser.add_argument(+ "--dry-run",+ action="store_true",+ help="Show what would change without writing anything.",+ )+ migrate_parser.add_argument(+ "-y",+ "--yes",+ action="store_true",+ help="Skip the interactive confirmation prompt.",+ )+# Search commandsearch_parser = subparsers.add_parser("search", help="Search documents")search_parser.add_argument("index_name", help="Index name")@@ -1880,7 +1910,32 @@ Examples:in paths]+ def _existing_index_id_scheme(self, index_path: str) -> Optional[str]:+ """Return the passage_id_scheme recorded in an existing index's meta.json.++ Returns None when the index doesn't exist yet or the field isn't+ recorded (older indexes pre-#330). Callers should treat None as+ "fall back to whatever the args say or the default".+ """+ meta_path = Path(index_path).with_suffix(".leann.meta.json")+ if not meta_path.exists():+ return None+ try:+ with open(meta_path, encoding="utf-8") as f:+ return json.load(f).get("passage_id_scheme")+ except Exception:+ return None+def _make_incremental_builder(self, args) -> "LeannBuilder":+ # For incremental updates, the existing index's scheme wins. Otherwise+ # IDs would mix schemes within one index, which breaks lookups.+ existing_scheme = self._existing_index_id_scheme(self.get_index_path(args.index_name))+ scheme = existing_scheme or getattr(args, "id_scheme", "sequential")+ if existing_scheme and getattr(args, "id_scheme", existing_scheme) != existing_scheme:+ print(+ f"Note: --id-scheme={args.id_scheme} ignored — index '{args.index_name}' "+ f"was built with passage_id_scheme={existing_scheme!r}, keeping that."+ )return LeannBuilder(backend_name=args.backend_name,embedding_model=args.embedding_model,@@ -1891,6 +1946,7 @@ Examples:is_compact=args.compact,is_recompute=args.recompute,num_threads=args.num_threads,+ passage_id_scheme=scheme,)def _incremental_add_only(@@ -2384,6 +2440,7 @@ Examples:is_compact=args.compact,is_recompute=args.recompute,num_threads=args.num_threads,+ passage_id_scheme=getattr(args, "id_scheme", "sequential"),)for chunk in all_texts:@@ -2504,6 +2561,150 @@ Examples:build_args = parser.parse_args(build_args_list)await self.build_index(build_args)+ def migrate_ids(self, args) -> None:+ """Rewrite an existing index's passage IDs to content-hash form.++ Migration is purely a Python-side rewrite — the vector graph isn't+ touched, so FAISS labels stay valid. What gets rewritten:+ - <index>.passages.jsonl : new IDs in each line's "id" field+ - <index>.passages.idx : new offset map keyed by new IDs+ - <index>.ids.txt : new label → ID mapping for FAISS+ - <index>.meta.json : passage_id_scheme = "content-hash"+ - <index>.bm25.sqlite : BM25 IDs, when an FTS5 artifact is configured++ Identical-text chunks collide on the same sha256 prefix; the later+ occurrence wins in the offset map (dedup). A `--preserve-duplicates`+ knob to suffix collisions can land separately.++ Irreversible. Prompts for confirmation unless --yes is passed.+ """+ import hashlib+ import shutil++ index_name = args.index_name+ index_path = self._resolve_index_path(index_name, purpose="migrate")+ if not index_path:+ return+ meta_path = Path(index_path).with_suffix(".leann.meta.json")+ if not meta_path.exists():+ print(f"Cannot migrate '{index_name}': metadata missing at {meta_path}.")+ return+ with open(meta_path, encoding="utf-8") as f:+ meta = json.load(f)+ current_scheme = meta.get("passage_id_scheme", "sequential")+ if current_scheme == "content-hash":+ print(f"Index '{index_name}' already uses content-hash IDs. Nothing to do.")+ return++ # Locate the sibling artifacts using the same conventions as build_index.+ index_dir = Path(index_path).parent+ index_base = Path(index_path).name+ passages_file = index_dir / f"{index_base}.passages.jsonl"+ offset_file = index_dir / f"{index_base}.passages.idx"+ base_no_leann = (+ index_base[: -len(".leann")] if index_base.endswith(".leann") else index_base+ )+ idmap_file = index_dir / f"{base_no_leann}.ids.txt"++ for p in (passages_file, offset_file):+ if not p.exists():+ print(f"Cannot migrate: required artifact missing at {p}.")+ return++ # Stream the passages to plan the rewrite and surface collision count+ # before committing to anything irreversible.+ old_ids: list[str] = []+ new_ids: list[str] = []+ with open(passages_file, encoding="utf-8") as f:+ for line in f:+ if not line.strip():+ continue+ data = json.loads(line)+ old_ids.append(data["id"])+ new_ids.append(hashlib.sha256(data["text"].encode("utf-8")).hexdigest()[:16])++ unique_new = len(set(new_ids))+ collisions = len(new_ids) - unique_new+ print(+ f"Migrating '{index_name}': {len(new_ids)} passages → {unique_new} unique "+ f"content-hash IDs ({collisions} collision(s) will dedup)."+ )++ if args.dry_run:+ print("(dry-run; not writing anything)")+ return+ if not args.yes:+ confirm = input(+ "Proceed? This rewrites passages.jsonl, .idx, .ids.txt, .meta.json. [y/N] "+ )+ if confirm.strip().lower() not in ("y", "yes"):+ print("Aborted.")+ return++ # Stage writes into siblings, then atomically rename.+ new_passages = passages_file.with_suffix(passages_file.suffix + ".migrate")+ new_offsets: dict[str, int] = {}+ rewritten_passages: list[dict[str, Any]] = []+ with (+ open(passages_file, encoding="utf-8") as src,+ open(new_passages, "w", encoding="utf-8") as dst,+ ):+ idx = 0+ for line in src:+ if not line.strip():+ continue+ data = json.loads(line)+ data["id"] = new_ids[idx]+ offset = dst.tell()+ json.dump(data, dst, ensure_ascii=False)+ dst.write("\n")+ new_offsets[new_ids[idx]] = offset+ rewritten_passages.append(data)+ idx += 1++ new_idx = offset_file.with_suffix(offset_file.suffix + ".migrate")+ with open(new_idx, "wb") as f:+ pickle.dump(new_offsets, f)++ new_idmap: Optional[Path] = None+ if idmap_file.exists():+ new_idmap = idmap_file.with_suffix(idmap_file.suffix + ".migrate")+ with open(new_idmap, "w", encoding="utf-8") as f:+ for nid in new_ids:+ f.write(nid + "\n")++ bm25_db_name = meta.get("bm25_db")+ should_rebuild_bm25 = meta.get("bm25_backend") == "fts5"+ if should_rebuild_bm25 and not bm25_db_name:+ bm25_db_name = f"{index_base}.bm25.sqlite"+ new_bm25: Optional[Path] = None+ bm25_file: Optional[Path] = None+ if should_rebuild_bm25 and bm25_db_name:+ bm25_file = index_dir / bm25_db_name+ new_bm25 = bm25_file.with_suffix(bm25_file.suffix + ".migrate")+ bm25_index = Fts5BM25Index(str(new_bm25))+ bm25_index.fit(rewritten_passages)+ bm25_index.close()++ shutil.move(str(new_passages), str(passages_file))+ shutil.move(str(new_idx), str(offset_file))+ if new_idmap is not None:+ shutil.move(str(new_idmap), str(idmap_file))+ if new_bm25 is not None and bm25_file is not None:+ shutil.move(str(new_bm25), str(bm25_file))++ meta["passage_id_scheme"] = "content-hash"+ meta["version"] = "1.1"+ if should_rebuild_bm25 and bm25_db_name:+ meta["bm25_backend"] = "fts5"+ meta["bm25_db"] = bm25_db_name+ with open(meta_path, "w", encoding="utf-8") as f:+ json.dump(meta, f, indent=2)++ print(+ f"✓ Migrated '{index_name}' to content-hash IDs. {collisions} collisions were deduped."+ )+async def watch_index(self, args):index_name = args.index_nameresolved = self._resolve_index_for_watch(index_name)@@ -3228,6 +3429,8 @@ Examples:await self.build_index(args)elif args.command == "watch":await self.watch_index(args)+ elif args.command == "migrate-ids":+ self.migrate_ids(args)elif args.command == "search":with suppress_cpp_output(suppress):await self.search_documents(args)diff --git a/tests/test_passage_id_scheme.py b/tests/test_passage_id_scheme.pynew file mode 100644index 0000000..e921d0a--- /dev/null+++ b/tests/test_passage_id_scheme.py@@ -0,0 +1,98 @@+import hashlib+import json+import pickle+from types import SimpleNamespace++from leann.api import Fts5BM25Index, LeannBuilder+from leann.cli import LeannCLI+++def _content_id(text: str) -> str:+ return hashlib.sha256(text.encode("utf-8")).hexdigest()[:16]+++def test_builder_content_hash_passage_ids_are_content_stable():+ builder = LeannBuilder(backend_name="hnsw", passage_id_scheme="content-hash")++ builder.add_text("same text", metadata={"source": "a.txt"})+ builder.add_text("same text", metadata={"source": "b.txt"})+ builder.add_text("different text", metadata={"source": "c.txt"})++ same_id = _content_id("same text")+ assert builder.chunks[0]["id"] == same_id+ assert builder.chunks[1]["id"] == same_id+ assert builder.chunks[2]["id"] == _content_id("different text")+++def test_migrate_ids_rewrites_passages_offsets_idmap_meta_and_bm25(tmp_path, monkeypatch):+ monkeypatch.chdir(tmp_path)+ index_dir = tmp_path / ".leann" / "indexes" / "sample"+ index_dir.mkdir(parents=True)++ passages = [+ {"id": "0", "text": "alpha beta", "metadata": {"source": "a.txt"}},+ {"id": "1", "text": "gamma delta", "metadata": {"source": "b.txt"}},+ ]+ passages_file = index_dir / "documents.leann.passages.jsonl"+ offsets = {}+ with open(passages_file, "w", encoding="utf-8") as f:+ for passage in passages:+ offsets[passage["id"]] = f.tell()+ json.dump(passage, f)+ f.write("\n")++ offset_file = index_dir / "documents.leann.passages.idx"+ with open(offset_file, "wb") as f:+ pickle.dump(offsets, f)++ idmap_file = index_dir / "documents.ids.txt"+ idmap_file.write_text("0\n1\n", encoding="utf-8")++ bm25_db = index_dir / "documents.leann.bm25.sqlite"+ bm25 = Fts5BM25Index(str(bm25_db))+ bm25.fit(passages)+ bm25.close()++ meta_path = index_dir / "documents.leann.meta.json"+ meta_path.write_text(+ json.dumps(+ {+ "version": "1.0",+ "backend_name": "hnsw",+ "embedding_model": "dummy",+ "dimensions": 3,+ "backend_kwargs": {},+ "embedding_mode": "sentence-transformers",+ "bm25_backend": "fts5",+ "bm25_db": "documents.leann.bm25.sqlite",+ "passage_id_scheme": "sequential",+ }+ ),+ encoding="utf-8",+ )++ cli = LeannCLI()+ cli.migrate_ids(SimpleNamespace(index_name="sample", dry_run=False, yes=True))++ expected_ids = [_content_id("alpha beta"), _content_id("gamma delta")]+ with open(passages_file, encoding="utf-8") as f:+ migrated_passages = [json.loads(line) for line in f if line.strip()]+ assert [p["id"] for p in migrated_passages] == expected_ids++ with open(offset_file, "rb") as f:+ migrated_offsets = pickle.load(f)+ assert set(migrated_offsets) == set(expected_ids)+ assert idmap_file.read_text(encoding="utf-8").splitlines() == expected_ids++ meta = json.loads(meta_path.read_text(encoding="utf-8"))+ assert meta["version"] == "1.1"+ assert meta["passage_id_scheme"] == "content-hash"+ assert meta["bm25_backend"] == "fts5"+ assert meta["bm25_db"] == "documents.leann.bm25.sqlite"++ migrated_bm25 = Fts5BM25Index(str(bm25_db))+ try:+ bm25_results = migrated_bm25.search("alpha", top_k=1)+ finally:+ migrated_bm25.close()+ assert [result.id for result in bm25_results] == [expected_ids[0]]