Skip to content

armgit / LEANN

Public Imported from GitHub
0

[MLsys2026 Best Paper]: https://arxiv.org/abs/2506.08276. RAG on Everything with LEANN. Enjoy 97% storage savings while running a fast, accurate, and 100% private RAG application on your personal device.

feat: support content-hash passage IDs

Abi2d430a445d16
diff --git a/packages/leann-core/src/leann/api.py b/packages/leann-core/src/leann/api.py
index e22633a..b5ee65d 100644
--- a/packages/leann-core/src/leann/api.py
+++ b/packages/leann-core/src/leann/api.py
@@ -30,6 +30,15 @@ from .registry import BACKEND_REGISTRY
logger = logging.getLogger(__name__)
+# Passage ID schemes recorded in <index>.meta.json["passage_id_scheme"].
+# - "sequential": today's default; IDs are str(insertion_index) (api.py:add_text).
+# - "content-hash": planned in #329; IDs are sha256(text)[:16], stable across
+# file moves and reorderings.
+# Older indexes have no passage_id_scheme field — readers must default to
+# "sequential" when the key is absent. See #329 for the rollout plan.
+PASSAGE_ID_SCHEME_SEQUENTIAL = "sequential"
+PASSAGE_ID_SCHEME_CONTENT_HASH = "content-hash"
+
def get_registered_backends() -> list[str]:
"""Get list of registered backend names."""
@@ -376,6 +385,7 @@ class LeannBuilder:
embedding_options: Optional[dict[str, Any]] = None,
prebuild_bm25: bool = False,
bm25_backend: str = "fts5",
+ passage_id_scheme: str = PASSAGE_ID_SCHEME_SEQUENTIAL,
**backend_kwargs,
):
if bm25_backend != "fts5":
@@ -383,6 +393,16 @@ class LeannBuilder:
bm25_backend = "fts5"
self.bm25_backend = bm25_backend
self.prebuild_bm25 = prebuild_bm25 or bm25_backend == "fts5"
+ if passage_id_scheme not in (
+ PASSAGE_ID_SCHEME_SEQUENTIAL,
+ PASSAGE_ID_SCHEME_CONTENT_HASH,
+ ):
+ raise ValueError(
+ f"Unknown passage_id_scheme: {passage_id_scheme!r}. "
+ f"Expected one of: {PASSAGE_ID_SCHEME_SEQUENTIAL!r}, "
+ f"{PASSAGE_ID_SCHEME_CONTENT_HASH!r}."
+ )
+ self.passage_id_scheme = passage_id_scheme
self.backend_name = backend_name
# Normalize incompatible combinations early (for consistent metadata)
if backend_name == "hnsw":
@@ -477,10 +497,23 @@ class LeannBuilder:
self.backend_kwargs = backend_kwargs
self.chunks: list[dict[str, Any]] = []
+ def _generate_passage_id(self, text: str) -> str:
+ """Generate a passage ID per the configured scheme.
+
+ sequential: str(insertion index) — fast, position-dependent, current default.
+ content-hash: sha256(text)[:16] — content-stable, dedup-friendly across
+ file moves and reorderings. See #329 for the design.
+ """
+ if self.passage_id_scheme == PASSAGE_ID_SCHEME_CONTENT_HASH:
+ import hashlib
+
+ return hashlib.sha256(text.encode("utf-8")).hexdigest()[:16]
+ return str(len(self.chunks))
+
def add_text(self, text: str, metadata: Optional[dict[str, Any]] = None):
if metadata is None:
metadata = {}
- passage_id = metadata.get("id", str(len(self.chunks)))
+ passage_id = metadata.get("id") or self._generate_passage_id(text)
chunk_data = {"id": passage_id, "text": text, "metadata": metadata}
self.chunks.append(chunk_data)
@@ -570,12 +603,13 @@ class LeannBuilder:
builder_instance.build(embeddings, string_ids, index_path, **current_backend_kwargs)
leann_meta_path = index_dir / f"{index_name}.meta.json"
meta_data = {
- "version": "1.0",
+ "version": "1.1",
"backend_name": self.backend_name,
"embedding_model": self.embedding_model,
"dimensions": self.dimensions,
"backend_kwargs": self.backend_kwargs,
"embedding_mode": self.embedding_mode,
+ "passage_id_scheme": self.passage_id_scheme,
"passage_sources": [
{
"type": "jsonl",
@@ -714,12 +748,13 @@ class LeannBuilder:
# Create metadata file
leann_meta_path = index_dir / f"{index_name}.meta.json"
meta_data = {
- "version": "1.0",
+ "version": "1.1",
"backend_name": self.backend_name,
"embedding_model": self.embedding_model,
"dimensions": self.dimensions,
"backend_kwargs": self.backend_kwargs,
"embedding_mode": self.embedding_mode,
+ "passage_id_scheme": self.passage_id_scheme,
"passage_sources": [
{
"type": "jsonl",
@@ -1177,6 +1212,12 @@ class LeannSearcher:
)
self.bm25_scorer: Optional[BM25Index] = None
+ # Surface the index's passage ID scheme so callers can introspect.
+ # Older indexes (pre-#330) don't record this field — they're sequential.
+ self.passage_id_scheme: str = self.meta_data.get(
+ "passage_id_scheme", PASSAGE_ID_SCHEME_SEQUENTIAL
+ )
+
# Optional one-shot warmup at construction time to hide cold-start latency.
if self._warmup:
self.warmup()
diff --git a/packages/leann-core/src/leann/cli.py b/packages/leann-core/src/leann/cli.py
index 0d50cf3..0b56acc 100644
--- a/packages/leann-core/src/leann/cli.py
+++ b/packages/leann-core/src/leann/cli.py
@@ -16,7 +16,7 @@ from llama_index.core import SimpleDirectoryReader
from llama_index.core.node_parser import SentenceSplitter
from tqdm import tqdm
-from .api import LeannBuilder, LeannChat, LeannSearcher
+from .api import Fts5BM25Index, LeannBuilder, LeannChat, LeannSearcher
from .embedding_server_manager import EmbeddingServerManager
from .interactive_utils import create_cli_session
from .registry import register_project_directory
@@ -347,6 +347,16 @@ Examples:
default=True,
help="Fall back to traditional chunking if AST chunking fails (default: True)",
)
+ build_parser.add_argument(
+ "--id-scheme",
+ choices=["sequential", "content-hash"],
+ default="sequential",
+ help=(
+ "How passage IDs are assigned. 'sequential' (default) keys by insertion "
+ "order; 'content-hash' uses sha256(text)[:16], stable across file moves "
+ "and reorderings. See #329."
+ ),
+ )
# Watch command
watch_parser = subparsers.add_parser(
@@ -371,6 +381,26 @@ Examples:
help="Report changes without rebuilding (original watch behavior)",
)
+ migrate_parser = subparsers.add_parser(
+ "migrate-ids",
+ help=(
+ "Rewrite an existing index's passage IDs to content-hash form. "
+ "Irreversible; back up the index first."
+ ),
+ )
+ migrate_parser.add_argument("index_name", help="Index name")
+ migrate_parser.add_argument(
+ "--dry-run",
+ action="store_true",
+ help="Show what would change without writing anything.",
+ )
+ migrate_parser.add_argument(
+ "-y",
+ "--yes",
+ action="store_true",
+ help="Skip the interactive confirmation prompt.",
+ )
+
# Search command
search_parser = subparsers.add_parser("search", help="Search documents")
search_parser.add_argument("index_name", help="Index name")
@@ -1880,7 +1910,32 @@ Examples:
in paths
]
+ def _existing_index_id_scheme(self, index_path: str) -> Optional[str]:
+ """Return the passage_id_scheme recorded in an existing index's meta.json.
+
+ Returns None when the index doesn't exist yet or the field isn't
+ recorded (older indexes pre-#330). Callers should treat None as
+ "fall back to whatever the args say or the default".
+ """
+ meta_path = Path(index_path).with_suffix(".leann.meta.json")
+ if not meta_path.exists():
+ return None
+ try:
+ with open(meta_path, encoding="utf-8") as f:
+ return json.load(f).get("passage_id_scheme")
+ except Exception:
+ return None
+
def _make_incremental_builder(self, args) -> "LeannBuilder":
+ # For incremental updates, the existing index's scheme wins. Otherwise
+ # IDs would mix schemes within one index, which breaks lookups.
+ existing_scheme = self._existing_index_id_scheme(self.get_index_path(args.index_name))
+ scheme = existing_scheme or getattr(args, "id_scheme", "sequential")
+ if existing_scheme and getattr(args, "id_scheme", existing_scheme) != existing_scheme:
+ print(
+ f"Note: --id-scheme={args.id_scheme} ignored — index '{args.index_name}' "
+ f"was built with passage_id_scheme={existing_scheme!r}, keeping that."
+ )
return LeannBuilder(
backend_name=args.backend_name,
embedding_model=args.embedding_model,
@@ -1891,6 +1946,7 @@ Examples:
is_compact=args.compact,
is_recompute=args.recompute,
num_threads=args.num_threads,
+ passage_id_scheme=scheme,
)
def _incremental_add_only(
@@ -2384,6 +2440,7 @@ Examples:
is_compact=args.compact,
is_recompute=args.recompute,
num_threads=args.num_threads,
+ passage_id_scheme=getattr(args, "id_scheme", "sequential"),
)
for chunk in all_texts:
@@ -2504,6 +2561,150 @@ Examples:
build_args = parser.parse_args(build_args_list)
await self.build_index(build_args)
+ def migrate_ids(self, args) -> None:
+ """Rewrite an existing index's passage IDs to content-hash form.
+
+ Migration is purely a Python-side rewrite — the vector graph isn't
+ touched, so FAISS labels stay valid. What gets rewritten:
+ - <index>.passages.jsonl : new IDs in each line's "id" field
+ - <index>.passages.idx : new offset map keyed by new IDs
+ - <index>.ids.txt : new label → ID mapping for FAISS
+ - <index>.meta.json : passage_id_scheme = "content-hash"
+ - <index>.bm25.sqlite : BM25 IDs, when an FTS5 artifact is configured
+
+ Identical-text chunks collide on the same sha256 prefix; the later
+ occurrence wins in the offset map (dedup). A `--preserve-duplicates`
+ knob to suffix collisions can land separately.
+
+ Irreversible. Prompts for confirmation unless --yes is passed.
+ """
+ import hashlib
+ import shutil
+
+ index_name = args.index_name
+ index_path = self._resolve_index_path(index_name, purpose="migrate")
+ if not index_path:
+ return
+ meta_path = Path(index_path).with_suffix(".leann.meta.json")
+ if not meta_path.exists():
+ print(f"Cannot migrate '{index_name}': metadata missing at {meta_path}.")
+ return
+ with open(meta_path, encoding="utf-8") as f:
+ meta = json.load(f)
+ current_scheme = meta.get("passage_id_scheme", "sequential")
+ if current_scheme == "content-hash":
+ print(f"Index '{index_name}' already uses content-hash IDs. Nothing to do.")
+ return
+
+ # Locate the sibling artifacts using the same conventions as build_index.
+ index_dir = Path(index_path).parent
+ index_base = Path(index_path).name
+ passages_file = index_dir / f"{index_base}.passages.jsonl"
+ offset_file = index_dir / f"{index_base}.passages.idx"
+ base_no_leann = (
+ index_base[: -len(".leann")] if index_base.endswith(".leann") else index_base
+ )
+ idmap_file = index_dir / f"{base_no_leann}.ids.txt"
+
+ for p in (passages_file, offset_file):
+ if not p.exists():
+ print(f"Cannot migrate: required artifact missing at {p}.")
+ return
+
+ # Stream the passages to plan the rewrite and surface collision count
+ # before committing to anything irreversible.
+ old_ids: list[str] = []
+ new_ids: list[str] = []
+ with open(passages_file, encoding="utf-8") as f:
+ for line in f:
+ if not line.strip():
+ continue
+ data = json.loads(line)
+ old_ids.append(data["id"])
+ new_ids.append(hashlib.sha256(data["text"].encode("utf-8")).hexdigest()[:16])
+
+ unique_new = len(set(new_ids))
+ collisions = len(new_ids) - unique_new
+ print(
+ f"Migrating '{index_name}': {len(new_ids)} passages → {unique_new} unique "
+ f"content-hash IDs ({collisions} collision(s) will dedup)."
+ )
+
+ if args.dry_run:
+ print("(dry-run; not writing anything)")
+ return
+ if not args.yes:
+ confirm = input(
+ "Proceed? This rewrites passages.jsonl, .idx, .ids.txt, .meta.json. [y/N] "
+ )
+ if confirm.strip().lower() not in ("y", "yes"):
+ print("Aborted.")
+ return
+
+ # Stage writes into siblings, then atomically rename.
+ new_passages = passages_file.with_suffix(passages_file.suffix + ".migrate")
+ new_offsets: dict[str, int] = {}
+ rewritten_passages: list[dict[str, Any]] = []
+ with (
+ open(passages_file, encoding="utf-8") as src,
+ open(new_passages, "w", encoding="utf-8") as dst,
+ ):
+ idx = 0
+ for line in src:
+ if not line.strip():
+ continue
+ data = json.loads(line)
+ data["id"] = new_ids[idx]
+ offset = dst.tell()
+ json.dump(data, dst, ensure_ascii=False)
+ dst.write("\n")
+ new_offsets[new_ids[idx]] = offset
+ rewritten_passages.append(data)
+ idx += 1
+
+ new_idx = offset_file.with_suffix(offset_file.suffix + ".migrate")
+ with open(new_idx, "wb") as f:
+ pickle.dump(new_offsets, f)
+
+ new_idmap: Optional[Path] = None
+ if idmap_file.exists():
+ new_idmap = idmap_file.with_suffix(idmap_file.suffix + ".migrate")
+ with open(new_idmap, "w", encoding="utf-8") as f:
+ for nid in new_ids:
+ f.write(nid + "\n")
+
+ bm25_db_name = meta.get("bm25_db")
+ should_rebuild_bm25 = meta.get("bm25_backend") == "fts5"
+ if should_rebuild_bm25 and not bm25_db_name:
+ bm25_db_name = f"{index_base}.bm25.sqlite"
+ new_bm25: Optional[Path] = None
+ bm25_file: Optional[Path] = None
+ if should_rebuild_bm25 and bm25_db_name:
+ bm25_file = index_dir / bm25_db_name
+ new_bm25 = bm25_file.with_suffix(bm25_file.suffix + ".migrate")
+ bm25_index = Fts5BM25Index(str(new_bm25))
+ bm25_index.fit(rewritten_passages)
+ bm25_index.close()
+
+ shutil.move(str(new_passages), str(passages_file))
+ shutil.move(str(new_idx), str(offset_file))
+ if new_idmap is not None:
+ shutil.move(str(new_idmap), str(idmap_file))
+ if new_bm25 is not None and bm25_file is not None:
+ shutil.move(str(new_bm25), str(bm25_file))
+
+ meta["passage_id_scheme"] = "content-hash"
+ meta["version"] = "1.1"
+ if should_rebuild_bm25 and bm25_db_name:
+ meta["bm25_backend"] = "fts5"
+ meta["bm25_db"] = bm25_db_name
+ with open(meta_path, "w", encoding="utf-8") as f:
+ json.dump(meta, f, indent=2)
+
+ print(
+ f"✓ Migrated '{index_name}' to content-hash IDs. {collisions} collisions were deduped."
+ )
+
async def watch_index(self, args):
index_name = args.index_name
resolved = self._resolve_index_for_watch(index_name)
@@ -3228,6 +3429,8 @@ Examples:
await self.build_index(args)
elif args.command == "watch":
await self.watch_index(args)
+ elif args.command == "migrate-ids":
+ self.migrate_ids(args)
elif args.command == "search":
with suppress_cpp_output(suppress):
await self.search_documents(args)
diff --git a/tests/test_passage_id_scheme.py b/tests/test_passage_id_scheme.py
new file mode 100644
index 0000000..e921d0a
--- /dev/null
+++ b/tests/test_passage_id_scheme.py
@@ -0,0 +1,98 @@
+import hashlib
+import json
+import pickle
+from types import SimpleNamespace
+
+from leann.api import Fts5BM25Index, LeannBuilder
+from leann.cli import LeannCLI
+
+
+def _content_id(text: str) -> str:
+ return hashlib.sha256(text.encode("utf-8")).hexdigest()[:16]
+
+
+def test_builder_content_hash_passage_ids_are_content_stable():
+ builder = LeannBuilder(backend_name="hnsw", passage_id_scheme="content-hash")
+
+ builder.add_text("same text", metadata={"source": "a.txt"})
+ builder.add_text("same text", metadata={"source": "b.txt"})
+ builder.add_text("different text", metadata={"source": "c.txt"})
+
+ same_id = _content_id("same text")
+ assert builder.chunks[0]["id"] == same_id
+ assert builder.chunks[1]["id"] == same_id
+ assert builder.chunks[2]["id"] == _content_id("different text")
+
+
+def test_migrate_ids_rewrites_passages_offsets_idmap_meta_and_bm25(tmp_path, monkeypatch):
+ monkeypatch.chdir(tmp_path)
+ index_dir = tmp_path / ".leann" / "indexes" / "sample"
+ index_dir.mkdir(parents=True)
+
+ passages = [
+ {"id": "0", "text": "alpha beta", "metadata": {"source": "a.txt"}},
+ {"id": "1", "text": "gamma delta", "metadata": {"source": "b.txt"}},
+ ]
+ passages_file = index_dir / "documents.leann.passages.jsonl"
+ offsets = {}
+ with open(passages_file, "w", encoding="utf-8") as f:
+ for passage in passages:
+ offsets[passage["id"]] = f.tell()
+ json.dump(passage, f)
+ f.write("\n")
+
+ offset_file = index_dir / "documents.leann.passages.idx"
+ with open(offset_file, "wb") as f:
+ pickle.dump(offsets, f)
+
+ idmap_file = index_dir / "documents.ids.txt"
+ idmap_file.write_text("0\n1\n", encoding="utf-8")
+
+ bm25_db = index_dir / "documents.leann.bm25.sqlite"
+ bm25 = Fts5BM25Index(str(bm25_db))
+ bm25.fit(passages)
+ bm25.close()
+
+ meta_path = index_dir / "documents.leann.meta.json"
+ meta_path.write_text(
+ json.dumps(
+ {
+ "version": "1.0",
+ "backend_name": "hnsw",
+ "embedding_model": "dummy",
+ "dimensions": 3,
+ "backend_kwargs": {},
+ "embedding_mode": "sentence-transformers",
+ "bm25_backend": "fts5",
+ "bm25_db": "documents.leann.bm25.sqlite",
+ "passage_id_scheme": "sequential",
+ }
+ ),
+ encoding="utf-8",
+ )
+
+ cli = LeannCLI()
+ cli.migrate_ids(SimpleNamespace(index_name="sample", dry_run=False, yes=True))
+
+ expected_ids = [_content_id("alpha beta"), _content_id("gamma delta")]
+ with open(passages_file, encoding="utf-8") as f:
+ migrated_passages = [json.loads(line) for line in f if line.strip()]
+ assert [p["id"] for p in migrated_passages] == expected_ids
+
+ with open(offset_file, "rb") as f:
+ migrated_offsets = pickle.load(f)
+ assert set(migrated_offsets) == set(expected_ids)
+ assert idmap_file.read_text(encoding="utf-8").splitlines() == expected_ids
+
+ meta = json.loads(meta_path.read_text(encoding="utf-8"))
+ assert meta["version"] == "1.1"
+ assert meta["passage_id_scheme"] == "content-hash"
+ assert meta["bm25_backend"] == "fts5"
+ assert meta["bm25_db"] == "documents.leann.bm25.sqlite"
+
+ migrated_bm25 = Fts5BM25Index(str(bm25_db))
+ try:
+ bm25_results = migrated_bm25.search("alpha", top_k=1)
+ finally:
+ migrated_bm25.close()
+ assert [result.id for result in bm25_results] == [expected_ids[0]]