Skip to content

armgit / LEANN

Public Imported from GitHub
0

[MLsys2026 Best Paper]: https://arxiv.org/abs/2506.08276. RAG on Everything with LEANN. Enjoy 97% storage savings while running a fast, accurate, and 100% private RAG application on your personal device.

Merge remote-tracking branch 'startrail/main' into feat/content-hash-passage-ids

Abib18df76486ea
diff --cc packages/leann-core/src/leann/api.py
index b5ee65d,c117abe..afec499
--- a/packages/leann-core/src/leann/api.py
+++ b/packages/leann-core/src/leann/api.py
@@@ -1212,12 -1177,11 +1212,17 @@@ class LeannSearcher
)
self.bm25_scorer: Optional[BM25Index] = None
+ # Surface the index's passage ID scheme so callers can introspect.
+ # Older indexes (pre-#330) don't record this field — they're sequential.
+ self.passage_id_scheme: str = self.meta_data.get(
+ "passage_id_scheme", PASSAGE_ID_SCHEME_SEQUENTIAL
+ )
+
+ # Optional query log path: set via LEANN_QUERY_LOG=<path>. When set, each
+ # search appends a JSON line containing the query, embedding (if computed),
+ # top_k, and result IDs/scores. Useful for offline benchmark replay.
+ self._query_log_path: Optional[str] = os.environ.get("LEANN_QUERY_LOG") or None
+
# Optional one-shot warmup at construction time to hide cold-start latency.
if self._warmup:
self.warmup()
diff --cc packages/leann-core/src/leann/cli.py
index 7b9c36c,27ba4b6..c96fd7c
--- a/packages/leann-core/src/leann/cli.py
+++ b/packages/leann-core/src/leann/cli.py
@@@ -381,26 -410,18 +420,38 @@@ Examples
help="Report changes without rebuilding (original watch behavior)",
)
+ migrate_parser = subparsers.add_parser(
+ "migrate-ids",
+ help=(
+ "Rewrite an existing index's passage IDs to content-hash form. "
+ "Irreversible; back up the index first."
+ ),
+ )
+ migrate_parser.add_argument("index_name", help="Index name")
+ migrate_parser.add_argument(
+ "--dry-run",
+ action="store_true",
+ help="Show what would change without writing anything.",
+ )
+ migrate_parser.add_argument(
+ "-y",
+ "--yes",
+ action="store_true",
+ help="Skip the interactive confirmation prompt.",
+ )
+
+ rebuild_parser = subparsers.add_parser(
+ "rebuild",
+ help="Rebuild an existing index using its stored config (delta by default; --force for full rebuild)",
+ )
+ rebuild_parser.add_argument("index_name", help="Index name")
+ rebuild_parser.add_argument(
+ "-f",
+ "--force",
+ action="store_true",
+ help="Full rebuild from scratch instead of incremental delta",
+ )
+
# Search command
search_parser = subparsers.add_parser("search", help="Search documents")
search_parser.add_argument("index_name", help="Index name")
@@@ -3429,8 -3482,9 +3683,11 @@@
await self.build_index(args)
elif args.command == "watch":
await self.watch_index(args)
+ elif args.command == "migrate-ids":
+ self.migrate_ids(args)
+ elif args.command == "rebuild":
+ with suppress_cpp_output(suppress):
+ await self.rebuild_index(args)
elif args.command == "search":
with suppress_cpp_output(suppress):
await self.search_documents(args)
diff --cc uv.lock
index a32f038,a32f038..cd7fa7e
--- a/uv.lock
+++ b/uv.lock
@@@ -2005,6 -2005,6 +2005,30 @@@ requires-dist =
]
provides-extras = ["query-server"]
++[[package]]
++name = "leann-backend-flashlib-ivf"
++version = "0.3.6"
++source = { editable = "packages/leann-backend-flashlib-ivf" }
++dependencies = [
++ { name = "flashlib" },
++ { name = "leann-core" },
++ { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13'" },
++ { name = "numpy", version = "2.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.13'" },
++ { name = "torch" },
++]
++
++[package.metadata]
++requires-dist = [
++ { name = "flashlib" },
++ { name = "leann-backend-hnsw", marker = "extra == 'query-server'", specifier = ">=0.3.6" },
++ { name = "leann-core", specifier = ">=0.3.6" },
++ { name = "msgpack", marker = "extra == 'query-server'", specifier = ">=1.0.0" },
++ { name = "numpy", specifier = ">=1.20.0" },
++ { name = "pyzmq", marker = "extra == 'query-server'", specifier = ">=23.0.0" },
++ { name = "torch" },
++]
++provides-extras = ["query-server"]
++
[[package]]
name = "leann-backend-hnsw"
version = "0.3.7"
@@@ -2158,6 -2158,6 +2182,9 @@@ documents =
flashlib = [
{ name = "leann-backend-flashlib" },
]
++flashlib-ivf = [
++ { name = "leann-backend-flashlib-ivf" },
++]
[package.dev-dependencies]
dev = [
@@@ -2190,6 -2190,6 +2217,7 @@@ requires-dist =
{ name = "ipykernel", specifier = "==6.29.5" },
{ name = "leann-backend-diskann", marker = "extra == 'diskann'", editable = "packages/leann-backend-diskann" },
{ name = "leann-backend-flashlib", marker = "extra == 'flashlib'", editable = "packages/leann-backend-flashlib" },
++ { name = "leann-backend-flashlib-ivf", marker = "extra == 'flashlib-ivf'", editable = "packages/leann-backend-flashlib-ivf" },
{ name = "leann-backend-hnsw", editable = "packages/leann-backend-hnsw" },
{ name = "leann-core", editable = "packages/leann-core" },
{ name = "llama-index", specifier = ">=0.12.44" },
@@@ -2230,7 -2230,7 +2258,7 @@@
{ name = "tree-sitter-typescript", specifier = ">=0.20.0" },
{ name = "typer", specifier = ">=0.12.3" },
]
--provides-extras = ["diskann", "flashlib", "documents"]
++provides-extras = ["diskann", "flashlib", "flashlib-ivf", "documents"]
[package.metadata.requires-dev]
dev = [