Skip to content

armgit / LEANN

Public Imported from GitHub
0

[MLsys2026 Best Paper]: https://arxiv.org/abs/2506.08276. RAG on Everything with LEANN. Enjoy 97% storage savings while running a fast, accurate, and 100% private RAG application on your personal device.

release: cut leann-core/leann-backend-hnsw/leann-backend-diskann 0.3.8 (#387)

Aakash Sureshe48e991c4746
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 2ce4387..2e4d5f2 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -40,3 +40,19 @@ fixes). Newest entries at the bottom.
at nprobe=32, ~6.5x lower single-query latency / ~75x higher batched throughput at
comparable recall (GPU latency stays ~flat while CPU grows linearly with nprobe).
- Docs: `docs/flashlib_backend_guide.md` gains a `flashlib_ivf` section.
+
+## 2026-07-24: `leann-core` 0.3.8 — HNSW full-rebuild fallback corpus wipe (#386)
+
+- Fixed a bug where, on the HNSW backend, `leann build` / `leann watch` falling
+ back to a full rebuild (any modified or removed file, since HNSW only
+ supports add-only incremental updates) rebuilt the index from only the
+ changed files instead of the full corpus, silently dropping every untouched
+ file. Root cause: the fallback reused `all_texts` from the incremental path
+ (which intentionally loads only the delta) instead of reloading the full
+ corpus via `load_documents(docs_paths, ...)`.
+- This was fixed on `main` in commit `956beb5` (merged via #279) but never
+ reached PyPI — `leann-core` 0.3.7 was still affected. Bumped `leann-core` to
+ 0.3.8 to ship the fix.
+- Added `tests/test_hnsw_rebuild_fallback.py`: regression test that builds an
+ HNSW index, modifies one file, and asserts untouched files are still present
+ in the rebuilt corpus.
diff --git a/packages/leann-backend-diskann/pyproject.toml b/packages/leann-backend-diskann/pyproject.toml
index 3dfa801..f3a073e 100644
--- a/packages/leann-backend-diskann/pyproject.toml
+++ b/packages/leann-backend-diskann/pyproject.toml
@@ -7,8 +7,8 @@ build-backend = "scikit_build_core.build"
[project]
name = "leann-backend-diskann"
-version = "0.3.7"
-dependencies = ["leann-core==0.3.7", "numpy", "protobuf>=3.19.0"]
+version = "0.3.8"
+dependencies = ["leann-core==0.3.8", "numpy", "protobuf>=3.19.0"]
[tool.scikit-build]
# Root CMake forces USE_TCMALLOC=OFF before DiskANN options (see CMakeLists.txt).
diff --git a/packages/leann-backend-hnsw/pyproject.toml b/packages/leann-backend-hnsw/pyproject.toml
index bc4a4ff..e2c0200 100644
--- a/packages/leann-backend-hnsw/pyproject.toml
+++ b/packages/leann-backend-hnsw/pyproject.toml
@@ -6,10 +6,10 @@ build-backend = "scikit_build_core.build"
[project]
name = "leann-backend-hnsw"
-version = "0.3.7"
+version = "0.3.8"
description = "Custom-built HNSW (Faiss) backend for the Leann toolkit."
dependencies = [
- "leann-core==0.3.7",
+ "leann-core==0.3.8",
"numpy",
"pyzmq>=23.0.0",
"msgpack>=1.0.0",
diff --git a/packages/leann-core/pyproject.toml b/packages/leann-core/pyproject.toml
index 3173282..1b45f1c 100644
--- a/packages/leann-core/pyproject.toml
+++ b/packages/leann-core/pyproject.toml
@@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta"
[project]
name = "leann-core"
-version = "0.3.7"
+version = "0.3.8"
description = "Core API and plugin system for LEANN"
readme = "README.md"
requires-python = ">=3.10"
diff --git a/tests/test_hnsw_rebuild_fallback.py b/tests/test_hnsw_rebuild_fallback.py
new file mode 100644
index 0000000..e2beb62
--- /dev/null
+++ b/tests/test_hnsw_rebuild_fallback.py
@@ -0,0 +1,111 @@
+"""Regression test for #386: HNSW full-rebuild fallback wiped the corpus.
+
+When HNSW can't apply an incremental update (any modified/removed file), `leann
+build` falls back to a full rebuild. The fallback must reload the *entire*
+corpus via `load_documents(docs_paths, ...)` — not reuse the partial
+`all_texts` from the incremental path, which only contains the changed files.
+Reusing the partial list silently drops every untouched file from the index.
+"""
+
+import asyncio
+import json
+import pickle
+from pathlib import Path
+
+from leann.cli import LeannCLI
+
+
+def test_hnsw_modify_triggers_full_rebuild_that_reloads_untouched_files(tmp_path, monkeypatch):
+ monkeypatch.chdir(tmp_path)
+ docs_root = tmp_path / "docs"
+ docs_root.mkdir()
+ keep_file = docs_root / "keep.md"
+ keep_file.write_text("keep me around", encoding="utf-8")
+ change_file = docs_root / "change.md"
+ change_file.write_text("original content", encoding="utf-8")
+
+ cli = LeannCLI()
+ index_dir = tmp_path / ".leann" / "indexes" / "sample"
+ index_dir.mkdir(parents=True)
+ meta_path = index_dir / "documents.leann.meta.json"
+ meta_path.write_text(
+ json.dumps(
+ {
+ "backend_name": "hnsw",
+ "embedding_model": "facebook/contriever",
+ "embedding_mode": "sentence-transformers",
+ "backend_kwargs": {"is_compact": False, "is_recompute": True},
+ }
+ ),
+ encoding="utf-8",
+ )
+ passages_file = index_dir / "documents.leann.passages.jsonl"
+ passages_file.write_text(
+ json.dumps({"id": "keep", "text": "keep me around", "metadata": {}}) + "\n",
+ encoding="utf-8",
+ )
+ with open(index_dir / "documents.leann.passages.idx", "wb") as f:
+ pickle.dump({"keep": 0}, f)
+
+ # Simulate the modification that forces HNSW off the add-only path.
+ change_file.write_text("modified content", encoding="utf-8")
+
+ class FakeSynchronizer:
+ def detect_changes(self):
+ return set(), set(), {str(change_file.resolve())}
+
+ def create_snapshot(self):
+ pass
+
+ load_documents_calls: list[list[str]] = []
+
+ def fake_load_documents(paths, _file_types, include_hidden=False, args=None):
+ load_documents_calls.append(list(paths))
+ if list(paths) == [str(change_file.resolve())]:
+ # Delta-only load taken on the incremental path before the
+ # fallback decision is made.
+ return [
+ {"text": "modified content", "metadata": {"file_path": str(change_file.resolve())}}
+ ]
+ # Full-corpus reload, expected once HNSW falls back to a full rebuild.
+ return [
+ {"text": "keep me around", "metadata": {"file_path": str(keep_file.resolve())}},
+ {"text": "modified content", "metadata": {"file_path": str(change_file.resolve())}},
+ ]
+
+ added_texts: list[str] = []
+
+ class RecordingBuilder:
+ def __init__(self, **_kwargs):
+ pass
+
+ def add_text(self, text, metadata=None):
+ added_texts.append(text)
+
+ def build_index(self, index_path):
+ Path(index_path).parent.mkdir(parents=True, exist_ok=True)
+
+ monkeypatch.setattr(cli, "_build_synchronizers", lambda *_a, **_k: [FakeSynchronizer()])
+ monkeypatch.setattr(cli, "load_documents", fake_load_documents)
+ monkeypatch.setattr(cli, "register_project_dir", lambda: None)
+ monkeypatch.setattr("leann.cli.LeannBuilder", RecordingBuilder)
+
+ args = cli.create_parser().parse_args(
+ [
+ "build",
+ "sample",
+ "--docs",
+ str(docs_root),
+ "--backend-name",
+ "hnsw",
+ ]
+ )
+
+ asyncio.run(cli.build_index(args))
+
+ # The fallback must have reloaded the full corpus, not just the delta.
+ assert [str(docs_root)] in load_documents_calls
+ assert "keep me around" in added_texts, (
+ "full-rebuild fallback dropped an untouched file from the corpus (issue #386)"
+ )
+ assert "modified content" in added_texts
diff --git a/uv.lock b/uv.lock
index cd7fa7e..b71861a 100644
--- a/uv.lock
+++ b/uv.lock
@@ -1965,7 +1965,7 @@ wheels = [
[[package]]
name = "leann-backend-diskann"
-version = "0.3.7"
+version = "0.3.8"
source = { editable = "packages/leann-backend-diskann" }
dependencies = [
{ name = "leann-core" },
@@ -1976,7 +1976,7 @@ dependencies = [
[package.metadata]
requires-dist = [
- { name = "leann-core", specifier = "==0.3.7" },
+ { name = "leann-core", specifier = "==0.3.8" },
{ name = "numpy" },
{ name = "protobuf", specifier = ">=3.19.0" },
]
@@ -2031,7 +2031,7 @@ provides-extras = ["query-server"]
[[package]]
name = "leann-backend-hnsw"
-version = "0.3.7"
+version = "0.3.8"
source = { editable = "packages/leann-backend-hnsw" }
dependencies = [
{ name = "leann-core" },
@@ -2043,7 +2043,7 @@ dependencies = [
[package.metadata]
requires-dist = [
- { name = "leann-core", specifier = "==0.3.7" },
+ { name = "leann-core", specifier = "==0.3.8" },
{ name = "msgpack", specifier = ">=1.0.0" },
{ name = "numpy" },
{ name = "pyzmq", specifier = ">=23.0.0" },
@@ -2051,7 +2051,7 @@ requires-dist = [
[[package]]
name = "leann-core"
-version = "0.3.7"
+version = "0.3.8"
source = { editable = "packages/leann-core" }
dependencies = [
{ name = "accelerate" },