[MLsys2026 Best Paper]: https://arxiv.org/abs/2506.08276. RAG on Everything with LEANN. Enjoy 97% storage savings while running a fast, accurate, and 100% private RAG application on your personal device.
Python aifaissgpt-osslangchainllama-indexllmlocalstorageoffline-firstollamaprivacypythonragretrieval-augmented-generationvector-databasevector-searchvectors
release: cut leann-core/leann-backend-hnsw/leann-backend-diskann 0.3.8 (#387)
Aakash Sureshe48e991c4746
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.mdindex 2ce4387..2e4d5f2 100644--- a/docs/CHANGELOG.md+++ b/docs/CHANGELOG.md@@ -40,3 +40,19 @@ fixes). Newest entries at the bottom.at nprobe=32, ~6.5x lower single-query latency / ~75x higher batched throughput atcomparable recall (GPU latency stays ~flat while CPU grows linearly with nprobe).- Docs: `docs/flashlib_backend_guide.md` gains a `flashlib_ivf` section.++## 2026-07-24: `leann-core` 0.3.8 — HNSW full-rebuild fallback corpus wipe (#386)++- Fixed a bug where, on the HNSW backend, `leann build` / `leann watch` falling+ back to a full rebuild (any modified or removed file, since HNSW only+ supports add-only incremental updates) rebuilt the index from only the+ changed files instead of the full corpus, silently dropping every untouched+ file. Root cause: the fallback reused `all_texts` from the incremental path+ (which intentionally loads only the delta) instead of reloading the full+ corpus via `load_documents(docs_paths, ...)`.+- This was fixed on `main` in commit `956beb5` (merged via #279) but never+ reached PyPI — `leann-core` 0.3.7 was still affected. Bumped `leann-core` to+ 0.3.8 to ship the fix.+- Added `tests/test_hnsw_rebuild_fallback.py`: regression test that builds an+ HNSW index, modifies one file, and asserts untouched files are still present+ in the rebuilt corpus.diff --git a/packages/leann-backend-diskann/pyproject.toml b/packages/leann-backend-diskann/pyproject.tomlindex 3dfa801..f3a073e 100644--- a/packages/leann-backend-diskann/pyproject.toml+++ b/packages/leann-backend-diskann/pyproject.toml@@ -7,8 +7,8 @@ build-backend = "scikit_build_core.build"[project]name = "leann-backend-diskann"-version = "0.3.7"-dependencies = ["leann-core==0.3.7", "numpy", "protobuf>=3.19.0"]+version = "0.3.8"+dependencies = ["leann-core==0.3.8", "numpy", "protobuf>=3.19.0"][tool.scikit-build]# Root CMake forces USE_TCMALLOC=OFF before DiskANN options (see CMakeLists.txt).diff --git a/packages/leann-backend-hnsw/pyproject.toml b/packages/leann-backend-hnsw/pyproject.tomlindex bc4a4ff..e2c0200 100644--- a/packages/leann-backend-hnsw/pyproject.toml+++ b/packages/leann-backend-hnsw/pyproject.toml@@ -6,10 +6,10 @@ build-backend = "scikit_build_core.build"[project]name = "leann-backend-hnsw"-version = "0.3.7"+version = "0.3.8"description = "Custom-built HNSW (Faiss) backend for the Leann toolkit."dependencies = [- "leann-core==0.3.7",+ "leann-core==0.3.8","numpy","pyzmq>=23.0.0","msgpack>=1.0.0",diff --git a/packages/leann-core/pyproject.toml b/packages/leann-core/pyproject.tomlindex 3173282..1b45f1c 100644--- a/packages/leann-core/pyproject.toml+++ b/packages/leann-core/pyproject.toml@@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta"[project]name = "leann-core"-version = "0.3.7"+version = "0.3.8"description = "Core API and plugin system for LEANN"readme = "README.md"requires-python = ">=3.10"diff --git a/tests/test_hnsw_rebuild_fallback.py b/tests/test_hnsw_rebuild_fallback.pynew file mode 100644index 0000000..e2beb62--- /dev/null+++ b/tests/test_hnsw_rebuild_fallback.py@@ -0,0 +1,111 @@+"""Regression test for #386: HNSW full-rebuild fallback wiped the corpus.++When HNSW can't apply an incremental update (any modified/removed file), `leann+build` falls back to a full rebuild. The fallback must reload the *entire*+corpus via `load_documents(docs_paths, ...)` — not reuse the partial+`all_texts` from the incremental path, which only contains the changed files.+Reusing the partial list silently drops every untouched file from the index.+"""++import asyncio+import json+import pickle+from pathlib import Path++from leann.cli import LeannCLI+++def test_hnsw_modify_triggers_full_rebuild_that_reloads_untouched_files(tmp_path, monkeypatch):+ monkeypatch.chdir(tmp_path)+ docs_root = tmp_path / "docs"+ docs_root.mkdir()+ keep_file = docs_root / "keep.md"+ keep_file.write_text("keep me around", encoding="utf-8")+ change_file = docs_root / "change.md"+ change_file.write_text("original content", encoding="utf-8")++ cli = LeannCLI()+ index_dir = tmp_path / ".leann" / "indexes" / "sample"+ index_dir.mkdir(parents=True)+ meta_path = index_dir / "documents.leann.meta.json"+ meta_path.write_text(+ json.dumps(+ {+ "backend_name": "hnsw",+ "embedding_model": "facebook/contriever",+ "embedding_mode": "sentence-transformers",+ "backend_kwargs": {"is_compact": False, "is_recompute": True},+ }+ ),+ encoding="utf-8",+ )+ passages_file = index_dir / "documents.leann.passages.jsonl"+ passages_file.write_text(+ json.dumps({"id": "keep", "text": "keep me around", "metadata": {}}) + "\n",+ encoding="utf-8",+ )+ with open(index_dir / "documents.leann.passages.idx", "wb") as f:+ pickle.dump({"keep": 0}, f)++ # Simulate the modification that forces HNSW off the add-only path.+ change_file.write_text("modified content", encoding="utf-8")++ class FakeSynchronizer:+ def detect_changes(self):+ return set(), set(), {str(change_file.resolve())}++ def create_snapshot(self):+ pass++ load_documents_calls: list[list[str]] = []++ def fake_load_documents(paths, _file_types, include_hidden=False, args=None):+ load_documents_calls.append(list(paths))+ if list(paths) == [str(change_file.resolve())]:+ # Delta-only load taken on the incremental path before the+ # fallback decision is made.+ return [+ {"text": "modified content", "metadata": {"file_path": str(change_file.resolve())}}+ ]+ # Full-corpus reload, expected once HNSW falls back to a full rebuild.+ return [+ {"text": "keep me around", "metadata": {"file_path": str(keep_file.resolve())}},+ {"text": "modified content", "metadata": {"file_path": str(change_file.resolve())}},+ ]++ added_texts: list[str] = []++ class RecordingBuilder:+ def __init__(self, **_kwargs):+ pass++ def add_text(self, text, metadata=None):+ added_texts.append(text)++ def build_index(self, index_path):+ Path(index_path).parent.mkdir(parents=True, exist_ok=True)++ monkeypatch.setattr(cli, "_build_synchronizers", lambda *_a, **_k: [FakeSynchronizer()])+ monkeypatch.setattr(cli, "load_documents", fake_load_documents)+ monkeypatch.setattr(cli, "register_project_dir", lambda: None)+ monkeypatch.setattr("leann.cli.LeannBuilder", RecordingBuilder)++ args = cli.create_parser().parse_args(+ [+ "build",+ "sample",+ "--docs",+ str(docs_root),+ "--backend-name",+ "hnsw",+ ]+ )++ asyncio.run(cli.build_index(args))++ # The fallback must have reloaded the full corpus, not just the delta.+ assert [str(docs_root)] in load_documents_calls+ assert "keep me around" in added_texts, (+ "full-rebuild fallback dropped an untouched file from the corpus (issue #386)"+ )+ assert "modified content" in added_textsdiff --git a/uv.lock b/uv.lockindex cd7fa7e..b71861a 100644--- a/uv.lock+++ b/uv.lock@@ -1965,7 +1965,7 @@ wheels = [[[package]]name = "leann-backend-diskann"-version = "0.3.7"+version = "0.3.8"source = { editable = "packages/leann-backend-diskann" }dependencies = [{ name = "leann-core" },@@ -1976,7 +1976,7 @@ dependencies = [[package.metadata]requires-dist = [- { name = "leann-core", specifier = "==0.3.7" },+ { name = "leann-core", specifier = "==0.3.8" },{ name = "numpy" },{ name = "protobuf", specifier = ">=3.19.0" },]@@ -2031,7 +2031,7 @@ provides-extras = ["query-server"][[package]]name = "leann-backend-hnsw"-version = "0.3.7"+version = "0.3.8"source = { editable = "packages/leann-backend-hnsw" }dependencies = [{ name = "leann-core" },@@ -2043,7 +2043,7 @@ dependencies = [[package.metadata]requires-dist = [- { name = "leann-core", specifier = "==0.3.7" },+ { name = "leann-core", specifier = "==0.3.8" },{ name = "msgpack", specifier = ">=1.0.0" },{ name = "numpy" },{ name = "pyzmq", specifier = ">=23.0.0" },@@ -2051,7 +2051,7 @@ requires-dist = [[[package]]name = "leann-core"-version = "0.3.7"+version = "0.3.8"source = { editable = "packages/leann-core" }dependencies = [{ name = "accelerate" },