Skip to content

armgit / LEANN

Public Imported from GitHub
0

[MLsys2026 Best Paper]: https://arxiv.org/abs/2506.08276. RAG on Everything with LEANN. Enjoy 97% storage savings while running a fast, accurate, and 100% private RAG application on your personal device.

Feat/ivf backend clean (#265)

Aakash Suresh5b8260357809
diff --git a/packages/leann-backend-ivf/README.md b/packages/leann-backend-ivf/README.md
new file mode 100644
index 0000000..bb76153
--- /dev/null
+++ b/packages/leann-backend-ivf/README.md
@@ -0,0 +1,52 @@
+# LEANN IVF Backend
+
+FAISS **IndexIVFFlat** backend for LEANN with **add** and **delete** APIs for incremental updates (#231, #89, #141).
+
+## Install
+
+```bash
+uv sync --package leann-backend-ivf
+# or
+pip install leann-backend-ivf
+```
+
+Requires `faiss-cpu`. For query embedding during search, the IVF searcher uses the same embedding server as HNSW; install `leann-backend-hnsw` if you use recompute/query embedding.
+
+## Build
+
+```bash
+leann build my-index --docs ./src --backend-name ivf
+```
+
+Optional backend kwargs (e.g. via API): `nlist` (default 100), `distance_metric` ("l2" or "cosine"/"mips").
+
+## Add / Delete API
+
+Follows the same idea as HNSW’s update path; use these for incremental workflows (e.g. after Merkle tree / file-change API):
+
+- **`add_vectors(index_path, embeddings, passage_ids)`**
+ Appends vectors to an existing IVF index. `embeddings`: `(N, D)` float32; `passage_ids`: list of N passage id strings (must not already exist).
+
+- **`remove_ids(index_path, passage_ids)`**
+ Removes vectors by passage id. Returns the number of vectors removed. Use after detecting changed chunk ids: delete old, then re-insert new.
+
+Example:
+
+```python
+from leann_backend_ivf import add_vectors, remove_ids
+
+# After file-change API says chunk id "abc123" changed:
+remove_ids("/path/to/index.leann", ["abc123"])
+# Re-embed and add new chunk with same or new id
+add_vectors("/path/to/index.leann", new_embeddings, ["abc123"])
+```
+
+## Core integration (#89, #141)
+
+`LeannBuilder.update_index(index_path, remove_passage_ids=None)` in leann-core supports IVF: it calls `add_vectors` for appends and, when `remove_passage_ids` is set, calls `remove_ids` first (e.g. from a file-change / Merkle API). Use for incremental build or reindex: detect changed chunk ids → `update_index(path, remove_passage_ids=changed_ids)` then add chunks for the new content.
+
+## Notes
+
+- IVF stores vectors in flat lists per cluster; no compact/recompute mode like HNSW. Good for frequent add/delete.
+- ID mapping is stored in `ivf_id_map.json` next to the index; passage ids are stable across add/delete.
+- Search uses `nprobe` (default from `complexity`, capped by `nlist`). Tune `nlist` at build time for speed/recall.
diff --git a/packages/leann-backend-ivf/leann_backend_ivf/__init__.py b/packages/leann-backend-ivf/leann_backend_ivf/__init__.py
new file mode 100644
index 0000000..006d822
--- /dev/null
+++ b/packages/leann-backend-ivf/leann_backend_ivf/__init__.py
@@ -0,0 +1,17 @@
+"""LEANN IVF backend: FAISS IndexIVFFlat with add/delete APIs for incremental updates."""
+
+from .ivf_backend import (
+ IVFBackend,
+ IVFBuilder,
+ IVFSearcher,
+ add_vectors,
+ remove_ids,
+)
+
+__all__ = [
+ "IVFBackend",
+ "IVFBuilder",
+ "IVFSearcher",
+ "add_vectors",
+ "remove_ids",
+]
diff --git a/packages/leann-backend-ivf/leann_backend_ivf/ivf_backend.py b/packages/leann-backend-ivf/leann_backend_ivf/ivf_backend.py
new file mode 100644
index 0000000..9ca45c1
--- /dev/null
+++ b/packages/leann-backend-ivf/leann_backend_ivf/ivf_backend.py
@@ -0,0 +1,276 @@
+"""
+IVF backend: FAISS IndexIVFFlat wrapped in IndexIDMap2 for stable add/delete by passage id.
+
+Provides clean add_vectors() and remove_ids() APIs (aligned with HNSW update_index flow)
+so the core can do incremental updates: detect changed chunk ids, delete and re-insert.
+"""
+
+import json
+import logging
+from pathlib import Path
+from typing import Any, Optional
+
+import numpy as np
+
+try:
+ import faiss
+except ImportError:
+ faiss = None # type: ignore[assignment]
+
+from leann.interface import (
+ LeannBackendBuilderInterface,
+ LeannBackendFactoryInterface,
+ LeannBackendSearcherInterface,
+)
+from leann.registry import register_backend
+from leann.searcher_base import BaseSearcher
+
+logger = logging.getLogger(__name__)
+
+ID_MAP_FILENAME = "ivf_id_map.json"
+
+
+def _check_faiss():
+ if faiss is None:
+ raise ImportError(
+ "faiss-cpu is required for IVF backend. Install with: pip install faiss-cpu"
+ )
+
+
+def _get_metric_map():
+ _check_faiss()
+ return {
+ "mips": faiss.METRIC_INNER_PRODUCT,
+ "l2": faiss.METRIC_L2,
+ "cosine": faiss.METRIC_INNER_PRODUCT,
+ }
+
+
+def _normalize_l2(data: np.ndarray) -> np.ndarray:
+ norms = np.linalg.norm(data, axis=1, keepdims=True)
+ norms[norms == 0] = 1
+ return data / norms
+
+
+def _load_id_map(index_dir: Path, index_prefix: str) -> tuple[dict[int, str], dict[str, int], int]:
+ """Load id_map.json. Returns (id_to_passage, passage_to_id, next_id)."""
+ path = index_dir / f"{index_prefix}.{ID_MAP_FILENAME}"
+ if not path.exists():
+ return {}, {}, 0
+ with open(path, encoding="utf-8") as f:
+ data = json.load(f)
+ id_to_passage = {int(k): v for k, v in data.get("id_to_passage", {}).items()}
+ passage_to_id = data.get("passage_to_id", {})
+ next_id = int(data.get("next_id", 0))
+ return id_to_passage, passage_to_id, next_id
+
+
+def _save_id_map(
+ index_dir: Path, index_prefix: str, id_to_passage: dict[int, str], next_id: int
+) -> None:
+ passage_to_id = {v: k for k, v in id_to_passage.items()}
+ path = index_dir / f"{index_prefix}.{ID_MAP_FILENAME}"
+ with open(path, "w", encoding="utf-8") as f:
+ json.dump(
+ {
+ "id_to_passage": {str(k): v for k, v in id_to_passage.items()},
+ "passage_to_id": passage_to_id,
+ "next_id": next_id,
+ },
+ f,
+ indent=2,
+ )
+
+
+@register_backend("ivf")
+class IVFBackend(LeannBackendFactoryInterface):
+ @staticmethod
+ def builder(**kwargs) -> LeannBackendBuilderInterface:
+ return IVFBuilder(**kwargs)
+
+ @staticmethod
+ def searcher(index_path: str, **kwargs) -> LeannBackendSearcherInterface:
+ return IVFSearcher(index_path, **kwargs)
+
+
+class IVFBuilder(LeannBackendBuilderInterface):
+ def __init__(self, **kwargs):
+ _check_faiss()
+ self.build_params = kwargs.copy()
+ self.nlist = self.build_params.setdefault("nlist", 100)
+ self.distance_metric = self.build_params.setdefault("distance_metric", "l2")
+ self.dimensions = self.build_params.get("dimensions")
+
+ def build(self, data: np.ndarray, ids: list[str], index_path: str, **kwargs) -> None:
+ _check_faiss()
+ path = Path(index_path)
+ index_dir = path.parent
+ index_prefix = path.stem
+ index_dir.mkdir(parents=True, exist_ok=True)
+
+ if data.dtype != np.float32:
+ data = data.astype(np.float32)
+ data = np.ascontiguousarray(data)
+ dim = self.dimensions or data.shape[1]
+ n = data.shape[0]
+ metric_enum = _get_metric_map().get(self.distance_metric.lower())
+ if metric_enum is None:
+ raise ValueError(f"Unsupported distance_metric '{self.distance_metric}'.")
+
+ if self.distance_metric.lower() == "cosine":
+ data = _normalize_l2(data)
+
+ quantizer = (
+ faiss.IndexFlatL2(dim) if metric_enum == faiss.METRIC_L2 else faiss.IndexFlatIP(dim)
+ )
+ ivf = faiss.IndexIVFFlat(quantizer, dim, self.nlist, metric_enum)
+ ivf.train(data)
+ index = faiss.IndexIDMap2(ivf)
+ faiss_ids = np.arange(n, dtype=np.int64)
+ index.add_with_ids(data, faiss_ids)
+
+ index_file = index_dir / f"{index_prefix}.index"
+ faiss.write_index(index, str(index_file))
+
+ id_to_passage = dict(enumerate(ids))
+ _save_id_map(index_dir, index_prefix, id_to_passage, next_id=n)
+
+
+class IVFSearcher(BaseSearcher):
+ def __init__(self, index_path: str, **kwargs):
+ # Use HNSW embedding server for query embedding if available (same as other backends)
+ super().__init__(
+ index_path,
+ backend_module_name="leann_backend_hnsw.hnsw_embedding_server",
+ **kwargs,
+ )
+ _check_faiss()
+ self.distance_metric = (
+ self.meta.get("backend_kwargs", {}).get("distance_metric", "l2").lower()
+ )
+ index_prefix = self.index_path.stem
+ index_file = self.index_dir / f"{index_prefix}.index"
+ if not index_file.exists():
+ raise FileNotFoundError(f"IVF index file not found at {index_file}")
+
+ self._index = faiss.read_index(str(index_file))
+ self._id_to_passage: dict[int, str] = {}
+ id_to_passage, _, _ = _load_id_map(self.index_dir, index_prefix)
+ self._id_to_passage = id_to_passage
+
+ def search(
+ self,
+ query: np.ndarray,
+ top_k: int,
+ complexity: int = 64,
+ nprobe: Optional[int] = None,
+ **kwargs,
+ ) -> dict[str, Any]:
+ _check_faiss()
+ if query.dtype != np.float32:
+ query = query.astype(np.float32)
+ if self.distance_metric == "cosine":
+ query = _normalize_l2(query)
+ nprobe = nprobe or min(complexity, self._index.nlist)
+ self._index.nprobe = nprobe
+ distances, label_rows = self._index.search(query, top_k)
+
+ def map_label(x: int) -> str:
+ return self._id_to_passage.get(int(x), str(x))
+
+ string_labels = [[map_label(int(lab)) for lab in row] for row in label_rows]
+ return {"labels": string_labels, "distances": distances}
+
+ def compute_query_embedding(
+ self,
+ query: str,
+ use_server_if_available: bool = True,
+ zmq_port: Optional[int] = None,
+ query_template: Optional[str] = None,
+ ) -> np.ndarray:
+ return super().compute_query_embedding(
+ query,
+ use_server_if_available=use_server_if_available,
+ zmq_port=zmq_port,
+ query_template=query_template,
+ )
+
+
+def add_vectors(index_path: str, embeddings: np.ndarray, passage_ids: list[str]) -> None:
+ """
+ Append vectors to an existing IVF index (same role as HNSW update_index add path).
+
+ Args:
+ index_path: Path to the .leann index (e.g. .../documents.leann).
+ embeddings: (N, D) float32 array.
+ passage_ids: List of N passage id strings (must not already exist in index).
+ """
+ _check_faiss()
+ path = Path(index_path)
+ index_dir = path.parent
+ index_prefix = path.stem
+ index_file = index_dir / f"{index_prefix}.index"
+ if not index_file.exists():
+ raise FileNotFoundError(f"IVF index not found: {index_file}")
+
+ embeddings = np.ascontiguousarray(embeddings.astype(np.float32))
+ id_to_passage, passage_to_id, next_id = _load_id_map(index_dir, index_prefix)
+ for pid in passage_ids:
+ if pid in passage_to_id:
+ raise ValueError(f"Passage id '{pid}' already exists in index.")
+
+ n = embeddings.shape[0]
+ if n != len(passage_ids):
+ raise ValueError("embeddings.shape[0] must equal len(passage_ids).")
+
+ index = faiss.read_index(str(index_file))
+ new_ids = np.arange(next_id, next_id + n, dtype=np.int64)
+ index.add_with_ids(embeddings, new_ids)
+ faiss.write_index(index, str(index_file))
+
+ for i, pid in enumerate(passage_ids):
+ id_to_passage[next_id + i] = pid
+ _save_id_map(index_dir, index_prefix, id_to_passage, next_id=next_id + n)
+ logger.info("IVF add_vectors: appended %d vectors, next_id=%d", n, next_id + n)
+
+
+def remove_ids(index_path: str, passage_ids: list[str]) -> int:
+ """
+ Remove vectors by passage id (for incremental update: delete changed chunks before re-insert).
+
+ Args:
+ index_path: Path to the .leann index.
+ passage_ids: List of passage id strings to remove.
+
+ Returns:
+ Number of vectors actually removed.
+ """
+ _check_faiss()
+ path = Path(index_path)
+ index_dir = path.parent
+ index_prefix = path.stem
+ index_file = index_dir / f"{index_prefix}.index"
+ if not index_file.exists():
+ raise FileNotFoundError(f"IVF index not found: {index_file}")
+
+ id_to_passage, passage_to_id, next_id = _load_id_map(index_dir, index_prefix)
+ to_remove_int: list[int] = []
+ for pid in passage_ids:
+ if pid in passage_to_id:
+ to_remove_int.append(passage_to_id[pid])
+ if not to_remove_int:
+ return 0
+
+ index = faiss.read_index(str(index_file))
+ sel = np.array(to_remove_int, dtype=np.int64)
+ nremoved = index.remove_ids(sel)
+ faiss.write_index(index, str(index_file))
+
+ for pid in passage_ids:
+ if pid in passage_to_id:
+ i = passage_to_id[pid]
+ id_to_passage.pop(i, None)
+ # passage_to_id is rebuilt from id_to_passage when we save; we don't decrease next_id so new adds get new ids
+ _save_id_map(index_dir, index_prefix, id_to_passage, next_id=next_id)
+ logger.info("IVF remove_ids: removed %d vectors (requested %d)", nremoved, len(passage_ids))
+ return nremoved
diff --git a/packages/leann-backend-ivf/pyproject.toml b/packages/leann-backend-ivf/pyproject.toml
new file mode 100644
index 0000000..caff74c
--- /dev/null
+++ b/packages/leann-backend-ivf/pyproject.toml
@@ -0,0 +1,19 @@
+[project]
+name = "leann-backend-ivf"
+version = "0.3.6"
+description = "FAISS IVF backend for LEANN with add/delete support for incremental updates."
+readme = "README.md"
+requires-python = ">=3.10"
+dependencies = [
+ "leann-core>=0.3.6",
+ "numpy>=1.20.0",
+ "faiss-cpu>=1.7.4",
+]
+
+[project.optional-dependencies]
+# Optional: use HNSW embedding server for query embedding (same as other backends)
+query-server = ["leann-backend-hnsw>=0.3.6", "pyzmq>=23.0.0", "msgpack>=1.0.0"]
+
+[tool.setuptools.packages.find]
+where = ["."]
+include = ["leann_backend_ivf*"]
diff --git a/packages/leann-core/src/leann/api.py b/packages/leann-core/src/leann/api.py
index 34440cf..cde098c 100644
--- a/packages/leann-core/src/leann/api.py
+++ b/packages/leann-core/src/leann/api.py
@@ -718,10 +718,12 @@ class LeannBuilder:
logger.info(f"Index built successfully from precomputed embeddings: {index_path}")
- def update_index(self, index_path: str):
- """Append new passages and vectors to an existing HNSW index."""
- if not self.chunks:
- raise ValueError("No new chunks provided for update.")
+ def update_index(self, index_path: str, remove_passage_ids: Optional[list[str]] = None) -> None:
+ """Append new passages and vectors to an existing index (HNSW or IVF).
+ For IVF, optional remove_passage_ids removes those ids first (e.g. from file-change API).
+ """
+ if not self.chunks and not remove_passage_ids:
+ raise ValueError("No new chunks or passage ids to remove provided for update.")
path = Path(index_path)
index_dir = path.parent
@@ -736,7 +738,7 @@ class LeannBuilder:
if not meta_path.exists() or not passages_file.exists() or not offset_file.exists():
raise FileNotFoundError("Index metadata or passage files are missing; cannot update.")
if not index_file.exists():
- raise FileNotFoundError(f"HNSW index file not found: {index_file}")
+ raise FileNotFoundError(f"Index file not found: {index_file}")
with open(meta_path, encoding="utf-8") as f:
meta = json.load(f)
@@ -746,12 +748,39 @@ class LeannBuilder:
f"Index was built with backend '{backend_name}', cannot update with '{self.backend_name}'."
)
+ with open(offset_file, "rb") as f:
+ offset_map: dict[str, int] = pickle.load(f)
+ existing_ids = set(offset_map.keys())
+
+ # IVF: optional delete (for reindex / file-change: remove then re-insert)
+ if remove_passage_ids and backend_name == "ivf":
+ try:
+ from leann_backend_ivf import remove_ids as ivf_remove_ids
+
+ ivf_remove_ids(str(path), remove_passage_ids)
+ except ImportError:
+ raise RuntimeError(
+ "IVF backend required for remove_ids. Install leann-backend-ivf."
+ )
+ for pid in remove_passage_ids:
+ offset_map.pop(pid, None)
+ with open(offset_file, "wb") as f:
+ pickle.dump(offset_map, f)
+
+ if not self.chunks:
+ meta["total_passages"] = len(offset_map)
+ with open(meta_path, "w", encoding="utf-8") as f:
+ json.dump(meta, f, indent=2)
+ self.chunks.clear()
+ return
+
meta_backend_kwargs = meta.get("backend_kwargs", {})
- index_is_compact = meta.get("is_compact", meta_backend_kwargs.get("is_compact", True))
- if index_is_compact:
- raise ValueError(
- "Compact HNSW indices do not support in-place updates. Rebuild required."
- )
+ if backend_name == "hnsw":
+ index_is_compact = meta.get("is_compact", meta_backend_kwargs.get("is_compact", True))
+ if index_is_compact:
+ raise ValueError(
+ "Compact HNSW indices do not support in-place updates. Rebuild required."
+ )
distance_metric = meta_backend_kwargs.get(
"distance_metric", self.backend_kwargs.get("distance_metric", "mips")
@@ -762,10 +791,6 @@ class LeannBuilder:
or self.backend_kwargs.get("is_recompute")
)
- with open(offset_file, "rb") as f:
- offset_map: dict[str, int] = pickle.load(f)
- existing_ids = set(offset_map.keys())
-
valid_chunks: list[dict[str, Any]] = []
for chunk in self.chunks:
text = chunk.get("text", "")
@@ -797,14 +822,69 @@ class LeannBuilder:
f"Dimension mismatch during update: existing index uses {expected_dim}, got {embedding_dim}."
)
- from leann_backend_hnsw import faiss
-
embeddings = np.ascontiguousarray(embeddings, dtype=np.float32)
if distance_metric == "cosine":
norms = np.linalg.norm(embeddings, axis=1, keepdims=True)
norms[norms == 0] = 1
embeddings = embeddings / norms
+ # IVF: add_vectors then append passages/offset (no ZMQ/server)
+ if backend_name == "ivf":
+ for i, chunk in enumerate(valid_chunks):
+ pid = chunk.get("id") or chunk.get("metadata", {}).get("id")
+ if not pid:
+ pid = str(len(offset_map) + i)
+ chunk.setdefault("metadata", {})["id"] = pid
+ chunk["id"] = pid
+ passage_ids = [c["id"] for c in valid_chunks]
+ try:
+ from leann_backend_ivf import add_vectors as ivf_add_vectors
+
+ ivf_add_vectors(str(path), embeddings, passage_ids)
+ except ImportError:
+ raise RuntimeError("IVF backend required. Install leann-backend-ivf.")
+ rollback_passages_size = passages_file.stat().st_size if passages_file.exists() else 0
+ offset_map_backup = offset_map.copy()
+ try:
+ with open(passages_file, "a", encoding="utf-8") as f:
+ for chunk in valid_chunks:
+ off = f.tell()
+ json.dump(
+ {
+ "id": chunk["id"],
+ "text": chunk["text"],
+ "metadata": chunk.get("metadata", {}),
+ },
+ f,
+ ensure_ascii=False,
+ )
+ f.write("\n")
+ offset_map[chunk["id"]] = off
+ with open(offset_file, "wb") as f:
+ pickle.dump(offset_map, f)
+ meta["total_passages"] = len(offset_map)
+ with open(meta_path, "w", encoding="utf-8") as f:
+ json.dump(meta, f, indent=2)
+ logger.info(
+ "Appended %d passages to IVF index '%s'. Total: %d",
+ len(valid_chunks),
+ index_path,
+ len(offset_map),
+ )
+ except Exception:
+ if passages_file.exists():
+ with open(passages_file, "rb+") as f:
+ f.truncate(rollback_passages_size)
+ offset_map = offset_map_backup
+ with open(offset_file, "wb") as f:
+ pickle.dump(offset_map, f)
+ raise
+ self.chunks.clear()
+ return
+
+ # HNSW path below
+ from leann_backend_hnsw import faiss
+
index = faiss.read_index(str(index_file))
if hasattr(index, "is_recompute"):
index.is_recompute = needs_recompute
diff --git a/packages/leann-core/src/leann/cli.py b/packages/leann-core/src/leann/cli.py
index 59ee874..62a1470 100644
--- a/packages/leann-core/src/leann/cli.py
+++ b/packages/leann-core/src/leann/cli.py
@@ -1,6 +1,7 @@
import argparse
import asyncio
import contextlib
+import hashlib
import io
import json
import os
@@ -25,6 +26,37 @@ from .settings import (
)
from .sync import FileSynchronizer
+# Manifest file for incremental build: maps source file path -> mtime
+SOURCES_MANIFEST_FILENAME = "documents.leann.sources.json"
+
+
+def _normalize_path(path: str) -> str:
+ """Return absolute path string for consistent manifest keys."""
+ if not path:
+ return path
+ return str(Path(path).resolve())
+
+
+def load_sources_manifest(index_dir: Path, index_name: str) -> dict[str, float]:
+ """Load sources manifest (path -> mtime). Returns {} if file missing or invalid."""
+ manifest_path = index_dir / SOURCES_MANIFEST_FILENAME
+ if not manifest_path.exists():
+ return {}
+ try:
+ with open(manifest_path, encoding="utf-8") as f:
+ data = json.load(f)
+ return data.get("sources", {})
+ except (json.JSONDecodeError, OSError):
+ return {}
+
+
+def save_sources_manifest(index_dir: Path, index_name: str, sources: dict[str, float]) -> None:
+ """Write sources manifest (path -> mtime)."""
+ manifest_path = index_dir / SOURCES_MANIFEST_FILENAME
+ index_dir.mkdir(parents=True, exist_ok=True)
+ with open(manifest_path, "w", encoding="utf-8") as f:
+ json.dump({"sources": sources}, f, indent=2)
+
@contextlib.contextmanager
def suppress_cpp_output(suppress: bool = True):
@@ -196,7 +228,7 @@ Examples:
"--backend-name",
type=str,
default="hnsw",
- choices=["hnsw", "diskann"],
+ choices=["hnsw", "diskann", "ivf"],
help="Backend to use (default: hnsw)",
)
build_parser.add_argument(
@@ -243,7 +275,10 @@ Examples:
help="Prompt template for queries (different from build template for task-specific models)",
)
build_parser.add_argument(
- "--force", "-f", action="store_true", help="Force rebuild existing index"
+ "--force",
+ "-f",
+ action="store_true",
+ help="Force full rebuild of existing index (without this, build does incremental update: add new files only)",
)
build_parser.add_argument(
"--graph-degree", type=int, default=32, help="Graph degree (default: 32)"
@@ -1727,10 +1762,6 @@ Examples:
for i, dir_path in enumerate(directories, 1):
print(f" {i}. {Path(dir_path).resolve()}")
- if index_dir.exists() and not args.force:
- print(f"Index '{index_name}' already exists. Use --force to rebuild.")
- return
-
# Configure chunking based on CLI args before loading documents
# Guard against invalid configurations
doc_chunk_size = max(1, int(args.doc_chunk_size))
@@ -1769,6 +1800,108 @@ Examples:
print("No documents found")
return
+ # Incremental build: index exists and no --force -> add only new files
+ if index_dir.exists() and not args.force:
+ manifest = load_sources_manifest(index_dir, "documents.leann")
+ if not manifest:
+ print("Index was built before incremental support. Use --force to rebuild.")
+ return
+ current_sources = {}
+ for c in all_texts:
+ path = c.get("metadata", {}).get("file_path") or c.get("metadata", {}).get("source")
+ if path and os.path.isfile(path):
+ current_sources[_normalize_path(path)] = os.path.getmtime(path)
+ new_paths = {p for p in current_sources if p not in manifest}
+ if not new_paths:
+ print("Index up to date. Use --force to rebuild from scratch.")
+ return
+ new_chunks = [
+ c
+ for c in all_texts
+ if _normalize_path(
+ c.get("metadata", {}).get("file_path")
+ or c.get("metadata", {}).get("source")
+ or ""
+ )
+ in new_paths
+ ]
+ meta_path = index_dir / "documents.leann.meta.json"
+ if not meta_path.exists():
+ print("Index metadata missing. Use --force to rebuild.")
+ return
+ with open(meta_path, encoding="utf-8") as f:
+ meta = json.load(f)
+ if meta.get("backend_name") != "hnsw":
+ print(
+ f"Incremental update only supported for HNSW backend (index uses '{meta.get('backend_name')}'). Use --force to rebuild."
+ )
+ return
+ if meta.get("is_compact", meta.get("backend_kwargs", {}).get("is_compact", True)):
+ print(
+ "Incremental update requires a non-compact index. Use --force and rebuild with --no-compact."
+ )
+ return
+ if (
+ meta.get("embedding_model") != args.embedding_model
+ or meta.get("embedding_mode") != args.embedding_mode
+ ):
+ print(
+ "Use the same --embedding-model and --embedding-mode as the existing index for incremental update."
+ )
+ return
+ from collections import defaultdict
+
+ by_path = defaultdict(list)
+ for c in new_chunks:
+ path = (
+ c.get("metadata", {}).get("file_path")
+ or c.get("metadata", {}).get("source")
+ or ""
+ )
+ by_path[_normalize_path(path)].append(c)
+ for path_key, path_chunks in by_path.items():
+ for idx, c in enumerate(path_chunks):
+ sid = hashlib.sha256(f"{path_key}:{idx}".encode()).hexdigest()[:16]
+ c.setdefault("metadata", {})["id"] = sid
+ c["id"] = sid
+ embedding_options_inc: dict[str, Any] = {}
+ if args.embedding_mode == "ollama":
+ embedding_options_inc["host"] = resolve_ollama_host(args.embedding_host)
+ elif args.embedding_mode == "openai":
+ embedding_options_inc["base_url"] = resolve_openai_base_url(args.embedding_api_base)
+ resolved_embedding_key = resolve_openai_api_key(args.embedding_api_key)
+ if resolved_embedding_key:
+ embedding_options_inc["api_key"] = resolved_embedding_key
+ if args.query_prompt_template:
+ if args.embedding_prompt_template:
+ embedding_options_inc["build_prompt_template"] = args.embedding_prompt_template
+ embedding_options_inc["query_prompt_template"] = args.query_prompt_template
+ elif args.embedding_prompt_template:
+ embedding_options_inc["prompt_template"] = args.embedding_prompt_template
+ builder_inc = LeannBuilder(
+ backend_name=args.backend_name,
+ embedding_model=args.embedding_model,
+ embedding_mode=args.embedding_mode,
+ embedding_options=embedding_options_inc or None,
+ graph_degree=args.graph_degree,
+ complexity=args.complexity,
+ is_compact=args.compact,
+ is_recompute=args.recompute,
+ num_threads=args.num_threads,
+ )
+ for chunk in new_chunks:
+ builder_inc.add_text(chunk["text"], metadata=chunk["metadata"])
+ print(
+ f"Updating index '{index_name}' with {len(new_chunks)} chunks from {len(new_paths)} new file(s)..."
+ )
+ builder_inc.update_index(index_path)
+ for p in new_paths:
+ manifest[p] = current_sources[p]
+ save_sources_manifest(index_dir, "documents.leann", manifest)
+ print(f"Index updated at {index_path}")
+ self.register_project_dir()
+ return
+
index_dir.mkdir(parents=True, exist_ok=True)
print(f"Building index '{index_name}' with {args.backend_name} backend...")
@@ -1806,6 +1939,14 @@ Examples:
builder.add_text(chunk["text"], metadata=chunk["metadata"])
builder.build_index(index_path)
+ # Write sources manifest for future incremental builds
+ full_sources = {}
+ for c in all_texts:
+ path = c.get("metadata", {}).get("file_path") or c.get("metadata", {}).get("source")
+ if path and os.path.isfile(path):
+ full_sources[_normalize_path(path)] = os.path.getmtime(path)
+ if full_sources:
+ save_sources_manifest(index_dir, "documents.leann", full_sources)
print(f"Index built at {index_path}")
sync_roots = self._resolve_sync_roots(docs_paths)
diff --git a/packages/leann-mcp/README.md b/packages/leann-mcp/README.md
index cde8d29..5e2055e 100644
--- a/packages/leann-mcp/README.md
+++ b/packages/leann-mcp/README.md
@@ -4,6 +4,8 @@ Transform your development workflow with intelligent code assistance using LEANN
For agent-facing discovery details, see `llms.txt` in the repository root.
+**Backend work (IVF, incremental build, reindex):** See [BACKEND.md](./BACKEND.md) for how the MCP server relates to backends and for notes on #231, #89, and #141.
+
## Prerequisites
Install LEANN globally for MCP integration (with default backend):
diff --git a/tests/test_incremental_build.py b/tests/test_incremental_build.py
new file mode 100644
index 0000000..7833993
--- /dev/null
+++ b/tests/test_incremental_build.py
@@ -0,0 +1,132 @@
+"""
+Tests for incremental build (Feature #89).
+
+When an index already exists and build is run again without --force,
+only new files are indexed and appended to the existing index (HNSW, non-compact only).
+"""
+
+import json
+import os
+from pathlib import Path
+
+import pytest
+from leann.cli import (
+ SOURCES_MANIFEST_FILENAME,
+ _normalize_path,
+ load_sources_manifest,
+ save_sources_manifest,
+)
+
+
+def test_normalize_path():
+ assert _normalize_path("") == ""
+ assert _normalize_path("/a/b") == "/a/b" or "a" in _normalize_path("/a/b")
+ # Resolve so relative becomes absolute
+ rel = "foo/bar"
+ out = _normalize_path(rel)
+ assert Path(out).is_absolute() or out == rel
+
+
+def test_load_sources_manifest_missing(tmp_path):
+ assert load_sources_manifest(tmp_path, "documents.leann") == {}
+
+
+def test_load_sources_manifest_empty(tmp_path):
+ (tmp_path / SOURCES_MANIFEST_FILENAME).write_text('{"sources": {}}', encoding="utf-8")
+ assert load_sources_manifest(tmp_path, "documents.leann") == {}
+
+
+def test_save_and_load_sources_manifest(tmp_path):
+ sources = {"/path/to/a": 12345.0, "/path/to/b": 67890.0}
+ save_sources_manifest(tmp_path, "documents.leann", sources)
+ loaded = load_sources_manifest(tmp_path, "documents.leann")
+ assert loaded == sources
+ raw = (tmp_path / SOURCES_MANIFEST_FILENAME).read_text(encoding="utf-8")
+ data = json.loads(raw)
+ assert data.get("sources") == sources
+
+
+ os.environ.get("CI") == "true",
+ reason="Skip in CI to avoid embedding/model load",
+)
+def test_incremental_build_adds_only_new_files(tmp_path):
+ """Build once with one file, add a second file, run build again without --force; index grows."""
+ from leann.cli import LeannCLI
+
+ docs_dir = tmp_path / "docs"
+ docs_dir.mkdir()
+ (docs_dir / "a.txt").write_text("First document content for indexing.", encoding="utf-8")
+
+ cli = LeannCLI()
+ # Override indexes_dir so we use tmp_path for index
+ cli.indexes_dir = tmp_path / ".leann" / "indexes"
+ cli.indexes_dir.mkdir(parents=True, exist_ok=True)
+ index_name = "incr_test"
+ index_dir = cli.indexes_dir / index_name
+ index_path = cli.get_index_path(index_name)
+
+ parser = cli.create_parser()
+ args = parser.parse_args(
+ [
+ "build",
+ index_name,
+ "--docs",
+ str(docs_dir),
+ "--backend-name",
+ "hnsw",
+ "--no-compact",
+ "--embedding-model",
+ "all-MiniLM-L6-v2",
+ "--embedding-mode",
+ "sentence-transformers",
+ "--force",
+ ]
+ )
+ import asyncio
+
+ asyncio.run(cli.build_index(args))
+ assert index_dir.exists()
+ meta_file = index_dir / "documents.leann.meta.json"
+ assert meta_file.exists()
+ manifest_file = index_dir / "documents.leann.sources.json"
+ assert manifest_file.exists(), "Full build should write sources manifest"
+ with open(manifest_file, encoding="utf-8") as f:
+ manifest = json.load(f)
+ assert len(manifest.get("sources", {})) == 1
+
+ # Add second file
+ (docs_dir / "b.txt").write_text("Second document content.", encoding="utf-8")
+
+ # Build again without --force (incremental)
+ args2 = parser.parse_args(
+ [
+ "build",
+ index_name,
+ "--docs",
+ str(docs_dir),
+ "--backend-name",
+ "hnsw",
+ "--no-compact",
+ "--embedding-model",
+ "all-MiniLM-L6-v2",
+ "--embedding-mode",
+ "sentence-transformers",
+ ]
+ )
+ asyncio.run(cli.build_index(args2))
+
+ with open(manifest_file, encoding="utf-8") as f:
+ manifest2 = json.load(f)
+ assert len(manifest2.get("sources", {})) == 2, (
+ "Manifest should list both files after incremental"
+ )
+
+ # Index should still be searchable
+ from leann.api import LeannSearcher
+
+ searcher = LeannSearcher(index_path)
+ results = searcher.search("Second document", top_k=3)
+ searcher.cleanup()
+ assert len(results) >= 1
+ assert "Second" in results[0].text or "document" in results[0].text