[MLsys2026 Best Paper]: https://arxiv.org/abs/2506.08276. RAG on Everything with LEANN. Enjoy 97% storage savings while running a fast, accurate, and 100% private RAG application on your personal device.
Python aifaissgpt-osslangchainllama-indexllmlocalstorageoffline-firstollamaprivacypythonragretrieval-augmented-generationvector-databasevector-searchvectors
Feat/ivf backend clean (#265)
Aakash Suresh5b8260357809
diff --git a/packages/leann-backend-ivf/README.md b/packages/leann-backend-ivf/README.mdnew file mode 100644index 0000000..bb76153--- /dev/null+++ b/packages/leann-backend-ivf/README.md@@ -0,0 +1,52 @@+# LEANN IVF Backend++FAISS **IndexIVFFlat** backend for LEANN with **add** and **delete** APIs for incremental updates (#231, #89, #141).++## Install++```bash+uv sync --package leann-backend-ivf+# or+pip install leann-backend-ivf+```++Requires `faiss-cpu`. For query embedding during search, the IVF searcher uses the same embedding server as HNSW; install `leann-backend-hnsw` if you use recompute/query embedding.++## Build++```bash+leann build my-index --docs ./src --backend-name ivf+```++Optional backend kwargs (e.g. via API): `nlist` (default 100), `distance_metric` ("l2" or "cosine"/"mips").++## Add / Delete API++Follows the same idea as HNSW’s update path; use these for incremental workflows (e.g. after Merkle tree / file-change API):++- **`add_vectors(index_path, embeddings, passage_ids)`**+ Appends vectors to an existing IVF index. `embeddings`: `(N, D)` float32; `passage_ids`: list of N passage id strings (must not already exist).++- **`remove_ids(index_path, passage_ids)`**+ Removes vectors by passage id. Returns the number of vectors removed. Use after detecting changed chunk ids: delete old, then re-insert new.++Example:++```python+from leann_backend_ivf import add_vectors, remove_ids++# After file-change API says chunk id "abc123" changed:+remove_ids("/path/to/index.leann", ["abc123"])+# Re-embed and add new chunk with same or new id+add_vectors("/path/to/index.leann", new_embeddings, ["abc123"])+```++## Core integration (#89, #141)++`LeannBuilder.update_index(index_path, remove_passage_ids=None)` in leann-core supports IVF: it calls `add_vectors` for appends and, when `remove_passage_ids` is set, calls `remove_ids` first (e.g. from a file-change / Merkle API). Use for incremental build or reindex: detect changed chunk ids → `update_index(path, remove_passage_ids=changed_ids)` then add chunks for the new content.++## Notes++- IVF stores vectors in flat lists per cluster; no compact/recompute mode like HNSW. Good for frequent add/delete.+- ID mapping is stored in `ivf_id_map.json` next to the index; passage ids are stable across add/delete.+- Search uses `nprobe` (default from `complexity`, capped by `nlist`). Tune `nlist` at build time for speed/recall.diff --git a/packages/leann-backend-ivf/leann_backend_ivf/__init__.py b/packages/leann-backend-ivf/leann_backend_ivf/__init__.pynew file mode 100644index 0000000..006d822--- /dev/null+++ b/packages/leann-backend-ivf/leann_backend_ivf/__init__.py@@ -0,0 +1,17 @@+"""LEANN IVF backend: FAISS IndexIVFFlat with add/delete APIs for incremental updates."""++from .ivf_backend import (+ IVFBackend,+ IVFBuilder,+ IVFSearcher,+ add_vectors,+ remove_ids,+)++__all__ = [+ "IVFBackend",+ "IVFBuilder",+ "IVFSearcher",+ "add_vectors",+ "remove_ids",+]diff --git a/packages/leann-backend-ivf/leann_backend_ivf/ivf_backend.py b/packages/leann-backend-ivf/leann_backend_ivf/ivf_backend.pynew file mode 100644index 0000000..9ca45c1--- /dev/null+++ b/packages/leann-backend-ivf/leann_backend_ivf/ivf_backend.py@@ -0,0 +1,276 @@+"""+IVF backend: FAISS IndexIVFFlat wrapped in IndexIDMap2 for stable add/delete by passage id.++Provides clean add_vectors() and remove_ids() APIs (aligned with HNSW update_index flow)+so the core can do incremental updates: detect changed chunk ids, delete and re-insert.+"""++import json+import logging+from pathlib import Path+from typing import Any, Optional++import numpy as np++try:+ import faiss+except ImportError:+ faiss = None # type: ignore[assignment]++from leann.interface import (+ LeannBackendBuilderInterface,+ LeannBackendFactoryInterface,+ LeannBackendSearcherInterface,+)+from leann.registry import register_backend+from leann.searcher_base import BaseSearcher++logger = logging.getLogger(__name__)++ID_MAP_FILENAME = "ivf_id_map.json"+++def _check_faiss():+ if faiss is None:+ raise ImportError(+ "faiss-cpu is required for IVF backend. Install with: pip install faiss-cpu"+ )+++def _get_metric_map():+ _check_faiss()+ return {+ "mips": faiss.METRIC_INNER_PRODUCT,+ "l2": faiss.METRIC_L2,+ "cosine": faiss.METRIC_INNER_PRODUCT,+ }+++def _normalize_l2(data: np.ndarray) -> np.ndarray:+ norms = np.linalg.norm(data, axis=1, keepdims=True)+ norms[norms == 0] = 1+ return data / norms+++def _load_id_map(index_dir: Path, index_prefix: str) -> tuple[dict[int, str], dict[str, int], int]:+ """Load id_map.json. Returns (id_to_passage, passage_to_id, next_id)."""+ path = index_dir / f"{index_prefix}.{ID_MAP_FILENAME}"+ if not path.exists():+ return {}, {}, 0+ with open(path, encoding="utf-8") as f:+ data = json.load(f)+ id_to_passage = {int(k): v for k, v in data.get("id_to_passage", {}).items()}+ passage_to_id = data.get("passage_to_id", {})+ next_id = int(data.get("next_id", 0))+ return id_to_passage, passage_to_id, next_id+++def _save_id_map(+ index_dir: Path, index_prefix: str, id_to_passage: dict[int, str], next_id: int+) -> None:+ passage_to_id = {v: k for k, v in id_to_passage.items()}+ path = index_dir / f"{index_prefix}.{ID_MAP_FILENAME}"+ with open(path, "w", encoding="utf-8") as f:+ json.dump(+ {+ "id_to_passage": {str(k): v for k, v in id_to_passage.items()},+ "passage_to_id": passage_to_id,+ "next_id": next_id,+ },+ f,+ indent=2,+ )+++@register_backend("ivf")+class IVFBackend(LeannBackendFactoryInterface):+ @staticmethod+ def builder(**kwargs) -> LeannBackendBuilderInterface:+ return IVFBuilder(**kwargs)++ @staticmethod+ def searcher(index_path: str, **kwargs) -> LeannBackendSearcherInterface:+ return IVFSearcher(index_path, **kwargs)+++class IVFBuilder(LeannBackendBuilderInterface):+ def __init__(self, **kwargs):+ _check_faiss()+ self.build_params = kwargs.copy()+ self.nlist = self.build_params.setdefault("nlist", 100)+ self.distance_metric = self.build_params.setdefault("distance_metric", "l2")+ self.dimensions = self.build_params.get("dimensions")++ def build(self, data: np.ndarray, ids: list[str], index_path: str, **kwargs) -> None:+ _check_faiss()+ path = Path(index_path)+ index_dir = path.parent+ index_prefix = path.stem+ index_dir.mkdir(parents=True, exist_ok=True)++ if data.dtype != np.float32:+ data = data.astype(np.float32)+ data = np.ascontiguousarray(data)+ dim = self.dimensions or data.shape[1]+ n = data.shape[0]+ metric_enum = _get_metric_map().get(self.distance_metric.lower())+ if metric_enum is None:+ raise ValueError(f"Unsupported distance_metric '{self.distance_metric}'.")++ if self.distance_metric.lower() == "cosine":+ data = _normalize_l2(data)++ quantizer = (+ faiss.IndexFlatL2(dim) if metric_enum == faiss.METRIC_L2 else faiss.IndexFlatIP(dim)+ )+ ivf = faiss.IndexIVFFlat(quantizer, dim, self.nlist, metric_enum)+ ivf.train(data)+ index = faiss.IndexIDMap2(ivf)+ faiss_ids = np.arange(n, dtype=np.int64)+ index.add_with_ids(data, faiss_ids)++ index_file = index_dir / f"{index_prefix}.index"+ faiss.write_index(index, str(index_file))++ id_to_passage = dict(enumerate(ids))+ _save_id_map(index_dir, index_prefix, id_to_passage, next_id=n)+++class IVFSearcher(BaseSearcher):+ def __init__(self, index_path: str, **kwargs):+ # Use HNSW embedding server for query embedding if available (same as other backends)+ super().__init__(+ index_path,+ backend_module_name="leann_backend_hnsw.hnsw_embedding_server",+ **kwargs,+ )+ _check_faiss()+ self.distance_metric = (+ self.meta.get("backend_kwargs", {}).get("distance_metric", "l2").lower()+ )+ index_prefix = self.index_path.stem+ index_file = self.index_dir / f"{index_prefix}.index"+ if not index_file.exists():+ raise FileNotFoundError(f"IVF index file not found at {index_file}")++ self._index = faiss.read_index(str(index_file))+ self._id_to_passage: dict[int, str] = {}+ id_to_passage, _, _ = _load_id_map(self.index_dir, index_prefix)+ self._id_to_passage = id_to_passage++ def search(+ self,+ query: np.ndarray,+ top_k: int,+ complexity: int = 64,+ nprobe: Optional[int] = None,+ **kwargs,+ ) -> dict[str, Any]:+ _check_faiss()+ if query.dtype != np.float32:+ query = query.astype(np.float32)+ if self.distance_metric == "cosine":+ query = _normalize_l2(query)+ nprobe = nprobe or min(complexity, self._index.nlist)+ self._index.nprobe = nprobe+ distances, label_rows = self._index.search(query, top_k)++ def map_label(x: int) -> str:+ return self._id_to_passage.get(int(x), str(x))++ string_labels = [[map_label(int(lab)) for lab in row] for row in label_rows]+ return {"labels": string_labels, "distances": distances}++ def compute_query_embedding(+ self,+ query: str,+ use_server_if_available: bool = True,+ zmq_port: Optional[int] = None,+ query_template: Optional[str] = None,+ ) -> np.ndarray:+ return super().compute_query_embedding(+ query,+ use_server_if_available=use_server_if_available,+ zmq_port=zmq_port,+ query_template=query_template,+ )+++def add_vectors(index_path: str, embeddings: np.ndarray, passage_ids: list[str]) -> None:+ """+ Append vectors to an existing IVF index (same role as HNSW update_index add path).++ Args:+ index_path: Path to the .leann index (e.g. .../documents.leann).+ embeddings: (N, D) float32 array.+ passage_ids: List of N passage id strings (must not already exist in index).+ """+ _check_faiss()+ path = Path(index_path)+ index_dir = path.parent+ index_prefix = path.stem+ index_file = index_dir / f"{index_prefix}.index"+ if not index_file.exists():+ raise FileNotFoundError(f"IVF index not found: {index_file}")++ embeddings = np.ascontiguousarray(embeddings.astype(np.float32))+ id_to_passage, passage_to_id, next_id = _load_id_map(index_dir, index_prefix)+ for pid in passage_ids:+ if pid in passage_to_id:+ raise ValueError(f"Passage id '{pid}' already exists in index.")++ n = embeddings.shape[0]+ if n != len(passage_ids):+ raise ValueError("embeddings.shape[0] must equal len(passage_ids).")++ index = faiss.read_index(str(index_file))+ new_ids = np.arange(next_id, next_id + n, dtype=np.int64)+ index.add_with_ids(embeddings, new_ids)+ faiss.write_index(index, str(index_file))++ for i, pid in enumerate(passage_ids):+ id_to_passage[next_id + i] = pid+ _save_id_map(index_dir, index_prefix, id_to_passage, next_id=next_id + n)+ logger.info("IVF add_vectors: appended %d vectors, next_id=%d", n, next_id + n)+++def remove_ids(index_path: str, passage_ids: list[str]) -> int:+ """+ Remove vectors by passage id (for incremental update: delete changed chunks before re-insert).++ Args:+ index_path: Path to the .leann index.+ passage_ids: List of passage id strings to remove.++ Returns:+ Number of vectors actually removed.+ """+ _check_faiss()+ path = Path(index_path)+ index_dir = path.parent+ index_prefix = path.stem+ index_file = index_dir / f"{index_prefix}.index"+ if not index_file.exists():+ raise FileNotFoundError(f"IVF index not found: {index_file}")++ id_to_passage, passage_to_id, next_id = _load_id_map(index_dir, index_prefix)+ to_remove_int: list[int] = []+ for pid in passage_ids:+ if pid in passage_to_id:+ to_remove_int.append(passage_to_id[pid])+ if not to_remove_int:+ return 0++ index = faiss.read_index(str(index_file))+ sel = np.array(to_remove_int, dtype=np.int64)+ nremoved = index.remove_ids(sel)+ faiss.write_index(index, str(index_file))++ for pid in passage_ids:+ if pid in passage_to_id:+ i = passage_to_id[pid]+ id_to_passage.pop(i, None)+ # passage_to_id is rebuilt from id_to_passage when we save; we don't decrease next_id so new adds get new ids+ _save_id_map(index_dir, index_prefix, id_to_passage, next_id=next_id)+ logger.info("IVF remove_ids: removed %d vectors (requested %d)", nremoved, len(passage_ids))+ return nremoveddiff --git a/packages/leann-backend-ivf/pyproject.toml b/packages/leann-backend-ivf/pyproject.tomlnew file mode 100644index 0000000..caff74c--- /dev/null+++ b/packages/leann-backend-ivf/pyproject.toml@@ -0,0 +1,19 @@+[project]+name = "leann-backend-ivf"+version = "0.3.6"+description = "FAISS IVF backend for LEANN with add/delete support for incremental updates."+readme = "README.md"+requires-python = ">=3.10"+dependencies = [+ "leann-core>=0.3.6",+ "numpy>=1.20.0",+ "faiss-cpu>=1.7.4",+]++[project.optional-dependencies]+# Optional: use HNSW embedding server for query embedding (same as other backends)+query-server = ["leann-backend-hnsw>=0.3.6", "pyzmq>=23.0.0", "msgpack>=1.0.0"]++[tool.setuptools.packages.find]+where = ["."]+include = ["leann_backend_ivf*"]diff --git a/packages/leann-core/src/leann/api.py b/packages/leann-core/src/leann/api.pyindex 34440cf..cde098c 100644--- a/packages/leann-core/src/leann/api.py+++ b/packages/leann-core/src/leann/api.py@@ -718,10 +718,12 @@ class LeannBuilder:logger.info(f"Index built successfully from precomputed embeddings: {index_path}")- def update_index(self, index_path: str):- """Append new passages and vectors to an existing HNSW index."""- if not self.chunks:- raise ValueError("No new chunks provided for update.")+ def update_index(self, index_path: str, remove_passage_ids: Optional[list[str]] = None) -> None:+ """Append new passages and vectors to an existing index (HNSW or IVF).+ For IVF, optional remove_passage_ids removes those ids first (e.g. from file-change API).+ """+ if not self.chunks and not remove_passage_ids:+ raise ValueError("No new chunks or passage ids to remove provided for update.")path = Path(index_path)index_dir = path.parent@@ -736,7 +738,7 @@ class LeannBuilder:if not meta_path.exists() or not passages_file.exists() or not offset_file.exists():raise FileNotFoundError("Index metadata or passage files are missing; cannot update.")if not index_file.exists():- raise FileNotFoundError(f"HNSW index file not found: {index_file}")+ raise FileNotFoundError(f"Index file not found: {index_file}")with open(meta_path, encoding="utf-8") as f:meta = json.load(f)@@ -746,12 +748,39 @@ class LeannBuilder:f"Index was built with backend '{backend_name}', cannot update with '{self.backend_name}'.")+ with open(offset_file, "rb") as f:+ offset_map: dict[str, int] = pickle.load(f)+ existing_ids = set(offset_map.keys())++ # IVF: optional delete (for reindex / file-change: remove then re-insert)+ if remove_passage_ids and backend_name == "ivf":+ try:+ from leann_backend_ivf import remove_ids as ivf_remove_ids++ ivf_remove_ids(str(path), remove_passage_ids)+ except ImportError:+ raise RuntimeError(+ "IVF backend required for remove_ids. Install leann-backend-ivf."+ )+ for pid in remove_passage_ids:+ offset_map.pop(pid, None)+ with open(offset_file, "wb") as f:+ pickle.dump(offset_map, f)++ if not self.chunks:+ meta["total_passages"] = len(offset_map)+ with open(meta_path, "w", encoding="utf-8") as f:+ json.dump(meta, f, indent=2)+ self.chunks.clear()+ return+meta_backend_kwargs = meta.get("backend_kwargs", {})- index_is_compact = meta.get("is_compact", meta_backend_kwargs.get("is_compact", True))- if index_is_compact:- raise ValueError(- "Compact HNSW indices do not support in-place updates. Rebuild required."- )+ if backend_name == "hnsw":+ index_is_compact = meta.get("is_compact", meta_backend_kwargs.get("is_compact", True))+ if index_is_compact:+ raise ValueError(+ "Compact HNSW indices do not support in-place updates. Rebuild required."+ )distance_metric = meta_backend_kwargs.get("distance_metric", self.backend_kwargs.get("distance_metric", "mips")@@ -762,10 +791,6 @@ class LeannBuilder:or self.backend_kwargs.get("is_recompute"))- with open(offset_file, "rb") as f:- offset_map: dict[str, int] = pickle.load(f)- existing_ids = set(offset_map.keys())-valid_chunks: list[dict[str, Any]] = []for chunk in self.chunks:text = chunk.get("text", "")@@ -797,14 +822,69 @@ class LeannBuilder:f"Dimension mismatch during update: existing index uses {expected_dim}, got {embedding_dim}.")- from leann_backend_hnsw import faiss-embeddings = np.ascontiguousarray(embeddings, dtype=np.float32)if distance_metric == "cosine":norms = np.linalg.norm(embeddings, axis=1, keepdims=True)norms[norms == 0] = 1embeddings = embeddings / norms+ # IVF: add_vectors then append passages/offset (no ZMQ/server)+ if backend_name == "ivf":+ for i, chunk in enumerate(valid_chunks):+ pid = chunk.get("id") or chunk.get("metadata", {}).get("id")+ if not pid:+ pid = str(len(offset_map) + i)+ chunk.setdefault("metadata", {})["id"] = pid+ chunk["id"] = pid+ passage_ids = [c["id"] for c in valid_chunks]+ try:+ from leann_backend_ivf import add_vectors as ivf_add_vectors++ ivf_add_vectors(str(path), embeddings, passage_ids)+ except ImportError:+ raise RuntimeError("IVF backend required. Install leann-backend-ivf.")+ rollback_passages_size = passages_file.stat().st_size if passages_file.exists() else 0+ offset_map_backup = offset_map.copy()+ try:+ with open(passages_file, "a", encoding="utf-8") as f:+ for chunk in valid_chunks:+ off = f.tell()+ json.dump(+ {+ "id": chunk["id"],+ "text": chunk["text"],+ "metadata": chunk.get("metadata", {}),+ },+ f,+ ensure_ascii=False,+ )+ f.write("\n")+ offset_map[chunk["id"]] = off+ with open(offset_file, "wb") as f:+ pickle.dump(offset_map, f)+ meta["total_passages"] = len(offset_map)+ with open(meta_path, "w", encoding="utf-8") as f:+ json.dump(meta, f, indent=2)+ logger.info(+ "Appended %d passages to IVF index '%s'. Total: %d",+ len(valid_chunks),+ index_path,+ len(offset_map),+ )+ except Exception:+ if passages_file.exists():+ with open(passages_file, "rb+") as f:+ f.truncate(rollback_passages_size)+ offset_map = offset_map_backup+ with open(offset_file, "wb") as f:+ pickle.dump(offset_map, f)+ raise+ self.chunks.clear()+ return++ # HNSW path below+ from leann_backend_hnsw import faiss+index = faiss.read_index(str(index_file))if hasattr(index, "is_recompute"):index.is_recompute = needs_recomputediff --git a/packages/leann-core/src/leann/cli.py b/packages/leann-core/src/leann/cli.pyindex 59ee874..62a1470 100644--- a/packages/leann-core/src/leann/cli.py+++ b/packages/leann-core/src/leann/cli.py@@ -1,6 +1,7 @@import argparseimport asyncioimport contextlib+import hashlibimport ioimport jsonimport os@@ -25,6 +26,37 @@ from .settings import ()from .sync import FileSynchronizer+# Manifest file for incremental build: maps source file path -> mtime+SOURCES_MANIFEST_FILENAME = "documents.leann.sources.json"+++def _normalize_path(path: str) -> str:+ """Return absolute path string for consistent manifest keys."""+ if not path:+ return path+ return str(Path(path).resolve())+++def load_sources_manifest(index_dir: Path, index_name: str) -> dict[str, float]:+ """Load sources manifest (path -> mtime). Returns {} if file missing or invalid."""+ manifest_path = index_dir / SOURCES_MANIFEST_FILENAME+ if not manifest_path.exists():+ return {}+ try:+ with open(manifest_path, encoding="utf-8") as f:+ data = json.load(f)+ return data.get("sources", {})+ except (json.JSONDecodeError, OSError):+ return {}+++def save_sources_manifest(index_dir: Path, index_name: str, sources: dict[str, float]) -> None:+ """Write sources manifest (path -> mtime)."""+ manifest_path = index_dir / SOURCES_MANIFEST_FILENAME+ index_dir.mkdir(parents=True, exist_ok=True)+ with open(manifest_path, "w", encoding="utf-8") as f:+ json.dump({"sources": sources}, f, indent=2)+@contextlib.contextmanagerdef suppress_cpp_output(suppress: bool = True):@@ -196,7 +228,7 @@ Examples:"--backend-name",type=str,default="hnsw",- choices=["hnsw", "diskann"],+ choices=["hnsw", "diskann", "ivf"],help="Backend to use (default: hnsw)",)build_parser.add_argument(@@ -243,7 +275,10 @@ Examples:help="Prompt template for queries (different from build template for task-specific models)",)build_parser.add_argument(- "--force", "-f", action="store_true", help="Force rebuild existing index"+ "--force",+ "-f",+ action="store_true",+ help="Force full rebuild of existing index (without this, build does incremental update: add new files only)",)build_parser.add_argument("--graph-degree", type=int, default=32, help="Graph degree (default: 32)"@@ -1727,10 +1762,6 @@ Examples:for i, dir_path in enumerate(directories, 1):print(f" {i}. {Path(dir_path).resolve()}")- if index_dir.exists() and not args.force:- print(f"Index '{index_name}' already exists. Use --force to rebuild.")- return-# Configure chunking based on CLI args before loading documents# Guard against invalid configurationsdoc_chunk_size = max(1, int(args.doc_chunk_size))@@ -1769,6 +1800,108 @@ Examples:print("No documents found")return+ # Incremental build: index exists and no --force -> add only new files+ if index_dir.exists() and not args.force:+ manifest = load_sources_manifest(index_dir, "documents.leann")+ if not manifest:+ print("Index was built before incremental support. Use --force to rebuild.")+ return+ current_sources = {}+ for c in all_texts:+ path = c.get("metadata", {}).get("file_path") or c.get("metadata", {}).get("source")+ if path and os.path.isfile(path):+ current_sources[_normalize_path(path)] = os.path.getmtime(path)+ new_paths = {p for p in current_sources if p not in manifest}+ if not new_paths:+ print("Index up to date. Use --force to rebuild from scratch.")+ return+ new_chunks = [+ c+ for c in all_texts+ if _normalize_path(+ c.get("metadata", {}).get("file_path")+ or c.get("metadata", {}).get("source")+ or ""+ )+ in new_paths+ ]+ meta_path = index_dir / "documents.leann.meta.json"+ if not meta_path.exists():+ print("Index metadata missing. Use --force to rebuild.")+ return+ with open(meta_path, encoding="utf-8") as f:+ meta = json.load(f)+ if meta.get("backend_name") != "hnsw":+ print(+ f"Incremental update only supported for HNSW backend (index uses '{meta.get('backend_name')}'). Use --force to rebuild."+ )+ return+ if meta.get("is_compact", meta.get("backend_kwargs", {}).get("is_compact", True)):+ print(+ "Incremental update requires a non-compact index. Use --force and rebuild with --no-compact."+ )+ return+ if (+ meta.get("embedding_model") != args.embedding_model+ or meta.get("embedding_mode") != args.embedding_mode+ ):+ print(+ "Use the same --embedding-model and --embedding-mode as the existing index for incremental update."+ )+ return+ from collections import defaultdict++ by_path = defaultdict(list)+ for c in new_chunks:+ path = (+ c.get("metadata", {}).get("file_path")+ or c.get("metadata", {}).get("source")+ or ""+ )+ by_path[_normalize_path(path)].append(c)+ for path_key, path_chunks in by_path.items():+ for idx, c in enumerate(path_chunks):+ sid = hashlib.sha256(f"{path_key}:{idx}".encode()).hexdigest()[:16]+ c.setdefault("metadata", {})["id"] = sid+ c["id"] = sid+ embedding_options_inc: dict[str, Any] = {}+ if args.embedding_mode == "ollama":+ embedding_options_inc["host"] = resolve_ollama_host(args.embedding_host)+ elif args.embedding_mode == "openai":+ embedding_options_inc["base_url"] = resolve_openai_base_url(args.embedding_api_base)+ resolved_embedding_key = resolve_openai_api_key(args.embedding_api_key)+ if resolved_embedding_key:+ embedding_options_inc["api_key"] = resolved_embedding_key+ if args.query_prompt_template:+ if args.embedding_prompt_template:+ embedding_options_inc["build_prompt_template"] = args.embedding_prompt_template+ embedding_options_inc["query_prompt_template"] = args.query_prompt_template+ elif args.embedding_prompt_template:+ embedding_options_inc["prompt_template"] = args.embedding_prompt_template+ builder_inc = LeannBuilder(+ backend_name=args.backend_name,+ embedding_model=args.embedding_model,+ embedding_mode=args.embedding_mode,+ embedding_options=embedding_options_inc or None,+ graph_degree=args.graph_degree,+ complexity=args.complexity,+ is_compact=args.compact,+ is_recompute=args.recompute,+ num_threads=args.num_threads,+ )+ for chunk in new_chunks:+ builder_inc.add_text(chunk["text"], metadata=chunk["metadata"])+ print(+ f"Updating index '{index_name}' with {len(new_chunks)} chunks from {len(new_paths)} new file(s)..."+ )+ builder_inc.update_index(index_path)+ for p in new_paths:+ manifest[p] = current_sources[p]+ save_sources_manifest(index_dir, "documents.leann", manifest)+ print(f"Index updated at {index_path}")+ self.register_project_dir()+ return+index_dir.mkdir(parents=True, exist_ok=True)print(f"Building index '{index_name}' with {args.backend_name} backend...")@@ -1806,6 +1939,14 @@ Examples:builder.add_text(chunk["text"], metadata=chunk["metadata"])builder.build_index(index_path)+ # Write sources manifest for future incremental builds+ full_sources = {}+ for c in all_texts:+ path = c.get("metadata", {}).get("file_path") or c.get("metadata", {}).get("source")+ if path and os.path.isfile(path):+ full_sources[_normalize_path(path)] = os.path.getmtime(path)+ if full_sources:+ save_sources_manifest(index_dir, "documents.leann", full_sources)print(f"Index built at {index_path}")sync_roots = self._resolve_sync_roots(docs_paths)diff --git a/packages/leann-mcp/README.md b/packages/leann-mcp/README.mdindex cde8d29..5e2055e 100644--- a/packages/leann-mcp/README.md+++ b/packages/leann-mcp/README.md@@ -4,6 +4,8 @@ Transform your development workflow with intelligent code assistance using LEANNFor agent-facing discovery details, see `llms.txt` in the repository root.+**Backend work (IVF, incremental build, reindex):** See [BACKEND.md](./BACKEND.md) for how the MCP server relates to backends and for notes on #231, #89, and #141.+## PrerequisitesInstall LEANN globally for MCP integration (with default backend):diff --git a/tests/test_incremental_build.py b/tests/test_incremental_build.pynew file mode 100644index 0000000..7833993--- /dev/null+++ b/tests/test_incremental_build.py@@ -0,0 +1,132 @@+"""+Tests for incremental build (Feature #89).++When an index already exists and build is run again without --force,+only new files are indexed and appended to the existing index (HNSW, non-compact only).+"""++import json+import os+from pathlib import Path++import pytest+from leann.cli import (+ SOURCES_MANIFEST_FILENAME,+ _normalize_path,+ load_sources_manifest,+ save_sources_manifest,+)+++def test_normalize_path():+ assert _normalize_path("") == ""+ assert _normalize_path("/a/b") == "/a/b" or "a" in _normalize_path("/a/b")+ # Resolve so relative becomes absolute+ rel = "foo/bar"+ out = _normalize_path(rel)+ assert Path(out).is_absolute() or out == rel+++def test_load_sources_manifest_missing(tmp_path):+ assert load_sources_manifest(tmp_path, "documents.leann") == {}+++def test_load_sources_manifest_empty(tmp_path):+ (tmp_path / SOURCES_MANIFEST_FILENAME).write_text('{"sources": {}}', encoding="utf-8")+ assert load_sources_manifest(tmp_path, "documents.leann") == {}+++def test_save_and_load_sources_manifest(tmp_path):+ sources = {"/path/to/a": 12345.0, "/path/to/b": 67890.0}+ save_sources_manifest(tmp_path, "documents.leann", sources)+ loaded = load_sources_manifest(tmp_path, "documents.leann")+ assert loaded == sources+ raw = (tmp_path / SOURCES_MANIFEST_FILENAME).read_text(encoding="utf-8")+ data = json.loads(raw)+ assert data.get("sources") == sources+++ os.environ.get("CI") == "true",+ reason="Skip in CI to avoid embedding/model load",+)+def test_incremental_build_adds_only_new_files(tmp_path):+ """Build once with one file, add a second file, run build again without --force; index grows."""+ from leann.cli import LeannCLI++ docs_dir = tmp_path / "docs"+ docs_dir.mkdir()+ (docs_dir / "a.txt").write_text("First document content for indexing.", encoding="utf-8")++ cli = LeannCLI()+ # Override indexes_dir so we use tmp_path for index+ cli.indexes_dir = tmp_path / ".leann" / "indexes"+ cli.indexes_dir.mkdir(parents=True, exist_ok=True)+ index_name = "incr_test"+ index_dir = cli.indexes_dir / index_name+ index_path = cli.get_index_path(index_name)++ parser = cli.create_parser()+ args = parser.parse_args(+ [+ "build",+ index_name,+ "--docs",+ str(docs_dir),+ "--backend-name",+ "hnsw",+ "--no-compact",+ "--embedding-model",+ "all-MiniLM-L6-v2",+ "--embedding-mode",+ "sentence-transformers",+ "--force",+ ]+ )+ import asyncio++ asyncio.run(cli.build_index(args))+ assert index_dir.exists()+ meta_file = index_dir / "documents.leann.meta.json"+ assert meta_file.exists()+ manifest_file = index_dir / "documents.leann.sources.json"+ assert manifest_file.exists(), "Full build should write sources manifest"+ with open(manifest_file, encoding="utf-8") as f:+ manifest = json.load(f)+ assert len(manifest.get("sources", {})) == 1++ # Add second file+ (docs_dir / "b.txt").write_text("Second document content.", encoding="utf-8")++ # Build again without --force (incremental)+ args2 = parser.parse_args(+ [+ "build",+ index_name,+ "--docs",+ str(docs_dir),+ "--backend-name",+ "hnsw",+ "--no-compact",+ "--embedding-model",+ "all-MiniLM-L6-v2",+ "--embedding-mode",+ "sentence-transformers",+ ]+ )+ asyncio.run(cli.build_index(args2))++ with open(manifest_file, encoding="utf-8") as f:+ manifest2 = json.load(f)+ assert len(manifest2.get("sources", {})) == 2, (+ "Manifest should list both files after incremental"+ )++ # Index should still be searchable+ from leann.api import LeannSearcher++ searcher = LeannSearcher(index_path)+ results = searcher.search("Second document", top_k=3)+ searcher.cleanup()+ assert len(results) >= 1+ assert "Second" in results[0].text or "document" in results[0].text