[MLsys2026 Best Paper]: https://arxiv.org/abs/2506.08276. RAG on Everything with LEANN. Enjoy 97% storage savings while running a fast, accurate, and 100% private RAG application on your personal device.
Python aifaissgpt-osslangchainllama-indexllmlocalstorageoffline-firstollamaprivacypythonragretrieval-augmented-generationvector-databasevector-searchvectors
ivf build wo rebuild (#273)
Yichuan Wang9890c35026ce
diff --git a/packages/leann-backend-ivf/leann_backend_ivf/ivf_backend.py b/packages/leann-backend-ivf/leann_backend_ivf/ivf_backend.pyindex 0f7e69f..ca893da 100644--- a/packages/leann-backend-ivf/leann_backend_ivf/ivf_backend.py+++ b/packages/leann-backend-ivf/leann_backend_ivf/ivf_backend.py@@ -1,8 +1,8 @@"""-IVF backend: FAISS IndexIVFFlat wrapped in IndexIDMap2 for stable add/delete by passage id.+IVF backend: FAISS IndexIVFFlat with DirectMap.Hashtable for add/remove by passage id.-Provides clean add_vectors() and remove_ids() APIs (aligned with HNSW update_index flow)-so the core can do incremental updates: detect changed chunk ids, delete and re-insert.+Uses IndexIVFFlat + DirectMap.Hashtable (no IndexIDMap2) so remove_ids works correctly.+Provides add_vectors() and remove_ids() for incremental updates."""import json@@ -125,12 +125,12 @@ class IVFBuilder(LeannBackendBuilderInterface):)ivf = faiss.IndexIVFFlat(quantizer, dim, self.nlist, metric_enum)ivf.train(data)- index = faiss.IndexIDMap2(ivf)+ ivf.set_direct_map_type(faiss.DirectMap.Hashtable)faiss_ids = np.arange(n, dtype=np.int64)- index.add_with_ids(data, faiss_ids)+ ivf.add_with_ids(data, faiss_ids)index_file = index_dir / f"{index_prefix}.index"- faiss.write_index(index, str(index_file))+ faiss.write_index(ivf, str(index_file))id_to_passage = dict(enumerate(ids))_save_id_map(index_dir, index_prefix, id_to_passage, next_id=n)diff --git a/packages/leann-core/src/leann/api.py b/packages/leann-core/src/leann/api.pyindex cde098c..743adc0 100644--- a/packages/leann-core/src/leann/api.py+++ b/packages/leann-core/src/leann/api.py@@ -1,1581 +1,1587 @@-"""-This file contains the core API for the LEANN project, now definitively updated-with the correct, original embedding logic from the user's reference code.-"""--import json-import logging-import os-import pickle-import re-import subprocess-import time-import warnings-from collections import Counter, defaultdict-from dataclasses import dataclass, field-from pathlib import Path-from typing import Any, Literal, Optional, Union--import numpy as np-from leann_backend_hnsw.convert_to_csr import prune_hnsw_embeddings_inplace--from leann.interactive_utils import create_api_session-from leann.interface import LeannBackendSearcherInterface--from .chat import get_llm-from .embedding_server_manager import EmbeddingServerManager-from .interface import LeannBackendFactoryInterface-from .metadata_filter import MetadataFilterEngine-from .registry import BACKEND_REGISTRY--logger = logging.getLogger(__name__)---def get_registered_backends() -> list[str]:- """Get list of registered backend names."""- return list(BACKEND_REGISTRY.keys())---def compute_embeddings(- chunks: list[str],- model_name: str,- mode: str = "sentence-transformers",- use_server: bool = True,- port: Optional[int] = None,- is_build=False,- provider_options: Optional[dict[str, Any]] = None,-) -> np.ndarray:- """- Computes embeddings using different backends.-- Args:- chunks: List of text chunks to embed- model_name: Name of the embedding model- mode: Embedding backend mode. Options:- - "sentence-transformers": Use sentence-transformers library (default)- - "mlx": Use MLX backend for Apple Silicon- - "openai": Use OpenAI embedding API- - "gemini": Use Google Gemini embedding API- use_server: Whether to use embedding server (True for search, False for build)-- Returns:- numpy array of embeddings- """- if use_server:- # Use embedding server (for search/query)- if port is None:- raise ValueError("port is required when use_server is True")- return compute_embeddings_via_server(chunks, model_name, port=port)- else:- # Use direct computation (for build_index)- from .embedding_compute import (- compute_embeddings as compute_embeddings_direct,- )-- return compute_embeddings_direct(- chunks,- model_name,- mode=mode,- is_build=is_build,- provider_options=provider_options,- )---def compute_embeddings_via_server(chunks: list[str], model_name: str, port: int) -> np.ndarray:- """Computes embeddings using sentence-transformers.-- Args:- chunks: List of text chunks to embed- model_name: Name of the sentence transformer model- """- logger.info(- f"Computing embeddings for {len(chunks)} chunks using SentenceTransformer model '{model_name}' (via embedding server)..."- )- import msgpack- import numpy as np- import zmq-- # Connect to embedding server- context = zmq.Context()- socket = context.socket(zmq.REQ)- socket.connect(f"tcp://localhost:{port}")-- # Send chunks to server for embedding computation- request = chunks- socket.send(msgpack.packb(request))-- # Receive embeddings from server- response = socket.recv()- embeddings_list = msgpack.unpackb(response)-- # Convert back to numpy array- embeddings = np.array(embeddings_list, dtype=np.float32)-- socket.close()- context.term()-- return embeddings---@dataclass-class SearchResult:- id: str- score: float- text: str- metadata: dict[str, Any] = field(default_factory=dict)---class PassageManager:- def __init__(- self, passage_sources: list[dict[str, Any]], metadata_file_path: Optional[str] = None- ):- self.offset_maps: dict[str, dict[str, int]] = {}- self.passage_files: dict[str, str] = {}- # Avoid materializing a single gigantic global map to reduce memory- # footprint on very large corpora (e.g., 60M+ passages). Instead, keep- # per-shard maps and do a lightweight per-shard lookup on demand.- self._total_count: int = 0- self.filter_engine = MetadataFilterEngine() # Initialize filter engine-- # Derive index base name for standard sibling fallbacks, e.g., <index_name>.passages.*- index_name_base = None- if metadata_file_path:- meta_name = Path(metadata_file_path).name- if meta_name.endswith(".meta.json"):- index_name_base = meta_name[: -len(".meta.json")]-- for source in passage_sources:- assert source["type"] == "jsonl", "only jsonl is supported"- passage_file = source.get("path", "")- index_file = source.get("index_path", "") # .idx file-- # Fix path resolution - relative paths should be relative to metadata file directory- def _resolve_candidates(- primary: str,- relative_key: str,- default_name: Optional[str],- source_dict: dict[str, Any],- ) -> list[Path]:- """- Build an ordered list of candidate paths. For relative paths specified in- metadata, prefer resolution relative to the metadata file directory first,- then fall back to CWD-based resolution, and finally to conventional- sibling defaults (e.g., <index_base>.passages.idx / .jsonl).- """- candidates: list[Path] = []- # 1) Primary path- if primary:- p = Path(primary)- if p.is_absolute():- candidates.append(p)- else:- # Prefer metadata-relative resolution for relative paths- if metadata_file_path:- candidates.append(Path(metadata_file_path).parent / p)- # Also consider CWD-relative as a fallback for legacy layouts- candidates.append(Path.cwd() / p)- # 2) metadata-relative explicit relative key (if present)- if metadata_file_path and source_dict.get(relative_key):- candidates.append(Path(metadata_file_path).parent / source_dict[relative_key])- # 3) metadata-relative standard sibling filename- if metadata_file_path and default_name:- candidates.append(Path(metadata_file_path).parent / default_name)- return candidates-- # Build candidate lists and pick first existing; otherwise keep last candidate for error message- idx_default = f"{index_name_base}.passages.idx" if index_name_base else None- idx_candidates = _resolve_candidates(- index_file, "index_path_relative", idx_default, source- )- pas_default = f"{index_name_base}.passages.jsonl" if index_name_base else None- pas_candidates = _resolve_candidates(passage_file, "path_relative", pas_default, source)-- def _pick_existing(cands: list[Path]) -> str:- for c in cands:- if c.exists():- return str(c.resolve())- # Fallback to last candidate (best guess) even if not exists; will error below- return str(cands[-1].resolve()) if cands else ""-- index_file = _pick_existing(idx_candidates)- passage_file = _pick_existing(pas_candidates)-- if not Path(index_file).exists():- raise FileNotFoundError(f"Passage index file not found: {index_file}")-- with open(index_file, "rb") as f:- offset_map: dict[str, int] = pickle.load(f)- self.offset_maps[passage_file] = offset_map- self.passage_files[passage_file] = passage_file- self._total_count += len(offset_map)-- def get_passage(self, passage_id: str) -> dict[str, Any]:- # Fast path: check each shard map (there are typically few shards).- # This avoids building a massive combined dict while keeping lookups- # bounded by the number of shards.- for passage_file, offset_map in self.offset_maps.items():- try:- offset = offset_map[passage_id]- with open(passage_file, encoding="utf-8") as f:- f.seek(offset)- return json.loads(f.readline())- except KeyError:- continue- raise KeyError(f"Passage ID not found: {passage_id}")-- def filter_search_results(- self,- search_results: list[SearchResult],- metadata_filters: Optional[dict[str, dict[str, Union[str, int, float, bool, list]]]],- ) -> list[SearchResult]:- """- Apply metadata filters to search results.-- Args:- search_results: List of SearchResult objects- metadata_filters: Filter specifications to apply-- Returns:- Filtered list of SearchResult objects- """- if not metadata_filters:- return search_results-- logger.debug(f"Applying metadata filters to {len(search_results)} results")-- # Convert SearchResult objects to dictionaries for the filter engine- result_dicts = []- for result in search_results:- result_dicts.append(- {- "id": result.id,- "score": result.score,- "text": result.text,- "metadata": result.metadata,- }- )-- # Apply filters using the filter engine- filtered_dicts = self.filter_engine.apply_filters(result_dicts, metadata_filters)-- # Convert back to SearchResult objects- filtered_results = []- for result_dict in filtered_dicts:- filtered_results.append(- SearchResult(- id=result_dict["id"],- score=result_dict["score"],- text=result_dict["text"],- metadata=result_dict["metadata"],- )- )-- logger.debug(f"Filtered results: {len(filtered_results)} remaining")- return filtered_results-- def __len__(self) -> int:- return self._total_count---class BM25Scorer:- def __init__(self, k1: float = 1.2, b: float = 0.75):- self.k1 = k1- self.b = b- self.doc_freqs = None # How many docs contain each term (DF)- self.doc_lengths = {} # How long each doc is (in words)- self.word_counts = {} # How many times each word appears in each doc (TF)- self.avg_doc_length = None- self.corpus_size = None- self.idlist = set() # List of all document IDs for easier searching-- def _tokenize(self, text: str) -> list[str]:- return re.sub(r"[^\w\s]", "", text).lower().split()-- def fit(self, documents: list[dict[str, Any]]):- """- Build BM25 statistics from a document corpus.- Must be called before scoring.- """- self.corpus_size = len(documents)- self.doc_lengths = {}- self.word_counts = {}- self.idlist = set()- doc_freqs = defaultdict(int)-- for doc_data in documents:- doc_id = doc_data["id"]- words = self._tokenize(doc_data["text"])- doc_length = len(words)- self.doc_lengths[doc_id] = doc_length-- unique_words = set(words)- for word in unique_words:- doc_freqs[word] += 1- self.word_counts[doc_id] = dict(Counter(words))- self.idlist.add(doc_id)-- self.doc_freqs = dict(doc_freqs)- self.avg_doc_length = sum(self.doc_lengths.values()) / len(self.doc_lengths)-- def score(self, query_words: list[str], document_id: str) -> float:- if (- self.doc_freqs is None- or self.doc_lengths == {}- or self.word_counts == {}- or self.avg_doc_length is None- or self.corpus_size is None- ):- raise ValueError("BM25 model not fitted. Call fit() before scoring.")-- passage_words = self.word_counts[document_id]- passage_length = sum(passage_words.values())- score = 0.0- for word in query_words:- if word not in self.doc_freqs:- continue- word_freq = passage_words[word] if word in passage_words else 0- idf = np.log(- (self.corpus_size - self.doc_freqs[word] + 0.5) / (self.doc_freqs[word] + 0.5) + 1- )- tf = (word_freq * (self.k1 + 1)) / (- word_freq + self.k1 * (1 - self.b + self.b * (passage_length / self.avg_doc_length))- )- score += idf * tf- return score-- def search(self, query: str, top_k: int = 5) -> list[SearchResult]:- query_words = self._tokenize(query)- scores = {doc_id: self.score(query_words, doc_id) for doc_id in self.idlist}- sorted_scores = sorted(scores.items(), key=lambda x: x[1], reverse=True)- return [- SearchResult(id=doc_id, score=score, text="", metadata={})- for doc_id, score in sorted_scores[:top_k]- ]---class LeannBuilder:- def __init__(- self,- backend_name: str,- embedding_model: str = "facebook/contriever",- dimensions: Optional[int] = None,- embedding_mode: str = "sentence-transformers",- embedding_options: Optional[dict[str, Any]] = None,- **backend_kwargs,- ):- self.backend_name = backend_name- # Normalize incompatible combinations early (for consistent metadata)- if backend_name == "hnsw":- is_recompute = backend_kwargs.get("is_recompute", True)- is_compact = backend_kwargs.get("is_compact", True)- if is_recompute is False and is_compact is True:- warnings.warn(- "HNSW with is_recompute=False requires non-compact storage. Forcing is_compact=False.",- UserWarning,- stacklevel=2,- )- backend_kwargs["is_compact"] = False-- backend_factory: Optional[LeannBackendFactoryInterface] = BACKEND_REGISTRY.get(backend_name)- if backend_factory is None:- raise ValueError(f"Backend '{backend_name}' not found or not registered.")- self.backend_factory = backend_factory- self.embedding_model = embedding_model- self.dimensions = dimensions- self.embedding_mode = embedding_mode- self.embedding_options = embedding_options or {}-- # Check if we need to use cosine distance for normalized embeddings- normalized_embeddings_models = {- # OpenAI models- ("openai", "text-embedding-ada-002"),- ("openai", "text-embedding-3-small"),- ("openai", "text-embedding-3-large"),- # Voyage AI models- ("voyage", "voyage-2"),- ("voyage", "voyage-3"),- ("voyage", "voyage-large-2"),- ("voyage", "voyage-multilingual-2"),- ("voyage", "voyage-code-2"),- # Cohere models- ("cohere", "embed-english-v3.0"),- ("cohere", "embed-multilingual-v3.0"),- ("cohere", "embed-english-light-v3.0"),- ("cohere", "embed-multilingual-light-v3.0"),- }-- # Also check for patterns in model names- is_normalized = False- current_model_lower = embedding_model.lower()- current_mode_lower = embedding_mode.lower()-- # Check exact matches- for mode, model in normalized_embeddings_models:- if (current_mode_lower == mode and current_model_lower == model) or (- mode in current_mode_lower and model in current_model_lower- ):- is_normalized = True- break-- # Check patterns- if not is_normalized:- # OpenAI patterns- if "openai" in current_mode_lower or "openai" in current_model_lower:- if any(- pattern in current_model_lower- for pattern in ["text-embedding", "ada", "3-small", "3-large"]- ):- is_normalized = True- # Voyage patterns- elif "voyage" in current_mode_lower or "voyage" in current_model_lower:- is_normalized = True- # Cohere patterns- elif "cohere" in current_mode_lower or "cohere" in current_model_lower:- if "embed" in current_model_lower:- is_normalized = True-- # Handle distance metric- if is_normalized and "distance_metric" not in backend_kwargs:- backend_kwargs["distance_metric"] = "cosine"- warnings.warn(- f"Detected normalized embeddings model '{embedding_model}' with mode '{embedding_mode}'. "- f"Automatically setting distance_metric='cosine' for optimal performance. "- f"Normalized embeddings (L2 norm = 1) should use cosine similarity instead of MIPS.",- UserWarning,- stacklevel=2,- )- elif is_normalized and backend_kwargs.get("distance_metric", "").lower() != "cosine":- current_metric = backend_kwargs.get("distance_metric", "mips")- warnings.warn(- f"Warning: Using '{current_metric}' distance metric with normalized embeddings model "- f"'{embedding_model}' may lead to suboptimal search results. "- f"Consider using 'cosine' distance metric for better performance.",- UserWarning,- stacklevel=2,- )-- self.backend_kwargs = backend_kwargs- self.chunks: list[dict[str, Any]] = []-- def add_text(self, text: str, metadata: Optional[dict[str, Any]] = None):- if metadata is None:- metadata = {}- passage_id = metadata.get("id", str(len(self.chunks)))- chunk_data = {"id": passage_id, "text": text, "metadata": metadata}- self.chunks.append(chunk_data)-- def build_index(self, index_path: str):- if not self.chunks:- raise ValueError("No chunks added.")-- # Filter out invalid/empty text chunks early to keep passage and embedding counts aligned- valid_chunks: list[dict[str, Any]] = []- skipped = 0- for chunk in self.chunks:- text = chunk.get("text", "")- if isinstance(text, str) and text.strip():- valid_chunks.append(chunk)- else:- skipped += 1- if skipped > 0:- print(- f"Warning: Skipping {skipped} empty/invalid text chunk(s). Processing {len(valid_chunks)} valid chunks"- )- self.chunks = valid_chunks- if not self.chunks:- raise ValueError("All provided chunks are empty or invalid. Nothing to index.")- if self.dimensions is None:- self.dimensions = len(- compute_embeddings(- ["dummy"],- self.embedding_model,- self.embedding_mode,- use_server=False,- provider_options=self.embedding_options,- )[0]- )- path = Path(index_path)- index_dir = path.parent- index_name = path.name- index_dir.mkdir(parents=True, exist_ok=True)- passages_file = index_dir / f"{index_name}.passages.jsonl"- offset_file = index_dir / f"{index_name}.passages.idx"- offset_map = {}- with open(passages_file, "w", encoding="utf-8") as f:- try:- from tqdm import tqdm-- chunk_iterator = tqdm(self.chunks, desc="Writing passages", unit="chunk")- except ImportError:- chunk_iterator = self.chunks-- for chunk in chunk_iterator:- offset = f.tell()- json.dump(- {- "id": chunk["id"],- "text": chunk["text"],- "metadata": chunk["metadata"],- },- f,- ensure_ascii=False,- )- f.write("\n")- offset_map[chunk["id"]] = offset- with open(offset_file, "wb") as f:- pickle.dump(offset_map, f)- texts_to_embed = [c["text"] for c in self.chunks]- embeddings = compute_embeddings(- texts_to_embed,- self.embedding_model,- self.embedding_mode,- use_server=False,- is_build=True,- provider_options=self.embedding_options,- )- string_ids = [chunk["id"] for chunk in self.chunks]- # Persist ID map alongside index so backends that return integer labels can remap to passage IDs- try:- idmap_file = (- index_dir- / f"{index_name[: -len('.leann')] if index_name.endswith('.leann') else index_name}.ids.txt"- )- with open(idmap_file, "w", encoding="utf-8") as f:- for sid in string_ids:- f.write(str(sid) + "\n")- except Exception:- pass- current_backend_kwargs = {**self.backend_kwargs, "dimensions": self.dimensions}- builder_instance = self.backend_factory.builder(**current_backend_kwargs)- builder_instance.build(embeddings, string_ids, index_path, **current_backend_kwargs)- leann_meta_path = index_dir / f"{index_name}.meta.json"- meta_data = {- "version": "1.0",- "backend_name": self.backend_name,- "embedding_model": self.embedding_model,- "dimensions": self.dimensions,- "backend_kwargs": self.backend_kwargs,- "embedding_mode": self.embedding_mode,- "passage_sources": [- {- "type": "jsonl",- # Preserve existing relative file names (backward-compatible)- "path": passages_file.name,- "index_path": offset_file.name,- # Add optional redundant relative keys for remote build portability (non-breaking)- "path_relative": passages_file.name,- "index_path_relative": offset_file.name,- }- ],- }-- if self.embedding_options:- meta_data["embedding_options"] = self.embedding_options-- # Add storage status flags for HNSW backend- if self.backend_name == "hnsw":- is_compact = self.backend_kwargs.get("is_compact", True)- is_recompute = self.backend_kwargs.get("is_recompute", True)- meta_data["is_compact"] = is_compact- meta_data["is_pruned"] = bool(is_recompute)- with open(leann_meta_path, "w", encoding="utf-8") as f:- json.dump(meta_data, f, indent=2)-- def build_index_from_embeddings(self, index_path: str, embeddings_file: str):- """- Build an index from pre-computed embeddings stored in a pickle file.-- Args:- index_path: Path where the index will be saved- embeddings_file: Path to pickle file containing (ids, embeddings) tuple- """- # Load pre-computed embeddings- with open(embeddings_file, "rb") as f:- data = pickle.load(f)-- if not isinstance(data, tuple) or len(data) != 2:- raise ValueError(- f"Invalid embeddings file format. Expected tuple with 2 elements, got {type(data)}"- )-- ids, embeddings = data-- if not isinstance(embeddings, np.ndarray):- raise ValueError(f"Expected embeddings to be numpy array, got {type(embeddings)}")-- if len(ids) != embeddings.shape[0]:- raise ValueError(- f"Mismatch between number of IDs ({len(ids)}) and embeddings ({embeddings.shape[0]})"- )-- # Validate/set dimensions- embedding_dim = embeddings.shape[1]- if self.dimensions is None:- self.dimensions = embedding_dim- elif self.dimensions != embedding_dim:- raise ValueError(f"Dimension mismatch: expected {self.dimensions}, got {embedding_dim}")-- logger.info(- f"Building index from precomputed embeddings: {len(ids)} items, {embedding_dim} dimensions"- )-- # Ensure we have text data for each embedding- if len(self.chunks) != len(ids):- # If no text chunks provided, create placeholder text entries- if not self.chunks:- logger.info("No text chunks provided, creating placeholder entries...")- for id_val in ids:- self.add_text(- f"Document {id_val}",- metadata={"id": str(id_val), "from_embeddings": True},- )- else:- raise ValueError(- f"Number of text chunks ({len(self.chunks)}) doesn't match number of embeddings ({len(ids)})"- )-- # Build file structure- path = Path(index_path)- index_dir = path.parent- index_name = path.name- index_dir.mkdir(parents=True, exist_ok=True)- passages_file = index_dir / f"{index_name}.passages.jsonl"- offset_file = index_dir / f"{index_name}.passages.idx"-- # Write passages and create offset map- offset_map = {}- with open(passages_file, "w", encoding="utf-8") as f:- for chunk in self.chunks:- offset = f.tell()- json.dump(- {- "id": chunk["id"],- "text": chunk["text"],- "metadata": chunk["metadata"],- },- f,- ensure_ascii=False,- )- f.write("\n")- offset_map[chunk["id"]] = offset-- with open(offset_file, "wb") as f:- pickle.dump(offset_map, f)-- # Build the vector index using precomputed embeddings- string_ids = [str(id_val) for id_val in ids]- # Persist ID map (order == embeddings order)- try:- idmap_file = (- index_dir- / f"{index_name[: -len('.leann')] if index_name.endswith('.leann') else index_name}.ids.txt"- )- with open(idmap_file, "w", encoding="utf-8") as f:- for sid in string_ids:- f.write(str(sid) + "\n")- except Exception:- pass- current_backend_kwargs = {**self.backend_kwargs, "dimensions": self.dimensions}- builder_instance = self.backend_factory.builder(**current_backend_kwargs)- builder_instance.build(embeddings, string_ids, index_path)-- # Create metadata file- leann_meta_path = index_dir / f"{index_name}.meta.json"- meta_data = {- "version": "1.0",- "backend_name": self.backend_name,- "embedding_model": self.embedding_model,- "dimensions": self.dimensions,- "backend_kwargs": self.backend_kwargs,- "embedding_mode": self.embedding_mode,- "passage_sources": [- {- "type": "jsonl",- # Preserve existing relative file names (backward-compatible)- "path": passages_file.name,- "index_path": offset_file.name,- # Add optional redundant relative keys for remote build portability (non-breaking)- "path_relative": passages_file.name,- "index_path_relative": offset_file.name,- }- ],- "built_from_precomputed_embeddings": True,- "embeddings_source": str(embeddings_file),- }-- if self.embedding_options:- meta_data["embedding_options"] = self.embedding_options-- # Add storage status flags for HNSW backend- if self.backend_name == "hnsw":- is_compact = self.backend_kwargs.get("is_compact", True)- is_recompute = self.backend_kwargs.get("is_recompute", True)- meta_data["is_compact"] = is_compact- meta_data["is_pruned"] = bool(is_recompute)-- with open(leann_meta_path, "w", encoding="utf-8") as f:- json.dump(meta_data, f, indent=2)-- logger.info(f"Index built successfully from precomputed embeddings: {index_path}")-- def update_index(self, index_path: str, remove_passage_ids: Optional[list[str]] = None) -> None:- """Append new passages and vectors to an existing index (HNSW or IVF).- For IVF, optional remove_passage_ids removes those ids first (e.g. from file-change API).- """- if not self.chunks and not remove_passage_ids:- raise ValueError("No new chunks or passage ids to remove provided for update.")-- path = Path(index_path)- index_dir = path.parent- index_name = path.name- index_prefix = path.stem-- meta_path = index_dir / f"{index_name}.meta.json"- passages_file = index_dir / f"{index_name}.passages.jsonl"- offset_file = index_dir / f"{index_name}.passages.idx"- index_file = index_dir / f"{index_prefix}.index"-- if not meta_path.exists() or not passages_file.exists() or not offset_file.exists():- raise FileNotFoundError("Index metadata or passage files are missing; cannot update.")- if not index_file.exists():- raise FileNotFoundError(f"Index file not found: {index_file}")-- with open(meta_path, encoding="utf-8") as f:- meta = json.load(f)- backend_name = meta.get("backend_name")- if backend_name != self.backend_name:- raise ValueError(- f"Index was built with backend '{backend_name}', cannot update with '{self.backend_name}'."- )-- with open(offset_file, "rb") as f:- offset_map: dict[str, int] = pickle.load(f)- existing_ids = set(offset_map.keys())-- # IVF: optional delete (for reindex / file-change: remove then re-insert)- if remove_passage_ids and backend_name == "ivf":- try:- from leann_backend_ivf import remove_ids as ivf_remove_ids-- ivf_remove_ids(str(path), remove_passage_ids)- except ImportError:- raise RuntimeError(- "IVF backend required for remove_ids. Install leann-backend-ivf."- )- for pid in remove_passage_ids:- offset_map.pop(pid, None)- with open(offset_file, "wb") as f:- pickle.dump(offset_map, f)-- if not self.chunks:- meta["total_passages"] = len(offset_map)- with open(meta_path, "w", encoding="utf-8") as f:- json.dump(meta, f, indent=2)- self.chunks.clear()- return-- meta_backend_kwargs = meta.get("backend_kwargs", {})- if backend_name == "hnsw":- index_is_compact = meta.get("is_compact", meta_backend_kwargs.get("is_compact", True))- if index_is_compact:- raise ValueError(- "Compact HNSW indices do not support in-place updates. Rebuild required."- )-- distance_metric = meta_backend_kwargs.get(- "distance_metric", self.backend_kwargs.get("distance_metric", "mips")- ).lower()- needs_recompute = bool(- meta.get("is_pruned")- or meta_backend_kwargs.get("is_recompute")- or self.backend_kwargs.get("is_recompute")- )-- valid_chunks: list[dict[str, Any]] = []- for chunk in self.chunks:- text = chunk.get("text", "")- if not isinstance(text, str) or not text.strip():- continue- metadata = chunk.setdefault("metadata", {})- passage_id = chunk.get("id") or metadata.get("id")- if passage_id and passage_id in existing_ids:- raise ValueError(f"Passage ID '{passage_id}' already exists in the index.")- valid_chunks.append(chunk)-- if not valid_chunks:- raise ValueError("No valid chunks to append.")-- texts_to_embed = [chunk["text"] for chunk in valid_chunks]- embeddings = compute_embeddings(- texts_to_embed,- self.embedding_model,- self.embedding_mode,- use_server=False,- is_build=True,- provider_options=self.embedding_options,- )-- embedding_dim = embeddings.shape[1]- expected_dim = meta.get("dimensions")- if expected_dim is not None and expected_dim != embedding_dim:- raise ValueError(- f"Dimension mismatch during update: existing index uses {expected_dim}, got {embedding_dim}."- )-- embeddings = np.ascontiguousarray(embeddings, dtype=np.float32)- if distance_metric == "cosine":- norms = np.linalg.norm(embeddings, axis=1, keepdims=True)- norms[norms == 0] = 1- embeddings = embeddings / norms-- # IVF: add_vectors then append passages/offset (no ZMQ/server)- if backend_name == "ivf":- for i, chunk in enumerate(valid_chunks):- pid = chunk.get("id") or chunk.get("metadata", {}).get("id")- if not pid:- pid = str(len(offset_map) + i)- chunk.setdefault("metadata", {})["id"] = pid- chunk["id"] = pid- passage_ids = [c["id"] for c in valid_chunks]- try:- from leann_backend_ivf import add_vectors as ivf_add_vectors-- ivf_add_vectors(str(path), embeddings, passage_ids)- except ImportError:- raise RuntimeError("IVF backend required. Install leann-backend-ivf.")- rollback_passages_size = passages_file.stat().st_size if passages_file.exists() else 0- offset_map_backup = offset_map.copy()- try:- with open(passages_file, "a", encoding="utf-8") as f:- for chunk in valid_chunks:- off = f.tell()- json.dump(- {- "id": chunk["id"],- "text": chunk["text"],- "metadata": chunk.get("metadata", {}),- },- f,- ensure_ascii=False,- )- f.write("\n")- offset_map[chunk["id"]] = off- with open(offset_file, "wb") as f:- pickle.dump(offset_map, f)- meta["total_passages"] = len(offset_map)- with open(meta_path, "w", encoding="utf-8") as f:- json.dump(meta, f, indent=2)- logger.info(- "Appended %d passages to IVF index '%s'. Total: %d",- len(valid_chunks),- index_path,- len(offset_map),- )- except Exception:- if passages_file.exists():- with open(passages_file, "rb+") as f:- f.truncate(rollback_passages_size)- offset_map = offset_map_backup- with open(offset_file, "wb") as f:- pickle.dump(offset_map, f)- raise- self.chunks.clear()- return-- # HNSW path below- from leann_backend_hnsw import faiss-- index = faiss.read_index(str(index_file))- if hasattr(index, "is_recompute"):- index.is_recompute = needs_recompute- print(f"index.is_recompute: {index.is_recompute}")- if getattr(index, "storage", None) is None:- if index.metric_type == faiss.METRIC_INNER_PRODUCT:- storage_index = faiss.IndexFlatIP(index.d)- else:- storage_index = faiss.IndexFlatL2(index.d)- index.storage = storage_index- index.own_fields = True- # Faiss expects storage.ntotal to reflect the existing graph's- # population (even if the vectors themselves were pruned from disk- # for recompute mode). When we attach a fresh IndexFlat here its- # ntotal starts at zero, which later causes IndexHNSW::add to- # believe new "preset" levels were provided and trips the- # `n0 + n == levels.size()` assertion. Seed the temporary storage- # with the current ntotal so Faiss maintains the proper offset for- # incoming vectors.- try:- storage_index.ntotal = index.ntotal- except AttributeError:- # Older Faiss builds may not expose ntotal as a writable- # attribute; in that case we fall back to the default behaviour.- pass- if index.d != embedding_dim:- raise ValueError(- f"Existing index dimension ({index.d}) does not match new embeddings ({embedding_dim})."- )-- passage_meta_mode = meta.get("embedding_mode", self.embedding_mode)- passage_provider_options = meta.get("embedding_options", self.embedding_options)-- base_id = index.ntotal- for offset, chunk in enumerate(valid_chunks):- new_id = str(base_id + offset)- chunk.setdefault("metadata", {})["id"] = new_id- chunk["id"] = new_id-- # Append passages/offsets before we attempt index.add so the ZMQ server- # can resolve newly assigned IDs during recompute. Keep rollback hooks- # so we can restore files if the update fails mid-way.- rollback_passages_size = passages_file.stat().st_size if passages_file.exists() else 0- offset_map_backup = offset_map.copy()-- try:- with open(passages_file, "a", encoding="utf-8") as f:- for chunk in valid_chunks:- offset = f.tell()- json.dump(- {- "id": chunk["id"],- "text": chunk["text"],- "metadata": chunk.get("metadata", {}),- },- f,- ensure_ascii=False,- )- f.write("\n")- offset_map[chunk["id"]] = offset-- with open(offset_file, "wb") as f:- pickle.dump(offset_map, f)-- server_manager: Optional[EmbeddingServerManager] = None- server_started = False- requested_zmq_port = int(os.getenv("LEANN_UPDATE_ZMQ_PORT", "5557"))-- try:- if needs_recompute:- server_manager = EmbeddingServerManager(- backend_module_name="leann_backend_hnsw.hnsw_embedding_server"- )- server_started, actual_port = server_manager.start_server(- port=requested_zmq_port,- model_name=self.embedding_model,- embedding_mode=passage_meta_mode,- passages_file=str(meta_path),- distance_metric=distance_metric,- use_daemon=False,- enable_warmup=False,- provider_options=passage_provider_options,- )- if not server_started:- raise RuntimeError(- "Failed to start HNSW embedding server for recompute update."- )- if actual_port != requested_zmq_port:- logger.warning(- "Embedding server started on port %s instead of requested %s. "- "Using reassigned port.",- actual_port,- requested_zmq_port,- )- if hasattr(index.hnsw, "set_zmq_port"):- index.hnsw.set_zmq_port(actual_port)- elif hasattr(index, "set_zmq_port"):- index.set_zmq_port(actual_port)-- if needs_recompute:- for i in range(embeddings.shape[0]):- print(f"add {i} embeddings")- index.add(1, faiss.swig_ptr(embeddings[i : i + 1]))- else:- index.add(embeddings.shape[0], faiss.swig_ptr(embeddings))- faiss.write_index(index, str(index_file))- finally:- if server_started and server_manager is not None:- server_manager.stop_server()-- except Exception:- # Roll back appended passages/offset map to keep files consistent.- if passages_file.exists():- with open(passages_file, "rb+") as f:- f.truncate(rollback_passages_size)- offset_map = offset_map_backup- with open(offset_file, "wb") as f:- pickle.dump(offset_map, f)- raise-- meta["total_passages"] = len(offset_map)- with open(meta_path, "w", encoding="utf-8") as f:- json.dump(meta, f, indent=2)-- logger.info(- "Appended %d passages to index '%s'. New total: %d",- len(valid_chunks),- index_path,- len(offset_map),- )-- self.chunks.clear()-- if needs_recompute:- prune_hnsw_embeddings_inplace(str(index_file))---class LeannSearcher:- def __init__(- self,- index_path: str,- enable_warmup: bool = True,- recompute_embeddings: bool = True,- use_daemon: bool = True,- daemon_ttl_seconds: int = 900,- **backend_kwargs,- ):- # Fix path resolution for Colab and other environments- if not Path(index_path).is_absolute():- index_path = str(Path(index_path).resolve())-- self.meta_path_str = f"{index_path}.meta.json"- if not Path(self.meta_path_str).exists():- parent_dir = Path(index_path).parent- print(- f"Leann metadata file not found at {self.meta_path_str}, and you may need to rm -rf {parent_dir}"- )- # highlight in red the filenotfound error- raise FileNotFoundError(- f"Leann metadata file not found at {self.meta_path_str}, \033[91m you may need to rm -rf {parent_dir}\033[0m"- )- with open(self.meta_path_str, encoding="utf-8") as f:- self.meta_data = json.load(f)- backend_name = self.meta_data["backend_name"]- self.embedding_model = self.meta_data["embedding_model"]- # Support both old and new format- self.embedding_mode = self.meta_data.get("embedding_mode", "sentence-transformers")- self.embedding_options = self.meta_data.get("embedding_options", {})- # Delegate portability handling to PassageManager- self.passage_manager = PassageManager(- self.meta_data.get("passage_sources", []), metadata_file_path=self.meta_path_str- )- # Preserve backend name for conditional parameter forwarding- self.backend_name = backend_name- backend_factory = BACKEND_REGISTRY.get(backend_name)- if backend_factory is None:- raise ValueError(f"Backend '{backend_name}' not found.")-- # Global recompute flag for this searcher (explicit knob, default True)- self.recompute_embeddings: bool = bool(recompute_embeddings)-- # Warmup flag: keep using the existing enable_warmup parameter,- # but default it to True so cold-start happens earlier.- self._warmup: bool = bool(enable_warmup)- self._use_daemon: bool = bool(use_daemon)- self._daemon_ttl_seconds: int = int(daemon_ttl_seconds)-- final_kwargs = {**self.meta_data.get("backend_kwargs", {}), **backend_kwargs}- final_kwargs["enable_warmup"] = self._warmup- final_kwargs["use_daemon"] = self._use_daemon- final_kwargs["daemon_ttl_seconds"] = self._daemon_ttl_seconds- if self.embedding_options:- final_kwargs.setdefault("embedding_options", self.embedding_options)- self.backend_impl: LeannBackendSearcherInterface = backend_factory.searcher(- index_path, **final_kwargs- )- self.bm25_scorer: Optional[BM25Scorer] = None-- # Optional one-shot warmup at construction time to hide cold-start latency.- if self._warmup:- self.warmup()-- def warmup(self) -> None:- """Warm up embedding path so first user query is faster."""- try:- _ = self.backend_impl.compute_query_embedding(- "__LEANN_WARMUP__",- use_server_if_available=self.recompute_embeddings,- )- except Exception as exc:- logger.warning(f"Warmup embedding failed (ignored): {exc}")-- def search(- self,- query: str,- top_k: int = 5,- complexity: int = 64,- beam_width: int = 1,- prune_ratio: float = 0.0,- recompute_embeddings: Optional[bool] = None,- pruning_strategy: Literal["global", "local", "proportional"] = "global",- expected_zmq_port: int = 5557,- metadata_filters: Optional[dict[str, dict[str, Union[str, int, float, bool, list]]]] = None,- batch_size: int = 0,- use_grep: bool = False,- gemma: float = 1.0,- provider_options: Optional[dict[str, Any]] = None,- **kwargs,- ) -> list[SearchResult]:- """- Search for nearest neighbors with optional metadata filtering.-- Args:- query: Text query to search for- top_k: Number of nearest neighbors to return- complexity: Search complexity/candidate list size, higher = more accurate but slower- beam_width: Number of parallel search paths/IO requests per iteration- prune_ratio: Ratio of neighbors to prune via approximate distance (0.0-1.0)- recompute_embeddings: (Deprecated) Per-call override for recompute mode.- Configure this at LeannSearcher(..., recompute_embeddings=...) instead.- pruning_strategy: Candidate selection strategy - "global" (default), "local", or "proportional"- expected_zmq_port: ZMQ port for embedding server communication- metadata_filters: Optional filters to apply to search results based on metadata.- Format: {"field_name": {"operator": value}}- Supported operators:- - Comparison: "==", "!=", "<", "<=", ">", ">="- - Membership: "in", "not_in"- - String: "contains", "starts_with", "ends_with"- Example: {"chapter": {"<=": 5}, "tags": {"in": ["fiction", "drama"]}}- gemma: Weight of vector search results in hybrid search (0.0-1.0), 1 = pure vector search, 0 = pure keyword search- **kwargs: Backend-specific parameters-- Returns:- List of SearchResult objects with text, metadata, and similarity scores- """- # Handle grep search- if use_grep:- return self._grep_search(query, top_k)-- logger.info("🔍 LeannSearcher.search() called:")- logger.info(f" Query: '{query}'")- logger.info(f" Top_k: {top_k}")- logger.info(f" Metadata filters: {metadata_filters}")- logger.info(f" Additional kwargs: {kwargs}")-- # Smart top_k detection and adjustment- # Use PassageManager length (sum of shard sizes) to avoid- # depending on a massive combined map- total_docs = len(self.passage_manager)- original_top_k = top_k- if top_k > total_docs:- top_k = total_docs- logger.warning(- f" ⚠️ Requested top_k ({original_top_k}) exceeds total documents ({total_docs})"- )- logger.warning(f" ✅ Auto-adjusted top_k to {top_k} to match available documents")-- # Handle pure keyword search- if gemma == 0.0:- start_time = time.time()- bm25_results = self._bm25_search(query, top_k)- # Convert BM25 results to the expected format- results = {- "labels": [[r.id for r in bm25_results]],- "distances": [[r.score for r in bm25_results]],- }- else:- # Perform vector search- zmq_port = None-- # Resolve effective recompute flag for this search.- if recompute_embeddings is not None:- logger.warning(- "LeannSearcher.search(..., recompute_embeddings=...) is deprecated and "- "will be removed in a future version. Configure recompute at "- "LeannSearcher(..., recompute_embeddings=...) instead."- )- effective_recompute = bool(recompute_embeddings)- else:- effective_recompute = self.recompute_embeddings-- start_time = time.time()- if effective_recompute:- zmq_port = self.backend_impl._ensure_server_running(- self.meta_path_str,- port=expected_zmq_port,- enable_warmup=self._warmup,- use_daemon=self._use_daemon,- daemon_ttl_seconds=self._daemon_ttl_seconds,- **kwargs,- )- del expected_zmq_port- zmq_time = time.time() - start_time- logger.info(f" Launching server time: {zmq_time} seconds")-- start_time = time.time()-- # Extract query template from stored embedding_options with fallback chain:- # 1. Check provider_options override (highest priority)- # 2. Check query_prompt_template (new format)- # 3. Check prompt_template (old format for backward compat)- # 4. None (no template)- query_template = None- if provider_options and "prompt_template" in provider_options:- query_template = provider_options["prompt_template"]- elif "query_prompt_template" in self.embedding_options:- query_template = self.embedding_options["query_prompt_template"]- elif "prompt_template" in self.embedding_options:- query_template = self.embedding_options["prompt_template"]-- query_embedding = self.backend_impl.compute_query_embedding(- query,- use_server_if_available=effective_recompute,- zmq_port=zmq_port,- query_template=query_template,- )- logger.info(f" Generated embedding shape: {query_embedding.shape}")- embedding_time = time.time() - start_time- logger.info(f" Embedding time: {embedding_time} seconds")-- start_time = time.time()- backend_search_kwargs: dict[str, Any] = {- "complexity": complexity,- "beam_width": beam_width,- "prune_ratio": prune_ratio,- "recompute_embeddings": effective_recompute,- "pruning_strategy": pruning_strategy,- "zmq_port": zmq_port,- }- # Only HNSW supports batching; forward conditionally- if self.backend_name == "hnsw":- backend_search_kwargs["batch_size"] = batch_size-- # Merge any extra kwargs last- backend_search_kwargs.update(kwargs)-- results = self.backend_impl.search(- query_embedding,- top_k,- **backend_search_kwargs,- )-- # Handle hybrid search- if 0.0 < gemma < 1.0:- logger.info(f" 🌟 Hybrid search enabled with gemma={gemma}")- BM25_WEIGHT = 1.0 - gemma- bm25_results = self._bm25_search(query, top_k)- hybrid_scores: dict[str, float] = {}- # Add vector search scores (weighted by gemma)- if "labels" in results and "distances" in results:- for doc_id, score in zip(results["labels"][0], results["distances"][0]):- hybrid_scores[doc_id] = gemma * score- # Add BM25 scores (weighted by BM25_WEIGHT)- for bm25_result in bm25_results:- doc_id = bm25_result.id- if doc_id in hybrid_scores:- hybrid_scores[doc_id] += BM25_WEIGHT * bm25_result.score- else:- hybrid_scores[doc_id] = BM25_WEIGHT * bm25_result.score-- sorted_hybrid = sorted(hybrid_scores.items(), key=lambda x: x[1], reverse=True)[:top_k]- results["labels"] = [[doc_id for doc_id, _ in sorted_hybrid]]- results["distances"] = [[score for _, score in sorted_hybrid]]-- logger.info(- f" Combined {len(hybrid_scores)} unique documents from vector and BM25 search"- )-- search_time = time.time() - start_time- logger.info(f" Search time in search() LEANN searcher: {search_time} seconds")- logger.info(f" Backend returned: labels={len(results.get('labels', [[]])[0])} results")-- enriched_results = []- if "labels" in results and "distances" in results:- logger.info(f" Processing {len(results['labels'][0])} passage IDs:")- # Python 3.9 does not support zip(strict=...); lengths are expected to match- for i, (string_id, dist) in enumerate(- zip(results["labels"][0], results["distances"][0])- ):- try:- passage_data = self.passage_manager.get_passage(string_id)- enriched_results.append(- SearchResult(- id=string_id,- score=dist,- text=passage_data["text"],- metadata=passage_data.get("metadata", {}),- )- )-- # Color codes for better logging- GREEN = "\033[92m"- BLUE = "\033[94m"- YELLOW = "\033[93m"- RESET = "\033[0m"-- # Truncate text for display (first 100 chars)- display_text = passage_data["text"]- logger.info(- f" {GREEN}✓{RESET} {BLUE}[{i + 1:2d}]{RESET} {YELLOW}ID:{RESET} '{string_id}' {YELLOW}Score:{RESET} {dist:.4f} {YELLOW}Text:{RESET} {display_text}"- )- except KeyError:- RED = "\033[91m"- RESET = "\033[0m"- logger.error(- f" {RED}✗{RESET} [{i + 1:2d}] ID: '{string_id}' -> {RED}ERROR: Passage not found!{RESET}"- )-- # Apply metadata filters if specified- if metadata_filters:- logger.info(f" 🔍 Applying metadata filters: {metadata_filters}")- enriched_results = self.passage_manager.filter_search_results(- enriched_results, metadata_filters- )-- # Define color codes outside the loop for final message- GREEN = "\033[92m"- RESET = "\033[0m"- logger.info(f" {GREEN}✓ Final enriched results: {len(enriched_results)} passages{RESET}")- return enriched_results-- def _init_bm25(self) -> None:- """Initialize BM25 scorer"""- self.bm25_scorer = BM25Scorer()- # Load all the files directly- passages = []- for passage_file in self.passage_manager.passage_files.values():- with open(passage_file, encoding="utf-8") as f:- for line in f:- if line.strip():- data = json.loads(line)- passages.append(data)- self.bm25_scorer.fit(passages)-- def _bm25_search(self, query: str, top_k: int = 5) -> list[SearchResult]:- """Perform BM25 search on raw passages"""- if self.bm25_scorer is None:- self._init_bm25()- logger.info(" BM25 scorer initialized")- scorer = self.bm25_scorer- if scorer is None:- raise RuntimeError("BM25 scorer failed to initialize")- return scorer.search(query, top_k)-- def _find_jsonl_file(self) -> Optional[str]:- """Find the .jsonl file containing raw passages for grep search"""- index_path = Path(self.meta_path_str).parent- potential_files = [- index_path / "documents.leann.passages.jsonl",- index_path.parent / "documents.leann.passages.jsonl",- ]-- for file_path in potential_files:- if file_path.exists():- return str(file_path)- return None-- def _grep_search(self, query: str, top_k: int = 5) -> list[SearchResult]:- """Perform grep-based search on raw passages"""- jsonl_file = self._find_jsonl_file()- if not jsonl_file:- raise FileNotFoundError("No .jsonl passages file found for grep search")-- try:- cmd = ["grep", "-i", "-n", query, jsonl_file]- result = subprocess.run(cmd, capture_output=True, text=True, check=False)-- if result.returncode == 1:- return []- elif result.returncode != 0:- raise RuntimeError(f"Grep failed: {result.stderr}")-- matches = []- for line in result.stdout.strip().split("\n"):- if not line:- continue- parts = line.split(":", 1)- if len(parts) != 2:- continue-- try:- data = json.loads(parts[1])- text = data.get("text", "")- score = text.lower().count(query.lower())-- matches.append(- SearchResult(- id=data.get("id", parts[0]),- text=text,- metadata=data.get("metadata", {}),- score=float(score),- )- )- except json.JSONDecodeError:- continue-- matches.sort(key=lambda x: x.score, reverse=True)- return matches[:top_k]-- except FileNotFoundError:- raise RuntimeError(- "grep command not found. Please install grep or use semantic search."- )-- def _python_regex_search(self, query: str, top_k: int = 5) -> list[SearchResult]:- """Fallback regex search"""- jsonl_file = self._find_jsonl_file()- if not jsonl_file:- raise FileNotFoundError("No .jsonl file found")-- pattern = re.compile(re.escape(query), re.IGNORECASE)- matches = []-- with open(jsonl_file, encoding="utf-8") as f:- for line_num, line in enumerate(f, 1):- if pattern.search(line):- try:- data = json.loads(line.strip())- matches.append(- SearchResult(- id=data.get("id", str(line_num)),- text=data.get("text", ""),- metadata=data.get("metadata", {}),- score=float(len(pattern.findall(data.get("text", "")))),- )- )- except json.JSONDecodeError:- continue-- matches.sort(key=lambda x: x.score, reverse=True)- return matches[:top_k]-- def cleanup(self):- """Explicitly cleanup embedding server resources.- This method should be called after you're done using the searcher,- especially in test environments or batch processing scenarios.- """- backend = getattr(self.backend_impl, "embedding_server_manager", None)- if backend is not None:- backend.stop_server()-- # Enable automatic cleanup patterns- def __enter__(self):- return self-- def __exit__(self, exc_type, exc, tb):- try:- self.cleanup()- except Exception:- pass-- def __del__(self):- try:- self.cleanup()- except Exception:- # Avoid noisy errors during interpreter shutdown- pass---class LeannChat:- def __init__(- self,- index_path: str,- llm_config: Optional[dict[str, Any]] = None,- enable_warmup: bool = False,- searcher: Optional[LeannSearcher] = None,- **kwargs,- ):- if searcher is None:- self.searcher = LeannSearcher(index_path, enable_warmup=enable_warmup, **kwargs)- self._owns_searcher = True- else:- self.searcher = searcher- self._owns_searcher = False- self.llm = get_llm(llm_config)-- def ask(- self,- question: str,- top_k: int = 5,- complexity: int = 64,- beam_width: int = 1,- prune_ratio: float = 0.0,- recompute_embeddings: bool = True,- pruning_strategy: Literal["global", "local", "proportional"] = "global",- llm_kwargs: Optional[dict[str, Any]] = None,- expected_zmq_port: int = 5557,- metadata_filters: Optional[dict[str, dict[str, Union[str, int, float, bool, list]]]] = None,- batch_size: int = 0,- use_grep: bool = False,- gemma: float = 1.0,- **search_kwargs,- ):- if llm_kwargs is None:- llm_kwargs = {}- search_time = time.time()- results = self.searcher.search(- question,- top_k=top_k,- complexity=complexity,- beam_width=beam_width,- prune_ratio=prune_ratio,- recompute_embeddings=recompute_embeddings,- pruning_strategy=pruning_strategy,- expected_zmq_port=expected_zmq_port,- metadata_filters=metadata_filters,- use_grep=use_grep,- gemma=gemma,- batch_size=batch_size,- **search_kwargs,- )- search_time = time.time() - search_time- logger.info(f" Search time: {search_time} seconds")- context = "\n\n".join([r.text for r in results])- prompt = (- "Here is some retrieved context that might help answer your question:\n\n"- f"{context}\n\n"- f"Question: {question}\n\n"- "Please provide the best answer you can based on this context and your knowledge."- )-- logger.info("The context provided to the LLM is:")- logger.info(f"{'Relevance':<10} | {'Chunk id':<10} | {'Content':<60} | {'Source':<80}")- logger.info("-" * 150)- for r in results:- chunk_relevance = f"{r.score:.3f}"- chunk_id = r.id- chunk_content = r.text[:60]- chunk_source = r.metadata.get("source", "")[:80]- logger.info(- f"{chunk_relevance:<10} | {chunk_id:<10} | {chunk_content:<60} | {chunk_source:<80}"- )- ask_time = time.time()- ans = self.llm.ask(prompt, **llm_kwargs)- ask_time = time.time() - ask_time- logger.info(f" Ask time: {ask_time} seconds")- return ans-- def start_interactive(self):- """Start interactive chat session."""- session = create_api_session()-- def handle_query(user_input: str):- response = self.ask(user_input)- print(f"Leann: {response}")-- session.run_interactive_loop(handle_query)-- def cleanup(self):- """Explicitly cleanup embedding server resources.-- This method should be called after you're done using the chat interface,- especially in test environments or batch processing scenarios.- """- # Only stop the embedding server if this LeannChat instance created the searcher.- # When a shared searcher is passed in, avoid shutting down the server to enable reuse.- if getattr(self, "_owns_searcher", False) and hasattr(self.searcher, "cleanup"):- self.searcher.cleanup()-- # Enable automatic cleanup patterns- def __enter__(self):- return self-- def __exit__(self, exc_type, exc, tb):- try:- self.cleanup()- except Exception:- pass-- def __del__(self):- try:- self.cleanup()- except Exception:- pass+"""+This file contains the core API for the LEANN project, now definitively updated+with the correct, original embedding logic from the user's reference code.+"""++import json+import logging+import os+import pickle+import re+import subprocess+import time+import warnings+from collections import Counter, defaultdict+from dataclasses import dataclass, field+from pathlib import Path+from typing import Any, Literal, Optional, Union++import numpy as np+from leann_backend_hnsw.convert_to_csr import prune_hnsw_embeddings_inplace++from leann.interactive_utils import create_api_session+from leann.interface import LeannBackendSearcherInterface++from .chat import get_llm+from .embedding_server_manager import EmbeddingServerManager+from .interface import LeannBackendFactoryInterface+from .metadata_filter import MetadataFilterEngine+from .registry import BACKEND_REGISTRY++logger = logging.getLogger(__name__)+++def get_registered_backends() -> list[str]:+ """Get list of registered backend names."""+ return list(BACKEND_REGISTRY.keys())+++def compute_embeddings(+ chunks: list[str],+ model_name: str,+ mode: str = "sentence-transformers",+ use_server: bool = True,+ port: Optional[int] = None,+ is_build=False,+ provider_options: Optional[dict[str, Any]] = None,+) -> np.ndarray:+ """+ Computes embeddings using different backends.++ Args:+ chunks: List of text chunks to embed+ model_name: Name of the embedding model+ mode: Embedding backend mode. Options:+ - "sentence-transformers": Use sentence-transformers library (default)+ - "mlx": Use MLX backend for Apple Silicon+ - "openai": Use OpenAI embedding API+ - "gemini": Use Google Gemini embedding API+ use_server: Whether to use embedding server (True for search, False for build)++ Returns:+ numpy array of embeddings+ """+ if use_server:+ # Use embedding server (for search/query)+ if port is None:+ raise ValueError("port is required when use_server is True")+ return compute_embeddings_via_server(chunks, model_name, port=port)+ else:+ # Use direct computation (for build_index)+ from .embedding_compute import (+ compute_embeddings as compute_embeddings_direct,+ )++ return compute_embeddings_direct(+ chunks,+ model_name,+ mode=mode,+ is_build=is_build,+ provider_options=provider_options,+ )+++def compute_embeddings_via_server(chunks: list[str], model_name: str, port: int) -> np.ndarray:+ """Computes embeddings using sentence-transformers.++ Args:+ chunks: List of text chunks to embed+ model_name: Name of the sentence transformer model+ """+ logger.info(+ f"Computing embeddings for {len(chunks)} chunks using SentenceTransformer model '{model_name}' (via embedding server)..."+ )+ import msgpack+ import numpy as np+ import zmq++ # Connect to embedding server+ context = zmq.Context()+ socket = context.socket(zmq.REQ)+ socket.connect(f"tcp://localhost:{port}")++ # Send chunks to server for embedding computation+ request = chunks+ socket.send(msgpack.packb(request))++ # Receive embeddings from server+ response = socket.recv()+ embeddings_list = msgpack.unpackb(response)++ # Convert back to numpy array+ embeddings = np.array(embeddings_list, dtype=np.float32)++ socket.close()+ context.term()++ return embeddings+++@dataclass+class SearchResult:+ id: str+ score: float+ text: str+ metadata: dict[str, Any] = field(default_factory=dict)+++class PassageManager:+ def __init__(+ self, passage_sources: list[dict[str, Any]], metadata_file_path: Optional[str] = None+ ):+ self.offset_maps: dict[str, dict[str, int]] = {}+ self.passage_files: dict[str, str] = {}+ # Avoid materializing a single gigantic global map to reduce memory+ # footprint on very large corpora (e.g., 60M+ passages). Instead, keep+ # per-shard maps and do a lightweight per-shard lookup on demand.+ self._total_count: int = 0+ self.filter_engine = MetadataFilterEngine() # Initialize filter engine++ # Derive index base name for standard sibling fallbacks, e.g., <index_name>.passages.*+ index_name_base = None+ if metadata_file_path:+ meta_name = Path(metadata_file_path).name+ if meta_name.endswith(".meta.json"):+ index_name_base = meta_name[: -len(".meta.json")]++ for source in passage_sources:+ assert source["type"] == "jsonl", "only jsonl is supported"+ passage_file = source.get("path", "")+ index_file = source.get("index_path", "") # .idx file++ # Fix path resolution - relative paths should be relative to metadata file directory+ def _resolve_candidates(+ primary: str,+ relative_key: str,+ default_name: Optional[str],+ source_dict: dict[str, Any],+ ) -> list[Path]:+ """+ Build an ordered list of candidate paths. For relative paths specified in+ metadata, prefer resolution relative to the metadata file directory first,+ then fall back to CWD-based resolution, and finally to conventional+ sibling defaults (e.g., <index_base>.passages.idx / .jsonl).+ """+ candidates: list[Path] = []+ # 1) Primary path+ if primary:+ p = Path(primary)+ if p.is_absolute():+ candidates.append(p)+ else:+ # Prefer metadata-relative resolution for relative paths+ if metadata_file_path:+ candidates.append(Path(metadata_file_path).parent / p)+ # Also consider CWD-relative as a fallback for legacy layouts+ candidates.append(Path.cwd() / p)+ # 2) metadata-relative explicit relative key (if present)+ if metadata_file_path and source_dict.get(relative_key):+ candidates.append(Path(metadata_file_path).parent / source_dict[relative_key])+ # 3) metadata-relative standard sibling filename+ if metadata_file_path and default_name:+ candidates.append(Path(metadata_file_path).parent / default_name)+ return candidates++ # Build candidate lists and pick first existing; otherwise keep last candidate for error message+ idx_default = f"{index_name_base}.passages.idx" if index_name_base else None+ idx_candidates = _resolve_candidates(+ index_file, "index_path_relative", idx_default, source+ )+ pas_default = f"{index_name_base}.passages.jsonl" if index_name_base else None+ pas_candidates = _resolve_candidates(passage_file, "path_relative", pas_default, source)++ def _pick_existing(cands: list[Path]) -> str:+ for c in cands:+ if c.exists():+ return str(c.resolve())+ # Fallback to last candidate (best guess) even if not exists; will error below+ return str(cands[-1].resolve()) if cands else ""++ index_file = _pick_existing(idx_candidates)+ passage_file = _pick_existing(pas_candidates)++ if not Path(index_file).exists():+ raise FileNotFoundError(f"Passage index file not found: {index_file}")++ with open(index_file, "rb") as f:+ offset_map: dict[str, int] = pickle.load(f)+ self.offset_maps[passage_file] = offset_map+ self.passage_files[passage_file] = passage_file+ self._total_count += len(offset_map)++ def get_passage(self, passage_id: str) -> dict[str, Any]:+ # Fast path: check each shard map (there are typically few shards).+ # This avoids building a massive combined dict while keeping lookups+ # bounded by the number of shards.+ for passage_file, offset_map in self.offset_maps.items():+ try:+ offset = offset_map[passage_id]+ with open(passage_file, encoding="utf-8") as f:+ f.seek(offset)+ return json.loads(f.readline())+ except KeyError:+ continue+ raise KeyError(f"Passage ID not found: {passage_id}")++ def filter_search_results(+ self,+ search_results: list[SearchResult],+ metadata_filters: Optional[dict[str, dict[str, Union[str, int, float, bool, list]]]],+ ) -> list[SearchResult]:+ """+ Apply metadata filters to search results.++ Args:+ search_results: List of SearchResult objects+ metadata_filters: Filter specifications to apply++ Returns:+ Filtered list of SearchResult objects+ """+ if not metadata_filters:+ return search_results++ logger.debug(f"Applying metadata filters to {len(search_results)} results")++ # Convert SearchResult objects to dictionaries for the filter engine+ result_dicts = []+ for result in search_results:+ result_dicts.append(+ {+ "id": result.id,+ "score": result.score,+ "text": result.text,+ "metadata": result.metadata,+ }+ )++ # Apply filters using the filter engine+ filtered_dicts = self.filter_engine.apply_filters(result_dicts, metadata_filters)++ # Convert back to SearchResult objects+ filtered_results = []+ for result_dict in filtered_dicts:+ filtered_results.append(+ SearchResult(+ id=result_dict["id"],+ score=result_dict["score"],+ text=result_dict["text"],+ metadata=result_dict["metadata"],+ )+ )++ logger.debug(f"Filtered results: {len(filtered_results)} remaining")+ return filtered_results++ def __len__(self) -> int:+ return self._total_count+++class BM25Scorer:+ def __init__(self, k1: float = 1.2, b: float = 0.75):+ self.k1 = k1+ self.b = b+ self.doc_freqs = None # How many docs contain each term (DF)+ self.doc_lengths = {} # How long each doc is (in words)+ self.word_counts = {} # How many times each word appears in each doc (TF)+ self.avg_doc_length = None+ self.corpus_size = None+ self.idlist = set() # List of all document IDs for easier searching++ def _tokenize(self, text: str) -> list[str]:+ return re.sub(r"[^\w\s]", "", text).lower().split()++ def fit(self, documents: list[dict[str, Any]]):+ """+ Build BM25 statistics from a document corpus.+ Must be called before scoring.+ """+ self.corpus_size = len(documents)+ self.doc_lengths = {}+ self.word_counts = {}+ self.idlist = set()+ doc_freqs = defaultdict(int)++ for doc_data in documents:+ doc_id = doc_data["id"]+ words = self._tokenize(doc_data["text"])+ doc_length = len(words)+ self.doc_lengths[doc_id] = doc_length++ unique_words = set(words)+ for word in unique_words:+ doc_freqs[word] += 1+ self.word_counts[doc_id] = dict(Counter(words))+ self.idlist.add(doc_id)++ self.doc_freqs = dict(doc_freqs)+ self.avg_doc_length = sum(self.doc_lengths.values()) / len(self.doc_lengths)++ def score(self, query_words: list[str], document_id: str) -> float:+ if (+ self.doc_freqs is None+ or self.doc_lengths == {}+ or self.word_counts == {}+ or self.avg_doc_length is None+ or self.corpus_size is None+ ):+ raise ValueError("BM25 model not fitted. Call fit() before scoring.")++ passage_words = self.word_counts[document_id]+ passage_length = sum(passage_words.values())+ score = 0.0+ for word in query_words:+ if word not in self.doc_freqs:+ continue+ word_freq = passage_words[word] if word in passage_words else 0+ idf = np.log(+ (self.corpus_size - self.doc_freqs[word] + 0.5) / (self.doc_freqs[word] + 0.5) + 1+ )+ tf = (word_freq * (self.k1 + 1)) / (+ word_freq + self.k1 * (1 - self.b + self.b * (passage_length / self.avg_doc_length))+ )+ score += idf * tf+ return score++ def search(self, query: str, top_k: int = 5) -> list[SearchResult]:+ query_words = self._tokenize(query)+ scores = {doc_id: self.score(query_words, doc_id) for doc_id in self.idlist}+ sorted_scores = sorted(scores.items(), key=lambda x: x[1], reverse=True)+ return [+ SearchResult(id=doc_id, score=score, text="", metadata={})+ for doc_id, score in sorted_scores[:top_k]+ ]+++class LeannBuilder:+ def __init__(+ self,+ backend_name: str,+ embedding_model: str = "facebook/contriever",+ dimensions: Optional[int] = None,+ embedding_mode: str = "sentence-transformers",+ embedding_options: Optional[dict[str, Any]] = None,+ **backend_kwargs,+ ):+ self.backend_name = backend_name+ # Normalize incompatible combinations early (for consistent metadata)+ if backend_name == "hnsw":+ is_recompute = backend_kwargs.get("is_recompute", True)+ is_compact = backend_kwargs.get("is_compact", True)+ if is_recompute is False and is_compact is True:+ warnings.warn(+ "HNSW with is_recompute=False requires non-compact storage. Forcing is_compact=False.",+ UserWarning,+ stacklevel=2,+ )+ backend_kwargs["is_compact"] = False++ backend_factory: Optional[LeannBackendFactoryInterface] = BACKEND_REGISTRY.get(backend_name)+ if backend_factory is None:+ raise ValueError(f"Backend '{backend_name}' not found or not registered.")+ self.backend_factory = backend_factory+ self.embedding_model = embedding_model+ self.dimensions = dimensions+ self.embedding_mode = embedding_mode+ self.embedding_options = embedding_options or {}++ # Check if we need to use cosine distance for normalized embeddings+ normalized_embeddings_models = {+ # OpenAI models+ ("openai", "text-embedding-ada-002"),+ ("openai", "text-embedding-3-small"),+ ("openai", "text-embedding-3-large"),+ # Voyage AI models+ ("voyage", "voyage-2"),+ ("voyage", "voyage-3"),+ ("voyage", "voyage-large-2"),+ ("voyage", "voyage-multilingual-2"),+ ("voyage", "voyage-code-2"),+ # Cohere models+ ("cohere", "embed-english-v3.0"),+ ("cohere", "embed-multilingual-v3.0"),+ ("cohere", "embed-english-light-v3.0"),+ ("cohere", "embed-multilingual-light-v3.0"),+ }++ # Also check for patterns in model names+ is_normalized = False+ current_model_lower = embedding_model.lower()+ current_mode_lower = embedding_mode.lower()++ # Check exact matches+ for mode, model in normalized_embeddings_models:+ if (current_mode_lower == mode and current_model_lower == model) or (+ mode in current_mode_lower and model in current_model_lower+ ):+ is_normalized = True+ break++ # Check patterns+ if not is_normalized:+ # OpenAI patterns+ if "openai" in current_mode_lower or "openai" in current_model_lower:+ if any(+ pattern in current_model_lower+ for pattern in ["text-embedding", "ada", "3-small", "3-large"]+ ):+ is_normalized = True+ # Voyage patterns+ elif "voyage" in current_mode_lower or "voyage" in current_model_lower:+ is_normalized = True+ # Cohere patterns+ elif "cohere" in current_mode_lower or "cohere" in current_model_lower:+ if "embed" in current_model_lower:+ is_normalized = True++ # Handle distance metric+ if is_normalized and "distance_metric" not in backend_kwargs:+ backend_kwargs["distance_metric"] = "cosine"+ warnings.warn(+ f"Detected normalized embeddings model '{embedding_model}' with mode '{embedding_mode}'. "+ f"Automatically setting distance_metric='cosine' for optimal performance. "+ f"Normalized embeddings (L2 norm = 1) should use cosine similarity instead of MIPS.",+ UserWarning,+ stacklevel=2,+ )+ elif is_normalized and backend_kwargs.get("distance_metric", "").lower() != "cosine":+ current_metric = backend_kwargs.get("distance_metric", "mips")+ warnings.warn(+ f"Warning: Using '{current_metric}' distance metric with normalized embeddings model "+ f"'{embedding_model}' may lead to suboptimal search results. "+ f"Consider using 'cosine' distance metric for better performance.",+ UserWarning,+ stacklevel=2,+ )++ self.backend_kwargs = backend_kwargs+ self.chunks: list[dict[str, Any]] = []++ def add_text(self, text: str, metadata: Optional[dict[str, Any]] = None):+ if metadata is None:+ metadata = {}+ passage_id = metadata.get("id", str(len(self.chunks)))+ chunk_data = {"id": passage_id, "text": text, "metadata": metadata}+ self.chunks.append(chunk_data)++ def build_index(self, index_path: str):+ if not self.chunks:+ raise ValueError("No chunks added.")++ # Filter out invalid/empty text chunks early to keep passage and embedding counts aligned+ valid_chunks: list[dict[str, Any]] = []+ skipped = 0+ for chunk in self.chunks:+ text = chunk.get("text", "")+ if isinstance(text, str) and text.strip():+ valid_chunks.append(chunk)+ else:+ skipped += 1+ if skipped > 0:+ print(+ f"Warning: Skipping {skipped} empty/invalid text chunk(s). Processing {len(valid_chunks)} valid chunks"+ )+ self.chunks = valid_chunks+ if not self.chunks:+ raise ValueError("All provided chunks are empty or invalid. Nothing to index.")+ if self.dimensions is None:+ self.dimensions = len(+ compute_embeddings(+ ["dummy"],+ self.embedding_model,+ self.embedding_mode,+ use_server=False,+ provider_options=self.embedding_options,+ )[0]+ )+ path = Path(index_path)+ index_dir = path.parent+ index_name = path.name+ index_dir.mkdir(parents=True, exist_ok=True)+ passages_file = index_dir / f"{index_name}.passages.jsonl"+ offset_file = index_dir / f"{index_name}.passages.idx"+ offset_map = {}+ with open(passages_file, "w", encoding="utf-8") as f:+ try:+ from tqdm import tqdm++ chunk_iterator = tqdm(self.chunks, desc="Writing passages", unit="chunk")+ except ImportError:+ chunk_iterator = self.chunks++ for chunk in chunk_iterator:+ offset = f.tell()+ json.dump(+ {+ "id": chunk["id"],+ "text": chunk["text"],+ "metadata": chunk["metadata"],+ },+ f,+ ensure_ascii=False,+ )+ f.write("\n")+ offset_map[chunk["id"]] = offset+ with open(offset_file, "wb") as f:+ pickle.dump(offset_map, f)+ texts_to_embed = [c["text"] for c in self.chunks]+ embeddings = compute_embeddings(+ texts_to_embed,+ self.embedding_model,+ self.embedding_mode,+ use_server=False,+ is_build=True,+ provider_options=self.embedding_options,+ )+ string_ids = [chunk["id"] for chunk in self.chunks]+ # Persist ID map alongside index so backends that return integer labels can remap to passage IDs+ try:+ idmap_file = (+ index_dir+ / f"{index_name[: -len('.leann')] if index_name.endswith('.leann') else index_name}.ids.txt"+ )+ with open(idmap_file, "w", encoding="utf-8") as f:+ for sid in string_ids:+ f.write(str(sid) + "\n")+ except Exception:+ pass+ current_backend_kwargs = {**self.backend_kwargs, "dimensions": self.dimensions}+ builder_instance = self.backend_factory.builder(**current_backend_kwargs)+ builder_instance.build(embeddings, string_ids, index_path, **current_backend_kwargs)+ leann_meta_path = index_dir / f"{index_name}.meta.json"+ meta_data = {+ "version": "1.0",+ "backend_name": self.backend_name,+ "embedding_model": self.embedding_model,+ "dimensions": self.dimensions,+ "backend_kwargs": self.backend_kwargs,+ "embedding_mode": self.embedding_mode,+ "passage_sources": [+ {+ "type": "jsonl",+ # Preserve existing relative file names (backward-compatible)+ "path": passages_file.name,+ "index_path": offset_file.name,+ # Add optional redundant relative keys for remote build portability (non-breaking)+ "path_relative": passages_file.name,+ "index_path_relative": offset_file.name,+ }+ ],+ }++ if self.embedding_options:+ meta_data["embedding_options"] = self.embedding_options++ # Add storage status flags for HNSW backend+ if self.backend_name == "hnsw":+ is_compact = self.backend_kwargs.get("is_compact", True)+ is_recompute = self.backend_kwargs.get("is_recompute", True)+ meta_data["is_compact"] = is_compact+ meta_data["is_pruned"] = bool(is_recompute)+ with open(leann_meta_path, "w", encoding="utf-8") as f:+ json.dump(meta_data, f, indent=2)++ def build_index_from_embeddings(self, index_path: str, embeddings_file: str):+ """+ Build an index from pre-computed embeddings stored in a pickle file.++ Args:+ index_path: Path where the index will be saved+ embeddings_file: Path to pickle file containing (ids, embeddings) tuple+ """+ # Load pre-computed embeddings+ with open(embeddings_file, "rb") as f:+ data = pickle.load(f)++ if not isinstance(data, tuple) or len(data) != 2:+ raise ValueError(+ f"Invalid embeddings file format. Expected tuple with 2 elements, got {type(data)}"+ )++ ids, embeddings = data++ if not isinstance(embeddings, np.ndarray):+ raise ValueError(f"Expected embeddings to be numpy array, got {type(embeddings)}")++ if len(ids) != embeddings.shape[0]:+ raise ValueError(+ f"Mismatch between number of IDs ({len(ids)}) and embeddings ({embeddings.shape[0]})"+ )++ # Validate/set dimensions+ embedding_dim = embeddings.shape[1]+ if self.dimensions is None:+ self.dimensions = embedding_dim+ elif self.dimensions != embedding_dim:+ raise ValueError(f"Dimension mismatch: expected {self.dimensions}, got {embedding_dim}")++ logger.info(+ f"Building index from precomputed embeddings: {len(ids)} items, {embedding_dim} dimensions"+ )++ # Ensure we have text data for each embedding+ if len(self.chunks) != len(ids):+ # If no text chunks provided, create placeholder text entries+ if not self.chunks:+ logger.info("No text chunks provided, creating placeholder entries...")+ for id_val in ids:+ self.add_text(+ f"Document {id_val}",+ metadata={"id": str(id_val), "from_embeddings": True},+ )+ else:+ raise ValueError(+ f"Number of text chunks ({len(self.chunks)}) doesn't match number of embeddings ({len(ids)})"+ )++ # Build file structure+ path = Path(index_path)+ index_dir = path.parent+ index_name = path.name+ index_dir.mkdir(parents=True, exist_ok=True)+ passages_file = index_dir / f"{index_name}.passages.jsonl"+ offset_file = index_dir / f"{index_name}.passages.idx"++ # Write passages and create offset map+ offset_map = {}+ with open(passages_file, "w", encoding="utf-8") as f:+ for chunk in self.chunks:+ offset = f.tell()+ json.dump(+ {+ "id": chunk["id"],+ "text": chunk["text"],+ "metadata": chunk["metadata"],+ },+ f,+ ensure_ascii=False,+ )+ f.write("\n")+ offset_map[chunk["id"]] = offset++ with open(offset_file, "wb") as f:+ pickle.dump(offset_map, f)++ # Build the vector index using precomputed embeddings+ string_ids = [str(id_val) for id_val in ids]+ # Persist ID map (order == embeddings order)+ try:+ idmap_file = (+ index_dir+ / f"{index_name[: -len('.leann')] if index_name.endswith('.leann') else index_name}.ids.txt"+ )+ with open(idmap_file, "w", encoding="utf-8") as f:+ for sid in string_ids:+ f.write(str(sid) + "\n")+ except Exception:+ pass+ current_backend_kwargs = {**self.backend_kwargs, "dimensions": self.dimensions}+ builder_instance = self.backend_factory.builder(**current_backend_kwargs)+ builder_instance.build(embeddings, string_ids, index_path)++ # Create metadata file+ leann_meta_path = index_dir / f"{index_name}.meta.json"+ meta_data = {+ "version": "1.0",+ "backend_name": self.backend_name,+ "embedding_model": self.embedding_model,+ "dimensions": self.dimensions,+ "backend_kwargs": self.backend_kwargs,+ "embedding_mode": self.embedding_mode,+ "passage_sources": [+ {+ "type": "jsonl",+ # Preserve existing relative file names (backward-compatible)+ "path": passages_file.name,+ "index_path": offset_file.name,+ # Add optional redundant relative keys for remote build portability (non-breaking)+ "path_relative": passages_file.name,+ "index_path_relative": offset_file.name,+ }+ ],+ "built_from_precomputed_embeddings": True,+ "embeddings_source": str(embeddings_file),+ }++ if self.embedding_options:+ meta_data["embedding_options"] = self.embedding_options++ # Add storage status flags for HNSW backend+ if self.backend_name == "hnsw":+ is_compact = self.backend_kwargs.get("is_compact", True)+ is_recompute = self.backend_kwargs.get("is_recompute", True)+ meta_data["is_compact"] = is_compact+ meta_data["is_pruned"] = bool(is_recompute)++ with open(leann_meta_path, "w", encoding="utf-8") as f:+ json.dump(meta_data, f, indent=2)++ logger.info(f"Index built successfully from precomputed embeddings: {index_path}")++ def update_index(self, index_path: str, remove_passage_ids: Optional[list[str]] = None) -> None:+ """Append new passages and vectors to an existing index (HNSW or IVF).+ For IVF, optional remove_passage_ids removes those ids first (e.g. from file-change API).+ """+ if not self.chunks and not remove_passage_ids:+ raise ValueError("No new chunks or passage ids to remove provided for update.")++ path = Path(index_path)+ index_dir = path.parent+ index_name = path.name+ index_prefix = path.stem++ meta_path = index_dir / f"{index_name}.meta.json"+ passages_file = index_dir / f"{index_name}.passages.jsonl"+ offset_file = index_dir / f"{index_name}.passages.idx"+ index_file = index_dir / f"{index_prefix}.index"++ if not meta_path.exists() or not passages_file.exists() or not offset_file.exists():+ raise FileNotFoundError("Index metadata or passage files are missing; cannot update.")+ if not index_file.exists():+ raise FileNotFoundError(f"Index file not found: {index_file}")++ with open(meta_path, encoding="utf-8") as f:+ meta = json.load(f)+ backend_name = meta.get("backend_name")+ if backend_name != self.backend_name:+ raise ValueError(+ f"Index was built with backend '{backend_name}', cannot update with '{self.backend_name}'."+ )++ with open(offset_file, "rb") as f:+ offset_map: dict[str, int] = pickle.load(f)+ existing_ids = set(offset_map.keys())++ # IVF: optional delete (for reindex / file-change: remove then re-insert)+ if remove_passage_ids and backend_name == "ivf":+ try:+ from leann_backend_ivf import remove_ids as ivf_remove_ids++ ivf_remove_ids(str(path), remove_passage_ids)+ except ImportError:+ raise RuntimeError(+ "IVF backend required for remove_ids. Install leann-backend-ivf."+ )+ for pid in remove_passage_ids:+ offset_map.pop(pid, None)+ existing_ids -= set(remove_passage_ids)+ with open(offset_file, "wb") as f:+ pickle.dump(offset_map, f)++ if not self.chunks:+ meta["total_passages"] = len(offset_map)+ with open(meta_path, "w", encoding="utf-8") as f:+ json.dump(meta, f, indent=2)+ self.chunks.clear()+ return++ meta_backend_kwargs = meta.get("backend_kwargs", {})+ if backend_name == "hnsw":+ index_is_compact = meta.get("is_compact", meta_backend_kwargs.get("is_compact", True))+ if index_is_compact:+ raise ValueError(+ "Compact HNSW indices do not support in-place updates. Rebuild required."+ )++ distance_metric = meta_backend_kwargs.get(+ "distance_metric", self.backend_kwargs.get("distance_metric", "mips")+ ).lower()+ needs_recompute = bool(+ meta.get("is_pruned")+ or meta_backend_kwargs.get("is_recompute")+ or self.backend_kwargs.get("is_recompute")+ )++ valid_chunks: list[dict[str, Any]] = []+ for chunk in self.chunks:+ text = chunk.get("text", "")+ if not isinstance(text, str) or not text.strip():+ continue+ metadata = chunk.setdefault("metadata", {})+ passage_id = chunk.get("id") or metadata.get("id")+ if passage_id and passage_id in existing_ids:+ raise ValueError(f"Passage ID '{passage_id}' already exists in the index.")+ valid_chunks.append(chunk)++ if not valid_chunks:+ # Remove-only or file emptied: we may have already removed ids, just update meta+ meta["total_passages"] = len(offset_map)+ with open(meta_path, "w", encoding="utf-8") as f:+ json.dump(meta, f, indent=2)+ self.chunks.clear()+ return++ texts_to_embed = [chunk["text"] for chunk in valid_chunks]+ embeddings = compute_embeddings(+ texts_to_embed,+ self.embedding_model,+ self.embedding_mode,+ use_server=False,+ is_build=True,+ provider_options=self.embedding_options,+ )++ embedding_dim = embeddings.shape[1]+ expected_dim = meta.get("dimensions")+ if expected_dim is not None and expected_dim != embedding_dim:+ raise ValueError(+ f"Dimension mismatch during update: existing index uses {expected_dim}, got {embedding_dim}."+ )++ embeddings = np.ascontiguousarray(embeddings, dtype=np.float32)+ if distance_metric == "cosine":+ norms = np.linalg.norm(embeddings, axis=1, keepdims=True)+ norms[norms == 0] = 1+ embeddings = embeddings / norms++ # IVF: add_vectors then append passages/offset (no ZMQ/server)+ if backend_name == "ivf":+ for i, chunk in enumerate(valid_chunks):+ pid = chunk.get("id") or chunk.get("metadata", {}).get("id")+ if not pid:+ pid = str(len(offset_map) + i)+ chunk.setdefault("metadata", {})["id"] = pid+ chunk["id"] = pid+ passage_ids = [c["id"] for c in valid_chunks]+ try:+ from leann_backend_ivf import add_vectors as ivf_add_vectors++ ivf_add_vectors(str(path), embeddings, passage_ids)+ except ImportError:+ raise RuntimeError("IVF backend required. Install leann-backend-ivf.")+ rollback_passages_size = passages_file.stat().st_size if passages_file.exists() else 0+ offset_map_backup = offset_map.copy()+ try:+ with open(passages_file, "a", encoding="utf-8") as f:+ for chunk in valid_chunks:+ off = f.tell()+ json.dump(+ {+ "id": chunk["id"],+ "text": chunk["text"],+ "metadata": chunk.get("metadata", {}),+ },+ f,+ ensure_ascii=False,+ )+ f.write("\n")+ offset_map[chunk["id"]] = off+ with open(offset_file, "wb") as f:+ pickle.dump(offset_map, f)+ meta["total_passages"] = len(offset_map)+ with open(meta_path, "w", encoding="utf-8") as f:+ json.dump(meta, f, indent=2)+ logger.info(+ "Appended %d passages to IVF index '%s'. Total: %d",+ len(valid_chunks),+ index_path,+ len(offset_map),+ )+ except Exception:+ if passages_file.exists():+ with open(passages_file, "rb+") as f:+ f.truncate(rollback_passages_size)+ offset_map = offset_map_backup+ with open(offset_file, "wb") as f:+ pickle.dump(offset_map, f)+ raise+ self.chunks.clear()+ return++ # HNSW path below+ from leann_backend_hnsw import faiss++ index = faiss.read_index(str(index_file))+ if hasattr(index, "is_recompute"):+ index.is_recompute = needs_recompute+ print(f"index.is_recompute: {index.is_recompute}")+ if getattr(index, "storage", None) is None:+ if index.metric_type == faiss.METRIC_INNER_PRODUCT:+ storage_index = faiss.IndexFlatIP(index.d)+ else:+ storage_index = faiss.IndexFlatL2(index.d)+ index.storage = storage_index+ index.own_fields = True+ # Faiss expects storage.ntotal to reflect the existing graph's+ # population (even if the vectors themselves were pruned from disk+ # for recompute mode). When we attach a fresh IndexFlat here its+ # ntotal starts at zero, which later causes IndexHNSW::add to+ # believe new "preset" levels were provided and trips the+ # `n0 + n == levels.size()` assertion. Seed the temporary storage+ # with the current ntotal so Faiss maintains the proper offset for+ # incoming vectors.+ try:+ storage_index.ntotal = index.ntotal+ except AttributeError:+ # Older Faiss builds may not expose ntotal as a writable+ # attribute; in that case we fall back to the default behaviour.+ pass+ if index.d != embedding_dim:+ raise ValueError(+ f"Existing index dimension ({index.d}) does not match new embeddings ({embedding_dim})."+ )++ passage_meta_mode = meta.get("embedding_mode", self.embedding_mode)+ passage_provider_options = meta.get("embedding_options", self.embedding_options)++ base_id = index.ntotal+ for offset, chunk in enumerate(valid_chunks):+ new_id = str(base_id + offset)+ chunk.setdefault("metadata", {})["id"] = new_id+ chunk["id"] = new_id++ # Append passages/offsets before we attempt index.add so the ZMQ server+ # can resolve newly assigned IDs during recompute. Keep rollback hooks+ # so we can restore files if the update fails mid-way.+ rollback_passages_size = passages_file.stat().st_size if passages_file.exists() else 0+ offset_map_backup = offset_map.copy()++ try:+ with open(passages_file, "a", encoding="utf-8") as f:+ for chunk in valid_chunks:+ offset = f.tell()+ json.dump(+ {+ "id": chunk["id"],+ "text": chunk["text"],+ "metadata": chunk.get("metadata", {}),+ },+ f,+ ensure_ascii=False,+ )+ f.write("\n")+ offset_map[chunk["id"]] = offset++ with open(offset_file, "wb") as f:+ pickle.dump(offset_map, f)++ server_manager: Optional[EmbeddingServerManager] = None+ server_started = False+ requested_zmq_port = int(os.getenv("LEANN_UPDATE_ZMQ_PORT", "5557"))++ try:+ if needs_recompute:+ server_manager = EmbeddingServerManager(+ backend_module_name="leann_backend_hnsw.hnsw_embedding_server"+ )+ server_started, actual_port = server_manager.start_server(+ port=requested_zmq_port,+ model_name=self.embedding_model,+ embedding_mode=passage_meta_mode,+ passages_file=str(meta_path),+ distance_metric=distance_metric,+ use_daemon=False,+ enable_warmup=False,+ provider_options=passage_provider_options,+ )+ if not server_started:+ raise RuntimeError(+ "Failed to start HNSW embedding server for recompute update."+ )+ if actual_port != requested_zmq_port:+ logger.warning(+ "Embedding server started on port %s instead of requested %s. "+ "Using reassigned port.",+ actual_port,+ requested_zmq_port,+ )+ if hasattr(index.hnsw, "set_zmq_port"):+ index.hnsw.set_zmq_port(actual_port)+ elif hasattr(index, "set_zmq_port"):+ index.set_zmq_port(actual_port)++ if needs_recompute:+ for i in range(embeddings.shape[0]):+ print(f"add {i} embeddings")+ index.add(1, faiss.swig_ptr(embeddings[i : i + 1]))+ else:+ index.add(embeddings.shape[0], faiss.swig_ptr(embeddings))+ faiss.write_index(index, str(index_file))+ finally:+ if server_started and server_manager is not None:+ server_manager.stop_server()++ except Exception:+ # Roll back appended passages/offset map to keep files consistent.+ if passages_file.exists():+ with open(passages_file, "rb+") as f:+ f.truncate(rollback_passages_size)+ offset_map = offset_map_backup+ with open(offset_file, "wb") as f:+ pickle.dump(offset_map, f)+ raise++ meta["total_passages"] = len(offset_map)+ with open(meta_path, "w", encoding="utf-8") as f:+ json.dump(meta, f, indent=2)++ logger.info(+ "Appended %d passages to index '%s'. New total: %d",+ len(valid_chunks),+ index_path,+ len(offset_map),+ )++ self.chunks.clear()++ if needs_recompute:+ prune_hnsw_embeddings_inplace(str(index_file))+++class LeannSearcher:+ def __init__(+ self,+ index_path: str,+ enable_warmup: bool = True,+ recompute_embeddings: bool = True,+ use_daemon: bool = True,+ daemon_ttl_seconds: int = 900,+ **backend_kwargs,+ ):+ # Fix path resolution for Colab and other environments+ if not Path(index_path).is_absolute():+ index_path = str(Path(index_path).resolve())++ self.meta_path_str = f"{index_path}.meta.json"+ if not Path(self.meta_path_str).exists():+ parent_dir = Path(index_path).parent+ print(+ f"Leann metadata file not found at {self.meta_path_str}, and you may need to rm -rf {parent_dir}"+ )+ # highlight in red the filenotfound error+ raise FileNotFoundError(+ f"Leann metadata file not found at {self.meta_path_str}, \033[91m you may need to rm -rf {parent_dir}\033[0m"+ )+ with open(self.meta_path_str, encoding="utf-8") as f:+ self.meta_data = json.load(f)+ backend_name = self.meta_data["backend_name"]+ self.embedding_model = self.meta_data["embedding_model"]+ # Support both old and new format+ self.embedding_mode = self.meta_data.get("embedding_mode", "sentence-transformers")+ self.embedding_options = self.meta_data.get("embedding_options", {})+ # Delegate portability handling to PassageManager+ self.passage_manager = PassageManager(+ self.meta_data.get("passage_sources", []), metadata_file_path=self.meta_path_str+ )+ # Preserve backend name for conditional parameter forwarding+ self.backend_name = backend_name+ backend_factory = BACKEND_REGISTRY.get(backend_name)+ if backend_factory is None:+ raise ValueError(f"Backend '{backend_name}' not found.")++ # Global recompute flag for this searcher (explicit knob, default True)+ self.recompute_embeddings: bool = bool(recompute_embeddings)++ # Warmup flag: keep using the existing enable_warmup parameter,+ # but default it to True so cold-start happens earlier.+ self._warmup: bool = bool(enable_warmup)+ self._use_daemon: bool = bool(use_daemon)+ self._daemon_ttl_seconds: int = int(daemon_ttl_seconds)++ final_kwargs = {**self.meta_data.get("backend_kwargs", {}), **backend_kwargs}+ final_kwargs["enable_warmup"] = self._warmup+ final_kwargs["use_daemon"] = self._use_daemon+ final_kwargs["daemon_ttl_seconds"] = self._daemon_ttl_seconds+ if self.embedding_options:+ final_kwargs.setdefault("embedding_options", self.embedding_options)+ self.backend_impl: LeannBackendSearcherInterface = backend_factory.searcher(+ index_path, **final_kwargs+ )+ self.bm25_scorer: Optional[BM25Scorer] = None++ # Optional one-shot warmup at construction time to hide cold-start latency.+ if self._warmup:+ self.warmup()++ def warmup(self) -> None:+ """Warm up embedding path so first user query is faster."""+ try:+ _ = self.backend_impl.compute_query_embedding(+ "__LEANN_WARMUP__",+ use_server_if_available=self.recompute_embeddings,+ )+ except Exception as exc:+ logger.warning(f"Warmup embedding failed (ignored): {exc}")++ def search(+ self,+ query: str,+ top_k: int = 5,+ complexity: int = 64,+ beam_width: int = 1,+ prune_ratio: float = 0.0,+ recompute_embeddings: Optional[bool] = None,+ pruning_strategy: Literal["global", "local", "proportional"] = "global",+ expected_zmq_port: int = 5557,+ metadata_filters: Optional[dict[str, dict[str, Union[str, int, float, bool, list]]]] = None,+ batch_size: int = 0,+ use_grep: bool = False,+ gemma: float = 1.0,+ provider_options: Optional[dict[str, Any]] = None,+ **kwargs,+ ) -> list[SearchResult]:+ """+ Search for nearest neighbors with optional metadata filtering.++ Args:+ query: Text query to search for+ top_k: Number of nearest neighbors to return+ complexity: Search complexity/candidate list size, higher = more accurate but slower+ beam_width: Number of parallel search paths/IO requests per iteration+ prune_ratio: Ratio of neighbors to prune via approximate distance (0.0-1.0)+ recompute_embeddings: (Deprecated) Per-call override for recompute mode.+ Configure this at LeannSearcher(..., recompute_embeddings=...) instead.+ pruning_strategy: Candidate selection strategy - "global" (default), "local", or "proportional"+ expected_zmq_port: ZMQ port for embedding server communication+ metadata_filters: Optional filters to apply to search results based on metadata.+ Format: {"field_name": {"operator": value}}+ Supported operators:+ - Comparison: "==", "!=", "<", "<=", ">", ">="+ - Membership: "in", "not_in"+ - String: "contains", "starts_with", "ends_with"+ Example: {"chapter": {"<=": 5}, "tags": {"in": ["fiction", "drama"]}}+ gemma: Weight of vector search results in hybrid search (0.0-1.0), 1 = pure vector search, 0 = pure keyword search+ **kwargs: Backend-specific parameters++ Returns:+ List of SearchResult objects with text, metadata, and similarity scores+ """+ # Handle grep search+ if use_grep:+ return self._grep_search(query, top_k)++ logger.info("🔍 LeannSearcher.search() called:")+ logger.info(f" Query: '{query}'")+ logger.info(f" Top_k: {top_k}")+ logger.info(f" Metadata filters: {metadata_filters}")+ logger.info(f" Additional kwargs: {kwargs}")++ # Smart top_k detection and adjustment+ # Use PassageManager length (sum of shard sizes) to avoid+ # depending on a massive combined map+ total_docs = len(self.passage_manager)+ original_top_k = top_k+ if top_k > total_docs:+ top_k = total_docs+ logger.warning(+ f" ⚠️ Requested top_k ({original_top_k}) exceeds total documents ({total_docs})"+ )+ logger.warning(f" ✅ Auto-adjusted top_k to {top_k} to match available documents")++ # Handle pure keyword search+ if gemma == 0.0:+ start_time = time.time()+ bm25_results = self._bm25_search(query, top_k)+ # Convert BM25 results to the expected format+ results = {+ "labels": [[r.id for r in bm25_results]],+ "distances": [[r.score for r in bm25_results]],+ }+ else:+ # Perform vector search+ zmq_port = None++ # Resolve effective recompute flag for this search.+ if recompute_embeddings is not None:+ logger.warning(+ "LeannSearcher.search(..., recompute_embeddings=...) is deprecated and "+ "will be removed in a future version. Configure recompute at "+ "LeannSearcher(..., recompute_embeddings=...) instead."+ )+ effective_recompute = bool(recompute_embeddings)+ else:+ effective_recompute = self.recompute_embeddings++ start_time = time.time()+ if effective_recompute:+ zmq_port = self.backend_impl._ensure_server_running(+ self.meta_path_str,+ port=expected_zmq_port,+ enable_warmup=self._warmup,+ use_daemon=self._use_daemon,+ daemon_ttl_seconds=self._daemon_ttl_seconds,+ **kwargs,+ )+ del expected_zmq_port+ zmq_time = time.time() - start_time+ logger.info(f" Launching server time: {zmq_time} seconds")++ start_time = time.time()++ # Extract query template from stored embedding_options with fallback chain:+ # 1. Check provider_options override (highest priority)+ # 2. Check query_prompt_template (new format)+ # 3. Check prompt_template (old format for backward compat)+ # 4. None (no template)+ query_template = None+ if provider_options and "prompt_template" in provider_options:+ query_template = provider_options["prompt_template"]+ elif "query_prompt_template" in self.embedding_options:+ query_template = self.embedding_options["query_prompt_template"]+ elif "prompt_template" in self.embedding_options:+ query_template = self.embedding_options["prompt_template"]++ query_embedding = self.backend_impl.compute_query_embedding(+ query,+ use_server_if_available=effective_recompute,+ zmq_port=zmq_port,+ query_template=query_template,+ )+ logger.info(f" Generated embedding shape: {query_embedding.shape}")+ embedding_time = time.time() - start_time+ logger.info(f" Embedding time: {embedding_time} seconds")++ start_time = time.time()+ backend_search_kwargs: dict[str, Any] = {+ "complexity": complexity,+ "beam_width": beam_width,+ "prune_ratio": prune_ratio,+ "recompute_embeddings": effective_recompute,+ "pruning_strategy": pruning_strategy,+ "zmq_port": zmq_port,+ }+ # Only HNSW supports batching; forward conditionally+ if self.backend_name == "hnsw":+ backend_search_kwargs["batch_size"] = batch_size++ # Merge any extra kwargs last+ backend_search_kwargs.update(kwargs)++ results = self.backend_impl.search(+ query_embedding,+ top_k,+ **backend_search_kwargs,+ )++ # Handle hybrid search+ if 0.0 < gemma < 1.0:+ logger.info(f" 🌟 Hybrid search enabled with gemma={gemma}")+ BM25_WEIGHT = 1.0 - gemma+ bm25_results = self._bm25_search(query, top_k)+ hybrid_scores: dict[str, float] = {}+ # Add vector search scores (weighted by gemma)+ if "labels" in results and "distances" in results:+ for doc_id, score in zip(results["labels"][0], results["distances"][0]):+ hybrid_scores[doc_id] = gemma * score+ # Add BM25 scores (weighted by BM25_WEIGHT)+ for bm25_result in bm25_results:+ doc_id = bm25_result.id+ if doc_id in hybrid_scores:+ hybrid_scores[doc_id] += BM25_WEIGHT * bm25_result.score+ else:+ hybrid_scores[doc_id] = BM25_WEIGHT * bm25_result.score++ sorted_hybrid = sorted(hybrid_scores.items(), key=lambda x: x[1], reverse=True)[:top_k]+ results["labels"] = [[doc_id for doc_id, _ in sorted_hybrid]]+ results["distances"] = [[score for _, score in sorted_hybrid]]++ logger.info(+ f" Combined {len(hybrid_scores)} unique documents from vector and BM25 search"+ )++ search_time = time.time() - start_time+ logger.info(f" Search time in search() LEANN searcher: {search_time} seconds")+ logger.info(f" Backend returned: labels={len(results.get('labels', [[]])[0])} results")++ enriched_results = []+ if "labels" in results and "distances" in results:+ logger.info(f" Processing {len(results['labels'][0])} passage IDs:")+ # Python 3.9 does not support zip(strict=...); lengths are expected to match+ for i, (string_id, dist) in enumerate(+ zip(results["labels"][0], results["distances"][0])+ ):+ try:+ passage_data = self.passage_manager.get_passage(string_id)+ enriched_results.append(+ SearchResult(+ id=string_id,+ score=dist,+ text=passage_data["text"],+ metadata=passage_data.get("metadata", {}),+ )+ )++ # Color codes for better logging+ GREEN = "\033[92m"+ BLUE = "\033[94m"+ YELLOW = "\033[93m"+ RESET = "\033[0m"++ # Truncate text for display (first 100 chars)+ display_text = passage_data["text"]+ logger.info(+ f" {GREEN}✓{RESET} {BLUE}[{i + 1:2d}]{RESET} {YELLOW}ID:{RESET} '{string_id}' {YELLOW}Score:{RESET} {dist:.4f} {YELLOW}Text:{RESET} {display_text}"+ )+ except KeyError:+ RED = "\033[91m"+ RESET = "\033[0m"+ logger.error(+ f" {RED}✗{RESET} [{i + 1:2d}] ID: '{string_id}' -> {RED}ERROR: Passage not found!{RESET}"+ )++ # Apply metadata filters if specified+ if metadata_filters:+ logger.info(f" 🔍 Applying metadata filters: {metadata_filters}")+ enriched_results = self.passage_manager.filter_search_results(+ enriched_results, metadata_filters+ )++ # Define color codes outside the loop for final message+ GREEN = "\033[92m"+ RESET = "\033[0m"+ logger.info(f" {GREEN}✓ Final enriched results: {len(enriched_results)} passages{RESET}")+ return enriched_results++ def _init_bm25(self) -> None:+ """Initialize BM25 scorer"""+ self.bm25_scorer = BM25Scorer()+ # Load all the files directly+ passages = []+ for passage_file in self.passage_manager.passage_files.values():+ with open(passage_file, encoding="utf-8") as f:+ for line in f:+ if line.strip():+ data = json.loads(line)+ passages.append(data)+ self.bm25_scorer.fit(passages)++ def _bm25_search(self, query: str, top_k: int = 5) -> list[SearchResult]:+ """Perform BM25 search on raw passages"""+ if self.bm25_scorer is None:+ self._init_bm25()+ logger.info(" BM25 scorer initialized")+ scorer = self.bm25_scorer+ if scorer is None:+ raise RuntimeError("BM25 scorer failed to initialize")+ return scorer.search(query, top_k)++ def _find_jsonl_file(self) -> Optional[str]:+ """Find the .jsonl file containing raw passages for grep search"""+ index_path = Path(self.meta_path_str).parent+ potential_files = [+ index_path / "documents.leann.passages.jsonl",+ index_path.parent / "documents.leann.passages.jsonl",+ ]++ for file_path in potential_files:+ if file_path.exists():+ return str(file_path)+ return None++ def _grep_search(self, query: str, top_k: int = 5) -> list[SearchResult]:+ """Perform grep-based search on raw passages"""+ jsonl_file = self._find_jsonl_file()+ if not jsonl_file:+ raise FileNotFoundError("No .jsonl passages file found for grep search")++ try:+ cmd = ["grep", "-i", "-n", query, jsonl_file]+ result = subprocess.run(cmd, capture_output=True, text=True, check=False)++ if result.returncode == 1:+ return []+ elif result.returncode != 0:+ raise RuntimeError(f"Grep failed: {result.stderr}")++ matches = []+ for line in result.stdout.strip().split("\n"):+ if not line:+ continue+ parts = line.split(":", 1)+ if len(parts) != 2:+ continue++ try:+ data = json.loads(parts[1])+ text = data.get("text", "")+ score = text.lower().count(query.lower())++ matches.append(+ SearchResult(+ id=data.get("id", parts[0]),+ text=text,+ metadata=data.get("metadata", {}),+ score=float(score),+ )+ )+ except json.JSONDecodeError:+ continue++ matches.sort(key=lambda x: x.score, reverse=True)+ return matches[:top_k]++ except FileNotFoundError:+ raise RuntimeError(+ "grep command not found. Please install grep or use semantic search."+ )++ def _python_regex_search(self, query: str, top_k: int = 5) -> list[SearchResult]:+ """Fallback regex search"""+ jsonl_file = self._find_jsonl_file()+ if not jsonl_file:+ raise FileNotFoundError("No .jsonl file found")++ pattern = re.compile(re.escape(query), re.IGNORECASE)+ matches = []++ with open(jsonl_file, encoding="utf-8") as f:+ for line_num, line in enumerate(f, 1):+ if pattern.search(line):+ try:+ data = json.loads(line.strip())+ matches.append(+ SearchResult(+ id=data.get("id", str(line_num)),+ text=data.get("text", ""),+ metadata=data.get("metadata", {}),+ score=float(len(pattern.findall(data.get("text", "")))),+ )+ )+ except json.JSONDecodeError:+ continue++ matches.sort(key=lambda x: x.score, reverse=True)+ return matches[:top_k]++ def cleanup(self):+ """Explicitly cleanup embedding server resources.+ This method should be called after you're done using the searcher,+ especially in test environments or batch processing scenarios.+ """+ backend = getattr(self.backend_impl, "embedding_server_manager", None)+ if backend is not None:+ backend.stop_server()++ # Enable automatic cleanup patterns+ def __enter__(self):+ return self++ def __exit__(self, exc_type, exc, tb):+ try:+ self.cleanup()+ except Exception:+ pass++ def __del__(self):+ try:+ self.cleanup()+ except Exception:+ # Avoid noisy errors during interpreter shutdown+ pass+++class LeannChat:+ def __init__(+ self,+ index_path: str,+ llm_config: Optional[dict[str, Any]] = None,+ enable_warmup: bool = False,+ searcher: Optional[LeannSearcher] = None,+ **kwargs,+ ):+ if searcher is None:+ self.searcher = LeannSearcher(index_path, enable_warmup=enable_warmup, **kwargs)+ self._owns_searcher = True+ else:+ self.searcher = searcher+ self._owns_searcher = False+ self.llm = get_llm(llm_config)++ def ask(+ self,+ question: str,+ top_k: int = 5,+ complexity: int = 64,+ beam_width: int = 1,+ prune_ratio: float = 0.0,+ recompute_embeddings: bool = True,+ pruning_strategy: Literal["global", "local", "proportional"] = "global",+ llm_kwargs: Optional[dict[str, Any]] = None,+ expected_zmq_port: int = 5557,+ metadata_filters: Optional[dict[str, dict[str, Union[str, int, float, bool, list]]]] = None,+ batch_size: int = 0,+ use_grep: bool = False,+ gemma: float = 1.0,+ **search_kwargs,+ ):+ if llm_kwargs is None:+ llm_kwargs = {}+ search_time = time.time()+ results = self.searcher.search(+ question,+ top_k=top_k,+ complexity=complexity,+ beam_width=beam_width,+ prune_ratio=prune_ratio,+ recompute_embeddings=recompute_embeddings,+ pruning_strategy=pruning_strategy,+ expected_zmq_port=expected_zmq_port,+ metadata_filters=metadata_filters,+ use_grep=use_grep,+ gemma=gemma,+ batch_size=batch_size,+ **search_kwargs,+ )+ search_time = time.time() - search_time+ logger.info(f" Search time: {search_time} seconds")+ context = "\n\n".join([r.text for r in results])+ prompt = (+ "Here is some retrieved context that might help answer your question:\n\n"+ f"{context}\n\n"+ f"Question: {question}\n\n"+ "Please provide the best answer you can based on this context and your knowledge."+ )++ logger.info("The context provided to the LLM is:")+ logger.info(f"{'Relevance':<10} | {'Chunk id':<10} | {'Content':<60} | {'Source':<80}")+ logger.info("-" * 150)+ for r in results:+ chunk_relevance = f"{r.score:.3f}"+ chunk_id = r.id+ chunk_content = r.text[:60]+ chunk_source = r.metadata.get("source", "")[:80]+ logger.info(+ f"{chunk_relevance:<10} | {chunk_id:<10} | {chunk_content:<60} | {chunk_source:<80}"+ )+ ask_time = time.time()+ ans = self.llm.ask(prompt, **llm_kwargs)+ ask_time = time.time() - ask_time+ logger.info(f" Ask time: {ask_time} seconds")+ return ans++ def start_interactive(self):+ """Start interactive chat session."""+ session = create_api_session()++ def handle_query(user_input: str):+ response = self.ask(user_input)+ print(f"Leann: {response}")++ session.run_interactive_loop(handle_query)++ def cleanup(self):+ """Explicitly cleanup embedding server resources.++ This method should be called after you're done using the chat interface,+ especially in test environments or batch processing scenarios.+ """+ # Only stop the embedding server if this LeannChat instance created the searcher.+ # When a shared searcher is passed in, avoid shutting down the server to enable reuse.+ if getattr(self, "_owns_searcher", False) and hasattr(self.searcher, "cleanup"):+ self.searcher.cleanup()++ # Enable automatic cleanup patterns+ def __enter__(self):+ return self++ def __exit__(self, exc_type, exc, tb):+ try:+ self.cleanup()+ except Exception:+ pass++ def __del__(self):+ try:+ self.cleanup()+ except Exception:+ passdiff --git a/packages/leann-core/src/leann/cli.py b/packages/leann-core/src/leann/cli.pyindex b4e4046..01e6c0b 100644--- a/packages/leann-core/src/leann/cli.py+++ b/packages/leann-core/src/leann/cli.py@@ -26,38 +26,14 @@ from .settings import ()from .sync import FileSynchronizer-# Manifest file for incremental build: maps source file path -> mtime-SOURCES_MANIFEST_FILENAME = "documents.leann.sources.json"-def _normalize_path(path: str) -> str:- """Return absolute path string for consistent manifest keys."""+ """Return absolute path string for consistent keys."""if not path:return pathreturn str(Path(path).resolve())-def load_sources_manifest(index_dir: Path, index_name: str) -> dict[str, float]:- """Load sources manifest (path -> mtime). Returns {} if file missing or invalid."""- manifest_path = index_dir / SOURCES_MANIFEST_FILENAME- if not manifest_path.exists():- return {}- try:- with open(manifest_path, encoding="utf-8") as f:- data = json.load(f)- return data.get("sources", {})- except (json.JSONDecodeError, OSError):- return {}---def save_sources_manifest(index_dir: Path, index_name: str, sources: dict[str, float]) -> None:- """Write sources manifest (path -> mtime)."""- manifest_path = index_dir / SOURCES_MANIFEST_FILENAME- index_dir.mkdir(parents=True, exist_ok=True)- with open(manifest_path, "w", encoding="utf-8") as f:- json.dump({"sources": sources}, f, indent=2)--@contextlib.contextmanagerdef suppress_cpp_output(suppress: bool = True):"""Context manager to suppress C++ stdout/stderr output from FAISS/HNSW@@ -367,8 +343,8 @@ Examples:watch_parser.add_argument("--interval",type=int,- default=5,- help="Poll interval in seconds (default: 5)",+ default=30,+ help="Poll interval in seconds (default: 30)",)watch_parser.add_argument("--once",@@ -1687,22 +1663,239 @@ Examples:roots.add(str(path_obj.parent))return sorted(roots)- def _initialize_file_synchronizers(+ def _create_synchronizers(self,+ index_dir: Path,roots: list[str],- include_extensions: Optional[list[str]],- ignore_patterns: Optional[list[str]],- ) -> None:+ include_extensions: Optional[list[str]] = None,+ ignore_patterns: Optional[list[str]] = None,+ ) -> list[FileSynchronizer]:+ """Create FileSynchronizers with snapshots stored in the index dir. Shared by build and watch."""+ synchronizers: list[FileSynchronizer] = []for root in roots:+ tag = hashlib.sha256(root.encode()).hexdigest()[:12]+ snapshot_path = str(index_dir / f"sync_{tag}.pickle")try:- FileSynchronizer(+ fs = FileSynchronizer(root_dir=root,ignore_patterns=ignore_patterns,include_extensions=include_extensions,- auto_load=True,+ snapshot_path=snapshot_path,)+ synchronizers.append(fs)except Exception as exc:- print(f"⚠️ Failed to initialize file synchronizer for {root}: {exc}")+ print(f"Warning: Failed to init synchronizer for {root}: {exc}")+ return synchronizers++ def _build_synchronizers(+ self,+ docs_paths: list[str],+ index_dir: Path,+ file_types: Optional[str] = None,+ include_hidden: bool = False,+ ) -> list[FileSynchronizer]:+ """Create FileSynchronizers for build from docs_paths."""+ roots = self._resolve_sync_roots(docs_paths)+ include_extensions = self._parse_file_types(file_types)+ ignore_patterns = self._sync_ignore_patterns(include_hidden)+ return self._create_synchronizers(index_dir, roots, include_extensions, ignore_patterns)++ def _detect_build_changes(+ self,+ synchronizers: list[FileSynchronizer],+ ) -> tuple[set[str], set[str], set[str]]:+ """Detect added/removed/modified files across all source roots using content hashes."""+ all_added: set[str] = set()+ all_removed: set[str] = set()+ all_modified: set[str] = set()+ for fs in synchronizers:+ added, removed, modified = fs.detect_changes()+ all_added.update(added)+ all_removed.update(removed)+ all_modified.update(modified)+ return all_added, all_removed, all_modified++ def _commit_synchronizers(self, synchronizers: list[FileSynchronizer]) -> None:+ """Persist all synchronizer snapshots after a successful build."""+ for fs in synchronizers:+ fs.commit()++ @staticmethod+ def _assign_chunk_ids(chunks: list[dict]) -> None:+ """Assign stable IDs to chunks based on their file path and position."""+ from collections import defaultdict++ by_path: dict[str, list] = defaultdict(list)+ for c in chunks:+ p = c.get("metadata", {}).get("file_path") or c.get("metadata", {}).get("source") or ""+ by_path[_normalize_path(p)].append(c)+ for path_key, path_chunks in by_path.items():+ for idx, c in enumerate(path_chunks):+ sid = hashlib.sha256(f"{path_key}:{idx}".encode()).hexdigest()[:16]+ c.setdefault("metadata", {})["id"] = sid+ c["id"] = sid++ def _chunks_for_paths(self, all_texts: list[dict], paths: set[str]) -> list[dict]:+ """Filter chunks belonging to the given file paths."""+ return [+ c+ for c in all_texts+ if _normalize_path(+ c.get("metadata", {}).get("file_path") or c.get("metadata", {}).get("source") or ""+ )+ in paths+ ]++ def _make_incremental_builder(self, args) -> "LeannBuilder":+ return LeannBuilder(+ backend_name=args.backend_name,+ embedding_model=args.embedding_model,+ embedding_mode=args.embedding_mode,+ embedding_options=self._build_embedding_options(args) or None,+ graph_degree=args.graph_degree,+ complexity=args.complexity,+ is_compact=args.compact,+ is_recompute=args.recompute,+ num_threads=args.num_threads,+ )++ def _incremental_add_only(+ self,+ index_path: str,+ all_texts: list[dict],+ args,+ new_paths: set[str],+ ) -> bool:+ """Add-only incremental update (works for HNSW and IVF)."""+ new_chunks = self._chunks_for_paths(all_texts, new_paths)+ if not new_chunks:+ return False+ self._assign_chunk_ids(new_chunks)+ builder = self._make_incremental_builder(args)+ for chunk in new_chunks:+ builder.add_text(chunk["text"], metadata=chunk["metadata"])+ print(+ f"Incremental update: adding {len(new_chunks)} chunks from {len(new_paths)} new file(s)..."+ )+ builder.update_index(index_path)+ print(f"Index updated at {index_path}")+ return True++ def _incremental_ivf_remove_only(+ self, index_path: str, index_dir: Path, removed_paths: set[str], args+ ) -> bool:+ """IVF remove-only fast path: remove chunk IDs without loading or chunking documents."""+ passages_file = index_dir / "documents.leann.passages.jsonl"+ if not passages_file.exists():+ return False+ chunk_ids_by_file = self._load_chunk_ids_by_file(passages_file)+ ids_to_remove: list[str] = []+ for p in removed_paths:+ ids_to_remove.extend(chunk_ids_by_file.get(p, []))+ if not ids_to_remove:+ return False+ print(+ f"Incremental IVF update (-{len(removed_paths)} removed): removing {len(ids_to_remove)} old chunks..."+ )+ builder = self._make_incremental_builder(args)+ builder.update_index(index_path, remove_passage_ids=ids_to_remove)+ print(f"Index updated at {index_path}")+ return True++ def _incremental_ivf_update(+ self,+ index_path: str,+ index_dir: Path,+ all_texts: list[dict],+ args,+ new_paths: set[str],+ removed_paths: set[str],+ modified_paths: set[str],+ ) -> bool:+ """IVF incremental update: remove old chunks for modified/removed files, add new chunks."""+ passages_file = index_dir / "documents.leann.passages.jsonl"+ chunk_ids_by_file = (+ self._load_chunk_ids_by_file(passages_file) if passages_file.exists() else {}+ )++ # Collect old chunk IDs to remove (modified + removed files)+ ids_to_remove: list[str] = []+ for p in modified_paths | removed_paths:+ ids_to_remove.extend(chunk_ids_by_file.get(p, []))++ # Collect new chunks to add (modified + new files)+ changed_paths = new_paths | modified_paths+ new_chunks = self._chunks_for_paths(all_texts, changed_paths)+ self._assign_chunk_ids(new_chunks)++ if not ids_to_remove and not new_chunks:+ return False++ builder = self._make_incremental_builder(args)+ for chunk in new_chunks:+ builder.add_text(chunk["text"], metadata=chunk["metadata"])++ parts = []+ if ids_to_remove:+ parts.append(f"removing {len(ids_to_remove)} old chunks")+ if new_chunks:+ parts.append(f"adding {len(new_chunks)} new chunks")+ file_parts = []+ if new_paths:+ file_parts.append(f"+{len(new_paths)} added")+ if modified_paths:+ file_parts.append(f"~{len(modified_paths)} modified")+ if removed_paths:+ file_parts.append(f"-{len(removed_paths)} removed")+ print(f"Incremental IVF update ({', '.join(file_parts)}): {', '.join(parts)}...")++ builder.update_index(+ index_path, remove_passage_ids=ids_to_remove if ids_to_remove else None+ )+ print(f"Index updated at {index_path}")+ return True++ @staticmethod+ def _log_rebuild_reason(+ meta: dict, args, new_paths: set, removed_paths: set, modified_paths: set+ ) -> None:+ """Print a human-readable explanation of why incremental update is not possible."""+ if removed_paths or modified_paths:+ reasons = []+ if removed_paths:+ reasons.append(f"{len(removed_paths)} file(s) removed")+ if modified_paths:+ reasons.append(f"{len(modified_paths)} file(s) modified")+ print(+ f"Incremental update not possible ({', '.join(reasons)}); falling back to full rebuild."+ )+ return++ blockers = []+ if meta.get("backend_name") not in ("hnsw", "ivf"):+ blockers.append(+ f"backend '{meta.get('backend_name')}' does not support incremental updates"+ )+ if meta.get("is_compact", meta.get("backend_kwargs", {}).get("is_compact", True)):+ blockers.append("index is compact (read-only); rebuild with --no-compact to enable")+ if meta.get("embedding_model") != args.embedding_model:+ blockers.append(+ f"embedding model changed ('{meta.get('embedding_model')}' -> '{args.embedding_model}')"+ )+ if meta.get("embedding_mode") != args.embedding_mode:+ blockers.append(+ f"embedding mode changed ('{meta.get('embedding_mode')}' -> '{args.embedding_mode}')"+ )+ if blockers:+ print(+ f"Incremental update not possible: {'; '.join(blockers)}. Falling back to full rebuild."+ )+ else:+ changes = []+ if new_paths:+ changes.append(f"+{len(new_paths)} added")+ summary = ", ".join(changes) if changes else "unknown reason"+ print(f"Full rebuild starting ({summary})...")def _write_sync_config(self,@@ -1838,161 +2031,144 @@ Examples:paragraph_separator="\n\n",)- all_texts = self.load_documents(- docs_paths, args.file_types, include_hidden=args.include_hidden, args=args+ # Detect changes first so we can skip load_documents for remove-only+ index_dir.mkdir(parents=True, exist_ok=True)+ synchronizers = self._build_synchronizers(+ docs_paths, index_dir, file_types=args.file_types, include_hidden=args.include_hidden)- if not all_texts:- print("No documents found")- return- # Idempotent build: detect changes and choose the minimal work path- if index_dir.exists() and not args.force:- manifest = load_sources_manifest(index_dir, "documents.leann")+ if index_dir.exists() and not args.force and synchronizers:meta_path = index_dir / "documents.leann.meta.json"+ new_paths, removed_paths, modified_paths = self._detect_build_changes(synchronizers)- current_sources: dict[str, float] = {}- for c in all_texts:- path = c.get("metadata", {}).get("file_path") or c.get("metadata", {}).get("source")- if path and os.path.isfile(path):- current_sources[_normalize_path(path)] = os.path.getmtime(path)-- if not manifest or not meta_path.exists():- print("Rebuilding index (legacy format detected, enabling incremental support)...")- else:- new_paths = {p for p in current_sources if p not in manifest}- removed_paths = {p for p in manifest if p not in current_sources}- modified_paths = {- p- for p in current_sources- if p in manifest and current_sources[p] != manifest[p]- }-- if not new_paths and not removed_paths and not modified_paths:- print("Index up to date.")- return+ if not new_paths and not removed_paths and not modified_paths:+ print("Index up to date.")+ return+ # Show change summary (add / modify / delete) before build+ change_parts = []+ if new_paths:+ change_parts.append(f"+{len(new_paths)} added")+ if modified_paths:+ change_parts.append(f"~{len(modified_paths)} modified")+ if removed_paths:+ change_parts.append(f"-{len(removed_paths)} removed")+ if change_parts:+ print(f"Changes: {', '.join(change_parts)}")++ if meta_path.exists():with open(meta_path, encoding="utf-8") as f:meta = json.load(f)- can_incremental = (+ backend_name = meta.get("backend_name")+ is_compact = meta.get(+ "is_compact", meta.get("backend_kwargs", {}).get("is_compact", True)+ )+ same_embedding = (+ meta.get("embedding_model") == args.embedding_model+ and meta.get("embedding_mode") == args.embedding_mode+ )++ # IVF supports remove+add, so it can handle modified and removed files incrementally+ can_ivf_update = backend_name == "ivf" and not is_compact and same_embedding+ # HNSW only supports add (no remove), so it needs add-only changes+ can_add_only = (not removed_pathsand not modified_paths- and meta.get("backend_name") in ("hnsw", "ivf")- and not meta.get(- "is_compact", meta.get("backend_kwargs", {}).get("is_compact", True)- )- and meta.get("embedding_model") == args.embedding_model- and meta.get("embedding_mode") == args.embedding_mode+ and backend_name in ("hnsw", "ivf")+ and not is_compact+ and same_embedding)- if not can_incremental and (removed_paths or modified_paths):- reasons = []- if removed_paths:- reasons.append(f"{len(removed_paths)} file(s) removed")- if modified_paths:- reasons.append(f"{len(modified_paths)} file(s) modified")- print(- f"Incremental update not possible ({', '.join(reasons)}); "- f"falling back to full rebuild."+ # Remove-only fast path: no load/chunk, just remove IDs from index+ if can_ivf_update and removed_paths and not new_paths and not modified_paths:+ result = self._incremental_ivf_remove_only(+ index_path, index_dir, removed_paths, args)- elif not can_incremental:- blockers = []- if meta.get("backend_name") not in ("hnsw", "ivf"):- blockers.append(- f"backend '{meta.get('backend_name')}' does not support incremental updates"- )- if meta.get(- "is_compact", meta.get("backend_kwargs", {}).get("is_compact", True)- ):- blockers.append(- "index is compact (read-only); rebuild with --no-compact to enable incremental updates"- )- if meta.get("embedding_model") != args.embedding_model:- blockers.append(- f"embedding model changed ('{meta.get('embedding_model')}' -> '{args.embedding_model}')"- )- if meta.get("embedding_mode") != args.embedding_mode:- blockers.append(- f"embedding mode changed ('{meta.get('embedding_mode')}' -> '{args.embedding_mode}')"- )- if blockers:- print(- f"Incremental update not possible: {'; '.join(blockers)}. Falling back to full rebuild."+ if result:+ self._commit_synchronizers(synchronizers)+ self._write_sync_config(+ index_dir,+ self._resolve_sync_roots(docs_paths),+ self._parse_file_types(args.file_types),+ self._sync_ignore_patterns(args.include_hidden),)+ self.register_project_dir()+ return++ # Load only changed files (no need to load/chunk the entire corpus)+ paths_to_load = new_paths | modified_paths+ all_texts = self.load_documents(+ list(paths_to_load),+ args.file_types,+ include_hidden=args.include_hidden,+ args=args,+ )+ # Proceed even when all_texts is empty (e.g. file emptied): we still need to remove old chunks+ if not all_texts and not (can_ivf_update and (modified_paths or removed_paths)):+ print("No documents found")+ return- if can_incremental and new_paths:- new_chunks = [- c- for c in all_texts- if _normalize_path(- c.get("metadata", {}).get("file_path")- or c.get("metadata", {}).get("source")- or ""- )- in new_paths- ]- from collections import defaultdict-- by_path: dict[str, list] = defaultdict(list)- for c in new_chunks:- path = (- c.get("metadata", {}).get("file_path")- or c.get("metadata", {}).get("source")- or ""- )- by_path[_normalize_path(path)].append(c)- for path_key, path_chunks in by_path.items():- for idx, c in enumerate(path_chunks):- sid = hashlib.sha256(f"{path_key}:{idx}".encode()).hexdigest()[:16]- c.setdefault("metadata", {})["id"] = sid- c["id"] = sid-- embedding_options_inc = self._build_embedding_options(args)- builder_inc = LeannBuilder(- backend_name=args.backend_name,- embedding_model=args.embedding_model,- embedding_mode=args.embedding_mode,- embedding_options=embedding_options_inc or None,- graph_degree=args.graph_degree,- complexity=args.complexity,- is_compact=args.compact,- is_recompute=args.recompute,- num_threads=args.num_threads,+ if can_ivf_update and (new_paths or modified_paths or removed_paths):+ result = self._incremental_ivf_update(+ index_path,+ index_dir,+ all_texts,+ args,+ new_paths,+ removed_paths,+ modified_paths,)- for chunk in new_chunks:- builder_inc.add_text(chunk["text"], metadata=chunk["metadata"])- print(- f"Incremental update: adding {len(new_chunks)} chunks from {len(new_paths)} new file(s)..."+ if result:+ self._commit_synchronizers(synchronizers)+ self._write_sync_config(+ index_dir,+ self._resolve_sync_roots(docs_paths),+ self._parse_file_types(args.file_types),+ self._sync_ignore_patterns(args.include_hidden),+ )+ self.register_project_dir()+ return++ elif can_add_only and new_paths:+ result = self._incremental_add_only(+ index_path,+ all_texts,+ args,+ new_paths,)- builder_inc.update_index(index_path)- for p in new_paths:- manifest[p] = current_sources[p]- save_sources_manifest(index_dir, "documents.leann", manifest)- print(f"Index updated at {index_path}")- self.register_project_dir()- return+ if result:+ self._commit_synchronizers(synchronizers)+ self._write_sync_config(+ index_dir,+ self._resolve_sync_roots(docs_paths),+ self._parse_file_types(args.file_types),+ self._sync_ignore_patterns(args.include_hidden),+ )+ self.register_project_dir()+ return+else:- changes = []- if new_paths:- changes.append(f"+{len(new_paths)} added")- if modified_paths:- changes.append(f"~{len(modified_paths)} modified")- if removed_paths:- changes.append(f"-{len(removed_paths)} removed")- summary = ", ".join(changes) if changes else "incompatible index format"- print(f"Full rebuild starting ({summary})...")+ self._log_rebuild_reason(meta, args, new_paths, removed_paths, modified_paths)- index_dir.mkdir(parents=True, exist_ok=True)+ # Full rebuild: load documents if not already loaded (first build or force)+ try:+ _ = all_texts+ except NameError:+ all_texts = self.load_documents(+ docs_paths, args.file_types, include_hidden=args.include_hidden, args=args+ )+ if not all_texts:+ print("No documents found")+ returnprint(f"Building index '{index_name}' with {args.backend_name} backend...")- embedding_options = self._build_embedding_options(args)-builder = LeannBuilder(backend_name=args.backend_name,embedding_model=args.embedding_model,embedding_mode=args.embedding_mode,- embedding_options=embedding_options or None,+ embedding_options=self._build_embedding_options(args) or None,graph_degree=args.graph_degree,complexity=args.complexity,is_compact=args.compact,@@ -2004,28 +2180,19 @@ Examples:builder.add_text(chunk["text"], metadata=chunk["metadata"])builder.build_index(index_path)- # Write sources manifest for future incremental builds- full_sources = {}- for c in all_texts:- path = c.get("metadata", {}).get("file_path") or c.get("metadata", {}).get("source")- if path and os.path.isfile(path):- full_sources[_normalize_path(path)] = os.path.getmtime(path)- if full_sources:- save_sources_manifest(index_dir, "documents.leann", full_sources)+ for fs in synchronizers:+ fs.create_snapshot()+ self._write_sync_config(+ index_dir,+ self._resolve_sync_roots(docs_paths),+ self._parse_file_types(args.file_types),+ self._sync_ignore_patterns(args.include_hidden),+ )print(f"Index built at {index_path}")-- sync_roots = self._resolve_sync_roots(docs_paths)- if sync_roots:- include_extensions = self._parse_file_types(args.file_types)- ignore_patterns = self._sync_ignore_patterns(args.include_hidden)- self._initialize_file_synchronizers(sync_roots, include_extensions, ignore_patterns)- self._write_sync_config(index_dir, sync_roots, include_extensions, ignore_patterns)-- # Register this project directory in global registryself.register_project_dir()def _watch_check_changes(self, index_name: str) -> tuple[set[str], set[str], set[str]]:- """Check for file changes in watched directories. Returns (added, removed, modified) paths."""+ """Check for file changes using the same snapshots as build (index_dir)."""resolved = self._resolve_index_for_watch(index_name)if not resolved:return set(), set(), set()@@ -2039,30 +2206,16 @@ Examples:config = json.load(f)roots = config.get("roots") or []- include_extensions = config.get("include_extensions")- ignore_patterns = config.get("ignore_patterns")-- added_paths: set[str] = set()- removed_paths: set[str] = set()- modified_paths: set[str] = set()-- for root in roots:- try:- fs = FileSynchronizer(- root_dir=root,- ignore_patterns=ignore_patterns,- include_extensions=include_extensions,- auto_load=True,- )- added, removed, modified = fs.check_for_changes()- except Exception as exc:- print(f"Warning: Failed to check {root}: {exc}")- continue- added_paths.update(added)- removed_paths.update(removed)- modified_paths.update(modified)+ if not roots:+ return set(), set(), set()- return added_paths, removed_paths, modified_paths+ synchronizers = self._create_synchronizers(+ index_dir,+ roots,+ include_extensions=config.get("include_extensions"),+ ignore_patterns=config.get("ignore_patterns"),+ )+ return self._detect_build_changes(synchronizers)def _watch_report_changes(self,@@ -2323,7 +2476,7 @@ Examples:if "last_modified_date" in result.metadata:print(f" Modified: {result.metadata['last_modified_date']}")- print(f" {result.text[:200]}...")+ print(f" {result.text}")print(f" Source: {result.metadata.get('source', '')}")print()diff --git a/packages/leann-core/src/leann/sync.py b/packages/leann-core/src/leann/sync.pyindex fe3f875..d0a69e5 100644--- a/packages/leann-core/src/leann/sync.py+++ b/packages/leann-core/src/leann/sync.py@@ -78,24 +78,32 @@ class FileSynchronizer:ignore_patterns: Optional[list] = None,include_extensions: Optional[list] = None,auto_load=True,+ snapshot_path: Optional[str] = None,):if not os.path.isdir(root_dir):raise ValueError("This is not a valid directory")self.root_dir = root_dirself.ignore_patterns = ignore_patternsself.include_extensions = include_extensions+ self._custom_snapshot_path = snapshot_path+ self._pending_tree: Optional[MerkleTree] = None+ self.tree: Optional[MerkleTree] = Noneif auto_load:self.load_snapshot()def generate_file_hashes(self):file_hashes = {}- reader = SimpleDirectoryReader(- self.root_dir,- recursive=True,- exclude=self.ignore_patterns,- required_exts=self.include_extensions,- exclude_empty=False,- )+ try:+ reader = SimpleDirectoryReader(+ self.root_dir,+ recursive=True,+ exclude=self.ignore_patterns,+ required_exts=self.include_extensions,+ exclude_empty=False,+ )+ except ValueError:+ # Empty directory — no files to hash+ return file_hashes# print('reader.iter_data() length', len(list(reader.iter_data())))for file in reader.iter_data():@@ -128,20 +136,40 @@ class FileSynchronizer:return tree- def check_for_changes(self):+ def detect_changes(self) -> tuple[list[str], list[str], list[str]]:+ """Detect changes without persisting. Call commit() after successful processing."""file_hashes = self.generate_file_hashes()new_tree = self.build_merkle_tree(file_hashes)+ self._pending_tree = new_tree++ if self.tree is None:+ return list(file_hashes.keys()), [], []- changes = self.tree.compare_with(new_tree)+ return self.tree.compare_with(new_tree)- if changes:- self.tree = new_tree+ def commit(self):+ """Persist the pending snapshot after successful processing."""+ if self._pending_tree is not None:+ self.tree = self._pending_tree+ self._pending_tree = Noneself.save_snapshot()+ def create_snapshot(self):+ """Build and persist a snapshot from the current file state (for initial / forced builds)."""+ file_hashes = self.generate_file_hashes()+ self.tree = self.build_merkle_tree(file_hashes)+ self.save_snapshot()++ def check_for_changes(self) -> tuple[list[str], list[str], list[str]]:+ """Detect and auto-commit changes (convenience wrapper)."""+ changes = self.detect_changes()+ self.commit()return changes@propertydef snapshot_path(self):+ if self._custom_snapshot_path:+ return self._custom_snapshot_pathreturn f"{self.root_dir}.sync_context.pickle"def save_snapshot(self):@@ -155,7 +183,4 @@ class FileSynchronizer:with open(self.snapshot_path, "rb") as f:self.tree = pickle.load(f)except FileNotFoundError:- file_hashes = self.generate_file_hashes()- self.tree = self.build_merkle_tree(file_hashes)- self.save_snapshot()- # yooooo+ self.tree = Nonediff --git a/tests/test_incremental_build.py b/tests/test_incremental_build.pyindex 7833993..a7922ed 100644--- a/tests/test_incremental_build.py+++ b/tests/test_incremental_build.py@@ -3,47 +3,103 @@ Tests for incremental build (Feature #89).When an index already exists and build is run again without --force,only new files are indexed and appended to the existing index (HNSW, non-compact only).+Change detection uses content-hash (merkle tree) via FileSynchronizer."""-import jsonimport osfrom pathlib import Pathimport pytest-from leann.cli import (- SOURCES_MANIFEST_FILENAME,- _normalize_path,- load_sources_manifest,- save_sources_manifest,-)+from leann.cli import _normalize_path+from leann.sync import FileSynchronizerdef test_normalize_path():assert _normalize_path("") == ""assert _normalize_path("/a/b") == "/a/b" or "a" in _normalize_path("/a/b")- # Resolve so relative becomes absoluterel = "foo/bar"out = _normalize_path(rel)assert Path(out).is_absolute() or out == rel-def test_load_sources_manifest_missing(tmp_path):- assert load_sources_manifest(tmp_path, "documents.leann") == {}---def test_load_sources_manifest_empty(tmp_path):- (tmp_path / SOURCES_MANIFEST_FILENAME).write_text('{"sources": {}}', encoding="utf-8")- assert load_sources_manifest(tmp_path, "documents.leann") == {}---def test_save_and_load_sources_manifest(tmp_path):- sources = {"/path/to/a": 12345.0, "/path/to/b": 67890.0}- save_sources_manifest(tmp_path, "documents.leann", sources)- loaded = load_sources_manifest(tmp_path, "documents.leann")- assert loaded == sources- raw = (tmp_path / SOURCES_MANIFEST_FILENAME).read_text(encoding="utf-8")- data = json.loads(raw)- assert data.get("sources") == sources+def test_file_synchronizer_detect_changes(tmp_path):+ """FileSynchronizer detect_changes returns all files as added when no snapshot exists."""+ docs = tmp_path / "docs"+ docs.mkdir()+ (docs / "a.txt").write_text("hello", encoding="utf-8")+ snapshot = str(tmp_path / "test.pickle")+ fs = FileSynchronizer(root_dir=str(docs), snapshot_path=snapshot)+ added, removed, modified = fs.detect_changes()+ assert len(added) == 1+ assert len(removed) == 0+ assert len(modified) == 0+++def test_file_synchronizer_no_changes_after_commit(tmp_path):+ """After commit, detect_changes should report no changes if files haven't changed."""+ docs = tmp_path / "docs"+ docs.mkdir()+ (docs / "a.txt").write_text("hello", encoding="utf-8")+ snapshot = str(tmp_path / "test.pickle")+ fs = FileSynchronizer(root_dir=str(docs), snapshot_path=snapshot)+ fs.detect_changes()+ fs.commit()++ fs2 = FileSynchronizer(root_dir=str(docs), snapshot_path=snapshot)+ added, removed, modified = fs2.detect_changes()+ assert not added and not removed and not modified+++def test_file_synchronizer_detects_new_file(tmp_path):+ """Adding a file after commit should be detected as added."""+ docs = tmp_path / "docs"+ docs.mkdir()+ (docs / "a.txt").write_text("hello", encoding="utf-8")+ snapshot = str(tmp_path / "test.pickle")+ fs = FileSynchronizer(root_dir=str(docs), snapshot_path=snapshot)+ fs.detect_changes()+ fs.commit()++ (docs / "b.txt").write_text("world", encoding="utf-8")+ fs2 = FileSynchronizer(root_dir=str(docs), snapshot_path=snapshot)+ added, removed, modified = fs2.detect_changes()+ assert len(added) == 1+ assert len(removed) == 0+ assert len(modified) == 0+++def test_file_synchronizer_detects_modification(tmp_path):+ """Changing file content should be detected as modified."""+ docs = tmp_path / "docs"+ docs.mkdir()+ (docs / "a.txt").write_text("hello", encoding="utf-8")+ snapshot = str(tmp_path / "test.pickle")+ fs = FileSynchronizer(root_dir=str(docs), snapshot_path=snapshot)+ fs.detect_changes()+ fs.commit()++ (docs / "a.txt").write_text("changed", encoding="utf-8")+ fs2 = FileSynchronizer(root_dir=str(docs), snapshot_path=snapshot)+ added, removed, modified = fs2.detect_changes()+ assert len(modified) == 1+ assert len(added) == 0+++def test_file_synchronizer_touch_no_false_positive(tmp_path):+ """Touching a file (mtime change, same content) should NOT report as modified."""+ docs = tmp_path / "docs"+ docs.mkdir()+ f = docs / "a.txt"+ f.write_text("hello", encoding="utf-8")+ snapshot = str(tmp_path / "test.pickle")+ fs = FileSynchronizer(root_dir=str(docs), snapshot_path=snapshot)+ fs.detect_changes()+ fs.commit()++ os.utime(f, None)+ fs2 = FileSynchronizer(root_dir=str(docs), snapshot_path=snapshot)+ added, removed, modified = fs2.detect_changes()+ assert not added and not removed and not modified@pytest.mark.skipif(@@ -52,6 +108,9 @@ def test_save_and_load_sources_manifest(tmp_path):)def test_incremental_build_adds_only_new_files(tmp_path):"""Build once with one file, add a second file, run build again without --force; index grows."""+ import asyncio++ from leann.api import LeannSearcherfrom leann.cli import LeannCLIdocs_dir = tmp_path / "docs"@@ -59,7 +118,6 @@ def test_incremental_build_adds_only_new_files(tmp_path):(docs_dir / "a.txt").write_text("First document content for indexing.", encoding="utf-8")cli = LeannCLI()- # Override indexes_dir so we use tmp_path for indexcli.indexes_dir = tmp_path / ".leann" / "indexes"cli.indexes_dir.mkdir(parents=True, exist_ok=True)index_name = "incr_test"@@ -83,17 +141,9 @@ def test_incremental_build_adds_only_new_files(tmp_path):"--force",])- import asyncio-asyncio.run(cli.build_index(args))assert index_dir.exists()- meta_file = index_dir / "documents.leann.meta.json"- assert meta_file.exists()- manifest_file = index_dir / "documents.leann.sources.json"- assert manifest_file.exists(), "Full build should write sources manifest"- with open(manifest_file, encoding="utf-8") as f:- manifest = json.load(f)- assert len(manifest.get("sources", {})) == 1+ assert (index_dir / "documents.leann.meta.json").exists()# Add second file(docs_dir / "b.txt").write_text("Second document content.", encoding="utf-8")@@ -116,17 +166,77 @@ def test_incremental_build_adds_only_new_files(tmp_path):)asyncio.run(cli.build_index(args2))- with open(manifest_file, encoding="utf-8") as f:- manifest2 = json.load(f)- assert len(manifest2.get("sources", {})) == 2, (- "Manifest should list both files after incremental"- )-- # Index should still be searchable- from leann.api import LeannSearcher-+ # Index should still be searchable and contain both filessearcher = LeannSearcher(index_path)results = searcher.search("Second document", top_k=3)searcher.cleanup()assert len(results) >= 1assert "Second" in results[0].text or "document" in results[0].text+++ os.environ.get("CI") == "true",+ reason="Skip in CI to avoid embedding/model load",+)+def test_ivf_incremental_add_then_remove_searchable(tmp_path):+ """IVF: add content, search finds it; delete content, incremental build, search no longer finds it."""+ import asyncio++ from leann.api import LeannSearcher+ from leann.cli import LeannCLI++ docs_dir = tmp_path / "docs"+ docs_dir.mkdir()+ unique_phrase = "XYZZY_PLUGH_INCR_TEST_12345"+ (docs_dir / "f.txt").write_text(+ f"Initial content.\n\n{unique_phrase}\n\nMore text here.",+ encoding="utf-8",+ )+ # Add extra files so IVF has enough training points (nlist=100)+ for i in range(110):+ (docs_dir / f"filler_{i:03d}.txt").write_text(+ f"Filler document {i} with some content to create enough chunks.",+ encoding="utf-8",+ )++ cli = LeannCLI()+ cli.indexes_dir = tmp_path / ".leann" / "indexes"+ cli.indexes_dir.mkdir(parents=True, exist_ok=True)+ index_name = "ivf_incr_test"+ index_path = cli.get_index_path(index_name)++ parser = cli.create_parser()+ build_args = [+ "build",+ index_name,+ "--docs",+ str(docs_dir),+ "--backend-name",+ "ivf",+ "--embedding-model",+ "all-MiniLM-L6-v2",+ "--embedding-mode",+ "sentence-transformers",+ ]++ asyncio.run(cli.build_index(parser.parse_args([*build_args, "--force"])))++ searcher = LeannSearcher(index_path)+ results = searcher.search(unique_phrase, top_k=5)+ searcher.cleanup()+ assert len(results) >= 1, "Added content should be searchable"+ assert unique_phrase in results[0].text++ # Remove the unique phrase from the file+ (docs_dir / "f.txt").write_text("Initial content.\n\nMore text here.", encoding="utf-8")++ # Incremental build (IVF remove+add)+ asyncio.run(cli.build_index(parser.parse_args(build_args)))++ # Deleted content should no longer be searchable+ searcher = LeannSearcher(index_path)+ results = searcher.search(unique_phrase, top_k=5)+ searcher.cleanup()+ assert all(unique_phrase not in r.text for r in results), (+ "Deleted content should not appear in search results"+ )