Skip to content

armgit / LEANN

Public Imported from GitHub
0

[MLsys2026 Best Paper]: https://arxiv.org/abs/2506.08276. RAG on Everything with LEANN. Enjoy 97% storage savings while running a fast, accurate, and 100% private RAG application on your personal device.

ivf build wo rebuild (#273)

Yichuan Wang9890c35026ce
diff --git a/packages/leann-backend-ivf/leann_backend_ivf/ivf_backend.py b/packages/leann-backend-ivf/leann_backend_ivf/ivf_backend.py
index 0f7e69f..ca893da 100644
--- a/packages/leann-backend-ivf/leann_backend_ivf/ivf_backend.py
+++ b/packages/leann-backend-ivf/leann_backend_ivf/ivf_backend.py
@@ -1,8 +1,8 @@
"""
-IVF backend: FAISS IndexIVFFlat wrapped in IndexIDMap2 for stable add/delete by passage id.
+IVF backend: FAISS IndexIVFFlat with DirectMap.Hashtable for add/remove by passage id.
-Provides clean add_vectors() and remove_ids() APIs (aligned with HNSW update_index flow)
-so the core can do incremental updates: detect changed chunk ids, delete and re-insert.
+Uses IndexIVFFlat + DirectMap.Hashtable (no IndexIDMap2) so remove_ids works correctly.
+Provides add_vectors() and remove_ids() for incremental updates.
"""
import json
@@ -125,12 +125,12 @@ class IVFBuilder(LeannBackendBuilderInterface):
)
ivf = faiss.IndexIVFFlat(quantizer, dim, self.nlist, metric_enum)
ivf.train(data)
- index = faiss.IndexIDMap2(ivf)
+ ivf.set_direct_map_type(faiss.DirectMap.Hashtable)
faiss_ids = np.arange(n, dtype=np.int64)
- index.add_with_ids(data, faiss_ids)
+ ivf.add_with_ids(data, faiss_ids)
index_file = index_dir / f"{index_prefix}.index"
- faiss.write_index(index, str(index_file))
+ faiss.write_index(ivf, str(index_file))
id_to_passage = dict(enumerate(ids))
_save_id_map(index_dir, index_prefix, id_to_passage, next_id=n)
diff --git a/packages/leann-core/src/leann/api.py b/packages/leann-core/src/leann/api.py
index cde098c..743adc0 100644
--- a/packages/leann-core/src/leann/api.py
+++ b/packages/leann-core/src/leann/api.py
@@ -1,1581 +1,1587 @@
-"""
-This file contains the core API for the LEANN project, now definitively updated
-with the correct, original embedding logic from the user's reference code.
-"""
-
-import json
-import logging
-import os
-import pickle
-import re
-import subprocess
-import time
-import warnings
-from collections import Counter, defaultdict
-from dataclasses import dataclass, field
-from pathlib import Path
-from typing import Any, Literal, Optional, Union
-
-import numpy as np
-from leann_backend_hnsw.convert_to_csr import prune_hnsw_embeddings_inplace
-
-from leann.interactive_utils import create_api_session
-from leann.interface import LeannBackendSearcherInterface
-
-from .chat import get_llm
-from .embedding_server_manager import EmbeddingServerManager
-from .interface import LeannBackendFactoryInterface
-from .metadata_filter import MetadataFilterEngine
-from .registry import BACKEND_REGISTRY
-
-logger = logging.getLogger(__name__)
-
-
-def get_registered_backends() -> list[str]:
- """Get list of registered backend names."""
- return list(BACKEND_REGISTRY.keys())
-
-
-def compute_embeddings(
- chunks: list[str],
- model_name: str,
- mode: str = "sentence-transformers",
- use_server: bool = True,
- port: Optional[int] = None,
- is_build=False,
- provider_options: Optional[dict[str, Any]] = None,
-) -> np.ndarray:
- """
- Computes embeddings using different backends.
-
- Args:
- chunks: List of text chunks to embed
- model_name: Name of the embedding model
- mode: Embedding backend mode. Options:
- - "sentence-transformers": Use sentence-transformers library (default)
- - "mlx": Use MLX backend for Apple Silicon
- - "openai": Use OpenAI embedding API
- - "gemini": Use Google Gemini embedding API
- use_server: Whether to use embedding server (True for search, False for build)
-
- Returns:
- numpy array of embeddings
- """
- if use_server:
- # Use embedding server (for search/query)
- if port is None:
- raise ValueError("port is required when use_server is True")
- return compute_embeddings_via_server(chunks, model_name, port=port)
- else:
- # Use direct computation (for build_index)
- from .embedding_compute import (
- compute_embeddings as compute_embeddings_direct,
- )
-
- return compute_embeddings_direct(
- chunks,
- model_name,
- mode=mode,
- is_build=is_build,
- provider_options=provider_options,
- )
-
-
-def compute_embeddings_via_server(chunks: list[str], model_name: str, port: int) -> np.ndarray:
- """Computes embeddings using sentence-transformers.
-
- Args:
- chunks: List of text chunks to embed
- model_name: Name of the sentence transformer model
- """
- logger.info(
- f"Computing embeddings for {len(chunks)} chunks using SentenceTransformer model '{model_name}' (via embedding server)..."
- )
- import msgpack
- import numpy as np
- import zmq
-
- # Connect to embedding server
- context = zmq.Context()
- socket = context.socket(zmq.REQ)
- socket.connect(f"tcp://localhost:{port}")
-
- # Send chunks to server for embedding computation
- request = chunks
- socket.send(msgpack.packb(request))
-
- # Receive embeddings from server
- response = socket.recv()
- embeddings_list = msgpack.unpackb(response)
-
- # Convert back to numpy array
- embeddings = np.array(embeddings_list, dtype=np.float32)
-
- socket.close()
- context.term()
-
- return embeddings
-
-
-@dataclass
-class SearchResult:
- id: str
- score: float
- text: str
- metadata: dict[str, Any] = field(default_factory=dict)
-
-
-class PassageManager:
- def __init__(
- self, passage_sources: list[dict[str, Any]], metadata_file_path: Optional[str] = None
- ):
- self.offset_maps: dict[str, dict[str, int]] = {}
- self.passage_files: dict[str, str] = {}
- # Avoid materializing a single gigantic global map to reduce memory
- # footprint on very large corpora (e.g., 60M+ passages). Instead, keep
- # per-shard maps and do a lightweight per-shard lookup on demand.
- self._total_count: int = 0
- self.filter_engine = MetadataFilterEngine() # Initialize filter engine
-
- # Derive index base name for standard sibling fallbacks, e.g., <index_name>.passages.*
- index_name_base = None
- if metadata_file_path:
- meta_name = Path(metadata_file_path).name
- if meta_name.endswith(".meta.json"):
- index_name_base = meta_name[: -len(".meta.json")]
-
- for source in passage_sources:
- assert source["type"] == "jsonl", "only jsonl is supported"
- passage_file = source.get("path", "")
- index_file = source.get("index_path", "") # .idx file
-
- # Fix path resolution - relative paths should be relative to metadata file directory
- def _resolve_candidates(
- primary: str,
- relative_key: str,
- default_name: Optional[str],
- source_dict: dict[str, Any],
- ) -> list[Path]:
- """
- Build an ordered list of candidate paths. For relative paths specified in
- metadata, prefer resolution relative to the metadata file directory first,
- then fall back to CWD-based resolution, and finally to conventional
- sibling defaults (e.g., <index_base>.passages.idx / .jsonl).
- """
- candidates: list[Path] = []
- # 1) Primary path
- if primary:
- p = Path(primary)
- if p.is_absolute():
- candidates.append(p)
- else:
- # Prefer metadata-relative resolution for relative paths
- if metadata_file_path:
- candidates.append(Path(metadata_file_path).parent / p)
- # Also consider CWD-relative as a fallback for legacy layouts
- candidates.append(Path.cwd() / p)
- # 2) metadata-relative explicit relative key (if present)
- if metadata_file_path and source_dict.get(relative_key):
- candidates.append(Path(metadata_file_path).parent / source_dict[relative_key])
- # 3) metadata-relative standard sibling filename
- if metadata_file_path and default_name:
- candidates.append(Path(metadata_file_path).parent / default_name)
- return candidates
-
- # Build candidate lists and pick first existing; otherwise keep last candidate for error message
- idx_default = f"{index_name_base}.passages.idx" if index_name_base else None
- idx_candidates = _resolve_candidates(
- index_file, "index_path_relative", idx_default, source
- )
- pas_default = f"{index_name_base}.passages.jsonl" if index_name_base else None
- pas_candidates = _resolve_candidates(passage_file, "path_relative", pas_default, source)
-
- def _pick_existing(cands: list[Path]) -> str:
- for c in cands:
- if c.exists():
- return str(c.resolve())
- # Fallback to last candidate (best guess) even if not exists; will error below
- return str(cands[-1].resolve()) if cands else ""
-
- index_file = _pick_existing(idx_candidates)
- passage_file = _pick_existing(pas_candidates)
-
- if not Path(index_file).exists():
- raise FileNotFoundError(f"Passage index file not found: {index_file}")
-
- with open(index_file, "rb") as f:
- offset_map: dict[str, int] = pickle.load(f)
- self.offset_maps[passage_file] = offset_map
- self.passage_files[passage_file] = passage_file
- self._total_count += len(offset_map)
-
- def get_passage(self, passage_id: str) -> dict[str, Any]:
- # Fast path: check each shard map (there are typically few shards).
- # This avoids building a massive combined dict while keeping lookups
- # bounded by the number of shards.
- for passage_file, offset_map in self.offset_maps.items():
- try:
- offset = offset_map[passage_id]
- with open(passage_file, encoding="utf-8") as f:
- f.seek(offset)
- return json.loads(f.readline())
- except KeyError:
- continue
- raise KeyError(f"Passage ID not found: {passage_id}")
-
- def filter_search_results(
- self,
- search_results: list[SearchResult],
- metadata_filters: Optional[dict[str, dict[str, Union[str, int, float, bool, list]]]],
- ) -> list[SearchResult]:
- """
- Apply metadata filters to search results.
-
- Args:
- search_results: List of SearchResult objects
- metadata_filters: Filter specifications to apply
-
- Returns:
- Filtered list of SearchResult objects
- """
- if not metadata_filters:
- return search_results
-
- logger.debug(f"Applying metadata filters to {len(search_results)} results")
-
- # Convert SearchResult objects to dictionaries for the filter engine
- result_dicts = []
- for result in search_results:
- result_dicts.append(
- {
- "id": result.id,
- "score": result.score,
- "text": result.text,
- "metadata": result.metadata,
- }
- )
-
- # Apply filters using the filter engine
- filtered_dicts = self.filter_engine.apply_filters(result_dicts, metadata_filters)
-
- # Convert back to SearchResult objects
- filtered_results = []
- for result_dict in filtered_dicts:
- filtered_results.append(
- SearchResult(
- id=result_dict["id"],
- score=result_dict["score"],
- text=result_dict["text"],
- metadata=result_dict["metadata"],
- )
- )
-
- logger.debug(f"Filtered results: {len(filtered_results)} remaining")
- return filtered_results
-
- def __len__(self) -> int:
- return self._total_count
-
-
-class BM25Scorer:
- def __init__(self, k1: float = 1.2, b: float = 0.75):
- self.k1 = k1
- self.b = b
- self.doc_freqs = None # How many docs contain each term (DF)
- self.doc_lengths = {} # How long each doc is (in words)
- self.word_counts = {} # How many times each word appears in each doc (TF)
- self.avg_doc_length = None
- self.corpus_size = None
- self.idlist = set() # List of all document IDs for easier searching
-
- def _tokenize(self, text: str) -> list[str]:
- return re.sub(r"[^\w\s]", "", text).lower().split()
-
- def fit(self, documents: list[dict[str, Any]]):
- """
- Build BM25 statistics from a document corpus.
- Must be called before scoring.
- """
- self.corpus_size = len(documents)
- self.doc_lengths = {}
- self.word_counts = {}
- self.idlist = set()
- doc_freqs = defaultdict(int)
-
- for doc_data in documents:
- doc_id = doc_data["id"]
- words = self._tokenize(doc_data["text"])
- doc_length = len(words)
- self.doc_lengths[doc_id] = doc_length
-
- unique_words = set(words)
- for word in unique_words:
- doc_freqs[word] += 1
- self.word_counts[doc_id] = dict(Counter(words))
- self.idlist.add(doc_id)
-
- self.doc_freqs = dict(doc_freqs)
- self.avg_doc_length = sum(self.doc_lengths.values()) / len(self.doc_lengths)
-
- def score(self, query_words: list[str], document_id: str) -> float:
- if (
- self.doc_freqs is None
- or self.doc_lengths == {}
- or self.word_counts == {}
- or self.avg_doc_length is None
- or self.corpus_size is None
- ):
- raise ValueError("BM25 model not fitted. Call fit() before scoring.")
-
- passage_words = self.word_counts[document_id]
- passage_length = sum(passage_words.values())
- score = 0.0
- for word in query_words:
- if word not in self.doc_freqs:
- continue
- word_freq = passage_words[word] if word in passage_words else 0
- idf = np.log(
- (self.corpus_size - self.doc_freqs[word] + 0.5) / (self.doc_freqs[word] + 0.5) + 1
- )
- tf = (word_freq * (self.k1 + 1)) / (
- word_freq + self.k1 * (1 - self.b + self.b * (passage_length / self.avg_doc_length))
- )
- score += idf * tf
- return score
-
- def search(self, query: str, top_k: int = 5) -> list[SearchResult]:
- query_words = self._tokenize(query)
- scores = {doc_id: self.score(query_words, doc_id) for doc_id in self.idlist}
- sorted_scores = sorted(scores.items(), key=lambda x: x[1], reverse=True)
- return [
- SearchResult(id=doc_id, score=score, text="", metadata={})
- for doc_id, score in sorted_scores[:top_k]
- ]
-
-
-class LeannBuilder:
- def __init__(
- self,
- backend_name: str,
- embedding_model: str = "facebook/contriever",
- dimensions: Optional[int] = None,
- embedding_mode: str = "sentence-transformers",
- embedding_options: Optional[dict[str, Any]] = None,
- **backend_kwargs,
- ):
- self.backend_name = backend_name
- # Normalize incompatible combinations early (for consistent metadata)
- if backend_name == "hnsw":
- is_recompute = backend_kwargs.get("is_recompute", True)
- is_compact = backend_kwargs.get("is_compact", True)
- if is_recompute is False and is_compact is True:
- warnings.warn(
- "HNSW with is_recompute=False requires non-compact storage. Forcing is_compact=False.",
- UserWarning,
- stacklevel=2,
- )
- backend_kwargs["is_compact"] = False
-
- backend_factory: Optional[LeannBackendFactoryInterface] = BACKEND_REGISTRY.get(backend_name)
- if backend_factory is None:
- raise ValueError(f"Backend '{backend_name}' not found or not registered.")
- self.backend_factory = backend_factory
- self.embedding_model = embedding_model
- self.dimensions = dimensions
- self.embedding_mode = embedding_mode
- self.embedding_options = embedding_options or {}
-
- # Check if we need to use cosine distance for normalized embeddings
- normalized_embeddings_models = {
- # OpenAI models
- ("openai", "text-embedding-ada-002"),
- ("openai", "text-embedding-3-small"),
- ("openai", "text-embedding-3-large"),
- # Voyage AI models
- ("voyage", "voyage-2"),
- ("voyage", "voyage-3"),
- ("voyage", "voyage-large-2"),
- ("voyage", "voyage-multilingual-2"),
- ("voyage", "voyage-code-2"),
- # Cohere models
- ("cohere", "embed-english-v3.0"),
- ("cohere", "embed-multilingual-v3.0"),
- ("cohere", "embed-english-light-v3.0"),
- ("cohere", "embed-multilingual-light-v3.0"),
- }
-
- # Also check for patterns in model names
- is_normalized = False
- current_model_lower = embedding_model.lower()
- current_mode_lower = embedding_mode.lower()
-
- # Check exact matches
- for mode, model in normalized_embeddings_models:
- if (current_mode_lower == mode and current_model_lower == model) or (
- mode in current_mode_lower and model in current_model_lower
- ):
- is_normalized = True
- break
-
- # Check patterns
- if not is_normalized:
- # OpenAI patterns
- if "openai" in current_mode_lower or "openai" in current_model_lower:
- if any(
- pattern in current_model_lower
- for pattern in ["text-embedding", "ada", "3-small", "3-large"]
- ):
- is_normalized = True
- # Voyage patterns
- elif "voyage" in current_mode_lower or "voyage" in current_model_lower:
- is_normalized = True
- # Cohere patterns
- elif "cohere" in current_mode_lower or "cohere" in current_model_lower:
- if "embed" in current_model_lower:
- is_normalized = True
-
- # Handle distance metric
- if is_normalized and "distance_metric" not in backend_kwargs:
- backend_kwargs["distance_metric"] = "cosine"
- warnings.warn(
- f"Detected normalized embeddings model '{embedding_model}' with mode '{embedding_mode}'. "
- f"Automatically setting distance_metric='cosine' for optimal performance. "
- f"Normalized embeddings (L2 norm = 1) should use cosine similarity instead of MIPS.",
- UserWarning,
- stacklevel=2,
- )
- elif is_normalized and backend_kwargs.get("distance_metric", "").lower() != "cosine":
- current_metric = backend_kwargs.get("distance_metric", "mips")
- warnings.warn(
- f"Warning: Using '{current_metric}' distance metric with normalized embeddings model "
- f"'{embedding_model}' may lead to suboptimal search results. "
- f"Consider using 'cosine' distance metric for better performance.",
- UserWarning,
- stacklevel=2,
- )
-
- self.backend_kwargs = backend_kwargs
- self.chunks: list[dict[str, Any]] = []
-
- def add_text(self, text: str, metadata: Optional[dict[str, Any]] = None):
- if metadata is None:
- metadata = {}
- passage_id = metadata.get("id", str(len(self.chunks)))
- chunk_data = {"id": passage_id, "text": text, "metadata": metadata}
- self.chunks.append(chunk_data)
-
- def build_index(self, index_path: str):
- if not self.chunks:
- raise ValueError("No chunks added.")
-
- # Filter out invalid/empty text chunks early to keep passage and embedding counts aligned
- valid_chunks: list[dict[str, Any]] = []
- skipped = 0
- for chunk in self.chunks:
- text = chunk.get("text", "")
- if isinstance(text, str) and text.strip():
- valid_chunks.append(chunk)
- else:
- skipped += 1
- if skipped > 0:
- print(
- f"Warning: Skipping {skipped} empty/invalid text chunk(s). Processing {len(valid_chunks)} valid chunks"
- )
- self.chunks = valid_chunks
- if not self.chunks:
- raise ValueError("All provided chunks are empty or invalid. Nothing to index.")
- if self.dimensions is None:
- self.dimensions = len(
- compute_embeddings(
- ["dummy"],
- self.embedding_model,
- self.embedding_mode,
- use_server=False,
- provider_options=self.embedding_options,
- )[0]
- )
- path = Path(index_path)
- index_dir = path.parent
- index_name = path.name
- index_dir.mkdir(parents=True, exist_ok=True)
- passages_file = index_dir / f"{index_name}.passages.jsonl"
- offset_file = index_dir / f"{index_name}.passages.idx"
- offset_map = {}
- with open(passages_file, "w", encoding="utf-8") as f:
- try:
- from tqdm import tqdm
-
- chunk_iterator = tqdm(self.chunks, desc="Writing passages", unit="chunk")
- except ImportError:
- chunk_iterator = self.chunks
-
- for chunk in chunk_iterator:
- offset = f.tell()
- json.dump(
- {
- "id": chunk["id"],
- "text": chunk["text"],
- "metadata": chunk["metadata"],
- },
- f,
- ensure_ascii=False,
- )
- f.write("\n")
- offset_map[chunk["id"]] = offset
- with open(offset_file, "wb") as f:
- pickle.dump(offset_map, f)
- texts_to_embed = [c["text"] for c in self.chunks]
- embeddings = compute_embeddings(
- texts_to_embed,
- self.embedding_model,
- self.embedding_mode,
- use_server=False,
- is_build=True,
- provider_options=self.embedding_options,
- )
- string_ids = [chunk["id"] for chunk in self.chunks]
- # Persist ID map alongside index so backends that return integer labels can remap to passage IDs
- try:
- idmap_file = (
- index_dir
- / f"{index_name[: -len('.leann')] if index_name.endswith('.leann') else index_name}.ids.txt"
- )
- with open(idmap_file, "w", encoding="utf-8") as f:
- for sid in string_ids:
- f.write(str(sid) + "\n")
- except Exception:
- pass
- current_backend_kwargs = {**self.backend_kwargs, "dimensions": self.dimensions}
- builder_instance = self.backend_factory.builder(**current_backend_kwargs)
- builder_instance.build(embeddings, string_ids, index_path, **current_backend_kwargs)
- leann_meta_path = index_dir / f"{index_name}.meta.json"
- meta_data = {
- "version": "1.0",
- "backend_name": self.backend_name,
- "embedding_model": self.embedding_model,
- "dimensions": self.dimensions,
- "backend_kwargs": self.backend_kwargs,
- "embedding_mode": self.embedding_mode,
- "passage_sources": [
- {
- "type": "jsonl",
- # Preserve existing relative file names (backward-compatible)
- "path": passages_file.name,
- "index_path": offset_file.name,
- # Add optional redundant relative keys for remote build portability (non-breaking)
- "path_relative": passages_file.name,
- "index_path_relative": offset_file.name,
- }
- ],
- }
-
- if self.embedding_options:
- meta_data["embedding_options"] = self.embedding_options
-
- # Add storage status flags for HNSW backend
- if self.backend_name == "hnsw":
- is_compact = self.backend_kwargs.get("is_compact", True)
- is_recompute = self.backend_kwargs.get("is_recompute", True)
- meta_data["is_compact"] = is_compact
- meta_data["is_pruned"] = bool(is_recompute)
- with open(leann_meta_path, "w", encoding="utf-8") as f:
- json.dump(meta_data, f, indent=2)
-
- def build_index_from_embeddings(self, index_path: str, embeddings_file: str):
- """
- Build an index from pre-computed embeddings stored in a pickle file.
-
- Args:
- index_path: Path where the index will be saved
- embeddings_file: Path to pickle file containing (ids, embeddings) tuple
- """
- # Load pre-computed embeddings
- with open(embeddings_file, "rb") as f:
- data = pickle.load(f)
-
- if not isinstance(data, tuple) or len(data) != 2:
- raise ValueError(
- f"Invalid embeddings file format. Expected tuple with 2 elements, got {type(data)}"
- )
-
- ids, embeddings = data
-
- if not isinstance(embeddings, np.ndarray):
- raise ValueError(f"Expected embeddings to be numpy array, got {type(embeddings)}")
-
- if len(ids) != embeddings.shape[0]:
- raise ValueError(
- f"Mismatch between number of IDs ({len(ids)}) and embeddings ({embeddings.shape[0]})"
- )
-
- # Validate/set dimensions
- embedding_dim = embeddings.shape[1]
- if self.dimensions is None:
- self.dimensions = embedding_dim
- elif self.dimensions != embedding_dim:
- raise ValueError(f"Dimension mismatch: expected {self.dimensions}, got {embedding_dim}")
-
- logger.info(
- f"Building index from precomputed embeddings: {len(ids)} items, {embedding_dim} dimensions"
- )
-
- # Ensure we have text data for each embedding
- if len(self.chunks) != len(ids):
- # If no text chunks provided, create placeholder text entries
- if not self.chunks:
- logger.info("No text chunks provided, creating placeholder entries...")
- for id_val in ids:
- self.add_text(
- f"Document {id_val}",
- metadata={"id": str(id_val), "from_embeddings": True},
- )
- else:
- raise ValueError(
- f"Number of text chunks ({len(self.chunks)}) doesn't match number of embeddings ({len(ids)})"
- )
-
- # Build file structure
- path = Path(index_path)
- index_dir = path.parent
- index_name = path.name
- index_dir.mkdir(parents=True, exist_ok=True)
- passages_file = index_dir / f"{index_name}.passages.jsonl"
- offset_file = index_dir / f"{index_name}.passages.idx"
-
- # Write passages and create offset map
- offset_map = {}
- with open(passages_file, "w", encoding="utf-8") as f:
- for chunk in self.chunks:
- offset = f.tell()
- json.dump(
- {
- "id": chunk["id"],
- "text": chunk["text"],
- "metadata": chunk["metadata"],
- },
- f,
- ensure_ascii=False,
- )
- f.write("\n")
- offset_map[chunk["id"]] = offset
-
- with open(offset_file, "wb") as f:
- pickle.dump(offset_map, f)
-
- # Build the vector index using precomputed embeddings
- string_ids = [str(id_val) for id_val in ids]
- # Persist ID map (order == embeddings order)
- try:
- idmap_file = (
- index_dir
- / f"{index_name[: -len('.leann')] if index_name.endswith('.leann') else index_name}.ids.txt"
- )
- with open(idmap_file, "w", encoding="utf-8") as f:
- for sid in string_ids:
- f.write(str(sid) + "\n")
- except Exception:
- pass
- current_backend_kwargs = {**self.backend_kwargs, "dimensions": self.dimensions}
- builder_instance = self.backend_factory.builder(**current_backend_kwargs)
- builder_instance.build(embeddings, string_ids, index_path)
-
- # Create metadata file
- leann_meta_path = index_dir / f"{index_name}.meta.json"
- meta_data = {
- "version": "1.0",
- "backend_name": self.backend_name,
- "embedding_model": self.embedding_model,
- "dimensions": self.dimensions,
- "backend_kwargs": self.backend_kwargs,
- "embedding_mode": self.embedding_mode,
- "passage_sources": [
- {
- "type": "jsonl",
- # Preserve existing relative file names (backward-compatible)
- "path": passages_file.name,
- "index_path": offset_file.name,
- # Add optional redundant relative keys for remote build portability (non-breaking)
- "path_relative": passages_file.name,
- "index_path_relative": offset_file.name,
- }
- ],
- "built_from_precomputed_embeddings": True,
- "embeddings_source": str(embeddings_file),
- }
-
- if self.embedding_options:
- meta_data["embedding_options"] = self.embedding_options
-
- # Add storage status flags for HNSW backend
- if self.backend_name == "hnsw":
- is_compact = self.backend_kwargs.get("is_compact", True)
- is_recompute = self.backend_kwargs.get("is_recompute", True)
- meta_data["is_compact"] = is_compact
- meta_data["is_pruned"] = bool(is_recompute)
-
- with open(leann_meta_path, "w", encoding="utf-8") as f:
- json.dump(meta_data, f, indent=2)
-
- logger.info(f"Index built successfully from precomputed embeddings: {index_path}")
-
- def update_index(self, index_path: str, remove_passage_ids: Optional[list[str]] = None) -> None:
- """Append new passages and vectors to an existing index (HNSW or IVF).
- For IVF, optional remove_passage_ids removes those ids first (e.g. from file-change API).
- """
- if not self.chunks and not remove_passage_ids:
- raise ValueError("No new chunks or passage ids to remove provided for update.")
-
- path = Path(index_path)
- index_dir = path.parent
- index_name = path.name
- index_prefix = path.stem
-
- meta_path = index_dir / f"{index_name}.meta.json"
- passages_file = index_dir / f"{index_name}.passages.jsonl"
- offset_file = index_dir / f"{index_name}.passages.idx"
- index_file = index_dir / f"{index_prefix}.index"
-
- if not meta_path.exists() or not passages_file.exists() or not offset_file.exists():
- raise FileNotFoundError("Index metadata or passage files are missing; cannot update.")
- if not index_file.exists():
- raise FileNotFoundError(f"Index file not found: {index_file}")
-
- with open(meta_path, encoding="utf-8") as f:
- meta = json.load(f)
- backend_name = meta.get("backend_name")
- if backend_name != self.backend_name:
- raise ValueError(
- f"Index was built with backend '{backend_name}', cannot update with '{self.backend_name}'."
- )
-
- with open(offset_file, "rb") as f:
- offset_map: dict[str, int] = pickle.load(f)
- existing_ids = set(offset_map.keys())
-
- # IVF: optional delete (for reindex / file-change: remove then re-insert)
- if remove_passage_ids and backend_name == "ivf":
- try:
- from leann_backend_ivf import remove_ids as ivf_remove_ids
-
- ivf_remove_ids(str(path), remove_passage_ids)
- except ImportError:
- raise RuntimeError(
- "IVF backend required for remove_ids. Install leann-backend-ivf."
- )
- for pid in remove_passage_ids:
- offset_map.pop(pid, None)
- with open(offset_file, "wb") as f:
- pickle.dump(offset_map, f)
-
- if not self.chunks:
- meta["total_passages"] = len(offset_map)
- with open(meta_path, "w", encoding="utf-8") as f:
- json.dump(meta, f, indent=2)
- self.chunks.clear()
- return
-
- meta_backend_kwargs = meta.get("backend_kwargs", {})
- if backend_name == "hnsw":
- index_is_compact = meta.get("is_compact", meta_backend_kwargs.get("is_compact", True))
- if index_is_compact:
- raise ValueError(
- "Compact HNSW indices do not support in-place updates. Rebuild required."
- )
-
- distance_metric = meta_backend_kwargs.get(
- "distance_metric", self.backend_kwargs.get("distance_metric", "mips")
- ).lower()
- needs_recompute = bool(
- meta.get("is_pruned")
- or meta_backend_kwargs.get("is_recompute")
- or self.backend_kwargs.get("is_recompute")
- )
-
- valid_chunks: list[dict[str, Any]] = []
- for chunk in self.chunks:
- text = chunk.get("text", "")
- if not isinstance(text, str) or not text.strip():
- continue
- metadata = chunk.setdefault("metadata", {})
- passage_id = chunk.get("id") or metadata.get("id")
- if passage_id and passage_id in existing_ids:
- raise ValueError(f"Passage ID '{passage_id}' already exists in the index.")
- valid_chunks.append(chunk)
-
- if not valid_chunks:
- raise ValueError("No valid chunks to append.")
-
- texts_to_embed = [chunk["text"] for chunk in valid_chunks]
- embeddings = compute_embeddings(
- texts_to_embed,
- self.embedding_model,
- self.embedding_mode,
- use_server=False,
- is_build=True,
- provider_options=self.embedding_options,
- )
-
- embedding_dim = embeddings.shape[1]
- expected_dim = meta.get("dimensions")
- if expected_dim is not None and expected_dim != embedding_dim:
- raise ValueError(
- f"Dimension mismatch during update: existing index uses {expected_dim}, got {embedding_dim}."
- )
-
- embeddings = np.ascontiguousarray(embeddings, dtype=np.float32)
- if distance_metric == "cosine":
- norms = np.linalg.norm(embeddings, axis=1, keepdims=True)
- norms[norms == 0] = 1
- embeddings = embeddings / norms
-
- # IVF: add_vectors then append passages/offset (no ZMQ/server)
- if backend_name == "ivf":
- for i, chunk in enumerate(valid_chunks):
- pid = chunk.get("id") or chunk.get("metadata", {}).get("id")
- if not pid:
- pid = str(len(offset_map) + i)
- chunk.setdefault("metadata", {})["id"] = pid
- chunk["id"] = pid
- passage_ids = [c["id"] for c in valid_chunks]
- try:
- from leann_backend_ivf import add_vectors as ivf_add_vectors
-
- ivf_add_vectors(str(path), embeddings, passage_ids)
- except ImportError:
- raise RuntimeError("IVF backend required. Install leann-backend-ivf.")
- rollback_passages_size = passages_file.stat().st_size if passages_file.exists() else 0
- offset_map_backup = offset_map.copy()
- try:
- with open(passages_file, "a", encoding="utf-8") as f:
- for chunk in valid_chunks:
- off = f.tell()
- json.dump(
- {
- "id": chunk["id"],
- "text": chunk["text"],
- "metadata": chunk.get("metadata", {}),
- },
- f,
- ensure_ascii=False,
- )
- f.write("\n")
- offset_map[chunk["id"]] = off
- with open(offset_file, "wb") as f:
- pickle.dump(offset_map, f)
- meta["total_passages"] = len(offset_map)
- with open(meta_path, "w", encoding="utf-8") as f:
- json.dump(meta, f, indent=2)
- logger.info(
- "Appended %d passages to IVF index '%s'. Total: %d",
- len(valid_chunks),
- index_path,
- len(offset_map),
- )
- except Exception:
- if passages_file.exists():
- with open(passages_file, "rb+") as f:
- f.truncate(rollback_passages_size)
- offset_map = offset_map_backup
- with open(offset_file, "wb") as f:
- pickle.dump(offset_map, f)
- raise
- self.chunks.clear()
- return
-
- # HNSW path below
- from leann_backend_hnsw import faiss
-
- index = faiss.read_index(str(index_file))
- if hasattr(index, "is_recompute"):
- index.is_recompute = needs_recompute
- print(f"index.is_recompute: {index.is_recompute}")
- if getattr(index, "storage", None) is None:
- if index.metric_type == faiss.METRIC_INNER_PRODUCT:
- storage_index = faiss.IndexFlatIP(index.d)
- else:
- storage_index = faiss.IndexFlatL2(index.d)
- index.storage = storage_index
- index.own_fields = True
- # Faiss expects storage.ntotal to reflect the existing graph's
- # population (even if the vectors themselves were pruned from disk
- # for recompute mode). When we attach a fresh IndexFlat here its
- # ntotal starts at zero, which later causes IndexHNSW::add to
- # believe new "preset" levels were provided and trips the
- # `n0 + n == levels.size()` assertion. Seed the temporary storage
- # with the current ntotal so Faiss maintains the proper offset for
- # incoming vectors.
- try:
- storage_index.ntotal = index.ntotal
- except AttributeError:
- # Older Faiss builds may not expose ntotal as a writable
- # attribute; in that case we fall back to the default behaviour.
- pass
- if index.d != embedding_dim:
- raise ValueError(
- f"Existing index dimension ({index.d}) does not match new embeddings ({embedding_dim})."
- )
-
- passage_meta_mode = meta.get("embedding_mode", self.embedding_mode)
- passage_provider_options = meta.get("embedding_options", self.embedding_options)
-
- base_id = index.ntotal
- for offset, chunk in enumerate(valid_chunks):
- new_id = str(base_id + offset)
- chunk.setdefault("metadata", {})["id"] = new_id
- chunk["id"] = new_id
-
- # Append passages/offsets before we attempt index.add so the ZMQ server
- # can resolve newly assigned IDs during recompute. Keep rollback hooks
- # so we can restore files if the update fails mid-way.
- rollback_passages_size = passages_file.stat().st_size if passages_file.exists() else 0
- offset_map_backup = offset_map.copy()
-
- try:
- with open(passages_file, "a", encoding="utf-8") as f:
- for chunk in valid_chunks:
- offset = f.tell()
- json.dump(
- {
- "id": chunk["id"],
- "text": chunk["text"],
- "metadata": chunk.get("metadata", {}),
- },
- f,
- ensure_ascii=False,
- )
- f.write("\n")
- offset_map[chunk["id"]] = offset
-
- with open(offset_file, "wb") as f:
- pickle.dump(offset_map, f)
-
- server_manager: Optional[EmbeddingServerManager] = None
- server_started = False
- requested_zmq_port = int(os.getenv("LEANN_UPDATE_ZMQ_PORT", "5557"))
-
- try:
- if needs_recompute:
- server_manager = EmbeddingServerManager(
- backend_module_name="leann_backend_hnsw.hnsw_embedding_server"
- )
- server_started, actual_port = server_manager.start_server(
- port=requested_zmq_port,
- model_name=self.embedding_model,
- embedding_mode=passage_meta_mode,
- passages_file=str(meta_path),
- distance_metric=distance_metric,
- use_daemon=False,
- enable_warmup=False,
- provider_options=passage_provider_options,
- )
- if not server_started:
- raise RuntimeError(
- "Failed to start HNSW embedding server for recompute update."
- )
- if actual_port != requested_zmq_port:
- logger.warning(
- "Embedding server started on port %s instead of requested %s. "
- "Using reassigned port.",
- actual_port,
- requested_zmq_port,
- )
- if hasattr(index.hnsw, "set_zmq_port"):
- index.hnsw.set_zmq_port(actual_port)
- elif hasattr(index, "set_zmq_port"):
- index.set_zmq_port(actual_port)
-
- if needs_recompute:
- for i in range(embeddings.shape[0]):
- print(f"add {i} embeddings")
- index.add(1, faiss.swig_ptr(embeddings[i : i + 1]))
- else:
- index.add(embeddings.shape[0], faiss.swig_ptr(embeddings))
- faiss.write_index(index, str(index_file))
- finally:
- if server_started and server_manager is not None:
- server_manager.stop_server()
-
- except Exception:
- # Roll back appended passages/offset map to keep files consistent.
- if passages_file.exists():
- with open(passages_file, "rb+") as f:
- f.truncate(rollback_passages_size)
- offset_map = offset_map_backup
- with open(offset_file, "wb") as f:
- pickle.dump(offset_map, f)
- raise
-
- meta["total_passages"] = len(offset_map)
- with open(meta_path, "w", encoding="utf-8") as f:
- json.dump(meta, f, indent=2)
-
- logger.info(
- "Appended %d passages to index '%s'. New total: %d",
- len(valid_chunks),
- index_path,
- len(offset_map),
- )
-
- self.chunks.clear()
-
- if needs_recompute:
- prune_hnsw_embeddings_inplace(str(index_file))
-
-
-class LeannSearcher:
- def __init__(
- self,
- index_path: str,
- enable_warmup: bool = True,
- recompute_embeddings: bool = True,
- use_daemon: bool = True,
- daemon_ttl_seconds: int = 900,
- **backend_kwargs,
- ):
- # Fix path resolution for Colab and other environments
- if not Path(index_path).is_absolute():
- index_path = str(Path(index_path).resolve())
-
- self.meta_path_str = f"{index_path}.meta.json"
- if not Path(self.meta_path_str).exists():
- parent_dir = Path(index_path).parent
- print(
- f"Leann metadata file not found at {self.meta_path_str}, and you may need to rm -rf {parent_dir}"
- )
- # highlight in red the filenotfound error
- raise FileNotFoundError(
- f"Leann metadata file not found at {self.meta_path_str}, \033[91m you may need to rm -rf {parent_dir}\033[0m"
- )
- with open(self.meta_path_str, encoding="utf-8") as f:
- self.meta_data = json.load(f)
- backend_name = self.meta_data["backend_name"]
- self.embedding_model = self.meta_data["embedding_model"]
- # Support both old and new format
- self.embedding_mode = self.meta_data.get("embedding_mode", "sentence-transformers")
- self.embedding_options = self.meta_data.get("embedding_options", {})
- # Delegate portability handling to PassageManager
- self.passage_manager = PassageManager(
- self.meta_data.get("passage_sources", []), metadata_file_path=self.meta_path_str
- )
- # Preserve backend name for conditional parameter forwarding
- self.backend_name = backend_name
- backend_factory = BACKEND_REGISTRY.get(backend_name)
- if backend_factory is None:
- raise ValueError(f"Backend '{backend_name}' not found.")
-
- # Global recompute flag for this searcher (explicit knob, default True)
- self.recompute_embeddings: bool = bool(recompute_embeddings)
-
- # Warmup flag: keep using the existing enable_warmup parameter,
- # but default it to True so cold-start happens earlier.
- self._warmup: bool = bool(enable_warmup)
- self._use_daemon: bool = bool(use_daemon)
- self._daemon_ttl_seconds: int = int(daemon_ttl_seconds)
-
- final_kwargs = {**self.meta_data.get("backend_kwargs", {}), **backend_kwargs}
- final_kwargs["enable_warmup"] = self._warmup
- final_kwargs["use_daemon"] = self._use_daemon
- final_kwargs["daemon_ttl_seconds"] = self._daemon_ttl_seconds
- if self.embedding_options:
- final_kwargs.setdefault("embedding_options", self.embedding_options)
- self.backend_impl: LeannBackendSearcherInterface = backend_factory.searcher(
- index_path, **final_kwargs
- )
- self.bm25_scorer: Optional[BM25Scorer] = None
-
- # Optional one-shot warmup at construction time to hide cold-start latency.
- if self._warmup:
- self.warmup()
-
- def warmup(self) -> None:
- """Warm up embedding path so first user query is faster."""
- try:
- _ = self.backend_impl.compute_query_embedding(
- "__LEANN_WARMUP__",
- use_server_if_available=self.recompute_embeddings,
- )
- except Exception as exc:
- logger.warning(f"Warmup embedding failed (ignored): {exc}")
-
- def search(
- self,
- query: str,
- top_k: int = 5,
- complexity: int = 64,
- beam_width: int = 1,
- prune_ratio: float = 0.0,
- recompute_embeddings: Optional[bool] = None,
- pruning_strategy: Literal["global", "local", "proportional"] = "global",
- expected_zmq_port: int = 5557,
- metadata_filters: Optional[dict[str, dict[str, Union[str, int, float, bool, list]]]] = None,
- batch_size: int = 0,
- use_grep: bool = False,
- gemma: float = 1.0,
- provider_options: Optional[dict[str, Any]] = None,
- **kwargs,
- ) -> list[SearchResult]:
- """
- Search for nearest neighbors with optional metadata filtering.
-
- Args:
- query: Text query to search for
- top_k: Number of nearest neighbors to return
- complexity: Search complexity/candidate list size, higher = more accurate but slower
- beam_width: Number of parallel search paths/IO requests per iteration
- prune_ratio: Ratio of neighbors to prune via approximate distance (0.0-1.0)
- recompute_embeddings: (Deprecated) Per-call override for recompute mode.
- Configure this at LeannSearcher(..., recompute_embeddings=...) instead.
- pruning_strategy: Candidate selection strategy - "global" (default), "local", or "proportional"
- expected_zmq_port: ZMQ port for embedding server communication
- metadata_filters: Optional filters to apply to search results based on metadata.
- Format: {"field_name": {"operator": value}}
- Supported operators:
- - Comparison: "==", "!=", "<", "<=", ">", ">="
- - Membership: "in", "not_in"
- - String: "contains", "starts_with", "ends_with"
- Example: {"chapter": {"<=": 5}, "tags": {"in": ["fiction", "drama"]}}
- gemma: Weight of vector search results in hybrid search (0.0-1.0), 1 = pure vector search, 0 = pure keyword search
- **kwargs: Backend-specific parameters
-
- Returns:
- List of SearchResult objects with text, metadata, and similarity scores
- """
- # Handle grep search
- if use_grep:
- return self._grep_search(query, top_k)
-
- logger.info("🔍 LeannSearcher.search() called:")
- logger.info(f" Query: '{query}'")
- logger.info(f" Top_k: {top_k}")
- logger.info(f" Metadata filters: {metadata_filters}")
- logger.info(f" Additional kwargs: {kwargs}")
-
- # Smart top_k detection and adjustment
- # Use PassageManager length (sum of shard sizes) to avoid
- # depending on a massive combined map
- total_docs = len(self.passage_manager)
- original_top_k = top_k
- if top_k > total_docs:
- top_k = total_docs
- logger.warning(
- f" ⚠️ Requested top_k ({original_top_k}) exceeds total documents ({total_docs})"
- )
- logger.warning(f" ✅ Auto-adjusted top_k to {top_k} to match available documents")
-
- # Handle pure keyword search
- if gemma == 0.0:
- start_time = time.time()
- bm25_results = self._bm25_search(query, top_k)
- # Convert BM25 results to the expected format
- results = {
- "labels": [[r.id for r in bm25_results]],
- "distances": [[r.score for r in bm25_results]],
- }
- else:
- # Perform vector search
- zmq_port = None
-
- # Resolve effective recompute flag for this search.
- if recompute_embeddings is not None:
- logger.warning(
- "LeannSearcher.search(..., recompute_embeddings=...) is deprecated and "
- "will be removed in a future version. Configure recompute at "
- "LeannSearcher(..., recompute_embeddings=...) instead."
- )
- effective_recompute = bool(recompute_embeddings)
- else:
- effective_recompute = self.recompute_embeddings
-
- start_time = time.time()
- if effective_recompute:
- zmq_port = self.backend_impl._ensure_server_running(
- self.meta_path_str,
- port=expected_zmq_port,
- enable_warmup=self._warmup,
- use_daemon=self._use_daemon,
- daemon_ttl_seconds=self._daemon_ttl_seconds,
- **kwargs,
- )
- del expected_zmq_port
- zmq_time = time.time() - start_time
- logger.info(f" Launching server time: {zmq_time} seconds")
-
- start_time = time.time()
-
- # Extract query template from stored embedding_options with fallback chain:
- # 1. Check provider_options override (highest priority)
- # 2. Check query_prompt_template (new format)
- # 3. Check prompt_template (old format for backward compat)
- # 4. None (no template)
- query_template = None
- if provider_options and "prompt_template" in provider_options:
- query_template = provider_options["prompt_template"]
- elif "query_prompt_template" in self.embedding_options:
- query_template = self.embedding_options["query_prompt_template"]
- elif "prompt_template" in self.embedding_options:
- query_template = self.embedding_options["prompt_template"]
-
- query_embedding = self.backend_impl.compute_query_embedding(
- query,
- use_server_if_available=effective_recompute,
- zmq_port=zmq_port,
- query_template=query_template,
- )
- logger.info(f" Generated embedding shape: {query_embedding.shape}")
- embedding_time = time.time() - start_time
- logger.info(f" Embedding time: {embedding_time} seconds")
-
- start_time = time.time()
- backend_search_kwargs: dict[str, Any] = {
- "complexity": complexity,
- "beam_width": beam_width,
- "prune_ratio": prune_ratio,
- "recompute_embeddings": effective_recompute,
- "pruning_strategy": pruning_strategy,
- "zmq_port": zmq_port,
- }
- # Only HNSW supports batching; forward conditionally
- if self.backend_name == "hnsw":
- backend_search_kwargs["batch_size"] = batch_size
-
- # Merge any extra kwargs last
- backend_search_kwargs.update(kwargs)
-
- results = self.backend_impl.search(
- query_embedding,
- top_k,
- **backend_search_kwargs,
- )
-
- # Handle hybrid search
- if 0.0 < gemma < 1.0:
- logger.info(f" 🌟 Hybrid search enabled with gemma={gemma}")
- BM25_WEIGHT = 1.0 - gemma
- bm25_results = self._bm25_search(query, top_k)
- hybrid_scores: dict[str, float] = {}
- # Add vector search scores (weighted by gemma)
- if "labels" in results and "distances" in results:
- for doc_id, score in zip(results["labels"][0], results["distances"][0]):
- hybrid_scores[doc_id] = gemma * score
- # Add BM25 scores (weighted by BM25_WEIGHT)
- for bm25_result in bm25_results:
- doc_id = bm25_result.id
- if doc_id in hybrid_scores:
- hybrid_scores[doc_id] += BM25_WEIGHT * bm25_result.score
- else:
- hybrid_scores[doc_id] = BM25_WEIGHT * bm25_result.score
-
- sorted_hybrid = sorted(hybrid_scores.items(), key=lambda x: x[1], reverse=True)[:top_k]
- results["labels"] = [[doc_id for doc_id, _ in sorted_hybrid]]
- results["distances"] = [[score for _, score in sorted_hybrid]]
-
- logger.info(
- f" Combined {len(hybrid_scores)} unique documents from vector and BM25 search"
- )
-
- search_time = time.time() - start_time
- logger.info(f" Search time in search() LEANN searcher: {search_time} seconds")
- logger.info(f" Backend returned: labels={len(results.get('labels', [[]])[0])} results")
-
- enriched_results = []
- if "labels" in results and "distances" in results:
- logger.info(f" Processing {len(results['labels'][0])} passage IDs:")
- # Python 3.9 does not support zip(strict=...); lengths are expected to match
- for i, (string_id, dist) in enumerate(
- zip(results["labels"][0], results["distances"][0])
- ):
- try:
- passage_data = self.passage_manager.get_passage(string_id)
- enriched_results.append(
- SearchResult(
- id=string_id,
- score=dist,
- text=passage_data["text"],
- metadata=passage_data.get("metadata", {}),
- )
- )
-
- # Color codes for better logging
- GREEN = "\033[92m"
- BLUE = "\033[94m"
- YELLOW = "\033[93m"
- RESET = "\033[0m"
-
- # Truncate text for display (first 100 chars)
- display_text = passage_data["text"]
- logger.info(
- f" {GREEN}✓{RESET} {BLUE}[{i + 1:2d}]{RESET} {YELLOW}ID:{RESET} '{string_id}' {YELLOW}Score:{RESET} {dist:.4f} {YELLOW}Text:{RESET} {display_text}"
- )
- except KeyError:
- RED = "\033[91m"
- RESET = "\033[0m"
- logger.error(
- f" {RED}✗{RESET} [{i + 1:2d}] ID: '{string_id}' -> {RED}ERROR: Passage not found!{RESET}"
- )
-
- # Apply metadata filters if specified
- if metadata_filters:
- logger.info(f" 🔍 Applying metadata filters: {metadata_filters}")
- enriched_results = self.passage_manager.filter_search_results(
- enriched_results, metadata_filters
- )
-
- # Define color codes outside the loop for final message
- GREEN = "\033[92m"
- RESET = "\033[0m"
- logger.info(f" {GREEN}✓ Final enriched results: {len(enriched_results)} passages{RESET}")
- return enriched_results
-
- def _init_bm25(self) -> None:
- """Initialize BM25 scorer"""
- self.bm25_scorer = BM25Scorer()
- # Load all the files directly
- passages = []
- for passage_file in self.passage_manager.passage_files.values():
- with open(passage_file, encoding="utf-8") as f:
- for line in f:
- if line.strip():
- data = json.loads(line)
- passages.append(data)
- self.bm25_scorer.fit(passages)
-
- def _bm25_search(self, query: str, top_k: int = 5) -> list[SearchResult]:
- """Perform BM25 search on raw passages"""
- if self.bm25_scorer is None:
- self._init_bm25()
- logger.info(" BM25 scorer initialized")
- scorer = self.bm25_scorer
- if scorer is None:
- raise RuntimeError("BM25 scorer failed to initialize")
- return scorer.search(query, top_k)
-
- def _find_jsonl_file(self) -> Optional[str]:
- """Find the .jsonl file containing raw passages for grep search"""
- index_path = Path(self.meta_path_str).parent
- potential_files = [
- index_path / "documents.leann.passages.jsonl",
- index_path.parent / "documents.leann.passages.jsonl",
- ]
-
- for file_path in potential_files:
- if file_path.exists():
- return str(file_path)
- return None
-
- def _grep_search(self, query: str, top_k: int = 5) -> list[SearchResult]:
- """Perform grep-based search on raw passages"""
- jsonl_file = self._find_jsonl_file()
- if not jsonl_file:
- raise FileNotFoundError("No .jsonl passages file found for grep search")
-
- try:
- cmd = ["grep", "-i", "-n", query, jsonl_file]
- result = subprocess.run(cmd, capture_output=True, text=True, check=False)
-
- if result.returncode == 1:
- return []
- elif result.returncode != 0:
- raise RuntimeError(f"Grep failed: {result.stderr}")
-
- matches = []
- for line in result.stdout.strip().split("\n"):
- if not line:
- continue
- parts = line.split(":", 1)
- if len(parts) != 2:
- continue
-
- try:
- data = json.loads(parts[1])
- text = data.get("text", "")
- score = text.lower().count(query.lower())
-
- matches.append(
- SearchResult(
- id=data.get("id", parts[0]),
- text=text,
- metadata=data.get("metadata", {}),
- score=float(score),
- )
- )
- except json.JSONDecodeError:
- continue
-
- matches.sort(key=lambda x: x.score, reverse=True)
- return matches[:top_k]
-
- except FileNotFoundError:
- raise RuntimeError(
- "grep command not found. Please install grep or use semantic search."
- )
-
- def _python_regex_search(self, query: str, top_k: int = 5) -> list[SearchResult]:
- """Fallback regex search"""
- jsonl_file = self._find_jsonl_file()
- if not jsonl_file:
- raise FileNotFoundError("No .jsonl file found")
-
- pattern = re.compile(re.escape(query), re.IGNORECASE)
- matches = []
-
- with open(jsonl_file, encoding="utf-8") as f:
- for line_num, line in enumerate(f, 1):
- if pattern.search(line):
- try:
- data = json.loads(line.strip())
- matches.append(
- SearchResult(
- id=data.get("id", str(line_num)),
- text=data.get("text", ""),
- metadata=data.get("metadata", {}),
- score=float(len(pattern.findall(data.get("text", "")))),
- )
- )
- except json.JSONDecodeError:
- continue
-
- matches.sort(key=lambda x: x.score, reverse=True)
- return matches[:top_k]
-
- def cleanup(self):
- """Explicitly cleanup embedding server resources.
- This method should be called after you're done using the searcher,
- especially in test environments or batch processing scenarios.
- """
- backend = getattr(self.backend_impl, "embedding_server_manager", None)
- if backend is not None:
- backend.stop_server()
-
- # Enable automatic cleanup patterns
- def __enter__(self):
- return self
-
- def __exit__(self, exc_type, exc, tb):
- try:
- self.cleanup()
- except Exception:
- pass
-
- def __del__(self):
- try:
- self.cleanup()
- except Exception:
- # Avoid noisy errors during interpreter shutdown
- pass
-
-
-class LeannChat:
- def __init__(
- self,
- index_path: str,
- llm_config: Optional[dict[str, Any]] = None,
- enable_warmup: bool = False,
- searcher: Optional[LeannSearcher] = None,
- **kwargs,
- ):
- if searcher is None:
- self.searcher = LeannSearcher(index_path, enable_warmup=enable_warmup, **kwargs)
- self._owns_searcher = True
- else:
- self.searcher = searcher
- self._owns_searcher = False
- self.llm = get_llm(llm_config)
-
- def ask(
- self,
- question: str,
- top_k: int = 5,
- complexity: int = 64,
- beam_width: int = 1,
- prune_ratio: float = 0.0,
- recompute_embeddings: bool = True,
- pruning_strategy: Literal["global", "local", "proportional"] = "global",
- llm_kwargs: Optional[dict[str, Any]] = None,
- expected_zmq_port: int = 5557,
- metadata_filters: Optional[dict[str, dict[str, Union[str, int, float, bool, list]]]] = None,
- batch_size: int = 0,
- use_grep: bool = False,
- gemma: float = 1.0,
- **search_kwargs,
- ):
- if llm_kwargs is None:
- llm_kwargs = {}
- search_time = time.time()
- results = self.searcher.search(
- question,
- top_k=top_k,
- complexity=complexity,
- beam_width=beam_width,
- prune_ratio=prune_ratio,
- recompute_embeddings=recompute_embeddings,
- pruning_strategy=pruning_strategy,
- expected_zmq_port=expected_zmq_port,
- metadata_filters=metadata_filters,
- use_grep=use_grep,
- gemma=gemma,
- batch_size=batch_size,
- **search_kwargs,
- )
- search_time = time.time() - search_time
- logger.info(f" Search time: {search_time} seconds")
- context = "\n\n".join([r.text for r in results])
- prompt = (
- "Here is some retrieved context that might help answer your question:\n\n"
- f"{context}\n\n"
- f"Question: {question}\n\n"
- "Please provide the best answer you can based on this context and your knowledge."
- )
-
- logger.info("The context provided to the LLM is:")
- logger.info(f"{'Relevance':<10} | {'Chunk id':<10} | {'Content':<60} | {'Source':<80}")
- logger.info("-" * 150)
- for r in results:
- chunk_relevance = f"{r.score:.3f}"
- chunk_id = r.id
- chunk_content = r.text[:60]
- chunk_source = r.metadata.get("source", "")[:80]
- logger.info(
- f"{chunk_relevance:<10} | {chunk_id:<10} | {chunk_content:<60} | {chunk_source:<80}"
- )
- ask_time = time.time()
- ans = self.llm.ask(prompt, **llm_kwargs)
- ask_time = time.time() - ask_time
- logger.info(f" Ask time: {ask_time} seconds")
- return ans
-
- def start_interactive(self):
- """Start interactive chat session."""
- session = create_api_session()
-
- def handle_query(user_input: str):
- response = self.ask(user_input)
- print(f"Leann: {response}")
-
- session.run_interactive_loop(handle_query)
-
- def cleanup(self):
- """Explicitly cleanup embedding server resources.
-
- This method should be called after you're done using the chat interface,
- especially in test environments or batch processing scenarios.
- """
- # Only stop the embedding server if this LeannChat instance created the searcher.
- # When a shared searcher is passed in, avoid shutting down the server to enable reuse.
- if getattr(self, "_owns_searcher", False) and hasattr(self.searcher, "cleanup"):
- self.searcher.cleanup()
-
- # Enable automatic cleanup patterns
- def __enter__(self):
- return self
-
- def __exit__(self, exc_type, exc, tb):
- try:
- self.cleanup()
- except Exception:
- pass
-
- def __del__(self):
- try:
- self.cleanup()
- except Exception:
- pass
+"""
+This file contains the core API for the LEANN project, now definitively updated
+with the correct, original embedding logic from the user's reference code.
+"""
+
+import json
+import logging
+import os
+import pickle
+import re
+import subprocess
+import time
+import warnings
+from collections import Counter, defaultdict
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Literal, Optional, Union
+
+import numpy as np
+from leann_backend_hnsw.convert_to_csr import prune_hnsw_embeddings_inplace
+
+from leann.interactive_utils import create_api_session
+from leann.interface import LeannBackendSearcherInterface
+
+from .chat import get_llm
+from .embedding_server_manager import EmbeddingServerManager
+from .interface import LeannBackendFactoryInterface
+from .metadata_filter import MetadataFilterEngine
+from .registry import BACKEND_REGISTRY
+
+logger = logging.getLogger(__name__)
+
+
+def get_registered_backends() -> list[str]:
+ """Get list of registered backend names."""
+ return list(BACKEND_REGISTRY.keys())
+
+
+def compute_embeddings(
+ chunks: list[str],
+ model_name: str,
+ mode: str = "sentence-transformers",
+ use_server: bool = True,
+ port: Optional[int] = None,
+ is_build=False,
+ provider_options: Optional[dict[str, Any]] = None,
+) -> np.ndarray:
+ """
+ Computes embeddings using different backends.
+
+ Args:
+ chunks: List of text chunks to embed
+ model_name: Name of the embedding model
+ mode: Embedding backend mode. Options:
+ - "sentence-transformers": Use sentence-transformers library (default)
+ - "mlx": Use MLX backend for Apple Silicon
+ - "openai": Use OpenAI embedding API
+ - "gemini": Use Google Gemini embedding API
+ use_server: Whether to use embedding server (True for search, False for build)
+
+ Returns:
+ numpy array of embeddings
+ """
+ if use_server:
+ # Use embedding server (for search/query)
+ if port is None:
+ raise ValueError("port is required when use_server is True")
+ return compute_embeddings_via_server(chunks, model_name, port=port)
+ else:
+ # Use direct computation (for build_index)
+ from .embedding_compute import (
+ compute_embeddings as compute_embeddings_direct,
+ )
+
+ return compute_embeddings_direct(
+ chunks,
+ model_name,
+ mode=mode,
+ is_build=is_build,
+ provider_options=provider_options,
+ )
+
+
+def compute_embeddings_via_server(chunks: list[str], model_name: str, port: int) -> np.ndarray:
+ """Computes embeddings using sentence-transformers.
+
+ Args:
+ chunks: List of text chunks to embed
+ model_name: Name of the sentence transformer model
+ """
+ logger.info(
+ f"Computing embeddings for {len(chunks)} chunks using SentenceTransformer model '{model_name}' (via embedding server)..."
+ )
+ import msgpack
+ import numpy as np
+ import zmq
+
+ # Connect to embedding server
+ context = zmq.Context()
+ socket = context.socket(zmq.REQ)
+ socket.connect(f"tcp://localhost:{port}")
+
+ # Send chunks to server for embedding computation
+ request = chunks
+ socket.send(msgpack.packb(request))
+
+ # Receive embeddings from server
+ response = socket.recv()
+ embeddings_list = msgpack.unpackb(response)
+
+ # Convert back to numpy array
+ embeddings = np.array(embeddings_list, dtype=np.float32)
+
+ socket.close()
+ context.term()
+
+ return embeddings
+
+
+@dataclass
+class SearchResult:
+ id: str
+ score: float
+ text: str
+ metadata: dict[str, Any] = field(default_factory=dict)
+
+
+class PassageManager:
+ def __init__(
+ self, passage_sources: list[dict[str, Any]], metadata_file_path: Optional[str] = None
+ ):
+ self.offset_maps: dict[str, dict[str, int]] = {}
+ self.passage_files: dict[str, str] = {}
+ # Avoid materializing a single gigantic global map to reduce memory
+ # footprint on very large corpora (e.g., 60M+ passages). Instead, keep
+ # per-shard maps and do a lightweight per-shard lookup on demand.
+ self._total_count: int = 0
+ self.filter_engine = MetadataFilterEngine() # Initialize filter engine
+
+ # Derive index base name for standard sibling fallbacks, e.g., <index_name>.passages.*
+ index_name_base = None
+ if metadata_file_path:
+ meta_name = Path(metadata_file_path).name
+ if meta_name.endswith(".meta.json"):
+ index_name_base = meta_name[: -len(".meta.json")]
+
+ for source in passage_sources:
+ assert source["type"] == "jsonl", "only jsonl is supported"
+ passage_file = source.get("path", "")
+ index_file = source.get("index_path", "") # .idx file
+
+ # Fix path resolution - relative paths should be relative to metadata file directory
+ def _resolve_candidates(
+ primary: str,
+ relative_key: str,
+ default_name: Optional[str],
+ source_dict: dict[str, Any],
+ ) -> list[Path]:
+ """
+ Build an ordered list of candidate paths. For relative paths specified in
+ metadata, prefer resolution relative to the metadata file directory first,
+ then fall back to CWD-based resolution, and finally to conventional
+ sibling defaults (e.g., <index_base>.passages.idx / .jsonl).
+ """
+ candidates: list[Path] = []
+ # 1) Primary path
+ if primary:
+ p = Path(primary)
+ if p.is_absolute():
+ candidates.append(p)
+ else:
+ # Prefer metadata-relative resolution for relative paths
+ if metadata_file_path:
+ candidates.append(Path(metadata_file_path).parent / p)
+ # Also consider CWD-relative as a fallback for legacy layouts
+ candidates.append(Path.cwd() / p)
+ # 2) metadata-relative explicit relative key (if present)
+ if metadata_file_path and source_dict.get(relative_key):
+ candidates.append(Path(metadata_file_path).parent / source_dict[relative_key])
+ # 3) metadata-relative standard sibling filename
+ if metadata_file_path and default_name:
+ candidates.append(Path(metadata_file_path).parent / default_name)
+ return candidates
+
+ # Build candidate lists and pick first existing; otherwise keep last candidate for error message
+ idx_default = f"{index_name_base}.passages.idx" if index_name_base else None
+ idx_candidates = _resolve_candidates(
+ index_file, "index_path_relative", idx_default, source
+ )
+ pas_default = f"{index_name_base}.passages.jsonl" if index_name_base else None
+ pas_candidates = _resolve_candidates(passage_file, "path_relative", pas_default, source)
+
+ def _pick_existing(cands: list[Path]) -> str:
+ for c in cands:
+ if c.exists():
+ return str(c.resolve())
+ # Fallback to last candidate (best guess) even if not exists; will error below
+ return str(cands[-1].resolve()) if cands else ""
+
+ index_file = _pick_existing(idx_candidates)
+ passage_file = _pick_existing(pas_candidates)
+
+ if not Path(index_file).exists():
+ raise FileNotFoundError(f"Passage index file not found: {index_file}")
+
+ with open(index_file, "rb") as f:
+ offset_map: dict[str, int] = pickle.load(f)
+ self.offset_maps[passage_file] = offset_map
+ self.passage_files[passage_file] = passage_file
+ self._total_count += len(offset_map)
+
+ def get_passage(self, passage_id: str) -> dict[str, Any]:
+ # Fast path: check each shard map (there are typically few shards).
+ # This avoids building a massive combined dict while keeping lookups
+ # bounded by the number of shards.
+ for passage_file, offset_map in self.offset_maps.items():
+ try:
+ offset = offset_map[passage_id]
+ with open(passage_file, encoding="utf-8") as f:
+ f.seek(offset)
+ return json.loads(f.readline())
+ except KeyError:
+ continue
+ raise KeyError(f"Passage ID not found: {passage_id}")
+
+ def filter_search_results(
+ self,
+ search_results: list[SearchResult],
+ metadata_filters: Optional[dict[str, dict[str, Union[str, int, float, bool, list]]]],
+ ) -> list[SearchResult]:
+ """
+ Apply metadata filters to search results.
+
+ Args:
+ search_results: List of SearchResult objects
+ metadata_filters: Filter specifications to apply
+
+ Returns:
+ Filtered list of SearchResult objects
+ """
+ if not metadata_filters:
+ return search_results
+
+ logger.debug(f"Applying metadata filters to {len(search_results)} results")
+
+ # Convert SearchResult objects to dictionaries for the filter engine
+ result_dicts = []
+ for result in search_results:
+ result_dicts.append(
+ {
+ "id": result.id,
+ "score": result.score,
+ "text": result.text,
+ "metadata": result.metadata,
+ }
+ )
+
+ # Apply filters using the filter engine
+ filtered_dicts = self.filter_engine.apply_filters(result_dicts, metadata_filters)
+
+ # Convert back to SearchResult objects
+ filtered_results = []
+ for result_dict in filtered_dicts:
+ filtered_results.append(
+ SearchResult(
+ id=result_dict["id"],
+ score=result_dict["score"],
+ text=result_dict["text"],
+ metadata=result_dict["metadata"],
+ )
+ )
+
+ logger.debug(f"Filtered results: {len(filtered_results)} remaining")
+ return filtered_results
+
+ def __len__(self) -> int:
+ return self._total_count
+
+
+class BM25Scorer:
+ def __init__(self, k1: float = 1.2, b: float = 0.75):
+ self.k1 = k1
+ self.b = b
+ self.doc_freqs = None # How many docs contain each term (DF)
+ self.doc_lengths = {} # How long each doc is (in words)
+ self.word_counts = {} # How many times each word appears in each doc (TF)
+ self.avg_doc_length = None
+ self.corpus_size = None
+ self.idlist = set() # List of all document IDs for easier searching
+
+ def _tokenize(self, text: str) -> list[str]:
+ return re.sub(r"[^\w\s]", "", text).lower().split()
+
+ def fit(self, documents: list[dict[str, Any]]):
+ """
+ Build BM25 statistics from a document corpus.
+ Must be called before scoring.
+ """
+ self.corpus_size = len(documents)
+ self.doc_lengths = {}
+ self.word_counts = {}
+ self.idlist = set()
+ doc_freqs = defaultdict(int)
+
+ for doc_data in documents:
+ doc_id = doc_data["id"]
+ words = self._tokenize(doc_data["text"])
+ doc_length = len(words)
+ self.doc_lengths[doc_id] = doc_length
+
+ unique_words = set(words)
+ for word in unique_words:
+ doc_freqs[word] += 1
+ self.word_counts[doc_id] = dict(Counter(words))
+ self.idlist.add(doc_id)
+
+ self.doc_freqs = dict(doc_freqs)
+ self.avg_doc_length = sum(self.doc_lengths.values()) / len(self.doc_lengths)
+
+ def score(self, query_words: list[str], document_id: str) -> float:
+ if (
+ self.doc_freqs is None
+ or self.doc_lengths == {}
+ or self.word_counts == {}
+ or self.avg_doc_length is None
+ or self.corpus_size is None
+ ):
+ raise ValueError("BM25 model not fitted. Call fit() before scoring.")
+
+ passage_words = self.word_counts[document_id]
+ passage_length = sum(passage_words.values())
+ score = 0.0
+ for word in query_words:
+ if word not in self.doc_freqs:
+ continue
+ word_freq = passage_words[word] if word in passage_words else 0
+ idf = np.log(
+ (self.corpus_size - self.doc_freqs[word] + 0.5) / (self.doc_freqs[word] + 0.5) + 1
+ )
+ tf = (word_freq * (self.k1 + 1)) / (
+ word_freq + self.k1 * (1 - self.b + self.b * (passage_length / self.avg_doc_length))
+ )
+ score += idf * tf
+ return score
+
+ def search(self, query: str, top_k: int = 5) -> list[SearchResult]:
+ query_words = self._tokenize(query)
+ scores = {doc_id: self.score(query_words, doc_id) for doc_id in self.idlist}
+ sorted_scores = sorted(scores.items(), key=lambda x: x[1], reverse=True)
+ return [
+ SearchResult(id=doc_id, score=score, text="", metadata={})
+ for doc_id, score in sorted_scores[:top_k]
+ ]
+
+
+class LeannBuilder:
+ def __init__(
+ self,
+ backend_name: str,
+ embedding_model: str = "facebook/contriever",
+ dimensions: Optional[int] = None,
+ embedding_mode: str = "sentence-transformers",
+ embedding_options: Optional[dict[str, Any]] = None,
+ **backend_kwargs,
+ ):
+ self.backend_name = backend_name
+ # Normalize incompatible combinations early (for consistent metadata)
+ if backend_name == "hnsw":
+ is_recompute = backend_kwargs.get("is_recompute", True)
+ is_compact = backend_kwargs.get("is_compact", True)
+ if is_recompute is False and is_compact is True:
+ warnings.warn(
+ "HNSW with is_recompute=False requires non-compact storage. Forcing is_compact=False.",
+ UserWarning,
+ stacklevel=2,
+ )
+ backend_kwargs["is_compact"] = False
+
+ backend_factory: Optional[LeannBackendFactoryInterface] = BACKEND_REGISTRY.get(backend_name)
+ if backend_factory is None:
+ raise ValueError(f"Backend '{backend_name}' not found or not registered.")
+ self.backend_factory = backend_factory
+ self.embedding_model = embedding_model
+ self.dimensions = dimensions
+ self.embedding_mode = embedding_mode
+ self.embedding_options = embedding_options or {}
+
+ # Check if we need to use cosine distance for normalized embeddings
+ normalized_embeddings_models = {
+ # OpenAI models
+ ("openai", "text-embedding-ada-002"),
+ ("openai", "text-embedding-3-small"),
+ ("openai", "text-embedding-3-large"),
+ # Voyage AI models
+ ("voyage", "voyage-2"),
+ ("voyage", "voyage-3"),
+ ("voyage", "voyage-large-2"),
+ ("voyage", "voyage-multilingual-2"),
+ ("voyage", "voyage-code-2"),
+ # Cohere models
+ ("cohere", "embed-english-v3.0"),
+ ("cohere", "embed-multilingual-v3.0"),
+ ("cohere", "embed-english-light-v3.0"),
+ ("cohere", "embed-multilingual-light-v3.0"),
+ }
+
+ # Also check for patterns in model names
+ is_normalized = False
+ current_model_lower = embedding_model.lower()
+ current_mode_lower = embedding_mode.lower()
+
+ # Check exact matches
+ for mode, model in normalized_embeddings_models:
+ if (current_mode_lower == mode and current_model_lower == model) or (
+ mode in current_mode_lower and model in current_model_lower
+ ):
+ is_normalized = True
+ break
+
+ # Check patterns
+ if not is_normalized:
+ # OpenAI patterns
+ if "openai" in current_mode_lower or "openai" in current_model_lower:
+ if any(
+ pattern in current_model_lower
+ for pattern in ["text-embedding", "ada", "3-small", "3-large"]
+ ):
+ is_normalized = True
+ # Voyage patterns
+ elif "voyage" in current_mode_lower or "voyage" in current_model_lower:
+ is_normalized = True
+ # Cohere patterns
+ elif "cohere" in current_mode_lower or "cohere" in current_model_lower:
+ if "embed" in current_model_lower:
+ is_normalized = True
+
+ # Handle distance metric
+ if is_normalized and "distance_metric" not in backend_kwargs:
+ backend_kwargs["distance_metric"] = "cosine"
+ warnings.warn(
+ f"Detected normalized embeddings model '{embedding_model}' with mode '{embedding_mode}'. "
+ f"Automatically setting distance_metric='cosine' for optimal performance. "
+ f"Normalized embeddings (L2 norm = 1) should use cosine similarity instead of MIPS.",
+ UserWarning,
+ stacklevel=2,
+ )
+ elif is_normalized and backend_kwargs.get("distance_metric", "").lower() != "cosine":
+ current_metric = backend_kwargs.get("distance_metric", "mips")
+ warnings.warn(
+ f"Warning: Using '{current_metric}' distance metric with normalized embeddings model "
+ f"'{embedding_model}' may lead to suboptimal search results. "
+ f"Consider using 'cosine' distance metric for better performance.",
+ UserWarning,
+ stacklevel=2,
+ )
+
+ self.backend_kwargs = backend_kwargs
+ self.chunks: list[dict[str, Any]] = []
+
+ def add_text(self, text: str, metadata: Optional[dict[str, Any]] = None):
+ if metadata is None:
+ metadata = {}
+ passage_id = metadata.get("id", str(len(self.chunks)))
+ chunk_data = {"id": passage_id, "text": text, "metadata": metadata}
+ self.chunks.append(chunk_data)
+
+ def build_index(self, index_path: str):
+ if not self.chunks:
+ raise ValueError("No chunks added.")
+
+ # Filter out invalid/empty text chunks early to keep passage and embedding counts aligned
+ valid_chunks: list[dict[str, Any]] = []
+ skipped = 0
+ for chunk in self.chunks:
+ text = chunk.get("text", "")
+ if isinstance(text, str) and text.strip():
+ valid_chunks.append(chunk)
+ else:
+ skipped += 1
+ if skipped > 0:
+ print(
+ f"Warning: Skipping {skipped} empty/invalid text chunk(s). Processing {len(valid_chunks)} valid chunks"
+ )
+ self.chunks = valid_chunks
+ if not self.chunks:
+ raise ValueError("All provided chunks are empty or invalid. Nothing to index.")
+ if self.dimensions is None:
+ self.dimensions = len(
+ compute_embeddings(
+ ["dummy"],
+ self.embedding_model,
+ self.embedding_mode,
+ use_server=False,
+ provider_options=self.embedding_options,
+ )[0]
+ )
+ path = Path(index_path)
+ index_dir = path.parent
+ index_name = path.name
+ index_dir.mkdir(parents=True, exist_ok=True)
+ passages_file = index_dir / f"{index_name}.passages.jsonl"
+ offset_file = index_dir / f"{index_name}.passages.idx"
+ offset_map = {}
+ with open(passages_file, "w", encoding="utf-8") as f:
+ try:
+ from tqdm import tqdm
+
+ chunk_iterator = tqdm(self.chunks, desc="Writing passages", unit="chunk")
+ except ImportError:
+ chunk_iterator = self.chunks
+
+ for chunk in chunk_iterator:
+ offset = f.tell()
+ json.dump(
+ {
+ "id": chunk["id"],
+ "text": chunk["text"],
+ "metadata": chunk["metadata"],
+ },
+ f,
+ ensure_ascii=False,
+ )
+ f.write("\n")
+ offset_map[chunk["id"]] = offset
+ with open(offset_file, "wb") as f:
+ pickle.dump(offset_map, f)
+ texts_to_embed = [c["text"] for c in self.chunks]
+ embeddings = compute_embeddings(
+ texts_to_embed,
+ self.embedding_model,
+ self.embedding_mode,
+ use_server=False,
+ is_build=True,
+ provider_options=self.embedding_options,
+ )
+ string_ids = [chunk["id"] for chunk in self.chunks]
+ # Persist ID map alongside index so backends that return integer labels can remap to passage IDs
+ try:
+ idmap_file = (
+ index_dir
+ / f"{index_name[: -len('.leann')] if index_name.endswith('.leann') else index_name}.ids.txt"
+ )
+ with open(idmap_file, "w", encoding="utf-8") as f:
+ for sid in string_ids:
+ f.write(str(sid) + "\n")
+ except Exception:
+ pass
+ current_backend_kwargs = {**self.backend_kwargs, "dimensions": self.dimensions}
+ builder_instance = self.backend_factory.builder(**current_backend_kwargs)
+ builder_instance.build(embeddings, string_ids, index_path, **current_backend_kwargs)
+ leann_meta_path = index_dir / f"{index_name}.meta.json"
+ meta_data = {
+ "version": "1.0",
+ "backend_name": self.backend_name,
+ "embedding_model": self.embedding_model,
+ "dimensions": self.dimensions,
+ "backend_kwargs": self.backend_kwargs,
+ "embedding_mode": self.embedding_mode,
+ "passage_sources": [
+ {
+ "type": "jsonl",
+ # Preserve existing relative file names (backward-compatible)
+ "path": passages_file.name,
+ "index_path": offset_file.name,
+ # Add optional redundant relative keys for remote build portability (non-breaking)
+ "path_relative": passages_file.name,
+ "index_path_relative": offset_file.name,
+ }
+ ],
+ }
+
+ if self.embedding_options:
+ meta_data["embedding_options"] = self.embedding_options
+
+ # Add storage status flags for HNSW backend
+ if self.backend_name == "hnsw":
+ is_compact = self.backend_kwargs.get("is_compact", True)
+ is_recompute = self.backend_kwargs.get("is_recompute", True)
+ meta_data["is_compact"] = is_compact
+ meta_data["is_pruned"] = bool(is_recompute)
+ with open(leann_meta_path, "w", encoding="utf-8") as f:
+ json.dump(meta_data, f, indent=2)
+
+ def build_index_from_embeddings(self, index_path: str, embeddings_file: str):
+ """
+ Build an index from pre-computed embeddings stored in a pickle file.
+
+ Args:
+ index_path: Path where the index will be saved
+ embeddings_file: Path to pickle file containing (ids, embeddings) tuple
+ """
+ # Load pre-computed embeddings
+ with open(embeddings_file, "rb") as f:
+ data = pickle.load(f)
+
+ if not isinstance(data, tuple) or len(data) != 2:
+ raise ValueError(
+ f"Invalid embeddings file format. Expected tuple with 2 elements, got {type(data)}"
+ )
+
+ ids, embeddings = data
+
+ if not isinstance(embeddings, np.ndarray):
+ raise ValueError(f"Expected embeddings to be numpy array, got {type(embeddings)}")
+
+ if len(ids) != embeddings.shape[0]:
+ raise ValueError(
+ f"Mismatch between number of IDs ({len(ids)}) and embeddings ({embeddings.shape[0]})"
+ )
+
+ # Validate/set dimensions
+ embedding_dim = embeddings.shape[1]
+ if self.dimensions is None:
+ self.dimensions = embedding_dim
+ elif self.dimensions != embedding_dim:
+ raise ValueError(f"Dimension mismatch: expected {self.dimensions}, got {embedding_dim}")
+
+ logger.info(
+ f"Building index from precomputed embeddings: {len(ids)} items, {embedding_dim} dimensions"
+ )
+
+ # Ensure we have text data for each embedding
+ if len(self.chunks) != len(ids):
+ # If no text chunks provided, create placeholder text entries
+ if not self.chunks:
+ logger.info("No text chunks provided, creating placeholder entries...")
+ for id_val in ids:
+ self.add_text(
+ f"Document {id_val}",
+ metadata={"id": str(id_val), "from_embeddings": True},
+ )
+ else:
+ raise ValueError(
+ f"Number of text chunks ({len(self.chunks)}) doesn't match number of embeddings ({len(ids)})"
+ )
+
+ # Build file structure
+ path = Path(index_path)
+ index_dir = path.parent
+ index_name = path.name
+ index_dir.mkdir(parents=True, exist_ok=True)
+ passages_file = index_dir / f"{index_name}.passages.jsonl"
+ offset_file = index_dir / f"{index_name}.passages.idx"
+
+ # Write passages and create offset map
+ offset_map = {}
+ with open(passages_file, "w", encoding="utf-8") as f:
+ for chunk in self.chunks:
+ offset = f.tell()
+ json.dump(
+ {
+ "id": chunk["id"],
+ "text": chunk["text"],
+ "metadata": chunk["metadata"],
+ },
+ f,
+ ensure_ascii=False,
+ )
+ f.write("\n")
+ offset_map[chunk["id"]] = offset
+
+ with open(offset_file, "wb") as f:
+ pickle.dump(offset_map, f)
+
+ # Build the vector index using precomputed embeddings
+ string_ids = [str(id_val) for id_val in ids]
+ # Persist ID map (order == embeddings order)
+ try:
+ idmap_file = (
+ index_dir
+ / f"{index_name[: -len('.leann')] if index_name.endswith('.leann') else index_name}.ids.txt"
+ )
+ with open(idmap_file, "w", encoding="utf-8") as f:
+ for sid in string_ids:
+ f.write(str(sid) + "\n")
+ except Exception:
+ pass
+ current_backend_kwargs = {**self.backend_kwargs, "dimensions": self.dimensions}
+ builder_instance = self.backend_factory.builder(**current_backend_kwargs)
+ builder_instance.build(embeddings, string_ids, index_path)
+
+ # Create metadata file
+ leann_meta_path = index_dir / f"{index_name}.meta.json"
+ meta_data = {
+ "version": "1.0",
+ "backend_name": self.backend_name,
+ "embedding_model": self.embedding_model,
+ "dimensions": self.dimensions,
+ "backend_kwargs": self.backend_kwargs,
+ "embedding_mode": self.embedding_mode,
+ "passage_sources": [
+ {
+ "type": "jsonl",
+ # Preserve existing relative file names (backward-compatible)
+ "path": passages_file.name,
+ "index_path": offset_file.name,
+ # Add optional redundant relative keys for remote build portability (non-breaking)
+ "path_relative": passages_file.name,
+ "index_path_relative": offset_file.name,
+ }
+ ],
+ "built_from_precomputed_embeddings": True,
+ "embeddings_source": str(embeddings_file),
+ }
+
+ if self.embedding_options:
+ meta_data["embedding_options"] = self.embedding_options
+
+ # Add storage status flags for HNSW backend
+ if self.backend_name == "hnsw":
+ is_compact = self.backend_kwargs.get("is_compact", True)
+ is_recompute = self.backend_kwargs.get("is_recompute", True)
+ meta_data["is_compact"] = is_compact
+ meta_data["is_pruned"] = bool(is_recompute)
+
+ with open(leann_meta_path, "w", encoding="utf-8") as f:
+ json.dump(meta_data, f, indent=2)
+
+ logger.info(f"Index built successfully from precomputed embeddings: {index_path}")
+
+ def update_index(self, index_path: str, remove_passage_ids: Optional[list[str]] = None) -> None:
+ """Append new passages and vectors to an existing index (HNSW or IVF).
+ For IVF, optional remove_passage_ids removes those ids first (e.g. from file-change API).
+ """
+ if not self.chunks and not remove_passage_ids:
+ raise ValueError("No new chunks or passage ids to remove provided for update.")
+
+ path = Path(index_path)
+ index_dir = path.parent
+ index_name = path.name
+ index_prefix = path.stem
+
+ meta_path = index_dir / f"{index_name}.meta.json"
+ passages_file = index_dir / f"{index_name}.passages.jsonl"
+ offset_file = index_dir / f"{index_name}.passages.idx"
+ index_file = index_dir / f"{index_prefix}.index"
+
+ if not meta_path.exists() or not passages_file.exists() or not offset_file.exists():
+ raise FileNotFoundError("Index metadata or passage files are missing; cannot update.")
+ if not index_file.exists():
+ raise FileNotFoundError(f"Index file not found: {index_file}")
+
+ with open(meta_path, encoding="utf-8") as f:
+ meta = json.load(f)
+ backend_name = meta.get("backend_name")
+ if backend_name != self.backend_name:
+ raise ValueError(
+ f"Index was built with backend '{backend_name}', cannot update with '{self.backend_name}'."
+ )
+
+ with open(offset_file, "rb") as f:
+ offset_map: dict[str, int] = pickle.load(f)
+ existing_ids = set(offset_map.keys())
+
+ # IVF: optional delete (for reindex / file-change: remove then re-insert)
+ if remove_passage_ids and backend_name == "ivf":
+ try:
+ from leann_backend_ivf import remove_ids as ivf_remove_ids
+
+ ivf_remove_ids(str(path), remove_passage_ids)
+ except ImportError:
+ raise RuntimeError(
+ "IVF backend required for remove_ids. Install leann-backend-ivf."
+ )
+ for pid in remove_passage_ids:
+ offset_map.pop(pid, None)
+ existing_ids -= set(remove_passage_ids)
+ with open(offset_file, "wb") as f:
+ pickle.dump(offset_map, f)
+
+ if not self.chunks:
+ meta["total_passages"] = len(offset_map)
+ with open(meta_path, "w", encoding="utf-8") as f:
+ json.dump(meta, f, indent=2)
+ self.chunks.clear()
+ return
+
+ meta_backend_kwargs = meta.get("backend_kwargs", {})
+ if backend_name == "hnsw":
+ index_is_compact = meta.get("is_compact", meta_backend_kwargs.get("is_compact", True))
+ if index_is_compact:
+ raise ValueError(
+ "Compact HNSW indices do not support in-place updates. Rebuild required."
+ )
+
+ distance_metric = meta_backend_kwargs.get(
+ "distance_metric", self.backend_kwargs.get("distance_metric", "mips")
+ ).lower()
+ needs_recompute = bool(
+ meta.get("is_pruned")
+ or meta_backend_kwargs.get("is_recompute")
+ or self.backend_kwargs.get("is_recompute")
+ )
+
+ valid_chunks: list[dict[str, Any]] = []
+ for chunk in self.chunks:
+ text = chunk.get("text", "")
+ if not isinstance(text, str) or not text.strip():
+ continue
+ metadata = chunk.setdefault("metadata", {})
+ passage_id = chunk.get("id") or metadata.get("id")
+ if passage_id and passage_id in existing_ids:
+ raise ValueError(f"Passage ID '{passage_id}' already exists in the index.")
+ valid_chunks.append(chunk)
+
+ if not valid_chunks:
+ # Remove-only or file emptied: we may have already removed ids, just update meta
+ meta["total_passages"] = len(offset_map)
+ with open(meta_path, "w", encoding="utf-8") as f:
+ json.dump(meta, f, indent=2)
+ self.chunks.clear()
+ return
+
+ texts_to_embed = [chunk["text"] for chunk in valid_chunks]
+ embeddings = compute_embeddings(
+ texts_to_embed,
+ self.embedding_model,
+ self.embedding_mode,
+ use_server=False,
+ is_build=True,
+ provider_options=self.embedding_options,
+ )
+
+ embedding_dim = embeddings.shape[1]
+ expected_dim = meta.get("dimensions")
+ if expected_dim is not None and expected_dim != embedding_dim:
+ raise ValueError(
+ f"Dimension mismatch during update: existing index uses {expected_dim}, got {embedding_dim}."
+ )
+
+ embeddings = np.ascontiguousarray(embeddings, dtype=np.float32)
+ if distance_metric == "cosine":
+ norms = np.linalg.norm(embeddings, axis=1, keepdims=True)
+ norms[norms == 0] = 1
+ embeddings = embeddings / norms
+
+ # IVF: add_vectors then append passages/offset (no ZMQ/server)
+ if backend_name == "ivf":
+ for i, chunk in enumerate(valid_chunks):
+ pid = chunk.get("id") or chunk.get("metadata", {}).get("id")
+ if not pid:
+ pid = str(len(offset_map) + i)
+ chunk.setdefault("metadata", {})["id"] = pid
+ chunk["id"] = pid
+ passage_ids = [c["id"] for c in valid_chunks]
+ try:
+ from leann_backend_ivf import add_vectors as ivf_add_vectors
+
+ ivf_add_vectors(str(path), embeddings, passage_ids)
+ except ImportError:
+ raise RuntimeError("IVF backend required. Install leann-backend-ivf.")
+ rollback_passages_size = passages_file.stat().st_size if passages_file.exists() else 0
+ offset_map_backup = offset_map.copy()
+ try:
+ with open(passages_file, "a", encoding="utf-8") as f:
+ for chunk in valid_chunks:
+ off = f.tell()
+ json.dump(
+ {
+ "id": chunk["id"],
+ "text": chunk["text"],
+ "metadata": chunk.get("metadata", {}),
+ },
+ f,
+ ensure_ascii=False,
+ )
+ f.write("\n")
+ offset_map[chunk["id"]] = off
+ with open(offset_file, "wb") as f:
+ pickle.dump(offset_map, f)
+ meta["total_passages"] = len(offset_map)
+ with open(meta_path, "w", encoding="utf-8") as f:
+ json.dump(meta, f, indent=2)
+ logger.info(
+ "Appended %d passages to IVF index '%s'. Total: %d",
+ len(valid_chunks),
+ index_path,
+ len(offset_map),
+ )
+ except Exception:
+ if passages_file.exists():
+ with open(passages_file, "rb+") as f:
+ f.truncate(rollback_passages_size)
+ offset_map = offset_map_backup
+ with open(offset_file, "wb") as f:
+ pickle.dump(offset_map, f)
+ raise
+ self.chunks.clear()
+ return
+
+ # HNSW path below
+ from leann_backend_hnsw import faiss
+
+ index = faiss.read_index(str(index_file))
+ if hasattr(index, "is_recompute"):
+ index.is_recompute = needs_recompute
+ print(f"index.is_recompute: {index.is_recompute}")
+ if getattr(index, "storage", None) is None:
+ if index.metric_type == faiss.METRIC_INNER_PRODUCT:
+ storage_index = faiss.IndexFlatIP(index.d)
+ else:
+ storage_index = faiss.IndexFlatL2(index.d)
+ index.storage = storage_index
+ index.own_fields = True
+ # Faiss expects storage.ntotal to reflect the existing graph's
+ # population (even if the vectors themselves were pruned from disk
+ # for recompute mode). When we attach a fresh IndexFlat here its
+ # ntotal starts at zero, which later causes IndexHNSW::add to
+ # believe new "preset" levels were provided and trips the
+ # `n0 + n == levels.size()` assertion. Seed the temporary storage
+ # with the current ntotal so Faiss maintains the proper offset for
+ # incoming vectors.
+ try:
+ storage_index.ntotal = index.ntotal
+ except AttributeError:
+ # Older Faiss builds may not expose ntotal as a writable
+ # attribute; in that case we fall back to the default behaviour.
+ pass
+ if index.d != embedding_dim:
+ raise ValueError(
+ f"Existing index dimension ({index.d}) does not match new embeddings ({embedding_dim})."
+ )
+
+ passage_meta_mode = meta.get("embedding_mode", self.embedding_mode)
+ passage_provider_options = meta.get("embedding_options", self.embedding_options)
+
+ base_id = index.ntotal
+ for offset, chunk in enumerate(valid_chunks):
+ new_id = str(base_id + offset)
+ chunk.setdefault("metadata", {})["id"] = new_id
+ chunk["id"] = new_id
+
+ # Append passages/offsets before we attempt index.add so the ZMQ server
+ # can resolve newly assigned IDs during recompute. Keep rollback hooks
+ # so we can restore files if the update fails mid-way.
+ rollback_passages_size = passages_file.stat().st_size if passages_file.exists() else 0
+ offset_map_backup = offset_map.copy()
+
+ try:
+ with open(passages_file, "a", encoding="utf-8") as f:
+ for chunk in valid_chunks:
+ offset = f.tell()
+ json.dump(
+ {
+ "id": chunk["id"],
+ "text": chunk["text"],
+ "metadata": chunk.get("metadata", {}),
+ },
+ f,
+ ensure_ascii=False,
+ )
+ f.write("\n")
+ offset_map[chunk["id"]] = offset
+
+ with open(offset_file, "wb") as f:
+ pickle.dump(offset_map, f)
+
+ server_manager: Optional[EmbeddingServerManager] = None
+ server_started = False
+ requested_zmq_port = int(os.getenv("LEANN_UPDATE_ZMQ_PORT", "5557"))
+
+ try:
+ if needs_recompute:
+ server_manager = EmbeddingServerManager(
+ backend_module_name="leann_backend_hnsw.hnsw_embedding_server"
+ )
+ server_started, actual_port = server_manager.start_server(
+ port=requested_zmq_port,
+ model_name=self.embedding_model,
+ embedding_mode=passage_meta_mode,
+ passages_file=str(meta_path),
+ distance_metric=distance_metric,
+ use_daemon=False,
+ enable_warmup=False,
+ provider_options=passage_provider_options,
+ )
+ if not server_started:
+ raise RuntimeError(
+ "Failed to start HNSW embedding server for recompute update."
+ )
+ if actual_port != requested_zmq_port:
+ logger.warning(
+ "Embedding server started on port %s instead of requested %s. "
+ "Using reassigned port.",
+ actual_port,
+ requested_zmq_port,
+ )
+ if hasattr(index.hnsw, "set_zmq_port"):
+ index.hnsw.set_zmq_port(actual_port)
+ elif hasattr(index, "set_zmq_port"):
+ index.set_zmq_port(actual_port)
+
+ if needs_recompute:
+ for i in range(embeddings.shape[0]):
+ print(f"add {i} embeddings")
+ index.add(1, faiss.swig_ptr(embeddings[i : i + 1]))
+ else:
+ index.add(embeddings.shape[0], faiss.swig_ptr(embeddings))
+ faiss.write_index(index, str(index_file))
+ finally:
+ if server_started and server_manager is not None:
+ server_manager.stop_server()
+
+ except Exception:
+ # Roll back appended passages/offset map to keep files consistent.
+ if passages_file.exists():
+ with open(passages_file, "rb+") as f:
+ f.truncate(rollback_passages_size)
+ offset_map = offset_map_backup
+ with open(offset_file, "wb") as f:
+ pickle.dump(offset_map, f)
+ raise
+
+ meta["total_passages"] = len(offset_map)
+ with open(meta_path, "w", encoding="utf-8") as f:
+ json.dump(meta, f, indent=2)
+
+ logger.info(
+ "Appended %d passages to index '%s'. New total: %d",
+ len(valid_chunks),
+ index_path,
+ len(offset_map),
+ )
+
+ self.chunks.clear()
+
+ if needs_recompute:
+ prune_hnsw_embeddings_inplace(str(index_file))
+
+
+class LeannSearcher:
+ def __init__(
+ self,
+ index_path: str,
+ enable_warmup: bool = True,
+ recompute_embeddings: bool = True,
+ use_daemon: bool = True,
+ daemon_ttl_seconds: int = 900,
+ **backend_kwargs,
+ ):
+ # Fix path resolution for Colab and other environments
+ if not Path(index_path).is_absolute():
+ index_path = str(Path(index_path).resolve())
+
+ self.meta_path_str = f"{index_path}.meta.json"
+ if not Path(self.meta_path_str).exists():
+ parent_dir = Path(index_path).parent
+ print(
+ f"Leann metadata file not found at {self.meta_path_str}, and you may need to rm -rf {parent_dir}"
+ )
+ # highlight in red the filenotfound error
+ raise FileNotFoundError(
+ f"Leann metadata file not found at {self.meta_path_str}, \033[91m you may need to rm -rf {parent_dir}\033[0m"
+ )
+ with open(self.meta_path_str, encoding="utf-8") as f:
+ self.meta_data = json.load(f)
+ backend_name = self.meta_data["backend_name"]
+ self.embedding_model = self.meta_data["embedding_model"]
+ # Support both old and new format
+ self.embedding_mode = self.meta_data.get("embedding_mode", "sentence-transformers")
+ self.embedding_options = self.meta_data.get("embedding_options", {})
+ # Delegate portability handling to PassageManager
+ self.passage_manager = PassageManager(
+ self.meta_data.get("passage_sources", []), metadata_file_path=self.meta_path_str
+ )
+ # Preserve backend name for conditional parameter forwarding
+ self.backend_name = backend_name
+ backend_factory = BACKEND_REGISTRY.get(backend_name)
+ if backend_factory is None:
+ raise ValueError(f"Backend '{backend_name}' not found.")
+
+ # Global recompute flag for this searcher (explicit knob, default True)
+ self.recompute_embeddings: bool = bool(recompute_embeddings)
+
+ # Warmup flag: keep using the existing enable_warmup parameter,
+ # but default it to True so cold-start happens earlier.
+ self._warmup: bool = bool(enable_warmup)
+ self._use_daemon: bool = bool(use_daemon)
+ self._daemon_ttl_seconds: int = int(daemon_ttl_seconds)
+
+ final_kwargs = {**self.meta_data.get("backend_kwargs", {}), **backend_kwargs}
+ final_kwargs["enable_warmup"] = self._warmup
+ final_kwargs["use_daemon"] = self._use_daemon
+ final_kwargs["daemon_ttl_seconds"] = self._daemon_ttl_seconds
+ if self.embedding_options:
+ final_kwargs.setdefault("embedding_options", self.embedding_options)
+ self.backend_impl: LeannBackendSearcherInterface = backend_factory.searcher(
+ index_path, **final_kwargs
+ )
+ self.bm25_scorer: Optional[BM25Scorer] = None
+
+ # Optional one-shot warmup at construction time to hide cold-start latency.
+ if self._warmup:
+ self.warmup()
+
+ def warmup(self) -> None:
+ """Warm up embedding path so first user query is faster."""
+ try:
+ _ = self.backend_impl.compute_query_embedding(
+ "__LEANN_WARMUP__",
+ use_server_if_available=self.recompute_embeddings,
+ )
+ except Exception as exc:
+ logger.warning(f"Warmup embedding failed (ignored): {exc}")
+
+ def search(
+ self,
+ query: str,
+ top_k: int = 5,
+ complexity: int = 64,
+ beam_width: int = 1,
+ prune_ratio: float = 0.0,
+ recompute_embeddings: Optional[bool] = None,
+ pruning_strategy: Literal["global", "local", "proportional"] = "global",
+ expected_zmq_port: int = 5557,
+ metadata_filters: Optional[dict[str, dict[str, Union[str, int, float, bool, list]]]] = None,
+ batch_size: int = 0,
+ use_grep: bool = False,
+ gemma: float = 1.0,
+ provider_options: Optional[dict[str, Any]] = None,
+ **kwargs,
+ ) -> list[SearchResult]:
+ """
+ Search for nearest neighbors with optional metadata filtering.
+
+ Args:
+ query: Text query to search for
+ top_k: Number of nearest neighbors to return
+ complexity: Search complexity/candidate list size, higher = more accurate but slower
+ beam_width: Number of parallel search paths/IO requests per iteration
+ prune_ratio: Ratio of neighbors to prune via approximate distance (0.0-1.0)
+ recompute_embeddings: (Deprecated) Per-call override for recompute mode.
+ Configure this at LeannSearcher(..., recompute_embeddings=...) instead.
+ pruning_strategy: Candidate selection strategy - "global" (default), "local", or "proportional"
+ expected_zmq_port: ZMQ port for embedding server communication
+ metadata_filters: Optional filters to apply to search results based on metadata.
+ Format: {"field_name": {"operator": value}}
+ Supported operators:
+ - Comparison: "==", "!=", "<", "<=", ">", ">="
+ - Membership: "in", "not_in"
+ - String: "contains", "starts_with", "ends_with"
+ Example: {"chapter": {"<=": 5}, "tags": {"in": ["fiction", "drama"]}}
+ gemma: Weight of vector search results in hybrid search (0.0-1.0), 1 = pure vector search, 0 = pure keyword search
+ **kwargs: Backend-specific parameters
+
+ Returns:
+ List of SearchResult objects with text, metadata, and similarity scores
+ """
+ # Handle grep search
+ if use_grep:
+ return self._grep_search(query, top_k)
+
+ logger.info("🔍 LeannSearcher.search() called:")
+ logger.info(f" Query: '{query}'")
+ logger.info(f" Top_k: {top_k}")
+ logger.info(f" Metadata filters: {metadata_filters}")
+ logger.info(f" Additional kwargs: {kwargs}")
+
+ # Smart top_k detection and adjustment
+ # Use PassageManager length (sum of shard sizes) to avoid
+ # depending on a massive combined map
+ total_docs = len(self.passage_manager)
+ original_top_k = top_k
+ if top_k > total_docs:
+ top_k = total_docs
+ logger.warning(
+ f" ⚠️ Requested top_k ({original_top_k}) exceeds total documents ({total_docs})"
+ )
+ logger.warning(f" ✅ Auto-adjusted top_k to {top_k} to match available documents")
+
+ # Handle pure keyword search
+ if gemma == 0.0:
+ start_time = time.time()
+ bm25_results = self._bm25_search(query, top_k)
+ # Convert BM25 results to the expected format
+ results = {
+ "labels": [[r.id for r in bm25_results]],
+ "distances": [[r.score for r in bm25_results]],
+ }
+ else:
+ # Perform vector search
+ zmq_port = None
+
+ # Resolve effective recompute flag for this search.
+ if recompute_embeddings is not None:
+ logger.warning(
+ "LeannSearcher.search(..., recompute_embeddings=...) is deprecated and "
+ "will be removed in a future version. Configure recompute at "
+ "LeannSearcher(..., recompute_embeddings=...) instead."
+ )
+ effective_recompute = bool(recompute_embeddings)
+ else:
+ effective_recompute = self.recompute_embeddings
+
+ start_time = time.time()
+ if effective_recompute:
+ zmq_port = self.backend_impl._ensure_server_running(
+ self.meta_path_str,
+ port=expected_zmq_port,
+ enable_warmup=self._warmup,
+ use_daemon=self._use_daemon,
+ daemon_ttl_seconds=self._daemon_ttl_seconds,
+ **kwargs,
+ )
+ del expected_zmq_port
+ zmq_time = time.time() - start_time
+ logger.info(f" Launching server time: {zmq_time} seconds")
+
+ start_time = time.time()
+
+ # Extract query template from stored embedding_options with fallback chain:
+ # 1. Check provider_options override (highest priority)
+ # 2. Check query_prompt_template (new format)
+ # 3. Check prompt_template (old format for backward compat)
+ # 4. None (no template)
+ query_template = None
+ if provider_options and "prompt_template" in provider_options:
+ query_template = provider_options["prompt_template"]
+ elif "query_prompt_template" in self.embedding_options:
+ query_template = self.embedding_options["query_prompt_template"]
+ elif "prompt_template" in self.embedding_options:
+ query_template = self.embedding_options["prompt_template"]
+
+ query_embedding = self.backend_impl.compute_query_embedding(
+ query,
+ use_server_if_available=effective_recompute,
+ zmq_port=zmq_port,
+ query_template=query_template,
+ )
+ logger.info(f" Generated embedding shape: {query_embedding.shape}")
+ embedding_time = time.time() - start_time
+ logger.info(f" Embedding time: {embedding_time} seconds")
+
+ start_time = time.time()
+ backend_search_kwargs: dict[str, Any] = {
+ "complexity": complexity,
+ "beam_width": beam_width,
+ "prune_ratio": prune_ratio,
+ "recompute_embeddings": effective_recompute,
+ "pruning_strategy": pruning_strategy,
+ "zmq_port": zmq_port,
+ }
+ # Only HNSW supports batching; forward conditionally
+ if self.backend_name == "hnsw":
+ backend_search_kwargs["batch_size"] = batch_size
+
+ # Merge any extra kwargs last
+ backend_search_kwargs.update(kwargs)
+
+ results = self.backend_impl.search(
+ query_embedding,
+ top_k,
+ **backend_search_kwargs,
+ )
+
+ # Handle hybrid search
+ if 0.0 < gemma < 1.0:
+ logger.info(f" 🌟 Hybrid search enabled with gemma={gemma}")
+ BM25_WEIGHT = 1.0 - gemma
+ bm25_results = self._bm25_search(query, top_k)
+ hybrid_scores: dict[str, float] = {}
+ # Add vector search scores (weighted by gemma)
+ if "labels" in results and "distances" in results:
+ for doc_id, score in zip(results["labels"][0], results["distances"][0]):
+ hybrid_scores[doc_id] = gemma * score
+ # Add BM25 scores (weighted by BM25_WEIGHT)
+ for bm25_result in bm25_results:
+ doc_id = bm25_result.id
+ if doc_id in hybrid_scores:
+ hybrid_scores[doc_id] += BM25_WEIGHT * bm25_result.score
+ else:
+ hybrid_scores[doc_id] = BM25_WEIGHT * bm25_result.score
+
+ sorted_hybrid = sorted(hybrid_scores.items(), key=lambda x: x[1], reverse=True)[:top_k]
+ results["labels"] = [[doc_id for doc_id, _ in sorted_hybrid]]
+ results["distances"] = [[score for _, score in sorted_hybrid]]
+
+ logger.info(
+ f" Combined {len(hybrid_scores)} unique documents from vector and BM25 search"
+ )
+
+ search_time = time.time() - start_time
+ logger.info(f" Search time in search() LEANN searcher: {search_time} seconds")
+ logger.info(f" Backend returned: labels={len(results.get('labels', [[]])[0])} results")
+
+ enriched_results = []
+ if "labels" in results and "distances" in results:
+ logger.info(f" Processing {len(results['labels'][0])} passage IDs:")
+ # Python 3.9 does not support zip(strict=...); lengths are expected to match
+ for i, (string_id, dist) in enumerate(
+ zip(results["labels"][0], results["distances"][0])
+ ):
+ try:
+ passage_data = self.passage_manager.get_passage(string_id)
+ enriched_results.append(
+ SearchResult(
+ id=string_id,
+ score=dist,
+ text=passage_data["text"],
+ metadata=passage_data.get("metadata", {}),
+ )
+ )
+
+ # Color codes for better logging
+ GREEN = "\033[92m"
+ BLUE = "\033[94m"
+ YELLOW = "\033[93m"
+ RESET = "\033[0m"
+
+ # Truncate text for display (first 100 chars)
+ display_text = passage_data["text"]
+ logger.info(
+ f" {GREEN}✓{RESET} {BLUE}[{i + 1:2d}]{RESET} {YELLOW}ID:{RESET} '{string_id}' {YELLOW}Score:{RESET} {dist:.4f} {YELLOW}Text:{RESET} {display_text}"
+ )
+ except KeyError:
+ RED = "\033[91m"
+ RESET = "\033[0m"
+ logger.error(
+ f" {RED}✗{RESET} [{i + 1:2d}] ID: '{string_id}' -> {RED}ERROR: Passage not found!{RESET}"
+ )
+
+ # Apply metadata filters if specified
+ if metadata_filters:
+ logger.info(f" 🔍 Applying metadata filters: {metadata_filters}")
+ enriched_results = self.passage_manager.filter_search_results(
+ enriched_results, metadata_filters
+ )
+
+ # Define color codes outside the loop for final message
+ GREEN = "\033[92m"
+ RESET = "\033[0m"
+ logger.info(f" {GREEN}✓ Final enriched results: {len(enriched_results)} passages{RESET}")
+ return enriched_results
+
+ def _init_bm25(self) -> None:
+ """Initialize BM25 scorer"""
+ self.bm25_scorer = BM25Scorer()
+ # Load all the files directly
+ passages = []
+ for passage_file in self.passage_manager.passage_files.values():
+ with open(passage_file, encoding="utf-8") as f:
+ for line in f:
+ if line.strip():
+ data = json.loads(line)
+ passages.append(data)
+ self.bm25_scorer.fit(passages)
+
+ def _bm25_search(self, query: str, top_k: int = 5) -> list[SearchResult]:
+ """Perform BM25 search on raw passages"""
+ if self.bm25_scorer is None:
+ self._init_bm25()
+ logger.info(" BM25 scorer initialized")
+ scorer = self.bm25_scorer
+ if scorer is None:
+ raise RuntimeError("BM25 scorer failed to initialize")
+ return scorer.search(query, top_k)
+
+ def _find_jsonl_file(self) -> Optional[str]:
+ """Find the .jsonl file containing raw passages for grep search"""
+ index_path = Path(self.meta_path_str).parent
+ potential_files = [
+ index_path / "documents.leann.passages.jsonl",
+ index_path.parent / "documents.leann.passages.jsonl",
+ ]
+
+ for file_path in potential_files:
+ if file_path.exists():
+ return str(file_path)
+ return None
+
+ def _grep_search(self, query: str, top_k: int = 5) -> list[SearchResult]:
+ """Perform grep-based search on raw passages"""
+ jsonl_file = self._find_jsonl_file()
+ if not jsonl_file:
+ raise FileNotFoundError("No .jsonl passages file found for grep search")
+
+ try:
+ cmd = ["grep", "-i", "-n", query, jsonl_file]
+ result = subprocess.run(cmd, capture_output=True, text=True, check=False)
+
+ if result.returncode == 1:
+ return []
+ elif result.returncode != 0:
+ raise RuntimeError(f"Grep failed: {result.stderr}")
+
+ matches = []
+ for line in result.stdout.strip().split("\n"):
+ if not line:
+ continue
+ parts = line.split(":", 1)
+ if len(parts) != 2:
+ continue
+
+ try:
+ data = json.loads(parts[1])
+ text = data.get("text", "")
+ score = text.lower().count(query.lower())
+
+ matches.append(
+ SearchResult(
+ id=data.get("id", parts[0]),
+ text=text,
+ metadata=data.get("metadata", {}),
+ score=float(score),
+ )
+ )
+ except json.JSONDecodeError:
+ continue
+
+ matches.sort(key=lambda x: x.score, reverse=True)
+ return matches[:top_k]
+
+ except FileNotFoundError:
+ raise RuntimeError(
+ "grep command not found. Please install grep or use semantic search."
+ )
+
+ def _python_regex_search(self, query: str, top_k: int = 5) -> list[SearchResult]:
+ """Fallback regex search"""
+ jsonl_file = self._find_jsonl_file()
+ if not jsonl_file:
+ raise FileNotFoundError("No .jsonl file found")
+
+ pattern = re.compile(re.escape(query), re.IGNORECASE)
+ matches = []
+
+ with open(jsonl_file, encoding="utf-8") as f:
+ for line_num, line in enumerate(f, 1):
+ if pattern.search(line):
+ try:
+ data = json.loads(line.strip())
+ matches.append(
+ SearchResult(
+ id=data.get("id", str(line_num)),
+ text=data.get("text", ""),
+ metadata=data.get("metadata", {}),
+ score=float(len(pattern.findall(data.get("text", "")))),
+ )
+ )
+ except json.JSONDecodeError:
+ continue
+
+ matches.sort(key=lambda x: x.score, reverse=True)
+ return matches[:top_k]
+
+ def cleanup(self):
+ """Explicitly cleanup embedding server resources.
+ This method should be called after you're done using the searcher,
+ especially in test environments or batch processing scenarios.
+ """
+ backend = getattr(self.backend_impl, "embedding_server_manager", None)
+ if backend is not None:
+ backend.stop_server()
+
+ # Enable automatic cleanup patterns
+ def __enter__(self):
+ return self
+
+ def __exit__(self, exc_type, exc, tb):
+ try:
+ self.cleanup()
+ except Exception:
+ pass
+
+ def __del__(self):
+ try:
+ self.cleanup()
+ except Exception:
+ # Avoid noisy errors during interpreter shutdown
+ pass
+
+
+class LeannChat:
+ def __init__(
+ self,
+ index_path: str,
+ llm_config: Optional[dict[str, Any]] = None,
+ enable_warmup: bool = False,
+ searcher: Optional[LeannSearcher] = None,
+ **kwargs,
+ ):
+ if searcher is None:
+ self.searcher = LeannSearcher(index_path, enable_warmup=enable_warmup, **kwargs)
+ self._owns_searcher = True
+ else:
+ self.searcher = searcher
+ self._owns_searcher = False
+ self.llm = get_llm(llm_config)
+
+ def ask(
+ self,
+ question: str,
+ top_k: int = 5,
+ complexity: int = 64,
+ beam_width: int = 1,
+ prune_ratio: float = 0.0,
+ recompute_embeddings: bool = True,
+ pruning_strategy: Literal["global", "local", "proportional"] = "global",
+ llm_kwargs: Optional[dict[str, Any]] = None,
+ expected_zmq_port: int = 5557,
+ metadata_filters: Optional[dict[str, dict[str, Union[str, int, float, bool, list]]]] = None,
+ batch_size: int = 0,
+ use_grep: bool = False,
+ gemma: float = 1.0,
+ **search_kwargs,
+ ):
+ if llm_kwargs is None:
+ llm_kwargs = {}
+ search_time = time.time()
+ results = self.searcher.search(
+ question,
+ top_k=top_k,
+ complexity=complexity,
+ beam_width=beam_width,
+ prune_ratio=prune_ratio,
+ recompute_embeddings=recompute_embeddings,
+ pruning_strategy=pruning_strategy,
+ expected_zmq_port=expected_zmq_port,
+ metadata_filters=metadata_filters,
+ use_grep=use_grep,
+ gemma=gemma,
+ batch_size=batch_size,
+ **search_kwargs,
+ )
+ search_time = time.time() - search_time
+ logger.info(f" Search time: {search_time} seconds")
+ context = "\n\n".join([r.text for r in results])
+ prompt = (
+ "Here is some retrieved context that might help answer your question:\n\n"
+ f"{context}\n\n"
+ f"Question: {question}\n\n"
+ "Please provide the best answer you can based on this context and your knowledge."
+ )
+
+ logger.info("The context provided to the LLM is:")
+ logger.info(f"{'Relevance':<10} | {'Chunk id':<10} | {'Content':<60} | {'Source':<80}")
+ logger.info("-" * 150)
+ for r in results:
+ chunk_relevance = f"{r.score:.3f}"
+ chunk_id = r.id
+ chunk_content = r.text[:60]
+ chunk_source = r.metadata.get("source", "")[:80]
+ logger.info(
+ f"{chunk_relevance:<10} | {chunk_id:<10} | {chunk_content:<60} | {chunk_source:<80}"
+ )
+ ask_time = time.time()
+ ans = self.llm.ask(prompt, **llm_kwargs)
+ ask_time = time.time() - ask_time
+ logger.info(f" Ask time: {ask_time} seconds")
+ return ans
+
+ def start_interactive(self):
+ """Start interactive chat session."""
+ session = create_api_session()
+
+ def handle_query(user_input: str):
+ response = self.ask(user_input)
+ print(f"Leann: {response}")
+
+ session.run_interactive_loop(handle_query)
+
+ def cleanup(self):
+ """Explicitly cleanup embedding server resources.
+
+ This method should be called after you're done using the chat interface,
+ especially in test environments or batch processing scenarios.
+ """
+ # Only stop the embedding server if this LeannChat instance created the searcher.
+ # When a shared searcher is passed in, avoid shutting down the server to enable reuse.
+ if getattr(self, "_owns_searcher", False) and hasattr(self.searcher, "cleanup"):
+ self.searcher.cleanup()
+
+ # Enable automatic cleanup patterns
+ def __enter__(self):
+ return self
+
+ def __exit__(self, exc_type, exc, tb):
+ try:
+ self.cleanup()
+ except Exception:
+ pass
+
+ def __del__(self):
+ try:
+ self.cleanup()
+ except Exception:
+ pass
diff --git a/packages/leann-core/src/leann/cli.py b/packages/leann-core/src/leann/cli.py
index b4e4046..01e6c0b 100644
--- a/packages/leann-core/src/leann/cli.py
+++ b/packages/leann-core/src/leann/cli.py
@@ -26,38 +26,14 @@ from .settings import (
)
from .sync import FileSynchronizer
-# Manifest file for incremental build: maps source file path -> mtime
-SOURCES_MANIFEST_FILENAME = "documents.leann.sources.json"
-
def _normalize_path(path: str) -> str:
- """Return absolute path string for consistent manifest keys."""
+ """Return absolute path string for consistent keys."""
if not path:
return path
return str(Path(path).resolve())
-def load_sources_manifest(index_dir: Path, index_name: str) -> dict[str, float]:
- """Load sources manifest (path -> mtime). Returns {} if file missing or invalid."""
- manifest_path = index_dir / SOURCES_MANIFEST_FILENAME
- if not manifest_path.exists():
- return {}
- try:
- with open(manifest_path, encoding="utf-8") as f:
- data = json.load(f)
- return data.get("sources", {})
- except (json.JSONDecodeError, OSError):
- return {}
-
-
-def save_sources_manifest(index_dir: Path, index_name: str, sources: dict[str, float]) -> None:
- """Write sources manifest (path -> mtime)."""
- manifest_path = index_dir / SOURCES_MANIFEST_FILENAME
- index_dir.mkdir(parents=True, exist_ok=True)
- with open(manifest_path, "w", encoding="utf-8") as f:
- json.dump({"sources": sources}, f, indent=2)
-
-
@contextlib.contextmanager
def suppress_cpp_output(suppress: bool = True):
"""Context manager to suppress C++ stdout/stderr output from FAISS/HNSW
@@ -367,8 +343,8 @@ Examples:
watch_parser.add_argument(
"--interval",
type=int,
- default=5,
- help="Poll interval in seconds (default: 5)",
+ default=30,
+ help="Poll interval in seconds (default: 30)",
)
watch_parser.add_argument(
"--once",
@@ -1687,22 +1663,239 @@ Examples:
roots.add(str(path_obj.parent))
return sorted(roots)
- def _initialize_file_synchronizers(
+ def _create_synchronizers(
self,
+ index_dir: Path,
roots: list[str],
- include_extensions: Optional[list[str]],
- ignore_patterns: Optional[list[str]],
- ) -> None:
+ include_extensions: Optional[list[str]] = None,
+ ignore_patterns: Optional[list[str]] = None,
+ ) -> list[FileSynchronizer]:
+ """Create FileSynchronizers with snapshots stored in the index dir. Shared by build and watch."""
+ synchronizers: list[FileSynchronizer] = []
for root in roots:
+ tag = hashlib.sha256(root.encode()).hexdigest()[:12]
+ snapshot_path = str(index_dir / f"sync_{tag}.pickle")
try:
- FileSynchronizer(
+ fs = FileSynchronizer(
root_dir=root,
ignore_patterns=ignore_patterns,
include_extensions=include_extensions,
- auto_load=True,
+ snapshot_path=snapshot_path,
)
+ synchronizers.append(fs)
except Exception as exc:
- print(f"⚠️ Failed to initialize file synchronizer for {root}: {exc}")
+ print(f"Warning: Failed to init synchronizer for {root}: {exc}")
+ return synchronizers
+
+ def _build_synchronizers(
+ self,
+ docs_paths: list[str],
+ index_dir: Path,
+ file_types: Optional[str] = None,
+ include_hidden: bool = False,
+ ) -> list[FileSynchronizer]:
+ """Create FileSynchronizers for build from docs_paths."""
+ roots = self._resolve_sync_roots(docs_paths)
+ include_extensions = self._parse_file_types(file_types)
+ ignore_patterns = self._sync_ignore_patterns(include_hidden)
+ return self._create_synchronizers(index_dir, roots, include_extensions, ignore_patterns)
+
+ def _detect_build_changes(
+ self,
+ synchronizers: list[FileSynchronizer],
+ ) -> tuple[set[str], set[str], set[str]]:
+ """Detect added/removed/modified files across all source roots using content hashes."""
+ all_added: set[str] = set()
+ all_removed: set[str] = set()
+ all_modified: set[str] = set()
+ for fs in synchronizers:
+ added, removed, modified = fs.detect_changes()
+ all_added.update(added)
+ all_removed.update(removed)
+ all_modified.update(modified)
+ return all_added, all_removed, all_modified
+
+ def _commit_synchronizers(self, synchronizers: list[FileSynchronizer]) -> None:
+ """Persist all synchronizer snapshots after a successful build."""
+ for fs in synchronizers:
+ fs.commit()
+
+ @staticmethod
+ def _assign_chunk_ids(chunks: list[dict]) -> None:
+ """Assign stable IDs to chunks based on their file path and position."""
+ from collections import defaultdict
+
+ by_path: dict[str, list] = defaultdict(list)
+ for c in chunks:
+ p = c.get("metadata", {}).get("file_path") or c.get("metadata", {}).get("source") or ""
+ by_path[_normalize_path(p)].append(c)
+ for path_key, path_chunks in by_path.items():
+ for idx, c in enumerate(path_chunks):
+ sid = hashlib.sha256(f"{path_key}:{idx}".encode()).hexdigest()[:16]
+ c.setdefault("metadata", {})["id"] = sid
+ c["id"] = sid
+
+ def _chunks_for_paths(self, all_texts: list[dict], paths: set[str]) -> list[dict]:
+ """Filter chunks belonging to the given file paths."""
+ return [
+ c
+ for c in all_texts
+ if _normalize_path(
+ c.get("metadata", {}).get("file_path") or c.get("metadata", {}).get("source") or ""
+ )
+ in paths
+ ]
+
+ def _make_incremental_builder(self, args) -> "LeannBuilder":
+ return LeannBuilder(
+ backend_name=args.backend_name,
+ embedding_model=args.embedding_model,
+ embedding_mode=args.embedding_mode,
+ embedding_options=self._build_embedding_options(args) or None,
+ graph_degree=args.graph_degree,
+ complexity=args.complexity,
+ is_compact=args.compact,
+ is_recompute=args.recompute,
+ num_threads=args.num_threads,
+ )
+
+ def _incremental_add_only(
+ self,
+ index_path: str,
+ all_texts: list[dict],
+ args,
+ new_paths: set[str],
+ ) -> bool:
+ """Add-only incremental update (works for HNSW and IVF)."""
+ new_chunks = self._chunks_for_paths(all_texts, new_paths)
+ if not new_chunks:
+ return False
+ self._assign_chunk_ids(new_chunks)
+ builder = self._make_incremental_builder(args)
+ for chunk in new_chunks:
+ builder.add_text(chunk["text"], metadata=chunk["metadata"])
+ print(
+ f"Incremental update: adding {len(new_chunks)} chunks from {len(new_paths)} new file(s)..."
+ )
+ builder.update_index(index_path)
+ print(f"Index updated at {index_path}")
+ return True
+
+ def _incremental_ivf_remove_only(
+ self, index_path: str, index_dir: Path, removed_paths: set[str], args
+ ) -> bool:
+ """IVF remove-only fast path: remove chunk IDs without loading or chunking documents."""
+ passages_file = index_dir / "documents.leann.passages.jsonl"
+ if not passages_file.exists():
+ return False
+ chunk_ids_by_file = self._load_chunk_ids_by_file(passages_file)
+ ids_to_remove: list[str] = []
+ for p in removed_paths:
+ ids_to_remove.extend(chunk_ids_by_file.get(p, []))
+ if not ids_to_remove:
+ return False
+ print(
+ f"Incremental IVF update (-{len(removed_paths)} removed): removing {len(ids_to_remove)} old chunks..."
+ )
+ builder = self._make_incremental_builder(args)
+ builder.update_index(index_path, remove_passage_ids=ids_to_remove)
+ print(f"Index updated at {index_path}")
+ return True
+
+ def _incremental_ivf_update(
+ self,
+ index_path: str,
+ index_dir: Path,
+ all_texts: list[dict],
+ args,
+ new_paths: set[str],
+ removed_paths: set[str],
+ modified_paths: set[str],
+ ) -> bool:
+ """IVF incremental update: remove old chunks for modified/removed files, add new chunks."""
+ passages_file = index_dir / "documents.leann.passages.jsonl"
+ chunk_ids_by_file = (
+ self._load_chunk_ids_by_file(passages_file) if passages_file.exists() else {}
+ )
+
+ # Collect old chunk IDs to remove (modified + removed files)
+ ids_to_remove: list[str] = []
+ for p in modified_paths | removed_paths:
+ ids_to_remove.extend(chunk_ids_by_file.get(p, []))
+
+ # Collect new chunks to add (modified + new files)
+ changed_paths = new_paths | modified_paths
+ new_chunks = self._chunks_for_paths(all_texts, changed_paths)
+ self._assign_chunk_ids(new_chunks)
+
+ if not ids_to_remove and not new_chunks:
+ return False
+
+ builder = self._make_incremental_builder(args)
+ for chunk in new_chunks:
+ builder.add_text(chunk["text"], metadata=chunk["metadata"])
+
+ parts = []
+ if ids_to_remove:
+ parts.append(f"removing {len(ids_to_remove)} old chunks")
+ if new_chunks:
+ parts.append(f"adding {len(new_chunks)} new chunks")
+ file_parts = []
+ if new_paths:
+ file_parts.append(f"+{len(new_paths)} added")
+ if modified_paths:
+ file_parts.append(f"~{len(modified_paths)} modified")
+ if removed_paths:
+ file_parts.append(f"-{len(removed_paths)} removed")
+ print(f"Incremental IVF update ({', '.join(file_parts)}): {', '.join(parts)}...")
+
+ builder.update_index(
+ index_path, remove_passage_ids=ids_to_remove if ids_to_remove else None
+ )
+ print(f"Index updated at {index_path}")
+ return True
+
+ @staticmethod
+ def _log_rebuild_reason(
+ meta: dict, args, new_paths: set, removed_paths: set, modified_paths: set
+ ) -> None:
+ """Print a human-readable explanation of why incremental update is not possible."""
+ if removed_paths or modified_paths:
+ reasons = []
+ if removed_paths:
+ reasons.append(f"{len(removed_paths)} file(s) removed")
+ if modified_paths:
+ reasons.append(f"{len(modified_paths)} file(s) modified")
+ print(
+ f"Incremental update not possible ({', '.join(reasons)}); falling back to full rebuild."
+ )
+ return
+
+ blockers = []
+ if meta.get("backend_name") not in ("hnsw", "ivf"):
+ blockers.append(
+ f"backend '{meta.get('backend_name')}' does not support incremental updates"
+ )
+ if meta.get("is_compact", meta.get("backend_kwargs", {}).get("is_compact", True)):
+ blockers.append("index is compact (read-only); rebuild with --no-compact to enable")
+ if meta.get("embedding_model") != args.embedding_model:
+ blockers.append(
+ f"embedding model changed ('{meta.get('embedding_model')}' -> '{args.embedding_model}')"
+ )
+ if meta.get("embedding_mode") != args.embedding_mode:
+ blockers.append(
+ f"embedding mode changed ('{meta.get('embedding_mode')}' -> '{args.embedding_mode}')"
+ )
+ if blockers:
+ print(
+ f"Incremental update not possible: {'; '.join(blockers)}. Falling back to full rebuild."
+ )
+ else:
+ changes = []
+ if new_paths:
+ changes.append(f"+{len(new_paths)} added")
+ summary = ", ".join(changes) if changes else "unknown reason"
+ print(f"Full rebuild starting ({summary})...")
def _write_sync_config(
self,
@@ -1838,161 +2031,144 @@ Examples:
paragraph_separator="\n\n",
)
- all_texts = self.load_documents(
- docs_paths, args.file_types, include_hidden=args.include_hidden, args=args
+ # Detect changes first so we can skip load_documents for remove-only
+ index_dir.mkdir(parents=True, exist_ok=True)
+ synchronizers = self._build_synchronizers(
+ docs_paths, index_dir, file_types=args.file_types, include_hidden=args.include_hidden
)
- if not all_texts:
- print("No documents found")
- return
- # Idempotent build: detect changes and choose the minimal work path
- if index_dir.exists() and not args.force:
- manifest = load_sources_manifest(index_dir, "documents.leann")
+ if index_dir.exists() and not args.force and synchronizers:
meta_path = index_dir / "documents.leann.meta.json"
+ new_paths, removed_paths, modified_paths = self._detect_build_changes(synchronizers)
- current_sources: dict[str, float] = {}
- for c in all_texts:
- path = c.get("metadata", {}).get("file_path") or c.get("metadata", {}).get("source")
- if path and os.path.isfile(path):
- current_sources[_normalize_path(path)] = os.path.getmtime(path)
-
- if not manifest or not meta_path.exists():
- print("Rebuilding index (legacy format detected, enabling incremental support)...")
- else:
- new_paths = {p for p in current_sources if p not in manifest}
- removed_paths = {p for p in manifest if p not in current_sources}
- modified_paths = {
- p
- for p in current_sources
- if p in manifest and current_sources[p] != manifest[p]
- }
-
- if not new_paths and not removed_paths and not modified_paths:
- print("Index up to date.")
- return
+ if not new_paths and not removed_paths and not modified_paths:
+ print("Index up to date.")
+ return
+ # Show change summary (add / modify / delete) before build
+ change_parts = []
+ if new_paths:
+ change_parts.append(f"+{len(new_paths)} added")
+ if modified_paths:
+ change_parts.append(f"~{len(modified_paths)} modified")
+ if removed_paths:
+ change_parts.append(f"-{len(removed_paths)} removed")
+ if change_parts:
+ print(f"Changes: {', '.join(change_parts)}")
+
+ if meta_path.exists():
with open(meta_path, encoding="utf-8") as f:
meta = json.load(f)
- can_incremental = (
+ backend_name = meta.get("backend_name")
+ is_compact = meta.get(
+ "is_compact", meta.get("backend_kwargs", {}).get("is_compact", True)
+ )
+ same_embedding = (
+ meta.get("embedding_model") == args.embedding_model
+ and meta.get("embedding_mode") == args.embedding_mode
+ )
+
+ # IVF supports remove+add, so it can handle modified and removed files incrementally
+ can_ivf_update = backend_name == "ivf" and not is_compact and same_embedding
+ # HNSW only supports add (no remove), so it needs add-only changes
+ can_add_only = (
not removed_paths
and not modified_paths
- and meta.get("backend_name") in ("hnsw", "ivf")
- and not meta.get(
- "is_compact", meta.get("backend_kwargs", {}).get("is_compact", True)
- )
- and meta.get("embedding_model") == args.embedding_model
- and meta.get("embedding_mode") == args.embedding_mode
+ and backend_name in ("hnsw", "ivf")
+ and not is_compact
+ and same_embedding
)
- if not can_incremental and (removed_paths or modified_paths):
- reasons = []
- if removed_paths:
- reasons.append(f"{len(removed_paths)} file(s) removed")
- if modified_paths:
- reasons.append(f"{len(modified_paths)} file(s) modified")
- print(
- f"Incremental update not possible ({', '.join(reasons)}); "
- f"falling back to full rebuild."
+ # Remove-only fast path: no load/chunk, just remove IDs from index
+ if can_ivf_update and removed_paths and not new_paths and not modified_paths:
+ result = self._incremental_ivf_remove_only(
+ index_path, index_dir, removed_paths, args
)
- elif not can_incremental:
- blockers = []
- if meta.get("backend_name") not in ("hnsw", "ivf"):
- blockers.append(
- f"backend '{meta.get('backend_name')}' does not support incremental updates"
- )
- if meta.get(
- "is_compact", meta.get("backend_kwargs", {}).get("is_compact", True)
- ):
- blockers.append(
- "index is compact (read-only); rebuild with --no-compact to enable incremental updates"
- )
- if meta.get("embedding_model") != args.embedding_model:
- blockers.append(
- f"embedding model changed ('{meta.get('embedding_model')}' -> '{args.embedding_model}')"
- )
- if meta.get("embedding_mode") != args.embedding_mode:
- blockers.append(
- f"embedding mode changed ('{meta.get('embedding_mode')}' -> '{args.embedding_mode}')"
- )
- if blockers:
- print(
- f"Incremental update not possible: {'; '.join(blockers)}. Falling back to full rebuild."
+ if result:
+ self._commit_synchronizers(synchronizers)
+ self._write_sync_config(
+ index_dir,
+ self._resolve_sync_roots(docs_paths),
+ self._parse_file_types(args.file_types),
+ self._sync_ignore_patterns(args.include_hidden),
)
+ self.register_project_dir()
+ return
+
+ # Load only changed files (no need to load/chunk the entire corpus)
+ paths_to_load = new_paths | modified_paths
+ all_texts = self.load_documents(
+ list(paths_to_load),
+ args.file_types,
+ include_hidden=args.include_hidden,
+ args=args,
+ )
+ # Proceed even when all_texts is empty (e.g. file emptied): we still need to remove old chunks
+ if not all_texts and not (can_ivf_update and (modified_paths or removed_paths)):
+ print("No documents found")
+ return
- if can_incremental and new_paths:
- new_chunks = [
- c
- for c in all_texts
- if _normalize_path(
- c.get("metadata", {}).get("file_path")
- or c.get("metadata", {}).get("source")
- or ""
- )
- in new_paths
- ]
- from collections import defaultdict
-
- by_path: dict[str, list] = defaultdict(list)
- for c in new_chunks:
- path = (
- c.get("metadata", {}).get("file_path")
- or c.get("metadata", {}).get("source")
- or ""
- )
- by_path[_normalize_path(path)].append(c)
- for path_key, path_chunks in by_path.items():
- for idx, c in enumerate(path_chunks):
- sid = hashlib.sha256(f"{path_key}:{idx}".encode()).hexdigest()[:16]
- c.setdefault("metadata", {})["id"] = sid
- c["id"] = sid
-
- embedding_options_inc = self._build_embedding_options(args)
- builder_inc = LeannBuilder(
- backend_name=args.backend_name,
- embedding_model=args.embedding_model,
- embedding_mode=args.embedding_mode,
- embedding_options=embedding_options_inc or None,
- graph_degree=args.graph_degree,
- complexity=args.complexity,
- is_compact=args.compact,
- is_recompute=args.recompute,
- num_threads=args.num_threads,
+ if can_ivf_update and (new_paths or modified_paths or removed_paths):
+ result = self._incremental_ivf_update(
+ index_path,
+ index_dir,
+ all_texts,
+ args,
+ new_paths,
+ removed_paths,
+ modified_paths,
)
- for chunk in new_chunks:
- builder_inc.add_text(chunk["text"], metadata=chunk["metadata"])
- print(
- f"Incremental update: adding {len(new_chunks)} chunks from {len(new_paths)} new file(s)..."
+ if result:
+ self._commit_synchronizers(synchronizers)
+ self._write_sync_config(
+ index_dir,
+ self._resolve_sync_roots(docs_paths),
+ self._parse_file_types(args.file_types),
+ self._sync_ignore_patterns(args.include_hidden),
+ )
+ self.register_project_dir()
+ return
+
+ elif can_add_only and new_paths:
+ result = self._incremental_add_only(
+ index_path,
+ all_texts,
+ args,
+ new_paths,
)
- builder_inc.update_index(index_path)
- for p in new_paths:
- manifest[p] = current_sources[p]
- save_sources_manifest(index_dir, "documents.leann", manifest)
- print(f"Index updated at {index_path}")
- self.register_project_dir()
- return
+ if result:
+ self._commit_synchronizers(synchronizers)
+ self._write_sync_config(
+ index_dir,
+ self._resolve_sync_roots(docs_paths),
+ self._parse_file_types(args.file_types),
+ self._sync_ignore_patterns(args.include_hidden),
+ )
+ self.register_project_dir()
+ return
+
else:
- changes = []
- if new_paths:
- changes.append(f"+{len(new_paths)} added")
- if modified_paths:
- changes.append(f"~{len(modified_paths)} modified")
- if removed_paths:
- changes.append(f"-{len(removed_paths)} removed")
- summary = ", ".join(changes) if changes else "incompatible index format"
- print(f"Full rebuild starting ({summary})...")
+ self._log_rebuild_reason(meta, args, new_paths, removed_paths, modified_paths)
- index_dir.mkdir(parents=True, exist_ok=True)
+ # Full rebuild: load documents if not already loaded (first build or force)
+ try:
+ _ = all_texts
+ except NameError:
+ all_texts = self.load_documents(
+ docs_paths, args.file_types, include_hidden=args.include_hidden, args=args
+ )
+ if not all_texts:
+ print("No documents found")
+ return
print(f"Building index '{index_name}' with {args.backend_name} backend...")
- embedding_options = self._build_embedding_options(args)
-
builder = LeannBuilder(
backend_name=args.backend_name,
embedding_model=args.embedding_model,
embedding_mode=args.embedding_mode,
- embedding_options=embedding_options or None,
+ embedding_options=self._build_embedding_options(args) or None,
graph_degree=args.graph_degree,
complexity=args.complexity,
is_compact=args.compact,
@@ -2004,28 +2180,19 @@ Examples:
builder.add_text(chunk["text"], metadata=chunk["metadata"])
builder.build_index(index_path)
- # Write sources manifest for future incremental builds
- full_sources = {}
- for c in all_texts:
- path = c.get("metadata", {}).get("file_path") or c.get("metadata", {}).get("source")
- if path and os.path.isfile(path):
- full_sources[_normalize_path(path)] = os.path.getmtime(path)
- if full_sources:
- save_sources_manifest(index_dir, "documents.leann", full_sources)
+ for fs in synchronizers:
+ fs.create_snapshot()
+ self._write_sync_config(
+ index_dir,
+ self._resolve_sync_roots(docs_paths),
+ self._parse_file_types(args.file_types),
+ self._sync_ignore_patterns(args.include_hidden),
+ )
print(f"Index built at {index_path}")
-
- sync_roots = self._resolve_sync_roots(docs_paths)
- if sync_roots:
- include_extensions = self._parse_file_types(args.file_types)
- ignore_patterns = self._sync_ignore_patterns(args.include_hidden)
- self._initialize_file_synchronizers(sync_roots, include_extensions, ignore_patterns)
- self._write_sync_config(index_dir, sync_roots, include_extensions, ignore_patterns)
-
- # Register this project directory in global registry
self.register_project_dir()
def _watch_check_changes(self, index_name: str) -> tuple[set[str], set[str], set[str]]:
- """Check for file changes in watched directories. Returns (added, removed, modified) paths."""
+ """Check for file changes using the same snapshots as build (index_dir)."""
resolved = self._resolve_index_for_watch(index_name)
if not resolved:
return set(), set(), set()
@@ -2039,30 +2206,16 @@ Examples:
config = json.load(f)
roots = config.get("roots") or []
- include_extensions = config.get("include_extensions")
- ignore_patterns = config.get("ignore_patterns")
-
- added_paths: set[str] = set()
- removed_paths: set[str] = set()
- modified_paths: set[str] = set()
-
- for root in roots:
- try:
- fs = FileSynchronizer(
- root_dir=root,
- ignore_patterns=ignore_patterns,
- include_extensions=include_extensions,
- auto_load=True,
- )
- added, removed, modified = fs.check_for_changes()
- except Exception as exc:
- print(f"Warning: Failed to check {root}: {exc}")
- continue
- added_paths.update(added)
- removed_paths.update(removed)
- modified_paths.update(modified)
+ if not roots:
+ return set(), set(), set()
- return added_paths, removed_paths, modified_paths
+ synchronizers = self._create_synchronizers(
+ index_dir,
+ roots,
+ include_extensions=config.get("include_extensions"),
+ ignore_patterns=config.get("ignore_patterns"),
+ )
+ return self._detect_build_changes(synchronizers)
def _watch_report_changes(
self,
@@ -2323,7 +2476,7 @@ Examples:
if "last_modified_date" in result.metadata:
print(f" Modified: {result.metadata['last_modified_date']}")
- print(f" {result.text[:200]}...")
+ print(f" {result.text}")
print(f" Source: {result.metadata.get('source', '')}")
print()
diff --git a/packages/leann-core/src/leann/sync.py b/packages/leann-core/src/leann/sync.py
index fe3f875..d0a69e5 100644
--- a/packages/leann-core/src/leann/sync.py
+++ b/packages/leann-core/src/leann/sync.py
@@ -78,24 +78,32 @@ class FileSynchronizer:
ignore_patterns: Optional[list] = None,
include_extensions: Optional[list] = None,
auto_load=True,
+ snapshot_path: Optional[str] = None,
):
if not os.path.isdir(root_dir):
raise ValueError("This is not a valid directory")
self.root_dir = root_dir
self.ignore_patterns = ignore_patterns
self.include_extensions = include_extensions
+ self._custom_snapshot_path = snapshot_path
+ self._pending_tree: Optional[MerkleTree] = None
+ self.tree: Optional[MerkleTree] = None
if auto_load:
self.load_snapshot()
def generate_file_hashes(self):
file_hashes = {}
- reader = SimpleDirectoryReader(
- self.root_dir,
- recursive=True,
- exclude=self.ignore_patterns,
- required_exts=self.include_extensions,
- exclude_empty=False,
- )
+ try:
+ reader = SimpleDirectoryReader(
+ self.root_dir,
+ recursive=True,
+ exclude=self.ignore_patterns,
+ required_exts=self.include_extensions,
+ exclude_empty=False,
+ )
+ except ValueError:
+ # Empty directory — no files to hash
+ return file_hashes
# print('reader.iter_data() length', len(list(reader.iter_data())))
for file in reader.iter_data():
@@ -128,20 +136,40 @@ class FileSynchronizer:
return tree
- def check_for_changes(self):
+ def detect_changes(self) -> tuple[list[str], list[str], list[str]]:
+ """Detect changes without persisting. Call commit() after successful processing."""
file_hashes = self.generate_file_hashes()
new_tree = self.build_merkle_tree(file_hashes)
+ self._pending_tree = new_tree
+
+ if self.tree is None:
+ return list(file_hashes.keys()), [], []
- changes = self.tree.compare_with(new_tree)
+ return self.tree.compare_with(new_tree)
- if changes:
- self.tree = new_tree
+ def commit(self):
+ """Persist the pending snapshot after successful processing."""
+ if self._pending_tree is not None:
+ self.tree = self._pending_tree
+ self._pending_tree = None
self.save_snapshot()
+ def create_snapshot(self):
+ """Build and persist a snapshot from the current file state (for initial / forced builds)."""
+ file_hashes = self.generate_file_hashes()
+ self.tree = self.build_merkle_tree(file_hashes)
+ self.save_snapshot()
+
+ def check_for_changes(self) -> tuple[list[str], list[str], list[str]]:
+ """Detect and auto-commit changes (convenience wrapper)."""
+ changes = self.detect_changes()
+ self.commit()
return changes
@property
def snapshot_path(self):
+ if self._custom_snapshot_path:
+ return self._custom_snapshot_path
return f"{self.root_dir}.sync_context.pickle"
def save_snapshot(self):
@@ -155,7 +183,4 @@ class FileSynchronizer:
with open(self.snapshot_path, "rb") as f:
self.tree = pickle.load(f)
except FileNotFoundError:
- file_hashes = self.generate_file_hashes()
- self.tree = self.build_merkle_tree(file_hashes)
- self.save_snapshot()
- # yooooo
+ self.tree = None
diff --git a/tests/test_incremental_build.py b/tests/test_incremental_build.py
index 7833993..a7922ed 100644
--- a/tests/test_incremental_build.py
+++ b/tests/test_incremental_build.py
@@ -3,47 +3,103 @@ Tests for incremental build (Feature #89).
When an index already exists and build is run again without --force,
only new files are indexed and appended to the existing index (HNSW, non-compact only).
+Change detection uses content-hash (merkle tree) via FileSynchronizer.
"""
-import json
import os
from pathlib import Path
import pytest
-from leann.cli import (
- SOURCES_MANIFEST_FILENAME,
- _normalize_path,
- load_sources_manifest,
- save_sources_manifest,
-)
+from leann.cli import _normalize_path
+from leann.sync import FileSynchronizer
def test_normalize_path():
assert _normalize_path("") == ""
assert _normalize_path("/a/b") == "/a/b" or "a" in _normalize_path("/a/b")
- # Resolve so relative becomes absolute
rel = "foo/bar"
out = _normalize_path(rel)
assert Path(out).is_absolute() or out == rel
-def test_load_sources_manifest_missing(tmp_path):
- assert load_sources_manifest(tmp_path, "documents.leann") == {}
-
-
-def test_load_sources_manifest_empty(tmp_path):
- (tmp_path / SOURCES_MANIFEST_FILENAME).write_text('{"sources": {}}', encoding="utf-8")
- assert load_sources_manifest(tmp_path, "documents.leann") == {}
-
-
-def test_save_and_load_sources_manifest(tmp_path):
- sources = {"/path/to/a": 12345.0, "/path/to/b": 67890.0}
- save_sources_manifest(tmp_path, "documents.leann", sources)
- loaded = load_sources_manifest(tmp_path, "documents.leann")
- assert loaded == sources
- raw = (tmp_path / SOURCES_MANIFEST_FILENAME).read_text(encoding="utf-8")
- data = json.loads(raw)
- assert data.get("sources") == sources
+def test_file_synchronizer_detect_changes(tmp_path):
+ """FileSynchronizer detect_changes returns all files as added when no snapshot exists."""
+ docs = tmp_path / "docs"
+ docs.mkdir()
+ (docs / "a.txt").write_text("hello", encoding="utf-8")
+ snapshot = str(tmp_path / "test.pickle")
+ fs = FileSynchronizer(root_dir=str(docs), snapshot_path=snapshot)
+ added, removed, modified = fs.detect_changes()
+ assert len(added) == 1
+ assert len(removed) == 0
+ assert len(modified) == 0
+
+
+def test_file_synchronizer_no_changes_after_commit(tmp_path):
+ """After commit, detect_changes should report no changes if files haven't changed."""
+ docs = tmp_path / "docs"
+ docs.mkdir()
+ (docs / "a.txt").write_text("hello", encoding="utf-8")
+ snapshot = str(tmp_path / "test.pickle")
+ fs = FileSynchronizer(root_dir=str(docs), snapshot_path=snapshot)
+ fs.detect_changes()
+ fs.commit()
+
+ fs2 = FileSynchronizer(root_dir=str(docs), snapshot_path=snapshot)
+ added, removed, modified = fs2.detect_changes()
+ assert not added and not removed and not modified
+
+
+def test_file_synchronizer_detects_new_file(tmp_path):
+ """Adding a file after commit should be detected as added."""
+ docs = tmp_path / "docs"
+ docs.mkdir()
+ (docs / "a.txt").write_text("hello", encoding="utf-8")
+ snapshot = str(tmp_path / "test.pickle")
+ fs = FileSynchronizer(root_dir=str(docs), snapshot_path=snapshot)
+ fs.detect_changes()
+ fs.commit()
+
+ (docs / "b.txt").write_text("world", encoding="utf-8")
+ fs2 = FileSynchronizer(root_dir=str(docs), snapshot_path=snapshot)
+ added, removed, modified = fs2.detect_changes()
+ assert len(added) == 1
+ assert len(removed) == 0
+ assert len(modified) == 0
+
+
+def test_file_synchronizer_detects_modification(tmp_path):
+ """Changing file content should be detected as modified."""
+ docs = tmp_path / "docs"
+ docs.mkdir()
+ (docs / "a.txt").write_text("hello", encoding="utf-8")
+ snapshot = str(tmp_path / "test.pickle")
+ fs = FileSynchronizer(root_dir=str(docs), snapshot_path=snapshot)
+ fs.detect_changes()
+ fs.commit()
+
+ (docs / "a.txt").write_text("changed", encoding="utf-8")
+ fs2 = FileSynchronizer(root_dir=str(docs), snapshot_path=snapshot)
+ added, removed, modified = fs2.detect_changes()
+ assert len(modified) == 1
+ assert len(added) == 0
+
+
+def test_file_synchronizer_touch_no_false_positive(tmp_path):
+ """Touching a file (mtime change, same content) should NOT report as modified."""
+ docs = tmp_path / "docs"
+ docs.mkdir()
+ f = docs / "a.txt"
+ f.write_text("hello", encoding="utf-8")
+ snapshot = str(tmp_path / "test.pickle")
+ fs = FileSynchronizer(root_dir=str(docs), snapshot_path=snapshot)
+ fs.detect_changes()
+ fs.commit()
+
+ os.utime(f, None)
+ fs2 = FileSynchronizer(root_dir=str(docs), snapshot_path=snapshot)
+ added, removed, modified = fs2.detect_changes()
+ assert not added and not removed and not modified
@pytest.mark.skipif(
@@ -52,6 +108,9 @@ def test_save_and_load_sources_manifest(tmp_path):
)
def test_incremental_build_adds_only_new_files(tmp_path):
"""Build once with one file, add a second file, run build again without --force; index grows."""
+ import asyncio
+
+ from leann.api import LeannSearcher
from leann.cli import LeannCLI
docs_dir = tmp_path / "docs"
@@ -59,7 +118,6 @@ def test_incremental_build_adds_only_new_files(tmp_path):
(docs_dir / "a.txt").write_text("First document content for indexing.", encoding="utf-8")
cli = LeannCLI()
- # Override indexes_dir so we use tmp_path for index
cli.indexes_dir = tmp_path / ".leann" / "indexes"
cli.indexes_dir.mkdir(parents=True, exist_ok=True)
index_name = "incr_test"
@@ -83,17 +141,9 @@ def test_incremental_build_adds_only_new_files(tmp_path):
"--force",
]
)
- import asyncio
-
asyncio.run(cli.build_index(args))
assert index_dir.exists()
- meta_file = index_dir / "documents.leann.meta.json"
- assert meta_file.exists()
- manifest_file = index_dir / "documents.leann.sources.json"
- assert manifest_file.exists(), "Full build should write sources manifest"
- with open(manifest_file, encoding="utf-8") as f:
- manifest = json.load(f)
- assert len(manifest.get("sources", {})) == 1
+ assert (index_dir / "documents.leann.meta.json").exists()
# Add second file
(docs_dir / "b.txt").write_text("Second document content.", encoding="utf-8")
@@ -116,17 +166,77 @@ def test_incremental_build_adds_only_new_files(tmp_path):
)
asyncio.run(cli.build_index(args2))
- with open(manifest_file, encoding="utf-8") as f:
- manifest2 = json.load(f)
- assert len(manifest2.get("sources", {})) == 2, (
- "Manifest should list both files after incremental"
- )
-
- # Index should still be searchable
- from leann.api import LeannSearcher
-
+ # Index should still be searchable and contain both files
searcher = LeannSearcher(index_path)
results = searcher.search("Second document", top_k=3)
searcher.cleanup()
assert len(results) >= 1
assert "Second" in results[0].text or "document" in results[0].text
+
+
+ os.environ.get("CI") == "true",
+ reason="Skip in CI to avoid embedding/model load",
+)
+def test_ivf_incremental_add_then_remove_searchable(tmp_path):
+ """IVF: add content, search finds it; delete content, incremental build, search no longer finds it."""
+ import asyncio
+
+ from leann.api import LeannSearcher
+ from leann.cli import LeannCLI
+
+ docs_dir = tmp_path / "docs"
+ docs_dir.mkdir()
+ unique_phrase = "XYZZY_PLUGH_INCR_TEST_12345"
+ (docs_dir / "f.txt").write_text(
+ f"Initial content.\n\n{unique_phrase}\n\nMore text here.",
+ encoding="utf-8",
+ )
+ # Add extra files so IVF has enough training points (nlist=100)
+ for i in range(110):
+ (docs_dir / f"filler_{i:03d}.txt").write_text(
+ f"Filler document {i} with some content to create enough chunks.",
+ encoding="utf-8",
+ )
+
+ cli = LeannCLI()
+ cli.indexes_dir = tmp_path / ".leann" / "indexes"
+ cli.indexes_dir.mkdir(parents=True, exist_ok=True)
+ index_name = "ivf_incr_test"
+ index_path = cli.get_index_path(index_name)
+
+ parser = cli.create_parser()
+ build_args = [
+ "build",
+ index_name,
+ "--docs",
+ str(docs_dir),
+ "--backend-name",
+ "ivf",
+ "--embedding-model",
+ "all-MiniLM-L6-v2",
+ "--embedding-mode",
+ "sentence-transformers",
+ ]
+
+ asyncio.run(cli.build_index(parser.parse_args([*build_args, "--force"])))
+
+ searcher = LeannSearcher(index_path)
+ results = searcher.search(unique_phrase, top_k=5)
+ searcher.cleanup()
+ assert len(results) >= 1, "Added content should be searchable"
+ assert unique_phrase in results[0].text
+
+ # Remove the unique phrase from the file
+ (docs_dir / "f.txt").write_text("Initial content.\n\nMore text here.", encoding="utf-8")
+
+ # Incremental build (IVF remove+add)
+ asyncio.run(cli.build_index(parser.parse_args(build_args)))
+
+ # Deleted content should no longer be searchable
+ searcher = LeannSearcher(index_path)
+ results = searcher.search(unique_phrase, top_k=5)
+ searcher.cleanup()
+ assert all(unique_phrase not in r.text for r in results), (
+ "Deleted content should not appear in search results"
+ )