Skip to content

armgit / LEANN

Public Imported from GitHub
0

[MLsys2026 Best Paper]: https://arxiv.org/abs/2506.08276. RAG on Everything with LEANN. Enjoy 97% storage savings while running a fast, accurate, and 100% private RAG application on your personal device.

feat: add hybrid search

Marcusb2e1f5a3de9a
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index c8bc3e7..b3e7040 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -1,17 +1,17 @@
-repos:
- - repo: https://github.com/pre-commit/pre-commit-hooks
- rev: v5.0.0
- hooks:
- - id: trailing-whitespace
- - id: end-of-file-fixer
- - id: check-yaml
- - id: check-added-large-files
- - id: check-merge-conflict
- - id: debug-statements
-
- - repo: https://github.com/astral-sh/ruff-pre-commit
- rev: v0.12.7 # Fixed version to match pyproject.toml
- hooks:
- - id: ruff
- args: [--fix, --exit-non-zero-on-fix]
- - id: ruff-format
+repos:
+ - repo: https://github.com/pre-commit/pre-commit-hooks
+ rev: v5.0.0
+ hooks:
+ - id: trailing-whitespace
+ - id: end-of-file-fixer
+ - id: check-yaml
+ - id: check-added-large-files
+ - id: check-merge-conflict
+ - id: debug-statements
+
+ - repo: https://github.com/astral-sh/ruff-pre-commit
+ rev: v0.12.7 # Fixed version to match pyproject.toml
+ hooks:
+ - id: ruff
+ args: [--fix, --exit-non-zero-on-fix]
+ - id: ruff-format
diff --git a/packages/leann-core/src/leann/api.py b/packages/leann-core/src/leann/api.py
index d64d433..48a1c5a 100644
--- a/packages/leann-core/src/leann/api.py
+++ b/packages/leann-core/src/leann/api.py
@@ -1,1306 +1,1436 @@
-"""
-This file contains the core API for the LEANN project, now definitively updated
-with the correct, original embedding logic from the user's reference code.
-"""
-
-import json
-import logging
-import os
-import pickle
-import re
-import subprocess
-import time
-import warnings
-from dataclasses import dataclass, field
-from pathlib import Path
-from typing import Any, Literal, Optional, Union
-
-import numpy as np
-from leann_backend_hnsw.convert_to_csr import prune_hnsw_embeddings_inplace
-
-from leann.interactive_utils import create_api_session
-from leann.interface import LeannBackendSearcherInterface
-
-from .chat import get_llm
-from .embedding_server_manager import EmbeddingServerManager
-from .interface import LeannBackendFactoryInterface
-from .metadata_filter import MetadataFilterEngine
-from .registry import BACKEND_REGISTRY
-
-logger = logging.getLogger(__name__)
-
-
-def get_registered_backends() -> list[str]:
- """Get list of registered backend names."""
- return list(BACKEND_REGISTRY.keys())
-
-
-def compute_embeddings(
- chunks: list[str],
- model_name: str,
- mode: str = "sentence-transformers",
- use_server: bool = True,
- port: Optional[int] = None,
- is_build=False,
- provider_options: Optional[dict[str, Any]] = None,
-) -> np.ndarray:
- """
- Computes embeddings using different backends.
-
- Args:
- chunks: List of text chunks to embed
- model_name: Name of the embedding model
- mode: Embedding backend mode. Options:
- - "sentence-transformers": Use sentence-transformers library (default)
- - "mlx": Use MLX backend for Apple Silicon
- - "openai": Use OpenAI embedding API
- - "gemini": Use Google Gemini embedding API
- use_server: Whether to use embedding server (True for search, False for build)
-
- Returns:
- numpy array of embeddings
- """
- if use_server:
- # Use embedding server (for search/query)
- if port is None:
- raise ValueError("port is required when use_server is True")
- return compute_embeddings_via_server(chunks, model_name, port=port)
- else:
- # Use direct computation (for build_index)
- from .embedding_compute import (
- compute_embeddings as compute_embeddings_direct,
- )
-
- return compute_embeddings_direct(
- chunks,
- model_name,
- mode=mode,
- is_build=is_build,
- provider_options=provider_options,
- )
-
-
-def compute_embeddings_via_server(chunks: list[str], model_name: str, port: int) -> np.ndarray:
- """Computes embeddings using sentence-transformers.
-
- Args:
- chunks: List of text chunks to embed
- model_name: Name of the sentence transformer model
- """
- logger.info(
- f"Computing embeddings for {len(chunks)} chunks using SentenceTransformer model '{model_name}' (via embedding server)..."
- )
- import msgpack
- import numpy as np
- import zmq
-
- # Connect to embedding server
- context = zmq.Context()
- socket = context.socket(zmq.REQ)
- socket.connect(f"tcp://localhost:{port}")
-
- # Send chunks to server for embedding computation
- request = chunks
- socket.send(msgpack.packb(request))
-
- # Receive embeddings from server
- response = socket.recv()
- embeddings_list = msgpack.unpackb(response)
-
- # Convert back to numpy array
- embeddings = np.array(embeddings_list, dtype=np.float32)
-
- socket.close()
- context.term()
-
- return embeddings
-
-
-@dataclass
-class SearchResult:
- id: str
- score: float
- text: str
- metadata: dict[str, Any] = field(default_factory=dict)
-
-
-class PassageManager:
- def __init__(
- self, passage_sources: list[dict[str, Any]], metadata_file_path: Optional[str] = None
- ):
- self.offset_maps: dict[str, dict[str, int]] = {}
- self.passage_files: dict[str, str] = {}
- # Avoid materializing a single gigantic global map to reduce memory
- # footprint on very large corpora (e.g., 60M+ passages). Instead, keep
- # per-shard maps and do a lightweight per-shard lookup on demand.
- self._total_count: int = 0
- self.filter_engine = MetadataFilterEngine() # Initialize filter engine
-
- # Derive index base name for standard sibling fallbacks, e.g., <index_name>.passages.*
- index_name_base = None
- if metadata_file_path:
- meta_name = Path(metadata_file_path).name
- if meta_name.endswith(".meta.json"):
- index_name_base = meta_name[: -len(".meta.json")]
-
- for source in passage_sources:
- assert source["type"] == "jsonl", "only jsonl is supported"
- passage_file = source.get("path", "")
- index_file = source.get("index_path", "") # .idx file
-
- # Fix path resolution - relative paths should be relative to metadata file directory
- def _resolve_candidates(
- primary: str,
- relative_key: str,
- default_name: Optional[str],
- source_dict: dict[str, Any],
- ) -> list[Path]:
- """
- Build an ordered list of candidate paths. For relative paths specified in
- metadata, prefer resolution relative to the metadata file directory first,
- then fall back to CWD-based resolution, and finally to conventional
- sibling defaults (e.g., <index_base>.passages.idx / .jsonl).
- """
- candidates: list[Path] = []
- # 1) Primary path
- if primary:
- p = Path(primary)
- if p.is_absolute():
- candidates.append(p)
- else:
- # Prefer metadata-relative resolution for relative paths
- if metadata_file_path:
- candidates.append(Path(metadata_file_path).parent / p)
- # Also consider CWD-relative as a fallback for legacy layouts
- candidates.append(Path.cwd() / p)
- # 2) metadata-relative explicit relative key (if present)
- if metadata_file_path and source_dict.get(relative_key):
- candidates.append(Path(metadata_file_path).parent / source_dict[relative_key])
- # 3) metadata-relative standard sibling filename
- if metadata_file_path and default_name:
- candidates.append(Path(metadata_file_path).parent / default_name)
- return candidates
-
- # Build candidate lists and pick first existing; otherwise keep last candidate for error message
- idx_default = f"{index_name_base}.passages.idx" if index_name_base else None
- idx_candidates = _resolve_candidates(
- index_file, "index_path_relative", idx_default, source
- )
- pas_default = f"{index_name_base}.passages.jsonl" if index_name_base else None
- pas_candidates = _resolve_candidates(passage_file, "path_relative", pas_default, source)
-
- def _pick_existing(cands: list[Path]) -> str:
- for c in cands:
- if c.exists():
- return str(c.resolve())
- # Fallback to last candidate (best guess) even if not exists; will error below
- return str(cands[-1].resolve()) if cands else ""
-
- index_file = _pick_existing(idx_candidates)
- passage_file = _pick_existing(pas_candidates)
-
- if not Path(index_file).exists():
- raise FileNotFoundError(f"Passage index file not found: {index_file}")
-
- with open(index_file, "rb") as f:
- offset_map: dict[str, int] = pickle.load(f)
- self.offset_maps[passage_file] = offset_map
- self.passage_files[passage_file] = passage_file
- self._total_count += len(offset_map)
-
- def get_passage(self, passage_id: str) -> dict[str, Any]:
- # Fast path: check each shard map (there are typically few shards).
- # This avoids building a massive combined dict while keeping lookups
- # bounded by the number of shards.
- for passage_file, offset_map in self.offset_maps.items():
- try:
- offset = offset_map[passage_id]
- with open(passage_file, encoding="utf-8") as f:
- f.seek(offset)
- return json.loads(f.readline())
- except KeyError:
- continue
- raise KeyError(f"Passage ID not found: {passage_id}")
-
- def filter_search_results(
- self,
- search_results: list[SearchResult],
- metadata_filters: Optional[dict[str, dict[str, Union[str, int, float, bool, list]]]],
- ) -> list[SearchResult]:
- """
- Apply metadata filters to search results.
-
- Args:
- search_results: List of SearchResult objects
- metadata_filters: Filter specifications to apply
-
- Returns:
- Filtered list of SearchResult objects
- """
- if not metadata_filters:
- return search_results
-
- logger.debug(f"Applying metadata filters to {len(search_results)} results")
-
- # Convert SearchResult objects to dictionaries for the filter engine
- result_dicts = []
- for result in search_results:
- result_dicts.append(
- {
- "id": result.id,
- "score": result.score,
- "text": result.text,
- "metadata": result.metadata,
- }
- )
-
- # Apply filters using the filter engine
- filtered_dicts = self.filter_engine.apply_filters(result_dicts, metadata_filters)
-
- # Convert back to SearchResult objects
- filtered_results = []
- for result_dict in filtered_dicts:
- filtered_results.append(
- SearchResult(
- id=result_dict["id"],
- score=result_dict["score"],
- text=result_dict["text"],
- metadata=result_dict["metadata"],
- )
- )
-
- logger.debug(f"Filtered results: {len(filtered_results)} remaining")
- return filtered_results
-
- def __len__(self) -> int:
- return self._total_count
-
-
-class LeannBuilder:
- def __init__(
- self,
- backend_name: str,
- embedding_model: str = "facebook/contriever",
- dimensions: Optional[int] = None,
- embedding_mode: str = "sentence-transformers",
- embedding_options: Optional[dict[str, Any]] = None,
- **backend_kwargs,
- ):
- self.backend_name = backend_name
- # Normalize incompatible combinations early (for consistent metadata)
- if backend_name == "hnsw":
- is_recompute = backend_kwargs.get("is_recompute", True)
- is_compact = backend_kwargs.get("is_compact", True)
- if is_recompute is False and is_compact is True:
- warnings.warn(
- "HNSW with is_recompute=False requires non-compact storage. Forcing is_compact=False.",
- UserWarning,
- stacklevel=2,
- )
- backend_kwargs["is_compact"] = False
-
- backend_factory: Optional[LeannBackendFactoryInterface] = BACKEND_REGISTRY.get(backend_name)
- if backend_factory is None:
- raise ValueError(f"Backend '{backend_name}' not found or not registered.")
- self.backend_factory = backend_factory
- self.embedding_model = embedding_model
- self.dimensions = dimensions
- self.embedding_mode = embedding_mode
- self.embedding_options = embedding_options or {}
-
- # Check if we need to use cosine distance for normalized embeddings
- normalized_embeddings_models = {
- # OpenAI models
- ("openai", "text-embedding-ada-002"),
- ("openai", "text-embedding-3-small"),
- ("openai", "text-embedding-3-large"),
- # Voyage AI models
- ("voyage", "voyage-2"),
- ("voyage", "voyage-3"),
- ("voyage", "voyage-large-2"),
- ("voyage", "voyage-multilingual-2"),
- ("voyage", "voyage-code-2"),
- # Cohere models
- ("cohere", "embed-english-v3.0"),
- ("cohere", "embed-multilingual-v3.0"),
- ("cohere", "embed-english-light-v3.0"),
- ("cohere", "embed-multilingual-light-v3.0"),
- }
-
- # Also check for patterns in model names
- is_normalized = False
- current_model_lower = embedding_model.lower()
- current_mode_lower = embedding_mode.lower()
-
- # Check exact matches
- for mode, model in normalized_embeddings_models:
- if (current_mode_lower == mode and current_model_lower == model) or (
- mode in current_mode_lower and model in current_model_lower
- ):
- is_normalized = True
- break
-
- # Check patterns
- if not is_normalized:
- # OpenAI patterns
- if "openai" in current_mode_lower or "openai" in current_model_lower:
- if any(
- pattern in current_model_lower
- for pattern in ["text-embedding", "ada", "3-small", "3-large"]
- ):
- is_normalized = True
- # Voyage patterns
- elif "voyage" in current_mode_lower or "voyage" in current_model_lower:
- is_normalized = True
- # Cohere patterns
- elif "cohere" in current_mode_lower or "cohere" in current_model_lower:
- if "embed" in current_model_lower:
- is_normalized = True
-
- # Handle distance metric
- if is_normalized and "distance_metric" not in backend_kwargs:
- backend_kwargs["distance_metric"] = "cosine"
- warnings.warn(
- f"Detected normalized embeddings model '{embedding_model}' with mode '{embedding_mode}'. "
- f"Automatically setting distance_metric='cosine' for optimal performance. "
- f"Normalized embeddings (L2 norm = 1) should use cosine similarity instead of MIPS.",
- UserWarning,
- stacklevel=2,
- )
- elif is_normalized and backend_kwargs.get("distance_metric", "").lower() != "cosine":
- current_metric = backend_kwargs.get("distance_metric", "mips")
- warnings.warn(
- f"Warning: Using '{current_metric}' distance metric with normalized embeddings model "
- f"'{embedding_model}' may lead to suboptimal search results. "
- f"Consider using 'cosine' distance metric for better performance.",
- UserWarning,
- stacklevel=2,
- )
-
- self.backend_kwargs = backend_kwargs
- self.chunks: list[dict[str, Any]] = []
-
- def add_text(self, text: str, metadata: Optional[dict[str, Any]] = None):
- if metadata is None:
- metadata = {}
- passage_id = metadata.get("id", str(len(self.chunks)))
- chunk_data = {"id": passage_id, "text": text, "metadata": metadata}
- self.chunks.append(chunk_data)
-
- def build_index(self, index_path: str):
- if not self.chunks:
- raise ValueError("No chunks added.")
-
- # Filter out invalid/empty text chunks early to keep passage and embedding counts aligned
- valid_chunks: list[dict[str, Any]] = []
- skipped = 0
- for chunk in self.chunks:
- text = chunk.get("text", "")
- if isinstance(text, str) and text.strip():
- valid_chunks.append(chunk)
- else:
- skipped += 1
- if skipped > 0:
- print(
- f"Warning: Skipping {skipped} empty/invalid text chunk(s). Processing {len(valid_chunks)} valid chunks"
- )
- self.chunks = valid_chunks
- if not self.chunks:
- raise ValueError("All provided chunks are empty or invalid. Nothing to index.")
- if self.dimensions is None:
- self.dimensions = len(
- compute_embeddings(
- ["dummy"],
- self.embedding_model,
- self.embedding_mode,
- use_server=False,
- provider_options=self.embedding_options,
- )[0]
- )
- path = Path(index_path)
- index_dir = path.parent
- index_name = path.name
- index_dir.mkdir(parents=True, exist_ok=True)
- passages_file = index_dir / f"{index_name}.passages.jsonl"
- offset_file = index_dir / f"{index_name}.passages.idx"
- offset_map = {}
- with open(passages_file, "w", encoding="utf-8") as f:
- try:
- from tqdm import tqdm
-
- chunk_iterator = tqdm(self.chunks, desc="Writing passages", unit="chunk")
- except ImportError:
- chunk_iterator = self.chunks
-
- for chunk in chunk_iterator:
- offset = f.tell()
- json.dump(
- {
- "id": chunk["id"],
- "text": chunk["text"],
- "metadata": chunk["metadata"],
- },
- f,
- ensure_ascii=False,
- )
- f.write("\n")
- offset_map[chunk["id"]] = offset
- with open(offset_file, "wb") as f:
- pickle.dump(offset_map, f)
- texts_to_embed = [c["text"] for c in self.chunks]
- embeddings = compute_embeddings(
- texts_to_embed,
- self.embedding_model,
- self.embedding_mode,
- use_server=False,
- is_build=True,
- provider_options=self.embedding_options,
- )
- string_ids = [chunk["id"] for chunk in self.chunks]
- # Persist ID map alongside index so backends that return integer labels can remap to passage IDs
- try:
- idmap_file = (
- index_dir
- / f"{index_name[: -len('.leann')] if index_name.endswith('.leann') else index_name}.ids.txt"
- )
- with open(idmap_file, "w", encoding="utf-8") as f:
- for sid in string_ids:
- f.write(str(sid) + "\n")
- except Exception:
- pass
- current_backend_kwargs = {**self.backend_kwargs, "dimensions": self.dimensions}
- builder_instance = self.backend_factory.builder(**current_backend_kwargs)
- builder_instance.build(embeddings, string_ids, index_path, **current_backend_kwargs)
- leann_meta_path = index_dir / f"{index_name}.meta.json"
- meta_data = {
- "version": "1.0",
- "backend_name": self.backend_name,
- "embedding_model": self.embedding_model,
- "dimensions": self.dimensions,
- "backend_kwargs": self.backend_kwargs,
- "embedding_mode": self.embedding_mode,
- "passage_sources": [
- {
- "type": "jsonl",
- # Preserve existing relative file names (backward-compatible)
- "path": passages_file.name,
- "index_path": offset_file.name,
- # Add optional redundant relative keys for remote build portability (non-breaking)
- "path_relative": passages_file.name,
- "index_path_relative": offset_file.name,
- }
- ],
- }
-
- if self.embedding_options:
- meta_data["embedding_options"] = self.embedding_options
-
- # Add storage status flags for HNSW backend
- if self.backend_name == "hnsw":
- is_compact = self.backend_kwargs.get("is_compact", True)
- is_recompute = self.backend_kwargs.get("is_recompute", True)
- meta_data["is_compact"] = is_compact
- meta_data["is_pruned"] = bool(is_recompute)
- with open(leann_meta_path, "w", encoding="utf-8") as f:
- json.dump(meta_data, f, indent=2)
-
- def build_index_from_embeddings(self, index_path: str, embeddings_file: str):
- """
- Build an index from pre-computed embeddings stored in a pickle file.
-
- Args:
- index_path: Path where the index will be saved
- embeddings_file: Path to pickle file containing (ids, embeddings) tuple
- """
- # Load pre-computed embeddings
- with open(embeddings_file, "rb") as f:
- data = pickle.load(f)
-
- if not isinstance(data, tuple) or len(data) != 2:
- raise ValueError(
- f"Invalid embeddings file format. Expected tuple with 2 elements, got {type(data)}"
- )
-
- ids, embeddings = data
-
- if not isinstance(embeddings, np.ndarray):
- raise ValueError(f"Expected embeddings to be numpy array, got {type(embeddings)}")
-
- if len(ids) != embeddings.shape[0]:
- raise ValueError(
- f"Mismatch between number of IDs ({len(ids)}) and embeddings ({embeddings.shape[0]})"
- )
-
- # Validate/set dimensions
- embedding_dim = embeddings.shape[1]
- if self.dimensions is None:
- self.dimensions = embedding_dim
- elif self.dimensions != embedding_dim:
- raise ValueError(f"Dimension mismatch: expected {self.dimensions}, got {embedding_dim}")
-
- logger.info(
- f"Building index from precomputed embeddings: {len(ids)} items, {embedding_dim} dimensions"
- )
-
- # Ensure we have text data for each embedding
- if len(self.chunks) != len(ids):
- # If no text chunks provided, create placeholder text entries
- if not self.chunks:
- logger.info("No text chunks provided, creating placeholder entries...")
- for id_val in ids:
- self.add_text(
- f"Document {id_val}",
- metadata={"id": str(id_val), "from_embeddings": True},
- )
- else:
- raise ValueError(
- f"Number of text chunks ({len(self.chunks)}) doesn't match number of embeddings ({len(ids)})"
- )
-
- # Build file structure
- path = Path(index_path)
- index_dir = path.parent
- index_name = path.name
- index_dir.mkdir(parents=True, exist_ok=True)
- passages_file = index_dir / f"{index_name}.passages.jsonl"
- offset_file = index_dir / f"{index_name}.passages.idx"
-
- # Write passages and create offset map
- offset_map = {}
- with open(passages_file, "w", encoding="utf-8") as f:
- for chunk in self.chunks:
- offset = f.tell()
- json.dump(
- {
- "id": chunk["id"],
- "text": chunk["text"],
- "metadata": chunk["metadata"],
- },
- f,
- ensure_ascii=False,
- )
- f.write("\n")
- offset_map[chunk["id"]] = offset
-
- with open(offset_file, "wb") as f:
- pickle.dump(offset_map, f)
-
- # Build the vector index using precomputed embeddings
- string_ids = [str(id_val) for id_val in ids]
- # Persist ID map (order == embeddings order)
- try:
- idmap_file = (
- index_dir
- / f"{index_name[: -len('.leann')] if index_name.endswith('.leann') else index_name}.ids.txt"
- )
- with open(idmap_file, "w", encoding="utf-8") as f:
- for sid in string_ids:
- f.write(str(sid) + "\n")
- except Exception:
- pass
- current_backend_kwargs = {**self.backend_kwargs, "dimensions": self.dimensions}
- builder_instance = self.backend_factory.builder(**current_backend_kwargs)
- builder_instance.build(embeddings, string_ids, index_path)
-
- # Create metadata file
- leann_meta_path = index_dir / f"{index_name}.meta.json"
- meta_data = {
- "version": "1.0",
- "backend_name": self.backend_name,
- "embedding_model": self.embedding_model,
- "dimensions": self.dimensions,
- "backend_kwargs": self.backend_kwargs,
- "embedding_mode": self.embedding_mode,
- "passage_sources": [
- {
- "type": "jsonl",
- # Preserve existing relative file names (backward-compatible)
- "path": passages_file.name,
- "index_path": offset_file.name,
- # Add optional redundant relative keys for remote build portability (non-breaking)
- "path_relative": passages_file.name,
- "index_path_relative": offset_file.name,
- }
- ],
- "built_from_precomputed_embeddings": True,
- "embeddings_source": str(embeddings_file),
- }
-
- if self.embedding_options:
- meta_data["embedding_options"] = self.embedding_options
-
- # Add storage status flags for HNSW backend
- if self.backend_name == "hnsw":
- is_compact = self.backend_kwargs.get("is_compact", True)
- is_recompute = self.backend_kwargs.get("is_recompute", True)
- meta_data["is_compact"] = is_compact
- meta_data["is_pruned"] = bool(is_recompute)
-
- with open(leann_meta_path, "w", encoding="utf-8") as f:
- json.dump(meta_data, f, indent=2)
-
- logger.info(f"Index built successfully from precomputed embeddings: {index_path}")
-
- def update_index(self, index_path: str):
- """Append new passages and vectors to an existing HNSW index."""
- if not self.chunks:
- raise ValueError("No new chunks provided for update.")
-
- path = Path(index_path)
- index_dir = path.parent
- index_name = path.name
- index_prefix = path.stem
-
- meta_path = index_dir / f"{index_name}.meta.json"
- passages_file = index_dir / f"{index_name}.passages.jsonl"
- offset_file = index_dir / f"{index_name}.passages.idx"
- index_file = index_dir / f"{index_prefix}.index"
-
- if not meta_path.exists() or not passages_file.exists() or not offset_file.exists():
- raise FileNotFoundError("Index metadata or passage files are missing; cannot update.")
- if not index_file.exists():
- raise FileNotFoundError(f"HNSW index file not found: {index_file}")
-
- with open(meta_path, encoding="utf-8") as f:
- meta = json.load(f)
- backend_name = meta.get("backend_name")
- if backend_name != self.backend_name:
- raise ValueError(
- f"Index was built with backend '{backend_name}', cannot update with '{self.backend_name}'."
- )
-
- meta_backend_kwargs = meta.get("backend_kwargs", {})
- index_is_compact = meta.get("is_compact", meta_backend_kwargs.get("is_compact", True))
- if index_is_compact:
- raise ValueError(
- "Compact HNSW indices do not support in-place updates. Rebuild required."
- )
-
- distance_metric = meta_backend_kwargs.get(
- "distance_metric", self.backend_kwargs.get("distance_metric", "mips")
- ).lower()
- needs_recompute = bool(
- meta.get("is_pruned")
- or meta_backend_kwargs.get("is_recompute")
- or self.backend_kwargs.get("is_recompute")
- )
-
- with open(offset_file, "rb") as f:
- offset_map: dict[str, int] = pickle.load(f)
- existing_ids = set(offset_map.keys())
-
- valid_chunks: list[dict[str, Any]] = []
- for chunk in self.chunks:
- text = chunk.get("text", "")
- if not isinstance(text, str) or not text.strip():
- continue
- metadata = chunk.setdefault("metadata", {})
- passage_id = chunk.get("id") or metadata.get("id")
- if passage_id and passage_id in existing_ids:
- raise ValueError(f"Passage ID '{passage_id}' already exists in the index.")
- valid_chunks.append(chunk)
-
- if not valid_chunks:
- raise ValueError("No valid chunks to append.")
-
- texts_to_embed = [chunk["text"] for chunk in valid_chunks]
- embeddings = compute_embeddings(
- texts_to_embed,
- self.embedding_model,
- self.embedding_mode,
- use_server=False,
- is_build=True,
- provider_options=self.embedding_options,
- )
-
- embedding_dim = embeddings.shape[1]
- expected_dim = meta.get("dimensions")
- if expected_dim is not None and expected_dim != embedding_dim:
- raise ValueError(
- f"Dimension mismatch during update: existing index uses {expected_dim}, got {embedding_dim}."
- )
-
- from leann_backend_hnsw import faiss # type: ignore
-
- embeddings = np.ascontiguousarray(embeddings, dtype=np.float32)
- if distance_metric == "cosine":
- norms = np.linalg.norm(embeddings, axis=1, keepdims=True)
- norms[norms == 0] = 1
- embeddings = embeddings / norms
-
- index = faiss.read_index(str(index_file))
- if hasattr(index, "is_recompute"):
- index.is_recompute = needs_recompute
- print(f"index.is_recompute: {index.is_recompute}")
- if getattr(index, "storage", None) is None:
- if index.metric_type == faiss.METRIC_INNER_PRODUCT:
- storage_index = faiss.IndexFlatIP(index.d)
- else:
- storage_index = faiss.IndexFlatL2(index.d)
- index.storage = storage_index
- index.own_fields = True
- # Faiss expects storage.ntotal to reflect the existing graph's
- # population (even if the vectors themselves were pruned from disk
- # for recompute mode). When we attach a fresh IndexFlat here its
- # ntotal starts at zero, which later causes IndexHNSW::add to
- # believe new "preset" levels were provided and trips the
- # `n0 + n == levels.size()` assertion. Seed the temporary storage
- # with the current ntotal so Faiss maintains the proper offset for
- # incoming vectors.
- try:
- storage_index.ntotal = index.ntotal
- except AttributeError:
- # Older Faiss builds may not expose ntotal as a writable
- # attribute; in that case we fall back to the default behaviour.
- pass
- if index.d != embedding_dim:
- raise ValueError(
- f"Existing index dimension ({index.d}) does not match new embeddings ({embedding_dim})."
- )
-
- passage_meta_mode = meta.get("embedding_mode", self.embedding_mode)
- passage_provider_options = meta.get("embedding_options", self.embedding_options)
-
- base_id = index.ntotal
- for offset, chunk in enumerate(valid_chunks):
- new_id = str(base_id + offset)
- chunk.setdefault("metadata", {})["id"] = new_id
- chunk["id"] = new_id
-
- # Append passages/offsets before we attempt index.add so the ZMQ server
- # can resolve newly assigned IDs during recompute. Keep rollback hooks
- # so we can restore files if the update fails mid-way.
- rollback_passages_size = passages_file.stat().st_size if passages_file.exists() else 0
- offset_map_backup = offset_map.copy()
-
- try:
- with open(passages_file, "a", encoding="utf-8") as f:
- for chunk in valid_chunks:
- offset = f.tell()
- json.dump(
- {
- "id": chunk["id"],
- "text": chunk["text"],
- "metadata": chunk.get("metadata", {}),
- },
- f,
- ensure_ascii=False,
- )
- f.write("\n")
- offset_map[chunk["id"]] = offset
-
- with open(offset_file, "wb") as f:
- pickle.dump(offset_map, f)
-
- server_manager: Optional[EmbeddingServerManager] = None
- server_started = False
- requested_zmq_port = int(os.getenv("LEANN_UPDATE_ZMQ_PORT", "5557"))
-
- try:
- if needs_recompute:
- server_manager = EmbeddingServerManager(
- backend_module_name="leann_backend_hnsw.hnsw_embedding_server"
- )
- server_started, actual_port = server_manager.start_server(
- port=requested_zmq_port,
- model_name=self.embedding_model,
- embedding_mode=passage_meta_mode,
- passages_file=str(meta_path),
- distance_metric=distance_metric,
- provider_options=passage_provider_options,
- )
- if not server_started:
- raise RuntimeError(
- "Failed to start HNSW embedding server for recompute update."
- )
- if actual_port != requested_zmq_port:
- logger.warning(
- "Embedding server started on port %s instead of requested %s. "
- "Using reassigned port.",
- actual_port,
- requested_zmq_port,
- )
- if hasattr(index.hnsw, "set_zmq_port"):
- index.hnsw.set_zmq_port(actual_port)
- elif hasattr(index, "set_zmq_port"):
- index.set_zmq_port(actual_port)
-
- if needs_recompute:
- for i in range(embeddings.shape[0]):
- print(f"add {i} embeddings")
- index.add(1, faiss.swig_ptr(embeddings[i : i + 1]))
- else:
- index.add(embeddings.shape[0], faiss.swig_ptr(embeddings))
- faiss.write_index(index, str(index_file))
- finally:
- if server_started and server_manager is not None:
- server_manager.stop_server()
-
- except Exception:
- # Roll back appended passages/offset map to keep files consistent.
- if passages_file.exists():
- with open(passages_file, "rb+") as f:
- f.truncate(rollback_passages_size)
- offset_map = offset_map_backup
- with open(offset_file, "wb") as f:
- pickle.dump(offset_map, f)
- raise
-
- meta["total_passages"] = len(offset_map)
- with open(meta_path, "w", encoding="utf-8") as f:
- json.dump(meta, f, indent=2)
-
- logger.info(
- "Appended %d passages to index '%s'. New total: %d",
- len(valid_chunks),
- index_path,
- len(offset_map),
- )
-
- self.chunks.clear()
-
- if needs_recompute:
- prune_hnsw_embeddings_inplace(str(index_file))
-
-
-class LeannSearcher:
- def __init__(self, index_path: str, enable_warmup: bool = False, **backend_kwargs):
- # Fix path resolution for Colab and other environments
- if not Path(index_path).is_absolute():
- index_path = str(Path(index_path).resolve())
-
- self.meta_path_str = f"{index_path}.meta.json"
- if not Path(self.meta_path_str).exists():
- parent_dir = Path(index_path).parent
- print(
- f"Leann metadata file not found at {self.meta_path_str}, and you may need to rm -rf {parent_dir}"
- )
- # highlight in red the filenotfound error
- raise FileNotFoundError(
- f"Leann metadata file not found at {self.meta_path_str}, \033[91m you may need to rm -rf {parent_dir}\033[0m"
- )
- with open(self.meta_path_str, encoding="utf-8") as f:
- self.meta_data = json.load(f)
- backend_name = self.meta_data["backend_name"]
- self.embedding_model = self.meta_data["embedding_model"]
- # Support both old and new format
- self.embedding_mode = self.meta_data.get("embedding_mode", "sentence-transformers")
- self.embedding_options = self.meta_data.get("embedding_options", {})
- # Delegate portability handling to PassageManager
- self.passage_manager = PassageManager(
- self.meta_data.get("passage_sources", []), metadata_file_path=self.meta_path_str
- )
- # Preserve backend name for conditional parameter forwarding
- self.backend_name = backend_name
- backend_factory = BACKEND_REGISTRY.get(backend_name)
- if backend_factory is None:
- raise ValueError(f"Backend '{backend_name}' not found.")
- final_kwargs = {**self.meta_data.get("backend_kwargs", {}), **backend_kwargs}
- final_kwargs["enable_warmup"] = enable_warmup
- if self.embedding_options:
- final_kwargs.setdefault("embedding_options", self.embedding_options)
- self.backend_impl: LeannBackendSearcherInterface = backend_factory.searcher(
- index_path, **final_kwargs
- )
-
- def search(
- self,
- query: str,
- top_k: int = 5,
- complexity: int = 64,
- beam_width: int = 1,
- prune_ratio: float = 0.0,
- recompute_embeddings: bool = True,
- pruning_strategy: Literal["global", "local", "proportional"] = "global",
- expected_zmq_port: int = 5557,
- metadata_filters: Optional[dict[str, dict[str, Union[str, int, float, bool, list]]]] = None,
- batch_size: int = 0,
- use_grep: bool = False,
- provider_options: Optional[dict[str, Any]] = None,
- **kwargs,
- ) -> list[SearchResult]:
- """
- Search for nearest neighbors with optional metadata filtering.
-
- Args:
- query: Text query to search for
- top_k: Number of nearest neighbors to return
- complexity: Search complexity/candidate list size, higher = more accurate but slower
- beam_width: Number of parallel search paths/IO requests per iteration
- prune_ratio: Ratio of neighbors to prune via approximate distance (0.0-1.0)
- recompute_embeddings: Whether to fetch fresh embeddings from server vs use stored codes
- pruning_strategy: Candidate selection strategy - "global" (default), "local", or "proportional"
- expected_zmq_port: ZMQ port for embedding server communication
- metadata_filters: Optional filters to apply to search results based on metadata.
- Format: {"field_name": {"operator": value}}
- Supported operators:
- - Comparison: "==", "!=", "<", "<=", ">", ">="
- - Membership: "in", "not_in"
- - String: "contains", "starts_with", "ends_with"
- Example: {"chapter": {"<=": 5}, "tags": {"in": ["fiction", "drama"]}}
- **kwargs: Backend-specific parameters
-
- Returns:
- List of SearchResult objects with text, metadata, and similarity scores
- """
- # Handle grep search
- if use_grep:
- return self._grep_search(query, top_k)
-
- logger.info("🔍 LeannSearcher.search() called:")
- logger.info(f" Query: '{query}'")
- logger.info(f" Top_k: {top_k}")
- logger.info(f" Metadata filters: {metadata_filters}")
- logger.info(f" Additional kwargs: {kwargs}")
-
- # Smart top_k detection and adjustment
- # Use PassageManager length (sum of shard sizes) to avoid
- # depending on a massive combined map
- total_docs = len(self.passage_manager)
- original_top_k = top_k
- if top_k > total_docs:
- top_k = total_docs
- logger.warning(
- f" ⚠️ Requested top_k ({original_top_k}) exceeds total documents ({total_docs})"
- )
- logger.warning(f" ✅ Auto-adjusted top_k to {top_k} to match available documents")
-
- zmq_port = None
-
- start_time = time.time()
- if recompute_embeddings:
- zmq_port = self.backend_impl._ensure_server_running(
- self.meta_path_str,
- port=expected_zmq_port,
- **kwargs,
- )
- del expected_zmq_port
- zmq_time = time.time() - start_time
- logger.info(f" Launching server time: {zmq_time} seconds")
-
- start_time = time.time()
-
- # Extract query template from stored embedding_options with fallback chain:
- # 1. Check provider_options override (highest priority)
- # 2. Check query_prompt_template (new format)
- # 3. Check prompt_template (old format for backward compat)
- # 4. None (no template)
- query_template = None
- if provider_options and "prompt_template" in provider_options:
- query_template = provider_options["prompt_template"]
- elif "query_prompt_template" in self.embedding_options:
- query_template = self.embedding_options["query_prompt_template"]
- elif "prompt_template" in self.embedding_options:
- query_template = self.embedding_options["prompt_template"]
-
- query_embedding = self.backend_impl.compute_query_embedding(
- query,
- use_server_if_available=recompute_embeddings,
- zmq_port=zmq_port,
- query_template=query_template,
- )
- logger.info(f" Generated embedding shape: {query_embedding.shape}")
- embedding_time = time.time() - start_time
- logger.info(f" Embedding time: {embedding_time} seconds")
-
- start_time = time.time()
- backend_search_kwargs: dict[str, Any] = {
- "complexity": complexity,
- "beam_width": beam_width,
- "prune_ratio": prune_ratio,
- "recompute_embeddings": recompute_embeddings,
- "pruning_strategy": pruning_strategy,
- "zmq_port": zmq_port,
- }
- # Only HNSW supports batching; forward conditionally
- if self.backend_name == "hnsw":
- backend_search_kwargs["batch_size"] = batch_size
-
- # Merge any extra kwargs last
- backend_search_kwargs.update(kwargs)
-
- results = self.backend_impl.search(
- query_embedding,
- top_k,
- **backend_search_kwargs,
- )
- search_time = time.time() - start_time
- logger.info(f" Search time in search() LEANN searcher: {search_time} seconds")
- logger.info(f" Backend returned: labels={len(results.get('labels', [[]])[0])} results")
-
- enriched_results = []
- if "labels" in results and "distances" in results:
- logger.info(f" Processing {len(results['labels'][0])} passage IDs:")
- # Python 3.9 does not support zip(strict=...); lengths are expected to match
- for i, (string_id, dist) in enumerate(
- zip(results["labels"][0], results["distances"][0])
- ):
- try:
- passage_data = self.passage_manager.get_passage(string_id)
- enriched_results.append(
- SearchResult(
- id=string_id,
- score=dist,
- text=passage_data["text"],
- metadata=passage_data.get("metadata", {}),
- )
- )
-
- # Color codes for better logging
- GREEN = "\033[92m"
- BLUE = "\033[94m"
- YELLOW = "\033[93m"
- RESET = "\033[0m"
-
- # Truncate text for display (first 100 chars)
- display_text = passage_data["text"]
- logger.info(
- f" {GREEN}✓{RESET} {BLUE}[{i + 1:2d}]{RESET} {YELLOW}ID:{RESET} '{string_id}' {YELLOW}Score:{RESET} {dist:.4f} {YELLOW}Text:{RESET} {display_text}"
- )
- except KeyError:
- RED = "\033[91m"
- RESET = "\033[0m"
- logger.error(
- f" {RED}✗{RESET} [{i + 1:2d}] ID: '{string_id}' -> {RED}ERROR: Passage not found!{RESET}"
- )
-
- # Apply metadata filters if specified
- if metadata_filters:
- logger.info(f" 🔍 Applying metadata filters: {metadata_filters}")
- enriched_results = self.passage_manager.filter_search_results(
- enriched_results, metadata_filters
- )
-
- # Define color codes outside the loop for final message
- GREEN = "\033[92m"
- RESET = "\033[0m"
- logger.info(f" {GREEN}✓ Final enriched results: {len(enriched_results)} passages{RESET}")
- return enriched_results
-
- def _find_jsonl_file(self) -> Optional[str]:
- """Find the .jsonl file containing raw passages for grep search"""
- index_path = Path(self.meta_path_str).parent
- potential_files = [
- index_path / "documents.leann.passages.jsonl",
- index_path.parent / "documents.leann.passages.jsonl",
- ]
-
- for file_path in potential_files:
- if file_path.exists():
- return str(file_path)
- return None
-
- def _grep_search(self, query: str, top_k: int = 5) -> list[SearchResult]:
- """Perform grep-based search on raw passages"""
- jsonl_file = self._find_jsonl_file()
- if not jsonl_file:
- raise FileNotFoundError("No .jsonl passages file found for grep search")
-
- try:
- cmd = ["grep", "-i", "-n", query, jsonl_file]
- result = subprocess.run(cmd, capture_output=True, text=True, check=False)
-
- if result.returncode == 1:
- return []
- elif result.returncode != 0:
- raise RuntimeError(f"Grep failed: {result.stderr}")
-
- matches = []
- for line in result.stdout.strip().split("\n"):
- if not line:
- continue
- parts = line.split(":", 1)
- if len(parts) != 2:
- continue
-
- try:
- data = json.loads(parts[1])
- text = data.get("text", "")
- score = text.lower().count(query.lower())
-
- matches.append(
- SearchResult(
- id=data.get("id", parts[0]),
- text=text,
- metadata=data.get("metadata", {}),
- score=float(score),
- )
- )
- except json.JSONDecodeError:
- continue
-
- matches.sort(key=lambda x: x.score, reverse=True)
- return matches[:top_k]
-
- except FileNotFoundError:
- raise RuntimeError(
- "grep command not found. Please install grep or use semantic search."
- )
-
- def _python_regex_search(self, query: str, top_k: int = 5) -> list[SearchResult]:
- """Fallback regex search"""
- jsonl_file = self._find_jsonl_file()
- if not jsonl_file:
- raise FileNotFoundError("No .jsonl file found")
-
- pattern = re.compile(re.escape(query), re.IGNORECASE)
- matches = []
-
- with open(jsonl_file, encoding="utf-8") as f:
- for line_num, line in enumerate(f, 1):
- if pattern.search(line):
- try:
- data = json.loads(line.strip())
- matches.append(
- SearchResult(
- id=data.get("id", str(line_num)),
- text=data.get("text", ""),
- metadata=data.get("metadata", {}),
- score=float(len(pattern.findall(data.get("text", "")))),
- )
- )
- except json.JSONDecodeError:
- continue
-
- matches.sort(key=lambda x: x.score, reverse=True)
- return matches[:top_k]
-
- def cleanup(self):
- """Explicitly cleanup embedding server resources.
- This method should be called after you're done using the searcher,
- especially in test environments or batch processing scenarios.
- """
- backend = getattr(self.backend_impl, "embedding_server_manager", None)
- if backend is not None:
- backend.stop_server()
-
- # Enable automatic cleanup patterns
- def __enter__(self):
- return self
-
- def __exit__(self, exc_type, exc, tb):
- try:
- self.cleanup()
- except Exception:
- pass
-
- def __del__(self):
- try:
- self.cleanup()
- except Exception:
- # Avoid noisy errors during interpreter shutdown
- pass
-
-
-class LeannChat:
- def __init__(
- self,
- index_path: str,
- llm_config: Optional[dict[str, Any]] = None,
- enable_warmup: bool = False,
- searcher: Optional[LeannSearcher] = None,
- **kwargs,
- ):
- if searcher is None:
- self.searcher = LeannSearcher(index_path, enable_warmup=enable_warmup, **kwargs)
- self._owns_searcher = True
- else:
- self.searcher = searcher
- self._owns_searcher = False
- self.llm = get_llm(llm_config)
-
- def ask(
- self,
- question: str,
- top_k: int = 5,
- complexity: int = 64,
- beam_width: int = 1,
- prune_ratio: float = 0.0,
- recompute_embeddings: bool = True,
- pruning_strategy: Literal["global", "local", "proportional"] = "global",
- llm_kwargs: Optional[dict[str, Any]] = None,
- expected_zmq_port: int = 5557,
- metadata_filters: Optional[dict[str, dict[str, Union[str, int, float, bool, list]]]] = None,
- batch_size: int = 0,
- use_grep: bool = False,
- **search_kwargs,
- ):
- if llm_kwargs is None:
- llm_kwargs = {}
- search_time = time.time()
- results = self.searcher.search(
- question,
- top_k=top_k,
- complexity=complexity,
- beam_width=beam_width,
- prune_ratio=prune_ratio,
- recompute_embeddings=recompute_embeddings,
- pruning_strategy=pruning_strategy,
- expected_zmq_port=expected_zmq_port,
- metadata_filters=metadata_filters,
- batch_size=batch_size,
- **search_kwargs,
- )
- search_time = time.time() - search_time
- logger.info(f" Search time: {search_time} seconds")
- context = "\n\n".join([r.text for r in results])
- prompt = (
- "Here is some retrieved context that might help answer your question:\n\n"
- f"{context}\n\n"
- f"Question: {question}\n\n"
- "Please provide the best answer you can based on this context and your knowledge."
- )
-
- logger.info("The context provided to the LLM is:")
- logger.info(f"{'Relevance':<10} | {'Chunk id':<10} | {'Content':<60} | {'Source':<80}")
- logger.info("-" * 150)
- for r in results:
- chunk_relevance = f"{r.score:.3f}"
- chunk_id = r.id
- chunk_content = r.text[:60]
- chunk_source = r.metadata.get("source", "")[:80]
- logger.info(
- f"{chunk_relevance:<10} | {chunk_id:<10} | {chunk_content:<60} | {chunk_source:<80}"
- )
- ask_time = time.time()
- ans = self.llm.ask(prompt, **llm_kwargs)
- ask_time = time.time() - ask_time
- logger.info(f" Ask time: {ask_time} seconds")
- return ans
-
- def start_interactive(self):
- """Start interactive chat session."""
- session = create_api_session()
-
- def handle_query(user_input: str):
- response = self.ask(user_input)
- print(f"Leann: {response}")
-
- session.run_interactive_loop(handle_query)
-
- def cleanup(self):
- """Explicitly cleanup embedding server resources.
-
- This method should be called after you're done using the chat interface,
- especially in test environments or batch processing scenarios.
- """
- # Only stop the embedding server if this LeannChat instance created the searcher.
- # When a shared searcher is passed in, avoid shutting down the server to enable reuse.
- if getattr(self, "_owns_searcher", False) and hasattr(self.searcher, "cleanup"):
- self.searcher.cleanup()
-
- # Enable automatic cleanup patterns
- def __enter__(self):
- return self
-
- def __exit__(self, exc_type, exc, tb):
- try:
- self.cleanup()
- except Exception:
- pass
-
- def __del__(self):
- try:
- self.cleanup()
- except Exception:
- pass
+"""
+This file contains the core API for the LEANN project, now definitively updated
+with the correct, original embedding logic from the user's reference code.
+"""
+
+import json
+import logging
+import os
+import pickle
+import re
+import subprocess
+import time
+import warnings
+from collections import Counter, defaultdict
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Literal, Optional, Union
+
+import numpy as np
+from leann_backend_hnsw.convert_to_csr import prune_hnsw_embeddings_inplace
+
+from leann.interactive_utils import create_api_session
+from leann.interface import LeannBackendSearcherInterface
+
+from .chat import get_llm
+from .embedding_server_manager import EmbeddingServerManager
+from .interface import LeannBackendFactoryInterface
+from .metadata_filter import MetadataFilterEngine
+from .registry import BACKEND_REGISTRY
+
+logger = logging.getLogger(__name__)
+
+
+def get_registered_backends() -> list[str]:
+ """Get list of registered backend names."""
+ return list(BACKEND_REGISTRY.keys())
+
+
+def compute_embeddings(
+ chunks: list[str],
+ model_name: str,
+ mode: str = "sentence-transformers",
+ use_server: bool = True,
+ port: Optional[int] = None,
+ is_build=False,
+ provider_options: Optional[dict[str, Any]] = None,
+) -> np.ndarray:
+ """
+ Computes embeddings using different backends.
+
+ Args:
+ chunks: List of text chunks to embed
+ model_name: Name of the embedding model
+ mode: Embedding backend mode. Options:
+ - "sentence-transformers": Use sentence-transformers library (default)
+ - "mlx": Use MLX backend for Apple Silicon
+ - "openai": Use OpenAI embedding API
+ - "gemini": Use Google Gemini embedding API
+ use_server: Whether to use embedding server (True for search, False for build)
+
+ Returns:
+ numpy array of embeddings
+ """
+ if use_server:
+ # Use embedding server (for search/query)
+ if port is None:
+ raise ValueError("port is required when use_server is True")
+ return compute_embeddings_via_server(chunks, model_name, port=port)
+ else:
+ # Use direct computation (for build_index)
+ from .embedding_compute import (
+ compute_embeddings as compute_embeddings_direct,
+ )
+
+ return compute_embeddings_direct(
+ chunks,
+ model_name,
+ mode=mode,
+ is_build=is_build,
+ provider_options=provider_options,
+ )
+
+
+def compute_embeddings_via_server(chunks: list[str], model_name: str, port: int) -> np.ndarray:
+ """Computes embeddings using sentence-transformers.
+
+ Args:
+ chunks: List of text chunks to embed
+ model_name: Name of the sentence transformer model
+ """
+ logger.info(
+ f"Computing embeddings for {len(chunks)} chunks using SentenceTransformer model '{model_name}' (via embedding server)..."
+ )
+ import msgpack
+ import numpy as np
+ import zmq
+
+ # Connect to embedding server
+ context = zmq.Context()
+ socket = context.socket(zmq.REQ)
+ socket.connect(f"tcp://localhost:{port}")
+
+ # Send chunks to server for embedding computation
+ request = chunks
+ socket.send(msgpack.packb(request))
+
+ # Receive embeddings from server
+ response = socket.recv()
+ embeddings_list = msgpack.unpackb(response)
+
+ # Convert back to numpy array
+ embeddings = np.array(embeddings_list, dtype=np.float32)
+
+ socket.close()
+ context.term()
+
+ return embeddings
+
+
+@dataclass
+class SearchResult:
+ id: str
+ score: float
+ text: str
+ metadata: dict[str, Any] = field(default_factory=dict)
+
+
+class PassageManager:
+ def __init__(
+ self, passage_sources: list[dict[str, Any]], metadata_file_path: Optional[str] = None
+ ):
+ self.offset_maps: dict[str, dict[str, int]] = {}
+ self.passage_files: dict[str, str] = {}
+ # Avoid materializing a single gigantic global map to reduce memory
+ # footprint on very large corpora (e.g., 60M+ passages). Instead, keep
+ # per-shard maps and do a lightweight per-shard lookup on demand.
+ self._total_count: int = 0
+ self.filter_engine = MetadataFilterEngine() # Initialize filter engine
+
+ # Derive index base name for standard sibling fallbacks, e.g., <index_name>.passages.*
+ index_name_base = None
+ if metadata_file_path:
+ meta_name = Path(metadata_file_path).name
+ if meta_name.endswith(".meta.json"):
+ index_name_base = meta_name[: -len(".meta.json")]
+
+ for source in passage_sources:
+ assert source["type"] == "jsonl", "only jsonl is supported"
+ passage_file = source.get("path", "")
+ index_file = source.get("index_path", "") # .idx file
+
+ # Fix path resolution - relative paths should be relative to metadata file directory
+ def _resolve_candidates(
+ primary: str,
+ relative_key: str,
+ default_name: Optional[str],
+ source_dict: dict[str, Any],
+ ) -> list[Path]:
+ """
+ Build an ordered list of candidate paths. For relative paths specified in
+ metadata, prefer resolution relative to the metadata file directory first,
+ then fall back to CWD-based resolution, and finally to conventional
+ sibling defaults (e.g., <index_base>.passages.idx / .jsonl).
+ """
+ candidates: list[Path] = []
+ # 1) Primary path
+ if primary:
+ p = Path(primary)
+ if p.is_absolute():
+ candidates.append(p)
+ else:
+ # Prefer metadata-relative resolution for relative paths
+ if metadata_file_path:
+ candidates.append(Path(metadata_file_path).parent / p)
+ # Also consider CWD-relative as a fallback for legacy layouts
+ candidates.append(Path.cwd() / p)
+ # 2) metadata-relative explicit relative key (if present)
+ if metadata_file_path and source_dict.get(relative_key):
+ candidates.append(Path(metadata_file_path).parent / source_dict[relative_key])
+ # 3) metadata-relative standard sibling filename
+ if metadata_file_path and default_name:
+ candidates.append(Path(metadata_file_path).parent / default_name)
+ return candidates
+
+ # Build candidate lists and pick first existing; otherwise keep last candidate for error message
+ idx_default = f"{index_name_base}.passages.idx" if index_name_base else None
+ idx_candidates = _resolve_candidates(
+ index_file, "index_path_relative", idx_default, source
+ )
+ pas_default = f"{index_name_base}.passages.jsonl" if index_name_base else None
+ pas_candidates = _resolve_candidates(passage_file, "path_relative", pas_default, source)
+
+ def _pick_existing(cands: list[Path]) -> str:
+ for c in cands:
+ if c.exists():
+ return str(c.resolve())
+ # Fallback to last candidate (best guess) even if not exists; will error below
+ return str(cands[-1].resolve()) if cands else ""
+
+ index_file = _pick_existing(idx_candidates)
+ passage_file = _pick_existing(pas_candidates)
+
+ if not Path(index_file).exists():
+ raise FileNotFoundError(f"Passage index file not found: {index_file}")
+
+ with open(index_file, "rb") as f:
+ offset_map: dict[str, int] = pickle.load(f)
+ self.offset_maps[passage_file] = offset_map
+ self.passage_files[passage_file] = passage_file
+ self._total_count += len(offset_map)
+
+ def get_passage(self, passage_id: str) -> dict[str, Any]:
+ # Fast path: check each shard map (there are typically few shards).
+ # This avoids building a massive combined dict while keeping lookups
+ # bounded by the number of shards.
+ for passage_file, offset_map in self.offset_maps.items():
+ try:
+ offset = offset_map[passage_id]
+ with open(passage_file, encoding="utf-8") as f:
+ f.seek(offset)
+ return json.loads(f.readline())
+ except KeyError:
+ continue
+ raise KeyError(f"Passage ID not found: {passage_id}")
+
+ def filter_search_results(
+ self,
+ search_results: list[SearchResult],
+ metadata_filters: Optional[dict[str, dict[str, Union[str, int, float, bool, list]]]],
+ ) -> list[SearchResult]:
+ """
+ Apply metadata filters to search results.
+
+ Args:
+ search_results: List of SearchResult objects
+ metadata_filters: Filter specifications to apply
+
+ Returns:
+ Filtered list of SearchResult objects
+ """
+ if not metadata_filters:
+ return search_results
+
+ logger.debug(f"Applying metadata filters to {len(search_results)} results")
+
+ # Convert SearchResult objects to dictionaries for the filter engine
+ result_dicts = []
+ for result in search_results:
+ result_dicts.append(
+ {
+ "id": result.id,
+ "score": result.score,
+ "text": result.text,
+ "metadata": result.metadata,
+ }
+ )
+
+ # Apply filters using the filter engine
+ filtered_dicts = self.filter_engine.apply_filters(result_dicts, metadata_filters)
+
+ # Convert back to SearchResult objects
+ filtered_results = []
+ for result_dict in filtered_dicts:
+ filtered_results.append(
+ SearchResult(
+ id=result_dict["id"],
+ score=result_dict["score"],
+ text=result_dict["text"],
+ metadata=result_dict["metadata"],
+ )
+ )
+
+ logger.debug(f"Filtered results: {len(filtered_results)} remaining")
+ return filtered_results
+
+ def __len__(self) -> int:
+ return self._total_count
+
+
+class BM25Scorer:
+ def __init__(self, k1: float = 1.2, b: float = 0.75):
+ self.k1 = k1
+ self.b = b
+ self.doc_freqs = None # How many docs contain each term (DF)
+ self.doc_lengths = {} # How long each doc is (in words)
+ self.word_counts = {} # How many times each word appears in each doc (TF)
+ self.avg_doc_length = None
+ self.corpus_size = None
+ self.idlist = set() # List of all document IDs for easier searching
+
+ def _tokenize(self, text: str) -> list[str]:
+ return re.sub(r"[^\w\s]", "", text).lower().split()
+
+ def fit(self, documents: list[dict[str, Any]]):
+ """
+ Build BM25 statistics from a document corpus.
+ Must be called before scoring.
+ """
+ self.corpus_size = len(documents)
+ self.doc_lengths = {}
+ self.word_counts = {}
+ self.idlist = set()
+ doc_freqs = defaultdict(int)
+
+ for doc_data in documents:
+ doc_id = doc_data["id"]
+ words = self._tokenize(doc_data["text"])
+ doc_length = len(words)
+ self.doc_lengths[doc_id] = doc_length
+
+ unique_words = set(words)
+ for word in unique_words:
+ doc_freqs[word] += 1
+ self.word_counts[doc_id] = dict(Counter(words))
+ self.idlist.add(doc_id)
+
+ self.doc_freqs = dict(doc_freqs)
+ self.avg_doc_length = sum(self.doc_lengths.values()) / len(self.doc_lengths)
+
+ def score(self, query_words: list[str], document_id: str) -> float:
+ if (
+ self.doc_freqs is None
+ or self.doc_lengths == {}
+ or self.word_counts == {}
+ or self.avg_doc_length is None
+ or self.corpus_size is None
+ ):
+ raise ValueError("BM25 model not fitted. Call fit() before scoring.")
+
+ passage_words = self.word_counts[document_id]
+ passage_length = sum(passage_words.values())
+ score = 0.0
+ for word in query_words:
+ if word not in self.doc_freqs:
+ continue
+ word_freq = passage_words[word] if word in passage_words else 0
+ idf = np.log(
+ (self.corpus_size - self.doc_freqs[word] + 0.5) / (self.doc_freqs[word] + 0.5) + 1
+ )
+ tf = (word_freq * (self.k1 + 1)) / (
+ word_freq + self.k1 * (1 - self.b + self.b * (passage_length / self.avg_doc_length))
+ )
+ score += idf * tf
+ return score
+
+ def search(self, query: str, top_k: int = 5) -> list[SearchResult]:
+ query_words = self._tokenize(query)
+ scores = {doc_id: self.score(query_words, doc_id) for doc_id in self.idlist}
+ sorted_scores = sorted(scores.items(), key=lambda x: x[1], reverse=True)
+ return [
+ SearchResult(id=doc_id, score=score, text="", metadata={})
+ for doc_id, score in sorted_scores[:top_k]
+ ]
+
+
+class LeannBuilder:
+ def __init__(
+ self,
+ backend_name: str,
+ embedding_model: str = "facebook/contriever",
+ dimensions: Optional[int] = None,
+ embedding_mode: str = "sentence-transformers",
+ embedding_options: Optional[dict[str, Any]] = None,
+ **backend_kwargs,
+ ):
+ self.backend_name = backend_name
+ # Normalize incompatible combinations early (for consistent metadata)
+ if backend_name == "hnsw":
+ is_recompute = backend_kwargs.get("is_recompute", True)
+ is_compact = backend_kwargs.get("is_compact", True)
+ if is_recompute is False and is_compact is True:
+ warnings.warn(
+ "HNSW with is_recompute=False requires non-compact storage. Forcing is_compact=False.",
+ UserWarning,
+ stacklevel=2,
+ )
+ backend_kwargs["is_compact"] = False
+
+ backend_factory: Optional[LeannBackendFactoryInterface] = BACKEND_REGISTRY.get(backend_name)
+ if backend_factory is None:
+ raise ValueError(f"Backend '{backend_name}' not found or not registered.")
+ self.backend_factory = backend_factory
+ self.embedding_model = embedding_model
+ self.dimensions = dimensions
+ self.embedding_mode = embedding_mode
+ self.embedding_options = embedding_options or {}
+
+ # Check if we need to use cosine distance for normalized embeddings
+ normalized_embeddings_models = {
+ # OpenAI models
+ ("openai", "text-embedding-ada-002"),
+ ("openai", "text-embedding-3-small"),
+ ("openai", "text-embedding-3-large"),
+ # Voyage AI models
+ ("voyage", "voyage-2"),
+ ("voyage", "voyage-3"),
+ ("voyage", "voyage-large-2"),
+ ("voyage", "voyage-multilingual-2"),
+ ("voyage", "voyage-code-2"),
+ # Cohere models
+ ("cohere", "embed-english-v3.0"),
+ ("cohere", "embed-multilingual-v3.0"),
+ ("cohere", "embed-english-light-v3.0"),
+ ("cohere", "embed-multilingual-light-v3.0"),
+ }
+
+ # Also check for patterns in model names
+ is_normalized = False
+ current_model_lower = embedding_model.lower()
+ current_mode_lower = embedding_mode.lower()
+
+ # Check exact matches
+ for mode, model in normalized_embeddings_models:
+ if (current_mode_lower == mode and current_model_lower == model) or (
+ mode in current_mode_lower and model in current_model_lower
+ ):
+ is_normalized = True
+ break
+
+ # Check patterns
+ if not is_normalized:
+ # OpenAI patterns
+ if "openai" in current_mode_lower or "openai" in current_model_lower:
+ if any(
+ pattern in current_model_lower
+ for pattern in ["text-embedding", "ada", "3-small", "3-large"]
+ ):
+ is_normalized = True
+ # Voyage patterns
+ elif "voyage" in current_mode_lower or "voyage" in current_model_lower:
+ is_normalized = True
+ # Cohere patterns
+ elif "cohere" in current_mode_lower or "cohere" in current_model_lower:
+ if "embed" in current_model_lower:
+ is_normalized = True
+
+ # Handle distance metric
+ if is_normalized and "distance_metric" not in backend_kwargs:
+ backend_kwargs["distance_metric"] = "cosine"
+ warnings.warn(
+ f"Detected normalized embeddings model '{embedding_model}' with mode '{embedding_mode}'. "
+ f"Automatically setting distance_metric='cosine' for optimal performance. "
+ f"Normalized embeddings (L2 norm = 1) should use cosine similarity instead of MIPS.",
+ UserWarning,
+ stacklevel=2,
+ )
+ elif is_normalized and backend_kwargs.get("distance_metric", "").lower() != "cosine":
+ current_metric = backend_kwargs.get("distance_metric", "mips")
+ warnings.warn(
+ f"Warning: Using '{current_metric}' distance metric with normalized embeddings model "
+ f"'{embedding_model}' may lead to suboptimal search results. "
+ f"Consider using 'cosine' distance metric for better performance.",
+ UserWarning,
+ stacklevel=2,
+ )
+
+ self.backend_kwargs = backend_kwargs
+ self.chunks: list[dict[str, Any]] = []
+
+ def add_text(self, text: str, metadata: Optional[dict[str, Any]] = None):
+ if metadata is None:
+ metadata = {}
+ passage_id = metadata.get("id", str(len(self.chunks)))
+ chunk_data = {"id": passage_id, "text": text, "metadata": metadata}
+ self.chunks.append(chunk_data)
+
+ def build_index(self, index_path: str):
+ if not self.chunks:
+ raise ValueError("No chunks added.")
+
+ # Filter out invalid/empty text chunks early to keep passage and embedding counts aligned
+ valid_chunks: list[dict[str, Any]] = []
+ skipped = 0
+ for chunk in self.chunks:
+ text = chunk.get("text", "")
+ if isinstance(text, str) and text.strip():
+ valid_chunks.append(chunk)
+ else:
+ skipped += 1
+ if skipped > 0:
+ print(
+ f"Warning: Skipping {skipped} empty/invalid text chunk(s). Processing {len(valid_chunks)} valid chunks"
+ )
+ self.chunks = valid_chunks
+ if not self.chunks:
+ raise ValueError("All provided chunks are empty or invalid. Nothing to index.")
+ if self.dimensions is None:
+ self.dimensions = len(
+ compute_embeddings(
+ ["dummy"],
+ self.embedding_model,
+ self.embedding_mode,
+ use_server=False,
+ provider_options=self.embedding_options,
+ )[0]
+ )
+ path = Path(index_path)
+ index_dir = path.parent
+ index_name = path.name
+ index_dir.mkdir(parents=True, exist_ok=True)
+ passages_file = index_dir / f"{index_name}.passages.jsonl"
+ offset_file = index_dir / f"{index_name}.passages.idx"
+ offset_map = {}
+ with open(passages_file, "w", encoding="utf-8") as f:
+ try:
+ from tqdm import tqdm
+
+ chunk_iterator = tqdm(self.chunks, desc="Writing passages", unit="chunk")
+ except ImportError:
+ chunk_iterator = self.chunks
+
+ for chunk in chunk_iterator:
+ offset = f.tell()
+ json.dump(
+ {
+ "id": chunk["id"],
+ "text": chunk["text"],
+ "metadata": chunk["metadata"],
+ },
+ f,
+ ensure_ascii=False,
+ )
+ f.write("\n")
+ offset_map[chunk["id"]] = offset
+ with open(offset_file, "wb") as f:
+ pickle.dump(offset_map, f)
+ texts_to_embed = [c["text"] for c in self.chunks]
+ embeddings = compute_embeddings(
+ texts_to_embed,
+ self.embedding_model,
+ self.embedding_mode,
+ use_server=False,
+ is_build=True,
+ provider_options=self.embedding_options,
+ )
+ string_ids = [chunk["id"] for chunk in self.chunks]
+ # Persist ID map alongside index so backends that return integer labels can remap to passage IDs
+ try:
+ idmap_file = (
+ index_dir
+ / f"{index_name[: -len('.leann')] if index_name.endswith('.leann') else index_name}.ids.txt"
+ )
+ with open(idmap_file, "w", encoding="utf-8") as f:
+ for sid in string_ids:
+ f.write(str(sid) + "\n")
+ except Exception:
+ pass
+ current_backend_kwargs = {**self.backend_kwargs, "dimensions": self.dimensions}
+ builder_instance = self.backend_factory.builder(**current_backend_kwargs)
+ builder_instance.build(embeddings, string_ids, index_path, **current_backend_kwargs)
+ leann_meta_path = index_dir / f"{index_name}.meta.json"
+ meta_data = {
+ "version": "1.0",
+ "backend_name": self.backend_name,
+ "embedding_model": self.embedding_model,
+ "dimensions": self.dimensions,
+ "backend_kwargs": self.backend_kwargs,
+ "embedding_mode": self.embedding_mode,
+ "passage_sources": [
+ {
+ "type": "jsonl",
+ # Preserve existing relative file names (backward-compatible)
+ "path": passages_file.name,
+ "index_path": offset_file.name,
+ # Add optional redundant relative keys for remote build portability (non-breaking)
+ "path_relative": passages_file.name,
+ "index_path_relative": offset_file.name,
+ }
+ ],
+ }
+
+ if self.embedding_options:
+ meta_data["embedding_options"] = self.embedding_options
+
+ # Add storage status flags for HNSW backend
+ if self.backend_name == "hnsw":
+ is_compact = self.backend_kwargs.get("is_compact", True)
+ is_recompute = self.backend_kwargs.get("is_recompute", True)
+ meta_data["is_compact"] = is_compact
+ meta_data["is_pruned"] = bool(is_recompute)
+ with open(leann_meta_path, "w", encoding="utf-8") as f:
+ json.dump(meta_data, f, indent=2)
+
+ def build_index_from_embeddings(self, index_path: str, embeddings_file: str):
+ """
+ Build an index from pre-computed embeddings stored in a pickle file.
+
+ Args:
+ index_path: Path where the index will be saved
+ embeddings_file: Path to pickle file containing (ids, embeddings) tuple
+ """
+ # Load pre-computed embeddings
+ with open(embeddings_file, "rb") as f:
+ data = pickle.load(f)
+
+ if not isinstance(data, tuple) or len(data) != 2:
+ raise ValueError(
+ f"Invalid embeddings file format. Expected tuple with 2 elements, got {type(data)}"
+ )
+
+ ids, embeddings = data
+
+ if not isinstance(embeddings, np.ndarray):
+ raise ValueError(f"Expected embeddings to be numpy array, got {type(embeddings)}")
+
+ if len(ids) != embeddings.shape[0]:
+ raise ValueError(
+ f"Mismatch between number of IDs ({len(ids)}) and embeddings ({embeddings.shape[0]})"
+ )
+
+ # Validate/set dimensions
+ embedding_dim = embeddings.shape[1]
+ if self.dimensions is None:
+ self.dimensions = embedding_dim
+ elif self.dimensions != embedding_dim:
+ raise ValueError(f"Dimension mismatch: expected {self.dimensions}, got {embedding_dim}")
+
+ logger.info(
+ f"Building index from precomputed embeddings: {len(ids)} items, {embedding_dim} dimensions"
+ )
+
+ # Ensure we have text data for each embedding
+ if len(self.chunks) != len(ids):
+ # If no text chunks provided, create placeholder text entries
+ if not self.chunks:
+ logger.info("No text chunks provided, creating placeholder entries...")
+ for id_val in ids:
+ self.add_text(
+ f"Document {id_val}",
+ metadata={"id": str(id_val), "from_embeddings": True},
+ )
+ else:
+ raise ValueError(
+ f"Number of text chunks ({len(self.chunks)}) doesn't match number of embeddings ({len(ids)})"
+ )
+
+ # Build file structure
+ path = Path(index_path)
+ index_dir = path.parent
+ index_name = path.name
+ index_dir.mkdir(parents=True, exist_ok=True)
+ passages_file = index_dir / f"{index_name}.passages.jsonl"
+ offset_file = index_dir / f"{index_name}.passages.idx"
+
+ # Write passages and create offset map
+ offset_map = {}
+ with open(passages_file, "w", encoding="utf-8") as f:
+ for chunk in self.chunks:
+ offset = f.tell()
+ json.dump(
+ {
+ "id": chunk["id"],
+ "text": chunk["text"],
+ "metadata": chunk["metadata"],
+ },
+ f,
+ ensure_ascii=False,
+ )
+ f.write("\n")
+ offset_map[chunk["id"]] = offset
+
+ with open(offset_file, "wb") as f:
+ pickle.dump(offset_map, f)
+
+ # Build the vector index using precomputed embeddings
+ string_ids = [str(id_val) for id_val in ids]
+ # Persist ID map (order == embeddings order)
+ try:
+ idmap_file = (
+ index_dir
+ / f"{index_name[: -len('.leann')] if index_name.endswith('.leann') else index_name}.ids.txt"
+ )
+ with open(idmap_file, "w", encoding="utf-8") as f:
+ for sid in string_ids:
+ f.write(str(sid) + "\n")
+ except Exception:
+ pass
+ current_backend_kwargs = {**self.backend_kwargs, "dimensions": self.dimensions}
+ builder_instance = self.backend_factory.builder(**current_backend_kwargs)
+ builder_instance.build(embeddings, string_ids, index_path)
+
+ # Create metadata file
+ leann_meta_path = index_dir / f"{index_name}.meta.json"
+ meta_data = {
+ "version": "1.0",
+ "backend_name": self.backend_name,
+ "embedding_model": self.embedding_model,
+ "dimensions": self.dimensions,
+ "backend_kwargs": self.backend_kwargs,
+ "embedding_mode": self.embedding_mode,
+ "passage_sources": [
+ {
+ "type": "jsonl",
+ # Preserve existing relative file names (backward-compatible)
+ "path": passages_file.name,
+ "index_path": offset_file.name,
+ # Add optional redundant relative keys for remote build portability (non-breaking)
+ "path_relative": passages_file.name,
+ "index_path_relative": offset_file.name,
+ }
+ ],
+ "built_from_precomputed_embeddings": True,
+ "embeddings_source": str(embeddings_file),
+ }
+
+ if self.embedding_options:
+ meta_data["embedding_options"] = self.embedding_options
+
+ # Add storage status flags for HNSW backend
+ if self.backend_name == "hnsw":
+ is_compact = self.backend_kwargs.get("is_compact", True)
+ is_recompute = self.backend_kwargs.get("is_recompute", True)
+ meta_data["is_compact"] = is_compact
+ meta_data["is_pruned"] = bool(is_recompute)
+
+ with open(leann_meta_path, "w", encoding="utf-8") as f:
+ json.dump(meta_data, f, indent=2)
+
+ logger.info(f"Index built successfully from precomputed embeddings: {index_path}")
+
+ def update_index(self, index_path: str):
+ """Append new passages and vectors to an existing HNSW index."""
+ if not self.chunks:
+ raise ValueError("No new chunks provided for update.")
+
+ path = Path(index_path)
+ index_dir = path.parent
+ index_name = path.name
+ index_prefix = path.stem
+
+ meta_path = index_dir / f"{index_name}.meta.json"
+ passages_file = index_dir / f"{index_name}.passages.jsonl"
+ offset_file = index_dir / f"{index_name}.passages.idx"
+ index_file = index_dir / f"{index_prefix}.index"
+
+ if not meta_path.exists() or not passages_file.exists() or not offset_file.exists():
+ raise FileNotFoundError("Index metadata or passage files are missing; cannot update.")
+ if not index_file.exists():
+ raise FileNotFoundError(f"HNSW index file not found: {index_file}")
+
+ with open(meta_path, encoding="utf-8") as f:
+ meta = json.load(f)
+ backend_name = meta.get("backend_name")
+ if backend_name != self.backend_name:
+ raise ValueError(
+ f"Index was built with backend '{backend_name}', cannot update with '{self.backend_name}'."
+ )
+
+ meta_backend_kwargs = meta.get("backend_kwargs", {})
+ index_is_compact = meta.get("is_compact", meta_backend_kwargs.get("is_compact", True))
+ if index_is_compact:
+ raise ValueError(
+ "Compact HNSW indices do not support in-place updates. Rebuild required."
+ )
+
+ distance_metric = meta_backend_kwargs.get(
+ "distance_metric", self.backend_kwargs.get("distance_metric", "mips")
+ ).lower()
+ needs_recompute = bool(
+ meta.get("is_pruned")
+ or meta_backend_kwargs.get("is_recompute")
+ or self.backend_kwargs.get("is_recompute")
+ )
+
+ with open(offset_file, "rb") as f:
+ offset_map: dict[str, int] = pickle.load(f)
+ existing_ids = set(offset_map.keys())
+
+ valid_chunks: list[dict[str, Any]] = []
+ for chunk in self.chunks:
+ text = chunk.get("text", "")
+ if not isinstance(text, str) or not text.strip():
+ continue
+ metadata = chunk.setdefault("metadata", {})
+ passage_id = chunk.get("id") or metadata.get("id")
+ if passage_id and passage_id in existing_ids:
+ raise ValueError(f"Passage ID '{passage_id}' already exists in the index.")
+ valid_chunks.append(chunk)
+
+ if not valid_chunks:
+ raise ValueError("No valid chunks to append.")
+
+ texts_to_embed = [chunk["text"] for chunk in valid_chunks]
+ embeddings = compute_embeddings(
+ texts_to_embed,
+ self.embedding_model,
+ self.embedding_mode,
+ use_server=False,
+ is_build=True,
+ provider_options=self.embedding_options,
+ )
+
+ embedding_dim = embeddings.shape[1]
+ expected_dim = meta.get("dimensions")
+ if expected_dim is not None and expected_dim != embedding_dim:
+ raise ValueError(
+ f"Dimension mismatch during update: existing index uses {expected_dim}, got {embedding_dim}."
+ )
+
+ from leann_backend_hnsw import faiss # type: ignore
+
+ embeddings = np.ascontiguousarray(embeddings, dtype=np.float32)
+ if distance_metric == "cosine":
+ norms = np.linalg.norm(embeddings, axis=1, keepdims=True)
+ norms[norms == 0] = 1
+ embeddings = embeddings / norms
+
+ index = faiss.read_index(str(index_file))
+ if hasattr(index, "is_recompute"):
+ index.is_recompute = needs_recompute
+ print(f"index.is_recompute: {index.is_recompute}")
+ if getattr(index, "storage", None) is None:
+ if index.metric_type == faiss.METRIC_INNER_PRODUCT:
+ storage_index = faiss.IndexFlatIP(index.d)
+ else:
+ storage_index = faiss.IndexFlatL2(index.d)
+ index.storage = storage_index
+ index.own_fields = True
+ # Faiss expects storage.ntotal to reflect the existing graph's
+ # population (even if the vectors themselves were pruned from disk
+ # for recompute mode). When we attach a fresh IndexFlat here its
+ # ntotal starts at zero, which later causes IndexHNSW::add to
+ # believe new "preset" levels were provided and trips the
+ # `n0 + n == levels.size()` assertion. Seed the temporary storage
+ # with the current ntotal so Faiss maintains the proper offset for
+ # incoming vectors.
+ try:
+ storage_index.ntotal = index.ntotal
+ except AttributeError:
+ # Older Faiss builds may not expose ntotal as a writable
+ # attribute; in that case we fall back to the default behaviour.
+ pass
+ if index.d != embedding_dim:
+ raise ValueError(
+ f"Existing index dimension ({index.d}) does not match new embeddings ({embedding_dim})."
+ )
+
+ passage_meta_mode = meta.get("embedding_mode", self.embedding_mode)
+ passage_provider_options = meta.get("embedding_options", self.embedding_options)
+
+ base_id = index.ntotal
+ for offset, chunk in enumerate(valid_chunks):
+ new_id = str(base_id + offset)
+ chunk.setdefault("metadata", {})["id"] = new_id
+ chunk["id"] = new_id
+
+ # Append passages/offsets before we attempt index.add so the ZMQ server
+ # can resolve newly assigned IDs during recompute. Keep rollback hooks
+ # so we can restore files if the update fails mid-way.
+ rollback_passages_size = passages_file.stat().st_size if passages_file.exists() else 0
+ offset_map_backup = offset_map.copy()
+
+ try:
+ with open(passages_file, "a", encoding="utf-8") as f:
+ for chunk in valid_chunks:
+ offset = f.tell()
+ json.dump(
+ {
+ "id": chunk["id"],
+ "text": chunk["text"],
+ "metadata": chunk.get("metadata", {}),
+ },
+ f,
+ ensure_ascii=False,
+ )
+ f.write("\n")
+ offset_map[chunk["id"]] = offset
+
+ with open(offset_file, "wb") as f:
+ pickle.dump(offset_map, f)
+
+ server_manager: Optional[EmbeddingServerManager] = None
+ server_started = False
+ requested_zmq_port = int(os.getenv("LEANN_UPDATE_ZMQ_PORT", "5557"))
+
+ try:
+ if needs_recompute:
+ server_manager = EmbeddingServerManager(
+ backend_module_name="leann_backend_hnsw.hnsw_embedding_server"
+ )
+ server_started, actual_port = server_manager.start_server(
+ port=requested_zmq_port,
+ model_name=self.embedding_model,
+ embedding_mode=passage_meta_mode,
+ passages_file=str(meta_path),
+ distance_metric=distance_metric,
+ provider_options=passage_provider_options,
+ )
+ if not server_started:
+ raise RuntimeError(
+ "Failed to start HNSW embedding server for recompute update."
+ )
+ if actual_port != requested_zmq_port:
+ logger.warning(
+ "Embedding server started on port %s instead of requested %s. "
+ "Using reassigned port.",
+ actual_port,
+ requested_zmq_port,
+ )
+ if hasattr(index.hnsw, "set_zmq_port"):
+ index.hnsw.set_zmq_port(actual_port)
+ elif hasattr(index, "set_zmq_port"):
+ index.set_zmq_port(actual_port)
+
+ if needs_recompute:
+ for i in range(embeddings.shape[0]):
+ print(f"add {i} embeddings")
+ index.add(1, faiss.swig_ptr(embeddings[i : i + 1]))
+ else:
+ index.add(embeddings.shape[0], faiss.swig_ptr(embeddings))
+ faiss.write_index(index, str(index_file))
+ finally:
+ if server_started and server_manager is not None:
+ server_manager.stop_server()
+
+ except Exception:
+ # Roll back appended passages/offset map to keep files consistent.
+ if passages_file.exists():
+ with open(passages_file, "rb+") as f:
+ f.truncate(rollback_passages_size)
+ offset_map = offset_map_backup
+ with open(offset_file, "wb") as f:
+ pickle.dump(offset_map, f)
+ raise
+
+ meta["total_passages"] = len(offset_map)
+ with open(meta_path, "w", encoding="utf-8") as f:
+ json.dump(meta, f, indent=2)
+
+ logger.info(
+ "Appended %d passages to index '%s'. New total: %d",
+ len(valid_chunks),
+ index_path,
+ len(offset_map),
+ )
+
+ self.chunks.clear()
+
+ if needs_recompute:
+ prune_hnsw_embeddings_inplace(str(index_file))
+
+
+class LeannSearcher:
+ def __init__(self, index_path: str, enable_warmup: bool = False, **backend_kwargs):
+ # Fix path resolution for Colab and other environments
+ if not Path(index_path).is_absolute():
+ index_path = str(Path(index_path).resolve())
+
+ self.meta_path_str = f"{index_path}.meta.json"
+ if not Path(self.meta_path_str).exists():
+ parent_dir = Path(index_path).parent
+ print(
+ f"Leann metadata file not found at {self.meta_path_str}, and you may need to rm -rf {parent_dir}"
+ )
+ # highlight in red the filenotfound error
+ raise FileNotFoundError(
+ f"Leann metadata file not found at {self.meta_path_str}, \033[91m you may need to rm -rf {parent_dir}\033[0m"
+ )
+ with open(self.meta_path_str, encoding="utf-8") as f:
+ self.meta_data = json.load(f)
+ backend_name = self.meta_data["backend_name"]
+ self.embedding_model = self.meta_data["embedding_model"]
+ # Support both old and new format
+ self.embedding_mode = self.meta_data.get("embedding_mode", "sentence-transformers")
+ self.embedding_options = self.meta_data.get("embedding_options", {})
+ # Delegate portability handling to PassageManager
+ self.passage_manager = PassageManager(
+ self.meta_data.get("passage_sources", []), metadata_file_path=self.meta_path_str
+ )
+ # Preserve backend name for conditional parameter forwarding
+ self.backend_name = backend_name
+ backend_factory = BACKEND_REGISTRY.get(backend_name)
+ if backend_factory is None:
+ raise ValueError(f"Backend '{backend_name}' not found.")
+ final_kwargs = {**self.meta_data.get("backend_kwargs", {}), **backend_kwargs}
+ final_kwargs["enable_warmup"] = enable_warmup
+ if self.embedding_options:
+ final_kwargs.setdefault("embedding_options", self.embedding_options)
+ self.backend_impl: LeannBackendSearcherInterface = backend_factory.searcher(
+ index_path, **final_kwargs
+ )
+ self.bm25_scorer = None
+
+ def search(
+ self,
+ query: str,
+ top_k: int = 5,
+ complexity: int = 64,
+ beam_width: int = 1,
+ prune_ratio: float = 0.0,
+ recompute_embeddings: bool = True,
+ pruning_strategy: Literal["global", "local", "proportional"] = "global",
+ expected_zmq_port: int = 5557,
+ metadata_filters: Optional[dict[str, dict[str, Union[str, int, float, bool, list]]]] = None,
+ batch_size: int = 0,
+ use_grep: bool = False,
+ gemma: float = 1.0,
+ provider_options: Optional[dict[str, Any]] = None,
+ **kwargs,
+ ) -> list[SearchResult]:
+ """
+ Search for nearest neighbors with optional metadata filtering.
+
+ Args:
+ query: Text query to search for
+ top_k: Number of nearest neighbors to return
+ complexity: Search complexity/candidate list size, higher = more accurate but slower
+ beam_width: Number of parallel search paths/IO requests per iteration
+ prune_ratio: Ratio of neighbors to prune via approximate distance (0.0-1.0)
+ recompute_embeddings: Whether to fetch fresh embeddings from server vs use stored codes
+ pruning_strategy: Candidate selection strategy - "global" (default), "local", or "proportional"
+ expected_zmq_port: ZMQ port for embedding server communication
+ metadata_filters: Optional filters to apply to search results based on metadata.
+ Format: {"field_name": {"operator": value}}
+ Supported operators:
+ - Comparison: "==", "!=", "<", "<=", ">", ">="
+ - Membership: "in", "not_in"
+ - String: "contains", "starts_with", "ends_with"
+ Example: {"chapter": {"<=": 5}, "tags": {"in": ["fiction", "drama"]}}
+ gemma: Weight of vector search results in hybrid search (0.0-1.0), 1 = pure vector search, 0 = pure keyword search
+ **kwargs: Backend-specific parameters
+
+ Returns:
+ List of SearchResult objects with text, metadata, and similarity scores
+ """
+ # Handle grep search
+ if use_grep:
+ return self._grep_search(query, top_k)
+
+ logger.info("🔍 LeannSearcher.search() called:")
+ logger.info(f" Query: '{query}'")
+ logger.info(f" Top_k: {top_k}")
+ logger.info(f" Metadata filters: {metadata_filters}")
+ logger.info(f" Additional kwargs: {kwargs}")
+
+ # Smart top_k detection and adjustment
+ # Use PassageManager length (sum of shard sizes) to avoid
+ # depending on a massive combined map
+ total_docs = len(self.passage_manager)
+ original_top_k = top_k
+ if top_k > total_docs:
+ top_k = total_docs
+ logger.warning(
+ f" ⚠️ Requested top_k ({original_top_k}) exceeds total documents ({total_docs})"
+ )
+ logger.warning(f" ✅ Auto-adjusted top_k to {top_k} to match available documents")
+
+ zmq_port = None
+
+ start_time = time.time()
+ if recompute_embeddings:
+ zmq_port = self.backend_impl._ensure_server_running(
+ self.meta_path_str,
+ port=expected_zmq_port,
+ **kwargs,
+ )
+ del expected_zmq_port
+ zmq_time = time.time() - start_time
+ logger.info(f" Launching server time: {zmq_time} seconds")
+
+ start_time = time.time()
+
+ # Extract query template from stored embedding_options with fallback chain:
+ # 1. Check provider_options override (highest priority)
+ # 2. Check query_prompt_template (new format)
+ # 3. Check prompt_template (old format for backward compat)
+ # 4. None (no template)
+ query_template = None
+ if provider_options and "prompt_template" in provider_options:
+ query_template = provider_options["prompt_template"]
+ elif "query_prompt_template" in self.embedding_options:
+ query_template = self.embedding_options["query_prompt_template"]
+ elif "prompt_template" in self.embedding_options:
+ query_template = self.embedding_options["prompt_template"]
+
+ query_embedding = self.backend_impl.compute_query_embedding(
+ query,
+ use_server_if_available=recompute_embeddings,
+ zmq_port=zmq_port,
+ query_template=query_template,
+ )
+ logger.info(f" Generated embedding shape: {query_embedding.shape}")
+ embedding_time = time.time() - start_time
+ logger.info(f" Embedding time: {embedding_time} seconds")
+
+ start_time = time.time()
+ backend_search_kwargs: dict[str, Any] = {
+ "complexity": complexity,
+ "beam_width": beam_width,
+ "prune_ratio": prune_ratio,
+ "recompute_embeddings": recompute_embeddings,
+ "pruning_strategy": pruning_strategy,
+ "zmq_port": zmq_port,
+ }
+ # Only HNSW supports batching; forward conditionally
+ if self.backend_name == "hnsw":
+ backend_search_kwargs["batch_size"] = batch_size
+
+ # Merge any extra kwargs last
+ backend_search_kwargs.update(kwargs)
+
+ results = self.backend_impl.search(
+ query_embedding,
+ top_k,
+ **backend_search_kwargs,
+ )
+
+ # Handle hybrid search
+ if gemma < 1.0:
+ logger.info(f" 🌟 Hybrid search enabled with gemma={gemma}")
+ BM25_WEIGHT = 1.0 - gemma
+ bm25_results = self._bm25_search(query, top_k)
+ hybrid_scores: dict[str, float] = {}
+ # Add vector search scores (weighted by gemma)
+ if "labels" in results and "distances" in results:
+ for doc_id, score in zip(results["labels"][0], results["distances"][0]):
+ hybrid_scores[doc_id] = gemma * score
+ # Add BM25 scores (weighted by BM25_WEIGHT)
+ for bm25_result in bm25_results:
+ doc_id = bm25_result.id
+ if doc_id in hybrid_scores:
+ hybrid_scores[doc_id] += BM25_WEIGHT * bm25_result.score
+ else:
+ hybrid_scores[doc_id] = BM25_WEIGHT * bm25_result.score
+
+ sorted_hybrid = sorted(hybrid_scores.items(), key=lambda x: x[1], reverse=True)[:top_k]
+ results["labels"] = [[doc_id for doc_id, _ in sorted_hybrid]]
+ results["distances"] = [[score for _, score in sorted_hybrid]]
+
+ logger.info(
+ f" Combined {len(hybrid_scores)} unique documents from vector and BM25 search"
+ )
+
+ search_time = time.time() - start_time
+ logger.info(f" Search time in search() LEANN searcher: {search_time} seconds")
+ logger.info(f" Backend returned: labels={len(results.get('labels', [[]])[0])} results")
+
+ enriched_results = []
+ if "labels" in results and "distances" in results:
+ logger.info(f" Processing {len(results['labels'][0])} passage IDs:")
+ # Python 3.9 does not support zip(strict=...); lengths are expected to match
+ for i, (string_id, dist) in enumerate(
+ zip(results["labels"][0], results["distances"][0])
+ ):
+ try:
+ passage_data = self.passage_manager.get_passage(string_id)
+ enriched_results.append(
+ SearchResult(
+ id=string_id,
+ score=dist,
+ text=passage_data["text"],
+ metadata=passage_data.get("metadata", {}),
+ )
+ )
+
+ # Color codes for better logging
+ GREEN = "\033[92m"
+ BLUE = "\033[94m"
+ YELLOW = "\033[93m"
+ RESET = "\033[0m"
+
+ # Truncate text for display (first 100 chars)
+ display_text = passage_data["text"]
+ logger.info(
+ f" {GREEN}✓{RESET} {BLUE}[{i + 1:2d}]{RESET} {YELLOW}ID:{RESET} '{string_id}' {YELLOW}Score:{RESET} {dist:.4f} {YELLOW}Text:{RESET} {display_text}"
+ )
+ except KeyError:
+ RED = "\033[91m"
+ RESET = "\033[0m"
+ logger.error(
+ f" {RED}✗{RESET} [{i + 1:2d}] ID: '{string_id}' -> {RED}ERROR: Passage not found!{RESET}"
+ )
+
+ # Apply metadata filters if specified
+ if metadata_filters:
+ logger.info(f" 🔍 Applying metadata filters: {metadata_filters}")
+ enriched_results = self.passage_manager.filter_search_results(
+ enriched_results, metadata_filters
+ )
+
+ # Define color codes outside the loop for final message
+ GREEN = "\033[92m"
+ RESET = "\033[0m"
+ logger.info(f" {GREEN}✓ Final enriched results: {len(enriched_results)} passages{RESET}")
+ return enriched_results
+
+ def _init_bm25(self) -> None:
+ """Initialize BM25 scorer"""
+ self.bm25_scorer = BM25Scorer()
+ # Load all the files directly
+ passages = []
+ for passage_file in self.passage_manager.passage_files.values():
+ with open(passage_file, encoding="utf-8") as f:
+ for line in f:
+ if line.strip():
+ data = json.loads(line)
+ passages.append(data)
+ self.bm25_scorer.fit(passages)
+
+ def _bm25_search(self, query: str, top_k: int = 5) -> list[SearchResult]:
+ """Perform BM25 search on raw passages"""
+ if not self.bm25_scorer:
+ self._init_bm25()
+ logger.info(" BM25 scorer initialized")
+ return self.bm25_scorer.search(query, top_k)
+
+ def _find_jsonl_file(self) -> Optional[str]:
+ """Find the .jsonl file containing raw passages for grep search"""
+ index_path = Path(self.meta_path_str).parent
+ potential_files = [
+ index_path / "documents.leann.passages.jsonl",
+ index_path.parent / "documents.leann.passages.jsonl",
+ ]
+
+ for file_path in potential_files:
+ if file_path.exists():
+ return str(file_path)
+ return None
+
+ def _grep_search(self, query: str, top_k: int = 5) -> list[SearchResult]:
+ """Perform grep-based search on raw passages"""
+ jsonl_file = self._find_jsonl_file()
+ if not jsonl_file:
+ raise FileNotFoundError("No .jsonl passages file found for grep search")
+
+ try:
+ cmd = ["grep", "-i", "-n", query, jsonl_file]
+ result = subprocess.run(cmd, capture_output=True, text=True, check=False)
+
+ if result.returncode == 1:
+ return []
+ elif result.returncode != 0:
+ raise RuntimeError(f"Grep failed: {result.stderr}")
+
+ matches = []
+ for line in result.stdout.strip().split("\n"):
+ if not line:
+ continue
+ parts = line.split(":", 1)
+ if len(parts) != 2:
+ continue
+
+ try:
+ data = json.loads(parts[1])
+ text = data.get("text", "")
+ score = text.lower().count(query.lower())
+
+ matches.append(
+ SearchResult(
+ id=data.get("id", parts[0]),
+ text=text,
+ metadata=data.get("metadata", {}),
+ score=float(score),
+ )
+ )
+ except json.JSONDecodeError:
+ continue
+
+ matches.sort(key=lambda x: x.score, reverse=True)
+ return matches[:top_k]
+
+ except FileNotFoundError:
+ raise RuntimeError(
+ "grep command not found. Please install grep or use semantic search."
+ )
+
+ def _python_regex_search(self, query: str, top_k: int = 5) -> list[SearchResult]:
+ """Fallback regex search"""
+ jsonl_file = self._find_jsonl_file()
+ if not jsonl_file:
+ raise FileNotFoundError("No .jsonl file found")
+
+ pattern = re.compile(re.escape(query), re.IGNORECASE)
+ matches = []
+
+ with open(jsonl_file, encoding="utf-8") as f:
+ for line_num, line in enumerate(f, 1):
+ if pattern.search(line):
+ try:
+ data = json.loads(line.strip())
+ matches.append(
+ SearchResult(
+ id=data.get("id", str(line_num)),
+ text=data.get("text", ""),
+ metadata=data.get("metadata", {}),
+ score=float(len(pattern.findall(data.get("text", "")))),
+ )
+ )
+ except json.JSONDecodeError:
+ continue
+
+ matches.sort(key=lambda x: x.score, reverse=True)
+ return matches[:top_k]
+
+ def cleanup(self):
+ """Explicitly cleanup embedding server resources.
+ This method should be called after you're done using the searcher,
+ especially in test environments or batch processing scenarios.
+ """
+ backend = getattr(self.backend_impl, "embedding_server_manager", None)
+ if backend is not None:
+ backend.stop_server()
+
+ # Enable automatic cleanup patterns
+ def __enter__(self):
+ return self
+
+ def __exit__(self, exc_type, exc, tb):
+ try:
+ self.cleanup()
+ except Exception:
+ pass
+
+ def __del__(self):
+ try:
+ self.cleanup()
+ except Exception:
+ # Avoid noisy errors during interpreter shutdown
+ pass
+
+
+class LeannChat:
+ def __init__(
+ self,
+ index_path: str,
+ llm_config: Optional[dict[str, Any]] = None,
+ enable_warmup: bool = False,
+ searcher: Optional[LeannSearcher] = None,
+ **kwargs,
+ ):
+ if searcher is None:
+ self.searcher = LeannSearcher(index_path, enable_warmup=enable_warmup, **kwargs)
+ self._owns_searcher = True
+ else:
+ self.searcher = searcher
+ self._owns_searcher = False
+ self.llm = get_llm(llm_config)
+
+ def ask(
+ self,
+ question: str,
+ top_k: int = 5,
+ complexity: int = 64,
+ beam_width: int = 1,
+ prune_ratio: float = 0.0,
+ recompute_embeddings: bool = True,
+ pruning_strategy: Literal["global", "local", "proportional"] = "global",
+ llm_kwargs: Optional[dict[str, Any]] = None,
+ expected_zmq_port: int = 5557,
+ metadata_filters: Optional[dict[str, dict[str, Union[str, int, float, bool, list]]]] = None,
+ batch_size: int = 0,
+ use_grep: bool = False,
+ gemma: float = 1.0,
+ **search_kwargs,
+ ):
+ if llm_kwargs is None:
+ llm_kwargs = {}
+ search_time = time.time()
+ results = self.searcher.search(
+ question,
+ top_k=top_k,
+ complexity=complexity,
+ beam_width=beam_width,
+ prune_ratio=prune_ratio,
+ recompute_embeddings=recompute_embeddings,
+ pruning_strategy=pruning_strategy,
+ expected_zmq_port=expected_zmq_port,
+ metadata_filters=metadata_filters,
+ use_grep=use_grep,
+ gemma=gemma,
+ batch_size=batch_size,
+ **search_kwargs,
+ )
+ search_time = time.time() - search_time
+ logger.info(f" Search time: {search_time} seconds")
+ context = "\n\n".join([r.text for r in results])
+ prompt = (
+ "Here is some retrieved context that might help answer your question:\n\n"
+ f"{context}\n\n"
+ f"Question: {question}\n\n"
+ "Please provide the best answer you can based on this context and your knowledge."
+ )
+
+ logger.info("The context provided to the LLM is:")
+ logger.info(f"{'Relevance':<10} | {'Chunk id':<10} | {'Content':<60} | {'Source':<80}")
+ logger.info("-" * 150)
+ for r in results:
+ chunk_relevance = f"{r.score:.3f}"
+ chunk_id = r.id
+ chunk_content = r.text[:60]
+ chunk_source = r.metadata.get("source", "")[:80]
+ logger.info(
+ f"{chunk_relevance:<10} | {chunk_id:<10} | {chunk_content:<60} | {chunk_source:<80}"
+ )
+ ask_time = time.time()
+ ans = self.llm.ask(prompt, **llm_kwargs)
+ ask_time = time.time() - ask_time
+ logger.info(f" Ask time: {ask_time} seconds")
+ return ans
+
+ def start_interactive(self):
+ """Start interactive chat session."""
+ session = create_api_session()
+
+ def handle_query(user_input: str):
+ response = self.ask(user_input)
+ print(f"Leann: {response}")
+
+ session.run_interactive_loop(handle_query)
+
+ def cleanup(self):
+ """Explicitly cleanup embedding server resources.
+
+ This method should be called after you're done using the chat interface,
+ especially in test environments or batch processing scenarios.
+ """
+ # Only stop the embedding server if this LeannChat instance created the searcher.
+ # When a shared searcher is passed in, avoid shutting down the server to enable reuse.
+ if getattr(self, "_owns_searcher", False) and hasattr(self.searcher, "cleanup"):
+ self.searcher.cleanup()
+
+ # Enable automatic cleanup patterns
+ def __enter__(self):
+ return self
+
+ def __exit__(self, exc_type, exc, tb):
+ try:
+ self.cleanup()
+ except Exception:
+ pass
+
+ def __del__(self):
+ try:
+ self.cleanup()
+ except Exception:
+ pass