[MLsys2026 Best Paper]: https://arxiv.org/abs/2506.08276. RAG on Everything with LEANN. Enjoy 97% storage savings while running a fast, accurate, and 100% private RAG application on your personal device.
Python aifaissgpt-osslangchainllama-indexllmlocalstorageoffline-firstollamaprivacypythonragretrieval-augmented-generationvector-databasevector-searchvectors
feat: allow build from existed embeddings
Andy Lee85b7ba01681e
diff --git a/examples/run_evaluation.py b/examples/run_evaluation.pyindex f80d7a6..6b07f09 100644--- a/examples/run_evaluation.py+++ b/examples/run_evaluation.py@@ -13,10 +13,10 @@ import sysimport numpy as npfrom typing import List-from leann.api import LeannSearcher+from leann.api import LeannSearcher, LeannBuilder-def download_data_if_needed(data_root: Path):+def download_data_if_needed(data_root: Path, download_embeddings: bool = False):"""Checks if the data directory exists, and if not, downloads it from HF Hub."""if not data_root.exists():print(f"Data directory '{data_root}' not found.")@@ -26,13 +26,32 @@ def download_data_if_needed(data_root: Path):try:from huggingface_hub import snapshot_download- snapshot_download(- repo_id="LEANN-RAG/leann-rag-evaluation-data",- repo_type="dataset",- local_dir=data_root,- local_dir_use_symlinks=False, # Recommended for Windows compatibility and simpler structure- )- print("Data download complete!")+ if download_embeddings:+ # Download everything including embeddings (large files)+ snapshot_download(+ repo_id="LEANN-RAG/leann-rag-evaluation-data",+ repo_type="dataset",+ local_dir=data_root,+ local_dir_use_symlinks=False,+ )+ print("Data download complete (including embeddings)!")+ else:+ # Download only specific folders, excluding embeddings+ allow_patterns = [+ "ground_truth/**",+ "indices/**",+ "queries/**",+ "*.md",+ "*.txt",+ ]+ snapshot_download(+ repo_id="LEANN-RAG/leann-rag-evaluation-data",+ repo_type="dataset",+ local_dir=data_root,+ local_dir_use_symlinks=False,+ allow_patterns=allow_patterns,+ )+ print("Data download complete (excluding embeddings)!")except ImportError:print("Error: huggingface_hub is not installed. Please install it to download the data:"@@ -44,6 +63,43 @@ def download_data_if_needed(data_root: Path):sys.exit(1)+def download_embeddings_if_needed(data_root: Path, dataset_type: str = None):+ """Download embeddings files specifically."""+ embeddings_dir = data_root / "embeddings"++ if dataset_type:+ # Check if specific dataset embeddings exist+ target_file = embeddings_dir / dataset_type / "passages_00.pkl"+ if target_file.exists():+ print(f"Embeddings for {dataset_type} already exist")+ return str(target_file)++ print("Downloading embeddings from HuggingFace Hub...")+ try:+ from huggingface_hub import snapshot_download++ # Download only embeddings folder+ snapshot_download(+ repo_id="LEANN-RAG/leann-rag-evaluation-data",+ repo_type="dataset",+ local_dir=data_root,+ local_dir_use_symlinks=False,+ allow_patterns=["embeddings/**/*.pkl"],+ )+ print("Embeddings download complete!")++ if dataset_type:+ target_file = embeddings_dir / dataset_type / "passages_00.pkl"+ if target_file.exists():+ return str(target_file)++ return str(embeddings_dir)++ except Exception as e:+ print(f"Error downloading embeddings: {e}")+ sys.exit(1)++# --- Helper Function to get Golden Passages ---def get_golden_texts(searcher: LeannSearcher, golden_ids: List[int]) -> set:"""@@ -72,12 +128,76 @@ def load_queries(file_path: Path) -> List[str]:return queries+def build_index_from_embeddings(+ embeddings_file: str, output_path: str, backend: str = "hnsw"+):+ """+ Build a LEANN index from pre-computed embeddings.++ Args:+ embeddings_file: Path to pickle file with (ids, embeddings) tuple+ output_path: Path where to save the index+ backend: Backend to use ("hnsw" or "diskann")+ """+ print(f"Building {backend} index from embeddings: {embeddings_file}")++ # Create builder with appropriate parameters+ if backend == "hnsw":+ builder_kwargs = {+ "M": 32, # Graph degree+ "efConstruction": 256, # Construction complexity+ "is_compact": True, # Use compact storage+ "is_recompute": True, # Enable pruning for better recall+ }+ elif backend == "diskann":+ builder_kwargs = {+ "complexity": 64,+ "graph_degree": 32,+ "search_memory_maximum": 8.0, # GB+ "build_memory_maximum": 16.0, # GB+ }+ else:+ builder_kwargs = {}++ builder = LeannBuilder(+ backend_name=backend,+ embedding_model="facebook/contriever-msmarco", # Model used to create embeddings+ dimensions=768, # Will be auto-detected from embeddings+ **builder_kwargs,+ )++ # Build index from precomputed embeddings+ builder.build_index_from_embeddings(output_path, embeddings_file)+ print(f"Index saved to: {output_path}")+ return output_path++def main():parser = argparse.ArgumentParser(description="Run recall evaluation on a LEANN index.")parser.add_argument(- "index_path", type=str, help="Path to the LEANN index to evaluate."+ "index_path",+ type=str,+ nargs="?",+ help="Path to the LEANN index to evaluate or build (optional).",+ )+ parser.add_argument(+ "--mode",+ choices=["evaluate", "build"],+ default="evaluate",+ help="Mode: 'evaluate' existing index or 'build' from embeddings",+ )+ parser.add_argument(+ "--embeddings-file",+ type=str,+ help="Path to embeddings pickle file (optional for build mode)",+ )+ parser.add_argument(+ "--backend",+ choices=["hnsw", "diskann"],+ default="hnsw",+ help="Backend to use for building index (default: hnsw)",)parser.add_argument("--num-queries", type=int, default=10, help="Number of queries to evaluate."@@ -96,8 +216,90 @@ def main():project_root = Path(__file__).resolve().parent.parentdata_root = project_root / "data"- # Automatically download data if it doesn't exist- download_data_if_needed(data_root)+ # Download data based on mode+ if args.mode == "build":+ # For building mode, we need embeddings+ download_data_if_needed(+ data_root, download_embeddings=False+ ) # Basic data first++ # Auto-detect dataset type and download embeddings+ if args.embeddings_file:+ embeddings_file = args.embeddings_file+ # Try to detect dataset type from embeddings file path+ if "rpj_wiki" in str(embeddings_file):+ dataset_type = "rpj_wiki"+ elif "dpr" in str(embeddings_file):+ dataset_type = "dpr"+ else:+ dataset_type = "dpr" # Default+ else:+ # Auto-detect from index path if provided, otherwise default to DPR+ if args.index_path:+ index_path_str = str(args.index_path)+ if "rpj_wiki" in index_path_str:+ dataset_type = "rpj_wiki"+ elif "dpr" in index_path_str:+ dataset_type = "dpr"+ else:+ dataset_type = "dpr" # Default to DPR+ else:+ dataset_type = "dpr" # Default to DPR++ embeddings_file = download_embeddings_if_needed(data_root, dataset_type)++ # Auto-generate index path if not provided+ if not args.index_path:+ indices_dir = data_root / "indices" / dataset_type+ indices_dir.mkdir(parents=True, exist_ok=True)+ args.index_path = str(indices_dir / f"{dataset_type}_from_embeddings")+ print(f"Auto-generated index path: {args.index_path}")++ print(f"Building index from embeddings: {embeddings_file}")+ built_index_path = build_index_from_embeddings(+ embeddings_file, args.index_path, args.backend+ )+ print(f"Index built successfully: {built_index_path}")++ # Ask if user wants to run evaluation+ eval_response = (+ input("Run evaluation on the built index? (y/n): ").strip().lower()+ )+ if eval_response != "y":+ print("Index building complete. Exiting.")+ return+ else:+ # For evaluation mode, don't need embeddings+ download_data_if_needed(data_root, download_embeddings=False)++ # Auto-detect index path if not provided+ if not args.index_path:+ # Default to using downloaded indices+ indices_dir = data_root / "indices"++ # Try common datasets in order of preference+ for dataset in ["dpr", "rpj_wiki"]:+ dataset_dir = indices_dir / dataset+ if dataset_dir.exists():+ # Look for index files+ index_files = list(dataset_dir.glob("*.index")) + list(+ dataset_dir.glob("*_disk.index")+ )+ if index_files:+ args.index_path = str(+ index_files[0].with_suffix("")+ ) # Remove .index extension+ print(f"Using index: {args.index_path}")+ break++ if not args.index_path:+ print(+ "No indices found. The data download should have included pre-built indices."+ )+ print(+ "Please check the data/indices/ directory or provide --index-path manually."+ )+ sys.exit(1)# Detect dataset type from index path to select the correct ground truthindex_path_str = str(args.index_path)diff --git a/packages/leann-core/src/leann/api.py b/packages/leann-core/src/leann/api.pyindex c6e43ab..529d817 100644--- a/packages/leann-core/src/leann/api.py+++ b/packages/leann-core/src/leann/api.py@@ -18,14 +18,14 @@ from .chat import get_llmdef compute_embeddings(- chunks: List[str],- model_name: str,+ chunks: List[str],+ model_name: str,mode: str = "sentence-transformers",- use_server: bool = True+ use_server: bool = True,) -> np.ndarray:"""Computes embeddings using different backends.-+Args:chunks: List of text chunks to embedmodel_name: Name of the embedding model@@ -34,40 +34,48 @@ def compute_embeddings(- "mlx": Use MLX backend for Apple Silicon- "openai": Use OpenAI embedding APIuse_server: Whether to use embedding server (True for search, False for build)-+Returns:numpy array of embeddings"""# Auto-detect mode based on model name if not explicitly setif mode == "sentence-transformers" and model_name.startswith("text-embedding-"):mode = "openai"-+if mode == "mlx":return compute_embeddings_mlx(chunks, model_name)elif mode == "openai":return compute_embeddings_openai(chunks, model_name)elif mode == "sentence-transformers":- return compute_embeddings_sentence_transformers(chunks, model_name, use_server=use_server)+ return compute_embeddings_sentence_transformers(+ chunks, model_name, use_server=use_server+ )else:- raise ValueError(f"Unsupported embedding mode: {mode}. Supported modes: sentence-transformers, mlx, openai")+ raise ValueError(+ f"Unsupported embedding mode: {mode}. Supported modes: sentence-transformers, mlx, openai"+ )-def compute_embeddings_sentence_transformers(chunks: List[str], model_name: str, use_server: bool = True) -> np.ndarray:+def compute_embeddings_sentence_transformers(+ chunks: List[str], model_name: str, use_server: bool = True+) -> np.ndarray:"""Computes embeddings using sentence-transformers.-+Args:chunks: List of text chunks to embedmodel_name: Name of the sentence transformer modeluse_server: If True, use embedding server (good for search). If False, use direct computation (good for build)."""if not use_server:- print(f"INFO: Computing embeddings for {len(chunks)} chunks using SentenceTransformer model '{model_name}' (direct)...")+ print(+ f"INFO: Computing embeddings for {len(chunks)} chunks using SentenceTransformer model '{model_name}' (direct)..."+ )return _compute_embeddings_sentence_transformers_direct(chunks, model_name)-+print(f"INFO: Computing embeddings for {len(chunks)} chunks using SentenceTransformer model '{model_name}' (via embedding server)...")-+# Use embedding server for sentence-transformers too# This avoids loading the model twice (once in API, once in server)try:@@ -76,49 +84,55 @@ def compute_embeddings_sentence_transformers(chunks: List[str], model_name: str,import msgpackimport numpy as npfrom .embedding_server_manager import EmbeddingServerManager-+# Ensure embedding server is runningport = 5557- server_manager = EmbeddingServerManager(backend_module_name="leann_backend_hnsw.hnsw_embedding_server")-+ server_manager = EmbeddingServerManager(+ backend_module_name="leann_backend_hnsw.hnsw_embedding_server"+ )+server_started = server_manager.start_server(port=port,model_name=model_name,embedding_mode="sentence-transformers",enable_warmup=False,)-+if not server_started:raise RuntimeError(f"Failed to start embedding server on port {port}")-+# Connect to embedding servercontext = zmq.Context()socket = context.socket(zmq.REQ)socket.connect(f"tcp://localhost:{port}")-+# Send chunks to server for embedding computationrequest = chunkssocket.send(msgpack.packb(request))-+# Receive embeddings from serverresponse = socket.recv()embeddings_list = msgpack.unpackb(response)-+# Convert back to numpy arrayembeddings = np.array(embeddings_list, dtype=np.float32)-+socket.close()context.term()-+return embeddings-+except Exception as e:# Fallback to direct sentence-transformers if server connection fails- print(f"Warning: Failed to connect to embedding server, falling back to direct computation: {e}")+ print(+ f"Warning: Failed to connect to embedding server, falling back to direct computation: {e}"+ )return _compute_embeddings_sentence_transformers_direct(chunks, model_name)-def _compute_embeddings_sentence_transformers_direct(chunks: List[str], model_name: str) -> np.ndarray:+def _compute_embeddings_sentence_transformers_direct(+ chunks: List[str], model_name: str+) -> np.ndarray:"""Direct sentence-transformers computation (fallback)."""try:from sentence_transformers import SentenceTransformer@@ -159,37 +173,40 @@ def compute_embeddings_openai(chunks: List[str], model_name: str) -> np.ndarray:raise RuntimeError("openai not available. Install with: uv pip install openai") from e-+# Get API key from environmentapi_key = os.getenv("OPENAI_API_KEY")if not api_key:raise RuntimeError("OPENAI_API_KEY environment variable not set")-+client = openai.OpenAI(api_key=api_key)-- print(f"INFO: Computing embeddings for {len(chunks)} chunks using OpenAI model '{model_name}'...")-++ print(+ f"INFO: Computing embeddings for {len(chunks)} chunks using OpenAI model '{model_name}'..."+ )+# OpenAI has a limit on batch size and input lengthmax_batch_size = 100 # Conservative batch sizeall_embeddings = []-+for i in range(0, len(chunks), max_batch_size):- batch_chunks = chunks[i:i + max_batch_size]- print(f"INFO: Processing batch {i//max_batch_size + 1}/{(len(chunks) + max_batch_size - 1)//max_batch_size}")-+ batch_chunks = chunks[i : i + max_batch_size]+ print(+ f"INFO: Processing batch {i // max_batch_size + 1}/{(len(chunks) + max_batch_size - 1) // max_batch_size}"+ )+try:- response = client.embeddings.create(- model=model_name,- input=batch_chunks- )+ response = client.embeddings.create(model=model_name, input=batch_chunks)batch_embeddings = [embedding.embedding for embedding in response.data]all_embeddings.extend(batch_embeddings)except Exception as e:print(f"ERROR: Failed to get embeddings for batch starting at {i}: {e}")raise-+embeddings = np.array(all_embeddings, dtype=np.float32)- print(f"INFO: Generated {len(embeddings)} embeddings with dimension {embeddings.shape[1]}")+ print(+ f"INFO: Generated {len(embeddings)} embeddings with dimension {embeddings.shape[1]}"+ )return embeddings@@ -308,7 +325,12 @@ class LeannBuilder:raise ValueError("No chunks added.")if self.dimensions is None:self.dimensions = len(- compute_embeddings(["dummy"], self.embedding_model, self.embedding_mode, use_server=False)[0]+ compute_embeddings(+ ["dummy"],+ self.embedding_model,+ self.embedding_mode,+ use_server=False,+ )[0])path = Path(index_path)index_dir = path.parent@@ -371,6 +393,129 @@ class LeannBuilder:with open(leann_meta_path, "w", encoding="utf-8") as f:json.dump(meta_data, f, indent=2)+ def build_index_from_embeddings(self, index_path: str, embeddings_file: str):+ """+ Build an index from pre-computed embeddings stored in a pickle file.++ Args:+ index_path: Path where the index will be saved+ embeddings_file: Path to pickle file containing (ids, embeddings) tuple+ """+ # Load pre-computed embeddings+ with open(embeddings_file, "rb") as f:+ data = pickle.load(f)++ if not isinstance(data, tuple) or len(data) != 2:+ raise ValueError(+ f"Invalid embeddings file format. Expected tuple with 2 elements, got {type(data)}"+ )++ ids, embeddings = data++ if not isinstance(embeddings, np.ndarray):+ raise ValueError(+ f"Expected embeddings to be numpy array, got {type(embeddings)}"+ )++ if len(ids) != embeddings.shape[0]:+ raise ValueError(+ f"Mismatch between number of IDs ({len(ids)}) and embeddings ({embeddings.shape[0]})"+ )++ # Validate/set dimensions+ embedding_dim = embeddings.shape[1]+ if self.dimensions is None:+ self.dimensions = embedding_dim+ elif self.dimensions != embedding_dim:+ raise ValueError(+ f"Dimension mismatch: expected {self.dimensions}, got {embedding_dim}"+ )++ print(+ f"Building index from precomputed embeddings: {len(ids)} items, {embedding_dim} dimensions"+ )++ # Ensure we have text data for each embedding+ if len(self.chunks) != len(ids):+ # If no text chunks provided, create placeholder text entries+ if not self.chunks:+ print("No text chunks provided, creating placeholder entries...")+ for id_val in ids:+ self.add_text(+ f"Document {id_val}",+ metadata={"id": str(id_val), "from_embeddings": True},+ )+ else:+ raise ValueError(+ f"Number of text chunks ({len(self.chunks)}) doesn't match number of embeddings ({len(ids)})"+ )++ # Build file structure+ path = Path(index_path)+ index_dir = path.parent+ index_name = path.name+ index_dir.mkdir(parents=True, exist_ok=True)+ passages_file = index_dir / f"{index_name}.passages.jsonl"+ offset_file = index_dir / f"{index_name}.passages.idx"++ # Write passages and create offset map+ offset_map = {}+ with open(passages_file, "w", encoding="utf-8") as f:+ for chunk in self.chunks:+ offset = f.tell()+ json.dump(+ {+ "id": chunk["id"],+ "text": chunk["text"],+ "metadata": chunk["metadata"],+ },+ f,+ ensure_ascii=False,+ )+ f.write("\n")+ offset_map[chunk["id"]] = offset++ with open(offset_file, "wb") as f:+ pickle.dump(offset_map, f)++ # Build the vector index using precomputed embeddings+ string_ids = [str(id_val) for id_val in ids]+ current_backend_kwargs = {**self.backend_kwargs, "dimensions": self.dimensions}+ builder_instance = self.backend_factory.builder(**current_backend_kwargs)+ builder_instance.build(embeddings, string_ids, index_path)++ # Create metadata file+ leann_meta_path = index_dir / f"{index_name}.meta.json"+ meta_data = {+ "version": "1.0",+ "backend_name": self.backend_name,+ "embedding_model": self.embedding_model,+ "dimensions": self.dimensions,+ "backend_kwargs": self.backend_kwargs,+ "embedding_mode": self.embedding_mode,+ "passage_sources": [+ {+ "type": "jsonl",+ "path": str(passages_file),+ "index_path": str(offset_file),+ }+ ],+ "built_from_precomputed_embeddings": True,+ "embeddings_source": str(embeddings_file),+ }++ # Add storage status flags for HNSW backend+ if self.backend_name == "hnsw":+ is_compact = self.backend_kwargs.get("is_compact", True)+ is_recompute = self.backend_kwargs.get("is_recompute", True)+ meta_data["is_compact"] = is_compact+ meta_data["is_pruned"] = is_compact and is_recompute++ with open(leann_meta_path, "w", encoding="utf-8") as f:+ json.dump(meta_data, f, indent=2)++ print(f"Index built successfully from precomputed embeddings: {index_path}")+class LeannSearcher:def __init__(self, index_path: str, enable_warmup: bool = False, **backend_kwargs):@@ -382,7 +527,9 @@ class LeannSearcher:backend_name = self.meta_data["backend_name"]self.embedding_model = self.meta_data["embedding_model"]# Support both old and new format- self.embedding_mode = self.meta_data.get("embedding_mode", "sentence-transformers")+ self.embedding_mode = self.meta_data.get(+ "embedding_mode", "sentence-transformers"+ )# Backward compatibility with use_mlxif self.meta_data.get("use_mlx", False):self.embedding_mode = "mlx"@@ -414,6 +561,7 @@ class LeannSearcher:# Use backend's compute_query_embedding method# This will automatically use embedding server if available and neededimport time+start_time = time.time()query_embedding = self.backend_impl.compute_query_embedding(query, zmq_port)@@ -513,7 +661,7 @@ class LeannChat:"Please provide the best answer you can based on this context and your knowledge.")- ans=self.llm.ask(prompt, **llm_kwargs)+ ans = self.llm.ask(prompt, **llm_kwargs)return ansdef start_interactive(self):