[MLsys2026 Best Paper]: https://arxiv.org/abs/2506.08276. RAG on Everything with LEANN. Enjoy 97% storage savings while running a fast, accurate, and 100% private RAG application on your personal device.
Python aifaissgpt-osslangchainllama-indexllmlocalstorageoffline-firstollamaprivacypythonragretrieval-augmented-generationvector-databasevector-searchvectors
Building, CLI tool & Embedding Server Fixed (#5)
Andy Lee1b6272ce0ef6
diff --git a/README.md b/README.mdindex dab40b5..1e0140c 100755--- a/README.md+++ b/README.md@@ -48,7 +48,7 @@ git submodule update --init --recursive**macOS:**```bash-brew install llvm libomp boost protobuf+brew install llvm libomp boost protobuf zeromqexport CC=$(brew --prefix llvm)/bin/clangexport CXX=$(brew --prefix llvm)/bin/clang++@@ -61,7 +61,7 @@ uv sync --extra diskann**Linux (Ubuntu/Debian):**```bash-sudo apt-get install libomp-dev libboost-all-dev protobuf-compiler libabsl-dev libmkl-full-dev libaio-dev+sudo apt-get install libomp-dev libboost-all-dev protobuf-compiler libabsl-dev libmkl-full-dev libaio-dev libzmq3-dev# Install with HNSW backend (default, recommended for most users)uv syncdiff --git a/packages/leann-backend-diskann/CMakeLists.txt b/packages/leann-backend-diskann/CMakeLists.txtindex ee9d932..2638282 100644--- a/packages/leann-backend-diskann/CMakeLists.txt+++ b/packages/leann-backend-diskann/CMakeLists.txt@@ -1,8 +1,8 @@-# packages/leann-backend-diskann/CMakeLists.txt (最终简化版)+# packages/leann-backend-diskann/CMakeLists.txt (simplified version)cmake_minimum_required(VERSION 3.20)project(leann_backend_diskann_wrapper)-# 告诉 CMake 直接进入 DiskANN 子模块并执行它自己的 CMakeLists.txt-# DiskANN 会自己处理所有事情,包括编译 Python 绑定+# Tell CMake to directly enter the DiskANN submodule and execute its own CMakeLists.txt+# DiskANN will handle everything itself, including compiling Python bindingsadd_subdirectory(src/third_party/DiskANN)diff --git a/packages/leann-backend-diskann/leann_backend_diskann/diskann_backend.py b/packages/leann-backend-diskann/leann_backend_diskann/diskann_backend.pyindex adf9182..bbd042d 100644--- a/packages/leann-backend-diskann/leann_backend_diskann/diskann_backend.py+++ b/packages/leann-backend-diskann/leann_backend_diskann/diskann_backend.py@@ -70,10 +70,6 @@ class DiskannBuilder(LeannBackendBuilderInterface):data_filename = f"{index_prefix}_data.bin"_write_vectors_to_bin(data, index_dir / data_filename)- label_map = {i: str_id for i, str_id in enumerate(ids)}- label_map_file = index_dir / "leann.labels.map"- with open(label_map_file, "wb") as f:- pickle.dump(label_map, f)build_kwargs = {**self.build_params, **kwargs}metric_enum = _get_diskann_metrics().get(@@ -211,10 +207,7 @@ class DiskannSearcher(BaseSearcher):)string_labels = [- [- self.label_map.get(int_label, f"unknown_{int_label}")- for int_label in batch_labels- ]+ [str(int_label) for int_label in batch_labels]for batch_labels in labels]diff --git a/packages/leann-backend-diskann/leann_backend_diskann/embedding_server.py b/packages/leann-backend-diskann/leann_backend_diskann/embedding_server.pyindex 089ec1f..04f7f56 100644--- a/packages/leann-backend-diskann/leann_backend_diskann/embedding_server.py+++ b/packages/leann-backend-diskann/leann_backend_diskann/embedding_server.py@@ -76,24 +76,11 @@ def load_passages_from_metadata(meta_file: str) -> SimplePassageLoader:finally:sys.path.pop(0)- # Load label map- passages_dir = Path(meta_file).parent- label_map_file = passages_dir / "leann.labels.map"-- if label_map_file.exists():- import pickle- with open(label_map_file, 'rb') as f:- label_map = pickle.load(f)- print(f"Loaded label map with {len(label_map)} entries")- else:- raise FileNotFoundError(f"Label map file not found: {label_map_file}")-- print(f"Initialized lazy passage loading for {len(label_map)} passages")+ print(f"Initialized lazy passage loading for {len(passage_manager.global_offset_map)} passages")class LazyPassageLoader(SimplePassageLoader):- def __init__(self, passage_manager, label_map):+ def __init__(self, passage_manager):self.passage_manager = passage_manager- self.label_map = label_map# Initialize parent with empty datasuper().__init__({})@@ -101,25 +88,22 @@ def load_passages_from_metadata(meta_file: str) -> SimplePassageLoader:"""Get passage by ID with lazy loading"""try:int_id = int(passage_id)- if int_id in self.label_map:- string_id = self.label_map[int_id]- passage_data = self.passage_manager.get_passage(string_id)- if passage_data and passage_data.get("text"):- return {"text": passage_data["text"]}- else:- raise RuntimeError(f"FATAL: Empty text for ID {int_id} -> {string_id}")+ string_id = str(int_id)+ passage_data = self.passage_manager.get_passage(string_id)+ if passage_data and passage_data.get("text"):+ return {"text": passage_data["text"]}else:- raise RuntimeError(f"FATAL: ID {int_id} not found in label_map")+ raise RuntimeError(f"FATAL: Empty text for ID {int_id} -> {string_id}")except Exception as e:raise RuntimeError(f"FATAL: Exception getting passage {passage_id}: {e}")def __len__(self) -> int:- return len(self.label_map)+ return len(self.passage_manager.global_offset_map)def keys(self):- return self.label_map.keys()+ return self.passage_manager.global_offset_map.keys()- loader = LazyPassageLoader(passage_manager, label_map)+ loader = LazyPassageLoader(passage_manager)loader._meta_path = meta_filereturn loader@@ -135,35 +119,15 @@ def load_passages_from_file(passages_file: str) -> SimplePassageLoader:if not passages_file.endswith('.jsonl'):raise ValueError(f"Expected .jsonl file format, got: {passages_file}")- # Load label map (int -> string_id)- passages_dir = Path(passages_file).parent- label_map_file = passages_dir / "leann.labels.map"-- label_map = {}- if label_map_file.exists():- with open(label_map_file, 'rb') as f:- label_map = pickle.load(f)- print(f"Loaded label map with {len(label_map)} entries")- else:- raise FileNotFoundError(f"Label map file not found: {label_map_file}")-- # Load passages by string ID- string_id_passages = {}+ # Load passages directly by their sequential IDs+ passages_data = {}with open(passages_file, 'r', encoding='utf-8') as f:for line in f:if line.strip():passage = json.loads(line)- string_id_passages[passage['id']] = passage['text']-- # Create int ID -> text mapping using label map- passages_data = {}- for int_id, string_id in label_map.items():- if string_id in string_id_passages:- passages_data[str(int_id)] = string_id_passages[string_id]- else:- print(f"WARNING: String ID {string_id} from label map not found in passages")+ passages_data[passage['id']] = passage['text']- print(f"Loaded {len(passages_data)} passages from JSONL file {passages_file} using label map")+ print(f"Loaded {len(passages_data)} passages from JSONL file {passages_file}")return SimplePassageLoader(passages_data)def create_embedding_server_thread(diff --git a/packages/leann-backend-diskann/pyproject.toml b/packages/leann-backend-diskann/pyproject.tomlindex 8f02feb..24637cc 100644--- a/packages/leann-backend-diskann/pyproject.toml+++ b/packages/leann-backend-diskann/pyproject.toml@@ -8,11 +8,11 @@ version = "0.1.0"dependencies = ["leann-core==0.1.0", "numpy"][tool.scikit-build]-# 关键:简化的 CMake 路径+# Key: simplified CMake pathcmake.source-dir = "third_party/DiskANN"-# 关键:Python 包在根目录,路径完全匹配+# Key: Python package in root directory, paths match exactlywheel.packages = ["leann_backend_diskann"]-# 使用默认的 redirect 模式+# Use default redirect modeeditable.mode = "redirect"cmake.build-type = "Release"build.verbose = truediff --git a/packages/leann-backend-hnsw/CMakeLists.txt b/packages/leann-backend-hnsw/CMakeLists.txtindex bcadd12..2b86b0a 100644--- a/packages/leann-backend-hnsw/CMakeLists.txt+++ b/packages/leann-backend-hnsw/CMakeLists.txt@@ -1,6 +1,7 @@-# 最终简化版cmake_minimum_required(VERSION 3.24)project(leann_backend_hnsw_wrapper)+set(CMAKE_C_COMPILER_WORKS 1)+set(CMAKE_CXX_COMPILER_WORKS 1)# Set OpenMP path for macOSif(APPLE)@@ -11,15 +12,9 @@ if(APPLE)set(OpenMP_omp_LIBRARY "/opt/homebrew/opt/libomp/lib/libomp.dylib")endif()-# Build ZeroMQ from source-set(ZMQ_BUILD_TESTS OFF CACHE BOOL "" FORCE)-set(ENABLE_DRAFTS OFF CACHE BOOL "" FORCE)-set(ENABLE_PRECOMPILED OFF CACHE BOOL "" FORCE)-set(WITH_PERF_TOOL OFF CACHE BOOL "" FORCE)-set(WITH_DOCS OFF CACHE BOOL "" FORCE)-set(BUILD_SHARED OFF CACHE BOOL "" FORCE)-set(BUILD_STATIC ON CACHE BOOL "" FORCE)-add_subdirectory(third_party/libzmq)+# Use system ZeroMQ instead of building from source+find_package(PkgConfig REQUIRED)+pkg_check_modules(ZMQ REQUIRED libzmq)# Add cppzmq headersinclude_directories(third_party/cppzmq)@@ -29,6 +24,7 @@ set(MSGPACK_USE_BOOST OFF CACHE BOOL "" FORCE)add_compile_definitions(MSGPACK_NO_BOOST)include_directories(third_party/msgpack-c/include)+# Faiss configuration - streamlined buildset(FAISS_ENABLE_PYTHON ON CACHE BOOL "" FORCE)set(FAISS_ENABLE_GPU OFF CACHE BOOL "" FORCE)set(FAISS_ENABLE_EXTRAS OFF CACHE BOOL "" FORCE)@@ -36,4 +32,24 @@ set(BUILD_TESTING OFF CACHE BOOL "" FORCE)set(FAISS_ENABLE_C_API OFF CACHE BOOL "" FORCE)set(FAISS_OPT_LEVEL "generic" CACHE STRING "" FORCE)+# Disable additional SIMD versions to speed up compilation+set(FAISS_ENABLE_AVX2 OFF CACHE BOOL "" FORCE)+set(FAISS_ENABLE_AVX512 OFF CACHE BOOL "" FORCE)++# Additional optimization options from INSTALL.md+set(CMAKE_BUILD_TYPE "Release" CACHE STRING "" FORCE)+set(BUILD_SHARED_LIBS OFF CACHE BOOL "" FORCE) # Static library is faster to build++# Avoid building demos and benchmarks+set(BUILD_DEMOS OFF CACHE BOOL "" FORCE)+set(BUILD_BENCHS OFF CACHE BOOL "" FORCE)++# NEW: Tell Faiss to only build the generic version+set(FAISS_BUILD_GENERIC ON CACHE BOOL "" FORCE)+set(FAISS_BUILD_AVX2 OFF CACHE BOOL "" FORCE)+set(FAISS_BUILD_AVX512 OFF CACHE BOOL "" FORCE)++# IMPORTANT: Disable building AVX versions to speed up compilation+set(FAISS_BUILD_AVX_VERSIONS OFF CACHE BOOL "" FORCE)+add_subdirectory(third_party/faiss)\ No newline at end of filediff --git a/packages/leann-backend-hnsw/leann_backend_hnsw/hnsw_backend.py b/packages/leann-backend-hnsw/leann_backend_hnsw/hnsw_backend.pyindex 3ceda37..b7061e1 100644--- a/packages/leann-backend-hnsw/leann_backend_hnsw/hnsw_backend.py+++ b/packages/leann-backend-hnsw/leann_backend_hnsw/hnsw_backend.py@@ -59,10 +59,6 @@ class HNSWBuilder(LeannBackendBuilderInterface):if data.dtype != np.float32:data = data.astype(np.float32)- label_map = {i: str_id for i, str_id in enumerate(ids)}- label_map_file = index_dir / "leann.labels.map"- with open(label_map_file, "wb") as f:- pickle.dump(label_map, f)metric_enum = get_metric_map().get(self.distance_metric.lower())if metric_enum is None:@@ -142,13 +138,6 @@ class HNSWSearcher(BaseSearcher):self._index = faiss.read_index(str(index_file), faiss.IO_FLAG_MMAP, hnsw_config)- # Load label mapping- label_map_file = self.index_dir / "leann.labels.map"- if not label_map_file.exists():- raise FileNotFoundError(f"Label map file not found at {label_map_file}")-- with open(label_map_file, "rb") as f:- self.label_map = pickle.load(f)def search(self,@@ -239,10 +228,7 @@ class HNSWSearcher(BaseSearcher):)string_labels = [- [- self.label_map.get(int_label, f"unknown_{int_label}")- for int_label in batch_labels- ]+ [str(int_label) for int_label in batch_labels]for batch_labels in labels]diff --git a/packages/leann-backend-hnsw/leann_backend_hnsw/hnsw_embedding_server.py b/packages/leann-backend-hnsw/leann_backend_hnsw/hnsw_embedding_server.pyindex 579f2bb..48f8e1b 100644--- a/packages/leann-backend-hnsw/leann_backend_hnsw/hnsw_embedding_server.py+++ b/packages/leann-backend-hnsw/leann_backend_hnsw/hnsw_embedding_server.py@@ -1,16 +1,11 @@-#!/usr/bin/env python3"""-HNSW-specific embedding server with removed config.py dependencies-Based on DiskANN embedding server architecture+HNSW-specific embedding server"""-import pickleimport argparseimport threadingimport time-from transformers import AutoTokenizer, AutoModelimport os-from contextlib import contextmanagerimport zmqimport numpy as npimport msgpack@@ -24,279 +19,50 @@ RED = "\033[91m"RESET = "\033[0m"# Set up logging based on environment variable-LOG_LEVEL = os.getenv('LEANN_LOG_LEVEL', 'INFO').upper()+LOG_LEVEL = os.getenv("LEANN_LOG_LEVEL", "INFO").upper()logging.basicConfig(level=getattr(logging, LOG_LEVEL, logging.INFO),- format='%(asctime)s - %(levelname)s - %(message)s'+ format="%(asctime)s - %(levelname)s - %(message)s",)logger = logging.getLogger(__name__)-def is_similarity_metric():- """- Check if the metric type is similarity-based (like inner product).- 0 = L2 (distance metric), 1 = Inner Product (similarity metric)- """- return True # 1 is METRIC_INNER_PRODUCT in FAISS---# Function for E5-style average pooling-import torch-from torch import Tensor-import torch.nn.functional as F--# Timing utilities-@contextmanager-def timer(name: str, sync_cuda: bool = True):- """Context manager for timing operations with optional CUDA sync"""- start_time = time.time()- if sync_cuda and torch.cuda.is_available():- torch.cuda.synchronize()- try:- yield- finally:- if sync_cuda and torch.cuda.is_available():- torch.cuda.synchronize()- elif sync_cuda and torch.backends.mps.is_available():- torch.mps.synchronize()- elapsed = time.time() - start_time- logger.info(f"⏱️ {name}: {elapsed:.4f}s")---def e5_average_pool(last_hidden_states: Tensor, attention_mask: Tensor) -> Tensor:- last_hidden = last_hidden_states.masked_fill(~attention_mask[..., None].bool(), 0.0)- return last_hidden.sum(dim=1) / attention_mask.sum(dim=1)[..., None]---class SimplePassageLoader:- """- Simple passage loader that replaces config.py dependencies- """-- def __init__(self, passages_data: Optional[Dict[str, Any]] = None):- self.passages_data = passages_data or {}- self._meta_path = ""-- def __getitem__(self, passage_id: Union[str, int]) -> Dict[str, str]:- """Get passage by ID"""- str_id = str(passage_id)- if str_id in self.passages_data:- return {"text": self.passages_data[str_id]}- else:- # Return empty text for missing passages- return {"text": ""}-- def __len__(self) -> int:- return len(self.passages_data)-- def keys(self):- return self.passages_data.keys()---def load_passages_from_metadata(meta_file: str) -> SimplePassageLoader:- """- Load passages using metadata file with PassageManager for lazy loading- """- # Load metadata to get passage sources- with open(meta_file, "r") as f:- meta = json.load(f)-- # Import PassageManager dynamically to avoid circular imports- # Find the leann package directory relative to this file- current_dir = Path(__file__).parent- leann_core_path = current_dir.parent.parent / "leann-core" / "src"- sys.path.insert(0, str(leann_core_path))-- try:- from leann.api import PassageManager-- passage_manager = PassageManager(meta["passage_sources"])- finally:- sys.path.pop(0)-- # Load label map- passages_dir = Path(meta_file).parent- label_map_file = passages_dir / "leann.labels.map"-- if label_map_file.exists():- import pickle-- with open(label_map_file, "rb") as f:- label_map = pickle.load(f)- print(f"Loaded label map with {len(label_map)} entries")- else:- raise FileNotFoundError(f"Label map file not found: {label_map_file}")-- print(f"Initialized lazy passage loading for {len(label_map)} passages")-- class LazyPassageLoader(SimplePassageLoader):- def __init__(self, passage_manager, label_map):- self.passage_manager = passage_manager- self.label_map = label_map- # Initialize parent with empty data- super().__init__({})-- def __getitem__(self, passage_id: Union[str, int]) -> Dict[str, str]:- """Get passage by ID with lazy loading"""- try:- int_id = int(passage_id)- if int_id in self.label_map:- string_id = self.label_map[int_id]- passage_data = self.passage_manager.get_passage(string_id)- if passage_data and passage_data.get("text"):- return {"text": passage_data["text"]}- else:- logger.debug(f"Empty text for ID {int_id} -> {string_id}")- return {"text": ""}- else:- logger.debug(f"ID {int_id} not found in label_map")- return {"text": ""}- except Exception as e:- logger.debug(f"Exception getting passage {passage_id}: {e}")- return {"text": ""}-- def __len__(self) -> int:- return len(self.label_map)-- def keys(self):- return self.label_map.keys()-- return LazyPassageLoader(passage_manager, label_map)--def create_hnsw_embedding_server(passages_file: Optional[str] = None,passages_data: Optional[Dict[str, str]] = None,- embeddings_file: Optional[str] = None,- use_fp16: bool = True,- use_int8: bool = False,- use_cuda_graphs: bool = False,zmq_port: int = 5555,- max_batch_size: int = 128,model_name: str = "sentence-transformers/all-mpnet-base-v2",- custom_max_length_param: Optional[int] = None,distance_metric: str = "mips",embedding_mode: str = "sentence-transformers",- enable_warmup: bool = False,):"""Create and start a ZMQ-based embedding server for HNSW backend.-- Args:- passages_file: Path to JSON file containing passage ID -> text mapping- passages_data: Direct passage data dict (alternative to passages_file)- embeddings_file: Path to pre-computed embeddings file (optional)- use_fp16: Whether to use FP16 precision- use_int8: Whether to use INT8 quantization- use_cuda_graphs: Whether to use CUDA graphs- zmq_port: ZMQ port to bind to- max_batch_size: Maximum batch size for processing- model_name: Transformer model name- custom_max_length_param: Custom max sequence length- distance_metric: The distance metric to use- enable_warmup: Whether to perform warmup requests on server start+ Simplified version using unified embedding computation module."""- # Handle different embedding modes directly in HNSW server-# Auto-detect mode based on model name if not explicitly set- if embedding_mode == "sentence-transformers" and model_name.startswith("text-embedding-"):+ if embedding_mode == "sentence-transformers" and model_name.startswith(+ "text-embedding-"+ ):embedding_mode = "openai"-- if embedding_mode == "openai":- print(f"Using OpenAI API mode for {model_name}")- tokenizer = None # No local tokenizer needed for OpenAI API- elif embedding_mode == "mlx":- print(f"Using MLX mode for {model_name}")- tokenizer = None # MLX handles tokenization separately- else: # sentence-transformers- print(f"Loading tokenizer for {model_name}...")- # Optimized tokenizer loading: try local first, then fallback- try:- tokenizer = AutoTokenizer.from_pretrained(- model_name,- use_fast=True, # Use fast tokenizer (better runtime perf)- local_files_only=True # Avoid network delays- )- print(f"Tokenizer loaded successfully! (local + fast)")- except Exception as e:- print(f"Local tokenizer failed ({e}), trying network download...")- tokenizer = AutoTokenizer.from_pretrained(- model_name,- use_fast=True # Use fast tokenizer- )- print(f"Tokenizer loaded successfully! (network)")- # Device setup- mps_available = hasattr(torch.backends, "mps") and torch.backends.mps.is_available()- cuda_available = torch.cuda.is_available()-- print(f"MPS available: {mps_available}")- print(f"CUDA available: {cuda_available}")+ print(f"Starting HNSW server on port {zmq_port} with model {model_name}")+ print(f"Using embedding mode: {embedding_mode}")- if cuda_available:- device = torch.device("cuda")- print("Using CUDA device")- elif mps_available:- device = torch.device("mps")- print("Using MPS device (Apple Silicon)")- else:- device = torch.device("cpu")- print("Using CPU device (no GPU acceleration available)")+ # Add leann-core to path for unified embedding computation+ current_dir = Path(__file__).parent+ leann_core_path = current_dir.parent.parent / "leann-core" / "src"+ sys.path.insert(0, str(leann_core_path))- # Load model to the appropriate device- print(f"Starting HNSW server on port {zmq_port} with model {model_name}")- print(f"Loading model {model_name}... (this may take a while if downloading)")+ try:+ from leann.embedding_compute import compute_embeddings+ from leann.api import PassageManager- if embedding_mode == "mlx":- # For MLX models, we need to use the MLX embedding computation- print("MLX model detected - using MLX backend for embeddings")- model = None # We'll handle MLX separately- elif embedding_mode == "openai":- # For OpenAI API, no local model needed- print("OpenAI API mode - no local model loading required")- model = None- else:- # Use optimized transformers loading for sentence-transformers models- print(f"Loading model with optimizations...")- try:- # Ultra-fast loading: preload config + fast_init- from transformers import AutoConfig- config = AutoConfig.from_pretrained(model_name, local_files_only=True)- model = AutoModel.from_pretrained(- model_name,- config=config,- torch_dtype=torch.float16, # Half precision for speed- low_cpu_mem_usage=True, # Reduce memory peaks- local_files_only=True, # Avoid network delays- _fast_init=True # Skip weight init checks- ).to(device).eval()- print(f"Model {model_name} loaded successfully! (ultra-fast)")- except Exception as e:- print(f"Ultra-fast loading failed ({e}), trying optimized...")- try:- # Fallback: regular optimized loading- model = AutoModel.from_pretrained(- model_name,- torch_dtype=torch.float16,- low_cpu_mem_usage=True,- local_files_only=True- ).to(device).eval()- print(f"Model {model_name} loaded successfully! (optimized)")- except Exception as e2:- print(f"Optimized loading failed ({e2}), trying network...")- try:- # Fallback: optimized network loading- model = AutoModel.from_pretrained(- model_name,- torch_dtype=torch.float16,- low_cpu_mem_usage=True- ).to(device).eval()- print(f"Model {model_name} loaded successfully! (network + optimized)")- except Exception as e3:- print(f"All optimized methods failed ({e3}), using standard...")- # Final fallback: standard loading- model = AutoModel.from_pretrained(model_name).to(device).eval()- print(f"Model {model_name} loaded successfully! (standard)")+ print("Successfully imported unified embedding computation module")+ except ImportError as e:+ print(f"ERROR: Failed to import embedding computation module: {e}")+ return+ finally:+ sys.path.pop(0)# Check port availabilityimport socket@@ -309,314 +75,19 @@ def create_hnsw_embedding_server(print(f"{RED}Port {zmq_port} is already in use{RESET}")return- # Apply model optimizations (similar to DiskANN version)- if use_fp16 and (cuda_available or mps_available):- model = model.half()- model = torch.compile(model)- print(f"Using FP16 precision with model: {model_name}")- elif use_int8:- print(- "- Using TorchAO for Int8 dynamic activation and Int8 weight quantization"- )- from torchao.quantization import (- quantize_,- Int8DynamicActivationInt8WeightConfig,- )-- quantize_(model, Int8DynamicActivationInt8WeightConfig())- model = torch.compile(model)- model.eval()- print("- Model successfully quantized and compiled")-- # Load passages- if passages_data:- passages = SimplePassageLoader(passages_data)- print(f"Using provided passages data: {len(passages)} passages")- elif passages_file:- # Check if it's a metadata file or a single passages file- if passages_file.endswith(".meta.json"):- passages = load_passages_from_metadata(passages_file)- # Store the meta path for future reference- passages._meta_path = passages_file- else:- # Try to find metadata file in same directory- passages_dir = Path(passages_file).parent- meta_files = list(passages_dir.glob("*.meta.json"))- if meta_files:- print(f"Found metadata file: {meta_files[0]}, using lazy loading")- passages = load_passages_from_metadata(str(meta_files[0]))- else:- # Fallback to original single file loading (will cause warnings)- print(- "WARNING: No metadata file found, using single file loading (may cause missing passage warnings)"- )- passages = (- SimplePassageLoader()- ) # Use empty loader to avoid massive warnings- else:- passages = SimplePassageLoader()- print("No passages provided, using empty loader")-- # Load embeddings if provided- _embeddings = None- if embeddings_file and os.path.exists(embeddings_file):- try:- with open(embeddings_file, "rb") as f:- _embeddings = pickle.load(f)- print(f"Loaded embeddings from {embeddings_file}")- except Exception as e:- print(f"Error loading embeddings: {e}")-- class DeviceTimer:- """Device event-based timer for accurate timing."""-- def __init__(self, name="", device=device):- self.name = name- self.device = device- self.start_time = 0- self.end_time = 0-- if cuda_available:- self.start_event = torch.cuda.Event(enable_timing=True)- self.end_event = torch.cuda.Event(enable_timing=True)- else:- self.start_event = None- self.end_event = None-- @contextmanager- def timing(self):- self.start()- yield- self.end()-- def start(self):- if cuda_available:- torch.cuda.synchronize()- self.start_event.record()- else:- if self.device.type == "mps":- torch.mps.synchronize()- self.start_time = time.time()-- def end(self):- if cuda_available:- self.end_event.record()- torch.cuda.synchronize()- else:- if self.device.type == "mps":- torch.mps.synchronize()- self.end_time = time.time()-- def elapsed_time(self):- if cuda_available:- return self.start_event.elapsed_time(self.end_event) / 1000.0- else:- return self.end_time - self.start_time-- def print_elapsed(self):- return # Disabled for now-- def _process_batch_mlx(texts_batch, ids_batch, missing_ids):- """Process a batch of texts using MLX backend"""- try:- # Import MLX embedding computation from main API- from leann.api import compute_embeddings-- # Compute embeddings using MLX- embeddings = compute_embeddings(texts_batch, model_name, mode="mlx", use_server=False)-- print(- f"[leann_backend_hnsw.hnsw_embedding_server LOG]: MLX embeddings computed for {len(texts_batch)} texts"- )- print(- f"[leann_backend_hnsw.hnsw_embedding_server LOG]: Embedding shape: {embeddings.shape}"- )-- return embeddings-- except Exception as e:- print(- f"[leann_backend_hnsw.hnsw_embedding_server LOG]: ERROR in MLX processing: {e}"- )- raise-- def process_batch(texts_batch, ids_batch, missing_ids):- """Process a batch of texts and return embeddings"""-- # Handle different embedding modes- if embedding_mode == "mlx":- return _process_batch_mlx(texts_batch, ids_batch, missing_ids)- elif embedding_mode == "openai":- with timer("OpenAI API call", sync_cuda=False):- from leann.api import compute_embeddings_openai- return compute_embeddings_openai(texts_batch, model_name)-- _is_e5_model = "e5" in model_name.lower()- _is_bge_model = "bge" in model_name.lower()- batch_size = len(texts_batch)-- # Allow empty texts to pass through (remove validation)-- # E5 model preprocessing- if _is_e5_model:- processed_texts_batch = [f"passage: {text}" for text in texts_batch]- else:- processed_texts_batch = texts_batch-- # Set max length- if _is_e5_model:- current_max_length = (- custom_max_length_param if custom_max_length_param is not None else 512- )- else:- current_max_length = (- custom_max_length_param if custom_max_length_param is not None else 256- )-- tokenize_timer = DeviceTimer("tokenization (batch)", device)- to_device_timer = DeviceTimer("transfer to device (batch)", device)- embed_timer = DeviceTimer("embedding (batch)", device)- pool_timer = DeviceTimer("pooling (batch)", device)- norm_timer = DeviceTimer("normalization (batch)", device)-- with tokenize_timer.timing():- encoded_batch = tokenizer(- processed_texts_batch,- padding="max_length",- truncation=True,- max_length=current_max_length,- return_tensors="pt",- return_token_type_ids=False,- )-- seq_length = encoded_batch["input_ids"].size(1)-- with to_device_timer.timing():- enc = {k: v.to(device) for k, v in encoded_batch.items()}-- with torch.no_grad():- with timer("Model forward pass"):- with embed_timer.timing():- out = model(enc["input_ids"], enc["attention_mask"])-- with timer("Pooling"):- with pool_timer.timing():- if _is_bge_model:- pooled_embeddings = out.last_hidden_state[:, 0]- elif not hasattr(out, "last_hidden_state"):- if isinstance(out, torch.Tensor) and len(out.shape) == 2:- pooled_embeddings = out- else:- print(- f"{RED}ERROR: Cannot determine how to pool. Output shape: {out.shape if isinstance(out, torch.Tensor) else 'N/A'}{RESET}"- )- hidden_dim = getattr(- model.config, "hidden_size", 384 if _is_e5_model else 768- )- pooled_embeddings = torch.zeros(- (batch_size, hidden_dim),- device=device,- dtype=enc["input_ids"].dtype- if hasattr(enc["input_ids"], "dtype")- else torch.float32,- )- elif _is_e5_model:- pooled_embeddings = e5_average_pool(- out.last_hidden_state, enc["attention_mask"]- )- else:- hidden_states = out.last_hidden_state- mask_expanded = (- enc["attention_mask"]- .unsqueeze(-1)- .expand(hidden_states.size())- .float()- )- sum_embeddings = torch.sum(hidden_states * mask_expanded, 1)- sum_mask = torch.clamp(mask_expanded.sum(1), min=1e-9)- pooled_embeddings = sum_embeddings / sum_mask-- final_embeddings = pooled_embeddings- if _is_e5_model or _is_bge_model:- with norm_timer.timing():- final_embeddings = F.normalize(pooled_embeddings, p=2, dim=1)-- if torch.isnan(final_embeddings).any() or torch.isinf(final_embeddings).any():- print(- f"{RED}!!! In process_batch: NaN or Inf detected in final_embeddings! "- f"Model: {model_name}, E5: {_is_e5_model}. IDs (sample): {ids_batch[:5]}...{RESET}"- )- dim_size = final_embeddings.shape[-1]- error_output = torch.zeros(- (batch_size, dim_size), device="cpu", dtype=torch.float32- ).numpy()- print(- f"{RED}Returning zero embeddings of shape ({batch_size}, {dim_size}) due to NaN/Inf.{RESET}"- )- return error_output-- return final_embeddings.cpu().numpy()-- def client_warmup(zmq_port):- """Perform client-side warmup"""- time.sleep(2)- print(f"Performing client-side warmup with model {model_name}...")-- # Get actual passage IDs from the loaded passages- sample_ids = []- if hasattr(passages, 'keys') and len(passages) > 0:- available_ids = list(passages.keys())- # Take up to 5 actual IDs, but at least 1- sample_ids = available_ids[:min(5, len(available_ids))]- print(f"Using actual passage IDs for warmup: {sample_ids}")- else:- print("No passages available for warmup, skipping warmup...")- return-- try:- context = zmq.Context()- socket = context.socket(zmq.REQ)- socket.connect(f"tcp://localhost:{zmq_port}")- socket.setsockopt(zmq.RCVTIMEO, 30000)- socket.setsockopt(zmq.SNDTIMEO, 30000)-- try:- ids_to_send = [int(x) for x in sample_ids]- except ValueError:- print("Warning: Could not convert sample IDs to integers, skipping warmup")- return-- if not ids_to_send:- print("Skipping warmup send.")- return-- request_payload = [ids_to_send]- request_bytes = msgpack.packb(request_payload)-- for i in range(3):- print(f"Sending warmup request {i + 1}/3 via ZMQ (MessagePack)...")- socket.send(request_bytes)- response_bytes = socket.recv()-- response_payload = msgpack.unpackb(response_bytes)- dimensions = response_payload[0]- embeddings_count = (- dimensions[0] if dimensions and len(dimensions) > 0 else 0- )- print(- f"Warmup request {i + 1}/3 successful, received {embeddings_count} embeddings"- )- time.sleep(0.1)-- print("Client-side MessagePack ZMQ warmup complete")- socket.close()- context.term()- except Exception as e:- print(f"Error during MessagePack ZMQ warmup: {e}")+ # Only support metadata file, fail fast for everything else+ if not passages_file or not passages_file.endswith(".meta.json"):+ raise ValueError("Only metadata files (.meta.json) are supported")++ # Load metadata to get passage sources+ with open(passages_file, "r") as f:+ meta = json.load(f)++ passages = PassageManager(meta["passage_sources"])+ print(f"Loaded PassageManager with {len(passages.global_offset_map)} passages from metadata")def zmq_server_thread():"""ZMQ server thread"""- nonlocal passages, model, tokenizer, model_name, embedding_modecontext = zmq.Context()socket = context.socket(zmq.REP)socket.bind(f"tcp://*:{zmq_port}")@@ -631,457 +102,139 @@ def create_hnsw_embedding_server(print(f"Received ZMQ request of size {len(message_bytes)} bytes")e2e_start = time.time()- lookup_timer = DeviceTimer("text lookup", device)-- try:- request_payload = msgpack.unpackb(message_bytes)- if isinstance(request_payload, list):- logger.debug(f"request_payload length: {len(request_payload)}")- for i, item in enumerate(request_payload):- print(- f"DEBUG: request_payload[{i}]: {type(item)} - {item if len(str(item)) < 100 else str(item)[:100] + '...'}"- )-- # Handle control messages for meta path and model management FIRST- if isinstance(request_payload, list) and len(request_payload) >= 1:- if request_payload[0] == "__QUERY_META_PATH__":- # Return the current meta path being used by the server- current_meta_path = (- getattr(passages, "_meta_path", "")- if hasattr(passages, "_meta_path")- else ""- )- response = [current_meta_path]- socket.send(msgpack.packb(response))- continue-- elif (- request_payload[0] == "__UPDATE_META_PATH__"- and len(request_payload) >= 2- ):- # Update the server's meta path and reload passages- new_meta_path = request_payload[1]- try:- print(- f"INFO: Updating server meta path to: {new_meta_path}"- )- # Reload passages from the new meta file- passages = load_passages_from_metadata(new_meta_path)- # Store the meta path for future queries- passages._meta_path = new_meta_path- response = ["SUCCESS"]- print(- f"INFO: Successfully updated meta path and reloaded {len(passages)} passages"- )- except Exception as e:- print(f"ERROR: Failed to update meta path: {e}")- response = ["FAILED", str(e)]- socket.send(msgpack.packb(response))- continue-- elif request_payload[0] == "__QUERY_MODEL__":- # Return the current model being used by the server- response = [model_name]- socket.send(msgpack.packb(response))- continue-- elif (- request_payload[0] == "__UPDATE_MODEL__"- and len(request_payload) >= 2- ):- # Update the server's embedding model- new_model_name = request_payload[1]- try:- print(- f"INFO: Updating server model from {model_name} to: {new_model_name}"- )-- # Clean up old model to free memory- logger.info("Releasing old model from memory...")- old_model = model- old_tokenizer = tokenizer-- # Load new tokenizer first (optimized)- print(f"Loading new tokenizer for {new_model_name}...")- try:- tokenizer = AutoTokenizer.from_pretrained(- new_model_name,- use_fast=True,- local_files_only=True- )- print(f"New tokenizer loaded! (local + fast)")- except:- tokenizer = AutoTokenizer.from_pretrained(- new_model_name,- use_fast=True- )- print(f"New tokenizer loaded! (network + fast)")-- # Load new model (optimized)- print(f"Loading new model {new_model_name}...")- try:- # Ultra-fast model switching- from transformers import AutoConfig- config = AutoConfig.from_pretrained(new_model_name, local_files_only=True)- model = AutoModel.from_pretrained(- new_model_name,- config=config,- torch_dtype=torch.float16,- low_cpu_mem_usage=True,- local_files_only=True,- _fast_init=True- )- print(f"New model loaded! (ultra-fast)")- except:- try:- model = AutoModel.from_pretrained(- new_model_name,- torch_dtype=torch.float16,- low_cpu_mem_usage=True,- local_files_only=True- )- print(f"New model loaded! (optimized)")- except:- try:- model = AutoModel.from_pretrained(- new_model_name,- torch_dtype=torch.float16,- low_cpu_mem_usage=True- )- print(f"New model loaded! (network + optimized)")- except:- model = AutoModel.from_pretrained(new_model_name)- print(f"New model loaded! (standard)")- model.to(device)- model.eval()-- # Now safely delete old model after new one is loaded- del old_model- del old_tokenizer-- # Clear GPU cache if available- if device.type == "cuda":- torch.cuda.empty_cache()- logger.info("Cleared CUDA cache")- elif device.type == "mps":- torch.mps.empty_cache()- logger.info("Cleared MPS cache")-- # Update model name- model_name = new_model_name-- # Re-detect embedding mode based on new model name- if model_name.startswith("text-embedding-"):- embedding_mode = "openai"- logger.info(f"Auto-detected embedding mode: openai for {model_name}")- else:- embedding_mode = "sentence-transformers"- logger.info(f"Auto-detected embedding mode: sentence-transformers for {model_name}")+ request_payload = msgpack.unpackb(message_bytes)- # Force garbage collection- import gc-- gc.collect()- logger.info("Memory cleanup completed")-- response = ["SUCCESS"]- print(- f"INFO: Successfully updated model to: {new_model_name}"- )- except Exception as e:- print(f"ERROR: Failed to update model: {e}")- response = ["FAILED", str(e)]- socket.send(msgpack.packb(response))- continue-- # Handle direct text embedding request (for OpenAI and sentence-transformers)- if isinstance(request_payload, list) and len(request_payload) > 0:- # Check if this is a direct text request (list of strings) and NOT a control message- if (all(isinstance(item, str) for item in request_payload) and- not request_payload[0].startswith("__")):- logger.info(f"Processing direct text embedding request for {len(request_payload)} texts in {embedding_mode} mode")-- try:- if embedding_mode == "openai":- from leann.api import compute_embeddings_openai- embeddings = compute_embeddings_openai(request_payload, model_name)- else:- # sentence-transformers mode - compute directly- with timer(f"Direct text embedding ({len(request_payload)} texts)"):- embeddings = process_batch(request_payload, [], [])-- response = embeddings.tolist()- socket.send(msgpack.packb(response))- e2e_end = time.time()- logger.info(f"⏱️ Text embedding E2E time: {e2e_end - e2e_start:.6f}s")- continue- except Exception as e:- logger.error(f"ERROR: Failed to compute {embedding_mode} embeddings: {e}")- socket.send(msgpack.packb([]))- continue-- # Handle distance calculation requests- if (- isinstance(request_payload, list)- and len(request_payload) == 2- and isinstance(request_payload[0], list)- and isinstance(request_payload[1], list)- ):- node_ids = request_payload[0]- query_vector = np.array(request_payload[1], dtype=np.float32)-- logger.debug("Distance calculation request received")- print(f" Node IDs: {node_ids}")- print(f" Query vector dim: {len(query_vector)}")- print(f" Passages loaded: {len(passages)}")-- # Get embeddings for node IDs- texts = []- missing_ids = []- with lookup_timer.timing():- for nid in node_ids:- logger.debug(f"Looking up passage ID {nid}")- try:- txtinfo = passages[nid]- if txtinfo is None:- print(- f"ERROR: Passage with ID {nid} returned None"- )- print(f"ERROR: txtinfo: {txtinfo}")- raise RuntimeError(- f"FATAL: Passage with ID {nid} returned None"- )- txt = txtinfo[- "text"- ] # Allow empty text to pass through- print(- f"DEBUG: Found text for ID {nid}, length: {len(txt)}"- )- texts.append(txt)- except KeyError:- print(- f"ERROR: Passage ID {nid} not found in passages dict"- )- print(- f"ERROR: Available passage IDs: {list(passages.keys())}..."- )- raise RuntimeError(- f"FATAL: Passage with ID {nid} not found"- )- except Exception as e:- print(- f"ERROR: Exception looking up passage ID {nid}: {e}"- )- raise- lookup_timer.print_elapsed()-- # Process embeddings in chunks if needed- all_node_embeddings = []- total_size = len(texts)-- if total_size > max_batch_size:- for i in range(0, total_size, max_batch_size):- end_idx = min(i + max_batch_size, total_size)- chunk_texts = texts[i:end_idx]- chunk_ids = node_ids[i:end_idx]-- embeddings_chunk = process_batch(- chunk_texts, chunk_ids, missing_ids- )- all_node_embeddings.append(embeddings_chunk)-- if cuda_available:- torch.cuda.empty_cache()- elif device.type == "mps":- torch.mps.empty_cache()-- node_embeddings = np.vstack(all_node_embeddings)- else:- node_embeddings = process_batch(- texts, node_ids, missing_ids- )-- # Calculate distances- query_tensor = torch.tensor(query_vector, device=device).float()- node_embeddings_tensor = torch.tensor(- node_embeddings, device=device- ).float()-- calc_timer = DeviceTimer("distance calculation", device)- with calc_timer.timing():- with torch.no_grad():- if distance_metric == "l2":- node_embeddings_np = (- node_embeddings_tensor.cpu()- .numpy()- .astype(np.float32)- )- query_np = (- query_tensor.cpu().numpy().astype(np.float32)- )- distances = np.sum(- np.square(- node_embeddings_np - query_np.reshape(1, -1)- ),- axis=1,- )- else: # mips or cosine- node_embeddings_np = (- node_embeddings_tensor.cpu().numpy()- )- query_np = query_tensor.cpu().numpy()- distances = -np.dot(node_embeddings_np, query_np)- calc_timer.print_elapsed()-- try:- response_payload = distances.flatten().tolist()- response_bytes = msgpack.packb(- [response_payload], use_single_float=True- )- print(- f"Sending distance response with {len(distances)} distances"- )- except Exception as pack_error:- print(- f"ERROR: Error packing MessagePack distance response: {pack_error}"- )- print(f"ERROR: distances shape: {distances.shape}")- print(f"ERROR: distances dtype: {distances.dtype}")- print(f"ERROR: distances content: {distances}")- print(f"ERROR: node_ids: {node_ids}")- print(f"ERROR: query_vector shape: {query_vector.shape}")- # Still return empty for now but with full error info- response_bytes = msgpack.packb([[]])-- socket.send(response_bytes)-- if device.type == "cuda":- torch.cuda.synchronize()- elif device.type == "mps":- torch.mps.synchronize()- e2e_end = time.time()+ # Handle direct text embedding request (for OpenAI and sentence-transformers)+ if isinstance(request_payload, list) and len(request_payload) > 0:+ # Check if this is a direct text request (list of strings)+ if all(isinstance(item, str) for item in request_payload):logger.info(- f"⏱️ Distance calculation E2E time: {e2e_end - e2e_start:.6f}s"+ f"Processing direct text embedding request for {len(request_payload)} texts in {embedding_mode} mode")- continue+ # Use unified embedding computation+ embeddings = compute_embeddings(+ request_payload, model_name, mode=embedding_mode+ )- # Standard embedding request (passage ID lookup)- if (- not isinstance(request_payload, list)- or len(request_payload) != 1- or not isinstance(request_payload[0], list)- ):- print(- f"Error: Invalid MessagePack request format. Expected [[ids...]] or [texts...], got: {type(request_payload)}"+ response = embeddings.tolist()+ socket.send(msgpack.packb(response))+ e2e_end = time.time()+ logger.info(+ f"⏱️ Text embedding E2E time: {e2e_end - e2e_start:.6f}s")- socket.send(msgpack.packb([[], []]))continue+ # Handle distance calculation requests+ if (+ isinstance(request_payload, list)+ and len(request_payload) == 2+ and isinstance(request_payload[0], list)+ and isinstance(request_payload[1], list)+ ):node_ids = request_payload[0]- print(f"Request for {len(node_ids)} node embeddings")+ query_vector = np.array(request_payload[1], dtype=np.float32)- except Exception as unpack_error:- print(f"Error unpacking MessagePack request: {unpack_error}")- socket.send(msgpack.packb([[], []]))- continue+ logger.debug("Distance calculation request received")+ print(f" Node IDs: {node_ids}")+ print(f" Query vector dim: {len(query_vector)}")- # Look up texts by node IDs- texts = []- missing_ids = []- with lookup_timer.timing():+ # Get embeddings for node IDs+ texts = []for nid in node_ids:try:- txtinfo = passages[nid]- if txtinfo is None or txtinfo["text"] == "":- raise RuntimeError(- f"FATAL: Passage with ID {nid} not found - failing fast"- )- else:- txt = txtinfo["text"]- except (KeyError, IndexError):- raise RuntimeError(- f"FATAL: Passage with ID {nid} not found - failing fast"- )- texts.append(txt)- lookup_timer.print_elapsed()-- if missing_ids:- print(f"Missing passages for IDs: {missing_ids}")-- # Process in chunks- total_size = len(texts)- print(- f"Total batch size: {total_size}, max_batch_size: {max_batch_size}"- )-- all_embeddings = []-- if total_size > max_batch_size:+ passage_data = passages.get_passage(str(nid))+ txt = passage_data["text"]+ texts.append(txt)+ except KeyError:+ print(f"ERROR: Passage ID {nid} not found")+ raise RuntimeError(f"FATAL: Passage with ID {nid} not found")+ except Exception as e:+ print(f"ERROR: Exception looking up passage ID {nid}: {e}")+ raise++ # Process embeddings+ embeddings = compute_embeddings(+ texts, model_name, mode=embedding_mode+ )print(- f"Splitting batch of size {total_size} into chunks of {max_batch_size}"+ f"INFO: Computed embeddings for {len(texts)} texts, shape: {embeddings.shape}")- for i in range(0, total_size, max_batch_size):- end_idx = min(i + max_batch_size, total_size)- print(- f"Processing chunk {i // max_batch_size + 1}/{(total_size + max_batch_size - 1) // max_batch_size}: items {i} to {end_idx - 1}"- )-- chunk_texts = texts[i:end_idx]- chunk_ids = node_ids[i:end_idx]- embeddings_chunk = process_batch(- chunk_texts, chunk_ids, missing_ids+ # Calculate distances+ if distance_metric == "l2":+ distances = np.sum(+ np.square(embeddings - query_vector.reshape(1, -1)), axis=1)- all_embeddings.append(embeddings_chunk)+ else: # mips or cosine+ distances = -np.dot(embeddings, query_vector)- if cuda_available:- torch.cuda.empty_cache()- elif device.type == "mps":- torch.mps.empty_cache()+ response_payload = distances.flatten().tolist()+ response_bytes = msgpack.packb(+ [response_payload], use_single_float=True+ )+ print(f"Sending distance response with {len(distances)} distances")- hidden = np.vstack(all_embeddings)- print(f"Combined embeddings shape: {hidden.shape}")- else:- hidden = process_batch(texts, node_ids, missing_ids)+ socket.send(response_bytes)+ e2e_end = time.time()+ logger.info(+ f"⏱️ Distance calculation E2E time: {e2e_end - e2e_start:.6f}s"+ )+ continue- # Serialization and response- ser_start = time.time()+ # Standard embedding request (passage ID lookup)+ if (+ not isinstance(request_payload, list)+ or len(request_payload) != 1+ or not isinstance(request_payload[0], list)+ ):+ print(+ f"Error: Invalid MessagePack request format. Expected [[ids...]] or [texts...], got: {type(request_payload)}"+ )+ socket.send(msgpack.packb([[], []]))+ continue+ node_ids = request_payload[0]+ print(f"Request for {len(node_ids)} node embeddings")++ # Look up texts by node IDs+ texts = []+ for nid in node_ids:+ try:+ passage_data = passages.get_passage(str(nid))+ txt = passage_data["text"]+ if not txt:+ raise RuntimeError(f"FATAL: Empty text for passage ID {nid}")+ texts.append(txt)+ except KeyError:+ raise RuntimeError(f"FATAL: Passage with ID {nid} not found")+ except Exception as e:+ print(f"ERROR: Exception looking up passage ID {nid}: {e}")+ raise++ # Process embeddings+ embeddings = compute_embeddings(texts, model_name, mode=embedding_mode)print(- f"DEBUG zmq_server_thread: Final 'hidden' array | Shape: {hidden.shape} | Dtype: {hidden.dtype} | Has NaN/Inf: {np.isnan(hidden).any() or np.isinf(hidden).any()}"+ f"INFO: Computed embeddings for {len(texts)} texts, shape: {embeddings.shape}")- if np.isnan(hidden).any() or np.isinf(hidden).any():++ # Serialization and response+ if np.isnan(embeddings).any() or np.isinf(embeddings).any():print(- f"{RED}!!! ERROR: NaN or Inf detected in final 'hidden' numpy array BEFORE sending! "- f"Requested IDs (sample): {node_ids[:5]}...{RESET}"+ f"{RED}!!! ERROR: NaN or Inf detected in embeddings! Requested IDs: {node_ids[:5]}...{RESET}")assert False- try:- hidden_contiguous_f32 = np.ascontiguousarray(- hidden, dtype=np.float32- )- response_payload = [- list(hidden_contiguous_f32.shape),- hidden_contiguous_f32.flatten().tolist(),- ]- response_bytes = msgpack.packb(- response_payload, use_single_float=True- )- except Exception as pack_error:- print(f"Error packing MessagePack response: {pack_error}")- response_bytes = msgpack.packb([[], []])+ hidden_contiguous_f32 = np.ascontiguousarray(+ embeddings, dtype=np.float32+ )+ response_payload = [+ list(hidden_contiguous_f32.shape),+ hidden_contiguous_f32.flatten().tolist(),+ ]+ response_bytes = msgpack.packb(response_payload, use_single_float=True)socket.send(response_bytes)- ser_end = time.time()-- print(f"Serialize time: {ser_end - ser_start:.6f} seconds")-- if device.type == "cuda":- torch.cuda.synchronize()- elif device.type == "mps":- torch.mps.synchronize()e2e_end = time.time()logger.info(f"⏱️ ZMQ E2E time: {e2e_end - e2e_start:.6f}s")@@ -1093,19 +246,7 @@ def create_hnsw_embedding_server(import tracebacktraceback.print_exc()- try:- socket.send(msgpack.packb([[], []]))- except:- pass-- # Start warmup and server threads- if enable_warmup and len(passages) > 0:- print(f"Warmup enabled: starting warmup thread")- warmup_thread = threading.Thread(target=client_warmup, args=(zmq_port,))- warmup_thread.daemon = True- warmup_thread.start()- else:- print(f"Warmup disabled or no passages available (enable_warmup={enable_warmup}, passages={len(passages)})")+ socket.send(msgpack.packb([[], []]))zmq_thread = threading.Thread(target=zmq_server_thread, daemon=True)zmq_thread.start()@@ -1128,74 +269,30 @@ if __name__ == "__main__":type=str,help="JSON file containing passage ID to text mapping",)- parser.add_argument(- "--embeddings-file",- type=str,- help="Pickle file containing pre-computed embeddings",- )- parser.add_argument("--use-fp16", action="store_true", default=False)- parser.add_argument("--use-int8", action="store_true", default=False)- parser.add_argument("--use-cuda-graphs", action="store_true", default=False)- parser.add_argument(- "--max-batch-size",- type=int,- default=128,- help="Maximum batch size before splitting",- )parser.add_argument("--model-name",type=str,default="sentence-transformers/all-mpnet-base-v2",help="Embedding model name",)- parser.add_argument(- "--custom-max-length",- type=int,- default=None,- help="Override model's default max sequence length",- )parser.add_argument("--distance-metric", type=str, default="mips", help="Distance metric to use")parser.add_argument(- "--embedding-mode",- type=str,- default="sentence-transformers",- choices=["sentence-transformers", "mlx", "openai"],- help="Embedding backend mode"- )- parser.add_argument(- "--use-mlx",- action="store_true",- default=False,- help="Use MLX for model inference (deprecated: use --embedding-mode mlx)",- )- parser.add_argument(- "--disable-warmup",- action="store_true",- default=False,- help="Disable warmup requests on server start",+ "--embedding-mode",+ type=str,+ default="sentence-transformers",+ choices=["sentence-transformers", "openai"],+ help="Embedding backend mode",)args = parser.parse_args()-- # Handle backward compatibility with use_mlx- embedding_mode = args.embedding_mode- if args.use_mlx:- embedding_mode = "mlx"# Create and start the HNSW embedding servercreate_hnsw_embedding_server(passages_file=args.passages_file,- embeddings_file=args.embeddings_file,- use_fp16=args.use_fp16,- use_int8=args.use_int8,- use_cuda_graphs=args.use_cuda_graphs,zmq_port=args.zmq_port,- max_batch_size=args.max_batch_size,model_name=args.model_name,- custom_max_length_param=args.custom_max_length,distance_metric=args.distance_metric,- embedding_mode=embedding_mode,- enable_warmup=not args.disable_warmup,+ embedding_mode=args.embedding_mode,)diff --git a/packages/leann-backend-hnsw/pyproject.toml b/packages/leann-backend-hnsw/pyproject.tomlindex 12df4d6..274f2b4 100644--- a/packages/leann-backend-hnsw/pyproject.toml+++ b/packages/leann-backend-hnsw/pyproject.toml@@ -15,4 +15,8 @@ wheel.packages = ["leann_backend_hnsw"]editable.mode = "redirect"cmake.build-type = "Release"build.verbose = true-build.tool-args = ["-j8"]\ No newline at end of file+build.tool-args = ["-j8"]++# CMake definitions to optimize compilation+[tool.scikit-build.cmake.define]+CMAKE_BUILD_PARALLEL_LEVEL = "8"\ No newline at end of filediff --git a/packages/leann-backend-hnsw/third_party/faiss b/packages/leann-backend-hnsw/third_party/faissindex 2547df4..ff22e2c 160000--- a/packages/leann-backend-hnsw/third_party/faiss+++ b/packages/leann-backend-hnsw/third_party/faiss@@ -1 +1 @@-Subproject commit 2547df4377ae097e2eabc9b019c15135b1fea2b4+Subproject commit ff22e2c86be1784c760265abe146b1ab0db90ebediff --git a/packages/leann-core/pyproject.toml b/packages/leann-core/pyproject.tomlindex 7f64793..08d2b4e 100644--- a/packages/leann-core/pyproject.toml+++ b/packages/leann-core/pyproject.toml@@ -15,5 +15,8 @@ dependencies = ["tqdm>=4.60.0"]+[project.scripts]+leann = "leann.cli:main"+[tool.setuptools.packages.find]where = ["src"]\ No newline at end of filediff --git a/packages/leann-core/src/leann/api.py b/packages/leann-core/src/leann/api.pyindex 832093d..d5f3a53 100644--- a/packages/leann-core/src/leann/api.py+++ b/packages/leann-core/src/leann/api.py@@ -9,9 +9,6 @@ import numpy as npfrom pathlib import Pathfrom typing import List, Dict, Any, Optional, Literalfrom dataclasses import dataclass, field-import uuid-import torch-from .registry import BACKEND_REGISTRYfrom .interface import LeannBackendFactoryInterfacefrom .chat import get_llm@@ -22,7 +19,7 @@ def compute_embeddings(model_name: str,mode: str = "sentence-transformers",use_server: bool = True,- use_mlx: bool = False # Backward compatibility: if True, override mode to 'mlx',+ port: int = 5557,) -> np.ndarray:"""Computes embeddings using different backends.@@ -39,251 +36,60 @@ def compute_embeddings(Returns:numpy array of embeddings"""- # Override mode for backward compatibility- if use_mlx:- mode = "mlx"-- # Auto-detect mode based on model name if not explicitly set- if mode == "sentence-transformers" and model_name.startswith("text-embedding-"):- mode = "openai"-- if mode == "mlx":- return compute_embeddings_mlx(chunks, model_name, batch_size=16)- elif mode == "openai":- return compute_embeddings_openai(chunks, model_name)- elif mode == "sentence-transformers":- return compute_embeddings_sentence_transformers(- chunks, model_name, use_server=use_server- )+ if use_server:+ # Use embedding server (for search/query)+ return compute_embeddings_via_server(chunks, model_name, port=port)else:- raise ValueError(- f"Unsupported embedding mode: {mode}. Supported modes: sentence-transformers, mlx, openai"+ # Use direct computation (for build_index)+ from .embedding_compute import (+ compute_embeddings as compute_embeddings_direct,+ )++ return compute_embeddings_direct(+ chunks,+ model_name,+ mode=mode,)-def compute_embeddings_sentence_transformers(- chunks: List[str], model_name: str, use_server: bool = True+def compute_embeddings_via_server(+ chunks: List[str], model_name: str, port: int) -> np.ndarray:"""Computes embeddings using sentence-transformers.Args:chunks: List of text chunks to embedmodel_name: Name of the sentence transformer model- use_server: If True, use embedding server (good for search). If False, use direct computation (good for build)."""- if not use_server:- print(- f"INFO: Computing embeddings for {len(chunks)} chunks using SentenceTransformer model '{model_name}' (direct)..."- )- return _compute_embeddings_sentence_transformers_direct(chunks, model_name)-print(f"INFO: Computing embeddings for {len(chunks)} chunks using SentenceTransformer model '{model_name}' (via embedding server)...")+ import zmq+ import msgpack+ import numpy as np- # Use embedding server for sentence-transformers too- # This avoids loading the model twice (once in API, once in server)- try:- # Import ZMQ client functionality and server manager- import zmq- import msgpack- import numpy as np- from .embedding_server_manager import EmbeddingServerManager-- # Ensure embedding server is running- port = 5557- server_manager = EmbeddingServerManager(- backend_module_name="leann_backend_hnsw.hnsw_embedding_server"- )-- server_started = server_manager.start_server(- port=port,- model_name=model_name,- embedding_mode="sentence-transformers",- enable_warmup=False,- )-- if not server_started:- raise RuntimeError(f"Failed to start embedding server on port {port}")-- # Connect to embedding server- context = zmq.Context()- socket = context.socket(zmq.REQ)- socket.connect(f"tcp://localhost:{port}")+ # Connect to embedding server+ context = zmq.Context()+ socket = context.socket(zmq.REQ)+ socket.connect(f"tcp://localhost:{port}")- # Send chunks to server for embedding computation- request = chunks- socket.send(msgpack.packb(request))+ # Send chunks to server for embedding computation+ request = chunks+ socket.send(msgpack.packb(request))- # Receive embeddings from server- response = socket.recv()- embeddings_list = msgpack.unpackb(response)+ # Receive embeddings from server+ response = socket.recv()+ embeddings_list = msgpack.unpackb(response)- # Convert back to numpy array- embeddings = np.array(embeddings_list, dtype=np.float32)+ # Convert back to numpy array+ embeddings = np.array(embeddings_list, dtype=np.float32)- socket.close()- context.term()+ socket.close()+ context.term()- return embeddings-- except Exception as e:- # Fallback to direct sentence-transformers if server connection fails- print(- f"Warning: Failed to connect to embedding server, falling back to direct computation: {e}"- )- return _compute_embeddings_sentence_transformers_direct(chunks, model_name)---def _compute_embeddings_sentence_transformers_direct(- chunks: List[str], model_name: str-) -> np.ndarray:- """Direct sentence-transformers computation (fallback)."""- try:- from sentence_transformers import SentenceTransformer- except ImportError as e:- raise RuntimeError(- "sentence-transformers not available. Install with: uv pip install sentence-transformers"- ) from e-- # Load model using sentence-transformers- model = SentenceTransformer(model_name)-- model = model.half()- print(- f"INFO: Computing embeddings for {len(chunks)} chunks using SentenceTransformer model '{model_name}' (direct)..."- )- # use acclerater GPU or MAC GPU-- if torch.cuda.is_available():- model = model.to("cuda")- elif torch.backends.mps.is_available():- model = model.to("mps")-- # Generate embeddings- # give use an warning if OOM here means we need to turn down the batch size- embeddings = model.encode(- chunks, convert_to_numpy=True, show_progress_bar=True, batch_size=16- )-- return embeddings---def compute_embeddings_openai(chunks: List[str], model_name: str) -> np.ndarray:- """Computes embeddings using OpenAI API."""- try:- import openai- import os- except ImportError as e:- raise RuntimeError(- "openai not available. Install with: uv pip install openai"- ) from e-- # Get API key from environment- api_key = os.getenv("OPENAI_API_KEY")- if not api_key:- raise RuntimeError("OPENAI_API_KEY environment variable not set")-- client = openai.OpenAI(api_key=api_key)-- print(- f"INFO: Computing embeddings for {len(chunks)} chunks using OpenAI model '{model_name}'..."- )-- # OpenAI has a limit on batch size and input length- max_batch_size = 100 # Conservative batch size- all_embeddings = []-- try:- from tqdm import tqdm- total_batches = (len(chunks) + max_batch_size - 1) // max_batch_size- batch_range = range(0, len(chunks), max_batch_size)- batch_iterator = tqdm(batch_range, desc="Computing embeddings", unit="batch", total=total_batches)- except ImportError:- # Fallback without progress bar- batch_iterator = range(0, len(chunks), max_batch_size)-- for i in batch_iterator:- batch_chunks = chunks[i:i + max_batch_size]-- try:- response = client.embeddings.create(model=model_name, input=batch_chunks)- batch_embeddings = [embedding.embedding for embedding in response.data]- all_embeddings.extend(batch_embeddings)- except Exception as e:- print(f"ERROR: Failed to get embeddings for batch starting at {i}: {e}")- raise-- embeddings = np.array(all_embeddings, dtype=np.float32)- print(- f"INFO: Generated {len(embeddings)} embeddings with dimension {embeddings.shape[1]}"- )return embeddings-def compute_embeddings_mlx(chunks: List[str], model_name: str, batch_size: int = 16) -> np.ndarray:- """Computes embeddings using an MLX model."""- try:- import mlx.core as mx- from mlx_lm.utils import load- from tqdm import tqdm- except ImportError as e:- raise RuntimeError(- "MLX or related libraries not available. Install with: uv pip install mlx mlx-lm"- ) from e-- print(- f"INFO: Computing embeddings for {len(chunks)} chunks using MLX model '{model_name}' with batch_size={batch_size}..."- )-- # Load model and tokenizer- model, tokenizer = load(model_name)-- # Process chunks in batches with progress bar- all_embeddings = []-- try:- from tqdm import tqdm- batch_iterator = tqdm(range(0, len(chunks), batch_size), desc="Computing embeddings", unit="batch")- except ImportError:- batch_iterator = range(0, len(chunks), batch_size)-- for i in batch_iterator:- batch_chunks = chunks[i:i + batch_size]-- # Tokenize all chunks in the batch- batch_token_ids = []- for chunk in batch_chunks:- token_ids = tokenizer.encode(chunk) # type: ignore- batch_token_ids.append(token_ids)-- # Pad sequences to the same length for batch processing- max_length = max(len(ids) for ids in batch_token_ids)- padded_token_ids = []- for token_ids in batch_token_ids:- # Pad with tokenizer.pad_token_id or 0- padded = token_ids + [0] * (max_length - len(token_ids))- padded_token_ids.append(padded)-- # Convert to MLX array with batch dimension- input_ids = mx.array(padded_token_ids)-- # Get embeddings for the batch- embeddings = model(input_ids)-- # Mean pooling for each sequence in the batch- pooled = embeddings.mean(axis=1) # Shape: (batch_size, hidden_size)-- # Convert batch embeddings to numpy- for j in range(len(batch_chunks)):- pooled_list = pooled[j].tolist() # Convert to list- pooled_numpy = np.array(pooled_list, dtype=np.float32)- all_embeddings.append(pooled_numpy)-- # Stack numpy arrays- return np.stack(all_embeddings)--@dataclassclass SearchResult:id: str@@ -344,14 +150,12 @@ class LeannBuilder:self.dimensions = dimensionsself.embedding_mode = embedding_modeself.backend_kwargs = backend_kwargs- if 'mlx' in self.embedding_model:- self.embedding_mode = "mlx"self.chunks: List[Dict[str, Any]] = []def add_text(self, text: str, metadata: Optional[Dict[str, Any]] = None):if metadata is None:metadata = {}- passage_id = metadata.get("id", str(uuid.uuid4()))+ passage_id = metadata.get("id", str(len(self.chunks)))chunk_data = {"id": passage_id, "text": text, "metadata": metadata}self.chunks.append(chunk_data)@@ -377,10 +181,13 @@ class LeannBuilder:with open(passages_file, "w", encoding="utf-8") as f:try:from tqdm import tqdm- chunk_iterator = tqdm(self.chunks, desc="Writing passages", unit="chunk")++ chunk_iterator = tqdm(+ self.chunks, desc="Writing passages", unit="chunk"+ )except ImportError:chunk_iterator = self.chunks-+for chunk in chunk_iterator:offset = f.tell()json.dump(@@ -398,7 +205,11 @@ class LeannBuilder:pickle.dump(offset_map, f)texts_to_embed = [c["text"] for c in self.chunks]embeddings = compute_embeddings(- texts_to_embed, self.embedding_model, self.embedding_mode, use_server=False+ texts_to_embed,+ self.embedding_model,+ self.embedding_mode,+ use_server=False,+ port=5557,)string_ids = [chunk["id"] for chunk in self.chunks]current_backend_kwargs = {**self.backend_kwargs, "dimensions": self.dimensions}diff --git a/packages/leann-core/src/leann/cli.py b/packages/leann-core/src/leann/cli.pynew file mode 100644index 0000000..854265b--- /dev/null+++ b/packages/leann-core/src/leann/cli.py@@ -0,0 +1,287 @@+#!/usr/bin/env python3+import argparse+import asyncio+import sys+from pathlib import Path+from typing import Optional+import os++from llama_index.core import SimpleDirectoryReader+from llama_index.core.node_parser import SentenceSplitter++from .api import LeannBuilder, LeannSearcher, LeannChat+++class LeannCLI:+ def __init__(self):+ self.indexes_dir = Path.home() / ".leann" / "indexes"+ self.indexes_dir.mkdir(parents=True, exist_ok=True)++ self.node_parser = SentenceSplitter(+ chunk_size=256, chunk_overlap=128, separator=" ", paragraph_separator="\n\n"+ )++ def get_index_path(self, index_name: str) -> str:+ index_dir = self.indexes_dir / index_name+ return str(index_dir / "documents.leann")++ def index_exists(self, index_name: str) -> bool:+ index_dir = self.indexes_dir / index_name+ meta_file = index_dir / "documents.leann.meta.json"+ return meta_file.exists()++ def create_parser(self) -> argparse.ArgumentParser:+ parser = argparse.ArgumentParser(+ prog="leann",+ description="LEANN - Local Enhanced AI Navigation",+ formatter_class=argparse.RawDescriptionHelpFormatter,+ epilog="""+Examples:+ leann build my-docs --docs ./documents # Build index named my-docs+ leann search my-docs "query" # Search in my-docs index+ leann ask my-docs "question" # Ask my-docs index+ leann list # List all stored indexes+ """+ )++ subparsers = parser.add_subparsers(dest="command", help="Available commands")++ # Build command+ build_parser = subparsers.add_parser("build", help="Build document index")+ build_parser.add_argument("index_name", help="Index name")+ build_parser.add_argument("--docs", type=str, required=True, help="Documents directory")+ build_parser.add_argument("--backend", type=str, default="hnsw", choices=["hnsw", "diskann"])+ build_parser.add_argument("--embedding-model", type=str, default="facebook/contriever")+ build_parser.add_argument("--force", "-f", action="store_true", help="Force rebuild")+ build_parser.add_argument("--graph-degree", type=int, default=32)+ build_parser.add_argument("--complexity", type=int, default=64)+ build_parser.add_argument("--num-threads", type=int, default=1)+ build_parser.add_argument("--compact", action="store_true", default=True)+ build_parser.add_argument("--recompute", action="store_true", default=True)++ # Search command+ search_parser = subparsers.add_parser("search", help="Search documents")+ search_parser.add_argument("index_name", help="Index name")+ search_parser.add_argument("query", help="Search query")+ search_parser.add_argument("--top-k", type=int, default=5)+ search_parser.add_argument("--complexity", type=int, default=64)+ search_parser.add_argument("--beam-width", type=int, default=1)+ search_parser.add_argument("--prune-ratio", type=float, default=0.0)+ search_parser.add_argument("--recompute-embeddings", action="store_true")+ search_parser.add_argument("--pruning-strategy", choices=["global", "local", "proportional"], default="global")++ # Ask command+ ask_parser = subparsers.add_parser("ask", help="Ask questions")+ ask_parser.add_argument("index_name", help="Index name")+ ask_parser.add_argument("--llm", type=str, default="ollama", choices=["simulated", "ollama", "hf", "openai"])+ ask_parser.add_argument("--model", type=str, default="qwen3:8b")+ ask_parser.add_argument("--host", type=str, default="http://localhost:11434")+ ask_parser.add_argument("--interactive", "-i", action="store_true")+ ask_parser.add_argument("--top-k", type=int, default=20)+ ask_parser.add_argument("--complexity", type=int, default=32)+ ask_parser.add_argument("--beam-width", type=int, default=1)+ ask_parser.add_argument("--prune-ratio", type=float, default=0.0)+ ask_parser.add_argument("--recompute-embeddings", action="store_true")+ ask_parser.add_argument("--pruning-strategy", choices=["global", "local", "proportional"], default="global")++ # List command+ list_parser = subparsers.add_parser("list", help="List all indexes")++ return parser++ def list_indexes(self):+ print("Stored LEANN indexes:")++ if not self.indexes_dir.exists():+ print("No indexes found. Use 'leann build <name> --docs <dir>' to create one.")+ return++ index_dirs = [d for d in self.indexes_dir.iterdir() if d.is_dir()]++ if not index_dirs:+ print("No indexes found. Use 'leann build <name> --docs <dir>' to create one.")+ return++ print(f"Found {len(index_dirs)} indexes:")+ for i, index_dir in enumerate(index_dirs, 1):+ index_name = index_dir.name+ status = "✓" if self.index_exists(index_name) else "✗"++ print(f" {i}. {index_name} [{status}]")+ if self.index_exists(index_name):+ meta_file = index_dir / "documents.leann.meta.json"+ size_mb = sum(f.stat().st_size for f in index_dir.iterdir() if f.is_file()) / (1024 * 1024)+ print(f" Size: {size_mb:.1f} MB")++ if index_dirs:+ example_name = index_dirs[0].name+ print(f"\nUsage:")+ print(f" leann search {example_name} \"your query\"")+ print(f" leann ask {example_name} --interactive")++ def load_documents(self, docs_dir: str):+ print(f"Loading documents from {docs_dir}...")++ documents = SimpleDirectoryReader(+ docs_dir,+ recursive=True,+ encoding="utf-8",+ required_exts=[".pdf", ".txt", ".md", ".docx"],+ ).load_data(show_progress=True)++ all_texts = []+ for doc in documents:+ nodes = self.node_parser.get_nodes_from_documents([doc])+ for node in nodes:+ all_texts.append(node.get_content())++ print(f"Loaded {len(documents)} documents, {len(all_texts)} chunks")+ return all_texts++ async def build_index(self, args):+ docs_dir = args.docs+ index_name = args.index_name+ index_dir = self.indexes_dir / index_name+ index_path = self.get_index_path(index_name)++ if index_dir.exists() and not args.force:+ print(f"Index '{index_name}' already exists. Use --force to rebuild.")+ return++ all_texts = self.load_documents(docs_dir)+ if not all_texts:+ print("No documents found")+ return++ index_dir.mkdir(parents=True, exist_ok=True)++ print(f"Building index '{index_name}' with {args.backend} backend...")++ builder = LeannBuilder(+ backend_name=args.backend,+ embedding_model=args.embedding_model,+ graph_degree=args.graph_degree,+ complexity=args.complexity,+ is_compact=args.compact,+ is_recompute=args.recompute,+ num_threads=args.num_threads,+ )++ for chunk_text in all_texts:+ builder.add_text(chunk_text)++ builder.build_index(index_path)+ print(f"Index built at {index_path}")++ async def search_documents(self, args):+ index_name = args.index_name+ query = args.query+ index_path = self.get_index_path(index_name)++ if not self.index_exists(index_name):+ print(f"Index '{index_name}' not found. Use 'leann build {index_name} --docs <dir>' to create it.")+ return++ searcher = LeannSearcher(index_path=index_path)+ results = searcher.search(+ query,+ top_k=args.top_k,+ complexity=args.complexity,+ beam_width=args.beam_width,+ prune_ratio=args.prune_ratio,+ recompute_embeddings=args.recompute_embeddings,+ pruning_strategy=args.pruning_strategy+ )++ print(f"Search results for '{query}' (top {len(results)}):")+ for i, result in enumerate(results, 1):+ print(f"{i}. Score: {result.score:.3f}")+ print(f" {result.text[:200]}...")+ print()++ async def ask_questions(self, args):+ index_name = args.index_name+ index_path = self.get_index_path(index_name)++ if not self.index_exists(index_name):+ print(f"Index '{index_name}' not found. Use 'leann build {index_name} --docs <dir>' to create it.")+ return++ print(f"Starting chat with index '{index_name}'...")+ print(f"Using {args.model} ({args.llm})")++ llm_config = {"type": args.llm, "model": args.model}+ if args.llm == "ollama":+ llm_config["host"] = args.host++ chat = LeannChat(index_path=index_path, llm_config=llm_config)++ if args.interactive:+ print("LEANN Assistant ready! Type 'quit' to exit")+ print("=" * 40)++ while True:+ user_input = input("\nYou: ").strip()+ if user_input.lower() in ['quit', 'exit', 'q']:+ print("Goodbye!")+ break++ if not user_input:+ continue++ response = chat.ask(+ user_input,+ top_k=args.top_k,+ complexity=args.complexity,+ beam_width=args.beam_width,+ prune_ratio=args.prune_ratio,+ recompute_embeddings=args.recompute_embeddings,+ pruning_strategy=args.pruning_strategy+ )+ print(f"LEANN: {response}")+ else:+ query = input("Enter your question: ").strip()+ if query:+ response = chat.ask(+ query,+ top_k=args.top_k,+ complexity=args.complexity,+ beam_width=args.beam_width,+ prune_ratio=args.prune_ratio,+ recompute_embeddings=args.recompute_embeddings,+ pruning_strategy=args.pruning_strategy+ )+ print(f"LEANN: {response}")++ async def run(self, args=None):+ parser = self.create_parser()++ if args is None:+ args = parser.parse_args()++ if not args.command:+ parser.print_help()+ return++ if args.command == "list":+ self.list_indexes()+ elif args.command == "build":+ await self.build_index(args)+ elif args.command == "search":+ await self.search_documents(args)+ elif args.command == "ask":+ await self.ask_questions(args)+ else:+ parser.print_help()+++def main():+ import dotenv+ dotenv.load_dotenv()++ cli = LeannCLI()+ asyncio.run(cli.run())+++if __name__ == "__main__":+ main()\ No newline at end of filediff --git a/packages/leann-core/src/leann/embedding_compute.py b/packages/leann-core/src/leann/embedding_compute.pynew file mode 100644index 0000000..20cef9f--- /dev/null+++ b/packages/leann-core/src/leann/embedding_compute.py@@ -0,0 +1,272 @@+"""+Unified embedding computation module+Consolidates all embedding computation logic using SentenceTransformer+Preserves all optimization parameters to ensure performance+"""++import numpy as np+import torch+from typing import List+import logging++logger = logging.getLogger(__name__)+++def compute_embeddings(+ texts: List[str], model_name: str, mode: str = "sentence-transformers"+) -> np.ndarray:+ """+ Unified embedding computation entry point++ Args:+ texts: List of texts to compute embeddings for+ model_name: Model name+ mode: Computation mode ('sentence-transformers', 'openai', 'mlx')++ Returns:+ Normalized embeddings array, shape: (len(texts), embedding_dim)+ """+ if mode == "sentence-transformers":+ return compute_embeddings_sentence_transformers(texts, model_name)+ elif mode == "openai":+ return compute_embeddings_openai(texts, model_name)+ elif mode == "mlx":+ return compute_embeddings_mlx(texts, model_name)+ else:+ raise ValueError(f"Unsupported embedding mode: {mode}")+++def compute_embeddings_sentence_transformers(+ texts: List[str],+ model_name: str,+ use_fp16: bool = True,+ device: str = "auto",+ batch_size: int = 32,+) -> np.ndarray:+ """+ Compute embeddings using SentenceTransformer+ Preserves all optimization parameters to ensure consistency with original embedding_server++ Args:+ texts: List of texts to compute embeddings for+ model_name: SentenceTransformer model name+ use_fp16: Whether to use FP16 precision+ device: Device selection ('auto', 'cuda', 'mps', 'cpu')+ batch_size: Batch size for processing++ Returns:+ Normalized embeddings array, shape: (len(texts), embedding_dim)+ """+ print(+ f"INFO: Computing embeddings for {len(texts)} texts using SentenceTransformer, model: '{model_name}'"+ )++ from sentence_transformers import SentenceTransformer++ # Auto-detect device+ if device == "auto":+ if torch.cuda.is_available():+ device = "cuda"+ elif hasattr(torch.backends, "mps") and torch.backends.mps.is_available():+ device = "mps"+ else:+ device = "cpu"++ print(f"INFO: Using device: {device}")++ # Prepare model and tokenizer optimization parameters (consistent with original embedding_server)+ model_kwargs = {+ "torch_dtype": torch.float16 if use_fp16 else torch.float32,+ "low_cpu_mem_usage": True,+ "_fast_init": True, # Skip weight initialization checks for faster loading+ }++ tokenizer_kwargs = {+ "use_fast": True, # Use fast tokenizer for better runtime performance+ }++ # Load SentenceTransformer (try local first, then network)+ print(f"INFO: Loading SentenceTransformer model: {model_name}")++ try:+ # Try local loading (avoid network delays)+ model_kwargs["local_files_only"] = True+ tokenizer_kwargs["local_files_only"] = True++ model = SentenceTransformer(+ model_name,+ device=device,+ model_kwargs=model_kwargs,+ tokenizer_kwargs=tokenizer_kwargs,+ local_files_only=True,+ )+ print("✅ Model loaded successfully! (local + optimized)")+ except Exception as e:+ print(f"Local loading failed ({e}), trying network download...")+ # Fallback to network loading+ model_kwargs["local_files_only"] = False+ tokenizer_kwargs["local_files_only"] = False++ model = SentenceTransformer(+ model_name,+ device=device,+ model_kwargs=model_kwargs,+ tokenizer_kwargs=tokenizer_kwargs,+ local_files_only=False,+ )+ print("✅ Model loaded successfully! (network + optimized)")++ # Apply additional optimizations (if supported)+ if use_fp16 and device in ["cuda", "mps"]:+ try:+ model = model.half()+ model = torch.compile(model)+ print(f"✅ Using FP16 precision and compile optimization: {model_name}")+ except Exception as e:+ print(+ f"FP16 or compile optimization failed, continuing with default settings: {e}"+ )++ # Compute embeddings (using SentenceTransformer's optimized implementation)+ print("INFO: Starting embedding computation...")++ embeddings = model.encode(+ texts,+ batch_size=batch_size,+ show_progress_bar=False, # Don't show progress bar in server environment+ convert_to_numpy=True,+ normalize_embeddings=False, # Keep consistent with original API behavior+ device=device,+ )++ print(+ f"INFO: Generated {len(embeddings)} embeddings, dimension: {embeddings.shape[1]}"+ )++ # Validate results+ if np.isnan(embeddings).any() or np.isinf(embeddings).any():+ raise RuntimeError(+ f"Detected NaN or Inf values in embeddings, model: {model_name}"+ )++ return embeddings+++def compute_embeddings_openai(texts: List[str], model_name: str) -> np.ndarray:+ """Compute embeddings using OpenAI API"""+ try:+ import openai+ import os+ except ImportError as e:+ raise ImportError(f"OpenAI package not installed: {e}")++ api_key = os.getenv("OPENAI_API_KEY")+ if not api_key:+ raise RuntimeError("OPENAI_API_KEY environment variable not set")++ client = openai.OpenAI(api_key=api_key)++ print(+ f"INFO: Computing embeddings for {len(texts)} texts using OpenAI API, model: '{model_name}'"+ )++ # OpenAI has limits on batch size and input length+ max_batch_size = 100 # Conservative batch size+ all_embeddings = []++ try:+ from tqdm import tqdm++ total_batches = (len(texts) + max_batch_size - 1) // max_batch_size+ batch_range = range(0, len(texts), max_batch_size)+ batch_iterator = tqdm(+ batch_range, desc="Computing embeddings", unit="batch", total=total_batches+ )+ except ImportError:+ # Fallback when tqdm is not available+ batch_iterator = range(0, len(texts), max_batch_size)++ for i in batch_iterator:+ batch_texts = texts[i : i + max_batch_size]++ try:+ response = client.embeddings.create(model=model_name, input=batch_texts)+ batch_embeddings = [embedding.embedding for embedding in response.data]+ all_embeddings.extend(batch_embeddings)+ except Exception as e:+ print(f"ERROR: Batch {i} failed: {e}")+ raise++ embeddings = np.array(all_embeddings, dtype=np.float32)+ print(+ f"INFO: Generated {len(embeddings)} embeddings, dimension: {embeddings.shape[1]}"+ )+ return embeddings+++def compute_embeddings_mlx(+ chunks: List[str], model_name: str, batch_size: int = 16+) -> np.ndarray:+ """Computes embeddings using an MLX model."""+ try:+ import mlx.core as mx+ from mlx_lm.utils import load+ from tqdm import tqdm+ except ImportError as e:+ raise RuntimeError(+ "MLX or related libraries not available. Install with: uv pip install mlx mlx-lm"+ ) from e++ print(+ f"INFO: Computing embeddings for {len(chunks)} chunks using MLX model '{model_name}' with batch_size={batch_size}..."+ )++ # Load model and tokenizer+ model, tokenizer = load(model_name)++ # Process chunks in batches with progress bar+ all_embeddings = []++ try:+ from tqdm import tqdm++ batch_iterator = tqdm(+ range(0, len(chunks), batch_size), desc="Computing embeddings", unit="batch"+ )+ except ImportError:+ batch_iterator = range(0, len(chunks), batch_size)++ for i in batch_iterator:+ batch_chunks = chunks[i : i + batch_size]++ # Tokenize all chunks in the batch+ batch_token_ids = []+ for chunk in batch_chunks:+ token_ids = tokenizer.encode(chunk) # type: ignore+ batch_token_ids.append(token_ids)++ # Pad sequences to the same length for batch processing+ max_length = max(len(ids) for ids in batch_token_ids)+ padded_token_ids = []+ for token_ids in batch_token_ids:+ # Pad with tokenizer.pad_token_id or 0+ padded = token_ids + [0] * (max_length - len(token_ids))+ padded_token_ids.append(padded)++ # Convert to MLX array with batch dimension+ input_ids = mx.array(padded_token_ids)++ # Get embeddings for the batch+ embeddings = model(input_ids)++ # Mean pooling for each sequence in the batch+ pooled = embeddings.mean(axis=1) # Shape: (batch_size, hidden_size)++ # Convert batch embeddings to numpy+ for j in range(len(batch_chunks)):+ pooled_list = pooled[j].tolist() # Convert to list+ pooled_numpy = np.array(pooled_list, dtype=np.float32)+ all_embeddings.append(pooled_numpy)++ # Stack numpy arrays+ return np.stack(all_embeddings)diff --git a/packages/leann-core/src/leann/embedding_server_manager.py b/packages/leann-core/src/leann/embedding_server_manager.pyindex 2022262..6a44160 100644--- a/packages/leann-core/src/leann/embedding_server_manager.py+++ b/packages/leann-core/src/leann/embedding_server_manager.py@@ -4,11 +4,10 @@ import atexitimport socketimport subprocessimport sys-import zmq-import msgpackfrom pathlib import Pathfrom typing import Optionalimport select+import psutildef _check_port(port: int) -> bool:@@ -17,151 +16,135 @@ def _check_port(port: int) -> bool:return s.connect_ex(("localhost", port)) == 0-def _check_server_meta_path(port: int, expected_meta_path: str) -> bool:+def _check_process_matches_config(+ port: int, expected_model: str, expected_passages_file: str+) -> bool:"""- Check if the existing server on the port is using the correct meta file.- Returns True if the server has the right meta path, False otherwise.+ Check if the process using the port matches our expected model and passages file.+ Returns True if matches, False otherwise."""try:- context = zmq.Context()- socket = context.socket(zmq.REQ)- socket.setsockopt(zmq.RCVTIMEO, 3000) # 3 second timeout- socket.connect(f"tcp://localhost:{port}")-- # Send a special control message to query the server's meta path- control_request = ["__QUERY_META_PATH__"]- request_bytes = msgpack.packb(control_request)- socket.send(request_bytes)-- # Wait for response- response_bytes = socket.recv()- response = msgpack.unpackb(response_bytes)-- socket.close()- context.term()-- # Check if the response contains the meta path and if it matches- if isinstance(response, list) and len(response) > 0:- server_meta_path = response[0]- # Normalize paths for comparison- expected_path = Path(expected_meta_path).resolve()- server_path = Path(server_meta_path).resolve() if server_meta_path else None- return server_path == expected_path+ for proc in psutil.process_iter(["pid", "cmdline"]):+ if not _is_process_listening_on_port(proc, port):+ continue+ cmdline = proc.info["cmdline"]+ if not cmdline:+ continue++ return _check_cmdline_matches_config(+ cmdline, port, expected_model, expected_passages_file+ )++ print(f"DEBUG: No process found listening on port {port}")return Falseexcept Exception as e:- print(f"WARNING: Could not query server meta path on port {port}: {e}")+ print(f"WARNING: Could not check process on port {port}: {e}")return False-def _update_server_meta_path(port: int, new_meta_path: str) -> bool:- """- Send a control message to update the server's meta path.- Returns True if successful, False otherwise.- """+def _is_process_listening_on_port(proc, port: int) -> bool:+ """Check if a process is listening on the given port."""try:- context = zmq.Context()- socket = context.socket(zmq.REQ)- socket.setsockopt(zmq.RCVTIMEO, 5000) # 5 second timeout- socket.connect(f"tcp://localhost:{port}")-- # Send a control message to update the meta path- control_request = ["__UPDATE_META_PATH__", new_meta_path]- request_bytes = msgpack.packb(control_request)- socket.send(request_bytes)-- # Wait for response- response_bytes = socket.recv()- response = msgpack.unpackb(response_bytes)-- socket.close()- context.term()-- # Check if the update was successful- if isinstance(response, list) and len(response) > 0:- return response[0] == "SUCCESS"-+ connections = proc.net_connections()+ for conn in connections:+ if conn.laddr.port == port and conn.status == psutil.CONN_LISTEN:+ return Truereturn False-- except Exception as e:- print(f"ERROR: Could not update server meta path on port {port}: {e}")+ except (psutil.NoSuchProcess, psutil.AccessDenied, psutil.ZombieProcess):return False-def _check_server_model(port: int, expected_model: str) -> bool:- """- Check if the existing server on the port is using the correct embedding model.- Returns True if the server has the right model, False otherwise.- """- try:- context = zmq.Context()- socket = context.socket(zmq.REQ)- socket.setsockopt(zmq.RCVTIMEO, 3000) # 3 second timeout- socket.connect(f"tcp://localhost:{port}")+def _check_cmdline_matches_config(+ cmdline: list, port: int, expected_model: str, expected_passages_file: str+) -> bool:+ """Check if command line matches our expected configuration."""+ cmdline_str = " ".join(cmdline)+ print(f"DEBUG: Found process on port {port}: {cmdline_str}")++ # Check if it's our embedding server+ is_embedding_server = any(+ server_type in cmdline_str+ for server_type in [+ "embedding_server",+ "leann_backend_diskann.embedding_server",+ "leann_backend_hnsw.hnsw_embedding_server",+ ]+ )++ if not is_embedding_server:+ print(f"DEBUG: Process on port {port} is not our embedding server")+ return False- # Send a special control message to query the server's model- control_request = ["__QUERY_MODEL__"]- request_bytes = msgpack.packb(control_request)- socket.send(request_bytes)+ # Check model name+ model_matches = _check_model_in_cmdline(cmdline, expected_model)- # Wait for response- response_bytes = socket.recv()- response = msgpack.unpackb(response_bytes)+ # Check passages file if provided+ passages_matches = _check_passages_in_cmdline(cmdline, expected_passages_file)- socket.close()- context.term()+ result = model_matches and passages_matches+ print(+ f"DEBUG: model_matches: {model_matches}, passages_matches: {passages_matches}, overall: {result}"+ )+ return result- # Check if the response contains the model name and if it matches- if isinstance(response, list) and len(response) > 0:- server_model = response[0]- return server_model == expected_model+def _check_model_in_cmdline(cmdline: list, expected_model: str) -> bool:+ """Check if the command line contains the expected model."""+ if "--model-name" not in cmdline:return False- except Exception as e:- print(f"WARNING: Could not query server model on port {port}: {e}")+ model_idx = cmdline.index("--model-name")+ if model_idx + 1 >= len(cmdline):return False+ actual_model = cmdline[model_idx + 1]+ return actual_model == expected_model-def _update_server_model(port: int, new_model: str) -> bool:- """- Send a control message to update the server's embedding model.- Returns True if successful, False otherwise.- """- try:- context = zmq.Context()- socket = context.socket(zmq.REQ)- socket.setsockopt(zmq.RCVTIMEO, 30000) # 30 second timeout for model loading- socket.setsockopt(zmq.SNDTIMEO, 5000) # 5 second timeout for sending- socket.connect(f"tcp://localhost:{port}")- # Send a control message to update the model- control_request = ["__UPDATE_MODEL__", new_model]- request_bytes = msgpack.packb(control_request)- socket.send(request_bytes)+def _check_passages_in_cmdline(cmdline: list, expected_passages_file: str) -> bool:+ """Check if the command line contains the expected passages file."""+ if "--passages-file" not in cmdline:+ return False # Expected but not found- # Wait for response- response_bytes = socket.recv()- response = msgpack.unpackb(response_bytes)+ passages_idx = cmdline.index("--passages-file")+ if passages_idx + 1 >= len(cmdline):+ return False- socket.close()- context.term()+ actual_passages = cmdline[passages_idx + 1]+ expected_path = Path(expected_passages_file).resolve()+ actual_path = Path(actual_passages).resolve()+ return actual_path == expected_path- # Check if the update was successful- if isinstance(response, list) and len(response) > 0:- return response[0] == "SUCCESS"- return False+def _find_compatible_port_or_next_available(+ start_port: int, model_name: str, passages_file: str, max_attempts: int = 100+) -> tuple[int, bool]:+ """+ Find a port that either has a compatible server or is available.+ Returns (port, is_compatible) where is_compatible indicates if we found a matching server.+ """+ for port in range(start_port, start_port + max_attempts):+ if not _check_port(port):+ # Port is available+ return port, False- except Exception as e:- print(f"ERROR: Could not update server model on port {port}: {e}")- return False+ # Port is in use, check if it's compatible+ if _check_process_matches_config(port, model_name, passages_file):+ print(f"✅ Found compatible server on port {port}")+ return port, True+ else:+ print(f"⚠️ Port {port} has incompatible server, trying next port...")++ raise RuntimeError(+ f"Could not find compatible or available port in range {start_port}-{start_port + max_attempts}"+ )class EmbeddingServerManager:"""- A generic manager for handling the lifecycle of a backend-specific embedding server process.+ A simplified manager for embedding server processes that avoids complex update mechanisms."""def __init__(self, backend_module_name: str):@@ -175,210 +158,162 @@ class EmbeddingServerManager:self.backend_module_name = backend_module_nameself.server_process: Optional[subprocess.Popen] = Noneself.server_port: Optional[int] = None- atexit.register(self.stop_server)-- def start_server(self, port: int, model_name: str, embedding_mode: str = "sentence-transformers", **kwargs) -> bool:+ self._atexit_registered = False++ def start_server(+ self,+ port: int,+ model_name: str,+ embedding_mode: str = "sentence-transformers",+ **kwargs,+ ) -> tuple[bool, int]:"""Starts the embedding server process.Args:- port (int): The ZMQ port for the server.+ port (int): The preferred ZMQ port for the server.model_name (str): The name of the embedding model to use.- **kwargs: Additional arguments for the server (e.g., passages_file, distance_metric, enable_warmup).+ **kwargs: Additional arguments for the server.Returns:- bool: True if the server is started successfully or already running, False otherwise.+ tuple[bool, int]: (success, actual_port_used)"""- if self.server_process and self.server_process.poll() is None:- # Even if we have a running process, check if model/meta path match- if self.server_port is not None:- port_in_use = _check_port(self.server_port)- if port_in_use:- print(- f"INFO: Checking compatibility of existing server process (PID {self.server_process.pid})"- )+ passages_file = kwargs.get("passages_file")+ assert isinstance(passages_file, str), "passages_file must be a string"- # Check model compatibility- model_matches = _check_server_model(self.server_port, model_name)- if model_matches:- print(- f"✅ Existing server already using correct model: {model_name}"- )-- # Still check meta path if provided- passages_file = kwargs.get("passages_file")- if passages_file and str(passages_file).endswith(- ".meta.json"- ):- meta_matches = _check_server_meta_path(- self.server_port, str(passages_file)- )- if not meta_matches:- print("⚠️ Updating meta path to: {passages_file}")- _update_server_meta_path(- self.server_port, str(passages_file)- )-- return True- else:- print(- f"⚠️ Existing server has different model. Attempting to update to: {model_name}"- )- if not _update_server_model(self.server_port, model_name):- print(- "❌ Failed to update existing server model. Restarting server..."- )- self.stop_server()- # Continue to start new server below- else:- print(- f"✅ Successfully updated existing server model to: {model_name}"- )-- # Also check meta path if provided- passages_file = kwargs.get("passages_file")- if passages_file and str(passages_file).endswith(- ".meta.json"- ):- meta_matches = _check_server_meta_path(- self.server_port, str(passages_file)- )- if not meta_matches:- print("⚠️ Updating meta path to: {passages_file}")- _update_server_meta_path(- self.server_port, str(passages_file)- )-- return True- else:- # Server process exists but port not responding - restart- print("⚠️ Server process exists but not responding. Restarting...")- self.stop_server()- # Continue to start new server below- else:- # No port stored - restart- print("⚠️ No port information stored. Restarting server...")- self.stop_server()- # Continue to start new server below-- if _check_port(port):- # Port is in use, check if it's using the correct meta file and model- passages_file = kwargs.get("passages_file")-- print(f"INFO: Port {port} is in use. Checking server compatibility...")-- # Check model compatibility first- model_matches = _check_server_model(port, model_name)- if model_matches:- print(- f"✅ Existing server on port {port} is using correct model: {model_name}"- )- else:- print(- f"⚠️ Existing server on port {port} has different model. Attempting to update to: {model_name}"- )- if not _update_server_model(port, model_name):- raise RuntimeError(- f"❌ Failed to update server model to {model_name}. Consider using a different port."- )- print(f"✅ Successfully updated server model to: {model_name}")-- # Check meta path compatibility if provided- if passages_file and str(passages_file).endswith(".meta.json"):- meta_matches = _check_server_meta_path(port, str(passages_file))- if not meta_matches:- print(- f"⚠️ Existing server on port {port} has different meta path. Attempting to update..."- )- if not _update_server_meta_path(port, str(passages_file)):- raise RuntimeError(- "❌ Failed to update server meta path. This may cause data synchronization issues."- )- print(- f"✅ Successfully updated server meta path to: {passages_file}"- )- else:- print(- f"✅ Existing server on port {port} is using correct meta path: {passages_file}"- )-- print(f"✅ Server on port {port} is compatible and ready to use.")- return True-- print(- f"INFO: Starting session-level embedding server for '{self.backend_module_name}'..."- )+ # Check if we have a compatible running server+ if self._has_compatible_running_server(model_name, passages_file):+ assert self.server_port is not None, (+ "a compatible running server should set server_port"+ )+ return True, self.server_port+ # Find available port (compatible or free)try:- command = [- sys.executable,- "-m",- self.backend_module_name,- "--zmq-port",- str(port),- "--model-name",- model_name,- ]-- # Add extra arguments for specific backends- if "passages_file" in kwargs and kwargs["passages_file"]:- command.extend(["--passages-file", str(kwargs["passages_file"])])- # if "distance_metric" in kwargs and kwargs["distance_metric"]:- # command.extend(["--distance-metric", kwargs["distance_metric"]])- if embedding_mode != "sentence-transformers":- command.extend(["--embedding-mode", embedding_mode])- if "enable_warmup" in kwargs and not kwargs["enable_warmup"]:- command.extend(["--disable-warmup"])-- project_root = Path(__file__).parent.parent.parent.parent.parent- print(f"INFO: Running command from project root: {project_root}")- print(f"INFO: Command: {' '.join(command)}") # Debug: show actual command-- self.server_process = subprocess.Popen(- command,- cwd=project_root,- stdout=subprocess.PIPE,- stderr=subprocess.STDOUT, # Merge stderr into stdout for easier monitoring- text=True,- encoding="utf-8",- bufsize=1, # Line buffered- universal_newlines=True,+ actual_port, is_compatible = _find_compatible_port_or_next_available(+ port, model_name, passages_file)- self.server_port = port- print(f"INFO: Server process started with PID: {self.server_process.pid}")-- max_wait, wait_interval = 120, 0.5- for _ in range(int(max_wait / wait_interval)):- if _check_port(port):- print("✅ Embedding server is up and ready for this session.")- log_thread = threading.Thread(target=self._log_monitor, daemon=True)- log_thread.start()- return True- if self.server_process.poll() is not None:- print(- "❌ ERROR: Server process terminated unexpectedly during startup."- )- self._print_recent_output()- return False- time.sleep(wait_interval)+ except RuntimeError as e:+ print(f"❌ {e}")+ return False, port++ if is_compatible:+ print(f"✅ Using existing compatible server on port {actual_port}")+ self.server_port = actual_port+ self.server_process = None # We don't own this process+ return True, actual_port++ if actual_port != port:+ print(f"⚠️ Using port {actual_port} instead of {port}")++ # Start new server+ return self._start_new_server(actual_port, model_name, embedding_mode, **kwargs)++ def _has_compatible_running_server(+ self, model_name: str, passages_file: str+ ) -> bool:+ """Check if we have a compatible running server."""+ if not (+ self.server_process+ and self.server_process.poll() is None+ and self.server_port+ ):+ return False+ if _check_process_matches_config(self.server_port, model_name, passages_file):print(- f"❌ ERROR: Server process failed to start listening within {max_wait} seconds."+ f"✅ Existing server process (PID {self.server_process.pid}) is compatible")- self.stop_server()- return False+ return True++ print("⚠️ Existing server process is incompatible. Should start a new server.")+ return False+ def _start_new_server(+ self, port: int, model_name: str, embedding_mode: str, **kwargs+ ) -> tuple[bool, int]:+ """Start a new embedding server on the given port."""+ print(f"INFO: Starting embedding server on port {port}...")++ command = self._build_server_command(port, model_name, embedding_mode, **kwargs)++ try:+ self._launch_server_process(command, port)+ return self._wait_for_server_ready(port)except Exception as e:- print(f"❌ ERROR: Failed to start embedding server process: {e}")- return False+ print(f"❌ ERROR: Failed to start embedding server: {e}")+ return False, port++ def _build_server_command(+ self, port: int, model_name: str, embedding_mode: str, **kwargs+ ) -> list:+ """Build the command to start the embedding server."""+ command = [+ sys.executable,+ "-m",+ self.backend_module_name,+ "--zmq-port",+ str(port),+ "--model-name",+ model_name,+ ]++ if kwargs.get("passages_file"):+ command.extend(["--passages-file", str(kwargs["passages_file"])])+ if embedding_mode != "sentence-transformers":+ command.extend(["--embedding-mode", embedding_mode])++ return command++ def _launch_server_process(self, command: list, port: int) -> None:+ """Launch the server process."""+ project_root = Path(__file__).parent.parent.parent.parent.parent+ print(f"INFO: Command: {' '.join(command)}")++ self.server_process = subprocess.Popen(+ command,+ cwd=project_root,+ stdout=subprocess.PIPE,+ stderr=subprocess.STDOUT,+ text=True,+ encoding="utf-8",+ bufsize=1,+ universal_newlines=True,+ )+ self.server_port = port+ print(f"INFO: Server process started with PID: {self.server_process.pid}")++ # Register atexit callback only when we actually start a process+ if not self._atexit_registered:+ # Use a lambda to avoid issues with bound methods+ atexit.register(lambda: self.stop_server() if self.server_process else None)+ self._atexit_registered = True++ def _wait_for_server_ready(self, port: int) -> tuple[bool, int]:+ """Wait for the server to be ready."""+ max_wait, wait_interval = 120, 0.5+ for _ in range(int(max_wait / wait_interval)):+ if _check_port(port):+ print("✅ Embedding server is ready!")+ threading.Thread(target=self._log_monitor, daemon=True).start()+ return True, port++ if self.server_process.poll() is not None:+ print("❌ ERROR: Server terminated during startup.")+ self._print_recent_output()+ return False, port++ time.sleep(wait_interval)++ print(f"❌ ERROR: Server failed to start within {max_wait} seconds.")+ self.stop_server()+ return False, portdef _print_recent_output(self):"""Print any recent output from the server process."""if not self.server_process or not self.server_process.stdout:returntry:- # Read any available output-if select.select([self.server_process.stdout], [], [], 0)[0]:output = self.server_process.stdout.read()if output:@@ -404,17 +339,26 @@ class EmbeddingServerManager:def stop_server(self):"""Stops the embedding server process if it's running."""- if self.server_process and self.server_process.poll() is None:+ if not self.server_process:+ return++ if self.server_process.poll() is not None:+ # Process already terminated+ self.server_process = None+ return++ print(+ f"INFO: Terminating server process (PID: {self.server_process.pid}) for backend {self.backend_module_name}..."+ )+ self.server_process.terminate()++ try:+ self.server_process.wait(timeout=5)+ print(f"INFO: Server process {self.server_process.pid} terminated.")+ except subprocess.TimeoutExpired:print(- f"INFO: Terminating session server process (PID: {self.server_process.pid})..."+ f"WARNING: Server process {self.server_process.pid} did not terminate gracefully, killing it.")- self.server_process.terminate()- try:- self.server_process.wait(timeout=5)- print("INFO: Server process terminated.")- except subprocess.TimeoutExpired:- print(- "WARNING: Server process did not terminate gracefully, killing it."- )- self.server_process.kill()+ self.server_process.kill()+self.server_process = Nonediff --git a/packages/leann-core/src/leann/registry.py b/packages/leann-core/src/leann/registry.pyindex bda797a..043a784 100644--- a/packages/leann-core/src/leann/registry.py+++ b/packages/leann-core/src/leann/registry.py@@ -7,30 +7,37 @@ import importlib.metadataif TYPE_CHECKING:from leann.interface import LeannBackendFactoryInterface-BACKEND_REGISTRY: Dict[str, 'LeannBackendFactoryInterface'] = {}+BACKEND_REGISTRY: Dict[str, "LeannBackendFactoryInterface"] = {}+def register_backend(name: str):"""A decorator to register a new backend class."""+def decorator(cls):print(f"INFO: Registering backend '{name}'")BACKEND_REGISTRY[name] = clsreturn cls+return decorator+def autodiscover_backends():"""Automatically discovers and imports all 'leann-backend-*' packages."""- print("INFO: Starting backend auto-discovery...")+ # print("INFO: Starting backend auto-discovery...")discovered_backends = []for dist in importlib.metadata.distributions():- dist_name = dist.metadata['name']- if dist_name.startswith('leann-backend-'):- backend_module_name = dist_name.replace('-', '_')+ dist_name = dist.metadata["name"]+ if dist_name.startswith("leann-backend-"):+ backend_module_name = dist_name.replace("-", "_")discovered_backends.append(backend_module_name)-- for backend_module_name in sorted(discovered_backends): # sort for deterministic loading++ for backend_module_name in sorted(+ discovered_backends+ ): # sort for deterministic loadingtry:importlib.import_module(backend_module_name)# Registration message is printed by the decoratorexcept ImportError as e:- print(f"WARN: Could not import backend module '{backend_module_name}': {e}")- print("INFO: Backend auto-discovery finished.")\ No newline at end of file+ # print(f"WARN: Could not import backend module '{backend_module_name}': {e}")+ pass+ # print("INFO: Backend auto-discovery finished.")diff --git a/packages/leann-core/src/leann/searcher_base.py b/packages/leann-core/src/leann/searcher_base.pyindex 0f40a85..dfa6c2d 100644--- a/packages/leann-core/src/leann/searcher_base.py+++ b/packages/leann-core/src/leann/searcher_base.py@@ -43,8 +43,6 @@ class BaseSearcher(LeannBackendSearcherInterface, ABC):"WARNING: embedding_model not found in meta.json. Recompute will fail.")- self.label_map = self._load_label_map()-self.embedding_server_manager = EmbeddingServerManager(backend_module_name=backend_module_name)@@ -58,17 +56,9 @@ class BaseSearcher(LeannBackendSearcherInterface, ABC):with open(meta_path, "r", encoding="utf-8") as f:return json.load(f)- def _load_label_map(self) -> Dict[int, str]:- """Loads the mapping from integer IDs to string IDs."""- label_map_file = self.index_dir / "leann.labels.map"- if not label_map_file.exists():- raise FileNotFoundError(f"Label map file not found: {label_map_file}")- with open(label_map_file, "rb") as f:- return pickle.load(f)-def _ensure_server_running(self, passages_source_file: str, port: int, **kwargs- ) -> None:+ ) -> int:"""Ensures the embedding server is running if recompute is needed.This is a helper for subclasses.@@ -79,8 +69,8 @@ class BaseSearcher(LeannBackendSearcherInterface, ABC):)embedding_mode = self.meta.get("embedding_mode", "sentence-transformers")-- server_started = self.embedding_server_manager.start_server(++ server_started, actual_port = self.embedding_server_manager.start_server(port=port,model_name=self.embedding_model,passages_file=passages_source_file,@@ -89,7 +79,11 @@ class BaseSearcher(LeannBackendSearcherInterface, ABC):enable_warmup=kwargs.get("enable_warmup", False),)if not server_started:- raise RuntimeError(f"Failed to start embedding server on port {port}")+ raise RuntimeError(+ f"Failed to start embedding server on port {actual_port}"+ )++ return actual_portdef compute_query_embedding(self, query: str, zmq_port: int = 5557, use_server_if_available: bool = True@@ -106,12 +100,16 @@ class BaseSearcher(LeannBackendSearcherInterface, ABC):Query embedding as numpy array"""# Try to use embedding server if available and requested- if (- use_server_if_available- and self.embedding_server_manager- and self.embedding_server_manager.server_process- ):+ if use_server_if_available:try:+ # Ensure we have a server with passages_file for compatibility+ passages_source_file = (+ self.index_dir / f"{self.index_path.name}.meta.json"+ )+ zmq_port = self._ensure_server_running(+ str(passages_source_file), zmq_port+ )+return self._compute_embedding_via_server([query], zmq_port)[0:1] # Return (1, D) shapediff --git a/pyproject.toml b/pyproject.tomlindex 3a0c027..b42dcbe 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -35,6 +35,7 @@ dependencies = ["llama-index-embeddings-huggingface>=0.5.5","mlx>=0.26.3","mlx-lm>=0.26.0",+ "psutil>=5.8.0",][project.optional-dependencies]diff --git a/uv.lock b/uv.lockindex aa661f6..6c84ad2 100644--- a/uv.lock+++ b/uv.lock@@ -1834,10 +1834,14 @@ source = { editable = "packages/leann-core" }dependencies = [{ name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" },{ name = "numpy", version = "2.3.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" },+ { name = "tqdm" },][package.metadata]-requires-dist = [{ name = "numpy", specifier = ">=1.20.0" }]+requires-dist = [+ { name = "numpy", specifier = ">=1.20.0" },+ { name = "tqdm", specifier = ">=4.60.0" },+][[package]]name = "leann-workspace"@@ -1851,7 +1855,6 @@ dependencies = [{ name = "flask" },{ name = "flask-compress" },{ name = "ipykernel" },- { name = "leann-backend-diskann" },{ name = "leann-backend-hnsw" },{ name = "leann-core" },{ name = "llama-index" },@@ -1867,6 +1870,7 @@ dependencies = [{ name = "ollama" },{ name = "openai" },{ name = "protobuf" },+ { name = "psutil" },{ name = "pypdf2" },{ name = "requests" },{ name = "sentence-transformers" },@@ -1884,6 +1888,9 @@ dev = [{ name = "pytest-cov" },{ name = "ruff" },]+diskann = [+ { name = "leann-backend-diskann" },+][package.metadata]requires-dist = [@@ -1896,7 +1903,7 @@ requires-dist = [{ name = "flask-compress" },{ name = "huggingface-hub", marker = "extra == 'dev'", specifier = ">=0.20.0" },{ name = "ipykernel", specifier = "==6.29.5" },- { name = "leann-backend-diskann", editable = "packages/leann-backend-diskann" },+ { name = "leann-backend-diskann", marker = "extra == 'diskann'", editable = "packages/leann-backend-diskann" },{ name = "leann-backend-hnsw", editable = "packages/leann-backend-hnsw" },{ name = "leann-core", editable = "packages/leann-core" },{ name = "llama-index", specifier = ">=0.12.44" },@@ -1912,6 +1919,7 @@ requires-dist = [{ name = "ollama" },{ name = "openai", specifier = ">=1.0.0" },{ name = "protobuf", specifier = "==4.25.3" },+ { name = "psutil", specifier = ">=5.8.0" },{ name = "pypdf2", specifier = ">=3.0.0" },{ name = "pytest", marker = "extra == 'dev'", specifier = ">=7.0" },{ name = "pytest-cov", marker = "extra == 'dev'", specifier = ">=4.0" },@@ -1922,7 +1930,7 @@ requires-dist = [{ name = "torch" },{ name = "tqdm" },]-provides-extras = ["dev"]+provides-extras = ["dev", "diskann"][[package]]name = "llama-cloud"