[MLsys2026 Best Paper]: https://arxiv.org/abs/2506.08276. RAG on Everything with LEANN. Enjoy 97% storage savings while running a fast, accurate, and 100% private RAG application on your personal device.
Python aifaissgpt-osslangchainllama-indexllmlocalstorageoffline-firstollamaprivacypythonragretrieval-augmented-generationvector-databasevector-searchvectors
refactor: check if current emb_server has correct passages/embedder
Andy Lee3b5a185e6041
diff --git a/examples/wechat_history_reader_leann.py b/examples/wechat_history_reader_leann.pyindex f46a1c2..f92dbb7 100644--- a/examples/wechat_history_reader_leann.py+++ b/examples/wechat_history_reader_leann.py@@ -14,43 +14,51 @@ dotenv.load_dotenv()# Default WeChat export directoryDEFAULT_WECHAT_EXPORT_DIR = "./wechat_export_direct"-def create_leann_index_from_multiple_wechat_exports(export_dirs: List[Path], index_path: str = "wechat_history_index.leann", max_count: int = -1):++def create_leann_index_from_multiple_wechat_exports(+ export_dirs: List[Path],+ index_path: str = "wechat_history_index.leann",+ max_count: int = -1,+):"""Create LEANN index from multiple WeChat export data sources.-+Args:export_dirs: List of Path objects pointing to WeChat export directoriesindex_path: Path to save the LEANN indexmax_count: Maximum number of chat entries to process per export"""print("Creating LEANN index from multiple WeChat export data sources...")-+# Load documents using WeChatHistoryReader from history_datafrom history_data.wechat_history import WeChatHistoryReader+reader = WeChatHistoryReader()-+INDEX_DIR = Path(index_path).parent-+if not INDEX_DIR.exists():print(f"--- Index directory not found, building new index ---")all_documents = []total_processed = 0-+# Process each WeChat export directoryfor i, export_dir in enumerate(export_dirs):- print(f"\nProcessing WeChat export {i+1}/{len(export_dirs)}: {export_dir}")-+ print(+ f"\nProcessing WeChat export {i + 1}/{len(export_dirs)}: {export_dir}"+ )+try:documents = reader.load_data(wechat_export_dir=str(export_dir),max_count=max_count,- concatenate_messages=False # Disable concatenation - one message per document+ concatenate_messages=False, # Disable concatenation - one message per document)if documents:print(f"Loaded {len(documents)} chat documents from {export_dir}")all_documents.extend(documents)total_processed += len(documents)-+# Check if we've reached the max countif max_count > 0 and total_processed >= max_count:print(f"Reached max count of {max_count} documents")@@ -60,16 +68,18 @@ def create_leann_index_from_multiple_wechat_exports(export_dirs: List[Path], indexcept Exception as e:print(f"Error processing {export_dir}: {e}")continue-+if not all_documents:print("No documents loaded from any source. Exiting.")return None-- print(f"\nTotal loaded {len(all_documents)} chat documents from {len(export_dirs)} exports")-++ print(+ f"\nTotal loaded {len(all_documents)} chat documents from {len(export_dirs)} exports"+ )+# Create text splitter with 256 chunk sizetext_splitter = SentenceSplitter(chunk_size=256, chunk_overlap=25)-+# Convert Documents to text strings and chunk themall_texts = []for doc in all_documents:@@ -77,43 +87,50 @@ def create_leann_index_from_multiple_wechat_exports(export_dirs: List[Path], indnodes = text_splitter.get_nodes_from_documents([doc])for node in nodes:all_texts.append(node.get_content())-- print(f"Created {len(all_texts)} text chunks from {len(all_documents)} documents")-++ print(+ f"Created {len(all_texts)} text chunks from {len(all_documents)} documents"+ )+# Create LEANN index directoryprint(f"--- Index directory not found, building new index ---")INDEX_DIR.mkdir(exist_ok=True)print(f"--- Building new LEANN index ---")-+print(f"\n[PHASE 1] Building Leann index...")# Use HNSW backend for better macOS compatibilitybuilder = LeannBuilder(backend_name="hnsw",- embedding_model="Qwen/Qwen3-Embedding-0.6B",- graph_degree=32,+ embedding_model="Qwen/Qwen3-Embedding-0.6B",+ graph_degree=32,complexity=64,is_compact=True,is_recompute=True,- num_threads=1 # Force single-threaded mode+ num_threads=1, # Force single-threaded mode)print(f"Adding {len(all_texts)} chat chunks to index...")for chunk_text in all_texts:builder.add_text(chunk_text)-+builder.build_index(index_path)print(f"\nLEANN index built at {index_path}!")else:print(f"--- Using existing index at {INDEX_DIR} ---")-+return index_path-def create_leann_index(export_dir: str = None, index_path: str = "wechat_history_index.leann", max_count: int = 1000):++def create_leann_index(+ export_dir: str = None,+ index_path: str = "wechat_history_index.leann",+ max_count: int = 1000,+):"""Create LEANN index from WeChat chat history data.-+Args:export_dir: Path to the WeChat export directory (optional, uses default if None)index_path: Path to save the LEANN index@@ -121,34 +138,35 @@ def create_leann_index(export_dir: str = None, index_path: str = "wechat_history"""print("Creating LEANN index from WeChat chat history data...")INDEX_DIR = Path(index_path).parent-+if not INDEX_DIR.exists():print(f"--- Index directory not found, building new index ---")INDEX_DIR.mkdir(exist_ok=True)print(f"--- Building new LEANN index ---")-+print(f"\n[PHASE 1] Building Leann index...")# Load documents using WeChatHistoryReader from history_datafrom history_data.wechat_history import WeChatHistoryReader+reader = WeChatHistoryReader()-+documents = reader.load_data(wechat_export_dir=export_dir,max_count=max_count,- concatenate_messages=False # Disable concatenation - one message per document+ concatenate_messages=False, # Disable concatenation - one message per document)-+if not documents:print("No documents loaded. Exiting.")return None-+print(f"Loaded {len(documents)} chat documents")-+# Create text splitter with 256 chunk sizetext_splitter = SentenceSplitter(chunk_size=256, chunk_overlap=25)-+# Convert Documents to text strings and chunk themall_texts = []for doc in documents:@@ -156,54 +174,55 @@ def create_leann_index(export_dir: str = None, index_path: str = "wechat_historynodes = text_splitter.get_nodes_from_documents([doc])for node in nodes:all_texts.append(node.get_content())-+print(f"Created {len(all_texts)} text chunks from {len(documents)} documents")-+# Create LEANN index directoryprint(f"--- Index directory not found, building new index ---")INDEX_DIR.mkdir(exist_ok=True)print(f"--- Building new LEANN index ---")-+print(f"\n[PHASE 1] Building Leann index...")# Use HNSW backend for better macOS compatibilitybuilder = LeannBuilder(backend_name="hnsw",embedding_model="mlx-community/Qwen3-Embedding-0.6B-4bit-DWQ", # MLX-optimized model- graph_degree=32,+ graph_degree=32,complexity=64,is_compact=True,is_recompute=True,- num_threads=1 # Force single-threaded mode+ num_threads=1, # Force single-threaded mode)print(f"Adding {len(all_texts)} chat chunks to index...")for chunk_text in all_texts:builder.add_text(chunk_text)-+builder.build_index(index_path)print(f"\nLEANN index built at {index_path}!")else:print(f"--- Using existing index at {INDEX_DIR} ---")-+return index_path+async def query_leann_index(index_path: str, query: str):"""Query the LEANN index.-+Args:index_path: Path to the LEANN indexquery: The query string"""print(f"\n[PHASE 2] Starting Leann chat session...")chat = LeannChat(index_path=index_path)-+print(f"You: {query}")chat_response = chat.ask(- query,- top_k=5,+ query,+ top_k=5,recompute_beighbor_embeddings=True,complexity=32,beam_width=1,@@ -212,52 +231,74 @@ async def query_leann_index(index_path: str, query: str):"model": "gpt-4o","api_key": os.getenv("OPENAI_API_KEY"),},- llm_kwargs={- "temperature": 0.0,- "max_tokens": 1000- }+ llm_kwargs={"temperature": 0.0, "max_tokens": 1000},)print(f"Leann: {chat_response}")+async def main():"""Main function with integrated WeChat export functionality."""-+# Parse command line arguments- parser = argparse.ArgumentParser(description='LEANN WeChat History Reader - Create and query WeChat chat history index')- parser.add_argument('--export-dir', type=str, default=DEFAULT_WECHAT_EXPORT_DIR,- help=f'Directory to store WeChat exports (default: {DEFAULT_WECHAT_EXPORT_DIR})')- parser.add_argument('--index-dir', type=str, default="./wechat_history_index_leann_test",- help='Directory to store the LEANN index (default: ./wechat_history_index_leann_test)')- parser.add_argument('--max-entries', type=int, default=5000,- help='Maximum number of chat entries to process (default: 5000)')- parser.add_argument('--query', type=str, default=None,- help='Single query to run (default: runs example queries)')- parser.add_argument('--force-export', action='store_true', default=False,- help='Force re-export of WeChat data even if exports exist')-+ parser = argparse.ArgumentParser(+ description="LEANN WeChat History Reader - Create and query WeChat chat history index"+ )+ parser.add_argument(+ "--export-dir",+ type=str,+ default=DEFAULT_WECHAT_EXPORT_DIR,+ help=f"Directory to store WeChat exports (default: {DEFAULT_WECHAT_EXPORT_DIR})",+ )+ parser.add_argument(+ "--index-dir",+ type=str,+ default="./wechat_history_index_leann_test",+ help="Directory to store the LEANN index (default: ./wechat_history_index_leann_test)",+ )+ parser.add_argument(+ "--max-entries",+ type=int,+ default=5000,+ help="Maximum number of chat entries to process (default: 5000)",+ )+ parser.add_argument(+ "--query",+ type=str,+ default=None,+ help="Single query to run (default: runs example queries)",+ )+ parser.add_argument(+ "--force-export",+ action="store_true",+ default=False,+ help="Force re-export of WeChat data even if exports exist",+ )+args = parser.parse_args()-+INDEX_DIR = Path(args.index_dir)INDEX_PATH = str(INDEX_DIR / "wechat_history.leann")-+print(f"Using WeChat export directory: {args.export_dir}")print(f"Index directory: {INDEX_DIR}")print(f"Max entries: {args.max_entries}")-+# Initialize WeChat reader with export capabilitiesfrom history_data.wechat_history import WeChatHistoryReader+reader = WeChatHistoryReader()-+# Find existing exports or create new ones using the centralized methodexport_dirs = reader.find_or_export_wechat_data(args.export_dir)-if not export_dirs:print("Failed to find or export WeChat data. Exiting.")return-+# Create or load the LEANN index from all sources- index_path = create_leann_index_from_multiple_wechat_exports(export_dirs, INDEX_PATH, max_count=args.max_entries)-+ index_path = create_leann_index_from_multiple_wechat_exports(+ export_dirs, INDEX_PATH, max_count=args.max_entries+ )+if index_path:if args.query:# Run single query@@ -267,10 +308,11 @@ async def main():queries = ["我想买魔术师约翰逊的球衣,给我一些对应聊天记录?",]-+for query in queries:- print("\n" + "="*60)+ print("\n" + "=" * 60)await query_leann_index(index_path, query)+if __name__ == "__main__":- asyncio.run(main())\ No newline at end of file+ asyncio.run(main())diff --git a/packages/leann-backend-diskann/leann_backend_diskann/embedding_server.py b/packages/leann-backend-diskann/leann_backend_diskann/embedding_server.pyindex 49442d0..f7a59cd 100644--- a/packages/leann-backend-diskann/leann_backend_diskann/embedding_server.py+++ b/packages/leann-backend-diskann/leann_backend_diskann/embedding_server.py@@ -14,6 +14,7 @@ import osfrom contextlib import contextmanagerimport zmqimport numpy as np+import msgpackfrom pathlib import PathRED = "\033[91m"@@ -26,6 +27,7 @@ class SimplePassageLoader:"""def __init__(self, passages_data: Optional[Dict[str, Any]] = None):self.passages_data = passages_data or {}+ self._meta_path = ''def __getitem__(self, passage_id: Union[str, int]) -> Dict[str, str]:"""Get passage by ID"""@@ -38,6 +40,9 @@ class SimplePassageLoader:def __len__(self) -> int:return len(self.passages_data)++ def keys(self):+ return self.passages_data.keys()def load_passages_from_metadata(meta_file: str) -> SimplePassageLoader:"""@@ -101,8 +106,13 @@ def load_passages_from_metadata(meta_file: str) -> SimplePassageLoader:def __len__(self) -> int:return len(self.label_map)++ def keys(self):+ return self.label_map.keys()- return LazyPassageLoader(passage_manager, label_map)+ loader = LazyPassageLoader(passage_manager, label_map)+ loader._meta_path = meta_file+ return loaderdef load_passages_from_file(passages_file: str) -> SimplePassageLoader:"""@@ -353,6 +363,100 @@ def create_embedding_server_thread(continueprint(f"INFO: Received ZMQ request from client {identity.hex()[:8]}, size {len(message)} bytes")+ # Handle control messages (MessagePack format)+ try:+ request_payload = msgpack.unpackb(message)+ if isinstance(request_payload, list) and len(request_payload) >= 1:+ if request_payload[0] == "__QUERY_META_PATH__":+ # Return the current meta path being used by the server+ current_meta_path = getattr(passages, '_meta_path', '') if hasattr(passages, '_meta_path') else ''+ response = [current_meta_path]+ socket.send_multipart([identity, b'', msgpack.packb(response)])+ continue++ elif request_payload[0] == "__UPDATE_META_PATH__" and len(request_payload) >= 2:+ # Update the server's meta path and reload passages+ new_meta_path = request_payload[1]+ try:+ print(f"INFO: Updating server meta path to: {new_meta_path}")+ # Reload passages from the new meta file+ passages = load_passages_from_metadata(new_meta_path)+ # Store the meta path for future queries+ passages._meta_path = new_meta_path+ response = ["SUCCESS"]+ print(f"INFO: Successfully updated meta path and reloaded {len(passages)} passages")+ except Exception as e:+ print(f"ERROR: Failed to update meta path: {e}")+ response = ["FAILED", str(e)]+ socket.send_multipart([identity, b'', msgpack.packb(response)])+ continue++ elif request_payload[0] == "__QUERY_MODEL__":+ # Return the current model being used by the server+ response = [model_name]+ socket.send_multipart([identity, b'', msgpack.packb(response)])+ continue++ elif request_payload[0] == "__UPDATE_MODEL__" and len(request_payload) >= 2:+ # Update the server's embedding model+ new_model_name = request_payload[1]+ try:+ print(f"INFO: Updating server model from {model_name} to: {new_model_name}")++ # Clean up old model to free memory+ if not use_mlx:+ print("INFO: Releasing old model from memory...")+ old_model = model+ old_tokenizer = tokenizer++ # Load new tokenizer first+ print(f"Loading new tokenizer for {new_model_name}...")+ tokenizer = AutoTokenizer.from_pretrained(new_model_name, use_fast=True)++ # Load new model+ print(f"Loading new model {new_model_name}...")+ model = AutoModel.from_pretrained(new_model_name).to(device).eval()++ # Optimize new model+ if cuda_available or mps_available:+ try:+ model = model.half()+ model = torch.compile(model)+ print(f"INFO: Using FP16 precision with model: {new_model_name}")+ except Exception as e:+ print(f"WARNING: Model optimization failed: {e}")++ # Now safely delete old model after new one is loaded+ del old_model+ del old_tokenizer++ # Clear GPU cache if available+ if device.type == "cuda":+ torch.cuda.empty_cache()+ print("INFO: Cleared CUDA cache")+ elif device.type == "mps":+ torch.mps.empty_cache()+ print("INFO: Cleared MPS cache")++ # Force garbage collection+ import gc+ gc.collect()+ print("INFO: Memory cleanup completed")++ # Update model name+ model_name = new_model_name++ response = ["SUCCESS"]+ print(f"INFO: Successfully updated model to: {new_model_name}")+ except Exception as e:+ print(f"ERROR: Failed to update model: {e}")+ response = ["FAILED", str(e)]+ socket.send_multipart([identity, b'', msgpack.packb(response)])+ continue+ except:+ # Not a control message, continue with normal protobuf processing+ pass+e2e_start = time.time()lookup_timer = DeviceTimer("text lookup")diff --git a/packages/leann-backend-hnsw/leann_backend_hnsw/hnsw_embedding_server.py b/packages/leann-backend-hnsw/leann_backend_hnsw/hnsw_embedding_server.pyindex e5d06f4..a27140f 100644--- a/packages/leann-backend-hnsw/leann_backend_hnsw/hnsw_embedding_server.py+++ b/packages/leann-backend-hnsw/leann_backend_hnsw/hnsw_embedding_server.py@@ -17,10 +17,12 @@ import msgpackimport jsonfrom pathlib import Pathfrom typing import Dict, Any, Optional, Union+import sysRED = "\033[91m"RESET = "\033[0m"+def is_similarity_metric():"""Check if the metric type is similarity-based (like inner product).@@ -28,22 +30,27 @@ def is_similarity_metric():"""return True # 1 is METRIC_INNER_PRODUCT in FAISS+# Function for E5-style average poolingimport torchfrom torch import Tensorimport torch.nn.functional as F+def e5_average_pool(last_hidden_states: Tensor, attention_mask: Tensor) -> Tensor:last_hidden = last_hidden_states.masked_fill(~attention_mask[..., None].bool(), 0.0)return last_hidden.sum(dim=1) / attention_mask.sum(dim=1)[..., None]+class SimplePassageLoader:"""Simple passage loader that replaces config.py dependencies"""+def __init__(self, passages_data: Optional[Dict[str, Any]] = None):self.passages_data = passages_data or {}-+ self._meta_path = ""+def __getitem__(self, passage_id: Union[str, int]) -> Dict[str, str]:"""Get passage by ID"""str_id = str(passage_id)@@ -52,54 +59,57 @@ class SimplePassageLoader:else:# Return empty text for missing passagesreturn {"text": ""}-+def __len__(self) -> int:return len(self.passages_data)+ def keys(self):+ return self.passages_data.keys()++def load_passages_from_metadata(meta_file: str) -> SimplePassageLoader:"""Load passages using metadata file with PassageManager for lazy loading"""# Load metadata to get passage sources- with open(meta_file, 'r') as f:+ with open(meta_file, "r") as f:meta = json.load(f)-+# Import PassageManager dynamically to avoid circular imports- import sys- import importlib.util-# Find the leann package directory relative to this filecurrent_dir = Path(__file__).parentleann_core_path = current_dir.parent.parent / "leann-core" / "src"sys.path.insert(0, str(leann_core_path))-+try:from leann.api import PassageManager- passage_manager = PassageManager(meta['passage_sources'])++ passage_manager = PassageManager(meta["passage_sources"])finally:sys.path.pop(0)-- # Load label map++ # Load label mappassages_dir = Path(meta_file).parentlabel_map_file = passages_dir / "leann.labels.map"-+if label_map_file.exists():import pickle- with open(label_map_file, 'rb') as f:++ with open(label_map_file, "rb") as f:label_map = pickle.load(f)print(f"Loaded label map with {len(label_map)} entries")else:raise FileNotFoundError(f"Label map file not found: {label_map_file}")-+print(f"Initialized lazy passage loading for {len(label_map)} passages")-+class LazyPassageLoader(SimplePassageLoader):def __init__(self, passage_manager, label_map):self.passage_manager = passage_managerself.label_map = label_map# Initialize parent with empty datasuper().__init__({})-+def __getitem__(self, passage_id: Union[str, int]) -> Dict[str, str]:"""Get passage by ID with lazy loading"""try:@@ -118,12 +128,16 @@ def load_passages_from_metadata(meta_file: str) -> SimplePassageLoader:except Exception as e:print(f"DEBUG: Exception getting passage {passage_id}: {e}")return {"text": ""}-+def __len__(self) -> int:return len(self.label_map)-++ def keys(self):+ return self.label_map.keys()+return LazyPassageLoader(passage_manager, label_map)+def create_hnsw_embedding_server(passages_file: Optional[str] = None,passages_data: Optional[Dict[str, str]] = None,@@ -139,7 +153,7 @@ def create_hnsw_embedding_server():"""Create and start a ZMQ-based embedding server for HNSW backend.-+Args:passages_file: Path to JSON file containing passage ID -> text mappingpassages_data: Direct passage data dict (alternative to passages_file)@@ -156,14 +170,14 @@ def create_hnsw_embedding_server(print(f"Loading tokenizer for {model_name}...")tokenizer = AutoTokenizer.from_pretrained(model_name, use_fast=True)print(f"Tokenizer loaded successfully!")-+# Device setup- mps_available = hasattr(torch.backends, 'mps') and torch.backends.mps.is_available()+ mps_available = hasattr(torch.backends, "mps") and torch.backends.mps.is_available()cuda_available = torch.cuda.is_available()-+print(f"MPS available: {mps_available}")print(f"CUDA available: {cuda_available}")-+if cuda_available:device = torch.device("cuda")print("Using CUDA device")@@ -173,7 +187,7 @@ def create_hnsw_embedding_server(else:device = torch.device("cpu")print("Using CPU device (no GPU acceleration available)")-+# Load model to the appropriate deviceprint(f"Starting HNSW server on port {zmq_port} with model {model_name}")print(f"Loading model {model_name}... (this may take a while if downloading)")@@ -182,9 +196,10 @@ def create_hnsw_embedding_server(# Check port availabilityimport socket+def check_port(port):with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:- return s.connect_ex(('localhost', port)) == 0+ return s.connect_ex(("localhost", port)) == 0if check_port(zmq_port):print(f"{RED}Port {zmq_port} is already in use{RESET}")@@ -196,8 +211,14 @@ def create_hnsw_embedding_server(model = torch.compile(model)print(f"Using FP16 precision with model: {model_name}")elif use_int8:- print("- Using TorchAO for Int8 dynamic activation and Int8 weight quantization")- from torchao.quantization import quantize_, Int8DynamicActivationInt8WeightConfig+ print(+ "- Using TorchAO for Int8 dynamic activation and Int8 weight quantization"+ )+ from torchao.quantization import (+ quantize_,+ Int8DynamicActivationInt8WeightConfig,+ )+quantize_(model, Int8DynamicActivationInt8WeightConfig())model = torch.compile(model)model.eval()@@ -209,8 +230,10 @@ def create_hnsw_embedding_server(print(f"Using provided passages data: {len(passages)} passages")elif passages_file:# Check if it's a metadata file or a single passages file- if passages_file.endswith('.meta.json'):+ if passages_file.endswith(".meta.json"):passages = load_passages_from_metadata(passages_file)+ # Store the meta path for future reference+ passages._meta_path = passages_fileelse:# Try to find metadata file in same directorypassages_dir = Path(passages_file).parent@@ -220,8 +243,12 @@ def create_hnsw_embedding_server(passages = load_passages_from_metadata(str(meta_files[0]))else:# Fallback to original single file loading (will cause warnings)- print("WARNING: No metadata file found, using single file loading (may cause missing passage warnings)")- passages = SimplePassageLoader() # Use empty loader to avoid massive warnings+ print(+ "WARNING: No metadata file found, using single file loading (may cause missing passage warnings)"+ )+ passages = (+ SimplePassageLoader()+ ) # Use empty loader to avoid massive warningselse:passages = SimplePassageLoader()print("No passages provided, using empty loader")@@ -238,12 +265,13 @@ def create_hnsw_embedding_server(class DeviceTimer:"""Device event-based timer for accurate timing."""+def __init__(self, name="", device=device):self.name = nameself.device = deviceself.start_time = 0self.end_time = 0-+if cuda_available:self.start_event = torch.cuda.Event(enable_timing=True)self.end_event = torch.cuda.Event(enable_timing=True)@@ -289,30 +317,31 @@ def create_hnsw_embedding_server(_is_e5_model = "e5" in model_name.lower()_is_bge_model = "bge" in model_name.lower()batch_size = len(texts_batch)-- # Validate no empty texts- for i, text in enumerate(texts_batch):- if not text or text.strip() == "":- raise RuntimeError(f"FATAL: Empty text at batch index {i}, ID: {ids_batch[i] if i < len(ids_batch) else 'unknown'}")-++ # Allow empty texts to pass through (remove validation)+# E5 model preprocessingif _is_e5_model:processed_texts_batch = [f"passage: {text}" for text in texts_batch]else:processed_texts_batch = texts_batch-+# Set max lengthif _is_e5_model:- current_max_length = custom_max_length_param if custom_max_length_param is not None else 512+ current_max_length = (+ custom_max_length_param if custom_max_length_param is not None else 512+ )else:- current_max_length = custom_max_length_param if custom_max_length_param is not None else 256-+ current_max_length = (+ custom_max_length_param if custom_max_length_param is not None else 256+ )+tokenize_timer = DeviceTimer("tokenization (batch)", device)to_device_timer = DeviceTimer("transfer to device (batch)", device)embed_timer = DeviceTimer("embedding (batch)", device)pool_timer = DeviceTimer("pooling (batch)", device)norm_timer = DeviceTimer("normalization (batch)", device)-+with tokenize_timer.timing():encoded_batch = tokenizer(processed_texts_batch,@@ -322,48 +351,71 @@ def create_hnsw_embedding_server(return_tensors="pt",return_token_type_ids=False,)-+seq_length = encoded_batch["input_ids"].size(1)-+with to_device_timer.timing():enc = {k: v.to(device) for k, v in encoded_batch.items()}-+with torch.no_grad():with embed_timer.timing():out = model(enc["input_ids"], enc["attention_mask"])-+with pool_timer.timing():if _is_bge_model:pooled_embeddings = out.last_hidden_state[:, 0]- elif not hasattr(out, 'last_hidden_state'):+ elif not hasattr(out, "last_hidden_state"):if isinstance(out, torch.Tensor) and len(out.shape) == 2:pooled_embeddings = outelse:- print(f"{RED}ERROR: Cannot determine how to pool. Output shape: {out.shape if isinstance(out, torch.Tensor) else 'N/A'}{RESET}")- hidden_dim = getattr(model.config, 'hidden_size', 384 if _is_e5_model else 768)- pooled_embeddings = torch.zeros((batch_size, hidden_dim), device=device, dtype=enc["input_ids"].dtype if hasattr(enc["input_ids"], "dtype") else torch.float32)+ print(+ f"{RED}ERROR: Cannot determine how to pool. Output shape: {out.shape if isinstance(out, torch.Tensor) else 'N/A'}{RESET}"+ )+ hidden_dim = getattr(+ model.config, "hidden_size", 384 if _is_e5_model else 768+ )+ pooled_embeddings = torch.zeros(+ (batch_size, hidden_dim),+ device=device,+ dtype=enc["input_ids"].dtype+ if hasattr(enc["input_ids"], "dtype")+ else torch.float32,+ )elif _is_e5_model:- pooled_embeddings = e5_average_pool(out.last_hidden_state, enc['attention_mask'])+ pooled_embeddings = e5_average_pool(+ out.last_hidden_state, enc["attention_mask"]+ )else:hidden_states = out.last_hidden_state- mask_expanded = enc["attention_mask"].unsqueeze(-1).expand(hidden_states.size()).float()+ mask_expanded = (+ enc["attention_mask"]+ .unsqueeze(-1)+ .expand(hidden_states.size())+ .float()+ )sum_embeddings = torch.sum(hidden_states * mask_expanded, 1)sum_mask = torch.clamp(mask_expanded.sum(1), min=1e-9)pooled_embeddings = sum_embeddings / sum_mask-+final_embeddings = pooled_embeddingsif _is_e5_model or _is_bge_model:with norm_timer.timing():final_embeddings = F.normalize(pooled_embeddings, p=2, dim=1)-+if torch.isnan(final_embeddings).any() or torch.isinf(final_embeddings).any():- print(f"{RED}!!! In process_batch: NaN or Inf detected in final_embeddings! "- f"Model: {model_name}, E5: {_is_e5_model}. IDs (sample): {ids_batch[:5]}...{RESET}")+ print(+ f"{RED}!!! In process_batch: NaN or Inf detected in final_embeddings! "+ f"Model: {model_name}, E5: {_is_e5_model}. IDs (sample): {ids_batch[:5]}...{RESET}"+ )dim_size = final_embeddings.shape[-1]- error_output = torch.zeros((batch_size, dim_size), device='cpu', dtype=torch.float32).numpy()- print(f"{RED}Returning zero embeddings of shape ({batch_size}, {dim_size}) due to NaN/Inf.{RESET}")+ error_output = torch.zeros(+ (batch_size, dim_size), device="cpu", dtype=torch.float32+ ).numpy()+ print(+ f"{RED}Returning zero embeddings of shape ({batch_size}, {dim_size}) due to NaN/Inf.{RESET}"+ )return error_output-+return final_embeddings.cpu().numpy()def client_warmup(zmq_port):@@ -371,7 +423,7 @@ def create_hnsw_embedding_server(time.sleep(2)print(f"Performing client-side warmup with model {model_name}...")sample_ids = ["1", "2", "3", "4", "5"]-+try:context = zmq.Context()socket = context.socket(zmq.REQ)@@ -379,12 +431,12 @@ def create_hnsw_embedding_server(socket.setsockopt(zmq.RCVTIMEO, 30000)socket.setsockopt(zmq.SNDTIMEO, 30000)- try:+ try:ids_to_send = [int(x) for x in sample_ids]- except ValueError:+ except ValueError:ids_to_send = []- if not ids_to_send:+ if not ids_to_send:print("Skipping warmup send.")return@@ -392,14 +444,18 @@ def create_hnsw_embedding_server(request_bytes = msgpack.packb(request_payload)for i in range(3):- print(f"Sending warmup request {i+1}/3 via ZMQ (MessagePack)...")+ print(f"Sending warmup request {i + 1}/3 via ZMQ (MessagePack)...")socket.send(request_bytes)response_bytes = socket.recv()response_payload = msgpack.unpackb(response_bytes)dimensions = response_payload[0]- embeddings_count = dimensions[0] if dimensions and len(dimensions) > 0 else 0- print(f"Warmup request {i+1}/3 successful, received {embeddings_count} embeddings")+ embeddings_count = (+ dimensions[0] if dimensions and len(dimensions) > 0 else 0+ )+ print(+ f"Warmup request {i + 1}/3 successful, received {embeddings_count} embeddings"+ )time.sleep(0.1)print("Client-side MessagePack ZMQ warmup complete")@@ -410,6 +466,7 @@ def create_hnsw_embedding_server(def zmq_server_thread():"""ZMQ server thread"""+ nonlocal passages, model, tokenizer, model_namecontext = zmq.Context()socket = context.socket(zmq.REP)socket.bind(f"tcp://*:{zmq_port}")@@ -428,94 +485,277 @@ def create_hnsw_embedding_server(try:request_payload = msgpack.unpackb(message_bytes)-+ print(f"DEBUG: Raw request_payload: {request_payload}")+ print(f"DEBUG: request_payload type: {type(request_payload)}")+ if isinstance(request_payload, list):+ print(f"DEBUG: request_payload length: {len(request_payload)}")+ for i, item in enumerate(request_payload):+ print(+ f"DEBUG: request_payload[{i}]: {type(item)} - {item if len(str(item)) < 100 else str(item)[:100] + '...'}"+ )++ # Handle control messages for meta path and model management+ if isinstance(request_payload, list) and len(request_payload) >= 1:+ if request_payload[0] == "__QUERY_META_PATH__":+ # Return the current meta path being used by the server+ current_meta_path = (+ getattr(passages, "_meta_path", "")+ if hasattr(passages, "_meta_path")+ else ""+ )+ response = [current_meta_path]+ socket.send(msgpack.packb(response))+ continue++ elif (+ request_payload[0] == "__UPDATE_META_PATH__"+ and len(request_payload) >= 2+ ):+ # Update the server's meta path and reload passages+ new_meta_path = request_payload[1]+ try:+ print(+ f"INFO: Updating server meta path to: {new_meta_path}"+ )+ # Reload passages from the new meta file+ passages = load_passages_from_metadata(new_meta_path)+ # Store the meta path for future queries+ passages._meta_path = new_meta_path+ response = ["SUCCESS"]+ print(+ f"INFO: Successfully updated meta path and reloaded {len(passages)} passages"+ )+ except Exception as e:+ print(f"ERROR: Failed to update meta path: {e}")+ response = ["FAILED", str(e)]+ socket.send(msgpack.packb(response))+ continue++ elif request_payload[0] == "__QUERY_MODEL__":+ # Return the current model being used by the server+ response = [model_name]+ socket.send(msgpack.packb(response))+ continue++ elif (+ request_payload[0] == "__UPDATE_MODEL__"+ and len(request_payload) >= 2+ ):+ # Update the server's embedding model+ new_model_name = request_payload[1]+ try:+ print(+ f"INFO: Updating server model from {model_name} to: {new_model_name}"+ )++ # Clean up old model to free memory+ print("INFO: Releasing old model from memory...")+ old_model = model+ old_tokenizer = tokenizer++ # Load new tokenizer first+ print(f"Loading new tokenizer for {new_model_name}...")+ tokenizer = AutoTokenizer.from_pretrained(+ new_model_name, use_fast=True+ )++ # Load new model+ print(f"Loading new model {new_model_name}...")+ model = AutoModel.from_pretrained(new_model_name)+ model.to(device)+ model.eval()++ # Now safely delete old model after new one is loaded+ del old_model+ del old_tokenizer++ # Clear GPU cache if available+ if device.type == "cuda":+ torch.cuda.empty_cache()+ print("INFO: Cleared CUDA cache")+ elif device.type == "mps":+ torch.mps.empty_cache()+ print("INFO: Cleared MPS cache")++ # Update model name+ model_name = new_model_name++ # Force garbage collection+ import gc++ gc.collect()+ print("INFO: Memory cleanup completed")++ response = ["SUCCESS"]+ print(+ f"INFO: Successfully updated model to: {new_model_name}"+ )+ except Exception as e:+ print(f"ERROR: Failed to update model: {e}")+ response = ["FAILED", str(e)]+ socket.send(msgpack.packb(response))+ continue+# Handle distance calculation requests- if isinstance(request_payload, list) and len(request_payload) == 2 and isinstance(request_payload[0], list) and isinstance(request_payload[1], list):+ if (+ isinstance(request_payload, list)+ and len(request_payload) == 2+ and isinstance(request_payload[0], list)+ and isinstance(request_payload[1], list)+ ):node_ids = request_payload[0]query_vector = np.array(request_payload[1], dtype=np.float32)-- print(f"Request for distance calculation: {len(node_ids)} nodes, query vector dim: {len(query_vector)}")-++ print("DEBUG: Distance calculation request received")+ print(f" Node IDs: {node_ids}")+ print(f" Query vector dim: {len(query_vector)}")+ print(f" Passages loaded: {len(passages)}")+# Get embeddings for node IDstexts = []missing_ids = []with lookup_timer.timing():for nid in node_ids:print(f"DEBUG: Looking up passage ID {nid}")- txtinfo = passages[nid]- if txtinfo is None or txtinfo["text"] == "":- raise RuntimeError(f"FATAL: Passage with ID {nid} returned empty text")- txt = txtinfo["text"]- print(f"DEBUG: Found text for ID {nid}, length: {len(txt)}")- texts.append(txt)+ try:+ txtinfo = passages[nid]+ if txtinfo is None:+ print(+ f"ERROR: Passage with ID {nid} returned None"+ )+ print(f"ERROR: txtinfo: {txtinfo}")+ raise RuntimeError(+ f"FATAL: Passage with ID {nid} returned None"+ )+ txt = txtinfo[+ "text"+ ] # Allow empty text to pass through+ print(+ f"DEBUG: Found text for ID {nid}, length: {len(txt)}"+ )+ texts.append(txt)+ except KeyError:+ print(+ f"ERROR: Passage ID {nid} not found in passages dict"+ )+ print(+ f"ERROR: Available passage IDs: {list(passages.keys())[:10]}..."+ )+ raise RuntimeError(+ f"FATAL: Passage with ID {nid} not found"+ )+ except Exception as e:+ print(+ f"ERROR: Exception looking up passage ID {nid}: {e}"+ )+ raiselookup_timer.print_elapsed()-+# Process embeddings in chunks if neededall_node_embeddings = []total_size = len(texts)-+if total_size > max_batch_size:for i in range(0, total_size, max_batch_size):end_idx = min(i + max_batch_size, total_size)chunk_texts = texts[i:end_idx]chunk_ids = node_ids[i:end_idx]-- embeddings_chunk = process_batch(chunk_texts, chunk_ids, missing_ids)++ embeddings_chunk = process_batch(+ chunk_texts, chunk_ids, missing_ids+ )all_node_embeddings.append(embeddings_chunk)-+if cuda_available:torch.cuda.empty_cache()elif device.type == "mps":torch.mps.empty_cache()-+node_embeddings = np.vstack(all_node_embeddings)else:- node_embeddings = process_batch(texts, node_ids, missing_ids)-+ node_embeddings = process_batch(+ texts, node_ids, missing_ids+ )+# Calculate distancesquery_tensor = torch.tensor(query_vector, device=device).float()- node_embeddings_tensor = torch.tensor(node_embeddings, device=device).float()-+ node_embeddings_tensor = torch.tensor(+ node_embeddings, device=device+ ).float()+calc_timer = DeviceTimer("distance calculation", device)with calc_timer.timing():with torch.no_grad():if distance_metric == "l2":- node_embeddings_np = node_embeddings_tensor.cpu().numpy().astype(np.float32)- query_np = query_tensor.cpu().numpy().astype(np.float32)- distances = np.sum(np.square(node_embeddings_np - query_np.reshape(1, -1)), axis=1)- else: # mips or cosine- node_embeddings_np = node_embeddings_tensor.cpu().numpy()+ node_embeddings_np = (+ node_embeddings_tensor.cpu()+ .numpy()+ .astype(np.float32)+ )+ query_np = (+ query_tensor.cpu().numpy().astype(np.float32)+ )+ distances = np.sum(+ np.square(+ node_embeddings_np - query_np.reshape(1, -1)+ ),+ axis=1,+ )+ else: # mips or cosine+ node_embeddings_np = (+ node_embeddings_tensor.cpu().numpy()+ )query_np = query_tensor.cpu().numpy()distances = -np.dot(node_embeddings_np, query_np)calc_timer.print_elapsed()-+try:response_payload = distances.flatten().tolist()- response_bytes = msgpack.packb([response_payload], use_single_float=True)- print(f"Sending distance response with {len(distances)} distances")+ response_bytes = msgpack.packb(+ [response_payload], use_single_float=True+ )+ print(+ f"Sending distance response with {len(distances)} distances"+ )except Exception as pack_error:- print(f"Error packing MessagePack distance response: {pack_error}")+ print(+ f"ERROR: Error packing MessagePack distance response: {pack_error}"+ )+ print(f"ERROR: distances shape: {distances.shape}")+ print(f"ERROR: distances dtype: {distances.dtype}")+ print(f"ERROR: distances content: {distances}")+ print(f"ERROR: node_ids: {node_ids}")+ print(f"ERROR: query_vector shape: {query_vector.shape}")+ # Still return empty for now but with full error inforesponse_bytes = msgpack.packb([[]])-+socket.send(response_bytes)-+if device.type == "cuda":torch.cuda.synchronize()elif device.type == "mps":torch.mps.synchronize()e2e_end = time.time()- print(f"Distance calculation E2E time: {e2e_end - e2e_start:.6f} seconds")+ print(+ f"Distance calculation E2E time: {e2e_end - e2e_start:.6f} seconds"+ )continue-+# Standard embedding request- if not isinstance(request_payload, list) or len(request_payload) != 1 or not isinstance(request_payload[0], list):- print(f"Error: Invalid MessagePack request format. Expected [[ids...]], got: {type(request_payload)}")+ if (+ not isinstance(request_payload, list)+ or len(request_payload) != 1+ or not isinstance(request_payload[0], list)+ ):+ print(+ f"Error: Invalid MessagePack request format. Expected [[ids...]], got: {type(request_payload)}"+ )socket.send(msgpack.packb([[], []]))continue-+node_ids = request_payload[0]print(f"Request for {len(node_ids)} node embeddings")-+except Exception as unpack_error:print(f"Error unpacking MessagePack request: {unpack_error}")socket.send(msgpack.packb([[], []]))@@ -529,11 +769,15 @@ def create_hnsw_embedding_server(try:txtinfo = passages[nid]if txtinfo is None or txtinfo["text"] == "":- raise RuntimeError(f"FATAL: Passage with ID {nid} not found - failing fast")+ raise RuntimeError(+ f"FATAL: Passage with ID {nid} not found - failing fast"+ )else:txt = txtinfo["text"]except (KeyError, IndexError):- raise RuntimeError(f"FATAL: Passage with ID {nid} not found - failing fast")+ raise RuntimeError(+ f"FATAL: Passage with ID {nid} not found - failing fast"+ )texts.append(txt)lookup_timer.print_elapsed()@@ -542,27 +786,35 @@ def create_hnsw_embedding_server(# Process in chunkstotal_size = len(texts)- print(f"Total batch size: {total_size}, max_batch_size: {max_batch_size}")-+ print(+ f"Total batch size: {total_size}, max_batch_size: {max_batch_size}"+ )+all_embeddings = []-+if total_size > max_batch_size:- print(f"Splitting batch of size {total_size} into chunks of {max_batch_size}")+ print(+ f"Splitting batch of size {total_size} into chunks of {max_batch_size}"+ )for i in range(0, total_size, max_batch_size):end_idx = min(i + max_batch_size, total_size)- print(f"Processing chunk {i//max_batch_size + 1}/{(total_size + max_batch_size - 1)//max_batch_size}: items {i} to {end_idx-1}")-+ print(+ f"Processing chunk {i // max_batch_size + 1}/{(total_size + max_batch_size - 1) // max_batch_size}: items {i} to {end_idx - 1}"+ )+chunk_texts = texts[i:end_idx]chunk_ids = node_ids[i:end_idx]-- embeddings_chunk = process_batch(chunk_texts, chunk_ids, missing_ids)++ embeddings_chunk = process_batch(+ chunk_texts, chunk_ids, missing_ids+ )all_embeddings.append(embeddings_chunk)-+if cuda_available:torch.cuda.empty_cache()elif device.type == "mps":torch.mps.empty_cache()-+hidden = np.vstack(all_embeddings)print(f"Combined embeddings shape: {hidden.shape}")else:@@ -571,22 +823,30 @@ def create_hnsw_embedding_server(# Serialization and responseser_start = time.time()- print(f"DEBUG zmq_server_thread: Final 'hidden' array | Shape: {hidden.shape} | Dtype: {hidden.dtype} | Has NaN/Inf: {np.isnan(hidden).any() or np.isinf(hidden).any()}")+ print(+ f"DEBUG zmq_server_thread: Final 'hidden' array | Shape: {hidden.shape} | Dtype: {hidden.dtype} | Has NaN/Inf: {np.isnan(hidden).any() or np.isinf(hidden).any()}"+ )if np.isnan(hidden).any() or np.isinf(hidden).any():- print(f"{RED}!!! ERROR: NaN or Inf detected in final 'hidden' numpy array BEFORE sending! "- f"Requested IDs (sample): {node_ids[:5]}...{RESET}")+ print(+ f"{RED}!!! ERROR: NaN or Inf detected in final 'hidden' numpy array BEFORE sending! "+ f"Requested IDs (sample): {node_ids[:5]}...{RESET}"+ )assert Falsetry:- hidden_contiguous_f32 = np.ascontiguousarray(hidden, dtype=np.float32)+ hidden_contiguous_f32 = np.ascontiguousarray(+ hidden, dtype=np.float32+ )response_payload = [list(hidden_contiguous_f32.shape),- hidden_contiguous_f32.flatten().tolist()+ hidden_contiguous_f32.flatten().tolist(),]- response_bytes = msgpack.packb(response_payload, use_single_float=True)+ response_bytes = msgpack.packb(+ response_payload, use_single_float=True+ )except Exception as pack_error:- print(f"Error packing MessagePack response: {pack_error}")- response_bytes = msgpack.packb([[], []])+ print(f"Error packing MessagePack response: {pack_error}")+ response_bytes = msgpack.packb([[], []])socket.send(response_bytes)ser_end = time.time()@@ -606,8 +866,9 @@ def create_hnsw_embedding_server(except Exception as e:print(f"Error in ZMQ server loop: {e}")import traceback+traceback.print_exc()- try:+ try:socket.send(msgpack.packb([[], []]))except:pass@@ -621,7 +882,7 @@ def create_hnsw_embedding_server(zmq_thread = threading.Thread(target=zmq_server_thread, daemon=True)zmq_thread.start()print(f"Started HNSW ZMQ server thread on port {zmq_port}")-+# Keep the main thread alivetry:while True:@@ -634,17 +895,41 @@ def create_hnsw_embedding_server(if __name__ == "__main__":parser = argparse.ArgumentParser(description="HNSW Embedding service")parser.add_argument("--zmq-port", type=int, default=5555, help="ZMQ port to run on")- parser.add_argument("--passages-file", type=str, help="JSON file containing passage ID to text mapping")- parser.add_argument("--embeddings-file", type=str, help="Pickle file containing pre-computed embeddings")+ parser.add_argument(+ "--passages-file",+ type=str,+ help="JSON file containing passage ID to text mapping",+ )+ parser.add_argument(+ "--embeddings-file",+ type=str,+ help="Pickle file containing pre-computed embeddings",+ )parser.add_argument("--use-fp16", action="store_true", default=False)parser.add_argument("--use-int8", action="store_true", default=False)parser.add_argument("--use-cuda-graphs", action="store_true", default=False)- parser.add_argument("--max-batch-size", type=int, default=128, help="Maximum batch size before splitting")- parser.add_argument("--model-name", type=str, default="sentence-transformers/all-mpnet-base-v2",- help="Embedding model name")- parser.add_argument("--custom-max-length", type=int, default=None, help="Override model's default max sequence length")- parser.add_argument("--distance-metric", type=str, default="mips", help="Distance metric to use")-+ parser.add_argument(+ "--max-batch-size",+ type=int,+ default=128,+ help="Maximum batch size before splitting",+ )+ parser.add_argument(+ "--model-name",+ type=str,+ default="sentence-transformers/all-mpnet-base-v2",+ help="Embedding model name",+ )+ parser.add_argument(+ "--custom-max-length",+ type=int,+ default=None,+ help="Override model's default max sequence length",+ )+ parser.add_argument(+ "--distance-metric", type=str, default="mips", help="Distance metric to use"+ )+args = parser.parse_args()# Create and start the HNSW embedding server@@ -659,4 +944,4 @@ if __name__ == "__main__":model_name=args.model_name,custom_max_length_param=args.custom_max_length,distance_metric=args.distance_metric,- )\ No newline at end of file+ )diff --git a/packages/leann-core/src/leann/embedding_server_manager.py b/packages/leann-core/src/leann/embedding_server_manager.pyindex 7205d87..9f5aaf3 100644--- a/packages/leann-core/src/leann/embedding_server_manager.py+++ b/packages/leann-core/src/leann/embedding_server_manager.py@@ -1,23 +1,169 @@--import osimport threadingimport timeimport atexitimport socketimport subprocessimport sys+import zmq+import msgpackfrom pathlib import Pathfrom typing import Optional+import select+def _check_port(port: int) -> bool:"""Check if a port is in use"""with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:- return s.connect_ex(('localhost', port)) == 0+ return s.connect_ex(("localhost", port)) == 0+++def _check_server_meta_path(port: int, expected_meta_path: str) -> bool:+ """+ Check if the existing server on the port is using the correct meta file.+ Returns True if the server has the right meta path, False otherwise.+ """+ try:+ context = zmq.Context()+ socket = context.socket(zmq.REQ)+ socket.setsockopt(zmq.RCVTIMEO, 3000) # 3 second timeout+ socket.connect(f"tcp://localhost:{port}")++ # Send a special control message to query the server's meta path+ control_request = ["__QUERY_META_PATH__"]+ request_bytes = msgpack.packb(control_request)+ socket.send(request_bytes)++ # Wait for response+ response_bytes = socket.recv()+ response = msgpack.unpackb(response_bytes)++ socket.close()+ context.term()++ # Check if the response contains the meta path and if it matches+ if isinstance(response, list) and len(response) > 0:+ server_meta_path = response[0]+ # Normalize paths for comparison+ expected_path = Path(expected_meta_path).resolve()+ server_path = Path(server_meta_path).resolve() if server_meta_path else None+ return server_path == expected_path++ return False++ except Exception as e:+ print(f"WARNING: Could not query server meta path on port {port}: {e}")+ return False+++def _update_server_meta_path(port: int, new_meta_path: str) -> bool:+ """+ Send a control message to update the server's meta path.+ Returns True if successful, False otherwise.+ """+ try:+ context = zmq.Context()+ socket = context.socket(zmq.REQ)+ socket.setsockopt(zmq.RCVTIMEO, 5000) # 5 second timeout+ socket.connect(f"tcp://localhost:{port}")++ # Send a control message to update the meta path+ control_request = ["__UPDATE_META_PATH__", new_meta_path]+ request_bytes = msgpack.packb(control_request)+ socket.send(request_bytes)++ # Wait for response+ response_bytes = socket.recv()+ response = msgpack.unpackb(response_bytes)++ socket.close()+ context.term()++ # Check if the update was successful+ if isinstance(response, list) and len(response) > 0:+ return response[0] == "SUCCESS"++ return False++ except Exception as e:+ print(f"ERROR: Could not update server meta path on port {port}: {e}")+ return False+++def _check_server_model(port: int, expected_model: str) -> bool:+ """+ Check if the existing server on the port is using the correct embedding model.+ Returns True if the server has the right model, False otherwise.+ """+ try:+ context = zmq.Context()+ socket = context.socket(zmq.REQ)+ socket.setsockopt(zmq.RCVTIMEO, 3000) # 3 second timeout+ socket.connect(f"tcp://localhost:{port}")++ # Send a special control message to query the server's model+ control_request = ["__QUERY_MODEL__"]+ request_bytes = msgpack.packb(control_request)+ socket.send(request_bytes)++ # Wait for response+ response_bytes = socket.recv()+ response = msgpack.unpackb(response_bytes)++ socket.close()+ context.term()++ # Check if the response contains the model name and if it matches+ if isinstance(response, list) and len(response) > 0:+ server_model = response[0]+ return server_model == expected_model++ return False++ except Exception as e:+ print(f"WARNING: Could not query server model on port {port}: {e}")+ return False+++def _update_server_model(port: int, new_model: str) -> bool:+ """+ Send a control message to update the server's embedding model.+ Returns True if successful, False otherwise.+ """+ try:+ context = zmq.Context()+ socket = context.socket(zmq.REQ)+ socket.setsockopt(zmq.RCVTIMEO, 30000) # 30 second timeout for model loading+ socket.setsockopt(zmq.SNDTIMEO, 5000) # 5 second timeout for sending+ socket.connect(f"tcp://localhost:{port}")++ # Send a control message to update the model+ control_request = ["__UPDATE_MODEL__", new_model]+ request_bytes = msgpack.packb(control_request)+ socket.send(request_bytes)++ # Wait for response+ response_bytes = socket.recv()+ response = msgpack.unpackb(response_bytes)++ socket.close()+ context.term()++ # Check if the update was successful+ if isinstance(response, list) and len(response) > 0:+ return response[0] == "SUCCESS"++ return False++ except Exception as e:+ print(f"ERROR: Could not update server model on port {port}: {e}")+ return False+class EmbeddingServerManager:"""A generic manager for handling the lifecycle of a backend-specific embedding server process."""+def __init__(self, backend_module_name: str):"""Initializes the manager for a specific backend.@@ -44,21 +190,119 @@ class EmbeddingServerManager:bool: True if the server is started successfully or already running, False otherwise."""if self.server_process and self.server_process.poll() is None:- print(f"INFO: Reusing existing server process for this session (PID {self.server_process.pid})")- return True+ # Even if we have a running process, check if model/meta path match+ if self.server_port is not None:+ port_in_use = _check_port(self.server_port)+ if port_in_use:+ print(+ f"INFO: Checking compatibility of existing server process (PID {self.server_process.pid})"+ )++ # Check model compatibility+ model_matches = _check_server_model(self.server_port, model_name)+ if not model_matches:+ print(+ f"⚠️ Existing server has different model. Attempting to update to: {model_name}"+ )+ if not _update_server_model(self.server_port, model_name):+ print(+ "❌ Failed to update existing server model. Restarting server..."+ )+ self.stop_server()+ # Continue to start new server below+ else:+ print(+ f"✅ Successfully updated existing server model to: {model_name}"+ )++ # Also check meta path if provided+ passages_file = kwargs.get("passages_file")+ if passages_file and str(passages_file).endswith(+ ".meta.json"+ ):+ meta_matches = _check_server_meta_path(+ self.server_port, str(passages_file)+ )+ if not meta_matches:+ print("⚠️ Updating meta path to: {passages_file}")+ _update_server_meta_path(+ self.server_port, str(passages_file)+ )++ return True+ else:+ print(+ f"✅ Existing server already using correct model: {model_name}"+ )+ return True+ else:+ # Server process exists but port not responding - restart+ print("⚠️ Server process exists but not responding. Restarting...")+ self.stop_server()+ # Continue to start new server below+ else:+ # No port stored - restart+ print("⚠️ No port information stored. Restarting server...")+ self.stop_server()+ # Continue to start new server belowif _check_port(port):- print(f"WARNING: Port {port} is already in use. Assuming an external server is running.")+ # Port is in use, check if it's using the correct meta file and model+ passages_file = kwargs.get("passages_file")++ print(f"INFO: Port {port} is in use. Checking server compatibility...")++ # Check model compatibility first+ model_matches = _check_server_model(port, model_name)+ if not model_matches:+ print(+ f"⚠️ Existing server on port {port} has different model. Attempting to update to: {model_name}"+ )+ if not _update_server_model(port, model_name):+ raise RuntimeError(+ f"❌ Failed to update server model to {model_name}. Consider using a different port."+ )+ print(f"✅ Successfully updated server model to: {model_name}")+ else:+ print(+ f"✅ Existing server on port {port} is using correct model: {model_name}"+ )++ # Check meta path compatibility if provided+ if passages_file and str(passages_file).endswith(".meta.json"):+ meta_matches = _check_server_meta_path(port, str(passages_file))+ if not meta_matches:+ print(+ f"⚠️ Existing server on port {port} has different meta path. Attempting to update..."+ )+ if not _update_server_meta_path(port, str(passages_file)):+ raise RuntimeError(+ "❌ Failed to update server meta path. This may cause data synchronization issues."+ )+ print(+ f"✅ Successfully updated server meta path to: {passages_file}"+ )+ else:+ print(+ f"✅ Existing server on port {port} is using correct meta path: {passages_file}"+ )++ print(f"✅ Server on port {port} is compatible and ready to use.")return True- print(f"INFO: Starting session-level embedding server for '{self.backend_module_name}'...")+ print(+ f"INFO: Starting session-level embedding server for '{self.backend_module_name}'..."+ )try:command = [sys.executable,- "-m", self.backend_module_name,- "--zmq-port", str(port),- "--model-name", model_name+ "-m",+ self.backend_module_name,+ "--zmq-port",+ str(port),+ "--model-name",+ model_name,]# Add extra arguments for specific backends@@ -76,9 +320,9 @@ class EmbeddingServerManager:stdout=subprocess.PIPE,stderr=subprocess.STDOUT, # Merge stderr into stdout for easier monitoringtext=True,- encoding='utf-8',+ encoding="utf-8",bufsize=1, # Line buffered- universal_newlines=True+ universal_newlines=True,)self.server_port = portprint(f"INFO: Server process started with PID: {self.server_process.pid}")@@ -86,17 +330,21 @@ class EmbeddingServerManager:max_wait, wait_interval = 120, 0.5for _ in range(int(max_wait / wait_interval)):if _check_port(port):- print(f"✅ Embedding server is up and ready for this session.")+ print("✅ Embedding server is up and ready for this session.")log_thread = threading.Thread(target=self._log_monitor, daemon=True)log_thread.start()return Trueif self.server_process.poll() is not None:- print("❌ ERROR: Server process terminated unexpectedly during startup.")+ print(+ "❌ ERROR: Server process terminated unexpectedly during startup."+ )self._print_recent_output()return Falsetime.sleep(wait_interval)- print(f"❌ ERROR: Server process failed to start listening within {max_wait} seconds.")+ print(+ f"❌ ERROR: Server process failed to start listening within {max_wait} seconds."+ )self.stop_server()return False@@ -110,8 +358,7 @@ class EmbeddingServerManager:returntry:# Read any available output- import select- import sys+if select.select([self.server_process.stdout], [], [], 0)[0]:output = self.server_process.stdout.read()if output:@@ -129,19 +376,25 @@ class EmbeddingServerManager:line = self.server_process.stdout.readline()if not line:break- print(f"[{self.backend_module_name} LOG]: {line.strip()}", flush=True)+ print(+ f"[{self.backend_module_name} LOG]: {line.strip()}", flush=True+ )except Exception as e:print(f"Log monitor error: {e}")def stop_server(self):"""Stops the embedding server process if it's running."""if self.server_process and self.server_process.poll() is None:- print(f"INFO: Terminating session server process (PID: {self.server_process.pid})...")+ print(+ f"INFO: Terminating session server process (PID: {self.server_process.pid})..."+ )self.server_process.terminate()try:self.server_process.wait(timeout=5)print("INFO: Server process terminated.")except subprocess.TimeoutExpired:- print("WARNING: Server process did not terminate gracefully, killing it.")+ print(+ "WARNING: Server process did not terminate gracefully, killing it."+ )self.server_process.kill()self.server_process = Nonediff --git a/pyproject.toml b/pyproject.tomlindex 6fcdbdf..3fa4b7b 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -32,6 +32,8 @@ dependencies = ["llama-index-node-parser-docling","ipykernel==6.29.5","msgpack>=1.1.1",+ "llama-index-vector-stores-faiss>=0.4.0",+ "llama-index-embeddings-huggingface>=0.5.5",][project.optional-dependencies]