Skip to content

armgit / LEANN

Public Imported from GitHub
0

[MLsys2026 Best Paper]: https://arxiv.org/abs/2506.08276. RAG on Everything with LEANN. Enjoy 97% storage savings while running a fast, accurate, and 100% private RAG application on your personal device.

Launch embedding server earlier (#176)

Andy Lee9003f913eabd
diff --git a/benchmarks/issue_159.py b/benchmarks/issue_159.py
new file mode 100644
index 0000000..43fd9a5
--- /dev/null
+++ b/benchmarks/issue_159.py
@@ -0,0 +1,98 @@
+#!/usr/bin/env python3
+"""
+Test script to reproduce issue #159: Slow search performance
+Configuration:
+- GPU: A10
+- embedding_model: BAAI/bge-large-zh-v1.5
+- data size: 180M text (~90K chunks)
+- backend: hnsw
+"""
+
+import os
+import time
+from pathlib import Path
+
+from leann.api import LeannBuilder, LeannSearcher
+
+os.environ["LEANN_LOG_LEVEL"] = "DEBUG"
+
+# Configuration matching the issue
+INDEX_PATH = "./test_issue_159.leann"
+EMBEDDING_MODEL = "BAAI/bge-large-zh-v1.5"
+BACKEND_NAME = "hnsw"
+
+
+def generate_test_data(num_chunks=90000, chunk_size=2000):
+ """Generate test data similar to 180MB text (~90K chunks)"""
+ # Each chunk is approximately 2000 characters
+ # 90K chunks * 2000 chars ≈ 180MB
+ chunks = []
+ base_text = (
+ "这是一个测试文档。LEANN是一个创新的向量数据库, 通过图基选择性重计算实现97%的存储节省。"
+ )
+
+ for i in range(num_chunks):
+ chunk = f"{base_text} 文档编号: {i}. " * (chunk_size // len(base_text) + 1)
+ chunks.append(chunk[:chunk_size])
+
+ return chunks
+
+
+def test_search_performance():
+ """Test search performance with different configurations"""
+ print("=" * 80)
+ print("Testing LEANN Search Performance (Issue #159)")
+ print("=" * 80)
+
+ meta_path = Path(f"{INDEX_PATH}.meta.json")
+ if meta_path.exists():
+ print(f"\n✓ Index already exists at {INDEX_PATH}")
+ print(" Skipping build phase. Delete the index to rebuild.")
+ else:
+ print("\n📦 Building index...")
+ print(f" Backend: {BACKEND_NAME}")
+ print(f" Embedding Model: {EMBEDDING_MODEL}")
+ print(" Generating test data (~90K chunks, ~180MB)...")
+
+ chunks = generate_test_data(num_chunks=90000)
+ print(f" Generated {len(chunks)} chunks")
+ print(f" Total text size: {sum(len(c) for c in chunks) / (1024 * 1024):.2f} MB")
+
+ builder = LeannBuilder(
+ backend_name=BACKEND_NAME,
+ embedding_model=EMBEDDING_MODEL,
+ )
+
+ print(" Adding chunks to builder...")
+ start_time = time.time()
+ for i, chunk in enumerate(chunks):
+ builder.add_text(chunk)
+ if (i + 1) % 10000 == 0:
+ print(f" Added {i + 1}/{len(chunks)} chunks...")
+
+ print(" Building index...")
+ build_start = time.time()
+ builder.build_index(INDEX_PATH)
+ build_time = time.time() - build_start
+ print(f" ✓ Index built in {build_time:.2f} seconds")
+
+ # Test search with different complexity values
+ print("\n🔍 Testing search performance...")
+ searcher = LeannSearcher(INDEX_PATH)
+
+ test_query = "LEANN向量数据库存储优化"
+
+ # Test with minimal complexity (8)
+ print("\n Test 4: Minimal complexity (8)")
+ print(f" Query: '{test_query}'")
+ start_time = time.time()
+ results = searcher.search(test_query, top_k=10, complexity=8)
+ search_time = time.time() - start_time
+ print(f" ✓ Search completed in {search_time:.2f} seconds")
+ print(f" Results: {len(results)} items")
+
+ print("\n" + "=" * 80)
+
+
+if __name__ == "__main__":
+ test_search_performance()
diff --git a/packages/leann-backend-hnsw/leann_backend_hnsw/hnsw_embedding_server.py b/packages/leann-backend-hnsw/leann_backend_hnsw/hnsw_embedding_server.py
index 882acbf..b55fa24 100644
--- a/packages/leann-backend-hnsw/leann_backend_hnsw/hnsw_embedding_server.py
+++ b/packages/leann-backend-hnsw/leann_backend_hnsw/hnsw_embedding_server.py
@@ -143,8 +143,6 @@ def create_hnsw_embedding_server(
pass
return str(nid)
- # (legacy ZMQ thread removed; using shutdown-capable server only)
-
def zmq_server_thread_with_shutdown(shutdown_event):
"""ZMQ server thread that respects shutdown signal.
@@ -158,225 +156,238 @@ def create_hnsw_embedding_server(
rep_socket.bind(f"tcp://*:{zmq_port}")
logger.info(f"HNSW ZMQ REP server listening on port {zmq_port}")
rep_socket.setsockopt(zmq.RCVTIMEO, 1000)
- # Keep sends from blocking during shutdown; fail fast and drop on close
rep_socket.setsockopt(zmq.SNDTIMEO, 1000)
rep_socket.setsockopt(zmq.LINGER, 0)
- # Track last request type/length for shape-correct fallbacks
- last_request_type = "unknown" # 'text' | 'distance' | 'embedding' | 'unknown'
+ last_request_type = "unknown"
last_request_length = 0
+ def _build_safe_fallback():
+ if last_request_type == "distance":
+ large_distance = 1e9
+ fallback_len = max(0, int(last_request_length))
+ return [[large_distance] * fallback_len]
+ if last_request_type == "embedding":
+ bsz = max(0, int(last_request_length))
+ dim = max(0, int(embedding_dim))
+ if dim > 0:
+ return [[bsz, dim], [0.0] * (bsz * dim)]
+ return [[0, 0], []]
+ if last_request_type == "text":
+ return []
+ return [[0, int(embedding_dim) if embedding_dim > 0 else 0], []]
+
+ def _handle_text_embedding(request: list[str]) -> None:
+ nonlocal last_request_type, last_request_length
+
+ e2e_start = time.time()
+ last_request_type = "text"
+ last_request_length = len(request)
+ embeddings = compute_embeddings(
+ request,
+ model_name,
+ mode=embedding_mode,
+ provider_options=PROVIDER_OPTIONS,
+ )
+ rep_socket.send(msgpack.packb(embeddings.tolist()))
+ e2e_end = time.time()
+ logger.info(f"⏱️ Direct text embedding E2E time: {e2e_end - e2e_start:.6f}s")
+
+ def _handle_distance_request(request: list[Any]) -> None:
+ nonlocal last_request_type, last_request_length
+
+ e2e_start = time.time()
+ node_ids = request[0]
+ if len(node_ids) == 1 and isinstance(node_ids[0], list):
+ node_ids = node_ids[0]
+ query_vector = np.array(request[1], dtype=np.float32)
+ last_request_type = "distance"
+ last_request_length = len(node_ids)
+
+ logger.debug("Distance calculation request received")
+ logger.debug(f" Node IDs: {node_ids}")
+ logger.debug(f" Query vector dim: {len(query_vector)}")
+
+ texts: list[str] = []
+ found_indices: list[int] = []
+ for idx, nid in enumerate(node_ids):
+ try:
+ passage_id = _map_node_id(nid)
+ passage_data = passages.get_passage(passage_id)
+ txt = passage_data.get("text", "")
+ if isinstance(txt, str) and len(txt) > 0:
+ texts.append(txt)
+ found_indices.append(idx)
+ else:
+ logger.error(f"Empty text for passage ID {passage_id}")
+ except KeyError:
+ logger.error(f"Passage ID {nid} not found")
+ except Exception as exc:
+ logger.error(f"Exception looking up passage ID {nid}: {exc}")
+
+ large_distance = 1e9
+ response_distances = [large_distance] * len(node_ids)
+
+ if texts:
+ try:
+ embeddings = compute_embeddings(
+ texts,
+ model_name,
+ mode=embedding_mode,
+ provider_options=PROVIDER_OPTIONS,
+ )
+ logger.info(
+ f"Computed embeddings for {len(texts)} texts, shape: {embeddings.shape}"
+ )
+ if distance_metric == "l2":
+ partial = np.sum(
+ np.square(embeddings - query_vector.reshape(1, -1)), axis=1
+ )
+ else:
+ partial = -np.dot(embeddings, query_vector)
+
+ for pos, dval in zip(found_indices, partial.flatten().tolist()):
+ response_distances[pos] = float(dval)
+ except Exception as exc:
+ logger.error(f"Distance computation error, using sentinels: {exc}")
+
+ rep_socket.send(msgpack.packb([response_distances], use_single_float=True))
+ e2e_end = time.time()
+ logger.info(f"⏱️ Distance calculation E2E time: {e2e_end - e2e_start:.6f}s")
+
+ def _handle_embedding_by_id(request: Any) -> None:
+ nonlocal last_request_type, last_request_length
+
+ if isinstance(request, list) and len(request) == 1 and isinstance(request[0], list):
+ node_ids = request[0]
+ elif isinstance(request, list):
+ node_ids = request
+ else:
+ node_ids = []
+
+ e2e_start = time.time()
+ last_request_type = "embedding"
+ last_request_length = len(node_ids)
+ logger.info(f"ZMQ received {len(node_ids)} node IDs for embedding fetch")
+
+ if embedding_dim <= 0:
+ dims = [0, 0]
+ flat_data: list[float] = []
+ else:
+ dims = [len(node_ids), embedding_dim]
+ flat_data = [0.0] * (dims[0] * dims[1])
+
+ texts: list[str] = []
+ found_indices: list[int] = []
+ for idx, nid in enumerate(node_ids):
+ try:
+ passage_id = _map_node_id(nid)
+ passage_data = passages.get_passage(passage_id)
+ txt = passage_data.get("text", "")
+ if isinstance(txt, str) and len(txt) > 0:
+ texts.append(txt)
+ found_indices.append(idx)
+ else:
+ logger.error(f"Empty text for passage ID {passage_id}")
+ except KeyError:
+ logger.error(f"Passage with ID {nid} not found")
+ except Exception as exc:
+ logger.error(f"Exception looking up passage ID {nid}: {exc}")
+
+ if texts:
+ try:
+ embeddings = compute_embeddings(
+ texts,
+ model_name,
+ mode=embedding_mode,
+ provider_options=PROVIDER_OPTIONS,
+ )
+ logger.info(
+ f"Computed embeddings for {len(texts)} texts, shape: {embeddings.shape}"
+ )
+
+ if np.isnan(embeddings).any() or np.isinf(embeddings).any():
+ logger.error(
+ f"NaN or Inf detected in embeddings! Requested IDs: {node_ids[:5]}..."
+ )
+ dims = [0, embedding_dim]
+ flat_data = []
+ else:
+ emb_f32 = np.ascontiguousarray(embeddings, dtype=np.float32)
+ flat = emb_f32.flatten().tolist()
+ for j, pos in enumerate(found_indices):
+ start = pos * embedding_dim
+ end = start + embedding_dim
+ if end <= len(flat_data):
+ flat_data[start:end] = flat[
+ j * embedding_dim : (j + 1) * embedding_dim
+ ]
+ except Exception as exc:
+ logger.error(f"Embedding computation error, returning zeros: {exc}")
+
+ response_payload = [dims, flat_data]
+ rep_socket.send(msgpack.packb(response_payload, use_single_float=True))
+ e2e_end = time.time()
+ logger.info(f"⏱️ Fallback Embed by Id E2E time: {e2e_end - e2e_start:.6f}s")
+
try:
while not shutdown_event.is_set():
try:
- e2e_start = time.time()
logger.debug("🔍 Waiting for ZMQ message...")
request_bytes = rep_socket.recv()
+ except zmq.Again:
+ continue
- # Rest of the processing logic (same as original)
+ try:
request = msgpack.unpackb(request_bytes)
+ except Exception as exc:
+ if shutdown_event.is_set():
+ logger.info("Shutdown in progress, ignoring ZMQ error")
+ break
+ logger.error(f"Error unpacking ZMQ message: {exc}")
+ try:
+ safe = _build_safe_fallback()
+ rep_socket.send(msgpack.packb(safe, use_single_float=True))
+ except Exception:
+ pass
+ continue
- if len(request) == 1 and request[0] == "__QUERY_MODEL__":
- response_bytes = msgpack.packb([model_name])
- rep_socket.send(response_bytes)
- continue
-
- # Handle direct text embedding request
+ try:
+ # Model query
if (
+ isinstance(request, list)
+ and len(request) == 1
+ and request[0] == "__QUERY_MODEL__"
+ ):
+ rep_socket.send(msgpack.packb([model_name]))
+ # Direct text embedding
+ elif (
isinstance(request, list)
and request
and all(isinstance(item, str) for item in request)
):
- last_request_type = "text"
- last_request_length = len(request)
- embeddings = compute_embeddings(
- request,
- model_name,
- mode=embedding_mode,
- provider_options=PROVIDER_OPTIONS,
- )
- rep_socket.send(msgpack.packb(embeddings.tolist()))
- e2e_end = time.time()
- logger.info(f"⏱️ Text embedding E2E time: {e2e_end - e2e_start:.6f}s")
- continue
-
- # Handle distance calculation request: [[ids], [query_vector]]
- if (
+ _handle_text_embedding(request)
+ # Distance calculation: [[ids], [query_vector]]
+ elif (
isinstance(request, list)
and len(request) == 2
and isinstance(request[0], list)
and isinstance(request[1], list)
):
- node_ids = request[0]
- # Handle nested [[ids]] shape defensively
- if len(node_ids) == 1 and isinstance(node_ids[0], list):
- node_ids = node_ids[0]
- query_vector = np.array(request[1], dtype=np.float32)
- last_request_type = "distance"
- last_request_length = len(node_ids)
-
- logger.debug("Distance calculation request received")
- logger.debug(f" Node IDs: {node_ids}")
- logger.debug(f" Query vector dim: {len(query_vector)}")
-
- # Gather texts for found ids
- texts: list[str] = []
- found_indices: list[int] = []
- for idx, nid in enumerate(node_ids):
- try:
- passage_id = _map_node_id(nid)
- passage_data = passages.get_passage(passage_id)
- txt = passage_data.get("text", "")
- if isinstance(txt, str) and len(txt) > 0:
- texts.append(txt)
- found_indices.append(idx)
- else:
- logger.error(f"Empty text for passage ID {passage_id}")
- except KeyError:
- logger.error(f"Passage ID {nid} not found")
- except Exception as e:
- logger.error(f"Exception looking up passage ID {nid}: {e}")
-
- # Prepare full-length response with large sentinel values
- large_distance = 1e9
- response_distances = [large_distance] * len(node_ids)
-
- if texts:
- try:
- embeddings = compute_embeddings(
- texts,
- model_name,
- mode=embedding_mode,
- provider_options=PROVIDER_OPTIONS,
- )
- logger.info(
- f"Computed embeddings for {len(texts)} texts, shape: {embeddings.shape}"
- )
- if distance_metric == "l2":
- partial = np.sum(
- np.square(embeddings - query_vector.reshape(1, -1)), axis=1
- )
- else: # mips or cosine
- partial = -np.dot(embeddings, query_vector)
-
- for pos, dval in zip(found_indices, partial.flatten().tolist()):
- response_distances[pos] = float(dval)
- except Exception as e:
- logger.error(f"Distance computation error, using sentinels: {e}")
-
- # Send response in expected shape [[distances]]
- rep_socket.send(msgpack.packb([response_distances], use_single_float=True))
- e2e_end = time.time()
- logger.info(f"⏱️ Distance calculation E2E time: {e2e_end - e2e_start:.6f}s")
- continue
-
- # Fallback: treat as embedding-by-id request
- if (
- isinstance(request, list)
- and len(request) == 1
- and isinstance(request[0], list)
- ):
- node_ids = request[0]
- elif isinstance(request, list):
- node_ids = request
- else:
- node_ids = []
- last_request_type = "embedding"
- last_request_length = len(node_ids)
- logger.info(f"ZMQ received {len(node_ids)} node IDs for embedding fetch")
-
- # Preallocate zero-filled flat data for robustness
- if embedding_dim <= 0:
- dims = [0, 0]
- flat_data: list[float] = []
- else:
- dims = [len(node_ids), embedding_dim]
- flat_data = [0.0] * (dims[0] * dims[1])
-
- # Collect texts for found ids
- texts: list[str] = []
- found_indices: list[int] = []
- for idx, nid in enumerate(node_ids):
- try:
- passage_id = _map_node_id(nid)
- passage_data = passages.get_passage(passage_id)
- txt = passage_data.get("text", "")
- if isinstance(txt, str) and len(txt) > 0:
- texts.append(txt)
- found_indices.append(idx)
- else:
- logger.error(f"Empty text for passage ID {passage_id}")
- except KeyError:
- logger.error(f"Passage with ID {nid} not found")
- except Exception as e:
- logger.error(f"Exception looking up passage ID {nid}: {e}")
-
- if texts:
- try:
- embeddings = compute_embeddings(
- texts,
- model_name,
- mode=embedding_mode,
- provider_options=PROVIDER_OPTIONS,
- )
- logger.info(
- f"Computed embeddings for {len(texts)} texts, shape: {embeddings.shape}"
- )
-
- if np.isnan(embeddings).any() or np.isinf(embeddings).any():
- logger.error(
- f"NaN or Inf detected in embeddings! Requested IDs: {node_ids[:5]}..."
- )
- dims = [0, embedding_dim]
- flat_data = []
- else:
- emb_f32 = np.ascontiguousarray(embeddings, dtype=np.float32)
- flat = emb_f32.flatten().tolist()
- for j, pos in enumerate(found_indices):
- start = pos * embedding_dim
- end = start + embedding_dim
- if end <= len(flat_data):
- flat_data[start:end] = flat[
- j * embedding_dim : (j + 1) * embedding_dim
- ]
- except Exception as e:
- logger.error(f"Embedding computation error, returning zeros: {e}")
-
- response_payload = [dims, flat_data]
- response_bytes = msgpack.packb(response_payload, use_single_float=True)
-
- rep_socket.send(response_bytes)
- e2e_end = time.time()
- logger.info(f"⏱️ ZMQ E2E time: {e2e_end - e2e_start:.6f}s")
-
- except zmq.Again:
- # Timeout - check shutdown_event and continue
- continue
- except Exception as e:
- if not shutdown_event.is_set():
- logger.error(f"Error in ZMQ server loop: {e}")
- # Shape-correct fallback
- try:
- if last_request_type == "distance":
- large_distance = 1e9
- fallback_len = max(0, int(last_request_length))
- safe = [[large_distance] * fallback_len]
- elif last_request_type == "embedding":
- bsz = max(0, int(last_request_length))
- dim = max(0, int(embedding_dim))
- safe = (
- [[bsz, dim], [0.0] * (bsz * dim)] if dim > 0 else [[0, 0], []]
- )
- elif last_request_type == "text":
- safe = [] # direct text embeddings expectation is a flat list
- else:
- safe = [[0, int(embedding_dim) if embedding_dim > 0 else 0], []]
- rep_socket.send(msgpack.packb(safe, use_single_float=True))
- except Exception:
- pass
+ _handle_distance_request(request)
+ # Embedding-by-id fallback
else:
+ _handle_embedding_by_id(request)
+ except Exception as exc:
+ if shutdown_event.is_set():
logger.info("Shutdown in progress, ignoring ZMQ error")
break
+ logger.error(f"Error in ZMQ server loop: {exc}")
+ try:
+ safe = _build_safe_fallback()
+ rep_socket.send(msgpack.packb(safe, use_single_float=True))
+ except Exception:
+ pass
finally:
try:
rep_socket.close(0)
diff --git a/packages/leann-backend-hnsw/third_party/faiss b/packages/leann-backend-hnsw/third_party/faiss
index e2d243c..301bf24 160000
--- a/packages/leann-backend-hnsw/third_party/faiss
+++ b/packages/leann-backend-hnsw/third_party/faiss
@@ -1 +1 @@
-Subproject commit e2d243c40ddc142b8c57c067c0441694f3c22121
+Subproject commit 301bf24f1467c6a09e8758cfb70aad856ce421db
diff --git a/packages/leann-core/src/leann/api.py b/packages/leann-core/src/leann/api.py
index d64d433..07806f0 100644
--- a/packages/leann-core/src/leann/api.py
+++ b/packages/leann-core/src/leann/api.py
@@ -864,7 +864,13 @@ class LeannBuilder:
class LeannSearcher:
- def __init__(self, index_path: str, enable_warmup: bool = False, **backend_kwargs):
+ def __init__(
+ self,
+ index_path: str,
+ enable_warmup: bool = True,
+ recompute_embeddings: bool = True,
+ **backend_kwargs,
+ ):
# Fix path resolution for Colab and other environments
if not Path(index_path).is_absolute():
index_path = str(Path(index_path).resolve())
@@ -895,14 +901,32 @@ class LeannSearcher:
backend_factory = BACKEND_REGISTRY.get(backend_name)
if backend_factory is None:
raise ValueError(f"Backend '{backend_name}' not found.")
+
+ # Global recompute flag for this searcher (explicit knob, default True)
+ self.recompute_embeddings: bool = bool(recompute_embeddings)
+
+ # Warmup flag: keep using the existing enable_warmup parameter,
+ # but default it to True so cold-start happens earlier.
+ self._warmup: bool = bool(enable_warmup)
+
final_kwargs = {**self.meta_data.get("backend_kwargs", {}), **backend_kwargs}
- final_kwargs["enable_warmup"] = enable_warmup
+ final_kwargs["enable_warmup"] = self._warmup
if self.embedding_options:
final_kwargs.setdefault("embedding_options", self.embedding_options)
self.backend_impl: LeannBackendSearcherInterface = backend_factory.searcher(
index_path, **final_kwargs
)
+ # Optional one-shot warmup at construction time to hide cold-start latency.
+ if self._warmup:
+ try:
+ _ = self.backend_impl.compute_query_embedding(
+ "__LEANN_WARMUP__",
+ use_server_if_available=self.recompute_embeddings,
+ )
+ except Exception as exc:
+ logger.warning(f"Warmup embedding failed (ignored): {exc}")
+
def search(
self,
query: str,
@@ -910,7 +934,7 @@ class LeannSearcher:
complexity: int = 64,
beam_width: int = 1,
prune_ratio: float = 0.0,
- recompute_embeddings: bool = True,
+ recompute_embeddings: Optional[bool] = None,
pruning_strategy: Literal["global", "local", "proportional"] = "global",
expected_zmq_port: int = 5557,
metadata_filters: Optional[dict[str, dict[str, Union[str, int, float, bool, list]]]] = None,
@@ -928,7 +952,8 @@ class LeannSearcher:
complexity: Search complexity/candidate list size, higher = more accurate but slower
beam_width: Number of parallel search paths/IO requests per iteration
prune_ratio: Ratio of neighbors to prune via approximate distance (0.0-1.0)
- recompute_embeddings: Whether to fetch fresh embeddings from server vs use stored codes
+ recompute_embeddings: (Deprecated) Per-call override for recompute mode.
+ Configure this at LeannSearcher(..., recompute_embeddings=...) instead.
pruning_strategy: Candidate selection strategy - "global" (default), "local", or "proportional"
expected_zmq_port: ZMQ port for embedding server communication
metadata_filters: Optional filters to apply to search results based on metadata.
@@ -967,8 +992,19 @@ class LeannSearcher:
zmq_port = None
+ # Resolve effective recompute flag for this search.
+ if recompute_embeddings is not None:
+ logger.warning(
+ "LeannSearcher.search(..., recompute_embeddings=...) is deprecated and "
+ "will be removed in a future version. Configure recompute at "
+ "LeannSearcher(..., recompute_embeddings=...) instead."
+ )
+ effective_recompute = bool(recompute_embeddings)
+ else:
+ effective_recompute = self.recompute_embeddings
+
start_time = time.time()
- if recompute_embeddings:
+ if effective_recompute:
zmq_port = self.backend_impl._ensure_server_running(
self.meta_path_str,
port=expected_zmq_port,
@@ -995,7 +1031,7 @@ class LeannSearcher:
query_embedding = self.backend_impl.compute_query_embedding(
query,
- use_server_if_available=recompute_embeddings,
+ use_server_if_available=effective_recompute,
zmq_port=zmq_port,
query_template=query_template,
)
@@ -1008,7 +1044,7 @@ class LeannSearcher:
"complexity": complexity,
"beam_width": beam_width,
"prune_ratio": prune_ratio,
- "recompute_embeddings": recompute_embeddings,
+ "recompute_embeddings": effective_recompute,
"pruning_strategy": pruning_strategy,
"zmq_port": zmq_port,
}
diff --git a/packages/leann-core/src/leann/embedding_compute.py b/packages/leann-core/src/leann/embedding_compute.py
index 70c1beb..04bb15c 100644
--- a/packages/leann-core/src/leann/embedding_compute.py
+++ b/packages/leann-core/src/leann/embedding_compute.py
@@ -335,6 +335,10 @@ def compute_embeddings(
Normalized embeddings array, shape: (len(texts), embedding_dim)
"""
provider_options = provider_options or {}
+ wrapper_start_time = time.time()
+ logger.debug(
+ f"[compute_embeddings] entry: mode={mode}, model='{model_name}', text_count={len(texts)}"
+ )
# Allow batch_size override from provider_options (disables adaptive_optimization)
if "batch_size" in provider_options:
@@ -342,7 +346,8 @@ def compute_embeddings(
adaptive_optimization = False # User-specified batch_size takes precedence
if mode == "sentence-transformers":
- return compute_embeddings_sentence_transformers(
+ inner_start_time = time.time()
+ result = compute_embeddings_sentence_transformers(
texts,
model_name,
is_build=is_build,
@@ -351,6 +356,14 @@ def compute_embeddings(
manual_tokenize=manual_tokenize,
max_length=max_length,
)
+ inner_end_time = time.time()
+ wrapper_end_time = time.time()
+ logger.debug(
+ "[compute_embeddings] sentence-transformers timings: "
+ f"inner={inner_end_time - inner_start_time:.6f}s, "
+ f"wrapper_total={wrapper_end_time - wrapper_start_time:.6f}s"
+ )
+ return result
elif mode == "openai":
return compute_embeddings_openai(
texts,
@@ -398,6 +411,7 @@ def compute_embeddings_sentence_transformers(
is_build: Whether this is a build operation (shows progress bar)
adaptive_optimization: Whether to use adaptive optimization based on batch size
"""
+ outer_start_time = time.time()
# Handle empty input
if not texts:
raise ValueError("Cannot compute embeddings for empty text list")
@@ -433,7 +447,14 @@ def compute_embeddings_sentence_transformers(
# Create cache key
cache_key = f"sentence_transformers_{model_name}_{device}_{use_fp16}_optimized"
+ pre_model_init_end_time = time.time()
+ logger.debug(
+ "compute_embeddings_sentence_transformers pre-model-init time "
+ f"(device/batch selection etc.): {pre_model_init_end_time - outer_start_time:.6f}s"
+ )
+
# Check if model is already cached
+ start_time = time.time()
if cache_key in _model_cache:
logger.info(f"Using cached optimized model: {model_name}")
model = _model_cache[cache_key]
@@ -574,10 +595,13 @@ def compute_embeddings_sentence_transformers(
_model_cache[cache_key] = model
logger.info(f"Model cached: {cache_key}")
- # Compute embeddings with optimized inference mode
- logger.info(
- f"Starting embedding computation... (batch_size: {batch_size}, manual_tokenize={manual_tokenize})"
- )
+ end_time = time.time()
+
+ # Compute embeddings with optimized inference mode
+ logger.info(
+ f"Starting embedding computation... (batch_size: {batch_size}, manual_tokenize={manual_tokenize})"
+ )
+ logger.info(f"start sentence transformers {model} takes {end_time - start_time}")
start_time = time.time()
if not manual_tokenize:
@@ -598,32 +622,46 @@ def compute_embeddings_sentence_transformers(
except Exception:
pass
else:
- # Manual tokenization + forward pass using HF AutoTokenizer/AutoModel
+ # Manual tokenization + forward pass using HF AutoTokenizer/AutoModel.
+ # This path is reserved for an aggressively optimized FP pipeline
+ # (no quantization), mainly for experimentation.
try:
from transformers import AutoModel, AutoTokenizer # type: ignore
except Exception as e:
raise ImportError(f"transformers is required for manual_tokenize=True: {e}")
- # Cache tokenizer and model
tok_cache_key = f"hf_tokenizer_{model_name}"
- mdl_cache_key = f"hf_model_{model_name}_{device}_{use_fp16}"
+ mdl_cache_key = f"hf_model_{model_name}_{device}_{use_fp16}_fp"
+
if tok_cache_key in _model_cache and mdl_cache_key in _model_cache:
hf_tokenizer = _model_cache[tok_cache_key]
hf_model = _model_cache[mdl_cache_key]
- logger.info("Using cached HF tokenizer/model for manual path")
+ logger.info("Using cached HF tokenizer/model for manual FP path")
else:
- logger.info("Loading HF tokenizer/model for manual tokenization path")
+ logger.info("Loading HF tokenizer/model for manual FP path")
hf_tokenizer = AutoTokenizer.from_pretrained(model_name, use_fast=True)
+
torch_dtype = torch.float16 if (use_fp16 and device == "cuda") else torch.float32
- hf_model = AutoModel.from_pretrained(model_name, torch_dtype=torch_dtype)
+ hf_model = AutoModel.from_pretrained(
+ model_name,
+ torch_dtype=torch_dtype,
+ )
hf_model.to(device)
+
hf_model.eval()
# Optional compile on supported devices
if device in ["cuda", "mps"]:
try:
- hf_model = torch.compile(hf_model, mode="reduce-overhead", dynamic=True) # type: ignore
- except Exception:
- pass
+ hf_model = torch.compile( # type: ignore
+ hf_model, mode="reduce-overhead", dynamic=True
+ )
+ logger.info(
+ f"Applied torch.compile to HF model for {model_name} "
+ f"(device={device}, dtype={torch_dtype})"
+ )
+ except Exception as exc:
+ logger.warning(f"torch.compile optimization failed: {exc}")
+
_model_cache[tok_cache_key] = hf_tokenizer
_model_cache[mdl_cache_key] = hf_model
@@ -649,7 +687,6 @@ def compute_embeddings_sentence_transformers(
for start_index in batch_iter:
end_index = min(start_index + batch_size, len(texts))
batch_texts = texts[start_index:end_index]
- tokenize_start_time = time.time()
inputs = hf_tokenizer(
batch_texts,
padding=True,
@@ -657,34 +694,17 @@ def compute_embeddings_sentence_transformers(
max_length=max_length,
return_tensors="pt",
)
- tokenize_end_time = time.time()
- logger.info(
- f"Tokenize time taken: {tokenize_end_time - tokenize_start_time} seconds"
- )
- # Print shapes of all input tensors for debugging
- for k, v in inputs.items():
- print(f"inputs[{k!r}] shape: {getattr(v, 'shape', type(v))}")
- to_device_start_time = time.time()
inputs = {k: v.to(device) for k, v in inputs.items()}
- to_device_end_time = time.time()
- logger.info(
- f"To device time taken: {to_device_end_time - to_device_start_time} seconds"
- )
- forward_start_time = time.time()
outputs = hf_model(**inputs)
- forward_end_time = time.time()
- logger.info(f"Forward time taken: {forward_end_time - forward_start_time} seconds")
last_hidden_state = outputs.last_hidden_state # (B, L, H)
attention_mask = inputs.get("attention_mask")
if attention_mask is None:
- # Fallback: assume all tokens are valid
pooled = last_hidden_state.mean(dim=1)
else:
mask = attention_mask.unsqueeze(-1).to(last_hidden_state.dtype)
masked = last_hidden_state * mask
lengths = mask.sum(dim=1).clamp(min=1)
pooled = masked.sum(dim=1) / lengths
- # Move to CPU float32
batch_embeddings = pooled.detach().to("cpu").float().numpy()
all_embeddings.append(batch_embeddings)
@@ -704,6 +724,12 @@ def compute_embeddings_sentence_transformers(
if np.isnan(embeddings).any() or np.isinf(embeddings).any():
raise RuntimeError(f"Detected NaN or Inf values in embeddings, model: {model_name}")
+ outer_end_time = time.time()
+ logger.debug(
+ "compute_embeddings_sentence_transformers total time "
+ f"(function entry -> return): {outer_end_time - outer_start_time:.6f}s"
+ )
+
return embeddings