[MLsys2026 Best Paper]: https://arxiv.org/abs/2506.08276. RAG on Everything with LEANN. Enjoy 97% storage savings while running a fast, accurate, and 100% private RAG application on your personal device.
Python aifaissgpt-osslangchainllama-indexllmlocalstorageoffline-firstollamaprivacypythonragretrieval-augmented-generationvector-databasevector-searchvectors
fix(embeddings): make CUDA batch size configurable for small GPUs (#368)
Aakash Suresh0ddc20fa979b
diff --git a/packages/leann-core/src/leann/cli.py b/packages/leann-core/src/leann/cli.pyindex c96fd7c..d565c73 100644--- a/packages/leann-core/src/leann/cli.py+++ b/packages/leann-core/src/leann/cli.py@@ -297,6 +297,12 @@ Examples:default=None,help="Prompt template to prepend to all texts for embedding (e.g., 'query: ' for search)",)+ build_parser.add_argument(+ "--embedding-batch-size",+ type=int,+ default=None,+ help="Embedding batch size for sentence-transformers (overrides device defaults; also set LEANN_CUDA_BATCH_SIZE env)",+ )build_parser.add_argument("--query-prompt-template",type=str,@@ -745,6 +751,12 @@ Examples:help="OpenAI-compatible embedding base URL",)p.add_argument("--embedding-api-key", type=str, default=None, help="Embedding API key")+ p.add_argument(+ "--embedding-batch-size",+ type=int,+ default=None,+ help="Embedding batch size for sentence-transformers",+ )p.add_argument("--max-count", type=int, default=1000, help="Max items to index (default: 1000)")@@ -1858,6 +1870,9 @@ Examples:opts["query_prompt_template"] = args.query_prompt_templateelif args.embedding_prompt_template:opts["prompt_template"] = args.embedding_prompt_template+ batch_size = getattr(args, "embedding_batch_size", None)+ if batch_size is not None:+ opts["batch_size"] = batch_sizereturn optsdef _build_config_from_args(diff --git a/packages/leann-core/src/leann/embedding_compute.py b/packages/leann-core/src/leann/embedding_compute.pyindex 7e51d4f..654c726 100644--- a/packages/leann-core/src/leann/embedding_compute.py+++ b/packages/leann-core/src/leann/embedding_compute.py@@ -320,6 +320,104 @@ def _query_lmstudio_context_limit(model_name: str, base_url: str) -> Optional[in# Global model cache to avoid repeated loading_model_cache: dict[str, Any] = {}+_DEFAULT_CUDA_BATCH_SIZE = 256+_DEFAULT_MPS_BATCH_SIZE = 128+++def _parse_positive_int_env(name: str, default: int) -> int:+ raw = os.getenv(name)+ if raw is None or raw.strip() == "":+ return default+ try:+ value = int(raw)+ except ValueError:+ logger.warning("Invalid %s=%r; using default %d", name, raw, default)+ return default+ if value < 1:+ logger.warning("%s must be >= 1; using default %d", name, default)+ return default+ return value+++def _resolve_cpu_thread_count() -> int:+ return _parse_positive_int_env("LEANN_CPU_THREADS", min(8, os.cpu_count() or 4))+++def _resolve_adaptive_batch_size(device: str, model_name: str) -> int:+ if device == "cuda":+ return _parse_positive_int_env("LEANN_CUDA_BATCH_SIZE", _DEFAULT_CUDA_BATCH_SIZE)+ if device == "mps":+ default = 32 if model_name == "Qwen/Qwen3-Embedding-0.6B" else _DEFAULT_MPS_BATCH_SIZE+ return _parse_positive_int_env("LEANN_MPS_BATCH_SIZE", default)+ return 32+++def _cap_cuda_batch_by_vram(requested: int, max_length: int = 512) -> int:+ auto = os.getenv("LEANN_CUDA_AUTO_BATCH", "1").lower()+ if auto in ("0", "false", "no"):+ return requested+ import torch++ if not torch.cuda.is_available():+ return requested+ try:+ free_bytes, _total = torch.cuda.mem_get_info()+ except Exception:+ return requested++ # Eager-attention peak memory scales ~O(seq^2) per sequence in the batch.+ bytes_per_seq = max(8_000_000, max_length * max_length * 32)+ budget = int(free_bytes * 0.2)+ max_by_vram = max(1, budget // bytes_per_seq)+ capped = min(requested, max_by_vram)+ if capped < requested:+ logger.info(+ "Capping CUDA embedding batch size %d -> %d (%.2f GiB free VRAM)",+ requested,+ capped,+ free_bytes / (1024**3),+ )+ return capped+++def _encode_with_oom_retry(+ model: _SentenceTransformerLike,+ texts: list[str],+ batch_size: int,+ *,+ is_build: bool,+ device: str,+) -> Any:+ import torch++ current = batch_size+ while True:+ try:+ with torch.inference_mode():+ return model.encode(+ texts,+ batch_size=current,+ show_progress_bar=is_build,+ convert_to_numpy=True,+ normalize_embeddings=False,+ device=device,+ )+ except RuntimeError as exc:+ oom = "out of memory" in str(exc).lower()+ if torch.cuda.is_available():+ oom = oom or isinstance(exc, torch.cuda.OutOfMemoryError)+ if not oom or current <= 1:+ raise+ if torch.cuda.is_available():+ torch.cuda.empty_cache()+ next_batch = max(1, current // 2)+ logger.warning(+ "CUDA OOM at embedding batch_size=%d; retrying with batch_size=%d",+ current,+ next_batch,+ )+ current = next_batch+def compute_embeddings(texts: list[str],@@ -449,14 +547,7 @@ def compute_embeddings_sentence_transformers(# Apply optimizations based on benchmark resultsif adaptive_optimization:- # Use optimal batch_size constants for different devices based on benchmark results- if device == "mps":- batch_size = 128 # MPS optimal batch size from benchmark- if model_name == "Qwen/Qwen3-Embedding-0.6B":- batch_size = 32- elif device == "cuda":- batch_size = 256 # CUDA optimal batch size- # Keep original batch_size for CPU+ batch_size = _resolve_adaptive_batch_size(device, model_name)# Create cache keycache_key = f"sentence_transformers_{model_name}_{device}_{use_fp16}_optimized"@@ -493,7 +584,7 @@ def compute_embeddings_sentence_transformers(passelif device == "cpu":# TODO: Haven't tested this yet- torch.set_num_threads(min(8, os.cpu_count() or 4))+ torch.set_num_threads(_resolve_cpu_thread_count())try:torch.backends.mkldnn.enabled = Trueexcept AttributeError:@@ -617,18 +708,19 @@ def compute_embeddings_sentence_transformers()logger.info(f"start sentence transformers {model} takes {end_time - start_time}")+ if device == "cuda" and adaptive_optimization:+ batch_size = _cap_cuda_batch_by_vram(batch_size, max_length=max_length)+start_time = time.time()if not manual_tokenize:# Use SentenceTransformer's optimized encode path (default)- with torch.inference_mode():- embeddings = model.encode(- texts,- batch_size=batch_size,- show_progress_bar=is_build, # Don't show progress bar in server environment- convert_to_numpy=True,- normalize_embeddings=False,- device=device,- )+ embeddings = _encode_with_oom_retry(+ model,+ texts,+ batch_size,+ is_build=is_build,+ device=device,+ )# Synchronize if CUDA to measure accurate wall timetry:if torch.cuda.is_available():diff --git a/tests/test_embedding_batch_size.py b/tests/test_embedding_batch_size.pynew file mode 100644index 0000000..207b75a--- /dev/null+++ b/tests/test_embedding_batch_size.py@@ -0,0 +1,69 @@+"""Tests for embedding batch size and CPU thread configuration."""++from unittest.mock import patch++from leann.embedding_compute import (+ _cap_cuda_batch_by_vram,+ _parse_positive_int_env,+ _resolve_adaptive_batch_size,+ _resolve_cpu_thread_count,+)+++def test_parse_positive_int_env_default(monkeypatch):+ monkeypatch.delenv("LEANN_TEST_INT", raising=False)+ assert _parse_positive_int_env("LEANN_TEST_INT", 256) == 256+++def test_parse_positive_int_env_override(monkeypatch):+ monkeypatch.setenv("LEANN_TEST_INT", "32")+ assert _parse_positive_int_env("LEANN_TEST_INT", 256) == 32+++def test_parse_positive_int_env_invalid(monkeypatch):+ monkeypatch.setenv("LEANN_TEST_INT", "not-a-number")+ assert _parse_positive_int_env("LEANN_TEST_INT", 256) == 256+++def test_resolve_adaptive_batch_size_cuda(monkeypatch):+ monkeypatch.setenv("LEANN_CUDA_BATCH_SIZE", "64")+ assert _resolve_adaptive_batch_size("cuda", "BAAI/bge-base-en-v1.5") == 64+++def test_resolve_adaptive_batch_size_mps_qwen(monkeypatch):+ monkeypatch.delenv("LEANN_MPS_BATCH_SIZE", raising=False)+ assert _resolve_adaptive_batch_size("mps", "Qwen/Qwen3-Embedding-0.6B") == 32+++def test_resolve_cpu_threads(monkeypatch):+ monkeypatch.setenv("LEANN_CPU_THREADS", "16")+ assert _resolve_cpu_thread_count() == 16+++def test_cap_cuda_batch_by_vram_disabled(monkeypatch):+ monkeypatch.setenv("LEANN_CUDA_AUTO_BATCH", "0")+ with patch("torch.cuda.is_available", return_value=True):+ with patch("torch.cuda.mem_get_info", return_value=(100, 1000)):+ assert _cap_cuda_batch_by_vram(256) == 256+++def test_cap_cuda_batch_by_vram_small_gpu(monkeypatch):+ monkeypatch.delenv("LEANN_CUDA_AUTO_BATCH", raising=False)+ # Typical free VRAM on a 4 GiB GPU after loading a base-sized encoder.+ one_gb = 1024**3+ with patch("torch.cuda.is_available", return_value=True):+ with patch("torch.cuda.mem_get_info", return_value=(one_gb, 4 * one_gb)):+ capped = _cap_cuda_batch_by_vram(256, max_length=512)+ assert capped < 256+ assert capped >= 1+++def test_cap_cuda_batch_by_vram_four_gb_gpu(monkeypatch):+ """Regression: 4 GiB RTX A1000 reports ~3.2 GiB free; cap should land near 76."""+ monkeypatch.delenv("LEANN_CUDA_AUTO_BATCH", raising=False)+ free_vram = int(3.2 * 1024**3)+ with patch("torch.cuda.is_available", return_value=True):+ with patch("torch.cuda.mem_get_info", return_value=(free_vram, 4 * 1024**3)):+ capped = _cap_cuda_batch_by_vram(256, max_length=512)+ assert capped <= 85+ assert capped >= 1