Skip to content

armgit / LEANN

Public Imported from GitHub
0

[MLsys2026 Best Paper]: https://arxiv.org/abs/2506.08276. RAG on Everything with LEANN. Enjoy 97% storage savings while running a fast, accurate, and 100% private RAG application on your personal device.

fix(embeddings): make CUDA batch size configurable for small GPUs (#368)

Aakash Suresh0ddc20fa979b
diff --git a/packages/leann-core/src/leann/cli.py b/packages/leann-core/src/leann/cli.py
index c96fd7c..d565c73 100644
--- a/packages/leann-core/src/leann/cli.py
+++ b/packages/leann-core/src/leann/cli.py
@@ -297,6 +297,12 @@ Examples:
default=None,
help="Prompt template to prepend to all texts for embedding (e.g., 'query: ' for search)",
)
+ build_parser.add_argument(
+ "--embedding-batch-size",
+ type=int,
+ default=None,
+ help="Embedding batch size for sentence-transformers (overrides device defaults; also set LEANN_CUDA_BATCH_SIZE env)",
+ )
build_parser.add_argument(
"--query-prompt-template",
type=str,
@@ -745,6 +751,12 @@ Examples:
help="OpenAI-compatible embedding base URL",
)
p.add_argument("--embedding-api-key", type=str, default=None, help="Embedding API key")
+ p.add_argument(
+ "--embedding-batch-size",
+ type=int,
+ default=None,
+ help="Embedding batch size for sentence-transformers",
+ )
p.add_argument(
"--max-count", type=int, default=1000, help="Max items to index (default: 1000)"
)
@@ -1858,6 +1870,9 @@ Examples:
opts["query_prompt_template"] = args.query_prompt_template
elif args.embedding_prompt_template:
opts["prompt_template"] = args.embedding_prompt_template
+ batch_size = getattr(args, "embedding_batch_size", None)
+ if batch_size is not None:
+ opts["batch_size"] = batch_size
return opts
def _build_config_from_args(
diff --git a/packages/leann-core/src/leann/embedding_compute.py b/packages/leann-core/src/leann/embedding_compute.py
index 7e51d4f..654c726 100644
--- a/packages/leann-core/src/leann/embedding_compute.py
+++ b/packages/leann-core/src/leann/embedding_compute.py
@@ -320,6 +320,104 @@ def _query_lmstudio_context_limit(model_name: str, base_url: str) -> Optional[in
# Global model cache to avoid repeated loading
_model_cache: dict[str, Any] = {}
+_DEFAULT_CUDA_BATCH_SIZE = 256
+_DEFAULT_MPS_BATCH_SIZE = 128
+
+
+def _parse_positive_int_env(name: str, default: int) -> int:
+ raw = os.getenv(name)
+ if raw is None or raw.strip() == "":
+ return default
+ try:
+ value = int(raw)
+ except ValueError:
+ logger.warning("Invalid %s=%r; using default %d", name, raw, default)
+ return default
+ if value < 1:
+ logger.warning("%s must be >= 1; using default %d", name, default)
+ return default
+ return value
+
+
+def _resolve_cpu_thread_count() -> int:
+ return _parse_positive_int_env("LEANN_CPU_THREADS", min(8, os.cpu_count() or 4))
+
+
+def _resolve_adaptive_batch_size(device: str, model_name: str) -> int:
+ if device == "cuda":
+ return _parse_positive_int_env("LEANN_CUDA_BATCH_SIZE", _DEFAULT_CUDA_BATCH_SIZE)
+ if device == "mps":
+ default = 32 if model_name == "Qwen/Qwen3-Embedding-0.6B" else _DEFAULT_MPS_BATCH_SIZE
+ return _parse_positive_int_env("LEANN_MPS_BATCH_SIZE", default)
+ return 32
+
+
+def _cap_cuda_batch_by_vram(requested: int, max_length: int = 512) -> int:
+ auto = os.getenv("LEANN_CUDA_AUTO_BATCH", "1").lower()
+ if auto in ("0", "false", "no"):
+ return requested
+ import torch
+
+ if not torch.cuda.is_available():
+ return requested
+ try:
+ free_bytes, _total = torch.cuda.mem_get_info()
+ except Exception:
+ return requested
+
+ # Eager-attention peak memory scales ~O(seq^2) per sequence in the batch.
+ bytes_per_seq = max(8_000_000, max_length * max_length * 32)
+ budget = int(free_bytes * 0.2)
+ max_by_vram = max(1, budget // bytes_per_seq)
+ capped = min(requested, max_by_vram)
+ if capped < requested:
+ logger.info(
+ "Capping CUDA embedding batch size %d -> %d (%.2f GiB free VRAM)",
+ requested,
+ capped,
+ free_bytes / (1024**3),
+ )
+ return capped
+
+
+def _encode_with_oom_retry(
+ model: _SentenceTransformerLike,
+ texts: list[str],
+ batch_size: int,
+ *,
+ is_build: bool,
+ device: str,
+) -> Any:
+ import torch
+
+ current = batch_size
+ while True:
+ try:
+ with torch.inference_mode():
+ return model.encode(
+ texts,
+ batch_size=current,
+ show_progress_bar=is_build,
+ convert_to_numpy=True,
+ normalize_embeddings=False,
+ device=device,
+ )
+ except RuntimeError as exc:
+ oom = "out of memory" in str(exc).lower()
+ if torch.cuda.is_available():
+ oom = oom or isinstance(exc, torch.cuda.OutOfMemoryError)
+ if not oom or current <= 1:
+ raise
+ if torch.cuda.is_available():
+ torch.cuda.empty_cache()
+ next_batch = max(1, current // 2)
+ logger.warning(
+ "CUDA OOM at embedding batch_size=%d; retrying with batch_size=%d",
+ current,
+ next_batch,
+ )
+ current = next_batch
+
def compute_embeddings(
texts: list[str],
@@ -449,14 +547,7 @@ def compute_embeddings_sentence_transformers(
# Apply optimizations based on benchmark results
if adaptive_optimization:
- # Use optimal batch_size constants for different devices based on benchmark results
- if device == "mps":
- batch_size = 128 # MPS optimal batch size from benchmark
- if model_name == "Qwen/Qwen3-Embedding-0.6B":
- batch_size = 32
- elif device == "cuda":
- batch_size = 256 # CUDA optimal batch size
- # Keep original batch_size for CPU
+ batch_size = _resolve_adaptive_batch_size(device, model_name)
# Create cache key
cache_key = f"sentence_transformers_{model_name}_{device}_{use_fp16}_optimized"
@@ -493,7 +584,7 @@ def compute_embeddings_sentence_transformers(
pass
elif device == "cpu":
# TODO: Haven't tested this yet
- torch.set_num_threads(min(8, os.cpu_count() or 4))
+ torch.set_num_threads(_resolve_cpu_thread_count())
try:
torch.backends.mkldnn.enabled = True
except AttributeError:
@@ -617,18 +708,19 @@ def compute_embeddings_sentence_transformers(
)
logger.info(f"start sentence transformers {model} takes {end_time - start_time}")
+ if device == "cuda" and adaptive_optimization:
+ batch_size = _cap_cuda_batch_by_vram(batch_size, max_length=max_length)
+
start_time = time.time()
if not manual_tokenize:
# Use SentenceTransformer's optimized encode path (default)
- with torch.inference_mode():
- embeddings = model.encode(
- texts,
- batch_size=batch_size,
- show_progress_bar=is_build, # Don't show progress bar in server environment
- convert_to_numpy=True,
- normalize_embeddings=False,
- device=device,
- )
+ embeddings = _encode_with_oom_retry(
+ model,
+ texts,
+ batch_size,
+ is_build=is_build,
+ device=device,
+ )
# Synchronize if CUDA to measure accurate wall time
try:
if torch.cuda.is_available():
diff --git a/tests/test_embedding_batch_size.py b/tests/test_embedding_batch_size.py
new file mode 100644
index 0000000..207b75a
--- /dev/null
+++ b/tests/test_embedding_batch_size.py
@@ -0,0 +1,69 @@
+"""Tests for embedding batch size and CPU thread configuration."""
+
+from unittest.mock import patch
+
+from leann.embedding_compute import (
+ _cap_cuda_batch_by_vram,
+ _parse_positive_int_env,
+ _resolve_adaptive_batch_size,
+ _resolve_cpu_thread_count,
+)
+
+
+def test_parse_positive_int_env_default(monkeypatch):
+ monkeypatch.delenv("LEANN_TEST_INT", raising=False)
+ assert _parse_positive_int_env("LEANN_TEST_INT", 256) == 256
+
+
+def test_parse_positive_int_env_override(monkeypatch):
+ monkeypatch.setenv("LEANN_TEST_INT", "32")
+ assert _parse_positive_int_env("LEANN_TEST_INT", 256) == 32
+
+
+def test_parse_positive_int_env_invalid(monkeypatch):
+ monkeypatch.setenv("LEANN_TEST_INT", "not-a-number")
+ assert _parse_positive_int_env("LEANN_TEST_INT", 256) == 256
+
+
+def test_resolve_adaptive_batch_size_cuda(monkeypatch):
+ monkeypatch.setenv("LEANN_CUDA_BATCH_SIZE", "64")
+ assert _resolve_adaptive_batch_size("cuda", "BAAI/bge-base-en-v1.5") == 64
+
+
+def test_resolve_adaptive_batch_size_mps_qwen(monkeypatch):
+ monkeypatch.delenv("LEANN_MPS_BATCH_SIZE", raising=False)
+ assert _resolve_adaptive_batch_size("mps", "Qwen/Qwen3-Embedding-0.6B") == 32
+
+
+def test_resolve_cpu_threads(monkeypatch):
+ monkeypatch.setenv("LEANN_CPU_THREADS", "16")
+ assert _resolve_cpu_thread_count() == 16
+
+
+def test_cap_cuda_batch_by_vram_disabled(monkeypatch):
+ monkeypatch.setenv("LEANN_CUDA_AUTO_BATCH", "0")
+ with patch("torch.cuda.is_available", return_value=True):
+ with patch("torch.cuda.mem_get_info", return_value=(100, 1000)):
+ assert _cap_cuda_batch_by_vram(256) == 256
+
+
+def test_cap_cuda_batch_by_vram_small_gpu(monkeypatch):
+ monkeypatch.delenv("LEANN_CUDA_AUTO_BATCH", raising=False)
+ # Typical free VRAM on a 4 GiB GPU after loading a base-sized encoder.
+ one_gb = 1024**3
+ with patch("torch.cuda.is_available", return_value=True):
+ with patch("torch.cuda.mem_get_info", return_value=(one_gb, 4 * one_gb)):
+ capped = _cap_cuda_batch_by_vram(256, max_length=512)
+ assert capped < 256
+ assert capped >= 1
+
+
+def test_cap_cuda_batch_by_vram_four_gb_gpu(monkeypatch):
+ """Regression: 4 GiB RTX A1000 reports ~3.2 GiB free; cap should land near 76."""
+ monkeypatch.delenv("LEANN_CUDA_AUTO_BATCH", raising=False)
+ free_vram = int(3.2 * 1024**3)
+ with patch("torch.cuda.is_available", return_value=True):
+ with patch("torch.cuda.mem_get_info", return_value=(free_vram, 4 * 1024**3)):
+ capped = _cap_cuda_batch_by_vram(256, max_length=512)
+ assert capped <= 85
+ assert capped >= 1