Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: allow long inputs for embedding models (#11721)

Nilay96697f2963a6
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py
index 74ab448c85c..b371f25f543 100644
--- a/studio/backend/core/inference/llama_cpp.py
+++ b/studio/backend/core/inference/llama_cpp.py
@@ -6598,6 +6598,21 @@ def _batch_ubatch_for_mmproj(
return n_batch, target
+def _embedding_batch_ubatch(
+ n_ctx: int,
+ n_batch: Optional[int],
+ n_ubatch: Optional[int],
+ extra_args: Optional[Iterable[str]],
+ env: Optional[Mapping[str, str]] = None,
+) -> tuple[Optional[int], Optional[int]]:
+ """Size an unset batch pair to the context for pooling that needs one micro-batch."""
+ _, _, batch_named, ubatch_named = _named_batch_sizes(extra_args, env, n_batch, n_ubatch)
+ if (batch_named and ubatch_named) or n_ctx <= _DEFAULT_LLAMA_N_UBATCH:
+ return n_batch, n_ubatch
+ # llama.cpp caps the micro-batch at the batch, so the unset side must grow too.
+ return (n_batch if batch_named else n_ctx), (n_ubatch if ubatch_named else n_ctx)
+
+
def _build_ngram_mod_flags(
caps: Optional[dict],
n_match: int = 24,
@@ -23849,6 +23864,16 @@ class LlamaCppBackend:
logger.info("Load cancelled after download phase")
return False
+ # MEAN/CLS inputs must fit one micro-batch (LAST splits). Before the projector raise,
+ # which would otherwise read as a user-set micro-batch.
+ if self._pooling_type in (1, 2):
+ n_batch, n_ubatch = _embedding_batch_ubatch(
+ resolve_requested_ctx(extra_args, n_ctx) or self._context_length or 0,
+ n_batch,
+ n_ubatch,
+ extra_args,
+ )
+
# Decide after downloading the projector and before pricing the load.
n_batch, n_ubatch = _batch_ubatch_for_mmproj(
_launch_required_ubatch(
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 7dcdbdac5cd..4b40df5eeb2 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -11590,7 +11590,11 @@ def _gguf_runtime_bytes(
over-reserves on purpose; a panel quoting a number to a user wants the other
one, since a smaller ``-c`` in the extras is the context the user gets."""
try:
- from core.inference.llama_cpp import _ASSUMED_MAX_VOCAB, _batch_ubatch_for_mmproj
+ from core.inference.llama_cpp import (
+ _ASSUMED_MAX_VOCAB,
+ _batch_ubatch_for_mmproj,
+ _embedding_batch_ubatch,
+ )
from core.inference.llama_cpp import effective_ctx_checkpoints_for_caps
from core.inference.llama_server_args import (
parse_ctx_override,
@@ -11608,13 +11612,6 @@ def _gguf_runtime_bytes(
)
except Exception as _rows_exc:
logger.debug("llama-server build probe failed: %s", _rows_exc)
- # Price the same batch sizes used by load_model.
- n_batch, n_ubatch = _batch_ubatch_for_mmproj(
- 0 if is_diffusion else launch_required_ubatch,
- n_batch,
- n_ubatch,
- llama_extra_args,
- )
# Carried out even when the cache cannot be sized: block_count is a separate
# key and is usually there, and a caller that loses it prices a manual offload
# split as fully GPU-resident (_gguf_offloaded_layer_fraction has nothing to
@@ -11657,6 +11654,15 @@ def _gguf_runtime_bytes(
)
if ctx <= 0:
return unknown
+ # Same batch sizes, in the same order, as load_model.
+ if getattr(probe, "_pooling_type", None) in (1, 2):
+ n_batch, n_ubatch = _embedding_batch_ubatch(ctx, n_batch, n_ubatch, llama_extra_args)
+ n_batch, n_ubatch = _batch_ubatch_for_mmproj(
+ 0 if is_diffusion else launch_required_ubatch,
+ n_batch,
+ n_ubatch,
+ llama_extra_args,
+ )
slots = max(1, n_parallel or 1)
planned_cache_types = _planned_main_cache_types(cache_type_kv, llama_extra_args)
# KV bytes take the heavier axis (conservative for storage); the dequant
diff --git a/studio/backend/tests/test_embedding_gguf_launch.py b/studio/backend/tests/test_embedding_gguf_launch.py
index dd16c09ab7b..4eded1f12bb 100644
--- a/studio/backend/tests/test_embedding_gguf_launch.py
+++ b/studio/backend/tests/test_embedding_gguf_launch.py
@@ -244,6 +244,54 @@ class TestLoadModelEmitsTheFlag:
assert f'"{name}"' in src
+class TestEmbeddingBatchSizedToContext:
+ """llama-server 500s on a MEAN/CLS input longer than --ubatch-size (512 by default)."""
+
+ def test_unset_pair_is_raised_to_the_context(self):
+ assert llama_cpp_module._embedding_batch_ubatch(2048, None, None, None, env = {}) == (
+ 2048,
+ 2048,
+ )
+
+ @pytest.mark.parametrize(
+ "n_batch, n_ubatch, extra_args, env, expected",
+ [
+ (1024, 256, None, {}, (1024, 256)),
+ (1024, None, None, {}, (1024, 8192)),
+ (None, 256, None, {}, (8192, 256)),
+ (None, None, ["-ub", "1024"], {}, (8192, None)),
+ (None, None, ["--batch-size=4096"], {}, (None, 8192)),
+ (None, None, None, {"LLAMA_ARG_UBATCH": "768"}, (8192, None)),
+ ],
+ )
+ def test_user_batch_sizes_are_kept(self, n_batch, n_ubatch, extra_args, env, expected):
+ assert (
+ llama_cpp_module._embedding_batch_ubatch(8192, n_batch, n_ubatch, extra_args, env = env)
+ == expected
+ )
+
+ @pytest.mark.parametrize("n_ctx", [0, 256, 512])
+ def test_context_within_the_default_micro_batch_is_left_alone(self, n_ctx):
+ assert llama_cpp_module._embedding_batch_ubatch(n_ctx, None, None, None, env = {}) == (
+ None,
+ None,
+ )
+
+ def test_load_model_applies_it_before_the_fit_for_non_last_pooling(self):
+ src = inspect.getsource(llama_cpp_module.LlamaCppBackend.load_model)
+ call = src.find("n_batch, n_ubatch = _embedding_batch_ubatch(")
+ assert call != -1, "load_model must size the embedding batch pair to the context"
+ assert (
+ src.find("if self._pooling_type in (1, 2):", call - 100, call) != -1
+ ), "only MEAN/CLS pooling needs the single micro-batch; LAST splits and NONE is refused"
+ assert call < src.find(
+ "n_batch, n_ubatch = _batch_ubatch_for_mmproj("
+ ), "a projector-raised micro-batch must not read as a user-set one"
+ assert call < src.find(
+ "_effective_ubatch = _ubatch_for_slots(n_parallel)"
+ ), "the raise must land before the fit prices the compute buffer"
+
+
@pytest.mark.parametrize("flag", ["--embedding", "--embeddings", "--pooling"])
def test_user_extra_args_still_cannot_pass_the_flag(flag):
# The denylist keeps a user-supplied --embedding off the chat server; the
diff --git a/studio/backend/tests/test_memory_estimate.py b/studio/backend/tests/test_memory_estimate.py
index b99b5154507..b5e180de1f8 100644
--- a/studio/backend/tests/test_memory_estimate.py
+++ b/studio/backend/tests/test_memory_estimate.py
@@ -295,6 +295,16 @@ class TestGgufRuntimeBytes:
assert runtime.n_ctx == _GQA_FIELDS["context_length"]
assert runtime.kv_bytes > 0
+ @pytest.mark.parametrize("pooling_type, priced_ubatch", [(1, 2048), (2, 2048), (3, None)])
+ def test_embedding_micro_batch_matches_the_launch(self, tmp_path, pooling_type, priced_ubatch):
+ gguf = _write_gguf(tmp_path, "qwen3", {**_GQA_FIELDS, "pooling_type": pooling_type})
+ assert ri._gguf_runtime_bytes(gguf, 2048).n_ubatch == priced_ubatch
+
+ def test_undeclared_embedding_pooling_keeps_the_default_micro_batch(self, tmp_path):
+ # llama.cpp runs a header without pooling_type as NONE, which /v1/embeddings refuses.
+ gguf = _write_gguf(tmp_path, "nomic-bert", _GQA_FIELDS)
+ assert ri._gguf_runtime_bytes(gguf, 2048).n_ubatch is None
+
def test_missing_dims_are_unknown_not_zero(self, dimless_gguf):
# THE case this NamedTuple exists for. kv_bytes == 0 here means "could not
# size", and the only thing separating it from a genuine zero is the flag.