Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: allow long inputs for embedding models (#11721)
Nilay96697f2963a6
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.pyindex 74ab448c85c..b371f25f543 100644--- a/studio/backend/core/inference/llama_cpp.py+++ b/studio/backend/core/inference/llama_cpp.py@@ -6598,6 +6598,21 @@ def _batch_ubatch_for_mmproj(return n_batch, target+def _embedding_batch_ubatch(+ n_ctx: int,+ n_batch: Optional[int],+ n_ubatch: Optional[int],+ extra_args: Optional[Iterable[str]],+ env: Optional[Mapping[str, str]] = None,+) -> tuple[Optional[int], Optional[int]]:+ """Size an unset batch pair to the context for pooling that needs one micro-batch."""+ _, _, batch_named, ubatch_named = _named_batch_sizes(extra_args, env, n_batch, n_ubatch)+ if (batch_named and ubatch_named) or n_ctx <= _DEFAULT_LLAMA_N_UBATCH:+ return n_batch, n_ubatch+ # llama.cpp caps the micro-batch at the batch, so the unset side must grow too.+ return (n_batch if batch_named else n_ctx), (n_ubatch if ubatch_named else n_ctx)++def _build_ngram_mod_flags(caps: Optional[dict],n_match: int = 24,@@ -23849,6 +23864,16 @@ class LlamaCppBackend:logger.info("Load cancelled after download phase")return False+ # MEAN/CLS inputs must fit one micro-batch (LAST splits). Before the projector raise,+ # which would otherwise read as a user-set micro-batch.+ if self._pooling_type in (1, 2):+ n_batch, n_ubatch = _embedding_batch_ubatch(+ resolve_requested_ctx(extra_args, n_ctx) or self._context_length or 0,+ n_batch,+ n_ubatch,+ extra_args,+ )+# Decide after downloading the projector and before pricing the load.n_batch, n_ubatch = _batch_ubatch_for_mmproj(_launch_required_ubatch(diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex 7dcdbdac5cd..4b40df5eeb2 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -11590,7 +11590,11 @@ def _gguf_runtime_bytes(over-reserves on purpose; a panel quoting a number to a user wants the otherone, since a smaller ``-c`` in the extras is the context the user gets."""try:- from core.inference.llama_cpp import _ASSUMED_MAX_VOCAB, _batch_ubatch_for_mmproj+ from core.inference.llama_cpp import (+ _ASSUMED_MAX_VOCAB,+ _batch_ubatch_for_mmproj,+ _embedding_batch_ubatch,+ )from core.inference.llama_cpp import effective_ctx_checkpoints_for_capsfrom core.inference.llama_server_args import (parse_ctx_override,@@ -11608,13 +11612,6 @@ def _gguf_runtime_bytes()except Exception as _rows_exc:logger.debug("llama-server build probe failed: %s", _rows_exc)- # Price the same batch sizes used by load_model.- n_batch, n_ubatch = _batch_ubatch_for_mmproj(- 0 if is_diffusion else launch_required_ubatch,- n_batch,- n_ubatch,- llama_extra_args,- )# Carried out even when the cache cannot be sized: block_count is a separate# key and is usually there, and a caller that loses it prices a manual offload# split as fully GPU-resident (_gguf_offloaded_layer_fraction has nothing to@@ -11657,6 +11654,15 @@ def _gguf_runtime_bytes()if ctx <= 0:return unknown+ # Same batch sizes, in the same order, as load_model.+ if getattr(probe, "_pooling_type", None) in (1, 2):+ n_batch, n_ubatch = _embedding_batch_ubatch(ctx, n_batch, n_ubatch, llama_extra_args)+ n_batch, n_ubatch = _batch_ubatch_for_mmproj(+ 0 if is_diffusion else launch_required_ubatch,+ n_batch,+ n_ubatch,+ llama_extra_args,+ )slots = max(1, n_parallel or 1)planned_cache_types = _planned_main_cache_types(cache_type_kv, llama_extra_args)# KV bytes take the heavier axis (conservative for storage); the dequantdiff --git a/studio/backend/tests/test_embedding_gguf_launch.py b/studio/backend/tests/test_embedding_gguf_launch.pyindex dd16c09ab7b..4eded1f12bb 100644--- a/studio/backend/tests/test_embedding_gguf_launch.py+++ b/studio/backend/tests/test_embedding_gguf_launch.py@@ -244,6 +244,54 @@ class TestLoadModelEmitsTheFlag:assert f'"{name}"' in src+class TestEmbeddingBatchSizedToContext:+ """llama-server 500s on a MEAN/CLS input longer than --ubatch-size (512 by default)."""++ def test_unset_pair_is_raised_to_the_context(self):+ assert llama_cpp_module._embedding_batch_ubatch(2048, None, None, None, env = {}) == (+ 2048,+ 2048,+ )++ @pytest.mark.parametrize(+ "n_batch, n_ubatch, extra_args, env, expected",+ [+ (1024, 256, None, {}, (1024, 256)),+ (1024, None, None, {}, (1024, 8192)),+ (None, 256, None, {}, (8192, 256)),+ (None, None, ["-ub", "1024"], {}, (8192, None)),+ (None, None, ["--batch-size=4096"], {}, (None, 8192)),+ (None, None, None, {"LLAMA_ARG_UBATCH": "768"}, (8192, None)),+ ],+ )+ def test_user_batch_sizes_are_kept(self, n_batch, n_ubatch, extra_args, env, expected):+ assert (+ llama_cpp_module._embedding_batch_ubatch(8192, n_batch, n_ubatch, extra_args, env = env)+ == expected+ )++ @pytest.mark.parametrize("n_ctx", [0, 256, 512])+ def test_context_within_the_default_micro_batch_is_left_alone(self, n_ctx):+ assert llama_cpp_module._embedding_batch_ubatch(n_ctx, None, None, None, env = {}) == (+ None,+ None,+ )++ def test_load_model_applies_it_before_the_fit_for_non_last_pooling(self):+ src = inspect.getsource(llama_cpp_module.LlamaCppBackend.load_model)+ call = src.find("n_batch, n_ubatch = _embedding_batch_ubatch(")+ assert call != -1, "load_model must size the embedding batch pair to the context"+ assert (+ src.find("if self._pooling_type in (1, 2):", call - 100, call) != -1+ ), "only MEAN/CLS pooling needs the single micro-batch; LAST splits and NONE is refused"+ assert call < src.find(+ "n_batch, n_ubatch = _batch_ubatch_for_mmproj("+ ), "a projector-raised micro-batch must not read as a user-set one"+ assert call < src.find(+ "_effective_ubatch = _ubatch_for_slots(n_parallel)"+ ), "the raise must land before the fit prices the compute buffer"++@pytest.mark.parametrize("flag", ["--embedding", "--embeddings", "--pooling"])def test_user_extra_args_still_cannot_pass_the_flag(flag):# The denylist keeps a user-supplied --embedding off the chat server; thediff --git a/studio/backend/tests/test_memory_estimate.py b/studio/backend/tests/test_memory_estimate.pyindex b99b5154507..b5e180de1f8 100644--- a/studio/backend/tests/test_memory_estimate.py+++ b/studio/backend/tests/test_memory_estimate.py@@ -295,6 +295,16 @@ class TestGgufRuntimeBytes:assert runtime.n_ctx == _GQA_FIELDS["context_length"]assert runtime.kv_bytes > 0+ @pytest.mark.parametrize("pooling_type, priced_ubatch", [(1, 2048), (2, 2048), (3, None)])+ def test_embedding_micro_batch_matches_the_launch(self, tmp_path, pooling_type, priced_ubatch):+ gguf = _write_gguf(tmp_path, "qwen3", {**_GQA_FIELDS, "pooling_type": pooling_type})+ assert ri._gguf_runtime_bytes(gguf, 2048).n_ubatch == priced_ubatch++ def test_undeclared_embedding_pooling_keeps_the_default_micro_batch(self, tmp_path):+ # llama.cpp runs a header without pooling_type as NONE, which /v1/embeddings refuses.+ gguf = _write_gguf(tmp_path, "nomic-bert", _GQA_FIELDS)+ assert ri._gguf_runtime_bytes(gguf, 2048).n_ubatch is None+def test_missing_dims_are_unknown_not_zero(self, dimless_gguf):# THE case this NamedTuple exists for. kv_bytes == 0 here means "could not# size", and the only thing separating it from a genuine zero is the flag.