Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: raise the micro-batch for the projectors that abort llama-server on an image (#10683)

Chakshu Dhannawatc3329b9dbf2d
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py
index 1f222636590..9f374b00c6d 100644
--- a/studio/backend/core/inference/llama_cpp.py
+++ b/studio/backend/core/inference/llama_cpp.py
@@ -89,6 +89,8 @@ from core.inference.llama_server_args import (
apply_model_memory_policy,
resolve_ctx_checkpoints,
extra_args_disable_mmproj,
+ extra_args_image_max_tokens,
+ extra_args_mmproj_auto,
extra_args_select_load_mode,
fit_is_enabled_in,
force_pageable_load,
@@ -4308,6 +4310,17 @@ def _strip_flag_pairs(args: Iterable[str], flags: frozenset[str]) -> list[str]:
# common_params defaults in the bundled llama.cpp runtime.
_DEFAULT_LLAMA_N_BATCH = 2048
_DEFAULT_LLAMA_N_UBATCH = 512
+# Non-causal image decoding requires one image chunk to fit in the micro-batch.
+# These are the per-image limits from clip.cpp. Exact values avoid excess VRAM use.
+_MMPROJ_NON_CAUSAL_IMAGE_TOKENS = {
+ "gemma4v": 1120,
+ "gemma4uv": 1120,
+ "gemma3": 256,
+ "deepseek4v": 384,
+}
+# A projector whose family we cannot read gets headroom instead of a ceiling.
+_MMPROJ_UNKNOWN_UBATCH = 2048
+_GEMMA4V_CAUSAL_TEXT_N_EMBD = frozenset({1536, 2560}) # E2B and E4B
_LLAMA_ARG_TRUE_VALUES = frozenset({"on", "enabled", "true", "1"})
_LLAMA_ARG_FALSE_VALUES = frozenset({"off", "disabled", "false", "0"})
_LLAMA_ARG_AUTO_VALUES = frozenset({"auto", "-1"})
@@ -5748,26 +5761,22 @@ def _extra_args_n_parallel(
return found
-def _extra_args_n_ubatch(
+def _named_batch_sizes(
extra_args: Optional[Iterable[str]],
env: Optional[Mapping[str, str]] = None,
- n_ctx: Optional[int] = None,
- *,
n_batch: Optional[int] = None,
n_ubatch: Optional[int] = None,
-) -> Optional[int]:
- """Effective ubatch after llama.cpp normalizes it, or None at defaults.
+) -> tuple[int, int, bool, bool]:
+ """Resolve batch sizes and whether each was explicitly set.
- Precedence mirrors the launched command line: env, then the first-class
- n_batch / n_ubatch fields (emitted as flags, so they beat env), then user
- extra_args (appended last, so they last-wins-override the emitted flags).
+ Precedence is environment, first-class fields, then extra arguments.
"""
values = {
"batch": _DEFAULT_LLAMA_N_BATCH,
"ubatch": _DEFAULT_LLAMA_N_UBATCH,
}
+ named = {"batch": False, "ubatch": False}
source_env = os.environ if env is None else env
- overridden = False
for key, env_name in (
("batch", "LLAMA_ARG_BATCH"),
("ubatch", "LLAMA_ARG_UBATCH"),
@@ -5776,16 +5785,16 @@ def _extra_args_n_ubatch(
if raw:
try:
values[key] = int(raw)
- overridden = True
+ named[key] = True
except (TypeError, ValueError):
pass
if n_batch is not None:
values["batch"] = int(n_batch)
- overridden = True
+ named["batch"] = True
if n_ubatch is not None:
values["ubatch"] = int(n_ubatch)
- overridden = True
+ named["ubatch"] = True
args = [str(a) for a in extra_args] if extra_args else []
flags = {
@@ -5803,10 +5812,26 @@ def _extra_args_n_ubatch(
value = inline if eq else (args[i + 1] if i + 1 < len(args) else "")
try:
values[key] = int(value)
- overridden = True
+ named[key] = True
except (TypeError, ValueError):
continue
- if not overridden:
+ return values["batch"], values["ubatch"], named["batch"], named["ubatch"]
+
+
+def _extra_args_n_ubatch(
+ extra_args: Optional[Iterable[str]],
+ env: Optional[Mapping[str, str]] = None,
+ n_ctx: Optional[int] = None,
+ *,
+ n_batch: Optional[int] = None,
+ n_ubatch: Optional[int] = None,
+) -> Optional[int]:
+ """Effective ubatch after llama.cpp normalizes it, or None at defaults."""
+ _batch, _ubatch, _batch_named, _ubatch_named = _named_batch_sizes(
+ extra_args, env, n_batch, n_ubatch
+ )
+ values = {"batch": _batch, "ubatch": _ubatch}
+ if not (_batch_named or _ubatch_named):
return None
# common_params stores signed values, then llama_context_params converts
@@ -5821,6 +5846,135 @@ def _extra_args_n_ubatch(
return effective
+def _read_gguf_embedding_length(path: Optional[str]) -> Optional[int]:
+ """``{arch}.embedding_length``, cached, or None. Thin import-local wrapper."""
+ if not path:
+ return None
+ try:
+ from utils.models.gguf_metadata import read_gguf_embedding_length
+ return read_gguf_embedding_length(str(path))
+ except Exception as e:
+ logger.debug(f"embedding length read failed: {e}")
+ return None
+
+
+def _unknown_projector_ubatch(
+ extra_args: Optional[Iterable[str]] = None, env: Optional[Mapping[str, str]] = None
+) -> int:
+ """Return conservative headroom for an unclassified projector."""
+ return max(_MMPROJ_UNKNOWN_UBATCH, extra_args_image_max_tokens(extra_args, env) or 0)
+
+
+def _mmproj_required_ubatch(
+ mmproj_path: Optional[str],
+ n_embd_text: Optional[int] = None,
+ extra_args: Optional[Iterable[str]] = None,
+ env: Optional[Mapping[str, str]] = None,
+) -> int:
+ """Return the required micro-batch, or 0 when the default is sufficient.
+
+ Inspect the image tower because audio-only projectors may still be marked as vision.
+ """
+ if not mmproj_path:
+ return 0
+ try:
+ from utils.models.gguf_metadata import (
+ mmproj_accepts_image,
+ read_mmproj_vision_projector_type,
+ )
+ if not mmproj_accepts_image(mmproj_path):
+ return 0
+ family = (read_mmproj_vision_projector_type(mmproj_path) or "").strip().lower()
+ except Exception as e:
+ logger.debug(f"mmproj capability read failed: {e}")
+ family = ""
+ custom = extra_args_image_max_tokens(extra_args, env) or 0
+ if not family:
+ return _unknown_projector_ubatch(extra_args, env)
+ if family not in _MMPROJ_NON_CAUSAL_IMAGE_TOKENS:
+ return 0
+ if family == "gemma4v" and n_embd_text in _GEMMA4V_CAUSAL_TEXT_N_EMBD:
+ return 0
+ # A custom image limit can exceed the family's built-in ceiling.
+ ceiling = max(_MMPROJ_NON_CAUSAL_IMAGE_TOKENS[family], custom)
+ return ceiling if ceiling > _DEFAULT_LLAMA_N_UBATCH else 0
+
+
+def _launch_required_ubatch(
+ mmproj_path: Optional[str],
+ n_embd_text: Optional[int] = None,
+ extra_args: Optional[Iterable[str]] = None,
+ *,
+ is_vision: bool = True,
+ vision_off: bool = False,
+ env: Optional[Mapping[str, str]] = None,
+) -> int:
+ """Return the largest micro-batch required by any possible launch projector.
+
+ Taking the maximum avoids duplicating llama.cpp's projector precedence and only
+ over-reserves when multiple image towers are supplied.
+ """
+ required = 0
+
+ # Pass-through arguments override managed projector flags.
+ override = _extra_args_device(extra_args, {"--mmproj", "-mm"})
+ if override:
+ required = max(
+ required, _mmproj_required_ubatch(str(override), n_embd_text, extra_args, env)
+ )
+
+ if not vision_off:
+ # Inherited projectors survive --no-mmproj but not the vision switch.
+ source_env = os.environ if env is None else env
+ if (source_env.get("LLAMA_ARG_MMPROJ_URL") or "").strip():
+ required = max(required, _unknown_projector_ubatch(extra_args, env))
+ inherited = (source_env.get("LLAMA_ARG_MMPROJ") or "").strip()
+ if inherited:
+ required = max(
+ required, _mmproj_required_ubatch(inherited, n_embd_text, extra_args, env)
+ )
+
+ if is_vision and not vision_off and not extra_args_disable_mmproj(extra_args):
+ if mmproj_path:
+ required = max(
+ required, _mmproj_required_ubatch(str(mmproj_path), n_embd_text, extra_args, env)
+ )
+
+ if not vision_off and extra_args_mmproj_auto(extra_args, env):
+ # llama-server's adjacent-projector search can disagree with ours.
+ required = max(
+ required,
+ _mmproj_required_ubatch(str(mmproj_path), n_embd_text, extra_args, env)
+ if mmproj_path
+ else _unknown_projector_ubatch(extra_args, env),
+ )
+
+ return required
+
+
+def _batch_ubatch_for_mmproj(
+ required_ubatch: int,
+ n_batch: Optional[int],
+ n_ubatch: Optional[int],
+ extra_args: Optional[Iterable[str]],
+ env: Optional[Mapping[str, str]] = None,
+) -> tuple[Optional[int], Optional[int]]:
+ """Raise an unset micro-batch, capped by the effective batch size."""
+ if required_ubatch <= 0:
+ return n_batch, n_ubatch
+ batch, ubatch, batch_named, ubatch_named = _named_batch_sizes(
+ extra_args, env, n_batch, n_ubatch
+ )
+ if ubatch_named:
+ return n_batch, n_ubatch
+ # Match llama_context_params, which converts the signed batch to uint32_t.
+ batch &= 0xFFFFFFFF
+ target = min(required_ubatch, batch)
+ if target <= ubatch:
+ return n_batch, n_ubatch
+ return n_batch, target
+
+
def _build_ngram_mod_flags(
caps: Optional[dict],
n_match: int = 24,
@@ -21226,6 +21380,26 @@ class LlamaCppBackend:
logger.info("Load cancelled after download phase")
return False
+ # Decide after downloading the projector and before pricing the load.
+ n_batch, n_ubatch = _batch_ubatch_for_mmproj(
+ _launch_required_ubatch(
+ None
+ if (disable_vision or not is_vision)
+ else self._resolve_launch_mmproj_path(
+ model_path = model_path,
+ mmproj_path = mmproj_path,
+ ),
+ # Use the same order-independent metadata read as the estimators.
+ _read_gguf_embedding_length(model_path),
+ extra_args,
+ is_vision = is_vision,
+ vision_off = disable_vision,
+ ),
+ n_batch,
+ n_ubatch,
+ extra_args,
+ )
+
# Backstop for everything the pre-teardown probes fail open on: refuse from the
# header rather than watching llama-server die as "failed to start".
non_chat = self._non_chat_gguf_refusal(model_path)
diff --git a/studio/backend/core/inference/llama_server_args.py b/studio/backend/core/inference/llama_server_args.py
index b5353e33247..14b8ccc8ca1 100644
--- a/studio/backend/core/inference/llama_server_args.py
+++ b/studio/backend/core/inference/llama_server_args.py
@@ -992,6 +992,59 @@ def extra_args_disable_mmproj(args: Optional[Iterable[str]]) -> bool:
return disabled
+def extra_args_image_max_tokens(
+ args: Optional[Iterable[str]], env: Optional[Mapping[str, str]] = None
+) -> Optional[int]:
+ """Return the effective positive ``--image-max-tokens`` value."""
+ found: Optional[int] = None
+ raw_env = (os.environ if env is None else env).get("LLAMA_ARG_IMAGE_MAX_TOKENS")
+ if raw_env:
+ try:
+ parsed_env = int(str(raw_env).strip())
+ except (TypeError, ValueError):
+ parsed_env = 0
+ if parsed_env > 0:
+ found = parsed_env
+ tokens = [str(a) for a in (args or ())]
+ for index, raw in enumerate(tokens):
+ if raw.startswith("--image-max-tokens="):
+ value = raw.partition("=")[2]
+ elif raw == "--image-max-tokens" and index + 1 < len(tokens):
+ value = tokens[index + 1]
+ else:
+ continue
+ try:
+ parsed = int(value)
+ except (TypeError, ValueError):
+ continue
+ if parsed > 0:
+ found = parsed
+ return found
+
+
+def extra_args_mmproj_auto(
+ args: Optional[Iterable[str]], env: Optional[Mapping[str, str]] = None
+) -> bool:
+ """Return whether llama-server will discover an adjacent projector."""
+ source_env = os.environ if env is None else env
+ enabled = False
+ if source_env.get("LLAMA_ARG_NO_MMPROJ_AUTO") is not None:
+ enabled = False
+ else:
+ raw = source_env.get("LLAMA_ARG_MMPROJ_AUTO")
+ if raw is not None:
+ enabled = str(raw).strip().lower() in _ENV_TRUE_VALUES
+ if not args:
+ return enabled
+ for raw in args:
+ flag = _flag_name(str(raw))
+ if flag in _MMPROJ_ENABLE_FLAGS:
+ enabled = True
+ elif flag in _MMPROJ_DISABLE_FLAGS:
+ enabled = False
+ return enabled
+
+
def strip_shadowing_flags(
args: Iterable[str],
*,
diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.py
index 005d2f17598..ed9d1999a02 100644
--- a/studio/backend/models/inference.py
+++ b/studio/backend/models/inference.py
@@ -187,9 +187,13 @@ class LoadRequest(BaseModel):
description = (
"Physical prompt micro-batch size for llama-server (--ubatch-size) "
f"for this load ({BATCH_MIN}..{BATCH_MAX}). Omit for the llama.cpp "
- "default (512). llama.cpp caps it at the batch size. Larger values "
- "speed up prompt processing at the cost of compute-buffer VRAM. "
- "Ignored for non-GGUF models."
+ "default (512), raised to a projector's own per-image ceiling when "
+ "its images would abort the server at 512: 1120 for Gemma 4, and "
+ "2048 for a projector whose family cannot be read. Every other "
+ "vision model keeps the 512 default. "
+ "llama.cpp caps it at the batch size. Larger values speed up prompt "
+ "processing at the cost of compute-buffer VRAM. Ignored for "
+ "non-GGUF models."
),
)
load_mode: Optional[Literal["auto", "none", "mmap", "mlock", "mmap+mlock", "dio"]] = Field(
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 397d1a8359e..5d2b5c1b542 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -1889,23 +1889,9 @@ def _openai_llama_admission_output_allowance(
def _extra_args_image_max_tokens(extra_args) -> Optional[int]:
- """The ``--image-max-tokens N`` a load passed through, or None. Last wins.
-
- Both spellings, since llama-server accepts ``--flag N`` and ``--flag=N``.
- """
- args = [str(arg) for arg in (extra_args or ())]
- found = None
- for index, raw in enumerate(args):
- if raw.startswith("--image-max-tokens="):
- value = raw.partition("=")[2]
- elif raw == "--image-max-tokens" and index + 1 < len(args):
- value = args[index + 1]
- else:
- continue
- parsed = _positive_int_or_none(value)
- if parsed is not None:
- found = parsed
- return found
+ """Return the effective ``--image-max-tokens`` value."""
+ from core.inference.llama_server_args import extra_args_image_max_tokens
+ return extra_args_image_max_tokens(extra_args)
# Embeddings one image may become, per projector family, from llama.cpp's own
@@ -10266,6 +10252,60 @@ def _inherited_ctx_size() -> int:
return 0
+def _launch_required_ubatch_for_config(
+ config,
+ llama_extra_args: Optional[list[str]] = None,
+ disable_vision: bool = False,
+) -> int:
+ """Return the required micro-batch for a local GGUF config."""
+ from core.inference.llama_cpp import _launch_required_ubatch
+ from utils.models.gguf_metadata import read_gguf_embedding_length
+
+ gguf_file = str(getattr(config, "gguf_file", "") or "")
+ own = getattr(config, "gguf_mmproj_file", None)
+ resolved = None
+ if own and getattr(config, "is_vision", False) and not disable_vision:
+ resolved = _probe_backend()._resolve_launch_mmproj_path(
+ model_path = gguf_file,
+ mmproj_path = str(own),
+ )
+ return _launch_required_ubatch(
+ resolved,
+ # The text embedding size identifies the causal Gemma 4 variants.
+ read_gguf_embedding_length(gguf_file) if gguf_file else None,
+ llama_extra_args,
+ is_vision = bool(getattr(config, "is_vision", False)),
+ vision_off = disable_vision,
+ )
+
+
+def _remote_required_ubatch(
+ config,
+ llama_extra_args: Optional[list[str]] = None,
+ disable_vision: bool = False,
+) -> int:
+ """Return a conservative micro-batch for an undownloaded GGUF config."""
+ from core.inference.llama_cpp import _launch_required_ubatch, extra_args_disable_mmproj
+
+ from core.inference.llama_cpp import _unknown_projector_ubatch
+
+ if (
+ bool(getattr(config, "is_vision", False))
+ and not disable_vision
+ and not extra_args_disable_mmproj(llama_extra_args)
+ ):
+ # Match the worst-case post-download allocation.
+ return _unknown_projector_ubatch(llama_extra_args)
+ # No repo projector in play, but the extras or the environment may still name one.
+ return _launch_required_ubatch(
+ None,
+ None,
+ llama_extra_args,
+ is_vision = False,
+ vision_off = disable_vision,
+ )
+
+
def _gguf_runtime_bytes(
gguf_path: str,
max_seq_length: int,
@@ -10280,6 +10320,7 @@ def _gguf_runtime_bytes(
is_diffusion: bool = False,
ctx_last_wins: bool = False,
model_identifier: Optional[str] = None,
+ launch_required_ubatch: int = 0,
) -> _GgufRuntimeBytes:
"""KV-cache and compute-buffer VRAM (bytes) at the larger of max_seq_length and
any `--ctx-size`/`-c` override, over n_parallel slots at the effective
@@ -10295,6 +10336,7 @@ def _gguf_runtime_bytes(
over-reserves on purpose; a panel quoting a number to a user wants the other
one, since a smaller ``-c`` in the extras is the context the user gets."""
try:
+ from core.inference.llama_cpp import _batch_ubatch_for_mmproj
from core.inference.llama_server_args import (
parse_ctx_override,
resolve_ctx_checkpoints,
@@ -10304,6 +10346,13 @@ def _gguf_runtime_bytes(
probe = _probe_backend()
probe._model_identifier = model_identifier
probe._read_gguf_metadata(gguf_path)
+ # Price the same batch sizes used by load_model.
+ n_batch, n_ubatch = _batch_ubatch_for_mmproj(
+ 0 if is_diffusion else launch_required_ubatch,
+ n_batch,
+ n_ubatch,
+ llama_extra_args,
+ )
# Carried out even when the cache cannot be sized: block_count is a separate
# key and is usually there, and a caller that loses it prices a manual offload
# split as fully GPU-resident (_gguf_offloaded_layer_fraction has nothing to
@@ -10583,6 +10632,7 @@ def _estimate_gguf_kv_gb(
n_devices: int = 1,
is_diffusion: bool = False,
model_identifier: Optional[str] = None,
+ launch_required_ubatch: int = 0,
) -> float:
"""``_gguf_runtime_bytes`` summed into GB, for the training guard.
@@ -10602,6 +10652,7 @@ def _estimate_gguf_kv_gb(
n_devices = n_devices,
is_diffusion = is_diffusion,
model_identifier = model_identifier,
+ launch_required_ubatch = launch_required_ubatch,
)
return (runtime.kv_bytes + runtime.compute_bytes) / (1024**3)
@@ -10615,16 +10666,27 @@ def _remote_gguf_compute_reserve_gb(
n_devices: int = 1,
tensor_parallel: bool = False,
is_diffusion: bool = False,
+ required_ubatch: int = 0,
) -> float:
"""Compute buffers a remote GGUF will reserve, in GB.
Split out of _estimate_gguf_required_gb so a caller that is pricing something
else, a drafter for instance, can hold it at zero the way it already holds
_estimate_gguf_kv_gb at zero. The arithmetic is unchanged.
+
+ ``required_ubatch`` matches the post-download launch.
"""
# remote dims are unreadable; only the kq mask, linear in ubatch x ctx, can be sized here
+ from core.inference.llama_cpp import _batch_ubatch_for_mmproj
from core.inference.llama_server_args import parse_ctx_override
+ n_batch, n_ubatch = _batch_ubatch_for_mmproj(
+ 0 if is_diffusion else required_ubatch,
+ n_batch,
+ n_ubatch,
+ llama_extra_args,
+ )
+
try:
ctx_override = parse_ctx_override(llama_extra_args) or 0
except Exception:
@@ -11012,6 +11074,9 @@ def _estimate_gguf_required_gb(
n_devices = n_devices,
is_diffusion = is_diffusion,
model_identifier = getattr(config, "identifier", None),
+ launch_required_ubatch = _launch_required_ubatch_for_config(
+ config, llama_extra_args, disable_vision
+ ),
)
repo = getattr(config, "gguf_hf_repo", None)
@@ -11066,6 +11131,7 @@ def _estimate_gguf_required_gb(
n_devices = n_devices,
tensor_parallel = tensor_parallel,
is_diffusion = is_diffusion,
+ required_ubatch = _remote_required_ubatch(config, llama_extra_args, disable_vision),
)
return total_gb
return None
@@ -11493,11 +11559,16 @@ def _gguf_resident_file_gb(
0,
llama_extra_args,
model_identifier = getattr(config, "identifier", None),
+ # Subtract the same runtime term added above.
+ launch_required_ubatch = _launch_required_ubatch_for_config(
+ config, llama_extra_args, disable_vision
+ ),
)
else:
context_term_gb = _remote_gguf_compute_reserve_gb(
llama_extra_args = llama_extra_args,
max_seq_length = 0,
+ required_ubatch = _remote_required_ubatch(config, llama_extra_args, disable_vision),
)
files_gb = max(0.0, required_gb - context_term_gb)
# Under the lock: the route body runs in an asyncio.to_thread worker, so two panel
@@ -12050,6 +12121,9 @@ def _gguf_memory_breakdown(
# An embedding model is recognised from its identifier, not its header, so the
# panel has to hand over the same one /load does or it prices a generation model.
model_identifier = getattr(config, "identifier", None),
+ launch_required_ubatch = _launch_required_ubatch_for_config(
+ config, llama_extra_args, disable_vision
+ ),
)
files_gb = _gguf_resident_file_gb(
config,
diff --git a/studio/backend/tests/test_gpu_init_crash_message.py b/studio/backend/tests/test_gpu_init_crash_message.py
index 7f9139a6985..2fac895a165 100644
--- a/studio/backend/tests/test_gpu_init_crash_message.py
+++ b/studio/backend/tests/test_gpu_init_crash_message.py
@@ -966,7 +966,8 @@ def test_env_projector_cpu_recovery_keeps_audio_input(monkeypatch, tmp_path):
)
assert loaded is True
- assert read == [str(projector)]
+ # The batch-size decision may inspect the same projector earlier in the load.
+ assert read and set(read) == {str(projector)}
assert backend._is_vision is True
assert backend._mmproj_has_audio is True
diff --git a/studio/backend/utils/models/gguf_metadata.py b/studio/backend/utils/models/gguf_metadata.py
index 3b6144aba24..759bcaa9f35 100644
--- a/studio/backend/utils/models/gguf_metadata.py
+++ b/studio/backend/utils/models/gguf_metadata.py
@@ -60,6 +60,8 @@ _NAMED_TENSOR_CACHE: Dict[Tuple[_CacheKey, str], Optional[bool]] = {}
# GGUF header dims for the staged UI in one cached pass (context_length, layer_count, moe_layer_count) so the staged sheet can size every slider before the model loads. None = unreadable / not a GGUF; the native ``{arch}.context_length`` the UI shows before a load is read from here via read_gguf_context_length.
_DIMS_CACHE: Dict[_CacheKey, Optional[Dict[str, Optional[int]]]] = {}
+# Read on its own: the only caller wants just this number, on every settings change.
+_N_EMBD_CACHE: Dict[_CacheKey, Optional[int]] = {}
# Cache the embedded speculative-head count separately for discovery, launch, and sizing.
@@ -179,6 +181,26 @@ def read_gguf_staged_dims(path: str) -> Optional[Dict[str, Optional[int]]]:
return result
+def read_gguf_embedding_length(path: str) -> Optional[int]:
+ """Return the cached ``{arch}.embedding_length`` value, if readable."""
+ key = _cache_key(path)
+ if key is None:
+ return None
+ with _CACHE_LOCK:
+ if key in _N_EMBD_CACHE:
+ return _N_EMBD_CACHE[key]
+ parsed = _parse_gguf_arch_uints(path, frozenset({"embedding_length"}))
+ result = (parsed or {}).get("embedding_length")
+ with _CACHE_LOCK:
+ while len(_N_EMBD_CACHE) >= _CACHE_MAX_ENTRIES:
+ try:
+ _N_EMBD_CACHE.pop(next(iter(_N_EMBD_CACHE)))
+ except StopIteration:
+ break
+ _N_EMBD_CACHE[key] = result
+ return result
+
+
def read_gguf_context_length(path: str) -> Optional[int]:
"""Native training context length (``{arch}.context_length``), or ``None``. Thin accessor over read_gguf_staged_dims."""
dims = read_gguf_staged_dims(path)
@@ -866,6 +888,13 @@ def read_mmproj_projector_type(path: str) -> Optional[str]:
return _read_gguf_string(path, "clip.projector_type")
+def read_mmproj_vision_projector_type(path: str) -> Optional[str]:
+ """Return the image tower family, falling back to the single-tower key."""
+ return _read_gguf_string(path, "clip.vision.projector_type") or _read_gguf_string(
+ path, "clip.projector_type"
+ )
+
+
def read_mmproj_vision_capability(path: str) -> Optional[bool]:
"""``clip.has_vision_encoder`` from an mmproj GGUF: ``True``/``False`` if
present, ``None`` if absent/unreadable."""
diff --git a/studio/frontend/src/features/model-picker/components/model-config-page.tsx b/studio/frontend/src/features/model-picker/components/model-config-page.tsx
index 0dc03a28296..4abc35f76f6 100644
--- a/studio/frontend/src/features/model-picker/components/model-config-page.tsx
+++ b/studio/frontend/src/features/model-picker/components/model-config-page.tsx
@@ -1389,9 +1389,10 @@ function GgufAdvancedSettings({
<span className={LABEL_CLASS}>Micro-batch Size</span>
<InfoHint>
Physical prompt micro-batch size (--ubatch-size). Leave blank for
- the llama.cpp default (512). Larger values speed up prompt
- processing but use more VRAM for the compute buffer; capped at the
- batch size.
+ the llama.cpp default (512), raised to 1120 on Gemma 4 vision
+ models, whose images do not fit in 512. Other vision models keep
+ the 512 default. Larger values speed up prompt processing but use
+ more VRAM for the compute buffer; capped at the batch size.
</InfoHint>
</div>
<input
diff --git a/tests/studio/test_llama_cpp_mmproj_batch.py b/tests/studio/test_llama_cpp_mmproj_batch.py
new file mode 100644
index 00000000000..3e82f7f3319
--- /dev/null
+++ b/tests/studio/test_llama_cpp_mmproj_batch.py
@@ -0,0 +1,373 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Regression tests for projector-aware llama.cpp micro-batch sizing."""
+
+import inspect
+
+import pytest
+
+from studio.backend.core.inference.llama_cpp import (
+ LlamaCppBackend,
+ _batch_ubatch_for_mmproj,
+ _launch_required_ubatch,
+ _mmproj_required_ubatch,
+ _MMPROJ_NON_CAUSAL_IMAGE_TOKENS,
+ _MMPROJ_UNKNOWN_UBATCH,
+)
+
+_GEMMA4 = _MMPROJ_NON_CAUSAL_IMAGE_TOKENS["gemma4uv"]
+
+
+def projector(monkeypatch, tmp_path):
+ """A readable mmproj whose family and image capability the test chooses."""
+
+ def _make(
+ family,
+ *,
+ accepts_image = True,
+ name = "mmproj.gguf",
+ ):
+ import utils.models.gguf_metadata as meta
+
+ path = tmp_path / name
+ path.write_bytes(b"")
+ families = getattr(_make, "_families", {})
+ images = getattr(_make, "_images", {})
+ families[str(path)] = family
+ images[str(path)] = accepts_image
+ _make._families, _make._images = families, images
+ monkeypatch.setattr(
+ meta, "read_mmproj_vision_projector_type", lambda p: families.get(str(p))
+ )
+ monkeypatch.setattr(meta, "mmproj_accepts_image", lambda p: images.get(str(p), True))
+ return str(path)
+
+ return _make
+
+
+class TestMmprojEmitsOversizedChunks:
+ """Which projector families can produce a chunk the 512 default cannot hold."""
+
+ @pytest.mark.parametrize(
+ "family, n_embd, expected",
+ [
+ # Non-causal at every text size, and 1120 tokens per image clears 512.
+ ("gemma4uv", 3840, _GEMMA4),
+ # Non-causal EXCEPT on E2B (n_embd 1536) and E4B (2560).
+ ("gemma4v", 3840, _GEMMA4),
+ ("gemma4v", 2560, 0),
+ ("gemma4v", 1536, 0),
+ # Non-causal but capped under the stock ubatch: 256 and 384.
+ ("gemma3", 2560, 0),
+ ("deepseek4v", 4096, 0),
+ # Causal, so no image size reaches the assert.
+ ("qwen3vl_merger", 2048, 0),
+ ("youtuvl", 4096, 0),
+ ("pixtral", 4096, 0),
+ ],
+ )
+ def test_families(self, projector, family, n_embd, expected):
+ assert _mmproj_required_ubatch(projector(family), n_embd) == expected
+
+ def test_no_projector(self):
+ assert _mmproj_required_ubatch(None) == 0
+
+ def test_an_audio_only_encoder_makes_no_image_chunk(self, projector):
+ # ModelConfig calls every discovered mmproj vision, so these reach here as
+ # vision and must still pay nothing.
+ path = projector("ultravox", accepts_image = False)
+ assert _mmproj_required_ubatch(path, 4096) == 0
+
+ def test_an_unnamed_family_is_assumed_oversized(self, projector):
+ # It could be a Gemma 4, and guessing wrong costs a crash, not an offload.
+ assert _mmproj_required_ubatch(projector(None), 3840) == _MMPROJ_UNKNOWN_UBATCH
+
+ def test_an_unreadable_file_is_assumed_oversized(self, monkeypatch):
+ import utils.models.gguf_metadata as meta
+
+ def _boom(path):
+ raise OSError("unreadable")
+
+ monkeypatch.setattr(meta, "mmproj_accepts_image", _boom)
+ assert _mmproj_required_ubatch("/m/mmproj-F16.gguf", 3840) == _MMPROJ_UNKNOWN_UBATCH
+
+
+class TestLaunchNeedsBiggerUbatch:
+ """Every projector source that can reach the child, asked as one question."""
+
+ def test_the_models_own_projector(self, projector):
+ assert _launch_required_ubatch(projector("gemma4uv"), 3840, env = {}) == _GEMMA4
+ assert _launch_required_ubatch(projector("qwen3vl_merger"), 2048, env = {}) == 0
+
+ def test_a_text_only_model(self, projector):
+ got = _launch_required_ubatch(None, 4096, is_vision = False, env = {})
+ assert got == 0
+
+ def test_the_vision_switch_drops_the_models_own(self, projector):
+ path = projector("gemma4uv")
+ assert _launch_required_ubatch(path, 3840, vision_off = True, env = {}) == 0
+
+ def test_no_mmproj_drops_the_models_own(self, projector):
+ path = projector("gemma4uv")
+ assert _launch_required_ubatch(path, 3840, ["--no-mmproj"], env = {}) == 0
+
+ def test_a_pass_through_projector_survives_both(self, projector):
+ # Appended after the managed flags and stripped by neither the switch nor
+ # --no-mmproj, so this opens an image tower regardless.
+ path = projector("gemma4uv")
+ for extra_kwargs in ({}, {"vision_off": True}):
+ got = _launch_required_ubatch(None, 3840, ["--mmproj", path], env = {}, **extra_kwargs)
+ assert got == _GEMMA4
+
+ def test_a_pass_through_projector_is_still_classified(self, projector):
+ path = projector("qwen3vl_merger")
+ assert _launch_required_ubatch(None, 2048, ["--mmproj", path], env = {}) == 0
+
+ def test_an_inherited_projector_survives_no_mmproj(self, projector):
+ # --no-mmproj empties the command line without clearing mmproj.path.
+ path = projector("gemma4uv")
+ got = _launch_required_ubatch(None, 3840, ["--no-mmproj"], env = {"LLAMA_ARG_MMPROJ": path})
+ assert got == _GEMMA4
+
+ def test_an_inherited_projector_is_still_classified(self, projector):
+ path = projector("qwen3vl_merger")
+ got = _launch_required_ubatch(None, 2048, env = {"LLAMA_ARG_MMPROJ": path})
+ assert got == 0
+
+ def test_an_unfetched_url_counts(self, projector):
+ # Nothing has downloaded it, so it cannot be classified.
+ env = {"LLAMA_ARG_MMPROJ_URL": "https://example.invalid/mmproj.gguf"}
+ got = _launch_required_ubatch(projector("qwen3vl_merger"), 2048, env = env)
+ assert got == _MMPROJ_UNKNOWN_UBATCH
+
+ def test_the_switch_scrubs_the_inherited_pair(self, projector):
+ for var in ("LLAMA_ARG_MMPROJ_URL", "LLAMA_ARG_MMPROJ"):
+ got = _launch_required_ubatch(
+ None, 3840, vision_off = True, env = {var: "https://example.invalid/x.gguf"}
+ )
+ assert got == 0
+
+ def test_mmproj_auto_with_nothing_resolved(self, projector):
+ # Discovery may open an adjacent projector this process was never told about.
+ assert (
+ _launch_required_ubatch(None, 3840, ["--mmproj-auto"], env = {}) == _MMPROJ_UNKNOWN_UBATCH
+ )
+ # Last-wins, exactly as llama-server parses the trio.
+ assert _launch_required_ubatch(None, 3840, ["--mmproj-auto", "--no-mmproj"], env = {}) == 0
+ assert (
+ _launch_required_ubatch(None, 3840, ["--no-mmproj", "--mmproj-auto"], env = {})
+ == _MMPROJ_UNKNOWN_UBATCH
+ )
+
+ def test_mmproj_auto_does_not_trust_this_processs_classification(self, projector):
+ # llama-server runs its own adjacent-projector search and the two can disagree,
+ # so a model this discovery called text-only can still open one.
+ got = _launch_required_ubatch(None, 3840, ["--mmproj-auto"], is_vision = False, env = {})
+ assert got == _MMPROJ_UNKNOWN_UBATCH
+ # With a projector to read, discovery finds that one: classify it rather than
+ # assume headroom.
+ got = _launch_required_ubatch(
+ projector("gemma4uv"), 3840, ["--mmproj-auto"], is_vision = False, env = {}
+ )
+ assert got == _GEMMA4
+
+ def test_the_vision_switch_still_exempts_mmproj_auto(self):
+ # The launch appends --no-mmproj-auto after the extras when it suppresses one.
+ got = _launch_required_ubatch(None, 3840, ["--mmproj-auto"], vision_off = True, env = {})
+ assert got == 0
+
+
+class TestBatchUbatchForMmproj:
+ """Turning the answer into the two flags, without undoing a size the user chose."""
+
+ def test_raised_when_nothing_else_sets_one(self):
+ # Only the micro-batch: llama.cpp's own 2048 batch default already exceeds it.
+ assert _batch_ubatch_for_mmproj(_GEMMA4, None, None, None, {}) == (None, _GEMMA4)
+
+ def test_untouched_when_no_projector_needs_it(self):
+ assert _batch_ubatch_for_mmproj(0, None, None, None, {}) == (None, None)
+
+ def test_an_explicit_micro_batch_is_preserved(self):
+ assert _batch_ubatch_for_mmproj(_GEMMA4, None, 1024, None, {}) == (None, 1024)
+
+ @pytest.mark.parametrize("flag", ["--ubatch-size", "-ub"])
+ def test_a_named_micro_batch_is_left_alone(self, flag):
+ assert _batch_ubatch_for_mmproj(_GEMMA4, None, None, [flag, "1024"], {}) == (None, None)
+
+ def test_an_env_micro_batch_is_left_alone(self):
+ assert _batch_ubatch_for_mmproj(
+ _GEMMA4, None, None, None, {"LLAMA_ARG_UBATCH": "1024"}
+ ) == (
+ None,
+ None,
+ )
+
+ @pytest.mark.parametrize(
+ "batch, expected",
+ [(4096, (4096, _GEMMA4)), (1024, (1024, 1024))],
+ )
+ def test_a_named_batch_caps_the_raise(self, batch, expected):
+ # mtmd cuts the image into n_batch chunks, so the batch caps how big the
+ # micro-batch must be; it does not cancel the raise.
+ assert _batch_ubatch_for_mmproj(_GEMMA4, batch, None, None, {}) == expected
+
+ def test_a_small_batch_already_holds_the_chunk(self):
+ # -b 256 makes every chunk 256, which the llama.cpp default 512 holds.
+ assert _batch_ubatch_for_mmproj(_GEMMA4, None, None, ["-b", "256"], {}) == (None, None)
+
+ def test_a_batch_in_the_extras_caps_the_raise(self):
+ # The field stays None so Unsloth emits no --batch-size; the extras keep theirs.
+ assert _batch_ubatch_for_mmproj(_GEMMA4, None, None, ["-b", "1024"], {}) == (None, 1024)
+
+ @pytest.mark.parametrize("source", ["extras", "env"])
+ def test_a_negative_batch_is_read_as_llama_cpp_reads_it(self, source):
+ # common_params stores the batch signed and llama_context_params casts it to
+ # uint32_t, so -1 reaches the child as 4294967295.
+ args, env = (["-b", "-1"], {}) if source == "extras" else (None, {"LLAMA_ARG_BATCH": "-1"})
+ assert _batch_ubatch_for_mmproj(_GEMMA4, None, None, args, env) == (None, _GEMMA4)
+
+
+def test_the_target_is_the_per_image_ceiling_not_a_round_number():
+ """Use exact image ceilings to avoid unnecessary compute-buffer VRAM."""
+ assert _MMPROJ_NON_CAUSAL_IMAGE_TOKENS == {
+ "gemma4v": 1120,
+ "gemma4uv": 1120,
+ "gemma3": 256,
+ "deepseek4v": 384,
+ }
+ # clip.cpp: set_limit_image_tokens(70, 1120) for both Gemma 4 towers.
+ assert _GEMMA4 == 1120
+ # Only a projector whose family cannot be read gets headroom instead.
+ assert _MMPROJ_UNKNOWN_UBATCH > _GEMMA4
+
+
+ "family, n_embd, custom, expected",
+ [
+ # Stock ceilings under the default micro-batch need nothing raised...
+ ("gemma3", 2560, None, 0),
+ ("deepseek4v", 4096, None, 0),
+ # ...but clip.cpp lets --image-max-tokens replace them, and the chunk grows
+ # with it, so a non-causal family under 512 today can be lifted past it.
+ ("gemma3", 2560, 1024, 1024),
+ ("deepseek4v", 4096, 4096, 4096),
+ # Below the stock micro-batch it changes nothing.
+ ("gemma3", 2560, 256, 0),
+ # A family ceiling already above it wins when the flag asks for less.
+ ("gemma4uv", 3840, 256, 1120),
+ # Causal families never reach the assert, whatever the flag says.
+ ("qwen3vl_merger", 2048, 8192, 0),
+ ("gemma4v", 2560, 8192, 0),
+ ],
+)
+def test_image_max_tokens_is_honoured_for_every_non_causal_family(
+ projector, family, n_embd, custom, expected
+):
+ extras = ["--image-max-tokens", str(custom)] if custom else None
+ assert _mmproj_required_ubatch(projector(family), n_embd, extras) == expected
+
+
+ "extras, env",
+ [
+ (["--image-max-tokens", "4096"], {"LLAMA_ARG_MMPROJ_URL": "https://example.invalid/m"}),
+ (["--mmproj-auto", "--image-max-tokens", "4096"], {}),
+ ],
+)
+def test_image_max_tokens_reaches_the_unclassifiable_projectors_too(extras, env):
+ """Custom limits also raise the fallback for unknown projectors."""
+ assert _launch_required_ubatch(None, 3840, extras, env = env) == 4096
+
+
+def test_the_remote_estimate_honours_a_custom_ceiling_too():
+ """Or the pre-download guard prices less than the post-download launch allocates."""
+ from types import SimpleNamespace
+
+ from studio.backend.routes.inference import _remote_required_ubatch
+
+ config = SimpleNamespace(is_vision = True, gguf_hf_repo = "owner/repo")
+ assert _remote_required_ubatch(config, None, False) == _MMPROJ_UNKNOWN_UBATCH
+ assert _remote_required_ubatch(config, ["--image-max-tokens", "4096"], False) == 4096
+
+
+ "env, expected",
+ [
+ # arg.cpp gives --image-max-tokens the LLAMA_ARG_IMAGE_MAX_TOKENS twin.
+ ({"LLAMA_ARG_IMAGE_MAX_TOKENS": "1024"}, 1024),
+ # A value the stock micro-batch already holds changes nothing.
+ ({"LLAMA_ARG_IMAGE_MAX_TOKENS": "256"}, 0),
+ ({}, 0),
+ ],
+)
+def test_the_image_ceiling_env_twin_is_honoured(projector, env, expected):
+ """gemma3 sits at 256 today, and the environment can lift it past the stock 512."""
+ got = _mmproj_required_ubatch(projector("gemma3"), 2560, None, env)
+ assert got == expected
+
+
+def test_argv_still_wins_over_the_image_ceiling_env_twin(projector):
+ # arg.cpp applies the environment first and argv after it.
+ env = {"LLAMA_ARG_IMAGE_MAX_TOKENS": "1024"}
+ got = _mmproj_required_ubatch(projector("gemma3"), 2560, ["--image-max-tokens", "2048"], env)
+ assert got == 2048
+
+
+ "env, expected",
+ [
+ # The LLAMA_ARG_MMPROJ_AUTO twin, parsed with llama.cpp's own truthy set.
+ ({"LLAMA_ARG_MMPROJ_AUTO": "1"}, _MMPROJ_UNKNOWN_UBATCH),
+ ({"LLAMA_ARG_MMPROJ_AUTO": "on"}, _MMPROJ_UNKNOWN_UBATCH),
+ ({"LLAMA_ARG_MMPROJ_AUTO": "0"}, 0),
+ # arg.cpp reads the NO_ form for any flag with a negative spelling, and its
+ # mere presence is falsey whatever it holds.
+ ({"LLAMA_ARG_MMPROJ_AUTO": "1", "LLAMA_ARG_NO_MMPROJ_AUTO": "anything"}, 0),
+ ({}, 0),
+ ],
+)
+def test_the_discovery_env_twin_is_honoured(env, expected):
+ assert _launch_required_ubatch(None, 3840, None, is_vision = False, env = env) == expected
+
+
+def test_the_batch_still_caps_what_is_emitted():
+ # The batch caps the chunk mtmd cuts, so it caps the micro-batch that must hold it.
+ assert _batch_ubatch_for_mmproj(4096, None, None, None, {})[1] == 2048
+
+
+def test_the_decision_lands_after_the_download_and_before_the_fit():
+ """Resolve the downloaded projector before pricing its micro-batch."""
+ source = inspect.getsource(LlamaCppBackend.load_model)
+ download = source.index("self._download_mmproj(")
+ decide = source.index("_batch_ubatch_for_mmproj(")
+ price = source.index("_ubatch_for_slots(n_parallel)")
+ assert download < decide < price
+ # A missing or mismatched projector must not add image overhead.
+ decision = source[decide : source.index("\n\n", decide)]
+ assert "self._resolve_launch_mmproj_path(" in decision
+
+
+def test_both_sides_read_the_embedding_length_the_same_way():
+ """Use the order-independent embedding reader on both sizing paths."""
+ source = inspect.getsource(LlamaCppBackend.load_model)
+ decision = source[
+ source.index("_batch_ubatch_for_mmproj(") : source.index(
+ "\n\n", source.index("_batch_ubatch_for_mmproj(")
+ )
+ ]
+ assert "_read_gguf_embedding_length(" in decision
+ assert "self._embedding_length" not in decision
+
+
+def test_the_estimators_ask_the_same_question_as_the_launch():
+ """Keep launch, admission, and displayed memory estimates consistent."""
+ from studio.backend.routes import inference as routes
+
+ assert "_launch_required_ubatch" in inspect.getsource(routes._launch_required_ubatch_for_config)
+ for fn in (routes._estimate_gguf_required_gb, routes._gguf_resident_file_gb):
+ body = inspect.getsource(fn)
+ assert "_launch_required_ubatch_for_config(" in body, fn.__name__
+ assert "_remote_required_ubatch(" in body, fn.__name__
+ assert "_launch_required_ubatch_for_config(" in inspect.getsource(routes._gguf_memory_breakdown)