Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: raise the micro-batch for the projectors that abort llama-server on an image (#10683)
Chakshu Dhannawatc3329b9dbf2d
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.pyindex 1f222636590..9f374b00c6d 100644--- a/studio/backend/core/inference/llama_cpp.py+++ b/studio/backend/core/inference/llama_cpp.py@@ -89,6 +89,8 @@ from core.inference.llama_server_args import (apply_model_memory_policy,resolve_ctx_checkpoints,extra_args_disable_mmproj,+ extra_args_image_max_tokens,+ extra_args_mmproj_auto,extra_args_select_load_mode,fit_is_enabled_in,force_pageable_load,@@ -4308,6 +4310,17 @@ def _strip_flag_pairs(args: Iterable[str], flags: frozenset[str]) -> list[str]:# common_params defaults in the bundled llama.cpp runtime._DEFAULT_LLAMA_N_BATCH = 2048_DEFAULT_LLAMA_N_UBATCH = 512+# Non-causal image decoding requires one image chunk to fit in the micro-batch.+# These are the per-image limits from clip.cpp. Exact values avoid excess VRAM use.+_MMPROJ_NON_CAUSAL_IMAGE_TOKENS = {+ "gemma4v": 1120,+ "gemma4uv": 1120,+ "gemma3": 256,+ "deepseek4v": 384,+}+# A projector whose family we cannot read gets headroom instead of a ceiling.+_MMPROJ_UNKNOWN_UBATCH = 2048+_GEMMA4V_CAUSAL_TEXT_N_EMBD = frozenset({1536, 2560}) # E2B and E4B_LLAMA_ARG_TRUE_VALUES = frozenset({"on", "enabled", "true", "1"})_LLAMA_ARG_FALSE_VALUES = frozenset({"off", "disabled", "false", "0"})_LLAMA_ARG_AUTO_VALUES = frozenset({"auto", "-1"})@@ -5748,26 +5761,22 @@ def _extra_args_n_parallel(return found-def _extra_args_n_ubatch(+def _named_batch_sizes(extra_args: Optional[Iterable[str]],env: Optional[Mapping[str, str]] = None,- n_ctx: Optional[int] = None,- *,n_batch: Optional[int] = None,n_ubatch: Optional[int] = None,-) -> Optional[int]:- """Effective ubatch after llama.cpp normalizes it, or None at defaults.+) -> tuple[int, int, bool, bool]:+ """Resolve batch sizes and whether each was explicitly set.- Precedence mirrors the launched command line: env, then the first-class- n_batch / n_ubatch fields (emitted as flags, so they beat env), then user- extra_args (appended last, so they last-wins-override the emitted flags).+ Precedence is environment, first-class fields, then extra arguments."""values = {"batch": _DEFAULT_LLAMA_N_BATCH,"ubatch": _DEFAULT_LLAMA_N_UBATCH,}+ named = {"batch": False, "ubatch": False}source_env = os.environ if env is None else env- overridden = Falsefor key, env_name in (("batch", "LLAMA_ARG_BATCH"),("ubatch", "LLAMA_ARG_UBATCH"),@@ -5776,16 +5785,16 @@ def _extra_args_n_ubatch(if raw:try:values[key] = int(raw)- overridden = True+ named[key] = Trueexcept (TypeError, ValueError):passif n_batch is not None:values["batch"] = int(n_batch)- overridden = True+ named["batch"] = Trueif n_ubatch is not None:values["ubatch"] = int(n_ubatch)- overridden = True+ named["ubatch"] = Trueargs = [str(a) for a in extra_args] if extra_args else []flags = {@@ -5803,10 +5812,26 @@ def _extra_args_n_ubatch(value = inline if eq else (args[i + 1] if i + 1 < len(args) else "")try:values[key] = int(value)- overridden = True+ named[key] = Trueexcept (TypeError, ValueError):continue- if not overridden:+ return values["batch"], values["ubatch"], named["batch"], named["ubatch"]+++def _extra_args_n_ubatch(+ extra_args: Optional[Iterable[str]],+ env: Optional[Mapping[str, str]] = None,+ n_ctx: Optional[int] = None,+ *,+ n_batch: Optional[int] = None,+ n_ubatch: Optional[int] = None,+) -> Optional[int]:+ """Effective ubatch after llama.cpp normalizes it, or None at defaults."""+ _batch, _ubatch, _batch_named, _ubatch_named = _named_batch_sizes(+ extra_args, env, n_batch, n_ubatch+ )+ values = {"batch": _batch, "ubatch": _ubatch}+ if not (_batch_named or _ubatch_named):return None# common_params stores signed values, then llama_context_params converts@@ -5821,6 +5846,135 @@ def _extra_args_n_ubatch(return effective+def _read_gguf_embedding_length(path: Optional[str]) -> Optional[int]:+ """``{arch}.embedding_length``, cached, or None. Thin import-local wrapper."""+ if not path:+ return None+ try:+ from utils.models.gguf_metadata import read_gguf_embedding_length+ return read_gguf_embedding_length(str(path))+ except Exception as e:+ logger.debug(f"embedding length read failed: {e}")+ return None+++def _unknown_projector_ubatch(+ extra_args: Optional[Iterable[str]] = None, env: Optional[Mapping[str, str]] = None+) -> int:+ """Return conservative headroom for an unclassified projector."""+ return max(_MMPROJ_UNKNOWN_UBATCH, extra_args_image_max_tokens(extra_args, env) or 0)+++def _mmproj_required_ubatch(+ mmproj_path: Optional[str],+ n_embd_text: Optional[int] = None,+ extra_args: Optional[Iterable[str]] = None,+ env: Optional[Mapping[str, str]] = None,+) -> int:+ """Return the required micro-batch, or 0 when the default is sufficient.++ Inspect the image tower because audio-only projectors may still be marked as vision.+ """+ if not mmproj_path:+ return 0+ try:+ from utils.models.gguf_metadata import (+ mmproj_accepts_image,+ read_mmproj_vision_projector_type,+ )+ if not mmproj_accepts_image(mmproj_path):+ return 0+ family = (read_mmproj_vision_projector_type(mmproj_path) or "").strip().lower()+ except Exception as e:+ logger.debug(f"mmproj capability read failed: {e}")+ family = ""+ custom = extra_args_image_max_tokens(extra_args, env) or 0+ if not family:+ return _unknown_projector_ubatch(extra_args, env)+ if family not in _MMPROJ_NON_CAUSAL_IMAGE_TOKENS:+ return 0+ if family == "gemma4v" and n_embd_text in _GEMMA4V_CAUSAL_TEXT_N_EMBD:+ return 0+ # A custom image limit can exceed the family's built-in ceiling.+ ceiling = max(_MMPROJ_NON_CAUSAL_IMAGE_TOKENS[family], custom)+ return ceiling if ceiling > _DEFAULT_LLAMA_N_UBATCH else 0+++def _launch_required_ubatch(+ mmproj_path: Optional[str],+ n_embd_text: Optional[int] = None,+ extra_args: Optional[Iterable[str]] = None,+ *,+ is_vision: bool = True,+ vision_off: bool = False,+ env: Optional[Mapping[str, str]] = None,+) -> int:+ """Return the largest micro-batch required by any possible launch projector.++ Taking the maximum avoids duplicating llama.cpp's projector precedence and only+ over-reserves when multiple image towers are supplied.+ """+ required = 0++ # Pass-through arguments override managed projector flags.+ override = _extra_args_device(extra_args, {"--mmproj", "-mm"})+ if override:+ required = max(+ required, _mmproj_required_ubatch(str(override), n_embd_text, extra_args, env)+ )++ if not vision_off:+ # Inherited projectors survive --no-mmproj but not the vision switch.+ source_env = os.environ if env is None else env+ if (source_env.get("LLAMA_ARG_MMPROJ_URL") or "").strip():+ required = max(required, _unknown_projector_ubatch(extra_args, env))+ inherited = (source_env.get("LLAMA_ARG_MMPROJ") or "").strip()+ if inherited:+ required = max(+ required, _mmproj_required_ubatch(inherited, n_embd_text, extra_args, env)+ )++ if is_vision and not vision_off and not extra_args_disable_mmproj(extra_args):+ if mmproj_path:+ required = max(+ required, _mmproj_required_ubatch(str(mmproj_path), n_embd_text, extra_args, env)+ )++ if not vision_off and extra_args_mmproj_auto(extra_args, env):+ # llama-server's adjacent-projector search can disagree with ours.+ required = max(+ required,+ _mmproj_required_ubatch(str(mmproj_path), n_embd_text, extra_args, env)+ if mmproj_path+ else _unknown_projector_ubatch(extra_args, env),+ )++ return required+++def _batch_ubatch_for_mmproj(+ required_ubatch: int,+ n_batch: Optional[int],+ n_ubatch: Optional[int],+ extra_args: Optional[Iterable[str]],+ env: Optional[Mapping[str, str]] = None,+) -> tuple[Optional[int], Optional[int]]:+ """Raise an unset micro-batch, capped by the effective batch size."""+ if required_ubatch <= 0:+ return n_batch, n_ubatch+ batch, ubatch, batch_named, ubatch_named = _named_batch_sizes(+ extra_args, env, n_batch, n_ubatch+ )+ if ubatch_named:+ return n_batch, n_ubatch+ # Match llama_context_params, which converts the signed batch to uint32_t.+ batch &= 0xFFFFFFFF+ target = min(required_ubatch, batch)+ if target <= ubatch:+ return n_batch, n_ubatch+ return n_batch, target++def _build_ngram_mod_flags(caps: Optional[dict],n_match: int = 24,@@ -21226,6 +21380,26 @@ class LlamaCppBackend:logger.info("Load cancelled after download phase")return False+ # Decide after downloading the projector and before pricing the load.+ n_batch, n_ubatch = _batch_ubatch_for_mmproj(+ _launch_required_ubatch(+ None+ if (disable_vision or not is_vision)+ else self._resolve_launch_mmproj_path(+ model_path = model_path,+ mmproj_path = mmproj_path,+ ),+ # Use the same order-independent metadata read as the estimators.+ _read_gguf_embedding_length(model_path),+ extra_args,+ is_vision = is_vision,+ vision_off = disable_vision,+ ),+ n_batch,+ n_ubatch,+ extra_args,+ )+# Backstop for everything the pre-teardown probes fail open on: refuse from the# header rather than watching llama-server die as "failed to start".non_chat = self._non_chat_gguf_refusal(model_path)diff --git a/studio/backend/core/inference/llama_server_args.py b/studio/backend/core/inference/llama_server_args.pyindex b5353e33247..14b8ccc8ca1 100644--- a/studio/backend/core/inference/llama_server_args.py+++ b/studio/backend/core/inference/llama_server_args.py@@ -992,6 +992,59 @@ def extra_args_disable_mmproj(args: Optional[Iterable[str]]) -> bool:return disabled+def extra_args_image_max_tokens(+ args: Optional[Iterable[str]], env: Optional[Mapping[str, str]] = None+) -> Optional[int]:+ """Return the effective positive ``--image-max-tokens`` value."""+ found: Optional[int] = None+ raw_env = (os.environ if env is None else env).get("LLAMA_ARG_IMAGE_MAX_TOKENS")+ if raw_env:+ try:+ parsed_env = int(str(raw_env).strip())+ except (TypeError, ValueError):+ parsed_env = 0+ if parsed_env > 0:+ found = parsed_env+ tokens = [str(a) for a in (args or ())]+ for index, raw in enumerate(tokens):+ if raw.startswith("--image-max-tokens="):+ value = raw.partition("=")[2]+ elif raw == "--image-max-tokens" and index + 1 < len(tokens):+ value = tokens[index + 1]+ else:+ continue+ try:+ parsed = int(value)+ except (TypeError, ValueError):+ continue+ if parsed > 0:+ found = parsed+ return found+++def extra_args_mmproj_auto(+ args: Optional[Iterable[str]], env: Optional[Mapping[str, str]] = None+) -> bool:+ """Return whether llama-server will discover an adjacent projector."""+ source_env = os.environ if env is None else env+ enabled = False+ if source_env.get("LLAMA_ARG_NO_MMPROJ_AUTO") is not None:+ enabled = False+ else:+ raw = source_env.get("LLAMA_ARG_MMPROJ_AUTO")+ if raw is not None:+ enabled = str(raw).strip().lower() in _ENV_TRUE_VALUES+ if not args:+ return enabled+ for raw in args:+ flag = _flag_name(str(raw))+ if flag in _MMPROJ_ENABLE_FLAGS:+ enabled = True+ elif flag in _MMPROJ_DISABLE_FLAGS:+ enabled = False+ return enabled++def strip_shadowing_flags(args: Iterable[str],*,diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.pyindex 005d2f17598..ed9d1999a02 100644--- a/studio/backend/models/inference.py+++ b/studio/backend/models/inference.py@@ -187,9 +187,13 @@ class LoadRequest(BaseModel):description = ("Physical prompt micro-batch size for llama-server (--ubatch-size) "f"for this load ({BATCH_MIN}..{BATCH_MAX}). Omit for the llama.cpp "- "default (512). llama.cpp caps it at the batch size. Larger values "- "speed up prompt processing at the cost of compute-buffer VRAM. "- "Ignored for non-GGUF models."+ "default (512), raised to a projector's own per-image ceiling when "+ "its images would abort the server at 512: 1120 for Gemma 4, and "+ "2048 for a projector whose family cannot be read. Every other "+ "vision model keeps the 512 default. "+ "llama.cpp caps it at the batch size. Larger values speed up prompt "+ "processing at the cost of compute-buffer VRAM. Ignored for "+ "non-GGUF models."),)load_mode: Optional[Literal["auto", "none", "mmap", "mlock", "mmap+mlock", "dio"]] = Field(diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex 397d1a8359e..5d2b5c1b542 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -1889,23 +1889,9 @@ def _openai_llama_admission_output_allowance(def _extra_args_image_max_tokens(extra_args) -> Optional[int]:- """The ``--image-max-tokens N`` a load passed through, or None. Last wins.-- Both spellings, since llama-server accepts ``--flag N`` and ``--flag=N``.- """- args = [str(arg) for arg in (extra_args or ())]- found = None- for index, raw in enumerate(args):- if raw.startswith("--image-max-tokens="):- value = raw.partition("=")[2]- elif raw == "--image-max-tokens" and index + 1 < len(args):- value = args[index + 1]- else:- continue- parsed = _positive_int_or_none(value)- if parsed is not None:- found = parsed- return found+ """Return the effective ``--image-max-tokens`` value."""+ from core.inference.llama_server_args import extra_args_image_max_tokens+ return extra_args_image_max_tokens(extra_args)# Embeddings one image may become, per projector family, from llama.cpp's own@@ -10266,6 +10252,60 @@ def _inherited_ctx_size() -> int:return 0+def _launch_required_ubatch_for_config(+ config,+ llama_extra_args: Optional[list[str]] = None,+ disable_vision: bool = False,+) -> int:+ """Return the required micro-batch for a local GGUF config."""+ from core.inference.llama_cpp import _launch_required_ubatch+ from utils.models.gguf_metadata import read_gguf_embedding_length++ gguf_file = str(getattr(config, "gguf_file", "") or "")+ own = getattr(config, "gguf_mmproj_file", None)+ resolved = None+ if own and getattr(config, "is_vision", False) and not disable_vision:+ resolved = _probe_backend()._resolve_launch_mmproj_path(+ model_path = gguf_file,+ mmproj_path = str(own),+ )+ return _launch_required_ubatch(+ resolved,+ # The text embedding size identifies the causal Gemma 4 variants.+ read_gguf_embedding_length(gguf_file) if gguf_file else None,+ llama_extra_args,+ is_vision = bool(getattr(config, "is_vision", False)),+ vision_off = disable_vision,+ )+++def _remote_required_ubatch(+ config,+ llama_extra_args: Optional[list[str]] = None,+ disable_vision: bool = False,+) -> int:+ """Return a conservative micro-batch for an undownloaded GGUF config."""+ from core.inference.llama_cpp import _launch_required_ubatch, extra_args_disable_mmproj++ from core.inference.llama_cpp import _unknown_projector_ubatch++ if (+ bool(getattr(config, "is_vision", False))+ and not disable_vision+ and not extra_args_disable_mmproj(llama_extra_args)+ ):+ # Match the worst-case post-download allocation.+ return _unknown_projector_ubatch(llama_extra_args)+ # No repo projector in play, but the extras or the environment may still name one.+ return _launch_required_ubatch(+ None,+ None,+ llama_extra_args,+ is_vision = False,+ vision_off = disable_vision,+ )++def _gguf_runtime_bytes(gguf_path: str,max_seq_length: int,@@ -10280,6 +10320,7 @@ def _gguf_runtime_bytes(is_diffusion: bool = False,ctx_last_wins: bool = False,model_identifier: Optional[str] = None,+ launch_required_ubatch: int = 0,) -> _GgufRuntimeBytes:"""KV-cache and compute-buffer VRAM (bytes) at the larger of max_seq_length andany `--ctx-size`/`-c` override, over n_parallel slots at the effective@@ -10295,6 +10336,7 @@ def _gguf_runtime_bytes(over-reserves on purpose; a panel quoting a number to a user wants the otherone, since a smaller ``-c`` in the extras is the context the user gets."""try:+ from core.inference.llama_cpp import _batch_ubatch_for_mmprojfrom core.inference.llama_server_args import (parse_ctx_override,resolve_ctx_checkpoints,@@ -10304,6 +10346,13 @@ def _gguf_runtime_bytes(probe = _probe_backend()probe._model_identifier = model_identifierprobe._read_gguf_metadata(gguf_path)+ # Price the same batch sizes used by load_model.+ n_batch, n_ubatch = _batch_ubatch_for_mmproj(+ 0 if is_diffusion else launch_required_ubatch,+ n_batch,+ n_ubatch,+ llama_extra_args,+ )# Carried out even when the cache cannot be sized: block_count is a separate# key and is usually there, and a caller that loses it prices a manual offload# split as fully GPU-resident (_gguf_offloaded_layer_fraction has nothing to@@ -10583,6 +10632,7 @@ def _estimate_gguf_kv_gb(n_devices: int = 1,is_diffusion: bool = False,model_identifier: Optional[str] = None,+ launch_required_ubatch: int = 0,) -> float:"""``_gguf_runtime_bytes`` summed into GB, for the training guard.@@ -10602,6 +10652,7 @@ def _estimate_gguf_kv_gb(n_devices = n_devices,is_diffusion = is_diffusion,model_identifier = model_identifier,+ launch_required_ubatch = launch_required_ubatch,)return (runtime.kv_bytes + runtime.compute_bytes) / (1024**3)@@ -10615,16 +10666,27 @@ def _remote_gguf_compute_reserve_gb(n_devices: int = 1,tensor_parallel: bool = False,is_diffusion: bool = False,+ required_ubatch: int = 0,) -> float:"""Compute buffers a remote GGUF will reserve, in GB.Split out of _estimate_gguf_required_gb so a caller that is pricing somethingelse, a drafter for instance, can hold it at zero the way it already holds_estimate_gguf_kv_gb at zero. The arithmetic is unchanged.++ ``required_ubatch`` matches the post-download launch."""# remote dims are unreadable; only the kq mask, linear in ubatch x ctx, can be sized here+ from core.inference.llama_cpp import _batch_ubatch_for_mmprojfrom core.inference.llama_server_args import parse_ctx_override+ n_batch, n_ubatch = _batch_ubatch_for_mmproj(+ 0 if is_diffusion else required_ubatch,+ n_batch,+ n_ubatch,+ llama_extra_args,+ )+try:ctx_override = parse_ctx_override(llama_extra_args) or 0except Exception:@@ -11012,6 +11074,9 @@ def _estimate_gguf_required_gb(n_devices = n_devices,is_diffusion = is_diffusion,model_identifier = getattr(config, "identifier", None),+ launch_required_ubatch = _launch_required_ubatch_for_config(+ config, llama_extra_args, disable_vision+ ),)repo = getattr(config, "gguf_hf_repo", None)@@ -11066,6 +11131,7 @@ def _estimate_gguf_required_gb(n_devices = n_devices,tensor_parallel = tensor_parallel,is_diffusion = is_diffusion,+ required_ubatch = _remote_required_ubatch(config, llama_extra_args, disable_vision),)return total_gbreturn None@@ -11493,11 +11559,16 @@ def _gguf_resident_file_gb(0,llama_extra_args,model_identifier = getattr(config, "identifier", None),+ # Subtract the same runtime term added above.+ launch_required_ubatch = _launch_required_ubatch_for_config(+ config, llama_extra_args, disable_vision+ ),)else:context_term_gb = _remote_gguf_compute_reserve_gb(llama_extra_args = llama_extra_args,max_seq_length = 0,+ required_ubatch = _remote_required_ubatch(config, llama_extra_args, disable_vision),)files_gb = max(0.0, required_gb - context_term_gb)# Under the lock: the route body runs in an asyncio.to_thread worker, so two panel@@ -12050,6 +12121,9 @@ def _gguf_memory_breakdown(# An embedding model is recognised from its identifier, not its header, so the# panel has to hand over the same one /load does or it prices a generation model.model_identifier = getattr(config, "identifier", None),+ launch_required_ubatch = _launch_required_ubatch_for_config(+ config, llama_extra_args, disable_vision+ ),)files_gb = _gguf_resident_file_gb(config,diff --git a/studio/backend/tests/test_gpu_init_crash_message.py b/studio/backend/tests/test_gpu_init_crash_message.pyindex 7f9139a6985..2fac895a165 100644--- a/studio/backend/tests/test_gpu_init_crash_message.py+++ b/studio/backend/tests/test_gpu_init_crash_message.py@@ -966,7 +966,8 @@ def test_env_projector_cpu_recovery_keeps_audio_input(monkeypatch, tmp_path):)assert loaded is True- assert read == [str(projector)]+ # The batch-size decision may inspect the same projector earlier in the load.+ assert read and set(read) == {str(projector)}assert backend._is_vision is Trueassert backend._mmproj_has_audio is Truediff --git a/studio/backend/utils/models/gguf_metadata.py b/studio/backend/utils/models/gguf_metadata.pyindex 3b6144aba24..759bcaa9f35 100644--- a/studio/backend/utils/models/gguf_metadata.py+++ b/studio/backend/utils/models/gguf_metadata.py@@ -60,6 +60,8 @@ _NAMED_TENSOR_CACHE: Dict[Tuple[_CacheKey, str], Optional[bool]] = {}# GGUF header dims for the staged UI in one cached pass (context_length, layer_count, moe_layer_count) so the staged sheet can size every slider before the model loads. None = unreadable / not a GGUF; the native ``{arch}.context_length`` the UI shows before a load is read from here via read_gguf_context_length._DIMS_CACHE: Dict[_CacheKey, Optional[Dict[str, Optional[int]]]] = {}+# Read on its own: the only caller wants just this number, on every settings change.+_N_EMBD_CACHE: Dict[_CacheKey, Optional[int]] = {}# Cache the embedded speculative-head count separately for discovery, launch, and sizing.@@ -179,6 +181,26 @@ def read_gguf_staged_dims(path: str) -> Optional[Dict[str, Optional[int]]]:return result+def read_gguf_embedding_length(path: str) -> Optional[int]:+ """Return the cached ``{arch}.embedding_length`` value, if readable."""+ key = _cache_key(path)+ if key is None:+ return None+ with _CACHE_LOCK:+ if key in _N_EMBD_CACHE:+ return _N_EMBD_CACHE[key]+ parsed = _parse_gguf_arch_uints(path, frozenset({"embedding_length"}))+ result = (parsed or {}).get("embedding_length")+ with _CACHE_LOCK:+ while len(_N_EMBD_CACHE) >= _CACHE_MAX_ENTRIES:+ try:+ _N_EMBD_CACHE.pop(next(iter(_N_EMBD_CACHE)))+ except StopIteration:+ break+ _N_EMBD_CACHE[key] = result+ return result++def read_gguf_context_length(path: str) -> Optional[int]:"""Native training context length (``{arch}.context_length``), or ``None``. Thin accessor over read_gguf_staged_dims."""dims = read_gguf_staged_dims(path)@@ -866,6 +888,13 @@ def read_mmproj_projector_type(path: str) -> Optional[str]:return _read_gguf_string(path, "clip.projector_type")+def read_mmproj_vision_projector_type(path: str) -> Optional[str]:+ """Return the image tower family, falling back to the single-tower key."""+ return _read_gguf_string(path, "clip.vision.projector_type") or _read_gguf_string(+ path, "clip.projector_type"+ )++def read_mmproj_vision_capability(path: str) -> Optional[bool]:"""``clip.has_vision_encoder`` from an mmproj GGUF: ``True``/``False`` ifpresent, ``None`` if absent/unreadable."""diff --git a/studio/frontend/src/features/model-picker/components/model-config-page.tsx b/studio/frontend/src/features/model-picker/components/model-config-page.tsxindex 0dc03a28296..4abc35f76f6 100644--- a/studio/frontend/src/features/model-picker/components/model-config-page.tsx+++ b/studio/frontend/src/features/model-picker/components/model-config-page.tsx@@ -1389,9 +1389,10 @@ function GgufAdvancedSettings({<span className={LABEL_CLASS}>Micro-batch Size</span><InfoHint>Physical prompt micro-batch size (--ubatch-size). Leave blank for- the llama.cpp default (512). Larger values speed up prompt- processing but use more VRAM for the compute buffer; capped at the- batch size.+ the llama.cpp default (512), raised to 1120 on Gemma 4 vision+ models, whose images do not fit in 512. Other vision models keep+ the 512 default. Larger values speed up prompt processing but use+ more VRAM for the compute buffer; capped at the batch size.</InfoHint></div><inputdiff --git a/tests/studio/test_llama_cpp_mmproj_batch.py b/tests/studio/test_llama_cpp_mmproj_batch.pynew file mode 100644index 00000000000..3e82f7f3319--- /dev/null+++ b/tests/studio/test_llama_cpp_mmproj_batch.py@@ -0,0 +1,373 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Regression tests for projector-aware llama.cpp micro-batch sizing."""++import inspect++import pytest++from studio.backend.core.inference.llama_cpp import (+ LlamaCppBackend,+ _batch_ubatch_for_mmproj,+ _launch_required_ubatch,+ _mmproj_required_ubatch,+ _MMPROJ_NON_CAUSAL_IMAGE_TOKENS,+ _MMPROJ_UNKNOWN_UBATCH,+)++_GEMMA4 = _MMPROJ_NON_CAUSAL_IMAGE_TOKENS["gemma4uv"]+++def projector(monkeypatch, tmp_path):+ """A readable mmproj whose family and image capability the test chooses."""++ def _make(+ family,+ *,+ accepts_image = True,+ name = "mmproj.gguf",+ ):+ import utils.models.gguf_metadata as meta++ path = tmp_path / name+ path.write_bytes(b"")+ families = getattr(_make, "_families", {})+ images = getattr(_make, "_images", {})+ families[str(path)] = family+ images[str(path)] = accepts_image+ _make._families, _make._images = families, images+ monkeypatch.setattr(+ meta, "read_mmproj_vision_projector_type", lambda p: families.get(str(p))+ )+ monkeypatch.setattr(meta, "mmproj_accepts_image", lambda p: images.get(str(p), True))+ return str(path)++ return _make+++class TestMmprojEmitsOversizedChunks:+ """Which projector families can produce a chunk the 512 default cannot hold."""++ @pytest.mark.parametrize(+ "family, n_embd, expected",+ [+ # Non-causal at every text size, and 1120 tokens per image clears 512.+ ("gemma4uv", 3840, _GEMMA4),+ # Non-causal EXCEPT on E2B (n_embd 1536) and E4B (2560).+ ("gemma4v", 3840, _GEMMA4),+ ("gemma4v", 2560, 0),+ ("gemma4v", 1536, 0),+ # Non-causal but capped under the stock ubatch: 256 and 384.+ ("gemma3", 2560, 0),+ ("deepseek4v", 4096, 0),+ # Causal, so no image size reaches the assert.+ ("qwen3vl_merger", 2048, 0),+ ("youtuvl", 4096, 0),+ ("pixtral", 4096, 0),+ ],+ )+ def test_families(self, projector, family, n_embd, expected):+ assert _mmproj_required_ubatch(projector(family), n_embd) == expected++ def test_no_projector(self):+ assert _mmproj_required_ubatch(None) == 0++ def test_an_audio_only_encoder_makes_no_image_chunk(self, projector):+ # ModelConfig calls every discovered mmproj vision, so these reach here as+ # vision and must still pay nothing.+ path = projector("ultravox", accepts_image = False)+ assert _mmproj_required_ubatch(path, 4096) == 0++ def test_an_unnamed_family_is_assumed_oversized(self, projector):+ # It could be a Gemma 4, and guessing wrong costs a crash, not an offload.+ assert _mmproj_required_ubatch(projector(None), 3840) == _MMPROJ_UNKNOWN_UBATCH++ def test_an_unreadable_file_is_assumed_oversized(self, monkeypatch):+ import utils.models.gguf_metadata as meta++ def _boom(path):+ raise OSError("unreadable")++ monkeypatch.setattr(meta, "mmproj_accepts_image", _boom)+ assert _mmproj_required_ubatch("/m/mmproj-F16.gguf", 3840) == _MMPROJ_UNKNOWN_UBATCH+++class TestLaunchNeedsBiggerUbatch:+ """Every projector source that can reach the child, asked as one question."""++ def test_the_models_own_projector(self, projector):+ assert _launch_required_ubatch(projector("gemma4uv"), 3840, env = {}) == _GEMMA4+ assert _launch_required_ubatch(projector("qwen3vl_merger"), 2048, env = {}) == 0++ def test_a_text_only_model(self, projector):+ got = _launch_required_ubatch(None, 4096, is_vision = False, env = {})+ assert got == 0++ def test_the_vision_switch_drops_the_models_own(self, projector):+ path = projector("gemma4uv")+ assert _launch_required_ubatch(path, 3840, vision_off = True, env = {}) == 0++ def test_no_mmproj_drops_the_models_own(self, projector):+ path = projector("gemma4uv")+ assert _launch_required_ubatch(path, 3840, ["--no-mmproj"], env = {}) == 0++ def test_a_pass_through_projector_survives_both(self, projector):+ # Appended after the managed flags and stripped by neither the switch nor+ # --no-mmproj, so this opens an image tower regardless.+ path = projector("gemma4uv")+ for extra_kwargs in ({}, {"vision_off": True}):+ got = _launch_required_ubatch(None, 3840, ["--mmproj", path], env = {}, **extra_kwargs)+ assert got == _GEMMA4++ def test_a_pass_through_projector_is_still_classified(self, projector):+ path = projector("qwen3vl_merger")+ assert _launch_required_ubatch(None, 2048, ["--mmproj", path], env = {}) == 0++ def test_an_inherited_projector_survives_no_mmproj(self, projector):+ # --no-mmproj empties the command line without clearing mmproj.path.+ path = projector("gemma4uv")+ got = _launch_required_ubatch(None, 3840, ["--no-mmproj"], env = {"LLAMA_ARG_MMPROJ": path})+ assert got == _GEMMA4++ def test_an_inherited_projector_is_still_classified(self, projector):+ path = projector("qwen3vl_merger")+ got = _launch_required_ubatch(None, 2048, env = {"LLAMA_ARG_MMPROJ": path})+ assert got == 0++ def test_an_unfetched_url_counts(self, projector):+ # Nothing has downloaded it, so it cannot be classified.+ env = {"LLAMA_ARG_MMPROJ_URL": "https://example.invalid/mmproj.gguf"}+ got = _launch_required_ubatch(projector("qwen3vl_merger"), 2048, env = env)+ assert got == _MMPROJ_UNKNOWN_UBATCH++ def test_the_switch_scrubs_the_inherited_pair(self, projector):+ for var in ("LLAMA_ARG_MMPROJ_URL", "LLAMA_ARG_MMPROJ"):+ got = _launch_required_ubatch(+ None, 3840, vision_off = True, env = {var: "https://example.invalid/x.gguf"}+ )+ assert got == 0++ def test_mmproj_auto_with_nothing_resolved(self, projector):+ # Discovery may open an adjacent projector this process was never told about.+ assert (+ _launch_required_ubatch(None, 3840, ["--mmproj-auto"], env = {}) == _MMPROJ_UNKNOWN_UBATCH+ )+ # Last-wins, exactly as llama-server parses the trio.+ assert _launch_required_ubatch(None, 3840, ["--mmproj-auto", "--no-mmproj"], env = {}) == 0+ assert (+ _launch_required_ubatch(None, 3840, ["--no-mmproj", "--mmproj-auto"], env = {})+ == _MMPROJ_UNKNOWN_UBATCH+ )++ def test_mmproj_auto_does_not_trust_this_processs_classification(self, projector):+ # llama-server runs its own adjacent-projector search and the two can disagree,+ # so a model this discovery called text-only can still open one.+ got = _launch_required_ubatch(None, 3840, ["--mmproj-auto"], is_vision = False, env = {})+ assert got == _MMPROJ_UNKNOWN_UBATCH+ # With a projector to read, discovery finds that one: classify it rather than+ # assume headroom.+ got = _launch_required_ubatch(+ projector("gemma4uv"), 3840, ["--mmproj-auto"], is_vision = False, env = {}+ )+ assert got == _GEMMA4++ def test_the_vision_switch_still_exempts_mmproj_auto(self):+ # The launch appends --no-mmproj-auto after the extras when it suppresses one.+ got = _launch_required_ubatch(None, 3840, ["--mmproj-auto"], vision_off = True, env = {})+ assert got == 0+++class TestBatchUbatchForMmproj:+ """Turning the answer into the two flags, without undoing a size the user chose."""++ def test_raised_when_nothing_else_sets_one(self):+ # Only the micro-batch: llama.cpp's own 2048 batch default already exceeds it.+ assert _batch_ubatch_for_mmproj(_GEMMA4, None, None, None, {}) == (None, _GEMMA4)++ def test_untouched_when_no_projector_needs_it(self):+ assert _batch_ubatch_for_mmproj(0, None, None, None, {}) == (None, None)++ def test_an_explicit_micro_batch_is_preserved(self):+ assert _batch_ubatch_for_mmproj(_GEMMA4, None, 1024, None, {}) == (None, 1024)++ @pytest.mark.parametrize("flag", ["--ubatch-size", "-ub"])+ def test_a_named_micro_batch_is_left_alone(self, flag):+ assert _batch_ubatch_for_mmproj(_GEMMA4, None, None, [flag, "1024"], {}) == (None, None)++ def test_an_env_micro_batch_is_left_alone(self):+ assert _batch_ubatch_for_mmproj(+ _GEMMA4, None, None, None, {"LLAMA_ARG_UBATCH": "1024"}+ ) == (+ None,+ None,+ )++ @pytest.mark.parametrize(+ "batch, expected",+ [(4096, (4096, _GEMMA4)), (1024, (1024, 1024))],+ )+ def test_a_named_batch_caps_the_raise(self, batch, expected):+ # mtmd cuts the image into n_batch chunks, so the batch caps how big the+ # micro-batch must be; it does not cancel the raise.+ assert _batch_ubatch_for_mmproj(_GEMMA4, batch, None, None, {}) == expected++ def test_a_small_batch_already_holds_the_chunk(self):+ # -b 256 makes every chunk 256, which the llama.cpp default 512 holds.+ assert _batch_ubatch_for_mmproj(_GEMMA4, None, None, ["-b", "256"], {}) == (None, None)++ def test_a_batch_in_the_extras_caps_the_raise(self):+ # The field stays None so Unsloth emits no --batch-size; the extras keep theirs.+ assert _batch_ubatch_for_mmproj(_GEMMA4, None, None, ["-b", "1024"], {}) == (None, 1024)++ @pytest.mark.parametrize("source", ["extras", "env"])+ def test_a_negative_batch_is_read_as_llama_cpp_reads_it(self, source):+ # common_params stores the batch signed and llama_context_params casts it to+ # uint32_t, so -1 reaches the child as 4294967295.+ args, env = (["-b", "-1"], {}) if source == "extras" else (None, {"LLAMA_ARG_BATCH": "-1"})+ assert _batch_ubatch_for_mmproj(_GEMMA4, None, None, args, env) == (None, _GEMMA4)+++def test_the_target_is_the_per_image_ceiling_not_a_round_number():+ """Use exact image ceilings to avoid unnecessary compute-buffer VRAM."""+ assert _MMPROJ_NON_CAUSAL_IMAGE_TOKENS == {+ "gemma4v": 1120,+ "gemma4uv": 1120,+ "gemma3": 256,+ "deepseek4v": 384,+ }+ # clip.cpp: set_limit_image_tokens(70, 1120) for both Gemma 4 towers.+ assert _GEMMA4 == 1120+ # Only a projector whose family cannot be read gets headroom instead.+ assert _MMPROJ_UNKNOWN_UBATCH > _GEMMA4+++ "family, n_embd, custom, expected",+ [+ # Stock ceilings under the default micro-batch need nothing raised...+ ("gemma3", 2560, None, 0),+ ("deepseek4v", 4096, None, 0),+ # ...but clip.cpp lets --image-max-tokens replace them, and the chunk grows+ # with it, so a non-causal family under 512 today can be lifted past it.+ ("gemma3", 2560, 1024, 1024),+ ("deepseek4v", 4096, 4096, 4096),+ # Below the stock micro-batch it changes nothing.+ ("gemma3", 2560, 256, 0),+ # A family ceiling already above it wins when the flag asks for less.+ ("gemma4uv", 3840, 256, 1120),+ # Causal families never reach the assert, whatever the flag says.+ ("qwen3vl_merger", 2048, 8192, 0),+ ("gemma4v", 2560, 8192, 0),+ ],+)+def test_image_max_tokens_is_honoured_for_every_non_causal_family(+ projector, family, n_embd, custom, expected+):+ extras = ["--image-max-tokens", str(custom)] if custom else None+ assert _mmproj_required_ubatch(projector(family), n_embd, extras) == expected+++ "extras, env",+ [+ (["--image-max-tokens", "4096"], {"LLAMA_ARG_MMPROJ_URL": "https://example.invalid/m"}),+ (["--mmproj-auto", "--image-max-tokens", "4096"], {}),+ ],+)+def test_image_max_tokens_reaches_the_unclassifiable_projectors_too(extras, env):+ """Custom limits also raise the fallback for unknown projectors."""+ assert _launch_required_ubatch(None, 3840, extras, env = env) == 4096+++def test_the_remote_estimate_honours_a_custom_ceiling_too():+ """Or the pre-download guard prices less than the post-download launch allocates."""+ from types import SimpleNamespace++ from studio.backend.routes.inference import _remote_required_ubatch++ config = SimpleNamespace(is_vision = True, gguf_hf_repo = "owner/repo")+ assert _remote_required_ubatch(config, None, False) == _MMPROJ_UNKNOWN_UBATCH+ assert _remote_required_ubatch(config, ["--image-max-tokens", "4096"], False) == 4096+++ "env, expected",+ [+ # arg.cpp gives --image-max-tokens the LLAMA_ARG_IMAGE_MAX_TOKENS twin.+ ({"LLAMA_ARG_IMAGE_MAX_TOKENS": "1024"}, 1024),+ # A value the stock micro-batch already holds changes nothing.+ ({"LLAMA_ARG_IMAGE_MAX_TOKENS": "256"}, 0),+ ({}, 0),+ ],+)+def test_the_image_ceiling_env_twin_is_honoured(projector, env, expected):+ """gemma3 sits at 256 today, and the environment can lift it past the stock 512."""+ got = _mmproj_required_ubatch(projector("gemma3"), 2560, None, env)+ assert got == expected+++def test_argv_still_wins_over_the_image_ceiling_env_twin(projector):+ # arg.cpp applies the environment first and argv after it.+ env = {"LLAMA_ARG_IMAGE_MAX_TOKENS": "1024"}+ got = _mmproj_required_ubatch(projector("gemma3"), 2560, ["--image-max-tokens", "2048"], env)+ assert got == 2048+++ "env, expected",+ [+ # The LLAMA_ARG_MMPROJ_AUTO twin, parsed with llama.cpp's own truthy set.+ ({"LLAMA_ARG_MMPROJ_AUTO": "1"}, _MMPROJ_UNKNOWN_UBATCH),+ ({"LLAMA_ARG_MMPROJ_AUTO": "on"}, _MMPROJ_UNKNOWN_UBATCH),+ ({"LLAMA_ARG_MMPROJ_AUTO": "0"}, 0),+ # arg.cpp reads the NO_ form for any flag with a negative spelling, and its+ # mere presence is falsey whatever it holds.+ ({"LLAMA_ARG_MMPROJ_AUTO": "1", "LLAMA_ARG_NO_MMPROJ_AUTO": "anything"}, 0),+ ({}, 0),+ ],+)+def test_the_discovery_env_twin_is_honoured(env, expected):+ assert _launch_required_ubatch(None, 3840, None, is_vision = False, env = env) == expected+++def test_the_batch_still_caps_what_is_emitted():+ # The batch caps the chunk mtmd cuts, so it caps the micro-batch that must hold it.+ assert _batch_ubatch_for_mmproj(4096, None, None, None, {})[1] == 2048+++def test_the_decision_lands_after_the_download_and_before_the_fit():+ """Resolve the downloaded projector before pricing its micro-batch."""+ source = inspect.getsource(LlamaCppBackend.load_model)+ download = source.index("self._download_mmproj(")+ decide = source.index("_batch_ubatch_for_mmproj(")+ price = source.index("_ubatch_for_slots(n_parallel)")+ assert download < decide < price+ # A missing or mismatched projector must not add image overhead.+ decision = source[decide : source.index("\n\n", decide)]+ assert "self._resolve_launch_mmproj_path(" in decision+++def test_both_sides_read_the_embedding_length_the_same_way():+ """Use the order-independent embedding reader on both sizing paths."""+ source = inspect.getsource(LlamaCppBackend.load_model)+ decision = source[+ source.index("_batch_ubatch_for_mmproj(") : source.index(+ "\n\n", source.index("_batch_ubatch_for_mmproj(")+ )+ ]+ assert "_read_gguf_embedding_length(" in decision+ assert "self._embedding_length" not in decision+++def test_the_estimators_ask_the_same_question_as_the_launch():+ """Keep launch, admission, and displayed memory estimates consistent."""+ from studio.backend.routes import inference as routes++ assert "_launch_required_ubatch" in inspect.getsource(routes._launch_required_ubatch_for_config)+ for fn in (routes._estimate_gguf_required_gb, routes._gguf_resident_file_gb):+ body = inspect.getsource(fn)+ assert "_launch_required_ubatch_for_config(" in body, fn.__name__+ assert "_remote_required_ubatch(" in body, fn.__name__+ assert "_launch_required_ubatch_for_config(" in inspect.getsource(routes._gguf_memory_breakdown)