Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Unsloth Studio / Desktop: support custom vision projectors (#11352)
Maheswar Kumar6c1cc50e84bc
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.pyindex b371f25f543..c2c021edadc 100644--- a/studio/backend/core/inference/llama_cpp.py+++ b/studio/backend/core/inference/llama_cpp.py@@ -6541,7 +6541,7 @@ def _launch_required_ubatch(# Pass-through arguments override managed projector flags.override = _extra_args_device(extra_args, {"--mmproj", "-mm"})- if override:+ if override and not vision_off and not extra_args_disable_mmproj(extra_args):required = max(required, _mmproj_required_ubatch(str(override), n_embd_text, extra_args, env))@@ -19884,12 +19884,18 @@ class LlamaCppBackend:return Falsedef _resolve_launch_mmproj_path(- self, *, model_path: str, mmproj_path: Optional[str]+ self,+ *,+ model_path: str,+ mmproj_path: Optional[str],+ extra_args: Optional[Iterable[str]] = None,) -> Optional[str]:- """Return mmproj_path iff it exists on disk AND matches the model family.-- None if mmproj_path is None, missing, or family-mismatched.- """+ """Resolve a projector, trusting an explicit path over filename heuristics."""+ override = _extra_args_device(extra_args, {"--mmproj", "-mm"})+ if override is not None:+ if not override or not Path(override).is_file():+ raise ValueError("The custom mmproj path must name an existing file.")+ return overrideif not mmproj_path:return None@@ -22490,17 +22496,15 @@ class LlamaCppBackend:@staticmethoddef _strip_mmproj_args(cmd: list[str]) -> list[str]:- """Return cmd without the '--mmproj <path>' pair (text-only retry).- Every other flag is preserved; a no-op when --mmproj is absent.- """+ """Remove explicit projector paths, including aliases, for managed emission or retry."""out: list[str] = []skip_value = Falsefor tok in cmd:if skip_value:skip_value = Falsecontinue- if tok == "--mmproj":- skip_value = True+ if _flag_name(tok) in {"--mmproj", "-mm"}:+ skip_value = "=" not in tokcontinueout.append(tok)return out@@ -23162,6 +23166,12 @@ class LlamaCppBackend:ctx_checkpoints = intent.ctx_checkpointscache_ram = intent.cache_ramextra_args = list(intent.extra_args) if intent.extra_args is not None else None+ custom_mmproj = _extra_args_device(extra_args, {"--mmproj", "-mm"})+ if custom_mmproj is not None:+ mmproj_path = self._resolve_launch_mmproj_path(+ model_path = gguf_path or "", mmproj_path = mmproj_path, extra_args = extra_args+ )+ is_vision = Truepreserve_multi_gpu_on_layer = intent.preserve_multi_gpu_on_layerreasoning_budget = intent.reasoning_budgetreasoning_budget_message = intent.reasoning_budget_message@@ -23882,6 +23892,7 @@ class LlamaCppBackend:else self._resolve_launch_mmproj_path(model_path = model_path,mmproj_path = mmproj_path,+ extra_args = extra_args,),# Use the same order-independent metadata read as the estimators._read_gguf_embedding_length(model_path),@@ -24250,6 +24261,7 @@ class LlamaCppBackend:launch_mmproj_path = self._resolve_launch_mmproj_path(model_path = model_path,mmproj_path = mmproj_path,+ extra_args = extra_args,)# The switch turns VISION off, and a projector is not always a# vision tower: ultravox, Voxtral and Qwen3-ASR declare an audio@@ -28506,7 +28518,7 @@ class LlamaCppBackend:# User pass-through args go last. Placement flags are removed# below when the Unsloth picker owns the GPU selection.if _mem_extras:- _emit_extra_args = list(_mem_extras)+ _emit_extra_args = self._strip_mmproj_args(list(_mem_extras))if _gpu_ids_own_device_flags:# gpu_ids owns placement, so remove competing device flags._before_device_strip = list(_emit_extra_args)@@ -32105,15 +32117,22 @@ class LlamaCppBackend:intent.model_identifier.lower()):return False- if intent.gguf_path is not None and self._gguf_path:+ custom_mmproj = _extra_args_device(intent.extra_args, {"--mmproj", "-mm"})+ source_path = intent.gguf_path+ if source_path is None and custom_mmproj is not None:+ if (self._hf_variant or "").lower() != (intent.hf_variant or "").lower():+ return False+ source_path = self._gguf_path+ if source_path is not None and self._gguf_path:resident_identity = getattr(self, "_gguf_load_identity", None)if resident_identity is not None:candidate_identity = LlamaCppBackend._gguf_load_source_identity(- intent.gguf_path, intent.mmproj_path+ source_path,+ custom_mmproj or intent.mmproj_path,)return candidate_identity == resident_identitytry:- return Path(self._gguf_path).resolve() == Path(intent.gguf_path).resolve()+ return Path(self._gguf_path).resolve() == Path(source_path).resolve()except OSError:return Falsereturn (self._hf_variant or "").lower() == (intent.hf_variant or "").lower()@@ -32137,7 +32156,7 @@ class LlamaCppBackend:the main ``--gpu-layers``. A drafter explicitly forced to CPU(--spec-draft-ngl 0 / --spec-draft-device cpu) doesn't count."""_args = [str(a) for a in cmd]- if any(a.startswith("--mmproj") for a in _args):+ if any(a.startswith("--mmproj") or _flag_name(a) == "-mm" for a in _args):# --no-mmproj-offload clears mmproj_use_gpu and clip.cpp gates the whole# GPU backend on it, so the projector holds no VRAM. Last flag wins._off = [a for a in _args if a in ("--mmproj-offload", "--no-mmproj-offload")]@@ -33853,6 +33872,8 @@ class LlamaCppBackend:return None_SIDECAR_WEIGHT_FLAGS = (+ "--mmproj",+ "-mm","--lora","--lora-scaled","--control-vector",diff --git a/studio/backend/core/inference/llama_server_args.py b/studio/backend/core/inference/llama_server_args.pyindex cd11322285f..716a9816f33 100644--- a/studio/backend/core/inference/llama_server_args.py+++ b/studio/backend/core/inference/llama_server_args.py@@ -82,7 +82,6 @@ _DENYLIST_GROUPS: tuple[frozenset[str], ...] = (frozenset({"-hfv", "-hfrv", "--hf-repo-v"}),frozenset({"-hffv", "--hf-file-v"}),frozenset({"-hft", "--hf-token"}),- frozenset({"-mm", "--mmproj"}),frozenset({"-mmu", "--mmproj-url"}),# Networking: Unsloth binds + proxies; retargeting orphans the proxy.frozenset({"--host"}),@@ -1815,13 +1814,7 @@ DENIED_ENV_VARS: tuple[str, ...] = ("LLAMA_ARG_UI_CONFIG_FILE","LLAMA_ARG_UI_MCP_PROXY","LLAMA_ARG_STATIC_PATH",- # Deliberately absent: LLAMA_ARG_MMPROJ and LLAMA_ARG_MMPROJ_URL. --mmproj is refused in the box because Unsloth- # resolves the projector itself, but the environment twin is an INPUT here: _launch_has_mmproj reads both to know- # the launch has a projector at all, which is what keeps the vision and audio state of a model loaded through an- # inherited one. Only the paravirtual CPU recovery drops them, where an unpinned projector is the corrupt path it- # is undoing. The pooling twins are absent for the opposite reason: load_model already pops LLAMA_ARG_POOLING /- # _RERANKING / _EMBEDDINGS itself. The multi-model server mode is absent too: a child holding its own model- # directory, preset and autoload policy is not the single model Unsloth launched and accounts for.+ # LLAMA_ARG_MMPROJ / _URL stay allowed: _launch_has_mmproj reads them as launch inputs."LLAMA_ARG_MODELS_DIR","LLAMA_ARG_MODELS_PRESET","LLAMA_ARG_MODELS_MAX",diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex b50d0f0954e..09dd5e28646 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -7518,6 +7518,19 @@ def _external_transcript_preview(response: Response) -> str:return ""+def _refuse_managed_custom_projector(extra_args: Optional[list[str]]) -> None:+ """A pass-through projector path skips account model access, so only the owner may name one."""+ from core.inference.llama_cpp import _extra_args_device+ if (+ account_access.managed_account()+ and _extra_args_device(extra_args, {"--mmproj", "-mm"}) is not None+ ):+ raise HTTPException(+ status_code = 403,+ detail = "A custom --mmproj path is available to this installation's owner only.",+ )++def _validate_native_gguf_companion(companion_path: str | None,gguf_path: str | None,@@ -11642,12 +11655,14 @@ def _remote_required_ubatch("""Return a conservative micro-batch for an undownloaded GGUF config."""from core.inference.llama_cpp import _launch_required_ubatch, extra_args_disable_mmproj- from core.inference.llama_cpp import _unknown_projector_ubatch+ from core.inference.llama_cpp import _extra_args_device, _unknown_projector_ubatch+ # A local --mmproj replaces the repo projector, so size from that file instead.if (bool(getattr(config, "is_vision", False))and not disable_visionand not extra_args_disable_mmproj(llama_extra_args)+ and _extra_args_device(llama_extra_args, {"--mmproj", "-mm"}) is None):# Match the worst-case post-download allocation.return _unknown_projector_ubatch(llama_extra_args)@@ -12341,6 +12356,7 @@ def _estimate_gguf_required_gb(# projector despite the switch, and dropping bytes that do get opened would# admit a load the running training job cannot afford, so ask the loader's own# question of the file. Same gate as the remote branch's include_mmproj.+ _mmproj_override = _extra_args_device(llama_extra_args, {"--mmproj", "-mm"})_sized_attrs = ["gguf_mmproj_file"]# Whether the CONFIGURED projector is one this launch opens. Bound before the# switch so the inherited-projector gate below can read it either way.@@ -12352,7 +12368,7 @@ def _estimate_gguf_required_gb(_dv_opens_projector = False_sized_attrs = []elif disable_vision:- _dv_mmproj = getattr(config, "gguf_mmproj_file", None)+ _dv_mmproj = _mmproj_override or getattr(config, "gguf_mmproj_file", None)_dv_opens_projector = Falseif _dv_mmproj:try:@@ -12372,11 +12388,9 @@ def _estimate_gguf_required_gb(# possibly much larger custom projector went free._mmproj_override_bytes = 0if _sized_attrs == ["gguf_mmproj_file"]:- _mmproj_override = _extra_args_device(llama_extra_args, {"--mmproj", "-mm"})if _mmproj_override and Path(_mmproj_override).is_file():_sized_attrs = []_mmproj_override_bytes = LlamaCppBackend._get_gguf_size_bytes(_mmproj_override)- total_bytes += _mmproj_override_bytes_sized_keys.add(_same_file_key(_mmproj_override))if not _charge_no_drafter:if dspark_requested:@@ -12470,7 +12484,7 @@ def _estimate_gguf_required_gb(# extras that skipped the resolve both leave it empty and let the inherited path# load -- so this asks what Unsloth emits, not what the config names._studio_mmproj_on_argv = bool(- getattr(config, "gguf_mmproj_file", None) and _dv_opens_projector+ (_mmproj_override or getattr(config, "gguf_mmproj_file", None)) and _dv_opens_projector)_env_mmproj_bytes = 0_env_mmproj = (os.environ.get("LLAMA_ARG_MMPROJ") or "").strip()@@ -12484,7 +12498,7 @@ def _estimate_gguf_required_gb(_env_mmproj_bytes = LlamaCppBackend._get_gguf_size_bytes(_env_mmproj)if total_bytes > 0:- return (total_bytes + _extras_bytes + _env_mmproj_bytes) / (+ return (total_bytes + _mmproj_override_bytes + _extras_bytes + _env_mmproj_bytes) / (1024**3) + _estimate_gguf_kv_gb(main,@@ -12524,7 +12538,7 @@ def _estimate_gguf_required_gb(# the file to ask. Under-charging is what would admit a chat load over# VRAM a training job needs, so an unknown projector is charged. The# local branch, holding the file, asks instead.- include_mmproj = bool(has_vision),+ include_mmproj = bool(has_vision) and _mmproj_override is None,# Remote, so which sidecar the repo ships is unknown until the# listing. Under Auto size both: a repo has one kind or the other,# the absent one contributes 0, and over-estimating is the safe@@ -12547,7 +12561,9 @@ def _estimate_gguf_required_gb(# Plus the caller's own --model-draft / --spec-draft-hf, if they named# one: this repo's listing cannot see it, local or remote, and it is# resident next to these weights.- total_gb = (main_bytes + companions + _extras_bytes) / (1024**3)+ total_gb = (main_bytes + companions + _mmproj_override_bytes + _extras_bytes) / (+ 1024**3+ )total_gb += _remote_gguf_compute_reserve_gb(llama_extra_args = llama_extra_args,max_seq_length = max_seq_length,@@ -16788,6 +16804,7 @@ async def _load_model_impl(extra_llama_args: Optional[list[str]] = (None if request.llama_extra_args is None else extra_llama_args)+ _refuse_managed_custom_projector(extra_llama_args)_reasoning_updates = {}_reasoning_budget_override = parse_reasoning_budget_override(extra_llama_args)@@ -18215,6 +18232,8 @@ async def validate_model(validate_extra_args(effective_extra_args)except ValueError as exc:raise HTTPException(status_code = 400, detail = str(exc)) from exc+ if getattr(request, "llama_extra_args", None) is not None:+ _refuse_managed_custom_projector(effective_extra_args)# Manual mode owns the offload flags, and /load translates an explicit -ngl# into the first-class field before it strips them. Doing that there and notdiff --git a/studio/backend/tests/test_detect_mmproj_file.py b/studio/backend/tests/test_detect_mmproj_file.pyindex 156ae347b03..2b84ac16c9e 100644--- a/studio/backend/tests/test_detect_mmproj_file.py+++ b/studio/backend/tests/test_detect_mmproj_file.py@@ -9,6 +9,8 @@ from pathlib import Pathimport struct+import pytest+from utils.models.model_config import (_detect_family_token,detect_mmproj_file,@@ -20,14 +22,17 @@ _GGUF_MAGIC = 0x46554747def _gguf_with_general(path: Path, fields: dict) -> Path:- """Write a minimal GGUF with only ``general.*`` string KVs."""+ """Write a minimal GGUF with string and unsigned integer KVs."""body = b""for k, v in fields.items():kb = k.encode("utf-8")- vb = v.encode("utf-8")body += struct.pack("<Q", len(kb)) + kb- body += struct.pack("<I", 8) # STRING vtype- body += struct.pack("<Q", len(vb)) + vb+ if isinstance(v, int):+ body += struct.pack("<II", 4, v)+ else:+ vb = v.encode("utf-8")+ body += struct.pack("<I", 8)+ body += struct.pack("<Q", len(vb)) + vbheader = struct.pack("<IIQQ", _GGUF_MAGIC, 3, 0, len(fields))path.parent.mkdir(parents = True, exist_ok = True)path.write_bytes(header + body)@@ -447,3 +452,49 @@ def test_a_draft_model_under_assets_is_not_mistaken_for_a_projector(tmp_path: Pamodel = _touch(tmp_path / "Qwen3.8-27B-UD-Q4_K_M.gguf")_touch(tmp_path / "assets" / "Qwen3.8-0.8B-draft-Q4_K_M.gguf")assert detect_mmproj_file(str(model)) is None++[email protected]("projection_dim", [5376, 2560, 0])[email protected]("identity_matches", [False, True])+def test_gemma4_finetune_pairs_by_functional_metadata(tmp_path, projection_dim, identity_matches):+ model = _gguf_with_general(+ tmp_path / "gembrain.gguf",+ {+ "general.architecture": "gemma4",+ "general.basename": "Gemma-4" if identity_matches else "Gemma-4-Gembrain",+ "general.base_model.0.repo_url": (+ "https://huggingface.co/google/gemma-4-31B-it"+ if identity_matches+ else "https://huggingface.co/community/Gembrain"+ ),+ "gemma4.embedding_length": 5376,+ },+ )+ projector = _gguf_with_general(+ tmp_path / "mmproj-gemma-4-31B-it-BF16.gguf",+ {+ "general.architecture": "clip",+ "general.type": "mmproj",+ "general.basename": "Gemma-4",+ "general.base_model.0.repo_url": "https://huggingface.co/google/gemma-4-31B-it",+ "clip.vision.projector_type": "gemma4v",+ "clip.vision.projection_dim": projection_dim,+ },+ )+ expected = (+ str(projector)+ if projection_dim == 5376 or (projection_dim == 0 and identity_matches)+ else None+ )+ assert detect_mmproj_file(str(model)) == expected+ if projection_dim:+ assert mmproj_matches_model_family(str(model), str(projector)) == (projection_dim == 5376)+++def test_mmproj_rejection_names_mismatched_fields(tmp_path, capsys):+ model = _gguf_with_general(tmp_path / "model.gguf", {"general.basename": "Foo"})+ _gguf_with_general(tmp_path / "mmproj.gguf", {"general.basename": "Bar"})+ assert detect_mmproj_file(str(model)) is None+ output = capsys.readouterr().out+ assert "general.basename" in output+ assert "Foo" in output and "Bar" in outputdiff --git a/studio/backend/tests/test_gpu_memory_mode.py b/studio/backend/tests/test_gpu_memory_mode.pyindex 7a8a0090b2b..940abc3b0ff 100644--- a/studio/backend/tests/test_gpu_memory_mode.py+++ b/studio/backend/tests/test_gpu_memory_mode.py@@ -1280,7 +1280,7 @@ def test_zero_offload_flag_false_without_companions():@pytest.mark.parametrize("companion",- ["--mmproj", "--model-draft", "-md", "--spec-draft-model", "-hfd"],+ ["--mmproj", "-mm", "--model-draft", "-md", "--spec-draft-model", "-hfd"],)def test_zero_offload_flag_true_with_companion(companion):# mmproj / a drafter offload to GPU regardless of --gpu-layers, so thediff --git a/studio/backend/tests/test_llama_server_args.py b/studio/backend/tests/test_llama_server_args.pyindex 0fb10c51021..4985fa95ebf 100644--- a/studio/backend/tests/test_llama_server_args.py+++ b/studio/backend/tests/test_llama_server_args.py@@ -184,8 +184,6 @@ def test_a_bare_positional_is_rejected():"--hf-file-v","-hft","--hf-token",- "-mm",- "--mmproj","-mmu","--mmproj-url",# Networking (Unsloth binds + proxies)@@ -1675,3 +1673,9 @@ def test_a_pageable_launch_keeps_every_token_including_its_lock():out, overridden = _lsa.force_pageable_load(list(argv), {})assert out == argv and overridden == []++[email protected]("flag", ["--mmproj", "-mm"])+def test_custom_projector_is_allowed(flag):+ args = [flag, "/models/custom projector.gguf"]+ assert validate_extra_args(args) == argsdiff --git a/studio/backend/tests/test_mmproj_placement_policy.py b/studio/backend/tests/test_mmproj_placement_policy.pyindex c9d77e68796..4326a293fb9 100644--- a/studio/backend/tests/test_mmproj_placement_policy.py+++ b/studio/backend/tests/test_mmproj_placement_policy.py@@ -16,6 +16,7 @@ import inspectimport osimport structimport subprocess+from dataclasses import replacefrom pathlib import Pathfrom types import SimpleNamespacefrom unittest.mock import patch@@ -1680,3 +1681,155 @@ def test_a_replayed_context_refits_for_a_cpu_pinned_drafter_too(tmp_path):assert replay_ctx == fresh_ctx < replayedassert ("--no-mmproj-offload" in replay) == ("--no-mmproj-offload" in fresh)++[email protected]("flag", ["--mmproj", "-mm"])[email protected]("vision_off", [False, True])+def test_custom_projector_replaces_discovery_and_respects_vision_switch(tmp_path, flag, vision_off):+ backend, gguf = _backend(tmp_path, memory = [(0, 12_000, 24_000)])+ backend._resolve_launch_mmproj_path = LlamaCppBackend._resolve_launch_mmproj_path.__get__(+ backend+ )+ gguf = gguf.rename(tmp_path / "Qwen-model.gguf")+ custom = _write_gguf(tmp_path / "gemma-custom projector.gguf")+ seen = []+ backend._mmproj_vram_bytes = lambda path: seen.append(path) or GIB+ cmd = _launch(+ backend,+ gguf,+ is_vision = False,+ disable_vision = vision_off,+ extra_args = [flag, str(custom)],+ )["cmd"]+ if vision_off:+ assert "--mmproj" not in cmd and "-mm" not in cmd+ else:+ assert cmd.count("--mmproj") == 1+ assert cmd[cmd.index("--mmproj") + 1] == str(custom)+ assert str(custom) in seen+++def test_vision_off_still_charges_a_custom_audio_only_projector(tmp_path, monkeypatch):+ model = _write_gguf(tmp_path / "model.gguf")+ custom = _write_gguf(tmp_path / "custom-audio.gguf")+ monkeypatch.setattr(_meta, "mmproj_accepts_image", lambda _path: False)+ config = SimpleNamespace(+ gguf_file = str(model),+ gguf_mmproj_file = None,+ gguf_mtp_file = None,+ gguf_dspark_file = None,+ gguf_dflash_file = None,+ gguf_hf_repo = None,+ gguf_variant = None,+ is_vision = False,+ )+ enabled = _estimate_gguf_required_gb(config, llama_extra_args = ["--mmproj", str(custom)])+ disabled = _estimate_gguf_required_gb(+ config,+ llama_extra_args = ["--mmproj", str(custom)],+ disable_vision = True,+ )+ assert enabled == disabled++[email protected]("vision_off", [False, True])+def test_custom_projector_does_not_hide_remote_model_bytes(tmp_path, monkeypatch, vision_off):+ import routes.inference as routes+ import utils.models.model_config as model_config++ custom = _write_gguf(tmp_path / "custom-vision.gguf")+ config = SimpleNamespace(+ gguf_file = None,+ gguf_mmproj_file = None,+ gguf_mtp_file = None,+ gguf_dspark_file = None,+ gguf_dflash_file = None,+ gguf_hf_repo = "org/vision-GGUF",+ gguf_variant = "Q4_K_M",+ is_vision = True,+ )+ monkeypatch.setattr(+ model_config,+ "list_gguf_variants",+ lambda *_a, **_kw: (+ [SimpleNamespace(quant = "Q4_K_M", size_bytes = 4 * GIB)],+ True,+ ),+ )+ seen = []+ monkeypatch.setattr(+ routes, "_remote_gguf_companion_bytes", lambda *_a, **kw: seen.append(kw) or 0+ )+ monkeypatch.setattr(routes, "_remote_gguf_compute_reserve_gb", lambda **_kw: 0)+ result = _estimate_gguf_required_gb(+ config,+ llama_extra_args = ["--mmproj", str(custom)],+ disable_vision = vision_off,+ speculative_type = "off",+ )+ expected = 4 * GIB + (0 if vision_off else custom.stat().st_size)+ assert result * GIB == expected+ assert seen[0]["include_mmproj"] is False+++def test_missing_custom_projector_is_rejected_before_unloading(tmp_path, monkeypatch):+ backend = LlamaCppBackend()+ unloaded = []+ monkeypatch.setattr(backend, "unload_model", lambda: unloaded.append(True))+ with pytest.raises(ValueError, match = "custom mmproj path"):+ backend.load_model(+ GgufLoadIntent(+ model_identifier = "test",+ gguf_path = str(tmp_path / "model.gguf"),+ extra_args = ("--mmproj", str(tmp_path / "missing.gguf")),+ )+ )+ assert unloaded == []++[email protected]("flag", ["--mmproj", "-mm"])+def test_hub_load_rechecks_custom_projector_replaced_in_place(tmp_path, flag):+ backend, gguf = _backend(tmp_path, memory = [(0, 12_000, 24_000)])+ backend._resolve_launch_mmproj_path = LlamaCppBackend._resolve_launch_mmproj_path.__get__(+ backend+ )+ custom = _write_gguf(tmp_path / "custom-projector.gguf")+ _launch(backend, gguf, is_vision = False, extra_args = [flag, str(custom)])+ backend._hf_variant = "Q4_K_M"+ intent = GgufLoadIntent(+ model_identifier = "test",+ hf_repo = "org/model",+ hf_variant = "Q4_K_M",+ extra_args = (flag, str(custom)),+ )+ assert backend.matches_load_source(intent)+ assert not backend.matches_load_source(replace(intent, hf_variant = "Q8_0"))+ replacement = _write_gguf(tmp_path / "replacement.gguf")+ replacement.replace(custom)+ assert not backend.matches_load_source(intent)+++def test_remote_ubatch_sizes_the_custom_projector_not_the_repo_one(tmp_path, monkeypatch):+ import routes.inference as routes++ custom = _write_gguf(tmp_path / "custom-audio.gguf")+ monkeypatch.setattr(_meta, "mmproj_accepts_image", lambda _path: False)+ config = SimpleNamespace(is_vision = True)+ assert routes._remote_required_ubatch(config, []) > 0+ assert routes._remote_required_ubatch(config, ["--mmproj", str(custom)]) == 0++[email protected]("flag", ["--mmproj", "-mm"])[email protected]("managed", [False, True])+def test_only_the_owner_may_name_a_custom_projector(monkeypatch, flag, managed):+ import routes.inference as routes+ from fastapi import HTTPException++ monkeypatch.setattr(routes.account_access, "managed_account", lambda: managed)+ routes._refuse_managed_custom_projector(["--ctx-size", "4096"])+ if not managed:+ routes._refuse_managed_custom_projector([flag, "/models/p.gguf"])+ return+ with pytest.raises(HTTPException) as err:+ routes._refuse_managed_custom_projector([flag, "/models/p.gguf"])+ assert err.value.status_code == 403diff --git a/studio/backend/tests/test_tp_vision_regression.py b/studio/backend/tests/test_tp_vision_regression.pyindex 3ff2fd9ecac..b736866f810 100644--- a/studio/backend/tests/test_tp_vision_regression.py+++ b/studio/backend/tests/test_tp_vision_regression.py@@ -385,9 +385,10 @@ def test_tensor_split_abort_raises_early_to_layer_fallback():# The strip is matched on the call, not on what is passed to it. What this# test is about is that the abort raises BEFORE the projector is thrown# away; which command the strip reads from is that code's own business.+ # `cmd = ` skips the extra-args strip, whose --mmproj the managed flag re-emits.for label, needle in (("the flash-attn-off retry", "_with_flash_attn_off"),- ("the text-only mmproj strip", "_strip_mmproj_args("),+ ("the text-only mmproj strip", "cmd = self._strip_mmproj_args("),):idx = src.find(needle)assert idx != -1, (diff --git a/studio/backend/utils/models/gguf_metadata.py b/studio/backend/utils/models/gguf_metadata.pyindex 759bcaa9f35..70778969377 100644--- a/studio/backend/utils/models/gguf_metadata.py+++ b/studio/backend/utils/models/gguf_metadata.py@@ -1009,6 +1009,26 @@ def _weight_url_looks_like_derivative_of_projector(weight_url: str, projector_urreturn _slug_extends_base(weight_slug, projector_slug)+def mmproj_functional_match(weight_path: str, mmproj_path: str) -> tuple[Optional[bool], str]:+ """Gemma 4 only: pair by projector type + projection dim, not branding. None = unknown."""+ if read_gguf_architecture(weight_path) != "gemma4":+ return None, ""+ projector_type = read_mmproj_vision_projector_type(mmproj_path)+ if projector_type != "gemma4v":+ return None, ""+ embedding_length = read_gguf_embedding_length(weight_path)+ dims = _parse_gguf_arch_uints(mmproj_path, frozenset({"vision.projection_dim"})) or {}+ projection_dim = dims.get("vision.projection_dim")+ if not embedding_length or not projection_dim:+ return None, ""+ if embedding_length != projection_dim:+ return False, (+ f"gemma4.embedding_length {embedding_length} != "+ f"clip.vision.projection_dim {projection_dim}"+ )+ return True, ""++def pairing_score(weight_meta: Optional[Dict[str, str]], mmproj_meta: Optional[Dict[str, str]]) -> int:diff --git a/studio/backend/utils/models/model_config.py b/studio/backend/utils/models/model_config.pyindex d90f4ceb32f..33747fe8ca7 100644--- a/studio/backend/utils/models/model_config.py+++ b/studio/backend/utils/models/model_config.py@@ -48,6 +48,7 @@ from utils.audio_tokens import (from utils.models.gguf_metadata import (is_mmproj_by_metadata,mmproj_accepts_image,+ mmproj_functional_match,pairing_score,read_gguf_general_metadata,read_gguf_nextn_predict_layers,@@ -1934,8 +1935,10 @@ def _detect_family_token(filename: str) -> Optional[str]:def mmproj_matches_model_family(model_path: str, mmproj_path: str) -> bool:- """Launcher guard: True unless both filenames carry recognised family- tokens that disagree."""+ """Prefer known functional metadata, falling back to filename family hints."""+ functional_match, _ = mmproj_functional_match(model_path, mmproj_path)+ if functional_match is not None:+ return functional_matchmodel_fam = _detect_family_token(Path(model_path).name)mmproj_fam = _detect_family_token(Path(mmproj_path).name)if model_fam is None or mmproj_fam is None:@@ -2193,8 +2196,24 @@ def detect_mmproj_file(for c in candidates:cand_meta = read_gguf_general_metadata(str(c))meta_score = pairing_score(weight_meta, cand_meta)+ functional_match, mismatch = mmproj_functional_match(str(p), str(c))+ if functional_match is False:+ logger.info(f"detect_mmproj_file: dropped {c.name} ({mismatch})")+ continue+ if functional_match is True:+ meta_score = max(meta_score, 50)if meta_score == -1:- logger.info(f"detect_mmproj_file: dropped {c.name} (metadata mismatch)")+ differences = "; ".join(+ f"{key} {weight_meta.get(key)!r} != {cand_meta.get(key)!r}"+ for key in (+ "general.base_model.0.repo_url",+ "general.basename",+ "general.base_model.0.organization",+ "general.organization",+ )+ if weight_meta.get(key) != cand_meta.get(key)+ )+ logger.info(f"detect_mmproj_file: dropped {c.name} (metadata mismatch: {differences})")continueif meta_score == 0 and model_family is not None:# Unrecognised candidate family is a wildcard (``mmproj-F16.gguf``).diff --git a/studio/frontend/src/lib/model-memory.ts b/studio/frontend/src/lib/model-memory.tsindex 301141179d8..22e2c0ba582 100644--- a/studio/frontend/src/lib/model-memory.ts+++ b/studio/frontend/src/lib/model-memory.ts@@ -387,6 +387,7 @@ export const RESIDENT_ADDING_ARGS = ["--control-vector","--control-vector-scaled","--mmproj",+ "-mm","--model-draft","-md","--spec-draft-hf",diff --git a/studio/frontend/tests/model-memory-arg-alias-parity.test.ts b/studio/frontend/tests/model-memory-arg-alias-parity.test.tsindex 6a06dad71ff..14b460582e3 100644--- a/studio/frontend/tests/model-memory-arg-alias-parity.test.ts+++ b/studio/frontend/tests/model-memory-arg-alias-parity.test.ts@@ -159,3 +159,12 @@ test("underscore spellings are classified like their dashed twins", () => {assert.equal(extraArgsOwnPlacement(["not_a_flag"]), false);assert.equal(extraArgsOwnPlacement(["--temp", "0.7"]), false);});++test("both projector spellings add resident files", () => {+ for (const flag of ["--mmproj", "-mm"]) {+ assert.ok(+ extraArgsAddResidentFiles([flag, "/models/p.gguf"]),+ `${flag} was not recognised`,+ );+ }+});diff --git a/tests/studio/test_llama_cpp_mmproj_batch.py b/tests/studio/test_llama_cpp_mmproj_batch.pyindex 3e82f7f3319..575dfb5987f 100644--- a/tests/studio/test_llama_cpp_mmproj_batch.py+++ b/tests/studio/test_llama_cpp_mmproj_batch.py@@ -113,13 +113,15 @@ class TestLaunchNeedsBiggerUbatch:path = projector("gemma4uv")assert _launch_required_ubatch(path, 3840, ["--no-mmproj"], env = {}) == 0- def test_a_pass_through_projector_survives_both(self, projector):- # Appended after the managed flags and stripped by neither the switch nor- # --no-mmproj, so this opens an image tower regardless.+ def test_a_pass_through_projector_obeys_both(self, projector):+ # The launch emits it as the managed projector, so the switch and --no-mmproj drop it.path = projector("gemma4uv")- for extra_kwargs in ({}, {"vision_off": True}):- got = _launch_required_ubatch(None, 3840, ["--mmproj", path], env = {}, **extra_kwargs)- assert got == _GEMMA4+ assert _launch_required_ubatch(None, 3840, ["--mmproj", path], env = {}) == _GEMMA4+ assert _launch_required_ubatch(None, 3840, ["-mm", path], env = {}) == _GEMMA4+ got = _launch_required_ubatch(None, 3840, ["--mmproj", path], env = {}, vision_off = True)+ assert got == 0+ got = _launch_required_ubatch(None, 3840, ["--mmproj", path, "--no-mmproj"], env = {})+ assert got == 0def test_a_pass_through_projector_is_still_classified(self, projector):path = projector("qwen3vl_merger")