Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Unsloth Studio / Desktop: support custom vision projectors (#11352)

Maheswar Kumar6c1cc50e84bc
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py
index b371f25f543..c2c021edadc 100644
--- a/studio/backend/core/inference/llama_cpp.py
+++ b/studio/backend/core/inference/llama_cpp.py
@@ -6541,7 +6541,7 @@ def _launch_required_ubatch(
# Pass-through arguments override managed projector flags.
override = _extra_args_device(extra_args, {"--mmproj", "-mm"})
- if override:
+ if override and not vision_off and not extra_args_disable_mmproj(extra_args):
required = max(
required, _mmproj_required_ubatch(str(override), n_embd_text, extra_args, env)
)
@@ -19884,12 +19884,18 @@ class LlamaCppBackend:
return False
def _resolve_launch_mmproj_path(
- self, *, model_path: str, mmproj_path: Optional[str]
+ self,
+ *,
+ model_path: str,
+ mmproj_path: Optional[str],
+ extra_args: Optional[Iterable[str]] = None,
) -> Optional[str]:
- """Return mmproj_path iff it exists on disk AND matches the model family.
-
- None if mmproj_path is None, missing, or family-mismatched.
- """
+ """Resolve a projector, trusting an explicit path over filename heuristics."""
+ override = _extra_args_device(extra_args, {"--mmproj", "-mm"})
+ if override is not None:
+ if not override or not Path(override).is_file():
+ raise ValueError("The custom mmproj path must name an existing file.")
+ return override
if not mmproj_path:
return None
@@ -22490,17 +22496,15 @@ class LlamaCppBackend:
@staticmethod
def _strip_mmproj_args(cmd: list[str]) -> list[str]:
- """Return cmd without the '--mmproj <path>' pair (text-only retry).
- Every other flag is preserved; a no-op when --mmproj is absent.
- """
+ """Remove explicit projector paths, including aliases, for managed emission or retry."""
out: list[str] = []
skip_value = False
for tok in cmd:
if skip_value:
skip_value = False
continue
- if tok == "--mmproj":
- skip_value = True
+ if _flag_name(tok) in {"--mmproj", "-mm"}:
+ skip_value = "=" not in tok
continue
out.append(tok)
return out
@@ -23162,6 +23166,12 @@ class LlamaCppBackend:
ctx_checkpoints = intent.ctx_checkpoints
cache_ram = intent.cache_ram
extra_args = list(intent.extra_args) if intent.extra_args is not None else None
+ custom_mmproj = _extra_args_device(extra_args, {"--mmproj", "-mm"})
+ if custom_mmproj is not None:
+ mmproj_path = self._resolve_launch_mmproj_path(
+ model_path = gguf_path or "", mmproj_path = mmproj_path, extra_args = extra_args
+ )
+ is_vision = True
preserve_multi_gpu_on_layer = intent.preserve_multi_gpu_on_layer
reasoning_budget = intent.reasoning_budget
reasoning_budget_message = intent.reasoning_budget_message
@@ -23882,6 +23892,7 @@ class LlamaCppBackend:
else self._resolve_launch_mmproj_path(
model_path = model_path,
mmproj_path = mmproj_path,
+ extra_args = extra_args,
),
# Use the same order-independent metadata read as the estimators.
_read_gguf_embedding_length(model_path),
@@ -24250,6 +24261,7 @@ class LlamaCppBackend:
launch_mmproj_path = self._resolve_launch_mmproj_path(
model_path = model_path,
mmproj_path = mmproj_path,
+ extra_args = extra_args,
)
# The switch turns VISION off, and a projector is not always a
# vision tower: ultravox, Voxtral and Qwen3-ASR declare an audio
@@ -28506,7 +28518,7 @@ class LlamaCppBackend:
# User pass-through args go last. Placement flags are removed
# below when the Unsloth picker owns the GPU selection.
if _mem_extras:
- _emit_extra_args = list(_mem_extras)
+ _emit_extra_args = self._strip_mmproj_args(list(_mem_extras))
if _gpu_ids_own_device_flags:
# gpu_ids owns placement, so remove competing device flags.
_before_device_strip = list(_emit_extra_args)
@@ -32105,15 +32117,22 @@ class LlamaCppBackend:
intent.model_identifier.lower()
):
return False
- if intent.gguf_path is not None and self._gguf_path:
+ custom_mmproj = _extra_args_device(intent.extra_args, {"--mmproj", "-mm"})
+ source_path = intent.gguf_path
+ if source_path is None and custom_mmproj is not None:
+ if (self._hf_variant or "").lower() != (intent.hf_variant or "").lower():
+ return False
+ source_path = self._gguf_path
+ if source_path is not None and self._gguf_path:
resident_identity = getattr(self, "_gguf_load_identity", None)
if resident_identity is not None:
candidate_identity = LlamaCppBackend._gguf_load_source_identity(
- intent.gguf_path, intent.mmproj_path
+ source_path,
+ custom_mmproj or intent.mmproj_path,
)
return candidate_identity == resident_identity
try:
- return Path(self._gguf_path).resolve() == Path(intent.gguf_path).resolve()
+ return Path(self._gguf_path).resolve() == Path(source_path).resolve()
except OSError:
return False
return (self._hf_variant or "").lower() == (intent.hf_variant or "").lower()
@@ -32137,7 +32156,7 @@ class LlamaCppBackend:
the main ``--gpu-layers``. A drafter explicitly forced to CPU
(--spec-draft-ngl 0 / --spec-draft-device cpu) doesn't count."""
_args = [str(a) for a in cmd]
- if any(a.startswith("--mmproj") for a in _args):
+ if any(a.startswith("--mmproj") or _flag_name(a) == "-mm" for a in _args):
# --no-mmproj-offload clears mmproj_use_gpu and clip.cpp gates the whole
# GPU backend on it, so the projector holds no VRAM. Last flag wins.
_off = [a for a in _args if a in ("--mmproj-offload", "--no-mmproj-offload")]
@@ -33853,6 +33872,8 @@ class LlamaCppBackend:
return None
_SIDECAR_WEIGHT_FLAGS = (
+ "--mmproj",
+ "-mm",
"--lora",
"--lora-scaled",
"--control-vector",
diff --git a/studio/backend/core/inference/llama_server_args.py b/studio/backend/core/inference/llama_server_args.py
index cd11322285f..716a9816f33 100644
--- a/studio/backend/core/inference/llama_server_args.py
+++ b/studio/backend/core/inference/llama_server_args.py
@@ -82,7 +82,6 @@ _DENYLIST_GROUPS: tuple[frozenset[str], ...] = (
frozenset({"-hfv", "-hfrv", "--hf-repo-v"}),
frozenset({"-hffv", "--hf-file-v"}),
frozenset({"-hft", "--hf-token"}),
- frozenset({"-mm", "--mmproj"}),
frozenset({"-mmu", "--mmproj-url"}),
# Networking: Unsloth binds + proxies; retargeting orphans the proxy.
frozenset({"--host"}),
@@ -1815,13 +1814,7 @@ DENIED_ENV_VARS: tuple[str, ...] = (
"LLAMA_ARG_UI_CONFIG_FILE",
"LLAMA_ARG_UI_MCP_PROXY",
"LLAMA_ARG_STATIC_PATH",
- # Deliberately absent: LLAMA_ARG_MMPROJ and LLAMA_ARG_MMPROJ_URL. --mmproj is refused in the box because Unsloth
- # resolves the projector itself, but the environment twin is an INPUT here: _launch_has_mmproj reads both to know
- # the launch has a projector at all, which is what keeps the vision and audio state of a model loaded through an
- # inherited one. Only the paravirtual CPU recovery drops them, where an unpinned projector is the corrupt path it
- # is undoing. The pooling twins are absent for the opposite reason: load_model already pops LLAMA_ARG_POOLING /
- # _RERANKING / _EMBEDDINGS itself. The multi-model server mode is absent too: a child holding its own model
- # directory, preset and autoload policy is not the single model Unsloth launched and accounts for.
+ # LLAMA_ARG_MMPROJ / _URL stay allowed: _launch_has_mmproj reads them as launch inputs.
"LLAMA_ARG_MODELS_DIR",
"LLAMA_ARG_MODELS_PRESET",
"LLAMA_ARG_MODELS_MAX",
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index b50d0f0954e..09dd5e28646 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -7518,6 +7518,19 @@ def _external_transcript_preview(response: Response) -> str:
return ""
+def _refuse_managed_custom_projector(extra_args: Optional[list[str]]) -> None:
+ """A pass-through projector path skips account model access, so only the owner may name one."""
+ from core.inference.llama_cpp import _extra_args_device
+ if (
+ account_access.managed_account()
+ and _extra_args_device(extra_args, {"--mmproj", "-mm"}) is not None
+ ):
+ raise HTTPException(
+ status_code = 403,
+ detail = "A custom --mmproj path is available to this installation's owner only.",
+ )
+
+
def _validate_native_gguf_companion(
companion_path: str | None,
gguf_path: str | None,
@@ -11642,12 +11655,14 @@ def _remote_required_ubatch(
"""Return a conservative micro-batch for an undownloaded GGUF config."""
from core.inference.llama_cpp import _launch_required_ubatch, extra_args_disable_mmproj
- from core.inference.llama_cpp import _unknown_projector_ubatch
+ from core.inference.llama_cpp import _extra_args_device, _unknown_projector_ubatch
+ # A local --mmproj replaces the repo projector, so size from that file instead.
if (
bool(getattr(config, "is_vision", False))
and not disable_vision
and not extra_args_disable_mmproj(llama_extra_args)
+ and _extra_args_device(llama_extra_args, {"--mmproj", "-mm"}) is None
):
# Match the worst-case post-download allocation.
return _unknown_projector_ubatch(llama_extra_args)
@@ -12341,6 +12356,7 @@ def _estimate_gguf_required_gb(
# projector despite the switch, and dropping bytes that do get opened would
# admit a load the running training job cannot afford, so ask the loader's own
# question of the file. Same gate as the remote branch's include_mmproj.
+ _mmproj_override = _extra_args_device(llama_extra_args, {"--mmproj", "-mm"})
_sized_attrs = ["gguf_mmproj_file"]
# Whether the CONFIGURED projector is one this launch opens. Bound before the
# switch so the inherited-projector gate below can read it either way.
@@ -12352,7 +12368,7 @@ def _estimate_gguf_required_gb(
_dv_opens_projector = False
_sized_attrs = []
elif disable_vision:
- _dv_mmproj = getattr(config, "gguf_mmproj_file", None)
+ _dv_mmproj = _mmproj_override or getattr(config, "gguf_mmproj_file", None)
_dv_opens_projector = False
if _dv_mmproj:
try:
@@ -12372,11 +12388,9 @@ def _estimate_gguf_required_gb(
# possibly much larger custom projector went free.
_mmproj_override_bytes = 0
if _sized_attrs == ["gguf_mmproj_file"]:
- _mmproj_override = _extra_args_device(llama_extra_args, {"--mmproj", "-mm"})
if _mmproj_override and Path(_mmproj_override).is_file():
_sized_attrs = []
_mmproj_override_bytes = LlamaCppBackend._get_gguf_size_bytes(_mmproj_override)
- total_bytes += _mmproj_override_bytes
_sized_keys.add(_same_file_key(_mmproj_override))
if not _charge_no_drafter:
if dspark_requested:
@@ -12470,7 +12484,7 @@ def _estimate_gguf_required_gb(
# extras that skipped the resolve both leave it empty and let the inherited path
# load -- so this asks what Unsloth emits, not what the config names.
_studio_mmproj_on_argv = bool(
- getattr(config, "gguf_mmproj_file", None) and _dv_opens_projector
+ (_mmproj_override or getattr(config, "gguf_mmproj_file", None)) and _dv_opens_projector
)
_env_mmproj_bytes = 0
_env_mmproj = (os.environ.get("LLAMA_ARG_MMPROJ") or "").strip()
@@ -12484,7 +12498,7 @@ def _estimate_gguf_required_gb(
_env_mmproj_bytes = LlamaCppBackend._get_gguf_size_bytes(_env_mmproj)
if total_bytes > 0:
- return (total_bytes + _extras_bytes + _env_mmproj_bytes) / (
+ return (total_bytes + _mmproj_override_bytes + _extras_bytes + _env_mmproj_bytes) / (
1024**3
) + _estimate_gguf_kv_gb(
main,
@@ -12524,7 +12538,7 @@ def _estimate_gguf_required_gb(
# the file to ask. Under-charging is what would admit a chat load over
# VRAM a training job needs, so an unknown projector is charged. The
# local branch, holding the file, asks instead.
- include_mmproj = bool(has_vision),
+ include_mmproj = bool(has_vision) and _mmproj_override is None,
# Remote, so which sidecar the repo ships is unknown until the
# listing. Under Auto size both: a repo has one kind or the other,
# the absent one contributes 0, and over-estimating is the safe
@@ -12547,7 +12561,9 @@ def _estimate_gguf_required_gb(
# Plus the caller's own --model-draft / --spec-draft-hf, if they named
# one: this repo's listing cannot see it, local or remote, and it is
# resident next to these weights.
- total_gb = (main_bytes + companions + _extras_bytes) / (1024**3)
+ total_gb = (main_bytes + companions + _mmproj_override_bytes + _extras_bytes) / (
+ 1024**3
+ )
total_gb += _remote_gguf_compute_reserve_gb(
llama_extra_args = llama_extra_args,
max_seq_length = max_seq_length,
@@ -16788,6 +16804,7 @@ async def _load_model_impl(
extra_llama_args: Optional[list[str]] = (
None if request.llama_extra_args is None else extra_llama_args
)
+ _refuse_managed_custom_projector(extra_llama_args)
_reasoning_updates = {}
_reasoning_budget_override = parse_reasoning_budget_override(extra_llama_args)
@@ -18215,6 +18232,8 @@ async def validate_model(
validate_extra_args(effective_extra_args)
except ValueError as exc:
raise HTTPException(status_code = 400, detail = str(exc)) from exc
+ if getattr(request, "llama_extra_args", None) is not None:
+ _refuse_managed_custom_projector(effective_extra_args)
# Manual mode owns the offload flags, and /load translates an explicit -ngl
# into the first-class field before it strips them. Doing that there and not
diff --git a/studio/backend/tests/test_detect_mmproj_file.py b/studio/backend/tests/test_detect_mmproj_file.py
index 156ae347b03..2b84ac16c9e 100644
--- a/studio/backend/tests/test_detect_mmproj_file.py
+++ b/studio/backend/tests/test_detect_mmproj_file.py
@@ -9,6 +9,8 @@ from pathlib import Path
import struct
+import pytest
+
from utils.models.model_config import (
_detect_family_token,
detect_mmproj_file,
@@ -20,14 +22,17 @@ _GGUF_MAGIC = 0x46554747
def _gguf_with_general(path: Path, fields: dict) -> Path:
- """Write a minimal GGUF with only ``general.*`` string KVs."""
+ """Write a minimal GGUF with string and unsigned integer KVs."""
body = b""
for k, v in fields.items():
kb = k.encode("utf-8")
- vb = v.encode("utf-8")
body += struct.pack("<Q", len(kb)) + kb
- body += struct.pack("<I", 8) # STRING vtype
- body += struct.pack("<Q", len(vb)) + vb
+ if isinstance(v, int):
+ body += struct.pack("<II", 4, v)
+ else:
+ vb = v.encode("utf-8")
+ body += struct.pack("<I", 8)
+ body += struct.pack("<Q", len(vb)) + vb
header = struct.pack("<IIQQ", _GGUF_MAGIC, 3, 0, len(fields))
path.parent.mkdir(parents = True, exist_ok = True)
path.write_bytes(header + body)
@@ -447,3 +452,49 @@ def test_a_draft_model_under_assets_is_not_mistaken_for_a_projector(tmp_path: Pa
model = _touch(tmp_path / "Qwen3.8-27B-UD-Q4_K_M.gguf")
_touch(tmp_path / "assets" / "Qwen3.8-0.8B-draft-Q4_K_M.gguf")
assert detect_mmproj_file(str(model)) is None
+
+
[email protected]("projection_dim", [5376, 2560, 0])
[email protected]("identity_matches", [False, True])
+def test_gemma4_finetune_pairs_by_functional_metadata(tmp_path, projection_dim, identity_matches):
+ model = _gguf_with_general(
+ tmp_path / "gembrain.gguf",
+ {
+ "general.architecture": "gemma4",
+ "general.basename": "Gemma-4" if identity_matches else "Gemma-4-Gembrain",
+ "general.base_model.0.repo_url": (
+ "https://huggingface.co/google/gemma-4-31B-it"
+ if identity_matches
+ else "https://huggingface.co/community/Gembrain"
+ ),
+ "gemma4.embedding_length": 5376,
+ },
+ )
+ projector = _gguf_with_general(
+ tmp_path / "mmproj-gemma-4-31B-it-BF16.gguf",
+ {
+ "general.architecture": "clip",
+ "general.type": "mmproj",
+ "general.basename": "Gemma-4",
+ "general.base_model.0.repo_url": "https://huggingface.co/google/gemma-4-31B-it",
+ "clip.vision.projector_type": "gemma4v",
+ "clip.vision.projection_dim": projection_dim,
+ },
+ )
+ expected = (
+ str(projector)
+ if projection_dim == 5376 or (projection_dim == 0 and identity_matches)
+ else None
+ )
+ assert detect_mmproj_file(str(model)) == expected
+ if projection_dim:
+ assert mmproj_matches_model_family(str(model), str(projector)) == (projection_dim == 5376)
+
+
+def test_mmproj_rejection_names_mismatched_fields(tmp_path, capsys):
+ model = _gguf_with_general(tmp_path / "model.gguf", {"general.basename": "Foo"})
+ _gguf_with_general(tmp_path / "mmproj.gguf", {"general.basename": "Bar"})
+ assert detect_mmproj_file(str(model)) is None
+ output = capsys.readouterr().out
+ assert "general.basename" in output
+ assert "Foo" in output and "Bar" in output
diff --git a/studio/backend/tests/test_gpu_memory_mode.py b/studio/backend/tests/test_gpu_memory_mode.py
index 7a8a0090b2b..940abc3b0ff 100644
--- a/studio/backend/tests/test_gpu_memory_mode.py
+++ b/studio/backend/tests/test_gpu_memory_mode.py
@@ -1280,7 +1280,7 @@ def test_zero_offload_flag_false_without_companions():
@pytest.mark.parametrize(
"companion",
- ["--mmproj", "--model-draft", "-md", "--spec-draft-model", "-hfd"],
+ ["--mmproj", "-mm", "--model-draft", "-md", "--spec-draft-model", "-hfd"],
)
def test_zero_offload_flag_true_with_companion(companion):
# mmproj / a drafter offload to GPU regardless of --gpu-layers, so the
diff --git a/studio/backend/tests/test_llama_server_args.py b/studio/backend/tests/test_llama_server_args.py
index 0fb10c51021..4985fa95ebf 100644
--- a/studio/backend/tests/test_llama_server_args.py
+++ b/studio/backend/tests/test_llama_server_args.py
@@ -184,8 +184,6 @@ def test_a_bare_positional_is_rejected():
"--hf-file-v",
"-hft",
"--hf-token",
- "-mm",
- "--mmproj",
"-mmu",
"--mmproj-url",
# Networking (Unsloth binds + proxies)
@@ -1675,3 +1673,9 @@ def test_a_pageable_launch_keeps_every_token_including_its_lock():
out, overridden = _lsa.force_pageable_load(list(argv), {})
assert out == argv and overridden == []
+
+
[email protected]("flag", ["--mmproj", "-mm"])
+def test_custom_projector_is_allowed(flag):
+ args = [flag, "/models/custom projector.gguf"]
+ assert validate_extra_args(args) == args
diff --git a/studio/backend/tests/test_mmproj_placement_policy.py b/studio/backend/tests/test_mmproj_placement_policy.py
index c9d77e68796..4326a293fb9 100644
--- a/studio/backend/tests/test_mmproj_placement_policy.py
+++ b/studio/backend/tests/test_mmproj_placement_policy.py
@@ -16,6 +16,7 @@ import inspect
import os
import struct
import subprocess
+from dataclasses import replace
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import patch
@@ -1680,3 +1681,155 @@ def test_a_replayed_context_refits_for_a_cpu_pinned_drafter_too(tmp_path):
assert replay_ctx == fresh_ctx < replayed
assert ("--no-mmproj-offload" in replay) == ("--no-mmproj-offload" in fresh)
+
+
[email protected]("flag", ["--mmproj", "-mm"])
[email protected]("vision_off", [False, True])
+def test_custom_projector_replaces_discovery_and_respects_vision_switch(tmp_path, flag, vision_off):
+ backend, gguf = _backend(tmp_path, memory = [(0, 12_000, 24_000)])
+ backend._resolve_launch_mmproj_path = LlamaCppBackend._resolve_launch_mmproj_path.__get__(
+ backend
+ )
+ gguf = gguf.rename(tmp_path / "Qwen-model.gguf")
+ custom = _write_gguf(tmp_path / "gemma-custom projector.gguf")
+ seen = []
+ backend._mmproj_vram_bytes = lambda path: seen.append(path) or GIB
+ cmd = _launch(
+ backend,
+ gguf,
+ is_vision = False,
+ disable_vision = vision_off,
+ extra_args = [flag, str(custom)],
+ )["cmd"]
+ if vision_off:
+ assert "--mmproj" not in cmd and "-mm" not in cmd
+ else:
+ assert cmd.count("--mmproj") == 1
+ assert cmd[cmd.index("--mmproj") + 1] == str(custom)
+ assert str(custom) in seen
+
+
+def test_vision_off_still_charges_a_custom_audio_only_projector(tmp_path, monkeypatch):
+ model = _write_gguf(tmp_path / "model.gguf")
+ custom = _write_gguf(tmp_path / "custom-audio.gguf")
+ monkeypatch.setattr(_meta, "mmproj_accepts_image", lambda _path: False)
+ config = SimpleNamespace(
+ gguf_file = str(model),
+ gguf_mmproj_file = None,
+ gguf_mtp_file = None,
+ gguf_dspark_file = None,
+ gguf_dflash_file = None,
+ gguf_hf_repo = None,
+ gguf_variant = None,
+ is_vision = False,
+ )
+ enabled = _estimate_gguf_required_gb(config, llama_extra_args = ["--mmproj", str(custom)])
+ disabled = _estimate_gguf_required_gb(
+ config,
+ llama_extra_args = ["--mmproj", str(custom)],
+ disable_vision = True,
+ )
+ assert enabled == disabled
+
+
[email protected]("vision_off", [False, True])
+def test_custom_projector_does_not_hide_remote_model_bytes(tmp_path, monkeypatch, vision_off):
+ import routes.inference as routes
+ import utils.models.model_config as model_config
+
+ custom = _write_gguf(tmp_path / "custom-vision.gguf")
+ config = SimpleNamespace(
+ gguf_file = None,
+ gguf_mmproj_file = None,
+ gguf_mtp_file = None,
+ gguf_dspark_file = None,
+ gguf_dflash_file = None,
+ gguf_hf_repo = "org/vision-GGUF",
+ gguf_variant = "Q4_K_M",
+ is_vision = True,
+ )
+ monkeypatch.setattr(
+ model_config,
+ "list_gguf_variants",
+ lambda *_a, **_kw: (
+ [SimpleNamespace(quant = "Q4_K_M", size_bytes = 4 * GIB)],
+ True,
+ ),
+ )
+ seen = []
+ monkeypatch.setattr(
+ routes, "_remote_gguf_companion_bytes", lambda *_a, **kw: seen.append(kw) or 0
+ )
+ monkeypatch.setattr(routes, "_remote_gguf_compute_reserve_gb", lambda **_kw: 0)
+ result = _estimate_gguf_required_gb(
+ config,
+ llama_extra_args = ["--mmproj", str(custom)],
+ disable_vision = vision_off,
+ speculative_type = "off",
+ )
+ expected = 4 * GIB + (0 if vision_off else custom.stat().st_size)
+ assert result * GIB == expected
+ assert seen[0]["include_mmproj"] is False
+
+
+def test_missing_custom_projector_is_rejected_before_unloading(tmp_path, monkeypatch):
+ backend = LlamaCppBackend()
+ unloaded = []
+ monkeypatch.setattr(backend, "unload_model", lambda: unloaded.append(True))
+ with pytest.raises(ValueError, match = "custom mmproj path"):
+ backend.load_model(
+ GgufLoadIntent(
+ model_identifier = "test",
+ gguf_path = str(tmp_path / "model.gguf"),
+ extra_args = ("--mmproj", str(tmp_path / "missing.gguf")),
+ )
+ )
+ assert unloaded == []
+
+
[email protected]("flag", ["--mmproj", "-mm"])
+def test_hub_load_rechecks_custom_projector_replaced_in_place(tmp_path, flag):
+ backend, gguf = _backend(tmp_path, memory = [(0, 12_000, 24_000)])
+ backend._resolve_launch_mmproj_path = LlamaCppBackend._resolve_launch_mmproj_path.__get__(
+ backend
+ )
+ custom = _write_gguf(tmp_path / "custom-projector.gguf")
+ _launch(backend, gguf, is_vision = False, extra_args = [flag, str(custom)])
+ backend._hf_variant = "Q4_K_M"
+ intent = GgufLoadIntent(
+ model_identifier = "test",
+ hf_repo = "org/model",
+ hf_variant = "Q4_K_M",
+ extra_args = (flag, str(custom)),
+ )
+ assert backend.matches_load_source(intent)
+ assert not backend.matches_load_source(replace(intent, hf_variant = "Q8_0"))
+ replacement = _write_gguf(tmp_path / "replacement.gguf")
+ replacement.replace(custom)
+ assert not backend.matches_load_source(intent)
+
+
+def test_remote_ubatch_sizes_the_custom_projector_not_the_repo_one(tmp_path, monkeypatch):
+ import routes.inference as routes
+
+ custom = _write_gguf(tmp_path / "custom-audio.gguf")
+ monkeypatch.setattr(_meta, "mmproj_accepts_image", lambda _path: False)
+ config = SimpleNamespace(is_vision = True)
+ assert routes._remote_required_ubatch(config, []) > 0
+ assert routes._remote_required_ubatch(config, ["--mmproj", str(custom)]) == 0
+
+
[email protected]("flag", ["--mmproj", "-mm"])
[email protected]("managed", [False, True])
+def test_only_the_owner_may_name_a_custom_projector(monkeypatch, flag, managed):
+ import routes.inference as routes
+ from fastapi import HTTPException
+
+ monkeypatch.setattr(routes.account_access, "managed_account", lambda: managed)
+ routes._refuse_managed_custom_projector(["--ctx-size", "4096"])
+ if not managed:
+ routes._refuse_managed_custom_projector([flag, "/models/p.gguf"])
+ return
+ with pytest.raises(HTTPException) as err:
+ routes._refuse_managed_custom_projector([flag, "/models/p.gguf"])
+ assert err.value.status_code == 403
diff --git a/studio/backend/tests/test_tp_vision_regression.py b/studio/backend/tests/test_tp_vision_regression.py
index 3ff2fd9ecac..b736866f810 100644
--- a/studio/backend/tests/test_tp_vision_regression.py
+++ b/studio/backend/tests/test_tp_vision_regression.py
@@ -385,9 +385,10 @@ def test_tensor_split_abort_raises_early_to_layer_fallback():
# The strip is matched on the call, not on what is passed to it. What this
# test is about is that the abort raises BEFORE the projector is thrown
# away; which command the strip reads from is that code's own business.
+ # `cmd = ` skips the extra-args strip, whose --mmproj the managed flag re-emits.
for label, needle in (
("the flash-attn-off retry", "_with_flash_attn_off"),
- ("the text-only mmproj strip", "_strip_mmproj_args("),
+ ("the text-only mmproj strip", "cmd = self._strip_mmproj_args("),
):
idx = src.find(needle)
assert idx != -1, (
diff --git a/studio/backend/utils/models/gguf_metadata.py b/studio/backend/utils/models/gguf_metadata.py
index 759bcaa9f35..70778969377 100644
--- a/studio/backend/utils/models/gguf_metadata.py
+++ b/studio/backend/utils/models/gguf_metadata.py
@@ -1009,6 +1009,26 @@ def _weight_url_looks_like_derivative_of_projector(weight_url: str, projector_ur
return _slug_extends_base(weight_slug, projector_slug)
+def mmproj_functional_match(weight_path: str, mmproj_path: str) -> tuple[Optional[bool], str]:
+ """Gemma 4 only: pair by projector type + projection dim, not branding. None = unknown."""
+ if read_gguf_architecture(weight_path) != "gemma4":
+ return None, ""
+ projector_type = read_mmproj_vision_projector_type(mmproj_path)
+ if projector_type != "gemma4v":
+ return None, ""
+ embedding_length = read_gguf_embedding_length(weight_path)
+ dims = _parse_gguf_arch_uints(mmproj_path, frozenset({"vision.projection_dim"})) or {}
+ projection_dim = dims.get("vision.projection_dim")
+ if not embedding_length or not projection_dim:
+ return None, ""
+ if embedding_length != projection_dim:
+ return False, (
+ f"gemma4.embedding_length {embedding_length} != "
+ f"clip.vision.projection_dim {projection_dim}"
+ )
+ return True, ""
+
+
def pairing_score(
weight_meta: Optional[Dict[str, str]], mmproj_meta: Optional[Dict[str, str]]
) -> int:
diff --git a/studio/backend/utils/models/model_config.py b/studio/backend/utils/models/model_config.py
index d90f4ceb32f..33747fe8ca7 100644
--- a/studio/backend/utils/models/model_config.py
+++ b/studio/backend/utils/models/model_config.py
@@ -48,6 +48,7 @@ from utils.audio_tokens import (
from utils.models.gguf_metadata import (
is_mmproj_by_metadata,
mmproj_accepts_image,
+ mmproj_functional_match,
pairing_score,
read_gguf_general_metadata,
read_gguf_nextn_predict_layers,
@@ -1934,8 +1935,10 @@ def _detect_family_token(filename: str) -> Optional[str]:
def mmproj_matches_model_family(model_path: str, mmproj_path: str) -> bool:
- """Launcher guard: True unless both filenames carry recognised family
- tokens that disagree."""
+ """Prefer known functional metadata, falling back to filename family hints."""
+ functional_match, _ = mmproj_functional_match(model_path, mmproj_path)
+ if functional_match is not None:
+ return functional_match
model_fam = _detect_family_token(Path(model_path).name)
mmproj_fam = _detect_family_token(Path(mmproj_path).name)
if model_fam is None or mmproj_fam is None:
@@ -2193,8 +2196,24 @@ def detect_mmproj_file(
for c in candidates:
cand_meta = read_gguf_general_metadata(str(c))
meta_score = pairing_score(weight_meta, cand_meta)
+ functional_match, mismatch = mmproj_functional_match(str(p), str(c))
+ if functional_match is False:
+ logger.info(f"detect_mmproj_file: dropped {c.name} ({mismatch})")
+ continue
+ if functional_match is True:
+ meta_score = max(meta_score, 50)
if meta_score == -1:
- logger.info(f"detect_mmproj_file: dropped {c.name} (metadata mismatch)")
+ differences = "; ".join(
+ f"{key} {weight_meta.get(key)!r} != {cand_meta.get(key)!r}"
+ for key in (
+ "general.base_model.0.repo_url",
+ "general.basename",
+ "general.base_model.0.organization",
+ "general.organization",
+ )
+ if weight_meta.get(key) != cand_meta.get(key)
+ )
+ logger.info(f"detect_mmproj_file: dropped {c.name} (metadata mismatch: {differences})")
continue
if meta_score == 0 and model_family is not None:
# Unrecognised candidate family is a wildcard (``mmproj-F16.gguf``).
diff --git a/studio/frontend/src/lib/model-memory.ts b/studio/frontend/src/lib/model-memory.ts
index 301141179d8..22e2c0ba582 100644
--- a/studio/frontend/src/lib/model-memory.ts
+++ b/studio/frontend/src/lib/model-memory.ts
@@ -387,6 +387,7 @@ export const RESIDENT_ADDING_ARGS = [
"--control-vector",
"--control-vector-scaled",
"--mmproj",
+ "-mm",
"--model-draft",
"-md",
"--spec-draft-hf",
diff --git a/studio/frontend/tests/model-memory-arg-alias-parity.test.ts b/studio/frontend/tests/model-memory-arg-alias-parity.test.ts
index 6a06dad71ff..14b460582e3 100644
--- a/studio/frontend/tests/model-memory-arg-alias-parity.test.ts
+++ b/studio/frontend/tests/model-memory-arg-alias-parity.test.ts
@@ -159,3 +159,12 @@ test("underscore spellings are classified like their dashed twins", () => {
assert.equal(extraArgsOwnPlacement(["not_a_flag"]), false);
assert.equal(extraArgsOwnPlacement(["--temp", "0.7"]), false);
});
+
+test("both projector spellings add resident files", () => {
+ for (const flag of ["--mmproj", "-mm"]) {
+ assert.ok(
+ extraArgsAddResidentFiles([flag, "/models/p.gguf"]),
+ `${flag} was not recognised`,
+ );
+ }
+});
diff --git a/tests/studio/test_llama_cpp_mmproj_batch.py b/tests/studio/test_llama_cpp_mmproj_batch.py
index 3e82f7f3319..575dfb5987f 100644
--- a/tests/studio/test_llama_cpp_mmproj_batch.py
+++ b/tests/studio/test_llama_cpp_mmproj_batch.py
@@ -113,13 +113,15 @@ class TestLaunchNeedsBiggerUbatch:
path = projector("gemma4uv")
assert _launch_required_ubatch(path, 3840, ["--no-mmproj"], env = {}) == 0
- def test_a_pass_through_projector_survives_both(self, projector):
- # Appended after the managed flags and stripped by neither the switch nor
- # --no-mmproj, so this opens an image tower regardless.
+ def test_a_pass_through_projector_obeys_both(self, projector):
+ # The launch emits it as the managed projector, so the switch and --no-mmproj drop it.
path = projector("gemma4uv")
- for extra_kwargs in ({}, {"vision_off": True}):
- got = _launch_required_ubatch(None, 3840, ["--mmproj", path], env = {}, **extra_kwargs)
- assert got == _GEMMA4
+ assert _launch_required_ubatch(None, 3840, ["--mmproj", path], env = {}) == _GEMMA4
+ assert _launch_required_ubatch(None, 3840, ["-mm", path], env = {}) == _GEMMA4
+ got = _launch_required_ubatch(None, 3840, ["--mmproj", path], env = {}, vision_off = True)
+ assert got == 0
+ got = _launch_required_ubatch(None, 3840, ["--mmproj", path, "--no-mmproj"], env = {})
+ assert got == 0
def test_a_pass_through_projector_is_still_classified(self, projector):
path = projector("qwen3vl_merger")