Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Fix GGUF vision capability selection (#11696)
Wasim Yousef Said6bcb8af48db6
diff --git a/studio/frontend/src/features/chat/chat-page.tsx b/studio/frontend/src/features/chat/chat-page.tsxindex 5ffe17fcb6b..baa2df7abed 100644--- a/studio/frontend/src/features/chat/chat-page.tsx+++ b/studio/frontend/src/features/chat/chat-page.tsx@@ -133,6 +133,7 @@ import {useChatArtifactsStore,useSelectedChatArtifact,} from "./artifacts/store";+import { isKnownTextOnlySelection } from "./utils/model-vision-capability";import type { ChatArtifact, ChatArtifactSurface } from "./artifacts/types";import { McpServersDialogMount } from "./mcp-composer-button";import { ChatSettingsPanel } from "./chat-settings-sheet";@@ -3372,7 +3373,11 @@ export function ChatPage({(model) => model.id === value,);showImageCompatibilityWarning =- hasImage && targetModel?.isVision === false;+ hasImage &&+ isKnownTextOnlySelection(+ { isVision: meta?.isVision, isGguf: meta?.isGguf },+ targetModel,+ );}}}@@ -3394,6 +3399,7 @@ export function ChatPage({expectedBytes: meta?.expectedBytes,downloadPresentation: meta?.downloadPresentation,isGguf: meta?.isGguf,+ isVision: meta?.isVision,isDiffusion: meta?.isDiffusion,config: meta?.config,nativePathToken: meta?.nativePathToken,diff --git a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.tsindex ca839ab2a5b..946ea49c840 100644--- a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts+++ b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts@@ -185,6 +185,8 @@ export type SelectedModelInput = {/** Direct local .gguf file (no HF variant / native token), still a GGUF source, so the staging* flow treats it as one. */isGguf?: boolean;+ /** Known model vision capability. Undefined means unknown, not text-only. */+ isVision?: boolean;/** Staged metadata confirmed the separate DiffusionGemma runner. */isDiffusion?: boolean;throwOnError?: boolean;diff --git a/studio/frontend/src/features/chat/utils/model-vision-capability.ts b/studio/frontend/src/features/chat/utils/model-vision-capability.tsnew file mode 100644index 00000000000..84cff70f632--- /dev/null+++ b/studio/frontend/src/features/chat/utils/model-vision-capability.ts@@ -0,0 +1,36 @@+// SPDX-License-Identifier: AGPL-3.0-only+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++export type ModelVisionCapability = {+ isVision?: boolean;+ isGguf?: boolean;+};++/**+ * Whether a selected model is authoritatively known to be text-only.+ *+ * A GGUF repo's catalog row can predate the variant metadata request and report+ * `isVision: false` even though the selected quant has an mmproj companion.+ * Prefer the variant-level hint when present; otherwise treat GGUF capability+ * as unknown rather than warning from stale catalog metadata.+ */+export function isKnownTextOnlySelection(+ selection: ModelVisionCapability,+ catalogModel?: ModelVisionCapability,+): boolean {+ if (selection.isVision !== undefined) {+ return selection.isVision === false;+ }+ if (selection.isGguf === true || catalogModel?.isGguf === true) {+ return false;+ }+ return catalogModel?.isVision === false;+}+++/** Preserve missing or malformed backend capability metadata as unknown. */+export function normalizeGgufVisionCapability(+ value: unknown,+): boolean | undefined {+ return typeof value === "boolean" ? value : undefined;+}diff --git a/studio/frontend/src/features/model-picker/components/model-selector/pickers.tsx b/studio/frontend/src/features/model-picker/components/model-selector/pickers.tsxindex ef899b7cbf4..391a1c13acb 100644--- a/studio/frontend/src/features/model-picker/components/model-selector/pickers.tsx+++ b/studio/frontend/src/features/model-picker/components/model-selector/pickers.tsx@@ -14,6 +14,7 @@ import {import { usePlatformStore } from "@/config/env";import { INVENTORY_FRESHNESS_WINDOW_MS } from "@/features/hub/inventory";import { ApiProviderLogo } from "@/features/chat";+import { normalizeGgufVisionCapability } from "@/features/chat/utils/model-vision-capability";import {type ScanFolderInfo,addScanFolder,@@ -1463,7 +1464,7 @@ function normalizeGgufVariantsResponse(): {variants: GgufVariantDetail[];defaultVariant: string | null;- hasVision: boolean;+ hasVision: boolean | undefined;contextLength: number | null;resolvedLocally: boolean;dependenciesResolved: boolean;@@ -1477,7 +1478,7 @@ function normalizeGgufVariantsResponse(typeof res?.default_variant === "string" && res.default_variant.length > 0? res.default_variant: null,- hasVision: res?.has_vision === true,+ hasVision: normalizeGgufVisionCapability(res?.has_vision),contextLength:typeof contextLength === "number" &&Number.isFinite(contextLength) &&@@ -1506,7 +1507,7 @@ function ggufVariantExpectedBytes(variant: GgufVariantDetail): number {* mounts the expander, so this is its only source. */interface SoleDownloadedQuant {variant: GgufVariantDetail;- hasVision: boolean;+ hasVision: boolean | undefined;}/** The repo's one complete quant, or null when Hub metadata cannot verify its dependencies. */@@ -1710,7 +1711,7 @@ function GgufVariantExpander({const deleteDisabled = variantActions?.deleteDisabled ?? false;const [variants, setVariants] = useState<GgufVariantDetail[] | null>(null);const [defaultVariant, setDefaultVariant] = useState<string | null>(null);- const [hasVision, setHasVision] = useState(false);+ const [hasVision, setHasVision] = useState<boolean | undefined>(undefined);// Native max context (GGUF metadata); only set once a variant is downloaded.const [nativeContext, setNativeContext] = useState<number | null>(null);const [loading, setLoading] = useState(true);@@ -1747,7 +1748,9 @@ function GgufVariantExpander({setVariants(normalized.variants);setDefaultVariant(normalized.defaultVariant);setHasVision(normalized.hasVision);- onHasVision?.(normalized.hasVision);+ if (normalized.hasVision !== undefined) {+ onHasVision?.(normalized.hasVision);+ }setNativeContext(normalized.contextLength);setResolvedLocally(normalized.resolvedLocally);})@@ -1769,6 +1772,10 @@ function GgufVariantExpander({const isLocalPath = /^(\/|\.{1,2}[\\/]|~[\\/]|[A-Za-z]:[\\/]|\\\\)/.test(repoId,);+ // Repo-level true only proves that some mmproj exists; the loader may reject+ // it for this quant/model family. Absence is authoritative text-only+ // evidence, while presence remains unknown until load.+ const variantVisionHint = hasVision === false ? false : undefined;// The prefix test cannot see a marker-less relative directory like "models/my-image-model",// so whether the checkpoint is on disk is asked of the listing, not of the spelling.const checkpointIsLocal = isLocalPath || resolvedLocally;@@ -1796,6 +1803,7 @@ function GgufVariantExpander({downloadPresentation,contextLength: isAvailable ? nativeContext : undefined,isGguf: true,+ isVision: variantVisionHint,pipelineTag,});},@@ -1807,6 +1815,7 @@ function GgufVariantExpander({sourceOverride,nativeContext,pipelineTag,+ variantVisionHint,],);@@ -2263,6 +2272,7 @@ function GgufVariantExpander({expectedBytes,contextLength: nativeContext,isGguf: true,+ isVision: variantVisionHint,})}/>@@ -4529,7 +4539,8 @@ export function HubModelPicker({const [pinnedQuantValidation, setPinnedQuantValidation] = useState<{validated: boolean;downloaded: ReadonlySet<string>;- }>({ validated: false, downloaded: new Set() });+ visionByRepo: ReadonlyMap<string, boolean>;+ }>({ validated: false, downloaded: new Set(), visionByRepo: new Map() });const prunePinnedQuantValidation = useCallback((repoId: string, quant: string) => {const key = pinKey(repoId, quant);@@ -4558,20 +4569,32 @@ export function HubModelPicker({hfToken || undefined,{ preferLocalCache: true },);- return normalizeGgufVariantsResponse(response)- .variants.filter((variant) => variant.downloaded === true)- .map((variant) => pinKey(repoId, variant.quant));+ const normalized = normalizeGgufVariantsResponse(response);+ return {+ repoId,+ hasVision: normalized.hasVision,+ downloaded: normalized.variants+ .filter((variant) => variant.downloaded === true)+ .map((variant) => pinKey(repoId, variant.quant)),+ };} catch {// If the backend cannot verify a quant, hiding the direct-load row is safer than claiming a// missing file is downloaded.- return [];+ return { repoId, hasVision: undefined, downloaded: [] };}}),).then((groups) => {if (!cancelled) {setPinnedQuantValidation({validated: true,- downloaded: new Set(groups.flat()),+ downloaded: new Set(groups.flatMap((group) => group.downloaded)),+ visionByRepo: new Map(+ groups.flatMap((group) =>+ group.hasVision === undefined+ ? []+ : [[group.repoId, group.hasVision] as const],+ ),+ ),});}});@@ -5640,6 +5663,12 @@ export function HubModelPicker({modelIdsMatchForPicker(loadedModelId, entry.repoId) &&!ggufVariantsMatchForPicker(activeGgufVariant, null) &&ggufVariantsMatchForPicker(activeGgufVariant, entry.quant);+ // A validated false is authoritative: this repo's variants carry no mmproj.+ // Anything else stays unknown rather than warning from a stale cache row.+ const pinnedVisionHint =+ pinnedQuantValidation.visionByRepo.get(entry.repoId) === false+ ? false+ : undefined;return (<divkey={optionKey}@@ -5678,6 +5707,7 @@ export function HubModelPicker({// The row loads one quant, so it is a GGUF pick like the expander's; without this the pages// asked for a pipeline, which a GGUF repo rejects. No filename: the pin stores a label.isGguf: true,+ isVision: pinnedVisionHint,pipelineTag:diffusionTaskById.get(entry.repoId.toLowerCase()) ?? null,})@@ -5697,6 +5727,7 @@ export function HubModelPicker({ggufVariant: entry.quant,isDownloaded: true,isGguf: true,+ isVision: pinnedVisionHint,pipelineTag:diffusionTaskById.get(entry.repoId.toLowerCase()) ?? null,})@@ -5781,6 +5812,10 @@ export function HubModelPicker({isDownloaded,expectedBytes,isGguf: true,+ // readSoleQuant already read the repo's mmproj verdict, and this collapsed row is+ // the only place it can reach the chat warning: forward it conservatively, as the+ // expander does, so a known text-only sole quant still warns.+ isVision: sole.hasVision === false ? false : undefined,pipelineTag: c.task ?? null,};return (diff --git a/studio/frontend/src/features/model-picker/components/model-selector/types.ts b/studio/frontend/src/features/model-picker/components/model-selector/types.tsindex b643297a76e..32d41d79a98 100644--- a/studio/frontend/src/features/model-picker/components/model-selector/types.ts+++ b/studio/frontend/src/features/model-picker/components/model-selector/types.ts@@ -64,6 +64,8 @@ export interface ModelSelectorChangeMeta {/** Direct local .gguf file picked without a variant (custom folder / LM Unsloth). Marks it as a* GGUF source for the deferred-load staging flow. */isGguf?: boolean;+ /** Known model vision capability. Undefined means unknown, not text-only. */+ isVision?: boolean;/** Staged metadata confirmed the separate DiffusionGemma runner. */isDiffusion?: boolean;config?: PerModelConfig;diff --git a/tests/studio/model_vision_capability.test.mjs b/tests/studio/model_vision_capability.test.mjsnew file mode 100644index 00000000000..1f65b2514f8--- /dev/null+++ b/tests/studio/model_vision_capability.test.mjs@@ -0,0 +1,68 @@+// SPDX-License-Identifier: AGPL-3.0-only+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++import assert from "node:assert/strict";+import test from "node:test";++import {+ isKnownTextOnlySelection,+ normalizeGgufVisionCapability,+} from "../../studio/frontend/src/features/chat/utils/model-vision-capability.ts";++test("undefined catalog target is not treated as text-only", () => {+ assert.equal(isKnownTextOnlySelection({}, undefined), false);+});++test("known vision GGUF suppresses the historical-image warning", () => {+ assert.equal(+ isKnownTextOnlySelection(+ { isGguf: true, isVision: true },+ { isGguf: true, isVision: false },+ ),+ false,+ );+});++test("known text-only GGUF emits the historical-image warning", () => {+ assert.equal(+ isKnownTextOnlySelection(+ { isGguf: true, isVision: false },+ { isGguf: true, isVision: false },+ ),+ true,+ );+});++test("unknown GGUF capability does not trust a stale catalog false", () => {+ assert.equal(+ isKnownTextOnlySelection(+ { isGguf: true },+ { isGguf: true, isVision: false },+ ),+ false,+ );+});+test("missing and malformed GGUF vision metadata remain unknown", () => {+ for (const value of [undefined, null, "false", 0]) {+ const isVision = normalizeGgufVisionCapability(value);+ assert.equal(isVision, undefined);+ assert.equal(+ isKnownTextOnlySelection({ isGguf: true, isVision }),+ false,+ );+ }+ assert.equal(normalizeGgufVisionCapability(false), false);+ assert.equal(+ isKnownTextOnlySelection({ isGguf: true, isVision: false }),+ true,+ );+});++++test("known text-only non-GGUF still emits the warning", () => {+ assert.equal(+ isKnownTextOnlySelection({}, { isGguf: false, isVision: false }),+ true,+ );+});diff --git a/tests/studio/test_model_picker_contracts.py b/tests/studio/test_model_picker_contracts.pyindex ef5fd98569e..4a46570bdef 100644--- a/tests/studio/test_model_picker_contracts.py+++ b/tests/studio/test_model_picker_contracts.py@@ -652,6 +652,33 @@ def test_variant_expander_refreshes_after_delete():assert del_confirm, "delete onConfirm must bump refreshKey after a successful delete"+def test_gguf_vision_capability_is_threaded_through_deferred_chat_load():+ """Variant metadata is more authoritative than the parent catalog row for GGUF+ vision support. Both the direct pick and the settings action must carry the hint,+ the pinned-quant row must forward its validated verdict, and the collapsed+ sole-quant row must not drop the mmproj answer it already read."""+ picker = _read("features/model-picker/components/model-selector/pickers.tsx")+ assert "const variantVisionHint = hasVision === false ? false : undefined;" in picker+ assert "hasVision: normalizeGgufVisionCapability(res?.has_vision)," in picker+ assert picker.count("isVision: variantVisionHint") >= 2+ assert "variantVisionHint," in picker+ assert "visionByRepo: ReadonlyMap<string, boolean>" in picker+ assert "pinnedQuantValidation.visionByRepo.get(entry.repoId) === false" in picker+ assert picker.count("isVision: pinnedVisionHint") >= 2+ assert "isVision: sole.hasVision === false ? false : undefined," in picker+ assert "cachedRepo?.has_vision === false" not in picker++ types = _read("features/model-picker/components/model-selector/types.ts")+ runtime = _read("features/chat/hooks/use-chat-model-runtime.ts")+ page = _read("features/chat/chat-page.tsx")+ assert "isVision?: boolean;" in types+ assert "isVision?: boolean;" in runtime+ assert "isVision: meta?.isVision" in page+ assert "isKnownTextOnlySelection(" in page+ assert "selection,\n contextKey:" in page+ assert "{ ...pending.selection, isDownloaded: true }" in page++def test_local_picker_rows_require_chat_capability():"""Local inventory rows can be classified non-chat (canChat false, e.g."""src = _read("features/model-picker/inventory/use-chat-picker-inventory.ts")diff --git a/tests/studio/test_model_vision_capability.py b/tests/studio/test_model_vision_capability.pynew file mode 100644index 00000000000..778a3797e32--- /dev/null+++ b/tests/studio/test_model_vision_capability.py@@ -0,0 +1,43 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Run the TypeScript vision-capability contract from the collected pytest suite.++The warning decision lives in ``studio/frontend`` which carries no JS test runner, so the+real module is executed under node's type stripping instead. On a tree whose chat warning+still reads only the catalog row (the pre-fix behaviour) the module is absent and this+fails, which is what keeps the fix attached to a red baseline.+"""++import shutil+import subprocess+from pathlib import Path++import pytest+++ROOT = Path(__file__).resolve().parents[2]+++def test_model_vision_capability_node_contract():+ if shutil.which("node") is None:+ pytest.skip("node not available")+ probe = subprocess.run(+ ["node", "--experimental-strip-types", "--version"],+ capture_output = True,+ text = True,+ timeout = 5,+ )+ if probe.returncode != 0:+ pytest.skip("node --experimental-strip-types not available")+ subprocess.run(+ [+ "node",+ "--experimental-strip-types",+ "--test",+ str(ROOT / "tests/studio/model_vision_capability.test.mjs"),+ ],+ cwd = ROOT,+ check = True,+ timeout = 30,+ )