Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Fix GGUF vision capability selection (#11696)

Wasim Yousef Said6bcb8af48db6
diff --git a/studio/frontend/src/features/chat/chat-page.tsx b/studio/frontend/src/features/chat/chat-page.tsx
index 5ffe17fcb6b..baa2df7abed 100644
--- a/studio/frontend/src/features/chat/chat-page.tsx
+++ b/studio/frontend/src/features/chat/chat-page.tsx
@@ -133,6 +133,7 @@ import {
useChatArtifactsStore,
useSelectedChatArtifact,
} from "./artifacts/store";
+import { isKnownTextOnlySelection } from "./utils/model-vision-capability";
import type { ChatArtifact, ChatArtifactSurface } from "./artifacts/types";
import { McpServersDialogMount } from "./mcp-composer-button";
import { ChatSettingsPanel } from "./chat-settings-sheet";
@@ -3372,7 +3373,11 @@ export function ChatPage({
(model) => model.id === value,
);
showImageCompatibilityWarning =
- hasImage && targetModel?.isVision === false;
+ hasImage &&
+ isKnownTextOnlySelection(
+ { isVision: meta?.isVision, isGguf: meta?.isGguf },
+ targetModel,
+ );
}
}
}
@@ -3394,6 +3399,7 @@ export function ChatPage({
expectedBytes: meta?.expectedBytes,
downloadPresentation: meta?.downloadPresentation,
isGguf: meta?.isGguf,
+ isVision: meta?.isVision,
isDiffusion: meta?.isDiffusion,
config: meta?.config,
nativePathToken: meta?.nativePathToken,
diff --git a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts
index ca839ab2a5b..946ea49c840 100644
--- a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts
+++ b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts
@@ -185,6 +185,8 @@ export type SelectedModelInput = {
/** Direct local .gguf file (no HF variant / native token), still a GGUF source, so the staging
* flow treats it as one. */
isGguf?: boolean;
+ /** Known model vision capability. Undefined means unknown, not text-only. */
+ isVision?: boolean;
/** Staged metadata confirmed the separate DiffusionGemma runner. */
isDiffusion?: boolean;
throwOnError?: boolean;
diff --git a/studio/frontend/src/features/chat/utils/model-vision-capability.ts b/studio/frontend/src/features/chat/utils/model-vision-capability.ts
new file mode 100644
index 00000000000..84cff70f632
--- /dev/null
+++ b/studio/frontend/src/features/chat/utils/model-vision-capability.ts
@@ -0,0 +1,36 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+export type ModelVisionCapability = {
+ isVision?: boolean;
+ isGguf?: boolean;
+};
+
+/**
+ * Whether a selected model is authoritatively known to be text-only.
+ *
+ * A GGUF repo's catalog row can predate the variant metadata request and report
+ * `isVision: false` even though the selected quant has an mmproj companion.
+ * Prefer the variant-level hint when present; otherwise treat GGUF capability
+ * as unknown rather than warning from stale catalog metadata.
+ */
+export function isKnownTextOnlySelection(
+ selection: ModelVisionCapability,
+ catalogModel?: ModelVisionCapability,
+): boolean {
+ if (selection.isVision !== undefined) {
+ return selection.isVision === false;
+ }
+ if (selection.isGguf === true || catalogModel?.isGguf === true) {
+ return false;
+ }
+ return catalogModel?.isVision === false;
+}
+
+
+/** Preserve missing or malformed backend capability metadata as unknown. */
+export function normalizeGgufVisionCapability(
+ value: unknown,
+): boolean | undefined {
+ return typeof value === "boolean" ? value : undefined;
+}
diff --git a/studio/frontend/src/features/model-picker/components/model-selector/pickers.tsx b/studio/frontend/src/features/model-picker/components/model-selector/pickers.tsx
index ef899b7cbf4..391a1c13acb 100644
--- a/studio/frontend/src/features/model-picker/components/model-selector/pickers.tsx
+++ b/studio/frontend/src/features/model-picker/components/model-selector/pickers.tsx
@@ -14,6 +14,7 @@ import {
import { usePlatformStore } from "@/config/env";
import { INVENTORY_FRESHNESS_WINDOW_MS } from "@/features/hub/inventory";
import { ApiProviderLogo } from "@/features/chat";
+import { normalizeGgufVisionCapability } from "@/features/chat/utils/model-vision-capability";
import {
type ScanFolderInfo,
addScanFolder,
@@ -1463,7 +1464,7 @@ function normalizeGgufVariantsResponse(
): {
variants: GgufVariantDetail[];
defaultVariant: string | null;
- hasVision: boolean;
+ hasVision: boolean | undefined;
contextLength: number | null;
resolvedLocally: boolean;
dependenciesResolved: boolean;
@@ -1477,7 +1478,7 @@ function normalizeGgufVariantsResponse(
typeof res?.default_variant === "string" && res.default_variant.length > 0
? res.default_variant
: null,
- hasVision: res?.has_vision === true,
+ hasVision: normalizeGgufVisionCapability(res?.has_vision),
contextLength:
typeof contextLength === "number" &&
Number.isFinite(contextLength) &&
@@ -1506,7 +1507,7 @@ function ggufVariantExpectedBytes(variant: GgufVariantDetail): number {
* mounts the expander, so this is its only source. */
interface SoleDownloadedQuant {
variant: GgufVariantDetail;
- hasVision: boolean;
+ hasVision: boolean | undefined;
}
/** The repo's one complete quant, or null when Hub metadata cannot verify its dependencies. */
@@ -1710,7 +1711,7 @@ function GgufVariantExpander({
const deleteDisabled = variantActions?.deleteDisabled ?? false;
const [variants, setVariants] = useState<GgufVariantDetail[] | null>(null);
const [defaultVariant, setDefaultVariant] = useState<string | null>(null);
- const [hasVision, setHasVision] = useState(false);
+ const [hasVision, setHasVision] = useState<boolean | undefined>(undefined);
// Native max context (GGUF metadata); only set once a variant is downloaded.
const [nativeContext, setNativeContext] = useState<number | null>(null);
const [loading, setLoading] = useState(true);
@@ -1747,7 +1748,9 @@ function GgufVariantExpander({
setVariants(normalized.variants);
setDefaultVariant(normalized.defaultVariant);
setHasVision(normalized.hasVision);
- onHasVision?.(normalized.hasVision);
+ if (normalized.hasVision !== undefined) {
+ onHasVision?.(normalized.hasVision);
+ }
setNativeContext(normalized.contextLength);
setResolvedLocally(normalized.resolvedLocally);
})
@@ -1769,6 +1772,10 @@ function GgufVariantExpander({
const isLocalPath = /^(\/|\.{1,2}[\\/]|~[\\/]|[A-Za-z]:[\\/]|\\\\)/.test(
repoId,
);
+ // Repo-level true only proves that some mmproj exists; the loader may reject
+ // it for this quant/model family. Absence is authoritative text-only
+ // evidence, while presence remains unknown until load.
+ const variantVisionHint = hasVision === false ? false : undefined;
// The prefix test cannot see a marker-less relative directory like "models/my-image-model",
// so whether the checkpoint is on disk is asked of the listing, not of the spelling.
const checkpointIsLocal = isLocalPath || resolvedLocally;
@@ -1796,6 +1803,7 @@ function GgufVariantExpander({
downloadPresentation,
contextLength: isAvailable ? nativeContext : undefined,
isGguf: true,
+ isVision: variantVisionHint,
pipelineTag,
});
},
@@ -1807,6 +1815,7 @@ function GgufVariantExpander({
sourceOverride,
nativeContext,
pipelineTag,
+ variantVisionHint,
],
);
@@ -2263,6 +2272,7 @@ function GgufVariantExpander({
expectedBytes,
contextLength: nativeContext,
isGguf: true,
+ isVision: variantVisionHint,
})
}
/>
@@ -4529,7 +4539,8 @@ export function HubModelPicker({
const [pinnedQuantValidation, setPinnedQuantValidation] = useState<{
validated: boolean;
downloaded: ReadonlySet<string>;
- }>({ validated: false, downloaded: new Set() });
+ visionByRepo: ReadonlyMap<string, boolean>;
+ }>({ validated: false, downloaded: new Set(), visionByRepo: new Map() });
const prunePinnedQuantValidation = useCallback(
(repoId: string, quant: string) => {
const key = pinKey(repoId, quant);
@@ -4558,20 +4569,32 @@ export function HubModelPicker({
hfToken || undefined,
{ preferLocalCache: true },
);
- return normalizeGgufVariantsResponse(response)
- .variants.filter((variant) => variant.downloaded === true)
- .map((variant) => pinKey(repoId, variant.quant));
+ const normalized = normalizeGgufVariantsResponse(response);
+ return {
+ repoId,
+ hasVision: normalized.hasVision,
+ downloaded: normalized.variants
+ .filter((variant) => variant.downloaded === true)
+ .map((variant) => pinKey(repoId, variant.quant)),
+ };
} catch {
// If the backend cannot verify a quant, hiding the direct-load row is safer than claiming a
// missing file is downloaded.
- return [];
+ return { repoId, hasVision: undefined, downloaded: [] };
}
}),
).then((groups) => {
if (!cancelled) {
setPinnedQuantValidation({
validated: true,
- downloaded: new Set(groups.flat()),
+ downloaded: new Set(groups.flatMap((group) => group.downloaded)),
+ visionByRepo: new Map(
+ groups.flatMap((group) =>
+ group.hasVision === undefined
+ ? []
+ : [[group.repoId, group.hasVision] as const],
+ ),
+ ),
});
}
});
@@ -5640,6 +5663,12 @@ export function HubModelPicker({
modelIdsMatchForPicker(loadedModelId, entry.repoId) &&
!ggufVariantsMatchForPicker(activeGgufVariant, null) &&
ggufVariantsMatchForPicker(activeGgufVariant, entry.quant);
+ // A validated false is authoritative: this repo's variants carry no mmproj.
+ // Anything else stays unknown rather than warning from a stale cache row.
+ const pinnedVisionHint =
+ pinnedQuantValidation.visionByRepo.get(entry.repoId) === false
+ ? false
+ : undefined;
return (
<div
key={optionKey}
@@ -5678,6 +5707,7 @@ export function HubModelPicker({
// The row loads one quant, so it is a GGUF pick like the expander's; without this the pages
// asked for a pipeline, which a GGUF repo rejects. No filename: the pin stores a label.
isGguf: true,
+ isVision: pinnedVisionHint,
pipelineTag:
diffusionTaskById.get(entry.repoId.toLowerCase()) ?? null,
})
@@ -5697,6 +5727,7 @@ export function HubModelPicker({
ggufVariant: entry.quant,
isDownloaded: true,
isGguf: true,
+ isVision: pinnedVisionHint,
pipelineTag:
diffusionTaskById.get(entry.repoId.toLowerCase()) ?? null,
})
@@ -5781,6 +5812,10 @@ export function HubModelPicker({
isDownloaded,
expectedBytes,
isGguf: true,
+ // readSoleQuant already read the repo's mmproj verdict, and this collapsed row is
+ // the only place it can reach the chat warning: forward it conservatively, as the
+ // expander does, so a known text-only sole quant still warns.
+ isVision: sole.hasVision === false ? false : undefined,
pipelineTag: c.task ?? null,
};
return (
diff --git a/studio/frontend/src/features/model-picker/components/model-selector/types.ts b/studio/frontend/src/features/model-picker/components/model-selector/types.ts
index b643297a76e..32d41d79a98 100644
--- a/studio/frontend/src/features/model-picker/components/model-selector/types.ts
+++ b/studio/frontend/src/features/model-picker/components/model-selector/types.ts
@@ -64,6 +64,8 @@ export interface ModelSelectorChangeMeta {
/** Direct local .gguf file picked without a variant (custom folder / LM Unsloth). Marks it as a
* GGUF source for the deferred-load staging flow. */
isGguf?: boolean;
+ /** Known model vision capability. Undefined means unknown, not text-only. */
+ isVision?: boolean;
/** Staged metadata confirmed the separate DiffusionGemma runner. */
isDiffusion?: boolean;
config?: PerModelConfig;
diff --git a/tests/studio/model_vision_capability.test.mjs b/tests/studio/model_vision_capability.test.mjs
new file mode 100644
index 00000000000..1f65b2514f8
--- /dev/null
+++ b/tests/studio/model_vision_capability.test.mjs
@@ -0,0 +1,68 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import {
+ isKnownTextOnlySelection,
+ normalizeGgufVisionCapability,
+} from "../../studio/frontend/src/features/chat/utils/model-vision-capability.ts";
+
+test("undefined catalog target is not treated as text-only", () => {
+ assert.equal(isKnownTextOnlySelection({}, undefined), false);
+});
+
+test("known vision GGUF suppresses the historical-image warning", () => {
+ assert.equal(
+ isKnownTextOnlySelection(
+ { isGguf: true, isVision: true },
+ { isGguf: true, isVision: false },
+ ),
+ false,
+ );
+});
+
+test("known text-only GGUF emits the historical-image warning", () => {
+ assert.equal(
+ isKnownTextOnlySelection(
+ { isGguf: true, isVision: false },
+ { isGguf: true, isVision: false },
+ ),
+ true,
+ );
+});
+
+test("unknown GGUF capability does not trust a stale catalog false", () => {
+ assert.equal(
+ isKnownTextOnlySelection(
+ { isGguf: true },
+ { isGguf: true, isVision: false },
+ ),
+ false,
+ );
+});
+test("missing and malformed GGUF vision metadata remain unknown", () => {
+ for (const value of [undefined, null, "false", 0]) {
+ const isVision = normalizeGgufVisionCapability(value);
+ assert.equal(isVision, undefined);
+ assert.equal(
+ isKnownTextOnlySelection({ isGguf: true, isVision }),
+ false,
+ );
+ }
+ assert.equal(normalizeGgufVisionCapability(false), false);
+ assert.equal(
+ isKnownTextOnlySelection({ isGguf: true, isVision: false }),
+ true,
+ );
+});
+
+
+
+test("known text-only non-GGUF still emits the warning", () => {
+ assert.equal(
+ isKnownTextOnlySelection({}, { isGguf: false, isVision: false }),
+ true,
+ );
+});
diff --git a/tests/studio/test_model_picker_contracts.py b/tests/studio/test_model_picker_contracts.py
index ef5fd98569e..4a46570bdef 100644
--- a/tests/studio/test_model_picker_contracts.py
+++ b/tests/studio/test_model_picker_contracts.py
@@ -652,6 +652,33 @@ def test_variant_expander_refreshes_after_delete():
assert del_confirm, "delete onConfirm must bump refreshKey after a successful delete"
+def test_gguf_vision_capability_is_threaded_through_deferred_chat_load():
+ """Variant metadata is more authoritative than the parent catalog row for GGUF
+ vision support. Both the direct pick and the settings action must carry the hint,
+ the pinned-quant row must forward its validated verdict, and the collapsed
+ sole-quant row must not drop the mmproj answer it already read."""
+ picker = _read("features/model-picker/components/model-selector/pickers.tsx")
+ assert "const variantVisionHint = hasVision === false ? false : undefined;" in picker
+ assert "hasVision: normalizeGgufVisionCapability(res?.has_vision)," in picker
+ assert picker.count("isVision: variantVisionHint") >= 2
+ assert "variantVisionHint," in picker
+ assert "visionByRepo: ReadonlyMap<string, boolean>" in picker
+ assert "pinnedQuantValidation.visionByRepo.get(entry.repoId) === false" in picker
+ assert picker.count("isVision: pinnedVisionHint") >= 2
+ assert "isVision: sole.hasVision === false ? false : undefined," in picker
+ assert "cachedRepo?.has_vision === false" not in picker
+
+ types = _read("features/model-picker/components/model-selector/types.ts")
+ runtime = _read("features/chat/hooks/use-chat-model-runtime.ts")
+ page = _read("features/chat/chat-page.tsx")
+ assert "isVision?: boolean;" in types
+ assert "isVision?: boolean;" in runtime
+ assert "isVision: meta?.isVision" in page
+ assert "isKnownTextOnlySelection(" in page
+ assert "selection,\n contextKey:" in page
+ assert "{ ...pending.selection, isDownloaded: true }" in page
+
+
def test_local_picker_rows_require_chat_capability():
"""Local inventory rows can be classified non-chat (canChat false, e.g."""
src = _read("features/model-picker/inventory/use-chat-picker-inventory.ts")
diff --git a/tests/studio/test_model_vision_capability.py b/tests/studio/test_model_vision_capability.py
new file mode 100644
index 00000000000..778a3797e32
--- /dev/null
+++ b/tests/studio/test_model_vision_capability.py
@@ -0,0 +1,43 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Run the TypeScript vision-capability contract from the collected pytest suite.
+
+The warning decision lives in ``studio/frontend`` which carries no JS test runner, so the
+real module is executed under node's type stripping instead. On a tree whose chat warning
+still reads only the catalog row (the pre-fix behaviour) the module is absent and this
+fails, which is what keeps the fix attached to a red baseline.
+"""
+
+import shutil
+import subprocess
+from pathlib import Path
+
+import pytest
+
+
+ROOT = Path(__file__).resolve().parents[2]
+
+
+def test_model_vision_capability_node_contract():
+ if shutil.which("node") is None:
+ pytest.skip("node not available")
+ probe = subprocess.run(
+ ["node", "--experimental-strip-types", "--version"],
+ capture_output = True,
+ text = True,
+ timeout = 5,
+ )
+ if probe.returncode != 0:
+ pytest.skip("node --experimental-strip-types not available")
+ subprocess.run(
+ [
+ "node",
+ "--experimental-strip-types",
+ "--test",
+ str(ROOT / "tests/studio/model_vision_capability.test.mjs"),
+ ],
+ cwd = ROOT,
+ check = True,
+ timeout = 30,
+ )