Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Recommend the repo's default GGUF quant instead of the largest that fits (#11606)
Daniel Hanedd54cb7044b
diff --git a/studio/frontend/src/features/model-picker/components/model-selector/model-catalog.ts b/studio/frontend/src/features/model-picker/components/model-selector/model-catalog.tsindex 77e0abe3b46..3b77e6a6919 100644--- a/studio/frontend/src/features/model-picker/components/model-selector/model-catalog.ts+++ b/studio/frontend/src/features/model-picker/components/model-selector/model-catalog.ts@@ -1133,15 +1133,18 @@ export function ggufFitIsComfortable(fit: GgufFitClass): boolean {return fit === "fits" || fit === "ram";}-/** What to recommend on a device whose budget is known: the largest quant that runs with room to- * spare, else the largest that runs at all, else the smallest. Quality the machine can actually- * hold, rather than whatever size the repo defaults to.+/** What to recommend on a device whose budget is known. With a repo default (`preferred`, the+ * backend's pick: UD-Q4_K_XL, else Q4_K_M, else Q4_K_S), that default wherever it loads, and+ * the largest smaller quant that does when it cannot. Spare memory is not a reason to star F16:+ * on a large card that recommended a 13 GiB F16 over a 3.9 GiB Q4_K_M. Without one, the largest+ * quant that runs with room to spare, else the largest that runs at all, else the smallest.** Null when no variant carries a size, since then there is nothing to weigh against the device* and the caller's repo default is the better guess. */export function recommendedQuantForDevice<T extends GgufVariantSizes>(variants: readonly T[],fitOf: (sizeBytes: number) => GgufFitClass,+ preferred: T | null = null,): T | null {// Ranked by the weights, since that is the quality on offer and the size the row shows. Judged on// the download footprint, which also covers the companion GGUFs the loader charges for: a vision@@ -1167,9 +1170,19 @@ export function recommendedQuantForDevice<T extends GgufVariantSizes>(? variant: best,);+ // Step down from the default when it cannot load, never up past it.+ const candidates =+ preferred && variants.includes(preferred)+ ? ggufVariantFitSizeBytes(preferred) <= 0 ||+ fitOfVariant(preferred) !== "oom"+ ? [preferred]+ : bySizeDesc.filter(+ (variant) => variant.size_bytes < preferred.size_bytes,+ )+ : bySizeDesc;return (- bySizeDesc.find((variant) => ggufFitIsComfortable(fitOfVariant(variant))) ??- bySizeDesc.find((variant) => fitOfVariant(variant) !== "oom") ??+ candidates.find((variant) => ggufFitIsComfortable(fitOfVariant(variant))) ??+ candidates.find((variant) => fitOfVariant(variant) !== "oom") ??smallestFootprint);}diff --git a/studio/frontend/src/features/model-picker/components/model-selector/pickers.tsx b/studio/frontend/src/features/model-picker/components/model-selector/pickers.tsxindex aac380245f8..26d53532e1d 100644--- a/studio/frontend/src/features/model-picker/components/model-selector/pickers.tsx+++ b/studio/frontend/src/features/model-picker/components/model-selector/pickers.tsx@@ -1854,9 +1854,8 @@ function GgufVariantExpander({[variantGroups, defaultVariant],);- // Each workflow gets its own recommendation: the largest quant the device runs with room to- // spare, else the largest that runs at all, else the smallest. The repo default only decides it- // when nothing has been measured, so a big machine is not held at the default's size.+ // Each workflow gets its own recommendation: the repo default (UD-Q4_K_XL, else Q4_K_M, else+ // Q4_K_S) wherever the device loads it, else the largest smaller quant that loads.const effectiveRecommendedByGroup = useMemo(() => {const recommended = new Map<string, string>();for (const group of variantGroups) {@@ -1866,7 +1865,9 @@ function GgufVariantExpander({continue;}// Null when the group carries no sizes at all, so there is nothing to measure.- const pick = recommendedQuantForDevice(group.variants, getGgufFit) ?? preferred;+ const pick =+ recommendedQuantForDevice(group.variants, getGgufFit, preferred) ??+ preferred;if (pick) recommended.set(group.key, pick.quant);}return recommended;diff --git a/studio/frontend/tests/recommended-quant-for-device.test.ts b/studio/frontend/tests/recommended-quant-for-device.test.tsindex 1eac7f336bc..78d78d10e59 100644--- a/studio/frontend/tests/recommended-quant-for-device.test.ts+++ b/studio/frontend/tests/recommended-quant-for-device.test.ts@@ -1,9 +1,10 @@// SPDX-License-Identifier: AGPL-3.0-only// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0-// The picker used to recommend whatever quant the repo defaults to, whenever it ran at all. On a-// machine with room for far more, that pinned the suggestion to the default's size: a 64 GiB box-// was told to run UD-Q4_K_XL at 18 GiB while UD-Q6_K_XL at 25 GiB fit with the reserve intact.+// With a repo default the picker recommends that default (UD-Q4_K_XL, else Q4_K_M, else Q4_K_S)+// wherever it loads, and steps down only when it cannot. Ranking by "largest that fits" starred+// the 13.25 GiB F16 of Qwen-Image-2.1-GGUF on any large card. Without a default it still ranks by+// the largest quant the device holds with room to spare.import assert from "node:assert/strict";import test from "node:test";@@ -39,7 +40,7 @@ test("a comfortable verdict is one that keeps its reserve", () => {assert.ok(!ggufFitIsComfortable("oom"));});-test("a roomy machine is recommended more than the repo default's size", () => {+test("with no repo default, a roomy machine gets the largest comfortable quant", () => {const pick = recommendedQuantForDevice(QUANTS, fitOn(0, 64));assert.equal(pick?.quant, "UD-Q6_K_XL");});@@ -102,7 +103,10 @@ test("a variant of unknown size does not outrank one that runs", () => {const fit = fitOn(24, 16);assert.equal(fit(0), "fits");for (const size of [22, 20.5]) {- assert.ok(!ggufFitIsComfortable(fit(size * GB)), `${size} GiB is comfortable`);+ assert.ok(+ !ggufFitIsComfortable(fit(size * GB)),+ `${size} GiB is comfortable`,+ );assert.notEqual(fit(size * GB), "oom", `${size} GiB does not run`);}assert.equal(recommendedQuantForDevice(listing, fit)?.quant, "UD-Q8_K_XL");@@ -122,7 +126,10 @@ test("a machine too small for anything still gets the smallest", () => {});test("nothing to choose from is not a choice", () => {- assert.equal(recommendedQuantForDevice([], () => "fits"), null);+ assert.equal(+ recommendedQuantForDevice([], () => "fits"),+ null,+ );});// Companions are picked per checkpoint, so download size does not have to rise@@ -149,3 +156,91 @@ test("the footprint fallback keeps the better quant when footprints tie", () =>"UD-Q6_K_XL",);});++/** unsloth/Qwen-Image-2.1-GGUF: no UD-Q4_K_XL, so the backend's default is Q4_K_M. */+const QWEN_IMAGE_21 = [+ variant("F16", 13.25),+ variant("Q2_K", 2.3),+ variant("Q3_K_S", 2.54),+ variant("Q3_K_M", 2.95),+ variant("Q3_K_XL", 3.36),+ variant("Q4_K_S", 3.64),+ variant("Q4_K_M", 3.91),+ variant("Q5_K_S", 4.19),+ variant("Q5_K_M", 5.02),+ variant("Q6_K", 5.84),+ variant("Q6_K_XL", 6.26),+ variant("Q8_0", 7.12),+];+const byQuant = (list: typeof QWEN_IMAGE_21, quant: string) =>+ list.find((q) => q.quant === quant) ?? null;++test("a large card is recommended the repo default, not F16", () => {+ const preferred = byQuant(QWEN_IMAGE_21, "Q4_K_M");+ for (const [gpuGb, ramGb] of [+ [24, 64],+ [80, 256],+ [180, 2000],+ ]) {+ const fit = fitOn(gpuGb, ramGb);+ assert.ok(+ ggufFitIsComfortable(fit(13.25 * GB)),+ "F16 fits, so the old rule starred it",+ );+ assert.equal(+ recommendedQuantForDevice(QWEN_IMAGE_21, fit, preferred)?.quant,+ "Q4_K_M",+ `${gpuGb} GiB card`,+ );+ }+});++test("the default stays recommended while it loads at all", () => {+ const preferred = byQuant(QWEN_IMAGE_21, "Q4_K_M");+ // Only runs by spilling to RAM, but it runs.+ const fit = (size: number) => (size <= 3.91 * GB ? "partial" : "oom");+ assert.equal(+ recommendedQuantForDevice(QWEN_IMAGE_21, fit, preferred)?.quant,+ "Q4_K_M",+ );+});++test("a default the device cannot load steps down, never up", () => {+ const preferred = byQuant(QWEN_IMAGE_21, "Q4_K_M");+ // Q4_K_S and smaller fit; the default and everything above it do not.+ const fit = (size: number) => (size <= 3.7 * GB ? "fits" : "oom");+ assert.equal(+ recommendedQuantForDevice(QWEN_IMAGE_21, fit, preferred)?.quant,+ "Q4_K_S",+ );+ // Only a spill fits below the default: the largest of those, not a larger quant that is OOM.+ const tight = (size: number) => (size <= 3.0 * GB ? "partial" : "oom");+ assert.equal(+ recommendedQuantForDevice(QWEN_IMAGE_21, tight, preferred)?.quant,+ "Q3_K_M",+ );+});++test("a UD-Q4_K_XL default is kept on a machine that holds every quant", () => {+ const preferred = byQuant(QUANTS, "UD-Q4_K_XL");+ assert.equal(+ recommendedQuantForDevice(QUANTS, fitOn(0, 256), preferred)?.quant,+ "UD-Q4_K_XL",+ );+});++test("nothing loads with a default either: the smallest footprint", () => {+ const preferred = byQuant(QWEN_IMAGE_21, "Q4_K_M");+ assert.equal(+ recommendedQuantForDevice(QWEN_IMAGE_21, () => "oom", preferred)?.quant,+ "Q2_K",+ );+});++test("a default that is not in this group is ignored", () => {+ const stranger = variant("Q4_K_M", 3.91);+ assert.equal(+ recommendedQuantForDevice(QUANTS, fitOn(0, 64), stranger)?.quant,+ "UD-Q6_K_XL",+ );+});