Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Recommend the repo's default GGUF quant instead of the largest that fits (#11606)

Daniel Hanedd54cb7044b
diff --git a/studio/frontend/src/features/model-picker/components/model-selector/model-catalog.ts b/studio/frontend/src/features/model-picker/components/model-selector/model-catalog.ts
index 77e0abe3b46..3b77e6a6919 100644
--- a/studio/frontend/src/features/model-picker/components/model-selector/model-catalog.ts
+++ b/studio/frontend/src/features/model-picker/components/model-selector/model-catalog.ts
@@ -1133,15 +1133,18 @@ export function ggufFitIsComfortable(fit: GgufFitClass): boolean {
return fit === "fits" || fit === "ram";
}
-/** What to recommend on a device whose budget is known: the largest quant that runs with room to
- * spare, else the largest that runs at all, else the smallest. Quality the machine can actually
- * hold, rather than whatever size the repo defaults to.
+/** What to recommend on a device whose budget is known. With a repo default (`preferred`, the
+ * backend's pick: UD-Q4_K_XL, else Q4_K_M, else Q4_K_S), that default wherever it loads, and
+ * the largest smaller quant that does when it cannot. Spare memory is not a reason to star F16:
+ * on a large card that recommended a 13 GiB F16 over a 3.9 GiB Q4_K_M. Without one, the largest
+ * quant that runs with room to spare, else the largest that runs at all, else the smallest.
*
* Null when no variant carries a size, since then there is nothing to weigh against the device
* and the caller's repo default is the better guess. */
export function recommendedQuantForDevice<T extends GgufVariantSizes>(
variants: readonly T[],
fitOf: (sizeBytes: number) => GgufFitClass,
+ preferred: T | null = null,
): T | null {
// Ranked by the weights, since that is the quality on offer and the size the row shows. Judged on
// the download footprint, which also covers the companion GGUFs the loader charges for: a vision
@@ -1167,9 +1170,19 @@ export function recommendedQuantForDevice<T extends GgufVariantSizes>(
? variant
: best,
);
+ // Step down from the default when it cannot load, never up past it.
+ const candidates =
+ preferred && variants.includes(preferred)
+ ? ggufVariantFitSizeBytes(preferred) <= 0 ||
+ fitOfVariant(preferred) !== "oom"
+ ? [preferred]
+ : bySizeDesc.filter(
+ (variant) => variant.size_bytes < preferred.size_bytes,
+ )
+ : bySizeDesc;
return (
- bySizeDesc.find((variant) => ggufFitIsComfortable(fitOfVariant(variant))) ??
- bySizeDesc.find((variant) => fitOfVariant(variant) !== "oom") ??
+ candidates.find((variant) => ggufFitIsComfortable(fitOfVariant(variant))) ??
+ candidates.find((variant) => fitOfVariant(variant) !== "oom") ??
smallestFootprint
);
}
diff --git a/studio/frontend/src/features/model-picker/components/model-selector/pickers.tsx b/studio/frontend/src/features/model-picker/components/model-selector/pickers.tsx
index aac380245f8..26d53532e1d 100644
--- a/studio/frontend/src/features/model-picker/components/model-selector/pickers.tsx
+++ b/studio/frontend/src/features/model-picker/components/model-selector/pickers.tsx
@@ -1854,9 +1854,8 @@ function GgufVariantExpander({
[variantGroups, defaultVariant],
);
- // Each workflow gets its own recommendation: the largest quant the device runs with room to
- // spare, else the largest that runs at all, else the smallest. The repo default only decides it
- // when nothing has been measured, so a big machine is not held at the default's size.
+ // Each workflow gets its own recommendation: the repo default (UD-Q4_K_XL, else Q4_K_M, else
+ // Q4_K_S) wherever the device loads it, else the largest smaller quant that loads.
const effectiveRecommendedByGroup = useMemo(() => {
const recommended = new Map<string, string>();
for (const group of variantGroups) {
@@ -1866,7 +1865,9 @@ function GgufVariantExpander({
continue;
}
// Null when the group carries no sizes at all, so there is nothing to measure.
- const pick = recommendedQuantForDevice(group.variants, getGgufFit) ?? preferred;
+ const pick =
+ recommendedQuantForDevice(group.variants, getGgufFit, preferred) ??
+ preferred;
if (pick) recommended.set(group.key, pick.quant);
}
return recommended;
diff --git a/studio/frontend/tests/recommended-quant-for-device.test.ts b/studio/frontend/tests/recommended-quant-for-device.test.ts
index 1eac7f336bc..78d78d10e59 100644
--- a/studio/frontend/tests/recommended-quant-for-device.test.ts
+++ b/studio/frontend/tests/recommended-quant-for-device.test.ts
@@ -1,9 +1,10 @@
// SPDX-License-Identifier: AGPL-3.0-only
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
-// The picker used to recommend whatever quant the repo defaults to, whenever it ran at all. On a
-// machine with room for far more, that pinned the suggestion to the default's size: a 64 GiB box
-// was told to run UD-Q4_K_XL at 18 GiB while UD-Q6_K_XL at 25 GiB fit with the reserve intact.
+// With a repo default the picker recommends that default (UD-Q4_K_XL, else Q4_K_M, else Q4_K_S)
+// wherever it loads, and steps down only when it cannot. Ranking by "largest that fits" starred
+// the 13.25 GiB F16 of Qwen-Image-2.1-GGUF on any large card. Without a default it still ranks by
+// the largest quant the device holds with room to spare.
import assert from "node:assert/strict";
import test from "node:test";
@@ -39,7 +40,7 @@ test("a comfortable verdict is one that keeps its reserve", () => {
assert.ok(!ggufFitIsComfortable("oom"));
});
-test("a roomy machine is recommended more than the repo default's size", () => {
+test("with no repo default, a roomy machine gets the largest comfortable quant", () => {
const pick = recommendedQuantForDevice(QUANTS, fitOn(0, 64));
assert.equal(pick?.quant, "UD-Q6_K_XL");
});
@@ -102,7 +103,10 @@ test("a variant of unknown size does not outrank one that runs", () => {
const fit = fitOn(24, 16);
assert.equal(fit(0), "fits");
for (const size of [22, 20.5]) {
- assert.ok(!ggufFitIsComfortable(fit(size * GB)), `${size} GiB is comfortable`);
+ assert.ok(
+ !ggufFitIsComfortable(fit(size * GB)),
+ `${size} GiB is comfortable`,
+ );
assert.notEqual(fit(size * GB), "oom", `${size} GiB does not run`);
}
assert.equal(recommendedQuantForDevice(listing, fit)?.quant, "UD-Q8_K_XL");
@@ -122,7 +126,10 @@ test("a machine too small for anything still gets the smallest", () => {
});
test("nothing to choose from is not a choice", () => {
- assert.equal(recommendedQuantForDevice([], () => "fits"), null);
+ assert.equal(
+ recommendedQuantForDevice([], () => "fits"),
+ null,
+ );
});
// Companions are picked per checkpoint, so download size does not have to rise
@@ -149,3 +156,91 @@ test("the footprint fallback keeps the better quant when footprints tie", () =>
"UD-Q6_K_XL",
);
});
+
+/** unsloth/Qwen-Image-2.1-GGUF: no UD-Q4_K_XL, so the backend's default is Q4_K_M. */
+const QWEN_IMAGE_21 = [
+ variant("F16", 13.25),
+ variant("Q2_K", 2.3),
+ variant("Q3_K_S", 2.54),
+ variant("Q3_K_M", 2.95),
+ variant("Q3_K_XL", 3.36),
+ variant("Q4_K_S", 3.64),
+ variant("Q4_K_M", 3.91),
+ variant("Q5_K_S", 4.19),
+ variant("Q5_K_M", 5.02),
+ variant("Q6_K", 5.84),
+ variant("Q6_K_XL", 6.26),
+ variant("Q8_0", 7.12),
+];
+const byQuant = (list: typeof QWEN_IMAGE_21, quant: string) =>
+ list.find((q) => q.quant === quant) ?? null;
+
+test("a large card is recommended the repo default, not F16", () => {
+ const preferred = byQuant(QWEN_IMAGE_21, "Q4_K_M");
+ for (const [gpuGb, ramGb] of [
+ [24, 64],
+ [80, 256],
+ [180, 2000],
+ ]) {
+ const fit = fitOn(gpuGb, ramGb);
+ assert.ok(
+ ggufFitIsComfortable(fit(13.25 * GB)),
+ "F16 fits, so the old rule starred it",
+ );
+ assert.equal(
+ recommendedQuantForDevice(QWEN_IMAGE_21, fit, preferred)?.quant,
+ "Q4_K_M",
+ `${gpuGb} GiB card`,
+ );
+ }
+});
+
+test("the default stays recommended while it loads at all", () => {
+ const preferred = byQuant(QWEN_IMAGE_21, "Q4_K_M");
+ // Only runs by spilling to RAM, but it runs.
+ const fit = (size: number) => (size <= 3.91 * GB ? "partial" : "oom");
+ assert.equal(
+ recommendedQuantForDevice(QWEN_IMAGE_21, fit, preferred)?.quant,
+ "Q4_K_M",
+ );
+});
+
+test("a default the device cannot load steps down, never up", () => {
+ const preferred = byQuant(QWEN_IMAGE_21, "Q4_K_M");
+ // Q4_K_S and smaller fit; the default and everything above it do not.
+ const fit = (size: number) => (size <= 3.7 * GB ? "fits" : "oom");
+ assert.equal(
+ recommendedQuantForDevice(QWEN_IMAGE_21, fit, preferred)?.quant,
+ "Q4_K_S",
+ );
+ // Only a spill fits below the default: the largest of those, not a larger quant that is OOM.
+ const tight = (size: number) => (size <= 3.0 * GB ? "partial" : "oom");
+ assert.equal(
+ recommendedQuantForDevice(QWEN_IMAGE_21, tight, preferred)?.quant,
+ "Q3_K_M",
+ );
+});
+
+test("a UD-Q4_K_XL default is kept on a machine that holds every quant", () => {
+ const preferred = byQuant(QUANTS, "UD-Q4_K_XL");
+ assert.equal(
+ recommendedQuantForDevice(QUANTS, fitOn(0, 256), preferred)?.quant,
+ "UD-Q4_K_XL",
+ );
+});
+
+test("nothing loads with a default either: the smallest footprint", () => {
+ const preferred = byQuant(QWEN_IMAGE_21, "Q4_K_M");
+ assert.equal(
+ recommendedQuantForDevice(QWEN_IMAGE_21, () => "oom", preferred)?.quant,
+ "Q2_K",
+ );
+});
+
+test("a default that is not in this group is ignored", () => {
+ const stranger = variant("Q4_K_M", 3.91);
+ assert.equal(
+ recommendedQuantForDevice(QUANTS, fitOn(0, 64), stranger)?.quant,
+ "UD-Q6_K_XL",
+ );
+});