Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: offer vLLM or SGLang when the default engine cannot run a quantized checkpoint (#13135)

Daniel Hanbb478c67fe36
diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.py
index 70f2ef69016..57b130cab47 100644
--- a/studio/backend/models/inference.py
+++ b/studio/backend/models/inference.py
@@ -758,6 +758,15 @@ class ValidateModelRequest(BaseModel):
)
+class ManagedEngineOffer(BaseModel):
+ """Optional engines that can run a checkpoint the Default engine cannot."""
+
+ quantization: str = Field(..., description = "quant_method from the checkpoint's config.json")
+ engines: List[Literal["vllm", "sglang"]] = Field(
+ ..., description = "Engines this host can run, in the order to offer them"
+ )
+
+
class TransformersUpgradeInfo(BaseModel):
"""A model architecture no installed transformers ships, but a newer release does."""
@@ -961,6 +970,11 @@ class ValidateModelResponse(BaseModel):
"loaded in place of the requested unsloth bnb-4bit repo (or of a LoRA's bnb base), "
"because MLX cannot read bitsandbytes weights. None when the pick loads as asked.",
)
+ managed_engine_offer: Optional[ManagedEngineOffer] = Field(
+ None,
+ description = "Set when the Default engine cannot run this checkpoint's quantization "
+ "but an optional engine can on this host; the UI offers to load it there.",
+ )
class EstimateMemoryRequest(BaseModel):
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 6d3ee5d50b4..2d1277fd6ba 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -16839,6 +16839,96 @@ def _vllm_engine_hint(engine: Optional[str]) -> str:
return ""
+# Quantizations the Default engine cannot run, with the packages that would let it (Studio ships none).
+_MANAGED_ENGINE_QUANTIZATIONS = {
+ "compressed-tensors": (),
+ "awq": ("gptqmodel", "awq"),
+ "gptq": ("gptqmodel", "auto_gptq"),
+}
+_OFFERED_ENGINES = ("vllm", "sglang")
+
+
+def _managed_engine_offer(
+ metadata, *, engine, is_gguf, is_lora, is_audio, supported
+) -> Optional[dict]:
+ """The engines to offer for a checkpoint the Default engine cannot run, or None.
+
+ ``supported(engine, quant_method)`` is asked last, so the GPU probe only runs for these
+ checkpoints. GGUF, adapters and audio models are refused by the optional engines."""
+ if (
+ engine not in (None, "auto")
+ or is_gguf
+ or is_lora
+ or is_audio
+ or not isinstance(metadata, dict)
+ ):
+ return None
+ text_config = metadata.get("text_config")
+ quant = metadata.get("quantization_config") or (
+ text_config.get("quantization_config") if isinstance(text_config, dict) else None
+ )
+ method = quant.get("quant_method") if isinstance(quant, dict) else None
+ if not isinstance(method, str):
+ return None
+ method = method.strip().lower()
+ if method not in _MANAGED_ENGINE_QUANTIZATIONS:
+ return None
+ import importlib.util
+
+ for module in _MANAGED_ENGINE_QUANTIZATIONS[method]:
+ try:
+ if importlib.util.find_spec(module) is not None:
+ return None
+ except (ImportError, ValueError):
+ pass
+ engines = [name for name in _OFFERED_ENGINES if supported(name, method)]
+ return {"quantization": method, "engines": engines} if engines else None
+
+
+def _managed_engine_offer_for(config, hf_token) -> Optional[dict]:
+ """``_managed_engine_offer`` from the checkpoint's own config.json; None on any failure."""
+ try:
+ if config.is_local:
+ path = Path(config.path) / "config.json"
+ else:
+ from huggingface_hub import hf_hub_download, try_to_load_from_cache
+
+ # Resolving the model already cached config.json; the Hub round trip costs ~45 ms per pick.
+ cached = try_to_load_from_cache(config.identifier, "config.json")
+ path = Path(
+ cached
+ if isinstance(cached, str)
+ else hf_hub_download(config.identifier, "config.json", token = hf_token)
+ )
+ metadata = json.loads(path.read_text(encoding = "utf-8"))
+ from core.inference.engine_install import _driver_rows, support_reason
+
+ def supported(name, method):
+ if support_reason(name) is not None:
+ return False
+ if name != "sglang" or method != "compressed-tensors":
+ return True
+ # SGLang 0.5.20's compressed-tensors NVFP4 needs SM 10.0; vLLM falls back to Marlin from 7.5.
+ try:
+ return any(
+ float(row[1]) >= 10.0 for row in _driver_rows(None) or () if len(row) == 2
+ )
+ except ValueError:
+ return False
+
+ return _managed_engine_offer(
+ metadata,
+ engine = "auto",
+ is_gguf = getattr(config, "is_gguf", False),
+ is_lora = getattr(config, "is_lora", False),
+ is_audio = getattr(config, "is_audio", False),
+ supported = supported,
+ )
+ except Exception as exc:
+ logger.debug("Managed engine offer check failed for '%s': %s", config.identifier, exc)
+ return None
+
+
def _diagnosis_text(msg: str) -> str:
"""``msg`` up to the startup-diagnostics block, which is not ours to read.
@@ -20122,6 +20212,16 @@ async def validate_model(
except Exception as e:
logger.debug("Header probe failed for %s: %s", model_log_label, e)
+ managed_engine_offer = None
+ if request.engine == "auto" and not is_gguf:
+ managed_engine_offer = await asyncio.to_thread(
+ _offline_guarded,
+ (model_identifier, config.identifier, getattr(config, "base_model", None)),
+ _managed_engine_offer_for,
+ config,
+ request.hf_token,
+ )
+
return restore_inventory_handles(
ValidateModelResponse(
valid = True,
@@ -20155,6 +20255,7 @@ async def validate_model(
requires_transformers_upgrade = transformers_upgrade is not None,
transformers_upgrade = transformers_upgrade,
mlx_loads_base_model = await asyncio.to_thread(_mlx_base_for_config, config),
+ managed_engine_offer = managed_engine_offer,
)
)
diff --git a/studio/backend/tests/test_managed_engine_offer.py b/studio/backend/tests/test_managed_engine_offer.py
new file mode 100644
index 00000000000..de18a08624e
--- /dev/null
+++ b/studio/backend/tests/test_managed_engine_offer.py
@@ -0,0 +1,236 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""/validate offers vLLM for checkpoints the Default engine cannot run (#11728).
+
+Only where the host can run vLLM, only for the Default engine, and only when the
+checkpoint's own config.json says compressed-tensors, or AWQ / GPTQ without the
+packages the Default engine would need for them.
+"""
+
+import importlib.util
+import json
+from pathlib import Path
+from types import SimpleNamespace
+from unittest.mock import patch
+
+import pytest
+
+from models.inference import ValidateModelResponse
+
+_BACKEND_ROOT = Path(__file__).resolve().parent.parent
+
+
+def _route():
+ spec = importlib.util.spec_from_file_location(
+ "inference_route_managed_engine_offer", _BACKEND_ROOT / "routes/inference.py"
+ )
+ module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(module)
+ return module
+
+
+route = _route()
+NVFP4 = {
+ "quantization_config": {"quant_method": "compressed-tensors", "format": "nvfp4-pack-quantized"}
+}
+
+
+def offer(
+ metadata,
+ supported = lambda name, method: True,
+ **overrides,
+):
+ kwargs = {
+ "engine": "auto",
+ "is_gguf": False,
+ "is_lora": False,
+ "is_audio": False,
+ "supported": supported,
+ }
+ kwargs.update(overrides)
+ return route._managed_engine_offer(metadata, **kwargs)
+
+
[email protected](autouse = True)
+def _no_awq_or_gptq_packages():
+ real = importlib.util.find_spec
+ missing = {"gptqmodel", "awq", "auto_gptq"}
+ with patch.object(
+ importlib.util,
+ "find_spec",
+ side_effect = lambda name, *a: None if name in missing else real(name, *a),
+ ):
+ yield
+
+
+def test_a_compressed_tensors_checkpoint_is_offered_vllm():
+ assert offer(NVFP4) == {"quantization": "compressed-tensors", "engines": ["vllm", "sglang"]}
+ # Vision checkpoints keep it under text_config.
+ assert offer({"text_config": NVFP4})["engines"] == ["vllm", "sglang"]
+ assert (
+ offer({"quantization_config": {"quant_method": " Compressed-Tensors "}})["quantization"]
+ == "compressed-tensors"
+ )
+
+
[email protected]("method", ["awq", "gptq", "AWQ"])
+def test_awq_and_gptq_are_offered_while_the_default_engine_has_no_kernels_for_them(method):
+ assert offer({"quantization_config": {"quant_method": method}})["engines"] == ["vllm", "sglang"]
+
+
+def test_awq_and_gptq_stay_on_the_default_engine_once_it_can_run_them():
+ with patch.object(importlib.util, "find_spec", return_value = object()):
+ assert offer({"quantization_config": {"quant_method": "gptq"}}) is None
+ assert offer({"quantization_config": {"quant_method": "awq"}}) is None
+ assert offer(NVFP4) is not None
+
+
+ "metadata",
+ [
+ {},
+ {"quantization_config": {"quant_method": "bitsandbytes"}},
+ {"quantization_config": {"quant_method": "fp8"}},
+ {"quantization_config": {}},
+ {"quantization_config": "compressed-tensors"},
+ {"quantization_config": {"quant_method": None}},
+ {"text_config": "x"},
+ None,
+ [],
+ ],
+)
+def test_nothing_is_offered_for_anything_else(metadata):
+ assert offer(metadata) is None
+
+
+def test_only_the_engines_this_host_can_run_are_offered():
+ asked = []
+ assert offer(NVFP4, supported = lambda *a: asked.append(a) or False) is None
+ assert asked == [("vllm", "compressed-tensors"), ("sglang", "compressed-tensors")]
+ assert offer(NVFP4, supported = lambda name, method: name == "sglang")["engines"] == ["sglang"]
+
+
+def test_the_gpu_probe_is_only_asked_about_these_checkpoints():
+ asked = []
+ offer(
+ {"quantization_config": {"quant_method": "bitsandbytes"}},
+ supported = lambda *a: asked.append(a),
+ )
+ offer(NVFP4, engine = "vllm", supported = lambda *a: asked.append(a))
+ assert asked == []
+
+
+ "overrides",
+ [
+ {"engine": "vllm"},
+ {"engine": "sglang"},
+ {"is_gguf": True},
+ {"is_lora": True},
+ {"is_audio": True},
+ ],
+)
+def test_nothing_is_offered_for_an_explicit_engine_a_gguf_an_adapter_or_audio(overrides):
+ assert offer(NVFP4, **overrides) is None
+
+
+def _config(tmp_path, metadata, **extra):
+ (tmp_path / "config.json").write_text(json.dumps(metadata), encoding = "utf-8")
+ return SimpleNamespace(is_local = True, path = str(tmp_path), identifier = "local/model", **extra)
+
+
+def _gpus(*capabilities):
+ return patch(
+ "core.inference.engine_install._driver_rows",
+ return_value = [["580.65", cap] for cap in capabilities],
+ )
+
+
+def test_the_check_reads_the_checkpoints_own_config(tmp_path):
+ with patch("core.inference.engine_install.support_reason", return_value = None), _gpus("10.0"):
+ assert route._managed_engine_offer_for(_config(tmp_path, NVFP4), None) == {
+ "quantization": "compressed-tensors",
+ "engines": ["vllm", "sglang"],
+ }
+ assert route._managed_engine_offer_for(_config(tmp_path, NVFP4, is_lora = True), None) is None
+
+
+ "reason",
+ [
+ "Checking for a supported NVIDIA GPU.",
+ "Managed engines currently require Linux x86_64 or Windows x64.",
+ "Requires an NVIDIA GPU with compute capability 8.0 or newer and driver 580 or newer.",
+ ],
+)
+def test_the_check_follows_the_engines_own_support_verdict(tmp_path, reason):
+ with patch("core.inference.engine_install.support_reason", return_value = reason):
+ assert route._managed_engine_offer_for(_config(tmp_path, NVFP4), None) is None
+
+
+def test_the_check_never_fails_validation(tmp_path):
+ missing = SimpleNamespace(is_local = True, path = str(tmp_path / "nope"), identifier = "x")
+ assert route._managed_engine_offer_for(missing, None) is None
+ (tmp_path / "config.json").write_text("{not json", encoding = "utf-8")
+ assert (
+ route._managed_engine_offer_for(
+ SimpleNamespace(is_local = True, path = str(tmp_path), identifier = "x"), None
+ )
+ is None
+ )
+ with patch("core.inference.engine_install.support_reason", side_effect = RuntimeError("probe")):
+ assert route._managed_engine_offer_for(_config(tmp_path, NVFP4), None) is None
+
+
+def test_the_response_carries_the_offer_and_defaults_to_none():
+ base = {"valid": True, "message": "ok"}
+ assert ValidateModelResponse(**base).managed_engine_offer is None
+ response = ValidateModelResponse(
+ **base, managed_engine_offer = {"quantization": "compressed-tensors", "engines": ["vllm"]}
+ )
+ assert response.model_dump()["managed_engine_offer"] == {
+ "quantization": "compressed-tensors",
+ "engines": ["vllm"],
+ }
+
+
+def test_sglang_is_offered_compressed_tensors_only_on_blackwell(tmp_path):
+ awq = {"quantization_config": {"quant_method": "awq"}}
+ with patch("core.inference.engine_install.support_reason", return_value = None):
+ with _gpus("9.0", "9.0"):
+ assert route._managed_engine_offer_for(_config(tmp_path, NVFP4), None)["engines"] == [
+ "vllm"
+ ]
+ assert route._managed_engine_offer_for(_config(tmp_path, awq), None)["engines"] == [
+ "vllm",
+ "sglang",
+ ]
+ with _gpus("8.0", "12.0"):
+ assert route._managed_engine_offer_for(_config(tmp_path, NVFP4), None)["engines"] == [
+ "vllm",
+ "sglang",
+ ]
+ with _gpus("bad"):
+ assert route._managed_engine_offer_for(_config(tmp_path, NVFP4), None)["engines"] == [
+ "vllm"
+ ]
+
+
+def test_audio_checkpoints_get_no_offer(tmp_path):
+ with patch("core.inference.engine_install.support_reason", return_value = None), _gpus("10.0"):
+ assert (
+ route._managed_engine_offer_for(_config(tmp_path, NVFP4, is_audio = True), None) is None
+ )
+
+
+def test_a_cached_hub_config_is_read_without_a_hub_round_trip(tmp_path):
+ (tmp_path / "config.json").write_text(json.dumps(NVFP4), encoding = "utf-8")
+ remote = SimpleNamespace(is_local = False, path = None, identifier = "unsloth/Qwen3.8-27B-NVFP4")
+ with (
+ patch("core.inference.engine_install.support_reason", return_value = None),
+ _gpus("10.0"),
+ patch("huggingface_hub.try_to_load_from_cache", return_value = str(tmp_path / "config.json")),
+ patch("huggingface_hub.hf_hub_download", side_effect = AssertionError("network")),
+ ):
+ assert route._managed_engine_offer_for(remote, None)["engines"] == ["vllm", "sglang"]
diff --git a/studio/frontend/src/app/routes/__root.tsx b/studio/frontend/src/app/routes/__root.tsx
index 509d7bb9c4f..333d57e5b3a 100644
--- a/studio/frontend/src/app/routes/__root.tsx
+++ b/studio/frontend/src/app/routes/__root.tsx
@@ -35,7 +35,10 @@ import { FIND_SCOPE_ATTRIBUTE, FindInPage } from "@/features/find-in-page";
import { HfTokenWarningDialog } from "@/features/hf-auth";
import { InterfaceZoom, zoomInterfaceFromMenu } from "@/features/interface-zoom";
import { bootstrapPersistedCredentials } from "@/features/credentials/bootstrap";
-import { SharedRunConfigLinkHandler } from "@/features/model-picker";
+import {
+ ManagedEngineOfferDialog,
+ SharedRunConfigLinkHandler,
+} from "@/features/model-picker";
import { backfillModelOverrides } from "@/features/model-picker/api/migrate-model-overrides";
import { hydratePins } from "@/lib/pins-mirror";
import { usePersonalizationSync } from "@/features/profile";
@@ -685,6 +688,7 @@ function RootLayout() {
<RemoteCodeConsentDialog />
<DownloadApprovalDialog />
<TransformersUpgradeDialog />
+ <ManagedEngineOfferDialog />
<LlmCompressorConsentDialog />
{/* At the root, not under /chat: a swap can start from the Hub too. */}
<StopRunningChatsDialog />
diff --git a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts
index 85a3ae8d24e..fd90146481c 100644
--- a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts
+++ b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts
@@ -167,6 +167,7 @@ import {
DEFAULT_MAX_SEQ_LENGTH,
DEFAULT_PER_MODEL_CONFIG,
applyPerModelConfigToRuntime,
+ confirmManagedEngineIfNeeded,
currentRuntimePerModelConfig,
isServedByMlx,
normalizeMaxSeqLength,
@@ -1809,9 +1810,9 @@ export function useChatModelRuntime() {
const { keepModelsLoaded, loadedModels: loadedNow, params: paramsNow } =
useChatRuntimeStore.getState();
// vLLM and SGLang always replace the loaded models, so they keep the running-chat prompt.
- const keepsOthers =
+ let keepsOthers =
keepModelsLoaded && !forceReload && (paramsNow.engine ?? "auto") === "auto";
- const switchingNote = keepsOthers ? "Keeping the loaded models." : "Switching models.";
+ let switchingNote = keepsOthers ? "Keeping the loaded models." : "Switching models.";
// Reloading one of several touches only its own slot, so only its chats stop.
const touchesOnlySelected =
forceReload && !isExternalModelId(paramsNow.checkpoint) && loadedNow.length > 1;
@@ -1862,7 +1863,7 @@ export function useChatModelRuntime() {
releasePreflightLifecycleLease();
throw error;
}
- const forceCancelActive = stopDecision.forceCancelActive;
+ let forceCancelActive = stopDecision.forceCancelActive;
const explicitIsLora =
typeof selection === "string" ? undefined : selection.isLora;
@@ -2009,6 +2010,8 @@ export function useChatModelRuntime() {
(typeof selection !== "string" ? selection.config?.engine : undefined) ??
useChatRuntimeStore.getState().params.engine ?? "auto";
const managedLoad = !isGguf && requestedEngine !== "auto";
+ // Set when the Default engine cannot run the checkpoint and the user picked vLLM / SGLang (#11728).
+ let engineSwitched = false;
let downloadComplete = isDownloaded || isCachedLora || managedLoad;
let cpuFallbackReason: CpuFallbackReason | null = null;
let mmprojFallbackReason: MmprojFallbackReason | null = null;
@@ -2273,6 +2276,11 @@ export function useChatModelRuntime() {
stateBeforeUnload.ctxCheckpoints,
cacheRam: pendingLoadConfig?.cacheRam ?? stateBeforeUnload.cacheRam,
};
+ let loadEngineFields = {
+ engine: stateBeforeUnload.params.engine ?? "auto",
+ engine_precision: stateBeforeUnload.params.enginePrecision ?? "auto",
+ engine_parallelism: stateBeforeUnload.params.engineParallelism ?? "tensor",
+ };
try {
// Lightweight pre-flight validation: avoid unloading a working model if the new identifier is
// clearly invalid.
@@ -2398,6 +2406,78 @@ export function useChatModelRuntime() {
// validateModel cannot be aborted either, and everything below writes shared
// state, so stop here if a replacement superseded this load meanwhile.
if (abortCtrl.signal.aborted) throw new Error("Cancelled");
+ const engineOffer =
+ !isGguf && loadEngineFields.engine === "auto"
+ ? validation.managed_engine_offer
+ : null;
+ if (engineOffer) {
+ const engine = await confirmManagedEngineIfNeeded(
+ modelId,
+ engineOffer,
+ abortCtrl.signal,
+ );
+ if (abortCtrl.signal.aborted) throw new Error("Cancelled");
+ if (!engine) {
+ throw new Error(
+ `${displayName} is quantized with ${engineOffer.quantization}, which the default engine cannot run. Set Inference engine to ${engineOffer.engines[0] === "sglang" ? "SGLang" : "vLLM"} in its run settings to load it.`,
+ );
+ }
+ loadEngineFields = {
+ engine,
+ engine_precision: "auto",
+ engine_parallelism: "tensor",
+ };
+ engineSwitched = true;
+ downloadComplete = true;
+ if (keepsOthers) {
+ // vLLM and SGLang replace every loaded model, so ask as any replacing load does.
+ stopDecision = await confirmStopRunningChatsIfNeeded(
+ "Loading a different model",
+ "reload",
+ );
+ if (abortCtrl.signal.aborted) throw new Error("Cancelled");
+ if (!stopDecision.proceed) {
+ // Declined before anything was unloaded: exit as a cancellation, not a failed load.
+ if (
+ modelSelectionIntentEpoch === loadIntentId &&
+ pendingReplacementRollback?.residentUnloaded === false
+ ) {
+ pendingReplacementRollback = null;
+ }
+ resetLoadingUiForRun(loadRun);
+ abortCtrl.abort();
+ throw new Error("Cancelled");
+ }
+ keepsOthers = false;
+ forceCancelActive = stopDecision.forceCancelActive;
+ loadRun.forceCancelActive = forceCancelActive;
+ loadingDescription = loadingDescription.replace(switchingNote, "Switching models.");
+ switchingNote = "Switching models.";
+ }
+ // Re-validate as that engine before unloading; a spent Desktop path lease relies on /load.
+ if (!nativePathToken) {
+ Object.assign(
+ validation,
+ await validateModel(
+ {
+ model_path: loadPath,
+ ...loadEngineFields,
+ hf_token: hfToken,
+ max_seq_length: validateMaxSeqLength,
+ load_in_4bit: false,
+ is_lora: isLora,
+ gguf_variant: ggufVariant ?? null,
+ cache_type_kv: loadKvCacheDtype,
+ tensor_parallel: loadTensorParallel,
+ disable_vision: loadDisableVision,
+ gpu_ids: validateGpuIds ?? undefined,
+ },
+ { signal: abortCtrl.signal },
+ ),
+ );
+ if (abortCtrl.signal.aborted) throw new Error("Cancelled");
+ }
+ }
if (validation.mlx_loads_base_model) {
mlxLoadProgress = true;
const mlxBaseDescription = isLora
@@ -2647,9 +2727,8 @@ export function useChatModelRuntime() {
const loadResponse = await loadModel({
model_path: loadPath,
- engine_precision: stateBeforeUnload.params.enginePrecision ?? "auto",
- engine_parallelism: stateBeforeUnload.params.engineParallelism ?? "tensor",
- engine: isGguf ? "auto" : (stateBeforeUnload.params.engine ?? "auto"),
+ ...loadEngineFields,
+ engine: isGguf ? "auto" : loadEngineFields.engine,
load_request_id: loadRun.requestId,
nativePathLease: loadNativePathLease,
hf_token: hfToken,
@@ -2659,7 +2738,7 @@ export function useChatModelRuntime() {
loadCustomContextLength,
loadMaxSeqLength,
),
- load_in_4bit: (stateBeforeUnload.params.engine ?? "auto") === "auto",
+ load_in_4bit: loadEngineFields.engine === "auto",
is_lora: isLora,
gguf_variant: ggufVariant ?? null,
trust_remote_code: trustRemoteCode,
@@ -3465,7 +3544,7 @@ export function useChatModelRuntime() {
if (progressInterval) clearInterval(progressInterval);
return;
}
- if (managedLoad && prog.bytes_total <= 0) {
+ if ((managedLoad || engineSwitched) && prog.bytes_total <= 0) {
const label = prog.phase === "warming_up"
? "Warming up inference kernels. The first load can take several minutes."
: prog.phase === "loading_weights"
diff --git a/studio/frontend/src/features/chat/types/api.ts b/studio/frontend/src/features/chat/types/api.ts
index da4f0fe96c1..0e30f98736a 100644
--- a/studio/frontend/src/features/chat/types/api.ts
+++ b/studio/frontend/src/features/chat/types/api.ts
@@ -2,6 +2,7 @@
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
import type { TransformersUpgradeInfo } from "@/features/transformers-upgrade";
+import type { ManagedEngineOffer } from "@/features/model-picker";
import type { CustomReasoningConfig } from "../custom-reasoning";
export type CpuFallbackReason = "vulkan_startup_crash";
@@ -167,6 +168,8 @@ export interface ValidateModelResponse {
transformers_upgrade?: TransformersUpgradeInfo | null;
/** Replacement repository for an MLX BNB model or adapter base. */
mlx_loads_base_model?: string | null;
+ /** The Default engine cannot run this quantization; these optional engines can on this host. */
+ managed_engine_offer?: ManagedEngineOffer | null;
}
export interface GgufVariantDetail {
diff --git a/studio/frontend/src/features/model-picker/components/inference-engines.tsx b/studio/frontend/src/features/model-picker/components/inference-engines.tsx
index 490de0f3a56..50404213764 100644
--- a/studio/frontend/src/features/model-picker/components/inference-engines.tsx
+++ b/studio/frontend/src/features/model-picker/components/inference-engines.tsx
@@ -30,7 +30,7 @@ import { useEngines } from "../hooks/use-engines";
const names = { vllm: "vLLM", sglang: "SGLang" };
-function EngineInstall({
+export function EngineInstall({
engine,
management = false,
onUse,
diff --git a/studio/frontend/src/features/model-picker/components/managed-engine-offer-dialog.tsx b/studio/frontend/src/features/model-picker/components/managed-engine-offer-dialog.tsx
new file mode 100644
index 00000000000..e5831458f20
--- /dev/null
+++ b/studio/frontend/src/features/model-picker/components/managed-engine-offer-dialog.tsx
@@ -0,0 +1,120 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import {
+ AlertDialog,
+ AlertDialogAction,
+ AlertDialogCancel,
+ AlertDialogContent,
+ AlertDialogDescription,
+ AlertDialogFooter,
+ AlertDialogHeader,
+ AlertDialogTitle,
+} from "@/components/ui/alert-dialog";
+import { Spinner } from "@/components/ui/spinner";
+import { useState } from "react";
+import { isEngineReady } from "../api/engines";
+import { useManagedEngineOfferStore } from "../hooks/managed-engine-offer";
+import { useEngines } from "../hooks/use-engines";
+import { EngineInstall } from "./inference-engines";
+
+const ENGINE_NAMES = { vllm: "vLLM", sglang: "SGLang" };
+const QUANTIZATION_NAMES: Record<string, string> = {
+ "compressed-tensors": "compressed-tensors",
+ awq: "AWQ",
+ gptq: "GPTQ",
+};
+
+/** Root-mounted: a Default-engine load only vLLM / SGLang can run waits here for an engine. */
+export function ManagedEngineOfferDialog() {
+ const open = useManagedEngineOfferStore((s) => s.open);
+ const modelName = useManagedEngineOfferStore((s) => s.modelName);
+ const offer = useManagedEngineOfferStore((s) => s.offer);
+ const resolve = useManagedEngineOfferStore((s) => s.resolve);
+ const { engines, error } = useEngines(open);
+ const offered = (offer?.engines ?? []).flatMap((name) => {
+ const engine = engines.find((row) => row.engine === name);
+ return engine ? [engine] : [];
+ });
+ const ready = offered.filter((engine) => isEngineReady(engine));
+ // Install sections stay mounted while open, so even an install that finishes between polls
+ // still reaches its success effect and hands the load to that engine.
+ const [shown, setShown] = useState<readonly string[]>([]);
+ const unshown = offered
+ .filter((engine) => !shown.includes(engine.engine))
+ .map((engine) => engine.engine);
+ if (!open && shown.length > 0) setShown([]);
+ else if (open && ready.length === 0 && unshown.length > 0)
+ setShown([...shown, ...unshown]);
+ const names = (offer?.engines ?? []).map((name) => ENGINE_NAMES[name]);
+ const displayName = modelName?.split("/").pop() || "This model";
+ const quantization =
+ QUANTIZATION_NAMES[offer?.quantization ?? ""] ?? offer?.quantization;
+
+ return (
+ <AlertDialog
+ open={open}
+ onOpenChange={(next) => {
+ if (!next) resolve(null);
+ }}
+ >
+ <AlertDialogContent className="max-w-lg">
+ <AlertDialogHeader className="min-w-0">
+ <AlertDialogTitle>
+ {displayName} needs {names.join(" or ")}
+ </AlertDialogTitle>
+ <AlertDialogDescription>
+ This model is quantized with {quantization}, which Unsloth's default
+ engine cannot run. {names.join(" and ")}{" "}
+ {names.length > 1 ? "can each" : "can"} run it on this computer
+ {ready.length > 0 ? "." : " once installed."}
+ </AlertDialogDescription>
+ </AlertDialogHeader>
+ {offered.length === 0 ? (
+ error ? (
+ // The check keeps retrying; say why it is stuck rather than spinning silently.
+ <p role="alert" className="text-ui-12 text-destructive">
+ Could not check installed engines: {error}
+ </p>
+ ) : (
+ <p className="flex items-center gap-2 text-ui-12 text-muted-foreground">
+ <Spinner className="size-3.5" />
+ Checking installed engines...
+ </p>
+ )
+ ) : (
+ offered
+ .filter(
+ (engine) => ready.length === 0 || shown.includes(engine.engine),
+ )
+ .map((engine) => (
+ <section
+ key={engine.engine}
+ aria-label={ENGINE_NAMES[engine.engine]}
+ className="space-y-2 rounded-lg border p-3"
+ >
+ <p className="text-ui-13 font-medium">
+ {ENGINE_NAMES[engine.engine]}
+ </p>
+ <EngineInstall
+ engine={engine}
+ onUse={() => resolve(engine.engine)}
+ />
+ </section>
+ ))
+ )}
+ <AlertDialogFooter>
+ <AlertDialogCancel>Cancel</AlertDialogCancel>
+ {ready.map((engine) => (
+ <AlertDialogAction
+ key={engine.engine}
+ onClick={() => resolve(engine.engine)}
+ >
+ Load with {ENGINE_NAMES[engine.engine]}
+ </AlertDialogAction>
+ ))}
+ </AlertDialogFooter>
+ </AlertDialogContent>
+ </AlertDialog>
+ );
+}
diff --git a/studio/frontend/src/features/model-picker/hooks/managed-engine-offer.ts b/studio/frontend/src/features/model-picker/hooks/managed-engine-offer.ts
new file mode 100644
index 00000000000..d8da5b2ba89
--- /dev/null
+++ b/studio/frontend/src/features/model-picker/hooks/managed-engine-offer.ts
@@ -0,0 +1,89 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import { create } from "zustand";
+
+export type ManagedEngine = "vllm" | "sglang";
+
+/** /validate's managed_engine_offer: the Default engine cannot run this quantization (#11728). */
+export interface ManagedEngineOffer {
+ quantization: string;
+ engines: ManagedEngine[];
+}
+
+type Resolver = (engine: ManagedEngine | null) => void;
+
+// One open offer; a newer request declines the older one.
+let pendingResolver: Resolver | null = null;
+
+interface ManagedEngineOfferStore {
+ open: boolean;
+ modelName: string | null;
+ offer: ManagedEngineOffer | null;
+ request: (
+ modelName: string,
+ offer: ManagedEngineOffer,
+ signal?: AbortSignal,
+ ) => Promise<ManagedEngine | null>;
+ resolve: (engine: ManagedEngine | null) => void;
+}
+
+export const useManagedEngineOfferStore = create<ManagedEngineOfferStore>()(
+ (set) => ({
+ open: false,
+ modelName: null,
+ offer: null,
+ request: (modelName, offer, signal) =>
+ new Promise<ManagedEngine | null>((resolve) => {
+ pendingResolver?.(null);
+ if (signal?.aborted) {
+ pendingResolver = null;
+ set({ open: false });
+ resolve(null);
+ return;
+ }
+ const settle: Resolver = (engine) => {
+ signal?.removeEventListener("abort", onAbort);
+ resolve(engine);
+ };
+ const onAbort = () => {
+ if (pendingResolver !== settle) return;
+ pendingResolver = null;
+ set({ open: false });
+ settle(null);
+ };
+ signal?.addEventListener("abort", onAbort, { once: true });
+ pendingResolver = settle;
+ set({ open: true, modelName, offer });
+ }),
+ resolve: (engine) => {
+ const resolver = pendingResolver;
+ pendingResolver = null;
+ set({ open: false });
+ resolver?.(engine);
+ },
+ }),
+);
+
+/** The engines of an offer this build can load with, in the backend's order. */
+export function offeredEngines(
+ offer: ManagedEngineOffer | null | undefined,
+): ManagedEngine[] {
+ return (offer?.engines ?? []).filter(
+ (engine): engine is ManagedEngine =>
+ engine === "vllm" || engine === "sglang",
+ );
+}
+
+/** Pause a Default-engine load on the offer dialog; resolves the engine to load with, or null. */
+export async function confirmManagedEngineIfNeeded(
+ modelName: string,
+ offer: ManagedEngineOffer | null | undefined,
+ signal?: AbortSignal,
+): Promise<ManagedEngine | null> {
+ const engines = offeredEngines(offer);
+ if (!offer || engines.length === 0) return null;
+ return useManagedEngineOfferStore
+ .getState()
+ .request(modelName, { ...offer, engines }, signal);
+}
diff --git a/studio/frontend/src/features/model-picker/index.ts b/studio/frontend/src/features/model-picker/index.ts
index 5b9626cda1e..496248ce255 100644
--- a/studio/frontend/src/features/model-picker/index.ts
+++ b/studio/frontend/src/features/model-picker/index.ts
@@ -37,6 +37,11 @@ export {
} from "./api/model-overrides";
export { useActiveModelConfig } from "./hooks/use-active-model-config";
export { useVllmAvailable } from "./hooks/use-vllm-available";
+export { ManagedEngineOfferDialog } from "./components/managed-engine-offer-dialog";
+export {
+ confirmManagedEngineIfNeeded,
+ type ManagedEngineOffer,
+} from "./hooks/managed-engine-offer";
export type {
DeletedModelRef,
ExternalConnectionRef,
diff --git a/studio/frontend/tests/chat-model-load-replacement.test.ts b/studio/frontend/tests/chat-model-load-replacement.test.ts
index 1346212a300..2d9bdcece17 100644
--- a/studio/frontend/tests/chat-model-load-replacement.test.ts
+++ b/studio/frontend/tests/chat-model-load-replacement.test.ts
@@ -47,7 +47,7 @@ const blocks = {
loadGate: section(runtime, "async function performLoad(): Promise<void> {", "const pendingLoadConfig ="),
stagedMetadata: section(runtime, "if (isGguf && isDiffusion === undefined) {", "const targetIsDiffusion = isDiffusion === true;"),
validation: section(runtime, "const validation = await validateModel({", "if (validation.mlx_loads_base_model) {"),
- recheck: section(runtime, "// Re-check the tracked picker for a load that was already starting", "const forceCancelActive = stopDecision.forceCancelActive;"),
+ recheck: section(runtime, "// Re-check the tracked picker for a load that was already starting", "let forceCancelActive = stopDecision.forceCancelActive;"),
terminal: section(runtime, "await performLoad();", "// Last act of this run's coroutine"),
failedCancel: section(section(runtime, "const cancelLoadRun = useCallback(", "const cancelLoadingWithCheckpointPolicy = useCallback("), "// The request failed, so reconcile against the backend before releasing the", "return false;"),
cancelReconcile: section(section(runtime, "const cancelLoadRun = useCallback(", "const cancelLoadingWithCheckpointPolicy = useCallback("), "if (\n (!run.loadAttemptPath", "activeLoadRunRef.current = releaseOwnedModelLoadRun("),
diff --git a/studio/frontend/tests/managed-engine-offer.test.ts b/studio/frontend/tests/managed-engine-offer.test.ts
new file mode 100644
index 00000000000..ec63c91c542
--- /dev/null
+++ b/studio/frontend/tests/managed-engine-offer.test.ts
@@ -0,0 +1,97 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+import assert from "node:assert/strict";
+import test from "node:test";
+import {
+ confirmManagedEngineIfNeeded,
+ offeredEngines,
+ useManagedEngineOfferStore,
+} from "../src/features/model-picker/hooks/managed-engine-offer.ts";
+import { readText } from "./helpers/kit.ts";
+
+// #11728: a Default-engine load of a compressed-tensors / AWQ / GPTQ checkpoint pauses on this offer.
+const OFFER = { quantization: "compressed-tensors", engines: ["vllm" as const] };
+const store = () => useManagedEngineOfferStore.getState();
+
+test("no offer, or an offer naming no engine this build knows, never opens the dialog", async () => {
+ assert.equal(await confirmManagedEngineIfNeeded("m", null), null);
+ assert.equal(await confirmManagedEngineIfNeeded("m", { quantization: "awq", engines: [] }), null);
+ const unknown = { quantization: "awq", engines: ["tgi"] } as unknown as typeof OFFER;
+ assert.equal(await confirmManagedEngineIfNeeded("m", unknown), null);
+ assert.equal(store().open, false);
+ assert.deepEqual(offeredEngines({ quantization: "gptq", engines: ["sglang", "vllm"] }), ["sglang", "vllm"]);
+});
+
+test("choosing the engine resumes the load with it; declining resumes with null", async () => {
+ const chosen = confirmManagedEngineIfNeeded("unsloth/Qwen3.8-27B-NVFP4", OFFER);
+ assert.equal(store().open, true);
+ assert.equal(store().modelName, "unsloth/Qwen3.8-27B-NVFP4");
+ assert.deepEqual(store().offer, OFFER);
+ store().resolve("vllm");
+ assert.equal(await chosen, "vllm");
+ assert.equal(store().open, false);
+
+ const declined = confirmManagedEngineIfNeeded("m", OFFER);
+ store().resolve(null);
+ assert.equal(await declined, null);
+ store().resolve("vllm");
+});
+
+test("a newer load declines the older one, so one click never resumes two loads", async () => {
+ const first = confirmManagedEngineIfNeeded("first", OFFER);
+ const second = confirmManagedEngineIfNeeded("second", OFFER);
+ assert.equal(await first, null);
+ assert.equal(store().modelName, "second");
+ store().resolve("vllm");
+ assert.equal(await second, "vllm");
+});
+
+test("a cancelled load closes its dialog and an already-cancelled one never opens it", async () => {
+ const controller = new AbortController();
+ const pending = confirmManagedEngineIfNeeded("m", OFFER, controller.signal);
+ assert.equal(store().open, true);
+ controller.abort();
+ assert.equal(await pending, null);
+ assert.equal(store().open, false);
+ assert.equal(await confirmManagedEngineIfNeeded("m", OFFER, controller.signal), null);
+ assert.equal(store().open, false);
+ const later = new AbortController();
+ const chosen = confirmManagedEngineIfNeeded("m", OFFER, later.signal);
+ store().resolve("vllm");
+ later.abort();
+ assert.equal(await chosen, "vllm");
+});
+
+test("the chat load asks before unloading anything and loads with the chosen engine", () => {
+ const source = readText("../src/features/chat/hooks/use-chat-model-runtime.ts");
+ const offerAt = source.indexOf("confirmManagedEngineIfNeeded(");
+ const unloadAt = source.indexOf("await unloadModel({ model_path: currentCheckpoint })");
+ assert.ok(offerAt > 0 && unloadAt > offerAt, "the offer must come before the preliminary unload");
+ assert.match(source, /loadModel\(\{\s*model_path: loadPath,\s*\.\.\.loadEngineFields,/);
+ const revalidateAt = source.indexOf("...loadEngineFields,", offerAt);
+ assert.ok(revalidateAt > offerAt && revalidateAt < unloadAt, "the chosen engine must be validated before the unload");
+ // Keep loaded models: the switch re-asks about running chats and then replaces, like any managed load.
+ const branch = source.slice(offerAt, unloadAt);
+ assert.match(branch, /if \(keepsOthers\) \{[\s\S]*?stopDecision = await confirmStopRunningChatsIfNeeded\([\s\S]*?keepsOthers = false;\s*forceCancelActive = stopDecision\.forceCancelActive;\s*loadRun\.forceCancelActive = forceCancelActive;/);
+ assert.match(readText("../src/app/routes/__root.tsx"), /<ManagedEngineOfferDialog \/>/);
+ // Declining the replacement prompt this engine needs is a cancellation, and the toast stops promising kept models.
+ const offerBranch = source.slice(offerAt, unloadAt);
+ assert.match(offerBranch, /abortCtrl\.abort\(\);\s*throw new Error\("Cancelled"\);/);
+ assert.match(offerBranch, /\},\s*\{ signal: abortCtrl\.signal \},\s*\),/);
+ assert.match(offerBranch, /loadingDescription = loadingDescription\.replace\(switchingNote, "Switching models\."\);/);
+});
+
+test("the offer dialog says when it cannot read the installed engines", () => {
+ const dialog = readText("../src/features/model-picker/components/managed-engine-offer-dialog.tsx");
+ assert.match(dialog, /const \{ engines, error \} = useEngines\(open\);/);
+ assert.match(dialog, /Could not check installed engines: \{error\}/);
+});
+
+test("an install started in the dialog stays mounted until it hands the load over", () => {
+ // Unmounting on the first ready poll would drop EngineInstall's success effect, and with it onUse,
+ // even for an install that finished between two polls.
+ const dialog = readText("../src/features/model-picker/components/managed-engine-offer-dialog.tsx");
+ assert.match(dialog, /ready\.length === 0 \|\| shown\.includes\(engine\.engine\)/);
+ assert.match(dialog, /open && ready\.length === 0 && unshown\.length > 0/);
+ assert.match(dialog, /onUse=\{\(\) => resolve\(engine\.engine\)\}/);
+});
diff --git a/studio/frontend/tests/resident-config-match.test.ts b/studio/frontend/tests/resident-config-match.test.ts
index e584400749f..60a1943592f 100644
--- a/studio/frontend/tests/resident-config-match.test.ts
+++ b/studio/frontend/tests/resident-config-match.test.ts
@@ -1994,7 +1994,7 @@ test("a pick asks the status about its own model and keeps or replaces the other
assert.equal(USE_CHAT_MODEL_RUNTIME.match(/await readPickStatus\(\)/g)?.length, 2);
assert.match(
USE_CHAT_MODEL_RUNTIME,
- /const keepsOthers =\s*keepModelsLoaded && !forceReload && \(paramsNow\.engine \?\? "auto"\) === "auto";[\s\S]*?const touchesOnlySelected =\s*forceReload && !isExternalModelId\(paramsNow\.checkpoint\) && loadedNow\.length > 1;/,
+ /let keepsOthers =\s*keepModelsLoaded && !forceReload && \(paramsNow\.engine \?\? "auto"\) === "auto";[\s\S]*?const touchesOnlySelected =\s*forceReload && !isExternalModelId\(paramsNow\.checkpoint\) && loadedNow\.length > 1;/,
);
assert.match(USE_CHAT_MODEL_RUNTIME, /touchesOnlySelected \? \(paramsNow\.checkpoint \?\? undefined\) : undefined,/);
assert.match(USE_CHAT_MODEL_RUNTIME, /stopQueuedRuns\(stopDecision, keepsOthers \|\| touchesOnlySelected\);/);
diff --git a/tests/studio/test_model_picker_contracts.py b/tests/studio/test_model_picker_contracts.py
index b00f764762b..c866f69074a 100644
--- a/tests/studio/test_model_picker_contracts.py
+++ b/tests/studio/test_model_picker_contracts.py
@@ -801,8 +801,8 @@ def test_a_pinned_cached_row_loads_from_the_id_the_backend_pinned():
assert (
'(typeof selection === "string" ? null : selection.loadId) || modelId' in runtime
), "loadPath must fall back to the id, so an unpinned pick is unchanged"
- # Staged metadata, validate and load: all three read the copy that loads.
- assert runtime.count("model_path: loadPath,") == 3
+ # Staged metadata, validate, the engine-switch revalidate and load all read the copy that loads.
+ assert runtime.count("model_path: loadPath,") == 4
assert "model_path: modelId," not in runtime
# A rollback reads the approval under the snapshot path, so store it under both keys.
assert "rememberApprovedRemoteCode(loadPath, approvedRemoteCodeFingerprint);" in runtime