Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: offer vLLM or SGLang when the default engine cannot run a quantized checkpoint (#13135)
Daniel Hanbb478c67fe36
diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.pyindex 70f2ef69016..57b130cab47 100644--- a/studio/backend/models/inference.py+++ b/studio/backend/models/inference.py@@ -758,6 +758,15 @@ class ValidateModelRequest(BaseModel):)+class ManagedEngineOffer(BaseModel):+ """Optional engines that can run a checkpoint the Default engine cannot."""++ quantization: str = Field(..., description = "quant_method from the checkpoint's config.json")+ engines: List[Literal["vllm", "sglang"]] = Field(+ ..., description = "Engines this host can run, in the order to offer them"+ )++class TransformersUpgradeInfo(BaseModel):"""A model architecture no installed transformers ships, but a newer release does."""@@ -961,6 +970,11 @@ class ValidateModelResponse(BaseModel):"loaded in place of the requested unsloth bnb-4bit repo (or of a LoRA's bnb base), ""because MLX cannot read bitsandbytes weights. None when the pick loads as asked.",)+ managed_engine_offer: Optional[ManagedEngineOffer] = Field(+ None,+ description = "Set when the Default engine cannot run this checkpoint's quantization "+ "but an optional engine can on this host; the UI offers to load it there.",+ )class EstimateMemoryRequest(BaseModel):diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex 6d3ee5d50b4..2d1277fd6ba 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -16839,6 +16839,96 @@ def _vllm_engine_hint(engine: Optional[str]) -> str:return ""+# Quantizations the Default engine cannot run, with the packages that would let it (Studio ships none).+_MANAGED_ENGINE_QUANTIZATIONS = {+ "compressed-tensors": (),+ "awq": ("gptqmodel", "awq"),+ "gptq": ("gptqmodel", "auto_gptq"),+}+_OFFERED_ENGINES = ("vllm", "sglang")+++def _managed_engine_offer(+ metadata, *, engine, is_gguf, is_lora, is_audio, supported+) -> Optional[dict]:+ """The engines to offer for a checkpoint the Default engine cannot run, or None.++ ``supported(engine, quant_method)`` is asked last, so the GPU probe only runs for these+ checkpoints. GGUF, adapters and audio models are refused by the optional engines."""+ if (+ engine not in (None, "auto")+ or is_gguf+ or is_lora+ or is_audio+ or not isinstance(metadata, dict)+ ):+ return None+ text_config = metadata.get("text_config")+ quant = metadata.get("quantization_config") or (+ text_config.get("quantization_config") if isinstance(text_config, dict) else None+ )+ method = quant.get("quant_method") if isinstance(quant, dict) else None+ if not isinstance(method, str):+ return None+ method = method.strip().lower()+ if method not in _MANAGED_ENGINE_QUANTIZATIONS:+ return None+ import importlib.util++ for module in _MANAGED_ENGINE_QUANTIZATIONS[method]:+ try:+ if importlib.util.find_spec(module) is not None:+ return None+ except (ImportError, ValueError):+ pass+ engines = [name for name in _OFFERED_ENGINES if supported(name, method)]+ return {"quantization": method, "engines": engines} if engines else None+++def _managed_engine_offer_for(config, hf_token) -> Optional[dict]:+ """``_managed_engine_offer`` from the checkpoint's own config.json; None on any failure."""+ try:+ if config.is_local:+ path = Path(config.path) / "config.json"+ else:+ from huggingface_hub import hf_hub_download, try_to_load_from_cache++ # Resolving the model already cached config.json; the Hub round trip costs ~45 ms per pick.+ cached = try_to_load_from_cache(config.identifier, "config.json")+ path = Path(+ cached+ if isinstance(cached, str)+ else hf_hub_download(config.identifier, "config.json", token = hf_token)+ )+ metadata = json.loads(path.read_text(encoding = "utf-8"))+ from core.inference.engine_install import _driver_rows, support_reason++ def supported(name, method):+ if support_reason(name) is not None:+ return False+ if name != "sglang" or method != "compressed-tensors":+ return True+ # SGLang 0.5.20's compressed-tensors NVFP4 needs SM 10.0; vLLM falls back to Marlin from 7.5.+ try:+ return any(+ float(row[1]) >= 10.0 for row in _driver_rows(None) or () if len(row) == 2+ )+ except ValueError:+ return False++ return _managed_engine_offer(+ metadata,+ engine = "auto",+ is_gguf = getattr(config, "is_gguf", False),+ is_lora = getattr(config, "is_lora", False),+ is_audio = getattr(config, "is_audio", False),+ supported = supported,+ )+ except Exception as exc:+ logger.debug("Managed engine offer check failed for '%s': %s", config.identifier, exc)+ return None++def _diagnosis_text(msg: str) -> str:"""``msg`` up to the startup-diagnostics block, which is not ours to read.@@ -20122,6 +20212,16 @@ async def validate_model(except Exception as e:logger.debug("Header probe failed for %s: %s", model_log_label, e)+ managed_engine_offer = None+ if request.engine == "auto" and not is_gguf:+ managed_engine_offer = await asyncio.to_thread(+ _offline_guarded,+ (model_identifier, config.identifier, getattr(config, "base_model", None)),+ _managed_engine_offer_for,+ config,+ request.hf_token,+ )+return restore_inventory_handles(ValidateModelResponse(valid = True,@@ -20155,6 +20255,7 @@ async def validate_model(requires_transformers_upgrade = transformers_upgrade is not None,transformers_upgrade = transformers_upgrade,mlx_loads_base_model = await asyncio.to_thread(_mlx_base_for_config, config),+ managed_engine_offer = managed_engine_offer,))diff --git a/studio/backend/tests/test_managed_engine_offer.py b/studio/backend/tests/test_managed_engine_offer.pynew file mode 100644index 00000000000..de18a08624e--- /dev/null+++ b/studio/backend/tests/test_managed_engine_offer.py@@ -0,0 +1,236 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""/validate offers vLLM for checkpoints the Default engine cannot run (#11728).++Only where the host can run vLLM, only for the Default engine, and only when the+checkpoint's own config.json says compressed-tensors, or AWQ / GPTQ without the+packages the Default engine would need for them.+"""++import importlib.util+import json+from pathlib import Path+from types import SimpleNamespace+from unittest.mock import patch++import pytest++from models.inference import ValidateModelResponse++_BACKEND_ROOT = Path(__file__).resolve().parent.parent+++def _route():+ spec = importlib.util.spec_from_file_location(+ "inference_route_managed_engine_offer", _BACKEND_ROOT / "routes/inference.py"+ )+ module = importlib.util.module_from_spec(spec)+ spec.loader.exec_module(module)+ return module+++route = _route()+NVFP4 = {+ "quantization_config": {"quant_method": "compressed-tensors", "format": "nvfp4-pack-quantized"}+}+++def offer(+ metadata,+ supported = lambda name, method: True,+ **overrides,+):+ kwargs = {+ "engine": "auto",+ "is_gguf": False,+ "is_lora": False,+ "is_audio": False,+ "supported": supported,+ }+ kwargs.update(overrides)+ return route._managed_engine_offer(metadata, **kwargs)++[email protected](autouse = True)+def _no_awq_or_gptq_packages():+ real = importlib.util.find_spec+ missing = {"gptqmodel", "awq", "auto_gptq"}+ with patch.object(+ importlib.util,+ "find_spec",+ side_effect = lambda name, *a: None if name in missing else real(name, *a),+ ):+ yield+++def test_a_compressed_tensors_checkpoint_is_offered_vllm():+ assert offer(NVFP4) == {"quantization": "compressed-tensors", "engines": ["vllm", "sglang"]}+ # Vision checkpoints keep it under text_config.+ assert offer({"text_config": NVFP4})["engines"] == ["vllm", "sglang"]+ assert (+ offer({"quantization_config": {"quant_method": " Compressed-Tensors "}})["quantization"]+ == "compressed-tensors"+ )++[email protected]("method", ["awq", "gptq", "AWQ"])+def test_awq_and_gptq_are_offered_while_the_default_engine_has_no_kernels_for_them(method):+ assert offer({"quantization_config": {"quant_method": method}})["engines"] == ["vllm", "sglang"]+++def test_awq_and_gptq_stay_on_the_default_engine_once_it_can_run_them():+ with patch.object(importlib.util, "find_spec", return_value = object()):+ assert offer({"quantization_config": {"quant_method": "gptq"}}) is None+ assert offer({"quantization_config": {"quant_method": "awq"}}) is None+ assert offer(NVFP4) is not None+++ "metadata",+ [+ {},+ {"quantization_config": {"quant_method": "bitsandbytes"}},+ {"quantization_config": {"quant_method": "fp8"}},+ {"quantization_config": {}},+ {"quantization_config": "compressed-tensors"},+ {"quantization_config": {"quant_method": None}},+ {"text_config": "x"},+ None,+ [],+ ],+)+def test_nothing_is_offered_for_anything_else(metadata):+ assert offer(metadata) is None+++def test_only_the_engines_this_host_can_run_are_offered():+ asked = []+ assert offer(NVFP4, supported = lambda *a: asked.append(a) or False) is None+ assert asked == [("vllm", "compressed-tensors"), ("sglang", "compressed-tensors")]+ assert offer(NVFP4, supported = lambda name, method: name == "sglang")["engines"] == ["sglang"]+++def test_the_gpu_probe_is_only_asked_about_these_checkpoints():+ asked = []+ offer(+ {"quantization_config": {"quant_method": "bitsandbytes"}},+ supported = lambda *a: asked.append(a),+ )+ offer(NVFP4, engine = "vllm", supported = lambda *a: asked.append(a))+ assert asked == []+++ "overrides",+ [+ {"engine": "vllm"},+ {"engine": "sglang"},+ {"is_gguf": True},+ {"is_lora": True},+ {"is_audio": True},+ ],+)+def test_nothing_is_offered_for_an_explicit_engine_a_gguf_an_adapter_or_audio(overrides):+ assert offer(NVFP4, **overrides) is None+++def _config(tmp_path, metadata, **extra):+ (tmp_path / "config.json").write_text(json.dumps(metadata), encoding = "utf-8")+ return SimpleNamespace(is_local = True, path = str(tmp_path), identifier = "local/model", **extra)+++def _gpus(*capabilities):+ return patch(+ "core.inference.engine_install._driver_rows",+ return_value = [["580.65", cap] for cap in capabilities],+ )+++def test_the_check_reads_the_checkpoints_own_config(tmp_path):+ with patch("core.inference.engine_install.support_reason", return_value = None), _gpus("10.0"):+ assert route._managed_engine_offer_for(_config(tmp_path, NVFP4), None) == {+ "quantization": "compressed-tensors",+ "engines": ["vllm", "sglang"],+ }+ assert route._managed_engine_offer_for(_config(tmp_path, NVFP4, is_lora = True), None) is None+++ "reason",+ [+ "Checking for a supported NVIDIA GPU.",+ "Managed engines currently require Linux x86_64 or Windows x64.",+ "Requires an NVIDIA GPU with compute capability 8.0 or newer and driver 580 or newer.",+ ],+)+def test_the_check_follows_the_engines_own_support_verdict(tmp_path, reason):+ with patch("core.inference.engine_install.support_reason", return_value = reason):+ assert route._managed_engine_offer_for(_config(tmp_path, NVFP4), None) is None+++def test_the_check_never_fails_validation(tmp_path):+ missing = SimpleNamespace(is_local = True, path = str(tmp_path / "nope"), identifier = "x")+ assert route._managed_engine_offer_for(missing, None) is None+ (tmp_path / "config.json").write_text("{not json", encoding = "utf-8")+ assert (+ route._managed_engine_offer_for(+ SimpleNamespace(is_local = True, path = str(tmp_path), identifier = "x"), None+ )+ is None+ )+ with patch("core.inference.engine_install.support_reason", side_effect = RuntimeError("probe")):+ assert route._managed_engine_offer_for(_config(tmp_path, NVFP4), None) is None+++def test_the_response_carries_the_offer_and_defaults_to_none():+ base = {"valid": True, "message": "ok"}+ assert ValidateModelResponse(**base).managed_engine_offer is None+ response = ValidateModelResponse(+ **base, managed_engine_offer = {"quantization": "compressed-tensors", "engines": ["vllm"]}+ )+ assert response.model_dump()["managed_engine_offer"] == {+ "quantization": "compressed-tensors",+ "engines": ["vllm"],+ }+++def test_sglang_is_offered_compressed_tensors_only_on_blackwell(tmp_path):+ awq = {"quantization_config": {"quant_method": "awq"}}+ with patch("core.inference.engine_install.support_reason", return_value = None):+ with _gpus("9.0", "9.0"):+ assert route._managed_engine_offer_for(_config(tmp_path, NVFP4), None)["engines"] == [+ "vllm"+ ]+ assert route._managed_engine_offer_for(_config(tmp_path, awq), None)["engines"] == [+ "vllm",+ "sglang",+ ]+ with _gpus("8.0", "12.0"):+ assert route._managed_engine_offer_for(_config(tmp_path, NVFP4), None)["engines"] == [+ "vllm",+ "sglang",+ ]+ with _gpus("bad"):+ assert route._managed_engine_offer_for(_config(tmp_path, NVFP4), None)["engines"] == [+ "vllm"+ ]+++def test_audio_checkpoints_get_no_offer(tmp_path):+ with patch("core.inference.engine_install.support_reason", return_value = None), _gpus("10.0"):+ assert (+ route._managed_engine_offer_for(_config(tmp_path, NVFP4, is_audio = True), None) is None+ )+++def test_a_cached_hub_config_is_read_without_a_hub_round_trip(tmp_path):+ (tmp_path / "config.json").write_text(json.dumps(NVFP4), encoding = "utf-8")+ remote = SimpleNamespace(is_local = False, path = None, identifier = "unsloth/Qwen3.8-27B-NVFP4")+ with (+ patch("core.inference.engine_install.support_reason", return_value = None),+ _gpus("10.0"),+ patch("huggingface_hub.try_to_load_from_cache", return_value = str(tmp_path / "config.json")),+ patch("huggingface_hub.hf_hub_download", side_effect = AssertionError("network")),+ ):+ assert route._managed_engine_offer_for(remote, None)["engines"] == ["vllm", "sglang"]diff --git a/studio/frontend/src/app/routes/__root.tsx b/studio/frontend/src/app/routes/__root.tsxindex 509d7bb9c4f..333d57e5b3a 100644--- a/studio/frontend/src/app/routes/__root.tsx+++ b/studio/frontend/src/app/routes/__root.tsx@@ -35,7 +35,10 @@ import { FIND_SCOPE_ATTRIBUTE, FindInPage } from "@/features/find-in-page";import { HfTokenWarningDialog } from "@/features/hf-auth";import { InterfaceZoom, zoomInterfaceFromMenu } from "@/features/interface-zoom";import { bootstrapPersistedCredentials } from "@/features/credentials/bootstrap";-import { SharedRunConfigLinkHandler } from "@/features/model-picker";+import {+ ManagedEngineOfferDialog,+ SharedRunConfigLinkHandler,+} from "@/features/model-picker";import { backfillModelOverrides } from "@/features/model-picker/api/migrate-model-overrides";import { hydratePins } from "@/lib/pins-mirror";import { usePersonalizationSync } from "@/features/profile";@@ -685,6 +688,7 @@ function RootLayout() {<RemoteCodeConsentDialog /><DownloadApprovalDialog /><TransformersUpgradeDialog />+ <ManagedEngineOfferDialog /><LlmCompressorConsentDialog />{/* At the root, not under /chat: a swap can start from the Hub too. */}<StopRunningChatsDialog />diff --git a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.tsindex 85a3ae8d24e..fd90146481c 100644--- a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts+++ b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts@@ -167,6 +167,7 @@ import {DEFAULT_MAX_SEQ_LENGTH,DEFAULT_PER_MODEL_CONFIG,applyPerModelConfigToRuntime,+ confirmManagedEngineIfNeeded,currentRuntimePerModelConfig,isServedByMlx,normalizeMaxSeqLength,@@ -1809,9 +1810,9 @@ export function useChatModelRuntime() {const { keepModelsLoaded, loadedModels: loadedNow, params: paramsNow } =useChatRuntimeStore.getState();// vLLM and SGLang always replace the loaded models, so they keep the running-chat prompt.- const keepsOthers =+ let keepsOthers =keepModelsLoaded && !forceReload && (paramsNow.engine ?? "auto") === "auto";- const switchingNote = keepsOthers ? "Keeping the loaded models." : "Switching models.";+ let switchingNote = keepsOthers ? "Keeping the loaded models." : "Switching models.";// Reloading one of several touches only its own slot, so only its chats stop.const touchesOnlySelected =forceReload && !isExternalModelId(paramsNow.checkpoint) && loadedNow.length > 1;@@ -1862,7 +1863,7 @@ export function useChatModelRuntime() {releasePreflightLifecycleLease();throw error;}- const forceCancelActive = stopDecision.forceCancelActive;+ let forceCancelActive = stopDecision.forceCancelActive;const explicitIsLora =typeof selection === "string" ? undefined : selection.isLora;@@ -2009,6 +2010,8 @@ export function useChatModelRuntime() {(typeof selection !== "string" ? selection.config?.engine : undefined) ??useChatRuntimeStore.getState().params.engine ?? "auto";const managedLoad = !isGguf && requestedEngine !== "auto";+ // Set when the Default engine cannot run the checkpoint and the user picked vLLM / SGLang (#11728).+ let engineSwitched = false;let downloadComplete = isDownloaded || isCachedLora || managedLoad;let cpuFallbackReason: CpuFallbackReason | null = null;let mmprojFallbackReason: MmprojFallbackReason | null = null;@@ -2273,6 +2276,11 @@ export function useChatModelRuntime() {stateBeforeUnload.ctxCheckpoints,cacheRam: pendingLoadConfig?.cacheRam ?? stateBeforeUnload.cacheRam,};+ let loadEngineFields = {+ engine: stateBeforeUnload.params.engine ?? "auto",+ engine_precision: stateBeforeUnload.params.enginePrecision ?? "auto",+ engine_parallelism: stateBeforeUnload.params.engineParallelism ?? "tensor",+ };try {// Lightweight pre-flight validation: avoid unloading a working model if the new identifier is// clearly invalid.@@ -2398,6 +2406,78 @@ export function useChatModelRuntime() {// validateModel cannot be aborted either, and everything below writes shared// state, so stop here if a replacement superseded this load meanwhile.if (abortCtrl.signal.aborted) throw new Error("Cancelled");+ const engineOffer =+ !isGguf && loadEngineFields.engine === "auto"+ ? validation.managed_engine_offer+ : null;+ if (engineOffer) {+ const engine = await confirmManagedEngineIfNeeded(+ modelId,+ engineOffer,+ abortCtrl.signal,+ );+ if (abortCtrl.signal.aborted) throw new Error("Cancelled");+ if (!engine) {+ throw new Error(+ `${displayName} is quantized with ${engineOffer.quantization}, which the default engine cannot run. Set Inference engine to ${engineOffer.engines[0] === "sglang" ? "SGLang" : "vLLM"} in its run settings to load it.`,+ );+ }+ loadEngineFields = {+ engine,+ engine_precision: "auto",+ engine_parallelism: "tensor",+ };+ engineSwitched = true;+ downloadComplete = true;+ if (keepsOthers) {+ // vLLM and SGLang replace every loaded model, so ask as any replacing load does.+ stopDecision = await confirmStopRunningChatsIfNeeded(+ "Loading a different model",+ "reload",+ );+ if (abortCtrl.signal.aborted) throw new Error("Cancelled");+ if (!stopDecision.proceed) {+ // Declined before anything was unloaded: exit as a cancellation, not a failed load.+ if (+ modelSelectionIntentEpoch === loadIntentId &&+ pendingReplacementRollback?.residentUnloaded === false+ ) {+ pendingReplacementRollback = null;+ }+ resetLoadingUiForRun(loadRun);+ abortCtrl.abort();+ throw new Error("Cancelled");+ }+ keepsOthers = false;+ forceCancelActive = stopDecision.forceCancelActive;+ loadRun.forceCancelActive = forceCancelActive;+ loadingDescription = loadingDescription.replace(switchingNote, "Switching models.");+ switchingNote = "Switching models.";+ }+ // Re-validate as that engine before unloading; a spent Desktop path lease relies on /load.+ if (!nativePathToken) {+ Object.assign(+ validation,+ await validateModel(+ {+ model_path: loadPath,+ ...loadEngineFields,+ hf_token: hfToken,+ max_seq_length: validateMaxSeqLength,+ load_in_4bit: false,+ is_lora: isLora,+ gguf_variant: ggufVariant ?? null,+ cache_type_kv: loadKvCacheDtype,+ tensor_parallel: loadTensorParallel,+ disable_vision: loadDisableVision,+ gpu_ids: validateGpuIds ?? undefined,+ },+ { signal: abortCtrl.signal },+ ),+ );+ if (abortCtrl.signal.aborted) throw new Error("Cancelled");+ }+ }if (validation.mlx_loads_base_model) {mlxLoadProgress = true;const mlxBaseDescription = isLora@@ -2647,9 +2727,8 @@ export function useChatModelRuntime() {const loadResponse = await loadModel({model_path: loadPath,- engine_precision: stateBeforeUnload.params.enginePrecision ?? "auto",- engine_parallelism: stateBeforeUnload.params.engineParallelism ?? "tensor",- engine: isGguf ? "auto" : (stateBeforeUnload.params.engine ?? "auto"),+ ...loadEngineFields,+ engine: isGguf ? "auto" : loadEngineFields.engine,load_request_id: loadRun.requestId,nativePathLease: loadNativePathLease,hf_token: hfToken,@@ -2659,7 +2738,7 @@ export function useChatModelRuntime() {loadCustomContextLength,loadMaxSeqLength,),- load_in_4bit: (stateBeforeUnload.params.engine ?? "auto") === "auto",+ load_in_4bit: loadEngineFields.engine === "auto",is_lora: isLora,gguf_variant: ggufVariant ?? null,trust_remote_code: trustRemoteCode,@@ -3465,7 +3544,7 @@ export function useChatModelRuntime() {if (progressInterval) clearInterval(progressInterval);return;}- if (managedLoad && prog.bytes_total <= 0) {+ if ((managedLoad || engineSwitched) && prog.bytes_total <= 0) {const label = prog.phase === "warming_up"? "Warming up inference kernels. The first load can take several minutes.": prog.phase === "loading_weights"diff --git a/studio/frontend/src/features/chat/types/api.ts b/studio/frontend/src/features/chat/types/api.tsindex da4f0fe96c1..0e30f98736a 100644--- a/studio/frontend/src/features/chat/types/api.ts+++ b/studio/frontend/src/features/chat/types/api.ts@@ -2,6 +2,7 @@// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0import type { TransformersUpgradeInfo } from "@/features/transformers-upgrade";+import type { ManagedEngineOffer } from "@/features/model-picker";import type { CustomReasoningConfig } from "../custom-reasoning";export type CpuFallbackReason = "vulkan_startup_crash";@@ -167,6 +168,8 @@ export interface ValidateModelResponse {transformers_upgrade?: TransformersUpgradeInfo | null;/** Replacement repository for an MLX BNB model or adapter base. */mlx_loads_base_model?: string | null;+ /** The Default engine cannot run this quantization; these optional engines can on this host. */+ managed_engine_offer?: ManagedEngineOffer | null;}export interface GgufVariantDetail {diff --git a/studio/frontend/src/features/model-picker/components/inference-engines.tsx b/studio/frontend/src/features/model-picker/components/inference-engines.tsxindex 490de0f3a56..50404213764 100644--- a/studio/frontend/src/features/model-picker/components/inference-engines.tsx+++ b/studio/frontend/src/features/model-picker/components/inference-engines.tsx@@ -30,7 +30,7 @@ import { useEngines } from "../hooks/use-engines";const names = { vllm: "vLLM", sglang: "SGLang" };-function EngineInstall({+export function EngineInstall({engine,management = false,onUse,diff --git a/studio/frontend/src/features/model-picker/components/managed-engine-offer-dialog.tsx b/studio/frontend/src/features/model-picker/components/managed-engine-offer-dialog.tsxnew file mode 100644index 00000000000..e5831458f20--- /dev/null+++ b/studio/frontend/src/features/model-picker/components/managed-engine-offer-dialog.tsx@@ -0,0 +1,120 @@+// SPDX-License-Identifier: AGPL-3.0-only+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++import {+ AlertDialog,+ AlertDialogAction,+ AlertDialogCancel,+ AlertDialogContent,+ AlertDialogDescription,+ AlertDialogFooter,+ AlertDialogHeader,+ AlertDialogTitle,+} from "@/components/ui/alert-dialog";+import { Spinner } from "@/components/ui/spinner";+import { useState } from "react";+import { isEngineReady } from "../api/engines";+import { useManagedEngineOfferStore } from "../hooks/managed-engine-offer";+import { useEngines } from "../hooks/use-engines";+import { EngineInstall } from "./inference-engines";++const ENGINE_NAMES = { vllm: "vLLM", sglang: "SGLang" };+const QUANTIZATION_NAMES: Record<string, string> = {+ "compressed-tensors": "compressed-tensors",+ awq: "AWQ",+ gptq: "GPTQ",+};++/** Root-mounted: a Default-engine load only vLLM / SGLang can run waits here for an engine. */+export function ManagedEngineOfferDialog() {+ const open = useManagedEngineOfferStore((s) => s.open);+ const modelName = useManagedEngineOfferStore((s) => s.modelName);+ const offer = useManagedEngineOfferStore((s) => s.offer);+ const resolve = useManagedEngineOfferStore((s) => s.resolve);+ const { engines, error } = useEngines(open);+ const offered = (offer?.engines ?? []).flatMap((name) => {+ const engine = engines.find((row) => row.engine === name);+ return engine ? [engine] : [];+ });+ const ready = offered.filter((engine) => isEngineReady(engine));+ // Install sections stay mounted while open, so even an install that finishes between polls+ // still reaches its success effect and hands the load to that engine.+ const [shown, setShown] = useState<readonly string[]>([]);+ const unshown = offered+ .filter((engine) => !shown.includes(engine.engine))+ .map((engine) => engine.engine);+ if (!open && shown.length > 0) setShown([]);+ else if (open && ready.length === 0 && unshown.length > 0)+ setShown([...shown, ...unshown]);+ const names = (offer?.engines ?? []).map((name) => ENGINE_NAMES[name]);+ const displayName = modelName?.split("/").pop() || "This model";+ const quantization =+ QUANTIZATION_NAMES[offer?.quantization ?? ""] ?? offer?.quantization;++ return (+ <AlertDialog+ open={open}+ onOpenChange={(next) => {+ if (!next) resolve(null);+ }}+ >+ <AlertDialogContent className="max-w-lg">+ <AlertDialogHeader className="min-w-0">+ <AlertDialogTitle>+ {displayName} needs {names.join(" or ")}+ </AlertDialogTitle>+ <AlertDialogDescription>+ This model is quantized with {quantization}, which Unsloth's default+ engine cannot run. {names.join(" and ")}{" "}+ {names.length > 1 ? "can each" : "can"} run it on this computer+ {ready.length > 0 ? "." : " once installed."}+ </AlertDialogDescription>+ </AlertDialogHeader>+ {offered.length === 0 ? (+ error ? (+ // The check keeps retrying; say why it is stuck rather than spinning silently.+ <p role="alert" className="text-ui-12 text-destructive">+ Could not check installed engines: {error}+ </p>+ ) : (+ <p className="flex items-center gap-2 text-ui-12 text-muted-foreground">+ <Spinner className="size-3.5" />+ Checking installed engines...+ </p>+ )+ ) : (+ offered+ .filter(+ (engine) => ready.length === 0 || shown.includes(engine.engine),+ )+ .map((engine) => (+ <section+ key={engine.engine}+ aria-label={ENGINE_NAMES[engine.engine]}+ className="space-y-2 rounded-lg border p-3"+ >+ <p className="text-ui-13 font-medium">+ {ENGINE_NAMES[engine.engine]}+ </p>+ <EngineInstall+ engine={engine}+ onUse={() => resolve(engine.engine)}+ />+ </section>+ ))+ )}+ <AlertDialogFooter>+ <AlertDialogCancel>Cancel</AlertDialogCancel>+ {ready.map((engine) => (+ <AlertDialogAction+ key={engine.engine}+ onClick={() => resolve(engine.engine)}+ >+ Load with {ENGINE_NAMES[engine.engine]}+ </AlertDialogAction>+ ))}+ </AlertDialogFooter>+ </AlertDialogContent>+ </AlertDialog>+ );+}diff --git a/studio/frontend/src/features/model-picker/hooks/managed-engine-offer.ts b/studio/frontend/src/features/model-picker/hooks/managed-engine-offer.tsnew file mode 100644index 00000000000..d8da5b2ba89--- /dev/null+++ b/studio/frontend/src/features/model-picker/hooks/managed-engine-offer.ts@@ -0,0 +1,89 @@+// SPDX-License-Identifier: AGPL-3.0-only+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++import { create } from "zustand";++export type ManagedEngine = "vllm" | "sglang";++/** /validate's managed_engine_offer: the Default engine cannot run this quantization (#11728). */+export interface ManagedEngineOffer {+ quantization: string;+ engines: ManagedEngine[];+}++type Resolver = (engine: ManagedEngine | null) => void;++// One open offer; a newer request declines the older one.+let pendingResolver: Resolver | null = null;++interface ManagedEngineOfferStore {+ open: boolean;+ modelName: string | null;+ offer: ManagedEngineOffer | null;+ request: (+ modelName: string,+ offer: ManagedEngineOffer,+ signal?: AbortSignal,+ ) => Promise<ManagedEngine | null>;+ resolve: (engine: ManagedEngine | null) => void;+}++export const useManagedEngineOfferStore = create<ManagedEngineOfferStore>()(+ (set) => ({+ open: false,+ modelName: null,+ offer: null,+ request: (modelName, offer, signal) =>+ new Promise<ManagedEngine | null>((resolve) => {+ pendingResolver?.(null);+ if (signal?.aborted) {+ pendingResolver = null;+ set({ open: false });+ resolve(null);+ return;+ }+ const settle: Resolver = (engine) => {+ signal?.removeEventListener("abort", onAbort);+ resolve(engine);+ };+ const onAbort = () => {+ if (pendingResolver !== settle) return;+ pendingResolver = null;+ set({ open: false });+ settle(null);+ };+ signal?.addEventListener("abort", onAbort, { once: true });+ pendingResolver = settle;+ set({ open: true, modelName, offer });+ }),+ resolve: (engine) => {+ const resolver = pendingResolver;+ pendingResolver = null;+ set({ open: false });+ resolver?.(engine);+ },+ }),+);++/** The engines of an offer this build can load with, in the backend's order. */+export function offeredEngines(+ offer: ManagedEngineOffer | null | undefined,+): ManagedEngine[] {+ return (offer?.engines ?? []).filter(+ (engine): engine is ManagedEngine =>+ engine === "vllm" || engine === "sglang",+ );+}++/** Pause a Default-engine load on the offer dialog; resolves the engine to load with, or null. */+export async function confirmManagedEngineIfNeeded(+ modelName: string,+ offer: ManagedEngineOffer | null | undefined,+ signal?: AbortSignal,+): Promise<ManagedEngine | null> {+ const engines = offeredEngines(offer);+ if (!offer || engines.length === 0) return null;+ return useManagedEngineOfferStore+ .getState()+ .request(modelName, { ...offer, engines }, signal);+}diff --git a/studio/frontend/src/features/model-picker/index.ts b/studio/frontend/src/features/model-picker/index.tsindex 5b9626cda1e..496248ce255 100644--- a/studio/frontend/src/features/model-picker/index.ts+++ b/studio/frontend/src/features/model-picker/index.ts@@ -37,6 +37,11 @@ export {} from "./api/model-overrides";export { useActiveModelConfig } from "./hooks/use-active-model-config";export { useVllmAvailable } from "./hooks/use-vllm-available";+export { ManagedEngineOfferDialog } from "./components/managed-engine-offer-dialog";+export {+ confirmManagedEngineIfNeeded,+ type ManagedEngineOffer,+} from "./hooks/managed-engine-offer";export type {DeletedModelRef,ExternalConnectionRef,diff --git a/studio/frontend/tests/chat-model-load-replacement.test.ts b/studio/frontend/tests/chat-model-load-replacement.test.tsindex 1346212a300..2d9bdcece17 100644--- a/studio/frontend/tests/chat-model-load-replacement.test.ts+++ b/studio/frontend/tests/chat-model-load-replacement.test.ts@@ -47,7 +47,7 @@ const blocks = {loadGate: section(runtime, "async function performLoad(): Promise<void> {", "const pendingLoadConfig ="),stagedMetadata: section(runtime, "if (isGguf && isDiffusion === undefined) {", "const targetIsDiffusion = isDiffusion === true;"),validation: section(runtime, "const validation = await validateModel({", "if (validation.mlx_loads_base_model) {"),- recheck: section(runtime, "// Re-check the tracked picker for a load that was already starting", "const forceCancelActive = stopDecision.forceCancelActive;"),+ recheck: section(runtime, "// Re-check the tracked picker for a load that was already starting", "let forceCancelActive = stopDecision.forceCancelActive;"),terminal: section(runtime, "await performLoad();", "// Last act of this run's coroutine"),failedCancel: section(section(runtime, "const cancelLoadRun = useCallback(", "const cancelLoadingWithCheckpointPolicy = useCallback("), "// The request failed, so reconcile against the backend before releasing the", "return false;"),cancelReconcile: section(section(runtime, "const cancelLoadRun = useCallback(", "const cancelLoadingWithCheckpointPolicy = useCallback("), "if (\n (!run.loadAttemptPath", "activeLoadRunRef.current = releaseOwnedModelLoadRun("),diff --git a/studio/frontend/tests/managed-engine-offer.test.ts b/studio/frontend/tests/managed-engine-offer.test.tsnew file mode 100644index 00000000000..ec63c91c542--- /dev/null+++ b/studio/frontend/tests/managed-engine-offer.test.ts@@ -0,0 +1,97 @@+// SPDX-License-Identifier: AGPL-3.0-only+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0+import assert from "node:assert/strict";+import test from "node:test";+import {+ confirmManagedEngineIfNeeded,+ offeredEngines,+ useManagedEngineOfferStore,+} from "../src/features/model-picker/hooks/managed-engine-offer.ts";+import { readText } from "./helpers/kit.ts";++// #11728: a Default-engine load of a compressed-tensors / AWQ / GPTQ checkpoint pauses on this offer.+const OFFER = { quantization: "compressed-tensors", engines: ["vllm" as const] };+const store = () => useManagedEngineOfferStore.getState();++test("no offer, or an offer naming no engine this build knows, never opens the dialog", async () => {+ assert.equal(await confirmManagedEngineIfNeeded("m", null), null);+ assert.equal(await confirmManagedEngineIfNeeded("m", { quantization: "awq", engines: [] }), null);+ const unknown = { quantization: "awq", engines: ["tgi"] } as unknown as typeof OFFER;+ assert.equal(await confirmManagedEngineIfNeeded("m", unknown), null);+ assert.equal(store().open, false);+ assert.deepEqual(offeredEngines({ quantization: "gptq", engines: ["sglang", "vllm"] }), ["sglang", "vllm"]);+});++test("choosing the engine resumes the load with it; declining resumes with null", async () => {+ const chosen = confirmManagedEngineIfNeeded("unsloth/Qwen3.8-27B-NVFP4", OFFER);+ assert.equal(store().open, true);+ assert.equal(store().modelName, "unsloth/Qwen3.8-27B-NVFP4");+ assert.deepEqual(store().offer, OFFER);+ store().resolve("vllm");+ assert.equal(await chosen, "vllm");+ assert.equal(store().open, false);++ const declined = confirmManagedEngineIfNeeded("m", OFFER);+ store().resolve(null);+ assert.equal(await declined, null);+ store().resolve("vllm");+});++test("a newer load declines the older one, so one click never resumes two loads", async () => {+ const first = confirmManagedEngineIfNeeded("first", OFFER);+ const second = confirmManagedEngineIfNeeded("second", OFFER);+ assert.equal(await first, null);+ assert.equal(store().modelName, "second");+ store().resolve("vllm");+ assert.equal(await second, "vllm");+});++test("a cancelled load closes its dialog and an already-cancelled one never opens it", async () => {+ const controller = new AbortController();+ const pending = confirmManagedEngineIfNeeded("m", OFFER, controller.signal);+ assert.equal(store().open, true);+ controller.abort();+ assert.equal(await pending, null);+ assert.equal(store().open, false);+ assert.equal(await confirmManagedEngineIfNeeded("m", OFFER, controller.signal), null);+ assert.equal(store().open, false);+ const later = new AbortController();+ const chosen = confirmManagedEngineIfNeeded("m", OFFER, later.signal);+ store().resolve("vllm");+ later.abort();+ assert.equal(await chosen, "vllm");+});++test("the chat load asks before unloading anything and loads with the chosen engine", () => {+ const source = readText("../src/features/chat/hooks/use-chat-model-runtime.ts");+ const offerAt = source.indexOf("confirmManagedEngineIfNeeded(");+ const unloadAt = source.indexOf("await unloadModel({ model_path: currentCheckpoint })");+ assert.ok(offerAt > 0 && unloadAt > offerAt, "the offer must come before the preliminary unload");+ assert.match(source, /loadModel\(\{\s*model_path: loadPath,\s*\.\.\.loadEngineFields,/);+ const revalidateAt = source.indexOf("...loadEngineFields,", offerAt);+ assert.ok(revalidateAt > offerAt && revalidateAt < unloadAt, "the chosen engine must be validated before the unload");+ // Keep loaded models: the switch re-asks about running chats and then replaces, like any managed load.+ const branch = source.slice(offerAt, unloadAt);+ assert.match(branch, /if \(keepsOthers\) \{[\s\S]*?stopDecision = await confirmStopRunningChatsIfNeeded\([\s\S]*?keepsOthers = false;\s*forceCancelActive = stopDecision\.forceCancelActive;\s*loadRun\.forceCancelActive = forceCancelActive;/);+ assert.match(readText("../src/app/routes/__root.tsx"), /<ManagedEngineOfferDialog \/>/);+ // Declining the replacement prompt this engine needs is a cancellation, and the toast stops promising kept models.+ const offerBranch = source.slice(offerAt, unloadAt);+ assert.match(offerBranch, /abortCtrl\.abort\(\);\s*throw new Error\("Cancelled"\);/);+ assert.match(offerBranch, /\},\s*\{ signal: abortCtrl\.signal \},\s*\),/);+ assert.match(offerBranch, /loadingDescription = loadingDescription\.replace\(switchingNote, "Switching models\."\);/);+});++test("the offer dialog says when it cannot read the installed engines", () => {+ const dialog = readText("../src/features/model-picker/components/managed-engine-offer-dialog.tsx");+ assert.match(dialog, /const \{ engines, error \} = useEngines\(open\);/);+ assert.match(dialog, /Could not check installed engines: \{error\}/);+});++test("an install started in the dialog stays mounted until it hands the load over", () => {+ // Unmounting on the first ready poll would drop EngineInstall's success effect, and with it onUse,+ // even for an install that finished between two polls.+ const dialog = readText("../src/features/model-picker/components/managed-engine-offer-dialog.tsx");+ assert.match(dialog, /ready\.length === 0 \|\| shown\.includes\(engine\.engine\)/);+ assert.match(dialog, /open && ready\.length === 0 && unshown\.length > 0/);+ assert.match(dialog, /onUse=\{\(\) => resolve\(engine\.engine\)\}/);+});diff --git a/studio/frontend/tests/resident-config-match.test.ts b/studio/frontend/tests/resident-config-match.test.tsindex e584400749f..60a1943592f 100644--- a/studio/frontend/tests/resident-config-match.test.ts+++ b/studio/frontend/tests/resident-config-match.test.ts@@ -1994,7 +1994,7 @@ test("a pick asks the status about its own model and keeps or replaces the otherassert.equal(USE_CHAT_MODEL_RUNTIME.match(/await readPickStatus\(\)/g)?.length, 2);assert.match(USE_CHAT_MODEL_RUNTIME,- /const keepsOthers =\s*keepModelsLoaded && !forceReload && \(paramsNow\.engine \?\? "auto"\) === "auto";[\s\S]*?const touchesOnlySelected =\s*forceReload && !isExternalModelId\(paramsNow\.checkpoint\) && loadedNow\.length > 1;/,+ /let keepsOthers =\s*keepModelsLoaded && !forceReload && \(paramsNow\.engine \?\? "auto"\) === "auto";[\s\S]*?const touchesOnlySelected =\s*forceReload && !isExternalModelId\(paramsNow\.checkpoint\) && loadedNow\.length > 1;/,);assert.match(USE_CHAT_MODEL_RUNTIME, /touchesOnlySelected \? \(paramsNow\.checkpoint \?\? undefined\) : undefined,/);assert.match(USE_CHAT_MODEL_RUNTIME, /stopQueuedRuns\(stopDecision, keepsOthers \|\| touchesOnlySelected\);/);diff --git a/tests/studio/test_model_picker_contracts.py b/tests/studio/test_model_picker_contracts.pyindex b00f764762b..c866f69074a 100644--- a/tests/studio/test_model_picker_contracts.py+++ b/tests/studio/test_model_picker_contracts.py@@ -801,8 +801,8 @@ def test_a_pinned_cached_row_loads_from_the_id_the_backend_pinned():assert ('(typeof selection === "string" ? null : selection.loadId) || modelId' in runtime), "loadPath must fall back to the id, so an unpinned pick is unchanged"- # Staged metadata, validate and load: all three read the copy that loads.- assert runtime.count("model_path: loadPath,") == 3+ # Staged metadata, validate, the engine-switch revalidate and load all read the copy that loads.+ assert runtime.count("model_path: loadPath,") == 4assert "model_path: modelId," not in runtime# A rollback reads the approval under the snapshot path, so store it under both keys.assert "rememberApprovedRemoteCode(loadPath, approvedRemoteCodeFingerprint);" in runtime