Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: KV cache dtype, GPU memory limit and seed for vLLM and SGLang loads (#13596)
Daniel Han742a2280638f
diff --git a/studio/backend/core/inference/engine_adapters.py b/studio/backend/core/inference/engine_adapters.pyindex 1648cd7ac09..6c15360c99b 100644--- a/studio/backend/core/inference/engine_adapters.py+++ b/studio/backend/core/inference/engine_adapters.py@@ -113,6 +113,9 @@ class EngineAdapter:served_model_name = None,):options = options or {}+ if options.get("gpu_memory_utilization"):+ # A user ceiling only lowers Studio's measured budget: above it the engine would OOM.+ memory_fraction = min(memory_fraction, options["gpu_memory_utilization"])tool_args = []if options.get("tool_parser"):tool_args = ["--tool-call-parser", options["tool_parser"]]@@ -242,6 +245,20 @@ class EngineAdapter:if self.name == "vllm" and options.get("max_num_seqs")else []),+ *(+ [+ "--kv-cache-dtype",+ # SGLang names the FP8 format; vLLM's "fp8" is the same E4M3 cache.+ "fp8_e4m3" if self.name == "sglang" else options["kv_cache_dtype"],+ ]+ if options.get("kv_cache_dtype")+ else []+ ),+ *(+ ["--random-seed" if self.name == "sglang" else "--seed", str(options["seed"])]+ if options.get("seed") is not None+ else []+ ),*(["--trust-remote-code"] if trust_remote_code else []),*self.extra_args,"--host",diff --git a/studio/backend/core/inference/managed_engine.py b/studio/backend/core/inference/managed_engine.pyindex c3f9f3ce35f..e586d33d972 100644--- a/studio/backend/core/inference/managed_engine.py+++ b/studio/backend/core/inference/managed_engine.py@@ -177,6 +177,9 @@ def validate_model(engine = "vllm",precision = "auto",parallelism = "tensor",+ kv_cache_dtype = "auto",+ seed = None,+ gpu_memory_utilization = None,) -> dict:"""Check plain metadata before releasing the previous resident model."""if config.is_local:@@ -239,6 +242,13 @@ def validate_model(or (engine == "vllm" and precision == "int4" and parallelism != "pipeline")else "auto",}+ # Unset options stay out of the dict, so a default load keeps today's argv and cache key.+ if kv_cache_dtype != "auto":+ options["kv_cache_dtype"] = kv_cache_dtype+ if seed is not None:+ options["seed"] = seed+ if gpu_memory_utilization is not None:+ options["gpu_memory_utilization"] = gpu_memory_utilizationif reasoning_parser:from .llama_cpp import detect_reasoning_flagsoptions["reasoning_parser"] = reasoning_parser@@ -250,9 +260,9 @@ def validate_model(else config.identifier,log_source = "managed engine",)- if precision == "fp8" and profile(engine)["platform"] == "cuda":- # Eager TorchAO FP8 on Ampere: Triton cannot compile its casts, SGLang online FP8 is invalid.+ if (precision == "fp8" or kv_cache_dtype == "fp8") and profile(engine)["platform"] == "cuda":from utils.hardware.nvidia import _nvidia_smi_executable+result = subprocess.run([_nvidia_smi_executable(),@@ -269,8 +279,23 @@ def validate_model(timeout = 60,check = True,)- options["disable_cuda_graph"] = any(float(cap) < 8.9 for cap in result.stdout.splitlines())+ caps = [float(cap) for cap in result.stdout.splitlines() if cap.strip()]+ if precision == "fp8":+ # Eager TorchAO FP8 on Ampere: Triton cannot compile its casts, SGLang online FP8 is invalid.+ options["disable_cuda_graph"] = any(cap < 8.9 for cap in caps)+ if kv_cache_dtype == "fp8" and any(cap < 8.0 for cap in caps):+ # vLLM 0.30 refuses it at startup on Turing ("native FP8 requires SM89+"); Ampere runs it.+ raise ValueError(+ "An FP8 KV cache needs an Ampere or newer NVIDIA GPU. "+ "Choose Model default for KV cache on this GPU."+ )metadata = metadata.get("text_config") or metadata+ if engine == "sglang" and kv_cache_dtype == "fp8" and metadata.get("kv_lora_rank"):+ # Studio pins SGLang to the Triton attention backend, whose MLA path has no FP8 KV cache.+ raise ValueError(+ "SGLang cannot use an FP8 KV cache with this model's attention. "+ "Choose Model default for KV cache, or use vLLM."+ )size = len(gpu_ids or [0])if size > 1 and parallelism == "tensor":for field in ("num_attention_heads", "hidden_size", "intermediate_size"):@@ -421,6 +446,8 @@ def _rocm_visibility(env: dict, gpu_ids) -> dict:# SGLang raises when its derived gRPC port (HTTP + 10000) exceeds 65535._PORT_LIMIT = 55535+# Launch options that change no compiled artifact, so they share the compile cache.+_UNCACHED_OPTIONS = frozenset(("seed", "gpu_memory_utilization"))def _free_port(limit: int | None = _PORT_LIMIT) -> int:@@ -737,7 +764,15 @@ class ManagedEngine:return hashlib.sha256(policy+ json.dumps(- [info.get("profile_digest"), model, self.context, gpu_ids, options],+ [+ info.get("profile_digest"),+ model,+ self.context,+ gpu_ids,+ {k: v for k, v in options.items() if k not in _UNCACHED_OPTIONS}+ if options+ else options,+ ],sort_keys = True,).encode()).hexdigest()[:16]diff --git a/studio/backend/core/inference/orchestrator.py b/studio/backend/core/inference/orchestrator.pyindex 2f38176dc3f..e8dac030283 100644--- a/studio/backend/core/inference/orchestrator.py+++ b/studio/backend/core/inference/orchestrator.py@@ -2474,6 +2474,9 @@ class InferenceOrchestrator:"engine": engine,"engine_parallelism": (options or {}).get("parallelism", "tensor"),"engine_precision": (options or {}).get("precision", "auto"),+ "engine_kv_cache_dtype": (options or {}).get("kv_cache_dtype", "auto"),+ "engine_seed": (options or {}).get("seed"),+ "engine_gpu_memory_utilization": (options or {}).get("gpu_memory_utilization"),"is_vision": (options or {}).get("is_vision", config.is_vision),"chat_template_info": {"accepts_multiple_images": bool(diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.pyindex b0eb9376bca..a9348c6eb5a 100644--- a/studio/backend/models/inference.py+++ b/studio/backend/models/inference.py@@ -61,6 +61,20 @@ class LoadRequest(BaseModel):engine_parallelism: Literal["tensor", "pipeline", "data"] = "tensor"engine_precision: Literal["auto", "bf16", "fp16", "int4", "int8", "fp8"] = "auto"+ engine_kv_cache_dtype: Literal["auto", "fp8"] = Field(+ "auto",+ description = "Managed engine KV cache dtype; 'fp8' halves KV cache memory (vLLM --kv-cache-dtype).",+ )+ engine_seed: Optional[int] = Field(+ None, ge = 0, le = 2**31 - 1, description = "Managed engine seed (vLLM --seed)."+ )+ engine_gpu_memory_utilization: Optional[float] = Field(+ None,+ ge = 0.05,+ le = 0.99,+ description = "Upper bound on the share of each selected GPU the managed engine may claim "+ "(vLLM --gpu-memory-utilization). Unset lets Studio size it from free memory.",+ )engine: Literal["auto", "vllm", "sglang"] = Field("auto",description = "Inference engine to use. 'auto' selects Studio's default backend; "@@ -599,6 +613,20 @@ class ValidateModelRequest(BaseModel):engine_parallelism: Literal["tensor", "pipeline", "data"] = "tensor"engine_precision: Literal["auto", "bf16", "fp16", "int4", "int8", "fp8"] = "auto"+ engine_kv_cache_dtype: Literal["auto", "fp8"] = Field(+ "auto",+ description = "Managed engine KV cache dtype; 'fp8' halves KV cache memory (vLLM --kv-cache-dtype).",+ )+ engine_seed: Optional[int] = Field(+ None, ge = 0, le = 2**31 - 1, description = "Managed engine seed (vLLM --seed)."+ )+ engine_gpu_memory_utilization: Optional[float] = Field(+ None,+ ge = 0.05,+ le = 0.99,+ description = "Upper bound on the share of each selected GPU the managed engine may claim "+ "(vLLM --gpu-memory-utilization). Unset lets Studio size it from free memory.",+ )engine: Literal["auto", "vllm", "sglang"] = "auto"model_path: str = Field(..., description = "Model identifier or local path")# The same inventory handle the picker was shown; see `resolve_inventory_handle`.@@ -1440,6 +1468,9 @@ class _InferenceRuntimeFields(BaseModel):engine_parallelism: Literal["tensor", "pipeline", "data"] = "tensor"engine_precision: Literal["auto", "bf16", "fp16", "int4", "int8", "fp8"] = "auto"+ engine_kv_cache_dtype: Literal["auto", "fp8"] = "auto"+ engine_seed: Optional[int] = None+ engine_gpu_memory_utilization: Optional[float] = Noneengine: Literal["auto", "vllm", "sglang"] = Field("auto",description = "Active inference engine. 'auto' denotes Studio's default backend; "diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex 3b7eca1feee..6d4b6f88a35 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -8460,6 +8460,7 @@ def _llama_runtime_fields(llama_backend: LlamaCppBackend) -> dict:engine = "auto",engine_parallelism = "tensor",engine_precision = "auto",+ **_MANAGED_ENGINE_LAUNCH_DEFAULTS,# Not MLX, so the MLX runtime fields report as absent.is_mlx = False,is_npu = False,@@ -17473,6 +17474,21 @@ def _inherit_resident_load_in_4bit(backend, request, model_identifier: str) -> Nrequest.load_in_4bit = bool(resident)+# Managed-engine launch options a load records and a status echoes; the value is "not set".+_MANAGED_ENGINE_LAUNCH_DEFAULTS = {+ "engine_kv_cache_dtype": "auto",+ "engine_seed": None,+ "engine_gpu_memory_utilization": None,+}+++def _managed_engine_launch_fields(model_info) -> dict:+ return {+ field: (model_info or {}).get(field, default)+ for field, default in _MANAGED_ENGINE_LAUNCH_DEFAULTS.items()+ }++def _non_gguf_runtime_settings_match(backend, request) -> bool:"""Whether the resident non-GGUF model already runs the request's load settings.@@ -17495,6 +17511,9 @@ def _non_gguf_runtime_settings_match(backend, request) -> bool:request, "engine_parallelism", "tensor"):return False+ for field, default in _MANAGED_ENGINE_LAUNCH_DEFAULTS.items():+ if entry.get(field, default) != getattr(request, field, default):+ return Falseif list(request.gpu_ids or [0]) != list(entry.get("gpu_ids_requested") or [0]):return Falseif "max_seq_length" in fields_set and int(request.max_seq_length or 0) > 0:@@ -18276,6 +18295,9 @@ async def _managed_engine_options(request, config, hf_token) -> dict:request.engine,request.engine_precision,request.engine_parallelism,+ kv_cache_dtype = getattr(request, "engine_kv_cache_dtype", "auto"),+ seed = getattr(request, "engine_seed", None),+ gpu_memory_utilization = getattr(request, "engine_gpu_memory_utilization", None),)except (ValueError, OSError) as exc:raise HTTPException(status_code = 400, detail = str(exc)) from exc@@ -18765,6 +18787,7 @@ async def _load_model_impl(engine = request.engine,engine_parallelism = request.engine_parallelism,engine_precision = request.engine_precision,+ **_managed_engine_launch_fields(_model_info),gpu_ids = _model_info.get("gpu_ids"),requested_gpu_ids = _model_info.get("gpu_ids_requested"),tensor_parallel = bool(_model_info.get("tensor_parallel", False)),@@ -19650,6 +19673,7 @@ async def _load_model_impl(engine = request.engine,engine_parallelism = request.engine_parallelism,engine_precision = request.engine_precision,+ **_managed_engine_launch_fields(_model_info),gpu_ids = _model_info.get("gpu_ids"),requested_gpu_ids = _model_info.get("gpu_ids_requested"),tensor_parallel = bool(_model_info.get("tensor_parallel", False)),@@ -22476,6 +22500,7 @@ async def _slot_status(current_subject: str):engine = model_info.get("engine", "auto"),engine_parallelism = model_info.get("engine_parallelism", "tensor"),engine_precision = model_info.get("engine_precision", "auto"),+ **_managed_engine_launch_fields(model_info),gpu_ids = model_info.get("gpu_ids"),tensor_parallel = bool(model_info.get("tensor_parallel", False)),active_model = _active,diff --git a/studio/backend/routes/settings.py b/studio/backend/routes/settings.pyindex 4dd27b5a079..bf32623e7e6 100644--- a/studio/backend/routes/settings.py+++ b/studio/backend/routes/settings.py@@ -996,6 +996,9 @@ class ModelOverridePayload(BaseModel):engine_parallelism: Optional[Literal["tensor", "pipeline", "data"]] = Noneengine_precision: Optional[Literal["auto", "bf16", "fp16", "int4", "int8", "fp8"]] = Noneengine: Optional[Literal["auto", "vllm", "sglang"]] = None+ engine_kv_cache_dtype: Optional[Literal["auto", "fp8"]] = None+ engine_seed: Optional[int] = Field(default = None, ge = 0, le = 2**31 - 1)+ engine_gpu_memory_utilization: Optional[float] = Field(default = None, ge = 0.05, le = 0.99)# None leaves the stored value alone (the UI has no control for flags); [] clears them.llama_extra_args: Optional[list[str]] = None# ge=1: the setter drops a falsy value, so reject 0 here instead of discarding it silently.@@ -2768,6 +2771,16 @@ def update_openai_auto_switch_override(engine = payload.engineif payload.engine is not None or is_removalelse get_model_override(target_id).get("engine"),+ **{+ field: getattr(payload, field)+ if field in fields_set or is_removal+ else get_model_override(target_id).get(field)+ for field in (+ "engine_kv_cache_dtype",+ "engine_seed",+ "engine_gpu_memory_utilization",+ )+ },max_seq_length = max_seq_length,custom_context_length = custom_context_length,kv_cache_dtype = payload.kv_cache_dtype,diff --git a/studio/backend/tests/test_chat_history_routes.py b/studio/backend/tests/test_chat_history_routes.pyindex b3329dd20b7..051194a9224 100644--- a/studio/backend/tests/test_chat_history_routes.py+++ b/studio/backend/tests/test_chat_history_routes.py@@ -673,7 +673,7 @@ def test_chat_inference_settings_covers_frontend_persisted_fields():block = re.search(r"interface InferenceParams \{(.*?)\n\}", source, re.DOTALL)assert block, "InferenceParams interface not found in runtime.ts"omitted = re.search(- r"PersistedInferenceParams = Partial<\s*Omit<InferenceParams,([^>]*)>", source+ r"PersistedInferenceParams = Partial<\s*Omit<\s*InferenceParams,([^>]*)>", source)assert omitted, "PersistedInferenceParams Omit<> not found in runtime.ts"persisted = set(re.findall(r"^\s*(\w+)\??:", block.group(1), re.M)) - set(diff --git a/studio/backend/tests/test_gguf_load_cache_reuse.py b/studio/backend/tests/test_gguf_load_cache_reuse.pyindex 51da3829967..ec9d5ea5110 100644--- a/studio/backend/tests/test_gguf_load_cache_reuse.py+++ b/studio/backend/tests/test_gguf_load_cache_reuse.py@@ -1178,6 +1178,9 @@ class TestLoadHubDownloadExclusion:"engine","engine_parallelism","engine_precision",+ "engine_kv_cache_dtype",+ "engine_seed",+ "engine_gpu_memory_utilization","requires_trust_remote_code","speculative_type","requested_parallel_slots",diff --git a/studio/backend/tests/test_managed_engine_launch_options.py b/studio/backend/tests/test_managed_engine_launch_options.pynew file mode 100644index 00000000000..f4286a1a912--- /dev/null+++ b/studio/backend/tests/test_managed_engine_launch_options.py@@ -0,0 +1,268 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""KV cache dtype, seed and GPU memory ceiling for the managed vLLM / SGLang engines."""++import json+from types import SimpleNamespace++import pytest++from core.inference.engine_adapters import ADAPTERS++# What validate_model returns for an unquantized text checkpoint with a tool parser.+_DEFAULT_OPTIONS = {+ "tool_parser": "hermes",+ "precision": "auto",+ "parallelism": "tensor",+ "is_vision": False,+ "quantization": None,+ "disable_cuda_graph": False,+ "load_format": "auto",+ "engine_version": "0.31.0",+}++# The argv main built for _DEFAULT_OPTIONS (launcher path at index 2 left out).+_MAIN_ARGV = {+ "vllm": [+ "/env/bin/python",+ "-I",+ "vllm.entrypoints.openai.api_server",+ "--model",+ "org/model",+ "--max-model-len",+ "4096",+ "--gpu-memory-utilization",+ "0.8",+ "--tensor-parallel-size",+ "1",+ "--tool-call-parser",+ "hermes",+ "--enable-auto-tool-choice",+ "--generation-config",+ "vllm",+ "--host",+ "127.0.0.1",+ "--port",+ "45678",+ "--served-model-name",+ "org/model",+ "--load-format",+ "auto",+ ],+ "sglang": [+ "/env/bin/python",+ "-I",+ "--model-path",+ "org/model",+ "--context-length",+ "4096",+ "--mem-fraction-static",+ "0.8",+ "--tensor-parallel-size",+ "1",+ "--tool-call-parser",+ "hermes",+ "--attention-backend",+ "triton",+ "--sampling-backend",+ "pytorch",+ "--host",+ "127.0.0.1",+ "--port",+ "45678",+ "--served-model-name",+ "org/model",+ "--load-format",+ "auto",+ ],+}+++def _argv(engine, **extra):+ argv = ADAPTERS[engine].command(+ "/env/bin/python",+ "org/model",+ 45678,+ "key",+ 4096,+ 0.8,+ 1,+ options = {**_DEFAULT_OPTIONS, **extra},+ )+ return argv[:2] + argv[3:]++[email protected]("engine", ["vllm", "sglang"])+def test_default_options_launch_the_same_argv_as_before(engine):+ assert _argv(engine) == _MAIN_ARGV[engine]+++ "engine, kv, seed", [("vllm", "fp8", "--seed"), ("sglang", "fp8_e4m3", "--random-seed")]+)+def test_kv_cache_dtype_and_seed_reach_each_engine(engine, kv, seed):+ argv = _argv(engine, kv_cache_dtype = "fp8", seed = 7)+ assert argv[argv.index("--kv-cache-dtype") + 1] == kv+ assert argv[argv.index(seed) + 1] == "7"+ zero = _argv(engine, seed = 0)+ assert zero[zero.index(seed) + 1] == "0"+++ "engine, flag", [("vllm", "--gpu-memory-utilization"), ("sglang", "--mem-fraction-static")]+)+def test_memory_ceiling_lowers_but_never_raises_the_measured_budget(engine, flag):+ lowered = _argv(engine, gpu_memory_utilization = 0.5)+ assert lowered[lowered.index(flag) + 1] == "0.5"+ kept = _argv(engine, gpu_memory_utilization = 0.95)+ assert kept[kept.index(flag) + 1] == "0.8"+++def _config(tmp_path):+ (tmp_path / "config.json").write_text(json.dumps({"num_attention_heads": 8}))+ return SimpleNamespace(is_local = True, path = str(tmp_path))+++def _compute_cap(monkeypatch, cap):+ from core.inference import managed_engine+ monkeypatch.setattr(managed_engine, "profile", lambda engine: {"platform": "cuda"})+ monkeypatch.setattr(+ managed_engine.subprocess, "run", lambda argv, **kwargs: SimpleNamespace(stdout = f"{cap}\n")+ )++[email protected]("engine", ["vllm", "sglang"])+def test_unset_options_stay_out_of_the_launch_options(tmp_path, engine, monkeypatch):+ from core.inference.managed_engine import validate_model++ _compute_cap(monkeypatch, "9.0")+ options = validate_model(_config(tmp_path), engine = engine)+ assert not {"kv_cache_dtype", "seed", "gpu_memory_utilization"} & set(options)+ options = validate_model(+ _config(tmp_path),+ engine = engine,+ kv_cache_dtype = "fp8",+ seed = 0,+ gpu_memory_utilization = 0.6,+ )+ assert (options["kv_cache_dtype"], options["seed"], options["gpu_memory_utilization"]) == (+ "fp8",+ 0,+ 0.6,+ )+++def test_seed_and_memory_ceiling_share_the_compile_cache():+ from core.inference.managed_engine import ManagedEngine++ engine = ManagedEngine("vllm")+ info = {"profile_digest": "d"}+ base = engine._cache_key(info, "m", [0], dict(_DEFAULT_OPTIONS))+ assert base == engine._cache_key(+ info, "m", [0], {**_DEFAULT_OPTIONS, "seed": 3, "gpu_memory_utilization": 0.5}+ )+ assert base != engine._cache_key(info, "m", [0], {**_DEFAULT_OPTIONS, "kv_cache_dtype": "fp8"})+ assert engine._cache_key(info, "m", [0], None) == engine._cache_key(info, "m", [0], None)+++ "field, value",+ [+ ("engine_kv_cache_dtype", "fp8"),+ ("engine_seed", 1),+ ("engine_gpu_memory_utilization", 0.5),+ ],+)+def test_a_changed_launch_option_reloads_the_resident_engine(field, value):+ from models.inference import LoadRequest+ from routes.inference import _non_gguf_runtime_settings_match++ backend = SimpleNamespace(+ active_model_name = "model",+ models = {"model": {"engine": "vllm", "gpu_ids_requested": [0]}},+ )+ request = LoadRequest(model_path = "model", engine = "vllm", gpu_ids = [0])+ assert _non_gguf_runtime_settings_match(backend, request)+ assert not _non_gguf_runtime_settings_match(backend, request.model_copy(update = {field: value}))+ backend.models["model"][field] = value+ assert _non_gguf_runtime_settings_match(backend, request.model_copy(update = {field: value}))+++def test_overrides_store_only_set_options_and_load_them_back():+ from utils.openai_auto_switch_settings import (+ model_override_load_kwargs,+ normalize_model_override,+ )++ assert normalize_model_override(+ {+ "engine": "vllm",+ "engine_kv_cache_dtype": "auto",+ "engine_seed": None,+ "engine_gpu_memory_utilization": None,+ }+ ) == {"engine": "vllm"}+ entry = normalize_model_override(+ {+ "engine": "vllm",+ "engine_kv_cache_dtype": "fp8",+ "engine_seed": 0,+ "engine_gpu_memory_utilization": 0.75,+ }+ )+ assert entry == {+ "engine": "vllm",+ "engine_kv_cache_dtype": "fp8",+ "engine_seed": 0,+ "engine_gpu_memory_utilization": 0.75,+ }+ assert normalize_model_override({"engine_gpu_memory_utilization": 1.5}) == {}+ kwargs = model_override_load_kwargs(entry, is_gguf = False)+ assert kwargs["engine_kv_cache_dtype"] == "fp8"+ assert kwargs["engine_seed"] == 0+ assert kwargs["engine_gpu_memory_utilization"] == 0.75+ assert "engine_seed" not in model_override_load_kwargs(entry, is_gguf = True)++[email protected]("cap,ok", [("7.5", False), ("8.0", True), ("8.9", True)])+def test_an_fp8_kv_cache_is_refused_before_ampere(tmp_path, monkeypatch, cap, ok):+ # Colab T4, vLLM 0.30: "FP8 KV cache is not supported ... on Tesla T4 (compute capability 7.5)".+ from core.inference.managed_engine import validate_model+ _compute_cap(monkeypatch, cap)+ if ok:+ assert validate_model(_config(tmp_path), kv_cache_dtype = "fp8")["kv_cache_dtype"] == "fp8"+ else:+ with pytest.raises(ValueError, match = "Ampere"):+ validate_model(_config(tmp_path), kv_cache_dtype = "fp8")+++def test_a_default_kv_cache_never_probes_the_gpu(tmp_path, monkeypatch):+ from core.inference import managed_engine++ def refuse(argv, **kwargs):+ raise AssertionError("nvidia-smi probed for a default KV cache")++ monkeypatch.setattr(managed_engine, "profile", lambda engine: {"platform": "cuda"})+ monkeypatch.setattr(managed_engine.subprocess, "run", refuse)+ managed_engine.validate_model(_config(tmp_path), kv_cache_dtype = "auto")++[email protected]("engine,refused", [("sglang", True), ("vllm", False)])+def test_sglang_refuses_an_fp8_kv_cache_on_mla_models(tmp_path, monkeypatch, engine, refused):+ # Studio pins SGLang's Triton attention backend, whose MLA path has no FP8 KV cache.+ from core.inference.managed_engine import validate_model++ _compute_cap(monkeypatch, "9.0")+ (tmp_path / "config.json").write_text(+ json.dumps({"num_attention_heads": 8, "kv_lora_rank": 512})+ )+ config = SimpleNamespace(is_local = True, path = str(tmp_path))+ if refused:+ with pytest.raises(ValueError, match = "MLA|attention"):+ validate_model(config, engine = engine, kv_cache_dtype = "fp8")+ else:+ assert (+ validate_model(config, engine = engine, kv_cache_dtype = "fp8")["kv_cache_dtype"] == "fp8"+ )+ assert "kv_cache_dtype" not in validate_model(config, engine = engine)diff --git a/studio/backend/utils/openai_auto_switch_settings.py b/studio/backend/utils/openai_auto_switch_settings.pyindex ad79892aa8d..ee2f8253b2a 100644--- a/studio/backend/utils/openai_auto_switch_settings.py+++ b/studio/backend/utils/openai_auto_switch_settings.py@@ -446,6 +446,18 @@ def normalize_model_override(entry["engine_precision"] = payload["engine_precision"]if payload.get("engine") in ("vllm", "sglang"):entry["engine"] = payload["engine"]+ if payload.get("engine_kv_cache_dtype") == "fp8":+ entry["engine_kv_cache_dtype"] = "fp8"+ seed = _bounded_int(payload.get("engine_seed"), minimum = 0, maximum = 2**31 - 1)+ if seed is not None:+ entry["engine_seed"] = seed+ utilization = payload.get("engine_gpu_memory_utilization")+ if (+ isinstance(utilization, (int, float))+ and not isinstance(utilization, bool)+ and 0.05 <= utilization <= 0.99+ ):+ entry["engine_gpu_memory_utilization"] = round(float(utilization), 3)extra_args = payload.get("llama_extra_args")if isinstance(extra_args, (list, tuple)) and extra_args:@@ -637,6 +649,9 @@ def model_override_load_kwargs(override: dict[str, Any], *, is_gguf: bool) -> dikwargs["engine"] = override["engine"]kwargs["engine_parallelism"] = override.get("engine_parallelism", "tensor")kwargs["engine_precision"] = override.get("engine_precision", "auto")+ for key in ("engine_kv_cache_dtype", "engine_seed", "engine_gpu_memory_utilization"):+ if key in override:+ kwargs[key] = override[key]kwargs["load_in_4bit"] = Falseif override.get("gpu_ids") is not None:kwargs["gpu_ids"] = override["gpu_ids"]diff --git a/studio/frontend/scripts/check-bundle-budget.ts b/studio/frontend/scripts/check-bundle-budget.tsindex ed1cfd6923b..658c15a1621 100644--- a/studio/frontend/scripts/check-bundle-budget.ts+++ b/studio/frontend/scripts/check-bundle-budget.ts@@ -80,8 +80,11 @@ export const BUDGET = {// no single import doing it. The largest step is the audio model selector fix (#13517, +2.9 KB raw /// +0.9 KB transfer); thinking traces (#13545), MLX precision folders (#13565) and vLLM thinking// effort (#13571) add the rest. Raised by that growth so main's margin stays where it was.- transferBytes: 2_025_000,- rawBytes: 6_737_000,+ // The KV cache, GPU memory limit and seed engine options (#13596) add 5.7 KB raw / 1.4 KB transfer+ // (6,573.5 -> 6,579.2 KB raw, 1,976.9 -> 1,978.3 KB transfer against abfe3142c, one machine and+ // build), inline in the eager chat and settings chunks like the other engine options; raised by that.+ transferBytes: 2_026_500,+ rawBytes: 6_743_000,};// The chunk count is reported but not budgeted. Splitting a page out of the entry raises it while lowering thediff --git a/studio/frontend/src/features/api-monitor/components/saved-model-settings.tsx b/studio/frontend/src/features/api-monitor/components/saved-model-settings.tsxindex 778237bfcb8..2fef1cc0bc3 100644--- a/studio/frontend/src/features/api-monitor/components/saved-model-settings.tsx+++ b/studio/frontend/src/features/api-monitor/components/saved-model-settings.tsx@@ -40,6 +40,17 @@ function describeOverride(override: ApiModelOverride): string[] {if (override.engine_parallelism && override.engine_parallelism !== "tensor") {parts.push(`${override.engine_parallelism} parallel`);}+ if (override.engine_kv_cache_dtype === "fp8") {+ parts.push("FP8 KV cache");+ }+ if (override.engine_seed != null) {+ parts.push(`seed ${override.engine_seed}`);+ }+ if (override.engine_gpu_memory_utilization != null) {+ parts.push(+ `${Math.round(override.engine_gpu_memory_utilization * 100)}% GPU memory`,+ );+ }if (override.custom_context_length) {parts.push(`${override.custom_context_length.toLocaleString()} context`);}diff --git a/studio/frontend/src/features/chat/api/chat-api.ts b/studio/frontend/src/features/chat/api/chat-api.tsindex ab03bbd1236..3b4b5ffd944 100644--- a/studio/frontend/src/features/chat/api/chat-api.ts+++ b/studio/frontend/src/features/chat/api/chat-api.ts@@ -487,6 +487,10 @@ export async function validateModel(engine: payload.engine ?? "auto",engine_precision: payload.engine_precision ?? "auto",engine_parallelism: payload.engine_parallelism ?? "tensor",+ engine_kv_cache_dtype: payload.engine_kv_cache_dtype ?? "auto",+ engine_seed: payload.engine_seed ?? null,+ engine_gpu_memory_utilization:+ payload.engine_gpu_memory_utilization ?? null,// Intended load settings so validate's preflight matches the follow-up /load.max_seq_length: payload.max_seq_length,load_in_4bit: payload.load_in_4bit,diff --git a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.tsindex c1048de6dbd..83d5bbe21da 100644--- a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts+++ b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts@@ -182,6 +182,11 @@ import {loadManagedLlamaFlags,} from "@/features/model-picker/api/llama-flags";import { usePlatformStore } from "@/config/env";+import {+ DEFAULT_ENGINE_LAUNCH_FIELDS,+ engineLaunchFieldsFrom,+ engineLaunchFieldsFromParams,+} from "../lib/engine-launch-fields";import type {ChatLoraSummary,ChatModelRow,@@ -2005,6 +2010,9 @@ export function useChatModelRuntime() {engine: "auto",enginePrecision: "auto",engineParallelism: "tensor",+ engineKvCacheDtype: "auto",+ engineSeed: null,+ engineGpuMemoryUtilization: null,});}const requestedEngine =@@ -2281,6 +2289,7 @@ export function useChatModelRuntime() {engine: stateBeforeUnload.params.engine ?? "auto",engine_precision: stateBeforeUnload.params.enginePrecision ?? "auto",engine_parallelism: stateBeforeUnload.params.engineParallelism ?? "tensor",+ ...engineLaunchFieldsFromParams(stateBeforeUnload.params),};try {// Lightweight pre-flight validation: avoid unloading a working model if the new identifier is@@ -2378,6 +2387,7 @@ export function useChatModelRuntime() {model_path: loadPath,engine_precision: stateBeforeUnload.params.enginePrecision ?? "auto",engine_parallelism: stateBeforeUnload.params.engineParallelism ?? "tensor",+ ...engineLaunchFieldsFromParams(stateBeforeUnload.params),engine: isGguf ? "auto" : (stateBeforeUnload.params.engine ?? "auto"),nativePathLease: validateNativePathLease,hf_token: hfToken,@@ -2463,6 +2473,7 @@ export function useChatModelRuntime() {engine,engine_precision: "auto",engine_parallelism: "tensor",+ ...DEFAULT_ENGINE_LAUNCH_FIELDS,};engineSwitched = true;downloadComplete = true;@@ -3105,6 +3116,7 @@ export function useChatModelRuntime() {loadedEngine: loadResponse.engine ?? "auto",loadedEnginePrecision: loadResponse.engine_precision ?? "auto",loadedEngineParallelism: loadResponse.engine_parallelism ?? "tensor",+ loadedEngineLaunch: engineLaunchFieldsFrom(loadResponse),});// Remembered so auto-load re-picks what the user ran, not the smallest. Native file-picker paths// need a signed, expiring lease, so they stay out.@@ -3158,6 +3170,7 @@ export function useChatModelRuntime() {engine: stateBeforeUnload.loadedEngine,engine_precision: stateBeforeUnload.loadedEnginePrecision,engine_parallelism: stateBeforeUnload.loadedEngineParallelism,+ ...stateBeforeUnload.loadedEngineLaunch,nativePathLease: rollbackNativePathLease,hf_token: hfToken,max_seq_length: rollbackMaxSeqLength,diff --git a/studio/frontend/src/features/chat/lib/apply-inference-status-to-store.ts b/studio/frontend/src/features/chat/lib/apply-inference-status-to-store.tsindex 48bdb82937a..bbfbdce9a88 100644--- a/studio/frontend/src/features/chat/lib/apply-inference-status-to-store.ts+++ b/studio/frontend/src/features/chat/lib/apply-inference-status-to-store.ts@@ -44,6 +44,7 @@ import { sameGpuSelection } from "@/hooks/gpu-selection";import { resolveBatchSizeSeed } from "./resolve-batch-size-seed";import { resolveChatTemplateSeed } from "./resolve-chat-template-seed";import { resolveCtxPinSeed } from "./resolve-ctx-pin-seed";+import { engineLaunchFieldsFrom } from "./engine-launch-fields";import { resolveLlamaExtraArgsSeed } from "./resolve-llama-extra-args-seed";import { shouldSeedVisionSwitch } from "./resolve-vision-switch-seed";@@ -148,6 +149,18 @@ export type ApplyInferenceStatusOptions = {adoptingExistingServerModel?: boolean;};+/** The resident engine launch settings, in the store's params shape. */+function residentEngineParams(status: InferenceStatusResponse) {+ return {+ engine: status.engine ?? "auto",+ enginePrecision: status.engine_precision ?? "auto",+ engineParallelism: status.engine_parallelism ?? "tensor",+ engineKvCacheDtype: status.engine_kv_cache_dtype ?? "auto",+ engineSeed: status.engine_seed ?? null,+ engineGpuMemoryUtilization: status.engine_gpu_memory_utilization ?? null,+ } as const;+}+/** Mirror refresh() hydration so adopted CLI models get reasoning/tools flags. */export function applyActiveModelStatusToStore(status: InferenceStatusResponse,@@ -163,13 +176,19 @@ export function applyActiveModelStatusToStore(loadedEngine: status.engine ?? "auto",loadedEnginePrecision: status.engine_precision ?? "auto",loadedEngineParallelism: status.engine_parallelism ?? "tensor",+ loadedEngineLaunch: engineLaunchFieldsFrom(status),});// Before the settings panel can open on it, which reads only the repo id.adoptCachedRepoConfig(checkpointId, status.gguf_variant ?? null);const store = useChatRuntimeStore.getState();- if ((store.params.engine ?? "auto") !== (status.engine ?? "auto") || (store.params.enginePrecision ?? "auto") !== (status.engine_precision ?? "auto") || (store.params.engineParallelism ?? "tensor") !== (status.engine_parallelism ?? "tensor")) {- store.setParams({ ...store.params, engine: status.engine ?? "auto", enginePrecision: status.engine_precision ?? "auto", engineParallelism: status.engine_parallelism ?? "tensor" });+ const residentEngine = residentEngineParams(status);+ if (+ (Object.keys(residentEngine) as (keyof typeof residentEngine)[]).some(+ (key) => (store.params[key] ?? null) !== residentEngine[key],+ )+ ) {+ store.setParams({ ...store.params, ...residentEngine });}const previousCheckpoint =options.previousCheckpoint ?? store.params.checkpoint;@@ -177,7 +196,7 @@ export function applyActiveModelStatusToStore(if (status.inference) {store.setParams(mergeBackendRecommendedInference({- current: { ...store.params, engine: status.engine ?? "auto", enginePrecision: status.engine_precision ?? "auto", engineParallelism: status.engine_parallelism ?? "tensor" },+ current: { ...store.params, ...residentEngine },response: status,modelId: checkpointId,presetSource: store.activePresetSource,diff --git a/studio/frontend/src/features/chat/lib/engine-launch-fields.ts b/studio/frontend/src/features/chat/lib/engine-launch-fields.tsnew file mode 100644index 00000000000..4d0f499605c--- /dev/null+++ b/studio/frontend/src/features/chat/lib/engine-launch-fields.ts@@ -0,0 +1,44 @@+// SPDX-License-Identifier: AGPL-3.0-only+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++import type { LoadModelRequest } from "../types/api";+import type { InferenceParams } from "../types/runtime";++/** vLLM / SGLang launch options beyond precision and parallelism; unset means the engine default. */+export type EngineLaunchFields = Required<+ Pick<+ LoadModelRequest,+ "engine_kv_cache_dtype" | "engine_seed" | "engine_gpu_memory_utilization"+ >+>;++export const DEFAULT_ENGINE_LAUNCH_FIELDS: EngineLaunchFields = {+ engine_kv_cache_dtype: "auto",+ engine_seed: null,+ engine_gpu_memory_utilization: null,+};++export function engineLaunchFieldsFromParams(+ params: Pick<+ InferenceParams,+ "engineKvCacheDtype" | "engineSeed" | "engineGpuMemoryUtilization"+ >,+): EngineLaunchFields {+ return {+ engine_kv_cache_dtype: params.engineKvCacheDtype ?? "auto",+ engine_seed: params.engineSeed ?? null,+ engine_gpu_memory_utilization: params.engineGpuMemoryUtilization ?? null,+ };+}++/** What a load or status response says the resident engine was launched with. */+export function engineLaunchFieldsFrom(+ response: Partial<EngineLaunchFields>,+): EngineLaunchFields {+ return {+ engine_kv_cache_dtype: response.engine_kv_cache_dtype ?? "auto",+ engine_seed: response.engine_seed ?? null,+ engine_gpu_memory_utilization:+ response.engine_gpu_memory_utilization ?? null,+ };+}diff --git a/studio/frontend/src/features/chat/lib/resident-config-match.ts b/studio/frontend/src/features/chat/lib/resident-config-match.tsindex 122b490aee5..6b2662f615a 100644--- a/studio/frontend/src/features/chat/lib/resident-config-match.ts+++ b/studio/frontend/src/features/chat/lib/resident-config-match.ts@@ -24,6 +24,9 @@ type ResidentRuntime = Pick<| "engine"| "engine_parallelism"| "engine_precision"+ | "engine_kv_cache_dtype"+ | "engine_seed"+ | "engine_gpu_memory_utilization"| "context_length"| "requested_context_length"| "cache_type_kv"@@ -612,7 +615,12 @@ export function residentRuntimeMatchesConfig(if (status.engine === "vllm" || status.engine === "sglang") {if ((status.engine_precision ?? "auto") !== (config.enginePrecision ?? "auto") ||- (status.engine_parallelism ?? "tensor") !== (config.engineParallelism ?? "tensor")+ (status.engine_parallelism ?? "tensor") !== (config.engineParallelism ?? "tensor") ||+ (status.engine_kv_cache_dtype ?? "auto") !==+ (config.engineKvCacheDtype ?? "auto") ||+ (status.engine_seed ?? null) !== (config.engineSeed ?? null) ||+ (status.engine_gpu_memory_utilization ?? null) !==+ (config.engineGpuMemoryUtilization ?? null))return false;if (diff --git a/studio/frontend/src/features/chat/shared-composer.tsx b/studio/frontend/src/features/chat/shared-composer.tsxindex 4b3497a336b..e828add7d77 100644--- a/studio/frontend/src/features/chat/shared-composer.tsx+++ b/studio/frontend/src/features/chat/shared-composer.tsx@@ -1,6 +1,11 @@// SPDX-License-Identifier: AGPL-3.0-only// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0+import {+ DEFAULT_ENGINE_LAUNCH_FIELDS,+ engineLaunchFieldsFrom,+ engineLaunchFieldsFromParams,+} from "./lib/engine-launch-fields";import { useChatArtifactsStore } from "./artifacts/store";import { mlxRuntimeStateFrom } from "./lib/mlx-runtime-state";import { offloadCountsFrom, offloadWarning } from "./lib/partial-offload";@@ -1755,6 +1760,7 @@ export function SharedComposer({engine: paneEngine,engine_precision: ownConfig.enginePrecision ?? "auto",engine_parallelism: ownConfig.engineParallelism ?? "tensor",+ ...engineLaunchFieldsFromParams(ownConfig),load_in_4bit: paneEngine === "auto",...(paneEngine !== "auto" && ownConfig.selectedGpuIds !== undefined? {@@ -1850,6 +1856,7 @@ export function SharedComposer({engine,engine_precision: "auto",engine_parallelism: "tensor",+ ...DEFAULT_ENGINE_LAUNCH_FIELDS,load_in_4bit: false,...(ownConfig.selectedGpuIds !== undefined? {@@ -2098,6 +2105,7 @@ export function SharedComposer({loadedEngine: resp.engine ?? "auto",loadedEnginePrecision: resp.engine_precision ?? "auto",loadedEngineParallelism: resp.engine_parallelism ?? "tensor",+ loadedEngineLaunch: engineLaunchFieldsFrom(resp),loadedDisableVision: resp.disable_vision ?? false,// Adopted from the echo like the knob above: this pane loaded its own model, so the editable// value must follow it or Advanced Settings shows the other pane's Vision state.diff --git a/studio/frontend/src/features/chat/stores/chat-runtime-store.ts b/studio/frontend/src/features/chat/stores/chat-runtime-store.tsindex 0466e3b2e44..49436a060dd 100644--- a/studio/frontend/src/features/chat/stores/chat-runtime-store.ts+++ b/studio/frontend/src/features/chat/stores/chat-runtime-store.ts@@ -126,6 +126,10 @@ import {CHAT_SPECULATIVE_TYPE_KEY,} from "./chat-runtime-keys";import { useExternalProvidersStore } from "./external-providers-store";+import {+ DEFAULT_ENGINE_LAUNCH_FIELDS,+ type EngineLaunchFields,+} from "../lib/engine-launch-fields";export {CHAT_GPU_MEMORY_MODE_KEY,@@ -2341,6 +2345,7 @@ type ChatRuntimeStore = {loadedEngine: "auto" | "vllm" | "sglang";loadedEnginePrecision: NonNullable<InferenceParams["enginePrecision"]>;loadedEngineParallelism: NonNullable<InferenceParams["engineParallelism"]>;+ loadedEngineLaunch: EngineLaunchFields;/** The backend's own is_gguf for the loaded model; null until one loads. Set wherever* loadedContextLength is, so a context never arrives unattributed. */loadedIsGguf: boolean | null;@@ -4224,6 +4229,7 @@ export const useChatRuntimeStore = create<ChatRuntimeStore>((set, get) => ({loadedEngine: "auto",loadedEnginePrecision: "auto",loadedEngineParallelism: "tensor",+ loadedEngineLaunch: DEFAULT_ENGINE_LAUNCH_FIELDS,loadedIsGguf: null,loadedIsMlx: null,loadedContextEnforced: null,diff --git a/studio/frontend/src/features/chat/types/api.ts b/studio/frontend/src/features/chat/types/api.tsindex 548cc0e1efb..d8d164700fc 100644--- a/studio/frontend/src/features/chat/types/api.ts+++ b/studio/frontend/src/features/chat/types/api.ts@@ -49,6 +49,9 @@ export interface ListLorasResponse {export interface LoadModelRequest {engine_parallelism?: "tensor" | "pipeline" | "data";engine_precision?: "auto" | "bf16" | "fp16" | "int4" | "int8" | "fp8";+ engine_kv_cache_dtype?: "auto" | "fp8";+ engine_seed?: number | null;+ engine_gpu_memory_utilization?: number | null;engine?: "auto" | "vllm" | "sglang";model_path: string;/** Opaque client attempt ID used to cancel only this in-flight load. */@@ -233,6 +236,9 @@ export function isMultimodalResponse(export interface LoadModelResponse {engine_parallelism?: "tensor" | "pipeline" | "data";engine_precision?: "auto" | "bf16" | "fp16" | "int4" | "int8" | "fp8";+ engine_kv_cache_dtype?: "auto" | "fp8";+ engine_seed?: number | null;+ engine_gpu_memory_utilization?: number | null;engine?: "auto" | "vllm" | "sglang";is_mlx?: boolean;evicted?: string[];@@ -361,6 +367,9 @@ export interface UnloadModelRequest {export interface InferenceStatusResponse {engine_parallelism?: "tensor" | "pipeline" | "data";engine_precision?: "auto" | "bf16" | "fp16" | "int4" | "int8" | "fp8";+ engine_kv_cache_dtype?: "auto" | "fp8";+ engine_seed?: number | null;+ engine_gpu_memory_utilization?: number | null;engine?: "auto" | "vllm" | "sglang";is_mlx?: boolean;is_npu?: boolean;diff --git a/studio/frontend/src/features/chat/types/runtime.ts b/studio/frontend/src/features/chat/types/runtime.tsindex 2f176f63958..0e3c03ce7b0 100644--- a/studio/frontend/src/features/chat/types/runtime.ts+++ b/studio/frontend/src/features/chat/types/runtime.ts@@ -6,6 +6,9 @@ export type MinPMode = "server-default" | "custom";export interface InferenceParams {engineParallelism?: "tensor" | "pipeline" | "data";enginePrecision?: "auto" | "bf16" | "fp16" | "int4" | "int8" | "fp8";+ engineKvCacheDtype?: "auto" | "fp8";+ engineSeed?: number | null;+ engineGpuMemoryUtilization?: number | null;engine?: "auto" | "vllm" | "sglang";temperature: number;topP: number;@@ -59,7 +62,16 @@ export function modelReadsSamplingSeed(/** The params that survive a reload. `checkpoint` names the model rather than being one of its* settings, so it is not one of them. */export type PersistedInferenceParams = Partial<- Omit<InferenceParams, "checkpoint" | "engine" | "enginePrecision" | "engineParallelism">+ Omit<+ InferenceParams,+ | "checkpoint"+ | "engine"+ | "enginePrecision"+ | "engineParallelism"+ | "engineKvCacheDtype"+ | "engineSeed"+ | "engineGpuMemoryUtilization"+ >>;export const DEFAULT_INFERENCE_PARAMS: InferenceParams = {diff --git a/studio/frontend/src/features/model-picker/api/model-overrides.ts b/studio/frontend/src/features/model-picker/api/model-overrides.tsindex 0c6c69bd01c..0ed4c3c50dc 100644--- a/studio/frontend/src/features/model-picker/api/model-overrides.ts+++ b/studio/frontend/src/features/model-picker/api/model-overrides.ts@@ -29,6 +29,9 @@ export interface ApiModelOverride {engine_parallelism?: "tensor" | "pipeline" | "data";engine_precision?: "auto" | "bf16" | "fp16" | "int4" | "int8" | "fp8";engine?: "auto" | "vllm" | "sglang";+ engine_kv_cache_dtype?: "auto" | "fp8";+ engine_seed?: number | null;+ engine_gpu_memory_utilization?: number | null;// biome-ignore lint/style/useNamingConvention: API schemallama_extra_args?: string[];// biome-ignore lint/style/useNamingConvention: API schema@@ -337,6 +340,13 @@ export function fromApiOverride(engineParallelism: override.engine_parallelism ?? local.engineParallelism ?? "tensor",enginePrecision:override.engine_precision ?? local.enginePrecision ?? "auto",+ engineKvCacheDtype:+ override.engine_kv_cache_dtype ?? local.engineKvCacheDtype ?? "auto",+ engineSeed: override.engine_seed ?? local.engineSeed ?? null,+ engineGpuMemoryUtilization:+ override.engine_gpu_memory_utilization ??+ local.engineGpuMemoryUtilization ??+ null,customContextLength: serverStatesPin? (override.custom_context_length ?? null): local.customContextLength,@@ -400,6 +410,9 @@ export function toApiOverride(config: PerModelConfig | null): ApiModelOverride {engine: config.engine ?? "auto",engine_precision: config.enginePrecision ?? "auto",engine_parallelism: config.engineParallelism ?? "tensor",+ engine_kv_cache_dtype: config.engineKvCacheDtype ?? "auto",+ engine_seed: config.engineSeed ?? null,+ engine_gpu_memory_utilization: config.engineGpuMemoryUtilization ?? null,};if (config.maxSeqLength && config.maxSeqLength > 0) {payload.max_seq_length = config.maxSeqLength;diff --git a/studio/frontend/src/features/model-picker/components/inference-engines.tsx b/studio/frontend/src/features/model-picker/components/inference-engines.tsxindex 59376712bf6..39c40832522 100644--- a/studio/frontend/src/features/model-picker/components/inference-engines.tsx+++ b/studio/frontend/src/features/model-picker/components/inference-engines.tsx@@ -2,6 +2,7 @@// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.import { Button } from "@/components/ui/button";+import { Input } from "@/components/ui/input";import { Switch } from "@/components/ui/switch";import {Select,@@ -37,6 +38,8 @@ const PRECISION_LABELS = [["int8", "INT8 (8-bit)"],["fp8", "FP8 (8-bit)"],] as const;+// Ceilings below Studio's measured budget; it never raises the engine past what is free.+const GPU_MEMORY_LIMITS = [0.95, 0.9, 0.8, 0.7, 0.5, 0.3] as const;export function EngineInstall({engine,@@ -293,6 +296,12 @@ export function InferenceEnginePicker({onParallelismChange,precision = "auto",onPrecisionChange,+ kvCacheDtype = "auto",+ onKvCacheDtypeChange,+ gpuMemoryUtilization = null,+ onGpuMemoryUtilizationChange,+ seed = null,+ onSeedChange,onGpuChange,}: {value: InferenceEngine;@@ -306,6 +315,12 @@ export function InferenceEnginePicker({onPrecisionChange: (precision: "auto" | "bf16" | "fp16" | "int4" | "int8" | "fp8",) => void;+ kvCacheDtype?: "auto" | "fp8";+ onKvCacheDtypeChange: (dtype: "auto" | "fp8") => void;+ gpuMemoryUtilization?: number | null;+ onGpuMemoryUtilizationChange: (fraction: number | null) => void;+ seed?: number | null;+ onSeedChange: (seed: number | null) => void;onGpuChange: (ids: number[]) => void;}) {const { engines, error } = useEngines();@@ -501,6 +516,113 @@ export function InferenceEnginePicker({{t("managedEngines.precisionHelp")}</p></div>+ <div className="space-y-2">+ <div className="flex items-center justify-between gap-3 text-ui-13">+ <span>{t("managedEngines.kvCache")}</span>+ <Select+ value={kvCacheDtype}+ onValueChange={(value) =>+ onKvCacheDtypeChange(value as typeof kvCacheDtype)+ }+ >+ <SelectTrigger+ className="w-44"+ aria-label={t("managedEngines.kvCache")}+ >+ <SelectValue />+ </SelectTrigger>+ <SelectContent>+ <SelectItem value="auto">+ {t("managedEngines.precisionAuto")}+ </SelectItem>+ <SelectItem value="fp8">+ {t("managedEngines.kvCacheFp8")}+ </SelectItem>+ </SelectContent>+ </Select>+ </div>+ <p className="text-ui-12 text-muted-foreground">+ {t("managedEngines.kvCacheHelp")}+ </p>+ </div>+ <div className="space-y-2">+ <div className="flex items-center justify-between gap-3 text-ui-13">+ <span>{t("managedEngines.gpuMemory")}</span>+ <Select+ value={+ gpuMemoryUtilization == null+ ? "auto"+ : String(gpuMemoryUtilization)+ }+ onValueChange={(value) =>+ onGpuMemoryUtilizationChange(+ value === "auto" ? null : Number(value),+ )+ }+ >+ <SelectTrigger+ className="w-44"+ aria-label={t("managedEngines.gpuMemory")}+ >+ <SelectValue />+ </SelectTrigger>+ <SelectContent>+ <SelectItem value="auto">+ {t("managedEngines.gpuMemoryAuto")}+ </SelectItem>+ {/* A stored value off the preset list stays selectable. */}+ {[+ ...new Set([+ ...GPU_MEMORY_LIMITS,+ ...(gpuMemoryUtilization == null+ ? []+ : [gpuMemoryUtilization]),+ ]),+ ]+ .sort((a, b) => b - a)+ .map((fraction) => (+ <SelectItem key={fraction} value={String(fraction)}>+ {`${Math.round(fraction * 100)}%`}+ </SelectItem>+ ))}+ </SelectContent>+ </Select>+ </div>+ <p className="text-ui-12 text-muted-foreground">+ {t("managedEngines.gpuMemoryHelp")}+ </p>+ </div>+ <div className="space-y-2">+ <div className="flex items-center justify-between gap-3 text-ui-13">+ <span>{t("managedEngines.seed")}</span>+ <Input+ className="w-44"+ type="number"+ inputMode="numeric"+ min={0}+ max={2 ** 31 - 1}+ step={1}+ aria-label={t("managedEngines.seed")}+ placeholder={t("managedEngines.seedPlaceholder")}+ value={seed ?? ""}+ onChange={(event) => {+ const raw = event.target.value.trim();+ const next = Number(raw);+ onSeedChange(+ raw !== "" &&+ Number.isInteger(next) &&+ next >= 0 &&+ next <= 2 ** 31 - 1+ ? next+ : null,+ );+ }}+ />+ </div>+ <p className="text-ui-12 text-muted-foreground">+ {t("managedEngines.seedHelp")}+ </p>+ </div><EngineInstallkey={selected.engine}engine={selected}diff --git a/studio/frontend/src/features/model-picker/components/model-config-page.tsx b/studio/frontend/src/features/model-picker/components/model-config-page.tsxindex 7bd9ed20498..0da6e44ea1c 100644--- a/studio/frontend/src/features/model-picker/components/model-config-page.tsx+++ b/studio/frontend/src/features/model-picker/components/model-config-page.tsx@@ -3700,7 +3700,7 @@ export function ModelConfigPage({/><div className="space-y-5">{!target.isGguf && !targetIsMlx && !targetIsNpu && !classifiedIsDiffusion && !target.meta.isLora && !target.meta.audioType && (- <InferenceEnginePicker parallelism={config.engineParallelism ?? "tensor"} onParallelismChange={engineParallelism => update({ engineParallelism })} precision={config.enginePrecision ?? "auto"} onPrecisionChange={enginePrecision => update({ enginePrecision })} value={config.engine ?? "auto"} onChange={engine => update({ engine })} onReadyChange={setEngineReady} onUse={handleRun} gpuIds={config.selectedGpuIds} onGpuChange={ids => update({ selectedGpuIds: ids, selectedGpuIndexKind: "physical" })} />+ <InferenceEnginePicker parallelism={config.engineParallelism ?? "tensor"} onParallelismChange={engineParallelism => update({ engineParallelism })} precision={config.enginePrecision ?? "auto"} onPrecisionChange={enginePrecision => update({ enginePrecision })} kvCacheDtype={config.engineKvCacheDtype ?? "auto"} onKvCacheDtypeChange={engineKvCacheDtype => update({ engineKvCacheDtype })} gpuMemoryUtilization={config.engineGpuMemoryUtilization ?? null} onGpuMemoryUtilizationChange={engineGpuMemoryUtilization => update({ engineGpuMemoryUtilization })} seed={config.engineSeed ?? null} onSeedChange={engineSeed => update({ engineSeed })} value={config.engine ?? "auto"} onChange={engine => update({ engine })} onReadyChange={setEngineReady} onUse={handleRun} gpuIds={config.selectedGpuIds} onGpuChange={ids => update({ selectedGpuIds: ids, selectedGpuIndexKind: "physical" })} />)}{audioRuntimeGguf ? (<p className="text-ui-12 leading-snug text-muted-foreground">diff --git a/studio/frontend/src/features/model-picker/hooks/use-active-model-config.ts b/studio/frontend/src/features/model-picker/hooks/use-active-model-config.tsindex e13c2cc731f..3e6df867463 100644--- a/studio/frontend/src/features/model-picker/hooks/use-active-model-config.ts+++ b/studio/frontend/src/features/model-picker/hooks/use-active-model-config.ts@@ -23,6 +23,13 @@ export function useActiveModelConfig(): ActiveModelConfigState {const engine = useChatRuntimeStore((s) => s.params.engine ?? "auto");const engineParallelism = useChatRuntimeStore((s) => s.params.engineParallelism ?? "tensor");const enginePrecision = useChatRuntimeStore((s) => s.params.enginePrecision ?? "auto");+ const engineKvCacheDtype = useChatRuntimeStore(+ (s) => s.params.engineKvCacheDtype ?? "auto",+ );+ const engineSeed = useChatRuntimeStore((s) => s.params.engineSeed ?? null);+ const engineGpuMemoryUtilization = useChatRuntimeStore(+ (s) => s.params.engineGpuMemoryUtilization ?? null,+ );const activeGgufVariant = useChatRuntimeStore((s) => s.activeGgufVariant);const loadedIsGguf = useChatRuntimeStore((s) => s.loadedIsGguf);const loadedIsMlx = useChatRuntimeStore((s) => s.loadedIsMlx);@@ -125,7 +132,14 @@ export function useActiveModelConfig(): ActiveModelConfigState {};if (!isGguf) {return engine === "vllm" || engine === "sglang"- ? { ...base, selectedGpuIds, selectedGpuIndexKind }+ ? {+ ...base,+ engineKvCacheDtype,+ engineSeed,+ engineGpuMemoryUtilization,+ selectedGpuIds,+ selectedGpuIndexKind,+ }: base;}return {@@ -149,6 +163,9 @@ export function useActiveModelConfig(): ActiveModelConfigState {engine,enginePrecision,engineParallelism,+ engineKvCacheDtype,+ engineSeed,+ engineGpuMemoryUtilization,customContextLength,kvCacheDtype,effectiveMlxKvQuant,diff --git a/studio/frontend/src/features/model-picker/model-config/apply-per-model-config.ts b/studio/frontend/src/features/model-picker/model-config/apply-per-model-config.tsindex f9d9cba8e59..e05a617f081 100644--- a/studio/frontend/src/features/model-picker/model-config/apply-per-model-config.ts+++ b/studio/frontend/src/features/model-picker/model-config/apply-per-model-config.ts@@ -39,13 +39,29 @@ export function applyPerModelConfigToRuntime(const engine = config.engine ?? "auto";const engineParallelism = config.engineParallelism ?? "tensor";const enginePrecision = config.enginePrecision ?? "auto";+ const engineKvCacheDtype = config.engineKvCacheDtype ?? "auto";+ const engineSeed = config.engineSeed ?? null;+ const engineGpuMemoryUtilization = config.engineGpuMemoryUtilization ?? null;if (maxSeqLength !== store.params.maxSeqLength ||engine !== (store.params.engine ?? "auto") ||enginePrecision !== (store.params.enginePrecision ?? "auto") ||- engineParallelism !== (store.params.engineParallelism ?? "tensor")+ engineParallelism !== (store.params.engineParallelism ?? "tensor") ||+ engineKvCacheDtype !== (store.params.engineKvCacheDtype ?? "auto") ||+ engineSeed !== (store.params.engineSeed ?? null) ||+ engineGpuMemoryUtilization !==+ (store.params.engineGpuMemoryUtilization ?? null)) {- store.setParams({ ...store.params, maxSeqLength, engine, enginePrecision, engineParallelism });+ store.setParams({+ ...store.params,+ maxSeqLength,+ engine,+ enginePrecision,+ engineParallelism,+ engineKvCacheDtype,+ engineSeed,+ engineGpuMemoryUtilization,+ });}const gpuSelection =config.selectedGpuIds !== undefined@@ -124,6 +140,9 @@ export function currentRuntimePerModelConfig(engine: s.params.engine ?? "auto",enginePrecision: s.params.enginePrecision ?? "auto",engineParallelism: s.params.engineParallelism ?? "tensor",+ engineKvCacheDtype: s.params.engineKvCacheDtype ?? "auto",+ engineSeed: s.params.engineSeed ?? null,+ engineGpuMemoryUtilization: s.params.engineGpuMemoryUtilization ?? null,customContextLength: s.customContextLength ?? null,maxSeqLength: options.includeMaxSeqLength? normalizeMaxSeqLength(s.params.maxSeqLength)@@ -177,6 +196,10 @@ export function perModelConfigsEqual((a.engine ?? "auto") === (b.engine ?? "auto") &&(a.enginePrecision ?? "auto") === (b.enginePrecision ?? "auto") &&(a.engineParallelism ?? "tensor") === (b.engineParallelism ?? "tensor") &&+ (a.engineKvCacheDtype ?? "auto") === (b.engineKvCacheDtype ?? "auto") &&+ (a.engineSeed ?? null) === (b.engineSeed ?? null) &&+ (a.engineGpuMemoryUtilization ?? null) ===+ (b.engineGpuMemoryUtilization ?? null) &&(a.customContextLength ?? null) === (b.customContextLength ?? null) &&normalizeMaxSeqLength(a.maxSeqLength) ===normalizeMaxSeqLength(b.maxSeqLength) &&diff --git a/studio/frontend/src/features/model-picker/model-config/config-signature.ts b/studio/frontend/src/features/model-picker/model-config/config-signature.tsindex 2b7645fe82a..c617f96e79e 100644--- a/studio/frontend/src/features/model-picker/model-config/config-signature.ts+++ b/studio/frontend/src/features/model-picker/model-config/config-signature.ts@@ -51,6 +51,9 @@ export function loadedConfigSignature(config.engine ?? "auto",config.enginePrecision ?? "auto",config.engineParallelism ?? "tensor",+ config.engineKvCacheDtype ?? "auto",+ config.engineSeed ?? "",+ config.engineGpuMemoryUtilization ?? "",config.customContextLength ?? "",config.maxSeqLength ?? "",config.kvCacheDtype ?? "",diff --git a/studio/frontend/src/features/model-picker/model-config/per-model-config.ts b/studio/frontend/src/features/model-picker/model-config/per-model-config.tsindex dcea694b9b6..2f7964e46be 100644--- a/studio/frontend/src/features/model-picker/model-config/per-model-config.ts+++ b/studio/frontend/src/features/model-picker/model-config/per-model-config.ts@@ -24,6 +24,9 @@ import {export interface PerModelConfig {engineParallelism?: "tensor" | "pipeline" | "data";enginePrecision?: "auto" | "bf16" | "fp16" | "int4" | "int8" | "fp8";+ engineKvCacheDtype?: "auto" | "fp8";+ engineSeed?: number | null;+ engineGpuMemoryUtilization?: number | null;engine?: "auto" | "vllm" | "sglang";customContextLength: number | null;maxSeqLength: number | null;@@ -74,6 +77,9 @@ export const DEFAULT_PER_MODEL_CONFIG: PerModelConfig = {engine: "auto",enginePrecision: "auto",engineParallelism: "tensor",+ engineKvCacheDtype: "auto",+ engineSeed: null,+ engineGpuMemoryUtilization: null,customContextLength: null,maxSeqLength: null,kvCacheDtype: null,@@ -96,6 +102,28 @@ export const DEFAULT_PER_MODEL_CONFIG: PerModelConfig = {chatTemplateOverride: null,};+/** vLLM --seed / SGLang --random-seed; the backend bounds it to a non-negative int32. */+export function normalizeEngineSeed(value: unknown): number | null {+ return typeof value === "number" &&+ Number.isInteger(value) &&+ value >= 0 &&+ value <= 2 ** 31 - 1+ ? value+ : null;+}++/** Ceiling on the share of each GPU a managed engine may claim, as the backend bounds it. */+export function normalizeEngineGpuMemoryUtilization(+ value: unknown,+): number | null {+ return typeof value === "number" &&+ Number.isFinite(value) &&+ value >= 0.05 &&+ value <= 0.99+ ? Math.round(value * 1000) / 1000+ : null;+}+// Mirrors llama_server_args.py PARALLEL_MIN/MAX; null = follow the server-wide default.export const N_PARALLEL_MIN = 1;export const N_PARALLEL_MAX = 64;@@ -1079,6 +1107,11 @@ function normalizeV1(partial: RawConfig): PerModelConfig {partial.engine === "vllm" || partial.engine === "sglang"? partial.engine: "auto",+ engineKvCacheDtype: partial.engineKvCacheDtype === "fp8" ? "fp8" : "auto",+ engineSeed: normalizeEngineSeed(partial.engineSeed),+ engineGpuMemoryUtilization: normalizeEngineGpuMemoryUtilization(+ partial.engineGpuMemoryUtilization,+ ),customContextLength:typeof partial.customContextLength === "number" &&Number.isFinite(partial.customContextLength) &&@@ -1357,6 +1390,9 @@ export function isDefaultConfig(config: PerModelConfig): boolean {(config.engine ?? "auto") === "auto" &&(config.enginePrecision ?? "auto") === "auto" &&(config.engineParallelism ?? "tensor") === "tensor" &&+ (config.engineKvCacheDtype ?? "auto") === "auto" &&+ config.engineSeed == null &&+ config.engineGpuMemoryUtilization == null &&config.customContextLength == null &&config.maxSeqLength == null &&(config.kvCacheDtype ?? null) === DEFAULT_PER_MODEL_CONFIG.kvCacheDtype &&diff --git a/studio/frontend/src/features/model-picker/sharing/fields.ts b/studio/frontend/src/features/model-picker/sharing/fields.tsindex bf0a9556809..471be173652 100644--- a/studio/frontend/src/features/model-picker/sharing/fields.ts+++ b/studio/frontend/src/features/model-picker/sharing/fields.ts@@ -50,6 +50,9 @@ export type SharedConfigKey = Exclude<| "engine"| "enginePrecision"| "engineParallelism"+ | "engineKvCacheDtype"+ | "engineSeed"+ | "engineGpuMemoryUtilization"// A named drafter can be a local directory.| "specDraftModel">;diff --git a/studio/frontend/src/i18n/locales/en.ts b/studio/frontend/src/i18n/locales/en.tsindex 8f3e6484ccc..fb3bd8e5588 100644--- a/studio/frontend/src/i18n/locales/en.ts+++ b/studio/frontend/src/i18n/locales/en.ts@@ -476,6 +476,15 @@ export const en = {precision: "Precision",precisionAuto: "Model default",precisionHelp: "Model default preserves stored precision. Other options convert unquantized weights when loading.",+ kvCache: "KV cache",+ kvCacheFp8: "FP8 (half the memory)",+ kvCacheHelp: "FP8 stores the context cache in 8 bits, fitting about twice the context in the same memory. Without calibrated scales in the checkpoint, answers can be slightly less accurate.",+ gpuMemory: "GPU memory limit",+ gpuMemoryAuto: "Automatic",+ gpuMemoryHelp: "The most of each selected GPU the engine may use. Automatic uses what is free; lower it to leave room for other programs.",+ seed: "Seed",+ seedPlaceholder: "Engine default",+ seedHelp: "Sets the engine's random seed. Concurrent chats can still change sampled replies.",default: "Default",installRequired: "install required",ownerRequired: "Ask the Studio owner to install this engine.",diff --git a/studio/frontend/src/i18n/locales/he.ts b/studio/frontend/src/i18n/locales/he.tsindex e3c479ba29c..cb598773c38 100644--- a/studio/frontend/src/i18n/locales/he.ts+++ b/studio/frontend/src/i18n/locales/he.ts@@ -502,6 +502,16 @@ export const he = {precisionAuto: "ברירת המחדל של המודל",precisionHelp:"ברירת המחדל של המודל שומרת על הדיוק המאוחסן. אפשרויות אחרות ממירות משקלים שלא עברו קוונטיזציה בעת הטעינה.",+ kvCache: "מטמון KV",+ kvCacheFp8: "FP8 (חצי מהזיכרון)",+ kvCacheHelp: "FP8 שומר את מטמון ההקשר ב-8 סיביות, כך שבערך פי שניים הקשר נכנס באותו זיכרון. בלי סקאלות מכוילות בצ'קפוינט, התשובות עלולות להיות מעט פחות מדויקות.",+ gpuMemory: "מגבלת זיכרון GPU",+ gpuMemoryAuto: "אוטומטי",+ gpuMemoryHelp:+ "החלק המרבי מכל GPU נבחר שהמנוע רשאי להשתמש בו. אוטומטי משתמש במה שפנוי; הורד אותו כדי להשאיר מקום לתוכניות אחרות.",+ seed: "זרע אקראי",+ seedPlaceholder: "ברירת המחדל של המנוע",+ seedHelp: "קובע את הזרע האקראי של המנוע. צ'אטים במקביל עדיין יכולים לשנות תשובות שנדגמו.",default: "ברירת מחדל",installRequired: "נדרשת התקנה",ownerRequired: "בקש מבעל ה-Studio להתקין מנוע זה.",diff --git a/studio/frontend/src/i18n/locales/sv.ts b/studio/frontend/src/i18n/locales/sv.tsindex 0475333a27b..cd4d74b7ff3 100644--- a/studio/frontend/src/i18n/locales/sv.ts+++ b/studio/frontend/src/i18n/locales/sv.ts@@ -509,6 +509,17 @@ export const sv = {precisionAuto: "Modellens standard",precisionHelp:"Modellens standard behåller lagrad precision. Andra alternativ konverterar okvantiserade vikter vid inläsning.",+ kvCache: "KV-cache",+ kvCacheFp8: "FP8 (halva minnet)",+ kvCacheHelp:+ "FP8 lagrar kontextcachen med 8 bitar och rymmer ungefär dubbelt så mycket kontext i samma minne. Utan kalibrerade skalor i checkpointen kan svaren bli något mindre exakta.",+ gpuMemory: "GPU-minnesgräns",+ gpuMemoryAuto: "Automatisk",+ gpuMemoryHelp:+ "Det mesta av varje vald GPU som motorn får använda. Automatisk använder det som är ledigt; sänk den för att lämna plats åt andra program.",+ seed: "Seed",+ seedPlaceholder: "Motorns standard",+ seedHelp: "Anger motorns slumpfrö. Samtidiga chattar kan fortfarande ändra samplade svar.",default: "Standard",installRequired: "installation krävs",ownerRequired: "Be Studio-ägaren att installera den här motorn.",diff --git a/studio/frontend/tests/compare-pane-managed-engine.test.ts b/studio/frontend/tests/compare-pane-managed-engine.test.tsindex 559e49a9477..a8b6165e183 100644--- a/studio/frontend/tests/compare-pane-managed-engine.test.ts+++ b/studio/frontend/tests/compare-pane-managed-engine.test.ts@@ -4,21 +4,33 @@import assert from "node:assert/strict";import test from "node:test";import { readSrc } from "./helpers/kit.ts";+import {+ DEFAULT_ENGINE_LAUNCH_FIELDS,+ engineLaunchFieldsFromParams,+} from "../src/features/chat/lib/engine-launch-fields.ts";const composer = readSrc("features/chat/shared-composer.tsx");const start = composer.indexOf("const paneEngine = ");const end = composer.indexOf("let loadTrustRemoteCode", start);assert.ok(start >= 0 && end > start);-const paneFields = new Function(+const paneFieldsBody = new Function("targetIsGguf","ownConfig","reconcilePersistedGpuIds",+ "engineLaunchFieldsFromParams",`${composer.slice(start, end)} return paneEngineFields;`,-) as (+);+const paneFields = (targetIsGguf: boolean,ownConfig: Record<string, unknown>,reconcile: (ids: number[] | null) => number[] | null,-) => Record<string, unknown>;+): Record<string, unknown> =>+ paneFieldsBody(+ targetIsGguf,+ ownConfig,+ reconcile,+ engineLaunchFieldsFromParams,+ );const reconcile = (ids: number[] | null) => ids;test("a compare pane loads its remembered vLLM or SGLang settings", () => {@@ -36,9 +48,23 @@ test("a compare pane loads its remembered vLLM or SGLang settings", () => {engine: "sglang",engine_precision: "int4",engine_parallelism: "pipeline",+ ...DEFAULT_ENGINE_LAUNCH_FIELDS,load_in_4bit: false,gpu_ids: [1, 2],});+ const tuned = paneFields(+ false,+ {+ engine: "vllm",+ engineKvCacheDtype: "fp8",+ engineSeed: 0,+ engineGpuMemoryUtilization: 0.5,+ },+ reconcile,+ );+ assert.equal(tuned.engine_kv_cache_dtype, "fp8");+ assert.equal(tuned.engine_seed, 0);+ assert.equal(tuned.engine_gpu_memory_utilization, 0.5);assert.equal(composer.match(/\.\.\.paneEngineFields,/g)?.length, 2);});diff --git a/studio/frontend/tests/managed-engine-launch-options.test.ts b/studio/frontend/tests/managed-engine-launch-options.test.tsnew file mode 100644index 00000000000..f9e6c0d9b61--- /dev/null+++ b/studio/frontend/tests/managed-engine-launch-options.test.ts@@ -0,0 +1,83 @@+// SPDX-License-Identifier: AGPL-3.0-only+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++/** KV cache dtype, seed and GPU memory limit for vLLM / SGLang loads. */++import assert from "node:assert/strict";+import test from "node:test";++import { registerBundlerResolver } from "./helpers/kit.ts";++registerBundlerResolver();+const { normalizePerModelConfig, isDefaultConfig, DEFAULT_PER_MODEL_CONFIG } = await import(+ "../src/features/model-picker/model-config/per-model-config.ts"+);+const { loadedConfigSignature } = await import(+ "../src/features/model-picker/model-config/config-signature.ts"+);+const { residentRuntimeMatchesConfig } = await import(+ "../src/features/chat/lib/resident-config-match.ts"+);++test("stored launch options survive normalization and bad ones fall back to unset", () => {+ const kept = normalizePerModelConfig({+ engine: "vllm",+ engineKvCacheDtype: "fp8",+ engineSeed: 0,+ engineGpuMemoryUtilization: 0.5,+ });+ assert.equal(kept.engineKvCacheDtype, "fp8");+ assert.equal(kept.engineSeed, 0);+ assert.equal(kept.engineGpuMemoryUtilization, 0.5);+ assert.equal(isDefaultConfig(kept), false);++ const dropped = normalizePerModelConfig({+ engineKvCacheDtype: "fp4",+ engineSeed: -1,+ engineGpuMemoryUtilization: 1.5,+ });+ assert.equal(dropped.engineKvCacheDtype, "auto");+ assert.equal(dropped.engineSeed, null);+ assert.equal(dropped.engineGpuMemoryUtilization, null);+ assert.equal(isDefaultConfig(normalizePerModelConfig({})), true);+});++test("a changed launch option changes the loaded-config signature", () => {+ const base = loadedConfigSignature(DEFAULT_PER_MODEL_CONFIG);+ for (const patch of [+ { engineKvCacheDtype: "fp8" as const },+ { engineSeed: 1 },+ { engineGpuMemoryUtilization: 0.7 },+ ]) {+ assert.notEqual(loadedConfigSignature({ ...DEFAULT_PER_MODEL_CONFIG, ...patch }), base);+ }+});++test("a resident engine launched with other options does not satisfy the pick", () => {+ const standing = {+ speculativeType: "auto",+ gpuMemoryMode: "auto" as const,+ gpuLayers: -1,+ nCpuMoe: 0,+ reconcileGpuIds: (ids: number[] | null) => ids,+ resolveContextLength: (value: number | null) => value ?? 0,+ parallelSlots: 1,+ splitRatio: null,+ normalizeSpeculative: (value: string | null | undefined) => value ?? null,+ };+ const config = { ...DEFAULT_PER_MODEL_CONFIG, engine: "vllm" as const, engineSeed: 3 };+ const status = { engine: "vllm" as const, engine_seed: 3 };+ assert.equal(residentRuntimeMatchesConfig(status, config, standing), true);+ assert.equal(+ residentRuntimeMatchesConfig({ ...status, engine_seed: 4 }, config, standing),+ false,+ );+ assert.equal(+ residentRuntimeMatchesConfig(+ { ...status, engine_kv_cache_dtype: "fp8" },+ config,+ standing,+ ),+ false,+ );+});diff --git a/studio/frontend/tests/managed-engine-offer-background-loads.test.ts b/studio/frontend/tests/managed-engine-offer-background-loads.test.tsindex 2456043cbcb..209adbd8de1 100644--- a/studio/frontend/tests/managed-engine-offer-background-loads.test.ts+++ b/studio/frontend/tests/managed-engine-offer-background-loads.test.ts@@ -6,6 +6,7 @@import assert from "node:assert/strict";import test from "node:test";import { readSrc } from "./helpers/kit.ts";+import { DEFAULT_ENGINE_LAUNCH_FIELDS } from "../src/features/chat/lib/engine-launch-fields.ts";const OFFER = {quantization: "compressed-tensors",@@ -69,7 +70,8 @@ const paneDecisionBody = new AsyncFunction("validation","deps",`const { sel, compareModelDisplayName, confirmManagedEngineIfNeeded, validateModel, panePayload,- throwIfCompareCancelled, compareSignal, ownConfig, reconcilePersistedGpuIds } = deps;+ throwIfCompareCancelled, compareSignal, ownConfig, reconcilePersistedGpuIds,+ DEFAULT_ENGINE_LAUNCH_FIELDS } = deps;let paneEngineFields = { engine: paneEngine };${composer.slice(paneStart, paneEnd)}return paneEngineFields;`,@@ -101,6 +103,7 @@ async function paneDecision(compareSignal: undefined,ownConfig,reconcilePersistedGpuIds: (ids: number[] | null) => ids,+ DEFAULT_ENGINE_LAUNCH_FIELDS,});return { fields, asked, revalidated, validation };}@@ -116,6 +119,7 @@ test("a default-engine compare pane offers vLLM and loads with the pick", asyncengine: "vllm",engine_precision: "auto",engine_parallelism: "tensor",+ ...DEFAULT_ENGINE_LAUNCH_FIELDS,load_in_4bit: false,});assert.equal(diff --git a/studio/frontend/tests/managed-engine-rollback.test.ts b/studio/frontend/tests/managed-engine-rollback.test.tsindex 52274e35bf7..727f2cb97b7 100644--- a/studio/frontend/tests/managed-engine-rollback.test.ts+++ b/studio/frontend/tests/managed-engine-rollback.test.ts@@ -4,16 +4,20 @@import assert from "node:assert/strict";import test from "node:test";import { readSrc } from "./helpers/kit.ts";+import { engineLaunchFieldsFrom } from "../src/features/chat/lib/engine-launch-fields.ts";const applier = readSrc("features/chat/lib/apply-inference-status-to-store.ts");const start = applier.indexOf(" residentCheckpoint: checkpointId,");const end = applier.indexOf(" });", start);assert.ok(start >= 0 && end > start);-const hydrate = new Function(+const hydrateBody = new Function("checkpointId","status",+ "engineLaunchFieldsFrom",`return { ${applier.slice(start, end)} };`,-) as (checkpointId: string, status: Record<string, unknown>) => Record<string, unknown>;+);+const hydrate = (checkpointId: string, status: Record<string, unknown>) =>+ hydrateBody(checkpointId, status, engineLaunchFieldsFrom) as Record<string, unknown>;const runtime = readSrc("features/chat/hooks/use-chat-model-runtime.ts");const rollbackStart = runtime.indexOf("engine: stateBeforeUnload.loadedEngine,");@@ -37,6 +41,7 @@ test("a failed switch restores the managed resident's precision and parallel modassert.equal(request.engine, "sglang");assert.equal(request.engine_precision, "bf16");assert.equal(request.engine_parallelism, "pipeline");+ assert.equal(request.engine_kv_cache_dtype, "auto");// The backend reads an explicit 4-bit request without a precision as INT4.assert.equal(request.load_in_4bit, false);});@@ -48,3 +53,16 @@ test("a default-backend resident still rolls back with 4-bit loading", () => {assert.equal(request.engine_parallelism, "tensor");assert.equal(request.load_in_4bit, true);});++test("a failed switch restores the managed resident's KV cache, seed and memory limit", () => {+ const resident = hydrate("org/model", {+ engine: "vllm",+ engine_kv_cache_dtype: "fp8",+ engine_seed: 0,+ engine_gpu_memory_utilization: 0.5,+ });+ const request = rollback(resident, undefined, null, 4096);+ assert.equal(request.engine_kv_cache_dtype, "fp8");+ assert.equal(request.engine_seed, 0);+ assert.equal(request.engine_gpu_memory_utilization, 0.5);+});diff --git a/studio/frontend/tests/resident-config-match-accelerator-matrix.test.ts b/studio/frontend/tests/resident-config-match-accelerator-matrix.test.tsindex d603d2708d4..07601ea1736 100644--- a/studio/frontend/tests/resident-config-match-accelerator-matrix.test.ts+++ b/studio/frontend/tests/resident-config-match-accelerator-matrix.test.ts@@ -491,6 +491,9 @@ test("every PerModelConfig field is either compared or deliberately excluded", ("maxSeqLength","enginePrecision","engineParallelism",+ "engineKvCacheDtype",+ "engineSeed",+ "engineGpuMemoryUtilization",// Qualifies selectedGpuIds rather than adding a dimension of its own: it is read, as// the reconciler's namespace argument, but /status has no field to compare it against."selectedGpuIndexKind",diff --git a/tests/studio/test_model_picker_contracts.py b/tests/studio/test_model_picker_contracts.pyindex 80fab39b392..2cdac1f908f 100644--- a/tests/studio/test_model_picker_contracts.py+++ b/tests/studio/test_model_picker_contracts.py@@ -519,10 +519,9 @@ def test_active_model_config_round_trips_gpu_fields():# Only GGUF carries the offload knobs; an optional engine keeps just its GPU pick.flat = " ".join(src.split())assert "if (!isGguf) {" in flat and ": base; }" in flat- assert (- 'engine === "vllm" || engine === "sglang" ? { ...base, selectedGpuIds, selectedGpuIndexKind }'- in flat- )+ managed = flat.split('engine === "vllm" || engine === "sglang" ? { ...base,', 1)+ assert len(managed) == 2+ assert "selectedGpuIds, selectedGpuIndexKind" in managed[1].split("}", 1)[0]assert "loadedLlamaExtraArgs != null" in srcassert "llamaExtraArgs: [...loadedLlamaExtraArgs]" in srcassert "useActiveModelConfig(" in _read("features/chat/chat-page.tsx")