Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: KV cache dtype, GPU memory limit and seed for vLLM and SGLang loads (#13596)

Daniel Han742a2280638f
diff --git a/studio/backend/core/inference/engine_adapters.py b/studio/backend/core/inference/engine_adapters.py
index 1648cd7ac09..6c15360c99b 100644
--- a/studio/backend/core/inference/engine_adapters.py
+++ b/studio/backend/core/inference/engine_adapters.py
@@ -113,6 +113,9 @@ class EngineAdapter:
served_model_name = None,
):
options = options or {}
+ if options.get("gpu_memory_utilization"):
+ # A user ceiling only lowers Studio's measured budget: above it the engine would OOM.
+ memory_fraction = min(memory_fraction, options["gpu_memory_utilization"])
tool_args = []
if options.get("tool_parser"):
tool_args = ["--tool-call-parser", options["tool_parser"]]
@@ -242,6 +245,20 @@ class EngineAdapter:
if self.name == "vllm" and options.get("max_num_seqs")
else []
),
+ *(
+ [
+ "--kv-cache-dtype",
+ # SGLang names the FP8 format; vLLM's "fp8" is the same E4M3 cache.
+ "fp8_e4m3" if self.name == "sglang" else options["kv_cache_dtype"],
+ ]
+ if options.get("kv_cache_dtype")
+ else []
+ ),
+ *(
+ ["--random-seed" if self.name == "sglang" else "--seed", str(options["seed"])]
+ if options.get("seed") is not None
+ else []
+ ),
*(["--trust-remote-code"] if trust_remote_code else []),
*self.extra_args,
"--host",
diff --git a/studio/backend/core/inference/managed_engine.py b/studio/backend/core/inference/managed_engine.py
index c3f9f3ce35f..e586d33d972 100644
--- a/studio/backend/core/inference/managed_engine.py
+++ b/studio/backend/core/inference/managed_engine.py
@@ -177,6 +177,9 @@ def validate_model(
engine = "vllm",
precision = "auto",
parallelism = "tensor",
+ kv_cache_dtype = "auto",
+ seed = None,
+ gpu_memory_utilization = None,
) -> dict:
"""Check plain metadata before releasing the previous resident model."""
if config.is_local:
@@ -239,6 +242,13 @@ def validate_model(
or (engine == "vllm" and precision == "int4" and parallelism != "pipeline")
else "auto",
}
+ # Unset options stay out of the dict, so a default load keeps today's argv and cache key.
+ if kv_cache_dtype != "auto":
+ options["kv_cache_dtype"] = kv_cache_dtype
+ if seed is not None:
+ options["seed"] = seed
+ if gpu_memory_utilization is not None:
+ options["gpu_memory_utilization"] = gpu_memory_utilization
if reasoning_parser:
from .llama_cpp import detect_reasoning_flags
options["reasoning_parser"] = reasoning_parser
@@ -250,9 +260,9 @@ def validate_model(
else config.identifier,
log_source = "managed engine",
)
- if precision == "fp8" and profile(engine)["platform"] == "cuda":
- # Eager TorchAO FP8 on Ampere: Triton cannot compile its casts, SGLang online FP8 is invalid.
+ if (precision == "fp8" or kv_cache_dtype == "fp8") and profile(engine)["platform"] == "cuda":
from utils.hardware.nvidia import _nvidia_smi_executable
+
result = subprocess.run(
[
_nvidia_smi_executable(),
@@ -269,8 +279,23 @@ def validate_model(
timeout = 60,
check = True,
)
- options["disable_cuda_graph"] = any(float(cap) < 8.9 for cap in result.stdout.splitlines())
+ caps = [float(cap) for cap in result.stdout.splitlines() if cap.strip()]
+ if precision == "fp8":
+ # Eager TorchAO FP8 on Ampere: Triton cannot compile its casts, SGLang online FP8 is invalid.
+ options["disable_cuda_graph"] = any(cap < 8.9 for cap in caps)
+ if kv_cache_dtype == "fp8" and any(cap < 8.0 for cap in caps):
+ # vLLM 0.30 refuses it at startup on Turing ("native FP8 requires SM89+"); Ampere runs it.
+ raise ValueError(
+ "An FP8 KV cache needs an Ampere or newer NVIDIA GPU. "
+ "Choose Model default for KV cache on this GPU."
+ )
metadata = metadata.get("text_config") or metadata
+ if engine == "sglang" and kv_cache_dtype == "fp8" and metadata.get("kv_lora_rank"):
+ # Studio pins SGLang to the Triton attention backend, whose MLA path has no FP8 KV cache.
+ raise ValueError(
+ "SGLang cannot use an FP8 KV cache with this model's attention. "
+ "Choose Model default for KV cache, or use vLLM."
+ )
size = len(gpu_ids or [0])
if size > 1 and parallelism == "tensor":
for field in ("num_attention_heads", "hidden_size", "intermediate_size"):
@@ -421,6 +446,8 @@ def _rocm_visibility(env: dict, gpu_ids) -> dict:
# SGLang raises when its derived gRPC port (HTTP + 10000) exceeds 65535.
_PORT_LIMIT = 55535
+# Launch options that change no compiled artifact, so they share the compile cache.
+_UNCACHED_OPTIONS = frozenset(("seed", "gpu_memory_utilization"))
def _free_port(limit: int | None = _PORT_LIMIT) -> int:
@@ -737,7 +764,15 @@ class ManagedEngine:
return hashlib.sha256(
policy
+ json.dumps(
- [info.get("profile_digest"), model, self.context, gpu_ids, options],
+ [
+ info.get("profile_digest"),
+ model,
+ self.context,
+ gpu_ids,
+ {k: v for k, v in options.items() if k not in _UNCACHED_OPTIONS}
+ if options
+ else options,
+ ],
sort_keys = True,
).encode()
).hexdigest()[:16]
diff --git a/studio/backend/core/inference/orchestrator.py b/studio/backend/core/inference/orchestrator.py
index 2f38176dc3f..e8dac030283 100644
--- a/studio/backend/core/inference/orchestrator.py
+++ b/studio/backend/core/inference/orchestrator.py
@@ -2474,6 +2474,9 @@ class InferenceOrchestrator:
"engine": engine,
"engine_parallelism": (options or {}).get("parallelism", "tensor"),
"engine_precision": (options or {}).get("precision", "auto"),
+ "engine_kv_cache_dtype": (options or {}).get("kv_cache_dtype", "auto"),
+ "engine_seed": (options or {}).get("seed"),
+ "engine_gpu_memory_utilization": (options or {}).get("gpu_memory_utilization"),
"is_vision": (options or {}).get("is_vision", config.is_vision),
"chat_template_info": {
"accepts_multiple_images": bool(
diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.py
index b0eb9376bca..a9348c6eb5a 100644
--- a/studio/backend/models/inference.py
+++ b/studio/backend/models/inference.py
@@ -61,6 +61,20 @@ class LoadRequest(BaseModel):
engine_parallelism: Literal["tensor", "pipeline", "data"] = "tensor"
engine_precision: Literal["auto", "bf16", "fp16", "int4", "int8", "fp8"] = "auto"
+ engine_kv_cache_dtype: Literal["auto", "fp8"] = Field(
+ "auto",
+ description = "Managed engine KV cache dtype; 'fp8' halves KV cache memory (vLLM --kv-cache-dtype).",
+ )
+ engine_seed: Optional[int] = Field(
+ None, ge = 0, le = 2**31 - 1, description = "Managed engine seed (vLLM --seed)."
+ )
+ engine_gpu_memory_utilization: Optional[float] = Field(
+ None,
+ ge = 0.05,
+ le = 0.99,
+ description = "Upper bound on the share of each selected GPU the managed engine may claim "
+ "(vLLM --gpu-memory-utilization). Unset lets Studio size it from free memory.",
+ )
engine: Literal["auto", "vllm", "sglang"] = Field(
"auto",
description = "Inference engine to use. 'auto' selects Studio's default backend; "
@@ -599,6 +613,20 @@ class ValidateModelRequest(BaseModel):
engine_parallelism: Literal["tensor", "pipeline", "data"] = "tensor"
engine_precision: Literal["auto", "bf16", "fp16", "int4", "int8", "fp8"] = "auto"
+ engine_kv_cache_dtype: Literal["auto", "fp8"] = Field(
+ "auto",
+ description = "Managed engine KV cache dtype; 'fp8' halves KV cache memory (vLLM --kv-cache-dtype).",
+ )
+ engine_seed: Optional[int] = Field(
+ None, ge = 0, le = 2**31 - 1, description = "Managed engine seed (vLLM --seed)."
+ )
+ engine_gpu_memory_utilization: Optional[float] = Field(
+ None,
+ ge = 0.05,
+ le = 0.99,
+ description = "Upper bound on the share of each selected GPU the managed engine may claim "
+ "(vLLM --gpu-memory-utilization). Unset lets Studio size it from free memory.",
+ )
engine: Literal["auto", "vllm", "sglang"] = "auto"
model_path: str = Field(..., description = "Model identifier or local path")
# The same inventory handle the picker was shown; see `resolve_inventory_handle`.
@@ -1440,6 +1468,9 @@ class _InferenceRuntimeFields(BaseModel):
engine_parallelism: Literal["tensor", "pipeline", "data"] = "tensor"
engine_precision: Literal["auto", "bf16", "fp16", "int4", "int8", "fp8"] = "auto"
+ engine_kv_cache_dtype: Literal["auto", "fp8"] = "auto"
+ engine_seed: Optional[int] = None
+ engine_gpu_memory_utilization: Optional[float] = None
engine: Literal["auto", "vllm", "sglang"] = Field(
"auto",
description = "Active inference engine. 'auto' denotes Studio's default backend; "
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 3b7eca1feee..6d4b6f88a35 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -8460,6 +8460,7 @@ def _llama_runtime_fields(llama_backend: LlamaCppBackend) -> dict:
engine = "auto",
engine_parallelism = "tensor",
engine_precision = "auto",
+ **_MANAGED_ENGINE_LAUNCH_DEFAULTS,
# Not MLX, so the MLX runtime fields report as absent.
is_mlx = False,
is_npu = False,
@@ -17473,6 +17474,21 @@ def _inherit_resident_load_in_4bit(backend, request, model_identifier: str) -> N
request.load_in_4bit = bool(resident)
+# Managed-engine launch options a load records and a status echoes; the value is "not set".
+_MANAGED_ENGINE_LAUNCH_DEFAULTS = {
+ "engine_kv_cache_dtype": "auto",
+ "engine_seed": None,
+ "engine_gpu_memory_utilization": None,
+}
+
+
+def _managed_engine_launch_fields(model_info) -> dict:
+ return {
+ field: (model_info or {}).get(field, default)
+ for field, default in _MANAGED_ENGINE_LAUNCH_DEFAULTS.items()
+ }
+
+
def _non_gguf_runtime_settings_match(backend, request) -> bool:
"""Whether the resident non-GGUF model already runs the request's load settings.
@@ -17495,6 +17511,9 @@ def _non_gguf_runtime_settings_match(backend, request) -> bool:
request, "engine_parallelism", "tensor"
):
return False
+ for field, default in _MANAGED_ENGINE_LAUNCH_DEFAULTS.items():
+ if entry.get(field, default) != getattr(request, field, default):
+ return False
if list(request.gpu_ids or [0]) != list(entry.get("gpu_ids_requested") or [0]):
return False
if "max_seq_length" in fields_set and int(request.max_seq_length or 0) > 0:
@@ -18276,6 +18295,9 @@ async def _managed_engine_options(request, config, hf_token) -> dict:
request.engine,
request.engine_precision,
request.engine_parallelism,
+ kv_cache_dtype = getattr(request, "engine_kv_cache_dtype", "auto"),
+ seed = getattr(request, "engine_seed", None),
+ gpu_memory_utilization = getattr(request, "engine_gpu_memory_utilization", None),
)
except (ValueError, OSError) as exc:
raise HTTPException(status_code = 400, detail = str(exc)) from exc
@@ -18765,6 +18787,7 @@ async def _load_model_impl(
engine = request.engine,
engine_parallelism = request.engine_parallelism,
engine_precision = request.engine_precision,
+ **_managed_engine_launch_fields(_model_info),
gpu_ids = _model_info.get("gpu_ids"),
requested_gpu_ids = _model_info.get("gpu_ids_requested"),
tensor_parallel = bool(_model_info.get("tensor_parallel", False)),
@@ -19650,6 +19673,7 @@ async def _load_model_impl(
engine = request.engine,
engine_parallelism = request.engine_parallelism,
engine_precision = request.engine_precision,
+ **_managed_engine_launch_fields(_model_info),
gpu_ids = _model_info.get("gpu_ids"),
requested_gpu_ids = _model_info.get("gpu_ids_requested"),
tensor_parallel = bool(_model_info.get("tensor_parallel", False)),
@@ -22476,6 +22500,7 @@ async def _slot_status(current_subject: str):
engine = model_info.get("engine", "auto"),
engine_parallelism = model_info.get("engine_parallelism", "tensor"),
engine_precision = model_info.get("engine_precision", "auto"),
+ **_managed_engine_launch_fields(model_info),
gpu_ids = model_info.get("gpu_ids"),
tensor_parallel = bool(model_info.get("tensor_parallel", False)),
active_model = _active,
diff --git a/studio/backend/routes/settings.py b/studio/backend/routes/settings.py
index 4dd27b5a079..bf32623e7e6 100644
--- a/studio/backend/routes/settings.py
+++ b/studio/backend/routes/settings.py
@@ -996,6 +996,9 @@ class ModelOverridePayload(BaseModel):
engine_parallelism: Optional[Literal["tensor", "pipeline", "data"]] = None
engine_precision: Optional[Literal["auto", "bf16", "fp16", "int4", "int8", "fp8"]] = None
engine: Optional[Literal["auto", "vllm", "sglang"]] = None
+ engine_kv_cache_dtype: Optional[Literal["auto", "fp8"]] = None
+ engine_seed: Optional[int] = Field(default = None, ge = 0, le = 2**31 - 1)
+ engine_gpu_memory_utilization: Optional[float] = Field(default = None, ge = 0.05, le = 0.99)
# None leaves the stored value alone (the UI has no control for flags); [] clears them.
llama_extra_args: Optional[list[str]] = None
# ge=1: the setter drops a falsy value, so reject 0 here instead of discarding it silently.
@@ -2768,6 +2771,16 @@ def update_openai_auto_switch_override(
engine = payload.engine
if payload.engine is not None or is_removal
else get_model_override(target_id).get("engine"),
+ **{
+ field: getattr(payload, field)
+ if field in fields_set or is_removal
+ else get_model_override(target_id).get(field)
+ for field in (
+ "engine_kv_cache_dtype",
+ "engine_seed",
+ "engine_gpu_memory_utilization",
+ )
+ },
max_seq_length = max_seq_length,
custom_context_length = custom_context_length,
kv_cache_dtype = payload.kv_cache_dtype,
diff --git a/studio/backend/tests/test_chat_history_routes.py b/studio/backend/tests/test_chat_history_routes.py
index b3329dd20b7..051194a9224 100644
--- a/studio/backend/tests/test_chat_history_routes.py
+++ b/studio/backend/tests/test_chat_history_routes.py
@@ -673,7 +673,7 @@ def test_chat_inference_settings_covers_frontend_persisted_fields():
block = re.search(r"interface InferenceParams \{(.*?)\n\}", source, re.DOTALL)
assert block, "InferenceParams interface not found in runtime.ts"
omitted = re.search(
- r"PersistedInferenceParams = Partial<\s*Omit<InferenceParams,([^>]*)>", source
+ r"PersistedInferenceParams = Partial<\s*Omit<\s*InferenceParams,([^>]*)>", source
)
assert omitted, "PersistedInferenceParams Omit<> not found in runtime.ts"
persisted = set(re.findall(r"^\s*(\w+)\??:", block.group(1), re.M)) - set(
diff --git a/studio/backend/tests/test_gguf_load_cache_reuse.py b/studio/backend/tests/test_gguf_load_cache_reuse.py
index 51da3829967..ec9d5ea5110 100644
--- a/studio/backend/tests/test_gguf_load_cache_reuse.py
+++ b/studio/backend/tests/test_gguf_load_cache_reuse.py
@@ -1178,6 +1178,9 @@ class TestLoadHubDownloadExclusion:
"engine",
"engine_parallelism",
"engine_precision",
+ "engine_kv_cache_dtype",
+ "engine_seed",
+ "engine_gpu_memory_utilization",
"requires_trust_remote_code",
"speculative_type",
"requested_parallel_slots",
diff --git a/studio/backend/tests/test_managed_engine_launch_options.py b/studio/backend/tests/test_managed_engine_launch_options.py
new file mode 100644
index 00000000000..f4286a1a912
--- /dev/null
+++ b/studio/backend/tests/test_managed_engine_launch_options.py
@@ -0,0 +1,268 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""KV cache dtype, seed and GPU memory ceiling for the managed vLLM / SGLang engines."""
+
+import json
+from types import SimpleNamespace
+
+import pytest
+
+from core.inference.engine_adapters import ADAPTERS
+
+# What validate_model returns for an unquantized text checkpoint with a tool parser.
+_DEFAULT_OPTIONS = {
+ "tool_parser": "hermes",
+ "precision": "auto",
+ "parallelism": "tensor",
+ "is_vision": False,
+ "quantization": None,
+ "disable_cuda_graph": False,
+ "load_format": "auto",
+ "engine_version": "0.31.0",
+}
+
+# The argv main built for _DEFAULT_OPTIONS (launcher path at index 2 left out).
+_MAIN_ARGV = {
+ "vllm": [
+ "/env/bin/python",
+ "-I",
+ "vllm.entrypoints.openai.api_server",
+ "--model",
+ "org/model",
+ "--max-model-len",
+ "4096",
+ "--gpu-memory-utilization",
+ "0.8",
+ "--tensor-parallel-size",
+ "1",
+ "--tool-call-parser",
+ "hermes",
+ "--enable-auto-tool-choice",
+ "--generation-config",
+ "vllm",
+ "--host",
+ "127.0.0.1",
+ "--port",
+ "45678",
+ "--served-model-name",
+ "org/model",
+ "--load-format",
+ "auto",
+ ],
+ "sglang": [
+ "/env/bin/python",
+ "-I",
+ "--model-path",
+ "org/model",
+ "--context-length",
+ "4096",
+ "--mem-fraction-static",
+ "0.8",
+ "--tensor-parallel-size",
+ "1",
+ "--tool-call-parser",
+ "hermes",
+ "--attention-backend",
+ "triton",
+ "--sampling-backend",
+ "pytorch",
+ "--host",
+ "127.0.0.1",
+ "--port",
+ "45678",
+ "--served-model-name",
+ "org/model",
+ "--load-format",
+ "auto",
+ ],
+}
+
+
+def _argv(engine, **extra):
+ argv = ADAPTERS[engine].command(
+ "/env/bin/python",
+ "org/model",
+ 45678,
+ "key",
+ 4096,
+ 0.8,
+ 1,
+ options = {**_DEFAULT_OPTIONS, **extra},
+ )
+ return argv[:2] + argv[3:]
+
+
[email protected]("engine", ["vllm", "sglang"])
+def test_default_options_launch_the_same_argv_as_before(engine):
+ assert _argv(engine) == _MAIN_ARGV[engine]
+
+
+ "engine, kv, seed", [("vllm", "fp8", "--seed"), ("sglang", "fp8_e4m3", "--random-seed")]
+)
+def test_kv_cache_dtype_and_seed_reach_each_engine(engine, kv, seed):
+ argv = _argv(engine, kv_cache_dtype = "fp8", seed = 7)
+ assert argv[argv.index("--kv-cache-dtype") + 1] == kv
+ assert argv[argv.index(seed) + 1] == "7"
+ zero = _argv(engine, seed = 0)
+ assert zero[zero.index(seed) + 1] == "0"
+
+
+ "engine, flag", [("vllm", "--gpu-memory-utilization"), ("sglang", "--mem-fraction-static")]
+)
+def test_memory_ceiling_lowers_but_never_raises_the_measured_budget(engine, flag):
+ lowered = _argv(engine, gpu_memory_utilization = 0.5)
+ assert lowered[lowered.index(flag) + 1] == "0.5"
+ kept = _argv(engine, gpu_memory_utilization = 0.95)
+ assert kept[kept.index(flag) + 1] == "0.8"
+
+
+def _config(tmp_path):
+ (tmp_path / "config.json").write_text(json.dumps({"num_attention_heads": 8}))
+ return SimpleNamespace(is_local = True, path = str(tmp_path))
+
+
+def _compute_cap(monkeypatch, cap):
+ from core.inference import managed_engine
+ monkeypatch.setattr(managed_engine, "profile", lambda engine: {"platform": "cuda"})
+ monkeypatch.setattr(
+ managed_engine.subprocess, "run", lambda argv, **kwargs: SimpleNamespace(stdout = f"{cap}\n")
+ )
+
+
[email protected]("engine", ["vllm", "sglang"])
+def test_unset_options_stay_out_of_the_launch_options(tmp_path, engine, monkeypatch):
+ from core.inference.managed_engine import validate_model
+
+ _compute_cap(monkeypatch, "9.0")
+ options = validate_model(_config(tmp_path), engine = engine)
+ assert not {"kv_cache_dtype", "seed", "gpu_memory_utilization"} & set(options)
+ options = validate_model(
+ _config(tmp_path),
+ engine = engine,
+ kv_cache_dtype = "fp8",
+ seed = 0,
+ gpu_memory_utilization = 0.6,
+ )
+ assert (options["kv_cache_dtype"], options["seed"], options["gpu_memory_utilization"]) == (
+ "fp8",
+ 0,
+ 0.6,
+ )
+
+
+def test_seed_and_memory_ceiling_share_the_compile_cache():
+ from core.inference.managed_engine import ManagedEngine
+
+ engine = ManagedEngine("vllm")
+ info = {"profile_digest": "d"}
+ base = engine._cache_key(info, "m", [0], dict(_DEFAULT_OPTIONS))
+ assert base == engine._cache_key(
+ info, "m", [0], {**_DEFAULT_OPTIONS, "seed": 3, "gpu_memory_utilization": 0.5}
+ )
+ assert base != engine._cache_key(info, "m", [0], {**_DEFAULT_OPTIONS, "kv_cache_dtype": "fp8"})
+ assert engine._cache_key(info, "m", [0], None) == engine._cache_key(info, "m", [0], None)
+
+
+ "field, value",
+ [
+ ("engine_kv_cache_dtype", "fp8"),
+ ("engine_seed", 1),
+ ("engine_gpu_memory_utilization", 0.5),
+ ],
+)
+def test_a_changed_launch_option_reloads_the_resident_engine(field, value):
+ from models.inference import LoadRequest
+ from routes.inference import _non_gguf_runtime_settings_match
+
+ backend = SimpleNamespace(
+ active_model_name = "model",
+ models = {"model": {"engine": "vllm", "gpu_ids_requested": [0]}},
+ )
+ request = LoadRequest(model_path = "model", engine = "vllm", gpu_ids = [0])
+ assert _non_gguf_runtime_settings_match(backend, request)
+ assert not _non_gguf_runtime_settings_match(backend, request.model_copy(update = {field: value}))
+ backend.models["model"][field] = value
+ assert _non_gguf_runtime_settings_match(backend, request.model_copy(update = {field: value}))
+
+
+def test_overrides_store_only_set_options_and_load_them_back():
+ from utils.openai_auto_switch_settings import (
+ model_override_load_kwargs,
+ normalize_model_override,
+ )
+
+ assert normalize_model_override(
+ {
+ "engine": "vllm",
+ "engine_kv_cache_dtype": "auto",
+ "engine_seed": None,
+ "engine_gpu_memory_utilization": None,
+ }
+ ) == {"engine": "vllm"}
+ entry = normalize_model_override(
+ {
+ "engine": "vllm",
+ "engine_kv_cache_dtype": "fp8",
+ "engine_seed": 0,
+ "engine_gpu_memory_utilization": 0.75,
+ }
+ )
+ assert entry == {
+ "engine": "vllm",
+ "engine_kv_cache_dtype": "fp8",
+ "engine_seed": 0,
+ "engine_gpu_memory_utilization": 0.75,
+ }
+ assert normalize_model_override({"engine_gpu_memory_utilization": 1.5}) == {}
+ kwargs = model_override_load_kwargs(entry, is_gguf = False)
+ assert kwargs["engine_kv_cache_dtype"] == "fp8"
+ assert kwargs["engine_seed"] == 0
+ assert kwargs["engine_gpu_memory_utilization"] == 0.75
+ assert "engine_seed" not in model_override_load_kwargs(entry, is_gguf = True)
+
+
[email protected]("cap,ok", [("7.5", False), ("8.0", True), ("8.9", True)])
+def test_an_fp8_kv_cache_is_refused_before_ampere(tmp_path, monkeypatch, cap, ok):
+ # Colab T4, vLLM 0.30: "FP8 KV cache is not supported ... on Tesla T4 (compute capability 7.5)".
+ from core.inference.managed_engine import validate_model
+ _compute_cap(monkeypatch, cap)
+ if ok:
+ assert validate_model(_config(tmp_path), kv_cache_dtype = "fp8")["kv_cache_dtype"] == "fp8"
+ else:
+ with pytest.raises(ValueError, match = "Ampere"):
+ validate_model(_config(tmp_path), kv_cache_dtype = "fp8")
+
+
+def test_a_default_kv_cache_never_probes_the_gpu(tmp_path, monkeypatch):
+ from core.inference import managed_engine
+
+ def refuse(argv, **kwargs):
+ raise AssertionError("nvidia-smi probed for a default KV cache")
+
+ monkeypatch.setattr(managed_engine, "profile", lambda engine: {"platform": "cuda"})
+ monkeypatch.setattr(managed_engine.subprocess, "run", refuse)
+ managed_engine.validate_model(_config(tmp_path), kv_cache_dtype = "auto")
+
+
[email protected]("engine,refused", [("sglang", True), ("vllm", False)])
+def test_sglang_refuses_an_fp8_kv_cache_on_mla_models(tmp_path, monkeypatch, engine, refused):
+ # Studio pins SGLang's Triton attention backend, whose MLA path has no FP8 KV cache.
+ from core.inference.managed_engine import validate_model
+
+ _compute_cap(monkeypatch, "9.0")
+ (tmp_path / "config.json").write_text(
+ json.dumps({"num_attention_heads": 8, "kv_lora_rank": 512})
+ )
+ config = SimpleNamespace(is_local = True, path = str(tmp_path))
+ if refused:
+ with pytest.raises(ValueError, match = "MLA|attention"):
+ validate_model(config, engine = engine, kv_cache_dtype = "fp8")
+ else:
+ assert (
+ validate_model(config, engine = engine, kv_cache_dtype = "fp8")["kv_cache_dtype"] == "fp8"
+ )
+ assert "kv_cache_dtype" not in validate_model(config, engine = engine)
diff --git a/studio/backend/utils/openai_auto_switch_settings.py b/studio/backend/utils/openai_auto_switch_settings.py
index ad79892aa8d..ee2f8253b2a 100644
--- a/studio/backend/utils/openai_auto_switch_settings.py
+++ b/studio/backend/utils/openai_auto_switch_settings.py
@@ -446,6 +446,18 @@ def normalize_model_override(
entry["engine_precision"] = payload["engine_precision"]
if payload.get("engine") in ("vllm", "sglang"):
entry["engine"] = payload["engine"]
+ if payload.get("engine_kv_cache_dtype") == "fp8":
+ entry["engine_kv_cache_dtype"] = "fp8"
+ seed = _bounded_int(payload.get("engine_seed"), minimum = 0, maximum = 2**31 - 1)
+ if seed is not None:
+ entry["engine_seed"] = seed
+ utilization = payload.get("engine_gpu_memory_utilization")
+ if (
+ isinstance(utilization, (int, float))
+ and not isinstance(utilization, bool)
+ and 0.05 <= utilization <= 0.99
+ ):
+ entry["engine_gpu_memory_utilization"] = round(float(utilization), 3)
extra_args = payload.get("llama_extra_args")
if isinstance(extra_args, (list, tuple)) and extra_args:
@@ -637,6 +649,9 @@ def model_override_load_kwargs(override: dict[str, Any], *, is_gguf: bool) -> di
kwargs["engine"] = override["engine"]
kwargs["engine_parallelism"] = override.get("engine_parallelism", "tensor")
kwargs["engine_precision"] = override.get("engine_precision", "auto")
+ for key in ("engine_kv_cache_dtype", "engine_seed", "engine_gpu_memory_utilization"):
+ if key in override:
+ kwargs[key] = override[key]
kwargs["load_in_4bit"] = False
if override.get("gpu_ids") is not None:
kwargs["gpu_ids"] = override["gpu_ids"]
diff --git a/studio/frontend/scripts/check-bundle-budget.ts b/studio/frontend/scripts/check-bundle-budget.ts
index ed1cfd6923b..658c15a1621 100644
--- a/studio/frontend/scripts/check-bundle-budget.ts
+++ b/studio/frontend/scripts/check-bundle-budget.ts
@@ -80,8 +80,11 @@ export const BUDGET = {
// no single import doing it. The largest step is the audio model selector fix (#13517, +2.9 KB raw /
// +0.9 KB transfer); thinking traces (#13545), MLX precision folders (#13565) and vLLM thinking
// effort (#13571) add the rest. Raised by that growth so main's margin stays where it was.
- transferBytes: 2_025_000,
- rawBytes: 6_737_000,
+ // The KV cache, GPU memory limit and seed engine options (#13596) add 5.7 KB raw / 1.4 KB transfer
+ // (6,573.5 -> 6,579.2 KB raw, 1,976.9 -> 1,978.3 KB transfer against abfe3142c, one machine and
+ // build), inline in the eager chat and settings chunks like the other engine options; raised by that.
+ transferBytes: 2_026_500,
+ rawBytes: 6_743_000,
};
// The chunk count is reported but not budgeted. Splitting a page out of the entry raises it while lowering the
diff --git a/studio/frontend/src/features/api-monitor/components/saved-model-settings.tsx b/studio/frontend/src/features/api-monitor/components/saved-model-settings.tsx
index 778237bfcb8..2fef1cc0bc3 100644
--- a/studio/frontend/src/features/api-monitor/components/saved-model-settings.tsx
+++ b/studio/frontend/src/features/api-monitor/components/saved-model-settings.tsx
@@ -40,6 +40,17 @@ function describeOverride(override: ApiModelOverride): string[] {
if (override.engine_parallelism && override.engine_parallelism !== "tensor") {
parts.push(`${override.engine_parallelism} parallel`);
}
+ if (override.engine_kv_cache_dtype === "fp8") {
+ parts.push("FP8 KV cache");
+ }
+ if (override.engine_seed != null) {
+ parts.push(`seed ${override.engine_seed}`);
+ }
+ if (override.engine_gpu_memory_utilization != null) {
+ parts.push(
+ `${Math.round(override.engine_gpu_memory_utilization * 100)}% GPU memory`,
+ );
+ }
if (override.custom_context_length) {
parts.push(`${override.custom_context_length.toLocaleString()} context`);
}
diff --git a/studio/frontend/src/features/chat/api/chat-api.ts b/studio/frontend/src/features/chat/api/chat-api.ts
index ab03bbd1236..3b4b5ffd944 100644
--- a/studio/frontend/src/features/chat/api/chat-api.ts
+++ b/studio/frontend/src/features/chat/api/chat-api.ts
@@ -487,6 +487,10 @@ export async function validateModel(
engine: payload.engine ?? "auto",
engine_precision: payload.engine_precision ?? "auto",
engine_parallelism: payload.engine_parallelism ?? "tensor",
+ engine_kv_cache_dtype: payload.engine_kv_cache_dtype ?? "auto",
+ engine_seed: payload.engine_seed ?? null,
+ engine_gpu_memory_utilization:
+ payload.engine_gpu_memory_utilization ?? null,
// Intended load settings so validate's preflight matches the follow-up /load.
max_seq_length: payload.max_seq_length,
load_in_4bit: payload.load_in_4bit,
diff --git a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts
index c1048de6dbd..83d5bbe21da 100644
--- a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts
+++ b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts
@@ -182,6 +182,11 @@ import {
loadManagedLlamaFlags,
} from "@/features/model-picker/api/llama-flags";
import { usePlatformStore } from "@/config/env";
+import {
+ DEFAULT_ENGINE_LAUNCH_FIELDS,
+ engineLaunchFieldsFrom,
+ engineLaunchFieldsFromParams,
+} from "../lib/engine-launch-fields";
import type {
ChatLoraSummary,
ChatModelRow,
@@ -2005,6 +2010,9 @@ export function useChatModelRuntime() {
engine: "auto",
enginePrecision: "auto",
engineParallelism: "tensor",
+ engineKvCacheDtype: "auto",
+ engineSeed: null,
+ engineGpuMemoryUtilization: null,
});
}
const requestedEngine =
@@ -2281,6 +2289,7 @@ export function useChatModelRuntime() {
engine: stateBeforeUnload.params.engine ?? "auto",
engine_precision: stateBeforeUnload.params.enginePrecision ?? "auto",
engine_parallelism: stateBeforeUnload.params.engineParallelism ?? "tensor",
+ ...engineLaunchFieldsFromParams(stateBeforeUnload.params),
};
try {
// Lightweight pre-flight validation: avoid unloading a working model if the new identifier is
@@ -2378,6 +2387,7 @@ export function useChatModelRuntime() {
model_path: loadPath,
engine_precision: stateBeforeUnload.params.enginePrecision ?? "auto",
engine_parallelism: stateBeforeUnload.params.engineParallelism ?? "tensor",
+ ...engineLaunchFieldsFromParams(stateBeforeUnload.params),
engine: isGguf ? "auto" : (stateBeforeUnload.params.engine ?? "auto"),
nativePathLease: validateNativePathLease,
hf_token: hfToken,
@@ -2463,6 +2473,7 @@ export function useChatModelRuntime() {
engine,
engine_precision: "auto",
engine_parallelism: "tensor",
+ ...DEFAULT_ENGINE_LAUNCH_FIELDS,
};
engineSwitched = true;
downloadComplete = true;
@@ -3105,6 +3116,7 @@ export function useChatModelRuntime() {
loadedEngine: loadResponse.engine ?? "auto",
loadedEnginePrecision: loadResponse.engine_precision ?? "auto",
loadedEngineParallelism: loadResponse.engine_parallelism ?? "tensor",
+ loadedEngineLaunch: engineLaunchFieldsFrom(loadResponse),
});
// Remembered so auto-load re-picks what the user ran, not the smallest. Native file-picker paths
// need a signed, expiring lease, so they stay out.
@@ -3158,6 +3170,7 @@ export function useChatModelRuntime() {
engine: stateBeforeUnload.loadedEngine,
engine_precision: stateBeforeUnload.loadedEnginePrecision,
engine_parallelism: stateBeforeUnload.loadedEngineParallelism,
+ ...stateBeforeUnload.loadedEngineLaunch,
nativePathLease: rollbackNativePathLease,
hf_token: hfToken,
max_seq_length: rollbackMaxSeqLength,
diff --git a/studio/frontend/src/features/chat/lib/apply-inference-status-to-store.ts b/studio/frontend/src/features/chat/lib/apply-inference-status-to-store.ts
index 48bdb82937a..bbfbdce9a88 100644
--- a/studio/frontend/src/features/chat/lib/apply-inference-status-to-store.ts
+++ b/studio/frontend/src/features/chat/lib/apply-inference-status-to-store.ts
@@ -44,6 +44,7 @@ import { sameGpuSelection } from "@/hooks/gpu-selection";
import { resolveBatchSizeSeed } from "./resolve-batch-size-seed";
import { resolveChatTemplateSeed } from "./resolve-chat-template-seed";
import { resolveCtxPinSeed } from "./resolve-ctx-pin-seed";
+import { engineLaunchFieldsFrom } from "./engine-launch-fields";
import { resolveLlamaExtraArgsSeed } from "./resolve-llama-extra-args-seed";
import { shouldSeedVisionSwitch } from "./resolve-vision-switch-seed";
@@ -148,6 +149,18 @@ export type ApplyInferenceStatusOptions = {
adoptingExistingServerModel?: boolean;
};
+/** The resident engine launch settings, in the store's params shape. */
+function residentEngineParams(status: InferenceStatusResponse) {
+ return {
+ engine: status.engine ?? "auto",
+ enginePrecision: status.engine_precision ?? "auto",
+ engineParallelism: status.engine_parallelism ?? "tensor",
+ engineKvCacheDtype: status.engine_kv_cache_dtype ?? "auto",
+ engineSeed: status.engine_seed ?? null,
+ engineGpuMemoryUtilization: status.engine_gpu_memory_utilization ?? null,
+ } as const;
+}
+
/** Mirror refresh() hydration so adopted CLI models get reasoning/tools flags. */
export function applyActiveModelStatusToStore(
status: InferenceStatusResponse,
@@ -163,13 +176,19 @@ export function applyActiveModelStatusToStore(
loadedEngine: status.engine ?? "auto",
loadedEnginePrecision: status.engine_precision ?? "auto",
loadedEngineParallelism: status.engine_parallelism ?? "tensor",
+ loadedEngineLaunch: engineLaunchFieldsFrom(status),
});
// Before the settings panel can open on it, which reads only the repo id.
adoptCachedRepoConfig(checkpointId, status.gguf_variant ?? null);
const store = useChatRuntimeStore.getState();
- if ((store.params.engine ?? "auto") !== (status.engine ?? "auto") || (store.params.enginePrecision ?? "auto") !== (status.engine_precision ?? "auto") || (store.params.engineParallelism ?? "tensor") !== (status.engine_parallelism ?? "tensor")) {
- store.setParams({ ...store.params, engine: status.engine ?? "auto", enginePrecision: status.engine_precision ?? "auto", engineParallelism: status.engine_parallelism ?? "tensor" });
+ const residentEngine = residentEngineParams(status);
+ if (
+ (Object.keys(residentEngine) as (keyof typeof residentEngine)[]).some(
+ (key) => (store.params[key] ?? null) !== residentEngine[key],
+ )
+ ) {
+ store.setParams({ ...store.params, ...residentEngine });
}
const previousCheckpoint =
options.previousCheckpoint ?? store.params.checkpoint;
@@ -177,7 +196,7 @@ export function applyActiveModelStatusToStore(
if (status.inference) {
store.setParams(
mergeBackendRecommendedInference({
- current: { ...store.params, engine: status.engine ?? "auto", enginePrecision: status.engine_precision ?? "auto", engineParallelism: status.engine_parallelism ?? "tensor" },
+ current: { ...store.params, ...residentEngine },
response: status,
modelId: checkpointId,
presetSource: store.activePresetSource,
diff --git a/studio/frontend/src/features/chat/lib/engine-launch-fields.ts b/studio/frontend/src/features/chat/lib/engine-launch-fields.ts
new file mode 100644
index 00000000000..4d0f499605c
--- /dev/null
+++ b/studio/frontend/src/features/chat/lib/engine-launch-fields.ts
@@ -0,0 +1,44 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import type { LoadModelRequest } from "../types/api";
+import type { InferenceParams } from "../types/runtime";
+
+/** vLLM / SGLang launch options beyond precision and parallelism; unset means the engine default. */
+export type EngineLaunchFields = Required<
+ Pick<
+ LoadModelRequest,
+ "engine_kv_cache_dtype" | "engine_seed" | "engine_gpu_memory_utilization"
+ >
+>;
+
+export const DEFAULT_ENGINE_LAUNCH_FIELDS: EngineLaunchFields = {
+ engine_kv_cache_dtype: "auto",
+ engine_seed: null,
+ engine_gpu_memory_utilization: null,
+};
+
+export function engineLaunchFieldsFromParams(
+ params: Pick<
+ InferenceParams,
+ "engineKvCacheDtype" | "engineSeed" | "engineGpuMemoryUtilization"
+ >,
+): EngineLaunchFields {
+ return {
+ engine_kv_cache_dtype: params.engineKvCacheDtype ?? "auto",
+ engine_seed: params.engineSeed ?? null,
+ engine_gpu_memory_utilization: params.engineGpuMemoryUtilization ?? null,
+ };
+}
+
+/** What a load or status response says the resident engine was launched with. */
+export function engineLaunchFieldsFrom(
+ response: Partial<EngineLaunchFields>,
+): EngineLaunchFields {
+ return {
+ engine_kv_cache_dtype: response.engine_kv_cache_dtype ?? "auto",
+ engine_seed: response.engine_seed ?? null,
+ engine_gpu_memory_utilization:
+ response.engine_gpu_memory_utilization ?? null,
+ };
+}
diff --git a/studio/frontend/src/features/chat/lib/resident-config-match.ts b/studio/frontend/src/features/chat/lib/resident-config-match.ts
index 122b490aee5..6b2662f615a 100644
--- a/studio/frontend/src/features/chat/lib/resident-config-match.ts
+++ b/studio/frontend/src/features/chat/lib/resident-config-match.ts
@@ -24,6 +24,9 @@ type ResidentRuntime = Pick<
| "engine"
| "engine_parallelism"
| "engine_precision"
+ | "engine_kv_cache_dtype"
+ | "engine_seed"
+ | "engine_gpu_memory_utilization"
| "context_length"
| "requested_context_length"
| "cache_type_kv"
@@ -612,7 +615,12 @@ export function residentRuntimeMatchesConfig(
if (status.engine === "vllm" || status.engine === "sglang") {
if (
(status.engine_precision ?? "auto") !== (config.enginePrecision ?? "auto") ||
- (status.engine_parallelism ?? "tensor") !== (config.engineParallelism ?? "tensor")
+ (status.engine_parallelism ?? "tensor") !== (config.engineParallelism ?? "tensor") ||
+ (status.engine_kv_cache_dtype ?? "auto") !==
+ (config.engineKvCacheDtype ?? "auto") ||
+ (status.engine_seed ?? null) !== (config.engineSeed ?? null) ||
+ (status.engine_gpu_memory_utilization ?? null) !==
+ (config.engineGpuMemoryUtilization ?? null)
)
return false;
if (
diff --git a/studio/frontend/src/features/chat/shared-composer.tsx b/studio/frontend/src/features/chat/shared-composer.tsx
index 4b3497a336b..e828add7d77 100644
--- a/studio/frontend/src/features/chat/shared-composer.tsx
+++ b/studio/frontend/src/features/chat/shared-composer.tsx
@@ -1,6 +1,11 @@
// SPDX-License-Identifier: AGPL-3.0-only
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+import {
+ DEFAULT_ENGINE_LAUNCH_FIELDS,
+ engineLaunchFieldsFrom,
+ engineLaunchFieldsFromParams,
+} from "./lib/engine-launch-fields";
import { useChatArtifactsStore } from "./artifacts/store";
import { mlxRuntimeStateFrom } from "./lib/mlx-runtime-state";
import { offloadCountsFrom, offloadWarning } from "./lib/partial-offload";
@@ -1755,6 +1760,7 @@ export function SharedComposer({
engine: paneEngine,
engine_precision: ownConfig.enginePrecision ?? "auto",
engine_parallelism: ownConfig.engineParallelism ?? "tensor",
+ ...engineLaunchFieldsFromParams(ownConfig),
load_in_4bit: paneEngine === "auto",
...(paneEngine !== "auto" && ownConfig.selectedGpuIds !== undefined
? {
@@ -1850,6 +1856,7 @@ export function SharedComposer({
engine,
engine_precision: "auto",
engine_parallelism: "tensor",
+ ...DEFAULT_ENGINE_LAUNCH_FIELDS,
load_in_4bit: false,
...(ownConfig.selectedGpuIds !== undefined
? {
@@ -2098,6 +2105,7 @@ export function SharedComposer({
loadedEngine: resp.engine ?? "auto",
loadedEnginePrecision: resp.engine_precision ?? "auto",
loadedEngineParallelism: resp.engine_parallelism ?? "tensor",
+ loadedEngineLaunch: engineLaunchFieldsFrom(resp),
loadedDisableVision: resp.disable_vision ?? false,
// Adopted from the echo like the knob above: this pane loaded its own model, so the editable
// value must follow it or Advanced Settings shows the other pane's Vision state.
diff --git a/studio/frontend/src/features/chat/stores/chat-runtime-store.ts b/studio/frontend/src/features/chat/stores/chat-runtime-store.ts
index 0466e3b2e44..49436a060dd 100644
--- a/studio/frontend/src/features/chat/stores/chat-runtime-store.ts
+++ b/studio/frontend/src/features/chat/stores/chat-runtime-store.ts
@@ -126,6 +126,10 @@ import {
CHAT_SPECULATIVE_TYPE_KEY,
} from "./chat-runtime-keys";
import { useExternalProvidersStore } from "./external-providers-store";
+import {
+ DEFAULT_ENGINE_LAUNCH_FIELDS,
+ type EngineLaunchFields,
+} from "../lib/engine-launch-fields";
export {
CHAT_GPU_MEMORY_MODE_KEY,
@@ -2341,6 +2345,7 @@ type ChatRuntimeStore = {
loadedEngine: "auto" | "vllm" | "sglang";
loadedEnginePrecision: NonNullable<InferenceParams["enginePrecision"]>;
loadedEngineParallelism: NonNullable<InferenceParams["engineParallelism"]>;
+ loadedEngineLaunch: EngineLaunchFields;
/** The backend's own is_gguf for the loaded model; null until one loads. Set wherever
* loadedContextLength is, so a context never arrives unattributed. */
loadedIsGguf: boolean | null;
@@ -4224,6 +4229,7 @@ export const useChatRuntimeStore = create<ChatRuntimeStore>((set, get) => ({
loadedEngine: "auto",
loadedEnginePrecision: "auto",
loadedEngineParallelism: "tensor",
+ loadedEngineLaunch: DEFAULT_ENGINE_LAUNCH_FIELDS,
loadedIsGguf: null,
loadedIsMlx: null,
loadedContextEnforced: null,
diff --git a/studio/frontend/src/features/chat/types/api.ts b/studio/frontend/src/features/chat/types/api.ts
index 548cc0e1efb..d8d164700fc 100644
--- a/studio/frontend/src/features/chat/types/api.ts
+++ b/studio/frontend/src/features/chat/types/api.ts
@@ -49,6 +49,9 @@ export interface ListLorasResponse {
export interface LoadModelRequest {
engine_parallelism?: "tensor" | "pipeline" | "data";
engine_precision?: "auto" | "bf16" | "fp16" | "int4" | "int8" | "fp8";
+ engine_kv_cache_dtype?: "auto" | "fp8";
+ engine_seed?: number | null;
+ engine_gpu_memory_utilization?: number | null;
engine?: "auto" | "vllm" | "sglang";
model_path: string;
/** Opaque client attempt ID used to cancel only this in-flight load. */
@@ -233,6 +236,9 @@ export function isMultimodalResponse(
export interface LoadModelResponse {
engine_parallelism?: "tensor" | "pipeline" | "data";
engine_precision?: "auto" | "bf16" | "fp16" | "int4" | "int8" | "fp8";
+ engine_kv_cache_dtype?: "auto" | "fp8";
+ engine_seed?: number | null;
+ engine_gpu_memory_utilization?: number | null;
engine?: "auto" | "vllm" | "sglang";
is_mlx?: boolean;
evicted?: string[];
@@ -361,6 +367,9 @@ export interface UnloadModelRequest {
export interface InferenceStatusResponse {
engine_parallelism?: "tensor" | "pipeline" | "data";
engine_precision?: "auto" | "bf16" | "fp16" | "int4" | "int8" | "fp8";
+ engine_kv_cache_dtype?: "auto" | "fp8";
+ engine_seed?: number | null;
+ engine_gpu_memory_utilization?: number | null;
engine?: "auto" | "vllm" | "sglang";
is_mlx?: boolean;
is_npu?: boolean;
diff --git a/studio/frontend/src/features/chat/types/runtime.ts b/studio/frontend/src/features/chat/types/runtime.ts
index 2f176f63958..0e3c03ce7b0 100644
--- a/studio/frontend/src/features/chat/types/runtime.ts
+++ b/studio/frontend/src/features/chat/types/runtime.ts
@@ -6,6 +6,9 @@ export type MinPMode = "server-default" | "custom";
export interface InferenceParams {
engineParallelism?: "tensor" | "pipeline" | "data";
enginePrecision?: "auto" | "bf16" | "fp16" | "int4" | "int8" | "fp8";
+ engineKvCacheDtype?: "auto" | "fp8";
+ engineSeed?: number | null;
+ engineGpuMemoryUtilization?: number | null;
engine?: "auto" | "vllm" | "sglang";
temperature: number;
topP: number;
@@ -59,7 +62,16 @@ export function modelReadsSamplingSeed(
/** The params that survive a reload. `checkpoint` names the model rather than being one of its
* settings, so it is not one of them. */
export type PersistedInferenceParams = Partial<
- Omit<InferenceParams, "checkpoint" | "engine" | "enginePrecision" | "engineParallelism">
+ Omit<
+ InferenceParams,
+ | "checkpoint"
+ | "engine"
+ | "enginePrecision"
+ | "engineParallelism"
+ | "engineKvCacheDtype"
+ | "engineSeed"
+ | "engineGpuMemoryUtilization"
+ >
>;
export const DEFAULT_INFERENCE_PARAMS: InferenceParams = {
diff --git a/studio/frontend/src/features/model-picker/api/model-overrides.ts b/studio/frontend/src/features/model-picker/api/model-overrides.ts
index 0c6c69bd01c..0ed4c3c50dc 100644
--- a/studio/frontend/src/features/model-picker/api/model-overrides.ts
+++ b/studio/frontend/src/features/model-picker/api/model-overrides.ts
@@ -29,6 +29,9 @@ export interface ApiModelOverride {
engine_parallelism?: "tensor" | "pipeline" | "data";
engine_precision?: "auto" | "bf16" | "fp16" | "int4" | "int8" | "fp8";
engine?: "auto" | "vllm" | "sglang";
+ engine_kv_cache_dtype?: "auto" | "fp8";
+ engine_seed?: number | null;
+ engine_gpu_memory_utilization?: number | null;
// biome-ignore lint/style/useNamingConvention: API schema
llama_extra_args?: string[];
// biome-ignore lint/style/useNamingConvention: API schema
@@ -337,6 +340,13 @@ export function fromApiOverride(
engineParallelism: override.engine_parallelism ?? local.engineParallelism ?? "tensor",
enginePrecision:
override.engine_precision ?? local.enginePrecision ?? "auto",
+ engineKvCacheDtype:
+ override.engine_kv_cache_dtype ?? local.engineKvCacheDtype ?? "auto",
+ engineSeed: override.engine_seed ?? local.engineSeed ?? null,
+ engineGpuMemoryUtilization:
+ override.engine_gpu_memory_utilization ??
+ local.engineGpuMemoryUtilization ??
+ null,
customContextLength: serverStatesPin
? (override.custom_context_length ?? null)
: local.customContextLength,
@@ -400,6 +410,9 @@ export function toApiOverride(config: PerModelConfig | null): ApiModelOverride {
engine: config.engine ?? "auto",
engine_precision: config.enginePrecision ?? "auto",
engine_parallelism: config.engineParallelism ?? "tensor",
+ engine_kv_cache_dtype: config.engineKvCacheDtype ?? "auto",
+ engine_seed: config.engineSeed ?? null,
+ engine_gpu_memory_utilization: config.engineGpuMemoryUtilization ?? null,
};
if (config.maxSeqLength && config.maxSeqLength > 0) {
payload.max_seq_length = config.maxSeqLength;
diff --git a/studio/frontend/src/features/model-picker/components/inference-engines.tsx b/studio/frontend/src/features/model-picker/components/inference-engines.tsx
index 59376712bf6..39c40832522 100644
--- a/studio/frontend/src/features/model-picker/components/inference-engines.tsx
+++ b/studio/frontend/src/features/model-picker/components/inference-engines.tsx
@@ -2,6 +2,7 @@
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
import { Button } from "@/components/ui/button";
+import { Input } from "@/components/ui/input";
import { Switch } from "@/components/ui/switch";
import {
Select,
@@ -37,6 +38,8 @@ const PRECISION_LABELS = [
["int8", "INT8 (8-bit)"],
["fp8", "FP8 (8-bit)"],
] as const;
+// Ceilings below Studio's measured budget; it never raises the engine past what is free.
+const GPU_MEMORY_LIMITS = [0.95, 0.9, 0.8, 0.7, 0.5, 0.3] as const;
export function EngineInstall({
engine,
@@ -293,6 +296,12 @@ export function InferenceEnginePicker({
onParallelismChange,
precision = "auto",
onPrecisionChange,
+ kvCacheDtype = "auto",
+ onKvCacheDtypeChange,
+ gpuMemoryUtilization = null,
+ onGpuMemoryUtilizationChange,
+ seed = null,
+ onSeedChange,
onGpuChange,
}: {
value: InferenceEngine;
@@ -306,6 +315,12 @@ export function InferenceEnginePicker({
onPrecisionChange: (
precision: "auto" | "bf16" | "fp16" | "int4" | "int8" | "fp8",
) => void;
+ kvCacheDtype?: "auto" | "fp8";
+ onKvCacheDtypeChange: (dtype: "auto" | "fp8") => void;
+ gpuMemoryUtilization?: number | null;
+ onGpuMemoryUtilizationChange: (fraction: number | null) => void;
+ seed?: number | null;
+ onSeedChange: (seed: number | null) => void;
onGpuChange: (ids: number[]) => void;
}) {
const { engines, error } = useEngines();
@@ -501,6 +516,113 @@ export function InferenceEnginePicker({
{t("managedEngines.precisionHelp")}
</p>
</div>
+ <div className="space-y-2">
+ <div className="flex items-center justify-between gap-3 text-ui-13">
+ <span>{t("managedEngines.kvCache")}</span>
+ <Select
+ value={kvCacheDtype}
+ onValueChange={(value) =>
+ onKvCacheDtypeChange(value as typeof kvCacheDtype)
+ }
+ >
+ <SelectTrigger
+ className="w-44"
+ aria-label={t("managedEngines.kvCache")}
+ >
+ <SelectValue />
+ </SelectTrigger>
+ <SelectContent>
+ <SelectItem value="auto">
+ {t("managedEngines.precisionAuto")}
+ </SelectItem>
+ <SelectItem value="fp8">
+ {t("managedEngines.kvCacheFp8")}
+ </SelectItem>
+ </SelectContent>
+ </Select>
+ </div>
+ <p className="text-ui-12 text-muted-foreground">
+ {t("managedEngines.kvCacheHelp")}
+ </p>
+ </div>
+ <div className="space-y-2">
+ <div className="flex items-center justify-between gap-3 text-ui-13">
+ <span>{t("managedEngines.gpuMemory")}</span>
+ <Select
+ value={
+ gpuMemoryUtilization == null
+ ? "auto"
+ : String(gpuMemoryUtilization)
+ }
+ onValueChange={(value) =>
+ onGpuMemoryUtilizationChange(
+ value === "auto" ? null : Number(value),
+ )
+ }
+ >
+ <SelectTrigger
+ className="w-44"
+ aria-label={t("managedEngines.gpuMemory")}
+ >
+ <SelectValue />
+ </SelectTrigger>
+ <SelectContent>
+ <SelectItem value="auto">
+ {t("managedEngines.gpuMemoryAuto")}
+ </SelectItem>
+ {/* A stored value off the preset list stays selectable. */}
+ {[
+ ...new Set([
+ ...GPU_MEMORY_LIMITS,
+ ...(gpuMemoryUtilization == null
+ ? []
+ : [gpuMemoryUtilization]),
+ ]),
+ ]
+ .sort((a, b) => b - a)
+ .map((fraction) => (
+ <SelectItem key={fraction} value={String(fraction)}>
+ {`${Math.round(fraction * 100)}%`}
+ </SelectItem>
+ ))}
+ </SelectContent>
+ </Select>
+ </div>
+ <p className="text-ui-12 text-muted-foreground">
+ {t("managedEngines.gpuMemoryHelp")}
+ </p>
+ </div>
+ <div className="space-y-2">
+ <div className="flex items-center justify-between gap-3 text-ui-13">
+ <span>{t("managedEngines.seed")}</span>
+ <Input
+ className="w-44"
+ type="number"
+ inputMode="numeric"
+ min={0}
+ max={2 ** 31 - 1}
+ step={1}
+ aria-label={t("managedEngines.seed")}
+ placeholder={t("managedEngines.seedPlaceholder")}
+ value={seed ?? ""}
+ onChange={(event) => {
+ const raw = event.target.value.trim();
+ const next = Number(raw);
+ onSeedChange(
+ raw !== "" &&
+ Number.isInteger(next) &&
+ next >= 0 &&
+ next <= 2 ** 31 - 1
+ ? next
+ : null,
+ );
+ }}
+ />
+ </div>
+ <p className="text-ui-12 text-muted-foreground">
+ {t("managedEngines.seedHelp")}
+ </p>
+ </div>
<EngineInstall
key={selected.engine}
engine={selected}
diff --git a/studio/frontend/src/features/model-picker/components/model-config-page.tsx b/studio/frontend/src/features/model-picker/components/model-config-page.tsx
index 7bd9ed20498..0da6e44ea1c 100644
--- a/studio/frontend/src/features/model-picker/components/model-config-page.tsx
+++ b/studio/frontend/src/features/model-picker/components/model-config-page.tsx
@@ -3700,7 +3700,7 @@ export function ModelConfigPage({
/>
<div className="space-y-5">
{!target.isGguf && !targetIsMlx && !targetIsNpu && !classifiedIsDiffusion && !target.meta.isLora && !target.meta.audioType && (
- <InferenceEnginePicker parallelism={config.engineParallelism ?? "tensor"} onParallelismChange={engineParallelism => update({ engineParallelism })} precision={config.enginePrecision ?? "auto"} onPrecisionChange={enginePrecision => update({ enginePrecision })} value={config.engine ?? "auto"} onChange={engine => update({ engine })} onReadyChange={setEngineReady} onUse={handleRun} gpuIds={config.selectedGpuIds} onGpuChange={ids => update({ selectedGpuIds: ids, selectedGpuIndexKind: "physical" })} />
+ <InferenceEnginePicker parallelism={config.engineParallelism ?? "tensor"} onParallelismChange={engineParallelism => update({ engineParallelism })} precision={config.enginePrecision ?? "auto"} onPrecisionChange={enginePrecision => update({ enginePrecision })} kvCacheDtype={config.engineKvCacheDtype ?? "auto"} onKvCacheDtypeChange={engineKvCacheDtype => update({ engineKvCacheDtype })} gpuMemoryUtilization={config.engineGpuMemoryUtilization ?? null} onGpuMemoryUtilizationChange={engineGpuMemoryUtilization => update({ engineGpuMemoryUtilization })} seed={config.engineSeed ?? null} onSeedChange={engineSeed => update({ engineSeed })} value={config.engine ?? "auto"} onChange={engine => update({ engine })} onReadyChange={setEngineReady} onUse={handleRun} gpuIds={config.selectedGpuIds} onGpuChange={ids => update({ selectedGpuIds: ids, selectedGpuIndexKind: "physical" })} />
)}
{audioRuntimeGguf ? (
<p className="text-ui-12 leading-snug text-muted-foreground">
diff --git a/studio/frontend/src/features/model-picker/hooks/use-active-model-config.ts b/studio/frontend/src/features/model-picker/hooks/use-active-model-config.ts
index e13c2cc731f..3e6df867463 100644
--- a/studio/frontend/src/features/model-picker/hooks/use-active-model-config.ts
+++ b/studio/frontend/src/features/model-picker/hooks/use-active-model-config.ts
@@ -23,6 +23,13 @@ export function useActiveModelConfig(): ActiveModelConfigState {
const engine = useChatRuntimeStore((s) => s.params.engine ?? "auto");
const engineParallelism = useChatRuntimeStore((s) => s.params.engineParallelism ?? "tensor");
const enginePrecision = useChatRuntimeStore((s) => s.params.enginePrecision ?? "auto");
+ const engineKvCacheDtype = useChatRuntimeStore(
+ (s) => s.params.engineKvCacheDtype ?? "auto",
+ );
+ const engineSeed = useChatRuntimeStore((s) => s.params.engineSeed ?? null);
+ const engineGpuMemoryUtilization = useChatRuntimeStore(
+ (s) => s.params.engineGpuMemoryUtilization ?? null,
+ );
const activeGgufVariant = useChatRuntimeStore((s) => s.activeGgufVariant);
const loadedIsGguf = useChatRuntimeStore((s) => s.loadedIsGguf);
const loadedIsMlx = useChatRuntimeStore((s) => s.loadedIsMlx);
@@ -125,7 +132,14 @@ export function useActiveModelConfig(): ActiveModelConfigState {
};
if (!isGguf) {
return engine === "vllm" || engine === "sglang"
- ? { ...base, selectedGpuIds, selectedGpuIndexKind }
+ ? {
+ ...base,
+ engineKvCacheDtype,
+ engineSeed,
+ engineGpuMemoryUtilization,
+ selectedGpuIds,
+ selectedGpuIndexKind,
+ }
: base;
}
return {
@@ -149,6 +163,9 @@ export function useActiveModelConfig(): ActiveModelConfigState {
engine,
enginePrecision,
engineParallelism,
+ engineKvCacheDtype,
+ engineSeed,
+ engineGpuMemoryUtilization,
customContextLength,
kvCacheDtype,
effectiveMlxKvQuant,
diff --git a/studio/frontend/src/features/model-picker/model-config/apply-per-model-config.ts b/studio/frontend/src/features/model-picker/model-config/apply-per-model-config.ts
index f9d9cba8e59..e05a617f081 100644
--- a/studio/frontend/src/features/model-picker/model-config/apply-per-model-config.ts
+++ b/studio/frontend/src/features/model-picker/model-config/apply-per-model-config.ts
@@ -39,13 +39,29 @@ export function applyPerModelConfigToRuntime(
const engine = config.engine ?? "auto";
const engineParallelism = config.engineParallelism ?? "tensor";
const enginePrecision = config.enginePrecision ?? "auto";
+ const engineKvCacheDtype = config.engineKvCacheDtype ?? "auto";
+ const engineSeed = config.engineSeed ?? null;
+ const engineGpuMemoryUtilization = config.engineGpuMemoryUtilization ?? null;
if (
maxSeqLength !== store.params.maxSeqLength ||
engine !== (store.params.engine ?? "auto") ||
enginePrecision !== (store.params.enginePrecision ?? "auto") ||
- engineParallelism !== (store.params.engineParallelism ?? "tensor")
+ engineParallelism !== (store.params.engineParallelism ?? "tensor") ||
+ engineKvCacheDtype !== (store.params.engineKvCacheDtype ?? "auto") ||
+ engineSeed !== (store.params.engineSeed ?? null) ||
+ engineGpuMemoryUtilization !==
+ (store.params.engineGpuMemoryUtilization ?? null)
) {
- store.setParams({ ...store.params, maxSeqLength, engine, enginePrecision, engineParallelism });
+ store.setParams({
+ ...store.params,
+ maxSeqLength,
+ engine,
+ enginePrecision,
+ engineParallelism,
+ engineKvCacheDtype,
+ engineSeed,
+ engineGpuMemoryUtilization,
+ });
}
const gpuSelection =
config.selectedGpuIds !== undefined
@@ -124,6 +140,9 @@ export function currentRuntimePerModelConfig(
engine: s.params.engine ?? "auto",
enginePrecision: s.params.enginePrecision ?? "auto",
engineParallelism: s.params.engineParallelism ?? "tensor",
+ engineKvCacheDtype: s.params.engineKvCacheDtype ?? "auto",
+ engineSeed: s.params.engineSeed ?? null,
+ engineGpuMemoryUtilization: s.params.engineGpuMemoryUtilization ?? null,
customContextLength: s.customContextLength ?? null,
maxSeqLength: options.includeMaxSeqLength
? normalizeMaxSeqLength(s.params.maxSeqLength)
@@ -177,6 +196,10 @@ export function perModelConfigsEqual(
(a.engine ?? "auto") === (b.engine ?? "auto") &&
(a.enginePrecision ?? "auto") === (b.enginePrecision ?? "auto") &&
(a.engineParallelism ?? "tensor") === (b.engineParallelism ?? "tensor") &&
+ (a.engineKvCacheDtype ?? "auto") === (b.engineKvCacheDtype ?? "auto") &&
+ (a.engineSeed ?? null) === (b.engineSeed ?? null) &&
+ (a.engineGpuMemoryUtilization ?? null) ===
+ (b.engineGpuMemoryUtilization ?? null) &&
(a.customContextLength ?? null) === (b.customContextLength ?? null) &&
normalizeMaxSeqLength(a.maxSeqLength) ===
normalizeMaxSeqLength(b.maxSeqLength) &&
diff --git a/studio/frontend/src/features/model-picker/model-config/config-signature.ts b/studio/frontend/src/features/model-picker/model-config/config-signature.ts
index 2b7645fe82a..c617f96e79e 100644
--- a/studio/frontend/src/features/model-picker/model-config/config-signature.ts
+++ b/studio/frontend/src/features/model-picker/model-config/config-signature.ts
@@ -51,6 +51,9 @@ export function loadedConfigSignature(
config.engine ?? "auto",
config.enginePrecision ?? "auto",
config.engineParallelism ?? "tensor",
+ config.engineKvCacheDtype ?? "auto",
+ config.engineSeed ?? "",
+ config.engineGpuMemoryUtilization ?? "",
config.customContextLength ?? "",
config.maxSeqLength ?? "",
config.kvCacheDtype ?? "",
diff --git a/studio/frontend/src/features/model-picker/model-config/per-model-config.ts b/studio/frontend/src/features/model-picker/model-config/per-model-config.ts
index dcea694b9b6..2f7964e46be 100644
--- a/studio/frontend/src/features/model-picker/model-config/per-model-config.ts
+++ b/studio/frontend/src/features/model-picker/model-config/per-model-config.ts
@@ -24,6 +24,9 @@ import {
export interface PerModelConfig {
engineParallelism?: "tensor" | "pipeline" | "data";
enginePrecision?: "auto" | "bf16" | "fp16" | "int4" | "int8" | "fp8";
+ engineKvCacheDtype?: "auto" | "fp8";
+ engineSeed?: number | null;
+ engineGpuMemoryUtilization?: number | null;
engine?: "auto" | "vllm" | "sglang";
customContextLength: number | null;
maxSeqLength: number | null;
@@ -74,6 +77,9 @@ export const DEFAULT_PER_MODEL_CONFIG: PerModelConfig = {
engine: "auto",
enginePrecision: "auto",
engineParallelism: "tensor",
+ engineKvCacheDtype: "auto",
+ engineSeed: null,
+ engineGpuMemoryUtilization: null,
customContextLength: null,
maxSeqLength: null,
kvCacheDtype: null,
@@ -96,6 +102,28 @@ export const DEFAULT_PER_MODEL_CONFIG: PerModelConfig = {
chatTemplateOverride: null,
};
+/** vLLM --seed / SGLang --random-seed; the backend bounds it to a non-negative int32. */
+export function normalizeEngineSeed(value: unknown): number | null {
+ return typeof value === "number" &&
+ Number.isInteger(value) &&
+ value >= 0 &&
+ value <= 2 ** 31 - 1
+ ? value
+ : null;
+}
+
+/** Ceiling on the share of each GPU a managed engine may claim, as the backend bounds it. */
+export function normalizeEngineGpuMemoryUtilization(
+ value: unknown,
+): number | null {
+ return typeof value === "number" &&
+ Number.isFinite(value) &&
+ value >= 0.05 &&
+ value <= 0.99
+ ? Math.round(value * 1000) / 1000
+ : null;
+}
+
// Mirrors llama_server_args.py PARALLEL_MIN/MAX; null = follow the server-wide default.
export const N_PARALLEL_MIN = 1;
export const N_PARALLEL_MAX = 64;
@@ -1079,6 +1107,11 @@ function normalizeV1(partial: RawConfig): PerModelConfig {
partial.engine === "vllm" || partial.engine === "sglang"
? partial.engine
: "auto",
+ engineKvCacheDtype: partial.engineKvCacheDtype === "fp8" ? "fp8" : "auto",
+ engineSeed: normalizeEngineSeed(partial.engineSeed),
+ engineGpuMemoryUtilization: normalizeEngineGpuMemoryUtilization(
+ partial.engineGpuMemoryUtilization,
+ ),
customContextLength:
typeof partial.customContextLength === "number" &&
Number.isFinite(partial.customContextLength) &&
@@ -1357,6 +1390,9 @@ export function isDefaultConfig(config: PerModelConfig): boolean {
(config.engine ?? "auto") === "auto" &&
(config.enginePrecision ?? "auto") === "auto" &&
(config.engineParallelism ?? "tensor") === "tensor" &&
+ (config.engineKvCacheDtype ?? "auto") === "auto" &&
+ config.engineSeed == null &&
+ config.engineGpuMemoryUtilization == null &&
config.customContextLength == null &&
config.maxSeqLength == null &&
(config.kvCacheDtype ?? null) === DEFAULT_PER_MODEL_CONFIG.kvCacheDtype &&
diff --git a/studio/frontend/src/features/model-picker/sharing/fields.ts b/studio/frontend/src/features/model-picker/sharing/fields.ts
index bf0a9556809..471be173652 100644
--- a/studio/frontend/src/features/model-picker/sharing/fields.ts
+++ b/studio/frontend/src/features/model-picker/sharing/fields.ts
@@ -50,6 +50,9 @@ export type SharedConfigKey = Exclude<
| "engine"
| "enginePrecision"
| "engineParallelism"
+ | "engineKvCacheDtype"
+ | "engineSeed"
+ | "engineGpuMemoryUtilization"
// A named drafter can be a local directory.
| "specDraftModel"
>;
diff --git a/studio/frontend/src/i18n/locales/en.ts b/studio/frontend/src/i18n/locales/en.ts
index 8f3e6484ccc..fb3bd8e5588 100644
--- a/studio/frontend/src/i18n/locales/en.ts
+++ b/studio/frontend/src/i18n/locales/en.ts
@@ -476,6 +476,15 @@ export const en = {
precision: "Precision",
precisionAuto: "Model default",
precisionHelp: "Model default preserves stored precision. Other options convert unquantized weights when loading.",
+ kvCache: "KV cache",
+ kvCacheFp8: "FP8 (half the memory)",
+ kvCacheHelp: "FP8 stores the context cache in 8 bits, fitting about twice the context in the same memory. Without calibrated scales in the checkpoint, answers can be slightly less accurate.",
+ gpuMemory: "GPU memory limit",
+ gpuMemoryAuto: "Automatic",
+ gpuMemoryHelp: "The most of each selected GPU the engine may use. Automatic uses what is free; lower it to leave room for other programs.",
+ seed: "Seed",
+ seedPlaceholder: "Engine default",
+ seedHelp: "Sets the engine's random seed. Concurrent chats can still change sampled replies.",
default: "Default",
installRequired: "install required",
ownerRequired: "Ask the Studio owner to install this engine.",
diff --git a/studio/frontend/src/i18n/locales/he.ts b/studio/frontend/src/i18n/locales/he.ts
index e3c479ba29c..cb598773c38 100644
--- a/studio/frontend/src/i18n/locales/he.ts
+++ b/studio/frontend/src/i18n/locales/he.ts
@@ -502,6 +502,16 @@ export const he = {
precisionAuto: "ברירת המחדל של המודל",
precisionHelp:
"ברירת המחדל של המודל שומרת על הדיוק המאוחסן. אפשרויות אחרות ממירות משקלים שלא עברו קוונטיזציה בעת הטעינה.",
+ kvCache: "מטמון KV",
+ kvCacheFp8: "FP8 (חצי מהזיכרון)",
+ kvCacheHelp: "FP8 שומר את מטמון ההקשר ב-8 סיביות, כך שבערך פי שניים הקשר נכנס באותו זיכרון. בלי סקאלות מכוילות בצ'קפוינט, התשובות עלולות להיות מעט פחות מדויקות.",
+ gpuMemory: "מגבלת זיכרון GPU",
+ gpuMemoryAuto: "אוטומטי",
+ gpuMemoryHelp:
+ "החלק המרבי מכל GPU נבחר שהמנוע רשאי להשתמש בו. אוטומטי משתמש במה שפנוי; הורד אותו כדי להשאיר מקום לתוכניות אחרות.",
+ seed: "זרע אקראי",
+ seedPlaceholder: "ברירת המחדל של המנוע",
+ seedHelp: "קובע את הזרע האקראי של המנוע. צ'אטים במקביל עדיין יכולים לשנות תשובות שנדגמו.",
default: "ברירת מחדל",
installRequired: "נדרשת התקנה",
ownerRequired: "בקש מבעל ה-Studio להתקין מנוע זה.",
diff --git a/studio/frontend/src/i18n/locales/sv.ts b/studio/frontend/src/i18n/locales/sv.ts
index 0475333a27b..cd4d74b7ff3 100644
--- a/studio/frontend/src/i18n/locales/sv.ts
+++ b/studio/frontend/src/i18n/locales/sv.ts
@@ -509,6 +509,17 @@ export const sv = {
precisionAuto: "Modellens standard",
precisionHelp:
"Modellens standard behåller lagrad precision. Andra alternativ konverterar okvantiserade vikter vid inläsning.",
+ kvCache: "KV-cache",
+ kvCacheFp8: "FP8 (halva minnet)",
+ kvCacheHelp:
+ "FP8 lagrar kontextcachen med 8 bitar och rymmer ungefär dubbelt så mycket kontext i samma minne. Utan kalibrerade skalor i checkpointen kan svaren bli något mindre exakta.",
+ gpuMemory: "GPU-minnesgräns",
+ gpuMemoryAuto: "Automatisk",
+ gpuMemoryHelp:
+ "Det mesta av varje vald GPU som motorn får använda. Automatisk använder det som är ledigt; sänk den för att lämna plats åt andra program.",
+ seed: "Seed",
+ seedPlaceholder: "Motorns standard",
+ seedHelp: "Anger motorns slumpfrö. Samtidiga chattar kan fortfarande ändra samplade svar.",
default: "Standard",
installRequired: "installation krävs",
ownerRequired: "Be Studio-ägaren att installera den här motorn.",
diff --git a/studio/frontend/tests/compare-pane-managed-engine.test.ts b/studio/frontend/tests/compare-pane-managed-engine.test.ts
index 559e49a9477..a8b6165e183 100644
--- a/studio/frontend/tests/compare-pane-managed-engine.test.ts
+++ b/studio/frontend/tests/compare-pane-managed-engine.test.ts
@@ -4,21 +4,33 @@
import assert from "node:assert/strict";
import test from "node:test";
import { readSrc } from "./helpers/kit.ts";
+import {
+ DEFAULT_ENGINE_LAUNCH_FIELDS,
+ engineLaunchFieldsFromParams,
+} from "../src/features/chat/lib/engine-launch-fields.ts";
const composer = readSrc("features/chat/shared-composer.tsx");
const start = composer.indexOf("const paneEngine = ");
const end = composer.indexOf("let loadTrustRemoteCode", start);
assert.ok(start >= 0 && end > start);
-const paneFields = new Function(
+const paneFieldsBody = new Function(
"targetIsGguf",
"ownConfig",
"reconcilePersistedGpuIds",
+ "engineLaunchFieldsFromParams",
`${composer.slice(start, end)} return paneEngineFields;`,
-) as (
+);
+const paneFields = (
targetIsGguf: boolean,
ownConfig: Record<string, unknown>,
reconcile: (ids: number[] | null) => number[] | null,
-) => Record<string, unknown>;
+): Record<string, unknown> =>
+ paneFieldsBody(
+ targetIsGguf,
+ ownConfig,
+ reconcile,
+ engineLaunchFieldsFromParams,
+ );
const reconcile = (ids: number[] | null) => ids;
test("a compare pane loads its remembered vLLM or SGLang settings", () => {
@@ -36,9 +48,23 @@ test("a compare pane loads its remembered vLLM or SGLang settings", () => {
engine: "sglang",
engine_precision: "int4",
engine_parallelism: "pipeline",
+ ...DEFAULT_ENGINE_LAUNCH_FIELDS,
load_in_4bit: false,
gpu_ids: [1, 2],
});
+ const tuned = paneFields(
+ false,
+ {
+ engine: "vllm",
+ engineKvCacheDtype: "fp8",
+ engineSeed: 0,
+ engineGpuMemoryUtilization: 0.5,
+ },
+ reconcile,
+ );
+ assert.equal(tuned.engine_kv_cache_dtype, "fp8");
+ assert.equal(tuned.engine_seed, 0);
+ assert.equal(tuned.engine_gpu_memory_utilization, 0.5);
assert.equal(composer.match(/\.\.\.paneEngineFields,/g)?.length, 2);
});
diff --git a/studio/frontend/tests/managed-engine-launch-options.test.ts b/studio/frontend/tests/managed-engine-launch-options.test.ts
new file mode 100644
index 00000000000..f9e6c0d9b61
--- /dev/null
+++ b/studio/frontend/tests/managed-engine-launch-options.test.ts
@@ -0,0 +1,83 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+/** KV cache dtype, seed and GPU memory limit for vLLM / SGLang loads. */
+
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import { registerBundlerResolver } from "./helpers/kit.ts";
+
+registerBundlerResolver();
+const { normalizePerModelConfig, isDefaultConfig, DEFAULT_PER_MODEL_CONFIG } = await import(
+ "../src/features/model-picker/model-config/per-model-config.ts"
+);
+const { loadedConfigSignature } = await import(
+ "../src/features/model-picker/model-config/config-signature.ts"
+);
+const { residentRuntimeMatchesConfig } = await import(
+ "../src/features/chat/lib/resident-config-match.ts"
+);
+
+test("stored launch options survive normalization and bad ones fall back to unset", () => {
+ const kept = normalizePerModelConfig({
+ engine: "vllm",
+ engineKvCacheDtype: "fp8",
+ engineSeed: 0,
+ engineGpuMemoryUtilization: 0.5,
+ });
+ assert.equal(kept.engineKvCacheDtype, "fp8");
+ assert.equal(kept.engineSeed, 0);
+ assert.equal(kept.engineGpuMemoryUtilization, 0.5);
+ assert.equal(isDefaultConfig(kept), false);
+
+ const dropped = normalizePerModelConfig({
+ engineKvCacheDtype: "fp4",
+ engineSeed: -1,
+ engineGpuMemoryUtilization: 1.5,
+ });
+ assert.equal(dropped.engineKvCacheDtype, "auto");
+ assert.equal(dropped.engineSeed, null);
+ assert.equal(dropped.engineGpuMemoryUtilization, null);
+ assert.equal(isDefaultConfig(normalizePerModelConfig({})), true);
+});
+
+test("a changed launch option changes the loaded-config signature", () => {
+ const base = loadedConfigSignature(DEFAULT_PER_MODEL_CONFIG);
+ for (const patch of [
+ { engineKvCacheDtype: "fp8" as const },
+ { engineSeed: 1 },
+ { engineGpuMemoryUtilization: 0.7 },
+ ]) {
+ assert.notEqual(loadedConfigSignature({ ...DEFAULT_PER_MODEL_CONFIG, ...patch }), base);
+ }
+});
+
+test("a resident engine launched with other options does not satisfy the pick", () => {
+ const standing = {
+ speculativeType: "auto",
+ gpuMemoryMode: "auto" as const,
+ gpuLayers: -1,
+ nCpuMoe: 0,
+ reconcileGpuIds: (ids: number[] | null) => ids,
+ resolveContextLength: (value: number | null) => value ?? 0,
+ parallelSlots: 1,
+ splitRatio: null,
+ normalizeSpeculative: (value: string | null | undefined) => value ?? null,
+ };
+ const config = { ...DEFAULT_PER_MODEL_CONFIG, engine: "vllm" as const, engineSeed: 3 };
+ const status = { engine: "vllm" as const, engine_seed: 3 };
+ assert.equal(residentRuntimeMatchesConfig(status, config, standing), true);
+ assert.equal(
+ residentRuntimeMatchesConfig({ ...status, engine_seed: 4 }, config, standing),
+ false,
+ );
+ assert.equal(
+ residentRuntimeMatchesConfig(
+ { ...status, engine_kv_cache_dtype: "fp8" },
+ config,
+ standing,
+ ),
+ false,
+ );
+});
diff --git a/studio/frontend/tests/managed-engine-offer-background-loads.test.ts b/studio/frontend/tests/managed-engine-offer-background-loads.test.ts
index 2456043cbcb..209adbd8de1 100644
--- a/studio/frontend/tests/managed-engine-offer-background-loads.test.ts
+++ b/studio/frontend/tests/managed-engine-offer-background-loads.test.ts
@@ -6,6 +6,7 @@
import assert from "node:assert/strict";
import test from "node:test";
import { readSrc } from "./helpers/kit.ts";
+import { DEFAULT_ENGINE_LAUNCH_FIELDS } from "../src/features/chat/lib/engine-launch-fields.ts";
const OFFER = {
quantization: "compressed-tensors",
@@ -69,7 +70,8 @@ const paneDecisionBody = new AsyncFunction(
"validation",
"deps",
`const { sel, compareModelDisplayName, confirmManagedEngineIfNeeded, validateModel, panePayload,
- throwIfCompareCancelled, compareSignal, ownConfig, reconcilePersistedGpuIds } = deps;
+ throwIfCompareCancelled, compareSignal, ownConfig, reconcilePersistedGpuIds,
+ DEFAULT_ENGINE_LAUNCH_FIELDS } = deps;
let paneEngineFields = { engine: paneEngine };
${composer.slice(paneStart, paneEnd)}
return paneEngineFields;`,
@@ -101,6 +103,7 @@ async function paneDecision(
compareSignal: undefined,
ownConfig,
reconcilePersistedGpuIds: (ids: number[] | null) => ids,
+ DEFAULT_ENGINE_LAUNCH_FIELDS,
});
return { fields, asked, revalidated, validation };
}
@@ -116,6 +119,7 @@ test("a default-engine compare pane offers vLLM and loads with the pick", async
engine: "vllm",
engine_precision: "auto",
engine_parallelism: "tensor",
+ ...DEFAULT_ENGINE_LAUNCH_FIELDS,
load_in_4bit: false,
});
assert.equal(
diff --git a/studio/frontend/tests/managed-engine-rollback.test.ts b/studio/frontend/tests/managed-engine-rollback.test.ts
index 52274e35bf7..727f2cb97b7 100644
--- a/studio/frontend/tests/managed-engine-rollback.test.ts
+++ b/studio/frontend/tests/managed-engine-rollback.test.ts
@@ -4,16 +4,20 @@
import assert from "node:assert/strict";
import test from "node:test";
import { readSrc } from "./helpers/kit.ts";
+import { engineLaunchFieldsFrom } from "../src/features/chat/lib/engine-launch-fields.ts";
const applier = readSrc("features/chat/lib/apply-inference-status-to-store.ts");
const start = applier.indexOf(" residentCheckpoint: checkpointId,");
const end = applier.indexOf(" });", start);
assert.ok(start >= 0 && end > start);
-const hydrate = new Function(
+const hydrateBody = new Function(
"checkpointId",
"status",
+ "engineLaunchFieldsFrom",
`return { ${applier.slice(start, end)} };`,
-) as (checkpointId: string, status: Record<string, unknown>) => Record<string, unknown>;
+);
+const hydrate = (checkpointId: string, status: Record<string, unknown>) =>
+ hydrateBody(checkpointId, status, engineLaunchFieldsFrom) as Record<string, unknown>;
const runtime = readSrc("features/chat/hooks/use-chat-model-runtime.ts");
const rollbackStart = runtime.indexOf("engine: stateBeforeUnload.loadedEngine,");
@@ -37,6 +41,7 @@ test("a failed switch restores the managed resident's precision and parallel mod
assert.equal(request.engine, "sglang");
assert.equal(request.engine_precision, "bf16");
assert.equal(request.engine_parallelism, "pipeline");
+ assert.equal(request.engine_kv_cache_dtype, "auto");
// The backend reads an explicit 4-bit request without a precision as INT4.
assert.equal(request.load_in_4bit, false);
});
@@ -48,3 +53,16 @@ test("a default-backend resident still rolls back with 4-bit loading", () => {
assert.equal(request.engine_parallelism, "tensor");
assert.equal(request.load_in_4bit, true);
});
+
+test("a failed switch restores the managed resident's KV cache, seed and memory limit", () => {
+ const resident = hydrate("org/model", {
+ engine: "vllm",
+ engine_kv_cache_dtype: "fp8",
+ engine_seed: 0,
+ engine_gpu_memory_utilization: 0.5,
+ });
+ const request = rollback(resident, undefined, null, 4096);
+ assert.equal(request.engine_kv_cache_dtype, "fp8");
+ assert.equal(request.engine_seed, 0);
+ assert.equal(request.engine_gpu_memory_utilization, 0.5);
+});
diff --git a/studio/frontend/tests/resident-config-match-accelerator-matrix.test.ts b/studio/frontend/tests/resident-config-match-accelerator-matrix.test.ts
index d603d2708d4..07601ea1736 100644
--- a/studio/frontend/tests/resident-config-match-accelerator-matrix.test.ts
+++ b/studio/frontend/tests/resident-config-match-accelerator-matrix.test.ts
@@ -491,6 +491,9 @@ test("every PerModelConfig field is either compared or deliberately excluded", (
"maxSeqLength",
"enginePrecision",
"engineParallelism",
+ "engineKvCacheDtype",
+ "engineSeed",
+ "engineGpuMemoryUtilization",
// Qualifies selectedGpuIds rather than adding a dimension of its own: it is read, as
// the reconciler's namespace argument, but /status has no field to compare it against.
"selectedGpuIndexKind",
diff --git a/tests/studio/test_model_picker_contracts.py b/tests/studio/test_model_picker_contracts.py
index 80fab39b392..2cdac1f908f 100644
--- a/tests/studio/test_model_picker_contracts.py
+++ b/tests/studio/test_model_picker_contracts.py
@@ -519,10 +519,9 @@ def test_active_model_config_round_trips_gpu_fields():
# Only GGUF carries the offload knobs; an optional engine keeps just its GPU pick.
flat = " ".join(src.split())
assert "if (!isGguf) {" in flat and ": base; }" in flat
- assert (
- 'engine === "vllm" || engine === "sglang" ? { ...base, selectedGpuIds, selectedGpuIndexKind }'
- in flat
- )
+ managed = flat.split('engine === "vllm" || engine === "sglang" ? { ...base,', 1)
+ assert len(managed) == 2
+ assert "selectedGpuIds, selectedGpuIndexKind" in managed[1].split("}", 1)[0]
assert "loadedLlamaExtraArgs != null" in src
assert "llamaExtraArgs: [...loadedLlamaExtraArgs]" in src
assert "useActiveModelConfig(" in _read("features/chat/chat-page.tsx")