Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
"""KV cache dtype, seed and GPU memory ceiling for the managed vLLM / SGLang engines."""
import json
from types import SimpleNamespace
import pytest
from core.inference.engine_adapters import ADAPTERS
# What validate_model returns for an unquantized text checkpoint with a tool parser.
_DEFAULT_OPTIONS = {
"tool_parser": "hermes",
"precision": "auto",
"parallelism": "tensor",
"is_vision": False,
"quantization": None,
"disable_cuda_graph": False,
"load_format": "auto",
"engine_version": "0.31.0",
}
# The argv main built for _DEFAULT_OPTIONS (launcher path at index 2 left out).
_MAIN_ARGV = {
"vllm": [
"/env/bin/python",
"-I",
"vllm.entrypoints.openai.api_server",
"--model",
"org/model",
"--max-model-len",
"4096",
"--gpu-memory-utilization",
"0.8",
"--tensor-parallel-size",
"1",
"--tool-call-parser",
"hermes",
"--enable-auto-tool-choice",
"--enable-prompt-tokens-details",
"--generation-config",
"vllm",
"--host",
"127.0.0.1",
"--port",
"45678",
"--served-model-name",
"org/model",
"--load-format",
"auto",
],
"sglang": [
"/env/bin/python",
"-I",
"--model-path",
"org/model",
"--context-length",
"4096",
"--mem-fraction-static",
"0.8",
"--tensor-parallel-size",
"1",
"--tool-call-parser",
"hermes",
"--enable-cache-report",
"--attention-backend",
"triton",
"--sampling-backend",
"pytorch",
"--host",
"127.0.0.1",
"--port",
"45678",
"--served-model-name",
"org/model",
"--load-format",
"auto",
],
}
def _argv(engine, **extra):
argv = ADAPTERS[engine].command(
"/env/bin/python",
"org/model",
45678,
"key",
4096,
0.8,
1,
options = {**_DEFAULT_OPTIONS, **extra},
)
return argv[:2] + argv[3:]
@pytest.mark.parametrize("engine", ["vllm", "sglang"])
def test_default_options_launch_the_same_argv_as_before(engine):
assert _argv(engine) == _MAIN_ARGV[engine]
@pytest.mark.parametrize(
"engine, kv, seed", [("vllm", "fp8", "--seed"), ("sglang", "fp8_e4m3", "--random-seed")]
)
def test_kv_cache_dtype_and_seed_reach_each_engine(engine, kv, seed):
argv = _argv(engine, kv_cache_dtype = "fp8", seed = 7)
assert argv[argv.index("--kv-cache-dtype") + 1] == kv
assert argv[argv.index(seed) + 1] == "7"
zero = _argv(engine, seed = 0)
assert zero[zero.index(seed) + 1] == "0"
@pytest.mark.parametrize(
"engine, flag", [("vllm", "--gpu-memory-utilization"), ("sglang", "--mem-fraction-static")]
)
def test_memory_ceiling_lowers_but_never_raises_the_measured_budget(engine, flag):
lowered = _argv(engine, gpu_memory_utilization = 0.5)
assert lowered[lowered.index(flag) + 1] == "0.5"
kept = _argv(engine, gpu_memory_utilization = 0.95)
assert kept[kept.index(flag) + 1] == "0.8"
def _config(tmp_path):
(tmp_path / "config.json").write_text(json.dumps({"num_attention_heads": 8}))
return SimpleNamespace(is_local = True, path = str(tmp_path))
def _compute_cap(monkeypatch, cap):
from core.inference import managed_engine
monkeypatch.setattr(managed_engine, "profile", lambda engine: {"platform": "cuda"})
monkeypatch.setattr(
managed_engine.subprocess, "run", lambda argv, **kwargs: SimpleNamespace(stdout = f"{cap}\n")
)
@pytest.mark.parametrize("engine", ["vllm", "sglang"])
def test_unset_options_stay_out_of_the_launch_options(tmp_path, engine, monkeypatch):
from core.inference.managed_engine import validate_model
_compute_cap(monkeypatch, "9.0")
options = validate_model(_config(tmp_path), engine = engine)
assert not {"kv_cache_dtype", "seed", "gpu_memory_utilization"} & set(options)
options = validate_model(
_config(tmp_path),
engine = engine,
kv_cache_dtype = "fp8",
seed = 0,
gpu_memory_utilization = 0.6,
)
assert (options["kv_cache_dtype"], options["seed"], options["gpu_memory_utilization"]) == (
"fp8",
0,
0.6,
)
def test_seed_and_memory_ceiling_share_the_compile_cache():
from core.inference.managed_engine import ManagedEngine
engine = ManagedEngine("vllm")
info = {"profile_digest": "d"}
base = engine._cache_key(info, "m", [0], dict(_DEFAULT_OPTIONS))
assert base == engine._cache_key(
info, "m", [0], {**_DEFAULT_OPTIONS, "seed": 3, "gpu_memory_utilization": 0.5}
)
assert base != engine._cache_key(info, "m", [0], {**_DEFAULT_OPTIONS, "kv_cache_dtype": "fp8"})
assert engine._cache_key(info, "m", [0], None) == engine._cache_key(info, "m", [0], None)
@pytest.mark.parametrize(
"field, value",
[
("engine_kv_cache_dtype", "fp8"),
("engine_seed", 1),
("engine_gpu_memory_utilization", 0.5),
],
)
def test_a_changed_launch_option_reloads_the_resident_engine(field, value):
from models.inference import LoadRequest
from routes.inference import _non_gguf_runtime_settings_match
backend = SimpleNamespace(
active_model_name = "model",
models = {"model": {"engine": "vllm", "gpu_ids_requested": [0]}},
)
request = LoadRequest(model_path = "model", engine = "vllm", gpu_ids = [0])
assert _non_gguf_runtime_settings_match(backend, request)
assert not _non_gguf_runtime_settings_match(backend, request.model_copy(update = {field: value}))
backend.models["model"][field] = value
assert _non_gguf_runtime_settings_match(backend, request.model_copy(update = {field: value}))
def test_overrides_store_only_set_options_and_load_them_back():
from utils.openai_auto_switch_settings import (
model_override_load_kwargs,
normalize_model_override,
)
assert normalize_model_override(
{
"engine": "vllm",
"engine_kv_cache_dtype": "auto",
"engine_seed": None,
"engine_gpu_memory_utilization": None,
}
) == {"engine": "vllm"}
entry = normalize_model_override(
{
"engine": "vllm",
"engine_kv_cache_dtype": "fp8",
"engine_seed": 0,
"engine_gpu_memory_utilization": 0.75,
}
)
assert entry == {
"engine": "vllm",
"engine_kv_cache_dtype": "fp8",
"engine_seed": 0,
"engine_gpu_memory_utilization": 0.75,
}
assert normalize_model_override({"engine_gpu_memory_utilization": 1.5}) == {}
kwargs = model_override_load_kwargs(entry, is_gguf = False)
assert kwargs["engine_kv_cache_dtype"] == "fp8"
assert kwargs["engine_seed"] == 0
assert kwargs["engine_gpu_memory_utilization"] == 0.75
assert "engine_seed" not in model_override_load_kwargs(entry, is_gguf = True)
@pytest.mark.parametrize("cap,ok", [("7.5", False), ("8.0", True), ("8.9", True)])
def test_an_fp8_kv_cache_is_refused_before_ampere(tmp_path, monkeypatch, cap, ok):
# Colab T4, vLLM 0.30: "FP8 KV cache is not supported ... on Tesla T4 (compute capability 7.5)".
from core.inference.managed_engine import validate_model
_compute_cap(monkeypatch, cap)
if ok:
assert validate_model(_config(tmp_path), kv_cache_dtype = "fp8")["kv_cache_dtype"] == "fp8"
else:
with pytest.raises(ValueError, match = "Ampere"):
validate_model(_config(tmp_path), kv_cache_dtype = "fp8")
def test_a_default_kv_cache_never_probes_the_gpu(tmp_path, monkeypatch):
from core.inference import managed_engine
def refuse(argv, **kwargs):
raise AssertionError("nvidia-smi probed for a default KV cache")
monkeypatch.setattr(managed_engine, "profile", lambda engine: {"platform": "cuda"})
monkeypatch.setattr(managed_engine.subprocess, "run", refuse)
managed_engine.validate_model(_config(tmp_path), kv_cache_dtype = "auto")
@pytest.mark.parametrize("engine,refused", [("sglang", True), ("vllm", False)])
def test_sglang_refuses_an_fp8_kv_cache_on_mla_models(tmp_path, monkeypatch, engine, refused):
# Studio pins SGLang's Triton attention backend, whose MLA path has no FP8 KV cache.
from core.inference.managed_engine import validate_model
_compute_cap(monkeypatch, "9.0")
(tmp_path / "config.json").write_text(
json.dumps({"num_attention_heads": 8, "kv_lora_rank": 512})
)
config = SimpleNamespace(is_local = True, path = str(tmp_path))
if refused:
with pytest.raises(ValueError, match = "MLA|attention"):
validate_model(config, engine = engine, kv_cache_dtype = "fp8")
else:
assert (
validate_model(config, engine = engine, kv_cache_dtype = "fp8")["kv_cache_dtype"] == "fp8"
)
assert "kv_cache_dtype" not in validate_model(config, engine = engine)