Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
"""Lifecycle and isolation tests using small environments and a real HTTP peer."""
import json
import threading
import sys
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
import os
import pytest
from core.inference import engine_install as install
from core.inference.managed_engine import ManagedEngine, launch_arguments
@pytest.fixture(autouse = True)
def nvidia_host(monkeypatch):
# These cover the CUDA profiles; on an AMD runner the real gpu_platform() answers "rocm".
monkeypatch.setattr(install, "gpu_platform", lambda: "cuda")
@pytest.fixture(autouse = True)
def visible_gpus(monkeypatch):
# CPU CI has no parent-visible GPUs; the real resolver tests the mask below.
from core.inference import managed_engine
monkeypatch.setattr(
managed_engine, "resolve_requested_gpu_ids", lambda ids: list(ids) if ids else [0]
)
@pytest.fixture
def isolated(monkeypatch, tmp_path):
monkeypatch.setattr(install, "engine_root", lambda: tmp_path)
monkeypatch.setattr(install, "support_reason", lambda engine = "vllm", **_: None)
monkeypatch.setattr(install, "_jobs", {})
monkeypatch.setattr(install, "_cancels", {})
monkeypatch.setattr(install, "_studio_packages", lambda: {})
return tmp_path
# Studio runs engines locally only on Linux; these fake its POSIX venv layout and timing.
_LOCAL_ENGINE_HOST = pytest.mark.skipif(
sys.platform != "linux", reason = "local engine host is Linux only"
)
_POSIX_ENGINE_LOCKS = pytest.mark.skipif(sys.platform == "win32", reason = "uses fcntl")
def active(
root,
engine = "vllm",
directory = "env-prior",
):
folder = root / engine / directory
(folder / "bin").mkdir(parents = True)
(folder / "bin" / "python").touch()
marker = root / engine / "active.json"
marker.write_text(json.dumps({"directory": directory, "version": "old"}))
return marker
def test_failed_update_preserves_active_environment(isolated, monkeypatch):
marker = active(isolated)
before = marker.read_bytes()
monkeypatch.setattr(install.shutil, "which", lambda _: "/uv")
def fail(*args):
raise RuntimeError("download failed")
monkeypatch.setattr(install, "_run", fail)
install._install("vllm", threading.Event())
assert marker.read_bytes() == before
assert install.status("vllm")["job"]["state"] == "error"
assert [p.name for p in marker.parent.iterdir() if p.is_dir()] == ["env-prior"]
@_LOCAL_ENGINE_HOST
def test_activation_only_after_check_and_keeps_previous(isolated, monkeypatch):
marker = active(isolated)
calls = []
monkeypatch.setattr(install.shutil, "which", lambda _: "/uv")
def run(engine, argv, cancel):
assert json.loads(marker.read_text())["directory"] == "env-prior"
calls.append(argv)
if argv[1] == "venv":
destination = Path(argv[-1])
(destination / "bin").mkdir(parents = True)
(destination / "bin" / "python").touch()
monkeypatch.setattr(install, "_run", run)
install._install("vllm", threading.Event())
result = install.installed("vllm")
assert result["directory"] != "env-prior"
assert result["previous_directory"] == "env-prior"
assert any("--require-hashes" in argv for argv in calls)
assert any(install._CHECK in argv for argv in calls)
assert install.status("vllm")["job"]["state"] == "success"
install.rollback("vllm")
assert install.installed("vllm")["directory"] == "env-prior"
install.rollback("vllm")
assert install.installed("vllm")["directory"] == result["directory"]
def fake_venv(engine, argv, cancel):
if argv[1] == "venv":
destination = Path(argv[-1])
(destination / "bin").mkdir(parents = True)
(destination / "bin" / "python").touch()
(destination / "lib" / "python3.13" / "site-packages").mkdir(parents = True)
def studio_with_engine_torch(monkeypatch, engine, **changes):
pins = {name: version for name, (version, _) in install._pins(engine).items()}
runtime = {"torch", "triton", "nvidia-cublas", "nvidia-nccl-cu13"}
packages = {name: pins[name] for name in (*runtime, "numpy", "fastapi")}
packages.update(changes)
monkeypatch.setattr(install, "_studio_packages", lambda: dict(packages))
monkeypatch.setattr(install, "_torch_runtime", lambda: runtime)
monkeypatch.setattr(install, "_studio_site", lambda: [])
monkeypatch.setattr(install.sys, "version_info", (3, 13, 0))
monkeypatch.setattr(install.sys.implementation, "name", "cpython")
monkeypatch.setattr(install.platform, "python_version", lambda: "3.13.0")
return packages
@_LOCAL_ENGINE_HOST
@pytest.mark.parametrize("engine", ["vllm", "sglang"])
def test_shared_environment_installs_only_what_studio_lacks(isolated, monkeypatch, engine):
studio = studio_with_engine_torch(monkeypatch, engine, numpy = "1.0.0")
monkeypatch.setattr(install.shutil, "which", lambda _: "/uv")
monkeypatch.setattr(install, "_run", fake_venv)
install._install(engine, threading.Event())
info = install.installed(engine)
assert info["shared"] is True
assert info["provided"] == {
name: version for name, version in studio.items() if name != "numpy"
}
packages = (Path(info["path"]) / "engine-requirements.txt").read_text()
assert not any(
line.startswith(("torch==", "triton==", "fastapi==")) for line in packages.splitlines()
)
assert any(line.startswith(engine + "==") for line in packages.splitlines())
assert any(line.startswith("numpy==") for line in packages.splitlines())
pth = Path(info["path"]) / "lib" / "python3.13" / "site-packages" / install._BASE_PTH
assert pth.read_text() == f"import {install._BASE_MODULE}\n"
assert "site.addsitedir(" in pth.with_name(f"{install._BASE_MODULE}.py").read_text()
assert install.status(engine)["current"] is True
studio["fastapi"] = "0.0.1"
status = install.status(engine)
assert status["installed"] is True and status["current"] is False
from core.inference import managed_engine
from types import SimpleNamespace
monkeypatch.setattr(managed_engine, "support_reason", lambda *args: None)
request = SimpleNamespace(gpu_ids = [0], gguf_variant = None, model_path = "m")
with pytest.raises(ValueError, match = "Repair it"):
managed_engine.validate_load(engine, request)
@pytest.mark.parametrize("engine", ["vllm", "sglang"])
def test_pytorch_index_build_of_the_locked_torch_is_shared(isolated, monkeypatch, engine):
pins = {name: version for name, (version, _) in install._pins(engine).items()}
labelled = {"torch": pins["torch"] + "+cu130", "torchvision": pins["torchvision"] + "+cu130"}
studio = studio_with_engine_torch(monkeypatch, engine, **labelled)
plan = install.install_plan(engine)
assert plan["shared"] is True
assert plan["provided"]["torch"] == studio["torch"]
assert plan["provided"]["torchvision"] == studio["torchvision"]
assert not any(
line.startswith(("torch==", "torchvision==", "triton==", "nvidia-cublas=="))
for line in plan["requirements"].splitlines()
)
studio_with_engine_torch(monkeypatch, engine, torch = pins["torch"] + "+cu128")
assert install.install_plan(engine)["shared"] is False
@pytest.mark.parametrize("engine", ["vllm", "sglang"])
def test_shared_engine_brings_its_own_nvcc_headers(isolated, monkeypatch, engine):
# Colab: torch 2.11's runtime matches, but its crt headers are 13.4.59 beside nvcc 13.0.88.
pins = {name: version for name, (version, _) in install._pins(engine).items()}
assert {"nvidia-cuda-nvcc", "nvidia-cuda-crt"} <= set(pins)
studio = studio_with_engine_torch(
monkeypatch,
engine,
**{"nvidia-cuda-crt": "13.4.59", "nvidia-cuda-nvcc": pins["nvidia-cuda-nvcc"]},
)
monkeypatch.setattr(
install, "_torch_runtime", lambda: {"torch", "triton", "nvidia-cublas", "nvidia-cuda-crt"}
)
monkeypatch.setattr(install, "_compat", lambda engine: {"nvidia-cuda-crt": [">=13"]})
plan = install.install_plan(engine)
assert plan["shared"] is True and plan["provided"]["torch"] == studio["torch"]
assert "nvidia-cuda-crt" not in plan["provided"]
# nvcc finds its headers beside its own binary, so a matching nvcc comes along too.
for name in install._TOOLCHAIN:
assert name not in plan["provided"]
assert f"{name}=={pins[name]}" in plan["requirements"]
toolchain = {name: pins[name] for name in install._TOOLCHAIN}
studio_with_engine_torch(monkeypatch, engine, **toolchain)
plan = install.install_plan(engine)
assert {name: plan["provided"][name] for name in install._TOOLCHAIN} == toolchain
def test_shared_engine_never_sees_studio_flashinfer(tmp_path):
import subprocess
import sys
studio, engine = tmp_path / "studio", tmp_path / "engine"
for name in ("flashinfer_jit_cache", "flashinfer_cubin", "studio_only"):
(studio / name).mkdir(parents = True)
(studio / name / "__init__.py").write_text("__version__ = '0.6.6+cu130'\n")
engine.mkdir()
(engine / f"{install._BASE_MODULE}.py").write_text(
install._STUDIO_BASE_SOURCE.format(paths = [str(studio)])
)
probe = (
f"import sys; sys.path.insert(0, {str(engine)!r}); import {install._BASE_MODULE}; "
"import importlib.util as u, pkgutil, studio_only; "
"print(u.find_spec('flashinfer_jit_cache'), u.find_spec('flashinfer_cubin'), "
f"sorted(m.name for m in pkgutil.iter_modules([{str(studio)!r}])))"
)
out = subprocess.run([sys.executable, "-I", "-S", "-c", probe], capture_output = True, text = True)
assert out.returncode == 0, out.stderr
assert out.stdout.strip() == "None None ['studio_only']"
def test_shared_engine_keeps_its_own_nvidia_tree_first(tmp_path):
# Colab's regular nvidia package hid the engine's crt headers from FlashInfer's JIT.
import subprocess
import sys
studio, engine = tmp_path / "studio", tmp_path / "engine"
(studio / "nvidia" / "studio_only").mkdir(parents = True)
(studio / "nvidia" / "__init__.py").write_text("")
(studio / "nvidia" / "studio_only" / "__init__.py").write_text("")
(engine / "nvidia" / "engine_only").mkdir(parents = True)
(engine / "nvidia" / "engine_only" / "__init__.py").write_text("")
(engine / f"{install._BASE_MODULE}.py").write_text(
install._STUDIO_BASE_SOURCE.format(paths = [str(studio)])
)
probe = (
f"import sys; sys.path.insert(0, {str(engine)!r}); import {install._BASE_MODULE}; "
"import nvidia, nvidia.engine_only, nvidia.studio_only; print(list(nvidia.__path__))"
)
out = subprocess.run([sys.executable, "-I", "-S", "-c", probe], capture_output = True, text = True)
assert out.returncode == 0, out.stderr
assert out.stdout.strip() == str([str(engine / "nvidia"), str(studio / "nvidia")])
def test_engine_cuda_home_is_the_locked_pip_nvcc(tmp_path, monkeypatch):
import os
env, studio = tmp_path / "env", tmp_path / "studio"
engine_cuda = env / "lib" / "python3.13" / "site-packages" / "nvidia" / "cu13"
studio_cuda = studio / "nvidia" / "cu13"
for path in (
engine_cuda / "bin" / "nvcc",
engine_cuda / "include" / "crt" / "host_defines.h",
studio_cuda / "lib" / "libcudart.so.13",
studio_cuda / "include" / "cuda_runtime.h",
):
path.parent.mkdir(parents = True, exist_ok = True)
path.touch()
(engine_cuda / "nvvm").mkdir()
monkeypatch.setattr(install, "_studio_site", lambda: [str(studio)])
assert install.cuda_environment({"path": str(env), "shared": True}) == {}
install.link_cuda_home(env, shared = True)
install.link_cuda_home(env, shared = True)
home = env / "cuda"
assert (home / "bin" / "nvcc").exists() and (home / "nvvm").resolve() == engine_cuda / "nvvm"
assert (home / "lib64" / "libcudart.so").resolve() == studio_cuda / "lib" / "libcudart.so.13"
cuda = install.cuda_environment({"path": str(env), "shared": True})
assert cuda == {"CUDA_HOME": str(home), "CPATH": str(home / "include")}
# One tree, so cuda_runtime.h's quote includes of crt/ resolve to the engine's release.
(studio_cuda / "include" / "crt").mkdir()
install.link_cuda_home(env, shared = True)
assert (
home / "include" / "cuda_runtime.h"
).resolve() == studio_cuda / "include" / "cuda_runtime.h"
assert (home / "include" / "crt").resolve() == engine_cuda / "include" / "crt"
assert sorted(os.listdir(home / "include")) == ["crt", "cuda_runtime.h"]
@_LOCAL_ENGINE_HOST
@pytest.mark.parametrize("engine", ["vllm", "sglang"])
def test_glibc_floor_matches_the_lock_platform(engine, monkeypatch):
assert "--python-platform x86_64-manylinux_2_34" in install.requirements(engine).read_text()
monkeypatch.setattr(install.platform, "system", lambda: "Linux")
monkeypatch.setattr(install.platform, "machine", lambda: "x86_64")
monkeypatch.setattr(install.platform, "libc_ver", lambda: ("glibc", "2.34"))
reason = install.support_reason(engine, wait = False)
assert reason is None or "glibc" not in reason
monkeypatch.setattr(install.platform, "libc_ver", lambda: ("glibc", "2.33"))
assert (
install.support_reason(engine, wait = False)
== f"{install.ENGINE_NAMES[engine]} requires glibc 2.34 or newer."
)
def fake_dist(site, name, version, *requires):
info = site / f"{name.replace('-', '_')}-{version}.dist-info"
info.mkdir(parents = True)
body = "".join(f"Requires-Dist: {r}\n" for r in requires)
(info / "METADATA").write_text(
f"Metadata-Version: 2.1\nName: {name}\nVersion: {version}\n{body}"
)
def test_shared_engine_reuses_studio_versions_its_dependencies_accept(
isolated, monkeypatch, tmp_path
):
pins = {name: version for name, (version, _) in install._pins("vllm").items()}
site = tmp_path / "studio-site"
fake_dist(site, "bitsandbytes", "9.0.0")
fake_dist(site, "scipy", "9.0.0", "numpy>=99")
fake_dist(site, "pillow", "1.0.0")
fake_dist(site, "flashinfer-python", "9.0.0")
studio = studio_with_engine_torch(
monkeypatch,
"vllm",
bitsandbytes = "9.0.0",
scipy = "9.0.0",
pillow = "1.0.0",
**{"flashinfer-python": "9.0.0"},
)
monkeypatch.setattr(install, "_studio_site", lambda: [str(site)])
lock = {**install._pins("vllm"), "scipy": ("1.18.0", "scipy==1.18.0\n")}
monkeypatch.setattr(install, "_pins", lambda engine: lock)
monkeypatch.setattr(
install, "_compat", lambda engine: {"pillow": [">=11"], "flashinfer-python": []}
)
plan = install.install_plan("vllm")
assert plan["provided"]["bitsandbytes"] == "9.0.0"
assert not any(line.startswith("bitsandbytes==") for line in plan["requirements"].splitlines())
assert "pillow" not in plan["provided"] and "scipy" not in plan["provided"]
# Studio's NVFP4 FlashInfer is hidden from engines, so it can never stand in.
assert "flashinfer-python" not in plan["provided"]
assert plan["provided"]["torch"] == studio["torch"] == pins["torch"]
monkeypatch.setattr(install, "_compat", lambda engine: {})
assert "bitsandbytes" not in install.install_plan("vllm")["provided"]
@pytest.mark.parametrize("engine", ["vllm", "sglang"])
@pytest.mark.parametrize("torchao, reused", [("0.18.0", False), ("0.17.0", True)])
def test_shared_engine_reuses_only_the_torchao_its_adapter_configs_use(
isolated, monkeypatch, tmp_path, engine, torchao, reused
):
# Nothing in the lock constrains torchao; Studio's adapter builds 0.17-API configs.
site = tmp_path / "studio-site"
fake_dist(site, "torchao", torchao)
studio_with_engine_torch(monkeypatch, engine, torchao = torchao)
monkeypatch.setattr(install, "_studio_site", lambda: [str(site)])
monkeypatch.setattr(install, "_compat", lambda engine: {"torchao": []})
plan = install.install_plan(engine)
assert plan["shared"]
assert ("torchao" in plan["provided"]) == reused
assert ("torchao==0.17.0" in plan["requirements"]) != reused
def test_existing_shared_engine_with_another_torchao_needs_repair(monkeypatch):
info = {
"shared": True,
"python": install.platform.python_version(),
"provided": {"torchao": "0.18.0"},
}
monkeypatch.setattr(install, "_studio_packages", lambda: {"torchao": "0.18.0"})
assert install.stale(info)
info["provided"] = {"torchao": "0.17.0"}
monkeypatch.setattr(install, "_studio_packages", lambda: {"torchao": "0.17.0"})
assert not install.stale(info)
@pytest.mark.parametrize("engine", ["vllm", "sglang"])
def test_compat_file_matches_its_lock(engine):
assert install._compat(engine), "regenerate with requirements/engines/engine_compat.py"
assert "flashinfer-cubin" not in install._pins(engine)
assert "flashinfer-cubin" in install.profile(engine)["omit"]
def test_no_lock_downloads_the_cuda12_cutlass_libraries(monkeypatch):
# Every release needs driver 580 (CUDA 13), so CUTLASS DSL never loads its cu12 flavour.
for engine, entry in install.PROFILES.items():
for release in entry["releases"]:
monkeypatch.setattr(install, "_release", lambda engine, release = release: release)
assert install.profile(engine)["driver"] >= 580
assert "nvidia-cutlass-dsl-libs-cu12" in install.profile(engine)["omit"]
assert "nvidia-cutlass-dsl-libs-cu12" not in install._pins(engine), release["lock"]
assert "nvidia-cutlass-dsl-libs-cu13" in install._pins(engine), release["lock"]
@pytest.mark.parametrize(
("engine", "studio_torch", "version"),
[
("vllm", "2.13.0+cu130", "0.30.0"),
("vllm", "2.11.0+cu130", "0.26.0"),
("vllm", "2.11.0", "0.26.0"),
("vllm", "2.12.1+cu130", "0.30.0"),
("vllm", None, "0.30.0"),
("sglang", "2.13.0+cu130", "0.5.20"),
("sglang", "2.11.0+cu130", "0.5.17"),
("sglang", "2.10.0+cu128", "0.5.20"),
],
)
@_LOCAL_ENGINE_HOST
def test_release_follows_studio_torch(monkeypatch, engine, studio_torch, version):
monkeypatch.setattr(
install, "_studio_packages", lambda: {"torch": studio_torch} if studio_torch else {}
)
chosen = install.profile(engine)
assert chosen["version"] == version
assert install._pins(engine)[engine][0] == version
assert install._pins(engine)["torch"][0] == chosen["torch"]
@_LOCAL_ENGINE_HOST
@pytest.mark.parametrize("engine", ["vllm", "sglang"])
def test_every_release_lock_is_current(monkeypatch, engine):
for release in install.PROFILES[engine]["releases"]:
monkeypatch.setattr(install, "_studio_packages", lambda r = release: {"torch": r["torch"]})
assert install.profile(engine)["version"] == release["version"]
assert install._compat_file(engine).get("sizes"), "regenerate with engine_compat.py"
assert install._pins(engine)[engine][0] == release["version"]
@_LOCAL_ENGINE_HOST
@pytest.mark.parametrize("engine", ["vllm", "sglang"])
def test_download_size_leaves_out_what_studio_shares(isolated, monkeypatch, engine):
full = install.download_bytes(engine)
studio_with_engine_torch(monkeypatch, engine)
shared = install.download_bytes(engine)
sizes = install._compat_file(engine)["sizes"]
assert install.install_plan(engine)["shared"]
assert full > shared > 0
assert full - shared >= sizes["torch"]
assert install.status(engine)["download_bytes"] == shared
@_LOCAL_ENGINE_HOST
def test_changed_studio_torch_uses_an_isolated_environment(isolated, monkeypatch):
studio_with_engine_torch(monkeypatch, "vllm", torch = "2.99.0")
monkeypatch.setattr(install.shutil, "which", lambda _: "/uv")
calls = []
def run(engine, argv, cancel):
calls.append(argv)
fake_venv(engine, argv, cancel)
monkeypatch.setattr(install, "_run", run)
install._install("vllm", threading.Event())
info = install.installed("vllm")
assert info["shared"] is False and info["provided"] == {}
packages = (Path(info["path"]) / "engine-requirements.txt").read_text()
assert packages == "".join(line for _, line in install._pins("vllm").values())
assert not (
Path(info["path"]) / "lib" / "python3.13" / "site-packages" / install._BASE_PTH
).exists()
assert install.status("vllm")["current"] is True
@_LOCAL_ENGINE_HOST
def test_rollback_refuses_a_shared_environment_studio_no_longer_matches(isolated, monkeypatch):
studio = studio_with_engine_torch(monkeypatch, "vllm")
monkeypatch.setattr(install.shutil, "which", lambda _: "/uv")
monkeypatch.setattr(install, "_run", fake_venv)
install._install("vllm", threading.Event())
install._install("vllm", threading.Event())
assert install.status("vllm")["can_rollback"] is True
studio["numpy"] = "0.0.1"
assert install.status("vllm")["can_rollback"] is False
with pytest.raises(RuntimeError, match = "no longer has"):
install.rollback("vllm")
def test_engine_check_sees_every_locked_version_and_requirement(tmp_path):
import os
import subprocess
import sys
site = tmp_path / "site"
(site / "demo-1.0.dist-info").mkdir(parents = True)
(site / "demo-1.0.dist-info" / "METADATA").write_text(
"Metadata-Version: 2.1\nName: demo\nVersion: 1.0\nRequires-Dist: absent-dependency>=1\n"
)
lock = tmp_path / "lock.txt"
env = {**os.environ, "PYTHONPATH": str(site)}
def check(pins, omitted = ""):
lock.write_text(pins)
return subprocess.run(
[sys.executable, "-c", install._CHECK, str(lock), omitted, "cu130"],
capture_output = True,
text = True,
env = env,
)
assert (
check("demo==2.0 \\\n --hash=sha256:00\n").stderr.strip()
== "demo==2.0 is required, found 1.0"
)
assert "demo requires absent-dependency>=1, found None" in check("demo==1.0\n").stderr
assert check("demo==1.0\n", "absent-dependency").returncode == 0
(site / "demo-1.0.dist-info" / "METADATA").write_text(
"Metadata-Version: 2.1\nName: demo\nVersion: 1.0+cu130\n"
)
assert check("demo==1.0\n").returncode == 0
@_LOCAL_ENGINE_HOST
def test_cancel_before_activation_keeps_previous(isolated, monkeypatch):
marker = active(isolated)
monkeypatch.setattr(install.shutil, "which", lambda _: "/uv")
cancelled = threading.Event()
def run(*args):
cancelled.set()
monkeypatch.setattr(install, "_run", run)
install._install("vllm", cancelled)
assert json.loads(marker.read_text())["directory"] == "env-prior"
assert install.status("vllm")["job"]["state"] == "cancelled"
@_LOCAL_ENGINE_HOST
@pytest.mark.parametrize("engine", ["vllm", "sglang"])
def test_explicit_rollback_allows_old_profile_until_replaced(isolated, monkeypatch, engine):
marker = active(isolated, engine)
monkeypatch.setattr(install.shutil, "which", lambda _: "/uv")
def run(engine, argv, cancel):
if argv[1] == "venv":
destination = Path(argv[-1])
(destination / "bin").mkdir(parents = True)
(destination / "bin" / "python").touch()
monkeypatch.setattr(install, "_run", run)
assert install.status(engine)["current"] is False
assert install.status(engine)["restored"] is False
install._install(engine, threading.Event())
assert install.status(engine)["current"] is True
install.rollback(engine)
monkeypatch.setattr(install, "_jobs", {})
assert install.status(engine)["current"] is False
assert install.status(engine)["restored"] is True
before = marker.read_bytes()
def fail(*args):
raise RuntimeError("download failed")
monkeypatch.setattr(install, "_run", fail)
install._install(engine, threading.Event())
assert marker.read_bytes() == before
assert install.status(engine)["restored"] is True
monkeypatch.setattr(install, "_run", run)
install._install(engine, threading.Event())
assert install.status(engine)["current"] is True
assert install.status(engine)["restored"] is False
def test_runtime_lease_blocks_removal(isolated):
marker = active(isolated)
with install.engine_lease("vllm"):
assert install.status("vllm")["in_use"] is True
with pytest.raises(RuntimeError, match = "another Unsloth"):
install.remove("vllm")
assert marker.exists()
install.remove("vllm")
assert not marker.exists()
def test_removal_keeps_shared_models_and_cache(isolated):
active(isolated)
shared = isolated / "shared-model.safetensors"
shared.write_bytes(b"model")
install.remove("vllm")
assert shared.read_bytes() == b"model"
def test_marker_cannot_escape_engine_root(isolated):
marker = active(isolated)
marker.write_text(json.dumps({"directory": "../env-other"}))
assert install.installed("vllm") is None
def test_unknown_engine_cannot_become_a_path(isolated):
with pytest.raises(ValueError):
install.remove("../vllm")
def test_installer_does_not_inherit_base_python_or_secrets(monkeypatch):
for key in ("HF_TOKEN", "PYTHONPATH", "VIRTUAL_ENV", "UV_OVERRIDE", "PIP_INDEX_URL"):
monkeypatch.setenv(key, "must-not-inherit")
env = install.install_environment()
assert not any(
env.get(key) == "must-not-inherit"
for key in (
"HF_TOKEN",
"PYTHONPATH",
"VIRTUAL_ENV",
"UV_OVERRIDE",
"PIP_INDEX_URL",
)
)
assert env["UV_LINK_MODE"] == "copy"
def test_only_driver_directories_survive_the_library_path(tmp_path):
driver, runtime = tmp_path / "lib64-nvidia", tmp_path / "cuda" / "lib64"
for folder, library in ((driver, "libcuda.so.1"), (runtime, "libcudart.so.13")):
folder.mkdir(parents = True)
(folder / library).write_bytes(b"")
joined = os.pathsep.join([str(runtime), "", "lib64-nvidia", str(driver)])
assert install.driver_library_path({"LD_LIBRARY_PATH": joined}) == str(driver)
assert install.driver_library_path({"LD_LIBRARY_PATH": str(runtime)}) is None
assert install.driver_library_path({}) is None
def test_installer_keeps_the_driver_its_checks_import(monkeypatch, tmp_path):
driver = tmp_path / "lib64-nvidia"
driver.mkdir()
(driver / "libcuda.so.1").write_bytes(b"")
monkeypatch.setenv("LD_LIBRARY_PATH", os.pathsep.join(["/opt/cuda-12/lib64", str(driver)]))
assert install.install_environment()["LD_LIBRARY_PATH"] == str(driver)
monkeypatch.setenv("LD_LIBRARY_PATH", "/opt/cuda-12/lib64")
assert "LD_LIBRARY_PATH" not in install.install_environment()
def test_installer_reuses_recorded_cache(monkeypatch, tmp_path):
from utils.paths import storage_roots
cache = tmp_path / "existing cache"
cache.mkdir()
(tmp_path / "uv-cache-dir").write_text(str(cache) + "\n")
monkeypatch.setattr(storage_roots, "cache_root", lambda: tmp_path)
monkeypatch.delenv("UV_CACHE_DIR", raising = False)
assert install.install_environment()["UV_CACHE_DIR"] == str(cache)
monkeypatch.setenv("UV_CACHE_DIR", str(tmp_path / "explicit"))
assert install.install_environment()["UV_CACHE_DIR"] == str(tmp_path / "explicit")
@pytest.mark.parametrize("image_python", [False, True])
def test_docker_engine_python_survives_image_replacement(isolated, monkeypatch, image_python):
import shutil
app = isolated / "app"
app.mkdir()
monkeypatch.setenv("UNSLOTH_STUDIO_APP", str(app))
monkeypatch.delenv("UV_PYTHON_INSTALL_DIR", raising = False)
if image_python:
(app / "python").mkdir()
linked_python = isolated / "linked-python"
linked_python.symlink_to(app / "python", target_is_directory = True)
monkeypatch.setenv("UV_PYTHON_INSTALL_DIR", str(linked_python))
# Model uv's interpreter symlink with a small file, then discard the app layer.
python_home = Path(install.install_environment()["UV_PYTHON_INSTALL_DIR"])
python_home.mkdir(parents = True, exist_ok = True)
interpreter = python_home / "python3.13"
interpreter.touch()
for engine in ("vllm", "sglang"):
marker = active(isolated, engine)
python = marker.parent / "env-prior" / "bin" / "python"
python.unlink()
python.symlink_to(interpreter)
shutil.rmtree(app)
app.mkdir()
for engine in ("vllm", "sglang"):
assert install.installed(engine) is not None
def test_docker_engine_python_respects_external_store(isolated, monkeypatch):
monkeypatch.setenv("UNSLOTH_STUDIO_APP", str(isolated / "app"))
external = str(isolated / "persistent-python")
monkeypatch.setenv("UV_PYTHON_INSTALL_DIR", external)
assert install.install_environment()["UV_PYTHON_INSTALL_DIR"] == external
def test_native_engine_python_keeps_uv_default(isolated, monkeypatch):
monkeypatch.delenv("UNSLOTH_STUDIO_APP", raising = False)
monkeypatch.delenv("UV_PYTHON_INSTALL_DIR", raising = False)
assert "UV_PYTHON_INSTALL_DIR" not in install.install_environment()
@pytest.mark.parametrize("engine", ["vllm", "sglang"])
def test_argv_owns_network_and_memory_settings(engine):
argv = launch_arguments(engine, "/env/bin/python", "org/model", 45678, "private", 4096)
assert argv[0:2] == ["/env/bin/python", "-I"]
assert argv[argv.index("--host") + 1] == "127.0.0.1"
assert "private" not in argv
assert "4096" in argv
assert "--trust-remote-code" not in argv
@pytest.fixture
def peer():
requests = []
waiting = threading.Event()
class Handler(BaseHTTPRequestHandler):
def log_message(self, *args):
pass
def do_POST(self):
body = json.loads(self.rfile.read(int(self.headers["Content-Length"])))
requests.append((self.path, self.headers.get("Authorization"), body))
self.send_response(200)
self.send_header("Content-Type", "text/event-stream")
self.end_headers()
try:
self.wfile.write(b'data: {"choices":[{"delta":{"content":"hello"}}]}\n\n')
self.wfile.flush()
if body.get("seed") == 42:
waiting.wait(10)
self.wfile.write(
b'data: {"choices":[{"delta":{"content":" world"},"finish_reason":"length"}]}\n\ndata: {"choices":[],"usage":{"prompt_tokens":3,"completion_tokens":2}}\n\ndata: [DONE]\n\n'
)
self.wfile.flush()
except (BrokenPipeError, ConnectionResetError):
pass
server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
thread = threading.Thread(target = server.serve_forever, daemon = True)
thread.start()
engine = ManagedEngine("vllm")
engine.base_url = f"http://127.0.0.1:{server.server_port}"
engine.model = "org/model"
yield engine, requests
waiting.set()
server.shutdown()
server.server_close()
thread.join(2)
@pytest.mark.parametrize("deadline,completes", [(0.3, False), (10, True)])
def test_engine_stream_waits_for_the_first_token_deadline(monkeypatch, deadline, completes):
import time
import httpx
from core.inference.engine_transport import stream_chat_events
class Handler(BaseHTTPRequestHandler):
def log_message(self, *args):
pass
def do_POST(self):
self.rfile.read(int(self.headers["Content-Length"]))
self.send_response(200)
self.send_header("Content-Type", "text/event-stream")
self.end_headers()
self.wfile.flush()
time.sleep(1)
try:
self.wfile.write(
b'data: {"choices":[{"delta":{"content":"hi"}}]}\n\ndata: [DONE]\n\n'
)
self.wfile.flush()
except (BrokenPipeError, ConnectionResetError):
pass
monkeypatch.setenv("UNSLOTH_OPENAI_COMPAT_FIRST_TOKEN_TIMEOUT", str(deadline))
server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
thread = threading.Thread(target = server.serve_forever, daemon = True)
thread.start()
try:
events = stream_chat_events(f"http://127.0.0.1:{server.server_port}", {}, {}, lambda: False)
if completes:
assert [e["choices"][0]["delta"]["content"] for e in events] == ["hi"]
else:
with pytest.raises(httpx.ReadTimeout):
list(events)
finally:
server.shutdown()
server.server_close()
thread.join(2)
def test_real_http_stream_and_usage(peer):
engine, requests = peer
stats = {}
assert list(
engine.generate(messages = [{"role": "user", "content": "Hi"}], stats_holder = stats)
) == ["hello", " world"]
assert stats["stats"]["usage"]["prompt_tokens"] == 3
assert stats["stats"]["finish_reason"] == "length"
path, auth, payload = requests[0]
assert path == "/v1/chat/completions"
assert auth == "Bearer " + engine.key
assert payload["chat_template_kwargs"]["enable_thinking"] is False
def test_cancellation_does_not_wait_for_next_token(peer):
import time
engine, _ = peer
cancel = threading.Event()
stream = engine.generate(messages = [], seed = 42, cancel_event = cancel)
assert next(stream) == "hello"
cancel.set()
start = time.monotonic()
assert list(stream) == []
assert time.monotonic() - start < 2
def test_structured_output_and_continuation_reach_the_engine(peer):
engine, requests = peer
schema = {
"type": "json_schema",
"json_schema": {"name": "city", "schema": {"type": "object"}},
}
partial = {"role": "assistant", "content": "Once upon"}
list(
engine.generate(
messages = [{"role": "user", "content": "Hi"}, partial],
response_format = schema,
continue_final_message = True,
)
)
body = requests[-1][2]
assert body["response_format"] == schema
assert body["messages"][-1] == partial
assert body["continue_final_message"] is True
assert body["add_generation_prompt"] is False
list(engine.generate(messages = [{"role": "user", "content": "Hi"}]))
assert not {"response_format", "continue_final_message", "add_generation_prompt"} & set(
requests[-1][2]
)
@pytest.mark.parametrize("engine", ["vllm", "sglang"])
def test_chat_completions_hand_structured_output_and_continuation_to_the_engine(
monkeypatch, engine
):
import asyncio
from types import SimpleNamespace
import routes.inference as inf
from fastapi import HTTPException
from core.inference.api_monitor import ApiMonitor
from models.inference import ChatCompletionRequest
from state.tool_policy import reset_tool_policy
calls = []
class Backend:
active_model_name = "m"
models = {"m": {"engine": engine, "context_length": 2048}}
def generate_chat_response(self, **kwargs):
calls.append(kwargs)
yield '{"city": "Tokyo"}'
def reset_generation_state(self, caller_cancel_event = None):
pass
class Request:
state = SimpleNamespace()
url = SimpleNamespace(path = "/v1/chat/completions")
method = "POST"
scope: dict = {}
headers: dict = {}
async def is_disconnected(self):
return False
reset_tool_policy()
monkeypatch.setattr(inf, "api_monitor", ApiMonitor(max_entries = 8))
monkeypatch.setattr(
inf,
"get_llama_cpp_backend",
lambda: SimpleNamespace(
is_loaded = False, supports_tools = False, is_vision = False, context_length = None
),
)
monkeypatch.setattr(inf, "get_inference_backend", lambda: Backend())
def chat(status = 200, **fields):
payload = ChatCompletionRequest(model = "m", stream = False, enable_tools = False, **fields)
try:
response = asyncio.run(
inf.openai_chat_completions(payload, request = Request(), current_subject = "u")
)
except HTTPException as exc:
assert exc.status_code == status, exc.detail
return exc.detail
assert response.status_code == status, response.body
return calls[-1]
schema = {"type": "json_schema", "json_schema": {"name": "c", "schema": {"type": "object"}}}
sent = chat(messages = [{"role": "user", "content": "City?"}], response_format = schema)
assert sent["response_format"] == schema
sent = chat(
messages = [
{"role": "user", "content": "Count to five."},
{"role": "assistant", "content": "one, two,"},
],
continue_final_message = True,
)
assert sent["continue_final_message"] is True
sent_before = len(calls)
refusal = chat(
status = 400,
messages = [
{"role": "user", "content": "Count to five."},
{"role": "assistant", "content": "", "reasoning_content": "The user wants"},
],
continue_final_message = True,
)
assert "mid-thought" in str(refusal)
assert len(calls) == sent_before
@pytest.mark.parametrize("stream", [False, True])
def test_studio_tool_execution_refuses_a_response_format_it_cannot_honor(monkeypatch, stream):
import asyncio
from types import SimpleNamespace
import routes.inference as inf
from fastapi import HTTPException
from core.inference.api_monitor import ApiMonitor
from models.inference import ChatCompletionRequest
from state.tool_policy import reset_tool_policy
class Backend:
active_model_name = "m"
models = {"m": {"engine": "vllm", "context_length": 2048, "supports_tools": True}}
_managed_engine = None
def reset_generation_state(self, caller_cancel_event = None):
pass
class Request:
state = SimpleNamespace()
url = SimpleNamespace(path = "/v1/chat/completions")
method = "POST"
scope: dict = {}
headers: dict = {}
async def is_disconnected(self):
return False
reset_tool_policy()
monkeypatch.setattr(inf, "api_monitor", ApiMonitor(max_entries = 8))
monkeypatch.setattr(
inf,
"get_llama_cpp_backend",
lambda: SimpleNamespace(
is_loaded = False, supports_tools = False, is_vision = False, context_length = None
),
)
monkeypatch.setattr(inf, "get_inference_backend", lambda: Backend())
payload = ChatCompletionRequest(
model = "m",
stream = stream,
enable_tools = True,
enabled_tools = ["python"],
permission_mode = "bypass",
messages = [{"role": "user", "content": "Compute 7*567 and answer as JSON."}],
response_format = {
"type": "json_schema",
"json_schema": {"name": "a", "schema": {"type": "object"}},
},
)
with pytest.raises(HTTPException) as refusal:
asyncio.run(inf.openai_chat_completions(payload, request = Request(), current_subject = "u"))
assert refusal.value.status_code == 400
assert refusal.value.detail["error"]["param"] == "response_format"
def test_studio_tool_loop_resumes_the_partial_it_was_given(monkeypatch):
import asyncio
from types import SimpleNamespace
import routes.inference as inf
import routes.managed_engine_chat as mec
from core.inference.api_monitor import ApiMonitor
from models.inference import ChatCompletionRequest
from state.tool_policy import reset_tool_policy
runs = []
class _Stop(Exception):
pass
def capture(transport, *, run, **kwargs):
runs.append(run)
raise _Stop
engine = ManagedEngine("vllm")
engine.base_url, engine.model, engine.context = "http://127.0.0.1:9", "m", 2048
class Backend:
active_model_name = "m"
models = {"m": {"engine": "vllm", "context_length": 2048, "supports_tools": True}}
_managed_engine = engine
def reset_generation_state(self, caller_cancel_event = None):
pass
class Request:
state = SimpleNamespace()
url = SimpleNamespace(path = "/v1/chat/completions")
method = "POST"
scope: dict = {}
headers: dict = {}
async def is_disconnected(self):
return False
reset_tool_policy()
monkeypatch.setattr(mec, "stream_with_studio_tools", capture)
monkeypatch.setattr(inf, "api_monitor", ApiMonitor(max_entries = 8))
monkeypatch.setattr(
inf,
"get_llama_cpp_backend",
lambda: SimpleNamespace(
is_loaded = False, supports_tools = False, is_vision = False, context_length = None
),
)
monkeypatch.setattr(inf, "get_inference_backend", lambda: Backend())
payload = ChatCompletionRequest(
model = "m",
stream = False,
enable_tools = True,
enabled_tools = ["python"],
permission_mode = "off",
continue_final_message = True,
messages = [
{"role": "user", "content": "Count to five."},
{"role": "assistant", "content": "one, two,"},
],
)
with pytest.raises(_Stop):
asyncio.run(inf.openai_chat_completions(payload, request = Request(), current_subject = "u"))
assert runs and runs[-1].continue_final_message is True
def test_unsupported_tools_are_not_silently_dropped(peer):
engine, requests = peer
with pytest.raises(ValueError, match = "Tools"):
list(engine.generate(messages = [], tools = [{"type": "function"}]))
assert requests == []
@_LOCAL_ENGINE_HOST
def test_interrupted_install_is_visible_after_restart(isolated):
(isolated / "vllm.job.json").write_text(json.dumps({"state": "running", "phase": "installing"}))
assert install.status("vllm")["job"]["state"] == "error"
assert "interrupted" in install.status("vllm")["job"]["message"]
def test_install_routes_require_owner(isolated, monkeypatch):
from fastapi import FastAPI, HTTPException
from fastapi.testclient import TestClient
from routes.engines import router
from auth import policy
from auth.authentication import get_current_subject
app = FastAPI()
app.include_router(router, prefix = "/api/engines")
app.dependency_overrides[get_current_subject] = lambda: "member"
def deny():
raise HTTPException(status_code = 403)
app.dependency_overrides[policy.require_owner] = deny
called = []
monkeypatch.setattr(install, "start_install", lambda name: called.append(name))
with TestClient(app) as client:
assert client.post("/api/engines/vllm/install").status_code == 403
assert client.delete("/api/engines/vllm").status_code == 403
assert client.post("/api/engines/vllm/cancel").status_code == 403
assert client.post("/api/engines/vllm/rollback").status_code == 403
assert called == []
def test_engine_routes_reap_a_crashed_engine_before_reporting(isolated, monkeypatch):
from fastapi import FastAPI
from fastapi.testclient import TestClient
from routes import inference as inference_routes
from routes.engines import router
from auth import policy
from auth.authentication import get_current_subject
lease = install.engine_lease("vllm")
lease.__enter__()
class Crashed:
_managed_engine = object()
reaped = 0
def reap_dead_managed_engine(self):
self.reaped += 1
lease.__exit__(None, None, None)
backend = Crashed()
monkeypatch.setattr(inference_routes, "_peek_inference_backend", lambda: backend)
app = FastAPI()
app.include_router(router, prefix = "/api/engines")
app.dependency_overrides[get_current_subject] = lambda: "owner"
app.dependency_overrides[policy.require_owner] = lambda: None
with TestClient(app) as client:
assert client.delete("/api/engines/vllm").status_code != 409
assert backend.reaped == 1
@pytest.mark.parametrize("engine", ["vllm", "sglang"])
def test_model_metadata_allows_native_vision_and_quantization(tmp_path, engine):
from types import SimpleNamespace
from core.inference.managed_engine import validate_model
config = SimpleNamespace(is_local = True, path = str(tmp_path))
path = tmp_path / "config.json"
path.write_text(
json.dumps(
{
"model_type": "qwen3_5",
"vision_config": {"hidden_size": 32},
"text_config": {"num_attention_heads": 8},
"quantization_config": {"quant_method": "awq"},
}
)
)
options = validate_model(config, gpu_ids = [0, 1], engine = engine)
assert options["is_vision"] and options["quantization"] == "awq"
with pytest.raises(ValueError, match = "already quantized"):
validate_model(config, engine = engine, precision = "int4")
path.write_text(
json.dumps({"quantization_config": {"quant_method": "bitsandbytes", "load_in_8bit": True}})
)
assert validate_model(config, engine = engine)["load_format"] == "bitsandbytes"
with pytest.raises(ValueError, match = "do not support tensor parallelism"):
validate_model(config, gpu_ids = [0, 1], engine = engine)
def test_engine_override_only_applies_to_safetensors():
from utils.openai_auto_switch_settings import (
normalize_model_override,
model_override_load_kwargs,
)
saved = normalize_model_override({"engine": "sglang"})
assert model_override_load_kwargs(saved, is_gguf = False) == {
"engine": "sglang",
"engine_precision": "auto",
"engine_parallelism": "tensor",
"load_in_4bit": False,
}
assert "engine" not in model_override_load_kwargs(saved, is_gguf = True)
assert normalize_model_override({"engine": "not-an-engine"}) == {}
def test_orchestrator_dispatches_managed_generation_without_worker(peer):
from core.inference.orchestrator import InferenceOrchestrator
engine, _ = peer
orchestrator = InferenceOrchestrator.__new__(InferenceOrchestrator)
orchestrator._managed_engine = engine
assert list(orchestrator._generate_inner(messages = [{"role": "user", "content": "Hello"}])) == [
"hello",
"hello world",
]
def test_an_engine_rejecting_the_schema_is_a_response_format_refusal():
from core.inference.orchestrator import InferenceOrchestrator
errors = ["Grammar error: Invalid type: ?"]
class Handler(BaseHTTPRequestHandler):
def log_message(self, *args):
pass
def do_POST(self):
self.rfile.read(int(self.headers["Content-Length"]))
self.send_response(200)
self.send_header("Content-Type", "text/event-stream")
self.end_headers()
error = {"message": errors[-1], "type": "BadRequestError", "code": 400}
self.wfile.write(f"data: {json.dumps({'error': error})}\n\n".encode())
self.wfile.flush()
server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
thread = threading.Thread(target = server.serve_forever, daemon = True)
thread.start()
try:
engine = ManagedEngine("vllm")
engine.base_url = f"http://127.0.0.1:{server.server_port}"
engine.model = "org/model"
orchestrator = InferenceOrchestrator.__new__(InferenceOrchestrator)
orchestrator._managed_engine = engine
messages = [{"role": "user", "content": "Hi"}]
schema = {"type": "json_schema", "json_schema": {"name": "x", "schema": {"type": "?"}}}
[refusal] = orchestrator._generate_inner(messages = messages, response_format = schema)
assert refusal.openai_param == "response_format"
assert "Grammar error" in refusal
[failure] = orchestrator._generate_inner(messages = messages)
assert failure.openai_param is None
assert failure == "Engine generation failed."
errors.append("This model's maximum context length is 2048 tokens.")
[overflow] = orchestrator._generate_inner(messages = messages, response_format = schema)
assert overflow.openai_param is None
assert overflow == "Engine generation failed."
finally:
server.shutdown()
server.server_close()
thread.join(2)
def test_managed_load_launches_the_validated_local_path(monkeypatch):
import threading as _threading
from types import SimpleNamespace
from core.inference import managed_engine
from core.inference.engine_adapters import ADAPTERS
from core.inference.orchestrator import InferenceOrchestrator
started = {}
class Engine:
context = 4096
def __init__(self, name):
pass
def start(
self,
model,
*args,
model_path = None,
):
started.update(model = model, model_path = model_path)
def alive(self):
return True
monkeypatch.setattr(managed_engine, "ManagedEngine", Engine)
from core.inference import worker
scanned = []
monkeypatch.setattr(
worker, "_run_security_gates", lambda targets, **kwargs: scanned.extend(targets) or True
)
orchestrator = InferenceOrchestrator.__new__(InferenceOrchestrator)
orchestrator._managed_engine = None
orchestrator._subprocess_shutdown_lock = _threading.RLock()
orchestrator._shutdown_subprocess = lambda *args, **kwargs: True
orchestrator.models, orchestrator.loading_models = {}, set()
orchestrator.active_model_name, orchestrator.load_generation = None, 0
config = SimpleNamespace(
identifier = "C:\\models\\foo", path = "/mnt/c/models/foo", is_local = True, is_vision = False
)
assert orchestrator._load_managed_engine("vllm", config, 4096, [0], None, None, None, False)
assert started == {"model": "C:\\models\\foo", "model_path": "/mnt/c/models/foo"}
assert scanned == ["/mnt/c/models/foo"]
assert orchestrator.active_model_name == "C:\\models\\foo"
args = ADAPTERS["vllm"].command(
"python", "/mnt/c/models/foo", 1, "key", 4096, 0.8, served_model_name = "C:\\models\\foo"
)
assert args[args.index("--model") + 1] == "/mnt/c/models/foo"
assert args[args.index("--served-model-name") + 1] == "C:\\models\\foo"
def test_validate_rejects_engine_settings_before_the_picker_unloads(monkeypatch, tmp_path):
import asyncio
from types import SimpleNamespace
from fastapi import HTTPException
from core.inference import managed_engine
from models.inference import ValidateModelRequest
from routes import inference as route
(tmp_path / "config.json").write_text(
json.dumps({"quantization_config": {"quant_method": "awq"}})
)
config = SimpleNamespace(
identifier = str(tmp_path),
path = str(tmp_path),
display_name = "model",
is_local = True,
is_gguf = False,
is_lora = False,
is_audio = False,
is_vision = False,
)
monkeypatch.setattr(managed_engine, "support_reason", lambda *args: None)
monkeypatch.setattr(
managed_engine,
"installed",
lambda engine: {"path": "/env", "profile_digest": install.profile_digest(engine)},
)
monkeypatch.setattr(
route,
"_resolve_model_identifier_for_request",
lambda *args, **kwargs: (str(tmp_path), str(tmp_path), False),
)
monkeypatch.setattr(route.ModelConfig, "from_identifier", lambda **kwargs: config)
request = ValidateModelRequest(model_path = str(tmp_path), engine = "vllm", engine_precision = "bf16")
with pytest.raises(HTTPException) as raised:
asyncio.run(route.validate_model(request, current_subject = "test-user"))
assert raised.value.status_code == 400
assert "already quantized" in raised.value.detail
def test_busy_install_does_not_overwrite_another_job(isolated):
job = {"state": "running", "phase": "installing", "message": "Downloading"}
(isolated / "vllm.job.json").write_text(json.dumps(job))
with install.engine_lease("vllm", exclusive = True):
with pytest.raises(RuntimeError, match = "another Unsloth"):
install.start_install("vllm")
assert install.status("vllm")["job"] == job
def test_another_instance_can_request_install_cancellation(isolated):
(isolated / "vllm.job.json").write_text(json.dumps({"state": "running"}))
with install.engine_lease("vllm", exclusive = True):
install.cancel_install("vllm")
assert (isolated / "vllm.cancel").exists()
def test_rollback_rejects_traversal(isolated):
marker = active(isolated)
info = json.loads(marker.read_text())
info["previous"] = {"directory": "../env-other"}
marker.write_text(json.dumps(info))
with pytest.raises(RuntimeError, match = "No previous"):
install.rollback("vllm")
assert install.installed("vllm")["directory"] == "env-prior"
def test_training_always_evicts_managed_engine(monkeypatch):
from routes import training_vram
monkeypatch.setattr(
training_vram, "summarize_resident_chat", lambda: {"any": True, "managed_engine": True}
)
monkeypatch.setattr(training_vram, "summarize_resident_stt", lambda: {"any": False})
monkeypatch.setattr(training_vram, "free_stt_model_for_training", lambda **kw: [])
monkeypatch.setattr(training_vram, "free_chat_models_for_training", lambda **kw: ["hf:model"])
def would_keep():
pytest.fail("Managed servers must be evicted even when idle VRAM appears sufficient")
assert training_vram.coordinate_models_for_training(would_keep) == ["hf:model"]
def test_failed_managed_stop_blocks_training(monkeypatch):
from types import SimpleNamespace
import core.inference
from routes.training_vram import free_chat_models_for_training
backend = SimpleNamespace(
active_model_name = "model",
loading_models = set(),
models = {"model": {}},
_managed_engine = object(),
_shutdown_subprocess = lambda: False,
)
monkeypatch.setattr(core.inference, "get_inference_backend", lambda: backend)
with pytest.raises(RuntimeError, match = "could not be stopped"):
free_chat_models_for_training("test")
assert backend.active_model_name == "model"
def test_failed_managed_stop_keeps_the_chat_gpu_claim_on_unload(monkeypatch):
import asyncio
from types import SimpleNamespace
from fastapi import HTTPException
from core.inference.orchestrator import InferenceOrchestrator
from models.inference import UnloadRequest
import routes.inference as inference_route
backend = InferenceOrchestrator.__new__(InferenceOrchestrator)
backend._subprocess_shutdown_lock = threading.RLock()
backend.active_model_name = "model"
backend.models = {"model": {"engine": "vllm"}}
backend.loading_models = set()
backend._managed_engine = SimpleNamespace(stop = lambda: False)
released = []
monkeypatch.setattr(inference_route, "get_inference_backend", lambda: backend)
monkeypatch.setattr(
inference_route,
"get_llama_cpp_backend",
lambda: SimpleNamespace(is_active = False, is_loaded = False, model_identifier = None),
)
monkeypatch.setattr(inference_route, "release_chat_gpu_claim", lambda: released.append(True))
with pytest.raises(HTTPException) as raised:
asyncio.run(inference_route._unload_model_impl(UnloadRequest(model_path = "model"), "tester"))
assert raised.value.status_code == 500
assert released == []
assert backend.active_model_name == "model"
def test_failed_managed_stop_during_load_cancel_keeps_chat_busy(monkeypatch):
from types import SimpleNamespace
import core.inference.gpu_arbiter as gpu_arbiter
import core.inference.llama_cpp as llama_cpp
from core.inference.orchestrator import InferenceOrchestrator
import routes.inference as inference_route
backend = InferenceOrchestrator.__new__(InferenceOrchestrator)
backend._subprocess_shutdown_lock = threading.RLock()
backend.active_model_name = None
backend.models = {}
backend.loading_models = {"model"}
backend._managed_engine = SimpleNamespace(stop = lambda: False)
with pytest.raises(RuntimeError, match = "did not stop"):
backend.cancel_load("model")
assert backend._managed_engine is not None
monkeypatch.setattr(inference_route, "_peek_inference_backend", lambda: backend)
monkeypatch.setattr(
inference_route,
"get_llama_cpp_backend",
lambda: SimpleNamespace(is_active = False, is_loaded = False),
)
monkeypatch.setattr(llama_cpp, "chat_load_active", lambda: False)
monkeypatch.setattr(gpu_arbiter, "release_if", lambda owner, idle: idle())
assert inference_route.release_chat_gpu_claim() is False
backend._managed_engine = None
assert inference_route.release_chat_gpu_claim() is True
def test_dead_managed_server_is_reaped_without_clearing_live_server(monkeypatch):
from types import SimpleNamespace
from core.inference.orchestrator import InferenceOrchestrator
backend = InferenceOrchestrator.__new__(InferenceOrchestrator)
backend._subprocess_shutdown_lock = threading.RLock()
backend.active_model_name = "model"
backend._managed_engine = SimpleNamespace(phase = "ready", alive = lambda: True)
stopped = []
backend._shutdown_subprocess = lambda: stopped.append(True)
backend.reap_dead_managed_engine()
assert stopped == []
backend._managed_engine.alive = lambda: False
backend.reap_dead_managed_engine()
assert stopped == [True]
@pytest.mark.parametrize("engine", ["vllm", "sglang"])
def test_adapter_uses_measured_memory_fraction(engine):
from core.inference.engine_adapters import ADAPTERS
args = ADAPTERS[engine].command("/env/python", "model", 40000, "key", 2048, 0.653)
option = "--gpu-memory-utilization" if engine == "vllm" else "--mem-fraction-static"
assert args[args.index(option) + 1] == "0.653"
def test_memory_budget_uses_selected_gpu_and_reserves_headroom(monkeypatch):
from types import SimpleNamespace
from core.inference import engine_adapters
from utils import vram_budget_settings
commands = []
def query(argv, **kwargs):
commands.append(argv)
return SimpleNamespace(stdout = "24576, 16384")
monkeypatch.setattr(engine_adapters.subprocess, "run", query)
monkeypatch.setattr(vram_budget_settings, "get_vram_budget_fraction", lambda: 0.97)
assert engine_adapters.gpu_memory_fraction([1]) == 0.645
assert commands[0][commands[0].index("--id") + 1] == "1"
# vLLM TorchAO weights overrun the reservation in sampler warmup, so keep 6 GiB more free.
assert engine_adapters.gpu_memory_fraction([1], 512 + 6144) == 0.395
monkeypatch.setattr(
engine_adapters.subprocess, "run", lambda *a, **kw: SimpleNamespace(stdout = "24576, 24000")
)
monkeypatch.setattr(vram_budget_settings, "get_vram_budget_fraction", lambda: 0.85)
assert engine_adapters.gpu_memory_fraction([1]) == 0.85
# A GGUF-only low budget must not shrink an engine that cannot offload to the CPU.
monkeypatch.setattr(vram_budget_settings, "get_vram_budget_fraction", lambda: 0.3)
assert engine_adapters.gpu_memory_fraction([1]) == engine_adapters.MANAGED_ENGINE_BUDGET_FLOOR
monkeypatch.setattr(
engine_adapters.subprocess, "run", lambda *a, **kw: SimpleNamespace(stdout = "24576, 256")
)
with pytest.raises(RuntimeError, match = "reserve memory"):
engine_adapters.gpu_memory_fraction([1])
def test_memory_reserve_grows_with_the_card(monkeypatch):
from types import SimpleNamespace
from core.inference import engine_adapters
from utils import vram_budget_settings
monkeypatch.setattr(vram_budget_settings, "get_vram_budget_fraction", lambda: 0.97)
share = engine_adapters.RESERVE_SHARE
# On an idle 96 GB RTX PRO 6000, a 3 GiB reserve left vLLM sampler warmup out of memory.
monkeypatch.setattr(
engine_adapters.subprocess, "run", lambda *a, **kw: SimpleNamespace(stdout = "97887, 97300")
)
fraction = engine_adapters.gpu_memory_fraction([0], 3072, share)
assert 97887 * (1 - fraction) >= 0.06 * 97887 - 1
assert fraction < engine_adapters.gpu_memory_fraction([0], 3072)
# A 24 GB L4 keeps the fixed reserve because 6% would be less than 3 GiB.
monkeypatch.setattr(
engine_adapters.subprocess, "run", lambda *a, **kw: SimpleNamespace(stdout = "23034, 22700")
)
assert engine_adapters.gpu_memory_fraction(
[0], 3072, share
) == engine_adapters.gpu_memory_fraction([0], 3072)
def test_engine_start_reserves_a_share_of_the_card():
import inspect
import re
from core.inference import managed_engine
# The local and the WSL launch both keep the share free.
source = inspect.getsource(managed_engine)
assert (
len(re.findall(r"memory_reserve_mib\(self\.engine, options\),\s*RESERVE_SHARE", source))
== 2
)
@_LOCAL_ENGINE_HOST
@pytest.mark.parametrize("gpu_ids", [[1], [1, 0]])
def test_server_outlives_short_lived_start_thread(isolated, monkeypatch, gpu_ids, tmp_path):
import os
import sys
import time
from types import SimpleNamespace
from core.inference import managed_engine
active(isolated)
monkeypatch.setattr(managed_engine, "gpu_memory_fraction", lambda *_: 0.8)
engine = ManagedEngine("vllm")
def command(python, model, port, key, context, memory, tensor_parallel_size):
assert tensor_parallel_size == len(gpu_ids)
code = (
"import os\n"
"from http.server import BaseHTTPRequestHandler, HTTPServer\n"
"class Handler(BaseHTTPRequestHandler):\n"
" def do_GET(self):\n"
" self.send_response(200); self.end_headers()\n"
" self.wfile.write('|'.join(os.environ.get(k, '') for k in ('CUDA_VISIBLE_DEVICES', 'TRITON_CACHE_DIR', 'FLASHINFER_WORKSPACE_BASE', 'LD_LIBRARY_PATH')).encode())\n"
" def log_message(self, *args): pass\n"
f"HTTPServer(('127.0.0.1', {port}), Handler).serve_forever()\n"
)
return [sys.executable, "-u", "-c", code]
engine.adapter = SimpleNamespace(
command = command,
progress = lambda _: None,
environment = lambda _: {},
key_environment = lambda _: {},
)
errors = []
# Colab reaches libcuda only via LD_LIBRARY_PATH; a CUDA runtime there must not follow.
driver, runtime = tmp_path / "lib64-nvidia", tmp_path / "cuda-12" / "lib64"
for folder, library in ((driver, "libcuda.so.1"), (runtime, "libcudart.so.12")):
folder.mkdir(parents = True)
(folder / library).write_bytes(b"")
def start():
try:
engine.start(
"model",
2048,
gpu_ids,
dict(
os.environ,
TRITON_CACHE_DIR = "/shared-cache",
FLASHINFER_WORKSPACE_BASE = "/home",
LD_LIBRARY_PATH = os.pathsep.join(map(str, (runtime, driver))),
),
)
except Exception as exc:
errors.append(exc)
thread = threading.Thread(target = start)
try:
thread.start()
thread.join(15)
assert not thread.is_alive() and not errors
time.sleep(0.2)
assert engine.alive()
import httpx
visible, cache_path, flashinfer_base, library_path = httpx.get(
engine.base_url, trust_env = False
).text.split("|")
assert library_path == str(driver)
assert flashinfer_base == install.installed("vllm")["path"]
assert visible == ",".join(map(str, gpu_ids))
assert Path(cache_path).parent.parent == isolated / "vllm" / "cache"
assert Path(cache_path).name == "triton"
with pytest.raises(RuntimeError):
install.remove("vllm")
finally:
assert engine.stop()
thread.join(5)
with install.engine_lease("vllm", exclusive = True):
pass
def test_shared_http_reader_closes_on_early_exit():
import asyncio
from core.inference.http_stream import closing_response_lines
closed = []
class Response:
async def aiter_lines(self):
try:
yield "data: first"
yield "data: second"
finally:
closed.append("iterator")
async def aclose(self):
closed.append("response")
async def consume():
lines = closing_response_lines(Response())
assert await anext(lines) == "data: first"
await lines.aclose()
asyncio.run(consume())
assert closed == ["response", "iterator"]
def test_quiet_installer_can_be_cancelled(isolated):
import sys
import time
cancel = threading.Event()
timer = threading.Timer(0.3, cancel.set)
timer.start()
started = time.monotonic()
try:
with pytest.raises(RuntimeError, match = "cancelled"):
install._run("vllm", [sys.executable, "-c", "import time; time.sleep(30)"], cancel)
assert time.monotonic() - started < 5
finally:
timer.cancel()
@_LOCAL_ENGINE_HOST
@pytest.mark.parametrize("kind", ["installer", "server"])
def test_shutdown_during_adoption_reaps_child(isolated, monkeypatch, kind):
import os
import sys
from types import SimpleNamespace
from core.inference import managed_engine
from utils import process_lifetime
shutting_down = threading.Event()
children = []
original_spawn = process_lifetime.spawn_on_lifetime_thread
original_adopt = process_lifetime.adopt_pid
def spawn(factory):
child = original_spawn(factory)
children.append(child)
return child
def adopt(pid):
original_adopt(pid)
shutting_down.set()
monkeypatch.setattr(process_lifetime, "spawn_on_lifetime_thread", spawn)
monkeypatch.setattr(process_lifetime, "adopt_pid", adopt)
monkeypatch.setattr(process_lifetime, "is_process_shutting_down", shutting_down.is_set)
command = [sys.executable, "-c", "import time; time.sleep(30)"]
engine = None
try:
with pytest.raises(RuntimeError, match = "shutting down"):
if kind == "installer":
install._run("vllm", command, threading.Event())
else:
active(isolated)
monkeypatch.setattr(managed_engine, "gpu_memory_fraction", lambda *_: 0.8)
engine = ManagedEngine("vllm")
engine.adapter = SimpleNamespace(
command = lambda *args: command,
environment = lambda _: {},
key_environment = lambda _: {},
)
engine.start("model", 2048, [0], dict(os.environ))
assert len(children) == 1
assert children[0].poll() is not None
assert children[0].stdout.closed
if engine is not None:
assert engine.process is None
with install.engine_lease("vllm", exclusive = True):
pass
finally:
for child in children:
if child.poll() is None:
process_lifetime.terminate_pid(child.pid, timeout = 5, owner_verified = True)
child.wait(timeout = 5)
process_lifetime.forget_pid(child.pid)
if engine is not None:
engine.stop()
@_POSIX_ENGINE_LOCKS
@pytest.mark.parametrize("supported", [True, False])
def test_package_clone_probe_falls_back_to_independent_copies(tmp_path, monkeypatch, supported):
import errno
import fcntl
cache = tmp_path / "cache"
destination = tmp_path / "environment"
destination.mkdir()
monkeypatch.setattr(install, "install_environment", lambda: {"UV_CACHE_DIR": str(cache)})
calls = []
def clone(target, operation, source):
calls.append(operation)
if not supported:
raise OSError(errno.EOPNOTSUPP, "No reflinks on this filesystem")
monkeypatch.setattr(fcntl, "ioctl", clone)
assert install.package_link_mode(destination) == ("clone" if supported else "copy")
assert calls == [0x40049409]
assert list(cache.iterdir()) == []
assert list(destination.iterdir()) == []
@pytest.mark.parametrize("engine", ["vllm", "sglang"])
def test_managed_capabilities_validate_in_load_and_status_responses(engine):
from types import SimpleNamespace
from routes.inference import _detect_safetensors_features
from models.inference import _InferenceRuntimeFields
backend = SimpleNamespace(active_model_name = "model", models = {"model": {"engine": engine}})
features = _detect_safetensors_features(backend, None)
runtime = _InferenceRuntimeFields(**features)
assert runtime.supports_reasoning is False
assert runtime.supports_tools is False
assert runtime.reasoning_style == "enable_thinking"
@pytest.mark.parametrize("limit", [None, -1, 0, 64, 4096, 8192])
def test_max_tokens_sentinel_is_not_sent_as_a_negative_limit(peer, limit):
engine, requests = peer
engine.context = 4096
assert list(engine.generate(messages = [{"role": "user", "content": "Hi"}], max_new_tokens = limit))
# A cap is first fitted to the window through /tokenize.
payload = next(body for path, _, body in requests if path.endswith("/chat/completions"))
if limit is not None and 0 < limit < engine.context:
assert payload["max_tokens"] == limit
else:
assert "max_tokens" not in payload
@pytest.mark.parametrize("engine", ["vllm", "sglang"])
def test_managed_gpu_change_does_not_reuse_resident(engine):
from types import SimpleNamespace
from models.inference import LoadRequest
from routes.inference import _non_gguf_runtime_settings_match
backend = SimpleNamespace(
active_model_name = "model",
models = {
"model": {
"engine": engine,
"gpu_ids_requested": [1],
"max_seq_length_requested": 4096,
}
},
)
request = LoadRequest(model_path = "model", engine = engine, gpu_ids = [1], max_seq_length = 4096)
assert _non_gguf_runtime_settings_match(backend, request)
assert not _non_gguf_runtime_settings_match(
backend, request.model_copy(update = {"gpu_ids": [0]})
)
assert not _non_gguf_runtime_settings_match(
backend, request.model_copy(update = {"max_seq_length": 8192})
)
@pytest.mark.parametrize("engine", ["vllm", "sglang"])
def test_managed_token_count_route_uses_engine_capabilities(monkeypatch, engine):
import asyncio
from types import SimpleNamespace
from fastapi import HTTPException
from models.inference import ChatCountTokensRequest
from routes import inference
calls = []
def count(messages, system_prompt, **kwargs):
calls.append(messages)
return 17, "model"
backend = SimpleNamespace(
active_model_name = "model",
models = {"model": {"engine": engine}},
load_generation = 1,
count_chat_tokens = count,
)
monkeypatch.setattr(inference, "get_inference_backend", lambda: backend)
payload = ChatCountTokensRequest(
model = "model", messages = [{"role": "user", "content": "Hi"}], enable_tools = False
)
if engine == "sglang":
with pytest.raises(HTTPException, match = "SGLang") as error:
asyncio.run(inference._mlx_count_chat_tokens(payload))
assert error.value.status_code == 503
assert not calls
else:
response = asyncio.run(inference._mlx_count_chat_tokens(payload))
assert json.loads(response.body)["input_tokens"] == 17
assert calls == [[{"role": "user", "content": "Hi"}]]
def test_managed_token_count_accepts_reasoning_turned_off(monkeypatch):
import asyncio
from types import SimpleNamespace
from fastapi import HTTPException
from models.inference import ChatCountTokensRequest
from routes import inference
backend = SimpleNamespace(
active_model_name = "model",
models = {"model": {"engine": "vllm"}},
load_generation = 1,
count_chat_tokens = lambda messages, system_prompt, **kwargs: (17, "model"),
)
monkeypatch.setattr(inference, "get_inference_backend", lambda: backend)
def request(effort):
return ChatCountTokensRequest(
model = "model",
messages = [{"role": "user", "content": "Hi"}],
enable_tools = False,
reasoning_effort = effort,
)
response = asyncio.run(inference._mlx_count_chat_tokens(request("none")))
assert json.loads(response.body)["input_tokens"] == 17
with pytest.raises(HTTPException) as error:
asyncio.run(inference._mlx_count_chat_tokens(request("high")))
assert error.value.status_code == 503
@pytest.mark.parametrize("engine", ["vllm", "sglang"])
@pytest.mark.parametrize("size", [1, 2, 4])
def test_tensor_parallel_launch_stays_on_selected_local_devices(engine, size):
args = launch_arguments(engine, "/env/python", "model", 40000, "key", 4096, 0.8, size)
assert args[args.index("--tensor-parallel-size") + 1] == str(size)
if engine == "vllm" and size > 1:
assert args[args.index("--distributed-executor-backend") + 1] == "mp"
if engine == "sglang":
assert ("--enable-p2p-check" in args) == (size > 1)
from core.inference.engine_adapters import ADAPTERS
multi_gpu = (
{
"vllm": {"VLLM_HOST_IP": "127.0.0.1"},
"sglang": {
"SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK": "0",
"FLASHINFER_USE_CUDA_NORM": "1",
},
}[engine]
if size > 1
else {}
)
always = {"TRITON_ALLOW_NON_CONSTEXPR_GLOBALS": "1"} if engine == "vllm" else {}
assert ADAPTERS[engine].environment(size) == {**always, **multi_gpu}
@pytest.mark.parametrize("engine", ["vllm", "sglang"])
def test_multi_gpu_preflight_checks_every_device(engine, monkeypatch):
from core.inference import managed_engine
from models.inference import LoadRequest
checked = []
info = {"path": "/env", "profile_digest": install.profile_digest(engine)}
monkeypatch.setattr(managed_engine, "installed", lambda _: info)
monkeypatch.setattr(managed_engine, "support_reason", lambda _, gpu: checked.append(gpu))
request = LoadRequest(model_path = "model", gpu_ids = [1, 0], engine = engine)
managed_engine.validate_load(engine, request)
assert checked == [1, 0]
info["profile_digest"] = "outdated"
with pytest.raises(ValueError, match = "Update"):
managed_engine.validate_load(engine, request)
info["restored"] = True
managed_engine.validate_load(engine, request)
managed_engine.validate_load(engine, request.model_copy(update = {"tensor_parallel": True}))
for ids in ([0, 0], [-1, 0]):
with pytest.raises(ValueError, match = "distinct"):
managed_engine.validate_load(engine, request.model_copy(update = {"gpu_ids": ids}))
monkeypatch.setattr(
managed_engine, "support_reason", lambda _, gpu: "Unsupported device" if gpu == 0 else None
)
with pytest.raises(ValueError, match = "GPU 0: Unsupported"):
managed_engine.validate_load(engine, request)
@pytest.mark.parametrize(("version", "refused"), [("0.5.20", True), ("0.5.17", False)])
def test_sglang_integer_precision_is_refused_before_unloading(monkeypatch, version, refused):
# Checked in validate_load, which runs before the resident model is torn down.
from core.inference import managed_engine
from models.inference import LoadRequest
info = {"path": "/env", "version": version, "profile_digest": install.profile_digest("sglang")}
monkeypatch.setattr(managed_engine, "installed", lambda _: info)
monkeypatch.setattr(managed_engine, "support_reason", lambda *a: None)
monkeypatch.setattr(managed_engine, "resolve_requested_gpu_ids", lambda ids: [0])
for precision in ("int8", "int4"):
request = LoadRequest(model_path = "m", engine = "sglang", engine_precision = precision)
if refused:
with pytest.raises(ValueError, match = "Choose FP8"):
managed_engine.validate_load("sglang", request)
else:
managed_engine.validate_load("sglang", request)
request = LoadRequest(model_path = "m", engine = "sglang", engine_precision = "fp8")
managed_engine.validate_load("sglang", request)
def test_engine_gpus_stay_inside_studios_visible_gpus(monkeypatch):
from core.inference import managed_engine
from models.inference import LoadRequest
from utils.hardware import hardware
monkeypatch.setattr(
managed_engine, "resolve_requested_gpu_ids", hardware.resolve_requested_gpu_ids
)
monkeypatch.setattr(hardware, "get_device", lambda: hardware.DeviceType.CUDA)
monkeypatch.setattr(hardware, "IS_ROCM", False)
monkeypatch.setattr(hardware, "get_physical_gpu_count", lambda: 4)
monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "2,3")
info = {"path": "/env", "profile_digest": install.profile_digest("vllm")}
monkeypatch.setattr(managed_engine, "installed", lambda _: info)
monkeypatch.setattr(managed_engine, "support_reason", lambda *args: None)
request = LoadRequest(model_path = "model", engine = "vllm")
assert managed_engine.validate_load("vllm", request) == [2]
explicit = request.model_copy(update = {"gpu_ids": [3, 2]})
assert managed_engine.validate_load("vllm", explicit) == [3, 2]
with pytest.raises(ValueError, match = "outside the parent-visible set"):
managed_engine.validate_load("vllm", request.model_copy(update = {"gpu_ids": [0]}))
monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "")
with pytest.raises(ValueError, match = "no GPU"):
managed_engine.validate_load("vllm", request)
def test_multi_gpu_memory_budget_uses_most_constrained_device(monkeypatch):
from types import SimpleNamespace
from core.inference import engine_adapters
from utils import vram_budget_settings
commands = []
def query(args, **kwargs):
commands.append(args)
return SimpleNamespace(stdout = "49152, 40000\n24576, 8192\n")
monkeypatch.setattr(engine_adapters.subprocess, "run", query)
monkeypatch.setattr(vram_budget_settings, "get_vram_budget_fraction", lambda: 0.97)
assert engine_adapters.gpu_memory_fraction([0, 1]) == 0.312
assert commands[0][commands[0].index("--id") + 1] == "0,1"
for output in ("49152, 40000", "49152, 40000\n24576, 256"):
monkeypatch.setattr(
engine_adapters.subprocess, "run", lambda *a, **kw: SimpleNamespace(stdout = output)
)
with pytest.raises(RuntimeError, match = "every selected GPU"):
engine_adapters.gpu_memory_fraction([0, 1])
@pytest.mark.parametrize(
"overrides,size,valid",
[
({}, 2, True),
({}, 4, True), # Two KV heads replicated over four ranks.
({}, 3, False),
({"num_attention_heads": 7}, 2, False),
({"intermediate_size": 1537}, 2, False),
({"hidden_size": 769}, 2, False),
({"num_key_value_heads": 6}, 4, False),
],
)
def test_tensor_parallel_model_validation_before_handoff(tmp_path, overrides, size, valid):
from types import SimpleNamespace
from core.inference.managed_engine import validate_model
metadata = {
"model_type": "qwen2",
"num_attention_heads": 12,
"num_key_value_heads": 2,
"hidden_size": 768,
"intermediate_size": 1536,
**overrides,
}
(tmp_path / "config.json").write_text(json.dumps(metadata))
config = SimpleNamespace(is_local = True, path = str(tmp_path))
if valid:
validate_model(config, gpu_ids = list(range(size)))
else:
with pytest.raises(ValueError, match = "GPUs"):
validate_model(config, gpu_ids = list(range(size)))
def test_installer_publishes_bounded_output_before_exit(isolated):
import sys
import time
errors = []
cancel = threading.Event()
code = "import time; print('Downloading torch (900 MiB)', flush=True); time.sleep(2); [print('Downloaded package-' + str(i), flush=True) for i in range(30)]"
def run():
try:
install._run("vllm", [sys.executable, "-u", "-c", code], cancel)
except Exception as exc:
errors.append(exc)
thread = threading.Thread(target = run)
thread.start()
try:
deadline = time.monotonic() + 2
while time.monotonic() < deadline:
path = isolated / "vllm.job.json"
if path.exists():
job = json.loads(path.read_text())
if job.get("activity") == "Downloading torch (900 MiB)":
break
time.sleep(0.02)
else:
pytest.fail("Installer output was not published while the subprocess was running")
assert thread.is_alive()
finally:
thread.join(10)
assert not thread.is_alive() and not errors
job = json.loads((isolated / "vllm.job.json").read_text())
assert len(job["log"]) == 20
assert job["activity"] == "Downloaded package-29"
@pytest.mark.parametrize("engine", ["vllm", "sglang"])
@pytest.mark.parametrize("precision", ["auto", "bf16", "fp16", "int4", "int8", "fp8"])
def test_native_precision_arguments(engine, precision):
from core.inference.engine_adapters import ADAPTERS
args = ADAPTERS[engine].command(
"python",
"model",
12345,
"key",
4096,
0.8,
options = {
"precision": precision,
"load_format": "bitsandbytes" if engine == "vllm" and precision == "int4" else "auto",
},
)
expected = {
"bf16": "bfloat16",
"fp16": "float16",
"int4": "bitsandbytes" if engine == "vllm" else "int4wo-32",
"fp8": "torchao" if engine == "vllm" else "fp8",
"int8": "torchao" if engine == "vllm" else "int8wo",
}
if precision == "auto":
assert "--quantization" not in args and "--dtype" not in args
else:
assert expected[precision] in args
assert args[args.index("--load-format") + 1] == (
"bitsandbytes" if engine == "vllm" and precision == "int4" else "auto"
)
# Only TorchAO weights leave SGLang's Blackwell BF16 GEMM for F.linear.
torchao = engine == "sglang" and precision in ("int4", "int8")
assert ("--bf16-gemm-backend" in args) is torchao
if torchao:
assert args[args.index("--bf16-gemm-backend") + 1] == "torch"
@pytest.mark.parametrize("encoded", [False, True])
def test_managed_images_keep_their_conversation_positions(peer, encoded):
from PIL import Image
engine, requests = peer
messages = [
{"role": "user", "content": [{"type": "image"}, {"type": "text", "text": "First?"}]},
{"role": "assistant", "content": "Red"},
{"role": "user", "content": [{"type": "text", "text": "Second?"}, {"type": "image"}]},
]
images = [Image.new("RGB", (4, 4), "red"), Image.new("RGB", (4, 4), "blue")]
if encoded:
from core.inference.orchestrator import InferenceOrchestrator
images = [InferenceOrchestrator._pil_to_base64(image) for image in images]
list(engine.generate(messages = messages, images = images))
sent = requests[0][2]["messages"]
assert sent[0]["content"][0]["type"] == "image_url"
assert sent[2]["content"][1]["type"] == "image_url"
assert sent[0]["content"][0]["image_url"] != sent[2]["content"][1]["image_url"]
assert messages[0]["content"][0]["type"] == "image"
@pytest.mark.parametrize("engine", ["vllm", "sglang"])
def test_prequantized_int8_uses_uncaptured_sglang_execution(tmp_path, engine):
from types import SimpleNamespace
from core.inference.managed_engine import validate_model
from core.inference.engine_adapters import ADAPTERS
(tmp_path / "config.json").write_text(
json.dumps({"quantization_config": {"quant_method": "bitsandbytes", "load_in_8bit": True}})
)
options = validate_model(SimpleNamespace(is_local = True, path = str(tmp_path)), engine = engine)
args = ADAPTERS[engine].command("python", "model", 12345, "key", 4096, 0.8, options = options)
assert ("--disable-cuda-graph" in args) == (engine == "sglang")
assert ("--disable-piecewise-cuda-graph" in args) == (engine == "sglang")
@pytest.mark.parametrize(
"capabilities, eager", [("8.9\n", False), ("8.6\n", True), ("8.9\n8.6\n", True)]
)
@pytest.mark.parametrize("engine", ["vllm", "sglang"])
def test_fp8_conversion_uses_eager_on_ampere(tmp_path, monkeypatch, capabilities, eager, engine):
from types import SimpleNamespace
from core.inference import managed_engine
(tmp_path / "config.json").write_text('{"model_type":"qwen2"}')
monkeypatch.setattr(
managed_engine.subprocess,
"run",
lambda *args, **kwargs: SimpleNamespace(stdout = capabilities),
)
options = managed_engine.validate_model(
SimpleNamespace(is_local = True, path = str(tmp_path)), engine = engine, precision = "fp8"
)
assert options["disable_cuda_graph"] is eager
@pytest.mark.parametrize("engine, load_format", [("vllm", "bitsandbytes"), ("sglang", "auto")])
def test_online_int4_uses_the_native_loader(tmp_path, engine, load_format):
from types import SimpleNamespace
from core.inference.managed_engine import validate_model
(tmp_path / "config.json").write_text('{"model_type":"qwen3_5"}')
options = validate_model(
SimpleNamespace(is_local = True, path = str(tmp_path)), engine = engine, precision = "int4"
)
assert options["load_format"] == load_format
@pytest.mark.parametrize("engine", ["vllm", "sglang"])
@pytest.mark.parametrize("mode", ["tensor", "pipeline", "data"])
@pytest.mark.parametrize("devices", [1, 2, 4])
def test_parallel_mode_uses_exactly_the_selected_devices(engine, mode, devices):
from core.inference.engine_adapters import ADAPTERS
args = ADAPTERS[engine].command(
"python",
"model",
12345,
"key",
2048,
0.8,
devices,
options = {"parallelism": mode},
)
sizes = {}
for name in ("tensor", "pipeline", "data"):
flag = f"--{name}-parallel-size"
sizes[name] = int(args[args.index(flag) + 1]) if flag in args else 1
assert sizes[mode] == devices
assert all(size == 1 for name, size in sizes.items() if name != mode)
assert args[args.index("--host") + 1] == "127.0.0.1"
if engine == "vllm" and mode == "data":
assert "vllm.entrypoints.cli.main" in args and "serve" in args
assert "--distributed-executor-backend" not in args
if devices > 1:
assert args[args.index("--data-parallel-backend") + 1] == "mp"
@pytest.mark.parametrize("engine", ["vllm", "sglang"])
@pytest.mark.parametrize("mode", ["pipeline", "data"])
def test_non_tensor_modes_do_not_require_divisible_heads(tmp_path, engine, mode):
from types import SimpleNamespace
from core.inference.managed_engine import validate_model
(tmp_path / "config.json").write_text(
json.dumps(
{
"text_config": {
"num_attention_heads": 3,
"hidden_size": 15,
"num_hidden_layers": 12,
},
}
)
)
config = SimpleNamespace(is_local = True, path = str(tmp_path))
with pytest.raises(ValueError, match = "cannot be split"):
validate_model(config, gpu_ids = [0, 1], engine = engine)
options = validate_model(config, gpu_ids = [0, 1], engine = engine, parallelism = mode)
assert options["parallelism"] == mode
@pytest.mark.parametrize("engine", ["vllm", "sglang"])
@pytest.mark.parametrize("mode", ["pipeline", "data"])
def test_non_tensor_modes_validate_prequantized_bitsandbytes(tmp_path, engine, mode):
from types import SimpleNamespace
from core.inference.managed_engine import validate_model
(tmp_path / "config.json").write_text(
json.dumps({"quantization_config": {"quant_method": "bitsandbytes"}})
)
config = SimpleNamespace(is_local = True, path = str(tmp_path))
if engine == "sglang" and mode == "pipeline":
with pytest.raises(ValueError, match = "SGLang cannot load prequantized BitsAndBytes"):
validate_model(config, gpu_ids = [0, 1], engine = engine, parallelism = mode)
assert (
validate_model(config, gpu_ids = [0], engine = engine, parallelism = mode)["load_format"]
== "bitsandbytes"
)
return
options = validate_model(
config,
gpu_ids = [0, 1],
engine = engine,
parallelism = mode,
)
assert options["parallelism"] == mode and options["load_format"] == "bitsandbytes"
@pytest.mark.parametrize("mode", ["pipeline", "data"])
def test_resident_engine_parallel_mode_requires_reload(mode):
from types import SimpleNamespace
from routes.inference import _non_gguf_runtime_settings_match
from models.inference import LoadRequest
backend = SimpleNamespace(
active_model_name = "model",
models = {
"model": {
"engine": "vllm",
"engine_parallelism": mode,
"gpu_ids_requested": [0, 1],
}
},
)
request = LoadRequest(
model_path = "model", engine = "vllm", engine_parallelism = mode, gpu_ids = [0, 1]
)
assert _non_gguf_runtime_settings_match(backend, request)
assert not _non_gguf_runtime_settings_match(
backend, request.model_copy(update = {"engine_parallelism": "tensor"})
)
def test_native_api_key_cannot_be_parsed_as_a_cli_option(monkeypatch):
import core.inference.managed_engine as managed
monkeypatch.setattr(managed.secrets, "token_urlsafe", lambda _: "--random-token")
assert managed.ManagedEngine("vllm").key == "studio---random-token"
def test_vllm_int4_pipeline_uses_native_torchao_loader(tmp_path):
from types import SimpleNamespace
from core.inference.managed_engine import validate_model
from core.inference.engine_adapters import ADAPTERS
(tmp_path / "config.json").write_text('{"model_type":"qwen2"}')
options = validate_model(
SimpleNamespace(is_local = True, path = str(tmp_path)),
engine = "vllm",
gpu_ids = [0, 1],
precision = "int4",
parallelism = "pipeline",
)
args = ADAPTERS["vllm"].command("python", "model", 12345, "key", 2048, 0.8, 2, options = options)
assert args[args.index("--load-format") + 1] == "auto"
assert args[args.index("--quantization") + 1] == "torchao"
config = json.loads(
json.loads(args[args.index("--hf-overrides") + 1])["quantization_config_dict_json"]
)
assert config["_type"] == "Int4WeightOnlyConfig"
assert config["_data"]["int4_choose_qparams_algorithm"]["_data"] == "HQQ"
def test_switching_from_managed_engine_keeps_default_precision():
from types import SimpleNamespace
from models.inference import LoadRequest
from routes.inference import _inherit_resident_load_in_4bit
backend = SimpleNamespace(
active_model_name = "model",
models = {"model": {"engine": "vllm", "load_in_4bit_requested": False}},
)
request = LoadRequest(model_path = "model")
_inherit_resident_load_in_4bit(backend, request, "model")
assert request.load_in_4bit is True
@pytest.mark.parametrize("url", ["/" * 64 + "etc/hostname", "/etc/hostname", "/9j/4AAQSkZJRg=="])
def test_bare_image_strings_reach_the_engine_as_data_urls(peer, url):
engine, requests = peer
messages = [{"role": "user", "content": [{"type": "image_url", "image_url": {"url": url}}]}]
list(engine.generate(messages = messages))
sent = requests[0][2]["messages"][0]["content"][0]["image_url"]["url"]
assert sent == "data:image/png;base64," + url
def test_non_base64_image_string_is_refused_before_the_engine(peer):
engine, requests = peer
messages = [
{"role": "user", "content": [{"type": "image_url", "image_url": {"url": "/tmp/a.png"}}]}
]
with pytest.raises(ValueError, match = "data URLs"):
list(engine.generate(messages = messages))
assert requests == []
@_POSIX_ENGINE_LOCKS
def test_status_probe_does_not_fail_a_concurrent_engine_lease(isolated):
import fcntl
import time
active(isolated)
probe_held = threading.Event()
def probe():
with (isolated / "vllm.lock").open("a") as handle:
fcntl.flock(handle, fcntl.LOCK_EX | fcntl.LOCK_NB)
probe_held.set()
time.sleep(0.2)
thread = threading.Thread(target = probe)
thread.start()
probe_held.wait(2)
with install.engine_lease("vllm"):
assert install.status("vllm")["in_use"]
thread.join()
assert not install.status("vllm")["in_use"]
def test_support_probe_waits_for_a_slow_driver_and_is_cached(monkeypatch):
from types import SimpleNamespace
calls = []
def slow_smi(argv, **kwargs):
calls.append(kwargs["timeout"])
if kwargs["timeout"] < 20:
raise install.subprocess.TimeoutExpired(argv, kwargs["timeout"])
return SimpleNamespace(returncode = 0, stdout = "590.48.01, 10.0\n")
monkeypatch.setattr(install, "_gpu_rows", {})
monkeypatch.setattr(install.platform, "system", lambda: "Linux")
monkeypatch.setattr(install.platform, "machine", lambda: "x86_64")
monkeypatch.setattr(install.platform, "libc_ver", lambda: ("glibc", "2.39"))
monkeypatch.setattr(install.subprocess, "run", slow_smi)
assert install.support_reason("vllm") is None
assert install.support_reason("sglang") is None
assert len(calls) == 1
def test_memory_and_fp8_probes_wait_for_a_slow_driver(tmp_path, monkeypatch):
from types import SimpleNamespace
from core.inference import engine_adapters, managed_engine
def slow_smi(argv, **kwargs):
if kwargs["timeout"] < 20:
raise managed_engine.subprocess.TimeoutExpired(argv, kwargs["timeout"])
query = next(arg for arg in argv if arg.startswith("--query-gpu="))
return SimpleNamespace(stdout = "8.9\n" if "compute_cap" in query else "81920, 80000\n")
monkeypatch.setattr(managed_engine.subprocess, "run", slow_smi)
assert 0.05 < engine_adapters.gpu_memory_fraction([0]) <= 1
(tmp_path / "config.json").write_text('{"model_type":"qwen2"}')
options = managed_engine.validate_model(
SimpleNamespace(is_local = True, path = str(tmp_path)), engine = "vllm", precision = "fp8"
)
assert options["disable_cuda_graph"] is False
@_LOCAL_ENGINE_HOST
@pytest.mark.parametrize("chatty", [True, False])
def test_startup_deadline_counts_engine_silence(isolated, monkeypatch, chatty):
import os
import sys
from types import SimpleNamespace
from core.inference import managed_engine
active(isolated)
monkeypatch.setattr(managed_engine, "gpu_memory_fraction", lambda *_: 0.8)
monkeypatch.setattr(managed_engine, "STARTUP_STALL_S", 5.0)
engine = ManagedEngine("vllm")
def command(python, model, port, key, context, memory, tensor_parallel_size):
code = (
"import time, sys\n"
"from http.server import BaseHTTPRequestHandler, HTTPServer\n"
f"for i in range(40):\n print('Downloading shard', i, flush = True) if {chatty} else None; time.sleep(0.25)\n"
"class Handler(BaseHTTPRequestHandler):\n"
" def do_GET(self):\n"
" self.send_response(200); self.end_headers()\n"
" def log_message(self, *args): pass\n"
f"HTTPServer(('127.0.0.1', {port}), Handler).serve_forever()\n"
)
return [sys.executable, "-u", "-c", code]
engine.adapter = SimpleNamespace(
command = command,
progress = lambda _: None,
environment = lambda _: {},
key_environment = lambda _: {},
)
try:
if chatty:
engine.start("model", 2048, [0], dict(os.environ))
assert engine.phase == "ready"
else:
with pytest.raises(RuntimeError, match = "timed out"):
engine.start("model", 2048, [0], dict(os.environ))
finally:
engine.stop()
@pytest.mark.parametrize(
"files, disabled",
[
(["vllm/third_party/deep_gemm/_C.cpython-312-x86_64-linux-gnu.so"], True),
(["vllm/third_party/deep_gemm/_C.cpython-313-x86_64-linux-gnu.so"], False),
(
[
"vllm/third_party/deep_gemm/_C.cpython-312-x86_64-linux-gnu.so",
"deep_gemm/__init__.py",
],
False,
),
([], False),
],
)
def test_vendored_deep_gemm_for_another_python_is_disabled(tmp_path, files, disabled):
from core.inference.managed_engine import _deep_gemm_unloadable
site = tmp_path / "lib" / "python3.13" / "site-packages"
for name in files:
(site / name).parent.mkdir(parents = True, exist_ok = True)
(site / name).touch()
site.mkdir(parents = True, exist_ok = True)
assert _deep_gemm_unloadable(str(tmp_path)) is disabled
def test_failed_support_probe_is_not_rerun_on_every_poll(monkeypatch):
calls = []
def hung_smi(argv, **kwargs):
calls.append(argv)
raise install.subprocess.TimeoutExpired(argv, kwargs["timeout"])
monkeypatch.setattr(install, "_gpu_rows", {})
monkeypatch.setattr(install.platform, "system", lambda: "Linux")
monkeypatch.setattr(install.platform, "machine", lambda: "x86_64")
monkeypatch.setattr(install.platform, "libc_ver", lambda: ("glibc", "2.39"))
monkeypatch.setattr(install.subprocess, "run", hung_smi)
for engine in ("vllm", "sglang", "vllm"):
assert install.support_reason(engine) is not None
assert len(calls) == 1
def test_status_never_waits_for_a_slow_support_probe(monkeypatch):
import time
from types import SimpleNamespace
release = threading.Event()
calls = []
def slow_smi(argv, **kwargs):
calls.append(argv)
release.wait(10)
return SimpleNamespace(returncode = 0, stdout = "590.48.01, 10.0\n")
monkeypatch.setattr(install, "_gpu_rows", {})
monkeypatch.setattr(install.platform, "system", lambda: "Linux")
monkeypatch.setattr(install.platform, "machine", lambda: "x86_64")
monkeypatch.setattr(install.platform, "libc_ver", lambda: ("glibc", "2.39"))
monkeypatch.setattr(install.subprocess, "run", slow_smi)
started = time.monotonic()
for _ in range(5):
assert install.support_reason("vllm", wait = False).startswith("Checking")
assert time.monotonic() - started < 2
release.set()
assert install.support_reason("vllm") is None
assert len(calls) == 1
@pytest.mark.parametrize("trust_remote_code", [False, True])
def test_managed_load_runs_the_worker_security_gates(monkeypatch, trust_remote_code):
import threading as _threading
from types import SimpleNamespace
from core.inference import managed_engine, worker
from core.inference.orchestrator import InferenceOrchestrator
started, gates = [], []
class Engine:
context = 4096
def __init__(self, name):
pass
def start(self, *args, **kwargs):
started.append(args)
def alive(self):
return True
def blocked(targets, *, resp_queue, **kwargs):
gates.append((targets, kwargs))
resp_queue.put({"type": "loaded", "success": False, "message": "flagged as CRITICAL"})
return False
monkeypatch.setattr(managed_engine, "ManagedEngine", Engine)
monkeypatch.setattr(worker, "_run_security_gates", blocked)
orchestrator = InferenceOrchestrator.__new__(InferenceOrchestrator)
orchestrator._managed_engine = None
orchestrator._subprocess_shutdown_lock = _threading.RLock()
orchestrator._shutdown_subprocess = lambda *args, **kwargs: True
orchestrator.models, orchestrator.loading_models = {}, set()
orchestrator.active_model_name, orchestrator.load_generation = None, 0
config = SimpleNamespace(identifier = "org/model", is_local = False, is_vision = False)
with pytest.raises(RuntimeError, match = "CRITICAL"):
orchestrator.load_model(
config,
hf_token = "tok",
trust_remote_code = trust_remote_code,
approved_remote_code_fingerprint = "fp",
subject = "user",
engine = "vllm",
)
assert started == [] and orchestrator.active_model_name is None
assert gates[0][0] == ["org/model"]
assert gates[0][1]["trust_remote_code"] is trust_remote_code
assert gates[0][1]["approved_fingerprint"] == "fp" and gates[0][1]["subject"] == "user"
@pytest.mark.parametrize(
"engine, options, reserve",
[
("vllm", {"precision": "int8"}, 3072 + 6144),
("vllm", {"precision": "fp8"}, 3072 + 6144),
("vllm", {"precision": "int4", "parallelism": "pipeline"}, 3072 + 6144),
("vllm", {"precision": "int4"}, 3072),
("vllm", {"precision": "bf16"}, 3072),
("vllm", None, 3072),
("sglang", {"precision": "int8"}, 4096),
("sglang", None, 4096),
],
)
def test_only_vllm_torchao_loads_keep_extra_headroom(engine, options, reserve):
from core.inference.engine_adapters import memory_reserve_mib
assert memory_reserve_mib(engine, options) == reserve
@pytest.mark.parametrize(
"engine, highest_passing",
# Highest passing on a Colab L4 (23034 MiB, Qwen2.5-0.5B); the old 512 MiB reserve gave 0.957.
[("vllm", 0.912), ("sglang", 0.890)],
)
def test_a_24_gb_card_leaves_the_headroom_measured_on_an_l4(monkeypatch, engine, highest_passing):
from types import SimpleNamespace
from core.inference import engine_adapters
from utils import vram_budget_settings
monkeypatch.setattr(
engine_adapters.subprocess, "run", lambda *a, **kw: SimpleNamespace(stdout = "23034, 22564")
)
monkeypatch.setattr(vram_budget_settings, "get_vram_budget_fraction", lambda: 0.97)
reserve = engine_adapters.memory_reserve_mib(engine, None)
assert engine_adapters.gpu_memory_fraction([0], reserve) < highest_passing
def test_structured_output_and_continuation_pass_the_managed_engine_gate():
from types import SimpleNamespace
from routes import inference as route
def payload(**fields):
base = dict(
use_adapter = None,
response_format = None,
continue_final_message = False,
enable_thinking = None,
preserve_thinking = None,
reasoning_effort = None,
model_extra = {},
)
return SimpleNamespace(**{**base, **fields})
assert not route._managed_engine_unsupported_controls(payload())
assert not route._managed_engine_unsupported_controls(payload(response_format = {"type": "text"}))
assert not route._managed_engine_unsupported_controls(
payload(response_format = {"type": "json_object"})
)
assert not route._managed_engine_unsupported_controls(payload(continue_final_message = True))
assert route._managed_engine_unsupported_controls(payload(use_adapter = True))
assert route._managed_engine_unsupported_controls(payload(reasoning_effort = "high"))
def test_an_audio_model_that_also_reads_images_is_refused_as_audio():
from types import SimpleNamespace
from fastapi import HTTPException
from routes import inference as route
config = SimpleNamespace(is_gguf = False, is_lora = False, is_audio = True, is_vision = True)
with pytest.raises(HTTPException) as refused:
route._reject_unsupported_managed_kind(SimpleNamespace(engine = "vllm"), config)
assert "detected as an audio model" in refused.value.detail
@_LOCAL_ENGINE_HOST
@pytest.mark.parametrize("engine", ["vllm", "sglang"])
def test_wsl_installs_the_newest_release_and_prices_the_distro(isolated, monkeypatch, engine):
from core.inference import wsl_host
studio_with_engine_torch(monkeypatch, engine)
monkeypatch.setattr(install, "_studio_packages", lambda: {"torch": "2.11.0+cu130"})
local = install.profile(engine)["version"]
monkeypatch.setattr(wsl_host, "active", lambda: True)
newest = install.PROFILES[engine]["releases"][0]
assert install.profile(engine)["version"] == newest["version"] != local
sizes = install._compat_file(engine)["sizes"]
omit = set(install.profile(engine)["omit"])
complete = sum(size or 0 for name, size in sizes.items() if name not in omit)
extra = wsl_host.ROOTFS["size"] + wsl_host.UV["size"]
monkeypatch.setattr(wsl_host, "summary", lambda: {"state": None, "distro": None})
assert install.download_bytes(engine) == complete + extra
monkeypatch.setattr(wsl_host, "summary", lambda: {"state": "ready", "distro": "UnslothStudio"})
assert install.download_bytes(engine) == complete
@pytest.mark.parametrize("version", ["0.5.20", "0.5.18"])
def test_sglang_without_torchao_refuses_int8_and_int4(version):
from core.inference.engine_adapters import ADAPTERS
for precision in ("int8", "int4"):
with pytest.raises(ValueError, match = "Choose FP8 or Model default"):
ADAPTERS["sglang"].command(
"python",
"model",
1,
"key",
4096,
0.8,
options = {"precision": precision, "engine_version": version},
)
args = ADAPTERS["sglang"].command(
"python",
"model",
1,
"key",
4096,
0.8,
options = {"precision": "fp8", "disable_cuda_graph": True, "engine_version": version},
)
assert "--torchao-config" not in args
assert args[args.index("--quantization") + 1] == "fp8"
assert "--disable-piecewise-cuda-graph" not in args
assert args[args.index("--cuda-graph-backend-prefill") + 1] == "disabled"
legacy = ADAPTERS["sglang"].command(
"python",
"model",
1,
"key",
4096,
0.8,
options = {"precision": "int8", "engine_version": "0.5.17"},
)
assert legacy[legacy.index("--torchao-config") + 1] == "int8wo"
def test_legacy_load_in_4bit_is_validated_as_int4(monkeypatch):
# An older client's load_in_4bit becomes INT4, and the SGLang refusal must see it first.
import asyncio
from fastapi import HTTPException
from core.inference import managed_engine
from models.inference import LoadRequest
from routes import inference as routes
info = {"path": "/env", "version": "0.5.20", "profile_digest": install.profile_digest("sglang")}
monkeypatch.setattr(managed_engine, "installed", lambda _: info)
monkeypatch.setattr(managed_engine, "support_reason", lambda *a: None)
monkeypatch.setattr(managed_engine, "resolve_requested_gpu_ids", lambda ids: [0])
request = LoadRequest(model_path = "m", engine = "sglang", load_in_4bit = True)
with pytest.raises(HTTPException, match = "Choose FP8"):
asyncio.run(routes._managed_engine_request(request))
vllm = {"path": "/env", "profile_digest": install.profile_digest("vllm")}
monkeypatch.setattr(managed_engine, "installed", lambda _: vllm)
out = asyncio.run(
routes._managed_engine_request(
LoadRequest(model_path = "m", engine = "vllm", load_in_4bit = True)
)
)
assert out.engine_precision == "int4" and out.load_in_4bit is False and out.gpu_ids == [0]
def test_vllm_argv_caps_max_num_seqs_only_when_asked():
from core.inference.engine_adapters import ADAPTERS
def argv(engine, options = None):
return ADAPTERS[engine].command(
"/env/bin/python", "org/model", 45678, "key", 4096, 0.8, options = options
)
capped = argv("vllm", {"max_num_seqs": 36})
assert capped[capped.index("--max-num-seqs") + 1] == "36"
assert "--max-num-seqs" not in argv("vllm")
assert "--max-num-seqs" not in argv("vllm", {"precision": "auto"})
assert "--max-num-seqs" not in argv("sglang", {"max_num_seqs": 36})
# The line vLLM 0.30.0 prints for Qwen3.8-27B-NVFP4 at a 32 GB RTX 5090's budget (#8861), measured on SM 12.0.
_MAMBA_LIMIT_LINE = (
"(EngineCore pid=34071) ValueError: max_num_seqs (256) exceeds available Mamba cache blocks (36). "
"Each decode sequence requires one Mamba cache block, so CUDA graph capture cannot proceed. "
"Please lower max_num_seqs to at most 36 or increase gpu_memory_utilization."
)
def _scripted_engine(isolated, monkeypatch, outcomes):
"""A vLLM ManagedEngine whose launches follow ``outcomes`` in order: "mamba" prints the
Mamba limit and exits, "crash" exits without it, "serve" answers /health."""
import sys
from types import SimpleNamespace
from core.inference import managed_engine
if not (isolated / "vllm" / "active.json").exists():
active(isolated)
monkeypatch.setattr(managed_engine, "gpu_memory_fraction", lambda *_: 0.8)
monkeypatch.setattr(managed_engine, "_SEQ_LIMITS", {})
engine = ManagedEngine("vllm")
launches = []
def command(
python,
model,
port,
key,
context,
memory,
tensor_parallel_size,
options = None,
**_,
):
launches.append(dict(options or {}))
outcome = outcomes[len(launches) - 1]
if outcome == "serve":
code = (
"from http.server import BaseHTTPRequestHandler, HTTPServer\n"
"class Handler(BaseHTTPRequestHandler):\n"
" def do_GET(self):\n"
" self.send_response(200); self.end_headers()\n"
" def log_message(self, *args): pass\n"
f"HTTPServer(('127.0.0.1', {port}), Handler).serve_forever()\n"
)
else:
lines = {
"mamba": [_MAMBA_LIMIT_LINE],
# Two data-parallel ranks with different free memory.
"mamba_dp": [_MAMBA_LIMIT_LINE, _MAMBA_LIMIT_LINE.replace("(36)", "(40)")],
}.get(outcome, ["RuntimeError: CUDA out of memory"])
code = f"import sys\nprint({chr(10).join(lines)!r}, flush = True)\nsys.exit(1)\n"
return [sys.executable, "-u", "-c", code]
engine.adapter = SimpleNamespace(
command = command,
progress = lambda _: None,
environment = lambda _: {},
key_environment = lambda _: {},
)
return engine, launches
@_LOCAL_ENGINE_HOST
def test_vllm_relaunches_once_at_the_mamba_block_count(isolated, monkeypatch):
import os
engine, launches = _scripted_engine(isolated, monkeypatch, ["mamba", "serve"])
try:
engine.start("model", 4096, [0], dict(os.environ), options = {"precision": "auto"})
assert engine.phase == "ready"
finally:
engine.stop()
assert [launch.get("max_num_seqs") for launch in launches] == [None, 36]
assert launches[1]["precision"] == "auto"
# A reload of the same config starts at the learned count instead of failing first.
from core.inference import managed_engine
learned = dict(managed_engine._SEQ_LIMITS)
engine, launches = _scripted_engine(isolated, monkeypatch, ["serve"])
monkeypatch.setattr(managed_engine, "_SEQ_LIMITS", learned)
try:
engine.start("model", 4096, [0], dict(os.environ), options = {"precision": "auto"})
finally:
engine.stop()
assert [launch.get("max_num_seqs") for launch in launches] == [36]
@_LOCAL_ENGINE_HOST
def test_vllm_relaunches_at_the_tightest_data_parallel_rank(isolated, monkeypatch):
import os
engine, launches = _scripted_engine(isolated, monkeypatch, ["mamba_dp", "serve"])
try:
engine.start("model", 4096, [0, 1], dict(os.environ), options = {"parallelism": "data"})
finally:
engine.stop()
assert [launch.get("max_num_seqs") for launch in launches] == [None, 36]
@_LOCAL_ENGINE_HOST
@pytest.mark.parametrize("outcomes", [["crash"], ["mamba", "mamba"]])
def test_vllm_does_not_retry_other_failures_or_twice(isolated, monkeypatch, outcomes):
import os
engine, launches = _scripted_engine(isolated, monkeypatch, outcomes)
with pytest.raises(RuntimeError, match = "Engine failed to start"):
engine.start("model", 4096, [0], dict(os.environ))
engine.stop()
assert len(launches) == len(outcomes)
@_LOCAL_ENGINE_HOST
def test_vllm_mamba_retry_respects_a_cancelled_load(isolated, monkeypatch):
import os
import threading
engine, launches = _scripted_engine(isolated, monkeypatch, ["mamba", "serve"])
cancel = threading.Event()
original = engine._start_once
def first_then_cancel(*args):
try:
return original(*args)
finally:
cancel.set()
monkeypatch.setattr(engine, "_start_once", first_then_cancel)
with pytest.raises(RuntimeError, match = "cancelled"):
engine.start("model", 4096, [0], dict(os.environ), cancel)
engine.stop()
assert len(launches) == 1
@_LOCAL_ENGINE_HOST
def test_vllm_mamba_retry_respects_an_external_stop(isolated, monkeypatch):
import os
from core.inference import managed_engine
engine, launches = _scripted_engine(isolated, monkeypatch, ["mamba", "serve"])
original = engine._start_once
def first_then_stop(*args):
try:
return original(*args)
except managed_engine._MambaSeqLimit:
# An unload or GPU handoff stops the engine without the load's cancel event.
assert engine.stop() is True
raise
monkeypatch.setattr(engine, "_start_once", first_then_stop)
with pytest.raises(RuntimeError, match = "cancelled"):
engine.start("model", 4096, [0], dict(os.environ))
engine.stop()
assert len(launches) == 1
@_LOCAL_ENGINE_HOST
def test_vllm_retries_once_without_a_drafter_that_breaks_startup(isolated, monkeypatch):
import os
from core.inference import engine_adapters
monkeypatch.setattr(
engine_adapters, "vllm_speculative_args", lambda _: ["--speculative-config", "{}"]
)
engine, launches = _scripted_engine(isolated, monkeypatch, ["crash", "serve"])
try:
engine.start("model", 4096, [0], dict(os.environ))
assert engine.phase == "ready"
finally:
engine.stop()
assert [launch.get("no_speculative_config") for launch in launches] == [None, True]
@_LOCAL_ENGINE_HOST
def test_vllm_drafter_retry_respects_an_external_stop(isolated, monkeypatch):
import os
from core.inference import engine_adapters, managed_engine
monkeypatch.setattr(
engine_adapters, "vllm_speculative_args", lambda _: ["--speculative-config", "{}"]
)
engine, launches = _scripted_engine(isolated, monkeypatch, ["crash", "serve"])
original = engine._start_once
def first_then_stop(*args):
try:
return original(*args)
except managed_engine._EngineExited:
assert engine.stop() is True
raise
monkeypatch.setattr(engine, "_start_once", first_then_stop)
with pytest.raises(RuntimeError, match = "cancelled"):
engine.start("model", 4096, [0], dict(os.environ))
engine.stop()
assert len(launches) == 1
@pytest.fixture
def kernel_host(monkeypatch):
rows = [["580.82.07", "8.0"]]
monkeypatch.setattr(install.sys, "platform", "linux")
monkeypatch.setattr(install.platform, "machine", lambda: "x86_64")
monkeypatch.setattr(install, "_driver_rows", lambda gpu_id, wait = True: rows)
monkeypatch.delenv("UNSLOTH_ENGINE_FLASHINFER_KERNELS", raising = False)
return rows
def test_every_vllm_lock_has_its_flashinfer_kernel_wheel(kernel_host, monkeypatch):
table = json.loads(install._FLASHINFER_KERNELS.read_text(encoding = "utf-8"))
for engine, entry in install.PROFILES.items():
for release in entry["releases"]:
monkeypatch.setattr(install, "_release", lambda engine, release = release: release)
version = install._pins(engine)["flashinfer-python"][0]
wheel = install.flashinfer_kernel_wheel(engine)
if engine != "vllm":
assert wheel is None, release["lock"]
continue
assert wheel == table[f"{version}+cu130"]["x86_64"], release["lock"]
assert f"/flashinfer_jit_cache-{version}+cu130-" in wheel["url"]
assert wheel["url"].endswith("_x86_64.whl")
assert len(wheel["sha256"]) == 64 and wheel["size"] > 0
@pytest.mark.parametrize(
"capability, runs",
[
("8.0", True),
("8.6", True),
("8.9", True),
("9.0", True),
("10.0", True),
("10.3", True),
("12.0", True),
("12.1", True),
("7.5", False),
("9.1", False),
("10.1", False),
("11.0", False),
("13.0", False),
],
)
def test_flashinfer_kernels_only_for_gpus_they_were_built_for(capability, runs):
arches = ["8.0", "8.9", "9.0a", "10.0a", "10.3a", "12.0f"]
assert install._kernels_run_on(arches, capability) is runs
def test_flashinfer_kernels_skip_hosts_they_cannot_serve(kernel_host, monkeypatch):
assert install.flashinfer_kernel_wheel("vllm") is not None
kernel_host.append(["580.82.07", "7.5"])
assert install.flashinfer_kernel_wheel("vllm") is None
kernel_host[:] = [["580.82.07", "12.0"], ["580.82.07", "9.0"]]
assert install.flashinfer_kernel_wheel("vllm") is not None
for unreadable in (None, [], [["580.82.07"]], [["580", "n/a"]], install._PENDING):
monkeypatch.setattr(install, "_driver_rows", lambda gpu_id, wait = True, r = unreadable: r)
assert install.flashinfer_kernel_wheel("vllm") is None
def test_flashinfer_kernels_only_for_linux_cuda_and_honour_the_switch(kernel_host, monkeypatch):
for value in ("0", "false", "off"):
monkeypatch.setenv("UNSLOTH_ENGINE_FLASHINFER_KERNELS", value)
assert install.flashinfer_kernel_wheel("vllm") is None
monkeypatch.setenv("UNSLOTH_ENGINE_FLASHINFER_KERNELS", "1")
assert install.flashinfer_kernel_wheel("vllm") is not None
monkeypatch.setattr(install.platform, "machine", lambda: "aarch64")
assert install.flashinfer_kernel_wheel("vllm") is None
monkeypatch.setattr(install.platform, "machine", lambda: "x86_64")
monkeypatch.setattr(install.sys, "platform", "win32")
assert install.flashinfer_kernel_wheel("vllm") is None
monkeypatch.setattr(install.sys, "platform", "linux")
monkeypatch.setattr(install, "gpu_platform", lambda: "rocm")
assert install.flashinfer_kernel_wheel("vllm") is None
monkeypatch.setattr(install, "gpu_platform", lambda: "cuda")
monkeypatch.setattr(
install, "_FLASHINFER_KERNELS", Path("/nonexistent/flashinfer-kernels.json")
)
assert install.flashinfer_kernel_wheel("vllm") is None
@_LOCAL_ENGINE_HOST
@pytest.mark.parametrize("engine", ["vllm", "sglang"])
def test_install_adds_the_flashinfer_kernel_wheel_before_the_checks(
isolated, kernel_host, monkeypatch, engine
):
monkeypatch.setattr(install.shutil, "which", lambda _: "/uv")
calls = []
def run(engine, argv, cancel):
calls.append(argv)
if "-r" in argv:
calls.append(Path(argv[argv.index("-r") + 1]).read_text())
fake_venv(engine, argv, cancel)
monkeypatch.setattr(install, "_run", run)
install._install(engine, threading.Event())
assert install.status(engine)["job"]["state"] == "success"
wheel = install.flashinfer_kernel_wheel(engine)
pip = [i for i, argv in enumerate(calls) if isinstance(argv, list) and "-r" in argv]
if engine == "sglang":
assert wheel is None and pip == []
assert "flashinfer_kernels" not in install.installed(engine)
return
assert len(pip) == 1
argv = calls[pip[0]]
assert argv[:3] == ["/uv", "pip", "install"]
assert {"--no-deps", "--require-hashes"} <= set(argv)
assert calls[pip[0] + 1] == (
f"flashinfer-jit-cache @ {wheel['url']} --hash=sha256:{wheel['sha256']}\n"
)
sync = next(i for i, a in enumerate(calls) if isinstance(a, list) and "sync" in a)
check = next(i for i, a in enumerate(calls) if isinstance(a, list) and install._CHECK in a)
assert sync < pip[0] < check
assert any(
isinstance(a, list) and "import flashinfer_jit_cache, flashinfer" in a for a in calls
)
assert install.installed(engine)["flashinfer_kernels"] is True
@_LOCAL_ENGINE_HOST
@pytest.mark.parametrize("failing", ["download", "import"])
def test_failed_flashinfer_kernels_keep_the_jit_engine(isolated, kernel_host, monkeypatch, failing):
monkeypatch.setattr(install.shutil, "which", lambda _: "/uv")
calls = []
def run(engine, argv, cancel):
calls.append(argv)
if failing == "download" and "-r" in argv:
raise RuntimeError("Engine installation failed. github.com unreachable")
if failing == "import" and "import flashinfer_jit_cache, flashinfer" in argv:
raise RuntimeError("Engine installation failed. version mismatch")
fake_venv(engine, argv, cancel)
monkeypatch.setattr(install, "_run", run)
install._install("vllm", threading.Event())
assert install.status("vllm")["job"]["state"] == "success"
assert "flashinfer_kernels" not in install.installed("vllm")
assert any(a[:3] == ["/uv", "pip", "uninstall"] and "flashinfer-jit-cache" in a for a in calls)
assert any(install._CHECK in a for a in calls)
@_LOCAL_ENGINE_HOST
def test_a_kernel_wheel_that_cannot_be_removed_keeps_the_active_environment(
isolated, kernel_host, monkeypatch
):
marker = active(isolated)
before = marker.read_bytes()
monkeypatch.setattr(install.shutil, "which", lambda _: "/uv")
def run(engine, argv, cancel):
if "import flashinfer_jit_cache, flashinfer" in argv or argv[:3] == [
"/uv",
"pip",
"uninstall",
]:
raise RuntimeError("Engine installation failed.")
fake_venv(engine, argv, cancel)
monkeypatch.setattr(install, "_run", run)
install._install("vllm", threading.Event())
assert install.status("vllm")["job"]["state"] == "error"
assert marker.read_bytes() == before
assert [p.name for p in marker.parent.iterdir() if p.is_dir()] == ["env-prior"]
def test_no_gpu_probe_without_a_kernel_wheel(kernel_host, monkeypatch):
probes = []
monkeypatch.setattr(install, "_driver_rows", lambda gpu_id, wait = True: probes.append(gpu_id))
monkeypatch.setattr(install.platform, "machine", lambda: "aarch64")
assert install.flashinfer_kernel_wheel("vllm") is None
assert install.flashinfer_kernel_wheel("sglang") is None
assert probes == []
@_LOCAL_ENGINE_HOST
def test_cancel_during_flashinfer_kernels_cancels_the_install(isolated, kernel_host, monkeypatch):
monkeypatch.setattr(install.shutil, "which", lambda _: "/uv")
cancel = threading.Event()
def run(engine, argv, event):
if "-r" in argv:
event.set()
raise RuntimeError("Installation cancelled.")
fake_venv(engine, argv, event)
monkeypatch.setattr(install, "_run", run)
install._install("vllm", cancel)
assert install.status("vllm")["job"]["state"] == "cancelled"
assert install.installed("vllm") is None
@_LOCAL_ENGINE_HOST
def test_switched_off_flashinfer_kernels_install_as_before(isolated, kernel_host, monkeypatch):
monkeypatch.setenv("UNSLOTH_ENGINE_FLASHINFER_KERNELS", "0")
monkeypatch.setattr(install.shutil, "which", lambda _: "/uv")
calls = []
def run(engine, argv, cancel):
calls.append(argv)
fake_venv(engine, argv, cancel)
monkeypatch.setattr(install, "_run", run)
install._install("vllm", threading.Event())
assert install.status("vllm")["job"]["state"] == "success"
assert not any("-r" in argv or "flashinfer-jit-cache" in argv for argv in calls)
assert "flashinfer_kernels" not in install.installed("vllm")
@_LOCAL_ENGINE_HOST
def test_download_size_counts_the_flashinfer_kernel_wheel(isolated, kernel_host, monkeypatch):
with_kernels = install.download_bytes("vllm")
monkeypatch.setenv("UNSLOTH_ENGINE_FLASHINFER_KERNELS", "0")
version = install._pins("vllm")["flashinfer-python"][0]
table = json.loads(install._FLASHINFER_KERNELS.read_text(encoding = "utf-8"))
assert (
with_kernels - install.download_bytes("vllm") == table[f"{version}+cu130"]["x86_64"]["size"]
)