Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
"""Opt-in engine envs share Studio torch/CUDA when locked, else isolated; none imported into Studio;
installs serialized; runtime leases block removing an env in use"""
from __future__ import annotations
import hashlib
import json
import os
import platform
import re
import shutil
import subprocess
from utils.subprocess_compat import windows_hidden_subprocess_kwargs
import sys
import sysconfig
import tempfile
import threading
import time
import uuid
from collections import deque
from contextlib import contextmanager, nullcontext
from pathlib import Path
PRECISIONS = ("auto", "bf16", "fp16", "int4", "int8", "fp8")
ENGINE_NAMES = {"vllm": "vLLM", "sglang": "SGLang"}
PROFILES = {
"vllm": {
"module": "vllm",
"cuda": "cu130",
"driver": 580,
# FlashInfer fetches trtllm kernels on demand; CUTLASS DSL loads the newest flavour driver 580 runs.
"omit": ("flashinfer-cubin", "nvidia-cutlass-dsl-libs-cu12"),
# vLLM samples with FlashInfer (nvcc JIT at first launch); Studio's SGLang flags never do.
"flashinfer_kernels": True,
# Newest first; each release pins one torch build (vLLM 0.27+ needs torch 2.13, 0.20-0.26 torch 2.11).
"releases": (
{"version": "0.30.0", "torch": "2.13.0", "lock": "vllm-linux-cu130-torch213"},
{"version": "0.26.0", "torch": "2.11.0", "lock": "vllm-linux-cu130"},
),
# AMD: vLLM's ROCm build, Python 3.12 only; its torch loads /opt/rocm, so always isolated.
"rocm": {
"cuda": None,
"driver": None,
"index": "https://wheels.vllm.ai/rocm/0.30.0/rocm723",
"python": (3, 12),
"glibc": (2, 39),
"rocm": (7, 2),
"gfx": (
"gfx90a",
"gfx942",
"gfx950",
"gfx1100",
"gfx1101",
"gfx1150",
"gfx1151",
"gfx1200",
"gfx1201",
),
"omit": (),
"platform": "rocm",
# The ROCm build has no bitsandbytes quantization, and TorchAO's packed INT4 is CUDA-only.
"bitsandbytes": False,
"precisions": tuple(p for p in PRECISIONS if p != "int4"),
"releases": ({"version": "0.30.0", "torch": "2.12.0", "lock": "vllm-linux-rocm723"},),
},
},
"sglang": {
"module": "sglang",
"cuda": "cu130",
"driver": 580,
# Excluded: outlines-core 0.1.26 has no py3.13 wheel; only --grammar-backend outlines needs it.
"omit": ("outlines", "outlines-core", "flashinfer-cubin", "nvidia-cutlass-dsl-libs-cu12"),
"releases": (
{
"version": "0.5.20",
"torch": "2.13.0",
"lock": "sglang-linux-cu130-torch213",
# cuda-tile 1.6.0rc5 is sdist-only and SGLang never imports it.
"omit": (
"outlines",
"outlines-core",
"flashinfer-cubin",
"nvidia-cutlass-dsl-libs-cu12",
"cuda-tile",
),
},
{"version": "0.5.17", "torch": "2.11.0", "lock": "sglang-linux-cu130"},
),
},
}
PYTHON = (3, 13)
_BASE_PTH = "zz_unsloth_studio_base.pth"
_BASE_MODULE = "_unsloth_studio_base"
# A jit-cache of another FlashInfer version fails the engine's flashinfer import.
_STUDIO_BASE_SOURCE = """import pkgutil, site, sys
_HIDDEN = ("flashinfer", "flashinfer_jit_cache", "flashinfer_cubin")
class _Hidden:
def __init__(self, finder):
self._finder = finder
def find_spec(self, name, target = None):
if name.partition(".")[0] in _HIDDEN:
return None
spec = self._finder.find_spec(name, target)
# A regular nvidia package here (Colab ships nvidia/__init__.py) would hide the engine's
# own nvidia/ tree; as a namespace portion both are found, the engine's first.
if name == "nvidia" and spec is not None and spec.submodule_search_locations:
spec.loader = None
return spec
def invalidate_caches(self):
self._finder.invalidate_caches()
def iter_modules(self, prefix = ""):
for info in pkgutil.iter_importer_modules(self._finder, prefix):
if info[0][len(prefix):] not in _HIDDEN:
yield info
for path in {paths!r}:
site.addsitedir(path)
for hook in sys.path_hooks:
try:
finder = hook(path)
except ImportError:
continue
sys.path_importer_cache[path] = _Hidden(finder)
break
"""
_REQUIREMENTS = Path(__file__).resolve().parents[2] / "requirements" / "engines"
_jobs: dict[str, dict] = {}
_cancels: dict[str, threading.Event] = {}
_lock = threading.RLock()
def engine_root() -> Path:
from utils.paths.storage_roots import studio_root
return studio_root() / "engines"
_kfd_has_amd_gpu: bool | None = None
def gpu_platform() -> str:
"""The GPU family the engine runs on: "rocm" for AMD, else "cuda". A host with both, and
every Windows host, follows Studio's own PyTorch."""
global _kfd_has_amd_gpu
from utils.hardware import hardware
# Status is read by idle Chat. The wheel's build family is enough to choose
# an engine profile; discovering it must not initialize the ML runtime.
if hardware.DETECTION_COMPLETE.is_set():
is_rocm = hardware.IS_ROCM
else:
from core._torchao_stub import _installed_torch_is_rocm, _module_is_rocm
torch = sys.modules.get("torch")
is_rocm = (
_module_is_rocm(torch)
if getattr(torch, "__version__", None) is not None
else _installed_torch_is_rocm()
)
if is_rocm:
return "rocm"
if platform.system() != "Linux" or os.path.exists("/proc/driver/nvidia/version"):
return "cuda"
# Asked many times per status poll; the GPUs do not change while Studio runs.
if _kfd_has_amd_gpu is None:
from utils.hardware.amd import amd_kfd_gpu_node_count
count = amd_kfd_gpu_node_count()
if count is None:
return "cuda"
_kfd_has_amd_gpu = count > 0
return "rocm" if _kfd_has_amd_gpu else "cuda"
def _flavor(engine: str) -> dict:
"""The engine's profile for this host's GPU platform, before a release is picked."""
base = {
"platform": "cuda",
"python": PYTHON,
"precisions": PRECISIONS,
"bitsandbytes": True,
**{key: value for key, value in PROFILES[engine].items() if key != "rocm"},
}
if gpu_platform() == "rocm" and "rocm" in PROFILES[engine]:
return {**base, **PROFILES[engine]["rocm"]}
return base
def _release(engine: str) -> dict:
"""The release built on Studio's own torch, so the engine can share it; otherwise the newest,
which then gets an isolated environment of its own."""
from . import wsl_host
flavor = _flavor(engine)
releases = flavor["releases"]
if wsl_host.active() or flavor["platform"] == "rocm":
# The WSL guest has no Studio torch to share, so it always gets the newest.
return releases[0]
torch = _studio_packages().get("torch")
for release in releases:
if _same_build(torch, release["torch"], flavor["cuda"]):
return release
return releases[0]
def profile(engine: str) -> dict:
if engine not in PROFILES:
raise ValueError("Unknown inference engine")
return {**_flavor(engine), **_release(engine)}
def _python(engine: str) -> tuple[int, int]:
return profile(engine)["python"]
def _venv_python_args(engine: str, guest: bool = False) -> list[str]:
"""uv venv's interpreter. Triton compiles its HIP driver module against the interpreter's
Python.h at run time, and Ubuntu's python3.12 has none without python3.12-dev, so the ROCm
environment always gets a uv-managed CPython, which ships its headers. Studio's own
interpreter serves only a local environment, never the WSL guest's."""
version = "{}.{}".format(*_python(engine))
if profile(engine)["platform"] == "rocm":
return ["--python", version, "--python-preference", "only-managed"]
if not guest and sys.version_info[:2] == _python(engine):
return ["--python", sys.executable]
return ["--python", version]
def requirements(engine: str) -> Path:
return _REQUIREMENTS / f"{profile(engine)['lock']}.txt"
def profile_digest(engine: str) -> str:
return hashlib.sha256(requirements(engine).read_bytes()).hexdigest()
def _normalize(name: str) -> str:
return re.sub(r"[-_.]+", "-", name).lower()
def _pins(engine: str) -> dict[str, tuple[str, str]]:
"""{name: (version, requirement line with its hashes)} from a uv-compiled lock."""
entries: dict[str, tuple[str, str]] = {}
name = None
for line in requirements(engine).read_text(encoding = "utf-8").splitlines(keepends = True):
match = re.match(r"([A-Za-z0-9][A-Za-z0-9_.\-]*)==([^\s;\\]+)", line)
if match:
name = _normalize(match.group(1))
entries[name] = (match.group(2), line)
elif name and line.startswith(" --hash="):
entries[name] = (entries[name][0], entries[name][1] + line)
else:
name = None
return entries
def _studio_site() -> list[str]:
paths = sysconfig.get_paths()
return sorted({paths["purelib"], paths["platlib"]})
_studio_seen: tuple[tuple, dict[str, str]] | None = None
def _studio_packages() -> dict[str, str]:
"""Distributions in Studio's own site-packages, excluding sidecars on sys.path. Status polls
read this several times; the scan is redone only when an install changes a site directory."""
global _studio_seen
import importlib.metadata as metadata
sites = _studio_site()
key = tuple((site, os.stat(site).st_mtime_ns) for site in sites if os.path.isdir(site))
if _studio_seen is None or _studio_seen[0] != key:
_studio_seen = (
key,
{
_normalize(dist.metadata["Name"]): dist.version
for dist in metadata.distributions(path = sites)
},
)
return dict(_studio_seen[1])
def _torch_runtime() -> set[str]:
"""Studio's torch and the native CUDA packages it loads, which one process can hold once."""
import importlib.metadata as metadata
from packaging.requirements import Requirement
names, pending = {"torch"}, [("torch", ())]
while pending:
name, extras = pending.pop()
dists = list(metadata.distributions(name = name, path = _studio_site()))
for raw in dists[0].requires or [] if dists else []:
requirement = Requirement(raw)
child = _normalize(requirement.name)
if (
child not in names
and child.startswith(("nvidia-", "cuda-toolkit", "triton"))
and (
requirement.marker is None
or any(requirement.marker.evaluate({"extra": e}) for e in ("", *extras))
)
):
names.add(child)
pending.append((child, tuple(requirement.extras)))
return names
# nvcc's cudafe stubs need crt headers of the same release, so share only when all match.
_TOOLCHAIN = frozenset({"nvidia-cuda-nvcc", "nvidia-cuda-crt", "nvidia-nvvm", "nvidia-cuda-cccl"})
def _same_build(installed: str | None, locked: str | None, cuda: str) -> bool:
"""PyTorch's index labels its wheels (2.11.0+cu130); the PyPI lock names the same build unlabelled."""
return installed is not None and installed in (locked, f"{locked}+{cuda}")
def _compat_file(engine: str) -> dict:
"""engine_compat.py's output for the lock, or {} for a missing or stale file."""
path = requirements(engine).with_suffix(".compat.json")
try:
data = json.loads(path.read_text(encoding = "utf-8"))
except (OSError, ValueError):
return {}
return data if data.get("lock_sha256") == profile_digest(engine) else {}
def _compat(engine: str) -> dict[str, list[str]]:
"""What the locked packages require of each other."""
return _compat_file(engine).get("requires", {})
def download_bytes(engine: str) -> int | None:
"""Wheel bytes an install of the selected release downloads: the lock minus what Studio provides."""
from . import wsl_host
sizes = _compat_file(engine).get("sizes")
if not sizes:
return None
wsl = wsl_host.active()
# A WSL engine is always a complete environment, plus the distro and uv on first setup.
provided = {} if wsl else install_plan(engine)["provided"]
omit = set(profile(engine).get("omit", ()))
total = sum(
size or 0 for name, size in sizes.items() if name not in provided and name not in omit
)
if not wsl and (kernels := flashinfer_kernel_wheel(engine, wait = False)):
total += kernels.get("size") or 0
summary = wsl_host.summary() if wsl else {}
if wsl and not summary.get("distro"):
total += wsl_host.ROOTFS["size"] + wsl_host.UV["size"]
if (
wsl
and profile(engine)["platform"] == "rocm"
and summary.get("rocm") != wsl_host.ROCM_RELEASE
):
total += wsl_host.ROCM_APT_BYTES + sum(
spec["size"] for spec in (wsl_host.ROCDXG, wsl_host.ROCDXG_SMI)
)
return total
# Studio's adapter configs use TorchAO 0.17's API; 0.18 removed their version 1.
_ADAPTER_REQUIRES = {"torchao": ["==0.17.0"]}
def _adapter_accepts(name: str, version: str) -> bool:
from packaging.specifiers import SpecifierSet
return all(
SpecifierSet(spec).contains(version, prereleases = True)
for spec in _ADAPTER_REQUIRES.get(name, ())
)
def _reusable(
engine: str, lock: dict, studio: dict[str, str], provided: dict[str, str]
) -> dict[str, str]:
"""Studio's own versions every engine dependency accepts, kept only while their own
requirements still hold against what the engine will see."""
from packaging.requirements import Requirement
from packaging.specifiers import SpecifierSet
import importlib.metadata as metadata
compat = _compat(engine)
if not compat:
return {}
hidden = {"flashinfer-python", "flashinfer-jit-cache", "flashinfer-cubin", *_TOOLCHAIN}
reuse = {
name: studio[name]
for name in lock
if name in studio
and name not in provided
and name not in hidden
and _adapter_accepts(name, studio[name])
and all(
SpecifierSet(spec).contains(studio[name], prereleases = True)
for spec in compat.get(name, ())
)
}
requires = {}
for name in reuse:
dists = list(metadata.distributions(name = name, path = _studio_site()))
requires[name] = [Requirement(r) for r in (dists[0].requires or [])] if dists else None
while True:
seen = {name: version for name, (version, _) in lock.items()} | provided | reuse
drop = {
name
for name in reuse
if requires[name] is None
or any(
_normalize(r.name) in seen
and not r.specifier.contains(seen[_normalize(r.name)], prereleases = True)
for r in requires[name]
if r.marker is None or r.marker.evaluate({"extra": ""})
)
}
if not drop:
return reuse
for name in drop:
del reuse[name]
def install_plan(engine: str) -> dict:
"""Split the lock into packages Studio already provides and the ones to install."""
from packaging.specifiers import SpecifierSet
lock = _pins(engine)
studio = _studio_packages()
cuda = profile(engine)["cuda"]
compat = _compat(engine)
runtime = _torch_runtime()
def fits(name: str) -> bool:
if _same_build(studio.get(name), lock.get(name, (None,))[0], cuda):
return True
# Torch is exact; range-pinned CUDA libs only need to satisfy every locked requirement.
specs = compat.get(name)
return bool(
name != "torch"
and name not in _TOOLCHAIN
and name in lock
and name in studio
and specs
and all(SpecifierSet(spec).contains(studio[name], prereleases = True) for spec in specs)
)
# The ROCm build's torch loads /opt/rocm rather than bundled libraries: never Studio's.
shared = (
profile(engine)["platform"] == "cuda"
and sys.implementation.name == "cpython"
and sys.version_info[:2] == _python(engine)
and "torch" in studio
and all(fits(name) for name in runtime - _TOOLCHAIN)
)
provided = {
name: studio[name]
for name, (version, _) in lock.items()
if shared
and (_same_build(studio.get(name), version, cuda) or (name in runtime and fits(name)))
}
if not all(name in provided for name in _TOOLCHAIN & lock.keys()):
provided = {name: version for name, version in provided.items() if name not in _TOOLCHAIN}
if shared:
provided |= _reusable(engine, lock, studio, provided)
return {
"shared": shared,
"provided": provided,
"requirements": "".join(line for name, (_, line) in lock.items() if name not in provided),
}
def _cuda_trees(env: Path, shared: bool) -> list[Path]:
sites = list(env.glob("lib/python3.*/site-packages"))
sites += [Path(path) for path in _studio_site()] if shared else []
return [site / "nvidia" / "cu13" for site in sites if (site / "nvidia" / "cu13").is_dir()]
def link_cuda_home(env: Path, shared: bool) -> None:
"""FlashInfer JIT-compiles some kernels; this CUDA_HOME is the locked pip nvcc, since a host may
have only the driver, or a toolkit of another CUDA major."""
home = env / "cuda"
(home / "lib64").mkdir(parents = True, exist_ok = True)
trees = _cuda_trees(env, shared)
# Engine entries first: Studio headers quote-include crt/ which must resolve to the engine's.
include = home / "include"
if include.is_symlink():
include.unlink()
include.mkdir(exist_ok = True)
for tree in trees:
if (tree / "include").is_dir():
for entry in (tree / "include").iterdir():
if not (include / entry.name).is_symlink():
(include / entry.name).symlink_to(entry)
for name, link in (
("bin", home / "bin"),
("nvvm", home / "nvvm"),
):
source = next(
(
tree / name
for tree in trees
if (tree / name / ("nvcc" if name == "bin" else "")).exists()
),
None,
)
if source is not None and not link.is_symlink():
link.symlink_to(source)
# nvcc links -lcudart, and the runtime wheel ships only the versioned soname.
cudart = next(
(
tree / "lib" / "libcudart.so.13"
for tree in trees
if (tree / "lib" / "libcudart.so.13").exists()
),
None,
)
if cudart is not None and not (home / "lib64" / "libcudart.so").is_symlink():
(home / "lib64" / "libcudart.so").symlink_to(cudart)
def cuda_environment(info: dict) -> dict[str, str]:
"""CUDA_HOME for the engine's JIT compiles; CPATH is its merged include tree for other compilers."""
if info.get("host") == "wsl":
return dict(info.get("cuda_environment") or {})
env = Path(info["path"])
if not (env / "cuda" / "bin" / "nvcc").exists():
return {}
return {"CUDA_HOME": str(env / "cuda"), "CPATH": str(env / "cuda" / "include")}
def stale(info: dict) -> bool:
"""True when Studio no longer provides what a shared engine environment was checked with."""
if not info.get("shared"):
return False
if info.get("python") != platform.python_version():
return True
studio = _studio_packages()
return any(
studio.get(name) != version or not _adapter_accepts(name, version)
for name, version in info.get("provided", {}).items()
)
def built_for_this_gpu(engine: str, info: dict) -> bool:
"""Whether an environment runs on this host's GPU platform; one from before AMD support is CUDA.
A restored environment skips the profile check, so this keeps it off the other vendor's GPU."""
return info.get("platform", "cuda") == profile(engine)["platform"]
# Runs in the engine's interpreter, so it checks both layers as the engine imports them.
_CHECK = r"""
import importlib.metadata as metadata, json, re, sys
from packaging.requirements import Requirement
def normalize(name):
return re.sub(r"[-_.]+", "-", name).lower()
omitted = set(sys.argv[2].split(",")) - {""}
cuda = sys.argv[3]
provided = json.loads(sys.argv[4]) if len(sys.argv) > 4 else {}
problems = []
for line in open(sys.argv[1], encoding = "utf-8"):
match = re.match(r"([A-Za-z0-9][A-Za-z0-9_.\-]*)==([^\s;\\]+)", line)
if not match:
continue
name, version = match.groups()
try:
found = metadata.version(name)
except metadata.PackageNotFoundError:
found = None
if found not in (version, f"{version}+{cuda}", provided.get(normalize(name))):
problems.append(f"{name}=={version} is required, found {found}")
continue
for raw in metadata.requires(name) or []:
requirement = Requirement(raw)
if normalize(requirement.name) in omitted or (
requirement.marker and not requirement.marker.evaluate({"extra": ""})
):
continue
try:
have = metadata.version(requirement.name)
except metadata.PackageNotFoundError:
have = None
if have is None or not requirement.specifier.contains(have, prereleases = True):
problems.append(f"{name} requires {requirement}, found {have}")
if problems:
sys.exit("\n".join(problems))
"""
_gpu_rows: dict = {}
_gpu_rows_lock = threading.Lock()
_gpu_probe_lock = threading.Lock()
_PENDING = object()
def _cached_rows(gpu_id):
with _gpu_rows_lock:
cached = _gpu_rows.get(gpu_id)
if cached is not None and (cached[1] is not None or time.monotonic() < cached[0]):
return cached[1]
return _PENDING
def _probe_rows(gpu_id):
rows = None
try:
from utils.hardware.nvidia import _nvidia_smi_executable
result = subprocess.run(
[
_nvidia_smi_executable(),
"--query-gpu=driver_version,compute_cap",
"--format=csv,noheader",
*(["--id", str(gpu_id)] if gpu_id is not None else []),
],
capture_output = True,
**windows_hidden_subprocess_kwargs(),
text = True,
encoding = "utf-8",
errors = "replace",
timeout = 60,
)
if result.returncode == 0:
rows = [line.split(",") for line in result.stdout.strip().splitlines()]
except (OSError, subprocess.TimeoutExpired):
pass
with _gpu_rows_lock:
_gpu_rows[gpu_id] = (time.monotonic() + 300, rows)
return rows
def _driver_rows(gpu_id: int | None, *, wait: bool = True):
"""nvidia-smi rows, cached per process; a failure is retried after 5 minutes. A loaded
multi-GPU host can take over 20s to answer, so ``wait=False`` (status polls) never runs the
probe in the caller: it starts one in the background and returns ``_PENDING``."""
rows = _cached_rows(gpu_id)
if rows is not _PENDING:
return rows
if not wait:
if _gpu_probe_lock.acquire(blocking = False):
def probe():
try:
_probe_rows(gpu_id)
finally:
_gpu_probe_lock.release()
threading.Thread(target = probe, daemon = True).start()
return _PENDING
with _gpu_probe_lock:
rows = _cached_rows(gpu_id)
return _probe_rows(gpu_id) if rows is _PENDING else rows
def support_reason(
engine: str = "vllm",
gpu_id: int | None = None,
*,
wait: bool = True,
) -> str | None:
from . import wsl_host
if wsl_host.active():
# WSL itself is not a prerequisite: installing the engine sets it up.
reason = wsl_host.support_reason()
if reason:
return reason
elif platform.system() != "Linux" or platform.machine() != "x86_64":
return "Managed engines currently require Linux x86_64 or Windows x64."
else:
glibc = profile(engine).get("glibc", (2, 34))
if tuple(int(x) for x in (platform.libc_ver()[1] or "0.0").split(".")[:2]) < glibc:
return f"{ENGINE_NAMES[engine]} requires glibc {glibc[0]}.{glibc[1]} or newer."
if gpu_platform() == "rocm":
return _rocm_reason(engine, gpu_id, wait)
rows = _driver_rows(gpu_id, wait = wait)
if rows is _PENDING:
return "Checking for a supported NVIDIA GPU."
try:
if any(
int(row[0].strip().split(".")[0]) >= profile(engine)["driver"] and float(row[1]) >= 8.0
for row in rows or ()
if len(row) == 2
):
return None
except ValueError:
pass
return f"Requires an NVIDIA GPU with compute capability 8.0 or newer and driver {profile(engine)['driver']} or newer."
ROCM_HOME = Path("/opt/rocm")
# What vLLM's ROCm torch and kernels load from the host (their RUNPATH is /opt/rocm/lib).
ROCM_LIBRARIES = (
"libamdhip64.so.7",
"libhiprtc.so.7",
"libMIOpen.so.1",
"librocblas.so.5",
"libhipblas.so.3",
"libhipblaslt.so.1",
"libhipfft.so.0",
"libhiprand.so.1",
"libhipsparse.so.4",
"libhipsparselt.so.0",
"libhipsolver.so.1",
"librocsolver.so.0",
"librccl.so.1",
"librocprofiler-sdk.so.1",
# rocprofiler-sdk loads it but does not depend on its package (hsa-amd-aqlprofile).
"libhsa-amd-aqlprofile64.so.1",
"libroctx64.so.4",
)
# Linked by that torch from outside ROCm: {soname: Ubuntu 24.04 package}.
ROCM_SYSTEM_LIBRARIES = {
"libmpi.so.40": "libopenmpi3t64",
"libmpi_cxx.so.40": "libopenmpi3t64",
"libnuma.so.1": "libnuma1",
}
def rocm_version() -> tuple[int, int] | None:
"""The ROCm release installed in /opt/rocm, where the ROCm engine's torch looks for it."""
try:
text = (ROCM_HOME / ".info" / "version").read_text(encoding = "utf-8")
except (OSError, UnicodeDecodeError):
return None
match = re.match(r"\s*(\d+)\.(\d+)", text)
return (int(match.group(1)), int(match.group(2))) if match else None
def rocm_loads_quantization(quant) -> bool:
"""Whether vLLM's ROCm build loads a checkpoint with this ``quantization_config``. Measured on
gfx1151 with vLLM 0.30.0: AWQ and integer compressed-tensors (W8A8, W4A16) load; GPTQ, FP8 and
NVFP4 checkpoints fail at engine start."""
if not isinstance(quant, dict) or not quant:
return True
method = str(quant.get("quant_method") or "").strip().lower()
if method in ("gptq", "fp8", "modelopt", "modelopt_fp4"):
return False
if method != "compressed-tensors":
return True
groups = quant.get("config_groups")
if not isinstance(groups, dict) or not groups:
return False
for group in groups.values():
if not isinstance(group, dict) or not isinstance(group.get("weights"), dict):
return False
for key in ("weights", "input_activations"):
spec = group.get(key)
if isinstance(spec, dict) and spec.get("type") != "int":
return False
return True
_rocm_arches: dict[int, str] | None = None
def _rocm_gpu_arches() -> dict[int, str]:
"""{physical GPU id: the gfx target it presents} from Studio's own ROCm torch, read once, or {}
when the ordinals cannot be mapped to physical ids. The engine inherits Studio's environment,
so an HSA_OVERRIDE_GFX_VERSION spoof applies to both."""
global _rocm_arches
if _rocm_arches is None:
_rocm_arches = {}
try:
import torch
from utils.hardware.hardware import (
_props_gfx_arch,
_rocm_device_ordinal_active,
_rocm_visibility_masks_are_stacked,
_torch_ordinal_physical_ids,
)
# Stacked or ordinal masks renumber the devices; the KFD topology then answers instead.
# AMD's SDK / Radeon wheels leave torch.version.hip unset; their label carries ROCm.
if (
(getattr(torch.version, "hip", None) or "rocm" in torch.__version__.lower())
and torch.cuda.is_available()
and not _rocm_device_ordinal_active()
and not _rocm_visibility_masks_are_stacked()
):
count = torch.cuda.device_count()
physical = _torch_ordinal_physical_ids(count) or list(range(count))
for ordinal, gpu_id in enumerate(physical[:count]):
arch = _props_gfx_arch(torch.cuda.get_device_properties(ordinal))
if arch:
_rocm_arches[gpu_id] = arch
except Exception:
_rocm_arches = {}
return _rocm_arches
def _known_rocm_gpu_arches(*, wait: bool) -> dict[int, str]:
"""Status uses existing runtime information or sysfs; a load may initialize torch.
Do not cache an empty provisional answer: first use must still run the real
probe, including GPU masks and HSA_OVERRIDE_GFX_VERSION.
"""
if wait or "torch" in sys.modules:
return _rocm_gpu_arches()
return _rocm_arches or {}
def _unsupported_amd_gpu(found: list[str]) -> str:
return (
"Requires an AMD Instinct MI200, MI300 or MI350, Radeon RX 7700 to 7900, Radeon RX 9000, or Ryzen AI Max or AI 300 GPU"
+ (f" (found {', '.join(found)})." if found else ".")
)
def _rocm_reason(
engine: str,
gpu_id: int | None,
wait: bool = True,
) -> str | None:
from . import wsl_host
if "rocm" not in PROFILES[engine]:
return f"{ENGINE_NAMES[engine]} requires an NVIDIA GPU. Use vLLM on AMD GPUs."
wanted = profile(engine)
if wsl_host.active():
# ROCm is checked inside WSL; a target torch or the driver already names (the selected GPU's) refuses first.
arches = _known_rocm_gpu_arches(wait = wait) if gpu_id is not None else {}
if gpu_id in arches:
known = [arches[gpu_id]]
else:
from utils.hardware.hardware import get_physical_gpu_inventory
known = [
device["gfx"]
for device in get_physical_gpu_inventory(block = wait).get("devices") or []
if device.get("vendor") == "amd" and device.get("gfx")
]
# Without a mapping for the selected GPU, another adapter's target is no evidence for it.
unmapped = gpu_id is not None and gpu_id not in arches
if known and not (all if unmapped else any)(target in wanted["gfx"] for target in known):
return _unsupported_amd_gpu(
[target for target in known if target not in wanted["gfx"]] if unmapped else known
)
return None
found = rocm_version()
# The ROCm 7 libraries the engine links keep one soname for every 7.x release.
if found is None or found[0] != wanted["rocm"][0] or found < wanted["rocm"]:
return "Requires ROCm {}.{} or a newer {}.x release installed in /opt/rocm".format(
*wanted["rocm"], wanted["rocm"][0]
) + (" (found {}.{}).".format(*found) if found else ".")
from utils.hardware.amd import (
_a_bare_soname_resolves,
amd_kfd_gpu_gfx_targets,
hsa_override_gfx_target,
)
missing = [soname for soname in ROCM_LIBRARIES if not (ROCM_HOME / "lib" / soname).exists()]
if missing:
return (
f"The ROCm installation in /opt/rocm is missing {', '.join(missing)}. Install the full "
"ROCm package (on Ubuntu: sudo apt install rocm), then retry."
)
missing = {
soname: package
for soname, package in ROCM_SYSTEM_LIBRARIES.items()
# The install and the engine both run without the caller's LD_LIBRARY_PATH.
if not (ROCM_HOME / "lib" / soname).exists()
and not _a_bare_soname_resolves(soname, with_ld_library_path = False, when_unknown = False)
}
if missing:
return (
f"Requires {', '.join(missing)}, which vLLM's AMD build links. "
f"On Ubuntu: sudo apt install {' '.join(dict.fromkeys(missing.values()))}"
)
if not (os.environ.get("CC") or shutil.which("gcc") or shutil.which("clang")):
# Triton builds its HIP driver module the first time the engine compiles a kernel.
return "Requires a C compiler, which vLLM's AMD build uses at run time (on Ubuntu: sudo apt install gcc)."
from utils.hardware.amd import amd_closed_nodes_block_the_runtime, amd_node_permission_hint
# HIP opens /dev/kfd and an AMD render node; a container mapping only one initialises no GPU.
if not os.access("/dev/kfd", os.R_OK | os.W_OK) or amd_closed_nodes_block_the_runtime():
return amd_node_permission_hint() or (
"Unsloth cannot open /dev/kfd and an AMD render node (/dev/dri/renderD*). Add your user "
"to the render and video groups, then sign in again."
)
# Each GPU's target as the engine sees it (HSA_OVERRIDE_GFX_VERSION included), else the kernel's.
arches = _known_rocm_gpu_arches(wait = wait)
if not arches and _rocm_arches is None:
kernel = amd_kfd_gpu_gfx_targets()
if os.environ.get("HSA_OVERRIDE_GFX_VERSION", "").strip():
# ROCr presents a readable global override for every GPU KFD lists; the kernel's target ignores it.
override = hsa_override_gfx_target()
if override is None or not kernel:
return "Checking for a supported AMD GPU."
arches = dict.fromkeys(range(len(kernel)), override)
elif kernel is None:
return "Checking for a supported AMD GPU."
if gpu_id is None:
selected = list(arches.values()) or amd_kfd_gpu_gfx_targets() or []
elif gpu_id in arches:
selected = [arches[gpu_id]]
else:
selected = (amd_kfd_gpu_gfx_targets() or [])[gpu_id : gpu_id + 1]
selected = [target for target in selected if target]
if gpu_id is not None and not selected:
# Nothing names this GPU's target; vLLM refuses one it has no kernels for itself.
return None
if any(target in wanted["gfx"] for target in selected):
return None
return _unsupported_amd_gpu(selected)
def _atomic_json(path: Path, data: dict) -> None:
tmp = path.with_name(path.name + "." + uuid.uuid4().hex + ".tmp")
try:
tmp.write_text(json.dumps(data), encoding = "utf-8")
# Windows readers (including status polling in another Studio process) can
# briefly deny deletion of the destination. Keep the previous JSON intact
# and retry the atomic rename; permanent permission errors still surface.
deadline = time.monotonic() + 1.0
while True:
try:
os.replace(tmp, path)
break
except OSError as exc:
if (
sys.platform != "win32"
or getattr(exc, "winerror", None) not in (5, 32, 33)
or time.monotonic() >= deadline
):
raise
time.sleep(0.01)
finally:
tmp.unlink(missing_ok = True)
@contextmanager
def engine_lease(
engine: str,
*,
exclusive: bool = False,
wait: float = 1.0,
):
"""``wait=0`` for status probes: real takers retry so a probe's instant hold never fails them."""
profile(engine)
lock, unlock = _file_lock()
root = engine_root()
root.mkdir(parents = True, exist_ok = True)
with (root / f"{engine}.lock").open("a", encoding = "utf-8") as handle:
deadline = time.monotonic() + wait
while True:
try:
lock(handle, exclusive)
break
except BlockingIOError:
if time.monotonic() >= deadline:
raise RuntimeError(
"This engine is running or being changed in another Unsloth instance."
) from None
time.sleep(0.05)
try:
yield
finally:
unlock(handle)
def _file_lock():
"""(lock, unlock) raising BlockingIOError when held: flock, or LockFileEx on Windows, which
also has shared and exclusive modes (msvcrt.locking is exclusive only)."""
if os.name != "nt":
import fcntl
return (
lambda handle, exclusive: fcntl.flock(
handle, (fcntl.LOCK_EX if exclusive else fcntl.LOCK_SH) | fcntl.LOCK_NB
),
lambda handle: fcntl.flock(handle, fcntl.LOCK_UN),
)
import ctypes
import msvcrt
from ctypes import wintypes
class Overlapped(ctypes.Structure):
_fields_ = [
("Internal", ctypes.c_void_p),
("InternalHigh", ctypes.c_void_p),
("Offset", wintypes.DWORD),
("OffsetHigh", wintypes.DWORD),
("hEvent", wintypes.HANDLE),
]
kernel32 = ctypes.WinDLL("kernel32", use_last_error = True)
def lock(handle, exclusive):
flags = 0x1 | (
0x2 if exclusive else 0
) # LOCKFILE_FAIL_IMMEDIATELY | LOCKFILE_EXCLUSIVE_LOCK
if not kernel32.LockFileEx(
wintypes.HANDLE(msvcrt.get_osfhandle(handle.fileno())),
flags,
0,
1,
0,
ctypes.byref(Overlapped()),
):
raise BlockingIOError(ctypes.get_last_error(), "lock held")
def unlock(handle):
kernel32.UnlockFileEx(
wintypes.HANDLE(msvcrt.get_osfhandle(handle.fileno())),
0,
1,
0,
ctypes.byref(Overlapped()),
)
return lock, unlock
def installed(engine: str) -> dict | None:
profile(engine)
root = engine_root() / engine
if root.is_symlink():
return None
try:
info = json.loads((root / "active.json").read_text(encoding = "utf-8"))
directory = info["directory"]
if (
not isinstance(directory, str)
or not directory.startswith("env-")
or Path(directory).name != directory
):
return None
if info.get("host") == "wsl":
# Lives in the distro; checking it would boot WSL on every status poll, so the load checks.
from .wsl_host import GUEST_ROOT
return {**info, "path": f"{GUEST_ROOT}/engines/{engine}/{directory}"}
path = root / directory
if path.is_symlink() or not (path / "bin" / "python").is_file():
return None
return {**info, "path": str(path)}
except (OSError, ValueError, KeyError, TypeError):
return None
def status(engine: str) -> dict:
info = installed(engine)
try:
job = json.loads((engine_root() / f"{engine}.job.json").read_text(encoding = "utf-8"))
except (OSError, ValueError):
job = {"state": "idle", "phase": None, "message": ""}
if job.get("state") == "running":
try:
with engine_lease(engine, wait = 0):
job = {
"state": "error",
"phase": None,
"message": "Installation was interrupted. Retry to repair it.",
}
except (RuntimeError, ImportError):
pass
outdated = bool(info and stale(info))
in_use = False
if info and job.get("state") != "running":
try:
with engine_lease(engine, exclusive = True, wait = 0):
pass
except RuntimeError:
in_use = True
except ImportError:
pass
wanted = profile(engine)
return {
"engine": engine,
"version": wanted["version"],
"installed_version": info.get("version") if info else None,
"installed": info is not None,
"in_use": in_use,
"current": bool(
info and info.get("profile_digest") == profile_digest(engine) and not outdated
),
"restored": bool(
info and info.get("restored") and not outdated and built_for_this_gpu(engine, info)
),
"shared": bool(info and info.get("shared")),
"can_rollback": bool(
info
and isinstance(info.get("previous"), dict)
and not stale(info["previous"])
and built_for_this_gpu(engine, info["previous"])
),
"unsupported_reason": support_reason(engine, wait = False),
"precisions": list(wanted["precisions"]),
"platform": gpu_platform(),
# Only priced while an install or update is on offer: the plan reads Studio's packages.
"download_bytes": None
if info and info.get("profile_digest") == profile_digest(engine) and not outdated
else _safe_download_bytes(engine),
"additional_disk_bytes": None,
"job": job,
**_host_status(),
}
def _host_status() -> dict:
from . import wsl_host
return {"host": "wsl", "wsl": wsl_host.summary()} if wsl_host.active() else {"host": "local"}
def _safe_download_bytes(engine: str) -> int | None:
try:
return download_bytes(engine)
except Exception:
return None
def _update(engine: str, **values) -> None:
with _lock:
job = _jobs.setdefault(engine, {})
if "phase" in values and values["phase"] != job.get("phase"):
values.setdefault("activity", "")
job.update(values)
root = engine_root()
root.mkdir(parents = True, exist_ok = True)
_atomic_json(root / f"{engine}.job.json", _jobs[engine])
def driver_library_path(env: dict[str, str]) -> str | None:
"""The LD_LIBRARY_PATH entries that hold the NVIDIA driver. Hosts such as Colab keep libcuda
outside the loader cache and reach it only this way; every other entry is dropped, so a CUDA
runtime Studio's environment points at never shadows the engine's own."""
entries = [
entry
for entry in env.get("LD_LIBRARY_PATH", "").split(os.pathsep)
if entry and os.path.isabs(entry) and (Path(entry) / "libcuda.so.1").exists()
]
return os.pathsep.join(entries) or None
def install_environment() -> dict[str, str]:
# Do not inherit Studio's resolver overrides, Python overlays or credentials.
env = {
key: value
for key, value in os.environ.items()
if key
in {
"HOME",
"PATH",
"LANG",
"LC_ALL",
"TMPDIR",
"SSL_CERT_FILE",
"SSL_CERT_DIR",
"HTTPS_PROXY",
"HTTP_PROXY",
"NO_PROXY",
"https_proxy",
"http_proxy",
"no_proxy",
# Where uv puts a managed Python; a custom Studio home keeps it inside the home.
"UV_PYTHON_INSTALL_DIR",
}
}
from utils.paths.storage_roots import cache_root
# Docker replaces the app tree (and /root) on update while engine venvs persist on the
# home volume, so their uv-managed interpreters must live beside them.
app = os.environ.get("UNSLOTH_STUDIO_APP")
python_home = env.get("UV_PYTHON_INSTALL_DIR")
if app and (not python_home or Path(python_home).resolve().is_relative_to(Path(app).resolve())):
env["UV_PYTHON_INSTALL_DIR"] = str(engine_root() / "python")
recorded_cache = None
try:
raw = (cache_root() / "uv-cache-dir").read_bytes().decode("utf-8-sig")
raw = raw.removesuffix("\n").removesuffix("\r")
if Path(raw).is_absolute() and Path(raw).is_dir() and os.access(raw, os.W_OK):
recorded_cache = raw
except (OSError, UnicodeError, ValueError):
pass
env["UV_CACHE_DIR"] = (
os.environ.get("UV_CACHE_DIR") or recorded_cache or str(cache_root() / "uv")
)
# The install checks import the engine, whose device detection needs the driver.
if driver := driver_library_path(dict(os.environ)):
env["LD_LIBRARY_PATH"] = driver
env["PYTHONNOUSERSITE"] = "1"
env["UV_NO_CONFIG"] = "1"
env["UV_CONCURRENT_DOWNLOADS"] = "4"
env["UV_HTTP_RETRIES"] = "5"
# uv's clone fallback can hardlink: opt into clones only after probing both filesystems.
env["UV_LINK_MODE"] = "copy"
return env
def package_link_mode(destination: Path) -> str:
"""Probe Linux reflinks across the actual install paths, without keeping files."""
import fcntl
cache = Path(install_environment()["UV_CACHE_DIR"])
try:
cache.mkdir(parents = True, exist_ok = True)
with (
tempfile.TemporaryFile(dir = cache) as source,
tempfile.TemporaryFile(dir = destination) as target,
):
source.write(b"Studio package clone probe")
source.flush()
fcntl.ioctl(target.fileno(), 0x40049409, source.fileno())
return "clone"
except OSError:
return "copy"
def _run(
engine: str,
argv: list[str],
cancel: threading.Event,
env: dict | None = None,
*,
stdin_pipe: bool = False,
) -> str:
from utils.process_lifetime import (
adopt_pid,
child_popen_kwargs,
terminate_pid,
forget_pid,
spawn_on_lifetime_thread,
is_process_shutting_down,
)
tail: deque[str] = deque(maxlen = 20)
if (engine_root() / f"{engine}.cancel").exists():
cancel.set()
if cancel.is_set() or is_process_shutting_down():
raise RuntimeError("Installation cancelled or Unsloth is shutting down.")
proc = spawn_on_lifetime_thread(
lambda: subprocess.Popen(
argv,
env = install_environment() if env is None else env,
stdin = subprocess.PIPE if stdin_pipe else None,
stdout = subprocess.PIPE,
stderr = subprocess.STDOUT,
text = True,
encoding = "utf-8",
errors = "replace",
start_new_session = True,
# A windowed Studio has no console; wsl.exe would otherwise open one.
creationflags = 0x08000000 if sys.platform == "win32" else 0,
**child_popen_kwargs(),
)
)
adopt_pid(proc.pid)
def drain():
from utils.log_redaction import redact_log_text
last_update = 0.0
activity = ""
for line in proc.stdout:
line = redact_log_text(line.rstrip())[-1000:]
if not line:
continue
tail.append(line)
if line.startswith(
(
"Downloading ",
"Downloaded ",
"Resolved ",
"Prepared ",
"Installed ",
"Uninstalled ",
)
):
activity = line
now = time.monotonic()
if now - last_update >= 1:
_update(engine, activity = activity, log = list(tail))
last_update = now
if tail:
_update(engine, activity = activity, log = list(tail))
reader = threading.Thread(target = drain, daemon = True)
deadline = time.monotonic() + 3600
try:
# A shutdown sweep may have finished between spawn and adoption; recheck even on clean exit.
if is_process_shutting_down() or cancel.is_set():
raise RuntimeError("Installation cancelled or Unsloth is shutting down.")
reader.start()
while proc.poll() is None:
if (engine_root() / f"{engine}.cancel").exists():
cancel.set()
if is_process_shutting_down():
cancel.set()
if cancel.wait(0.2):
raise RuntimeError("Installation cancelled.")
if time.monotonic() > deadline:
raise RuntimeError(
"Installation timed out. Retry when the connection is available."
)
reader.join(timeout = 2)
if cancel.is_set() or is_process_shutting_down():
raise RuntimeError("Installation cancelled or Unsloth is shutting down.")
if proc.returncode:
raise RuntimeError("Engine installation failed. " + "\n".join(tail))
return "\n".join(tail)
finally:
if proc.stdin is not None:
# Closing the WSL runner's pipe stops its guest process group, including
# apt/dpkg children; terminating wsl.exe alone can leave them running.
try:
proc.stdin.close()
proc.wait(timeout = 15)
except (OSError, subprocess.TimeoutExpired):
pass
if proc.poll() is None:
terminate_pid(proc.pid, timeout = 5, owner_verified = True)
proc.wait(timeout = 10)
forget_pid(proc.pid)
if reader.ident is not None:
reader.join(timeout = 2)
proc.stdout.close()
def _index_arguments(engine: str) -> list[str]:
"""PyPI, plus the engine's own index first for the builds only it publishes (vLLM's ROCm torch)."""
extra = profile(engine).get("index")
return [
"--index-url",
"https://pypi.org/simple",
*(["--extra-index-url", extra] if extra else []),
]
# flashinfer.ai/whl/cu130 jit-cache per locked FlashInfer; UNSLOTH_ENGINE_FLASHINFER_KERNELS=0 skips.
_FLASHINFER_KERNELS = _REQUIREMENTS / "flashinfer-kernels.json"
def _kernels_run_on(arches: list[str], capability: str) -> bool:
"""SASS only, no PTX: `X.Ya` runs on exactly X.Y, `X.Y` and `X.Yf` on X.Y and later X minors."""
major, minor = (int(part) for part in capability.strip().split(".")[:2])
for arch in arches:
built = tuple(int(part) for part in arch.rstrip("af").split("."))
if (major, minor) == built or (
not arch.endswith("a") and major == built[0] and minor > built[1]
):
return True
return False
def flashinfer_kernel_wheel(engine: str, *, wait: bool = True) -> dict | None:
"""The jit-cache wheel if it has SASS for every GPU here (a missing arch fails, never JITs)."""
if os.environ.get("UNSLOTH_ENGINE_FLASHINFER_KERNELS", "").strip().lower() in (
"0",
"false",
"no",
"off",
):
return None
wanted = profile(engine)
if (
not wanted.get("flashinfer_kernels")
or sys.platform != "linux"
or wanted["platform"] != "cuda"
or not wanted["cuda"]
):
return None
version = _pins(engine).get("flashinfer-python", (None,))[0]
try:
entry = json.loads(_FLASHINFER_KERNELS.read_text(encoding = "utf-8")).get(
f"{version}+{wanted['cuda']}", {}
)
wheel = entry.get(platform.machine())
if not wheel:
return None
rows = _driver_rows(None, wait = wait)
if (
rows is _PENDING
or not rows
or not all(_kernels_run_on(entry["arches"], row[1]) for row in rows)
):
return None
except (OSError, ValueError, AttributeError, KeyError, IndexError, TypeError):
return None
return wheel
def _install_flashinfer_kernels(
engine: str, uv: str, python: str, destination: Path, cancel: threading.Event
) -> bool:
"""Best effort: a failure removes the wheel and keeps JIT; one that cannot be removed fails."""
wheel = flashinfer_kernel_wheel(engine)
if wheel is None:
return False
requirement = destination / "flashinfer-kernels.txt"
requirement.write_text(
f"flashinfer-jit-cache @ {wheel['url']} --hash=sha256:{wheel['sha256']}\n", encoding = "utf-8"
)
_update(engine, message = "Downloading FlashInfer's prebuilt kernels")
try:
_run(
engine,
[
uv,
"pip",
"install",
"--python",
python,
"--link-mode",
package_link_mode(destination),
"--no-deps",
"--require-hashes",
"--only-binary",
":all:",
"-r",
str(requirement),
],
cancel,
)
_run(engine, [python, "-I", "-c", "import flashinfer_jit_cache, flashinfer"], cancel)
return True
except RuntimeError:
if cancel.is_set():
raise
_run(engine, [uv, "pip", "uninstall", "--python", python, "flashinfer-jit-cache"], cancel)
return False
def _smoke_source(engine: str) -> str:
"""Imports the engine and the precision libraries its lock carries, on the locked torch build."""
wanted = profile(engine)
pins = _pins(engine)
modules = [
wanted["module"],
"torch",
*(name for name in ("bitsandbytes", "torchao") if name in pins),
]
runtime = (
"assert torch.version.hip"
if wanted["platform"] == "rocm"
else "assert torch.version.cuda == '13.0'"
)
return (
"".join(f"import {module}\n" for module in modules)
+ f"assert torch.__version__.split('+')[0] == {pins['torch'][0].split('+')[0]!r}\n"
+ runtime
+ "\n"
)
def _install(
engine: str,
cancel: threading.Event,
lease = None,
) -> None:
destination = None
try:
with nullcontext() if lease is not None else engine_lease(engine, exclusive = True):
_update(engine, state = "running", phase = "creating", message = "Preparing installation")
reason = support_reason(engine)
if reason:
raise RuntimeError(reason)
from . import wsl_host
if wsl_host.active():
_install_wsl(engine, cancel)
return
uv = shutil.which("uv")
if uv is None:
raise RuntimeError("uv is unavailable. Repair the Unsloth installation and retry.")
root = engine_root() / engine
if root.is_symlink():
raise RuntimeError("Engine directory must not be a symbolic link.")
root.mkdir(parents = True, exist_ok = True)
digest = profile_digest(engine)
destination = root / ("env-" + uuid.uuid4().hex)
plan = install_plan(engine)
_update(
engine,
phase = "creating",
message = "Preparing an engine environment on Unsloth's PyTorch"
if plan["shared"]
else "Preparing an isolated Python environment",
)
_run(engine, [uv, "venv", *_venv_python_args(engine), str(destination)], cancel)
python = str(destination / "bin" / "python")
packages = destination / "engine-requirements.txt"
packages.write_text(plan["requirements"], encoding = "utf-8")
_update(
engine, phase = "installing", message = "Downloading and installing engine packages"
)
_run(
engine,
[
uv,
"pip",
"sync",
"--python",
python,
"--link-mode",
package_link_mode(destination),
"--require-hashes",
"--only-binary",
":all:",
*_index_arguments(engine),
str(packages),
],
cancel,
)
if plan["shared"]:
# Appended after the engine's own site-packages, so its pins win.
site = (
destination / "lib" / "python{}.{}".format(*_python(engine)) / "site-packages"
)
(site / f"{_BASE_MODULE}.py").write_text(
_STUDIO_BASE_SOURCE.format(paths = _studio_site()), encoding = "utf-8"
)
(site / _BASE_PTH).write_text(f"import {_BASE_MODULE}\n", encoding = "utf-8")
if profile(engine)["platform"] == "cuda":
link_cuda_home(destination, plan["shared"])
kernels = _install_flashinfer_kernels(engine, uv, python, destination, cancel)
_update(engine, phase = "checking", message = "Checking the installed engine")
_run(
engine,
[
python,
"-I",
"-c",
_CHECK,
str(requirements(engine)),
",".join(profile(engine).get("omit", ())),
profile(engine)["cuda"] or "",
json.dumps(plan["provided"]),
],
cancel,
)
_run(engine, [python, "-I", "-c", _smoke_source(engine)], cancel)
from .engine_adapters import ADAPTERS
module = ADAPTERS[engine].module
_run(engine, [python, "-I", "-m", module, "--help"], cancel)
if (engine_root() / f"{engine}.cancel").exists():
cancel.set()
if cancel.is_set():
raise RuntimeError("Installation cancelled.")
prior = installed(engine)
if profile_digest(engine) != digest:
raise RuntimeError(
"Unsloth's engine profile changed during installation. Retry to use the updated profile."
)
_atomic_json(
root / "active.json",
{
"directory": destination.name,
"version": profile(engine)["version"],
"profile_digest": digest,
"shared": plan["shared"],
"platform": profile(engine)["platform"],
"python": platform.python_version(),
"provided": plan["provided"],
**({"flashinfer_kernels": True} if kernels else {}),
"studio_prefix": sys.prefix,
"previous_directory": prior["directory"] if prior else None,
"previous": {
k: v
for k, v in prior.items()
if k not in ("path", "previous", "previous_directory")
}
if prior
else None,
},
)
# Keep the active env and one previous version; every runtime lease (any Studio) is excluded.
keep = {destination.name, prior["directory"] if prior else None}
for old in root.glob("env-*"):
if old.name not in keep and old.is_dir() and not old.is_symlink():
shutil.rmtree(old, ignore_errors = True)
destination = None
_record_manifest(engine)
_update(engine, state = "success", phase = "ready", message = "Engine installed")
except Exception as exc:
from .wsl_host import Waiting
_update(
engine,
state = "waiting"
if isinstance(exc, Waiting)
else "cancelled"
if cancel.is_set()
else "error",
phase = None,
message = str(exc),
)
finally:
if destination is not None:
shutil.rmtree(destination, ignore_errors = True)
if lease is not None:
lease.__exit__(None, None, None)
# Runs inside WSL: the parts of link_cuda_home Windows cannot inspect.
_GUEST_FINALIZE = r"""
import json, sys
from pathlib import Path
env = Path(sys.argv[1])
site = next(env.glob("lib/python3.*/site-packages"))
tree = site / "nvidia" / "cu13"
home = env / "cuda"
(home / "lib64").mkdir(parents = True, exist_ok = True)
for name in ("bin", "nvvm", "include"):
if (tree / name).exists() and not (home / name).is_symlink():
(home / name).symlink_to(tree / name)
cudart = tree / "lib" / "libcudart.so.13"
if cudart.exists() and not (home / "lib64" / "libcudart.so").is_symlink():
(home / "lib64" / "libcudart.so").symlink_to(cudart)
vendored = site / "vllm" / "third_party" / "deep_gemm"
tag = "cpython-3" + site.parent.name.removeprefix("python3.")
print(json.dumps({
"cuda_environment": {"CUDA_HOME": str(home), "CPATH": str(tree / "include")}
if (home / "bin" / "nvcc").exists() else {},
"deep_gemm_unloadable": not (site / "deep_gemm").is_dir() and vendored.is_dir()
and not any(vendored.glob(f"_C.{tag}-*.so")) and not any(vendored.glob("_C.abi3*.so")),
}))
"""
_PROXIES = ("HTTPS_PROXY", "HTTP_PROXY", "NO_PROXY", "https_proxy", "http_proxy", "no_proxy")
def _install_wsl(engine: str, cancel: threading.Event) -> None:
"""The same hash-pinned lock and checks as on Linux, run inside Studio's WSL distro. Always
isolated: Studio's Windows packages cannot serve a Linux interpreter."""
from . import wsl_host
guest_root = wsl_host.GUEST_ROOT
rocm = profile(engine)["platform"] == "rocm"
progress = lambda text: _update(engine, activity = text)
_update(engine, phase = "preparing_wsl", message = "Setting up the Unsloth WSL environment")
wsl_host.prepare(progress, cancel, profile(engine)["platform"])
if cancel.is_set():
raise RuntimeError("Installation cancelled.")
root = engine_root() / engine
root.mkdir(parents = True, exist_ok = True)
digest = profile_digest(engine)
directory = "env-" + uuid.uuid4().hex
base = f"{guest_root}/engines/{engine}"
destination = f"{base}/{directory}"
python = f"{destination}/bin/python"
uv = f"{guest_root}/bin/uv"
env = {
"PATH": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
"PYTHONNOUSERSITE": "1",
"UV_CACHE_DIR": f"{guest_root}/uv-cache",
"UV_PYTHON_INSTALL_DIR": f"{guest_root}/python",
"UV_NO_CONFIG": "1",
"UV_CONCURRENT_DOWNLOADS": "4",
"UV_HTTP_RETRIES": "5",
# Cache and environments share the distro's ext4 disk.
"UV_LINK_MODE": "hardlink",
"DEBIAN_FRONTEND": "noninteractive",
**(wsl_host.rocm_environment() if rocm else {}),
}
secrets = {key: os.environ[key] for key in _PROXIES if os.environ.get(key)}
def guest_run(argv):
command, windows_env = wsl_host.guest_command(
[f"{guest_root}/bin/run-engine", *argv], env = env, secrets = secrets
)
return _run(engine, command, cancel, env = windows_env, stdin_pipe = True)
try:
wsl_host.ensure_build_tools(guest_run, progress)
if rocm:
_install_wsl_rocm(engine, guest_run, progress, cancel)
_update(engine, phase = "creating", message = "Preparing an isolated Python environment")
guest_run([uv, "venv", *_venv_python_args(engine, guest = True), destination])
_update(engine, phase = "installing", message = "Downloading and installing engine packages")
lock = wsl_host.to_guest_path(requirements(engine))
guest_run(
[
uv,
"pip",
"sync",
"--python",
python,
"--require-hashes",
"--only-binary",
":all:",
*_index_arguments(engine),
lock,
]
)
_update(engine, phase = "checking", message = "Checking the installed engine")
from .engine_adapters import ADAPTERS
scripts = {
"check.py": _CHECK,
"finalize.py": _GUEST_FINALIZE,
"smoke.py": _smoke_source(engine),
}
for name, source in scripts.items():
wsl_host.put(f"{destination}/{name}", source)
guest_run(
[
python,
"-I",
f"{destination}/check.py",
lock,
",".join(profile(engine).get("omit", ())),
profile(engine)["cuda"] or "",
"{}",
]
)
guest_run([python, "-I", f"{destination}/smoke.py"])
guest_run([python, "-I", "-m", ADAPTERS[engine].module, "--help"])
facts = json.loads(
wsl_host.guest([python, "-I", f"{destination}/finalize.py", destination])
.strip()
.splitlines()[-1]
)
if (engine_root() / f"{engine}.cancel").exists():
cancel.set()
if cancel.is_set():
raise RuntimeError("Installation cancelled.")
if profile_digest(engine) != digest:
raise RuntimeError(
"Unsloth's engine profile changed during installation. Retry to use the updated profile."
)
prior = installed(engine)
_atomic_json(
root / "active.json",
{
"directory": directory,
"version": profile(engine)["version"],
"profile_digest": digest,
"shared": False,
"platform": profile(engine)["platform"],
"python": "{}.{}".format(*_python(engine)),
"provided": {},
"host": "wsl",
"distro": wsl_host.distro_name(),
**facts,
"previous_directory": prior["directory"] if prior else None,
"previous": {
k: v
for k, v in prior.items()
if k not in ("path", "previous", "previous_directory")
}
if prior
else None,
},
)
# Committed: from here a failed cleanup must never delete the environment just activated.
destination = None
keep = [directory, *([prior["directory"]] if prior else [])]
try:
wsl_host.guest(
["find", base, "-mindepth", "1", "-maxdepth", "1", "-name", "env-*"]
+ [arg for name in keep for arg in ("!", "-name", name)]
+ ["-exec", "rm", "-rf", "{}", "+"]
)
except (OSError, RuntimeError, subprocess.TimeoutExpired):
pass
finally:
if destination is not None:
try:
wsl_host.guest(["rm", "-rf", destination])
except (OSError, RuntimeError, subprocess.TimeoutExpired):
pass
_record_manifest(engine)
_update(engine, state = "success", phase = "ready", message = "Engine installed")
def _install_wsl_rocm(engine: str, guest_run, progress, cancel: threading.Event) -> None:
"""ROCm's userspace and the DXG bridge in the distro, then the GPU check the host cannot make."""
from . import wsl_host
_update(
engine, phase = "preparing_rocm", message = "Installing AMD ROCm in the Unsloth WSL environment"
)
key = wsl_host.ROCM_APT_KEY
if wsl_host._sha256(key) != wsl_host.ROCM_APT_KEY_SHA256:
raise RuntimeError(
"Unsloth's copy of AMD's ROCm repository key is damaged. Reinstall Unsloth."
)
packages = [
wsl_host.download(spec, spec["url"].rsplit("/", 1)[1], progress, cancel)
for spec in (wsl_host.ROCDXG, wsl_host.ROCDXG_SMI)
]
setup = f"{wsl_host.GUEST_ROOT}/bin/setup-rocm"
wsl_host.put(setup, wsl_host.ROCM_SETUP, "755")
guest_run([setup, *(wsl_host.to_guest_path(path) for path in (key, *packages))])
found = sorted(
set(
re.findall(
r"\bgfx[0-9a-f]+\b",
wsl_host.guest(
["/opt/rocm/bin/rocminfo"], env = wsl_host.rocm_environment(), timeout = 300
),
)
)
)
if not any(target in profile(engine)["gfx"] for target in found):
raise RuntimeError(_unsupported_amd_gpu(found))
wsl_host.write_state(rocm = wsl_host.ROCM_RELEASE)
def start_install(engine: str) -> dict:
profile(engine)
with _lock:
if _jobs.get(engine, {}).get("state") == "running":
return status(engine)
lease = engine_lease(engine, exclusive = True)
lease.__enter__()
cancel = threading.Event()
_cancels[engine] = cancel
try:
(engine_root() / f"{engine}.cancel").unlink(missing_ok = True)
_update(
engine,
state = "running",
phase = "queued",
message = "Preparing installation",
activity = "",
log = [],
)
threading.Thread(target = _install, args = (engine, cancel, lease), daemon = True).start()
except Exception:
lease.__exit__(None, None, None)
raise
return status(engine)
def cancel_install(engine: str) -> dict:
profile(engine)
with _lock:
event = _cancels.get(engine)
if event:
event.set()
if status(engine)["job"].get("state") == "running":
(engine_root() / f"{engine}.cancel").touch()
return status(engine)
def remove(engine: str) -> dict:
with engine_lease(engine, exclusive = True):
root = engine_root() / engine
if root.is_symlink():
raise RuntimeError("Engine directory must not be a symbolic link.")
from . import wsl_host
if (
wsl_host.active()
and not wsl_host.distro_ready()
and wsl_host.wsl_exe() is not None
and wsl_host.distro_name() in wsl_host.registered_distros()
):
# Forgetting the engine now would strand its files inside the distro.
raise RuntimeError("WSL is not responding, so the engine cannot be removed. Try again.")
if wsl_host.active() and wsl_host.distro_ready():
# The compile caches live beside the environments, not under them.
wsl_host.guest(
[
"rm",
"-rf",
f"{wsl_host.GUEST_ROOT}/engines/{engine}",
f"{wsl_host.GUEST_ROOT}/cache/{engine}",
]
)
shutil.rmtree(root, ignore_errors = False) if root.exists() else None
with _lock:
_jobs.pop(engine, None)
(engine_root() / f"{engine}.job.json").unlink(missing_ok = True)
(engine_root() / f"{engine}.cancel").unlink(missing_ok = True)
_record_manifest(engine)
return status(engine)
def remove_wsl_environment() -> list[dict]:
"""Unregister the private distro. Every engine lives in it, so all must be idle."""
from contextlib import ExitStack
from . import wsl_host
if not wsl_host.active():
raise RuntimeError("Unsloth only creates a WSL environment on Windows.")
with ExitStack() as stack:
for engine in PROFILES:
stack.enter_context(engine_lease(engine, exclusive = True))
wsl_host.unregister()
for engine in PROFILES:
root = engine_root() / engine
if root.exists() and not root.is_symlink():
shutil.rmtree(root)
(engine_root() / f"{engine}.job.json").unlink(missing_ok = True)
with _lock:
_jobs.pop(engine, None)
for engine in PROFILES:
_record_manifest(engine)
return [status(engine) for engine in PROFILES]
def rollback(engine: str) -> dict:
with engine_lease(engine, exclusive = True):
info = installed(engine)
previous = info.get("previous") if info else None
directory = previous.get("directory") if isinstance(previous, dict) else None
if (
not isinstance(directory, str)
or not directory.startswith("env-")
or Path(directory).name != directory
):
raise RuntimeError("No previous engine installation is available.")
root = engine_root() / engine
path = root / directory
if previous.get("host") == "wsl":
from . import wsl_host
try:
wsl_host.guest(
["test", "-f", f"{wsl_host.GUEST_ROOT}/engines/{engine}/{directory}/bin/python"]
)
present = True
except (OSError, RuntimeError, subprocess.TimeoutExpired):
present = False
else:
present = not path.is_symlink() and (path / "bin" / "python").is_file()
if not present:
raise RuntimeError(
"The previous engine installation is unavailable. Repair the engine instead."
)
if stale(previous):
raise RuntimeError(
"The previous engine installation was built on packages Unsloth no longer has. Repair the engine instead."
)
if not built_for_this_gpu(engine, previous):
raise RuntimeError(
"The previous engine installation was built for another GPU platform. Update the engine instead."
)
_atomic_json(
root / "active.json",
{
**previous,
"restored": True,
"previous_directory": info["directory"],
"previous": {
k: v
for k, v in info.items()
if k not in ("path", "previous", "previous_directory")
},
},
)
_update(
engine, state = "success", phase = "ready", message = "Previous engine installation restored"
)
_record_manifest(engine)
return status(engine)
def _record_manifest(engine: str) -> None:
try:
from studio.install_manifest import update_manifest
info = installed(engine)
from utils.paths.storage_roots import studio_root
update_manifest(
root = studio_root(),
**{
f"optional_engine_{engine}": {
"installed": info is not None,
"version": info.get("version") if info else None,
"profile_digest": info.get("profile_digest") if info else None,
}
},
)
except ImportError:
pass