Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""Device resolver on CPU with torch faked; both the studio-layer and torch-probe fallback paths."""
from __future__ import annotations
import sys
import types
from typing import Optional
import pytest
from core.inference import diffusion_device as dd
class _FakeDtype:
def __init__(self, name: str) -> None:
self.name = name
def __eq__(self, other: object) -> bool:
return isinstance(other, _FakeDtype) and other.name == self.name
def __hash__(self) -> int:
return hash(self.name)
def __repr__(self) -> str:
return f"torch.{self.name}"
BF16 = _FakeDtype("bfloat16")
FP16 = _FakeDtype("float16")
FP32 = _FakeDtype("float32")
class _FiniteResult:
def __init__(self, finite: bool) -> None:
self._finite = finite
def all(self) -> "_FiniteResult":
return self
def item(self) -> bool:
return self._finite
class _FakeTensor:
def __init__(self, finite: bool = True) -> None:
self._finite = finite
def __add__(self, other: object) -> "_FakeTensor":
return self
def float(self) -> "_FakeTensor":
return self
def _make_torch(
*,
cuda_available: bool = False,
capability = (8, 0),
capability_raises: bool = False,
bf16_supported: bool = False,
hip = None,
mps_available: bool = False,
mps_probe: str = "pass", # "pass" | "raise" | "nonfinite"
xpu_available = None, # None -> no xpu attr; True/False -> present
xpu_bf16: bool = False,
device_count: int = 1,
free_vram_by_index: Optional[dict] = None,
set_device_calls: Optional[list] = None,
) -> types.ModuleType:
torch = types.ModuleType("torch")
torch.bfloat16 = BF16
torch.float16 = FP16
torch.float32 = FP32
torch.version = types.SimpleNamespace(hip = hip)
free_vram_by_index = free_vram_by_index or {}
def _set_device(index):
if set_device_calls is not None:
set_device_calls.append(index)
# Same optional-device signature as torch's own, so a probe of a SELECTED card is answered rather than raising.
def _get_cap(device = None):
if capability_raises:
raise RuntimeError("no capability")
return capability
torch.cuda = types.SimpleNamespace(
is_available = lambda: cuda_available,
get_device_capability = _get_cap,
is_bf16_supported = lambda: bf16_supported,
device_count = lambda: device_count,
mem_get_info = lambda index = None: (free_vram_by_index.get(index, 0), 0),
set_device = _set_device,
)
mps_ns = types.SimpleNamespace(is_available = lambda: mps_available)
torch.backends = types.SimpleNamespace(mps = mps_ns)
def _ones(*_a, **_k):
if mps_probe == "raise":
raise RuntimeError("bf16 unsupported on this MPS")
return _FakeTensor(finite = (mps_probe == "pass"))
torch.ones = _ones
torch.isfinite = lambda t: _FiniteResult(getattr(t, "_finite", True))
if xpu_available is not None:
torch.xpu = types.SimpleNamespace(
is_available = lambda: xpu_available,
is_bf16_supported = lambda: xpu_bf16,
)
return torch
def _install(
monkeypatch,
torch,
*,
studio_device = None,
is_rocm = False,
hardware_fails = False,
):
"""Install the fake torch and either a fake or failing `utils.hardware`."""
monkeypatch.setitem(sys.modules, "torch", torch)
if hardware_fails:
# Force `from utils.hardware import ...` to raise, exercising the torch-probe fallback.
monkeypatch.setitem(sys.modules, "utils.hardware", None)
return
class _DT:
CUDA = "cuda"
XPU = "xpu"
MLX = "mlx"
CPU = "cpu"
fake_uh = types.ModuleType("utils.hardware")
fake_uh.DeviceType = _DT
fake_uh.get_device = lambda: studio_device
fake_uh.hardware = types.SimpleNamespace(IS_ROCM = is_rocm)
monkeypatch.setitem(sys.modules, "utils.hardware", fake_uh)
def test_cuda_ampere_bf16(monkeypatch):
torch = _make_torch(cuda_available = True, capability = (8, 0))
_install(monkeypatch, torch, studio_device = "cuda")
t = dd.resolve_diffusion_device_target()
assert (t.device, t.dtype, t.backend, t.vendor) == ("cuda", BF16, "cuda", "nvidia")
assert (
t.supports_model_cpu_offload
and t.supports_default_torch_compile
and t.supports_pinned_transfer
)
def test_cuda_pre_ampere_fp16(monkeypatch):
torch = _make_torch(cuda_available = True, capability = (7, 5), bf16_supported = True)
_install(monkeypatch, torch, studio_device = "cuda")
t = dd.resolve_diffusion_device_target()
# is_bf16_supported() is True (emulated) but capability < 8, so fp16.
assert t.dtype == FP16 and t.backend == "cuda"
def test_cuda_capability_raises_falls_back_fp16(monkeypatch):
torch = _make_torch(cuda_available = True, capability_raises = True)
_install(monkeypatch, torch, studio_device = "cuda")
t = dd.resolve_diffusion_device_target()
assert t.dtype == FP16 and t.device == "cuda"
def test_cuda_studio_says_cuda_but_unavailable_is_cpu(monkeypatch):
torch = _make_torch(cuda_available = False)
_install(monkeypatch, torch, studio_device = "cuda")
t = dd.resolve_diffusion_device_target()
assert t.device == "cpu" and t.dtype == FP32
def test_rocm_target(monkeypatch):
torch = _make_torch(cuda_available = True, bf16_supported = True)
_install(monkeypatch, torch, studio_device = "cuda", is_rocm = True)
t = dd.resolve_diffusion_device_target()
assert (t.device, t.backend, t.vendor) == ("cuda", "rocm", "amd")
assert t.dtype == BF16
assert t.supports_default_torch_compile is True
def test_rocm_target_compile_kill_switch(monkeypatch):
monkeypatch.setenv("UNSLOTH_DIFFUSION_ROCM_COMPILE", "0")
torch = _make_torch(cuda_available = True, bf16_supported = True)
_install(monkeypatch, torch, studio_device = "cuda", is_rocm = True)
t = dd.resolve_diffusion_device_target()
assert t.backend == "rocm" and t.supports_default_torch_compile is False
def test_rocm_without_bf16_uses_fp16(monkeypatch):
torch = _make_torch(cuda_available = True, bf16_supported = False)
_install(monkeypatch, torch, studio_device = "cuda", is_rocm = True)
t = dd.resolve_diffusion_device_target()
assert t.dtype == FP16 and t.backend == "rocm"
def test_xpu_bf16(monkeypatch):
torch = _make_torch(xpu_available = True, xpu_bf16 = True)
_install(monkeypatch, torch, studio_device = "xpu")
t = dd.resolve_diffusion_device_target()
assert (t.device, t.backend, t.vendor, t.dtype) == ("xpu", "xpu", "intel", BF16)
assert (
t.supports_model_cpu_offload
and not t.supports_default_torch_compile
and not t.supports_pinned_transfer
)
def test_xpu_without_bf16_fp16(monkeypatch):
torch = _make_torch(xpu_available = True, xpu_bf16 = False)
_install(monkeypatch, torch, studio_device = "xpu")
t = dd.resolve_diffusion_device_target()
assert t.device == "xpu" and t.dtype == FP16
def test_mps_probe_pass_bf16(monkeypatch):
torch = _make_torch(mps_available = True, mps_probe = "pass")
_install(monkeypatch, torch, studio_device = "mlx")
t = dd.resolve_diffusion_device_target()
assert (t.device, t.backend, t.vendor, t.dtype) == ("mps", "mps", "apple", BF16)
assert not t.supports_model_cpu_offload
def test_mps_probe_raises_uses_fp32_not_fp16(monkeypatch):
torch = _make_torch(mps_available = True, mps_probe = "raise")
_install(monkeypatch, torch, studio_device = "mlx")
t = dd.resolve_diffusion_device_target()
assert t.device == "mps" and t.dtype == FP32
def test_mps_probe_nonfinite_uses_fp32(monkeypatch):
torch = _make_torch(mps_available = True, mps_probe = "nonfinite")
_install(monkeypatch, torch, studio_device = "mlx")
t = dd.resolve_diffusion_device_target()
assert t.device == "mps" and t.dtype == FP32
def test_studio_cpu_on_apple_prefers_mps(monkeypatch):
torch = _make_torch(mps_available = True, mps_probe = "pass")
_install(monkeypatch, torch, studio_device = "cpu")
t = dd.resolve_diffusion_device_target()
assert t.device == "mps" and t.dtype == BF16
def test_cpu_when_nothing_available(monkeypatch):
torch = _make_torch(mps_available = False)
_install(monkeypatch, torch, studio_device = "cpu")
t = dd.resolve_diffusion_device_target()
assert (t.device, t.backend, t.vendor, t.dtype) == ("cpu", "cpu", None, FP32)
assert not any(
(t.supports_model_cpu_offload, t.supports_default_torch_compile, t.supports_pinned_transfer)
)
def test_fallback_cuda(monkeypatch):
torch = _make_torch(cuda_available = True, capability = (9, 0))
_install(monkeypatch, torch, hardware_fails = True)
t = dd.resolve_diffusion_device_target()
assert t.device == "cuda" and t.dtype == BF16 and t.backend == "cuda"
def test_fallback_rocm_via_torch_hip(monkeypatch):
torch = _make_torch(cuda_available = True, bf16_supported = True, hip = "6.2")
_install(monkeypatch, torch, hardware_fails = True)
t = dd.resolve_diffusion_device_target()
assert t.backend == "rocm" and t.vendor == "amd"
def test_fallback_xpu(monkeypatch):
torch = _make_torch(cuda_available = False, xpu_available = True, xpu_bf16 = True)
_install(monkeypatch, torch, hardware_fails = True)
t = dd.resolve_diffusion_device_target()
assert t.device == "xpu" and t.dtype == BF16
def test_fallback_mps(monkeypatch):
torch = _make_torch(cuda_available = False, mps_available = True, mps_probe = "pass")
_install(monkeypatch, torch, hardware_fails = True)
t = dd.resolve_diffusion_device_target()
assert t.device == "mps" and t.dtype == BF16
def test_fallback_cpu(monkeypatch):
torch = _make_torch(cuda_available = False, mps_available = False)
_install(monkeypatch, torch, hardware_fails = True)
t = dd.resolve_diffusion_device_target()
assert t.device == "cpu" and t.dtype == FP32
def test_from_torch_device_cuda(monkeypatch):
torch = _make_torch()
monkeypatch.setitem(sys.modules, "torch", torch)
t = dd.diffusion_device_target_from_torch_device("cuda:0", FP32)
assert (t.device, t.backend, t.vendor, t.dtype) == ("cuda", "cuda", "nvidia", FP32)
assert t.is_cuda_torch_device
def test_from_torch_device_tag_only_rocm(monkeypatch):
# AMD SDK / Radeon wheels: no torch.version.hip, ROCm only in __version__. Must stay ROCm so the
# probe and UNSLOTH_DIFFUSION_ROCM_COMPILE apply.
torch = _make_torch()
torch.__version__ = "2.9.0+rocm7.1.0"
monkeypatch.setitem(sys.modules, "torch", torch)
monkeypatch.setenv("UNSLOTH_DIFFUSION_ROCM_COMPILE", "0")
t = dd.diffusion_device_target_from_torch_device("cuda:1", BF16)
assert (t.backend, t.vendor, t.ordinal) == ("rocm", "amd", 1)
assert t.supports_default_torch_compile is False
def test_from_torch_device_mps_and_cpu(monkeypatch):
torch = _make_torch()
monkeypatch.setitem(sys.modules, "torch", torch)
mps = dd.diffusion_device_target_from_torch_device("mps", FP16)
assert mps.device == "mps" and not mps.supports_model_cpu_offload
cpu = dd.diffusion_device_target_from_torch_device("cpu", FP32)
assert cpu.device == "cpu" and cpu.vendor is None
@pytest.mark.parametrize(
"dtype,expected", [(BF16, "bfloat16"), (FP16, "float16"), (FP32, "float32")]
)
def test_public_dict_dtype_string(dtype, expected):
t = dd.DiffusionDeviceTarget(
device = "cuda",
dtype = dtype,
backend = "cuda",
vendor = "nvidia",
supports_model_cpu_offload = True,
supports_default_torch_compile = True,
supports_pinned_transfer = True,
)
d = t.as_public_dict()
assert d["dtype"] == expected and "torch." not in d["dtype"]
def test_only_mps_lacks_float64(monkeypatch):
torch = _make_torch(mps_available = True, mps_probe = "pass")
_install(monkeypatch, torch)
assert dd.resolve_diffusion_device_target().supports_float64 is False
for device in ("cuda", "xpu", "cpu"):
assert dd.diffusion_device_target_from_torch_device(device, FP32).supports_float64 is True
assert dd.diffusion_device_target_from_torch_device("mps", FP32).supports_float64 is False
def test_float64_is_built_on_the_device_unless_it_lacks_it():
for device in ("cuda", "cuda:1", "xpu", "cpu", "meta"):
assert dd.float64_device(device) is device
# Callers pass torch.device objects, which only stringify to the device name.
mps = type("Device", (), {"__str__": lambda self: "mps:0"})()
for device in ("mps", "mps:0", mps):
assert dd.float64_device(device) == "cpu"
class _RopeModule:
def __init__(self, double_precision = True):
self.double_precision = double_precision
class _Component:
def __init__(self, *mods):
self._mods = mods
def modules(self):
return iter(self._mods)
class _Pipe:
def __init__(self, **components):
self.components = components
def _mps_target():
return dd.diffusion_device_target_from_torch_device("mps", FP32)
def _cuda_target():
return dd.diffusion_device_target_from_torch_device("cuda", FP32)
def test_force_float32_rope_demotes_every_component_on_mps():
# Connectors and transformer both carry RoPE, so every module must be demoted.
conn, dit_a, dit_b = _RopeModule(), _RopeModule(), _RopeModule()
pipe = _Pipe(connectors = _Component(conn), transformer = _Component(dit_a, dit_b))
assert dd.force_float32_rope(pipe, _mps_target()) == 3
assert not any(m.double_precision for m in (conn, dit_a, dit_b))
def test_force_float32_rope_leaves_float64_devices_untouched():
rope = _RopeModule()
pipe = _Pipe(transformer = _Component(rope))
assert dd.force_float32_rope(pipe, _cuda_target()) == 0
assert rope.double_precision is True
def test_force_float32_rope_skips_modules_without_the_flag():
already_off = _RopeModule(double_precision = False)
plain = object()
pipe = _Pipe(vae = _Component(already_off, plain))
assert dd.force_float32_rope(pipe, _mps_target()) == 0
def test_force_float32_rope_tolerates_non_module_components():
rope = _RopeModule()
pipe = _Pipe(scheduler = object(), tokenizer = None, transformer = _Component(rope))
assert dd.force_float32_rope(pipe, _mps_target()) == 1
assert rope.double_precision is False
def test_the_video_loader_demotes_rope():
# Call must be unconditional (helper no-ops on float64); AST catches and/ternary guards.
import ast
from pathlib import Path
src = (Path(__file__).resolve().parent.parent / "core/inference/video.py").read_text(
encoding = "utf-8"
)
loader = next(
n
for n in ast.walk(ast.parse(src))
if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef)) and n.name == "load_pipeline"
)
def _is_rope_call(node):
return (
isinstance(node, ast.Call)
and isinstance(node.func, ast.Name)
and node.func.id == "force_float32_rope"
)
conditional = {
id(inner)
for node in ast.walk(loader)
if isinstance(node, (ast.If, ast.IfExp, ast.BoolOp))
for inner in ast.walk(node)
}
assert any(
isinstance(n, ast.Expr) and _is_rope_call(n.value) and id(n) not in conditional
for n in ast.walk(loader)
), (
"load_pipeline does not reach force_float32_rope unconditionally, so the demotion is "
"either gone or behind a guard -- and a guard here can only be wrong, since the helper "
"already no-ops wherever float64 works"
)
# ── Frame padding by concatenation on Metal ───────────────────────────
def _frame_pad_pipe(factor_t: int, name: str = "AvgDown3D"):
torch = pytest.importorskip("torch")
class Shortcut(torch.nn.Module):
"""The diffusers shortcut's padding, recording how much of it ``F.pad`` was left to do."""
def __init__(self):
super().__init__()
self.factor_t = factor_t
self.padded = []
def forward(self, x):
pad_t = (self.factor_t - x.shape[2] % self.factor_t) % self.factor_t
self.padded.append(pad_t)
return torch.nn.functional.pad(x, (0, 0, 0, 0, pad_t, 0))
Shortcut.__name__ = name
shortcut = Shortcut()
return torch, shortcut, _Pipe(vae = torch.nn.Sequential(shortcut), scheduler = object())
@pytest.mark.parametrize("name", ["AvgDown3D", "QwenImage21AvgDown3D"])
@pytest.mark.parametrize("factor_t, frames", [(2, 1), (2, 3), (4, 5), (2, 4)])
def test_frame_pad_fix_pads_by_concatenation_on_mps(factor_t, frames, name):
torch, shortcut, pipe = _frame_pad_pipe(factor_t, name)
x = torch.arange(2 * 3 * frames * 2 * 2, dtype = torch.float32).reshape(2, 3, frames, 2, 2)
expected = shortcut(x)
assert dd.install_frame_pad_fix(pipe, _mps_target()) == 1
assert dd.install_frame_pad_fix(pipe, _mps_target()) == 0 # one hook, however often a load runs
shortcut.padded.clear()
assert torch.equal(shortcut(x), expected)
assert shortcut.padded == [0] # nothing left for the pad Metal gets wrong
def test_frame_pad_fix_is_metal_only():
torch, shortcut, pipe = _frame_pad_pipe(2)
assert dd.install_frame_pad_fix(pipe, _cuda_target()) == 0
shortcut(torch.zeros(1, 1, 1, 2, 2))
assert shortcut.padded == [1]
def test_the_image_loader_installs_the_frame_pad_fix():
from pathlib import Path
src = (Path(__file__).resolve().parent.parent / "core/inference/diffusion.py").read_text(
encoding = "utf-8"
)
assert "install_frame_pad_fix(pipe, target, logger = logger)" in src
def test_the_image_loader_syncs_every_decoder_call():
import ast
import inspect
import textwrap
from core.inference import diffusion
src = textwrap.dedent(
inspect.getsource(inspect.unwrap(diffusion.DiffusionBackend.load_pipeline))
)
calls = [
node
for node in ast.walk(ast.parse(src))
if isinstance(node, ast.Call) and getattr(node.func, "id", None) == "install_vae_sync"
]
assert len(calls) == 1
def _target(device: str) -> dd.DiffusionDeviceTarget:
return dd.DiffusionDeviceTarget(
device = device,
dtype = FP32,
backend = device,
vendor = None,
supports_model_cpu_offload = False,
supports_default_torch_compile = False,
supports_pinned_transfer = False,
)
class _FakeDecoder:
"""A VAE decoder module, following nn.Module: a hook returning non-None replaces the output."""
def __init__(self) -> None:
self.hooks: list = []
def register_forward_hook(self, hook):
self.hooks.append(hook)
def decode(self, calls: int) -> list:
outputs = []
for index in range(calls):
out = f"out{index}"
for hook in self.hooks:
replacement = hook(self, (), out)
if replacement is not None:
out = replacement
outputs.append(out)
return outputs
def _pipe_with(decoder, encoder = None) -> types.SimpleNamespace:
return types.SimpleNamespace(vae = types.SimpleNamespace(decoder = decoder, encoder = encoder))
def _mps_torch() -> types.ModuleType:
torch = types.ModuleType("torch")
torch.syncs = 0
def _bump():
torch.syncs += 1
torch.mps = types.SimpleNamespace(synchronize = _bump)
return torch
@pytest.mark.parametrize("device", ["cuda", "xpu", "cpu"])
def test_vae_sync_is_metal_only(monkeypatch, device):
monkeypatch.setitem(sys.modules, "torch", _mps_torch())
decoder = _FakeDecoder()
assert dd.install_vae_sync(_pipe_with(decoder), _target(device)) is False
assert decoder.hooks == []
def test_vae_sync_runs_after_every_decoder_call_and_keeps_its_output(monkeypatch):
# An nn.Module forward hook that returns non-None REPLACES the output; this one must not.
torch = _mps_torch()
monkeypatch.setitem(sys.modules, "torch", torch)
decoder = _FakeDecoder()
assert dd.install_vae_sync(_pipe_with(decoder), _target("mps")) is True
assert decoder.decode(3) == ["out0", "out1", "out2"]
assert torch.syncs == 3
def test_vae_sync_runs_after_every_encoder_call_too(monkeypatch):
torch = _mps_torch()
monkeypatch.setitem(sys.modules, "torch", torch)
decoder, encoder = _FakeDecoder(), _FakeDecoder()
assert dd.install_vae_sync(_pipe_with(decoder, encoder), _target("mps")) is True
assert encoder.decode(2) == ["out0", "out1"]
assert torch.syncs == 2
decoder.decode(1)
assert torch.syncs == 3
lone = _FakeDecoder()
assert dd.install_vae_sync(_pipe_with(object(), lone), _target("mps")) is True
lone.decode(1)
assert torch.syncs == 4
@pytest.mark.parametrize("pipe", [types.SimpleNamespace(), _pipe_with(None), _pipe_with(object())])
def test_vae_sync_no_op_without_a_hookable_decoder(monkeypatch, pipe):
monkeypatch.setitem(sys.modules, "torch", _mps_torch())
assert dd.install_vae_sync(pipe, _target("mps")) is False
def test_vae_sync_survives_a_synchronize_that_raises(monkeypatch):
torch = _mps_torch()
def _boom():
raise RuntimeError("Torch not compiled with MPS enabled")
torch.mps.synchronize = _boom
monkeypatch.setitem(sys.modules, "torch", torch)
decoder = _FakeDecoder()
assert dd.install_vae_sync(_pipe_with(decoder), _target("mps")) is True
assert decoder.decode(2) == ["out0", "out1"]
def _mask(
monkeypatch,
visible,
*,
physical_count = None,
):
"""Stub the hardware layer's parent-visible view, the mask `gpu_ids` is expressed against."""
import utils.hardware.hardware as hw
monkeypatch.setattr(
hw,
"_get_parent_visible_gpu_spec",
lambda: {"raw": None, "numeric_ids": list(visible), "supports_explicit_gpu_ids": True},
)
monkeypatch.setattr(
hw, "get_physical_gpu_count", lambda: physical_count or (max(visible) + 1 if visible else 0)
)
def test_no_selection_leaves_the_target_on_the_default_device(monkeypatch):
calls: list = []
torch = _make_torch(cuda_available = True, capability = (8, 0), set_device_calls = calls)
_install(monkeypatch, torch, studio_device = "cuda")
t = dd.resolve_diffusion_device_target()
assert t.ordinal is None
assert t.torch_device == "cuda"
dd.apply_diffusion_device_ordinal(t)
assert calls == []
def test_a_single_card_pick_is_honoured_exactly(monkeypatch):
calls: list = []
torch = _make_torch(
cuda_available = True, capability = (8, 0), device_count = 2, set_device_calls = calls
)
_install(monkeypatch, torch, studio_device = "cuda")
_mask(monkeypatch, [0, 1])
t = dd.resolve_diffusion_device_target(ordinal = dd.resolve_selected_cuda_ordinal([1]))
assert t.ordinal == 1
# The device string stays BARE: is_cuda / memory / speed / attention all compare it by value.
assert t.device == "cuda"
assert t.is_cuda_torch_device is True
assert t.torch_device == "cuda:1"
dd.apply_diffusion_device_ordinal(t)
assert calls == [1]
def test_physical_ids_are_translated_through_the_visibility_mask(monkeypatch):
# CUDA_VISIBLE_DEVICES=4,5: physical 4 and 5 are valid but torch sees 0 and 1.
torch = _make_torch(cuda_available = True, capability = (8, 0), device_count = 2)
_install(monkeypatch, torch, studio_device = "cuda")
_mask(monkeypatch, [4, 5], physical_count = 8)
assert dd.resolve_selected_cuda_ordinal([4]) == 0
assert dd.resolve_selected_cuda_ordinal([5]) == 1
with pytest.raises(ValueError):
dd.resolve_selected_cuda_ordinal([0])
_mask(monkeypatch, [1, 0], physical_count = 2)
assert dd.resolve_selected_cuda_ordinal([1]) == 0
assert dd.resolve_selected_cuda_ordinal([0]) == 1
def test_several_cards_resolve_to_the_one_with_the_most_free_vram(monkeypatch):
# Ordinal 0 is the SMALL card here, so taking the first id picks the wrong GPU.
torch = _make_torch(
cuda_available = True,
capability = (8, 0),
device_count = 2,
free_vram_by_index = {0: 6 * 1024**3, 1: 15 * 1024**3},
)
_install(monkeypatch, torch, studio_device = "cuda")
_mask(monkeypatch, [0, 1])
assert dd.resolve_selected_cuda_ordinal([0, 1]) == 1
torch.cuda.mem_get_info = lambda index = None: (8 * 1024**3, 0)
assert dd.resolve_selected_cuda_ordinal([0, 1]) == 0
def test_free_vram_is_read_on_the_torch_ordinal_not_the_physical_id(monkeypatch):
# Under a mask the two differ, and querying the physical id would rank the wrong cards.
seen: list = []
torch = _make_torch(cuda_available = True, capability = (8, 0), device_count = 2)
torch.cuda.mem_get_info = lambda index = None: (seen.append(index), 1 << 30)[1:]
_install(monkeypatch, torch, studio_device = "cuda")
_mask(monkeypatch, [4, 5], physical_count = 8)
dd.resolve_selected_cuda_ordinal([4, 5])
assert seen == [0, 1]
def test_an_unreadable_card_sorts_last_rather_than_failing_the_load(monkeypatch):
torch = _make_torch(
cuda_available = True,
capability = (8, 0),
device_count = 3,
free_vram_by_index = {2: 4 * 1024**3},
)
_install(monkeypatch, torch, studio_device = "cuda")
_mask(monkeypatch, [0, 1, 2])
assert dd.resolve_selected_cuda_ordinal([0, 2]) == 2
assert dd.resolve_selected_cuda_ordinal([0, 1]) == 0
def test_an_index_this_host_does_not_have_is_refused(monkeypatch):
torch = _make_torch(cuda_available = True, capability = (8, 0), device_count = 2)
_install(monkeypatch, torch, studio_device = "cuda")
_mask(monkeypatch, [0, 1])
with pytest.raises(ValueError):
dd.resolve_selected_cuda_ordinal([5])
with pytest.raises(ValueError):
dd.resolve_selected_cuda_ordinal([-1])
assert dd.resolve_selected_cuda_ordinal([]) is None
assert dd.resolve_selected_cuda_ordinal(None) is None
def test_the_capability_probe_asks_about_the_selected_card(monkeypatch):
torch = _make_torch(cuda_available = True, device_count = 2)
seen: list = []
_NOTHING = object()
def _cap(device = _NOTHING):
seen.append(device)
return (7, 5) if device in (_NOTHING, 0) else (8, 9)
torch.cuda.get_device_capability = _cap
_install(monkeypatch, torch, studio_device = "cuda")
assert dd.resolve_diffusion_device_target(ordinal = 1).dtype == BF16
assert seen == [1]
# Never probe with no argument: a stub or older build would raise into the fp16 fallback.
seen.clear()
assert dd.resolve_diffusion_device_target().dtype == FP16
assert seen == [_NOTHING]
def test_an_indexed_override_string_keeps_its_card(monkeypatch):
torch = _make_torch(cuda_available = True, capability = (8, 0))
_install(monkeypatch, torch, studio_device = "cuda")
t = dd.diffusion_device_target_from_torch_device("cuda:1", BF16)
assert (t.device, t.ordinal, t.torch_device) == ("cuda", 1, "cuda:1")
assert dd.diffusion_device_target_from_torch_device("cuda", BF16).ordinal is None
def test_a_selection_is_ignored_where_physical_indices_mean_nothing(monkeypatch):
torch = _make_torch(mps_available = True)
_install(monkeypatch, torch, studio_device = "mlx")
assert dd.resolve_diffusion_device_target(ordinal = 1).ordinal is None
def test_the_device_scope_restores_the_previous_card(monkeypatch):
# Route preflights run on a pooled executor, so a leftover pin leaks into the next request.
calls: list = []
torch = _make_torch(cuda_available = True, device_count = 2, set_device_calls = calls)
class _Scope:
def __init__(self, index):
calls.append(("enter", index))
def __enter__(self):
return self
def __exit__(self, *_exc):
calls.append(("exit", None))
return False
torch.cuda.device = _Scope
monkeypatch.setitem(sys.modules, "torch", torch)
with dd.diffusion_device_scope(1):
pass
assert calls == [("enter", 1), ("exit", None)]
calls.clear()
with dd.diffusion_device_scope(None):
pass
assert calls == []
def test_the_rocm_bf16_probe_asks_about_the_selected_card(monkeypatch):
# is_bf16_supported() takes no device, so the selected card must be made current to ask it.
torch = _make_torch(cuda_available = True, hip = "6.0", device_count = 2)
scoped: list = []
class _Scope:
def __init__(self, index):
scoped.append(index)
def __enter__(self):
return self
def __exit__(self, *_exc):
return False
torch.cuda.device = _Scope
torch.cuda.is_bf16_supported = lambda: bool(scoped and scoped[-1] == 1)
_install(monkeypatch, torch, studio_device = "cuda", is_rocm = True)
assert dd.resolve_diffusion_device_target(ordinal = 1).dtype == BF16
assert scoped == [1]
def test_the_device_scope_lets_the_body_exception_through(monkeypatch):
# Catching around the yield makes contextlib raise "generator didn't stop after throw()".
torch = _make_torch(cuda_available = True, device_count = 2)
class _Scope:
def __init__(self, index):
pass
def __enter__(self):
return self
def __exit__(self, *_exc):
return False
torch.cuda.device = _Scope
monkeypatch.setitem(sys.modules, "torch", torch)
with pytest.raises(RuntimeError, match = "the real refusal"):
with dd.diffusion_device_scope(1):
raise RuntimeError("the real refusal")
def test_the_device_scope_still_runs_the_body_on_an_unusable_index(monkeypatch):
torch = _make_torch(cuda_available = True, device_count = 2)
def _boom(_index):
raise RuntimeError("invalid device ordinal")
torch.cuda.device = _boom
monkeypatch.setitem(sys.modules, "torch", torch)
ran = []
with dd.diffusion_device_scope(9):
ran.append(True)
assert ran == [True]
def test_the_placed_ordinal_records_the_card_an_automatic_load_used(monkeypatch):
# Pooled workers keep a pinned load's card; the placed card is recorded to put the worker back.
current = [3]
torch = _make_torch(cuda_available = True, device_count = 4)
torch.cuda.current_device = lambda: current[0]
torch.cuda.set_device = lambda index: current.__setitem__(0, index)
monkeypatch.setitem(sys.modules, "torch", torch)
_install(monkeypatch, torch, studio_device = "cuda")
automatic = dd.resolve_diffusion_device_target()
assert automatic.ordinal is None
assert dd.placed_cuda_ordinal(automatic) == 3
selected = dd.resolve_diffusion_device_target(ordinal = 1)
assert dd.placed_cuda_ordinal(selected) == 1
cpu_torch = _make_torch(cuda_available = False)
monkeypatch.setitem(sys.modules, "torch", cpu_torch)
_install(monkeypatch, cpu_torch, studio_device = "cpu")
assert dd.placed_cuda_ordinal(dd.resolve_diffusion_device_target()) is None
def test_pinning_an_automatic_load_puts_a_shared_worker_back(monkeypatch):
current = [0]
torch = _make_torch(cuda_available = True, device_count = 4)
torch.cuda.current_device = lambda: current[0]
torch.cuda.set_device = lambda index: current.__setitem__(0, index)
monkeypatch.setitem(sys.modules, "torch", torch)
_install(monkeypatch, torch, studio_device = "cuda")
dd.apply_diffusion_device_ordinal(dd.resolve_diffusion_device_target(ordinal = 2))
assert current == [2]
dd.pin_cuda_ordinal(0)
assert current == [0]
dd.pin_cuda_ordinal(None)
assert current == [0]
def test_a_multi_card_pick_declines_to_rank_when_ranking_is_barred(monkeypatch):
# Plan routes must not open a CUDA context while a trainer holds the cards.
torch = _make_torch(cuda_available = True, device_count = 4, free_vram_by_index = {0: 1, 1: 2})
probed: list = []
torch.cuda.mem_get_info = lambda index = None: (probed.append(index), (1, 2))[1]
monkeypatch.setitem(sys.modules, "torch", torch)
_install(monkeypatch, torch, studio_device = "cuda")
import utils.hardware.hardware as hw
monkeypatch.setattr(hw, "get_parent_visible_gpu_ids", lambda: [0, 1, 2, 3])
monkeypatch.setattr(hw, "get_physical_gpu_count", lambda: 4)
assert dd.resolve_selected_cuda_ordinal([2], allow_ranking = False) == 2
assert probed == []
assert dd.resolve_selected_cuda_ordinal([0, 1], allow_ranking = False) is None
assert probed == []
with pytest.raises(ValueError):
dd.resolve_selected_cuda_ordinal([9], allow_ranking = False)
# ── MPS allocation cap ────────────────────────────────────────────────
GIB = 1024**3
@pytest.mark.parametrize(
"recommended_gib, ram_gib, expected_cap_gib",
[
(5.33, 8, 6.5), # 8 GB Mac: RAM - 1.5 GiB floor reserve
(10.67, 16, 14.4), # RAM - 10%
(96, 128, 115.2),
(5.0, 5.5, 5.0), # reserve would go below the working set: never cap under it
],
)
def test_mps_ratio_caps_below_ram_never_below_working_set(
recommended_gib, ram_gib, expected_cap_gib
):
ratio = dd.mps_high_watermark_ratio(int(recommended_gib * GIB), int(ram_gib * GIB))
cap_gib = ratio * recommended_gib
assert cap_gib == pytest.approx(expected_cap_gib, rel = 1e-3)
assert cap_gib < ram_gib
assert 1.0 <= ratio <= 2.0
assert cap_gib < 1.7 * recommended_gib # tighter than torch's own default, which is above RAM
def test_mps_ratio_unknown_readings_mean_no_cap():
assert dd.mps_high_watermark_ratio(None, 16 * GIB) is None
assert dd.mps_high_watermark_ratio(10 * GIB, 0) is None
assert dd.mps_high_watermark_ratio("x", 16 * GIB) is None
def _mps_cap_torch(
recorded,
*,
recommended = 10 * GIB,
fraction_raises = False,
):
torch = _make_torch(mps_available = True, mps_probe = "pass")
def _set(fraction):
if fraction_raises:
raise AttributeError("old torch")
recorded.append(fraction)
torch.mps = types.SimpleNamespace(
recommended_max_memory = lambda: recommended, set_per_process_memory_fraction = _set
)
return torch
def test_mps_target_sets_finite_cap(monkeypatch):
monkeypatch.delenv(dd.MPS_HIGH_WATERMARK_ENV, raising = False)
monkeypatch.delenv("UNSLOTH_DIFFUSION_ALLOW_OVERSIZED_LOAD", raising = False)
monkeypatch.setattr(dd, "_total_ram_bytes", lambda: 16 * GIB)
recorded = []
_install(monkeypatch, _mps_cap_torch(recorded), studio_device = "mlx")
assert dd.resolve_diffusion_device_target().device == "mps"
assert recorded == [pytest.approx(14.4 / 10, rel = 1e-3)]
@pytest.mark.parametrize(
"env",
[
{dd.MPS_HIGH_WATERMARK_ENV: "0.0"}, # explicit "no cap" stays no cap
{dd.MPS_HIGH_WATERMARK_ENV: "1.2"},
{"UNSLOTH_DIFFUSION_ALLOW_OVERSIZED_LOAD": "1"},
],
)
def test_mps_cap_respects_user_override(monkeypatch, env):
monkeypatch.delenv(dd.MPS_HIGH_WATERMARK_ENV, raising = False)
monkeypatch.delenv("UNSLOTH_DIFFUSION_ALLOW_OVERSIZED_LOAD", raising = False)
for k, v in env.items():
monkeypatch.setenv(k, v)
monkeypatch.setattr(dd, "_total_ram_bytes", lambda: 16 * GIB)
recorded = []
_install(monkeypatch, _mps_cap_torch(recorded), studio_device = "mlx")
assert dd.resolve_diffusion_device_target().device == "mps"
assert recorded == []
if dd.MPS_HIGH_WATERMARK_ENV in env:
import os
assert os.environ[dd.MPS_HIGH_WATERMARK_ENV] == env[dd.MPS_HIGH_WATERMARK_ENV]
def test_mps_cap_failure_keeps_target(monkeypatch):
monkeypatch.delenv(dd.MPS_HIGH_WATERMARK_ENV, raising = False)
monkeypatch.delenv("UNSLOTH_DIFFUSION_ALLOW_OVERSIZED_LOAD", raising = False)
monkeypatch.setattr(dd, "_total_ram_bytes", lambda: 16 * GIB)
_install(monkeypatch, _mps_cap_torch([], fraction_raises = True), studio_device = "mlx")
t = dd.resolve_diffusion_device_target()
assert t.device == "mps" and t.dtype == BF16
_PROBE_ENV = "UNSLOTH_DIFFUSION_XPU_FP64_PROBE"
def _fake_xpu_torch(has_fp64 = None, raises = False):
import torch
def props(index):
if raises:
raise RuntimeError("no device properties")
return (
types.SimpleNamespace()
if has_fp64 is None
else types.SimpleNamespace(has_fp64 = has_fp64)
)
xpu = types.SimpleNamespace(
is_bf16_supported = lambda: True,
current_device = lambda: 0,
get_device_properties = props,
)
return types.SimpleNamespace(xpu = xpu, bfloat16 = torch.bfloat16, float16 = torch.float16)
@pytest.mark.parametrize(
"has_fp64, raises, env, want",
[
(False, False, None, False), # Arc A-series
(True, False, None, True),
(None, False, None, True),
(False, True, None, True),
(False, False, "0", True),
],
)
def test_xpu_target_reports_the_devices_float64(monkeypatch, has_fp64, raises, env, want):
if env is None:
monkeypatch.delenv(_PROBE_ENV, raising = False)
else:
monkeypatch.setenv(_PROBE_ENV, env)
target = dd._xpu_target(_fake_xpu_torch(has_fp64, raises))
assert target.device == "xpu"
assert target.supports_float64 is want
def test_fp64_less_xpu_demotes_ltx2_rope_to_float32(monkeypatch):
torch = pytest.importorskip("torch")
ltx2 = pytest.importorskip("diffusers.models.transformers.transformer_ltx2")
monkeypatch.delenv(_PROBE_ENV, raising = False)
rope = ltx2.LTX2AudioVideoRotaryPosEmbed(dim = 96, num_attention_heads = 2)
assert rope.double_precision
pipe = types.SimpleNamespace(components = {"transformer": torch.nn.Sequential(rope)})
coords = rope.prepare_video_coords(1, 4, 8, 8, device = "cpu")
oracle = rope(coords, device = "cpu")
assert dd.force_float32_rope(pipe, dd._xpu_target(_fake_xpu_torch(True))) == 0
assert rope.double_precision
assert dd.force_float32_rope(pipe, dd._xpu_target(_fake_xpu_torch(False))) == 1
assert not rope.double_precision
got = rope(coords, device = "cpu")
for a, b in zip(got, oracle):
assert a.dtype == b.dtype == torch.float32
assert 0 < float((a - b).abs().max()) < 1e-2
assert dd.float64_device("cpu") == "cpu"
def test_float64_device_probes_the_operands_xpu(monkeypatch):
"""``float64_device`` must probe the operand's own XPU index, not the current card."""
import torch
asked = []
def props(index):
asked.append(index)
return types.SimpleNamespace(has_fp64 = index != 1)
xpu = types.SimpleNamespace(current_device = lambda: 1, get_device_properties = props)
monkeypatch.setattr(torch, "xpu", xpu, raising = False)
monkeypatch.delenv(_PROBE_ENV, raising = False)
assert dd.float64_device("xpu:1") == "cpu"
assert dd.float64_device("xpu:0") == "xpu:0"
assert dd.float64_device("xpu") == "cpu"
assert asked == [1, 0, 1]
monkeypatch.setenv(_PROBE_ENV, "0")
assert dd.float64_device("xpu:1") == "xpu:1"