Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""sd.cpp graph-cut VRAM budget and the H3 CUDA compute-capability fallback.
The committed H3 flags keep ``--max-vram -1`` (sd.cpp's own free-VRAM reading) as a placeholder; each process launch
replaces it with an explicit budget read by Studio right before the spawn, and a device OOM under that budget is
retried once at a smaller one. Hermetic: no binary, no GPU, no socket."""
from __future__ import annotations
import sys
import threading
from pathlib import Path
import pytest
from core.inference import sd_cpp_args as args_mod
from core.inference import sd_cpp_backend
from core.inference import sd_cpp_server as srv
from core.inference import video as video_mod
from core.inference import video_minimax_h3 as h3
from core.inference.sd_cpp_args import SdCppModelFiles, SdCppVideoGenParams
from core.inference.video import VideoBackend
_TESTS_DIR = str(Path(__file__).resolve().parent)
if _TESTS_DIR not in sys.path:
sys.path.insert(0, _TESTS_DIR)
GiB = 1 << 30
# The failure the offload audit recorded (H3 Q4_K, 12 GB card, another process moved VRAM mid-render).
_AUDIT_OOM = (
"sd-server generation failed: generation_failed: generate_video returned no results\n"
"Last output:\n"
"[E] ggml: ggml_backend_cuda_buffer_type_alloc_buffer: allocating 210.08 MiB on device 0: "
"cudaMalloc failed: out of memory\n"
"[E] ggml: alloc_tensor_range: failed to allocate CUDA0 buffer of size 220288512\n"
"[E] minimax_h3 failed to capture graph cut tensor: ggml_runner_cut:minimax_h3.blocks.7|hidden_states|\n"
"[E] minimax_h3 segment 11/54 (minimax_h3.blocks.7) failed during output caching\n"
)
_FILES = SdCppModelFiles(
diffusion_model = "/m/h3.gguf",
vae = "/m/vae.safetensors",
audio_vae = "/m/audio_vae.safetensors",
llm = "/m/te.gguf",
)
_COMMITTED = ("--offload-to-cpu", "--diffusion-fa", "--max-vram", "-1", "--stream-layers")
@pytest.fixture(autouse = True)
def _no_kill_switch(monkeypatch):
monkeypatch.delenv(args_mod.GRAPH_CUT_EXPLICIT_BUDGET_ENV, raising = False)
def test_budget_is_free_minus_reserve_rounded_down():
assert args_mod.graph_cut_budget_gib(None) is None
assert args_mod.graph_cut_budget_gib(int(8.0 * GiB)) == 6.0
assert args_mod.graph_cut_budget_gib(int(7.999 * GiB)) == 5.99
# Under the floor after the reserve: keep sd.cpp's own reading.
assert args_mod.graph_cut_budget_gib(int(2.9 * GiB)) is None
assert args_mod.graph_cut_budget_gib(int(8 * GiB), scale = 0.5) == 3.0
def test_only_the_auto_placeholder_is_replaced():
assert args_mod.with_max_vram_budget(_COMMITTED, 6.25)[3] == "6.25"
per_device = args_mod.with_max_vram_budget(_COMMITTED, {"cuda0": 6.0, "cuda1": 12.5})
assert per_device[3] == "cuda0=6.00,cuda1=12.50"
# An explicit value (the user's, or an earlier launch's) is never widened or rewritten.
explicit = ["--max-vram", "4"]
assert args_mod.with_max_vram_budget(explicit, 20.0) == explicit
assert args_mod.with_max_vram_budget(["--max-vram", "cuda0=4"], 20.0) == [
"--max-vram",
"cuda0=4",
]
assert args_mod.with_max_vram_budget(["--offload-to-cpu"], 6.0) == ["--offload-to-cpu"]
assert args_mod.with_max_vram_budget(_COMMITTED, None) == list(_COMMITTED)
assert args_mod.with_max_vram_budget(_COMMITTED, {}) == list(_COMMITTED)
def test_explicit_budget_round_trips_and_scales():
assert args_mod.explicit_max_vram_gib(_COMMITTED) is None
assert args_mod.explicit_max_vram_gib(["--max-vram", "6.25"]) == 6.25
assert args_mod.explicit_max_vram_gib(["--max-vram", "0"]) is None
assert args_mod.explicit_max_vram_gib(["--max-vram", "cuda0=6,cuda1=12.5"]) == {
"cuda0": 6.0,
"cuda1": 12.5,
}
assert args_mod.explicit_max_vram_gib(["--max-vram", "garbage"]) is None
assert args_mod.scaled_vram_budget(8.0, 0.75) == 6.0
assert args_mod.scaled_vram_budget({"cuda0": 8.0, "cuda1": 4.0}, 0.75) == {
"cuda0": 6.0,
"cuda1": 3.0,
}
# Below the floor a smaller budget is not a working one.
assert args_mod.scaled_vram_budget(1.2, 0.75) is None
assert args_mod.scaled_vram_budget({"cuda0": 8.0, "cuda1": 1.2}, 0.75) is None
def test_device_oom_text():
assert args_mod.is_device_oom_text(_AUDIT_OOM)
assert args_mod.is_device_oom_text(
"ggml_vulkan: vk::Device::allocateMemory: ErrorOutOfDeviceMemory"
)
assert args_mod.is_device_oom_text("hipMalloc failed: out of memory")
assert not args_mod.is_device_oom_text(
"sd-cli exited 1. Last output:\n[E] unknown argument --foo"
)
assert not args_mod.is_device_oom_text(None)
def test_a_host_allocation_failure_is_not_a_device_oom():
# A smaller --max-vram keeps more on the host: it cannot cure exhausted RAM.
for text in (
"sd-server generation failed\n[E] host malloc failed: out of memory",
"terminate called after throwing an instance of 'std::bad_alloc'",
"[E] vk::Device::allocateMemory: ErrorOutOfHostMemory out_of_host_memory",
"load failed: out of memory",
):
assert not args_mod.is_device_oom_text(text), text
# The graph-cut executor's own segment capacity failures are device ones.
assert args_mod.is_device_oom_text(
"[E] minimax_h3 segment 3/54 (minimax_h3.blocks.1) failed during workspace capacity check"
)
assert args_mod.is_device_oom_text("CUDA error: out of memory\n current device: 0")
class _Rt:
def __init__(self, free):
self.calls = 0
self._free = free
def probe():
self.calls += 1
if isinstance(self._free, Exception):
raise self._free
return self._free
self.vram_free_probe = probe
def test_launch_flags_read_the_card_at_launch():
rt = _Rt(int(10 * GiB))
out = video_mod._h3_budgeted_launch_flags(rt, _COMMITTED)
assert out[3] == "8.00" and rt.calls == 1
# A retry cap is a ceiling, never a floor.
assert video_mod._h3_budgeted_launch_flags(rt, _COMMITTED, cap = 5.5)[3] == "5.50"
assert video_mod._h3_budgeted_launch_flags(rt, _COMMITTED, cap = 50.0)[3] == "8.00"
rt = _Rt({"cuda0": int(10 * GiB), "cuda1": int(16 * GiB)})
assert video_mod._h3_budgeted_launch_flags(rt, _COMMITTED)[3] == "cuda0=8.00,cuda1=14.00"
@pytest.mark.parametrize("free", [None, int(2 * GiB), RuntimeError("nvidia-smi timed out")])
def test_unknown_or_tiny_reading_keeps_the_placeholder(free):
rt = _Rt(free)
assert video_mod._h3_budgeted_launch_flags(rt, _COMMITTED) == list(_COMMITTED)
@pytest.mark.parametrize("free", [None, RuntimeError("nvidia-smi timed out")])
def test_an_unreadable_card_still_retries_below_the_budget_that_failed(free):
assert video_mod._h3_budgeted_launch_flags(_Rt(free), _COMMITTED, cap = 4.5)[3] == "4.50"
def test_a_known_small_reading_never_takes_a_larger_retry_cap():
# 2 GiB free now: a 4.5 / 6 GiB cap from the earlier launch would exceed the card; sd.cpp's own -1 does not.
for cap in (4.5, 6.0, {"cuda0": 6.0}):
assert video_mod._h3_budgeted_launch_flags(_Rt(int(2 * GiB)), _COMMITTED, cap = cap) == list(
_COMMITTED
)
rt = _Rt({"cuda0": int(10 * GiB), "cuda1": int(2 * GiB)})
assert video_mod._h3_budgeted_launch_flags(
rt, _COMMITTED, cap = {"cuda0": 6.0, "cuda1": 6.0}
) == list(_COMMITTED)
def test_a_uma_pinned_weight_reserve_is_never_replaced():
"""#13786: a negative --max-vram other than the -1 placeholder is sd.cpp's reading less the GTT-pinned weights; an
explicit free-minus-2-GiB budget in its place would hand those weights' memory back to the graph."""
uma = ["--offload-to-cpu", "--diffusion-fa", "--max-vram", "-17.05", "--stream-layers"]
rt = _Rt(int(22 * GiB))
assert video_mod._h3_budgeted_launch_flags(rt, uma) == uma
assert video_mod._h3_budgeted_launch_flags(rt, uma, cap = 6.0) == uma
assert rt.calls == 0
assert args_mod.with_max_vram_budget(uma, 8.0) == uma
assert args_mod.explicit_max_vram_gib(uma) is None
assert video_mod._h3_retry_vram_cap(uma, _AUDIT_OOM) is None
assert args_mod.max_vram_is_placeholder("-1") and not args_mod.max_vram_is_placeholder("-17.05")
def test_kill_switch_and_no_probe_keep_the_placeholder(monkeypatch):
class _NoProbe:
pass
assert video_mod._h3_budgeted_launch_flags(_NoProbe(), _COMMITTED) == list(_COMMITTED)
monkeypatch.setenv(args_mod.GRAPH_CUT_EXPLICIT_BUDGET_ENV, "0")
rt = _Rt(int(10 * GiB))
assert video_mod._h3_budgeted_launch_flags(rt, _COMMITTED) == list(_COMMITTED)
assert rt.calls == 0
def test_flags_without_graph_cut_are_untouched():
rt = _Rt(int(10 * GiB))
flags = ["--offload-to-cpu", "--diffusion-fa"]
assert video_mod._h3_budgeted_launch_flags(rt, flags) == flags
assert rt.calls == 0
def test_retry_cap():
launched = ["--offload-to-cpu", "--max-vram", "8.00"]
assert video_mod._h3_retry_vram_cap(launched, _AUDIT_OOM) == 6.0
assert video_mod._h3_retry_vram_cap(launched, "sd-cli exited 1: bad flag") is None
assert (
video_mod._h3_retry_vram_cap(list(_COMMITTED), _AUDIT_OOM) is None
) # sd.cpp's own reading: no retry
assert video_mod._h3_retry_vram_cap(None, _AUDIT_OOM) is None
def test_slot_launches_with_the_budget_but_reuses_on_the_placeholder(monkeypatch):
started = []
class _Srv:
def __init__(self, binary):
self.alive = False
def start(
self,
files,
*,
offload = None,
threads = None,
env = None,
extra_args = None,
):
started.append(list(offload))
self.alive = True
def is_alive(self):
return self.alive
def stop(self):
self.alive = False
monkeypatch.setattr(srv, "SdCppServer", _Srv)
monkeypatch.setattr("core.inference.sd_cpp_engine.is_managed_binary", lambda b: False)
slot = h3.H3NativeServerSlot("/x/sd-server", _FILES, _COMMITTED)
budgets = iter(["8.00", "7.00"])
def transform(flags):
out = list(flags)
out[out.index("--max-vram") + 1] = next(budgets)
return out
first = slot.get(_COMMITTED, launch_transform = transform)
# Same committed flags, a different launch-time budget: the live server is reused, not respawned.
assert slot.get(_COMMITTED, launch_transform = transform) is first
assert started == [
["--offload-to-cpu", "--diffusion-fa", "--max-vram", "8.00", "--stream-layers"]
]
assert slot.launched_flags[3] == "8.00"
assert slot.alive_signature()[2] == _COMMITTED
slot.stop()
class _Server:
def __init__(self, outcomes):
self.outcomes = outcomes
self.alive = True
def is_alive(self):
return self.alive
def vid_gen(
self,
payload,
*,
on_step = None,
cancel_event = None,
):
outcome = self.outcomes.pop(0)
if isinstance(outcome, Exception):
raise outcome
return outcome
class _Slot:
def __init__(self, server):
self.server = server
self.disabled_reason = None
self.launched_flags = None
self.launches = []
self.stops = []
def get(
self,
flags = None,
env = None,
*,
cancel_event = None,
pre_spawn = None,
launch_transform = None,
):
if pre_spawn is not None:
flags = list(pre_spawn())
self.launched_flags = tuple(launch_transform(tuple(flags)))
self.launches.append(self.launched_flags)
return self.server
def begin_render(self):
pass
def end_render(self):
pass
def stop(self, reason = "stopped"):
self.stops.append(reason)
def disable(self, reason):
self.disabled_reason = reason
def release(self, reason):
return True
def alive_signature(self):
return None
class _ServerRuntime:
def __init__(self, slot, free):
self.server_slot = slot
self.offload_flags = _COMMITTED
self.env = ()
self.vram_free_probe = lambda: free
def _params():
return SdCppVideoGenParams(
prompt = "p", width = 864, height = 480, num_frames = 39, fps = 24, steps = 4, cfg_scale = 1.0, seed = 1
)
def _server_render(slot, free, tmp_path):
return VideoBackend._h3_native_server_render(
None,
_ServerRuntime(slot, free),
_params(),
output_path = tmp_path / "out.avi",
on_log = lambda line: None,
cancel = threading.Event(),
)
def test_server_render_retries_once_smaller_on_device_oom(tmp_path, monkeypatch):
monkeypatch.delenv(h3.H3_NATIVE_SERVER_ENV, raising = False)
slot = _Slot(_Server([RuntimeError(_AUDIT_OOM), b"AVI"]))
out = _server_render(slot, int(10 * GiB), tmp_path)
assert out.read_bytes() == b"AVI"
budgets = [launch[launch.index("--max-vram") + 1] for launch in slot.launches]
assert budgets == ["8.00", "6.00"]
assert len(slot.stops) == 1 and "smaller VRAM budget" in slot.stops[0]
def test_server_render_retries_only_once(tmp_path, monkeypatch):
monkeypatch.delenv(h3.H3_NATIVE_SERVER_ENV, raising = False)
slot = _Slot(_Server([RuntimeError(_AUDIT_OOM), RuntimeError(_AUDIT_OOM)]))
with pytest.raises(RuntimeError, match = "out of memory"):
_server_render(slot, int(10 * GiB), tmp_path)
assert len(slot.launches) == 2
def test_server_render_does_not_fall_back_to_sd_cli_after_the_retry_dies(tmp_path, monkeypatch):
monkeypatch.delenv(h3.H3_NATIVE_SERVER_ENV, raising = False)
server = _Server([RuntimeError(_AUDIT_OOM), RuntimeError(_AUDIT_OOM)])
vid_gen = server.vid_gen
def dies_on_retry(*args, **kwargs):
if len(server.outcomes) == 1:
server.alive = False
return vid_gen(*args, **kwargs)
server.vid_gen = dies_on_retry
slot = _Slot(server)
with pytest.raises(RuntimeError, match = "out of memory"):
_server_render(slot, int(10 * GiB), tmp_path)
assert len(slot.launches) == 2
assert slot.disabled_reason is None
def test_server_render_does_not_fall_back_when_the_retry_start_runs_out_of_memory(
tmp_path, monkeypatch
):
monkeypatch.delenv(h3.H3_NATIVE_SERVER_ENV, raising = False)
slot = _Slot(_Server([RuntimeError(_AUDIT_OOM)]))
get = slot.get
def oom_on_second_start(*args, **kwargs):
server = get(*args, **kwargs)
if len(slot.launches) == 2:
raise RuntimeError(_AUDIT_OOM)
return server
slot.get = oom_on_second_start
with pytest.raises(RuntimeError, match = "out of memory"):
_server_render(slot, int(10 * GiB), tmp_path)
assert len(slot.launches) == 2
assert slot.disabled_reason is None
def test_server_render_does_not_restart_over_a_pending_release(tmp_path, monkeypatch):
monkeypatch.delenv(h3.H3_NATIVE_SERVER_ENV, raising = False)
slot = _Slot(_Server([RuntimeError(_AUDIT_OOM)]))
slot.release_pending = "another model is loading"
with pytest.raises(RuntimeError, match = "out of memory"):
_server_render(slot, int(10 * GiB), tmp_path)
assert len(slot.launches) == 1
def test_a_server_start_that_runs_out_of_memory_retries_once_smaller_on_the_server(
tmp_path, monkeypatch
):
monkeypatch.delenv(h3.H3_NATIVE_SERVER_ENV, raising = False)
slot = _Slot(_Server([b"AVI"]))
get = slot.get
def oom_on_first_start(*args, **kwargs):
server = get(*args, **kwargs)
slot.attempted_flags = slot.launched_flags
if len(slot.launches) == 1:
slot.launched_flags = None # a failed start never records launched flags
raise RuntimeError(_AUDIT_OOM)
return server
slot.get = oom_on_first_start
out = _server_render(slot, int(10 * GiB), tmp_path)
assert out.read_bytes() == b"AVI"
budgets = [launch[launch.index("--max-vram") + 1] for launch in slot.launches]
assert budgets == ["8.00", "6.00"]
assert slot.disabled_reason is None # the server path is kept, not given up for the session
def test_restart_for_retry_keeps_a_release_that_landed_before_the_stop(monkeypatch):
stopped = []
class _Srv:
def __init__(self, binary):
self.alive = False
def start(
self,
files,
*,
offload = None,
threads = None,
env = None,
extra_args = None,
):
self.alive = True
def is_alive(self):
return self.alive
def stop(self):
stopped.append(1)
self.alive = False
monkeypatch.setattr(srv, "SdCppServer", _Srv)
monkeypatch.setattr("core.inference.sd_cpp_engine.is_managed_binary", lambda b: False)
slot = h3.H3NativeServerSlot("/x/sd-server", _FILES, _COMMITTED)
slot.get(_COMMITTED)
slot.begin_render()
assert slot.release("another model needs the memory") is True # busy: left pending
assert slot.restart_for_retry("device out of memory") is False
assert slot.release_pending == "another model needs the memory" and stopped == []
assert (
slot.end_render() == "another model needs the memory"
) # the release is honoured at the end
assert stopped == [1]
# With nothing pending the retry stops the server in the same locked step.
slot.get(_COMMITTED)
assert slot.restart_for_retry("device out of memory") is True and slot.release_pending is None
# A release landing in the gap between the retry's stop and its respawn still stops the fresh server at the end.
slot.begin_render()
assert slot.release("another load") is True and slot.release_pending == "another load"
slot.get(_COMMITTED)
assert slot.end_render() == "another load" and stopped == [1, 1, 1]
# A release left with no server to stop does not outlive the render (it would refuse the next retry).
slot.begin_render()
slot.release("late")
assert slot.end_render() is None and slot.release_pending is None
_H3_FLAGS = ("--offload-to-cpu", "--diffusion-fa", "--max-vram", "-1", "--stream-layers")
def _h3_telemetry(monkeypatch, *, rocm, smi):
"""48 GiB card with 8 used: amd-smi / nvidia-smi rows when ``smi``, else only torch can read it."""
from utils.hardware import hardware
touched = []
monkeypatch.setattr(hardware, "get_device", lambda: hardware.DeviceType.CUDA)
monkeypatch.setattr(hardware, "IS_ROCM", rocm)
monkeypatch.setattr(
hardware, "_get_parent_visible_gpu_spec", lambda: {"raw": "0", "numeric_ids": [0]}
)
monkeypatch.setattr(hardware, "get_parent_visible_gpu_ids", lambda: [0])
row = {"index": 0, "visible_ordinal": 0, "vram_total_gb": 48.0, "vram_used_gb": 8.0}
monkeypatch.setattr(
hardware,
"_smi_visible_utilization",
lambda spec: {"available": True, "devices": [dict(row)], "index_kind": "physical"}
if smi
else None,
)
monkeypatch.setattr(
hardware,
"_reconcile_rocm_unified_memory",
lambda *_a: touched.append("reconcile"),
)
def per_device(idx):
touched.append("torch")
return [{"index": 0, "visible_ordinal": 0, "total_gb": 48.0, "used_gb": 8.0}]
monkeypatch.setattr(hardware, "_torch_get_per_device_info", per_device)
monkeypatch.setattr(hardware, "_overlay_system_wide_vram", lambda devices: None)
# No KFD topology, so only amd-smi + torch can answer.
monkeypatch.setattr(hardware, "_rocm_kfd_gpu_pci_ids", lambda: [])
return touched
def test_h3_card_probes_read_nothing_on_rocm_without_a_context(monkeypatch):
touched = _h3_telemetry(monkeypatch, rocm = True, smi = True)
assert video_mod._h3_card_memory_bytes("cuda", 0, allow_context = False) == (None, None)
assert video_mod._h3_visible_card_count("cuda") == 0
assert touched == []
_ROCM_MASKS = (
"HIP_VISIBLE_DEVICES",
"ROCR_VISIBLE_DEVICES",
"CUDA_VISIBLE_DEVICES",
"GPU_DEVICE_ORDINAL",
)
def _rocm_linux_host(
monkeypatch,
cards,
boards = None,
):
"""``cards`` = [(pci, gfx target, (used_gb, total_gb) or None)] in KFD order; torch and amd-smi are tripwires."""
from utils.hardware import amd, hardware
for name in _ROCM_MASKS:
monkeypatch.delenv(name, raising = False)
monkeypatch.setattr(hardware, "get_device", lambda: hardware.DeviceType.CUDA)
monkeypatch.setattr(hardware, "IS_ROCM", True)
monkeypatch.setattr(hardware.platform, "system", lambda: "Linux")
monkeypatch.setattr(hardware, "_rocm_kfd_gpu_pci_ids", lambda: [c[0] for c in cards])
monkeypatch.setattr(amd, "amd_kfd_gpu_gfx_targets", lambda *a, **k: [c[1] for c in cards])
monkeypatch.setattr(
hardware,
"_rocm_linux_sysfs_vram_by_pci_gb",
lambda: {c[0]: c[2] for c in cards if c[2] is not None},
)
monkeypatch.setattr(
hardware,
"_rocm_linux_amdgpu_cards",
lambda: [
(bdf, i, f"/sys/class/drm/card{i}/device")
for i, bdf in enumerate(boards or dict.fromkeys(c[0] for c in cards))
],
)
def _no_torch(*a, **k):
pytest.fail("the no-context probe touched torch")
import torch
for name in ("mem_get_info", "get_device_properties", "init", "_lazy_init", "device_count"):
monkeypatch.setattr(torch.cuda, name, _no_torch)
for name in (
"_torch_get_device_module",
"_torch_get_per_device_info",
"_torch_get_device_inventory",
"_torch_get_physical_gpu_count",
"_smi_visible_utilization",
):
monkeypatch.setattr(hardware, name, _no_torch)
def test_h3_card_probes_read_rocm_sysfs_without_a_context(monkeypatch):
_rocm_linux_host(monkeypatch, [("0000:03:00.0", "gfx1100", (4.0, 24.0))])
assert video_mod._h3_card_memory_bytes("cuda", 0, allow_context = False) == (20 * GiB, 24 * GiB)
assert video_mod._h3_visible_card_count("cuda") == 1
assert (
video_mod._h3_card_free_bytes("cuda", 0, single_card_only = True, allow_context = False)
== 20 * GiB
)
def test_h3_card_probes_on_masked_rocm_read_nothing(monkeypatch):
_rocm_linux_host(monkeypatch, [("0000:03:00.0", "gfx1100", (4.0, 24.0))])
assert video_mod._h3_card_memory_bytes("cuda", 0, allow_context = False) == (20 * GiB, 24 * GiB)
for name in _ROCM_MASKS:
monkeypatch.setenv(name, "0")
assert video_mod._h3_card_memory_bytes("cuda", 0, allow_context = False) == (None, None), name
assert video_mod._h3_visible_card_count("cuda") == 0, name
monkeypatch.delenv(name)
def test_h3_card_probes_never_report_an_apu_carve_out(monkeypatch):
_rocm_linux_host(
monkeypatch,
[
("0000:03:00.0", "gfx1100", (4.0, 24.0)),
("0000:c5:00.0", "gfx1151", (1.0, 0.5)),
("0000:0b:00.0", "gfx942", (1.0, 192.0)),
],
)
assert video_mod._h3_card_memory_bytes("cuda", 0, allow_context = False) == (20 * GiB, 24 * GiB)
assert video_mod._h3_card_memory_bytes("cuda", 1, allow_context = False) == (None, None)
assert video_mod._h3_card_memory_bytes("cuda", 2, allow_context = False) == (None, None)
assert video_mod._h3_visible_card_count("cuda") == 3
# A partitioned card (two ordinals on one PCI address) reads nothing.
_rocm_linux_host(
monkeypatch,
[("0000:03:00.0", "gfx950", (4.0, 24.0)), ("0000:03:00.0", "gfx950", (4.0, 24.0))],
)
assert video_mod._h3_card_memory_bytes("cuda", 0, allow_context = False) == (None, None)
assert video_mod._h3_visible_card_count("cuda") == 0
def test_h3_card_probes_never_report_a_partition_the_whole_board(monkeypatch):
# MI355X DPX: partition 1 decodes to 03:00.1 (no amdgpu device); partition 0 must not read the whole board.
_rocm_linux_host(
monkeypatch,
[("0000:03:00.0", "gfx950", (8.0, 288.0)), ("0000:03:00.1", "gfx950", None)],
boards = ["0000:03:00.0"],
)
assert video_mod._h3_card_memory_bytes("cuda", 0, allow_context = False) == (None, None)
assert video_mod._h3_visible_card_count("cuda") == 0
# The same board unpartitioned (SPX) still reads.
_rocm_linux_host(monkeypatch, [("0000:03:00.0", "gfx950", (8.0, 288.0))])
assert video_mod._h3_card_memory_bytes("cuda", 0, allow_context = False) == (280 * GiB, 288 * GiB)
def test_h3_card_probes_on_cuda_still_read_nvidia_smi(monkeypatch):
from utils.hardware import hardware
monkeypatch.setattr(hardware, "get_device", lambda: hardware.DeviceType.CUDA)
monkeypatch.setattr(hardware, "IS_ROCM", False)
monkeypatch.setattr(
hardware,
"_get_parent_visible_gpu_spec",
lambda: {"raw": None, "numeric_ids": None, "supports_explicit_gpu_ids": True},
)
monkeypatch.setattr(hardware, "_reconcile_cuda_integrated_memory", lambda *a, **k: None)
monkeypatch.setattr(
hardware,
"_smi_visible_utilization",
lambda spec: {
"available": True,
"devices": [
{"index": 0, "visible_ordinal": 0, "vram_used_gb": 2.0, "vram_total_gb": 16.0}
],
"index_kind": "relative",
"parent_visible_gpu_ids": [],
},
)
monkeypatch.setattr(
hardware,
"_rocm_context_free_visible_devices",
lambda: pytest.fail("ROCm sysfs read on a CUDA host"),
raising = False,
)
assert video_mod._h3_card_memory_bytes("cuda", 0, allow_context = False) == (14 * GiB, 16 * GiB)
assert video_mod._h3_visible_card_count("cuda") == 1
def test_h3_residency_reads_fall_back_to_torch_on_rocm(monkeypatch):
# The residency and pressure reads may create a context, as they did before the launch budget.
_h3_telemetry(monkeypatch, rocm = True, smi = True)
assert video_mod._h3_card_memory_bytes("cuda", 0) == (40 * GiB, 48 * GiB)
@pytest.mark.parametrize(
"rocm, smi", [(True, True), (False, False)], ids = ["rocm_amd_smi", "nvidia_no_smi"]
)
def test_h3_auto_runs_resident_when_the_card_fits(monkeypatch, rocm, smi):
"""48 GiB total, 40 free, 39 needed: Auto drops the streaming flags, as it did before the launch budget."""
_h3_telemetry(monkeypatch, rocm = rocm, smi = smi)
flags, resident = h3.h3_native_render_flags(
_H3_FLAGS,
memory_mode = "auto",
free_bytes = video_mod._h3_card_free_bytes("cuda", 0),
need_bytes = 39 * GiB,
env = {},
)
assert resident is True
assert "--offload-to-cpu" not in flags and "--stream-layers" not in flags
def test_a_slot_start_records_what_it_attempted_even_when_it_fails(monkeypatch):
class _Srv:
def __init__(self, binary):
pass
def start(
self,
files,
*,
offload = None,
threads = None,
env = None,
extra_args = None,
):
raise RuntimeError(_AUDIT_OOM)
def is_alive(self):
return False
def stop(self):
pass
monkeypatch.setattr(srv, "SdCppServer", _Srv)
monkeypatch.setattr("core.inference.sd_cpp_engine.is_managed_binary", lambda b: False)
slot = h3.H3NativeServerSlot("/x/sd-server", _FILES, _COMMITTED)
with pytest.raises(RuntimeError):
slot.get(_COMMITTED, launch_transform = lambda f: [*f[:3], "8.00", *f[4:]])
assert slot.launched_flags is None and slot.attempted_flags[3] == "8.00"
def test_server_render_does_not_retry_other_failures_or_auto_budget(tmp_path, monkeypatch):
monkeypatch.delenv(h3.H3_NATIVE_SERVER_ENV, raising = False)
slot = _Slot(_Server([RuntimeError("sd-server generation failed: bad prompt")]))
with pytest.raises(RuntimeError, match = "bad prompt"):
_server_render(slot, int(10 * GiB), tmp_path)
assert len(slot.launches) == 1
# Unreadable card: "-1" as before, and an OOM under it is not retried here (nothing smaller to try).
slot = _Slot(_Server([RuntimeError(_AUDIT_OOM)]))
with pytest.raises(RuntimeError, match = "out of memory"):
_server_render(slot, None, tmp_path)
assert [launch[3] for launch in slot.launches] == ["-1"]
def test_one_shot_retries_once_smaller_on_device_oom():
rt = _Rt(int(12 * GiB))
seen = []
def run(flags):
seen.append(flags[flags.index("--max-vram") + 1])
if len(seen) == 1:
raise RuntimeError("sd-cli exited 1. Last output:\n" + _AUDIT_OOM)
return "/out.avi"
assert (
video_mod._h3_one_shot_with_vram_retry(rt, run, _COMMITTED, threading.Event()) == "/out.avi"
)
assert seen == ["10.00", "7.50"]
def bad(flags):
raise RuntimeError("sd-cli exited 1. Last output:\n[E] unknown argument")
with pytest.raises(RuntimeError, match = "unknown argument"):
video_mod._h3_one_shot_with_vram_retry(rt, bad, _COMMITTED, threading.Event())
def _load(monkeypatch, tmp_path, **kw):
from test_video_backend import _SAGE_HELP, _load_h3_native_offload
return _load_h3_native_offload(monkeypatch, tmp_path, help_text = _SAGE_HELP, **kw)
def test_load_commits_the_placeholder_and_a_probe(monkeypatch, tmp_path):
from test_video_backend import _GRAPH_CUT_HELP, _load_h3_native_offload
reads = []
monkeypatch.setattr(
video_mod,
"_h3_card_free_bytes",
lambda device, ordinal, **kw: reads.append((device, ordinal)) or int(9 * GiB),
)
state, offload = _load_h3_native_offload(monkeypatch, tmp_path, help_text = _GRAPH_CUT_HELP)
assert offload[offload.index("--max-vram") + 1] == "-1"
probe = state.pipe.vram_free_probe
assert probe is not None
assert (
video_mod._h3_budgeted_launch_flags(state.pipe, offload)[offload.index("--max-vram") + 1]
== "7.00"
)
assert reads and reads[-1][0] == "cuda"
@pytest.mark.parametrize(
"torch_cc, reason",
[
((10, 0), "sd.cpp BF16 cuBLAS matmuls"),
((8, 9), "sd.cpp BF16 cuBLAS matmuls"),
((7, 5), "sd.cpp exact kernels"),
],
)
def test_unreadable_listing_falls_back_to_torch_cc(monkeypatch, tmp_path, torch_cc, reason):
"""--list-devices unreadable (timed out / non-zero exit): nothing is cached, the verdict keeps the GPU, and the
capability comes from torch instead of silently routing a sm80+ card to the exact kernels."""
monkeypatch.delenv("UNSLOTH_H3_SAGE_ATTN", raising = False)
monkeypatch.delenv("GGML_CUDA_QUANT_CUBLAS_MIN_BATCH", raising = False)
monkeypatch.setattr(sd_cpp_backend, "_LAST_DEVICE_LISTING", {})
monkeypatch.setattr(sd_cpp_backend, "_remember_device_listing", lambda *_a: None)
asked = []
monkeypatch.setattr(
sd_cpp_backend,
"torch_cuda_compute_capability",
lambda ordinal: asked.append(ordinal) or torch_cc,
)
state, _offload = _load(monkeypatch, tmp_path, devices = "", installed_accelerator = "cuda")
assert asked, "torch was not consulted"
assert state.resolved["speed_mode"]["reason"] == reason
assert (dict(state.pipe.env).get("GGML_CUDA_QUANT_CUBLAS_MIN_BATCH") is not None) == (
torch_cc >= (8, 0)
)
def test_base_behaviour_without_fallback_is_exact_kernels(monkeypatch, tmp_path):
"""The defect itself: the same unreadable listing with torch giving no answer routes to the exact kernels."""
monkeypatch.delenv("UNSLOTH_H3_SAGE_ATTN", raising = False)
monkeypatch.delenv("GGML_CUDA_QUANT_CUBLAS_MIN_BATCH", raising = False)
monkeypatch.setattr(sd_cpp_backend, "_LAST_DEVICE_LISTING", {})
monkeypatch.setattr(sd_cpp_backend, "_remember_device_listing", lambda *_a: None)
monkeypatch.setattr(sd_cpp_backend, "torch_cuda_compute_capability", lambda ordinal: None)
state, _offload = _load(monkeypatch, tmp_path, devices = "", installed_accelerator = "cuda")
assert state.resolved["speed_mode"]["reason"] == "sd.cpp exact kernels"
def test_torch_is_not_consulted_when_the_listing_was_read(monkeypatch, tmp_path):
from test_video_backend import _cuda_devices
monkeypatch.delenv("GGML_CUDA_QUANT_CUBLAS_MIN_BATCH", raising = False)
monkeypatch.setattr(sd_cpp_backend, "_LAST_DEVICE_LISTING", {})
monkeypatch.setattr(
sd_cpp_backend,
"torch_cuda_compute_capability",
lambda ordinal: pytest.fail("torch consulted"),
)
state, _ = _load(
monkeypatch, tmp_path, devices = _cuda_devices("7.5"), installed_accelerator = "cuda"
)
assert state.resolved["speed_mode"]["reason"] == "sd.cpp exact kernels"
def test_cc_fallback_rules(monkeypatch):
monkeypatch.setattr(sd_cpp_backend, "_LAST_DEVICE_LISTING", {})
monkeypatch.setattr(
sd_cpp_backend,
"torch_cuda_compute_capability",
lambda ordinal: (9, 0) if ordinal in (0, 1) else None,
)
kw = dict(binary = "/x/sd-cli", native_device = "cuda", listed_accelerator = True)
fb = video_mod._h3_cuda_cc_fallback
assert fb((8, 6), accelerator_names = ("cuda",), device_name = "CUDA0", **kw) == (8, 6)
assert fb(None, accelerator_names = ("cuda",), device_name = "CUDA1", **kw) == (9, 0)
assert fb(None, accelerator_names = ("cuda",), device_name = None, **kw) == (9, 0)
# Not a CUDA build, not a CUDA device name, a CPU target: unknown stays unknown.
assert fb(None, accelerator_names = ("vulkan",), device_name = None, **kw) is None
assert fb(None, accelerator_names = ("rocm",), device_name = "ROCm0", **kw) is None
assert fb(None, accelerator_names = ("cuda",), device_name = "Vulkan0", **kw) is None
assert (
fb(
None,
binary = "/x/sd-cli",
native_device = "cpu",
listed_accelerator = True,
accelerator_names = ("cuda",),
device_name = None,
)
is None
)
def test_cc_fallback_reads_the_card_sd_cpp_runs_on_and_only_an_owned_cuda_build(monkeypatch):
monkeypatch.setattr(sd_cpp_backend, "_LAST_DEVICE_LISTING", {})
asked = []
monkeypatch.setattr(
sd_cpp_backend,
"torch_cuda_compute_capability",
lambda ordinal: asked.append(ordinal) or {0: (7, 5), 1: (8, 9)}.get(ordinal),
)
kw = dict(binary = "/x/sd-cli", native_device = "cuda", listed_accelerator = True)
fb = video_mod._h3_cuda_cc_fallback
# No device name means no --backend pin: sd.cpp renders on CUDA0, so that card is read (not the lowest across cards).
assert fb(None, accelerator_names = ("cuda",), device_name = None, **kw) == (7, 5)
assert asked == [0]
assert fb(None, accelerator_names = ("cuda",), device_name = "CUDA1", **kw) == (8, 9)
assert asked == [0, 1]
# No install record (an SD_CLI_PATH build): its accelerator is unknown, so no CUDA route from torch.
assert fb(None, accelerator_names = (None,), device_name = None, **kw) is None
# The requested ordinal is not an input: it names a card sd.cpp only uses through a resolved pin.
import inspect
assert "native_ordinal" not in inspect.signature(fb).parameters
def test_a_stalled_torch_capability_reading_is_bounded(monkeypatch):
import time
monkeypatch.setattr(
sd_cpp_backend, "_torch_cuda_compute_capability", lambda ordinal: time.sleep(5) or (9, 0)
)
t0 = time.monotonic()
assert sd_cpp_backend.torch_cuda_compute_capability(0, timeout_s = 0.2) is None
assert time.monotonic() - t0 < 2.0
monkeypatch.setattr(sd_cpp_backend, "_torch_cuda_compute_capability", lambda ordinal: (9, 0))
assert sd_cpp_backend.torch_cuda_compute_capability(0) == (9, 0)
def test_an_unpinned_render_on_several_cards_takes_no_scalar_budget(monkeypatch):
monkeypatch.setattr(
video_mod,
"_h3_card_memory_bytes",
lambda device, ordinal, **_k: (int(9 * GiB), int(24 * GiB)),
)
monkeypatch.setattr(video_mod, "_h3_visible_card_count", lambda device: 2)
assert video_mod._h3_card_free_bytes("cuda", None, single_card_only = True) is None
monkeypatch.setattr(video_mod, "_h3_visible_card_count", lambda device: 1)
assert video_mod._h3_card_free_bytes("cuda", None, single_card_only = True) == int(9 * GiB)
monkeypatch.setattr(video_mod, "_h3_visible_card_count", lambda device: 2)
assert video_mod._h3_card_free_bytes("cuda", 1) == int(9 * GiB)
def test_torch_cc_helper_is_none_off_cuda(monkeypatch):
import torch
monkeypatch.setattr(torch.version, "cuda", None, raising = False)
assert sd_cpp_backend.torch_cuda_compute_capability(0) is None
monkeypatch.setattr(torch.version, "cuda", "13.0", raising = False)
monkeypatch.setattr(torch.version, "hip", "7.0", raising = False)
assert sd_cpp_backend.torch_cuda_compute_capability(0) is None
def test_h3_card_probes_never_fall_back_to_a_torch_context(monkeypatch):
from utils.hardware import hardware
monkeypatch.setattr(hardware, "get_device", lambda: hardware.DeviceType.CUDA)
monkeypatch.setattr(hardware, "_smi_visible_utilization", lambda spec: None)
monkeypatch.setattr(hardware, "IS_ROCM", False)
monkeypatch.setattr(hardware, "get_parent_visible_gpu_ids", lambda: [0])
touched = []
monkeypatch.setattr(
hardware, "_torch_get_per_device_info", lambda idx: touched.append(idx) or []
)
assert video_mod._h3_visible_card_count("cuda") == 0
assert video_mod._h3_card_memory_bytes("cuda", 0, allow_context = False) == (None, None)
assert video_mod._h3_card_free_bytes("cuda", 0, allow_context = False) is None
assert touched == []
def test_image_vram_refusal_text_is_the_model_manager_declining_a_segment():
segment = (
"[E] MOSS-Image segment 1/43 (moss_image.layers.0) failed during {} --- ggml_runner.cpp:934"
)
for stage in ("weight preparation", "workspace capacity check", "allocated capacity check"):
assert args_mod.is_vram_refusal_text(segment.format(stage))
# The model manager's own warning beside it is not an allocation failure.
assert args_mod.is_vram_refusal_text(
"[W] model manager cannot make enough memory available on CUDA0: need 512.63 MB device / 0.63 MB "
"budget, available 0.00 MB device / 5549.38 MB budget --- model_manager.cpp:2101\n"
+ segment.format("weight preparation")
)
assert not args_mod.is_vram_refusal_text("generation_failed: invalid sampler")
assert not args_mod.is_vram_refusal_text(None)
def test_image_vram_refusal_text_leaves_failed_allocations_alone():
segment = "[E] MOSS-Image segment 12/43 (moss_image.layers.11) failed during {} --- ggml_runner.cpp:934"
allocator = (
"[E] ggml: ggml_backend_cuda_buffer_type_alloc_buffer: allocating 210.08 MiB on device 0: "
"cudaMalloc failed: out of memory\n"
"[E] ggml: alloc_tensor_range: failed to allocate CUDA0 buffer of size 220288512\n"
)
assert not args_mod.is_vram_refusal_text(segment.format("workspace allocation"))
assert not args_mod.is_vram_refusal_text(allocator + segment.format("workspace allocation"))
assert not args_mod.is_vram_refusal_text(allocator + segment.format("weight preparation"))
assert not args_mod.is_vram_refusal_text(
"vk::Device::allocateMemory: ErrorOutOfDeviceMemory\n"
+ segment.format("weight preparation")
)
assert not args_mod.is_vram_refusal_text(_AUDIT_OOM)
for host in (
"[E] ggml: ggml_backend_cpu_buffer_type_alloc_buffer: failed to allocate buffer of size 220288512\n",
"[E] model manager alloc params backend buffer failed, size = 210.08MB --- model_manager.cpp:964\n",
"[E] vk::Device::allocateMemory: ErrorOutOfHostMemory\n",
"[E] ggml_vulkan: host malloc failed\n",
):
assert not args_mod.is_vram_refusal_text(host + segment.format("weight preparation"))
assert args_mod.is_device_oom_text(_AUDIT_OOM)
assert args_mod.is_device_oom_text(segment.format("workspace allocation"))
assert not args_mod.is_device_oom_text(segment.format("weight preparation"))
def test_image_vram_retry_budget_is_half_the_planned_free_vram():
assert args_mod.image_vram_retry_budget({"CUDA0": 6666}) == "3.25"
assert args_mod.image_vram_retry_budget({"Vulkan0": 24176}) == "11.80"
assert args_mod.image_vram_retry_budget({"CUDA0": 2048}) == "1.00"
assert args_mod.image_vram_retry_budget({"CUDA0": 2047}) is None
assert args_mod.image_vram_retry_budget({"CUDA0": 0}) is None
assert args_mod.image_vram_retry_budget({}) is None
assert args_mod.image_vram_retry_budget(None) is None
def test_image_vram_retry_budget_is_per_card_and_needs_the_render_cards():
plan = {"CUDA0": 45979, "CUDA1": 5771}
assert args_mod.image_vram_retry_budget(plan) == "CUDA0=22.45,CUDA1=2.81"
assert args_mod.image_vram_retry_budget(plan, {"cuda1"}) == "CUDA0=22.45,CUDA1=2.81"
# A card under the floor is left out, unless the render is pinned to it or it is missing from the plan.
assert (
args_mod.image_vram_retry_budget({"CUDA0": 45979, "CUDA1": 1800}, {"cuda0"})
== "CUDA0=22.45"
)
assert args_mod.image_vram_retry_budget({"CUDA0": 45979, "CUDA1": 1800}, {"cuda1"}) is None
assert args_mod.image_vram_retry_budget({"CUDA0": 45979}, {"cuda1"}) is None
def test_planned_free_vram_is_read_from_the_auto_fit_device_lines():
plan = (
"[I] auto-fit plan: --- backend_fit.cpp:334",
"[I] devices: --- backend_fit.cpp:335",
"[I] CUDA0 NVIDIA RTX 6000 Ada Generation free 45979 MiB, budget 45467 MiB --- backend_fit.cpp:337",
"[I] CUDA1 NVIDIA GeForce RTX 3090 free 5771 MiB, budget 5259 MiB --- backend_fit.cpp:337",
"[I] RAM free 69220 MiB, params budget 62298 MiB --- backend_fit.cpp:344",
"[I] DiT params 5228 MiB, compute reserve 2048 MiB -> compute CUDA0, params CUDA0 --- backend_fit.cpp:357",
)
found = {}
for line in plan:
match = srv._AUTO_FIT_DEVICE_FREE_RE.search(line)
if match is not None:
found[match.group(1)] = int(match.group(2))
assert found == {"CUDA0": 45979, "CUDA1": 5771}