Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio H3: let Auto residency see the card again on ROCm and NVIDIA without nvidia-smi (#14820)
Daniel Hanb8513d2efa35
diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.pyindex f777a5f3923..81302a7e8c5 100644--- a/studio/backend/core/inference/video.py+++ b/studio/backend/core/inference/video.py@@ -2527,17 +2527,30 @@ def _h3_free_device_bytes(device: str) -> Optional[int]:def _h3_card_memory_bytes(- device: Optional[str], ordinal: Optional[int]+ device: Optional[str],+ ordinal: Optional[int],+ *,+ allow_context: bool = True,) -> tuple[Optional[int], Optional[int]]:- """(free, total) VRAM of the H3 render card via nvidia-smi / amd-smi (no CUDA context), or (None, None). No pin =- ordinal 0."""+ """(free, total) VRAM of the H3 render card, or (None, None). No pin = ordinal 0. nvidia-smi / amd-smi / sysfs+ first (no CUDA context); torch only when none answers and ``allow_context``, as the residency and pressure reads+ always did. The pre-spawn budget passes False: it never pays for a context."""if device != "cuda":return None, None+ found = _h3_read_card_memory_bytes(ordinal, no_context = True)+ if found == (None, None) and allow_context:+ found = _h3_read_card_memory_bytes(ordinal, no_context = False)+ return found+++def _h3_read_card_memory_bytes(+ ordinal: Optional[int], *, no_context: bool+) -> tuple[Optional[int], Optional[int]]:try:from utils.hardware import get_visible_gpu_utilization, gpu_querywith gpu_query.fresh_reads():- devices = get_visible_gpu_utilization(no_context = True).get("devices", [])+ devices = get_visible_gpu_utilization(no_context = no_context).get("devices", [])want = 0 if ordinal is None else int(ordinal)for entry in devices:if entry.get("visible_ordinal") != want:@@ -2558,12 +2571,14 @@ def _h3_card_free_bytes(ordinal: Optional[int],*,single_card_only: bool = False,+ allow_context: bool = True,) -> Optional[int]:"""Free VRAM of the H3 render card, or None. ``single_card_only``: None on multi-card boxes, where one card's- scalar budget applied to every card would steer sd.cpp's own device pick."""+ scalar budget applied to every card would steer sd.cpp's own device pick. ``allow_context``: see+ ``_h3_card_memory_bytes``."""if single_card_only and _h3_visible_card_count(device) != 1:return None- return _h3_card_memory_bytes(device, ordinal)[0]+ return _h3_card_memory_bytes(device, ordinal, allow_context = allow_context)[0]def _h3_visible_card_count(device: Optional[str]) -> Optional[int]:@@ -5626,6 +5641,7 @@ class VideoBackend:native_device,native_ordinal if native_device_name else None,single_card_only = not native_device_name,+ allow_context = False,)if native_device != "cpu"else Nonediff --git a/studio/backend/tests/test_sd_cpp_vram_budget.py b/studio/backend/tests/test_sd_cpp_vram_budget.pyindex 0a09d9c44b5..5c62043962a 100644--- a/studio/backend/tests/test_sd_cpp_vram_budget.py+++ b/studio/backend/tests/test_sd_cpp_vram_budget.py@@ -489,18 +489,72 @@ def test_restart_for_retry_keeps_a_release_that_landed_before_the_stop(monkeypatassert slot.end_render() is None and slot.release_pending is None-def test_h3_card_probes_read_nothing_on_rocm_without_a_context(monkeypatch):+_H3_FLAGS = ("--offload-to-cpu", "--diffusion-fa", "--max-vram", "-1", "--stream-layers")+++def _h3_telemetry(monkeypatch, *, rocm, smi):+ """48 GiB card with 8 used: amd-smi / nvidia-smi rows when ``smi``, else only torch can read it."""from utils.hardware import hardware+ touched = []monkeypatch.setattr(hardware, "get_device", lambda: hardware.DeviceType.CUDA)- monkeypatch.setattr(hardware, "IS_ROCM", True)+ monkeypatch.setattr(hardware, "IS_ROCM", rocm)+ monkeypatch.setattr(+ hardware, "_get_parent_visible_gpu_spec", lambda: {"raw": "0", "numeric_ids": [0]}+ )+ monkeypatch.setattr(hardware, "get_parent_visible_gpu_ids", lambda: [0])+ row = {"index": 0, "visible_ordinal": 0, "vram_total_gb": 48.0, "vram_used_gb": 8.0}monkeypatch.setattr(hardware,"_smi_visible_utilization",- lambda spec: pytest.fail("amd-smi rows need a torch context"),+ lambda spec: {"available": True, "devices": [dict(row)], "index_kind": "physical"}+ if smi+ else None,+ )+ monkeypatch.setattr(+ hardware,+ "_reconcile_rocm_unified_memory",+ lambda *_a: touched.append("reconcile"),)- assert video_mod._h3_card_memory_bytes("cuda", 0) == (None, None)++ def per_device(idx):+ touched.append("torch")+ return [{"index": 0, "visible_ordinal": 0, "total_gb": 48.0, "used_gb": 8.0}]++ monkeypatch.setattr(hardware, "_torch_get_per_device_info", per_device)+ monkeypatch.setattr(hardware, "_overlay_system_wide_vram", lambda devices: None)+ return touched+++def test_h3_card_probes_read_nothing_on_rocm_without_a_context(monkeypatch):+ # The pre-spawn budget never creates a context, and ROCm's amd-smi rows need one.+ touched = _h3_telemetry(monkeypatch, rocm = True, smi = True)+ assert video_mod._h3_card_memory_bytes("cuda", 0, allow_context = False) == (None, None)assert video_mod._h3_visible_card_count("cuda") == 0+ assert touched == []+++def test_h3_residency_reads_fall_back_to_torch_on_rocm(monkeypatch):+ # The residency and pressure reads may create a context, as they did before the launch budget.+ _h3_telemetry(monkeypatch, rocm = True, smi = True)+ assert video_mod._h3_card_memory_bytes("cuda", 0) == (40 * GiB, 48 * GiB)+++ "rocm, smi", [(True, True), (False, False)], ids = ["rocm_amd_smi", "nvidia_no_smi"]+)+def test_h3_auto_runs_resident_when_the_card_fits(monkeypatch, rocm, smi):+ """48 GiB total, 40 free, 39 needed: Auto drops the streaming flags, as it did before the launch budget."""+ _h3_telemetry(monkeypatch, rocm = rocm, smi = smi)+ flags, resident = h3.h3_native_render_flags(+ _H3_FLAGS,+ memory_mode = "auto",+ free_bytes = video_mod._h3_card_free_bytes("cuda", 0),+ need_bytes = 39 * GiB,+ env = {},+ )+ assert resident is True+ assert "--offload-to-cpu" not in flags and "--stream-layers" not in flagsdef test_a_slot_start_records_what_it_attempted_even_when_it_fails(monkeypatch):@@ -716,7 +770,9 @@ def test_a_stalled_torch_capability_reading_is_bounded(monkeypatch):def test_an_unpinned_render_on_several_cards_takes_no_scalar_budget(monkeypatch):monkeypatch.setattr(- video_mod, "_h3_card_memory_bytes", lambda device, ordinal: (int(9 * GiB), int(24 * GiB))+ video_mod,+ "_h3_card_memory_bytes",+ lambda device, ordinal, **_k: (int(9 * GiB), int(24 * GiB)),)monkeypatch.setattr(video_mod, "_h3_visible_card_count", lambda device: 2)assert video_mod._h3_card_free_bytes("cuda", None, single_card_only = True) is None@@ -748,5 +804,6 @@ def test_h3_card_probes_never_fall_back_to_a_torch_context(monkeypatch):hardware, "_torch_get_per_device_info", lambda idx: touched.append(idx) or [])assert video_mod._h3_visible_card_count("cuda") == 0- assert video_mod._h3_card_memory_bytes("cuda", 0) == (None, None)+ assert video_mod._h3_card_memory_bytes("cuda", 0, allow_context = False) == (None, None)+ assert video_mod._h3_card_free_bytes("cuda", 0, allow_context = False) is Noneassert touched == []