Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: read MiniMax-H3 card VRAM on ROCm without a torch context (#14732)

Daniel Han97a4208b6c78
diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.py
index eff8afd8e8f..3331afbbc1f 100644
--- a/studio/backend/core/inference/video.py
+++ b/studio/backend/core/inference/video.py
@@ -2694,7 +2694,7 @@ def _h3_card_memory_bytes(
*,
allow_context: bool = True,
) -> tuple[Optional[int], Optional[int]]:
- """(free, total) VRAM of the H3 render card, or (None, None). No pin = ordinal 0. nvidia-smi / amd-smi / sysfs
+ """(free, total) VRAM of the H3 render card, or (None, None). No pin = ordinal 0. nvidia-smi / ROCm KFD + sysfs
first (no CUDA context); torch only when none answers and ``allow_context``, as the residency and pressure reads
always did. The pre-spawn budget passes False: it never pays for a context. Free is less the user's VRAM Budget
reserve (raw at the default), so the sd.cpp resident-vs-offload pick, its ``--max-vram`` probe and the
diff --git a/studio/backend/tests/test_sd_cpp_vram_budget.py b/studio/backend/tests/test_sd_cpp_vram_budget.py
index 5c62043962a..11b0db90636 100644
--- a/studio/backend/tests/test_sd_cpp_vram_budget.py
+++ b/studio/backend/tests/test_sd_cpp_vram_budget.py
@@ -523,17 +523,161 @@ def _h3_telemetry(monkeypatch, *, rocm, smi):
monkeypatch.setattr(hardware, "_torch_get_per_device_info", per_device)
monkeypatch.setattr(hardware, "_overlay_system_wide_vram", lambda devices: None)
+ # No KFD topology, so only amd-smi + torch can answer.
+ monkeypatch.setattr(hardware, "_rocm_kfd_gpu_pci_ids", lambda: [])
return touched
def test_h3_card_probes_read_nothing_on_rocm_without_a_context(monkeypatch):
- # The pre-spawn budget never creates a context, and ROCm's amd-smi rows need one.
touched = _h3_telemetry(monkeypatch, rocm = True, smi = True)
assert video_mod._h3_card_memory_bytes("cuda", 0, allow_context = False) == (None, None)
assert video_mod._h3_visible_card_count("cuda") == 0
assert touched == []
+_ROCM_MASKS = (
+ "HIP_VISIBLE_DEVICES",
+ "ROCR_VISIBLE_DEVICES",
+ "CUDA_VISIBLE_DEVICES",
+ "GPU_DEVICE_ORDINAL",
+)
+
+
+def _rocm_linux_host(
+ monkeypatch,
+ cards,
+ boards = None,
+):
+ """``cards`` = [(pci, gfx target, (used_gb, total_gb) or None)] in KFD order; torch and amd-smi are tripwires."""
+ from utils.hardware import amd, hardware
+
+ for name in _ROCM_MASKS:
+ monkeypatch.delenv(name, raising = False)
+ monkeypatch.setattr(hardware, "get_device", lambda: hardware.DeviceType.CUDA)
+ monkeypatch.setattr(hardware, "IS_ROCM", True)
+ monkeypatch.setattr(hardware.platform, "system", lambda: "Linux")
+ monkeypatch.setattr(hardware, "_rocm_kfd_gpu_pci_ids", lambda: [c[0] for c in cards])
+ monkeypatch.setattr(amd, "amd_kfd_gpu_gfx_targets", lambda *a, **k: [c[1] for c in cards])
+ monkeypatch.setattr(
+ hardware,
+ "_rocm_linux_sysfs_vram_by_pci_gb",
+ lambda: {c[0]: c[2] for c in cards if c[2] is not None},
+ )
+ monkeypatch.setattr(
+ hardware,
+ "_rocm_linux_amdgpu_cards",
+ lambda: [
+ (bdf, i, f"/sys/class/drm/card{i}/device")
+ for i, bdf in enumerate(boards or dict.fromkeys(c[0] for c in cards))
+ ],
+ )
+
+ def _no_torch(*a, **k):
+ pytest.fail("the no-context probe touched torch")
+
+ import torch
+
+ for name in ("mem_get_info", "get_device_properties", "init", "_lazy_init", "device_count"):
+ monkeypatch.setattr(torch.cuda, name, _no_torch)
+ for name in (
+ "_torch_get_device_module",
+ "_torch_get_per_device_info",
+ "_torch_get_device_inventory",
+ "_torch_get_physical_gpu_count",
+ "_smi_visible_utilization",
+ ):
+ monkeypatch.setattr(hardware, name, _no_torch)
+
+
+def test_h3_card_probes_read_rocm_sysfs_without_a_context(monkeypatch):
+ _rocm_linux_host(monkeypatch, [("0000:03:00.0", "gfx1100", (4.0, 24.0))])
+ assert video_mod._h3_card_memory_bytes("cuda", 0, allow_context = False) == (20 * GiB, 24 * GiB)
+ assert video_mod._h3_visible_card_count("cuda") == 1
+ assert (
+ video_mod._h3_card_free_bytes("cuda", 0, single_card_only = True, allow_context = False)
+ == 20 * GiB
+ )
+
+
+def test_h3_card_probes_on_masked_rocm_read_nothing(monkeypatch):
+ _rocm_linux_host(monkeypatch, [("0000:03:00.0", "gfx1100", (4.0, 24.0))])
+ assert video_mod._h3_card_memory_bytes("cuda", 0, allow_context = False) == (20 * GiB, 24 * GiB)
+ for name in _ROCM_MASKS:
+ monkeypatch.setenv(name, "0")
+ assert video_mod._h3_card_memory_bytes("cuda", 0, allow_context = False) == (None, None), name
+ assert video_mod._h3_visible_card_count("cuda") == 0, name
+ monkeypatch.delenv(name)
+
+
+def test_h3_card_probes_never_report_an_apu_carve_out(monkeypatch):
+ _rocm_linux_host(
+ monkeypatch,
+ [
+ ("0000:03:00.0", "gfx1100", (4.0, 24.0)),
+ ("0000:c5:00.0", "gfx1151", (1.0, 0.5)),
+ ("0000:0b:00.0", "gfx942", (1.0, 192.0)),
+ ],
+ )
+ assert video_mod._h3_card_memory_bytes("cuda", 0, allow_context = False) == (20 * GiB, 24 * GiB)
+ assert video_mod._h3_card_memory_bytes("cuda", 1, allow_context = False) == (None, None)
+ assert video_mod._h3_card_memory_bytes("cuda", 2, allow_context = False) == (None, None)
+ assert video_mod._h3_visible_card_count("cuda") == 3
+ # A partitioned card (two ordinals on one PCI address) reads nothing.
+ _rocm_linux_host(
+ monkeypatch,
+ [("0000:03:00.0", "gfx950", (4.0, 24.0)), ("0000:03:00.0", "gfx950", (4.0, 24.0))],
+ )
+ assert video_mod._h3_card_memory_bytes("cuda", 0, allow_context = False) == (None, None)
+ assert video_mod._h3_visible_card_count("cuda") == 0
+
+
+def test_h3_card_probes_never_report_a_partition_the_whole_board(monkeypatch):
+ # MI355X DPX: partition 1 decodes to 03:00.1 (no amdgpu device); partition 0 must not read the whole board.
+ _rocm_linux_host(
+ monkeypatch,
+ [("0000:03:00.0", "gfx950", (8.0, 288.0)), ("0000:03:00.1", "gfx950", None)],
+ boards = ["0000:03:00.0"],
+ )
+ assert video_mod._h3_card_memory_bytes("cuda", 0, allow_context = False) == (None, None)
+ assert video_mod._h3_visible_card_count("cuda") == 0
+ # The same board unpartitioned (SPX) still reads.
+ _rocm_linux_host(monkeypatch, [("0000:03:00.0", "gfx950", (8.0, 288.0))])
+ assert video_mod._h3_card_memory_bytes("cuda", 0, allow_context = False) == (280 * GiB, 288 * GiB)
+
+
+def test_h3_card_probes_on_cuda_still_read_nvidia_smi(monkeypatch):
+ from utils.hardware import hardware
+
+ monkeypatch.setattr(hardware, "get_device", lambda: hardware.DeviceType.CUDA)
+ monkeypatch.setattr(hardware, "IS_ROCM", False)
+ monkeypatch.setattr(
+ hardware,
+ "_get_parent_visible_gpu_spec",
+ lambda: {"raw": None, "numeric_ids": None, "supports_explicit_gpu_ids": True},
+ )
+ monkeypatch.setattr(hardware, "_reconcile_cuda_integrated_memory", lambda *a, **k: None)
+ monkeypatch.setattr(
+ hardware,
+ "_smi_visible_utilization",
+ lambda spec: {
+ "available": True,
+ "devices": [
+ {"index": 0, "visible_ordinal": 0, "vram_used_gb": 2.0, "vram_total_gb": 16.0}
+ ],
+ "index_kind": "relative",
+ "parent_visible_gpu_ids": [],
+ },
+ )
+ monkeypatch.setattr(
+ hardware,
+ "_rocm_context_free_visible_devices",
+ lambda: pytest.fail("ROCm sysfs read on a CUDA host"),
+ raising = False,
+ )
+ assert video_mod._h3_card_memory_bytes("cuda", 0, allow_context = False) == (14 * GiB, 16 * GiB)
+ assert video_mod._h3_visible_card_count("cuda") == 1
+
+
def test_h3_residency_reads_fall_back_to_torch_on_rocm(monkeypatch):
# The residency and pressure reads may create a context, as they did before the launch budget.
_h3_telemetry(monkeypatch, rocm = True, smi = True)
diff --git a/studio/backend/utils/hardware/hardware.py b/studio/backend/utils/hardware/hardware.py
index 93b2650d0e0..80964175ff8 100644
--- a/studio/backend/utils/hardware/hardware.py
+++ b/studio/backend/utils/hardware/hardware.py
@@ -5230,17 +5230,90 @@ def _overlay_system_wide_vram(devices: list[Dict[str, Any]]) -> None:
_apply_system_wide_vram(devices, _rocm_system_wide_vram_by_index(devices))
+# Discrete-only gfx targets: an APU's sysfs total is the BIOS carve-out, so unlisted targets read unknown. gfx942 is
+# excluded because MI300A (an APU) shares it with MI300X.
+_ROCM_DISCRETE_ONLY_ARCHES = frozenset(
+ {
+ "gfx803",
+ "gfx900",
+ "gfx906",
+ "gfx908",
+ "gfx90a",
+ "gfx950",
+ "gfx1010",
+ "gfx1011",
+ "gfx1012",
+ "gfx1030",
+ "gfx1031",
+ "gfx1032",
+ "gfx1034",
+ "gfx1100",
+ "gfx1101",
+ "gfx1102",
+ "gfx1200",
+ "gfx1201",
+ }
+)
+
+
+def _rocm_context_free_visible_devices() -> list[Dict[str, Any]]:
+ """Linux ROCm per-device VRAM from KFD topology + DRM sysfs, joined by PCI address, with no torch call. Empty under
+ any visibility mask, a KFD reader disagreement, a shared PCI address or a KFD address with no amdgpu device (a
+ partitioned card); non-discrete targets keep their row with used / total None."""
+ if platform.system() != "Linux" or _rocm_visibility_mask_active():
+ return []
+ pci_ids = _rocm_kfd_gpu_pci_ids()
+ if not pci_ids or len(set(p.lower() for p in pci_ids)) != len(pci_ids):
+ return []
+ # A partitioned card ORs each partition's node id into its KFD location_id (kfd_topology_add_device), so partitions
+ # decode to PCI functions with no amdgpu device of their own while partition 0 matches the whole-board sysfs total.
+ if not {p.lower() for p in pci_ids} <= {c[0].lower() for c in _rocm_linux_amdgpu_cards()}:
+ return []
+ try:
+ from . import amd
+ targets = amd.amd_kfd_gpu_gfx_targets()
+ except Exception as e:
+ logger.debug("KFD gfx target read failed: %s", e)
+ return []
+ if targets is None or len(targets) != len(pci_ids):
+ return []
+ vram_by_pci = _rocm_linux_sysfs_vram_by_pci_gb()
+ devices = []
+ for ordinal, (bdf, target) in enumerate(zip(pci_ids, targets)):
+ entry = vram_by_pci.get(bdf.lower()) if target in _ROCM_DISCRETE_ONLY_ARCHES else None
+ used, total = entry if entry is not None else (None, None)
+ devices.append(
+ {
+ "index": ordinal,
+ "index_kind": "physical",
+ "visible_ordinal": ordinal,
+ "gpu_utilization_pct": None,
+ "temperature_c": None,
+ "vram_used_gb": used,
+ "vram_total_gb": total,
+ "vram_utilization_pct": round((used / total) * 100, 1)
+ if used is not None and total
+ else None,
+ "power_draw_w": None,
+ "power_limit_w": None,
+ "power_utilization_pct": None,
+ }
+ )
+ return devices
+
+
def get_visible_gpu_utilization(*, no_context: bool = False) -> Dict[str, Any]:
- """``no_context``: skip torch's per-device reading (it leaves a resident driver context); devices are then empty
- when nvidia-smi cannot answer, and always on ROCm, whose amd-smi rows need that reading."""
+ """``no_context``: never ask torch (it leaves a resident driver context): CUDA devices are empty without nvidia-smi,
+ ROCm ones come from _rocm_context_free_visible_devices."""
device = get_device()
if no_context and IS_ROCM and device == DeviceType.CUDA:
+ devices = _rocm_context_free_visible_devices()
return {
- "available": False,
+ "available": bool(devices),
"backend": _backend_label(device),
- "parent_visible_gpu_ids": [],
- "devices": [],
- "index_kind": "relative",
+ "parent_visible_gpu_ids": [d["index"] for d in devices],
+ "devices": devices,
+ "index_kind": "physical" if devices else "relative",
}
if device == DeviceType.CUDA: