Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: read MiniMax-H3 card VRAM on ROCm without a torch context (#14732)
Daniel Han97a4208b6c78
diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.pyindex eff8afd8e8f..3331afbbc1f 100644--- a/studio/backend/core/inference/video.py+++ b/studio/backend/core/inference/video.py@@ -2694,7 +2694,7 @@ def _h3_card_memory_bytes(*,allow_context: bool = True,) -> tuple[Optional[int], Optional[int]]:- """(free, total) VRAM of the H3 render card, or (None, None). No pin = ordinal 0. nvidia-smi / amd-smi / sysfs+ """(free, total) VRAM of the H3 render card, or (None, None). No pin = ordinal 0. nvidia-smi / ROCm KFD + sysfsfirst (no CUDA context); torch only when none answers and ``allow_context``, as the residency and pressure readsalways did. The pre-spawn budget passes False: it never pays for a context. Free is less the user's VRAM Budgetreserve (raw at the default), so the sd.cpp resident-vs-offload pick, its ``--max-vram`` probe and thediff --git a/studio/backend/tests/test_sd_cpp_vram_budget.py b/studio/backend/tests/test_sd_cpp_vram_budget.pyindex 5c62043962a..11b0db90636 100644--- a/studio/backend/tests/test_sd_cpp_vram_budget.py+++ b/studio/backend/tests/test_sd_cpp_vram_budget.py@@ -523,17 +523,161 @@ def _h3_telemetry(monkeypatch, *, rocm, smi):monkeypatch.setattr(hardware, "_torch_get_per_device_info", per_device)monkeypatch.setattr(hardware, "_overlay_system_wide_vram", lambda devices: None)+ # No KFD topology, so only amd-smi + torch can answer.+ monkeypatch.setattr(hardware, "_rocm_kfd_gpu_pci_ids", lambda: [])return toucheddef test_h3_card_probes_read_nothing_on_rocm_without_a_context(monkeypatch):- # The pre-spawn budget never creates a context, and ROCm's amd-smi rows need one.touched = _h3_telemetry(monkeypatch, rocm = True, smi = True)assert video_mod._h3_card_memory_bytes("cuda", 0, allow_context = False) == (None, None)assert video_mod._h3_visible_card_count("cuda") == 0assert touched == []+_ROCM_MASKS = (+ "HIP_VISIBLE_DEVICES",+ "ROCR_VISIBLE_DEVICES",+ "CUDA_VISIBLE_DEVICES",+ "GPU_DEVICE_ORDINAL",+)+++def _rocm_linux_host(+ monkeypatch,+ cards,+ boards = None,+):+ """``cards`` = [(pci, gfx target, (used_gb, total_gb) or None)] in KFD order; torch and amd-smi are tripwires."""+ from utils.hardware import amd, hardware++ for name in _ROCM_MASKS:+ monkeypatch.delenv(name, raising = False)+ monkeypatch.setattr(hardware, "get_device", lambda: hardware.DeviceType.CUDA)+ monkeypatch.setattr(hardware, "IS_ROCM", True)+ monkeypatch.setattr(hardware.platform, "system", lambda: "Linux")+ monkeypatch.setattr(hardware, "_rocm_kfd_gpu_pci_ids", lambda: [c[0] for c in cards])+ monkeypatch.setattr(amd, "amd_kfd_gpu_gfx_targets", lambda *a, **k: [c[1] for c in cards])+ monkeypatch.setattr(+ hardware,+ "_rocm_linux_sysfs_vram_by_pci_gb",+ lambda: {c[0]: c[2] for c in cards if c[2] is not None},+ )+ monkeypatch.setattr(+ hardware,+ "_rocm_linux_amdgpu_cards",+ lambda: [+ (bdf, i, f"/sys/class/drm/card{i}/device")+ for i, bdf in enumerate(boards or dict.fromkeys(c[0] for c in cards))+ ],+ )++ def _no_torch(*a, **k):+ pytest.fail("the no-context probe touched torch")++ import torch++ for name in ("mem_get_info", "get_device_properties", "init", "_lazy_init", "device_count"):+ monkeypatch.setattr(torch.cuda, name, _no_torch)+ for name in (+ "_torch_get_device_module",+ "_torch_get_per_device_info",+ "_torch_get_device_inventory",+ "_torch_get_physical_gpu_count",+ "_smi_visible_utilization",+ ):+ monkeypatch.setattr(hardware, name, _no_torch)+++def test_h3_card_probes_read_rocm_sysfs_without_a_context(monkeypatch):+ _rocm_linux_host(monkeypatch, [("0000:03:00.0", "gfx1100", (4.0, 24.0))])+ assert video_mod._h3_card_memory_bytes("cuda", 0, allow_context = False) == (20 * GiB, 24 * GiB)+ assert video_mod._h3_visible_card_count("cuda") == 1+ assert (+ video_mod._h3_card_free_bytes("cuda", 0, single_card_only = True, allow_context = False)+ == 20 * GiB+ )+++def test_h3_card_probes_on_masked_rocm_read_nothing(monkeypatch):+ _rocm_linux_host(monkeypatch, [("0000:03:00.0", "gfx1100", (4.0, 24.0))])+ assert video_mod._h3_card_memory_bytes("cuda", 0, allow_context = False) == (20 * GiB, 24 * GiB)+ for name in _ROCM_MASKS:+ monkeypatch.setenv(name, "0")+ assert video_mod._h3_card_memory_bytes("cuda", 0, allow_context = False) == (None, None), name+ assert video_mod._h3_visible_card_count("cuda") == 0, name+ monkeypatch.delenv(name)+++def test_h3_card_probes_never_report_an_apu_carve_out(monkeypatch):+ _rocm_linux_host(+ monkeypatch,+ [+ ("0000:03:00.0", "gfx1100", (4.0, 24.0)),+ ("0000:c5:00.0", "gfx1151", (1.0, 0.5)),+ ("0000:0b:00.0", "gfx942", (1.0, 192.0)),+ ],+ )+ assert video_mod._h3_card_memory_bytes("cuda", 0, allow_context = False) == (20 * GiB, 24 * GiB)+ assert video_mod._h3_card_memory_bytes("cuda", 1, allow_context = False) == (None, None)+ assert video_mod._h3_card_memory_bytes("cuda", 2, allow_context = False) == (None, None)+ assert video_mod._h3_visible_card_count("cuda") == 3+ # A partitioned card (two ordinals on one PCI address) reads nothing.+ _rocm_linux_host(+ monkeypatch,+ [("0000:03:00.0", "gfx950", (4.0, 24.0)), ("0000:03:00.0", "gfx950", (4.0, 24.0))],+ )+ assert video_mod._h3_card_memory_bytes("cuda", 0, allow_context = False) == (None, None)+ assert video_mod._h3_visible_card_count("cuda") == 0+++def test_h3_card_probes_never_report_a_partition_the_whole_board(monkeypatch):+ # MI355X DPX: partition 1 decodes to 03:00.1 (no amdgpu device); partition 0 must not read the whole board.+ _rocm_linux_host(+ monkeypatch,+ [("0000:03:00.0", "gfx950", (8.0, 288.0)), ("0000:03:00.1", "gfx950", None)],+ boards = ["0000:03:00.0"],+ )+ assert video_mod._h3_card_memory_bytes("cuda", 0, allow_context = False) == (None, None)+ assert video_mod._h3_visible_card_count("cuda") == 0+ # The same board unpartitioned (SPX) still reads.+ _rocm_linux_host(monkeypatch, [("0000:03:00.0", "gfx950", (8.0, 288.0))])+ assert video_mod._h3_card_memory_bytes("cuda", 0, allow_context = False) == (280 * GiB, 288 * GiB)+++def test_h3_card_probes_on_cuda_still_read_nvidia_smi(monkeypatch):+ from utils.hardware import hardware++ monkeypatch.setattr(hardware, "get_device", lambda: hardware.DeviceType.CUDA)+ monkeypatch.setattr(hardware, "IS_ROCM", False)+ monkeypatch.setattr(+ hardware,+ "_get_parent_visible_gpu_spec",+ lambda: {"raw": None, "numeric_ids": None, "supports_explicit_gpu_ids": True},+ )+ monkeypatch.setattr(hardware, "_reconcile_cuda_integrated_memory", lambda *a, **k: None)+ monkeypatch.setattr(+ hardware,+ "_smi_visible_utilization",+ lambda spec: {+ "available": True,+ "devices": [+ {"index": 0, "visible_ordinal": 0, "vram_used_gb": 2.0, "vram_total_gb": 16.0}+ ],+ "index_kind": "relative",+ "parent_visible_gpu_ids": [],+ },+ )+ monkeypatch.setattr(+ hardware,+ "_rocm_context_free_visible_devices",+ lambda: pytest.fail("ROCm sysfs read on a CUDA host"),+ raising = False,+ )+ assert video_mod._h3_card_memory_bytes("cuda", 0, allow_context = False) == (14 * GiB, 16 * GiB)+ assert video_mod._h3_visible_card_count("cuda") == 1++def test_h3_residency_reads_fall_back_to_torch_on_rocm(monkeypatch):# The residency and pressure reads may create a context, as they did before the launch budget._h3_telemetry(monkeypatch, rocm = True, smi = True)diff --git a/studio/backend/utils/hardware/hardware.py b/studio/backend/utils/hardware/hardware.pyindex 93b2650d0e0..80964175ff8 100644--- a/studio/backend/utils/hardware/hardware.py+++ b/studio/backend/utils/hardware/hardware.py@@ -5230,17 +5230,90 @@ def _overlay_system_wide_vram(devices: list[Dict[str, Any]]) -> None:_apply_system_wide_vram(devices, _rocm_system_wide_vram_by_index(devices))+# Discrete-only gfx targets: an APU's sysfs total is the BIOS carve-out, so unlisted targets read unknown. gfx942 is+# excluded because MI300A (an APU) shares it with MI300X.+_ROCM_DISCRETE_ONLY_ARCHES = frozenset(+ {+ "gfx803",+ "gfx900",+ "gfx906",+ "gfx908",+ "gfx90a",+ "gfx950",+ "gfx1010",+ "gfx1011",+ "gfx1012",+ "gfx1030",+ "gfx1031",+ "gfx1032",+ "gfx1034",+ "gfx1100",+ "gfx1101",+ "gfx1102",+ "gfx1200",+ "gfx1201",+ }+)+++def _rocm_context_free_visible_devices() -> list[Dict[str, Any]]:+ """Linux ROCm per-device VRAM from KFD topology + DRM sysfs, joined by PCI address, with no torch call. Empty under+ any visibility mask, a KFD reader disagreement, a shared PCI address or a KFD address with no amdgpu device (a+ partitioned card); non-discrete targets keep their row with used / total None."""+ if platform.system() != "Linux" or _rocm_visibility_mask_active():+ return []+ pci_ids = _rocm_kfd_gpu_pci_ids()+ if not pci_ids or len(set(p.lower() for p in pci_ids)) != len(pci_ids):+ return []+ # A partitioned card ORs each partition's node id into its KFD location_id (kfd_topology_add_device), so partitions+ # decode to PCI functions with no amdgpu device of their own while partition 0 matches the whole-board sysfs total.+ if not {p.lower() for p in pci_ids} <= {c[0].lower() for c in _rocm_linux_amdgpu_cards()}:+ return []+ try:+ from . import amd+ targets = amd.amd_kfd_gpu_gfx_targets()+ except Exception as e:+ logger.debug("KFD gfx target read failed: %s", e)+ return []+ if targets is None or len(targets) != len(pci_ids):+ return []+ vram_by_pci = _rocm_linux_sysfs_vram_by_pci_gb()+ devices = []+ for ordinal, (bdf, target) in enumerate(zip(pci_ids, targets)):+ entry = vram_by_pci.get(bdf.lower()) if target in _ROCM_DISCRETE_ONLY_ARCHES else None+ used, total = entry if entry is not None else (None, None)+ devices.append(+ {+ "index": ordinal,+ "index_kind": "physical",+ "visible_ordinal": ordinal,+ "gpu_utilization_pct": None,+ "temperature_c": None,+ "vram_used_gb": used,+ "vram_total_gb": total,+ "vram_utilization_pct": round((used / total) * 100, 1)+ if used is not None and total+ else None,+ "power_draw_w": None,+ "power_limit_w": None,+ "power_utilization_pct": None,+ }+ )+ return devices++def get_visible_gpu_utilization(*, no_context: bool = False) -> Dict[str, Any]:- """``no_context``: skip torch's per-device reading (it leaves a resident driver context); devices are then empty- when nvidia-smi cannot answer, and always on ROCm, whose amd-smi rows need that reading."""+ """``no_context``: never ask torch (it leaves a resident driver context): CUDA devices are empty without nvidia-smi,+ ROCm ones come from _rocm_context_free_visible_devices."""device = get_device()if no_context and IS_ROCM and device == DeviceType.CUDA:+ devices = _rocm_context_free_visible_devices()return {- "available": False,+ "available": bool(devices),"backend": _backend_label(device),- "parent_visible_gpu_ids": [],- "devices": [],- "index_kind": "relative",+ "parent_visible_gpu_ids": [d["index"] for d in devices],+ "devices": devices,+ "index_kind": "physical" if devices else "relative",}if device == DeviceType.CUDA: