Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Unsloth Studio (AMD): show each GPU's live VRAM and utilization under its own HIP id (#12961)

Daniel Han154a5eaeb3c5
diff --git a/studio/backend/tests/conftest.py b/studio/backend/tests/conftest.py
index 0a5a9d3a404..76f6a077d6e 100644
--- a/studio/backend/tests/conftest.py
+++ b/studio/backend/tests/conftest.py
@@ -211,6 +211,10 @@ def _reset_gpu_query_cache():
if hw is not None and hasattr(hw, "_last_good_visible_info"):
with hw._last_good_visible_lock:
hw._last_good_visible_info.clear()
+ amd = sys.modules.get("utils.hardware.amd")
+ if amd is not None and hasattr(amd, "_hip_id_map_lock"):
+ with amd._hip_id_map_lock:
+ amd._hip_id_map_cache = None
_reset()
yield
diff --git a/studio/backend/tests/test_rocm_live_telemetry_hip_ids.py b/studio/backend/tests/test_rocm_live_telemetry_hip_ids.py
new file mode 100644
index 00000000000..54aab5a35ba
--- /dev/null
+++ b/studio/backend/tests/test_rocm_live_telemetry_hip_ids.py
@@ -0,0 +1,112 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""ROCm live telemetry keys each amd-smi row by its HIP id (amd-smi, torch and the device are mocked)."""
+
+from __future__ import annotations
+
+import pytest
+
+from utils.hardware import amd
+from utils.hardware import hardware as hw
+from utils.hardware.hardware import DeviceType
+
+
+def _metric(*gpus):
+ return [
+ {
+ "gpu": idx,
+ "usage": {"gfx_activity": util},
+ "mem_usage": {
+ "used_vram": {"value": used, "unit": "MB"},
+ "total_vram": {"value": total, "unit": "MB"},
+ },
+ }
+ for idx, used, total, util in gpus
+ ]
+
+
+# amd-smi gpu 0 is a 16 GiB card that HIP calls 1; amd-smi gpu 1 is a 96 GiB card HIP calls 0.
+_TWO_CARDS = _metric((0, 1024, 16384, 5), (1, 2048, 98304, 90))
+_REVERSED = [{"gpu": 0, "hip_id": 1}, {"gpu": 1, "hip_id": 0}]
+
+
+def rocm(monkeypatch):
+ for var in ("HIP_VISIBLE_DEVICES", "ROCR_VISIBLE_DEVICES", "CUDA_VISIBLE_DEVICES"):
+ monkeypatch.delenv(var, raising = False)
+ monkeypatch.delenv("GPU_DEVICE_ORDINAL", raising = False)
+ monkeypatch.setattr(hw, "IS_ROCM", True)
+ monkeypatch.setattr(hw, "get_device", lambda: DeviceType.CUDA)
+ monkeypatch.setattr(hw, "_reconcile_rocm_unified_memory", lambda *a, **k: None)
+ monkeypatch.setattr(hw, "_apply_system_wide_vram", lambda *a, **k: None, raising = False)
+
+ def _host(physical, hip_visible, metric, enumeration):
+ monkeypatch.setattr(hw, "get_physical_gpu_count", lambda: physical)
+ monkeypatch.setattr(amd, "get_physical_gpu_count", lambda: physical)
+ monkeypatch.setattr(hw, "_torch_get_physical_gpu_count", lambda: hip_visible)
+
+ def _run(*args, **kwargs):
+ return enumeration if args and args[0] == "list" else metric
+
+ monkeypatch.setattr(amd, "_run_amd_smi", _run)
+
+ return _host
+
+
+def _by_index(result):
+ return {d["index"]: (d["vram_total_gb"], d["gpu_utilization_pct"]) for d in result["devices"]}
+
+
[email protected]("entry", ["visible", "payload"])
+def test_reordered_host_reports_each_card_under_its_hip_id(rocm, entry):
+ rocm(2, 2, _TWO_CARDS, _REVERSED)
+ if entry == "visible":
+ result = hw.get_visible_gpu_utilization()
+ else:
+ result = {"devices": hw.get_gpu_utilization()["devices"]}
+ assert _by_index(result) == {0: (96.0, 90), 1: (16.0, 5)}
+
+
+def test_hip_mask_names_the_card_amd_smi_calls_by_another_id(rocm, monkeypatch):
+ monkeypatch.setenv("HIP_VISIBLE_DEVICES", "0")
+ rocm(2, 1, _TWO_CARDS, _REVERSED)
+ result = hw.get_visible_gpu_utilization()
+ assert _by_index(result) == {0: (96.0, 90)}
+ assert result["parent_visible_gpu_ids"] == [0]
+ assert result["devices"][0]["visible_ordinal"] == 0
+
+
+def test_identity_mapping_is_unchanged(rocm):
+ rocm(2, 2, _TWO_CARDS, [{"gpu": 0, "hip_id": 0}, {"gpu": 1, "hip_id": 1}])
+ assert _by_index(hw.get_visible_gpu_utilization()) == {0: (16.0, 5), 1: (96.0, 90)}
+
+
+def test_without_list_e_the_query_is_unchanged(rocm):
+ # amd-smi before ROCm 6.4.0 has no `list -e`: keep today's untranslated rows.
+ rocm(2, 2, _TWO_CARDS, None)
+ assert _by_index(hw.get_visible_gpu_utilization()) == {0: (16.0, 5), 1: (96.0, 90)}
+
+
+def test_stacked_masks_keep_the_untranslated_query(rocm, monkeypatch):
+ monkeypatch.setattr(hw.sys, "platform", "linux")
+ monkeypatch.setenv("ROCR_VISIBLE_DEVICES", "0,1")
+ monkeypatch.setenv("HIP_VISIBLE_DEVICES", "1")
+ rocm(2, 1, _TWO_CARDS, _REVERSED)
+ assert _by_index(hw.get_visible_gpu_utilization()) == {1: (96.0, 90)}
+
+
+def test_single_gpu_is_unchanged(rocm):
+ rocm(1, 1, _metric((0, 1024, 16384, 5)), None)
+ assert _by_index(hw.get_visible_gpu_utilization()) == {0: (16.0, 5)}
+
+
[email protected]("enumeration", [_REVERSED, None])
+def test_list_e_is_spawned_once_across_polls(rocm, monkeypatch, enumeration):
+ rocm(2, 2, _TWO_CARDS, enumeration)
+ stub, calls = amd._run_amd_smi, []
+ monkeypatch.setattr(amd, "_run_amd_smi", lambda *a, **k: calls.append(a) or stub(*a, **k))
+ first = _by_index(hw.get_visible_gpu_utilization())
+ assert _by_index(hw.get_visible_gpu_utilization()) == first
+ assert calls.count(("list", "-e")) == 1
+ assert calls.count(("metric",)) == 2
diff --git a/studio/backend/utils/hardware/amd.py b/studio/backend/utils/hardware/amd.py
index e41134f6b47..7cbc569074e 100644
--- a/studio/backend/utils/hardware/amd.py
+++ b/studio/backend/utils/hardware/amd.py
@@ -14,6 +14,8 @@ import shutil
import stat
import subprocess
import sys
+import threading
+import time
from pathlib import PurePath
from typing import Any, Optional
@@ -379,8 +381,24 @@ def get_gpu_vram_mib() -> dict[int, tuple[int, int]]:
return get_gpu_vram_report()[0]
+_HIP_ID_MAP_NONE_TTL_S = 300.0
+_hip_id_map_lock = threading.Lock()
+_hip_id_map_cache: Optional[tuple[float, Optional[dict[int, int]]]] = None
+
+
def get_hip_id_by_gpu_index() -> Optional[dict[int, int]]:
- """{amd-smi gpu id: HIP device id}, or None when the mapping is not readable. Two index spaces, one number: amd-smi's gpu id is an enumeration index in discovery order over its KFD/sysfs view, while HIP's is what ``HIP_VISIBLE_DEVICES`` names and what torch reports as ``cuda:N``, derived from the KFD node id instead (``hip_id = node_id - smallest_node_id``). They coincide on most hosts and not on all of them, so the number cannot be carried from one space to the other without this call. ``amd-smi list -e`` is the mapping AMD publishes for exactly this ("mapping physical-to-logical GPU IDs"), added in ROCm 6.4.0. None when any device lacks a usable id (an older CLI rejects ``-e`` outright, and ``hip_id`` reads "N/A" when the library cannot reach the device's KFD node) so callers decline rather than assume the identity mapping."""
+ """{amd-smi gpu id: HIP device id}, or None when the mapping is not readable. Two index spaces, one number: amd-smi's gpu id is an enumeration index in discovery order over its KFD/sysfs view, while HIP's is what ``HIP_VISIBLE_DEVICES`` names and what torch reports as ``cuda:N``, derived from the KFD node id instead (``hip_id = node_id - smallest_node_id``). They coincide on most hosts and not on all of them, so the number cannot be carried from one space to the other without this call. ``amd-smi list -e`` is the mapping AMD publishes for exactly this ("mapping physical-to-logical GPU IDs"), added in ROCm 6.4.0. None when any device lacks a usable id (an older CLI rejects ``-e`` outright, and ``hip_id`` reads "N/A" when the library cannot reach the device's KFD node) so callers decline rather than assume the identity mapping. Cached for the process; an unreadable answer is retried after five minutes."""
+ global _hip_id_map_cache
+ with _hip_id_map_lock:
+ cached = _hip_id_map_cache
+ if cached is None or (
+ cached[1] is None and time.monotonic() - cached[0] >= _HIP_ID_MAP_NONE_TTL_S
+ ):
+ cached = _hip_id_map_cache = (time.monotonic(), _read_hip_id_by_gpu_index())
+ return None if cached[1] is None else dict(cached[1])
+
+
+def _read_hip_id_by_gpu_index() -> Optional[dict[int, int]]:
data = _run_amd_smi("list", "-e", count_failures = False)
if data is None:
return None
diff --git a/studio/backend/utils/hardware/hardware.py b/studio/backend/utils/hardware/hardware.py
index e067a2eb2f1..ce42da1d60d 100644
--- a/studio/backend/utils/hardware/hardware.py
+++ b/studio/backend/utils/hardware/hardware.py
@@ -2046,6 +2046,23 @@ def _amd_smi_ids_for_hip_ids(hip_ids: Optional[list[int]]) -> Optional[list[int]
return [hip_to_smi[hip_id] for hip_id in hip_ids]
+def _smi_visible_utilization(parent_visible_spec: Dict[str, Any]) -> Optional[Dict[str, Any]]:
+ """SMI visible-GPU utilization keyed by the parent's ids. amd-smi filters by its own gpu ids, not HIP's, so translate when the mapping is readable."""
+ ids = parent_visible_spec["numeric_ids"]
+ smi_ids = _amd_smi_ids_for_hip_ids(ids) if IS_ROCM and ids else None
+ result = _smi_query(
+ "get_visible_gpu_utilization",
+ ids if smi_ids is None else smi_ids,
+ parent_cuda_visible_devices = parent_visible_spec["raw"],
+ )
+ if result is not None and smi_ids is not None and smi_ids != ids:
+ hip_by_smi = dict(zip(smi_ids, ids))
+ for device in result.get("devices", []):
+ device["index"] = hip_by_smi.get(device.get("index"), device.get("index"))
+ result["parent_visible_gpu_ids"] = list(ids)
+ return result
+
+
def _free_in_torch_scope(total_bytes: int, used_gb: float) -> int:
"""Driver used-memory turned into free bytes, in torch's allocatable scope. Subtract from torch's total, never the driver's: NVIDIA's total also spans a reserved framebuffer torch can never hand out (726 MiB on a B200), so driver_total - used reports a full card as free. Clamped both ends, since the parsers accept signed values and a negative used would advertise more free than the card has."""
return min(total_bytes, max(0, total_bytes - round(used_gb * (1024**3))))
@@ -3792,11 +3809,7 @@ def get_gpu_utilization() -> Dict[str, Any]:
if device == DeviceType.CUDA:
parent_visible_spec = _get_parent_visible_gpu_spec()
- result = _smi_query(
- "get_visible_gpu_utilization",
- parent_visible_spec["numeric_ids"],
- parent_cuda_visible_devices = parent_visible_spec["raw"],
- )
+ result = _smi_visible_utilization(parent_visible_spec)
if result is not None and "devices" in result:
devices = result["devices"]
numeric_ids = parent_visible_spec.get("numeric_ids")
@@ -4409,11 +4422,7 @@ def get_visible_gpu_utilization() -> Dict[str, Any]:
if device == DeviceType.CUDA:
parent_visible_spec = _get_parent_visible_gpu_spec()
- result = _smi_query(
- "get_visible_gpu_utilization",
- parent_visible_spec["numeric_ids"],
- parent_cuda_visible_devices = parent_visible_spec["raw"],
- )
+ result = _smi_visible_utilization(parent_visible_spec)
if result is not None:
result["backend"] = _backend_label(device)
numeric_ids = parent_visible_spec.get("numeric_ids")
diff --git a/tests/studio/install/test_rocm_support.py b/tests/studio/install/test_rocm_support.py
index 6cc35977101..6216296949a 100644
--- a/tests/studio/install/test_rocm_support.py
+++ b/tests/studio/install/test_rocm_support.py
@@ -4407,9 +4407,11 @@ class TestHardwareAmdBranching:
source = hw_path.read_text(encoding = "utf-8")
func_start = source.find("def get_gpu_utilization")
func_body = source[func_start : source.find("\ndef ", func_start + 1)]
- assert "_smi_query(" in func_body
- assert '"get_visible_gpu_utilization"' in func_body
+ assert "_smi_visible_utilization(" in func_body
assert "_reconcile_rocm_unified_memory" in func_body
+ helper_start = source.find("def _smi_visible_utilization")
+ helper = source[helper_start : source.find("\ndef ", helper_start + 1)]
+ assert re.search(r'_smi_query\(\s*"get_visible_gpu_utilization"', helper)
smi = source[
source.find("def _smi_query") : source.find("\ndef ", source.find("def _smi_query") + 1)
]
@@ -4422,10 +4424,10 @@ class TestHardwareAmdBranching:
source = hw_path.read_text(encoding = "utf-8")
func_start = source.find("def get_visible_gpu_utilization")
func_body = source[func_start : source.find("\ndef ", func_start + 1)]
- # The dispatcher call may wrap; allow whitespace before the func name arg.
- import re as _re
-
- assert _re.search(r'_smi_query\(\s*"get_visible_gpu_utilization"', func_body)
+ assert "_smi_visible_utilization(" in func_body
+ helper_start = source.find("def _smi_visible_utilization")
+ helper = source[helper_start : source.find("\ndef ", helper_start + 1)]
+ assert re.search(r'_smi_query\(\s*"get_visible_gpu_utilization"', helper)
smi = source[
source.find("def _smi_query") : source.find("\ndef ", source.find("def _smi_query") + 1)
]