Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Unsloth Studio (AMD): show each GPU's live VRAM and utilization under its own HIP id (#12961)
Daniel Han154a5eaeb3c5
diff --git a/studio/backend/tests/conftest.py b/studio/backend/tests/conftest.pyindex 0a5a9d3a404..76f6a077d6e 100644--- a/studio/backend/tests/conftest.py+++ b/studio/backend/tests/conftest.py@@ -211,6 +211,10 @@ def _reset_gpu_query_cache():if hw is not None and hasattr(hw, "_last_good_visible_info"):with hw._last_good_visible_lock:hw._last_good_visible_info.clear()+ amd = sys.modules.get("utils.hardware.amd")+ if amd is not None and hasattr(amd, "_hip_id_map_lock"):+ with amd._hip_id_map_lock:+ amd._hip_id_map_cache = None_reset()yielddiff --git a/studio/backend/tests/test_rocm_live_telemetry_hip_ids.py b/studio/backend/tests/test_rocm_live_telemetry_hip_ids.pynew file mode 100644index 00000000000..54aab5a35ba--- /dev/null+++ b/studio/backend/tests/test_rocm_live_telemetry_hip_ids.py@@ -0,0 +1,112 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""ROCm live telemetry keys each amd-smi row by its HIP id (amd-smi, torch and the device are mocked)."""++from __future__ import annotations++import pytest++from utils.hardware import amd+from utils.hardware import hardware as hw+from utils.hardware.hardware import DeviceType+++def _metric(*gpus):+ return [+ {+ "gpu": idx,+ "usage": {"gfx_activity": util},+ "mem_usage": {+ "used_vram": {"value": used, "unit": "MB"},+ "total_vram": {"value": total, "unit": "MB"},+ },+ }+ for idx, used, total, util in gpus+ ]+++# amd-smi gpu 0 is a 16 GiB card that HIP calls 1; amd-smi gpu 1 is a 96 GiB card HIP calls 0.+_TWO_CARDS = _metric((0, 1024, 16384, 5), (1, 2048, 98304, 90))+_REVERSED = [{"gpu": 0, "hip_id": 1}, {"gpu": 1, "hip_id": 0}]+++def rocm(monkeypatch):+ for var in ("HIP_VISIBLE_DEVICES", "ROCR_VISIBLE_DEVICES", "CUDA_VISIBLE_DEVICES"):+ monkeypatch.delenv(var, raising = False)+ monkeypatch.delenv("GPU_DEVICE_ORDINAL", raising = False)+ monkeypatch.setattr(hw, "IS_ROCM", True)+ monkeypatch.setattr(hw, "get_device", lambda: DeviceType.CUDA)+ monkeypatch.setattr(hw, "_reconcile_rocm_unified_memory", lambda *a, **k: None)+ monkeypatch.setattr(hw, "_apply_system_wide_vram", lambda *a, **k: None, raising = False)++ def _host(physical, hip_visible, metric, enumeration):+ monkeypatch.setattr(hw, "get_physical_gpu_count", lambda: physical)+ monkeypatch.setattr(amd, "get_physical_gpu_count", lambda: physical)+ monkeypatch.setattr(hw, "_torch_get_physical_gpu_count", lambda: hip_visible)++ def _run(*args, **kwargs):+ return enumeration if args and args[0] == "list" else metric++ monkeypatch.setattr(amd, "_run_amd_smi", _run)++ return _host+++def _by_index(result):+ return {d["index"]: (d["vram_total_gb"], d["gpu_utilization_pct"]) for d in result["devices"]}++[email protected]("entry", ["visible", "payload"])+def test_reordered_host_reports_each_card_under_its_hip_id(rocm, entry):+ rocm(2, 2, _TWO_CARDS, _REVERSED)+ if entry == "visible":+ result = hw.get_visible_gpu_utilization()+ else:+ result = {"devices": hw.get_gpu_utilization()["devices"]}+ assert _by_index(result) == {0: (96.0, 90), 1: (16.0, 5)}+++def test_hip_mask_names_the_card_amd_smi_calls_by_another_id(rocm, monkeypatch):+ monkeypatch.setenv("HIP_VISIBLE_DEVICES", "0")+ rocm(2, 1, _TWO_CARDS, _REVERSED)+ result = hw.get_visible_gpu_utilization()+ assert _by_index(result) == {0: (96.0, 90)}+ assert result["parent_visible_gpu_ids"] == [0]+ assert result["devices"][0]["visible_ordinal"] == 0+++def test_identity_mapping_is_unchanged(rocm):+ rocm(2, 2, _TWO_CARDS, [{"gpu": 0, "hip_id": 0}, {"gpu": 1, "hip_id": 1}])+ assert _by_index(hw.get_visible_gpu_utilization()) == {0: (16.0, 5), 1: (96.0, 90)}+++def test_without_list_e_the_query_is_unchanged(rocm):+ # amd-smi before ROCm 6.4.0 has no `list -e`: keep today's untranslated rows.+ rocm(2, 2, _TWO_CARDS, None)+ assert _by_index(hw.get_visible_gpu_utilization()) == {0: (16.0, 5), 1: (96.0, 90)}+++def test_stacked_masks_keep_the_untranslated_query(rocm, monkeypatch):+ monkeypatch.setattr(hw.sys, "platform", "linux")+ monkeypatch.setenv("ROCR_VISIBLE_DEVICES", "0,1")+ monkeypatch.setenv("HIP_VISIBLE_DEVICES", "1")+ rocm(2, 1, _TWO_CARDS, _REVERSED)+ assert _by_index(hw.get_visible_gpu_utilization()) == {1: (96.0, 90)}+++def test_single_gpu_is_unchanged(rocm):+ rocm(1, 1, _metric((0, 1024, 16384, 5)), None)+ assert _by_index(hw.get_visible_gpu_utilization()) == {0: (16.0, 5)}++[email protected]("enumeration", [_REVERSED, None])+def test_list_e_is_spawned_once_across_polls(rocm, monkeypatch, enumeration):+ rocm(2, 2, _TWO_CARDS, enumeration)+ stub, calls = amd._run_amd_smi, []+ monkeypatch.setattr(amd, "_run_amd_smi", lambda *a, **k: calls.append(a) or stub(*a, **k))+ first = _by_index(hw.get_visible_gpu_utilization())+ assert _by_index(hw.get_visible_gpu_utilization()) == first+ assert calls.count(("list", "-e")) == 1+ assert calls.count(("metric",)) == 2diff --git a/studio/backend/utils/hardware/amd.py b/studio/backend/utils/hardware/amd.pyindex e41134f6b47..7cbc569074e 100644--- a/studio/backend/utils/hardware/amd.py+++ b/studio/backend/utils/hardware/amd.py@@ -14,6 +14,8 @@ import shutilimport statimport subprocessimport sys+import threading+import timefrom pathlib import PurePathfrom typing import Any, Optional@@ -379,8 +381,24 @@ def get_gpu_vram_mib() -> dict[int, tuple[int, int]]:return get_gpu_vram_report()[0]+_HIP_ID_MAP_NONE_TTL_S = 300.0+_hip_id_map_lock = threading.Lock()+_hip_id_map_cache: Optional[tuple[float, Optional[dict[int, int]]]] = None++def get_hip_id_by_gpu_index() -> Optional[dict[int, int]]:- """{amd-smi gpu id: HIP device id}, or None when the mapping is not readable. Two index spaces, one number: amd-smi's gpu id is an enumeration index in discovery order over its KFD/sysfs view, while HIP's is what ``HIP_VISIBLE_DEVICES`` names and what torch reports as ``cuda:N``, derived from the KFD node id instead (``hip_id = node_id - smallest_node_id``). They coincide on most hosts and not on all of them, so the number cannot be carried from one space to the other without this call. ``amd-smi list -e`` is the mapping AMD publishes for exactly this ("mapping physical-to-logical GPU IDs"), added in ROCm 6.4.0. None when any device lacks a usable id (an older CLI rejects ``-e`` outright, and ``hip_id`` reads "N/A" when the library cannot reach the device's KFD node) so callers decline rather than assume the identity mapping."""+ """{amd-smi gpu id: HIP device id}, or None when the mapping is not readable. Two index spaces, one number: amd-smi's gpu id is an enumeration index in discovery order over its KFD/sysfs view, while HIP's is what ``HIP_VISIBLE_DEVICES`` names and what torch reports as ``cuda:N``, derived from the KFD node id instead (``hip_id = node_id - smallest_node_id``). They coincide on most hosts and not on all of them, so the number cannot be carried from one space to the other without this call. ``amd-smi list -e`` is the mapping AMD publishes for exactly this ("mapping physical-to-logical GPU IDs"), added in ROCm 6.4.0. None when any device lacks a usable id (an older CLI rejects ``-e`` outright, and ``hip_id`` reads "N/A" when the library cannot reach the device's KFD node) so callers decline rather than assume the identity mapping. Cached for the process; an unreadable answer is retried after five minutes."""+ global _hip_id_map_cache+ with _hip_id_map_lock:+ cached = _hip_id_map_cache+ if cached is None or (+ cached[1] is None and time.monotonic() - cached[0] >= _HIP_ID_MAP_NONE_TTL_S+ ):+ cached = _hip_id_map_cache = (time.monotonic(), _read_hip_id_by_gpu_index())+ return None if cached[1] is None else dict(cached[1])+++def _read_hip_id_by_gpu_index() -> Optional[dict[int, int]]:data = _run_amd_smi("list", "-e", count_failures = False)if data is None:return Nonediff --git a/studio/backend/utils/hardware/hardware.py b/studio/backend/utils/hardware/hardware.pyindex e067a2eb2f1..ce42da1d60d 100644--- a/studio/backend/utils/hardware/hardware.py+++ b/studio/backend/utils/hardware/hardware.py@@ -2046,6 +2046,23 @@ def _amd_smi_ids_for_hip_ids(hip_ids: Optional[list[int]]) -> Optional[list[int]return [hip_to_smi[hip_id] for hip_id in hip_ids]+def _smi_visible_utilization(parent_visible_spec: Dict[str, Any]) -> Optional[Dict[str, Any]]:+ """SMI visible-GPU utilization keyed by the parent's ids. amd-smi filters by its own gpu ids, not HIP's, so translate when the mapping is readable."""+ ids = parent_visible_spec["numeric_ids"]+ smi_ids = _amd_smi_ids_for_hip_ids(ids) if IS_ROCM and ids else None+ result = _smi_query(+ "get_visible_gpu_utilization",+ ids if smi_ids is None else smi_ids,+ parent_cuda_visible_devices = parent_visible_spec["raw"],+ )+ if result is not None and smi_ids is not None and smi_ids != ids:+ hip_by_smi = dict(zip(smi_ids, ids))+ for device in result.get("devices", []):+ device["index"] = hip_by_smi.get(device.get("index"), device.get("index"))+ result["parent_visible_gpu_ids"] = list(ids)+ return result++def _free_in_torch_scope(total_bytes: int, used_gb: float) -> int:"""Driver used-memory turned into free bytes, in torch's allocatable scope. Subtract from torch's total, never the driver's: NVIDIA's total also spans a reserved framebuffer torch can never hand out (726 MiB on a B200), so driver_total - used reports a full card as free. Clamped both ends, since the parsers accept signed values and a negative used would advertise more free than the card has."""return min(total_bytes, max(0, total_bytes - round(used_gb * (1024**3))))@@ -3792,11 +3809,7 @@ def get_gpu_utilization() -> Dict[str, Any]:if device == DeviceType.CUDA:parent_visible_spec = _get_parent_visible_gpu_spec()- result = _smi_query(- "get_visible_gpu_utilization",- parent_visible_spec["numeric_ids"],- parent_cuda_visible_devices = parent_visible_spec["raw"],- )+ result = _smi_visible_utilization(parent_visible_spec)if result is not None and "devices" in result:devices = result["devices"]numeric_ids = parent_visible_spec.get("numeric_ids")@@ -4409,11 +4422,7 @@ def get_visible_gpu_utilization() -> Dict[str, Any]:if device == DeviceType.CUDA:parent_visible_spec = _get_parent_visible_gpu_spec()- result = _smi_query(- "get_visible_gpu_utilization",- parent_visible_spec["numeric_ids"],- parent_cuda_visible_devices = parent_visible_spec["raw"],- )+ result = _smi_visible_utilization(parent_visible_spec)if result is not None:result["backend"] = _backend_label(device)numeric_ids = parent_visible_spec.get("numeric_ids")diff --git a/tests/studio/install/test_rocm_support.py b/tests/studio/install/test_rocm_support.pyindex 6cc35977101..6216296949a 100644--- a/tests/studio/install/test_rocm_support.py+++ b/tests/studio/install/test_rocm_support.py@@ -4407,9 +4407,11 @@ class TestHardwareAmdBranching:source = hw_path.read_text(encoding = "utf-8")func_start = source.find("def get_gpu_utilization")func_body = source[func_start : source.find("\ndef ", func_start + 1)]- assert "_smi_query(" in func_body- assert '"get_visible_gpu_utilization"' in func_body+ assert "_smi_visible_utilization(" in func_bodyassert "_reconcile_rocm_unified_memory" in func_body+ helper_start = source.find("def _smi_visible_utilization")+ helper = source[helper_start : source.find("\ndef ", helper_start + 1)]+ assert re.search(r'_smi_query\(\s*"get_visible_gpu_utilization"', helper)smi = source[source.find("def _smi_query") : source.find("\ndef ", source.find("def _smi_query") + 1)]@@ -4422,10 +4424,10 @@ class TestHardwareAmdBranching:source = hw_path.read_text(encoding = "utf-8")func_start = source.find("def get_visible_gpu_utilization")func_body = source[func_start : source.find("\ndef ", func_start + 1)]- # The dispatcher call may wrap; allow whitespace before the func name arg.- import re as _re-- assert _re.search(r'_smi_query\(\s*"get_visible_gpu_utilization"', func_body)+ assert "_smi_visible_utilization(" in func_body+ helper_start = source.find("def _smi_visible_utilization")+ helper = source[helper_start : source.find("\ndef ", helper_start + 1)]+ assert re.search(r'_smi_query\(\s*"get_visible_gpu_utilization"', helper)smi = source[source.find("def _smi_query") : source.find("\ndef ", source.find("def _smi_query") + 1)]