Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: reuse Windows ROCm Get-Counter readings for display polls like amd-smi (#17306)

Daniel Han3b6339d1f74f
diff --git a/studio/backend/tests/test_rocm_windows_counter_display_cache.py b/studio/backend/tests/test_rocm_windows_counter_display_cache.py
new file mode 100644
index 00000000000..99760428ad8
--- /dev/null
+++ b/studio/backend/tests/test_rocm_windows_counter_display_cache.py
@@ -0,0 +1,260 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Windows ROCm Get-Counter reads: display polls share one combined PowerShell call per window; fit reads stay fresh."""
+
+import threading
+import time
+import types
+
+import pytest
+
+from utils import gpu_memory_events
+from utils.hardware import gpu_query
+from utils.hardware import hardware as hw
+
+_DED = "luid_0x00000000_0x0000d1a2_phys_0|1073741824\nluid_0x00000000_0x0000d1a3_phys_0|0\n"
+_SHARED = "luid_0x00000000_0x0000d1a2_phys_0|2048\n"
+_UTIL = "pid_42_luid_0x00000000_0x0000d1a2_phys_0_eng_0_engtype_3D|37.5\n"
+# One Get-Counter spawn on the Radeon 8060S CI box took about 2 s.
+_SPAWN_S = 2.0
+
+
+class _Clock:
+ """Stands in for ``hw.time``: monotonic() is a fake clock, everything else is the real module."""
+
+ def __init__(self):
+ self.now = 1000.0
+
+ def monotonic(self):
+ return self.now
+
+ def __getattr__(self, name):
+ return getattr(time, name)
+
+
+def _tagged(tag, body):
+ return "".join(f"{tag}|{line}\n" for line in body.splitlines())
+
+
+def counters(monkeypatch):
+ monkeypatch.setattr(hw, "_rocm_win_counter_cache", {}, raising = False)
+ monkeypatch.setattr(hw.platform, "system", lambda: "Windows")
+ monkeypatch.setattr(gpu_query, "enabled", lambda: True)
+ monkeypatch.setattr(gpu_query, "ttl_for", lambda kind: 3.0)
+ clock = _Clock()
+ monkeypatch.setattr(hw, "time", clock)
+ calls = []
+ state = types.SimpleNamespace(calls = calls, clock = clock, spawn_s = _SPAWN_S, drop = ())
+
+ def run(argv, **kwargs):
+ script = argv[-1]
+ calls.append(script)
+ clock.now += state.spawn_s
+ if script.count("Get-Counter") != 1:
+ raise AssertionError(script)
+ if "Get-Counter -Counter" in script:
+ parts = {"D": _DED, "S": _SHARED, "U": _UTIL}
+ out = "".join(_tagged(t, body) for t, body in parts.items() if t not in state.drop)
+ elif "GPU Engine" in script:
+ out = _UTIL
+ elif "Shared Usage" in script:
+ out = _SHARED
+ else:
+ out = _DED
+ return types.SimpleNamespace(returncode = 0, stdout = out or "__NONE__\n", stderr = "")
+
+ monkeypatch.setattr(hw.subprocess, "run", run)
+ return state
+
+
+def _combined(calls):
+ return [c for c in calls if "Get-Counter -Counter" in c]
+
+
+def _read_all():
+ return (
+ hw._rocm_windows_perf_counter_vram_by_adapter(),
+ hw._rocm_windows_perf_counter_vram_by_adapter("Shared Usage"),
+ hw._rocm_windows_perf_counter_gpu_util_pct(),
+ hw._rocm_windows_perf_counter_gpu_util_pct(0xD1A2),
+ )
+
+
+def test_back_to_back_display_polls_spawn_once(counters):
+ fresh = _read_all()
+ assert len(counters.calls) == 4 and not _combined(counters.calls)
+ with gpu_query.display_reads(max_stale = 0):
+ for _ in range(10):
+ assert _read_all() == fresh
+ # Ten polls of all three counters: one combined spawn, against 30 per-counter spawns before.
+ assert len(counters.calls) == 4 + 1
+ assert len(_combined(counters.calls)) == 1
+ assert fresh[0] == [
+ ("luid_0x00000000_0x0000d1a2_phys_0", 1073741824.0),
+ ("luid_0x00000000_0x0000d1a3_phys_0", 0.0),
+ ]
+ assert fresh[1] == [("luid_0x00000000_0x0000d1a2_phys_0", 2048.0)]
+ assert fresh[2] == fresh[3] == 37.5
+
+
+def test_reading_outlives_its_own_probe(counters):
+ # 2 s spawns, 1 s between polls: the reading lives 4 x 2 s = 8 s from completion, not the 3 s TTL.
+ with gpu_query.display_reads(max_stale = 0):
+ spawned_at = []
+ for poll in range(20):
+ before = len(counters.calls)
+ _read_all()
+ if len(counters.calls) > before:
+ spawned_at.append(poll)
+ counters.clock.now += 1.0
+ assert spawned_at == [0, 8, 16]
+ assert len(counters.calls) == 3
+
+
+def test_window_is_the_ttl_for_fast_spawns_and_capped_for_slow_ones(counters):
+ counters.spawn_s = 0.1
+ with gpu_query.display_reads(max_stale = 0):
+ _read_all()
+ counters.clock.now += 2.9
+ _read_all()
+ assert len(counters.calls) == 1
+ counters.clock.now += 0.1
+ _read_all()
+ assert len(counters.calls) == 2
+ counters.spawn_s = 4.5
+ counters.clock.now += 3.0
+ _read_all()
+ assert len(counters.calls) == 3
+ counters.clock.now += 9.9
+ _read_all()
+ assert len(counters.calls) == 3
+ counters.clock.now += 0.1
+ _read_all()
+ assert len(counters.calls) == 4
+
+
+def test_fit_and_fresh_reads_always_spawn_their_own_counter(counters):
+ with gpu_query.display_reads(max_stale = 0):
+ hw._rocm_windows_perf_counter_vram_by_adapter()
+ assert len(counters.calls) == 1
+ for _ in range(3):
+ hw._rocm_windows_perf_counter_vram_by_adapter()
+ assert len(counters.calls) == 4
+ with gpu_query.display_reads(max_stale = 0), gpu_query.fresh_reads():
+ hw._rocm_windows_perf_counter_vram_by_adapter()
+ hw._rocm_windows_perf_counter_gpu_util_pct()
+ assert len(counters.calls) == 6
+ assert len(_combined(counters.calls)) == 1
+ assert all(c.startswith("$s=(Get-Counter '") for c in counters.calls[1:])
+
+
+def test_display_cache_drops_on_memory_event_and_ttl_zero(counters, monkeypatch):
+ with gpu_query.display_reads(max_stale = 0):
+ hw._rocm_windows_perf_counter_vram_by_adapter()
+ hw._rocm_windows_perf_counter_gpu_util_pct()
+ assert len(counters.calls) == 1
+ gpu_memory_events.invalidate_gpu_memory("model loaded")
+ hw._rocm_windows_perf_counter_vram_by_adapter()
+ assert len(counters.calls) == 2
+ monkeypatch.setattr(gpu_query, "ttl_for", lambda kind: 0.0)
+ hw._rocm_windows_perf_counter_vram_by_adapter()
+ hw._rocm_windows_perf_counter_vram_by_adapter()
+ assert len(counters.calls) == 4
+ assert len(_combined(counters.calls)) == 2
+
+
+def test_failed_combined_reads_fall_back_and_are_never_cached(counters, monkeypatch):
+ def failing(argv, **kwargs):
+ counters.calls.append(argv[-1])
+ return types.SimpleNamespace(returncode = 1, stdout = "", stderr = "boom")
+
+ monkeypatch.setattr(hw.subprocess, "run", failing)
+ with gpu_query.display_reads(max_stale = 0):
+ assert hw._rocm_windows_perf_counter_vram_by_adapter() is None
+ assert hw._rocm_windows_perf_counter_vram_by_adapter() is None
+ assert hw._rocm_windows_perf_counter_gpu_util_pct() is None
+ # One combined attempt, then own calls: nothing failed is served.
+ assert len(_combined(counters.calls)) == 1
+ assert len(counters.calls) == 4
+ assert hw._rocm_win_counter_cache["display"][3] == {}
+ counters.clock.now += hw._ROCM_WIN_COUNTER_EMPTY_RETRY_S - 1
+ with gpu_query.display_reads(max_stale = 0):
+ assert hw._rocm_windows_perf_counter_vram_by_adapter() is None
+ assert len(_combined(counters.calls)) == 1
+ # The empty reading still expires, and a GPU memory event drops it too.
+ counters.clock.now += 1
+ with gpu_query.display_reads(max_stale = 0):
+ assert hw._rocm_windows_perf_counter_vram_by_adapter() is None
+ gpu_memory_events.invalidate_gpu_memory("model loaded")
+ assert hw._rocm_windows_perf_counter_vram_by_adapter() is None
+ assert len(_combined(counters.calls)) == 3
+
+
+def test_a_hung_combined_call_is_not_retried_by_every_read(counters, monkeypatch):
+ def hung(argv, **kwargs):
+ counters.calls.append(argv[-1])
+ counters.clock.now += 5.0
+ raise hw.subprocess.TimeoutExpired(argv, 5)
+
+ monkeypatch.setattr(hw.subprocess, "run", hung)
+ with gpu_query.display_reads(max_stale = 0):
+ assert _read_all() == (None, None, None, None)
+ # Base ran 4 own calls (20 s); one combined attempt rides on top (25 s), not one per read (40 s).
+ assert len(_combined(counters.calls)) == 1
+ assert len(counters.calls) == 5
+
+
+def test_a_counter_missing_from_the_combined_reading_reads_its_own(counters):
+ counters.drop = ("U",)
+ fresh = _read_all()
+ with gpu_query.display_reads(max_stale = 0):
+ for _ in range(2):
+ assert _read_all() == fresh
+ combined = _combined(counters.calls)
+ own_util = [c for c in counters.calls[4:] if c not in combined and "GPU Engine" in c]
+ assert len(combined) == 1
+ assert len(own_util) == 4
+ assert len(counters.calls) == 4 + 1 + 4
+
+
+def test_concurrent_display_polls_share_one_spawn(counters, monkeypatch):
+ real_run = hw.subprocess.run
+ gate = threading.Event()
+
+ def slow(argv, **kwargs):
+ gate.wait(5)
+ return real_run(argv, **kwargs)
+
+ monkeypatch.setattr(hw.subprocess, "run", slow)
+ results = []
+
+ def poll():
+ with gpu_query.display_reads(max_stale = 0):
+ results.append(_read_all())
+
+ threads = [threading.Thread(target = poll) for _ in range(4)]
+ for t in threads:
+ t.start()
+ gate.set()
+ for t in threads:
+ t.join(10)
+ assert len(results) == 4 and len(set(map(repr, results))) == 1
+ assert len(counters.calls) == 1
+
+
+def test_combined_script_keeps_each_counter_path_and_format():
+ ps = hw._rocm_windows_combined_counter_ps()
+ assert ps.count("Get-Counter") == 1
+ for _, (tag, path, value) in hw._ROCM_WIN_DISPLAY_COUNTERS.items():
+ assert f"'{path}'" in ps
+ assert f"'{tag}|{{0}}|{{1}}' -f $_.InstanceName,{value}" in ps
+ split = hw._rocm_windows_split_combined(
+ _tagged("D", _DED) + _tagged("S", _SHARED) + _tagged("U", _UTIL) + "noise\n"
+ )
+ assert split == {
+ ("vram", "Dedicated Usage"): _DED,
+ ("vram", "Shared Usage"): _SHARED,
+ ("util", "3D"): _UTIL,
+ }
diff --git a/studio/backend/tests/test_rocm_windows_vram_7072.py b/studio/backend/tests/test_rocm_windows_vram_7072.py
index cb7fa159c8f..a90374d91a8 100644
--- a/studio/backend/tests/test_rocm_windows_vram_7072.py
+++ b/studio/backend/tests/test_rocm_windows_vram_7072.py
@@ -2165,3 +2165,70 @@ def test_a_krackan_point_2_with_no_arch_attrs_or_flag_is_unified_by_name(win_roc
assert hw._rocm_props_unified_status(_P()) is True
assert hw._rocm_props_are_positively_unified(_P()) is True
+
+
+def test_display_polls_share_one_combined_counter_read(win_rocm_solo, monkeypatch):
+ """2 s Get-Counter spawns: one combined spawn serves every counter for 4x its duration."""
+ import time
+
+ from utils.hardware import gpu_query
+
+ class Clock:
+ now = 500.0
+
+ def monotonic(self):
+ return self.now
+
+ def __getattr__(self, name):
+ return getattr(time, name)
+
+ clock = Clock()
+ monkeypatch.setattr(hw, "time", clock)
+ monkeypatch.setattr(hw, "_rocm_win_counter_cache", {}, raising = False)
+ monkeypatch.setenv("UNSLOTH_GPU_QUERY_CACHE", "1")
+ monkeypatch.delenv("UNSLOTH_GPU_QUERY_DISPLAY_TTL", raising = False)
+ adapters = _adapter_output(SOLO_ADAPTERS)
+ engines = _engine_output(_DEFAULT_ENGINES)
+ fake = _subprocess_run(adapter_output = adapters, util_output = engines)
+ spawns = []
+
+ def run(cmd, *a, **k):
+ script = cmd[-1] if isinstance(cmd, list) else str(cmd)
+ if "Get-Counter" not in script:
+ return fake(cmd, *a, **k)
+ spawns.append(script)
+ clock.now += 2.0
+ if "Get-Counter -Counter" in script:
+ out = "".join(
+ f"{tag}|{line}\n"
+ for tag, body in (("D", adapters), ("S", adapters), ("U", engines))
+ for line in body.splitlines()
+ )
+ return subprocess.CompletedProcess(args = cmd, returncode = 0, stdout = out, stderr = "")
+ return fake(cmd, *a, **k)
+
+ monkeypatch.setattr(hw.subprocess, "run", run)
+
+ def poll():
+ return hw.get_visible_gpu_utilization(), hw.get_gpu_utilization()
+
+ fresh = poll()
+ per_poll = len(spawns)
+ assert per_poll >= 3
+ assert not any("Get-Counter -Counter" in s for s in spawns)
+ with gpu_query.display_reads(max_stale = 0):
+ for _ in range(10):
+ assert poll() == fresh
+ # Ten back-to-back polls: one spawn, against 10 * per_poll before.
+ assert len(spawns) - per_poll == 1
+ # The reading lives 4 x 2 s from completion: still served at 7.9 s, refreshed at 8 s.
+ clock.now += 7.9
+ assert poll() == fresh
+ assert len(spawns) - per_poll == 1
+ clock.now += 0.1
+ assert poll() == fresh
+ assert len(spawns) - per_poll == 2
+ before = len(spawns)
+ assert poll() == fresh
+ assert len(spawns) - before == per_poll
+ assert not any("Get-Counter -Counter" in s for s in spawns[before:])
diff --git a/studio/backend/utils/hardware/hardware.py b/studio/backend/utils/hardware/hardware.py
index ae8ce0fddf5..0f740d20615 100644
--- a/studio/backend/utils/hardware/hardware.py
+++ b/studio/backend/utils/hardware/hardware.py
@@ -3620,6 +3620,125 @@ def _engine_instance_luid(instance_name: str) -> Optional[int]:
return _parse_adapter_luid(instance_name[head:])
+_rocm_win_counter_lock = threading.Lock()
+# (completed_at, gpu_memory_events generation, spawn seconds, {key: stdout})
+_rocm_win_counter_cache: dict[str, tuple[float, int, float, dict[tuple[str, str], str]]] = {}
+_ROCM_WIN_COUNTER_SPACING = 4.0
+_ROCM_WIN_COUNTER_MAX_AGE_S = 10.0
+# Empty combined reading (localized names, hung PowerShell): each counter runs its own call until this retry.
+_ROCM_WIN_COUNTER_EMPTY_RETRY_S = 60.0
+_ROCM_WIN_DISPLAY_COUNTERS: dict[tuple[str, str], tuple[str, str, str]] = {
+ ("vram", "Dedicated Usage"): (
+ "D",
+ "\\GPU Adapter Memory(*)\\Dedicated Usage",
+ "[int64]$_.CookedValue",
+ ),
+ ("vram", "Shared Usage"): (
+ "S",
+ "\\GPU Adapter Memory(*)\\Shared Usage",
+ "[int64]$_.CookedValue",
+ ),
+ ("util", "3D"): (
+ "U",
+ "\\GPU Engine(*engtype_3D*)\\Utilization Percentage",
+ "$_.CookedValue",
+ ),
+}
+
+
+def _rocm_windows_combined_counter_ps() -> str:
+ paths = ",".join(f"'{path}'" for _, path, _ in _ROCM_WIN_DISPLAY_COUNTERS.values())
+ branches = "else".join(
+ f"if($p.EndsWith('{path[path.rindex(chr(92)):].lower()}'))"
+ f"{{'{tag}|{{0}}|{{1}}' -f $_.InstanceName,{value}}}"
+ for tag, path, value in _ROCM_WIN_DISPLAY_COUNTERS.values()
+ )
+ return (
+ f"$s=(Get-Counter -Counter {paths} -ErrorAction SilentlyContinue).CounterSamples;"
+ f"if($s){{$s|ForEach-Object{{$p=$_.Path.ToLower();{branches}}}}}"
+ "else{'__NONE__'}"
+ )
+
+
+def _rocm_windows_split_combined(stdout: str) -> dict[tuple[str, str], str]:
+ by_tag: dict[str, list[str]] = {}
+ for line in stdout.splitlines():
+ tag, sep, rest = line.strip().partition("|")
+ if sep:
+ by_tag.setdefault(tag, []).append(rest)
+ return {
+ key: "\n".join(by_tag[tag]) + "\n"
+ for key, (tag, _, _) in _ROCM_WIN_DISPLAY_COUNTERS.items()
+ if by_tag.get(tag)
+ }
+
+
+def _rocm_windows_counter_stdout(key: tuple[str, str], ps: str) -> Optional[str]:
+ """stdout of ``ps`` or None. Display reads share one combined Get-Counter, kept max(TTL, 4x spawn time) until a GPU memory event; misses fall back to ``ps``, failures are never served."""
+ try:
+ from utils import gpu_memory_events
+ from utils.hardware import gpu_query
+
+ use_cache = gpu_query.display_cache_allowed() and key in _ROCM_WIN_DISPLAY_COUNTERS
+ ttl = gpu_query.ttl_for(gpu_query.DISPLAY)
+ except Exception:
+ use_cache = False
+ if use_cache and ttl > 0:
+ stdout = None
+ try:
+ with _rocm_win_counter_lock:
+ cached = _rocm_win_counter_cache.get("display")
+ generation = gpu_memory_events.generation()
+ if (
+ cached is None
+ or cached[1] != generation
+ or (
+ time.monotonic() - cached[0]
+ >= (
+ min(
+ _ROCM_WIN_COUNTER_MAX_AGE_S,
+ max(ttl, _ROCM_WIN_COUNTER_SPACING * cached[2]),
+ )
+ if cached[3]
+ else _ROCM_WIN_COUNTER_EMPTY_RETRY_S
+ )
+ )
+ ):
+ _rocm_win_counter_cache.pop("display", None)
+ started = time.monotonic()
+ try:
+ combined = _rocm_windows_counter_stdout_uncached(
+ _rocm_windows_combined_counter_ps()
+ )
+ except Exception:
+ combined = None
+ done = time.monotonic()
+ readings = _rocm_windows_split_combined(combined) if combined else {}
+ cached = (done, generation, done - started, readings)
+ _rocm_win_counter_cache["display"] = cached
+ if cached is not None:
+ stdout = cached[3].get(key)
+ except Exception:
+ stdout = None
+ if stdout is not None:
+ return stdout
+ return _rocm_windows_counter_stdout_uncached(ps)
+
+
+def _rocm_windows_counter_stdout_uncached(ps: str) -> Optional[str]:
+ r = subprocess.run(
+ ["powershell", "-NoProfile", "-NonInteractive", "-Command", ps],
+ capture_output = True,
+ text = True,
+ encoding = "utf-8",
+ errors = "replace",
+ timeout = 5,
+ )
+ if r.returncode != 0 or not r.stdout.strip():
+ return None
+ return r.stdout
+
+
def _rocm_windows_perf_counter_gpu_util_pct(luid: Optional[int] = None) -> Optional[float]:
"""AMD GPU compute utilization via Windows Performance Counters (3D engine nodes). ``luid`` narrows the sum to one adapter's engines, matched here rather than in the counter path so tests can reach it."""
if platform.system() != "Windows":
@@ -3631,19 +3750,12 @@ def _rocm_windows_perf_counter_gpu_util_pct(luid: Optional[int] = None) -> Optio
"if($s){$s|ForEach-Object{'{0}|{1}' -f $_.InstanceName,$_.CookedValue}}"
"else{'__NONE__'}"
)
- r = subprocess.run(
- ["powershell", "-NoProfile", "-NonInteractive", "-Command", ps],
- capture_output = True,
- text = True,
- encoding = "utf-8",
- errors = "replace",
- timeout = 5,
- )
- if r.returncode != 0 or not r.stdout.strip():
+ stdout = _rocm_windows_counter_stdout(("util", "3D"), ps)
+ if stdout is None:
return None
total = 0.0
matched = False
- for line in r.stdout.splitlines():
+ for line in stdout.splitlines():
line = line.strip()
if not line or line == "__NONE__" or "|" not in line:
continue
@@ -3808,18 +3920,11 @@ def _rocm_windows_perf_counter_vram_by_adapter(
"if($s){$s|ForEach-Object{'{0}|{1}' -f $_.InstanceName,[int64]$_.CookedValue}}"
"else{'__NONE__'}"
)
- r = subprocess.run(
- ["powershell", "-NoProfile", "-NonInteractive", "-Command", ps],
- capture_output = True,
- text = True,
- encoding = "utf-8",
- errors = "replace",
- timeout = 5,
- )
- if r.returncode != 0 or not r.stdout.strip():
+ stdout = _rocm_windows_counter_stdout(("vram", counter), ps)
+ if stdout is None:
return None
adapters: list[tuple[str, float]] = []
- for line in r.stdout.splitlines():
+ for line in stdout.splitlines():
line = line.strip()
if not line or line == "__NONE__" or "|" not in line:
continue