Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: reuse Windows ROCm Get-Counter readings for display polls like amd-smi (#17306)
Daniel Han3b6339d1f74f
diff --git a/studio/backend/tests/test_rocm_windows_counter_display_cache.py b/studio/backend/tests/test_rocm_windows_counter_display_cache.pynew file mode 100644index 00000000000..99760428ad8--- /dev/null+++ b/studio/backend/tests/test_rocm_windows_counter_display_cache.py@@ -0,0 +1,260 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Windows ROCm Get-Counter reads: display polls share one combined PowerShell call per window; fit reads stay fresh."""++import threading+import time+import types++import pytest++from utils import gpu_memory_events+from utils.hardware import gpu_query+from utils.hardware import hardware as hw++_DED = "luid_0x00000000_0x0000d1a2_phys_0|1073741824\nluid_0x00000000_0x0000d1a3_phys_0|0\n"+_SHARED = "luid_0x00000000_0x0000d1a2_phys_0|2048\n"+_UTIL = "pid_42_luid_0x00000000_0x0000d1a2_phys_0_eng_0_engtype_3D|37.5\n"+# One Get-Counter spawn on the Radeon 8060S CI box took about 2 s.+_SPAWN_S = 2.0+++class _Clock:+ """Stands in for ``hw.time``: monotonic() is a fake clock, everything else is the real module."""++ def __init__(self):+ self.now = 1000.0++ def monotonic(self):+ return self.now++ def __getattr__(self, name):+ return getattr(time, name)+++def _tagged(tag, body):+ return "".join(f"{tag}|{line}\n" for line in body.splitlines())+++def counters(monkeypatch):+ monkeypatch.setattr(hw, "_rocm_win_counter_cache", {}, raising = False)+ monkeypatch.setattr(hw.platform, "system", lambda: "Windows")+ monkeypatch.setattr(gpu_query, "enabled", lambda: True)+ monkeypatch.setattr(gpu_query, "ttl_for", lambda kind: 3.0)+ clock = _Clock()+ monkeypatch.setattr(hw, "time", clock)+ calls = []+ state = types.SimpleNamespace(calls = calls, clock = clock, spawn_s = _SPAWN_S, drop = ())++ def run(argv, **kwargs):+ script = argv[-1]+ calls.append(script)+ clock.now += state.spawn_s+ if script.count("Get-Counter") != 1:+ raise AssertionError(script)+ if "Get-Counter -Counter" in script:+ parts = {"D": _DED, "S": _SHARED, "U": _UTIL}+ out = "".join(_tagged(t, body) for t, body in parts.items() if t not in state.drop)+ elif "GPU Engine" in script:+ out = _UTIL+ elif "Shared Usage" in script:+ out = _SHARED+ else:+ out = _DED+ return types.SimpleNamespace(returncode = 0, stdout = out or "__NONE__\n", stderr = "")++ monkeypatch.setattr(hw.subprocess, "run", run)+ return state+++def _combined(calls):+ return [c for c in calls if "Get-Counter -Counter" in c]+++def _read_all():+ return (+ hw._rocm_windows_perf_counter_vram_by_adapter(),+ hw._rocm_windows_perf_counter_vram_by_adapter("Shared Usage"),+ hw._rocm_windows_perf_counter_gpu_util_pct(),+ hw._rocm_windows_perf_counter_gpu_util_pct(0xD1A2),+ )+++def test_back_to_back_display_polls_spawn_once(counters):+ fresh = _read_all()+ assert len(counters.calls) == 4 and not _combined(counters.calls)+ with gpu_query.display_reads(max_stale = 0):+ for _ in range(10):+ assert _read_all() == fresh+ # Ten polls of all three counters: one combined spawn, against 30 per-counter spawns before.+ assert len(counters.calls) == 4 + 1+ assert len(_combined(counters.calls)) == 1+ assert fresh[0] == [+ ("luid_0x00000000_0x0000d1a2_phys_0", 1073741824.0),+ ("luid_0x00000000_0x0000d1a3_phys_0", 0.0),+ ]+ assert fresh[1] == [("luid_0x00000000_0x0000d1a2_phys_0", 2048.0)]+ assert fresh[2] == fresh[3] == 37.5+++def test_reading_outlives_its_own_probe(counters):+ # 2 s spawns, 1 s between polls: the reading lives 4 x 2 s = 8 s from completion, not the 3 s TTL.+ with gpu_query.display_reads(max_stale = 0):+ spawned_at = []+ for poll in range(20):+ before = len(counters.calls)+ _read_all()+ if len(counters.calls) > before:+ spawned_at.append(poll)+ counters.clock.now += 1.0+ assert spawned_at == [0, 8, 16]+ assert len(counters.calls) == 3+++def test_window_is_the_ttl_for_fast_spawns_and_capped_for_slow_ones(counters):+ counters.spawn_s = 0.1+ with gpu_query.display_reads(max_stale = 0):+ _read_all()+ counters.clock.now += 2.9+ _read_all()+ assert len(counters.calls) == 1+ counters.clock.now += 0.1+ _read_all()+ assert len(counters.calls) == 2+ counters.spawn_s = 4.5+ counters.clock.now += 3.0+ _read_all()+ assert len(counters.calls) == 3+ counters.clock.now += 9.9+ _read_all()+ assert len(counters.calls) == 3+ counters.clock.now += 0.1+ _read_all()+ assert len(counters.calls) == 4+++def test_fit_and_fresh_reads_always_spawn_their_own_counter(counters):+ with gpu_query.display_reads(max_stale = 0):+ hw._rocm_windows_perf_counter_vram_by_adapter()+ assert len(counters.calls) == 1+ for _ in range(3):+ hw._rocm_windows_perf_counter_vram_by_adapter()+ assert len(counters.calls) == 4+ with gpu_query.display_reads(max_stale = 0), gpu_query.fresh_reads():+ hw._rocm_windows_perf_counter_vram_by_adapter()+ hw._rocm_windows_perf_counter_gpu_util_pct()+ assert len(counters.calls) == 6+ assert len(_combined(counters.calls)) == 1+ assert all(c.startswith("$s=(Get-Counter '") for c in counters.calls[1:])+++def test_display_cache_drops_on_memory_event_and_ttl_zero(counters, monkeypatch):+ with gpu_query.display_reads(max_stale = 0):+ hw._rocm_windows_perf_counter_vram_by_adapter()+ hw._rocm_windows_perf_counter_gpu_util_pct()+ assert len(counters.calls) == 1+ gpu_memory_events.invalidate_gpu_memory("model loaded")+ hw._rocm_windows_perf_counter_vram_by_adapter()+ assert len(counters.calls) == 2+ monkeypatch.setattr(gpu_query, "ttl_for", lambda kind: 0.0)+ hw._rocm_windows_perf_counter_vram_by_adapter()+ hw._rocm_windows_perf_counter_vram_by_adapter()+ assert len(counters.calls) == 4+ assert len(_combined(counters.calls)) == 2+++def test_failed_combined_reads_fall_back_and_are_never_cached(counters, monkeypatch):+ def failing(argv, **kwargs):+ counters.calls.append(argv[-1])+ return types.SimpleNamespace(returncode = 1, stdout = "", stderr = "boom")++ monkeypatch.setattr(hw.subprocess, "run", failing)+ with gpu_query.display_reads(max_stale = 0):+ assert hw._rocm_windows_perf_counter_vram_by_adapter() is None+ assert hw._rocm_windows_perf_counter_vram_by_adapter() is None+ assert hw._rocm_windows_perf_counter_gpu_util_pct() is None+ # One combined attempt, then own calls: nothing failed is served.+ assert len(_combined(counters.calls)) == 1+ assert len(counters.calls) == 4+ assert hw._rocm_win_counter_cache["display"][3] == {}+ counters.clock.now += hw._ROCM_WIN_COUNTER_EMPTY_RETRY_S - 1+ with gpu_query.display_reads(max_stale = 0):+ assert hw._rocm_windows_perf_counter_vram_by_adapter() is None+ assert len(_combined(counters.calls)) == 1+ # The empty reading still expires, and a GPU memory event drops it too.+ counters.clock.now += 1+ with gpu_query.display_reads(max_stale = 0):+ assert hw._rocm_windows_perf_counter_vram_by_adapter() is None+ gpu_memory_events.invalidate_gpu_memory("model loaded")+ assert hw._rocm_windows_perf_counter_vram_by_adapter() is None+ assert len(_combined(counters.calls)) == 3+++def test_a_hung_combined_call_is_not_retried_by_every_read(counters, monkeypatch):+ def hung(argv, **kwargs):+ counters.calls.append(argv[-1])+ counters.clock.now += 5.0+ raise hw.subprocess.TimeoutExpired(argv, 5)++ monkeypatch.setattr(hw.subprocess, "run", hung)+ with gpu_query.display_reads(max_stale = 0):+ assert _read_all() == (None, None, None, None)+ # Base ran 4 own calls (20 s); one combined attempt rides on top (25 s), not one per read (40 s).+ assert len(_combined(counters.calls)) == 1+ assert len(counters.calls) == 5+++def test_a_counter_missing_from_the_combined_reading_reads_its_own(counters):+ counters.drop = ("U",)+ fresh = _read_all()+ with gpu_query.display_reads(max_stale = 0):+ for _ in range(2):+ assert _read_all() == fresh+ combined = _combined(counters.calls)+ own_util = [c for c in counters.calls[4:] if c not in combined and "GPU Engine" in c]+ assert len(combined) == 1+ assert len(own_util) == 4+ assert len(counters.calls) == 4 + 1 + 4+++def test_concurrent_display_polls_share_one_spawn(counters, monkeypatch):+ real_run = hw.subprocess.run+ gate = threading.Event()++ def slow(argv, **kwargs):+ gate.wait(5)+ return real_run(argv, **kwargs)++ monkeypatch.setattr(hw.subprocess, "run", slow)+ results = []++ def poll():+ with gpu_query.display_reads(max_stale = 0):+ results.append(_read_all())++ threads = [threading.Thread(target = poll) for _ in range(4)]+ for t in threads:+ t.start()+ gate.set()+ for t in threads:+ t.join(10)+ assert len(results) == 4 and len(set(map(repr, results))) == 1+ assert len(counters.calls) == 1+++def test_combined_script_keeps_each_counter_path_and_format():+ ps = hw._rocm_windows_combined_counter_ps()+ assert ps.count("Get-Counter") == 1+ for _, (tag, path, value) in hw._ROCM_WIN_DISPLAY_COUNTERS.items():+ assert f"'{path}'" in ps+ assert f"'{tag}|{{0}}|{{1}}' -f $_.InstanceName,{value}" in ps+ split = hw._rocm_windows_split_combined(+ _tagged("D", _DED) + _tagged("S", _SHARED) + _tagged("U", _UTIL) + "noise\n"+ )+ assert split == {+ ("vram", "Dedicated Usage"): _DED,+ ("vram", "Shared Usage"): _SHARED,+ ("util", "3D"): _UTIL,+ }diff --git a/studio/backend/tests/test_rocm_windows_vram_7072.py b/studio/backend/tests/test_rocm_windows_vram_7072.pyindex cb7fa159c8f..a90374d91a8 100644--- a/studio/backend/tests/test_rocm_windows_vram_7072.py+++ b/studio/backend/tests/test_rocm_windows_vram_7072.py@@ -2165,3 +2165,70 @@ def test_a_krackan_point_2_with_no_arch_attrs_or_flag_is_unified_by_name(win_rocassert hw._rocm_props_unified_status(_P()) is Trueassert hw._rocm_props_are_positively_unified(_P()) is True+++def test_display_polls_share_one_combined_counter_read(win_rocm_solo, monkeypatch):+ """2 s Get-Counter spawns: one combined spawn serves every counter for 4x its duration."""+ import time++ from utils.hardware import gpu_query++ class Clock:+ now = 500.0++ def monotonic(self):+ return self.now++ def __getattr__(self, name):+ return getattr(time, name)++ clock = Clock()+ monkeypatch.setattr(hw, "time", clock)+ monkeypatch.setattr(hw, "_rocm_win_counter_cache", {}, raising = False)+ monkeypatch.setenv("UNSLOTH_GPU_QUERY_CACHE", "1")+ monkeypatch.delenv("UNSLOTH_GPU_QUERY_DISPLAY_TTL", raising = False)+ adapters = _adapter_output(SOLO_ADAPTERS)+ engines = _engine_output(_DEFAULT_ENGINES)+ fake = _subprocess_run(adapter_output = adapters, util_output = engines)+ spawns = []++ def run(cmd, *a, **k):+ script = cmd[-1] if isinstance(cmd, list) else str(cmd)+ if "Get-Counter" not in script:+ return fake(cmd, *a, **k)+ spawns.append(script)+ clock.now += 2.0+ if "Get-Counter -Counter" in script:+ out = "".join(+ f"{tag}|{line}\n"+ for tag, body in (("D", adapters), ("S", adapters), ("U", engines))+ for line in body.splitlines()+ )+ return subprocess.CompletedProcess(args = cmd, returncode = 0, stdout = out, stderr = "")+ return fake(cmd, *a, **k)++ monkeypatch.setattr(hw.subprocess, "run", run)++ def poll():+ return hw.get_visible_gpu_utilization(), hw.get_gpu_utilization()++ fresh = poll()+ per_poll = len(spawns)+ assert per_poll >= 3+ assert not any("Get-Counter -Counter" in s for s in spawns)+ with gpu_query.display_reads(max_stale = 0):+ for _ in range(10):+ assert poll() == fresh+ # Ten back-to-back polls: one spawn, against 10 * per_poll before.+ assert len(spawns) - per_poll == 1+ # The reading lives 4 x 2 s from completion: still served at 7.9 s, refreshed at 8 s.+ clock.now += 7.9+ assert poll() == fresh+ assert len(spawns) - per_poll == 1+ clock.now += 0.1+ assert poll() == fresh+ assert len(spawns) - per_poll == 2+ before = len(spawns)+ assert poll() == fresh+ assert len(spawns) - before == per_poll+ assert not any("Get-Counter -Counter" in s for s in spawns[before:])diff --git a/studio/backend/utils/hardware/hardware.py b/studio/backend/utils/hardware/hardware.pyindex ae8ce0fddf5..0f740d20615 100644--- a/studio/backend/utils/hardware/hardware.py+++ b/studio/backend/utils/hardware/hardware.py@@ -3620,6 +3620,125 @@ def _engine_instance_luid(instance_name: str) -> Optional[int]:return _parse_adapter_luid(instance_name[head:])+_rocm_win_counter_lock = threading.Lock()+# (completed_at, gpu_memory_events generation, spawn seconds, {key: stdout})+_rocm_win_counter_cache: dict[str, tuple[float, int, float, dict[tuple[str, str], str]]] = {}+_ROCM_WIN_COUNTER_SPACING = 4.0+_ROCM_WIN_COUNTER_MAX_AGE_S = 10.0+# Empty combined reading (localized names, hung PowerShell): each counter runs its own call until this retry.+_ROCM_WIN_COUNTER_EMPTY_RETRY_S = 60.0+_ROCM_WIN_DISPLAY_COUNTERS: dict[tuple[str, str], tuple[str, str, str]] = {+ ("vram", "Dedicated Usage"): (+ "D",+ "\\GPU Adapter Memory(*)\\Dedicated Usage",+ "[int64]$_.CookedValue",+ ),+ ("vram", "Shared Usage"): (+ "S",+ "\\GPU Adapter Memory(*)\\Shared Usage",+ "[int64]$_.CookedValue",+ ),+ ("util", "3D"): (+ "U",+ "\\GPU Engine(*engtype_3D*)\\Utilization Percentage",+ "$_.CookedValue",+ ),+}+++def _rocm_windows_combined_counter_ps() -> str:+ paths = ",".join(f"'{path}'" for _, path, _ in _ROCM_WIN_DISPLAY_COUNTERS.values())+ branches = "else".join(+ f"if($p.EndsWith('{path[path.rindex(chr(92)):].lower()}'))"+ f"{{'{tag}|{{0}}|{{1}}' -f $_.InstanceName,{value}}}"+ for tag, path, value in _ROCM_WIN_DISPLAY_COUNTERS.values()+ )+ return (+ f"$s=(Get-Counter -Counter {paths} -ErrorAction SilentlyContinue).CounterSamples;"+ f"if($s){{$s|ForEach-Object{{$p=$_.Path.ToLower();{branches}}}}}"+ "else{'__NONE__'}"+ )+++def _rocm_windows_split_combined(stdout: str) -> dict[tuple[str, str], str]:+ by_tag: dict[str, list[str]] = {}+ for line in stdout.splitlines():+ tag, sep, rest = line.strip().partition("|")+ if sep:+ by_tag.setdefault(tag, []).append(rest)+ return {+ key: "\n".join(by_tag[tag]) + "\n"+ for key, (tag, _, _) in _ROCM_WIN_DISPLAY_COUNTERS.items()+ if by_tag.get(tag)+ }+++def _rocm_windows_counter_stdout(key: tuple[str, str], ps: str) -> Optional[str]:+ """stdout of ``ps`` or None. Display reads share one combined Get-Counter, kept max(TTL, 4x spawn time) until a GPU memory event; misses fall back to ``ps``, failures are never served."""+ try:+ from utils import gpu_memory_events+ from utils.hardware import gpu_query++ use_cache = gpu_query.display_cache_allowed() and key in _ROCM_WIN_DISPLAY_COUNTERS+ ttl = gpu_query.ttl_for(gpu_query.DISPLAY)+ except Exception:+ use_cache = False+ if use_cache and ttl > 0:+ stdout = None+ try:+ with _rocm_win_counter_lock:+ cached = _rocm_win_counter_cache.get("display")+ generation = gpu_memory_events.generation()+ if (+ cached is None+ or cached[1] != generation+ or (+ time.monotonic() - cached[0]+ >= (+ min(+ _ROCM_WIN_COUNTER_MAX_AGE_S,+ max(ttl, _ROCM_WIN_COUNTER_SPACING * cached[2]),+ )+ if cached[3]+ else _ROCM_WIN_COUNTER_EMPTY_RETRY_S+ )+ )+ ):+ _rocm_win_counter_cache.pop("display", None)+ started = time.monotonic()+ try:+ combined = _rocm_windows_counter_stdout_uncached(+ _rocm_windows_combined_counter_ps()+ )+ except Exception:+ combined = None+ done = time.monotonic()+ readings = _rocm_windows_split_combined(combined) if combined else {}+ cached = (done, generation, done - started, readings)+ _rocm_win_counter_cache["display"] = cached+ if cached is not None:+ stdout = cached[3].get(key)+ except Exception:+ stdout = None+ if stdout is not None:+ return stdout+ return _rocm_windows_counter_stdout_uncached(ps)+++def _rocm_windows_counter_stdout_uncached(ps: str) -> Optional[str]:+ r = subprocess.run(+ ["powershell", "-NoProfile", "-NonInteractive", "-Command", ps],+ capture_output = True,+ text = True,+ encoding = "utf-8",+ errors = "replace",+ timeout = 5,+ )+ if r.returncode != 0 or not r.stdout.strip():+ return None+ return r.stdout++def _rocm_windows_perf_counter_gpu_util_pct(luid: Optional[int] = None) -> Optional[float]:"""AMD GPU compute utilization via Windows Performance Counters (3D engine nodes). ``luid`` narrows the sum to one adapter's engines, matched here rather than in the counter path so tests can reach it."""if platform.system() != "Windows":@@ -3631,19 +3750,12 @@ def _rocm_windows_perf_counter_gpu_util_pct(luid: Optional[int] = None) -> Optio"if($s){$s|ForEach-Object{'{0}|{1}' -f $_.InstanceName,$_.CookedValue}}""else{'__NONE__'}")- r = subprocess.run(- ["powershell", "-NoProfile", "-NonInteractive", "-Command", ps],- capture_output = True,- text = True,- encoding = "utf-8",- errors = "replace",- timeout = 5,- )- if r.returncode != 0 or not r.stdout.strip():+ stdout = _rocm_windows_counter_stdout(("util", "3D"), ps)+ if stdout is None:return Nonetotal = 0.0matched = False- for line in r.stdout.splitlines():+ for line in stdout.splitlines():line = line.strip()if not line or line == "__NONE__" or "|" not in line:continue@@ -3808,18 +3920,11 @@ def _rocm_windows_perf_counter_vram_by_adapter("if($s){$s|ForEach-Object{'{0}|{1}' -f $_.InstanceName,[int64]$_.CookedValue}}""else{'__NONE__'}")- r = subprocess.run(- ["powershell", "-NoProfile", "-NonInteractive", "-Command", ps],- capture_output = True,- text = True,- encoding = "utf-8",- errors = "replace",- timeout = 5,- )- if r.returncode != 0 or not r.stdout.strip():+ stdout = _rocm_windows_counter_stdout(("vram", counter), ps)+ if stdout is None:return Noneadapters: list[tuple[str, float]] = []- for line in r.stdout.splitlines():+ for line in stdout.splitlines():line = line.strip()if not line or line == "__NONE__" or "|" not in line:continue