A GPU task manager for Windows, Linux, and WSL: per-process VRAM, GPU activity, and LLM model/KV-cache insights (Windows AMD and NVIDIA).
Python
Explain unmatched Windows GPU counters in GPU-PV guests
Scott Hanselman752b5de68268
diff --git a/README.md b/README.mdindex e254948..1328a8d 100644--- a/README.md+++ b/README.md@@ -426,6 +426,20 @@ driver did not expose that reading; it does not by itself indicate a hardware faMissing per-process memory or engine samples likewise show `N/A`/`null`; the paged estimate isunavailable when any of its input counters is missing.+On GPU-paravirtualized VMs, including Windows Sandbox, Windows may expose GPU counter LUIDs that+do not match the selected DXGI adapter. When that happens, the counters cannot safely be assigned+to the adapter: the process list and usage are unavailable, not zero. The text and TUI explain+the mismatch; JSON reports it in `counter_status` and sets `counter_adapter_matched` to `false`+(`null` when the check does not apply).+GPU-PV guests may also omit adapter memory counters. Counter values that are actually reported as+zero remain zero; a zero alone does not establish that a counter is unavailable.++Live check: Windows Sandbox on Windows 11 build 26100, hosted by an RTX 4080 SUPER with NVIDIA+driver 630.20. `D3DKMTEnumAdapters2/3` exposed the PDH LUID `0x4BF1` as a software adapter with a+different adapter/render GUID from DXGI's `0x4BB7`; it was not a proven alias for the NVIDIA GPU.+The portable build's `--once` explanation and `--json` unmatched status were verified in the guest.+Other Hyper-V GPU-PV configurations have not been validated.+### Comparing snapshotsTo see which memory bucket actually grew when you load a model or change the context length, savediff --git a/src/gputaskman/app.py b/src/gputaskman/app.pyindex 69e566a..e34835b 100644--- a/src/gputaskman/app.py+++ b/src/gputaskman/app.py@@ -356,6 +356,8 @@ class GpuTaskManager(App):legend.append("\n " + wsl_note(s), style="grey62")if s.backend == "metal":legend.append("\n " + METAL_NOTE, style="grey62")+ if s.counter_status:+ legend.append("\n " + s.counter_status, style="grey62")if s.process_status:legend.append(f"\n DRM process scan: {s.process_status}.", style="grey62")diff --git a/src/gputaskman/collector.py b/src/gputaskman/collector.pyindex 041adcc..7761842 100644--- a/src/gputaskman/collector.py+++ b/src/gputaskman/collector.py@@ -117,6 +117,8 @@ class Snapshot:wsl_split: str = "" # WSL only: why Linux rows have no (inferred) VRAM, when they don't# False on an APU (amdgpu iGPU): its GTT is ordinary GPU memory, not spill from a full VRAM.shared_is_spill: bool = True+ counter_status: str = "" # Windows counter availability or adapter matching problem+ counter_adapter_matched: bool | None = None # Windows PDH LUID matched the selected DXGI adapter@propertydef host_data(self) -> bool:@@ -124,7 +126,7 @@ class Snapshot:@propertydef attributed(self) -> int | None:- if any("dedicated" not in p.gpu.available for p in self.procs):+ if self.counter_adapter_matched is False or any("dedicated" not in p.gpu.available for p in self.procs):return Nonereturn int(sum(p.gpu.dedicated for p in self.procs))@@ -263,6 +265,14 @@ class Collector:)snap.host_status = self.host_status+ if BACKEND == "wddm" and self.adapter:+ snap.counter_adapter_matched = luid in per_luid or luid in adapters+ if not snap.counter_adapter_matched:+ snap.counter_status = ("Windows GPU counters could not be matched to the selected adapter; "+ "VRAM and processes are unavailable. This can occur in GPU-PV VMs.")+ elif a is None or a.dedicated is None or a.shared is None:+ snap.counter_status = ("Some Windows adapter memory counters are unavailable; "+ "GPU-PV VMs may not expose them.")if self.drm_card is not None:snap.process_status = self.counters.drm.statuslive = [(pid, g) for pid, g in procs.items() if _is_live(g)]diff --git a/src/gputaskman/report.py b/src/gputaskman/report.pyindex c3eb54a..8deae02 100644--- a/src/gputaskman/report.py+++ b/src/gputaskman/report.py@@ -140,6 +140,7 @@ def to_dict(s: Snapshot) -> dict:"attributed": s.attributed, "backend": s.backend, "shared_is_spill": s.shared_is_spill,"host_status": s.host_status or None,"process_status": s.process_status or None,+ "counter_status": s.counter_status or None, "counter_adapter_matched": s.counter_adapter_matched,"wsl_split": s.wsl_split or None, "device": nv, "processes": procs,}@@ -168,6 +169,8 @@ def to_text(s: Snapshot, limit: int = 30) -> str:out.append(memory)if s.backend == "wsl":out.append(wsl_note(s))+ if s.counter_status:+ out.append(s.counter_status)if nv and s.backend != "metal":rx = nv.pcie_rx_kbs / 1024 if nv.pcie_rx_kbs is not None else Nonetx = nv.pcie_tx_kbs / 1024 if nv.pcie_tx_kbs is not None else Nonediff --git a/tests/test_core.py b/tests/test_core.pyindex 6d4af0c..807417b 100644--- a/tests/test_core.py+++ b/tests/test_core.py@@ -11,7 +11,7 @@ import unittestimport xml.etree.ElementTree as ETfrom datetime import datetime, timezonefrom pathlib import Path-from unittest.mock import patch+from unittest.mock import Mock, patchfrom rich.console import Consolefrom rich.table import Table@@ -21,14 +21,14 @@ from gputaskman.amdsmi import AmdSmifrom gputaskman.adl import Adl, _Metricsfrom gputaskman.app import GpuTaskManager, spark, stacked_barfrom gputaskman.cli import main-from gputaskman.collector import ProcRow, Snapshot, select_adapter, select_nv_device+from gputaskman.collector import Collector, ProcRow, Snapshot, select_adapter, select_nv_devicefrom gputaskman.dxgi import Adapterfrom gputaskman.gguf import GgufInfo, read_gguffrom gputaskman.handy import HandyEvidence, inspect_handy, _log_cache, _settings_cachefrom gputaskman.llm import (LlmView, ServerInfo, build_view, is_gpu_device, llama_server_models, lmstudio_models,parse_flags, parse_load_log, probe_server)from gputaskman.nvml import NvDevice, Nvml-from gputaskman.pdh import GpuCounters, ProcGpu+from gputaskman.pdh import AdapterGpu, GpuCounters, ProcGpufrom gputaskman.procinfo import (AI_KINDS, MappedModel, ProcDetail, ProcInspector, _is_model_file, _match_runtimes,_models_from_cmdline, classify, installed_models, make_model)from gputaskman.report import to_dict, to_text@@ -220,6 +220,67 @@ class DeviceSelectionTests(unittest.TestCase):class MemoryTests(unittest.TestCase):+ def test_windows_counter_luid_mismatch_is_not_attributed_to_selected_adapter(self):+ c = Collector.__new__(Collector)+ c.adapter = Adapter(0x4bb7, "GPU", 0x10DE, 16 * GiB, 0, False)+ c.adapters = [c.adapter]+ c.gpu_index, c.scan_budget, c.host, c.host_status = 0, None, None, ""+ c.nvml = Mock(ok=False)+ c.counters = Mock(outside={})+ c.counters.sample.return_value = (+ {0x4bf1: {4: ProcGpu(dedicated=GiB, engines={"compute": 10})}},+ {0x4bf1: AdapterGpu(dedicated=GiB, shared=0)})+ c.inspector, c.prober = Mock(), Mock()+ with patch("gputaskman.collector.BACKEND", "wddm"):+ s = c.sample()+ self.assertIsNone(s.vram_used)+ self.assertIsNone(s.attributed)+ self.assertEqual(s.procs, [])+ self.assertFalse(s.counter_adapter_matched)+ self.assertIn("could not be matched", s.counter_status)+ self.assertIn("GPU-PV", to_text(s))+ data = to_dict(s)+ self.assertIsNone(data["attributed"])+ self.assertFalse(data["counter_adapter_matched"])+ self.assertEqual(data["counter_status"], s.counter_status)+ output = io.StringIO()+ Console(file=output, width=160).print(GpuTaskManager().render_device(s))+ self.assertIn("could not be matched", output.getvalue())++ def test_windows_matching_counter_zero_is_not_unavailable(self):+ c = Collector.__new__(Collector)+ c.adapter = Adapter(1, "GPU", 0x10DE, 16 * GiB, 0, False)+ c.adapters = [c.adapter]+ c.gpu_index, c.scan_budget, c.host, c.host_status = 0, None, None, ""+ c.nvml = Mock(ok=False)+ c.counters = Mock(outside={})+ c.counters.sample.return_value = (+ {2: {4: ProcGpu(dedicated=GiB)}},+ {2: AdapterGpu(dedicated=GiB, shared=GiB), 1: AdapterGpu(dedicated=0, shared=0)})+ c.inspector, c.prober = Mock(), Mock()+ with patch("gputaskman.collector.BACKEND", "wddm"):+ s = c.sample()+ self.assertEqual((s.vram_used, s.shared_used), (0, 0))+ self.assertEqual(s.procs, [])+ self.assertTrue(s.counter_adapter_matched)+ self.assertEqual(s.counter_status, "")++ def test_windows_missing_adapter_counter_explains_unavailable_memory(self):+ c = Collector.__new__(Collector)+ c.adapter = Adapter(1, "GPU", 0x10DE, 16 * GiB, 0, False)+ c.adapters = [c.adapter]+ c.gpu_index, c.scan_budget, c.host, c.host_status = 0, None, None, ""+ c.nvml = Mock(ok=False)+ c.counters = Mock(outside={})+ c.counters.sample.return_value = ({1: {}}, {})+ c.inspector, c.prober = Mock(), Mock()+ with patch("gputaskman.collector.BACKEND", "wddm"):+ s = c.sample()+ self.assertTrue(s.counter_adapter_matched)+ self.assertIsNone(s.vram_used)+ self.assertIn("adapter memory counters are unavailable", s.counter_status)+ self.assertIn("adapter memory counters are unavailable", to_text(s))+def test_failed_collection_does_not_reuse_previous_counters(self):counters = GpuCounters.__new__(GpuCounters)counters._query = None