A GPU task manager for Windows, Linux, and WSL: per-process VRAM, GPU activity, and LLM model/KV-cache insights (Windows AMD and NVIDIA).
Python
Fix Intel PMU device binding and complete engine coverage
Scott Hanselman49fc145093b8
diff --git a/README.md b/README.mdindex 9d45e51..de2519f 100644--- a/README.md+++ b/README.md@@ -256,6 +256,10 @@ Device memory bandwidth and the PCIe link are `n/a`: i915 has no device-wide bansysfs link fields can report the power-on default instead of the live link. Device GPU load and video loadcome from the i915 performance counters and need root; power comes from RAPL's uncore zone on an integratedGPU and needs root too.+The load readings sample every exposed `*-busy` engine instance, including compute and additional+video/copy engines, and use the busiest instance rather than averaging across idle engines. Discrete+PMUs are matched by PCI address (`i915_<PCI BDF>`); the legacy `i915` PMU is used only for an integrated+card. If multiple cards resolve to the same PMU, their load stays `N/A` with an attribution reason.`xe` cards are detected, but its fdinfo uses other keys (`drm-cycles-*`, `drm-resident-vram0`/`-gtt`), so itsper-process rows show `n/a` until they are decoded on hardware.@@ -277,6 +281,8 @@ Live check of the root-only readings on the same machine (`intel_gpu_top` from inearby window; scenes vary, and uncore is approximate while the CPU is active). A VA-API `ffmpeg` decode loopdrives the video engine to 99.9% on both (`intel_gpu_top` Video 99.4-99.9%). Without root the three stay `N/A`with a reason, while the GT clock, throttle and per-process rows keep working.+Per-PCI PMU selection and compute/secondary-engine coverage are regression-tested with synthetic+counters, **not yet validated on hardware**; the live check above covers only the Vivobook's exercised engines.**WSL** is different, because Windows owns the GPU. Linux cannot see how much VRAM its ownprocesses use: `nvidia-smi` lists no processes, NVML's per-process samples carry *Windows* PIDsdiff --git a/src/gputaskman/drm.py b/src/gputaskman/drm.pyindex 0089603..43e11c8 100644--- a/src/gputaskman/drm.py+++ b/src/gputaskman/drm.py@@ -520,7 +520,7 @@ def read_device(card: DrmCard, index: int) -> DeviceTelemetry:dev.gfx_clock = curdev.sm_clock_max = _int(os.path.join(gt, "rps_RP0_freq_mhz"))dev.throttle = _intel_throttle(gt)- util, video = _intel_pmu_rates(d)+ util, video = _intel_pmu_rates(card)dev.util_gpu = utildev.video = videoif dev.power_w is None and card.integrated:@@ -582,8 +582,6 @@ _INTEL_ENERGY_STATE: dict[str, tuple[int, float]] = {} # hwmon path -> (last uJ# never be opened records why ("denied", "missing" or "shared") instead of retrying a failing syscall# every refresh; RAPL failures are cached the same way. See intel_counter_status for what each means._PERF_NR = {"x86_64": 298, "aarch64": 241} # __NR_perf_event_open-_INTEL_PMU_ENGINES = (("render", "rcs0-busy"), ("copy", "bcs0-busy"),- ("video", "vcs0-busy"), ("video-enhance", "vecs0-busy"))_INTEL_PMU_STATE: dict[str, dict] = {} # card sysfs dir -> {"fds", "prev", "fail"}_INTEL_RAPL_PREV: dict[str, tuple[int, float]] = {} # energy_uj path -> (last uJ, last read time)_INTEL_RAPL_FAIL: str | None = None # why the uncore zone can't be read: denied, missing, or None@@ -612,8 +610,21 @@ def _perf_open(pmu_type: int, config: int, cpu: int) -> tuple[int | None, str |return None, ("denied" if err in (errno.EACCES, errno.EPERM) else "missing")-def _intel_pmu_spec(pmu_root: str = "/sys/bus/event_source/devices/i915") -> tuple[int, int, dict[str, int]] | None:- """(PMU type, CPU, {engine kind: event config}) from the i915 PMU in sysfs; None when absent."""+def _intel_pmu_path(card: DrmCard, root: str = "/sys/bus/event_source/devices") -> str | None:+ """Bind the PCI-named discrete PMU, or the legacy integrated i915 PMU; never use i915 for xe."""+ if card.driver != "i915":+ return None+ discrete = os.path.join(root, "i915_" + card.pdev.replace(":", "_"))+ if os.path.isdir(discrete):+ return discrete+ integrated = os.path.join(root, "i915")+ return integrated if card.integrated and os.path.isdir(integrated) else None+++def _intel_pmu_spec(pmu_root: str | None) -> tuple[int, int, dict[str, int]] | None:+ """(PMU type, CPU, {engine instance: config}); None if any busy counter can't be decoded."""+ if pmu_root is None:+ return Noneroot = pmu_rootpmu_type = _int(os.path.join(root, "type"))mask = _read(os.path.join(root, "cpumask"))@@ -623,15 +634,21 @@ def _intel_pmu_spec(pmu_root: str = "/sys/bus/event_source/devices/i915") -> tupcpu = int(re.split(r"[-,]", mask.strip())[0], 0)except ValueError:return None+ try:+ events = sorted(os.listdir(os.path.join(root, "events")))+ except OSError:+ return Noneconfigs = {}- for kind, event in _INTEL_PMU_ENGINES:+ for event in events:+ if not event.endswith("-busy"):+ continuevalue = _read(os.path.join(root, "events", event))if not value or "=" not in value:- continue+ return Nonetry:- configs[kind] = int(value.split("=", 1)[1].strip(), 16)+ configs[event[:-len("-busy")]] = int(value.split("=", 1)[1].strip(), 16)except ValueError:- continue+ return Nonereturn (pmu_type, cpu, configs) if configs else None@@ -643,17 +660,18 @@ def _read_counter(fd: int) -> int | None:return None-def _intel_pmu_rates(sysfs: str, now: int | None = None) -> tuple[float | None, float | None]:+def _intel_pmu_rates(card: DrmCard, now: int | None = None) -> tuple[float | None, float | None]:"""Whole-GPU busy % (busiest engine) and video-engine busy % for an Intel card; (None, None) whenunreadable. The first sample only arms the baseline (a rate needs an interval); a counter steppingback re-arms it instead of reporting a negative load."""if now is None:now = time.monotonic_ns()+ sysfs = card.sysfsstate = _INTEL_PMU_STATE.get(sysfs)if state is None:state = {"fds": {}, "prev": {}, "fail": None}_INTEL_PMU_STATE[sysfs] = state- spec = _intel_pmu_spec()+ spec = _intel_pmu_spec(_intel_pmu_path(card))if spec is None:state["fail"] = "missing"return None, None@@ -698,8 +716,10 @@ def _intel_pmu_rates(sysfs: str, now: int | None = None) -> tuple[float | None,busy[kind] = min(100.0, delta * 100.0 / elapsed)if not busy:return None, None- video = max([v for k, v in busy.items() if k in ("video", "video-enhance")], default=None)- util = max(busy.values(), default=None)+ video_engines = [k for k in state["fds"] if k.startswith(("vcs", "vecs"))]+ video = (max((busy[k] for k in video_engines), default=None)+ if all(k in busy for k in video_engines) else None)+ util = max(busy.values()) if len(busy) == len(state["fds"]) else Nonereturn util, video@@ -775,7 +795,7 @@ def intel_counter_status(card: DrmCard, nv=None) -> str:para = _read("/proc/sys/kernel/perf_event_paranoid") or "?"parts.append(f"GPU and video load need perf privileges this user lacks (perf_event_paranoid={para})")elif pmu_fail == "missing":- parts.append("no i915 performance counters on this kernel")+ parts.append("no readable supported i915 performance counters for this GPU")elif pmu_fail == "shared":parts.append("performance counters can't attribute load with multiple Intel GPUs")hwmon_power = nv is not None and nv.power_kind == "Board"@@ -873,16 +893,16 @@ class DrmCounters:for card in intel_cards:if card.integrated:_intel_rapl_w() # arm the power baseline; RAPL is package-global, unaffected by sharing- if len(intel_cards) > 1:- # One i915 PMU for the whole driver: its counters can't be attributed to either card,- # so neither gets device load instead of both showing the same numbers.- for card in intel_cards:- _INTEL_PMU_STATE.setdefault(card.sysfs, {"fds": {}, "prev": {}, "fail": "shared"})- return+ pmu_paths = {card.sysfs: _intel_pmu_path(card) for card in intel_cards}for card in intel_cards:+ path = pmu_paths[card.sysfs]+ if path is not None and list(pmu_paths.values()).count(path) > 1:+ # Only a genuinely shared PMU is ambiguous; discrete PMUs are named by PCI address.+ _INTEL_PMU_STATE.setdefault(card.sysfs, {"fds": {}, "prev": {}, "fail": "shared"})+ continue# Arm the load baseline too: --once sleeps a second after Collector init,# so its single sample is already a rate over a real interval.- _intel_pmu_rates(card.sysfs)+ _intel_pmu_rates(card)def _adapters(self) -> dict[int, AdapterGpu]:out: dict[int, AdapterGpu] = {}diff --git a/tests/test_drm.py b/tests/test_drm.pyindex 506931b..f90237f 100644--- a/tests/test_drm.py+++ b/tests/test_drm.py@@ -1511,7 +1511,7 @@ class IntelScanTests(unittest.TestCase):with patch.object(drm, "_intel_pmu_rates") as pmu, \patch.object(drm, "_intel_rapl_w") as rapl:counters.start()- pmu.assert_called_once_with(card.sysfs)+ pmu.assert_called_once_with(card)rapl.assert_called_once_with()def test_device_used_sums_each_client_once_and_is_unknown_when_incomplete(self):@@ -1534,6 +1534,9 @@ class IntelPmuTests(unittest.TestCase):def setUp(self):drm._INTEL_PMU_STATE.clear()+ self.card = drm.DrmCard(key=32, pdev="0000:00:02.0", sysfs="/x/card1/device",+ driver="i915", vendor_id=0x8086, device_id=0xA7AD, name="Intel Graphics",+ vram_total=0, integrated=True, intel=True)def _pmu(self, root, events=("rcs0-busy", "bcs0-busy", "vcs0-busy"), cpumask="0\n"):(root / "type").write_text("28\n")@@ -1549,7 +1552,7 @@ class IntelPmuTests(unittest.TestCase):self._pmu(root)pmu_type, cpu, configs = drm._intel_pmu_spec(str(root))self.assertEqual((pmu_type, cpu), (28, 0))- self.assertEqual(configs, {"render": 0, "copy": 0x1000, "video": 0x2000})+ self.assertEqual(configs, {"rcs0": 0, "bcs0": 0x1000, "vcs0": 0x2000})def test_spec_missing_is_unknown(self):with tempfile.TemporaryDirectory() as tmp:@@ -1559,47 +1562,131 @@ class IntelPmuTests(unittest.TestCase):(root / "cpumask").write_text("not-a-cpu\n")self.assertIsNone(drm._intel_pmu_spec(str(root)))+ def test_compute_and_secondary_engine_activity_is_not_lost(self):+ events = ("rcs0-busy", "ccs0-busy", "bcs1-busy", "vcs0-busy", "vcs1-busy", "vecs1-busy")+ for active in ("ccs0", "bcs1", "vcs1", "vecs1"):+ with self.subTest(active=active), tempfile.TemporaryDirectory() as tmp:+ drm._INTEL_PMU_STATE.clear()+ root = Path(tmp)+ self._pmu(root, events=events)+ values = {}+ with patch.object(drm, "_intel_pmu_path", return_value=str(root)), \+ patch.object(drm, "_perf_open", side_effect=lambda typ, config, cpu: (config + 10, None)), \+ patch.object(drm, "_read_counter", side_effect=lambda fd: values.get(fd, 0)):+ self.assertEqual(drm._intel_pmu_rates(self.card, now=0), (None, None))+ config = drm._intel_pmu_spec(str(root))[2][active]+ values[config + 10] = 850_000_000+ video = 85.0 if active.startswith(("vcs", "vecs")) else 0.0+ self.assertEqual(drm._intel_pmu_rates(self.card, now=1_000_000_000), (85.0, video))++ def test_unreadable_busy_event_does_not_leave_partial_coverage(self):+ with tempfile.TemporaryDirectory() as tmp:+ root = Path(tmp)+ self._pmu(root, events=("rcs0-busy", "ccs0-busy"))+ (root / "events" / "ccs0-busy").write_text("unparseable\n")+ self.assertIsNone(drm._intel_pmu_spec(str(root)))+ (root / "events" / "ccs0-busy").write_text("config=not-hex\n")+ self.assertIsNone(drm._intel_pmu_spec(str(root)))++ def test_discovery_uses_pci_identity_and_never_assigns_igpu_pmu_to_discrete_or_xe(self):+ with tempfile.TemporaryDirectory() as tmp:+ root = Path(tmp)+ legacy = root / "i915"+ legacy.mkdir()+ self.assertEqual(drm._intel_pmu_path(self.card, str(root)), str(legacy))+ self.card.pdev = "0000:03:00.0"+ self.card.integrated = False+ self.assertIsNone(drm._intel_pmu_path(self.card, str(root)))+ discrete = root / "i915_0000_03_00.0"+ discrete.mkdir()+ self.assertEqual(drm._intel_pmu_path(self.card, str(root)), str(discrete))+ self.card.integrated = True # a small BAR can initially look integrated+ self.assertEqual(drm._intel_pmu_path(self.card, str(root)), str(discrete))+ self.card.driver = "xe"+ self.assertIsNone(drm._intel_pmu_path(self.card, str(root)))++ def test_distinct_pmus_sample_each_card_independently(self):+ with tempfile.TemporaryDirectory() as tmp:+ root = Path(tmp)+ proc = root / "proc"+ proc.mkdir()+ a = self.card+ b = drm.DrmCard(key=2, pdev="0000:03:00.0", sysfs="/x/card2/device", driver="i915",+ vendor_id=0x8086, device_id=0x56A0, name="Intel Arc A770",+ vram_total=16 * GiB, integrated=False, intel=True)+ igpu = root / "i915"+ dgpu = root / "i915_0000_03_00.0"+ igpu.mkdir()+ dgpu.mkdir()+ self._pmu(igpu, events=("rcs0-busy",))+ self._pmu(dgpu, events=("ccs0-busy",))+ (dgpu / "type").write_text("29\n")+ path_for = drm._intel_pmu_path+ counters = drm.DrmCounters(found=[a, b], proc=str(proc), clock=Clock())+ values = {28: 0, 29: 0}+ with patch.object(drm, "_intel_pmu_path", side_effect=lambda card: path_for(card, str(root))), \+ patch.object(drm, "_intel_rapl_w"), \+ patch.object(drm, "_perf_open", side_effect=lambda typ, config, cpu: (typ, None)) as opened, \+ patch.object(drm, "_read_counter", side_effect=lambda fd: values[fd]), \+ patch.object(drm.time, "monotonic_ns", return_value=0), \+ patch.object(drm.os, "close") as closed:+ counters.start()+ self.assertEqual(opened.call_args_list, [unittest.mock.call(28, 0, 0),+ unittest.mock.call(29, 0, 0)])+ values.update({28: 200_000_000, 29: 900_000_000})+ self.assertEqual(drm._intel_pmu_rates(a, now=1_000_000_000), (20.0, None))+ self.assertEqual(drm._intel_pmu_rates(b, now=1_000_000_000), (90.0, None))+ counters.close()+ self.assertCountEqual([c.args[0] for c in closed.call_args_list], [28, 29])+def test_rates_need_two_samples(self):with tempfile.TemporaryDirectory() as tmp:root = Path(tmp)self._pmu(root)- with patch.object(drm, "_intel_pmu_spec", return_value=(28, 0, {"render": 0, "copy": 0x1000})), \+ with patch.object(drm, "_intel_pmu_spec", return_value=(28, 0, {"rcs0": 0, "bcs0": 0x1000})), \patch.object(drm, "_perf_open", side_effect=[(10, None), (11, None)]) as opened, \patch.object(drm, "_read_counter", side_effect=[0, 0, 500_000_000, 250_000_000]):- self.assertEqual(drm._intel_pmu_rates("/x/card1/device", now=0), (None, None))+ self.assertEqual(drm._intel_pmu_rates(self.card, now=0), (None, None))self.assertEqual(opened.call_count, 2)- self.assertEqual(drm._intel_pmu_rates("/x/card1/device", now=1_000_000_000), (50.0, None))+ self.assertEqual(drm._intel_pmu_rates(self.card, now=1_000_000_000), (50.0, None))# "copy" is not video; with no video engine the video figure stays unknownself.assertEqual(drm._INTEL_PMU_STATE["/x/card1/device"]["fail"], None)def test_video_is_the_video_engines_busy_share(self):with tempfile.TemporaryDirectory() as tmp:- with patch.object(drm, "_intel_pmu_spec", return_value=(28, 0, {"video": 0, "video-enhance": 1})), \+ with patch.object(drm, "_intel_pmu_spec", return_value=(28, 0, {"vcs0": 0, "vecs0": 1})), \patch.object(drm, "_perf_open", side_effect=[(10, None), (11, None)]), \patch.object(drm, "_read_counter", side_effect=[0, 0, 800_000_000, 200_000_000]):- drm._intel_pmu_rates("/x/card1/device", now=0)- self.assertEqual(drm._intel_pmu_rates("/x/card1/device", now=1_000_000_000), (80.0, 80.0))+ drm._intel_pmu_rates(self.card, now=0)+ self.assertEqual(drm._intel_pmu_rates(self.card, now=1_000_000_000), (80.0, 80.0))def test_stepped_back_counter_re_arms(self):with tempfile.TemporaryDirectory() as tmp:- with patch.object(drm, "_intel_pmu_spec", return_value=(28, 0, {"render": 0})), \+ with patch.object(drm, "_intel_pmu_spec", return_value=(28, 0, {"rcs0": 0})), \patch.object(drm, "_perf_open", return_value=(10, None)), \patch.object(drm, "_read_counter", side_effect=[1_000_000_000, 900_000_000, 1_500_000_000]) as reads:- drm._intel_pmu_rates("/x/card1/device", now=0)- self.assertEqual(drm._intel_pmu_rates("/x/card1/device", now=1_000_000_000), (None, None))- self.assertEqual(drm._intel_pmu_rates("/x/card1/device", now=2_000_000_000), (60.0, None))+ drm._intel_pmu_rates(self.card, now=0)+ self.assertEqual(drm._intel_pmu_rates(self.card, now=1_000_000_000), (None, None))+ self.assertEqual(drm._intel_pmu_rates(self.card, now=2_000_000_000), (60.0, None))self.assertEqual(reads.call_count, 3)+ def test_reset_engine_does_not_make_other_idle_engines_look_like_idle_gpu(self):+ with patch.object(drm, "_intel_pmu_spec", return_value=(28, 0, {"ccs0": 0, "vcs0": 1})), \+ patch.object(drm, "_perf_open", side_effect=[(10, None), (11, None)]), \+ patch.object(drm, "_read_counter", side_effect=[100, 0, 99, 0]):+ drm._intel_pmu_rates(self.card, now=0)+ self.assertEqual(drm._intel_pmu_rates(self.card, now=1_000_000_000), (None, 0.0))+def test_denied_open_reports_denied(self):with tempfile.TemporaryDirectory() as tmp:- with patch.object(drm, "_intel_pmu_spec", return_value=(28, 0, {"render": 0})), \+ with patch.object(drm, "_intel_pmu_spec", return_value=(28, 0, {"rcs0": 0})), \patch.object(drm, "_perf_open", return_value=(None, "denied")):- self.assertEqual(drm._intel_pmu_rates("/x/card1/device"), (None, None))+ self.assertEqual(drm._intel_pmu_rates(self.card), (None, None))self.assertEqual(drm._INTEL_PMU_STATE["/x/card1/device"]["fail"], "denied")def test_missing_spec_reports_missing(self):with patch.object(drm, "_intel_pmu_spec", return_value=None):- self.assertEqual(drm._intel_pmu_rates("/x/card1/device"), (None, None))+ self.assertEqual(drm._intel_pmu_rates(self.card), (None, None))self.assertEqual(drm._INTEL_PMU_STATE["/x/card1/device"]["fail"], "missing")def test_unsupported_arch_reports_missing_not_denied(self):@@ -1608,16 +1695,14 @@ class IntelPmuTests(unittest.TestCase):self.assertEqual(drm._perf_open(28, 0, 0), (None, "missing"))def test_busy_cannot_exceed_wall_time(self):- with patch.object(drm, "_intel_pmu_spec", return_value=(28, 0, {"render": 0})), \+ with patch.object(drm, "_intel_pmu_spec", return_value=(28, 0, {"rcs0": 0})), \patch.object(drm, "_perf_open", return_value=(10, None)), \patch.object(drm, "_read_counter", side_effect=[0, 3_000_000_000]):- drm._intel_pmu_rates("/x/card1/device", now=0)- util, _video = drm._intel_pmu_rates("/x/card1/device", now=1_000_000_000)+ drm._intel_pmu_rates(self.card, now=0)+ util, _video = drm._intel_pmu_rates(self.card, now=1_000_000_000)self.assertEqual(util, 100.0)- def test_two_intel_gpus_share_no_pmu(self):- # One i915 PMU for the whole driver: with two Intel cards neither gets device load,- # instead of both showing the same numbers.+ def test_ambiguous_legacy_pmu_is_not_shared_between_cards(self):with tempfile.TemporaryDirectory() as tmp:proc = Path(tmp) / "proc"proc.mkdir()@@ -1626,9 +1711,12 @@ class IntelPmuTests(unittest.TestCase):vram_total=0, integrated=True, intel=True)b = drm.DrmCard(key=2, pdev="0000:03:00.0", sysfs=str(Path(tmp) / "b"), driver="i915",vendor_id=0x8086, device_id=0x56A0, name="Intel Arc A770",- vram_total=16 * GiB, integrated=False, intel=True)+ vram_total=0, integrated=True, intel=True)counters = drm.DrmCounters(found=[a, b], proc=str(proc), clock=Clock())- counters.start()+ with patch.object(drm, "_intel_pmu_path", return_value="/pmu/i915"), \+ patch.object(drm, "_intel_rapl_w"), patch.object(drm, "_perf_open") as opened:+ counters.start()+ opened.assert_not_called()self.assertEqual(drm._INTEL_PMU_STATE[str(Path(tmp) / "a")]["fail"], "shared")self.assertEqual(drm._INTEL_PMU_STATE[str(Path(tmp) / "b")]["fail"], "shared")