Skip to content

A GPU task manager for Windows, Linux, and WSL: per-process VRAM, GPU activity, and LLM model/KV-cache insights (Windows AMD and NVIDIA).

Python

Fix Intel PMU device binding and complete engine coverage

Scott Hanselman49fc145093b8
diff --git a/README.md b/README.md
index 9d45e51..de2519f 100644
--- a/README.md
+++ b/README.md
@@ -256,6 +256,10 @@ Device memory bandwidth and the PCIe link are `n/a`: i915 has no device-wide ban
sysfs link fields can report the power-on default instead of the live link. Device GPU load and video load
come from the i915 performance counters and need root; power comes from RAPL's uncore zone on an integrated
GPU and needs root too.
+The load readings sample every exposed `*-busy` engine instance, including compute and additional
+video/copy engines, and use the busiest instance rather than averaging across idle engines. Discrete
+PMUs are matched by PCI address (`i915_<PCI BDF>`); the legacy `i915` PMU is used only for an integrated
+card. If multiple cards resolve to the same PMU, their load stays `N/A` with an attribution reason.
`xe` cards are detected, but its fdinfo uses other keys (`drm-cycles-*`, `drm-resident-vram0`/`-gtt`), so its
per-process rows show `n/a` until they are decoded on hardware.
@@ -277,6 +281,8 @@ Live check of the root-only readings on the same machine (`intel_gpu_top` from i
nearby window; scenes vary, and uncore is approximate while the CPU is active). A VA-API `ffmpeg` decode loop
drives the video engine to 99.9% on both (`intel_gpu_top` Video 99.4-99.9%). Without root the three stay `N/A`
with a reason, while the GT clock, throttle and per-process rows keep working.
+Per-PCI PMU selection and compute/secondary-engine coverage are regression-tested with synthetic
+counters, **not yet validated on hardware**; the live check above covers only the Vivobook's exercised engines.
**WSL** is different, because Windows owns the GPU. Linux cannot see how much VRAM its own
processes use: `nvidia-smi` lists no processes, NVML's per-process samples carry *Windows* PIDs
diff --git a/src/gputaskman/drm.py b/src/gputaskman/drm.py
index 0089603..43e11c8 100644
--- a/src/gputaskman/drm.py
+++ b/src/gputaskman/drm.py
@@ -520,7 +520,7 @@ def read_device(card: DrmCard, index: int) -> DeviceTelemetry:
dev.gfx_clock = cur
dev.sm_clock_max = _int(os.path.join(gt, "rps_RP0_freq_mhz"))
dev.throttle = _intel_throttle(gt)
- util, video = _intel_pmu_rates(d)
+ util, video = _intel_pmu_rates(card)
dev.util_gpu = util
dev.video = video
if dev.power_w is None and card.integrated:
@@ -582,8 +582,6 @@ _INTEL_ENERGY_STATE: dict[str, tuple[int, float]] = {} # hwmon path -> (last uJ
# never be opened records why ("denied", "missing" or "shared") instead of retrying a failing syscall
# every refresh; RAPL failures are cached the same way. See intel_counter_status for what each means.
_PERF_NR = {"x86_64": 298, "aarch64": 241} # __NR_perf_event_open
-_INTEL_PMU_ENGINES = (("render", "rcs0-busy"), ("copy", "bcs0-busy"),
- ("video", "vcs0-busy"), ("video-enhance", "vecs0-busy"))
_INTEL_PMU_STATE: dict[str, dict] = {} # card sysfs dir -> {"fds", "prev", "fail"}
_INTEL_RAPL_PREV: dict[str, tuple[int, float]] = {} # energy_uj path -> (last uJ, last read time)
_INTEL_RAPL_FAIL: str | None = None # why the uncore zone can't be read: denied, missing, or None
@@ -612,8 +610,21 @@ def _perf_open(pmu_type: int, config: int, cpu: int) -> tuple[int | None, str |
return None, ("denied" if err in (errno.EACCES, errno.EPERM) else "missing")
-def _intel_pmu_spec(pmu_root: str = "/sys/bus/event_source/devices/i915") -> tuple[int, int, dict[str, int]] | None:
- """(PMU type, CPU, {engine kind: event config}) from the i915 PMU in sysfs; None when absent."""
+def _intel_pmu_path(card: DrmCard, root: str = "/sys/bus/event_source/devices") -> str | None:
+ """Bind the PCI-named discrete PMU, or the legacy integrated i915 PMU; never use i915 for xe."""
+ if card.driver != "i915":
+ return None
+ discrete = os.path.join(root, "i915_" + card.pdev.replace(":", "_"))
+ if os.path.isdir(discrete):
+ return discrete
+ integrated = os.path.join(root, "i915")
+ return integrated if card.integrated and os.path.isdir(integrated) else None
+
+
+def _intel_pmu_spec(pmu_root: str | None) -> tuple[int, int, dict[str, int]] | None:
+ """(PMU type, CPU, {engine instance: config}); None if any busy counter can't be decoded."""
+ if pmu_root is None:
+ return None
root = pmu_root
pmu_type = _int(os.path.join(root, "type"))
mask = _read(os.path.join(root, "cpumask"))
@@ -623,15 +634,21 @@ def _intel_pmu_spec(pmu_root: str = "/sys/bus/event_source/devices/i915") -> tup
cpu = int(re.split(r"[-,]", mask.strip())[0], 0)
except ValueError:
return None
+ try:
+ events = sorted(os.listdir(os.path.join(root, "events")))
+ except OSError:
+ return None
configs = {}
- for kind, event in _INTEL_PMU_ENGINES:
+ for event in events:
+ if not event.endswith("-busy"):
+ continue
value = _read(os.path.join(root, "events", event))
if not value or "=" not in value:
- continue
+ return None
try:
- configs[kind] = int(value.split("=", 1)[1].strip(), 16)
+ configs[event[:-len("-busy")]] = int(value.split("=", 1)[1].strip(), 16)
except ValueError:
- continue
+ return None
return (pmu_type, cpu, configs) if configs else None
@@ -643,17 +660,18 @@ def _read_counter(fd: int) -> int | None:
return None
-def _intel_pmu_rates(sysfs: str, now: int | None = None) -> tuple[float | None, float | None]:
+def _intel_pmu_rates(card: DrmCard, now: int | None = None) -> tuple[float | None, float | None]:
"""Whole-GPU busy % (busiest engine) and video-engine busy % for an Intel card; (None, None) when
unreadable. The first sample only arms the baseline (a rate needs an interval); a counter stepping
back re-arms it instead of reporting a negative load."""
if now is None:
now = time.monotonic_ns()
+ sysfs = card.sysfs
state = _INTEL_PMU_STATE.get(sysfs)
if state is None:
state = {"fds": {}, "prev": {}, "fail": None}
_INTEL_PMU_STATE[sysfs] = state
- spec = _intel_pmu_spec()
+ spec = _intel_pmu_spec(_intel_pmu_path(card))
if spec is None:
state["fail"] = "missing"
return None, None
@@ -698,8 +716,10 @@ def _intel_pmu_rates(sysfs: str, now: int | None = None) -> tuple[float | None,
busy[kind] = min(100.0, delta * 100.0 / elapsed)
if not busy:
return None, None
- video = max([v for k, v in busy.items() if k in ("video", "video-enhance")], default=None)
- util = max(busy.values(), default=None)
+ video_engines = [k for k in state["fds"] if k.startswith(("vcs", "vecs"))]
+ video = (max((busy[k] for k in video_engines), default=None)
+ if all(k in busy for k in video_engines) else None)
+ util = max(busy.values()) if len(busy) == len(state["fds"]) else None
return util, video
@@ -775,7 +795,7 @@ def intel_counter_status(card: DrmCard, nv=None) -> str:
para = _read("/proc/sys/kernel/perf_event_paranoid") or "?"
parts.append(f"GPU and video load need perf privileges this user lacks (perf_event_paranoid={para})")
elif pmu_fail == "missing":
- parts.append("no i915 performance counters on this kernel")
+ parts.append("no readable supported i915 performance counters for this GPU")
elif pmu_fail == "shared":
parts.append("performance counters can't attribute load with multiple Intel GPUs")
hwmon_power = nv is not None and nv.power_kind == "Board"
@@ -873,16 +893,16 @@ class DrmCounters:
for card in intel_cards:
if card.integrated:
_intel_rapl_w() # arm the power baseline; RAPL is package-global, unaffected by sharing
- if len(intel_cards) > 1:
- # One i915 PMU for the whole driver: its counters can't be attributed to either card,
- # so neither gets device load instead of both showing the same numbers.
- for card in intel_cards:
- _INTEL_PMU_STATE.setdefault(card.sysfs, {"fds": {}, "prev": {}, "fail": "shared"})
- return
+ pmu_paths = {card.sysfs: _intel_pmu_path(card) for card in intel_cards}
for card in intel_cards:
+ path = pmu_paths[card.sysfs]
+ if path is not None and list(pmu_paths.values()).count(path) > 1:
+ # Only a genuinely shared PMU is ambiguous; discrete PMUs are named by PCI address.
+ _INTEL_PMU_STATE.setdefault(card.sysfs, {"fds": {}, "prev": {}, "fail": "shared"})
+ continue
# Arm the load baseline too: --once sleeps a second after Collector init,
# so its single sample is already a rate over a real interval.
- _intel_pmu_rates(card.sysfs)
+ _intel_pmu_rates(card)
def _adapters(self) -> dict[int, AdapterGpu]:
out: dict[int, AdapterGpu] = {}
diff --git a/tests/test_drm.py b/tests/test_drm.py
index 506931b..f90237f 100644
--- a/tests/test_drm.py
+++ b/tests/test_drm.py
@@ -1511,7 +1511,7 @@ class IntelScanTests(unittest.TestCase):
with patch.object(drm, "_intel_pmu_rates") as pmu, \
patch.object(drm, "_intel_rapl_w") as rapl:
counters.start()
- pmu.assert_called_once_with(card.sysfs)
+ pmu.assert_called_once_with(card)
rapl.assert_called_once_with()
def test_device_used_sums_each_client_once_and_is_unknown_when_incomplete(self):
@@ -1534,6 +1534,9 @@ class IntelPmuTests(unittest.TestCase):
def setUp(self):
drm._INTEL_PMU_STATE.clear()
+ self.card = drm.DrmCard(key=32, pdev="0000:00:02.0", sysfs="/x/card1/device",
+ driver="i915", vendor_id=0x8086, device_id=0xA7AD, name="Intel Graphics",
+ vram_total=0, integrated=True, intel=True)
def _pmu(self, root, events=("rcs0-busy", "bcs0-busy", "vcs0-busy"), cpumask="0\n"):
(root / "type").write_text("28\n")
@@ -1549,7 +1552,7 @@ class IntelPmuTests(unittest.TestCase):
self._pmu(root)
pmu_type, cpu, configs = drm._intel_pmu_spec(str(root))
self.assertEqual((pmu_type, cpu), (28, 0))
- self.assertEqual(configs, {"render": 0, "copy": 0x1000, "video": 0x2000})
+ self.assertEqual(configs, {"rcs0": 0, "bcs0": 0x1000, "vcs0": 0x2000})
def test_spec_missing_is_unknown(self):
with tempfile.TemporaryDirectory() as tmp:
@@ -1559,47 +1562,131 @@ class IntelPmuTests(unittest.TestCase):
(root / "cpumask").write_text("not-a-cpu\n")
self.assertIsNone(drm._intel_pmu_spec(str(root)))
+ def test_compute_and_secondary_engine_activity_is_not_lost(self):
+ events = ("rcs0-busy", "ccs0-busy", "bcs1-busy", "vcs0-busy", "vcs1-busy", "vecs1-busy")
+ for active in ("ccs0", "bcs1", "vcs1", "vecs1"):
+ with self.subTest(active=active), tempfile.TemporaryDirectory() as tmp:
+ drm._INTEL_PMU_STATE.clear()
+ root = Path(tmp)
+ self._pmu(root, events=events)
+ values = {}
+ with patch.object(drm, "_intel_pmu_path", return_value=str(root)), \
+ patch.object(drm, "_perf_open", side_effect=lambda typ, config, cpu: (config + 10, None)), \
+ patch.object(drm, "_read_counter", side_effect=lambda fd: values.get(fd, 0)):
+ self.assertEqual(drm._intel_pmu_rates(self.card, now=0), (None, None))
+ config = drm._intel_pmu_spec(str(root))[2][active]
+ values[config + 10] = 850_000_000
+ video = 85.0 if active.startswith(("vcs", "vecs")) else 0.0
+ self.assertEqual(drm._intel_pmu_rates(self.card, now=1_000_000_000), (85.0, video))
+
+ def test_unreadable_busy_event_does_not_leave_partial_coverage(self):
+ with tempfile.TemporaryDirectory() as tmp:
+ root = Path(tmp)
+ self._pmu(root, events=("rcs0-busy", "ccs0-busy"))
+ (root / "events" / "ccs0-busy").write_text("unparseable\n")
+ self.assertIsNone(drm._intel_pmu_spec(str(root)))
+ (root / "events" / "ccs0-busy").write_text("config=not-hex\n")
+ self.assertIsNone(drm._intel_pmu_spec(str(root)))
+
+ def test_discovery_uses_pci_identity_and_never_assigns_igpu_pmu_to_discrete_or_xe(self):
+ with tempfile.TemporaryDirectory() as tmp:
+ root = Path(tmp)
+ legacy = root / "i915"
+ legacy.mkdir()
+ self.assertEqual(drm._intel_pmu_path(self.card, str(root)), str(legacy))
+ self.card.pdev = "0000:03:00.0"
+ self.card.integrated = False
+ self.assertIsNone(drm._intel_pmu_path(self.card, str(root)))
+ discrete = root / "i915_0000_03_00.0"
+ discrete.mkdir()
+ self.assertEqual(drm._intel_pmu_path(self.card, str(root)), str(discrete))
+ self.card.integrated = True # a small BAR can initially look integrated
+ self.assertEqual(drm._intel_pmu_path(self.card, str(root)), str(discrete))
+ self.card.driver = "xe"
+ self.assertIsNone(drm._intel_pmu_path(self.card, str(root)))
+
+ def test_distinct_pmus_sample_each_card_independently(self):
+ with tempfile.TemporaryDirectory() as tmp:
+ root = Path(tmp)
+ proc = root / "proc"
+ proc.mkdir()
+ a = self.card
+ b = drm.DrmCard(key=2, pdev="0000:03:00.0", sysfs="/x/card2/device", driver="i915",
+ vendor_id=0x8086, device_id=0x56A0, name="Intel Arc A770",
+ vram_total=16 * GiB, integrated=False, intel=True)
+ igpu = root / "i915"
+ dgpu = root / "i915_0000_03_00.0"
+ igpu.mkdir()
+ dgpu.mkdir()
+ self._pmu(igpu, events=("rcs0-busy",))
+ self._pmu(dgpu, events=("ccs0-busy",))
+ (dgpu / "type").write_text("29\n")
+ path_for = drm._intel_pmu_path
+ counters = drm.DrmCounters(found=[a, b], proc=str(proc), clock=Clock())
+ values = {28: 0, 29: 0}
+ with patch.object(drm, "_intel_pmu_path", side_effect=lambda card: path_for(card, str(root))), \
+ patch.object(drm, "_intel_rapl_w"), \
+ patch.object(drm, "_perf_open", side_effect=lambda typ, config, cpu: (typ, None)) as opened, \
+ patch.object(drm, "_read_counter", side_effect=lambda fd: values[fd]), \
+ patch.object(drm.time, "monotonic_ns", return_value=0), \
+ patch.object(drm.os, "close") as closed:
+ counters.start()
+ self.assertEqual(opened.call_args_list, [unittest.mock.call(28, 0, 0),
+ unittest.mock.call(29, 0, 0)])
+ values.update({28: 200_000_000, 29: 900_000_000})
+ self.assertEqual(drm._intel_pmu_rates(a, now=1_000_000_000), (20.0, None))
+ self.assertEqual(drm._intel_pmu_rates(b, now=1_000_000_000), (90.0, None))
+ counters.close()
+ self.assertCountEqual([c.args[0] for c in closed.call_args_list], [28, 29])
+
def test_rates_need_two_samples(self):
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
self._pmu(root)
- with patch.object(drm, "_intel_pmu_spec", return_value=(28, 0, {"render": 0, "copy": 0x1000})), \
+ with patch.object(drm, "_intel_pmu_spec", return_value=(28, 0, {"rcs0": 0, "bcs0": 0x1000})), \
patch.object(drm, "_perf_open", side_effect=[(10, None), (11, None)]) as opened, \
patch.object(drm, "_read_counter", side_effect=[0, 0, 500_000_000, 250_000_000]):
- self.assertEqual(drm._intel_pmu_rates("/x/card1/device", now=0), (None, None))
+ self.assertEqual(drm._intel_pmu_rates(self.card, now=0), (None, None))
self.assertEqual(opened.call_count, 2)
- self.assertEqual(drm._intel_pmu_rates("/x/card1/device", now=1_000_000_000), (50.0, None))
+ self.assertEqual(drm._intel_pmu_rates(self.card, now=1_000_000_000), (50.0, None))
# "copy" is not video; with no video engine the video figure stays unknown
self.assertEqual(drm._INTEL_PMU_STATE["/x/card1/device"]["fail"], None)
def test_video_is_the_video_engines_busy_share(self):
with tempfile.TemporaryDirectory() as tmp:
- with patch.object(drm, "_intel_pmu_spec", return_value=(28, 0, {"video": 0, "video-enhance": 1})), \
+ with patch.object(drm, "_intel_pmu_spec", return_value=(28, 0, {"vcs0": 0, "vecs0": 1})), \
patch.object(drm, "_perf_open", side_effect=[(10, None), (11, None)]), \
patch.object(drm, "_read_counter", side_effect=[0, 0, 800_000_000, 200_000_000]):
- drm._intel_pmu_rates("/x/card1/device", now=0)
- self.assertEqual(drm._intel_pmu_rates("/x/card1/device", now=1_000_000_000), (80.0, 80.0))
+ drm._intel_pmu_rates(self.card, now=0)
+ self.assertEqual(drm._intel_pmu_rates(self.card, now=1_000_000_000), (80.0, 80.0))
def test_stepped_back_counter_re_arms(self):
with tempfile.TemporaryDirectory() as tmp:
- with patch.object(drm, "_intel_pmu_spec", return_value=(28, 0, {"render": 0})), \
+ with patch.object(drm, "_intel_pmu_spec", return_value=(28, 0, {"rcs0": 0})), \
patch.object(drm, "_perf_open", return_value=(10, None)), \
patch.object(drm, "_read_counter", side_effect=[1_000_000_000, 900_000_000, 1_500_000_000]) as reads:
- drm._intel_pmu_rates("/x/card1/device", now=0)
- self.assertEqual(drm._intel_pmu_rates("/x/card1/device", now=1_000_000_000), (None, None))
- self.assertEqual(drm._intel_pmu_rates("/x/card1/device", now=2_000_000_000), (60.0, None))
+ drm._intel_pmu_rates(self.card, now=0)
+ self.assertEqual(drm._intel_pmu_rates(self.card, now=1_000_000_000), (None, None))
+ self.assertEqual(drm._intel_pmu_rates(self.card, now=2_000_000_000), (60.0, None))
self.assertEqual(reads.call_count, 3)
+ def test_reset_engine_does_not_make_other_idle_engines_look_like_idle_gpu(self):
+ with patch.object(drm, "_intel_pmu_spec", return_value=(28, 0, {"ccs0": 0, "vcs0": 1})), \
+ patch.object(drm, "_perf_open", side_effect=[(10, None), (11, None)]), \
+ patch.object(drm, "_read_counter", side_effect=[100, 0, 99, 0]):
+ drm._intel_pmu_rates(self.card, now=0)
+ self.assertEqual(drm._intel_pmu_rates(self.card, now=1_000_000_000), (None, 0.0))
+
def test_denied_open_reports_denied(self):
with tempfile.TemporaryDirectory() as tmp:
- with patch.object(drm, "_intel_pmu_spec", return_value=(28, 0, {"render": 0})), \
+ with patch.object(drm, "_intel_pmu_spec", return_value=(28, 0, {"rcs0": 0})), \
patch.object(drm, "_perf_open", return_value=(None, "denied")):
- self.assertEqual(drm._intel_pmu_rates("/x/card1/device"), (None, None))
+ self.assertEqual(drm._intel_pmu_rates(self.card), (None, None))
self.assertEqual(drm._INTEL_PMU_STATE["/x/card1/device"]["fail"], "denied")
def test_missing_spec_reports_missing(self):
with patch.object(drm, "_intel_pmu_spec", return_value=None):
- self.assertEqual(drm._intel_pmu_rates("/x/card1/device"), (None, None))
+ self.assertEqual(drm._intel_pmu_rates(self.card), (None, None))
self.assertEqual(drm._INTEL_PMU_STATE["/x/card1/device"]["fail"], "missing")
def test_unsupported_arch_reports_missing_not_denied(self):
@@ -1608,16 +1695,14 @@ class IntelPmuTests(unittest.TestCase):
self.assertEqual(drm._perf_open(28, 0, 0), (None, "missing"))
def test_busy_cannot_exceed_wall_time(self):
- with patch.object(drm, "_intel_pmu_spec", return_value=(28, 0, {"render": 0})), \
+ with patch.object(drm, "_intel_pmu_spec", return_value=(28, 0, {"rcs0": 0})), \
patch.object(drm, "_perf_open", return_value=(10, None)), \
patch.object(drm, "_read_counter", side_effect=[0, 3_000_000_000]):
- drm._intel_pmu_rates("/x/card1/device", now=0)
- util, _video = drm._intel_pmu_rates("/x/card1/device", now=1_000_000_000)
+ drm._intel_pmu_rates(self.card, now=0)
+ util, _video = drm._intel_pmu_rates(self.card, now=1_000_000_000)
self.assertEqual(util, 100.0)
- def test_two_intel_gpus_share_no_pmu(self):
- # One i915 PMU for the whole driver: with two Intel cards neither gets device load,
- # instead of both showing the same numbers.
+ def test_ambiguous_legacy_pmu_is_not_shared_between_cards(self):
with tempfile.TemporaryDirectory() as tmp:
proc = Path(tmp) / "proc"
proc.mkdir()
@@ -1626,9 +1711,12 @@ class IntelPmuTests(unittest.TestCase):
vram_total=0, integrated=True, intel=True)
b = drm.DrmCard(key=2, pdev="0000:03:00.0", sysfs=str(Path(tmp) / "b"), driver="i915",
vendor_id=0x8086, device_id=0x56A0, name="Intel Arc A770",
- vram_total=16 * GiB, integrated=False, intel=True)
+ vram_total=0, integrated=True, intel=True)
counters = drm.DrmCounters(found=[a, b], proc=str(proc), clock=Clock())
- counters.start()
+ with patch.object(drm, "_intel_pmu_path", return_value="/pmu/i915"), \
+ patch.object(drm, "_intel_rapl_w"), patch.object(drm, "_perf_open") as opened:
+ counters.start()
+ opened.assert_not_called()
self.assertEqual(drm._INTEL_PMU_STATE[str(Path(tmp) / "a")]["fail"], "shared")
self.assertEqual(drm._INTEL_PMU_STATE[str(Path(tmp) / "b")]["fail"], "shared")