A GPU task manager for Windows, Linux, and WSL: per-process VRAM, GPU activity, and LLM model/KV-cache insights (Windows AMD and NVIDIA).
Python
Address first review pass: shared PMU, status accuracy, clamping, cleanup
Martin Pluskal88398ef4e7d9
diff --git a/src/gputaskman/collector.py b/src/gputaskman/collector.pyindex c967126..fb0b78c 100644--- a/src/gputaskman/collector.py+++ b/src/gputaskman/collector.py@@ -352,7 +352,7 @@ class Collector:if self.drm_card.intel and nv is not None:from .drm import intel_counter_status- snap.counter_status = intel_counter_status(self.drm_card)+ snap.counter_status = intel_counter_status(self.drm_card, nv)live = [(pid, g) for pid, g in procs.items() if _is_live(g)]live.sort(key=lambda x: -(x[1].shown_vram or 0))for pid, g in live:diff --git a/src/gputaskman/drm.py b/src/gputaskman/drm.pyindex dea3d37..81d97db 100644--- a/src/gputaskman/drm.py+++ b/src/gputaskman/drm.py@@ -577,30 +577,39 @@ _INTEL_ENERGY_STATE: dict[str, tuple[int, float]] = {} # hwmon path -> (last uJ# Whole-GPU load and power for Intel (i915) need privileges a normal user lacks, so they live apart# from the sysfs readings above: the i915 performance counters (engine busy time) via perf_event_open-# and GPU power via RAPL's uncore energy zone. File descriptors stay open for the process's life-# (like Adl/NVML handles); the previous readings sit beside them. A card whose counters can never be-# opened records why ("denied" or "missing") instead of retrying a failing syscall every refresh.+# and GPU power via RAPL's uncore energy zone. File descriptors stay open for the process's life and+# are closed by DrmCounters.close(); the previous readings sit beside them. A card whose counters can+# never be opened records why ("denied", "missing" or "shared") instead of retrying a failing syscall+# every refresh; RAPL failures are cached the same way. See intel_counter_status for what each means._PERF_NR = {"x86_64": 298, "aarch64": 241} # __NR_perf_event_open_INTEL_PMU_ENGINES = (("render", "rcs0-busy"), ("copy", "bcs0-busy"),("video", "vcs0-busy"), ("video-enhance", "vecs0-busy"))_INTEL_PMU_STATE: dict[str, dict] = {} # card sysfs dir -> {"fds", "prev", "fail"}-_INTEL_RAPL_STATE: dict[str, dict] = {} # energy_uj path -> {"max", "prev", "fail"}+_INTEL_RAPL_PREV: dict[str, tuple[int, float]] = {} # energy_uj path -> (last uJ, last read time)+_INTEL_RAPL_FAIL: str | None = None # why the uncore zone can't be read: denied, missing, or None-_LIBC = ctypes.CDLL(None, use_errno=True)-_LIBC.syscall.restype = ctypes.c_long--def _perf_open(pmu_type: int, config: int, cpu: int) -> int | None:- """One system-wide perf counter fd, or None. The explicit c_long wrappers matter: plain ints- would cross the variadic call boundary as 32-bit values."""+def _perf_open(pmu_type: int, config: int, cpu: int) -> tuple[int | None, str | None]:+ """One system-wide perf counter fd, or (None, reason). The explicit c_long wrappers matter: plain+ ints would cross the variadic call boundary as 32-bit values. Loading libc lazily (and catching+ its absence) keeps importing this module safe where syscall(2) doesn't exist."""nr = _PERF_NR.get(platform.machine())if nr is None:- return None+ return None, "missing" # unsupported architecture: unknown, not denied+ try:+ libc = ctypes.CDLL(None, use_errno=True)+ syscall = libc.syscall+ except (OSError, AttributeError):+ return None, "missing" # no syscall export (e.g. Windows): the counters don't exist here+ syscall.restype = ctypes.c_longattr = ctypes.create_string_buffer(64) # PERF_ATTR_SIZE_VER0: type/size/config is all counting needsstruct.pack_into("IIQ", attr, 0, pmu_type, 64, config)- fd = _LIBC.syscall(ctypes.c_long(nr), attr, ctypes.c_long(-1),- ctypes.c_long(cpu), ctypes.c_long(-1), ctypes.c_ulong(0))- return fd if fd >= 0 else None+ fd = syscall(ctypes.c_long(nr), attr, ctypes.c_long(-1),+ ctypes.c_long(cpu), ctypes.c_long(-1), ctypes.c_ulong(0))+ if fd >= 0:+ return fd, None+ err = ctypes.get_errno()+ return None, ("denied" if err in (errno.EACCES, errno.EPERM) else "missing")def _intel_pmu_spec(pmu_root: str = "/sys/bus/event_source/devices/i915") -> tuple[int, int, dict[str, int]] | None:@@ -611,7 +620,7 @@ def _intel_pmu_spec(pmu_root: str = "/sys/bus/event_source/devices/i915") -> tupif pmu_type is None or not mask:return Nonetry:- cpu = int(re.split(r"[-,]", mask.strip())[0])+ cpu = int(re.split(r"[-,]", mask.strip())[0], 0)except ValueError:return Noneconfigs = {}@@ -650,13 +659,15 @@ def _intel_pmu_rates(sysfs: str, now: int | None = None) -> tuple[float | None,return None, Nonepmu_type, cpu, configs = specfor kind, config in configs.items():- fd = _perf_open(pmu_type, config, cpu)+ fd, fail = _perf_open(pmu_type, config, cpu)if fd is None:for open_fd in state["fds"].values():- os.close(open_fd)+ try:+ os.close(open_fd)+ except OSError:+ passstate["fds"] = {}- err = ctypes.get_errno()- state["fail"] = "denied" if err in (errno.EACCES, errno.EPERM) else "missing"+ state["fail"] = failreturn None, Nonestate["fds"][kind] = fdif not state["fds"]:@@ -682,7 +693,9 @@ def _intel_pmu_rates(sysfs: str, now: int | None = None) -> tuple[float | None,delta, elapsed = value - prev[0], now - prev[1]if elapsed <= 0 or delta < 0:continue # torn read or stepped-back counter: unknown, not a negative load- busy[kind] = delta * 100.0 / elapsed+ # An engine can't be busier than the wall clock it ran against; clamp the skew instead of+ # displaying an impossible load, like the per-process path does.+ busy[kind] = min(100.0, delta * 100.0 / elapsed)if not busy:return None, Nonevideo = max([v for k, v in busy.items() if k in ("video", "video-enhance")], default=None)@@ -716,52 +729,46 @@ def _intel_rapl_zone(powercap_root: str = "/sys/class/powercap") -> tuple[str, idef _intel_rapl_w(now: float | None = None) -> float | None:"""GPU (uncore) power in watts from RAPL's cumulative energy counter; None when unreadable.- The first sample only arms the baseline. A wrapped counter adds its range (unknown range with a- wrapped counter is unknown, not a negative power draw)."""+ The first sample only arms the baseline. A stepped-back counter (reset, suspend) re-arms it+ instead of reporting a spike, like the PMU path. Denied and missing failures are cached: the+ zone won't appear mid-run."""+ global _INTEL_RAPL_FAIL+ if _INTEL_RAPL_FAIL is not None:+ return Noneif now is None:now = time.perf_counter()zone = _intel_rapl_zone()if zone is None:+ _INTEL_RAPL_FAIL = "missing"return None- energy_path, max_range = zone- state = _INTEL_RAPL_STATE.get(energy_path)- if state is None:- state = {"prev": None, "fail": None}- _INTEL_RAPL_STATE[energy_path] = state+ energy_path, _max_range = zonetry:with open(energy_path, encoding="utf-8") as f:energy = int(f.read().strip())- except (OSError, ValueError):- try:- with open(energy_path, encoding="utf-8") as f:- f.read(1)- state["fail"] = "missing" # readable but unparseable: a broken counter, not denial- except OSError as err:- state["fail"] = "denied" if err.errno in (errno.EACCES, errno.EPERM) else "missing"+ except OSError as err:+ _INTEL_RAPL_FAIL = "denied" if err.errno in (errno.EACCES, errno.EPERM) else "missing"return None- prev = state["prev"]- state["prev"] = (energy, now)+ except ValueError:+ _INTEL_RAPL_FAIL = "missing" # readable but unparseable: a broken counter, not denial+ return None+ prev = _INTEL_RAPL_PREV.get(energy_path)+ _INTEL_RAPL_PREV[energy_path] = (energy, now)if prev is None:return Nonedelta, seconds = energy - prev[0], now - prev[1]- if seconds <= 0:+ if seconds <= 0 or delta < 0:return None- if delta < 0:- if max_range is None:- return None- delta += max_range- return delta / 1e6 / seconds if delta >= 0 else None+ return delta / 1e6 / seconds-def intel_counter_status(card: DrmCard) -> str:+def intel_counter_status(card: DrmCard, nv=None) -> str:"""Why Intel whole-GPU load, video load or power are N/A, for Snapshot.counter_status; "" when- all present (or not Intel). Unknown stays explained, never a bare blank."""+ all present (or not Intel). Unknown stays explained, never a bare blank. The nv reading tells+ which source actually served power, so a working hwmon sensor is never blamed on RAPL."""if not card.intel:return ""pmu_fail = _INTEL_PMU_STATE.get(card.sysfs, {}).get("fail")- states = [s for s in _INTEL_RAPL_STATE.values()]- rapl_fail = states[0]["fail"] if states else None- if pmu_fail is None and rapl_fail is None:+ if pmu_fail is None and _INTEL_RAPL_FAIL is None:return ""parts = []if pmu_fail == "denied":@@ -769,10 +776,13 @@ def intel_counter_status(card: DrmCard) -> str:parts.append(f"GPU and video load need perf privileges this user lacks (perf_event_paranoid={para})")elif pmu_fail == "missing":parts.append("no i915 performance counters on this kernel")- if card.integrated:- if rapl_fail == "denied":+ elif pmu_fail == "shared":+ parts.append("performance counters can't attribute load with multiple Intel GPUs")+ hwmon_power = nv is not None and nv.power_kind == "Board"+ if card.integrated and not hwmon_power:+ if _INTEL_RAPL_FAIL == "denied":parts.append("GPU power needs root (RAPL energy is root-only)")- elif rapl_fail == "missing":+ elif _INTEL_RAPL_FAIL == "missing":parts.append("no RAPL uncore zone found")if not parts:return ""@@ -859,7 +869,14 @@ class DrmCounters:budget, self.scan_budget = self.scan_budget, Noneself.sample()self.scan_budget = budget- for card in self.cards:+ intel_cards = [c for c in self.cards if c.intel]+ if len(intel_cards) > 1:+ # One i915 PMU for the whole driver: its counters can't be attributed to either card,+ # so neither gets device load instead of both showing the same numbers.+ for card in intel_cards:+ _INTEL_PMU_STATE.setdefault(card.sysfs, {"fds": {}, "prev": {}, "fail": "shared"})+ return+ for card in intel_cards:if card.intel:# Arm the device-wide baselines too: --once sleeps a second after Collector init,# so its single sample is already a rate over a real interval.@@ -1141,4 +1158,11 @@ class DrmCounters:return procs, adaptersdef close(self) -> None:- """No handles are held."""\ No newline at end of file+ """Close any open Intel PMU file descriptors."""+ for sysfs in list(_INTEL_PMU_STATE):+ for fd in _INTEL_PMU_STATE[sysfs].get("fds", {}).values():+ try:+ os.close(fd)+ except OSError:+ pass+ del _INTEL_PMU_STATE[sysfs]\ No newline at end of filediff --git a/tests/test_drm.py b/tests/test_drm.pyindex 5df11eb..8a82e6b 100644--- a/tests/test_drm.py+++ b/tests/test_drm.py@@ -4,6 +4,7 @@ import osimport sysimport tempfileimport unittest+from types import SimpleNamespacefrom unittest.mock import patchfrom pathlib import Path@@ -1563,7 +1564,7 @@ class IntelPmuTests(unittest.TestCase):root = Path(tmp)self._pmu(root)with patch.object(drm, "_intel_pmu_spec", return_value=(28, 0, {"render": 0, "copy": 0x1000})), \- patch.object(drm, "_perf_open", side_effect=[10, 11]) as opened, \+ patch.object(drm, "_perf_open", side_effect=[(10, None), (11, None)]) as opened, \patch.object(drm, "_read_counter", side_effect=[0, 0, 500_000_000, 250_000_000]):self.assertEqual(drm._intel_pmu_rates("/x/card1/device", now=0), (None, None))self.assertEqual(opened.call_count, 2)@@ -1574,7 +1575,7 @@ class IntelPmuTests(unittest.TestCase):def test_video_is_the_video_engines_busy_share(self):with tempfile.TemporaryDirectory() as tmp:with patch.object(drm, "_intel_pmu_spec", return_value=(28, 0, {"video": 0, "video-enhance": 1})), \- patch.object(drm, "_perf_open", side_effect=[10, 11]), \+ patch.object(drm, "_perf_open", side_effect=[(10, None), (11, None)]), \patch.object(drm, "_read_counter", side_effect=[0, 0, 800_000_000, 200_000_000]):drm._intel_pmu_rates("/x/card1/device", now=0)self.assertEqual(drm._intel_pmu_rates("/x/card1/device", now=1_000_000_000), (80.0, 80.0))@@ -1583,7 +1584,7 @@ class IntelPmuTests(unittest.TestCase):def test_stepped_back_counter_re_arms(self):with tempfile.TemporaryDirectory() as tmp:with patch.object(drm, "_intel_pmu_spec", return_value=(28, 0, {"render": 0})), \- patch.object(drm, "_perf_open", return_value=10), \+ patch.object(drm, "_perf_open", return_value=(10, None)), \patch.object(drm, "_read_counter", side_effect=[1_000_000_000, 900_000_000, 1_500_000_000]) as reads:drm._intel_pmu_rates("/x/card1/device", now=0)self.assertEqual(drm._intel_pmu_rates("/x/card1/device", now=1_000_000_000), (None, None))@@ -1593,8 +1594,7 @@ class IntelPmuTests(unittest.TestCase):def test_denied_open_reports_denied(self):with tempfile.TemporaryDirectory() as tmp:with patch.object(drm, "_intel_pmu_spec", return_value=(28, 0, {"render": 0})), \- patch.object(drm, "_perf_open", return_value=None), \- patch.object(drm.ctypes, "get_errno", return_value=13):+ patch.object(drm, "_perf_open", return_value=(None, "denied")):self.assertEqual(drm._intel_pmu_rates("/x/card1/device"), (None, None))self.assertEqual(drm._INTEL_PMU_STATE["/x/card1/device"]["fail"], "denied")@@ -1603,16 +1603,50 @@ class IntelPmuTests(unittest.TestCase):self.assertEqual(drm._intel_pmu_rates("/x/card1/device"), (None, None))self.assertEqual(drm._INTEL_PMU_STATE["/x/card1/device"]["fail"], "missing")+ def test_unsupported_arch_reports_missing_not_denied(self):+ with patch.object(drm.platform, "machine", return_value="riscv"), \+ patch.object(drm.ctypes, "get_errno", return_value=13):+ self.assertEqual(drm._perf_open(28, 0, 0), (None, "missing"))++ def test_busy_cannot_exceed_wall_time(self):+ with patch.object(drm, "_intel_pmu_spec", return_value=(28, 0, {"render": 0})), \+ patch.object(drm, "_perf_open", return_value=(10, None)), \+ patch.object(drm, "_read_counter", side_effect=[0, 3_000_000_000]):+ drm._intel_pmu_rates("/x/card1/device", now=0)+ util, _video = drm._intel_pmu_rates("/x/card1/device", now=1_000_000_000)+ self.assertEqual(util, 100.0)++ def test_two_intel_gpus_share_no_pmu(self):+ # One i915 PMU for the whole driver: with two Intel cards neither gets device load,+ # instead of both showing the same numbers.+ with tempfile.TemporaryDirectory() as tmp:+ proc = Path(tmp) / "proc"+ proc.mkdir()+ a = drm.DrmCard(key=1, pdev="0000:00:02.0", sysfs=str(Path(tmp) / "a"), driver="i915",+ vendor_id=0x8086, device_id=0xA782, name="Intel UHD Graphics",+ vram_total=0, integrated=True, intel=True)+ b = drm.DrmCard(key=2, pdev="0000:03:00.0", sysfs=str(Path(tmp) / "b"), driver="i915",+ vendor_id=0x8086, device_id=0x56A0, name="Intel Arc A770",+ vram_total=16 * GiB, integrated=False, intel=True)+ counters = drm.DrmCounters(found=[a, b], proc=str(proc), clock=Clock())+ counters.start()+ self.assertEqual(drm._INTEL_PMU_STATE[str(Path(tmp) / "a")]["fail"], "shared")+ self.assertEqual(drm._INTEL_PMU_STATE[str(Path(tmp) / "b")]["fail"], "shared")+class IntelRaplTests(unittest.TestCase):"""GPU power from RAPL's uncore zone, against a fake powercap tree."""def setUp(self):- drm._INTEL_RAPL_STATE.clear()+ drm._INTEL_RAPL_PREV.clear()+ drm._INTEL_RAPL_FAIL = Nonedef _zone(self, root, energy="1000000\n", max_range="100000000\n", name="uncore"):package = root / "intel-rapl:0"package.mkdir(parents=True, exist_ok=True)+ sibling = package / "intel-rapl:0:0"+ sibling.mkdir(exist_ok=True)+ (sibling / "name").write_text("core\n") # a named non-uncore zone the search must skipzone = package / "intel-rapl:0:1"zone.mkdir(exist_ok=True)(zone / "name").write_text(name + "\n")@@ -1627,7 +1661,6 @@ class IntelRaplTests(unittest.TestCase):with tempfile.TemporaryDirectory() as tmp:root = Path(tmp)energy = self._zone(root)- (root / "intel-rapl:0" / "intel-rapl:0:0" / "name").parent.mkdir(parents=True, exist_ok=True)self.assertEqual(drm._intel_rapl_zone(str(root))[0], energy)@unittest.skipIf(sys.platform == "win32", "powercap sysfs names contain a colon")@@ -1648,26 +1681,30 @@ class IntelRaplTests(unittest.TestCase):Path(energy).write_text("6000000\n") # 5 J in 1 sself.assertAlmostEqual(drm._intel_rapl_w(now=1.0), 5.0)- def test_wrapped_counter_adds_its_range(self):+ def test_stepped_back_counter_re_arms(self):with tempfile.TemporaryDirectory() as tmp:energy = self._zone(Path(tmp))- with patch.object(drm, "_intel_rapl_zone", return_value=(energy, 10_000_000)):- self.assertIsNone(drm._intel_rapl_w(now=0.0))- Path(energy).write_text("1000\n") # wrapped past 10 mJ- self.assertAlmostEqual(drm._intel_rapl_w(now=1.0), 9.001)-- def test_wrapped_counter_without_range_is_unknown(self):- with tempfile.TemporaryDirectory() as tmp:- energy = self._zone(Path(tmp), max_range=None)- with patch.object(drm, "_intel_rapl_zone", return_value=(energy, None)):+ with patch.object(drm, "_intel_rapl_zone", return_value=(energy, 100_000_000)):self.assertIsNone(drm._intel_rapl_w(now=0.0))- Path(energy).write_text("1000\n")+ Path(energy).write_text("1000\n") # reset, not a negative power drawself.assertIsNone(drm._intel_rapl_w(now=1.0))+ Path(energy).write_text("6001000\n")+ self.assertAlmostEqual(drm._intel_rapl_w(now=2.0), 6.0)- def test_unreadable_zone_reports_denied(self):- with patch.object(drm, "_intel_rapl_zone", return_value=("/no/energy_uj", 10_000_000)):+ def test_missing_zone_reports_missing(self):+ with patch.object(drm, "_intel_rapl_zone", return_value=None):self.assertIsNone(drm._intel_rapl_w(now=0.0))- self.assertEqual(drm._INTEL_RAPL_STATE["/no/energy_uj"]["fail"], "missing")+ self.assertEqual(drm._INTEL_RAPL_FAIL, "missing")++ def test_unreadable_zone_reports_denied(self):+ if os.geteuid() == 0:+ self.skipTest("root bypasses file permissions")+ with tempfile.TemporaryDirectory() as tmp:+ energy = self._zone(Path(tmp))+ os.chmod(energy, 0)+ with patch.object(drm, "_intel_rapl_zone", return_value=(energy, 10_000_000)):+ self.assertIsNone(drm._intel_rapl_w(now=0.0))+ self.assertEqual(drm._INTEL_RAPL_FAIL, "denied")@unittest.skipIf(sys.platform == "win32", "powercap sysfs names contain a colon")def test_unparseable_zone_reports_missing(self):@@ -1675,7 +1712,7 @@ class IntelRaplTests(unittest.TestCase):energy = self._zone(Path(tmp), energy="lots\n")with patch.object(drm, "_intel_rapl_zone", return_value=(energy, 10_000_000)):self.assertIsNone(drm._intel_rapl_w(now=0.0))- self.assertEqual(drm._INTEL_RAPL_STATE[energy]["fail"], "missing")+ self.assertEqual(drm._INTEL_RAPL_FAIL, "missing")class IntelCounterStatusTests(unittest.TestCase):@@ -1688,41 +1725,54 @@ class IntelCounterStatusTests(unittest.TestCase):def setUp(self):drm._INTEL_PMU_STATE.clear()- drm._INTEL_RAPL_STATE.clear()+ drm._INTEL_RAPL_PREV.clear()+ drm._INTEL_RAPL_FAIL = Nonedef test_not_intel_is_silent(self):card = drm.DrmCard(key=0, pdev="0000:03:00.0", sysfs="/x/amd/device", driver="amdgpu",vendor_id=0x1002, device_id=0x7480, name="AMD GPU", vram_total=8 * GiB,integrated=False)- self.assertEqual(drm.intel_counter_status(card), "")+ self.assertEqual(drm.intel_counter_status(card, None), "")def test_denied_names_privileges(self):drm._INTEL_PMU_STATE["/x/card1/device"] = {"fail": "denied"}- drm._INTEL_RAPL_STATE["/z/energy_uj"] = {"fail": "denied", "prev": None}+ drm._INTEL_RAPL_FAIL = "denied"with patch.object(drm, "_read", return_value="2"):- status = drm.intel_counter_status(self._card())+ status = drm.intel_counter_status(self._card(), None)self.assertIn("perf_event_paranoid=2", status)self.assertIn("root", status)self.assertIn("unknown, not zero", status)def test_missing_names_the_gap(self):drm._INTEL_PMU_STATE["/x/card1/device"] = {"fail": "missing"}- drm._INTEL_RAPL_STATE["/z/energy_uj"] = {"fail": "missing", "prev": None}- status = drm.intel_counter_status(self._card())+ drm._INTEL_RAPL_FAIL = "missing"+ status = drm.intel_counter_status(self._card(), None)self.assertIn("performance counters", status)self.assertIn("uncore", status)+ def test_hwmon_power_needs_no_rapal_reason(self):+ # A discrete card's hwmon sensor already answered: RAPL was never needed, so its+ # failure (cached from the unconditional start() priming) must not be reported.+ drm._INTEL_RAPL_FAIL = "denied"+ nv = SimpleNamespace(power_kind="Board")+ self.assertEqual(drm.intel_counter_status(self._card(), nv), "")+def test_all_present_is_silent(self):- self.assertEqual(drm.intel_counter_status(self._card()), "")+ self.assertEqual(drm.intel_counter_status(self._card(), None), "")def test_discrete_card_omits_rapal(self):drm._INTEL_PMU_STATE["/x/card1/device"] = {"fail": "denied"}- drm._INTEL_RAPL_STATE["/z/energy_uj"] = {"fail": "denied", "prev": None}+ drm._INTEL_RAPL_FAIL = "denied"with patch.object(drm, "_read", return_value="2"):- status = drm.intel_counter_status(self._card(integrated=False))+ status = drm.intel_counter_status(self._card(integrated=False), None)self.assertIn("load", status)self.assertNotIn("RAPL", status)+ def test_shared_pmu_is_refused(self):+ drm._INTEL_PMU_STATE["/x/card1/device"] = {"fail": "shared"}+ status = drm.intel_counter_status(self._card(), None)+ self.assertIn("multiple Intel GPUs", status)+class IntelNoteTests(unittest.TestCase):"""Intel's device VRAM used is a sum over the clients gputaskman can inspect and its total is a PCI window size,