A GPU task manager for Windows, Linux, and WSL: per-process VRAM, GPU activity, and LLM model/KV-cache insights (Windows AMD and NVIDIA).
Python
Address review: LLM view on integrated GPUs, unknown throttle
Martin Pluskal5e7209fe4149
diff --git a/src/gputaskman/collector.py b/src/gputaskman/collector.pyindex 9cfed6b..f2befa7 100644--- a/src/gputaskman/collector.py+++ b/src/gputaskman/collector.py@@ -180,6 +180,17 @@ def select_nv_device(adapter: Adapter | None, devices: list[NvDevice], gpu_indexreturn matches[0] if len(matches) == 1 else None+def _llm_wanted(row, shared_is_spill: bool) -> bool:+ """Whether a process row gets an LLM breakdown: an AI-kind process with models (or kind LLM),+ not diffusion, and with measured GPU memory. Where spilled system memory is a counter of its own+ (WDDM, discrete DRM cards) both the dedicated and shared counters must be present; on an integrated+ GPU system memory is the memory and there is no spill counter, so dedicated alone suffices."""+ if row.detail.is_diffusion or row.kind not in AI_KINDS or not (row.detail.models or row.kind == "LLM"):+ return False+ need = {"dedicated", "shared"} if shared_is_spill else {"dedicated"}+ return need <= row.gpu.available++class Collector:DEEP_TOP_N = 25 # memory-map candidates: the top N VRAM users plus anything busy_mac_driver = "" # "macOS 26.3.1" on the metal backend, read once at startup@@ -333,8 +344,7 @@ class Collector:continueif os.path.basename(row.detail.exe).lower() in ("handy.exe", "handy"):row.handy = inspect_handy(row.detail.create_time)- if (row.detail.is_diffusion or row.kind not in AI_KINDS or not (row.detail.models or row.kind == "LLM")- or not {"dedicated", "shared"} <= row.gpu.available):+ if not _llm_wanted(row, snap.shared_is_spill):continueports = self.inspector.exclusive_ports(row.pid)key = api_key(row.detail.cmdline)@@ -342,7 +352,7 @@ class Collector:host = self.prober.get(self._parent_ports(row.pid)) # Ollama / LM Studio front-end processlog = ollama_log_path() if row in ollama_runners and len(ollama_runners) == 1 else None# On an APU, GTT is the GPU's ordinary memory (often most of it), not spill: count it as held memory.- held = (row.gpu.shown_vram or 0) + (0 if snap.shared_is_spill else row.gpu.shared)+ held = (row.gpu.shown_vram or 0) + (0 if snap.shared_is_spill else (row.gpu.shared or 0))row.llm = build_view(row.detail, held, row.gpu.shared if snap.shared_is_spill else 0,own or host, log, host=host if own else None,attributed=row.gpu.shown_vram is not None)diff --git a/src/gputaskman/drm.py b/src/gputaskman/drm.pyindex 298a266..6202142 100644--- a/src/gputaskman/drm.py+++ b/src/gputaskman/drm.py@@ -61,8 +61,9 @@ class DrmCard:# BAR) window, an estimate of the card's memory; 0 means unknown (an integrated GPU, or a discrete card# whose window is under 1 GiB). Per-process bytes still come from fdinfo.intel: bool = False- # Intel only: used local memory, stamped in each DrmCounters.sample() as the sum of the card's- # clients' local bytes (shared buffers can double-count); None = no readable client, unknown.+ # Intel only: used memory, stamped in each DrmCounters.sample() as the sum of the card's+ # clients' resident bytes (local bytes on a discrete card, system bytes on an integrated one;+ # shared buffers can double-count); None = no readable client, unknown.vram_used: int | None = None@@ -410,12 +411,22 @@ _INTEL_THROTTLE = ()-def _intel_throttle(gt: str) -> list[str]:- """Names of the limits currently throttling an Intel GPU; empty when none is (or none is readable).+def _intel_throttle(gt: str) -> list[str] | None:+ """Names of the limits currently throttling an Intel GPU; empty when none is active.- _int already turns an unreadable or unparseable node into None, so those are skipped: unknown,- not inactive."""- return [label for node, label in _INTEL_THROTTLE if _int(os.path.join(gt, node))]+ _int already turns an unreadable or unparseable node into None, so those are skipped. None when no+ node was readable at all: unknown, not "no throttling" (an empty list means readable and idle).+ """+ seen = False+ active = []+ for node, label in _INTEL_THROTTLE:+ value = _int(os.path.join(gt, node))+ if value is None:+ continue+ seen = True+ if value:+ active.append(label)+ return active if seen else Nonedef read_device(card: DrmCard, index: int) -> DeviceTelemetry:@@ -833,6 +844,9 @@ class DrmCounters:proc.attributed = Falseproc.available -= {"dedicated", "shared"}elif integrated_intel:+ # Unified memory: the same system bytes are the charge (vram_bytes) and the reachable+ # reference (gtt_resident_bytes). Both are measured and labelled; diff.py compares+ # per-field and never sums them.proc.dedicated = proc.resident_gttelse:proc.dedicated = proc.resident_vramdiff --git a/tests/test_core.py b/tests/test_core.pyindex bc61ee0..cd6188a 100644--- a/tests/test_core.py+++ b/tests/test_core.py@@ -24,7 +24,7 @@ from gputaskman.amdsmi import AmdSmifrom gputaskman.adl import Adl, _Metricsfrom gputaskman.app import GpuTaskManager, spark, stacked_barfrom gputaskman.cli import main-from gputaskman.collector import Collector, ProcRow, Snapshot, select_adapter, select_nv_device+from gputaskman.collector import Collector, ProcRow, Snapshot, _llm_wanted, select_adapter, select_nv_devicefrom gputaskman.dxgi import Adapterfrom gputaskman.gguf import GgufInfo, read_gguffrom gputaskman.handy import HandyEvidence, inspect_handy, _log_cache, _settings_cache@@ -1108,6 +1108,36 @@ class ChartTests(unittest.TestCase):self.assertTrue(any(span.style == colors["Mem activity"] for span in metrics.spans))+class LlmWantedTests(unittest.TestCase):+ """The LLM-breakdown gate: what measured memory a row needs on spill vs integrated GPUs."""++ def _row(self, kind="LLM", models=("m.gguf",), diffusion=False, available=("dedicated", "shared")):+ return SimpleNamespace(+ kind=kind,+ detail=SimpleNamespace(is_diffusion=diffusion, models=list(models)),+ gpu=SimpleNamespace(available=set(available)),+ )++ def test_spill_backend_needs_both_counters(self):+ self.assertTrue(_llm_wanted(self._row(), True))+ row = self._row(available=("dedicated",))+ self.assertFalse(_llm_wanted(row, True)) # discrete card without a spill counter: no breakdown+ row = self._row(available=("shared",))+ self.assertFalse(_llm_wanted(row, True))++ def test_integrated_gpu_needs_only_dedicated(self):+ # An integrated GPU has no spill counter; a provably private client's memory suffices.+ self.assertTrue(_llm_wanted(self._row(available=("dedicated",)), False))+ row = self._row(available=set())+ self.assertFalse(_llm_wanted(row, False))++ def test_non_ai_and_diffusion_rows_are_out(self):+ self.assertFalse(_llm_wanted(self._row(kind="3D/game"), False))+ self.assertFalse(_llm_wanted(self._row(diffusion=True), True))+ # Kind LLM alone still wants a view when its counters are measured.+ self.assertTrue(_llm_wanted(self._row(kind="LLM", models=()), True))++class UiTests(unittest.IsolatedAsyncioTestCase):def setUp(self):# These snapshots are NVIDIA/WDDM ones: render them with that backend's charts on every OS (macOS too).diff --git a/tests/test_drm.py b/tests/test_drm.pyindex 54dfab0..b5e1a00 100644--- a/tests/test_drm.py+++ b/tests/test_drm.py@@ -457,7 +457,6 @@ class ReadDeviceTests(unittest.TestCase):(gt / "rps_act_freq_mhz").write_text("300\n")(gt / "rps_cur_freq_mhz").write_text("350\n")(gt / "rps_RP0_freq_mhz").write_text("1250\n")- (gt / "throttle_reason_status").write_text("1\n")(gt / "throttle_reason_pl1").write_text("1\n")(gt / "throttle_reason_pl2").write_text("0\n")(gt / "throttle_reason_thermal").write_text("0\n")@@ -489,7 +488,23 @@ class ReadDeviceTests(unittest.TestCase):d = drm.read_device(card, 0)self.assertEqual(d.sm_clock, 300)- self.assertEqual(d.throttle, [])+ # No throttle node is readable here: unknown, not "no throttling".+ self.assertIsNone(d.throttle)++ def test_read_device_intel_throttle_all_idle_is_empty_not_unknown(self):+ with tempfile.TemporaryDirectory() as tmp:+ card_sysfs = Path(tmp) / "card1"+ dev = card_sysfs / "device"+ dev.mkdir(parents=True)+ gt = card_sysfs / "gt" / "gt0"+ gt.mkdir(parents=True)+ (gt / "throttle_reason_pl1").write_text("0\n")+ (gt / "throttle_reason_thermal").write_text("0\n")++ card = drm.DrmCard(key=32, pdev="0000:00:02.0", sysfs=str(dev), driver="i915",+ vendor_id=0x8086, device_id=0xA7AD, name="Intel Intel Graphics",+ vram_total=0, integrated=True, intel=True)+ self.assertEqual(drm.read_device(card, 0).throttle, [])def test_intel_throttle_skips_unreadable_nodes(self):with tempfile.TemporaryDirectory() as tmp:@@ -1348,6 +1363,30 @@ class IntelScanTests(unittest.TestCase):# The stamped device total sums resident system memory (shared buffers can double-count it).self.assertEqual(adapters[card.key].dedicated, 37476 * KiB)+ def test_mixed_discrete_and_integrated_cards_charge_each_by_its_rule(self):+ disc_pdev = "0000:03:00.0"+ igpu_pdev = "0000:00:02.0"+ with tempfile.TemporaryDirectory() as tmp:+ proc = Path(tmp) / "proc"+ proc.mkdir()+ disc = drm.DrmCard(key=drm.pci_key(disc_pdev), pdev=disc_pdev, sysfs=str(Path(tmp) / "d" / "device"),+ driver="i915", vendor_id=0x8086, device_id=0x56A0, name="Intel Arc A770",+ vram_total=16 * GiB, integrated=False, intel=True)+ igpu = drm.DrmCard(key=drm.pci_key(igpu_pdev), pdev=igpu_pdev, sysfs=str(Path(tmp) / "i" / "device"),+ driver="i915", vendor_id=0x8086, device_id=0xA782, name="Intel UHD Graphics",+ vram_total=0, integrated=True, intel=True)+ counters = drm.DrmCounters(found=[disc, igpu], proc=str(proc), clock=Clock())+ _write_fd(proc, 100, 5, _i915_fdinfo(1, disc_pdev, local_mib=64, system_mib=8))+ _write_fd(proc, 200, 6, I915_IGPU_FDINFO)+ procs, adapters = counters.sample()+ g_disc = procs[disc.key][100]+ self.assertEqual((g_disc.dedicated, g_disc.shared), (64 * MiB, 8 * MiB))+ g_igpu = procs[igpu.key][200]+ self.assertEqual(g_igpu.dedicated, 60 * MiB)+ self.assertNotIn("shared", g_igpu.available)+ self.assertEqual(adapters[disc.key].dedicated, 64 * MiB)+ self.assertEqual(adapters[igpu.key].dedicated, 60 * MiB)+def test_device_used_sums_each_client_once_and_is_unknown_when_incomplete(self):with tempfile.TemporaryDirectory() as tmp:card, proc, counters = self._setup(tmp)