Skip to content

A GPU task manager for Windows, Linux, and WSL: per-process VRAM, GPU activity, and LLM model/KV-cache insights (Windows AMD and NVIDIA).

Python

Address review: LLM view on integrated GPUs, unknown throttle

Martin Pluskal5e7209fe4149
diff --git a/src/gputaskman/collector.py b/src/gputaskman/collector.py
index 9cfed6b..f2befa7 100644
--- a/src/gputaskman/collector.py
+++ b/src/gputaskman/collector.py
@@ -180,6 +180,17 @@ def select_nv_device(adapter: Adapter | None, devices: list[NvDevice], gpu_index
return matches[0] if len(matches) == 1 else None
+def _llm_wanted(row, shared_is_spill: bool) -> bool:
+ """Whether a process row gets an LLM breakdown: an AI-kind process with models (or kind LLM),
+ not diffusion, and with measured GPU memory. Where spilled system memory is a counter of its own
+ (WDDM, discrete DRM cards) both the dedicated and shared counters must be present; on an integrated
+ GPU system memory is the memory and there is no spill counter, so dedicated alone suffices."""
+ if row.detail.is_diffusion or row.kind not in AI_KINDS or not (row.detail.models or row.kind == "LLM"):
+ return False
+ need = {"dedicated", "shared"} if shared_is_spill else {"dedicated"}
+ return need <= row.gpu.available
+
+
class Collector:
DEEP_TOP_N = 25 # memory-map candidates: the top N VRAM users plus anything busy
_mac_driver = "" # "macOS 26.3.1" on the metal backend, read once at startup
@@ -333,8 +344,7 @@ class Collector:
continue
if os.path.basename(row.detail.exe).lower() in ("handy.exe", "handy"):
row.handy = inspect_handy(row.detail.create_time)
- if (row.detail.is_diffusion or row.kind not in AI_KINDS or not (row.detail.models or row.kind == "LLM")
- or not {"dedicated", "shared"} <= row.gpu.available):
+ if not _llm_wanted(row, snap.shared_is_spill):
continue
ports = self.inspector.exclusive_ports(row.pid)
key = api_key(row.detail.cmdline)
@@ -342,7 +352,7 @@ class Collector:
host = self.prober.get(self._parent_ports(row.pid)) # Ollama / LM Studio front-end process
log = ollama_log_path() if row in ollama_runners and len(ollama_runners) == 1 else None
# On an APU, GTT is the GPU's ordinary memory (often most of it), not spill: count it as held memory.
- held = (row.gpu.shown_vram or 0) + (0 if snap.shared_is_spill else row.gpu.shared)
+ held = (row.gpu.shown_vram or 0) + (0 if snap.shared_is_spill else (row.gpu.shared or 0))
row.llm = build_view(row.detail, held, row.gpu.shared if snap.shared_is_spill else 0,
own or host, log, host=host if own else None,
attributed=row.gpu.shown_vram is not None)
diff --git a/src/gputaskman/drm.py b/src/gputaskman/drm.py
index 298a266..6202142 100644
--- a/src/gputaskman/drm.py
+++ b/src/gputaskman/drm.py
@@ -61,8 +61,9 @@ class DrmCard:
# BAR) window, an estimate of the card's memory; 0 means unknown (an integrated GPU, or a discrete card
# whose window is under 1 GiB). Per-process bytes still come from fdinfo.
intel: bool = False
- # Intel only: used local memory, stamped in each DrmCounters.sample() as the sum of the card's
- # clients' local bytes (shared buffers can double-count); None = no readable client, unknown.
+ # Intel only: used memory, stamped in each DrmCounters.sample() as the sum of the card's
+ # clients' resident bytes (local bytes on a discrete card, system bytes on an integrated one;
+ # shared buffers can double-count); None = no readable client, unknown.
vram_used: int | None = None
@@ -410,12 +411,22 @@ _INTEL_THROTTLE = (
)
-def _intel_throttle(gt: str) -> list[str]:
- """Names of the limits currently throttling an Intel GPU; empty when none is (or none is readable).
+def _intel_throttle(gt: str) -> list[str] | None:
+ """Names of the limits currently throttling an Intel GPU; empty when none is active.
- _int already turns an unreadable or unparseable node into None, so those are skipped: unknown,
- not inactive."""
- return [label for node, label in _INTEL_THROTTLE if _int(os.path.join(gt, node))]
+ _int already turns an unreadable or unparseable node into None, so those are skipped. None when no
+ node was readable at all: unknown, not "no throttling" (an empty list means readable and idle).
+ """
+ seen = False
+ active = []
+ for node, label in _INTEL_THROTTLE:
+ value = _int(os.path.join(gt, node))
+ if value is None:
+ continue
+ seen = True
+ if value:
+ active.append(label)
+ return active if seen else None
def read_device(card: DrmCard, index: int) -> DeviceTelemetry:
@@ -833,6 +844,9 @@ class DrmCounters:
proc.attributed = False
proc.available -= {"dedicated", "shared"}
elif integrated_intel:
+ # Unified memory: the same system bytes are the charge (vram_bytes) and the reachable
+ # reference (gtt_resident_bytes). Both are measured and labelled; diff.py compares
+ # per-field and never sums them.
proc.dedicated = proc.resident_gtt
else:
proc.dedicated = proc.resident_vram
diff --git a/tests/test_core.py b/tests/test_core.py
index bc61ee0..cd6188a 100644
--- a/tests/test_core.py
+++ b/tests/test_core.py
@@ -24,7 +24,7 @@ from gputaskman.amdsmi import AmdSmi
from gputaskman.adl import Adl, _Metrics
from gputaskman.app import GpuTaskManager, spark, stacked_bar
from gputaskman.cli import main
-from gputaskman.collector import Collector, ProcRow, Snapshot, select_adapter, select_nv_device
+from gputaskman.collector import Collector, ProcRow, Snapshot, _llm_wanted, select_adapter, select_nv_device
from gputaskman.dxgi import Adapter
from gputaskman.gguf import GgufInfo, read_gguf
from gputaskman.handy import HandyEvidence, inspect_handy, _log_cache, _settings_cache
@@ -1108,6 +1108,36 @@ class ChartTests(unittest.TestCase):
self.assertTrue(any(span.style == colors["Mem activity"] for span in metrics.spans))
+class LlmWantedTests(unittest.TestCase):
+ """The LLM-breakdown gate: what measured memory a row needs on spill vs integrated GPUs."""
+
+ def _row(self, kind="LLM", models=("m.gguf",), diffusion=False, available=("dedicated", "shared")):
+ return SimpleNamespace(
+ kind=kind,
+ detail=SimpleNamespace(is_diffusion=diffusion, models=list(models)),
+ gpu=SimpleNamespace(available=set(available)),
+ )
+
+ def test_spill_backend_needs_both_counters(self):
+ self.assertTrue(_llm_wanted(self._row(), True))
+ row = self._row(available=("dedicated",))
+ self.assertFalse(_llm_wanted(row, True)) # discrete card without a spill counter: no breakdown
+ row = self._row(available=("shared",))
+ self.assertFalse(_llm_wanted(row, True))
+
+ def test_integrated_gpu_needs_only_dedicated(self):
+ # An integrated GPU has no spill counter; a provably private client's memory suffices.
+ self.assertTrue(_llm_wanted(self._row(available=("dedicated",)), False))
+ row = self._row(available=set())
+ self.assertFalse(_llm_wanted(row, False))
+
+ def test_non_ai_and_diffusion_rows_are_out(self):
+ self.assertFalse(_llm_wanted(self._row(kind="3D/game"), False))
+ self.assertFalse(_llm_wanted(self._row(diffusion=True), True))
+ # Kind LLM alone still wants a view when its counters are measured.
+ self.assertTrue(_llm_wanted(self._row(kind="LLM", models=()), True))
+
+
class UiTests(unittest.IsolatedAsyncioTestCase):
def setUp(self):
# These snapshots are NVIDIA/WDDM ones: render them with that backend's charts on every OS (macOS too).
diff --git a/tests/test_drm.py b/tests/test_drm.py
index 54dfab0..b5e1a00 100644
--- a/tests/test_drm.py
+++ b/tests/test_drm.py
@@ -457,7 +457,6 @@ class ReadDeviceTests(unittest.TestCase):
(gt / "rps_act_freq_mhz").write_text("300\n")
(gt / "rps_cur_freq_mhz").write_text("350\n")
(gt / "rps_RP0_freq_mhz").write_text("1250\n")
- (gt / "throttle_reason_status").write_text("1\n")
(gt / "throttle_reason_pl1").write_text("1\n")
(gt / "throttle_reason_pl2").write_text("0\n")
(gt / "throttle_reason_thermal").write_text("0\n")
@@ -489,7 +488,23 @@ class ReadDeviceTests(unittest.TestCase):
d = drm.read_device(card, 0)
self.assertEqual(d.sm_clock, 300)
- self.assertEqual(d.throttle, [])
+ # No throttle node is readable here: unknown, not "no throttling".
+ self.assertIsNone(d.throttle)
+
+ def test_read_device_intel_throttle_all_idle_is_empty_not_unknown(self):
+ with tempfile.TemporaryDirectory() as tmp:
+ card_sysfs = Path(tmp) / "card1"
+ dev = card_sysfs / "device"
+ dev.mkdir(parents=True)
+ gt = card_sysfs / "gt" / "gt0"
+ gt.mkdir(parents=True)
+ (gt / "throttle_reason_pl1").write_text("0\n")
+ (gt / "throttle_reason_thermal").write_text("0\n")
+
+ card = drm.DrmCard(key=32, pdev="0000:00:02.0", sysfs=str(dev), driver="i915",
+ vendor_id=0x8086, device_id=0xA7AD, name="Intel Intel Graphics",
+ vram_total=0, integrated=True, intel=True)
+ self.assertEqual(drm.read_device(card, 0).throttle, [])
def test_intel_throttle_skips_unreadable_nodes(self):
with tempfile.TemporaryDirectory() as tmp:
@@ -1348,6 +1363,30 @@ class IntelScanTests(unittest.TestCase):
# The stamped device total sums resident system memory (shared buffers can double-count it).
self.assertEqual(adapters[card.key].dedicated, 37476 * KiB)
+ def test_mixed_discrete_and_integrated_cards_charge_each_by_its_rule(self):
+ disc_pdev = "0000:03:00.0"
+ igpu_pdev = "0000:00:02.0"
+ with tempfile.TemporaryDirectory() as tmp:
+ proc = Path(tmp) / "proc"
+ proc.mkdir()
+ disc = drm.DrmCard(key=drm.pci_key(disc_pdev), pdev=disc_pdev, sysfs=str(Path(tmp) / "d" / "device"),
+ driver="i915", vendor_id=0x8086, device_id=0x56A0, name="Intel Arc A770",
+ vram_total=16 * GiB, integrated=False, intel=True)
+ igpu = drm.DrmCard(key=drm.pci_key(igpu_pdev), pdev=igpu_pdev, sysfs=str(Path(tmp) / "i" / "device"),
+ driver="i915", vendor_id=0x8086, device_id=0xA782, name="Intel UHD Graphics",
+ vram_total=0, integrated=True, intel=True)
+ counters = drm.DrmCounters(found=[disc, igpu], proc=str(proc), clock=Clock())
+ _write_fd(proc, 100, 5, _i915_fdinfo(1, disc_pdev, local_mib=64, system_mib=8))
+ _write_fd(proc, 200, 6, I915_IGPU_FDINFO)
+ procs, adapters = counters.sample()
+ g_disc = procs[disc.key][100]
+ self.assertEqual((g_disc.dedicated, g_disc.shared), (64 * MiB, 8 * MiB))
+ g_igpu = procs[igpu.key][200]
+ self.assertEqual(g_igpu.dedicated, 60 * MiB)
+ self.assertNotIn("shared", g_igpu.available)
+ self.assertEqual(adapters[disc.key].dedicated, 64 * MiB)
+ self.assertEqual(adapters[igpu.key].dedicated, 60 * MiB)
+
def test_device_used_sums_each_client_once_and_is_unknown_when_incomplete(self):
with tempfile.TemporaryDirectory() as tmp:
card, proc, counters = self._setup(tmp)