A GPU task manager for Windows, Linux, and WSL: per-process VRAM, GPU activity, and LLM model/KV-cache insights (Windows AMD and NVIDIA).
Python
Release v0.3.3
Scott Hanselman9bc64590d152
diff --git a/README.md b/README.mdindex 721fb25..3519ef9 100644--- a/README.md+++ b/README.md@@ -420,12 +420,24 @@ scale, temperature uses 0–120°C, and the power and clock scales are displayedThe dashboard and inspector scroll independently in short terminals so the process list stays usable.Model/runtime discovery is throttled, so newly loaded models may take about 15 seconds to appear.+On multi-GPU systems, the TUI stacks a header and trend charts for each GPU and adds a **GPU**+column to the combined process list. A process using two GPUs has a separate row for each;+selecting a row inspects its memory on that GPU. Single-GPU layouts are unchanged.+Memory totals and histories stay separate per device; model files and runtime buffer logs+can span devices and do not prove per-device model placement. Discovery's 150 ms scan budget+is divided across GPUs, though each GPU has its own counter and telemetry queries.+This layout is covered by synthetic tests, **not yet validated on multi-GPU hardware**.+Under multi-GPU WSL, host attribution is disabled with a reason rather than risking matching+two devices by memory usage; Linux processes are listed once with GPU `?`, because `/dev/dxg`+does not identify their GPU. `--gpu N` retains the existing single-device host matching.+`--list-gpus` lists selectable hardware adapters with their vendor and LUID. `--gpu N` selectsone of these adapters: NVIDIA adapters are listed first, followed by other vendors, with eachgroup ordered by dedicated memory. An out-of-rangeindex is rejected rather than silently selecting another device. Compute-only devices such as anNPU are not graphics adapters. AMD and NVIDIA hardware coverage is described below; configurations-with multiple GPUs and mixed vendors need further validation.+with multiple GPUs and mixed vendors need further hardware validation. `--once` and `--json`+still default to GPU 0; use `--gpu N` to collect another device without changing the snapshot format.Device telemetry is matched to WDDM adapters by PCI location, not display name or enumeration order.If a unique match cannot be established, telemetry is unavailable; buffer logs can still span devices,so model-buffer attribution across GPUs is not yet reliable. This conservative matching also appliesdiff --git a/pyproject.toml b/pyproject.tomlindex b03116d..44fa8f8 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -1,6 +1,6 @@[project]name = "gputaskman"-version = "0.3.2"+version = "0.3.3"description = "An htop for what's inside your GPU: per-process VRAM, engine load, and LLM weights / KV cache breakdown"readme = "README.md"license = "MIT"diff --git a/src/gputaskman/app.py b/src/gputaskman/app.pyindex e34835b..7d613ee 100644--- a/src/gputaskman/app.py+++ b/src/gputaskman/app.py@@ -3,8 +3,8 @@from __future__ import annotationsimport os-import timefrom collections import deque+from contextlib import ExitStackfrom rich.console import Groupfrom rich.table import Table@@ -17,7 +17,7 @@ from textual.screen import ModalScreenfrom textual.widgets import DataTable, Footer, Markdown, Select, Staticfrom textual.worker import get_current_worker-from .collector import BACKEND, Collector, ProcRow, Snapshot+from .collector import BACKEND, Collector, ProcRow, Snapshot, enumerate_adapters, ordered_adaptersfrom .llm import gbfrom .nvml import NvDevice, format_metricfrom .report import (DRM_RESIDENT_NOTE, METAL_NOTE, WSL_EMPTY, drm_note, has_engine_counters, has_spill_counters, has_wddm_counters,@@ -174,6 +174,7 @@ class GpuTaskManager(App):#chart-controls { height: 1; }#chart-label { width: 8; text-style: bold; }#chart-mode { width: 24; }+ #chart-mode SelectOverlay { border: solid $accent; padding: 0 1; }#chart-hint { width: 1fr; color: $text-muted; }#charts { height: auto; }#chart-note { height: auto; color: $text-muted; }@@ -203,12 +204,15 @@ class GpuTaskManager(App):}CHARTS = tuple(CHART_COLORS)- def __init__(self, gpu_index: int = 0, interval: float = 1.0, host: bool = True) -> None:+ def __init__(self, gpu_index: int | None = None, interval: float = 1.0, host: bool = True) -> None:super().__init__()self.gpu_index = gpu_indexself.interval = intervalself.host = hostself.snap: Snapshot | None = None+ self.snapshots: list[Snapshot] = []+ self._histories: dict[int, dict[str, deque]] = {}+ self._row_snapshots: dict[str, Snapshot] = {}self.selected_key: str | None = None # ProcRow.key: Linux and Windows host PIDs can collideself.sort_idx = 0self.show_all = False@@ -228,7 +232,8 @@ class GpuTaskManager(App):def selected_pid(self) -> int | None:"""PID of the selected local process; None for Windows (W:) and outside (H:) rows."""key = self.selected_key- return int(key) if key and key.isdigit() else None+ pid = key.rsplit(":", 1)[-1] if key else ""+ return int(pid) if pid.isdigit() else None@selected_pid.setterdef selected_pid(self, pid: int | None) -> None:@@ -262,31 +267,70 @@ class GpuTaskManager(App):@work(thread=True, exclusive=True)def poll(self) -> None:worker = get_current_worker()- collector = Collector(self.gpu_index, host=self.host)- time.sleep(0.5)- try:+ indices = ([self.gpu_index] if self.gpu_index is not None else+ list(range(len(ordered_adapters(enumerate_adapters())))) or [0])+ with ExitStack() as stack:+ collectors = []+ for index in indices:+ collector = Collector(index, scan_budget=Collector.SCAN_BUDGET / len(indices),+ host=self.host and not (BACKEND == "wsl" and len(indices) > 1))+ if BACKEND == "wsl" and len(indices) > 1 and self.host:+ collector.host_status = "off (multi-GPU WSL host matching is not validated; use --gpu N)"+ stack.callback(collector.close)+ collectors.append(collector)+ worker.cancelled_event.wait(0.5)while not worker.is_cancelled:if not self.paused:- snap = collector.sample()- self.call_from_thread(self.apply, snap)- time.sleep(self.interval)- finally:- collector.close()+ snapshots = [collector.sample() for collector in collectors]+ if not worker.is_cancelled:+ self.call_from_thread(self.apply_all, snapshots)+ worker.cancelled_event.wait(self.interval)# ------------------------------------------------------------------ renderingdef apply(self, snap: Snapshot) -> None:+ self.apply_all([snap])++ def apply_all(self, snapshots: list[Snapshot]) -> None:+ if not self.is_running:+ return # A worker may finish sampling after shutdown has removed the widgets.+ self.snapshots = snapshots+ multi = len(snapshots) > 1+ table = self.query_one("#procs", DataTable)+ if multi and "GPU" not in table.columns:+ table.add_column("GPU", key="GPU")+ elif not multi and "GPU" in table.columns:+ table.remove_column("GPU")+ self._row_snapshots = {}+ for index, snap in enumerate(snapshots):+ for row in snap.procs:+ row.gpu_index = index if multi else None+ self._row_snapshots[row.key] = snap+ self._use_history(index)+ self._record_sample(snap)+ self._use_history(0)+ self.snap = snapshots[0]+ if not self._auto_selected:+ llm = next((r for r in sorted(+ (r for s in snapshots for r in s.procs), key=self.SORTS[0][1]) if r.llm), None)+ if llm:+ self.selected_key = llm.key+ self._auto_selected = True+ self.render_dashboard()+ self.render_table(self.snap)+ self.render_inspector()++ def _use_history(self, index: int) -> None:+ names = ("util", "clock", "bw", "vram", "temp", "power", "enc", "dec")+ if index not in self._histories:+ self._histories[index] = {name: deque(maxlen=60) for name in names}+ for name, history in self._histories[index].items():+ setattr(self, f"hist_{name}", history)++ def _record_sample(self, snap: Snapshot) -> None:self.snap = snapnv = snap.nvamd = bool(snap.adapter and snap.adapter.vendor_id == 0x1002)- if getattr(self, "_amd_chart_labels", None) != amd:- self._amd_chart_labels = amd- selector = self.query_one("#chart-mode", Select)- selected = selector.value- selector.set_options([("All charts", -1), *(- ("GFX clock" if amd and self.CHARTS[index] == "SM clock" else self.CHARTS[index], index)- for index in self.available_charts)]) # macOS offers only the charts IOKit has data for- selector.value = selectedif nv:self.hist_util.append(nv.util_gpu)self.hist_clock.append(nv.gfx_clock if amd else nv.sm_clock)@@ -301,15 +345,27 @@ class GpuTaskManager(App):history.append(None)self.hist_vram.append(100 * snap.vram_used / snap.vram_totalif snap.vram_total and snap.vram_used is not None else None)- if not self._auto_selected:- llm = next((r for r in sorted(snap.procs, key=self.SORTS[0][1]) if r.llm), None)- if llm:- self.selected_key = llm.key- self._auto_selected = True- self.query_one("#device", Static).update(self.render_device(snap))++ def render_dashboard(self) -> None:+ amd = bool(self.snapshots[0].adapter and self.snapshots[0].adapter.vendor_id == 0x1002)+ clock_label = "SM/GFX clock" if len(self.snapshots) > 1 else "GFX clock" if amd else "SM clock"+ if getattr(self, "_clock_label", None) != clock_label:+ self._clock_label = clock_label+ selector = self.query_one("#chart-mode", Select)+ selected = selector.value+ selector.set_options([("All charts", -1), *(+ (clock_label if self.CHARTS[index] == "SM clock" else self.CHARTS[index], index)+ for index in self.available_charts)])+ selector.value = selected+ self._amd_chart_labels = amd+ devices = []+ for index, snap in enumerate(self.snapshots):+ if len(self.snapshots) > 1:+ devices.append(Text(f"GPU {index}", style="bold"))+ devices.append(self.render_device(snap))+ self.query_one("#device", Static).update(+ Group(*devices) if len(self.snapshots) > 1 else devices[0])self.render_charts()- self.render_table(snap)- self.render_inspector()def render_device(self, s: Snapshot) -> Group:nv = s.nv@@ -415,9 +471,28 @@ class GpuTaskManager(App):return self.hist_dec, 100, "%", colordef render_charts(self) -> None:- if self.snap is None:+ if len(self.snapshots) > 1:+ charts = []+ notes = []+ for index, snap in enumerate(self.snapshots):+ self.snap = snap+ self._use_history(index)+ self._amd_chart_labels = bool(snap.adapter and snap.adapter.vendor_id == 0x1002)+ grid, note = self._chart_content()+ charts.extend([Text(f"GPU {index}", style="bold"), grid])+ notes.append(f"GPU {index}: {note}")+ self.query_one("#charts", Static).update(Group(*charts))+ self.query_one("#chart-note", Static).update("\n".join(notes))+ self.snap = self.snapshots[0]+ self._use_history(0)+ self._amd_chart_labels = bool(self.snap.adapter and self.snap.adapter.vendor_id == 0x1002)return- widget = self.query_one("#charts", Static)+ if self.snap is not None:+ grid, note = self._chart_content()+ self.query_one("#charts", Static).update(grid)+ self.query_one("#chart-note", Static).update(note)++ def _chart_content(self) -> tuple[Table, str]:# Reserve the border, padding, and scrollbar even before child resize events arrive.width = max(1, self.size.width - 6)indices = list(self.available_charts) if self.chart_idx == -1 else [self.chart_idx]@@ -439,19 +514,18 @@ class GpuTaskManager(App):cells.append(cell)for start in range(0, len(cells), columns):grid.add_row(*cells[start:start + columns])- widget.update(grid)_, power_scale, _, _ = self.chart_series(self.CHARTS.index("Power"), self.snap.nv)_, clock_scale, _, _ = self.chart_series(self.CHARTS.index("SM clock"), self.snap.nv)- self.query_one("#chart-note", Static).update(+ note = ("Recent samples · blank = no reading" if self.snap.backend == "metal" elsef"Recent samples · blank = no reading · power 0–{power_scale:.0f} W · "f"{'GFX' if getattr(self, '_amd_chart_labels', False) else 'SM'} clock 0–{clock_scale:.0f} MHz")+ return grid, notedef on_resize(self) -> None:if self.is_mounted and self.snap is not None:- self.query_one("#device", Static).update(self.render_device(self.snap))- self.call_after_refresh(self.render_charts)+ self.call_after_refresh(self.render_dashboard)def visible_rows(self, s: Snapshot) -> list[ProcRow]:rows = s.procs@@ -470,7 +544,12 @@ class GpuTaskManager(App):def render_table(self, s: Snapshot) -> None:t = self.query_one("#procs", DataTable)- rows = self.visible_rows(s)+ rows = [r for snap in self.snapshots for r in self.visible_rows(snap)] if len(self.snapshots) > 1 else self.visible_rows(s)+ if len(self.snapshots) > 1:+ rows = [r for r in rows if not (+ self._row_snapshots[r.key].backend == "wsl" and not r.host and r.gpu_index != 0)]+ if len(self.snapshots) > 1 and not any(snap.backend == "wsl" for snap in self.snapshots):+ rows.sort(key=self.SORTS[self.sort_idx][1])keep = self.selected_keyt.clear()@@ -480,6 +559,7 @@ class GpuTaskManager(App):return Text(f"{v:.0f}" if v >= 0.5 else "", justify="right", style="bold" if v >= 20 else "")for r in rows:+ s = self._row_snapshots.get(r.key, s)e = r.gpu.engineswddm = has_wddm_counters(s, r)engines = has_engine_counters(s, r)@@ -503,6 +583,8 @@ class GpuTaskManager(App):pct(e.get("copy", 0) if split else None),pct(e.get("decode", 0) + e.get("encode", 0) + e.get("video", 0) if split else None),Text(r.inside, style="magenta" if r.llm else "grey62", overflow="ellipsis", no_wrap=True),+ *([Text("?" if s.backend == "wsl" and not r.host else str(r.gpu_index))]+ if len(self.snapshots) > 1 else []),key=r.key,)if keep is not None:@@ -516,13 +598,15 @@ class GpuTaskManager(App):def render_inspector(self) -> None:panel = self.query_one("#inspector", Static)- s = self.snap+ s = self._row_snapshots.get(self.selected_key, self.snap)r = next((x for x in s.procs if x.key == self.selected_key), None) if s else Noneif r is None:panel.update(Text("Select a process (↑/↓) to see what's inside its GPU memory. Press ? to learn what's knowable.", style="grey62"))returnwhere = {"windows": "Windows PID ", "outside": "host PID "}.get(r.origin, "") if r.host else ""panel.border_title = f"{r.name} ({where}{r.pid}) - {r.kind}"+ if r.gpu_index is not None:+ panel.border_title += " - GPU unknown" if s.backend == "wsl" and not r.host else f" - GPU {r.gpu_index}"out: list = []e = r.gpu.engineswddm = has_wddm_counters(s, r)@@ -571,6 +655,9 @@ class GpuTaskManager(App):v = r.llmif v:+ if r.gpu_index is not None:+ out.append(Text("Model files, launch flags and runtime buffer logs can span GPUs. "+ "The breakdown below is not a measured per-device model placement.", style="grey70"))width = max(min(self.size.width - 8, 100), 20)total = max(r.gpu.shown_vram or 0, sum(x.bytes for x in v.segments), 1)out.append(stacked_bar([(x.bytes, SEG_STYLE[x.kind]) for x in v.segments], total, width))@@ -669,7 +756,7 @@ class GpuTaskManager(App):def action_pause(self) -> None:self.paused = not self.pausedif self.snap:- self.query_one("#device", Static).update(self.render_device(self.snap))+ self.render_dashboard()def action_help(self) -> None:self.push_screen(HelpScreen())diff --git a/src/gputaskman/cli.py b/src/gputaskman/cli.pyindex 00bf573..8ef2cea 100644--- a/src/gputaskman/cli.py+++ b/src/gputaskman/cli.py@@ -34,13 +34,13 @@ def main() -> None:epilog="gputaskman diff BEFORE.json AFTER.json compares two --json snapshots.")ap.add_argument("--once", action="store_true", help="print one snapshot and exit (no TUI)")ap.add_argument("--json", action="store_true", help="print one JSON snapshot and exit (implies --once)")- ap.add_argument("--gpu", type=int, default=0, help="GPU index (default 0)")+ ap.add_argument("--gpu", type=int, help="show only this GPU (TUI default: all; one-shot default: 0)")ap.add_argument("--list-gpus", action="store_true", help="list selectable GPU adapters and exit")ap.add_argument("--no-host", action="store_true",help="WSL: don't read Windows host GPU counters through interop (WSL VM total, Windows apps)")ap.add_argument("-i", "--interval", type=float, default=1.0, help="refresh seconds (default 1)")args = ap.parse_args()- if args.gpu < 0:+ if args.gpu is not None and args.gpu < 0:ap.error("--gpu must be a non-negative index")if not math.isfinite(args.interval) or args.interval <= 0:ap.error("--interval must be a finite number greater than zero")@@ -68,7 +68,7 @@ def main() -> None:from .report import dumptry:- c = Collector(args.gpu, scan_budget=None, host=not args.no_host) # one-shot: identify everything now+ c = Collector(args.gpu or 0, scan_budget=None, host=not args.no_host) # one-shot: identify everything nowexcept ValueError as error:ap.error(str(error))try:diff --git a/src/gputaskman/collector.py b/src/gputaskman/collector.pyindex 7761842..9d3cd63 100644--- a/src/gputaskman/collector.py+++ b/src/gputaskman/collector.py@@ -51,11 +51,13 @@ class ProcRow:handy: HandyEvidence | None = Nonehost: bool = False # pid is from another PID namespace, never a local one (see origin)origin: str = "" # with host: "windows" (a Windows process seen from WSL) | "outside" (outside this container)+ gpu_index: int | None = None@propertydef key(self) -> str:"""Unique row id: PIDs from other namespaces and local PIDs can collide."""- return f"{self.origin[0]}{self.pid}" if self.host else str(self.pid)+ pid = f"{self.origin[0]}{self.pid}" if self.host else str(self.pid)+ return f"{self.gpu_index}:{pid}" if self.gpu_index is not None else pid@propertydef pid_label(self) -> str:diff --git a/tests/test_core.py b/tests/test_core.pyindex 807417b..bae2de6 100644--- a/tests/test_core.py+++ b/tests/test_core.py@@ -1,12 +1,14 @@from __future__ import annotationsimport contextlib+import asyncioimport ctypesimport ioimport jsonimport osfrom types import SimpleNamespaceimport tempfile+import threadingimport unittestimport xml.etree.ElementTree as ETfrom datetime import datetime, timezone@@ -1032,6 +1034,9 @@ class UiTests(unittest.IsolatedAsyncioTestCase):await pilot.click("#chart-mode", offset=(2, 0))selector = app.query_one("#chart-mode", Select)self.assertTrue(selector.expanded)+ overlay = selector.query_one("SelectOverlay")+ self.assertEqual(overlay.styles.border_top[0], "solid")+ self.assertEqual(overlay.styles.padding.left, 1)await pilot.press("down", "down", "enter")await pilot.pause()self.assertEqual(selector.value, 1)@@ -1043,6 +1048,103 @@ class UiTests(unittest.IsolatedAsyncioTestCase):for name in app.CHARTS:self.assertIn(name, self.chart_text(app))+ async def test_multi_gpu_rows_histories_and_selection_stay_separate(self):+ with patch.object(GpuTaskManager, "poll"):+ app = GpuTaskManager()+ async with app.run_test(size=(140, 42)) as pilot:+ first, second = snapshot(), snapshot()+ first.nv = NvDevice(index=0, util_gpu=10)+ second.nv = NvDevice(index=1, util_gpu=80)+ second.procs[0].gpu.dedicated = 3 * GiB+ second.procs[0].gpu.available = {"dedicated"}+ app.apply_all([first, second])+ await pilot.pause()+ table = app.query_one("#procs")+ self.assertEqual(table.row_count, 4)+ self.assertEqual(table.get_cell("0:10", "GPU").plain, "0")+ self.assertEqual(table.get_cell("1:10", "GPU").plain, "1")+ self.assertEqual(table.get_cell("1:10", "VRAM").plain, "3.00 GB")+ self.assertEqual(table.get_cell("1:10", "GPU%").plain, "n/a")+ self.assertEqual(list(app._histories[0]["util"]), [10])+ self.assertEqual(list(app._histories[1]["util"]), [80])+ self.assertIn("GPU 0", self.chart_text(app))+ self.assertIn("GPU 1", self.chart_text(app))+ app.selected_key = "1:10"+ app.render_inspector()+ self.assertEqual(app.selected_pid, 10)+ self.assertIn("GPU 1", str(app.query_one("#inspector").border_title))+ app.apply_all([first, second])+ await pilot.pause()+ self.assertEqual(app.selected_key, "1:10")+ await pilot.press("c", "p")+ await pilot.pause()+ await pilot.resize_terminal(80, 24)+ await pilot.pause()+ self.assertIn("GPU 1", self.chart_text(app))+ self.assertEqual(list(app._histories[1]["util"]), [80, 80])+ self.assertGreaterEqual(table.region.height, 4)+ app.apply(first)+ await pilot.pause()+ self.assertNotIn("GPU", table.columns)+ self.assertEqual(table.row_count, 2)++ async def test_poll_collects_all_adapters_and_closes_each_collector(self):+ adapters = [Adapter(1, "first", 0x10DE, 16 * GiB, 0, False),+ Adapter(2, "second", 0x10DE, 8 * GiB, 0, False)]+ with patch("gputaskman.app.enumerate_adapters", return_value=adapters), \+ patch("gputaskman.app.Collector") as factory:+ factory.SCAN_BUDGET = 0.15+ collectors = [Mock(), Mock()]+ closed = [threading.Event(), threading.Event()]+ for collector, event in zip(collectors, closed):+ collector.sample.side_effect = snapshot+ collector.close.side_effect = event.set+ factory.side_effect = collectors+ app = GpuTaskManager(interval=0.05)+ async with app.run_test(size=(140, 42)) as pilot:+ await pilot.pause(0.8)+ self.assertEqual(len(app.snapshots), 2)+ self.assertEqual(factory.call_args_list[0].args, (0,))+ self.assertEqual(factory.call_args_list[1].args, (1,))+ self.assertEqual(factory.call_args_list[0].kwargs["scan_budget"], 0.075)+ for collector, event in zip(collectors, closed):+ self.assertTrue(await asyncio.to_thread(event.wait, 2))+ collector.close.assert_called_once()++ async def test_explicit_gpu_keeps_single_device_layout(self):+ with patch("gputaskman.app.Collector") as factory:+ factory.SCAN_BUDGET = 0.15+ factory.return_value.sample.side_effect = snapshot+ closed = threading.Event()+ factory.return_value.close.side_effect = closed.set+ app = GpuTaskManager(gpu_index=1, interval=0.05)+ async with app.run_test(size=(140, 42)) as pilot:+ await pilot.pause(0.8)+ self.assertEqual(len(app.snapshots), 1)+ self.assertNotIn("GPU", app.query_one("#procs").columns)+ self.assertEqual(factory.call_args.args, (1,))+ self.assertTrue(await asyncio.to_thread(closed.wait, 2))+ factory.return_value.close.assert_called_once()++ async def test_multi_gpu_wsl_does_not_claim_process_device_ownership(self):+ with patch.object(GpuTaskManager, "poll"):+ app = GpuTaskManager()+ async with app.run_test(size=(140, 42)) as pilot:+ first, second = snapshot(), snapshot()+ for s in (first, second):+ s.backend = "wsl"+ for row in s.procs:+ row.gpu = ProcGpu(attributed=False)+ app.apply_all([first, second])+ await pilot.pause()+ table = app.query_one("#procs")+ self.assertEqual(table.row_count, 2)+ self.assertEqual(table.get_cell("0:10", "GPU").plain, "?")+ self.assertEqual(table.get_cell("0:10", "VRAM").plain, "n/a")+ app.selected_key = "0:10"+ app.render_inspector()+ self.assertIn("GPU unknown", str(app.query_one("#inspector").border_title))+async def test_chart_layout_reflows_while_paused_and_keeps_processes_visible(self):with patch.object(GpuTaskManager, "poll"):app = GpuTaskManager()diff --git a/uv.lock b/uv.lockindex bfacf43..b35cb81 100644--- a/uv.lock+++ b/uv.lock@@ -13,7 +13,7 @@ wheels = [[[package]]name = "gputaskman"-version = "0.3.2"+version = "0.3.3"source = { editable = "." }dependencies = [{ name = "nvidia-ml-py" },