A GPU task manager for Windows, Linux, and WSL: per-process VRAM, GPU activity, and LLM model/KV-cache insights (Windows AMD and NVIDIA).
Python
Add optional slim monitoring view
Scott Hanselman32be75e4aefb
diff --git a/README.md b/README.mdindex 3519ef9..844aee8 100644--- a/README.md+++ b/README.md@@ -406,12 +406,22 @@ uv run --locked gputaskman --help| `s` | Cycle sort: VRAM, GPU%, spill, paged, name || `c` | Cycle all charts or a single metric (also available in the **Charts** dropdown) || `a` | Switch between top and all collected processes |+| `m` | Toggle full diagnostics / slim monitoring || `p` | Pause / resume collection || `?` | Explain the counters and their limits || Escape | Close help || `q` | Quit |A wide terminal (roughly 140 columns or more) makes the full table easier to read.+The TUI starts in **Full diagnostics** mode. Press `m` (or click **Slim/Full** in the+footer) for **Slim monitoring**: each GPU header hides driver/CUDA versions, PCI bus+address, driver model, PCIe generation/width, compute capability, power/clock limits,+performance state, and secondary memory/video clocks. GPU identity, memory bars and+provenance/availability warnings, utilization, PCIe throughput, temperature, current power+and SM/GFX clock, throttling alerts, charts, processes, and process inspection remain.+The header border names the active mode; press `m` again to restore the full details.+This is a session-only display toggle, applies to every GPU, and does not change collection+or one-shot text/JSON output.All eight history charts are shown by default: GPU, SM clock (GFX clock on AMD), VRAM, memory activity,temperature, power, encoder, and decoder. They stack in narrow terminals and share rows asthe window widens. Click the **Charts** dropdown to focus on one metric or return to **All charts**.diff --git a/src/gputaskman/app.py b/src/gputaskman/app.pyindex 7d613ee..c8e68e3 100644--- a/src/gputaskman/app.py+++ b/src/gputaskman/app.py@@ -58,6 +58,11 @@ EXPLAIN = {HELP_MD = """\# What can be known about GPU memory?+Press **m** (or click **Slim/Full** in the footer) to switch between full diagnostics+and slim monitoring. Slim hides hardware identifiers and secondary telemetry details;+live readings, charts, processes, inspection, and provenance/availability warnings stay visible.+This display-only choice applies to all GPUs and resets to full when the app restarts.+| Layer | Source | Works for ||---|---|---|| **Device** telemetry | NVML, AMD ADL, or optional AMD SMI; amdgpu sysfs on Linux | availability varies by GPU and driver |@@ -186,6 +191,7 @@ class GpuTaskManager(App):Binding("s", "cycle_sort", "Sort"),Binding("c", "cycle_chart", "Chart"),Binding("a", "toggle_all", "All/Top"),+ Binding("m", "toggle_monitoring", "Slim/Full"),Binding("p", "pause", "Pause"),Binding("question_mark", "help", "Explain"),]@@ -217,6 +223,7 @@ class GpuTaskManager(App):self.sort_idx = 0self.show_all = Falseself.paused = False+ self.monitoring = Falseself.hist_util: deque = deque(maxlen=60)self.hist_clock: deque = deque(maxlen=60)self.hist_bw: deque = deque(maxlen=60)@@ -363,14 +370,18 @@ class GpuTaskManager(App):if len(self.snapshots) > 1:devices.append(Text(f"GPU {index}", style="bold"))devices.append(self.render_device(snap))- self.query_one("#device", Static).update(+ device = self.query_one("#device", Static)+ device.border_title = "Slim monitoring" if self.monitoring else "Full diagnostics"+ device.update(Group(*devices) if len(self.snapshots) > 1 else devices[0])self.render_charts()def render_device(self, s: Snapshot) -> Group:nv = s.nvname = s.adapter.name if s.adapter else (nv.name if nv else "GPU")- if s.backend == "metal":+ if self.monitoring:+ head = Text(name, style="bold")+ elif s.backend == "metal":head = Text.assemble((name, "bold"), f" {s.driver} ", ("Metal", "bold yellow"), " unified memory")else:info = (f" driver {s.driver}" if s.driver else "") + (f" CUDA {s.cuda}" if s.cuda else "")@@ -436,12 +447,19 @@ class GpuTaskManager(App):clock_label = (f"gfx {format_metric(nv.gfx_clock)} MHz" if nv.source.startswith("AMD ") elsef"SM {format_metric(nv.sm_clock)}/{format_metric(nv.sm_clock_max)} MHz")power_kind = f" {nv.power_kind}" if nv.power_kind else ""- st = Text(f"{format_metric(nv.temp, unit='°C')} {format_metric(nv.power_w, '.0f')}/"- f"{format_metric(nv.power_limit_w, '.0f')} W{power_kind} {pstate} "- f"{clock_label} "- f"mem {format_metric(nv.mem_clock)} MHz video {format_metric(nv.video_clock)} MHz "- f"PCIe gen{format_metric(nv.pcie_gen)} x{format_metric(nv.pcie_width)} "- + (f"CC {nv.compute_cap or 'N/A'}" if nv.source == "NVML" else nv.source), style="grey70")+ if self.monitoring:+ clock = nv.gfx_clock if nv.source.startswith("AMD ") else nv.sm_clock+ label = "gfx" if nv.source.startswith("AMD ") else "SM"+ st = Text(f"{format_metric(nv.temp, unit='°C')} "+ f"{format_metric(nv.power_w, '.0f')} W{power_kind} "+ f"{label} {format_metric(clock)} MHz", style="grey70")+ else:+ st = Text(f"{format_metric(nv.temp, unit='°C')} {format_metric(nv.power_w, '.0f')}/"+ f"{format_metric(nv.power_limit_w, '.0f')} W{power_kind} {pstate} "+ f"{clock_label} "+ f"mem {format_metric(nv.mem_clock)} MHz video {format_metric(nv.video_clock)} MHz "+ f"PCIe gen{format_metric(nv.pcie_gen)} x{format_metric(nv.pcie_width)} "+ + (f"CC {nv.compute_cap or 'N/A'}" if nv.source == "NVML" else nv.source), style="grey70")if nv.throttle:st.append(" throttle: " + ",".join(nv.throttle), style="yellow")lines.append(st)@@ -758,5 +776,10 @@ class GpuTaskManager(App):if self.snap:self.render_dashboard()+ def action_toggle_monitoring(self) -> None:+ self.monitoring = not self.monitoring+ if self.snap:+ self.render_dashboard()+def action_help(self) -> None:self.push_screen(HelpScreen())diff --git a/tests/test_core.py b/tests/test_core.pyindex bae2de6..4652fcd 100644--- a/tests/test_core.py+++ b/tests/test_core.py@@ -887,6 +887,132 @@ class UiTests(unittest.IsolatedAsyncioTestCase):Console(file=output, width=widget.content_size.width, color_system=None).print(widget.content)return output.getvalue()+ async def test_monitoring_toggle_hides_diagnostics_without_changing_live_state(self):+ s = snapshot()+ s.driver, s.cuda = "621.01", "13.5"+ s.nv = NvDevice(index=0, name="Test GPU", driver_model="WDDM", pci_bus="00000000:01:00.0",+ util_gpu=47, util_mem=44, temp=59, power_w=126, power_limit_w=300,+ sm_clock=2595, sm_clock_max=3090, mem_clock=15801, video_clock=2242,+ pstate=1, pcie_gen=5, pcie_width=8, compute_cap="12.0",+ pcie_rx_kbs=35 * 1024, pcie_tx_kbs=31 * 1024, throttle=["thermal"])+ s.counter_status = "Counters unavailable for one protected process"+ with patch.object(GpuTaskManager, "poll"):+ app = GpuTaskManager()+ async with app.run_test(size=(180, 50)) as pilot:+ app.apply(s)+ await pilot.pause()+ device = app.query_one("#device", Static)+ full = "\n".join(line.plain for line in device.content.renderables)+ self.assertFalse(app.monitoring)+ self.assertEqual(str(device.border_title), "Full diagnostics")+ for diagnostic in ("driver 621.01", "CUDA 13.5", "WDDM", "00000000:01:00.0",+ "PCIe gen5 x8", "CC 12.0", "126/300 W", "P1",+ "2595/3090 MHz", "mem 15801 MHz", "video 2242 MHz"):+ self.assertIn(diagnostic, full)+ await pilot.press("down", "p")+ selected = app.selected_key+ histories = list(app.hist_util)+ table = app.query_one("#procs")+ row_count = table.row_count+ charts = self.chart_text(app)+ inspector = app.query_one("#inspector", Static).content+ await pilot.press("m")+ await pilot.pause()+ slim = "\n".join(line.plain for line in device.content.renderables)+ self.assertTrue(app.monitoring)+ self.assertEqual(str(device.border_title), "Slim monitoring")+ for hidden in ("621.01", "13.5", "WDDM", "00000000:01:00.0", "PCIe gen", "CC ",+ "/300 W", "P1", "/3090", "mem 15801", "video 2242"):+ self.assertNotIn(hidden, slim)+ for live in ("Test GPU", "VRAM", "10.00 GB / 16.00 GB", "shared(RAM) 1.00 GB",+ "GPU 47%", "Mem activity 44%", "PCIe↓", "59°C", "126 W", "SM 2595 MHz",+ "throttle: thermal", s.counter_status, "[PAUSED]"):+ self.assertIn(live, slim)+ self.assertEqual(app.selected_key, selected)+ self.assertEqual(list(app.hist_util), histories)+ self.assertEqual(table.row_count, row_count)+ self.assertEqual(self.chart_text(app), charts)+ self.assertIs(app.query_one("#inspector", Static).content, inspector)+ await pilot.press("m")+ await pilot.pause()+ restored = "\n".join(line.plain for line in device.content.renderables)+ self.assertEqual(restored.replace(" [PAUSED]", ""), full)++ async def test_monitoring_applies_to_every_gpu_and_survives_new_samples(self):+ with patch.object(GpuTaskManager, "poll"):+ app = GpuTaskManager()+ async with app.run_test(size=(140, 42)) as pilot:+ await pilot.press("m")+ first, second = snapshot(), snapshot()+ first.driver = second.driver = "test-driver"+ first.nv = NvDevice(index=0, name="First GPU", util_gpu=10)+ second.nv = NvDevice(index=1, name="Second GPU", util_gpu=80)+ app.apply_all([first, second])+ await pilot.pause()+ device = app.query_one("#device", Static)+ rendered = io.StringIO()+ Console(file=rendered, width=140).print(device.content)+ for name in ("GPU 0", "GPU 1", "First GPU", "Second GPU"):+ self.assertIn(name, rendered.getvalue())+ self.assertNotIn("test-driver", rendered.getvalue())+ self.assertNotIn("PCIe gen", rendered.getvalue())+ self.assertEqual(app.query_one("#procs").row_count, 4)+ self.assertEqual(list(app._histories[0]["util"]), [10])+ self.assertEqual(list(app._histories[1]["util"]), [80])+ first.nv.util_gpu = 20+ app.apply_all([first, second])+ await pilot.pause()+ self.assertTrue(app.monitoring)+ self.assertEqual(list(app._histories[0]["util"]), [10, 20])+ self.assertEqual(list(app._histories[1]["util"]), [80, 80])+ await pilot.resize_terminal(80, 24)+ await pilot.pause()+ self.assertGreaterEqual(app.query_one("#procs").region.height, 4)+ self.assertLessEqual(app.query_one("#procs").region.bottom, app.query_one(Footer).region.y)+ screenshot = "".join(ET.fromstring(app.export_screenshot()).itertext())+ self.assertIn("Slim/Full", screenshot)+ await pilot.press("m")+ await pilot.pause()+ rendered = io.StringIO()+ Console(file=rendered, width=140).print(device.content)+ self.assertIn("test-driver", rendered.getvalue())++ async def test_monitoring_keeps_missing_values_and_backend_provenance(self):+ with patch.object(GpuTaskManager, "poll"):+ app = GpuTaskManager()+ async with app.run_test(size=(140, 42)) as pilot:+ await pilot.press("m")+ for backend in ("wddm", "wsl", "nvml", "metal"):+ with self.subTest(backend=backend):+ s = snapshot()+ s.backend, s.driver = backend, "hidden-driver"+ s.vram_used = None+ s.nv = NvDevice(index=0, util_gpu=0)+ s.counter_status = "Counter pairing unavailable"+ app.apply(s)+ await pilot.pause()+ lines = "\n".join(line.plain for line in app.render_device(s).renderables)+ self.assertNotIn("hidden-driver", lines)+ self.assertIn("N/A / 16.00 GB", lines)+ self.assertIn("GPU 0%", lines)+ self.assertIn(s.counter_status, lines)+ self.assertNotIn("None", lines)+ if backend != "metal":+ self.assertIn("N/A N/A W SM N/A MHz", lines)+ s = snapshot()+ s.nv = NvDevice(index=0, source="AMD ADL", util_gpu=25, gfx_clock=2200,+ power_w=115, power_kind="ASIC")+ app.apply(s)+ await pilot.pause()+ lines = "\n".join(line.plain for line in app.render_device(s).renderables)+ self.assertIn("AMD gfx 25%", lines)+ self.assertIn("gfx 2200 MHz", lines)+ self.assertIn("115 W ASIC", lines)+ s.nv = None+ app.apply(s)+ await pilot.pause()+ self.assertIn("VRAM", "\n".join(line.plain for line in app.render_device(s).renderables))+async def test_wsl_sort_selection_and_bar_use_inferred_vram_without_double_counting(self):s = snapshot()s.backend = "wsl"