Skip to content

A GPU task manager for Windows, Linux, and WSL: per-process VRAM, GPU activity, and LLM model/KV-cache insights (Windows AMD and NVIDIA).

Python

Add optional slim monitoring view

Scott Hanselman32be75e4aefb
diff --git a/README.md b/README.md
index 3519ef9..844aee8 100644
--- a/README.md
+++ b/README.md
@@ -406,12 +406,22 @@ uv run --locked gputaskman --help
| `s` | Cycle sort: VRAM, GPU%, spill, paged, name |
| `c` | Cycle all charts or a single metric (also available in the **Charts** dropdown) |
| `a` | Switch between top and all collected processes |
+| `m` | Toggle full diagnostics / slim monitoring |
| `p` | Pause / resume collection |
| `?` | Explain the counters and their limits |
| Escape | Close help |
| `q` | Quit |
A wide terminal (roughly 140 columns or more) makes the full table easier to read.
+The TUI starts in **Full diagnostics** mode. Press `m` (or click **Slim/Full** in the
+footer) for **Slim monitoring**: each GPU header hides driver/CUDA versions, PCI bus
+address, driver model, PCIe generation/width, compute capability, power/clock limits,
+performance state, and secondary memory/video clocks. GPU identity, memory bars and
+provenance/availability warnings, utilization, PCIe throughput, temperature, current power
+and SM/GFX clock, throttling alerts, charts, processes, and process inspection remain.
+The header border names the active mode; press `m` again to restore the full details.
+This is a session-only display toggle, applies to every GPU, and does not change collection
+or one-shot text/JSON output.
All eight history charts are shown by default: GPU, SM clock (GFX clock on AMD), VRAM, memory activity,
temperature, power, encoder, and decoder. They stack in narrow terminals and share rows as
the window widens. Click the **Charts** dropdown to focus on one metric or return to **All charts**.
diff --git a/src/gputaskman/app.py b/src/gputaskman/app.py
index 7d613ee..c8e68e3 100644
--- a/src/gputaskman/app.py
+++ b/src/gputaskman/app.py
@@ -58,6 +58,11 @@ EXPLAIN = {
HELP_MD = """\
# What can be known about GPU memory?
+Press **m** (or click **Slim/Full** in the footer) to switch between full diagnostics
+and slim monitoring. Slim hides hardware identifiers and secondary telemetry details;
+live readings, charts, processes, inspection, and provenance/availability warnings stay visible.
+This display-only choice applies to all GPUs and resets to full when the app restarts.
+
| Layer | Source | Works for |
|---|---|---|
| **Device** telemetry | NVML, AMD ADL, or optional AMD SMI; amdgpu sysfs on Linux | availability varies by GPU and driver |
@@ -186,6 +191,7 @@ class GpuTaskManager(App):
Binding("s", "cycle_sort", "Sort"),
Binding("c", "cycle_chart", "Chart"),
Binding("a", "toggle_all", "All/Top"),
+ Binding("m", "toggle_monitoring", "Slim/Full"),
Binding("p", "pause", "Pause"),
Binding("question_mark", "help", "Explain"),
]
@@ -217,6 +223,7 @@ class GpuTaskManager(App):
self.sort_idx = 0
self.show_all = False
self.paused = False
+ self.monitoring = False
self.hist_util: deque = deque(maxlen=60)
self.hist_clock: deque = deque(maxlen=60)
self.hist_bw: deque = deque(maxlen=60)
@@ -363,14 +370,18 @@ class GpuTaskManager(App):
if len(self.snapshots) > 1:
devices.append(Text(f"GPU {index}", style="bold"))
devices.append(self.render_device(snap))
- self.query_one("#device", Static).update(
+ device = self.query_one("#device", Static)
+ device.border_title = "Slim monitoring" if self.monitoring else "Full diagnostics"
+ device.update(
Group(*devices) if len(self.snapshots) > 1 else devices[0])
self.render_charts()
def render_device(self, s: Snapshot) -> Group:
nv = s.nv
name = s.adapter.name if s.adapter else (nv.name if nv else "GPU")
- if s.backend == "metal":
+ if self.monitoring:
+ head = Text(name, style="bold")
+ elif s.backend == "metal":
head = Text.assemble((name, "bold"), f" {s.driver} ", ("Metal", "bold yellow"), " unified memory")
else:
info = (f" driver {s.driver}" if s.driver else "") + (f" CUDA {s.cuda}" if s.cuda else "")
@@ -436,12 +447,19 @@ class GpuTaskManager(App):
clock_label = (f"gfx {format_metric(nv.gfx_clock)} MHz" if nv.source.startswith("AMD ") else
f"SM {format_metric(nv.sm_clock)}/{format_metric(nv.sm_clock_max)} MHz")
power_kind = f" {nv.power_kind}" if nv.power_kind else ""
- st = Text(f"{format_metric(nv.temp, unit='°C')} {format_metric(nv.power_w, '.0f')}/"
- f"{format_metric(nv.power_limit_w, '.0f')} W{power_kind} {pstate} "
- f"{clock_label} "
- f"mem {format_metric(nv.mem_clock)} MHz video {format_metric(nv.video_clock)} MHz "
- f"PCIe gen{format_metric(nv.pcie_gen)} x{format_metric(nv.pcie_width)} "
- + (f"CC {nv.compute_cap or 'N/A'}" if nv.source == "NVML" else nv.source), style="grey70")
+ if self.monitoring:
+ clock = nv.gfx_clock if nv.source.startswith("AMD ") else nv.sm_clock
+ label = "gfx" if nv.source.startswith("AMD ") else "SM"
+ st = Text(f"{format_metric(nv.temp, unit='°C')} "
+ f"{format_metric(nv.power_w, '.0f')} W{power_kind} "
+ f"{label} {format_metric(clock)} MHz", style="grey70")
+ else:
+ st = Text(f"{format_metric(nv.temp, unit='°C')} {format_metric(nv.power_w, '.0f')}/"
+ f"{format_metric(nv.power_limit_w, '.0f')} W{power_kind} {pstate} "
+ f"{clock_label} "
+ f"mem {format_metric(nv.mem_clock)} MHz video {format_metric(nv.video_clock)} MHz "
+ f"PCIe gen{format_metric(nv.pcie_gen)} x{format_metric(nv.pcie_width)} "
+ + (f"CC {nv.compute_cap or 'N/A'}" if nv.source == "NVML" else nv.source), style="grey70")
if nv.throttle:
st.append(" throttle: " + ",".join(nv.throttle), style="yellow")
lines.append(st)
@@ -758,5 +776,10 @@ class GpuTaskManager(App):
if self.snap:
self.render_dashboard()
+ def action_toggle_monitoring(self) -> None:
+ self.monitoring = not self.monitoring
+ if self.snap:
+ self.render_dashboard()
+
def action_help(self) -> None:
self.push_screen(HelpScreen())
diff --git a/tests/test_core.py b/tests/test_core.py
index bae2de6..4652fcd 100644
--- a/tests/test_core.py
+++ b/tests/test_core.py
@@ -887,6 +887,132 @@ class UiTests(unittest.IsolatedAsyncioTestCase):
Console(file=output, width=widget.content_size.width, color_system=None).print(widget.content)
return output.getvalue()
+ async def test_monitoring_toggle_hides_diagnostics_without_changing_live_state(self):
+ s = snapshot()
+ s.driver, s.cuda = "621.01", "13.5"
+ s.nv = NvDevice(index=0, name="Test GPU", driver_model="WDDM", pci_bus="00000000:01:00.0",
+ util_gpu=47, util_mem=44, temp=59, power_w=126, power_limit_w=300,
+ sm_clock=2595, sm_clock_max=3090, mem_clock=15801, video_clock=2242,
+ pstate=1, pcie_gen=5, pcie_width=8, compute_cap="12.0",
+ pcie_rx_kbs=35 * 1024, pcie_tx_kbs=31 * 1024, throttle=["thermal"])
+ s.counter_status = "Counters unavailable for one protected process"
+ with patch.object(GpuTaskManager, "poll"):
+ app = GpuTaskManager()
+ async with app.run_test(size=(180, 50)) as pilot:
+ app.apply(s)
+ await pilot.pause()
+ device = app.query_one("#device", Static)
+ full = "\n".join(line.plain for line in device.content.renderables)
+ self.assertFalse(app.monitoring)
+ self.assertEqual(str(device.border_title), "Full diagnostics")
+ for diagnostic in ("driver 621.01", "CUDA 13.5", "WDDM", "00000000:01:00.0",
+ "PCIe gen5 x8", "CC 12.0", "126/300 W", "P1",
+ "2595/3090 MHz", "mem 15801 MHz", "video 2242 MHz"):
+ self.assertIn(diagnostic, full)
+ await pilot.press("down", "p")
+ selected = app.selected_key
+ histories = list(app.hist_util)
+ table = app.query_one("#procs")
+ row_count = table.row_count
+ charts = self.chart_text(app)
+ inspector = app.query_one("#inspector", Static).content
+ await pilot.press("m")
+ await pilot.pause()
+ slim = "\n".join(line.plain for line in device.content.renderables)
+ self.assertTrue(app.monitoring)
+ self.assertEqual(str(device.border_title), "Slim monitoring")
+ for hidden in ("621.01", "13.5", "WDDM", "00000000:01:00.0", "PCIe gen", "CC ",
+ "/300 W", "P1", "/3090", "mem 15801", "video 2242"):
+ self.assertNotIn(hidden, slim)
+ for live in ("Test GPU", "VRAM", "10.00 GB / 16.00 GB", "shared(RAM) 1.00 GB",
+ "GPU 47%", "Mem activity 44%", "PCIe↓", "59°C", "126 W", "SM 2595 MHz",
+ "throttle: thermal", s.counter_status, "[PAUSED]"):
+ self.assertIn(live, slim)
+ self.assertEqual(app.selected_key, selected)
+ self.assertEqual(list(app.hist_util), histories)
+ self.assertEqual(table.row_count, row_count)
+ self.assertEqual(self.chart_text(app), charts)
+ self.assertIs(app.query_one("#inspector", Static).content, inspector)
+ await pilot.press("m")
+ await pilot.pause()
+ restored = "\n".join(line.plain for line in device.content.renderables)
+ self.assertEqual(restored.replace(" [PAUSED]", ""), full)
+
+ async def test_monitoring_applies_to_every_gpu_and_survives_new_samples(self):
+ with patch.object(GpuTaskManager, "poll"):
+ app = GpuTaskManager()
+ async with app.run_test(size=(140, 42)) as pilot:
+ await pilot.press("m")
+ first, second = snapshot(), snapshot()
+ first.driver = second.driver = "test-driver"
+ first.nv = NvDevice(index=0, name="First GPU", util_gpu=10)
+ second.nv = NvDevice(index=1, name="Second GPU", util_gpu=80)
+ app.apply_all([first, second])
+ await pilot.pause()
+ device = app.query_one("#device", Static)
+ rendered = io.StringIO()
+ Console(file=rendered, width=140).print(device.content)
+ for name in ("GPU 0", "GPU 1", "First GPU", "Second GPU"):
+ self.assertIn(name, rendered.getvalue())
+ self.assertNotIn("test-driver", rendered.getvalue())
+ self.assertNotIn("PCIe gen", rendered.getvalue())
+ self.assertEqual(app.query_one("#procs").row_count, 4)
+ self.assertEqual(list(app._histories[0]["util"]), [10])
+ self.assertEqual(list(app._histories[1]["util"]), [80])
+ first.nv.util_gpu = 20
+ app.apply_all([first, second])
+ await pilot.pause()
+ self.assertTrue(app.monitoring)
+ self.assertEqual(list(app._histories[0]["util"]), [10, 20])
+ self.assertEqual(list(app._histories[1]["util"]), [80, 80])
+ await pilot.resize_terminal(80, 24)
+ await pilot.pause()
+ self.assertGreaterEqual(app.query_one("#procs").region.height, 4)
+ self.assertLessEqual(app.query_one("#procs").region.bottom, app.query_one(Footer).region.y)
+ screenshot = "".join(ET.fromstring(app.export_screenshot()).itertext())
+ self.assertIn("Slim/Full", screenshot)
+ await pilot.press("m")
+ await pilot.pause()
+ rendered = io.StringIO()
+ Console(file=rendered, width=140).print(device.content)
+ self.assertIn("test-driver", rendered.getvalue())
+
+ async def test_monitoring_keeps_missing_values_and_backend_provenance(self):
+ with patch.object(GpuTaskManager, "poll"):
+ app = GpuTaskManager()
+ async with app.run_test(size=(140, 42)) as pilot:
+ await pilot.press("m")
+ for backend in ("wddm", "wsl", "nvml", "metal"):
+ with self.subTest(backend=backend):
+ s = snapshot()
+ s.backend, s.driver = backend, "hidden-driver"
+ s.vram_used = None
+ s.nv = NvDevice(index=0, util_gpu=0)
+ s.counter_status = "Counter pairing unavailable"
+ app.apply(s)
+ await pilot.pause()
+ lines = "\n".join(line.plain for line in app.render_device(s).renderables)
+ self.assertNotIn("hidden-driver", lines)
+ self.assertIn("N/A / 16.00 GB", lines)
+ self.assertIn("GPU 0%", lines)
+ self.assertIn(s.counter_status, lines)
+ self.assertNotIn("None", lines)
+ if backend != "metal":
+ self.assertIn("N/A N/A W SM N/A MHz", lines)
+ s = snapshot()
+ s.nv = NvDevice(index=0, source="AMD ADL", util_gpu=25, gfx_clock=2200,
+ power_w=115, power_kind="ASIC")
+ app.apply(s)
+ await pilot.pause()
+ lines = "\n".join(line.plain for line in app.render_device(s).renderables)
+ self.assertIn("AMD gfx 25%", lines)
+ self.assertIn("gfx 2200 MHz", lines)
+ self.assertIn("115 W ASIC", lines)
+ s.nv = None
+ app.apply(s)
+ await pilot.pause()
+ self.assertIn("VRAM", "\n".join(line.plain for line in app.render_device(s).renderables))
+
async def test_wsl_sort_selection_and_bar_use_inferred_vram_without_double_counting(self):
s = snapshot()
s.backend = "wsl"