A GPU task manager for Windows, Linux, and WSL: per-process VRAM, GPU activity, and LLM model/KV-cache insights (Windows AMD and NVIDIA).
Python
Address PR review comments
Sameh Khamisd361d5fe785f
diff --git a/src/gputaskman/app.py b/src/gputaskman/app.pyindex ad0863c..320c170 100644--- a/src/gputaskman/app.py+++ b/src/gputaskman/app.py@@ -175,7 +175,7 @@ class GpuTaskManager(App):Binding("p", "pause", "Pause"),Binding("question_mark", "help", "Explain"),]- SORTS = [("VRAM", lambda r: -r.gpu.dedicated), ("GPU%", lambda r: -r.gpu.gpu_pct),+ SORTS = [("VRAM", lambda r: -r.gpu.shown_vram), ("GPU%", lambda r: -r.gpu.gpu_pct),("spill", lambda r: -r.gpu.shared), ("paged", lambda r: -r.gpu.paged_out),("name", lambda r: r.name.lower())]CHART_COLORS = {@@ -279,7 +279,7 @@ class GpuTaskManager(App):history.append(None)self.hist_vram.append(100 * snap.vram_used / snap.vram_total if snap.vram_total else None)if not self._auto_selected:- llm = next((r for r in sorted(snap.procs, key=lambda r: -r.gpu.dedicated) if r.llm), None)+ llm = next((r for r in sorted(snap.procs, key=lambda r: -r.gpu.shown_vram) if r.llm), None)if llm:self.selected_key = llm.keyself._auto_selected = True@@ -415,7 +415,7 @@ class GpuTaskManager(App):rows = sorted(rows, key=self.SORTS[self.sort_idx][1])vm = next((i for i, r in enumerate(rows) if r.kind == "wsl-vm"), None)if vm is not None and self.SORTS[self.sort_idx][0] == "VRAM":- # Linux rows have no VRAM of their own here; keep them under the WSL VM row whose total they share.+ # Linux rows share the WSL VM row's total: keep them under it, ordered by their inferred sizes.linux = [r for r in rows if not r.gpu.attributed and not r.host]rest = [r for r in rows if r not in linux]vm = rest.index(rows[vm])@@ -501,7 +501,7 @@ class GpuTaskManager(App):v = r.llmif v:width = max(min(self.size.width - 8, 100), 20)- total = max(r.gpu.dedicated, sum(x.bytes for x in v.segments), 1)+ total = max(r.gpu.shown_vram, sum(x.bytes for x in v.segments), 1)out.append(stacked_bar([(x.bytes, SEG_STYLE[x.kind]) for x in v.segments], total, width))legend = Text()for x in v.segments:diff --git a/src/gputaskman/collector.py b/src/gputaskman/collector.pyindex 1cfe583..1d414dd 100644--- a/src/gputaskman/collector.py+++ b/src/gputaskman/collector.py@@ -207,7 +207,8 @@ class Collector:snap.host_status = self.host_statuslive = [(pid, g) for pid, g in procs.items() if _is_live(g)]- live.sort(key=lambda x: -x[1].dedicated)+ # WSL Linux rows are ranked by last sample's inferred size, so the deep-scan budget reaches them too+ live.sort(key=lambda x: -(x[1].dedicated or self._inferred.get(x[0], 0.0)))deep_pids = {pid for pid, _ in live[: self.DEEP_TOP_N]} | {pid for pid, g in live if g.gpu_pct >= 2}deadline = None if self.scan_budget is None else time.perf_counter() + self.scan_budgetollama_runners = []diff --git a/src/gputaskman/pdh.py b/src/gputaskman/pdh.pyindex 1f9635f..1f183e2 100644--- a/src/gputaskman/pdh.py+++ b/src/gputaskman/pdh.py@@ -80,6 +80,11 @@ class ProcGpu:inferred: float | None = Noneinferred_by: str = "" # "consistent" (only one explanation fits) | "order" (start-order guess)+ @property+ def shown_vram(self) -> float:+ """The VRAM figure a row displays: measured, else inferred (WSL), else 0 for sorting."""+ return self.dedicated if self.attributed else (self.inferred or 0.0)+@propertydef paged_out(self) -> float:"""Commit minus local/non-local usage: a non-residency estimate, not an eviction trace."""diff --git a/src/gputaskman/report.py b/src/gputaskman/report.pyindex 71a9083..4c54f4d 100644--- a/src/gputaskman/report.py+++ b/src/gputaskman/report.py@@ -53,13 +53,20 @@ def to_dict(s: Snapshot) -> dict:nv = dataclasses.asdict(s.nv) if s.nv else Noneprocs = []for p in s.procs:+ # Unknown is null, not 0: WSL Linux rows have no per-process counters, and spill/commit only exist+ # in WDDM (native Windows, or Windows processes seen from WSL).+ known = p.gpu.attributed+ wddm = known and (s.backend == "wddm" or p.origin == "windows")item = {"pid": p.pid, "host": p.origin if p.host else None, "name": p.name, "kind": p.kind, "runtimes": p.detail.runtimes,"vram_attributed": p.gpu.attributed, "vram_inferred_bytes": p.gpu.inferred,"vram_inferred_by": p.gpu.inferred_by or None,- "vram_bytes": int(p.gpu.dedicated) if p.gpu.attributed else None, "shared_bytes": int(p.gpu.shared),- "committed_bytes": int(p.gpu.committed), "paged_out_bytes": int(p.gpu.paged_out),- "gpu_pct": round(p.gpu.gpu_pct, 1), "engines": {k: round(v, 1) for k, v in p.gpu.engines.items() if v},+ "vram_bytes": int(p.gpu.dedicated) if known else None,+ "shared_bytes": int(p.gpu.shared) if wddm else None,+ "committed_bytes": int(p.gpu.committed) if wddm else None,+ "paged_out_bytes": int(p.gpu.paged_out) if wddm else None,+ "gpu_pct": round(p.gpu.gpu_pct, 1) if known else None,+ "engines": {k: round(v, 1) for k, v in p.gpu.engines.items() if v} if known else None,"ports": p.detail.ports, "inside": p.inside,"model_files": [dataclasses.asdict(m) for m in p.detail.models],"installed_models": [dataclasses.asdict(m) for m in p.detail.candidates],@@ -142,7 +149,7 @@ def to_text(s: Snapshot, limit: int = 30) -> str:if not p.llm:continueout.append("")- out.append(f"== {p.name} ({p.pid}) VRAM {gb(p.gpu.dedicated)}")+ out.append(f"== {p.name} ({p.pid_label}) VRAM {vram_label(p.gpu)}")for i, g in enumerate(p.llm.models):kind = p.llm.kind_label(i)ev = p.llm.files[i].evidence if i < len(p.llm.files) else ""diff --git a/src/gputaskman/wslhost.py b/src/gputaskman/wslhost.pyindex 36982f6..854d660 100644--- a/src/gputaskman/wslhost.py+++ b/src/gputaskman/wslhost.py@@ -314,8 +314,8 @@ class ContextTracker:PENDING = 5.0 # seconds a new process may take to create its contextGONE = 5.0 # seconds an exited process's context may linger in the (older) host sampleTOL_MIN = 64 * MiB+ SELF_MAX = 1 * MiB # an NVML-only context (this monitor) holds 0 bytesTOL_REL = 0.10- SAVE_EVERY = 60.0def __init__(self, path: str | None = None, boot_id: str | None = None, self_pid: object = None) -> None:self.path = path or _default_cache()@@ -324,7 +324,6 @@ class ContextTracker:self.luid: int | None = Noneself.cached_foreign: list[float] = []self._reset()- self._saved = 0.0try:with open(self.path, encoding="utf-8") as f:data = json.load(f)@@ -381,11 +380,7 @@ class ContextTracker:best, best_cost = out, costreturn best- def _save(self, force: bool = False) -> None:- now = time.time()- if not force and now - self._saved < self.SAVE_EVERY:- return- self._saved = now+ def _save(self) -> None:try:os.makedirs(os.path.dirname(self.path), exist_ok=True)tmp = f"{self.path}.{os.getpid()}.tmp"@@ -451,8 +446,12 @@ class ContextTracker:if e.label is None:e.label, e.sure = FOREIGN, Trueelif first:- if self.self_pid in start and cur and cur[0].value <= self.TOL_MIN:- cur[0].label, cur[0].sure = self.self_pid, True # our NVML context: newest when we start+ # Our NVML-only context is 0 bytes and newest at startup, unless another process started after us:+ # its CUDA context could then be the newest one (and small while it initializes).+ later = any(t >= start[self.self_pid] for p, t in start.items() if p != self.self_pid) \+ if self.self_pid in start else True+ if not later and cur and cur[0].value <= self.SELF_MAX:+ cur[0].label, cur[0].sure = self.self_pid, Trueif set(start) == {self.self_pid} and cur and cur[0].label == self.self_pid:for e in cur[1:]: # nothing else in this distro holds the GPU: the rest is foreigne.label, e.sure = FOREIGN, True@@ -493,11 +492,12 @@ class ContextTracker:e.label, e.same = pid, sameself.entries = cur+ # Foreign labels only come from evidence, so the cache follows them, including down to none: a cached+ # context that exited must not later claim a local context of the same size.foreign = [e.value for e in cur if e.label == FOREIGN and e.sure]- if not start or foreign:- changed = sorted(foreign) != sorted(self.cached_foreign)+ if sorted(foreign) != sorted(self.cached_foreign):self.cached_foreign = foreign- self._save(force=changed and not start)+ self._save()out = {e.label: ContextGuess(e.value, "consistent" if e.sure or e.same else "order")for e in cur if e.label in start}return out, reasondiff --git a/tests/test_linux.py b/tests/test_linux.pyindex 6b235ac..0e69949 100644--- a/tests/test_linux.py+++ b/tests/test_linux.py@@ -284,6 +284,28 @@ class UnattributedReportTests(unittest.TestCase):self.assertIn("WSL:", text)self.assertNotIn("shared (in system RAM)", text)+ def test_json_unknown_counters_are_null_not_zero(self):+ snap = self.snapshot()+ win = ProcRow(2744, "dwm.exe", "compositor", ProcGpu(dedicated=GiB, shared=6 << 20, committed=2 * GiB),+ ProcDetail(2744, "dwm.exe"), host=True, origin="windows")+ snap.procs.append(win)+ linux_row, win_row = to_dict(snap)["processes"]+ for k in ("shared_bytes", "committed_bytes", "paged_out_bytes", "gpu_pct", "engines"):+ self.assertIsNone(linux_row[k], k)+ self.assertEqual((win_row["shared_bytes"], win_row["committed_bytes"]), (6 << 20, 2 * GiB))+ self.assertEqual(win_row["gpu_pct"], 0.0) # measured idle+ snap.backend = "nvml" # native Linux: NVML measures VRAM and load, but there is no spill/commit+ snap.procs = [ProcRow(547, "python3", "AI", ProcGpu(dedicated=GiB), ProcDetail(547, "python3"))]+ row = to_dict(snap)["processes"][0]+ self.assertEqual((row["vram_bytes"], row["gpu_pct"], row["shared_bytes"]), (GiB, 0.0, None))++ def test_wsl_rows_sort_by_the_size_they_show(self):+ from gputaskman.app import GpuTaskManager+ rows = [ProcRow(p, "python3", "AI", ProcGpu(attributed=False, inferred=v, inferred_by="consistent"),+ ProcDetail(p, "python3")) for p, v in ((1, GiB), (2, 5 * GiB), (3, None))]+ key = GpuTaskManager.SORTS[0][1]+ self.assertEqual([r.pid for r in sorted(rows, key=key)], [2, 1, 3])+def test_empty_wsl_view_explains_itself(self):snap = self.snapshot()snap.procs = []diff --git a/tests/test_wslhost.py b/tests/test_wslhost.pyindex 0f92373..13cf4f1 100644--- a/tests/test_wslhost.py+++ b/tests/test_wslhost.py@@ -168,6 +168,21 @@ class TrackerTests(unittest.TestCase):h.remove(("A", 950)) # the oldest job exits; the others keep their contextsself.check(self.step(t, [3.05, 0.4, 0.0, 0.0], h), {"F1": (0.4, "*"), "B": (3.05, "*")})+ def test_a_workload_started_after_us_is_not_taken_for_our_context(self):+ t = self.make()+ h = [("S", 900), ("W", 950)] # W created its CUDA context between our NVML init and the first sample+ self.check(self.step(t, [20 / 1024, 0.0], h), {"W": (0.02, "?")})+ self.check(self.step(t, [5.0, 0.0], h), {"W": (5.0, "?")})++ def test_cache_forgets_a_foreign_context_that_exited(self):+ t = self.make()+ self.check(self.step(t, [0.0, 2.0], [("S", 900)]), {}) # only us: 2.0 is foreign+ self.check(self.step(t, [0.0], [("S", 900)]), {}) # it exits+ with open(self.path, encoding="utf-8") as f:+ self.assertEqual(json.load(f)["foreign"], [])+ t = self.make() # next run, same boot: a local 2 GiB job must not be taken for the old foreign one+ self.check(self.step(t, [0.0, 2.0], [("S", 900), ("A", 800)]), {"A": (2.0, "*")})+def test_equal_sizes_are_right_now_but_never_evidence_later(self):t = self.make()h = [("S", 900), ("A", 13), ("B", 14)]