Skip to content

A GPU task manager for Windows, Linux, and WSL: per-process VRAM, GPU activity, and LLM model/KV-cache insights (Windows AMD and NVIDIA).

Python

Address PR review comments

Sameh Khamisd361d5fe785f
diff --git a/src/gputaskman/app.py b/src/gputaskman/app.py
index ad0863c..320c170 100644
--- a/src/gputaskman/app.py
+++ b/src/gputaskman/app.py
@@ -175,7 +175,7 @@ class GpuTaskManager(App):
Binding("p", "pause", "Pause"),
Binding("question_mark", "help", "Explain"),
]
- SORTS = [("VRAM", lambda r: -r.gpu.dedicated), ("GPU%", lambda r: -r.gpu.gpu_pct),
+ SORTS = [("VRAM", lambda r: -r.gpu.shown_vram), ("GPU%", lambda r: -r.gpu.gpu_pct),
("spill", lambda r: -r.gpu.shared), ("paged", lambda r: -r.gpu.paged_out),
("name", lambda r: r.name.lower())]
CHART_COLORS = {
@@ -279,7 +279,7 @@ class GpuTaskManager(App):
history.append(None)
self.hist_vram.append(100 * snap.vram_used / snap.vram_total if snap.vram_total else None)
if not self._auto_selected:
- llm = next((r for r in sorted(snap.procs, key=lambda r: -r.gpu.dedicated) if r.llm), None)
+ llm = next((r for r in sorted(snap.procs, key=lambda r: -r.gpu.shown_vram) if r.llm), None)
if llm:
self.selected_key = llm.key
self._auto_selected = True
@@ -415,7 +415,7 @@ class GpuTaskManager(App):
rows = sorted(rows, key=self.SORTS[self.sort_idx][1])
vm = next((i for i, r in enumerate(rows) if r.kind == "wsl-vm"), None)
if vm is not None and self.SORTS[self.sort_idx][0] == "VRAM":
- # Linux rows have no VRAM of their own here; keep them under the WSL VM row whose total they share.
+ # Linux rows share the WSL VM row's total: keep them under it, ordered by their inferred sizes.
linux = [r for r in rows if not r.gpu.attributed and not r.host]
rest = [r for r in rows if r not in linux]
vm = rest.index(rows[vm])
@@ -501,7 +501,7 @@ class GpuTaskManager(App):
v = r.llm
if v:
width = max(min(self.size.width - 8, 100), 20)
- total = max(r.gpu.dedicated, sum(x.bytes for x in v.segments), 1)
+ total = max(r.gpu.shown_vram, sum(x.bytes for x in v.segments), 1)
out.append(stacked_bar([(x.bytes, SEG_STYLE[x.kind]) for x in v.segments], total, width))
legend = Text()
for x in v.segments:
diff --git a/src/gputaskman/collector.py b/src/gputaskman/collector.py
index 1cfe583..1d414dd 100644
--- a/src/gputaskman/collector.py
+++ b/src/gputaskman/collector.py
@@ -207,7 +207,8 @@ class Collector:
snap.host_status = self.host_status
live = [(pid, g) for pid, g in procs.items() if _is_live(g)]
- live.sort(key=lambda x: -x[1].dedicated)
+ # WSL Linux rows are ranked by last sample's inferred size, so the deep-scan budget reaches them too
+ live.sort(key=lambda x: -(x[1].dedicated or self._inferred.get(x[0], 0.0)))
deep_pids = {pid for pid, _ in live[: self.DEEP_TOP_N]} | {pid for pid, g in live if g.gpu_pct >= 2}
deadline = None if self.scan_budget is None else time.perf_counter() + self.scan_budget
ollama_runners = []
diff --git a/src/gputaskman/pdh.py b/src/gputaskman/pdh.py
index 1f9635f..1f183e2 100644
--- a/src/gputaskman/pdh.py
+++ b/src/gputaskman/pdh.py
@@ -80,6 +80,11 @@ class ProcGpu:
inferred: float | None = None
inferred_by: str = "" # "consistent" (only one explanation fits) | "order" (start-order guess)
+ @property
+ def shown_vram(self) -> float:
+ """The VRAM figure a row displays: measured, else inferred (WSL), else 0 for sorting."""
+ return self.dedicated if self.attributed else (self.inferred or 0.0)
+
@property
def paged_out(self) -> float:
"""Commit minus local/non-local usage: a non-residency estimate, not an eviction trace."""
diff --git a/src/gputaskman/report.py b/src/gputaskman/report.py
index 71a9083..4c54f4d 100644
--- a/src/gputaskman/report.py
+++ b/src/gputaskman/report.py
@@ -53,13 +53,20 @@ def to_dict(s: Snapshot) -> dict:
nv = dataclasses.asdict(s.nv) if s.nv else None
procs = []
for p in s.procs:
+ # Unknown is null, not 0: WSL Linux rows have no per-process counters, and spill/commit only exist
+ # in WDDM (native Windows, or Windows processes seen from WSL).
+ known = p.gpu.attributed
+ wddm = known and (s.backend == "wddm" or p.origin == "windows")
item = {
"pid": p.pid, "host": p.origin if p.host else None, "name": p.name, "kind": p.kind, "runtimes": p.detail.runtimes,
"vram_attributed": p.gpu.attributed, "vram_inferred_bytes": p.gpu.inferred,
"vram_inferred_by": p.gpu.inferred_by or None,
- "vram_bytes": int(p.gpu.dedicated) if p.gpu.attributed else None, "shared_bytes": int(p.gpu.shared),
- "committed_bytes": int(p.gpu.committed), "paged_out_bytes": int(p.gpu.paged_out),
- "gpu_pct": round(p.gpu.gpu_pct, 1), "engines": {k: round(v, 1) for k, v in p.gpu.engines.items() if v},
+ "vram_bytes": int(p.gpu.dedicated) if known else None,
+ "shared_bytes": int(p.gpu.shared) if wddm else None,
+ "committed_bytes": int(p.gpu.committed) if wddm else None,
+ "paged_out_bytes": int(p.gpu.paged_out) if wddm else None,
+ "gpu_pct": round(p.gpu.gpu_pct, 1) if known else None,
+ "engines": {k: round(v, 1) for k, v in p.gpu.engines.items() if v} if known else None,
"ports": p.detail.ports, "inside": p.inside,
"model_files": [dataclasses.asdict(m) for m in p.detail.models],
"installed_models": [dataclasses.asdict(m) for m in p.detail.candidates],
@@ -142,7 +149,7 @@ def to_text(s: Snapshot, limit: int = 30) -> str:
if not p.llm:
continue
out.append("")
- out.append(f"== {p.name} ({p.pid}) VRAM {gb(p.gpu.dedicated)}")
+ out.append(f"== {p.name} ({p.pid_label}) VRAM {vram_label(p.gpu)}")
for i, g in enumerate(p.llm.models):
kind = p.llm.kind_label(i)
ev = p.llm.files[i].evidence if i < len(p.llm.files) else ""
diff --git a/src/gputaskman/wslhost.py b/src/gputaskman/wslhost.py
index 36982f6..854d660 100644
--- a/src/gputaskman/wslhost.py
+++ b/src/gputaskman/wslhost.py
@@ -314,8 +314,8 @@ class ContextTracker:
PENDING = 5.0 # seconds a new process may take to create its context
GONE = 5.0 # seconds an exited process's context may linger in the (older) host sample
TOL_MIN = 64 * MiB
+ SELF_MAX = 1 * MiB # an NVML-only context (this monitor) holds 0 bytes
TOL_REL = 0.10
- SAVE_EVERY = 60.0
def __init__(self, path: str | None = None, boot_id: str | None = None, self_pid: object = None) -> None:
self.path = path or _default_cache()
@@ -324,7 +324,6 @@ class ContextTracker:
self.luid: int | None = None
self.cached_foreign: list[float] = []
self._reset()
- self._saved = 0.0
try:
with open(self.path, encoding="utf-8") as f:
data = json.load(f)
@@ -381,11 +380,7 @@ class ContextTracker:
best, best_cost = out, cost
return best
- def _save(self, force: bool = False) -> None:
- now = time.time()
- if not force and now - self._saved < self.SAVE_EVERY:
- return
- self._saved = now
+ def _save(self) -> None:
try:
os.makedirs(os.path.dirname(self.path), exist_ok=True)
tmp = f"{self.path}.{os.getpid()}.tmp"
@@ -451,8 +446,12 @@ class ContextTracker:
if e.label is None:
e.label, e.sure = FOREIGN, True
elif first:
- if self.self_pid in start and cur and cur[0].value <= self.TOL_MIN:
- cur[0].label, cur[0].sure = self.self_pid, True # our NVML context: newest when we start
+ # Our NVML-only context is 0 bytes and newest at startup, unless another process started after us:
+ # its CUDA context could then be the newest one (and small while it initializes).
+ later = any(t >= start[self.self_pid] for p, t in start.items() if p != self.self_pid) \
+ if self.self_pid in start else True
+ if not later and cur and cur[0].value <= self.SELF_MAX:
+ cur[0].label, cur[0].sure = self.self_pid, True
if set(start) == {self.self_pid} and cur and cur[0].label == self.self_pid:
for e in cur[1:]: # nothing else in this distro holds the GPU: the rest is foreign
e.label, e.sure = FOREIGN, True
@@ -493,11 +492,12 @@ class ContextTracker:
e.label, e.same = pid, same
self.entries = cur
+ # Foreign labels only come from evidence, so the cache follows them, including down to none: a cached
+ # context that exited must not later claim a local context of the same size.
foreign = [e.value for e in cur if e.label == FOREIGN and e.sure]
- if not start or foreign:
- changed = sorted(foreign) != sorted(self.cached_foreign)
+ if sorted(foreign) != sorted(self.cached_foreign):
self.cached_foreign = foreign
- self._save(force=changed and not start)
+ self._save()
out = {e.label: ContextGuess(e.value, "consistent" if e.sure or e.same else "order")
for e in cur if e.label in start}
return out, reason
diff --git a/tests/test_linux.py b/tests/test_linux.py
index 6b235ac..0e69949 100644
--- a/tests/test_linux.py
+++ b/tests/test_linux.py
@@ -284,6 +284,28 @@ class UnattributedReportTests(unittest.TestCase):
self.assertIn("WSL:", text)
self.assertNotIn("shared (in system RAM)", text)
+ def test_json_unknown_counters_are_null_not_zero(self):
+ snap = self.snapshot()
+ win = ProcRow(2744, "dwm.exe", "compositor", ProcGpu(dedicated=GiB, shared=6 << 20, committed=2 * GiB),
+ ProcDetail(2744, "dwm.exe"), host=True, origin="windows")
+ snap.procs.append(win)
+ linux_row, win_row = to_dict(snap)["processes"]
+ for k in ("shared_bytes", "committed_bytes", "paged_out_bytes", "gpu_pct", "engines"):
+ self.assertIsNone(linux_row[k], k)
+ self.assertEqual((win_row["shared_bytes"], win_row["committed_bytes"]), (6 << 20, 2 * GiB))
+ self.assertEqual(win_row["gpu_pct"], 0.0) # measured idle
+ snap.backend = "nvml" # native Linux: NVML measures VRAM and load, but there is no spill/commit
+ snap.procs = [ProcRow(547, "python3", "AI", ProcGpu(dedicated=GiB), ProcDetail(547, "python3"))]
+ row = to_dict(snap)["processes"][0]
+ self.assertEqual((row["vram_bytes"], row["gpu_pct"], row["shared_bytes"]), (GiB, 0.0, None))
+
+ def test_wsl_rows_sort_by_the_size_they_show(self):
+ from gputaskman.app import GpuTaskManager
+ rows = [ProcRow(p, "python3", "AI", ProcGpu(attributed=False, inferred=v, inferred_by="consistent"),
+ ProcDetail(p, "python3")) for p, v in ((1, GiB), (2, 5 * GiB), (3, None))]
+ key = GpuTaskManager.SORTS[0][1]
+ self.assertEqual([r.pid for r in sorted(rows, key=key)], [2, 1, 3])
+
def test_empty_wsl_view_explains_itself(self):
snap = self.snapshot()
snap.procs = []
diff --git a/tests/test_wslhost.py b/tests/test_wslhost.py
index 0f92373..13cf4f1 100644
--- a/tests/test_wslhost.py
+++ b/tests/test_wslhost.py
@@ -168,6 +168,21 @@ class TrackerTests(unittest.TestCase):
h.remove(("A", 950)) # the oldest job exits; the others keep their contexts
self.check(self.step(t, [3.05, 0.4, 0.0, 0.0], h), {"F1": (0.4, "*"), "B": (3.05, "*")})
+ def test_a_workload_started_after_us_is_not_taken_for_our_context(self):
+ t = self.make()
+ h = [("S", 900), ("W", 950)] # W created its CUDA context between our NVML init and the first sample
+ self.check(self.step(t, [20 / 1024, 0.0], h), {"W": (0.02, "?")})
+ self.check(self.step(t, [5.0, 0.0], h), {"W": (5.0, "?")})
+
+ def test_cache_forgets_a_foreign_context_that_exited(self):
+ t = self.make()
+ self.check(self.step(t, [0.0, 2.0], [("S", 900)]), {}) # only us: 2.0 is foreign
+ self.check(self.step(t, [0.0], [("S", 900)]), {}) # it exits
+ with open(self.path, encoding="utf-8") as f:
+ self.assertEqual(json.load(f)["foreign"], [])
+ t = self.make() # next run, same boot: a local 2 GiB job must not be taken for the old foreign one
+ self.check(self.step(t, [0.0, 2.0], [("S", 900), ("A", 800)]), {"A": (2.0, "*")})
+
def test_equal_sizes_are_right_now_but_never_evidence_later(self):
t = self.make()
h = [("S", 900), ("A", 13), ("B", 14)]