Skip to content

A GPU task manager for Windows, Linux, and WSL: per-process VRAM, GPU activity, and LLM model/KV-cache insights (Windows AMD and NVIDIA).

Python

Fix review findings from Claude and Codex

Sameh Khamis05f5a08e4193
diff --git a/src/gputaskman/app.py b/src/gputaskman/app.py
index 034458f..ad0863c 100644
--- a/src/gputaskman/app.py
+++ b/src/gputaskman/app.py
@@ -146,16 +146,6 @@ class HelpScreen(ModalScreen):
HelpScreen > VerticalScroll { width: 140; max-width: 95%; height: 85%; border: thick $accent; background: $surface; padding: 1 2; }
"""
- @property
- def selected_pid(self) -> int | None:
- """PID of the selected local (non-host) process."""
- key = self.selected_key
- return int(key) if key and not key.startswith("w") else None
-
- @selected_pid.setter
- def selected_pid(self, pid: int | None) -> None:
- self.selected_key = None if pid is None else str(pid)
-
def compose(self) -> ComposeResult:
with VerticalScroll():
yield Markdown(HELP_MD)
@@ -223,9 +213,9 @@ class GpuTaskManager(App):
@property
def selected_pid(self) -> int | None:
- """PID of the selected local (non-host) process."""
+ """PID of the selected local process; None for Windows (W:) and outside (H:) rows."""
key = self.selected_key
- return int(key) if key and not key.startswith("w") else None
+ return int(key) if key and key.isdigit() else None
@selected_pid.setter
def selected_pid(self, pid: int | None) -> None:
@@ -478,7 +468,7 @@ class GpuTaskManager(App):
panel.border_title = f"{r.name} ({where}{r.pid}) - {r.kind}"
out: list = []
e = r.gpu.engines
- wddm = s.backend == "wddm" or r.host # spill and commit counters only exist in WDDM
+ wddm = s.backend == "wddm" or r.origin == "windows" # spill and commit counters only exist in WDDM
engines = " ".join(f"{k} {v:.0f}%" for k, v in e.items() if v >= 0.5) or "idle"
hdr = Text.assemble(("VRAM ", "bold"), (vram_label(r.gpu), "bold"), " spill(RAM) ",
(gb(r.gpu.shared) if wddm else "n/a", "bold red" if r.gpu.shared > 256 << 20 else ""),
diff --git a/src/gputaskman/collector.py b/src/gputaskman/collector.py
index 7b84a95..1cfe583 100644
--- a/src/gputaskman/collector.py
+++ b/src/gputaskman/collector.py
@@ -171,6 +171,7 @@ class Collector:
self._host_luid: int | None = None # host adapter for this GPU, matched once and kept
self.tracker = ContextTracker(self_pid=os.getpid()) if self.host is not None else None
self._own_start = time.time() # our NVML context is created now, not when the process started
+ self._inferred: dict[int, float] = {} # WSL: last inferred VRAM per Linux PID
self._host_restarted = time.time()
def _parent_ports(self, pid: int, depth: int = 2) -> list[int]:
@@ -212,7 +213,8 @@ class Collector:
ollama_runners = []
for pid, g in live: # largest VRAM first, so the budget goes to what matters
d = self.inspector.get(pid)
- mem = g.dedicated + g.shared
+ # WSL: Linux rows carry no counter of their own; the last inferred size triggers rescans instead
+ mem = g.dedicated + g.shared or self._inferred.get(pid, 0.0)
if (pid in deep_pids and (deadline is None or time.perf_counter() < deadline)
and self.inspector.needs_scan(d, mem)):
self.inspector.scan(d, mem)
@@ -300,6 +302,7 @@ class Collector:
holders = [(r.pid, r.detail.create_time) for r in linux] + [(os.getpid(), self._own_start)]
guesses, reason = self.tracker.update(luid, contexts, holders)
snap.wsl_split = reason
+ self._inferred = {pid: g.bytes for pid, g in guesses.items()}
for r in linux:
g = guesses.get(r.pid)
if g is not None:
diff --git a/src/gputaskman/linux.py b/src/gputaskman/linux.py
index 56dcf52..cf27143 100644
--- a/src/gputaskman/linux.py
+++ b/src/gputaskman/linux.py
@@ -49,9 +49,17 @@ def _bus_id(handle) -> str:
def _nvml_devices() -> list[tuple[int, object]]:
+ """(adapter key, handle) per GPU NVML can open; a lost or failing GPU is skipped, not fatal."""
out = []
- for i in range(N.nvmlDeviceGetCount()):
- h = N.nvmlDeviceGetHandleByIndex(i)
+ try:
+ count = N.nvmlDeviceGetCount()
+ except Exception:
+ return out
+ for i in range(count):
+ try:
+ h = N.nvmlDeviceGetHandleByIndex(i)
+ except Exception:
+ continue
try:
key = pci_key(_bus_id(h))
except Exception:
@@ -86,6 +94,21 @@ def enumerate_adapters() -> list[Adapter]:
return out
+PROC_PID_INIT_INO = 0xEFFFFFFC # the kernel's fixed inode number for the initial (host) PID namespace
+
+
+@lru_cache(maxsize=1)
+def in_child_pid_namespace() -> bool | None:
+ """True inside a container with its own PID namespace, where NVML reports host PIDs. False in the host
+ namespace (bare metal, or a container run with --pid=host), where NVML's PIDs are this /proc's PIDs.
+ None when it can't be told."""
+ try:
+ link = os.readlink("/proc/self/ns/pid") # "pid:[4026531836]"
+ return int(link.split("[", 1)[1].rstrip("]")) != PROC_PID_INIT_INO
+ except (OSError, IndexError, ValueError):
+ return None
+
+
def dxg_pids(exclude: set[int] | None = None) -> set[int]:
"""Linux PIDs with /dev/dxg open: the processes that use the GPU under WSL. Other users' processes
are only visible when running as root."""
@@ -267,31 +290,37 @@ class LinuxCounters:
return procs, adapters
def _resolve_namespaces(self, procs: dict[int, dict[int, ProcGpu]]) -> None:
- """Re-key NVML's PIDs to local ones. Same namespace (bare metal, or a container sharing the host
- PID namespace): the PID exists here and holds an NVIDIA device. Otherwise (a container):
- match by creation order with PidMatcher."""
+ """Re-key NVML's PIDs to local ones. In the host PID namespace (bare metal, --pid=host) they already
+ are local, including other users' processes whose /proc/<pid>/fd this monitor can't read. In a
+ container's own namespace every NVML PID is a host PID, even one that happens to equal a local
+ PID, so all of them are matched by creation order with PidMatcher."""
self.outside = {}
nv_pids = {pid for per in procs.values() for pid in per}
- if not nv_pids:
+ child = in_child_pid_namespace()
+ if child is None: # unknown: PIDs that don't exist here can't be local
+ unresolved = sorted(pid for pid in nv_pids if not os.path.exists(f"/proc/{pid}"))
+ else:
+ unresolved = sorted(nv_pids) if child else []
+ if not unresolved:
self._matcher.update([], [])
return
holders = nvidia_holders()
- unresolved = sorted(pid for pid in nv_pids if pid not in holders)
- if not unresolved:
- return
me = os.getpid()
- local = sorted(p for p in holders - nv_pids if p != me and has_cuda(p))
+ local = sorted(p for p in holders - (nv_pids - set(unresolved)) if p != me and has_cuda(p))
compute = [pid for pid in unresolved if pid in self._compute_pids] # CUDA, so local libcuda users
matched, _ = self._matcher.update(compute, local)
+ unresolved = set(unresolved)
for key, per in procs.items():
+ moved: dict[int, ProcGpu] = {} # kept apart: a local PID may equal a host PID not yet moved
for pid in [p for p in per if p in unresolved]:
g = per.pop(pid)
if pid in matched:
lpid, confidence = matched[pid]
g.inferred_by = confidence
- per[lpid] = g
+ moved[lpid] = g
else:
self.outside.setdefault(key, {})[pid] = g
+ per.update(moved)
def close(self) -> None:
if self.ok:
diff --git a/src/gputaskman/llm.py b/src/gputaskman/llm.py
index 3b9a520..57430c6 100644
--- a/src/gputaskman/llm.py
+++ b/src/gputaskman/llm.py
@@ -398,8 +398,8 @@ def build_view(d: ProcDetail, vram: float, shared: float, server: ServerInfo | N
if rest is None:
if kv:
v.segments.append(Segment(f"KV cache ({v.n_ctx:,} ctx, {v.kv_type}) est.", kv, "kv"))
- v.notes.append("This process's VRAM total is unknown (WSL charges it to the VM on the Windows host); "
- "segments are estimates from the model file and launch flags.")
+ v.notes.append("This process's VRAM total is unknown (WSL charges it to the VM on the Windows host, "
+ "or the driver hides per-process memory); segments are estimates from the model file and launch flags.")
elif kv and kv <= rest:
swa = main.sliding_window if main else None
v.segments.append(Segment(f"KV cache ({v.n_ctx:,} ctx, {v.kv_type}) est.{' max' if swa else ''}", kv, "kv"))
diff --git a/src/gputaskman/procinfo.py b/src/gputaskman/procinfo.py
index 7f879c8..7d589bf 100644
--- a/src/gputaskman/procinfo.py
+++ b/src/gputaskman/procinfo.py
@@ -225,7 +225,7 @@ def installed_models(exe: str, now: float | None = None) -> list[MappedModel]:
hit = _installed.get(key)
if hit and now - hit[0] < INSTALLED_TTL:
return hit[1]
- stem = os.path.splitext(os.path.basename(key))[0]
+ stem = os.path.splitext(os.path.basename(key))[0].lower() # normcase only lowercases on Windows
roots = []
if not _is_system_path(key):
roots.append(os.path.dirname(exe))
diff --git a/src/gputaskman/wslhost.py b/src/gputaskman/wslhost.py
index ef8ebf9..c9372e4 100644
--- a/src/gputaskman/wslhost.py
+++ b/src/gputaskman/wslhost.py
@@ -32,6 +32,7 @@ _POWERSHELL_FALLBACK = "/mnt/c/Windows/System32/WindowsPowerShell/v1.0/powershel
# Counter paths are English (PdhAddEnglishCounter), so this works on localized Windows too.
_SCRIPT = r"""
$ErrorActionPreference = 'Stop'
+[Console]::OutputEncoding = New-Object System.Text.UTF8Encoding $false # not the OEM code page
Add-Type -TypeDefinition @'
using System;
using System.Runtime.InteropServices;
@@ -213,7 +214,7 @@ class HostStream:
encoded = base64.b64encode(_SCRIPT.encode("utf-16-le")).decode("ascii")
self._proc = popen([exe, "-NoProfile", "-NonInteractive", "-EncodedCommand", encoded],
stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL,
- text=True, bufsize=1)
+ text=True, encoding="utf-8", errors="replace", bufsize=1)
except (OSError, ValueError) as error:
self.status = f"unavailable ({error})"
self._ready.set()
@@ -221,15 +222,23 @@ class HostStream:
threading.Thread(target=self._read, name="wsl-host-counters", daemon=True).start()
def _read(self) -> None:
- for line in self._proc.stdout:
- sample = parse_line(line)
- if sample is not None and (sample.procs or sample.adapters):
- self._latest = sample
- self.status = "ok"
- self._ready.set()
- if self.status != "off":
- self.status = "unavailable (counter stream ended)"
- self._ready.set()
+ try:
+ for line in self._proc.stdout:
+ try:
+ sample = parse_line(line)
+ except Exception: # one malformed line must not end the stream
+ continue
+ if sample is not None and (sample.procs or sample.adapters):
+ self._latest = sample
+ self.status = "ok"
+ self._ready.set()
+ except Exception:
+ pass
+ finally:
+ # always report the end, so the collector restarts the stream instead of waiting forever
+ if self.status != "off":
+ self.status = "unavailable (counter stream ended)"
+ self._ready.set()
def wait(self, timeout: float) -> bool:
return self._ready.wait(timeout)
@@ -284,6 +293,7 @@ class _Entry:
value: float
label: object = None # holder pid | FOREIGN | ("gone", pid): its holder exited, waiting to vanish
sure: bool = False # established by an event or elimination, never by a guess
+ same: bool = False # this sample only: all candidates are the same size, so the bytes are right either way
class ContextTracker:
@@ -411,10 +421,12 @@ class ContextTracker:
for pid in start:
if pid not in self.seen:
self.seen[pid] = now
+ self.gone.pop(pid, None) # reused PID: not the process that exited
if first:
self.initial.add(pid)
for pid in [p for p in self.seen if p not in start]:
del self.seen[pid]
+ self.initial.discard(pid) # a reused PID is a new process, not one that was running at startup
self.gone[pid] = now
for e in cur:
if e.label in self.gone and not isinstance(e.label, tuple):
@@ -464,13 +476,17 @@ class ContextTracker:
if len(free) < len(unsure):
reason = (f"{len(unsure)} Linux GPU process(es) without a VM context to match "
"(one may not have created its GPU context yet)")
- elif len(free) == len(unsure) == 1:
+ # A process seen moments ago may not have created its context yet; the one free context could
+ # then be a foreign one (WSLg, another distro), so wait for its own context instead of eliminating.
+ recent = any(p not in self.initial and now - self.seen[p] <= self.PENDING for p in unsure)
+ if len(free) == len(unsure) == 1 and not recent:
free[0].label, free[0].sure = unsure[0], True
else:
- same = len(free) == len(unsure) and all(abs(e.value - free[0].value) <= self._tol(free[0].value)
- for e in free)
+ same = (len(free) == len(unsure) > 1 and not recent
+ and all(abs(e.value - free[0].value) <= self._tol(free[0].value) for e in free))
for pid, e in zip(unsure, free): # newest process <-> newest context; leftovers are foreign
- e.label, e.sure = pid, same
+ # Equal sizes make the shown bytes right, not the owner: never carried as evidence.
+ e.label, e.same = pid, same
self.entries = cur
foreign = [e.value for e in cur if e.label == FOREIGN and e.sure]
@@ -478,6 +494,6 @@ class ContextTracker:
changed = sorted(foreign) != sorted(self.cached_foreign)
self.cached_foreign = foreign
self._save(force=changed and not start)
- out = {e.label: ContextGuess(e.value, "consistent" if e.sure else "order")
+ out = {e.label: ContextGuess(e.value, "consistent" if e.sure or e.same else "order")
for e in cur if e.label in start}
return out, reason
diff --git a/tests/test_linux.py b/tests/test_linux.py
index 39abb41..6b235ac 100644
--- a/tests/test_linux.py
+++ b/tests/test_linux.py
@@ -103,9 +103,10 @@ class PdhImportTests(unittest.TestCase):
class LinuxCountersTests(unittest.TestCase):
def setUp(self):
# Same PID namespace (bare metal): NVML's PIDs are local processes holding an NVIDIA device.
- holders = patch.object(linux, "nvidia_holders", return_value={100, 101, 200, 300, 555})
- holders.start()
- self.addCleanup(holders.stop)
+ for p in (patch.object(linux, "nvidia_holders", return_value={100, 101, 200, 300, 555}),
+ patch.object(linux, "in_child_pid_namespace", return_value=False)):
+ p.start()
+ self.addCleanup(p.stop)
def counters(self, fake, wsl):
with patch.object(linux, "N", fake), patch.object(linux, "is_wsl", return_value=wsl):
@@ -161,6 +162,32 @@ class LinuxCountersTests(unittest.TestCase):
self.assertEqual([(a.luid, a.name, a.dedicated_vram) for a in adapters],
[(KEY, "NVIDIA GeForce RTX 3090 Ti", 24 * GiB)])
+ def test_a_failing_gpu_is_skipped_not_fatal(self):
+ fake = FakeNvml()
+ fake.nvmlDeviceGetCount = lambda: 2
+ real = fake.nvmlDeviceGetHandleByIndex
+ def handle(i):
+ if i == 0:
+ raise RuntimeError("NVML_ERROR_GPU_IS_LOST")
+ return real(0)
+ fake.nvmlDeviceGetHandleByIndex = handle
+ with patch.object(linux, "N", fake):
+ self.assertEqual(len(linux.enumerate_adapters()), 1)
+ fake.nvmlDeviceGetCount = lambda: (_ for _ in ()).throw(RuntimeError("NVML_ERROR_UNKNOWN"))
+ with patch.object(linux, "N", fake):
+ self.assertEqual(linux.enumerate_adapters(), [])
+
+ def test_other_users_processes_stay_local_on_bare_metal(self):
+ # Without root, another user's /proc/<pid>/fd is unreadable, so it isn't an NVIDIA holder.
+ fake = FakeNvml(compute=[proc(6000, 20 * GiB)])
+ c = self.counters(fake, wsl=False)
+ with patch.object(linux, "N", fake), patch.object(linux, "nvidia_holders", return_value={100}), \
+ patch.object(linux, "has_cuda", return_value=True):
+ procs, _ = c.sample()
+ self.assertEqual((procs[KEY][6000].dedicated, procs[KEY][6000].inferred_by), (20 * GiB, ""))
+ self.assertNotIn(100, procs[KEY])
+ self.assertEqual(c.outside, {})
+
class ContainerTests(unittest.TestCase):
"""Measured on a Vast.ai Docker container: NVML lists host PIDs (2029701...) while the processes are
@@ -168,9 +195,12 @@ class ContainerTests(unittest.TestCase):
def sample(self, c, fake, holders, cuda):
with patch.object(linux, "N", fake), patch.object(linux, "nvidia_holders", return_value=set(holders)), \
- patch.object(linux, "has_cuda", side_effect=lambda pid: pid in cuda):
+ patch.object(linux, "has_cuda", side_effect=lambda pid: pid in cuda), \
+ patch.object(linux, "in_child_pid_namespace", return_value=self.child):
return c.sample()[0][KEY]
+ child = True
+
def test_host_pids_are_matched_to_container_pids_by_creation_order(self):
fake = FakeNvml(compute=[proc(2029701, GiB), proc(2029747, 2 * GiB)],
graphics=[proc(1145, 58 << 20), proc(1360, 21 << 20)],
@@ -199,10 +229,32 @@ class ContainerTests(unittest.TestCase):
fake = FakeNvml(compute=[proc(547, GiB)])
with patch.object(linux, "N", fake), patch.object(linux, "is_wsl", return_value=False):
c = linux.LinuxCounters()
+ self.child = False
per = self.sample(c, fake, holders={547}, cuda={547})
self.assertEqual((per[547].dedicated, per[547].inferred_by), (GiB, ""))
self.assertEqual(c.outside, {})
+ def test_a_host_pid_equal_to_a_local_pid_is_still_a_host_pid(self):
+ # Host PIDs 100 and 200 are container PIDs 10 and 100: local PID 100 is not host PID 100.
+ fake = FakeNvml(compute=[proc(100, GiB), proc(200, 4 * GiB)])
+ with patch.object(linux, "N", fake), patch.object(linux, "is_wsl", return_value=False):
+ c = linux.LinuxCounters()
+ per = self.sample(c, fake, holders={10, 100}, cuda={10, 100})
+ self.assertEqual({pid: (g.dedicated, g.inferred_by) for pid, g in per.items()},
+ {10: (GiB, "consistent"), 100: (4 * GiB, "consistent")})
+
+ def test_namespace_detection(self):
+ linux.in_child_pid_namespace.cache_clear()
+ self.addCleanup(linux.in_child_pid_namespace.cache_clear)
+ with patch.object(linux.os, "readlink", return_value="pid:[4026531836]"):
+ self.assertFalse(linux.in_child_pid_namespace())
+ linux.in_child_pid_namespace.cache_clear()
+ with patch.object(linux.os, "readlink", return_value="pid:[4026532481]"):
+ self.assertTrue(linux.in_child_pid_namespace())
+ linux.in_child_pid_namespace.cache_clear()
+ with patch.object(linux.os, "readlink", side_effect=OSError):
+ self.assertIsNone(linux.in_child_pid_namespace())
+
def test_outside_rows_never_borrow_a_local_identity(self):
from gputaskman.collector import ProcRow as Row
row = Row(1145, "(outside this container)", "external", ProcGpu(dedicated=58 << 20),
diff --git a/tests/test_wslhost.py b/tests/test_wslhost.py
index ed52ded..dcc9a38 100644
--- a/tests/test_wslhost.py
+++ b/tests/test_wslhost.py
@@ -99,6 +99,27 @@ class InteropTests(unittest.TestCase):
self.assertIsNotNone(h.latest())
self.assertEqual(calls[0][:4], ["powershell.exe", "-NoProfile", "-NonInteractive", "-EncodedCommand"])
+ def test_a_bad_byte_ends_the_stream_visibly_so_it_restarts(self):
+ class BadStdout:
+ def __iter__(self):
+ yield SAMPLE + "\n"
+ raise UnicodeDecodeError("utf-8", b"\x82", 0, 1, "invalid start byte")
+
+ class FakeProc:
+ def __init__(self, args, **kw):
+ self.encoding = kw.get("encoding")
+ self.stdout = BadStdout()
+
+ def poll(self):
+ return 0
+
+ h = HostStream(exe="powershell.exe", popen=FakeProc)
+ deadline = time.time() + 2
+ while not h.ended and time.time() < deadline:
+ time.sleep(0.01)
+ self.assertTrue(h.ended)
+ self.assertEqual(h._proc.encoding, "utf-8")
+
def test_missing_powershell_is_reported(self):
with patch("gputaskman.wslhost.powershell_path", return_value=None):
h = HostStream()
@@ -147,6 +168,28 @@ class TrackerTests(unittest.TestCase):
h.remove(("A", 950)) # the oldest job exits; the others keep their contexts
self.check(self.step(t, [3.05, 0.4, 0.0, 0.0], h), {"F1": (0.4, "*"), "B": (3.05, "*")})
+ def test_equal_sizes_are_right_now_but_never_evidence_later(self):
+ t = self.make()
+ h = [("S", 900), ("A", 13), ("B", 14)]
+ self.check(self.step(t, [0.0, 0.2, 0.2], h), {"B": (0.2, "*"), "A": (0.2, "*")})
+ # one of them loads a model: which one is unknown, so neither keeps the "*"
+ self.check(self.step(t, [0.0, 4.0, 0.2], h), {"B": (4.0, "?"), "A": (0.2, "?")})
+
+ def test_new_process_waits_for_its_own_context_instead_of_eliminating(self):
+ t = self.make()
+ h = [("S", 900)]
+ self.check(self.step(t, [0.0, 0.15], h), {}) # 0.15: WSLg, not yet known as foreign
+ h.append(("A", 1001))
+ self.check(self.step(t, [0.0, 0.15], h), {"A": (0.15, "?")}) # A's context isn't there yet
+ self.check(self.step(t, [6.0, 0.0, 0.15], h), {"A": (6.0, "*")})
+
+ def test_a_reused_pid_is_a_new_process(self):
+ t = self.make()
+ self.check(self.step(t, [0.0, 1.0], [("S", 900), ("P", 10)]), {"P": (1.0, "*")})
+ self.check(self.step(t, [0.0], [("S", 900)]), {})
+ self.check(self.step(t, [2.0, 0.0], [("S", 900), ("P", 2000)]), {"P": (2.0, "*")})
+ self.check(self.step(t, [2.0, 0.0], [("S", 900), ("P", 2000)]), {"P": (2.0, "*")})
+
def test_already_running_processes_are_guesses_until_evidence(self):
t = self.make()
h = [("S", 900), ("C", 14), ("B", 13)]
@@ -238,6 +281,7 @@ class HostRowTests(unittest.TestCase):
c._host_restarted = time.time()
c.tracker = ContextTracker(path=os.path.join(self.tmp.name, "ctx.json"), boot_id="boot", self_pid=os.getpid())
c._own_start = time.time()
+ c._inferred = {}
return c
def setUp(self):