A GPU task manager for Windows, Linux, and WSL: per-process VRAM, GPU activity, and LLM model/KV-cache insights (Windows AMD and NVIDIA).
Python
Fix review findings from Claude and Codex
Sameh Khamis05f5a08e4193
diff --git a/src/gputaskman/app.py b/src/gputaskman/app.pyindex 034458f..ad0863c 100644--- a/src/gputaskman/app.py+++ b/src/gputaskman/app.py@@ -146,16 +146,6 @@ class HelpScreen(ModalScreen):HelpScreen > VerticalScroll { width: 140; max-width: 95%; height: 85%; border: thick $accent; background: $surface; padding: 1 2; }"""- @property- def selected_pid(self) -> int | None:- """PID of the selected local (non-host) process."""- key = self.selected_key- return int(key) if key and not key.startswith("w") else None-- @selected_pid.setter- def selected_pid(self, pid: int | None) -> None:- self.selected_key = None if pid is None else str(pid)-def compose(self) -> ComposeResult:with VerticalScroll():yield Markdown(HELP_MD)@@ -223,9 +213,9 @@ class GpuTaskManager(App):@propertydef selected_pid(self) -> int | None:- """PID of the selected local (non-host) process."""+ """PID of the selected local process; None for Windows (W:) and outside (H:) rows."""key = self.selected_key- return int(key) if key and not key.startswith("w") else None+ return int(key) if key and key.isdigit() else None@selected_pid.setterdef selected_pid(self, pid: int | None) -> None:@@ -478,7 +468,7 @@ class GpuTaskManager(App):panel.border_title = f"{r.name} ({where}{r.pid}) - {r.kind}"out: list = []e = r.gpu.engines- wddm = s.backend == "wddm" or r.host # spill and commit counters only exist in WDDM+ wddm = s.backend == "wddm" or r.origin == "windows" # spill and commit counters only exist in WDDMengines = " ".join(f"{k} {v:.0f}%" for k, v in e.items() if v >= 0.5) or "idle"hdr = Text.assemble(("VRAM ", "bold"), (vram_label(r.gpu), "bold"), " spill(RAM) ",(gb(r.gpu.shared) if wddm else "n/a", "bold red" if r.gpu.shared > 256 << 20 else ""),diff --git a/src/gputaskman/collector.py b/src/gputaskman/collector.pyindex 7b84a95..1cfe583 100644--- a/src/gputaskman/collector.py+++ b/src/gputaskman/collector.py@@ -171,6 +171,7 @@ class Collector:self._host_luid: int | None = None # host adapter for this GPU, matched once and keptself.tracker = ContextTracker(self_pid=os.getpid()) if self.host is not None else Noneself._own_start = time.time() # our NVML context is created now, not when the process started+ self._inferred: dict[int, float] = {} # WSL: last inferred VRAM per Linux PIDself._host_restarted = time.time()def _parent_ports(self, pid: int, depth: int = 2) -> list[int]:@@ -212,7 +213,8 @@ class Collector:ollama_runners = []for pid, g in live: # largest VRAM first, so the budget goes to what mattersd = self.inspector.get(pid)- mem = g.dedicated + g.shared+ # WSL: Linux rows carry no counter of their own; the last inferred size triggers rescans instead+ mem = g.dedicated + g.shared or self._inferred.get(pid, 0.0)if (pid in deep_pids and (deadline is None or time.perf_counter() < deadline)and self.inspector.needs_scan(d, mem)):self.inspector.scan(d, mem)@@ -300,6 +302,7 @@ class Collector:holders = [(r.pid, r.detail.create_time) for r in linux] + [(os.getpid(), self._own_start)]guesses, reason = self.tracker.update(luid, contexts, holders)snap.wsl_split = reason+ self._inferred = {pid: g.bytes for pid, g in guesses.items()}for r in linux:g = guesses.get(r.pid)if g is not None:diff --git a/src/gputaskman/linux.py b/src/gputaskman/linux.pyindex 56dcf52..cf27143 100644--- a/src/gputaskman/linux.py+++ b/src/gputaskman/linux.py@@ -49,9 +49,17 @@ def _bus_id(handle) -> str:def _nvml_devices() -> list[tuple[int, object]]:+ """(adapter key, handle) per GPU NVML can open; a lost or failing GPU is skipped, not fatal."""out = []- for i in range(N.nvmlDeviceGetCount()):- h = N.nvmlDeviceGetHandleByIndex(i)+ try:+ count = N.nvmlDeviceGetCount()+ except Exception:+ return out+ for i in range(count):+ try:+ h = N.nvmlDeviceGetHandleByIndex(i)+ except Exception:+ continuetry:key = pci_key(_bus_id(h))except Exception:@@ -86,6 +94,21 @@ def enumerate_adapters() -> list[Adapter]:return out+PROC_PID_INIT_INO = 0xEFFFFFFC # the kernel's fixed inode number for the initial (host) PID namespace+++@lru_cache(maxsize=1)+def in_child_pid_namespace() -> bool | None:+ """True inside a container with its own PID namespace, where NVML reports host PIDs. False in the host+ namespace (bare metal, or a container run with --pid=host), where NVML's PIDs are this /proc's PIDs.+ None when it can't be told."""+ try:+ link = os.readlink("/proc/self/ns/pid") # "pid:[4026531836]"+ return int(link.split("[", 1)[1].rstrip("]")) != PROC_PID_INIT_INO+ except (OSError, IndexError, ValueError):+ return None++def dxg_pids(exclude: set[int] | None = None) -> set[int]:"""Linux PIDs with /dev/dxg open: the processes that use the GPU under WSL. Other users' processesare only visible when running as root."""@@ -267,31 +290,37 @@ class LinuxCounters:return procs, adaptersdef _resolve_namespaces(self, procs: dict[int, dict[int, ProcGpu]]) -> None:- """Re-key NVML's PIDs to local ones. Same namespace (bare metal, or a container sharing the host- PID namespace): the PID exists here and holds an NVIDIA device. Otherwise (a container):- match by creation order with PidMatcher."""+ """Re-key NVML's PIDs to local ones. In the host PID namespace (bare metal, --pid=host) they already+ are local, including other users' processes whose /proc/<pid>/fd this monitor can't read. In a+ container's own namespace every NVML PID is a host PID, even one that happens to equal a local+ PID, so all of them are matched by creation order with PidMatcher."""self.outside = {}nv_pids = {pid for per in procs.values() for pid in per}- if not nv_pids:+ child = in_child_pid_namespace()+ if child is None: # unknown: PIDs that don't exist here can't be local+ unresolved = sorted(pid for pid in nv_pids if not os.path.exists(f"/proc/{pid}"))+ else:+ unresolved = sorted(nv_pids) if child else []+ if not unresolved:self._matcher.update([], [])returnholders = nvidia_holders()- unresolved = sorted(pid for pid in nv_pids if pid not in holders)- if not unresolved:- returnme = os.getpid()- local = sorted(p for p in holders - nv_pids if p != me and has_cuda(p))+ local = sorted(p for p in holders - (nv_pids - set(unresolved)) if p != me and has_cuda(p))compute = [pid for pid in unresolved if pid in self._compute_pids] # CUDA, so local libcuda usersmatched, _ = self._matcher.update(compute, local)+ unresolved = set(unresolved)for key, per in procs.items():+ moved: dict[int, ProcGpu] = {} # kept apart: a local PID may equal a host PID not yet movedfor pid in [p for p in per if p in unresolved]:g = per.pop(pid)if pid in matched:lpid, confidence = matched[pid]g.inferred_by = confidence- per[lpid] = g+ moved[lpid] = gelse:self.outside.setdefault(key, {})[pid] = g+ per.update(moved)def close(self) -> None:if self.ok:diff --git a/src/gputaskman/llm.py b/src/gputaskman/llm.pyindex 3b9a520..57430c6 100644--- a/src/gputaskman/llm.py+++ b/src/gputaskman/llm.py@@ -398,8 +398,8 @@ def build_view(d: ProcDetail, vram: float, shared: float, server: ServerInfo | Nif rest is None:if kv:v.segments.append(Segment(f"KV cache ({v.n_ctx:,} ctx, {v.kv_type}) est.", kv, "kv"))- v.notes.append("This process's VRAM total is unknown (WSL charges it to the VM on the Windows host); "- "segments are estimates from the model file and launch flags.")+ v.notes.append("This process's VRAM total is unknown (WSL charges it to the VM on the Windows host, "+ "or the driver hides per-process memory); segments are estimates from the model file and launch flags.")elif kv and kv <= rest:swa = main.sliding_window if main else Nonev.segments.append(Segment(f"KV cache ({v.n_ctx:,} ctx, {v.kv_type}) est.{' max' if swa else ''}", kv, "kv"))diff --git a/src/gputaskman/procinfo.py b/src/gputaskman/procinfo.pyindex 7f879c8..7d589bf 100644--- a/src/gputaskman/procinfo.py+++ b/src/gputaskman/procinfo.py@@ -225,7 +225,7 @@ def installed_models(exe: str, now: float | None = None) -> list[MappedModel]:hit = _installed.get(key)if hit and now - hit[0] < INSTALLED_TTL:return hit[1]- stem = os.path.splitext(os.path.basename(key))[0]+ stem = os.path.splitext(os.path.basename(key))[0].lower() # normcase only lowercases on Windowsroots = []if not _is_system_path(key):roots.append(os.path.dirname(exe))diff --git a/src/gputaskman/wslhost.py b/src/gputaskman/wslhost.pyindex ef8ebf9..c9372e4 100644--- a/src/gputaskman/wslhost.py+++ b/src/gputaskman/wslhost.py@@ -32,6 +32,7 @@ _POWERSHELL_FALLBACK = "/mnt/c/Windows/System32/WindowsPowerShell/v1.0/powershel# Counter paths are English (PdhAddEnglishCounter), so this works on localized Windows too._SCRIPT = r"""$ErrorActionPreference = 'Stop'+[Console]::OutputEncoding = New-Object System.Text.UTF8Encoding $false # not the OEM code pageAdd-Type -TypeDefinition @'using System;using System.Runtime.InteropServices;@@ -213,7 +214,7 @@ class HostStream:encoded = base64.b64encode(_SCRIPT.encode("utf-16-le")).decode("ascii")self._proc = popen([exe, "-NoProfile", "-NonInteractive", "-EncodedCommand", encoded],stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL,- text=True, bufsize=1)+ text=True, encoding="utf-8", errors="replace", bufsize=1)except (OSError, ValueError) as error:self.status = f"unavailable ({error})"self._ready.set()@@ -221,15 +222,23 @@ class HostStream:threading.Thread(target=self._read, name="wsl-host-counters", daemon=True).start()def _read(self) -> None:- for line in self._proc.stdout:- sample = parse_line(line)- if sample is not None and (sample.procs or sample.adapters):- self._latest = sample- self.status = "ok"- self._ready.set()- if self.status != "off":- self.status = "unavailable (counter stream ended)"- self._ready.set()+ try:+ for line in self._proc.stdout:+ try:+ sample = parse_line(line)+ except Exception: # one malformed line must not end the stream+ continue+ if sample is not None and (sample.procs or sample.adapters):+ self._latest = sample+ self.status = "ok"+ self._ready.set()+ except Exception:+ pass+ finally:+ # always report the end, so the collector restarts the stream instead of waiting forever+ if self.status != "off":+ self.status = "unavailable (counter stream ended)"+ self._ready.set()def wait(self, timeout: float) -> bool:return self._ready.wait(timeout)@@ -284,6 +293,7 @@ class _Entry:value: floatlabel: object = None # holder pid | FOREIGN | ("gone", pid): its holder exited, waiting to vanishsure: bool = False # established by an event or elimination, never by a guess+ same: bool = False # this sample only: all candidates are the same size, so the bytes are right either wayclass ContextTracker:@@ -411,10 +421,12 @@ class ContextTracker:for pid in start:if pid not in self.seen:self.seen[pid] = now+ self.gone.pop(pid, None) # reused PID: not the process that exitedif first:self.initial.add(pid)for pid in [p for p in self.seen if p not in start]:del self.seen[pid]+ self.initial.discard(pid) # a reused PID is a new process, not one that was running at startupself.gone[pid] = nowfor e in cur:if e.label in self.gone and not isinstance(e.label, tuple):@@ -464,13 +476,17 @@ class ContextTracker:if len(free) < len(unsure):reason = (f"{len(unsure)} Linux GPU process(es) without a VM context to match ""(one may not have created its GPU context yet)")- elif len(free) == len(unsure) == 1:+ # A process seen moments ago may not have created its context yet; the one free context could+ # then be a foreign one (WSLg, another distro), so wait for its own context instead of eliminating.+ recent = any(p not in self.initial and now - self.seen[p] <= self.PENDING for p in unsure)+ if len(free) == len(unsure) == 1 and not recent:free[0].label, free[0].sure = unsure[0], Trueelse:- same = len(free) == len(unsure) and all(abs(e.value - free[0].value) <= self._tol(free[0].value)- for e in free)+ same = (len(free) == len(unsure) > 1 and not recent+ and all(abs(e.value - free[0].value) <= self._tol(free[0].value) for e in free))for pid, e in zip(unsure, free): # newest process <-> newest context; leftovers are foreign- e.label, e.sure = pid, same+ # Equal sizes make the shown bytes right, not the owner: never carried as evidence.+ e.label, e.same = pid, sameself.entries = curforeign = [e.value for e in cur if e.label == FOREIGN and e.sure]@@ -478,6 +494,6 @@ class ContextTracker:changed = sorted(foreign) != sorted(self.cached_foreign)self.cached_foreign = foreignself._save(force=changed and not start)- out = {e.label: ContextGuess(e.value, "consistent" if e.sure else "order")+ out = {e.label: ContextGuess(e.value, "consistent" if e.sure or e.same else "order")for e in cur if e.label in start}return out, reasondiff --git a/tests/test_linux.py b/tests/test_linux.pyindex 39abb41..6b235ac 100644--- a/tests/test_linux.py+++ b/tests/test_linux.py@@ -103,9 +103,10 @@ class PdhImportTests(unittest.TestCase):class LinuxCountersTests(unittest.TestCase):def setUp(self):# Same PID namespace (bare metal): NVML's PIDs are local processes holding an NVIDIA device.- holders = patch.object(linux, "nvidia_holders", return_value={100, 101, 200, 300, 555})- holders.start()- self.addCleanup(holders.stop)+ for p in (patch.object(linux, "nvidia_holders", return_value={100, 101, 200, 300, 555}),+ patch.object(linux, "in_child_pid_namespace", return_value=False)):+ p.start()+ self.addCleanup(p.stop)def counters(self, fake, wsl):with patch.object(linux, "N", fake), patch.object(linux, "is_wsl", return_value=wsl):@@ -161,6 +162,32 @@ class LinuxCountersTests(unittest.TestCase):self.assertEqual([(a.luid, a.name, a.dedicated_vram) for a in adapters],[(KEY, "NVIDIA GeForce RTX 3090 Ti", 24 * GiB)])+ def test_a_failing_gpu_is_skipped_not_fatal(self):+ fake = FakeNvml()+ fake.nvmlDeviceGetCount = lambda: 2+ real = fake.nvmlDeviceGetHandleByIndex+ def handle(i):+ if i == 0:+ raise RuntimeError("NVML_ERROR_GPU_IS_LOST")+ return real(0)+ fake.nvmlDeviceGetHandleByIndex = handle+ with patch.object(linux, "N", fake):+ self.assertEqual(len(linux.enumerate_adapters()), 1)+ fake.nvmlDeviceGetCount = lambda: (_ for _ in ()).throw(RuntimeError("NVML_ERROR_UNKNOWN"))+ with patch.object(linux, "N", fake):+ self.assertEqual(linux.enumerate_adapters(), [])++ def test_other_users_processes_stay_local_on_bare_metal(self):+ # Without root, another user's /proc/<pid>/fd is unreadable, so it isn't an NVIDIA holder.+ fake = FakeNvml(compute=[proc(6000, 20 * GiB)])+ c = self.counters(fake, wsl=False)+ with patch.object(linux, "N", fake), patch.object(linux, "nvidia_holders", return_value={100}), \+ patch.object(linux, "has_cuda", return_value=True):+ procs, _ = c.sample()+ self.assertEqual((procs[KEY][6000].dedicated, procs[KEY][6000].inferred_by), (20 * GiB, ""))+ self.assertNotIn(100, procs[KEY])+ self.assertEqual(c.outside, {})+class ContainerTests(unittest.TestCase):"""Measured on a Vast.ai Docker container: NVML lists host PIDs (2029701...) while the processes are@@ -168,9 +195,12 @@ class ContainerTests(unittest.TestCase):def sample(self, c, fake, holders, cuda):with patch.object(linux, "N", fake), patch.object(linux, "nvidia_holders", return_value=set(holders)), \- patch.object(linux, "has_cuda", side_effect=lambda pid: pid in cuda):+ patch.object(linux, "has_cuda", side_effect=lambda pid: pid in cuda), \+ patch.object(linux, "in_child_pid_namespace", return_value=self.child):return c.sample()[0][KEY]+ child = True+def test_host_pids_are_matched_to_container_pids_by_creation_order(self):fake = FakeNvml(compute=[proc(2029701, GiB), proc(2029747, 2 * GiB)],graphics=[proc(1145, 58 << 20), proc(1360, 21 << 20)],@@ -199,10 +229,32 @@ class ContainerTests(unittest.TestCase):fake = FakeNvml(compute=[proc(547, GiB)])with patch.object(linux, "N", fake), patch.object(linux, "is_wsl", return_value=False):c = linux.LinuxCounters()+ self.child = Falseper = self.sample(c, fake, holders={547}, cuda={547})self.assertEqual((per[547].dedicated, per[547].inferred_by), (GiB, ""))self.assertEqual(c.outside, {})+ def test_a_host_pid_equal_to_a_local_pid_is_still_a_host_pid(self):+ # Host PIDs 100 and 200 are container PIDs 10 and 100: local PID 100 is not host PID 100.+ fake = FakeNvml(compute=[proc(100, GiB), proc(200, 4 * GiB)])+ with patch.object(linux, "N", fake), patch.object(linux, "is_wsl", return_value=False):+ c = linux.LinuxCounters()+ per = self.sample(c, fake, holders={10, 100}, cuda={10, 100})+ self.assertEqual({pid: (g.dedicated, g.inferred_by) for pid, g in per.items()},+ {10: (GiB, "consistent"), 100: (4 * GiB, "consistent")})++ def test_namespace_detection(self):+ linux.in_child_pid_namespace.cache_clear()+ self.addCleanup(linux.in_child_pid_namespace.cache_clear)+ with patch.object(linux.os, "readlink", return_value="pid:[4026531836]"):+ self.assertFalse(linux.in_child_pid_namespace())+ linux.in_child_pid_namespace.cache_clear()+ with patch.object(linux.os, "readlink", return_value="pid:[4026532481]"):+ self.assertTrue(linux.in_child_pid_namespace())+ linux.in_child_pid_namespace.cache_clear()+ with patch.object(linux.os, "readlink", side_effect=OSError):+ self.assertIsNone(linux.in_child_pid_namespace())+def test_outside_rows_never_borrow_a_local_identity(self):from gputaskman.collector import ProcRow as Rowrow = Row(1145, "(outside this container)", "external", ProcGpu(dedicated=58 << 20),diff --git a/tests/test_wslhost.py b/tests/test_wslhost.pyindex ed52ded..dcc9a38 100644--- a/tests/test_wslhost.py+++ b/tests/test_wslhost.py@@ -99,6 +99,27 @@ class InteropTests(unittest.TestCase):self.assertIsNotNone(h.latest())self.assertEqual(calls[0][:4], ["powershell.exe", "-NoProfile", "-NonInteractive", "-EncodedCommand"])+ def test_a_bad_byte_ends_the_stream_visibly_so_it_restarts(self):+ class BadStdout:+ def __iter__(self):+ yield SAMPLE + "\n"+ raise UnicodeDecodeError("utf-8", b"\x82", 0, 1, "invalid start byte")++ class FakeProc:+ def __init__(self, args, **kw):+ self.encoding = kw.get("encoding")+ self.stdout = BadStdout()++ def poll(self):+ return 0++ h = HostStream(exe="powershell.exe", popen=FakeProc)+ deadline = time.time() + 2+ while not h.ended and time.time() < deadline:+ time.sleep(0.01)+ self.assertTrue(h.ended)+ self.assertEqual(h._proc.encoding, "utf-8")+def test_missing_powershell_is_reported(self):with patch("gputaskman.wslhost.powershell_path", return_value=None):h = HostStream()@@ -147,6 +168,28 @@ class TrackerTests(unittest.TestCase):h.remove(("A", 950)) # the oldest job exits; the others keep their contextsself.check(self.step(t, [3.05, 0.4, 0.0, 0.0], h), {"F1": (0.4, "*"), "B": (3.05, "*")})+ def test_equal_sizes_are_right_now_but_never_evidence_later(self):+ t = self.make()+ h = [("S", 900), ("A", 13), ("B", 14)]+ self.check(self.step(t, [0.0, 0.2, 0.2], h), {"B": (0.2, "*"), "A": (0.2, "*")})+ # one of them loads a model: which one is unknown, so neither keeps the "*"+ self.check(self.step(t, [0.0, 4.0, 0.2], h), {"B": (4.0, "?"), "A": (0.2, "?")})++ def test_new_process_waits_for_its_own_context_instead_of_eliminating(self):+ t = self.make()+ h = [("S", 900)]+ self.check(self.step(t, [0.0, 0.15], h), {}) # 0.15: WSLg, not yet known as foreign+ h.append(("A", 1001))+ self.check(self.step(t, [0.0, 0.15], h), {"A": (0.15, "?")}) # A's context isn't there yet+ self.check(self.step(t, [6.0, 0.0, 0.15], h), {"A": (6.0, "*")})++ def test_a_reused_pid_is_a_new_process(self):+ t = self.make()+ self.check(self.step(t, [0.0, 1.0], [("S", 900), ("P", 10)]), {"P": (1.0, "*")})+ self.check(self.step(t, [0.0], [("S", 900)]), {})+ self.check(self.step(t, [2.0, 0.0], [("S", 900), ("P", 2000)]), {"P": (2.0, "*")})+ self.check(self.step(t, [2.0, 0.0], [("S", 900), ("P", 2000)]), {"P": (2.0, "*")})+def test_already_running_processes_are_guesses_until_evidence(self):t = self.make()h = [("S", 900), ("C", 14), ("B", 13)]@@ -238,6 +281,7 @@ class HostRowTests(unittest.TestCase):c._host_restarted = time.time()c.tracker = ContextTracker(path=os.path.join(self.tmp.name, "ctx.json"), boot_id="boot", self_pid=os.getpid())c._own_start = time.time()+ c._inferred = {}return cdef setUp(self):