A GPU task manager for Windows, Linux, and WSL: per-process VRAM, GPU activity, and LLM model/KV-cache insights (Windows AMD and NVIDIA).
Python
Stop faulting NVML reads and hanging shutdown after GPU re-enable
Scott Hanselmane157c218e961
diff --git a/README.md b/README.mdindex ac10dba..af6361a 100644--- a/README.md+++ b/README.md@@ -453,7 +453,14 @@ failures back off to at most one reconstruction per minute; a reappearing or chaDiscovery errors retain the last known topology with a warning, not invented zero usage.Pausing suspends discovery as well as sampling; resume performs an overdue check.This recovery is covered by mocks, **not validated with hardware disable/re-enable** on the-reporter's hybrid AMD/NVIDIA laptop or locally. It is not a claim of physical hot-swap support.+reporter's hybrid AMD/NVIDIA laptop. On a three-GPU desktop (RTX 5060 Ti, RTX 3060, AMD iGPU;+Windows 11 build 26200, NVIDIA driver 610.88), disabling and re-enabling the RTX 3060 recovered+its slot, history, VRAM and processes, but NVML reads for that GPU then faulted inside `nvml.dll`+and `nvmlShutdown()` hung on exit. A fresh process read it normally. gputaskman therefore stops+reading NVML for a GPU whose reads fault, shows "restart gputaskman to restore it", and skips+`nvmlShutdown()` once an NVIDIA adapter has gone away (process exit releases the session). That+mitigation is covered by mocks and not yet validated on that hardware. It is not a claim of+physical hot-swap support.Other backends and one-shot output retain startup-only discovery.Under multi-GPU WSL, host attribution is disabled with a reason rather than risking matchingtwo devices by memory usage; Linux processes are listed once with GPU `?`, because `/dev/dxg`diff --git a/src/gputaskman/collector.py b/src/gputaskman/collector.pyindex 4daa530..ea15405 100644--- a/src/gputaskman/collector.py+++ b/src/gputaskman/collector.py@@ -15,6 +15,7 @@ from .adl import Adlfrom .dxgi import VENDOR_NVIDIA, Adapterfrom .handy import HandyEvidence, inspect_handyfrom .llm import LlmView, ServerProber, build_view, gb, ollama_log_path+from . import nvml as nvml_modfrom .nvml import Nvml, NvDevicefrom .pdh import ProcGpufrom .procinfo import AI_KINDS, ProcDetail, ProcInspector, classify@@ -122,7 +123,7 @@ class Snapshot:shared_is_spill: bool = Truecounter_status: str = "" # Windows counter availability or adapter matching problemcounter_adapter_matched: bool | None = None # Windows PDH LUID matched the selected DXGI adapter-+ nv_lost: bool = False # NVML faulted for this GPU after a disable/re-enable; restart restores it@propertydef host_data(self) -> bool:return self.host_status.startswith("ok")@@ -285,6 +286,12 @@ class Collector:elif a is None or a.dedicated is None or a.shared is None:snap.counter_status = ("Some Windows adapter memory counters are unavailable; ""GPU-PV VMs may not expose them.")+ if (nv is None and self.adapter.vendor_id == VENDOR_NVIDIA and self.adapter.pci_address+ and any(_bdf(b) == self.adapter.pci_address for b in nvml_mod.LOST)):+ snap.nv_lost = True+ snap.counter_status = " ".join(filter(None, (+ snap.counter_status, "NVIDIA telemetry stopped after this GPU was re-enabled; "+ "restart gputaskman to restore it.")))if self.drm_card is not None:snap.process_status = self.counters.drm.statuslive = [(pid, g) for pid, g in procs.items() if _is_live(g)]diff --git a/src/gputaskman/nvml.py b/src/gputaskman/nvml.pyindex 00b6b46..65f9738 100644--- a/src/gputaskman/nvml.py+++ b/src/gputaskman/nvml.py@@ -24,6 +24,17 @@ _THROTTLE = {}_RESET_ERRORS = (1, 9, 15) # uninitialized, driver not loaded, GPU lost+# NVML is one session per process. After Windows disables and re-enables a GPU, reads for that GPU+# can fault inside nvml.dll and nvmlShutdown() can hang (reported on an RTX 3060, driver 610.88).+LOST: set[str] = set() # PCI bus IDs whose reads faulted; never read again in this process+_unsafe = False+++def mark_unsafe() -> None:+ """An NVIDIA adapter went away: skip nvmlShutdown() from now on and let process exit clean up."""+ global _unsafe+ _unsafe = True+def _try(fn, *a, default=None):try:@@ -39,6 +50,7 @@ class Nvml:self.cuda_version = ""self.handles = []self.needs_reset = False+ self._faulted = Falseself.refresh()def refresh(self) -> None:@@ -78,6 +90,9 @@ class Nvml:def _read(self, fn, *args, default=None):try:return fn(*args)+ except OSError: # access violation inside nvml.dll, not an NVML error code+ self._faulted = True+ return defaultexcept Exception as error:if getattr(error, "value", None) in _RESET_ERRORS:self.needs_reset = True@@ -86,11 +101,14 @@ class Nvml:def sample(self) -> list[NvDevice]:out = []for i, h in self.handles:+ self._faulted = Falsed = NvDevice(index=i)d.name = self._read(N.nvmlDeviceGetName, h, default="?")pci = self._read(N.nvmlDeviceGetPciInfo, h)if pci is not None:d.pci_bus = pci.busId.decode() if isinstance(pci.busId, bytes) else str(pci.busId)+ if d.pci_bus in LOST or self._faulted:+ continuedm = self._read(N.nvmlDeviceGetDriverModel, h)if dm:d.driver_model = {0: "WDDM", 1: "TCC", 2: "MCDM"}.get(dm[0], str(dm[0]))@@ -128,10 +146,16 @@ class Nvml:cc = self._read(N.nvmlDeviceGetCudaComputeCapability, h)if cc:d.compute_cap = f"{cc[0]}.{cc[1]}"+ if self._faulted:+ if d.pci_bus:+ LOST.add(d.pci_bus)+ mark_unsafe()+ continueout.append(d)return outdef close(self) -> None:if self.ok:- _try(N.nvmlShutdown)+ if not _unsafe:+ _try(N.nvmlShutdown)self.ok = Falsediff --git a/src/gputaskman/recovery.py b/src/gputaskman/recovery.pyindex 6b3211d..2e05702 100644--- a/src/gputaskman/recovery.py+++ b/src/gputaskman/recovery.py@@ -5,8 +5,9 @@ from __future__ import annotationsimport timefrom dataclasses import dataclass-from .collector import Collector, Snapshot, ordered_adapters, select_adapter+from .collector import VENDOR_NVIDIA, Collector, Snapshot, ordered_adapters, select_adapterfrom .dxgi import Adapter, enumerate_adapters+from .nvml import mark_unsafe@dataclass@@ -80,6 +81,8 @@ class LiveCollectors:remaining = list(adapters)for slot in self.slots:old = slot.adapter+ if old is not None and old.vendor_id == VENDOR_NVIDIA and all(a.luid != old.luid for a in adapters):+ mark_unsafe()match = next((a for a in remaining if old and a.luid == old.luid), None)if match is None and old is not None:location = _location(old)@@ -144,7 +147,7 @@ class LiveCollectors:try:snap = slot.collector.sample()slot.retry = (snap.counter_adapter_matched is False- or (slot.had_telemetry and snap.nv is None))+ or (slot.had_telemetry and snap.nv is None and not snap.nv_lost))slot.had_telemetry |= snap.nv is not Noneif not slot.retry:slot.retry_after = slot.retry_delay = 0.0diff --git a/tests/test_core.py b/tests/test_core.pyindex 1d86de6..a62ea8a 100644--- a/tests/test_core.py+++ b/tests/test_core.py@@ -250,6 +250,25 @@ class MemoryTests(unittest.TestCase):Console(file=output, width=160).print(GpuTaskManager().render_device(s))self.assertIn("could not be matched", output.getvalue())+ def test_lost_nvml_telemetry_is_explained_not_silent(self):+ c = Collector.__new__(Collector)+ c.adapter = Adapter(1, "RTX 3060", 0x10DE, 12 * GiB, 0, False, (5, 0, 0))+ c.adapters = [c.adapter]+ c.gpu_index, c.scan_budget, c.host, c.host_status = 0, None, None, ""+ c.nvml = Mock(ok=True, driver="", cuda_version="")+ c.nvml.sample.return_value = []+ c.counters = Mock(outside={})+ c.counters.sample.return_value = ({1: {}}, {1: AdapterGpu(dedicated=GiB, shared=0)})+ c.inspector, c.prober = Mock(), Mock()+ with patch("gputaskman.collector.BACKEND", "wddm"), \+ patch("gputaskman.collector.sys.platform", "win32"), \+ patch("gputaskman.nvml.LOST", {"00000000:05:00.0"}):+ s = c.sample()+ self.assertIsNone(s.nv)+ self.assertTrue(s.nv_lost)+ self.assertIn("restart gputaskman", s.counter_status)+ self.assertEqual(s.vram_used, GiB)+def test_windows_matching_counter_zero_is_not_unavailable(self):c = Collector.__new__(Collector)c.adapter = Adapter(1, "GPU", 0x10DE, 16 * GiB, 0, False)diff --git a/tests/test_recovery.py b/tests/test_recovery.pyindex 738923c..4a27ebb 100644--- a/tests/test_recovery.py+++ b/tests/test_recovery.py@@ -4,6 +4,7 @@ from types import SimpleNamespacefrom unittest import IsolatedAsyncioTestCase, TestCasefrom unittest.mock import Mock, patch+from gputaskman import nvmlfrom gputaskman.app import GpuTaskManagerfrom gputaskman.collector import Collector, ProcRow, Snapshot, select_nv_devicefrom gputaskman.dxgi import Adapter, enumerate_adapters@@ -28,6 +29,10 @@ def snapshot(a, value=10):class RecoveryTests(TestCase):def setUp(self):+ for name, value in (("_unsafe", False), ("LOST", set())):+ patcher = patch(f"gputaskman.nvml.{name}", value)+ patcher.start()+ self.addCleanup(patcher.stop)self.first, self.second = adapter(1), adapter(2, 0x1002, 2)self.topology = [self.first, self.second]self.created = []@@ -71,6 +76,27 @@ class RecoveryTests(TestCase):self.assertEqual(len(self.created), 4)self.assertAlmostEqual(survivor.scan_budget, Collector.SCAN_BUDGET / 3)+ def test_nvidia_adapter_loss_marks_nvml_unsafe_and_lost_telemetry_does_not_rebuild(self):+ live = self.live()+ self.topology = [self.second]+ self.rediscover(live)+ self.assertTrue(nvml._unsafe)+ readded = adapter(11)+ self.topology = [readded, self.second]+ self.rediscover(live)+ lost = snapshot(readded)+ lost.nv, lost.nv_lost = None, True+ live.slots[0].had_telemetry = True+ self.created[-1].sample.side_effect = lambda: lost+ live.sample(now=live.next_discovery - 1)+ self.assertFalse(live.slots[0].retry)++ def test_amd_adapter_loss_keeps_nvml_shutdown(self):+ live = self.live()+ self.topology = [self.first]+ self.rediscover(live)+ self.assertFalse(nvml._unsafe)+def test_explicit_gpu_pins_original_identity_not_current_index(self):live = self.live(1)self.topology = [self.first]@@ -264,6 +290,12 @@ class RecoveryTests(TestCase):class NvmlRecoveryTests(TestCase):+ def setUp(self):+ for name, value in (("_unsafe", False), ("LOST", set())):+ patcher = patch(f"gputaskman.nvml.{name}", value)+ patcher.start()+ self.addCleanup(patcher.stop)+def api(self):return SimpleNamespace(nvmlInit=Mock(), nvmlShutdown=Mock(), nvmlDeviceGetCount=Mock(return_value=2),@@ -439,6 +471,36 @@ class NvmlRecoveryTests(TestCase):self.assertFalse(nv.needs_reset)nv.close()+ def test_access_violation_stops_reading_that_gpu_and_skips_hanging_shutdown(self):+ class Api(SimpleNamespace):+ def __getattr__(self, name):+ return 0 if name.startswith("NVML_") else lambda *args: None++ def temperature(handle, _kind):+ if handle == "second":+ raise OSError("exception: access violation reading 0x20C25D65908")+ return 40++ api = Api(**vars(self.api()))+ api.nvmlDeviceGetPciInfo = Mock(side_effect=lambda h: SimpleNamespace(+ busId=b"00000000:01:00.0" if h == "first" else b"00000000:05:00.0"))+ api.nvmlDeviceGetTemperature = Mock(side_effect=temperature)+ with patch("gputaskman.nvml.N", api):+ nv = Nvml()+ self.assertEqual([(d.pci_bus, d.temp) for d in nv.sample()], [("00000000:01:00.0", 40)])+ self.assertEqual(nvml.LOST, {"00000000:05:00.0"})+ calls = api.nvmlDeviceGetTemperature.call_count+ self.assertEqual(len(nv.sample()), 1)+ self.assertEqual(api.nvmlDeviceGetTemperature.call_count, calls + 1) # the lost GPU is skipped+ nv.close()+ api.nvmlShutdown.assert_not_called()++ def test_healthy_session_still_shuts_down(self):+ api = self.api()+ with patch("gputaskman.nvml.N", api):+ Nvml().close()+ api.nvmlShutdown.assert_called_once()+class DiscoveryTests(TestCase):def test_discovery_error_is_not_confused_with_empty_topology(self):