Skip to content

A GPU task manager for Windows, Linux, and WSL: per-process VRAM, GPU activity, and LLM model/KV-cache insights (Windows AMD and NVIDIA).

Python

Stop faulting NVML reads and hanging shutdown after GPU re-enable

Scott Hanselmane157c218e961
diff --git a/README.md b/README.md
index ac10dba..af6361a 100644
--- a/README.md
+++ b/README.md
@@ -453,7 +453,14 @@ failures back off to at most one reconstruction per minute; a reappearing or cha
Discovery errors retain the last known topology with a warning, not invented zero usage.
Pausing suspends discovery as well as sampling; resume performs an overdue check.
This recovery is covered by mocks, **not validated with hardware disable/re-enable** on the
-reporter's hybrid AMD/NVIDIA laptop or locally. It is not a claim of physical hot-swap support.
+reporter's hybrid AMD/NVIDIA laptop. On a three-GPU desktop (RTX 5060 Ti, RTX 3060, AMD iGPU;
+Windows 11 build 26200, NVIDIA driver 610.88), disabling and re-enabling the RTX 3060 recovered
+its slot, history, VRAM and processes, but NVML reads for that GPU then faulted inside `nvml.dll`
+and `nvmlShutdown()` hung on exit. A fresh process read it normally. gputaskman therefore stops
+reading NVML for a GPU whose reads fault, shows "restart gputaskman to restore it", and skips
+`nvmlShutdown()` once an NVIDIA adapter has gone away (process exit releases the session). That
+mitigation is covered by mocks and not yet validated on that hardware. It is not a claim of
+physical hot-swap support.
Other backends and one-shot output retain startup-only discovery.
Under multi-GPU WSL, host attribution is disabled with a reason rather than risking matching
two devices by memory usage; Linux processes are listed once with GPU `?`, because `/dev/dxg`
diff --git a/src/gputaskman/collector.py b/src/gputaskman/collector.py
index 4daa530..ea15405 100644
--- a/src/gputaskman/collector.py
+++ b/src/gputaskman/collector.py
@@ -15,6 +15,7 @@ from .adl import Adl
from .dxgi import VENDOR_NVIDIA, Adapter
from .handy import HandyEvidence, inspect_handy
from .llm import LlmView, ServerProber, build_view, gb, ollama_log_path
+from . import nvml as nvml_mod
from .nvml import Nvml, NvDevice
from .pdh import ProcGpu
from .procinfo import AI_KINDS, ProcDetail, ProcInspector, classify
@@ -122,7 +123,7 @@ class Snapshot:
shared_is_spill: bool = True
counter_status: str = "" # Windows counter availability or adapter matching problem
counter_adapter_matched: bool | None = None # Windows PDH LUID matched the selected DXGI adapter
-
+ nv_lost: bool = False # NVML faulted for this GPU after a disable/re-enable; restart restores it
@property
def host_data(self) -> bool:
return self.host_status.startswith("ok")
@@ -285,6 +286,12 @@ class Collector:
elif a is None or a.dedicated is None or a.shared is None:
snap.counter_status = ("Some Windows adapter memory counters are unavailable; "
"GPU-PV VMs may not expose them.")
+ if (nv is None and self.adapter.vendor_id == VENDOR_NVIDIA and self.adapter.pci_address
+ and any(_bdf(b) == self.adapter.pci_address for b in nvml_mod.LOST)):
+ snap.nv_lost = True
+ snap.counter_status = " ".join(filter(None, (
+ snap.counter_status, "NVIDIA telemetry stopped after this GPU was re-enabled; "
+ "restart gputaskman to restore it.")))
if self.drm_card is not None:
snap.process_status = self.counters.drm.status
live = [(pid, g) for pid, g in procs.items() if _is_live(g)]
diff --git a/src/gputaskman/nvml.py b/src/gputaskman/nvml.py
index 00b6b46..65f9738 100644
--- a/src/gputaskman/nvml.py
+++ b/src/gputaskman/nvml.py
@@ -24,6 +24,17 @@ _THROTTLE = {
}
_RESET_ERRORS = (1, 9, 15) # uninitialized, driver not loaded, GPU lost
+# NVML is one session per process. After Windows disables and re-enables a GPU, reads for that GPU
+# can fault inside nvml.dll and nvmlShutdown() can hang (reported on an RTX 3060, driver 610.88).
+LOST: set[str] = set() # PCI bus IDs whose reads faulted; never read again in this process
+_unsafe = False
+
+
+def mark_unsafe() -> None:
+ """An NVIDIA adapter went away: skip nvmlShutdown() from now on and let process exit clean up."""
+ global _unsafe
+ _unsafe = True
+
def _try(fn, *a, default=None):
try:
@@ -39,6 +50,7 @@ class Nvml:
self.cuda_version = ""
self.handles = []
self.needs_reset = False
+ self._faulted = False
self.refresh()
def refresh(self) -> None:
@@ -78,6 +90,9 @@ class Nvml:
def _read(self, fn, *args, default=None):
try:
return fn(*args)
+ except OSError: # access violation inside nvml.dll, not an NVML error code
+ self._faulted = True
+ return default
except Exception as error:
if getattr(error, "value", None) in _RESET_ERRORS:
self.needs_reset = True
@@ -86,11 +101,14 @@ class Nvml:
def sample(self) -> list[NvDevice]:
out = []
for i, h in self.handles:
+ self._faulted = False
d = NvDevice(index=i)
d.name = self._read(N.nvmlDeviceGetName, h, default="?")
pci = self._read(N.nvmlDeviceGetPciInfo, h)
if pci is not None:
d.pci_bus = pci.busId.decode() if isinstance(pci.busId, bytes) else str(pci.busId)
+ if d.pci_bus in LOST or self._faulted:
+ continue
dm = self._read(N.nvmlDeviceGetDriverModel, h)
if dm:
d.driver_model = {0: "WDDM", 1: "TCC", 2: "MCDM"}.get(dm[0], str(dm[0]))
@@ -128,10 +146,16 @@ class Nvml:
cc = self._read(N.nvmlDeviceGetCudaComputeCapability, h)
if cc:
d.compute_cap = f"{cc[0]}.{cc[1]}"
+ if self._faulted:
+ if d.pci_bus:
+ LOST.add(d.pci_bus)
+ mark_unsafe()
+ continue
out.append(d)
return out
def close(self) -> None:
if self.ok:
- _try(N.nvmlShutdown)
+ if not _unsafe:
+ _try(N.nvmlShutdown)
self.ok = False
diff --git a/src/gputaskman/recovery.py b/src/gputaskman/recovery.py
index 6b3211d..2e05702 100644
--- a/src/gputaskman/recovery.py
+++ b/src/gputaskman/recovery.py
@@ -5,8 +5,9 @@ from __future__ import annotations
import time
from dataclasses import dataclass
-from .collector import Collector, Snapshot, ordered_adapters, select_adapter
+from .collector import VENDOR_NVIDIA, Collector, Snapshot, ordered_adapters, select_adapter
from .dxgi import Adapter, enumerate_adapters
+from .nvml import mark_unsafe
@dataclass
@@ -80,6 +81,8 @@ class LiveCollectors:
remaining = list(adapters)
for slot in self.slots:
old = slot.adapter
+ if old is not None and old.vendor_id == VENDOR_NVIDIA and all(a.luid != old.luid for a in adapters):
+ mark_unsafe()
match = next((a for a in remaining if old and a.luid == old.luid), None)
if match is None and old is not None:
location = _location(old)
@@ -144,7 +147,7 @@ class LiveCollectors:
try:
snap = slot.collector.sample()
slot.retry = (snap.counter_adapter_matched is False
- or (slot.had_telemetry and snap.nv is None))
+ or (slot.had_telemetry and snap.nv is None and not snap.nv_lost))
slot.had_telemetry |= snap.nv is not None
if not slot.retry:
slot.retry_after = slot.retry_delay = 0.0
diff --git a/tests/test_core.py b/tests/test_core.py
index 1d86de6..a62ea8a 100644
--- a/tests/test_core.py
+++ b/tests/test_core.py
@@ -250,6 +250,25 @@ class MemoryTests(unittest.TestCase):
Console(file=output, width=160).print(GpuTaskManager().render_device(s))
self.assertIn("could not be matched", output.getvalue())
+ def test_lost_nvml_telemetry_is_explained_not_silent(self):
+ c = Collector.__new__(Collector)
+ c.adapter = Adapter(1, "RTX 3060", 0x10DE, 12 * GiB, 0, False, (5, 0, 0))
+ c.adapters = [c.adapter]
+ c.gpu_index, c.scan_budget, c.host, c.host_status = 0, None, None, ""
+ c.nvml = Mock(ok=True, driver="", cuda_version="")
+ c.nvml.sample.return_value = []
+ c.counters = Mock(outside={})
+ c.counters.sample.return_value = ({1: {}}, {1: AdapterGpu(dedicated=GiB, shared=0)})
+ c.inspector, c.prober = Mock(), Mock()
+ with patch("gputaskman.collector.BACKEND", "wddm"), \
+ patch("gputaskman.collector.sys.platform", "win32"), \
+ patch("gputaskman.nvml.LOST", {"00000000:05:00.0"}):
+ s = c.sample()
+ self.assertIsNone(s.nv)
+ self.assertTrue(s.nv_lost)
+ self.assertIn("restart gputaskman", s.counter_status)
+ self.assertEqual(s.vram_used, GiB)
+
def test_windows_matching_counter_zero_is_not_unavailable(self):
c = Collector.__new__(Collector)
c.adapter = Adapter(1, "GPU", 0x10DE, 16 * GiB, 0, False)
diff --git a/tests/test_recovery.py b/tests/test_recovery.py
index 738923c..4a27ebb 100644
--- a/tests/test_recovery.py
+++ b/tests/test_recovery.py
@@ -4,6 +4,7 @@ from types import SimpleNamespace
from unittest import IsolatedAsyncioTestCase, TestCase
from unittest.mock import Mock, patch
+from gputaskman import nvml
from gputaskman.app import GpuTaskManager
from gputaskman.collector import Collector, ProcRow, Snapshot, select_nv_device
from gputaskman.dxgi import Adapter, enumerate_adapters
@@ -28,6 +29,10 @@ def snapshot(a, value=10):
class RecoveryTests(TestCase):
def setUp(self):
+ for name, value in (("_unsafe", False), ("LOST", set())):
+ patcher = patch(f"gputaskman.nvml.{name}", value)
+ patcher.start()
+ self.addCleanup(patcher.stop)
self.first, self.second = adapter(1), adapter(2, 0x1002, 2)
self.topology = [self.first, self.second]
self.created = []
@@ -71,6 +76,27 @@ class RecoveryTests(TestCase):
self.assertEqual(len(self.created), 4)
self.assertAlmostEqual(survivor.scan_budget, Collector.SCAN_BUDGET / 3)
+ def test_nvidia_adapter_loss_marks_nvml_unsafe_and_lost_telemetry_does_not_rebuild(self):
+ live = self.live()
+ self.topology = [self.second]
+ self.rediscover(live)
+ self.assertTrue(nvml._unsafe)
+ readded = adapter(11)
+ self.topology = [readded, self.second]
+ self.rediscover(live)
+ lost = snapshot(readded)
+ lost.nv, lost.nv_lost = None, True
+ live.slots[0].had_telemetry = True
+ self.created[-1].sample.side_effect = lambda: lost
+ live.sample(now=live.next_discovery - 1)
+ self.assertFalse(live.slots[0].retry)
+
+ def test_amd_adapter_loss_keeps_nvml_shutdown(self):
+ live = self.live()
+ self.topology = [self.first]
+ self.rediscover(live)
+ self.assertFalse(nvml._unsafe)
+
def test_explicit_gpu_pins_original_identity_not_current_index(self):
live = self.live(1)
self.topology = [self.first]
@@ -264,6 +290,12 @@ class RecoveryTests(TestCase):
class NvmlRecoveryTests(TestCase):
+ def setUp(self):
+ for name, value in (("_unsafe", False), ("LOST", set())):
+ patcher = patch(f"gputaskman.nvml.{name}", value)
+ patcher.start()
+ self.addCleanup(patcher.stop)
+
def api(self):
return SimpleNamespace(
nvmlInit=Mock(), nvmlShutdown=Mock(), nvmlDeviceGetCount=Mock(return_value=2),
@@ -439,6 +471,36 @@ class NvmlRecoveryTests(TestCase):
self.assertFalse(nv.needs_reset)
nv.close()
+ def test_access_violation_stops_reading_that_gpu_and_skips_hanging_shutdown(self):
+ class Api(SimpleNamespace):
+ def __getattr__(self, name):
+ return 0 if name.startswith("NVML_") else lambda *args: None
+
+ def temperature(handle, _kind):
+ if handle == "second":
+ raise OSError("exception: access violation reading 0x20C25D65908")
+ return 40
+
+ api = Api(**vars(self.api()))
+ api.nvmlDeviceGetPciInfo = Mock(side_effect=lambda h: SimpleNamespace(
+ busId=b"00000000:01:00.0" if h == "first" else b"00000000:05:00.0"))
+ api.nvmlDeviceGetTemperature = Mock(side_effect=temperature)
+ with patch("gputaskman.nvml.N", api):
+ nv = Nvml()
+ self.assertEqual([(d.pci_bus, d.temp) for d in nv.sample()], [("00000000:01:00.0", 40)])
+ self.assertEqual(nvml.LOST, {"00000000:05:00.0"})
+ calls = api.nvmlDeviceGetTemperature.call_count
+ self.assertEqual(len(nv.sample()), 1)
+ self.assertEqual(api.nvmlDeviceGetTemperature.call_count, calls + 1) # the lost GPU is skipped
+ nv.close()
+ api.nvmlShutdown.assert_not_called()
+
+ def test_healthy_session_still_shuts_down(self):
+ api = self.api()
+ with patch("gputaskman.nvml.N", api):
+ Nvml().close()
+ api.nvmlShutdown.assert_called_once()
+
class DiscoveryTests(TestCase):
def test_discovery_error_is_not_confused_with_empty_topology(self):