Skip to content

A GPU task manager for Windows, Linux, and WSL: per-process VRAM, GPU activity, and LLM model/KV-cache insights (Windows AMD and NVIDIA).

Python

Recover GPUs after Windows adapter changes

Scott Hanselmand1d0db5203f1
diff --git a/README.md b/README.md
index 844aee8..ac10dba 100644
--- a/README.md
+++ b/README.md
@@ -437,6 +437,24 @@ Memory totals and histories stay separate per device; model files and runtime bu
can span devices and do not prove per-device model placement. Discovery's 150 ms scan budget
is divided across GPUs, though each GPU has its own counter and telemetry queries.
This layout is covered by synthetic tests, **not yet validated on multi-GPU hardware**.
+On Windows, the live TUI checks DXGI adapter topology every five seconds in the existing
+sampling worker (or on the next sample with a longer interval). Removed adapters retain their
+session GPU number with unavailable readings; surviving adapters keep their collectors and
+histories, and newly discovered adapters are appended. Re-enabling an adapter rebuilds its
+providers. LUIDs identify adapters; if a LUID changes, only a unique PCI location **and**
+matching vendor/description can reconnect the old slot. Without that evidence, the old slot
+remains unavailable rather than transferring its history or process ownership to another GPU.
+`--gpu N` pins the adapter selected at startup, not whichever adapter later occupies index N;
+it does not start monitoring other newly discovered devices. Session GPU numbers therefore
+need not match a fresh `--list-gpus` after a topology change.
+NVML handles are reacquired at the same low-frequency cadence; missing counters, lost telemetry
+or sampling errors trigger provider reconstruction at the next successful discovery. Persistent
+failures back off to at most one reconstruction per minute; a reappearing or changed adapter retries immediately.
+Discovery errors retain the last known topology with a warning, not invented zero usage.
+Pausing suspends discovery as well as sampling; resume performs an overdue check.
+This recovery is covered by mocks, **not validated with hardware disable/re-enable** on the
+reporter's hybrid AMD/NVIDIA laptop or locally. It is not a claim of physical hot-swap support.
+Other backends and one-shot output retain startup-only discovery.
Under multi-GPU WSL, host attribution is disabled with a reason rather than risking matching
two devices by memory usage; Linux processes are listed once with GPU `?`, because `/dev/dxg`
does not identify their GPU. `--gpu N` retains the existing single-device host matching.
@@ -648,6 +666,8 @@ Identification is designed to stay cheap. Measured on one Windows machine:
| Probe | Cost | Used |
|---|---|---|
| Process list, command line, listening ports | under 1 ms | Once per process / every 5 s |
+| Windows adapter discovery (DXGI + PCI location, no telemetry sample) | ~0.40 ms median; first/cold call up to 14.1 ms in 30 reads (Windows 11 build 26340, RTX 4080 SUPER, NVIDIA 630.20) | TUI only, every 5 s; synchronous OS calls, not a hard latency bound. Other drivers/multi-GPU laptops unmeasured; provider reconstruction after changes/errors has additional driver-dependent cost |
+| Windows NVML handle reacquisition (no telemetry sample) | ~8.7 ms median, 9.3 ms max in 30 reads on the same machine | Every 5 s per active collector; retries initialization after NVML session-loss errors. Other drivers unmeasured |
| Memory-map scan (DLLs + mapped files) | 1-50 ms per process | New processes, a 64 MiB / 10% memory change, or every 2 min; at most 150 ms per refresh |
| GGUF header, model-store manifests | ms, cached | Once per file / every 60 s |
| Local API probes | ms, local only | llama-server `/v1/models` only when its model path changes |
diff --git a/src/gputaskman/app.py b/src/gputaskman/app.py
index c8e68e3..83054c1 100644
--- a/src/gputaskman/app.py
+++ b/src/gputaskman/app.py
@@ -20,6 +20,7 @@ from textual.worker import get_current_worker
from .collector import BACKEND, Collector, ProcRow, Snapshot, enumerate_adapters, ordered_adapters
from .llm import gb
from .nvml import NvDevice, format_metric
+from .recovery import LiveCollectors
from .report import (DRM_RESIDENT_NOTE, METAL_NOTE, WSL_EMPTY, drm_note, has_engine_counters, has_spill_counters, has_wddm_counters,
inferred_note, vram_label, wsl_note)
@@ -274,6 +275,19 @@ class GpuTaskManager(App):
@work(thread=True, exclusive=True)
def poll(self) -> None:
worker = get_current_worker()
+ if BACKEND == "wddm":
+ live = LiveCollectors(self.gpu_index, collector_factory=Collector, discover=enumerate_adapters)
+ try:
+ worker.cancelled_event.wait(0.5)
+ while not worker.is_cancelled:
+ if not self.paused:
+ snapshots = live.sample()
+ if not worker.is_cancelled:
+ self.call_from_thread(self.apply_all, snapshots)
+ worker.cancelled_event.wait(self.interval)
+ finally:
+ live.close()
+ return
indices = ([self.gpu_index] if self.gpu_index is not None else
list(range(len(ordered_adapters(enumerate_adapters())))) or [0])
with ExitStack() as stack:
@@ -301,8 +315,13 @@ class GpuTaskManager(App):
def apply_all(self, snapshots: list[Snapshot]) -> None:
if not self.is_running:
return # A worker may finish sampling after shutdown has removed the widgets.
- self.snapshots = snapshots
multi = len(snapshots) > 1
+ if self.snapshots and multi != (len(self.snapshots) > 1) and self.selected_key is not None:
+ if multi:
+ self.selected_key = f"0:{self.selected_key}"
+ else:
+ self.selected_key = self.selected_key[2:] if self.selected_key.startswith("0:") else None
+ self.snapshots = snapshots
table = self.query_one("#procs", DataTable)
if multi and "GPU" not in table.columns:
table.add_column("GPU", key="GPU")
diff --git a/src/gputaskman/collector.py b/src/gputaskman/collector.py
index 9d3cd63..4daa530 100644
--- a/src/gputaskman/collector.py
+++ b/src/gputaskman/collector.py
@@ -6,6 +6,7 @@ import os
import sys
import time
from dataclasses import dataclass, field
+from contextlib import ExitStack
import psutil
@@ -172,7 +173,8 @@ def select_nv_device(adapter: Adapter | None, devices: list[NvDevice], gpu_index
if len(matches) == 1:
return matches[0]
return None
- return devices[gpu_index] if gpu_index < len(devices) else None
+ matches = [d for d in devices if d.index == gpu_index]
+ return matches[0] if len(matches) == 1 else None
class Collector:
@@ -185,13 +187,20 @@ class Collector:
def backend(self) -> str:
return "drm" if self.drm_card is not None else BACKEND
- def __init__(self, gpu_index: int = 0, scan_budget: float | None = SCAN_BUDGET, host: bool = True) -> None:
- self.adapters = [a for a in enumerate_adapters() if not a.software]
+ def __init__(self, gpu_index: int = 0, scan_budget: float | None = SCAN_BUDGET, host: bool = True,
+ adapters: list[Adapter] | None = None) -> None:
+ self.adapters = [a for a in (enumerate_adapters() if adapters is None else adapters) if not a.software]
self.adapter = select_adapter(self.adapters, gpu_index)
- self.counters = GpuCounters()
- self.nvml = Nvml()
- self.amdsmi = AmdSmi(self.adapter, self.adapters)
- self.adl = Adl(self.adapter, self.adapters)
+ with ExitStack() as resources:
+ self.counters = GpuCounters()
+ resources.callback(self.counters.close)
+ self.nvml = Nvml()
+ resources.callback(self.nvml.close)
+ self.amdsmi = AmdSmi(self.adapter, self.adapters)
+ resources.callback(self.amdsmi.close)
+ self.adl = Adl(self.adapter, self.adapters)
+ resources.callback(self.adl.close)
+ resources.pop_all()
self._mac_driver = f"macOS {macos_version()}".strip() if BACKEND == "metal" else ""
# Native Linux on an AMD GPU: device metrics from amdgpu's sysfs, processes from DRM fdinfo.
drm = getattr(self.counters, "drm", None)
@@ -271,7 +280,8 @@ class Collector:
snap.counter_adapter_matched = luid in per_luid or luid in adapters
if not snap.counter_adapter_matched:
snap.counter_status = ("Windows GPU counters could not be matched to the selected adapter; "
- "VRAM and processes are unavailable. This can occur in GPU-PV VMs.")
+ "VRAM and processes are unavailable. The adapter may be temporarily "
+ "unavailable, or GPU-PV VMs may not expose these counters.")
elif a is None or a.dedicated is None or a.shared is None:
snap.counter_status = ("Some Windows adapter memory counters are unavailable; "
"GPU-PV VMs may not expose them.")
@@ -386,9 +396,10 @@ class Collector:
r.gpu.inferred, r.gpu.inferred_by = g.bytes, g.confidence
def close(self) -> None:
- self.counters.close()
- self.nvml.close()
- self.amdsmi.close()
- self.adl.close()
- if self.host is not None:
- self.host.close()
+ with ExitStack() as resources:
+ if self.host is not None:
+ resources.callback(self.host.close)
+ resources.callback(self.adl.close)
+ resources.callback(self.amdsmi.close)
+ resources.callback(self.nvml.close)
+ resources.callback(self.counters.close)
diff --git a/src/gputaskman/dxgi.py b/src/gputaskman/dxgi.py
index 93bb675..9c60742 100644
--- a/src/gputaskman/dxgi.py
+++ b/src/gputaskman/dxgi.py
@@ -88,13 +88,17 @@ def adapter_pci_address(luid: int) -> tuple[int, int, int] | None:
return None
-def enumerate_adapters() -> list[Adapter]:
+def enumerate_adapters(*, strict: bool = False) -> list[Adapter]:
try:
dxgi = ctypes.WinDLL("dxgi")
except OSError:
+ if strict:
+ raise
return []
factory = ctypes.c_void_p()
if dxgi.CreateDXGIFactory1(ctypes.byref(_IID_IDXGIFactory1), ctypes.byref(factory)) != 0:
+ if strict:
+ raise OSError("DXGI adapter discovery failed")
return []
out: list[Adapter] = []
try:
@@ -102,11 +106,15 @@ def enumerate_adapters() -> list[Adapter]:
i = 0
while True:
adapter = ctypes.c_void_p()
- if enum_adapters1(factory, i, ctypes.byref(adapter)) != 0:
- break # DXGI_ERROR_NOT_FOUND
+ status = enum_adapters1(factory, i, ctypes.byref(adapter))
+ if status != 0:
+ if strict and status & 0xffffffff != 0x887A0002: # DXGI_ERROR_NOT_FOUND
+ raise OSError("DXGI adapter enumeration failed")
+ break
try:
desc = _DESC1()
- if _vcall(adapter, 10, ctypes.c_long, ctypes.POINTER(_DESC1))(adapter, ctypes.byref(desc)) == 0:
+ status = _vcall(adapter, 10, ctypes.c_long, ctypes.POINTER(_DESC1))(adapter, ctypes.byref(desc))
+ if status == 0:
luid = ((desc.AdapterLuid.HighPart & 0xffffffff) << 32) | desc.AdapterLuid.LowPart
out.append(Adapter(
luid=luid,
@@ -117,6 +125,8 @@ def enumerate_adapters() -> list[Adapter]:
software=bool(desc.Flags & 2),
pci_address=adapter_pci_address(luid),
))
+ elif strict:
+ raise OSError("DXGI adapter description unavailable")
finally:
_vcall(adapter, 2, wintypes.ULONG)(adapter) # Release
i += 1
diff --git a/src/gputaskman/nvml.py b/src/gputaskman/nvml.py
index 39d936d..00b6b46 100644
--- a/src/gputaskman/nvml.py
+++ b/src/gputaskman/nvml.py
@@ -22,6 +22,7 @@ _THROTTLE = {
0x40: "hw-thermal",
0x80: "power-brake",
}
+_RESET_ERRORS = (1, 9, 15) # uninitialized, driver not loaded, GPU lost
def _try(fn, *a, default=None):
@@ -37,63 +38,94 @@ class Nvml:
self.driver = ""
self.cuda_version = ""
self.handles = []
+ self.needs_reset = False
+ self.refresh()
+
+ def refresh(self) -> None:
+ """Reacquire handles without sampling telemetry or restarting a healthy NVML session."""
if N is None:
return
+ if self.needs_reset:
+ self.close()
+ self.needs_reset = False
+ if not self.ok:
+ try:
+ N.nvmlInit()
+ except Exception:
+ return
+ self.ok = True
try:
- N.nvmlInit()
+ handles = []
+ reset_needed = False
+ for i in range(N.nvmlDeviceGetCount()):
+ try:
+ handles.append((i, N.nvmlDeviceGetHandleByIndex(i)))
+ except Exception as error:
+ reset_needed |= getattr(error, "value", None) in _RESET_ERRORS
+ continue # one unavailable device must not hide its surviving peers
+ self.handles = handles
+ if not handles and reset_needed:
+ self.needs_reset = True
except Exception:
+ self.handles = []
+ self.close() # next low-frequency refresh retries driver initialization
return
- self.ok = True
self.driver = _try(N.nvmlSystemGetDriverVersion, default="") or ""
cv = _try(N.nvmlSystemGetCudaDriverVersion_v2) or _try(N.nvmlSystemGetCudaDriverVersion)
if cv:
self.cuda_version = f"{cv // 1000}.{(cv % 1000) // 10}"
- for i in range(_try(N.nvmlDeviceGetCount, default=0)):
- self.handles.append(N.nvmlDeviceGetHandleByIndex(i))
+
+ def _read(self, fn, *args, default=None):
+ try:
+ return fn(*args)
+ except Exception as error:
+ if getattr(error, "value", None) in _RESET_ERRORS:
+ self.needs_reset = True
+ return default
def sample(self) -> list[NvDevice]:
out = []
- for i, h in enumerate(self.handles):
+ for i, h in self.handles:
d = NvDevice(index=i)
- d.name = _try(N.nvmlDeviceGetName, h, default="?")
- pci = _try(N.nvmlDeviceGetPciInfo, h)
+ d.name = self._read(N.nvmlDeviceGetName, h, default="?")
+ pci = self._read(N.nvmlDeviceGetPciInfo, h)
if pci is not None:
d.pci_bus = pci.busId.decode() if isinstance(pci.busId, bytes) else str(pci.busId)
- dm = _try(N.nvmlDeviceGetDriverModel, h)
+ dm = self._read(N.nvmlDeviceGetDriverModel, h)
if dm:
d.driver_model = {0: "WDDM", 1: "TCC", 2: "MCDM"}.get(dm[0], str(dm[0]))
- mem = _try(N.nvmlDeviceGetMemoryInfo, h, N.nvmlMemory_v2) or _try(N.nvmlDeviceGetMemoryInfo, h)
+ mem = self._read(N.nvmlDeviceGetMemoryInfo, h, N.nvmlMemory_v2) or self._read(N.nvmlDeviceGetMemoryInfo, h)
if mem is not None:
d.mem_total, d.mem_used = mem.total, mem.used
d.mem_reserved = getattr(mem, "reserved", 0)
- u = _try(N.nvmlDeviceGetUtilizationRates, h)
+ u = self._read(N.nvmlDeviceGetUtilizationRates, h)
if u is not None:
d.util_gpu, d.util_mem = u.gpu, u.memory
- enc = _try(N.nvmlDeviceGetEncoderUtilization, h)
- dec = _try(N.nvmlDeviceGetDecoderUtilization, h)
+ enc = self._read(N.nvmlDeviceGetEncoderUtilization, h)
+ dec = self._read(N.nvmlDeviceGetDecoderUtilization, h)
d.enc = enc[0] if enc else None
d.dec = dec[0] if dec else None
- d.temp = _try(N.nvmlDeviceGetTemperature, h, N.NVML_TEMPERATURE_GPU)
- p = _try(N.nvmlDeviceGetPowerUsage, h)
+ d.temp = self._read(N.nvmlDeviceGetTemperature, h, N.NVML_TEMPERATURE_GPU)
+ p = self._read(N.nvmlDeviceGetPowerUsage, h)
d.power_w = p / 1000 if p is not None else None
- pl = _try(N.nvmlDeviceGetEnforcedPowerLimit, h)
+ pl = self._read(N.nvmlDeviceGetEnforcedPowerLimit, h)
d.power_limit_w = pl / 1000 if pl is not None else None
- d.fan = _try(N.nvmlDeviceGetFanSpeed, h)
- d.pstate = _try(N.nvmlDeviceGetPerformanceState, h)
- d.sm_clock = _try(N.nvmlDeviceGetClockInfo, h, N.NVML_CLOCK_SM)
- d.sm_clock_max = _try(N.nvmlDeviceGetMaxClockInfo, h, N.NVML_CLOCK_SM)
- d.mem_clock = _try(N.nvmlDeviceGetClockInfo, h, N.NVML_CLOCK_MEM)
- d.video_clock = _try(N.nvmlDeviceGetClockInfo, h, N.NVML_CLOCK_VIDEO)
- d.pcie_rx_kbs = _try(N.nvmlDeviceGetPcieThroughput, h, N.NVML_PCIE_UTIL_RX_BYTES)
- d.pcie_tx_kbs = _try(N.nvmlDeviceGetPcieThroughput, h, N.NVML_PCIE_UTIL_TX_BYTES)
- d.pcie_gen = _try(N.nvmlDeviceGetCurrPcieLinkGeneration, h)
- d.pcie_width = _try(N.nvmlDeviceGetCurrPcieLinkWidth, h)
- reasons = _try(N.nvmlDeviceGetCurrentClocksEventReasons, h)
+ d.fan = self._read(N.nvmlDeviceGetFanSpeed, h)
+ d.pstate = self._read(N.nvmlDeviceGetPerformanceState, h)
+ d.sm_clock = self._read(N.nvmlDeviceGetClockInfo, h, N.NVML_CLOCK_SM)
+ d.sm_clock_max = self._read(N.nvmlDeviceGetMaxClockInfo, h, N.NVML_CLOCK_SM)
+ d.mem_clock = self._read(N.nvmlDeviceGetClockInfo, h, N.NVML_CLOCK_MEM)
+ d.video_clock = self._read(N.nvmlDeviceGetClockInfo, h, N.NVML_CLOCK_VIDEO)
+ d.pcie_rx_kbs = self._read(N.nvmlDeviceGetPcieThroughput, h, N.NVML_PCIE_UTIL_RX_BYTES)
+ d.pcie_tx_kbs = self._read(N.nvmlDeviceGetPcieThroughput, h, N.NVML_PCIE_UTIL_TX_BYTES)
+ d.pcie_gen = self._read(N.nvmlDeviceGetCurrPcieLinkGeneration, h)
+ d.pcie_width = self._read(N.nvmlDeviceGetCurrPcieLinkWidth, h)
+ reasons = self._read(N.nvmlDeviceGetCurrentClocksEventReasons, h)
if reasons is None:
- reasons = _try(N.nvmlDeviceGetCurrentClocksThrottleReasons, h)
+ reasons = self._read(N.nvmlDeviceGetCurrentClocksThrottleReasons, h)
if reasons:
d.throttle = [n for bit, n in _THROTTLE.items() if reasons & bit]
- cc = _try(N.nvmlDeviceGetCudaComputeCapability, h)
+ cc = self._read(N.nvmlDeviceGetCudaComputeCapability, h)
if cc:
d.compute_cap = f"{cc[0]}.{cc[1]}"
out.append(d)
diff --git a/src/gputaskman/recovery.py b/src/gputaskman/recovery.py
new file mode 100644
index 0000000..6b3211d
--- /dev/null
+++ b/src/gputaskman/recovery.py
@@ -0,0 +1,166 @@
+"""Low-frequency Windows discovery in the existing sampling worker, with stable display slots."""
+
+from __future__ import annotations
+
+import time
+from dataclasses import dataclass
+
+from .collector import Collector, Snapshot, ordered_adapters, select_adapter
+from .dxgi import Adapter, enumerate_adapters
+
+
+@dataclass
+class _Slot:
+ adapter: Adapter | None
+ collector: Collector | None = None
+ retry: bool = False
+ had_telemetry: bool = False
+ retry_after: float = 0.0
+ retry_delay: float = 0.0
+ warning: str = ""
+
+
+def _location(adapter: Adapter) -> tuple | None:
+ # LUIDs may change after re-enabling. A unique PCI location plus device description
+ # can bridge that change; never match by name or enumeration index alone.
+ if adapter.pci_address is not None:
+ return adapter.pci_address, adapter.vendor_id, adapter.name
+ return None
+
+
+class LiveCollectors:
+ DISCOVERY_INTERVAL = 5.0
+ MAX_RETRY_INTERVAL = 60.0
+
+ def __init__(self, gpu_index: int | None = None, *, collector_factory=Collector,
+ discover=enumerate_adapters) -> None:
+ self.factory = collector_factory
+ self.discover = discover
+ self.gpu_index = gpu_index
+ self.discovery_error = ""
+ try:
+ self.adapters = ordered_adapters(discover(strict=True))
+ except OSError as error:
+ self.adapters = []
+ self.discovery_error = f"Windows adapter discovery unavailable ({error}); retaining last known topology."
+ selected = ([select_adapter(self.adapters, gpu_index)] if gpu_index is not None and not self.discovery_error
+ else self.adapters or [None])
+ self.slots = [_Slot(a) for a in selected]
+ now = time.monotonic()
+ self.next_discovery = now + self.DISCOVERY_INTERVAL
+ self._reconcile(self.adapters, now)
+
+ def _close_slot(self, slot: _Slot) -> None:
+ if slot.collector is not None:
+ collector = slot.collector
+ slot.collector = None
+ try:
+ collector.close()
+ except (OSError, ValueError) as error:
+ slot.warning = f"GPU provider cleanup failed: {error}."
+
+ def _schedule_retry(self, slot: _Slot, now: float) -> None:
+ slot.retry_delay = min(slot.retry_delay * 2 or self.DISCOVERY_INTERVAL, self.MAX_RETRY_INTERVAL)
+ slot.retry_after = now + slot.retry_delay
+
+ def _start_slot(self, slot: _Slot, adapters: list[Adapter], now: float, *, retrying: bool = False) -> None:
+ if retrying:
+ self._schedule_retry(slot, now)
+ try:
+ slot.collector = self.factory(adapters.index(slot.adapter), adapters=adapters,
+ scan_budget=Collector.SCAN_BUDGET / max(len(adapters), 1))
+ slot.retry = False
+ except (OSError, ValueError) as error:
+ slot.warning = " ".join(filter(None, (slot.warning, f"GPU provider initialization failed: {error}.")))
+ slot.retry = True
+ if not retrying:
+ self._schedule_retry(slot, now)
+
+ def _reconcile(self, adapters: list[Adapter], now: float) -> None:
+ remaining = list(adapters)
+ for slot in self.slots:
+ old = slot.adapter
+ match = next((a for a in remaining if old and a.luid == old.luid), None)
+ if match is None and old is not None:
+ location = _location(old)
+ candidates = [a for a in remaining if location is not None and _location(a) == location]
+ if (len(candidates) == 1
+ and sum(a.pci_address == old.pci_address for a in adapters) == 1
+ and sum(s.adapter.pci_address == old.pci_address
+ for s in self.slots if s.adapter) == 1):
+ match = candidates[0]
+ if old is None and remaining:
+ index = self.gpu_index or 0
+ if index < len(adapters):
+ match = adapters[index]
+ if match is None:
+ self._close_slot(slot)
+ slot.retry = False
+ slot.retry_after = slot.retry_delay = 0.0
+ continue
+ remaining.remove(match)
+ changed = old != match
+ slot.adapter = match
+ if changed:
+ self._close_slot(slot)
+ slot.retry = False
+ slot.retry_after = slot.retry_delay = 0.0
+ retrying = slot.retry and now >= slot.retry_after
+ if retrying:
+ self._close_slot(slot)
+ if slot.collector is None and (not slot.retry or retrying):
+ self._start_slot(slot, adapters, now, retrying=retrying)
+ elif slot.collector is not None:
+ slot.collector.adapters = adapters
+ slot.collector.nvml.refresh()
+ if self.gpu_index is None:
+ for adapter in remaining:
+ self.slots.append(_Slot(adapter))
+ if remaining:
+ # Existing slots have already been reconciled; initialize only the new ones.
+ for slot in self.slots[-len(remaining):]:
+ self._start_slot(slot, adapters, now)
+ active = sum(s.collector is not None for s in self.slots)
+ for slot in self.slots:
+ if slot.collector:
+ slot.collector.scan_budget = Collector.SCAN_BUDGET / max(active, 1)
+ self.adapters = adapters
+
+ def sample(self, now: float | None = None) -> list[Snapshot]:
+ now = time.monotonic() if now is None else now
+ if now >= self.next_discovery:
+ self.next_discovery = now + self.DISCOVERY_INTERVAL
+ try:
+ adapters = ordered_adapters(self.discover(strict=True))
+ except OSError as error:
+ self.discovery_error = f"Windows adapter discovery unavailable ({error}); retaining last known topology."
+ else:
+ self.discovery_error = ""
+ self._reconcile(adapters, now)
+ snapshots = []
+ for slot in self.slots:
+ snap = None
+ if slot.collector is not None:
+ try:
+ snap = slot.collector.sample()
+ slot.retry = (snap.counter_adapter_matched is False
+ or (slot.had_telemetry and snap.nv is None))
+ slot.had_telemetry |= snap.nv is not None
+ if not slot.retry:
+ slot.retry_after = slot.retry_delay = 0.0
+ except (OSError, ValueError) as error:
+ slot.retry = True
+ slot.warning = f"GPU sampling failed: {error}."
+ if snap is None:
+ snap = Snapshot(time.time(), slot.adapter, self.adapters, None, "", "",
+ slot.adapter.dedicated_vram if slot.adapter else 0, None, None,
+ counter_status="Adapter unavailable; waiting for Windows rediscovery.",
+ counter_adapter_matched=False)
+ snap.counter_status = " ".join(filter(None, (snap.counter_status, slot.warning, self.discovery_error)))
+ slot.warning = ""
+ snapshots.append(snap)
+ return snapshots
+
+ def close(self) -> None:
+ for slot in self.slots:
+ self._close_slot(slot)
diff --git a/tests/test_core.py b/tests/test_core.py
index 4652fcd..1d86de6 100644
--- a/tests/test_core.py
+++ b/tests/test_core.py
@@ -240,6 +240,7 @@ class MemoryTests(unittest.TestCase):
self.assertEqual(s.procs, [])
self.assertFalse(s.counter_adapter_matched)
self.assertIn("could not be matched", s.counter_status)
+ self.assertIn("temporarily unavailable", s.counter_status)
self.assertIn("GPU-PV", to_text(s))
data = to_dict(s)
self.assertIsNone(data["attributed"])
@@ -1238,7 +1239,10 @@ class UiTests(unittest.IsolatedAsyncioTestCase):
collector.close.assert_called_once()
async def test_explicit_gpu_keeps_single_device_layout(self):
- with patch("gputaskman.app.Collector") as factory:
+ adapters = [Adapter(1, "first", 0x10DE, 16 * GiB, 0, False),
+ Adapter(2, "second", 0x10DE, 8 * GiB, 0, False)]
+ with patch("gputaskman.app.enumerate_adapters", return_value=adapters), \
+ patch("gputaskman.app.Collector") as factory:
factory.SCAN_BUDGET = 0.15
factory.return_value.sample.side_effect = snapshot
closed = threading.Event()
diff --git a/tests/test_recovery.py b/tests/test_recovery.py
new file mode 100644
index 0000000..5c0ba11
--- /dev/null
+++ b/tests/test_recovery.py
@@ -0,0 +1,533 @@
+"""Hardware-independent Windows disable/re-enable and provider recovery sequences."""
+
+from types import SimpleNamespace
+from unittest import IsolatedAsyncioTestCase, TestCase
+from unittest.mock import Mock, patch
+
+from gputaskman.app import GpuTaskManager
+from gputaskman.collector import Collector, ProcRow, Snapshot, select_nv_device
+from gputaskman.dxgi import Adapter, enumerate_adapters
+from gputaskman.nvml import Nvml
+from gputaskman.pdh import ProcGpu
+from gputaskman.procinfo import ProcDetail
+from gputaskman.recovery import LiveCollectors
+from gputaskman.report import to_dict
+from gputaskman.telemetry import DeviceTelemetry
+
+
+def adapter(luid, vendor=0x10DE, bus=1):
+ return Adapter(luid, f"device at {bus}", vendor, 4 << 30, 0, False, (bus, 0, 0))
+
+
+def snapshot(a, value=10):
+ return Snapshot(1, a, [a], DeviceTelemetry(index=0, util_gpu=value), "", "",
+ a.dedicated_vram, value << 20, 0,
+ [ProcRow(10, "app", "app", ProcGpu(dedicated=value << 20), ProcDetail(10, "app"))],
+ counter_adapter_matched=True)
+
+
+class RecoveryTests(TestCase):
+ def setUp(self):
+ self.first, self.second = adapter(1), adapter(2, 0x1002, 2)
+ self.topology = [self.first, self.second]
+ self.created = []
+ self.discover = Mock(side_effect=lambda **_kwargs: list(self.topology))
+
+ def factory(index, *, adapters, scan_budget):
+ a = adapters[index]
+ c = Mock(adapter=a, scan_budget=scan_budget)
+ c.sample.side_effect = lambda: snapshot(a, a.luid)
+ self.created.append(c)
+ return c
+
+ self.factory = Mock(side_effect=factory)
+
+ def live(self, index=None):
+ live = LiveCollectors(index, collector_factory=self.factory, discover=self.discover)
+ self.addCleanup(live.close)
+ return live
+
+ def rediscover(self, live):
+ return live.sample(now=live.next_discovery)
+
+ def test_removed_readded_and_new_adapter_keep_slots_and_surviving_collector(self):
+ live = self.live()
+ survivor = self.created[1]
+ self.topology = [self.second]
+ snaps = self.rediscover(live)
+ self.assertIsNone(snaps[0].vram_used)
+ self.assertIsNone(snaps[0].attributed)
+ self.assertEqual(snaps[0].procs, [])
+ self.assertFalse(snaps[0].counter_adapter_matched)
+ self.assertEqual(snaps[1].adapter, self.second)
+ self.created[0].close.assert_called_once()
+ self.assertIs(live.slots[1].collector, survivor)
+ # Re-enabling can produce a new LUID; bridge only a unique PCI+description match.
+ readded, new = adapter(11), adapter(3, bus=3)
+ self.topology = [new, self.second, readded]
+ snaps = self.rediscover(live)
+ self.assertEqual([s.adapter.luid for s in snaps], [11, 2, 3])
+ self.assertIs(live.slots[1].collector, survivor)
+ self.assertEqual(len(self.created), 4)
+ self.assertAlmostEqual(survivor.scan_budget, Collector.SCAN_BUDGET / 3)
+
+ def test_explicit_gpu_pins_original_identity_not_current_index(self):
+ live = self.live(1)
+ self.topology = [self.first]
+ snaps = self.rediscover(live)
+ self.assertEqual(len(snaps), 1)
+ self.assertEqual(snaps[0].adapter, self.second)
+ self.assertIsNone(snaps[0].vram_used)
+ self.topology = [adapter(12, 0x1002, 2), self.first, adapter(3, bus=3)]
+ self.assertEqual(self.rediscover(live)[0].adapter.luid, 12)
+ self.assertEqual(len(live.slots), 1)
+
+ def test_reordering_does_not_recreate_collectors_or_swap_ownership(self):
+ live = self.live()
+ original = list(self.created)
+ self.topology.reverse()
+ snaps = self.rediscover(live)
+ self.assertEqual([s.adapter.luid for s in snaps], [1, 2])
+ self.assertEqual([s.collector for s in live.slots], original)
+ self.assertEqual(len(self.created), 2)
+
+ def test_discovery_failure_keeps_surviving_samples_and_retries_later(self):
+ live = self.live()
+ self.discover.side_effect = OSError("temporary DXGI failure")
+ snaps = self.rediscover(live)
+ self.assertEqual([s.vram_used for s in snaps], [1 << 20, 2 << 20])
+ self.assertIn("discovery unavailable", snaps[0].counter_status)
+ self.discover.side_effect = lambda **_kwargs: list(self.topology)
+ self.assertEqual(self.rediscover(live)[0].counter_status, "")
+ self.assertEqual(len(self.created), 2)
+
+ def test_sample_failure_is_unknown_and_rebuilds_only_failed_provider(self):
+ live = self.live()
+ first, second = self.created
+ first.sample.side_effect = OSError("lost provider")
+ snaps = live.sample(now=live.next_discovery - 1)
+ self.assertIsNone(snaps[0].attributed)
+ self.assertEqual(snaps[1].vram_used, 2 << 20)
+ first.close.side_effect = OSError("cleanup failed")
+ snaps = self.rediscover(live)
+ self.assertEqual(snaps[0].vram_used, 1 << 20)
+ self.assertIn("cleanup failed", snaps[0].counter_status)
+ self.assertIs(live.slots[1].collector, second)
+ first.close.assert_called_once()
+
+ def test_unmatched_counters_trigger_bounded_rebuild(self):
+ live = self.live()
+ failed = snapshot(self.first)
+ failed.counter_adapter_matched = False
+ self.created[0].sample.side_effect = lambda: failed
+ live.sample(now=live.next_discovery - 1)
+ self.assertEqual(len(self.created), 2)
+ self.rediscover(live)
+ self.assertEqual(len(self.created), 3)
+
+ def test_persistent_missing_counters_back_off_and_recover(self):
+ live = self.live()
+ failed = snapshot(self.first)
+ failed.counter_adapter_matched = False
+ self.created[0].sample.side_effect = lambda: failed
+ original_factory = self.factory.side_effect
+
+ def failing_factory(*args, **kwargs):
+ collector = original_factory(*args, **kwargs)
+ if collector.adapter.luid == self.first.luid:
+ collector.sample.side_effect = lambda: failed
+ return collector
+
+ self.factory.side_effect = failing_factory
+ live.sample(now=live.next_discovery - 1)
+ for _ in range(12):
+ self.rediscover(live)
+ self.assertEqual(sum(c.adapter.luid == self.first.luid for c in self.created), 5)
+ self.assertEqual(sum(c.adapter.luid == self.second.luid for c in self.created), 1)
+
+ recovered = self.created[-1]
+ recovered.sample.side_effect = lambda: snapshot(self.first)
+ self.assertTrue(live.sample(now=live.next_discovery - 1)[0].counter_adapter_matched)
+ recovered.sample.side_effect = lambda: failed
+ live.sample(now=live.next_discovery - 1)
+ self.rediscover(live)
+ self.assertEqual(sum(c.adapter.luid == self.first.luid for c in self.created), 6)
+
+ def test_returning_adapter_retries_immediately_during_backoff(self):
+ live = self.live()
+ failed = snapshot(self.first)
+ failed.counter_adapter_matched = False
+ self.created[0].sample.side_effect = lambda: failed
+ live.sample(now=live.next_discovery - 1)
+ self.rediscover(live)
+ self.created[-1].sample.side_effect = lambda: failed
+ live.sample(now=live.next_discovery - 1)
+ self.rediscover(live)
+ self.created[-1].sample.side_effect = lambda: failed
+ live.sample(now=live.next_discovery - 1)
+
+ self.topology = [self.second]
+ self.assertIsNone(self.rediscover(live)[0].vram_used)
+ self.topology = [self.first, self.second]
+ self.assertEqual(self.rediscover(live)[0].vram_used, 1 << 20)
+
+ def test_discovery_runs_only_every_five_seconds_without_catchup_burst(self):
+ live = self.live()
+ for _ in range(20):
+ live.sample(now=live.next_discovery - 1)
+ self.discover.assert_called_once_with(strict=True)
+ self.rediscover(live)
+ self.assertEqual(self.discover.call_count, 2)
+ live.sample(now=live.next_discovery + 100)
+ self.assertEqual(self.discover.call_count, 3)
+
+ def test_no_hardware_at_startup_can_discover_first_device(self):
+ self.topology = []
+ live = self.live(0)
+ self.assertIsNone(live.sample()[0].vram_used)
+ self.topology = [self.first]
+ self.assertEqual(self.rediscover(live)[0].adapter, self.first)
+
+ def test_initial_discovery_error_recovers_without_binding_wrong_explicit_index(self):
+ self.discover.side_effect = OSError("temporary DXGI failure")
+ live = self.live(1)
+ self.assertIsNone(live.sample()[0].adapter)
+ self.discover.side_effect = lambda **_kwargs: list(self.topology)
+ self.assertEqual(self.rediscover(live)[0].adapter, self.second)
+
+ def test_out_of_range_explicit_gpu_still_rejected(self):
+ with self.assertRaisesRegex(ValueError, "out of range"):
+ self.live(2)
+ self.factory.assert_not_called()
+
+ def test_ambiguous_pci_match_never_reassigns_removed_device(self):
+ live = self.live(0)
+ self.topology = [adapter(11), adapter(12)]
+ snap = self.rediscover(live)[0]
+ self.assertEqual(snap.adapter, self.first)
+ self.assertIsNone(snap.vram_used)
+
+ def test_duplicate_pci_address_is_ambiguous_even_with_different_vendors(self):
+ live = self.live(0)
+ self.topology = [adapter(11), adapter(12, 0x1002)]
+ snap = self.rediscover(live)[0]
+ self.assertEqual(snap.adapter, self.first)
+ self.assertIsNone(snap.vram_used)
+
+ def test_missing_pci_does_not_match_by_name(self):
+ self.first.pci_address = None
+ live = self.live(0)
+ replacement = adapter(11)
+ replacement.pci_address = None
+ self.topology = [replacement]
+ self.assertIsNone(self.rediscover(live)[0].vram_used)
+
+ def test_lost_telemetry_rebuilds_provider_without_recreating_survivor(self):
+ live = self.live()
+ live.sample()
+ failed = snapshot(self.first)
+ failed.nv = None
+ self.created[0].sample.side_effect = lambda: failed
+ live.sample()
+ self.rediscover(live)
+ self.assertEqual(len(self.created), 3)
+ self.assertIs(live.slots[1].collector, self.created[1])
+
+ def test_collector_creation_error_retries_and_keeps_survivor(self):
+ survivor = Mock()
+ survivor.sample.return_value = snapshot(self.second)
+ self.factory.side_effect = [OSError("provider unavailable"), survivor]
+ live = self.live()
+ self.assertIs(live.slots[1].collector, survivor)
+ replacement = Mock()
+ replacement.sample.return_value = snapshot(self.first)
+ self.factory.side_effect = [replacement]
+ self.assertEqual(self.rediscover(live)[0].vram_used, 10 << 20)
+ self.assertIs(live.slots[1].collector, survivor)
+
+ def test_close_failure_does_not_skip_other_collectors(self):
+ live = self.live()
+ self.created[0].close.side_effect = OSError("device gone")
+ live.close()
+ for collector in self.created:
+ collector.close.assert_called_once()
+ live.close()
+ for collector in self.created:
+ collector.close.assert_called_once()
+
+ def test_software_adapter_is_not_added_as_a_gpu(self):
+ live = self.live()
+ software = adapter(30)
+ software.software = True
+ self.topology.append(software)
+ self.assertEqual(len(self.rediscover(live)), 2)
+
+
+class NvmlRecoveryTests(TestCase):
+ def api(self):
+ return SimpleNamespace(
+ nvmlInit=Mock(), nvmlShutdown=Mock(), nvmlDeviceGetCount=Mock(return_value=2),
+ nvmlDeviceGetHandleByIndex=Mock(side_effect=["first", "second"]),
+ nvmlSystemGetDriverVersion=Mock(return_value="test"),
+ nvmlSystemGetCudaDriverVersion_v2=Mock(return_value=12000),
+ nvmlSystemGetCudaDriverVersion=Mock(return_value=12000),
+ )
+
+ def test_rebind_handles_without_reinitializing_healthy_session(self):
+ api = self.api()
+ with patch("gputaskman.nvml.N", api):
+ nv = Nvml()
+ api.nvmlDeviceGetHandleByIndex.side_effect = ["new-first", "new-second"]
+ nv.refresh()
+ self.assertEqual(nv.handles, [(0, "new-first"), (1, "new-second")])
+ api.nvmlInit.assert_called_once()
+ api.nvmlShutdown.assert_not_called()
+ nv.close()
+
+ def test_one_failed_handle_does_not_hide_other_devices(self):
+ api = self.api()
+ api.nvmlDeviceGetHandleByIndex.side_effect = [OSError("GPU lost"), "second"]
+ with patch("gputaskman.nvml.N", api):
+ nv = Nvml()
+ self.assertEqual(nv.handles, [(1, "second")])
+ nv.close()
+
+ def test_all_lost_handles_request_reset_without_resetting_surviving_peers(self):
+ class LostGpu(Exception):
+ value = 15
+
+ api = self.api()
+ api.nvmlDeviceGetHandleByIndex.side_effect = [LostGpu(), "second"]
+ with patch("gputaskman.nvml.N", api):
+ nv = Nvml()
+ self.assertFalse(nv.needs_reset)
+ api.nvmlDeviceGetHandleByIndex.side_effect = [LostGpu(), LostGpu()]
+ nv.refresh()
+ self.assertTrue(nv.needs_reset)
+ api.nvmlDeviceGetHandleByIndex.side_effect = ["first", "second"]
+ nv.refresh()
+ self.assertEqual(nv.handles, [(0, "first"), (1, "second")])
+ self.assertEqual(api.nvmlInit.call_count, 2)
+ api.nvmlShutdown.assert_called_once()
+ nv.close()
+
+ def test_count_error_clears_stale_handles_and_next_refresh_recovers(self):
+ api = self.api()
+ with patch("gputaskman.nvml.N", api):
+ nv = Nvml()
+ api.nvmlDeviceGetCount.side_effect = OSError("driver lost")
+ nv.refresh()
+ self.assertFalse(nv.ok)
+ self.assertEqual(nv.handles, [])
+ api.nvmlDeviceGetCount.side_effect = None
+ api.nvmlDeviceGetHandleByIndex.side_effect = ["new-first", "new-second"]
+ nv.refresh()
+ self.assertTrue(nv.ok)
+ self.assertEqual(nv.handles, [(0, "new-first"), (1, "new-second")])
+ nv.close()
+
+ def test_failed_initialization_can_retry(self):
+ api = self.api()
+ api.nvmlInit.side_effect = OSError("driver unavailable")
+ with patch("gputaskman.nvml.N", api):
+ nv = Nvml()
+ self.assertFalse(nv.ok)
+ api.nvmlInit.side_effect = None
+ nv.refresh()
+ self.assertTrue(nv.ok)
+ nv.close()
+
+ def test_failed_first_handle_preserves_survivor_index_json_and_fallback_selection(self):
+ class Api(SimpleNamespace):
+ def __getattr__(self, name):
+ return 0 if name.startswith("NVML_") else lambda *args: None
+
+ api = Api(**vars(self.api()))
+ api.nvmlDeviceGetHandleByIndex.side_effect = [OSError("GPU lost"), "second"]
+ api.nvmlDeviceGetName = Mock(return_value="surviving GPU")
+ api.nvmlDeviceGetPciInfo = Mock(return_value=SimpleNamespace(busId="00000000:02:00.0"))
+ with patch("gputaskman.nvml.N", api):
+ nv = Nvml()
+ try:
+ devices = nv.sample()
+ self.assertEqual(len(devices), 1)
+ survivor = devices[0]
+ self.assertEqual(survivor.index, 1)
+ snap = snapshot(adapter(2))
+ snap.adapter = None
+ snap.nv = survivor
+ self.assertEqual(to_dict(snap)["device"]["index"], 1)
+ self.assertIsNone(select_nv_device(None, devices, 0))
+ self.assertIs(select_nv_device(None, devices, 1), survivor)
+ self.assertIsNone(select_nv_device(None, devices, 2))
+ api.nvmlDeviceGetHandleByIndex.side_effect = ["new-first", "new-second"]
+ nv.refresh()
+ recovered = nv.sample()
+ self.assertEqual([d.index for d in recovered], [0, 1])
+ self.assertIs(select_nv_device(None, recovered, 0), recovered[0])
+ self.assertIs(select_nv_device(None, recovered, 1), recovered[1])
+ finally:
+ nv.close()
+
+ def test_adapter_none_fallback_uses_index_even_when_devices_are_reordered(self):
+ first = DeviceTelemetry(index=0)
+ second = DeviceTelemetry(index=1)
+ self.assertIs(select_nv_device(None, [second, first], 0), first)
+ self.assertIs(select_nv_device(None, [second, first], 1), second)
+ self.assertIsNone(select_nv_device(None, [first, second], -1))
+ self.assertIsNone(select_nv_device(None, [], 0))
+
+ def test_lost_pci_identity_requests_session_reset_on_refresh(self):
+ api = self.api()
+ with patch("gputaskman.nvml.N", api):
+ nv = Nvml()
+ nv.needs_reset = True
+ api.nvmlDeviceGetHandleByIndex.side_effect = ["new-first", "new-second"]
+ nv.refresh()
+ self.assertEqual(api.nvmlInit.call_count, 2)
+ api.nvmlShutdown.assert_called_once()
+ nv.close()
+
+ def test_unsupported_pci_query_does_not_restart_healthy_nvml(self):
+ class Api(SimpleNamespace):
+ def __getattr__(self, name):
+ return 0 if name.startswith("NVML_") else lambda *args: None
+
+ class Unsupported(Exception):
+ value = 3
+
+ api = Api(**vars(self.api()))
+ api.nvmlDeviceGetPciInfo = Mock(side_effect=Unsupported())
+ with patch("gputaskman.nvml.N", api):
+ nv = Nvml()
+ nv.sample()
+ self.assertFalse(nv.needs_reset)
+ api.nvmlDeviceGetHandleByIndex.side_effect = ["first", "second"]
+ nv.refresh()
+ api.nvmlInit.assert_called_once()
+ api.nvmlShutdown.assert_not_called()
+ nv.close()
+
+ def test_gpu_lost_metric_error_resets_even_when_pci_identity_still_responds(self):
+ class Api(SimpleNamespace):
+ def __getattr__(self, name):
+ return 0 if name.startswith("NVML_") else lambda *args: None
+
+ class Error(Exception):
+ value = 15
+
+ api = Api(**vars(self.api()))
+ api.nvmlDeviceGetPciInfo = Mock(return_value=SimpleNamespace(busId="00000000:01:00.0"))
+ api.nvmlDeviceGetUtilizationRates = Mock(side_effect=Error("GPU lost"))
+ with patch("gputaskman.nvml.N", api):
+ nv = Nvml()
+ self.assertIsNone(nv.sample()[0].util_gpu)
+ self.assertTrue(nv.needs_reset)
+ api.nvmlDeviceGetHandleByIndex.side_effect = ["new-first", "new-second"]
+ nv.refresh()
+ api.nvmlShutdown.assert_called_once()
+ nv.close()
+
+ def test_unsupported_optional_metric_does_not_request_reset(self):
+ class Error(Exception):
+ value = 3
+
+ api = self.api()
+ with patch("gputaskman.nvml.N", api):
+ nv = Nvml()
+ self.assertIsNone(nv._read(Mock(side_effect=Error("not supported"))))
+ self.assertFalse(nv.needs_reset)
+ nv.close()
+
+
+class DiscoveryTests(TestCase):
+ def test_discovery_error_is_not_confused_with_empty_topology(self):
+ with patch("gputaskman.dxgi.ctypes.WinDLL", side_effect=OSError("DXGI unavailable"), create=True):
+ self.assertEqual(enumerate_adapters(), [])
+ with self.assertRaises(OSError):
+ enumerate_adapters(strict=True)
+
+ def test_factory_error_is_reported_to_live_discovery(self):
+ dll = SimpleNamespace(CreateDXGIFactory1=Mock(return_value=-1))
+ with patch("gputaskman.dxgi.ctypes.WinDLL", return_value=dll, create=True):
+ with self.assertRaisesRegex(OSError, "discovery failed"):
+ enumerate_adapters(strict=True)
+
+ def test_enumeration_error_is_not_reported_as_successful_empty_list(self):
+ dll = SimpleNamespace(CreateDXGIFactory1=Mock(return_value=0))
+ with patch("gputaskman.dxgi.ctypes.WinDLL", return_value=dll, create=True), \
+ patch("gputaskman.dxgi._vcall", side_effect=[Mock(return_value=-1), Mock()]):
+ with self.assertRaisesRegex(OSError, "enumeration failed"):
+ enumerate_adapters(strict=True)
+
+
+class CollectorResourceTests(TestCase):
+ def test_partial_provider_initialization_closes_previously_created_resources(self):
+ with patch("gputaskman.collector.GpuCounters") as counters, \
+ patch("gputaskman.collector.Nvml") as nvml, \
+ patch("gputaskman.collector.AmdSmi") as amd, \
+ patch("gputaskman.collector.Adl", side_effect=OSError("device unavailable")):
+ with self.assertRaises(OSError):
+ Collector(adapters=[adapter(1)])
+ counters.return_value.close.assert_called_once()
+ nvml.return_value.close.assert_called_once()
+ amd.return_value.close.assert_called_once()
+
+ def test_close_failure_still_releases_all_providers(self):
+ c = Collector.__new__(Collector)
+ c.host = None
+ c.counters, c.nvml, c.amdsmi, c.adl = Mock(), Mock(), Mock(), Mock()
+ c.counters.close.side_effect = OSError("query lost")
+ with self.assertRaises(OSError):
+ c.close()
+ for provider in (c.counters, c.nvml, c.amdsmi, c.adl):
+ provider.close.assert_called_once()
+
+
+class RecoveryUiTests(IsolatedAsyncioTestCase):
+ async def test_histories_and_row_ownership_stay_in_stable_slots_across_recovery(self):
+ first, second = adapter(1), adapter(2, 0x1002, 2)
+ topology = [first, second]
+
+ def factory(index, *, adapters, **_kwargs):
+ a = adapters[index]
+ c = Mock()
+ c.sample.side_effect = lambda: snapshot(a, a.luid)
+ return c
+
+ live = LiveCollectors(collector_factory=factory, discover=lambda **_kwargs: list(topology))
+ try:
+ with patch.object(GpuTaskManager, "poll"):
+ app = GpuTaskManager()
+ app.show_all = True
+ async with app.run_test() as pilot:
+ app.apply_all(live.sample())
+ await pilot.pause()
+ app.selected_key = "1:10"
+ topology[:] = [second]
+ app.apply_all(live.sample(now=live.next_discovery))
+ await pilot.pause()
+ topology[:] = [second, adapter(11)]
+ app.apply_all(live.sample(now=live.next_discovery))
+ await pilot.pause()
+ self.assertEqual(list(app._histories[0]["util"]), [1, None, 11])
+ self.assertEqual(list(app._histories[1]["util"]), [2, 2, 2])
+ self.assertEqual(app.selected_key, "1:10")
+ self.assertEqual(app._row_snapshots["1:10"].adapter, second)
+ self.assertEqual(app._row_snapshots["0:10"].adapter.luid, 11)
+ finally:
+ live.close()
+
+ async def test_new_second_gpu_preserves_selected_process_on_original_gpu(self):
+ first, second = adapter(1), adapter(2, 0x1002, 2)
+ with patch.object(GpuTaskManager, "poll"):
+ app = GpuTaskManager()
+ app.show_all = True
+ async with app.run_test() as pilot:
+ app.apply(snapshot(first))
+ await pilot.pause()
+ app.selected_key = "10"
+ app.apply_all([snapshot(first), snapshot(second, 80)])
+ await pilot.pause()
+ self.assertEqual(app.selected_key, "0:10")
+ self.assertEqual(app._row_snapshots[app.selected_key].adapter, first)