Skip to content

A GPU task manager for Windows, Linux, and WSL: per-process VRAM, GPU activity, and LLM model/KV-cache insights (Windows AMD and NVIDIA).

Python

Fix Ascend chip accounting and telemetry

Scott Hanselman7f88442fa96d
diff --git a/README.md b/README.md
index 12d2db8..2ac6020 100644
--- a/README.md
+++ b/README.md
@@ -118,8 +118,8 @@ skips the consistency check, so do not use it after changing dependencies.
Tagged releases include portable Linux builds: `gputaskman-VERSION-linux-x64.tar.gz` and
`gputaskman-VERSION-linux-arm64.tar.gz`. They need glibc 2.35 or newer (Ubuntu 22.04, Debian 12,
-or newer) and an NVIDIA driver (or, on native Linux, an AMD GPU on the `amdgpu` driver or an Intel GPU on `i915`), and work
-natively, in containers, and in WSL:
+or newer) and an NVIDIA driver (or, on native Linux, an AMD GPU on the `amdgpu` driver, an Intel GPU on `i915`,
+or a Huawei Ascend 310P with `npu-smi`). NVIDIA works natively, in containers, and in WSL:
```bash
tar -xzf gputaskman-VERSION-linux-x64.tar.gz
@@ -137,6 +137,13 @@ uv run --locked gputaskman
What can be measured depends on where it runs, and the UI says which mode it is in.
+On native Linux, Ascend chips are listed individually. Their device memory and per-process memory
+come from `npu-smi`; AI-core and memory-bandwidth activity come from its watch stream, not
+per-process engine counters. Memory refreshes every 10 seconds, and a failed refresh or stale watch
+is reported as unavailable. The contributor reported testing an Atlas 300I Duo with a vLLM
+workload; driver version and these subsequent fixes have not been independently validated on hardware.
+Ascend container and PID-namespace behavior is not validated.
+
| | Native Linux | WSL, interop on (default) | WSL, interop off or `--no-host` |
|---|---|---|---|
| Device totals, load, temperature, power, clocks | NVML | NVML | NVML |
@@ -855,6 +862,7 @@ Identification is designed to stay cheap. Measured on one Windows machine:
| vLLM `/v1/models` | 1.32 ms median, 21.36 ms max in 30 loopback fixture GETs; cached lookup ~0.0004 ms (Windows 11 build 28000, ARM64; no vLLM runtime/GPU workload) | One GET, 0.5 s timeout and 4 MiB response cap; success cached 3 s, misses/401 cached 30 s. No redirects; actual Linux/vLLM latency unmeasured |
| `psutil.open_files()` | 0.6-2.5 s per process | Never |
| AMD and Intel on Linux: DRM fdinfo scan (`readlink` every fd, read fdinfo of `/dev/dri` ones) | ~9 ms for ~8,000 fds (AMD, Linux desktop; Intel not measured) | Every refresh, only when an amdgpu or Intel card is shown; listing fds is bounded to 50 ms in the TUI and resumes next refresh, known GPU fds are still read |
+| Ascend on Linux: `npu-smi` | Contributor reports ~0.5 s per command on an Atlas 300I Duo; not independently measured | Discovery is cached for 120 s. Each selected chip runs one `info` and one `proc-mem` command every 10 s and owns a watch process; multiple selected chips add their synchronous command costs |
| macOS: `ioreg` device stats + Metal clients | ~10 ms | Every refresh |
| macOS: memory-map scan (`proc_pidinfo` region walk) | ~4 ms per process | Same policy as the memory-map scan above |
| macOS: `footprint` GPU memory | ~70 ms per process, background thread | Busy and new processes every ~2 s, idle ones every 30 s (54 runs in 40 s with 27 GPU processes, M4 Mac mini) |
diff --git a/src/gputaskman/ascend.py b/src/gputaskman/ascend.py
index d5f6936..c87a945 100644
--- a/src/gputaskman/ascend.py
+++ b/src/gputaskman/ascend.py
@@ -11,8 +11,8 @@ appears as two adapters, one per chip, each with its own DDR bank.
from __future__ import annotations
-import os
import re
+import shutil
import subprocess
import threading
import time
@@ -33,26 +33,23 @@ def _npu_path() -> str | None:
"""Path to npu-smi, or None when the tool (and likely the driver) is absent."""
global _NPUSMI
if _NPUSMI is None:
- for candidate in ("npu-smi", "/usr/local/bin/npu-smi"):
- try:
- subprocess.run([candidate, "-h"], capture_output=True, timeout=5)
- _NPUSMI = candidate
- break
- except (OSError, subprocess.SubprocessError):
- continue
+ _NPUSMI = shutil.which("npu-smi") or shutil.which("/usr/local/bin/npu-smi")
return _NPUSMI
def _run(args: list[str], timeout: float = 10.0) -> str:
- """npu-smi output, or '' on any failure (missing tool, error exit, timeout)."""
+ """Return npu-smi output; callers must distinguish failure from an empty result."""
path = _npu_path()
if path is None:
- return ""
+ raise OSError("npu-smi not found")
try:
- r = subprocess.run([path, *args], capture_output=True, text=True, timeout=timeout)
- except (OSError, subprocess.SubprocessError):
- return ""
- return r.stdout if r.returncode == 0 else ""
+ r = subprocess.run([path, *args], capture_output=True, text=True, errors="replace",
+ timeout=timeout, check=True)
+ except subprocess.TimeoutExpired as error:
+ raise OSError("npu-smi timed out") from error
+ except subprocess.CalledProcessError as error:
+ raise OSError(f"npu-smi exited with status {error.returncode}") from error
+ return r.stdout
@dataclass
@@ -82,7 +79,7 @@ def parse_info(text: str) -> list[dict]:
if len(cells) < 3:
continue
m = re.match(r"^(\d+)\s+(\d+)$", cells[0])
- if m and re.match(r"^\d+:\d+:\d+\.\d+$", cells[1]):
+ if m and re.fullmatch(r"[0-9a-fA-F]+:[0-9a-fA-F]+:[0-9a-fA-F]+\.[0-9a-fA-F]+", cells[1]):
if head is None:
continue
hm = re.match(r"^(\d+)\s+(\S+)$", head[0])
@@ -96,7 +93,7 @@ def parse_info(text: str) -> list[dict]:
"health": head[1], "power": ht[0] if ht else None,
"temp": int(ht[1]) if len(ht) > 1 and ht[1].lstrip("-").isdigit() else None,
"chip": int(m.group(1)), "device": int(m.group(2)), "bus": cells[1],
- "aicore": int(cells[2].split()[0]) if cells[2].split() else None,
+ "aicore": int(cells[2].split()[0]) if cells[2].split() and cells[2].split()[0].isdigit() else None,
"mem_used": int(mt.group(1)) if mt else None,
"mem_total": int(mt.group(2)) if mt else None,
})
@@ -123,10 +120,15 @@ def parse_watch_line(line: str) -> dict | None:
def parse_proc_mem(text: str) -> list[tuple[int, str, int]]:
- """`npu-smi info -t proc-mem` rows -> (pid, name, megabytes)."""
+ """`npu-smi info -t proc-mem` rows -> (pid, name, bytes)."""
out = []
- for m in re.finditer(r"Process id:(\d+)\s+Process name:(\S+)\s+Process memory\(MB\):(\d+)", text):
- out.append((int(m.group(1)), m.group(2), int(m.group(3))))
+ for m in re.finditer(
+ r"Process id:\s*(\d+)\s+Process name:\s*(\S+)\s+Process memory\((B|MB)\):\s*(\d+)", text):
+ size = int(m.group(4)) * (_MB if m.group(3) == "MB" else 1)
+ out.append((int(m.group(1)), m.group(2), size))
+ if len(out) != text.count("Process id:") or (not out and
+ not re.search(r"no (?:running )?process(?:es)? (?:found|running)", text, re.IGNORECASE)):
+ raise ValueError("unrecognized npu-smi process output")
return out
@@ -136,13 +138,19 @@ def _pci_key(pdev: str) -> int:
def probe() -> list[AscendChip]:
"""Enumerate every Ascend chip npu-smi can see (two subprocess calls, ~1s)."""
- text = _run(["info"])
- if not text:
+ if _npu_path() is None:
+ return []
+ try:
+ text = _run(["info"])
+ except OSError:
return []
chips = []
for c in parse_info(text):
product = ""
- board = _run(["info", "-t", "board", "-i", str(c["npu"])])
+ try:
+ board = _run(["info", "-t", "board", "-i", str(c["npu"])])
+ except OSError:
+ board = ""
pm = re.search(r"Product Name\s*:\s*(.+)", board)
if pm:
product = pm.group(1).strip()
@@ -152,7 +160,7 @@ def probe() -> list[AscendChip]:
chip = AscendChip(npu=c["npu"], chip=c["chip"], name=name, pdev=c["bus"], product=product)
if c["mem_total"] is not None:
chip.mem_total = c["mem_total"] * _MB
- chip.mem_used = (c["mem_used"] or 0) * _MB
+ chip.mem_used = c["mem_used"] * _MB if c["mem_used"] is not None else None
chips.append(chip)
return chips
@@ -162,19 +170,26 @@ class AscendCounters:
PROC_REFRESH_S = 10.0 # proc-mem is ~0.5s per chip; don't pay it every sample
INFO_REFRESH_S = 10.0 # absolute memory refresh (watch only carries percentages)
+ WATCH_STALE_S = 3.0
+ WATCH_RETRY_S = 10.0
def __init__(self, chips: list[AscendChip] | None = None) -> None:
self.ok = False
self.chips = chips if chips is not None else probe()
self.ok = bool(self.chips)
self.by_key = {(_pci_key(c.pdev) << 8) | c.chip: c for c in self.chips}
- self._watch: dict[tuple[int, int], dict] = {} # (npu, chip) -> latest watch row
- self._watch_dead = False
+ self._watch: dict[tuple[int, int], tuple[float, dict]] = {}
self._lock = threading.Lock()
- self._last_proc = 0.0
- self._last_info = 0.0
- self._procs: dict[int, ProcGpu] = {}
+ self._last_proc = float("-inf")
+ self._last_info = float("-inf")
+ self._last_watch_start = float("-inf")
+ self._procs: dict[int, dict[int, ProcGpu]] = {}
self._thread: threading.Thread | None = None
+ self._watch_process: subprocess.Popen | None = None
+ self._info_error = ""
+ self._proc_error = ""
+ self._watch_error = "npu-smi watch starting"
+ self.process_available = False
def adapters(self) -> list[Adapter]:
return [Adapter(luid=key, name=c.name, vendor_id=VENDOR_HUAWEI,
@@ -184,66 +199,92 @@ class AscendCounters:
chip_key = staticmethod(_pci_key) # exposed so the Collector can match adapter keys
def start(self) -> None:
- """Spawn the watch-stream reader (once). Cheap: one npu-smi process, a line per second."""
- if self._thread is not None or not self.ok:
+ """Start or retry the watch process without leaving an unowned child."""
+ now = time.monotonic()
+ if not self.ok or (self._thread is not None and self._thread.is_alive()):
return
- self._thread = threading.Thread(target=self._watch_loop, daemon=True)
- self._thread.start()
-
- def _watch_loop(self) -> None:
+ if now - self._last_watch_start < self.WATCH_RETRY_S:
+ return
+ self._last_watch_start = now
path = _npu_path()
if path is None:
- self._watch_dead = True
+ self._watch_error = "npu-smi watch unavailable: npu-smi not found"
return
try:
- proc = subprocess.Popen([path, "info", "watch"], stdout=subprocess.PIPE,
- stderr=subprocess.DEVNULL, text=True)
- except OSError:
- self._watch_dead = True
+ args = [path, "info", "watch"]
+ if len(self.chips) == 1:
+ args += ["-i", str(self.chips[0].npu), "-c", str(self.chips[0].chip)]
+ self._watch_process = subprocess.Popen(args, stdout=subprocess.PIPE,
+ stderr=subprocess.DEVNULL, text=True, errors="replace")
+ except OSError as error:
+ self._watch_error = f"npu-smi watch unavailable: {error}"
return
+ self._watch_error = "npu-smi watch starting"
+ self._thread = threading.Thread(target=self._watch_loop, args=(self._watch_process,), daemon=True)
+ self._thread.start()
+
+ def _watch_loop(self, proc: subprocess.Popen) -> None:
assert proc.stdout is not None
- for line in proc.stdout:
- row = parse_watch_line(line)
- if row:
- with self._lock:
- self._watch[(row["npu"], row["chip"])] = row
- self._watch_dead = True # stream ended (tool exit or broken pipe)
try:
- proc.wait(timeout=5)
- except subprocess.SubprocessError:
- proc.kill()
+ for line in proc.stdout:
+ row = parse_watch_line(line)
+ if row:
+ with self._lock:
+ self._watch[(row["npu"], row["chip"])] = (time.monotonic(), row)
+ self._watch_error = ""
+ finally:
+ proc.stdout.close()
+ proc.wait()
+ with self._lock:
+ self._watch.clear()
+ self._watch_error = "npu-smi watch stopped"
def _refresh(self, now: float) -> None:
"""Slow cadence: absolute memory from `info`, per-process memory from proc-mem."""
self.start()
if now - self._last_info >= self.INFO_REFRESH_S:
self._last_info = now
- for c in parse_info(_run(["info"])):
- chip = self.by_key.get((_pci_key(c["bus"]) << 8) | c["chip"]) if c["bus"] else None
- if chip and c["mem_total"] is not None:
- chip.mem_total = c["mem_total"] * _MB
- chip.mem_used = (c["mem_used"] or 0) * _MB
+ try:
+ rows = parse_info(_run(["info"]))
+ by_key = {(_pci_key(c["bus"]) << 8) | c["chip"]: c for c in rows}
+ for key, chip in self.by_key.items():
+ c = by_key.get(key)
+ chip.mem_used = c["mem_used"] * _MB if c and c["mem_used"] is not None else None
+ if c and c["mem_total"] is not None:
+ chip.mem_total = c["mem_total"] * _MB
+ self._info_error = "" if all(c.mem_used is not None for c in self.chips) else \
+ "npu-smi device memory unavailable: selected chip missing or output unrecognized"
+ except OSError as error:
+ for chip in self.chips:
+ chip.mem_used = None
+ self._info_error = f"npu-smi device memory unavailable: {error}"
if now - self._last_proc >= self.PROC_REFRESH_S:
self._last_proc = now
- procs: dict[int, ProcGpu] = {}
- for c in self.chips:
- for pid, name, mb in parse_proc_mem(
- _run(["info", "-t", "proc-mem", "-i", str(c.npu), "-c", str(c.chip)])):
- g = procs.setdefault(pid, ProcGpu())
- g.dedicated += mb * _MB
- self._procs = procs
+ self._procs = {}
+ try:
+ for key, c in self.by_key.items():
+ per: dict[int, ProcGpu] = {}
+ for pid, name, size in parse_proc_mem(
+ _run(["info", "-t", "proc-mem", "-i", str(c.npu), "-c", str(c.chip)])):
+ g = per.setdefault(pid, ProcGpu(available={"dedicated"}))
+ g.dedicated += size
+ self._procs[key] = per
+ self.process_available = True
+ self._proc_error = ""
+ except (OSError, ValueError) as error:
+ self._procs = {}
+ self.process_available = False
+ self._proc_error = f"npu-smi process memory unavailable: {error}"
def sample(self) -> tuple[dict[int, dict[int, ProcGpu]], dict[int, AdapterGpu]]:
"""({luid: {pid: ProcGpu}}, {luid: AdapterGpu}); per-chip process memory."""
if not self.ok:
return {}, {}
- self._refresh(time.time())
+ self._refresh(time.monotonic())
procs: dict[int, dict[int, ProcGpu]] = defaultdict(dict)
adapters: dict[int, AdapterGpu] = defaultdict(AdapterGpu)
for key, chip in self.by_key.items():
- per = procs[key]
- for pid, g in getattr(self, "_procs", {}).items():
- per[pid] = ProcGpu(dedicated=g.dedicated, available={"dedicated"})
+ procs[key] = self._procs.get(key, {})
a = adapters[key]
if chip.mem_used is not None:
a.dedicated = float(chip.mem_used)
@@ -251,28 +292,32 @@ class AscendCounters:
def watch_row(self, chip: AscendChip) -> dict | None:
with self._lock:
- return self._watch.get((chip.npu, chip.chip))
+ entry = self._watch.get((chip.npu, chip.chip))
+ if entry is None or time.monotonic() - entry[0] > self.WATCH_STALE_S:
+ return None
+ return entry[1]
+
+ @property
+ def status(self) -> str:
+ with self._lock:
+ watch_error = self._watch_error
+ if (not watch_error or (watch_error == "npu-smi watch starting" and
+ time.monotonic() - self._last_watch_start > self.WATCH_STALE_S)) \
+ and any(self.watch_row(chip) is None for chip in self.chips):
+ watch_error = "npu-smi watch data unavailable or stale"
+ return ". ".join(filter(None, (self._info_error, self._proc_error, watch_error)))
def read_device(self, chip: AscendChip, index: int) -> DeviceTelemetry:
"""NVIDIA-shaped telemetry for one chip; watch data is live, sysfs fills the PCIe link."""
row = self.watch_row(chip)
- if row is None and not self._watch_dead:
- self.start()
- time.sleep(0.2) # give the just-started stream a moment (one-shot mode)
- row = self.watch_row(chip)
- if row is None: # watch unavailable: one synchronous `info` (covers all chips)
- for c in parse_info(_run(["info"])):
- if c["npu"] == chip.npu and c["chip"] == chip.chip:
- row = {"temp": c["temp"], "ai_core": c["aicore"] or 0, "ai_cpu": 0,
- "ctrl_cpu": 0, "mem_pct": 0, "mem_bw": 0}
dev = DeviceTelemetry(
index=index, name=chip.name, pci_bus=chip.pdev, driver_model="devdrv",
source="npu-smi", mem_total=chip.mem_total or None, mem_used=chip.mem_used,
)
if row:
dev.temp = row.get("temp")
- dev.util_gpu = float(row.get("ai_core") or 0)
- dev.util_mem = float(row.get("mem_bw") or 0) or None
+ dev.util_gpu = float(row["ai_core"])
+ dev.util_mem = float(row["mem_bw"])
speed = _read_sysfs(f"/sys/bus/pci/devices/{chip.pdev}/current_link_speed")
if speed:
gen = {"2.5": 1, "5.0": 2, "8.0": 3, "16.0": 4, "32.0": 5}.get(speed.split()[0])
@@ -281,7 +326,19 @@ class AscendCounters:
return dev
def close(self) -> None:
- pass # the watch thread is a daemon; it dies with the process
+ proc = self._watch_process
+ if proc is not None and proc.poll() is None:
+ try:
+ proc.terminate()
+ except ProcessLookupError:
+ pass # the child exited after poll()
+ try:
+ proc.wait(timeout=5)
+ except subprocess.TimeoutExpired:
+ proc.kill()
+ proc.wait()
+ if self._thread is not None:
+ self._thread.join(timeout=5)
def _read_sysfs(path: str) -> str | None:
diff --git a/src/gputaskman/collector.py b/src/gputaskman/collector.py
index c75d2a1..7a8e323 100644
--- a/src/gputaskman/collector.py
+++ b/src/gputaskman/collector.py
@@ -128,6 +128,7 @@ class Snapshot:
shared_is_spill: bool = True
counter_status: str = "" # Windows counter availability or adapter matching problem
counter_adapter_matched: bool | None = None # Windows PDH LUID matched the selected DXGI adapter
+ process_memory_available: bool = True
nv_lost: bool = False # NVML faulted for this GPU after a disable/re-enable; restart restores it
@property
def host_data(self) -> bool:
@@ -135,7 +136,8 @@ class Snapshot:
@property
def attributed(self) -> int | None:
- if self.counter_adapter_matched is False or any("dedicated" not in p.gpu.available for p in self.procs):
+ if not self.process_memory_available or self.counter_adapter_matched is False or any(
+ "dedicated" not in p.gpu.available for p in self.procs):
return None
return int(sum(p.gpu.dedicated for p in self.procs))
@@ -204,7 +206,7 @@ class Collector:
@property
def backend(self) -> str:
- if self.ascend_chip is not None:
+ if BACKEND == "nvml" and self.adapter and self.adapter.vendor_id == VENDOR_HUAWEI:
return "ascend" # Huawei NPU: npu-smi is the source, not NVML/DRM
return "drm" if self.drm_card is not None else BACKEND
@@ -234,18 +236,19 @@ class Collector:
drm.start() # engine baseline over every process
if scan_budget is None:
drm.scan_budget = None # one-shot output sees every process
- # Huawei Ascend NPU (no DRM): per-chip processes from npu-smi, live metrics from a
- # `npu-smi info watch` stream. Owned by LinuxCounters (one watch stream for all chips).
+ # Huawei Ascend NPU (no DRM): one selected chip and its owned watch process.
self.ascend_chip = None
self.ascend = None
- if (BACKEND == "posix" or sys.platform == "linux") and self.adapter \
- and self.adapter.vendor_id == VENDOR_HUAWEI:
- asc = getattr(self.counters, "ascend", None)
- if asc is not None and asc.ok:
- self.ascend = asc
- self.ascend_chip = next(
- (c for c in asc.chips
- if (asc.chip_key(c.pdev) << 8) | c.chip == self.adapter.luid), None)
+ if BACKEND == "nvml" and self.adapter and self.adapter.vendor_id == VENDOR_HUAWEI:
+ from dataclasses import replace
+ from .ascend import AscendCounters, cached_chips
+
+ chip = next((c for c in cached_chips()
+ if (AscendCounters.chip_key(c.pdev) << 8) | c.chip == self.adapter.luid), None)
+ if chip is not None:
+ self.ascend_chip = replace(chip)
+ self.ascend = AscendCounters(chips=[self.ascend_chip])
+ self.counters.ascend = self.ascend
self.inspector = ProcInspector()
self.prober = ServerProber()
self.scan_budget = scan_budget # None = scan every candidate now (one-shot output)
@@ -326,11 +329,11 @@ class Collector:
from .drm import read_device
nv = read_device(self.drm_card, self.gpu_index)
+ elif self.ascend_chip is not None:
+ nv = self.ascend.read_device(self.ascend_chip, self.gpu_index)
elif self.adapter and self.adapter.vendor_id != VENDOR_NVIDIA:
# NVML can't describe this GPU, and a sweep of every NVIDIA device costs ~60 ms per device.
nv = (self.adl.sample() or self.amdsmi.sample()) if self.adapter.vendor_id == VENDOR_AMD else None
- elif self.ascend_chip is not None:
- nv = self.ascend.read_device(self.ascend_chip, self.gpu_index)
else:
nvs = self.nvml.sample() if self.nvml.ok else []
nv = select_nv_device(self.adapter, nvs, self.gpu_index, self.adapters)
@@ -366,6 +369,12 @@ class Collector:
snap.counter_status = " ".join(filter(None, (snap.counter_status, self.adapter.identity_status)))
if self.drm_card is not None:
snap.process_status = self.counters.drm.status
+ if self.ascend is not None:
+ snap.process_memory_available = self.ascend.process_available
+ snap.counter_status = self.ascend.status
+ elif self.backend == "ascend":
+ snap.process_memory_available = False
+ snap.counter_status = "Ascend chip unavailable: npu-smi could not match the selected adapter"
live = [(pid, g) for pid, g in procs.items() if _is_live(g)]
live.sort(key=lambda x: -(x[1].shown_vram or 0))
for pid, g in live:
diff --git a/src/gputaskman/linux.py b/src/gputaskman/linux.py
index 29207f2..d09a324 100644
--- a/src/gputaskman/linux.py
+++ b/src/gputaskman/linux.py
@@ -78,13 +78,10 @@ def enumerate_adapters() -> list[Adapter]:
seen = {a.luid for a in out}
out += [a for a in drm.adapters(drm.cards()) if a.luid not in seen]
# Ascend NPUs (no DRM): dual-chip cards yield one adapter per chip.
- try:
- from . import ascend
+ from . import ascend
- seen |= {a.luid for a in out}
- out += [a for a in ascend.cached_chips_adapters() if a.luid not in seen]
- except Exception:
- pass # no npu-smi: the backend is simply absent
+ seen |= {a.luid for a in out}
+ out += [a for a in ascend.cached_chips_adapters() if a.luid not in seen]
return out
@@ -244,15 +241,8 @@ class LinuxCounters:
drm = DrmCounters() # cheap: only lists cards; the Collector starts it if an AMD GPU is selected
self.drm = drm if drm.ok else None
- # Huawei Ascend NPUs (no DRM): npu-smi-based, own watch stream + slow proc-mem refresh.
+ # A Collector attaches an Ascend provider only when an Ascend chip is selected.
self.ascend = None
- if not self.wsl:
- try:
- from .ascend import AscendCounters
-
- self.ascend = AscendCounters()
- except Exception:
- self.ascend = None
if N is None:
return
try:
@@ -371,6 +361,8 @@ class LinuxCounters:
def close(self) -> None:
if self.drm is not None:
self.drm.close()
+ if self.ascend is not None:
+ self.ascend.close()
if self.ok:
try:
N.nvmlShutdown()
diff --git a/tests/test_ascend.py b/tests/test_ascend.py
new file mode 100644
index 0000000..0fd8d07
--- /dev/null
+++ b/tests/test_ascend.py
@@ -0,0 +1,174 @@
+from __future__ import annotations
+
+import subprocess
+import sys
+import time
+import unittest
+from unittest.mock import Mock, patch
+
+from gputaskman import ascend, linux
+from gputaskman.collector import Collector
+from gputaskman.telemetry import DeviceTelemetry
+
+
+MB = 1024 * 1024
+INFO = (
+ "| 0 310P3 | OK | NA 45 0 / 0 |\n"
+ "| 0 0 | 0000:0a:00.0 | 0 12 / 100 |\n"
+ "| 0 310P3 | OK | NA 45 0 / 0 |\n"
+ "| 1 1 | 0000:0b:00.0 | 0 24 / 100 |\n"
+)
+
+
+def proc(pid: int, mb: int) -> str:
+ return f"Process id:{pid} Process name:worker Process memory(MB):{mb}"
+
+
+class AscendTests(unittest.TestCase):
+ def setUp(self):
+ self.chips = [
+ ascend.AscendChip(0, 0, "chip 0", "0000:0a:00.0", 100 * MB),
+ ascend.AscendChip(0, 1, "chip 1", "0000:0b:00.0", 100 * MB),
+ ]
+ self.counters = ascend.AscendCounters(self.chips)
+
+ def test_hexadecimal_pci_and_unavailable_core(self):
+ rows = ascend.parse_info(INFO.replace("0 12 / 100", "NA 12 / 100"))
+ self.assertEqual([r["bus"] for r in rows], ["0000:0a:00.0", "0000:0b:00.0"])
+ self.assertIsNone(rows[0]["aicore"])
+
+ def test_per_chip_attribution_never_combines_or_copies_other_chip(self):
+ def run(args):
+ if args == ["info"]:
+ return INFO
+ return proc(12, 10) if args[-1] == "0" else proc(12, 20) + "\n" + proc(13, 3)
+
+ with patch("gputaskman.ascend._run", side_effect=run), patch.object(self.counters, "start"):
+ per, adapters = self.counters.sample()
+ key0, key1 = self.counters.by_key
+ self.assertEqual({pid: gpu.dedicated for pid, gpu in per[key0].items()}, {12: 10 * MB})
+ self.assertEqual({pid: gpu.dedicated for pid, gpu in per[key1].items()},
+ {12: 20 * MB, 13: 3 * MB})
+ self.assertEqual(adapters[key0].dedicated, 12 * MB)
+ self.assertTrue(self.counters.process_available)
+
+ def test_failed_refresh_clears_previous_values_and_recovers(self):
+ def run(args):
+ return INFO if args == ["info"] else proc(12, 10)
+
+ with patch("gputaskman.ascend._run", side_effect=run), patch.object(self.counters, "start"):
+ self.counters._refresh(100)
+ with patch("gputaskman.ascend._run", side_effect=OSError("timed out")), \
+ patch.object(self.counters, "start"):
+ self.counters._refresh(111)
+ self.assertEqual(self.counters._procs, {})
+ self.assertFalse(self.counters.process_available)
+ self.assertTrue(all(c.mem_used is None for c in self.chips))
+ self.assertIn("device memory unavailable", self.counters.status)
+ self.assertIn("process memory unavailable", self.counters.status)
+ with patch("gputaskman.ascend._run", side_effect=run), patch.object(self.counters, "start"):
+ self.counters._refresh(122)
+ self.assertTrue(self.counters.process_available)
+ self.assertNotIn("memory unavailable", self.counters.status)
+
+ def test_bad_process_record_is_not_treated_as_no_processes(self):
+ with self.assertRaises(ValueError):
+ ascend.parse_proc_mem("Process id:7 Process name:worker Process memory(MB):NA")
+ with self.assertRaises(ValueError):
+ ascend.parse_proc_mem("")
+ self.assertEqual(ascend.parse_proc_mem("No running processes found"), [])
+
+ def test_driver_byte_and_megabyte_process_values(self):
+ data = "Process id:38329 Process name:hlt_host_devmm_ Process memory(B):100683776\n" + proc(7, 3)
+ self.assertEqual(ascend.parse_proc_mem(data),
+ [(38329, "hlt_host_devmm_", 100683776), (7, "worker", 3 * MB)])
+
+ def test_watch_zero_and_stale_sample(self):
+ chip = self.chips[0]
+ self.counters._watch[(chip.npu, chip.chip)] = (
+ 10.0, {"temp": 40, "ai_core": 0, "mem_bw": 0})
+ self.counters._watch_error = ""
+ with patch("gputaskman.ascend.time.monotonic", return_value=11):
+ dev = self.counters.read_device(chip, 0)
+ self.assertEqual((dev.util_gpu, dev.util_mem), (0.0, 0.0))
+ with patch("gputaskman.ascend.time.monotonic", return_value=14):
+ dev = self.counters.read_device(chip, 0)
+ self.assertIsNone(dev.util_gpu)
+ self.assertIsNone(dev.util_mem)
+ self.assertIn("stale", self.counters.status)
+
+ def test_close_reaps_watch_child(self):
+ self.counters = ascend.AscendCounters([self.chips[1]])
+ popen = subprocess.Popen
+ commands = []
+
+ def start_child(*args, **kwargs):
+ commands.append(args[0])
+ return popen([sys.executable, "-u", "-c",
+ "import time; print('0 1 0 40 1 0 0 0 0', flush=True); time.sleep(60)"],
+ **kwargs)
+
+ with patch("gputaskman.ascend._npu_path", return_value="npu-smi"), \
+ patch("gputaskman.ascend.subprocess.Popen", side_effect=start_child):
+ self.counters.start()
+ child = self.counters._watch_process
+ try:
+ deadline = time.monotonic() + 5
+ while self.counters.watch_row(self.chips[1]) is None and time.monotonic() < deadline:
+ time.sleep(0.01)
+ self.assertIsNotNone(self.counters.watch_row(self.chips[1]))
+ finally:
+ self.counters.close()
+ self.assertIsNotNone(child.poll())
+ self.assertFalse(self.counters._thread.is_alive())
+ self.assertEqual(commands, [["npu-smi", "info", "watch", "-i", "0", "-c", "1"]])
+
+ def test_linux_counters_do_not_probe_ascend_without_selection(self):
+ with patch("gputaskman.linux.is_wsl", return_value=False), \
+ patch("gputaskman.drm.DrmCounters") as drm, \
+ patch("gputaskman.linux.N", None), patch("gputaskman.ascend.probe") as probe:
+ drm.return_value.ok = False
+ counters = linux.LinuxCounters()
+ counters.sample()
+ counters.close()
+ self.assertIsNone(counters.ascend)
+ probe.assert_not_called()
+
+ def test_collector_attaches_only_selected_chip(self):
+ adapter = self.counters.adapters()[1]
+ counters = Mock(drm=None)
+ with patch("gputaskman.collector.BACKEND", "nvml"), \
+ patch("gputaskman.collector.GpuCounters", return_value=counters), \
+ patch("gputaskman.collector.Nvml"), patch("gputaskman.collector.AmdSmi"), \
+ patch("gputaskman.collector.Adl"), \
+ patch("gputaskman.ascend.cached_chips", return_value=self.chips):
+ collector = Collector(adapters=[adapter])
+ self.assertEqual(len(collector.ascend.chips), 1)
+ self.assertEqual(collector.ascend.chips[0].chip, 1)
+ self.assertIsNot(collector.ascend.chips[0], self.chips[1])
+ self.assertIs(counters.ascend, collector.ascend)
+
+ def test_collector_dispatches_and_propagates_unavailable_processes(self):
+ c = Collector.__new__(Collector)
+ chip = self.chips[0]
+ c.adapter = self.counters.adapters()[0]
+ c.adapters = [c.adapter]
+ c.ascend_chip = chip
+ c.ascend = Mock(process_available=False, status="process memory unavailable")
+ c.ascend.read_device.return_value = DeviceTelemetry(index=0, name=chip.name, source="npu-smi")
+ c.counters = Mock(outside={})
+ c.counters.sample.return_value = ({}, {})
+ c.nvml = Mock(ok=False)
+ c.gpu_index, c.scan_budget, c.host, c.host_status = 0, None, None, ""
+ c.inspector, c.prober = Mock(), Mock()
+ with patch("gputaskman.collector.BACKEND", "nvml"):
+ snap = c.sample()
+ self.assertEqual(snap.backend, "ascend")
+ self.assertEqual(snap.nv.source, "npu-smi")
+ self.assertIsNone(snap.attributed)
+ self.assertIn("process memory unavailable", snap.counter_status)
+ c.ascend.read_device.assert_called_once_with(chip, 0)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tests/test_linux.py b/tests/test_linux.py
index d5243d2..efe7041 100644
--- a/tests/test_linux.py
+++ b/tests/test_linux.py
@@ -26,6 +26,7 @@ def setUpModule() -> None:
def tearDownModule() -> None:
_no_drm.stop()
+ _no_ascend.stop()
NOT_AVAILABLE = (1 << 64) - 1 # NVML_VALUE_NOT_AVAILABLE as an unsigned long long