A GPU task manager for Windows, Linux, and WSL: per-process VRAM, GPU activity, and LLM model/KV-cache insights (Windows AMD and NVIDIA).
Python
Fix Ascend chip accounting and telemetry
Scott Hanselman7f88442fa96d
diff --git a/README.md b/README.mdindex 12d2db8..2ac6020 100644--- a/README.md+++ b/README.md@@ -118,8 +118,8 @@ skips the consistency check, so do not use it after changing dependencies.Tagged releases include portable Linux builds: `gputaskman-VERSION-linux-x64.tar.gz` and`gputaskman-VERSION-linux-arm64.tar.gz`. They need glibc 2.35 or newer (Ubuntu 22.04, Debian 12,-or newer) and an NVIDIA driver (or, on native Linux, an AMD GPU on the `amdgpu` driver or an Intel GPU on `i915`), and work-natively, in containers, and in WSL:+or newer) and an NVIDIA driver (or, on native Linux, an AMD GPU on the `amdgpu` driver, an Intel GPU on `i915`,+or a Huawei Ascend 310P with `npu-smi`). NVIDIA works natively, in containers, and in WSL:```bashtar -xzf gputaskman-VERSION-linux-x64.tar.gz@@ -137,6 +137,13 @@ uv run --locked gputaskmanWhat can be measured depends on where it runs, and the UI says which mode it is in.+On native Linux, Ascend chips are listed individually. Their device memory and per-process memory+come from `npu-smi`; AI-core and memory-bandwidth activity come from its watch stream, not+per-process engine counters. Memory refreshes every 10 seconds, and a failed refresh or stale watch+is reported as unavailable. The contributor reported testing an Atlas 300I Duo with a vLLM+workload; driver version and these subsequent fixes have not been independently validated on hardware.+Ascend container and PID-namespace behavior is not validated.+| | Native Linux | WSL, interop on (default) | WSL, interop off or `--no-host` ||---|---|---|---|| Device totals, load, temperature, power, clocks | NVML | NVML | NVML |@@ -855,6 +862,7 @@ Identification is designed to stay cheap. Measured on one Windows machine:| vLLM `/v1/models` | 1.32 ms median, 21.36 ms max in 30 loopback fixture GETs; cached lookup ~0.0004 ms (Windows 11 build 28000, ARM64; no vLLM runtime/GPU workload) | One GET, 0.5 s timeout and 4 MiB response cap; success cached 3 s, misses/401 cached 30 s. No redirects; actual Linux/vLLM latency unmeasured || `psutil.open_files()` | 0.6-2.5 s per process | Never || AMD and Intel on Linux: DRM fdinfo scan (`readlink` every fd, read fdinfo of `/dev/dri` ones) | ~9 ms for ~8,000 fds (AMD, Linux desktop; Intel not measured) | Every refresh, only when an amdgpu or Intel card is shown; listing fds is bounded to 50 ms in the TUI and resumes next refresh, known GPU fds are still read |+| Ascend on Linux: `npu-smi` | Contributor reports ~0.5 s per command on an Atlas 300I Duo; not independently measured | Discovery is cached for 120 s. Each selected chip runs one `info` and one `proc-mem` command every 10 s and owns a watch process; multiple selected chips add their synchronous command costs || macOS: `ioreg` device stats + Metal clients | ~10 ms | Every refresh || macOS: memory-map scan (`proc_pidinfo` region walk) | ~4 ms per process | Same policy as the memory-map scan above || macOS: `footprint` GPU memory | ~70 ms per process, background thread | Busy and new processes every ~2 s, idle ones every 30 s (54 runs in 40 s with 27 GPU processes, M4 Mac mini) |diff --git a/src/gputaskman/ascend.py b/src/gputaskman/ascend.pyindex d5f6936..c87a945 100644--- a/src/gputaskman/ascend.py+++ b/src/gputaskman/ascend.py@@ -11,8 +11,8 @@ appears as two adapters, one per chip, each with its own DDR bank.from __future__ import annotations-import osimport re+import shutilimport subprocessimport threadingimport time@@ -33,26 +33,23 @@ def _npu_path() -> str | None:"""Path to npu-smi, or None when the tool (and likely the driver) is absent."""global _NPUSMIif _NPUSMI is None:- for candidate in ("npu-smi", "/usr/local/bin/npu-smi"):- try:- subprocess.run([candidate, "-h"], capture_output=True, timeout=5)- _NPUSMI = candidate- break- except (OSError, subprocess.SubprocessError):- continue+ _NPUSMI = shutil.which("npu-smi") or shutil.which("/usr/local/bin/npu-smi")return _NPUSMIdef _run(args: list[str], timeout: float = 10.0) -> str:- """npu-smi output, or '' on any failure (missing tool, error exit, timeout)."""+ """Return npu-smi output; callers must distinguish failure from an empty result."""path = _npu_path()if path is None:- return ""+ raise OSError("npu-smi not found")try:- r = subprocess.run([path, *args], capture_output=True, text=True, timeout=timeout)- except (OSError, subprocess.SubprocessError):- return ""- return r.stdout if r.returncode == 0 else ""+ r = subprocess.run([path, *args], capture_output=True, text=True, errors="replace",+ timeout=timeout, check=True)+ except subprocess.TimeoutExpired as error:+ raise OSError("npu-smi timed out") from error+ except subprocess.CalledProcessError as error:+ raise OSError(f"npu-smi exited with status {error.returncode}") from error+ return r.stdout@dataclass@@ -82,7 +79,7 @@ def parse_info(text: str) -> list[dict]:if len(cells) < 3:continuem = re.match(r"^(\d+)\s+(\d+)$", cells[0])- if m and re.match(r"^\d+:\d+:\d+\.\d+$", cells[1]):+ if m and re.fullmatch(r"[0-9a-fA-F]+:[0-9a-fA-F]+:[0-9a-fA-F]+\.[0-9a-fA-F]+", cells[1]):if head is None:continuehm = re.match(r"^(\d+)\s+(\S+)$", head[0])@@ -96,7 +93,7 @@ def parse_info(text: str) -> list[dict]:"health": head[1], "power": ht[0] if ht else None,"temp": int(ht[1]) if len(ht) > 1 and ht[1].lstrip("-").isdigit() else None,"chip": int(m.group(1)), "device": int(m.group(2)), "bus": cells[1],- "aicore": int(cells[2].split()[0]) if cells[2].split() else None,+ "aicore": int(cells[2].split()[0]) if cells[2].split() and cells[2].split()[0].isdigit() else None,"mem_used": int(mt.group(1)) if mt else None,"mem_total": int(mt.group(2)) if mt else None,})@@ -123,10 +120,15 @@ def parse_watch_line(line: str) -> dict | None:def parse_proc_mem(text: str) -> list[tuple[int, str, int]]:- """`npu-smi info -t proc-mem` rows -> (pid, name, megabytes)."""+ """`npu-smi info -t proc-mem` rows -> (pid, name, bytes)."""out = []- for m in re.finditer(r"Process id:(\d+)\s+Process name:(\S+)\s+Process memory\(MB\):(\d+)", text):- out.append((int(m.group(1)), m.group(2), int(m.group(3))))+ for m in re.finditer(+ r"Process id:\s*(\d+)\s+Process name:\s*(\S+)\s+Process memory\((B|MB)\):\s*(\d+)", text):+ size = int(m.group(4)) * (_MB if m.group(3) == "MB" else 1)+ out.append((int(m.group(1)), m.group(2), size))+ if len(out) != text.count("Process id:") or (not out and+ not re.search(r"no (?:running )?process(?:es)? (?:found|running)", text, re.IGNORECASE)):+ raise ValueError("unrecognized npu-smi process output")return out@@ -136,13 +138,19 @@ def _pci_key(pdev: str) -> int:def probe() -> list[AscendChip]:"""Enumerate every Ascend chip npu-smi can see (two subprocess calls, ~1s)."""- text = _run(["info"])- if not text:+ if _npu_path() is None:+ return []+ try:+ text = _run(["info"])+ except OSError:return []chips = []for c in parse_info(text):product = ""- board = _run(["info", "-t", "board", "-i", str(c["npu"])])+ try:+ board = _run(["info", "-t", "board", "-i", str(c["npu"])])+ except OSError:+ board = ""pm = re.search(r"Product Name\s*:\s*(.+)", board)if pm:product = pm.group(1).strip()@@ -152,7 +160,7 @@ def probe() -> list[AscendChip]:chip = AscendChip(npu=c["npu"], chip=c["chip"], name=name, pdev=c["bus"], product=product)if c["mem_total"] is not None:chip.mem_total = c["mem_total"] * _MB- chip.mem_used = (c["mem_used"] or 0) * _MB+ chip.mem_used = c["mem_used"] * _MB if c["mem_used"] is not None else Nonechips.append(chip)return chips@@ -162,19 +170,26 @@ class AscendCounters:PROC_REFRESH_S = 10.0 # proc-mem is ~0.5s per chip; don't pay it every sampleINFO_REFRESH_S = 10.0 # absolute memory refresh (watch only carries percentages)+ WATCH_STALE_S = 3.0+ WATCH_RETRY_S = 10.0def __init__(self, chips: list[AscendChip] | None = None) -> None:self.ok = Falseself.chips = chips if chips is not None else probe()self.ok = bool(self.chips)self.by_key = {(_pci_key(c.pdev) << 8) | c.chip: c for c in self.chips}- self._watch: dict[tuple[int, int], dict] = {} # (npu, chip) -> latest watch row- self._watch_dead = False+ self._watch: dict[tuple[int, int], tuple[float, dict]] = {}self._lock = threading.Lock()- self._last_proc = 0.0- self._last_info = 0.0- self._procs: dict[int, ProcGpu] = {}+ self._last_proc = float("-inf")+ self._last_info = float("-inf")+ self._last_watch_start = float("-inf")+ self._procs: dict[int, dict[int, ProcGpu]] = {}self._thread: threading.Thread | None = None+ self._watch_process: subprocess.Popen | None = None+ self._info_error = ""+ self._proc_error = ""+ self._watch_error = "npu-smi watch starting"+ self.process_available = Falsedef adapters(self) -> list[Adapter]:return [Adapter(luid=key, name=c.name, vendor_id=VENDOR_HUAWEI,@@ -184,66 +199,92 @@ class AscendCounters:chip_key = staticmethod(_pci_key) # exposed so the Collector can match adapter keysdef start(self) -> None:- """Spawn the watch-stream reader (once). Cheap: one npu-smi process, a line per second."""- if self._thread is not None or not self.ok:+ """Start or retry the watch process without leaving an unowned child."""+ now = time.monotonic()+ if not self.ok or (self._thread is not None and self._thread.is_alive()):return- self._thread = threading.Thread(target=self._watch_loop, daemon=True)- self._thread.start()-- def _watch_loop(self) -> None:+ if now - self._last_watch_start < self.WATCH_RETRY_S:+ return+ self._last_watch_start = nowpath = _npu_path()if path is None:- self._watch_dead = True+ self._watch_error = "npu-smi watch unavailable: npu-smi not found"returntry:- proc = subprocess.Popen([path, "info", "watch"], stdout=subprocess.PIPE,- stderr=subprocess.DEVNULL, text=True)- except OSError:- self._watch_dead = True+ args = [path, "info", "watch"]+ if len(self.chips) == 1:+ args += ["-i", str(self.chips[0].npu), "-c", str(self.chips[0].chip)]+ self._watch_process = subprocess.Popen(args, stdout=subprocess.PIPE,+ stderr=subprocess.DEVNULL, text=True, errors="replace")+ except OSError as error:+ self._watch_error = f"npu-smi watch unavailable: {error}"return+ self._watch_error = "npu-smi watch starting"+ self._thread = threading.Thread(target=self._watch_loop, args=(self._watch_process,), daemon=True)+ self._thread.start()++ def _watch_loop(self, proc: subprocess.Popen) -> None:assert proc.stdout is not None- for line in proc.stdout:- row = parse_watch_line(line)- if row:- with self._lock:- self._watch[(row["npu"], row["chip"])] = row- self._watch_dead = True # stream ended (tool exit or broken pipe)try:- proc.wait(timeout=5)- except subprocess.SubprocessError:- proc.kill()+ for line in proc.stdout:+ row = parse_watch_line(line)+ if row:+ with self._lock:+ self._watch[(row["npu"], row["chip"])] = (time.monotonic(), row)+ self._watch_error = ""+ finally:+ proc.stdout.close()+ proc.wait()+ with self._lock:+ self._watch.clear()+ self._watch_error = "npu-smi watch stopped"def _refresh(self, now: float) -> None:"""Slow cadence: absolute memory from `info`, per-process memory from proc-mem."""self.start()if now - self._last_info >= self.INFO_REFRESH_S:self._last_info = now- for c in parse_info(_run(["info"])):- chip = self.by_key.get((_pci_key(c["bus"]) << 8) | c["chip"]) if c["bus"] else None- if chip and c["mem_total"] is not None:- chip.mem_total = c["mem_total"] * _MB- chip.mem_used = (c["mem_used"] or 0) * _MB+ try:+ rows = parse_info(_run(["info"]))+ by_key = {(_pci_key(c["bus"]) << 8) | c["chip"]: c for c in rows}+ for key, chip in self.by_key.items():+ c = by_key.get(key)+ chip.mem_used = c["mem_used"] * _MB if c and c["mem_used"] is not None else None+ if c and c["mem_total"] is not None:+ chip.mem_total = c["mem_total"] * _MB+ self._info_error = "" if all(c.mem_used is not None for c in self.chips) else \+ "npu-smi device memory unavailable: selected chip missing or output unrecognized"+ except OSError as error:+ for chip in self.chips:+ chip.mem_used = None+ self._info_error = f"npu-smi device memory unavailable: {error}"if now - self._last_proc >= self.PROC_REFRESH_S:self._last_proc = now- procs: dict[int, ProcGpu] = {}- for c in self.chips:- for pid, name, mb in parse_proc_mem(- _run(["info", "-t", "proc-mem", "-i", str(c.npu), "-c", str(c.chip)])):- g = procs.setdefault(pid, ProcGpu())- g.dedicated += mb * _MB- self._procs = procs+ self._procs = {}+ try:+ for key, c in self.by_key.items():+ per: dict[int, ProcGpu] = {}+ for pid, name, size in parse_proc_mem(+ _run(["info", "-t", "proc-mem", "-i", str(c.npu), "-c", str(c.chip)])):+ g = per.setdefault(pid, ProcGpu(available={"dedicated"}))+ g.dedicated += size+ self._procs[key] = per+ self.process_available = True+ self._proc_error = ""+ except (OSError, ValueError) as error:+ self._procs = {}+ self.process_available = False+ self._proc_error = f"npu-smi process memory unavailable: {error}"def sample(self) -> tuple[dict[int, dict[int, ProcGpu]], dict[int, AdapterGpu]]:"""({luid: {pid: ProcGpu}}, {luid: AdapterGpu}); per-chip process memory."""if not self.ok:return {}, {}- self._refresh(time.time())+ self._refresh(time.monotonic())procs: dict[int, dict[int, ProcGpu]] = defaultdict(dict)adapters: dict[int, AdapterGpu] = defaultdict(AdapterGpu)for key, chip in self.by_key.items():- per = procs[key]- for pid, g in getattr(self, "_procs", {}).items():- per[pid] = ProcGpu(dedicated=g.dedicated, available={"dedicated"})+ procs[key] = self._procs.get(key, {})a = adapters[key]if chip.mem_used is not None:a.dedicated = float(chip.mem_used)@@ -251,28 +292,32 @@ class AscendCounters:def watch_row(self, chip: AscendChip) -> dict | None:with self._lock:- return self._watch.get((chip.npu, chip.chip))+ entry = self._watch.get((chip.npu, chip.chip))+ if entry is None or time.monotonic() - entry[0] > self.WATCH_STALE_S:+ return None+ return entry[1]++ @property+ def status(self) -> str:+ with self._lock:+ watch_error = self._watch_error+ if (not watch_error or (watch_error == "npu-smi watch starting" and+ time.monotonic() - self._last_watch_start > self.WATCH_STALE_S)) \+ and any(self.watch_row(chip) is None for chip in self.chips):+ watch_error = "npu-smi watch data unavailable or stale"+ return ". ".join(filter(None, (self._info_error, self._proc_error, watch_error)))def read_device(self, chip: AscendChip, index: int) -> DeviceTelemetry:"""NVIDIA-shaped telemetry for one chip; watch data is live, sysfs fills the PCIe link."""row = self.watch_row(chip)- if row is None and not self._watch_dead:- self.start()- time.sleep(0.2) # give the just-started stream a moment (one-shot mode)- row = self.watch_row(chip)- if row is None: # watch unavailable: one synchronous `info` (covers all chips)- for c in parse_info(_run(["info"])):- if c["npu"] == chip.npu and c["chip"] == chip.chip:- row = {"temp": c["temp"], "ai_core": c["aicore"] or 0, "ai_cpu": 0,- "ctrl_cpu": 0, "mem_pct": 0, "mem_bw": 0}dev = DeviceTelemetry(index=index, name=chip.name, pci_bus=chip.pdev, driver_model="devdrv",source="npu-smi", mem_total=chip.mem_total or None, mem_used=chip.mem_used,)if row:dev.temp = row.get("temp")- dev.util_gpu = float(row.get("ai_core") or 0)- dev.util_mem = float(row.get("mem_bw") or 0) or None+ dev.util_gpu = float(row["ai_core"])+ dev.util_mem = float(row["mem_bw"])speed = _read_sysfs(f"/sys/bus/pci/devices/{chip.pdev}/current_link_speed")if speed:gen = {"2.5": 1, "5.0": 2, "8.0": 3, "16.0": 4, "32.0": 5}.get(speed.split()[0])@@ -281,7 +326,19 @@ class AscendCounters:return devdef close(self) -> None:- pass # the watch thread is a daemon; it dies with the process+ proc = self._watch_process+ if proc is not None and proc.poll() is None:+ try:+ proc.terminate()+ except ProcessLookupError:+ pass # the child exited after poll()+ try:+ proc.wait(timeout=5)+ except subprocess.TimeoutExpired:+ proc.kill()+ proc.wait()+ if self._thread is not None:+ self._thread.join(timeout=5)def _read_sysfs(path: str) -> str | None:diff --git a/src/gputaskman/collector.py b/src/gputaskman/collector.pyindex c75d2a1..7a8e323 100644--- a/src/gputaskman/collector.py+++ b/src/gputaskman/collector.py@@ -128,6 +128,7 @@ class Snapshot:shared_is_spill: bool = Truecounter_status: str = "" # Windows counter availability or adapter matching problemcounter_adapter_matched: bool | None = None # Windows PDH LUID matched the selected DXGI adapter+ process_memory_available: bool = Truenv_lost: bool = False # NVML faulted for this GPU after a disable/re-enable; restart restores it@propertydef host_data(self) -> bool:@@ -135,7 +136,8 @@ class Snapshot:@propertydef attributed(self) -> int | None:- if self.counter_adapter_matched is False or any("dedicated" not in p.gpu.available for p in self.procs):+ if not self.process_memory_available or self.counter_adapter_matched is False or any(+ "dedicated" not in p.gpu.available for p in self.procs):return Nonereturn int(sum(p.gpu.dedicated for p in self.procs))@@ -204,7 +206,7 @@ class Collector:@propertydef backend(self) -> str:- if self.ascend_chip is not None:+ if BACKEND == "nvml" and self.adapter and self.adapter.vendor_id == VENDOR_HUAWEI:return "ascend" # Huawei NPU: npu-smi is the source, not NVML/DRMreturn "drm" if self.drm_card is not None else BACKEND@@ -234,18 +236,19 @@ class Collector:drm.start() # engine baseline over every processif scan_budget is None:drm.scan_budget = None # one-shot output sees every process- # Huawei Ascend NPU (no DRM): per-chip processes from npu-smi, live metrics from a- # `npu-smi info watch` stream. Owned by LinuxCounters (one watch stream for all chips).+ # Huawei Ascend NPU (no DRM): one selected chip and its owned watch process.self.ascend_chip = Noneself.ascend = None- if (BACKEND == "posix" or sys.platform == "linux") and self.adapter \- and self.adapter.vendor_id == VENDOR_HUAWEI:- asc = getattr(self.counters, "ascend", None)- if asc is not None and asc.ok:- self.ascend = asc- self.ascend_chip = next(- (c for c in asc.chips- if (asc.chip_key(c.pdev) << 8) | c.chip == self.adapter.luid), None)+ if BACKEND == "nvml" and self.adapter and self.adapter.vendor_id == VENDOR_HUAWEI:+ from dataclasses import replace+ from .ascend import AscendCounters, cached_chips++ chip = next((c for c in cached_chips()+ if (AscendCounters.chip_key(c.pdev) << 8) | c.chip == self.adapter.luid), None)+ if chip is not None:+ self.ascend_chip = replace(chip)+ self.ascend = AscendCounters(chips=[self.ascend_chip])+ self.counters.ascend = self.ascendself.inspector = ProcInspector()self.prober = ServerProber()self.scan_budget = scan_budget # None = scan every candidate now (one-shot output)@@ -326,11 +329,11 @@ class Collector:from .drm import read_devicenv = read_device(self.drm_card, self.gpu_index)+ elif self.ascend_chip is not None:+ nv = self.ascend.read_device(self.ascend_chip, self.gpu_index)elif self.adapter and self.adapter.vendor_id != VENDOR_NVIDIA:# NVML can't describe this GPU, and a sweep of every NVIDIA device costs ~60 ms per device.nv = (self.adl.sample() or self.amdsmi.sample()) if self.adapter.vendor_id == VENDOR_AMD else None- elif self.ascend_chip is not None:- nv = self.ascend.read_device(self.ascend_chip, self.gpu_index)else:nvs = self.nvml.sample() if self.nvml.ok else []nv = select_nv_device(self.adapter, nvs, self.gpu_index, self.adapters)@@ -366,6 +369,12 @@ class Collector:snap.counter_status = " ".join(filter(None, (snap.counter_status, self.adapter.identity_status)))if self.drm_card is not None:snap.process_status = self.counters.drm.status+ if self.ascend is not None:+ snap.process_memory_available = self.ascend.process_available+ snap.counter_status = self.ascend.status+ elif self.backend == "ascend":+ snap.process_memory_available = False+ snap.counter_status = "Ascend chip unavailable: npu-smi could not match the selected adapter"live = [(pid, g) for pid, g in procs.items() if _is_live(g)]live.sort(key=lambda x: -(x[1].shown_vram or 0))for pid, g in live:diff --git a/src/gputaskman/linux.py b/src/gputaskman/linux.pyindex 29207f2..d09a324 100644--- a/src/gputaskman/linux.py+++ b/src/gputaskman/linux.py@@ -78,13 +78,10 @@ def enumerate_adapters() -> list[Adapter]:seen = {a.luid for a in out}out += [a for a in drm.adapters(drm.cards()) if a.luid not in seen]# Ascend NPUs (no DRM): dual-chip cards yield one adapter per chip.- try:- from . import ascend+ from . import ascend- seen |= {a.luid for a in out}- out += [a for a in ascend.cached_chips_adapters() if a.luid not in seen]- except Exception:- pass # no npu-smi: the backend is simply absent+ seen |= {a.luid for a in out}+ out += [a for a in ascend.cached_chips_adapters() if a.luid not in seen]return out@@ -244,15 +241,8 @@ class LinuxCounters:drm = DrmCounters() # cheap: only lists cards; the Collector starts it if an AMD GPU is selectedself.drm = drm if drm.ok else None- # Huawei Ascend NPUs (no DRM): npu-smi-based, own watch stream + slow proc-mem refresh.+ # A Collector attaches an Ascend provider only when an Ascend chip is selected.self.ascend = None- if not self.wsl:- try:- from .ascend import AscendCounters-- self.ascend = AscendCounters()- except Exception:- self.ascend = Noneif N is None:returntry:@@ -371,6 +361,8 @@ class LinuxCounters:def close(self) -> None:if self.drm is not None:self.drm.close()+ if self.ascend is not None:+ self.ascend.close()if self.ok:try:N.nvmlShutdown()diff --git a/tests/test_ascend.py b/tests/test_ascend.pynew file mode 100644index 0000000..0fd8d07--- /dev/null+++ b/tests/test_ascend.py@@ -0,0 +1,174 @@+from __future__ import annotations++import subprocess+import sys+import time+import unittest+from unittest.mock import Mock, patch++from gputaskman import ascend, linux+from gputaskman.collector import Collector+from gputaskman.telemetry import DeviceTelemetry+++MB = 1024 * 1024+INFO = (+ "| 0 310P3 | OK | NA 45 0 / 0 |\n"+ "| 0 0 | 0000:0a:00.0 | 0 12 / 100 |\n"+ "| 0 310P3 | OK | NA 45 0 / 0 |\n"+ "| 1 1 | 0000:0b:00.0 | 0 24 / 100 |\n"+)+++def proc(pid: int, mb: int) -> str:+ return f"Process id:{pid} Process name:worker Process memory(MB):{mb}"+++class AscendTests(unittest.TestCase):+ def setUp(self):+ self.chips = [+ ascend.AscendChip(0, 0, "chip 0", "0000:0a:00.0", 100 * MB),+ ascend.AscendChip(0, 1, "chip 1", "0000:0b:00.0", 100 * MB),+ ]+ self.counters = ascend.AscendCounters(self.chips)++ def test_hexadecimal_pci_and_unavailable_core(self):+ rows = ascend.parse_info(INFO.replace("0 12 / 100", "NA 12 / 100"))+ self.assertEqual([r["bus"] for r in rows], ["0000:0a:00.0", "0000:0b:00.0"])+ self.assertIsNone(rows[0]["aicore"])++ def test_per_chip_attribution_never_combines_or_copies_other_chip(self):+ def run(args):+ if args == ["info"]:+ return INFO+ return proc(12, 10) if args[-1] == "0" else proc(12, 20) + "\n" + proc(13, 3)++ with patch("gputaskman.ascend._run", side_effect=run), patch.object(self.counters, "start"):+ per, adapters = self.counters.sample()+ key0, key1 = self.counters.by_key+ self.assertEqual({pid: gpu.dedicated for pid, gpu in per[key0].items()}, {12: 10 * MB})+ self.assertEqual({pid: gpu.dedicated for pid, gpu in per[key1].items()},+ {12: 20 * MB, 13: 3 * MB})+ self.assertEqual(adapters[key0].dedicated, 12 * MB)+ self.assertTrue(self.counters.process_available)++ def test_failed_refresh_clears_previous_values_and_recovers(self):+ def run(args):+ return INFO if args == ["info"] else proc(12, 10)++ with patch("gputaskman.ascend._run", side_effect=run), patch.object(self.counters, "start"):+ self.counters._refresh(100)+ with patch("gputaskman.ascend._run", side_effect=OSError("timed out")), \+ patch.object(self.counters, "start"):+ self.counters._refresh(111)+ self.assertEqual(self.counters._procs, {})+ self.assertFalse(self.counters.process_available)+ self.assertTrue(all(c.mem_used is None for c in self.chips))+ self.assertIn("device memory unavailable", self.counters.status)+ self.assertIn("process memory unavailable", self.counters.status)+ with patch("gputaskman.ascend._run", side_effect=run), patch.object(self.counters, "start"):+ self.counters._refresh(122)+ self.assertTrue(self.counters.process_available)+ self.assertNotIn("memory unavailable", self.counters.status)++ def test_bad_process_record_is_not_treated_as_no_processes(self):+ with self.assertRaises(ValueError):+ ascend.parse_proc_mem("Process id:7 Process name:worker Process memory(MB):NA")+ with self.assertRaises(ValueError):+ ascend.parse_proc_mem("")+ self.assertEqual(ascend.parse_proc_mem("No running processes found"), [])++ def test_driver_byte_and_megabyte_process_values(self):+ data = "Process id:38329 Process name:hlt_host_devmm_ Process memory(B):100683776\n" + proc(7, 3)+ self.assertEqual(ascend.parse_proc_mem(data),+ [(38329, "hlt_host_devmm_", 100683776), (7, "worker", 3 * MB)])++ def test_watch_zero_and_stale_sample(self):+ chip = self.chips[0]+ self.counters._watch[(chip.npu, chip.chip)] = (+ 10.0, {"temp": 40, "ai_core": 0, "mem_bw": 0})+ self.counters._watch_error = ""+ with patch("gputaskman.ascend.time.monotonic", return_value=11):+ dev = self.counters.read_device(chip, 0)+ self.assertEqual((dev.util_gpu, dev.util_mem), (0.0, 0.0))+ with patch("gputaskman.ascend.time.monotonic", return_value=14):+ dev = self.counters.read_device(chip, 0)+ self.assertIsNone(dev.util_gpu)+ self.assertIsNone(dev.util_mem)+ self.assertIn("stale", self.counters.status)++ def test_close_reaps_watch_child(self):+ self.counters = ascend.AscendCounters([self.chips[1]])+ popen = subprocess.Popen+ commands = []++ def start_child(*args, **kwargs):+ commands.append(args[0])+ return popen([sys.executable, "-u", "-c",+ "import time; print('0 1 0 40 1 0 0 0 0', flush=True); time.sleep(60)"],+ **kwargs)++ with patch("gputaskman.ascend._npu_path", return_value="npu-smi"), \+ patch("gputaskman.ascend.subprocess.Popen", side_effect=start_child):+ self.counters.start()+ child = self.counters._watch_process+ try:+ deadline = time.monotonic() + 5+ while self.counters.watch_row(self.chips[1]) is None and time.monotonic() < deadline:+ time.sleep(0.01)+ self.assertIsNotNone(self.counters.watch_row(self.chips[1]))+ finally:+ self.counters.close()+ self.assertIsNotNone(child.poll())+ self.assertFalse(self.counters._thread.is_alive())+ self.assertEqual(commands, [["npu-smi", "info", "watch", "-i", "0", "-c", "1"]])++ def test_linux_counters_do_not_probe_ascend_without_selection(self):+ with patch("gputaskman.linux.is_wsl", return_value=False), \+ patch("gputaskman.drm.DrmCounters") as drm, \+ patch("gputaskman.linux.N", None), patch("gputaskman.ascend.probe") as probe:+ drm.return_value.ok = False+ counters = linux.LinuxCounters()+ counters.sample()+ counters.close()+ self.assertIsNone(counters.ascend)+ probe.assert_not_called()++ def test_collector_attaches_only_selected_chip(self):+ adapter = self.counters.adapters()[1]+ counters = Mock(drm=None)+ with patch("gputaskman.collector.BACKEND", "nvml"), \+ patch("gputaskman.collector.GpuCounters", return_value=counters), \+ patch("gputaskman.collector.Nvml"), patch("gputaskman.collector.AmdSmi"), \+ patch("gputaskman.collector.Adl"), \+ patch("gputaskman.ascend.cached_chips", return_value=self.chips):+ collector = Collector(adapters=[adapter])+ self.assertEqual(len(collector.ascend.chips), 1)+ self.assertEqual(collector.ascend.chips[0].chip, 1)+ self.assertIsNot(collector.ascend.chips[0], self.chips[1])+ self.assertIs(counters.ascend, collector.ascend)++ def test_collector_dispatches_and_propagates_unavailable_processes(self):+ c = Collector.__new__(Collector)+ chip = self.chips[0]+ c.adapter = self.counters.adapters()[0]+ c.adapters = [c.adapter]+ c.ascend_chip = chip+ c.ascend = Mock(process_available=False, status="process memory unavailable")+ c.ascend.read_device.return_value = DeviceTelemetry(index=0, name=chip.name, source="npu-smi")+ c.counters = Mock(outside={})+ c.counters.sample.return_value = ({}, {})+ c.nvml = Mock(ok=False)+ c.gpu_index, c.scan_budget, c.host, c.host_status = 0, None, None, ""+ c.inspector, c.prober = Mock(), Mock()+ with patch("gputaskman.collector.BACKEND", "nvml"):+ snap = c.sample()+ self.assertEqual(snap.backend, "ascend")+ self.assertEqual(snap.nv.source, "npu-smi")+ self.assertIsNone(snap.attributed)+ self.assertIn("process memory unavailable", snap.counter_status)+ c.ascend.read_device.assert_called_once_with(chip, 0)+++if __name__ == "__main__":+ unittest.main()diff --git a/tests/test_linux.py b/tests/test_linux.pyindex d5243d2..efe7041 100644--- a/tests/test_linux.py+++ b/tests/test_linux.py@@ -26,6 +26,7 @@ def setUpModule() -> None:def tearDownModule() -> None:_no_drm.stop()+ _no_ascend.stop()NOT_AVAILABLE = (1 << 64) - 1 # NVML_VALUE_NOT_AVAILABLE as an unsigned long long