Skip to content

A GPU task manager for Windows, Linux, and WSL: per-process VRAM, GPU activity, and LLM model/KV-cache insights (Windows AMD and NVIDIA).

Python

Add Huawei Ascend NPU backend (310P) on Linux

WLTBAgent39e337f53b77
diff --git a/src/gputaskman/ascend.py b/src/gputaskman/ascend.py
new file mode 100644
index 0000000..d5f6936
--- /dev/null
+++ b/src/gputaskman/ascend.py
@@ -0,0 +1,320 @@
+"""Huawei Ascend NPU backend (Linux): adapters and telemetry from npu-smi, per-process memory
+from `npu-smi info -t proc-mem`.
+
+The 310P-family NPUs don't use DRM: the host driver (devdrv) exposes /dev/davinci* character
+devices, and telemetry lives behind the npu-smi management tool (PCI vendor 0x19E5). npu-smi
+calls cost ~0.5s each, so live utilization comes from a persistent `npu-smi info watch` stream
+(one line per chip per second) that a reader thread keeps running; absolute memory and
+per-process memory are refreshed on a slow cadence. A dual-chip card (e.g. Atlas 300I Duo)
+appears as two adapters, one per chip, each with its own DDR bank.
+"""
+
+from __future__ import annotations
+
+import os
+import re
+import subprocess
+import threading
+import time
+from collections import defaultdict
+from dataclasses import dataclass
+
+from .dxgi import Adapter
+from .pdh import AdapterGpu, ProcGpu
+from .telemetry import DeviceTelemetry
+
+VENDOR_HUAWEI = 0x19E5
+_MB = 1024 * 1024
+
+_NPUSMI = None # resolved lazily; see _npu_path
+
+
+def _npu_path() -> str | None:
+ """Path to npu-smi, or None when the tool (and likely the driver) is absent."""
+ global _NPUSMI
+ if _NPUSMI is None:
+ for candidate in ("npu-smi", "/usr/local/bin/npu-smi"):
+ try:
+ subprocess.run([candidate, "-h"], capture_output=True, timeout=5)
+ _NPUSMI = candidate
+ break
+ except (OSError, subprocess.SubprocessError):
+ continue
+ return _NPUSMI
+
+
+def _run(args: list[str], timeout: float = 10.0) -> str:
+ """npu-smi output, or '' on any failure (missing tool, error exit, timeout)."""
+ path = _npu_path()
+ if path is None:
+ return ""
+ try:
+ r = subprocess.run([path, *args], capture_output=True, text=True, timeout=timeout)
+ except (OSError, subprocess.SubprocessError):
+ return ""
+ return r.stdout if r.returncode == 0 else ""
+
+
+@dataclass
+class AscendChip:
+ npu: int # npu-smi card ID (-i)
+ chip: int # chip ID within the card (-c)
+ name: str
+ pdev: str # PCI bus id, "0000:05:00.0"
+ mem_total: int = 0 # bytes
+ mem_used: int | None = None
+ product: str = ""
+
+
+def parse_info(text: str) -> list[dict]:
+ """Parse `npu-smi info` into one dict per chip.
+
+ Each chip is two table rows sharing a cell layout (leading/trailing | stripped, split on |):
+ head: | 3 310P3 | OK | NA 45 0 / 0 | npu name | health | power temp hugepages
+ tail: | 0 0 | 0000:05:00.0 | 0 1522 / 47302 | chip dev | bus | aicore mem-usage
+ """
+ chips: list[dict] = []
+ head: list[str] | None = None
+ for line in text.splitlines():
+ if not line.startswith("|"):
+ continue
+ cells = [c.strip() for c in line.strip().strip("|").split("|")]
+ if len(cells) < 3:
+ continue
+ m = re.match(r"^(\d+)\s+(\d+)$", cells[0])
+ if m and re.match(r"^\d+:\d+:\d+\.\d+$", cells[1]):
+ if head is None:
+ continue
+ hm = re.match(r"^(\d+)\s+(\S+)$", head[0])
+ if not hm:
+ head = None
+ continue
+ mt = re.search(r"(\d+)\s*/\s*(\d+)", cells[2])
+ ht = head[2].split()
+ chips.append({
+ "npu": int(hm.group(1)), "name": hm.group(2),
+ "health": head[1], "power": ht[0] if ht else None,
+ "temp": int(ht[1]) if len(ht) > 1 and ht[1].lstrip("-").isdigit() else None,
+ "chip": int(m.group(1)), "device": int(m.group(2)), "bus": cells[1],
+ "aicore": int(cells[2].split()[0]) if cells[2].split() else None,
+ "mem_used": int(mt.group(1)) if mt else None,
+ "mem_total": int(mt.group(2)) if mt else None,
+ })
+ head = None
+ elif re.match(r"^(\d+)\s+(\S+)$", cells[0]) and not cells[1].startswith("0000:"):
+ head = cells
+ return chips
+
+
+_WATCH_RE = re.compile(
+ r"^\s*(\d+)\s+(\d+)\s+(\S+)\s+(-?\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s*$")
+
+
+def parse_watch_line(line: str) -> dict | None:
+ """One `npu-smi info watch` data row: NpuID ChipId Pwr Temp AI-Core AI-Cpu CtrlCpu Mem% MemBW%."""
+ m = _WATCH_RE.match(line)
+ if not m:
+ return None
+ return {
+ "npu": int(m.group(1)), "chip": int(m.group(2)), "temp": int(m.group(4)),
+ "ai_core": int(m.group(5)), "ai_cpu": int(m.group(6)), "ctrl_cpu": int(m.group(7)),
+ "mem_pct": int(m.group(8)), "mem_bw": int(m.group(9)),
+ }
+
+
+def parse_proc_mem(text: str) -> list[tuple[int, str, int]]:
+ """`npu-smi info -t proc-mem` rows -> (pid, name, megabytes)."""
+ out = []
+ for m in re.finditer(r"Process id:(\d+)\s+Process name:(\S+)\s+Process memory\(MB\):(\d+)", text):
+ out.append((int(m.group(1)), m.group(2), int(m.group(3))))
+ return out
+
+
+def _pci_key(pdev: str) -> int:
+ return int(re.sub(r"[^0-9a-f]", "", pdev.lower()) or "0", 16)
+
+
+def probe() -> list[AscendChip]:
+ """Enumerate every Ascend chip npu-smi can see (two subprocess calls, ~1s)."""
+ text = _run(["info"])
+ if not text:
+ return []
+ chips = []
+ for c in parse_info(text):
+ product = ""
+ board = _run(["info", "-t", "board", "-i", str(c["npu"])])
+ pm = re.search(r"Product Name\s*:\s*(.+)", board)
+ if pm:
+ product = pm.group(1).strip()
+ name = f"Ascend {c['name']} chip {c['chip']}"
+ if product:
+ name += f" ({product})"
+ chip = AscendChip(npu=c["npu"], chip=c["chip"], name=name, pdev=c["bus"], product=product)
+ if c["mem_total"] is not None:
+ chip.mem_total = c["mem_total"] * _MB
+ chip.mem_used = (c["mem_used"] or 0) * _MB
+ chips.append(chip)
+ return chips
+
+
+class AscendCounters:
+ """Same interface as drm.DrmCounters: adapters + sample() -> per-chip per-process usage."""
+
+ PROC_REFRESH_S = 10.0 # proc-mem is ~0.5s per chip; don't pay it every sample
+ INFO_REFRESH_S = 10.0 # absolute memory refresh (watch only carries percentages)
+
+ def __init__(self, chips: list[AscendChip] | None = None) -> None:
+ self.ok = False
+ self.chips = chips if chips is not None else probe()
+ self.ok = bool(self.chips)
+ self.by_key = {(_pci_key(c.pdev) << 8) | c.chip: c for c in self.chips}
+ self._watch: dict[tuple[int, int], dict] = {} # (npu, chip) -> latest watch row
+ self._watch_dead = False
+ self._lock = threading.Lock()
+ self._last_proc = 0.0
+ self._last_info = 0.0
+ self._procs: dict[int, ProcGpu] = {}
+ self._thread: threading.Thread | None = None
+
+ def adapters(self) -> list[Adapter]:
+ return [Adapter(luid=key, name=c.name, vendor_id=VENDOR_HUAWEI,
+ dedicated_vram=c.mem_total, shared_system=0, software=False)
+ for key, c in self.by_key.items()]
+
+ chip_key = staticmethod(_pci_key) # exposed so the Collector can match adapter keys
+
+ def start(self) -> None:
+ """Spawn the watch-stream reader (once). Cheap: one npu-smi process, a line per second."""
+ if self._thread is not None or not self.ok:
+ return
+ self._thread = threading.Thread(target=self._watch_loop, daemon=True)
+ self._thread.start()
+
+ def _watch_loop(self) -> None:
+ path = _npu_path()
+ if path is None:
+ self._watch_dead = True
+ return
+ try:
+ proc = subprocess.Popen([path, "info", "watch"], stdout=subprocess.PIPE,
+ stderr=subprocess.DEVNULL, text=True)
+ except OSError:
+ self._watch_dead = True
+ return
+ assert proc.stdout is not None
+ for line in proc.stdout:
+ row = parse_watch_line(line)
+ if row:
+ with self._lock:
+ self._watch[(row["npu"], row["chip"])] = row
+ self._watch_dead = True # stream ended (tool exit or broken pipe)
+ try:
+ proc.wait(timeout=5)
+ except subprocess.SubprocessError:
+ proc.kill()
+
+ def _refresh(self, now: float) -> None:
+ """Slow cadence: absolute memory from `info`, per-process memory from proc-mem."""
+ self.start()
+ if now - self._last_info >= self.INFO_REFRESH_S:
+ self._last_info = now
+ for c in parse_info(_run(["info"])):
+ chip = self.by_key.get((_pci_key(c["bus"]) << 8) | c["chip"]) if c["bus"] else None
+ if chip and c["mem_total"] is not None:
+ chip.mem_total = c["mem_total"] * _MB
+ chip.mem_used = (c["mem_used"] or 0) * _MB
+ if now - self._last_proc >= self.PROC_REFRESH_S:
+ self._last_proc = now
+ procs: dict[int, ProcGpu] = {}
+ for c in self.chips:
+ for pid, name, mb in parse_proc_mem(
+ _run(["info", "-t", "proc-mem", "-i", str(c.npu), "-c", str(c.chip)])):
+ g = procs.setdefault(pid, ProcGpu())
+ g.dedicated += mb * _MB
+ self._procs = procs
+
+ def sample(self) -> tuple[dict[int, dict[int, ProcGpu]], dict[int, AdapterGpu]]:
+ """({luid: {pid: ProcGpu}}, {luid: AdapterGpu}); per-chip process memory."""
+ if not self.ok:
+ return {}, {}
+ self._refresh(time.time())
+ procs: dict[int, dict[int, ProcGpu]] = defaultdict(dict)
+ adapters: dict[int, AdapterGpu] = defaultdict(AdapterGpu)
+ for key, chip in self.by_key.items():
+ per = procs[key]
+ for pid, g in getattr(self, "_procs", {}).items():
+ per[pid] = ProcGpu(dedicated=g.dedicated, available={"dedicated"})
+ a = adapters[key]
+ if chip.mem_used is not None:
+ a.dedicated = float(chip.mem_used)
+ return procs, adapters
+
+ def watch_row(self, chip: AscendChip) -> dict | None:
+ with self._lock:
+ return self._watch.get((chip.npu, chip.chip))
+
+ def read_device(self, chip: AscendChip, index: int) -> DeviceTelemetry:
+ """NVIDIA-shaped telemetry for one chip; watch data is live, sysfs fills the PCIe link."""
+ row = self.watch_row(chip)
+ if row is None and not self._watch_dead:
+ self.start()
+ time.sleep(0.2) # give the just-started stream a moment (one-shot mode)
+ row = self.watch_row(chip)
+ if row is None: # watch unavailable: one synchronous `info` (covers all chips)
+ for c in parse_info(_run(["info"])):
+ if c["npu"] == chip.npu and c["chip"] == chip.chip:
+ row = {"temp": c["temp"], "ai_core": c["aicore"] or 0, "ai_cpu": 0,
+ "ctrl_cpu": 0, "mem_pct": 0, "mem_bw": 0}
+ dev = DeviceTelemetry(
+ index=index, name=chip.name, pci_bus=chip.pdev, driver_model="devdrv",
+ source="npu-smi", mem_total=chip.mem_total or None, mem_used=chip.mem_used,
+ )
+ if row:
+ dev.temp = row.get("temp")
+ dev.util_gpu = float(row.get("ai_core") or 0)
+ dev.util_mem = float(row.get("mem_bw") or 0) or None
+ speed = _read_sysfs(f"/sys/bus/pci/devices/{chip.pdev}/current_link_speed")
+ if speed:
+ gen = {"2.5": 1, "5.0": 2, "8.0": 3, "16.0": 4, "32.0": 5}.get(speed.split()[0])
+ dev.pcie_gen = gen
+ dev.pcie_width = _int_sysfs(f"/sys/bus/pci/devices/{chip.pdev}/current_link_width")
+ return dev
+
+ def close(self) -> None:
+ pass # the watch thread is a daemon; it dies with the process
+
+
+def _read_sysfs(path: str) -> str | None:
+ try:
+ with open(path, encoding="utf-8", errors="replace") as f:
+ return f.read().strip()
+ except OSError:
+ return None
+
+
+def _int_sysfs(path: str) -> int | None:
+ v = _read_sysfs(path)
+ try:
+ return int(v) if v else None
+ except ValueError:
+ return None
+
+
+# Module-level TTL cache: app.py re-enumerates adapters every TUI cycle, and probe() costs ~1s.
+_cache: list[AscendChip] | None = None
+_cache_ts = 0.0
+_CACHE_TTL = 120.0
+
+
+def cached_chips() -> list[AscendChip]:
+ global _cache, _cache_ts
+ now = time.time()
+ if _cache is None or now - _cache_ts > _CACHE_TTL:
+ _cache = probe()
+ _cache_ts = now
+ return _cache
+
+
+def cached_chips_adapters() -> list[Adapter]:
+ """Adapters for the cached chip list (cheap: no npu-smi call once chips are cached)."""
+ return AscendCounters(chips=cached_chips()).adapters()
diff --git a/src/gputaskman/collector.py b/src/gputaskman/collector.py
index 0e4b624..c75d2a1 100644
--- a/src/gputaskman/collector.py
+++ b/src/gputaskman/collector.py
@@ -12,7 +12,7 @@ import psutil
from .amdsmi import AmdSmi, _bdf
from .adl import Adl
-from .dxgi import VENDOR_AMD, VENDOR_NVIDIA, Adapter
+from .dxgi import VENDOR_AMD, VENDOR_NVIDIA, VENDOR_HUAWEI, Adapter
from .handy import HandyEvidence, inspect_handy
from .llm import LlmView, ServerProber, api_key, build_view, gb, ollama_log_path
from . import nvml as nvml_mod
@@ -120,8 +120,8 @@ class Snapshot:
vram_used: int | None # WDDM adapter-wide dedicated usage
shared_used: int | None
procs: list[ProcRow] = field(default_factory=list)
- backend: str = "wddm" # wddm (Windows) | nvml (native Linux, NVIDIA) | drm (native Linux, AMD or Intel) | wsl (device-wide only) | metal (macOS)
- process_status: str = "" # drm backend only: what the DRM process scan couldn't cover; empty when it saw everything
+ backend: str = "wddm" # wddm (Windows) | nvml (native Linux, NVIDIA) | drm (native Linux, AMD/Intel) | ascend (Huawei NPU) | wsl (device-wide only) | metal (macOS)
+ process_status: str = "" # amdgpu only: what the DRM process scan couldn't cover; empty when it saw everything
host_status: str = "" # WSL only: Windows host counters via interop -- ok | starting | off (...) | ...
wsl_split: str = "" # WSL only: why Linux rows have no (inferred) VRAM, when they don't
# False on an APU (amdgpu iGPU): its GTT is ordinary GPU memory, not spill from a full VRAM.
@@ -199,9 +199,13 @@ class Collector:
_mac_driver = "" # "macOS 26.3.1" on the metal backend, read once at startup
SCAN_BUDGET = 0.15 # seconds of memory-map scanning per sample; leftovers wait for the next sample
drm_card = None # the selected GPU's AMD or Intel card, when it is one (drm.DrmCard)
+ ascend = None # AscendCounters, when LinuxCounters owns one (shared watch stream)
+ ascend_chip = None # the selected GPU's Ascend chip, when it is one (ascend.AscendChip)
@property
def backend(self) -> str:
+ if self.ascend_chip is not None:
+ return "ascend" # Huawei NPU: npu-smi is the source, not NVML/DRM
return "drm" if self.drm_card is not None else BACKEND
def __init__(self, gpu_index: int = 0, scan_budget: float | None = SCAN_BUDGET, host: bool = True,
@@ -230,6 +234,18 @@ class Collector:
drm.start() # engine baseline over every process
if scan_budget is None:
drm.scan_budget = None # one-shot output sees every process
+ # Huawei Ascend NPU (no DRM): per-chip processes from npu-smi, live metrics from a
+ # `npu-smi info watch` stream. Owned by LinuxCounters (one watch stream for all chips).
+ self.ascend_chip = None
+ self.ascend = None
+ if (BACKEND == "posix" or sys.platform == "linux") and self.adapter \
+ and self.adapter.vendor_id == VENDOR_HUAWEI:
+ asc = getattr(self.counters, "ascend", None)
+ if asc is not None and asc.ok:
+ self.ascend = asc
+ self.ascend_chip = next(
+ (c for c in asc.chips
+ if (asc.chip_key(c.pdev) << 8) | c.chip == self.adapter.luid), None)
self.inspector = ProcInspector()
self.prober = ServerProber()
self.scan_budget = scan_budget # None = scan every candidate now (one-shot output)
@@ -313,6 +329,8 @@ class Collector:
elif self.adapter and self.adapter.vendor_id != VENDOR_NVIDIA:
# NVML can't describe this GPU, and a sweep of every NVIDIA device costs ~60 ms per device.
nv = (self.adl.sample() or self.amdsmi.sample()) if self.adapter.vendor_id == VENDOR_AMD else None
+ elif self.ascend_chip is not None:
+ nv = self.ascend.read_device(self.ascend_chip, self.gpu_index)
else:
nvs = self.nvml.sample() if self.nvml.ok else []
nv = select_nv_device(self.adapter, nvs, self.gpu_index, self.adapters)
diff --git a/src/gputaskman/dxgi.py b/src/gputaskman/dxgi.py
index 3c44d9d..e63bf1c 100644
--- a/src/gputaskman/dxgi.py
+++ b/src/gputaskman/dxgi.py
@@ -33,7 +33,9 @@ class _GUID(ctypes.Structure):
_IID_IDXGIFactory1 = _GUID(0x770AAE78, 0xF26F, 0x4DBA, (ctypes.c_ubyte * 8)(0xA8, 0x29, 0x25, 0x3C, 0x83, 0xD1, 0xB3, 0x87))
VENDOR_NVIDIA = 0x10DE
VENDOR_AMD = 0x1002
-VENDORS = {0x10DE: "NVIDIA", 0x1002: "AMD", 0x8086: "Intel", 0x1414: "Microsoft", 0x5143: "Qualcomm"}
+VENDOR_HUAWEI = 0x19E5 # Ascend NPUs (telemetry via npu-smi, not DRM)
+VENDORS = {0x10DE: "NVIDIA", 0x1002: "AMD", 0x8086: "Intel", 0x19E5: "Ascend",
+ 0x1414: "Microsoft", 0x5143: "Qualcomm"}
@dataclass
diff --git a/src/gputaskman/linux.py b/src/gputaskman/linux.py
index a85c0d7..29207f2 100644
--- a/src/gputaskman/linux.py
+++ b/src/gputaskman/linux.py
@@ -69,13 +69,22 @@ def _nvml_devices() -> list[tuple[int, object]]:
def enumerate_adapters() -> list[Adapter]:
- """NVIDIA GPUs through NVML, plus (native Linux only) AMD and Intel GPUs through DRM."""
+ """NVIDIA GPUs through NVML, plus (native Linux only) AMD and Intel GPUs through DRM,
+ and Huawei Ascend NPUs through npu-smi."""
out = _nvml_adapters()
if not is_wsl():
from . import drm
seen = {a.luid for a in out}
out += [a for a in drm.adapters(drm.cards()) if a.luid not in seen]
+ # Ascend NPUs (no DRM): dual-chip cards yield one adapter per chip.
+ try:
+ from . import ascend
+
+ seen |= {a.luid for a in out}
+ out += [a for a in ascend.cached_chips_adapters() if a.luid not in seen]
+ except Exception:
+ pass # no npu-smi: the backend is simply absent
return out
@@ -235,6 +244,15 @@ class LinuxCounters:
drm = DrmCounters() # cheap: only lists cards; the Collector starts it if an AMD GPU is selected
self.drm = drm if drm.ok else None
+ # Huawei Ascend NPUs (no DRM): npu-smi-based, own watch stream + slow proc-mem refresh.
+ self.ascend = None
+ if not self.wsl:
+ try:
+ from .ascend import AscendCounters
+
+ self.ascend = AscendCounters()
+ except Exception:
+ self.ascend = None
if N is None:
return
try:
@@ -311,6 +329,10 @@ class LinuxCounters:
drm_procs, drm_adapters = self.drm.sample()
procs.update(drm_procs)
adapters.update(drm_adapters)
+ if self.ascend is not None and self.ascend.ok:
+ asc_procs, asc_adapters = self.ascend.sample()
+ procs.update(asc_procs)
+ adapters.update(asc_adapters)
return procs, adapters
def _resolve_namespaces(self, procs: dict[int, dict[int, ProcGpu]]) -> None:
diff --git a/tests/test_linux.py b/tests/test_linux.py
index ad9419f..d5243d2 100644
--- a/tests/test_linux.py
+++ b/tests/test_linux.py
@@ -16,10 +16,12 @@ from gputaskman.report import to_dict, to_text
GiB = 1 << 30
_no_drm = patch("gputaskman.drm.cards", return_value=[]) # keep the test machine's real AMD GPUs out of NVML tests
+_no_ascend = patch("gputaskman.ascend.probe", return_value=[]) # and its real Ascend NPUs out too
def setUpModule() -> None:
_no_drm.start()
+ _no_ascend.start()
def tearDownModule() -> None: