A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.
Python artificial-intelligencedagdata-sciencedata-visualizationdataflowdeveloper-toolsmachine-learningnotebookspipelinepythonreactivesqlweb-app
fix(health): detect AMD GPU stats from amd-smi when rocm-smi is absent (#11568)
Hitaishi55448963dde1
diff --git a/marimo/_server/api/endpoints/health.py b/marimo/_server/api/endpoints/health.pyindex fa2b48fa13..d669707677 100644--- a/marimo/_server/api/endpoints/health.py+++ b/marimo/_server/api/endpoints/health.py@@ -373,6 +373,8 @@ async def usage(request: Request) -> JSONResponse:gpu_stats = _parse_nvidia_smi_stats()elif gpu_available == "rocm":gpu_stats = _parse_rocm_smi_stats()+ elif gpu_available == "amd":+ gpu_stats = _parse_amd_smi_stats()return JSONResponse({@@ -430,9 +432,16 @@ _AMD_GPU_STATS_CMD = ["--json",]+# Newer ROCm ships amd-smi only; rocm-smi is gone. Prefer this over+# `metric --json`: on APUs (UMA) `metric` reports only the small dedicated+# carveout as total_vram (2048 MB here), while `monitor --vram-usage` reports+# the shared pool the GPU can actually use.+_AMD_SMI_STATS_CMD = ["amd-smi", "monitor", "--vram-usage", "--json"]+_AMD_SMI_STATIC_CMD = ["amd-smi", "static", "--json"]+@lru_cache(maxsize=1)-def _is_gpu_available() -> Literal["nvidia", "rocm", False]:+def _is_gpu_available() -> Literal["nvidia", "rocm", "amd", False]:import subprocessif DependencyManager.which("nvidia-smi"):@@ -459,8 +468,21 @@ def _is_gpu_available() -> Literal["nvidia", "rocm", False]:except (subprocess.SubprocessError, FileNotFoundError):pass+ if DependencyManager.which("amd-smi"):+ try:+ subprocess.run(+ _AMD_SMI_STATS_CMD,+ capture_output=True,+ text=True,+ check=True,+ )+ return "amd"+ except (subprocess.SubprocessError, FileNotFoundError):+ pass+LOGGER.warning(- "Neither nvidia-smi nor rocm-smi succeeded, cannot get GPU stats"+ "Neither nvidia-smi, rocm-smi, nor amd-smi succeeded, "+ "cannot get GPU stats")return False@@ -560,3 +582,93 @@ def _parse_rocm_smi_stats() -> list[dict[str, Any]]:})return stats+++def _amd_smi_bytes(field: Any) -> int:+ """Convert an amd-smi field (`{"value": 6, "unit": "MB"}`) to bytes."""+ if not isinstance(field, dict):+ return 0+ value: Any = field.get("value")+ unit = str(field.get("unit") or "B")+ try:+ number = float(value)+ except (TypeError, ValueError):+ return 0+ scale = {"B": 1, "KB": 1024, "MB": 1024**2, "GB": 1024**3}+ return int(number * scale.get(unit, 1))+++def _amd_smi_json(stdout: str) -> Any:+ """Parse amd-smi JSON, tolerating leading warning lines on stdout."""+ import json++ start = min(+ (i for i in (stdout.find("{"), stdout.find("[")) if i != -1),+ default=-1,+ )+ if start == -1:+ raise ValueError("No JSON found in amd-smi output")+ return json.loads(stdout[start:])+++def _parse_amd_smi_stats() -> list[dict[str, Any]]:+ import subprocess++ # amd-smi monitor reports memory but not the product name; static has it.+ names: dict[int, str] = {}+ try:+ result = subprocess.run(+ _AMD_SMI_STATIC_CMD,+ capture_output=True,+ text=True,+ check=True,+ )+ for gpu in _amd_smi_json(result.stdout).get("gpu_data", []):+ asic = gpu.get("asic") or {}+ names[int(gpu.get("gpu", 0))] = (+ asic.get("market_name") or "AMD GPU"+ )+ except (+ subprocess.SubprocessError,+ FileNotFoundError,+ ValueError,+ AttributeError,+ ) as e:+ LOGGER.warning("Failed to extract AMD GPU names: %s", e)++ try:+ result = subprocess.run(+ _AMD_SMI_STATS_CMD,+ capture_output=True,+ text=True,+ check=True,+ )+ data = _amd_smi_json(result.stdout)+ except (subprocess.SubprocessError, FileNotFoundError, ValueError) as e:+ LOGGER.warning("Failed to extract AMD GPU stats: %s", e)+ return []++ stats: list[dict[str, Any]] = []+ seen: set[int] = set()+ for gpu in data if isinstance(data, list) else []:+ index = int(gpu.get("gpu", 0))+ # monitor emits one row per xcp; keep the first for each GPU.+ if index in seen:+ continue+ seen.add(index)+ total = _amd_smi_bytes(gpu.get("vram_total"))+ used = _amd_smi_bytes(gpu.get("vram_used"))+ total = max(total, used)+ stats.append(+ {+ "index": index,+ "name": names.get(index, "AMD GPU").strip(),+ "memory": {+ "total": total,+ "used": used,+ "free": total - used,+ "percent": (used / total) * 100 if total > 0 else 0,+ },+ }+ )+ return statsdiff --git a/tests/_server/api/endpoints/test_health.py b/tests/_server/api/endpoints/test_health.pyindex 5774a5d077..6169fd05b2 100644--- a/tests/_server/api/endpoints/test_health.py+++ b/tests/_server/api/endpoints/test_health.py@@ -1,8 +1,9 @@# Copyright 2026 Marimo. All rights reserved.from __future__ import annotations+import subprocessfrom typing import TYPE_CHECKING-from unittest.mock import patch+from unittest.mock import Mock, patchimport pytest@@ -227,6 +228,105 @@ def test_usage_rocm_gpu(client: TestClient) -> None:assert memory["percent"] == pytest.approx(1069268992 / 8573157376 * 100)+def test_usage_amd_smi_gpu(client: TestClient) -> None:+ import json++ # `amd-smi static --json`: the product name lives under asic.market_name.+ static_output = json.dumps(+ {+ "gpu_data": [+ {+ "gpu": 0,+ "asic": {"market_name": "AMD Radeon 8060S Graphics"},+ }+ ]+ }+ )+ # `amd-smi monitor --vram-usage --json`: values carry their own unit, and+ # on APUs (UMA) this reports the shared pool, not the small carveout that+ # `amd-smi metric` reports as total_vram. Numbers below are from a+ # Strix Halo APU: 28672 MB (28 GiB) total, 89 MB used.+ monitor_output = json.dumps(+ [+ {+ "gpu": 0,+ "xcp": 0,+ "vram_used": {"value": 89, "unit": "MB"},+ "vram_free": {"value": 28583, "unit": "MB"},+ "vram_total": {"value": 28, "unit": "GB"},+ },+ # One row per xcp; the same GPU must not be listed twice.+ {+ "gpu": 0,+ "xcp": 1,+ "vram_used": {"value": 0, "unit": "MB"},+ "vram_free": {"value": 28672, "unit": "MB"},+ "vram_total": {"value": 28, "unit": "GB"},+ },+ ]+ )++ with (+ patch(+ "marimo._server.api.endpoints.health._is_gpu_available",+ return_value="amd",+ ),+ patch("subprocess.run") as mock_run,+ ):+ mock_run.side_effect = [+ Mock(stdout=static_output, stderr=""),+ Mock(stdout=monitor_output, stderr=""),+ ]+ response = client.get("/api/usage", headers=token_header())+ assert response.status_code == 200+ gpu = response.json()["gpu"]+ assert len(gpu) == 1+ assert gpu[0]["index"] == 0+ assert gpu[0]["name"] == "AMD Radeon 8060S Graphics"+ memory = gpu[0]["memory"]+ assert memory["total"] == 28 * 1024**3+ assert memory["used"] == 89 * 1024**2+ assert memory["free"] == 28583 * 1024**2+ assert memory["percent"] == pytest.approx(89 / 28672 * 100)+++def test_usage_amd_smi_gpu_without_static(client: TestClient) -> None:+ import json++ monitor_output = json.dumps(+ [+ {+ "gpu": 0,+ "xcp": 0,+ "vram_used": {"value": 1, "unit": "GB"},+ "vram_total": {"value": 24, "unit": "GB"},+ }+ ]+ )++ with (+ patch(+ "marimo._server.api.endpoints.health._is_gpu_available",+ return_value="amd",+ ),+ patch("subprocess.run") as mock_run,+ ):+ # `amd-smi static` is unavailable, so the name falls back.+ mock_run.side_effect = [+ subprocess.CalledProcessError(1, "amd-smi"),+ Mock(stdout=monitor_output, stderr=""),+ ]+ response = client.get("/api/usage", headers=token_header())+ assert response.status_code == 200+ gpu = response.json()["gpu"]+ assert len(gpu) == 1+ assert gpu[0]["name"] == "AMD GPU"+ memory = gpu[0]["memory"]+ assert memory["total"] == 24 * 1024**3+ assert memory["used"] == 1 * 1024**3+ assert memory["free"] == 23 * 1024**3++@with_session(SESSION_ID)def test_read_code(client: TestClient) -> None:response = client.get("/api/status/connections", headers=HEADERS)