Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

fix(health): detect AMD GPU stats from amd-smi when rocm-smi is absent (#11568)

Hitaishi55448963dde1
diff --git a/marimo/_server/api/endpoints/health.py b/marimo/_server/api/endpoints/health.py
index fa2b48fa13..d669707677 100644
--- a/marimo/_server/api/endpoints/health.py
+++ b/marimo/_server/api/endpoints/health.py
@@ -373,6 +373,8 @@ async def usage(request: Request) -> JSONResponse:
gpu_stats = _parse_nvidia_smi_stats()
elif gpu_available == "rocm":
gpu_stats = _parse_rocm_smi_stats()
+ elif gpu_available == "amd":
+ gpu_stats = _parse_amd_smi_stats()
return JSONResponse(
{
@@ -430,9 +432,16 @@ _AMD_GPU_STATS_CMD = [
"--json",
]
+# Newer ROCm ships amd-smi only; rocm-smi is gone. Prefer this over
+# `metric --json`: on APUs (UMA) `metric` reports only the small dedicated
+# carveout as total_vram (2048 MB here), while `monitor --vram-usage` reports
+# the shared pool the GPU can actually use.
+_AMD_SMI_STATS_CMD = ["amd-smi", "monitor", "--vram-usage", "--json"]
+_AMD_SMI_STATIC_CMD = ["amd-smi", "static", "--json"]
+
@lru_cache(maxsize=1)
-def _is_gpu_available() -> Literal["nvidia", "rocm", False]:
+def _is_gpu_available() -> Literal["nvidia", "rocm", "amd", False]:
import subprocess
if DependencyManager.which("nvidia-smi"):
@@ -459,8 +468,21 @@ def _is_gpu_available() -> Literal["nvidia", "rocm", False]:
except (subprocess.SubprocessError, FileNotFoundError):
pass
+ if DependencyManager.which("amd-smi"):
+ try:
+ subprocess.run(
+ _AMD_SMI_STATS_CMD,
+ capture_output=True,
+ text=True,
+ check=True,
+ )
+ return "amd"
+ except (subprocess.SubprocessError, FileNotFoundError):
+ pass
+
LOGGER.warning(
- "Neither nvidia-smi nor rocm-smi succeeded, cannot get GPU stats"
+ "Neither nvidia-smi, rocm-smi, nor amd-smi succeeded, "
+ "cannot get GPU stats"
)
return False
@@ -560,3 +582,93 @@ def _parse_rocm_smi_stats() -> list[dict[str, Any]]:
}
)
return stats
+
+
+def _amd_smi_bytes(field: Any) -> int:
+ """Convert an amd-smi field (`{"value": 6, "unit": "MB"}`) to bytes."""
+ if not isinstance(field, dict):
+ return 0
+ value: Any = field.get("value")
+ unit = str(field.get("unit") or "B")
+ try:
+ number = float(value)
+ except (TypeError, ValueError):
+ return 0
+ scale = {"B": 1, "KB": 1024, "MB": 1024**2, "GB": 1024**3}
+ return int(number * scale.get(unit, 1))
+
+
+def _amd_smi_json(stdout: str) -> Any:
+ """Parse amd-smi JSON, tolerating leading warning lines on stdout."""
+ import json
+
+ start = min(
+ (i for i in (stdout.find("{"), stdout.find("[")) if i != -1),
+ default=-1,
+ )
+ if start == -1:
+ raise ValueError("No JSON found in amd-smi output")
+ return json.loads(stdout[start:])
+
+
+def _parse_amd_smi_stats() -> list[dict[str, Any]]:
+ import subprocess
+
+ # amd-smi monitor reports memory but not the product name; static has it.
+ names: dict[int, str] = {}
+ try:
+ result = subprocess.run(
+ _AMD_SMI_STATIC_CMD,
+ capture_output=True,
+ text=True,
+ check=True,
+ )
+ for gpu in _amd_smi_json(result.stdout).get("gpu_data", []):
+ asic = gpu.get("asic") or {}
+ names[int(gpu.get("gpu", 0))] = (
+ asic.get("market_name") or "AMD GPU"
+ )
+ except (
+ subprocess.SubprocessError,
+ FileNotFoundError,
+ ValueError,
+ AttributeError,
+ ) as e:
+ LOGGER.warning("Failed to extract AMD GPU names: %s", e)
+
+ try:
+ result = subprocess.run(
+ _AMD_SMI_STATS_CMD,
+ capture_output=True,
+ text=True,
+ check=True,
+ )
+ data = _amd_smi_json(result.stdout)
+ except (subprocess.SubprocessError, FileNotFoundError, ValueError) as e:
+ LOGGER.warning("Failed to extract AMD GPU stats: %s", e)
+ return []
+
+ stats: list[dict[str, Any]] = []
+ seen: set[int] = set()
+ for gpu in data if isinstance(data, list) else []:
+ index = int(gpu.get("gpu", 0))
+ # monitor emits one row per xcp; keep the first for each GPU.
+ if index in seen:
+ continue
+ seen.add(index)
+ total = _amd_smi_bytes(gpu.get("vram_total"))
+ used = _amd_smi_bytes(gpu.get("vram_used"))
+ total = max(total, used)
+ stats.append(
+ {
+ "index": index,
+ "name": names.get(index, "AMD GPU").strip(),
+ "memory": {
+ "total": total,
+ "used": used,
+ "free": total - used,
+ "percent": (used / total) * 100 if total > 0 else 0,
+ },
+ }
+ )
+ return stats
diff --git a/tests/_server/api/endpoints/test_health.py b/tests/_server/api/endpoints/test_health.py
index 5774a5d077..6169fd05b2 100644
--- a/tests/_server/api/endpoints/test_health.py
+++ b/tests/_server/api/endpoints/test_health.py
@@ -1,8 +1,9 @@
# Copyright 2026 Marimo. All rights reserved.
from __future__ import annotations
+import subprocess
from typing import TYPE_CHECKING
-from unittest.mock import patch
+from unittest.mock import Mock, patch
import pytest
@@ -227,6 +228,105 @@ def test_usage_rocm_gpu(client: TestClient) -> None:
assert memory["percent"] == pytest.approx(1069268992 / 8573157376 * 100)
+def test_usage_amd_smi_gpu(client: TestClient) -> None:
+ import json
+
+ # `amd-smi static --json`: the product name lives under asic.market_name.
+ static_output = json.dumps(
+ {
+ "gpu_data": [
+ {
+ "gpu": 0,
+ "asic": {"market_name": "AMD Radeon 8060S Graphics"},
+ }
+ ]
+ }
+ )
+ # `amd-smi monitor --vram-usage --json`: values carry their own unit, and
+ # on APUs (UMA) this reports the shared pool, not the small carveout that
+ # `amd-smi metric` reports as total_vram. Numbers below are from a
+ # Strix Halo APU: 28672 MB (28 GiB) total, 89 MB used.
+ monitor_output = json.dumps(
+ [
+ {
+ "gpu": 0,
+ "xcp": 0,
+ "vram_used": {"value": 89, "unit": "MB"},
+ "vram_free": {"value": 28583, "unit": "MB"},
+ "vram_total": {"value": 28, "unit": "GB"},
+ },
+ # One row per xcp; the same GPU must not be listed twice.
+ {
+ "gpu": 0,
+ "xcp": 1,
+ "vram_used": {"value": 0, "unit": "MB"},
+ "vram_free": {"value": 28672, "unit": "MB"},
+ "vram_total": {"value": 28, "unit": "GB"},
+ },
+ ]
+ )
+
+ with (
+ patch(
+ "marimo._server.api.endpoints.health._is_gpu_available",
+ return_value="amd",
+ ),
+ patch("subprocess.run") as mock_run,
+ ):
+ mock_run.side_effect = [
+ Mock(stdout=static_output, stderr=""),
+ Mock(stdout=monitor_output, stderr=""),
+ ]
+ response = client.get("/api/usage", headers=token_header())
+ assert response.status_code == 200
+ gpu = response.json()["gpu"]
+ assert len(gpu) == 1
+ assert gpu[0]["index"] == 0
+ assert gpu[0]["name"] == "AMD Radeon 8060S Graphics"
+ memory = gpu[0]["memory"]
+ assert memory["total"] == 28 * 1024**3
+ assert memory["used"] == 89 * 1024**2
+ assert memory["free"] == 28583 * 1024**2
+ assert memory["percent"] == pytest.approx(89 / 28672 * 100)
+
+
+def test_usage_amd_smi_gpu_without_static(client: TestClient) -> None:
+ import json
+
+ monitor_output = json.dumps(
+ [
+ {
+ "gpu": 0,
+ "xcp": 0,
+ "vram_used": {"value": 1, "unit": "GB"},
+ "vram_total": {"value": 24, "unit": "GB"},
+ }
+ ]
+ )
+
+ with (
+ patch(
+ "marimo._server.api.endpoints.health._is_gpu_available",
+ return_value="amd",
+ ),
+ patch("subprocess.run") as mock_run,
+ ):
+ # `amd-smi static` is unavailable, so the name falls back.
+ mock_run.side_effect = [
+ subprocess.CalledProcessError(1, "amd-smi"),
+ Mock(stdout=monitor_output, stderr=""),
+ ]
+ response = client.get("/api/usage", headers=token_header())
+ assert response.status_code == 200
+ gpu = response.json()["gpu"]
+ assert len(gpu) == 1
+ assert gpu[0]["name"] == "AMD GPU"
+ memory = gpu[0]["memory"]
+ assert memory["total"] == 24 * 1024**3
+ assert memory["used"] == 1 * 1024**3
+ assert memory["free"] == 23 * 1024**3
+
+
@with_session(SESSION_ID)
def test_read_code(client: TestClient) -> None:
response = client.get("/api/status/connections", headers=HEADERS)