Modern Docker Management, Designed for Everyone
Go composecontainer-managementcontainersdockerdocker-composedocker-managementgoself-hostedsveltekittypescriptweb-ui
package system
import (
"bytes"
"context"
"encoding/csv"
"errors"
"fmt"
"log/slog"
"os"
"os/exec"
"strconv"
"strings"
"sync"
"sync/atomic"
"time"
systemtypes "github.com/getarcaneapp/arcane/types/v2/system"
"github.com/samber/hot"
kit "go.getarcane.app/kit/pkg"
"golang.org/x/sync/singleflight"
)
// AMDGPUSysfsPath is the sysfs base used to discover AMD GPUs.
const (
AMDGPUSysfsPath = "/sys/class/drm"
// gpuDetectionTTL bounds how long a successful detection result is reused before re-detecting.
gpuDetectionTTL = 30 * time.Second
// gpuStatsTTL bounds how often the vendor tool is invoked: reading VRAM
// stats forks a process (nvidia-smi, intel_gpu_top), which is far too
// expensive to repeat on every sampler tick.
gpuStatsTTL = 3 * time.Second
)
// GPUMonitor probes for an attached GPU (NVIDIA / AMD / Intel) and reports VRAM usage.
// Detection is cached for gpuDetectionTTL; once a vendor is detected, subsequent Stats
// calls invoke the vendor-specific tool directly.
type GPUMonitor struct {
configuredType string
detectionMu sync.Mutex
detectionCache *hot.HotCache[struct{}, gpuDetection]
detectionDone bool
// statsCache holds the last vendor-tool result (success or failure) for
// gpuStatsTTL so concurrent consumers and fast sampler ticks share one
// process fork instead of each launching their own.
statsCache atomic.Pointer[gpuStatsSnapshotInternal]
statsGroup singleflight.Group
announcedType string
enabled bool
}
type gpuStatsSnapshotInternal struct {
stats []systemtypes.GPUStats
err error
sampledAt time.Time
}
type gpuDetection struct {
detected bool
gpuType string
toolPath string
}
// NewGPUMonitor creates a monitor. enabled gates Stats; when false, Stats returns
// (nil, nil). configuredType is the user-pinned vendor ("nvidia"|"amd"|"intel"|"auto"|"")
// — anything else falls back to auto-detection.
func NewGPUMonitor(enabled bool, configuredType string) *GPUMonitor {
return &GPUMonitor{
enabled: enabled,
configuredType: configuredType,
detectionCache: hot.NewHotCache[struct{}, gpuDetection](hot.LRU, 1).
WithTTL(gpuDetectionTTL).
Build(),
}
}
// Enabled reports whether GPU monitoring is on.
func (m *GPUMonitor) Enabled() bool { return m.enabled }
// Stats returns per-GPU VRAM stats. Returns (nil, 0, nil) when monitoring is disabled
// or no GPU is detected; vendor-specific errors are propagated otherwise.
// Results (including failures) are cached for gpuStatsTTL, so callers may
// receive stats up to that many seconds old.
func (m *GPUMonitor) Stats(ctx context.Context) ([]systemtypes.GPUStats, error) {
if !m.enabled {
return nil, nil
}
if snapshot := m.statsCache.Load(); snapshot != nil && time.Since(snapshot.sampledAt) < gpuStatsTTL {
return snapshot.stats, snapshot.err
}
result, err, _ := m.statsGroup.Do("stats", func() (any, error) {
previous := m.statsCache.Load()
if previous != nil && time.Since(previous.sampledAt) < gpuStatsTTL {
return previous.stats, previous.err
}
stats, statsErr := m.statsUncachedInternal(ctx)
if previousErr := kit.FromPtr(previous).err; statsErr != nil && (previousErr == nil || previousErr.Error() != statsErr.Error()) {
slog.WarnContext(ctx, "GPU stats unavailable", "error", statsErr)
}
m.statsCache.Store(&gpuStatsSnapshotInternal{stats: stats, err: statsErr, sampledAt: time.Now()})
return stats, statsErr
})
stats, _ := result.([]systemtypes.GPUStats)
return stats, err
}
func (m *GPUMonitor) statsUncachedInternal(ctx context.Context) ([]systemtypes.GPUStats, error) {
m.detectionMu.Lock()
done := m.detectionDone
m.detectionMu.Unlock()
if !done {
if err := m.detectInternal(ctx); err != nil {
return nil, err
}
}
if detection, found, _ := m.detectionCache.Get(struct{}{}); found && detection.detected {
return m.statsForTypeInternal(ctx, detection.gpuType)
}
if err := m.detectInternal(ctx); err != nil {
return nil, err
}
detection, found, _ := m.detectionCache.Get(struct{}{})
if !found || detection.gpuType == "" {
return nil, errors.New("no supported GPU found")
}
return m.statsForTypeInternal(ctx, detection.gpuType)
}
func (m *GPUMonitor) statsForTypeInternal(ctx context.Context, gpuType string) ([]systemtypes.GPUStats, error) {
switch gpuType {
case "nvidia":
return getNvidiaStatsInternal(ctx)
case "amd":
return getAMDStatsInternal(ctx)
case "intel":
return getIntelStatsInternal(ctx)
default:
return nil, errors.New("no supported GPU found")
}
}
// markDetected records a successful detection.
func (m *GPUMonitor) markDetectedInternal(gpuType, toolPath string) bool {
m.detectionCache.Set(struct{}{}, gpuDetection{detected: true, gpuType: gpuType, toolPath: toolPath})
m.detectionDone = true
announce := m.announcedType != gpuType
m.announcedType = gpuType
return announce
}
// detect runs vendor probing under detectionMu. The configuredType pin is honored when set
// to a known vendor; otherwise vendors are tried in order: nvidia → amd → intel.
func (m *GPUMonitor) detectInternal(ctx context.Context) error {
m.detectionMu.Lock()
defer m.detectionMu.Unlock()
if t := m.configuredType; t != "" && t != "auto" {
switch t {
case "nvidia":
if path, err := exec.LookPath("nvidia-smi"); err == nil {
if m.markDetectedInternal("nvidia", path) {
slog.InfoContext(ctx, "Using configured GPU type", "type", "nvidia")
}
return nil
}
return errors.New("nvidia-smi not found but GPU_TYPE set to nvidia")
case "amd":
if HasAMDGPU() {
if m.markDetectedInternal("amd", AMDGPUSysfsPath) {
slog.InfoContext(ctx, "Using configured GPU type", "type", "amd")
}
return nil
}
return errors.New("AMD GPU not found in sysfs but GPU_TYPE set to amd")
case "intel":
if path, err := exec.LookPath("intel_gpu_top"); err == nil {
if m.markDetectedInternal("intel", path) {
slog.InfoContext(ctx, "Using configured GPU type", "type", "intel")
}
return nil
}
return errors.New("intel_gpu_top not found but GPU_TYPE set to intel")
default:
slog.WarnContext(ctx, "Invalid GPU_TYPE specified, falling back to auto-detection", "gpuType", t)
}
}
if path, err := exec.LookPath("nvidia-smi"); err == nil {
if m.markDetectedInternal("nvidia", path) {
slog.InfoContext(ctx, "NVIDIA GPU detected", "tool", "nvidia-smi", "path", path)
}
return nil
}
if HasAMDGPU() {
if m.markDetectedInternal("amd", AMDGPUSysfsPath) {
slog.InfoContext(ctx, "AMD GPU detected", "method", "sysfs", "path", AMDGPUSysfsPath)
}
return nil
}
if path, err := exec.LookPath("intel_gpu_top"); err == nil {
if m.markDetectedInternal("intel", path) {
slog.InfoContext(ctx, "Intel GPU detected", "tool", "intel_gpu_top", "path", path)
}
return nil
}
m.detectionDone = true
return errors.New("no supported GPU found")
}
// HasAMDGPU reports whether a card with mem_info_vram_total exists under AMDGPUSysfsPath.
func HasAMDGPU() bool {
// Reads fixed system paths under /sys; not under any acfs root.
entries, err := os.ReadDir(AMDGPUSysfsPath)
if err != nil {
return false
}
for _, entry := range entries {
name := entry.Name()
if !strings.HasPrefix(name, "card") || strings.Contains(name, "-") {
continue
}
if _, statErr := os.Stat(fmt.Sprintf("%s/%s/device/mem_info_vram_total", AMDGPUSysfsPath, name)); statErr == nil {
return true
}
}
return false
}
// readSysfsValueInternal parses a numeric value from a sysfs file.
func readSysfsValueInternal(path string) (uint64, error) {
// Reads fixed system paths under /sys; not under any acfs root.
data, err := os.ReadFile(path)
if err != nil {
return 0, err
}
return strconv.ParseUint(strings.TrimSpace(string(data)), 10, 64)
}
func getNvidiaStatsInternal(ctx context.Context) ([]systemtypes.GPUStats, error) {
ctx, cancel := context.WithTimeout(ctx, 3*time.Second)
defer cancel()
cmd := exec.CommandContext(ctx, "nvidia-smi",
"--query-gpu=index,name,memory.used,memory.total",
"--format=csv,noheader,nounits")
output, err := cmd.Output()
if err != nil {
return nil, fmt.Errorf("nvidia-smi execution failed: %w", err)
}
reader := csv.NewReader(bytes.NewReader(output))
reader.TrimLeadingSpace = true
records, err := reader.ReadAll()
if err != nil {
return nil, fmt.Errorf("failed to parse nvidia-smi output: %w", err)
}
var stats []systemtypes.GPUStats
for _, record := range records {
if len(record) < 4 {
continue
}
index, atoiErr := strconv.Atoi(strings.TrimSpace(record[0]))
if atoiErr != nil {
slog.WarnContext(ctx, "Failed to parse GPU index", "value", record[0])
continue
}
memUsed, atoiErr := strconv.ParseFloat(strings.TrimSpace(record[2]), 64)
if atoiErr != nil {
slog.WarnContext(ctx, "Failed to parse memory used", "value", record[2])
continue
}
memTotal, atoiErr := strconv.ParseFloat(strings.TrimSpace(record[3]), 64)
if atoiErr != nil {
slog.WarnContext(ctx, "Failed to parse memory total", "value", record[3])
continue
}
stats = append(stats, systemtypes.GPUStats{
Name: strings.TrimSpace(record[1]),
Index: index,
MemoryUsed: memUsed * 1024 * 1024,
MemoryTotal: memTotal * 1024 * 1024,
})
}
if len(stats) == 0 {
return nil, errors.New("no GPU data parsed from nvidia-smi")
}
slog.DebugContext(ctx, "Collected NVIDIA GPU stats", "gpuCount", len(stats))
return stats, nil
}
func getAMDStatsInternal(ctx context.Context) ([]systemtypes.GPUStats, error) {
// Reads fixed system paths under /sys; not under any acfs root.
entries, err := os.ReadDir(AMDGPUSysfsPath)
if err != nil {
slog.WarnContext(ctx, "Failed to read DRM sysfs directory", "error", err)
return nil, fmt.Errorf("failed to read sysfs: %w", err)
}
var stats []systemtypes.GPUStats
index := 0
for _, entry := range entries {
name := entry.Name()
if !strings.HasPrefix(name, "card") || strings.Contains(name, "-") {
continue
}
devicePath := fmt.Sprintf("%s/%s/device", AMDGPUSysfsPath, name)
memTotalBytes, readSysfsValueErr := readSysfsValueInternal(devicePath + "/mem_info_vram_total")
if readSysfsValueErr != nil {
continue
}
memUsedBytes, readSysfsValueErr := readSysfsValueInternal(devicePath + "/mem_info_vram_used")
if readSysfsValueErr != nil {
slog.WarnContext(ctx, "Failed to read AMD GPU memory used", "card", name, "error", readSysfsValueErr)
continue
}
stats = append(stats, systemtypes.GPUStats{
Name: fmt.Sprintf("AMD GPU %d", index),
Index: index,
MemoryUsed: float64(memUsedBytes),
MemoryTotal: float64(memTotalBytes),
})
index++
}
if len(stats) == 0 {
return nil, errors.New("no AMD GPU data found in sysfs")
}
slog.DebugContext(ctx, "Collected AMD GPU stats", "gpuCount", len(stats))
return stats, nil
}
func getIntelStatsInternal(ctx context.Context) ([]systemtypes.GPUStats, error) {
stats := []systemtypes.GPUStats{
{Name: "Intel GPU", Index: 0, MemoryUsed: 0, MemoryTotal: 0},
}
slog.DebugContext(ctx, "Intel GPU detected but detailed stats not yet implemented")
return stats, nil
}