Skip to content

Modern Docker Management, Designed for Everyone

10.9 KB
Raw Blame
package system

import (
	"bytes"
	"context"
	"encoding/csv"
	"errors"
	"fmt"
	"log/slog"
	"os"
	"os/exec"
	"strconv"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	systemtypes "github.com/getarcaneapp/arcane/types/v2/system"
	"github.com/samber/hot"
	kit "go.getarcane.app/kit/pkg"
	"golang.org/x/sync/singleflight"
)

// AMDGPUSysfsPath is the sysfs base used to discover AMD GPUs.
const (
	AMDGPUSysfsPath = "/sys/class/drm"

	// gpuDetectionTTL bounds how long a successful detection result is reused before re-detecting.
	gpuDetectionTTL = 30 * time.Second

	// gpuStatsTTL bounds how often the vendor tool is invoked: reading VRAM
	// stats forks a process (nvidia-smi, intel_gpu_top), which is far too
	// expensive to repeat on every sampler tick.
	gpuStatsTTL = 3 * time.Second
)

// GPUMonitor probes for an attached GPU (NVIDIA / AMD / Intel) and reports VRAM usage.
// Detection is cached for gpuDetectionTTL; once a vendor is detected, subsequent Stats
// calls invoke the vendor-specific tool directly.
type GPUMonitor struct {
	configuredType string

	detectionMu    sync.Mutex
	detectionCache *hot.HotCache[struct{}, gpuDetection]
	detectionDone  bool

	// statsCache holds the last vendor-tool result (success or failure) for
	// gpuStatsTTL so concurrent consumers and fast sampler ticks share one
	// process fork instead of each launching their own.
	statsCache atomic.Pointer[gpuStatsSnapshotInternal]
	statsGroup singleflight.Group

	announcedType string

	enabled bool
}

type gpuStatsSnapshotInternal struct {
	stats     []systemtypes.GPUStats
	err       error
	sampledAt time.Time
}

type gpuDetection struct {
	detected bool
	gpuType  string
	toolPath string
}

// NewGPUMonitor creates a monitor. enabled gates Stats; when false, Stats returns
// (nil, nil). configuredType is the user-pinned vendor ("nvidia"|"amd"|"intel"|"auto"|"")
// — anything else falls back to auto-detection.
func NewGPUMonitor(enabled bool, configuredType string) *GPUMonitor {
	return &GPUMonitor{
		enabled:        enabled,
		configuredType: configuredType,
		detectionCache: hot.NewHotCache[struct{}, gpuDetection](hot.LRU, 1).
			WithTTL(gpuDetectionTTL).
			Build(),
	}
}

// Enabled reports whether GPU monitoring is on.
func (m *GPUMonitor) Enabled() bool { return m.enabled }

// Stats returns per-GPU VRAM stats. Returns (nil, 0, nil) when monitoring is disabled
// or no GPU is detected; vendor-specific errors are propagated otherwise.
// Results (including failures) are cached for gpuStatsTTL, so callers may
// receive stats up to that many seconds old.
func (m *GPUMonitor) Stats(ctx context.Context) ([]systemtypes.GPUStats, error) {
	if !m.enabled {
		return nil, nil
	}

	if snapshot := m.statsCache.Load(); snapshot != nil && time.Since(snapshot.sampledAt) < gpuStatsTTL {
		return snapshot.stats, snapshot.err
	}

	result, err, _ := m.statsGroup.Do("stats", func() (any, error) {
		previous := m.statsCache.Load()
		if previous != nil && time.Since(previous.sampledAt) < gpuStatsTTL {
			return previous.stats, previous.err
		}
		stats, statsErr := m.statsUncachedInternal(ctx)
		if previousErr := kit.FromPtr(previous).err; statsErr != nil && (previousErr == nil || previousErr.Error() != statsErr.Error()) {
			slog.WarnContext(ctx, "GPU stats unavailable", "error", statsErr)
		}
		m.statsCache.Store(&gpuStatsSnapshotInternal{stats: stats, err: statsErr, sampledAt: time.Now()})
		return stats, statsErr
	})
	stats, _ := result.([]systemtypes.GPUStats)
	return stats, err
}

func (m *GPUMonitor) statsUncachedInternal(ctx context.Context) ([]systemtypes.GPUStats, error) {
	m.detectionMu.Lock()
	done := m.detectionDone
	m.detectionMu.Unlock()
	if !done {
		if err := m.detectInternal(ctx); err != nil {
			return nil, err
		}
	}

	if detection, found, _ := m.detectionCache.Get(struct{}{}); found && detection.detected {
		return m.statsForTypeInternal(ctx, detection.gpuType)
	}

	if err := m.detectInternal(ctx); err != nil {
		return nil, err
	}

	detection, found, _ := m.detectionCache.Get(struct{}{})
	if !found || detection.gpuType == "" {
		return nil, errors.New("no supported GPU found")
	}
	return m.statsForTypeInternal(ctx, detection.gpuType)
}

func (m *GPUMonitor) statsForTypeInternal(ctx context.Context, gpuType string) ([]systemtypes.GPUStats, error) {
	switch gpuType {
	case "nvidia":
		return getNvidiaStatsInternal(ctx)
	case "amd":
		return getAMDStatsInternal(ctx)
	case "intel":
		return getIntelStatsInternal(ctx)
	default:
		return nil, errors.New("no supported GPU found")
	}
}

// markDetected records a successful detection.
func (m *GPUMonitor) markDetectedInternal(gpuType, toolPath string) bool {
	m.detectionCache.Set(struct{}{}, gpuDetection{detected: true, gpuType: gpuType, toolPath: toolPath})
	m.detectionDone = true
	announce := m.announcedType != gpuType
	m.announcedType = gpuType
	return announce
}

// detect runs vendor probing under detectionMu. The configuredType pin is honored when set
// to a known vendor; otherwise vendors are tried in order: nvidia → amd → intel.
func (m *GPUMonitor) detectInternal(ctx context.Context) error {
	m.detectionMu.Lock()
	defer m.detectionMu.Unlock()

	if t := m.configuredType; t != "" && t != "auto" {
		switch t {
		case "nvidia":
			if path, err := exec.LookPath("nvidia-smi"); err == nil {
				if m.markDetectedInternal("nvidia", path) {
					slog.InfoContext(ctx, "Using configured GPU type", "type", "nvidia")
				}
				return nil
			}
			return errors.New("nvidia-smi not found but GPU_TYPE set to nvidia")
		case "amd":
			if HasAMDGPU() {
				if m.markDetectedInternal("amd", AMDGPUSysfsPath) {
					slog.InfoContext(ctx, "Using configured GPU type", "type", "amd")
				}
				return nil
			}
			return errors.New("AMD GPU not found in sysfs but GPU_TYPE set to amd")
		case "intel":
			if path, err := exec.LookPath("intel_gpu_top"); err == nil {
				if m.markDetectedInternal("intel", path) {
					slog.InfoContext(ctx, "Using configured GPU type", "type", "intel")
				}
				return nil
			}
			return errors.New("intel_gpu_top not found but GPU_TYPE set to intel")
		default:
			slog.WarnContext(ctx, "Invalid GPU_TYPE specified, falling back to auto-detection", "gpuType", t)
		}
	}

	if path, err := exec.LookPath("nvidia-smi"); err == nil {
		if m.markDetectedInternal("nvidia", path) {
			slog.InfoContext(ctx, "NVIDIA GPU detected", "tool", "nvidia-smi", "path", path)
		}
		return nil
	}
	if HasAMDGPU() {
		if m.markDetectedInternal("amd", AMDGPUSysfsPath) {
			slog.InfoContext(ctx, "AMD GPU detected", "method", "sysfs", "path", AMDGPUSysfsPath)
		}
		return nil
	}
	if path, err := exec.LookPath("intel_gpu_top"); err == nil {
		if m.markDetectedInternal("intel", path) {
			slog.InfoContext(ctx, "Intel GPU detected", "tool", "intel_gpu_top", "path", path)
		}
		return nil
	}

	m.detectionDone = true
	return errors.New("no supported GPU found")
}

// HasAMDGPU reports whether a card with mem_info_vram_total exists under AMDGPUSysfsPath.
func HasAMDGPU() bool {
	// Reads fixed system paths under /sys; not under any acfs root.
	entries, err := os.ReadDir(AMDGPUSysfsPath)
	if err != nil {
		return false
	}
	for _, entry := range entries {
		name := entry.Name()
		if !strings.HasPrefix(name, "card") || strings.Contains(name, "-") {
			continue
		}
		if _, statErr := os.Stat(fmt.Sprintf("%s/%s/device/mem_info_vram_total", AMDGPUSysfsPath, name)); statErr == nil {
			return true
		}
	}
	return false
}

// readSysfsValueInternal parses a numeric value from a sysfs file.
func readSysfsValueInternal(path string) (uint64, error) {
	// Reads fixed system paths under /sys; not under any acfs root.
	data, err := os.ReadFile(path)
	if err != nil {
		return 0, err
	}
	return strconv.ParseUint(strings.TrimSpace(string(data)), 10, 64)
}

func getNvidiaStatsInternal(ctx context.Context) ([]systemtypes.GPUStats, error) {
	ctx, cancel := context.WithTimeout(ctx, 3*time.Second)
	defer cancel()

	cmd := exec.CommandContext(ctx, "nvidia-smi",
		"--query-gpu=index,name,memory.used,memory.total",
		"--format=csv,noheader,nounits")

	output, err := cmd.Output()
	if err != nil {
		return nil, fmt.Errorf("nvidia-smi execution failed: %w", err)
	}

	reader := csv.NewReader(bytes.NewReader(output))
	reader.TrimLeadingSpace = true
	records, err := reader.ReadAll()
	if err != nil {
		return nil, fmt.Errorf("failed to parse nvidia-smi output: %w", err)
	}

	var stats []systemtypes.GPUStats
	for _, record := range records {
		if len(record) < 4 {
			continue
		}
		index, atoiErr := strconv.Atoi(strings.TrimSpace(record[0]))
		if atoiErr != nil {
			slog.WarnContext(ctx, "Failed to parse GPU index", "value", record[0])
			continue
		}
		memUsed, atoiErr := strconv.ParseFloat(strings.TrimSpace(record[2]), 64)
		if atoiErr != nil {
			slog.WarnContext(ctx, "Failed to parse memory used", "value", record[2])
			continue
		}
		memTotal, atoiErr := strconv.ParseFloat(strings.TrimSpace(record[3]), 64)
		if atoiErr != nil {
			slog.WarnContext(ctx, "Failed to parse memory total", "value", record[3])
			continue
		}
		stats = append(stats, systemtypes.GPUStats{
			Name:        strings.TrimSpace(record[1]),
			Index:       index,
			MemoryUsed:  memUsed * 1024 * 1024,
			MemoryTotal: memTotal * 1024 * 1024,
		})
	}

	if len(stats) == 0 {
		return nil, errors.New("no GPU data parsed from nvidia-smi")
	}

	slog.DebugContext(ctx, "Collected NVIDIA GPU stats", "gpuCount", len(stats))
	return stats, nil
}

func getAMDStatsInternal(ctx context.Context) ([]systemtypes.GPUStats, error) {
	// Reads fixed system paths under /sys; not under any acfs root.
	entries, err := os.ReadDir(AMDGPUSysfsPath)
	if err != nil {
		slog.WarnContext(ctx, "Failed to read DRM sysfs directory", "error", err)
		return nil, fmt.Errorf("failed to read sysfs: %w", err)
	}

	var stats []systemtypes.GPUStats
	index := 0
	for _, entry := range entries {
		name := entry.Name()
		if !strings.HasPrefix(name, "card") || strings.Contains(name, "-") {
			continue
		}

		devicePath := fmt.Sprintf("%s/%s/device", AMDGPUSysfsPath, name)
		memTotalBytes, readSysfsValueErr := readSysfsValueInternal(devicePath + "/mem_info_vram_total")
		if readSysfsValueErr != nil {
			continue
		}
		memUsedBytes, readSysfsValueErr := readSysfsValueInternal(devicePath + "/mem_info_vram_used")
		if readSysfsValueErr != nil {
			slog.WarnContext(ctx, "Failed to read AMD GPU memory used", "card", name, "error", readSysfsValueErr)
			continue
		}

		stats = append(stats, systemtypes.GPUStats{
			Name:        fmt.Sprintf("AMD GPU %d", index),
			Index:       index,
			MemoryUsed:  float64(memUsedBytes),
			MemoryTotal: float64(memTotalBytes),
		})
		index++
	}

	if len(stats) == 0 {
		return nil, errors.New("no AMD GPU data found in sysfs")
	}

	slog.DebugContext(ctx, "Collected AMD GPU stats", "gpuCount", len(stats))
	return stats, nil
}

func getIntelStatsInternal(ctx context.Context) ([]systemtypes.GPUStats, error) {
	stats := []systemtypes.GPUStats{
		{Name: "Intel GPU", Index: 0, MemoryUsed: 0, MemoryTotal: 0},
	}
	slog.DebugContext(ctx, "Intel GPU detected but detailed stats not yet implemented")
	return stats, nil
}