Lightweight server monitoring with historical data, docker stats, and alerts.
//go:build linux
package agent
import (
"bufio"
"errors"
"fmt"
"io"
"io/fs"
"log/slog"
"os"
"path/filepath"
"strconv"
"strings"
"sync"
"syscall"
"time"
"github.com/henrygd/beszel/agent/utils"
"github.com/henrygd/beszel/internal/entities/system"
)
var amdSysfsSleep = time.Sleep
var amdgpuNameCache = struct {
sync.RWMutex
hits map[string]string
misses map[string]struct{}
}{
hits: make(map[string]string),
misses: make(map[string]struct{}),
}
// hasAmdSysfs returns true if any AMD GPU sysfs nodes are found
func (gm *GPUManager) hasAmdSysfs() bool {
cards, err := filepath.Glob(filepath.Join(drmSysfsRoot, "card*/device/vendor"))
if err != nil {
return false
}
for _, vendorPath := range cards {
vendor, err := utils.ReadStringFileLimited(vendorPath, 64)
if err == nil && vendor == "0x1002" {
return true
}
}
return false
}
// collectAmdStats collects AMD GPU metrics directly from sysfs to avoid the overhead of rocm-smi
func (gm *GPUManager) collectAmdStats() error {
sysfsPollInterval := 3000 * time.Millisecond
cards, err := filepath.Glob(filepath.Join(drmSysfsRoot, "card*"))
if err != nil {
return err
}
var amdGpuPaths []string
for _, card := range cards {
// Ignore symbolic links and non-main card directories
if strings.Contains(filepath.Base(card), "-") || !isAmdGpu(card) {
continue
}
amdGpuPaths = append(amdGpuPaths, card)
}
if len(amdGpuPaths) == 0 {
return errNoValidData
}
slog.Debug("Using sysfs for AMD GPU data collection")
failures := 0
for {
hasData := false
for _, cardPath := range amdGpuPaths {
if gm.updateAmdGpuData(cardPath) {
hasData = true
}
}
if !hasData {
failures++
// Keep polling so transient sysfs failures can recover without an agent restart,
// but only warn for the first retries to avoid flooding the log.
log := slog.Debug
if failures <= maxFailureRetries {
log = slog.Warn
}
log("No AMD GPU data from sysfs", "failures", failures)
amdSysfsSleep(retryWaitTime)
continue
}
failures = 0
amdSysfsSleep(sysfsPollInterval)
}
}
// isAmdGpu checks whether a DRM card path belongs to AMD vendor ID 0x1002.
func isAmdGpu(cardPath string) bool {
vendor, err := utils.ReadStringFileLimited(filepath.Join(cardPath, "device/vendor"), 64)
if err != nil {
return false
}
return vendor == "0x1002"
}
// unifiedMemoryMaxVram is the dedicated VRAM size below which a device without a
// gpu_metrics table is taken for an APU (typical carve-out sizes are 512 MiB to 1 GiB).
const unifiedMemoryMaxVram = 2 << 30
// amdMemoryKind is the cached result of classifying an AMD device's memory.
type amdMemoryKind uint8
const (
amdMemoryUnknown amdMemoryKind = iota // not classified yet; gpu_metrics is read again on the next poll
amdMemoryDiscrete // gpu_metrics format revision 1
amdMemoryUnified // gpu_metrics format revision 2 or 3
amdMemoryBySize // no usable gpu_metrics table; decide by dedicated VRAM size
)
// amdMemoryKindCache holds the memory kind per device path, since it never changes for a device.
var amdMemoryKindCache = struct {
sync.Mutex
kinds map[string]amdMemoryKind
}{kinds: make(map[string]amdMemoryKind)}
// amdGpuIsUnifiedMemory reports whether the device shares memory with the CPU (an APU).
// The gpu_metrics header is read only until it yields a stable answer, which is then cached.
func amdGpuIsUnifiedMemory(devicePath string, vramTotal float64) bool {
amdMemoryKindCache.Lock()
kind := amdMemoryKindCache.kinds[devicePath]
amdMemoryKindCache.Unlock()
if kind == amdMemoryUnknown {
kind = detectAmdMemoryKind(devicePath)
if kind != amdMemoryUnknown {
amdMemoryKindCache.Lock()
amdMemoryKindCache.kinds[devicePath] = kind
amdMemoryKindCache.Unlock()
}
}
switch kind {
case amdMemoryDiscrete:
return false
case amdMemoryUnified:
return true
default:
// No gpu_metrics table (older ASICs) or a transient read failure: fall back to the
// size of the dedicated VRAM. A failed VRAM read (0) is not taken for an APU.
return vramTotal > 0 && vramTotal < unifiedMemoryMaxVram
}
}
// detectAmdMemoryKind classifies a device from its gpu_metrics table.
//
// amdgpu exposes the gpu_metrics table in sysfs, whose first four bytes are a
// metrics_table_header (u16 structure_size, u8 format_revision, u8 content_revision).
// Format revision 1 is used by discrete cards (gpu_metrics_v1_x), 2 and 3 by APUs
// (gpu_metrics_v2_x / v3_x). Returns amdMemoryUnknown only for failures that may be
// transient, so the table is read again on the next poll.
func detectAmdMemoryKind(devicePath string) amdMemoryKind {
header := make([]byte, 4)
n, err := readSysfsHead(filepath.Join(devicePath, "gpu_metrics"), header)
switch {
case err == nil && n == len(header):
switch header[2] {
case 1:
return amdMemoryDiscrete
case 2, 3:
return amdMemoryUnified
}
return amdMemoryBySize
case err == nil, errors.Is(err, fs.ErrNotExist), errors.Is(err, syscall.EOPNOTSUPP):
// short table, no table, or not supported by this ASIC: none of these change at runtime
return amdMemoryBySize
}
return amdMemoryUnknown
}
// updateAmdGpuData reads GPU metrics from sysfs and updates the GPU data map.
// Returns true if at least some data was successfully read.
func (gm *GPUManager) updateAmdGpuData(cardPath string) bool {
devicePath := filepath.Join(cardPath, "device")
id := filepath.Base(cardPath)
// Read all sysfs values first (no lock needed - these can be slow)
usage, usageErr := readSysfsFloat(filepath.Join(devicePath, "gpu_busy_percent"))
memUsed, memUsedErr := readSysfsFloat(filepath.Join(devicePath, "mem_info_vram_used"))
memTotal, _ := readSysfsFloat(filepath.Join(devicePath, "mem_info_vram_total"))
// On a unified memory device (APU) the GPU mostly uses system memory through GTT, so add it to the
// memory used and total (https://github.com/henrygd/beszel/issues/1569#issuecomment-3837640484).
// On a discrete card GTT is just mappable system RAM, not VRAM, and must not inflate the numbers.
if amdGpuIsUnifiedMemory(devicePath, memTotal) {
if gttUsed, err := readSysfsFloat(filepath.Join(devicePath, "mem_info_gtt_used")); err == nil && gttUsed > 0 {
if gttTotal, err := readSysfsFloat(filepath.Join(devicePath, "mem_info_gtt_total")); err == nil {
memUsed += gttUsed
memTotal += gttTotal
}
}
}
var temp, power float64
hwmons, _ := filepath.Glob(filepath.Join(devicePath, "hwmon/hwmon*"))
for _, hwmonDir := range hwmons {
if t, err := readSysfsFloat(filepath.Join(hwmonDir, "temp1_input")); err == nil {
temp = t / 1000.0
}
if p, err := readSysfsFloat(filepath.Join(hwmonDir, "power1_average")); err == nil {
power += p / 1000000.0
} else if p, err := readSysfsFloat(filepath.Join(hwmonDir, "power1_input")); err == nil {
power += p / 1000000.0
}
}
// Check if we got any meaningful data
if usageErr != nil && memUsedErr != nil && temp == 0 {
return false
}
// Single lock to update all values atomically
gm.Lock()
defer gm.Unlock()
gpu, ok := gm.GpuDataMap[id]
if !ok {
gpu = &system.GPUData{Name: getAmdGpuName(devicePath)}
gm.GpuDataMap[id] = gpu
}
if usageErr == nil {
gpu.Usage += usage
}
gpu.MemoryUsed = utils.BytesToMegabytes(memUsed)
gpu.MemoryTotal = utils.BytesToMegabytes(memTotal)
gpu.Temperature = temp
gpu.Power += power
gpu.Count++
return true
}
// readSysfsFloat reads and parses a numeric value from a sysfs file.
// Files that fail slowly or are unsupported are skipped for a while (see sysfsBackoff).
func readSysfsFloat(path string) (float64, error) {
now := time.Now()
if !amdSysfsBackoff.allow(path, now) {
return 0, errSysfsBackoff
}
val, err := utils.ReadStringFileLimited(path, 64)
amdSysfsBackoff.record(path, err, time.Since(now), now)
if err != nil {
slog.Debug("Failed to read sysfs value", "path", path, "error", err)
return 0, err
}
return strconv.ParseFloat(val, 64)
}
// readSysfsHead reads up to len(buf) bytes from the start of a binary sysfs file,
// honoring the same backoff as readSysfsFloat. A file shorter than buf is not an error.
func readSysfsHead(path string, buf []byte) (int, error) {
now := time.Now()
if !amdSysfsBackoff.allow(path, now) {
return 0, errSysfsBackoff
}
n, err := readFileHead(path, buf)
amdSysfsBackoff.record(path, err, time.Since(now), now)
if err != nil {
slog.Debug("Failed to read sysfs file", "path", path, "error", err)
}
return n, err
}
func readFileHead(path string, buf []byte) (int, error) {
f, err := os.Open(path)
if err != nil {
return 0, err
}
defer f.Close()
n, err := io.ReadFull(f, buf)
if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) {
err = nil
}
return n, err
}
const (
// sysfsSlowReadThreshold is how long a read may take to fail before the file is backed off.
// Healthy amdgpu reads take well under a millisecond.
sysfsSlowReadThreshold = 500 * time.Millisecond
sysfsBackoffMin = 30 * time.Second
sysfsBackoffMax = 10 * time.Minute
)
var errSysfsBackoff = errors.New("sysfs read skipped during backoff")
// amdSysfsBackoff tracks AMD sysfs files to skip after slow failures or unsupported reads.
var amdSysfsBackoff = newSysfsBackoff()
// sysfsBackoff skips sysfs files whose reads fail slowly or with EOPNOTSUPP.
// Slow successes are not backed off, since a stalled host can delay a healthy read.
// When the amdgpu driver can't reach the SMU, reads can busy-wait in the kernel for
// several seconds before failing, which burns a CPU core on every poll
// (https://github.com/henrygd/beszel/issues/2699).
type sysfsBackoff struct {
sync.Mutex
entries map[string]*sysfsBackoffEntry
}
type sysfsBackoffEntry struct {
until time.Time
delay time.Duration
}
func newSysfsBackoff() *sysfsBackoff {
return &sysfsBackoff{entries: make(map[string]*sysfsBackoffEntry)}
}
// allow reports whether path may be read at now.
func (b *sysfsBackoff) allow(path string, now time.Time) bool {
b.Lock()
defer b.Unlock()
entry, ok := b.entries[path]
return !ok || !now.Before(entry.until)
}
// record updates the backoff state for path after a read that took elapsed.
// Slow failures and unsupported reads double the delay up to sysfsBackoffMax; a success resets it.
func (b *sysfsBackoff) record(path string, err error, elapsed time.Duration, now time.Time) {
b.Lock()
defer b.Unlock()
if err == nil {
delete(b.entries, path)
return
}
if elapsed < sysfsSlowReadThreshold && !errors.Is(err, syscall.EOPNOTSUPP) {
return
}
entry, ok := b.entries[path]
if !ok {
entry = &sysfsBackoffEntry{delay: sysfsBackoffMin}
b.entries[path] = entry
} else {
entry.delay = min(entry.delay*2, sysfsBackoffMax)
}
entry.until = now.Add(entry.delay)
slog.Warn("Backing off slow or unsupported sysfs file", "path", path, "elapsed", elapsed, "error", err, "retry", entry.delay)
}
// normalizeHexID normalizes hex IDs by trimming spaces, lowercasing, and dropping 0x.
func normalizeHexID(id string) string {
return strings.TrimPrefix(strings.ToLower(strings.TrimSpace(id)), "0x")
}
// cacheKeyForAmdgpu builds the cache key for a device and optional revision.
func cacheKeyForAmdgpu(deviceID, revisionID string) string {
if revisionID != "" {
return deviceID + ":" + revisionID
}
return deviceID
}
// lookupAmdgpuNameInFile resolves an AMDGPU name from amdgpu.ids by device/revision.
func lookupAmdgpuNameInFile(deviceID, revisionID, filePath string) (name string, exact bool, found bool) {
file, err := os.Open(filePath)
if err != nil {
return "", false, false
}
defer file.Close()
var byDevice string
scanner := bufio.NewScanner(file)
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
if line == "" || strings.HasPrefix(line, "#") {
continue
}
parts := strings.SplitN(line, ",", 3)
if len(parts) != 3 {
continue
}
dev := normalizeHexID(parts[0])
rev := normalizeHexID(parts[1])
productName := strings.TrimSpace(parts[2])
if dev == "" || productName == "" || dev != deviceID {
continue
}
if byDevice == "" {
byDevice = productName
}
if revisionID != "" && rev == revisionID {
return productName, true, true
}
}
if byDevice != "" {
return byDevice, false, true
}
return "", false, false
}
// getCachedAmdgpuName returns cached hit/miss status for the given device/revision.
func getCachedAmdgpuName(deviceID, revisionID string) (name string, found bool, done bool) {
// Build the list of cache keys to check. We always look up the exact device+revision key.
// When revisionID is set, we also look up deviceID alone, since the cache may store a
// device-only fallback when we couldn't resolve the exact revision.
keys := []string{cacheKeyForAmdgpu(deviceID, revisionID)}
if revisionID != "" {
keys = append(keys, deviceID)
}
knownMisses := 0
amdgpuNameCache.RLock()
defer amdgpuNameCache.RUnlock()
for _, key := range keys {
if name, ok := amdgpuNameCache.hits[key]; ok {
return name, true, true
}
if _, ok := amdgpuNameCache.misses[key]; ok {
knownMisses++
}
}
// done=true means "don't bother doing slow lookup": we either found a name (above) or
// every key we checked was already a known miss, so we've tried before and failed.
return "", false, knownMisses == len(keys)
}
// normalizeAmdgpuName trims standard suffixes from AMDGPU product names.
func normalizeAmdgpuName(name string) string {
for _, suffix := range []string{" Graphics", " Series"} {
name = strings.TrimSuffix(name, suffix)
}
return name
}
// cacheAmdgpuName stores a resolved AMDGPU name in the lookup cache.
func cacheAmdgpuName(deviceID, revisionID, name string, exact bool) {
name = normalizeAmdgpuName(name)
amdgpuNameCache.Lock()
defer amdgpuNameCache.Unlock()
if exact && revisionID != "" {
amdgpuNameCache.hits[cacheKeyForAmdgpu(deviceID, revisionID)] = name
}
amdgpuNameCache.hits[deviceID] = name
}
// cacheMissingAmdgpuName records unresolved device/revision lookups.
func cacheMissingAmdgpuName(deviceID, revisionID string) {
amdgpuNameCache.Lock()
defer amdgpuNameCache.Unlock()
amdgpuNameCache.misses[deviceID] = struct{}{}
if revisionID != "" {
amdgpuNameCache.misses[cacheKeyForAmdgpu(deviceID, revisionID)] = struct{}{}
}
}
// getAmdGpuName attempts to get a descriptive GPU name.
// First tries product_name (rarely available), then looks up the PCI device ID.
// Falls back to showing the raw device ID if not found in the lookup table.
func getAmdGpuName(devicePath string) string {
// Try product_name first (works for some enterprise GPUs)
if prod, err := utils.ReadStringFileLimited(filepath.Join(devicePath, "product_name"), 128); err == nil {
return prod
}
// Read PCI device ID and look it up
if deviceID, err := utils.ReadStringFileLimited(filepath.Join(devicePath, "device"), 64); err == nil {
id := normalizeHexID(deviceID)
revision := ""
if rev, revErr := utils.ReadStringFileLimited(filepath.Join(devicePath, "revision"), 64); revErr == nil {
revision = normalizeHexID(rev)
}
if name, found, done := getCachedAmdgpuName(id, revision); found {
return name
} else if !done {
if name, exact, ok := lookupAmdgpuNameInFile(id, revision, "/usr/share/libdrm/amdgpu.ids"); ok {
cacheAmdgpuName(id, revision, name, exact)
return normalizeAmdgpuName(name)
}
cacheMissingAmdgpuName(id, revision)
}
return fmt.Sprintf("AMD GPU (%s)", id)
}
return "AMD GPU"
}