Skip to content

Lightweight server monitoring with historical data, docker stats, and alerts.

fix(agent): keep amd_sysfs collector polling after repeated read failures (#2687)

cobalt-kite12784fb2fdbb6b
diff --git a/agent/gpu_amd_linux.go b/agent/gpu_amd_linux.go
index a6b249737..54a1c4093 100644
--- a/agent/gpu_amd_linux.go
+++ b/agent/gpu_amd_linux.go
@@ -17,6 +17,8 @@ import (
"github.com/henrygd/beszel/internal/entities/system"
)
+var amdSysfsSleep = time.Sleep
+
var amdgpuNameCache = struct {
sync.RWMutex
hits map[string]string
@@ -28,7 +30,7 @@ var amdgpuNameCache = struct {
// hasAmdSysfs returns true if any AMD GPU sysfs nodes are found
func (gm *GPUManager) hasAmdSysfs() bool {
- cards, err := filepath.Glob("/sys/class/drm/card*/device/vendor")
+ cards, err := filepath.Glob(filepath.Join(drmSysfsRoot, "card*/device/vendor"))
if err != nil {
return false
}
@@ -44,7 +46,7 @@ func (gm *GPUManager) hasAmdSysfs() bool {
// collectAmdStats collects AMD GPU metrics directly from sysfs to avoid the overhead of rocm-smi
func (gm *GPUManager) collectAmdStats() error {
sysfsPollInterval := 3000 * time.Millisecond
- cards, err := filepath.Glob("/sys/class/drm/card*")
+ cards, err := filepath.Glob(filepath.Join(drmSysfsRoot, "card*"))
if err != nil {
return err
}
@@ -74,15 +76,18 @@ func (gm *GPUManager) collectAmdStats() error {
}
if !hasData {
failures++
- if failures > maxFailureRetries {
- return errNoValidData
+ // Keep polling so transient sysfs failures can recover without an agent restart,
+ // but only warn for the first retries to avoid flooding the log.
+ log := slog.Debug
+ if failures <= maxFailureRetries {
+ log = slog.Warn
}
- slog.Warn("No AMD GPU data from sysfs", "failures", failures)
- time.Sleep(retryWaitTime)
+ log("No AMD GPU data from sysfs", "failures", failures)
+ amdSysfsSleep(retryWaitTime)
continue
}
failures = 0
- time.Sleep(sysfsPollInterval)
+ amdSysfsSleep(sysfsPollInterval)
}
}
diff --git a/agent/gpu_amd_linux_test.go b/agent/gpu_amd_linux_test.go
index 76b286cc2..bbcaadbcf 100644
--- a/agent/gpu_amd_linux_test.go
+++ b/agent/gpu_amd_linux_test.go
@@ -5,7 +5,9 @@ package agent
import (
"os"
"path/filepath"
+ "runtime"
"testing"
+ "time"
"github.com/henrygd/beszel/agent/utils"
"github.com/henrygd/beszel/internal/entities/system"
@@ -13,6 +15,72 @@ import (
"github.com/stretchr/testify/require"
)
+func TestCollectAmdStatsRecoversAfterReadFailures(t *testing.T) {
+ root := t.TempDir()
+ oldRoot, oldSleep := drmSysfsRoot, amdSysfsSleep
+ drmSysfsRoot = root
+ t.Cleanup(func() {
+ drmSysfsRoot, amdSysfsSleep = oldRoot, oldSleep
+ })
+
+ devicePath := filepath.Join(root, "card0", "device")
+ require.NoError(t, os.MkdirAll(devicePath, 0o755))
+ require.NoError(t, os.WriteFile(filepath.Join(devicePath, "vendor"), []byte("0x1002"), 0o644))
+ metrics := map[string]string{
+ "gpu_busy_percent": "25",
+ "mem_info_vram_used": "1073741824",
+ "mem_info_vram_total": "2147483648",
+ }
+ writeMetrics := func() {
+ for name, value := range metrics {
+ require.NoError(t, os.WriteFile(filepath.Join(devicePath, name), []byte(value), 0o644))
+ }
+ }
+ writeMetrics()
+
+ gm := &GPUManager{GpuDataMap: map[string]*system.GPUData{"card0": {Name: "AMD Test GPU"}}}
+ failures := 0
+ removed := false
+ recovered := false
+ amdSysfsSleep = func(delay time.Duration) {
+ if !removed {
+ assert.Equal(t, 1.0, gm.GpuDataMap["card0"].Count)
+ assert.Equal(t, 3*time.Second, delay)
+ for name := range metrics {
+ require.NoError(t, os.Remove(filepath.Join(devicePath, name)))
+ }
+ removed = true
+ return
+ }
+ if gm.GpuDataMap["card0"].Count == 2 {
+ assert.Equal(t, 3*time.Second, delay)
+ recovered = true
+ // Stop the collector goroutine after observing the recovered sample.
+ runtime.Goexit()
+ }
+ assert.Equal(t, retryWaitTime, delay)
+ failures++
+ if failures == maxFailureRetries+2 {
+ writeMetrics()
+ } else if failures > maxFailureRetries+2 {
+ runtime.Goexit()
+ }
+ }
+
+ done := make(chan struct{})
+ var err error
+ go func() {
+ defer close(done)
+ err = gm.collectAmdStats()
+ }()
+ <-done
+ require.NoError(t, err, "collector must keep polling through prolonged sysfs read failures")
+ require.True(t, recovered, "collector must resume when sysfs metrics are restored")
+ assert.Equal(t, maxFailureRetries+2, failures)
+ assert.Equal(t, 50.0, gm.GpuDataMap["card0"].Usage)
+ assert.Equal(t, utils.BytesToMegabytes(1073741824), gm.GpuDataMap["card0"].MemoryUsed)
+}
+
func TestNormalizeHexID(t *testing.T) {
tests := []struct {
in string