Lightweight server monitoring with historical data, docker stats, and alerts.
fix(agent): keep amd_sysfs collector polling after repeated read failures (#2687)
cobalt-kite12784fb2fdbb6b
diff --git a/agent/gpu_amd_linux.go b/agent/gpu_amd_linux.goindex a6b249737..54a1c4093 100644--- a/agent/gpu_amd_linux.go+++ b/agent/gpu_amd_linux.go@@ -17,6 +17,8 @@ import ("github.com/henrygd/beszel/internal/entities/system")+var amdSysfsSleep = time.Sleep+var amdgpuNameCache = struct {sync.RWMutexhits map[string]string@@ -28,7 +30,7 @@ var amdgpuNameCache = struct {// hasAmdSysfs returns true if any AMD GPU sysfs nodes are foundfunc (gm *GPUManager) hasAmdSysfs() bool {- cards, err := filepath.Glob("/sys/class/drm/card*/device/vendor")+ cards, err := filepath.Glob(filepath.Join(drmSysfsRoot, "card*/device/vendor"))if err != nil {return false}@@ -44,7 +46,7 @@ func (gm *GPUManager) hasAmdSysfs() bool {// collectAmdStats collects AMD GPU metrics directly from sysfs to avoid the overhead of rocm-smifunc (gm *GPUManager) collectAmdStats() error {sysfsPollInterval := 3000 * time.Millisecond- cards, err := filepath.Glob("/sys/class/drm/card*")+ cards, err := filepath.Glob(filepath.Join(drmSysfsRoot, "card*"))if err != nil {return err}@@ -74,15 +76,18 @@ func (gm *GPUManager) collectAmdStats() error {}if !hasData {failures++- if failures > maxFailureRetries {- return errNoValidData+ // Keep polling so transient sysfs failures can recover without an agent restart,+ // but only warn for the first retries to avoid flooding the log.+ log := slog.Debug+ if failures <= maxFailureRetries {+ log = slog.Warn}- slog.Warn("No AMD GPU data from sysfs", "failures", failures)- time.Sleep(retryWaitTime)+ log("No AMD GPU data from sysfs", "failures", failures)+ amdSysfsSleep(retryWaitTime)continue}failures = 0- time.Sleep(sysfsPollInterval)+ amdSysfsSleep(sysfsPollInterval)}}diff --git a/agent/gpu_amd_linux_test.go b/agent/gpu_amd_linux_test.goindex 76b286cc2..bbcaadbcf 100644--- a/agent/gpu_amd_linux_test.go+++ b/agent/gpu_amd_linux_test.go@@ -5,7 +5,9 @@ package agentimport ("os""path/filepath"+ "runtime""testing"+ "time""github.com/henrygd/beszel/agent/utils""github.com/henrygd/beszel/internal/entities/system"@@ -13,6 +15,72 @@ import ("github.com/stretchr/testify/require")+func TestCollectAmdStatsRecoversAfterReadFailures(t *testing.T) {+ root := t.TempDir()+ oldRoot, oldSleep := drmSysfsRoot, amdSysfsSleep+ drmSysfsRoot = root+ t.Cleanup(func() {+ drmSysfsRoot, amdSysfsSleep = oldRoot, oldSleep+ })++ devicePath := filepath.Join(root, "card0", "device")+ require.NoError(t, os.MkdirAll(devicePath, 0o755))+ require.NoError(t, os.WriteFile(filepath.Join(devicePath, "vendor"), []byte("0x1002"), 0o644))+ metrics := map[string]string{+ "gpu_busy_percent": "25",+ "mem_info_vram_used": "1073741824",+ "mem_info_vram_total": "2147483648",+ }+ writeMetrics := func() {+ for name, value := range metrics {+ require.NoError(t, os.WriteFile(filepath.Join(devicePath, name), []byte(value), 0o644))+ }+ }+ writeMetrics()++ gm := &GPUManager{GpuDataMap: map[string]*system.GPUData{"card0": {Name: "AMD Test GPU"}}}+ failures := 0+ removed := false+ recovered := false+ amdSysfsSleep = func(delay time.Duration) {+ if !removed {+ assert.Equal(t, 1.0, gm.GpuDataMap["card0"].Count)+ assert.Equal(t, 3*time.Second, delay)+ for name := range metrics {+ require.NoError(t, os.Remove(filepath.Join(devicePath, name)))+ }+ removed = true+ return+ }+ if gm.GpuDataMap["card0"].Count == 2 {+ assert.Equal(t, 3*time.Second, delay)+ recovered = true+ // Stop the collector goroutine after observing the recovered sample.+ runtime.Goexit()+ }+ assert.Equal(t, retryWaitTime, delay)+ failures+++ if failures == maxFailureRetries+2 {+ writeMetrics()+ } else if failures > maxFailureRetries+2 {+ runtime.Goexit()+ }+ }++ done := make(chan struct{})+ var err error+ go func() {+ defer close(done)+ err = gm.collectAmdStats()+ }()+ <-done+ require.NoError(t, err, "collector must keep polling through prolonged sysfs read failures")+ require.True(t, recovered, "collector must resume when sysfs metrics are restored")+ assert.Equal(t, maxFailureRetries+2, failures)+ assert.Equal(t, 50.0, gm.GpuDataMap["card0"].Usage)+ assert.Equal(t, utils.BytesToMegabytes(1073741824), gm.GpuDataMap["card0"].MemoryUsed)+}+func TestNormalizeHexID(t *testing.T) {tests := []struct {in string