Lightweight server monitoring with historical data, docker stats, and alerts.
fix(agent): SKIP_GPU excludes GPU hwmon from temperatures and fans (#2313)
Miłosz Kolber6141b15f0359
diff --git a/agent/agent.go b/agent/agent.goindex 454c565e5..998a32a7e 100644--- a/agent/agent.go+++ b/agent/agent.go@@ -252,7 +252,11 @@ func (a *Agent) gatherStats(options common.DataRequestOptions) *system.CombinedD// Start initializes and starts the agent with optional WebSocket connectionfunc (a *Agent) Start(serverOptions ServerOptions) error {a.keys = serverOptions.Keys- return a.connectionManager.Start(serverOptions)+ err := a.connectionManager.Start(serverOptions)+ if err != nil {+ a.cleanupSensorShadow()+ }+ return err}func (a *Agent) getFingerprint() string {diff --git a/agent/connection_manager.go b/agent/connection_manager.goindex 0f29bd24a..8152817ee 100644--- a/agent/connection_manager.go+++ b/agent/connection_manager.go@@ -155,6 +155,7 @@ func (c *ConnectionManager) stop() error {_ = c.agent.StopServer()c.agent.monitorManager.Stop()c.closeWebSocket()+ c.agent.cleanupSensorShadow()return health.CleanUp()}diff --git a/agent/fans.go b/agent/fans.goindex 2562d8d22..7b312a583 100644--- a/agent/fans.go+++ b/agent/fans.go@@ -12,7 +12,7 @@ import ()type fanSensor struct {- key, path string+ key, path, chip string}var getFanSensors = newFanSensorCache(hwmonRoot)@@ -34,6 +34,10 @@ func (a *Agent) updateFans(systemStats *system.Stats) {slog.Debug("Error reading fans", "err", err)return}+ // Filter before reading fan*_input: each read can wake an idle GPU.+ if a.sensorConfig != nil && a.sensorConfig.skipGPU {+ sensors = filterGpuFans(sensors)+ }fans := readFanSensors(sensors)if len(fans) == 0 {return@@ -100,7 +104,7 @@ func discoverHwmonFans(root string) ([]fanSensor, error) {if label != "" {key = chipName + "_" + label}- sensors = append(sensors, fanSensor{key, inputPath})+ sensors = append(sensors, fanSensor{key, inputPath, chipName})}}return sensors, nil@@ -115,3 +119,15 @@ func readFanSensors(sensors []fanSensor) map[string]uint16 {}return fans}++// filterGpuFans drops GPU chips without touching the shared cache backing array.+func filterGpuFans(sensors []fanSensor) []fanSensor {+ kept := make([]fanSensor, 0, len(sensors))+ for _, sensor := range sensors {+ if isGpuChipName(sensor.chip) {+ continue+ }+ kept = append(kept, sensor)+ }+ return kept+}diff --git a/agent/fans_test.go b/agent/fans_test.goindex 3bdb480d6..113dc0095 100644--- a/agent/fans_test.go+++ b/agent/fans_test.go@@ -103,3 +103,20 @@ func TestFanDiscoveryCache(t *testing.T) {fans = readFanSensors(sensors)assert.Equal(t, map[string]uint16{"chip_fan1": 1200}, fans)}++func TestFilterGpuFans(t *testing.T) {+ root := t.TempDir()+ writeFile(t, filepath.Join(root, "hwmon0", "name"), "xe\n")+ writeFile(t, filepath.Join(root, "hwmon0", "fan1_input"), "1200\n")+ writeFile(t, filepath.Join(root, "hwmon1", "name"), "nct6798\n")+ writeFile(t, filepath.Join(root, "hwmon1", "fan1_input"), "800\n")++ discovered, err := discoverHwmonFans(root)+ require.NoError(t, err)+ require.Len(t, discovered, 2)++ filtered := filterGpuFans(discovered)+ require.Len(t, filtered, 1)+ assert.Equal(t, "nct6798_fan1", filtered[0].key)+ assert.Len(t, discovered, 2)+}diff --git a/agent/gpu.go b/agent/gpu.goindex d1b7b4221..270cf4e93 100644--- a/agent/gpu.go+++ b/agent/gpu.go@@ -454,8 +454,8 @@ func (gm *GPUManager) storeSnapshot(id string, gpu *system.GPUData, cacheKey uin// It only reports capability presence and does not apply policy decisions.func (gm *GPUManager) discoverGpuCapabilities() gpuCapabilities {caps := gpuCapabilities{- hasAmdSysfs: gm.hasAmdSysfs(),- hasXe: gm.hasXe(),+ hasAmdSysfs: gm.hasAmdSysfs(),+ hasXe: gm.hasXe(),hasIntelSysfs: gm.hasIntelSysfs(),}if _, err := exec.LookPath(nvidiaSmiCmd); err == nil {@@ -750,9 +750,36 @@ func (gm *GPUManager) resolveLegacyCollectorPriority(caps gpuCapabilities) []colreturn priorities}+// gpuHwmonChips are hwmon chip names belonging to GPUs. Sensor reads on some+// of these drivers (notably Intel Xe, where each read is a runtime PM resume)+// wake the card, so SKIP_GPU must avoid touching them, not just hide them.+var gpuHwmonChips = []string{"xe", "i915", "amdgpu", "radeon", "nvidia", "nouveau"}++func isGpuChipName(name string) bool {+ name = strings.ToLower(strings.TrimSpace(name))+ for _, chip := range gpuHwmonChips {+ if name == chip {+ return true+ }+ }+ return false+}++// SensorKeys are "<chip>" or "<chip>_<label>".+func isGpuSensorKey(key string) bool {+ key = strings.ToLower(strings.TrimSpace(key))+ for _, chip := range gpuHwmonChips {+ if key == chip || strings.HasPrefix(key, chip+"_") {+ return true+ }+ }+ return false+}+// NewGPUManager creates and initializes a new GPUManagerfunc NewGPUManager() (*GPUManager, error) {if skipGPU, _ := utils.GetEnv("SKIP_GPU"); skipGPU == "true" {+ slog.Info("SKIP_GPU enabled, skipping GPU monitoring (collectors, temperatures, and fans)")return nil, nil}var gm GPUManagerdiff --git a/agent/sensors.go b/agent/sensors.goindex 8a6a6728a..23428ea63 100644--- a/agent/sensors.go+++ b/agent/sensors.go@@ -5,7 +5,9 @@ import ("errors""fmt""log/slog"+ "os""path"+ "path/filepath""runtime""strconv""strings"@@ -32,6 +34,8 @@ type SensorConfig struct {isBlacklist boolhasWildcards boolskipCollection bool+ skipGPU bool+ sensorShadow stringfirstRun bool}@@ -41,13 +45,14 @@ func (a *Agent) newSensorConfig() *SensorConfig {sensorsEnvVal, sensorsSet := utils.GetEnv("SENSORS")skipCollection := sensorsSet && sensorsEnvVal == ""sensorsTimeout, _ := utils.GetEnv("SENSORS_TIMEOUT")+ skipGPU, _ := utils.GetEnv("SKIP_GPU")- return a.newSensorConfigWithEnv(primarySensor, sysSensors, sensorsEnvVal, sensorsTimeout, skipCollection)+ return a.newSensorConfigWithEnv(primarySensor, sysSensors, sensorsEnvVal, sensorsTimeout, skipCollection, skipGPU == "true")}// newSensorConfigWithEnv creates a SensorConfig with the provided environment variables// sensorsSet indicates if the SENSORS environment variable was explicitly set (even to empty string)-func (a *Agent) newSensorConfigWithEnv(primarySensor, sysSensors, sensorsEnvVal, sensorsTimeout string, skipCollection bool) *SensorConfig {+func (a *Agent) newSensorConfigWithEnv(primarySensor, sysSensors, sensorsEnvVal, sensorsTimeout string, skipCollection, skipGPU bool) *SensorConfig {timeout := 2 * time.Secondif sensorsTimeout != "" {if d, err := time.ParseDuration(sensorsTimeout); err == nil {@@ -62,6 +67,7 @@ func (a *Agent) newSensorConfigWithEnv(primarySensor, sysSensors, sensorsEnvVal,primarySensor: primarySensor,timeout: timeout,skipCollection: skipCollection,+ skipGPU: skipGPU,firstRun: true,sensors: make(map[string]struct{}),}@@ -73,6 +79,19 @@ func (a *Agent) newSensorConfigWithEnv(primarySensor, sysSensors, sensorsEnvVal,common.EnvKey, common.EnvMap{common.HostSysEnvKey: sysSensors},)}+ if skipGPU && runtime.GOOS == "linux" {+ // gopsutil reads every temp*_input before results can be filtered, so+ // point it at a shadow tree built from the effective sysfs root instead.+ if shadow, err := buildNonGpuSysShadow(effectiveSysRoot(config.context)); err == nil {+ slog.Info("SKIP_GPU enabled, using non-GPU sensor sysfs shadow", "path", shadow)+ config.sensorShadow = shadow+ config.context = context.WithValue(config.context,+ common.EnvKey, common.EnvMap{common.HostSysEnvKey: shadow},+ )+ } else {+ slog.Warn("SKIP_GPU sensor shadow unavailable, falling back to post-read filtering", "err", err)+ }+ }// handle blacklistif strings.HasPrefix(sensorsEnvVal, "-") {@@ -149,6 +168,9 @@ func (a *Agent) updateTemperatures(systemStats *system.Stats) {if !isValidSensor(sensorName, a.sensorConfig) {continue}+ if a.sensorConfig.skipGPU && isGpuSensorKey(sensorName) {+ continue+ }// set dashboard temperatureswitch a.sensorConfig.primarySensor {case "":@@ -245,3 +267,102 @@ func scaleTemperature(temp float64) float64 {}return scaled100}++// effectiveSysRoot mirrors gopsutil's HostSys lookup, which lives in its+// internal package: context override, then HOST_SYS env, then /sys.+func effectiveSysRoot(ctx context.Context) string {+ if envMap, ok := ctx.Value(common.EnvKey).(common.EnvMap); ok {+ if v := envMap[common.HostSysEnvKey]; v != "" {+ return v+ }+ }+ if v := os.Getenv("HOST_SYS"); v != "" {+ return v+ }+ return "/sys"+}++func (config *SensorConfig) cleanupSensorShadow() {+ if config.sensorShadow == "" {+ return+ }+ if err := os.RemoveAll(config.sensorShadow); err != nil {+ slog.Warn("Error removing sensor sysfs shadow", "path", config.sensorShadow, "err", err)+ return+ }+ config.sensorShadow = ""+}++func (a *Agent) cleanupSensorShadow() {+ if a.sensorConfig != nil {+ a.sensorConfig.cleanupSensorShadow()+ }+}++func isGpuThermalZone(zoneType string) bool {+ zoneType = strings.ToLower(strings.TrimSpace(zoneType))+ return isGpuChipName(zoneType) || strings.Contains(zoneType, "gpu")+}++// buildNonGpuSysShadow links non-GPU sensor directories into a temp dir. Only+// static chip names and thermal-zone types are read; no sensor values are touched.+func buildNonGpuSysShadow(sysRoot string) (string, error) {+ shadow, err := os.MkdirTemp("", "beszel-sensors-*")+ if err != nil {+ return "", err+ }+ shadowHwmon := filepath.Join(shadow, "class", "hwmon")+ if err := os.MkdirAll(shadowHwmon, 0o755); err != nil {+ os.RemoveAll(shadow)+ return "", err+ }+ entries, err := os.ReadDir(filepath.Join(sysRoot, "class", "hwmon"))+ if err != nil && !os.IsNotExist(err) {+ os.RemoveAll(shadow)+ return "", err+ }+ for _, entry := range entries {+ chipDir := filepath.Join(sysRoot, "class", "hwmon", entry.Name())+ // Some hwmon devices expose name under device/ (gopsutil's CentOS fallback).+ name, ok := utils.ReadStringFileOK(filepath.Join(chipDir, "name"))+ if !ok {+ name, ok = utils.ReadStringFileOK(filepath.Join(chipDir, "device", "name"))+ }+ if !ok || isGpuChipName(name) {+ continue+ }+ if err := os.Symlink(chipDir, filepath.Join(shadowHwmon, entry.Name())); err != nil {+ os.RemoveAll(shadow)+ return "", err+ }+ }++ thermalEntries, err := os.ReadDir(filepath.Join(sysRoot, "class", "thermal"))+ if err != nil {+ if os.IsNotExist(err) {+ return shadow, nil+ }+ os.RemoveAll(shadow)+ return "", err+ }+ shadowThermal := filepath.Join(shadow, "class", "thermal")+ if err := os.MkdirAll(shadowThermal, 0o755); err != nil {+ os.RemoveAll(shadow)+ return "", err+ }+ for _, entry := range thermalEntries {+ if !strings.HasPrefix(entry.Name(), "thermal_zone") {+ continue+ }+ zoneDir := filepath.Join(sysRoot, "class", "thermal", entry.Name())+ zoneType, ok := utils.ReadStringFileOK(filepath.Join(zoneDir, "type"))+ if !ok || isGpuThermalZone(zoneType) {+ continue+ }+ if err := os.Symlink(zoneDir, filepath.Join(shadowThermal, entry.Name())); err != nil {+ os.RemoveAll(shadow)+ return "", err+ }+ }+ return shadow, nil+}diff --git a/agent/sensors_test.go b/agent/sensors_test.goindex f46c5247d..0523bc6f9 100644--- a/agent/sensors_test.go+++ b/agent/sensors_test.go@@ -5,6 +5,8 @@ package agentimport ("context""fmt"+ "os"+ "path/filepath""testing""time"@@ -328,7 +330,7 @@ func TestNewSensorConfigWithEnv(t *testing.T) {for _, tt := range tests {t.Run(tt.name, func(t *testing.T) {- result := agent.newSensorConfigWithEnv(tt.primarySensor, tt.sysSensors, tt.sensors, tt.sensorsTimeout, tt.skipCollection)+ result := agent.newSensorConfigWithEnv(tt.primarySensor, tt.sysSensors, tt.sensors, tt.sensorsTimeout, tt.skipCollection, false)// Check primary sensorassert.Equal(t, tt.expectedConfig.primarySensor, result.primarySensor)@@ -620,3 +622,143 @@ func TestUpdateTemperaturesSkipsOnTimeout(t *testing.T) {assert.Equal(t, 0.0, agent.systemInfo.DashboardTemp)assert.Equal(t, map[string]float64{}, stats.Temperatures)}++func TestIsGpuSensorKey(t *testing.T) {+ for _, key := range []string{"xe", "XE_temp1", "amdgpu_edge", "NVIDIA"} {+ assert.True(t, isGpuSensorKey(key), key)+ }+ for _, key := range []string{"coretemp_core_0", "acpitz", "xen_temp", "myxe", ""} {+ assert.False(t, isGpuSensorKey(key), key)+ }+}++func TestSkipGpuSensorShadow(t *testing.T) {+ sysRoot := t.TempDir()+ writeFile(t, filepath.Join(sysRoot, "class", "hwmon", "hwmon0", "name"), "coretemp\n")+ writeFile(t, filepath.Join(sysRoot, "class", "hwmon", "hwmon0", "temp1_input"), "55000\n")+ writeFile(t, filepath.Join(sysRoot, "class", "hwmon", "hwmon1", "name"), "xe\n")+ writeFile(t, filepath.Join(sysRoot, "class", "hwmon", "hwmon1", "temp1_input"), "48000\n")+ writeFile(t, filepath.Join(sysRoot, "class", "thermal", "thermal_zone0", "type"), "cpu-thermal\n")+ writeFile(t, filepath.Join(sysRoot, "class", "thermal", "thermal_zone0", "temp"), "55000\n")+ writeFile(t, filepath.Join(sysRoot, "class", "thermal", "thermal_zone1", "type"), "gpu\n")+ writeFile(t, filepath.Join(sysRoot, "class", "thermal", "thermal_zone1", "temp"), "48000\n")++ shadow, err := buildNonGpuSysShadow(sysRoot)+ require.NoError(t, err)+ t.Cleanup(func() { os.RemoveAll(shadow) })++ assert.FileExists(t, filepath.Join(shadow, "class", "hwmon", "hwmon0", "temp1_input"))+ assert.NoFileExists(t, filepath.Join(shadow, "class", "hwmon", "hwmon1"))+ assert.FileExists(t, filepath.Join(shadow, "class", "thermal", "thermal_zone0", "temp"))+ assert.NoFileExists(t, filepath.Join(shadow, "class", "thermal", "thermal_zone1"))+}++func TestSkipGpuSensorShadowDeviceName(t *testing.T) {+ sysRoot := t.TempDir()+ writeFile(t, filepath.Join(sysRoot, "class", "hwmon", "hwmon0", "device", "name"), "coretemp\n")+ writeFile(t, filepath.Join(sysRoot, "class", "hwmon", "hwmon0", "device", "temp1_input"), "55000\n")+ writeFile(t, filepath.Join(sysRoot, "class", "hwmon", "hwmon1", "device", "name"), "xe\n")+ writeFile(t, filepath.Join(sysRoot, "class", "hwmon", "hwmon1", "device", "temp1_input"), "48000\n")++ shadow, err := buildNonGpuSysShadow(sysRoot)+ require.NoError(t, err)+ t.Cleanup(func() { os.RemoveAll(shadow) })++ assert.FileExists(t, filepath.Join(shadow, "class", "hwmon", "hwmon0", "device", "temp1_input"))+ assert.NoFileExists(t, filepath.Join(shadow, "class", "hwmon", "hwmon1"))+}++func TestSkipGpuSensorShadowKeepsThermalZonesWithoutNonGpuHwmon(t *testing.T) {+ sysRoot := t.TempDir()+ writeFile(t, filepath.Join(sysRoot, "class", "hwmon", "hwmon0", "name"), "xe\n")+ writeFile(t, filepath.Join(sysRoot, "class", "hwmon", "hwmon0", "temp1_input"), "48000\n")+ writeFile(t, filepath.Join(sysRoot, "class", "thermal", "thermal_zone0", "type"), "cpu-thermal\n")+ writeFile(t, filepath.Join(sysRoot, "class", "thermal", "thermal_zone0", "temp"), "55000\n")+ writeFile(t, filepath.Join(sysRoot, "class", "thermal", "thermal_zone1", "type"), "gpu\n")+ writeFile(t, filepath.Join(sysRoot, "class", "thermal", "thermal_zone1", "temp"), "48000\n")++ shadow, err := buildNonGpuSysShadow(sysRoot)+ require.NoError(t, err)+ t.Cleanup(func() { os.RemoveAll(shadow) })++ hwmonTemps, err := filepath.Glob(filepath.Join(shadow, "class", "hwmon", "hwmon*", "temp*_input"))+ require.NoError(t, err)+ assert.Empty(t, hwmonTemps)+ assert.FileExists(t, filepath.Join(shadow, "class", "thermal", "thermal_zone0", "temp"))+ assert.NoFileExists(t, filepath.Join(shadow, "class", "thermal", "thermal_zone1"))+}++func TestNewSensorConfigSkipGpuWiresShadow(t *testing.T) {+ t.Setenv("SKIP_GPU", "true")++ agent := &Agent{}+ config := agent.newSensorConfig()++ assert.True(t, config.skipGPU)+ envMap, ok := config.context.Value(common.EnvKey).(common.EnvMap)+ require.True(t, ok, "SKIP_GPU should point the sensor context at a sysfs shadow")+ shadow, ok := envMap[common.HostSysEnvKey]+ require.True(t, ok)+ assert.DirExists(t, filepath.Join(shadow, "class", "hwmon"))+ assert.Equal(t, shadow, config.sensorShadow)+ config.cleanupSensorShadow()+ assert.NoDirExists(t, shadow)+ assert.Empty(t, config.sensorShadow)+}++func TestSkipGpuShadowUsesSysSensorsRoot(t *testing.T) {+ sysRoot := t.TempDir()+ writeFile(t, filepath.Join(sysRoot, "class", "hwmon", "hwmon0", "name"), "coretemp\n")+ writeFile(t, filepath.Join(sysRoot, "class", "hwmon", "hwmon0", "temp1_input"), "55000\n")+ writeFile(t, filepath.Join(sysRoot, "class", "hwmon", "hwmon1", "name"), "xe\n")+ writeFile(t, filepath.Join(sysRoot, "class", "hwmon", "hwmon1", "temp1_input"), "48000\n")++ agent := &Agent{}+ config := agent.newSensorConfigWithEnv("", sysRoot, "", "", false, true)+ t.Cleanup(config.cleanupSensorShadow)++ envMap, ok := config.context.Value(common.EnvKey).(common.EnvMap)+ require.True(t, ok, "SKIP_GPU should point the sensor context at a sysfs shadow")+ shadow, ok := envMap[common.HostSysEnvKey]+ require.True(t, ok)+ require.NotEqual(t, sysRoot, shadow, "shadow must not be the SYS_SENSORS tree itself")++ target, err := os.Readlink(filepath.Join(shadow, "class", "hwmon", "hwmon0"))+ require.NoError(t, err)+ assert.Equal(t, filepath.Join(sysRoot, "class", "hwmon", "hwmon0"), target)+ assert.NoFileExists(t, filepath.Join(shadow, "class", "hwmon", "hwmon1"))+}++func TestUpdateTemperaturesSkipGpu(t *testing.T) {+ originalGetSensorTemps := getSensorTemps+ t.Cleanup(func() {+ getSensorTemps = originalGetSensorTemps+ })+ getSensorTemps = func(ctx context.Context) ([]sensors.TemperatureStat, error) {+ return []sensors.TemperatureStat{+ {SensorKey: "coretemp_core_0", Temperature: 55},+ {SensorKey: "XE", Temperature: 48},+ }, nil+ }++ newAgent := func(skipGPU bool) *Agent {+ agent := &Agent{+ systemInfo: system.Info{},+ sensorConfig: &SensorConfig{+ context: context.Background(),+ timeout: 2 * time.Second,+ sensors: map[string]struct{}{},+ skipGPU: skipGPU,+ },+ }+ return agent+ }++ stats := &system.Stats{}+ newAgent(true).updateTemperatures(stats)+ assert.Equal(t, map[string]float64{"coretemp_core_0": 55}, stats.Temperatures)++ stats = &system.Stats{}+ newAgent(false).updateTemperatures(stats)+ assert.Len(t, stats.Temperatures, 2)+}