Skip to content

Lightweight server monitoring with historical data, docker stats, and alerts.

fix(agent): SKIP_GPU excludes GPU hwmon from temperatures and fans (#2313)

Miłosz Kolber6141b15f0359
diff --git a/agent/agent.go b/agent/agent.go
index 454c565e5..998a32a7e 100644
--- a/agent/agent.go
+++ b/agent/agent.go
@@ -252,7 +252,11 @@ func (a *Agent) gatherStats(options common.DataRequestOptions) *system.CombinedD
// Start initializes and starts the agent with optional WebSocket connection
func (a *Agent) Start(serverOptions ServerOptions) error {
a.keys = serverOptions.Keys
- return a.connectionManager.Start(serverOptions)
+ err := a.connectionManager.Start(serverOptions)
+ if err != nil {
+ a.cleanupSensorShadow()
+ }
+ return err
}
func (a *Agent) getFingerprint() string {
diff --git a/agent/connection_manager.go b/agent/connection_manager.go
index 0f29bd24a..8152817ee 100644
--- a/agent/connection_manager.go
+++ b/agent/connection_manager.go
@@ -155,6 +155,7 @@ func (c *ConnectionManager) stop() error {
_ = c.agent.StopServer()
c.agent.monitorManager.Stop()
c.closeWebSocket()
+ c.agent.cleanupSensorShadow()
return health.CleanUp()
}
diff --git a/agent/fans.go b/agent/fans.go
index 2562d8d22..7b312a583 100644
--- a/agent/fans.go
+++ b/agent/fans.go
@@ -12,7 +12,7 @@ import (
)
type fanSensor struct {
- key, path string
+ key, path, chip string
}
var getFanSensors = newFanSensorCache(hwmonRoot)
@@ -34,6 +34,10 @@ func (a *Agent) updateFans(systemStats *system.Stats) {
slog.Debug("Error reading fans", "err", err)
return
}
+ // Filter before reading fan*_input: each read can wake an idle GPU.
+ if a.sensorConfig != nil && a.sensorConfig.skipGPU {
+ sensors = filterGpuFans(sensors)
+ }
fans := readFanSensors(sensors)
if len(fans) == 0 {
return
@@ -100,7 +104,7 @@ func discoverHwmonFans(root string) ([]fanSensor, error) {
if label != "" {
key = chipName + "_" + label
}
- sensors = append(sensors, fanSensor{key, inputPath})
+ sensors = append(sensors, fanSensor{key, inputPath, chipName})
}
}
return sensors, nil
@@ -115,3 +119,15 @@ func readFanSensors(sensors []fanSensor) map[string]uint16 {
}
return fans
}
+
+// filterGpuFans drops GPU chips without touching the shared cache backing array.
+func filterGpuFans(sensors []fanSensor) []fanSensor {
+ kept := make([]fanSensor, 0, len(sensors))
+ for _, sensor := range sensors {
+ if isGpuChipName(sensor.chip) {
+ continue
+ }
+ kept = append(kept, sensor)
+ }
+ return kept
+}
diff --git a/agent/fans_test.go b/agent/fans_test.go
index 3bdb480d6..113dc0095 100644
--- a/agent/fans_test.go
+++ b/agent/fans_test.go
@@ -103,3 +103,20 @@ func TestFanDiscoveryCache(t *testing.T) {
fans = readFanSensors(sensors)
assert.Equal(t, map[string]uint16{"chip_fan1": 1200}, fans)
}
+
+func TestFilterGpuFans(t *testing.T) {
+ root := t.TempDir()
+ writeFile(t, filepath.Join(root, "hwmon0", "name"), "xe\n")
+ writeFile(t, filepath.Join(root, "hwmon0", "fan1_input"), "1200\n")
+ writeFile(t, filepath.Join(root, "hwmon1", "name"), "nct6798\n")
+ writeFile(t, filepath.Join(root, "hwmon1", "fan1_input"), "800\n")
+
+ discovered, err := discoverHwmonFans(root)
+ require.NoError(t, err)
+ require.Len(t, discovered, 2)
+
+ filtered := filterGpuFans(discovered)
+ require.Len(t, filtered, 1)
+ assert.Equal(t, "nct6798_fan1", filtered[0].key)
+ assert.Len(t, discovered, 2)
+}
diff --git a/agent/gpu.go b/agent/gpu.go
index d1b7b4221..270cf4e93 100644
--- a/agent/gpu.go
+++ b/agent/gpu.go
@@ -454,8 +454,8 @@ func (gm *GPUManager) storeSnapshot(id string, gpu *system.GPUData, cacheKey uin
// It only reports capability presence and does not apply policy decisions.
func (gm *GPUManager) discoverGpuCapabilities() gpuCapabilities {
caps := gpuCapabilities{
- hasAmdSysfs: gm.hasAmdSysfs(),
- hasXe: gm.hasXe(),
+ hasAmdSysfs: gm.hasAmdSysfs(),
+ hasXe: gm.hasXe(),
hasIntelSysfs: gm.hasIntelSysfs(),
}
if _, err := exec.LookPath(nvidiaSmiCmd); err == nil {
@@ -750,9 +750,36 @@ func (gm *GPUManager) resolveLegacyCollectorPriority(caps gpuCapabilities) []col
return priorities
}
+// gpuHwmonChips are hwmon chip names belonging to GPUs. Sensor reads on some
+// of these drivers (notably Intel Xe, where each read is a runtime PM resume)
+// wake the card, so SKIP_GPU must avoid touching them, not just hide them.
+var gpuHwmonChips = []string{"xe", "i915", "amdgpu", "radeon", "nvidia", "nouveau"}
+
+func isGpuChipName(name string) bool {
+ name = strings.ToLower(strings.TrimSpace(name))
+ for _, chip := range gpuHwmonChips {
+ if name == chip {
+ return true
+ }
+ }
+ return false
+}
+
+// SensorKeys are "<chip>" or "<chip>_<label>".
+func isGpuSensorKey(key string) bool {
+ key = strings.ToLower(strings.TrimSpace(key))
+ for _, chip := range gpuHwmonChips {
+ if key == chip || strings.HasPrefix(key, chip+"_") {
+ return true
+ }
+ }
+ return false
+}
+
// NewGPUManager creates and initializes a new GPUManager
func NewGPUManager() (*GPUManager, error) {
if skipGPU, _ := utils.GetEnv("SKIP_GPU"); skipGPU == "true" {
+ slog.Info("SKIP_GPU enabled, skipping GPU monitoring (collectors, temperatures, and fans)")
return nil, nil
}
var gm GPUManager
diff --git a/agent/sensors.go b/agent/sensors.go
index 8a6a6728a..23428ea63 100644
--- a/agent/sensors.go
+++ b/agent/sensors.go
@@ -5,7 +5,9 @@ import (
"errors"
"fmt"
"log/slog"
+ "os"
"path"
+ "path/filepath"
"runtime"
"strconv"
"strings"
@@ -32,6 +34,8 @@ type SensorConfig struct {
isBlacklist bool
hasWildcards bool
skipCollection bool
+ skipGPU bool
+ sensorShadow string
firstRun bool
}
@@ -41,13 +45,14 @@ func (a *Agent) newSensorConfig() *SensorConfig {
sensorsEnvVal, sensorsSet := utils.GetEnv("SENSORS")
skipCollection := sensorsSet && sensorsEnvVal == ""
sensorsTimeout, _ := utils.GetEnv("SENSORS_TIMEOUT")
+ skipGPU, _ := utils.GetEnv("SKIP_GPU")
- return a.newSensorConfigWithEnv(primarySensor, sysSensors, sensorsEnvVal, sensorsTimeout, skipCollection)
+ return a.newSensorConfigWithEnv(primarySensor, sysSensors, sensorsEnvVal, sensorsTimeout, skipCollection, skipGPU == "true")
}
// newSensorConfigWithEnv creates a SensorConfig with the provided environment variables
// sensorsSet indicates if the SENSORS environment variable was explicitly set (even to empty string)
-func (a *Agent) newSensorConfigWithEnv(primarySensor, sysSensors, sensorsEnvVal, sensorsTimeout string, skipCollection bool) *SensorConfig {
+func (a *Agent) newSensorConfigWithEnv(primarySensor, sysSensors, sensorsEnvVal, sensorsTimeout string, skipCollection, skipGPU bool) *SensorConfig {
timeout := 2 * time.Second
if sensorsTimeout != "" {
if d, err := time.ParseDuration(sensorsTimeout); err == nil {
@@ -62,6 +67,7 @@ func (a *Agent) newSensorConfigWithEnv(primarySensor, sysSensors, sensorsEnvVal,
primarySensor: primarySensor,
timeout: timeout,
skipCollection: skipCollection,
+ skipGPU: skipGPU,
firstRun: true,
sensors: make(map[string]struct{}),
}
@@ -73,6 +79,19 @@ func (a *Agent) newSensorConfigWithEnv(primarySensor, sysSensors, sensorsEnvVal,
common.EnvKey, common.EnvMap{common.HostSysEnvKey: sysSensors},
)
}
+ if skipGPU && runtime.GOOS == "linux" {
+ // gopsutil reads every temp*_input before results can be filtered, so
+ // point it at a shadow tree built from the effective sysfs root instead.
+ if shadow, err := buildNonGpuSysShadow(effectiveSysRoot(config.context)); err == nil {
+ slog.Info("SKIP_GPU enabled, using non-GPU sensor sysfs shadow", "path", shadow)
+ config.sensorShadow = shadow
+ config.context = context.WithValue(config.context,
+ common.EnvKey, common.EnvMap{common.HostSysEnvKey: shadow},
+ )
+ } else {
+ slog.Warn("SKIP_GPU sensor shadow unavailable, falling back to post-read filtering", "err", err)
+ }
+ }
// handle blacklist
if strings.HasPrefix(sensorsEnvVal, "-") {
@@ -149,6 +168,9 @@ func (a *Agent) updateTemperatures(systemStats *system.Stats) {
if !isValidSensor(sensorName, a.sensorConfig) {
continue
}
+ if a.sensorConfig.skipGPU && isGpuSensorKey(sensorName) {
+ continue
+ }
// set dashboard temperature
switch a.sensorConfig.primarySensor {
case "":
@@ -245,3 +267,102 @@ func scaleTemperature(temp float64) float64 {
}
return scaled100
}
+
+// effectiveSysRoot mirrors gopsutil's HostSys lookup, which lives in its
+// internal package: context override, then HOST_SYS env, then /sys.
+func effectiveSysRoot(ctx context.Context) string {
+ if envMap, ok := ctx.Value(common.EnvKey).(common.EnvMap); ok {
+ if v := envMap[common.HostSysEnvKey]; v != "" {
+ return v
+ }
+ }
+ if v := os.Getenv("HOST_SYS"); v != "" {
+ return v
+ }
+ return "/sys"
+}
+
+func (config *SensorConfig) cleanupSensorShadow() {
+ if config.sensorShadow == "" {
+ return
+ }
+ if err := os.RemoveAll(config.sensorShadow); err != nil {
+ slog.Warn("Error removing sensor sysfs shadow", "path", config.sensorShadow, "err", err)
+ return
+ }
+ config.sensorShadow = ""
+}
+
+func (a *Agent) cleanupSensorShadow() {
+ if a.sensorConfig != nil {
+ a.sensorConfig.cleanupSensorShadow()
+ }
+}
+
+func isGpuThermalZone(zoneType string) bool {
+ zoneType = strings.ToLower(strings.TrimSpace(zoneType))
+ return isGpuChipName(zoneType) || strings.Contains(zoneType, "gpu")
+}
+
+// buildNonGpuSysShadow links non-GPU sensor directories into a temp dir. Only
+// static chip names and thermal-zone types are read; no sensor values are touched.
+func buildNonGpuSysShadow(sysRoot string) (string, error) {
+ shadow, err := os.MkdirTemp("", "beszel-sensors-*")
+ if err != nil {
+ return "", err
+ }
+ shadowHwmon := filepath.Join(shadow, "class", "hwmon")
+ if err := os.MkdirAll(shadowHwmon, 0o755); err != nil {
+ os.RemoveAll(shadow)
+ return "", err
+ }
+ entries, err := os.ReadDir(filepath.Join(sysRoot, "class", "hwmon"))
+ if err != nil && !os.IsNotExist(err) {
+ os.RemoveAll(shadow)
+ return "", err
+ }
+ for _, entry := range entries {
+ chipDir := filepath.Join(sysRoot, "class", "hwmon", entry.Name())
+ // Some hwmon devices expose name under device/ (gopsutil's CentOS fallback).
+ name, ok := utils.ReadStringFileOK(filepath.Join(chipDir, "name"))
+ if !ok {
+ name, ok = utils.ReadStringFileOK(filepath.Join(chipDir, "device", "name"))
+ }
+ if !ok || isGpuChipName(name) {
+ continue
+ }
+ if err := os.Symlink(chipDir, filepath.Join(shadowHwmon, entry.Name())); err != nil {
+ os.RemoveAll(shadow)
+ return "", err
+ }
+ }
+
+ thermalEntries, err := os.ReadDir(filepath.Join(sysRoot, "class", "thermal"))
+ if err != nil {
+ if os.IsNotExist(err) {
+ return shadow, nil
+ }
+ os.RemoveAll(shadow)
+ return "", err
+ }
+ shadowThermal := filepath.Join(shadow, "class", "thermal")
+ if err := os.MkdirAll(shadowThermal, 0o755); err != nil {
+ os.RemoveAll(shadow)
+ return "", err
+ }
+ for _, entry := range thermalEntries {
+ if !strings.HasPrefix(entry.Name(), "thermal_zone") {
+ continue
+ }
+ zoneDir := filepath.Join(sysRoot, "class", "thermal", entry.Name())
+ zoneType, ok := utils.ReadStringFileOK(filepath.Join(zoneDir, "type"))
+ if !ok || isGpuThermalZone(zoneType) {
+ continue
+ }
+ if err := os.Symlink(zoneDir, filepath.Join(shadowThermal, entry.Name())); err != nil {
+ os.RemoveAll(shadow)
+ return "", err
+ }
+ }
+ return shadow, nil
+}
diff --git a/agent/sensors_test.go b/agent/sensors_test.go
index f46c5247d..0523bc6f9 100644
--- a/agent/sensors_test.go
+++ b/agent/sensors_test.go
@@ -5,6 +5,8 @@ package agent
import (
"context"
"fmt"
+ "os"
+ "path/filepath"
"testing"
"time"
@@ -328,7 +330,7 @@ func TestNewSensorConfigWithEnv(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
- result := agent.newSensorConfigWithEnv(tt.primarySensor, tt.sysSensors, tt.sensors, tt.sensorsTimeout, tt.skipCollection)
+ result := agent.newSensorConfigWithEnv(tt.primarySensor, tt.sysSensors, tt.sensors, tt.sensorsTimeout, tt.skipCollection, false)
// Check primary sensor
assert.Equal(t, tt.expectedConfig.primarySensor, result.primarySensor)
@@ -620,3 +622,143 @@ func TestUpdateTemperaturesSkipsOnTimeout(t *testing.T) {
assert.Equal(t, 0.0, agent.systemInfo.DashboardTemp)
assert.Equal(t, map[string]float64{}, stats.Temperatures)
}
+
+func TestIsGpuSensorKey(t *testing.T) {
+ for _, key := range []string{"xe", "XE_temp1", "amdgpu_edge", "NVIDIA"} {
+ assert.True(t, isGpuSensorKey(key), key)
+ }
+ for _, key := range []string{"coretemp_core_0", "acpitz", "xen_temp", "myxe", ""} {
+ assert.False(t, isGpuSensorKey(key), key)
+ }
+}
+
+func TestSkipGpuSensorShadow(t *testing.T) {
+ sysRoot := t.TempDir()
+ writeFile(t, filepath.Join(sysRoot, "class", "hwmon", "hwmon0", "name"), "coretemp\n")
+ writeFile(t, filepath.Join(sysRoot, "class", "hwmon", "hwmon0", "temp1_input"), "55000\n")
+ writeFile(t, filepath.Join(sysRoot, "class", "hwmon", "hwmon1", "name"), "xe\n")
+ writeFile(t, filepath.Join(sysRoot, "class", "hwmon", "hwmon1", "temp1_input"), "48000\n")
+ writeFile(t, filepath.Join(sysRoot, "class", "thermal", "thermal_zone0", "type"), "cpu-thermal\n")
+ writeFile(t, filepath.Join(sysRoot, "class", "thermal", "thermal_zone0", "temp"), "55000\n")
+ writeFile(t, filepath.Join(sysRoot, "class", "thermal", "thermal_zone1", "type"), "gpu\n")
+ writeFile(t, filepath.Join(sysRoot, "class", "thermal", "thermal_zone1", "temp"), "48000\n")
+
+ shadow, err := buildNonGpuSysShadow(sysRoot)
+ require.NoError(t, err)
+ t.Cleanup(func() { os.RemoveAll(shadow) })
+
+ assert.FileExists(t, filepath.Join(shadow, "class", "hwmon", "hwmon0", "temp1_input"))
+ assert.NoFileExists(t, filepath.Join(shadow, "class", "hwmon", "hwmon1"))
+ assert.FileExists(t, filepath.Join(shadow, "class", "thermal", "thermal_zone0", "temp"))
+ assert.NoFileExists(t, filepath.Join(shadow, "class", "thermal", "thermal_zone1"))
+}
+
+func TestSkipGpuSensorShadowDeviceName(t *testing.T) {
+ sysRoot := t.TempDir()
+ writeFile(t, filepath.Join(sysRoot, "class", "hwmon", "hwmon0", "device", "name"), "coretemp\n")
+ writeFile(t, filepath.Join(sysRoot, "class", "hwmon", "hwmon0", "device", "temp1_input"), "55000\n")
+ writeFile(t, filepath.Join(sysRoot, "class", "hwmon", "hwmon1", "device", "name"), "xe\n")
+ writeFile(t, filepath.Join(sysRoot, "class", "hwmon", "hwmon1", "device", "temp1_input"), "48000\n")
+
+ shadow, err := buildNonGpuSysShadow(sysRoot)
+ require.NoError(t, err)
+ t.Cleanup(func() { os.RemoveAll(shadow) })
+
+ assert.FileExists(t, filepath.Join(shadow, "class", "hwmon", "hwmon0", "device", "temp1_input"))
+ assert.NoFileExists(t, filepath.Join(shadow, "class", "hwmon", "hwmon1"))
+}
+
+func TestSkipGpuSensorShadowKeepsThermalZonesWithoutNonGpuHwmon(t *testing.T) {
+ sysRoot := t.TempDir()
+ writeFile(t, filepath.Join(sysRoot, "class", "hwmon", "hwmon0", "name"), "xe\n")
+ writeFile(t, filepath.Join(sysRoot, "class", "hwmon", "hwmon0", "temp1_input"), "48000\n")
+ writeFile(t, filepath.Join(sysRoot, "class", "thermal", "thermal_zone0", "type"), "cpu-thermal\n")
+ writeFile(t, filepath.Join(sysRoot, "class", "thermal", "thermal_zone0", "temp"), "55000\n")
+ writeFile(t, filepath.Join(sysRoot, "class", "thermal", "thermal_zone1", "type"), "gpu\n")
+ writeFile(t, filepath.Join(sysRoot, "class", "thermal", "thermal_zone1", "temp"), "48000\n")
+
+ shadow, err := buildNonGpuSysShadow(sysRoot)
+ require.NoError(t, err)
+ t.Cleanup(func() { os.RemoveAll(shadow) })
+
+ hwmonTemps, err := filepath.Glob(filepath.Join(shadow, "class", "hwmon", "hwmon*", "temp*_input"))
+ require.NoError(t, err)
+ assert.Empty(t, hwmonTemps)
+ assert.FileExists(t, filepath.Join(shadow, "class", "thermal", "thermal_zone0", "temp"))
+ assert.NoFileExists(t, filepath.Join(shadow, "class", "thermal", "thermal_zone1"))
+}
+
+func TestNewSensorConfigSkipGpuWiresShadow(t *testing.T) {
+ t.Setenv("SKIP_GPU", "true")
+
+ agent := &Agent{}
+ config := agent.newSensorConfig()
+
+ assert.True(t, config.skipGPU)
+ envMap, ok := config.context.Value(common.EnvKey).(common.EnvMap)
+ require.True(t, ok, "SKIP_GPU should point the sensor context at a sysfs shadow")
+ shadow, ok := envMap[common.HostSysEnvKey]
+ require.True(t, ok)
+ assert.DirExists(t, filepath.Join(shadow, "class", "hwmon"))
+ assert.Equal(t, shadow, config.sensorShadow)
+ config.cleanupSensorShadow()
+ assert.NoDirExists(t, shadow)
+ assert.Empty(t, config.sensorShadow)
+}
+
+func TestSkipGpuShadowUsesSysSensorsRoot(t *testing.T) {
+ sysRoot := t.TempDir()
+ writeFile(t, filepath.Join(sysRoot, "class", "hwmon", "hwmon0", "name"), "coretemp\n")
+ writeFile(t, filepath.Join(sysRoot, "class", "hwmon", "hwmon0", "temp1_input"), "55000\n")
+ writeFile(t, filepath.Join(sysRoot, "class", "hwmon", "hwmon1", "name"), "xe\n")
+ writeFile(t, filepath.Join(sysRoot, "class", "hwmon", "hwmon1", "temp1_input"), "48000\n")
+
+ agent := &Agent{}
+ config := agent.newSensorConfigWithEnv("", sysRoot, "", "", false, true)
+ t.Cleanup(config.cleanupSensorShadow)
+
+ envMap, ok := config.context.Value(common.EnvKey).(common.EnvMap)
+ require.True(t, ok, "SKIP_GPU should point the sensor context at a sysfs shadow")
+ shadow, ok := envMap[common.HostSysEnvKey]
+ require.True(t, ok)
+ require.NotEqual(t, sysRoot, shadow, "shadow must not be the SYS_SENSORS tree itself")
+
+ target, err := os.Readlink(filepath.Join(shadow, "class", "hwmon", "hwmon0"))
+ require.NoError(t, err)
+ assert.Equal(t, filepath.Join(sysRoot, "class", "hwmon", "hwmon0"), target)
+ assert.NoFileExists(t, filepath.Join(shadow, "class", "hwmon", "hwmon1"))
+}
+
+func TestUpdateTemperaturesSkipGpu(t *testing.T) {
+ originalGetSensorTemps := getSensorTemps
+ t.Cleanup(func() {
+ getSensorTemps = originalGetSensorTemps
+ })
+ getSensorTemps = func(ctx context.Context) ([]sensors.TemperatureStat, error) {
+ return []sensors.TemperatureStat{
+ {SensorKey: "coretemp_core_0", Temperature: 55},
+ {SensorKey: "XE", Temperature: 48},
+ }, nil
+ }
+
+ newAgent := func(skipGPU bool) *Agent {
+ agent := &Agent{
+ systemInfo: system.Info{},
+ sensorConfig: &SensorConfig{
+ context: context.Background(),
+ timeout: 2 * time.Second,
+ sensors: map[string]struct{}{},
+ skipGPU: skipGPU,
+ },
+ }
+ return agent
+ }
+
+ stats := &system.Stats{}
+ newAgent(true).updateTemperatures(stats)
+ assert.Equal(t, map[string]float64{"coretemp_core_0": 55}, stats.Temperatures)
+
+ stats = &system.Stats{}
+ newAgent(false).updateTemperatures(stats)
+ assert.Len(t, stats.Temperatures, 2)
+}