Skip to content

Lightweight server monitoring with historical data, docker stats, and alerts.

fix(hub): skip null entries from agents to prevent hub crash (#2834)

Sven van Ginkel87d61ea2b140
diff --git a/internal/alerts/alerts_system.go b/internal/alerts/alerts_system.go
index 39cf3ddc8..8de3ccd9f 100644
--- a/internal/alerts/alerts_system.go
+++ b/internal/alerts/alerts_system.go
@@ -72,6 +72,9 @@ func (am *AlertManager) HandleSystemAlerts(systemRecord *core.Record, data *syst
case "Disk":
maxUsedPct := data.Info.DiskPct
for _, fs := range data.Stats.ExtraFs {
+ if fs == nil {
+ continue
+ }
usedPct := fs.DiskUsed / fs.DiskTotal * 100
if usedPct > maxUsedPct {
maxUsedPct = usedPct
diff --git a/internal/hub/systems/nil_entries_test.go b/internal/hub/systems/nil_entries_test.go
new file mode 100644
index 000000000..a781a20c2
--- /dev/null
+++ b/internal/hub/systems/nil_entries_test.go
@@ -0,0 +1,122 @@
+//go:build testing
+
+package systems
+
+import (
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/blang/semver"
+ "github.com/fxamacker/cbor/v2"
+ "github.com/henrygd/beszel/internal/common"
+ "github.com/henrygd/beszel/internal/entities/container"
+ "github.com/henrygd/beszel/internal/entities/smart"
+ "github.com/henrygd/beszel/internal/entities/system"
+ "github.com/henrygd/beszel/internal/hub/ws"
+ "github.com/lxzan/gws"
+ "github.com/pocketbase/dbx"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+func TestSanitizeNilEntries(t *testing.T) {
+ data := &system.CombinedData{
+ Containers: []*container.Stats{nil, {Id: "abc"}, nil},
+ Stats: system.Stats{
+ ExtraFs: map[string]*system.FsStats{"bad": nil, "good": {DiskTotal: 1}},
+ ZfsPools: map[string]*system.ZfsPool{"bad": nil},
+ },
+ }
+ sanitizeNilEntries(data)
+ assert.Len(t, data.Containers, 1)
+ assert.Equal(t, "abc", data.Containers[0].Id)
+ assert.Len(t, data.Stats.ExtraFs, 1)
+ assert.Contains(t, data.Stats.ExtraFs, "good")
+ assert.Empty(t, data.Stats.ZfsPools)
+
+ // only nils: container slice becomes empty, so Containers[0] is never read
+ data = &system.CombinedData{Containers: []*container.Stats{nil}}
+ sanitizeNilEntries(data)
+ assert.Empty(t, data.Containers)
+
+ assert.NotPanics(t, func() { sanitizeNilEntries(nil) })
+}
+
+func TestExtractPowerMetricsNilAttribute(t *testing.T) {
+ assert.NotPanics(t, func() {
+ hours, cycles := extractPowerMetrics([]*smart.SmartAttribute{nil, {Name: "PowerOnHours", RawValue: 5}})
+ assert.Equal(t, uint64(5), hours)
+ assert.Equal(t, uint64(0), cycles)
+ })
+}
+
+// fixedDataClient answers every GetData request with the same payload.
+type fixedDataClient struct {
+ gws.BuiltinEventHandler
+ data *system.CombinedData
+}
+
+func (c *fixedDataClient) OnMessage(conn *gws.Conn, message *gws.Message) {
+ defer message.Close()
+ var req common.HubRequest[cbor.RawMessage]
+ if err := cbor.Unmarshal(message.Bytes(), &req); err != nil || req.Action != common.GetData {
+ return
+ }
+ // nil pointers in the payload encode as CBOR nulls, as a buggy agent would send
+ response, _ := cbor.Marshal(common.AgentResponse{Id: req.Id, SystemData: c.data})
+ _ = conn.WriteMessage(gws.OpcodeBinary, response)
+}
+
+// A full poll with null containers and filesystems used to panic in
+// createRecords (#2831). The nulls must be gone before sys.data is published.
+func TestUpdateDropsNullEntriesFromAgent(t *testing.T) {
+ sys, app := newTestSystemWithHub(t)
+
+ agentData := &system.CombinedData{
+ Containers: []*container.Stats{nil, {Id: "abc", Name: "web"}},
+ Stats: system.Stats{
+ ExtraFs: map[string]*system.FsStats{"bad": nil, "sdb": {DiskTotal: 100, DiskUsed: 40}},
+ },
+ }
+
+ connections := make(chan *ws.WsConn, 1)
+ upgrader := gws.NewUpgrader(&monitorSyncServer{}, nil)
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ conn, err := upgrader.Upgrade(w, r)
+ if err != nil {
+ t.Error(err)
+ return
+ }
+ wsConn := ws.NewWsConnection(conn, semver.MustParse("0.20.0"))
+ conn.Session().Store("wsConn", wsConn)
+ connections <- wsConn
+ conn.ReadLoop()
+ }))
+ t.Cleanup(server.Close)
+
+ conn, _, err := gws.NewClient(&fixedDataClient{data: agentData}, &gws.ClientOption{Addr: "ws" + strings.TrimPrefix(server.URL, "http")})
+ require.NoError(t, err)
+ t.Cleanup(func() { _ = conn.NetConn().Close() })
+ go conn.ReadLoop()
+
+ select {
+ case sys.WsConn = <-connections:
+ case <-time.After(3 * time.Second):
+ t.Fatal("websocket connection was not established")
+ }
+
+ require.NotPanics(t, func() { require.NoError(t, sys.update()) })
+
+ require.Len(t, sys.data.Containers, 1)
+ assert.Equal(t, "abc", sys.data.Containers[0].Id)
+ assert.NotContains(t, sys.data.Stats.ExtraFs, "bad")
+ assert.Contains(t, sys.data.Stats.ExtraFs, "sdb")
+
+ count, err := app.CountRecords("container_stats", dbx.HashExp{"system": sys.Id})
+ require.NoError(t, err)
+ assert.EqualValues(t, 1, count)
+}
diff --git a/internal/hub/systems/system.go b/internal/hub/systems/system.go
index 58c9ca94a..4cab4997a 100644
--- a/internal/hub/systems/system.go
+++ b/internal/hub/systems/system.go
@@ -7,7 +7,9 @@ import (
"errors"
"fmt"
"hash/fnv"
+ "maps"
"math/rand"
+ "slices"
"strings"
"sync"
"sync/atomic"
@@ -184,6 +186,8 @@ func (sys *System) update() error {
return err
}
+ // Drop null entries
+ sanitizeNilEntries(data)
// ensure deprecated fields from older agents are migrated to current fields
migrateDeprecatedFields(data, !sys.detailsFetched.Load())
sys.data = data
@@ -372,6 +376,17 @@ func (sys *System) createRecords(data *system.CombinedData) (*core.Record, error
return systemRecord, err
}
+// sanitizeNilEntries removes null elements from agent-supplied slices and maps
+// so later code can dereference them safely. Agent payloads can contain nulls.
+func sanitizeNilEntries(data *system.CombinedData) {
+ if data == nil {
+ return
+ }
+ data.Containers = slices.DeleteFunc(data.Containers, func(c *container.Stats) bool { return c == nil })
+ maps.DeleteFunc(data.Stats.ExtraFs, func(_ string, fs *system.FsStats) bool { return fs == nil })
+ maps.DeleteFunc(data.Stats.ZfsPools, func(_ string, p *system.ZfsPool) bool { return p == nil })
+}
+
func createSystemDetailsRecord(app core.App, data *system.Details, systemId string) error {
collectionName := "system_details"
params := dbx.Params{
diff --git a/internal/hub/systems/system_smart.go b/internal/hub/systems/system_smart.go
index 9a6db8de4..390a7552d 100644
--- a/internal/hub/systems/system_smart.go
+++ b/internal/hub/systems/system_smart.go
@@ -3,6 +3,7 @@ package systems
import (
"database/sql"
"errors"
+ "slices"
"strings"
"time"
@@ -131,6 +132,7 @@ func (sys *System) upsertSmartDeviceRecord(app core.App, collection *core.Collec
name = deviceKey
}
+ device.Attributes = slices.DeleteFunc(device.Attributes, func(a *smart.SmartAttribute) bool { return a == nil })
powerOnHours, powerCycles := extractPowerMetrics(device.Attributes)
record.Set("system", sys.Id)
record.Set("name", name)
@@ -151,6 +153,9 @@ func (sys *System) upsertSmartDeviceRecord(app core.App, collection *core.Collec
// extractPowerMetrics extracts power on hours and power cycles from SMART attributes
func extractPowerMetrics(attributes []*smart.SmartAttribute) (powerOnHours, powerCycles uint64) {
for _, attr := range attributes {
+ if attr == nil {
+ continue
+ }
nameLower := strings.ToLower(attr.Name)
if powerOnHours == 0 && (strings.Contains(nameLower, "poweronhours") || strings.Contains(nameLower, "power_on_hours")) {
powerOnHours = attr.RawValue
diff --git a/internal/records/nil_extrafs_test.go b/internal/records/nil_extrafs_test.go
new file mode 100644
index 000000000..28a0d7b54
--- /dev/null
+++ b/internal/records/nil_extrafs_test.go
@@ -0,0 +1,23 @@
+//go:build testing
+
+package records_test
+
+import (
+ "testing"
+
+ "github.com/henrygd/beszel/internal/entities/system"
+ "github.com/henrygd/beszel/internal/records"
+
+ "github.com/stretchr/testify/assert"
+)
+
+func TestAverageSystemStatsSlice_NilExtraFsEntry(t *testing.T) {
+ input := []system.Stats{
+ {ExtraFs: map[string]*system.FsStats{"bad": nil, "sda": {DiskTotal: 100, DiskUsed: 50}}},
+ {ExtraFs: map[string]*system.FsStats{"sda": {DiskTotal: 100, DiskUsed: 70}}},
+ }
+ var result system.Stats
+ assert.NotPanics(t, func() { result = records.AverageSystemStatsSlice(input) })
+ assert.Equal(t, 60.0, result.ExtraFs["sda"].DiskUsed)
+ assert.NotContains(t, result.ExtraFs, "bad")
+}
diff --git a/internal/records/records.go b/internal/records/records.go
index 80b670537..c22f83eb7 100644
--- a/internal/records/records.go
+++ b/internal/records/records.go
@@ -414,6 +414,9 @@ func AverageSystemStatsSlice(records []system.Stats) system.Stats {
sum.ExtraFs = make(map[string]*system.FsStats, len(stats.ExtraFs))
}
for key, value := range stats.ExtraFs {
+ if value == nil {
+ continue
+ }
if _, ok := sum.ExtraFs[key]; !ok {
sum.ExtraFs[key] = &system.FsStats{}
}