Lightweight server monitoring with historical data, docker stats, and alerts.
fix(hub): skip null entries from agents to prevent hub crash (#2834)
Sven van Ginkel87d61ea2b140
diff --git a/internal/alerts/alerts_system.go b/internal/alerts/alerts_system.goindex 39cf3ddc8..8de3ccd9f 100644--- a/internal/alerts/alerts_system.go+++ b/internal/alerts/alerts_system.go@@ -72,6 +72,9 @@ func (am *AlertManager) HandleSystemAlerts(systemRecord *core.Record, data *systcase "Disk":maxUsedPct := data.Info.DiskPctfor _, fs := range data.Stats.ExtraFs {+ if fs == nil {+ continue+ }usedPct := fs.DiskUsed / fs.DiskTotal * 100if usedPct > maxUsedPct {maxUsedPct = usedPctdiff --git a/internal/hub/systems/nil_entries_test.go b/internal/hub/systems/nil_entries_test.gonew file mode 100644index 000000000..a781a20c2--- /dev/null+++ b/internal/hub/systems/nil_entries_test.go@@ -0,0 +1,122 @@+//go:build testing++package systems++import (+ "net/http"+ "net/http/httptest"+ "strings"+ "testing"+ "time"++ "github.com/blang/semver"+ "github.com/fxamacker/cbor/v2"+ "github.com/henrygd/beszel/internal/common"+ "github.com/henrygd/beszel/internal/entities/container"+ "github.com/henrygd/beszel/internal/entities/smart"+ "github.com/henrygd/beszel/internal/entities/system"+ "github.com/henrygd/beszel/internal/hub/ws"+ "github.com/lxzan/gws"+ "github.com/pocketbase/dbx"++ "github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require"+)++func TestSanitizeNilEntries(t *testing.T) {+ data := &system.CombinedData{+ Containers: []*container.Stats{nil, {Id: "abc"}, nil},+ Stats: system.Stats{+ ExtraFs: map[string]*system.FsStats{"bad": nil, "good": {DiskTotal: 1}},+ ZfsPools: map[string]*system.ZfsPool{"bad": nil},+ },+ }+ sanitizeNilEntries(data)+ assert.Len(t, data.Containers, 1)+ assert.Equal(t, "abc", data.Containers[0].Id)+ assert.Len(t, data.Stats.ExtraFs, 1)+ assert.Contains(t, data.Stats.ExtraFs, "good")+ assert.Empty(t, data.Stats.ZfsPools)++ // only nils: container slice becomes empty, so Containers[0] is never read+ data = &system.CombinedData{Containers: []*container.Stats{nil}}+ sanitizeNilEntries(data)+ assert.Empty(t, data.Containers)++ assert.NotPanics(t, func() { sanitizeNilEntries(nil) })+}++func TestExtractPowerMetricsNilAttribute(t *testing.T) {+ assert.NotPanics(t, func() {+ hours, cycles := extractPowerMetrics([]*smart.SmartAttribute{nil, {Name: "PowerOnHours", RawValue: 5}})+ assert.Equal(t, uint64(5), hours)+ assert.Equal(t, uint64(0), cycles)+ })+}++// fixedDataClient answers every GetData request with the same payload.+type fixedDataClient struct {+ gws.BuiltinEventHandler+ data *system.CombinedData+}++func (c *fixedDataClient) OnMessage(conn *gws.Conn, message *gws.Message) {+ defer message.Close()+ var req common.HubRequest[cbor.RawMessage]+ if err := cbor.Unmarshal(message.Bytes(), &req); err != nil || req.Action != common.GetData {+ return+ }+ // nil pointers in the payload encode as CBOR nulls, as a buggy agent would send+ response, _ := cbor.Marshal(common.AgentResponse{Id: req.Id, SystemData: c.data})+ _ = conn.WriteMessage(gws.OpcodeBinary, response)+}++// A full poll with null containers and filesystems used to panic in+// createRecords (#2831). The nulls must be gone before sys.data is published.+func TestUpdateDropsNullEntriesFromAgent(t *testing.T) {+ sys, app := newTestSystemWithHub(t)++ agentData := &system.CombinedData{+ Containers: []*container.Stats{nil, {Id: "abc", Name: "web"}},+ Stats: system.Stats{+ ExtraFs: map[string]*system.FsStats{"bad": nil, "sdb": {DiskTotal: 100, DiskUsed: 40}},+ },+ }++ connections := make(chan *ws.WsConn, 1)+ upgrader := gws.NewUpgrader(&monitorSyncServer{}, nil)+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ conn, err := upgrader.Upgrade(w, r)+ if err != nil {+ t.Error(err)+ return+ }+ wsConn := ws.NewWsConnection(conn, semver.MustParse("0.20.0"))+ conn.Session().Store("wsConn", wsConn)+ connections <- wsConn+ conn.ReadLoop()+ }))+ t.Cleanup(server.Close)++ conn, _, err := gws.NewClient(&fixedDataClient{data: agentData}, &gws.ClientOption{Addr: "ws" + strings.TrimPrefix(server.URL, "http")})+ require.NoError(t, err)+ t.Cleanup(func() { _ = conn.NetConn().Close() })+ go conn.ReadLoop()++ select {+ case sys.WsConn = <-connections:+ case <-time.After(3 * time.Second):+ t.Fatal("websocket connection was not established")+ }++ require.NotPanics(t, func() { require.NoError(t, sys.update()) })++ require.Len(t, sys.data.Containers, 1)+ assert.Equal(t, "abc", sys.data.Containers[0].Id)+ assert.NotContains(t, sys.data.Stats.ExtraFs, "bad")+ assert.Contains(t, sys.data.Stats.ExtraFs, "sdb")++ count, err := app.CountRecords("container_stats", dbx.HashExp{"system": sys.Id})+ require.NoError(t, err)+ assert.EqualValues(t, 1, count)+}diff --git a/internal/hub/systems/system.go b/internal/hub/systems/system.goindex 58c9ca94a..4cab4997a 100644--- a/internal/hub/systems/system.go+++ b/internal/hub/systems/system.go@@ -7,7 +7,9 @@ import ("errors""fmt""hash/fnv"+ "maps""math/rand"+ "slices""strings""sync""sync/atomic"@@ -184,6 +186,8 @@ func (sys *System) update() error {return err}+ // Drop null entries+ sanitizeNilEntries(data)// ensure deprecated fields from older agents are migrated to current fieldsmigrateDeprecatedFields(data, !sys.detailsFetched.Load())sys.data = data@@ -372,6 +376,17 @@ func (sys *System) createRecords(data *system.CombinedData) (*core.Record, errorreturn systemRecord, err}+// sanitizeNilEntries removes null elements from agent-supplied slices and maps+// so later code can dereference them safely. Agent payloads can contain nulls.+func sanitizeNilEntries(data *system.CombinedData) {+ if data == nil {+ return+ }+ data.Containers = slices.DeleteFunc(data.Containers, func(c *container.Stats) bool { return c == nil })+ maps.DeleteFunc(data.Stats.ExtraFs, func(_ string, fs *system.FsStats) bool { return fs == nil })+ maps.DeleteFunc(data.Stats.ZfsPools, func(_ string, p *system.ZfsPool) bool { return p == nil })+}+func createSystemDetailsRecord(app core.App, data *system.Details, systemId string) error {collectionName := "system_details"params := dbx.Params{diff --git a/internal/hub/systems/system_smart.go b/internal/hub/systems/system_smart.goindex 9a6db8de4..390a7552d 100644--- a/internal/hub/systems/system_smart.go+++ b/internal/hub/systems/system_smart.go@@ -3,6 +3,7 @@ package systemsimport ("database/sql""errors"+ "slices""strings""time"@@ -131,6 +132,7 @@ func (sys *System) upsertSmartDeviceRecord(app core.App, collection *core.Collecname = deviceKey}+ device.Attributes = slices.DeleteFunc(device.Attributes, func(a *smart.SmartAttribute) bool { return a == nil })powerOnHours, powerCycles := extractPowerMetrics(device.Attributes)record.Set("system", sys.Id)record.Set("name", name)@@ -151,6 +153,9 @@ func (sys *System) upsertSmartDeviceRecord(app core.App, collection *core.Collec// extractPowerMetrics extracts power on hours and power cycles from SMART attributesfunc extractPowerMetrics(attributes []*smart.SmartAttribute) (powerOnHours, powerCycles uint64) {for _, attr := range attributes {+ if attr == nil {+ continue+ }nameLower := strings.ToLower(attr.Name)if powerOnHours == 0 && (strings.Contains(nameLower, "poweronhours") || strings.Contains(nameLower, "power_on_hours")) {powerOnHours = attr.RawValuediff --git a/internal/records/nil_extrafs_test.go b/internal/records/nil_extrafs_test.gonew file mode 100644index 000000000..28a0d7b54--- /dev/null+++ b/internal/records/nil_extrafs_test.go@@ -0,0 +1,23 @@+//go:build testing++package records_test++import (+ "testing"++ "github.com/henrygd/beszel/internal/entities/system"+ "github.com/henrygd/beszel/internal/records"++ "github.com/stretchr/testify/assert"+)++func TestAverageSystemStatsSlice_NilExtraFsEntry(t *testing.T) {+ input := []system.Stats{+ {ExtraFs: map[string]*system.FsStats{"bad": nil, "sda": {DiskTotal: 100, DiskUsed: 50}}},+ {ExtraFs: map[string]*system.FsStats{"sda": {DiskTotal: 100, DiskUsed: 70}}},+ }+ var result system.Stats+ assert.NotPanics(t, func() { result = records.AverageSystemStatsSlice(input) })+ assert.Equal(t, 60.0, result.ExtraFs["sda"].DiskUsed)+ assert.NotContains(t, result.ExtraFs, "bad")+}diff --git a/internal/records/records.go b/internal/records/records.goindex 80b670537..c22f83eb7 100644--- a/internal/records/records.go+++ b/internal/records/records.go@@ -414,6 +414,9 @@ func AverageSystemStatsSlice(records []system.Stats) system.Stats {sum.ExtraFs = make(map[string]*system.FsStats, len(stats.ExtraFs))}for key, value := range stats.ExtraFs {+ if value == nil {+ continue+ }if _, ok := sum.ExtraFs[key]; !ok {sum.ExtraFs[key] = &system.FsStats{}}