Lightweight server monitoring with historical data, docker stats, and alerts.
fix(agent): report failed services that were never active (#2776, #2777)
Martin Stenröse46abf3576cba
diff --git a/agent/systemd.go b/agent/systemd.goindex d24d77910..cefbc8937 100644--- a/agent/systemd.go+++ b/agent/systemd.go@@ -226,6 +226,33 @@ func (sm *systemdManager) getServiceStats(conn *dbus.Conn, refresh bool) []*systreturn services}+// keepUnit reports whether a unit should be listed. Units that are loaded but+// have never been active are skipped. Failed units are kept without calling+// activeEnterTimestamp: a service that fails while starting, such as a+// Type=oneshot service without RemainAfterExit, never becomes active, so its+// ActiveEnterTimestamp is never set (#2776).+//+// When a skipped unit is inactive again, its entry from an earlier update is+// removed so that a failed oneshot which has since succeeded is not kept as+// failed. While it is running again, its last state is kept so that the failed+// count doesn't drop mid-run. The caller must hold the lock.+func (sm *systemdManager) keepUnit(unit dbus.UnitStatus, activeEnterTimestamp func() (any, error)) (bool, error) {+ if unit.ActiveState == "failed" {+ return true, nil+ }+ value, err := activeEnterTimestamp()+ if err != nil {+ return false, err+ }+ if ts, ok := value.(uint64); ok && ts != 0 && ts != math.MaxUint64 {+ return true, nil+ }+ if unit.ActiveState == "inactive" {+ delete(sm.serviceStatsMap, unit.Name)+ }+ return false, nil+}+// updateServiceStats updates the statistics for a single systemd service.func (sm *systemdManager) updateServiceStats(conn *dbus.Conn, unit dbus.UnitStatus) (*systemd.Service, error) {sm.Lock()@@ -233,14 +260,20 @@ func (sm *systemdManager) updateServiceStats(conn *dbus.Conn, unit dbus.UnitStatctx := context.Background()- // if service has never been active (no active since time), skip it- if activeEnterTsProp, err := conn.GetUnitTypePropertyContext(ctx, unit.Name, "Unit", "ActiveEnterTimestamp"); err == nil {- if ts, ok := activeEnterTsProp.Value.Value().(uint64); !ok || ts == 0 || ts == math.MaxUint64 {- return nil, errNoActiveTime+ // skip services that have never been active (no active since time), see keepUnit+ keep, err := sm.keepUnit(unit, func() (any, error) {+ prop, err := conn.GetUnitTypePropertyContext(ctx, unit.Name, "Unit", "ActiveEnterTimestamp")+ if err != nil {+ return nil, err}- } else {+ return prop.Value.Value(), nil+ })+ if err != nil {return nil, err}+ if !keep {+ return nil, errNoActiveTime+ }service, serviceExists := sm.serviceStatsMap[unit.Name]if !serviceExists {diff --git a/agent/systemd_test.go b/agent/systemd_test.goindex d8c6a7353..1fc3e0843 100644--- a/agent/systemd_test.go+++ b/agent/systemd_test.go@@ -4,12 +4,14 @@ package agentimport ("errors"+ "math""os""os/exec""path/filepath""strings""testing"+ "github.com/coreos/go-systemd/v22/dbus""github.com/henrygd/beszel/internal/entities/systemd""github.com/stretchr/testify/assert")@@ -72,6 +74,67 @@ func TestServiceUnitName(t *testing.T) {}}+func TestKeepUnit(t *testing.T) {+ tests := []struct {+ name string+ activeState string+ timestamp any // ActiveEnterTimestamp value+ hadEntry bool // unit has a failed entry from an earlier update+ wantKeep bool+ wantLookup bool // ActiveEnterTimestamp is read+ wantEntry bool // failed entry is still there afterwards+ }{+ {name: "active unit with timestamp", activeState: "active", timestamp: uint64(1762500000000000), wantKeep: true, wantLookup: true},+ {name: "never-run inactive unit", activeState: "inactive", timestamp: uint64(0), wantLookup: true},+ {name: "failed oneshot with timestamp 0", activeState: "failed", timestamp: uint64(0), wantKeep: true},+ {name: "timestamp MaxUint64", activeState: "inactive", timestamp: uint64(math.MaxUint64), wantLookup: true},+ {name: "timestamp not a uint64", activeState: "inactive", timestamp: "unexpected", wantLookup: true},+ {name: "recovered oneshot removes stale entry", activeState: "inactive", timestamp: uint64(0), hadEntry: true, wantLookup: true},+ {name: "failed oneshot running again keeps entry", activeState: "activating", timestamp: uint64(0), hadEntry: true, wantLookup: true, wantEntry: true},+ }++ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ sm := &systemdManager{serviceStatsMap: map[string]*systemd.Service{}}+ if tt.hadEntry {+ sm.serviceStatsMap["backup.service"] = &systemd.Service{Name: "backup", State: systemd.StatusFailed}+ }+ lookedUp := false++ keep, err := sm.keepUnit(dbus.UnitStatus{Name: "backup.service", ActiveState: tt.activeState}, func() (any, error) {+ lookedUp = true+ return tt.timestamp, nil+ })++ assert.NoError(t, err)+ assert.Equal(t, tt.wantKeep, keep)+ assert.Equal(t, tt.wantLookup, lookedUp)+ assert.Equal(t, tt.wantEntry, sm.serviceStatsMap["backup.service"] != nil)+ wantCount := 0+ if tt.wantEntry {+ wantCount = 1+ }+ assert.Equal(t, wantCount, sm.getServiceStatsCount())+ assert.EqualValues(t, wantCount, sm.getFailedServiceCount())+ })+ }++ t.Run("lookup error keeps existing entry", func(t *testing.T) {+ sm := &systemdManager{serviceStatsMap: map[string]*systemd.Service{+ "backup.service": {Name: "backup", State: systemd.StatusFailed},+ }}+ lookupErr := errors.New("dbus error")++ keep, err := sm.keepUnit(dbus.UnitStatus{Name: "backup.service", ActiveState: "inactive"}, func() (any, error) {+ return nil, lookupErr+ })++ assert.ErrorIs(t, err, lookupErr)+ assert.False(t, keep)+ assert.Contains(t, sm.serviceStatsMap, "backup.service")+ })+}+func TestCanReadSystemJournal(t *testing.T) {tests := []struct {name string