Skip to content

Lightweight server monitoring with historical data, docker stats, and alerts.

fix(agent): report failed services that were never active (#2776, #2777)

Martin Stenröse46abf3576cba
diff --git a/agent/systemd.go b/agent/systemd.go
index d24d77910..cefbc8937 100644
--- a/agent/systemd.go
+++ b/agent/systemd.go
@@ -226,6 +226,33 @@ func (sm *systemdManager) getServiceStats(conn *dbus.Conn, refresh bool) []*syst
return services
}
+// keepUnit reports whether a unit should be listed. Units that are loaded but
+// have never been active are skipped. Failed units are kept without calling
+// activeEnterTimestamp: a service that fails while starting, such as a
+// Type=oneshot service without RemainAfterExit, never becomes active, so its
+// ActiveEnterTimestamp is never set (#2776).
+//
+// When a skipped unit is inactive again, its entry from an earlier update is
+// removed so that a failed oneshot which has since succeeded is not kept as
+// failed. While it is running again, its last state is kept so that the failed
+// count doesn't drop mid-run. The caller must hold the lock.
+func (sm *systemdManager) keepUnit(unit dbus.UnitStatus, activeEnterTimestamp func() (any, error)) (bool, error) {
+ if unit.ActiveState == "failed" {
+ return true, nil
+ }
+ value, err := activeEnterTimestamp()
+ if err != nil {
+ return false, err
+ }
+ if ts, ok := value.(uint64); ok && ts != 0 && ts != math.MaxUint64 {
+ return true, nil
+ }
+ if unit.ActiveState == "inactive" {
+ delete(sm.serviceStatsMap, unit.Name)
+ }
+ return false, nil
+}
+
// updateServiceStats updates the statistics for a single systemd service.
func (sm *systemdManager) updateServiceStats(conn *dbus.Conn, unit dbus.UnitStatus) (*systemd.Service, error) {
sm.Lock()
@@ -233,14 +260,20 @@ func (sm *systemdManager) updateServiceStats(conn *dbus.Conn, unit dbus.UnitStat
ctx := context.Background()
- // if service has never been active (no active since time), skip it
- if activeEnterTsProp, err := conn.GetUnitTypePropertyContext(ctx, unit.Name, "Unit", "ActiveEnterTimestamp"); err == nil {
- if ts, ok := activeEnterTsProp.Value.Value().(uint64); !ok || ts == 0 || ts == math.MaxUint64 {
- return nil, errNoActiveTime
+ // skip services that have never been active (no active since time), see keepUnit
+ keep, err := sm.keepUnit(unit, func() (any, error) {
+ prop, err := conn.GetUnitTypePropertyContext(ctx, unit.Name, "Unit", "ActiveEnterTimestamp")
+ if err != nil {
+ return nil, err
}
- } else {
+ return prop.Value.Value(), nil
+ })
+ if err != nil {
return nil, err
}
+ if !keep {
+ return nil, errNoActiveTime
+ }
service, serviceExists := sm.serviceStatsMap[unit.Name]
if !serviceExists {
diff --git a/agent/systemd_test.go b/agent/systemd_test.go
index d8c6a7353..1fc3e0843 100644
--- a/agent/systemd_test.go
+++ b/agent/systemd_test.go
@@ -4,12 +4,14 @@ package agent
import (
"errors"
+ "math"
"os"
"os/exec"
"path/filepath"
"strings"
"testing"
+ "github.com/coreos/go-systemd/v22/dbus"
"github.com/henrygd/beszel/internal/entities/systemd"
"github.com/stretchr/testify/assert"
)
@@ -72,6 +74,67 @@ func TestServiceUnitName(t *testing.T) {
}
}
+func TestKeepUnit(t *testing.T) {
+ tests := []struct {
+ name string
+ activeState string
+ timestamp any // ActiveEnterTimestamp value
+ hadEntry bool // unit has a failed entry from an earlier update
+ wantKeep bool
+ wantLookup bool // ActiveEnterTimestamp is read
+ wantEntry bool // failed entry is still there afterwards
+ }{
+ {name: "active unit with timestamp", activeState: "active", timestamp: uint64(1762500000000000), wantKeep: true, wantLookup: true},
+ {name: "never-run inactive unit", activeState: "inactive", timestamp: uint64(0), wantLookup: true},
+ {name: "failed oneshot with timestamp 0", activeState: "failed", timestamp: uint64(0), wantKeep: true},
+ {name: "timestamp MaxUint64", activeState: "inactive", timestamp: uint64(math.MaxUint64), wantLookup: true},
+ {name: "timestamp not a uint64", activeState: "inactive", timestamp: "unexpected", wantLookup: true},
+ {name: "recovered oneshot removes stale entry", activeState: "inactive", timestamp: uint64(0), hadEntry: true, wantLookup: true},
+ {name: "failed oneshot running again keeps entry", activeState: "activating", timestamp: uint64(0), hadEntry: true, wantLookup: true, wantEntry: true},
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ sm := &systemdManager{serviceStatsMap: map[string]*systemd.Service{}}
+ if tt.hadEntry {
+ sm.serviceStatsMap["backup.service"] = &systemd.Service{Name: "backup", State: systemd.StatusFailed}
+ }
+ lookedUp := false
+
+ keep, err := sm.keepUnit(dbus.UnitStatus{Name: "backup.service", ActiveState: tt.activeState}, func() (any, error) {
+ lookedUp = true
+ return tt.timestamp, nil
+ })
+
+ assert.NoError(t, err)
+ assert.Equal(t, tt.wantKeep, keep)
+ assert.Equal(t, tt.wantLookup, lookedUp)
+ assert.Equal(t, tt.wantEntry, sm.serviceStatsMap["backup.service"] != nil)
+ wantCount := 0
+ if tt.wantEntry {
+ wantCount = 1
+ }
+ assert.Equal(t, wantCount, sm.getServiceStatsCount())
+ assert.EqualValues(t, wantCount, sm.getFailedServiceCount())
+ })
+ }
+
+ t.Run("lookup error keeps existing entry", func(t *testing.T) {
+ sm := &systemdManager{serviceStatsMap: map[string]*systemd.Service{
+ "backup.service": {Name: "backup", State: systemd.StatusFailed},
+ }}
+ lookupErr := errors.New("dbus error")
+
+ keep, err := sm.keepUnit(dbus.UnitStatus{Name: "backup.service", ActiveState: "inactive"}, func() (any, error) {
+ return nil, lookupErr
+ })
+
+ assert.ErrorIs(t, err, lookupErr)
+ assert.False(t, keep)
+ assert.Contains(t, sm.serviceStatsMap, "backup.service")
+ })
+}
+
func TestCanReadSystemJournal(t *testing.T) {
tests := []struct {
name string