Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Guard Docker update alerts against the daily resolve loop

rcourtman0de21d8931d9
diff --git a/internal/alerts/docker_update_daily_cycle_test.go b/internal/alerts/docker_update_daily_cycle_test.go
new file mode 100644
index 0000000000..f66d018355
--- /dev/null
+++ b/internal/alerts/docker_update_daily_cycle_test.go
@@ -0,0 +1,168 @@
+package alerts
+
+import (
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "github.com/rcourtman/pulse-go-rewrite/internal/alerts/eventlog"
+ "github.com/rcourtman/pulse-go-rewrite/internal/models"
+)
+
+// ageDockerUpdateStateForTest moves the manager's recorded instants back by d,
+// which is how the wall clock advancing by d looks from inside the manager.
+// The alerts package reads time.Now() directly, so a multi-day run is
+// simulated by ageing state rather than by sleeping.
+func ageDockerUpdateStateForTest(m *Manager, d time.Duration) {
+ m.mu.Lock()
+ defer m.mu.Unlock()
+ for _, alert := range m.activeAlerts {
+ if alert == nil {
+ continue
+ }
+ alert.StartTime = alert.StartTime.Add(-d)
+ alert.LastSeen = alert.LastSeen.Add(-d)
+ if alert.LastNotified != nil {
+ at := alert.LastNotified.Add(-d)
+ alert.LastNotified = &at
+ }
+ }
+ for _, tracked := range []map[string]time.Time{m.dockerUpdateFirstSeen, m.dockerUpdateFirstSeenByIdentity, m.dockerUpdateLastObserved} {
+ for key, at := range tracked {
+ tracked[key] = at.Add(-d)
+ }
+ }
+}
+
+// An update that stays asserted must hold one open alert for days. Reports
+// arrive every 30 minutes and the hourly housekeeping runs between them, as on
+// a live server.
+//
+// #2353: on v6.3.2 the hourly sweep expired update tracking by first
+// detection, the next report restarted the delay below its threshold without
+// refreshing the open alert, and the stale-alert sweep resolved it a day
+// later, just before the restarted delay re-fired it. Two independent repairs
+// now hold the alert open; each case below fails when one of them regresses:
+// - tracking expiry must follow the latest observation, or a 48h delay
+// restarts every day and never opens;
+// - an open alert's start must stand in for tracking a restart lost, or the
+// restarted delay stops refreshing the alert and the stale-alert sweep
+// resolves it before the delay re-opens it.
+func TestDockerUpdateAlertDoesNotCycleDailyWhileAsserted(t *testing.T) {
+ cases := []struct {
+ name string
+ delayHours int
+ trackingLost time.Duration // restart: in-memory tracking dropped, saved alert kept
+ simulateHours int
+ }{
+ {name: "24h delay", delayHours: 24, simulateHours: 96},
+ {name: "48h delay", delayHours: 48, simulateHours: 120},
+ {name: "24h delay across a restart", delayHours: 24, trackingLost: 30 * time.Hour, simulateHours: 96},
+ }
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ m := newTestManager(t)
+ m.mu.Lock()
+ m.config.DockerDefaults.UpdateAlertDelayHours = tc.delayHours
+ m.mu.Unlock()
+
+ host := models.DockerHost{ID: "host", AgentID: "agent", Hostname: "host", Containers: []models.DockerContainer{{
+ ID: "container", Name: "web", Image: "mongo:7", State: "running",
+ UpdateStatus: &models.DockerContainerUpdateStatus{
+ UpdateAvailable: true, CurrentDigest: "sha256:old", LatestDigest: "sha256:new", LastChecked: time.Now(),
+ },
+ }}}
+
+ var firings, resolutions atomic.Int32
+ m.SubscribeLifecycleCallback(func(event LifecycleEvent) {
+ switch event.Type {
+ case eventlog.TypeFired, eventlog.TypeRefired:
+ firings.Add(1)
+ case eventlog.TypeResolved:
+ resolutions.Add(1)
+ }
+ })
+
+ delay := time.Duration(tc.delayHours) * time.Hour
+ const step = 15 * time.Minute
+ var openedStart time.Time
+ for elapsed := time.Duration(0); elapsed <= time.Duration(tc.simulateHours)*time.Hour; elapsed += step {
+ if elapsed > 0 {
+ ageDockerUpdateStateForTest(m, step)
+ }
+ if tc.trackingLost > 0 && elapsed == tc.trackingLost {
+ m.mu.Lock()
+ clear(m.dockerUpdateFirstSeen)
+ clear(m.dockerUpdateFirstSeenByIdentity)
+ clear(m.dockerUpdateLastObserved)
+ m.mu.Unlock()
+ }
+ // The server is down for the two hours before the restart:
+ // no reports and no housekeeping.
+ down := tc.trackingLost > 0 && elapsed >= tc.trackingLost-2*time.Hour && elapsed < tc.trackingLost
+ switch {
+ case down:
+ case elapsed%time.Hour == 0, elapsed%time.Hour == 30*time.Minute:
+ m.CheckDockerHost(host)
+ case elapsed%time.Hour == 15*time.Minute:
+ m.cleanupStaleMaps()
+ }
+ active := m.GetActiveAlerts()
+ if elapsed < delay {
+ if len(active) != 0 {
+ t.Fatalf("%s: alert opened before its %s delay: %+v", elapsed, delay, active)
+ }
+ continue
+ }
+ if len(active) != 1 {
+ t.Fatalf("%s: asserted update alert not open (firings=%d resolutions=%d)", elapsed, firings.Load(), resolutions.Load())
+ }
+ // Ageing moves the start back with the clock, so start+elapsed
+ // is fixed for as long as the occurrence continues.
+ start := active[0].StartTime.Add(elapsed)
+ if openedStart.IsZero() {
+ openedStart = start
+ } else if d := start.Sub(openedStart); d > time.Second || d < -time.Second {
+ t.Fatalf("%s: incident restarted (start moved by %s)", elapsed, d)
+ }
+ }
+ if firings.Load() != 1 || resolutions.Load() != 0 {
+ t.Fatalf("firings=%d resolutions=%d, want one opening and no resolution", firings.Load(), resolutions.Load())
+ }
+
+ // A genuine update still resolves.
+ host.Containers[0].UpdateStatus = &models.DockerContainerUpdateStatus{
+ UpdateAvailable: false, CurrentDigest: "sha256:new", LatestDigest: "sha256:new", LastChecked: time.Now(),
+ }
+ m.CheckDockerHost(host)
+ if got := m.GetActiveAlerts(); len(got) != 0 || resolutions.Load() != 1 {
+ t.Fatalf("current image did not resolve the alert: %+v resolutions=%d", got, resolutions.Load())
+ }
+ })
+ }
+}
+
+// Removing the container still resolves an open update alert.
+func TestDockerUpdateAlertResolvesWhenContainerRemoved(t *testing.T) {
+ m := newTestManager(t)
+ m.mu.Lock()
+ m.config.DockerDefaults.UpdateAlertDelayHours = 24
+ m.mu.Unlock()
+ host := models.DockerHost{ID: "host", AgentID: "agent", Hostname: "host", Containers: []models.DockerContainer{{
+ ID: "container", Name: "web", Image: "mongo:7", State: "running",
+ UpdateStatus: &models.DockerContainerUpdateStatus{
+ UpdateAvailable: true, CurrentDigest: "sha256:old", LatestDigest: "sha256:new", LastChecked: time.Now(),
+ },
+ }}}
+ m.CheckDockerHost(host)
+ ageDockerUpdateStateForTest(m, 25*time.Hour)
+ m.CheckDockerHost(host)
+ if got := m.GetActiveAlerts(); len(got) != 1 {
+ t.Fatalf("pending update past its delay did not open: %+v", got)
+ }
+ host.Containers = nil
+ m.CheckDockerHost(host)
+ if got := m.GetActiveAlerts(); len(got) != 0 {
+ t.Fatalf("removed container kept its update alert: %+v", got)
+ }
+}