Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Reset guest shadow runs on healthy powered-state observations

pulse-triage[bot]2415a22c51e1
diff --git a/internal/alerts/guest.go b/internal/alerts/guest.go
index b9fc3b2723..dacc37dcbc 100644
--- a/internal/alerts/guest.go
+++ b/internal/alerts/guest.go
@@ -565,6 +565,11 @@ func (m *Manager) clearGuestPoweredOffAlert(guestID, name string) {
}
}()
defer m.mu.Unlock()
+ // Healthy and intentionally stopped observations end the shadow's run as
+ // well, even when the live alert is only pending (and the lookup below
+ // returns early). Otherwise a later stopped run can inherit old grace or
+ // confirmation evidence and falsely disagree with the live manager.
+ defer m.shadowObserveRecoveryNoLock(guestID, canonicalPoweredStateSpecID(guestID), alertID, 1)
// Reset confirmation count when guest comes back online
if decision := m.evaluateIntentNoLock(guestID, "", string(AlertIntentSignalOffline), alertID, m.policyNow().UTC(), false, BackupIntentContext{}); decision.StateChanged {
diff --git a/internal/alerts/shadow_guest_recovery_test.go b/internal/alerts/shadow_guest_recovery_test.go
new file mode 100644
index 0000000000..bfb2e5259b
--- /dev/null
+++ b/internal/alerts/shadow_guest_recovery_test.go
@@ -0,0 +1,173 @@
+package alerts
+
+import (
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "github.com/rcourtman/pulse-go-rewrite/internal/alerts/eventlog"
+ "github.com/rcourtman/pulse-go-rewrite/internal/alerts/reducer"
+ "github.com/rcourtman/pulse-go-rewrite/internal/models"
+)
+
+const shadowGuestID = "pve:node:101"
+
+func shadowGuestManager(t *testing.T, grace bool) (*Manager, *atomic.Int64) {
+ t.Helper()
+ m := newShadowFeedManager(t)
+ configureUnifiedEvalManager(t, m, unifiedEvalBaseConfig())
+ epoch := time.Date(2026, 10, 25, 12, 0, 0, 0, time.UTC)
+ tick := &atomic.Int64{}
+ m.mu.Lock()
+ m.now = func() time.Time { return epoch.Add(time.Duration(tick.Load())) }
+ m.intentClock = func() time.Duration { return time.Duration(tick.Load()) }
+ m.mu.Unlock()
+ if grace {
+ document := NewAlertIntentPolicyDocument()
+ seconds := 600
+ document.Defaults[string(AlertIntentSignalOffline)] = AlertIntentRule{GraceSeconds: &seconds}
+ if err := m.LoadIntentPolicies(document); err != nil {
+ t.Fatal(err)
+ }
+ }
+ return m, tick
+}
+
+func observeShadowGuest(m *Manager, kind, status string, onBoot bool) {
+ if kind == "container" {
+ m.CheckGuest(models.Container{
+ ID: shadowGuestID, VMID: 101, Name: "database", Node: "node", Instance: "pve",
+ Status: status, OnBoot: &onBoot,
+ }, "pve")
+ return
+ }
+ m.CheckGuest(models.VM{
+ ID: shadowGuestID, VMID: 101, Name: "database", Node: "node", Instance: "pve",
+ Status: status, OnBoot: &onBoot,
+ }, "pve")
+}
+
+func assertShadowGuestParity(t *testing.T, m *Manager, wantFiring bool) {
+ t.Helper()
+ active := m.GetActiveAlerts()
+ if len(active) != 0 && (len(active) != 1 || active[0].ResourceID != shadowGuestID) {
+ t.Fatalf("unexpected live alerts: %+v", active)
+ }
+ if got := len(active) == 1; got != wantFiring {
+ t.Fatalf("live firing = %t, want %t", got, wantFiring)
+ }
+ if got := m.ShadowDivergences(); got != 0 {
+ t.Fatalf("false shadow divergences = %d: %+v", got, shadowDivergenceEvents(t, m))
+ }
+ if events := shadowDivergenceEvents(t, m); len(events) != 0 {
+ t.Fatalf("false shadow events: %+v", events)
+ }
+ m.mu.RLock()
+ defer m.mu.RUnlock()
+ core, coreOK := m.core.Incident(shadowGuestID, canonicalPoweredStateSpecID(shadowGuestID))
+ shadow, shadowOK := m.shadow.state.Incident(shadowGuestID, canonicalPoweredStateSpecID(shadowGuestID))
+ if coreOK != shadowOK || (coreOK && (core.State != shadow.State || core.Confirmations != shadow.Confirmations ||
+ !core.PendingSince.Equal(shadow.PendingSince) || !core.StartedAt.Equal(shadow.StartedAt))) {
+ t.Fatalf("guest reducers differ: core=%+v (exists=%t), shadow=%+v (exists=%t)", core, coreOK, shadow, shadowOK)
+ }
+}
+
+// Regression for #3280: a running, paused, suspended or deliberately stopped
+// guest interrupts the first pending run. A later stop must satisfy its own
+// full grace/confirmation run; the shadow must not reuse the old evidence.
+func TestShadowGuestRecoveryRestartsPendingRun(t *testing.T) {
+ for _, kind := range []string{"vm", "container"} {
+ for _, policy := range []string{"600-second-grace", "two-confirmations"} {
+ for _, healthy := range []string{"running", "paused", "suspended", "onboot-disabled"} {
+ t.Run(kind+"/"+policy+"/"+healthy, func(t *testing.T) {
+ grace := policy == "600-second-grace"
+ m, tick := shadowGuestManager(t, grace)
+ var notifications atomic.Int64
+ m.SetAlertCallback(func(*Alert) { notifications.Add(1) })
+ observeShadowGuest(m, kind, "stopped", true)
+ assertShadowGuestParity(t, m, false)
+ tick.Store(int64(5 * time.Minute))
+ if healthy == "onboot-disabled" {
+ observeShadowGuest(m, kind, "stopped", false)
+ } else {
+ observeShadowGuest(m, kind, healthy, true)
+ }
+ tick.Store(int64(10 * time.Minute))
+ observeShadowGuest(m, kind, "stopped", true)
+ assertShadowGuestParity(t, m, false)
+ if notifications.Load() != 0 || len(m.GetAlertHistory(100)) != 0 {
+ t.Fatal("interrupted grace/confirmation run fired or notified")
+ }
+ if grace {
+ tick.Store(int64(20*time.Minute - time.Second))
+ observeShadowGuest(m, kind, "stopped", true)
+ assertShadowGuestParity(t, m, false)
+ }
+ tick.Store(int64(20 * time.Minute))
+ observeShadowGuest(m, kind, "stopped", true)
+ assertShadowGuestParity(t, m, true)
+ if got := notifications.Load(); got != 1 {
+ t.Fatalf("notifications after genuine activation = %d, want 1", got)
+ }
+ if events := queryAlertEvents(t, m, eventlog.Filter{Types: []string{eventlog.TypeFired}}); len(events) != 1 {
+ t.Fatalf("genuine firing events = %d, want 1", len(events))
+ }
+ })
+ }
+ }
+ }
+}
+
+// Exercise active and restart-seeded shadow state as well as no-active-alert
+// recovery. Expected-offline continues to hold activation after recovery;
+// unsuppression must still wait the full new grace and fire normally.
+func TestShadowGuestRecoveryAfterFiringAndFeedRestart(t *testing.T) {
+ for _, kind := range []string{"vm", "container"} {
+ for _, restart := range []bool{false, true} {
+ name := kind + "/continuous-feed"
+ if restart {
+ name = kind + "/reseeded-feed"
+ }
+ t.Run(name, func(t *testing.T) {
+ m, tick := shadowGuestManager(t, true)
+ observeShadowGuest(m, kind, "stopped", true)
+ tick.Store(int64(10 * time.Minute))
+ observeShadowGuest(m, kind, "stopped", true)
+ assertShadowGuestParity(t, m, true)
+ if err := m.AcknowledgeAlert(canonicalPoweredStateStateID(shadowGuestID), "operator"); err != nil {
+ t.Fatal(err)
+ }
+ if restart {
+ m.EnableShadowFeed()
+ }
+ tick.Store(int64(11 * time.Minute))
+ observeShadowGuest(m, kind, "running", true)
+ assertShadowGuestParity(t, m, false)
+ m.SetOperatorIntentContextResolver(func(string, time.Time) (OperatorIntentContext, bool) {
+ return OperatorIntentContext{MonitoringMode: "expected_offline"}, true
+ })
+ tick.Store(int64(12 * time.Minute))
+ observeShadowGuest(m, kind, "stopped", true)
+ assertShadowGuestParity(t, m, false)
+ tick.Store(int64(13 * time.Minute))
+ observeShadowGuest(m, kind, "running", true)
+ m.SetOperatorIntentContextResolver(nil)
+ tick.Store(int64(14 * time.Minute))
+ observeShadowGuest(m, kind, "stopped", true)
+ assertShadowGuestParity(t, m, false)
+ tick.Store(int64(24*time.Minute - time.Second))
+ observeShadowGuest(m, kind, "stopped", true)
+ assertShadowGuestParity(t, m, false)
+ tick.Store(int64(24 * time.Minute))
+ observeShadowGuest(m, kind, "stopped", true)
+ assertShadowGuestParity(t, m, true)
+ m.mu.RLock()
+ incident, ok := m.shadow.state.Incident(shadowGuestID, canonicalPoweredStateSpecID(shadowGuestID))
+ m.mu.RUnlock()
+ if !ok || incident.State != reducer.StateFiring || !incident.Acknowledged {
+ t.Fatalf("re-fire lost existing acknowledgement semantics: %+v", incident)
+ }
+ })
+ }
+ }
+}