Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Revalidate restored backup alerts before startup delivery
rcourtmanbea763bc4c11
diff --git a/internal/alerts/active_cleanup.go b/internal/alerts/active_cleanup.goindex 9d0a19af68..ea78202cf7 100644--- a/internal/alerts/active_cleanup.go+++ b/internal/alerts/active_cleanup.go@@ -341,6 +341,7 @@ func (m *Manager) ClearActiveAlerts() {return}m.activeAlerts = make(map[string]*Alert)+ m.restoredBackupNotifications = nilm.activeAlertAlias = make(map[string]string)m.core.Reset()m.intentPending = make(map[string]IntentPendingState)diff --git a/internal/alerts/active_lifecycle.go b/internal/alerts/active_lifecycle.goindex 3771f98205..c2d150d324 100644--- a/internal/alerts/active_lifecycle.go+++ b/internal/alerts/active_lifecycle.go@@ -663,6 +663,9 @@ func (m *Manager) removeActiveAlertNoLock(alertID string) {}if exists {delete(m.activeAlerts, key)+ if currentAlert != nil {+ delete(m.restoredBackupNotifications, canonicalTrackingKeyForAlert(currentAlert))+ }m.removeActiveRecoveryAlert(currentAlert, key)}diff --git a/internal/alerts/active_persistence.go b/internal/alerts/active_persistence.goindex d5c0c122a1..66c7e35f25 100644--- a/internal/alerts/active_persistence.go+++ b/internal/alerts/active_persistence.go@@ -431,6 +431,7 @@ func (m *Manager) restoreActiveAlertSnapshots(alerts []*Alert, source string, rem.mu.Lock()if replace {m.activeAlerts = make(map[string]*Alert)+ m.restoredBackupNotifications = nilm.activeAlertAlias = make(map[string]string)m.ackState = make(map[string]ackRecord)m.ackStateByCanonical = make(map[string]ackRecord)@@ -526,7 +527,14 @@ func (m *Manager) restoreActiveAlertSnapshots(alerts []*Alert, source string, rerestoredCount++if alert.Level == AlertLevelCritical {- restoredCritical = append(restoredCritical, alert.Clone())+ if alert.Type == "backup-age" {+ if m.restoredBackupNotifications == nil {+ m.restoredBackupNotifications = make(map[string]time.Time)+ }+ m.restoredBackupNotifications[canonicalTrackingKeyForAlert(alert)] = alert.StartTime+ } else {+ restoredCritical = append(restoredCritical, alert.Clone())+ }}}if err := m.loadIntentPendingNoLock(); err != nil {diff --git a/internal/alerts/canonical_lifecycle.go b/internal/alerts/canonical_lifecycle.goindex b6f8386273..f4a7c614f4 100644--- a/internal/alerts/canonical_lifecycle.go+++ b/internal/alerts/canonical_lifecycle.go@@ -823,8 +823,14 @@ func (m *Manager) evaluateCanonicalStatefulAlert(params canonicalStatefulAlertPareturn result, true}- if primary == reducer.EventSeverityChanged && params.NotifyOnSeverityChange {- if params.AddToHistoryOnSeverityChange {+ // Restored backup incidents do not dispatch on a startup timer. Their+ // first valid overdue evaluation must resume delivery even when the+ // reducer keeps the same occurrence and severity.+ backupRevalidated := m.revalidateBackupNotificationNoLock(alert)+ severityChanged := primary == reducer.EventSeverityChanged && params.NotifyOnSeverityChange+ if severityChanged ||+ (backupRevalidated && m.shouldNotifyAfterCooldown(alert)) {+ if severityChanged && params.AddToHistoryOnSeverityChange {m.historyManager.AddAlertTransition(*alert)}if params.RateLimit && !m.checkRateLimit(trackingKey) {diff --git a/internal/alerts/canonical_stateful_test.go b/internal/alerts/canonical_stateful_test.goindex d76ced7073..d5e7118854 100644--- a/internal/alerts/canonical_stateful_test.go+++ b/internal/alerts/canonical_stateful_test.go@@ -1,12 +1,14 @@package alertsimport (+ "fmt""testing""time""github.com/rcourtman/pulse-go-rewrite/internal/alerts/eventlog"alertspecs "github.com/rcourtman/pulse-go-rewrite/internal/alerts/specs""github.com/rcourtman/pulse-go-rewrite/internal/models"+ "github.com/rcourtman/pulse-go-rewrite/internal/recovery""github.com/rcourtman/pulse-go-rewrite/internal/storagehealth""github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources""github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"@@ -448,3 +450,312 @@ func TestCheckDiskHealthWearoutFlappingDoesNotRepeatNotifications(t *testing.T)t.Fatalf("genuine recurrence produced %d total dispatches, want two", got)}}++// Replays obsolete PVE backup identities and timestamps. The live+// PBS inventory is deliberately evaluated after the production restore timer,+// matching a restart whose first successful backup check is still pending.+func TestRestoredBackupRequiresFreshEvaluation(t *testing.T) {+ for _, sqlite := range []bool{false, true} {+ name := "recovery-mirror"+ if sqlite {+ name = "sqlite-authority"+ }+ t.Run(name, func(t *testing.T) {+ dir := t.TempDir()+ now := time.Now().UTC()+ lastSeen := now.Add(-4 * 24 * time.Hour)+ var fixtures []*Alert+ for i, subject := range []string{"res-vm-legacy-1", "res-vm-legacy-2"} {+ resourceID := "backup-subject:" + subject+ specID := resourceID + "-backup-age"+ backupTime := now.Add(-94*24*time.Hour + time.Duration(i)*15*time.Minute)+ fixtures = append(fixtures, &Alert{+ ID: resourceID + "::" + specID, Type: "backup-age", Level: AlertLevelCritical,+ ResourceID: resourceID, ResourceName: fmt.Sprintf("backup-server-%d backup", i+1),+ CanonicalSpecID: specID, CanonicalKind: "posture-threshold",+ CanonicalState: resourceID + "::" + specID,+ Message: "Backup via proxmox-pve is 90.0 days old", Value: 90, Threshold: 84,+ StartTime: backupTime.Add(84 * 24 * time.Hour), LastSeen: lastSeen,+ Metadata: map[string]interface{}{+ "source": "PVE", "providers": []string{"proxmox-pve"},+ "lastBackupTime": backupTime, "ageDays": float64(90), "thresholdDays": float64(84),+ "guestVmid": 0, "guestInstance": "", "guestNode": "", "guestType": "",+ "guestName": fmt.Sprintf("backup-server-%d", i+1), "orphaned": false,+ "resourceType": "vm", "rollupId": "res:" + subject[4:],+ "canonicalAlertKind": "posture-threshold", "canonicalSpecID": specID,+ },+ })+ }+ writeActiveRecoveryFixture(t, dir, fixtures)+ seed := NewManagerWithDataDir(dir)+ if sqlite {+ seed.EnableEventLog()+ if !seed.activeStateAuthoritative.Load() {+ t.Fatal("failed to establish SQLite authority")+ }+ }+ if err := seed.SaveActiveAlerts(); err != nil {+ t.Fatal(err)+ }+ seed.Stop()++ m := NewManagerWithDataDir(dir)+ t.Cleanup(m.Stop)+ if sqlite {+ m.EnableEventLog()+ }+ m.UpdateConfig(AlertConfig{Enabled: true, ActivationState: ActivationActive,+ TimeThresholds: map[string]int{},+ BackupDefaults: BackupAlertConfig{Enabled: true, WarningDays: 70, CriticalDays: 84}})+ delivered := make(chan *Alert, 8)+ m.SetAlertCallback(func(a *Alert) { delivered <- a.Clone() })+ for _, a := range fixtures {+ if !testHasActiveAlert(t, m, a.ID) {+ t.Fatalf("the historical alert was not restored: %s", a.ID)+ }+ }++ // Exercise the real ten-second startup timer without changing source.+ select {+ case a := <-delivered:+ t.Errorf("restored stale backup warning dispatched before any fresh backup evaluation: id=%s lastSeen=%s lastBackupTime=%v",+ a.ID, a.LastSeen.Format(time.RFC3339), a.Metadata["lastBackupTime"])+ case <-time.After(12 * time.Second):+ }++ // The current PBS rollups use different resource identities.+ var rollups []recovery.ProtectionRollup+ guests := make(map[string]GuestLookup)+ byVMID := make(map[string][]GuestLookup)+ for i, resourceID := range []string{"vm-current-1", "vm-current-2"} {+ vmid := 101 + i+ node := fmt.Sprintf("pve%02d", i+1)+ guest := GuestLookup{Name: fmt.Sprintf("backup-server-%d", i+1), Instance: "pve01", Node: node,+ VMID: vmid, Type: "qemu", ResourceID: resourceID}+ guests[BuildGuestKey("pve01", node, vmid)] = guest+ byVMID[fmt.Sprint(vmid)] = []GuestLookup{guest}+ success := now.Add(-2*24*time.Hour + time.Duration(i)*15*time.Minute)+ rollups = append(rollups, recovery.ProtectionRollup{RollupID: "res:" + resourceID,+ SubjectRef: &recovery.ExternalRef{Type: "proxmox-vm", Namespace: "pve01", Name: guest.Name,+ ID: fmt.Sprintf("pve01:%s:%d", node, vmid), Class: node},+ LastSuccessAt: &success, LastOutcome: recovery.OutcomeSuccess,+ Providers: []recovery.Provider{recovery.ProviderProxmoxPBS}})+ }+ m.CheckBackups(rollups, guests, byVMID)+ for _, a := range fixtures {+ if testHasActiveAlert(t, m, a.ID) {+ t.Errorf("fresh current PBS inventory did not clear historical warning: %s", a.ID)+ }+ }+ select {+ case a := <-delivered:+ t.Fatalf("cleared historical backup emitted a notification: %s", a.ID)+ case <-time.After(100 * time.Millisecond):+ }+ })+ }+}++func startupBackupConfig() AlertConfig {+ return AlertConfig{Enabled: true, ActivationState: ActivationActive,+ TimeThresholds: map[string]int{},+ BackupDefaults: BackupAlertConfig{Enabled: true, WarningDays: 70, CriticalDays: 84}}+}++// Seed through normal backup evaluation so revalidation must retain the same+// canonical identity, acknowledgement, occurrence, and notification receipt.+func restartOverdueBackup(t *testing.T, sqlite bool, edit func(*Alert)) (*Manager, []recovery.ProtectionRollup, *Alert) {+ t.Helper()+ dir := t.TempDir()+ success := time.Now().Add(-94 * 24 * time.Hour)+ rollups := []recovery.ProtectionRollup{{RollupID: "res:vm-startup-test",+ SubjectRef: &recovery.ExternalRef{Type: "proxmox-vm", Namespace: "test", ID: "test:node:101"},+ LastSuccessAt: &success, LastOutcome: recovery.OutcomeSuccess,+ Providers: []recovery.Provider{recovery.ProviderProxmoxPBS}}}+ seed := NewManagerWithDataDir(dir)+ if sqlite {+ seed.EnableEventLog()+ }+ seed.UpdateConfig(startupBackupConfig())+ seed.CheckBackups(rollups, nil, nil)+ active := seed.GetActiveAlerts()+ if len(active) != 1 || active[0].Level != AlertLevelCritical {+ seed.Stop()+ t.Fatalf("expected one overdue backup incident, got %+v", active)+ }+ if edit != nil {+ seed.mu.Lock()+ current, _ := seed.getActiveAlertNoLock(active[0].ID)+ edit(current)+ seed.setActiveAlertNoLock(current.ID, current)+ seed.mu.Unlock()+ }+ if err := seed.SaveActiveAlerts(); err != nil {+ seed.Stop()+ t.Fatal(err)+ }+ seed.Stop()+ m := NewManagerWithDataDir(dir)+ t.Cleanup(m.Stop)+ if sqlite {+ m.EnableEventLog()+ if !m.activeStateAuthoritative.Load() {+ t.Fatal("failed to restore SQLite active authority")+ }+ }+ m.UpdateConfig(startupBackupConfig())+ restored := m.GetActiveAlerts()+ if len(restored) != 1 {+ t.Fatalf("expected retained overdue backup incident, got %+v", restored)+ }+ return m, rollups, restored[0].Clone()+}++func TestRestoredBackupRevalidationDeliversOverdueOccurrence(t *testing.T) {+ for _, sqlite := range []bool{false, true} {+ t.Run(fmt.Sprintf("sqlite=%t", sqlite), func(t *testing.T) {+ m, rollups, restored := restartOverdueBackup(t, sqlite, nil)+ delivered := make(chan *Alert, 4)+ m.SetAlertCallback(func(a *Alert) { delivered <- a })+ if !m.ShouldSuppressNotification(restored) {+ t.Fatal("restored snapshot can notify before a backup evaluation")+ }+ m.CheckBackups(rollups, nil, nil)+ select {+ case a := <-delivered:+ if a.ID != restored.ID || !a.StartTime.Equal(restored.StartTime) || !a.LastSeen.After(restored.LastSeen) {+ t.Fatalf("revalidation did not preserve the occurrence with fresh evidence: %+v", a)+ }+ case <-time.After(2 * time.Second):+ t.Fatal("genuinely overdue backup did not notify after revalidation")+ }+ m.CheckBackups(rollups, nil, nil)+ select {+ case a := <-delivered:+ t.Fatalf("unchanged overdue occurrence notified twice: %s", a.ID)+ case <-time.After(100 * time.Millisecond):+ }+ if active := m.GetActiveAlerts(); len(active) != 1 || active[0].LastNotified == nil {+ t.Fatalf("revalidated notification receipt was not retained: %+v", active)+ }+ })+ }+}++func TestRestoredBackupWithoutValidEvidenceRetainsIncidentAndEscalation(t *testing.T) {+ m, rollups, restored := restartOverdueBackup(t, true, nil)+ delivered := make(chan *Alert, 4)+ m.SetAlertCallback(func(a *Alert) { delivered <- a })+ m.SetEscalateCallback(func(a *Alert, _ int) { delivered <- a })+ cfg := startupBackupConfig()+ cfg.Schedule.Escalation = EscalationConfig{Enabled: true, Levels: []EscalationLevel{{After: 5}}}+ m.UpdateConfig(cfg)+ spec, err := buildCanonicalPostureThresholdSpec(restored.CanonicalSpecID, restored.ResourceID,+ restored.ResourceName, "vm", "backup-age-days", 70, 84, "", 0, 0, false)+ if err != nil {+ t.Fatal(err)+ }+ if _, valid := m.evaluateCanonicalStatefulAlert(canonicalStatefulAlertParams{+ Spec: spec, AlertID: restored.ID, AlertType: "backup-age",+ Evidence: alertspecs.AlertEvidence{ObservedAt: time.Now()},+ }); valid {+ t.Fatal("invalid backup evidence was accepted")+ }+ m.mu.Lock()+ if m.dispatchAlert(restored, false) {+ m.mu.Unlock()+ t.Fatal("restored snapshot dispatched without valid evidence")+ }+ m.mu.Unlock()+ m.checkEscalations()+ if _, _, eligible := m.PrepareEscalationNotification(restored, 1); eligible {+ t.Fatal("restored backup snapshot bypassed revalidation through escalation")+ }+ diagnosis, found := m.DiagnoseAlertDelivery(restored.ID)+ if !found || diagnosis.Reason != alertDeliveryReasonBackupRevalidation {+ t.Fatalf("delivery diagnosis did not explain the startup hold: %+v", diagnosis)+ }+ active := m.GetActiveAlerts()+ if len(active) != 1 || !active[0].StartTime.Equal(restored.StartTime) || active[0].LastEscalation != 0 || active[0].LastNotified != nil {+ t.Fatalf("missing evidence mutated or resolved the retained incident: %+v", active)+ }+ select {+ case <-delivered:+ t.Fatal("missing evidence sent a firing or escalation notification")+ case <-time.After(100 * time.Millisecond):+ }+ // Once collection succeeds, neither the overdue notification nor the+ // previously held escalation level may be permanently lost.+ m.CheckBackups(rollups, nil, nil)+ select {+ case <-delivered:+ case <-time.After(2 * time.Second):+ t.Fatal("overdue notification was lost after collection succeeded")+ }+ m.checkEscalations()+ if _, _, eligible := m.PrepareEscalationNotification(restored, 1); !eligible {+ t.Fatal("freshly revalidated backup cannot escalate")+ }+}++func TestRestoredBackupRevalidationRespectsAcknowledgementAndCooldown(t *testing.T) {+ for _, policy := range []string{"acknowledged", "cooldown", "first-notification-only", "snoozed"} {+ t.Run(policy, func(t *testing.T) {+ m, rollups, restored := restartOverdueBackup(t, false, func(a *Alert) {+ if policy == "acknowledged" {+ a.Acknowledged = true+ a.AckUser = "operator"+ } else if policy != "snoozed" {+ now := time.Now()+ a.LastNotified = &now+ }+ })+ if policy == "snoozed" {+ if err := m.SnoozeAlert(restored.ID, "operator", time.Now().Add(time.Hour)); err != nil {+ t.Fatal(err)+ }+ }+ cfg := startupBackupConfig()+ if policy == "cooldown" {+ cfg.Schedule.Cooldown = 60+ }+ m.UpdateConfig(cfg)+ delivered := make(chan *Alert, 2)+ m.SetAlertCallback(func(a *Alert) { delivered <- a })+ m.CheckBackups(rollups, nil, nil)+ select {+ case a := <-delivered:+ t.Fatalf("revalidation bypassed %s: %+v", policy, a)+ case <-time.After(100 * time.Millisecond):+ }+ })+ }+}++func TestRestoredBackupRevalidationPreservesQuietHoursReplay(t *testing.T) {+ m, rollups, restored := restartOverdueBackup(t, false, nil)+ cfg := startupBackupConfig()+ cfg.Schedule.QuietHours = QuietHours{Enabled: true, Start: "00:00", End: "23:59", Timezone: "UTC",+ Days: map[string]bool{"monday": true, "tuesday": true, "wednesday": true, "thursday": true,+ "friday": true, "saturday": true, "sunday": true},+ Suppress: QuietHoursSuppression{Storage: true}}+ m.UpdateConfig(cfg)+ if !m.ShouldSuppressNotification(restored) || hasQuietHoursNotificationReplay(restored) {+ t.Fatal("startup hold queued stale backup evidence for quiet-hours replay")+ }+ // Fresh evidence downgrades this retained occurrence to a warning, which+ // normal quiet-hours policy defers. Critical backup-age alerts bypass it.+ warningBackupTime := time.Now().Add(-75 * 24 * time.Hour)+ rollups[0].LastSuccessAt = &warningBackupTime+ delivered := make(chan *Alert, 2)+ m.SetAlertCallback(func(a *Alert) { delivered <- a })+ m.CheckBackups(rollups, nil, nil)+ select {+ case a := <-delivered:+ if a.Level != AlertLevelWarning || !hasQuietHoursNotificationReplay(a) {+ t.Fatal("revalidated overdue backup lost normal quiet-hours replay metadata")+ }+ case <-time.After(2 * time.Second):+ t.Fatal("revalidated backup did not enter normal quiet-hours delivery")+ }+}diff --git a/internal/alerts/escalation.go b/internal/alerts/escalation.goindex 5e747492ed..8e8bbcc59b 100644--- a/internal/alerts/escalation.go+++ b/internal/alerts/escalation.go@@ -49,6 +49,9 @@ func (m *Manager) checkEscalations() {if alert == nil || alert.Acknowledged || isMonitorOnlyAlert(alert) || isSupportedInfrastructureSymptom(alert) {continue}+ if m.awaitsBackupRevalidationNoLock(alert) {+ continue+ }if _, snoozed := alertSnoozeUntil(alert, now); snoozed {continue}@@ -133,6 +136,9 @@ func (m *Manager) PrepareEscalationNotification(snapshot *Alert, level int) (*Alif !ok || active == nil || !active.StartTime.Equal(snapshot.StartTime) || active.Acknowledged || isMonitorOnlyAlert(active) || isSupportedInfrastructureSymptom(active) {return nil, EscalationLevel{}, false}+ if m.awaitsBackupRevalidationNoLock(active) {+ return nil, EscalationLevel{}, false+ }if _, snoozed := alertSnoozeUntil(active, m.policyNow()); snoozed {return nil, EscalationLevel{}, false}diff --git a/internal/alerts/manager.go b/internal/alerts/manager.goindex 79aa6dc464..ef7021c080 100644--- a/internal/alerts/manager.go+++ b/internal/alerts/manager.go@@ -133,6 +133,9 @@ type Manager struct {activeRecoveryReadable atomic.BoolactiveRecoveryWriteBlock atomic.BoolrestoredAlertEpoch atomic.Uint64+ // Restored backup occurrences stay visible, but cannot notify until a+ // successful backup evaluation confirms they are still overdue. Under mu.+ restoredBackupNotifications map[string]time.Time// Shadow-mode reducer feed (Phase 1 capstone). Nil until// EnableShadowFeed; all access is under m.mu.diff --git a/internal/alerts/notification_policy.go b/internal/alerts/notification_policy.goindex 2fc298450e..ddb9032c64 100644--- a/internal/alerts/notification_policy.go+++ b/internal/alerts/notification_policy.go@@ -172,11 +172,39 @@ func (m *Manager) expireSuppressionNoLock(trackingKey string, now time.Time) booreturn true}+const alertDeliveryReasonBackupRevalidation = "awaiting_backup_evaluation"++// Caller holds mu. The occurrence check prevents a deferred startup snapshot+// from suppressing a later alert that happens to reuse the same identity.+func (m *Manager) awaitsBackupRevalidationNoLock(alert *Alert) bool {+ if alert == nil || alert.Type != "backup-age" {+ return false+ }+ startedAt, pending := m.restoredBackupNotifications[canonicalTrackingKeyForAlert(alert)]+ return pending && startedAt.Equal(alert.StartTime)+}++// A valid, matched canonical evaluation releases the startup delivery hold.+// Recovery/removal retires the hold through the active lifecycle owner instead.+func (m *Manager) revalidateBackupNotificationNoLock(alert *Alert) bool {+ if !m.awaitsBackupRevalidationNoLock(alert) {+ return false+ }+ delete(m.restoredBackupNotifications, canonicalTrackingKeyForAlert(alert))+ return true+}+func (m *Manager) dispatchAlert(alert *Alert, async bool) bool {callbacks := m.getAlertCallbacks()if len(callbacks) == 0 || alert == nil {return false}+ if m.awaitsBackupRevalidationNoLock(alert) {+ m.recordAlertEvent(eventlog.TypeNotificationSuppressed, alert, "",+ alertDeliveryReasonBackupRevalidation,+ "Restored backup alert is awaiting a successful backup evaluation.", nil)+ return false+ }// Don't dispatch notifications for acknowledged alertsif alert.Acknowledged {@@ -640,6 +668,9 @@ func (m *Manager) ShouldSuppressNotification(alert *Alert) bool {m.mu.RLock()defer m.mu.RUnlock()+ if m.awaitsBackupRevalidationNoLock(alert) {+ return true+ }if _, snoozed := alertSnoozeUntil(alert, m.policyNow()); snoozed {clearQuietHoursNotificationReplay(alert)return true@@ -858,6 +889,8 @@ func (m *Manager) diagnoseActiveAlertLocked(alert *Alert) AlertDeliveryDiagnosisdiagnosis.RecentAlertsInHour = m.rateLimitCountLocked(trackingKey, now)switch {+ case m.awaitsBackupRevalidationNoLock(alert):+ diagnosis.setSuppressed(alertDeliveryReasonBackupRevalidation, "Restored backup alert is awaiting a successful backup evaluation.")case alert.Acknowledged:diagnosis.setSuppressed(AlertDeliveryReasonAcknowledged, "Alert is acknowledged, so firing notifications are suppressed.")case isSupportedInfrastructureSymptom(alert):