Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Check PBS expiry alerts by behaviour rather than context spelling
pulse-triage[bot]b4ac15a3a569
diff --git a/internal/monitoring/canonical_guardrails_test.go b/internal/monitoring/canonical_guardrails_test.goindex 3176a447f4..64693a1ef0 100644--- a/internal/monitoring/canonical_guardrails_test.go+++ b/internal/monitoring/canonical_guardrails_test.go@@ -1361,7 +1361,6 @@ func TestBackupOrphanDetectionUsesCanonicalInventoryReadinessScope(t *testing.T)"func (m *Monitor) updatePVEBackupTemplateSubjectsFromClusterResources(instanceName string, resources []proxmox.ClusterResource) {","func (m *Monitor) backupInventoryScopeForAlerts() *alerts.BackupInventoryScope {","m.checkBackupAlerts(ctx)",- "m.checkBackupAlerts(context.Background())",},"recovery_rollups.go": {"m.alertManager.CheckBackupsWithInventory(rollups, guestsByKey, guestsByVMID, m.backupInventoryScopeForAlerts())",@@ -1393,6 +1392,10 @@ func TestBackupOrphanDetectionUsesCanonicalInventoryReadinessScope(t *testing.T)}}}+ // Inventory-readiness scope still owns orphan detection. The old spelling+ // check also required an unbounded PBS alert read; exercise its actual+ // expiry/ordinary-recovery behaviour instead of preserving that choice.+ exercisePBSBudgetExpiredAlertEvaluation(t)}func TestStoragePollingUsesCanonicalPoolMetadataForZFSAttachment(t *testing.T) {diff --git a/internal/monitoring/pbs_backup_budget_test.go b/internal/monitoring/pbs_backup_budget_test.goindex 5298701be9..c3a8a867ce 100644--- a/internal/monitoring/pbs_backup_budget_test.go+++ b/internal/monitoring/pbs_backup_budget_test.go@@ -15,6 +15,7 @@ import ("testing""time"+ "github.com/rcourtman/pulse-go-rewrite/internal/alerts""github.com/rcourtman/pulse-go-rewrite/internal/models""github.com/rcourtman/pulse-go-rewrite/internal/operationaltrust""github.com/rcourtman/pulse-go-rewrite/internal/recovery"@@ -276,3 +277,53 @@ func TestPBSBudgetAcceptedEmptyDoesNotProveUnreadAbsence(t *testing.T) {})}}++func exercisePBSBudgetExpiredAlertEvaluation(t *testing.T) {+ f := newPBSBudgetFixture(t, "vm", "datastore")+ f.at.Store(time.Now().Add(-4 * 24 * time.Hour).Unix())+ f.collect(t, context.Background())+ am := alerts.NewManagerWithDataDir(t.TempDir())+ t.Cleanup(am.Stop)+ cfg := am.GetConfig()+ cfg.Enabled, cfg.BackupDefaults.Enabled = true, true+ cfg.ActivationState = alerts.ActivationActive+ cfg.BackupDefaults.WarningDays, cfg.BackupDefaults.CriticalDays = 1, 2+ am.UpdateConfig(cfg)+ f.m.alertManager = am+ f.m.checkBackupAlerts(context.Background())+ if self, ages := backupEvaluationAlerts(t, f.m); self != nil || ages != 3 {+ t.Fatalf("wanted backup-age controls self=%v ages=%d", self, ages)+ }+ // Simulate an already-visible evaluation warning, not an actual store+ // outage or a new diagnostic request. Collection expiry must not clear it.+ am.RaiseSystemAlert(alerts.SystemAlertInput{Type: alerts.BackupEvaluationAlertType,+ Level: alerts.AlertLevelWarning, Message: "Fixture standing evaluation warning", Fingerprint: "fixture-standing"})+ before := map[string]alerts.Alert{}+ for _, alert := range am.GetActiveAlerts() {+ before[alert.ID] = alert+ }+ f.at.Store(time.Now().Add(-time.Hour).Unix())+ f.phase.Store(1)+ ctx, cancel := context.WithTimeout(context.Background(), time.Second)+ defer cancel()+ f.collect(t, ctx)+ if ctx.Err() != context.DeadlineExceeded {+ t.Fatal("fixture did not spend the collection budget")+ }+ active := am.GetActiveAlerts()+ if len(active) != len(before) {+ t.Error("spent collection budget evaluated/cleared standing alerts")+ }+ for _, alert := range active {+ old, ok := before[alert.ID]+ if !ok || old.Level != alert.Level || old.Message != alert.Message || !old.StartTime.Equal(alert.StartTime) {+ t.Error("spent collection budget replaced a standing alert")+ }+ }+ // A separate ordinary evaluation may use the accepted fresh point. It+ // resolves only its age and the evaluation warning, not unread siblings.+ f.m.checkBackupAlerts(context.Background())+ if self, ages := backupEvaluationAlerts(t, f.m); self != nil || ages != 2 {+ t.Errorf("ordinary evaluation self=%v age alerts=%d; want only the two unread workloads", self, ages)+ }+}