Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Check PBS expiry alerts by behaviour rather than context spelling

pulse-triage[bot]b4ac15a3a569
diff --git a/internal/monitoring/canonical_guardrails_test.go b/internal/monitoring/canonical_guardrails_test.go
index 3176a447f4..64693a1ef0 100644
--- a/internal/monitoring/canonical_guardrails_test.go
+++ b/internal/monitoring/canonical_guardrails_test.go
@@ -1361,7 +1361,6 @@ func TestBackupOrphanDetectionUsesCanonicalInventoryReadinessScope(t *testing.T)
"func (m *Monitor) updatePVEBackupTemplateSubjectsFromClusterResources(instanceName string, resources []proxmox.ClusterResource) {",
"func (m *Monitor) backupInventoryScopeForAlerts() *alerts.BackupInventoryScope {",
"m.checkBackupAlerts(ctx)",
- "m.checkBackupAlerts(context.Background())",
},
"recovery_rollups.go": {
"m.alertManager.CheckBackupsWithInventory(rollups, guestsByKey, guestsByVMID, m.backupInventoryScopeForAlerts())",
@@ -1393,6 +1392,10 @@ func TestBackupOrphanDetectionUsesCanonicalInventoryReadinessScope(t *testing.T)
}
}
}
+ // Inventory-readiness scope still owns orphan detection. The old spelling
+ // check also required an unbounded PBS alert read; exercise its actual
+ // expiry/ordinary-recovery behaviour instead of preserving that choice.
+ exercisePBSBudgetExpiredAlertEvaluation(t)
}
func TestStoragePollingUsesCanonicalPoolMetadataForZFSAttachment(t *testing.T) {
diff --git a/internal/monitoring/pbs_backup_budget_test.go b/internal/monitoring/pbs_backup_budget_test.go
index 5298701be9..c3a8a867ce 100644
--- a/internal/monitoring/pbs_backup_budget_test.go
+++ b/internal/monitoring/pbs_backup_budget_test.go
@@ -15,6 +15,7 @@ import (
"testing"
"time"
+ "github.com/rcourtman/pulse-go-rewrite/internal/alerts"
"github.com/rcourtman/pulse-go-rewrite/internal/models"
"github.com/rcourtman/pulse-go-rewrite/internal/operationaltrust"
"github.com/rcourtman/pulse-go-rewrite/internal/recovery"
@@ -276,3 +277,53 @@ func TestPBSBudgetAcceptedEmptyDoesNotProveUnreadAbsence(t *testing.T) {
})
}
}
+
+func exercisePBSBudgetExpiredAlertEvaluation(t *testing.T) {
+ f := newPBSBudgetFixture(t, "vm", "datastore")
+ f.at.Store(time.Now().Add(-4 * 24 * time.Hour).Unix())
+ f.collect(t, context.Background())
+ am := alerts.NewManagerWithDataDir(t.TempDir())
+ t.Cleanup(am.Stop)
+ cfg := am.GetConfig()
+ cfg.Enabled, cfg.BackupDefaults.Enabled = true, true
+ cfg.ActivationState = alerts.ActivationActive
+ cfg.BackupDefaults.WarningDays, cfg.BackupDefaults.CriticalDays = 1, 2
+ am.UpdateConfig(cfg)
+ f.m.alertManager = am
+ f.m.checkBackupAlerts(context.Background())
+ if self, ages := backupEvaluationAlerts(t, f.m); self != nil || ages != 3 {
+ t.Fatalf("wanted backup-age controls self=%v ages=%d", self, ages)
+ }
+ // Simulate an already-visible evaluation warning, not an actual store
+ // outage or a new diagnostic request. Collection expiry must not clear it.
+ am.RaiseSystemAlert(alerts.SystemAlertInput{Type: alerts.BackupEvaluationAlertType,
+ Level: alerts.AlertLevelWarning, Message: "Fixture standing evaluation warning", Fingerprint: "fixture-standing"})
+ before := map[string]alerts.Alert{}
+ for _, alert := range am.GetActiveAlerts() {
+ before[alert.ID] = alert
+ }
+ f.at.Store(time.Now().Add(-time.Hour).Unix())
+ f.phase.Store(1)
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+ defer cancel()
+ f.collect(t, ctx)
+ if ctx.Err() != context.DeadlineExceeded {
+ t.Fatal("fixture did not spend the collection budget")
+ }
+ active := am.GetActiveAlerts()
+ if len(active) != len(before) {
+ t.Error("spent collection budget evaluated/cleared standing alerts")
+ }
+ for _, alert := range active {
+ old, ok := before[alert.ID]
+ if !ok || old.Level != alert.Level || old.Message != alert.Message || !old.StartTime.Equal(alert.StartTime) {
+ t.Error("spent collection budget replaced a standing alert")
+ }
+ }
+ // A separate ordinary evaluation may use the accepted fresh point. It
+ // resolves only its age and the evaluation warning, not unread siblings.
+ f.m.checkBackupAlerts(context.Background())
+ if self, ages := backupEvaluationAlerts(t, f.m); self != nil || ages != 2 {
+ t.Errorf("ordinary evaluation self=%v age alerts=%d; want only the two unread workloads", self, ages)
+ }
+}