Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Keep spent collection budgets out of backup alert reads
pulse-triage[bot]289198a61eb5
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.mdindex 99409eaced..0423e60df8 100644--- a/docs/release-control/v6/internal/subsystems/monitoring.md+++ b/docs/release-control/v6/internal/subsystems/monitoring.md@@ -33,8 +33,11 @@ Explicit request cancellation withdraws the enumeration. Dropping an expiredchild deadline for local publication does not drop the connection owner:runtime shutdown/replacement, client replacement and retirement still preventlate publication. Unexpired publication retains its request cancellation signal.-Independent connections, local-disk identity, permissions,-alerts, guest operations and provider budgets are unchanged. Every node's+A spent collection budget is not a recovery-store read failure: it does not+start an already-expired backup-alert read, invent an evaluation-outage warning+or clear existing wanted alerts. The next ordinary evaluation remains unchanged.+Independent connections, local-disk identity, permissions, guest operations and+provider budgets are unchanged. Every node'sordinary storage reads remain; an accepted shared view is not substituted foranother node's access evidence.diff --git a/docs/release-control/v6/internal/subsystems/storage-recovery.md b/docs/release-control/v6/internal/subsystems/storage-recovery.mdindex 03b2747eee..5b83946a55 100644--- a/docs/release-control/v6/internal/subsystems/storage-recovery.md+++ b/docs/release-control/v6/internal/subsystems/storage-recovery.md@@ -37,8 +37,11 @@ Explicit request cancellation withdraws the enumeration. Dropping an expiredchild deadline for local publication does not drop the connection owner:runtime shutdown/replacement, client replacement and retirement still preventlate publication. Unexpired publication retains its request cancellation signal.-Independent connections, local-disk identity, permissions,-alerts, guest operations and provider budgets are unchanged. Every node's+A spent collection budget is not a recovery-store read failure: it does not+start an already-expired backup-alert read, invent an evaluation-outage warning+or clear existing wanted alerts. The next ordinary evaluation remains unchanged.+Independent connections, local-disk identity, permissions, guest operations and+provider budgets are unchanged. Every node'sordinary storage reads remain; an accepted shared view is not substituted foranother node's access evidence.diff --git a/internal/monitoring/monitor_backups.go b/internal/monitoring/monitor_backups.goindex 853f70a272..b11d70479d 100644--- a/internal/monitoring/monitor_backups.go+++ b/internal/monitoring/monitor_backups.go@@ -770,7 +770,12 @@ func (m *Monitor) pollStorageBackupsWithNodes(ctx context.Context, instanceName// Sync backup times to VMs/Containers and republish them to canonical resources.m.syncGuestBackupTimesAndResourceStore()- m.checkBackupAlerts(ctx)+ // Optional collection expiry is not a failed recovery-store read. Keep+ // existing alerts for the next ordinary evaluation rather than issuing an+ // already-expired read that would manufacture an evaluation-outage alert.+ if ctx.Err() == nil {+ m.checkBackupAlerts(ctx)+ }// Only a complete accepted enumeration can clear an earlier access// warning. An unread scope has not demonstrated permission recovery.diff --git a/internal/monitoring/pve_storage_budget_test.go b/internal/monitoring/pve_storage_budget_test.goindex fc1d3771a9..28fee2134d 100644--- a/internal/monitoring/pve_storage_budget_test.go+++ b/internal/monitoring/pve_storage_budget_test.go@@ -10,6 +10,7 @@ import ("testing""time"+ "github.com/rcourtman/pulse-go-rewrite/internal/alerts""github.com/rcourtman/pulse-go-rewrite/internal/config""github.com/rcourtman/pulse-go-rewrite/internal/models""github.com/rcourtman/pulse-go-rewrite/internal/operationaltrust"@@ -323,3 +324,40 @@ func TestPVEStorageBudgetUnreadNodeKeepsUnscopedPBSConfirmation(t *testing.T) {})}}++// The collection budget cannot stand in for evidence that the recovery reader+// failed. Preserve wanted alerts (and a real prior evaluation warning) until+// an ordinary evaluation runs with a live context.+func TestPVEStorageBudgetDoesNotInventAlertReadFailure(t *testing.T) {+ for _, priorWarning := range []bool{false, true} {+ t.Run(fmt.Sprintf("prior-warning=%t", priorWarning), func(t *testing.T) {+ f, c := newStorageBudgetFixture(t, "qemu")+ pollStorageBudget(t, f, c, context.Background())+ evaluator, _ := backupEvaluationMonitor(t)+ f.m.alertManager = evaluator.alertManager+ f.m.alertManager.RaiseSystemAlert(alerts.SystemAlertInput{Type: "fixture-wanted", Level: alerts.AlertLevelCritical, Fingerprint: "wanted-condition", Message: "Existing wanted condition"})+ if priorWarning {+ f.m.alertManager.RaiseSystemAlert(alerts.SystemAlertInput{Type: alerts.BackupEvaluationAlertType, Level: alerts.AlertLevelWarning, Fingerprint: "prior-observed-read-failure", Message: "Prior observed reader failure"})+ }+ before := f.m.alertManager.GetActiveAlerts()+ c.phase, c.at = "content/node-b/local", c.at.Add(15*time.Minute)+ ctx, cancel := context.WithTimeout(context.Background(), time.Second)+ defer cancel()+ pollStorageBudget(t, f, c, ctx)+ after := f.m.alertManager.GetActiveAlerts()+ if len(after) != len(before) {+ t.Fatalf("collection expiry invented or cleared an alert: before=%+v after=%+v", before, after)+ }+ byID := make(map[string]alerts.Alert, len(after))+ for _, alert := range after {+ byID[alert.ID] = alert+ }+ for _, old := range before {+ got, ok := byID[old.ID]+ if !ok || got.Type != old.Type || got.Level != old.Level || got.Message != old.Message || !got.StartTime.Equal(old.StartTime) || !reflect.DeepEqual(got.Metadata, old.Metadata) {+ t.Errorf("collection budget changed an existing alert condition: before=%+v after=%+v", old, got)+ }+ }+ })+ }+}