Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Keep spent collection budgets out of backup alert reads

pulse-triage[bot]289198a61eb5
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md
index 99409eaced..0423e60df8 100644
--- a/docs/release-control/v6/internal/subsystems/monitoring.md
+++ b/docs/release-control/v6/internal/subsystems/monitoring.md
@@ -33,8 +33,11 @@ Explicit request cancellation withdraws the enumeration. Dropping an expired
child deadline for local publication does not drop the connection owner:
runtime shutdown/replacement, client replacement and retirement still prevent
late publication. Unexpired publication retains its request cancellation signal.
-Independent connections, local-disk identity, permissions,
-alerts, guest operations and provider budgets are unchanged. Every node's
+A spent collection budget is not a recovery-store read failure: it does not
+start an already-expired backup-alert read, invent an evaluation-outage warning
+or clear existing wanted alerts. The next ordinary evaluation remains unchanged.
+Independent connections, local-disk identity, permissions, guest operations and
+provider budgets are unchanged. Every node's
ordinary storage reads remain; an accepted shared view is not substituted for
another node's access evidence.
diff --git a/docs/release-control/v6/internal/subsystems/storage-recovery.md b/docs/release-control/v6/internal/subsystems/storage-recovery.md
index 03b2747eee..5b83946a55 100644
--- a/docs/release-control/v6/internal/subsystems/storage-recovery.md
+++ b/docs/release-control/v6/internal/subsystems/storage-recovery.md
@@ -37,8 +37,11 @@ Explicit request cancellation withdraws the enumeration. Dropping an expired
child deadline for local publication does not drop the connection owner:
runtime shutdown/replacement, client replacement and retirement still prevent
late publication. Unexpired publication retains its request cancellation signal.
-Independent connections, local-disk identity, permissions,
-alerts, guest operations and provider budgets are unchanged. Every node's
+A spent collection budget is not a recovery-store read failure: it does not
+start an already-expired backup-alert read, invent an evaluation-outage warning
+or clear existing wanted alerts. The next ordinary evaluation remains unchanged.
+Independent connections, local-disk identity, permissions, guest operations and
+provider budgets are unchanged. Every node's
ordinary storage reads remain; an accepted shared view is not substituted for
another node's access evidence.
diff --git a/internal/monitoring/monitor_backups.go b/internal/monitoring/monitor_backups.go
index 853f70a272..b11d70479d 100644
--- a/internal/monitoring/monitor_backups.go
+++ b/internal/monitoring/monitor_backups.go
@@ -770,7 +770,12 @@ func (m *Monitor) pollStorageBackupsWithNodes(ctx context.Context, instanceName
// Sync backup times to VMs/Containers and republish them to canonical resources.
m.syncGuestBackupTimesAndResourceStore()
- m.checkBackupAlerts(ctx)
+ // Optional collection expiry is not a failed recovery-store read. Keep
+ // existing alerts for the next ordinary evaluation rather than issuing an
+ // already-expired read that would manufacture an evaluation-outage alert.
+ if ctx.Err() == nil {
+ m.checkBackupAlerts(ctx)
+ }
// Only a complete accepted enumeration can clear an earlier access
// warning. An unread scope has not demonstrated permission recovery.
diff --git a/internal/monitoring/pve_storage_budget_test.go b/internal/monitoring/pve_storage_budget_test.go
index fc1d3771a9..28fee2134d 100644
--- a/internal/monitoring/pve_storage_budget_test.go
+++ b/internal/monitoring/pve_storage_budget_test.go
@@ -10,6 +10,7 @@ import (
"testing"
"time"
+ "github.com/rcourtman/pulse-go-rewrite/internal/alerts"
"github.com/rcourtman/pulse-go-rewrite/internal/config"
"github.com/rcourtman/pulse-go-rewrite/internal/models"
"github.com/rcourtman/pulse-go-rewrite/internal/operationaltrust"
@@ -323,3 +324,40 @@ func TestPVEStorageBudgetUnreadNodeKeepsUnscopedPBSConfirmation(t *testing.T) {
})
}
}
+
+// The collection budget cannot stand in for evidence that the recovery reader
+// failed. Preserve wanted alerts (and a real prior evaluation warning) until
+// an ordinary evaluation runs with a live context.
+func TestPVEStorageBudgetDoesNotInventAlertReadFailure(t *testing.T) {
+ for _, priorWarning := range []bool{false, true} {
+ t.Run(fmt.Sprintf("prior-warning=%t", priorWarning), func(t *testing.T) {
+ f, c := newStorageBudgetFixture(t, "qemu")
+ pollStorageBudget(t, f, c, context.Background())
+ evaluator, _ := backupEvaluationMonitor(t)
+ f.m.alertManager = evaluator.alertManager
+ f.m.alertManager.RaiseSystemAlert(alerts.SystemAlertInput{Type: "fixture-wanted", Level: alerts.AlertLevelCritical, Fingerprint: "wanted-condition", Message: "Existing wanted condition"})
+ if priorWarning {
+ f.m.alertManager.RaiseSystemAlert(alerts.SystemAlertInput{Type: alerts.BackupEvaluationAlertType, Level: alerts.AlertLevelWarning, Fingerprint: "prior-observed-read-failure", Message: "Prior observed reader failure"})
+ }
+ before := f.m.alertManager.GetActiveAlerts()
+ c.phase, c.at = "content/node-b/local", c.at.Add(15*time.Minute)
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+ defer cancel()
+ pollStorageBudget(t, f, c, ctx)
+ after := f.m.alertManager.GetActiveAlerts()
+ if len(after) != len(before) {
+ t.Fatalf("collection expiry invented or cleared an alert: before=%+v after=%+v", before, after)
+ }
+ byID := make(map[string]alerts.Alert, len(after))
+ for _, alert := range after {
+ byID[alert.ID] = alert
+ }
+ for _, old := range before {
+ got, ok := byID[old.ID]
+ if !ok || got.Type != old.Type || got.Level != old.Level || got.Message != old.Message || !got.StartTime.Equal(old.StartTime) || !reflect.DeepEqual(got.Metadata, old.Metadata) {
+ t.Errorf("collection budget changed an existing alert condition: before=%+v after=%+v", old, got)
+ }
+ }
+ })
+ }
+}