Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Verify storage gap containment through the notification queue

pulse-triage[bot]6ffc02e5c502
diff --git a/docs/release-control/v6/internal/subsystems/alerts.md b/docs/release-control/v6/internal/subsystems/alerts.md
index 081be8700b..e09fb70937 100644
--- a/docs/release-control/v6/internal/subsystems/alerts.md
+++ b/docs/release-control/v6/internal/subsystems/alerts.md
@@ -3899,3 +3899,10 @@ reconfirmation and empty intent checkpoint. Storage routes in
intent grace. Existing restart, empty-capacity and predictive-capacity controls
remain required. These source controls are not native storage recovery or
installed destination-delivery evidence.
+
+`TestStorageConnectivityGapWebhook` also exercises the production Monitor
+callbacks, ordinary notification queue and guest-local HTTP receiver: the gap
+and first fresh healthy poll send no recovery; confirmed recovery sends one
+resolved receipt with the original occurrence identity, and the queue drains
+without failed/DLQ work. This is synthetic callback/receiver acceptance, not a
+native appliance or external webhook-provider result.
diff --git a/internal/monitoring/monitor_storage_webhook_test.go b/internal/monitoring/monitor_storage_webhook_test.go
index c3d9257adb..1226b24408 100644
--- a/internal/monitoring/monitor_storage_webhook_test.go
+++ b/internal/monitoring/monitor_storage_webhook_test.go
@@ -132,3 +132,104 @@ func TestStorageEmptyRecoveryWebhook(t *testing.T) {
case <-time.After(150 * time.Millisecond):
}
}
+
+// The callback and durable queue must not tell a receiver that an outage has
+// recovered by joining healthy observations across missing status evidence.
+func TestStorageConnectivityGapWebhook(t *testing.T) {
+ type payload struct {
+ Event string `json:"event"`
+ Alerts []alerts.Alert `json:"alerts"`
+ }
+ received := make(chan payload, 8)
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ defer r.Body.Close()
+ var p payload
+ if r.Method != http.MethodPost || json.NewDecoder(r.Body).Decode(&p) != nil {
+ t.Error("invalid webhook request")
+ http.Error(w, "invalid request", http.StatusBadRequest)
+ return
+ }
+ received <- p
+ w.WriteHeader(http.StatusOK)
+ }))
+ defer server.Close()
+ nm := notifications.NewNotificationManagerWithDataDir("", t.TempDir())
+ defer nm.Stop()
+ nm.SetGroupingWindow(0)
+ nm.SetNotifyOnResolve(true)
+ if err := nm.UpdateAllowedPrivateCIDRs("127.0.0.1/32"); err != nil {
+ t.Fatal(err)
+ }
+ nm.AddWebhook(notifications.WebhookConfig{ID: "storage-gap", URL: server.URL, Enabled: true})
+ manager := alerts.NewManagerWithDataDir(t.TempDir())
+ defer manager.Stop()
+ manager.UpdateConfig(alerts.AlertConfig{Enabled: true, ActivationState: alerts.ActivationActive,
+ TimeThresholds: map[string]int{"storage": 0}, StorageDefault: alerts.HysteresisThreshold{Trigger: 80, Clear: 70}})
+ monitor := newPBSHealthAuthorityMonitor(nil)
+ monitor.alertManager, monitor.notificationMgr = manager, nm
+ monitor.wireExternalAlertCallbacks(nil)
+ take := func() payload {
+ t.Helper()
+ select {
+ case p := <-received:
+ return p
+ case <-time.After(5 * time.Second):
+ t.Fatal("missing webhook receipt")
+ return payload{}
+ }
+ }
+ quiet := func() {
+ t.Helper()
+ select {
+ case p := <-received:
+ t.Fatalf("unconfirmed recovery sent a webhook: %+v", p)
+ case <-time.After(150 * time.Millisecond):
+ }
+ }
+ storage := models.Storage{ID: "pbs-gap-backups", Name: "backups", Type: "pbs", Status: "unavailable"}
+ for range 2 {
+ manager.CheckStorage(storage)
+ }
+ fired := take()
+ if fired.Event == "resolved" || len(fired.Alerts) != 1 || fired.Alerts[0].Type != "offline" {
+ t.Fatalf("unexpected firing receipt: %+v", fired)
+ }
+ original := fired.Alerts[0]
+ storage.Status = "available"
+ manager.CheckStorage(storage)
+ storage.Status = "unknown"
+ manager.CheckStorage(storage)
+ storage.Status = "available"
+ manager.CheckStorage(storage)
+ active := manager.GetActiveAlerts()
+ if len(active) != 1 || active[0].ID != original.ID || !active[0].StartTime.Equal(original.StartTime) {
+ t.Fatalf("non-consecutive recovery changed the original outage: %+v", active)
+ }
+ quiet()
+ manager.CheckStorage(storage)
+ recovered := take()
+ if recovered.Event != "resolved" || len(recovered.Alerts) != 1 || recovered.Alerts[0].ID != original.ID || !recovered.Alerts[0].StartTime.Equal(original.StartTime) {
+ t.Fatalf("bad confirmed-recovery receipt: %+v", recovered)
+ }
+ deadline := time.Now().Add(5 * time.Second)
+ for {
+ stats, err := nm.GetQueueStats()
+ if err != nil {
+ t.Fatal(err)
+ }
+ if stats["pending"]+stats["sending"] == 0 {
+ if stats["failed"]+stats["dlq"] != 0 {
+ t.Fatalf("notification delivery failed: %v", stats)
+ }
+ break
+ }
+ if time.Now().After(deadline) {
+ t.Fatalf("notification queue did not drain: %v", stats)
+ }
+ time.Sleep(time.Millisecond)
+ }
+ for range 3 {
+ manager.CheckStorage(storage)
+ }
+ quiet()
+}