Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Verify storage gap containment through the notification queue
pulse-triage[bot]6ffc02e5c502
diff --git a/docs/release-control/v6/internal/subsystems/alerts.md b/docs/release-control/v6/internal/subsystems/alerts.mdindex 081be8700b..e09fb70937 100644--- a/docs/release-control/v6/internal/subsystems/alerts.md+++ b/docs/release-control/v6/internal/subsystems/alerts.md@@ -3899,3 +3899,10 @@ reconfirmation and empty intent checkpoint. Storage routes inintent grace. Existing restart, empty-capacity and predictive-capacity controlsremain required. These source controls are not native storage recovery orinstalled destination-delivery evidence.++`TestStorageConnectivityGapWebhook` also exercises the production Monitor+callbacks, ordinary notification queue and guest-local HTTP receiver: the gap+and first fresh healthy poll send no recovery; confirmed recovery sends one+resolved receipt with the original occurrence identity, and the queue drains+without failed/DLQ work. This is synthetic callback/receiver acceptance, not a+native appliance or external webhook-provider result.diff --git a/internal/monitoring/monitor_storage_webhook_test.go b/internal/monitoring/monitor_storage_webhook_test.goindex c3d9257adb..1226b24408 100644--- a/internal/monitoring/monitor_storage_webhook_test.go+++ b/internal/monitoring/monitor_storage_webhook_test.go@@ -132,3 +132,104 @@ func TestStorageEmptyRecoveryWebhook(t *testing.T) {case <-time.After(150 * time.Millisecond):}}++// The callback and durable queue must not tell a receiver that an outage has+// recovered by joining healthy observations across missing status evidence.+func TestStorageConnectivityGapWebhook(t *testing.T) {+ type payload struct {+ Event string `json:"event"`+ Alerts []alerts.Alert `json:"alerts"`+ }+ received := make(chan payload, 8)+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ defer r.Body.Close()+ var p payload+ if r.Method != http.MethodPost || json.NewDecoder(r.Body).Decode(&p) != nil {+ t.Error("invalid webhook request")+ http.Error(w, "invalid request", http.StatusBadRequest)+ return+ }+ received <- p+ w.WriteHeader(http.StatusOK)+ }))+ defer server.Close()+ nm := notifications.NewNotificationManagerWithDataDir("", t.TempDir())+ defer nm.Stop()+ nm.SetGroupingWindow(0)+ nm.SetNotifyOnResolve(true)+ if err := nm.UpdateAllowedPrivateCIDRs("127.0.0.1/32"); err != nil {+ t.Fatal(err)+ }+ nm.AddWebhook(notifications.WebhookConfig{ID: "storage-gap", URL: server.URL, Enabled: true})+ manager := alerts.NewManagerWithDataDir(t.TempDir())+ defer manager.Stop()+ manager.UpdateConfig(alerts.AlertConfig{Enabled: true, ActivationState: alerts.ActivationActive,+ TimeThresholds: map[string]int{"storage": 0}, StorageDefault: alerts.HysteresisThreshold{Trigger: 80, Clear: 70}})+ monitor := newPBSHealthAuthorityMonitor(nil)+ monitor.alertManager, monitor.notificationMgr = manager, nm+ monitor.wireExternalAlertCallbacks(nil)+ take := func() payload {+ t.Helper()+ select {+ case p := <-received:+ return p+ case <-time.After(5 * time.Second):+ t.Fatal("missing webhook receipt")+ return payload{}+ }+ }+ quiet := func() {+ t.Helper()+ select {+ case p := <-received:+ t.Fatalf("unconfirmed recovery sent a webhook: %+v", p)+ case <-time.After(150 * time.Millisecond):+ }+ }+ storage := models.Storage{ID: "pbs-gap-backups", Name: "backups", Type: "pbs", Status: "unavailable"}+ for range 2 {+ manager.CheckStorage(storage)+ }+ fired := take()+ if fired.Event == "resolved" || len(fired.Alerts) != 1 || fired.Alerts[0].Type != "offline" {+ t.Fatalf("unexpected firing receipt: %+v", fired)+ }+ original := fired.Alerts[0]+ storage.Status = "available"+ manager.CheckStorage(storage)+ storage.Status = "unknown"+ manager.CheckStorage(storage)+ storage.Status = "available"+ manager.CheckStorage(storage)+ active := manager.GetActiveAlerts()+ if len(active) != 1 || active[0].ID != original.ID || !active[0].StartTime.Equal(original.StartTime) {+ t.Fatalf("non-consecutive recovery changed the original outage: %+v", active)+ }+ quiet()+ manager.CheckStorage(storage)+ recovered := take()+ if recovered.Event != "resolved" || len(recovered.Alerts) != 1 || recovered.Alerts[0].ID != original.ID || !recovered.Alerts[0].StartTime.Equal(original.StartTime) {+ t.Fatalf("bad confirmed-recovery receipt: %+v", recovered)+ }+ deadline := time.Now().Add(5 * time.Second)+ for {+ stats, err := nm.GetQueueStats()+ if err != nil {+ t.Fatal(err)+ }+ if stats["pending"]+stats["sending"] == 0 {+ if stats["failed"]+stats["dlq"] != 0 {+ t.Fatalf("notification delivery failed: %v", stats)+ }+ break+ }+ if time.Now().After(deadline) {+ t.Fatalf("notification queue did not drain: %v", stats)+ }+ time.Sleep(time.Millisecond)+ }+ for range 3 {+ manager.CheckStorage(storage)+ }+ quiet()+}