Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Reconcile delivery failures after releasing alert gates
pulse-triage[bot]d500b4f21152
diff --git a/docs/release-control/v6/internal/subsystems/notifications.md b/docs/release-control/v6/internal/subsystems/notifications.mdindex 9081f935bf..8089a5c57b 100644--- a/docs/release-control/v6/internal/subsystems/notifications.md+++ b/docs/release-control/v6/internal/subsystems/notifications.md@@ -15,6 +15,27 @@## Purpose+### Delivery health reconciliation follows the complete send transition++The queue's automatic send worker records the retry/dead-letter outcome and+its per-attempt audit before releasing the firing alert's delivery gate and+announcing the health change. The monitoring callback can therefore inspect+the complete attempt or take the same alert's cancellation gate without+blocking delivery on its own held gate. Configuration failures remain immediate+dead letters; transient failures keep their existing budget and backoff.+Skipped policy delivery remains a cancellation with no invented provider audit.+Explicit operator status/retry/dismiss actions still reconcile immediately+following their own committed transitions.++`queue_delivery_callback_test.go` checks configuration, transient and exhausted+failures through the real delivery processor, exact committed status, attempt+counts/classes, operational links, one callback and re-entrant writer-gate+acquisition. Existing disabled-delivery, restart/resolution/retry and queue+health tests retain policy cancellation and operator action coverage; the+monitor startup test retains saved-destination restoration and its audit check.+This changes callback sequencing only: no SQLite driver, schema/SQL, provider+policy, credential, SSRF boundary or notification payload is changed.+### Delivery verdicts survive session cleanup and interrupted diagnosticsA successful final SMTP DATA reply establishes server acceptance. Plain SMTP,diff --git a/internal/notifications/queue.go b/internal/notifications/queue.goindex 9835972871..e6fca002b2 100644--- a/internal/notifications/queue.go+++ b/internal/notifications/queue.go@@ -848,11 +848,18 @@ func (nq *NotificationQueue) enqueueLocked(notif *QueuedNotification) error {// UpdateStatus updates the status of a queued notification without incrementing attemptsfunc (nq *NotificationQueue) UpdateStatus(id string, status NotificationQueueStatus, errorMsg string) error {+ return nq.updateStatus(id, status, errorMsg, true)+}++// Delivery workers defer reconciliation until their audit is recorded and+// their per-alert gates are released. Explicit operator transitions still+// reconcile immediately after committing the status.+func (nq *NotificationQueue) updateStatus(id string, status NotificationQueueStatus, errorMsg string, notifyHealth bool) error {nq.mu.Lock()err := nq.updateNotificationStatusNoLock(id, status, errorMsg, time.Now())nq.mu.Unlock()- if err == nil && notificationStatusChangesDeliveryHealth(status) {+ if err == nil && notifyHealth && notificationStatusChangesDeliveryHealth(status) {nq.notifyDeliveryHealthChanged()}return err@@ -1218,6 +1225,10 @@ func (nq *NotificationQueue) scanNotification(rows interface{ Scan(...any) error// ScheduleRetry schedules a notification for retry with exponential backoff.// Cancelled and delivered rows are final, even for stale operator requests.func (nq *NotificationQueue) ScheduleRetry(id string, attempt int) error {+ return nq.scheduleRetry(id, attempt, true)+}++func (nq *NotificationQueue) scheduleRetry(id string, attempt int, notifyHealth bool) error {backoff := calculateBackoff(attempt)nextRetry := time.Now().Add(backoff)@@ -1272,7 +1283,9 @@ func (nq *NotificationQueue) ScheduleRetry(id string, attempt int) error {Time("nextRetry", nextRetry).Msg("notification retry scheduled")- nq.notifyDeliveryHealthChanged()+ if notifyHealth {+ nq.notifyDeliveryHealthChanged()+ }return nil}@@ -1881,7 +1894,7 @@ func (nq *NotificationQueue) processNotification(notif *QueuedNotification) {retryable := failureClass.Retryable()if notif.Attempts >= notif.MaxAttempts || !retryable {// Move to DLQ- if dlqErr := nq.MoveToDLQ(notif.ID, errorMsg); dlqErr != nil {+ if dlqErr := nq.updateStatus(notif.ID, QueueStatusDLQ, errorMsg, false); dlqErr != nil {log.Error().Err(dlqErr).Str("component", "notification_queue").@@ -1892,6 +1905,7 @@ func (nq *NotificationQueue) processNotification(notif *QueuedNotification) {Int("maxAttempts", notif.MaxAttempts).Msg("Failed to move notification to DLQ")} else {+ healthChanged = truecompletedAt := time.Now()notif.Status = QueueStatusDLQnotif.CompletedAt = &completedAt@@ -1918,7 +1932,7 @@ func (nq *NotificationQueue) processNotification(notif *QueuedNotification) {}} else {// Schedule retry- if retryErr := nq.ScheduleRetry(notif.ID, notif.Attempts); retryErr != nil {+ if retryErr := nq.scheduleRetry(notif.ID, notif.Attempts, false); retryErr != nil {log.Error().Err(retryErr).Str("component", "notification_queue").@@ -1929,6 +1943,7 @@ func (nq *NotificationQueue) processNotification(notif *QueuedNotification) {Int("maxAttempts", notif.MaxAttempts).Msg("Failed to schedule retry")} else {+ healthChanged = truenotif.Status = QueueStatusPendingnotif.Links = transitionNotificationLinks(notif.Links,diff --git a/internal/notifications/queue_delivery_callback_test.go b/internal/notifications/queue_delivery_callback_test.gonew file mode 100644index 0000000000..0ef60f6cc7--- /dev/null+++ b/internal/notifications/queue_delivery_callback_test.go@@ -0,0 +1,97 @@+package notifications++import (+ "errors"+ "testing"+ "time"++ "github.com/rcourtman/pulse-go-rewrite/internal/alerts"+ "github.com/rcourtman/pulse-go-rewrite/internal/operationaltrust"+)++// A health callback can resolve an incident and acquire its delivery gate.+// Every send-failure transition must release that gate and finish its audit+// before calling the owner, just like policy cancellation already does.+func TestQueuedFailureCallbackSeesCommittedAuditAndReleasedDeliveryGate(t *testing.T) {+ for _, tc := range []struct {+ name string+ err error+ attempts int+ max int+ status NotificationQueueStatus+ failures int+ }{+ {"configuration", errors.New("invalid configuration: synthetic blocked destination"), 0, 3, QueueStatusDLQ, 1},+ {"transient", errors.New("connection refused: synthetic unavailable destination"), 0, 3, QueueStatusPending, 0},+ {"exhausted", errors.New("connection refused: synthetic unavailable destination"), 2, 3, QueueStatusDLQ, 1},+ } {+ t.Run(tc.name, func(t *testing.T) {+ now := time.Now()+ q := newQuietReplayQueue(t, t.TempDir(), &now)+ n := &QueuedNotification{+ ID: "callback-order", Type: "webhook", Status: QueueStatusPending,+ Config: []byte(`{}`), Attempts: tc.attempts, MaxAttempts: tc.max,+ Alerts: []*alerts.Alert{{ID: "incident"}},+ Links: []operationaltrust.NotificationLink{{DestinationID: "ops", OperationalRecordID: "incident",+ TransitionID: "firing", LifecycleState: operationaltrust.OperationalOpen, CauseKey: "cpu"}},+ }+ if err := q.Enqueue(n); err != nil {+ t.Fatal(err)+ }+ q.SetProcessor(func(*QueuedNotification) error { return tc.err })+ callbacks := 0+ q.SetDeliveryHealthChangedCallback(func() {+ callbacks+++ stats, err := q.GetTelemetryStats(time.Time{})+ if err != nil {+ t.Error(err)+ return+ }+ if stats.Attempts != 1 || stats.Deliveries != 0 || stats.Failures != tc.failures {+ t.Errorf("callback saw incomplete audit: %+v", stats)+ }+ if tc.name == "configuration" && stats.FailureClasses.Configuration != 1 ||+ tc.name == "exhausted" && stats.FailureClasses.Connectivity != 1 {+ t.Errorf("callback lost the terminal failure class: %+v", stats)+ }+ q.deliveryGateMu.Lock()+ gate := q.deliveryGates["incident"]+ q.deliveryGateMu.Unlock()+ if gate != nil {+ // Fail rather than hang when this regression is deliberately+ // restored. A callback must be free to take the writer gate.+ if !gate.mu.TryLock() {+ t.Error("callback retained the firing alert delivery gate")+ return+ }+ gate.mu.Unlock()+ }+ release := q.acquireAlertDeliveryGates([]string{"incident"}, true)+ defer release()+ rows, err := q.GetQueueStats()+ if err != nil || rows[string(tc.status)] != 1 {+ t.Errorf("callback did not see committed %s: %v, %v", tc.status, rows, err)+ }+ links, err := q.getNotificationLinks(n.ID)+ if err != nil || len(links) != 1 {+ t.Errorf("callback lost operational links: %v, %v", links, err)+ return+ }+ wantState := operationaltrust.NotificationDeadLetter+ if tc.status == QueueStatusPending {+ wantState = operationaltrust.NotificationRetrying+ }+ if links[0].DeliveryState != wantState || links[0].AttemptedAt == nil {+ t.Errorf("callback saw incomplete attempt linkage: %+v", links[0])+ }+ })+ q.processNotification(n)+ if callbacks != 1 {+ t.Fatalf("health callbacks=%d, want 1", callbacks)+ }+ if n.Status != tc.status || n.Attempts != tc.attempts+1 {+ t.Fatalf("status=%s attempts=%d, want %s/%d", n.Status, n.Attempts, tc.status, tc.attempts+1)+ }+ })+ }+}