Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Reconcile delivery failures after releasing alert gates

pulse-triage[bot]d500b4f21152
diff --git a/docs/release-control/v6/internal/subsystems/notifications.md b/docs/release-control/v6/internal/subsystems/notifications.md
index 9081f935bf..8089a5c57b 100644
--- a/docs/release-control/v6/internal/subsystems/notifications.md
+++ b/docs/release-control/v6/internal/subsystems/notifications.md
@@ -15,6 +15,27 @@
## Purpose
+### Delivery health reconciliation follows the complete send transition
+
+The queue's automatic send worker records the retry/dead-letter outcome and
+its per-attempt audit before releasing the firing alert's delivery gate and
+announcing the health change. The monitoring callback can therefore inspect
+the complete attempt or take the same alert's cancellation gate without
+blocking delivery on its own held gate. Configuration failures remain immediate
+dead letters; transient failures keep their existing budget and backoff.
+Skipped policy delivery remains a cancellation with no invented provider audit.
+Explicit operator status/retry/dismiss actions still reconcile immediately
+following their own committed transitions.
+
+`queue_delivery_callback_test.go` checks configuration, transient and exhausted
+failures through the real delivery processor, exact committed status, attempt
+counts/classes, operational links, one callback and re-entrant writer-gate
+acquisition. Existing disabled-delivery, restart/resolution/retry and queue
+health tests retain policy cancellation and operator action coverage; the
+monitor startup test retains saved-destination restoration and its audit check.
+This changes callback sequencing only: no SQLite driver, schema/SQL, provider
+policy, credential, SSRF boundary or notification payload is changed.
+
### Delivery verdicts survive session cleanup and interrupted diagnostics
A successful final SMTP DATA reply establishes server acceptance. Plain SMTP,
diff --git a/internal/notifications/queue.go b/internal/notifications/queue.go
index 9835972871..e6fca002b2 100644
--- a/internal/notifications/queue.go
+++ b/internal/notifications/queue.go
@@ -848,11 +848,18 @@ func (nq *NotificationQueue) enqueueLocked(notif *QueuedNotification) error {
// UpdateStatus updates the status of a queued notification without incrementing attempts
func (nq *NotificationQueue) UpdateStatus(id string, status NotificationQueueStatus, errorMsg string) error {
+ return nq.updateStatus(id, status, errorMsg, true)
+}
+
+// Delivery workers defer reconciliation until their audit is recorded and
+// their per-alert gates are released. Explicit operator transitions still
+// reconcile immediately after committing the status.
+func (nq *NotificationQueue) updateStatus(id string, status NotificationQueueStatus, errorMsg string, notifyHealth bool) error {
nq.mu.Lock()
err := nq.updateNotificationStatusNoLock(id, status, errorMsg, time.Now())
nq.mu.Unlock()
- if err == nil && notificationStatusChangesDeliveryHealth(status) {
+ if err == nil && notifyHealth && notificationStatusChangesDeliveryHealth(status) {
nq.notifyDeliveryHealthChanged()
}
return err
@@ -1218,6 +1225,10 @@ func (nq *NotificationQueue) scanNotification(rows interface{ Scan(...any) error
// ScheduleRetry schedules a notification for retry with exponential backoff.
// Cancelled and delivered rows are final, even for stale operator requests.
func (nq *NotificationQueue) ScheduleRetry(id string, attempt int) error {
+ return nq.scheduleRetry(id, attempt, true)
+}
+
+func (nq *NotificationQueue) scheduleRetry(id string, attempt int, notifyHealth bool) error {
backoff := calculateBackoff(attempt)
nextRetry := time.Now().Add(backoff)
@@ -1272,7 +1283,9 @@ func (nq *NotificationQueue) ScheduleRetry(id string, attempt int) error {
Time("nextRetry", nextRetry).
Msg("notification retry scheduled")
- nq.notifyDeliveryHealthChanged()
+ if notifyHealth {
+ nq.notifyDeliveryHealthChanged()
+ }
return nil
}
@@ -1881,7 +1894,7 @@ func (nq *NotificationQueue) processNotification(notif *QueuedNotification) {
retryable := failureClass.Retryable()
if notif.Attempts >= notif.MaxAttempts || !retryable {
// Move to DLQ
- if dlqErr := nq.MoveToDLQ(notif.ID, errorMsg); dlqErr != nil {
+ if dlqErr := nq.updateStatus(notif.ID, QueueStatusDLQ, errorMsg, false); dlqErr != nil {
log.Error().
Err(dlqErr).
Str("component", "notification_queue").
@@ -1892,6 +1905,7 @@ func (nq *NotificationQueue) processNotification(notif *QueuedNotification) {
Int("maxAttempts", notif.MaxAttempts).
Msg("Failed to move notification to DLQ")
} else {
+ healthChanged = true
completedAt := time.Now()
notif.Status = QueueStatusDLQ
notif.CompletedAt = &completedAt
@@ -1918,7 +1932,7 @@ func (nq *NotificationQueue) processNotification(notif *QueuedNotification) {
}
} else {
// Schedule retry
- if retryErr := nq.ScheduleRetry(notif.ID, notif.Attempts); retryErr != nil {
+ if retryErr := nq.scheduleRetry(notif.ID, notif.Attempts, false); retryErr != nil {
log.Error().
Err(retryErr).
Str("component", "notification_queue").
@@ -1929,6 +1943,7 @@ func (nq *NotificationQueue) processNotification(notif *QueuedNotification) {
Int("maxAttempts", notif.MaxAttempts).
Msg("Failed to schedule retry")
} else {
+ healthChanged = true
notif.Status = QueueStatusPending
notif.Links = transitionNotificationLinks(
notif.Links,
diff --git a/internal/notifications/queue_delivery_callback_test.go b/internal/notifications/queue_delivery_callback_test.go
new file mode 100644
index 0000000000..0ef60f6cc7
--- /dev/null
+++ b/internal/notifications/queue_delivery_callback_test.go
@@ -0,0 +1,97 @@
+package notifications
+
+import (
+ "errors"
+ "testing"
+ "time"
+
+ "github.com/rcourtman/pulse-go-rewrite/internal/alerts"
+ "github.com/rcourtman/pulse-go-rewrite/internal/operationaltrust"
+)
+
+// A health callback can resolve an incident and acquire its delivery gate.
+// Every send-failure transition must release that gate and finish its audit
+// before calling the owner, just like policy cancellation already does.
+func TestQueuedFailureCallbackSeesCommittedAuditAndReleasedDeliveryGate(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ err error
+ attempts int
+ max int
+ status NotificationQueueStatus
+ failures int
+ }{
+ {"configuration", errors.New("invalid configuration: synthetic blocked destination"), 0, 3, QueueStatusDLQ, 1},
+ {"transient", errors.New("connection refused: synthetic unavailable destination"), 0, 3, QueueStatusPending, 0},
+ {"exhausted", errors.New("connection refused: synthetic unavailable destination"), 2, 3, QueueStatusDLQ, 1},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ now := time.Now()
+ q := newQuietReplayQueue(t, t.TempDir(), &now)
+ n := &QueuedNotification{
+ ID: "callback-order", Type: "webhook", Status: QueueStatusPending,
+ Config: []byte(`{}`), Attempts: tc.attempts, MaxAttempts: tc.max,
+ Alerts: []*alerts.Alert{{ID: "incident"}},
+ Links: []operationaltrust.NotificationLink{{DestinationID: "ops", OperationalRecordID: "incident",
+ TransitionID: "firing", LifecycleState: operationaltrust.OperationalOpen, CauseKey: "cpu"}},
+ }
+ if err := q.Enqueue(n); err != nil {
+ t.Fatal(err)
+ }
+ q.SetProcessor(func(*QueuedNotification) error { return tc.err })
+ callbacks := 0
+ q.SetDeliveryHealthChangedCallback(func() {
+ callbacks++
+ stats, err := q.GetTelemetryStats(time.Time{})
+ if err != nil {
+ t.Error(err)
+ return
+ }
+ if stats.Attempts != 1 || stats.Deliveries != 0 || stats.Failures != tc.failures {
+ t.Errorf("callback saw incomplete audit: %+v", stats)
+ }
+ if tc.name == "configuration" && stats.FailureClasses.Configuration != 1 ||
+ tc.name == "exhausted" && stats.FailureClasses.Connectivity != 1 {
+ t.Errorf("callback lost the terminal failure class: %+v", stats)
+ }
+ q.deliveryGateMu.Lock()
+ gate := q.deliveryGates["incident"]
+ q.deliveryGateMu.Unlock()
+ if gate != nil {
+ // Fail rather than hang when this regression is deliberately
+ // restored. A callback must be free to take the writer gate.
+ if !gate.mu.TryLock() {
+ t.Error("callback retained the firing alert delivery gate")
+ return
+ }
+ gate.mu.Unlock()
+ }
+ release := q.acquireAlertDeliveryGates([]string{"incident"}, true)
+ defer release()
+ rows, err := q.GetQueueStats()
+ if err != nil || rows[string(tc.status)] != 1 {
+ t.Errorf("callback did not see committed %s: %v, %v", tc.status, rows, err)
+ }
+ links, err := q.getNotificationLinks(n.ID)
+ if err != nil || len(links) != 1 {
+ t.Errorf("callback lost operational links: %v, %v", links, err)
+ return
+ }
+ wantState := operationaltrust.NotificationDeadLetter
+ if tc.status == QueueStatusPending {
+ wantState = operationaltrust.NotificationRetrying
+ }
+ if links[0].DeliveryState != wantState || links[0].AttemptedAt == nil {
+ t.Errorf("callback saw incomplete attempt linkage: %+v", links[0])
+ }
+ })
+ q.processNotification(n)
+ if callbacks != 1 {
+ t.Fatalf("health callbacks=%d, want 1", callbacks)
+ }
+ if n.Status != tc.status || n.Attempts != tc.attempts+1 {
+ t.Fatalf("status=%s attempts=%d, want %s/%d", n.Status, n.Attempts, tc.status, tc.attempts+1)
+ }
+ })
+ }
+}