Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

7.9 KB
Raw Blame
package notifications

import (
	"bytes"
	"encoding/json"
	"io"
	"net/http"
	"reflect"
	"sync"
	"testing"
	"time"

	"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
	"github.com/rcourtman/pulse-go-rewrite/internal/operationaltrust"
)

func TestQueuedWebhookEligibilityUsesSavedIDBeforeURL(t *testing.T) {
	original := WebhookConfig{ID: "removed", URL: "https://notify.example.test/topic", Enabled: true}
	sibling := original
	sibling.ID = "replacement"
	idless := original
	idless.ID = ""
	disabled := original
	disabled.Enabled = false
	edited := original
	edited.URL = "https://notify.example.test/new-topic"
	tests := []struct {
		name    string
		queued  WebhookConfig
		current []WebhookConfig
		want    bool
	}{
		{"same identity", original, []WebhookConfig{original}, true},
		{"same identity edited URL preserves snapshot", original, []WebhookConfig{edited}, true},
		{"removed ID cannot borrow sibling URL", original, []WebhookConfig{sibling}, false},
		{"removed ID cannot borrow ID-less URL", original, []WebhookConfig{idless}, false},
		{"disabled ID cannot borrow enabled sibling", original, []WebhookConfig{sibling, disabled}, false},
		{"disabled ID first cannot borrow sibling", original, []WebhookConfig{disabled, sibling}, false},
		{"queued disabled", disabled, []WebhookConfig{original}, false},
		{"empty current list", original, nil, false},
		{"legacy ID-less matching URL", idless, []WebhookConfig{sibling}, true},
		{"legacy ID-less disabled URL", idless, []WebhookConfig{disabled}, false},
		{"legacy ID-less different URL", idless, []WebhookConfig{edited}, false},
		{"legacy missing URL", WebhookConfig{Enabled: true}, []WebhookConfig{sibling}, false},
	}
	for _, tc := range tests {
		t.Run(tc.name, func(t *testing.T) {
			manager := &NotificationManager{webhooks: tc.current}
			if got := manager.isQueuedWebhookStillEnabled(tc.queued); got != tc.want {
				t.Errorf("eligible=%v, want %v", got, tc.want)
			}
		})
	}
}

// A URL is a transport address, not a destination identity. Removing a known
// destination must revoke its pending and operator-retried work even when a
// different destination uses that same URL, before or after reopening the queue.
func TestQueuedWebhookDeletionDoesNotTransferWorkToSameURL(t *testing.T) {
	for _, service := range []string{"generic", "ntfy"} {
		for _, suffix := range []string{"", "_resolved"} {
			t.Run(service+suffix, func(t *testing.T) {
				var mu sync.Mutex
				var received []string
				server := newIPv4HTTPServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
					_, _ = io.Copy(io.Discard, r.Body)
					mu.Lock()
					received = append(received, r.Header.Get("Authorization"))
					mu.Unlock()
					if r.Header.Get("Authorization") == "Bearer synthetic-removed" {
						w.WriteHeader(http.StatusForbidden)
						return
					}
					w.WriteHeader(http.StatusAccepted)
				}))
				defer server.Close()
				original := WebhookConfig{ID: "removed", Name: "Removed destination", Enabled: true,
					Service: service, URL: server.URL + "/topic", Headers: map[string]string{"Authorization": "Bearer synthetic-removed"}}
				sibling := copyWebhookConfig(original)
				sibling.ID, sibling.Name = "replacement", "Replacement destination"
				sibling.Headers["Authorization"] = "Bearer synthetic-replacement"
				dir, now := t.TempDir(), time.Now()
				open := func(hooks []WebhookConfig) *NotificationManager {
					q := newQuietReplayQueue(t, dir, &now)
					m := &NotificationManager{enabled: true, queue: q, webhooks: copyWebhookConfigs(hooks),
						webhookClient: server.Client(), webhookRateLimits: make(map[string]*webhookRateLimit),
						lastNotified: make(map[string]notificationRecord), deliveryReceipts: make(map[string]struct{})}
					if err := m.UpdateAllowedPrivateCIDRs("127.0.0.1/32"); err != nil {
						t.Fatal(err)
					}
					q.SetProcessor(m.ProcessQueuedNotification)
					return m
				}
				m := open([]WebhookConfig{original, sibling})
				config, err := json.Marshal(original)
				if err != nil {
					t.Fatal(err)
				}
				for _, id := range []string{"waiting", "previous-failure"} {
					a := quietReplayAlert(id, alerts.AlertLevelWarning, now.Add(-time.Minute))
					n := &QueuedNotification{ID: id, Type: "webhook" + suffix, Status: QueueStatusPending,
						Config: config, MaxAttempts: 1, DestinationID: "webhook:removed", Alerts: []*alerts.Alert{a}}
					if err := m.queue.Enqueue(n); err != nil {
						t.Fatal(err)
					}
				}
				m.queue.processNotification(loadQuietReplay(t, m.queue, "previous-failure"))
				if row := loadQuietReplay(t, m.queue, "previous-failure"); row.Status != QueueStatusDLQ {
					t.Fatalf("initial status=%s", row.Status)
				}
				if err := m.DeleteWebhook(original.ID); err != nil {
					t.Fatal(err)
				}
				saved := m.GetWebhooks()
				if err := m.queue.db.Close(); err != nil {
					t.Fatal(err)
				}
				m = open(saved)
				if count, err := m.queue.RetryTerminalFailures(); err != nil || count != 1 {
					t.Fatalf("retry=%d, %v; want 1", count, err)
				}
				callbacks := 0
				m.queue.SetDeliveryHealthChangedCallback(func() { callbacks++ })
				for _, id := range []string{"waiting", "previous-failure"} {
					m.queue.processNotification(loadQuietReplay(t, m.queue, id))
					row := loadQuietReplay(t, m.queue, id)
					if row.Status != QueueStatusCancelled || row.CompletedAt == nil {
						t.Errorf("%s: status=%s completed=%v; want cancellation", id, row.Status, row.CompletedAt)
					}
					if !bytes.Equal(row.Config, config) {
						t.Errorf("%s: retained configuration rewritten", id)
					}
					if len(row.Links) != 1 || row.Links[0].DeliveryState != operationaltrust.NotificationCancelled || row.Links[0].DestinationID != "webhook:removed" {
						t.Errorf("%s: cancellation lost original link: %+v", id, row.Links)
					}
					var successes, audits, receipts int
					for query, into := range map[string]*int{
						"SELECT count(*) FROM notification_audit WHERE notification_id = ?":      &audits,
						"SELECT count(*) FROM notification_delivery_receipts WHERE alert_id = ?": &receipts,
					} {
						if err := m.queue.db.QueryRow(query, id).Scan(into); err != nil {
							t.Fatal(err)
						}
					}
					if err := m.queue.db.QueryRow("SELECT count(*) FROM notification_audit WHERE notification_id = ? AND success = 1", id).Scan(&successes); err != nil {
						t.Fatal(err)
					}
					wantAudits := 0
					if id == "previous-failure" {
						wantAudits = 1
					}
					if successes != 0 || audits != wantAudits || receipts != 0 {
						t.Errorf("%s: audit=%d successes=%d receipts=%d; want %d/0/0", id, audits, successes, receipts, wantAudits)
					}
				}
				if callbacks != 2 {
					t.Errorf("health callbacks=%d, want 2", callbacks)
				}
				if count, err := m.queue.RetryTerminalFailures(); err != nil || count != 0 {
					t.Errorf("cancelled work retried: %d, %v", count, err)
				}
				stats, err := m.queue.GetQueueStats()
				if err != nil || stats["cancelled"] != 2 || stats["sent"] != 0 || !m.DeliveryHealth().Healthy {
					t.Errorf("post-deletion stats=%v error=%v health=%+v", stats, err, m.DeliveryHealth())
				}
				// Normal work for the still-enabled destination must continue.
				m.SendAlert(quietReplayAlert("new-incident", alerts.AlertLevelWarning, now))
				pending, err := m.queue.GetPending(10)
				if err != nil || len(pending) != 1 {
					t.Fatalf("fresh pending=%d, %v", len(pending), err)
				}
				m.queue.processNotification(pending[0])
				if row := loadQuietReplay(t, m.queue, pending[0].ID); row.Status != QueueStatusSent {
					t.Errorf("replacement's new work=%s, want sent", row.Status)
				}
				mu.Lock()
				actual := append([]string(nil), received...)
				mu.Unlock()
				if want := []string{"Bearer synthetic-removed", "Bearer synthetic-replacement"}; !reflect.DeepEqual(actual, want) {
					t.Errorf("receiver credentials=%v, want %v", actual, want)
				}
				t.Log("original failure retained; deleted pending/retried work cancelled across restart without requests, audits or success receipts; new replacement work delivered")
			})
		}
	}
}