Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

5.8 KB
Raw Blame
package notifications

import (
	"encoding/json"
	"fmt"
	"net/http"
	"sort"
	"sync"
	"testing"
	"time"

	"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
	"github.com/stretchr/testify/require"
)

// Shutdown is a restart boundary, not an operator decision to stop delivery.
// Five real sends occupy every queue worker while a sixth admitted job waits
// for a slot, which is where shutdown finds an ordinary backlog. The waiting
// job must survive for the next start rather than be cancelled as "delivery
// disabled", and sends completing during shutdown must keep durable firing
// receipts so their recoveries are still delivered after restart.
func TestManagerStopKeepsQueuedDeliveryForRestart(t *testing.T) {
	type message struct {
		Event  string          `json:"event"`
		Alerts []*alerts.Alert `json:"alerts"`
	}
	received := make(chan message, 16)
	release := make(chan struct{})
	server := newIPv4HTTPServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		defer r.Body.Close()
		var msg message
		if err := json.NewDecoder(r.Body).Decode(&msg); err != nil || len(msg.Alerts) != 1 {
			t.Errorf("decode delivery: %v (%d alerts)", err, len(msg.Alerts))
			w.WriteHeader(http.StatusBadRequest)
			return
		}
		select {
		case received <- msg:
		default:
			t.Error("unexpected excess deliveries")
		}
		<-release
		w.WriteHeader(http.StatusOK)
	}))
	t.Cleanup(server.Close)
	var releaseOnce sync.Once
	releaseSends := func() { releaseOnce.Do(func() { close(release) }) }

	dir := t.TempDir()
	hook := WebhookConfig{ID: "ops", URL: server.URL, Enabled: true, Service: "generic"}
	open := func() *NotificationManager {
		m := NewNotificationManagerWithDeferredQueue("", dir)
		t.Cleanup(m.Stop)
		m.webhookClient = server.Client()
		require.NoError(t, m.UpdateAllowedPrivateCIDRs("127.0.0.1/32"))
		m.SetGroupingConfig(false, 0, false, false)
		m.AddWebhook(hook)
		return m
	}
	m := open()
	// Registered after the manager, so held sends are released before its Stop.
	t.Cleanup(releaseSends)

	start := time.Now().UTC().Add(-time.Minute)
	admitted := map[string]*alerts.Alert{}
	for i := range 6 {
		alert := quietReplayAlert(fmt.Sprintf("shutdown-%d", i), alerts.AlertLevelWarning, start)
		admitted[alert.ID] = alert
		m.SendAlert(alert)
	}
	pending, err := m.GetQueue().GetPending(20)
	require.NoError(t, err)
	require.Len(t, pending, 6, "every job must be admitted before workers start")

	q := m.GetQueue()
	m.StartQueueProcessing()
	inFlight := map[string]bool{}
	for range 5 {
		select {
		case msg := <-received:
			inFlight[msg.Alerts[0].ID] = true
		case <-time.After(10 * time.Second):
			t.Fatal("five queued sends did not occupy the worker pool")
		}
	}

	stopped := make(chan struct{})
	go func() {
		m.Stop()
		close(stopped)
	}()
	// Either half of shutdown may come first; release the held sends only
	// once the manager has begun to stop.
	stopBegun := func() bool {
		select {
		case <-q.stopChan:
			return true
		default:
			return !m.IsEnabled()
		}
	}
	deadline := time.Now().Add(10 * time.Second)
	for !stopBegun() {
		if time.Now().After(deadline) {
			t.Fatal("manager shutdown did not begin")
		}
		time.Sleep(time.Millisecond)
	}
	releaseSends()
	select {
	case <-stopped:
	case <-time.After(10 * time.Second):
		t.Fatal("manager shutdown did not finish its in-flight sends")
	}
	select {
	case extra := <-received:
		t.Fatalf("shutdown started a new provider attempt: %+v", extra)
	default:
	}

	m = open()
	q = m.GetQueue()
	var waiting *QueuedNotification
	for _, row := range quietReplayRows(t, q) {
		require.Len(t, row.Alerts, 1)
		if inFlight[row.Alerts[0].ID] {
			require.Equal(t, QueueStatusSent, row.Status, "a send completed during shutdown keeps its outcome")
			require.Equal(t, 1, row.Attempts)
			continue
		}
		require.Nil(t, waiting, "exactly one job waited for a worker")
		waiting = row
	}
	require.NotNil(t, waiting)

	delivered := make([]string, 0, len(inFlight))
	for id := range inFlight {
		delivered = append(delivered, id)
	}
	sort.Strings(delivered)
	for _, id := range delivered {
		jobs := buildNotificationDeliveryJobs(EmailConfig{}, []WebhookConfig{hook}, AppriseConfig{}, []*alerts.Alert{admitted[id]}, eventResolved, time.Now())
		if len(m.filterResolvedJobsByDeliveryReceipt(jobs)) != 1 {
			t.Errorf("SHUTDOWN_RECEIPT_LOST: %s reached the receiver during shutdown, but its recovery is suppressed after restart", id)
		}
	}
	if waiting.Status != QueueStatusPending || waiting.Attempts != 0 {
		lastError := ""
		if waiting.LastError != nil {
			lastError = *waiting.LastError
		}
		t.Fatalf("SHUTDOWN_CANCELLED_UNCLAIMED: job for %s that waited for a worker became %s after %d attempt(s) (%q); it must stay pending for the next start",
			waiting.Alerts[0].ID, waiting.Status, waiting.Attempts, lastError)
	}

	m.StartQueueProcessing()
	select {
	case msg := <-received:
		require.Equal(t, waiting.Alerts[0].ID, msg.Alerts[0].ID)
	case <-time.After(10 * time.Second):
		t.Fatal("the job kept across shutdown was not delivered after restart")
	}
	deadline = time.Now().Add(10 * time.Second)
	for loadQuietReplay(t, q, waiting.ID).Status != QueueStatusSent {
		if time.Now().After(deadline) {
			t.Fatal("the job kept across shutdown was not recorded as sent")
		}
		time.Sleep(time.Millisecond)
	}

	// The user-visible end of a receipt: a recovery after restart still
	// reaches the destination that received the firing during shutdown.
	recovered := delivered[0]
	m.SendResolvedAlert(&alerts.ResolvedAlert{Alert: admitted[recovered], ResolvedTime: time.Now()})
	select {
	case msg := <-received:
		require.Equal(t, "resolved", msg.Event)
		require.Equal(t, recovered, msg.Alerts[0].ID)
	case <-time.After(10 * time.Second):
		t.Fatalf("recovery for %s was not delivered after restart", recovered)
	}
	select {
	case extra := <-received:
		t.Fatalf("duplicate provider message: %+v", extra)
	default:
	}
}