Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

12.9 KB
Raw Blame
package notifications

import (
	"net/http"
	"sync"
	"testing"
	"time"

	"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
	"github.com/stretchr/testify/require"
)

// admissionBarrier holds the first serialisation of the alert that carries it.
// Enqueue serialises an alert's payload after the admission path has read the
// queue and before it writes the row, so the barrier pauses admitted work at
// the point where Stop could otherwise close storage underneath it.
type admissionBarrier struct {
	once    sync.Once
	entered chan struct{}
	release chan struct{}
}

func newAdmissionBarrier() *admissionBarrier {
	return &admissionBarrier{entered: make(chan struct{}), release: make(chan struct{})}
}

func (b *admissionBarrier) MarshalJSON() ([]byte, error) {
	b.once.Do(func() {
		close(b.entered)
		<-b.release
	})
	return []byte(`"released"`), nil
}

// shutdownAdmissionFixture is a deferred manager on persistent storage whose
// queue is never activated, so only the work under test takes the queue lock
// while Stop runs. Direct delivery is captured, never run.
type shutdownAdmissionFixture struct {
	t       *testing.T
	dir     string
	hook    WebhookConfig
	spawned chan func()
}

func newShutdownAdmissionFixture(t *testing.T) *shutdownAdmissionFixture {
	t.Helper()
	f := &shutdownAdmissionFixture{
		t:       t,
		dir:     t.TempDir(),
		hook:    WebhookConfig{ID: "ops", URL: "http://127.0.0.1:9/never-contacted", Enabled: true, Service: "generic"},
		spawned: make(chan func(), 8),
	}
	origSpawn := spawnAsync
	spawnAsync = func(deliver func()) { f.spawned <- deliver }
	t.Cleanup(func() { spawnAsync = origSpawn })
	return f
}

func (f *shutdownAdmissionFixture) open(configure func(*NotificationManager)) *NotificationManager {
	f.t.Helper()
	m := NewNotificationManagerWithDeferredQueue("", f.dir)
	f.t.Cleanup(m.Stop)
	// No wall-clock batch may contend for the queue lock in this window.
	m.GetQueue().processorTicker.Stop()
	m.SetGroupingConfig(false, 0, false, false)
	m.AddWebhook(f.hook)
	if configure != nil {
		configure(m)
	}
	return m
}

// stopPassedAdmission reports that Stop has gone as far as it can while
// admitted work is held: it closed storage without waiting for that work, or
// it stopped the queue workers and is waiting with delivery still permitted.
func stopPassedAdmission(m *NotificationManager, q *NotificationQueue) bool {
	if q.db.Ping() != nil {
		return true
	}
	select {
	case <-q.stopChan:
		return m.IsEnabled()
	default:
		return false
	}
}

func awaitShutdownCondition(t *testing.T, what string, condition func() bool) {
	t.Helper()
	deadline := time.Now().Add(10 * time.Second)
	for !condition() {
		if time.Now().After(deadline) {
			t.Fatalf("timed out waiting for %s", what)
		}
		time.Sleep(time.Millisecond)
	}
}

func awaitShutdownSignal(t *testing.T, what string, signal <-chan struct{}) {
	t.Helper()
	select {
	case <-signal:
	case <-time.After(10 * time.Second):
		t.Fatalf("timed out waiting for %s", what)
	}
}

// An alert, recovery or timer group admitted before Stop must be written
// before Stop closes storage, and an attempt made once Stop has begun must be
// refused rather than wait on the queue or fall back to direct delivery. Two
// concurrent Stop calls share one shutdown.
func TestStopWaitsForAdmittedNotificationWrites(t *testing.T) {
	for _, path := range []string{"firing", "recovery", "group-timer"} {
		t.Run(path, func(t *testing.T) {
			f := newShutdownAdmissionFixture(t)
			m := f.open(func(m *NotificationManager) {
				if path == "group-timer" {
					// sendGroupedAlerts is the timer callback; a long window keeps
					// the real timer from also running it.
					m.SetGroupingConfig(true, 3600, false, false)
				}
			})
			q := m.GetQueue()
			barrier := newAdmissionBarrier()
			alert := quietReplayAlert("admitted-"+path, alerts.AlertLevelWarning, time.Now().UTC().Add(-time.Minute))
			alert.Metadata = map[string]interface{}{"barrier": barrier}

			admitted := make(chan struct{})
			switch path {
			case "firing":
				go func() {
					defer close(admitted)
					m.SendAlert(alert)
				}()
			case "recovery":
				m.recordSuccessfulDelivery(notificationDeliveryJob{Type: "webhook", Event: eventAlert,
					Alerts: []*alerts.Alert{alert}, WebhookConfig: &f.hook}, time.Now())
				go func() {
					defer close(admitted)
					m.SendResolvedAlert(&alerts.ResolvedAlert{Alert: alert, ResolvedTime: time.Now()})
				}()
			case "group-timer":
				m.SendAlert(alert)
				go func() {
					defer close(admitted)
					m.sendGroupedAlerts()
				}()
			}
			awaitShutdownSignal(t, "admitted work to reach queue serialisation", barrier.entered)

			stopped := make(chan struct{}, 2)
			for range 2 {
				go func() {
					m.Stop()
					stopped <- struct{}{}
				}()
			}
			awaitShutdownCondition(t, "Stop to pass the admitted work", func() bool { return stopPassedAdmission(m, q) })

			late := quietReplayAlert("late-"+path, alerts.AlertLevelWarning, time.Now().UTC())
			refused := make(chan struct{})
			go func() {
				defer close(refused)
				m.SendAlert(late)
				m.SendResolvedAlert(&alerts.ResolvedAlert{Alert: late, ResolvedTime: time.Now()})
			}()
			awaitShutdownSignal(t, "an attempt made during Stop to be refused", refused)

			close(barrier.release)
			awaitShutdownSignal(t, "admitted work to finish", admitted)
			for range 2 {
				select {
				case <-stopped:
				case <-time.After(10 * time.Second):
					t.Fatal("Stop did not finish after admitted work completed")
				}
			}
			select {
			case <-f.spawned:
				t.Errorf("SHUTDOWN_ADMISSION_DIRECT_SEND: %s admitted before Stop could not be written because Stop closed the queue under it, and went to direct delivery after shutdown", path)
			default:
			}

			rows := quietReplayRows(t, f.open(nil).GetQueue())
			if len(rows) != 1 || len(rows[0].Alerts) != 1 || rows[0].Alerts[0].ID != alert.ID ||
				rows[0].Status != QueueStatusPending || rows[0].Attempts != 0 {
				t.Fatalf("SHUTDOWN_ADMISSION_LOST: %s admitted before Stop is not pending after restart: %+v", path, rows)
			}
		})
	}
}

// Operator pause and channel disable keep their meaning while Stop drains:
// they cancel queued work, including an alert admitted just before Stop.
func TestStopDrainKeepsPauseAndChannelDisable(t *testing.T) {
	email := EmailConfig{Enabled: true, SMTPHost: "127.0.0.1", SMTPPort: 9,
		From: "[email protected]", To: []string{"[email protected]"}}
	for _, tc := range []struct {
		name      string
		configure func(*NotificationManager)
		act       func(*NotificationManager)
		visible   func(*NotificationManager) bool
		reason    string
	}{
		{
			name:    "pause",
			act:     func(m *NotificationManager) { m.SetEnabled(false) },
			visible: func(m *NotificationManager) bool { return !m.IsEnabled() },
			reason:  "Notifications disabled",
		},
		{
			name: "email-disabled",
			configure: func(m *NotificationManager) {
				m.SetEmailConfig(email)
				_ = m.DeleteWebhook("ops")
			},
			act: func(m *NotificationManager) {
				disabled := email
				disabled.Enabled = false
				m.SetEmailConfig(disabled)
			},
			visible: func(m *NotificationManager) bool { return !m.GetEmailConfig().Enabled },
			reason:  "Email notifications disabled",
		},
	} {
		t.Run(tc.name, func(t *testing.T) {
			f := newShutdownAdmissionFixture(t)
			m := f.open(tc.configure)
			q := m.GetQueue()
			barrier := newAdmissionBarrier()
			alert := quietReplayAlert("paused-"+tc.name, alerts.AlertLevelWarning, time.Now().UTC().Add(-time.Minute))
			alert.Metadata = map[string]interface{}{"barrier": barrier}
			admitted := make(chan struct{})
			go func() {
				defer close(admitted)
				m.SendAlert(alert)
			}()
			awaitShutdownSignal(t, "admitted work to reach queue serialisation", barrier.entered)

			stopped := make(chan struct{})
			go func() {
				m.Stop()
				close(stopped)
			}()
			awaitShutdownCondition(t, "Stop to pass the admitted work", func() bool { return stopPassedAdmission(m, q) })

			// The change takes its lease in the critical section that publishes
			// it, so once it is visible its cancellation is owed to the queue.
			acted := make(chan struct{})
			go func() {
				defer close(acted)
				tc.act(m)
			}()
			awaitShutdownCondition(t, tc.name+" to take effect", func() bool { return tc.visible(m) })
			close(barrier.release)
			awaitShutdownSignal(t, "admitted work to finish", admitted)
			awaitShutdownSignal(t, tc.name+" to finish", acted)
			awaitShutdownSignal(t, "Stop to finish", stopped)

			rows := quietReplayRows(t, f.open(nil).GetQueue())
			if len(rows) != 1 || rows[0].Status != QueueStatusCancelled ||
				rows[0].LastError == nil || *rows[0].LastError != tc.reason {
				t.Fatalf("SHUTDOWN_POLICY_LOST: %s during the drain did not cancel the alert admitted before Stop: %+v", tc.name, rows)
			}
		})
	}
}

// A deferred queue that never started keeps admitted rows for the next start,
// and Stop checkpoints the buffered group without starting provider delivery.
func TestStopNeverStartedQueueAndBufferedGroup(t *testing.T) {
	f := newShutdownAdmissionFixture(t)
	m := f.open(nil)
	start := time.Now().UTC().Add(-time.Minute)
	m.SendAlert(quietReplayAlert("queued-1", alerts.AlertLevelWarning, start))
	m.SendAlert(quietReplayAlert("queued-2", alerts.AlertLevelWarning, start))
	m.SetGroupingConfig(true, 3600, false, false)
	m.SendAlert(quietReplayAlert("buffered", alerts.AlertLevelWarning, start))

	stopped := make(chan struct{})
	go func() {
		m.Stop()
		close(stopped)
	}()
	awaitShutdownSignal(t, "Stop of a never-started queue", stopped)
	// A timer that runs after Stop finds nothing to admit.
	m.sendGroupedAlerts()

	select {
	case <-f.spawned:
		t.Fatal("stopped manager handed work to direct delivery")
	default:
	}
	statuses := map[string]NotificationQueueStatus{}
	rows := quietReplayRows(t, f.open(nil).GetQueue())
	for _, row := range rows {
		if row.Attempts != 0 {
			t.Errorf("never-started queue consumed an attempt: %+v", row)
		}
		for _, alert := range row.Alerts {
			statuses[alert.ID] = row.Status
		}
	}
	if len(rows) != 3 || statuses["queued-1"] != QueueStatusPending || statuses["queued-2"] != QueueStatusPending || statuses["buffered"] != QueueStatusPending {
		t.Fatalf("never-started queue lost admitted rows across Stop: %+v", rows)
	}
	for _, row := range rows {
		if row.Alerts[0].ID == "buffered" {
			require.NotNil(t, row.NextRetryAt)
			require.True(t, row.NextRetryAt.After(time.Now().Add(3500*time.Second)), "shutdown must not flush or reset the original deadline")
		}
	}
}

// Direct delivery started before Stop is owned: Stop waits for it, so its
// firing receipt is durable and the recovery still reaches the destination
// after restart. Direct delivery cannot start once Stop has finished.
func TestStopOwnsDirectFallbackDelivery(t *testing.T) {
	received := make(chan struct{}, 4)
	release := make(chan struct{})
	server := newIPv4HTTPServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		received <- struct{}{}
		<-release
		w.WriteHeader(http.StatusOK)
	}))
	t.Cleanup(server.Close)
	var releaseOnce sync.Once
	releaseDelivery := func() { releaseOnce.Do(func() { close(release) }) }

	dir := t.TempDir()
	hook := WebhookConfig{ID: "ops", URL: server.URL + "/direct", Enabled: true, Service: "generic"}
	open := func() *NotificationManager {
		m := NewNotificationManagerWithDeferredQueue("", dir)
		t.Cleanup(m.Stop)
		m.webhookClient = server.Client()
		require.NoError(t, m.UpdateAllowedPrivateCIDRs("127.0.0.1/32"))
		m.SetGroupingConfig(false, 0, false, false)
		m.AddWebhook(hook)
		return m
	}
	m := open()
	// Registered after the manager, so a held delivery is released before Stop.
	t.Cleanup(releaseDelivery)
	q := m.GetQueue()
	// A storage failure, not shutdown, sends this alert down the direct path.
	_, err := q.db.Exec(`CREATE TRIGGER reject_admission BEFORE INSERT ON notification_queue
  BEGIN SELECT RAISE(ABORT, 'fixture queue write failed'); END`)
	require.NoError(t, err)

	alert := quietReplayAlert("direct", alerts.AlertLevelWarning, time.Now().UTC().Add(-time.Minute))
	m.SendAlert(alert)
	select {
	case <-received:
	case <-time.After(10 * time.Second):
		t.Fatal("enqueue failure did not start direct delivery")
	}

	stopped := make(chan struct{})
	go func() {
		m.Stop()
		close(stopped)
	}()
	awaitShutdownCondition(t, "Stop to pass the direct delivery", func() bool { return stopPassedAdmission(m, q) })
	releaseDelivery()
	awaitShutdownSignal(t, "Stop to finish", stopped)

	spawned := 0
	origSpawn := spawnAsync
	spawnAsync = func(func()) { spawned++ }
	m.dispatchNotificationJobAsync(notificationDeliveryJob{Type: "webhook", Event: eventAlert,
		Alerts: []*alerts.Alert{alert}, WebhookConfig: &hook}, "direct delivery after Stop")
	spawnAsync = origSpawn
	if spawned != 0 {
		t.Error("FALLBACK_AFTER_SHUTDOWN: direct delivery started after Stop returned")
	}

	jobs := buildNotificationDeliveryJobs(EmailConfig{}, []WebhookConfig{hook}, AppriseConfig{}, []*alerts.Alert{alert}, eventResolved, time.Now())
	if len(open().filterResolvedJobsByDeliveryReceipt(jobs)) != 1 {
		t.Fatal("FALLBACK_RECEIPT_LOST: direct delivery accepted during shutdown left no durable firing receipt, so its recovery is suppressed after restart")
	}
}