Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
package notifications
import (
"encoding/json"
"fmt"
"io"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
)
func waitResolvedContract(t *testing.T, predicate func() bool) {
t.Helper()
deadline := time.Now().Add(12 * time.Second)
for !predicate() {
if time.Now().After(deadline) {
t.Fatal("notification condition not reached")
}
time.Sleep(5 * time.Millisecond)
}
}
// Ordinary firing establishes real receipts. Restart reloads those receipts;
// no receipt or delivery job is manufactured by this test.
func TestResolvedGroupingOrdinaryRestart(t *testing.T) {
for _, stage := range []string{"same_process", "after_firing", "pending_recovery", "late_arrival"} {
t.Run(stage, func(t *testing.T) {
type payload struct {
Event string `json:"event"`
Alerts []*alerts.Alert `json:"alerts"`
receivedAt time.Time
}
received := make(chan payload, 64)
server := newResolvedContractServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
var p payload
if err := json.NewDecoder(r.Body).Decode(&p); err != nil {
t.Error(err)
}
p.receivedAt = time.Now()
received <- p
w.WriteHeader(http.StatusOK)
}))
defer server.Close()
dir := t.TempDir()
hook := WebhookConfig{ID: "ops", Name: "ops", URL: server.URL, Enabled: true}
open := func() *NotificationManager {
m := NewNotificationManagerWithDataDir("", dir)
m.webhookClient = server.Client()
if err := m.UpdateAllowedPrivateCIDRs("127.0.0.1/32"); err != nil {
t.Fatal(err)
}
m.AddWebhook(hook)
m.SetNotifyOnResolve(true)
m.SetGroupingConfig(true, 1, false, false)
t.Cleanup(m.Stop)
return m
}
m := open()
batch := make([]*alerts.Alert, 15)
for i := range batch {
batch[i] = &alerts.Alert{ID: fmt.Sprintf("burst-%02d", i), ResourceName: fmt.Sprintf("guest-%02d", i), Type: "cpu", Level: alerts.AlertLevelWarning, StartTime: time.Now().Add(-time.Minute)}
m.SendAlert(batch[i])
}
job := notificationDeliveryJob{Type: "webhook", Event: eventResolved, Alerts: batch, WebhookConfig: &hook}
waitResolvedContract(t, func() bool {
jobs := m.filterResolvedJobsByDeliveryReceipt([]notificationDeliveryJob{job})
return len(jobs) == 1 && len(jobs[0].Alerts) == 15
})
firing := <-received
if len(firing.Alerts) != 15 {
t.Fatalf("firing batch = %d", len(firing.Alerts))
}
if stage == "after_firing" {
m.Stop()
m = open()
}
// A newly configured destination must not receive recoveries for firings
// delivered only to ops, even when sharing its endpoint.
unseenHook := hook
unseenHook.ID = "never-delivered"
m.AddWebhook(unseenHook)
resolvedAt := time.Now().Truncate(time.Second)
submittedAt := make(map[string]time.Time, len(batch))
for i, a := range batch {
if m.CancelAlert(a.ID) {
t.Fatal("delivered occurrence cancelled as unannounced")
}
submittedAt[a.ID] = time.Now()
m.SendResolvedAlert(&alerts.ResolvedAlert{Alert: a, ResolvedTime: resolvedAt.Add(time.Duration(i) * time.Second)})
if stage == "late_arrival" && i == 0 {
// A completed enqueue has at most one extra second of rounding.
// Arrivals after its deadline belong to a new fixed window, even
// when the ordinary dispatcher has not yet claimed the first group.
time.Sleep(2100 * time.Millisecond)
}
}
burstFitsWindow := time.Since(submittedAt[batch[0].ID]) < time.Second
m.SendResolvedAlert(&alerts.ResolvedAlert{Alert: &alerts.Alert{ID: "unannounced", ResourceName: "unannounced", StartTime: time.Now()}, ResolvedTime: resolvedAt})
if stage == "pending_recovery" {
m.Stop()
m = open()
m.AddWebhook(unseenHook)
}
// Check receiver time against admission, not against the end of the
// loop/restart: slow emission can legitimately span fixed windows.
// A fast burst must still be one group, and all occurrences must be
// delivered exactly once with their original recovery metadata.
seen := make(map[string]bool, len(batch))
groups := 0
timer := time.NewTimer(12 * time.Second)
defer timer.Stop()
for len(seen) < len(batch) {
var recovery payload
select {
case recovery = <-received:
case <-timer.C:
t.Fatalf("grouped recovery missing occurrences: delivered=%d expected=%d", len(seen), len(batch))
}
if recovery.Event != "resolved" || len(recovery.Alerts) == 0 {
t.Fatalf("recovery event=%s alerts=%d", recovery.Event, len(recovery.Alerts))
}
if burstFitsWindow && len(recovery.Alerts) != len(batch) {
t.Fatalf("burst admitted within one window was split: alerts=%d", len(recovery.Alerts))
}
if stage == "late_arrival" && len(seen) == 0 && len(recovery.Alerts) != 1 {
t.Fatal("late arrivals extended the expired first group")
}
firstAdmission := submittedAt[recovery.Alerts[0].ID]
if firstAdmission.IsZero() || recovery.receivedAt.Before(firstAdmission.Add(time.Second)) {
t.Fatal("resolved delivery bypassed the first occurrence's grouping window")
}
groups++
previousIndex := -1
for _, a := range recovery.Alerts {
i := -1
for j, original := range batch {
if a.ID == original.ID {
i = j
break
}
}
if i < 0 || i <= previousIndex || seen[a.ID] || !a.StartTime.Equal(batch[i].StartTime) || a.Metadata[metadataResolvedAt] != resolvedAt.Add(time.Duration(i)*time.Second).Format(time.RFC3339) {
t.Fatalf("duplicate, unannounced or altered recovery occurrence: id=%s", a.ID)
}
seen[a.ID] = true
previousIndex = i
}
}
t.Logf("ordinary recovery: stage=%s groups=%d occurrences=%d burstFitsWindow=%t", stage, groups, len(seen), burstFitsWindow)
waitResolvedContract(t, func() bool { return len(m.filterResolvedJobsByDeliveryReceipt([]notificationDeliveryJob{job})) == 0 })
stats, err := m.GetQueue().GetQueueStats()
if err != nil {
t.Fatal(err)
}
if stats["dlq"] != 0 || stats["failed"] != 0 {
t.Fatalf("unexpected delivery failures: %v", stats)
}
m.Stop()
select {
case p := <-received:
t.Fatalf("extra recovery: %+v", p)
default:
}
})
}
}
func TestResolvedGroupedWebhookNamesEveryAlert(t *testing.T) {
for _, service := range []string{"generic", "discord", "slack", "telegram", "teams", "teams-adaptive", "pushover", "gotify", "ntfy", "mattermost", "custom"} {
t.Run(service, func(t *testing.T) {
var body string
server := newResolvedContractServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
b, _ := io.ReadAll(r.Body)
body = string(b)
w.WriteHeader(http.StatusOK)
}))
defer server.Close()
m := NewNotificationManagerWithDataDir("", t.TempDir())
defer m.Stop()
m.webhookClient = server.Client()
if err := m.UpdateAllowedPrivateCIDRs("127.0.0.1/32"); err != nil {
t.Fatal(err)
}
hook := WebhookConfig{ID: "ops", Name: "ops", URL: server.URL + "?chat_id=1234", Enabled: true, Service: service}
if service == "custom" {
hook.Service = "generic"
hook.Template = `{"message":"{{.Message}}","count":{{.AlertCount}}}`
}
batch := contractGroupedAlerts()
if err := m.sendResolvedWebhook(hook, batch, time.Now()); err != nil {
t.Fatal(err)
}
for _, a := range batch {
if !strings.Contains(body, a.ResourceName) {
t.Errorf("%s missing %s: %s", service, a.ResourceName, body)
}
}
})
}
}
// Exercise HTTP payloads without a listener or external network. Persistent
// queue operations remain real; this adapter replaces only provider transport.
type resolvedContractTransport struct{ handler http.Handler }
func (tr resolvedContractTransport) RoundTrip(r *http.Request) (*http.Response, error) {
recorder := httptest.NewRecorder()
tr.handler.ServeHTTP(recorder, r)
return recorder.Result(), nil
}
type resolvedContractServer struct {
URL string
client *http.Client
}
func newResolvedContractServer(handler http.Handler) *resolvedContractServer {
return &resolvedContractServer{URL: "http://127.0.0.1:12345", client: &http.Client{Transport: resolvedContractTransport{handler}}}
}
func (s *resolvedContractServer) Client() *http.Client { return s.client }
func (s *resolvedContractServer) Close() {}
// Disabled grouping remains immediate. With the same 15-alert workload this
// deliberately hits the unchanged internal 10/minute destination limit; grouped
// recovery above sends one request instead and creates no dead letters.
func TestResolvedGroupingDisabledBurstRetainsRateLimit(t *testing.T) {
server := newResolvedContractServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) }))
m := NewNotificationManagerWithDataDir("", t.TempDir())
defer m.Stop()
m.webhookClient = server.Client()
if err := m.UpdateAllowedPrivateCIDRs("127.0.0.1/32"); err != nil {
t.Fatal(err)
}
hook := WebhookConfig{ID: "ops", Name: "ops", URL: server.URL, Enabled: true}
m.AddWebhook(hook)
m.SetNotifyOnResolve(true)
m.SetGroupingConfig(true, 1, false, false)
batch := make([]*alerts.Alert, 15)
for i := range batch {
batch[i] = &alerts.Alert{ID: fmt.Sprintf("limit-%d", i), ResourceName: fmt.Sprintf("limit-%d", i), StartTime: time.Now().Add(-time.Minute), Level: alerts.AlertLevelWarning}
m.SendAlert(batch[i])
}
waitResolvedContract(t, func() bool { stats, err := m.GetQueue().GetQueueStats(); return err == nil && stats["sent"] == 1 })
m.SetGroupingConfig(false, 1, false, false)
for _, a := range batch {
m.CancelAlert(a.ID)
m.SendResolvedAlert(&alerts.ResolvedAlert{Alert: a, ResolvedTime: time.Now()})
}
waitResolvedContract(t, func() bool {
stats, err := m.GetQueue().GetQueueStats()
return err == nil && stats["dlq"] == 6 && stats["sent"] == 10
})
dlq, err := m.GetQueue().GetDLQ(20)
if err != nil {
t.Fatal(err)
}
if len(dlq) != 6 {
t.Fatalf("dead letters=%d", len(dlq))
}
for _, job := range dlq {
if job.Attempts != 3 || (job.LastError == nil || !strings.Contains(*job.LastError, "rate limit exceeded")) {
t.Errorf("wrong retry outcome: %+v", job)
}
}
}
// A restart must not terminally cancel a due persisted delivery before the
// owner has applied saved destination configuration. Manager construction
// happens before saved webhooks/email/Apprise are loaded in the monitor
// startup path; the worker must not treat the still-empty destination list as
// a permanent policy decision.
func TestRestartKeepsDueDeliveryPendingUntilConfigured(t *testing.T) {
received := make(chan struct{}, 4)
server := newResolvedContractServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
received <- struct{}{}
w.WriteHeader(http.StatusOK)
}))
defer server.Close()
dir := t.TempDir()
hook := WebhookConfig{ID: "ops", Name: "ops", URL: server.URL, Enabled: true}
configJSON, err := json.Marshal(hook)
if err != nil {
t.Fatal(err)
}
seed, err := NewNotificationQueue(dir)
if err != nil {
t.Fatal(err)
}
due := time.Now().Add(-time.Second)
if err := seed.Enqueue(&QueuedNotification{
ID: "restart-due",
Type: "webhook_resolved",
Status: QueueStatusPending,
Config: configJSON,
Alerts: []*alerts.Alert{{ID: "restart-due", ResourceName: "restart-due", StartTime: time.Now().Add(-time.Minute)}},
MaxAttempts: 3,
NextRetryAt: &due,
}); err != nil {
t.Fatal(err)
}
if err := seed.Stop(); err != nil {
t.Fatal(err)
}
// Production order: construct the manager, then apply saved configuration.
m := NewNotificationManagerWithDeferredQueue("", dir)
defer m.Stop()
m.webhookClient = server.Client()
if err := m.UpdateAllowedPrivateCIDRs("127.0.0.1/32"); err != nil {
t.Fatal(err)
}
// Force the same batch path the five-second ticker uses while saved
// configuration is still absent. It must leave the due row pending.
queue := m.GetQueue()
if queue == nil {
t.Fatal("queue unavailable")
}
queue.processBatch()
var status string
if err := queue.db.QueryRow(`SELECT status FROM notification_queue WHERE id = 'restart-due'`).Scan(&status); err != nil {
t.Fatal(err)
}
if status != string(QueueStatusPending) {
t.Fatalf("due delivery status before configuration = %s, want pending", status)
}
m.AddWebhook(hook)
m.StartQueueProcessing()
select {
case <-received:
case <-time.After(5 * time.Second):
stats, _ := queue.GetQueueStats()
t.Fatalf("configured due delivery not sent: %v", stats)
}
}
func TestResolvedGroupingPagerDutyKeepsIndividualKeys(t *testing.T) {
hook := WebhookConfig{ID: "pd", Service: "pagerduty", Enabled: true}
batch := contractGroupedAlerts()
jobs := buildNotificationDeliveryJobs(EmailConfig{}, []WebhookConfig{hook}, AppriseConfig{}, batch, eventResolved, time.Now())
if len(jobs) != len(batch) {
t.Fatalf("PagerDuty jobs=%d, want %d independent dedup keys", len(jobs), len(batch))
}
for i, job := range jobs {
if len(job.Alerts) != 1 || job.Alerts[0].ID != batch[i].ID {
t.Fatalf("wrong PagerDuty occurrence: %+v", job)
}
}
}