Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
package notifications
import (
"encoding/json"
"fmt"
"io"
"net/http"
"sync/atomic"
"testing"
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestEnhancedWebhook(t *testing.T) {
nm := NewNotificationManager("http://pulse.local")
_ = nm.UpdateAllowedPrivateCIDRs("127.0.0.1")
server := newIPv4HTTPServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
w.Write([]byte("ok"))
}))
defer server.Close()
webhook := EnhancedWebhookConfig{
WebhookConfig: WebhookConfig{
Name: "Test Webhook",
URL: server.URL,
},
Service: "discord",
PayloadTemplate: "{}",
}
status, resp, err := nm.TestEnhancedWebhook(webhook)
assert.NoError(t, err)
assert.Equal(t, http.StatusOK, status)
assert.Equal(t, "Response body withheld (2 bytes read)", resp)
}
func TestGotifyPresetTestDelivery(t *testing.T) {
nm := NewNotificationManager("https://pulse.example")
require.NoError(t, nm.UpdateAllowedPrivateCIDRs("127.0.0.1"))
server := newIPv4HTTPServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
assert.Equal(t, http.MethodPost, r.Method)
assert.Equal(t, "gotify-app-token", r.URL.Query().Get("token"))
assert.Equal(t, "application/json", r.Header.Get("Content-Type"))
var payload struct {
Message string `json:"message"`
Title string `json:"title"`
Priority int `json:"priority"`
Extras map[string]interface{} `json:"extras"`
}
require.NoError(t, json.NewDecoder(r.Body).Decode(&payload))
assert.NotEmpty(t, payload.Message)
assert.NotEmpty(t, payload.Title)
assert.Positive(t, payload.Priority)
assert.Contains(t, payload.Extras, "client::display")
assert.Contains(t, payload.Extras, "pulse::alert")
w.WriteHeader(http.StatusOK)
_, _ = w.Write([]byte(`{"id":1}`))
}))
defer server.Close()
basic := WebhookConfig{
Name: "Operations Gotify",
URL: server.URL + "/message?token=gotify-app-token",
Method: http.MethodPost,
Enabled: true,
Service: "gotify",
}
status, response, err := nm.TestEnhancedWebhook(BuildEnhancedWebhookTestConfig(basic, "gotify"))
require.NoError(t, err)
assert.Equal(t, http.StatusOK, status)
assert.Equal(t, "Response body withheld (8 bytes read)", response)
}
func TestShouldSendWebhook(t *testing.T) {
nm := &NotificationManager{}
tests := []struct {
name string
webhook EnhancedWebhookConfig
alert *alerts.Alert
expected bool
}{
{
name: "Match Level",
webhook: EnhancedWebhookConfig{
FilterRules: WebhookFilterRules{
Levels: []string{"critical"},
},
},
alert: &alerts.Alert{
Level: alerts.AlertLevelCritical,
},
expected: true,
},
{
name: "No Match Level",
webhook: EnhancedWebhookConfig{
FilterRules: WebhookFilterRules{
Levels: []string{"critical"},
},
},
alert: &alerts.Alert{
Level: alerts.AlertLevelWarning,
},
expected: false,
},
{
name: "Match Type",
webhook: EnhancedWebhookConfig{
FilterRules: WebhookFilterRules{
Types: []string{"cpu"},
},
},
alert: &alerts.Alert{
Type: "cpu",
},
expected: true,
},
{
name: "Match Node",
webhook: EnhancedWebhookConfig{
FilterRules: WebhookFilterRules{
Nodes: []string{"pve-01"},
},
},
alert: &alerts.Alert{
Node: "pve-01",
},
expected: true,
},
{
name: "Match ResourceType",
webhook: EnhancedWebhookConfig{
FilterRules: WebhookFilterRules{
ResourceTypes: []string{"vm"},
},
},
alert: &alerts.Alert{
Metadata: map[string]interface{}{
"resourceType": "vm",
},
},
expected: true,
},
{
name: "Empty Rules Match Everything",
webhook: EnhancedWebhookConfig{
FilterRules: WebhookFilterRules{},
},
alert: &alerts.Alert{
Level: alerts.AlertLevelWarning,
},
expected: true,
},
{
name: "No Match Node",
webhook: EnhancedWebhookConfig{
FilterRules: WebhookFilterRules{
Nodes: []string{"pve-01"},
},
},
alert: &alerts.Alert{
Node: "pve-02",
},
expected: false,
},
{
name: "No Match ResourceType",
webhook: EnhancedWebhookConfig{
FilterRules: WebhookFilterRules{
ResourceTypes: []string{"vm"},
},
},
alert: &alerts.Alert{
Metadata: map[string]interface{}{
"resourceType": "container",
},
},
expected: false,
},
{
name: "Missing ResourceType Metadata",
webhook: EnhancedWebhookConfig{
FilterRules: WebhookFilterRules{
ResourceTypes: []string{"vm"},
},
},
alert: &alerts.Alert{
Metadata: map[string]interface{}{},
},
expected: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
result := nm.shouldSendWebhook(tt.webhook, tt.alert)
assert.Equal(t, tt.expected, result)
})
}
}
func TestParseRetryAfterBackoff(t *testing.T) {
now := time.Date(2026, time.March, 13, 19, 0, 0, 0, time.UTC)
tests := []struct {
name string
retryAfter string
want time.Duration
ok bool
}{
{
name: "delta seconds",
retryAfter: "3",
want: 3 * time.Second,
ok: true,
},
{
name: "delta seconds with whitespace",
retryAfter: " 3 ",
want: 3 * time.Second,
ok: true,
},
{
name: "negative delta treated as immediate",
retryAfter: "-5",
want: 0,
ok: true,
},
{
name: "http date",
retryAfter: "Fri, 13 Mar 2026 19:00:02 GMT",
want: 2 * time.Second,
ok: true,
},
{
name: "http date with whitespace",
retryAfter: " Fri, 13 Mar 2026 19:00:02 GMT ",
want: 2 * time.Second,
ok: true,
},
{
name: "delta seconds capped at max backoff",
retryAfter: "999",
want: WebhookMaxBackoff,
ok: true,
},
{
name: "large seconds cannot overflow duration",
retryAfter: "9223372036854775807",
want: WebhookMaxBackoff,
ok: true,
},
{
name: "large negative seconds stay immediate",
retryAfter: "-9223372036854775807",
want: 0,
ok: true,
},
{
name: "http date capped at max backoff",
retryAfter: "Fri, 13 Mar 2026 19:05:00 GMT",
want: WebhookMaxBackoff,
ok: true,
},
{
name: "past http date treated as immediate",
retryAfter: "Fri, 13 Mar 2026 18:59:58 GMT",
want: 0,
ok: true,
},
{
name: "invalid value",
retryAfter: "not-a-number",
want: 0,
ok: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, ok := parseRetryAfterBackoff(tt.retryAfter, now)
assert.Equal(t, tt.ok, ok)
assert.Equal(t, tt.want, got)
})
}
}
// Even repeated valid zero-delay hints must not reset the transport retry budget.
func TestSendWebhookWithRetry_RateLimitExhaustion(t *testing.T) {
for _, retries := range []int{1, 2, 0, -1} {
t.Run(fmt.Sprint(retries), func(t *testing.T) {
nm := NewNotificationManager("http://pulse.local")
t.Cleanup(nm.Stop)
require.NoError(t, nm.UpdateAllowedPrivateCIDRs("127.0.0.1"))
var attempts atomic.Int32
payload := []byte(`{"test":true}`)
server := newIPv4HTTPServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
attempts.Add(1)
assert.Equal(t, "limited:alert", r.Header.Get("X-Pulse-Event-ID"))
body, err := io.ReadAll(r.Body)
assert.NoError(t, err)
assert.Equal(t, payload, body)
w.Header().Set("Retry-After", "0")
w.WriteHeader(http.StatusTooManyRequests)
}))
defer server.Close()
webhook := EnhancedWebhookConfig{
WebhookConfig: WebhookConfig{Name: "Limited webhook", URL: server.URL},
RetryEnabled: true,
RetryCount: retries,
}
wantRetries := retries
if wantRetries <= 0 {
wantRetries = WebhookDefaultRetries
}
err := nm.sendWebhookWithRetry(webhook, payload, "limited:alert")
require.Error(t, err)
assert.Contains(t, err.Error(), fmt.Sprintf("after %d attempts", wantRetries+1))
assert.Equal(t, int32(wantRetries+1), attempts.Load())
history := nm.GetWebhookHistory()
require.Len(t, history, 1, "one failed delivery, not one history entry per HTTP attempt")
assert.False(t, history[0].Success)
assert.Equal(t, http.StatusTooManyRequests, history[0].StatusCode)
assert.Equal(t, wantRetries, history[0].RetryAttempts)
assert.Equal(t, len(payload), history[0].PayloadSize)
assert.Contains(t, history[0].ErrorMessage, "HTTP 429")
})
}
}
// A 503 may advertise a recovery delay longer than our initial backoff.
// Observe request spacing, not merely eventual success, to detect ignored hints.
func TestSendWebhookWithRetry_ServiceUnavailableRetryAfter(t *testing.T) {
for _, tc := range []struct {
name string
header string
minimum time.Duration
}{
{"valid", "2", 2 * time.Second},
{"invalid", "not-a-number", WebhookInitialBackoff},
} {
t.Run(tc.name, func(t *testing.T) {
nm := NewNotificationManager("http://pulse.local")
t.Cleanup(nm.Stop)
require.NoError(t, nm.UpdateAllowedPrivateCIDRs("127.0.0.1"))
attemptTimes := make(chan time.Time, 2)
attempts := 0
server := newIPv4HTTPServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
attempts++
attemptTimes <- time.Now()
if attempts == 1 {
w.Header().Set("Retry-After", tc.header)
w.WriteHeader(http.StatusServiceUnavailable)
return
}
w.WriteHeader(http.StatusOK)
}))
defer server.Close()
webhook := EnhancedWebhookConfig{
WebhookConfig: WebhookConfig{Name: "Service recovery", URL: server.URL},
RetryEnabled: true,
RetryCount: 1,
}
require.NoError(t, nm.sendWebhookWithRetry(webhook, []byte("{}"), "service:alert"))
require.Len(t, attemptTimes, 2)
first, second := <-attemptTimes, <-attemptTimes
assert.GreaterOrEqual(t, second.Sub(first), tc.minimum)
})
}
}
func TestSendWebhookWithRetry_429RetryAfter(t *testing.T) {
nm := NewNotificationManager("http://pulse.local")
_ = nm.UpdateAllowedPrivateCIDRs("127.0.0.1")
attempts := 0
server := newIPv4HTTPServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
attempts++
if attempts == 1 {
w.Header().Set("Retry-After", "1")
w.WriteHeader(http.StatusTooManyRequests)
return
}
w.WriteHeader(http.StatusOK)
}))
defer server.Close()
webhook := EnhancedWebhookConfig{
WebhookConfig: WebhookConfig{
Name: "Test Webhook",
URL: server.URL,
},
RetryEnabled: true,
RetryCount: 2,
}
err := nm.sendWebhookWithRetry(webhook, []byte(`{"test":true}`), "test-alert:alert")
assert.NoError(t, err)
assert.Equal(t, 2, attempts)
}
func TestSendWebhookWithRetry_InvalidRetryAfterFallsBackToExponentialBackoff(t *testing.T) {
nm := NewNotificationManager("http://pulse.local")
_ = nm.UpdateAllowedPrivateCIDRs("127.0.0.1")
attempts := 0
attemptTimes := make([]time.Time, 0, 2)
server := newIPv4HTTPServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
attempts++
attemptTimes = append(attemptTimes, time.Now())
if attempts == 1 {
w.Header().Set("Retry-After", "not-a-number")
w.WriteHeader(http.StatusTooManyRequests)
return
}
w.WriteHeader(http.StatusOK)
}))
defer server.Close()
webhook := EnhancedWebhookConfig{
WebhookConfig: WebhookConfig{
Name: "Test Webhook",
URL: server.URL,
},
RetryEnabled: true,
RetryCount: 1,
}
start := time.Now()
err := nm.sendWebhookWithRetry(webhook, []byte(`{"test":true}`), "test-alert:alert")
elapsed := time.Since(start)
assert.NoError(t, err)
assert.Equal(t, 2, attempts)
if assert.Len(t, attemptTimes, 2) {
assert.GreaterOrEqual(t, attemptTimes[1].Sub(attemptTimes[0]), 900*time.Millisecond)
}
assert.GreaterOrEqual(t, elapsed, 900*time.Millisecond)
}
func TestSendWebhookWithRetry_RetriesRequestTimeoutResponses(t *testing.T) {
nm := NewNotificationManager("http://pulse.local")
_ = nm.UpdateAllowedPrivateCIDRs("127.0.0.1")
attempts := 0
server := newIPv4HTTPServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
attempts++
if attempts == 1 {
w.WriteHeader(http.StatusRequestTimeout)
return
}
w.WriteHeader(http.StatusOK)
}))
defer server.Close()
webhook := EnhancedWebhookConfig{
WebhookConfig: WebhookConfig{
Name: "Timeout Webhook",
URL: server.URL,
},
RetryEnabled: true,
RetryCount: 1,
}
err := nm.sendWebhookWithRetry(webhook, []byte(`{"test":true}`), "test-alert:alert")
assert.NoError(t, err)
assert.Equal(t, 2, attempts)
}
func TestSendWebhookWithRetry_RetriesMisdirectedRequestResponses(t *testing.T) {
nm := NewNotificationManager("http://pulse.local")
_ = nm.UpdateAllowedPrivateCIDRs("127.0.0.1")
attempts := 0
server := newIPv4HTTPServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
attempts++
if attempts == 1 {
w.WriteHeader(http.StatusMisdirectedRequest)
return
}
w.WriteHeader(http.StatusOK)
}))
defer server.Close()
webhook := EnhancedWebhookConfig{
WebhookConfig: WebhookConfig{
Name: "Misdirected Webhook",
URL: server.URL,
},
RetryEnabled: true,
RetryCount: 1,
}
err := nm.sendWebhookWithRetry(webhook, []byte(`{"test":true}`), "test-alert:alert")
assert.NoError(t, err)
assert.Equal(t, 2, attempts)
}
func TestSendWebhookWithRetry_RetriesLockedResponses(t *testing.T) {
nm := NewNotificationManager("http://pulse.local")
_ = nm.UpdateAllowedPrivateCIDRs("127.0.0.1")
attempts := 0
server := newIPv4HTTPServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
attempts++
if attempts == 1 {
w.WriteHeader(http.StatusLocked)
return
}
w.WriteHeader(http.StatusOK)
}))
defer server.Close()
webhook := EnhancedWebhookConfig{
WebhookConfig: WebhookConfig{
Name: "Locked Webhook",
URL: server.URL,
},
RetryEnabled: true,
RetryCount: 1,
}
err := nm.sendWebhookWithRetry(webhook, []byte(`{"test":true}`), "test-alert:alert")
assert.NoError(t, err)
assert.Equal(t, 2, attempts)
}
func TestSendWebhookWithRetry_RetriesTooEarlyResponses(t *testing.T) {
nm := NewNotificationManager("http://pulse.local")
_ = nm.UpdateAllowedPrivateCIDRs("127.0.0.1")
attempts := 0
server := newIPv4HTTPServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
attempts++
if attempts == 1 {
w.WriteHeader(http.StatusTooEarly)
return
}
w.WriteHeader(http.StatusOK)
}))
defer server.Close()
webhook := EnhancedWebhookConfig{
WebhookConfig: WebhookConfig{
Name: "Too Early Webhook",
URL: server.URL,
},
RetryEnabled: true,
RetryCount: 1,
}
err := nm.sendWebhookWithRetry(webhook, []byte(`{"test":true}`), "test-alert:alert")
assert.NoError(t, err)
assert.Equal(t, 2, attempts)
}
func TestSendWebhookWithRetry_StopsOnNonRetryableErrorAfterRetryable(t *testing.T) {
nm := NewNotificationManager("http://pulse.local")
_ = nm.UpdateAllowedPrivateCIDRs("127.0.0.1")
attempts := 0
server := newIPv4HTTPServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
attempts++
switch attempts {
case 1:
// Keep test fast by making the retry delay zero.
w.Header().Set("Retry-After", "0")
w.WriteHeader(http.StatusTooManyRequests)
case 2:
w.WriteHeader(http.StatusBadRequest)
default:
w.WriteHeader(http.StatusOK)
}
}))
defer server.Close()
webhook := EnhancedWebhookConfig{
WebhookConfig: WebhookConfig{
Name: "Test Webhook",
URL: server.URL,
},
RetryEnabled: true,
RetryCount: 5,
}
err := nm.sendWebhookWithRetry(webhook, []byte(`{"test":true}`), "test-alert:alert")
assert.Error(t, err)
assert.Contains(t, err.Error(), "HTTP 400")
assert.Contains(t, err.Error(), "after 2 attempts")
assert.Equal(t, 2, attempts)
}
func TestSendWebhookWithRetry_PreservesSuccessfulStatusCodeInHistory(t *testing.T) {
nm := NewNotificationManager("http://pulse.local")
_ = nm.UpdateAllowedPrivateCIDRs("127.0.0.1")
server := newIPv4HTTPServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusAccepted)
}))
defer server.Close()
webhook := EnhancedWebhookConfig{
WebhookConfig: WebhookConfig{
Name: "Accepted Webhook",
URL: server.URL,
},
RetryEnabled: true,
RetryCount: 1,
}
err := nm.sendWebhookWithRetry(webhook, []byte(`{"test":true}`), "test-alert:alert")
assert.NoError(t, err)
history := nm.GetWebhookHistory()
if assert.Len(t, history, 1) {
assert.Equal(t, http.StatusAccepted, history[0].StatusCode)
assert.True(t, history[0].Success)
assert.Equal(t, "Accepted Webhook", history[0].WebhookName)
}
}
func TestSendWebhookWithRetry_PreservesFailedStatusCodeInHistory(t *testing.T) {
nm := NewNotificationManager("http://pulse.local")
_ = nm.UpdateAllowedPrivateCIDRs("127.0.0.1")
server := newIPv4HTTPServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusBadRequest)
w.Write([]byte("bad request"))
}))
defer server.Close()
webhook := EnhancedWebhookConfig{
WebhookConfig: WebhookConfig{
Name: "Rejected Webhook",
URL: server.URL,
},
RetryEnabled: true,
RetryCount: 0,
}
err := nm.sendWebhookWithRetry(webhook, []byte(`{"test":true}`), "test-alert:alert")
assert.Error(t, err)
assert.Contains(t, err.Error(), "HTTP 400")
history := nm.GetWebhookHistory()
if assert.Len(t, history, 1) {
assert.Equal(t, http.StatusBadRequest, history[0].StatusCode)
assert.False(t, history[0].Success)
assert.Equal(t, "Rejected Webhook", history[0].WebhookName)
}
}
func TestSendEnhancedWebhook_TemplateRendering(t *testing.T) {
nm := NewNotificationManager("http://pulse.local")
_ = nm.UpdateAllowedPrivateCIDRs("127.0.0.1")
receivedPayload := ""
server := newIPv4HTTPServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
body, _ := io.ReadAll(r.Body)
receivedPayload = string(body)
w.WriteHeader(http.StatusOK)
}))
defer server.Close()
webhook := EnhancedWebhookConfig{
WebhookConfig: WebhookConfig{
Name: "Templated Webhook",
URL: server.URL,
},
PayloadTemplate: `{"alert":"{{.ID}}", "node":"{{.Node}}"}`,
}
alert := &alerts.Alert{
ID: "ALERT-123",
Node: "pve-node",
}
err := nm.SendEnhancedWebhook(webhook, alert)
assert.NoError(t, err)
assert.Contains(t, receivedPayload, "ALERT-123")
assert.Contains(t, receivedPayload, "pve-node")
}
func TestSendEnhancedWebhook_SpecialServices(t *testing.T) {
nm := NewNotificationManager("http://pulse.local")
_ = nm.UpdateAllowedPrivateCIDRs("127.0.0.1")
t.Run("Telegram ChatID", func(t *testing.T) {
server := newIPv4HTTPServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
}))
defer server.Close()
webhook := EnhancedWebhookConfig{
WebhookConfig: WebhookConfig{
URL: server.URL + "?chat_id=12345",
},
Service: "telegram",
PayloadTemplate: "{}",
}
err := nm.SendEnhancedWebhook(webhook, &alerts.Alert{})
assert.NoError(t, err)
})
t.Run("PagerDuty RoutingKey", func(t *testing.T) {
server := newIPv4HTTPServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
}))
defer server.Close()
webhook := EnhancedWebhookConfig{
WebhookConfig: WebhookConfig{
URL: server.URL,
Headers: map[string]string{"routing_key": "abc"},
},
Service: "pagerduty",
PayloadTemplate: "{}",
}
err := nm.SendEnhancedWebhook(webhook, &alerts.Alert{})
assert.NoError(t, err)
})
}
func TestEnhancedWebhook_ntfy(t *testing.T) {
nm := NewNotificationManager("http://pulse.local")
_ = nm.UpdateAllowedPrivateCIDRs("127.0.0.1")
server := newIPv4HTTPServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
assert.Equal(t, "text/plain", r.Header.Get("Content-Type"))
assert.NotEmpty(t, r.Header.Get("Title"))
assert.NotEmpty(t, r.Header.Get("Priority"))
assert.NotEmpty(t, r.Header.Get("Tags"))
w.WriteHeader(http.StatusOK)
}))
defer server.Close()
webhook := EnhancedWebhookConfig{
WebhookConfig: WebhookConfig{
URL: server.URL,
},
Service: "ntfy",
// ntfy doesn't need valid JSON
PayloadTemplate: "Test Alert",
}
status, _, err := nm.TestEnhancedWebhook(webhook)
assert.NoError(t, err)
assert.Equal(t, 200, status)
}
func TestPrepareEnhancedWebhookExecution_UsesCanonicalDeliveryContext(t *testing.T) {
nm := NewNotificationManager("http://pulse.local")
webhook := EnhancedWebhookConfig{
WebhookConfig: WebhookConfig{
Name: "Telegram Test",
URL: "https://example.com/hook?chat_id=12345",
},
Service: "telegram",
PayloadTemplate: `{"chat_id":"{{.ChatID}}","alert":"{{.ID}}"}`,
}
alert := &alerts.Alert{
ID: "alert-123",
Node: "node-a",
}
prepared, payload, err := nm.prepareEnhancedWebhookExecution(webhook, alert)
require.NoError(t, err)
assert.Equal(t, "https://example.com/hook?chat_id=12345", prepared.URL)
assert.JSONEq(t, `{"chat_id":"12345","alert":"alert-123"}`, string(payload))
}
func TestPrepareEnhancedWebhookExecution_NtfyPreservesPlainTextPayload(t *testing.T) {
nm := NewNotificationManager("http://pulse.local")
webhook := EnhancedWebhookConfig{
WebhookConfig: WebhookConfig{
Name: "ntfy Test",
URL: "https://example.com/topic",
},
Service: "ntfy",
PayloadTemplate: "Alert for {{.ResourceName}} on {{.Node}}",
}
alert := &alerts.Alert{
ResourceName: "vm-101",
Node: "pve-01",
}
prepared, payload, err := nm.prepareEnhancedWebhookExecution(webhook, alert)
require.NoError(t, err)
assert.Equal(t, "https://example.com/topic", prepared.URL)
assert.Equal(t, "Alert for vm-101 on pve-01", string(payload))
}
func TestPrepareEnhancedWebhookExecution_NtfyRendersBuiltinPresetTemplate(t *testing.T) {
nm := NewNotificationManager("http://pulse.local")
var preset *WebhookTemplate
for i := range GetWebhookTemplates() {
tmpl := GetWebhookTemplates()[i]
if tmpl.Service == "ntfy" {
preset = &tmpl
break
}
}
require.NotNil(t, preset, "built-in ntfy webhook template missing")
webhook := EnhancedWebhookConfig{
WebhookConfig: WebhookConfig{
Name: "ntfy Preset Test",
URL: "https://example.com/topic",
},
Service: "ntfy",
PayloadTemplate: preset.PayloadTemplate,
}
alert := &alerts.Alert{
ID: "alert-123",
Type: "cpu",
Level: "warning",
ResourceName: "vm-101",
Node: "pve-01",
Message: "CPU usage above threshold",
Value: 92.5,
Threshold: 90,
}
prepared, payload, err := nm.prepareEnhancedWebhookExecution(webhook, alert)
require.NoError(t, err)
assert.Equal(t, "https://example.com/topic", prepared.URL)
assert.Contains(t, string(payload), "WARNING: vm-101 on pve-01")
assert.Contains(t, string(payload), "- Type: Cpu")
}
func TestBuildEnhancedWebhookTestConfig_UsesServiceTemplateOwnership(t *testing.T) {
basic := WebhookConfig{
Name: "Discord test",
URL: "https://example.com/hook",
Headers: map[string]string{
"X-Custom": "value",
},
}
webhook := BuildEnhancedWebhookTestConfig(basic, "discord")
assert.Equal(t, "discord", webhook.Service)
assert.NotEmpty(t, webhook.PayloadTemplate)
assert.Equal(t, "value", webhook.Headers["X-Custom"])
assert.Equal(t, "application/json", webhook.Headers["Content-Type"])
}
func TestBuildEnhancedWebhookTestConfig_FallsBackToGenericTemplate(t *testing.T) {
basic := WebhookConfig{
Name: "Unknown service test",
URL: "https://example.com/hook",
}
webhook := BuildEnhancedWebhookTestConfig(basic, "unknown-service")
assert.Equal(t, "unknown-service", webhook.Service)
assert.Contains(t, webhook.PayloadTemplate, `"source": "pulse-monitoring"`)
assert.Contains(t, webhook.PayloadTemplate, `"resourceName": "{{.ResourceName}}"`)
}
func TestEnhancedWebhook_UsesCanonicalTransportSanitization(t *testing.T) {
nm := NewNotificationManager("http://pulse.local")
_ = nm.UpdateAllowedPrivateCIDRs("127.0.0.1")
var gotQuery string
var gotBody []byte
server := newIPv4HTTPServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
gotQuery = r.URL.RawQuery
body, err := io.ReadAll(r.Body)
require.NoError(t, err)
gotBody = body
w.WriteHeader(http.StatusOK)
w.Write([]byte("ok"))
}))
defer server.Close()
webhook := EnhancedWebhookConfig{
WebhookConfig: WebhookConfig{
Name: "Telegram Test",
URL: server.URL + "/hook?chat_id=12345",
},
Service: "telegram",
PayloadTemplate: `{"chat_id":"{{.ChatID}}","alert":"{{.ID}}"}`,
}
alert := &alerts.Alert{
ID: "alert-telegram-1",
Node: "node-a",
}
err := nm.SendEnhancedWebhook(webhook, alert)
require.NoError(t, err)
assert.Empty(t, gotQuery)
assert.JSONEq(t, `{"chat_id":"12345","alert":"alert-telegram-1"}`, string(gotBody))
}
func TestIsRetryableWebhookErrorEnhanced(t *testing.T) {
tests := []struct {
err string
expected bool
}{
{"timeout", true},
{"connection refused", true},
{"status 429", true},
{"http 429", true},
{"status 502", true},
{"status 404", false},
{"status 400", false},
{"webhook returned HTTP 400: Bad Request", false},
}
for _, tt := range tests {
t.Run(tt.err, func(t *testing.T) {
result := isRetryableWebhookError(fmt.Errorf("%s", tt.err))
assert.Equal(t, tt.expected, result)
})
}
}
// A response-specific zero delay must not erase backoff for later failures.
func TestSendWebhookWithRetry_ZeroRetryAfterPreservesLaterBackoff(t *testing.T) {
for _, firstStatus := range []int{http.StatusTooManyRequests, http.StatusServiceUnavailable} {
for _, status := range []int{http.StatusServiceUnavailable, http.StatusTooManyRequests} {
t.Run(fmt.Sprintf("%d_then_%d", firstStatus, status), func(t *testing.T) {
nm := NewNotificationManager("http://pulse.local")
t.Cleanup(nm.Stop)
require.NoError(t, nm.UpdateAllowedPrivateCIDRs("127.0.0.1"))
attemptTimes := make(chan time.Time, 3)
attempts := 0
server := newIPv4HTTPServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
attempts++
attemptTimes <- time.Now()
switch attempts {
case 1:
w.Header().Set("Retry-After", "0")
w.WriteHeader(firstStatus)
case 2:
w.WriteHeader(status)
default:
w.WriteHeader(http.StatusOK)
}
}))
defer server.Close()
webhook := EnhancedWebhookConfig{
WebhookConfig: WebhookConfig{Name: "Backoff regression", URL: server.URL},
RetryEnabled: true,
RetryCount: 2,
}
require.NoError(t, nm.sendWebhookWithRetry(webhook, []byte("{}"), "backoff:alert"))
require.Len(t, attemptTimes, 3)
<-attemptTimes
second, third := <-attemptTimes, <-attemptTimes
assert.GreaterOrEqual(t, third.Sub(second), 2*WebhookInitialBackoff,
"later failure must retain exponential backoff after a zero Retry-After")
})
}
}
}