Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
package notifications
import (
"errors"
"fmt"
"net/http"
"regexp"
"strconv"
"strings"
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
"github.com/rs/zerolog/log"
)
var webhookHTTPStatusPattern = regexp.MustCompile(`\b(?:status|http)\s+(\d{3})\b`)
// EnhancedWebhookConfig extends WebhookConfig with template support
type EnhancedWebhookConfig struct {
WebhookConfig
Service string `json:"service"` // discord, slack, teams, pagerduty, generic
PayloadTemplate string `json:"payloadTemplate"` // Go template for payload
RetryEnabled bool `json:"retryEnabled"`
RetryCount int `json:"retryCount"`
FilterRules WebhookFilterRules `json:"filterRules"`
CustomFields map[string]interface{} `json:"customFields"` // For template variables
ResponseLogging bool `json:"responseLogging"` // Log structured response metadata, never provider text
}
// WebhookFilterRules defines filtering for this webhook
type WebhookFilterRules struct {
Levels []string `json:"levels"` // Only send these levels
Types []string `json:"types"` // Only send these alert types
Nodes []string `json:"nodes"` // Only send from these nodes
ResourceTypes []string `json:"resourceTypes"` // vm, container, storage, etc
}
// WebhookPayloadData contains all data available for templates
type WebhookPayloadData struct {
// Alert fields
ID string
MessageKey string
Level string
Type string
ResourceName string
ResourceID string
Node string
NodeDisplayName string
Instance string
// AlertURL opens this alert in Pulse's Alert History, or the Alerts page
// for a grouped notification; it falls back to Instance when no public
// URL is configured.
AlertURL string
Message string
Value float64
Threshold float64
ValueFormatted string
ThresholdFormatted string
StartTime string
Duration string
Timestamp string
ResourceType string
Acknowledged bool
AckTime string
AckUser string
// Event context (alert vs resolved)
Event string // "alert" or "resolved"
ResolvedAt string // RFC3339 timestamp when alert was resolved
ResolvedAtISO string // Same as ResolvedAt (alias for template consistency)
// NotRecovered marks a resolved notification in which at least one close
// was not a recovery (an alert that moved to a linked Pulse agent), so
// templates can drop recovery colours and check marks.
NotRecovered bool
// Tenant identity (multi-tenant orgs and MSP client runtimes).
// Empty on plain single-tenant installs.
TenantID string
TenantName string
// Additional context
Metadata map[string]interface{}
CustomFields map[string]interface{}
AlertCount int
Alerts []*alerts.Alert // For grouped alerts
ChatID string // For Telegram webhooks
Mention string // Platform-specific mention text
}
func canonicalWebhookConfigForEnhanced(webhook EnhancedWebhookConfig) WebhookConfig {
canonical := webhook.WebhookConfig
if strings.TrimSpace(webhook.Service) != "" {
canonical.Service = webhook.Service
}
if strings.TrimSpace(webhook.PayloadTemplate) != "" {
canonical.Template = webhook.PayloadTemplate
}
return canonical
}
func BuildEnhancedWebhookTestConfig(basicWebhook WebhookConfig, requestedService string) EnhancedWebhookConfig {
webhook := EnhancedWebhookConfig{
WebhookConfig: basicWebhook,
Service: "generic",
RetryEnabled: false,
}
if len(basicWebhook.CustomFields) > 0 {
customFields := make(map[string]interface{}, len(basicWebhook.CustomFields))
for key, value := range basicWebhook.CustomFields {
customFields[key] = value
}
webhook.CustomFields = customFields
}
if basicWebhook.Template != "" {
webhook.PayloadTemplate = basicWebhook.Template
}
if strings.TrimSpace(requestedService) != "" {
webhook.Service = requestedService
webhook.WebhookConfig.Service = requestedService
} else if strings.TrimSpace(basicWebhook.Service) != "" {
webhook.Service = basicWebhook.Service
}
if webhook.PayloadTemplate == "" {
templates := GetWebhookTemplates()
for _, tmpl := range templates {
if tmpl.Service != webhook.Service {
continue
}
webhook.PayloadTemplate = tmpl.PayloadTemplate
webhook.WebhookConfig = withWebhookTemplateHeaders(webhook.WebhookConfig, tmpl.Headers)
break
}
}
if webhook.PayloadTemplate == "" {
webhook.PayloadTemplate = `{
"alert": {
"id": "{{.ID}}",
"type": "{{.Type}}",
"level": "{{.Level}}",
"resourceName": "{{.ResourceName}}",
"node": "{{.Node}}",
"message": "{{.Message}}",
"value": {{.Value}},
"threshold": {{.Threshold}}
},
"source": "pulse-monitoring",
"timestamp": {{.Timestamp}}
}`
}
return webhook
}
func (n *NotificationManager) prepareEnhancedWebhookExecution(webhook EnhancedWebhookConfig, alert *alerts.Alert) (EnhancedWebhookConfig, []byte, error) {
data := n.prepareWebhookData(alert, webhook.CustomFields)
canonical := canonicalWebhookConfigForEnhanced(webhook)
var err error
canonical, data, err = n.prepareWebhookDeliveryContext(canonical, data)
if err != nil {
return webhook, nil, fmt.Errorf("prepare enhanced webhook context: %w", err)
}
webhook.WebhookConfig = canonical
payload, err := n.renderWebhookPayloadJSON(canonical, data, webhookRenderModeSingle, func() ([]byte, error) {
if strings.TrimSpace(webhook.PayloadTemplate) == "" {
return nil, fmt.Errorf("enhanced webhook requires payload template or known service template")
}
return n.generatePayloadFromTemplateWithService(webhook.PayloadTemplate, data, webhook.Service)
})
if err != nil {
return webhook, nil, fmt.Errorf("render enhanced webhook payload: %w", err)
}
return webhook, payload, nil
}
// SendEnhancedWebhook sends a webhook with template support
func (n *NotificationManager) SendEnhancedWebhook(webhook EnhancedWebhookConfig, alert *alerts.Alert) error {
// Check filters
if !n.shouldSendWebhook(webhook, alert) {
log.Debug().
Str("webhook", webhook.Name).
Str("alertID", alert.ID).
Msg("alert filtered out by webhook rules")
return nil
}
webhook, payload, err := n.prepareEnhancedWebhookExecution(webhook, alert)
if err != nil {
return err
}
if !n.checkWebhookRateLimit(webhook.URL) {
log.Warn().
Str("webhook", webhook.Name).
Str("url", RedactWebhookURLSecrets(webhook.URL)).
Msg("webhook request dropped due to rate limiting")
return fmt.Errorf("rate limit exceeded for webhook %s", webhook.Name)
}
eventID := webhookEventID(alert.ID, "alert")
// Send with retry logic
if webhook.RetryEnabled {
return n.sendWebhookWithRetry(webhook, payload, eventID)
}
_, err = n.executeEnhancedWebhookRequest(webhook, payload, WebhookTimeout, "Pulse-Monitoring/2.0", eventID)
if err != nil {
return fmt.Errorf("send webhook %q once: %w", webhook.Name, err)
}
return nil
}
// NOTE: prepareWebhookData is now defined in notifications.go to avoid duplication
// NOTE: generatePayloadFromTemplate is now defined in notifications.go to avoid duplication
// shouldSendWebhook checks if alert matches webhook filter rules
func (n *NotificationManager) shouldSendWebhook(webhook EnhancedWebhookConfig, alert *alerts.Alert) bool {
rules := webhook.FilterRules
// Check level filter
if len(rules.Levels) > 0 {
found := false
for _, level := range rules.Levels {
if string(alert.Level) == level {
found = true
break
}
}
if !found {
return false
}
}
// Check type filter
if len(rules.Types) > 0 {
found := false
for _, alertType := range rules.Types {
if alert.Type == alertType {
found = true
break
}
}
if !found {
return false
}
}
// Check node filter
if len(rules.Nodes) > 0 {
found := false
for _, node := range rules.Nodes {
if alert.Node == node {
found = true
break
}
}
if !found {
return false
}
}
// Check resource type filter
if len(rules.ResourceTypes) > 0 {
resourceType, ok := alert.Metadata["resourceType"].(string)
if !ok {
return false
}
found := false
for _, rt := range rules.ResourceTypes {
if resourceType == rt {
found = true
break
}
}
if !found {
return false
}
}
return true
}
// sendWebhookWithRetry implements exponential backoff retry with enhanced error tracking
// Note: When used with the persistent queue, retry behavior is layered:
// - Transport: one initial attempt plus RetryCount retries (non-positive counts use WebhookDefaultRetries)
// - Queue: up to MaxAttempts delivery attempts with exponential backoff
// The HTTP attempt ceiling is (effective RetryCount + 1) * MaxAttempts when
// each queue delivery uses this transport. Permanent errors can stop earlier;
// exhausting either retry budget does not guarantee delivery.
func (n *NotificationManager) sendWebhookWithRetry(webhook EnhancedWebhookConfig, payload []byte, eventID string) error {
maxRetries := webhook.RetryCount
if maxRetries <= 0 {
maxRetries = WebhookDefaultRetries
}
var lastErr error
var lastResp *webhookHTTPResult
backoff := WebhookInitialBackoff
retryableErrors := 0
totalAttempts := 0
for attempt := 0; attempt <= maxRetries; attempt++ {
if attempt > 0 {
// Check for Retry-After header from previous response (overrides backoff)
// Service-unavailable responses can advertise recovery time too
// (RFC 9110 section 10.2.3). Keep the same bounded parser and budget.
if lastResp != nil && (lastResp.statusCode == http.StatusTooManyRequests || lastResp.statusCode == http.StatusServiceUnavailable) {
usedBackoff := false
if retryAfter := lastResp.headers.Get("Retry-After"); retryAfter != "" {
if customBackoff, ok := parseRetryAfterBackoff(retryAfter, time.Now()); ok {
log.Debug().
Str("webhook", webhook.Name).
Dur("retryAfter", customBackoff).
Msg("using Retry-After header for backoff")
time.Sleep(customBackoff)
// The header overrides this wait only. Preserve the exponential
// schedule so a zero delay cannot erase later failure backoff.
usedBackoff = true
} else {
log.Debug().
Str("webhook", webhook.Name).
Msg("invalid Retry-After header; falling back to exponential backoff")
}
}
if !usedBackoff {
// No Retry-After, use exponential backoff
log.Debug().
Str("webhook", webhook.Name).
Int("attempt", attempt).
Int("maxRetries", maxRetries).
Dur("backoff", backoff).
Msg("retrying webhook after backoff")
time.Sleep(backoff)
}
} else {
// No supported retry hint, use exponential backoff
log.Debug().
Str("webhook", webhook.Name).
Int("attempt", attempt).
Int("maxRetries", maxRetries).
Dur("backoff", backoff).
Msg("retrying webhook after backoff")
time.Sleep(backoff)
}
// Exponential backoff for next iteration
backoff *= 2
if backoff > WebhookMaxBackoff {
backoff = WebhookMaxBackoff
}
}
resp, err := n.executeEnhancedWebhookRequest(webhook, payload, WebhookTimeout, "Pulse-Monitoring/2.0", eventID)
totalAttempts = attempt + 1
lastResp = resp
if err == nil {
if attempt > 0 {
log.Info().
Str("webhook", webhook.Name).
Int("attempt", attempt).
Int("totalAttempts", attempt+1).
Msg("webhook succeeded after retry")
}
// Log successful delivery
log.Debug().
Str("webhook", webhook.Name).
Str("service", webhook.Service).
Int("payloadSize", len(payload)).
Msg("webhook delivered successfully")
// Track successful delivery
statusCode := http.StatusOK
if resp != nil {
statusCode = resp.statusCode
}
delivery := WebhookDelivery{
WebhookName: webhook.Name,
WebhookURL: webhook.URL,
Service: webhook.Service,
AlertIdentifier: "enhanced", // Enhanced webhooks are service-level deliveries, not alert-bound.
Timestamp: time.Now(),
StatusCode: statusCode,
Success: true,
RetryAttempts: attempt,
PayloadSize: len(payload),
}
n.addWebhookDelivery(delivery)
return nil
}
lastErr = err
// Determine if error is retryable
isRetryable := isRetryableWebhookError(err)
if isRetryable {
retryableErrors++
}
log.Warn().
Err(err).
Str("webhook", webhook.Name).
Str("service", webhook.Service).
Int("attempt", attempt+1).
Int("maxRetries", maxRetries+1).
Bool("retryable", isRetryable).
Msg("webhook attempt failed")
// If error is not retryable, stop immediately regardless of attempt.
if !isRetryable {
log.Error().
Err(err).
Str("webhook", webhook.Name).
Msg("non-retryable webhook error - not attempting retry")
break
}
}
if totalAttempts == 0 {
totalAttempts = maxRetries + 1
}
retryAttempts := totalAttempts - 1
statusCode := 0
if lastResp != nil {
statusCode = lastResp.statusCode
}
// Final error logging with summary
log.Error().
Err(lastErr).
Str("webhook", webhook.Name).
Str("service", webhook.Service).
Int("totalAttempts", totalAttempts).
Int("retryableErrors", retryableErrors).
Msg("webhook delivery failed after all retry attempts")
// Track failed delivery
delivery := WebhookDelivery{
WebhookName: webhook.Name,
WebhookURL: webhook.URL,
Service: webhook.Service,
AlertIdentifier: "enhanced", // Enhanced webhooks are service-level deliveries, not alert-bound.
Timestamp: time.Now(),
StatusCode: statusCode,
Success: false,
ErrorMessage: lastErr.Error(),
RetryAttempts: retryAttempts,
PayloadSize: len(payload),
}
n.addWebhookDelivery(delivery)
return fmt.Errorf("webhook failed after %d attempts: %w", totalAttempts, lastErr)
}
func parseRetryAfterBackoff(retryAfter string, now time.Time) (time.Duration, bool) {
retryAfter = strings.TrimSpace(retryAfter)
clampBackoff := func(backoff time.Duration) time.Duration {
if backoff < 0 {
return 0
}
if backoff > WebhookMaxBackoff {
return WebhookMaxBackoff
}
return backoff
}
if seconds, err := strconv.ParseInt(retryAfter, 10, 64); err == nil {
// Clamp in seconds before conversion: multiplying an untrusted value
// by time.Second can overflow and turn a long delay into no delay.
if seconds <= 0 {
return 0, true
}
if seconds > int64(WebhookMaxBackoff/time.Second) {
return WebhookMaxBackoff, true
}
return clampBackoff(time.Duration(seconds) * time.Second), true
}
if retryAt, err := http.ParseTime(retryAfter); err == nil {
return clampBackoff(retryAt.Sub(now)), true
}
return 0, false
}
// isRetryableWebhookError determines if a webhook error should trigger a retry
func isRetryableWebhookError(err error) bool {
// A local security/configuration refusal is authoritative too. In
// particular, retrying an origin-changing redirect cannot make it safe.
var declared *NotificationFailureError
if errors.As(err, &declared) && declared.Class != "" {
return declared.Class.Retryable()
}
// An explicit HTTP rejection takes precedence over diagnostic body text:
// a 403 mentioning "timeout" is not a transport timeout. Errors without
// a recognised status (including network failures) remain retryable below.
if statusCode, ok := webhookErrorStatusCode(err); ok {
switch statusCode {
case http.StatusRequestTimeout, http.StatusMisdirectedRequest, http.StatusLocked, http.StatusTooEarly, http.StatusTooManyRequests:
return true
}
if statusCode >= 500 && statusCode <= 599 {
return true
}
if statusCode >= 400 && statusCode <= 499 {
return false
}
}
// Default to retryable for unknown errors
return true
}
func webhookErrorStatusCode(err error) (int, bool) {
if err == nil {
return 0, false
}
match := webhookHTTPStatusPattern.FindStringSubmatch(strings.ToLower(err.Error()))
if len(match) != 2 {
return 0, false
}
statusCode, parseErr := strconv.Atoi(match[1])
if parseErr != nil {
return 0, false
}
return statusCode, true
}
// executeEnhancedWebhookRequest sends a single enhanced webhook request and returns response metadata.
func (n *NotificationManager) executeEnhancedWebhookRequest(webhook EnhancedWebhookConfig, payload []byte, timeout time.Duration, userAgent string, eventID string) (*webhookHTTPResult, error) {
canonical := canonicalWebhookConfigForEnhanced(webhook)
return n.executeWebhookRequest(canonical, payload, webhookRequestOptions{
eventID: eventID,
timeout: timeout,
userAgent: userAgent,
responseLogging: webhook.ResponseLogging,
validateURL: true,
})
}
// NOTE: formatWebhookDuration is now defined in notifications.go to avoid duplication
// TestEnhancedWebhook tests a webhook with a specific payload
func (n *NotificationManager) TestEnhancedWebhook(webhook EnhancedWebhookConfig) (int, string, error) {
testAlert := buildNotificationTestAlert()
testAlert.ID = "test-" + time.Now().Format("20060102-150405")
if n.publicURL != "" {
testAlert.Instance = n.publicURL
}
var err error
webhook, payload, err := n.prepareEnhancedWebhookExecution(webhook, testAlert)
if err != nil {
return 0, "", err
}
if webhook.Service == "ntfy" {
canonical := withNtfyAlertHeaders(canonicalWebhookConfigForEnhanced(webhook), []*alerts.Alert{testAlert})
webhook.Headers = canonical.Headers
}
result, err := n.executeEnhancedWebhookRequest(webhook, payload, WebhookTestTimeout, "Pulse-Monitoring/2.0 (Test)", webhookEventID(testAlert.ID, "alert"))
if err != nil {
if result != nil {
return result.statusCode, result.responseSummary(), err
}
return 0, "", err
}
return result.statusCode, result.responseSummary(), nil
}