Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

17.5 KB
Raw Blame
package notifications

import (
	"errors"
	"fmt"
	"net/http"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
	"github.com/rs/zerolog/log"
)

var webhookHTTPStatusPattern = regexp.MustCompile(`\b(?:status|http)\s+(\d{3})\b`)

// EnhancedWebhookConfig extends WebhookConfig with template support
type EnhancedWebhookConfig struct {
	WebhookConfig
	Service         string                 `json:"service"`         // discord, slack, teams, pagerduty, generic
	PayloadTemplate string                 `json:"payloadTemplate"` // Go template for payload
	RetryEnabled    bool                   `json:"retryEnabled"`
	RetryCount      int                    `json:"retryCount"`
	FilterRules     WebhookFilterRules     `json:"filterRules"`
	CustomFields    map[string]interface{} `json:"customFields"`    // For template variables
	ResponseLogging bool                   `json:"responseLogging"` // Log structured response metadata, never provider text
}

// WebhookFilterRules defines filtering for this webhook
type WebhookFilterRules struct {
	Levels        []string `json:"levels"`        // Only send these levels
	Types         []string `json:"types"`         // Only send these alert types
	Nodes         []string `json:"nodes"`         // Only send from these nodes
	ResourceTypes []string `json:"resourceTypes"` // vm, container, storage, etc
}

// WebhookPayloadData contains all data available for templates
type WebhookPayloadData struct {
	// Alert fields
	ID              string
	MessageKey      string
	Level           string
	Type            string
	ResourceName    string
	ResourceID      string
	Node            string
	NodeDisplayName string
	Instance        string
	// AlertURL opens this alert in Pulse's Alert History, or the Alerts page
	// for a grouped notification; it falls back to Instance when no public
	// URL is configured.
	AlertURL           string
	Message            string
	Value              float64
	Threshold          float64
	ValueFormatted     string
	ThresholdFormatted string
	StartTime          string
	Duration           string
	Timestamp          string
	ResourceType       string
	Acknowledged       bool
	AckTime            string
	AckUser            string

	// Event context (alert vs resolved)
	Event         string // "alert" or "resolved"
	ResolvedAt    string // RFC3339 timestamp when alert was resolved
	ResolvedAtISO string // Same as ResolvedAt (alias for template consistency)
	// NotRecovered marks a resolved notification in which at least one close
	// was not a recovery (an alert that moved to a linked Pulse agent), so
	// templates can drop recovery colours and check marks.
	NotRecovered bool

	// Tenant identity (multi-tenant orgs and MSP client runtimes).
	// Empty on plain single-tenant installs.
	TenantID   string
	TenantName string

	// Additional context
	Metadata     map[string]interface{}
	CustomFields map[string]interface{}
	AlertCount   int
	Alerts       []*alerts.Alert // For grouped alerts
	ChatID       string          // For Telegram webhooks
	Mention      string          // Platform-specific mention text
}

func canonicalWebhookConfigForEnhanced(webhook EnhancedWebhookConfig) WebhookConfig {
	canonical := webhook.WebhookConfig
	if strings.TrimSpace(webhook.Service) != "" {
		canonical.Service = webhook.Service
	}
	if strings.TrimSpace(webhook.PayloadTemplate) != "" {
		canonical.Template = webhook.PayloadTemplate
	}
	return canonical
}

func BuildEnhancedWebhookTestConfig(basicWebhook WebhookConfig, requestedService string) EnhancedWebhookConfig {
	webhook := EnhancedWebhookConfig{
		WebhookConfig: basicWebhook,
		Service:       "generic",
		RetryEnabled:  false,
	}

	if len(basicWebhook.CustomFields) > 0 {
		customFields := make(map[string]interface{}, len(basicWebhook.CustomFields))
		for key, value := range basicWebhook.CustomFields {
			customFields[key] = value
		}
		webhook.CustomFields = customFields
	}

	if basicWebhook.Template != "" {
		webhook.PayloadTemplate = basicWebhook.Template
	}

	if strings.TrimSpace(requestedService) != "" {
		webhook.Service = requestedService
		webhook.WebhookConfig.Service = requestedService
	} else if strings.TrimSpace(basicWebhook.Service) != "" {
		webhook.Service = basicWebhook.Service
	}

	if webhook.PayloadTemplate == "" {
		templates := GetWebhookTemplates()
		for _, tmpl := range templates {
			if tmpl.Service != webhook.Service {
				continue
			}
			webhook.PayloadTemplate = tmpl.PayloadTemplate
			webhook.WebhookConfig = withWebhookTemplateHeaders(webhook.WebhookConfig, tmpl.Headers)
			break
		}
	}

	if webhook.PayloadTemplate == "" {
		webhook.PayloadTemplate = `{
			"alert": {
				"id": "{{.ID}}",
				"type": "{{.Type}}",
				"level": "{{.Level}}",
				"resourceName": "{{.ResourceName}}",
				"node": "{{.Node}}",
				"message": "{{.Message}}",
				"value": {{.Value}},
				"threshold": {{.Threshold}}
			},
			"source": "pulse-monitoring",
			"timestamp": {{.Timestamp}}
		}`
	}

	return webhook
}

func (n *NotificationManager) prepareEnhancedWebhookExecution(webhook EnhancedWebhookConfig, alert *alerts.Alert) (EnhancedWebhookConfig, []byte, error) {
	data := n.prepareWebhookData(alert, webhook.CustomFields)
	canonical := canonicalWebhookConfigForEnhanced(webhook)

	var err error
	canonical, data, err = n.prepareWebhookDeliveryContext(canonical, data)
	if err != nil {
		return webhook, nil, fmt.Errorf("prepare enhanced webhook context: %w", err)
	}
	webhook.WebhookConfig = canonical

	payload, err := n.renderWebhookPayloadJSON(canonical, data, webhookRenderModeSingle, func() ([]byte, error) {
		if strings.TrimSpace(webhook.PayloadTemplate) == "" {
			return nil, fmt.Errorf("enhanced webhook requires payload template or known service template")
		}
		return n.generatePayloadFromTemplateWithService(webhook.PayloadTemplate, data, webhook.Service)
	})
	if err != nil {
		return webhook, nil, fmt.Errorf("render enhanced webhook payload: %w", err)
	}

	return webhook, payload, nil
}

// SendEnhancedWebhook sends a webhook with template support
func (n *NotificationManager) SendEnhancedWebhook(webhook EnhancedWebhookConfig, alert *alerts.Alert) error {
	// Check filters
	if !n.shouldSendWebhook(webhook, alert) {
		log.Debug().
			Str("webhook", webhook.Name).
			Str("alertID", alert.ID).
			Msg("alert filtered out by webhook rules")
		return nil
	}

	webhook, payload, err := n.prepareEnhancedWebhookExecution(webhook, alert)
	if err != nil {
		return err
	}

	if !n.checkWebhookRateLimit(webhook.URL) {
		log.Warn().
			Str("webhook", webhook.Name).
			Str("url", RedactWebhookURLSecrets(webhook.URL)).
			Msg("webhook request dropped due to rate limiting")
		return fmt.Errorf("rate limit exceeded for webhook %s", webhook.Name)
	}

	eventID := webhookEventID(alert.ID, "alert")

	// Send with retry logic
	if webhook.RetryEnabled {
		return n.sendWebhookWithRetry(webhook, payload, eventID)
	}

	_, err = n.executeEnhancedWebhookRequest(webhook, payload, WebhookTimeout, "Pulse-Monitoring/2.0", eventID)
	if err != nil {
		return fmt.Errorf("send webhook %q once: %w", webhook.Name, err)
	}
	return nil
}

// NOTE: prepareWebhookData is now defined in notifications.go to avoid duplication
// NOTE: generatePayloadFromTemplate is now defined in notifications.go to avoid duplication

// shouldSendWebhook checks if alert matches webhook filter rules
func (n *NotificationManager) shouldSendWebhook(webhook EnhancedWebhookConfig, alert *alerts.Alert) bool {
	rules := webhook.FilterRules

	// Check level filter
	if len(rules.Levels) > 0 {
		found := false
		for _, level := range rules.Levels {
			if string(alert.Level) == level {
				found = true
				break
			}
		}
		if !found {
			return false
		}
	}

	// Check type filter
	if len(rules.Types) > 0 {
		found := false
		for _, alertType := range rules.Types {
			if alert.Type == alertType {
				found = true
				break
			}
		}
		if !found {
			return false
		}
	}

	// Check node filter
	if len(rules.Nodes) > 0 {
		found := false
		for _, node := range rules.Nodes {
			if alert.Node == node {
				found = true
				break
			}
		}
		if !found {
			return false
		}
	}

	// Check resource type filter
	if len(rules.ResourceTypes) > 0 {
		resourceType, ok := alert.Metadata["resourceType"].(string)
		if !ok {
			return false
		}
		found := false
		for _, rt := range rules.ResourceTypes {
			if resourceType == rt {
				found = true
				break
			}
		}
		if !found {
			return false
		}
	}

	return true
}

// sendWebhookWithRetry implements exponential backoff retry with enhanced error tracking
// Note: When used with the persistent queue, retry behavior is layered:
// - Transport: one initial attempt plus RetryCount retries (non-positive counts use WebhookDefaultRetries)
// - Queue: up to MaxAttempts delivery attempts with exponential backoff
// The HTTP attempt ceiling is (effective RetryCount + 1) * MaxAttempts when
// each queue delivery uses this transport. Permanent errors can stop earlier;
// exhausting either retry budget does not guarantee delivery.
func (n *NotificationManager) sendWebhookWithRetry(webhook EnhancedWebhookConfig, payload []byte, eventID string) error {
	maxRetries := webhook.RetryCount
	if maxRetries <= 0 {
		maxRetries = WebhookDefaultRetries
	}

	var lastErr error
	var lastResp *webhookHTTPResult
	backoff := WebhookInitialBackoff
	retryableErrors := 0
	totalAttempts := 0

	for attempt := 0; attempt <= maxRetries; attempt++ {
		if attempt > 0 {
			// Check for Retry-After header from previous response (overrides backoff)
			// Service-unavailable responses can advertise recovery time too
			// (RFC 9110 section 10.2.3). Keep the same bounded parser and budget.
			if lastResp != nil && (lastResp.statusCode == http.StatusTooManyRequests || lastResp.statusCode == http.StatusServiceUnavailable) {
				usedBackoff := false
				if retryAfter := lastResp.headers.Get("Retry-After"); retryAfter != "" {
					if customBackoff, ok := parseRetryAfterBackoff(retryAfter, time.Now()); ok {
						log.Debug().
							Str("webhook", webhook.Name).
							Dur("retryAfter", customBackoff).
							Msg("using Retry-After header for backoff")
						time.Sleep(customBackoff)
						// The header overrides this wait only. Preserve the exponential
						// schedule so a zero delay cannot erase later failure backoff.
						usedBackoff = true
					} else {
						log.Debug().
							Str("webhook", webhook.Name).
							Msg("invalid Retry-After header; falling back to exponential backoff")
					}
				}
				if !usedBackoff {
					// No Retry-After, use exponential backoff
					log.Debug().
						Str("webhook", webhook.Name).
						Int("attempt", attempt).
						Int("maxRetries", maxRetries).
						Dur("backoff", backoff).
						Msg("retrying webhook after backoff")
					time.Sleep(backoff)
				}
			} else {
				// No supported retry hint, use exponential backoff
				log.Debug().
					Str("webhook", webhook.Name).
					Int("attempt", attempt).
					Int("maxRetries", maxRetries).
					Dur("backoff", backoff).
					Msg("retrying webhook after backoff")
				time.Sleep(backoff)
			}

			// Exponential backoff for next iteration
			backoff *= 2
			if backoff > WebhookMaxBackoff {
				backoff = WebhookMaxBackoff
			}
		}

		resp, err := n.executeEnhancedWebhookRequest(webhook, payload, WebhookTimeout, "Pulse-Monitoring/2.0", eventID)
		totalAttempts = attempt + 1
		lastResp = resp
		if err == nil {
			if attempt > 0 {
				log.Info().
					Str("webhook", webhook.Name).
					Int("attempt", attempt).
					Int("totalAttempts", attempt+1).
					Msg("webhook succeeded after retry")
			}
			// Log successful delivery
			log.Debug().
				Str("webhook", webhook.Name).
				Str("service", webhook.Service).
				Int("payloadSize", len(payload)).
				Msg("webhook delivered successfully")

			// Track successful delivery
			statusCode := http.StatusOK
			if resp != nil {
				statusCode = resp.statusCode
			}
			delivery := WebhookDelivery{
				WebhookName:     webhook.Name,
				WebhookURL:      webhook.URL,
				Service:         webhook.Service,
				AlertIdentifier: "enhanced", // Enhanced webhooks are service-level deliveries, not alert-bound.
				Timestamp:       time.Now(),
				StatusCode:      statusCode,
				Success:         true,
				RetryAttempts:   attempt,
				PayloadSize:     len(payload),
			}
			n.addWebhookDelivery(delivery)

			return nil
		}

		lastErr = err

		// Determine if error is retryable
		isRetryable := isRetryableWebhookError(err)
		if isRetryable {
			retryableErrors++
		}

		log.Warn().
			Err(err).
			Str("webhook", webhook.Name).
			Str("service", webhook.Service).
			Int("attempt", attempt+1).
			Int("maxRetries", maxRetries+1).
			Bool("retryable", isRetryable).
			Msg("webhook attempt failed")

		// If error is not retryable, stop immediately regardless of attempt.
		if !isRetryable {
			log.Error().
				Err(err).
				Str("webhook", webhook.Name).
				Msg("non-retryable webhook error - not attempting retry")
			break
		}
	}

	if totalAttempts == 0 {
		totalAttempts = maxRetries + 1
	}
	retryAttempts := totalAttempts - 1
	statusCode := 0
	if lastResp != nil {
		statusCode = lastResp.statusCode
	}

	// Final error logging with summary
	log.Error().
		Err(lastErr).
		Str("webhook", webhook.Name).
		Str("service", webhook.Service).
		Int("totalAttempts", totalAttempts).
		Int("retryableErrors", retryableErrors).
		Msg("webhook delivery failed after all retry attempts")

	// Track failed delivery
	delivery := WebhookDelivery{
		WebhookName:     webhook.Name,
		WebhookURL:      webhook.URL,
		Service:         webhook.Service,
		AlertIdentifier: "enhanced", // Enhanced webhooks are service-level deliveries, not alert-bound.
		Timestamp:       time.Now(),
		StatusCode:      statusCode,
		Success:         false,
		ErrorMessage:    lastErr.Error(),
		RetryAttempts:   retryAttempts,
		PayloadSize:     len(payload),
	}
	n.addWebhookDelivery(delivery)

	return fmt.Errorf("webhook failed after %d attempts: %w", totalAttempts, lastErr)
}

func parseRetryAfterBackoff(retryAfter string, now time.Time) (time.Duration, bool) {
	retryAfter = strings.TrimSpace(retryAfter)

	clampBackoff := func(backoff time.Duration) time.Duration {
		if backoff < 0 {
			return 0
		}
		if backoff > WebhookMaxBackoff {
			return WebhookMaxBackoff
		}
		return backoff
	}

	if seconds, err := strconv.ParseInt(retryAfter, 10, 64); err == nil {
		// Clamp in seconds before conversion: multiplying an untrusted value
		// by time.Second can overflow and turn a long delay into no delay.
		if seconds <= 0 {
			return 0, true
		}
		if seconds > int64(WebhookMaxBackoff/time.Second) {
			return WebhookMaxBackoff, true
		}
		return clampBackoff(time.Duration(seconds) * time.Second), true
	}

	if retryAt, err := http.ParseTime(retryAfter); err == nil {
		return clampBackoff(retryAt.Sub(now)), true
	}

	return 0, false
}

// isRetryableWebhookError determines if a webhook error should trigger a retry
func isRetryableWebhookError(err error) bool {
	// A local security/configuration refusal is authoritative too. In
	// particular, retrying an origin-changing redirect cannot make it safe.
	var declared *NotificationFailureError
	if errors.As(err, &declared) && declared.Class != "" {
		return declared.Class.Retryable()
	}
	// An explicit HTTP rejection takes precedence over diagnostic body text:
	// a 403 mentioning "timeout" is not a transport timeout. Errors without
	// a recognised status (including network failures) remain retryable below.
	if statusCode, ok := webhookErrorStatusCode(err); ok {
		switch statusCode {
		case http.StatusRequestTimeout, http.StatusMisdirectedRequest, http.StatusLocked, http.StatusTooEarly, http.StatusTooManyRequests:
			return true
		}
		if statusCode >= 500 && statusCode <= 599 {
			return true
		}
		if statusCode >= 400 && statusCode <= 499 {
			return false
		}
	}

	// Default to retryable for unknown errors
	return true
}

func webhookErrorStatusCode(err error) (int, bool) {
	if err == nil {
		return 0, false
	}

	match := webhookHTTPStatusPattern.FindStringSubmatch(strings.ToLower(err.Error()))
	if len(match) != 2 {
		return 0, false
	}

	statusCode, parseErr := strconv.Atoi(match[1])
	if parseErr != nil {
		return 0, false
	}
	return statusCode, true
}

// executeEnhancedWebhookRequest sends a single enhanced webhook request and returns response metadata.
func (n *NotificationManager) executeEnhancedWebhookRequest(webhook EnhancedWebhookConfig, payload []byte, timeout time.Duration, userAgent string, eventID string) (*webhookHTTPResult, error) {
	canonical := canonicalWebhookConfigForEnhanced(webhook)
	return n.executeWebhookRequest(canonical, payload, webhookRequestOptions{
		eventID:         eventID,
		timeout:         timeout,
		userAgent:       userAgent,
		responseLogging: webhook.ResponseLogging,
		validateURL:     true,
	})
}

// NOTE: formatWebhookDuration is now defined in notifications.go to avoid duplication

// TestEnhancedWebhook tests a webhook with a specific payload
func (n *NotificationManager) TestEnhancedWebhook(webhook EnhancedWebhookConfig) (int, string, error) {
	testAlert := buildNotificationTestAlert()
	testAlert.ID = "test-" + time.Now().Format("20060102-150405")
	if n.publicURL != "" {
		testAlert.Instance = n.publicURL
	}

	var err error
	webhook, payload, err := n.prepareEnhancedWebhookExecution(webhook, testAlert)
	if err != nil {
		return 0, "", err
	}

	if webhook.Service == "ntfy" {
		canonical := withNtfyAlertHeaders(canonicalWebhookConfigForEnhanced(webhook), []*alerts.Alert{testAlert})
		webhook.Headers = canonical.Headers
	}

	result, err := n.executeEnhancedWebhookRequest(webhook, payload, WebhookTestTimeout, "Pulse-Monitoring/2.0 (Test)", webhookEventID(testAlert.ID, "alert"))
	if err != nil {
		if result != nil {
			return result.statusCode, result.responseSummary(), err
		}
		return 0, "", err
	}

	return result.statusCode, result.responseSummary(), nil
}