Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

145 KB
Raw Blame
package notifications

import (
	"bytes"
	"context"
	"crypto/hmac"
	"crypto/sha256"
	"crypto/tls"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"html"
	"math"
	"net"
	"net/http"
	"net/url"
	"os"
	"os/exec"
	"sort"
	"strconv"
	"strings"
	"sync"
	"text/template"
	"time"

	"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
	"github.com/rcourtman/pulse-go-rewrite/internal/securityutil"
	pkgsecurityutil "github.com/rcourtman/pulse-go-rewrite/pkg/securityutil"
	"github.com/rs/zerolog/log"
)

// Webhook configuration constants
const (
	// HTTP client settings
	WebhookTimeout          = 30 * time.Second
	WebhookMaxResponseSize  = 1 * 1024 * 1024 // 1 MB max response size
	WebhookMaxRedirects     = 3               // Maximum number of redirects to follow
	WebhookTestTimeout      = 10 * time.Second
	webhookDNSLookupTimeout = 5 * time.Second

	// Retry settings
	WebhookInitialBackoff = 1 * time.Second
	WebhookMaxBackoff     = 30 * time.Second
	WebhookDefaultRetries = 3

	// History settings
	WebhookHistoryMaxSize = 100

	// Rate limiting settings
	WebhookRateLimitWindow = 1 * time.Minute // Time window for rate limiting
	WebhookRateLimitMax    = 10              // Max requests per window per webhook
)

const (
	queueTypeSuffixResolved = "_resolved"
	metadataResolvedAt      = "resolvedAt"

	// Apprise timeout bounds (seconds), applied during config normalization.
	appriseDefaultTimeoutSecs = 15
	appriseMaxTimeoutSecs     = 120
	appriseMinTimeoutSecs     = 5
)

// notificationEvent represents the type of notification being processed.
// It distinguishes between new alerts and resolved/cleared alerts.
type notificationEvent string

const (
	eventAlert    notificationEvent = "alert"
	eventResolved notificationEvent = "resolved"
)

type notificationDeliveryJob struct {
	Type          string
	Event         notificationEvent
	Alerts        []*alerts.Alert
	ResolvedAt    time.Time
	EmailConfig   *EmailConfig
	WebhookConfig *WebhookConfig
	AppriseConfig *AppriseConfig
}

type notificationQueueBucket struct {
	job         notificationDeliveryJob
	nextRetryAt *time.Time
}

type notificationDeliveryTarget string

const (
	notificationDeliveryTargetAll     notificationDeliveryTarget = "all"
	notificationDeliveryTargetEmail   notificationDeliveryTarget = "email"
	notificationDeliveryTargetWebhook notificationDeliveryTarget = "webhook"
	notificationDeliveryTargetApprise notificationDeliveryTarget = "apprise"
)

type alertSendOptions struct {
	bypassCooldown bool
	immediate      bool
	target         notificationDeliveryTarget
	destinationIDs []string
}

// createSecureWebhookClient creates an HTTP client with security controls
func (n *NotificationManager) createSecureWebhookClient(timeout time.Duration) *http.Client {
	return n.createSecureWebhookClientWithTLS(timeout, false)
}

// createSecureWebhookClientWithTLS creates a secure HTTP client with optional TLS verification override.
func (n *NotificationManager) createSecureWebhookClientWithTLS(timeout time.Duration, skipTLSVerify bool) *http.Client {
	return n.createSecureWebhookClientWithURLDiagnostics(timeout, skipTLSVerify, RedactWebhookURLSecrets)
}

func (n *NotificationManager) createSecureWebhookClientWithURLDiagnostics(timeout time.Duration, skipTLSVerify bool, diagnosticURL func(string) string) *http.Client {
	// dedicated transport that pins DNS resolution to prevent rebinding
	transport := &http.Transport{
		// Proxy intentionally nil — outbound proxies would bypass DialContext
		// SSRF checks by resolving the target on the proxy side.
		DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
			// Extract hostname and port
			host, port, err := net.SplitHostPort(addr)
			if err != nil {
				return nil, fmt.Errorf("parse webhook address %q: %w", addr, err)
			}

			// Validate IP if it's already an IP
			if ip := net.ParseIP(host); ip != nil {
				if ip.IsUnspecified() {
					return nil, fmt.Errorf("blocked unspecified IP: %s", ip)
				}
				if isPrivateIP(ip) && !n.isIPInAllowlist(ip) {
					return nil, fmt.Errorf("blocked private IP: %s", ip)
				}
				return pkgsecurityutil.DialPinnedIPs(ctx, network, port, []net.IP{ip}, 10*time.Second)
			}

			// Resolve hostname
			ips, err := n.lookupWebhookHostIPs(ctx, host)
			if err != nil {
				return nil, fmt.Errorf("resolve webhook host %q: %w", host, err)
			}

			// Collect every permitted IP in resolution order
			var permittedIPs []net.IP
			for _, ip := range ips {
				if !isPrivateIP(ip) || n.isIPInAllowlist(ip) {
					permittedIPs = append(permittedIPs, ip)
				}
			}

			if len(permittedIPs) == 0 {
				return nil, fmt.Errorf("hostname %s resolves to blocked private IPs", host)
			}

			// Log if we filtered some IPs
			if len(ips) > 1 {
				log.Debug().
					Str("host", host).
					Int("resolved", len(ips)).
					Int("permitted", len(permittedIPs)).
					Msg("dns resolution pinned for webhook security")
			}

			return pkgsecurityutil.DialPinnedIPs(ctx, network, port, permittedIPs, 10*time.Second)
		},
		ForceAttemptHTTP2:     true,
		MaxIdleConns:          100,
		IdleConnTimeout:       90 * time.Second,
		TLSHandshakeTimeout:   10 * time.Second,
		ExpectContinueTimeout: 1 * time.Second,
	}
	if skipTLSVerify {
		transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
	}

	return &http.Client{
		Timeout:   timeout,
		Transport: transport,
		CheckRedirect: func(req *http.Request, via []*http.Request) error {
			// SSRF admission is not permission to forward private alert bodies,
			// custom authentication headers or signatures to a different receiver.
			if len(via) == 0 || via[0] == nil || !sameNotificationOrigin(via[0].URL, req.URL) {
				return FailWithClass(NotificationFailureConfiguration, errNotificationRedirectOrigin)
			}
			// Origin refusal takes precedence even when the redirect budget is
			// exhausted: a foreign Location must not enter diagnostics or become
			// an unknown/retryable error instead of a local configuration failure.
			if len(via) >= WebhookMaxRedirects {
				return fmt.Errorf("stopped after %d redirects", WebhookMaxRedirects)
			}
			// Re-validate strictly on an admitted redirect, including DNS/IP
			// changes. Foreign origins are rejected before resolution or logging.
			return n.validateWebhookURLWithContext(req.Context(), req.URL.String(), diagnosticURL)
		},
	}
}

// TestNodeInfo contains information about nodes for test notifications
type TestNodeInfo struct {
	NodeName    string
	InstanceURL string
}

// WebhookDelivery tracks webhook delivery attempts for debugging
type WebhookDelivery struct {
	WebhookName     string    `json:"webhookName"`
	WebhookURL      string    `json:"webhookUrl"`
	Service         string    `json:"service"`
	AlertIdentifier string    `json:"alertIdentifier,omitempty"`
	Timestamp       time.Time `json:"timestamp"`
	StatusCode      int       `json:"statusCode"`
	Success         bool      `json:"success"`
	ErrorMessage    string    `json:"errorMessage,omitempty"`
	RetryAttempts   int       `json:"retryAttempts"`
	PayloadSize     int       `json:"payloadSize"`
}

// webhookRateLimit tracks rate limiting for webhook deliveries
type webhookRateLimit struct {
	lastSent  time.Time
	sentCount int
}

type notificationLifecycleState int

const (
	notificationLifecycleRunning notificationLifecycleState = iota
	notificationLifecycleDraining
	notificationLifecycleClosed
)

// notificationLifecycle owns work that holds the queue outside the manager
// lock. Each lease is taken in the critical section that reads the queue, so
// a lease holder always writes to an open queue. Stop drains the lifecycle,
// refusing new alerts while cancellations, operator pause and direct delivery
// already owed continue, then closes it once every lease has ended.
type notificationLifecycle struct {
	mu     sync.Mutex
	idle   sync.Cond
	state  notificationLifecycleState
	leases int
}

// admit leases new alert or recovery work. Stop refuses it from the start.
func (l *notificationLifecycle) admit() bool {
	return l.acquire(notificationLifecycleRunning)
}

// join leases work that must still reach an open queue while Stop drains:
// cancellation, operator pause or channel disable, and direct delivery
// started by admitted work. It is refused only once the drain has finished.
func (l *notificationLifecycle) join() bool {
	return l.acquire(notificationLifecycleDraining)
}

func (l *notificationLifecycle) acquire(latest notificationLifecycleState) bool {
	l.mu.Lock()
	defer l.mu.Unlock()
	if l.state > latest {
		return false
	}
	l.leases++
	return true
}

func (l *notificationLifecycle) release() {
	l.mu.Lock()
	l.leases--
	if l.leases == 0 {
		l.idle.Broadcast()
	}
	l.mu.Unlock()
}

// accepting reports whether new work may still be buffered for later
// admission.
func (l *notificationLifecycle) accepting() bool {
	l.mu.Lock()
	defer l.mu.Unlock()
	return l.state == notificationLifecycleRunning
}

func (l *notificationLifecycle) drain() {
	l.mu.Lock()
	if l.state == notificationLifecycleRunning {
		l.state = notificationLifecycleDraining
	}
	l.mu.Unlock()
}

// close waits for every lease to end, then refuses all later work.
func (l *notificationLifecycle) close() {
	l.mu.Lock()
	defer l.mu.Unlock()
	if l.idle.L == nil {
		l.idle.L = &l.mu
	}
	for l.leases > 0 {
		l.idle.Wait()
	}
	l.state = notificationLifecycleClosed
}

// NotificationManager handles sending notifications
type NotificationManager struct {
	mu                 sync.RWMutex
	stopOnce           sync.Once
	cleanupWG          sync.WaitGroup
	lifecycle          notificationLifecycle // Admission and shutdown ownership of queue and direct delivery work
	emailConfig        EmailConfig
	emailManager       *EnhancedEmailManager // Shared email manager for rate limiting
	webhooks           []WebhookConfig
	appriseConfig      AppriseConfig
	enabled            bool
	cooldown           time.Duration
	initialTarget      notificationDeliveryTarget
	notifyOnResolve    bool
	lastNotified       map[string]notificationRecord
	deliveryReceipts   map[string]struct{}
	groupWindow        time.Duration
	groupingEnabled    bool
	pendingAlerts      []*alerts.Alert
	pendingAlertIndex  map[pendingFiringOccurrence]int
	groupTimer         *time.Timer
	groupDeadline      time.Time // First admission fixes this window; shutdown must not restart or shorten it
	pendingResolved    []*alerts.Alert
	resolvedGroupTimer *time.Timer
	groupByNode        bool
	publicURL          string // Full URL to access Pulse
	groupByGuest       bool
	webhookHistory     []WebhookDelivery            // Keep last 100 webhook deliveries for debugging
	webhookRateLimits  map[string]*webhookRateLimit // Track rate limits per webhook URL
	webhookRateMu      sync.Mutex                   // Separate mutex for webhook rate limiting
	webhookRateCleanup time.Time                    // Last cleanup time for webhook rate limit entries
	appriseExec        appriseExecFunc
	queue              *NotificationQueue                              // Persistent notification queue
	tenantID           string                                          // Tenant identity stamped into webhook payloads (env default)
	tenantName         string                                          // Tenant display name stamped into webhook payloads (env default)
	tenantResolver     func() (id string, name string)                 // Org-backed tenant identity override (multi-tenant)
	webhookClient      *http.Client                                    // Shared HTTP client for webhooks
	stopCleanup        chan struct{}                                   // Signal to stop cleanup goroutine
	cleanupDone        chan struct{}                                   // Signals cleanup goroutine exit during shutdown
	allowedPrivateNets []*net.IPNet                                    // Parsed CIDR ranges allowed for private webhook targets
	allowedPrivateMu   sync.RWMutex                                    // Protects allowedPrivateNets
	resolveWebhookIPs  func(context.Context, string) ([]net.IP, error) // Test seam; nil uses the context-aware default resolver
}

type webhookHTTPResult struct {
	statusCode           int
	headers              http.Header
	responseBytes        int64
	responseLimitReached bool
	responseIncomplete   bool
}

type webhookRequestOptions struct {
	alertType       string
	eventID         string // Legacy X-Pulse-Event-ID correlation header, not a unique delivery ID
	timeout         time.Duration
	userAgent       string
	responseLogging bool
	validateURL     bool
}

// spawnAsync is used for fire-and-forget notification delivery.
// It exists so tests can disable or inline async behavior without relying on sleeps.
var spawnAsync = func(f func()) { go f() }

type appriseExecFunc func(ctx context.Context, args []string) ([]byte, error)

// copyEmailConfig returns a defensive copy of EmailConfig including its slices to avoid data races.
func copyEmailConfig(cfg EmailConfig) EmailConfig {
	copy := cfg
	if len(cfg.To) > 0 {
		copy.To = append([]string(nil), cfg.To...)
	}
	if len(cfg.TagFilter) > 0 {
		copy.TagFilter = append([]string(nil), cfg.TagFilter...)
	}
	return copy
}

// copyWebhookConfigs deep-copies webhook configurations to isolate concurrent writers from background senders.
func copyWebhookConfigs(webhooks []WebhookConfig) []WebhookConfig {
	if len(webhooks) == 0 {
		return nil
	}

	copies := make([]WebhookConfig, 0, len(webhooks))
	for _, webhook := range webhooks {
		clone := webhook
		if len(clone.Headers) > 0 {
			headers := make(map[string]string, len(webhook.Headers))
			for k, v := range clone.Headers {
				headers[k] = v
			}
			clone.Headers = headers
		}
		if len(clone.CustomFields) > 0 {
			custom := make(map[string]string, len(clone.CustomFields))
			for k, v := range clone.CustomFields {
				custom[k] = v
			}
			clone.CustomFields = custom
		}
		if len(clone.TagFilter) > 0 {
			clone.TagFilter = append([]string(nil), clone.TagFilter...)
		}
		copies = append(copies, clone)
	}

	return copies
}

func copyWebhookConfig(webhook WebhookConfig) WebhookConfig {
	clones := copyWebhookConfigs([]WebhookConfig{webhook})
	if len(clones) == 0 {
		return WebhookConfig{}
	}
	return clones[0]
}

// NormalizeWebhookConfig canonicalizes service-specific webhook configuration fields
// at ingress and persistence boundaries. Runtime delivery paths should already
// carry canonical webhook state and must not rely on legacy alias migration here.
func NormalizeWebhookConfig(webhook WebhookConfig) WebhookConfig {
	normalized := webhook
	normalized.TagFilter = normalizeNotificationTagFilter(normalized.TagFilter)
	normalized.MinimumSeverity = normalizeNotificationMinimumSeverity(normalized.MinimumSeverity)
	if len(normalized.TagFilter) == 0 {
		normalized.TagMode = ""
	} else {
		normalized.TagMode = normalizeNotificationTagMode(normalized.TagMode)
	}
	if strings.EqualFold(strings.TrimSpace(normalized.Service), "pushover") {
		normalized.CustomFields = normalizePushoverWebhookCustomFields(normalized.CustomFields)
	}
	normalized.Headers = normalizeWebhookHeaders(normalized.Headers)
	return normalized
}

// normalizeWebhookHeaders drops headers with a blank name or value. A cleared
// header field in the editor means "no header"; sending it empty would replace
// the default Content-Type and leave receivers unable to parse the body.
func normalizeWebhookHeaders(headers map[string]string) map[string]string {
	if len(headers) == 0 {
		return headers
	}
	normalized := make(map[string]string, len(headers))
	for key, value := range headers {
		if strings.TrimSpace(key) != "" && strings.TrimSpace(value) != "" {
			normalized[key] = value
		}
	}
	return normalized
}

func normalizePushoverWebhookCustomFields(fields map[string]string) map[string]string {
	if len(fields) == 0 {
		return nil
	}

	normalized := make(map[string]string, len(fields))
	for key, value := range fields {
		normalized[key] = value
	}

	token := strings.TrimSpace(normalized["token"])
	if token == "" {
		if legacy := strings.TrimSpace(normalized["app_token"]); legacy != "" {
			normalized["token"] = legacy
		}
	}

	user := strings.TrimSpace(normalized["user"])
	if user == "" {
		if legacy := strings.TrimSpace(normalized["user_token"]); legacy != "" {
			normalized["user"] = legacy
		}
	}

	delete(normalized, "app_token")
	delete(normalized, "user_token")

	if len(normalized) == 0 {
		return nil
	}
	return normalized
}

func copyAppriseConfig(cfg AppriseConfig) AppriseConfig {
	copy := cfg
	if len(cfg.Targets) > 0 {
		copy.Targets = append([]string(nil), cfg.Targets...)
	}
	return copy
}

// annotateResolvedMetadata stores the resolution timestamp on the alert metadata for queue persistence.
func annotateResolvedMetadata(alert *alerts.Alert, resolvedAt time.Time) {
	if alert == nil {
		return
	}
	if alert.Metadata == nil {
		alert.Metadata = make(map[string]interface{})
	}
	alert.Metadata[metadataResolvedAt] = resolvedAt.Format(time.RFC3339)
}

// resolvedAlertMessage is the one-line account of a resolved alert. A close
// that was not a recovery, such as a node alert that moved to the Pulse agent
// now monitoring the machine, says what happened instead of "now healthy".
func resolvedAlertMessage(alert *alerts.Alert) string {
	if summary := resolvedAlertNotRecoveredSummary(alert); summary != "" {
		return summary
	}
	return fmt.Sprintf("%s on %s is now healthy", alert.ResourceName, alert.Node)
}

// resolvedAlertNotRecoveredSummary names the alert in its resolution summary,
// so one of several closes for a machine says which alert moved. It is empty
// for an ordinary recovery.
func resolvedAlertNotRecoveredSummary(alert *alerts.Alert) string {
	label := alertTypeDisplay(alert.Type)
	if label != "" {
		label += " alert"
	}
	return alert.Resolution.Describe(label)
}

// resolvedAlertsIncludeNotRecovered reports whether any close in a resolved
// notification was not a recovery, so recovery visuals must not be used.
func resolvedAlertsIncludeNotRecovered(alertList []*alerts.Alert) bool {
	for _, alert := range alertList {
		if alert != nil && alert.Resolution.Outcome() != "" {
			return true
		}
	}
	return false
}

// resolvedAlertListEntry names one alert in a grouped recovery list.
func resolvedAlertListEntry(alert *alerts.Alert) string {
	entry := fmt.Sprintf("%s on %s", alert.ResourceName, alert.Node)
	if outcome := alert.Resolution.Outcome(); outcome != "" {
		entry += ", " + outcome
	}
	return entry
}

func quietHoursReplayAtForAlert(alert *alerts.Alert, now time.Time) *time.Time {
	if alert == nil || alert.Metadata == nil {
		return nil
	}
	raw, ok := alert.Metadata[alerts.MetadataQuietHoursReplayAt]
	if !ok {
		return nil
	}

	var parsed time.Time
	switch value := raw.(type) {
	case string:
		ts, err := time.Parse(time.RFC3339, strings.TrimSpace(value))
		if err != nil {
			return nil
		}
		parsed = ts
	case time.Time:
		parsed = value
	case float64:
		if value <= 0 {
			return nil
		}
		parsed = time.Unix(int64(value), 0)
	default:
		return nil
	}

	if !parsed.After(now) {
		return nil
	}
	parsed = parsed.UTC()
	return &parsed
}

func notificationQueueBucketsForJob(job notificationDeliveryJob, now time.Time) []notificationQueueBucket {
	if len(job.Alerts) == 0 {
		return []notificationQueueBucket{{job: job}}
	}

	buckets := make([]notificationQueueBucket, 0, len(job.Alerts))
	bucketIndexes := make(map[string]int, len(job.Alerts))
	for _, alert := range job.Alerts {
		nextRetryAt := quietHoursReplayAtForAlert(alert, now)
		key := "immediate"
		if nextRetryAt != nil {
			key = nextRetryAt.Format(time.RFC3339Nano)
		}

		index, exists := bucketIndexes[key]
		if !exists {
			bucketJob := job
			bucketJob.Alerts = nil
			bucket := notificationQueueBucket{job: bucketJob}
			if nextRetryAt != nil {
				replayAt := *nextRetryAt
				bucket.nextRetryAt = &replayAt
			}
			index = len(buckets)
			bucketIndexes[key] = index
			buckets = append(buckets, bucket)
		}
		buckets[index].job.Alerts = append(buckets[index].job.Alerts, alert)
	}

	return buckets
}

// NormalizeAppriseConfig cleans and normalizes Apprise configuration values.
func NormalizeAppriseConfig(cfg AppriseConfig) AppriseConfig {
	normalized := cfg
	normalized.MinimumSeverity = normalizeNotificationMinimumSeverity(normalized.MinimumSeverity)

	mode := strings.ToLower(strings.TrimSpace(string(normalized.Mode)))
	switch mode {
	case string(AppriseModeHTTP):
		normalized.Mode = AppriseModeHTTP
	default:
		normalized.Mode = AppriseModeCLI
	}

	normalized.CLIPath = "apprise" // Force default binary for security

	if normalized.TimeoutSeconds <= 0 {
		normalized.TimeoutSeconds = appriseDefaultTimeoutSecs
	} else if normalized.TimeoutSeconds > appriseMaxTimeoutSecs {
		normalized.TimeoutSeconds = appriseMaxTimeoutSecs
	} else if normalized.TimeoutSeconds < appriseMinTimeoutSecs {
		normalized.TimeoutSeconds = appriseMinTimeoutSecs
	}

	cleanTargets := make([]string, 0, len(normalized.Targets))
	seen := make(map[string]struct{}, len(normalized.Targets))
	for _, target := range normalized.Targets {
		trimmed := strings.TrimSpace(target)
		if trimmed == "" {
			continue
		}
		lower := strings.ToLower(trimmed)
		if _, exists := seen[lower]; exists {
			continue
		}
		seen[lower] = struct{}{}
		cleanTargets = append(cleanTargets, trimmed)
	}
	normalized.Targets = cleanTargets

	normalized.ServerURL = strings.TrimSpace(normalized.ServerURL)
	normalized.ServerURL = strings.TrimRight(normalized.ServerURL, "/")

	normalized.ConfigKey = strings.TrimSpace(normalized.ConfigKey)

	normalized.APIKey = strings.TrimSpace(normalized.APIKey)
	normalized.APIKeyHeader = strings.TrimSpace(normalized.APIKeyHeader)
	if normalized.APIKeyHeader == "" {
		normalized.APIKeyHeader = "X-API-KEY"
	}

	switch normalized.Mode {
	case AppriseModeCLI:
		if len(normalized.Targets) == 0 {
			normalized.Enabled = false
		}
	case AppriseModeHTTP:
		if normalized.ServerURL == "" {
			normalized.Enabled = false
		}
	}

	return normalized
}

func defaultAppriseExec(ctx context.Context, args []string) ([]byte, error) {
	cmd := exec.CommandContext(ctx, "apprise", args...)
	return cmd.CombinedOutput()
}

type notificationRecord struct {
	lastSent   time.Time
	alertStart time.Time
	level      alerts.AlertLevel // Highest delivered severity for this occurrence.
}

// Alert represents an alert (interface to avoid circular dependency)
type Alert interface {
	GetID() string
	GetResourceName() string
	GetType() string
	GetLevel() string
	GetValue() float64
	GetThreshold() float64
	GetMessage() string
	GetNode() string
	GetInstance() string
	GetStartTime() time.Time
}

// EmailConfig holds email notification settings
type EmailConfig struct {
	Enabled         bool     `json:"enabled"`
	Provider        string   `json:"provider"` // Email provider name (Gmail, SendGrid, etc.)
	SMTPHost        string   `json:"server"`   // Changed from smtpHost to server for frontend consistency
	SMTPPort        int      `json:"port"`     // Changed from smtpPort to port for frontend consistency
	Username        string   `json:"username"`
	Password        string   `json:"password"`
	From            string   `json:"from"`
	To              []string `json:"to"`
	TLS             bool     `json:"tls"`
	StartTLS        bool     `json:"startTLS"`  // STARTTLS support
	RateLimit       int      `json:"rateLimit"` // Max emails per minute (0 = default 60)
	TagFilter       []string `json:"tagFilter,omitempty"`
	TagMode         string   `json:"tagFilterMode,omitempty"`   // "all" (default) or "any"
	MinimumSeverity string   `json:"minimumSeverity,omitempty"` // "all" (default), "warning", or "critical"
}

// WebhookConfig holds webhook settings
type WebhookConfig struct {
	ID              string            `json:"id"`
	Name            string            `json:"name"`
	URL             string            `json:"url"`
	Method          string            `json:"method"`
	Headers         map[string]string `json:"headers"`
	Enabled         bool              `json:"enabled"`
	Service         string            `json:"service"`  // discord, slack, teams, etc.
	Template        string            `json:"template"` // Custom payload template
	CustomFields    map[string]string `json:"customFields,omitempty"`
	Mention         string            `json:"mention,omitempty"` // Platform-specific mention (e.g., @everyone, @channel, <@USER_ID>)
	TagFilter       []string          `json:"tagFilter,omitempty"`
	TagMode         string            `json:"tagFilterMode,omitempty"`   // "all" (default) or "any"
	MinimumSeverity string            `json:"minimumSeverity,omitempty"` // "all" (default), "warning", or "critical"
	// SigningSecret enables HMAC-SHA256 signing of outbound deliveries.
	// When set, requests carry X-Pulse-Timestamp and X-Pulse-Signature
	// (v1=hex(hmac_sha256(secret, timestamp + "." + body))) so receivers
	// can verify origin and integrity.
	SigningSecret string `json:"signingSecret,omitempty"`
}

// AppriseMode identifies how Pulse should deliver notifications through Apprise.
type AppriseMode string

const (
	AppriseModeCLI  AppriseMode = "cli"
	AppriseModeHTTP AppriseMode = "http"
)

const (
	defaultSMTPPort       = 587
	defaultEmailRateLimit = 60
)

// AppriseConfig holds Apprise notification settings.
type AppriseConfig struct {
	Enabled         bool        `json:"enabled"`
	Mode            AppriseMode `json:"mode,omitempty"`
	Targets         []string    `json:"targets"`
	CLIPath         string      `json:"cliPath,omitempty"`
	TimeoutSeconds  int         `json:"timeoutSeconds,omitempty"`
	ServerURL       string      `json:"serverUrl,omitempty"`
	ConfigKey       string      `json:"configKey,omitempty"`
	APIKey          string      `json:"apiKey,omitempty"`
	APIKeyHeader    string      `json:"apiKeyHeader,omitempty"`
	SkipTLSVerify   bool        `json:"skipTlsVerify,omitempty"`
	MinimumSeverity string      `json:"minimumSeverity,omitempty"` // "all" (default), "warning", or "critical"
}

func normalizeEmailConfig(cfg EmailConfig) EmailConfig {
	normalized := cfg
	normalized.Provider = strings.TrimSpace(normalized.Provider)
	normalized.SMTPHost = strings.TrimSpace(normalized.SMTPHost)
	normalized.Username = strings.TrimSpace(normalized.Username)
	normalized.From = strings.TrimSpace(normalized.From)
	normalized.TagFilter = normalizeNotificationTagFilter(normalized.TagFilter)
	normalized.MinimumSeverity = normalizeNotificationMinimumSeverity(normalized.MinimumSeverity)
	if len(normalized.TagFilter) == 0 {
		normalized.TagMode = ""
	} else {
		normalized.TagMode = normalizeNotificationTagMode(normalized.TagMode)
	}

	if normalized.SMTPPort <= 0 || normalized.SMTPPort > 65535 {
		log.Warn().
			Int("smtpPort", normalized.SMTPPort).
			Int("defaultPort", defaultSMTPPort).
			Msg("Invalid SMTP port in email config, using default")
		normalized.SMTPPort = defaultSMTPPort
	}

	if normalized.RateLimit < 0 {
		log.Warn().
			Int("rateLimit", normalized.RateLimit).
			Msg("Invalid negative email rate limit, using default behavior")
		normalized.RateLimit = 0
	}

	if len(normalized.To) > 0 {
		cleaned := make([]string, 0, len(normalized.To))
		seen := make(map[string]struct{}, len(normalized.To))
		for _, recipient := range normalized.To {
			trimmed := strings.TrimSpace(recipient)
			if trimmed == "" {
				continue
			}
			key := strings.ToLower(trimmed)
			if _, exists := seen[key]; exists {
				continue
			}
			seen[key] = struct{}{}
			cleaned = append(cleaned, trimmed)
		}
		normalized.To = cleaned
	}

	return normalized
}

func effectiveEmailRateLimit(configured int) int {
	if configured <= 0 {
		return defaultEmailRateLimit
	}
	return configured
}

// NewNotificationManager creates a new notification manager using the global data directory.
// For multi-tenant deployments, use NewNotificationManagerWithDataDir instead.
func NewNotificationManager(publicURL string) *NotificationManager {
	return NewNotificationManagerWithDataDir(publicURL, "")
}

// NewNotificationManagerWithDataDir creates a new notification manager with a custom data directory.
// This enables tenant-scoped notification queue persistence in multi-tenant deployments.
// If dataDir is empty, it uses the global data directory.
func NewNotificationManagerWithDataDir(publicURL string, dataDir string) *NotificationManager {
	return newNotificationManagerWithDataDir(publicURL, dataDir, false)
}

// NewNotificationManagerWithDeferredQueue creates a manager whose persisted
// deliveries stay idle until StartQueueProcessing is called. Owners that load
// saved destinations after construction must use this form so a queue ticker
// cannot interpret incomplete startup configuration as a removed destination.
func NewNotificationManagerWithDeferredQueue(publicURL string, dataDir string) *NotificationManager {
	return newNotificationManagerWithDataDir(publicURL, dataDir, true)
}

func newNotificationManagerWithDataDir(publicURL string, dataDir string, deferQueueProcessing bool) *NotificationManager {
	cleanURL := strings.TrimRight(strings.TrimSpace(publicURL), "/")
	if cleanURL != "" {
		log.Info().Str("publicURL", cleanURL).Msg("notification manager initialized with public URL")
	} else {
		log.Info().Msg("notification manager initialized without public URL - webhook links may not work")
	}

	// Initialize persistent queue with tenant-specific data directory
	queue, err := NewNotificationQueue(dataDir)
	if err != nil {
		log.Error().Err(err).Msg("failed to initialize persistent notification queue, falling back to in-memory queue")
		queue, err = NewInMemoryNotificationQueue()
		if err != nil {
			log.Error().Err(err).Msg("failed to initialize in-memory notification queue, notifications will be unavailable")
			queue = nil
		}
	}

	nm := &NotificationManager{
		enabled:          true,
		cooldown:         5 * time.Minute,
		initialTarget:    notificationDeliveryTargetAll,
		notifyOnResolve:  true,
		lastNotified:     make(map[string]notificationRecord),
		deliveryReceipts: make(map[string]struct{}),
		webhooks:         []WebhookConfig{},
		appriseConfig: AppriseConfig{
			Enabled:        false,
			Mode:           AppriseModeCLI,
			Targets:        []string{},
			CLIPath:        "apprise",
			TimeoutSeconds: 15,
			APIKeyHeader:   "X-API-KEY",
		},
		groupWindow:       30 * time.Second,
		groupingEnabled:   true,
		tenantID:          strings.TrimSpace(os.Getenv("PULSE_TENANT_ID")),
		tenantName:        strings.TrimSpace(os.Getenv("PULSE_TENANT_NAME")),
		pendingAlerts:     make([]*alerts.Alert, 0),
		groupByNode:       true,
		groupByGuest:      false,
		webhookHistory:    make([]WebhookDelivery, 0, WebhookHistoryMaxSize),
		webhookRateLimits: make(map[string]*webhookRateLimit),
		publicURL:         cleanURL,
		appriseExec:       defaultAppriseExec,
		queue:             queue,
		stopCleanup:       make(chan struct{}),
		cleanupDone:       make(chan struct{}),
	}

	// Create webhook client after NotificationManager is initialized
	nm.webhookClient = nm.createSecureWebhookClient(WebhookTimeout)

	// Ordinary callers retain the historical autonomous queue. A bootstrap
	// owner leaves the processor absent until every saved destination is loaded;
	// not waking the worker alone cannot guard against its periodic ticker.
	if queue != nil && !deferQueueProcessing {
		queue.installProcessor(nm.ProcessQueuedNotification)
	}

	// Start periodic cleanup of old lastNotified entries (every 1 hour)
	nm.cleanupWG.Add(1)
	go nm.cleanupOldNotificationRecords()

	return nm
}

// StartQueueProcessing activates persisted deliveries after the owner has
// restored its complete notification policy and destination configuration.
func (n *NotificationManager) StartQueueProcessing() {
	n.mu.RLock()
	queue := n.queue
	n.mu.RUnlock()
	if queue != nil {
		queue.SetProcessor(n.ProcessQueuedNotification)
	}
}

// SetQuietHoursPolicyProvider binds persistent replay to the alert owner's
// current schedule. Install it before StartQueueProcessing during bootstrap.
// Each provider call must return a lock-independent, read-only evaluator.
func (n *NotificationManager) SetQuietHoursPolicyProvider(provider func() func(*alerts.Alert, time.Time) *time.Time) {
	n.mu.RLock()
	queue := n.queue
	n.mu.RUnlock()
	if queue != nil {
		queue.mu.Lock()
		queue.quietHoursPolicy = provider
		queue.mu.Unlock()
	}
}

// SetTenantIdentityResolver installs an org-backed resolver for the tenant
// identity stamped into webhook payloads. It overrides the
// PULSE_TENANT_ID / PULSE_TENANT_NAME environment defaults; multi-tenant
// deployments use it so each org's manager reports that org's identity.
func (n *NotificationManager) SetTenantIdentityResolver(resolver func() (id string, name string)) {
	n.mu.Lock()
	n.tenantResolver = resolver
	n.mu.Unlock()
}

// TenantIdentity returns the tenant ID and display name stamped into webhook
// payloads. The name falls back to the ID so receivers always get a usable
// label when any tenant identity is configured at all.
func (n *NotificationManager) TenantIdentity() (string, string) {
	n.mu.RLock()
	resolver := n.tenantResolver
	id, name := n.tenantID, n.tenantName
	n.mu.RUnlock()

	if resolver != nil {
		if rid, rname := resolver(); strings.TrimSpace(rid) != "" {
			id, name = strings.TrimSpace(rid), strings.TrimSpace(rname)
		}
	}
	if name == "" {
		name = id
	}
	return id, name
}

// SetPublicURL updates the public URL used for webhook payloads.
func (n *NotificationManager) SetPublicURL(publicURL string) {
	trimmed := strings.TrimRight(strings.TrimSpace(publicURL), "/")
	if trimmed == "" {
		return
	}

	n.mu.Lock()
	if n.publicURL == trimmed {
		n.mu.Unlock()
		return
	}
	n.publicURL = trimmed
	n.mu.Unlock()

	log.Info().Str("publicURL", trimmed).Msg("notification manager public URL updated")
}

// GetPublicURL returns the configured public URL for notifications.
func (n *NotificationManager) GetPublicURL() string {
	n.mu.RLock()
	defer n.mu.RUnlock()
	return n.publicURL
}

// SetEmailConfig updates email configuration
func (n *NotificationManager) SetEmailConfig(config EmailConfig) {
	n.mu.Lock()
	config = normalizeEmailConfig(copyEmailConfig(config))
	n.emailConfig = config

	// Recreate email manager with new config to preserve rate limiting state
	rateLimit := effectiveEmailRateLimit(config.RateLimit)
	providerConfig := EmailProviderConfig{
		EmailConfig:   config,
		Provider:      "",
		MaxRetries:    3,
		RetryDelay:    5,
		RateLimit:     rateLimit,
		StartTLS:      config.StartTLS,
		SkipTLSVerify: false,
		AuthRequired:  config.Username != "" && config.Password != "",
	}
	n.emailManager = NewEnhancedEmailManager(providerConfig)
	var queue *NotificationQueue
	if !config.Enabled && n.queue != nil && n.lifecycle.join() {
		queue = n.queue
	}
	n.mu.Unlock()

	if queue != nil {
		defer n.lifecycle.release()
		if err := queue.CancelByTypes([]string{"email", "email" + queueTypeSuffixResolved}, "Email notifications disabled"); err != nil {
			log.Error().Err(err).Msg("failed to cancel queued email notifications after disabling email delivery")
		}
	}
}

// SetAppriseConfig updates Apprise configuration.
func (n *NotificationManager) SetAppriseConfig(config AppriseConfig) {
	n.mu.Lock()
	n.appriseConfig = NormalizeAppriseConfig(config)
	var queue *NotificationQueue
	if !n.appriseConfig.Enabled && n.queue != nil && n.lifecycle.join() {
		queue = n.queue
	}
	n.mu.Unlock()

	if queue != nil {
		defer n.lifecycle.release()
		if err := queue.CancelByTypes([]string{"apprise", "apprise" + queueTypeSuffixResolved}, "Apprise notifications disabled"); err != nil {
			log.Error().Err(err).Msg("failed to cancel queued Apprise notifications after disabling delivery")
		}
	}
}

// GetAppriseConfig returns a copy of the Apprise configuration.
func (n *NotificationManager) GetAppriseConfig() AppriseConfig {
	n.mu.RLock()
	defer n.mu.RUnlock()
	return copyAppriseConfig(n.appriseConfig)
}

// SetCooldown updates the cooldown duration
func (n *NotificationManager) SetCooldown(minutes int) {
	n.mu.Lock()
	defer n.mu.Unlock()
	if minutes < 0 {
		minutes = 0
	}
	n.cooldown = time.Duration(minutes) * time.Minute
	log.Info().Int("minutes", minutes).Msg("updated notification cooldown")
}

// SetNotifyOnResolve toggles whether resolved alerts send notifications.
func (n *NotificationManager) SetNotifyOnResolve(enabled bool) {
	n.mu.Lock()
	was := n.notifyOnResolve
	n.notifyOnResolve = enabled
	if !enabled {
		n.takePendingResolvedLocked()
	}
	n.mu.Unlock()

	if was != enabled {
		log.Info().Bool("enabled", enabled).Msg("updated resolved alert notifications")
	}
}

// SetInitialNotifyTarget selects destinations for firing and grouped
// notifications. Matching recovery delivery follows firing delivery receipts,
// while escalation levels retain their own per-level routing.
func (n *NotificationManager) SetInitialNotifyTarget(target string) {
	normalized := normalizeNotificationDeliveryTarget(target)
	n.mu.Lock()
	was := n.initialTarget
	n.initialTarget = normalized
	n.mu.Unlock()

	if was != normalized {
		log.Info().Str("target", string(normalized)).Msg("updated initial alert notification target")
	}
}

// GetInitialNotifyTarget returns the normalized initial delivery target.
func (n *NotificationManager) GetInitialNotifyTarget() string {
	n.mu.RLock()
	defer n.mu.RUnlock()
	return string(n.initialTarget)
}

// GetNotifyOnResolve returns whether resolved alerts trigger notifications.
func (n *NotificationManager) GetNotifyOnResolve() bool {
	n.mu.RLock()
	defer n.mu.RUnlock()
	return n.notifyOnResolve
}

// SetGroupingWindow updates the grouping window duration
func (n *NotificationManager) SetGroupingWindow(seconds int) {
	n.mu.Lock()
	defer n.mu.Unlock()
	if seconds < 0 {
		seconds = 0
	}
	n.groupWindow = time.Duration(seconds) * time.Second
	log.Info().Int("seconds", seconds).Msg("updated notification grouping window")
}

// SetGroupingConfig atomically applies the complete grouping policy. Disabling
// grouping (or selecting a zero-second window) flushes already-pending alerts
// as individual deliveries so a live config change cannot merge or lose them.
func (n *NotificationManager) SetGroupingConfig(enabled bool, seconds int, byNode, byGuest bool) {
	if seconds < 0 {
		seconds = 0
	}

	n.mu.Lock()
	n.groupingEnabled = enabled
	n.groupWindow = time.Duration(seconds) * time.Second
	n.groupByNode = byNode
	n.groupByGuest = byGuest

	var pending []*alerts.Alert
	var resolved []*alerts.Alert
	var emailConfig EmailConfig
	var webhooks []WebhookConfig
	var appriseConfig AppriseConfig
	var initialTarget notificationDeliveryTarget
	var queue *NotificationQueue
	if (!enabled || seconds == 0) && n.lifecycle.accepting() {
		resolved = n.takePendingResolvedLocked()
		// Only a live manager may flush to delivery; Stop checkpoints any
		// remaining buffer without bypassing its deadline.
		if len(n.pendingAlerts) > 0 && n.lifecycle.admit() {
			pending = append(pending, n.pendingAlerts...)
		}
		clear(n.pendingAlerts)
		n.pendingAlerts = n.pendingAlerts[:0]
		n.pendingAlertIndex = nil
		if n.groupTimer != nil {
			n.groupTimer.Stop()
			n.groupTimer = nil
			n.groupDeadline = time.Time{}
		}
		emailConfig = copyEmailConfig(n.emailConfig)
		webhooks = copyWebhookConfigs(n.webhooks)
		appriseConfig = copyAppriseConfig(n.appriseConfig)
		initialTarget = n.initialTarget
		queue = n.queue
	}
	n.mu.Unlock()
	if len(pending) > 0 {
		defer n.lifecycle.release()
	}

	for _, alert := range resolved {
		n.dispatchResolvedAlerts([]*alerts.Alert{alert})
	}
	for _, alert := range pending {
		n.dispatchFiringAlerts(emailConfig, webhooks, appriseConfig, []*alerts.Alert{alert}, initialTarget, nil, queue)
	}

	log.Info().
		Bool("enabled", enabled).
		Int("seconds", seconds).
		Bool("byNode", byNode).
		Bool("byGuest", byGuest).
		Int("flushedIndividually", len(pending)).
		Msg("updated notification grouping configuration")
}

// SetGroupingOptions updates grouping options
func (n *NotificationManager) SetGroupingOptions(byNode, byGuest bool) {
	n.mu.Lock()
	defer n.mu.Unlock()
	n.groupByNode = byNode
	n.groupByGuest = byGuest
	log.Info().Bool("byNode", byNode).Bool("byGuest", byGuest).Msg("updated notification grouping options")
}

// AddWebhook adds a webhook configuration
func (n *NotificationManager) AddWebhook(webhook WebhookConfig) {
	n.mu.Lock()
	defer n.mu.Unlock()
	n.webhooks = append(n.webhooks, copyWebhookConfig(NormalizeWebhookConfig(webhook)))
}

// UpdateWebhook updates an existing webhook
func (n *NotificationManager) UpdateWebhook(webhookID string, webhook WebhookConfig) error {
	n.mu.Lock()
	defer n.mu.Unlock()

	for i, w := range n.webhooks {
		if w.ID == webhookID {
			n.webhooks[i] = copyWebhookConfig(NormalizeWebhookConfig(webhook))
			return nil
		}
	}
	return fmt.Errorf("webhook not found: %s", webhookID)
}

// DeleteWebhook removes a webhook
func (n *NotificationManager) DeleteWebhook(webhookID string) error {
	n.mu.Lock()
	defer n.mu.Unlock()

	for i, w := range n.webhooks {
		if w.ID == webhookID {
			n.webhooks = append(n.webhooks[:i], n.webhooks[i+1:]...)
			return nil
		}
	}
	return fmt.Errorf("webhook not found: %s", webhookID)
}

// GetWebhooks returns all webhook configurations
func (n *NotificationManager) GetWebhooks() []WebhookConfig {
	n.mu.RLock()
	defer n.mu.RUnlock()

	if len(n.webhooks) == 0 {
		return []WebhookConfig{}
	}

	return copyWebhookConfigs(n.webhooks)
}

// GetEmailConfig returns the email configuration
func (n *NotificationManager) GetEmailConfig() EmailConfig {
	n.mu.RLock()
	defer n.mu.RUnlock()
	return copyEmailConfig(n.emailConfig)
}

// GetQueue returns the notification queue
func (n *NotificationManager) GetQueue() *NotificationQueue {
	n.mu.RLock()
	defer n.mu.RUnlock()
	return n.queue
}

// SetEnabled toggles notification delivery globally for this runtime instance.
func (n *NotificationManager) SetEnabled(enabled bool) {
	var (
		queue   *NotificationQueue
		changed bool
	)

	n.mu.Lock()
	changed = n.enabled != enabled
	n.enabled = enabled
	if !enabled {
		n.takePendingResolvedLocked()
		clear(n.pendingAlerts)
		n.pendingAlerts = n.pendingAlerts[:0]
		n.pendingAlertIndex = nil
		if n.groupTimer != nil {
			n.groupTimer.Stop()
			n.groupTimer = nil
			n.groupDeadline = time.Time{}
		}
		if n.queue != nil && n.lifecycle.join() {
			queue = n.queue
		}
	}
	n.mu.Unlock()

	if changed {
		log.Info().Bool("enabled", enabled).Msg("updated notification manager enabled state")
	}

	if queue != nil {
		defer n.lifecycle.release()
		types := []string{
			"email", "email" + queueTypeSuffixResolved,
			"webhook", "webhook" + queueTypeSuffixResolved,
			"apprise", "apprise" + queueTypeSuffixResolved,
		}
		if err := queue.CancelByTypes(types, "Notifications disabled"); err != nil {
			log.Error().Err(err).Msg("failed to cancel queued notifications after disabling notification manager")
		}
	}
}

// IsEnabled reports whether notification delivery is currently enabled.
func (n *NotificationManager) IsEnabled() bool {
	n.mu.RLock()
	defer n.mu.RUnlock()
	return n.enabled
}

// SendAlert sends notifications for a newly fired or explicitly re-notified alert.
func (n *NotificationManager) SendAlert(alert *alerts.Alert) {
	n.mu.RLock()
	target := n.initialTarget
	n.mu.RUnlock()
	n.sendAlert(alert, alertSendOptions{target: target})
}

// SendEscalatedAlert sends a scheduled escalation notification. Escalation
// cadence is owned by the alert schedule, so delivery cooldown must not suppress
// a level that the alert manager has already decided is due.
func (n *NotificationManager) SendEscalatedAlert(alert *alerts.Alert, notifyTarget string) {
	n.SendEscalatedAlertToDestinations(alert, notifyTarget, nil)
}

// SendEscalatedAlertToDestinations sends one due escalation to exact logical
// destinations when provided. Exact selection is intentionally fail-closed:
// stale or unknown IDs never widen into a channel-wide or all-destination send.
func (n *NotificationManager) SendEscalatedAlertToDestinations(alert *alerts.Alert, notifyTarget string, destinationIDs []string) {
	n.sendAlert(alert, alertSendOptions{
		bypassCooldown: true,
		immediate:      true,
		target:         normalizeNotificationDeliveryTarget(notifyTarget),
		destinationIDs: normalizeEscalationDestinationIDs(destinationIDs),
	})
}

func (n *NotificationManager) sendAlert(alert *alerts.Alert, options alertSendOptions) {
	if alert == nil {
		return
	}
	alert = alert.Clone()
	if alert == nil {
		return
	}
	if options.target == "" {
		options.target = notificationDeliveryTargetAll
	}

	n.mu.Lock()

	log.Info().
		Str("alertID", alert.ID).
		Bool("enabled", n.enabled).
		Int("webhooks", len(n.webhooks)).
		Bool("emailEnabled", n.emailConfig.Enabled).
		Msg("send alert called")

	if !n.enabled {
		log.Debug().Msg("notifications disabled, skipping")
		n.mu.Unlock()
		return
	}
	// The lease covers the queue read below until its write completes, so Stop
	// cannot close storage under an admitted alert.
	if !n.lifecycle.admit() {
		log.Debug().Str("alertID", alert.ID).Msg("notification manager stopping, alert not admitted")
		n.mu.Unlock()
		return
	}
	defer n.lifecycle.release()

	// Check cooldown. A disabled cooldown means "do not send repeat
	// notifications for the same active alert occurrence". A higher severity is
	// new information, not a repeat. Scheduled escalation retains its explicit
	// bypass because its cadence is owned by the alert schedule.
	record, exists := n.lastNotified[alert.ID]
	severityIncreased := notificationSeverityRank(record.level) > 0 && notificationSeverityRank(alert.Level) > notificationSeverityRank(record.level)
	if !options.bypassCooldown && !severityIncreased && exists && record.alertStart.Equal(alert.StartTime) && (n.cooldown <= 0 || time.Since(record.lastSent) < n.cooldown) {
		elapsed := time.Since(record.lastSent)
		remainingCooldown := time.Duration(0)
		if n.cooldown > elapsed {
			remainingCooldown = n.cooldown - elapsed
		}
		log.Info().
			Str("alertID", alert.ID).
			Str("resourceName", alert.ResourceName).
			Str("type", alert.Type).
			Dur("timeSince", elapsed).
			Dur("cooldown", n.cooldown).
			Dur("remainingCooldown", remainingCooldown).
			Msg("alert notification in cooldown for active alert - notification suppressed")
		n.mu.Unlock()
		return
	}

	if options.immediate || !n.groupingEnabled || n.groupWindow <= 0 {
		emailConfig := copyEmailConfig(n.emailConfig)
		webhooks := copyWebhookConfigs(n.webhooks)
		appriseConfig := copyAppriseConfig(n.appriseConfig)
		queue := n.queue
		n.mu.Unlock()

		n.dispatchFiringAlerts(emailConfig, webhooks, appriseConfig, []*alerts.Alert{alert}, options.target, options.destinationIDs, queue)
		return
	}

	log.Info().
		Str("alertID", alert.ID).
		Str("resourceName", alert.ResourceName).
		Str("type", alert.Type).
		Float64("value", alert.Value).
		Float64("threshold", alert.Threshold).
		Bool("inCooldown", exists).
		Msg("alert passed cooldown check - adding to pending notifications")

	// Group occurrences, not callback invocations. A repeat before delivery has
	// no cooldown receipt yet, but must not inflate the batch's count or payload.
	n.addPendingFiringAlertLocked(alert)

	// If this is the first alert in the group, start the timer
	if n.groupTimer == nil {
		n.groupDeadline = time.Now().Add(n.groupWindow)
		n.groupTimer = time.AfterFunc(time.Until(n.groupDeadline), func() {
			n.sendGroupedAlerts()
		})
		log.Debug().
			Int("pendingCount", len(n.pendingAlerts)).
			Dur("groupWindow", n.groupWindow).
			Msg("started alert grouping timer")
	}
	n.mu.Unlock()
}

type pendingFiringOccurrence struct {
	id    string
	start time.Time
}

func pendingFiringKey(alert *alerts.Alert) (pendingFiringOccurrence, bool) {
	if alert == nil || alert.ID == "" || alert.StartTime.IsZero() {
		// Incomplete legacy identity cannot prove that two callbacks refer to
		// the same occurrence. Preserve both rather than silently lose one.
		return pendingFiringOccurrence{}, false
	}
	// time.Time's map equality includes location and monotonic clock data;
	// occurrence identity must use instant equality, as cancellation does.
	return pendingFiringOccurrence{alert.ID, alert.StartTime.UTC().Round(0)}, true
}

// addPendingFiringAlertLocked takes an already-cloned snapshot under n.mu.
// The index belongs only to the current grouping window; cancellation rebuilds
// it lazily, and every flush/disable discards it. No delivery is recorded here.
func (n *NotificationManager) addPendingFiringAlertLocked(alert *alerts.Alert) {
	if key, known := pendingFiringKey(alert); known {
		if n.pendingAlertIndex == nil {
			n.pendingAlertIndex = make(map[pendingFiringOccurrence]int, len(n.pendingAlerts)+1)
			for index, pending := range n.pendingAlerts {
				if key, known := pendingFiringKey(pending); known {
					n.pendingAlertIndex[key] = index
				}
			}
		}
		if index, exists := n.pendingAlertIndex[key]; exists {
			previous := n.pendingAlerts[index]
			// Keep the newest observation's complete snapshot, including its
			// operational links. Equal-time callbacks prefer higher severity;
			// unknown observation time cannot evict a known newer observation.
			if previous.LastSeen.After(alert.LastSeen) ||
				(previous.LastSeen.Equal(alert.LastSeen) && notificationSeverityRank(previous.Level) > notificationSeverityRank(alert.Level)) {
				return
			}
			n.pendingAlerts[index] = alert
			return
		}
		n.pendingAlertIndex[key] = len(n.pendingAlerts)
	}
	n.pendingAlerts = append(n.pendingAlerts, alert)
}

// Unknown levels cannot opt into the severity cooldown exception.
func notificationSeverityRank(level alerts.AlertLevel) int {
	switch level {
	case alerts.AlertLevelInfo:
		return 1
	case alerts.AlertLevelWarning:
		return 2
	case alerts.AlertLevelCritical:
		return 3
	default:
		return 0
	}
}

func (n *NotificationManager) markAlertsNotified(alertsToSend []*alerts.Alert, sentAt time.Time) {
	n.mu.Lock()
	if n.lastNotified == nil {
		n.lastNotified = make(map[string]notificationRecord)
	}
	for _, alert := range alertsToSend {
		if alert == nil {
			continue
		}
		level := alert.Level
		lastSent := sentAt
		if previous, ok := n.lastNotified[alert.ID]; ok {
			// Destinations and retries can complete out of order. An older
			// occurrence still owns its delivery receipt, but must not replace
			// the current occurrence's repeat-delivery cooldown. Unknown legacy
			// starts likewise cannot evict a known occurrence.
			if previous.alertStart.After(alert.StartTime) {
				continue
			}
			if previous.alertStart.Equal(alert.StartTime) {
				// A late warning cannot re-enable critical delivery, and an
				// earlier completion cannot shorten the same occurrence's hold.
				if notificationSeverityRank(previous.level) > notificationSeverityRank(level) {
					level = previous.level
				}
				if previous.lastSent.After(lastSent) {
					lastSent = previous.lastSent
				}
			}
		}
		n.lastNotified[alert.ID] = notificationRecord{
			level:      level,
			lastSent:   lastSent,
			alertStart: alert.StartTime,
		}
	}
	n.mu.Unlock()
}

func (n *NotificationManager) dispatchFiringAlerts(
	emailConfig EmailConfig,
	webhooks []WebhookConfig,
	appriseConfig AppriseConfig,
	alertsToSend []*alerts.Alert,
	target notificationDeliveryTarget,
	destinationIDs []string,
	queue *NotificationQueue,
) {
	jobs := buildNotificationDeliveryJobsForSelection(
		emailConfig,
		webhooks,
		appriseConfig,
		alertsToSend,
		eventAlert,
		time.Time{},
		target,
		destinationIDs,
	)
	if len(jobs) == 0 {
		if len(destinationIDs) == 0 {
			n.markAlertsNotified(alertsToSend, time.Now())
		}
		return
	}
	if queue != nil {
		n.enqueueNotificationJobs(queue, jobs)
		return
	}
	n.dispatchNotificationJobsAsync(jobs)
}

func notificationDeliveryDestinationKey(job notificationDeliveryJob) string {
	var identity string
	switch job.Type {
	case "email":
		if job.EmailConfig == nil {
			return ""
		}
		recipients := append([]string(nil), job.EmailConfig.To...)
		if len(recipients) == 0 && strings.TrimSpace(job.EmailConfig.From) != "" {
			recipients = append(recipients, job.EmailConfig.From)
		}
		for i := range recipients {
			recipients[i] = strings.ToLower(strings.TrimSpace(recipients[i]))
		}
		sort.Strings(recipients)
		identity = strings.Join(recipients, "\x1f")
	case "webhook":
		if job.WebhookConfig == nil {
			return ""
		}
		// Include both the logical configuration ID and the actual endpoint.
		// Reusing an ID after changing its URL must not send a recovery to an
		// endpoint that never received the firing occurrence.
		webhookID := strings.TrimSpace(job.WebhookConfig.ID)
		webhookURL := strings.TrimSpace(job.WebhookConfig.URL)
		if webhookID == "" && webhookURL == "" {
			return ""
		}
		identity = strings.Join([]string{webhookID, webhookURL}, "\x1f")
	case "apprise":
		if job.AppriseConfig == nil {
			return ""
		}
		targets := append([]string(nil), job.AppriseConfig.Targets...)
		for i := range targets {
			targets[i] = strings.TrimSpace(targets[i])
		}
		sort.Strings(targets)
		identity = strings.Join([]string{
			string(job.AppriseConfig.Mode),
			strings.TrimSpace(job.AppriseConfig.ServerURL),
			strings.TrimSpace(job.AppriseConfig.ConfigKey),
			strings.Join(targets, "\x1f"),
		}, "\x1e")
	default:
		return ""
	}
	if identity == "" {
		return ""
	}
	digest := sha256.Sum256([]byte(job.Type + "\x00" + identity))
	return job.Type + ":" + hex.EncodeToString(digest[:16])
}

func notificationDeliveryReceiptKey(alert *alerts.Alert, destinationKey string) string {
	if alert == nil || alert.ID == "" || alert.StartTime.IsZero() || destinationKey == "" {
		return ""
	}
	return alert.ID + "\x00" + strconv.FormatInt(alert.StartTime.UnixNano(), 10) + "\x00" + destinationKey
}

func (n *NotificationManager) recordSuccessfulDelivery(job notificationDeliveryJob, deliveredAt time.Time) {
	destinationKey := notificationDeliveryDestinationKey(job)
	if destinationKey == "" {
		return
	}
	queue := n.GetQueue()
	for _, alert := range job.Alerts {
		if alert == nil || alert.StartTime.IsZero() {
			continue
		}
		if queue != nil {
			var err error
			if job.Event == eventResolved {
				err = queue.DeleteDeliveryReceipt(alert.ID, alert.StartTime, destinationKey)
			} else {
				err = queue.RecordDeliveryReceipt(alert.ID, alert.StartTime, destinationKey, deliveredAt)
			}
			if err != nil {
				log.Error().Err(err).Str("alertID", alert.ID).Str("destination", destinationKey).Msg("Failed to update notification delivery receipt")
			}
			continue
		}

		key := notificationDeliveryReceiptKey(alert, destinationKey)
		if key == "" {
			continue
		}
		n.mu.Lock()
		if n.deliveryReceipts == nil {
			n.deliveryReceipts = make(map[string]struct{})
		}
		if job.Event == eventResolved {
			delete(n.deliveryReceipts, key)
		} else {
			n.deliveryReceipts[key] = struct{}{}
		}
		n.mu.Unlock()
	}
	if job.Event == eventAlert {
		// Publish the cooldown/delivered marker only after the per-destination
		// receipts are durable. Resolution can race the delivery goroutine and
		// uses this marker as proof that receipt recording has completed.
		n.markAlertsNotified(job.Alerts, deliveredAt)
	}
}

func (n *NotificationManager) filterResolvedJobsByDeliveryReceipt(jobs []notificationDeliveryJob) []notificationDeliveryJob {
	queue := n.GetQueue()
	filtered := make([]notificationDeliveryJob, 0, len(jobs))
	for _, job := range jobs {
		destinationKey := notificationDeliveryDestinationKey(job)
		if destinationKey == "" {
			continue
		}
		eligibleAlerts := make([]*alerts.Alert, 0, len(job.Alerts))
		for _, alert := range job.Alerts {
			if alert == nil || alert.StartTime.IsZero() {
				continue
			}
			eligible := false
			if queue != nil {
				var err error
				eligible, err = queue.HasDeliveryReceipt(alert.ID, alert.StartTime, destinationKey)
				if err != nil {
					log.Error().Err(err).Str("alertID", alert.ID).Str("destination", destinationKey).Msg("Failed to read notification delivery receipt; suppressing recovery")
					continue
				}
			} else {
				key := notificationDeliveryReceiptKey(alert, destinationKey)
				n.mu.RLock()
				_, eligible = n.deliveryReceipts[key]
				n.mu.RUnlock()
			}
			if eligible {
				eligibleAlerts = append(eligibleAlerts, alert)
				continue
			}
			log.Debug().Str("alertID", alert.ID).Str("destination", destinationKey).Msg("Resolved notification suppressed because this destination did not receive the firing occurrence")
		}
		if len(eligibleAlerts) == 0 {
			continue
		}
		job.Alerts = eligibleAlerts
		filtered = append(filtered, job)
	}
	return filtered
}

// SendResolvedAlert applies the same grouping window to recoveries as firings.
// The two event types have separate buffers; a recovery can never be rendered
// as a firing alert. Normal operation persists the grouping window in the
// delivery queue; the in-memory buffer is only the queue-unavailable fallback.
func (n *NotificationManager) SendResolvedAlert(resolved *alerts.ResolvedAlert) {
	if resolved == nil || resolved.Alert == nil {
		return
	}

	// Clone the alert so downstream goroutines cannot mutate shared state.
	alertCopy := resolved.Alert.Clone()
	if alertCopy == nil {
		return
	}

	resolvedAt := resolved.ResolvedTime
	if resolvedAt.IsZero() {
		resolvedAt = time.Now()
	}
	annotateResolvedMetadata(alertCopy, resolvedAt)

	n.mu.Lock()
	if !n.enabled || !n.notifyOnResolve || !n.lifecycle.accepting() {
		n.mu.Unlock()
		return
	}
	if n.queue == nil && n.groupingEnabled && n.groupWindow > 0 {
		// Repeated reconciliation of one occurrence must not duplicate batch rows.
		for _, pending := range n.pendingResolved {
			if pending.ID == alertCopy.ID && pending.StartTime.Equal(alertCopy.StartTime) {
				n.mu.Unlock()
				return
			}
		}
		n.pendingResolved = append(n.pendingResolved, alertCopy)
		if n.resolvedGroupTimer == nil {
			n.resolvedGroupTimer = time.AfterFunc(n.groupWindow, n.sendGroupedResolvedAlerts)
		}
		n.mu.Unlock()
		return
	}
	n.mu.Unlock()
	n.dispatchResolvedAlerts([]*alerts.Alert{alertCopy})
}

// takePendingResolvedLocked drains only the recovery buffer. Call with mu held.
func (n *NotificationManager) takePendingResolvedLocked() []*alerts.Alert {
	pending := n.pendingResolved
	n.pendingResolved = nil
	if n.resolvedGroupTimer != nil {
		n.resolvedGroupTimer.Stop()
		n.resolvedGroupTimer = nil
	}
	return pending
}

func (n *NotificationManager) sendGroupedResolvedAlerts() {
	n.mu.Lock()
	pending := n.takePendingResolvedLocked()
	n.mu.Unlock()
	n.dispatchResolvedAlerts(pending)
}

func (n *NotificationManager) dispatchResolvedAlerts(alertList []*alerts.Alert) {
	if len(alertList) == 0 {
		return
	}
	n.mu.RLock()
	enabled := n.enabled && n.notifyOnResolve
	emailConfig := copyEmailConfig(n.emailConfig)
	webhooks := copyWebhookConfigs(n.webhooks)
	appriseConfig := copyAppriseConfig(n.appriseConfig)
	queue := n.queue
	// The lease precedes the receipt read below, which already uses the queue.
	admitted := enabled && n.lifecycle.admit()
	n.mu.RUnlock()
	if !admitted {
		return
	}
	defer n.lifecycle.release()
	jobs := buildNotificationDeliveryJobs(emailConfig, webhooks, appriseConfig, alertList, eventResolved, resolvedTimeFromAlerts(alertList))
	// Filter each destination independently at dispatch, not merely when the
	// alert enters the window. Never announce a recovery for an undelivered firing.
	jobs = n.filterResolvedJobsByDeliveryReceipt(jobs)
	if queue != nil {
		n.enqueueNotificationJobs(queue, jobs)
	} else {
		n.dispatchNotificationJobsAsync(jobs)
	}
}

// CancelAlert removes pending notifications for a resolved alert. It reports
// whether the alert's firing notification was cancelled before any delivery:
// true means the user never received the firing notification (it was still in
// the grouping window or waiting in the queue), so sending a recovery
// notification would reference an alert that was never announced.
func (n *NotificationManager) CancelAlert(alertID string) bool {
	return n.cancelAlert(alertID, nil)
}

// CancelResolvedAlert cancels firing work for one resolved occurrence, without
// deleting a newer occurrence's grouping entries or delivery cooldown.
// ID-wide cancellation remains available for explicit administrative callers.
func (n *NotificationManager) CancelResolvedAlert(alert *alerts.Alert) bool {
	if alert == nil || alert.ID == "" || alert.StartTime.IsZero() {
		return false
	}
	return n.cancelAlert(alert.ID, &alert.StartTime)
}

func (n *NotificationManager) cancelAlert(alertID string, start *time.Time) bool {
	n.mu.Lock()
	queue := n.queue
	// Cancellation still runs while Stop drains; its lease keeps the queue
	// open until the cancellation is committed.
	if queue != nil && !n.lifecycle.join() {
		queue = nil
	}
	record, firingDelivered := n.lastNotified[alertID]
	firingDelivered = firingDelivered && (start == nil || record.alertStart.Equal(*start))

	removed := 0
	if len(n.pendingAlerts) > 0 {
		filtered := n.pendingAlerts[:0]
		for _, pending := range n.pendingAlerts {
			if pending == nil {
				continue
			}
			if pending.ID == alertID && (start == nil || pending.StartTime.Equal(*start)) {
				removed++
				continue
			}
			filtered = append(filtered, pending)
		}

		for i := len(filtered); i < len(n.pendingAlerts); i++ {
			n.pendingAlerts[i] = nil
		}

		n.pendingAlerts = filtered
		if removed > 0 {
			n.pendingAlertIndex = nil
		}

		if len(n.pendingAlerts) == 0 && n.groupTimer != nil {
			if n.groupTimer.Stop() {
				log.Debug().Str("alertID", alertID).Msg("stopped grouping timer after alert cancellation")
			}
			n.groupTimer = nil
			n.groupDeadline = time.Time{}
		}
	}

	// Clean up cooldown record for resolved alert
	if start == nil || record.alertStart.Equal(*start) {
		delete(n.lastNotified, alertID)
	}
	remaining := len(n.pendingAlerts)
	n.mu.Unlock()

	// Cancel any queued notifications containing this alert
	cancelledPending := 0
	if queue != nil {
		defer n.lifecycle.release()
		var err error
		if start == nil {
			cancelledPending, err = queue.CancelByAlertIdentifiers([]string{alertID})
		} else {
			cancelledPending, err = queue.CancelByAlertOccurrence(alertID, *start)
		}
		if err != nil {
			log.Error().Err(err).Str("alertID", alertID).Msg("failed to cancel queued notifications")
		}
	}

	firingNeverDelivered := !firingDelivered && (removed > 0 || cancelledPending > 0)

	log.Debug().
		Str("alertID", alertID).
		Int("remaining", remaining).
		Bool("firingNeverDelivered", firingNeverDelivered).
		Msg("removed resolved alert from pending notifications and cooldown map")

	return firingNeverDelivered
}

// sendGroupedAlerts sends all pending alerts as a group
func (n *NotificationManager) sendGroupedAlerts() {
	n.mu.Lock()
	if len(n.pendingAlerts) == 0 {
		n.mu.Unlock()
		return
	}
	// A timer racing Stop must leave the buffer for the shutdown checkpoint,
	// not dispatch it directly or lose it after admission has closed.
	if !n.lifecycle.admit() {
		n.mu.Unlock()
		return
	}
	defer n.lifecycle.release()

	// Copy alerts to send
	alertsToSend := make([]*alerts.Alert, len(n.pendingAlerts))
	copy(alertsToSend, n.pendingAlerts)

	// Clear pending alerts
	clear(n.pendingAlerts)
	n.pendingAlerts = n.pendingAlerts[:0]
	n.pendingAlertIndex = nil
	if n.groupTimer != nil {
		n.groupTimer.Stop()
	}
	n.groupTimer = nil
	n.groupDeadline = time.Time{}

	log.Info().
		Int("alertCount", len(alertsToSend)).
		Msg("sending grouped alert notifications")

	// Snapshot configuration while holding the lock to avoid races with concurrent updates
	emailConfig := copyEmailConfig(n.emailConfig)
	webhooks := copyWebhookConfigs(n.webhooks)
	appriseConfig := copyAppriseConfig(n.appriseConfig)
	initialTarget := n.initialTarget
	queue := n.queue
	n.mu.Unlock()

	n.dispatchFiringAlerts(emailConfig, webhooks, appriseConfig, alertsToSend, initialTarget, nil, queue)
}

func buildNotificationDeliveryJobs(emailConfig EmailConfig, webhooks []WebhookConfig, appriseConfig AppriseConfig, alertsToSend []*alerts.Alert, event notificationEvent, resolvedAt time.Time) []notificationDeliveryJob {
	return buildNotificationDeliveryJobsForTarget(
		emailConfig,
		webhooks,
		appriseConfig,
		alertsToSend,
		event,
		resolvedAt,
		notificationDeliveryTargetAll,
	)
}

func buildNotificationDeliveryJobsForTarget(
	emailConfig EmailConfig,
	webhooks []WebhookConfig,
	appriseConfig AppriseConfig,
	alertsToSend []*alerts.Alert,
	event notificationEvent,
	resolvedAt time.Time,
	target notificationDeliveryTarget,
) []notificationDeliveryJob {
	return buildNotificationDeliveryJobsForSelection(
		emailConfig,
		webhooks,
		appriseConfig,
		alertsToSend,
		event,
		resolvedAt,
		target,
		nil,
	)
}

func buildNotificationDeliveryJobsForSelection(
	emailConfig EmailConfig,
	webhooks []WebhookConfig,
	appriseConfig AppriseConfig,
	alertsToSend []*alerts.Alert,
	event notificationEvent,
	resolvedAt time.Time,
	target notificationDeliveryTarget,
	destinationIDs []string,
) []notificationDeliveryJob {
	if target == "" {
		target = notificationDeliveryTargetAll
	}
	destinationIDs = normalizeEscalationDestinationIDs(destinationIDs)
	exactSelection := len(destinationIDs) > 0
	selected := make(map[string]struct{}, len(destinationIDs))
	for _, id := range destinationIDs {
		selected[id] = struct{}{}
	}
	isSelected := func(id string) bool {
		_, ok := selected[id]
		return ok
	}

	jobs := make([]notificationDeliveryJob, 0, 2+len(webhooks))
	if emailConfig.Enabled && ((exactSelection && isSelected("email")) || (!exactSelection && (target == notificationDeliveryTargetAll || target == notificationDeliveryTargetEmail))) {
		routedAlerts := routeNotificationAlerts(alertsToSend, emailConfig.TagFilter, emailConfig.TagMode, emailConfig.MinimumSeverity, event)
		if len(routedAlerts) > 0 {
			emailCopy := emailConfig
			jobs = append(jobs, notificationDeliveryJob{
				Type:        "email",
				Event:       event,
				Alerts:      routedAlerts,
				ResolvedAt:  resolvedAt,
				EmailConfig: &emailCopy,
			})
		}
	}
	if exactSelection || target == notificationDeliveryTargetAll || target == notificationDeliveryTargetWebhook {
		for _, webhook := range webhooks {
			if !webhook.Enabled || (exactSelection && !isSelected("webhook:"+strings.TrimSpace(webhook.ID))) {
				continue
			}
			routedAlerts := routeNotificationAlerts(alertsToSend, webhook.TagFilter, webhook.TagMode, webhook.MinimumSeverity, event)
			if len(routedAlerts) > 0 {
				webhookCopy := webhook
				// PagerDuty Events v2 resolves one dedup_key, not a list.
				// Never consume other occurrences' receipts for that request.
				if event == eventResolved && webhook.Service == "pagerduty" && strings.TrimSpace(webhook.Template) == "" {
					for _, alert := range routedAlerts {
						jobs = append(jobs, notificationDeliveryJob{Type: "webhook", Event: event, Alerts: []*alerts.Alert{alert}, ResolvedAt: resolvedAt, WebhookConfig: &webhookCopy})
					}
					continue
				}
				jobs = append(jobs, notificationDeliveryJob{
					Type:          "webhook",
					Event:         event,
					Alerts:        routedAlerts,
					ResolvedAt:    resolvedAt,
					WebhookConfig: &webhookCopy,
				})
			}
		}
	}
	if appriseConfig.Enabled && ((exactSelection && isSelected("apprise")) || (!exactSelection && (target == notificationDeliveryTargetAll || target == notificationDeliveryTargetApprise))) {
		routedAlerts := routeNotificationAlerts(alertsToSend, nil, "", appriseConfig.MinimumSeverity, event)
		if len(routedAlerts) > 0 {
			appriseCopy := appriseConfig
			jobs = append(jobs, notificationDeliveryJob{
				Type:          "apprise",
				Event:         event,
				Alerts:        routedAlerts,
				ResolvedAt:    resolvedAt,
				AppriseConfig: &appriseCopy,
			})
		}
	}
	return jobs
}

func normalizeEscalationDestinationIDs(destinationIDs []string) []string {
	return alerts.NormalizeEscalationDestinationIDs(destinationIDs)
}

func queueTypeForNotificationDeliveryJob(job notificationDeliveryJob) string {
	queueType := job.Type
	if job.Event == eventResolved {
		queueType += queueTypeSuffixResolved
	}
	return queueType
}

func normalizeNotificationDeliveryTarget(raw string) notificationDeliveryTarget {
	switch strings.ToLower(strings.TrimSpace(raw)) {
	case "", string(notificationDeliveryTargetAll):
		return notificationDeliveryTargetAll
	case string(notificationDeliveryTargetEmail):
		return notificationDeliveryTargetEmail
	case string(notificationDeliveryTargetWebhook), "webhooks":
		return notificationDeliveryTargetWebhook
	case string(notificationDeliveryTargetApprise):
		return notificationDeliveryTargetApprise
	default:
		return notificationDeliveryTargetAll
	}
}

func configJSONForNotificationDeliveryJob(job notificationDeliveryJob) ([]byte, error) {
	switch job.Type {
	case "email":
		if job.EmailConfig == nil {
			return nil, fmt.Errorf("missing email config")
		}
		return json.Marshal(*job.EmailConfig)
	case "webhook":
		if job.WebhookConfig == nil {
			return nil, fmt.Errorf("missing webhook config")
		}
		return json.Marshal(*job.WebhookConfig)
	case "apprise":
		if job.AppriseConfig == nil {
			return nil, FailfWithClass(NotificationFailureConfiguration, "missing apprise config")
		}
		return json.Marshal(*job.AppriseConfig)
	default:
		return nil, fmt.Errorf("unknown notification type: %s", job.Type)
	}
}

func notificationDestinationIDForJob(job notificationDeliveryJob) string {
	parts := []string{job.Type}
	switch job.Type {
	case "email":
		if job.EmailConfig != nil {
			recipients := append([]string(nil), job.EmailConfig.To...)
			sort.Strings(recipients)
			parts = append(
				parts,
				job.EmailConfig.Provider,
				job.EmailConfig.SMTPHost,
				strconv.Itoa(job.EmailConfig.SMTPPort),
				job.EmailConfig.From,
				strings.Join(recipients, "\x00"),
			)
		}
	case "webhook":
		if job.WebhookConfig != nil {
			if id := strings.TrimSpace(job.WebhookConfig.ID); id != "" {
				return "webhook:" + id
			}
			parts = append(
				parts,
				job.WebhookConfig.Name,
				job.WebhookConfig.Service,
				job.WebhookConfig.URL,
			)
		}
	case "apprise":
		if job.AppriseConfig != nil {
			targets := append([]string(nil), job.AppriseConfig.Targets...)
			sort.Strings(targets)
			parts = append(
				parts,
				string(job.AppriseConfig.Mode),
				job.AppriseConfig.ServerURL,
				job.AppriseConfig.ConfigKey,
				strings.Join(targets, "\x00"),
			)
		}
	}
	sum := sha256.Sum256([]byte(strings.Join(parts, "\x00")))
	return "destination:" + hex.EncodeToString(sum[:16])
}

func (n *NotificationManager) enqueueNotificationJobs(queue *NotificationQueue, jobs []notificationDeliveryJob) bool {
	if queue == nil {
		return false
	}

	n.mu.RLock()
	groupWindow := n.groupWindow
	if !n.groupingEnabled {
		groupWindow = 0
	}
	n.mu.RUnlock()
	anyFailed := false
	for _, job := range jobs {
		configJSON, err := configJSONForNotificationDeliveryJob(job)
		if err != nil {
			n.logNotificationJobError(job, err, "failed to marshal config for queue")
			continue
		}

		for _, bucket := range notificationQueueBucketsForJob(job, time.Now()) {
			notif := &QueuedNotification{
				Type:          queueTypeForNotificationDeliveryJob(bucket.job),
				DestinationID: notificationDestinationIDForJob(bucket.job),
				Alerts:        bucket.job.Alerts,
				Config:        configJSON,
				MaxAttempts:   3,
				NextRetryAt:   bucket.nextRetryAt,
			}
			var enqueueErr error
			if job.Event == eventResolved && groupWindow > 0 && !(job.WebhookConfig != nil && job.WebhookConfig.Service == "pagerduty" && strings.TrimSpace(job.WebhookConfig.Template) == "") {
				enqueueErr = queue.enqueueResolvedGroup(notif, groupWindow)
			} else {
				enqueueErr = queue.Enqueue(notif)
			}
			if err := enqueueErr; err != nil {
				anyFailed = true
				n.logNotificationJobError(bucket.job, err, "failed to enqueue notification - falling back to direct send")
				n.dispatchNotificationJobAsync(bucket.job, "failed to send notification after queue enqueue failure")
				continue
			}

			logger := log.Debug().
				Str("type", bucket.job.Type).
				Str("event", string(bucket.job.Event)).
				Int("alertCount", len(bucket.job.Alerts))
			if notif.NextRetryAt != nil {
				logger = logger.Time("nextRetryAt", *notif.NextRetryAt)
			}
			if bucket.job.WebhookConfig != nil {
				logger = logger.Str("webhookName", bucket.job.WebhookConfig.Name)
			}
			logger.Msg("enqueued notification delivery job")
		}
	}

	if anyFailed && len(jobs) > 0 && jobs[0].Event == eventResolved {
		log.Debug().Msg("at least one resolved notification enqueue failed; direct sends were triggered")
	}
	return anyFailed
}

func (n *NotificationManager) dispatchNotificationJobsAsync(jobs []notificationDeliveryJob) {
	for _, job := range jobs {
		if job.Type == "email" && job.EmailConfig != nil {
			log.Info().
				Int("alertCount", len(job.Alerts)).
				Str("smtpHost", job.EmailConfig.SMTPHost).
				Int("smtpPort", job.EmailConfig.SMTPPort).
				Strs("recipients", job.EmailConfig.To).
				Msg("email notifications enabled - dispatching delivery job")
		}
		n.dispatchNotificationJobAsync(job, "failed to send notification")
	}
}

func (n *NotificationManager) sendNotificationsDirect(emailConfig EmailConfig, webhooks []WebhookConfig, appriseConfig AppriseConfig, alertsToSend []*alerts.Alert) {
	n.dispatchNotificationJobsAsync(buildNotificationDeliveryJobs(emailConfig, webhooks, appriseConfig, alertsToSend, eventAlert, time.Time{}))
}

func (n *NotificationManager) sendResolvedNotificationsDirect(emailConfig EmailConfig, webhooks []WebhookConfig, appriseConfig AppriseConfig, alertsToSend []*alerts.Alert, resolvedAt time.Time) {
	if len(alertsToSend) == 0 {
		return
	}
	n.dispatchNotificationJobsAsync(buildNotificationDeliveryJobs(emailConfig, webhooks, appriseConfig, alertsToSend, eventResolved, resolvedAt))
}

func (n *NotificationManager) dispatchNotificationJobAsync(job notificationDeliveryJob, failureMessage string) {
	// Direct delivery stays owned until it finishes, so Stop waits for it and
	// nothing is sent, or recorded only in memory, after Stop returns.
	if !n.lifecycle.join() {
		log.Warn().
			Str("type", job.Type).
			Str("event", string(job.Event)).
			Int("alertCount", len(job.Alerts)).
			Msg("notification manager stopped; direct delivery refused")
		return
	}
	spawnAsync(func() {
		defer n.lifecycle.release()
		if err := n.deliverNotificationJob(job); err != nil {
			n.logNotificationJobError(job, err, failureMessage)
			return
		}
		n.recordSuccessfulDelivery(job, time.Now())
	})
}

func (n *NotificationManager) deliverNotificationJob(job notificationDeliveryJob) error {
	switch job.Type {
	case "email":
		if job.EmailConfig == nil {
			return fmt.Errorf("missing email config")
		}
		if job.Event == eventResolved {
			return n.sendResolvedEmail(*job.EmailConfig, job.Alerts, job.ResolvedAt)
		}
		return n.sendGroupedEmail(*job.EmailConfig, job.Alerts)
	case "webhook":
		if job.WebhookConfig == nil {
			return fmt.Errorf("missing webhook config")
		}
		if job.Event == eventResolved {
			return n.sendResolvedWebhook(*job.WebhookConfig, job.Alerts, job.ResolvedAt)
		}
		return n.sendGroupedWebhook(*job.WebhookConfig, job.Alerts)
	case "apprise":
		if job.AppriseConfig == nil {
			return FailfWithClass(NotificationFailureConfiguration, "missing apprise config")
		}
		if job.Event == eventResolved {
			return n.sendResolvedApprise(*job.AppriseConfig, job.Alerts, job.ResolvedAt)
		}
		return n.sendGroupedApprise(*job.AppriseConfig, job.Alerts)
	default:
		return fmt.Errorf("unknown notification type: %s", job.Type)
	}
}

func (n *NotificationManager) logNotificationJobError(job notificationDeliveryJob, err error, message string) {
	logger := log.Error().
		Err(err).
		Str("type", job.Type).
		Str("event", string(job.Event))
	if job.WebhookConfig != nil {
		logger = logger.Str("webhookName", job.WebhookConfig.Name)
	}
	logger.Msg(message)
}

// alertListThreadID returns the incident thread ID when the email covers
// exactly one alert. Grouped emails don't thread: firing and resolved
// batches rarely contain the same alert set, so a group-level reference
// would attach messages to the wrong thread more often than the right one.
func alertListThreadID(alertList []*alerts.Alert) string {
	if len(alertList) != 1 || alertList[0] == nil {
		return ""
	}
	return alertThreadMessageID(alertList[0].ID, alertList[0].StartTime)
}

// sendGroupedEmail sends a grouped email notification
func (n *NotificationManager) sendGroupedEmail(config EmailConfig, alertList []*alerts.Alert) error {

	// Don't check for recipients here - sendHTMLEmail handles empty recipients
	// by using the From address as the recipient

	// Generate email using template
	subject, htmlBody, textBody := EmailTemplate(alertList, false, n.publicURL)

	// Send using HTML-aware method
	return n.sendThreadedHTMLEmailWithError(subject, htmlBody, textBody, alertListThreadID(alertList), config)
}

func (n *NotificationManager) sendResolvedEmail(config EmailConfig, alertList []*alerts.Alert, resolvedAt time.Time) error {
	if len(alertList) == 0 {
		return fmt.Errorf("no alerts to send")
	}

	subject, htmlBody, textBody := buildResolvedNotificationContent(alertList, resolvedAt, n.publicURL)
	if subject == "" && textBody == "" {
		return fmt.Errorf("failed to build resolved email content")
	}
	subject = resolvedEmailSubject(alertList)

	return n.sendThreadedHTMLEmailWithError(subject, htmlBody, textBody, alertListThreadID(alertList), config)
}

func (n *NotificationManager) sendGroupedApprise(config AppriseConfig, alertList []*alerts.Alert) error {
	if len(alertList) == 0 {
		return fmt.Errorf("no alerts to send")
	}

	cfg := NormalizeAppriseConfig(config)
	if !cfg.Enabled {
		return FailfWithClass(NotificationFailureConfiguration, "apprise not enabled")
	}

	title, body, notifyType := buildApprisePayload(alertList, n.publicURL)
	if title == "" && body == "" {
		return fmt.Errorf("failed to build apprise payload")
	}

	switch cfg.Mode {
	case AppriseModeHTTP:
		if err := n.sendAppriseViaHTTP(cfg, title, body, notifyType); err != nil {
			log.Warn().
				Err(err).
				Str("mode", string(cfg.Mode)).
				Msg("failed to send Apprise notification via API")
			return fmt.Errorf("apprise HTTP send failed: %w", err)
		}
	default:
		if err := n.sendAppriseViaCLI(cfg, title, body); err != nil {
			log.Warn().
				Err(err).
				Str("mode", string(cfg.Mode)).
				Int("targetCount", len(cfg.Targets)).
				Msg("failed to send Apprise notification")
			return fmt.Errorf("apprise CLI send failed: %w", err)
		}
	}
	return nil
}

func buildApprisePayload(alertList []*alerts.Alert, publicURL string) (string, string, string) {
	validAlerts := make([]*alerts.Alert, 0, len(alertList))
	var primary *alerts.Alert
	for _, alert := range alertList {
		if alert == nil {
			continue
		}
		if primary == nil {
			primary = alert
		}
		validAlerts = append(validAlerts, alert)
	}

	if len(validAlerts) == 0 || primary == nil {
		return "", "", "info"
	}

	title := fmt.Sprintf("Pulse alert: %s", primary.ResourceName)
	if len(validAlerts) > 1 {
		title = fmt.Sprintf("Pulse alerts (%d)", len(validAlerts))
	}

	var bodyBuilder strings.Builder
	bodyBuilder.WriteString(primary.Message)
	bodyBuilder.WriteString("\n\n")

	for _, alert := range validAlerts {
		bodyBuilder.WriteString(fmt.Sprintf("[%s] %s", strings.ToUpper(string(alert.Level)), alert.ResourceName))
		bodyBuilder.WriteString(fmt.Sprintf(" — value %.2f (threshold %.2f)\n", alert.Value, alert.Threshold))
		if alert.Node != "" {
			bodyBuilder.WriteString(fmt.Sprintf("Node: %s\n", alertNodeDisplay(alert)))
		}
		if alert.Instance != "" && alert.Instance != alert.Node {
			bodyBuilder.WriteString(fmt.Sprintf("Instance: %s\n", alert.Instance))
		}
		if link := alertLinkURL(publicURL, alert); link != "" {
			bodyBuilder.WriteString("View in Pulse: " + link + "\n")
		}
		bodyBuilder.WriteString("\n")
	}

	if publicURL != "" {
		bodyBuilder.WriteString("Dashboard: " + publicURL + "\n")
	}

	return title, bodyBuilder.String(), resolveAppriseNotificationType(validAlerts)
}

func buildResolvedNotificationContent(alertList []*alerts.Alert, resolvedAt time.Time, publicURL string) (string, string, string) {
	validAlerts := make([]*alerts.Alert, 0, len(alertList))
	var primary *alerts.Alert
	for _, alert := range alertList {
		if alert == nil {
			continue
		}
		if primary == nil {
			primary = alert
		}
		validAlerts = append(validAlerts, alert)
	}

	if len(validAlerts) == 0 || primary == nil {
		return "", "", ""
	}

	if resolvedAt.IsZero() {
		resolvedAt = time.Now()
	}
	resolvedLabel := resolvedAt.Format(time.RFC3339)

	title := fmt.Sprintf("Pulse alert resolved: %s", primary.ResourceName)
	if primary.Resolution.Outcome() != "" {
		title = fmt.Sprintf("Pulse alert moved: %s", primary.ResourceName)
	}
	if len(validAlerts) > 1 {
		title = fmt.Sprintf("Pulse alerts resolved (%d)", len(validAlerts))
	}

	var bodyBuilder strings.Builder
	bodyBuilder.WriteString("Resolved at ")
	bodyBuilder.WriteString(resolvedLabel)
	bodyBuilder.WriteString("\n\n")

	for _, alert := range validAlerts {
		bodyBuilder.WriteString(fmt.Sprintf("[%s] %s\n", strings.ToUpper(string(alert.Level)), alert.ResourceName))
		if alert.Message != "" {
			bodyBuilder.WriteString(alert.Message)
			bodyBuilder.WriteString("\n")
		}
		if summary := resolvedAlertNotRecoveredSummary(alert); summary != "" {
			bodyBuilder.WriteString(summary)
			bodyBuilder.WriteString("\n")
		}
		if !alert.StartTime.IsZero() {
			bodyBuilder.WriteString("Started: ")
			bodyBuilder.WriteString(alert.StartTime.Format(time.RFC3339))
			bodyBuilder.WriteString("\n")
		}
		bodyBuilder.WriteString("Cleared: ")
		bodyBuilder.WriteString(resolvedLabel)
		bodyBuilder.WriteString("\n")
		if alert.Node != "" {
			bodyBuilder.WriteString("Node: ")
			bodyBuilder.WriteString(alertNodeDisplay(alert))
			bodyBuilder.WriteString("\n")
		}
		if alert.Instance != "" && alert.Instance != alert.Node {
			bodyBuilder.WriteString("Instance: ")
			bodyBuilder.WriteString(alert.Instance)
			bodyBuilder.WriteString("\n")
		}
		if alert.Threshold != 0 || alert.Value != 0 {
			bodyBuilder.WriteString(fmt.Sprintf("Last value %.2f (threshold %.2f)\n", alert.Value, alert.Threshold))
		}
		if link := alertLinkURL(publicURL, alert); link != "" {
			bodyBuilder.WriteString("View in Pulse: ")
			bodyBuilder.WriteString(link)
			bodyBuilder.WriteString("\n")
		}
		bodyBuilder.WriteString("\n")
	}

	if publicURL != "" {
		bodyBuilder.WriteString("Dashboard: ")
		bodyBuilder.WriteString(publicURL)
		bodyBuilder.WriteString("\n")
	}

	textBody := bodyBuilder.String()
	htmlBody := "<pre style=\"font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, \\\"Liberation Mono\\\", \\\"Courier New\\\", monospace\">" +
		html.EscapeString(textBody) + "</pre>"

	return title, htmlBody, textBody
}

func resolveAppriseNotificationType(alertList []*alerts.Alert) string {
	notifyType := "info"
	for _, alert := range alertList {
		if alert == nil {
			continue
		}
		switch alert.Level {
		case alerts.AlertLevelCritical:
			return "failure"
		case alerts.AlertLevelWarning:
			notifyType = "warning"
		}
	}
	return notifyType
}

func (n *NotificationManager) sendAppriseViaCLI(cfg AppriseConfig, title, body string) error {
	if len(cfg.Targets) == 0 {
		return FailfWithClass(NotificationFailureConfiguration, "no Apprise targets configured for CLI delivery")
	}

	ctx, cancel := context.WithTimeout(context.Background(), time.Duration(cfg.TimeoutSeconds)*time.Second)
	defer cancel()

	args := []string{"-t", title, "-b", body}
	args = append(args, cfg.Targets...)

	execFn := n.appriseExec
	if execFn == nil {
		execFn = defaultAppriseExec
	}

	output, err := execFn(ctx, args)
	if err != nil {
		if len(output) > 0 {
			log.Debug().
				Str("cliPath", "apprise").
				Int("targetCount", len(cfg.Targets)).
				Int("outputBytes", len(output)).
				Msg("apprise CLI output (error)")
		}
		return safeAppriseError("execute apprise CLI", err)
	}

	if len(output) > 0 {
		log.Debug().
			Str("cliPath", "apprise").
			Int("targetCount", len(cfg.Targets)).
			Int("outputBytes", len(output)).
			Msg("apprise CLI output")
	}
	return nil
}

func (n *NotificationManager) validatedWebhookBaseURL(rawURL string) (*url.URL, error) {
	baseURL, err := securityutil.NormalizeHTTPBaseURL(rawURL, "")
	if err != nil {
		return nil, err
	}
	if err := n.ValidateWebhookURL(baseURL.String()); err != nil {
		return nil, err
	}
	return baseURL, nil
}

func (n *NotificationManager) sendAppriseViaHTTP(cfg AppriseConfig, title, body, notifyType string) error {
	if cfg.ServerURL == "" {
		return fmt.Errorf("apprise server URL is not configured")
	}

	serverURL := cfg.ServerURL
	lowerURL := strings.ToLower(serverURL)
	if !strings.HasPrefix(lowerURL, "http://") && !strings.HasPrefix(lowerURL, "https://") {
		return fmt.Errorf("apprise server URL must start with http or https")
	}

	validatedBaseURL, err := securityutil.NormalizeHTTPBaseURL(serverURL, "")
	if err == nil {
		err = n.validateWebhookURL(validatedBaseURL.String(), appriseDiagnosticURL)
	}
	if err != nil {
		err = safeAppriseError("apprise server URL validation failed", err)
		log.Error().
			Err(err).
			Msg("apprise server URL validation failed - possible SSRF attempt")
		return err
	}

	ctx, cancel := context.WithTimeout(context.Background(), time.Duration(cfg.TimeoutSeconds)*time.Second)
	defer cancel()

	notifyEndpoint := "/notify"
	if cfg.ConfigKey != "" {
		notifyEndpoint = "/notify/" + url.PathEscape(cfg.ConfigKey)
	}

	targetURL, err := securityutil.ResolveRelativeURL(validatedBaseURL, notifyEndpoint)
	if err != nil {
		return safeAppriseError("apprise server URL validation failed", err)
	}

	payload := map[string]any{
		"body":  body,
		"title": title,
	}
	if len(cfg.Targets) > 0 {
		payload["urls"] = cfg.Targets
	}
	if notifyType != "" {
		payload["type"] = notifyType
	}

	payloadBytes, err := json.Marshal(payload)
	if err != nil {
		return fmt.Errorf("failed to marshal Apprise payload: %w", err)
	}

	req, err := securityutil.NewValidatedRequestWithContext(ctx, http.MethodPost, targetURL, bytes.NewReader(payloadBytes))
	if err != nil {
		return safeAppriseError("failed to create Apprise request", err)
	}
	req.Header.Set("Content-Type", "application/json")
	req.Header.Set("Accept", "application/json")

	if cfg.APIKey != "" {
		if cfg.APIKeyHeader == "" {
			req.Header.Set("X-API-KEY", cfg.APIKey)
		} else {
			req.Header.Set(cfg.APIKeyHeader, cfg.APIKey)
		}
	}

	client := n.createSecureWebhookClientWithURLDiagnostics(
		time.Duration(cfg.TimeoutSeconds)*time.Second,
		validatedBaseURL.Scheme == "https" && cfg.SkipTLSVerify,
		appriseDiagnosticURL,
	)

	resp, err := client.Do(req)
	if err != nil {
		return safeAppriseError("failed to reach Apprise server", err)
	}
	defer resp.Body.Close()

	// Share the bounded response verdict with webhooks. Receiving 2xx headers
	// does not make a failed body read successful delivery or a durable receipt.
	result, err := readWebhookResponse(resp)
	if err != nil {
		operation := "read Apprise server response"
		if resp.StatusCode < 200 || resp.StatusCode >= 300 {
			operation = fmt.Sprintf("apprise server returned HTTP %d", resp.StatusCode)
		}
		return safeAppriseError(operation, err)
	}

	if result.responseBytes > 0 {
		log.Debug().
			Str("mode", string(cfg.Mode)).
			Int("statusCode", resp.StatusCode).
			Int64("responseBytes", result.responseBytes).
			Msg("apprise API response")
	}

	return nil
}

func (n *NotificationManager) sendResolvedApprise(config AppriseConfig, alertList []*alerts.Alert, resolvedAt time.Time) error {
	if len(alertList) == 0 {
		return fmt.Errorf("no alerts to send")
	}

	cfg := NormalizeAppriseConfig(config)
	if !cfg.Enabled {
		return FailfWithClass(NotificationFailureConfiguration, "apprise not enabled")
	}

	title, _, body := buildResolvedNotificationContent(alertList, resolvedAt, n.publicURL)
	if title == "" && body == "" {
		return fmt.Errorf("failed to build resolved apprise payload")
	}

	switch cfg.Mode {
	case AppriseModeHTTP:
		if err := n.sendAppriseViaHTTP(cfg, title, body, "info"); err != nil {
			log.Warn().
				Err(err).
				Str("mode", string(cfg.Mode)).
				Msg("failed to send resolved Apprise notification via API")
			return fmt.Errorf("apprise HTTP send failed: %w", err)
		}
	default:
		if err := n.sendAppriseViaCLI(cfg, title, body); err != nil {
			log.Warn().
				Err(err).
				Str("mode", string(cfg.Mode)).
				Int("targetCount", len(cfg.Targets)).
				Msg("failed to send resolved Apprise notification")
			return fmt.Errorf("apprise CLI send failed: %w", err)
		}
	}
	return nil
}

// sendEmail sends an email notification
func (n *NotificationManager) sendSingleEmailWithError(alert *alerts.Alert, config EmailConfig) error {
	subject, htmlBody, textBody := EmailTemplate([]*alerts.Alert{alert}, true, n.publicURL)
	return n.sendThreadedHTMLEmailWithError(subject, htmlBody, textBody, alertListThreadID([]*alerts.Alert{alert}), config)
}

func (n *NotificationManager) sendEmail(alert *alerts.Alert) {
	n.mu.RLock()
	config := n.emailConfig
	n.mu.RUnlock()

	if err := n.sendSingleEmailWithError(alert, config); err != nil {
		log.Error().
			Err(err).
			Str("alertID", alert.ID).
			Msg("failed to send alert email notification")
	}
}

func effectiveEmailRecipients(config EmailConfig) []string {
	recipients := config.To
	if len(recipients) == 0 && config.From != "" {
		recipients = []string{config.From}
	}
	return recipients
}

func newEmailDeliveryManager(config EmailConfig, recipients []string) *EnhancedEmailManager {
	rl := effectiveEmailRateLimit(config.RateLimit)
	return NewEnhancedEmailManager(EmailProviderConfig{
		EmailConfig: EmailConfig{
			Provider:  config.Provider,
			From:      config.From,
			To:        recipients,
			SMTPHost:  config.SMTPHost,
			SMTPPort:  config.SMTPPort,
			Username:  config.Username,
			Password:  config.Password,
			TLS:       config.TLS,
			StartTLS:  config.StartTLS,
			RateLimit: config.RateLimit,
		},
		Provider:      config.Provider,
		StartTLS:      config.StartTLS,
		MaxRetries:    2,
		RetryDelay:    3,
		RateLimit:     rl,
		SkipTLSVerify: false,
		AuthRequired:  config.Username != "" && config.Password != "",
	})
}

func emailDeliveryTransportMatches(manager *EnhancedEmailManager, config EmailConfig) bool {
	if manager == nil {
		return false
	}
	persisted := manager.config.EmailConfig
	persistedProvider := manager.config.Provider
	if persistedProvider == "" {
		persistedProvider = persisted.Provider
	}
	return persisted.SMTPHost == config.SMTPHost &&
		persisted.SMTPPort == config.SMTPPort &&
		persisted.Username == config.Username &&
		persisted.Password == config.Password &&
		persisted.TLS == config.TLS &&
		persisted.StartTLS == config.StartTLS &&
		persistedProvider == config.Provider
}

func (n *NotificationManager) emailDeliveryManager(config EmailConfig) (*EnhancedEmailManager, []string) {
	recipients := effectiveEmailRecipients(config)

	n.mu.RLock()
	manager := n.emailManager
	n.mu.RUnlock()

	if manager == nil {
		return newEmailDeliveryManager(config, recipients), recipients
	}

	if !emailDeliveryTransportMatches(manager, config) {
		return newEmailDeliveryManager(config, recipients), recipients
	}

	// Concurrent queue workers and reports may share an SMTP transport while
	// carrying different admitted envelopes. Keep the cached manager immutable:
	// otherwise preparing one delivery changes another delivery's recipients,
	// sender and rate policy while it is being sent or waiting to retry.
	deliveryConfig := manager.config
	deliveryConfig.EmailConfig = copyEmailConfig(config)
	deliveryConfig.To = append([]string(nil), recipients...)
	deliveryConfig.Provider = config.Provider
	deliveryConfig.StartTLS = config.StartTLS
	deliveryConfig.RateLimit = effectiveEmailRateLimit(config.RateLimit)
	deliveryConfig.AuthRequired = config.Username != "" && config.Password != ""
	delivery := NewEnhancedEmailManager(deliveryConfig)
	// Isolating envelopes must not give every delivery a fresh rate budget.
	delivery.rateLimit = manager.rateLimit
	return delivery, recipients
}

// sendHTMLEmailWithError sends an HTML email with multipart content and returns any error
func (n *NotificationManager) sendHTMLEmailWithError(subject, htmlBody, textBody string, config EmailConfig) error {
	return n.sendThreadedHTMLEmailWithError(subject, htmlBody, textBody, "", config)
}

// sendThreadedHTMLEmailWithError is sendHTMLEmailWithError plus an optional
// incident thread ID (see alertThreadMessageID).
func (n *NotificationManager) sendThreadedHTMLEmailWithError(subject, htmlBody, textBody, threadID string, config EmailConfig) error {
	config = normalizeEmailConfig(config)

	recipients := effectiveEmailRecipients(config)
	if len(config.To) == 0 && config.From != "" {
		log.Info().
			Str("from", config.From).
			Msg("using From address as recipient since To is empty")
	}

	manager, recipients := n.emailDeliveryManager(config)

	log.Info().
		Str("smtp", fmt.Sprintf("%s:%d", config.SMTPHost, config.SMTPPort)).
		Str("from", config.From).
		Strs("to", recipients).
		Bool("hasAuth", manager.config.AuthRequired).
		Bool("startTLS", manager.config.StartTLS).
		Msg("attempting to send email via SMTP with enhanced support")

	err := manager.SendEmailThreaded(subject, htmlBody, textBody, threadID)

	if err != nil {
		log.Error().
			Err(err).
			Str("smtp", fmt.Sprintf("%s:%d", config.SMTPHost, config.SMTPPort)).
			Strs("recipients", recipients).
			Msg("failed to send email notification")
		return fmt.Errorf("failed to send email: %w", err)
	}

	log.Info().
		Strs("recipients", recipients).
		Int("recipientCount", len(recipients)).
		Msg("email notification sent successfully")
	return nil
}

type webhookRenderMode string

const (
	webhookRenderModeSingle   webhookRenderMode = "single"
	webhookRenderModeGrouped  webhookRenderMode = "grouped"
	webhookRenderModeResolved webhookRenderMode = "resolved"
)

func (n *NotificationManager) prepareWebhookDeliveryContext(webhook WebhookConfig, data WebhookPayloadData) (WebhookConfig, WebhookPayloadData, error) {
	renderedURL, renderErr := renderWebhookURL(webhook.URL, data)
	if renderErr != nil {
		return webhook, data, fmt.Errorf("render webhook URL template: %w", renderErr)
	}
	webhook.URL = renderedURL

	switch webhook.Service {
	case "telegram":
		chatID, chatErr := extractTelegramChatID(renderedURL)
		if chatErr != nil {
			return webhook, data, fmt.Errorf("extract Telegram chat_id: %w", chatErr)
		}
		if chatID != "" {
			data.ChatID = chatID
		}
	case "pagerduty":
		if data.CustomFields == nil {
			data.CustomFields = make(map[string]interface{})
		}
		if routingKey, ok := webhook.Headers["routing_key"]; ok {
			data.CustomFields["routing_key"] = routingKey
		}
	}

	// Without a custom template Pulse renders the service's built-in body, so
	// the template's headers describe that body. The stored headers are only a
	// copy the UI seeded at creation, and an edit can blank or replace them;
	// Telegram then ignores the JSON and answers "message text is empty"
	// (#2540). Test sends always applied these headers, so they passed while
	// every real alert failed.
	if strings.TrimSpace(webhook.Template) == "" {
		if tmpl, ok := builtInWebhookServiceTemplate(webhook.Service); ok {
			webhook = withWebhookTemplateHeaders(webhook, tmpl.Headers)
		}
	}

	return webhook, data, nil
}

// builtInWebhookServiceTemplate returns the template Pulse renders for a
// service without a custom template. Generic webhooks send the fallback
// payload instead, so they have none.
func builtInWebhookServiceTemplate(service string) (WebhookTemplate, bool) {
	if service == "" || service == "generic" {
		return WebhookTemplate{}, false
	}
	for _, tmpl := range GetWebhookTemplates() {
		if tmpl.Service == service {
			return tmpl, true
		}
	}
	return WebhookTemplate{}, false
}

// withWebhookTemplateHeaders returns a copy of webhook whose headers carry the
// template's static headers, replacing stored values under any spelling of the
// same name. Templated values are skipped, as executeWebhookRequest skips them.
func withWebhookTemplateHeaders(webhook WebhookConfig, templateHeaders map[string]string) WebhookConfig {
	webhook = copyWebhookConfig(webhook)
	for key, value := range templateHeaders {
		if strings.Contains(value, "{{") {
			continue
		}
		if webhook.Headers == nil {
			webhook.Headers = make(map[string]string, len(templateHeaders))
		}
		for stored := range webhook.Headers {
			if strings.EqualFold(stored, key) {
				delete(webhook.Headers, stored)
			}
		}
		webhook.Headers[key] = value
	}
	return webhook
}

func webhookServiceTemplate(webhook WebhookConfig, mode webhookRenderMode) (string, bool) {
	if webhook.Service == "" || webhook.Service == "generic" {
		return "", false
	}

	for _, tmpl := range GetWebhookTemplates() {
		if tmpl.Service != webhook.Service {
			continue
		}
		switch mode {
		case webhookRenderModeResolved:
			if tmpl.ResolvedPayloadTemplate != "" {
				return tmpl.ResolvedPayloadTemplate, true
			}
		default:
			if tmpl.PayloadTemplate != "" {
				return tmpl.PayloadTemplate, true
			}
		}
		break
	}

	return "", false
}

func (n *NotificationManager) renderWebhookPayloadJSON(webhook WebhookConfig, data WebhookPayloadData, mode webhookRenderMode, fallback func() ([]byte, error)) ([]byte, error) {
	if webhook.Template != "" && strings.TrimSpace(webhook.Template) != "" {
		jsonData, err := n.generatePayloadFromTemplateWithService(webhook.Template, data, webhook.Service)
		if err != nil {
			return nil, fmt.Errorf("generate webhook payload from custom template: %w", err)
		}
		return jsonData, nil
	}

	if templateStr, ok := webhookServiceTemplate(webhook, mode); ok {
		jsonData, err := n.generatePayloadFromTemplateWithService(templateStr, data, webhook.Service)
		if err != nil {
			return nil, fmt.Errorf("generate webhook payload for service %s: %w", webhook.Service, err)
		}
		return jsonData, nil
	}

	return fallback()
}

func withNtfyAlertHeaders(webhook WebhookConfig, alertList []*alerts.Alert) WebhookConfig {
	webhook = copyWebhookConfig(webhook)
	if webhook.Headers == nil {
		webhook.Headers = make(map[string]string)
	}

	level := alerts.AlertLevelInfo
	var primary *alerts.Alert
	for _, alert := range alertList {
		if alert == nil {
			continue
		}
		if primary == nil {
			primary = alert
		}
		alertLevel := alerts.NormalizeAlertLevel(alert.Level)
		if alertLevel == alerts.AlertLevelCritical {
			level = alerts.AlertLevelCritical
			break
		}
		if alertLevel == alerts.AlertLevelWarning {
			level = alerts.AlertLevelWarning
		}
	}

	levelLabel := "INFO"
	priority := "default"
	severityTag := "information_source"
	if level == alerts.AlertLevelCritical {
		levelLabel = "CRITICAL"
		priority = "urgent"
		severityTag = "rotating_light"
	} else if level == alerts.AlertLevelWarning {
		levelLabel = "WARNING"
		priority = "high"
		severityTag = "warning"
	}

	titleSubject := fmt.Sprintf("%d alerts", len(alertList))
	typeTag := "grouped"
	if len(alertList) == 1 && primary != nil {
		titleSubject = primary.ResourceName
		if primary.Node != "" {
			titleSubject += " on " + primary.Node
		}
		if primary.Type != "" {
			typeTag = primary.Type
		}
	}

	webhook.Headers["Content-Type"] = "text/plain"
	webhook.Headers["Title"] = fmt.Sprintf("%s: %s", levelLabel, titleSubject)
	webhook.Headers["Priority"] = priority
	webhook.Headers["Tags"] = fmt.Sprintf("%s,pulse,%s", severityTag, typeTag)
	return webhook
}

// sendGroupedWebhook sends a grouped webhook notification
func (n *NotificationManager) sendGroupedWebhook(webhook WebhookConfig, alertList []*alerts.Alert) error {
	if len(alertList) == 0 {
		return fmt.Errorf("no alerts to send")
	}

	// Create a shallow copy of the primary alert to avoid mutating the original memory
	// when we modify the message for grouped summaries.
	originalPrimary := alertList[0]
	alertCopy := *originalPrimary
	primaryAlert := &alertCopy
	usesRenderedAlert := strings.TrimSpace(webhook.Template) != "" || (webhook.Service != "" && webhook.Service != "generic")
	if usesRenderedAlert && len(alertList) > 1 {
		otherAlerts := make([]string, 0, len(alertList)-1)
		for i := 1; i < len(alertList); i++ {
			if alertList[i] == nil {
				continue
			}
			alertLabel := alertList[i].ResourceName
			if alertList[i].Node != "" {
				alertLabel += " on " + alertList[i].Node
			}
			if alertList[i].Message != "" {
				otherAlerts = append(otherAlerts, fmt.Sprintf("• %s: %s", alertLabel, alertList[i].Message))
			} else {
				otherAlerts = append(otherAlerts, fmt.Sprintf("• %s: %.1f%%", alertLabel, alertList[i].Value))
			}
		}
		if len(otherAlerts) > 0 {
			if webhook.Service == "discord" {
				primaryAlert.Message = fmt.Sprintf("%s | %d alerts: %s", primaryAlert.Message, len(alertList), strings.Join(otherAlerts, ", "))
			} else if strings.TrimSpace(webhook.Template) != "" {
				primaryAlert.Message = fmt.Sprintf("%s\\n\\nAll %d alerts:\\n%s", primaryAlert.Message, len(alertList), strings.Join(otherAlerts, "\\n"))
			} else {
				primaryAlert.Message = fmt.Sprintf("%s\n\nAll %d alerts:\n%s", primaryAlert.Message, len(alertList), strings.Join(otherAlerts, "\n"))
			}
		}
	}

	// Prepare template data only after enriching the primary alert. Preparing it
	// first captured the original message and silently omitted grouped alerts
	// from built-in service payloads.
	customFields := convertWebhookCustomFields(webhook.CustomFields)
	data := n.prepareWebhookData(primaryAlert, customFields)
	data.AlertCount = len(alertList)
	data.Alerts = alertList
	data.AlertURL = firstNonEmpty(alertsPageURL(n.publicURL, alertList, false), data.Instance)
	data.Mention = webhook.Mention
	if webhook.Service == "ntfy" {
		webhook = withNtfyAlertHeaders(webhook, alertList)
	}

	var err error
	webhook, data, err = n.prepareWebhookDeliveryContext(webhook, data)
	if err != nil {
		return fmt.Errorf("prepare grouped webhook context: %w", err)
	}

	jsonData, err := n.renderWebhookPayloadJSON(webhook, data, webhookRenderModeGrouped, func() ([]byte, error) {
		payload := map[string]interface{}{
			"alerts":    alertList,
			"count":     len(alertList),
			"timestamp": time.Now().Unix(),
			"source":    "pulse-monitoring",
			"grouped":   true,
		}
		if link := alertsPageURL(n.publicURL, alertList, false); link != "" {
			payload["alertUrl"] = link
		}
		return json.Marshal(payload)
	})
	if err != nil {
		return fmt.Errorf("render grouped webhook payload: %w", err)
	}

	// Send using same request logic
	return n.sendWebhookRequest(webhook, jsonData, "grouped", webhookEventID(data.ID, data.Event))
}

func (n *NotificationManager) sendResolvedWebhook(webhook WebhookConfig, alertList []*alerts.Alert, resolvedAt time.Time) error {
	if len(alertList) == 0 {
		return fmt.Errorf("no alerts to send")
	}

	if !webhook.Enabled {
		return fmt.Errorf("webhook is disabled")
	}

	if resolvedAt.IsZero() {
		resolvedAt = time.Now()
	}

	// ntfy needs plain-text body + headers, not JSON
	if webhook.Service == "ntfy" {
		return n.sendResolvedWebhookNtfy(webhook, alertList, resolvedAt)
	}

	// Use the first non-nil alert for template data (most common case is single-alert recovery)
	var alert *alerts.Alert
	for _, a := range alertList {
		if a != nil {
			alert = a
			break
		}
	}
	if alert == nil {
		return fmt.Errorf("all alerts in resolved list are nil")
	}

	// Prepare template data using the same pipeline as the firing path
	customFields := convertWebhookCustomFields(webhook.CustomFields)
	data := n.prepareWebhookData(alert, customFields)

	// Carry the configured mention into the resolved payload too, matching the
	// firing/grouped paths so a configured @everyone/@channel is not dropped on
	// resolved notifications (#1118).
	data.Mention = webhook.Mention

	// Override fields for resolved context
	data.Event = "resolved"
	data.ResolvedAt = resolvedAt.Format(time.RFC3339)
	data.ResolvedAtISO = resolvedAt.Format(time.RFC3339)
	data.Duration = formatWebhookDuration(resolvedAt.Sub(alert.StartTime))
	data.Message = resolvedAlertMessage(alert)
	data.NotRecovered = resolvedAlertsIncludeNotRecovered(alertList)
	data.AlertCount = len(alertList)
	data.Alerts = alertList
	data.AlertURL = firstNonEmpty(alertsPageURL(n.publicURL, alertList, true), data.Instance)
	if len(alertList) > 1 {
		names := make([]string, 0, len(alertList))
		for _, a := range alertList {
			if a != nil {
				names = append(names, resolvedAlertListEntry(a))
			}
		}
		data.Message = fmt.Sprintf("%d alerts resolved: %s", len(alertList), strings.Join(names, "; "))
	}

	var err error
	webhook, data, err = n.prepareWebhookDeliveryContext(webhook, data)
	if err != nil {
		return fmt.Errorf("prepare resolved webhook context: %w", err)
	}

	jsonData, err := n.renderWebhookPayloadJSON(webhook, data, webhookRenderModeResolved, func() ([]byte, error) {
		payload := map[string]interface{}{
			"event":         string(eventResolved),
			"alerts":        alertList,
			"count":         len(alertList),
			"resolvedAt":    resolvedAt.Unix(),
			"resolvedAtIso": resolvedAt.Format(time.RFC3339),
			"source":        "pulse-monitoring",
		}

		if n.publicURL != "" {
			payload["dashboard"] = n.publicURL
		}

		if len(alertList) == 1 {
			payload["alertIdentifier"] = alert.ID
		}
		if link := alertsPageURL(n.publicURL, alertList, true); link != "" {
			payload["alertUrl"] = link
		}
		return json.Marshal(payload)
	})
	if err != nil {
		return fmt.Errorf("render resolved webhook payload: %w", err)
	}

	return n.sendWebhookRequest(webhook, jsonData, "resolved", webhookEventID(data.ID, data.Event))
}

// sendResolvedWebhookNtfy sends a resolved webhook formatted for ntfy (plain text + headers)
func (n *NotificationManager) sendResolvedWebhookNtfy(webhook WebhookConfig, alertList []*alerts.Alert, resolvedAt time.Time) error {
	// Re-validate webhook URL
	if err := n.ValidateWebhookURL(webhook.URL); err != nil {
		return fmt.Errorf("webhook URL validation failed: %w", err)
	}

	if !n.checkWebhookRateLimit(webhook.URL) {
		return fmt.Errorf("rate limit exceeded for webhook %s", webhook.Name)
	}

	// Build plain-text body and title. A single alert that moved rather than
	// recovered says so in both, and no batch holding such a close carries
	// the green check.
	var body strings.Builder
	title := "RESOLVED"
	tags := "white_check_mark,pulse,resolved"
	if resolvedAlertsIncludeNotRecovered(alertList) {
		tags = "arrow_right,pulse,resolved"
	}
	if len(alertList) == 1 && alertList[0] != nil {
		a := alertList[0]
		title = fmt.Sprintf("RESOLVED: %s", a.ResourceName)
		if summary := resolvedAlertNotRecoveredSummary(a); summary != "" {
			title = fmt.Sprintf("MOVED: %s", a.ResourceName)
			body.WriteString(summary)
		} else {
			fmt.Fprintf(&body, "Resolved: %s", resolvedAlertMessage(a))
		}
	} else {
		title = fmt.Sprintf("RESOLVED: %d alerts", len(alertList))
		fmt.Fprintf(&body, "%d alerts resolved at %s:\n", len(alertList), resolvedAt.Format(time.RFC822))
		for _, a := range alertList {
			if a != nil {
				fmt.Fprintf(&body, "- %s\n", resolvedAlertListEntry(a))
			}
		}
	}

	method := webhook.Method
	if method == "" {
		method = "POST"
	}

	req, err := http.NewRequest(method, webhook.URL, bytes.NewBufferString(body.String()))
	if err != nil {
		return fmt.Errorf("failed to create ntfy request: %w", err)
	}

	req.Header.Set("Content-Type", "text/plain")
	req.Header.Set("Title", title)
	req.Header.Set("Priority", "default")
	req.Header.Set("Tags", tags)
	req.Header.Set("User-Agent", "Pulse-Monitoring/2.0")

	// Apply any custom headers from webhook config
	for key, value := range webhook.Headers {
		if !strings.Contains(value, "{{") {
			req.Header.Set(key, value)
		}
	}

	resp, err := n.webhookClient.Do(req)
	if err != nil {
		err = redactWebhookTransportError(err)
		log.Error().
			Err(err).
			Str("webhook", webhook.Name).
			Msg("failed to send resolved ntfy webhook")
		return fmt.Errorf("failed to send ntfy webhook: %w", err)
	}
	defer resp.Body.Close()

	result, err := readWebhookResponse(resp)
	if err == nil {
		log.Info().
			Str("webhook", webhook.Name).
			Str("service", "ntfy").
			Str("type", "resolved").
			Int("status", resp.StatusCode).
			Int("alertCount", len(alertList)).
			Msg("resolved ntfy webhook sent successfully")
		return nil
	}

	log.Warn().
		Err(err).
		Str("webhook", webhook.Name).
		Str("service", "ntfy").
		Int("status", resp.StatusCode).
		Int64("responseBytes", result.responseBytes).
		Bool("responseLimitReached", result.responseLimitReached).
		Bool("responseIncomplete", result.responseIncomplete).
		Msg("resolved ntfy webhook response did not complete successfully")
	return fmt.Errorf("ntfy %w", err)
}

// checkWebhookRateLimit checks if a webhook can be sent based on rate limits
func (n *NotificationManager) checkWebhookRateLimit(webhookURL string) bool {
	n.webhookRateMu.Lock()
	defer n.webhookRateMu.Unlock()

	now := time.Now()
	n.cleanupWebhookRateLimitsLocked(now)
	limit, exists := n.webhookRateLimits[webhookURL]

	if !exists {
		// First time sending to this webhook
		n.webhookRateLimits[webhookURL] = &webhookRateLimit{
			lastSent:  now,
			sentCount: 1,
		}
		return true
	}

	// Check if we're still in the rate limit window
	if now.Sub(limit.lastSent) > WebhookRateLimitWindow {
		// Window expired, reset counter
		limit.lastSent = now
		limit.sentCount = 1
		return true
	}

	// Still in window, check if we've exceeded the limit
	if limit.sentCount >= WebhookRateLimitMax {
		log.Warn().
			Str("webhookURL", RedactWebhookURLSecrets(webhookURL)).
			Int("sentCount", limit.sentCount).
			Dur("window", WebhookRateLimitWindow).
			Msg("webhook rate limit exceeded, dropping request")
		return false
	}

	// Increment counter and allow
	limit.sentCount++
	return true
}

func (n *NotificationManager) cleanupWebhookRateLimitsLocked(now time.Time) {
	if now.Sub(n.webhookRateCleanup) < WebhookRateLimitWindow {
		return
	}

	cutoff := now.Add(-WebhookRateLimitWindow)
	cleaned := 0
	for webhookURL, limit := range n.webhookRateLimits {
		if limit.lastSent.Before(cutoff) {
			delete(n.webhookRateLimits, webhookURL)
			cleaned++
		}
	}
	n.webhookRateCleanup = now

	if cleaned > 0 {
		log.Debug().
			Int("cleaned", cleaned).
			Int("remaining", len(n.webhookRateLimits)).
			Msg("Cleaned up stale webhook rate limit entries")
	}
}

// sendWebhookRequest sends the actual webhook request
func normalizeWebhookRequestURL(webhook WebhookConfig) string {
	// For Telegram webhooks, strip chat_id from URL if present.
	// The chat_id belongs in the payload, not in the request URL.
	if webhook.Service == "telegram" && strings.Contains(webhook.URL, "chat_id=") {
		if u, err := url.Parse(webhook.URL); err == nil {
			q := u.Query()
			q.Del("chat_id")
			u.RawQuery = q.Encode()
			return u.String()
		}
	}
	return webhook.URL
}

func (n *NotificationManager) executeWebhookRequest(webhook WebhookConfig, payload []byte, opts webhookRequestOptions) (*webhookHTTPResult, error) {
	if opts.validateURL {
		if err := n.ValidateWebhookURL(webhook.URL); err != nil {
			return nil, fmt.Errorf("webhook URL validation failed: %w", err)
		}
	}

	method := webhook.Method
	if method == "" {
		method = "POST"
	}

	targetURL, err := securityutil.NormalizeAbsoluteHTTPURL(normalizeWebhookRequestURL(webhook))
	if err != nil {
		return nil, fmt.Errorf("failed to normalize webhook request URL: %w", err)
	}

	req, err := securityutil.NewValidatedRequestWithContext(context.Background(), method, targetURL, bytes.NewBuffer(payload))
	if err != nil {
		return nil, fmt.Errorf("failed to create webhook request: %w", err)
	}

	req.Header.Set("Content-Type", "application/json")
	req.Header.Set("User-Agent", opts.userAgent)
	if webhook.Service == "ntfy" {
		req.Header.Set("Content-Type", "text/plain")
	}

	for key, value := range webhook.Headers {
		if strings.Contains(value, "{{") {
			continue
		}
		req.Header.Set(key, value)
	}

	// Delivery integrity headers are set after custom headers so user-provided
	// header maps cannot shadow them.
	if opts.eventID != "" {
		req.Header.Set("X-Pulse-Event-ID", opts.eventID)
	}
	if secret := strings.TrimSpace(webhook.SigningSecret); secret != "" {
		timestamp := strconv.FormatInt(time.Now().Unix(), 10)
		req.Header.Set("X-Pulse-Timestamp", timestamp)
		req.Header.Set("X-Pulse-Signature", "v1="+signWebhookPayload(secret, timestamp, payload))
	}

	client := n.webhookClient
	if client == nil || (opts.timeout > 0 && opts.timeout != WebhookTimeout) {
		client = n.createSecureWebhookClient(opts.timeout)
	}

	resp, err := client.Do(req)
	if err != nil {
		return nil, fmt.Errorf("failed to send webhook: %w", redactWebhookTransportError(err))
	}
	defer resp.Body.Close()

	result, err := readWebhookResponse(resp)
	if result.responseLimitReached {
		log.Warn().
			Str("webhook", webhook.Name).
			Int64("responseBytes", result.responseBytes).
			Int("maxSize", WebhookMaxResponseSize).
			Msg("webhook response read limit reached; body withheld")
	}

	if opts.responseLogging || resp.StatusCode < 200 || resp.StatusCode >= 300 {
		log.Debug().
			Str("webhook", webhook.Name).
			Int("status", resp.StatusCode).
			Int64("responseBytes", result.responseBytes).
			Bool("responseLimitReached", result.responseLimitReached).
			Bool("responseIncomplete", result.responseIncomplete).
			Msg("webhook response")
	}
	return result, err
}

func (n *NotificationManager) sendWebhookRequest(webhook WebhookConfig, jsonData []byte, alertType string, eventID string) error {
	// Re-validate webhook URL to prevent DNS rebinding attacks
	if err := n.ValidateWebhookURL(webhook.URL); err != nil {
		log.Error().
			Err(err).
			Str("webhook", webhook.Name).
			Str("url", RedactWebhookURLSecrets(webhook.URL)).
			Msg("webhook URL validation failed at send time - possible DNS rebinding")
		return fmt.Errorf("webhook URL validation failed: %w", err)
	}

	// Check rate limit before sending
	if !n.checkWebhookRateLimit(webhook.URL) {
		log.Warn().
			Str("webhook", webhook.Name).
			Str("url", RedactWebhookURLSecrets(webhook.URL)).
			Msg("Webhook request dropped due to rate limiting")
		return fmt.Errorf("rate limit exceeded for webhook %s", webhook.Name)
	}

	result, err := n.executeWebhookRequest(webhook, jsonData, webhookRequestOptions{
		alertType:       alertType,
		eventID:         eventID,
		timeout:         WebhookTimeout,
		userAgent:       "Pulse-Monitoring/2.0",
		responseLogging: false,
		validateURL:     false,
	})
	if err != nil {
		log.Error().
			Err(err).
			Str("webhook", webhook.Name).
			Str("type", alertType).
			Msg("failed to send webhook")
		return err
	}

	if result.statusCode >= 200 && result.statusCode < 300 {
		log.Info().
			Str("webhook", webhook.Name).
			Str("service", webhook.Service).
			Str("type", alertType).
			Int("status", result.statusCode).
			Int("payloadSize", len(jsonData)).
			Msg("webhook notification sent successfully")

		if result.responseBytes > 0 {
			log.Debug().
				Str("webhook", webhook.Name).
				Int64("responseBytes", result.responseBytes).
				Bool("responseLimitReached", result.responseLimitReached).
				Msg("webhook response body withheld")
		}
		return nil
	} else {
		log.Warn().
			Str("webhook", webhook.Name).
			Str("service", webhook.Service).
			Str("type", alertType).
			Int("status", result.statusCode).
			Msg("webhook returned non-success status")
		return FailfWithClass(ClassFromHTTPStatus(result.statusCode), "webhook returned HTTP %d (response body withheld)", result.statusCode)
	}
}

func convertWebhookCustomFields(fields map[string]string) map[string]interface{} {
	if len(fields) == 0 {
		return nil
	}

	converted := make(map[string]interface{}, len(fields))
	for key, value := range fields {
		converted[key] = value
	}
	return converted
}

func isEmptyInterface(value interface{}) bool {
	switch v := value.(type) {
	case string:
		return strings.TrimSpace(v) == ""
	case fmt.Stringer:
		return strings.TrimSpace(v.String()) == ""
	case nil:
		return true
	default:
		return false
	}
}

// signWebhookPayload computes the hex HMAC-SHA256 of "timestamp.body" with
// the webhook's signing secret. Binding the timestamp into the MAC lets
// receivers reject replayed deliveries.
func signWebhookPayload(secret, timestamp string, payload []byte) string {
	mac := hmac.New(sha256.New, []byte(secret))
	mac.Write([]byte(timestamp))
	mac.Write([]byte("."))
	mac.Write(payload)
	return hex.EncodeToString(mac.Sum(nil))
}

// webhookEventID builds the legacy correlation header, stable across retries.
// It omits occurrence start, severity and grouped membership, so it is not a
// unique incident/delivery identity or a sufficient durable deduplication key.
func webhookEventID(alertID, event string) string {
	if alertID == "" {
		return ""
	}
	if event == "" {
		event = "alert"
	}
	return alertID + ":" + event
}

// prepareWebhookData prepares data for template rendering
func (n *NotificationManager) prepareWebhookData(alert *alerts.Alert, customFields map[string]interface{}) WebhookPayloadData {
	duration := time.Since(alert.StartTime)

	// Construct full Pulse URL if publicURL is configured
	// The Instance field should contain the full URL to the Pulse dashboard
	instance := ""
	if n.publicURL != "" {
		// Remove trailing slash from publicURL if present
		instance = strings.TrimRight(n.publicURL, "/")
	} else if alert.Instance != "" && (strings.HasPrefix(alert.Instance, "http://") || strings.HasPrefix(alert.Instance, "https://")) {
		// If publicURL is not set but alert.Instance contains a full URL, use it
		instance = alert.Instance
	}

	resourceType := ""
	messageKey := strings.TrimSpace(alert.Type)
	if alert.Metadata != nil {
		if rt, ok := alert.Metadata["resourceType"].(string); ok {
			resourceType = rt
		}
		if kind, ok := alert.Metadata["canonicalAlertKind"].(string); ok && strings.TrimSpace(kind) != "" {
			messageKey = strings.TrimSpace(kind)
			if alertType := strings.TrimSpace(alert.Type); alertType != "" {
				messageKey += "." + alertType
			}
		}
	}

	var metadataCopy map[string]interface{}
	if alert.Metadata != nil {
		metadataCopy = make(map[string]interface{}, len(alert.Metadata))
		for k, v := range alert.Metadata {
			metadataCopy[k] = v
		}
	}

	var ackTime string
	if alert.AckTime != nil {
		ackTime = alert.AckTime.Format(time.RFC3339)
	}

	// Round Value and Threshold to 1 decimal place for cleaner webhook payloads
	roundedValue := math.Round(alert.Value*10) / 10
	roundedThreshold := math.Round(alert.Threshold*10) / 10

	tenantID, tenantName := n.TenantIdentity()

	return WebhookPayloadData{
		ID:                 alert.ID,
		MessageKey:         messageKey,
		Level:              string(alert.Level),
		Type:               alert.Type,
		ResourceName:       alert.ResourceName,
		ResourceID:         alert.ResourceID,
		Node:               alert.Node,
		NodeDisplayName:    alertNodeDisplay(alert),
		Instance:           instance,
		AlertURL:           firstNonEmpty(alertLinkURL(n.publicURL, alert), instance),
		Message:            alert.Message,
		Value:              roundedValue,
		Threshold:          roundedThreshold,
		ValueFormatted:     formatMetricValue(alert.Type, alert.Value),
		ThresholdFormatted: formatMetricThreshold(alert.Type, alert.Threshold),
		StartTime:          alert.StartTime.Format(time.RFC3339),
		Duration:           formatWebhookDuration(duration),
		Timestamp:          time.Now().Format(time.RFC3339),
		ResourceType:       resourceType,
		Acknowledged:       alert.Acknowledged,
		AckTime:            ackTime,
		AckUser:            alert.AckUser,
		Event:              "alert",
		TenantID:           tenantID,
		TenantName:         tenantName,
		Metadata:           metadataCopy,
		CustomFields:       customFields,
		AlertCount:         1,
	}
}

func templateFuncMap() template.FuncMap {
	return template.FuncMap{
		"title": func(s string) string {
			if s == "" {
				return s
			}
			return strings.ToUpper(s[:1]) + strings.ToLower(s[1:])
		},
		"jsonString": func(v interface{}) string {
			encoded, err := json.Marshal(v)
			if err != nil {
				return ""
			}
			if len(encoded) >= 2 && encoded[0] == '"' && encoded[len(encoded)-1] == '"' {
				return string(encoded[1 : len(encoded)-1])
			}
			return string(encoded)
		},
		"upper":     strings.ToUpper,
		"lower":     strings.ToLower,
		"printf":    fmt.Sprintf,
		"urlquery":  template.URLQueryEscaper,
		"urlencode": template.URLQueryEscaper,
		"urlpath":   url.PathEscape,
		"pathescape": func(s string) string {
			return url.PathEscape(s)
		},
	}
}

// generatePayloadFromTemplateWithService renders the payload using Go templates with service-specific handling
func (n *NotificationManager) generatePayloadFromTemplateWithService(templateStr string, data WebhookPayloadData, service string) ([]byte, error) {
	tmpl, err := template.New("webhook").Funcs(templateFuncMap()).Parse(templateStr)
	if err != nil {
		return nil, failNotificationConfiguration("invalid template", err)
	}

	var buf bytes.Buffer
	if err := tmpl.Execute(&buf, data); err != nil {
		return nil, failNotificationConfiguration("template execution failed", err)
	}

	// Skip JSON validation for services that use plain text payloads
	if service == "ntfy" {
		// ntfy uses plain text, not JSON
		return buf.Bytes(), nil
	}

	// Validate that the generated payload is valid JSON for other services
	var jsonCheck interface{}
	if err := json.Unmarshal(buf.Bytes(), &jsonCheck); err != nil {
		log.Error().
			Int("payloadBytes", buf.Len()).
			Msg("generated webhook payload is invalid JSON")
		return nil, failNotificationConfiguration("template produced invalid JSON", err)
	}

	return buf.Bytes(), nil
}

// renderWebhookURL applies template rendering to webhook URLs and ensures the result is a valid URL
func renderWebhookURL(urlTemplate string, data WebhookPayloadData) (string, error) {
	trimmed := strings.TrimSpace(urlTemplate)
	if trimmed == "" {
		return "", fmt.Errorf("webhook URL cannot be empty")
	}

	if !strings.Contains(trimmed, "{{") {
		return trimmed, nil
	}

	tmpl, err := template.New("webhook_url").Funcs(templateFuncMap()).Parse(trimmed)
	if err != nil {
		return "", fmt.Errorf("invalid webhook URL template: %w", err)
	}

	var buf bytes.Buffer
	if err := tmpl.Execute(&buf, data); err != nil {
		return "", fmt.Errorf("webhook URL template execution failed: %w", err)
	}

	rendered := strings.TrimSpace(buf.String())
	if rendered == "" {
		return "", fmt.Errorf("webhook URL template produced empty URL")
	}

	parsed, err := url.Parse(rendered)
	if err != nil {
		return "", fmt.Errorf("webhook URL template produced invalid URL: %w", err)
	}

	if parsed.Scheme == "" || parsed.Host == "" {
		return "", fmt.Errorf("webhook URL template produced invalid URL: missing scheme or host")
	}

	return parsed.String(), nil
}

// formatWebhookDuration formats a duration in a human-readable way
func formatWebhookDuration(d time.Duration) string {
	if d < time.Minute {
		return fmt.Sprintf("%ds", int(d.Seconds()))
	} else if d < time.Hour {
		return fmt.Sprintf("%dm", int(d.Minutes()))
	} else if d < 24*time.Hour {
		return fmt.Sprintf("%dh %dm", int(d.Hours()), int(d.Minutes())%60)
	} else {
		days := int(d.Hours()) / 24
		hours := int(d.Hours()) % 24
		return fmt.Sprintf("%dd %dh", days, hours)
	}
}

// extractTelegramChatID extracts and validates the chat_id from a Telegram webhook URL
func extractTelegramChatID(webhookURL string) (string, error) {
	if !strings.Contains(webhookURL, "chat_id=") {
		return "", fmt.Errorf("telegram webhook URL missing chat_id parameter")
	}

	u, err := url.Parse(webhookURL)
	if err != nil {
		return "", fmt.Errorf("invalid URL format: %w", err)
	}

	chatID := u.Query().Get("chat_id")
	if chatID == "" {
		return "", fmt.Errorf("chat_id parameter is empty")
	}

	// Validate that chat_id is numeric (Telegram chat IDs are always numeric)
	// Handle negative IDs (group chats) and positive IDs (private chats)
	if strings.HasPrefix(chatID, "-") {
		if !isNumeric(chatID[1:]) {
			return "", fmt.Errorf("chat_id must be numeric, got: %s", chatID)
		}
	} else if !isNumeric(chatID) {
		return "", fmt.Errorf("chat_id must be numeric, got: %s", chatID)
	}

	return chatID, nil
}

// isNumeric checks if a string contains only digits
func isNumeric(s string) bool {
	for _, char := range s {
		if char < '0' || char > '9' {
			return false
		}
	}
	return len(s) > 0
}

// ValidateWebhookURL validates that a webhook URL is safe and properly formed
func (n *NotificationManager) ValidateWebhookURL(webhookURL string) error {
	return n.validateWebhookURL(webhookURL, RedactWebhookURLSecrets)
}

// Validation and redirect security are shared; only diagnostic URL rendering
// differs for Apprise, whose endpoint path may itself be a configuration key.
func (n *NotificationManager) validateWebhookURL(webhookURL string, diagnosticURL func(string) string) error {
	return n.validateWebhookURLWithContext(context.Background(), webhookURL, diagnosticURL)
}

func (n *NotificationManager) validateWebhookURLWithContext(ctx context.Context, webhookURL string, diagnosticURL func(string) string) error {
	if webhookURL == "" {
		return fmt.Errorf("webhook URL cannot be empty")
	}

	u, err := url.Parse(webhookURL)
	if err != nil {
		return fmt.Errorf("invalid URL format: %w", err)
	}
	if u.User != nil {
		return fmt.Errorf("webhook URL userinfo is not allowed")
	}

	// Must be HTTP or HTTPS
	if u.Scheme != "http" && u.Scheme != "https" {
		return fmt.Errorf("webhook URL must use http or https protocol")
	}

	// Get hostname for validation
	host := u.Hostname()
	if host == "" {
		return fmt.Errorf("webhook URL missing hostname")
	}
	if ip := net.ParseIP(host); ip != nil && ip.IsUnspecified() {
		return fmt.Errorf("webhook URLs pointing to unspecified addresses are not allowed")
	}

	// Block localhost and loopback addresses (SSRF protection) unless allowlisted
	if host == "localhost" || host == "127.0.0.1" || host == "::1" || strings.HasPrefix(host, "127.") {
		// Check if localhost is in the allowlist
		localhostIP := net.ParseIP("127.0.0.1")
		if !n.isIPInAllowlist(localhostIP) {
			return fmt.Errorf("webhook URLs pointing to localhost are not allowed for security reasons")
		}
		log.Debug().
			Str("host", host).
			Str("url", diagnosticURL(webhookURL)).
			Msg("localhost webhook URL allowed via allowlist")
	}

	// Block link-local addresses
	if strings.HasPrefix(host, "169.254.") || strings.HasPrefix(host, "fe80:") {
		return fmt.Errorf("webhook URLs pointing to link-local addresses are not allowed")
	}

	// Resolve hostname to IPs and check for private ranges (DNS rebinding protection)
	ips, err := n.lookupWebhookHostIPs(ctx, host)
	if err != nil {
		// DNS resolution failed - reject for security
		return fmt.Errorf("failed to resolve webhook hostname %s: %w (DNS resolution required for security)", host, err)
	}

	// Check all resolved IPs for private ranges
	for _, ip := range ips {
		if isPrivateIP(ip) {
			// Check if this private IP is in the allowlist
			if n.isIPInAllowlist(ip) {
				log.Debug().
					Str("ip", ip.String()).
					Str("url", diagnosticURL(webhookURL)).
					Msg("webhook URL resolves to private IP in allowlist")
			} else {
				return fmt.Errorf("webhook URL resolves to private IP %s - private networks are not allowed for security (configure allowlist in System Settings)", ip.String())
			}
		}
	}

	// Block common metadata service endpoints (cloud providers)
	metadataHosts := []string{
		"169.254.169.254", // AWS, Azure, GCP metadata
		"metadata.google.internal",
		"metadata.goog",
	}
	for _, metadataHost := range metadataHosts {
		if host == metadataHost {
			return fmt.Errorf("webhook URLs pointing to cloud metadata services are not allowed")
		}
	}

	// Ensure hostname is not just an IP address without proper DNS
	// This helps prevent SSRF attacks using numeric IPs to bypass filters
	if u.Scheme == "https" && isNumericIP(host) {
		log.Warn().
			Str("url", diagnosticURL(webhookURL)).
			Msg("webhook URL uses numeric IP with HTTPS - certificate validation may fail")
	}

	return nil
}

// isPrivateIP reports whether ip, or any IPv4 destination an IPv6 transition
// address (NAT64, 6to4, Teredo and friends) embeds in it, is in a private or
// special-use range. Without the embedded check 64:ff9b::a9fe:a9fe passes as a
// public address while a NAT64 gateway delivers it to 169.254.169.254.
func isPrivateIP(ip net.IP) bool {
	if isPrivateIPLiteral(ip) {
		return true
	}
	for _, embedded := range pkgsecurityutil.EmbeddedIPv4Candidates(ip) {
		if isPrivateIPLiteral(embedded) {
			return true
		}
	}
	return false
}

// isPrivateIPLiteral checks the address exactly as given against the private
// and special-use ranges.
func isPrivateIPLiteral(ip net.IP) bool {
	// Private and special-use ranges that should not be reachable via webhooks.
	privateRanges := []string{
		"10.0.0.0/8",      // RFC1918
		"172.16.0.0/12",   // RFC1918
		"192.168.0.0/16",  // RFC1918
		"127.0.0.0/8",     // Loopback
		"169.254.0.0/16",  // Link-local
		"100.64.0.0/10",   // CGNAT (RFC6598)
		"198.18.0.0/15",   // Benchmarking (RFC2544)
		"0.0.0.0/8",       // "This" network (RFC1122)
		"192.0.0.0/24",    // IETF protocol assignments (RFC6890)
		"192.0.2.0/24",    // Documentation TEST-NET-1 (RFC5737)
		"198.51.100.0/24", // Documentation TEST-NET-2 (RFC5737)
		"203.0.113.0/24",  // Documentation TEST-NET-3 (RFC5737)
		"::1/128",         // IPv6 loopback
		"::/128",          // IPv6 unspecified (RFC4291)
		"fe80::/10",       // IPv6 link-local
		"fc00::/7",        // IPv6 unique local
	}

	for _, cidr := range privateRanges {
		_, ipnet, err := net.ParseCIDR(cidr)
		if err != nil {
			continue
		}
		if ipnet.Contains(ip) {
			return true
		}
	}
	return false
}

// isNumericIP checks if a string is a numeric IP address
func isNumericIP(host string) bool {
	// Simple check: if it contains only digits, dots, and colons, it's likely an IP
	for _, char := range host {
		if !(char >= '0' && char <= '9') && char != '.' && char != ':' {
			return false
		}
	}
	return len(host) > 0 && (strings.Contains(host, ".") || strings.Contains(host, ":"))
}

// UpdateAllowedPrivateCIDRs parses and updates the list of allowed private CIDR ranges for webhooks
// ParseAllowedPrivateCIDRs validates a comma-separated CIDR string and returns
// the parsed networks. It does NOT modify any runtime state — use
// ApplyAllowedPrivateCIDRs to commit the result.
func ParseAllowedPrivateCIDRs(cidrsString string) ([]*net.IPNet, error) {
	if cidrsString == "" {
		return nil, nil
	}

	cidrs := strings.Split(cidrsString, ",")
	var parsedNets []*net.IPNet

	for _, cidr := range cidrs {
		cidr = strings.TrimSpace(cidr)
		if cidr == "" {
			continue
		}

		// Support bare IPs by adding /32 or /128
		if !strings.Contains(cidr, "/") {
			ip := net.ParseIP(cidr)
			if ip == nil {
				return nil, fmt.Errorf("invalid IP address: %s", cidr)
			}
			if ip.To4() != nil {
				cidr = cidr + "/32"
			} else {
				cidr = cidr + "/128"
			}
		}

		_, ipNet, err := net.ParseCIDR(cidr)
		if err != nil {
			return nil, fmt.Errorf("invalid CIDR range %s: %w", cidr, err)
		}
		if cidrContainsUnspecifiedIP(ipNet) {
			return nil, fmt.Errorf("CIDR range %s includes an unspecified address", cidr)
		}

		parsedNets = append(parsedNets, ipNet)
	}

	return parsedNets, nil
}

func cidrContainsUnspecifiedIP(ipNet *net.IPNet) bool {
	if ipNet == nil {
		return false
	}
	return ipNet.Contains(net.IPv4zero) || ipNet.Contains(net.IPv6zero)
}

// ApplyAllowedPrivateCIDRs atomically replaces the runtime allowlist with
// pre-validated networks (from ParseAllowedPrivateCIDRs).
func (n *NotificationManager) ApplyAllowedPrivateCIDRs(cidrsString string, nets []*net.IPNet) {
	n.allowedPrivateMu.Lock()
	defer n.allowedPrivateMu.Unlock()

	n.allowedPrivateNets = nets
	if len(nets) == 0 {
		log.Info().Msg("webhook private IP allowlist cleared - all private IPs blocked")
	} else {
		log.Info().
			Str("cidrs", cidrsString).
			Int("count", len(nets)).
			Msg("webhook private IP allowlist updated")
	}
}

// UpdateAllowedPrivateCIDRs validates and applies CIDR changes in one call.
// Kept for backward compatibility; prefer ParseAllowedPrivateCIDRs + ApplyAllowedPrivateCIDRs
// when you need persist-before-mutate semantics.
func (n *NotificationManager) UpdateAllowedPrivateCIDRs(cidrsString string) error {
	nets, err := ParseAllowedPrivateCIDRs(cidrsString)
	if err != nil {
		return err
	}
	n.ApplyAllowedPrivateCIDRs(cidrsString, nets)
	return nil
}

// lookupWebhookHostIPs bounds DNS work for both admission and pinned dialing.
func (n *NotificationManager) lookupWebhookHostIPs(ctx context.Context, host string) ([]net.IP, error) {
	if ip := net.ParseIP(host); ip != nil {
		return []net.IP{ip}, nil
	}
	resolveCtx, cancel := context.WithTimeout(ctx, webhookDNSLookupTimeout)
	defer cancel()
	var ips []net.IP
	var err error
	if n.resolveWebhookIPs != nil {
		ips, err = n.resolveWebhookIPs(resolveCtx, host)
	} else {
		ips, err = net.DefaultResolver.LookupIP(resolveCtx, "ip", host)
	}
	if err != nil {
		return nil, err
	}
	if len(ips) == 0 {
		return nil, &net.DNSError{Err: "no addresses", Name: host, IsNotFound: true}
	}
	return ips, nil
}

// isIPInAllowlist checks if an IP is in the configured allowlist
func (n *NotificationManager) isIPInAllowlist(ip net.IP) bool {
	n.allowedPrivateMu.RLock()
	defer n.allowedPrivateMu.RUnlock()

	// No allowlist means block all private IPs
	if len(n.allowedPrivateNets) == 0 {
		return false
	}

	// Check if IP is in any allowed range
	for _, ipNet := range n.allowedPrivateNets {
		if ipNet.Contains(ip) {
			return true
		}
	}

	return false
}

// addWebhookDelivery adds a webhook delivery record to the history
func (n *NotificationManager) addWebhookDelivery(delivery WebhookDelivery) {
	n.mu.Lock()
	defer n.mu.Unlock()

	// Add to history
	n.webhookHistory = append(n.webhookHistory, delivery)

	// Keep only last 100 entries
	if len(n.webhookHistory) > 100 {
		// Remove oldest entry
		n.webhookHistory = n.webhookHistory[1:]
	}
}

// GetWebhookHistory returns recent webhook delivery history
func (n *NotificationManager) GetWebhookHistory() []WebhookDelivery {
	n.mu.RLock()
	defer n.mu.RUnlock()

	// Return a copy to avoid concurrent access issues
	history := make([]WebhookDelivery, len(n.webhookHistory))
	copy(history, n.webhookHistory)
	return history
}

func buildNotificationTestAlert() *alerts.Alert {
	return &alerts.Alert{
		ID:           "test-alert",
		Type:         "cpu",
		Level:        "warning",
		ResourceID:   "test-resource",
		ResourceName: "Test Resource",
		Node:         "pve-node-01",
		Instance:     "https://192.168.1.100:8006",
		Message:      "This is a test alert from Pulse Monitoring to verify your notification settings are working correctly",
		Value:        95.5,
		Threshold:    90,
		StartTime:    time.Now().Add(-5 * time.Minute), // Show it's been active for 5 minutes
		LastSeen:     time.Now(),
		Metadata: map[string]interface{}{
			"resourceType": "vm",
		},
	}
}

// GetQueueStats returns statistics about the notification queue
func (n *NotificationManager) GetQueueStats() (map[string]int, error) {
	n.mu.RLock()
	queue := n.queue
	n.mu.RUnlock()

	if queue == nil {
		return nil, fmt.Errorf("notification queue not initialized")
	}
	return queue.GetQueueStats()
}

// GetTelemetryStats returns only aggregate delivery outcomes from the
// persistent queue. It never exposes destinations, alert content, or IDs.
func (n *NotificationManager) GetTelemetryStats(since time.Time) (TelemetryStats, error) {
	n.mu.RLock()
	queue := n.queue
	n.mu.RUnlock()

	if queue == nil {
		return TelemetryStats{}, nil
	}
	return queue.GetTelemetryStats(since)
}

// SendTestNotification sends a test notification
func (n *NotificationManager) SendTestNotification(method string) error {
	testAlert := buildNotificationTestAlert()

	switch method {
	case "email":
		log.Info().
			Bool("enabled", n.emailConfig.Enabled).
			Str("smtp", n.emailConfig.SMTPHost).
			Int("port", n.emailConfig.SMTPPort).
			Str("from", n.emailConfig.From).
			Int("toCount", len(n.emailConfig.To)).
			Msg("testing email notification")
		if !n.emailConfig.Enabled {
			return fmt.Errorf("email notifications are not enabled")
		}
		return n.sendSingleEmailWithError(testAlert, n.emailConfig)
	case "webhook":
		n.mu.RLock()
		if len(n.webhooks) == 0 {
			n.mu.RUnlock()
			return fmt.Errorf("no webhooks configured")
		}
		// Find first enabled webhook and copy it before releasing lock
		var webhookToTest *WebhookConfig
		for _, webhook := range n.webhooks {
			if webhook.Enabled {
				// Copy webhook to avoid race condition
				webhookCopy := webhook
				webhookToTest = &webhookCopy
				break
			}
		}
		n.mu.RUnlock()

		if webhookToTest == nil {
			return fmt.Errorf("no enabled webhooks found")
		}
		return n.SendTestWebhook(*webhookToTest)
	case "apprise":
		n.mu.RLock()
		appriseConfig := n.appriseConfig
		n.mu.RUnlock()

		log.Info().
			Bool("enabled", appriseConfig.Enabled).
			Str("mode", string(appriseConfig.Mode)).
			Int("targetCount", len(appriseConfig.Targets)).
			Msg("testing Apprise notification")

		if !appriseConfig.Enabled {
			return fmt.Errorf("apprise notifications are not enabled")
		}

		// Use sendGroupedApprise with a single test alert
		return n.sendGroupedApprise(appriseConfig, []*alerts.Alert{testAlert})
	default:
		return fmt.Errorf("unknown notification method: %s", method)
	}
}

// SendTestAppriseWithConfig sends a test Apprise notification using provided config
func (n *NotificationManager) SendTestAppriseWithConfig(config AppriseConfig) error {
	cfg := NormalizeAppriseConfig(config)

	log.Info().
		Bool("enabled", cfg.Enabled).
		Str("mode", string(cfg.Mode)).
		Int("targetCount", len(cfg.Targets)).
		Bool("hasServerURL", cfg.ServerURL != "").
		Msg("testing Apprise notification with provided config")

	if !cfg.Enabled {
		switch cfg.Mode {
		case AppriseModeCLI:
			return fmt.Errorf("apprise notifications are not enabled in the provided configuration: at least one target is required for CLI mode")
		case AppriseModeHTTP:
			return fmt.Errorf("apprise notifications are not enabled in the provided configuration: server URL is required for API mode")
		default:
			return fmt.Errorf("apprise notifications are not enabled in the provided configuration")
		}
	}

	return n.sendGroupedApprise(cfg, []*alerts.Alert{buildNotificationTestAlert()})
}

// SendTestWebhook sends a test notification to a specific webhook
func (n *NotificationManager) SendTestWebhook(webhook WebhookConfig) error {
	_, _, err := n.TestEnhancedWebhook(BuildEnhancedWebhookTestConfig(webhook, webhook.Service))
	return err
}

// SendTestNotificationWithConfig sends a test notification using provided config
func (n *NotificationManager) SendTestNotificationWithConfig(method string, config *EmailConfig, nodeInfo *TestNodeInfo) error {
	// Use actual node info if provided, otherwise use defaults
	nodeName := "test-node"
	instanceURL := n.publicURL
	if instanceURL == "" {
		instanceURL = "https://proxmox.local:8006"
	}
	if nodeInfo != nil {
		if nodeInfo.NodeName != "" {
			nodeName = nodeInfo.NodeName
		}
		if nodeInfo.InstanceURL != "" {
			instanceURL = nodeInfo.InstanceURL
		}
	}

	testAlert := &alerts.Alert{
		ID:           "test-alert",
		Type:         "cpu",
		Level:        "warning",
		ResourceID:   "test-email-config",
		ResourceName: "Email Configuration Test",
		Node:         nodeName,
		Instance:     instanceURL,
		Message:      "This is a test alert to verify your email notification settings are working correctly",
		Value:        85.5,
		Threshold:    80,
		StartTime:    time.Now(),
		LastSeen:     time.Now(),
		Metadata: map[string]interface{}{
			"resourceType": "test",
		},
	}

	switch method {
	case "email":
		if config == nil {
			return fmt.Errorf("email configuration is required")
		}

		log.Info().
			Bool("enabled", config.Enabled).
			Str("smtp", config.SMTPHost).
			Int("port", config.SMTPPort).
			Str("from", config.From).
			Int("toCount", len(config.To)).
			Strs("to", config.To).
			Bool("smtpEmpty", config.SMTPHost == "").
			Bool("fromEmpty", config.From == "").
			Msg("testing email notification with provided config")

		if !config.Enabled {
			return fmt.Errorf("email notifications are not enabled in the provided configuration")
		}

		if config.SMTPHost == "" || config.From == "" {
			return fmt.Errorf("email configuration is incomplete: SMTP host and from address are required")
		}

		return n.sendSingleEmailWithError(testAlert, *config)

	default:
		return fmt.Errorf("unsupported method for config-based testing: %s", method)
	}
}

func normalizeQueueType(notifType string) (string, notificationEvent) {
	if strings.HasSuffix(notifType, queueTypeSuffixResolved) {
		return strings.TrimSuffix(notifType, queueTypeSuffixResolved), eventResolved
	}
	return notifType, eventAlert
}

func resolvedTimeFromAlerts(alerts []*alerts.Alert) time.Time {
	for _, alert := range alerts {
		if alert == nil || alert.Metadata == nil {
			continue
		}
		raw, ok := alert.Metadata[metadataResolvedAt]
		if !ok {
			continue
		}
		switch ts := raw.(type) {
		case string:
			if parsed, err := time.Parse(time.RFC3339, ts); err == nil {
				return parsed
			}
		case float64:
			if ts > 0 {
				return time.Unix(int64(ts), 0)
			}
		}
	}

	return time.Now()
}

// ProcessQueuedNotification processes a notification from the persistent queue
func (n *NotificationManager) ProcessQueuedNotification(notif *QueuedNotification) error {
	baseType, event := normalizeQueueType(notif.Type)

	log.Debug().
		Str("notificationID", notif.ID).
		Str("type", baseType).
		Str("event", string(event)).
		Int("alertCount", len(notif.Alerts)).
		Msg("processing queued notification")

	var (
		err          error
		deliveredJob notificationDeliveryJob
	)
	switch baseType {
	case "email":
		var emailConfig EmailConfig
		if err = json.Unmarshal(notif.Config, &emailConfig); err != nil {
			return failNotificationConfiguration("failed to unmarshal email config", err)
		}
		currentEmailConfig := n.GetEmailConfig()
		if !n.IsEnabled() || !currentEmailConfig.Enabled {
			log.Info().
				Str("notificationID", notif.ID).
				Str("type", baseType).
				Str("event", string(event)).
				Msg("skipping queued email notification because email delivery is disabled")
			return ErrNotificationDeliverySkipped
		}
		deliveredJob = notificationDeliveryJob{
			Type:        "email",
			Event:       event,
			Alerts:      notif.Alerts,
			ResolvedAt:  resolvedTimeFromAlerts(notif.Alerts),
			EmailConfig: &emailConfig,
		}
		err = n.deliverNotificationJob(deliveredJob)

	case "webhook":
		var webhookConfig WebhookConfig
		if err = json.Unmarshal(notif.Config, &webhookConfig); err != nil {
			return failNotificationConfiguration("failed to unmarshal webhook config", err)
		}
		if !n.IsEnabled() || !n.isQueuedWebhookStillEnabled(webhookConfig) {
			log.Info().
				Str("notificationID", notif.ID).
				Str("type", baseType).
				Str("event", string(event)).
				Str("webhookID", webhookConfig.ID).
				Msg("skipping queued webhook notification because delivery is disabled")
			return ErrNotificationDeliverySkipped
		}
		deliveredJob = notificationDeliveryJob{
			Type:          "webhook",
			Event:         event,
			Alerts:        notif.Alerts,
			ResolvedAt:    resolvedTimeFromAlerts(notif.Alerts),
			WebhookConfig: &webhookConfig,
		}
		err = n.deliverNotificationJob(deliveredJob)

	case "apprise":
		var appriseConfig AppriseConfig
		if err = json.Unmarshal(notif.Config, &appriseConfig); err != nil {
			return failNotificationConfiguration("failed to unmarshal apprise config", err)
		}
		currentAppriseConfig := n.GetAppriseConfig()
		if !n.IsEnabled() || !currentAppriseConfig.Enabled {
			log.Info().
				Str("notificationID", notif.ID).
				Str("type", baseType).
				Str("event", string(event)).
				Msg("skipping queued Apprise notification because delivery is disabled")
			return ErrNotificationDeliverySkipped
		}
		deliveredJob = notificationDeliveryJob{
			Type:          "apprise",
			Event:         event,
			Alerts:        notif.Alerts,
			ResolvedAt:    resolvedTimeFromAlerts(notif.Alerts),
			AppriseConfig: &appriseConfig,
		}
		err = n.deliverNotificationJob(deliveredJob)

	default:
		return FailfWithClass(NotificationFailureConfiguration, "unknown notification type")
	}

	if err == nil {
		n.recordSuccessfulDelivery(deliveredJob, time.Now())
	}

	if err != nil {
		return fmt.Errorf("process queued %s notification %q (%s): %w", baseType, notif.ID, event, err)
	}
	return nil
}

func (n *NotificationManager) isQueuedWebhookStillEnabled(queuedWebhook WebhookConfig) bool {
	if !queuedWebhook.Enabled {
		return false
	}

	currentWebhooks := n.GetWebhooks()
	if len(currentWebhooks) == 0 {
		return false
	}

	if queuedWebhook.ID != "" {
		for _, current := range currentWebhooks {
			if current.ID == queuedWebhook.ID {
				return current.Enabled
			}
		}
		// A removed destination cannot borrow another destination's permission
		// just because it shares the URL. Retained work carries the old payload
		// and credentials; only legacy work without an ID may fall back to URL.
		return false
	}

	if queuedWebhook.URL == "" {
		return false
	}

	for _, current := range currentWebhooks {
		if current.URL == queuedWebhook.URL {
			return current.Enabled
		}
	}

	return false
}

// cleanupOldNotificationRecords periodically cleans up old entries from lastNotified map
func (n *NotificationManager) cleanupOldNotificationRecords() {
	if n.cleanupDone != nil {
		defer close(n.cleanupDone)
	}

	ticker := time.NewTicker(1 * time.Hour)
	defer ticker.Stop()

	for {
		select {
		case <-ticker.C:
			n.mu.Lock()
			now := time.Now()
			cutoff := now.Add(-24 * time.Hour)
			cleaned := 0

			for alertID, record := range n.lastNotified {
				// Remove entries older than 24 hours
				if record.lastSent.Before(cutoff) {
					delete(n.lastNotified, alertID)
					cleaned++
				}
			}

			if cleaned > 0 {
				log.Debug().
					Int("cleaned", cleaned).
					Int("remaining", len(n.lastNotified)).
					Msg("cleaned up old notification cooldown records")
			}
			n.mu.Unlock()
		case <-n.stopCleanup:
			// Stop cleanup when manager is stopped
			return
		}
	}
}

// Stop gracefully stops the notification manager
func (n *NotificationManager) Stop() {
	n.stopOnce.Do(func() {
		// Refuse new admissions and stop the firing timer, but keep its snapshots
		// until the queue is quiescent. Shutdown checkpoints the accepted batch
		// without sending it or waiting for its grouping deadline.
		n.mu.Lock()
		n.lifecycle.drain()
		n.takePendingResolvedLocked()
		if n.groupTimer != nil {
			n.groupTimer.Stop()
			n.groupTimer = nil
		}
		queue := n.queue
		n.mu.Unlock()

		// Shutdown is a restart boundary, not a delivery verdict. Quiesce the
		// queue while this manager still permits delivery and records receipts
		// durably: a queued job reaching the processor after delivery is
		// withdrawn below would be cancelled as disabled, and a send completing
		// after the queue is detached would keep its firing receipt only in
		// memory. Rows no worker has begun stay pending for the next start.
		if queue != nil {
			queue.stopProcessing()
		}
		// With workers stopped, no checkpoint can race a provider attempt. Keep
		// n.mu across the buffer-to-queue handoff: a concurrent resolution/pause
		// either removes the snapshot first or cancels the committed queue row.
		// Neither can fall between removing the buffer and persisting the row.
		n.mu.Lock()
		n.checkpointPendingFiringGroupLocked(queue)
		n.mu.Unlock()

		// Wait for admitted writes, cancellations, operator changes and direct
		// deliveries. No lease can be taken once this returns.
		n.lifecycle.close()

		n.mu.Lock()
		n.enabled = false
		cleanupDone := n.cleanupDone
		client := n.webhookClient

		// Stop cleanup goroutine. Do not nil out the channel — the cleanup
		// goroutine reads n.stopCleanup in its select loop, and nilling it
		// between close and the goroutine re-entering select causes a race
		// where the goroutine blocks on a nil channel forever.
		if n.stopCleanup != nil {
			close(n.stopCleanup)
		}

		// Nil out queue before unlocking
		n.queue = nil
		n.mu.Unlock()

		// Wait for cleanup goroutine outside the lock to avoid deadlock.
		if cleanupDone != nil {
			<-cleanupDone
		}

		// Stop the notification queue if it exists
		if queue != nil {
			if err := queue.Stop(); err != nil {
				log.Warn().Err(err).Msg("Notification queue stop returned error")
			}
		}

		// Explicitly release pooled webhook connections during shutdown.
		if client != nil {
			client.CloseIdleConnections()
		}

		log.Info().Msg("NotificationManager stopped")
	})
}