Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

45.2 KB
Raw Blame
package alerting

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"net/http"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/rcourtman/pulse-go-rewrite/internal/api/apicontext"
	"github.com/rcourtman/pulse-go-rewrite/internal/api/apihttp"
	"github.com/rcourtman/pulse-go-rewrite/internal/config"
	"github.com/rcourtman/pulse-go-rewrite/internal/monitoring"
	"github.com/rcourtman/pulse-go-rewrite/internal/notifications"
	"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
	"github.com/rcourtman/pulse-go-rewrite/internal/utils"
	internalauth "github.com/rcourtman/pulse-go-rewrite/pkg/auth"
	"github.com/rs/zerolog/log"
)

const (
	notificationTestRequestBodyLimit = 64 * 1024
	webhookTestRequestBodyLimit      = 64 * 1024
	completedQueueRetentionDays      = 7
	deadLetterQueueRetentionDays     = 30
)

// NotificationManager defines the interface for notification management operations.
type NotificationManager interface {
	GetEmailConfig() notifications.EmailConfig
	SetEmailConfig(notifications.EmailConfig)
	GetAppriseConfig() notifications.AppriseConfig
	SetAppriseConfig(notifications.AppriseConfig)
	GetWebhooks() []notifications.WebhookConfig
	ValidateWebhookURL(string) error
	AddWebhook(notifications.WebhookConfig)
	UpdateWebhook(string, notifications.WebhookConfig) error
	DeleteWebhook(string) error
	SendTestWebhook(notifications.WebhookConfig) error
	SendTestNotificationWithConfig(string, *notifications.EmailConfig, *notifications.TestNodeInfo) error
	SendTestAppriseWithConfig(notifications.AppriseConfig) error
	SendTestNotification(string) error
	GetWebhookHistory() []notifications.WebhookDelivery
	TestEnhancedWebhook(notifications.EnhancedWebhookConfig) (int, string, error)
	GetQueueStats() (map[string]int, error)
	GetTelemetryStats(time.Time) (notifications.TelemetryStats, error)
	GetDeliveryLog(time.Time, int) ([]notifications.DeliveryLogEntry, error)
	IsEnabled() bool
}

// NotificationConfigPersistence defines the interface for saving notification configuration.
type NotificationConfigPersistence interface {
	SaveEmailConfig(notifications.EmailConfig) error
	SaveAppriseConfig(notifications.AppriseConfig) error
	SaveWebhooks([]notifications.WebhookConfig) error
	IsEncryptionEnabled() bool
}

// NotificationMonitor defines the interface for monitoring operations used by notification handlers.
type NotificationMonitor interface {
	GetNotificationManager() NotificationManager
	GetConfigPersistence() NotificationConfigPersistence
}

// NotificationHandlers handles notification-related HTTP endpoints
type NotificationHandlers struct {
	stateMu        sync.RWMutex
	configMu       sync.Mutex
	mtMonitor      *monitoring.MultiTenantMonitor
	defaultMonitor NotificationMonitor
	readState      unifiedresources.ReadState
}

// NewNotificationHandlers creates new notification handlers
func NewNotificationHandlers(mtm *monitoring.MultiTenantMonitor, monitor NotificationMonitor) *NotificationHandlers {
	// If mtm is provided, try to populate defaultMonitor from "default" org if not provided.
	if monitor == nil && mtm != nil {
		if m, err := mtm.GetMonitor("default"); err == nil {
			monitor = NewNotificationMonitorWrapper(m)
		}
	}
	return &NotificationHandlers{
		mtMonitor:      mtm,
		defaultMonitor: monitor,
	}
}

// SetMonitor updates the monitor reference for notification handlers.
func (h *NotificationHandlers) SetMonitor(m NotificationMonitor) {
	h.stateMu.Lock()
	defer h.stateMu.Unlock()
	h.defaultMonitor = m
}

// SetMultiTenantMonitor updates the multi-tenant monitor reference
func (h *NotificationHandlers) SetMultiTenantMonitor(mtm *monitoring.MultiTenantMonitor) {
	var defaultMonitor NotificationMonitor
	if mtm != nil {
		if m, err := mtm.GetMonitor("default"); err == nil {
			defaultMonitor = NewNotificationMonitorWrapper(m)
		}
	}

	h.stateMu.Lock()
	defer h.stateMu.Unlock()
	h.mtMonitor = mtm
	if defaultMonitor != nil {
		h.defaultMonitor = defaultMonitor
	}
}

// SetReadState updates the ReadState reference for notification handlers.
func (h *NotificationHandlers) SetReadState(rs unifiedresources.ReadState) {
	h.stateMu.Lock()
	defer h.stateMu.Unlock()
	h.readState = rs
}

func (h *NotificationHandlers) getMonitor(ctx context.Context) NotificationMonitor {
	h.stateMu.RLock()
	mtMonitor := h.mtMonitor
	defaultMonitor := h.defaultMonitor
	h.stateMu.RUnlock()

	orgID := strings.TrimSpace(apicontext.OrgID(ctx))
	if orgID == "" || orgID == "default" {
		if mtMonitor != nil {
			if m, err := mtMonitor.GetMonitor(orgID); err == nil && m != nil {
				return NewNotificationMonitorWrapper(m)
			}
		}
		return defaultMonitor
	}

	// Never fall back to the default organization's runtime for another
	// organization: its requests would read or change the wrong tenant.
	if mtMonitor == nil {
		log.Warn().Str("org_id", orgID).Msg("Tenant monitor manager unavailable for notification request")
		return nil
	}
	m, err := mtMonitor.GetMonitor(orgID)
	if err != nil || m == nil {
		log.Warn().Err(err).Str("org_id", orgID).Msg("Failed to resolve tenant monitor for notification request")
		return nil
	}
	return NewNotificationMonitorWrapper(m)
}

// requireMonitor resolves the request organization's monitor, writing a 503
// and returning nil when it is unavailable.
func (h *NotificationHandlers) requireMonitor(w http.ResponseWriter, r *http.Request) NotificationMonitor {
	monitor := h.getMonitor(r.Context())
	if monitor == nil {
		http.Error(w, "Tenant monitor is not available", http.StatusServiceUnavailable)
		return nil
	}
	return monitor
}

// MonitorForContext resolves the tenant-scoped notification runtime used by
// handlers and extensions.
func (h *NotificationHandlers) MonitorForContext(ctx context.Context) NotificationMonitor {
	return h.getMonitor(ctx)
}

// getReadState returns the tenant-scoped ReadState for the request context.
// It resolves the correct monitor for the org and returns its ReadState.
// Falls back to the handler-level readState only for default/empty org.
// Non-default orgs fail closed (return nil) if tenant ReadState is unavailable.
func (h *NotificationHandlers) getReadState(ctx context.Context) unifiedresources.ReadState {
	h.stateMu.RLock()
	mtMonitor := h.mtMonitor
	fallback := h.readState
	h.stateMu.RUnlock()

	orgID := apicontext.OrgID(ctx)
	if mtMonitor != nil {
		if m, err := mtMonitor.GetMonitor(orgID); err == nil && m != nil {
			if rs := m.GetUnifiedReadState(); rs != nil {
				return rs
			}
		}
		// Fail closed for non-default orgs: never leak default-org metadata.
		if orgID != "" && orgID != "default" {
			return nil
		}
	}
	return fallback
}

// GetEmailConfig returns the current email configuration
func (h *NotificationHandlers) GetEmailConfig(w http.ResponseWriter, r *http.Request) {
	monitor := h.requireMonitor(w, r)
	if monitor == nil {
		return
	}
	config := monitor.GetNotificationManager().GetEmailConfig()

	// For security, don't return the password
	config.Password = ""

	w.Header().Set("Content-Type", "application/json")
	json.NewEncoder(w).Encode(config)
}

// UpdateEmailConfig updates the email configuration
func (h *NotificationHandlers) UpdateEmailConfig(w http.ResponseWriter, r *http.Request) {
	h.configMu.Lock()
	defer h.configMu.Unlock()

	// Limit request body to 32KB to prevent memory exhaustion
	r.Body = http.MaxBytesReader(w, r.Body, 32*1024)

	// Read raw body for debugging
	body, err := io.ReadAll(r.Body)
	if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	// NEVER log the body as it contains passwords
	log.Info().
		Msg("Received email config update")

	// Parse strict subset to check for presence of fields
	var presenceCheck struct {
		RateLimit       *int      `json:"rateLimit"`
		TagFilter       *[]string `json:"tagFilter"`
		TagMode         *string   `json:"tagFilterMode"`
		MinimumSeverity *string   `json:"minimumSeverity"`
	}
	if err := json.Unmarshal(body, &presenceCheck); err != nil {
		// Non-fatal, just means we can't do presence check
	}

	var config notifications.EmailConfig
	if err := json.Unmarshal(body, &config); err != nil {
		log.Error().Err(err).Msg("Failed to parse email config") // Don't log body with passwords
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	monitor := h.requireMonitor(w, r)
	if monitor == nil {
		return
	}
	manager := monitor.GetNotificationManager()
	existingConfig := manager.GetEmailConfig()

	// If password is empty, preserve the existing password
	if config.Password == "" {
		config.Password = existingConfig.Password
	}

	// If rateLimit was NOT provided (nil in presence check), preserve existing
	if presenceCheck.RateLimit == nil {
		config.RateLimit = existingConfig.RateLimit
	}
	if presenceCheck.TagFilter == nil {
		config.TagFilter = existingConfig.TagFilter
	}
	if presenceCheck.TagMode == nil {
		config.TagMode = existingConfig.TagMode
	}
	if presenceCheck.MinimumSeverity == nil {
		config.MinimumSeverity = existingConfig.MinimumSeverity
	}

	log.Info().
		Bool("enabled", config.Enabled).
		Str("smtp", config.SMTPHost).
		Str("from", config.From).
		Int("toCount", len(config.To)).
		Bool("hasPassword", config.Password != "").
		Int("rateLimit", config.RateLimit).
		Msg("Parsed email config")

	// Durable state is the publication boundary. Applying the live config first
	// can cancel queued deliveries or change routing even when the update cannot
	// survive a restart.
	if err := monitor.GetConfigPersistence().SaveEmailConfig(config); err != nil {
		log.Error().Err(err).Msg("Failed to save email configuration")
		http.Error(w, "failed to save email configuration", http.StatusInternalServerError)
		return
	}
	manager.SetEmailConfig(config)

	w.Header().Set("Content-Type", "application/json")
	json.NewEncoder(w).Encode(map[string]string{"status": "success"})
}

// appriseConfigResponse is the API projection of an AppriseConfig. The stored
// API key never leaves the server; callers only learn whether one is saved,
// matching how the email handler blanks the SMTP password.
type appriseConfigResponse struct {
	notifications.AppriseConfig
	HasAPIKey bool `json:"hasApiKey"`
}

func redactAppriseConfig(config notifications.AppriseConfig) appriseConfigResponse {
	response := appriseConfigResponse{AppriseConfig: config, HasAPIKey: config.APIKey != ""}
	response.APIKey = ""
	return response
}

// GetAppriseConfig returns the current Apprise configuration.
func (h *NotificationHandlers) GetAppriseConfig(w http.ResponseWriter, r *http.Request) {
	monitor := h.requireMonitor(w, r)
	if monitor == nil {
		return
	}
	config := monitor.GetNotificationManager().GetAppriseConfig()

	w.Header().Set("Content-Type", "application/json")
	if err := json.NewEncoder(w).Encode(redactAppriseConfig(config)); err != nil {
		log.Error().Err(err).Msg("Failed to encode Apprise configuration response")
	}
}

// UpdateAppriseConfig updates the Apprise configuration.
func (h *NotificationHandlers) UpdateAppriseConfig(w http.ResponseWriter, r *http.Request) {
	h.configMu.Lock()
	defer h.configMu.Unlock()

	// Limit request body to 64KB to prevent memory exhaustion
	r.Body = http.MaxBytesReader(w, r.Body, 64*1024)

	body, err := io.ReadAll(r.Body)
	if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	var config notifications.AppriseConfig
	if err := json.Unmarshal(body, &config); err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}
	var presenceCheck struct {
		MinimumSeverity *string `json:"minimumSeverity"`
	}
	_ = json.Unmarshal(body, &presenceCheck)
	monitor := h.requireMonitor(w, r)
	if monitor == nil {
		return
	}
	manager := monitor.GetNotificationManager()
	existingConfig := manager.GetAppriseConfig()

	// An empty API key means "keep the saved key": responses never include the
	// stored value, so the settings form cannot round-trip it.
	if config.APIKey == "" {
		config.APIKey = existingConfig.APIKey
	}
	if presenceCheck.MinimumSeverity == nil {
		config.MinimumSeverity = existingConfig.MinimumSeverity
	}

	diagnosticConfig := notifications.NormalizeAppriseConfig(config)
	log.Info().
		Bool("enabled", diagnosticConfig.Enabled).
		Str("mode", string(diagnosticConfig.Mode)).
		Int("targetCount", len(diagnosticConfig.Targets)).
		Bool("hasServerURL", diagnosticConfig.ServerURL != "").
		Bool("hasConfigKey", diagnosticConfig.ConfigKey != "").
		Bool("hasApiKey", diagnosticConfig.APIKey != "").
		Bool("skipTlsVerify", diagnosticConfig.SkipTLSVerify).
		Int("timeoutSeconds", diagnosticConfig.TimeoutSeconds).
		Msg("Parsed Apprise configuration update")

	if err := monitor.GetConfigPersistence().SaveAppriseConfig(config); err != nil {
		log.Error().Err(err).Msg("Failed to save Apprise configuration")
		http.Error(w, "failed to save Apprise configuration", http.StatusInternalServerError)
		return
	}
	manager.SetAppriseConfig(config)

	normalized := manager.GetAppriseConfig()

	w.Header().Set("Content-Type", "application/json")
	if err := json.NewEncoder(w).Encode(redactAppriseConfig(normalized)); err != nil {
		log.Error().Err(err).Msg("Failed to encode Apprise configuration response")
	}
}

// GetWebhooks returns all webhook configurations with secrets masked
func (h *NotificationHandlers) GetWebhooks(w http.ResponseWriter, r *http.Request) {
	monitor := h.requireMonitor(w, r)
	if monitor == nil {
		return
	}
	webhooks := monitor.GetNotificationManager().GetWebhooks()

	maskedWebhooks := make([]map[string]interface{}, len(webhooks))
	for i, webhook := range webhooks {
		maskedWebhooks[i] = maskedWebhookResponse(webhook)
	}

	w.Header().Set("Content-Type", "application/json")
	json.NewEncoder(w).Encode(maskedWebhooks)
}

// maskedWebhookSecret stands in for every stored header, custom field and
// signing secret value the webhook API returns. A request that sends it back
// for a saved webhook means "keep the saved value"; see
// restoreMaskedWebhookSecrets.
const maskedWebhookSecret = "***REDACTED***"

// maskedWebhookResponse is the single response shape for webhook list, create
// and update: header and custom field values and the signing secret are
// masked, so no response returns a stored secret.
func maskedWebhookResponse(webhook notifications.WebhookConfig) map[string]interface{} {
	whMap := map[string]interface{}{
		"id":      webhook.ID,
		"name":    webhook.Name,
		"url":     webhook.URL,
		"method":  webhook.Method,
		"enabled": webhook.Enabled,
		"service": webhook.Service,
	}

	// Mask headers - only show keys, not values
	if len(webhook.Headers) > 0 {
		maskedHeaders := make(map[string]string)
		for key := range webhook.Headers {
			maskedHeaders[key] = maskedWebhookSecret
		}
		whMap["headers"] = maskedHeaders
	}

	// Mask custom fields - only show keys, not values
	if len(webhook.CustomFields) > 0 {
		maskedFields := make(map[string]string)
		for key := range webhook.CustomFields {
			maskedFields[key] = maskedWebhookSecret
		}
		whMap["customFields"] = maskedFields
	}

	// Include template if present
	if webhook.Template != "" {
		whMap["template"] = webhook.Template
	}

	// Include the configured mention so the UI can render it after reload (#1118)
	if webhook.Mention != "" {
		whMap["mention"] = webhook.Mention
	}

	if len(webhook.TagFilter) > 0 {
		whMap["tagFilter"] = append([]string(nil), webhook.TagFilter...)
		whMap["tagFilterMode"] = webhook.TagMode
	}
	whMap["minimumSeverity"] = webhook.MinimumSeverity

	// Signal that a signing secret is configured without revealing it
	if webhook.SigningSecret != "" {
		whMap["signingSecret"] = maskedWebhookSecret
	}

	return whMap
}

// restoreMaskedWebhookSecrets is the shared Create/Update/Test policy. A mask
// means keep this saved value, not discard the field or store the placeholder.
// Reject missing/ambiguous saved identities before persistence or any test send.
func restoreMaskedWebhookSecrets(incoming, saved notifications.WebhookConfig) (notifications.WebhookConfig, error) {
	var err error
	incoming.Headers, err = restoreStrictMaskedWebhookValues(incoming.Headers, saved.Headers, true)
	if err != nil {
		return notifications.WebhookConfig{}, err
	}
	incoming.CustomFields, err = restoreStrictMaskedWebhookValues(incoming.CustomFields, saved.CustomFields, false)
	if err != nil {
		return notifications.WebhookConfig{}, err
	}
	if incoming.SigningSecret == maskedWebhookSecret {
		if saved.SigningSecret == "" || saved.SigningSecret == maskedWebhookSecret {
			return notifications.WebhookConfig{}, fmt.Errorf("no saved value for masked signing secret")
		}
		incoming.SigningSecret = saved.SigningSecret
	}
	return notifications.NormalizeWebhookConfig(incoming), nil
}

// CreateWebhook creates a new webhook
func (h *NotificationHandlers) CreateWebhook(w http.ResponseWriter, r *http.Request) {
	h.configMu.Lock()
	defer h.configMu.Unlock()

	// Limit request body to 64KB to prevent memory exhaustion
	r.Body = http.MaxBytesReader(w, r.Body, 64*1024)

	// Read the raw body to preserve all fields
	bodyBytes, err := io.ReadAll(r.Body)
	if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	var webhook notifications.WebhookConfig
	if err := json.Unmarshal(bodyBytes, &webhook); err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}
	webhook, err = restoreMaskedWebhookSecrets(webhook, notifications.WebhookConfig{})
	if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	monitor := h.requireMonitor(w, r)
	if monitor == nil {
		return
	}
	manager := monitor.GetNotificationManager()

	// Validate webhook URL
	if err := manager.ValidateWebhookURL(webhook.URL); err != nil {
		http.Error(w, fmt.Sprintf("Invalid webhook URL: %v", err), http.StatusBadRequest)
		return
	}

	// Generate ID if not provided
	if webhook.ID == "" {
		webhook.ID = utils.GenerateID("webhook")
	}

	// Persist the complete candidate inventory before exposing the destination
	// to live alert delivery.
	webhooks := append(manager.GetWebhooks(), webhook)
	if err := monitor.GetConfigPersistence().SaveWebhooks(webhooks); err != nil {
		log.Error().Err(err).Msg("Failed to save webhooks")
		http.Error(w, "failed to save webhook configuration", http.StatusInternalServerError)
		return
	}
	manager.AddWebhook(webhook)

	if err := utils.WriteJSONResponse(w, maskedWebhookResponse(webhook)); err != nil {
		log.Error().Err(err).Msg("Failed to write webhook creation response")
	}
}

// restoreStrictMaskedWebhookValues resolves only submitted placeholders. The editor
// sends a replacement map: explicit edits and omitted keys must not be undone
// just because another value is still masked. Header names are case-insensitive;
// custom-field names are not. Never store a placeholder without a saved value.
func restoreStrictMaskedWebhookValues(incoming, existing map[string]string, headers bool) (map[string]string, error) {
	if incoming == nil {
		return nil, nil
	}
	// Build the full HTTP identity index, even if an exact spelling exists:
	// a legacy case-varied alias must not override or conceal a conflicting value.
	savedHeaders := make(map[string]string)
	ambiguousHeaders := make(map[string]bool)
	if headers {
		for key, value := range existing {
			identity := strings.ToLower(key)
			if previous, found := savedHeaders[identity]; found && previous != value {
				ambiguousHeaders[identity] = true
			}
			savedHeaders[identity] = value
		}
	}
	resolved := make(map[string]string, len(incoming))
	incomingHeaders := make(map[string]string)
	for key, value := range incoming {
		identity := strings.ToLower(key)
		if value == maskedWebhookSecret {
			stored, found := existing[key]
			if headers {
				if ambiguousHeaders[identity] {
					return nil, fmt.Errorf("ambiguous saved value for masked header %q", key)
				}
				stored, found = savedHeaders[identity]
			}
			if !found || stored == maskedWebhookSecret {
				return nil, fmt.Errorf("no saved value for masked field %q", key)
			}
			value = stored
		}
		if headers {
			if previous, found := incomingHeaders[identity]; found && previous != value {
				return nil, fmt.Errorf("ambiguous submitted values for header %q", key)
			}
			incomingHeaders[identity] = value
		}
		resolved[key] = value
	}
	return resolved, nil
}

// UpdateWebhook updates an existing webhook.
func (h *NotificationHandlers) UpdateWebhook(w http.ResponseWriter, r *http.Request) {
	h.configMu.Lock()
	defer h.configMu.Unlock()

	// Extract webhook ID from URL path
	// Path is like /api/notifications/webhooks/{id} after routing
	path := strings.TrimPrefix(r.URL.Path, "/api/notifications/webhooks/")
	webhookID := path

	if webhookID == "" {
		http.Error(w, "Invalid URL - missing webhook ID", http.StatusBadRequest)
		return
	}

	// Limit request body to 64KB to prevent memory exhaustion
	r.Body = http.MaxBytesReader(w, r.Body, 64*1024)

	// Read the raw body to preserve all fields
	bodyBytes, err := io.ReadAll(r.Body)
	if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	var webhook notifications.WebhookConfig
	if err := json.Unmarshal(bodyBytes, &webhook); err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}
	var routingPresence struct {
		TagFilter       *[]string `json:"tagFilter"`
		TagMode         *string   `json:"tagFilterMode"`
		MinimumSeverity *string   `json:"minimumSeverity"`
	}
	_ = json.Unmarshal(bodyBytes, &routingPresence)

	monitor := h.requireMonitor(w, r)
	if monitor == nil {
		return
	}
	manager := monitor.GetNotificationManager()

	// Restore each masked value returned by GetWebhooks, not its entire map.
	existingWebhooks := manager.GetWebhooks()
	found := false
	for _, existing := range existingWebhooks {
		if existing.ID == webhookID {
			found = true
			if routingPresence.TagFilter == nil {
				webhook.TagFilter = existing.TagFilter
			}
			if routingPresence.TagMode == nil {
				webhook.TagMode = existing.TagMode
			}
			if routingPresence.MinimumSeverity == nil {
				webhook.MinimumSeverity = existing.MinimumSeverity
			}
			webhook, err = restoreMaskedWebhookSecrets(webhook, existing)
			if err != nil {
				http.Error(w, err.Error(), http.StatusBadRequest)
				return
			}
			break
		}
	}
	if !found {
		http.Error(w, fmt.Sprintf("webhook not found: %s", webhookID), http.StatusNotFound)
		return
	}

	// Validate webhook URL
	if err := manager.ValidateWebhookURL(webhook.URL); err != nil {
		http.Error(w, fmt.Sprintf("Invalid webhook URL: %v", err), http.StatusBadRequest)
		return
	}

	webhook.ID = webhookID
	candidateWebhooks := make([]notifications.WebhookConfig, len(existingWebhooks))
	copy(candidateWebhooks, existingWebhooks)
	for i := range candidateWebhooks {
		if candidateWebhooks[i].ID == webhookID {
			candidateWebhooks[i] = webhook
			break
		}
	}
	if err := monitor.GetConfigPersistence().SaveWebhooks(candidateWebhooks); err != nil {
		log.Error().Err(err).Msg("Failed to save webhooks")
		http.Error(w, "failed to save webhook configuration", http.StatusInternalServerError)
		return
	}
	if err := manager.UpdateWebhook(webhookID, webhook); err != nil {
		// The handler serializes destination writes, so this indicates an
		// out-of-band mutation. Restore the prior durable inventory rather than
		// leave restart-time state ahead of the live manager.
		if rollbackErr := monitor.GetConfigPersistence().SaveWebhooks(existingWebhooks); rollbackErr != nil {
			log.Error().Err(rollbackErr).Msg("Failed to roll back webhook configuration")
		}
		http.Error(w, "failed to publish webhook configuration", http.StatusInternalServerError)
		return
	}

	if err := utils.WriteJSONResponse(w, maskedWebhookResponse(webhook)); err != nil {
		log.Error().Err(err).Str("webhookID", webhookID).Msg("Failed to write webhook update response")
	}
}

// DeleteWebhook deletes a webhook
func (h *NotificationHandlers) DeleteWebhook(w http.ResponseWriter, r *http.Request) {
	h.configMu.Lock()
	defer h.configMu.Unlock()

	// Extract webhook ID from URL path
	// Path is like /api/notifications/webhooks/{id} after routing
	path := strings.TrimPrefix(r.URL.Path, "/api/notifications/webhooks/")
	webhookID := path

	if webhookID == "" {
		http.Error(w, "Invalid URL - missing webhook ID", http.StatusBadRequest)
		return
	}

	monitor := h.requireMonitor(w, r)
	if monitor == nil {
		return
	}
	manager := monitor.GetNotificationManager()
	existingWebhooks := manager.GetWebhooks()
	candidateWebhooks := make([]notifications.WebhookConfig, 0, len(existingWebhooks))
	found := false
	for _, webhook := range existingWebhooks {
		if webhook.ID == webhookID {
			found = true
			continue
		}
		candidateWebhooks = append(candidateWebhooks, webhook)
	}
	if !found {
		http.Error(w, fmt.Sprintf("webhook not found: %s", webhookID), http.StatusNotFound)
		return
	}

	if err := monitor.GetConfigPersistence().SaveWebhooks(candidateWebhooks); err != nil {
		log.Error().Err(err).Msg("Failed to save webhooks")
		http.Error(w, "failed to save webhook configuration", http.StatusInternalServerError)
		return
	}
	if err := manager.DeleteWebhook(webhookID); err != nil {
		if rollbackErr := monitor.GetConfigPersistence().SaveWebhooks(existingWebhooks); rollbackErr != nil {
			log.Error().Err(rollbackErr).Msg("Failed to roll back webhook configuration")
		}
		http.Error(w, "failed to publish webhook configuration", http.StatusInternalServerError)
		return
	}

	// The destination editor accepts a mutation only after success is explicit.
	// Keep status for existing clients; acknowledge only after durable/live removal.
	if err := utils.WriteJSONResponse(w, map[string]interface{}{"success": true, "status": "success"}); err != nil {
		log.Error().Err(err).Str("webhookID", webhookID).Msg("Failed to write webhook deletion response")
	}
}

// classifyNotificationError maps raw Go errors to user-friendly summaries.
// Returns a user-facing message and the original error string as detail.
func classifyNotificationError(err error) (summary, detail string) {
	raw := err.Error()
	detail = raw
	lower := strings.ToLower(raw)

	switch {
	case strings.Contains(lower, "connection refused"):
		summary = "Could not connect to the server — check host, port, and firewall settings"
	case strings.Contains(lower, "no such host"):
		summary = "Server hostname not found — check the server address"
	case strings.Contains(lower, "i/o timeout") || strings.Contains(lower, "deadline exceeded"):
		summary = "Connection timed out — the server may be unreachable or the port may be blocked"
	case strings.Contains(lower, "x509:") || strings.Contains(lower, "certificate"):
		summary = "TLS certificate error — check certificate settings or try enabling 'Skip TLS Verify'"
	case strings.Contains(lower, "535") || strings.Contains(lower, "authentication"):
		summary = "Authentication failed — check username and password"
	case strings.Contains(lower, "executable file not found"):
		summary = "Required program not found — ensure it is installed on the server"
	case strings.Contains(lower, "permission denied"):
		summary = "Permission denied — the server process lacks access to the required resource"
	case strings.Contains(lower, "eof"):
		summary = "The server closed the connection unexpectedly — check if TLS/StartTLS settings are correct"
	default:
		summary = raw
		detail = ""
	}
	return summary, detail
}

// writeTestNotificationError writes a JSON error response for test notification failures.
func writeTestNotificationError(w http.ResponseWriter, err error, statusCode int) {
	summary, detail := classifyNotificationError(err)
	resp := map[string]string{"error": summary}
	if detail != "" {
		resp["detail"] = detail
	}
	w.Header().Set("Content-Type", "application/json")
	w.WriteHeader(statusCode)
	json.NewEncoder(w).Encode(resp)
}

// TestNotification sends a test notification
func (h *NotificationHandlers) TestNotification(w http.ResponseWriter, r *http.Request) {
	r.Body = http.MaxBytesReader(w, r.Body, notificationTestRequestBodyLimit)

	// Read body for debugging
	body, err := io.ReadAll(r.Body)
	if err != nil {
		var maxBytesErr *http.MaxBytesError
		if errors.As(err, &maxBytesErr) {
			http.Error(w, "Request body too large", http.StatusRequestEntityTooLarge)
			return
		}
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	// NEVER log the body as it contains passwords
	log.Info().
		Msg("Test notification request received")

	var req struct {
		Method    string          `json:"method"`              // "email", "webhook", or "apprise"
		Type      string          `json:"type"`                // Alternative field name used by frontend
		Config    json.RawMessage `json:"config,omitempty"`    // Optional config for testing (email or apprise)
		WebhookID string          `json:"webhookId,omitempty"` // For webhook testing
	}

	if err := json.Unmarshal(body, &req); err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	// Support both "method" and "type" field names
	if req.Method == "" && req.Type != "" {
		req.Method = req.Type
	}

	// Get actual node info from ReadState (unified resources registry)
	var nodeInfo *notifications.TestNodeInfo

	if rs := h.getReadState(r.Context()); rs != nil {
		if nodes := rs.Nodes(); len(nodes) > 0 {
			nodeInfo = &notifications.TestNodeInfo{
				NodeName:    nodes[0].Name(),
				InstanceURL: nodes[0].Instance(),
			}
		}
	}

	monitor := h.requireMonitor(w, r)
	if monitor == nil {
		return
	}

	// Handle webhook testing
	if req.Method == "webhook" && req.WebhookID != "" {
		log.Info().
			Str("webhookId", req.WebhookID).
			Msg("Testing specific webhook")

		// Get the webhook by ID and test it
		webhooks := monitor.GetNotificationManager().GetWebhooks()
		var foundWebhook *notifications.WebhookConfig
		for _, wh := range webhooks {
			if wh.ID == req.WebhookID {
				foundWebhook = &wh
				break
			}
		}

		if foundWebhook == nil {
			http.Error(w, "Webhook not found", http.StatusNotFound)
			return
		}

		// Send test webhook
		if err := monitor.GetNotificationManager().SendTestWebhook(*foundWebhook); err != nil {
			writeTestNotificationError(w, err, http.StatusBadRequest)
			return
		}
	} else if req.Method == "email" && len(req.Config) > 0 {
		var emailConfig notifications.EmailConfig
		if err := json.Unmarshal(req.Config, &emailConfig); err != nil {
			http.Error(w, fmt.Sprintf("Invalid email config: %v", err), http.StatusBadRequest)
			return
		}

		// If password is empty, use the saved password
		if emailConfig.Password == "" {
			savedConfig := monitor.GetNotificationManager().GetEmailConfig()
			emailConfig.Password = savedConfig.Password
		}

		log.Info().
			Bool("enabled", emailConfig.Enabled).
			Str("smtp", emailConfig.SMTPHost).
			Str("from", emailConfig.From).
			Int("toCount", len(emailConfig.To)).
			Strs("to", emailConfig.To).
			Bool("hasPassword", emailConfig.Password != "").
			Msg("Testing email with provided config")

		if err := monitor.GetNotificationManager().SendTestNotificationWithConfig(req.Method, &emailConfig, nodeInfo); err != nil {
			writeTestNotificationError(w, err, http.StatusBadRequest)
			return
		}
	} else if req.Method == "apprise" && len(req.Config) > 0 {
		var appriseConfig notifications.AppriseConfig
		if err := json.Unmarshal(req.Config, &appriseConfig); err != nil {
			http.Error(w, fmt.Sprintf("Invalid Apprise config: %v", err), http.StatusBadRequest)
			return
		}

		// If the API key is empty, use the saved key
		if appriseConfig.APIKey == "" {
			savedConfig := monitor.GetNotificationManager().GetAppriseConfig()
			appriseConfig.APIKey = savedConfig.APIKey
		}

		if err := monitor.GetNotificationManager().SendTestAppriseWithConfig(appriseConfig); err != nil {
			writeTestNotificationError(w, err, http.StatusBadRequest)
			return
		}
	} else {
		// Use saved config
		if err := monitor.GetNotificationManager().SendTestNotification(req.Method); err != nil {
			writeTestNotificationError(w, err, http.StatusBadRequest)
			return
		}
	}

	// A test send bypasses the activation gate real alerts honor, so a bare
	// success here is exactly how installs end up believing delivery works
	// while every real alert is suppressed. Say so in the result itself.
	response := map[string]interface{}{
		"status":  "success",
		"message": "Test notification sent",
	}
	if !monitor.GetNotificationManager().IsEnabled() {
		response["deliveryPaused"] = true
		response["message"] = "Test notification sent, but alert delivery is paused: real alerts are not being sent"
	}
	w.Header().Set("Content-Type", "application/json")
	json.NewEncoder(w).Encode(response)
}

// GetWebhookTemplates returns available webhook templates
func (h *NotificationHandlers) GetWebhookTemplates(w http.ResponseWriter, r *http.Request) {
	templates := notifications.GetWebhookTemplates()

	w.Header().Set("Content-Type", "application/json")
	json.NewEncoder(w).Encode(templates)
}

// GetWebhookHistory returns recent webhook delivery history with URLs redacted
func (h *NotificationHandlers) GetWebhookHistory(w http.ResponseWriter, r *http.Request) {
	monitor := h.requireMonitor(w, r)
	if monitor == nil {
		return
	}
	history := monitor.GetNotificationManager().GetWebhookHistory()

	// Redact secrets from URLs in history
	for i := range history {
		history[i].WebhookURL = redactSecretsFromURL(history[i].WebhookURL)
		// Note: ResponseBody is not stored in WebhookDelivery struct
		// Error messages are already limited in length
	}

	w.Header().Set("Content-Type", "application/json")
	json.NewEncoder(w).Encode(history)
}

// redactSecretsFromURL is retained as the API package boundary for delivery
// history while the canonical redaction policy lives with webhook execution.
func redactSecretsFromURL(urlStr string) string {
	return notifications.RedactWebhookURLSecrets(urlStr)
}

// GetEmailProviders returns available email providers
func (h *NotificationHandlers) GetEmailProviders(w http.ResponseWriter, r *http.Request) {
	providers := notifications.GetEmailProviders()

	w.Header().Set("Content-Type", "application/json")
	json.NewEncoder(w).Encode(providers)
}

// TestWebhook tests a webhook configuration
func (h *NotificationHandlers) TestWebhook(w http.ResponseWriter, r *http.Request) {
	// First try to decode as basic webhook config
	var basicWebhook notifications.WebhookConfig

	r.Body = http.MaxBytesReader(w, r.Body, webhookTestRequestBodyLimit)
	bodyBytes, err := io.ReadAll(r.Body)
	if err != nil {
		var maxBytesErr *http.MaxBytesError
		if errors.As(err, &maxBytesErr) {
			http.Error(w, "Request body too large", http.StatusRequestEntityTooLarge)
			return
		}
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	if err := json.Unmarshal(bodyBytes, &basicWebhook); err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}
	// Testing an edit of a saved webhook sends the form, whose untouched
	// secrets are the masked values from GetWebhooks. Test with the saved
	// values, as the saved webhook would send; this route requires the same
	// settings:write scope as UpdateWebhook, which keeps them the same way.
	monitor := h.requireMonitor(w, r)
	if monitor == nil {
		return
	}
	saved := notifications.WebhookConfig{}
	if basicWebhook.ID != "" {
		for _, candidate := range monitor.GetNotificationManager().GetWebhooks() {
			if candidate.ID == basicWebhook.ID {
				saved = candidate
				break
			}
		}
	}
	basicWebhook, err = restoreMaskedWebhookSecrets(basicWebhook, saved)
	if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	// Try to extract service from body if present
	var serviceCheck struct {
		Service string `json:"service"`
	}
	if err := json.Unmarshal(bodyBytes, &serviceCheck); err == nil && serviceCheck.Service != "" {
		basicWebhook.Service = serviceCheck.Service
	}

	webhook := notifications.BuildEnhancedWebhookTestConfig(basicWebhook, serviceCheck.Service)

	log.Info().
		Str("service", webhook.Service).
		Str("url", notifications.RedactWebhookURLSecrets(webhook.URL)).
		Str("name", webhook.Name).
		Msg("Testing webhook")

	// Test the webhook
	status, response, err := monitor.GetNotificationManager().TestEnhancedWebhook(webhook)

	result := map[string]interface{}{
		"status":   status,
		"response": response,
	}

	if err != nil {
		summary, detail := classifyNotificationError(err)
		result["error"] = summary
		if detail != "" {
			result["detail"] = detail
		}
		w.WriteHeader(http.StatusBadRequest)
	} else if status < 200 || status >= 300 {
		// HTTP error from webhook endpoint
		result["error"] = fmt.Sprintf("Webhook returned HTTP %d: %s", status, response)
		result["success"] = false
		w.WriteHeader(http.StatusBadRequest)
	} else {
		result["success"] = true
		// Same honesty as TestNotification: a webhook test bypasses the
		// activation gate, so its success must not imply live alerts flow.
		if !monitor.GetNotificationManager().IsEnabled() {
			result["deliveryPaused"] = true
		}
	}

	w.Header().Set("Content-Type", "application/json")
	json.NewEncoder(w).Encode(result)
}

// GetNotificationHealth returns health status of notification system
func (h *NotificationHandlers) GetNotificationHealth(w http.ResponseWriter, r *http.Request) {
	monitor := h.requireMonitor(w, r)
	if monitor == nil {
		return
	}
	manager := monitor.GetNotificationManager()

	failureClasses := notifications.NotificationFailureClassCounts{}.AsMap()
	failureClassesAvailable := false
	if deliveryStats, statsErr := manager.GetTelemetryStats(
		time.Now().Add(-completedQueueRetentionDays * 24 * time.Hour),
	); statsErr != nil {
		log.Warn().Err(statsErr).Msg("Failed to get notification failure classes for health check")
	} else {
		failureClasses = deliveryStats.FailureClasses.AsMap()
		failureClassesAvailable = true
	}

	queueStats := make(map[string]interface{})
	stats, err := manager.GetQueueStats()
	if err != nil {
		log.Warn().Err(err).Msg("Failed to get queue stats for health check")
		queueStats = map[string]interface{}{
			"healthy":                         false,
			"status":                          "unavailable",
			"attention_required":              0,
			"reason_codes":                    []string{"queue_stats_unavailable"},
			"completed_retention_days":        completedQueueRetentionDays,
			"dead_letter_retention_days":      deadLetterQueueRetentionDays,
			"counts_are_retention_bounded":    true,
			"retry_attempts_affect_health":    false,
			"terminal_failures_affect_health": true,
			"failure_classes_7d":              failureClasses,
			"failure_classes_available":       failureClassesAvailable,
			"failure_class_window_days":       completedQueueRetentionDays,
		}
	} else {
		healthy, attentionRequired, reasonCodes := classifyNotificationQueueHealth(stats)
		status := "healthy"
		if !healthy {
			status = "degraded"
		}
		queueStats = map[string]interface{}{
			"pending":                         stats["pending"],
			"sending":                         stats["sending"],
			"sent":                            stats["sent"],
			"failed":                          stats["failed"],
			"dlq":                             stats["dlq"],
			"healthy":                         healthy,
			"status":                          status,
			"attention_required":              attentionRequired,
			"reason_codes":                    reasonCodes,
			"completed_retention_days":        completedQueueRetentionDays,
			"dead_letter_retention_days":      deadLetterQueueRetentionDays,
			"counts_are_retention_bounded":    true,
			"retry_attempts_affect_health":    false,
			"terminal_failures_affect_health": true,
			"failure_classes_7d":              failureClasses,
			"failure_classes_available":       failureClassesAvailable,
			"failure_class_window_days":       completedQueueRetentionDays,
		}
	}

	// Get config status
	emailCfg := manager.GetEmailConfig()
	webhooks := manager.GetWebhooks()

	health := map[string]interface{}{
		"queue": queueStats,
		"email": map[string]interface{}{
			"enabled":    emailCfg.Enabled,
			"configured": emailCfg.SMTPHost != "",
		},
		"webhooks": map[string]interface{}{
			"total":   len(webhooks),
			"enabled": countEnabledWebhooks(webhooks),
		},
		"encryption": map[string]interface{}{
			"enabled": monitor.GetConfigPersistence().IsEncryptionEnabled(),
		},
		"overall_healthy": queueStats["healthy"] == true,
	}

	w.Header().Set("Content-Type", "application/json")
	json.NewEncoder(w).Encode(health)
}

// GetDeliveryLog returns recent recorded delivery attempts, newest first.
// This is the per-attempt evidence behind the aggregate health verdict: what
// fired, which destination it went to, and what happened. Entries share the
// queue's retention windows, so the payload names the window rather than
// presenting itself as lifetime history.
func (h *NotificationHandlers) GetDeliveryLog(w http.ResponseWriter, r *http.Request) {
	monitor := h.requireMonitor(w, r)
	if monitor == nil {
		return
	}
	manager := monitor.GetNotificationManager()

	limit := 0
	if raw := strings.TrimSpace(r.URL.Query().Get("limit")); raw != "" {
		parsed, err := strconv.Atoi(raw)
		if err != nil || parsed <= 0 {
			http.Error(w, "limit must be a positive integer", http.StatusBadRequest)
			return
		}
		limit = parsed
	}

	entries, err := manager.GetDeliveryLog(
		time.Now().Add(-deadLetterQueueRetentionDays*24*time.Hour), limit,
	)
	if err != nil {
		log.Warn().Err(err).Msg("Failed to read notification delivery log")
		http.Error(w, "Delivery log unavailable", http.StatusServiceUnavailable)
		return
	}

	// Webhook failure text can embed the destination URL, and destination URLs
	// can embed credentials.
	for i := range entries {
		if entries[i].ErrorMessage != "" {
			entries[i].ErrorMessage = notifications.RedactWebhookDiagnosticSecrets(entries[i].ErrorMessage)
		}
	}

	w.Header().Set("Content-Type", "application/json")
	json.NewEncoder(w).Encode(map[string]interface{}{
		"entries":                       entries,
		"window_days":                   deadLetterQueueRetentionDays,
		"completed_retention_days":      completedQueueRetentionDays,
		"dead_letter_retention_days":    deadLetterQueueRetentionDays,
		"entries_are_retention_bounded": true,
	})
}

// classifyNotificationQueueHealth delegates to the canonical rule in
// internal/notifications so this endpoint and the monitoring loop that raises
// the notification-delivery system alert cannot drift apart.
func classifyNotificationQueueHealth(stats map[string]int) (bool, int, []string) {
	health := notifications.ClassifyQueueHealth(stats)
	return health.Healthy, health.AttentionRequired, health.ReasonCodes
}

func countEnabledWebhooks(webhooks []notifications.WebhookConfig) int {
	count := 0
	for _, wh := range webhooks {
		if wh.Enabled {
			count++
		}
	}
	return count
}

// HandleNotifications routes notification requests to appropriate handlers
func (h *NotificationHandlers) HandleNotifications(w http.ResponseWriter, r *http.Request) {
	path := strings.TrimPrefix(r.URL.Path, "/api/notifications")

	requireAnyScope := func(required string, scopes ...string) bool {
		record := internalauth.GetAPIToken(r.Context())
		if record == nil {
			return true
		}
		for _, scope := range scopes {
			if scope != "" && record.HasScope(scope) {
				return true
			}
		}
		apihttp.RespondMissingScope(w, required)
		return false
	}

	switch {
	case path == "/email" && r.Method == http.MethodGet:
		if !requireAnyScope(config.ScopeSettingsRead, config.ScopeSettingsRead, config.ScopeSettingsWrite) {
			return
		}
		h.GetEmailConfig(w, r)
	case path == "/email" && r.Method == http.MethodPut:
		if !requireAnyScope(config.ScopeSettingsWrite, config.ScopeSettingsWrite) {
			return
		}
		h.UpdateEmailConfig(w, r)
	case path == "/apprise" && r.Method == http.MethodGet:
		if !requireAnyScope(config.ScopeSettingsRead, config.ScopeSettingsRead, config.ScopeSettingsWrite) {
			return
		}
		h.GetAppriseConfig(w, r)
	case path == "/apprise" && r.Method == http.MethodPut:
		if !requireAnyScope(config.ScopeSettingsWrite, config.ScopeSettingsWrite) {
			return
		}
		h.UpdateAppriseConfig(w, r)
	case path == "/webhooks" && r.Method == http.MethodGet:
		if !requireAnyScope(config.ScopeSettingsRead, config.ScopeSettingsRead, config.ScopeSettingsWrite) {
			return
		}
		h.GetWebhooks(w, r)
	case path == "/webhooks" && r.Method == http.MethodPost:
		if !requireAnyScope(config.ScopeSettingsWrite, config.ScopeSettingsWrite) {
			return
		}
		h.CreateWebhook(w, r)
	case path == "/webhooks/test" && r.Method == http.MethodPost:
		if !requireAnyScope(config.ScopeSettingsWrite, config.ScopeSettingsWrite) {
			return
		}
		h.TestWebhook(w, r)
	case strings.HasPrefix(path, "/webhooks/") && r.Method == http.MethodPut:
		if !requireAnyScope(config.ScopeSettingsWrite, config.ScopeSettingsWrite) {
			return
		}
		h.UpdateWebhook(w, r)
	case strings.HasPrefix(path, "/webhooks/") && r.Method == http.MethodDelete:
		if !requireAnyScope(config.ScopeSettingsWrite, config.ScopeSettingsWrite) {
			return
		}
		h.DeleteWebhook(w, r)
	case path == "/webhook-templates" && r.Method == http.MethodGet:
		if !requireAnyScope(config.ScopeSettingsRead, config.ScopeSettingsRead, config.ScopeSettingsWrite) {
			return
		}
		h.GetWebhookTemplates(w, r)
	case path == "/webhook-history" && r.Method == http.MethodGet:
		if !requireAnyScope(config.ScopeSettingsRead, config.ScopeSettingsRead, config.ScopeSettingsWrite) {
			return
		}
		h.GetWebhookHistory(w, r)
	case path == "/email-providers" && r.Method == http.MethodGet:
		if !requireAnyScope(config.ScopeSettingsRead, config.ScopeSettingsRead, config.ScopeSettingsWrite) {
			return
		}
		h.GetEmailProviders(w, r)
	case path == "/test" && r.Method == http.MethodPost:
		if !requireAnyScope(config.ScopeSettingsWrite, config.ScopeSettingsWrite) {
			return
		}
		h.TestNotification(w, r)
	case path == "/health" && r.Method == http.MethodGet:
		if !requireAnyScope(config.ScopeSettingsRead, config.ScopeSettingsRead, config.ScopeSettingsWrite) {
			return
		}
		h.GetNotificationHealth(w, r)
	case path == "/delivery-log" && r.Method == http.MethodGet:
		if !requireAnyScope(config.ScopeSettingsRead, config.ScopeSettingsRead, config.ScopeSettingsWrite) {
			return
		}
		h.GetDeliveryLog(w, r)
	default:
		http.Error(w, "Not found", http.StatusNotFound)
	}
}