Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

53.9 KB
Raw Blame
package alerting

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"net/http"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/rcourtman/pulse-go-rewrite/internal/ai/memory"
	"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
	"github.com/rcourtman/pulse-go-rewrite/internal/alerts/eventlog"
	"github.com/rcourtman/pulse-go-rewrite/internal/api/apicontext"
	"github.com/rcourtman/pulse-go-rewrite/internal/api/apihttp"
	"github.com/rcourtman/pulse-go-rewrite/internal/config"
	"github.com/rcourtman/pulse-go-rewrite/internal/mock"
	"github.com/rcourtman/pulse-go-rewrite/internal/models"
	"github.com/rcourtman/pulse-go-rewrite/internal/monitoring"
	"github.com/rcourtman/pulse-go-rewrite/internal/notifications"
	"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
	"github.com/rcourtman/pulse-go-rewrite/internal/utils"
	"github.com/rcourtman/pulse-go-rewrite/internal/websocket"
	"github.com/rs/zerolog/log"
)

// AlertManager defines the interface for alert management operations.
type AlertManager interface {
	GetConfig() alerts.AlertConfig
	UpdateConfig(alerts.AlertConfig)
	ApplyConfigUpdate(update []byte) (alerts.AlertConfig, error)
	GetActiveAlerts() []alerts.Alert
	DiagnoseAlertDelivery(alertIdentifier string) (alerts.AlertDeliveryDiagnosis, bool)
	DiagnoseActiveAlertDeliveries() []alerts.AlertDeliveryDiagnosis
	AlertEvents(filter eventlog.Filter) ([]eventlog.Event, error)
	NotifyExistingAlert(id string)
	ClearAlertHistory() error
	UnacknowledgeAlert(id string) error
	AcknowledgeAlert(id, user string) error
	SnoozeAlert(id, user string, until time.Time) error
	UnsnoozeAlert(id, user string) error
	ClearAlert(id string) bool
	GetAlertHistory(limit int) []alerts.Alert
	GetAlertHistorySince(since time.Time, limit int) []alerts.Alert
}

// ConfigPersistence defines the interface for saving configuration.
type ConfigPersistence interface {
	SaveAlertConfig(alerts.AlertConfig) error
}

type alertIntentPolicyManager interface {
	GetIntentPolicies() alerts.AlertIntentPolicyDocument
	LoadIntentPolicies(alerts.AlertIntentPolicyDocument) error
	UpdateIntentPolicies(alerts.AlertIntentPolicyDocument) (alerts.AlertIntentPolicyDocument, error)
	PreviewIntentPolicy(alerts.AlertIntentPolicyPreviewRequest) (alerts.AlertIntentPolicyPreview, error)
}

type alertIntentPolicyPersistence interface {
	SaveAlertIntentPolicies(alerts.AlertIntentPolicyDocument) error
}

// AlertMonitor defines the interface for monitoring operations used by alert handlers.
type AlertMonitor interface {
	GetAlertManager() AlertManager
	GetConfigPersistence() ConfigPersistence
	GetIncidentStore() *memory.IncidentStore
	GetNotificationManager() *notifications.NotificationManager
	DeadManConfig() notifications.DeadManConfig
	UpdateDeadManConfig(notifications.DeadManConfig) error
	DeadManStatus() monitoring.DeadManStatus
	SyncAlertState()
	BuildFrontendState() models.StateFrontend
}

// AlertHandlers handles alert-related HTTP endpoints
type AlertHandlers struct {
	stateMu        sync.RWMutex
	intentPolicyMu sync.Mutex
	configSaveMu   sync.Mutex
	mtMonitor      *monitoring.MultiTenantMonitor
	defaultMonitor AlertMonitor
	wsHub          *websocket.Hub
}

// NewAlertHandlers creates new alert handlers
func NewAlertHandlers(mtm *monitoring.MultiTenantMonitor, monitor AlertMonitor, wsHub *websocket.Hub) *AlertHandlers {
	// If mtm is provided, try to populate defaultMonitor from "default" org if not provided.
	if monitor == nil && mtm != nil {
		if m, err := mtm.GetMonitor("default"); err == nil {
			monitor = NewAlertMonitorWrapper(m)
		}
	}
	return &AlertHandlers{
		mtMonitor:      mtm,
		defaultMonitor: monitor,
		wsHub:          wsHub,
	}
}

// SetMultiTenantMonitor updates the multi-tenant monitor reference
func (h *AlertHandlers) SetMultiTenantMonitor(mtm *monitoring.MultiTenantMonitor) {
	var defaultMonitor AlertMonitor
	if mtm != nil {
		if m, err := mtm.GetMonitor("default"); err == nil {
			defaultMonitor = NewAlertMonitorWrapper(m)
		}
	}

	h.stateMu.Lock()
	defer h.stateMu.Unlock()
	h.mtMonitor = mtm
	if defaultMonitor != nil {
		h.defaultMonitor = defaultMonitor
	}
}

// SetMonitor updates the monitor reference for alert handlers.
func (h *AlertHandlers) SetMonitor(m AlertMonitor) {
	h.stateMu.Lock()
	defer h.stateMu.Unlock()
	h.defaultMonitor = m
}

func (h *AlertHandlers) getMonitor(ctx context.Context) AlertMonitor {
	h.stateMu.RLock()
	mtMonitor := h.mtMonitor
	defaultMonitor := h.defaultMonitor
	h.stateMu.RUnlock()

	orgID := strings.TrimSpace(apicontext.OrgID(ctx))
	if orgID == "" || orgID == "default" {
		if mtMonitor != nil {
			if m, err := mtMonitor.GetMonitor(orgID); err == nil && m != nil {
				return NewAlertMonitorWrapper(m)
			}
		}
		return defaultMonitor
	}

	// Never fall back to the default organization's runtime for another
	// organization: its requests would read or change the wrong tenant.
	if mtMonitor == nil {
		log.Warn().Str("org_id", orgID).Msg("Tenant monitor manager unavailable for alert request")
		return nil
	}
	m, err := mtMonitor.GetMonitor(orgID)
	if err != nil || m == nil {
		log.Warn().Err(err).Str("org_id", orgID).Msg("Failed to resolve tenant monitor for alert request")
		return nil
	}
	return NewAlertMonitorWrapper(m)
}

// requireMonitor resolves the request organization's monitor, writing a 503
// and returning nil when it is unavailable.
func (h *AlertHandlers) requireMonitor(w http.ResponseWriter, r *http.Request) AlertMonitor {
	monitor := h.getMonitor(r.Context())
	if monitor == nil {
		http.Error(w, "Tenant monitor is not available", http.StatusServiceUnavailable)
		return nil
	}
	return monitor
}

// MonitorForContext resolves the tenant-scoped alert runtime used by handlers.
// It is the supported extension boundary for integrations that need the same
// tenant selection semantics as HTTP alert routes.
func (h *AlertHandlers) MonitorForContext(ctx context.Context) AlertMonitor {
	return h.getMonitor(ctx)
}

func (h *AlertHandlers) broadcastStateForContext(ctx context.Context) {
	if h.wsHub == nil {
		return
	}

	orgID := apicontext.OrgID(ctx)
	if orgID != "" {
		if h.wsHub.GetTenantClientCount(orgID) == 0 {
			return
		}
	} else if h.wsHub.GetClientCount() == 0 {
		return
	}

	monitor := h.getMonitor(ctx)
	if monitor == nil {
		return
	}
	if orgID != "" {
		h.wsHub.BroadcastCurrentStateToTenant(orgID)
		return
	}
	h.wsHub.BroadcastCurrentState()
}

// validateAlertIdentifier validates an alert identifier for security.
// Alert identifiers may contain user-supplied data (e.g., Docker hostnames), so we allow
// printable ASCII characters while blocking control characters and path traversal.
func validateAlertIdentifier(alertIdentifier string) bool {
	// Guard against empty strings or extremely large payloads that could impact memory usage.
	if len(alertIdentifier) == 0 || len(alertIdentifier) > 500 {
		return false
	}

	// Reject attempts to traverse directories via crafted path segments.
	if strings.Contains(alertIdentifier, "../") || strings.Contains(alertIdentifier, "/..") {
		return false
	}

	// Allow printable ASCII characters (32-126) to support user-supplied identifiers
	// like Docker hostnames that may contain parentheses, brackets, etc.
	// Reject control characters (0-31) and DEL (127) which could cause logging issues.
	for _, r := range alertIdentifier {
		if r < 32 || r > 126 {
			return false
		}
	}

	// Reject IDs that start or end with spaces (likely malformed)
	if alertIdentifier[0] == ' ' || alertIdentifier[len(alertIdentifier)-1] == ' ' {
		return false
	}

	return true
}

// GetAlertConfig returns the current alert configuration
func (h *AlertHandlers) GetAlertConfig(w http.ResponseWriter, r *http.Request) {
	monitor := h.requireMonitor(w, r)
	if monitor == nil {
		return
	}
	config := monitor.GetAlertManager().GetConfig()

	if err := utils.WriteJSONResponse(w, config); err != nil {
		log.Error().Err(err).Msg("Failed to write alert config response")
	}
}

// GetDeadManStatus returns the live external-watchdog read model without
// echoing the secret-bearing ping URL.
func (h *AlertHandlers) GetDeadManStatus(w http.ResponseWriter, r *http.Request) {
	monitor := h.requireMonitor(w, r)
	if monitor == nil {
		return
	}
	if err := utils.WriteJSONResponse(w, monitor.DeadManStatus()); err != nil {
		log.Error().Err(err).Msg("Failed to write dead-man status response")
	}
}

const deadManRedactedPingURL = "***REDACTED***"

type deadManConfigView struct {
	PingURL    string `json:"pingUrl"`
	Configured bool   `json:"configured"`
}

// GetDeadManConfig returns only a presence marker for the credential-bearing
// ping URL. The sentinel supports an edit round-trip without disclosing it.
func (h *AlertHandlers) GetDeadManConfig(w http.ResponseWriter, r *http.Request) {
	monitor := h.requireMonitor(w, r)
	if monitor == nil {
		return
	}
	configured := strings.TrimSpace(monitor.DeadManConfig().PingURL) != ""
	view := deadManConfigView{Configured: configured}
	if configured {
		view.PingURL = deadManRedactedPingURL
	}
	if err := utils.WriteJSONResponse(w, view); err != nil {
		log.Error().Err(err).Msg("Failed to write dead-man config response")
	}
}

// UpdateDeadManConfig validates and durably encrypts the watchdog destination
// before its live worker is reconfigured.
func (h *AlertHandlers) UpdateDeadManConfig(w http.ResponseWriter, r *http.Request) {
	monitor := h.requireMonitor(w, r)
	if monitor == nil {
		return
	}
	r.Body = http.MaxBytesReader(w, r.Body, notifications.MaxDeadManPingURLLength+1024)
	var request deadManConfigView
	if err := json.NewDecoder(r.Body).Decode(&request); err != nil {
		http.Error(w, "Invalid request body", http.StatusBadRequest)
		return
	}
	pingURL := strings.TrimSpace(request.PingURL)
	if pingURL == deadManRedactedPingURL {
		pingURL = monitor.DeadManConfig().PingURL
	}
	config := notifications.DeadManConfig{PingURL: pingURL}
	if err := notifications.ValidateDeadManPingURL(config.PingURL); err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}
	if err := monitor.UpdateDeadManConfig(config); err != nil {
		log.Error().Err(err).Msg("Failed to update dead-man configuration")
		http.Error(w, "Failed to save external watchdog configuration", http.StatusInternalServerError)
		return
	}
	if err := utils.WriteJSONResponse(w, map[string]interface{}{
		"success":    true,
		"configured": config.PingURL != "",
	}); err != nil {
		log.Error().Err(err).Msg("Failed to write dead-man config update response")
	}
}

// UpdateAlertConfig updates the alert configuration
func (h *AlertHandlers) UpdateAlertConfig(w http.ResponseWriter, r *http.Request) {
	// Config size scales with the fleet: every per-resource toggle adds an
	// Overrides entry keyed by a long resource ID, so 64KB rejected saves
	// from instances with a few hundred disabled containers (#1601).
	r.Body = http.MaxBytesReader(w, r.Body, 1<<20)

	body, err := io.ReadAll(r.Body)
	if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}
	// One save at a time, so a slower save cannot write an older config to
	// disk after a newer one was applied.
	h.configSaveMu.Lock()
	defer h.configSaveMu.Unlock()

	// Keys the client left out keep their stored values, so a thresholds
	// save no longer turns off flapping detection or alert TTL cleanup.
	monitor := h.requireMonitor(w, r)
	if monitor == nil {
		return
	}
	updatedConfig, err := monitor.GetAlertManager().ApplyConfigUpdate(body)
	if errors.Is(err, alerts.ErrConfigSnapshot) {
		log.Error().Err(err).Msg("Failed to snapshot applied alert configuration")
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	// Update notification manager with schedule settings
	notificationMgr := monitor.GetNotificationManager()
	notificationMgr.SetEnabled(updatedConfig.Enabled && updatedConfig.ActivationState == alerts.ActivationActive)
	notificationMgr.SetCooldown(updatedConfig.Schedule.Cooldown)
	notificationMgr.SetGroupingConfig(
		updatedConfig.Schedule.Grouping.Enabled,
		updatedConfig.Schedule.Grouping.Window,
		updatedConfig.Schedule.Grouping.ByNode,
		updatedConfig.Schedule.Grouping.ByGuest,
	)
	notificationMgr.SetInitialNotifyTarget(updatedConfig.Schedule.InitialNotify)
	notificationMgr.SetNotifyOnResolve(updatedConfig.Schedule.NotifyOnResolve)

	// Save to persistent storage. Failure here used to be swallowed (logged
	// and reported "success"), which let the in-memory state diverge from
	// what was actually persisted. On the next restart or config reload,
	// the override silently vanished. Surface the error so the client can
	// show a real save-failed signal.
	if err := monitor.GetConfigPersistence().SaveAlertConfig(updatedConfig); err != nil {
		log.Error().Err(err).Msg("Failed to save alert configuration")
		http.Error(w, fmt.Sprintf("Failed to save alert configuration: %v", err), http.StatusInternalServerError)
		return
	}

	if err := utils.WriteJSONResponse(w, map[string]interface{}{
		"success": true,
		"message": "Alert configuration updated successfully",
	}); err != nil {
		log.Error().Err(err).Msg("Failed to write alert config update response")
	}
}

// ActivateAlerts activates alert notifications
func (h *AlertHandlers) ActivateAlerts(w http.ResponseWriter, r *http.Request) {
	// Get current config
	monitor := h.requireMonitor(w, r)
	if monitor == nil {
		return
	}
	config := monitor.GetAlertManager().GetConfig()

	// Check if already active
	if config.ActivationState == alerts.ActivationActive {
		if err := utils.WriteJSONResponse(w, map[string]interface{}{
			"success":        true,
			"message":        "Alerts already activated",
			"state":          string(config.ActivationState),
			"activationTime": config.ActivationTime,
		}); err != nil {
			log.Error().Err(err).Msg("Failed to write activate response")
		}
		return
	}

	// Activate notifications
	now := time.Now()
	config.ActivationState = alerts.ActivationActive
	config.ActivationTime = &now

	// Update config
	monitor.GetAlertManager().UpdateConfig(config)
	monitor.GetNotificationManager().SetEnabled(
		config.Enabled && config.ActivationState == alerts.ActivationActive,
	)

	// Save to persistent storage
	if err := monitor.GetConfigPersistence().SaveAlertConfig(config); err != nil {
		log.Error().Err(err).Msg("Failed to save alert configuration after activation")
		http.Error(w, "Failed to save configuration", http.StatusInternalServerError)
		return
	}

	// Notify about existing critical alerts after activation
	activeAlerts := monitor.GetAlertManager().GetActiveAlerts()
	criticalCount := 0
	for _, alert := range activeAlerts {
		if alert.Level == alerts.AlertLevelCritical && !alert.Acknowledged {
			// Re-dispatch critical alerts to trigger notifications
			monitor.GetAlertManager().NotifyExistingAlert(alert.ID)
			criticalCount++
		}
	}

	if criticalCount > 0 {
		log.Info().
			Int("criticalAlerts", criticalCount).
			Msg("Sent notifications for existing critical alerts after activation")
	}

	log.Info().Msg("Alert notifications activated")

	if err := utils.WriteJSONResponse(w, map[string]interface{}{
		"success":        true,
		"message":        "Alert notifications activated",
		"state":          string(config.ActivationState),
		"activationTime": config.ActivationTime,
	}); err != nil {
		log.Error().Err(err).Msg("Failed to write activate response")
	}
}

// GetActiveAlerts returns all active alerts
func (h *AlertHandlers) GetActiveAlerts(w http.ResponseWriter, r *http.Request) {
	monitor := h.requireMonitor(w, r)
	if monitor == nil {
		return
	}
	alerts := monitor.GetAlertManager().GetActiveAlerts()

	if err := utils.WriteJSONResponse(w, alerts); err != nil {
		log.Error().Err(err).Msg("Failed to write active alerts response")
	}
}

func (h *AlertHandlers) GetAlertIntentPolicies(w http.ResponseWriter, r *http.Request) {
	h.intentPolicyMu.Lock()
	defer h.intentPolicyMu.Unlock()
	monitor := h.requireMonitor(w, r)
	if monitor == nil {
		return
	}
	manager, ok := monitor.GetAlertManager().(alertIntentPolicyManager)
	if !ok {
		http.Error(w, "alert intent policies are unavailable", http.StatusServiceUnavailable)
		return
	}
	if err := utils.WriteJSONResponse(w, manager.GetIntentPolicies()); err != nil {
		log.Error().Err(err).Msg("Failed to write alert intent policies response")
	}
}

func (h *AlertHandlers) UpdateAlertIntentPolicies(w http.ResponseWriter, r *http.Request) {
	// Per-resource policy rules scale with the fleet the same way alert
	// config overrides do, so this shares their 1MB bound.
	r.Body = http.MaxBytesReader(w, r.Body, 1<<20)
	var document alerts.AlertIntentPolicyDocument
	decoder := json.NewDecoder(r.Body)
	decoder.DisallowUnknownFields()
	if err := decoder.Decode(&document); err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}
	if err := decoder.Decode(&struct{}{}); err != io.EOF {
		http.Error(w, "request body must contain one JSON object", http.StatusBadRequest)
		return
	}
	// Treat revision check, in-memory install, and durable save as one API
	// transaction. This also prevents a failed earlier writer from rolling
	// back a later successful update.
	h.intentPolicyMu.Lock()
	defer h.intentPolicyMu.Unlock()
	monitor := h.requireMonitor(w, r)
	if monitor == nil {
		return
	}
	manager, ok := monitor.GetAlertManager().(alertIntentPolicyManager)
	if !ok {
		http.Error(w, "alert intent policies are unavailable", http.StatusServiceUnavailable)
		return
	}
	persistence, ok := monitor.GetConfigPersistence().(alertIntentPolicyPersistence)
	if !ok {
		http.Error(w, "alert intent policy persistence is unavailable", http.StatusServiceUnavailable)
		return
	}
	previous := manager.GetIntentPolicies()
	updated, err := manager.UpdateIntentPolicies(document)
	if err != nil {
		if errors.Is(err, alerts.ErrAlertIntentPolicyRevisionConflict) {
			http.Error(w, err.Error(), http.StatusConflict)
			return
		}
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}
	if err := persistence.SaveAlertIntentPolicies(updated); err != nil {
		if rollbackErr := manager.LoadIntentPolicies(previous); rollbackErr != nil {
			log.Error().Err(rollbackErr).Msg("Failed to roll back in-memory alert intent policies after persistence failure")
		}
		http.Error(w, fmt.Sprintf("Failed to save alert intent policies: %v", err), http.StatusInternalServerError)
		return
	}
	if err := utils.WriteJSONResponse(w, updated); err != nil {
		log.Error().Err(err).Msg("Failed to write updated alert intent policies response")
	}
}

func (h *AlertHandlers) PreviewAlertIntentPolicy(w http.ResponseWriter, r *http.Request) {
	r.Body = http.MaxBytesReader(w, r.Body, 64*1024)
	var request alerts.AlertIntentPolicyPreviewRequest
	decoder := json.NewDecoder(r.Body)
	decoder.DisallowUnknownFields()
	if err := decoder.Decode(&request); err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}
	if err := decoder.Decode(&struct{}{}); err != io.EOF {
		http.Error(w, "request body must contain one JSON object", http.StatusBadRequest)
		return
	}
	h.intentPolicyMu.Lock()
	defer h.intentPolicyMu.Unlock()
	monitor := h.requireMonitor(w, r)
	if monitor == nil {
		return
	}
	manager, ok := monitor.GetAlertManager().(alertIntentPolicyManager)
	if !ok {
		http.Error(w, "alert intent policy preview is unavailable", http.StatusServiceUnavailable)
		return
	}
	preview, err := manager.PreviewIntentPolicy(request)
	if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}
	if err := utils.WriteJSONResponse(w, preview); err != nil {
		log.Error().Err(err).Msg("Failed to write alert intent policy preview response")
	}
}

// GetAlertDeliveryDiagnosis explains current notification delivery policy
// without sending a notification or mutating delivery state. With an
// alertIdentifier query parameter it returns one diagnosis object; without
// one it returns the diagnosis array for every active alert in a single
// pass so list surfaces do not need a request per alert.
func (h *AlertHandlers) GetAlertDeliveryDiagnosis(w http.ResponseWriter, r *http.Request) {
	if r.Method != http.MethodGet {
		http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
		return
	}

	alertIdentifier := strings.TrimSpace(r.URL.Query().Get("alertIdentifier"))
	if alertIdentifier == "" {
		alertIdentifier = strings.TrimSpace(r.URL.Query().Get("id"))
	}
	if alertIdentifier != "" && !validateAlertIdentifier(alertIdentifier) {
		http.Error(w, "Invalid alert identifier", http.StatusBadRequest)
		return
	}

	monitor := h.requireMonitor(w, r)
	if monitor == nil {
		return
	}

	manager := monitor.GetAlertManager()
	if manager == nil {
		http.Error(w, "alert manager is not initialized", http.StatusServiceUnavailable)
		return
	}

	if alertIdentifier == "" {
		diagnoses := manager.DiagnoseActiveAlertDeliveries()
		if diagnoses == nil {
			diagnoses = []alerts.AlertDeliveryDiagnosis{}
		}
		if err := utils.WriteJSONResponse(w, diagnoses); err != nil {
			log.Error().Err(err).Msg("Failed to write alert delivery diagnoses response")
		}
		return
	}

	diagnosis, exists := manager.DiagnoseAlertDelivery(alertIdentifier)
	if !exists {
		http.Error(w, "alert not found", http.StatusNotFound)
		return
	}

	if err := utils.WriteJSONResponse(w, diagnosis); err != nil {
		log.Error().Err(err).Msg("Failed to write alert delivery diagnosis response")
	}
}

// GetAlertEvents returns entries from the append-only alert event log:
// lifecycle transitions and notification decisions, including suppressions
// with reasons. Newest first.
func (h *AlertHandlers) GetAlertEvents(w http.ResponseWriter, r *http.Request) {
	if r.Method != http.MethodGet {
		http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
		return
	}

	query := r.URL.Query()
	filter := eventlog.Filter{
		AlertID: strings.TrimSpace(query.Get("alertIdentifier")),
	}
	if filter.AlertID != "" && !validateAlertIdentifier(filter.AlertID) {
		http.Error(w, "Invalid alert identifier", http.StatusBadRequest)
		return
	}
	for _, raw := range query["type"] {
		for _, part := range strings.Split(raw, ",") {
			if trimmed := strings.TrimSpace(part); trimmed != "" {
				filter.Types = append(filter.Types, trimmed)
			}
		}
	}
	if since := strings.TrimSpace(query.Get("since")); since != "" {
		parsed, err := time.Parse(time.RFC3339, since)
		if err != nil {
			http.Error(w, "since must be RFC3339", http.StatusBadRequest)
			return
		}
		filter.Since = parsed
	}
	if limitRaw := strings.TrimSpace(query.Get("limit")); limitRaw != "" {
		limit, err := strconv.Atoi(limitRaw)
		if err != nil || limit <= 0 {
			http.Error(w, "limit must be a positive integer", http.StatusBadRequest)
			return
		}
		filter.Limit = limit
	}

	monitor := h.requireMonitor(w, r)
	if monitor == nil {
		return
	}
	manager := monitor.GetAlertManager()
	if manager == nil {
		http.Error(w, "alert manager is not initialized", http.StatusServiceUnavailable)
		return
	}

	events, err := manager.AlertEvents(filter)
	if err != nil {
		log.Error().Err(err).Msg("Failed to query alert event log")
		http.Error(w, "failed to query alert events", http.StatusInternalServerError)
		return
	}
	if events == nil {
		events = []eventlog.Event{}
	}
	if err := utils.WriteJSONResponse(w, events); err != nil {
		log.Error().Err(err).Msg("Failed to write alert events response")
	}
}

// GetAlertHistory returns alert history
func (h *AlertHandlers) GetAlertHistory(w http.ResponseWriter, r *http.Request) {
	query := r.URL.Query()

	limit := 100
	if limitStr := query.Get("limit"); limitStr != "" {
		l, err := strconv.Atoi(limitStr)
		switch {
		case err != nil:
			log.Warn().Str("limit", limitStr).Msg("Invalid limit parameter, using default")
		case l < 0:
			http.Error(w, "limit must be non-negative", http.StatusBadRequest)
			return
		case l == 0:
			limit = 0
		case l > 10000:
			log.Warn().Int("limit", l).Msg("Limit exceeds maximum, capping at 10000")
			limit = 10000
		default:
			limit = l
		}
	}

	offset := 0
	if offsetStr := query.Get("offset"); offsetStr != "" {
		if o, err := strconv.Atoi(offsetStr); err == nil {
			if o < 0 {
				http.Error(w, "offset must be non-negative", http.StatusBadRequest)
				return
			}
			offset = o
		} else {
			log.Warn().Str("offset", offsetStr).Msg("Invalid offset parameter, ignoring")
		}
	}

	var startTime *time.Time
	if startStr := query.Get("startTime"); startStr != "" {
		parsed, err := time.Parse(time.RFC3339, startStr)
		if err != nil {
			http.Error(w, "invalid startTime parameter", http.StatusBadRequest)
			return
		}
		startTime = &parsed
	}

	var endTime *time.Time
	if endStr := query.Get("endTime"); endStr != "" {
		parsed, err := time.Parse(time.RFC3339, endStr)
		if err != nil {
			http.Error(w, "invalid endTime parameter", http.StatusBadRequest)
			return
		}
		endTime = &parsed
	}

	if startTime != nil && endTime != nil && endTime.Before(*startTime) {
		http.Error(w, "endTime must be after startTime", http.StatusBadRequest)
		return
	}

	severity := strings.ToLower(strings.TrimSpace(query.Get("severity")))
	switch severity {
	case "", "all":
		severity = ""
	case "info", "warning", "critical":
	default:
		log.Warn().Str("severity", severity).Msg("Invalid severity filter, ignoring")
		severity = ""
	}

	resourceID := strings.TrimSpace(query.Get("resourceId"))

	matchesFilters := func(alertTime time.Time, alertLevel string, alertResourceID string) bool {
		if startTime != nil && alertTime.Before(*startTime) {
			return false
		}
		if endTime != nil && alertTime.After(*endTime) {
			return false
		}
		if severity != "" && !strings.EqualFold(alertLevel, severity) {
			return false
		}
		if resourceID != "" && alertResourceID != resourceID {
			return false
		}
		return true
	}

	trimAlerts := func(alerts []alerts.Alert) []alerts.Alert {
		if offset > 0 {
			if offset >= len(alerts) {
				return alerts[:0]
			}
			alerts = alerts[offset:]
		}
		if limit > 0 && len(alerts) > limit {
			alerts = alerts[:limit]
		}
		return alerts
	}

	trimMockAlerts := func(alerts []models.Alert) []models.Alert {
		if offset > 0 {
			if offset >= len(alerts) {
				return alerts[:0]
			}
			alerts = alerts[offset:]
		}
		if limit > 0 && len(alerts) > limit {
			alerts = alerts[:limit]
		}
		return alerts
	}

	// Check if mock mode is enabled
	mockEnabled := mock.IsMockEnabled()
	log.Debug().Bool("mockEnabled", mockEnabled).Msg("GetAlertHistory: mock mode status")

	fetchLimit := limit
	if fetchLimit > 0 && offset > 0 {
		fetchLimit += offset
	}

	if mockEnabled {
		mockHistory := mock.GetMockAlertHistory(fetchLimit)
		filtered := make([]models.Alert, 0, len(mockHistory))
		for _, alert := range mockHistory {
			if matchesFilters(alert.StartTime, alert.Level, alert.ResourceID) {
				filtered = append(filtered, alert)
			}
		}
		filtered = trimMockAlerts(filtered)
		if err := utils.WriteJSONResponse(w, filtered); err != nil {
			log.Error().Err(err).Msg("Failed to write mock alert history response")
		}
		return
	}

	monitor := h.requireMonitor(w, r)
	if monitor == nil {
		return
	}

	manager := monitor.GetAlertManager()
	var history []alerts.Alert
	if startTime != nil {
		history = manager.GetAlertHistorySince(*startTime, fetchLimit)
	} else {
		history = manager.GetAlertHistory(fetchLimit)
	}

	filtered := make([]alerts.Alert, 0, len(history))
	for _, alert := range history {
		if matchesFilters(alert.StartTime, string(alert.Level), alert.ResourceID) {
			filtered = append(filtered, alert)
		}
	}
	filtered = trimAlerts(filtered)

	if err := utils.WriteJSONResponse(w, filtered); err != nil {
		log.Error().Err(err).Msg("Failed to write alert history response")
	}
}

// GetAlertIncidentTimeline returns the incident timeline for an alert or resource.
func (h *AlertHandlers) GetAlertIncidentTimeline(w http.ResponseWriter, r *http.Request) {
	if r.Method != http.MethodGet {
		http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
		return
	}

	query := r.URL.Query()
	alertID := strings.TrimSpace(query.Get("alertIdentifier"))
	resourceID := strings.TrimSpace(query.Get("resource_id"))
	startedAtRaw := strings.TrimSpace(query.Get("started_at"))
	if startedAtRaw == "" {
		startedAtRaw = strings.TrimSpace(query.Get("start_time"))
	}
	limit := 0
	if rawLimit := strings.TrimSpace(query.Get("limit")); rawLimit != "" {
		if parsed, err := strconv.Atoi(rawLimit); err == nil && parsed > 0 {
			limit = parsed
		}
	}

	monitor := h.requireMonitor(w, r)
	if monitor == nil {
		return
	}
	if alertID != "" {
		if !validateAlertIdentifier(alertID) {
			http.Error(w, "Invalid alert identifier", http.StatusBadRequest)
			return
		}
		var startedAt time.Time
		if startedAtRaw != "" {
			parsed, err := time.Parse(time.RFC3339, startedAtRaw)
			if err != nil {
				http.Error(w, "Invalid started_at time", http.StatusBadRequest)
				return
			}
			startedAt = parsed
		}
		if mockIncident := mock.GetMockAlertIncidentTimeline(alertID, startedAt); mockIncident != nil {
			if err := utils.WriteJSONResponse(w, exportIncident(mockIncident)); err != nil {
				log.Error().Err(err).Msg("Failed to write mock incident timeline response")
			}
			return
		}

		store := monitor.GetIncidentStore()
		if store == nil {
			http.Error(w, "Incident store unavailable", http.StatusServiceUnavailable)
			return
		}

		page, err := store.QueryIncidents(memory.IncidentQuery{AlertIdentifier: alertID, StartedAt: startedAt, Limit: 1})
		if err != nil {
			http.Error(w, "Incident history unavailable", http.StatusServiceUnavailable)
			return
		}
		var incident *memory.Incident
		if len(page.Incidents) > 0 {
			incident = page.Incidents[0]
		}

		if !startedAt.IsZero() && (incident == nil || len(incident.Events) == 0) {
			if alert, resolvedAt := findAlertOccurrenceForTimeline(monitor.GetAlertManager(), alertID, startedAt); alert != nil {
				incident = store.EnsureAlertOccurrence(alert, resolvedAt)
			}
		}
		if err := utils.WriteJSONResponse(w, exportIncident(incident)); err != nil {
			log.Error().Err(err).Msg("Failed to write incident timeline response")
		}
		return
	}

	if resourceID != "" {
		if len(resourceID) > 500 {
			http.Error(w, "Invalid resource ID", http.StatusBadRequest)
			return
		}
		if mock.IsMockEnabled() {
			incidents := mock.GetMockAlertIncidentsForResource(resourceID, limit)
			if err := utils.WriteJSONResponse(w, exportIncidents(incidents)); err != nil {
				log.Error().Err(err).Msg("Failed to write mock incident list response")
			}
			return
		}
		store := monitor.GetIncidentStore()
		if store == nil {
			http.Error(w, "Incident store unavailable", http.StatusServiceUnavailable)
			return
		}
		page, err := store.QueryIncidents(memory.IncidentQuery{ResourceID: resourceID, Limit: limit})
		if err != nil {
			http.Error(w, "Incident history unavailable", http.StatusServiceUnavailable)
			return
		}
		if err := utils.WriteJSONResponse(w, exportIncidents(page.Incidents)); err != nil {
			log.Error().Err(err).Msg("Failed to write incident list response")
		}
		return
	}

	http.Error(w, "Missing alertIdentifier or resource_id", http.StatusBadRequest)
}

func findAlertOccurrenceForTimeline(manager AlertManager, alertID string, startedAt time.Time) (*alerts.Alert, *time.Time) {
	if manager == nil || strings.TrimSpace(alertID) == "" {
		return nil, nil
	}

	for _, active := range manager.GetActiveAlerts() {
		if alertOccurrenceMatches(&active, alertID, startedAt) {
			return active.Clone(), nil
		}
	}

	for _, historical := range manager.GetAlertHistory(0) {
		if !alertOccurrenceMatches(&historical, alertID, startedAt) {
			continue
		}
		resolvedAt := historical.LastSeen
		if historical.OperationalRecord != nil && historical.OperationalRecord.ResolvedAt != nil && !historical.OperationalRecord.ResolvedAt.IsZero() {
			resolvedAt = *historical.OperationalRecord.ResolvedAt
		}
		if resolvedAt.IsZero() {
			return historical.Clone(), nil
		}
		if !historical.StartTime.IsZero() && resolvedAt.Before(historical.StartTime) {
			resolvedAt = historical.StartTime
		}
		return historical.Clone(), &resolvedAt
	}

	return nil, nil
}

func alertOccurrenceMatches(alert *alerts.Alert, alertID string, startedAt time.Time) bool {
	if alert == nil || strings.TrimSpace(alert.ID) != strings.TrimSpace(alertID) {
		return false
	}
	if startedAt.IsZero() || alert.StartTime.IsZero() {
		return true
	}
	delta := alert.StartTime.Sub(startedAt)
	if delta < 0 {
		delta = -delta
	}
	return delta <= time.Second
}

// SaveAlertIncidentNote stores a user note in the incident timeline.
func (h *AlertHandlers) SaveAlertIncidentNote(w http.ResponseWriter, r *http.Request) {
	if r.Method != http.MethodPost {
		http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
		return
	}

	r.Body = http.MaxBytesReader(w, r.Body, 8*1024)
	var req struct {
		AlertIdentifier string `json:"alertIdentifier"`
		IncidentID      string `json:"incident_id"`
		Note            string `json:"note"`
		User            string `json:"user,omitempty"`
	}
	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
		http.Error(w, "Invalid request body", http.StatusBadRequest)
		return
	}

	req.AlertIdentifier = strings.TrimSpace(req.AlertIdentifier)
	req.IncidentID = strings.TrimSpace(req.IncidentID)
	req.Note = strings.TrimSpace(req.Note)
	req.User = strings.TrimSpace(req.User)

	alertIdentifier := req.AlertIdentifier

	if alertIdentifier == "" && req.IncidentID == "" {
		http.Error(w, "alertIdentifier or incident_id is required", http.StatusBadRequest)
		return
	}
	if alertIdentifier != "" && !validateAlertIdentifier(alertIdentifier) {
		http.Error(w, "Invalid alert identifier", http.StatusBadRequest)
		return
	}
	if req.Note == "" {
		http.Error(w, "note is required", http.StatusBadRequest)
		return
	}
	if mock.AddMockAlertIncidentNote(alertIdentifier, req.IncidentID, req.Note, req.User) {
		if err := utils.WriteJSONResponse(w, map[string]interface{}{"success": true}); err != nil {
			log.Error().Err(err).Msg("Failed to write mock incident note response")
		}
		return
	}

	monitor := h.requireMonitor(w, r)
	if monitor == nil {
		return
	}
	store := monitor.GetIncidentStore()
	if store == nil {
		http.Error(w, "Incident store unavailable", http.StatusServiceUnavailable)
		return
	}

	if ok := store.RecordNote(alertIdentifier, req.IncidentID, req.Note, req.User); !ok {
		http.Error(w, "Failed to save note", http.StatusBadRequest)
		return
	}

	if err := utils.WriteJSONResponse(w, map[string]interface{}{
		"success": true,
	}); err != nil {
		log.Error().Err(err).Msg("Failed to write incident note response")
	}
}

type incidentEventView struct {
	Source    string                           `json:"source,omitempty"`
	Evidence  *unifiedresources.ResourceChange `json:"evidence,omitempty"`
	ID        string                           `json:"id"`
	Type      memory.IncidentEventType         `json:"type"`
	Timestamp time.Time                        `json:"timestamp"`
	Summary   string                           `json:"summary"`
	Details   map[string]interface{}           `json:"details"`
}

func (v incidentEventView) NormalizeCollections() incidentEventView {
	if v.Details == nil {
		v.Details = map[string]interface{}{}
	}
	return v
}

type incidentView struct {
	History         *memory.IncidentHistoryCoverage `json:"history,omitempty"`
	ID              string                          `json:"id"`
	AlertIdentifier string                          `json:"alertIdentifier"`
	AlertType       string                          `json:"alertType"`
	Level           string                          `json:"level"`
	ResourceID      string                          `json:"resourceId"`
	ResourceName    string                          `json:"resourceName"`
	ResourceType    string                          `json:"resourceType,omitempty"`
	Node            string                          `json:"node,omitempty"`
	Instance        string                          `json:"instance,omitempty"`
	Message         string                          `json:"message,omitempty"`
	Status          memory.IncidentStatus           `json:"status"`
	OpenedAt        time.Time                       `json:"openedAt"`
	ClosedAt        *time.Time                      `json:"closedAt,omitempty"`
	Acknowledged    bool                            `json:"acknowledged"`
	AckUser         string                          `json:"ackUser,omitempty"`
	AckTime         *time.Time                      `json:"ackTime,omitempty"`
	Events          []incidentEventView             `json:"events"`
}

func emptyIncidentView() incidentView {
	return incidentView{}.NormalizeCollections()
}

func (v incidentView) NormalizeCollections() incidentView {
	if v.Events == nil {
		v.Events = []incidentEventView{}
	}
	for i := range v.Events {
		v.Events[i] = v.Events[i].NormalizeCollections()
	}
	return v
}

func exportIncident(incident *memory.Incident) *incidentView {
	if incident == nil {
		return nil
	}
	events := make([]incidentEventView, 0, len(incident.Events))
	for _, event := range incident.Events {
		events = append(events, incidentEventView{
			Source: event.Source, Evidence: event.Evidence,
			ID:        event.ID,
			Type:      event.Type,
			Timestamp: event.Timestamp,
			Summary:   event.Summary,
			Details:   event.Details,
		}.NormalizeCollections())
	}
	alertIdentifier := strings.TrimSpace(incident.AlertIdentifier)
	view := emptyIncidentView()
	view.ID = incident.ID
	view.History = incident.History
	view.AlertIdentifier = alertIdentifier
	view.AlertType = incident.AlertType
	view.Level = incident.Level
	view.ResourceID = incident.ResourceID
	view.ResourceName = incident.ResourceName
	view.ResourceType = incident.ResourceType
	view.Node = incident.Node
	view.Instance = incident.Instance
	view.Message = incident.Message
	view.Status = incident.Status
	view.OpenedAt = incident.OpenedAt
	view.ClosedAt = incident.ClosedAt
	view.Acknowledged = incident.Acknowledged
	view.AckUser = incident.AckUser
	view.AckTime = incident.AckTime
	view.Events = events
	returnValue := view.NormalizeCollections()
	return &returnValue
}

func exportIncidents(incidents []*memory.Incident) []*incidentView {
	if len(incidents) == 0 {
		return []*incidentView{}
	}
	exported := make([]*incidentView, 0, len(incidents))
	for _, incident := range incidents {
		if view := exportIncident(incident); view != nil {
			exported = append(exported, view)
		}
	}
	return exported
}

// ClearAlertHistory clears all alert history
func (h *AlertHandlers) ClearAlertHistory(w http.ResponseWriter, r *http.Request) {
	monitor := h.requireMonitor(w, r)
	if monitor == nil {
		return
	}
	if err := monitor.GetAlertManager().ClearAlertHistory(); err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	if err := utils.WriteJSONResponse(w, map[string]string{"status": "success", "message": "Alert history cleared"}); err != nil {
		log.Error().Err(err).Msg("Failed to write alert history clear confirmation")
	}
}

// alertIdentifierRequest is used for endpoints that accept one canonical alert identifier in the request body.
type alertIdentifierRequest struct {
	AlertIdentifier string `json:"alertIdentifier"`
}

type snoozeAlertRequest struct {
	AlertIdentifier string    `json:"alertIdentifier"`
	Until           time.Time `json:"until"`
}

func (r alertIdentifierRequest) identifier() string {
	return strings.TrimSpace(r.AlertIdentifier)
}

type alertIdentifiersRequest struct {
	AlertIdentifiers []string `json:"alertIdentifiers"`
	User             string   `json:"user,omitempty"`
}

func (r alertIdentifiersRequest) identifiers() []string {
	raw := r.AlertIdentifiers
	identifiers := make([]string, 0, len(raw))
	for _, identifier := range raw {
		trimmed := strings.TrimSpace(identifier)
		if trimmed != "" {
			identifiers = append(identifiers, trimmed)
		}
	}
	return identifiers
}

// AcknowledgeAlertByBody acknowledges an alert using a canonical alert identifier from the request body.
// POST /api/alerts/acknowledge with {"alertIdentifier": "alert-id"}
func (h *AlertHandlers) AcknowledgeAlertByBody(w http.ResponseWriter, r *http.Request) {
	var req alertIdentifierRequest
	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
		log.Error().Err(err).Msg("Failed to decode acknowledge request body")
		http.Error(w, "Invalid request body", http.StatusBadRequest)
		return
	}

	alertID := req.identifier()
	if alertID == "" {
		http.Error(w, "Alert identifier is required", http.StatusBadRequest)
		return
	}

	if !validateAlertIdentifier(alertID) {
		log.Error().Str("alertIdentifier", alertID).Msg("Invalid alert identifier")
		http.Error(w, "Invalid alert identifier", http.StatusBadRequest)
		return
	}

	// Get the authenticated user from the auth middleware (set in X-Authenticated-User header)
	user := w.Header().Get("X-Authenticated-User")
	if user == "" {
		user = "unknown" // Fallback if auth header not set (shouldn't happen in normal flow)
	}

	monitor := h.requireMonitor(w, r)
	if monitor == nil {
		return
	}
	if err := monitor.GetAlertManager().AcknowledgeAlert(alertID, user); err != nil {
		log.Error().Err(err).Str("alertID", alertID).Msg("Failed to acknowledge alert")
		http.Error(w, err.Error(), http.StatusNotFound)
		return
	}

	monitor.SyncAlertState()

	log.Info().Str("alertID", alertID).Str("user", user).Msg("Alert acknowledged successfully")

	if err := utils.WriteJSONResponse(w, map[string]bool{"success": true}); err != nil {
		log.Error().Err(err).Str("alertID", alertID).Msg("Failed to write acknowledge response")
	}

	if h.wsHub != nil {
		ctx := r.Context()
		go func() {
			h.broadcastStateForContext(ctx)
		}()
	}
}

// UnacknowledgeAlertByBody unacknowledges an alert using a canonical alert identifier from the request body.
// POST /api/alerts/unacknowledge with {"alertIdentifier": "alert-id"}
func (h *AlertHandlers) UnacknowledgeAlertByBody(w http.ResponseWriter, r *http.Request) {
	var req alertIdentifierRequest
	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
		log.Error().Err(err).Msg("Failed to decode unacknowledge request body")
		http.Error(w, "Invalid request body", http.StatusBadRequest)
		return
	}

	alertID := req.identifier()
	if alertID == "" {
		http.Error(w, "Alert identifier is required", http.StatusBadRequest)
		return
	}

	if !validateAlertIdentifier(alertID) {
		log.Error().Str("alertIdentifier", alertID).Msg("Invalid alert identifier")
		http.Error(w, "Invalid alert identifier", http.StatusBadRequest)
		return
	}

	monitor := h.requireMonitor(w, r)
	if monitor == nil {
		return
	}
	if err := monitor.GetAlertManager().UnacknowledgeAlert(alertID); err != nil {
		log.Error().Err(err).Str("alertID", alertID).Msg("Failed to unacknowledge alert")
		http.Error(w, err.Error(), http.StatusNotFound)
		return
	}

	monitor.SyncAlertState()

	log.Info().Str("alertID", alertID).Msg("Alert unacknowledged successfully")

	if err := utils.WriteJSONResponse(w, map[string]bool{"success": true}); err != nil {
		log.Error().Err(err).Str("alertID", alertID).Msg("Failed to write unacknowledge response")
	}

	if h.wsHub != nil {
		ctx := r.Context()
		go func() {
			h.broadcastStateForContext(ctx)
		}()
	}
}

// SnoozeAlertByBody pauses this occurrence's notifications and escalations.
// POST /api/alerts/snooze with {"alertIdentifier":"alert-id","until":"RFC3339"}
func (h *AlertHandlers) SnoozeAlertByBody(w http.ResponseWriter, r *http.Request) {
	r.Body = http.MaxBytesReader(w, r.Body, 16<<10)
	var req snoozeAlertRequest
	decoder := json.NewDecoder(r.Body)
	decoder.DisallowUnknownFields()
	if err := decoder.Decode(&req); err != nil {
		http.Error(w, "Invalid request body", http.StatusBadRequest)
		return
	}
	alertID := strings.TrimSpace(req.AlertIdentifier)
	if alertID == "" || !validateAlertIdentifier(alertID) || req.Until.IsZero() {
		http.Error(w, "Valid alert identifier and snooze expiry are required", http.StatusBadRequest)
		return
	}
	user := w.Header().Get("X-Authenticated-User")
	if user == "" {
		user = "unknown"
	}
	monitor := h.requireMonitor(w, r)
	if monitor == nil {
		return
	}
	if err := monitor.GetAlertManager().SnoozeAlert(alertID, user, req.Until); err != nil {
		status := http.StatusBadRequest
		if errors.Is(err, alerts.ErrAlertNotFound) {
			status = http.StatusNotFound
		}
		http.Error(w, err.Error(), status)
		return
	}
	monitor.SyncAlertState()
	if err := utils.WriteJSONResponse(w, map[string]any{"success": true, "snoozedUntil": req.Until.UTC()}); err != nil {
		log.Error().Err(err).Str("alertID", alertID).Msg("Failed to write snooze response")
	}
	if h.wsHub != nil {
		ctx := r.Context()
		go h.broadcastStateForContext(ctx)
	}
}

// UnsnoozeAlertByBody resumes normal policy without resolving the alert.
func (h *AlertHandlers) UnsnoozeAlertByBody(w http.ResponseWriter, r *http.Request) {
	var req alertIdentifierRequest
	if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 16<<10)).Decode(&req); err != nil {
		http.Error(w, "Invalid request body", http.StatusBadRequest)
		return
	}
	alertID := req.identifier()
	if alertID == "" || !validateAlertIdentifier(alertID) {
		http.Error(w, "Valid alert identifier is required", http.StatusBadRequest)
		return
	}
	user := w.Header().Get("X-Authenticated-User")
	if user == "" {
		user = "unknown"
	}
	monitor := h.requireMonitor(w, r)
	if monitor == nil {
		return
	}
	if err := monitor.GetAlertManager().UnsnoozeAlert(alertID, user); err != nil {
		status := http.StatusBadRequest
		if errors.Is(err, alerts.ErrAlertNotFound) {
			status = http.StatusNotFound
		}
		http.Error(w, err.Error(), status)
		return
	}
	monitor.SyncAlertState()
	if err := utils.WriteJSONResponse(w, map[string]bool{"success": true}); err != nil {
		log.Error().Err(err).Str("alertID", alertID).Msg("Failed to write unsnooze response")
	}
	if h.wsHub != nil {
		ctx := r.Context()
		go h.broadcastStateForContext(ctx)
	}
}

// ClearAlertByBody clears an alert using a canonical alert identifier from the request body.
// POST /api/alerts/clear with {"alertIdentifier": "alert-id"}
func (h *AlertHandlers) ClearAlertByBody(w http.ResponseWriter, r *http.Request) {
	var req alertIdentifierRequest
	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
		log.Error().Err(err).Msg("Failed to decode clear request body")
		http.Error(w, "Invalid request body", http.StatusBadRequest)
		return
	}

	alertID := req.identifier()
	if alertID == "" {
		http.Error(w, "Alert identifier is required", http.StatusBadRequest)
		return
	}

	if !validateAlertIdentifier(alertID) {
		log.Error().Str("alertIdentifier", alertID).Msg("Invalid alert identifier")
		http.Error(w, "Invalid alert identifier", http.StatusBadRequest)
		return
	}

	monitor := h.requireMonitor(w, r)
	if monitor == nil {
		return
	}
	if !monitor.GetAlertManager().ClearAlert(alertID) {
		http.Error(w, "Alert not found", http.StatusNotFound)
		return
	}

	monitor.SyncAlertState()

	log.Info().Str("alertID", alertID).Msg("Alert cleared successfully")

	if err := utils.WriteJSONResponse(w, map[string]bool{"success": true}); err != nil {
		log.Error().Err(err).Str("alertID", alertID).Msg("Failed to write clear response")
	}

	if h.wsHub != nil {
		ctx := r.Context()
		go func() {
			h.broadcastStateForContext(ctx)
		}()
	}
}

// BulkAcknowledgeAlerts acknowledges multiple alerts at once
func (h *AlertHandlers) BulkAcknowledgeAlerts(w http.ResponseWriter, r *http.Request) {
	// Identifier lists scale with active alert count, and an alert flood is
	// exactly when ack-all gets used, so 32KB starved the one path meant to
	// recover from it (#1601).
	r.Body = http.MaxBytesReader(w, r.Body, 1<<20)

	var request alertIdentifiersRequest

	if err := json.NewDecoder(r.Body).Decode(&request); err != nil {
		http.Error(w, "Invalid request body", http.StatusBadRequest)
		return
	}

	alertIdentifiers := request.identifiers()
	if len(alertIdentifiers) == 0 {
		http.Error(w, "No alert identifiers provided", http.StatusBadRequest)
		return
	}

	// Get the authenticated user from the auth middleware - ignore request.User to prevent spoofing
	user := w.Header().Get("X-Authenticated-User")
	if user == "" {
		user = "unknown" // Fallback if auth header not set (shouldn't happen in normal flow)
	}

	var (
		results    []map[string]interface{}
		anySuccess bool
	)
	monitor := h.requireMonitor(w, r)
	if monitor == nil {
		return
	}
	for _, alertID := range alertIdentifiers {
		result := map[string]interface{}{
			"alertIdentifier": alertID,
			"success":         true,
		}
		if err := monitor.GetAlertManager().AcknowledgeAlert(alertID, user); err != nil {
			result["success"] = false
			result["error"] = err.Error()
		} else {
			anySuccess = true
		}
		results = append(results, result)
	}

	if anySuccess {
		monitor.SyncAlertState()
	}

	// Send response immediately
	if err := utils.WriteJSONResponse(w, map[string]interface{}{
		"results": results,
	}); err != nil {
		log.Error().Err(err).Msg("Failed to write bulk acknowledge response")
	}

	// Broadcast updated state to all WebSocket clients if any alerts were acknowledged
	// Do this in a goroutine to avoid blocking the HTTP response
	if h.wsHub != nil && anySuccess {
		ctx := r.Context()
		go func() {
			h.broadcastStateForContext(ctx)
			log.Debug().Msg("Broadcasted state after bulk alert acknowledgment")
		}()
	}
}

// BulkClearAlerts clears multiple alerts at once
func (h *AlertHandlers) BulkClearAlerts(w http.ResponseWriter, r *http.Request) {
	// Same fleet-scaling bound as BulkAcknowledgeAlerts.
	r.Body = http.MaxBytesReader(w, r.Body, 1<<20)

	var request alertIdentifiersRequest

	if err := json.NewDecoder(r.Body).Decode(&request); err != nil {
		http.Error(w, "Invalid request body", http.StatusBadRequest)
		return
	}

	alertIdentifiers := request.identifiers()
	if len(alertIdentifiers) == 0 {
		http.Error(w, "No alert identifiers provided", http.StatusBadRequest)
		return
	}

	var (
		results    []map[string]interface{}
		anySuccess bool
	)
	monitor := h.requireMonitor(w, r)
	if monitor == nil {
		return
	}
	for _, alertID := range alertIdentifiers {
		result := map[string]interface{}{
			"alertIdentifier": alertID,
			"success":         true,
		}
		if monitor.GetAlertManager().ClearAlert(alertID) {
			anySuccess = true
		} else {
			result["success"] = false
			result["error"] = "alert not found"
		}
		results = append(results, result)
	}

	if anySuccess {
		monitor.SyncAlertState()
	}

	// Send response immediately
	if err := utils.WriteJSONResponse(w, map[string]interface{}{
		"results": results,
	}); err != nil {
		log.Error().Err(err).Msg("Failed to write bulk clear response")
	}

	// Broadcast updated state to all WebSocket clients after response
	// Do this in a goroutine to avoid blocking the HTTP response
	if h.wsHub != nil && anySuccess {
		ctx := r.Context()
		go func() {
			h.broadcastStateForContext(ctx)
			log.Debug().Msg("Broadcasted state after bulk alert clear")
		}()
	}
}

// HandleAlerts routes alert requests to appropriate handlers
func (h *AlertHandlers) HandleAlerts(w http.ResponseWriter, r *http.Request) {
	path := strings.TrimPrefix(r.URL.Path, "/api/alerts/")

	log.Debug().
		Str("originalPath", r.URL.Path).
		Str("trimmedPath", path).
		Str("method", r.Method).
		Msg("HandleAlerts routing request")

	switch {
	case path == "config" && r.Method == http.MethodGet:
		if !apihttp.EnsureScope(w, r, config.ScopeMonitoringRead) {
			return
		}
		h.GetAlertConfig(w, r)
	case path == "config" && r.Method == http.MethodPut:
		if !apihttp.EnsureScope(w, r, config.ScopeMonitoringWrite) {
			return
		}
		h.UpdateAlertConfig(w, r)
	case path == "deadman/status" && r.Method == http.MethodGet:
		if !apihttp.EnsureScope(w, r, config.ScopeMonitoringRead) {
			return
		}
		h.GetDeadManStatus(w, r)
	case path == "deadman/config" && r.Method == http.MethodGet:
		if !apihttp.EnsureScope(w, r, config.ScopeMonitoringRead) {
			return
		}
		h.GetDeadManConfig(w, r)
	case path == "deadman/config" && r.Method == http.MethodPut:
		if !apihttp.EnsureScope(w, r, config.ScopeMonitoringWrite) {
			return
		}
		h.UpdateDeadManConfig(w, r)
	case path == "intent-policies" && r.Method == http.MethodGet:
		if !apihttp.EnsureScope(w, r, config.ScopeMonitoringRead) {
			return
		}
		h.GetAlertIntentPolicies(w, r)
	case path == "intent-policies" && r.Method == http.MethodPut:
		if !apihttp.EnsureScope(w, r, config.ScopeMonitoringWrite) {
			return
		}
		h.UpdateAlertIntentPolicies(w, r)
	case path == "intent-policies/preview" && r.Method == http.MethodPost:
		if !apihttp.EnsureScope(w, r, config.ScopeMonitoringRead) {
			return
		}
		h.PreviewAlertIntentPolicy(w, r)
	case path == "activate" && r.Method == http.MethodPost:
		if !apihttp.EnsureScope(w, r, config.ScopeMonitoringWrite) {
			return
		}
		h.ActivateAlerts(w, r)
	case path == "active" && r.Method == http.MethodGet:
		if !apihttp.EnsureScope(w, r, config.ScopeMonitoringRead) {
			return
		}
		h.GetActiveAlerts(w, r)
	case path == "delivery-diagnosis" && r.Method == http.MethodGet:
		if !apihttp.EnsureScope(w, r, config.ScopeMonitoringRead) {
			return
		}
		h.GetAlertDeliveryDiagnosis(w, r)
	case path == "events" && r.Method == http.MethodGet:
		if !apihttp.EnsureScope(w, r, config.ScopeMonitoringRead) {
			return
		}
		h.GetAlertEvents(w, r)
	case path == "history" && r.Method == http.MethodGet:
		if !apihttp.EnsureScope(w, r, config.ScopeMonitoringRead) {
			return
		}
		h.GetAlertHistory(w, r)
	case path == "incidents" && r.Method == http.MethodGet:
		if !apihttp.EnsureScope(w, r, config.ScopeMonitoringRead) {
			return
		}
		h.GetAlertIncidentTimeline(w, r)
	case path == "incidents/note" && r.Method == http.MethodPost:
		if !apihttp.EnsureScope(w, r, config.ScopeMonitoringWrite) {
			return
		}
		h.SaveAlertIncidentNote(w, r)
	case path == "history" && r.Method == http.MethodDelete:
		if !apihttp.EnsureScope(w, r, config.ScopeMonitoringWrite) {
			return
		}
		h.ClearAlertHistory(w, r)
	case path == "bulk/acknowledge" && r.Method == http.MethodPost:
		if !apihttp.EnsureScope(w, r, config.ScopeMonitoringWrite) {
			return
		}
		h.BulkAcknowledgeAlerts(w, r)
	case path == "bulk/clear" && r.Method == http.MethodPost:
		if !apihttp.EnsureScope(w, r, config.ScopeMonitoringWrite) {
			return
		}
		h.BulkClearAlerts(w, r)
	// Body-based endpoints (preferred - avoids URL encoding issues with reverse proxies)
	case path == "acknowledge" && r.Method == http.MethodPost:
		if !apihttp.EnsureScope(w, r, config.ScopeMonitoringWrite) {
			return
		}
		h.AcknowledgeAlertByBody(w, r)
	case path == "unacknowledge" && r.Method == http.MethodPost:
		if !apihttp.EnsureScope(w, r, config.ScopeMonitoringWrite) {
			return
		}
		h.UnacknowledgeAlertByBody(w, r)
	case path == "snooze" && r.Method == http.MethodPost:
		if !apihttp.EnsureScope(w, r, config.ScopeMonitoringWrite) {
			return
		}
		h.SnoozeAlertByBody(w, r)
	case path == "unsnooze" && r.Method == http.MethodPost:
		if !apihttp.EnsureScope(w, r, config.ScopeMonitoringWrite) {
			return
		}
		h.UnsnoozeAlertByBody(w, r)
	case path == "clear" && r.Method == http.MethodPost:
		if !apihttp.EnsureScope(w, r, config.ScopeMonitoringWrite) {
			return
		}
		h.ClearAlertByBody(w, r)
	default:
		http.Error(w, "Not found", http.StatusNotFound)
	}
}