Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

37.4 KB
Raw Blame
package alerts

import (
	"sort"
	"strings"
	"time"

	"github.com/rcourtman/pulse-go-rewrite/internal/alerts/eventlog"
	"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
	"github.com/rs/zerolog/log"
)

const (
	// MetadataQuietHoursSuppressed marks notifications that were held by alert
	// quiet-hours policy and must be replayed by the notification queue.
	MetadataQuietHoursSuppressed = "quietHoursSuppressed"
	// MetadataQuietHoursSuppressionReason records the quiet-hours rule that
	// deferred notification delivery.
	MetadataQuietHoursSuppressionReason = "quietHoursSuppressionReason"
	// MetadataQuietHoursReplayAt stores the earliest RFC3339 UTC time that the
	// notification queue may retry delivery.
	MetadataQuietHoursReplayAt = "quietHoursReplayAt"
)

const (
	AlertDeliveryStatusWouldSend  = "would_send"
	AlertDeliveryStatusDeferred   = "deferred"
	AlertDeliveryStatusSuppressed = "suppressed"

	AlertDeliveryReasonReady                 = "ready"
	AlertDeliveryReasonAcknowledged          = "acknowledged"
	AlertDeliveryReasonSnoozed               = "snoozed"
	AlertDeliveryReasonNotificationsDisabled = "notifications_disabled"
	AlertDeliveryReasonNotificationsInactive = "notifications_inactive"
	AlertDeliveryReasonSuppressionWindow     = "suppression_window"
	AlertDeliveryReasonFlapping              = "flapping"
	AlertDeliveryReasonRateLimited           = "rate_limited"
	AlertDeliveryReasonCooldown              = "cooldown"
	AlertDeliveryReasonQuietHours            = "quiet_hours"
	AlertDeliveryReasonMonitorOnly           = "monitor_only"
	AlertDeliveryReasonCorrelatedPrimary     = "correlated_primary"
)

// AlertDeliveryDiagnosis is a read-only projection of the alert manager's
// current delivery policy for one active alert. It is intentionally
// non-mutating: it does not dispatch callbacks or advance flapping/rate-limit
// counters.
type AlertDeliveryDiagnosis struct {
	AlertIdentifier         string     `json:"alertIdentifier"`
	AlertID                 string     `json:"alertId"`
	TrackingKey             string     `json:"trackingKey"`
	Status                  string     `json:"status"`
	Reason                  string     `json:"reason"`
	Message                 string     `json:"message"`
	AlertType               string     `json:"alertType"`
	Level                   AlertLevel `json:"level"`
	ResourceID              string     `json:"resourceId,omitempty"`
	ResourceName            string     `json:"resourceName,omitempty"`
	Node                    string     `json:"node,omitempty"`
	NotificationsEnabled    bool       `json:"notificationsEnabled"`
	ActivationState         string     `json:"activationState"`
	CooldownMinutes         int        `json:"cooldownMinutes"`
	MaxAlertsHour           int        `json:"maxAlertsHour"`
	RecentAlertsInHour      int        `json:"recentAlertsInHour"`
	FlappingActive          bool       `json:"flappingActive"`
	FlappingHistoryInWindow int        `json:"flappingHistoryInWindow"`
	FlappingThreshold       int        `json:"flappingThreshold"`
	FlappingWindowSeconds   int        `json:"flappingWindowSeconds"`
	LastNotified            *time.Time `json:"lastNotified,omitempty"`
	NextEligibleAt          *time.Time `json:"nextEligibleAt,omitempty"`
	QuietHoursReplayAt      *time.Time `json:"quietHoursReplayAt,omitempty"`
	SuppressedUntil         *time.Time `json:"suppressedUntil,omitempty"`
}

// checkFlappingLocked detects alert flapping and returns whether the alert
// should be suppressed and whether this call is the first transition into
// the flapping state for the current cooldown window. justTransitioned is
// the signal callers use to fire a one-shot postmortem hook on first
// detection -- it is true only on the call that flips
// flappingActive[trackingKey] from false to true.
//
// It modifies flappingHistory, flappingActive, and suppressedUntil maps.
// IMPORTANT: Caller MUST hold m.mu before calling this function. The
// transition signal is returned so the caller can dispatch the
// flapping-detected callback OUTSIDE the lock (typically via a goroutine).
func (m *Manager) checkFlappingLocked(trackingKey string) (suppress bool, justTransitioned bool) {
	if !m.config.FlappingEnabled {
		return false, false
	}

	now := time.Now()
	windowDuration := time.Duration(m.config.FlappingWindowSeconds) * time.Second

	// Honour an active cooldown before anything else. The cooldown recorded
	// below used to be write-only on this path: suppression was decided purely
	// by the sliding-window count, so it ended the moment the count dipped
	// under the threshold instead of holding for FlappingCooldownMinutes. A
	// resource oscillating just under the threshold rate was therefore never
	// damped at all, while the flapping postmortem told the operator that
	// notifications had been suppressed for the configured cooldown and
	// suggested raising a value that changed nothing.
	if until, cooling := m.suppressedUntil[trackingKey]; cooling {
		if now.Before(until) {
			return true, false
		}
		m.expireSuppressionNoLock(trackingKey, now)
	}

	// Record this state change
	m.flappingHistory[trackingKey] = append(m.flappingHistory[trackingKey], now)

	// Remove state changes outside the window
	history := m.flappingHistory[trackingKey]
	validHistory := []time.Time{}
	for _, t := range history {
		if now.Sub(t) <= windowDuration {
			validHistory = append(validHistory, t)
		}
	}
	// Keep enough observations to reach the configured threshold. A fixed
	// ten-entry cap silently disabled flapping suppression for accepted
	// thresholds above ten. Retain the old diagnostic allowance for smaller
	// thresholds, while still bounding history by configuration rather than
	// the number of dispatch attempts.
	maxFlappingHistory := max(10, m.config.FlappingThreshold)
	if len(validHistory) > maxFlappingHistory {
		validHistory = validHistory[len(validHistory)-maxFlappingHistory:]
	}
	m.flappingHistory[trackingKey] = validHistory

	// Check if we've exceeded the threshold
	if len(validHistory) >= m.config.FlappingThreshold {
		// Mark as flapping
		if !m.flappingActive[trackingKey] {
			log.Warn().
				Str("trackingKey", trackingKey).
				Int("stateChanges", len(validHistory)).
				Int("threshold", m.config.FlappingThreshold).
				Int("windowSeconds", m.config.FlappingWindowSeconds).
				Msg("Flapping detected - suppressing alert")

			m.flappingActive[trackingKey] = true

			// Set cooldown period
			cooldownDuration := time.Duration(m.config.FlappingCooldownMinutes) * time.Minute
			m.suppressedUntil[trackingKey] = now.Add(cooldownDuration)

			// Record suppression metric
			if recordAlertSuppressed != nil {
				recordAlertSuppressed("flapping")
			}
			return true, true
		}
		return true, false
	}

	return false, false
}

// expireSuppressionNoLock retires a served suppression and its episode together.
// Cleanup must use the same reset as dispatch: deleting only the deadline leaves
// a latched flapping episode that can suppress delivery without ever rearming a
// cooldown or notifying the flapping callback. Caller MUST hold m.mu.
func (m *Manager) expireSuppressionNoLock(trackingKey string, now time.Time) bool {
	until, exists := m.suppressedUntil[trackingKey]
	if !exists || now.Before(until) {
		return false
	}
	delete(m.suppressedUntil, trackingKey)
	delete(m.flappingActive, trackingKey)
	delete(m.flappingHistory, trackingKey)
	return true
}

const alertDeliveryReasonBackupRevalidation = "awaiting_backup_evaluation"

// Caller holds mu. The occurrence check prevents a deferred startup snapshot
// from suppressing a later alert that happens to reuse the same identity.
func (m *Manager) awaitsBackupRevalidationNoLock(alert *Alert) bool {
	if alert == nil || alert.Type != "backup-age" {
		return false
	}
	startedAt, pending := m.restoredBackupNotifications[canonicalTrackingKeyForAlert(alert)]
	return pending && startedAt.Equal(alert.StartTime)
}

// A valid, matched canonical evaluation releases the startup delivery hold.
// Recovery/removal retires the hold through the active lifecycle owner instead.
func (m *Manager) revalidateBackupNotificationNoLock(alert *Alert) bool {
	if !m.awaitsBackupRevalidationNoLock(alert) {
		return false
	}
	delete(m.restoredBackupNotifications, canonicalTrackingKeyForAlert(alert))
	return true
}

func (m *Manager) dispatchAlert(alert *Alert, async bool) bool {
	callbacks := m.getAlertCallbacks()
	if len(callbacks) == 0 || alert == nil {
		return false
	}
	if m.awaitsBackupRevalidationNoLock(alert) {
		m.recordAlertEvent(eventlog.TypeNotificationSuppressed, alert, "",
			alertDeliveryReasonBackupRevalidation,
			"Restored backup alert is awaiting a successful backup evaluation.", nil)
		return false
	}

	// Don't dispatch notifications for acknowledged alerts
	if alert.Acknowledged {
		log.Debug().
			Str("alertID", alert.ID).
			Str("ackUser", alert.AckUser).
			Msg("Alert notification suppressed - already acknowledged")
		m.recordAlertEvent(eventlog.TypeNotificationSuppressed, alert, "",
			AlertDeliveryReasonAcknowledged, "Notification suppressed: alert is acknowledged.", nil)
		return false
	}
	if until, snoozed := alertSnoozeUntil(alert, m.policyNow()); snoozed {
		details := map[string]string(nil)
		if until != nil {
			details = map[string]string{"until": until.Format(time.RFC3339Nano)}
		}
		m.recordAlertEvent(eventlog.TypeNotificationSuppressed, alert, "",
			AlertDeliveryReasonSnoozed, "Notification suppressed: alert is snoozed.",
			details)
		return false
	}
	if isSupportedInfrastructureSymptom(alert) {
		m.recordAlertEvent(eventlog.TypeNotificationSuppressed, alert, "",
			AlertDeliveryReasonCorrelatedPrimary,
			"Notification suppressed: a supported primary infrastructure alert owns delivery for this incident group.",
			map[string]string{"primaryAlertId": alert.Correlation.PrimaryAlertID})
		return false
	}
	// Resource monitoring policy is a terminal suppression, unlike quiet hours.
	// An alert already active when the policy changes must not be queued for
	// replay or reach a callback before reconciliation clears it.
	if suppressed, reason := m.operatorSuppressionForAlertNoLock(alert, m.policyNow().UTC()); suppressed {
		clearQuietHoursNotificationReplay(alert)
		m.recordAlertEvent(eventlog.TypeNotificationSuppressed, alert, "", reason,
			"Notification suppressed by resource monitoring policy.", nil)
		return false
	}
	if isMonitorOnlyAlert(alert) {
		clearQuietHoursNotificationReplay(alert)
		m.recordAlertEvent(eventlog.TypeNotificationSuppressed, alert, "",
			AlertDeliveryReasonMonitorOnly, "Notification suppressed: alert is monitor-only.", nil)
		return false
	}

	trackingKey := canonicalTrackingKeyForAlert(alert)

	// Check for flapping (caller must hold m.mu). When this call is the
	// first transition into the flapping state for the current cooldown
	// window, fire the flapping-detected callback so a postmortem patrol
	// can be triggered. The callback runs in its own goroutine because
	// the caller of dispatchAlert holds m.mu and the callback is allowed
	// to take its own locks or re-enter the alerts package.
	suppress, justTransitioned := m.checkFlappingLocked(trackingKey)
	if justTransitioned {
		m.recordAlertEvent(eventlog.TypeFlappingDetected, alert, "",
			AlertDeliveryReasonFlapping, "Alert entered flapping suppression.",
			map[string]string{"trackingKey": trackingKey})
		cb := m.callbacks.flappingDetectedCallback()
		if cb != nil {
			alertCopy := cloneAlertForOutput(alert)
			go func(a *Alert, key string, fn func(*Alert, string)) {
				defer func() {
					if r := recover(); r != nil {
						log.Error().
							Interface("panic", r).
							Str("alertID", a.ID).
							Str("trackingKey", key).
							Msg("Panic in onFlappingDetected callback")
					}
				}()
				fn(a, key)
			}(alertCopy, trackingKey, cb)
		}
	}
	if suppress {
		log.Debug().
			Str("alertID", alert.ID).
			Str("trackingKey", trackingKey).
			Msg("Alert suppressed due to flapping")
		m.recordAlertEvent(eventlog.TypeNotificationSuppressed, alert, "",
			AlertDeliveryReasonFlapping, "Notification suppressed: alert is flapping.",
			map[string]string{"trackingKey": trackingKey})
		return false
	}

	// Check activation state - only dispatch notifications if active
	if m.config.ActivationState != ActivationActive {
		log.Debug().
			Str("alertID", alert.ID).
			Str("activationState", string(m.config.ActivationState)).
			Msg("Alert notification suppressed - not activated")
		m.recordAlertEvent(eventlog.TypeNotificationSuppressed, alert, "",
			AlertDeliveryReasonNotificationsInactive, "Notification suppressed: alert delivery is not turned on.",
			map[string]string{"activationState": string(m.config.ActivationState)})
		return false
	}

	if suppressed, reason := m.shouldSuppressNotification(alert); suppressed {
		replayAt := m.quietHoursReplayAt()
		markQuietHoursNotificationReplay(alert, reason, replayAt)
		log.Debug().
			Str("alertID", alert.ID).
			Str("type", alert.Type).
			Str("level", string(alert.Level)).
			Str("quietHoursRule", reason).
			Time("replayAt", replayAt).
			Msg("Alert notification deferred during quiet hours")
		deferredReason := AlertDeliveryReasonQuietHours
		if reason != "" {
			deferredReason += ":" + reason
		}
		m.recordAlertEvent(eventlog.TypeNotificationDeferred, alert, "",
			deferredReason, "Notification deferred by quiet hours; the queue replays it when they end.",
			map[string]string{"replayAt": replayAt.UTC().Format(time.RFC3339)})
	} else {
		clearQuietHoursNotificationReplay(alert)
	}

	// Record metric for fired alert
	if recordAlertFired != nil {
		recordAlertFired(alert)
	}

	notifiedAt := time.Now()
	m.recordAlertNotifiedNoLock(alert, notifiedAt)
	m.saveActiveAlertsAsync("alert-dispatch")
	m.recordAlertEvent(eventlog.TypeNotificationDispatched, alert, "",
		AlertDeliveryReasonReady, "Notification dispatched to the delivery pipeline.", nil)

	alertCopy := cloneAlertForOutput(alert)
	if async {
		go func(a *Alert, fns []func(*Alert)) {
			defer func() {
				if r := recover(); r != nil {
					log.Error().
						Interface("panic", r).
						Str("alertID", a.ID).
						Str("type", a.Type).
						Msg("Panic in onAlert callback")
				}
			}()
			for _, callback := range fns {
				callback(a)
			}
		}(alertCopy, callbacks)
	} else {
		// Synchronous calls also need panic recovery to prevent service crash
		func(fns []func(*Alert)) {
			defer func() {
				if r := recover(); r != nil {
					log.Error().
						Interface("panic", r).
						Str("alertID", alertCopy.ID).
						Str("type", alertCopy.Type).
						Msg("Panic in onAlert callback (synchronous)")
				}
			}()
			for _, callback := range fns {
				callback(alertCopy)
			}
		}(callbacks)
	}
	return true
}

func (m *Manager) recordAlertNotifiedNoLock(alert *Alert, notifiedAt time.Time) {
	if alert == nil {
		return
	}

	setAlertLastNotified(alert, notifiedAt)

	if trackingKey := canonicalTrackingKeyForAlert(alert); trackingKey != "" {
		if active, exists := m.getActiveAlertNoLock(trackingKey); exists && active != nil {
			setAlertLastNotified(active, notifiedAt)
			return
		}
	}
	if alert.ID != "" {
		if active, exists := m.getActiveAlertNoLock(alert.ID); exists && active != nil {
			setAlertLastNotified(active, notifiedAt)
		}
	}
}

func setAlertLastNotified(alert *Alert, notifiedAt time.Time) {
	if alert == nil {
		return
	}
	t := notifiedAt
	alert.LastNotified = &t
}

func isMonitorOnlyAlert(alert *Alert) bool {
	if alert == nil || alert.Metadata == nil {
		return false
	}

	if value, ok := alert.Metadata["monitorOnly"]; ok {
		switch v := value.(type) {
		case bool:
			return v
		case string:
			return strings.EqualFold(v, "true")
		}
	}
	return false
}

func isSupportedInfrastructureSymptom(alert *Alert) bool {
	return alert != nil &&
		alert.Correlation != nil &&
		alert.Correlation.Kind == AlertCorrelationKindInfrastructureIncident &&
		alert.Correlation.Role == AlertCorrelationRoleSupporting &&
		alert.Correlation.Inference == AlertCorrelationInferenceSupportedCause &&
		strings.TrimSpace(alert.Correlation.PrimaryAlertID) != ""
}

// quietHoursClock compares civil clock minutes, not date-constructed instants.
// time.Date chooses only one side of a repeated hour and normalizes missing
// minutes to a different hour, neither of which is the user's daily schedule.
type quietHoursClock struct {
	location *time.Location
	start    int
	end      int
	days     [7]bool
}

func (m *Manager) quietHoursClock() (quietHoursClock, bool) {
	quiet := m.config.Schedule.QuietHours
	if !quiet.Enabled {
		return quietHoursClock{}, false
	}
	loc := m.quietHoursLoc
	if loc == nil {
		var err error
		loc, err = time.LoadLocation(quiet.Timezone)
		if err != nil {
			log.Warn().Err(err).Str("timezone", quiet.Timezone).Msg("failed to load timezone, using local time")
			loc = time.Local
		}
		// Do not populate the cache here: public suppression helpers can call
		// this path while holding only a read lock. UpdateConfig owns the cache.
	}
	start, err := time.Parse("15:04", quiet.Start)
	if err != nil {
		log.Warn().Err(err).Str("start", quiet.Start).Msg("failed to parse quiet hours start time")
		return quietHoursClock{}, false
	}
	end, err := time.Parse("15:04", quiet.End)
	if err != nil {
		log.Warn().Err(err).Str("end", quiet.End).Msg("failed to parse quiet hours end time")
		return quietHoursClock{}, false
	}
	clock := quietHoursClock{location: loc, start: start.Hour()*60 + start.Minute(), end: end.Hour()*60 + end.Minute()}
	for i, day := range [...]string{"sunday", "monday", "tuesday", "wednesday", "thursday", "friday", "saturday"} {
		clock.days[i] = quiet.Days[day]
	}
	return clock, true
}

func (clock quietHoursClock) contains(at time.Time) bool {
	local := at.In(clock.location)
	if !clock.days[local.Weekday()] {
		return false
	}
	minute := local.Hour()*60 + local.Minute()
	if clock.start > clock.end {
		return minute >= clock.start || minute <= clock.end
	}
	return minute >= clock.start && minute <= clock.end
}

// isInQuietHours checks the selected local calendar day and includes the whole
// configured end minute, including both copies of it during a clock rollback.
func (m *Manager) isInQuietHours() bool {
	clock, valid := m.quietHoursClock()
	return valid && clock.contains(m.policyNow())
}

func quietHoursCategoryForAlert(alert *Alert) string {
	if alert == nil {
		return ""
	}

	switch alert.Type {
	case "cpu", "memory", "disk", "diskRead", "diskWrite", "networkIn", "networkOut", "temperature":
		return "performance"
	case "queue-depth", "queue-deferred", "queue-hold", "message-age",
		"docker-container-health", "docker-container-restart-loop",
		"docker-container-oom-kill", "docker-container-memory-limit":
		return "performance"
	case "usage", "disk-health", "disk-wearout", "zfs-pool-state", "zfs-pool-errors", "zfs-device", "storage-incident", "backup-storage-incident", "backup-posture-incident":
		return "storage"
	case "resource-incident":
		if metadataStringValue(alert.Metadata, "incidentCategory") == unifiedresources.IncidentCategoryAvailability {
			return "offline"
		}
		return "performance"
	case "connectivity", "offline", "powered-off", "docker-host-offline", connectionDegradedAlertType:
		return "offline"
	}

	if strings.HasPrefix(alert.Type, "docker-container-") {
		if alert.Type == "docker-container-state" {
			return "offline"
		}
		return "performance"
	}

	return ""
}

func (m *Manager) shouldSuppressNotification(alert *Alert) (bool, string) {
	// Only quiet-hours matches are replayable. Operator policy is checked by
	// each caller before this helper, because it must drop rather than defer.
	if !m.isInQuietHours() {
		return false, ""
	}
	return quietHoursSuppressionForAlert(alert, m.config.Schedule.QuietHours.Suppress)
}

func quietHoursSuppressionForAlert(alert *Alert, suppress QuietHoursSuppression) (bool, string) {
	if alert == nil {
		return false, ""
	}

	if alert.Level != AlertLevelCritical {
		return true, "non-critical"
	}

	category := quietHoursCategoryForAlert(alert)
	switch category {
	case "performance":
		if suppress.Performance {
			return true, category
		}
	case "storage":
		if suppress.Storage {
			return true, category
		}
	case "offline":
		if suppress.Offline {
			return true, category
		}
	}

	return false, ""
}

func (m *Manager) quietHoursReplayAt() time.Time {
	now := m.policyNow()
	clock, valid := m.quietHoursClock()
	if !valid {
		return now.Add(time.Minute).UTC()
	}
	return clock.replayAt(now)
}

// QuietHoursNotificationPolicy returns an immutable snapshot of the current
// quiet-hours rule. The delivery owner obtains it before taking queue or
// per-alert delivery locks, so evaluation never calls back into this manager
// while cancellation is waiting for those locks. It does not mutate alert
// lifecycle, acknowledgement, metadata or destination selection.
func (m *Manager) QuietHoursNotificationPolicy() func(*Alert, time.Time) *time.Time {
	m.mu.RLock()
	clock, valid := m.quietHoursClock()
	suppress := m.config.Schedule.QuietHours.Suppress
	m.mu.RUnlock()
	return func(alert *Alert, now time.Time) *time.Time {
		if !valid || !clock.contains(now) {
			return nil
		}
		if suppressed, _ := quietHoursSuppressionForAlert(alert, suppress); !suppressed {
			return nil
		}
		replayAt := clock.replayAt(now)
		return &replayAt
	}
}

func (clock quietHoursClock) replayAt(now time.Time) time.Time {
	if !clock.contains(now) {
		return now.Add(time.Minute).UTC()
	}

	// A full-day window has no non-quiet clock minute. Preserve its existing
	// daily revalidation boundary. The queue must check policy again there,
	// not invent an end to a continuous schedule or attempt a provider send.
	if (clock.end-clock.start+24*60)%(24*60) == 24*60-1 {
		local := now.In(clock.location)
		endExclusive := time.Date(local.Year(), local.Month(), local.Day(), clock.end/60, clock.end%60, 0, 0, clock.location).Add(time.Minute)
		if clock.start > clock.end && local.Hour()*60+local.Minute() >= clock.start {
			endExclusive = endExclusive.AddDate(0, 0, 1)
		}
		if endExclusive.After(now) {
			return endExclusive.UTC()
		}
		return now.Add(time.Minute).UTC()
	}

	// Walk real minutes through the current window, rather than constructing
	// its end on a local date. This handles repeated ends, skipped ends, half-
	// hour DST transitions and a midnight leading into an unselected day. A
	// non-full-day window has an exit within three real days even when a
	// forward clock change skips its entire gap on the first day.
	candidate := now.Truncate(time.Minute).Add(time.Minute)
	limit := candidate.Add(72 * time.Hour)
	for candidate.Before(limit) {
		if !clock.contains(candidate) {
			return candidate.UTC()
		}
		candidate = candidate.Add(time.Minute)
	}
	log.Warn().Msg("could not find quiet-hours window end within three days")
	return now.Add(time.Minute).UTC()
}

func markQuietHoursNotificationReplay(alert *Alert, reason string, replayAt time.Time) {
	if alert == nil {
		return
	}
	if alert.Metadata == nil {
		alert.Metadata = make(map[string]interface{}, 3)
	}
	alert.Metadata[MetadataQuietHoursSuppressed] = true
	alert.Metadata[MetadataQuietHoursSuppressionReason] = reason
	alert.Metadata[MetadataQuietHoursReplayAt] = replayAt.UTC().Format(time.RFC3339)
}

func clearQuietHoursNotificationReplay(alert *Alert) {
	if alert == nil || alert.Metadata == nil {
		return
	}
	delete(alert.Metadata, MetadataQuietHoursSuppressed)
	delete(alert.Metadata, MetadataQuietHoursSuppressionReason)
	delete(alert.Metadata, MetadataQuietHoursReplayAt)
}

func hasQuietHoursNotificationReplay(alert *Alert) bool {
	if alert == nil || alert.Metadata == nil {
		return false
	}
	if replayAt, ok := alert.Metadata[MetadataQuietHoursReplayAt].(string); ok && strings.TrimSpace(replayAt) != "" {
		return true
	}
	if suppressed, ok := alert.Metadata[MetadataQuietHoursSuppressed].(bool); ok {
		return suppressed
	}
	return false
}

// ShouldSuppressNotification checks whether a notification must be dropped at
// the alert-policy layer. Quiet-hours matches are replayable, so this helper
// annotates the alert for notification-queue retry and returns false to let
// bypass callers hand the alert to the delivery owner.
func (m *Manager) ShouldSuppressNotification(alert *Alert) bool {
	if alert == nil {
		return false
	}

	m.mu.RLock()
	defer m.mu.RUnlock()
	if m.awaitsBackupRevalidationNoLock(alert) {
		return true
	}
	if _, snoozed := alertSnoozeUntil(alert, m.policyNow()); snoozed {
		clearQuietHoursNotificationReplay(alert)
		return true
	}
	if isSupportedInfrastructureSymptom(alert) {
		clearQuietHoursNotificationReplay(alert)
		return true
	}

	if suppressed, reason := m.operatorSuppressionForAlertNoLock(alert, time.Now().UTC()); suppressed {
		clearQuietHoursNotificationReplay(alert)
		log.Debug().
			Str("alertID", alert.ID).
			Str("type", alert.Type).
			Str("operatorPolicy", reason).
			Msg("Notification suppressed by resource monitoring policy")
		return true
	}
	if isMonitorOnlyAlert(alert) {
		clearQuietHoursNotificationReplay(alert)
		return true
	}

	suppressed, reason := m.shouldSuppressNotification(alert)
	if suppressed {
		replayAt := m.quietHoursReplayAt()
		markQuietHoursNotificationReplay(alert, reason, replayAt)
		log.Debug().
			Str("alertID", alert.ID).
			Str("type", alert.Type).
			Str("level", string(alert.Level)).
			Str("quietHoursRule", reason).
			Time("replayAt", replayAt).
			Msg("Notification deferred during quiet hours")
		return false
	}
	clearQuietHoursNotificationReplay(alert)
	return false
}

// ShouldSuppressResolvedNotification checks if a recovery notification should be suppressed.
// Recovery notifications keep the existing quiet-hours rule unless the firing
// notification was already deferred into the owned queue replay path; in that
// case the resolved notification is also allowed to enter the queue so the
// close-the-loop delivery is not lost.
func (m *Manager) ShouldSuppressResolvedNotification(alert *Alert) bool {
	if alert == nil {
		return false
	}

	m.mu.RLock()
	defer m.mu.RUnlock()

	if alert.Acknowledged {
		log.Debug().
			Str("alertID", alert.ID).
			Str("type", alert.Type).
			Msg("Recovery notification suppressed for acknowledged alert")
		return true
	}
	if _, snoozed := alertSnoozeUntil(alert, m.policyNow()); snoozed {
		clearQuietHoursNotificationReplay(alert)
		log.Debug().Str("alertID", alert.ID).Msg("Recovery notification suppressed for snoozed alert")
		return true
	}
	if suppressed, reason := m.operatorSuppressionForAlertNoLock(alert, time.Now().UTC()); suppressed {
		clearQuietHoursNotificationReplay(alert)
		log.Debug().
			Str("alertID", alert.ID).
			Str("type", alert.Type).
			Str("operatorPolicy", reason).
			Msg("Recovery notification suppressed by resource monitoring policy")
		return true
	}
	if isMonitorOnlyAlert(alert) {
		clearQuietHoursNotificationReplay(alert)
		return true
	}

	quietHoursReplay := hasQuietHoursNotificationReplay(alert)
	if alert.LastNotified == nil && !quietHoursReplay {
		log.Debug().
			Str("alertID", alert.ID).
			Str("type", alert.Type).
			Msg("Recovery notification suppressed because firing notification was never sent")
		return true
	}

	suppressed, reason := m.shouldSuppressNotification(alert)
	if suppressed {
		if quietHoursReplay {
			log.Debug().
				Str("alertID", alert.ID).
				Str("type", alert.Type).
				Str("level", string(alert.Level)).
				Str("quietHoursRule", reason).
				Msg("Recovery notification allowed into quiet-hours replay queue")
			return false
		}
		log.Debug().
			Str("alertID", alert.ID).
			Str("type", alert.Type).
			Str("level", string(alert.Level)).
			Str("quietHoursRule", reason).
			Msg("Recovery notification suppressed during quiet hours")
	}
	return suppressed
}

// shouldNotifyAfterCooldown decides whether a re-notification can be sent for
// an existing alert based on the configured cooldown.
//
// Cooldown semantics:
//   - cooldown > 0: re-notify after that many minutes have passed since the
//     last notification.
//   - cooldown <= 0: cooldown is disabled, so only the first notification for
//     an alert occurrence is allowed. The alert evaluation loop runs every
//     metric tick, so treating disabled cooldown as "always allow" causes
//     repeated notifications for the same active alert.
//
// Level-escalation re-notifications are handled separately at the call site.
// Returns true if a notification should be sent, false otherwise.
func (m *Manager) shouldNotifyAfterCooldown(alert *Alert) bool {
	if alert.LastNotified == nil {
		return true
	}

	if m.config.Schedule.Cooldown <= 0 {
		return false
	}

	cooldownDuration := time.Duration(m.config.Schedule.Cooldown) * time.Minute
	timeSinceLastNotification := time.Since(*alert.LastNotified)

	return timeSinceLastNotification >= cooldownDuration
}

// DiagnoseAlertDelivery reports the current notification policy outcome for
// one active alert without sending a notification or updating any delivery
// tracking maps.
func (m *Manager) DiagnoseAlertDelivery(alertIdentifier string) (AlertDeliveryDiagnosis, bool) {
	diagnosis := AlertDeliveryDiagnosis{
		AlertIdentifier: strings.TrimSpace(alertIdentifier),
		Status:          AlertDeliveryStatusSuppressed,
		Reason:          "not_found",
	}
	if m == nil || diagnosis.AlertIdentifier == "" {
		return diagnosis, false
	}

	m.mu.Lock()
	defer m.mu.Unlock()

	alert, exists := m.getActiveAlertNoLock(diagnosis.AlertIdentifier)
	if !exists || alert == nil {
		return diagnosis, false
	}

	resolved := m.diagnoseActiveAlertLocked(alert)
	resolved.AlertIdentifier = diagnosis.AlertIdentifier
	return resolved, true
}

// DiagnoseActiveAlertDeliveries reports the delivery policy outcome for every
// active alert in one pass, with AlertIdentifier set to the canonical alert
// ID. Like DiagnoseAlertDelivery it is read-only: it sends nothing and
// advances no delivery tracking state.
func (m *Manager) DiagnoseActiveAlertDeliveries() []AlertDeliveryDiagnosis {
	if m == nil {
		return nil
	}

	m.mu.Lock()
	defer m.mu.Unlock()

	diagnoses := make([]AlertDeliveryDiagnosis, 0, len(m.activeAlerts))
	for _, alert := range m.activeAlerts {
		if alert == nil {
			continue
		}
		diagnosis := m.diagnoseActiveAlertLocked(alert)
		diagnosis.AlertIdentifier = alert.ID
		diagnoses = append(diagnoses, diagnosis)
	}
	sort.Slice(diagnoses, func(i, j int) bool {
		return diagnoses[i].AlertID < diagnoses[j].AlertID
	})
	return diagnoses
}

// diagnoseActiveAlertLocked computes the delivery diagnosis for one active
// alert. Caller MUST hold m.mu.
func (m *Manager) diagnoseActiveAlertLocked(alert *Alert) AlertDeliveryDiagnosis {
	var diagnosis AlertDeliveryDiagnosis
	now := m.policyNow()
	trackingKey := canonicalTrackingKeyForAlert(alert)
	diagnosis.AlertID = alert.ID
	diagnosis.TrackingKey = trackingKey
	diagnosis.AlertType = alert.Type
	diagnosis.Level = alert.Level
	diagnosis.ResourceID = alert.ResourceID
	diagnosis.ResourceName = alert.ResourceName
	diagnosis.Node = alert.Node
	diagnosis.NotificationsEnabled = m.config.Enabled && m.config.ActivationState == ActivationActive
	diagnosis.ActivationState = string(m.config.ActivationState)
	diagnosis.CooldownMinutes = m.config.Schedule.Cooldown
	diagnosis.MaxAlertsHour = m.config.Schedule.MaxAlertsHour
	diagnosis.FlappingThreshold = m.config.FlappingThreshold
	diagnosis.FlappingWindowSeconds = m.config.FlappingWindowSeconds
	if alert.LastNotified != nil {
		lastNotified := *alert.LastNotified
		diagnosis.LastNotified = &lastNotified
	}

	diagnosis.FlappingActive, diagnosis.FlappingHistoryInWindow, diagnosis.SuppressedUntil = m.flappingSnapshotLocked(trackingKey, now)
	diagnosis.RecentAlertsInHour = m.rateLimitCountLocked(trackingKey, now)

	switch {
	case m.awaitsBackupRevalidationNoLock(alert):
		diagnosis.setSuppressed(alertDeliveryReasonBackupRevalidation, "Restored backup alert is awaiting a successful backup evaluation.")
	case alert.Acknowledged:
		diagnosis.setSuppressed(AlertDeliveryReasonAcknowledged, "Alert is acknowledged, so firing notifications are suppressed.")
	case isSupportedInfrastructureSymptom(alert):
		diagnosis.setSuppressed(AlertDeliveryReasonCorrelatedPrimary, "A supported primary infrastructure alert owns notification delivery for this grouped symptom.")
	case func() bool {
		until, snoozed := alertSnoozeUntil(alert, now)
		if !snoozed {
			return false
		}
		diagnosis.SuppressedUntil = until
		diagnosis.setSuppressed(AlertDeliveryReasonSnoozed, "Alert is snoozed, so notifications and escalations are paused until the selected time.")
		return true
	}():
	case func() bool {
		suppressed, reason := m.operatorSuppressionForAlertNoLock(alert, now.UTC())
		if !suppressed {
			return false
		}
		diagnosis.setSuppressed(reason, "Alert notification is suppressed by the resource monitoring policy.")
		return true
	}():
	case isMonitorOnlyAlert(alert):
		diagnosis.setSuppressed(AlertDeliveryReasonMonitorOnly, "Alert is marked monitor-only, so it stays visible without notification delivery.")
	case !m.config.Enabled:
		diagnosis.setSuppressed(AlertDeliveryReasonNotificationsDisabled, "Alert notifications are disabled in the alert configuration.")
	case m.config.ActivationState != ActivationActive:
		diagnosis.setSuppressed(AlertDeliveryReasonNotificationsInactive, "Alert notifications are not active yet.")
	case diagnosis.SuppressedUntil != nil && diagnosis.SuppressedUntil.After(now):
		reason := AlertDeliveryReasonSuppressionWindow
		message := "Alert delivery is inside an active suppression window."
		if diagnosis.FlappingActive {
			reason = AlertDeliveryReasonFlapping
			message = "Alert delivery is suppressed because this alert is flapping."
		}
		diagnosis.setSuppressed(reason, message)
	case m.rateLimitExceededLocked(diagnosis.RecentAlertsInHour):
		diagnosis.setSuppressed(AlertDeliveryReasonRateLimited, "Alert delivery has reached the configured per-hour rate limit.")
	case alert.LastNotified != nil && !m.cooldownAllowsDeliveryLocked(alert, now):
		diagnosis.setSuppressed(AlertDeliveryReasonCooldown, "Alert delivery is waiting for the configured re-notification cooldown.")
		if next := m.nextCooldownEligibleAtLocked(alert); next != nil {
			diagnosis.NextEligibleAt = next
		}
	case func() bool {
		suppressed, reason := m.shouldSuppressNotification(alert)
		if !suppressed {
			return false
		}
		replayAt := m.quietHoursReplayAt()
		diagnosis.QuietHoursReplayAt = &replayAt
		if reason != "" {
			diagnosis.Reason = AlertDeliveryReasonQuietHours + ":" + reason
		} else {
			diagnosis.Reason = AlertDeliveryReasonQuietHours
		}
		diagnosis.Status = AlertDeliveryStatusDeferred
		diagnosis.Message = "Alert delivery is deferred by quiet-hours policy and will be replayed later."
		return true
	}():
	default:
		diagnosis.Status = AlertDeliveryStatusWouldSend
		diagnosis.Reason = AlertDeliveryReasonReady
		diagnosis.Message = "Alert delivery is currently eligible for notification delivery."
	}

	return diagnosis
}

func (d *AlertDeliveryDiagnosis) setSuppressed(reason, message string) {
	d.Status = AlertDeliveryStatusSuppressed
	d.Reason = reason
	d.Message = message
}

func (m *Manager) policyNow() time.Time {
	if m != nil && m.now != nil {
		return m.now()
	}
	return time.Now()
}

func (m *Manager) flappingSnapshotLocked(trackingKey string, now time.Time) (bool, int, *time.Time) {
	if trackingKey == "" || !m.config.FlappingEnabled {
		return false, 0, nil
	}

	windowDuration := time.Duration(m.config.FlappingWindowSeconds) * time.Second
	validCount := 0
	for _, t := range m.flappingHistory[trackingKey] {
		if now.Sub(t) <= windowDuration {
			validCount++
		}
	}

	var suppressedUntil *time.Time
	if until, ok := m.suppressedUntil[trackingKey]; ok {
		u := until
		suppressedUntil = &u
	}

	return m.flappingActive[trackingKey], validCount, suppressedUntil
}

func (m *Manager) rateLimitCountLocked(trackingKey string, now time.Time) int {
	if trackingKey == "" || m.config.Schedule.MaxAlertsHour <= 0 {
		return 0
	}
	cutoff := now.Add(-1 * time.Hour)
	recentCount := 0
	for _, t := range m.alertRateLimit[trackingKey] {
		if t.After(cutoff) {
			recentCount++
		}
	}
	return recentCount
}

func (m *Manager) rateLimitExceededLocked(recentCount int) bool {
	return m.config.Schedule.MaxAlertsHour > 0 && recentCount >= m.config.Schedule.MaxAlertsHour
}

func (m *Manager) cooldownAllowsDeliveryLocked(alert *Alert, now time.Time) bool {
	if alert == nil || alert.LastNotified == nil {
		return true
	}
	if m.config.Schedule.Cooldown <= 0 {
		return false
	}
	return !now.Before(alert.LastNotified.Add(time.Duration(m.config.Schedule.Cooldown) * time.Minute))
}

func (m *Manager) nextCooldownEligibleAtLocked(alert *Alert) *time.Time {
	if alert == nil || alert.LastNotified == nil || m.config.Schedule.Cooldown <= 0 {
		return nil
	}
	next := alert.LastNotified.Add(time.Duration(m.config.Schedule.Cooldown) * time.Minute)
	return &next
}

func (m *Manager) allowNotificationByRateLimit(trackingKey string, alert *Alert, reason string) bool {
	if trackingKey == "" && alert != nil {
		trackingKey = canonicalTrackingKeyForAlert(alert)
	}
	if trackingKey == "" && alert != nil {
		trackingKey = alert.ID
	}
	if m.checkRateLimit(trackingKey) {
		return true
	}

	log.Debug().
		Str("alertID", alertIDForLog(alert)).
		Str("trackingKey", trackingKey).
		Str("reason", reason).
		Int("maxPerHour", m.config.Schedule.MaxAlertsHour).
		Msg("Alert notification suppressed due to rate limit")
	return false
}

func alertIDForLog(alert *Alert) string {
	if alert == nil {
		return ""
	}
	return alert.ID
}

// checkRateLimit checks if an alert has exceeded rate limit
func (m *Manager) checkRateLimit(alertID string) bool {
	if m.config.Schedule.MaxAlertsHour <= 0 {
		return true // No rate limit
	}

	now := time.Now()
	cutoff := now.Add(-1 * time.Hour)

	// Clean old entries and count recent alerts
	var recentAlerts []time.Time
	if times, exists := m.alertRateLimit[alertID]; exists {
		for _, t := range times {
			if t.After(cutoff) {
				recentAlerts = append(recentAlerts, t)
			}
		}
	}

	// Check if we've hit the limit
	if len(recentAlerts) >= m.config.Schedule.MaxAlertsHour {
		return false
	}

	// Add current time
	recentAlerts = append(recentAlerts, now)
	m.alertRateLimit[alertID] = recentAlerts

	return true
}