Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
package alerts
import (
"sort"
"strings"
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/alerts/eventlog"
"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
"github.com/rs/zerolog/log"
)
const (
// MetadataQuietHoursSuppressed marks notifications that were held by alert
// quiet-hours policy and must be replayed by the notification queue.
MetadataQuietHoursSuppressed = "quietHoursSuppressed"
// MetadataQuietHoursSuppressionReason records the quiet-hours rule that
// deferred notification delivery.
MetadataQuietHoursSuppressionReason = "quietHoursSuppressionReason"
// MetadataQuietHoursReplayAt stores the earliest RFC3339 UTC time that the
// notification queue may retry delivery.
MetadataQuietHoursReplayAt = "quietHoursReplayAt"
)
const (
AlertDeliveryStatusWouldSend = "would_send"
AlertDeliveryStatusDeferred = "deferred"
AlertDeliveryStatusSuppressed = "suppressed"
AlertDeliveryReasonReady = "ready"
AlertDeliveryReasonAcknowledged = "acknowledged"
AlertDeliveryReasonSnoozed = "snoozed"
AlertDeliveryReasonNotificationsDisabled = "notifications_disabled"
AlertDeliveryReasonNotificationsInactive = "notifications_inactive"
AlertDeliveryReasonSuppressionWindow = "suppression_window"
AlertDeliveryReasonFlapping = "flapping"
AlertDeliveryReasonRateLimited = "rate_limited"
AlertDeliveryReasonCooldown = "cooldown"
AlertDeliveryReasonQuietHours = "quiet_hours"
AlertDeliveryReasonMonitorOnly = "monitor_only"
AlertDeliveryReasonCorrelatedPrimary = "correlated_primary"
)
// AlertDeliveryDiagnosis is a read-only projection of the alert manager's
// current delivery policy for one active alert. It is intentionally
// non-mutating: it does not dispatch callbacks or advance flapping/rate-limit
// counters.
type AlertDeliveryDiagnosis struct {
AlertIdentifier string `json:"alertIdentifier"`
AlertID string `json:"alertId"`
TrackingKey string `json:"trackingKey"`
Status string `json:"status"`
Reason string `json:"reason"`
Message string `json:"message"`
AlertType string `json:"alertType"`
Level AlertLevel `json:"level"`
ResourceID string `json:"resourceId,omitempty"`
ResourceName string `json:"resourceName,omitempty"`
Node string `json:"node,omitempty"`
NotificationsEnabled bool `json:"notificationsEnabled"`
ActivationState string `json:"activationState"`
CooldownMinutes int `json:"cooldownMinutes"`
MaxAlertsHour int `json:"maxAlertsHour"`
RecentAlertsInHour int `json:"recentAlertsInHour"`
FlappingActive bool `json:"flappingActive"`
FlappingHistoryInWindow int `json:"flappingHistoryInWindow"`
FlappingThreshold int `json:"flappingThreshold"`
FlappingWindowSeconds int `json:"flappingWindowSeconds"`
LastNotified *time.Time `json:"lastNotified,omitempty"`
NextEligibleAt *time.Time `json:"nextEligibleAt,omitempty"`
QuietHoursReplayAt *time.Time `json:"quietHoursReplayAt,omitempty"`
SuppressedUntil *time.Time `json:"suppressedUntil,omitempty"`
}
// checkFlappingLocked detects alert flapping and returns whether the alert
// should be suppressed and whether this call is the first transition into
// the flapping state for the current cooldown window. justTransitioned is
// the signal callers use to fire a one-shot postmortem hook on first
// detection -- it is true only on the call that flips
// flappingActive[trackingKey] from false to true.
//
// It modifies flappingHistory, flappingActive, and suppressedUntil maps.
// IMPORTANT: Caller MUST hold m.mu before calling this function. The
// transition signal is returned so the caller can dispatch the
// flapping-detected callback OUTSIDE the lock (typically via a goroutine).
func (m *Manager) checkFlappingLocked(trackingKey string) (suppress bool, justTransitioned bool) {
if !m.config.FlappingEnabled {
return false, false
}
now := time.Now()
windowDuration := time.Duration(m.config.FlappingWindowSeconds) * time.Second
// Honour an active cooldown before anything else. The cooldown recorded
// below used to be write-only on this path: suppression was decided purely
// by the sliding-window count, so it ended the moment the count dipped
// under the threshold instead of holding for FlappingCooldownMinutes. A
// resource oscillating just under the threshold rate was therefore never
// damped at all, while the flapping postmortem told the operator that
// notifications had been suppressed for the configured cooldown and
// suggested raising a value that changed nothing.
if until, cooling := m.suppressedUntil[trackingKey]; cooling {
if now.Before(until) {
return true, false
}
m.expireSuppressionNoLock(trackingKey, now)
}
// Record this state change
m.flappingHistory[trackingKey] = append(m.flappingHistory[trackingKey], now)
// Remove state changes outside the window
history := m.flappingHistory[trackingKey]
validHistory := []time.Time{}
for _, t := range history {
if now.Sub(t) <= windowDuration {
validHistory = append(validHistory, t)
}
}
// Keep enough observations to reach the configured threshold. A fixed
// ten-entry cap silently disabled flapping suppression for accepted
// thresholds above ten. Retain the old diagnostic allowance for smaller
// thresholds, while still bounding history by configuration rather than
// the number of dispatch attempts.
maxFlappingHistory := max(10, m.config.FlappingThreshold)
if len(validHistory) > maxFlappingHistory {
validHistory = validHistory[len(validHistory)-maxFlappingHistory:]
}
m.flappingHistory[trackingKey] = validHistory
// Check if we've exceeded the threshold
if len(validHistory) >= m.config.FlappingThreshold {
// Mark as flapping
if !m.flappingActive[trackingKey] {
log.Warn().
Str("trackingKey", trackingKey).
Int("stateChanges", len(validHistory)).
Int("threshold", m.config.FlappingThreshold).
Int("windowSeconds", m.config.FlappingWindowSeconds).
Msg("Flapping detected - suppressing alert")
m.flappingActive[trackingKey] = true
// Set cooldown period
cooldownDuration := time.Duration(m.config.FlappingCooldownMinutes) * time.Minute
m.suppressedUntil[trackingKey] = now.Add(cooldownDuration)
// Record suppression metric
if recordAlertSuppressed != nil {
recordAlertSuppressed("flapping")
}
return true, true
}
return true, false
}
return false, false
}
// expireSuppressionNoLock retires a served suppression and its episode together.
// Cleanup must use the same reset as dispatch: deleting only the deadline leaves
// a latched flapping episode that can suppress delivery without ever rearming a
// cooldown or notifying the flapping callback. Caller MUST hold m.mu.
func (m *Manager) expireSuppressionNoLock(trackingKey string, now time.Time) bool {
until, exists := m.suppressedUntil[trackingKey]
if !exists || now.Before(until) {
return false
}
delete(m.suppressedUntil, trackingKey)
delete(m.flappingActive, trackingKey)
delete(m.flappingHistory, trackingKey)
return true
}
const alertDeliveryReasonBackupRevalidation = "awaiting_backup_evaluation"
// Caller holds mu. The occurrence check prevents a deferred startup snapshot
// from suppressing a later alert that happens to reuse the same identity.
func (m *Manager) awaitsBackupRevalidationNoLock(alert *Alert) bool {
if alert == nil || alert.Type != "backup-age" {
return false
}
startedAt, pending := m.restoredBackupNotifications[canonicalTrackingKeyForAlert(alert)]
return pending && startedAt.Equal(alert.StartTime)
}
// A valid, matched canonical evaluation releases the startup delivery hold.
// Recovery/removal retires the hold through the active lifecycle owner instead.
func (m *Manager) revalidateBackupNotificationNoLock(alert *Alert) bool {
if !m.awaitsBackupRevalidationNoLock(alert) {
return false
}
delete(m.restoredBackupNotifications, canonicalTrackingKeyForAlert(alert))
return true
}
func (m *Manager) dispatchAlert(alert *Alert, async bool) bool {
callbacks := m.getAlertCallbacks()
if len(callbacks) == 0 || alert == nil {
return false
}
if m.awaitsBackupRevalidationNoLock(alert) {
m.recordAlertEvent(eventlog.TypeNotificationSuppressed, alert, "",
alertDeliveryReasonBackupRevalidation,
"Restored backup alert is awaiting a successful backup evaluation.", nil)
return false
}
// Don't dispatch notifications for acknowledged alerts
if alert.Acknowledged {
log.Debug().
Str("alertID", alert.ID).
Str("ackUser", alert.AckUser).
Msg("Alert notification suppressed - already acknowledged")
m.recordAlertEvent(eventlog.TypeNotificationSuppressed, alert, "",
AlertDeliveryReasonAcknowledged, "Notification suppressed: alert is acknowledged.", nil)
return false
}
if until, snoozed := alertSnoozeUntil(alert, m.policyNow()); snoozed {
details := map[string]string(nil)
if until != nil {
details = map[string]string{"until": until.Format(time.RFC3339Nano)}
}
m.recordAlertEvent(eventlog.TypeNotificationSuppressed, alert, "",
AlertDeliveryReasonSnoozed, "Notification suppressed: alert is snoozed.",
details)
return false
}
if isSupportedInfrastructureSymptom(alert) {
m.recordAlertEvent(eventlog.TypeNotificationSuppressed, alert, "",
AlertDeliveryReasonCorrelatedPrimary,
"Notification suppressed: a supported primary infrastructure alert owns delivery for this incident group.",
map[string]string{"primaryAlertId": alert.Correlation.PrimaryAlertID})
return false
}
// Resource monitoring policy is a terminal suppression, unlike quiet hours.
// An alert already active when the policy changes must not be queued for
// replay or reach a callback before reconciliation clears it.
if suppressed, reason := m.operatorSuppressionForAlertNoLock(alert, m.policyNow().UTC()); suppressed {
clearQuietHoursNotificationReplay(alert)
m.recordAlertEvent(eventlog.TypeNotificationSuppressed, alert, "", reason,
"Notification suppressed by resource monitoring policy.", nil)
return false
}
if isMonitorOnlyAlert(alert) {
clearQuietHoursNotificationReplay(alert)
m.recordAlertEvent(eventlog.TypeNotificationSuppressed, alert, "",
AlertDeliveryReasonMonitorOnly, "Notification suppressed: alert is monitor-only.", nil)
return false
}
trackingKey := canonicalTrackingKeyForAlert(alert)
// Check for flapping (caller must hold m.mu). When this call is the
// first transition into the flapping state for the current cooldown
// window, fire the flapping-detected callback so a postmortem patrol
// can be triggered. The callback runs in its own goroutine because
// the caller of dispatchAlert holds m.mu and the callback is allowed
// to take its own locks or re-enter the alerts package.
suppress, justTransitioned := m.checkFlappingLocked(trackingKey)
if justTransitioned {
m.recordAlertEvent(eventlog.TypeFlappingDetected, alert, "",
AlertDeliveryReasonFlapping, "Alert entered flapping suppression.",
map[string]string{"trackingKey": trackingKey})
cb := m.callbacks.flappingDetectedCallback()
if cb != nil {
alertCopy := cloneAlertForOutput(alert)
go func(a *Alert, key string, fn func(*Alert, string)) {
defer func() {
if r := recover(); r != nil {
log.Error().
Interface("panic", r).
Str("alertID", a.ID).
Str("trackingKey", key).
Msg("Panic in onFlappingDetected callback")
}
}()
fn(a, key)
}(alertCopy, trackingKey, cb)
}
}
if suppress {
log.Debug().
Str("alertID", alert.ID).
Str("trackingKey", trackingKey).
Msg("Alert suppressed due to flapping")
m.recordAlertEvent(eventlog.TypeNotificationSuppressed, alert, "",
AlertDeliveryReasonFlapping, "Notification suppressed: alert is flapping.",
map[string]string{"trackingKey": trackingKey})
return false
}
// Check activation state - only dispatch notifications if active
if m.config.ActivationState != ActivationActive {
log.Debug().
Str("alertID", alert.ID).
Str("activationState", string(m.config.ActivationState)).
Msg("Alert notification suppressed - not activated")
m.recordAlertEvent(eventlog.TypeNotificationSuppressed, alert, "",
AlertDeliveryReasonNotificationsInactive, "Notification suppressed: alert delivery is not turned on.",
map[string]string{"activationState": string(m.config.ActivationState)})
return false
}
if suppressed, reason := m.shouldSuppressNotification(alert); suppressed {
replayAt := m.quietHoursReplayAt()
markQuietHoursNotificationReplay(alert, reason, replayAt)
log.Debug().
Str("alertID", alert.ID).
Str("type", alert.Type).
Str("level", string(alert.Level)).
Str("quietHoursRule", reason).
Time("replayAt", replayAt).
Msg("Alert notification deferred during quiet hours")
deferredReason := AlertDeliveryReasonQuietHours
if reason != "" {
deferredReason += ":" + reason
}
m.recordAlertEvent(eventlog.TypeNotificationDeferred, alert, "",
deferredReason, "Notification deferred by quiet hours; the queue replays it when they end.",
map[string]string{"replayAt": replayAt.UTC().Format(time.RFC3339)})
} else {
clearQuietHoursNotificationReplay(alert)
}
// Record metric for fired alert
if recordAlertFired != nil {
recordAlertFired(alert)
}
notifiedAt := time.Now()
m.recordAlertNotifiedNoLock(alert, notifiedAt)
m.saveActiveAlertsAsync("alert-dispatch")
m.recordAlertEvent(eventlog.TypeNotificationDispatched, alert, "",
AlertDeliveryReasonReady, "Notification dispatched to the delivery pipeline.", nil)
alertCopy := cloneAlertForOutput(alert)
if async {
go func(a *Alert, fns []func(*Alert)) {
defer func() {
if r := recover(); r != nil {
log.Error().
Interface("panic", r).
Str("alertID", a.ID).
Str("type", a.Type).
Msg("Panic in onAlert callback")
}
}()
for _, callback := range fns {
callback(a)
}
}(alertCopy, callbacks)
} else {
// Synchronous calls also need panic recovery to prevent service crash
func(fns []func(*Alert)) {
defer func() {
if r := recover(); r != nil {
log.Error().
Interface("panic", r).
Str("alertID", alertCopy.ID).
Str("type", alertCopy.Type).
Msg("Panic in onAlert callback (synchronous)")
}
}()
for _, callback := range fns {
callback(alertCopy)
}
}(callbacks)
}
return true
}
func (m *Manager) recordAlertNotifiedNoLock(alert *Alert, notifiedAt time.Time) {
if alert == nil {
return
}
setAlertLastNotified(alert, notifiedAt)
if trackingKey := canonicalTrackingKeyForAlert(alert); trackingKey != "" {
if active, exists := m.getActiveAlertNoLock(trackingKey); exists && active != nil {
setAlertLastNotified(active, notifiedAt)
return
}
}
if alert.ID != "" {
if active, exists := m.getActiveAlertNoLock(alert.ID); exists && active != nil {
setAlertLastNotified(active, notifiedAt)
}
}
}
func setAlertLastNotified(alert *Alert, notifiedAt time.Time) {
if alert == nil {
return
}
t := notifiedAt
alert.LastNotified = &t
}
func isMonitorOnlyAlert(alert *Alert) bool {
if alert == nil || alert.Metadata == nil {
return false
}
if value, ok := alert.Metadata["monitorOnly"]; ok {
switch v := value.(type) {
case bool:
return v
case string:
return strings.EqualFold(v, "true")
}
}
return false
}
func isSupportedInfrastructureSymptom(alert *Alert) bool {
return alert != nil &&
alert.Correlation != nil &&
alert.Correlation.Kind == AlertCorrelationKindInfrastructureIncident &&
alert.Correlation.Role == AlertCorrelationRoleSupporting &&
alert.Correlation.Inference == AlertCorrelationInferenceSupportedCause &&
strings.TrimSpace(alert.Correlation.PrimaryAlertID) != ""
}
// quietHoursClock compares civil clock minutes, not date-constructed instants.
// time.Date chooses only one side of a repeated hour and normalizes missing
// minutes to a different hour, neither of which is the user's daily schedule.
type quietHoursClock struct {
location *time.Location
start int
end int
days [7]bool
}
func (m *Manager) quietHoursClock() (quietHoursClock, bool) {
quiet := m.config.Schedule.QuietHours
if !quiet.Enabled {
return quietHoursClock{}, false
}
loc := m.quietHoursLoc
if loc == nil {
var err error
loc, err = time.LoadLocation(quiet.Timezone)
if err != nil {
log.Warn().Err(err).Str("timezone", quiet.Timezone).Msg("failed to load timezone, using local time")
loc = time.Local
}
// Do not populate the cache here: public suppression helpers can call
// this path while holding only a read lock. UpdateConfig owns the cache.
}
start, err := time.Parse("15:04", quiet.Start)
if err != nil {
log.Warn().Err(err).Str("start", quiet.Start).Msg("failed to parse quiet hours start time")
return quietHoursClock{}, false
}
end, err := time.Parse("15:04", quiet.End)
if err != nil {
log.Warn().Err(err).Str("end", quiet.End).Msg("failed to parse quiet hours end time")
return quietHoursClock{}, false
}
clock := quietHoursClock{location: loc, start: start.Hour()*60 + start.Minute(), end: end.Hour()*60 + end.Minute()}
for i, day := range [...]string{"sunday", "monday", "tuesday", "wednesday", "thursday", "friday", "saturday"} {
clock.days[i] = quiet.Days[day]
}
return clock, true
}
func (clock quietHoursClock) contains(at time.Time) bool {
local := at.In(clock.location)
if !clock.days[local.Weekday()] {
return false
}
minute := local.Hour()*60 + local.Minute()
if clock.start > clock.end {
return minute >= clock.start || minute <= clock.end
}
return minute >= clock.start && minute <= clock.end
}
// isInQuietHours checks the selected local calendar day and includes the whole
// configured end minute, including both copies of it during a clock rollback.
func (m *Manager) isInQuietHours() bool {
clock, valid := m.quietHoursClock()
return valid && clock.contains(m.policyNow())
}
func quietHoursCategoryForAlert(alert *Alert) string {
if alert == nil {
return ""
}
switch alert.Type {
case "cpu", "memory", "disk", "diskRead", "diskWrite", "networkIn", "networkOut", "temperature":
return "performance"
case "queue-depth", "queue-deferred", "queue-hold", "message-age",
"docker-container-health", "docker-container-restart-loop",
"docker-container-oom-kill", "docker-container-memory-limit":
return "performance"
case "usage", "disk-health", "disk-wearout", "zfs-pool-state", "zfs-pool-errors", "zfs-device", "storage-incident", "backup-storage-incident", "backup-posture-incident":
return "storage"
case "resource-incident":
if metadataStringValue(alert.Metadata, "incidentCategory") == unifiedresources.IncidentCategoryAvailability {
return "offline"
}
return "performance"
case "connectivity", "offline", "powered-off", "docker-host-offline", connectionDegradedAlertType:
return "offline"
}
if strings.HasPrefix(alert.Type, "docker-container-") {
if alert.Type == "docker-container-state" {
return "offline"
}
return "performance"
}
return ""
}
func (m *Manager) shouldSuppressNotification(alert *Alert) (bool, string) {
// Only quiet-hours matches are replayable. Operator policy is checked by
// each caller before this helper, because it must drop rather than defer.
if !m.isInQuietHours() {
return false, ""
}
return quietHoursSuppressionForAlert(alert, m.config.Schedule.QuietHours.Suppress)
}
func quietHoursSuppressionForAlert(alert *Alert, suppress QuietHoursSuppression) (bool, string) {
if alert == nil {
return false, ""
}
if alert.Level != AlertLevelCritical {
return true, "non-critical"
}
category := quietHoursCategoryForAlert(alert)
switch category {
case "performance":
if suppress.Performance {
return true, category
}
case "storage":
if suppress.Storage {
return true, category
}
case "offline":
if suppress.Offline {
return true, category
}
}
return false, ""
}
func (m *Manager) quietHoursReplayAt() time.Time {
now := m.policyNow()
clock, valid := m.quietHoursClock()
if !valid {
return now.Add(time.Minute).UTC()
}
return clock.replayAt(now)
}
// QuietHoursNotificationPolicy returns an immutable snapshot of the current
// quiet-hours rule. The delivery owner obtains it before taking queue or
// per-alert delivery locks, so evaluation never calls back into this manager
// while cancellation is waiting for those locks. It does not mutate alert
// lifecycle, acknowledgement, metadata or destination selection.
func (m *Manager) QuietHoursNotificationPolicy() func(*Alert, time.Time) *time.Time {
m.mu.RLock()
clock, valid := m.quietHoursClock()
suppress := m.config.Schedule.QuietHours.Suppress
m.mu.RUnlock()
return func(alert *Alert, now time.Time) *time.Time {
if !valid || !clock.contains(now) {
return nil
}
if suppressed, _ := quietHoursSuppressionForAlert(alert, suppress); !suppressed {
return nil
}
replayAt := clock.replayAt(now)
return &replayAt
}
}
func (clock quietHoursClock) replayAt(now time.Time) time.Time {
if !clock.contains(now) {
return now.Add(time.Minute).UTC()
}
// A full-day window has no non-quiet clock minute. Preserve its existing
// daily revalidation boundary. The queue must check policy again there,
// not invent an end to a continuous schedule or attempt a provider send.
if (clock.end-clock.start+24*60)%(24*60) == 24*60-1 {
local := now.In(clock.location)
endExclusive := time.Date(local.Year(), local.Month(), local.Day(), clock.end/60, clock.end%60, 0, 0, clock.location).Add(time.Minute)
if clock.start > clock.end && local.Hour()*60+local.Minute() >= clock.start {
endExclusive = endExclusive.AddDate(0, 0, 1)
}
if endExclusive.After(now) {
return endExclusive.UTC()
}
return now.Add(time.Minute).UTC()
}
// Walk real minutes through the current window, rather than constructing
// its end on a local date. This handles repeated ends, skipped ends, half-
// hour DST transitions and a midnight leading into an unselected day. A
// non-full-day window has an exit within three real days even when a
// forward clock change skips its entire gap on the first day.
candidate := now.Truncate(time.Minute).Add(time.Minute)
limit := candidate.Add(72 * time.Hour)
for candidate.Before(limit) {
if !clock.contains(candidate) {
return candidate.UTC()
}
candidate = candidate.Add(time.Minute)
}
log.Warn().Msg("could not find quiet-hours window end within three days")
return now.Add(time.Minute).UTC()
}
func markQuietHoursNotificationReplay(alert *Alert, reason string, replayAt time.Time) {
if alert == nil {
return
}
if alert.Metadata == nil {
alert.Metadata = make(map[string]interface{}, 3)
}
alert.Metadata[MetadataQuietHoursSuppressed] = true
alert.Metadata[MetadataQuietHoursSuppressionReason] = reason
alert.Metadata[MetadataQuietHoursReplayAt] = replayAt.UTC().Format(time.RFC3339)
}
func clearQuietHoursNotificationReplay(alert *Alert) {
if alert == nil || alert.Metadata == nil {
return
}
delete(alert.Metadata, MetadataQuietHoursSuppressed)
delete(alert.Metadata, MetadataQuietHoursSuppressionReason)
delete(alert.Metadata, MetadataQuietHoursReplayAt)
}
func hasQuietHoursNotificationReplay(alert *Alert) bool {
if alert == nil || alert.Metadata == nil {
return false
}
if replayAt, ok := alert.Metadata[MetadataQuietHoursReplayAt].(string); ok && strings.TrimSpace(replayAt) != "" {
return true
}
if suppressed, ok := alert.Metadata[MetadataQuietHoursSuppressed].(bool); ok {
return suppressed
}
return false
}
// ShouldSuppressNotification checks whether a notification must be dropped at
// the alert-policy layer. Quiet-hours matches are replayable, so this helper
// annotates the alert for notification-queue retry and returns false to let
// bypass callers hand the alert to the delivery owner.
func (m *Manager) ShouldSuppressNotification(alert *Alert) bool {
if alert == nil {
return false
}
m.mu.RLock()
defer m.mu.RUnlock()
if m.awaitsBackupRevalidationNoLock(alert) {
return true
}
if _, snoozed := alertSnoozeUntil(alert, m.policyNow()); snoozed {
clearQuietHoursNotificationReplay(alert)
return true
}
if isSupportedInfrastructureSymptom(alert) {
clearQuietHoursNotificationReplay(alert)
return true
}
if suppressed, reason := m.operatorSuppressionForAlertNoLock(alert, time.Now().UTC()); suppressed {
clearQuietHoursNotificationReplay(alert)
log.Debug().
Str("alertID", alert.ID).
Str("type", alert.Type).
Str("operatorPolicy", reason).
Msg("Notification suppressed by resource monitoring policy")
return true
}
if isMonitorOnlyAlert(alert) {
clearQuietHoursNotificationReplay(alert)
return true
}
suppressed, reason := m.shouldSuppressNotification(alert)
if suppressed {
replayAt := m.quietHoursReplayAt()
markQuietHoursNotificationReplay(alert, reason, replayAt)
log.Debug().
Str("alertID", alert.ID).
Str("type", alert.Type).
Str("level", string(alert.Level)).
Str("quietHoursRule", reason).
Time("replayAt", replayAt).
Msg("Notification deferred during quiet hours")
return false
}
clearQuietHoursNotificationReplay(alert)
return false
}
// ShouldSuppressResolvedNotification checks if a recovery notification should be suppressed.
// Recovery notifications keep the existing quiet-hours rule unless the firing
// notification was already deferred into the owned queue replay path; in that
// case the resolved notification is also allowed to enter the queue so the
// close-the-loop delivery is not lost.
func (m *Manager) ShouldSuppressResolvedNotification(alert *Alert) bool {
if alert == nil {
return false
}
m.mu.RLock()
defer m.mu.RUnlock()
if alert.Acknowledged {
log.Debug().
Str("alertID", alert.ID).
Str("type", alert.Type).
Msg("Recovery notification suppressed for acknowledged alert")
return true
}
if _, snoozed := alertSnoozeUntil(alert, m.policyNow()); snoozed {
clearQuietHoursNotificationReplay(alert)
log.Debug().Str("alertID", alert.ID).Msg("Recovery notification suppressed for snoozed alert")
return true
}
if suppressed, reason := m.operatorSuppressionForAlertNoLock(alert, time.Now().UTC()); suppressed {
clearQuietHoursNotificationReplay(alert)
log.Debug().
Str("alertID", alert.ID).
Str("type", alert.Type).
Str("operatorPolicy", reason).
Msg("Recovery notification suppressed by resource monitoring policy")
return true
}
if isMonitorOnlyAlert(alert) {
clearQuietHoursNotificationReplay(alert)
return true
}
quietHoursReplay := hasQuietHoursNotificationReplay(alert)
if alert.LastNotified == nil && !quietHoursReplay {
log.Debug().
Str("alertID", alert.ID).
Str("type", alert.Type).
Msg("Recovery notification suppressed because firing notification was never sent")
return true
}
suppressed, reason := m.shouldSuppressNotification(alert)
if suppressed {
if quietHoursReplay {
log.Debug().
Str("alertID", alert.ID).
Str("type", alert.Type).
Str("level", string(alert.Level)).
Str("quietHoursRule", reason).
Msg("Recovery notification allowed into quiet-hours replay queue")
return false
}
log.Debug().
Str("alertID", alert.ID).
Str("type", alert.Type).
Str("level", string(alert.Level)).
Str("quietHoursRule", reason).
Msg("Recovery notification suppressed during quiet hours")
}
return suppressed
}
// shouldNotifyAfterCooldown decides whether a re-notification can be sent for
// an existing alert based on the configured cooldown.
//
// Cooldown semantics:
// - cooldown > 0: re-notify after that many minutes have passed since the
// last notification.
// - cooldown <= 0: cooldown is disabled, so only the first notification for
// an alert occurrence is allowed. The alert evaluation loop runs every
// metric tick, so treating disabled cooldown as "always allow" causes
// repeated notifications for the same active alert.
//
// Level-escalation re-notifications are handled separately at the call site.
// Returns true if a notification should be sent, false otherwise.
func (m *Manager) shouldNotifyAfterCooldown(alert *Alert) bool {
if alert.LastNotified == nil {
return true
}
if m.config.Schedule.Cooldown <= 0 {
return false
}
cooldownDuration := time.Duration(m.config.Schedule.Cooldown) * time.Minute
timeSinceLastNotification := time.Since(*alert.LastNotified)
return timeSinceLastNotification >= cooldownDuration
}
// DiagnoseAlertDelivery reports the current notification policy outcome for
// one active alert without sending a notification or updating any delivery
// tracking maps.
func (m *Manager) DiagnoseAlertDelivery(alertIdentifier string) (AlertDeliveryDiagnosis, bool) {
diagnosis := AlertDeliveryDiagnosis{
AlertIdentifier: strings.TrimSpace(alertIdentifier),
Status: AlertDeliveryStatusSuppressed,
Reason: "not_found",
}
if m == nil || diagnosis.AlertIdentifier == "" {
return diagnosis, false
}
m.mu.Lock()
defer m.mu.Unlock()
alert, exists := m.getActiveAlertNoLock(diagnosis.AlertIdentifier)
if !exists || alert == nil {
return diagnosis, false
}
resolved := m.diagnoseActiveAlertLocked(alert)
resolved.AlertIdentifier = diagnosis.AlertIdentifier
return resolved, true
}
// DiagnoseActiveAlertDeliveries reports the delivery policy outcome for every
// active alert in one pass, with AlertIdentifier set to the canonical alert
// ID. Like DiagnoseAlertDelivery it is read-only: it sends nothing and
// advances no delivery tracking state.
func (m *Manager) DiagnoseActiveAlertDeliveries() []AlertDeliveryDiagnosis {
if m == nil {
return nil
}
m.mu.Lock()
defer m.mu.Unlock()
diagnoses := make([]AlertDeliveryDiagnosis, 0, len(m.activeAlerts))
for _, alert := range m.activeAlerts {
if alert == nil {
continue
}
diagnosis := m.diagnoseActiveAlertLocked(alert)
diagnosis.AlertIdentifier = alert.ID
diagnoses = append(diagnoses, diagnosis)
}
sort.Slice(diagnoses, func(i, j int) bool {
return diagnoses[i].AlertID < diagnoses[j].AlertID
})
return diagnoses
}
// diagnoseActiveAlertLocked computes the delivery diagnosis for one active
// alert. Caller MUST hold m.mu.
func (m *Manager) diagnoseActiveAlertLocked(alert *Alert) AlertDeliveryDiagnosis {
var diagnosis AlertDeliveryDiagnosis
now := m.policyNow()
trackingKey := canonicalTrackingKeyForAlert(alert)
diagnosis.AlertID = alert.ID
diagnosis.TrackingKey = trackingKey
diagnosis.AlertType = alert.Type
diagnosis.Level = alert.Level
diagnosis.ResourceID = alert.ResourceID
diagnosis.ResourceName = alert.ResourceName
diagnosis.Node = alert.Node
diagnosis.NotificationsEnabled = m.config.Enabled && m.config.ActivationState == ActivationActive
diagnosis.ActivationState = string(m.config.ActivationState)
diagnosis.CooldownMinutes = m.config.Schedule.Cooldown
diagnosis.MaxAlertsHour = m.config.Schedule.MaxAlertsHour
diagnosis.FlappingThreshold = m.config.FlappingThreshold
diagnosis.FlappingWindowSeconds = m.config.FlappingWindowSeconds
if alert.LastNotified != nil {
lastNotified := *alert.LastNotified
diagnosis.LastNotified = &lastNotified
}
diagnosis.FlappingActive, diagnosis.FlappingHistoryInWindow, diagnosis.SuppressedUntil = m.flappingSnapshotLocked(trackingKey, now)
diagnosis.RecentAlertsInHour = m.rateLimitCountLocked(trackingKey, now)
switch {
case m.awaitsBackupRevalidationNoLock(alert):
diagnosis.setSuppressed(alertDeliveryReasonBackupRevalidation, "Restored backup alert is awaiting a successful backup evaluation.")
case alert.Acknowledged:
diagnosis.setSuppressed(AlertDeliveryReasonAcknowledged, "Alert is acknowledged, so firing notifications are suppressed.")
case isSupportedInfrastructureSymptom(alert):
diagnosis.setSuppressed(AlertDeliveryReasonCorrelatedPrimary, "A supported primary infrastructure alert owns notification delivery for this grouped symptom.")
case func() bool {
until, snoozed := alertSnoozeUntil(alert, now)
if !snoozed {
return false
}
diagnosis.SuppressedUntil = until
diagnosis.setSuppressed(AlertDeliveryReasonSnoozed, "Alert is snoozed, so notifications and escalations are paused until the selected time.")
return true
}():
case func() bool {
suppressed, reason := m.operatorSuppressionForAlertNoLock(alert, now.UTC())
if !suppressed {
return false
}
diagnosis.setSuppressed(reason, "Alert notification is suppressed by the resource monitoring policy.")
return true
}():
case isMonitorOnlyAlert(alert):
diagnosis.setSuppressed(AlertDeliveryReasonMonitorOnly, "Alert is marked monitor-only, so it stays visible without notification delivery.")
case !m.config.Enabled:
diagnosis.setSuppressed(AlertDeliveryReasonNotificationsDisabled, "Alert notifications are disabled in the alert configuration.")
case m.config.ActivationState != ActivationActive:
diagnosis.setSuppressed(AlertDeliveryReasonNotificationsInactive, "Alert notifications are not active yet.")
case diagnosis.SuppressedUntil != nil && diagnosis.SuppressedUntil.After(now):
reason := AlertDeliveryReasonSuppressionWindow
message := "Alert delivery is inside an active suppression window."
if diagnosis.FlappingActive {
reason = AlertDeliveryReasonFlapping
message = "Alert delivery is suppressed because this alert is flapping."
}
diagnosis.setSuppressed(reason, message)
case m.rateLimitExceededLocked(diagnosis.RecentAlertsInHour):
diagnosis.setSuppressed(AlertDeliveryReasonRateLimited, "Alert delivery has reached the configured per-hour rate limit.")
case alert.LastNotified != nil && !m.cooldownAllowsDeliveryLocked(alert, now):
diagnosis.setSuppressed(AlertDeliveryReasonCooldown, "Alert delivery is waiting for the configured re-notification cooldown.")
if next := m.nextCooldownEligibleAtLocked(alert); next != nil {
diagnosis.NextEligibleAt = next
}
case func() bool {
suppressed, reason := m.shouldSuppressNotification(alert)
if !suppressed {
return false
}
replayAt := m.quietHoursReplayAt()
diagnosis.QuietHoursReplayAt = &replayAt
if reason != "" {
diagnosis.Reason = AlertDeliveryReasonQuietHours + ":" + reason
} else {
diagnosis.Reason = AlertDeliveryReasonQuietHours
}
diagnosis.Status = AlertDeliveryStatusDeferred
diagnosis.Message = "Alert delivery is deferred by quiet-hours policy and will be replayed later."
return true
}():
default:
diagnosis.Status = AlertDeliveryStatusWouldSend
diagnosis.Reason = AlertDeliveryReasonReady
diagnosis.Message = "Alert delivery is currently eligible for notification delivery."
}
return diagnosis
}
func (d *AlertDeliveryDiagnosis) setSuppressed(reason, message string) {
d.Status = AlertDeliveryStatusSuppressed
d.Reason = reason
d.Message = message
}
func (m *Manager) policyNow() time.Time {
if m != nil && m.now != nil {
return m.now()
}
return time.Now()
}
func (m *Manager) flappingSnapshotLocked(trackingKey string, now time.Time) (bool, int, *time.Time) {
if trackingKey == "" || !m.config.FlappingEnabled {
return false, 0, nil
}
windowDuration := time.Duration(m.config.FlappingWindowSeconds) * time.Second
validCount := 0
for _, t := range m.flappingHistory[trackingKey] {
if now.Sub(t) <= windowDuration {
validCount++
}
}
var suppressedUntil *time.Time
if until, ok := m.suppressedUntil[trackingKey]; ok {
u := until
suppressedUntil = &u
}
return m.flappingActive[trackingKey], validCount, suppressedUntil
}
func (m *Manager) rateLimitCountLocked(trackingKey string, now time.Time) int {
if trackingKey == "" || m.config.Schedule.MaxAlertsHour <= 0 {
return 0
}
cutoff := now.Add(-1 * time.Hour)
recentCount := 0
for _, t := range m.alertRateLimit[trackingKey] {
if t.After(cutoff) {
recentCount++
}
}
return recentCount
}
func (m *Manager) rateLimitExceededLocked(recentCount int) bool {
return m.config.Schedule.MaxAlertsHour > 0 && recentCount >= m.config.Schedule.MaxAlertsHour
}
func (m *Manager) cooldownAllowsDeliveryLocked(alert *Alert, now time.Time) bool {
if alert == nil || alert.LastNotified == nil {
return true
}
if m.config.Schedule.Cooldown <= 0 {
return false
}
return !now.Before(alert.LastNotified.Add(time.Duration(m.config.Schedule.Cooldown) * time.Minute))
}
func (m *Manager) nextCooldownEligibleAtLocked(alert *Alert) *time.Time {
if alert == nil || alert.LastNotified == nil || m.config.Schedule.Cooldown <= 0 {
return nil
}
next := alert.LastNotified.Add(time.Duration(m.config.Schedule.Cooldown) * time.Minute)
return &next
}
func (m *Manager) allowNotificationByRateLimit(trackingKey string, alert *Alert, reason string) bool {
if trackingKey == "" && alert != nil {
trackingKey = canonicalTrackingKeyForAlert(alert)
}
if trackingKey == "" && alert != nil {
trackingKey = alert.ID
}
if m.checkRateLimit(trackingKey) {
return true
}
log.Debug().
Str("alertID", alertIDForLog(alert)).
Str("trackingKey", trackingKey).
Str("reason", reason).
Int("maxPerHour", m.config.Schedule.MaxAlertsHour).
Msg("Alert notification suppressed due to rate limit")
return false
}
func alertIDForLog(alert *Alert) string {
if alert == nil {
return ""
}
return alert.ID
}
// checkRateLimit checks if an alert has exceeded rate limit
func (m *Manager) checkRateLimit(alertID string) bool {
if m.config.Schedule.MaxAlertsHour <= 0 {
return true // No rate limit
}
now := time.Now()
cutoff := now.Add(-1 * time.Hour)
// Clean old entries and count recent alerts
var recentAlerts []time.Time
if times, exists := m.alertRateLimit[alertID]; exists {
for _, t := range times {
if t.After(cutoff) {
recentAlerts = append(recentAlerts, t)
}
}
}
// Check if we've hit the limit
if len(recentAlerts) >= m.config.Schedule.MaxAlertsHour {
return false
}
// Add current time
recentAlerts = append(recentAlerts, now)
m.alertRateLimit[alertID] = recentAlerts
return true
}