Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
package alerts
import (
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/operationaltrust"
"github.com/rs/zerolog/log"
)
// escalationChecker runs periodically to check for alerts that need escalation and cleanup
func (m *Manager) escalationChecker() {
ticker := time.NewTicker(1 * time.Minute)
cleanupTicker := time.NewTicker(10 * time.Minute) // Run cleanup every 10 minutes
defer ticker.Stop()
defer cleanupTicker.Stop()
for {
select {
case <-ticker.C:
m.checkEscalations()
case <-cleanupTicker.C:
m.Cleanup(24 * time.Hour) // Clean up acknowledged alerts older than 24 hours
case <-m.escalationStop:
return
}
}
}
// checkEscalations checks all active alerts for escalation
func (m *Manager) checkEscalations() {
m.expireOperationalSuppressions()
m.mu.Lock()
defer m.mu.Unlock()
// Respect global alert and activation controls before escalating.
// Escalations should never bypass a user disabling alerts.
if !m.config.Enabled || m.config.ActivationState != ActivationActive {
return
}
if !m.config.Schedule.Escalation.Enabled {
return
}
now := m.policyNow()
for _, alert := range m.activeAlerts {
// A supported symptom has a primary delivery owner. Like monitor-only
// alerts, it stays visible without its own escalation or critical repeat.
if alert == nil || alert.Acknowledged || isMonitorOnlyAlert(alert) || isSupportedInfrastructureSymptom(alert) {
continue
}
if m.awaitsBackupRevalidationNoLock(alert) {
continue
}
if _, snoozed := alertSnoozeUntil(alert, now); snoozed {
continue
}
// Check each escalation level
escalationBaseline := alert.StartTime
if transition := alert.LatestTransition; transition != nil &&
transition.Cause == operationaltrust.TransitionSuppressionExpired &&
transition.At.After(escalationBaseline) {
// Snoozed escalation levels are deliberately skipped, not replayed in
// a burst. After resume, the remaining escalation schedule starts
// from the explicit resume transition while incident age is preserved.
escalationBaseline = transition.At
}
for i, level := range m.config.Schedule.Escalation.Levels {
// Skip if we've already escalated to this level
if alert.LastEscalation >= i+1 {
continue
}
// Check if it's time to escalate
escalateTime := escalationBaseline.Add(time.Duration(level.After) * time.Minute)
if now.After(escalateTime) {
// Update alert escalation state
alert.LastEscalation = i + 1
alert.EscalationTimes = append(alert.EscalationTimes, now)
log.Info().
Str("alertID", alert.ID).
Str("trackingKey", canonicalTrackingKeyForAlert(alert)).
Int("level", i+1).
Str("notify", level.Notify).
Msg("Alert escalated")
// Trigger escalation callback
m.safeCallEscalateCallback(alert, i+1)
}
}
escalation := m.config.Schedule.Escalation
if !escalation.RepeatCritical || alert.Level != AlertLevelCritical || len(escalation.Levels) == 0 || alert.LastEscalation < len(escalation.Levels) {
continue
}
repeatEvery := time.Duration(escalation.RepeatEvery) * time.Minute
if repeatEvery <= 0 {
continue
}
lastLevel := escalation.Levels[len(escalation.Levels)-1]
repeatBaseline := escalationBaseline.Add(time.Duration(lastLevel.After) * time.Minute)
if count := len(alert.EscalationTimes); count > 0 && alert.EscalationTimes[count-1].After(repeatBaseline) {
repeatBaseline = alert.EscalationTimes[count-1]
}
if !now.After(repeatBaseline.Add(repeatEvery)) {
continue
}
alert.EscalationTimes = append(alert.EscalationTimes, now)
log.Info().
Str("alertID", alert.ID).
Str("trackingKey", canonicalTrackingKeyForAlert(alert)).
Int("level", len(escalation.Levels)).
Int("repeatEveryMinutes", escalation.RepeatEvery).
Msg("Critical alert escalation repeated")
m.safeCallEscalationRepeatCallback(alert, len(escalation.Levels))
}
}
// PrepareEscalationNotification revalidates an asynchronously dispatched
// escalation against the current policy and occurrence. The callback snapshot
// is not authority to send after recovery, acknowledgement or a policy edit.
// This does not recall a notification already handed to a destination.
func (m *Manager) PrepareEscalationNotification(snapshot *Alert, level int) (*Alert, EscalationLevel, bool) {
if m == nil || snapshot == nil {
return nil, EscalationLevel{}, false
}
m.mu.Lock()
defer m.mu.Unlock()
if !m.config.Enabled || m.config.ActivationState != ActivationActive || !m.config.Schedule.Escalation.Enabled || level <= 0 || level > len(m.config.Schedule.Escalation.Levels) {
return nil, EscalationLevel{}, false
}
active, ok := m.getActiveAlertNoLock(snapshot.ID)
if !ok || active == nil || !active.StartTime.Equal(snapshot.StartTime) || active.Acknowledged || isMonitorOnlyAlert(active) || isSupportedInfrastructureSymptom(active) {
return nil, EscalationLevel{}, false
}
if m.awaitsBackupRevalidationNoLock(active) {
return nil, EscalationLevel{}, false
}
if _, snoozed := alertSnoozeUntil(active, m.policyNow()); snoozed {
return nil, EscalationLevel{}, false
}
target := m.config.Schedule.Escalation.Levels[level-1]
target.DestinationIDs = append([]string(nil), target.DestinationIDs...)
return cloneAlertForOutput(active), target, true
}