Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
package alerts
import (
"time"
"github.com/rs/zerolog/log"
)
// trackingMapCleanup periodically cleans up stale entries from tracking maps
// to prevent unbounded memory growth from deleted/decommissioned resources.
func (m *Manager) trackingMapCleanup() {
ticker := time.NewTicker(1 * time.Hour)
defer ticker.Stop()
for {
select {
case <-ticker.C:
m.cleanupStaleMaps()
case <-m.cleanupStop:
return
}
}
}
// cleanupStaleMaps removes stale entries from tracking maps.
// Entries are considered stale if they haven't been updated in 24 hours
// and don't correspond to any active alert.
func (m *Manager) cleanupStaleMaps() {
m.mu.Lock()
defer m.mu.Unlock()
now := time.Now()
staleThreshold := StaleTrackingThreshold
cleaned := 0
cleaned += m.core.PruneStalePending(now.Add(-staleThreshold))
// A native sensor's link identity is kept only while its run is pending.
for resourceID := range m.hostNativeSensorPendingCtx {
if !m.hostNativeSensorPendingNoLock(resourceID) {
delete(m.hostNativeSensorPendingCtx, resourceID)
cleaned++
}
}
for alertID, history := range m.flappingHistory {
if !m.hasActiveAlertNoLock(alertID) {
if len(history) == 0 || now.Sub(history[len(history)-1]) > staleThreshold {
delete(m.flappingHistory, alertID)
delete(m.flappingActive, alertID)
cleaned++
}
}
}
for alertID := range m.suppressedUntil {
if m.expireSuppressionNoLock(alertID, now) {
cleaned++
}
}
for trackingKey, pending := range m.intentPending {
if m.hasActiveAlertNoLock(trackingKey) {
continue
}
if pending.LastObservedAt.IsZero() || now.Sub(pending.LastObservedAt) > staleThreshold {
m.clearIntentPendingNoLock(trackingKey)
cleaned++
}
}
for containerID, record := range m.dockerRestartTracking {
if record != nil && now.Sub(record.lastChecked) > staleThreshold {
delete(m.dockerRestartTracking, containerID)
cleaned++
}
}
for containerID, firstSeen := range m.dockerUpdateFirstSeen {
lastObserved := m.dockerUpdateLastObserved[containerID]
if lastObserved.IsZero() {
lastObserved = firstSeen
}
if now.Sub(lastObserved) > staleThreshold {
delete(m.dockerUpdateFirstSeen, containerID)
delete(m.dockerUpdateLastObserved, containerID)
cleaned++
}
}
for containerID, firstSeen := range m.dockerUpdateFirstSeenByIdentity {
lastObserved := m.dockerUpdateLastObserved[containerID]
if lastObserved.IsZero() {
lastObserved = firstSeen
}
if now.Sub(lastObserved) > staleThreshold {
delete(m.dockerUpdateFirstSeenByIdentity, containerID)
delete(m.dockerUpdateLastObserved, containerID)
cleaned++
}
}
for resourceID, snapshot := range m.smartCounterSnapshots {
if snapshot.LastObserved.IsZero() || now.Sub(snapshot.LastObserved) > staleThreshold {
delete(m.smartCounterSnapshots, resourceID)
cleaned++
}
}
rateLimitThreshold := RateLimitCleanupWindow
for resourceID, times := range m.alertRateLimit {
var recent []time.Time
for _, t := range times {
if now.Sub(t) < rateLimitThreshold {
recent = append(recent, t)
}
}
if len(recent) == 0 {
delete(m.alertRateLimit, resourceID)
cleaned++
} else if len(recent) < len(times) {
m.alertRateLimit[resourceID] = recent
}
}
suppressWindow := time.Duration(m.config.SuppressionWindow) * time.Minute
if suppressWindow <= 0 {
suppressWindow = 5 * time.Minute
}
for alertID, alert := range m.recentAlerts {
if now.Sub(alert.LastSeen) > suppressWindow {
delete(m.recentAlerts, alertID)
cleaned++
}
}
for alertID, record := range m.ackState {
if !m.hasActiveAlertNoLock(alertID) {
checkTime := record.inactiveAt
if checkTime.IsZero() {
checkTime = record.time
}
if now.Sub(checkTime) > staleThreshold {
delete(m.ackState, alertID)
cleaned++
}
}
}
for canonicalID, record := range m.ackStateByCanonical {
// As with legacy records and Cleanup, an active incident owns its
// acknowledgement until recovery starts the inactive-retention window.
if m.hasActiveAlertTrackingKeyNoLock(canonicalID) {
continue
}
checkTime := record.inactiveAt
if checkTime.IsZero() {
checkTime = record.time
}
if now.Sub(checkTime) > staleThreshold {
delete(m.ackStateByCanonical, canonicalID)
cleaned++
}
}
staleAlerts := make([]string, 0)
for storageKey, alert := range m.activeAlerts {
alertID := effectiveAlertID(alert, storageKey)
// A threshold alert holding below its trigger is still observed
// even though LastSeen stays at its last breach.
if alert != nil && now.Sub(alert.lastObservedAt()) > staleThreshold {
staleAlerts = append(staleAlerts, alertID)
}
}
staleResolved := 0
for _, alertID := range staleAlerts {
alert, exists := m.getActiveAlertNoLock(alertID)
if !exists || alert == nil {
continue
}
log.Info().
Str("alertID", alertID).
Str("resourceName", alert.ResourceName).
Time("lastSeen", alert.LastSeen).
Dur("staleFor", now.Sub(alert.LastSeen)).
Msg("Auto-resolving stale alert - resource no longer being monitored")
m.clearAlertNoLock(alertID)
cleaned++
staleResolved++
}
if staleResolved > 0 {
m.saveActiveAlertsAsync("stale cleanup")
log.Info().
Int("count", staleResolved).
Msg("Auto-resolved stale alerts")
}
if cleaned > 0 {
log.Debug().
Int("entriesCleaned", cleaned).
Msg("Cleaned stale entries from alert tracking maps")
}
}