Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

11.7 KB
Raw Blame
package alerts

import (
	"strings"
	"time"

	alertconfig "github.com/rcourtman/pulse-go-rewrite/internal/alerts/config"
	alertspecs "github.com/rcourtman/pulse-go-rewrite/internal/alerts/specs"
	"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
	"github.com/rs/zerolog/log"
)

// Cleanup removes old acknowledged alerts and cleans up tracking maps.
func (m *Manager) Cleanup(maxAge time.Duration) {
	m.mu.Lock()
	now := time.Now()
	var autoAcked []*Alert

	lastSeenTooOld := func(alert *Alert, cutoff time.Duration) bool {
		if alert == nil {
			return true
		}
		lastSeen := alert.lastObservedAt()
		if lastSeen.IsZero() {
			lastSeen = alert.StartTime
		}
		return now.Sub(lastSeen) > cutoff
	}

	if m.config.AutoAcknowledgeAfterHours > 0 {
		autoAckThreshold := time.Duration(m.config.AutoAcknowledgeAfterHours) * time.Hour
		for id, alert := range m.activeAlerts {
			if !alert.Acknowledged && now.Sub(alert.StartTime) > autoAckThreshold {
				log.Info().
					Str("alertID", id).
					Dur("age", now.Sub(alert.StartTime)).
					Msg("Auto-acknowledging old alert")
				alert.Acknowledged = true
				ackTime := now
				alert.AckTime = &ackTime
				alert.AckUser = "system-auto"
				// Preserve the same recurrence retention and reducer state as a
				// manual acknowledgement, not just the active alert's display.
				m.setAckRecordNoLock(alert, id, ackRecord{
					acknowledged: true,
					user:         "system-auto",
					time:         ackTime,
				})
				m.mirrorAcknowledgeNoLock(alert, "system-auto", ackTime)
				autoAcked = append(autoAcked, alert.Clone())

				if recordAlertAcknowledged != nil {
					recordAlertAcknowledged()
				}
			}
		}
	}

	if m.config.MaxAcknowledgedAgeDays > 0 {
		acknowledgedTTL := time.Duration(m.config.MaxAcknowledgedAgeDays) * 24 * time.Hour
		for id, alert := range m.activeAlerts {
			if alert.Acknowledged && alert.AckTime != nil &&
				now.Sub(*alert.AckTime) > acknowledgedTTL &&
				lastSeenTooOld(alert, acknowledgedTTL) {
				log.Info().
					Str("alertID", id).
					Dur("age", now.Sub(*alert.AckTime)).
					Msg("Cleaning up old acknowledged alert (TTL)")
				m.removeActiveAlertNoLock(id)
			}
		}
	}

	if m.config.MaxAlertAgeDays > 0 {
		alertTTL := time.Duration(m.config.MaxAlertAgeDays) * 24 * time.Hour
		for id, alert := range m.activeAlerts {
			// Retention is housekeeping for inactive alerts, not recovery
			// evidence. As with acknowledged alerts, keep a condition still
			// observed; deleting it would re-fire on the next failing poll.
			if !alert.Acknowledged && now.Sub(alert.StartTime) > alertTTL &&
				lastSeenTooOld(alert, alertTTL) {
				log.Info().
					Str("alertID", id).
					Dur("age", now.Sub(alert.StartTime)).
					Msg("Cleaning up old unacknowledged alert (TTL)")
				m.removeActiveAlertNoLock(id)
			}
		}
	}

	for id, alert := range m.activeAlerts {
		if alert.Acknowledged && alert.AckTime != nil &&
			now.Sub(*alert.AckTime) > maxAge &&
			lastSeenTooOld(alert, maxAge) {
			m.removeActiveAlertNoLock(id)
		}
	}

	ackStateTTL := 1 * time.Hour
	for id, record := range m.ackState {
		if !m.hasActiveAlertNoLock(id) {
			checkTime := record.inactiveAt
			if checkTime.IsZero() {
				checkTime = record.time
			}
			if now.Sub(checkTime) > ackStateTTL {
				delete(m.ackState, id)
			}
		}
	}
	for canonicalID, record := range m.ackStateByCanonical {
		if m.hasActiveAlertTrackingKeyNoLock(canonicalID) {
			continue
		}
		checkTime := record.inactiveAt
		if checkTime.IsZero() {
			checkTime = record.time
		}
		if now.Sub(checkTime) > ackStateTTL {
			delete(m.ackStateByCanonical, canonicalID)
		}
	}

	suppressionWindow := time.Duration(m.config.SuppressionWindow) * time.Minute
	if suppressionWindow == 0 {
		suppressionWindow = 5 * time.Minute
	}
	for id, alert := range m.recentAlerts {
		if now.Sub(alert.StartTime) > suppressionWindow {
			delete(m.recentAlerts, id)
		}
	}

	for id := range m.suppressedUntil {
		m.expireSuppressionNoLock(id, now)
	}

	cutoff := now.Add(-1 * time.Hour)
	for alertID, times := range m.alertRateLimit {
		var recentTimes []time.Time
		for _, t := range times {
			if t.After(cutoff) {
				recentTimes = append(recentTimes, t)
			}
		}
		if len(recentTimes) == 0 {
			delete(m.alertRateLimit, alertID)
		} else {
			m.alertRateLimit[alertID] = recentTimes
		}
	}

	m.resolvedMutex.Lock()
	m.pruneRecentlyResolvedUnlocked(now)
	m.resolvedMutex.Unlock()

	// Pending confirmation/delay runs whose resource stopped being observed
	// mid-run would otherwise linger in the core forever.
	m.core.PruneStalePending(now.Add(-10 * time.Minute))

	flappingCleanupAge := 1 * time.Hour
	for alertID := range m.flappingHistory {
		if !m.hasActiveAlertTrackingKeyNoLock(alertID) {
			if suppressUntil, suppressed := m.suppressedUntil[alertID]; !suppressed || now.After(suppressUntil.Add(flappingCleanupAge)) {
				delete(m.flappingHistory, alertID)
				delete(m.flappingActive, alertID)
				log.Debug().
					Str("alertID", alertID).
					Msg("Cleaned up flapping history for inactive alert")
			}
		}
	}

	for resourceID, record := range m.dockerRestartTracking {
		if now.Sub(record.lastChecked) > 24*time.Hour {
			delete(m.dockerRestartTracking, resourceID)
			log.Debug().
				Str("resourceID", resourceID).
				Msg("Cleaned up stale Docker restart tracking entry")
		}
	}

	staleTrackerAge := 24 * time.Hour
	for pmgID, tracker := range m.pmgAnomalyTrackers {
		if tracker != nil && !tracker.LastSampleTime.IsZero() {
			if now.Sub(tracker.LastSampleTime) > staleTrackerAge {
				delete(m.pmgAnomalyTrackers, pmgID)
				log.Debug().
					Str("pmgID", pmgID).
					Time("lastSampleTime", tracker.LastSampleTime).
					Msg("Cleaned up stale PMG anomaly tracker")
			}
		}
	}

	staleHistoryAge := 7 * 24 * time.Hour
	for pmgID, snapshots := range m.pmgQuarantineHistory {
		if len(snapshots) == 0 {
			delete(m.pmgQuarantineHistory, pmgID)
			log.Debug().
				Str("pmgID", pmgID).
				Msg("Cleaned up empty PMG quarantine history")
			continue
		}

		lastSnapshot := snapshots[len(snapshots)-1]
		if now.Sub(lastSnapshot.Timestamp) > staleHistoryAge {
			delete(m.pmgQuarantineHistory, pmgID)
			log.Debug().
				Str("pmgID", pmgID).
				Time("lastSnapshot", lastSnapshot.Timestamp).
				Msg("Cleaned up stale PMG quarantine history")
		}
	}

	for resourceID, snapshot := range m.smartCounterSnapshots {
		if snapshot.LastObserved.IsZero() || now.Sub(snapshot.LastObserved) > staleTrackerAge {
			delete(m.smartCounterSnapshots, resourceID)
			log.Debug().
				Str("resourceID", resourceID).
				Msg("Cleaned up stale SMART counter snapshot")
		}
	}

	m.mu.Unlock()

	if len(autoAcked) > 0 {
		m.saveActiveAlertsAsync("auto acknowledge")
	}
	for _, alert := range autoAcked {
		m.safeCallAcknowledgedCallback(alert, "system-auto")
	}
}

func alertMetadataResourceType(alert *Alert) string {
	if alert == nil || alert.Metadata == nil {
		return ""
	}
	if value, ok := alert.Metadata["resourceType"].(string); ok {
		return strings.ToLower(strings.TrimSpace(value))
	}
	return ""
}

func shouldPreserveAlertOutsideNodeCleanup(alertID string, alert *Alert) bool {
	if alert == nil {
		return true
	}
	// System alerts have no node, so the empty-node removal below would
	// silently delete them every sweep. The delivery-health alert then
	// re-raised as new on its next evaluation, notifying again each cycle
	// with no recovery in between (#1721).
	if IsSystemAlert(alert) {
		return true
	}
	if strings.HasPrefix(alertID, "docker-") || strings.HasPrefix(alert.ResourceID, "docker:") {
		return true
	}
	if strings.HasPrefix(alertID, "pbs-") || alert.Type == "pbs-offline" {
		return true
	}
	if strings.HasPrefix(alert.ResourceID, "agent:") {
		return true
	}
	if alert.CanonicalKind == string(alertspecs.AlertSpecKindProviderIncident) {
		return true
	}
	// Kubernetes, TrueNAS and vSphere metric alerts carry a display label such
	// as "TrueNAS Disk" or "Kubernetes Pod" in resourceType, which the platform
	// words below never matched, and a cluster or host in Node rather than a
	// Proxmox node. Every Proxmox poll used to delete them silently, and the
	// next evaluation raised them again as new alerts.
	if metadataStringValue(alert.Metadata, alertPlatformTypeKey) != "" ||
		isUnifiedModernPlatformAlertType(alertconfig.CanonicalAlertResourceType(alertMetadataResourceType(alert))) {
		return true
	}

	switch alertMetadataResourceType(alert) {
	case string(unifiedresources.ResourceTypeAgent),
		string(unifiedresources.ResourceTypeAppContainer),
		string(unifiedresources.ResourceTypeDockerService),
		string(unifiedresources.ResourceTypeK8sCluster),
		string(unifiedresources.ResourceTypeK8sNode),
		string(unifiedresources.ResourceTypePod),
		string(unifiedresources.ResourceTypeK8sDeployment),
		string(unifiedresources.ResourceTypeStorage),
		string(unifiedresources.ResourceTypePBS),
		string(unifiedresources.ResourceTypePMG),
		string(unifiedresources.ResourceTypeCeph),
		string(unifiedresources.ResourceTypePhysicalDisk),
		string(unifiedresources.ResourceTypeNetworkShare),
		string(unifiedresources.ResourceTypeNetworkEndpoint),
		"docker",
		"host",
		"kubernetes",
		"k8s",
		"truenas",
		"vmware":
		return true
	default:
		return false
	}
}

// CleanupAlertsForNodes removes stale Proxmox node-scoped alerts.
func (m *Manager) CleanupAlertsForNodes(existingNodes map[string]bool) {
	m.mu.Lock()
	defer m.mu.Unlock()

	log.Debug().
		Int("totalAlerts", len(m.activeAlerts)).
		Int("existingNodes", len(existingNodes)).
		Interface("nodes", existingNodes).
		Msg("Starting alert cleanup for non-existent nodes")

	removedCount := 0
	for storageKey, alert := range m.activeAlerts {
		alertID := effectiveAlertID(alert, storageKey)
		if alert == nil {
			continue
		}

		if alert.CanonicalKind == string(alertspecs.AlertSpecKindConnectivity) && strings.HasPrefix(alert.ResourceID, "pbs") {
			continue
		}
		if shouldPreserveAlertOutsideNodeCleanup(alertID, alert) {
			continue
		}

		node := alert.Node
		if node == "" || !existingNodes[node] {
			m.removeActiveAlertNoLock(alertID)
			removedCount++
			log.Debug().Str("alertID", alertID).Str("node", node).Msg("removed alert for non-existent node")
		}
	}

	if removedCount > 0 {
		log.Debug().Int("removed", removedCount).Int("remaining", len(m.activeAlerts)).Msg("cleaned up alerts for non-existent nodes")
		m.saveActiveAlertsAsync("node cleanup")
	} else {
		log.Debug().Msg("no alerts needed cleanup")
	}
}

// ClearActiveAlerts removes all active and pending alerts, resetting the manager state.
func (m *Manager) ClearActiveAlerts() {
	m.mu.Lock()
	if len(m.activeAlerts) == 0 && len(m.intentPending) == 0 && len(m.smartCounterSnapshots) == 0 && len(m.hostDiskTempPendingContexts) == 0 && len(m.hostNativeSensorPendingCtx) == 0 {
		m.mu.Unlock()
		return
	}
	m.activeAlerts = make(map[string]*Alert)
	m.restoredBackupNotifications = nil
	m.activeAlertAlias = make(map[string]string)
	for _, state := range m.mirrorStatesNoLock() {
		state.Reset()
	}
	m.hostDiskTempPendingContexts = nil
	m.hostNativeSensorPendingCtx = nil
	m.intentPending = make(map[string]IntentPendingState)
	m.intentRuntimeTicks = make(map[string]time.Duration)
	m.recentAlerts = make(map[string]*Alert)
	m.suppressedUntil = make(map[string]time.Time)
	m.alertRateLimit = make(map[string][]time.Time)
	m.dockerRestartTracking = make(map[string]*dockerRestartRecord)
	m.dockerUpdateFirstSeen = make(map[string]time.Time)
	m.dockerUpdateFirstSeenByIdentity = make(map[string]time.Time)
	m.dockerUpdateLastObserved = make(map[string]time.Time)
	m.smartCounterSnapshots = make(map[string]smartCounterSnapshot)
	m.ackState = make(map[string]ackRecord)
	m.ackStateByCanonical = make(map[string]ackRecord)
	m.mu.Unlock()

	m.resolvedMutex.Lock()
	m.recentlyResolved = make(map[string]*ResolvedAlert)
	m.resolvedAlias = make(map[string]string)
	m.resolvedMutex.Unlock()

	log.Info().Msg("cleared all active and pending alerts")

	m.saveActiveAlertsAsync("clear active alerts")
}