Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

5.9 KB
Raw Blame
package alerts

import "time"

func activeAlertStorageKey(alert *Alert, fallback string) string {
	if fallback == "" && alert == nil {
		return ""
	}
	if alert != nil {
		if identity := deriveCanonicalIdentity(alert); identity.State != "" {
			return identity.State
		}
		if alert.ID != "" {
			return alert.ID
		}
	}
	return fallback
}

func effectiveAlertID(alert *Alert, fallback string) string {
	if alert != nil {
		if identity := deriveCanonicalIdentity(alert); identity.State != "" {
			return identity.State
		}
		if alert.ID != "" {
			return alert.ID
		}
	}
	return fallback
}

func (m *Manager) registerActiveAlertAliasNoLock(storageKey string, alert *Alert) {
	if m == nil || alert == nil || storageKey == "" {
		return
	}
	if m.activeAlertAlias == nil {
		m.activeAlertAlias = make(map[string]string)
	}
	backfillCanonicalIdentity(alert)
	if alias := effectiveAlertID(alert, ""); alias != "" && alias != storageKey {
		m.activeAlertAlias[alias] = storageKey
	}
	if alert.CanonicalState != "" && alert.CanonicalState != storageKey {
		m.activeAlertAlias[alert.CanonicalState] = storageKey
	}
}

func (m *Manager) resolveActiveAlertKeyNoLock(id string) (string, bool) {
	if _, ok := m.activeAlerts[id]; ok {
		return id, true
	}
	key, ok := m.activeAlertAlias[id]
	if !ok {
		return "", false
	}
	if _, exists := m.activeAlerts[key]; !exists {
		delete(m.activeAlertAlias, id)
		return "", false
	}
	return key, true
}

func (m *Manager) resolveActiveAlertKeyByCanonicalStateNoLock(id string) (string, bool) {
	for storageKey, alert := range m.activeAlerts {
		if alert == nil {
			continue
		}
		if deriveCanonicalIdentity(alert).State != id {
			continue
		}
		if m.activeAlertAlias == nil {
			m.activeAlertAlias = make(map[string]string)
		}
		m.activeAlertAlias[id] = storageKey
		return storageKey, true
	}
	return "", false
}

func (m *Manager) getActiveAlertNoLock(id string) (*Alert, bool) {
	key, ok := m.resolveActiveAlertKeyNoLock(id)
	if !ok {
		key, ok = m.resolveActiveAlertKeyByCanonicalStateNoLock(id)
		if !ok {
			return nil, false
		}
	}
	alert := m.activeAlerts[key]
	if alert == nil {
		return nil, false
	}
	return alert, true
}

func (m *Manager) hasActiveAlertNoLock(id string) bool {
	if _, ok := m.resolveActiveAlertKeyNoLock(id); ok {
		return true
	}
	_, ok := m.resolveActiveAlertKeyByCanonicalStateNoLock(id)
	return ok
}

// setActiveAlertNoLock reports whether the alert was admitted. Detector callers
// must check admission before recording a firing lifecycle or dispatching it.
func (m *Manager) setActiveAlertNoLock(storageKey string, alert *Alert) bool {
	if storageKey == "" || alert == nil {
		return false
	}
	backfillCanonicalIdentity(alert)
	if suppressed, _ := m.operatorSuppressionForAlertNoLock(alert, time.Now().UTC()); suppressed {
		return false
	}
	ensureOperationalContract(alert, time.Now())
	requestedKey := storageKey
	storageKey = activeAlertStorageKey(alert, storageKey)
	for _, staleKey := range []string{requestedKey, alert.ID, alert.CanonicalState} {
		if staleKey == "" || staleKey == storageKey {
			continue
		}
		if existing, ok := m.activeAlerts[staleKey]; ok {
			delete(m.activeAlerts, staleKey)
			m.unregisterActiveAlertAliasNoLock(staleKey, existing)
			m.removeActiveRecoveryAlert(existing, staleKey)
		}
	}
	m.activeAlerts[storageKey] = alert
	m.registerActiveAlertAliasNoLock(storageKey, alert)
	m.setActiveRecoveryAlert(alert, storageKey)
	if m.historyManager != nil {
		m.historyManager.UpdateAlertOperationalContractForAlert(alert)
	}
	return true
}

func (m *Manager) unregisterActiveAlertAliasNoLock(storageKey string, alert *Alert) {
	if m == nil || storageKey == "" {
		return
	}
	if alert != nil {
		backfillCanonicalIdentity(alert)
		if alias := effectiveAlertID(alert, ""); alias != "" && alias != storageKey {
			delete(m.activeAlertAlias, alias)
		}
		if alert.CanonicalState != "" && alert.CanonicalState != storageKey {
			delete(m.activeAlertAlias, alert.CanonicalState)
		}
	}
}

func (m *Manager) registerResolvedAliasUnlocked(storageKey string, resolved *ResolvedAlert) {
	if m == nil || resolved == nil || resolved.Alert == nil || storageKey == "" {
		return
	}
	if m.resolvedAlias == nil {
		m.resolvedAlias = make(map[string]string)
	}
	// Resolved alerts may already be visible to asynchronous lifecycle
	// callbacks. Derive aliases without mutating the shared alert snapshot.
	if alias := effectiveAlertID(resolved.Alert, ""); alias != "" && alias != storageKey {
		m.resolvedAlias[alias] = storageKey
	}
}

func (m *Manager) unregisterResolvedAliasUnlocked(storageKey string, resolved *ResolvedAlert) {
	if m == nil || storageKey == "" {
		return
	}
	if resolved != nil && resolved.Alert != nil {
		if alias := effectiveAlertID(resolved.Alert, ""); alias != "" && alias != storageKey {
			delete(m.resolvedAlias, alias)
		}
	}
}

func (m *Manager) removeResolvedAlertUnlocked(id string) (*ResolvedAlert, bool) {
	resolved, ok := m.getResolvedAlertNoLock(id)
	if !ok || resolved == nil {
		return nil, false
	}

	storageKey := id
	if resolved.Alert != nil {
		storageKey = activeAlertStorageKey(resolved.Alert, id)
	}
	if key, exists := m.resolvedAlias[id]; exists {
		storageKey = key
	}

	delete(m.recentlyResolved, storageKey)
	m.unregisterResolvedAliasUnlocked(storageKey, resolved)
	return resolved, true
}

func (m *Manager) getResolvedAlertNoLock(id string) (*ResolvedAlert, bool) {
	if resolved, ok := m.recentlyResolved[id]; ok && resolved != nil {
		return resolved, true
	}
	key, ok := m.resolvedAlias[id]
	if !ok {
		for storageKey, resolved := range m.recentlyResolved {
			if resolved == nil || resolved.Alert == nil {
				continue
			}
			if deriveCanonicalIdentity(resolved.Alert).State != id {
				continue
			}
			if m.resolvedAlias == nil {
				m.resolvedAlias = make(map[string]string)
			}
			m.resolvedAlias[id] = storageKey
			return resolved, true
		}
		return nil, false
	}
	resolved, exists := m.recentlyResolved[key]
	if !exists || resolved == nil {
		delete(m.resolvedAlias, id)
		return nil, false
	}
	return resolved, true
}