Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
package alerts
import "time"
func activeAlertStorageKey(alert *Alert, fallback string) string {
if fallback == "" && alert == nil {
return ""
}
if alert != nil {
if identity := deriveCanonicalIdentity(alert); identity.State != "" {
return identity.State
}
if alert.ID != "" {
return alert.ID
}
}
return fallback
}
func effectiveAlertID(alert *Alert, fallback string) string {
if alert != nil {
if identity := deriveCanonicalIdentity(alert); identity.State != "" {
return identity.State
}
if alert.ID != "" {
return alert.ID
}
}
return fallback
}
func (m *Manager) registerActiveAlertAliasNoLock(storageKey string, alert *Alert) {
if m == nil || alert == nil || storageKey == "" {
return
}
if m.activeAlertAlias == nil {
m.activeAlertAlias = make(map[string]string)
}
backfillCanonicalIdentity(alert)
if alias := effectiveAlertID(alert, ""); alias != "" && alias != storageKey {
m.activeAlertAlias[alias] = storageKey
}
if alert.CanonicalState != "" && alert.CanonicalState != storageKey {
m.activeAlertAlias[alert.CanonicalState] = storageKey
}
}
func (m *Manager) resolveActiveAlertKeyNoLock(id string) (string, bool) {
if _, ok := m.activeAlerts[id]; ok {
return id, true
}
key, ok := m.activeAlertAlias[id]
if !ok {
return "", false
}
if _, exists := m.activeAlerts[key]; !exists {
delete(m.activeAlertAlias, id)
return "", false
}
return key, true
}
func (m *Manager) resolveActiveAlertKeyByCanonicalStateNoLock(id string) (string, bool) {
for storageKey, alert := range m.activeAlerts {
if alert == nil {
continue
}
if deriveCanonicalIdentity(alert).State != id {
continue
}
if m.activeAlertAlias == nil {
m.activeAlertAlias = make(map[string]string)
}
m.activeAlertAlias[id] = storageKey
return storageKey, true
}
return "", false
}
func (m *Manager) getActiveAlertNoLock(id string) (*Alert, bool) {
key, ok := m.resolveActiveAlertKeyNoLock(id)
if !ok {
key, ok = m.resolveActiveAlertKeyByCanonicalStateNoLock(id)
if !ok {
return nil, false
}
}
alert := m.activeAlerts[key]
if alert == nil {
return nil, false
}
return alert, true
}
func (m *Manager) hasActiveAlertNoLock(id string) bool {
if _, ok := m.resolveActiveAlertKeyNoLock(id); ok {
return true
}
_, ok := m.resolveActiveAlertKeyByCanonicalStateNoLock(id)
return ok
}
// setActiveAlertNoLock reports whether the alert was admitted. Detector callers
// must check admission before recording a firing lifecycle or dispatching it.
func (m *Manager) setActiveAlertNoLock(storageKey string, alert *Alert) bool {
if storageKey == "" || alert == nil {
return false
}
backfillCanonicalIdentity(alert)
if suppressed, _ := m.operatorSuppressionForAlertNoLock(alert, time.Now().UTC()); suppressed {
return false
}
ensureOperationalContract(alert, time.Now())
requestedKey := storageKey
storageKey = activeAlertStorageKey(alert, storageKey)
for _, staleKey := range []string{requestedKey, alert.ID, alert.CanonicalState} {
if staleKey == "" || staleKey == storageKey {
continue
}
if existing, ok := m.activeAlerts[staleKey]; ok {
delete(m.activeAlerts, staleKey)
m.unregisterActiveAlertAliasNoLock(staleKey, existing)
m.removeActiveRecoveryAlert(existing, staleKey)
}
}
m.activeAlerts[storageKey] = alert
m.registerActiveAlertAliasNoLock(storageKey, alert)
m.setActiveRecoveryAlert(alert, storageKey)
if m.historyManager != nil {
m.historyManager.UpdateAlertOperationalContractForAlert(alert)
}
return true
}
func (m *Manager) unregisterActiveAlertAliasNoLock(storageKey string, alert *Alert) {
if m == nil || storageKey == "" {
return
}
if alert != nil {
backfillCanonicalIdentity(alert)
if alias := effectiveAlertID(alert, ""); alias != "" && alias != storageKey {
delete(m.activeAlertAlias, alias)
}
if alert.CanonicalState != "" && alert.CanonicalState != storageKey {
delete(m.activeAlertAlias, alert.CanonicalState)
}
}
}
func (m *Manager) registerResolvedAliasUnlocked(storageKey string, resolved *ResolvedAlert) {
if m == nil || resolved == nil || resolved.Alert == nil || storageKey == "" {
return
}
if m.resolvedAlias == nil {
m.resolvedAlias = make(map[string]string)
}
// Resolved alerts may already be visible to asynchronous lifecycle
// callbacks. Derive aliases without mutating the shared alert snapshot.
if alias := effectiveAlertID(resolved.Alert, ""); alias != "" && alias != storageKey {
m.resolvedAlias[alias] = storageKey
}
}
func (m *Manager) unregisterResolvedAliasUnlocked(storageKey string, resolved *ResolvedAlert) {
if m == nil || storageKey == "" {
return
}
if resolved != nil && resolved.Alert != nil {
if alias := effectiveAlertID(resolved.Alert, ""); alias != "" && alias != storageKey {
delete(m.resolvedAlias, alias)
}
}
}
func (m *Manager) removeResolvedAlertUnlocked(id string) (*ResolvedAlert, bool) {
resolved, ok := m.getResolvedAlertNoLock(id)
if !ok || resolved == nil {
return nil, false
}
storageKey := id
if resolved.Alert != nil {
storageKey = activeAlertStorageKey(resolved.Alert, id)
}
if key, exists := m.resolvedAlias[id]; exists {
storageKey = key
}
delete(m.recentlyResolved, storageKey)
m.unregisterResolvedAliasUnlocked(storageKey, resolved)
return resolved, true
}
func (m *Manager) getResolvedAlertNoLock(id string) (*ResolvedAlert, bool) {
if resolved, ok := m.recentlyResolved[id]; ok && resolved != nil {
return resolved, true
}
key, ok := m.resolvedAlias[id]
if !ok {
for storageKey, resolved := range m.recentlyResolved {
if resolved == nil || resolved.Alert == nil {
continue
}
if deriveCanonicalIdentity(resolved.Alert).State != id {
continue
}
if m.resolvedAlias == nil {
m.resolvedAlias = make(map[string]string)
}
m.resolvedAlias[id] = storageKey
return resolved, true
}
return nil, false
}
resolved, exists := m.recentlyResolved[key]
if !exists || resolved == nil {
delete(m.resolvedAlias, id)
return nil, false
}
return resolved, true
}