Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
package alerts
import (
"encoding/json"
"fmt"
"github.com/rcourtman/pulse-go-rewrite/internal/alerts/eventlog"
"sort"
"strconv"
"strings"
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/models"
)
// GetActiveAlerts returns all active alerts
func (m *Manager) GetActiveAlerts() []Alert {
if m == nil {
return nil
}
m.mu.RLock()
defer m.mu.RUnlock()
alerts := make([]Alert, 0, len(m.activeAlerts))
for _, alert := range m.activeAlerts {
a := *cloneAlertForOutput(alert)
// Ensure display name is current (handles upgrades, renames, and
// alerts created before the cache was populated).
if dn := m.resolveNodeDisplayName(a.Instance, a.Node); dn != "" {
a.NodeDisplayName = dn
}
alerts = append(alerts, a)
}
sort.Slice(alerts, func(i, j int) bool {
if left, right := alertSeveritySortRank(alerts[i]), alertSeveritySortRank(alerts[j]); left != right {
return left > right
}
if left, right := alertProtectionSortRank(alerts[i]), alertProtectionSortRank(alerts[j]); left != right {
return left > right
}
if left, right := alertRecoverabilitySortRank(alerts[i]), alertRecoverabilitySortRank(alerts[j]); left != right {
return left > right
}
if left, right := alertImpactSortRank(alerts[i]), alertImpactSortRank(alerts[j]); left != right {
return left > right
}
if left, right := alertTypeSortRank(alerts[i]), alertTypeSortRank(alerts[j]); left != right {
return left > right
}
if !alerts[i].StartTime.Equal(alerts[j].StartTime) {
return alerts[i].StartTime.Before(alerts[j].StartTime)
}
if alerts[i].Node != alerts[j].Node {
return alerts[i].Node < alerts[j].Node
}
return alerts[i].ID < alerts[j].ID
})
return alerts
}
func alertProtectionSortRank(alert Alert) int {
switch {
case metadataBoolValue(alert.Metadata, "protectionReduced"):
return 2
case metadataBoolValue(alert.Metadata, "rebuildInProgress"):
return 1
default:
return 0
}
}
func alertSeveritySortRank(alert Alert) int {
switch alert.Level {
case AlertLevelCritical:
return 2
case AlertLevelWarning:
return 1
case AlertLevelInfo:
return 0
default:
return -1
}
}
func alertImpactSortRank(alert Alert) int {
if alert.Metadata == nil {
return 0
}
return metadataIntValue(alert.Metadata["consumerCount"])
}
func alertRecoverabilitySortRank(alert Alert) int {
switch {
case metadataBoolValue(alert.Metadata, "backupTarget") && metadataIntValue(alert.Metadata["protectedWorkloadCount"]) > 0:
return 2
case metadataBoolValue(alert.Metadata, "backupServer") && metadataIntValue(alert.Metadata["protectedWorkloadCount"]) > 0:
return 2
case metadataBoolValue(alert.Metadata, "backupTarget"):
return 1
case metadataBoolValue(alert.Metadata, "backupServer") && metadataIntValue(alert.Metadata["affectedDatastoreCount"]) > 0:
return 1
case metadataBoolValue(alert.Metadata, "backupServer"):
return 1
default:
return 0
}
}
func alertTypeSortRank(alert Alert) int {
switch alert.Type {
case "backup-posture-incident":
return 6
case "backup-storage-incident":
return 5
case "storage-incident", "zfs-pool-state", "zfs-pool-errors":
return 4
case "resource-incident":
return 4
case "disk-health", "disk-wearout", "zfs-device":
return 3
case "offline", "connectivity", "powered-off", "docker-host-offline":
return 2
default:
return 1
}
}
func metadataIntValue(value interface{}) int {
switch v := value.(type) {
case int:
return v
case int8:
return int(v)
case int16:
return int(v)
case int32:
return int(v)
case int64:
return int(v)
case uint:
return int(v)
case uint8:
return int(v)
case uint16:
return int(v)
case uint32:
return int(v)
case uint64:
return int(v)
case float32:
return int(v)
case float64:
return int(v)
case json.Number:
if parsed, err := v.Int64(); err == nil {
return int(parsed)
}
case string:
if parsed, err := strconv.Atoi(strings.TrimSpace(v)); err == nil {
return parsed
}
}
return 0
}
func metadataFloatValue(metadata map[string]interface{}, key string) (float64, bool) {
if metadata == nil {
return 0, false
}
value, ok := metadata[key]
if !ok {
return 0, false
}
return numericConditionValue(value)
}
func metadataStringValue(metadata map[string]interface{}, key string) string {
if metadata == nil {
return ""
}
value, ok := metadata[key]
if !ok {
return ""
}
switch v := value.(type) {
case string:
return strings.TrimSpace(v)
case []byte:
return strings.TrimSpace(string(v))
case json.Number:
return strings.TrimSpace(v.String())
default:
return strings.TrimSpace(fmt.Sprint(v))
}
}
// metadataStringsValue reads a string list from alert metadata. A live alert
// holds a []string; one reloaded from disk holds a []interface{}.
func metadataStringsValue(metadata map[string]interface{}, key string) ([]string, bool) {
switch values := metadata[key].(type) {
case []string:
return append([]string(nil), values...), true
case []interface{}:
result := make([]string, 0, len(values))
for _, value := range values {
if s, ok := value.(string); ok {
result = append(result, s)
}
}
return result, true
}
return nil, false
}
func metadataBoolValue(metadata map[string]interface{}, key string) bool {
if metadata == nil {
return false
}
value, ok := metadata[key]
if !ok {
return false
}
switch v := value.(type) {
case bool:
return v
case string:
switch strings.ToLower(strings.TrimSpace(v)) {
case "1", "true", "yes", "on":
return true
}
case int:
return v != 0
case int8:
return v != 0
case int16:
return v != 0
case int32:
return v != 0
case int64:
return v != 0
case uint:
return v != 0
case uint8:
return v != 0
case uint16:
return v != 0
case uint32:
return v != 0
case uint64:
return v != 0
case float32:
return v != 0
case float64:
return v != 0
case json.Number:
if parsed, err := v.Int64(); err == nil {
return parsed != 0
}
}
return false
}
// NotifyExistingAlert re-dispatches a notification for an existing active alert
// Used when activation state changes from pending to active
func (m *Manager) NotifyExistingAlert(alertID string) {
m.mu.Lock()
defer m.mu.Unlock()
alert, exists := m.getActiveAlertNoLock(alertID)
if !exists {
return
}
// Dispatch notification for existing alert while holding lock
// dispatchAlert expects caller to hold m.mu for checkFlapping safety
m.dispatchAlert(alert, true)
}
// GetRecentlyResolved returns recently resolved alerts
func (m *Manager) GetRecentlyResolved() []models.ResolvedAlert {
if m == nil {
return nil
}
m.resolvedMutex.RLock()
resolvedSnapshot := make([]*ResolvedAlert, 0, len(m.recentlyResolved))
for _, alert := range m.recentlyResolved {
resolvedSnapshot = append(resolvedSnapshot, &ResolvedAlert{
Alert: cloneAlertForOutput(alert.Alert),
ResolvedTime: alert.ResolvedTime,
})
}
m.resolvedMutex.RUnlock()
resolved := make([]models.ResolvedAlert, 0, len(resolvedSnapshot))
m.mu.RLock()
defer m.mu.RUnlock()
for _, alert := range resolvedSnapshot {
exported := cloneAlertForOutput(alert.Alert)
nodeDisplayName := exported.NodeDisplayName
if current := m.resolveNodeDisplayName(exported.Instance, exported.Node); current != "" {
nodeDisplayName = current
}
resolved = append(resolved, models.ResolvedAlert{
Alert: models.Alert{
ID: exported.ID,
Type: exported.Type,
Level: string(exported.Level),
ResourceID: exported.ResourceID,
ResourceName: exported.ResourceName,
Node: exported.Node,
NodeDisplayName: nodeDisplayName,
Instance: exported.Instance,
Message: exported.Message,
Summary: exported.Summary,
Value: exported.Value,
Threshold: exported.Threshold,
StartTime: exported.StartTime,
Acknowledged: exported.Acknowledged,
// exported is a deep clone, so the map is already private.
Metadata: exported.Metadata,
Resolution: modelAlertResolution(exported.Resolution),
},
ResolvedTime: alert.ResolvedTime,
})
}
return resolved
}
// modelAlertResolution projects a non-recovery close into the state model, so
// snapshot and assistant readers can tell a handover from a recovery.
func modelAlertResolution(resolution *AlertResolution) *models.AlertResolution {
if resolution == nil {
return nil
}
return &models.AlertResolution{
Reason: string(resolution.Reason),
SuccessorResourceID: resolution.SuccessorResourceID,
SuccessorName: resolution.SuccessorName,
Summary: resolution.Summary(),
}
}
// GetResolvedAlert returns a copy of a recently resolved alert by ID.
func (m *Manager) GetResolvedAlert(alertID string) *ResolvedAlert {
// Write lock: getResolvedAlertNoLock backfills the resolvedAlias map on a
// canonical-identity miss, so concurrent read-locked callers would race
// each other on that write.
m.resolvedMutex.Lock()
resolved, ok := m.getResolvedAlertNoLock(alertID)
if !ok || resolved == nil || resolved.Alert == nil {
m.resolvedMutex.Unlock()
return nil
}
result := &ResolvedAlert{
Alert: cloneAlertForOutput(resolved.Alert),
ResolvedTime: resolved.ResolvedTime,
}
m.resolvedMutex.Unlock()
m.mu.RLock()
if current := m.resolveNodeDisplayName(result.Alert.Instance, result.Alert.Node); current != "" {
result.Alert.NodeDisplayName = current
}
m.mu.RUnlock()
return result
}
// GetAlertHistory returns alert history. With an event log enabled the
// list is projected from the log's lifecycle snapshots (the authority);
// the in-memory JSON-loaded history is the fallback for managers without
// a log.
func (m *Manager) GetAlertHistory(limit int) []Alert {
if projected, ok := m.AlertHistoryFromEvents(time.Time{}, limit); ok {
return projected
}
return m.applyCurrentNodeDisplayNames(canonicalizeAlertHistoryForOutput(m.historyManager.GetAllHistory(limit)))
}
// GetAlertHistorySince returns alert history entries created after the provided time.
func (m *Manager) GetAlertHistorySince(since time.Time, limit int) []Alert {
if since.IsZero() {
return m.GetAlertHistory(limit)
}
if projected, ok := m.AlertHistoryFromEvents(since, limit); ok {
return projected
}
return m.applyCurrentNodeDisplayNames(canonicalizeAlertHistoryForOutput(m.historyManager.GetHistory(since, limit)))
}
func (m *Manager) applyCurrentNodeDisplayNames(alerts []Alert) []Alert {
m.mu.RLock()
defer m.mu.RUnlock()
for idx := range alerts {
if displayName := m.resolveNodeDisplayName(alerts[idx].Instance, alerts[idx].Node); displayName != "" {
alerts[idx].NodeDisplayName = displayName
}
}
return alerts
}
// ClearAlertHistory clears all alert history. The event log stays
// append-only: the clear is a tombstone event, and the projection ignores
// lifecycle events that precede it.
func (m *Manager) ClearAlertHistory() error {
if store := m.eventLogStore(); store != nil {
if err := store.ImportEvents([]eventlog.Event{{
Type: eventlog.TypeHistoryCleared,
AlertID: "history",
Message: "Alert history cleared by the user.",
}}); err != nil {
return err
}
}
return m.historyManager.ClearAllHistory()
}
// OnAlertHistory registers a callback to be called when alerts are added to history.
// This enables external systems like pattern detection to track alerts.
func (m *Manager) OnAlertHistory(cb AlertCallback) {
if m.historyManager != nil {
m.historyManager.OnAlert(cb)
}
}