Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
package alerts
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"sort"
"strconv"
"strings"
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/alerts/eventlog"
"github.com/rcourtman/pulse-go-rewrite/internal/alerts/reducer"
"github.com/rcourtman/pulse-go-rewrite/internal/utils"
"github.com/rs/zerolog/log"
)
func (m *Manager) saveActiveAlertsAsync(context string) {
if m.queueActiveAlertsSave(context) {
go m.runActiveAlertsSaveWorker(m.SaveActiveAlerts)
}
}
// queueActiveAlertsSave reserves at most one worker and one follow-up snapshot.
// Callers may hold m.mu; never wait for a checkpoint under stopMu.
func (m *Manager) queueActiveAlertsSave(context string) bool {
m.stopMu.Lock()
defer m.stopMu.Unlock()
if m.stopping {
return false
}
m.activeSavePending = true
m.activeSaveContext = context
if m.activeSaveRunning {
return false
}
m.activeSaveRunning = true
// Add under the shutdown admission lock, before Stop can begin waiting.
m.workerWG.Add(1)
return true
}
// Each pass takes a fresh snapshot. Requests arriving during a pass mark
// one follow-up, not another goroutine/full snapshot per mutation. No timer
// delays alert persistence, and lifecycle events remain independently durable.
func (m *Manager) runActiveAlertsSaveWorker(save func() error) {
defer m.workerWG.Done()
for {
m.stopMu.Lock()
if m.stopping || !m.activeSavePending {
m.activeSaveRunning = false
m.activeSavePending = false
m.activeSaveContext = ""
m.stopMu.Unlock()
// Stop drains this in-flight worker and owns the final synchronous
// checkpoint; do not drain redundant queued snapshots at shutdown.
return
}
context := m.activeSaveContext
m.activeSavePending = false
m.activeSaveContext = ""
m.stopMu.Unlock()
// Recover per pass so a failed checkpoint cannot strand later requests
// or leave worker admission permanently occupied. Retry only on another
// request (or the existing periodic/shutdown save), never in a busy loop.
func() {
defer func() {
if r := recover(); r != nil {
log.Error().
Interface("panic", r).
Str("context", context).
Msg("Panic in SaveActiveAlerts goroutine")
}
}()
if err := save(); err != nil {
log.Error().
Err(err).
Str("context", context).
Msg("Failed to save active alerts")
}
}()
}
}
// SaveActiveAlerts persists active alerts to disk.
func (m *Manager) SaveActiveAlerts() error {
// Serialize snapshots and writes so concurrent async saves cannot
// overwrite newer state with an older snapshot.
m.saveMu.Lock()
defer m.saveMu.Unlock()
const maxCheckpointAttempts = 8
for attempt := 0; attempt < maxCheckpointAttempts; attempt++ {
failureEpoch := m.activeStateFailureEpoch.Load()
store := m.eventLogStore()
if store != nil && m.activeStateDegraded() {
alerts := m.snapshotActiveAlerts()
markerErr, mirrorErr := m.writeDegradedActiveState(alerts, errors.New("SQLite active alert state remains degraded"))
intentErr := m.saveIntentPendingSnapshot()
if err := errors.Join(markerErr, mirrorErr, intentErr); err != nil {
return fmt.Errorf("failed to checkpoint degraded active alert recovery state: %w", err)
}
return nil
}
revision := int64(0)
if store != nil {
var err error
revision, err = store.ActiveStateRevision()
if err != nil {
alerts := m.snapshotActiveAlerts()
markerErr, mirrorErr := m.writeDegradedActiveState(alerts, err)
if recoveryErr := errors.Join(markerErr, mirrorErr); recoveryErr != nil {
return errors.Join(err, recoveryErr)
}
return fmt.Errorf("failed to read SQLite active alert revision: %w", err)
}
}
alerts := m.snapshotActiveAlerts()
mirrorErr := m.writeActiveAlertsRecoveryMirror(alerts)
intentErr := m.saveIntentPendingSnapshot()
if store == nil {
if err := errors.Join(mirrorErr, intentErr); err != nil {
return err
}
log.Debug().Int("count", len(alerts)).Msg("saved active alert recovery mirror")
return nil
}
snapshots, err := activeStateSnapshots(alerts)
if err != nil {
markerErr, degradedMirrorErr := m.writeDegradedActiveState(alerts, err)
if recoveryErr := errors.Join(markerErr, degradedMirrorErr); recoveryErr != nil {
return errors.Join(err, recoveryErr)
}
return err
}
replaced, err := store.ReplaceActiveStateIfRevision(snapshots, revision)
if err != nil {
markerErr, degradedMirrorErr := m.writeDegradedActiveState(alerts, err)
if recoveryErr := errors.Join(markerErr, degradedMirrorErr); recoveryErr != nil {
return errors.Join(err, recoveryErr)
}
return fmt.Errorf("failed to checkpoint active alerts in SQLite: %w", err)
}
if !replaced {
// A lifecycle event committed after this snapshot began. Retry from
// the manager's current state instead of overwriting that newer event.
continue
}
if m.activeStateFailureEpoch.Load() != failureEpoch {
// A lifecycle append failed while this checkpoint was in flight.
// Retry from current manager state so a stale snapshot cannot clear
// the degradation marker or replace the recovery projection.
continue
}
m.activeStateAuthoritative.Store(true)
if err := errors.Join(mirrorErr, intentErr); err != nil {
return fmt.Errorf("SQLite active alert checkpoint succeeded but recovery persistence failed: %w", err)
}
log.Debug().Int("count", len(alerts)).Msg("saved active alerts to durable state and recovery mirror")
return nil
}
return fmt.Errorf("active alert checkpoint changed during %d consecutive attempts", maxCheckpointAttempts)
}
func (m *Manager) snapshotActiveAlerts() []*Alert {
m.mu.RLock()
alerts := make([]*Alert, 0, len(m.activeAlerts))
for _, alert := range m.activeAlerts {
if alert == nil {
continue
}
clone := alert.Clone()
backfillCanonicalIdentity(clone)
clone.ID = exportedAlertID(clone, clone.ID)
// The live status changes on every poll while a threshold alert
// holds; keeping it out of the checkpoint keeps an unchanged alert
// set byte-identical, so the recovery mirror is not rewritten.
clone.MetricStatus = nil
alerts = append(alerts, clone)
}
m.mu.RUnlock()
return alerts
}
func (m *Manager) writeActiveAlertsRecoveryMirror(alerts []*Alert) error {
m.recoveryMirrorMu.Lock()
defer m.recoveryMirrorMu.Unlock()
return m.writeActiveAlertsRecoveryMirrorLocked(alerts)
}
func (m *Manager) writeActiveAlertsRecoveryMirrorLocked(alerts []*Alert) error {
if m.activeRecoveryWriteBlock.Load() {
return fmt.Errorf("active alert recovery mirror is write-blocked because its startup source was unreadable")
}
alertsDir := m.getAlertsDir()
if err := os.MkdirAll(alertsDir, alertsDirPerm); err != nil {
return fmt.Errorf("failed to create alerts directory: %w", err)
}
// Even chmod to the existing mode dirties directory metadata on Linux.
// Preserve the no-change checkpoint path while still repairing permissions.
info, err := os.Stat(alertsDir)
if err != nil {
return fmt.Errorf("failed to stat alerts directory: %w", err)
}
if info.Mode().Perm() != alertsDirPerm || info.Mode()&(os.ModeSetuid|os.ModeSetgid|os.ModeSticky) != 0 {
if err := os.Chmod(alertsDir, alertsDirPerm); err != nil {
return fmt.Errorf("failed to set alerts directory permissions: %w", err)
}
}
// Snapshots originate from maps. Canonicalise the complete records rather
// than relying on iteration order or assuming legacy IDs are unique.
var records []json.RawMessage
if alerts != nil {
records = make([]json.RawMessage, 0, len(alerts))
}
for _, alert := range alerts {
record, err := json.Marshal(alert)
if err != nil {
return fmt.Errorf("failed to marshal active alerts: %w", err)
}
records = append(records, record)
}
sort.Slice(records, func(i, j int) bool { return bytes.Compare(records[i], records[j]) < 0 })
data, err := json.Marshal(records)
if err != nil {
return fmt.Errorf("failed to marshal active alerts: %w", err)
}
finalFile := filepath.Join(alertsDir, "active-alerts.json")
// Compare the actual file, not a cached hash: missing/corrupt files and
// failed writes must be retried. Never accept a symlink or insecure mode
// as a completed checkpoint. Bound reads by the expected snapshot size.
if info, err := os.Lstat(finalFile); err == nil && info.Mode().IsRegular() &&
info.Mode().Perm() == alertsFilePerm && info.Size() == int64(len(data)) {
if file, err := os.Open(finalFile); err == nil {
existing, readErr := io.ReadAll(io.LimitReader(file, int64(len(data))+1))
closeErr := file.Close()
if readErr == nil && closeErr == nil && bytes.Equal(existing, data) {
// Also retry a directory sync that failed after an earlier rename.
return syncActiveAlertsDirectory(alertsDir)
}
}
}
// Write to temporary file first, then rename. Use a unique temp file so
// periodic saves, explicit saves, and shutdown saves cannot race on a name.
tmpFile, err := os.CreateTemp(alertsDir, "active-alerts-*.json.tmp")
if err != nil {
return fmt.Errorf("failed to create temp file: %w", err)
}
tmpName := tmpFile.Name()
cleanupTemp := true
defer func() {
if !cleanupTemp {
return
}
if err := os.Remove(tmpName); err != nil && !os.IsNotExist(err) {
log.Warn().Err(err).Str("file", tmpName).Msg("Failed to remove temp active alerts file")
}
}()
if _, err := tmpFile.Write(data); err != nil {
writeErr := fmt.Errorf("failed to write active alerts temp file %s: %w", tmpName, err)
if closeErr := tmpFile.Close(); closeErr != nil {
closeErr = fmt.Errorf("failed to close temp file %s after write failure: %w", tmpName, closeErr)
return fmt.Errorf("failed to persist active alerts: %w", errors.Join(writeErr, closeErr))
}
return writeErr
}
if err := tmpFile.Chmod(alertsFilePerm); err != nil {
if closeErr := tmpFile.Close(); closeErr != nil {
log.Warn().Err(closeErr).Str("file", tmpName).Msg("Failed to close temp file after chmod error")
}
return fmt.Errorf("failed to set active alerts temp file permissions: %w", err)
}
if err := tmpFile.Sync(); err != nil {
if closeErr := tmpFile.Close(); closeErr != nil {
return fmt.Errorf("failed to sync and close active alerts temp file %s: %w", tmpName, errors.Join(err, closeErr))
}
return fmt.Errorf("failed to sync active alerts temp file %s: %w", tmpName, err)
}
if err := tmpFile.Close(); err != nil {
return fmt.Errorf("failed to close active alerts temp file %s: %w", tmpName, err)
}
if err := replaceActiveAlertsFile(tmpName, finalFile); err != nil {
return fmt.Errorf("failed to rename active alerts file from %s to %s: %w", tmpName, finalFile, err)
}
if err := os.Chmod(finalFile, alertsFilePerm); err != nil {
return fmt.Errorf("failed to set active alerts file permissions: %w", err)
}
if err := syncActiveAlertsDirectory(alertsDir); err != nil {
return err
}
return nil
}
func (m *Manager) setActiveRecoveryAlert(alert *Alert, fallback string) {
if m == nil || alert == nil {
return
}
clone := alert.Clone()
backfillCanonicalIdentity(clone)
clone.ID = exportedAlertID(clone, clone.ID)
clone.MetricStatus = nil
storageKey := activeAlertStorageKey(clone, fallback)
if storageKey == "" {
return
}
m.activeRecoveryMu.Lock()
if m.activeRecoveryState == nil {
m.activeRecoveryState = make(map[string]*Alert)
}
identity := effectiveAlertID(clone, storageKey)
for key, existing := range m.activeRecoveryState {
if key == storageKey || key == fallback ||
(existing != nil && effectiveAlertID(existing, key) == identity) {
delete(m.activeRecoveryState, key)
}
}
m.activeRecoveryState[storageKey] = clone
m.activeRecoveryMu.Unlock()
}
func (m *Manager) removeActiveRecoveryAlert(alert *Alert, fallback string) {
if m == nil {
return
}
identity := effectiveAlertID(alert, fallback)
canonicalState := ""
if alert != nil {
canonicalState = deriveCanonicalIdentity(alert).State
}
m.activeRecoveryMu.Lock()
for key, existing := range m.activeRecoveryState {
if key == fallback || key == identity || key == canonicalState ||
(existing != nil && effectiveAlertID(existing, key) == identity) {
delete(m.activeRecoveryState, key)
}
}
m.activeRecoveryMu.Unlock()
}
func (m *Manager) activeRecoverySnapshot() []*Alert {
m.activeRecoveryMu.RLock()
keys := make([]string, 0, len(m.activeRecoveryState))
for key := range m.activeRecoveryState {
keys = append(keys, key)
}
sort.Strings(keys)
alerts := make([]*Alert, 0, len(keys))
for _, key := range keys {
if alert := m.activeRecoveryState[key]; alert != nil {
alerts = append(alerts, alert.Clone())
}
}
m.activeRecoveryMu.RUnlock()
return alerts
}
func (m *Manager) checkpointActiveRecoveryAfterDurableFailure(cause error) error {
if m == nil {
return nil
}
m.recoveryMirrorMu.Lock()
alerts := m.activeRecoverySnapshot()
markerErr := m.writeActiveStateDegradedRecovery(alerts, cause)
mirrorErr := m.writeActiveAlertsRecoveryMirrorLocked(alerts)
m.recoveryMirrorMu.Unlock()
m.activeStateAuthoritative.Store(false)
return errors.Join(markerErr, mirrorErr)
}
func (m *Manager) writeDegradedActiveState(alerts []*Alert, cause error) (markerErr, mirrorErr error) {
if m == nil {
return nil, nil
}
m.activeStateAuthoritative.Store(false)
m.recoveryMirrorMu.Lock()
defer m.recoveryMirrorMu.Unlock()
markerErr = m.writeActiveStateDegradedRecovery(alerts, cause)
mirrorErr = m.writeActiveAlertsRecoveryMirrorLocked(alerts)
return markerErr, mirrorErr
}
// LoadActiveAlerts restores active alerts from disk.
func (m *Manager) LoadActiveAlerts() error {
m.activeRecoveryReadable.Store(false)
m.activeRecoveryWriteBlock.Store(false)
alertsFile := filepath.Join(m.getAlertsDir(), "active-alerts.json")
data, err := readLimitedRegularFile(alertsFile, maxActiveAlertsFileSizeBytes)
if err != nil {
if errors.Is(err, os.ErrNotExist) {
m.activeRecoveryReadable.Store(true)
log.Info().Msg("No active alerts file found, starting fresh")
m.mu.Lock()
defer m.mu.Unlock()
if err := m.loadIntentPendingNoLock(); err != nil {
return err
}
m.seedReducerCoreNoLock()
return nil
}
m.activeRecoveryWriteBlock.Store(true)
return fmt.Errorf("failed to read active alerts: %w", err)
}
var alerts []*Alert
if err := json.Unmarshal(data, &alerts); err != nil {
m.activeRecoveryWriteBlock.Store(true)
return fmt.Errorf("failed to unmarshal active alerts: %w", err)
}
if err := os.Chmod(alertsFile, alertsFilePerm); err != nil && !os.IsNotExist(err) {
log.Warn().Err(err).Str("file", alertsFile).Msg("Failed to harden active alerts file permissions")
}
if err := m.restoreActiveAlertSnapshots(alerts, "JSON recovery mirror", false); err != nil {
return err
}
m.activeRecoveryReadable.Store(true)
return nil
}
func (m *Manager) restoreActiveAlertSnapshots(alerts []*Alert, source string, replace bool) error {
m.mu.Lock()
if replace {
m.activeAlerts = make(map[string]*Alert)
m.restoredBackupNotifications = nil
m.activeAlertAlias = make(map[string]string)
m.ackState = make(map[string]ackRecord)
m.ackStateByCanonical = make(map[string]ackRecord)
m.core = reducer.NewState()
}
restoredCount := 0
duplicateCount := 0
identityMigrationCount := 0
seen := make(map[string]bool)
restoredCritical := make([]*Alert, 0)
for _, alert := range alerts {
if alert == nil {
continue
}
alert.Level = NormalizeAlertLevel(alert.Level)
// Migrate legacy guest alert IDs (instance-node-VMID -> instance-VMID).
isGuestAlert := strings.Contains(alert.Type, "cpu") || strings.Contains(alert.Type, "memory") ||
strings.Contains(alert.Type, "disk") || strings.Contains(alert.Type, "network") ||
alert.Type == "guest-offline"
if isGuestAlert {
parts := strings.Split(alert.ResourceID, "-")
if alert.Node != "" && len(parts) >= 2 {
var newResourceID string
vmidStr := parts[len(parts)-1]
if _, err := strconv.Atoi(vmidStr); err == nil {
if len(parts) == 3 && alert.Instance != "" && alert.Instance != alert.Node {
newResourceID = fmt.Sprintf("%s-%s", alert.Instance, vmidStr)
} else if len(parts) == 2 && alert.Instance == alert.Node {
newResourceID = fmt.Sprintf("%s-%s", alert.Instance, vmidStr)
}
if newResourceID != "" && newResourceID != alert.ResourceID {
log.Info().
Str("oldID", alert.ResourceID).
Str("newID", newResourceID).
Str("alertType", alert.Type).
Msg("Migrating active alert from legacy guest ID format")
oldResourceID := alert.ResourceID
alert.ResourceID = newResourceID
alert.ID = strings.Replace(alert.ID, oldResourceID, newResourceID, 1)
alert.CanonicalSpecID = ""
alert.CanonicalState = ""
alert.CanonicalKind = ""
if alert.Metadata != nil {
delete(alert.Metadata, "canonicalSpecID")
delete(alert.Metadata, "canonicalAlertKind")
}
}
}
}
}
legacyID := alert.ID
backfillCanonicalIdentity(alert)
alert.ID = exportedAlertID(alert, alert.ID)
if alert.ID != legacyID {
identityMigrationCount++
}
if seen[alert.ID] {
duplicateCount++
log.Warn().Str("alertID", alert.ID).Msg("skipping duplicate alert during restore")
continue
}
seen[alert.ID] = true
if !m.setActiveAlertNoLock(alert.ID, alert) {
continue
}
if legacyID != alert.ID {
if m.activeAlertAlias == nil {
m.activeAlertAlias = make(map[string]string)
}
m.activeAlertAlias[legacyID] = activeAlertStorageKey(alert, alert.ID)
}
if alert.Acknowledged {
ackTime := alert.StartTime
if alert.AckTime != nil {
ackTime = *alert.AckTime
}
m.setAckRecordNoLock(alert, alert.ID, ackRecord{
acknowledged: true,
user: alert.AckUser,
time: ackTime,
})
}
restoredCount++
if alert.Level == AlertLevelCritical {
if alert.Type == "backup-age" {
if m.restoredBackupNotifications == nil {
m.restoredBackupNotifications = make(map[string]time.Time)
}
m.restoredBackupNotifications[canonicalTrackingKeyForAlert(alert)] = alert.StartTime
} else {
restoredCritical = append(restoredCritical, alert.Clone())
}
}
}
if err := m.loadIntentPendingNoLock(); err != nil {
m.mu.Unlock()
return err
}
// Rebuild the reducer from the exact restored active set. Active incidents
// never expire merely because Pulse was restarted; only fresh healthy
// evidence or an explicit operator action may resolve them.
m.seedReducerCoreNoLock()
m.mu.Unlock()
log.Info().
Int("restored", restoredCount).
Int("total", len(alerts)).
Int("duplicates", duplicateCount).
Int("identityMigrations", identityMigrationCount).
Str("source", source).
Msg("Restored active alerts")
epoch := m.restoredAlertEpoch.Add(1)
for _, alert := range restoredCritical {
m.scheduleRestoredCriticalAlert(alert, epoch)
}
if identityMigrationCount > 0 {
// The asynchronous save waits for the load lock, then atomically replaces
// the legacy file with the canonical IDs just restored into memory.
m.saveActiveAlertsAsync("canonical-identity-migration")
}
return nil
}
func activeStateSnapshots(alerts []*Alert) ([]eventlog.ActiveStateSnapshot, error) {
snapshots := make([]eventlog.ActiveStateSnapshot, 0, len(alerts))
for _, alert := range alerts {
if alert == nil {
continue
}
snapshot, err := json.Marshal(alert)
if err != nil {
return nil, fmt.Errorf("failed to marshal active alert %q for SQLite: %w", alert.ID, err)
}
snapshots = append(snapshots, eventlog.ActiveStateSnapshot{
AlertID: alert.ID,
OccurrenceStartedAt: alert.StartTime,
UpdatedAt: time.Now().UTC(),
Snapshot: snapshot,
})
}
return snapshots, nil
}
func (m *Manager) scheduleRestoredCriticalAlert(alert *Alert, epoch uint64) {
if m == nil || alert == nil {
return
}
go func(a *Alert) {
delay := time.NewTimer(10 * time.Second)
defer func() {
if !delay.Stop() {
select {
case <-delay.C:
default:
}
}
}()
select {
case <-delay.C:
if m.restoredAlertEpoch.Load() != epoch {
return
}
log.Info().
Str("alertID", a.ID).
Str("resource", a.ResourceName).
Msg("Attempting to send notification for restored critical alert")
m.mu.Lock()
if active, ok := m.getActiveAlertNoLock(a.ID); ok && active.StartTime.Equal(a.StartTime) {
m.dispatchAlert(active, false)
}
m.mu.Unlock()
case <-m.escalationStop:
log.Debug().
Str("alertID", a.ID).
Msg("Cancelled startup notification due to shutdown")
}
}(alert.Clone())
}
func (m *Manager) getAlertsDir() string {
if strings.TrimSpace(m.alertsDir) != "" {
return m.alertsDir
}
return filepath.Join(utils.GetDataDir(), "alerts")
}
// periodicSaveAlerts saves active alerts to disk periodically.
func (m *Manager) periodicSaveAlerts() {
ticker := time.NewTicker(1 * time.Minute)
defer ticker.Stop()
for {
select {
case <-ticker.C:
if err := m.SaveActiveAlerts(); err != nil {
log.Error().Err(err).Msg("failed to save active alerts during periodic save")
}
case <-m.escalationStop:
return
}
}
}