Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

7.1 KB
Raw Blame
package alerts

import (
	"github.com/rcourtman/pulse-go-rewrite/internal/alerts/eventlog"
	"strings"
	"time"

	"github.com/rs/zerolog/log"
)

// System-scoped alerts report on Pulse itself rather than on a monitored
// resource. They exist because some failures cannot be observed from the
// outside by the person running Pulse: the clearest case is notification
// delivery, where the channel that would have carried the warning is the thing
// that broke. Routing these through the ordinary alert pipeline puts them in
// the alert list and the navigation badge, which is the only escalation path
// that does not depend on delivery working.
const (
	// SystemAlertResourceName is what the alert list shows in place of a
	// monitored resource name.
	SystemAlertResourceName = "Pulse"

	// SystemAlertIDPrefix namespaces system alert IDs so they cannot collide
	// with a resource-derived alert ID.
	SystemAlertIDPrefix = "pulse-system-"

	// NotificationDeliveryAlertType is the first system alert: configured
	// notification destinations are not delivering.
	NotificationDeliveryAlertType = "notification-delivery"

	// BackupEvaluationAlertType reports unavailable backup-age evaluation,
	// independently of whether backup artifacts themselves are stale.
	BackupEvaluationAlertType = "backup-evaluation"

	// DeadManDeliveryAlertType reports that Pulse is healthy but cannot reach
	// the configured external watchdog.
	DeadManDeliveryAlertType = "deadman-delivery"
	// DeadManMonitoringStalledAlertType reports that the watchdog worker is
	// alive but the canonical monitoring loop has stopped making progress.
	DeadManMonitoringStalledAlertType = "deadman-monitoring-stalled"
	// DeadManInterruptionAlertType records a monitoring availability gap found
	// when Pulse restarts.
	DeadManInterruptionAlertType = "deadman-interruption"
	// DeadManStateAlertType reports loss of the durable restart-gap record.
	DeadManStateAlertType = "deadman-state"
)

// SystemAlertInput describes a system-scoped condition. Type is required and
// identifies the condition; the same Type always maps to the same alert, so
// repeated raises update one alert rather than accumulating.
type SystemAlertInput struct {
	Type    string
	Level   AlertLevel
	Message string
	// Fingerprint, when set, identifies the notify-worthy state of the
	// condition. A re-raise whose level and fingerprint both match the
	// standing alert refreshes the message and metadata silently, so a
	// message carrying a moving counter does not notify on every tick.
	// Callers that leave it empty keep the message itself as the change
	// signal.
	Fingerprint string
	Metadata    map[string]interface{}
}

// systemAlertFingerprintKey stores the raise fingerprint on the alert metadata
// so the next raise can compare against it.
const systemAlertFingerprintKey = "systemAlertFingerprint"

// SystemAlertID returns the stable alert ID for a system alert type.
func SystemAlertID(alertType string) string {
	alertType = strings.TrimSpace(alertType)
	if alertType == "" {
		return ""
	}
	return SystemAlertIDPrefix + alertType
}

// IsSystemAlert reports whether an alert is system-scoped.
func IsSystemAlert(alert *Alert) bool {
	if alert == nil {
		return false
	}
	return strings.HasPrefix(alert.ID, SystemAlertIDPrefix)
}

// RaiseSystemAlert raises a system-scoped alert, or refreshes one that is
// already standing. It is deliberately idempotent: re-raising an unchanged
// condition only advances LastSeen and does not notify again, so a condition
// that is re-evaluated on a timer cannot turn into a notification storm. A
// change of level or message is treated as new information and does notify.
//
// It reports whether this call raised the alert or changed it.
func (m *Manager) RaiseSystemAlert(input SystemAlertInput) bool {
	alertType := strings.TrimSpace(input.Type)
	if alertType == "" {
		return false
	}
	alertID := SystemAlertID(alertType)
	level := NormalizeAlertLevel(input.Level)

	m.mu.Lock()

	now := time.Now()
	existing, exists := m.getActiveAlertNoLock(alertID)
	if exists && existing != nil {
		unchanged := existing.Level == level
		if unchanged {
			existingFingerprint, _ := existing.Metadata[systemAlertFingerprintKey].(string)
			if input.Fingerprint != "" || existingFingerprint != "" {
				unchanged = existingFingerprint == input.Fingerprint
			} else {
				unchanged = existing.Message == input.Message
			}
		}
		existing.LastSeen = now
		if unchanged {
			// Same condition state: keep the presentation current (a message
			// may carry counters) without treating it as new information.
			existing.Message = input.Message
			existing.Metadata = systemAlertMetadata(alertType, input.Fingerprint, input.Metadata)
			m.setActiveAlertNoLock(alertID, existing)
			m.mu.Unlock()
			m.saveActiveAlertsAsync("system-alert-refresh")
			return false
		}

		existing.Level = level
		existing.Message = input.Message
		existing.Metadata = systemAlertMetadata(alertType, input.Fingerprint, input.Metadata)
		m.setActiveAlertNoLock(alertID, existing)
		// dispatchAlert reads flapping and schedule state through the primary
		// lock, so it must run before the unlock rather than after.
		m.dispatchAlert(existing, true)
		m.mu.Unlock()

		m.saveActiveAlertsAsync("system-alert-update")
		log.Info().
			Str("alertID", alertID).
			Str("level", string(level)).
			Msg("System alert updated")
		return true
	}

	alert := &Alert{
		ID:   alertID,
		Type: alertType,
		// A system alert has no monitored resource. ResourceID stays empty so
		// surfaces that link an alert back to a resource skip that affordance
		// rather than offering a link to nothing.
		ResourceID:   "",
		ResourceName: SystemAlertResourceName,
		Level:        level,
		Message:      input.Message,
		StartTime:    now,
		LastSeen:     now,
		Metadata:     systemAlertMetadata(alertType, input.Fingerprint, input.Metadata),
	}

	m.setActiveAlertNoLock(alertID, alert)
	if m.historyManager != nil {
		m.historyManager.AddAlert(*alert)
	}
	m.recordAlertEvent(eventlog.TypeFired, alert, alertID, "system-alert", input.Message, nil)
	m.dispatchAlert(alert, true)
	m.mu.Unlock()

	m.saveActiveAlertsAsync("system-alert-raise")
	log.Warn().
		Str("alertID", alertID).
		Str("level", string(level)).
		Str("message", input.Message).
		Msg("System alert raised")
	return true
}

// ClearSystemAlert clears a standing system alert and reports whether one was
// cleared. Clearing an alert that is not standing is a no-op.
func (m *Manager) ClearSystemAlert(alertType string) bool {
	alertID := SystemAlertID(alertType)
	if alertID == "" {
		return false
	}
	if !m.ClearAlert(alertID) {
		return false
	}
	log.Info().
		Str("alertID", alertID).
		Msg("System alert cleared")
	return true
}

// systemAlertMetadata stamps the marker that surfaces use to tell a
// system-scoped alert apart from a resource alert, without letting a caller
// overwrite it.
func systemAlertMetadata(alertType, fingerprint string, extra map[string]interface{}) map[string]interface{} {
	metadata := make(map[string]interface{}, len(extra)+3)
	for key, value := range extra {
		metadata[key] = value
	}
	metadata["systemAlert"] = true
	metadata["systemAlertType"] = alertType
	if fingerprint != "" {
		metadata[systemAlertFingerprintKey] = fingerprint
	}
	return metadata
}