Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

32.0 KB
Raw Blame
package alerts

import (
	"strings"
	"time"

	"github.com/rcourtman/pulse-go-rewrite/internal/alerts/eventlog"
	"github.com/rcourtman/pulse-go-rewrite/internal/alerts/reducer"
	alertspecs "github.com/rcourtman/pulse-go-rewrite/internal/alerts/specs"
	"github.com/rcourtman/pulse-go-rewrite/internal/operationaltrust"
	"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
	"github.com/rs/zerolog/log"
)

type canonicalLifecycleAlertParams struct {
	Spec                 alertspecs.ResourceAlertSpec
	Evidence             alertspecs.AlertEvidence
	IntentSignal         string
	PolicyDisabledNoLock func() bool
	AlertID              string
	AlertType            string
	ResourceID           string
	ResourceName         string
	Node                 string
	Instance             string
	Message              string
	// Summary is Message without its subject; see Alert.Summary.
	Summary                      string
	Value                        float64
	Threshold                    float64
	Correlation                  *AlertCorrelation
	Metadata                     map[string]interface{}
	AddToRecent                  bool
	AddToHistory                 bool
	RateLimit                    bool
	DispatchAsync                bool
	NotifyOnSeverityChange       bool
	AddToHistoryOnSeverityChange bool
	IntentBackup                 BackupIntentContext
}

type canonicalStatefulAlertParams struct {
	Spec                  alertspecs.ResourceAlertSpec
	Evidence              alertspecs.AlertEvidence
	RecoveryConfirmations int
	AlertID               string
	AlertType             string
	ResourceID            string
	ResourceName          string
	Node                  string
	Instance              string
	Message               string
	// Summary is Message without its subject; see Alert.Summary.
	Summary                      string
	Value                        float64
	Threshold                    float64
	StartTimeOverride            time.Time
	Metadata                     map[string]interface{}
	AddToRecent                  bool
	AddToHistory                 bool
	MessageBuilder               func(alertspecs.EvaluationResult) (string, float64, float64)
	RateLimit                    bool
	NotifyOnSeverityChange       bool
	AddToHistoryOnSeverityChange bool
	DispatchAsync                bool
}

func buildCanonicalConnectivitySpec(resourceID, title string, resourceType unifiedresources.ResourceType, severity AlertLevel, confirmations int, disabled bool) (alertspecs.ResourceAlertSpec, error) {
	spec := alertspecs.ResourceAlertSpec{
		ID:                    resourceID + "-connectivity",
		ResourceID:            resourceID,
		ResourceType:          resourceType,
		Kind:                  alertspecs.AlertSpecKindConnectivity,
		Severity:              canonicalAlertSeverity(severity),
		Title:                 title,
		Disabled:              disabled,
		ConfirmationsRequired: confirmations,
		Connectivity: &alertspecs.ConnectivitySpec{
			Signal:    "status",
			LostAfter: time.Second,
		},
	}

	return spec, spec.Validate()
}

func buildCanonicalPoweredStateSpec(resourceID, title string, resourceType unifiedresources.ResourceType, severity AlertLevel, confirmations int, disabled bool) (alertspecs.ResourceAlertSpec, error) {
	spec := alertspecs.ResourceAlertSpec{
		ID:                    resourceID + "-powered-state",
		ResourceID:            resourceID,
		ResourceType:          resourceType,
		Kind:                  alertspecs.AlertSpecKindPoweredState,
		Severity:              canonicalAlertSeverity(severity),
		Title:                 title,
		Disabled:              disabled,
		ConfirmationsRequired: confirmations,
		PoweredState: &alertspecs.PoweredStateSpec{
			Expected: alertspecs.PowerStateOn,
		},
	}

	return spec, spec.Validate()
}

func buildCanonicalDiscreteStateSpec(resourceID, title string, resourceType unifiedresources.ResourceType, severity AlertLevel, confirmations int, disabled bool, stateKey string, triggerStates []string) (alertspecs.ResourceAlertSpec, error) {
	spec := alertspecs.ResourceAlertSpec{
		ID:                    resourceID + "-" + stateKey,
		ResourceID:            resourceID,
		ResourceType:          resourceType,
		Kind:                  alertspecs.AlertSpecKindDiscreteState,
		Severity:              canonicalAlertSeverity(severity),
		Title:                 title,
		Disabled:              disabled,
		ConfirmationsRequired: confirmations,
		DiscreteState: &alertspecs.DiscreteStateSpec{
			StateKey:      stateKey,
			TriggerStates: append([]string(nil), triggerStates...),
		},
	}

	return spec, spec.Validate()
}

func buildCanonicalServiceGapSpec(resourceID, title string, resourceType unifiedresources.ResourceType, service string, warningPercent, criticalPercent float64, disabled bool) (alertspecs.ResourceAlertSpec, error) {
	if criticalPercent > 0 && warningPercent > 0 && criticalPercent < warningPercent {
		warningPercent = criticalPercent
	}
	spec := alertspecs.ResourceAlertSpec{
		ID:           resourceID + "-service-gap",
		ResourceID:   resourceID,
		ResourceType: resourceType,
		Kind:         alertspecs.AlertSpecKindServiceGap,
		Severity:     alertspecs.AlertSeverityWarning,
		Title:        title,
		Disabled:     disabled,
		ServiceGap: &alertspecs.ServiceGapSpec{
			Service:         service,
			WarningPercent:  warningPercent,
			CriticalPercent: criticalPercent,
		},
	}

	return spec, spec.Validate()
}

func buildCanonicalSeverityThresholdSpecWithDirection(specID, resourceID, title string, resourceType unifiedresources.ResourceType, metric string, direction alertspecs.ThresholdDirection, warning, critical float64, disabled bool) (alertspecs.ResourceAlertSpec, error) {
	spec := alertspecs.ResourceAlertSpec{
		ID:           specID,
		ResourceID:   resourceID,
		ResourceType: resourceType,
		Kind:         alertspecs.AlertSpecKindSeverityThreshold,
		Severity:     alertspecs.AlertSeverityWarning,
		Title:        title,
		Disabled:     disabled,
		SeverityThreshold: &alertspecs.SeverityThresholdSpec{
			Metric:    metric,
			Direction: direction,
			Warning:   warning,
			Critical:  critical,
		},
	}

	return spec, spec.Validate()
}

func buildCanonicalSeverityThresholdSpec(specID, resourceID, title string, resourceType unifiedresources.ResourceType, metric string, warning, critical float64, disabled bool) (alertspecs.ResourceAlertSpec, error) {
	return buildCanonicalSeverityThresholdSpecWithDirection(specID, resourceID, title, resourceType, metric, alertspecs.ThresholdDirectionAbove, warning, critical, disabled)
}

func buildCanonicalSeverityThresholdSpecWithRecovery(specID, resourceID, title string, resourceType unifiedresources.ResourceType, metric string, warning, critical float64, recovery *float64, disabled bool) (alertspecs.ResourceAlertSpec, error) {
	spec, err := buildCanonicalSeverityThresholdSpec(specID, resourceID, title, resourceType, metric, warning, critical, disabled)
	if err != nil {
		return spec, err
	}
	spec.SeverityThreshold.Recovery = recovery
	return spec, spec.Validate()
}

func buildCanonicalChangeThresholdSpec(specID, resourceID, title string, resourceType unifiedresources.ResourceType, metric string, warningCurrent, criticalCurrent, warningDelta, criticalDelta, warningPercent, criticalPercent float64, window time.Duration, disabled bool) (alertspecs.ResourceAlertSpec, error) {
	spec := alertspecs.ResourceAlertSpec{
		ID:           specID,
		ResourceID:   resourceID,
		ResourceType: resourceType,
		Kind:         alertspecs.AlertSpecKindChangeThreshold,
		Severity:     alertspecs.AlertSeverityWarning,
		Title:        title,
		Disabled:     disabled,
		ChangeThreshold: &alertspecs.ChangeThresholdSpec{
			Metric:          metric,
			ReferenceWindow: window,
			WarningCurrent:  warningCurrent,
			CriticalCurrent: criticalCurrent,
			WarningDelta:    warningDelta,
			CriticalDelta:   criticalDelta,
			WarningPercent:  warningPercent,
			CriticalPercent: criticalPercent,
		},
	}

	return spec, spec.Validate()
}

func buildCanonicalBaselineAnomalySpec(specID, resourceID, title string, resourceType unifiedresources.ResourceType, metric string, confirmations int, disabled bool) (alertspecs.ResourceAlertSpec, error) {
	spec := alertspecs.ResourceAlertSpec{
		ID:                    specID,
		ResourceID:            resourceID,
		ResourceType:          resourceType,
		Kind:                  alertspecs.AlertSpecKindBaselineAnomaly,
		Severity:              alertspecs.AlertSeverityWarning,
		Title:                 title,
		Disabled:              disabled,
		ConfirmationsRequired: confirmations,
		BaselineAnomaly: &alertspecs.BaselineAnomalySpec{
			Metric:             metric,
			QuietBaseline:      40,
			WarningRatio:       1.8,
			CriticalRatio:      2.5,
			WarningDelta:       150,
			CriticalDelta:      300,
			QuietWarningDelta:  60,
			QuietCriticalDelta: 120,
		},
	}

	return spec, spec.Validate()
}

func buildCanonicalHealthAssessmentSpec(specID, resourceID, title string, resourceType unifiedresources.ResourceType, signal string, codes []string, disabled bool) (alertspecs.ResourceAlertSpec, error) {
	spec := alertspecs.ResourceAlertSpec{
		ID:           specID,
		ResourceID:   resourceID,
		ResourceType: resourceType,
		Kind:         alertspecs.AlertSpecKindHealthAssessment,
		Severity:     alertspecs.AlertSeverityWarning,
		Title:        title,
		Disabled:     disabled,
		HealthAssessment: &alertspecs.HealthAssessmentSpec{
			Signal: signal,
			Codes:  append([]string(nil), codes...),
		},
	}

	return spec, spec.Validate()
}

func buildCanonicalPostureThresholdSpec(specID, resourceID, title string, resourceType unifiedresources.ResourceType, ageMetric string, warningAge, criticalAge float64, sizeMetric string, warningSize, criticalSize float64, disabled bool) (alertspecs.ResourceAlertSpec, error) {
	spec := alertspecs.ResourceAlertSpec{
		ID:           specID,
		ResourceID:   resourceID,
		ResourceType: resourceType,
		Kind:         alertspecs.AlertSpecKindPostureThreshold,
		Severity:     alertspecs.AlertSeverityWarning,
		Title:        title,
		Disabled:     disabled,
		PostureThreshold: &alertspecs.PostureThresholdSpec{
			AgeMetric:    ageMetric,
			WarningAge:   warningAge,
			CriticalAge:  criticalAge,
			SizeMetric:   sizeMetric,
			WarningSize:  warningSize,
			CriticalSize: criticalSize,
		},
	}

	return spec, spec.Validate()
}

func canonicalAlertSeverity(level AlertLevel) alertspecs.AlertSeverity {
	switch level {
	case AlertLevelCritical:
		return alertspecs.AlertSeverityCritical
	case AlertLevelInfo:
		return alertspecs.AlertSeverityInfo
	default:
		return alertspecs.AlertSeverityWarning
	}
}

func (m *Manager) evaluateCanonicalLifecycleAlert(params canonicalLifecycleAlertParams) (alertspecs.EvaluationResult, bool) {
	if params.Evidence.ObservedAt.IsZero() {
		params.Evidence.ObservedAt = time.Now()
	}

	m.mu.Lock()
	migratedAlertIdentity := false
	defer func() {
		if migratedAlertIdentity {
			m.saveActiveAlertsAsync("guest lifecycle alert node move")
		}
	}()
	defer m.mu.Unlock()

	// Recheck mutable policy while holding the same lock used for lifecycle
	// state and dispatch. This closes the save-vs-dispatch race where policy
	// could be disabled after a detector's initial snapshot but before it
	// activated the alert.
	if params.PolicyDisabledNoLock != nil && params.PolicyDisabledNoLock() {
		params.Spec.Disabled = true
	}

	storageKey := canonicalTrackingKeyForSpec(params.Spec, params.AlertID)
	trackingKey := storageKey

	// Shadow feed: after this evaluation settles (any exit), replay the same
	// observation through the shadow reducer and compare outcomes. Runs
	// before the deferred unlock (LIFO), so it still holds m.mu.
	var shadowIntent *reducer.DiscreteIntent
	shadowEligible := false
	defer func() {
		if shadowEligible {
			m.shadowObserveLifecycleNoLock(params.Spec, params.Evidence, shadowIntent, storageKey)
		}
	}()

	var existing *Alert
	if current, ok := m.getActiveAlertNoLock(storageKey); ok {
		existing = current
	} else if migrated := m.migrateGuestAlertNoLock(storageKey, params.Spec.ID, string(params.Spec.Kind), params.Spec.ResourceID, params.ResourceName, params.Node, params.Instance, string(params.Spec.ResourceType)); migrated != nil {
		existing = migrated
		migratedAlertIdentity = true
	}

	// Validate the spec and evidence exactly as the evaluator did, so
	// malformed input is skipped rather than misread as a recovery.
	if err := params.Spec.Validate(); err != nil {
		log.Warn().
			Err(err).
			Str("alertID", storageKey).
			Str("resourceID", params.ResourceID).
			Str("specID", params.Spec.ID).
			Msg("Skipping invalid canonical lifecycle evaluation")
		return alertspecs.EvaluationResult{}, false
	}
	if err := alertspecs.ValidateEvidence(params.Spec.Kind, params.Evidence); err != nil {
		log.Warn().
			Err(err).
			Str("alertID", storageKey).
			Str("resourceID", params.ResourceID).
			Str("specID", params.Spec.ID).
			Msg("Skipping invalid canonical lifecycle evidence")
		return alertspecs.EvaluationResult{}, false
	}

	matched, matchSeverity, matchReason := alertspecs.Match(params.Spec, params.Evidence)

	// Intent context resolves before the transition, exactly when the old
	// engine consulted it: only while no alert is active. Its bookkeeping
	// (intentPending, preview surfaces) is preserved; the gate itself now
	// lives in the reducer.
	intentSignal := params.IntentSignal
	if intentSignal == "" && (params.Spec.Kind == alertspecs.AlertSpecKindConnectivity || params.Spec.Kind == alertspecs.AlertSpecKindPoweredState) {
		intentSignal = string(AlertIntentSignalOffline)
	}
	var intent *reducer.DiscreteIntent
	if intentSignal != "" && existing == nil {
		decision := m.evaluateIntentNoLock(params.Spec.ResourceID, string(params.Spec.ResourceType), intentSignal, storageKey, params.Evidence.ObservedAt, matched, params.IntentBackup)
		intent = &reducer.DiscreteIntent{
			Explicit:     decision.Effective.Explicit,
			GraceSeconds: decision.Effective.GraceSeconds,
			// Operator suppression only: the backup-offline deferral is
			// modeled independently below so the reducer computes its own
			// hold rather than echoing the manager's.
			OperatorSuppressed: decision.Suppressed && strings.HasPrefix(decision.Reason, "operator_"),
			OperatorReason:     decision.Reason,
		}
		if backupOffline := decision.Effective.BackupOffline; backupOffline != nil && backupOffline.Enabled && intentSignal == string(AlertIntentSignalOffline) {
			intent.BackupEnabled = true
			intent.BackupActive = params.IntentBackup.Active
			intent.BackupPostGraceSeconds = backupOffline.PostGraceSeconds
			intent.BackupMaxDeferralSeconds = backupOffline.MaxDeferralSeconds
		}
		shadowIntent = intent
		if decision.StateChanged {
			m.saveActiveAlertsAsync("lifecycle intent state")
		}
	}

	severity := reducer.SeverityWarning
	if matchSeverity == alertspecs.AlertSeverityCritical {
		severity = reducer.SeverityCritical
	}

	// An existing alert the core does not know about (guest migration,
	// direct injection, any unhooked path) means it was firing: adopt it,
	// exactly as the old engine reconstructed previous state as firing
	// whenever an active alert existed.
	if existing != nil {
		if _, known := m.core.Incident(params.Spec.ResourceID, params.Spec.ID); !known {
			ackAt := time.Time{}
			if existing.AckTime != nil {
				ackAt = *existing.AckTime
			}
			m.core.SeedFiringIncident(
				params.Spec.ResourceID,
				params.Spec.ID,
				shadowSeverityForLevel(existing.Level),
				existing.StartTime,
				existing.Acknowledged,
				existing.AckUser,
				ackAt,
			)
		}
	}

	events := m.core.ApplyDiscrete(reducer.DiscreteSignal{
		ResourceID:       params.Spec.ResourceID,
		Key:              params.Spec.ID,
		Matched:          matched,
		Severity:         severity,
		RuntimeTick:      m.intentTickNoLock(),
		RuntimeTickValid: true,
		ObservedAt:       params.Evidence.ObservedAt,
	}, reducer.DiscreteRule{
		Confirmations: specConfirmationsRequired(params.Spec),
		Disabled:      params.Spec.Disabled,
		Intent:        intent,
	})
	primary := reducer.EventType("")
	if len(events) > 0 {
		primary = events[0].Type
	}

	shadowEligible = true

	incident, hasIncident := m.core.Incident(params.Spec.ResourceID, params.Spec.ID)

	// Synthesize the evaluator-shaped result the callers consume.
	result := alertspecs.EvaluationResult{}
	result.State.SpecID = params.Spec.ID
	result.State.LastObservedAt = params.Evidence.ObservedAt
	result.State.Reason = matchReason

	transition := func(kind alertspecs.EvaluationTransitionKind, from, to alertspecs.AlertState) *alertspecs.EvaluationTransition {
		return &alertspecs.EvaluationTransition{
			Kind:       kind,
			SpecID:     params.Spec.ID,
			ResourceID: params.Spec.ResourceID,
			From:       from,
			To:         to,
			At:         params.Evidence.ObservedAt,
			Severity:   matchSeverity,
			Reason:     matchReason,
			Evidence:   params.Evidence,
		}
	}

	if hasIncident && incident.State == reducer.StatePending {
		result.State.State = alertspecs.AlertStatePending
		result.State.Severity = matchSeverity
		result.State.ConsecutiveMatches = incident.Confirmations
		result.State.FirstMatchedAt = incident.PendingSince
		if intent != nil && intent.OperatorReason != "" {
			result.State.Reason = intent.OperatorReason
		}
		if primary == reducer.EventPending {
			result.Transition = transition(alertspecs.EvaluationTransitionPending, alertspecs.AlertStateClear, alertspecs.AlertStatePending)
		}
		return result, true
	}

	if hasIncident && incident.State == reducer.StateFiring {
		level := AlertLevelWarning
		if incident.Severity == reducer.SeverityCritical {
			level = AlertLevelCritical
		}
		alert := &Alert{
			ID:           storageKey,
			Type:         params.AlertType,
			Level:        level,
			ResourceID:   params.Spec.ResourceID,
			ResourceName: params.ResourceName,
			Node:         params.Node,
			Instance:     params.Instance,
			Message:      params.Message,
			Summary:      params.Summary,
			Value:        params.Value,
			Threshold:    params.Threshold,
			StartTime:    incident.StartedAt,
			LastSeen:     params.Evidence.ObservedAt,
			Correlation:  cloneAlertCorrelation(params.Correlation),
			Metadata:     cloneMetadata(params.Metadata),
		}
		if alert.Metadata == nil {
			alert.Metadata = make(map[string]interface{}, 2)
		}
		if _, ok := alert.Metadata["resourceType"]; !ok && params.Spec.ResourceType != "" {
			alert.Metadata["resourceType"] = string(params.Spec.ResourceType)
		}
		applyCanonicalIdentity(alert, params.Spec.ID, string(params.Spec.Kind))
		applyCanonicalOperationalEvidence(alert, params.Spec, params.Evidence, time.Now())
		m.preserveAlertState(storageKey, alert)
		// The reducer core is authoritative for occurrence start and
		// acknowledgement restoration.
		alert.StartTime = incident.StartedAt
		alert.Acknowledged = incident.Acknowledged
		alert.AckUser = incident.AckUser
		if incident.Acknowledged && !incident.AckAt.IsZero() {
			ackAt := incident.AckAt
			alert.AckTime = &ackAt
		} else if !incident.Acknowledged {
			alert.AckTime = nil
			alert.AckUser = ""
		}
		if !m.setActiveAlertNoLock(storageKey, alert) {
			result.State.State = alertspecs.AlertStateSuppressed
			result.State.Reason = "operator-suppressed"
			return result, true
		}
		if params.AddToRecent {
			m.recentAlerts[trackingKey] = alert
		}

		result.State.State = alertspecs.AlertStateFiring
		result.State.Severity = matchSeverity
		result.State.ConsecutiveMatches = incident.Confirmations
		result.State.FirstMatchedAt = incident.StartedAt
		result.State.ActiveSince = incident.StartedAt

		if existing != nil {
			if primary == reducer.EventSeverityChanged {
				result.Transition = transition(alertspecs.EvaluationTransitionSeverityChanged, alertspecs.AlertStateFiring, alertspecs.AlertStateFiring)
				if params.AddToHistoryOnSeverityChange {
					m.historyManager.AddAlertTransition(*alert)
				}
				if params.NotifyOnSeverityChange {
					if !params.RateLimit || m.checkRateLimit(trackingKey) {
						m.dispatchAlert(alert, params.DispatchAsync)
					}
				}
			}
			return result, true
		}

		// Activation. Intent bookkeeping closes out exactly as before.
		if intent != nil && intent.Explicit {
			m.clearIntentPendingNoLock(storageKey)
			m.saveActiveAlertsAsync("lifecycle intent activated")
		}
		result.Transition = transition(alertspecs.EvaluationTransitionActivated, alertspecs.AlertStatePending, alertspecs.AlertStateFiring)

		firedType := eventlog.TypeFired
		if primary == reducer.EventRefired {
			firedType = eventlog.TypeRefired
		}
		m.recordAlertEvent(firedType, alert, storageKey, matchReason, params.Message, nil)

		// Consume the recently-resolved entry for map hygiene and the
		// new-occurrence history distinction; the reducer already decided
		// whether this is a reactivation (EventRefired) and restored the
		// start time.
		_, _, _, hadResolvedOccurrence := m.consumeRecentlyResolvedForRefireWithPrimaryLock(storageKey, time.Now())
		if primary == reducer.EventRefired {
			if params.AddToHistory {
				m.historyManager.UpdateAlertLastSeenForAlert(alert, alert.LastSeen)
			}
			log.Debug().
				Str("alertID", storageKey).
				Msg("Alert re-fired within cooldown, reactivated without new history entry")
			if params.RateLimit && !m.checkRateLimit(trackingKey) {
				return result, true
			}
			m.dispatchAlert(alert, params.DispatchAsync)
			return result, true
		}

		if params.AddToHistory {
			if hadResolvedOccurrence {
				m.historyManager.AddAlertTransition(*alert)
			} else {
				m.historyManager.AddAlert(*alert)
			}
		}

		if params.RateLimit && !m.checkRateLimit(trackingKey) {
			log.Debug().
				Str("alertID", storageKey).
				Str("trackingKey", trackingKey).
				Int("maxPerHour", m.config.Schedule.MaxAlertsHour).
				Msg("Lifecycle alert notification suppressed due to rate limit")
			return result, true
		}

		m.dispatchAlert(alert, params.DispatchAsync)
		return result, true
	}

	// Clear: the core holds no incident for this key.
	result.State.State = alertspecs.AlertStateClear
	if existing == nil {
		return result, true
	}

	result.Transition = transition(alertspecs.EvaluationTransitionRecovered, alertspecs.AlertStateFiring, alertspecs.AlertStateClear)
	m.removeActiveAlertNoLock(storageKey)
	recoveryEvidence, hasRecoveryEvidence := canonicalAlertEvidenceEnvelope(
		params.Spec,
		params.Evidence,
		existing.Instance,
		time.Now(),
	)
	var recoveryEvidenceRef *operationaltrust.EvidenceEnvelope
	if hasRecoveryEvidence {
		recoveryEvidenceRef = &recoveryEvidence
	}
	resolvedAlert := m.newResolvedAlert(
		existing,
		params.Evidence.ObservedAt,
		recoveryEvidenceRef,
	)
	m.addRecentlyResolvedWithPrimaryLock(resolvedAlert)
	m.safeCallResolvedAlertCallback(resolvedAlert, storageKey, true)
	return result, true
}

func (m *Manager) evaluateCanonicalStatefulAlert(params canonicalStatefulAlertParams) (alertspecs.EvaluationResult, bool) {
	if params.Evidence.ObservedAt.IsZero() {
		params.Evidence.ObservedAt = time.Now()
	}

	m.mu.Lock()
	migratedAlertIdentity := false
	defer func() {
		if migratedAlertIdentity {
			m.saveActiveAlertsAsync("guest stateful alert node move")
		}
	}()
	defer m.mu.Unlock()

	storageKey := canonicalTrackingKeyForSpec(params.Spec, params.AlertID)
	trackingKey := storageKey

	var existing *Alert
	if current, ok := m.getActiveAlertNoLock(storageKey); ok {
		existing = current
	} else if migrated := m.migrateGuestAlertNoLock(storageKey, params.Spec.ID, string(params.Spec.Kind), params.Spec.ResourceID, params.ResourceName, params.Node, params.Instance, string(params.Spec.ResourceType)); migrated != nil {
		existing = migrated
		migratedAlertIdentity = true
	}

	if err := params.Spec.Validate(); err != nil {
		log.Warn().
			Err(err).
			Str("alertID", storageKey).
			Str("resourceID", params.ResourceID).
			Str("specID", params.Spec.ID).
			Msg("Skipping invalid canonical stateful evaluation")
		return alertspecs.EvaluationResult{}, false
	}
	if err := alertspecs.ValidateEvidence(params.Spec.Kind, params.Evidence); err != nil {
		log.Warn().
			Err(err).
			Str("alertID", storageKey).
			Str("resourceID", params.ResourceID).
			Str("specID", params.Spec.ID).
			Msg("Skipping invalid canonical stateful evidence")
		return alertspecs.EvaluationResult{}, false
	}

	matched, matchSeverity, matchReason := alertspecs.Match(params.Spec, params.Evidence)

	// An existing alert the core does not know about means it was firing:
	// adopt it, as everywhere else.
	if existing != nil {
		if _, known := m.core.Incident(params.Spec.ResourceID, params.Spec.ID); !known {
			ackAt := time.Time{}
			if existing.AckTime != nil {
				ackAt = *existing.AckTime
			}
			m.core.SeedFiringIncident(params.Spec.ResourceID, params.Spec.ID, shadowSeverityForLevel(existing.Level), existing.StartTime, existing.Acknowledged, existing.AckUser, ackAt)
		}
	}

	severity := reducer.SeverityWarning
	if matchSeverity == alertspecs.AlertSeverityCritical {
		severity = reducer.SeverityCritical
	}
	events := m.core.ApplyDiscrete(reducer.DiscreteSignal{
		ResourceID: params.Spec.ResourceID,
		Key:        params.Spec.ID,
		Matched:    matched,
		Severity:   severity,
		ObservedAt: params.Evidence.ObservedAt,
	}, reducer.DiscreteRule{
		Confirmations:         specConfirmationsRequired(params.Spec),
		RecoveryConfirmations: params.RecoveryConfirmations,
		Disabled:              params.Spec.Disabled,
	})
	primary := reducer.EventType("")
	if len(events) > 0 {
		primary = events[0].Type
	}
	incident, hasIncident := m.core.Incident(params.Spec.ResourceID, params.Spec.ID)

	result := alertspecs.EvaluationResult{}
	result.State.SpecID = params.Spec.ID
	result.State.LastObservedAt = params.Evidence.ObservedAt
	result.State.Reason = matchReason
	transition := func(kind alertspecs.EvaluationTransitionKind, from, to alertspecs.AlertState) *alertspecs.EvaluationTransition {
		return &alertspecs.EvaluationTransition{
			Kind:       kind,
			SpecID:     params.Spec.ID,
			ResourceID: params.Spec.ResourceID,
			From:       from,
			To:         to,
			At:         params.Evidence.ObservedAt,
			Severity:   matchSeverity,
			Reason:     matchReason,
			Evidence:   params.Evidence,
		}
	}

	switch {
	case hasIncident && incident.State == reducer.StatePending:
		result.State.State = alertspecs.AlertStatePending
		result.State.Severity = matchSeverity
		result.State.ConsecutiveMatches = incident.Confirmations
		result.State.FirstMatchedAt = incident.PendingSince
		if primary == reducer.EventPending {
			result.Transition = transition(alertspecs.EvaluationTransitionPending, alertspecs.AlertStateClear, alertspecs.AlertStatePending)
		}
		return result, true
	case hasIncident && incident.State == reducer.StateFiring:
		level := AlertLevelWarning
		if incident.Severity == reducer.SeverityCritical {
			level = AlertLevelCritical
		}
		result.State.State = alertspecs.AlertStateFiring
		result.State.Severity = matchSeverity
		result.State.ConsecutiveMatches = incident.Confirmations
		result.State.FirstMatchedAt = incident.StartedAt
		result.State.ActiveSince = incident.StartedAt
		// A recovery still inside its confirmation window is not new failing
		// evidence. Keep the previous alert value and message rather than
		// projecting a healthy reading onto a still-active warning.
		if !matched {
			return result, true
		}
		switch {
		case existing == nil:
			result.Transition = transition(alertspecs.EvaluationTransitionActivated, alertspecs.AlertStatePending, alertspecs.AlertStateFiring)
		case primary == reducer.EventSeverityChanged:
			result.Transition = transition(alertspecs.EvaluationTransitionSeverityChanged, alertspecs.AlertStateFiring, alertspecs.AlertStateFiring)
		}
		message := params.Message
		summary := params.Summary
		value := params.Value
		threshold := params.Threshold
		if params.MessageBuilder != nil {
			message, value, threshold = params.MessageBuilder(result)
			// A built message has no matching summary; rows show it whole.
			summary = ""
		}
		// The stateful family stamps occurrences at the observation (or the
		// caller's override, e.g. a backup timestamp), not the pending run
		// start — preserved from the pre-cutover engine.
		startTime := params.Evidence.ObservedAt
		if !params.StartTimeOverride.IsZero() {
			startTime = params.StartTimeOverride
		}
		alert := &Alert{
			ID:           storageKey,
			Type:         params.AlertType,
			Level:        level,
			ResourceID:   params.Spec.ResourceID,
			ResourceName: params.ResourceName,
			Node:         params.Node,
			Instance:     params.Instance,
			Message:      message,
			Summary:      summary,
			Value:        value,
			Threshold:    threshold,
			StartTime:    startTime,
			LastSeen:     params.Evidence.ObservedAt,
			Metadata:     cloneMetadata(params.Metadata),
		}
		if alert.Metadata == nil {
			alert.Metadata = make(map[string]interface{}, 2)
		}
		if _, ok := alert.Metadata["resourceType"]; !ok && params.Spec.ResourceType != "" {
			alert.Metadata["resourceType"] = string(params.Spec.ResourceType)
		}
		applyCanonicalIdentity(alert, params.Spec.ID, string(params.Spec.Kind))
		applyCanonicalOperationalEvidence(alert, params.Spec, params.Evidence, time.Now())
		m.preserveAlertState(storageKey, alert)
		if !m.setActiveAlertNoLock(storageKey, alert) {
			result.State.State = alertspecs.AlertStateSuppressed
			result.State.Reason = "operator-suppressed"
			result.State.ActiveSince = time.Time{}
			result.Transition = nil
			return result, true
		}
		if params.AddToRecent {
			m.recentAlerts[trackingKey] = alert
		}

		if existing == nil {
			_, _, _, hadResolvedOccurrence := m.consumeRecentlyResolvedForRefireWithPrimaryLock(storageKey, time.Now())
			if primary == reducer.EventRefired {
				if !incident.StartedAt.IsZero() {
					alert.StartTime = incident.StartedAt
				}
				if params.AddToHistory {
					m.historyManager.UpdateAlertLastSeenForAlert(alert, alert.LastSeen)
				}
				log.Debug().
					Str("alertID", storageKey).
					Msg("Stateful alert re-fired within cooldown, reactivated without new history entry")
				if params.RateLimit && !m.checkRateLimit(trackingKey) {
					return result, true
				}
				m.dispatchAlert(alert, params.DispatchAsync)
				return result, true
			}

			m.recordAlertEvent(eventlog.TypeFired, alert, storageKey, matchReason, message, nil)
			if params.AddToHistory {
				if hadResolvedOccurrence {
					m.historyManager.AddAlertTransition(*alert)
				} else {
					m.historyManager.AddAlert(*alert)
				}
			}
			if params.RateLimit && !m.checkRateLimit(trackingKey) {
				log.Debug().
					Str("alertID", storageKey).
					Str("trackingKey", trackingKey).
					Int("maxPerHour", m.config.Schedule.MaxAlertsHour).
					Msg("Stateful alert notification suppressed due to rate limit")
				return result, true
			}
			m.dispatchAlert(alert, params.DispatchAsync)
			return result, true
		}

		// Restored backup incidents do not dispatch on a startup timer. Their
		// first valid overdue evaluation must resume delivery even when the
		// reducer keeps the same occurrence and severity.
		backupRevalidated := m.revalidateBackupNotificationNoLock(alert)
		severityChanged := primary == reducer.EventSeverityChanged && params.NotifyOnSeverityChange
		if severityChanged ||
			(backupRevalidated && m.shouldNotifyAfterCooldown(alert)) {
			if severityChanged && params.AddToHistoryOnSeverityChange {
				m.historyManager.AddAlertTransition(*alert)
			}
			if params.RateLimit && !m.checkRateLimit(trackingKey) {
				log.Debug().
					Str("alertID", storageKey).
					Str("trackingKey", trackingKey).
					Int("maxPerHour", m.config.Schedule.MaxAlertsHour).
					Msg("Stateful escalation notification suppressed due to rate limit")
				return result, true
			}
			m.dispatchAlert(alert, params.DispatchAsync)
		}
		m.setActiveAlertNoLock(storageKey, alert)
		return result, true
	default:
		result.State.State = alertspecs.AlertStateClear
		if existing == nil {
			return result, true
		}
		result.Transition = transition(alertspecs.EvaluationTransitionRecovered, alertspecs.AlertStateFiring, alertspecs.AlertStateClear)

		m.removeActiveAlertNoLock(storageKey)
		recoveryEvidence, hasRecoveryEvidence := canonicalAlertEvidenceEnvelope(
			params.Spec,
			params.Evidence,
			existing.Instance,
			time.Now(),
		)
		var recoveryEvidenceRef *operationaltrust.EvidenceEnvelope
		if hasRecoveryEvidence {
			recoveryEvidenceRef = &recoveryEvidence
		}
		resolvedAlert := m.newResolvedAlert(
			existing,
			params.Evidence.ObservedAt,
			recoveryEvidenceRef,
		)
		m.addRecentlyResolvedWithPrimaryLock(resolvedAlert)
		m.safeCallResolvedAlertCallback(resolvedAlert, storageKey, true)
		return result, true
	}
}