Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

19.5 KB
Raw Blame
package alerts

import (
	"errors"
	"fmt"
	"strings"
	"time"
)

var ErrAlertIntentPolicyRevisionConflict = errors.New("alert_intent_policy_revision_conflict")

type OperatorIntentContext struct {
	IntentionallyOffline bool       `json:"intentionallyOffline"`
	MonitoringMode       string     `json:"monitoringMode"`
	LifecycleState       string     `json:"lifecycleState"`
	MaintenanceStartAt   *time.Time `json:"maintenanceStartAt,omitempty"`
	MaintenanceEndAt     *time.Time `json:"maintenanceEndAt,omitempty"`
	MaintenanceReason    string     `json:"maintenanceReason,omitempty"`
	MaintenanceSourceID  string     `json:"maintenanceSourceId,omitempty"`
	MaintenanceInherited bool       `json:"maintenanceInherited,omitempty"`
	MaintenanceScope     string     `json:"maintenanceScope,omitempty"`
}

func (c OperatorIntentContext) suppressionForSignal(signal string) (bool, string) {
	mode := strings.ToLower(strings.TrimSpace(c.MonitoringMode))
	if mode == "" && c.IntentionallyOffline {
		mode = "expected_offline"
	}
	lifecycle := strings.ToLower(strings.TrimSpace(c.LifecycleState))
	if lifecycle == "retired" {
		return true, "operator_retired"
	}
	if mode == "muted" {
		return true, "operator_muted"
	}
	if mode == "expected_offline" {
		switch strings.ToLower(strings.TrimSpace(signal)) {
		case string(AlertIntentSignalOffline), string(AlertIntentSignalAvailability):
			return true, "operator_expected_offline"
		}
	}
	return false, ""
}

func (c OperatorIntentContext) MaintenanceActiveAt(now time.Time) bool {
	return c.MaintenanceStartAt != nil && c.MaintenanceEndAt != nil &&
		!now.Before(*c.MaintenanceStartAt) && now.Before(*c.MaintenanceEndAt)
}

type BackupIntentContext struct {
	Active     bool      `json:"active"`
	ObservedAt time.Time `json:"observedAt,omitempty"`
	Evidence   string    `json:"evidence,omitempty"`
}

type OperatorIntentContextResolver func(resourceID string, now time.Time) (OperatorIntentContext, bool)
type BackupIntentContextResolver func(resourceID, instance, node string, vmid int, now time.Time) (BackupIntentContext, bool)
type ResourceIntentIdentityResolver func(resourceID string) (canonicalID string, found bool)

type IntentPendingState struct {
	TrackingKey             string     `json:"trackingKey"`
	ResourceID              string     `json:"resourceId"`
	ResourceType            string     `json:"resourceType"`
	Signal                  string     `json:"signal"`
	FirstMatchedAt          time.Time  `json:"firstMatchedAt"`
	LastObservedAt          time.Time  `json:"lastObservedAt"`
	ElapsedNanos            int64      `json:"elapsedNanos,omitempty"`
	BackupActive            bool       `json:"backupActive,omitempty"`
	BackupEndedAt           *time.Time `json:"backupEndedAt,omitempty"`
	BackupEndedElapsedNanos *int64     `json:"backupEndedElapsedNanos,omitempty"`
	BackupEvidence          string     `json:"backupEvidence,omitempty"`
}

type EffectiveAlertIntentPolicy struct {
	GraceSeconds       int                        `json:"graceSeconds"`
	HonorOperatorState bool                       `json:"honorOperatorState"`
	BackupOffline      *BackupOfflineIntentPolicy `json:"backupOffline,omitempty"`
	Sources            map[string]string          `json:"sources"`
	Explicit           bool                       `json:"explicit"`
}

type AlertIntentPolicyPreviewRequest struct {
	ResourceID       string     `json:"resourceId"`
	ResourceType     string     `json:"resourceType"`
	Signal           string     `json:"signal"`
	ConditionActive  bool       `json:"conditionActive"`
	FirstMatchedAt   *time.Time `json:"firstMatchedAt,omitempty"`
	BackupActive     *bool      `json:"backupActive,omitempty"`
	BackupObservedAt *time.Time `json:"backupObservedAt,omitempty"`
}

type AlertIntentPolicyPreviewContext struct {
	Kind       string     `json:"kind"`
	Active     bool       `json:"active"`
	Evidence   string     `json:"evidence,omitempty"`
	ObservedAt *time.Time `json:"observedAt,omitempty"`
	ExpiresAt  *time.Time `json:"expiresAt,omitempty"`
}

type AlertIntentPolicyPreview struct {
	ResourceID     string                            `json:"resourceId"`
	ResourceType   string                            `json:"resourceType"`
	Signal         string                            `json:"signal"`
	Status         string                            `json:"status"`
	Reason         string                            `json:"reason"`
	Effective      EffectiveAlertIntentPolicy        `json:"effective"`
	FirstMatchedAt *time.Time                        `json:"firstMatchedAt,omitempty"`
	EligibleAt     *time.Time                        `json:"eligibleAt,omitempty"`
	HardCapAt      *time.Time                        `json:"hardCapAt,omitempty"`
	RemainingSecs  int64                             `json:"remainingSeconds,omitempty"`
	Contexts       []AlertIntentPolicyPreviewContext `json:"contexts"`
	Warnings       []string                          `json:"warnings"`
}

type intentDecision struct {
	ShouldActivate bool
	Pending        bool
	Suppressed     bool
	Reason         string
	EligibleAt     time.Time
	HardCapAt      time.Time
	StateChanged   bool
	Effective      EffectiveAlertIntentPolicy
}

func (m *Manager) SetOperatorIntentContextResolver(resolver OperatorIntentContextResolver) {
	if m == nil {
		return
	}
	m.mu.Lock()
	m.operatorIntentResolver = resolver
	m.mu.Unlock()
}

func (m *Manager) SetBackupIntentContextResolver(resolver BackupIntentContextResolver) {
	if m == nil {
		return
	}
	m.mu.Lock()
	m.backupIntentResolver = resolver
	m.mu.Unlock()
}

func (m *Manager) SetResourceIntentIdentityResolver(resolver ResourceIntentIdentityResolver) {
	if m == nil {
		return
	}
	m.mu.Lock()
	m.resourceIntentResolver = resolver
	m.mu.Unlock()
}

// LoadIntentPolicies installs a persisted policy document without changing its
// revision. API writes should use UpdateIntentPolicies.
func (m *Manager) LoadIntentPolicies(document AlertIntentPolicyDocument) error {
	if err := ValidateAlertIntentPolicyDocument(document); err != nil {
		return err
	}
	if m == nil {
		return errors.New("alert manager is nil")
	}
	m.mu.Lock()
	m.intentPolicies = NormalizeAlertIntentPolicyDocument(document)
	m.mu.Unlock()
	return nil
}

func (m *Manager) GetIntentPolicies() AlertIntentPolicyDocument {
	if m == nil {
		return NewAlertIntentPolicyDocument()
	}
	m.mu.RLock()
	document := NormalizeAlertIntentPolicyDocument(m.intentPolicies)
	m.mu.RUnlock()
	return document
}

func (m *Manager) UpdateIntentPolicies(document AlertIntentPolicyDocument) (AlertIntentPolicyDocument, error) {
	if err := ValidateAlertIntentPolicyDocument(document); err != nil {
		return AlertIntentPolicyDocument{}, err
	}
	if m == nil {
		return AlertIntentPolicyDocument{}, errors.New("alert manager is nil")
	}

	now := m.policyNow().UTC()
	m.mu.Lock()
	defer m.mu.Unlock()
	if document.Revision != m.intentPolicies.Revision {
		return AlertIntentPolicyDocument{}, fmt.Errorf("%w: got %d want %d", ErrAlertIntentPolicyRevisionConflict, document.Revision, m.intentPolicies.Revision)
	}
	normalized := NormalizeAlertIntentPolicyDocument(document)
	normalized.Revision = m.intentPolicies.Revision + 1
	normalized.UpdatedAt = &now
	m.intentPolicies = normalized
	return NormalizeAlertIntentPolicyDocument(normalized), nil
}

func (m *Manager) resolveEffectiveIntentPolicyNoLock(resourceID, resourceType, signal string) EffectiveAlertIntentPolicy {
	resourceID = strings.TrimSpace(resourceID)
	resourceType = strings.ToLower(strings.TrimSpace(resourceType))
	signal = strings.ToLower(strings.TrimSpace(signal))
	effective := EffectiveAlertIntentPolicy{
		HonorOperatorState: true,
		Sources:            make(map[string]string),
	}

	if strings.HasPrefix(signal, "metric.") {
		metric := strings.TrimPrefix(signal, "metric.")
		if delay, source, ok := m.getLegacyTimeThresholdWithSource(resourceType, metric); ok {
			effective.GraceSeconds = delay
			effective.Sources["graceSeconds"] = source
		}
	}

	apply := func(rule AlertIntentRule, source string) {
		if rule.GraceSeconds != nil {
			effective.GraceSeconds = *rule.GraceSeconds
			effective.Sources["graceSeconds"] = source
			effective.Explicit = true
		}
		if rule.HonorOperatorState != nil {
			effective.HonorOperatorState = *rule.HonorOperatorState
			effective.Sources["honorOperatorState"] = source
			effective.Explicit = true
		}
		if rule.BackupOffline != nil {
			backup := *rule.BackupOffline
			effective.BackupOffline = &backup
			effective.Sources["backupOffline"] = source
			effective.Explicit = true
		}
	}
	applyRules := func(rules map[string]AlertIntentRule, source string) {
		if rule, ok := rules[string(AlertIntentSignalDefault)]; ok {
			apply(rule, source+".*")
		}
		if rule, ok := rules[signal]; ok {
			apply(rule, source+"."+signal)
		}
	}

	applyRules(m.intentPolicies.Defaults, "defaults")
	typeKeys := CanonicalResourceTypeKeys(resourceType)
	for i := len(typeKeys) - 1; i >= 0; i-- {
		typeKey := typeKeys[i]
		if rules, ok := m.intentPolicies.ResourceTypes[typeKey]; ok {
			applyRules(rules, "resourceTypes."+typeKey)
		}
	}
	resourceIDs := make([]string, 0, 2)
	if m.resourceIntentResolver != nil {
		if canonicalID, ok := m.resourceIntentResolver(resourceID); ok {
			canonicalID = strings.TrimSpace(canonicalID)
			if canonicalID != "" && canonicalID != resourceID {
				resourceIDs = append(resourceIDs, canonicalID)
			}
		}
	}
	resourceIDs = append(resourceIDs, resourceID)
	for _, candidateID := range resourceIDs {
		if rules, ok := m.intentPolicies.Resources[candidateID]; ok {
			applyRules(rules, "resources."+candidateID)
		}
	}
	if _, ok := effective.Sources["graceSeconds"]; !ok {
		effective.Sources["graceSeconds"] = "factory"
	}
	if _, ok := effective.Sources["honorOperatorState"]; !ok {
		effective.Sources["honorOperatorState"] = "factory"
	}
	return effective
}

func (m *Manager) ResolveEffectiveIntentPolicy(resourceID, resourceType, signal string) EffectiveAlertIntentPolicy {
	if m == nil {
		return EffectiveAlertIntentPolicy{HonorOperatorState: true, Sources: map[string]string{"graceSeconds": "factory", "honorOperatorState": "factory"}}
	}
	m.mu.RLock()
	effective := m.resolveEffectiveIntentPolicyNoLock(resourceID, resourceType, signal)
	m.mu.RUnlock()
	return effective
}

func (m *Manager) intentTickNoLock() time.Duration {
	if m.intentClock == nil {
		return 0
	}
	return m.intentClock()
}

func (m *Manager) clearIntentPendingNoLock(trackingKey string) bool {
	_, pending := m.intentPending[trackingKey]
	_, ticking := m.intentRuntimeTicks[trackingKey]
	delete(m.intentPending, trackingKey)
	delete(m.intentRuntimeTicks, trackingKey)
	return pending || ticking
}

func (m *Manager) advanceIntentElapsedNoLock(trackingKey string, state *IntentPendingState, tick time.Duration) {
	if state == nil {
		return
	}
	if previous, ok := m.intentRuntimeTicks[trackingKey]; ok && tick >= previous {
		delta := tick - previous
		if delta > 0 && state.ElapsedNanos <= int64(^uint64(0)>>1)-int64(delta) {
			state.ElapsedNanos += int64(delta)
		}
	}
	m.intentRuntimeTicks[trackingKey] = tick
}

func durationUntil(total, elapsed time.Duration) time.Duration {
	if total <= elapsed {
		return 0
	}
	return total - elapsed
}

func absoluteDuration(value time.Duration) time.Duration {
	if value < 0 {
		return -value
	}
	return value
}

func intentTimePointer(value time.Time) *time.Time {
	return &value
}

func (m *Manager) evaluateIntentNoLock(resourceID, resourceType, signal, trackingKey string, _ time.Time, conditionActive bool, backup BackupIntentContext) intentDecision {
	effective := m.resolveEffectiveIntentPolicyNoLock(resourceID, resourceType, signal)
	decision := intentDecision{Effective: effective}
	if !conditionActive {
		if m.clearIntentPendingNoLock(trackingKey) {
			decision.StateChanged = true
		}
		decision.Reason = "condition_clear"
		return decision
	}
	observedAt := m.policyNow().UTC()
	tick := m.intentTickNoLock()
	state, exists := m.intentPending[trackingKey]
	if !exists || state.FirstMatchedAt.IsZero() {
		state = IntentPendingState{
			TrackingKey: trackingKey, ResourceID: resourceID, ResourceType: resourceType,
			Signal: signal, FirstMatchedAt: observedAt,
		}
		m.intentRuntimeTicks[trackingKey] = tick
		decision.StateChanged = true
	} else {
		previousElapsed := state.ElapsedNanos
		m.advanceIntentElapsedNoLock(trackingKey, &state, tick)
		if state.ElapsedNanos != previousElapsed {
			decision.StateChanged = true
		}
	}
	state.LastObservedAt = observedAt
	elapsed := time.Duration(state.ElapsedNanos)
	projectedFirstMatchedAt := observedAt.Add(-elapsed)
	if absoluteDuration(state.FirstMatchedAt.Add(elapsed).Sub(observedAt)) > time.Minute {
		state.FirstMatchedAt = projectedFirstMatchedAt
		if state.BackupEndedElapsedNanos != nil {
			backupEndedElapsed := time.Duration(*state.BackupEndedElapsedNanos)
			state.BackupEndedAt = intentTimePointer(observedAt.Add(-(elapsed - backupEndedElapsed)))
		}
		decision.StateChanged = true
	}

	if effective.HonorOperatorState && m.operatorIntentResolver != nil {
		if operator, ok := m.operatorIntentResolver(resourceID, observedAt); ok {
			if operator.MaintenanceActiveAt(observedAt) {
				m.intentPending[trackingKey] = state
				decision.Pending = true
				decision.Suppressed = true
				decision.Reason = "operator_maintenance"
				if operator.MaintenanceEndAt != nil {
					decision.EligibleAt = *operator.MaintenanceEndAt
				}
				return decision
			}
			if suppressed, reason := operator.suppressionForSignal(signal); suppressed {
				m.intentPending[trackingKey] = state
				decision.Pending = true
				decision.Suppressed = true
				decision.Reason = reason
				return decision
			}
		}
	}
	if !effective.Explicit {
		if m.clearIntentPendingNoLock(trackingKey) {
			decision.StateChanged = true
		}
		decision.ShouldActivate = true
		return decision
	}

	eligibleElapsed := time.Duration(effective.GraceSeconds) * time.Second
	if effective.BackupOffline != nil && effective.BackupOffline.Enabled && signal == string(AlertIntentSignalOffline) {
		if backup.Active {
			if !state.BackupActive || state.BackupEvidence != backup.Evidence {
				decision.StateChanged = true
			}
			state.BackupActive = true
			state.BackupEndedAt = nil
			state.BackupEndedElapsedNanos = nil
			state.BackupEvidence = backup.Evidence
		} else if state.BackupActive {
			endedAt := observedAt
			endedElapsed := state.ElapsedNanos
			state.BackupActive = false
			state.BackupEndedAt = &endedAt
			state.BackupEndedElapsedNanos = &endedElapsed
			decision.StateChanged = true
		}

		capElapsed := time.Duration(effective.BackupOffline.MaxDeferralSeconds) * time.Second
		capAt := observedAt.Add(durationUntil(capElapsed, elapsed))
		decision.HardCapAt = capAt
		if state.BackupActive && elapsed < capElapsed {
			m.intentPending[trackingKey] = state
			decision.Pending = true
			decision.Suppressed = true
			decision.Reason = "backup_active"
			decision.EligibleAt = capAt
			return decision
		}
		if state.BackupEndedElapsedNanos != nil {
			postEligibleElapsed := time.Duration(*state.BackupEndedElapsedNanos) + time.Duration(effective.BackupOffline.PostGraceSeconds)*time.Second
			if postEligibleElapsed > eligibleElapsed {
				eligibleElapsed = postEligibleElapsed
			}
		}
		if eligibleElapsed > capElapsed {
			eligibleElapsed = capElapsed
		}
		if elapsed >= capElapsed {
			decision.Reason = "backup_grace_cap_exceeded"
		}
	}

	m.intentPending[trackingKey] = state
	eligibleAt := observedAt.Add(durationUntil(eligibleElapsed, elapsed))
	decision.EligibleAt = eligibleAt
	if elapsed < eligibleElapsed {
		decision.Pending = true
		if decision.Reason == "" {
			decision.Reason = "grace_period"
		}
		return decision
	}
	decision.ShouldActivate = true
	if decision.Reason == "" {
		decision.Reason = "eligible"
	}
	return decision
}

func (m *Manager) PreviewIntentPolicy(request AlertIntentPolicyPreviewRequest) (AlertIntentPolicyPreview, error) {
	request.ResourceID = strings.TrimSpace(request.ResourceID)
	request.ResourceType = strings.ToLower(strings.TrimSpace(request.ResourceType))
	request.Signal = strings.ToLower(strings.TrimSpace(request.Signal))
	if request.ResourceID == "" || request.ResourceType == "" || request.Signal == "" {
		return AlertIntentPolicyPreview{}, errors.New("resourceId, resourceType, and signal are required")
	}
	if !ValidAlertIntentSignal(request.Signal) {
		return AlertIntentPolicyPreview{}, fmt.Errorf("unsupported alert intent signal %q", request.Signal)
	}
	now := m.policyNow().UTC()
	trackingKey := "preview:" + request.ResourceID + ":" + request.Signal
	backup := BackupIntentContext{}
	if request.BackupActive != nil {
		backup.Active = *request.BackupActive
		backup.Evidence = "preview"
		if request.BackupObservedAt != nil {
			backup.ObservedAt = request.BackupObservedAt.UTC()
		}
	}

	m.mu.Lock()
	previous, hadPrevious := m.intentPending[trackingKey]
	previousTick, hadPreviousTick := m.intentRuntimeTicks[trackingKey]
	if request.FirstMatchedAt != nil {
		firstMatchedAt := request.FirstMatchedAt.UTC()
		elapsed := now.Sub(firstMatchedAt)
		if elapsed < 0 {
			elapsed = 0
		}
		m.intentPending[trackingKey] = IntentPendingState{
			Signal: request.Signal, FirstMatchedAt: firstMatchedAt, LastObservedAt: now, ElapsedNanos: int64(elapsed),
		}
		m.intentRuntimeTicks[trackingKey] = m.intentTickNoLock()
	}
	decision := m.evaluateIntentNoLock(request.ResourceID, request.ResourceType, request.Signal, trackingKey, now, request.ConditionActive, backup)
	state, stateExists := m.intentPending[trackingKey]
	operator := OperatorIntentContext{}
	operatorFound := false
	if decision.Effective.HonorOperatorState && m.operatorIntentResolver != nil {
		operator, operatorFound = m.operatorIntentResolver(request.ResourceID, now)
	}
	if hadPrevious {
		m.intentPending[trackingKey] = previous
	} else {
		delete(m.intentPending, trackingKey)
	}
	if hadPreviousTick {
		m.intentRuntimeTicks[trackingKey] = previousTick
	} else {
		delete(m.intentRuntimeTicks, trackingKey)
	}
	m.mu.Unlock()

	preview := AlertIntentPolicyPreview{
		ResourceID: request.ResourceID, ResourceType: request.ResourceType, Signal: request.Signal,
		Effective: decision.Effective, Contexts: []AlertIntentPolicyPreviewContext{}, Warnings: []string{},
	}
	if stateExists && !state.FirstMatchedAt.IsZero() {
		first := state.FirstMatchedAt
		preview.FirstMatchedAt = &first
	}
	if !decision.EligibleAt.IsZero() {
		eligible := decision.EligibleAt
		preview.EligibleAt = &eligible
		if eligible.After(now) {
			preview.RemainingSecs = int64(eligible.Sub(now).Seconds())
		}
	}
	if !decision.HardCapAt.IsZero() {
		capAt := decision.HardCapAt
		preview.HardCapAt = &capAt
	}
	switch {
	case !request.ConditionActive:
		preview.Status, preview.Reason = "clear", "condition_clear"
	case decision.Suppressed:
		preview.Status, preview.Reason = "expected_transient", decision.Reason
	case decision.Pending:
		preview.Status, preview.Reason = "pending_grace", decision.Reason
	default:
		preview.Status, preview.Reason = "would_activate", decision.Reason
	}
	if request.BackupActive != nil {
		ctx := AlertIntentPolicyPreviewContext{Kind: "backup", Active: *request.BackupActive, Evidence: backup.Evidence}
		if !backup.ObservedAt.IsZero() {
			observedAt := backup.ObservedAt
			ctx.ObservedAt = &observedAt
		}
		preview.Contexts = append(preview.Contexts, ctx)
	}
	if operatorFound {
		suppressed, _ := operator.suppressionForSignal(request.Signal)
		active := suppressed || operator.MaintenanceActiveAt(now)
		ctx := AlertIntentPolicyPreviewContext{Kind: "operator_state", Active: active, Evidence: operator.MaintenanceReason}
		if operator.MaintenanceEndAt != nil {
			expiresAt := operator.MaintenanceEndAt.UTC()
			ctx.ExpiresAt = &expiresAt
		}
		preview.Contexts = append(preview.Contexts, ctx)
	}
	if request.Signal == string(AlertIntentSignalAvailability) {
		preview.Warnings = append(preview.Warnings, "Availability probe failure thresholds are evaluated before alert grace.")
	}
	return preview, nil
}