Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

10.4 KB
Raw Blame
package alerts

import (
	"strings"
	"time"
	"unicode"
	"unicode/utf8"

	"github.com/rcourtman/pulse-go-rewrite/internal/models"
	"github.com/rcourtman/pulse-go-rewrite/internal/operationaltrust"
)

type AlertCorrelationKind string

const (
	AlertCorrelationKindSharedSystem           AlertCorrelationKind = "shared-system"
	AlertCorrelationKindInfrastructureIncident AlertCorrelationKind = "infrastructure-incident"
)

type AlertCorrelationRole string

const (
	AlertCorrelationRolePrimary    AlertCorrelationRole = "primary"
	AlertCorrelationRoleSupporting AlertCorrelationRole = "supporting"
)

type AlertFailureClass string

const (
	AlertFailureClassRuntime             AlertFailureClass = "runtime"
	AlertFailureClassNetworkPath         AlertFailureClass = "network-path"
	AlertFailureClassApplicationResponse AlertFailureClass = "application-response"
	AlertFailureClassCertificate         AlertFailureClass = "certificate"
	AlertFailureClassDependency          AlertFailureClass = "dependency"
	AlertFailureClassEvidenceCoverage    AlertFailureClass = "evidence-coverage"
)

type AlertCorrelationInference string

const (
	AlertCorrelationInferenceSupportedCause AlertCorrelationInference = "supported-cause"
	AlertCorrelationInferenceObservationSet AlertCorrelationInference = "observation-set"
)

// AlertCorrelationObservation is a bounded, content-free description of one
// independently evaluated detector in an incident synthesis group. It keeps
// the source alert and its evidence addressable without copying arbitrary
// provider payloads into the correlation contract.
type AlertCorrelationObservation struct {
	AlertID      string            `json:"alertId"`
	ResourceID   string            `json:"resourceId"`
	ResourceName string            `json:"resourceName"`
	FailureClass AlertFailureClass `json:"failureClass"`
	Level        AlertLevel        `json:"level"`
	ObservedAt   time.Time         `json:"observedAt"`
	EvidenceIDs  []string          `json:"evidenceIds,omitempty"`
}

// AlertCorrelation is presentation-only incident context. It lets clients
// present independently evaluated alerts as signals from one verified system
// without merging their canonical detector lifecycles.
type AlertCorrelation struct {
	Key                 string                        `json:"key"`
	Kind                AlertCorrelationKind          `json:"kind"`
	Role                AlertCorrelationRole          `json:"role"`
	Reason              string                        `json:"reason"`
	FailureClass        AlertFailureClass             `json:"failureClass,omitempty"`
	Inference           AlertCorrelationInference     `json:"inference,omitempty"`
	PrimaryAlertID      string                        `json:"primaryAlertId,omitempty"`
	PrimaryResourceID   string                        `json:"primaryResourceId,omitempty"`
	AffectedResourceIDs []string                      `json:"affectedResourceIds,omitempty"`
	Observations        []AlertCorrelationObservation `json:"observations,omitempty"`
}

// NewSharedSystemAlertCorrelation returns a bounded shared-system correlation.
// Invalid or incomplete identity fails open so unrelated alerts remain separate.
func NewSharedSystemAlertCorrelation(key string, role AlertCorrelationRole, reason string) *AlertCorrelation {
	key = strings.TrimSpace(key)
	reason = strings.TrimSpace(reason)
	if key == "" || reason == "" {
		return nil
	}
	if role != AlertCorrelationRolePrimary && role != AlertCorrelationRoleSupporting {
		return nil
	}
	return &AlertCorrelation{
		Key:    key,
		Kind:   AlertCorrelationKindSharedSystem,
		Role:   role,
		Reason: reason,
	}
}

func cloneAlertCorrelation(correlation *AlertCorrelation) *AlertCorrelation {
	if correlation == nil {
		return nil
	}
	if correlation.Kind == AlertCorrelationKindSharedSystem {
		return NewSharedSystemAlertCorrelation(correlation.Key, correlation.Role, correlation.Reason)
	}
	if correlation.Kind != AlertCorrelationKindInfrastructureIncident ||
		strings.TrimSpace(correlation.Key) == "" ||
		strings.TrimSpace(correlation.Reason) == "" ||
		(correlation.Role != AlertCorrelationRolePrimary && correlation.Role != AlertCorrelationRoleSupporting) ||
		(correlation.Inference != AlertCorrelationInferenceSupportedCause && correlation.Inference != AlertCorrelationInferenceObservationSet) {
		return nil
	}
	clone := *correlation
	clone.AffectedResourceIDs = append([]string(nil), correlation.AffectedResourceIDs...)
	if len(correlation.Observations) > 0 {
		clone.Observations = make([]AlertCorrelationObservation, len(correlation.Observations))
		for index := range correlation.Observations {
			clone.Observations[index] = correlation.Observations[index]
			clone.Observations[index].EvidenceIDs = append([]string(nil), correlation.Observations[index].EvidenceIDs...)
		}
	}
	return &clone
}

// Alert represents an active alert
type Alert struct {
	ID              string     `json:"id"`
	Type            string     `json:"type"` // cpu, memory, disk, etc.
	Level           AlertLevel `json:"level"`
	ResourceID      string     `json:"resourceId"` // guest or node ID
	CanonicalSpecID string     `json:"canonicalSpecId,omitempty"`
	CanonicalKind   string     `json:"canonicalKind,omitempty"`
	CanonicalState  string     `json:"canonicalState,omitempty"`
	ResourceName    string     `json:"resourceName"`
	Node            string     `json:"node"`
	NodeDisplayName string     `json:"nodeDisplayName,omitempty"`
	Instance        string     `json:"instance"`
	Message         string     `json:"message"`
	// Summary is the condition without its subject ("Running 1 of 3 desired
	// tasks", "Offline") for rows that already name the resource. Empty when
	// the message has no separate subject; Message stays the full sentence for
	// notifications and detail views.
	Summary           string                                 `json:"summary,omitempty"`
	Value             float64                                `json:"value"`
	Threshold         float64                                `json:"threshold"`
	StartTime         time.Time                              `json:"startTime"`
	LastSeen          time.Time                              `json:"lastSeen"`
	Acknowledged      bool                                   `json:"acknowledged"`
	AckTime           *time.Time                             `json:"ackTime,omitempty"`
	AckUser           string                                 `json:"ackUser,omitempty"`
	Correlation       *AlertCorrelation                      `json:"correlation,omitempty"`
	Resolution        *AlertResolution                       `json:"resolution,omitempty"` // set only when a close was not a recovery
	Metadata          map[string]interface{}                 `json:"metadata,omitempty"`
	LastNotified      *time.Time                             `json:"lastNotified,omitempty"`
	LastEscalation    int                                    `json:"lastEscalation,omitempty"`
	EscalationTimes   []time.Time                            `json:"escalationTimes,omitempty"`
	OperationalRecord *operationaltrust.OperationalRecord    `json:"operationalRecord,omitempty"`
	LatestTransition  *operationaltrust.LifecycleTransition  `json:"latestTransition,omitempty"`
	Transitions       []operationaltrust.LifecycleTransition `json:"transitions,omitempty"`
	Evidence          []operationaltrust.EvidenceEnvelope    `json:"evidence,omitempty"`
	// MetricStatus is the live evaluation behind an open threshold alert. It
	// is volatile and stripped from durable snapshots.
	MetricStatus *models.MetricAlertStatus `json:"metricStatus,omitempty"`
}

// lastObservedAt is when the alert's condition was last evaluated. A
// threshold alert held below its trigger keeps LastSeen at the last breach,
// so its live status carries the newer observation.
func (a *Alert) lastObservedAt() time.Time {
	if a == nil {
		return time.Time{}
	}
	observed := a.LastSeen
	if a.MetricStatus != nil && a.MetricStatus.ObservedAt.After(observed) {
		observed = a.MetricStatus.ObservedAt
	}
	return observed
}

// Clone returns a deep copy of the alert so it can be safely shared across goroutines.
func (a *Alert) Clone() *Alert {
	if a == nil {
		return nil
	}

	clone := *a

	if a.AckTime != nil {
		t := *a.AckTime
		clone.AckTime = &t
	}

	if a.LastNotified != nil {
		t := *a.LastNotified
		clone.LastNotified = &t
	}

	if a.Correlation != nil {
		clone.Correlation = cloneAlertCorrelation(a.Correlation)
	}

	clone.Resolution = a.Resolution.Clone()

	if len(a.EscalationTimes) > 0 {
		clone.EscalationTimes = append([]time.Time(nil), a.EscalationTimes...)
	}

	if a.Metadata != nil {
		clone.Metadata = cloneMetadata(a.Metadata)
	}

	clone.MetricStatus = a.MetricStatus.Clone()

	if a.OperationalRecord != nil {
		value := a.OperationalRecord.Clone()
		clone.OperationalRecord = &value
	}

	if a.LatestTransition != nil {
		value := a.LatestTransition.Clone()
		clone.LatestTransition = &value
	}

	if len(a.Transitions) > 0 {
		clone.Transitions = make([]operationaltrust.LifecycleTransition, len(a.Transitions))
		for index := range a.Transitions {
			clone.Transitions[index] = a.Transitions[index].Clone()
		}
	}

	if len(a.Evidence) > 0 {
		clone.Evidence = make([]operationaltrust.EvidenceEnvelope, len(a.Evidence))
		for index := range a.Evidence {
			clone.Evidence[index] = a.Evidence[index].Clone()
		}
	}

	return &clone
}

func cloneMetadata(src map[string]interface{}) map[string]interface{} {
	if src == nil {
		return nil
	}

	dst := make(map[string]interface{}, len(src))
	for k, v := range src {
		dst[k] = cloneMetadataValue(v)
	}
	return dst
}

func cloneMetadataValue(val interface{}) interface{} {
	switch v := val.(type) {
	case map[string]interface{}:
		return cloneMetadata(v)
	case map[string]string:
		m := make(map[string]interface{}, len(v))
		for key, value := range v {
			m[key] = value
		}
		return m
	case []interface{}:
		arr := make([]interface{}, len(v))
		for i, elem := range v {
			arr[i] = cloneMetadataValue(elem)
		}
		return arr
	case []string:
		arr := make([]string, len(v))
		copy(arr, v)
		return arr
	case []int:
		arr := make([]int, len(v))
		copy(arr, v)
		return arr
	case []float64:
		arr := make([]float64, len(v))
		copy(arr, v)
		return arr
	default:
		return v
	}
}

// ResolvedAlert represents a recently resolved alert
type ResolvedAlert struct {
	*Alert
	ResolvedTime time.Time `json:"resolvedTime"`
}

// capitalizeFirst upper-cases a condition's first letter so it can stand alone
// as an alert Summary ("paused" reads "Paused"; "Exited (137)" is unchanged).
func capitalizeFirst(s string) string {
	if s == "" {
		return s
	}
	r, size := utf8.DecodeRuneInString(s)
	return string(unicode.ToUpper(r)) + s[size:]
}