Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

29.7 KB
Raw Blame
package ai

import (
	"errors"
	"fmt"
	"math"
	"sort"
	"strconv"
	"strings"
	"time"

	"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
	"github.com/rcourtman/pulse-go-rewrite/internal/models"
	"github.com/rcourtman/pulse-go-rewrite/internal/operationaltrust"
	recoverymodel "github.com/rcourtman/pulse-go-rewrite/internal/recovery/model"
	"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
)

const (
	DefaultAttentionPageSize = 50
	MaxAttentionPageSize     = 200
)

type AttentionFilter string

const (
	AttentionFilterActive       AttentionFilter = "active"
	AttentionFilterOpen         AttentionFilter = "open"
	AttentionFilterAcknowledged AttentionFilter = "acknowledged"
	AttentionFilterSuppressed   AttentionFilter = "suppressed"
	AttentionFilterUncertain    AttentionFilter = "stale_unknown"
	AttentionFilterResolved     AttentionFilter = "resolved"
	AttentionFilterAll          AttentionFilter = "all"
)

func (filter AttentionFilter) Valid() bool {
	switch filter {
	case AttentionFilterActive,
		AttentionFilterOpen,
		AttentionFilterAcknowledged,
		AttentionFilterSuppressed,
		AttentionFilterUncertain,
		AttentionFilterResolved,
		AttentionFilterAll:
		return true
	default:
		return false
	}
}

type AttentionVerificationState string

const (
	AttentionVerificationNotAvailable AttentionVerificationState = "not_available"
	AttentionVerificationPending      AttentionVerificationState = "pending"
	AttentionVerificationSucceeded    AttentionVerificationState = "succeeded"
	AttentionVerificationFailed       AttentionVerificationState = "failed"
	AttentionVerificationUnknown      AttentionVerificationState = "unknown"
)

type AttentionActionOffer struct {
	ActionID              string   `json:"actionId,omitempty"`
	TargetResourceID      string   `json:"targetResourceId"`
	Capability            string   `json:"capability"`
	Kind                  string   `json:"kind"`
	Label                 string   `json:"label"`
	Mode                  string   `json:"mode"`
	Risk                  string   `json:"risk"`
	Approval              string   `json:"approval"`
	Eligibility           string   `json:"eligibility"`
	Reasons               []string `json:"reasons"`
	EvidenceIDs           []string `json:"evidenceIds"`
	ExpectedPostcondition string   `json:"expectedPostcondition"`
	VerificationPolicy    string   `json:"verificationPolicy"`
	RequiresApproval      bool     `json:"requiresApproval"`
}

type AttentionResource struct {
	ResourceID string `json:"resourceId"`
}

// AttentionFlapping summarises an item whose lifecycle keeps moving between
// open and resolved. It uses the same window and threshold as finding
// flapping (findings_storm_throttler.go) so alerts and Patrol findings agree
// on the label. The full transition list stays in the detail timeline; this
// is the collapsed operator-facing summary.
type AttentionFlapping struct {
	TransitionCount   int       `json:"transitionCount"`
	WindowHours       int       `json:"windowHours"`
	FirstTransitionAt time.Time `json:"firstTransitionAt"`
	LastTransitionAt  time.Time `json:"lastTransitionAt"`
}

type AttentionItem struct {
	ID                   string                                `json:"id"`
	OperationalRecordID  string                                `json:"operationalRecordId"`
	SubjectResourceID    string                                `json:"subjectResourceId"`
	SubjectResourceName  string                                `json:"subjectResourceName"`
	SubjectResourceType  string                                `json:"subjectResourceType,omitempty"`
	Kind                 string                                `json:"kind"`
	Title                string                                `json:"title"`
	PlainLanguageSummary string                                `json:"plainLanguageSummary"`
	Severity             operationaltrust.OperationalSeverity  `json:"severity"`
	State                operationaltrust.OperationalState     `json:"state"`
	FirstObservedAt      time.Time                             `json:"firstObservedAt"`
	LastObservedAt       time.Time                             `json:"lastObservedAt"`
	EvidenceFreshness    operationaltrust.EvidenceFreshness    `json:"evidenceFreshness"`
	EvidenceCompleteness operationaltrust.EvidenceCompleteness `json:"evidenceCompleteness"`
	Impact               string                                `json:"impact,omitempty"`
	ProtectionPosture    *recoverymodel.ProtectionPosture      `json:"protectionPosture,omitempty"`
	RelatedResources     []AttentionResource                   `json:"relatedResources"`
	RecommendedNextStep  string                                `json:"recommendedNextStep,omitempty"`
	AvailableActions     []AttentionActionOffer                `json:"availableActions"`
	VerificationState    AttentionVerificationState            `json:"verificationState"`
	Flapping             *AttentionFlapping                    `json:"flapping,omitempty"`
}

type AttentionItemDetail struct {
	Item              AttentionItem                          `json:"item"`
	OperationalRecord operationaltrust.OperationalRecord     `json:"operationalRecord"`
	Timeline          []operationaltrust.LifecycleTransition `json:"timeline"`
	Evidence          []operationaltrust.EvidenceEnvelope    `json:"evidence"`
}

type AttentionSummary struct {
	ActiveCount       int       `json:"activeCount"`
	OpenCount         int       `json:"openCount"`
	AcknowledgedCount int       `json:"acknowledgedCount"`
	SuppressedCount   int       `json:"suppressedCount"`
	UncertainCount    int       `json:"uncertainCount"`
	ResolvedCount     int       `json:"resolvedCount"`
	Calm              bool      `json:"calm"`
	CoverageState     string    `json:"coverageState"`
	EvaluatedAt       time.Time `json:"evaluatedAt"`
}

type AttentionProjection struct {
	Details []AttentionItemDetail `json:"details"`
	Summary AttentionSummary      `json:"summary"`
}

// ProjectAttentionItems is the single Patrol read-model projection over the
// canonical alert lifecycle. It does not inspect loose alert metadata to
// invent work and it never creates a second writable lifecycle. An item's
// subject is the unified resource its record's subject reference resolves to
// in current inventory (AttentionSubjectResourceID), the identity Patrol
// findings, protection postures and resource actions carry; the operational
// record keeps the alert's own reference. A nil subjects resolver keeps every
// record's reference.
func ProjectAttentionItems(
	active []alerts.Alert,
	history []alerts.Alert,
	postures map[string]recoverymodel.ProtectionPosture,
	subjects unifiedresources.AlertResourceReferenceResolver,
	now time.Time,
) AttentionProjection {
	if now.IsZero() {
		now = time.Now().UTC()
	}
	now = now.UTC()

	records := make(map[string]alerts.Alert, len(active)+len(history))
	for _, alert := range history {
		addAttentionAlert(records, alert, false)
	}
	for _, alert := range active {
		addAttentionAlert(records, alert, true)
	}

	details := make([]AttentionItemDetail, 0, len(records))
	for _, alert := range records {
		detail, ok := projectAttentionAlert(alert, postures, subjects, now)
		if !ok {
			continue
		}
		details = append(details, detail)
	}
	sort.SliceStable(details, func(i, j int) bool {
		return attentionItemLess(details[i].Item, details[j].Item)
	})

	return AttentionProjection{
		Details: details,
		Summary: summarizeAttention(details, now),
	}
}

func addAttentionAlert(records map[string]alerts.Alert, alert alerts.Alert, active bool) {
	if alert.OperationalRecord == nil {
		return
	}
	if !active && alert.OperationalRecord.State != operationaltrust.OperationalResolved {
		return
	}
	recordID := strings.TrimSpace(alert.OperationalRecord.ID)
	if recordID == "" {
		return
	}
	existing, found := records[recordID]
	if !found || active || attentionAlertNewer(alert, existing) {
		records[recordID] = *alert.Clone()
	}
}

func attentionAlertNewer(candidate, existing alerts.Alert) bool {
	if candidate.OperationalRecord == nil {
		return false
	}
	if existing.OperationalRecord == nil {
		return true
	}
	if !candidate.OperationalRecord.StateChangedAt.Equal(existing.OperationalRecord.StateChangedAt) {
		return candidate.OperationalRecord.StateChangedAt.After(existing.OperationalRecord.StateChangedAt)
	}
	return candidate.OperationalRecord.LastObservedAt.After(existing.OperationalRecord.LastObservedAt)
}

// AttentionSubjectResourceID names the unified resource an alert lifecycle
// record's subject reference ("docker:<host>/<container>", a Proxmox source
// ID, "agent:<host>/disk:<label>") resolves to in current inventory, or the
// reference itself when inventory does not bind it.
func AttentionSubjectResourceID(
	subjects unifiedresources.AlertResourceReferenceResolver,
	reference string,
) string {
	reference = strings.TrimSpace(reference)
	if subjects == nil || reference == "" {
		return reference
	}
	if binding, _ := subjects.ResolveAlertResourceReference(reference); binding.ResourceID != "" {
		return binding.ResourceID
	}
	return reference
}

func projectAttentionAlert(
	alert alerts.Alert,
	postures map[string]recoverymodel.ProtectionPosture,
	subjects unifiedresources.AlertResourceReferenceResolver,
	now time.Time,
) (AttentionItemDetail, bool) {
	if alert.OperationalRecord == nil {
		return AttentionItemDetail{}, false
	}
	record := alert.OperationalRecord.Clone()
	if err := record.Validate(); err != nil {
		return AttentionItemDetail{}, false
	}

	evidence := cloneAttentionEvidence(alert.Evidence)
	timeline := cloneAttentionTimeline(alert.Transitions)
	freshness, completeness := summarizeAttentionEvidence(record.State, evidence, now)
	resourceName := firstAttentionText(alert.ResourceName, alert.Instance, record.SubjectResourceID)
	title := attentionTitle(alert, resourceName)
	summary := firstAttentionText(heldMetricAlertSummary(alert, now), alert.Message, record.ImpactSummary, title)

	related := make([]AttentionResource, 0, len(record.RelatedResourceIDs))
	for _, resourceID := range canonicalAttentionIDs(record.RelatedResourceIDs) {
		related = append(related, AttentionResource{ResourceID: resourceID})
	}

	subjectResourceID := AttentionSubjectResourceID(subjects, record.SubjectResourceID)
	var posture *recoverymodel.ProtectionPosture
	if candidate, found := postures[subjectResourceID]; found {
		value := candidate.Clone()
		posture = &value
	}

	item := AttentionItem{
		ID:                   record.ID,
		OperationalRecordID:  record.ID,
		SubjectResourceID:    subjectResourceID,
		SubjectResourceName:  resourceName,
		SubjectResourceType:  attentionResourceType(alert),
		Kind:                 strings.TrimSpace(alert.Type),
		Title:                title,
		PlainLanguageSummary: summary,
		Severity:             record.Severity,
		State:                record.State,
		FirstObservedAt:      record.FirstObservedAt,
		LastObservedAt:       record.LastObservedAt,
		EvidenceFreshness:    freshness,
		EvidenceCompleteness: completeness,
		Impact:               record.ImpactSummary,
		ProtectionPosture:    posture,
		RelatedResources:     related,
		RecommendedNextStep:  record.RecommendedNextStep,
		AvailableActions:     []AttentionActionOffer{},
		VerificationState:    AttentionVerificationNotAvailable,
		Flapping:             attentionFlapping(timeline, now),
	}
	return AttentionItemDetail{
		Item:              item,
		OperationalRecord: record,
		Timeline:          timeline,
		Evidence:          evidence,
	}, true
}

// attentionFlapping collapses open/resolved churn in the lifecycle timeline
// into one summary when it crosses the shared flapping threshold. Only
// transitions that open or resolve the record count; acknowledgement,
// suppression, and evidence refreshes are operator or detector bookkeeping,
// not the condition coming and going.
func attentionFlapping(
	timeline []operationaltrust.LifecycleTransition,
	now time.Time,
) *AttentionFlapping {
	transitions := make([]time.Time, 0, len(timeline))
	for _, transition := range timeline {
		if !attentionTransitionIsFlap(transition) {
			continue
		}
		transitions = append(transitions, transition.At)
	}
	summary := summarizeFlapTransitions(transitions, now)
	if summary == nil {
		return nil
	}
	return &AttentionFlapping{
		TransitionCount:   summary.TransitionCount,
		WindowHours:       summary.WindowHours,
		FirstTransitionAt: summary.FirstTransitionAt,
		LastTransitionAt:  summary.LastTransitionAt,
	}
}

func attentionTransitionIsFlap(transition operationaltrust.LifecycleTransition) bool {
	if transition.To == operationaltrust.OperationalResolved {
		return transition.From != operationaltrust.OperationalResolved
	}
	if transition.To == operationaltrust.OperationalOpen {
		return transition.From == operationaltrust.OperationalResolved
	}
	return false
}

func summarizeAttentionEvidence(
	state operationaltrust.OperationalState,
	evidence []operationaltrust.EvidenceEnvelope,
	now time.Time,
) (operationaltrust.EvidenceFreshness, operationaltrust.EvidenceCompleteness) {
	if state == operationaltrust.OperationalStale {
		return operationaltrust.EvidenceStale, worstAttentionCompleteness(evidence)
	}
	if state == operationaltrust.OperationalUnknown {
		return operationaltrust.EvidenceFreshnessUnknown, worstAttentionCompleteness(evidence)
	}
	if len(evidence) == 0 {
		return operationaltrust.EvidenceFreshnessUnknown, operationaltrust.EvidenceUnavailable
	}

	freshness := operationaltrust.EvidenceFresh
	completeness := operationaltrust.EvidenceComplete
	for _, envelope := range evidence {
		switch envelope.FreshnessAt(now) {
		case operationaltrust.EvidenceStale:
			freshness = operationaltrust.EvidenceStale
		case operationaltrust.EvidenceFreshnessUnknown:
			if freshness != operationaltrust.EvidenceStale {
				freshness = operationaltrust.EvidenceFreshnessUnknown
			}
		}
		switch envelope.Completeness {
		case operationaltrust.EvidenceUnavailable:
			completeness = operationaltrust.EvidenceUnavailable
		case operationaltrust.EvidencePartial:
			if completeness != operationaltrust.EvidenceUnavailable {
				completeness = operationaltrust.EvidencePartial
			}
		}
	}
	return freshness, completeness
}

func worstAttentionCompleteness(
	evidence []operationaltrust.EvidenceEnvelope,
) operationaltrust.EvidenceCompleteness {
	if len(evidence) == 0 {
		return operationaltrust.EvidenceUnavailable
	}
	completeness := operationaltrust.EvidenceComplete
	for _, envelope := range evidence {
		if envelope.Completeness == operationaltrust.EvidenceUnavailable {
			return operationaltrust.EvidenceUnavailable
		}
		if envelope.Completeness == operationaltrust.EvidencePartial {
			completeness = operationaltrust.EvidencePartial
		}
	}
	return completeness
}

func FilterAttentionDetails(
	details []AttentionItemDetail,
	filter AttentionFilter,
) ([]AttentionItemDetail, error) {
	if filter == "" {
		filter = AttentionFilterActive
	}
	if !filter.Valid() {
		return nil, fmt.Errorf("invalid attention filter %q", filter)
	}
	filtered := make([]AttentionItemDetail, 0, len(details))
	for _, detail := range details {
		if attentionStateMatches(detail.Item.State, filter) {
			filtered = append(filtered, detail)
		}
	}
	return filtered, nil
}

func PaginateAttentionDetails(
	details []AttentionItemDetail,
	page int,
	limit int,
) ([]AttentionItemDetail, error) {
	if page < 1 {
		return nil, errors.New("attention page must be at least one")
	}
	if limit < 1 || limit > MaxAttentionPageSize {
		return nil, fmt.Errorf("attention limit must be between 1 and %d", MaxAttentionPageSize)
	}
	start := (page - 1) * limit
	if start >= len(details) {
		return []AttentionItemDetail{}, nil
	}
	end := start + limit
	if end > len(details) {
		end = len(details)
	}
	return append([]AttentionItemDetail(nil), details[start:end]...), nil
}

func attentionStateMatches(state operationaltrust.OperationalState, filter AttentionFilter) bool {
	switch filter {
	case AttentionFilterActive:
		return attentionStateCountsAsActive(state)
	case AttentionFilterOpen:
		return state == operationaltrust.OperationalOpen ||
			state == operationaltrust.OperationalObserving ||
			state == operationaltrust.OperationalResolving
	case AttentionFilterAcknowledged:
		return state == operationaltrust.OperationalAcknowledged
	case AttentionFilterSuppressed:
		return state == operationaltrust.OperationalSuppressed
	case AttentionFilterUncertain:
		return state == operationaltrust.OperationalStale ||
			state == operationaltrust.OperationalUnknown
	case AttentionFilterResolved:
		return state == operationaltrust.OperationalResolved
	case AttentionFilterAll:
		return true
	default:
		return false
	}
}

func attentionStateCountsAsActive(state operationaltrust.OperationalState) bool {
	switch state {
	case operationaltrust.OperationalObserving,
		operationaltrust.OperationalOpen,
		operationaltrust.OperationalResolving,
		operationaltrust.OperationalStale,
		operationaltrust.OperationalUnknown:
		return true
	default:
		return false
	}
}

func summarizeAttention(details []AttentionItemDetail, now time.Time) AttentionSummary {
	summary := AttentionSummary{
		CoverageState: "current",
		EvaluatedAt:   now,
	}
	for _, detail := range details {
		switch detail.Item.State {
		case operationaltrust.OperationalAcknowledged:
			summary.AcknowledgedCount++
		case operationaltrust.OperationalSuppressed:
			summary.SuppressedCount++
		case operationaltrust.OperationalStale, operationaltrust.OperationalUnknown:
			summary.UncertainCount++
			summary.ActiveCount++
		case operationaltrust.OperationalResolved:
			summary.ResolvedCount++
		default:
			if attentionStateCountsAsActive(detail.Item.State) {
				summary.ActiveCount++
				summary.OpenCount++
			}
		}
	}
	summary.Calm = summary.ActiveCount == 0
	return summary
}

func attentionItemLess(left, right AttentionItem) bool {
	if l, r := attentionSeverityRank(left.Severity), attentionSeverityRank(right.Severity); l != r {
		return l > r
	}
	if l, r := len(left.RelatedResources), len(right.RelatedResources); l != r {
		return l > r
	}
	if l, r := attentionProtectionRank(left.ProtectionPosture), attentionProtectionRank(right.ProtectionPosture); l != r {
		return l > r
	}
	if l, r := attentionFreshnessRank(left.EvidenceFreshness), attentionFreshnessRank(right.EvidenceFreshness); l != r {
		return l > r
	}
	if !left.FirstObservedAt.Equal(right.FirstObservedAt) {
		return left.FirstObservedAt.Before(right.FirstObservedAt)
	}
	return left.ID < right.ID
}

func attentionSeverityRank(severity operationaltrust.OperationalSeverity) int {
	switch severity {
	case operationaltrust.SeverityCritical:
		return 3
	case operationaltrust.SeverityWarning:
		return 2
	case operationaltrust.SeverityInfo:
		return 1
	default:
		return 0
	}
}

func attentionProtectionRank(posture *recoverymodel.ProtectionPosture) int {
	if posture == nil {
		return 2
	}
	switch posture.State {
	case recoverymodel.ProtectionStateAttention:
		return 4
	case recoverymodel.ProtectionStateUnprotected:
		return 3
	case recoverymodel.ProtectionStateUnknown:
		return 2
	default:
		return 0
	}
}

func attentionFreshnessRank(freshness operationaltrust.EvidenceFreshness) int {
	switch freshness {
	case operationaltrust.EvidenceFresh:
		return 2
	case operationaltrust.EvidenceStale:
		return 1
	default:
		return 0
	}
}

func attentionTitle(alert alerts.Alert, resourceName string) string {
	alertType := strings.TrimSpace(alert.Type)
	if alertType == "" {
		return "Issue on " + resourceName
	}
	if incidentTitle := attentionIncidentTitle(alert); incidentTitle != "" {
		return incidentTitle + " on " + resourceName
	}
	switch strings.ToLower(alertType) {
	case "usage":
		if strings.EqualFold(attentionResourceType(alert), "storage") {
			return "Storage usage on " + resourceName
		}
		return "Resource usage on " + resourceName
	case "cpu":
		return "CPU usage on " + resourceName
	case "disk":
		return "Disk usage on " + resourceName
	case "memory":
		return "Memory usage on " + resourceName
	}
	words := strings.Fields(strings.NewReplacer("-", " ", "_", " ").Replace(alertType))
	for i := range words {
		word := strings.ToLower(words[i])
		switch word {
		case "api", "cpu", "dns", "ip", "vm", "zfs":
			words[i] = strings.ToUpper(word)
		case "io":
			words[i] = "I/O"
		default:
			words[i] = word
			if i == 0 {
				words[i] = strings.ToUpper(word[:1]) + word[1:]
			}
		}
	}
	return strings.Join(words, " ") + " on " + resourceName
}

func attentionIncidentTitle(alert alerts.Alert) string {
	normalizedType := strings.NewReplacer("-", "_", " ", "_").Replace(
		strings.ToLower(strings.TrimSpace(alert.Type)),
	)
	if normalizedType != "resource_incident" && normalizedType != "storage_incident" {
		return ""
	}
	message := strings.TrimSpace(alert.Message)
	if strings.Contains(strings.ToLower(message), "pg degraded") {
		return "Ceph placement groups degraded"
	}

	switch attentionIncidentCode(alert) {
	case "vmware_alarm_state":
		// The message is the alarm's name, with the consumers it affects
		// appended on storage. Strip only that appended summary: a custom
		// alarm name may itself contain ". " or parentheses.
		issue := message
		summaryIndex := -1
		for _, consumerMarker := range []string{". Affects ", ". Puts backups for "} {
			summaryIndex = max(summaryIndex, strings.LastIndex(issue, consumerMarker))
		}
		if summaryIndex >= 0 {
			issue = issue[:summaryIndex]
		}
		// Alerts recorded before the summary became the alarm name carry
		// "Network network-302 has VMware alarm <name> (yellow)", and resolved
		// ones keep that text in history.
		const legacyAlarmMarker = " has VMware alarm "
		if markerIndex := strings.Index(issue, legacyAlarmMarker); markerIndex >= 0 {
			issue = issue[markerIndex+len(legacyAlarmMarker):]
			for _, status := range []string{"red", "yellow", "green", "gray", "active"} {
				if trimmed, found := strings.CutSuffix(issue, " ("+status+")"); found {
					issue = trimmed
					break
				}
			}
		}
		if issue = strings.TrimSpace(strings.TrimSuffix(issue, ".")); issue != "" {
			return issue
		}
	case "vmware_health_state":
		if normalizedType == "storage_incident" {
			return "VMware datastore health"
		}
		return "VMware host health"
	}
	if normalizedType == "storage_incident" {
		return "Storage issue"
	}
	return "Infrastructure issue"
}

func attentionIncidentCode(alert alerts.Alert) string {
	if alert.Metadata != nil {
		if value, ok := alert.Metadata["incidentCode"].(string); ok {
			return strings.TrimSpace(value)
		}
	}
	return ""
}

func attentionResourceType(alert alerts.Alert) string {
	if alert.Metadata != nil {
		if value, ok := alert.Metadata["resourceType"].(string); ok {
			return strings.TrimSpace(value)
		}
	}
	return ""
}

// attentionMetricStatusStaleAfter matches the frontend's
// METRIC_ALERT_STATUS_STALE_MS: an older evaluation is no longer "now".
const attentionMetricStatusStaleAfter = 10 * time.Minute

// heldMetricAlertSummary describes a threshold alert that is still open below
// its trigger from the evaluator's live status, in the words the frontend's
// metric alert presentation uses. The alert's Message keeps the last reading
// that met the trigger, so on its own it reports a value the resource may no
// longer have. A breaching alert keeps its Message, which tracks the current
// breach, and a missing or stale status returns "" so the caller falls back
// to it.
func heldMetricAlertSummary(alert alerts.Alert, now time.Time) string {
	status := alert.MetricStatus
	if status == nil {
		return ""
	}
	if status.Phase != models.MetricAlertPhaseLatched && status.Phase != models.MetricAlertPhaseRecovering {
		return ""
	}
	if !attentionFinite(status.Value) || !attentionFinite(status.Trigger) || !attentionFinite(status.Recovery) {
		return ""
	}
	if status.ObservedAt.IsZero() || now.Sub(status.ObservedAt) > attentionMetricStatusStaleAfter {
		return ""
	}

	reading := attentionMetricReading(alert, status)
	recovery := formatAttentionMetricValue(status.Recovery, status.Unit)
	delay := status.RecoveryDelaySeconds
	if status.Phase == models.MetricAlertPhaseRecovering {
		if delay <= 0 {
			return fmt.Sprintf("%s now, recovering. Clears at %s or lower.", reading, recovery)
		}
		progress := ""
		if elapsed := status.RecoveryElapsedSeconds; elapsed > 0 {
			progress = ", " + formatAttentionElapsed(min(elapsed, delay)) + " so far"
		}
		return fmt.Sprintf("%s now, recovering. Clears after %s at %s or lower%s.",
			reading, formatAttentionSeconds(delay), recovery, progress)
	}
	hold := ""
	if delay > 0 {
		hold = " and stays there for " + formatAttentionSeconds(delay)
	}
	return fmt.Sprintf("%s now, back under the %s alert level. Stays open until it reaches %s or lower%s.",
		reading, formatAttentionMetricValue(status.Trigger, status.Unit), recovery, hold)
}

func attentionMetricReading(alert alerts.Alert, status *models.MetricAlertStatus) string {
	label := attentionMetricLabel(alert)
	value := formatAttentionMetricValue(status.Value, status.Unit)
	if status.EvaluationWindowSeconds <= 0 {
		return label + " " + value
	}
	latest := ""
	if status.RawValue != nil && attentionFinite(*status.RawValue) {
		latest = ", latest " + formatAttentionMetricValue(*status.RawValue, status.Unit)
	}
	return fmt.Sprintf("%s averaged %s over %s%s",
		label, value, formatAttentionSeconds(status.EvaluationWindowSeconds), latest)
}

// attentionMetricLabel names the metric the way the alert type label does. A
// guest raises one disk alert per disk, all named after the guest, so a disk
// reading also names its disk, as the alert message does.
func attentionMetricLabel(alert alerts.Alert) string {
	var label string
	switch alert.Type {
	case "cpu":
		label = "CPU"
	case "memory":
		label = "Memory"
	case "disk", "disk-usage":
		label = "Disk"
	case "usage":
		label = "Usage"
	case "diskRead":
		label = "Disk Read"
	case "diskWrite":
		label = "Disk Write"
	case "networkIn":
		label = "Network In"
	case "networkOut":
		label = "Network Out"
	case "temperature", "disk_temperature", "diskTemperature":
		label = "Temperature"
	default:
		label = firstAttentionText(alert.Type, "Reading")
	}
	if alert.Type == "disk" {
		if disk, ok := alert.Metadata["label"].(string); ok && strings.TrimSpace(disk) != "" {
			label += " (" + strings.TrimSpace(disk) + ")"
		}
	}
	return label
}

func formatAttentionMetricValue(value float64, unit string) string {
	switch unit {
	case "°C":
		return formatAttentionDecimal(attentionRoundHalfUp(value), 0) + "°C"
	case "%":
		if math.Abs(value) >= 10 {
			return formatAttentionDecimal(attentionRoundHalfUp(value), 0) + "%"
		}
		return formatAttentionDecimal(value, 1) + "%"
	case "":
		return formatAttentionDecimal(value, 1)
	default:
		return formatAttentionDecimal(value, 1) + " " + unit
	}
}

// formatAttentionDecimal rounds like the frontend's Number(value.toFixed(n)):
// from the float's exact value, so 2.55 (stored just under it) reads 2.5, and
// an exact tie such as 4.25 rounds away from zero where FormatFloat would
// round it to even. No trailing ".0".
func formatAttentionDecimal(value float64, places int) string {
	scale := math.Pow10(places)
	scaled := value * scale
	var rounded float64
	if math.FMA(value, scale, -scaled) == 0 && math.Abs(scaled-math.Trunc(scaled)) == 0.5 {
		rounded = math.Round(scaled) / scale
	} else {
		parsed, err := strconv.ParseFloat(strconv.FormatFloat(value, 'f', places, 64), 64)
		if err != nil {
			parsed = value
		}
		rounded = parsed
	}
	if rounded == 0 {
		rounded = 0 // a template literal prints -0 as "0"
	}
	return strconv.FormatFloat(rounded, 'f', -1, 64)
}

// attentionRoundHalfUp rounds like JavaScript's Math.round: ties go toward
// positive infinity, so -10.5 rounds to -10 where math.Round gives -11.
func attentionRoundHalfUp(value float64) float64 {
	floor := math.Floor(value)
	if value-floor >= 0.5 {
		return floor + 1
	}
	return floor
}

func formatAttentionSeconds(seconds int) string {
	plural := func(n float64, unit string) string {
		if n == 1 {
			return "1 " + unit
		}
		return strconv.FormatFloat(n, 'f', -1, 64) + " " + unit + "s"
	}
	switch {
	case seconds < 60:
		return plural(float64(seconds), "second")
	case seconds < 3600:
		return plural(math.Round(float64(seconds)/60), "minute")
	default:
		hours, err := strconv.ParseFloat(formatAttentionDecimal(float64(seconds)/3600, 1), 64)
		if err != nil {
			hours = float64(seconds) / 3600
		}
		return plural(hours, "hour")
	}
}

// formatAttentionElapsed rounds progress down, so a run four minutes and
// fifty seconds into a five minute delay does not read as finished.
func formatAttentionElapsed(seconds int) string {
	switch {
	case seconds < 60:
		return formatAttentionSeconds(seconds)
	case seconds < 3600:
		return formatAttentionSeconds(seconds / 60 * 60)
	default:
		return formatAttentionSeconds(seconds / 360 * 360)
	}
}

func attentionFinite(value float64) bool {
	return !math.IsNaN(value) && !math.IsInf(value, 0)
}

func firstAttentionText(values ...string) string {
	for _, value := range values {
		if normalized := strings.TrimSpace(value); normalized != "" {
			return normalized
		}
	}
	return ""
}

func canonicalAttentionIDs(values []string) []string {
	unique := make(map[string]struct{}, len(values))
	for _, value := range values {
		if normalized := strings.TrimSpace(value); normalized != "" {
			unique[normalized] = struct{}{}
		}
	}
	result := make([]string, 0, len(unique))
	for value := range unique {
		result = append(result, value)
	}
	sort.Strings(result)
	return result
}

func cloneAttentionEvidence(
	values []operationaltrust.EvidenceEnvelope,
) []operationaltrust.EvidenceEnvelope {
	result := make([]operationaltrust.EvidenceEnvelope, len(values))
	for i := range values {
		result[i] = values[i].Clone()
	}
	sort.Slice(result, func(i, j int) bool {
		if !result[i].ObservedAt.Equal(result[j].ObservedAt) {
			return result[i].ObservedAt.Before(result[j].ObservedAt)
		}
		return result[i].ID < result[j].ID
	})
	return result
}

func cloneAttentionTimeline(
	values []operationaltrust.LifecycleTransition,
) []operationaltrust.LifecycleTransition {
	result := make([]operationaltrust.LifecycleTransition, len(values))
	for i := range values {
		result[i] = values[i].Clone()
	}
	sort.Slice(result, func(i, j int) bool {
		if !result[i].At.Equal(result[j].At) {
			return result[i].At.Before(result[j].At)
		}
		return result[i].ID < result[j].ID
	})
	return result
}