Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

29.6 KB
Raw Blame
package recovery

import (
	"fmt"
	"sort"
	"strings"
	"time"

	"github.com/rcourtman/pulse-go-rewrite/internal/operationaltrust"
)

const (
	DefaultProtectionFreshnessWindow    = 7 * 24 * time.Hour
	DefaultProtectionVerificationWindow = 7 * 24 * time.Hour
)

var DefaultProtectionPosturePolicy = ProtectionPosturePolicy{
	FreshnessWindow:     DefaultProtectionFreshnessWindow,
	VerificationWindow:  DefaultProtectionVerificationWindow,
	RequireVerification: false,
}

func ProviderScopeForPoint(point RecoveryPoint) string {
	if scope := strings.TrimSpace(point.ProviderScope); scope != "" {
		return scope
	}
	switch point.Provider {
	case ProviderProxmoxPBS:
		if point.RepositoryRef != nil {
			if scope := strings.TrimSpace(point.RepositoryRef.Namespace); scope != "" {
				return scope
			}
		}
	case ProviderProxmoxPVE:
		if scope := recoveryDetailString(point, "instance"); scope != "" {
			return scope
		}
	case ProviderKubernetes:
		if scope := recoveryDetailString(point, "k8sClusterId"); scope != "" {
			return scope
		}
	case ProviderTrueNAS:
		if scope := recoveryDetailString(point, "connectionId"); scope != "" {
			return scope
		}
	}
	if point.SubjectRef != nil {
		if scope := strings.TrimSpace(point.SubjectRef.Namespace); scope != "" {
			return scope
		}
	}
	return "provider-default"
}

func NewRecoveryPointEvidence(
	point RecoveryPoint,
	collector string,
	ingestedAt time.Time,
) (*operationaltrust.EvidenceEnvelope, error) {
	collector = strings.TrimSpace(collector)
	if collector == "" {
		return nil, fmt.Errorf("recovery evidence collector is required")
	}
	observedAt := recoveryPointObservedAt(point)
	if observedAt.IsZero() {
		return nil, fmt.Errorf("recovery point observation time is required")
	}
	if ingestedAt.IsZero() {
		return nil, fmt.Errorf("recovery point ingestion time is required")
	}

	scope := ProviderScopeForPoint(point)
	source := operationaltrust.EvidenceSource{
		Provider:  string(point.Provider),
		Collector: collector,
		Instance:  scope,
	}
	subject := operationaltrust.EvidenceSubject{}
	if resourceID := strings.TrimSpace(point.SubjectResourceID); resourceID != "" {
		subject.ResourceID = resourceID
	} else {
		subject.ProviderRef = SubjectKeyForPoint(point)
		subject.ProviderScope = scope
	}
	evidenceID, err := operationaltrust.NewEvidenceID(
		source,
		subject,
		observedAt,
		point.ID,
	)
	if err != nil {
		return nil, err
	}

	envelope := &operationaltrust.EvidenceEnvelope{
		ID:           evidenceID,
		Source:       source,
		Subject:      subject,
		ObservedAt:   observedAt,
		IngestedAt:   ingestedAt.UTC(),
		Completeness: operationaltrust.EvidenceComplete,
		Confidence:   operationaltrust.EvidenceConfirmed,
		Permissions:  operationaltrust.EvidencePermissionsSufficient,
		PayloadRef: &operationaltrust.EvidencePayloadRef{
			Kind: "recovery-point",
			ID:   strings.TrimSpace(point.ID),
		},
	}
	if subject.ResourceID != "" && point.Provider == ProviderProxmoxPBS {
		index := DeriveIndex(point)
		matched := map[string]string{
			"providerScope": scope,
		}
		if value := strings.TrimSpace(index.ItemType); value != "" {
			matched["itemType"] = value
		}
		if value := strings.TrimSpace(index.EntityIDLabel); value != "" {
			matched["entityId"] = value
		}
		envelope.Confidence = operationaltrust.EvidenceInferred
		envelope.Reason = &operationaltrust.EvidenceReason{
			Code:    "provider_identity_correlation",
			Message: "The PBS subject was linked through a unique provider-scoped guest match.",
		}
		envelope.Correlation = &operationaltrust.IdentityCorrelation{
			Rule:           "provider_scoped_guest_identity",
			MatchedFields:  matched,
			CandidateCount: 1,
		}
	}
	if err := envelope.Validate(); err != nil {
		return nil, err
	}
	return envelope, nil
}

func NewProtectionProviderObservation(
	provider Provider,
	source string,
	scope string,
	jobState Outcome,
	historyCompleteness ProtectionHistoryCompleteness,
	permissions operationaltrust.EvidencePermissions,
	verificationExpected bool,
	observedAt time.Time,
	ingestedAt time.Time,
	reason *operationaltrust.EvidenceReason,
) (ProtectionProviderObservation, error) {
	source = strings.TrimSpace(source)
	scope = strings.TrimSpace(scope)
	if observedAt.IsZero() || ingestedAt.IsZero() {
		return ProtectionProviderObservation{}, fmt.Errorf(
			"protection provider observation times are required",
		)
	}
	// Provider observations are persisted in millisecond-precision SQLite
	// columns. Canonicalize the envelope and its identity to that precision so
	// a write/read round trip cannot make valid evidence fail exact timestamp
	// validation.
	observedAt = time.UnixMilli(observedAt.UTC().UnixMilli()).UTC()
	ingestedAt = time.UnixMilli(ingestedAt.UTC().UnixMilli()).UTC()
	evidenceSource := operationaltrust.EvidenceSource{
		Provider:  strings.TrimSpace(string(provider)),
		Collector: source,
		Instance:  scope,
	}
	evidenceSubject := operationaltrust.EvidenceSubject{
		ProviderRef:   strings.TrimSpace(string(provider)) + ":" + scope,
		ProviderScope: scope,
	}
	evidenceID, err := operationaltrust.NewEvidenceID(
		evidenceSource,
		evidenceSubject,
		observedAt,
		fmt.Sprintf(
			"%s:%s:%s:%s",
			jobState,
			historyCompleteness,
			permissions,
			scope,
		),
	)
	if err != nil {
		return ProtectionProviderObservation{}, err
	}

	completeness := operationaltrust.EvidenceUnavailable
	switch historyCompleteness {
	case ProtectionHistoryComplete:
		completeness = operationaltrust.EvidenceComplete
	case ProtectionHistoryPartial:
		completeness = operationaltrust.EvidencePartial
	}
	confidence := operationaltrust.EvidenceUnknown
	if historyCompleteness == ProtectionHistoryComplete &&
		permissions == operationaltrust.EvidencePermissionsSufficient {
		confidence = operationaltrust.EvidenceConfirmed
	}
	if (completeness != operationaltrust.EvidenceComplete ||
		confidence != operationaltrust.EvidenceConfirmed ||
		permissions != operationaltrust.EvidencePermissionsSufficient) &&
		reason == nil {
		reason = &operationaltrust.EvidenceReason{
			Code:    "provider_history_limited",
			Message: "Provider history could not support a complete protection assertion.",
		}
	}

	observation := ProtectionProviderObservation{
		ID:                   evidenceID,
		Provider:             provider,
		Source:               source,
		Scope:                scope,
		JobState:             jobState,
		HistoryCompleteness:  historyCompleteness,
		Permissions:          permissions,
		VerificationExpected: verificationExpected,
		ObservedAt:           observedAt.UTC(),
		IngestedAt:           ingestedAt.UTC(),
		Evidence: operationaltrust.EvidenceEnvelope{
			ID:           evidenceID,
			Source:       evidenceSource,
			Subject:      evidenceSubject,
			ObservedAt:   observedAt.UTC(),
			IngestedAt:   ingestedAt.UTC(),
			Completeness: completeness,
			Confidence:   confidence,
			Reason:       reason,
			Permissions:  permissions,
			PayloadRef: &operationaltrust.EvidencePayloadRef{
				Kind: "protection-provider-observation",
				ID:   evidenceID,
			},
		},
	}
	if err := observation.Validate(); err != nil {
		return ProtectionProviderObservation{}, err
	}
	return observation, nil
}

func BuildProtectionPostureFromPointsAt(
	subjectResourceID string,
	points []RecoveryPoint,
	observations []ProtectionProviderObservation,
	policy ProtectionPosturePolicy,
	now time.Time,
) ProtectionPosture {
	subjectResourceID = strings.TrimSpace(subjectResourceID)
	type aggregate struct {
		summary     ProtectionProviderSummary
		lastEventAt time.Time
	}
	byProviderScope := make(map[string]*aggregate)

	latestObservations := make(map[string]ProtectionProviderObservation)
	for _, observation := range observations {
		if err := observation.Validate(); err != nil {
			continue
		}
		key := providerScopeKey(observation.Provider, observation.Scope)
		current, exists := latestObservations[key]
		if !exists ||
			observation.ObservedAt.After(current.ObservedAt) ||
			(observation.ObservedAt.Equal(current.ObservedAt) && observation.ID > current.ID) {
			latestObservations[key] = observation.Clone()
		}
	}

	for _, point := range points {
		// Use the same identity normalization as SubjectKeyForPoint. A stored
		// Proxmox point can still carry a recognised legacy node-scoped ID
		// even though its recovery rollup is keyed to the canonical guest.
		if CanonicalSubjectResourceID(point.Provider, point.SubjectResourceID, point.SubjectRef) != subjectResourceID {
			continue
		}
		scope := ProviderScopeForPoint(point)
		key := providerScopeKey(point.Provider, scope)
		agg := byProviderScope[key]
		if agg == nil {
			agg = &aggregate{
				summary: ProtectionProviderSummary{
					Provider:            point.Provider,
					Source:              "legacy-recovery-point",
					Scope:               scope,
					JobState:            OutcomeUnknown,
					HistoryCompleteness: ProtectionHistoryUnknown,
					Permissions:         operationaltrust.EvidencePermissionsUnknown,
				},
			}
			byProviderScope[key] = agg
		}

		eventAt := recoveryPointObservedAt(point)
		if !eventAt.IsZero() {
			if agg.summary.LastAttemptAt == nil || eventAt.After(*agg.summary.LastAttemptAt) {
				value := eventAt.UTC()
				agg.summary.LastAttemptAt = &value
			}
			if agg.lastEventAt.IsZero() || eventAt.After(agg.lastEventAt) {
				agg.lastEventAt = eventAt
				if validRecoveryOutcome(point.Outcome) {
					agg.summary.JobState = point.Outcome
				} else {
					agg.summary.JobState = OutcomeUnknown
				}
			}
		}

		switch point.Kind {
		case KindBackup:
			agg.summary.BackupPointCount++
			observation, observed := latestObservations[key]
			confirmed := observed && pointConfirmedByObservation(point, observation)
			if point.Outcome == OutcomeSuccess && !eventAt.IsZero() {
				agg.summary.LastSuccessAt = latestProtectionTime(agg.summary.LastSuccessAt, &eventAt)
				if confirmed {
					agg.summary.ConfirmedLastSuccessAt = latestProtectionTime(
						agg.summary.ConfirmedLastSuccessAt,
						&eventAt,
					)
				}
			}
			if point.Outcome == OutcomeSuccess &&
				point.Verified != nil &&
				*point.Verified &&
				!eventAt.IsZero() {
				agg.summary.LastVerifiedAt = latestProtectionTime(agg.summary.LastVerifiedAt, &eventAt)
				if confirmed {
					agg.summary.ConfirmedLastVerifiedAt = latestProtectionTime(
						agg.summary.ConfirmedLastVerifiedAt,
						&eventAt,
					)
				}
			}
		case KindSnapshot:
			agg.summary.SnapshotPointCount++
		}
		if value := strings.TrimSpace(point.RepositoryResourceID); value != "" {
			agg.summary.RepositoryResourceIDs = append(
				agg.summary.RepositoryResourceIDs,
				value,
			)
		}
		if point.Evidence != nil {
			agg.summary.EvidenceIDs = append(agg.summary.EvidenceIDs, point.Evidence.ID)
			if source := strings.TrimSpace(point.Evidence.Source.Collector); source != "" {
				agg.summary.Source = source
			}
		}
	}

	for key, agg := range byProviderScope {
		observation, ok := latestObservations[key]
		if !ok {
			continue
		}
		agg.summary.Source = observation.Source
		agg.summary.HistoryCompleteness = observation.HistoryCompleteness
		agg.summary.Permissions = observation.Permissions
		agg.summary.VerificationExpected = observation.VerificationExpected
		agg.summary.UnreadScopes = cloneUnreadScopes(observation.UnreadScopes)
		agg.summary.EvidenceIDs = append(agg.summary.EvidenceIDs, observation.Evidence.ID)
		if !observation.ObservedAt.Before(agg.lastEventAt) &&
			observation.JobState != OutcomeUnknown {
			agg.summary.JobState = observation.JobState
		}
	}

	summaries := make([]ProtectionProviderSummary, 0, len(byProviderScope))
	for _, agg := range byProviderScope {
		agg.summary.RepositoryResourceIDs = sortedUnique(agg.summary.RepositoryResourceIDs)
		agg.summary.EvidenceIDs = sortedUnique(agg.summary.EvidenceIDs)
		summaries = append(summaries, agg.summary)
	}
	return DeriveProtectionPostureAt(subjectResourceID, summaries, policy, now)
}

// recoveryPointMigrationCollector marks evidence the store backfilled for
// legacy points. It was never produced by a provider read.
const recoveryPointMigrationCollector = "recovery-point-migration"

// pointConfirmedByObservation reports whether the provider collection behind
// observation re-read this point. Collectors renew a point's evidence only for
// rows a poll actually returned, stamped with that poll's observation time, so
// evidence ingested no earlier than the observation proves the point still
// existed on a scope that collection read. A point retained from an unread
// scope keeps its older ingestion time and may have been pruned since.
func pointConfirmedByObservation(
	point RecoveryPoint,
	observation ProtectionProviderObservation,
) bool {
	evidence := point.Evidence
	if evidence == nil || evidence.IngestedAt.IsZero() {
		return false
	}
	if strings.TrimSpace(evidence.Source.Collector) == recoveryPointMigrationCollector ||
		strings.TrimSpace(evidence.Source.Provider) != strings.TrimSpace(string(observation.Provider)) {
		return false
	}
	return !evidence.IngestedAt.Before(observation.ObservedAt)
}

func DeriveProtectionPostureAt(
	subjectResourceID string,
	summaries []ProtectionProviderSummary,
	policy ProtectionPosturePolicy,
	now time.Time,
) ProtectionPosture {
	subjectResourceID = strings.TrimSpace(subjectResourceID)
	if err := policy.Validate(); err != nil {
		operationaltrust.GetMetrics().ObserveProtectionEvaluationFailure("invalid_policy")
		policy = DefaultProtectionPosturePolicy
	}
	if now.IsZero() {
		now = time.Now().UTC()
	} else {
		now = now.UTC()
	}

	posture := ProtectionPosture{
		SubjectResourceID:     subjectResourceID,
		State:                 ProtectionStateUnknown,
		Freshness:             ProtectionFreshnessUnknown,
		Verification:          ProtectionVerificationUnknown,
		Coverage:              ProtectionCoverageUnknown,
		ProviderStates:        []ProtectionProviderState{},
		RepositoryResourceIDs: []string{},
		EvidenceIDs:           []string{},
		Reason:                ProtectionReasonNoHistory,
		Explanation:           "Pulse has no readable backup history for this resource, so it cannot say whether it is protected.",
		EvaluatedAt:           now,
	}

	hasUnknownBlock := false
	hasPartial := false
	hasCompleteHistory := false
	hasQualifyingSuccess := false
	hasSupportedQualifyingSuccess := false
	hasInvalidatingFailure := false
	hasSnapshotsOnly := false
	var unreadScopes []ProtectionUnreadScope
	verificationExpected := policy.RequireVerification
	supportedVerificationExpected := policy.RequireVerification
	var supportedLastSuccessAt *time.Time
	var supportedLastVerifiedAt *time.Time

	for _, summary := range summaries {
		if strings.TrimSpace(string(summary.Provider)) == "" ||
			strings.TrimSpace(summary.Scope) == "" {
			continue
		}
		if !validRecoveryOutcome(summary.JobState) {
			summary.JobState = OutcomeUnknown
		}
		if !summary.HistoryCompleteness.Valid() {
			summary.HistoryCompleteness = ProtectionHistoryUnknown
		}
		if !validProtectionPermissions(summary.Permissions) {
			summary.Permissions = operationaltrust.EvidencePermissionsUnknown
		}
		summary.Source = strings.TrimSpace(summary.Source)
		if summary.Source == "" {
			summary.Source = "legacy-recovery-point"
		}
		summary.RepositoryResourceIDs = sortedUnique(summary.RepositoryResourceIDs)
		summary.EvidenceIDs = sortedUnique(summary.EvidenceIDs)

		state := ProtectionProviderState{
			Provider:             summary.Provider,
			Source:               summary.Source,
			Scope:                strings.TrimSpace(summary.Scope),
			JobState:             summary.JobState,
			HistoryCompleteness:  summary.HistoryCompleteness,
			Permissions:          summary.Permissions,
			LastAttemptAt:        cloneProtectionTime(summary.LastAttemptAt),
			LastSuccessAt:        cloneProtectionTime(summary.LastSuccessAt),
			LastVerifiedAt:       cloneProtectionTime(summary.LastVerifiedAt),
			EvidenceIDs:          append([]string(nil), summary.EvidenceIDs...),
			VerificationExpected: summary.VerificationExpected,
		}
		if summary.HistoryCompleteness != ProtectionHistoryComplete {
			state.UnreadScopes = NormalizeProtectionUnreadScopes(summary.UnreadScopes)
			unreadScopes = append(unreadScopes, state.UnreadScopes...)
		}
		posture.ProviderStates = append(posture.ProviderStates, state)
		posture.RepositoryResourceIDs = append(
			posture.RepositoryResourceIDs,
			summary.RepositoryResourceIDs...,
		)
		posture.EvidenceIDs = append(posture.EvidenceIDs, summary.EvidenceIDs...)

		posture.LastAttemptAt = latestProtectionTime(posture.LastAttemptAt, summary.LastAttemptAt)
		posture.LastSuccessfulPointAt = latestProtectionTime(
			posture.LastSuccessfulPointAt,
			summary.LastSuccessAt,
		)
		posture.LastVerifiedAt = latestProtectionTime(posture.LastVerifiedAt, summary.LastVerifiedAt)

		switch summary.HistoryCompleteness {
		case ProtectionHistoryComplete:
			hasCompleteHistory = true
		case ProtectionHistoryPartial:
			hasPartial = true
		case ProtectionHistoryUnavailable, ProtectionHistoryUnknown:
			hasUnknownBlock = true
		}
		switch summary.Permissions {
		case operationaltrust.EvidencePermissionsSufficient:
		case operationaltrust.EvidencePermissionsPartial:
			hasPartial = true
		case operationaltrust.EvidencePermissionsUnknown:
			// Partial history already says part of the scope went unread;
			// unknown access describes that unread part, not the history
			// this source did return.
			if summary.HistoryCompleteness != ProtectionHistoryPartial {
				hasUnknownBlock = true
			}
		case operationaltrust.EvidencePermissionsDenied:
			hasUnknownBlock = true
		}

		if summary.LastSuccessAt != nil {
			hasQualifyingSuccess = true
		}
		if supportedSuccessAt, supportedVerifiedAt := supportedProtectionTimes(summary); supportedSuccessAt != nil {
			hasSupportedQualifyingSuccess = true
			supportedLastSuccessAt = latestProtectionTime(
				supportedLastSuccessAt,
				supportedSuccessAt,
			)
			supportedLastVerifiedAt = latestProtectionTime(
				supportedLastVerifiedAt,
				supportedVerifiedAt,
			)
			if summary.VerificationExpected {
				supportedVerificationExpected = true
			}
		}
		if summary.SnapshotPointCount > 0 && summary.BackupPointCount == 0 {
			hasSnapshotsOnly = true
		}
		if summary.VerificationExpected {
			verificationExpected = true
		}
		if summary.JobState == OutcomeFailed &&
			summary.LastAttemptAt != nil &&
			(summary.LastSuccessAt == nil ||
				!summary.LastAttemptAt.Before(*summary.LastSuccessAt)) {
			hasInvalidatingFailure = true
		}
	}

	sort.Slice(posture.ProviderStates, func(i, j int) bool {
		return providerStateSortKey(posture.ProviderStates[i]) <
			providerStateSortKey(posture.ProviderStates[j])
	})
	posture.RepositoryResourceIDs = sortedUnique(posture.RepositoryResourceIDs)
	posture.EvidenceIDs = sortedUnique(posture.EvidenceIDs)

	stateLastSuccessAt := posture.LastSuccessfulPointAt
	stateLastVerifiedAt := posture.LastVerifiedAt
	stateVerificationExpected := verificationExpected
	if hasSupportedQualifyingSuccess {
		stateLastSuccessAt = supportedLastSuccessAt
		stateLastVerifiedAt = supportedLastVerifiedAt
		stateVerificationExpected = supportedVerificationExpected
	}
	// A negative age is clock uncertainty, not a recent backup. Retain the
	// provider's timestamps as evidence, but never turn them into a current
	// or verified assessment until they are within the evaluation's past.
	if stateLastSuccessAt != nil && !stateLastSuccessAt.After(now) {
		if now.Sub(*stateLastSuccessAt) <= policy.FreshnessWindow {
			posture.Freshness = ProtectionFreshnessCurrent
		} else {
			posture.Freshness = ProtectionFreshnessStale
		}
	}
	if stateLastSuccessAt != nil && !stateLastSuccessAt.After(now) {
		switch {
		case stateLastVerifiedAt != nil && stateLastVerifiedAt.After(now):
			// Keep verification unknown; a future date proves no recency.
		case stateLastVerifiedAt != nil &&
			now.Sub(*stateLastVerifiedAt) <= policy.VerificationWindow:
			posture.Verification = ProtectionVerificationVerified
		case stateLastVerifiedAt != nil:
			posture.Verification = ProtectionVerificationStale
		case stateVerificationExpected:
			posture.Verification = ProtectionVerificationUnverified
		}
	}

	switch {
	case len(posture.ProviderStates) == 0:
		posture.Coverage = ProtectionCoverageUnknown
	case hasUnknownBlock:
		posture.Coverage = ProtectionCoverageUnknown
	case hasPartial:
		posture.Coverage = ProtectionCoveragePartial
	case hasCompleteHistory && !hasQualifyingSuccess:
		posture.Coverage = ProtectionCoverageNone
	default:
		posture.Coverage = ProtectionCoverageComplete
	}

	switch {
	case len(posture.ProviderStates) == 0:
		// Keep the initialized unknown explanation.
	case hasSupportedQualifyingSuccess &&
		posture.Freshness == ProtectionFreshnessCurrent &&
		!hasInvalidatingFailure &&
		(!stateVerificationExpected || posture.Verification == ProtectionVerificationVerified):
		// A backup the latest collection read stays a backup when some other
		// storage could not be read; coverage carries that gap.
		posture.State = ProtectionStateProtected
		posture.Reason = ProtectionReasonRecent
		switch {
		case posture.Verification == ProtectionVerificationVerified:
			posture.Reason = ProtectionReasonRecentVerified
			posture.Explanation = "A recent backup is available and it was verified recently."
		case hasPartial || hasUnknownBlock:
			posture.Explanation = "A recent backup is available."
		default:
			posture.Explanation = "A recent backup is available, and Pulse can see the full backup history."
		}
		if names := describeUnreadScopes(unreadScopes); names != "" {
			posture.Explanation += " Pulse could not read " + names + ", but that does not change this backup."
		} else if hasPartial {
			posture.Explanation += " Part of the backup history could not be read, but that does not change this backup."
		} else if hasUnknownBlock {
			posture.Explanation += " Another backup source could not be read, but that does not change this backup."
		}
	case hasSupportedQualifyingSuccess:
		posture.State = ProtectionStateAttention
		switch {
		case hasInvalidatingFailure:
			posture.Reason = ProtectionReasonSourceFailed
			posture.Explanation = "A backup exists, but one of its backup sources has reported a failure."
		case stateLastSuccessAt != nil && stateLastSuccessAt.After(now):
			posture.Reason = ProtectionReasonBackupFutureDated
			posture.Explanation = "The newest backup is dated in the future, so Pulse cannot confirm its age. Check the backup source and Pulse server clocks."
		case posture.Freshness == ProtectionFreshnessStale:
			posture.Reason = ProtectionReasonStale
			posture.Explanation = fmt.Sprintf("The newest confirmed backup is older than %s.", formatPostureWindow(policy.FreshnessWindow))
		case stateVerificationExpected && stateLastVerifiedAt != nil && stateLastVerifiedAt.After(now):
			posture.Reason = ProtectionReasonVerificationFutureDated
			posture.Explanation = "The backup verification is dated in the future, so Pulse cannot confirm it. Check the backup source and Pulse server clocks."
		case stateVerificationExpected && posture.Verification != ProtectionVerificationVerified:
			posture.Reason = ProtectionReasonVerificationOverdue
			posture.Explanation = "A recent backup exists, but its verification is missing or overdue."
		default:
			posture.Reason = ProtectionReasonHistoryPartial
			posture.Explanation = "A backup exists, but Pulse can see only part of the backup history or cannot read all of it."
		}
	case hasUnknownBlock:
		posture.State = ProtectionStateUnknown
		posture.Reason = ProtectionReasonHistoryUnreadable
		posture.Explanation = "Pulse cannot read the backup history (it is unavailable or Pulse lacks permission), so it cannot say whether this is protected."
	case hasQualifyingSuccess || hasPartial:
		posture.State = ProtectionStateAttention
		posture.Reason = ProtectionReasonHistoryPartial
		if hasQualifyingSuccess {
			posture.Explanation = "A backup exists, but Pulse can see only part of the backup history or cannot read all of it."
		} else {
			posture.Explanation = "Pulse can see only part of the backup history, so it cannot confirm a backup."
		}
		if names := describeUnreadScopes(unreadScopes); names != "" {
			posture.Explanation += " Pulse could not read " + names + "."
		}
	case hasCompleteHistory:
		posture.State = ProtectionStateUnprotected
		posture.Reason = ProtectionReasonNoBackup
		if hasSnapshotsOnly {
			posture.Reason = ProtectionReasonSnapshotsOnly
			posture.Explanation = "Only snapshots were found. A snapshot is not a separate backup, so it does not count."
		} else {
			posture.Explanation = "Pulse can see the full backup history, but found no successful, completed backup for this resource."
		}
	default:
		posture.State = ProtectionStateUnknown
	}

	operationaltrust.GetMetrics().ObserveProtectionEvaluation(string(posture.State))
	return posture
}

// supportedProtectionTimes returns the success and verification times a
// provider summary can stand behind. Complete history with sufficient access
// supports every point. A partial collection supports only the points it
// re-read; denied or unavailable history supports none.
func supportedProtectionTimes(summary ProtectionProviderSummary) (*time.Time, *time.Time) {
	switch {
	case summary.LastSuccessAt != nil &&
		summary.HistoryCompleteness == ProtectionHistoryComplete &&
		summary.Permissions == operationaltrust.EvidencePermissionsSufficient:
		return summary.LastSuccessAt, summary.LastVerifiedAt
	case summary.ConfirmedLastSuccessAt != nil &&
		summary.HistoryCompleteness == ProtectionHistoryPartial &&
		(summary.Permissions == operationaltrust.EvidencePermissionsPartial ||
			summary.Permissions == operationaltrust.EvidencePermissionsUnknown):
		return summary.ConfirmedLastSuccessAt, summary.ConfirmedLastVerifiedAt
	default:
		return nil, nil
	}
}

// describeUnreadScopes names unread provider scopes for an explanation, e.g.
// "pbs-offsite on delly". It names at most three and counts the rest.
func describeUnreadScopes(scopes []ProtectionUnreadScope) string {
	scopes = NormalizeProtectionUnreadScopes(scopes)
	if len(scopes) == 0 {
		return ""
	}
	names := make([]string, 0, len(scopes))
	for _, scope := range scopes {
		names = append(names, DescribeProtectionUnreadScope(scope))
	}
	const named = 3
	if len(names) > named {
		rest := len(names) - named
		names = append(names[:named], fmt.Sprintf("%d more", rest))
	}
	if len(names) == 1 {
		return names[0]
	}
	return strings.Join(names[:len(names)-1], ", ") + " and " + names[len(names)-1]
}

// DescribeProtectionUnreadScope renders one unread scope in operator terms.
func DescribeProtectionUnreadScope(scope ProtectionUnreadScope) string {
	switch scope.Kind {
	case ProtectionUnreadScopeNode:
		return "the storage list on node " + scope.Node
	case ProtectionUnreadScopeStorage:
		if scope.Node != "" {
			return scope.Storage + " on " + scope.Node
		}
		return scope.Storage
	case ProtectionUnreadScopeDatastore:
		return "the namespace list of datastore " + scope.Storage
	case ProtectionUnreadScopeNamespace:
		if scope.Namespace == "" {
			return "the root namespace of datastore " + scope.Storage
		}
		return "namespace " + scope.Namespace + " on datastore " + scope.Storage
	default:
		return ""
	}
}

func cloneUnreadScopes(scopes []ProtectionUnreadScope) []ProtectionUnreadScope {
	if len(scopes) == 0 {
		return nil
	}
	return append([]ProtectionUnreadScope(nil), scopes...)
}

func recoveryPointObservedAt(point RecoveryPoint) time.Time {
	if point.CompletedAt != nil && !point.CompletedAt.IsZero() {
		return point.CompletedAt.UTC()
	}
	if point.StartedAt != nil && !point.StartedAt.IsZero() {
		return point.StartedAt.UTC()
	}
	return time.Time{}
}

func recoveryDetailString(point RecoveryPoint, key string) string {
	if point.Details == nil {
		return ""
	}
	value, _ := point.Details[key].(string)
	return strings.TrimSpace(value)
}

func providerScopeKey(provider Provider, scope string) string {
	return strings.TrimSpace(string(provider)) + "\x00" + strings.TrimSpace(scope)
}

func providerStateSortKey(state ProtectionProviderState) string {
	return strings.Join([]string{
		strings.TrimSpace(string(state.Provider)),
		strings.TrimSpace(state.Scope),
		strings.TrimSpace(state.Source),
	}, "\x00")
}

func sortedUnique(values []string) []string {
	set := make(map[string]struct{}, len(values))
	for _, value := range values {
		if value = strings.TrimSpace(value); value != "" {
			set[value] = struct{}{}
		}
	}
	out := make([]string, 0, len(set))
	for value := range set {
		out = append(out, value)
	}
	sort.Strings(out)
	return out
}

func latestProtectionTime(current, candidate *time.Time) *time.Time {
	if candidate == nil {
		return cloneProtectionTime(current)
	}
	if current == nil || candidate.After(*current) {
		return cloneProtectionTime(candidate)
	}
	return cloneProtectionTime(current)
}

func cloneProtectionTime(value *time.Time) *time.Time {
	if value == nil {
		return nil
	}
	clone := value.UTC()
	return &clone
}

func validRecoveryOutcome(value Outcome) bool {
	switch value {
	case OutcomeSuccess, OutcomeWarning, OutcomeFailed, OutcomeRunning, OutcomeUnknown:
		return true
	default:
		return false
	}
}

func validProtectionPermissions(value operationaltrust.EvidencePermissions) bool {
	switch value {
	case operationaltrust.EvidencePermissionsSufficient,
		operationaltrust.EvidencePermissionsPartial,
		operationaltrust.EvidencePermissionsDenied,
		operationaltrust.EvidencePermissionsUnknown:
		return true
	default:
		return false
	}
}

// formatPostureWindow renders the freshness window the way the explanation
// reads it: whole days, then whole hours, otherwise minutes, so a sub-hour or
// fractional window is never rounded into a claim it did not make.
func formatPostureWindow(window time.Duration) string {
	plural := func(n int, unit string) string {
		if n == 1 {
			return fmt.Sprintf("1 %s", unit)
		}
		return fmt.Sprintf("%d %ss", n, unit)
	}
	switch {
	case window >= 24*time.Hour && window%(24*time.Hour) == 0:
		return plural(int(window/(24*time.Hour)), "day")
	case window >= time.Hour && window%time.Hour == 0:
		return plural(int(window/time.Hour), "hour")
	case window > 0:
		minutes := int((window + time.Minute - 1) / time.Minute)
		return plural(minutes, "minute")
	default:
		return "0 minutes"
	}
}