Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
package recovery
import (
"fmt"
"sort"
"strings"
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/operationaltrust"
)
const (
DefaultProtectionFreshnessWindow = 7 * 24 * time.Hour
DefaultProtectionVerificationWindow = 7 * 24 * time.Hour
)
var DefaultProtectionPosturePolicy = ProtectionPosturePolicy{
FreshnessWindow: DefaultProtectionFreshnessWindow,
VerificationWindow: DefaultProtectionVerificationWindow,
RequireVerification: false,
}
func ProviderScopeForPoint(point RecoveryPoint) string {
if scope := strings.TrimSpace(point.ProviderScope); scope != "" {
return scope
}
switch point.Provider {
case ProviderProxmoxPBS:
if point.RepositoryRef != nil {
if scope := strings.TrimSpace(point.RepositoryRef.Namespace); scope != "" {
return scope
}
}
case ProviderProxmoxPVE:
if scope := recoveryDetailString(point, "instance"); scope != "" {
return scope
}
case ProviderKubernetes:
if scope := recoveryDetailString(point, "k8sClusterId"); scope != "" {
return scope
}
case ProviderTrueNAS:
if scope := recoveryDetailString(point, "connectionId"); scope != "" {
return scope
}
}
if point.SubjectRef != nil {
if scope := strings.TrimSpace(point.SubjectRef.Namespace); scope != "" {
return scope
}
}
return "provider-default"
}
func NewRecoveryPointEvidence(
point RecoveryPoint,
collector string,
ingestedAt time.Time,
) (*operationaltrust.EvidenceEnvelope, error) {
collector = strings.TrimSpace(collector)
if collector == "" {
return nil, fmt.Errorf("recovery evidence collector is required")
}
observedAt := recoveryPointObservedAt(point)
if observedAt.IsZero() {
return nil, fmt.Errorf("recovery point observation time is required")
}
if ingestedAt.IsZero() {
return nil, fmt.Errorf("recovery point ingestion time is required")
}
scope := ProviderScopeForPoint(point)
source := operationaltrust.EvidenceSource{
Provider: string(point.Provider),
Collector: collector,
Instance: scope,
}
subject := operationaltrust.EvidenceSubject{}
if resourceID := strings.TrimSpace(point.SubjectResourceID); resourceID != "" {
subject.ResourceID = resourceID
} else {
subject.ProviderRef = SubjectKeyForPoint(point)
subject.ProviderScope = scope
}
evidenceID, err := operationaltrust.NewEvidenceID(
source,
subject,
observedAt,
point.ID,
)
if err != nil {
return nil, err
}
envelope := &operationaltrust.EvidenceEnvelope{
ID: evidenceID,
Source: source,
Subject: subject,
ObservedAt: observedAt,
IngestedAt: ingestedAt.UTC(),
Completeness: operationaltrust.EvidenceComplete,
Confidence: operationaltrust.EvidenceConfirmed,
Permissions: operationaltrust.EvidencePermissionsSufficient,
PayloadRef: &operationaltrust.EvidencePayloadRef{
Kind: "recovery-point",
ID: strings.TrimSpace(point.ID),
},
}
if subject.ResourceID != "" && point.Provider == ProviderProxmoxPBS {
index := DeriveIndex(point)
matched := map[string]string{
"providerScope": scope,
}
if value := strings.TrimSpace(index.ItemType); value != "" {
matched["itemType"] = value
}
if value := strings.TrimSpace(index.EntityIDLabel); value != "" {
matched["entityId"] = value
}
envelope.Confidence = operationaltrust.EvidenceInferred
envelope.Reason = &operationaltrust.EvidenceReason{
Code: "provider_identity_correlation",
Message: "The PBS subject was linked through a unique provider-scoped guest match.",
}
envelope.Correlation = &operationaltrust.IdentityCorrelation{
Rule: "provider_scoped_guest_identity",
MatchedFields: matched,
CandidateCount: 1,
}
}
if err := envelope.Validate(); err != nil {
return nil, err
}
return envelope, nil
}
func NewProtectionProviderObservation(
provider Provider,
source string,
scope string,
jobState Outcome,
historyCompleteness ProtectionHistoryCompleteness,
permissions operationaltrust.EvidencePermissions,
verificationExpected bool,
observedAt time.Time,
ingestedAt time.Time,
reason *operationaltrust.EvidenceReason,
) (ProtectionProviderObservation, error) {
source = strings.TrimSpace(source)
scope = strings.TrimSpace(scope)
if observedAt.IsZero() || ingestedAt.IsZero() {
return ProtectionProviderObservation{}, fmt.Errorf(
"protection provider observation times are required",
)
}
// Provider observations are persisted in millisecond-precision SQLite
// columns. Canonicalize the envelope and its identity to that precision so
// a write/read round trip cannot make valid evidence fail exact timestamp
// validation.
observedAt = time.UnixMilli(observedAt.UTC().UnixMilli()).UTC()
ingestedAt = time.UnixMilli(ingestedAt.UTC().UnixMilli()).UTC()
evidenceSource := operationaltrust.EvidenceSource{
Provider: strings.TrimSpace(string(provider)),
Collector: source,
Instance: scope,
}
evidenceSubject := operationaltrust.EvidenceSubject{
ProviderRef: strings.TrimSpace(string(provider)) + ":" + scope,
ProviderScope: scope,
}
evidenceID, err := operationaltrust.NewEvidenceID(
evidenceSource,
evidenceSubject,
observedAt,
fmt.Sprintf(
"%s:%s:%s:%s",
jobState,
historyCompleteness,
permissions,
scope,
),
)
if err != nil {
return ProtectionProviderObservation{}, err
}
completeness := operationaltrust.EvidenceUnavailable
switch historyCompleteness {
case ProtectionHistoryComplete:
completeness = operationaltrust.EvidenceComplete
case ProtectionHistoryPartial:
completeness = operationaltrust.EvidencePartial
}
confidence := operationaltrust.EvidenceUnknown
if historyCompleteness == ProtectionHistoryComplete &&
permissions == operationaltrust.EvidencePermissionsSufficient {
confidence = operationaltrust.EvidenceConfirmed
}
if (completeness != operationaltrust.EvidenceComplete ||
confidence != operationaltrust.EvidenceConfirmed ||
permissions != operationaltrust.EvidencePermissionsSufficient) &&
reason == nil {
reason = &operationaltrust.EvidenceReason{
Code: "provider_history_limited",
Message: "Provider history could not support a complete protection assertion.",
}
}
observation := ProtectionProviderObservation{
ID: evidenceID,
Provider: provider,
Source: source,
Scope: scope,
JobState: jobState,
HistoryCompleteness: historyCompleteness,
Permissions: permissions,
VerificationExpected: verificationExpected,
ObservedAt: observedAt.UTC(),
IngestedAt: ingestedAt.UTC(),
Evidence: operationaltrust.EvidenceEnvelope{
ID: evidenceID,
Source: evidenceSource,
Subject: evidenceSubject,
ObservedAt: observedAt.UTC(),
IngestedAt: ingestedAt.UTC(),
Completeness: completeness,
Confidence: confidence,
Reason: reason,
Permissions: permissions,
PayloadRef: &operationaltrust.EvidencePayloadRef{
Kind: "protection-provider-observation",
ID: evidenceID,
},
},
}
if err := observation.Validate(); err != nil {
return ProtectionProviderObservation{}, err
}
return observation, nil
}
func BuildProtectionPostureFromPointsAt(
subjectResourceID string,
points []RecoveryPoint,
observations []ProtectionProviderObservation,
policy ProtectionPosturePolicy,
now time.Time,
) ProtectionPosture {
subjectResourceID = strings.TrimSpace(subjectResourceID)
type aggregate struct {
summary ProtectionProviderSummary
lastEventAt time.Time
}
byProviderScope := make(map[string]*aggregate)
latestObservations := make(map[string]ProtectionProviderObservation)
for _, observation := range observations {
if err := observation.Validate(); err != nil {
continue
}
key := providerScopeKey(observation.Provider, observation.Scope)
current, exists := latestObservations[key]
if !exists ||
observation.ObservedAt.After(current.ObservedAt) ||
(observation.ObservedAt.Equal(current.ObservedAt) && observation.ID > current.ID) {
latestObservations[key] = observation.Clone()
}
}
for _, point := range points {
// Use the same identity normalization as SubjectKeyForPoint. A stored
// Proxmox point can still carry a recognised legacy node-scoped ID
// even though its recovery rollup is keyed to the canonical guest.
if CanonicalSubjectResourceID(point.Provider, point.SubjectResourceID, point.SubjectRef) != subjectResourceID {
continue
}
scope := ProviderScopeForPoint(point)
key := providerScopeKey(point.Provider, scope)
agg := byProviderScope[key]
if agg == nil {
agg = &aggregate{
summary: ProtectionProviderSummary{
Provider: point.Provider,
Source: "legacy-recovery-point",
Scope: scope,
JobState: OutcomeUnknown,
HistoryCompleteness: ProtectionHistoryUnknown,
Permissions: operationaltrust.EvidencePermissionsUnknown,
},
}
byProviderScope[key] = agg
}
eventAt := recoveryPointObservedAt(point)
if !eventAt.IsZero() {
if agg.summary.LastAttemptAt == nil || eventAt.After(*agg.summary.LastAttemptAt) {
value := eventAt.UTC()
agg.summary.LastAttemptAt = &value
}
if agg.lastEventAt.IsZero() || eventAt.After(agg.lastEventAt) {
agg.lastEventAt = eventAt
if validRecoveryOutcome(point.Outcome) {
agg.summary.JobState = point.Outcome
} else {
agg.summary.JobState = OutcomeUnknown
}
}
}
switch point.Kind {
case KindBackup:
agg.summary.BackupPointCount++
observation, observed := latestObservations[key]
confirmed := observed && pointConfirmedByObservation(point, observation)
if point.Outcome == OutcomeSuccess && !eventAt.IsZero() {
agg.summary.LastSuccessAt = latestProtectionTime(agg.summary.LastSuccessAt, &eventAt)
if confirmed {
agg.summary.ConfirmedLastSuccessAt = latestProtectionTime(
agg.summary.ConfirmedLastSuccessAt,
&eventAt,
)
}
}
if point.Outcome == OutcomeSuccess &&
point.Verified != nil &&
*point.Verified &&
!eventAt.IsZero() {
agg.summary.LastVerifiedAt = latestProtectionTime(agg.summary.LastVerifiedAt, &eventAt)
if confirmed {
agg.summary.ConfirmedLastVerifiedAt = latestProtectionTime(
agg.summary.ConfirmedLastVerifiedAt,
&eventAt,
)
}
}
case KindSnapshot:
agg.summary.SnapshotPointCount++
}
if value := strings.TrimSpace(point.RepositoryResourceID); value != "" {
agg.summary.RepositoryResourceIDs = append(
agg.summary.RepositoryResourceIDs,
value,
)
}
if point.Evidence != nil {
agg.summary.EvidenceIDs = append(agg.summary.EvidenceIDs, point.Evidence.ID)
if source := strings.TrimSpace(point.Evidence.Source.Collector); source != "" {
agg.summary.Source = source
}
}
}
for key, agg := range byProviderScope {
observation, ok := latestObservations[key]
if !ok {
continue
}
agg.summary.Source = observation.Source
agg.summary.HistoryCompleteness = observation.HistoryCompleteness
agg.summary.Permissions = observation.Permissions
agg.summary.VerificationExpected = observation.VerificationExpected
agg.summary.UnreadScopes = cloneUnreadScopes(observation.UnreadScopes)
agg.summary.EvidenceIDs = append(agg.summary.EvidenceIDs, observation.Evidence.ID)
if !observation.ObservedAt.Before(agg.lastEventAt) &&
observation.JobState != OutcomeUnknown {
agg.summary.JobState = observation.JobState
}
}
summaries := make([]ProtectionProviderSummary, 0, len(byProviderScope))
for _, agg := range byProviderScope {
agg.summary.RepositoryResourceIDs = sortedUnique(agg.summary.RepositoryResourceIDs)
agg.summary.EvidenceIDs = sortedUnique(agg.summary.EvidenceIDs)
summaries = append(summaries, agg.summary)
}
return DeriveProtectionPostureAt(subjectResourceID, summaries, policy, now)
}
// recoveryPointMigrationCollector marks evidence the store backfilled for
// legacy points. It was never produced by a provider read.
const recoveryPointMigrationCollector = "recovery-point-migration"
// pointConfirmedByObservation reports whether the provider collection behind
// observation re-read this point. Collectors renew a point's evidence only for
// rows a poll actually returned, stamped with that poll's observation time, so
// evidence ingested no earlier than the observation proves the point still
// existed on a scope that collection read. A point retained from an unread
// scope keeps its older ingestion time and may have been pruned since.
func pointConfirmedByObservation(
point RecoveryPoint,
observation ProtectionProviderObservation,
) bool {
evidence := point.Evidence
if evidence == nil || evidence.IngestedAt.IsZero() {
return false
}
if strings.TrimSpace(evidence.Source.Collector) == recoveryPointMigrationCollector ||
strings.TrimSpace(evidence.Source.Provider) != strings.TrimSpace(string(observation.Provider)) {
return false
}
return !evidence.IngestedAt.Before(observation.ObservedAt)
}
func DeriveProtectionPostureAt(
subjectResourceID string,
summaries []ProtectionProviderSummary,
policy ProtectionPosturePolicy,
now time.Time,
) ProtectionPosture {
subjectResourceID = strings.TrimSpace(subjectResourceID)
if err := policy.Validate(); err != nil {
operationaltrust.GetMetrics().ObserveProtectionEvaluationFailure("invalid_policy")
policy = DefaultProtectionPosturePolicy
}
if now.IsZero() {
now = time.Now().UTC()
} else {
now = now.UTC()
}
posture := ProtectionPosture{
SubjectResourceID: subjectResourceID,
State: ProtectionStateUnknown,
Freshness: ProtectionFreshnessUnknown,
Verification: ProtectionVerificationUnknown,
Coverage: ProtectionCoverageUnknown,
ProviderStates: []ProtectionProviderState{},
RepositoryResourceIDs: []string{},
EvidenceIDs: []string{},
Reason: ProtectionReasonNoHistory,
Explanation: "Pulse has no readable backup history for this resource, so it cannot say whether it is protected.",
EvaluatedAt: now,
}
hasUnknownBlock := false
hasPartial := false
hasCompleteHistory := false
hasQualifyingSuccess := false
hasSupportedQualifyingSuccess := false
hasInvalidatingFailure := false
hasSnapshotsOnly := false
var unreadScopes []ProtectionUnreadScope
verificationExpected := policy.RequireVerification
supportedVerificationExpected := policy.RequireVerification
var supportedLastSuccessAt *time.Time
var supportedLastVerifiedAt *time.Time
for _, summary := range summaries {
if strings.TrimSpace(string(summary.Provider)) == "" ||
strings.TrimSpace(summary.Scope) == "" {
continue
}
if !validRecoveryOutcome(summary.JobState) {
summary.JobState = OutcomeUnknown
}
if !summary.HistoryCompleteness.Valid() {
summary.HistoryCompleteness = ProtectionHistoryUnknown
}
if !validProtectionPermissions(summary.Permissions) {
summary.Permissions = operationaltrust.EvidencePermissionsUnknown
}
summary.Source = strings.TrimSpace(summary.Source)
if summary.Source == "" {
summary.Source = "legacy-recovery-point"
}
summary.RepositoryResourceIDs = sortedUnique(summary.RepositoryResourceIDs)
summary.EvidenceIDs = sortedUnique(summary.EvidenceIDs)
state := ProtectionProviderState{
Provider: summary.Provider,
Source: summary.Source,
Scope: strings.TrimSpace(summary.Scope),
JobState: summary.JobState,
HistoryCompleteness: summary.HistoryCompleteness,
Permissions: summary.Permissions,
LastAttemptAt: cloneProtectionTime(summary.LastAttemptAt),
LastSuccessAt: cloneProtectionTime(summary.LastSuccessAt),
LastVerifiedAt: cloneProtectionTime(summary.LastVerifiedAt),
EvidenceIDs: append([]string(nil), summary.EvidenceIDs...),
VerificationExpected: summary.VerificationExpected,
}
if summary.HistoryCompleteness != ProtectionHistoryComplete {
state.UnreadScopes = NormalizeProtectionUnreadScopes(summary.UnreadScopes)
unreadScopes = append(unreadScopes, state.UnreadScopes...)
}
posture.ProviderStates = append(posture.ProviderStates, state)
posture.RepositoryResourceIDs = append(
posture.RepositoryResourceIDs,
summary.RepositoryResourceIDs...,
)
posture.EvidenceIDs = append(posture.EvidenceIDs, summary.EvidenceIDs...)
posture.LastAttemptAt = latestProtectionTime(posture.LastAttemptAt, summary.LastAttemptAt)
posture.LastSuccessfulPointAt = latestProtectionTime(
posture.LastSuccessfulPointAt,
summary.LastSuccessAt,
)
posture.LastVerifiedAt = latestProtectionTime(posture.LastVerifiedAt, summary.LastVerifiedAt)
switch summary.HistoryCompleteness {
case ProtectionHistoryComplete:
hasCompleteHistory = true
case ProtectionHistoryPartial:
hasPartial = true
case ProtectionHistoryUnavailable, ProtectionHistoryUnknown:
hasUnknownBlock = true
}
switch summary.Permissions {
case operationaltrust.EvidencePermissionsSufficient:
case operationaltrust.EvidencePermissionsPartial:
hasPartial = true
case operationaltrust.EvidencePermissionsUnknown:
// Partial history already says part of the scope went unread;
// unknown access describes that unread part, not the history
// this source did return.
if summary.HistoryCompleteness != ProtectionHistoryPartial {
hasUnknownBlock = true
}
case operationaltrust.EvidencePermissionsDenied:
hasUnknownBlock = true
}
if summary.LastSuccessAt != nil {
hasQualifyingSuccess = true
}
if supportedSuccessAt, supportedVerifiedAt := supportedProtectionTimes(summary); supportedSuccessAt != nil {
hasSupportedQualifyingSuccess = true
supportedLastSuccessAt = latestProtectionTime(
supportedLastSuccessAt,
supportedSuccessAt,
)
supportedLastVerifiedAt = latestProtectionTime(
supportedLastVerifiedAt,
supportedVerifiedAt,
)
if summary.VerificationExpected {
supportedVerificationExpected = true
}
}
if summary.SnapshotPointCount > 0 && summary.BackupPointCount == 0 {
hasSnapshotsOnly = true
}
if summary.VerificationExpected {
verificationExpected = true
}
if summary.JobState == OutcomeFailed &&
summary.LastAttemptAt != nil &&
(summary.LastSuccessAt == nil ||
!summary.LastAttemptAt.Before(*summary.LastSuccessAt)) {
hasInvalidatingFailure = true
}
}
sort.Slice(posture.ProviderStates, func(i, j int) bool {
return providerStateSortKey(posture.ProviderStates[i]) <
providerStateSortKey(posture.ProviderStates[j])
})
posture.RepositoryResourceIDs = sortedUnique(posture.RepositoryResourceIDs)
posture.EvidenceIDs = sortedUnique(posture.EvidenceIDs)
stateLastSuccessAt := posture.LastSuccessfulPointAt
stateLastVerifiedAt := posture.LastVerifiedAt
stateVerificationExpected := verificationExpected
if hasSupportedQualifyingSuccess {
stateLastSuccessAt = supportedLastSuccessAt
stateLastVerifiedAt = supportedLastVerifiedAt
stateVerificationExpected = supportedVerificationExpected
}
// A negative age is clock uncertainty, not a recent backup. Retain the
// provider's timestamps as evidence, but never turn them into a current
// or verified assessment until they are within the evaluation's past.
if stateLastSuccessAt != nil && !stateLastSuccessAt.After(now) {
if now.Sub(*stateLastSuccessAt) <= policy.FreshnessWindow {
posture.Freshness = ProtectionFreshnessCurrent
} else {
posture.Freshness = ProtectionFreshnessStale
}
}
if stateLastSuccessAt != nil && !stateLastSuccessAt.After(now) {
switch {
case stateLastVerifiedAt != nil && stateLastVerifiedAt.After(now):
// Keep verification unknown; a future date proves no recency.
case stateLastVerifiedAt != nil &&
now.Sub(*stateLastVerifiedAt) <= policy.VerificationWindow:
posture.Verification = ProtectionVerificationVerified
case stateLastVerifiedAt != nil:
posture.Verification = ProtectionVerificationStale
case stateVerificationExpected:
posture.Verification = ProtectionVerificationUnverified
}
}
switch {
case len(posture.ProviderStates) == 0:
posture.Coverage = ProtectionCoverageUnknown
case hasUnknownBlock:
posture.Coverage = ProtectionCoverageUnknown
case hasPartial:
posture.Coverage = ProtectionCoveragePartial
case hasCompleteHistory && !hasQualifyingSuccess:
posture.Coverage = ProtectionCoverageNone
default:
posture.Coverage = ProtectionCoverageComplete
}
switch {
case len(posture.ProviderStates) == 0:
// Keep the initialized unknown explanation.
case hasSupportedQualifyingSuccess &&
posture.Freshness == ProtectionFreshnessCurrent &&
!hasInvalidatingFailure &&
(!stateVerificationExpected || posture.Verification == ProtectionVerificationVerified):
// A backup the latest collection read stays a backup when some other
// storage could not be read; coverage carries that gap.
posture.State = ProtectionStateProtected
posture.Reason = ProtectionReasonRecent
switch {
case posture.Verification == ProtectionVerificationVerified:
posture.Reason = ProtectionReasonRecentVerified
posture.Explanation = "A recent backup is available and it was verified recently."
case hasPartial || hasUnknownBlock:
posture.Explanation = "A recent backup is available."
default:
posture.Explanation = "A recent backup is available, and Pulse can see the full backup history."
}
if names := describeUnreadScopes(unreadScopes); names != "" {
posture.Explanation += " Pulse could not read " + names + ", but that does not change this backup."
} else if hasPartial {
posture.Explanation += " Part of the backup history could not be read, but that does not change this backup."
} else if hasUnknownBlock {
posture.Explanation += " Another backup source could not be read, but that does not change this backup."
}
case hasSupportedQualifyingSuccess:
posture.State = ProtectionStateAttention
switch {
case hasInvalidatingFailure:
posture.Reason = ProtectionReasonSourceFailed
posture.Explanation = "A backup exists, but one of its backup sources has reported a failure."
case stateLastSuccessAt != nil && stateLastSuccessAt.After(now):
posture.Reason = ProtectionReasonBackupFutureDated
posture.Explanation = "The newest backup is dated in the future, so Pulse cannot confirm its age. Check the backup source and Pulse server clocks."
case posture.Freshness == ProtectionFreshnessStale:
posture.Reason = ProtectionReasonStale
posture.Explanation = fmt.Sprintf("The newest confirmed backup is older than %s.", formatPostureWindow(policy.FreshnessWindow))
case stateVerificationExpected && stateLastVerifiedAt != nil && stateLastVerifiedAt.After(now):
posture.Reason = ProtectionReasonVerificationFutureDated
posture.Explanation = "The backup verification is dated in the future, so Pulse cannot confirm it. Check the backup source and Pulse server clocks."
case stateVerificationExpected && posture.Verification != ProtectionVerificationVerified:
posture.Reason = ProtectionReasonVerificationOverdue
posture.Explanation = "A recent backup exists, but its verification is missing or overdue."
default:
posture.Reason = ProtectionReasonHistoryPartial
posture.Explanation = "A backup exists, but Pulse can see only part of the backup history or cannot read all of it."
}
case hasUnknownBlock:
posture.State = ProtectionStateUnknown
posture.Reason = ProtectionReasonHistoryUnreadable
posture.Explanation = "Pulse cannot read the backup history (it is unavailable or Pulse lacks permission), so it cannot say whether this is protected."
case hasQualifyingSuccess || hasPartial:
posture.State = ProtectionStateAttention
posture.Reason = ProtectionReasonHistoryPartial
if hasQualifyingSuccess {
posture.Explanation = "A backup exists, but Pulse can see only part of the backup history or cannot read all of it."
} else {
posture.Explanation = "Pulse can see only part of the backup history, so it cannot confirm a backup."
}
if names := describeUnreadScopes(unreadScopes); names != "" {
posture.Explanation += " Pulse could not read " + names + "."
}
case hasCompleteHistory:
posture.State = ProtectionStateUnprotected
posture.Reason = ProtectionReasonNoBackup
if hasSnapshotsOnly {
posture.Reason = ProtectionReasonSnapshotsOnly
posture.Explanation = "Only snapshots were found. A snapshot is not a separate backup, so it does not count."
} else {
posture.Explanation = "Pulse can see the full backup history, but found no successful, completed backup for this resource."
}
default:
posture.State = ProtectionStateUnknown
}
operationaltrust.GetMetrics().ObserveProtectionEvaluation(string(posture.State))
return posture
}
// supportedProtectionTimes returns the success and verification times a
// provider summary can stand behind. Complete history with sufficient access
// supports every point. A partial collection supports only the points it
// re-read; denied or unavailable history supports none.
func supportedProtectionTimes(summary ProtectionProviderSummary) (*time.Time, *time.Time) {
switch {
case summary.LastSuccessAt != nil &&
summary.HistoryCompleteness == ProtectionHistoryComplete &&
summary.Permissions == operationaltrust.EvidencePermissionsSufficient:
return summary.LastSuccessAt, summary.LastVerifiedAt
case summary.ConfirmedLastSuccessAt != nil &&
summary.HistoryCompleteness == ProtectionHistoryPartial &&
(summary.Permissions == operationaltrust.EvidencePermissionsPartial ||
summary.Permissions == operationaltrust.EvidencePermissionsUnknown):
return summary.ConfirmedLastSuccessAt, summary.ConfirmedLastVerifiedAt
default:
return nil, nil
}
}
// describeUnreadScopes names unread provider scopes for an explanation, e.g.
// "pbs-offsite on delly". It names at most three and counts the rest.
func describeUnreadScopes(scopes []ProtectionUnreadScope) string {
scopes = NormalizeProtectionUnreadScopes(scopes)
if len(scopes) == 0 {
return ""
}
names := make([]string, 0, len(scopes))
for _, scope := range scopes {
names = append(names, DescribeProtectionUnreadScope(scope))
}
const named = 3
if len(names) > named {
rest := len(names) - named
names = append(names[:named], fmt.Sprintf("%d more", rest))
}
if len(names) == 1 {
return names[0]
}
return strings.Join(names[:len(names)-1], ", ") + " and " + names[len(names)-1]
}
// DescribeProtectionUnreadScope renders one unread scope in operator terms.
func DescribeProtectionUnreadScope(scope ProtectionUnreadScope) string {
switch scope.Kind {
case ProtectionUnreadScopeNode:
return "the storage list on node " + scope.Node
case ProtectionUnreadScopeStorage:
if scope.Node != "" {
return scope.Storage + " on " + scope.Node
}
return scope.Storage
case ProtectionUnreadScopeDatastore:
return "the namespace list of datastore " + scope.Storage
case ProtectionUnreadScopeNamespace:
if scope.Namespace == "" {
return "the root namespace of datastore " + scope.Storage
}
return "namespace " + scope.Namespace + " on datastore " + scope.Storage
default:
return ""
}
}
func cloneUnreadScopes(scopes []ProtectionUnreadScope) []ProtectionUnreadScope {
if len(scopes) == 0 {
return nil
}
return append([]ProtectionUnreadScope(nil), scopes...)
}
func recoveryPointObservedAt(point RecoveryPoint) time.Time {
if point.CompletedAt != nil && !point.CompletedAt.IsZero() {
return point.CompletedAt.UTC()
}
if point.StartedAt != nil && !point.StartedAt.IsZero() {
return point.StartedAt.UTC()
}
return time.Time{}
}
func recoveryDetailString(point RecoveryPoint, key string) string {
if point.Details == nil {
return ""
}
value, _ := point.Details[key].(string)
return strings.TrimSpace(value)
}
func providerScopeKey(provider Provider, scope string) string {
return strings.TrimSpace(string(provider)) + "\x00" + strings.TrimSpace(scope)
}
func providerStateSortKey(state ProtectionProviderState) string {
return strings.Join([]string{
strings.TrimSpace(string(state.Provider)),
strings.TrimSpace(state.Scope),
strings.TrimSpace(state.Source),
}, "\x00")
}
func sortedUnique(values []string) []string {
set := make(map[string]struct{}, len(values))
for _, value := range values {
if value = strings.TrimSpace(value); value != "" {
set[value] = struct{}{}
}
}
out := make([]string, 0, len(set))
for value := range set {
out = append(out, value)
}
sort.Strings(out)
return out
}
func latestProtectionTime(current, candidate *time.Time) *time.Time {
if candidate == nil {
return cloneProtectionTime(current)
}
if current == nil || candidate.After(*current) {
return cloneProtectionTime(candidate)
}
return cloneProtectionTime(current)
}
func cloneProtectionTime(value *time.Time) *time.Time {
if value == nil {
return nil
}
clone := value.UTC()
return &clone
}
func validRecoveryOutcome(value Outcome) bool {
switch value {
case OutcomeSuccess, OutcomeWarning, OutcomeFailed, OutcomeRunning, OutcomeUnknown:
return true
default:
return false
}
}
func validProtectionPermissions(value operationaltrust.EvidencePermissions) bool {
switch value {
case operationaltrust.EvidencePermissionsSufficient,
operationaltrust.EvidencePermissionsPartial,
operationaltrust.EvidencePermissionsDenied,
operationaltrust.EvidencePermissionsUnknown:
return true
default:
return false
}
}
// formatPostureWindow renders the freshness window the way the explanation
// reads it: whole days, then whole hours, otherwise minutes, so a sub-hour or
// fractional window is never rounded into a claim it did not make.
func formatPostureWindow(window time.Duration) string {
plural := func(n int, unit string) string {
if n == 1 {
return fmt.Sprintf("1 %s", unit)
}
return fmt.Sprintf("%d %ss", n, unit)
}
switch {
case window >= 24*time.Hour && window%(24*time.Hour) == 0:
return plural(int(window/(24*time.Hour)), "day")
case window >= time.Hour && window%time.Hour == 0:
return plural(int(window/time.Hour), "hour")
case window > 0:
minutes := int((window + time.Minute - 1) / time.Minute)
return plural(minutes, "minute")
default:
return "0 minutes"
}
}