Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
package recovery
import (
"fmt"
"reflect"
"strings"
"testing"
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/operationaltrust"
"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
)
func protectionTestSummary(
now time.Time,
lastSuccess *time.Time,
) ProtectionProviderSummary {
return ProtectionProviderSummary{
Provider: ProviderProxmoxPBS,
Source: "pbs-backup-enumeration",
Scope: "pbs-main",
JobState: OutcomeSuccess,
HistoryCompleteness: ProtectionHistoryComplete,
Permissions: operationaltrust.EvidencePermissionsSufficient,
VerificationExpected: true,
LastAttemptAt: cloneProtectionTime(lastSuccess),
LastSuccessAt: cloneProtectionTime(lastSuccess),
LastVerifiedAt: cloneProtectionTime(lastSuccess),
BackupPointCount: 1,
RepositoryResourceIDs: []string{
"repository:pbs-main/store-a",
},
EvidenceIDs: []string{"evidence:point-1", "evidence:provider-1"},
}
}
func TestDeriveProtectionPostureTruthTable(t *testing.T) {
t.Parallel()
now := time.Date(2026, 7, 19, 8, 0, 0, 0, time.UTC)
recent := now.Add(-2 * time.Hour)
stale := now.Add(-8 * 24 * time.Hour)
policy := DefaultProtectionPosturePolicy
tests := []struct {
name string
summaries []ProtectionProviderSummary
wantState ProtectionState
wantFresh ProtectionFreshness
wantVerify ProtectionVerification
wantCoverage ProtectionCoverage
wantText string
wantReason ProtectionReason
}{
{
name: "current verified PBS backup is protected",
summaries: []ProtectionProviderSummary{protectionTestSummary(now, &recent)},
wantState: ProtectionStateProtected,
wantFresh: ProtectionFreshnessCurrent,
wantVerify: ProtectionVerificationVerified,
wantCoverage: ProtectionCoverageComplete,
wantText: "A recent backup is available",
wantReason: ProtectionReasonRecentVerified,
},
{
name: "stale success needs attention",
summaries: []ProtectionProviderSummary{
protectionTestSummary(now, &stale),
},
wantState: ProtectionStateAttention,
wantFresh: ProtectionFreshnessStale,
wantVerify: ProtectionVerificationStale,
wantCoverage: ProtectionCoverageComplete,
wantText: "newest confirmed backup is older than 7 days",
wantReason: ProtectionReasonStale,
},
{
name: "missing expected verification needs attention",
summaries: func() []ProtectionProviderSummary {
summary := protectionTestSummary(now, &recent)
summary.LastVerifiedAt = nil
return []ProtectionProviderSummary{summary}
}(),
wantState: ProtectionStateAttention,
wantFresh: ProtectionFreshnessCurrent,
wantVerify: ProtectionVerificationUnverified,
wantCoverage: ProtectionCoverageComplete,
wantText: "verification is missing or overdue",
wantReason: ProtectionReasonVerificationOverdue,
},
{
name: "newer provider failure invalidates protected claim",
summaries: func() []ProtectionProviderSummary {
summary := protectionTestSummary(now, &recent)
failedAt := now.Add(-time.Hour)
summary.JobState = OutcomeFailed
summary.LastAttemptAt = &failedAt
return []ProtectionProviderSummary{summary}
}(),
wantState: ProtectionStateAttention,
wantFresh: ProtectionFreshnessCurrent,
wantVerify: ProtectionVerificationVerified,
wantCoverage: ProtectionCoverageComplete,
wantText: "one of its backup sources has reported a failure",
wantReason: ProtectionReasonSourceFailed,
},
{
name: "complete snapshot-only history is unprotected",
summaries: []ProtectionProviderSummary{
{
Provider: ProviderProxmoxPVE,
Source: "pve-snapshot-enumeration",
Scope: "pve-main",
JobState: OutcomeSuccess,
HistoryCompleteness: ProtectionHistoryComplete,
Permissions: operationaltrust.EvidencePermissionsSufficient,
SnapshotPointCount: 3,
EvidenceIDs: []string{"evidence:snapshot"},
},
},
wantState: ProtectionStateUnprotected,
wantFresh: ProtectionFreshnessUnknown,
wantVerify: ProtectionVerificationUnknown,
wantCoverage: ProtectionCoverageNone,
wantText: "A snapshot is not a separate backup",
wantReason: ProtectionReasonSnapshotsOnly,
},
{
name: "partial provider history is attention when a backup exists",
summaries: func() []ProtectionProviderSummary {
summary := protectionTestSummary(now, &recent)
summary.HistoryCompleteness = ProtectionHistoryPartial
summary.Permissions = operationaltrust.EvidencePermissionsPartial
return []ProtectionProviderSummary{summary}
}(),
wantState: ProtectionStateAttention,
wantFresh: ProtectionFreshnessCurrent,
wantVerify: ProtectionVerificationVerified,
wantCoverage: ProtectionCoveragePartial,
wantText: "only part of the backup history",
wantReason: ProtectionReasonHistoryPartial,
},
{
name: "partial history without any backup does not claim one exists",
summaries: func() []ProtectionProviderSummary {
summary := protectionTestSummary(now, nil)
summary.HistoryCompleteness = ProtectionHistoryPartial
summary.BackupPointCount = 0
return []ProtectionProviderSummary{summary}
}(),
wantState: ProtectionStateAttention,
wantFresh: ProtectionFreshnessUnknown,
wantVerify: ProtectionVerificationUnknown,
wantCoverage: ProtectionCoveragePartial,
wantText: "cannot confirm a backup",
wantReason: ProtectionReasonHistoryPartial,
},
{
name: "partial history with unknown access is attention when a backup exists",
summaries: func() []ProtectionProviderSummary {
summary := protectionTestSummary(now, &recent)
summary.HistoryCompleteness = ProtectionHistoryPartial
summary.Permissions = operationaltrust.EvidencePermissionsUnknown
return []ProtectionProviderSummary{summary}
}(),
wantState: ProtectionStateAttention,
wantFresh: ProtectionFreshnessCurrent,
wantVerify: ProtectionVerificationVerified,
wantCoverage: ProtectionCoveragePartial,
wantText: "only part of the backup history",
wantReason: ProtectionReasonHistoryPartial,
},
{
name: "partial history with unknown access and no backup does not claim one exists",
summaries: func() []ProtectionProviderSummary {
summary := protectionTestSummary(now, nil)
summary.HistoryCompleteness = ProtectionHistoryPartial
summary.Permissions = operationaltrust.EvidencePermissionsUnknown
summary.BackupPointCount = 0
return []ProtectionProviderSummary{summary}
}(),
wantState: ProtectionStateAttention,
wantFresh: ProtectionFreshnessUnknown,
wantVerify: ProtectionVerificationUnknown,
wantCoverage: ProtectionCoveragePartial,
wantText: "cannot confirm a backup",
wantReason: ProtectionReasonHistoryPartial,
},
{
name: "unavailable history with unknown access is unknown",
summaries: func() []ProtectionProviderSummary {
summary := protectionTestSummary(now, &recent)
summary.HistoryCompleteness = ProtectionHistoryUnavailable
summary.Permissions = operationaltrust.EvidencePermissionsUnknown
return []ProtectionProviderSummary{summary}
}(),
wantState: ProtectionStateUnknown,
wantFresh: ProtectionFreshnessCurrent,
wantVerify: ProtectionVerificationVerified,
wantCoverage: ProtectionCoverageUnknown,
wantText: "cannot read the backup history",
wantReason: ProtectionReasonHistoryUnreadable,
},
{
name: "permission denied is unknown",
summaries: func() []ProtectionProviderSummary {
summary := protectionTestSummary(now, &recent)
summary.HistoryCompleteness = ProtectionHistoryUnavailable
summary.Permissions = operationaltrust.EvidencePermissionsDenied
return []ProtectionProviderSummary{summary}
}(),
wantState: ProtectionStateUnknown,
wantFresh: ProtectionFreshnessCurrent,
wantVerify: ProtectionVerificationVerified,
wantCoverage: ProtectionCoverageUnknown,
wantText: "cannot read the backup history",
wantReason: ProtectionReasonHistoryUnreadable,
},
{
name: "confirmed PBS recovery is not invalidated by an unknown legacy provider",
summaries: func() []ProtectionProviderSummary {
confirmed := protectionTestSummary(now, &recent)
legacy := ProtectionProviderSummary{
Provider: ProviderProxmoxPVE,
Source: "legacy-recovery-point",
Scope: "pve-main",
JobState: OutcomeSuccess,
HistoryCompleteness: ProtectionHistoryUnknown,
Permissions: operationaltrust.EvidencePermissionsUnknown,
LastAttemptAt: &recent,
LastSuccessAt: &recent,
BackupPointCount: 1,
EvidenceIDs: []string{"evidence:legacy-pve"},
}
return []ProtectionProviderSummary{confirmed, legacy}
}(),
wantState: ProtectionStateProtected,
wantFresh: ProtectionFreshnessCurrent,
wantVerify: ProtectionVerificationVerified,
wantCoverage: ProtectionCoverageUnknown,
wantText: "does not change this backup",
wantReason: ProtectionReasonRecentVerified,
},
{
name: "no provider evidence is unknown",
summaries: nil,
wantState: ProtectionStateUnknown,
wantFresh: ProtectionFreshnessUnknown,
wantVerify: ProtectionVerificationUnknown,
wantCoverage: ProtectionCoverageUnknown,
wantText: "no readable backup history",
wantReason: ProtectionReasonNoHistory,
},
}
for _, test := range tests {
test := test
t.Run(test.name, func(t *testing.T) {
t.Parallel()
got := DeriveProtectionPostureAt(
"resource:vm-100",
test.summaries,
policy,
now,
)
if got.State != test.wantState {
t.Fatalf("state = %q, want %q; posture=%#v", got.State, test.wantState, got)
}
if got.Freshness != test.wantFresh {
t.Fatalf("freshness = %q, want %q", got.Freshness, test.wantFresh)
}
if got.Verification != test.wantVerify {
t.Fatalf("verification = %q, want %q", got.Verification, test.wantVerify)
}
if got.Coverage != test.wantCoverage {
t.Fatalf("coverage = %q, want %q", got.Coverage, test.wantCoverage)
}
if !strings.Contains(got.Explanation, test.wantText) {
t.Fatalf("explanation = %q, want substring %q", got.Explanation, test.wantText)
}
if got.Reason != test.wantReason {
t.Fatalf("reason = %q, want %q", got.Reason, test.wantReason)
}
if err := got.Validate(); err != nil {
t.Fatalf("Validate() error = %v", err)
}
})
}
}
func TestBuildProtectionPostureFromPointsUsesLatestProviderObservation(t *testing.T) {
t.Parallel()
now := time.Date(2026, 7, 19, 8, 0, 0, 0, time.UTC)
completedAt := now.Add(-time.Hour)
verified := true
point := RecoveryPoint{
ID: "pbs-backup:vm-100-2026-07-19",
Provider: ProviderProxmoxPBS,
Kind: KindBackup,
Mode: ModeRemote,
Outcome: OutcomeSuccess,
CompletedAt: &completedAt,
Verified: &verified,
SubjectResourceID: "resource:vm-100",
ProviderScope: "pbs-main",
}
envelope, err := NewRecoveryPointEvidence(point, "pbs-backup-inventory", now)
if err != nil {
t.Fatalf("NewRecoveryPointEvidence() error = %v", err)
}
point.Evidence = envelope
older, err := NewProtectionProviderObservation(
ProviderProxmoxPBS,
"pbs-backup-enumeration",
"pbs-main",
OutcomeFailed,
ProtectionHistoryUnavailable,
operationaltrust.EvidencePermissionsUnknown,
true,
now.Add(-2*time.Hour),
now.Add(-2*time.Hour),
&operationaltrust.EvidenceReason{Code: "pbs_timeout"},
)
if err != nil {
t.Fatalf("older observation error = %v", err)
}
current, err := NewProtectionProviderObservation(
ProviderProxmoxPBS,
"pbs-backup-enumeration",
"pbs-main",
OutcomeSuccess,
ProtectionHistoryComplete,
operationaltrust.EvidencePermissionsSufficient,
true,
now,
now,
nil,
)
if err != nil {
t.Fatalf("current observation error = %v", err)
}
got := BuildProtectionPostureFromPointsAt(
"resource:vm-100",
[]RecoveryPoint{point},
[]ProtectionProviderObservation{current, older},
DefaultProtectionPosturePolicy,
now,
)
if got.State != ProtectionStateProtected {
t.Fatalf("state = %q, want protected; posture=%#v", got.State, got)
}
if len(got.ProviderStates) != 1 {
t.Fatalf("provider states = %d, want 1", len(got.ProviderStates))
}
if got.ProviderStates[0].HistoryCompleteness != ProtectionHistoryComplete {
t.Fatalf(
"history completeness = %q, want complete",
got.ProviderStates[0].HistoryCompleteness,
)
}
if len(got.EvidenceIDs) != 2 {
t.Fatalf("evidence ids = %#v, want point and latest provider evidence", got.EvidenceIDs)
}
}
func TestBuildProtectionPostureDoesNotTreatSuccessfulTaskAsAvailableBackup(t *testing.T) {
t.Parallel()
now := time.Date(2026, 8, 4, 8, 0, 0, 0, time.UTC)
completedAt := now.Add(-time.Hour)
point := RecoveryPoint{
ID: "pve-task:vm-100-2026-08-04",
Provider: ProviderProxmoxPVE,
Kind: KindOther,
Mode: ModeLocal,
Outcome: OutcomeSuccess,
CompletedAt: &completedAt,
SubjectResourceID: "resource:vm-100",
ProviderScope: "pve-main",
}
envelope, err := NewRecoveryPointEvidence(point, "pve-backup-task-inventory", now)
if err != nil {
t.Fatalf("NewRecoveryPointEvidence() error = %v", err)
}
point.Evidence = envelope
observation, err := NewProtectionProviderObservation(
ProviderProxmoxPVE,
"pve-backup-enumeration",
"pve-main",
OutcomeSuccess,
ProtectionHistoryComplete,
operationaltrust.EvidencePermissionsSufficient,
false,
now,
now,
nil,
)
if err != nil {
t.Fatalf("NewProtectionProviderObservation() error = %v", err)
}
got := BuildProtectionPostureFromPointsAt(
"resource:vm-100",
[]RecoveryPoint{point},
[]ProtectionProviderObservation{observation},
DefaultProtectionPosturePolicy,
now,
)
if got.State != ProtectionStateUnprotected {
t.Fatalf("state = %q, want unprotected; posture=%#v", got.State, got)
}
if got.LastSuccessfulPointAt != nil {
t.Fatalf("last successful point = %v, want nil for task-only evidence", got.LastSuccessfulPointAt)
}
if !strings.Contains(got.Explanation, "found no successful, completed backup") {
t.Fatalf("explanation = %q, want no qualifying backup", got.Explanation)
}
if got.Reason != ProtectionReasonNoBackup {
t.Fatalf("reason = %q, want %q", got.Reason, ProtectionReasonNoBackup)
}
}
func TestNewProtectionProviderObservationRequiresTypedLimitation(t *testing.T) {
t.Parallel()
now := time.Date(2026, 7, 19, 8, 0, 0, 0, time.UTC)
observation, err := NewProtectionProviderObservation(
ProviderProxmoxPBS,
"pbs-backup-enumeration",
"pbs-main",
OutcomeFailed,
ProtectionHistoryUnavailable,
operationaltrust.EvidencePermissionsDenied,
true,
now,
now,
&operationaltrust.EvidenceReason{
Code: "pbs_access_denied",
Message: "PBS did not authorize backup history enumeration.",
},
)
if err != nil {
t.Fatalf("NewProtectionProviderObservation() error = %v", err)
}
if observation.Evidence.Reason == nil ||
observation.Evidence.Reason.Code != "pbs_access_denied" {
t.Fatalf("reason = %#v, want pbs_access_denied", observation.Evidence.Reason)
}
if observation.Evidence.Permissions != operationaltrust.EvidencePermissionsDenied {
t.Fatalf("permissions = %q, want denied", observation.Evidence.Permissions)
}
}
func TestNewProtectionProviderObservationCanonicalizesPersistentTimestampPrecision(t *testing.T) {
t.Parallel()
observedAt := time.Date(2026, 8, 4, 22, 0, 1, 752103663, time.UTC)
ingestedAt := observedAt.Add(321 * time.Microsecond)
observation, err := NewProtectionProviderObservation(
ProviderProxmoxPVE,
"pve-backup-enumeration",
"homelab",
OutcomeSuccess,
ProtectionHistoryComplete,
operationaltrust.EvidencePermissionsSufficient,
false,
observedAt,
ingestedAt,
nil,
)
if err != nil {
t.Fatalf("NewProtectionProviderObservation() error = %v", err)
}
wantObservedAt := time.UnixMilli(observedAt.UnixMilli()).UTC()
wantIngestedAt := time.UnixMilli(ingestedAt.UnixMilli()).UTC()
if !observation.ObservedAt.Equal(wantObservedAt) ||
!observation.Evidence.ObservedAt.Equal(wantObservedAt) {
t.Fatalf(
"observed times = %s / %s, want %s",
observation.ObservedAt,
observation.Evidence.ObservedAt,
wantObservedAt,
)
}
if !observation.IngestedAt.Equal(wantIngestedAt) ||
!observation.Evidence.IngestedAt.Equal(wantIngestedAt) {
t.Fatalf(
"ingested times = %s / %s, want %s",
observation.IngestedAt,
observation.Evidence.IngestedAt,
wantIngestedAt,
)
}
}
func TestDeriveProtectionPostureFailsClosedToDefaultPolicy(t *testing.T) {
t.Parallel()
now := time.Date(2026, 7, 19, 8, 0, 0, 0, time.UTC)
recent := now.Add(-time.Hour)
posture := DeriveProtectionPostureAt(
"resource:vm-100",
[]ProtectionProviderSummary{protectionTestSummary(now, &recent)},
ProtectionPosturePolicy{},
now,
)
if posture.State != ProtectionStateProtected {
t.Fatalf("state = %q, want protected after default-policy fallback", posture.State)
}
if posture.Freshness != ProtectionFreshnessCurrent {
t.Fatalf("freshness = %q, want current after default-policy fallback", posture.Freshness)
}
}
func TestFormatPostureWindowReadsAsDaysOrHours(t *testing.T) {
for _, tc := range []struct {
window time.Duration
want string
}{
{7 * 24 * time.Hour, "7 days"},
{24 * time.Hour, "1 day"},
{36 * time.Hour, "36 hours"},
{time.Hour, "1 hour"},
{30 * time.Minute, "30 minutes"},
{90 * time.Minute, "90 minutes"},
{time.Minute, "1 minute"},
{0, "0 minutes"},
} {
if got := formatPostureWindow(tc.window); got != tc.want {
t.Fatalf("formatPostureWindow(%s) = %q, want %q", tc.window, got, tc.want)
}
}
}
// Canonical posture and rollups must select the same subject. Historical
// node-scoped aliases are not another guest, nor permission to use another
// installation's backup merely because the VMID/name is the same.
func TestBuildProtectionPostureUsesCanonicalPointIdentity(t *testing.T) {
t.Parallel()
now := time.Date(2026, 10, 25, 12, 0, 0, 0, time.UTC)
stale, fresh := now.Add(-17*24*time.Hour), now.Add(-time.Hour)
for _, kind := range []struct {
typ unifiedresources.ResourceType
ref string
}{
{unifiedresources.ResourceTypeVM, "proxmox-vm"},
{unifiedresources.ResourceTypeSystemContainer, "proxmox-lxc"},
} {
for _, provider := range []Provider{ProviderProxmoxPVE, ProviderProxmoxPBS} {
t.Run(fmt.Sprintf("%s/%s", provider, kind.typ), func(t *testing.T) {
const instance = "estate-a"
source := instance + ":old-node:101"
canonical := unifiedresources.ProxmoxGuestCanonicalID(kind.typ, instance, 101)
aliases := []string{source, unifiedresources.SourceSpecificID(kind.typ, unifiedresources.SourceProxmox, source)}
if kind.typ == unifiedresources.ResourceTypeSystemContainer {
aliases = append(aliases, strings.Replace(aliases[1], "system-container-", "lxc-", 1))
}
observation, err := NewProtectionProviderObservation(provider, "backup-enumeration", instance, OutcomeSuccess, ProtectionHistoryComplete, operationaltrust.EvidencePermissionsSufficient, false, now, now, nil)
if err != nil {
t.Fatal(err)
}
baseline := RecoveryPoint{ID: "old-backup", Provider: provider, ProviderScope: instance, Kind: KindBackup, Mode: ModeLocal, Outcome: OutcomeSuccess, CompletedAt: &stale, SubjectResourceID: canonical}
for _, alias := range aliases {
t.Run(alias, func(t *testing.T) {
latest := baseline
latest.ID = "fresh-backup"
latest.CompletedAt = &fresh
latest.SubjectResourceID = alias
latest.SubjectRef = &ExternalRef{Type: kind.ref, ID: source, Namespace: instance, Name: "same-name"}
if SubjectKeyForPoint(latest) != SubjectKeyForPoint(baseline) {
t.Fatal("control does not share a canonical rollup")
}
got := BuildProtectionPostureFromPointsAt(canonical, []RecoveryPoint{baseline, latest}, []ProtectionProviderObservation{observation}, DefaultProtectionPosturePolicy, now)
if got.State != ProtectionStateProtected || got.Freshness != ProtectionFreshnessCurrent || got.LastSuccessfulPointAt == nil || !got.LastSuccessfulPointAt.Equal(fresh) {
t.Fatalf("fresh legacy point ignored: state=%s freshness=%s success=%v", got.State, got.Freshness, got.LastSuccessfulPointAt)
}
// Running files and successful task records remain distinct from a
// completed backup, even when their identity is a recognised alias.
for _, limiting := range []RecoveryPoint{
func() RecoveryPoint { p := latest; p.Outcome = OutcomeRunning; return p }(),
func() RecoveryPoint { p := latest; p.Kind = KindOther; return p }(),
func() RecoveryPoint {
p := latest
p.SubjectResourceID = unifiedresources.ProxmoxGuestCanonicalID(kind.typ, "estate-b", 101)
p.SubjectRef = &ExternalRef{Type: kind.ref, ID: "estate-b:node:101", Namespace: "estate-b", Name: "same-name"}
return p
}(),
func() RecoveryPoint {
p := latest
p.SubjectResourceID = unifiedresources.ProxmoxGuestCanonicalID(kind.typ, instance, 102)
p.SubjectRef = &ExternalRef{Type: kind.ref, ID: instance + ":node:102", Namespace: instance}
return p
}(),
} {
got := BuildProtectionPostureFromPointsAt(canonical, []RecoveryPoint{baseline, limiting}, []ProtectionProviderObservation{observation}, DefaultProtectionPosturePolicy, now)
if got.State != ProtectionStateAttention || got.LastSuccessfulPointAt == nil || !got.LastSuccessfulPointAt.Equal(stale) {
t.Fatalf("unqualified/foreign point renewed protection: %+v", got)
}
}
partial := observation.Clone()
partial.HistoryCompleteness = ProtectionHistoryPartial
partial.Evidence.Completeness = operationaltrust.EvidencePartial
partial.Evidence.Reason = &operationaltrust.EvidenceReason{Code: "partial_inventory"}
got = BuildProtectionPostureFromPointsAt(canonical, []RecoveryPoint{baseline, latest}, []ProtectionProviderObservation{partial}, DefaultProtectionPosturePolicy, now)
if got.State == ProtectionStateProtected || got.Coverage != ProtectionCoveragePartial {
t.Fatalf("alias bypassed limited collection: %+v", got)
}
})
}
})
}
}
}
var clockBoundReason = map[string]ProtectionReason{
"future-backup": ProtectionReasonBackupFutureDated,
"future-backup-without-verification": ProtectionReasonBackupFutureDated,
"one-nanosecond-ahead": ProtectionReasonBackupFutureDated,
"future-verification": ProtectionReasonVerificationFutureDated,
"optional-future-verification": ProtectionReasonRecent,
"exactly-now": ProtectionReasonRecentVerified,
"exact-window-boundary": ProtectionReasonRecentVerified,
"expired-window": ProtectionReasonStale,
"missing-verification": ProtectionReasonVerificationOverdue,
}
func TestProtectionPostureClockBounds(t *testing.T) {
t.Parallel()
now := time.Date(2026, 10, 25, 12, 0, 0, 0, time.FixedZone("source", 3600))
recent := now.Add(-time.Hour)
future := now.Add(48 * time.Hour)
atBoundary := now.Add(-DefaultProtectionFreshnessWindow)
tooOld := atBoundary.Add(-time.Nanosecond)
justAhead := now.Add(time.Nanosecond)
for _, tc := range []struct {
name string
success time.Time
verified time.Time
require bool
wantState ProtectionState
wantFresh ProtectionFreshness
wantVerified ProtectionVerification
wantText string
}{
{"future-backup", future, future, true, ProtectionStateAttention, ProtectionFreshnessUnknown, ProtectionVerificationUnknown, "newest backup is dated in the future"},
{"future-backup-without-verification", future, time.Time{}, false, ProtectionStateAttention, ProtectionFreshnessUnknown, ProtectionVerificationUnknown, "newest backup is dated in the future"},
{"one-nanosecond-ahead", justAhead, justAhead, true, ProtectionStateAttention, ProtectionFreshnessUnknown, ProtectionVerificationUnknown, "newest backup is dated in the future"},
{"future-verification", recent, future, true, ProtectionStateAttention, ProtectionFreshnessCurrent, ProtectionVerificationUnknown, "verification is dated in the future"},
{"optional-future-verification", recent, future, false, ProtectionStateProtected, ProtectionFreshnessCurrent, ProtectionVerificationUnknown, "A recent backup is available"},
{"exactly-now", now, now, true, ProtectionStateProtected, ProtectionFreshnessCurrent, ProtectionVerificationVerified, "verified recently"},
{"exact-window-boundary", atBoundary, atBoundary, true, ProtectionStateProtected, ProtectionFreshnessCurrent, ProtectionVerificationVerified, "verified recently"},
{"expired-window", tooOld, tooOld, true, ProtectionStateAttention, ProtectionFreshnessStale, ProtectionVerificationStale, "older than 7 days"},
{"missing-verification", recent, time.Time{}, true, ProtectionStateAttention, ProtectionFreshnessCurrent, ProtectionVerificationUnverified, "missing or overdue"},
} {
t.Run(tc.name, func(t *testing.T) {
summary := protectionTestSummary(now, &tc.success)
summary.VerificationExpected = tc.require
summary.LastVerifiedAt = nil
if !tc.verified.IsZero() {
summary.LastVerifiedAt = &tc.verified
}
before := summary
posture := DeriveProtectionPostureAt("resource:vm-100", []ProtectionProviderSummary{summary}, DefaultProtectionPosturePolicy, now)
if posture.State != tc.wantState || posture.Freshness != tc.wantFresh || posture.Verification != tc.wantVerified {
t.Fatalf("state=%s freshness=%s verification=%s; want %s/%s/%s", posture.State, posture.Freshness, posture.Verification, tc.wantState, tc.wantFresh, tc.wantVerified)
}
if !strings.Contains(posture.Explanation, tc.wantText) {
t.Errorf("explanation=%q, want %q", posture.Explanation, tc.wantText)
}
if want := clockBoundReason[tc.name]; posture.Reason != want {
t.Errorf("reason=%q, want %q", posture.Reason, want)
}
if posture.Coverage != ProtectionCoverageComplete || !posture.LastSuccessfulPointAt.Equal(tc.success) || len(posture.ProviderStates) != 1 || !posture.ProviderStates[0].LastSuccessAt.Equal(tc.success) || !reflect.DeepEqual(summary, before) {
t.Error("clock assessment changed source evidence or collection coverage")
}
if tc.verified.IsZero() != (posture.LastVerifiedAt == nil) || (posture.LastVerifiedAt != nil && !posture.LastVerifiedAt.Equal(tc.verified)) {
t.Error("clock assessment rewrote the reported verification date")
}
if err := posture.Validate(); err != nil {
t.Fatal(err)
}
})
}
}
func TestProtectionPostureFuturePointsRetainAccessBoundaries(t *testing.T) {
t.Parallel()
now := time.Date(2026, 10, 25, 11, 0, 0, 0, time.UTC)
recent, future := now.Add(-time.Hour), now.Add(48*time.Hour)
verified := true
for _, provider := range []Provider{ProviderProxmoxPVE, ProviderProxmoxPBS} {
t.Run(string(provider), func(t *testing.T) {
point := RecoveryPoint{ID: "backup", Provider: provider, ProviderScope: "source-a", Kind: KindBackup, Mode: ModeRemote, Outcome: OutcomeSuccess, CompletedAt: &future, Verified: &verified, SubjectResourceID: "resource:vm-100"}
observation, err := NewProtectionProviderObservation(provider, "inventory", "source-a", OutcomeSuccess, ProtectionHistoryComplete, operationaltrust.EvidencePermissionsSufficient, true, now, now, nil)
if err != nil {
t.Fatal(err)
}
build := func(points []RecoveryPoint, observations []ProtectionProviderObservation) ProtectionPosture {
return BuildProtectionPostureFromPointsAt(point.SubjectResourceID, points, observations, DefaultProtectionPosturePolicy, now)
}
posture := build([]RecoveryPoint{point}, []ProtectionProviderObservation{observation})
if posture.State != ProtectionStateAttention || posture.Freshness != ProtectionFreshnessUnknown || posture.Verification != ProtectionVerificationUnknown || !posture.LastSuccessfulPointAt.Equal(future) {
t.Fatalf("future point claimed current protection or lost evidence: %+v", posture)
}
denied, err := NewProtectionProviderObservation(provider, "inventory", "source-a", OutcomeFailed, ProtectionHistoryUnavailable, operationaltrust.EvidencePermissionsDenied, true, now, now, nil)
if err != nil {
t.Fatal(err)
}
if got := build([]RecoveryPoint{point}, []ProtectionProviderObservation{denied}); got.State != ProtectionStateUnknown || got.Coverage != ProtectionCoverageUnknown {
t.Fatalf("clock issue overrode unavailable access: %+v", got)
}
// An unreadable source's future date cannot invalidate a separately
// readable, recent backup or make its verification appear newer.
good := point
good.ID, good.ProviderScope, good.CompletedAt = "good-backup", "source-b", &recent
goodObservation, err := NewProtectionProviderObservation(provider, "inventory", "source-b", OutcomeSuccess, ProtectionHistoryComplete, operationaltrust.EvidencePermissionsSufficient, true, now, now, nil)
if err != nil {
t.Fatal(err)
}
got := build([]RecoveryPoint{point, good}, []ProtectionProviderObservation{denied, goodObservation})
if got.State != ProtectionStateProtected || got.Verification != ProtectionVerificationVerified || got.Freshness != ProtectionFreshnessCurrent || got.Coverage != ProtectionCoverageUnknown {
t.Fatalf("independently supported backup was invalidated: %+v", got)
}
point.CompletedAt = &recent
if got := build([]RecoveryPoint{point}, []ProtectionProviderObservation{observation}); got.State != ProtectionStateProtected || got.Freshness != ProtectionFreshnessCurrent || got.Verification != ProtectionVerificationVerified {
t.Fatalf("corrected source date did not recover: %+v", got)
}
})
}
}
// A complete inventory with no successful backup need not be empty. Existing
// running/failed/uncertain artifacts remain evidence without supplying a usable
// completed backup, a success time or a protected state.
func TestProtectionExplanationDoesNotEraseUnsuccessfulBackupEvidence(t *testing.T) {
t.Parallel()
now := time.Date(2026, 10, 31, 9, 0, 0, 0, time.UTC)
started := now.Add(-time.Hour)
for _, outcome := range []Outcome{OutcomeRunning, OutcomeFailed, OutcomeWarning, OutcomeUnknown} {
t.Run(string(outcome), func(t *testing.T) {
point := RecoveryPoint{
ID: "pbs-backup:observed", Provider: ProviderProxmoxPBS,
Kind: KindBackup, Mode: ModeLocal, Outcome: outcome,
StartedAt: &started, SubjectResourceID: "resource:vm-100", ProviderScope: "pbs-main",
}
envelope, err := NewRecoveryPointEvidence(point, "pbs-backup-inventory", now)
if err != nil {
t.Fatal(err)
}
point.Evidence = envelope
observation, err := NewProtectionProviderObservation(ProviderProxmoxPBS,
"pbs-backup-enumeration", "pbs-main", OutcomeSuccess, ProtectionHistoryComplete,
operationaltrust.EvidencePermissionsSufficient, false, now, now, nil)
if err != nil {
t.Fatal(err)
}
got := BuildProtectionPostureFromPointsAt(point.SubjectResourceID, []RecoveryPoint{point},
[]ProtectionProviderObservation{observation}, DefaultProtectionPosturePolicy, now)
if got.State != ProtectionStateUnprotected || got.Coverage != ProtectionCoverageNone || got.LastSuccessfulPointAt != nil {
t.Fatalf("non-successful point changed protection/success: %+v", got)
}
if len(got.ProviderStates) != 1 || len(got.EvidenceIDs) != 2 || len(got.ProviderStates[0].EvidenceIDs) != 2 {
t.Fatalf("observed backup evidence lost: %+v", got)
}
if !strings.Contains(got.Explanation, "no successful, completed backup") || strings.Contains(got.Explanation, "found no backup") {
t.Fatalf("retained %s artifact described as absent: %q", outcome, got.Explanation)
}
})
}
}
// A partial provider collection supports exactly the points it re-read. A
// retained point from an unread scope has no newer evidence and may have been
// pruned since, so it cannot stand behind a protected claim.
func TestBuildProtectionPosturePartialCollectionConfirmsOnlyReReadPoints(t *testing.T) {
t.Parallel()
now := time.Date(2026, 11, 1, 12, 0, 0, 0, time.UTC)
observedAt := now.Add(-time.Minute)
backupAt := now.Add(-3 * time.Hour)
verified := true
unread := []ProtectionUnreadScope{{Kind: ProtectionUnreadScopeStorage, Node: "delly", Storage: "pbs-energy-offsite"}}
for _, provider := range []Provider{ProviderProxmoxPVE, ProviderProxmoxPBS} {
for _, permissions := range []operationaltrust.EvidencePermissions{
operationaltrust.EvidencePermissionsUnknown,
operationaltrust.EvidencePermissionsPartial,
} {
t.Run(fmt.Sprintf("%s/%s", provider, permissions), func(t *testing.T) {
observation, err := NewProtectionProviderObservation(provider, "inventory", "source-a", OutcomeWarning, ProtectionHistoryPartial, permissions, true, observedAt, observedAt, nil)
if err != nil {
t.Fatal(err)
}
observation = observation.WithUnreadScopes(unread)
withEvidence := func(collector string, ingestedAt time.Time) RecoveryPoint {
point := RecoveryPoint{ID: "backup", Provider: provider, ProviderScope: "source-a", Kind: KindBackup, Mode: ModeRemote, Outcome: OutcomeSuccess, CompletedAt: &backupAt, Verified: &verified, SubjectResourceID: "resource:vm-100"}
evidence, err := NewRecoveryPointEvidence(point, collector, ingestedAt)
if err != nil {
t.Fatal(err)
}
point.Evidence = evidence
return point
}
build := func(point RecoveryPoint) ProtectionPosture {
return BuildProtectionPostureFromPointsAt("resource:vm-100", []RecoveryPoint{point}, []ProtectionProviderObservation{observation}, DefaultProtectionPosturePolicy, now)
}
got := build(withEvidence("inventory", observedAt))
if got.State != ProtectionStateProtected || got.Coverage != ProtectionCoveragePartial || got.Verification != ProtectionVerificationVerified {
t.Fatalf("re-read point = %s/%s/%s, want protected/partial/verified", got.State, got.Coverage, got.Verification)
}
if !strings.Contains(got.Explanation, "Pulse could not read pbs-energy-offsite on delly, but that does not change this backup.") {
t.Errorf("explanation = %q", got.Explanation)
}
if len(got.ProviderStates) != 1 || !reflect.DeepEqual(got.ProviderStates[0].UnreadScopes, unread) {
t.Errorf("provider state unread scopes = %+v", got.ProviderStates)
}
if err := got.Validate(); err != nil {
t.Fatal(err)
}
for name, point := range map[string]RecoveryPoint{
"retained": withEvidence("inventory", observedAt.Add(-time.Millisecond)),
"migration": withEvidence(recoveryPointMigrationCollector, now),
"legacy": func() RecoveryPoint {
p := withEvidence("inventory", observedAt)
p.Evidence = nil
return p
}(),
} {
got := build(point)
if got.State != ProtectionStateAttention || got.Coverage != ProtectionCoveragePartial {
t.Errorf("%s point = %s/%s, want attention/partial", name, got.State, got.Coverage)
}
if !strings.Contains(got.Explanation, "Pulse could not read pbs-energy-offsite on delly.") {
t.Errorf("%s explanation does not name the unread storage: %q", name, got.Explanation)
}
}
})
}
}
}
// Denied or unavailable history confirms nothing, even for a renewed point.
func TestProtectionPostureUnavailableHistoryConfirmsNothing(t *testing.T) {
t.Parallel()
now := time.Date(2026, 11, 1, 12, 0, 0, 0, time.UTC)
recent := now.Add(-time.Hour)
for _, history := range []ProtectionHistoryCompleteness{ProtectionHistoryUnavailable, ProtectionHistoryUnknown} {
summary := protectionTestSummary(now, &recent)
summary.HistoryCompleteness = history
summary.Permissions = operationaltrust.EvidencePermissionsUnknown
summary.ConfirmedLastSuccessAt = &recent
summary.ConfirmedLastVerifiedAt = &recent
if got := DeriveProtectionPostureAt("resource:vm-100", []ProtectionProviderSummary{summary}, DefaultProtectionPosturePolicy, now); got.State != ProtectionStateUnknown {
t.Errorf("%s history = %s, want unknown", history, got.State)
}
}
summary := protectionTestSummary(now, &recent)
summary.HistoryCompleteness = ProtectionHistoryPartial
summary.Permissions = operationaltrust.EvidencePermissionsDenied
summary.ConfirmedLastSuccessAt = &recent
if got := DeriveProtectionPostureAt("resource:vm-100", []ProtectionProviderSummary{summary}, DefaultProtectionPosturePolicy, now); got.State == ProtectionStateProtected {
t.Errorf("denied access claimed protection: %+v", got)
}
}
func TestDescribeUnreadScopesNamesAtMostThree(t *testing.T) {
t.Parallel()
scopes := []ProtectionUnreadScope{
{Kind: ProtectionUnreadScopeNode, Node: "pve2"},
{Kind: ProtectionUnreadScopeStorage, Storage: "nfs"},
{Kind: ProtectionUnreadScopeNamespace, Storage: "archive"},
{Kind: ProtectionUnreadScopeNamespace, Storage: "archive", Namespace: "tenant"},
{Kind: ProtectionUnreadScopeDatastore, Storage: "archive"},
{Kind: "bogus", Storage: "ignored"},
}
want := "the namespace list of datastore archive, the root namespace of datastore archive, namespace tenant on datastore archive and 2 more"
if got := describeUnreadScopes(scopes); got != want {
t.Fatalf("describeUnreadScopes() = %q\nwant %q", got, want)
}
if got := describeUnreadScopes(scopes[:2]); got != "the storage list on node pve2 and nfs" {
t.Fatalf("two scopes = %q", got)
}
}