Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Keep future backup dates out of current protection
pulse-triage[bot]d7f138cea6f8
diff --git a/docs/release-control/v6/internal/subsystems/storage-recovery.md b/docs/release-control/v6/internal/subsystems/storage-recovery.mdindex f217582b91..42057b19ba 100644--- a/docs/release-control/v6/internal/subsystems/storage-recovery.md+++ b/docs/release-control/v6/internal/subsystems/storage-recovery.md@@ -21,6 +21,24 @@## Purpose+### Future backup dates cannot establish current protection++Protection freshness and verification must fall at or before their evaluation+time as well as within the existing policy windows. A future-dated backup has+Unknown freshness and needs Attention; required future-dated verification is+Unknown and needs Attention rather than a verified claim. Explanations name+the clock uncertainty. Optional verification cannot claim Verified from a+future date, but does not invalidate an otherwise current backup.++Reported dates, artifacts and collection/access evidence are retained unchanged;+clock uncertainty neither proves absence nor overrides denied access. Normal+dates recover through the existing read and poll paths. The registered+`posture_test.go` clock controls cover both policy boundaries and PVE/PBS+source/access separation. `pve_backup_posture_clock_test.go` drives the real+synthetic poll, async ingest, requested and state-filtered posture reads, and+clock correction. No clock mutation, backup execution or native recovery is+established by these fixtures.+### Canonical posture recognises recovery's Proxmox subject aliasesProtection posture selects stored points and publishes its subject ID usingdiff --git a/internal/monitoring/pve_backup_posture_clock_test.go b/internal/monitoring/pve_backup_posture_clock_test.gonew file mode 100644index 0000000000..f5bdd19033--- /dev/null+++ b/internal/monitoring/pve_backup_posture_clock_test.go@@ -0,0 +1,65 @@+package monitoring++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/rcourtman/pulse-go-rewrite/internal/recovery"+ "github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"+)++type postureClockClient struct {+ PVEClientInterface+ times map[string]time.Time+}++func (c *postureClockClient) GetStorage(context.Context, string) ([]proxmox.Storage, error) {+ return []proxmox.Storage{{Storage: "local", Type: "dir", Content: "backup", Active: 1, Enabled: 1}}, nil+}++func (c *postureClockClient) GetStorageContent(_ context.Context, node, _ string) ([]proxmox.StorageContent, error) {+ id := 101+ if node == "node-b" {+ id = 102+ }+ return []proxmox.StorageContent{{Volid: fmt.Sprintf("local:backup/vzdump-qemu-%d.vma.zst", id), VMID: id, CTime: c.times[node].Unix(), Size: 2048, Content: "backup", Format: "vma.zst"}}, nil+}++func TestPVEBackupPostureFutureInventory(t *testing.T) {+ f := newPVEReconciliationFixture(t)+ now := time.Now().UTC().Truncate(time.Second)+ future, recent := now.Add(48*time.Hour), now.Add(-time.Hour)+ client := &postureClockClient{times: map[string]time.Time{"node-a": future, "node-b": recent}}+ poll := func() {+ f.m.pollStorageBackupsWithNodes(context.Background(), "cluster", client, f.nodes, f.online)+ f.wait(t)+ }+ poll()+ for _, posture := range f.postures(t) {+ if posture.SubjectResourceID == f.ids[0] {+ if posture.State != recovery.ProtectionStateAttention || posture.Freshness != recovery.ProtectionFreshnessUnknown || !posture.LastSuccessfulPointAt.Equal(future) {+ t.Errorf("future provider date claimed protection or lost its evidence: %+v", posture)+ }+ } else if posture.State != recovery.ProtectionStateProtected {+ t.Errorf("independent current backup lost protection: %+v", posture)+ }+ }+ for _, state := range []recovery.ProtectionState{recovery.ProtectionStateAttention, recovery.ProtectionStateProtected} {+ postures, total, err := f.store.ListProtectionPostures(context.Background(), recovery.ProtectionPostureQuery{State: state})+ if err != nil || total != 1 || len(postures) != 1 || (state == recovery.ProtectionStateAttention && postures[0].SubjectResourceID != f.ids[0]) {+ t.Errorf("state-filtered %s read: total=%d len=%d err=%v", state, total, len(postures), err)+ }+ }+ if len(f.points(t)) != 2 || len(f.m.state.GetSnapshot().PVEBackups.StorageBackups) != 2 {+ t.Fatal("uncertain clock discarded provider artifacts")+ }+ client.times["node-a"] = recent+ poll()+ for _, posture := range f.postures(t) {+ if posture.State != recovery.ProtectionStateProtected || !posture.LastSuccessfulPointAt.Equal(recent) {+ t.Errorf("normal provider clock did not recover: %+v", posture)+ }+ }+}diff --git a/internal/recovery/posture.go b/internal/recovery/posture.goindex 6b2174b324..34f3596ab6 100644--- a/internal/recovery/posture.go+++ b/internal/recovery/posture.go@@ -522,15 +522,20 @@ func DeriveProtectionPostureAt(stateLastVerifiedAt = supportedLastVerifiedAtstateVerificationExpected = supportedVerificationExpected}- if stateLastSuccessAt != nil {+ // A negative age is clock uncertainty, not a recent backup. Retain the+ // provider's timestamps as evidence, but never turn them into a current+ // or verified assessment until they are within the evaluation's past.+ if stateLastSuccessAt != nil && !stateLastSuccessAt.After(now) {if now.Sub(*stateLastSuccessAt) <= policy.FreshnessWindow {posture.Freshness = ProtectionFreshnessCurrent} else {posture.Freshness = ProtectionFreshnessStale}}- if stateLastSuccessAt != nil {+ if stateLastSuccessAt != nil && !stateLastSuccessAt.After(now) {switch {+ case stateLastVerifiedAt != nil && stateLastVerifiedAt.After(now):+ // Keep verification unknown; a future date proves no recency.case stateLastVerifiedAt != nil &&now.Sub(*stateLastVerifiedAt) <= policy.VerificationWindow:posture.Verification = ProtectionVerificationVerified@@ -576,8 +581,12 @@ func DeriveProtectionPostureAt(switch {case hasInvalidatingFailure:posture.Explanation = "A backup exists, but one of its backup sources has reported a failure."+ case stateLastSuccessAt != nil && stateLastSuccessAt.After(now):+ posture.Explanation = "The newest backup is dated in the future, so Pulse cannot confirm its age. Check the backup source and Pulse server clocks."case posture.Freshness == ProtectionFreshnessStale:posture.Explanation = fmt.Sprintf("The newest confirmed backup is older than %s.", formatPostureWindow(policy.FreshnessWindow))+ case stateVerificationExpected && stateLastVerifiedAt != nil && stateLastVerifiedAt.After(now):+ posture.Explanation = "The backup verification is dated in the future, so Pulse cannot confirm it. Check the backup source and Pulse server clocks."case stateVerificationExpected && posture.Verification != ProtectionVerificationVerified:posture.Explanation = "A recent backup exists, but its verification is missing or overdue."default:diff --git a/internal/recovery/posture_test.go b/internal/recovery/posture_test.goindex d8ff825511..1f33181638 100644--- a/internal/recovery/posture_test.go+++ b/internal/recovery/posture_test.go@@ -2,6 +2,7 @@ package recoveryimport ("fmt"+ "reflect""strings""testing""time"@@ -552,3 +553,105 @@ func TestBuildProtectionPostureUsesCanonicalPointIdentity(t *testing.T) {}}}++func TestProtectionPostureClockBounds(t *testing.T) {+ t.Parallel()+ now := time.Date(2026, 10, 25, 12, 0, 0, 0, time.FixedZone("source", 3600))+ recent := now.Add(-time.Hour)+ future := now.Add(48 * time.Hour)+ atBoundary := now.Add(-DefaultProtectionFreshnessWindow)+ tooOld := atBoundary.Add(-time.Nanosecond)+ justAhead := now.Add(time.Nanosecond)+ for _, tc := range []struct {+ name string+ success time.Time+ verified time.Time+ require bool+ wantState ProtectionState+ wantFresh ProtectionFreshness+ wantVerified ProtectionVerification+ wantText string+ }{+ {"future-backup", future, future, true, ProtectionStateAttention, ProtectionFreshnessUnknown, ProtectionVerificationUnknown, "newest backup is dated in the future"},+ {"future-backup-without-verification", future, time.Time{}, false, ProtectionStateAttention, ProtectionFreshnessUnknown, ProtectionVerificationUnknown, "newest backup is dated in the future"},+ {"one-nanosecond-ahead", justAhead, justAhead, true, ProtectionStateAttention, ProtectionFreshnessUnknown, ProtectionVerificationUnknown, "newest backup is dated in the future"},+ {"future-verification", recent, future, true, ProtectionStateAttention, ProtectionFreshnessCurrent, ProtectionVerificationUnknown, "verification is dated in the future"},+ {"optional-future-verification", recent, future, false, ProtectionStateProtected, ProtectionFreshnessCurrent, ProtectionVerificationUnknown, "A recent backup is available"},+ {"exactly-now", now, now, true, ProtectionStateProtected, ProtectionFreshnessCurrent, ProtectionVerificationVerified, "verified recently"},+ {"exact-window-boundary", atBoundary, atBoundary, true, ProtectionStateProtected, ProtectionFreshnessCurrent, ProtectionVerificationVerified, "verified recently"},+ {"expired-window", tooOld, tooOld, true, ProtectionStateAttention, ProtectionFreshnessStale, ProtectionVerificationStale, "older than 7 days"},+ {"missing-verification", recent, time.Time{}, true, ProtectionStateAttention, ProtectionFreshnessCurrent, ProtectionVerificationUnverified, "missing or overdue"},+ } {+ t.Run(tc.name, func(t *testing.T) {+ summary := protectionTestSummary(now, &tc.success)+ summary.VerificationExpected = tc.require+ summary.LastVerifiedAt = nil+ if !tc.verified.IsZero() {+ summary.LastVerifiedAt = &tc.verified+ }+ before := summary+ posture := DeriveProtectionPostureAt("resource:vm-100", []ProtectionProviderSummary{summary}, DefaultProtectionPosturePolicy, now)+ if posture.State != tc.wantState || posture.Freshness != tc.wantFresh || posture.Verification != tc.wantVerified {+ t.Fatalf("state=%s freshness=%s verification=%s; want %s/%s/%s", posture.State, posture.Freshness, posture.Verification, tc.wantState, tc.wantFresh, tc.wantVerified)+ }+ if !strings.Contains(posture.Explanation, tc.wantText) {+ t.Errorf("explanation=%q, want %q", posture.Explanation, tc.wantText)+ }+ if posture.Coverage != ProtectionCoverageComplete || !posture.LastSuccessfulPointAt.Equal(tc.success) || len(posture.ProviderStates) != 1 || !posture.ProviderStates[0].LastSuccessAt.Equal(tc.success) || !reflect.DeepEqual(summary, before) {+ t.Error("clock assessment changed source evidence or collection coverage")+ }+ if tc.verified.IsZero() != (posture.LastVerifiedAt == nil) || (posture.LastVerifiedAt != nil && !posture.LastVerifiedAt.Equal(tc.verified)) {+ t.Error("clock assessment rewrote the reported verification date")+ }+ if err := posture.Validate(); err != nil {+ t.Fatal(err)+ }+ })+ }+}++func TestProtectionPostureFuturePointsRetainAccessBoundaries(t *testing.T) {+ t.Parallel()+ now := time.Date(2026, 10, 25, 11, 0, 0, 0, time.UTC)+ recent, future := now.Add(-time.Hour), now.Add(48*time.Hour)+ verified := true+ for _, provider := range []Provider{ProviderProxmoxPVE, ProviderProxmoxPBS} {+ t.Run(string(provider), func(t *testing.T) {+ point := RecoveryPoint{ID: "backup", Provider: provider, ProviderScope: "source-a", Kind: KindBackup, Mode: ModeRemote, Outcome: OutcomeSuccess, CompletedAt: &future, Verified: &verified, SubjectResourceID: "resource:vm-100"}+ observation, err := NewProtectionProviderObservation(provider, "inventory", "source-a", OutcomeSuccess, ProtectionHistoryComplete, operationaltrust.EvidencePermissionsSufficient, true, now, now, nil)+ if err != nil {+ t.Fatal(err)+ }+ build := func(points []RecoveryPoint, observations []ProtectionProviderObservation) ProtectionPosture {+ return BuildProtectionPostureFromPointsAt(point.SubjectResourceID, points, observations, DefaultProtectionPosturePolicy, now)+ }+ posture := build([]RecoveryPoint{point}, []ProtectionProviderObservation{observation})+ if posture.State != ProtectionStateAttention || posture.Freshness != ProtectionFreshnessUnknown || posture.Verification != ProtectionVerificationUnknown || !posture.LastSuccessfulPointAt.Equal(future) {+ t.Fatalf("future point claimed current protection or lost evidence: %+v", posture)+ }+ denied, err := NewProtectionProviderObservation(provider, "inventory", "source-a", OutcomeFailed, ProtectionHistoryUnavailable, operationaltrust.EvidencePermissionsDenied, true, now, now, nil)+ if err != nil {+ t.Fatal(err)+ }+ if got := build([]RecoveryPoint{point}, []ProtectionProviderObservation{denied}); got.State != ProtectionStateUnknown || got.Coverage != ProtectionCoverageUnknown {+ t.Fatalf("clock issue overrode unavailable access: %+v", got)+ }+ // An unreadable source's future date cannot invalidate a separately+ // readable, recent backup or make its verification appear newer.+ good := point+ good.ID, good.ProviderScope, good.CompletedAt = "good-backup", "source-b", &recent+ goodObservation, err := NewProtectionProviderObservation(provider, "inventory", "source-b", OutcomeSuccess, ProtectionHistoryComplete, operationaltrust.EvidencePermissionsSufficient, true, now, now, nil)+ if err != nil {+ t.Fatal(err)+ }+ got := build([]RecoveryPoint{point, good}, []ProtectionProviderObservation{denied, goodObservation})+ if got.State != ProtectionStateProtected || got.Verification != ProtectionVerificationVerified || got.Freshness != ProtectionFreshnessCurrent || got.Coverage != ProtectionCoverageUnknown {+ t.Fatalf("independently supported backup was invalidated: %+v", got)+ }+ point.CompletedAt = &recent+ if got := build([]RecoveryPoint{point}, []ProtectionProviderObservation{observation}); got.State != ProtectionStateProtected || got.Freshness != ProtectionFreshnessCurrent || got.Verification != ProtectionVerificationVerified {+ t.Fatalf("corrected source date did not recover: %+v", got)+ }+ })+ }+}