Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Keep future backup dates out of current protection

pulse-triage[bot]d7f138cea6f8
diff --git a/docs/release-control/v6/internal/subsystems/storage-recovery.md b/docs/release-control/v6/internal/subsystems/storage-recovery.md
index f217582b91..42057b19ba 100644
--- a/docs/release-control/v6/internal/subsystems/storage-recovery.md
+++ b/docs/release-control/v6/internal/subsystems/storage-recovery.md
@@ -21,6 +21,24 @@
## Purpose
+### Future backup dates cannot establish current protection
+
+Protection freshness and verification must fall at or before their evaluation
+time as well as within the existing policy windows. A future-dated backup has
+Unknown freshness and needs Attention; required future-dated verification is
+Unknown and needs Attention rather than a verified claim. Explanations name
+the clock uncertainty. Optional verification cannot claim Verified from a
+future date, but does not invalidate an otherwise current backup.
+
+Reported dates, artifacts and collection/access evidence are retained unchanged;
+clock uncertainty neither proves absence nor overrides denied access. Normal
+dates recover through the existing read and poll paths. The registered
+`posture_test.go` clock controls cover both policy boundaries and PVE/PBS
+source/access separation. `pve_backup_posture_clock_test.go` drives the real
+synthetic poll, async ingest, requested and state-filtered posture reads, and
+clock correction. No clock mutation, backup execution or native recovery is
+established by these fixtures.
+
### Canonical posture recognises recovery's Proxmox subject aliases
Protection posture selects stored points and publishes its subject ID using
diff --git a/internal/monitoring/pve_backup_posture_clock_test.go b/internal/monitoring/pve_backup_posture_clock_test.go
new file mode 100644
index 0000000000..f5bdd19033
--- /dev/null
+++ b/internal/monitoring/pve_backup_posture_clock_test.go
@@ -0,0 +1,65 @@
+package monitoring
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/rcourtman/pulse-go-rewrite/internal/recovery"
+ "github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"
+)
+
+type postureClockClient struct {
+ PVEClientInterface
+ times map[string]time.Time
+}
+
+func (c *postureClockClient) GetStorage(context.Context, string) ([]proxmox.Storage, error) {
+ return []proxmox.Storage{{Storage: "local", Type: "dir", Content: "backup", Active: 1, Enabled: 1}}, nil
+}
+
+func (c *postureClockClient) GetStorageContent(_ context.Context, node, _ string) ([]proxmox.StorageContent, error) {
+ id := 101
+ if node == "node-b" {
+ id = 102
+ }
+ return []proxmox.StorageContent{{Volid: fmt.Sprintf("local:backup/vzdump-qemu-%d.vma.zst", id), VMID: id, CTime: c.times[node].Unix(), Size: 2048, Content: "backup", Format: "vma.zst"}}, nil
+}
+
+func TestPVEBackupPostureFutureInventory(t *testing.T) {
+ f := newPVEReconciliationFixture(t)
+ now := time.Now().UTC().Truncate(time.Second)
+ future, recent := now.Add(48*time.Hour), now.Add(-time.Hour)
+ client := &postureClockClient{times: map[string]time.Time{"node-a": future, "node-b": recent}}
+ poll := func() {
+ f.m.pollStorageBackupsWithNodes(context.Background(), "cluster", client, f.nodes, f.online)
+ f.wait(t)
+ }
+ poll()
+ for _, posture := range f.postures(t) {
+ if posture.SubjectResourceID == f.ids[0] {
+ if posture.State != recovery.ProtectionStateAttention || posture.Freshness != recovery.ProtectionFreshnessUnknown || !posture.LastSuccessfulPointAt.Equal(future) {
+ t.Errorf("future provider date claimed protection or lost its evidence: %+v", posture)
+ }
+ } else if posture.State != recovery.ProtectionStateProtected {
+ t.Errorf("independent current backup lost protection: %+v", posture)
+ }
+ }
+ for _, state := range []recovery.ProtectionState{recovery.ProtectionStateAttention, recovery.ProtectionStateProtected} {
+ postures, total, err := f.store.ListProtectionPostures(context.Background(), recovery.ProtectionPostureQuery{State: state})
+ if err != nil || total != 1 || len(postures) != 1 || (state == recovery.ProtectionStateAttention && postures[0].SubjectResourceID != f.ids[0]) {
+ t.Errorf("state-filtered %s read: total=%d len=%d err=%v", state, total, len(postures), err)
+ }
+ }
+ if len(f.points(t)) != 2 || len(f.m.state.GetSnapshot().PVEBackups.StorageBackups) != 2 {
+ t.Fatal("uncertain clock discarded provider artifacts")
+ }
+ client.times["node-a"] = recent
+ poll()
+ for _, posture := range f.postures(t) {
+ if posture.State != recovery.ProtectionStateProtected || !posture.LastSuccessfulPointAt.Equal(recent) {
+ t.Errorf("normal provider clock did not recover: %+v", posture)
+ }
+ }
+}
diff --git a/internal/recovery/posture.go b/internal/recovery/posture.go
index 6b2174b324..34f3596ab6 100644
--- a/internal/recovery/posture.go
+++ b/internal/recovery/posture.go
@@ -522,15 +522,20 @@ func DeriveProtectionPostureAt(
stateLastVerifiedAt = supportedLastVerifiedAt
stateVerificationExpected = supportedVerificationExpected
}
- if stateLastSuccessAt != nil {
+ // A negative age is clock uncertainty, not a recent backup. Retain the
+ // provider's timestamps as evidence, but never turn them into a current
+ // or verified assessment until they are within the evaluation's past.
+ if stateLastSuccessAt != nil && !stateLastSuccessAt.After(now) {
if now.Sub(*stateLastSuccessAt) <= policy.FreshnessWindow {
posture.Freshness = ProtectionFreshnessCurrent
} else {
posture.Freshness = ProtectionFreshnessStale
}
}
- if stateLastSuccessAt != nil {
+ if stateLastSuccessAt != nil && !stateLastSuccessAt.After(now) {
switch {
+ case stateLastVerifiedAt != nil && stateLastVerifiedAt.After(now):
+ // Keep verification unknown; a future date proves no recency.
case stateLastVerifiedAt != nil &&
now.Sub(*stateLastVerifiedAt) <= policy.VerificationWindow:
posture.Verification = ProtectionVerificationVerified
@@ -576,8 +581,12 @@ func DeriveProtectionPostureAt(
switch {
case hasInvalidatingFailure:
posture.Explanation = "A backup exists, but one of its backup sources has reported a failure."
+ case stateLastSuccessAt != nil && stateLastSuccessAt.After(now):
+ posture.Explanation = "The newest backup is dated in the future, so Pulse cannot confirm its age. Check the backup source and Pulse server clocks."
case posture.Freshness == ProtectionFreshnessStale:
posture.Explanation = fmt.Sprintf("The newest confirmed backup is older than %s.", formatPostureWindow(policy.FreshnessWindow))
+ case stateVerificationExpected && stateLastVerifiedAt != nil && stateLastVerifiedAt.After(now):
+ posture.Explanation = "The backup verification is dated in the future, so Pulse cannot confirm it. Check the backup source and Pulse server clocks."
case stateVerificationExpected && posture.Verification != ProtectionVerificationVerified:
posture.Explanation = "A recent backup exists, but its verification is missing or overdue."
default:
diff --git a/internal/recovery/posture_test.go b/internal/recovery/posture_test.go
index d8ff825511..1f33181638 100644
--- a/internal/recovery/posture_test.go
+++ b/internal/recovery/posture_test.go
@@ -2,6 +2,7 @@ package recovery
import (
"fmt"
+ "reflect"
"strings"
"testing"
"time"
@@ -552,3 +553,105 @@ func TestBuildProtectionPostureUsesCanonicalPointIdentity(t *testing.T) {
}
}
}
+
+func TestProtectionPostureClockBounds(t *testing.T) {
+ t.Parallel()
+ now := time.Date(2026, 10, 25, 12, 0, 0, 0, time.FixedZone("source", 3600))
+ recent := now.Add(-time.Hour)
+ future := now.Add(48 * time.Hour)
+ atBoundary := now.Add(-DefaultProtectionFreshnessWindow)
+ tooOld := atBoundary.Add(-time.Nanosecond)
+ justAhead := now.Add(time.Nanosecond)
+ for _, tc := range []struct {
+ name string
+ success time.Time
+ verified time.Time
+ require bool
+ wantState ProtectionState
+ wantFresh ProtectionFreshness
+ wantVerified ProtectionVerification
+ wantText string
+ }{
+ {"future-backup", future, future, true, ProtectionStateAttention, ProtectionFreshnessUnknown, ProtectionVerificationUnknown, "newest backup is dated in the future"},
+ {"future-backup-without-verification", future, time.Time{}, false, ProtectionStateAttention, ProtectionFreshnessUnknown, ProtectionVerificationUnknown, "newest backup is dated in the future"},
+ {"one-nanosecond-ahead", justAhead, justAhead, true, ProtectionStateAttention, ProtectionFreshnessUnknown, ProtectionVerificationUnknown, "newest backup is dated in the future"},
+ {"future-verification", recent, future, true, ProtectionStateAttention, ProtectionFreshnessCurrent, ProtectionVerificationUnknown, "verification is dated in the future"},
+ {"optional-future-verification", recent, future, false, ProtectionStateProtected, ProtectionFreshnessCurrent, ProtectionVerificationUnknown, "A recent backup is available"},
+ {"exactly-now", now, now, true, ProtectionStateProtected, ProtectionFreshnessCurrent, ProtectionVerificationVerified, "verified recently"},
+ {"exact-window-boundary", atBoundary, atBoundary, true, ProtectionStateProtected, ProtectionFreshnessCurrent, ProtectionVerificationVerified, "verified recently"},
+ {"expired-window", tooOld, tooOld, true, ProtectionStateAttention, ProtectionFreshnessStale, ProtectionVerificationStale, "older than 7 days"},
+ {"missing-verification", recent, time.Time{}, true, ProtectionStateAttention, ProtectionFreshnessCurrent, ProtectionVerificationUnverified, "missing or overdue"},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ summary := protectionTestSummary(now, &tc.success)
+ summary.VerificationExpected = tc.require
+ summary.LastVerifiedAt = nil
+ if !tc.verified.IsZero() {
+ summary.LastVerifiedAt = &tc.verified
+ }
+ before := summary
+ posture := DeriveProtectionPostureAt("resource:vm-100", []ProtectionProviderSummary{summary}, DefaultProtectionPosturePolicy, now)
+ if posture.State != tc.wantState || posture.Freshness != tc.wantFresh || posture.Verification != tc.wantVerified {
+ t.Fatalf("state=%s freshness=%s verification=%s; want %s/%s/%s", posture.State, posture.Freshness, posture.Verification, tc.wantState, tc.wantFresh, tc.wantVerified)
+ }
+ if !strings.Contains(posture.Explanation, tc.wantText) {
+ t.Errorf("explanation=%q, want %q", posture.Explanation, tc.wantText)
+ }
+ if posture.Coverage != ProtectionCoverageComplete || !posture.LastSuccessfulPointAt.Equal(tc.success) || len(posture.ProviderStates) != 1 || !posture.ProviderStates[0].LastSuccessAt.Equal(tc.success) || !reflect.DeepEqual(summary, before) {
+ t.Error("clock assessment changed source evidence or collection coverage")
+ }
+ if tc.verified.IsZero() != (posture.LastVerifiedAt == nil) || (posture.LastVerifiedAt != nil && !posture.LastVerifiedAt.Equal(tc.verified)) {
+ t.Error("clock assessment rewrote the reported verification date")
+ }
+ if err := posture.Validate(); err != nil {
+ t.Fatal(err)
+ }
+ })
+ }
+}
+
+func TestProtectionPostureFuturePointsRetainAccessBoundaries(t *testing.T) {
+ t.Parallel()
+ now := time.Date(2026, 10, 25, 11, 0, 0, 0, time.UTC)
+ recent, future := now.Add(-time.Hour), now.Add(48*time.Hour)
+ verified := true
+ for _, provider := range []Provider{ProviderProxmoxPVE, ProviderProxmoxPBS} {
+ t.Run(string(provider), func(t *testing.T) {
+ point := RecoveryPoint{ID: "backup", Provider: provider, ProviderScope: "source-a", Kind: KindBackup, Mode: ModeRemote, Outcome: OutcomeSuccess, CompletedAt: &future, Verified: &verified, SubjectResourceID: "resource:vm-100"}
+ observation, err := NewProtectionProviderObservation(provider, "inventory", "source-a", OutcomeSuccess, ProtectionHistoryComplete, operationaltrust.EvidencePermissionsSufficient, true, now, now, nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ build := func(points []RecoveryPoint, observations []ProtectionProviderObservation) ProtectionPosture {
+ return BuildProtectionPostureFromPointsAt(point.SubjectResourceID, points, observations, DefaultProtectionPosturePolicy, now)
+ }
+ posture := build([]RecoveryPoint{point}, []ProtectionProviderObservation{observation})
+ if posture.State != ProtectionStateAttention || posture.Freshness != ProtectionFreshnessUnknown || posture.Verification != ProtectionVerificationUnknown || !posture.LastSuccessfulPointAt.Equal(future) {
+ t.Fatalf("future point claimed current protection or lost evidence: %+v", posture)
+ }
+ denied, err := NewProtectionProviderObservation(provider, "inventory", "source-a", OutcomeFailed, ProtectionHistoryUnavailable, operationaltrust.EvidencePermissionsDenied, true, now, now, nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if got := build([]RecoveryPoint{point}, []ProtectionProviderObservation{denied}); got.State != ProtectionStateUnknown || got.Coverage != ProtectionCoverageUnknown {
+ t.Fatalf("clock issue overrode unavailable access: %+v", got)
+ }
+ // An unreadable source's future date cannot invalidate a separately
+ // readable, recent backup or make its verification appear newer.
+ good := point
+ good.ID, good.ProviderScope, good.CompletedAt = "good-backup", "source-b", &recent
+ goodObservation, err := NewProtectionProviderObservation(provider, "inventory", "source-b", OutcomeSuccess, ProtectionHistoryComplete, operationaltrust.EvidencePermissionsSufficient, true, now, now, nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ got := build([]RecoveryPoint{point, good}, []ProtectionProviderObservation{denied, goodObservation})
+ if got.State != ProtectionStateProtected || got.Verification != ProtectionVerificationVerified || got.Freshness != ProtectionFreshnessCurrent || got.Coverage != ProtectionCoverageUnknown {
+ t.Fatalf("independently supported backup was invalidated: %+v", got)
+ }
+ point.CompletedAt = &recent
+ if got := build([]RecoveryPoint{point}, []ProtectionProviderObservation{observation}); got.State != ProtectionStateProtected || got.Freshness != ProtectionFreshnessCurrent || got.Verification != ProtectionVerificationVerified {
+ t.Fatalf("corrected source date did not recover: %+v", got)
+ }
+ })
+ }
+}