Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Keep future backup clocks out of verified rollup claims

pulse-triage[bot]6fdfd1b2eba7
diff --git a/docs/release-control/v6/internal/subsystems/storage-recovery.md b/docs/release-control/v6/internal/subsystems/storage-recovery.md
index c88a249966..56791f981f 100644
--- a/docs/release-control/v6/internal/subsystems/storage-recovery.md
+++ b/docs/release-control/v6/internal/subsystems/storage-recovery.md
@@ -643,6 +643,23 @@ synthetic poll, async ingest, requested and state-filtered posture reads, and
clock correction. No clock mutation, backup execution or native recovery is
established by these fixtures.
+The existing recovery rollup API uses the same upper clock boundary. Its shared
+`ComputeVerifyIntentAt` classifier returns `unknown` when the success or newest
+verification date is later than evaluation, instead of a false `verified` or a
+stale-age claim. The existing seven-day inclusive verification window and
+`verified`/`stale`/`unknown` wire values are unchanged. Both persisted and in-memory
+rollups retain the original success, verification and attempt timestamps and
+subject/provider scope; a normal corrected source date recovers on the next read.
+This is a machine-API assessment correction, not a new frontend surface or an
+installed/native backup recovery claim.
+
+`recovery_test.go` covers future, absent, invalid, equal-time and window-boundary
+inputs, extreme timestamps, five provider types and in-place correction without
+rewriting evidence. `recovery_handlers_test.go` exercises real persisted writes,
+org-isolated rollup reads and the canonical/compatibility HTTP payload through
+future and corrected dates. The current frontend does not consume this rollup
+intent; Coverage's separately assessed posture and its pixels are unchanged.
+
### Canonical posture recognises recovery's Proxmox subject aliases
Protection posture selects stored points and publishes its subject ID using
diff --git a/internal/api/recovery_handlers_test.go b/internal/api/recovery_handlers_test.go
index fe8c6d66f1..baee5216e1 100644
--- a/internal/api/recovery_handlers_test.go
+++ b/internal/api/recovery_handlers_test.go
@@ -1,9 +1,11 @@
package api
import (
+ "bytes"
"context"
"database/sql"
"encoding/json"
+ "fmt"
"net/http"
"net/http/httptest"
"net/url"
@@ -703,3 +705,112 @@ func joinCSV(parts []string) string {
func timePtr(t time.Time) *time.Time {
return &t
}
+
+// Exercise the shared classifier through real persisted writes, tenant-scoped
+// rollup reads and the public wire projection. Future source dates are evidence,
+// not verified protection; ordinary date correction must recover in place.
+func TestHandleListRollupsFutureClockEvidenceAndCorrection(t *testing.T) {
+ setMockModeForTest(t, false)
+ now := time.Now().UTC().Truncate(time.Millisecond)
+ recent, future := now.Add(-time.Hour), now.Add(24*time.Hour)
+ for _, provider := range []recovery.Provider{recovery.ProviderProxmoxPVE, recovery.ProviderProxmoxPBS} {
+ for _, verified := range []bool{false, true} {
+ t.Run(fmt.Sprintf("%s/verified=%t", provider, verified), func(t *testing.T) {
+ manager := recoverymanager.New(config.NewMultiTenantPersistence(t.TempDir()))
+ store, err := manager.StoreForOrg("default")
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() { _ = store.Close() })
+ other, err := manager.StoreForOrg("clock-other")
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() { _ = other.Close() })
+ point := recovery.RecoveryPoint{ID: "clock-point", Provider: provider, ProviderScope: "clock-source", Kind: recovery.KindBackup, Mode: recovery.ModeRemote,
+ Outcome: recovery.OutcomeSuccess, CompletedAt: &future, Verified: &verified, SubjectResourceID: "clock-guest"}
+ otherPoint := point
+ otherPoint.CompletedAt = &recent
+ if err := other.UpsertPoints(context.Background(), []recovery.RecoveryPoint{otherPoint}); err != nil {
+ t.Fatal(err)
+ }
+ handler := NewRecoveryHandlers(manager)
+ read := func(org string) recoveryRollupPayload {
+ t.Helper()
+ request := httptest.NewRequest(http.MethodGet, "/api/recovery/rollups?limit=500", nil)
+ request = request.WithContext(context.WithValue(request.Context(), OrgIDContextKey, org))
+ response := httptest.NewRecorder()
+ handler.HandleListRollups(response, request)
+ if response.Code != http.StatusOK {
+ t.Fatalf("status=%d, body=%s", response.Code, response.Body.String())
+ }
+ var payload struct {
+ Data []recoveryRollupPayload `json:"data"`
+ Meta struct {
+ Total int `json:"total"`
+ } `json:"meta"`
+ }
+ if err := json.Unmarshal(response.Body.Bytes(), &payload); err != nil {
+ t.Fatal(err)
+ }
+ if len(payload.Data) != 1 || payload.Meta.Total != 1 {
+ t.Fatalf("response=%s", response.Body.String())
+ }
+ return payload.Data[0]
+ }
+ check := func(got recoveryRollupPayload, intent recovery.VerifyIntent, at time.Time) {
+ t.Helper()
+ if got.VerifyIntent != intent {
+ t.Errorf("intent=%q, want %q", got.VerifyIntent, intent)
+ }
+ if got.RollupID != "res:clock-guest" || got.ItemResourceID != "clock-guest" || got.SubjectResourceID != got.ItemResourceID || len(got.Providers) != 1 || got.Providers[0] != provider || len(got.Platforms) != 1 || got.Platforms[0] != provider {
+ t.Errorf("canonical/compatibility scope changed: %+v", got)
+ }
+ if got.LastSuccessAt == nil || !got.LastSuccessAt.Equal(at) || got.LastAttemptAt == nil || !got.LastAttemptAt.Equal(at) || got.LastOutcome != recovery.OutcomeSuccess {
+ t.Errorf("source evidence changed: %+v", got)
+ }
+ if verified {
+ if got.LastVerifiedAt == nil || !got.LastVerifiedAt.Equal(at) {
+ t.Errorf("verification date changed: %+v", got)
+ }
+ } else if got.LastVerifiedAt != nil {
+ t.Error("unobserved verification gained a timestamp")
+ }
+ }
+ if err := store.UpsertPoints(context.Background(), []recovery.RecoveryPoint{point}); err != nil {
+ t.Fatal(err)
+ }
+ before, err := json.Marshal(point)
+ if err != nil {
+ t.Fatal(err)
+ }
+ check(read("default"), recovery.VerifyIntentUnknown, future)
+ check(read("default"), recovery.VerifyIntentUnknown, future)
+ wantCorrected := recovery.VerifyIntentStale
+ if verified {
+ wantCorrected = recovery.VerifyIntentVerified
+ }
+ check(read("clock-other"), wantCorrected, recent)
+ stored, total, err := store.ListPoints(context.Background(), recovery.ListPointsOptions{Page: 1, Limit: 50})
+ if err != nil || total != 1 || len(stored) != 1 {
+ t.Fatalf("point read: %d/%d/%v", total, len(stored), err)
+ }
+ // Persistence adds a derived display projection, but the measured completion,
+ // verification, source identity and scope must survive these read-only calls.
+ if stored[0].CompletedAt == nil || !stored[0].CompletedAt.Equal(future) || stored[0].Verified == nil || *stored[0].Verified != verified || stored[0].ID != point.ID || stored[0].ProviderScope != point.ProviderScope {
+ t.Fatalf("stored evidence changed: %+v", stored[0])
+ }
+ after, err := json.Marshal(point)
+ if err != nil || !bytes.Equal(before, after) {
+ t.Fatal("read changed the source input")
+ }
+ point.CompletedAt = &recent
+ if err := store.UpsertPoints(context.Background(), []recovery.RecoveryPoint{point}); err != nil {
+ t.Fatal(err)
+ }
+ check(read("default"), wantCorrected, recent)
+ check(read("clock-other"), wantCorrected, recent)
+ })
+ }
+ }
+}
diff --git a/internal/recovery/model/types.go b/internal/recovery/model/types.go
index 27970b9611..8a1e56ce11 100644
--- a/internal/recovery/model/types.go
+++ b/internal/recovery/model/types.go
@@ -121,13 +121,13 @@ type VerifyIntent string
const (
// VerifyIntentVerified means a verification-bearing successful recovery
- // point exists within the staleness window.
+ // point exists at or before evaluation time within the staleness window.
VerifyIntentVerified VerifyIntent = "verified"
// VerifyIntentStale means a successful backup exists for the subject but
// no verification-bearing point has landed within the staleness window.
VerifyIntentStale VerifyIntent = "stale"
- // VerifyIntentUnknown means there is no information one way or the other
- // (no successful backup, or no verification-bearing points at all).
+ // VerifyIntentUnknown means there is no successful backup, or a reported
+ // success/verification date is in the future and its age cannot be assessed.
VerifyIntentUnknown VerifyIntent = "unknown"
)
diff --git a/internal/recovery/recovery_test.go b/internal/recovery/recovery_test.go
index 4c82821cc1..f8512f23ca 100644
--- a/internal/recovery/recovery_test.go
+++ b/internal/recovery/recovery_test.go
@@ -1,6 +1,8 @@
package recovery
import (
+ "math"
+ "reflect"
"strings"
"testing"
"time"
@@ -1247,3 +1249,87 @@ func TestComputeVerifyIntentAt_BoundaryEqualsWindow(t *testing.T) {
t.Fatalf("beyond boundary lastVerifiedAt = nil, want non-nil so UI can render history")
}
}
+
+func TestRollupVerificationClockBoundaries(t *testing.T) {
+ now := time.Date(2026, 10, 28, 23, 0, 0, 0, time.UTC)
+ recent := now.Add(-time.Hour).UnixMilli()
+ boundary := now.Add(-BackupVerifyStaleWindow).UnixMilli()
+ for _, tc := range []struct {
+ name string
+ success, verified int64
+ want VerifyIntent
+ }{
+ {"recent", recent, recent, VerifyIntentVerified},
+ {"at-evaluation", now.UnixMilli(), now.UnixMilli(), VerifyIntentVerified},
+ {"inclusive-window", boundary, boundary, VerifyIntentVerified},
+ {"past-window", recent, boundary - 1, VerifyIntentStale},
+ {"no-verification", recent, 0, VerifyIntentStale},
+ {"invalid-verification", recent, -1, VerifyIntentStale},
+ {"no-success", 0, recent, VerifyIntentUnknown},
+ {"invalid-success", -1, recent, VerifyIntentUnknown},
+ {"future-verification", recent, now.UnixMilli() + 1, VerifyIntentUnknown},
+ {"future-success", now.UnixMilli() + 1, recent, VerifyIntentUnknown},
+ {"future-success-unverified", now.UnixMilli() + 1, 0, VerifyIntentUnknown},
+ {"both-future", now.UnixMilli() + 1, now.UnixMilli() + 1, VerifyIntentUnknown},
+ {"extreme-future-verification", recent, math.MaxInt64, VerifyIntentUnknown},
+ {"extreme-future-success", math.MaxInt64, recent, VerifyIntentUnknown},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ intent, verified := ComputeVerifyIntentAt(tc.success, tc.verified, now)
+ if intent != tc.want {
+ t.Errorf("intent=%q, want %q", intent, tc.want)
+ }
+ if tc.verified <= 0 {
+ if verified != nil {
+ t.Fatalf("unobserved verification became %v", verified)
+ }
+ } else if verified == nil || verified.UnixMilli() != tc.verified {
+ t.Fatalf("source verification date rewritten: %v, want %d", verified, tc.verified)
+ }
+ })
+ }
+}
+
+func TestRollupFutureDateCorrectionPreservesEvidence(t *testing.T) {
+ now := time.Date(2026, 10, 28, 23, 0, 0, 0, time.UTC)
+ recent, future := now.Add(-time.Hour), now.Add(time.Hour)
+ verified := true
+ for _, provider := range []Provider{ProviderProxmoxPVE, ProviderProxmoxPBS, ProviderProxmoxPMG, ProviderTrueNAS, ProviderKubernetes} {
+ t.Run(string(provider), func(t *testing.T) {
+ point := RecoveryPoint{ID: "clock-backup", Provider: provider, Kind: KindBackup, Mode: ModeRemote, Outcome: OutcomeSuccess,
+ CompletedAt: &future, Verified: &verified, SubjectResourceID: "clock-guest", ProviderScope: "clock-estate"}
+ points := []RecoveryPoint{point}
+ before := []RecoveryPoint{point}
+ rollups := BuildRollupsFromPointsAt(points, now)
+ if len(rollups) != 1 {
+ t.Fatalf("rollups=%+v", rollups)
+ }
+ r := rollups[0]
+ if r.VerifyIntent != VerifyIntentUnknown {
+ t.Errorf("future intent=%q, want unknown", r.VerifyIntent)
+ }
+ if r.LastSuccessAt == nil || !r.LastSuccessAt.Equal(future) || r.LastVerifiedAt == nil || !r.LastVerifiedAt.Equal(future) || r.LastAttemptAt == nil || !r.LastAttemptAt.Equal(future) {
+ t.Fatalf("future evidence lost: %+v", r)
+ }
+ if r.RollupID != "res:clock-guest" || r.SubjectResourceID != point.SubjectResourceID || len(r.Providers) != 1 || r.Providers[0] != provider || !reflect.DeepEqual(points, before) {
+ t.Fatalf("scope, identity or input changed: %+v", r)
+ }
+ // Merely collecting the same point again must not invent a new date.
+ again := BuildRollupsFromPointsAt(points, now)
+ if !reflect.DeepEqual(rollups, again) {
+ t.Fatal("ordinary reread changed evidence")
+ }
+ // Only a corrected source date (or actual time reaching the reported instant)
+ // makes this same verified point eligible for a verified assessment.
+ points[0].CompletedAt = &recent
+ corrected := BuildRollupsFromPointsAt(points, now)[0]
+ if corrected.VerifyIntent != VerifyIntentVerified || corrected.LastVerifiedAt == nil || !corrected.LastVerifiedAt.Equal(recent) || corrected.RollupID != r.RollupID {
+ t.Fatalf("corrected date did not recover in place: %+v", corrected)
+ }
+ atReportedTime := BuildRollupsFromPointsAt(before, future)[0]
+ if atReportedTime.VerifyIntent != VerifyIntentVerified || !atReportedTime.LastVerifiedAt.Equal(future) {
+ t.Fatal("evaluation did not recover at reported time")
+ }
+ })
+ }
+}
diff --git a/internal/recovery/rollups.go b/internal/recovery/rollups.go
index e1ee167898..cbbec8091d 100644
--- a/internal/recovery/rollups.go
+++ b/internal/recovery/rollups.go
@@ -21,10 +21,10 @@ const BackupVerifyStaleWindow = 7 * 24 * time.Hour
// Semantics:
// - verified : a verification-bearing point exists within window of now
// - stale : a successful backup exists, but no verification within window
-// - unknown : no successful backup at all (or no info)
+// - unknown : no successful backup, or a success/verification date is in the future
//
// LastVerifiedAt is returned regardless of intent (when known) so consumers
-// can surface "last verified X days ago" alongside the badge.
+// can retain the source date even when clock uncertainty prevents an assessment.
func ComputeVerifyIntentAt(lastSuccessMs, lastVerifiedMs int64, now time.Time) (VerifyIntent, *time.Time) {
windowMs := BackupVerifyStaleWindow.Milliseconds()
nowMs := now.UTC().UnixMilli()
@@ -35,7 +35,10 @@ func ComputeVerifyIntentAt(lastSuccessMs, lastVerifiedMs int64, now time.Time) (
lastVerifiedAt = &t
}
- if lastSuccessMs <= 0 {
+ // A future date proves no recency. Keep the reported date as evidence,
+ // but align the rollup with posture: clock uncertainty is neither verified
+ // nor stale. Check before subtraction so extreme dates cannot overflow age.
+ if lastSuccessMs <= 0 || lastSuccessMs > nowMs || lastVerifiedMs > nowMs {
return VerifyIntentUnknown, lastVerifiedAt
}