Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Keep future backup clocks out of verified rollup claims
pulse-triage[bot]6fdfd1b2eba7
diff --git a/docs/release-control/v6/internal/subsystems/storage-recovery.md b/docs/release-control/v6/internal/subsystems/storage-recovery.mdindex c88a249966..56791f981f 100644--- a/docs/release-control/v6/internal/subsystems/storage-recovery.md+++ b/docs/release-control/v6/internal/subsystems/storage-recovery.md@@ -643,6 +643,23 @@ synthetic poll, async ingest, requested and state-filtered posture reads, andclock correction. No clock mutation, backup execution or native recovery isestablished by these fixtures.+The existing recovery rollup API uses the same upper clock boundary. Its shared+`ComputeVerifyIntentAt` classifier returns `unknown` when the success or newest+verification date is later than evaluation, instead of a false `verified` or a+stale-age claim. The existing seven-day inclusive verification window and+`verified`/`stale`/`unknown` wire values are unchanged. Both persisted and in-memory+rollups retain the original success, verification and attempt timestamps and+subject/provider scope; a normal corrected source date recovers on the next read.+This is a machine-API assessment correction, not a new frontend surface or an+installed/native backup recovery claim.++`recovery_test.go` covers future, absent, invalid, equal-time and window-boundary+inputs, extreme timestamps, five provider types and in-place correction without+rewriting evidence. `recovery_handlers_test.go` exercises real persisted writes,+org-isolated rollup reads and the canonical/compatibility HTTP payload through+future and corrected dates. The current frontend does not consume this rollup+intent; Coverage's separately assessed posture and its pixels are unchanged.+### Canonical posture recognises recovery's Proxmox subject aliasesProtection posture selects stored points and publishes its subject ID usingdiff --git a/internal/api/recovery_handlers_test.go b/internal/api/recovery_handlers_test.goindex fe8c6d66f1..baee5216e1 100644--- a/internal/api/recovery_handlers_test.go+++ b/internal/api/recovery_handlers_test.go@@ -1,9 +1,11 @@package apiimport (+ "bytes""context""database/sql""encoding/json"+ "fmt""net/http""net/http/httptest""net/url"@@ -703,3 +705,112 @@ func joinCSV(parts []string) string {func timePtr(t time.Time) *time.Time {return &t}++// Exercise the shared classifier through real persisted writes, tenant-scoped+// rollup reads and the public wire projection. Future source dates are evidence,+// not verified protection; ordinary date correction must recover in place.+func TestHandleListRollupsFutureClockEvidenceAndCorrection(t *testing.T) {+ setMockModeForTest(t, false)+ now := time.Now().UTC().Truncate(time.Millisecond)+ recent, future := now.Add(-time.Hour), now.Add(24*time.Hour)+ for _, provider := range []recovery.Provider{recovery.ProviderProxmoxPVE, recovery.ProviderProxmoxPBS} {+ for _, verified := range []bool{false, true} {+ t.Run(fmt.Sprintf("%s/verified=%t", provider, verified), func(t *testing.T) {+ manager := recoverymanager.New(config.NewMultiTenantPersistence(t.TempDir()))+ store, err := manager.StoreForOrg("default")+ if err != nil {+ t.Fatal(err)+ }+ t.Cleanup(func() { _ = store.Close() })+ other, err := manager.StoreForOrg("clock-other")+ if err != nil {+ t.Fatal(err)+ }+ t.Cleanup(func() { _ = other.Close() })+ point := recovery.RecoveryPoint{ID: "clock-point", Provider: provider, ProviderScope: "clock-source", Kind: recovery.KindBackup, Mode: recovery.ModeRemote,+ Outcome: recovery.OutcomeSuccess, CompletedAt: &future, Verified: &verified, SubjectResourceID: "clock-guest"}+ otherPoint := point+ otherPoint.CompletedAt = &recent+ if err := other.UpsertPoints(context.Background(), []recovery.RecoveryPoint{otherPoint}); err != nil {+ t.Fatal(err)+ }+ handler := NewRecoveryHandlers(manager)+ read := func(org string) recoveryRollupPayload {+ t.Helper()+ request := httptest.NewRequest(http.MethodGet, "/api/recovery/rollups?limit=500", nil)+ request = request.WithContext(context.WithValue(request.Context(), OrgIDContextKey, org))+ response := httptest.NewRecorder()+ handler.HandleListRollups(response, request)+ if response.Code != http.StatusOK {+ t.Fatalf("status=%d, body=%s", response.Code, response.Body.String())+ }+ var payload struct {+ Data []recoveryRollupPayload `json:"data"`+ Meta struct {+ Total int `json:"total"`+ } `json:"meta"`+ }+ if err := json.Unmarshal(response.Body.Bytes(), &payload); err != nil {+ t.Fatal(err)+ }+ if len(payload.Data) != 1 || payload.Meta.Total != 1 {+ t.Fatalf("response=%s", response.Body.String())+ }+ return payload.Data[0]+ }+ check := func(got recoveryRollupPayload, intent recovery.VerifyIntent, at time.Time) {+ t.Helper()+ if got.VerifyIntent != intent {+ t.Errorf("intent=%q, want %q", got.VerifyIntent, intent)+ }+ if got.RollupID != "res:clock-guest" || got.ItemResourceID != "clock-guest" || got.SubjectResourceID != got.ItemResourceID || len(got.Providers) != 1 || got.Providers[0] != provider || len(got.Platforms) != 1 || got.Platforms[0] != provider {+ t.Errorf("canonical/compatibility scope changed: %+v", got)+ }+ if got.LastSuccessAt == nil || !got.LastSuccessAt.Equal(at) || got.LastAttemptAt == nil || !got.LastAttemptAt.Equal(at) || got.LastOutcome != recovery.OutcomeSuccess {+ t.Errorf("source evidence changed: %+v", got)+ }+ if verified {+ if got.LastVerifiedAt == nil || !got.LastVerifiedAt.Equal(at) {+ t.Errorf("verification date changed: %+v", got)+ }+ } else if got.LastVerifiedAt != nil {+ t.Error("unobserved verification gained a timestamp")+ }+ }+ if err := store.UpsertPoints(context.Background(), []recovery.RecoveryPoint{point}); err != nil {+ t.Fatal(err)+ }+ before, err := json.Marshal(point)+ if err != nil {+ t.Fatal(err)+ }+ check(read("default"), recovery.VerifyIntentUnknown, future)+ check(read("default"), recovery.VerifyIntentUnknown, future)+ wantCorrected := recovery.VerifyIntentStale+ if verified {+ wantCorrected = recovery.VerifyIntentVerified+ }+ check(read("clock-other"), wantCorrected, recent)+ stored, total, err := store.ListPoints(context.Background(), recovery.ListPointsOptions{Page: 1, Limit: 50})+ if err != nil || total != 1 || len(stored) != 1 {+ t.Fatalf("point read: %d/%d/%v", total, len(stored), err)+ }+ // Persistence adds a derived display projection, but the measured completion,+ // verification, source identity and scope must survive these read-only calls.+ if stored[0].CompletedAt == nil || !stored[0].CompletedAt.Equal(future) || stored[0].Verified == nil || *stored[0].Verified != verified || stored[0].ID != point.ID || stored[0].ProviderScope != point.ProviderScope {+ t.Fatalf("stored evidence changed: %+v", stored[0])+ }+ after, err := json.Marshal(point)+ if err != nil || !bytes.Equal(before, after) {+ t.Fatal("read changed the source input")+ }+ point.CompletedAt = &recent+ if err := store.UpsertPoints(context.Background(), []recovery.RecoveryPoint{point}); err != nil {+ t.Fatal(err)+ }+ check(read("default"), wantCorrected, recent)+ check(read("clock-other"), wantCorrected, recent)+ })+ }+ }+}diff --git a/internal/recovery/model/types.go b/internal/recovery/model/types.goindex 27970b9611..8a1e56ce11 100644--- a/internal/recovery/model/types.go+++ b/internal/recovery/model/types.go@@ -121,13 +121,13 @@ type VerifyIntent stringconst (// VerifyIntentVerified means a verification-bearing successful recovery- // point exists within the staleness window.+ // point exists at or before evaluation time within the staleness window.VerifyIntentVerified VerifyIntent = "verified"// VerifyIntentStale means a successful backup exists for the subject but// no verification-bearing point has landed within the staleness window.VerifyIntentStale VerifyIntent = "stale"- // VerifyIntentUnknown means there is no information one way or the other- // (no successful backup, or no verification-bearing points at all).+ // VerifyIntentUnknown means there is no successful backup, or a reported+ // success/verification date is in the future and its age cannot be assessed.VerifyIntentUnknown VerifyIntent = "unknown")diff --git a/internal/recovery/recovery_test.go b/internal/recovery/recovery_test.goindex 4c82821cc1..f8512f23ca 100644--- a/internal/recovery/recovery_test.go+++ b/internal/recovery/recovery_test.go@@ -1,6 +1,8 @@package recoveryimport (+ "math"+ "reflect""strings""testing""time"@@ -1247,3 +1249,87 @@ func TestComputeVerifyIntentAt_BoundaryEqualsWindow(t *testing.T) {t.Fatalf("beyond boundary lastVerifiedAt = nil, want non-nil so UI can render history")}}++func TestRollupVerificationClockBoundaries(t *testing.T) {+ now := time.Date(2026, 10, 28, 23, 0, 0, 0, time.UTC)+ recent := now.Add(-time.Hour).UnixMilli()+ boundary := now.Add(-BackupVerifyStaleWindow).UnixMilli()+ for _, tc := range []struct {+ name string+ success, verified int64+ want VerifyIntent+ }{+ {"recent", recent, recent, VerifyIntentVerified},+ {"at-evaluation", now.UnixMilli(), now.UnixMilli(), VerifyIntentVerified},+ {"inclusive-window", boundary, boundary, VerifyIntentVerified},+ {"past-window", recent, boundary - 1, VerifyIntentStale},+ {"no-verification", recent, 0, VerifyIntentStale},+ {"invalid-verification", recent, -1, VerifyIntentStale},+ {"no-success", 0, recent, VerifyIntentUnknown},+ {"invalid-success", -1, recent, VerifyIntentUnknown},+ {"future-verification", recent, now.UnixMilli() + 1, VerifyIntentUnknown},+ {"future-success", now.UnixMilli() + 1, recent, VerifyIntentUnknown},+ {"future-success-unverified", now.UnixMilli() + 1, 0, VerifyIntentUnknown},+ {"both-future", now.UnixMilli() + 1, now.UnixMilli() + 1, VerifyIntentUnknown},+ {"extreme-future-verification", recent, math.MaxInt64, VerifyIntentUnknown},+ {"extreme-future-success", math.MaxInt64, recent, VerifyIntentUnknown},+ } {+ t.Run(tc.name, func(t *testing.T) {+ intent, verified := ComputeVerifyIntentAt(tc.success, tc.verified, now)+ if intent != tc.want {+ t.Errorf("intent=%q, want %q", intent, tc.want)+ }+ if tc.verified <= 0 {+ if verified != nil {+ t.Fatalf("unobserved verification became %v", verified)+ }+ } else if verified == nil || verified.UnixMilli() != tc.verified {+ t.Fatalf("source verification date rewritten: %v, want %d", verified, tc.verified)+ }+ })+ }+}++func TestRollupFutureDateCorrectionPreservesEvidence(t *testing.T) {+ now := time.Date(2026, 10, 28, 23, 0, 0, 0, time.UTC)+ recent, future := now.Add(-time.Hour), now.Add(time.Hour)+ verified := true+ for _, provider := range []Provider{ProviderProxmoxPVE, ProviderProxmoxPBS, ProviderProxmoxPMG, ProviderTrueNAS, ProviderKubernetes} {+ t.Run(string(provider), func(t *testing.T) {+ point := RecoveryPoint{ID: "clock-backup", Provider: provider, Kind: KindBackup, Mode: ModeRemote, Outcome: OutcomeSuccess,+ CompletedAt: &future, Verified: &verified, SubjectResourceID: "clock-guest", ProviderScope: "clock-estate"}+ points := []RecoveryPoint{point}+ before := []RecoveryPoint{point}+ rollups := BuildRollupsFromPointsAt(points, now)+ if len(rollups) != 1 {+ t.Fatalf("rollups=%+v", rollups)+ }+ r := rollups[0]+ if r.VerifyIntent != VerifyIntentUnknown {+ t.Errorf("future intent=%q, want unknown", r.VerifyIntent)+ }+ if r.LastSuccessAt == nil || !r.LastSuccessAt.Equal(future) || r.LastVerifiedAt == nil || !r.LastVerifiedAt.Equal(future) || r.LastAttemptAt == nil || !r.LastAttemptAt.Equal(future) {+ t.Fatalf("future evidence lost: %+v", r)+ }+ if r.RollupID != "res:clock-guest" || r.SubjectResourceID != point.SubjectResourceID || len(r.Providers) != 1 || r.Providers[0] != provider || !reflect.DeepEqual(points, before) {+ t.Fatalf("scope, identity or input changed: %+v", r)+ }+ // Merely collecting the same point again must not invent a new date.+ again := BuildRollupsFromPointsAt(points, now)+ if !reflect.DeepEqual(rollups, again) {+ t.Fatal("ordinary reread changed evidence")+ }+ // Only a corrected source date (or actual time reaching the reported instant)+ // makes this same verified point eligible for a verified assessment.+ points[0].CompletedAt = &recent+ corrected := BuildRollupsFromPointsAt(points, now)[0]+ if corrected.VerifyIntent != VerifyIntentVerified || corrected.LastVerifiedAt == nil || !corrected.LastVerifiedAt.Equal(recent) || corrected.RollupID != r.RollupID {+ t.Fatalf("corrected date did not recover in place: %+v", corrected)+ }+ atReportedTime := BuildRollupsFromPointsAt(before, future)[0]+ if atReportedTime.VerifyIntent != VerifyIntentVerified || !atReportedTime.LastVerifiedAt.Equal(future) {+ t.Fatal("evaluation did not recover at reported time")+ }+ })+ }+}diff --git a/internal/recovery/rollups.go b/internal/recovery/rollups.goindex e1ee167898..cbbec8091d 100644--- a/internal/recovery/rollups.go+++ b/internal/recovery/rollups.go@@ -21,10 +21,10 @@ const BackupVerifyStaleWindow = 7 * 24 * time.Hour// Semantics:// - verified : a verification-bearing point exists within window of now// - stale : a successful backup exists, but no verification within window-// - unknown : no successful backup at all (or no info)+// - unknown : no successful backup, or a success/verification date is in the future//// LastVerifiedAt is returned regardless of intent (when known) so consumers-// can surface "last verified X days ago" alongside the badge.+// can retain the source date even when clock uncertainty prevents an assessment.func ComputeVerifyIntentAt(lastSuccessMs, lastVerifiedMs int64, now time.Time) (VerifyIntent, *time.Time) {windowMs := BackupVerifyStaleWindow.Milliseconds()nowMs := now.UTC().UnixMilli()@@ -35,7 +35,10 @@ func ComputeVerifyIntentAt(lastSuccessMs, lastVerifiedMs int64, now time.Time) (lastVerifiedAt = &t}- if lastSuccessMs <= 0 {+ // A future date proves no recency. Keep the reported date as evidence,+ // but align the rollup with posture: clock uncertainty is neither verified+ // nor stale. Check before subtraction so extreme dates cannot overflow age.+ if lastSuccessMs <= 0 || lastSuccessMs > nowMs || lastVerifiedMs > nowMs {return VerifyIntentUnknown, lastVerifiedAt}