Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Reject PBS snapshots outside their requested group
pulse-triage[bot]352121a8b3a8
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.mdindex d4db08a560..49fccc5f34 100644--- a/docs/release-control/v6/internal/subsystems/monitoring.md+++ b/docs/release-control/v6/internal/subsystems/monitoring.md@@ -17,6 +17,28 @@## Purpose+### PBS snapshot reads retain their exact requested group++A snapshot array is accepted only when every row's native backup kind and+opaque ID exactly match the requested group, with no repeated native snapshot+time. Reject the whole response before sorting, bounding, conversion or cache+renewal; neither discard only bad rows nor rewrite them to the request's subject.+Case, whitespace, numeric spelling and host IDs remain significant. Errors name+the row and condition, never snapshot payloads. Valid distinct rows retain their+order and facts; explicit empty arrays remain authoritative.++`backup_snapshot_identity_test.go` covers VM, CT and host controls, missing and+foreign subjects, identical/conflicting repeats, mixed arrays and a bad row+beyond the collector's retention cap. `pbs_snapshot_identity_test.go` crosses+the real HTTP reader, ordinary poll, raw inventory, cache, asynchronous recovery+ingest and protection read in cold/warm VM/CT lifecycles. Rejected reads retain+only prior accepted history without renewing it or claiming complete protection;+independent namespaces progress, and an ordinary valid or explicit empty read+recovers/reconciles without a reset. No backup, guest, restore, auth or provider+operation changes; this is malformed-response safety, not incident causality or+original installed recovery.++### CORE per-disk throughput keeps its native meaningThe existing reporting batch retains each selected disk's read/write rate evendiff --git a/docs/release-control/v6/internal/subsystems/registry.json b/docs/release-control/v6/internal/subsystems/registry.jsonindex 1445b51f5d..f57b22d34d 100644--- a/docs/release-control/v6/internal/subsystems/registry.json+++ b/docs/release-control/v6/internal/subsystems/registry.json@@ -6846,6 +6846,7 @@"test_prefixes": [],"exact_files": ["pkg/pbs/auth_session_concurrency_test.go",+ "pkg/pbs/backup_snapshot_identity_test.go","pkg/pbs/client_http_test.go","pkg/pbs/client_test.go","pkg/pbs/host_diagnostics_regression_test.go",diff --git a/docs/release-control/v6/internal/subsystems/storage-recovery.md b/docs/release-control/v6/internal/subsystems/storage-recovery.mdindex 700845be31..c967f1fb79 100644--- a/docs/release-control/v6/internal/subsystems/storage-recovery.md+++ b/docs/release-control/v6/internal/subsystems/storage-recovery.md@@ -21,6 +21,27 @@## Purpose+### PBS snapshot group contradictions cannot renew recovery evidence++Every returned PBS snapshot must belong to its exact requested kind/opaque ID,+and each native snapshot time may occur only once within that group response.+The PBS client rejects an ambiguous response as a whole, including invalid rows+beyond the live retention cap. Existing malformed-read handling retains accepted+raw/history evidence without cache or point renewal, marks collection partial+rather than complete or access-denied, and never reconciles absence from that+read. Independent readable namespaces remain usable. Only ordinary valid reads+restore complete evidence; explicit empty arrays still reconcile confirmed+absence. No guest, backup, restore, retention policy, access or schema changes.++The normal-suite client `backup_snapshot_identity_test.go` and collector+`pbs_snapshot_identity_test.go` cover exact subjects, opaque host IDs, duplicates,+cold/warm VM/CT reads, historical preservation, cache/retry, independent scope,+valid recovery and authoritative emptiness. Desktop/narrow browser proof must+consume actual collector outputs through Backups inventory/posture clients,+inspect retained evidence, uncertainty, recurrence and recovery with keyboard+and touch. This proves truthful inspection, not original guest/native recovery.++### Fleet protection reads bound their request fan-outCoverage reads keep the existing 200-subject batch size and issue at most fourdiff --git a/internal/monitoring/pbs_snapshot_identity_test.go b/internal/monitoring/pbs_snapshot_identity_test.gonew file mode 100644index 0000000000..33b5a13235--- /dev/null+++ b/internal/monitoring/pbs_snapshot_identity_test.go@@ -0,0 +1,127 @@+package monitoring++import (+ "encoding/json"+ "fmt"+ "reflect"+ "testing"+ "time"++ "github.com/rcourtman/pulse-go-rewrite/internal/operationaltrust"+ "github.com/rcourtman/pulse-go-rewrite/internal/recovery"+)++func conflictingPBSSnapshotBody(kind, failure string, at int64) string {+ row := map[string]any{"backup-type": kind, "backup-id": "101", "backup-time": at, "size": 1024, "files": []string{"index.json.blob"}, "verification": map[string]any{"state": "ok"}, "comment": "untrusted-new-file"}+ rows := []map[string]any{row}+ switch failure {+ case "wrong-kind":+ if kind == "vm" {+ row["backup-type"] = "ct"+ } else {+ row["backup-type"] = "vm"+ }+ case "wrong-id":+ row["backup-id"] = "100"+ case "missing-kind":+ delete(row, "backup-type")+ case "missing-id":+ delete(row, "backup-id")+ case "identical-repeat":+ rows = append(rows, row)+ case "conflicting-repeat":+ rows = append(rows, map[string]any{"backup-type": kind, "backup-id": "101", "backup-time": at, "size": 2048, "verification": map[string]any{"state": "failed"}})+ }+ b, _ := json.Marshal(map[string]any{"data": rows})+ return string(b)+}++func TestPBSConflictingSnapshotIdentityRetainsUnreadScope(t *testing.T) {+ for _, kind := range []string{"vm", "ct"} {+ for _, failure := range []string{"wrong-kind", "wrong-id", "missing-kind", "missing-id", "identical-repeat", "conflicting-repeat"} {+ for _, warm := range []bool{false, true} {+ t.Run(fmt.Sprintf("%s/%s/warm-%t", kind, failure, warm), func(t *testing.T) {+ f := newPBSNamespaceContinuityFixture(t, kind)+ if warm {+ f.collect(t, true, nil)+ }+ before, points := f.live(), f.points(t, "pbs-main")+ key := pbsBackupGroupKey{datastore: "archive", namespace: "tenant", backupType: kind, backupID: "101"}+ cacheAt := f.m.pbsBackupCacheTimeFor("pbs-main", key)+ f.rootAt.Add(15 * 60)+ f.tenantAt.Add(15 * 60)+ f.inventoryResponse.Store(&pbsInventoryResponse{stage: "snapshots", body: conflictingPBSSnapshotBody(kind, failure, f.tenantAt.Load())})+ for range 2 {+ f.collect(t, true, nil)+ raw := f.m.state.GetSnapshot().PBSBackups+ want := 1+ if warm {+ want = 2+ }+ if len(raw) != want || f.live()["100"].BackupTime.Unix() != f.rootAt.Load() {+ t.Errorf("invalid read displaced independent scope or invented files: %+v", raw)+ }+ for _, row := range raw {+ if row.Namespace != "tenant" {+ continue+ }+ if !warm || !reflect.DeepEqual(row, before["101"]) {+ t.Errorf("malformed group published or rewrote raw history: %+v", row)+ }+ }+ if !f.m.pbsBackupCacheTimeFor("pbs-main", key).Equal(cacheAt) {+ t.Error("invalid response renewed cache")+ }+ for _, prior := range points {+ if prior.Details["namespace"] != "tenant" {+ continue+ }+ found := false+ for _, now := range f.points(t, "pbs-main") {+ found = found || reflect.DeepEqual(prior, now)+ }+ if !found {+ t.Error("unread historical point was deleted or renewed")+ }+ }+ for _, p := range f.postures(t) {+ if p.State == recovery.ProtectionStateProtected {+ t.Errorf("invalid group advertised protected: %+v", p)+ }+ for _, provider := range p.ProviderStates {+ if provider.HistoryCompleteness != recovery.ProtectionHistoryPartial || provider.Permissions != operationaltrust.EvidencePermissionsUnknown {+ t.Errorf("malformed read treated as complete or access refusal: %+v", provider)+ }+ }+ }+ }+ if f.failedReads.Load() != 2 {+ t.Errorf("malformed response caused extra reads or poisoned TTL reuse: %d", f.failedReads.Load())+ }+ f.inventoryResponse.Store(nil)+ f.collect(t, true, nil)+ if len(f.live()) != 2 || f.live()["101"].BackupTime.Unix() != f.tenantAt.Load() || !f.m.pbsBackupCacheTimeFor("pbs-main", key).After(cacheAt) {+ t.Error("ordinary valid read did not recover without cache reset")+ }+ for _, p := range f.postures(t) {+ if p.State != recovery.ProtectionStateProtected {+ t.Errorf("valid recovery not protected: %+v", p)+ }+ }+ // A valid empty snapshot array is still authoritative for this+ // group; the independent root namespace must survive it.+ f.tenantAt.Add(int64(time.Hour / time.Second))+ f.inventoryResponse.Store(&pbsInventoryResponse{stage: "snapshots", body: `{"data":[]}`})+ f.collect(t, true, nil)+ if len(f.live()) != 1 || f.live()["100"].VMID != "100" {+ t.Error("confirmed empty snapshots did not remove only their own group")+ }+ points = f.points(t, "pbs-main")+ if len(points) != 1 || points[0].Details["namespace"] != "" {+ t.Error("complete empty snapshot read did not reconcile only its own group")+ }+ })+ }+ }+ }+}diff --git a/pkg/pbs/backup_snapshot_identity_test.go b/pkg/pbs/backup_snapshot_identity_test.gonew file mode 100644index 0000000000..ef2ce41f1e--- /dev/null+++ b/pkg/pbs/backup_snapshot_identity_test.go@@ -0,0 +1,86 @@+package pbs++import (+ "context"+ "encoding/json"+ "net/http"+ "reflect"+ "strings"+ "testing"+)++func TestBackupSnapshotsRequireRequestedUniqueIdentity(t *testing.T) {+ for _, group := range []struct{ kind, id string }{{"vm", "101"}, {"ct", "101"}, {"host", "host-A.example"}} {+ row := func(kind, id string, at int64) BackupSnapshot {+ return BackupSnapshot{BackupType: kind, BackupID: id, BackupTime: at, Size: 1024, Comment: "private-payload-marker"}+ }+ valid := row(group.kind, group.id, 1793293200)+ wrongKind := "vm"+ if group.kind == wrongKind {+ wrongKind = "ct"+ }+ for _, tc := range []struct {+ name string+ rows []BackupSnapshot+ valid bool+ }{+ {"empty", []BackupSnapshot{}, true},+ {"matching", []BackupSnapshot{valid}, true},+ {"distinct-times-in-response-order", []BackupSnapshot{valid, row(group.kind, group.id, valid.BackupTime-3600)}, true},+ {"wrong-kind", []BackupSnapshot{row(wrongKind, group.id, valid.BackupTime)}, false},+ {"wrong-id", []BackupSnapshot{row(group.kind, group.id+"0", valid.BackupTime)}, false},+ {"missing-kind", []BackupSnapshot{row("", group.id, valid.BackupTime)}, false},+ {"missing-id", []BackupSnapshot{row(group.kind, "", valid.BackupTime)}, false},+ {"no-kind-normalisation", []BackupSnapshot{row(strings.ToUpper(group.kind), group.id, valid.BackupTime)}, false},+ {"no-id-normalisation", []BackupSnapshot{row(group.kind, " "+group.id, valid.BackupTime)}, false},+ {"mixed-valid-and-invalid", []BackupSnapshot{valid, row(wrongKind, group.id, valid.BackupTime-3600)}, false},+ {"identical-repeat", []BackupSnapshot{valid, valid}, false},+ {"conflicting-repeat", []BackupSnapshot{valid, {BackupType: group.kind, BackupID: group.id, BackupTime: valid.BackupTime, Size: 999}}, false},+ } {+ t.Run(group.kind+"/"+tc.name, func(t *testing.T) {+ reads := 0+ client, server := newTestClient(t, func(w http.ResponseWriter, r *http.Request) {+ reads+++ q := r.URL.Query()+ if r.Method != http.MethodGet || r.URL.Path != "/api2/json/admin/datastore/archive/snapshots" || q.Get("ns") != "tenant/one" || q.Get("backup-type") != group.kind || q.Get("backup-id") != group.id {+ t.Errorf("request scope/method changed: %s %s", r.Method, r.URL)+ }+ _ = json.NewEncoder(w).Encode(map[string]any{"data": tc.rows})+ })+ defer server.Close()+ got, err := client.ListBackupSnapshots(context.Background(), "archive", "tenant/one", group.kind, group.id)+ if tc.valid {+ if err != nil || !reflect.DeepEqual(got, tc.rows) {+ t.Fatalf("valid response changed: rows=%+v err=%v", got, err)+ }+ } else {+ if err == nil || len(got) != 0 {+ t.Fatalf("conflicting group admitted: rows=%+v err=%v", got, err)+ }+ if err != nil && strings.Contains(err.Error(), "private-payload-marker") {+ t.Error("error exposed snapshot payload")+ }+ }+ if reads != 1 {+ t.Errorf("admission caused %d requests, want one", reads)+ }+ })+ }+ }+}++func TestBackupSnapshotIdentityAdmissionCoversEntireResponse(t *testing.T) {+ rows := make([]BackupSnapshot, 101)+ for i := range rows {+ rows[i] = BackupSnapshot{BackupType: "vm", BackupID: "101", BackupTime: int64(1793293200 - i)}+ }+ rows[len(rows)-1].BackupID = "102"+ client, server := newTestClient(t, func(w http.ResponseWriter, r *http.Request) {+ _ = json.NewEncoder(w).Encode(map[string]any{"data": rows})+ })+ defer server.Close()+ got, err := client.ListBackupSnapshots(context.Background(), "archive", "", "vm", "101")+ if err == nil || len(got) != 0 {+ t.Fatalf("invalid row beyond retained group cap admitted: count=%d err=%v", len(got), err)+ }+}diff --git a/pkg/pbs/client.go b/pkg/pbs/client.goindex 384e8d0c70..182cee1b2d 100644--- a/pkg/pbs/client.go+++ b/pkg/pbs/client.go@@ -2059,6 +2059,22 @@ func (c *Client) ListBackupSnapshots(ctx context.Context, datastore string, namereturn nil, fmt.Errorf("snapshot list response contains no data array")}+ // The request selects one exact group. A foreign row or repeated native+ // snapshot identity makes the whole read ambiguous, not partially usable+ // history or an authoritative empty group. Validate before callers sort,+ // cap or cache rows; their existing malformed-read path retains old evidence+ // without renewing it. Do not put provider payloads in the error.+ seen := make(map[int64]struct{}, len(result.Data))+ for i, snapshot := range result.Data {+ if snapshot.BackupType != backupType || snapshot.BackupID != backupID {+ return nil, fmt.Errorf("snapshot list response row %d does not match requested backup group", i)+ }+ if _, exists := seen[snapshot.BackupTime]; exists {+ return nil, fmt.Errorf("snapshot list response row %d repeats a snapshot time", i)+ }+ seen[snapshot.BackupTime] = struct{}{}+ }+return result.Data, nil}