Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Reject PBS snapshots outside their requested group

pulse-triage[bot]352121a8b3a8
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md
index d4db08a560..49fccc5f34 100644
--- a/docs/release-control/v6/internal/subsystems/monitoring.md
+++ b/docs/release-control/v6/internal/subsystems/monitoring.md
@@ -17,6 +17,28 @@
## Purpose
+### PBS snapshot reads retain their exact requested group
+
+A snapshot array is accepted only when every row's native backup kind and
+opaque ID exactly match the requested group, with no repeated native snapshot
+time. Reject the whole response before sorting, bounding, conversion or cache
+renewal; neither discard only bad rows nor rewrite them to the request's subject.
+Case, whitespace, numeric spelling and host IDs remain significant. Errors name
+the row and condition, never snapshot payloads. Valid distinct rows retain their
+order and facts; explicit empty arrays remain authoritative.
+
+`backup_snapshot_identity_test.go` covers VM, CT and host controls, missing and
+foreign subjects, identical/conflicting repeats, mixed arrays and a bad row
+beyond the collector's retention cap. `pbs_snapshot_identity_test.go` crosses
+the real HTTP reader, ordinary poll, raw inventory, cache, asynchronous recovery
+ingest and protection read in cold/warm VM/CT lifecycles. Rejected reads retain
+only prior accepted history without renewing it or claiming complete protection;
+independent namespaces progress, and an ordinary valid or explicit empty read
+recovers/reconciles without a reset. No backup, guest, restore, auth or provider
+operation changes; this is malformed-response safety, not incident causality or
+original installed recovery.
+
+
### CORE per-disk throughput keeps its native meaning
The existing reporting batch retains each selected disk's read/write rate even
diff --git a/docs/release-control/v6/internal/subsystems/registry.json b/docs/release-control/v6/internal/subsystems/registry.json
index 1445b51f5d..f57b22d34d 100644
--- a/docs/release-control/v6/internal/subsystems/registry.json
+++ b/docs/release-control/v6/internal/subsystems/registry.json
@@ -6846,6 +6846,7 @@
"test_prefixes": [],
"exact_files": [
"pkg/pbs/auth_session_concurrency_test.go",
+ "pkg/pbs/backup_snapshot_identity_test.go",
"pkg/pbs/client_http_test.go",
"pkg/pbs/client_test.go",
"pkg/pbs/host_diagnostics_regression_test.go",
diff --git a/docs/release-control/v6/internal/subsystems/storage-recovery.md b/docs/release-control/v6/internal/subsystems/storage-recovery.md
index 700845be31..c967f1fb79 100644
--- a/docs/release-control/v6/internal/subsystems/storage-recovery.md
+++ b/docs/release-control/v6/internal/subsystems/storage-recovery.md
@@ -21,6 +21,27 @@
## Purpose
+### PBS snapshot group contradictions cannot renew recovery evidence
+
+Every returned PBS snapshot must belong to its exact requested kind/opaque ID,
+and each native snapshot time may occur only once within that group response.
+The PBS client rejects an ambiguous response as a whole, including invalid rows
+beyond the live retention cap. Existing malformed-read handling retains accepted
+raw/history evidence without cache or point renewal, marks collection partial
+rather than complete or access-denied, and never reconciles absence from that
+read. Independent readable namespaces remain usable. Only ordinary valid reads
+restore complete evidence; explicit empty arrays still reconcile confirmed
+absence. No guest, backup, restore, retention policy, access or schema changes.
+
+The normal-suite client `backup_snapshot_identity_test.go` and collector
+`pbs_snapshot_identity_test.go` cover exact subjects, opaque host IDs, duplicates,
+cold/warm VM/CT reads, historical preservation, cache/retry, independent scope,
+valid recovery and authoritative emptiness. Desktop/narrow browser proof must
+consume actual collector outputs through Backups inventory/posture clients,
+inspect retained evidence, uncertainty, recurrence and recovery with keyboard
+and touch. This proves truthful inspection, not original guest/native recovery.
+
+
### Fleet protection reads bound their request fan-out
Coverage reads keep the existing 200-subject batch size and issue at most four
diff --git a/internal/monitoring/pbs_snapshot_identity_test.go b/internal/monitoring/pbs_snapshot_identity_test.go
new file mode 100644
index 0000000000..33b5a13235
--- /dev/null
+++ b/internal/monitoring/pbs_snapshot_identity_test.go
@@ -0,0 +1,127 @@
+package monitoring
+
+import (
+ "encoding/json"
+ "fmt"
+ "reflect"
+ "testing"
+ "time"
+
+ "github.com/rcourtman/pulse-go-rewrite/internal/operationaltrust"
+ "github.com/rcourtman/pulse-go-rewrite/internal/recovery"
+)
+
+func conflictingPBSSnapshotBody(kind, failure string, at int64) string {
+ row := map[string]any{"backup-type": kind, "backup-id": "101", "backup-time": at, "size": 1024, "files": []string{"index.json.blob"}, "verification": map[string]any{"state": "ok"}, "comment": "untrusted-new-file"}
+ rows := []map[string]any{row}
+ switch failure {
+ case "wrong-kind":
+ if kind == "vm" {
+ row["backup-type"] = "ct"
+ } else {
+ row["backup-type"] = "vm"
+ }
+ case "wrong-id":
+ row["backup-id"] = "100"
+ case "missing-kind":
+ delete(row, "backup-type")
+ case "missing-id":
+ delete(row, "backup-id")
+ case "identical-repeat":
+ rows = append(rows, row)
+ case "conflicting-repeat":
+ rows = append(rows, map[string]any{"backup-type": kind, "backup-id": "101", "backup-time": at, "size": 2048, "verification": map[string]any{"state": "failed"}})
+ }
+ b, _ := json.Marshal(map[string]any{"data": rows})
+ return string(b)
+}
+
+func TestPBSConflictingSnapshotIdentityRetainsUnreadScope(t *testing.T) {
+ for _, kind := range []string{"vm", "ct"} {
+ for _, failure := range []string{"wrong-kind", "wrong-id", "missing-kind", "missing-id", "identical-repeat", "conflicting-repeat"} {
+ for _, warm := range []bool{false, true} {
+ t.Run(fmt.Sprintf("%s/%s/warm-%t", kind, failure, warm), func(t *testing.T) {
+ f := newPBSNamespaceContinuityFixture(t, kind)
+ if warm {
+ f.collect(t, true, nil)
+ }
+ before, points := f.live(), f.points(t, "pbs-main")
+ key := pbsBackupGroupKey{datastore: "archive", namespace: "tenant", backupType: kind, backupID: "101"}
+ cacheAt := f.m.pbsBackupCacheTimeFor("pbs-main", key)
+ f.rootAt.Add(15 * 60)
+ f.tenantAt.Add(15 * 60)
+ f.inventoryResponse.Store(&pbsInventoryResponse{stage: "snapshots", body: conflictingPBSSnapshotBody(kind, failure, f.tenantAt.Load())})
+ for range 2 {
+ f.collect(t, true, nil)
+ raw := f.m.state.GetSnapshot().PBSBackups
+ want := 1
+ if warm {
+ want = 2
+ }
+ if len(raw) != want || f.live()["100"].BackupTime.Unix() != f.rootAt.Load() {
+ t.Errorf("invalid read displaced independent scope or invented files: %+v", raw)
+ }
+ for _, row := range raw {
+ if row.Namespace != "tenant" {
+ continue
+ }
+ if !warm || !reflect.DeepEqual(row, before["101"]) {
+ t.Errorf("malformed group published or rewrote raw history: %+v", row)
+ }
+ }
+ if !f.m.pbsBackupCacheTimeFor("pbs-main", key).Equal(cacheAt) {
+ t.Error("invalid response renewed cache")
+ }
+ for _, prior := range points {
+ if prior.Details["namespace"] != "tenant" {
+ continue
+ }
+ found := false
+ for _, now := range f.points(t, "pbs-main") {
+ found = found || reflect.DeepEqual(prior, now)
+ }
+ if !found {
+ t.Error("unread historical point was deleted or renewed")
+ }
+ }
+ for _, p := range f.postures(t) {
+ if p.State == recovery.ProtectionStateProtected {
+ t.Errorf("invalid group advertised protected: %+v", p)
+ }
+ for _, provider := range p.ProviderStates {
+ if provider.HistoryCompleteness != recovery.ProtectionHistoryPartial || provider.Permissions != operationaltrust.EvidencePermissionsUnknown {
+ t.Errorf("malformed read treated as complete or access refusal: %+v", provider)
+ }
+ }
+ }
+ }
+ if f.failedReads.Load() != 2 {
+ t.Errorf("malformed response caused extra reads or poisoned TTL reuse: %d", f.failedReads.Load())
+ }
+ f.inventoryResponse.Store(nil)
+ f.collect(t, true, nil)
+ if len(f.live()) != 2 || f.live()["101"].BackupTime.Unix() != f.tenantAt.Load() || !f.m.pbsBackupCacheTimeFor("pbs-main", key).After(cacheAt) {
+ t.Error("ordinary valid read did not recover without cache reset")
+ }
+ for _, p := range f.postures(t) {
+ if p.State != recovery.ProtectionStateProtected {
+ t.Errorf("valid recovery not protected: %+v", p)
+ }
+ }
+ // A valid empty snapshot array is still authoritative for this
+ // group; the independent root namespace must survive it.
+ f.tenantAt.Add(int64(time.Hour / time.Second))
+ f.inventoryResponse.Store(&pbsInventoryResponse{stage: "snapshots", body: `{"data":[]}`})
+ f.collect(t, true, nil)
+ if len(f.live()) != 1 || f.live()["100"].VMID != "100" {
+ t.Error("confirmed empty snapshots did not remove only their own group")
+ }
+ points = f.points(t, "pbs-main")
+ if len(points) != 1 || points[0].Details["namespace"] != "" {
+ t.Error("complete empty snapshot read did not reconcile only its own group")
+ }
+ })
+ }
+ }
+ }
+}
diff --git a/pkg/pbs/backup_snapshot_identity_test.go b/pkg/pbs/backup_snapshot_identity_test.go
new file mode 100644
index 0000000000..ef2ce41f1e
--- /dev/null
+++ b/pkg/pbs/backup_snapshot_identity_test.go
@@ -0,0 +1,86 @@
+package pbs
+
+import (
+ "context"
+ "encoding/json"
+ "net/http"
+ "reflect"
+ "strings"
+ "testing"
+)
+
+func TestBackupSnapshotsRequireRequestedUniqueIdentity(t *testing.T) {
+ for _, group := range []struct{ kind, id string }{{"vm", "101"}, {"ct", "101"}, {"host", "host-A.example"}} {
+ row := func(kind, id string, at int64) BackupSnapshot {
+ return BackupSnapshot{BackupType: kind, BackupID: id, BackupTime: at, Size: 1024, Comment: "private-payload-marker"}
+ }
+ valid := row(group.kind, group.id, 1793293200)
+ wrongKind := "vm"
+ if group.kind == wrongKind {
+ wrongKind = "ct"
+ }
+ for _, tc := range []struct {
+ name string
+ rows []BackupSnapshot
+ valid bool
+ }{
+ {"empty", []BackupSnapshot{}, true},
+ {"matching", []BackupSnapshot{valid}, true},
+ {"distinct-times-in-response-order", []BackupSnapshot{valid, row(group.kind, group.id, valid.BackupTime-3600)}, true},
+ {"wrong-kind", []BackupSnapshot{row(wrongKind, group.id, valid.BackupTime)}, false},
+ {"wrong-id", []BackupSnapshot{row(group.kind, group.id+"0", valid.BackupTime)}, false},
+ {"missing-kind", []BackupSnapshot{row("", group.id, valid.BackupTime)}, false},
+ {"missing-id", []BackupSnapshot{row(group.kind, "", valid.BackupTime)}, false},
+ {"no-kind-normalisation", []BackupSnapshot{row(strings.ToUpper(group.kind), group.id, valid.BackupTime)}, false},
+ {"no-id-normalisation", []BackupSnapshot{row(group.kind, " "+group.id, valid.BackupTime)}, false},
+ {"mixed-valid-and-invalid", []BackupSnapshot{valid, row(wrongKind, group.id, valid.BackupTime-3600)}, false},
+ {"identical-repeat", []BackupSnapshot{valid, valid}, false},
+ {"conflicting-repeat", []BackupSnapshot{valid, {BackupType: group.kind, BackupID: group.id, BackupTime: valid.BackupTime, Size: 999}}, false},
+ } {
+ t.Run(group.kind+"/"+tc.name, func(t *testing.T) {
+ reads := 0
+ client, server := newTestClient(t, func(w http.ResponseWriter, r *http.Request) {
+ reads++
+ q := r.URL.Query()
+ if r.Method != http.MethodGet || r.URL.Path != "/api2/json/admin/datastore/archive/snapshots" || q.Get("ns") != "tenant/one" || q.Get("backup-type") != group.kind || q.Get("backup-id") != group.id {
+ t.Errorf("request scope/method changed: %s %s", r.Method, r.URL)
+ }
+ _ = json.NewEncoder(w).Encode(map[string]any{"data": tc.rows})
+ })
+ defer server.Close()
+ got, err := client.ListBackupSnapshots(context.Background(), "archive", "tenant/one", group.kind, group.id)
+ if tc.valid {
+ if err != nil || !reflect.DeepEqual(got, tc.rows) {
+ t.Fatalf("valid response changed: rows=%+v err=%v", got, err)
+ }
+ } else {
+ if err == nil || len(got) != 0 {
+ t.Fatalf("conflicting group admitted: rows=%+v err=%v", got, err)
+ }
+ if err != nil && strings.Contains(err.Error(), "private-payload-marker") {
+ t.Error("error exposed snapshot payload")
+ }
+ }
+ if reads != 1 {
+ t.Errorf("admission caused %d requests, want one", reads)
+ }
+ })
+ }
+ }
+}
+
+func TestBackupSnapshotIdentityAdmissionCoversEntireResponse(t *testing.T) {
+ rows := make([]BackupSnapshot, 101)
+ for i := range rows {
+ rows[i] = BackupSnapshot{BackupType: "vm", BackupID: "101", BackupTime: int64(1793293200 - i)}
+ }
+ rows[len(rows)-1].BackupID = "102"
+ client, server := newTestClient(t, func(w http.ResponseWriter, r *http.Request) {
+ _ = json.NewEncoder(w).Encode(map[string]any{"data": rows})
+ })
+ defer server.Close()
+ got, err := client.ListBackupSnapshots(context.Background(), "archive", "", "vm", "101")
+ if err == nil || len(got) != 0 {
+ t.Fatalf("invalid row beyond retained group cap admitted: count=%d err=%v", len(got), err)
+ }
+}
diff --git a/pkg/pbs/client.go b/pkg/pbs/client.go
index 384e8d0c70..182cee1b2d 100644
--- a/pkg/pbs/client.go
+++ b/pkg/pbs/client.go
@@ -2059,6 +2059,22 @@ func (c *Client) ListBackupSnapshots(ctx context.Context, datastore string, name
return nil, fmt.Errorf("snapshot list response contains no data array")
}
+ // The request selects one exact group. A foreign row or repeated native
+ // snapshot identity makes the whole read ambiguous, not partially usable
+ // history or an authoritative empty group. Validate before callers sort,
+ // cap or cache rows; their existing malformed-read path retains old evidence
+ // without renewing it. Do not put provider payloads in the error.
+ seen := make(map[int64]struct{}, len(result.Data))
+ for i, snapshot := range result.Data {
+ if snapshot.BackupType != backupType || snapshot.BackupID != backupID {
+ return nil, fmt.Errorf("snapshot list response row %d does not match requested backup group", i)
+ }
+ if _, exists := seen[snapshot.BackupTime]; exists {
+ return nil, fmt.Errorf("snapshot list response row %d repeats a snapshot time", i)
+ }
+ seen[snapshot.BackupTime] = struct{}{}
+ }
+
return result.Data, nil
}