Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Require explicit PBS backup inventory arrays before reconciling absence

pulse-triage[bot]72e777d962d5
diff --git a/internal/monitoring/pbs_inventory_envelope_test.go b/internal/monitoring/pbs_inventory_envelope_test.go
new file mode 100644
index 0000000000..8bed6e500f
--- /dev/null
+++ b/internal/monitoring/pbs_inventory_envelope_test.go
@@ -0,0 +1,194 @@
+package monitoring
+
+import (
+ "context"
+ "crypto/sha256"
+ "encoding/json"
+ "fmt"
+ "os"
+ "reflect"
+ "testing"
+ "time"
+
+ "github.com/rcourtman/pulse-go-rewrite/internal/models"
+ "github.com/rcourtman/pulse-go-rewrite/internal/operationaltrust"
+ "github.com/rcourtman/pulse-go-rewrite/internal/recovery"
+)
+
+type pbsInventoryResponse struct {
+ stage string
+ body string
+}
+
+func (f *pbsNamespaceContinuityFixture) collectInventoryResponse(t *testing.T, stage, body string) {
+ t.Helper()
+ f.inventoryResponse.Store(&pbsInventoryResponse{stage: stage, body: body})
+ var discoveryErr error
+ if stage == "namespace" {
+ _, discoveryErr = f.client.ListNamespaces(context.Background(), "archive", "", 0)
+ if discoveryErr == nil {
+ t.Error("namespace discovery accepted a missing inventory array")
+ }
+ }
+ f.collect(t, stage != "namespace", discoveryErr)
+}
+
+func TestPBSMissingInventoryArrayRetainsUnreadScope(t *testing.T) {
+ for _, kind := range []string{"vm", "ct"} {
+ for _, stage := range []string{"namespace", "groups", "snapshots"} {
+ for _, body := range []string{`null`, `{}`, `{"data":null}`} {
+ for _, warm := range []bool{false, true} {
+ t.Run(fmt.Sprintf("%s/%s/%s/warm=%t", kind, stage, body, warm), func(t *testing.T) {
+ f := newPBSNamespaceContinuityFixture(t, kind)
+ if warm {
+ f.collect(t, true, nil)
+ }
+ before, points := f.live(), f.points(t, "pbs-main")
+ key := pbsBackupGroupKey{datastore: "archive", namespace: "tenant", backupType: kind, backupID: "101"}
+ cacheAt := f.m.pbsBackupCacheTimeFor("pbs-main", key)
+ f.rootAt.Add(15 * 60)
+ f.tenantAt.Add(15 * 60) // Require a snapshot read at the existing TTL.
+ for range 2 {
+ f.collectInventoryResponse(t, stage, body)
+ rows := f.live()
+ wantCount := 1
+ if warm {
+ wantCount = 2
+ if !reflect.DeepEqual(rows["101"], before["101"]) {
+ t.Error("unread file was removed or changed")
+ }
+ }
+ if len(rows) != wantCount || rows["100"].BackupTime.Unix() != f.rootAt.Load() {
+ t.Errorf("unread scope invented/lost files or held back its readable sibling: %+v", rows)
+ }
+ if !f.m.pbsBackupCacheTimeFor("pbs-main", key).Equal(cacheAt) {
+ t.Error("missing inventory renewed the snapshot cache")
+ }
+ for _, prior := range points {
+ if prior.Details["namespace"] != "tenant" {
+ continue
+ }
+ found := false
+ for _, now := range f.points(t, "pbs-main") {
+ found = found || (prior.ID == now.ID && reflect.DeepEqual(prior, now))
+ }
+ if !found {
+ t.Error("unread historical point was removed or its evidence renewed")
+ }
+ }
+ for _, p := range f.postures(t) {
+ if p.State == recovery.ProtectionStateProtected || len(p.ProviderStates) != 1 ||
+ p.ProviderStates[0].HistoryCompleteness != recovery.ProtectionHistoryPartial ||
+ p.ProviderStates[0].Permissions != operationaltrust.EvidencePermissionsUnknown {
+ t.Errorf("missing inventory invented complete protection or denied access: %+v", p)
+ }
+ }
+ }
+ if f.failedReads.Load() != 2 {
+ t.Error("invalid inventory caused an extra request or hid a failed read")
+ }
+ f.inventoryResponse.Store(nil)
+ f.collect(t, true, nil)
+ if f.live()["101"].BackupTime.Unix() != f.tenantAt.Load() {
+ t.Error("ordinary valid read did not restore inventory")
+ }
+ for _, p := range f.postures(t) {
+ if p.State != recovery.ProtectionStateProtected {
+ t.Error("complete ordinary recovery did not restore protection")
+ }
+ }
+ f.rootEmpty.Store(true)
+ f.tenantEmpty.Store(true)
+ f.collect(t, true, nil)
+ if len(f.live()) != 0 || len(f.points(t, "pbs-main")) != 0 {
+ t.Error("explicit empty arrays did not reconcile confirmed absence")
+ }
+ })
+ }
+ }
+ }
+ }
+}
+
+func TestPBSMissingInventoryArrayKeepsOtherConnection(t *testing.T) {
+ f := newPBSNamespaceContinuityFixture(t, "vm")
+ f.collect(t, true, nil)
+ // Populate a second connection through the same ordinary synthetic reader.
+ f.m.pollPBSBackups(context.Background(), "pbs-other", f.client, []models.PBSDatastore{{Name: "archive"}}, nil)
+ f.wait(t)
+ before := f.m.state.GetSnapshot().PBSBackups
+ points := f.points(t, "pbs-other")
+ f.collectInventoryResponse(t, "groups", `{"data":null}`)
+ var retained []models.PBSBackup
+ for _, row := range f.m.state.GetSnapshot().PBSBackups {
+ if row.Instance == "pbs-other" {
+ retained = append(retained, row)
+ }
+ }
+ var original []models.PBSBackup
+ for _, row := range before {
+ if row.Instance == "pbs-other" {
+ original = append(original, row)
+ }
+ }
+ if len(original) != 1 || !reflect.DeepEqual(original, retained) || !reflect.DeepEqual(points, f.points(t, "pbs-other")) {
+ t.Error("invalid inventory altered another connection's files or point evidence")
+ }
+}
+
+// This normal-suite lifecycle can export its actual poll/ingest/posture results
+// for an evidence-only production Backups/API-hook desktop/narrow browser pass.
+func TestPBSMissingInventoryArrayConsumerLifecycle(t *testing.T) {
+ f := newPBSNamespaceContinuityFixture(t, "vm")
+ frames := map[string]any{}
+ for _, phase := range []string{"observed", "missing-namespaces", "null-groups", "null-snapshots", "recovered", "confirmed-empty"} {
+ switch phase {
+ case "missing-namespaces":
+ f.rootAt.Add(15 * 60)
+ f.collectInventoryResponse(t, "namespace", `{}`)
+ case "null-groups":
+ f.collectInventoryResponse(t, "groups", `{"data":null}`)
+ case "null-snapshots":
+ f.tenantAt.Add(15 * 60)
+ f.collectInventoryResponse(t, "snapshots", `{"data":null}`)
+ default:
+ f.inventoryResponse.Store(nil)
+ if phase == "confirmed-empty" {
+ f.rootEmpty.Store(true)
+ f.tenantEmpty.Store(true)
+ }
+ f.collect(t, true, nil)
+ }
+ postures := f.postures(t)
+ for _, p := range postures {
+ healthy := phase == "observed" || phase == "recovered"
+ if healthy != (p.State == recovery.ProtectionStateProtected) {
+ t.Fatalf("phase %s posture=%+v", phase, p)
+ }
+ }
+ want := 2
+ if phase == "confirmed-empty" {
+ want = 0
+ }
+ if len(f.live()) != want || len(f.points(t, "pbs-main")) != want {
+ t.Fatalf("phase %s lost or invented backup facts", phase)
+ }
+ frames[phase] = map[string]any{"pbs": map[string]any{"data": map[string]any{"backups": f.m.state.GetSnapshot().PBSBackups}}, "postures": map[string]any{"data": postures, "policy": recovery.DefaultProtectionPosturePolicy.Payload()}, "points": f.points(t, "pbs-main")}
+ }
+ if os.Getenv("PULSE_PBS_INVENTORY_FRAMES") != "1" {
+ return
+ }
+ hashes := map[string]string{}
+ for _, file := range []string{"monitor_backups.go", "backup_guard.go", "../../pkg/pbs/client.go"} {
+ content, err := os.ReadFile(file)
+ if err != nil {
+ t.Fatal(err)
+ }
+ hashes[file] = fmt.Sprintf("%x", sha256.Sum256(content))
+ }
+ data, err := json.Marshal(map[string]any{"runtime_sha256": hashes, "observed_at": time.Now().UTC(), "resource_ids": f.ids, "frames": frames})
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Log("PBS_INVENTORY_FRAMES=" + string(data))
+}
diff --git a/internal/monitoring/pbs_namespace_continuity_test.go b/internal/monitoring/pbs_namespace_continuity_test.go
index 5285231612..d3cdac5b5d 100644
--- a/internal/monitoring/pbs_namespace_continuity_test.go
+++ b/internal/monitoring/pbs_namespace_continuity_test.go
@@ -27,6 +27,7 @@ type pbsNamespaceContinuityFixture struct {
rootReads, tenantReads atomic.Int32
tenantSnapshotStatus, namespaceStatus atomic.Int32
failedReads atomic.Int32
+ inventoryResponse atomic.Pointer[pbsInventoryResponse]
}
func newPBSNamespaceContinuityFixture(t *testing.T, kind string) *pbsNamespaceContinuityFixture {
@@ -35,6 +36,13 @@ func newPBSNamespaceContinuityFixture(t *testing.T, kind string) *pbsNamespaceCo
f.rootAt.Store(f.backupAt)
f.tenantAt.Store(f.backupAt)
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if response := f.inventoryResponse.Load(); response != nil &&
+ r.URL.Path == "/api2/json/admin/datastore/archive/"+response.stage &&
+ (response.stage == "namespace" || r.URL.Query().Get("ns") == "tenant") {
+ f.failedReads.Add(1)
+ _, _ = w.Write([]byte(response.body))
+ return
+ }
id, at, status, empty := "100", f.rootAt.Load(), f.rootStatus.Load(), f.rootEmpty.Load()
if r.URL.Query().Get("ns") == "tenant" {
id, at, status, empty = "101", f.tenantAt.Load(), f.tenantStatus.Load(), f.tenantEmpty.Load()
diff --git a/pkg/pbs/backup_inventory_envelope_test.go b/pkg/pbs/backup_inventory_envelope_test.go
new file mode 100644
index 0000000000..68b9084b69
--- /dev/null
+++ b/pkg/pbs/backup_inventory_envelope_test.go
@@ -0,0 +1,56 @@
+package pbs
+
+import (
+ "context"
+ "net/http"
+ "testing"
+)
+
+// A successful HTTP status without an inventory array is an unread scope, not
+// an accepted empty listing. In particular, decoding JSON null is not a parse
+// error in Go and must not silently authorize backup-history reconciliation.
+func TestBackupInventoryRequiresExplicitArray(t *testing.T) {
+ readers := []struct {
+ name string
+ read func(*Client) (int, error)
+ }{
+ {"namespaces", func(c *Client) (int, error) {
+ rows, err := c.ListNamespaces(context.Background(), "archive", "tenant", 2)
+ return len(rows), err
+ }},
+ {"groups", func(c *Client) (int, error) {
+ rows, err := c.ListBackupGroups(context.Background(), "archive", "tenant")
+ return len(rows), err
+ }},
+ {"snapshots", func(c *Client) (int, error) {
+ rows, err := c.ListBackupSnapshots(context.Background(), "archive", "tenant", "vm", "101")
+ return len(rows), err
+ }},
+ }
+ for _, reader := range readers {
+ for _, body := range []string{`null`, `{}`, `{"data":null}`, `{"unrelated":[]}`, `{"data":{}}`, `{"data":false}`, `{"data":`, `{"data":[]}`} {
+ t.Run(reader.name+"/"+body, func(t *testing.T) {
+ reads := 0
+ client, server := newTestClient(t, func(w http.ResponseWriter, r *http.Request) {
+ reads++
+ if r.Method != http.MethodGet {
+ t.Errorf("inventory read made a %s request", r.Method)
+ }
+ _, _ = w.Write([]byte(body))
+ })
+ defer server.Close()
+ count, err := reader.read(client)
+ if body == `{"data":[]}` {
+ if err != nil || count != 0 {
+ t.Fatalf("explicit empty array rejected: count=%d err=%v", count, err)
+ }
+ } else if err == nil || count != 0 {
+ t.Fatalf("unread scope accepted: count=%d err=%v", count, err)
+ }
+ if reads != 1 {
+ t.Fatalf("inventory admission caused %d requests, want one", reads)
+ }
+ })
+ }
+ }
+}
diff --git a/pkg/pbs/client.go b/pkg/pbs/client.go
index 4118e0501f..28bc0c7469 100644
--- a/pkg/pbs/client.go
+++ b/pkg/pbs/client.go
@@ -1465,6 +1465,8 @@ func (c *Client) getJobConfigMap(ctx context.Context, path string) (map[string]i
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
return nil, err
}
+ // Match datastore discovery: missing/null data is unread, not an empty
+ // scope that can authorize removal of retained backup history.
if result.Data == nil {
result.Data = map[string]interface{}{}
}
@@ -1964,6 +1966,9 @@ func (c *Client) ListNamespaces(ctx context.Context, datastore string, parentNam
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
return nil, err
}
+ if result.Data == nil {
+ return nil, fmt.Errorf("namespace list response contains no data array")
+ }
return result.Data, nil
}
@@ -2005,6 +2010,9 @@ func (c *Client) ListBackupGroups(ctx context.Context, datastore string, namespa
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
return nil, fmt.Errorf("failed to decode backup groups: %w", err)
}
+ if result.Data == nil {
+ return nil, fmt.Errorf("backup group list response contains no data array")
+ }
log.Debug().
Str("namespace", namespace).
@@ -2047,6 +2055,9 @@ func (c *Client) ListBackupSnapshots(ctx context.Context, datastore string, name
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
return nil, fmt.Errorf("failed to decode snapshots: %w", err)
}
+ if result.Data == nil {
+ return nil, fmt.Errorf("snapshot list response contains no data array")
+ }
return result.Data, nil
}