Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Require explicit PBS backup inventory arrays before reconciling absence
pulse-triage[bot]72e777d962d5
diff --git a/internal/monitoring/pbs_inventory_envelope_test.go b/internal/monitoring/pbs_inventory_envelope_test.gonew file mode 100644index 0000000000..8bed6e500f--- /dev/null+++ b/internal/monitoring/pbs_inventory_envelope_test.go@@ -0,0 +1,194 @@+package monitoring++import (+ "context"+ "crypto/sha256"+ "encoding/json"+ "fmt"+ "os"+ "reflect"+ "testing"+ "time"++ "github.com/rcourtman/pulse-go-rewrite/internal/models"+ "github.com/rcourtman/pulse-go-rewrite/internal/operationaltrust"+ "github.com/rcourtman/pulse-go-rewrite/internal/recovery"+)++type pbsInventoryResponse struct {+ stage string+ body string+}++func (f *pbsNamespaceContinuityFixture) collectInventoryResponse(t *testing.T, stage, body string) {+ t.Helper()+ f.inventoryResponse.Store(&pbsInventoryResponse{stage: stage, body: body})+ var discoveryErr error+ if stage == "namespace" {+ _, discoveryErr = f.client.ListNamespaces(context.Background(), "archive", "", 0)+ if discoveryErr == nil {+ t.Error("namespace discovery accepted a missing inventory array")+ }+ }+ f.collect(t, stage != "namespace", discoveryErr)+}++func TestPBSMissingInventoryArrayRetainsUnreadScope(t *testing.T) {+ for _, kind := range []string{"vm", "ct"} {+ for _, stage := range []string{"namespace", "groups", "snapshots"} {+ for _, body := range []string{`null`, `{}`, `{"data":null}`} {+ for _, warm := range []bool{false, true} {+ t.Run(fmt.Sprintf("%s/%s/%s/warm=%t", kind, stage, body, warm), func(t *testing.T) {+ f := newPBSNamespaceContinuityFixture(t, kind)+ if warm {+ f.collect(t, true, nil)+ }+ before, points := f.live(), f.points(t, "pbs-main")+ key := pbsBackupGroupKey{datastore: "archive", namespace: "tenant", backupType: kind, backupID: "101"}+ cacheAt := f.m.pbsBackupCacheTimeFor("pbs-main", key)+ f.rootAt.Add(15 * 60)+ f.tenantAt.Add(15 * 60) // Require a snapshot read at the existing TTL.+ for range 2 {+ f.collectInventoryResponse(t, stage, body)+ rows := f.live()+ wantCount := 1+ if warm {+ wantCount = 2+ if !reflect.DeepEqual(rows["101"], before["101"]) {+ t.Error("unread file was removed or changed")+ }+ }+ if len(rows) != wantCount || rows["100"].BackupTime.Unix() != f.rootAt.Load() {+ t.Errorf("unread scope invented/lost files or held back its readable sibling: %+v", rows)+ }+ if !f.m.pbsBackupCacheTimeFor("pbs-main", key).Equal(cacheAt) {+ t.Error("missing inventory renewed the snapshot cache")+ }+ for _, prior := range points {+ if prior.Details["namespace"] != "tenant" {+ continue+ }+ found := false+ for _, now := range f.points(t, "pbs-main") {+ found = found || (prior.ID == now.ID && reflect.DeepEqual(prior, now))+ }+ if !found {+ t.Error("unread historical point was removed or its evidence renewed")+ }+ }+ for _, p := range f.postures(t) {+ if p.State == recovery.ProtectionStateProtected || len(p.ProviderStates) != 1 ||+ p.ProviderStates[0].HistoryCompleteness != recovery.ProtectionHistoryPartial ||+ p.ProviderStates[0].Permissions != operationaltrust.EvidencePermissionsUnknown {+ t.Errorf("missing inventory invented complete protection or denied access: %+v", p)+ }+ }+ }+ if f.failedReads.Load() != 2 {+ t.Error("invalid inventory caused an extra request or hid a failed read")+ }+ f.inventoryResponse.Store(nil)+ f.collect(t, true, nil)+ if f.live()["101"].BackupTime.Unix() != f.tenantAt.Load() {+ t.Error("ordinary valid read did not restore inventory")+ }+ for _, p := range f.postures(t) {+ if p.State != recovery.ProtectionStateProtected {+ t.Error("complete ordinary recovery did not restore protection")+ }+ }+ f.rootEmpty.Store(true)+ f.tenantEmpty.Store(true)+ f.collect(t, true, nil)+ if len(f.live()) != 0 || len(f.points(t, "pbs-main")) != 0 {+ t.Error("explicit empty arrays did not reconcile confirmed absence")+ }+ })+ }+ }+ }+ }+}++func TestPBSMissingInventoryArrayKeepsOtherConnection(t *testing.T) {+ f := newPBSNamespaceContinuityFixture(t, "vm")+ f.collect(t, true, nil)+ // Populate a second connection through the same ordinary synthetic reader.+ f.m.pollPBSBackups(context.Background(), "pbs-other", f.client, []models.PBSDatastore{{Name: "archive"}}, nil)+ f.wait(t)+ before := f.m.state.GetSnapshot().PBSBackups+ points := f.points(t, "pbs-other")+ f.collectInventoryResponse(t, "groups", `{"data":null}`)+ var retained []models.PBSBackup+ for _, row := range f.m.state.GetSnapshot().PBSBackups {+ if row.Instance == "pbs-other" {+ retained = append(retained, row)+ }+ }+ var original []models.PBSBackup+ for _, row := range before {+ if row.Instance == "pbs-other" {+ original = append(original, row)+ }+ }+ if len(original) != 1 || !reflect.DeepEqual(original, retained) || !reflect.DeepEqual(points, f.points(t, "pbs-other")) {+ t.Error("invalid inventory altered another connection's files or point evidence")+ }+}++// This normal-suite lifecycle can export its actual poll/ingest/posture results+// for an evidence-only production Backups/API-hook desktop/narrow browser pass.+func TestPBSMissingInventoryArrayConsumerLifecycle(t *testing.T) {+ f := newPBSNamespaceContinuityFixture(t, "vm")+ frames := map[string]any{}+ for _, phase := range []string{"observed", "missing-namespaces", "null-groups", "null-snapshots", "recovered", "confirmed-empty"} {+ switch phase {+ case "missing-namespaces":+ f.rootAt.Add(15 * 60)+ f.collectInventoryResponse(t, "namespace", `{}`)+ case "null-groups":+ f.collectInventoryResponse(t, "groups", `{"data":null}`)+ case "null-snapshots":+ f.tenantAt.Add(15 * 60)+ f.collectInventoryResponse(t, "snapshots", `{"data":null}`)+ default:+ f.inventoryResponse.Store(nil)+ if phase == "confirmed-empty" {+ f.rootEmpty.Store(true)+ f.tenantEmpty.Store(true)+ }+ f.collect(t, true, nil)+ }+ postures := f.postures(t)+ for _, p := range postures {+ healthy := phase == "observed" || phase == "recovered"+ if healthy != (p.State == recovery.ProtectionStateProtected) {+ t.Fatalf("phase %s posture=%+v", phase, p)+ }+ }+ want := 2+ if phase == "confirmed-empty" {+ want = 0+ }+ if len(f.live()) != want || len(f.points(t, "pbs-main")) != want {+ t.Fatalf("phase %s lost or invented backup facts", phase)+ }+ frames[phase] = map[string]any{"pbs": map[string]any{"data": map[string]any{"backups": f.m.state.GetSnapshot().PBSBackups}}, "postures": map[string]any{"data": postures, "policy": recovery.DefaultProtectionPosturePolicy.Payload()}, "points": f.points(t, "pbs-main")}+ }+ if os.Getenv("PULSE_PBS_INVENTORY_FRAMES") != "1" {+ return+ }+ hashes := map[string]string{}+ for _, file := range []string{"monitor_backups.go", "backup_guard.go", "../../pkg/pbs/client.go"} {+ content, err := os.ReadFile(file)+ if err != nil {+ t.Fatal(err)+ }+ hashes[file] = fmt.Sprintf("%x", sha256.Sum256(content))+ }+ data, err := json.Marshal(map[string]any{"runtime_sha256": hashes, "observed_at": time.Now().UTC(), "resource_ids": f.ids, "frames": frames})+ if err != nil {+ t.Fatal(err)+ }+ t.Log("PBS_INVENTORY_FRAMES=" + string(data))+}diff --git a/internal/monitoring/pbs_namespace_continuity_test.go b/internal/monitoring/pbs_namespace_continuity_test.goindex 5285231612..d3cdac5b5d 100644--- a/internal/monitoring/pbs_namespace_continuity_test.go+++ b/internal/monitoring/pbs_namespace_continuity_test.go@@ -27,6 +27,7 @@ type pbsNamespaceContinuityFixture struct {rootReads, tenantReads atomic.Int32tenantSnapshotStatus, namespaceStatus atomic.Int32failedReads atomic.Int32+ inventoryResponse atomic.Pointer[pbsInventoryResponse]}func newPBSNamespaceContinuityFixture(t *testing.T, kind string) *pbsNamespaceContinuityFixture {@@ -35,6 +36,13 @@ func newPBSNamespaceContinuityFixture(t *testing.T, kind string) *pbsNamespaceCof.rootAt.Store(f.backupAt)f.tenantAt.Store(f.backupAt)server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ if response := f.inventoryResponse.Load(); response != nil &&+ r.URL.Path == "/api2/json/admin/datastore/archive/"+response.stage &&+ (response.stage == "namespace" || r.URL.Query().Get("ns") == "tenant") {+ f.failedReads.Add(1)+ _, _ = w.Write([]byte(response.body))+ return+ }id, at, status, empty := "100", f.rootAt.Load(), f.rootStatus.Load(), f.rootEmpty.Load()if r.URL.Query().Get("ns") == "tenant" {id, at, status, empty = "101", f.tenantAt.Load(), f.tenantStatus.Load(), f.tenantEmpty.Load()diff --git a/pkg/pbs/backup_inventory_envelope_test.go b/pkg/pbs/backup_inventory_envelope_test.gonew file mode 100644index 0000000000..68b9084b69--- /dev/null+++ b/pkg/pbs/backup_inventory_envelope_test.go@@ -0,0 +1,56 @@+package pbs++import (+ "context"+ "net/http"+ "testing"+)++// A successful HTTP status without an inventory array is an unread scope, not+// an accepted empty listing. In particular, decoding JSON null is not a parse+// error in Go and must not silently authorize backup-history reconciliation.+func TestBackupInventoryRequiresExplicitArray(t *testing.T) {+ readers := []struct {+ name string+ read func(*Client) (int, error)+ }{+ {"namespaces", func(c *Client) (int, error) {+ rows, err := c.ListNamespaces(context.Background(), "archive", "tenant", 2)+ return len(rows), err+ }},+ {"groups", func(c *Client) (int, error) {+ rows, err := c.ListBackupGroups(context.Background(), "archive", "tenant")+ return len(rows), err+ }},+ {"snapshots", func(c *Client) (int, error) {+ rows, err := c.ListBackupSnapshots(context.Background(), "archive", "tenant", "vm", "101")+ return len(rows), err+ }},+ }+ for _, reader := range readers {+ for _, body := range []string{`null`, `{}`, `{"data":null}`, `{"unrelated":[]}`, `{"data":{}}`, `{"data":false}`, `{"data":`, `{"data":[]}`} {+ t.Run(reader.name+"/"+body, func(t *testing.T) {+ reads := 0+ client, server := newTestClient(t, func(w http.ResponseWriter, r *http.Request) {+ reads+++ if r.Method != http.MethodGet {+ t.Errorf("inventory read made a %s request", r.Method)+ }+ _, _ = w.Write([]byte(body))+ })+ defer server.Close()+ count, err := reader.read(client)+ if body == `{"data":[]}` {+ if err != nil || count != 0 {+ t.Fatalf("explicit empty array rejected: count=%d err=%v", count, err)+ }+ } else if err == nil || count != 0 {+ t.Fatalf("unread scope accepted: count=%d err=%v", count, err)+ }+ if reads != 1 {+ t.Fatalf("inventory admission caused %d requests, want one", reads)+ }+ })+ }+ }+}diff --git a/pkg/pbs/client.go b/pkg/pbs/client.goindex 4118e0501f..28bc0c7469 100644--- a/pkg/pbs/client.go+++ b/pkg/pbs/client.go@@ -1465,6 +1465,8 @@ func (c *Client) getJobConfigMap(ctx context.Context, path string) (map[string]iif err := json.NewDecoder(resp.Body).Decode(&result); err != nil {return nil, err}+ // Match datastore discovery: missing/null data is unread, not an empty+ // scope that can authorize removal of retained backup history.if result.Data == nil {result.Data = map[string]interface{}{}}@@ -1964,6 +1966,9 @@ func (c *Client) ListNamespaces(ctx context.Context, datastore string, parentNamif err := json.NewDecoder(resp.Body).Decode(&result); err != nil {return nil, err}+ if result.Data == nil {+ return nil, fmt.Errorf("namespace list response contains no data array")+ }return result.Data, nil}@@ -2005,6 +2010,9 @@ func (c *Client) ListBackupGroups(ctx context.Context, datastore string, namespaif err := json.NewDecoder(resp.Body).Decode(&result); err != nil {return nil, fmt.Errorf("failed to decode backup groups: %w", err)}+ if result.Data == nil {+ return nil, fmt.Errorf("backup group list response contains no data array")+ }log.Debug().Str("namespace", namespace).@@ -2047,6 +2055,9 @@ func (c *Client) ListBackupSnapshots(ctx context.Context, datastore string, nameif err := json.NewDecoder(resp.Body).Decode(&result); err != nil {return nil, fmt.Errorf("failed to decode snapshots: %w", err)}+ if result.Data == nil {+ return nil, fmt.Errorf("snapshot list response contains no data array")+ }return result.Data, nil}