Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Preserve accepted storage backup evidence through later poll expiry
pulse-triage[bot]b89e14a8925b
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.mdindex 28cd25a709..779a3a990e 100644--- a/docs/release-control/v6/internal/subsystems/monitoring.md+++ b/docs/release-control/v6/internal/subsystems/monitoring.md@@ -17,6 +17,33 @@## Purpose+### Accepted storage listings survive a later child-budget timeout++PVE storage polling commits listings accepted before a later node, content,+retry-wait or assignment read exhausts the child budget. A nil-error response+returned after expiry is unread, not a successful empty listing. No queued+provider read starts after expiry. Remaining known local repositories and+unlisted nodes retain their last live rows without renewing durable evidence;+only accepted rows advance. An accepted empty repository can remove its raw+rows, but incomplete instance-wide history cannot reconcile durable absence or+claim complete protection. PBS attribution without node provenance survives an+unread node until an accepted storage or complete enumeration replaces it.++Explicit request cancellation withdraws the enumeration. Dropping an expired+child deadline for local publication does not drop the connection owner:+runtime shutdown/replacement, client replacement and retirement still prevent+late publication. Independent connections, local-disk identity, permissions,+alerts, guest operations and provider budgets are unchanged. Every node's+ordinary storage reads remain; an accepted shared view is not substituted for+another node's access evidence.++`pve_storage_budget_test.go` covers VM/LXC, warm/cold live inventory, all five+expiry phases, exact retained/advanced raw and durable fields, accepted empty+membership, ordinary complete recovery, PBS attribution and owner withdrawal.+Source-bound desktop/narrow browser proof consumes emitted inventory/posture+frames through the existing Backups surface; this is not original installed+or frozen-guest recovery.+### Guest snapshot collection bounds waiting work as well as provider readsPVE guest snapshot polling uses at most eight collector workers, plus onediff --git a/docs/release-control/v6/internal/subsystems/registry.json b/docs/release-control/v6/internal/subsystems/registry.jsonindex f193c1b4c9..b4fd0c1578 100644--- a/docs/release-control/v6/internal/subsystems/registry.json+++ b/docs/release-control/v6/internal/subsystems/registry.json@@ -6457,6 +6457,7 @@"test_prefixes": [],"exact_files": ["internal/monitoring/pve_backup_lifecycle_test.go",+ "internal/monitoring/pve_storage_budget_test.go","internal/monitoring/pve_backup_scheduler_test.go"]},@@ -6955,6 +6956,7 @@"internal/monitoring/monitor_full_coverage_test.go","internal/monitoring/pbs_protection_observation_test.go","internal/monitoring/pve_backup_lifecycle_test.go",+ "internal/monitoring/pve_storage_budget_test.go","internal/monitoring/pve_protection_observation_test.go","internal/monitoring/recovery_ingest_test.go","internal/monitoring/snapshot_poll_workers_test.go",@@ -7061,6 +7063,7 @@"internal/monitoring/physical_disk_roundtrip_test.go","internal/monitoring/proxmox_large_cluster_poll_budget_test.go","internal/monitoring/pve_backup_lifecycle_test.go",+ "internal/monitoring/pve_storage_budget_test.go","internal/monitoring/pve_backup_scheduler_test.go","internal/monitoring/pve_protection_observation_test.go","internal/monitoring/ratetracker_test.go",diff --git a/docs/release-control/v6/internal/subsystems/storage-recovery.md b/docs/release-control/v6/internal/subsystems/storage-recovery.mdindex 16885690ae..93044ceeae 100644--- a/docs/release-control/v6/internal/subsystems/storage-recovery.md+++ b/docs/release-control/v6/internal/subsystems/storage-recovery.md@@ -21,6 +21,33 @@## Purpose+### Accepted storage listings survive a later child-budget timeout++PVE storage polling commits listings accepted before a later node, content,+retry-wait or assignment read exhausts the child budget. A nil-error response+returned after expiry is unread, not a successful empty listing. No queued+provider read starts after expiry. Remaining known local repositories and+unlisted nodes retain their last live rows without renewing durable evidence;+only accepted rows advance. An accepted empty repository can remove its raw+rows, but incomplete instance-wide history cannot reconcile durable absence or+claim complete protection. PBS attribution without node provenance survives an+unread node until an accepted storage or complete enumeration replaces it.++Explicit request cancellation withdraws the enumeration. Dropping an expired+child deadline for local publication does not drop the connection owner:+runtime shutdown/replacement, client replacement and retirement still prevent+late publication. Independent connections, local-disk identity, permissions,+alerts, guest operations and provider budgets are unchanged. Every node's+ordinary storage reads remain; an accepted shared view is not substituted for+another node's access evidence.++`pve_storage_budget_test.go` covers VM/LXC, warm/cold live inventory, all five+expiry phases, exact retained/advanced raw and durable fields, accepted empty+membership, ordinary complete recovery, PBS attribution and owner withdrawal.+Source-bound desktop/narrow browser proof consumes emitted inventory/posture+frames through the existing Backups surface; this is not original installed+or frozen-guest recovery.+### Coverage duration requires observed task timingThe existing Coverage task disclosure derives a duration only from positive,diff --git a/internal/monitoring/monitor_backups.go b/internal/monitoring/monitor_backups.goindex ee20bd9cc4..b3575600f2 100644--- a/internal/monitoring/monitor_backups.go+++ b/internal/monitoring/monitor_backups.go@@ -318,10 +318,9 @@ func (m *Monitor) pollStorageBackupsWithNodes(ctx context.Context, instanceName// For each node, get storage and check contentfor _, node := range nodes {- if ctx.Err() != nil {- return- }- if nodeEffectiveStatus[node.Node] != "online" {+ if ctx.Err() != nil || nodeEffectiveStatus[node.Node] != "online" {+ // A spent child budget is unread scope, not an empty listing.+ // Finish classifying the remaining nodes without another API call.unreadNodes[node.Node] = struct{}{}for _, storageName := range storageNamesForNode(readState, instanceName, node.Node) {confirmationPreserveNeeded[storageName] = struct{}{}@@ -333,13 +332,16 @@ func (m *Monitor) pollStorageBackupsWithNodes(ctx context.Context, instanceNamevar storages []proxmox.Storagevar err error+ storageAttempts:for attempt := 1; attempt <= 2; attempt++ {if ctx.Err() != nil {- return+ err = ctx.Err()+ break}storages, err = client.GetStorage(ctx, node.Node)if ctx.Err() != nil {- return+ err = ctx.Err()+ break}if err == nil {break // Success@@ -359,7 +361,8 @@ func (m *Monitor) pollStorageBackupsWithNodes(ctx context.Context, instanceNameselect {case <-ctx.Done():timer.Stop()- return+ err = ctx.Err()+ break storageAttemptscase <-timer.C:}continue@@ -390,9 +393,6 @@ func (m *Monitor) pollStorageBackupsWithNodes(ctx context.Context, instanceName// For each storage that can contain backups or templatesfor _, storage := range storages {- if ctx.Err() != nil {- return- }// Check if storage supports backup contentif !strings.Contains(storage.Content, "backup") {continue@@ -417,10 +417,15 @@ func (m *Monitor) pollStorageBackupsWithNodes(ctx context.Context, instanceNamecontinue}- // Get storage content- contents, err := client.GetStorageContent(ctx, node.Node, storage.Storage)- if ctx.Err() != nil {- return+ // Classify every unattempted storage after expiry, but never start a+ // late read or accept a nil-error response returned after its budget.+ var contents []proxmox.StorageContent+ err := ctx.Err()+ if err == nil {+ contents, err = client.GetStorageContent(ctx, node.Node, storage.Storage)+ if ctx.Err() != nil {+ err = ctx.Err()+ }}if err != nil {monErr := errors.NewMonitorError(errors.ErrorTypeAPI, "get_storage_content", instanceName, err).WithNode(node.Node)@@ -579,7 +584,7 @@ func (m *Monitor) pollStorageBackupsWithNodes(ctx context.Context, instanceName}}- if ctx.Err() != nil {+ if ctx.Err() != nil && ctx.Err() != context.DeadlineExceeded {return}@@ -589,22 +594,27 @@ func (m *Monitor) pollStorageBackupsWithNodes(ctx context.Context, instanceName// datacenter catalogue, once per poll; failed/missing config is not absence.storageAssignments := make(map[string]string)for scope := range skippedLocalStorages {+ if ctx.Err() != nil {+ break+ }if _, readElsewhere := storageSuccess[scope.storage]; !readElsewhere {continue}- if configured, err := client.GetAllStorage(ctx); err == nil {+ if configured, err := client.GetAllStorage(ctx); err == nil && ctx.Err() == nil {for _, storage := range configured {storageAssignments[storage.Storage] = storage.Nodes}}break}- if ctx.Err() != nil {- // A cancelled enumeration cannot publish a successful empty inventory- // or renew provider evidence for a payload we did not accept.+ if ctx.Err() != nil && ctx.Err() != context.DeadlineExceeded {+ // Explicit cancellation still withdraws the entire enumeration.return}- unlock, current := m.beginPVEBackupPublication(ctx, instanceName)+ // A later storage/assignment timeout cannot discard earlier accepted+ // listings. Drop only the spent child budget: the owner value still fences+ // runtime shutdown, connection replacement and retirement atomically.+ unlock, current := m.beginPVEBackupPublication(context.WithoutCancel(ctx), instanceName)if !current {return}@@ -627,6 +637,14 @@ func (m *Monitor) pollStorageBackupsWithNodes(ctx context.Context, instanceName// Resolve name-wide skips/shared failures after every node, regardless of// order. A local disk failure remains scoped to its own node below.+ if len(unreadNodes) > 0 {+ // Confirmation rows have no node provenance. An unread node cannot+ // establish their absence, even after its live storage inventory was+ // replaced. A successful read of that storage below still wins.+ for _, confirmation := range m.state.PBSGuestConfirmationsForInstance(instanceName) {+ confirmationPreserveNeeded[confirmation.Storage] = struct{}{}+ }+ }for storageName := range storageSuccess {delete(storagePreserveNeeded, storageName)}diff --git a/internal/monitoring/pve_storage_budget_test.go b/internal/monitoring/pve_storage_budget_test.gonew file mode 100644index 0000000000..9d2650f918--- /dev/null+++ b/internal/monitoring/pve_storage_budget_test.go@@ -0,0 +1,316 @@+package monitoring++import (+ "context"+ "crypto/sha256"+ "encoding/json"+ "fmt"+ "os"+ "reflect"+ "testing"+ "time"++ "github.com/rcourtman/pulse-go-rewrite/internal/config"+ "github.com/rcourtman/pulse-go-rewrite/internal/models"+ "github.com/rcourtman/pulse-go-rewrite/internal/operationaltrust"+ "github.com/rcourtman/pulse-go-rewrite/internal/recovery"+ "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"+ "github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"+)++// Ordinary storage reads only: no guest-agent, backup operation or installed+// target. Identically named local repositories remain independent node scopes.+type storageBudgetClient struct {+ PVEClientInterface+ at time.Time+ kind string+ phase string+ nodeErr error+ calls []string+ emptyFirst bool+ afterLate func()+}++func (c *storageBudgetClient) before(ctx context.Context, phase string) {+ c.calls = append(c.calls, phase)+ if phase == c.phase {+ <-ctx.Done()+ if c.afterLate != nil {+ c.afterLate()+ }+ }+}++func (c *storageBudgetClient) GetStorage(ctx context.Context, node string) ([]proxmox.Storage, error) {+ c.before(ctx, "storage/"+node)+ if node == "node-b" && c.nodeErr != nil {+ return nil, c.nodeErr+ }+ if c.phase == "retry" && node == "node-b" {+ return nil, fmt.Errorf("storage query timeout")+ }+ storages := []proxmox.Storage{{Storage: "local", Type: "dir", Content: "backup", Active: 1, Enabled: 1}}+ if c.phase == "assignment" && node == "node-b" {+ storages[0].Active, storages[0].Enabled = 0, 0+ }+ if node == "node-a" {+ storages = append(storages,+ proxmox.Storage{Storage: "tail", Type: "dir", Content: "backup", Active: 1, Enabled: 1},+ proxmox.Storage{Storage: "last", Type: "dir", Content: "backup", Active: 1, Enabled: 1})+ }+ return storages, nil+}++func (c *storageBudgetClient) GetStorageContent(ctx context.Context, node, storage string) ([]proxmox.StorageContent, error) {+ c.before(ctx, "content/"+node+"/"+storage)+ if c.emptyFirst && node == "node-a" && storage == "local" {+ return nil, nil+ }+ id := map[string]int{"node-a": 101, "node-b": 102, "node-c": 103}[node]+ return []proxmox.StorageContent{{Volid: fmt.Sprintf("%s:backup/vzdump-%s-%d-fixture.vma.zst", storage, c.kind, id),+ VMID: id, CTime: c.at.Unix(), Size: 4096, Content: "backup", Format: "vma.zst", Notes: "accepted " + c.at.Format(time.RFC3339)}}, nil+}++func (c *storageBudgetClient) GetAllStorage(ctx context.Context) ([]proxmox.Storage, error) {+ c.before(ctx, "assignment")+ // This late nil-error configuration would incorrectly excuse node-b if+ // it were accepted after expiry. The unknown local disk must be retained.+ return []proxmox.Storage{{Storage: "local", Nodes: "node-a,node-c"}}, nil+}++func newStorageBudgetFixture(t *testing.T, kind string) (*pveReconciliationFixture, *storageBudgetClient) {+ t.Helper()+ f := newPVEReconciliationFixture(t)+ f.nodes = append(f.nodes, proxmox.Node{Node: "node-c"})+ f.online["node-c"] = "online"+ f.m.state.UpdateVMs(nil)+ f.ids = nil+ for i, node := range []string{"node-a", "node-b", "node-c"} {+ id := 101 + i+ resourceType := unifiedresources.ResourceTypeVM+ if kind == "lxc" {+ resourceType = unifiedresources.ResourceTypeSystemContainer+ f.m.state.UpdateContainersForInstance("cluster", append(f.m.state.GetSnapshot().Containers,+ models.Container{ID: makeGuestID("cluster", node, id), Instance: "cluster", Node: node, VMID: id, Name: fmt.Sprintf("guest-%d", id), Status: "running", Type: "lxc"}))+ } else {+ f.m.state.UpdateVMsForInstance("cluster", append(f.m.state.GetSnapshot().VMs,+ models.VM{ID: makeGuestID("cluster", node, id), Instance: "cluster", Node: node, VMID: id, Name: fmt.Sprintf("guest-%d", id), Status: "running"}))+ }+ f.ids = append(f.ids, unifiedresources.ProxmoxGuestCanonicalID(resourceType, "cluster", id))+ }+ c := &storageBudgetClient{at: f.client.at, kind: kind}+ f.m.pveClients = map[string]PVEClientInterface{"cluster": c}+ f.m.runtimeCtx = context.Background()+ f.m.state.UpdateStorageBackupsForInstance("other-cluster", []models.StorageBackup{{ID: "peer", Instance: "other-cluster", StorageNode: "node-a", Node: "node-a", Storage: "local", VMID: 101, Type: kind, Time: c.at, CTime: c.at.Unix()}})+ return f, c+}++func pollStorageBudget(t *testing.T, f *pveReconciliationFixture, c *storageBudgetClient, ctx context.Context) {+ t.Helper()+ f.m.pollStorageBackupsWithNodes(ctx, "cluster", c, f.nodes, f.online)+ f.wait(t)+}++func storageBudgetFrame(t *testing.T, f *pveReconciliationFixture) map[string]any {+ t.Helper()+ return map[string]any{"pve": map[string]any{"data": f.m.state.GetSnapshot().PVEBackups},+ "postures": map[string]any{"data": f.postures(t), "policy": recovery.DefaultProtectionPosturePolicy.Payload()},+ "points": f.points(t)}+}++func TestPVEStorageBudgetPublishesAcceptedListings(t *testing.T) {+ for _, kind := range []string{"qemu", "lxc"} {+ for _, phase := range []string{"content/node-a/tail", "storage/node-b", "content/node-b/local", "retry", "assignment"} {+ for _, cold := range []bool{false, true} {+ t.Run(fmt.Sprintf("%s/%s/cold=%t", kind, phase, cold), func(t *testing.T) {+ f, c := newStorageBudgetFixture(t, kind)+ pollStorageBudget(t, f, c, context.Background())+ before := f.points(t)+ if len(before) != 5 {+ t.Fatalf("healthy control points=%d, want 5", len(before))+ }+ frames := map[string]any{"observed": storageBudgetFrame(t, f)}+ priorRaw := make(map[string]models.StorageBackup)+ for _, raw := range f.m.state.GetSnapshot().PVEBackups.StorageBackups {+ priorRaw[raw.ID] = raw+ }+ if cold {+ f.m.state.UpdateStorageBackupsForInstance("cluster", nil)+ }+ c.at = c.at.Add(15 * time.Minute)+ c.phase, c.calls = phase, nil+ // The timeout starts only after fixture/storage preparation.+ ctx, cancel := context.WithTimeout(context.Background(), time.Second)+ defer cancel()+ pollStorageBudget(t, f, c, ctx)+ if ctx.Err() != context.DeadlineExceeded {+ t.Fatal("fixture did not exhaust the actual child deadline")+ }+ if len(c.calls) == 0 || (phase != "retry" && c.calls[len(c.calls)-1] != phase) || (phase == "retry" && c.calls[len(c.calls)-1] != "storage/node-b") {+ t.Fatalf("queued provider reads started after expiry: %+v", c.calls)+ }+ points := f.points(t)+ if len(points) != len(before) {+ t.Fatalf("partial poll deleted durable history: got=%d want=%d", len(points), len(before))+ }+ accepted := func(raw models.StorageBackup) bool {+ return raw.StorageNode == "node-a" && (raw.Storage == "local" || phase != "content/node-a/tail") || raw.StorageNode == "node-c" && phase == "assignment"+ }+ for id, old := range before {+ var raw models.StorageBackup+ for _, candidate := range priorRaw {+ if "pve-backup:"+candidate.ID == id {+ raw = candidate+ break+ }+ }+ if raw.ID == "" {+ t.Fatalf("missing known point origin for %s", id)+ }+ if accepted(raw) {+ if points[id].CompletedAt == nil || !points[id].CompletedAt.Equal(c.at) || reflect.DeepEqual(old, points[id]) {+ t.Error("later timeout discarded an earlier accepted archive observation")+ }+ } else if !reflect.DeepEqual(old, points[id]) {+ t.Error("unread/late local disk renewed its prior durable evidence")+ }+ }+ rawRows := f.m.state.GetSnapshot().PVEBackups.StorageBackups+ expectedRows := 0+ for _, old := range priorRaw {+ if old.Instance == "other-cluster" || !cold || accepted(old) {+ expectedRows+++ }+ }+ if len(rawRows) != expectedRows {+ t.Errorf("partial raw membership: got %d, want %d", len(rawRows), expectedRows)+ }+ for _, raw := range rawRows {+ old, known := priorRaw[raw.ID]+ if !known {+ t.Fatalf("invented raw record: %s", raw.ID)+ }+ if raw.Instance == "other-cluster" || !accepted(raw) {+ if (cold && raw.Instance == "cluster") || !reflect.DeepEqual(raw, old) {+ t.Error("unread or independent raw record was invented or freshened")+ }+ } else {+ expected := old+ expected.Time, expected.CTime = c.at, c.at.Unix()+ expected.Notes = "accepted " + c.at.Format(time.RFC3339)+ if !reflect.DeepEqual(raw, expected) {+ t.Error("accepted raw archive did not advance exactly its observed fields")+ }+ }+ }+ f.assertHistory(t, recovery.ProtectionHistoryPartial, operationaltrust.EvidencePermissionsUnknown)+ frames["partial"] = storageBudgetFrame(t, f)+ c.phase, c.calls = "", nil+ pollStorageBudget(t, f, c, context.Background())+ for _, point := range f.points(t) {+ if point.CompletedAt == nil || !point.CompletedAt.Equal(c.at) {+ t.Error("ordinary later poll did not recover every local disk")+ }+ }+ f.assertHistory(t, recovery.ProtectionHistoryComplete, operationaltrust.EvidencePermissionsSufficient)+ frames["recovered"] = storageBudgetFrame(t, f)+ if os.Getenv("PULSE_STORAGE_BUDGET_FRAMES") == "1" && kind == "qemu" && phase == "content/node-b/local" && !cold {+ source, err := os.ReadFile("monitor_backups.go")+ if err != nil {+ t.Fatal(err)+ }+ data, err := json.Marshal(map[string]any{"runtime_sha256": fmt.Sprintf("%x", sha256.Sum256(source)), "frames": frames, "resource_ids": f.ids, "vms": f.m.state.GetSnapshot().VMs})+ if err != nil {+ t.Fatal(err)+ }+ t.Log("PVE_STORAGE_BUDGET_FRAME=" + string(data))+ }+ })+ }+ }+ }+}++func TestPVEStorageBudgetAcceptedEmptyDoesNotKeepRawRows(t *testing.T) {+ f, c := newStorageBudgetFixture(t, "qemu")+ pollStorageBudget(t, f, c, context.Background())+ c.phase, c.emptyFirst = "storage/node-b", true+ ctx, cancel := context.WithTimeout(context.Background(), time.Second)+ defer cancel()+ pollStorageBudget(t, f, c, ctx)+ for _, raw := range f.m.state.GetSnapshot().PVEBackups.StorageBackups {+ if raw.Instance == "cluster" && raw.StorageNode == "node-a" && raw.Storage == "local" {+ t.Error("accepted empty scope retained its obsolete raw row")+ }+ }+ if len(f.points(t)) != 5 {+ t.Error("incomplete instance-wide enumeration reconciled durable absence")+ }+ c.phase = ""+ pollStorageBudget(t, f, c, context.Background())+ if len(f.points(t)) != 4 {+ t.Error("ordinary complete enumeration did not reconcile confirmed absence")+ }+}++func TestPVEStorageBudgetOwnerWithdrawalStillBlocksPublication(t *testing.T) {+ for _, transition := range []string{"client", "runtime", "retire"} {+ t.Run(transition, func(t *testing.T) {+ f, c := newStorageBudgetFixture(t, "qemu")+ pollStorageBudget(t, f, c, context.Background())+ before := f.points(t)+ owner := f.m.beginPVEBackupPoll("cluster", c, time.Time{}, time.Now())+ if owner == nil {+ t.Fatal("missing backup owner")+ }+ defer f.m.finishPVEBackupPoll("cluster", owner)+ ctx, cancel := context.WithTimeout(context.WithValue(owner.ctx, pveBackupPollContextKey{}, owner), time.Second)+ defer cancel()+ c.phase, c.at = "content/node-b/local", c.at.Add(15*time.Minute)+ var expected models.PVEBackups+ c.afterLate = func() {+ switch transition {+ case "client":+ f.m.replacePVEClient("cluster", &storageBudgetClient{})+ case "runtime":+ f.m.setRuntimeContext(context.Background(), nil)+ case "retire":+ f.m.retirePVEInstanceRuntime("cluster")+ }+ expected = f.m.state.GetSnapshot().PVEBackups+ }+ pollStorageBudget(t, f, c, ctx)+ if !reflect.DeepEqual(expected, f.m.state.GetSnapshot().PVEBackups) || !reflect.DeepEqual(before, f.points(t)) {+ t.Error("expired scan published after explicit owner withdrawal")+ }+ })+ }+}++func TestPVEStorageBudgetUnreadNodeKeepsUnscopedPBSConfirmation(t *testing.T) {+ for _, mode := range []string{"deadline", "denied"} {+ t.Run(mode, func(t *testing.T) {+ f, c := newStorageBudgetFixture(t, "qemu")+ pollStorageBudget(t, f, c, context.Background())+ f.m.config = &config.Config{PBSInstances: []config.PBSInstance{{Name: "pbs"}}}+ prior := []models.PBSGuestConfirmation{{Storage: "not-in-live-storage-cache", BackupType: "vm", VMID: 102, Time: c.at.Unix()}}+ f.m.state.UpdatePBSGuestConfirmationsForInstance("cluster", prior)+ if mode == "deadline" {+ c.phase = "storage/node-b"+ } else {+ c.nodeErr = fmt.Errorf("403 forbidden")+ }+ ctx, cancel := context.WithTimeout(context.Background(), time.Second)+ defer cancel()+ pollStorageBudget(t, f, c, ctx)+ if !reflect.DeepEqual(prior, f.m.state.PBSGuestConfirmationsForInstance("cluster")) {+ t.Error("unread node erased prior PBS attribution without an accepted listing")+ }+ c.phase, c.nodeErr = "", nil+ pollStorageBudget(t, f, c, context.Background())+ if len(f.m.state.PBSGuestConfirmationsForInstance("cluster")) != 0 {+ t.Error("complete read failed to remove absent PBS attribution")+ }+ })+ }+}