Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Preserve accepted storage backup evidence through later poll expiry

pulse-triage[bot]b89e14a8925b
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md
index 28cd25a709..779a3a990e 100644
--- a/docs/release-control/v6/internal/subsystems/monitoring.md
+++ b/docs/release-control/v6/internal/subsystems/monitoring.md
@@ -17,6 +17,33 @@
## Purpose
+### Accepted storage listings survive a later child-budget timeout
+
+PVE storage polling commits listings accepted before a later node, content,
+retry-wait or assignment read exhausts the child budget. A nil-error response
+returned after expiry is unread, not a successful empty listing. No queued
+provider read starts after expiry. Remaining known local repositories and
+unlisted nodes retain their last live rows without renewing durable evidence;
+only accepted rows advance. An accepted empty repository can remove its raw
+rows, but incomplete instance-wide history cannot reconcile durable absence or
+claim complete protection. PBS attribution without node provenance survives an
+unread node until an accepted storage or complete enumeration replaces it.
+
+Explicit request cancellation withdraws the enumeration. Dropping an expired
+child deadline for local publication does not drop the connection owner:
+runtime shutdown/replacement, client replacement and retirement still prevent
+late publication. Independent connections, local-disk identity, permissions,
+alerts, guest operations and provider budgets are unchanged. Every node's
+ordinary storage reads remain; an accepted shared view is not substituted for
+another node's access evidence.
+
+`pve_storage_budget_test.go` covers VM/LXC, warm/cold live inventory, all five
+expiry phases, exact retained/advanced raw and durable fields, accepted empty
+membership, ordinary complete recovery, PBS attribution and owner withdrawal.
+Source-bound desktop/narrow browser proof consumes emitted inventory/posture
+frames through the existing Backups surface; this is not original installed
+or frozen-guest recovery.
+
### Guest snapshot collection bounds waiting work as well as provider reads
PVE guest snapshot polling uses at most eight collector workers, plus one
diff --git a/docs/release-control/v6/internal/subsystems/registry.json b/docs/release-control/v6/internal/subsystems/registry.json
index f193c1b4c9..b4fd0c1578 100644
--- a/docs/release-control/v6/internal/subsystems/registry.json
+++ b/docs/release-control/v6/internal/subsystems/registry.json
@@ -6457,6 +6457,7 @@
"test_prefixes": [],
"exact_files": [
"internal/monitoring/pve_backup_lifecycle_test.go",
+ "internal/monitoring/pve_storage_budget_test.go",
"internal/monitoring/pve_backup_scheduler_test.go"
]
},
@@ -6955,6 +6956,7 @@
"internal/monitoring/monitor_full_coverage_test.go",
"internal/monitoring/pbs_protection_observation_test.go",
"internal/monitoring/pve_backup_lifecycle_test.go",
+ "internal/monitoring/pve_storage_budget_test.go",
"internal/monitoring/pve_protection_observation_test.go",
"internal/monitoring/recovery_ingest_test.go",
"internal/monitoring/snapshot_poll_workers_test.go",
@@ -7061,6 +7063,7 @@
"internal/monitoring/physical_disk_roundtrip_test.go",
"internal/monitoring/proxmox_large_cluster_poll_budget_test.go",
"internal/monitoring/pve_backup_lifecycle_test.go",
+ "internal/monitoring/pve_storage_budget_test.go",
"internal/monitoring/pve_backup_scheduler_test.go",
"internal/monitoring/pve_protection_observation_test.go",
"internal/monitoring/ratetracker_test.go",
diff --git a/docs/release-control/v6/internal/subsystems/storage-recovery.md b/docs/release-control/v6/internal/subsystems/storage-recovery.md
index 16885690ae..93044ceeae 100644
--- a/docs/release-control/v6/internal/subsystems/storage-recovery.md
+++ b/docs/release-control/v6/internal/subsystems/storage-recovery.md
@@ -21,6 +21,33 @@
## Purpose
+### Accepted storage listings survive a later child-budget timeout
+
+PVE storage polling commits listings accepted before a later node, content,
+retry-wait or assignment read exhausts the child budget. A nil-error response
+returned after expiry is unread, not a successful empty listing. No queued
+provider read starts after expiry. Remaining known local repositories and
+unlisted nodes retain their last live rows without renewing durable evidence;
+only accepted rows advance. An accepted empty repository can remove its raw
+rows, but incomplete instance-wide history cannot reconcile durable absence or
+claim complete protection. PBS attribution without node provenance survives an
+unread node until an accepted storage or complete enumeration replaces it.
+
+Explicit request cancellation withdraws the enumeration. Dropping an expired
+child deadline for local publication does not drop the connection owner:
+runtime shutdown/replacement, client replacement and retirement still prevent
+late publication. Independent connections, local-disk identity, permissions,
+alerts, guest operations and provider budgets are unchanged. Every node's
+ordinary storage reads remain; an accepted shared view is not substituted for
+another node's access evidence.
+
+`pve_storage_budget_test.go` covers VM/LXC, warm/cold live inventory, all five
+expiry phases, exact retained/advanced raw and durable fields, accepted empty
+membership, ordinary complete recovery, PBS attribution and owner withdrawal.
+Source-bound desktop/narrow browser proof consumes emitted inventory/posture
+frames through the existing Backups surface; this is not original installed
+or frozen-guest recovery.
+
### Coverage duration requires observed task timing
The existing Coverage task disclosure derives a duration only from positive,
diff --git a/internal/monitoring/monitor_backups.go b/internal/monitoring/monitor_backups.go
index ee20bd9cc4..b3575600f2 100644
--- a/internal/monitoring/monitor_backups.go
+++ b/internal/monitoring/monitor_backups.go
@@ -318,10 +318,9 @@ func (m *Monitor) pollStorageBackupsWithNodes(ctx context.Context, instanceName
// For each node, get storage and check content
for _, node := range nodes {
- if ctx.Err() != nil {
- return
- }
- if nodeEffectiveStatus[node.Node] != "online" {
+ if ctx.Err() != nil || nodeEffectiveStatus[node.Node] != "online" {
+ // A spent child budget is unread scope, not an empty listing.
+ // Finish classifying the remaining nodes without another API call.
unreadNodes[node.Node] = struct{}{}
for _, storageName := range storageNamesForNode(readState, instanceName, node.Node) {
confirmationPreserveNeeded[storageName] = struct{}{}
@@ -333,13 +332,16 @@ func (m *Monitor) pollStorageBackupsWithNodes(ctx context.Context, instanceName
var storages []proxmox.Storage
var err error
+ storageAttempts:
for attempt := 1; attempt <= 2; attempt++ {
if ctx.Err() != nil {
- return
+ err = ctx.Err()
+ break
}
storages, err = client.GetStorage(ctx, node.Node)
if ctx.Err() != nil {
- return
+ err = ctx.Err()
+ break
}
if err == nil {
break // Success
@@ -359,7 +361,8 @@ func (m *Monitor) pollStorageBackupsWithNodes(ctx context.Context, instanceName
select {
case <-ctx.Done():
timer.Stop()
- return
+ err = ctx.Err()
+ break storageAttempts
case <-timer.C:
}
continue
@@ -390,9 +393,6 @@ func (m *Monitor) pollStorageBackupsWithNodes(ctx context.Context, instanceName
// For each storage that can contain backups or templates
for _, storage := range storages {
- if ctx.Err() != nil {
- return
- }
// Check if storage supports backup content
if !strings.Contains(storage.Content, "backup") {
continue
@@ -417,10 +417,15 @@ func (m *Monitor) pollStorageBackupsWithNodes(ctx context.Context, instanceName
continue
}
- // Get storage content
- contents, err := client.GetStorageContent(ctx, node.Node, storage.Storage)
- if ctx.Err() != nil {
- return
+ // Classify every unattempted storage after expiry, but never start a
+ // late read or accept a nil-error response returned after its budget.
+ var contents []proxmox.StorageContent
+ err := ctx.Err()
+ if err == nil {
+ contents, err = client.GetStorageContent(ctx, node.Node, storage.Storage)
+ if ctx.Err() != nil {
+ err = ctx.Err()
+ }
}
if err != nil {
monErr := errors.NewMonitorError(errors.ErrorTypeAPI, "get_storage_content", instanceName, err).WithNode(node.Node)
@@ -579,7 +584,7 @@ func (m *Monitor) pollStorageBackupsWithNodes(ctx context.Context, instanceName
}
}
- if ctx.Err() != nil {
+ if ctx.Err() != nil && ctx.Err() != context.DeadlineExceeded {
return
}
@@ -589,22 +594,27 @@ func (m *Monitor) pollStorageBackupsWithNodes(ctx context.Context, instanceName
// datacenter catalogue, once per poll; failed/missing config is not absence.
storageAssignments := make(map[string]string)
for scope := range skippedLocalStorages {
+ if ctx.Err() != nil {
+ break
+ }
if _, readElsewhere := storageSuccess[scope.storage]; !readElsewhere {
continue
}
- if configured, err := client.GetAllStorage(ctx); err == nil {
+ if configured, err := client.GetAllStorage(ctx); err == nil && ctx.Err() == nil {
for _, storage := range configured {
storageAssignments[storage.Storage] = storage.Nodes
}
}
break
}
- if ctx.Err() != nil {
- // A cancelled enumeration cannot publish a successful empty inventory
- // or renew provider evidence for a payload we did not accept.
+ if ctx.Err() != nil && ctx.Err() != context.DeadlineExceeded {
+ // Explicit cancellation still withdraws the entire enumeration.
return
}
- unlock, current := m.beginPVEBackupPublication(ctx, instanceName)
+ // A later storage/assignment timeout cannot discard earlier accepted
+ // listings. Drop only the spent child budget: the owner value still fences
+ // runtime shutdown, connection replacement and retirement atomically.
+ unlock, current := m.beginPVEBackupPublication(context.WithoutCancel(ctx), instanceName)
if !current {
return
}
@@ -627,6 +637,14 @@ func (m *Monitor) pollStorageBackupsWithNodes(ctx context.Context, instanceName
// Resolve name-wide skips/shared failures after every node, regardless of
// order. A local disk failure remains scoped to its own node below.
+ if len(unreadNodes) > 0 {
+ // Confirmation rows have no node provenance. An unread node cannot
+ // establish their absence, even after its live storage inventory was
+ // replaced. A successful read of that storage below still wins.
+ for _, confirmation := range m.state.PBSGuestConfirmationsForInstance(instanceName) {
+ confirmationPreserveNeeded[confirmation.Storage] = struct{}{}
+ }
+ }
for storageName := range storageSuccess {
delete(storagePreserveNeeded, storageName)
}
diff --git a/internal/monitoring/pve_storage_budget_test.go b/internal/monitoring/pve_storage_budget_test.go
new file mode 100644
index 0000000000..9d2650f918
--- /dev/null
+++ b/internal/monitoring/pve_storage_budget_test.go
@@ -0,0 +1,316 @@
+package monitoring
+
+import (
+ "context"
+ "crypto/sha256"
+ "encoding/json"
+ "fmt"
+ "os"
+ "reflect"
+ "testing"
+ "time"
+
+ "github.com/rcourtman/pulse-go-rewrite/internal/config"
+ "github.com/rcourtman/pulse-go-rewrite/internal/models"
+ "github.com/rcourtman/pulse-go-rewrite/internal/operationaltrust"
+ "github.com/rcourtman/pulse-go-rewrite/internal/recovery"
+ "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
+ "github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"
+)
+
+// Ordinary storage reads only: no guest-agent, backup operation or installed
+// target. Identically named local repositories remain independent node scopes.
+type storageBudgetClient struct {
+ PVEClientInterface
+ at time.Time
+ kind string
+ phase string
+ nodeErr error
+ calls []string
+ emptyFirst bool
+ afterLate func()
+}
+
+func (c *storageBudgetClient) before(ctx context.Context, phase string) {
+ c.calls = append(c.calls, phase)
+ if phase == c.phase {
+ <-ctx.Done()
+ if c.afterLate != nil {
+ c.afterLate()
+ }
+ }
+}
+
+func (c *storageBudgetClient) GetStorage(ctx context.Context, node string) ([]proxmox.Storage, error) {
+ c.before(ctx, "storage/"+node)
+ if node == "node-b" && c.nodeErr != nil {
+ return nil, c.nodeErr
+ }
+ if c.phase == "retry" && node == "node-b" {
+ return nil, fmt.Errorf("storage query timeout")
+ }
+ storages := []proxmox.Storage{{Storage: "local", Type: "dir", Content: "backup", Active: 1, Enabled: 1}}
+ if c.phase == "assignment" && node == "node-b" {
+ storages[0].Active, storages[0].Enabled = 0, 0
+ }
+ if node == "node-a" {
+ storages = append(storages,
+ proxmox.Storage{Storage: "tail", Type: "dir", Content: "backup", Active: 1, Enabled: 1},
+ proxmox.Storage{Storage: "last", Type: "dir", Content: "backup", Active: 1, Enabled: 1})
+ }
+ return storages, nil
+}
+
+func (c *storageBudgetClient) GetStorageContent(ctx context.Context, node, storage string) ([]proxmox.StorageContent, error) {
+ c.before(ctx, "content/"+node+"/"+storage)
+ if c.emptyFirst && node == "node-a" && storage == "local" {
+ return nil, nil
+ }
+ id := map[string]int{"node-a": 101, "node-b": 102, "node-c": 103}[node]
+ return []proxmox.StorageContent{{Volid: fmt.Sprintf("%s:backup/vzdump-%s-%d-fixture.vma.zst", storage, c.kind, id),
+ VMID: id, CTime: c.at.Unix(), Size: 4096, Content: "backup", Format: "vma.zst", Notes: "accepted " + c.at.Format(time.RFC3339)}}, nil
+}
+
+func (c *storageBudgetClient) GetAllStorage(ctx context.Context) ([]proxmox.Storage, error) {
+ c.before(ctx, "assignment")
+ // This late nil-error configuration would incorrectly excuse node-b if
+ // it were accepted after expiry. The unknown local disk must be retained.
+ return []proxmox.Storage{{Storage: "local", Nodes: "node-a,node-c"}}, nil
+}
+
+func newStorageBudgetFixture(t *testing.T, kind string) (*pveReconciliationFixture, *storageBudgetClient) {
+ t.Helper()
+ f := newPVEReconciliationFixture(t)
+ f.nodes = append(f.nodes, proxmox.Node{Node: "node-c"})
+ f.online["node-c"] = "online"
+ f.m.state.UpdateVMs(nil)
+ f.ids = nil
+ for i, node := range []string{"node-a", "node-b", "node-c"} {
+ id := 101 + i
+ resourceType := unifiedresources.ResourceTypeVM
+ if kind == "lxc" {
+ resourceType = unifiedresources.ResourceTypeSystemContainer
+ f.m.state.UpdateContainersForInstance("cluster", append(f.m.state.GetSnapshot().Containers,
+ models.Container{ID: makeGuestID("cluster", node, id), Instance: "cluster", Node: node, VMID: id, Name: fmt.Sprintf("guest-%d", id), Status: "running", Type: "lxc"}))
+ } else {
+ f.m.state.UpdateVMsForInstance("cluster", append(f.m.state.GetSnapshot().VMs,
+ models.VM{ID: makeGuestID("cluster", node, id), Instance: "cluster", Node: node, VMID: id, Name: fmt.Sprintf("guest-%d", id), Status: "running"}))
+ }
+ f.ids = append(f.ids, unifiedresources.ProxmoxGuestCanonicalID(resourceType, "cluster", id))
+ }
+ c := &storageBudgetClient{at: f.client.at, kind: kind}
+ f.m.pveClients = map[string]PVEClientInterface{"cluster": c}
+ f.m.runtimeCtx = context.Background()
+ f.m.state.UpdateStorageBackupsForInstance("other-cluster", []models.StorageBackup{{ID: "peer", Instance: "other-cluster", StorageNode: "node-a", Node: "node-a", Storage: "local", VMID: 101, Type: kind, Time: c.at, CTime: c.at.Unix()}})
+ return f, c
+}
+
+func pollStorageBudget(t *testing.T, f *pveReconciliationFixture, c *storageBudgetClient, ctx context.Context) {
+ t.Helper()
+ f.m.pollStorageBackupsWithNodes(ctx, "cluster", c, f.nodes, f.online)
+ f.wait(t)
+}
+
+func storageBudgetFrame(t *testing.T, f *pveReconciliationFixture) map[string]any {
+ t.Helper()
+ return map[string]any{"pve": map[string]any{"data": f.m.state.GetSnapshot().PVEBackups},
+ "postures": map[string]any{"data": f.postures(t), "policy": recovery.DefaultProtectionPosturePolicy.Payload()},
+ "points": f.points(t)}
+}
+
+func TestPVEStorageBudgetPublishesAcceptedListings(t *testing.T) {
+ for _, kind := range []string{"qemu", "lxc"} {
+ for _, phase := range []string{"content/node-a/tail", "storage/node-b", "content/node-b/local", "retry", "assignment"} {
+ for _, cold := range []bool{false, true} {
+ t.Run(fmt.Sprintf("%s/%s/cold=%t", kind, phase, cold), func(t *testing.T) {
+ f, c := newStorageBudgetFixture(t, kind)
+ pollStorageBudget(t, f, c, context.Background())
+ before := f.points(t)
+ if len(before) != 5 {
+ t.Fatalf("healthy control points=%d, want 5", len(before))
+ }
+ frames := map[string]any{"observed": storageBudgetFrame(t, f)}
+ priorRaw := make(map[string]models.StorageBackup)
+ for _, raw := range f.m.state.GetSnapshot().PVEBackups.StorageBackups {
+ priorRaw[raw.ID] = raw
+ }
+ if cold {
+ f.m.state.UpdateStorageBackupsForInstance("cluster", nil)
+ }
+ c.at = c.at.Add(15 * time.Minute)
+ c.phase, c.calls = phase, nil
+ // The timeout starts only after fixture/storage preparation.
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+ defer cancel()
+ pollStorageBudget(t, f, c, ctx)
+ if ctx.Err() != context.DeadlineExceeded {
+ t.Fatal("fixture did not exhaust the actual child deadline")
+ }
+ if len(c.calls) == 0 || (phase != "retry" && c.calls[len(c.calls)-1] != phase) || (phase == "retry" && c.calls[len(c.calls)-1] != "storage/node-b") {
+ t.Fatalf("queued provider reads started after expiry: %+v", c.calls)
+ }
+ points := f.points(t)
+ if len(points) != len(before) {
+ t.Fatalf("partial poll deleted durable history: got=%d want=%d", len(points), len(before))
+ }
+ accepted := func(raw models.StorageBackup) bool {
+ return raw.StorageNode == "node-a" && (raw.Storage == "local" || phase != "content/node-a/tail") || raw.StorageNode == "node-c" && phase == "assignment"
+ }
+ for id, old := range before {
+ var raw models.StorageBackup
+ for _, candidate := range priorRaw {
+ if "pve-backup:"+candidate.ID == id {
+ raw = candidate
+ break
+ }
+ }
+ if raw.ID == "" {
+ t.Fatalf("missing known point origin for %s", id)
+ }
+ if accepted(raw) {
+ if points[id].CompletedAt == nil || !points[id].CompletedAt.Equal(c.at) || reflect.DeepEqual(old, points[id]) {
+ t.Error("later timeout discarded an earlier accepted archive observation")
+ }
+ } else if !reflect.DeepEqual(old, points[id]) {
+ t.Error("unread/late local disk renewed its prior durable evidence")
+ }
+ }
+ rawRows := f.m.state.GetSnapshot().PVEBackups.StorageBackups
+ expectedRows := 0
+ for _, old := range priorRaw {
+ if old.Instance == "other-cluster" || !cold || accepted(old) {
+ expectedRows++
+ }
+ }
+ if len(rawRows) != expectedRows {
+ t.Errorf("partial raw membership: got %d, want %d", len(rawRows), expectedRows)
+ }
+ for _, raw := range rawRows {
+ old, known := priorRaw[raw.ID]
+ if !known {
+ t.Fatalf("invented raw record: %s", raw.ID)
+ }
+ if raw.Instance == "other-cluster" || !accepted(raw) {
+ if (cold && raw.Instance == "cluster") || !reflect.DeepEqual(raw, old) {
+ t.Error("unread or independent raw record was invented or freshened")
+ }
+ } else {
+ expected := old
+ expected.Time, expected.CTime = c.at, c.at.Unix()
+ expected.Notes = "accepted " + c.at.Format(time.RFC3339)
+ if !reflect.DeepEqual(raw, expected) {
+ t.Error("accepted raw archive did not advance exactly its observed fields")
+ }
+ }
+ }
+ f.assertHistory(t, recovery.ProtectionHistoryPartial, operationaltrust.EvidencePermissionsUnknown)
+ frames["partial"] = storageBudgetFrame(t, f)
+ c.phase, c.calls = "", nil
+ pollStorageBudget(t, f, c, context.Background())
+ for _, point := range f.points(t) {
+ if point.CompletedAt == nil || !point.CompletedAt.Equal(c.at) {
+ t.Error("ordinary later poll did not recover every local disk")
+ }
+ }
+ f.assertHistory(t, recovery.ProtectionHistoryComplete, operationaltrust.EvidencePermissionsSufficient)
+ frames["recovered"] = storageBudgetFrame(t, f)
+ if os.Getenv("PULSE_STORAGE_BUDGET_FRAMES") == "1" && kind == "qemu" && phase == "content/node-b/local" && !cold {
+ source, err := os.ReadFile("monitor_backups.go")
+ if err != nil {
+ t.Fatal(err)
+ }
+ data, err := json.Marshal(map[string]any{"runtime_sha256": fmt.Sprintf("%x", sha256.Sum256(source)), "frames": frames, "resource_ids": f.ids, "vms": f.m.state.GetSnapshot().VMs})
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Log("PVE_STORAGE_BUDGET_FRAME=" + string(data))
+ }
+ })
+ }
+ }
+ }
+}
+
+func TestPVEStorageBudgetAcceptedEmptyDoesNotKeepRawRows(t *testing.T) {
+ f, c := newStorageBudgetFixture(t, "qemu")
+ pollStorageBudget(t, f, c, context.Background())
+ c.phase, c.emptyFirst = "storage/node-b", true
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+ defer cancel()
+ pollStorageBudget(t, f, c, ctx)
+ for _, raw := range f.m.state.GetSnapshot().PVEBackups.StorageBackups {
+ if raw.Instance == "cluster" && raw.StorageNode == "node-a" && raw.Storage == "local" {
+ t.Error("accepted empty scope retained its obsolete raw row")
+ }
+ }
+ if len(f.points(t)) != 5 {
+ t.Error("incomplete instance-wide enumeration reconciled durable absence")
+ }
+ c.phase = ""
+ pollStorageBudget(t, f, c, context.Background())
+ if len(f.points(t)) != 4 {
+ t.Error("ordinary complete enumeration did not reconcile confirmed absence")
+ }
+}
+
+func TestPVEStorageBudgetOwnerWithdrawalStillBlocksPublication(t *testing.T) {
+ for _, transition := range []string{"client", "runtime", "retire"} {
+ t.Run(transition, func(t *testing.T) {
+ f, c := newStorageBudgetFixture(t, "qemu")
+ pollStorageBudget(t, f, c, context.Background())
+ before := f.points(t)
+ owner := f.m.beginPVEBackupPoll("cluster", c, time.Time{}, time.Now())
+ if owner == nil {
+ t.Fatal("missing backup owner")
+ }
+ defer f.m.finishPVEBackupPoll("cluster", owner)
+ ctx, cancel := context.WithTimeout(context.WithValue(owner.ctx, pveBackupPollContextKey{}, owner), time.Second)
+ defer cancel()
+ c.phase, c.at = "content/node-b/local", c.at.Add(15*time.Minute)
+ var expected models.PVEBackups
+ c.afterLate = func() {
+ switch transition {
+ case "client":
+ f.m.replacePVEClient("cluster", &storageBudgetClient{})
+ case "runtime":
+ f.m.setRuntimeContext(context.Background(), nil)
+ case "retire":
+ f.m.retirePVEInstanceRuntime("cluster")
+ }
+ expected = f.m.state.GetSnapshot().PVEBackups
+ }
+ pollStorageBudget(t, f, c, ctx)
+ if !reflect.DeepEqual(expected, f.m.state.GetSnapshot().PVEBackups) || !reflect.DeepEqual(before, f.points(t)) {
+ t.Error("expired scan published after explicit owner withdrawal")
+ }
+ })
+ }
+}
+
+func TestPVEStorageBudgetUnreadNodeKeepsUnscopedPBSConfirmation(t *testing.T) {
+ for _, mode := range []string{"deadline", "denied"} {
+ t.Run(mode, func(t *testing.T) {
+ f, c := newStorageBudgetFixture(t, "qemu")
+ pollStorageBudget(t, f, c, context.Background())
+ f.m.config = &config.Config{PBSInstances: []config.PBSInstance{{Name: "pbs"}}}
+ prior := []models.PBSGuestConfirmation{{Storage: "not-in-live-storage-cache", BackupType: "vm", VMID: 102, Time: c.at.Unix()}}
+ f.m.state.UpdatePBSGuestConfirmationsForInstance("cluster", prior)
+ if mode == "deadline" {
+ c.phase = "storage/node-b"
+ } else {
+ c.nodeErr = fmt.Errorf("403 forbidden")
+ }
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+ defer cancel()
+ pollStorageBudget(t, f, c, ctx)
+ if !reflect.DeepEqual(prior, f.m.state.PBSGuestConfirmationsForInstance("cluster")) {
+ t.Error("unread node erased prior PBS attribution without an accepted listing")
+ }
+ c.phase, c.nodeErr = "", nil
+ pollStorageBudget(t, f, c, context.Background())
+ if len(f.m.state.PBSGuestConfirmationsForInstance("cluster")) != 0 {
+ t.Error("complete read failed to remove absent PBS attribution")
+ }
+ })
+ }
+}