Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Stop expired backup storage scans before retrying more nodes
pulse-triage[bot]307408e0b3ea
diff --git a/internal/monitoring/monitor_backups.go b/internal/monitoring/monitor_backups.goindex 9871605e63..08f53d57ed 100644--- a/internal/monitoring/monitor_backups.go+++ b/internal/monitoring/monitor_backups.go@@ -175,6 +175,9 @@ func (m *Monitor) backupInventoryScopeForAlerts() *alerts.BackupInventoryScope {}func (m *Monitor) pollStorageBackupsWithNodes(ctx context.Context, instanceName string, client PVEClientInterface, nodes []proxmox.Node, nodeEffectiveStatus map[string]string) {+ if ctx.Err() != nil {+ return+ }var allBackups []models.StorageBackupvar pbsConfirmations []models.PBSGuestConfirmation@@ -225,6 +228,9 @@ func (m *Monitor) pollStorageBackupsWithNodes(ctx context.Context, instanceName// For each node, get storage and check contentfor _, node := range nodes {+ if ctx.Err() != nil {+ return+ }if nodeEffectiveStatus[node.Node] != "online" {unreadNodes[node.Node] = struct{}{}for _, storageName := range storageNamesForNode(readState, instanceName, node.Node) {@@ -238,7 +244,13 @@ func (m *Monitor) pollStorageBackupsWithNodes(ctx context.Context, instanceNamevar err errorfor attempt := 1; attempt <= 2; attempt++ {+ if ctx.Err() != nil {+ return+ }storages, err = client.GetStorage(ctx, node.Node)+ if ctx.Err() != nil {+ return+ }if err == nil {break // Success}@@ -250,9 +262,16 @@ func (m *Monitor) pollStorageBackupsWithNodes(ctx context.Context, instanceNamelog.Warn().Str("node", node.Node).Str("instance", instanceName).- Msg("Storage query timed out, retrying with extended timeout...")- // Give it a bit more time on retry- time.Sleep(2 * time.Second)+ Msg("Storage query timed out, retrying within the remaining backup poll budget")+ // Do not add two seconds per remaining node after the+ // shared budget expires or the monitor is stopped.+ timer := time.NewTimer(2 * time.Second)+ select {+ case <-ctx.Done():+ timer.Stop()+ return+ case <-timer.C:+ }continue}}@@ -281,6 +300,9 @@ func (m *Monitor) pollStorageBackupsWithNodes(ctx context.Context, instanceName// For each storage that can contain backups or templatesfor _, storage := range storages {+ if ctx.Err() != nil {+ return+ }// Check if storage supports backup contentif !strings.Contains(storage.Content, "backup") {continue@@ -307,6 +329,9 @@ func (m *Monitor) pollStorageBackupsWithNodes(ctx context.Context, instanceName// Get storage contentcontents, err := client.GetStorageContent(ctx, node.Node, storage.Storage)+ if ctx.Err() != nil {+ return+ }if err != nil {monErr := errors.NewMonitorError(errors.ErrorTypeAPI, "get_storage_content", instanceName, err).WithNode(node.Node)// Check if this is a permission errordiff --git a/internal/monitoring/pve_storage_cancellation_test.go b/internal/monitoring/pve_storage_cancellation_test.gonew file mode 100644index 0000000000..792ba51a70--- /dev/null+++ b/internal/monitoring/pve_storage_cancellation_test.go@@ -0,0 +1,235 @@+package monitoring++import (+ "context"+ "crypto/sha256"+ "encoding/json"+ "fmt"+ "os"+ "reflect"+ "testing"+ "testing/synctest"+ "time"++ "github.com/rcourtman/pulse-go-rewrite/internal/models"+ "github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"+)++type cancellableStorageClient struct {+ mockPVEClientExtra+ storage func(context.Context, string) ([]proxmox.Storage, error)+ content func(context.Context, string, string) ([]proxmox.StorageContent, error)+ calls []string+}++func (c *cancellableStorageClient) GetStorage(ctx context.Context, node string) ([]proxmox.Storage, error) {+ c.calls = append(c.calls, "storage/"+node)+ return c.storage(ctx, node)+}++func (c *cancellableStorageClient) GetStorageContent(ctx context.Context, node, storage string) ([]proxmox.StorageContent, error) {+ c.calls = append(c.calls, "content/"+node+"/"+storage)+ return c.content(ctx, node, storage)+}++func storageCancellationFixture() (*Monitor, []proxmox.Node, map[string]string) {+ m := &Monitor{state: models.NewState(), backupPermissionWarnings: map[string]string{"cluster": "previous access warning"}}+ m.state.UpdateStorageBackupsForInstance("cluster", []models.StorageBackup{{ID: "accepted-file", Instance: "cluster", Storage: "archive", VMID: 101, Time: time.Now().Add(-time.Hour)}})+ m.state.UpdatePBSGuestConfirmationsForInstance("cluster", []models.PBSGuestConfirmation{{Storage: "pbs", VMID: 101, Time: 1000}})+ nodes, online := make([]proxmox.Node, 64), make(map[string]string)+ for i := range nodes {+ nodes[i].Node = fmt.Sprintf("node-%02d", i)+ online[nodes[i].Node] = "online"+ }+ return m, nodes, online+}++func cancellationTestStorages() []proxmox.Storage {+ return []proxmox.Storage{+ {Storage: "archive", Type: "dir", Content: "backup", Active: 1, Enabled: 1},+ {Storage: "archive-two", Type: "dir", Content: "backup", Active: 1, Enabled: 1},+ }+}++// Virtual time advances the unchanged production two-second retry, so this+// catches a cluster-sized deadline overrun without real waits or shorter delays.+func TestPVEStoragePollCancellationStopsRequests(t *testing.T) {+ for _, phase := range []string{"before-poll", "storage-error", "storage-success", "retry-wait", "retry-deadline", "content-error", "content-success", "storage-deadline"} {+ t.Run(phase, func(t *testing.T) {+ synctest.Test(t, func(t *testing.T) {+ m, nodes, online := storageCancellationFixture()+ before := m.state.GetSnapshot().PVEBackups+ confirmations := m.state.PBSGuestConfirmationsForInstance("cluster")+ ctx, cancel := context.WithCancel(context.Background())+ defer cancel()+ wantElapsed, wantCalls := time.Duration(0), 1+ c := &cancellableStorageClient{}+ c.storage = func(ctx context.Context, _ string) ([]proxmox.Storage, error) {+ switch phase {+ case "storage-error":+ cancel()+ return nil, ctx.Err()+ case "storage-success":+ cancel()+ case "retry-wait", "retry-deadline":+ return nil, fmt.Errorf("provider request timeout")+ case "storage-deadline":+ <-ctx.Done()+ return nil, ctx.Err()+ }+ return cancellationTestStorages(), nil+ }+ c.content = func(context.Context, string, string) ([]proxmox.StorageContent, error) {+ cancel()+ if phase == "content-error" {+ return nil, fmt.Errorf("403 forbidden")+ }+ return nil, nil+ }+ switch phase {+ case "before-poll":+ cancel()+ wantCalls = 0+ case "retry-wait":+ go func() { time.Sleep(100 * time.Millisecond); cancel() }()+ wantElapsed = 100 * time.Millisecond+ case "retry-deadline", "storage-deadline":+ cancel()+ ctx, cancel = context.WithTimeout(context.Background(), 100*time.Millisecond)+ defer cancel()+ wantElapsed = 100 * time.Millisecond+ case "content-error", "content-success":+ wantCalls = 2+ }+ started := time.Now()+ m.pollStorageBackupsWithNodes(ctx, "cluster", c, nodes, online)+ if elapsed := time.Since(started); elapsed != wantElapsed || len(c.calls) != wantCalls {+ t.Errorf("cancelled poll kept waiting or issuing reads: elapsed=%s want=%s calls=%d want=%d", elapsed, wantElapsed, len(c.calls), wantCalls)+ }+ if !reflect.DeepEqual(before, m.state.GetSnapshot().PVEBackups) || !reflect.DeepEqual(confirmations, m.state.PBSGuestConfirmationsForInstance("cluster")) || m.backupPermissionWarnings["cluster"] != "previous access warning" {+ t.Error("cancelled poll changed accepted inventory, confirmation or access evidence")+ }+ t.Logf("cancellation phase=%s elapsed=%s requests=%d", phase, time.Since(started), len(c.calls))+ })+ })+ }+}++func TestPVEStoragePollRetryKeepsBoundedPolicy(t *testing.T) {+ for _, tc := range []struct {+ name string+ message string+ recover bool+ retry bool+ }{+ {"timeout-recovers", "request timeout", true, true},+ {"deadline-recovers", "provider deadline exceeded", true, true},+ {"timeout-exhausted", "request timeout", false, true},+ {"permission-no-retry", "403 forbidden", false, false},+ {"transient-no-retry", "503 unavailable", false, false},+ } {+ t.Run(tc.name, func(t *testing.T) {+ synctest.Test(t, func(t *testing.T) {+ m, nodes, online := storageCancellationFixture()+ c := &cancellableStorageClient{}+ c.storage = func(context.Context, string) ([]proxmox.Storage, error) {+ if tc.recover && len(c.calls) == 2 {+ return nil, nil // An accepted empty catalogue is still meaningful.+ }+ return nil, fmt.Errorf("%s", tc.message)+ }+ started := time.Now()+ m.pollStorageBackupsWithNodes(context.Background(), "cluster", c, nodes[:1], online)+ wantCalls, wantElapsed := 1, time.Duration(0)+ if tc.retry {+ wantCalls, wantElapsed = 2, 2*time.Second+ }+ if len(c.calls) != wantCalls || time.Since(started) != wantElapsed {+ t.Fatalf("live retry policy changed: elapsed=%s calls=%v", time.Since(started), c.calls)+ }+ if got := len(m.state.GetSnapshot().PVEBackups.StorageBackups); (tc.recover && got != 0) || (!tc.recover && got != 1) {+ t.Fatalf("accepted-empty/failure preservation changed: live backups=%d", got)+ }+ })+ })+ }+}++type storageBudgetSnapshotClient struct {+ *cancellableStorageClient+ snapshotAt time.Time+ snapshots []proxmox.Snapshot+}++func (c *storageBudgetSnapshotClient) GetBackupTasks(context.Context) ([]proxmox.Task, error) {+ return nil, nil+}++func (c *storageBudgetSnapshotClient) GetVMSnapshots(ctx context.Context, _ string, _ int) ([]proxmox.Snapshot, error) {+ c.snapshotAt = time.Now()+ return c.snapshots, ctx.Err()+}++func (c *storageBudgetSnapshotClient) GetContainerSnapshots(ctx context.Context, node string, vmid int) ([]proxmox.Snapshot, error) {+ return c.GetVMSnapshots(ctx, node, vmid)+}++// Exercise the real task -> storage -> snapshot handoff with 64 nodes, including+// a finite parent budget. The old per-node sleeps exhausted that parent before+// snapshots could start. No guest command, backup operation or database is used.+func TestPVEStorageBudgetHandsOffToSnapshots(t *testing.T) {+ for _, kind := range []string{"qemu", "lxc"} {+ for _, finiteParent := range []bool{false, true} {+ t.Run(fmt.Sprintf("%s/finite-parent=%t", kind, finiteParent), func(t *testing.T) {+ synctest.Test(t, func(t *testing.T) {+ m, nodes, online := storageCancellationFixture()+ if kind == "qemu" {+ m.state.UpdateVMsForInstance("cluster", []models.VM{{ID: "cluster-node-00-101", VMID: 101, Node: "node-00", Instance: "cluster", Name: "guest-101"}})+ } else {+ m.state.UpdateContainersForInstance("cluster", []models.Container{{ID: "cluster-node-00-101", VMID: 101, Node: "node-00", Instance: "cluster", Name: "guest-101"}})+ }+ started := time.Now()+ c := &storageBudgetSnapshotClient{cancellableStorageClient: &cancellableStorageClient{}, snapshots: []proxmox.Snapshot{{Name: "after-storage-timeout", SnapTime: started.Add(-time.Minute).Unix(), Description: "Snapshot listing recovered within its own budget"}}}+ c.storage = func(ctx context.Context, _ string) ([]proxmox.Storage, error) {+ if !c.snapshotAt.IsZero() {+ return nil, nil // Independent snapshot-size lookup, not an expired backup retry.+ }+ <-ctx.Done()+ return nil, ctx.Err()+ }+ parent, cancel := context.WithCancel(context.Background())+ if finiteParent {+ cancel()+ parent, cancel = context.WithTimeout(context.Background(), 5*time.Second)+ }+ defer cancel()+ before := m.state.GetSnapshot().PVEBackups+ m.pollPVEBackupsAndSnapshots(parent, "cluster", c, nodes, online, time.Second)+ synctest.Wait()+ after := m.state.GetSnapshot().PVEBackups+ if elapsed := c.snapshotAt.Sub(started); elapsed != time.Second {+ t.Errorf("snapshot discovery started after %s, want the unchanged 1s backup budget", elapsed)+ }+ if time.Since(started) != time.Second || len(c.calls) != 2 || len(after.GuestSnapshots) != 1 {+ t.Errorf("snapshot handoff lost its bounded independent read: elapsed=%s storage reads=%d snapshots=%d", time.Since(started), len(c.calls), len(after.GuestSnapshots))+ }+ if !reflect.DeepEqual(before.StorageBackups, after.StorageBackups) {+ t.Error("exhausted storage scan replaced its last accepted files")+ }+ t.Logf("snapshot handoff kind=%s finite-parent=%t elapsed=%s storage-reads=%d snapshots=%d", kind, finiteParent, time.Since(started), len(c.calls), len(after.GuestSnapshots))+ if os.Getenv("PULSE_STORAGE_BUDGET_FRAMES") == "1" && finiteParent {+ source, err := os.ReadFile("monitor_backups.go")+ if err != nil {+ t.Fatal(err)+ }+ data, err := json.Marshal(map[string]any{"kind": kind, "runtime_sha256": fmt.Sprintf("%x", sha256.Sum256(source)), "observed_at": time.Now().UTC(), "before": before, "after": after})+ if err != nil {+ t.Fatal(err)+ }+ t.Log("STORAGE_BUDGET_FRAME=" + string(data))+ }+ })+ })+ }+ }+}