Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Bound waiting work in Proxmox guest snapshot polling
pulse-triage[bot]1ac1b2f5a094
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.mdindex bbe09d599c..28cd25a709 100644--- a/docs/release-control/v6/internal/subsystems/monitoring.md+++ b/docs/release-control/v6/internal/subsystems/monitoring.md@@ -17,6 +17,29 @@## Purpose+### Guest snapshot collection bounds waiting work as well as provider reads++PVE guest snapshot polling uses at most eight collector workers, plus one+dispatcher and one result closer, independent of the guest count. Both VM and+LXC targets use the same queue; templates and other connections are excluded.+The result channel is bounded to eight entries rather than the estate size.+The existing guest-scaled 60-second-to-four-minute deadline and eight-way read+concurrency are unchanged. Cancellation stops queued work before another+provider call; it does not turn an unattempted guest into an empty listing.++Accepted listings survive later cancellation, while failed, late and queued+listings preserve their exact previous live rows. Complete successful reads+still replace membership normally; independent connection rows, source identity,+alert/recovery evidence rules and connection/runtime publication fences remain+unchanged. No guest operation or extra provider request is added.++`snapshot_poll_workers_test.go` measures suspended collector goroutines and+active reads at 0, 1, 8, 9 and 512 guests, checks exactly-once VM/LXC admission,+template/connection isolation and complete output, and exercises queued+cancellation, deadline expiry, accepted/late listings and ordinary recovery.+Its deterministic synthetic listings measure collector fan-out, not installed+memory, CPU or original guest recovery.+### Sensor units come from native subfeature keysOnly `tempN_input` keys (one or more ASCII digits) supply Celsius in the shareddiff --git a/docs/release-control/v6/internal/subsystems/registry.json b/docs/release-control/v6/internal/subsystems/registry.jsonindex 9984f6cfb6..f193c1b4c9 100644--- a/docs/release-control/v6/internal/subsystems/registry.json+++ b/docs/release-control/v6/internal/subsystems/registry.json@@ -6957,6 +6957,7 @@"internal/monitoring/pve_backup_lifecycle_test.go","internal/monitoring/pve_protection_observation_test.go","internal/monitoring/recovery_ingest_test.go",+ "internal/monitoring/snapshot_poll_workers_test.go","internal/monitoring/vzdump_cached_guest_test.go","internal/monitoring/vzdump_job_log_scheduling_test.go","internal/recovery/mapper/proxmox/mapper_test.go",@@ -7065,6 +7066,7 @@"internal/monitoring/ratetracker_test.go","internal/monitoring/recovery_rollups_resilience_test.go","internal/monitoring/reload_test.go",+ "internal/monitoring/snapshot_poll_workers_test.go","internal/monitoring/truenas_poller_test.go","internal/unifiedresources/code_standards_test.go"]diff --git a/internal/monitoring/monitor_backups.go b/internal/monitoring/monitor_backups.goindex 8e460d9d41..ee20bd9cc4 100644--- a/internal/monitoring/monitor_backups.go+++ b/internal/monitoring/monitor_backups.go@@ -25,6 +25,7 @@ import (const (pbsBackupSnapshotFetchWorkers = 5+ pveGuestSnapshotFetchWorkers = 8// runningVzdumpClockSkew is the slack applied when deciding whether a// backup file's ctime falls inside a running vzdump task's window, so a// small clock offset between Pulse and the node cannot misclassify the@@ -1297,10 +1298,9 @@ func (m *Monitor) pollGuestSnapshots(ctx context.Context, instanceName string, c}const (- minSnapshotTimeout = 60 * time.Second- maxSnapshotTimeout = 4 * time.Minute- snapshotTimeoutPerGuest = 2 * time.Second- maxConcurrentGuestSnapshotPolls = 8+ minSnapshotTimeout = 60 * time.Second+ maxSnapshotTimeout = 4 * time.Minute+ snapshotTimeoutPerGuest = 2 * time.Second)timeout := minSnapshotTimeout@@ -1405,49 +1405,61 @@ func (m *Monitor) pollGuestSnapshots(ctx context.Context, instanceName string, creturn client.GetVMSnapshots(snapshotCtx, target.node, target.vmid)}- results := make(chan guestSnapshotPollResult, len(targets))+ // Bound waiting goroutines as well as active provider reads. A semaphore+ // inside one goroutine per guest still leaves thousands of suspended+ // collectors on a large estate. The dispatcher stops on cancellation, and+ // unattempted targets remain unpolled so their prior listings survive.+ queue := make(chan guestSnapshotPollTarget)+ results := make(chan guestSnapshotPollResult, pveGuestSnapshotFetchWorkers)var wg sync.WaitGroup- sem := make(chan struct{}, maxConcurrentGuestSnapshotPolls)-- for _, target := range targets {- target := target+ for i := 0; i < min(len(targets), pveGuestSnapshotFetchWorkers); i++ {wg.Add(1)go func() {defer wg.Done()-- select {- case sem <- struct{}{}:- case <-snapshotCtx.Done():- results <- guestSnapshotPollResult{target: target, err: snapshotCtx.Err()}- return- }- defer func() { <-sem }()-- snapshots, err := fetchSnapshots(target)- // A provider returning nil error after cancellation did not supply- // an accepted listing. Optional sizing can exhaust its budget later,- // but cannot retroactively invalidate a listing accepted here.- if err == nil {- err = snapshotCtx.Err()- }- result := guestSnapshotPollResult{target: target, err: err}- if err == nil {- result.polled = true- result.snapshots = make([]models.GuestSnapshot, 0, len(snapshots))- for _, snap := range snapshots {- result.snapshots = append(result.snapshots, buildSnapshot(target, snap))+ for target := range queue {+ // Dispatch and cancellation can become ready together. A queued+ // target is not permission to start another expired provider read.+ if snapshotCtx.Err() != nil {+ return+ }+ snapshots, err := fetchSnapshots(target)+ // A provider returning nil error after cancellation did not supply+ // an accepted listing. Optional sizing can exhaust its budget later,+ // but cannot retroactively invalidate a listing accepted here.+ if err == nil {+ err = snapshotCtx.Err()+ }+ result := guestSnapshotPollResult{target: target, err: err}+ if err == nil {+ result.polled = true+ result.snapshots = make([]models.GuestSnapshot, 0, len(snapshots))+ for _, snap := range snapshots {+ result.snapshots = append(result.snapshots, buildSnapshot(target, snap))+ }+ // Only an accepted listing establishes membership, including an+ // empty one. LXC 596/unavailable errors are not deletion evidence;+ // leave failed guests unpolled so their last listing is retained.}- // Only an accepted listing establishes membership, including an- // empty one. LXC 596/unavailable errors are not deletion evidence;- // leave failed guests unpolled so their last listing is retained.+ results <- result}-- results <- result}()}-- wg.Wait()- close(results)+ wg.Add(1)+ go func() {+ defer wg.Done()+ defer close(queue)+ for _, target := range targets {+ select {+ case queue <- target:+ case <-snapshotCtx.Done():+ return+ }+ }+ }()+ go func() {+ wg.Wait()+ close(results)+ }()for result := range results {if result.polled {@@ -1488,6 +1500,9 @@ func (m *Monitor) pollGuestSnapshots(ctx context.Context, instanceName string, cMsg("Failed to get VM snapshots")}}+ // The dispatcher can stop without a result for each unattempted guest.+ // Those targets are unknown, not successful empty listings.+ deadlineExceeded = deadlineExceeded || snapshotCtx.Err() != nil// Retained live rows are continuity, not newly accepted provider evidence.// Keep their count separate before merging so a failed listing cannot renewdiff --git a/internal/monitoring/snapshot_poll_workers_test.go b/internal/monitoring/snapshot_poll_workers_test.gonew file mode 100644index 0000000000..c69e82f5b2--- /dev/null+++ b/internal/monitoring/snapshot_poll_workers_test.go@@ -0,0 +1,217 @@+package monitoring++import (+ "context"+ "fmt"+ "runtime"+ "strings"+ "sync"+ "testing"+ "testing/synctest"+ "time"++ "github.com/rcourtman/pulse-go-rewrite/internal/models"+ "github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"+ "github.com/stretchr/testify/require"+)++// Synthetic API listings only: no guest agent, task operation or installed host.+type snapshotPoolClient struct {+ mockPVEClientExtra+ read func(context.Context, string, int) ([]proxmox.Snapshot, error)+}++func (c *snapshotPoolClient) GetVMSnapshots(ctx context.Context, _ string, id int) ([]proxmox.Snapshot, error) {+ return c.read(ctx, "qemu", id)+}+func (c *snapshotPoolClient) GetContainerSnapshots(ctx context.Context, _ string, id int) ([]proxmox.Snapshot, error) {+ return c.read(ctx, "lxc", id)+}+func (c *snapshotPoolClient) GetStorage(context.Context, string) ([]proxmox.Storage, error) {+ return nil, nil // Optional sizing is not the measurement.+}++func snapshotPoolMonitor(count int) (*Monitor, map[string]models.GuestSnapshot) {+ m := &Monitor{state: models.NewState()}+ var vms []models.VM+ var cts []models.Container+ previous := make(map[string]models.GuestSnapshot, count)+ var snapshots []models.GuestSnapshot+ for i := 0; i < count; i++ {+ id := 1000 + i+ kind := "qemu"+ if i%2 == 0 {+ vms = append(vms, models.VM{ID: fmt.Sprintf("site-node-%d", id), Instance: "site", Node: "node", VMID: id, Name: fmt.Sprintf("vm%d", id)})+ } else {+ kind = "lxc"+ cts = append(cts, models.Container{ID: fmt.Sprintf("site-node-%d", id), Instance: "site", Node: "node", VMID: id, Name: fmt.Sprintf("ct%d", id)})+ }+ snap := models.GuestSnapshot{ID: fmt.Sprintf("site-node-%d-prior", id), Instance: "site", Node: "node", VMID: id, Type: kind, Name: "prior", Time: time.Unix(1700000000, 0), Description: "last accepted listing", SizeBytes: 2048}+ previous[fmt.Sprintf("%s/%d", kind, id)] = snap+ snapshots = append(snapshots, snap)+ }+ // Templates and another connection must never enter this collector's queue.+ vms = append(vms, models.VM{ID: "site-node-1", Instance: "site", Node: "node", VMID: 1, Template: true})+ cts = append(cts, models.Container{ID: "site-node-2", Instance: "site", Node: "node", VMID: 2, Template: true})+ m.state.UpdateVMsForInstance("site", vms)+ m.state.UpdateContainersForInstance("site", cts)+ m.state.UpdateVMsForInstance("peer", []models.VM{{ID: "peer-node-1000", Instance: "peer", Node: "node", VMID: 1000}})+ m.state.UpdateGuestSnapshotsForInstance("site", snapshots)+ m.state.UpdateGuestSnapshotsForInstance("peer", []models.GuestSnapshot{{ID: "peer-prior", Instance: "peer", Node: "node", VMID: 1000, Type: "qemu", Name: "independent", Time: time.Unix(1690000000, 0)}})+ return m, previous+}++// Count only this collector's goroutines, not runtime/GC/testing workers. No+// stacks are logged or retained. Both the old semaphore and new pool are+// measured by the same frame prefix, without relying on compiler func numbers.+func snapshotCollectorGoroutines(t *testing.T) int {+ t.Helper()+ buf := make([]byte, 2<<20)+ n := runtime.Stack(buf, true)+ require.Less(t, n, len(buf), "goroutine measurement was truncated")+ count := 0+ for _, stack := range strings.Split(string(buf[:n]), "\n\n") {+ if strings.Contains(stack, ".(*Monitor).pollGuestSnapshots.func") {+ count+++ }+ }+ return count+}++func TestSnapshotPollWorkersBoundWaitingAndActiveWork(t *testing.T) {+ for _, count := range []int{0, 1, 8, 9, 512} {+ t.Run(fmt.Sprint(count), func(t *testing.T) {+ synctest.Test(t, func(t *testing.T) {+ m, previous := snapshotPoolMonitor(count)+ gate := make(chan struct{})+ var mu sync.Mutex+ calls := make(map[string]int, count)+ active, maxActive := 0, 0+ c := &snapshotPoolClient{read: func(_ context.Context, kind string, id int) ([]proxmox.Snapshot, error) {+ mu.Lock()+ calls[fmt.Sprintf("%s/%d", kind, id)]+++ active+++ maxActive = max(maxActive, active)+ mu.Unlock()+ <-gate+ mu.Lock()+ active--+ mu.Unlock()+ return []proxmox.Snapshot{{Name: "fresh", SnapTime: 1700000100, Description: "accepted"}}, nil+ }}+ before := snapshotCollectorGoroutines(t)+ done := make(chan struct{})+ go func() {+ m.pollGuestSnapshots(context.Background(), "site", c)+ close(done)+ }()+ synctest.Wait()+ waiting := snapshotCollectorGoroutines(t) - before+ // Release even when an assertion fails, so the old-source control+ // can finish and report its real resource-bound failure cleanly.+ close(gate)+ synctest.Wait()+ <-done+ t.Logf("guests=%d collector_goroutines=%d max_active_reads=%d", count, waiting, maxActive)+ require.LessOrEqual(t, waiting, 10, "eight workers plus dispatcher/closer, not one waiting goroutine per guest")+ require.Equal(t, min(count, 8), maxActive, "preserve eight-way provider concurrency")+ require.Len(t, calls, count)+ for key := range previous {+ require.Equal(t, 1, calls[key], "each eligible guest needs exactly one read: %s", key)+ }+ rows := m.state.GetSnapshot().PVEBackups.GuestSnapshots+ require.Len(t, rows, count+1, "fresh target listings plus untouched peer")+ for _, row := range rows {+ if row.Instance == "peer" {+ require.Equal(t, "independent", row.Name)+ continue+ }+ require.Equal(t, "fresh", row.Name)+ require.Equal(t, time.Unix(1700000100, 0), row.Time)+ require.Equal(t, "accepted", row.Description)+ }+ })+ })+ }+}++func TestSnapshotPollWorkersCancelQueuedReadsWithoutLosingAcceptedListings(t *testing.T) {+ for _, stop := range []string{"cancel", "deadline"} {+ t.Run(stop, func(t *testing.T) {+ synctest.Test(t, func(t *testing.T) {+ m, previous := snapshotPoolMonitor(512)+ ctx, cancel := context.WithTimeout(context.Background(), time.Minute)+ defer cancel()+ var mu sync.Mutex+ calls := make(map[string]int)+ accepted := ""+ c := &snapshotPoolClient{read: func(ctx context.Context, kind string, id int) ([]proxmox.Snapshot, error) {+ key := fmt.Sprintf("%s/%d", kind, id)+ mu.Lock()+ calls[key]+++ first := accepted == ""+ if first {+ accepted = key+ }+ mu.Unlock()+ if first {+ return []proxmox.Snapshot{{Name: "accepted-before-stop", SnapTime: 1700000100}}, nil+ }+ <-ctx.Done()+ // A nil-error late body must not masquerade as an accepted+ // result or authorize removal of the last known listing.+ return []proxmox.Snapshot{{Name: "late-unaccepted", SnapTime: 1700000200}}, nil+ }}+ done := make(chan struct{})+ go func() {+ m.pollGuestSnapshots(ctx, "site", c)+ close(done)+ }()+ synctest.Wait()+ beforeStop := len(calls)+ require.Equal(t, 9, beforeStop, "one accepted listing and eight blocked reads")+ if stop == "deadline" {+ time.Sleep(time.Minute)+ } else {+ cancel()+ }+ synctest.Wait()+ <-done+ require.Len(t, calls, beforeStop, "queued guests must not start reads after cancellation")+ rows := m.state.GetSnapshot().PVEBackups.GuestSnapshots+ require.Len(t, rows, len(previous)+1)+ for _, row := range rows {+ if row.Instance == "peer" {+ require.Equal(t, "independent", row.Name)+ continue+ }+ key := fmt.Sprintf("%s/%d", row.Type, row.VMID)+ if key == accepted {+ require.Equal(t, "accepted-before-stop", row.Name)+ require.Equal(t, time.Unix(1700000100, 0), row.Time)+ } else {+ require.Equal(t, previous[key], row, "unread/late listing must preserve exact continuity: %s", key)+ }+ }+ // A later ordinary observation drains every eligible guest. No+ // restart, wider credentials or extra read path is involved.+ c.read = func(_ context.Context, kind string, id int) ([]proxmox.Snapshot, error) {+ mu.Lock()+ calls[fmt.Sprintf("%s/%d", kind, id)]+++ mu.Unlock()+ return []proxmox.Snapshot{{Name: "recovered", SnapTime: 1700000300}}, nil+ }+ m.pollGuestSnapshots(context.Background(), "site", c)+ require.Len(t, calls, len(previous))+ rows = m.state.GetSnapshot().PVEBackups.GuestSnapshots+ require.Len(t, rows, len(previous)+1)+ for _, row := range rows {+ if row.Instance == "site" {+ require.Equal(t, "recovered", row.Name)+ require.Equal(t, time.Unix(1700000300, 0), row.Time)+ }+ }+ })+ })+ }+}