Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Bound waiting work in Proxmox guest snapshot polling

pulse-triage[bot]1ac1b2f5a094
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md
index bbe09d599c..28cd25a709 100644
--- a/docs/release-control/v6/internal/subsystems/monitoring.md
+++ b/docs/release-control/v6/internal/subsystems/monitoring.md
@@ -17,6 +17,29 @@
## Purpose
+### Guest snapshot collection bounds waiting work as well as provider reads
+
+PVE guest snapshot polling uses at most eight collector workers, plus one
+dispatcher and one result closer, independent of the guest count. Both VM and
+LXC targets use the same queue; templates and other connections are excluded.
+The result channel is bounded to eight entries rather than the estate size.
+The existing guest-scaled 60-second-to-four-minute deadline and eight-way read
+concurrency are unchanged. Cancellation stops queued work before another
+provider call; it does not turn an unattempted guest into an empty listing.
+
+Accepted listings survive later cancellation, while failed, late and queued
+listings preserve their exact previous live rows. Complete successful reads
+still replace membership normally; independent connection rows, source identity,
+alert/recovery evidence rules and connection/runtime publication fences remain
+unchanged. No guest operation or extra provider request is added.
+
+`snapshot_poll_workers_test.go` measures suspended collector goroutines and
+active reads at 0, 1, 8, 9 and 512 guests, checks exactly-once VM/LXC admission,
+template/connection isolation and complete output, and exercises queued
+cancellation, deadline expiry, accepted/late listings and ordinary recovery.
+Its deterministic synthetic listings measure collector fan-out, not installed
+memory, CPU or original guest recovery.
+
### Sensor units come from native subfeature keys
Only `tempN_input` keys (one or more ASCII digits) supply Celsius in the shared
diff --git a/docs/release-control/v6/internal/subsystems/registry.json b/docs/release-control/v6/internal/subsystems/registry.json
index 9984f6cfb6..f193c1b4c9 100644
--- a/docs/release-control/v6/internal/subsystems/registry.json
+++ b/docs/release-control/v6/internal/subsystems/registry.json
@@ -6957,6 +6957,7 @@
"internal/monitoring/pve_backup_lifecycle_test.go",
"internal/monitoring/pve_protection_observation_test.go",
"internal/monitoring/recovery_ingest_test.go",
+ "internal/monitoring/snapshot_poll_workers_test.go",
"internal/monitoring/vzdump_cached_guest_test.go",
"internal/monitoring/vzdump_job_log_scheduling_test.go",
"internal/recovery/mapper/proxmox/mapper_test.go",
@@ -7065,6 +7066,7 @@
"internal/monitoring/ratetracker_test.go",
"internal/monitoring/recovery_rollups_resilience_test.go",
"internal/monitoring/reload_test.go",
+ "internal/monitoring/snapshot_poll_workers_test.go",
"internal/monitoring/truenas_poller_test.go",
"internal/unifiedresources/code_standards_test.go"
]
diff --git a/internal/monitoring/monitor_backups.go b/internal/monitoring/monitor_backups.go
index 8e460d9d41..ee20bd9cc4 100644
--- a/internal/monitoring/monitor_backups.go
+++ b/internal/monitoring/monitor_backups.go
@@ -25,6 +25,7 @@ import (
const (
pbsBackupSnapshotFetchWorkers = 5
+ pveGuestSnapshotFetchWorkers = 8
// runningVzdumpClockSkew is the slack applied when deciding whether a
// backup file's ctime falls inside a running vzdump task's window, so a
// small clock offset between Pulse and the node cannot misclassify the
@@ -1297,10 +1298,9 @@ func (m *Monitor) pollGuestSnapshots(ctx context.Context, instanceName string, c
}
const (
- minSnapshotTimeout = 60 * time.Second
- maxSnapshotTimeout = 4 * time.Minute
- snapshotTimeoutPerGuest = 2 * time.Second
- maxConcurrentGuestSnapshotPolls = 8
+ minSnapshotTimeout = 60 * time.Second
+ maxSnapshotTimeout = 4 * time.Minute
+ snapshotTimeoutPerGuest = 2 * time.Second
)
timeout := minSnapshotTimeout
@@ -1405,49 +1405,61 @@ func (m *Monitor) pollGuestSnapshots(ctx context.Context, instanceName string, c
return client.GetVMSnapshots(snapshotCtx, target.node, target.vmid)
}
- results := make(chan guestSnapshotPollResult, len(targets))
+ // Bound waiting goroutines as well as active provider reads. A semaphore
+ // inside one goroutine per guest still leaves thousands of suspended
+ // collectors on a large estate. The dispatcher stops on cancellation, and
+ // unattempted targets remain unpolled so their prior listings survive.
+ queue := make(chan guestSnapshotPollTarget)
+ results := make(chan guestSnapshotPollResult, pveGuestSnapshotFetchWorkers)
var wg sync.WaitGroup
- sem := make(chan struct{}, maxConcurrentGuestSnapshotPolls)
-
- for _, target := range targets {
- target := target
+ for i := 0; i < min(len(targets), pveGuestSnapshotFetchWorkers); i++ {
wg.Add(1)
go func() {
defer wg.Done()
-
- select {
- case sem <- struct{}{}:
- case <-snapshotCtx.Done():
- results <- guestSnapshotPollResult{target: target, err: snapshotCtx.Err()}
- return
- }
- defer func() { <-sem }()
-
- snapshots, err := fetchSnapshots(target)
- // A provider returning nil error after cancellation did not supply
- // an accepted listing. Optional sizing can exhaust its budget later,
- // but cannot retroactively invalidate a listing accepted here.
- if err == nil {
- err = snapshotCtx.Err()
- }
- result := guestSnapshotPollResult{target: target, err: err}
- if err == nil {
- result.polled = true
- result.snapshots = make([]models.GuestSnapshot, 0, len(snapshots))
- for _, snap := range snapshots {
- result.snapshots = append(result.snapshots, buildSnapshot(target, snap))
+ for target := range queue {
+ // Dispatch and cancellation can become ready together. A queued
+ // target is not permission to start another expired provider read.
+ if snapshotCtx.Err() != nil {
+ return
+ }
+ snapshots, err := fetchSnapshots(target)
+ // A provider returning nil error after cancellation did not supply
+ // an accepted listing. Optional sizing can exhaust its budget later,
+ // but cannot retroactively invalidate a listing accepted here.
+ if err == nil {
+ err = snapshotCtx.Err()
+ }
+ result := guestSnapshotPollResult{target: target, err: err}
+ if err == nil {
+ result.polled = true
+ result.snapshots = make([]models.GuestSnapshot, 0, len(snapshots))
+ for _, snap := range snapshots {
+ result.snapshots = append(result.snapshots, buildSnapshot(target, snap))
+ }
+ // Only an accepted listing establishes membership, including an
+ // empty one. LXC 596/unavailable errors are not deletion evidence;
+ // leave failed guests unpolled so their last listing is retained.
}
- // Only an accepted listing establishes membership, including an
- // empty one. LXC 596/unavailable errors are not deletion evidence;
- // leave failed guests unpolled so their last listing is retained.
+ results <- result
}
-
- results <- result
}()
}
-
- wg.Wait()
- close(results)
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ defer close(queue)
+ for _, target := range targets {
+ select {
+ case queue <- target:
+ case <-snapshotCtx.Done():
+ return
+ }
+ }
+ }()
+ go func() {
+ wg.Wait()
+ close(results)
+ }()
for result := range results {
if result.polled {
@@ -1488,6 +1500,9 @@ func (m *Monitor) pollGuestSnapshots(ctx context.Context, instanceName string, c
Msg("Failed to get VM snapshots")
}
}
+ // The dispatcher can stop without a result for each unattempted guest.
+ // Those targets are unknown, not successful empty listings.
+ deadlineExceeded = deadlineExceeded || snapshotCtx.Err() != nil
// Retained live rows are continuity, not newly accepted provider evidence.
// Keep their count separate before merging so a failed listing cannot renew
diff --git a/internal/monitoring/snapshot_poll_workers_test.go b/internal/monitoring/snapshot_poll_workers_test.go
new file mode 100644
index 0000000000..c69e82f5b2
--- /dev/null
+++ b/internal/monitoring/snapshot_poll_workers_test.go
@@ -0,0 +1,217 @@
+package monitoring
+
+import (
+ "context"
+ "fmt"
+ "runtime"
+ "strings"
+ "sync"
+ "testing"
+ "testing/synctest"
+ "time"
+
+ "github.com/rcourtman/pulse-go-rewrite/internal/models"
+ "github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"
+ "github.com/stretchr/testify/require"
+)
+
+// Synthetic API listings only: no guest agent, task operation or installed host.
+type snapshotPoolClient struct {
+ mockPVEClientExtra
+ read func(context.Context, string, int) ([]proxmox.Snapshot, error)
+}
+
+func (c *snapshotPoolClient) GetVMSnapshots(ctx context.Context, _ string, id int) ([]proxmox.Snapshot, error) {
+ return c.read(ctx, "qemu", id)
+}
+func (c *snapshotPoolClient) GetContainerSnapshots(ctx context.Context, _ string, id int) ([]proxmox.Snapshot, error) {
+ return c.read(ctx, "lxc", id)
+}
+func (c *snapshotPoolClient) GetStorage(context.Context, string) ([]proxmox.Storage, error) {
+ return nil, nil // Optional sizing is not the measurement.
+}
+
+func snapshotPoolMonitor(count int) (*Monitor, map[string]models.GuestSnapshot) {
+ m := &Monitor{state: models.NewState()}
+ var vms []models.VM
+ var cts []models.Container
+ previous := make(map[string]models.GuestSnapshot, count)
+ var snapshots []models.GuestSnapshot
+ for i := 0; i < count; i++ {
+ id := 1000 + i
+ kind := "qemu"
+ if i%2 == 0 {
+ vms = append(vms, models.VM{ID: fmt.Sprintf("site-node-%d", id), Instance: "site", Node: "node", VMID: id, Name: fmt.Sprintf("vm%d", id)})
+ } else {
+ kind = "lxc"
+ cts = append(cts, models.Container{ID: fmt.Sprintf("site-node-%d", id), Instance: "site", Node: "node", VMID: id, Name: fmt.Sprintf("ct%d", id)})
+ }
+ snap := models.GuestSnapshot{ID: fmt.Sprintf("site-node-%d-prior", id), Instance: "site", Node: "node", VMID: id, Type: kind, Name: "prior", Time: time.Unix(1700000000, 0), Description: "last accepted listing", SizeBytes: 2048}
+ previous[fmt.Sprintf("%s/%d", kind, id)] = snap
+ snapshots = append(snapshots, snap)
+ }
+ // Templates and another connection must never enter this collector's queue.
+ vms = append(vms, models.VM{ID: "site-node-1", Instance: "site", Node: "node", VMID: 1, Template: true})
+ cts = append(cts, models.Container{ID: "site-node-2", Instance: "site", Node: "node", VMID: 2, Template: true})
+ m.state.UpdateVMsForInstance("site", vms)
+ m.state.UpdateContainersForInstance("site", cts)
+ m.state.UpdateVMsForInstance("peer", []models.VM{{ID: "peer-node-1000", Instance: "peer", Node: "node", VMID: 1000}})
+ m.state.UpdateGuestSnapshotsForInstance("site", snapshots)
+ m.state.UpdateGuestSnapshotsForInstance("peer", []models.GuestSnapshot{{ID: "peer-prior", Instance: "peer", Node: "node", VMID: 1000, Type: "qemu", Name: "independent", Time: time.Unix(1690000000, 0)}})
+ return m, previous
+}
+
+// Count only this collector's goroutines, not runtime/GC/testing workers. No
+// stacks are logged or retained. Both the old semaphore and new pool are
+// measured by the same frame prefix, without relying on compiler func numbers.
+func snapshotCollectorGoroutines(t *testing.T) int {
+ t.Helper()
+ buf := make([]byte, 2<<20)
+ n := runtime.Stack(buf, true)
+ require.Less(t, n, len(buf), "goroutine measurement was truncated")
+ count := 0
+ for _, stack := range strings.Split(string(buf[:n]), "\n\n") {
+ if strings.Contains(stack, ".(*Monitor).pollGuestSnapshots.func") {
+ count++
+ }
+ }
+ return count
+}
+
+func TestSnapshotPollWorkersBoundWaitingAndActiveWork(t *testing.T) {
+ for _, count := range []int{0, 1, 8, 9, 512} {
+ t.Run(fmt.Sprint(count), func(t *testing.T) {
+ synctest.Test(t, func(t *testing.T) {
+ m, previous := snapshotPoolMonitor(count)
+ gate := make(chan struct{})
+ var mu sync.Mutex
+ calls := make(map[string]int, count)
+ active, maxActive := 0, 0
+ c := &snapshotPoolClient{read: func(_ context.Context, kind string, id int) ([]proxmox.Snapshot, error) {
+ mu.Lock()
+ calls[fmt.Sprintf("%s/%d", kind, id)]++
+ active++
+ maxActive = max(maxActive, active)
+ mu.Unlock()
+ <-gate
+ mu.Lock()
+ active--
+ mu.Unlock()
+ return []proxmox.Snapshot{{Name: "fresh", SnapTime: 1700000100, Description: "accepted"}}, nil
+ }}
+ before := snapshotCollectorGoroutines(t)
+ done := make(chan struct{})
+ go func() {
+ m.pollGuestSnapshots(context.Background(), "site", c)
+ close(done)
+ }()
+ synctest.Wait()
+ waiting := snapshotCollectorGoroutines(t) - before
+ // Release even when an assertion fails, so the old-source control
+ // can finish and report its real resource-bound failure cleanly.
+ close(gate)
+ synctest.Wait()
+ <-done
+ t.Logf("guests=%d collector_goroutines=%d max_active_reads=%d", count, waiting, maxActive)
+ require.LessOrEqual(t, waiting, 10, "eight workers plus dispatcher/closer, not one waiting goroutine per guest")
+ require.Equal(t, min(count, 8), maxActive, "preserve eight-way provider concurrency")
+ require.Len(t, calls, count)
+ for key := range previous {
+ require.Equal(t, 1, calls[key], "each eligible guest needs exactly one read: %s", key)
+ }
+ rows := m.state.GetSnapshot().PVEBackups.GuestSnapshots
+ require.Len(t, rows, count+1, "fresh target listings plus untouched peer")
+ for _, row := range rows {
+ if row.Instance == "peer" {
+ require.Equal(t, "independent", row.Name)
+ continue
+ }
+ require.Equal(t, "fresh", row.Name)
+ require.Equal(t, time.Unix(1700000100, 0), row.Time)
+ require.Equal(t, "accepted", row.Description)
+ }
+ })
+ })
+ }
+}
+
+func TestSnapshotPollWorkersCancelQueuedReadsWithoutLosingAcceptedListings(t *testing.T) {
+ for _, stop := range []string{"cancel", "deadline"} {
+ t.Run(stop, func(t *testing.T) {
+ synctest.Test(t, func(t *testing.T) {
+ m, previous := snapshotPoolMonitor(512)
+ ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
+ defer cancel()
+ var mu sync.Mutex
+ calls := make(map[string]int)
+ accepted := ""
+ c := &snapshotPoolClient{read: func(ctx context.Context, kind string, id int) ([]proxmox.Snapshot, error) {
+ key := fmt.Sprintf("%s/%d", kind, id)
+ mu.Lock()
+ calls[key]++
+ first := accepted == ""
+ if first {
+ accepted = key
+ }
+ mu.Unlock()
+ if first {
+ return []proxmox.Snapshot{{Name: "accepted-before-stop", SnapTime: 1700000100}}, nil
+ }
+ <-ctx.Done()
+ // A nil-error late body must not masquerade as an accepted
+ // result or authorize removal of the last known listing.
+ return []proxmox.Snapshot{{Name: "late-unaccepted", SnapTime: 1700000200}}, nil
+ }}
+ done := make(chan struct{})
+ go func() {
+ m.pollGuestSnapshots(ctx, "site", c)
+ close(done)
+ }()
+ synctest.Wait()
+ beforeStop := len(calls)
+ require.Equal(t, 9, beforeStop, "one accepted listing and eight blocked reads")
+ if stop == "deadline" {
+ time.Sleep(time.Minute)
+ } else {
+ cancel()
+ }
+ synctest.Wait()
+ <-done
+ require.Len(t, calls, beforeStop, "queued guests must not start reads after cancellation")
+ rows := m.state.GetSnapshot().PVEBackups.GuestSnapshots
+ require.Len(t, rows, len(previous)+1)
+ for _, row := range rows {
+ if row.Instance == "peer" {
+ require.Equal(t, "independent", row.Name)
+ continue
+ }
+ key := fmt.Sprintf("%s/%d", row.Type, row.VMID)
+ if key == accepted {
+ require.Equal(t, "accepted-before-stop", row.Name)
+ require.Equal(t, time.Unix(1700000100, 0), row.Time)
+ } else {
+ require.Equal(t, previous[key], row, "unread/late listing must preserve exact continuity: %s", key)
+ }
+ }
+ // A later ordinary observation drains every eligible guest. No
+ // restart, wider credentials or extra read path is involved.
+ c.read = func(_ context.Context, kind string, id int) ([]proxmox.Snapshot, error) {
+ mu.Lock()
+ calls[fmt.Sprintf("%s/%d", kind, id)]++
+ mu.Unlock()
+ return []proxmox.Snapshot{{Name: "recovered", SnapTime: 1700000300}}, nil
+ }
+ m.pollGuestSnapshots(context.Background(), "site", c)
+ require.Len(t, calls, len(previous))
+ rows = m.state.GetSnapshot().PVEBackups.GuestSnapshots
+ require.Len(t, rows, len(previous)+1)
+ for _, row := range rows {
+ if row.Instance == "site" {
+ require.Equal(t, "recovered", row.Name)
+ require.Equal(t, time.Unix(1700000300, 0), row.Time)
+ }
+ }
+ })
+ })
+ }
+}