Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

fix(mock): keep PVE guest sightings fresh on slow mock ticks

rcourtmanf69860b15337
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md
index 664c7bf48c..fdcdcf1aac 100644
--- a/docs/release-control/v6/internal/subsystems/monitoring.md
+++ b/docs/release-control/v6/internal/subsystems/monitoring.md
@@ -3764,6 +3764,14 @@ service-history reads plus denial/recovery without fabricated samples.
resource timestamps between cohorts, cover every node within twenty seconds,
and refresh provider-backed fixtures only once per full rotation so one demo
tick cannot manufacture an estate-wide WebSocket delta.
+ The twenty-second coverage is a sighting budget, not a side effect of the
+ two-second tick: at a slower `PULSE_MOCK_UPDATE_INTERVAL` the PVE estate
+ rotates through the largest divisor of the ten cohorts that still sights
+ every node and its guests within thirty seconds (`proxmoxCohortCount`), down
+ to every node per tick. A tick slower than that budget derives the mock
+ Proxmox stale threshold from `mock.ProxmoxRefreshInterval`, never below the
+ configured PVE derivation, so healthy running guests on a long-running
+ mock stack never read as `telemetry_stale`.
Mock replication timing follows that same refresh instead of staying at its
startup stamp: each job keeps a stable phase inside its schedule window and
is rewritten only when that phase passes, so a healthy job always last
diff --git a/docs/release-control/v6/internal/subsystems/performance-and-scalability.md b/docs/release-control/v6/internal/subsystems/performance-and-scalability.md
index 3e467ac6d5..5cdb2e7699 100644
--- a/docs/release-control/v6/internal/subsystems/performance-and-scalability.md
+++ b/docs/release-control/v6/internal/subsystems/performance-and-scalability.md
@@ -814,6 +814,10 @@ leaving broadcast headroom to coalesce several sampler invalidations. A cohort
must not rebase unchanged guest, node, storage, or disk evidence, and the
provider-backed fixture set refreshes once per full rotation rather than on
every tick.
+A slower mock tick rotates the PVE estate through a divisor of those cohorts
+that keeps every node sighted within thirty seconds, down to every node per
+tick, so per-second churn never exceeds the two-second default while running
+guests stay inside the Proxmox freshness threshold.
Mock replication timing is advanced in that same full-rotation refresh, and
only a job whose schedule phase has passed is rewritten, so keeping demo
replication on schedule adds no per-tick change for jobs that did not run.
diff --git a/internal/mock/fixture_graph.go b/internal/mock/fixture_graph.go
index 0eb602949b..9cfd529433 100644
--- a/internal/mock/fixture_graph.go
+++ b/internal/mock/fixture_graph.go
@@ -95,9 +95,14 @@ func (g *FixtureGraph) UpdateMetricCohort(
}
setMockUpdateInterval(cfg.UpdateInterval)
- selectedNodes := make(map[string]struct{}, (len(g.State.Nodes)+cohortCount-1)/cohortCount)
+ // PVE sightings rotate through a divisor of cohortCount, so a slow tick
+ // still sights every node and its guests inside the Proxmox freshness
+ // budget instead of once per full (tick x cohortCount) rotation.
+ proxmoxCohorts := proxmoxCohortCount(currentMockUpdateInterval(), cohortCount)
+ proxmoxCohortIndex := cohortIndex % proxmoxCohorts
+ selectedNodes := make(map[string]struct{}, (len(g.State.Nodes)+proxmoxCohorts-1)/proxmoxCohorts)
for index := range g.State.Nodes {
- if index%cohortCount != cohortIndex {
+ if index%proxmoxCohorts != proxmoxCohortIndex {
continue
}
selectedNodes[g.State.Nodes[index].Name] = struct{}{}
@@ -119,7 +124,7 @@ func (g *FixtureGraph) UpdateMetricCohort(
updateFixtureStateMetricsSelectedAt(&g.State, cfg, now, fixtureMetricSelection{
proxmoxNodeNames: selectedNodes,
includeSupplemental: includeSupplemental,
- uptimeStep: currentMockUpdateStepInt64() * int64(cohortCount),
+ uptimeStep: currentMockUpdateStepInt64() * int64(proxmoxCohorts),
supplementalUptimeStep: currentMockUpdateStepInt64() * supplementalTicks,
})
if includeSupplemental {
diff --git a/internal/mock/integration.go b/internal/mock/integration.go
index 8ae4c87c58..bf09ea0592 100644
--- a/internal/mock/integration.go
+++ b/internal/mock/integration.go
@@ -43,6 +43,8 @@ var (
// seconds while bounding each realtime delta to roughly one tenth of a large
// demo estate. The WebSocket publisher may coalesce several sampler ticks into
// one broadcast, so this leaves enough headroom for that union to remain small.
+// Slower ticks rotate the PVE estate through fewer of them; see
+// proxmoxCohortCount.
const mockMetricCohortCount = 10
func init() {
@@ -95,6 +97,39 @@ func currentMockUpdateStepInt64() int64 {
// never let supplemental fixtures age beyond this budget between rebases.
const supplementalFreshnessBudget = 120 * time.Second
+// proxmoxSightingBudget bounds how long a PVE node and its guests go between
+// mock sightings. It is half the unified registry's minimum Proxmox stale
+// threshold, so a rotation inside it never brushes against that threshold.
+const proxmoxSightingBudget = 30 * time.Second
+
+// proxmoxCohortCount returns how many of the update loop's cohorts the PVE
+// estate rotates through at a tick interval. The ten two-second cohorts sight
+// every node within twenty seconds, but the count is fixed, so a slower tick
+// stretches the rotation (150s at the public demo's 15s interval) past the
+// 60s Proxmox stale threshold and running guests read as stale sources for
+// most of it. The PVE rotation therefore uses the largest divisor of
+// cohortCount whose rotation fits proxmoxSightingBudget, down to sighting
+// every node each tick. A divisor keeps the loop's cohort index mapping each
+// node to exactly one tick of every PVE rotation, and the per-second delta
+// stays no larger than the default configuration's.
+func proxmoxCohortCount(interval time.Duration, cohortCount int) int {
+ for count := cohortCount; count > 1; count-- {
+ if cohortCount%count == 0 && interval*time.Duration(count) <= proxmoxSightingBudget {
+ return count
+ }
+ }
+ return 1
+}
+
+// ProxmoxRefreshInterval returns the wall-clock cadence at which the mock
+// update loop sights each PVE node and its guests. Stale-threshold derivation
+// uses it the same way the Proxmox poller's interval is used, which matters
+// once a tick alone is slower than proxmoxSightingBudget.
+func ProxmoxRefreshInterval() time.Duration {
+ interval := currentMockUpdateInterval()
+ return interval * time.Duration(proxmoxCohortCount(interval, mockMetricCohortCount))
+}
+
// supplementalRefreshEveryTick reports whether the update loop must rebase
// provider-backed fixtures on every tick. At fast tick rates one rebase per
// cohort rotation keeps them comfortably fresh while bounding per-tick socket
diff --git a/internal/mock/platform_fixtures_test.go b/internal/mock/platform_fixtures_test.go
index 33bb5e9067..ccde16a834 100644
--- a/internal/mock/platform_fixtures_test.go
+++ b/internal/mock/platform_fixtures_test.go
@@ -1032,6 +1032,39 @@ func TestSupplementalRefreshCadenceTracksFreshnessBudget(t *testing.T) {
}
}
+func TestProxmoxRefreshCadenceTracksSightingBudget(t *testing.T) {
+ t.Cleanup(func() { setMockUpdateInterval(DefaultConfig.UpdateInterval) })
+
+ cases := []struct {
+ name string
+ interval time.Duration
+ wantCohorts int
+ wantInterval time.Duration
+ }{
+ {"default rotation keeps every cohort", 2 * time.Second, mockMetricCohortCount, 20 * time.Second},
+ {"rotation at the budget keeps every cohort", 3 * time.Second, mockMetricCohortCount, 30 * time.Second},
+ {"rotation past the budget drops to a divisor", 4 * time.Second, 5, 20 * time.Second},
+ {"public demo interval rotates two cohorts", 15 * time.Second, 2, 30 * time.Second},
+ {"tick past half the budget sights every node per tick", 16 * time.Second, 1, 16 * time.Second},
+ {"tick past the budget sights every node per tick", 60 * time.Second, 1, 60 * time.Second},
+ }
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ got := proxmoxCohortCount(tc.interval, mockMetricCohortCount)
+ if got != tc.wantCohorts {
+ t.Fatalf("proxmoxCohortCount(%s, %d) = %d, want %d", tc.interval, mockMetricCohortCount, got, tc.wantCohorts)
+ }
+ if mockMetricCohortCount%got != 0 {
+ t.Fatalf("proxmoxCohortCount(%s) = %d does not divide %d", tc.interval, got, mockMetricCohortCount)
+ }
+ setMockUpdateInterval(tc.interval)
+ if got := ProxmoxRefreshInterval(); got != tc.wantInterval {
+ t.Fatalf("ProxmoxRefreshInterval() = %s at %s ticks, want %s", got, tc.interval, tc.wantInterval)
+ }
+ })
+ }
+}
+
func TestFixtureGraphSlowTicksKeepSupplementalFixturesFresh(t *testing.T) {
t.Cleanup(func() { setMockUpdateInterval(DefaultConfig.UpdateInterval) })
@@ -1071,6 +1104,61 @@ func TestFixtureGraphSlowTicksKeepSupplementalFixturesFresh(t *testing.T) {
}
}
+// The public demo runs PULSE_MOCK_UPDATE_INTERVAL=15s. A fixed ten-cohort
+// rotation sighted each PVE node's guests only every 150s there, while the
+// registry judges Proxmox sightings stale after 60s, so most running guests
+// drifted into Attention as telemetry_stale within minutes of startup.
+func TestFixtureGraphSlowTicksKeepProxmoxGuestSightingsFresh(t *testing.T) {
+ t.Cleanup(func() { setMockUpdateInterval(DefaultConfig.UpdateInterval) })
+
+ for _, interval := range []time.Duration{2 * time.Second, 7 * time.Second, 15 * time.Second, 60 * time.Second} {
+ t.Run(interval.String(), func(t *testing.T) {
+ cfg := DefaultConfig
+ cfg.NodeCount = 8
+ cfg.VMsPerNode = 6
+ cfg.LXCsPerNode = 4
+ cfg.DockerHostCount = 0
+ cfg.GenericHostCount = 0
+ cfg.K8sClusterCount = 0
+ cfg.UpdateInterval = interval
+
+ base := time.Date(2026, 10, 22, 12, 0, 0, 0, time.UTC)
+ graph := buildFixtureGraph(cfg, base)
+ online := make(map[string]struct{}, len(graph.State.Nodes))
+ for _, node := range graph.State.Nodes {
+ if node.Status == "online" && node.ConnectionHealth != "offline" && node.Uptime > 0 {
+ online[node.Name] = struct{}{}
+ }
+ }
+ if len(online) == 0 {
+ t.Fatal("fixture estate has no online PVE node")
+ }
+
+ bound := proxmoxSightingBudget
+ if interval > bound {
+ bound = interval
+ }
+ // Drive the update loop's own cohort sequence through two full
+ // rotations and measure every online guest's sighting age just
+ // before each tick lands, when it is oldest.
+ for tick := 1; tick <= 2*mockMetricCohortCount; tick++ {
+ now := base.Add(time.Duration(tick) * interval)
+ for _, vm := range graph.State.VMs {
+ if _, ok := online[vm.Node]; ok && now.Sub(vm.LastSeen) > bound {
+ t.Fatalf("tick %d: VM %s on %s last sighted %s ago, want within %s", tick, vm.ID, vm.Node, now.Sub(vm.LastSeen), bound)
+ }
+ }
+ for _, container := range graph.State.Containers {
+ if _, ok := online[container.Node]; ok && now.Sub(container.LastSeen) > bound {
+ t.Fatalf("tick %d: container %s on %s last sighted %s ago, want within %s", tick, container.ID, container.Node, now.Sub(container.LastSeen), bound)
+ }
+ }
+ graph.UpdateMetricCohort(cfg, now, (tick-1)%mockMetricCohortCount, mockMetricCohortCount)
+ }
+ })
+ }
+}
+
func TestFixtureGraphMetricCohortProducesSparseUnifiedResourceChanges(t *testing.T) {
cfg := DefaultConfig
cfg.NodeCount = 50
diff --git a/internal/monitoring/monitor_host_agents_test.go b/internal/monitoring/monitor_host_agents_test.go
index 797f10d78f..5332e2f6b9 100644
--- a/internal/monitoring/monitor_host_agents_test.go
+++ b/internal/monitoring/monitor_host_agents_test.go
@@ -6044,8 +6044,10 @@ func TestMockUnifiedViewAppliesOperatorManualLinks(t *testing.T) {
if len(thresholds) != 0 {
t.Fatalf("mock seed borrowed live thresholds: %v", thresholds)
}
- if got := adapter.StaleThresholds()[unifiedresources.SourceProxmox]; got != 4*time.Minute {
- t.Fatalf("live adapter threshold = %v, want configured four minutes", got)
+ // The held five-minute mock tick sights every PVE node once per tick,
+ // so that cadence widens the configured four minutes to ten.
+ if got := adapter.StaleThresholds()[unifiedresources.SourceProxmox]; got != 10*time.Minute {
+ t.Fatalf("live adapter threshold = %v, want ten minutes from the mock PVE cadence", got)
}
rest := unifiedresources.NewRegistryWithStaleThresholds(store, thresholds)
rest.IngestResources(seed)
diff --git a/internal/monitoring/resource_stale_thresholds.go b/internal/monitoring/resource_stale_thresholds.go
index f5dc5cbb0a..7f51b37fe9 100644
--- a/internal/monitoring/resource_stale_thresholds.go
+++ b/internal/monitoring/resource_stale_thresholds.go
@@ -55,21 +55,36 @@ func resourcePollIntervalsForConfig(cfg *config.Config) resourcePollIntervals {
// Monitor.resourceStaleThresholds instead, which honours the runtime polling
// overrides its scheduler reads.
func ResourceStaleThresholdsForConfig(cfg *config.Config) map[unifiedresources.DataSource]time.Duration {
- return resourceStaleThresholdsForConfig(cfg, mock.IsMockEnabled(), mock.SupplementalRefreshInterval)
+ return resourceStaleThresholdsForConfig(cfg, mock.IsMockEnabled(), currentMockResourceCadences())
+}
+
+// mockResourceCadences are the mock update loop's wall-clock refresh
+// cadences, which stand in for poll intervals while mock mode is on. A nil
+// cadence leaves its sources on the polled derivation.
+type mockResourceCadences struct {
+ proxmox func() time.Duration
+ supplemental func() time.Duration
+}
+
+func currentMockResourceCadences() mockResourceCadences {
+ return mockResourceCadences{
+ proxmox: mock.ProxmoxRefreshInterval,
+ supplemental: mock.SupplementalRefreshInterval,
+ }
}
func resourceStaleThresholdsForConfig(
cfg *config.Config,
mockEnabled bool,
- mockSupplementalCadence func() time.Duration,
+ mockCadences mockResourceCadences,
) map[unifiedresources.DataSource]time.Duration {
- return resourceStaleThresholdsForIntervals(resourcePollIntervalsForConfig(cfg), mockEnabled, mockSupplementalCadence)
+ return resourceStaleThresholdsForIntervals(resourcePollIntervalsForConfig(cfg), mockEnabled, mockCadences)
}
func resourceStaleThresholdsForIntervals(
intervals resourcePollIntervals,
mockEnabled bool,
- mockSupplementalCadence func() time.Duration,
+ mockCadences mockResourceCadences,
) map[unifiedresources.DataSource]time.Duration {
thresholds := map[unifiedresources.DataSource]time.Duration{
unifiedresources.SourceProxmox: resourceStaleThresholdForPollInterval(
@@ -91,15 +106,29 @@ func resourceStaleThresholdsForIntervals(
// way the entries above derive theirs, so a slow mock tick (e.g. a large
// PULSE_MOCK_UPDATE_INTERVAL on the public demo) does not flag every
// provider-owned row as a stale source between refreshes.
- if mockEnabled && mockSupplementalCadence != nil {
+ if mockEnabled && mockCadences.supplemental != nil {
supplemental := resourceStaleThresholdForPollInterval(
- mockSupplementalCadence(),
+ mockCadences.supplemental(),
defaultPlatformResourceStaleThreshold,
)
thresholds[unifiedresources.SourceTrueNAS] = supplemental
thresholds[unifiedresources.SourceVMware] = supplemental
thresholds[unifiedresources.SourceAvailability] = supplemental
}
+ // Mock PVE nodes and guests are sighted on the update loop's Proxmox
+ // cohort cadence, not the configured PVE poll interval. That cadence
+ // stays inside the default threshold unless a single tick is slower than
+ // its budget; then the threshold derives from it, never dropping below
+ // the configured derivation.
+ if mockEnabled && mockCadences.proxmox != nil {
+ proxmox := resourceStaleThresholdForPollInterval(
+ mockCadences.proxmox(),
+ defaultProxmoxResourceStaleThreshold,
+ )
+ if proxmox > thresholds[unifiedresources.SourceProxmox] {
+ thresholds[unifiedresources.SourceProxmox] = proxmox
+ }
+ }
return thresholds
}
@@ -166,7 +195,7 @@ func (m *Monitor) resourceStaleThresholds() map[unifiedresources.DataSource]time
return resourceStaleThresholdsForIntervals(
m.resourcePollIntervals(),
mock.IsMockEnabled(),
- mock.SupplementalRefreshInterval,
+ currentMockResourceCadences(),
)
}
diff --git a/internal/monitoring/resource_stale_thresholds_test.go b/internal/monitoring/resource_stale_thresholds_test.go
index 71507dc5cb..3cf53d6d2a 100644
--- a/internal/monitoring/resource_stale_thresholds_test.go
+++ b/internal/monitoring/resource_stale_thresholds_test.go
@@ -4,6 +4,7 @@ import (
"testing"
"time"
+ "github.com/rcourtman/pulse-go-rewrite/internal/config"
"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
)
@@ -15,10 +16,10 @@ func TestResourceStaleThresholdsDeriveMockSupplementalCadence(t *testing.T) {
}
t.Run("mock disabled leaves supplemental sources on registry defaults", func(t *testing.T) {
- thresholds := resourceStaleThresholdsForConfig(nil, false, func() time.Duration {
+ thresholds := resourceStaleThresholdsForConfig(nil, false, mockResourceCadences{supplemental: func() time.Duration {
t.Fatal("supplemental cadence must not be consulted while mock mode is off")
return 0
- })
+ }})
for _, source := range supplementalSources {
if _, ok := thresholds[source]; ok {
t.Fatalf("unexpected %s threshold override with mock disabled", source)
@@ -27,9 +28,9 @@ func TestResourceStaleThresholdsDeriveMockSupplementalCadence(t *testing.T) {
})
t.Run("fast mock cadence keeps the default platform threshold", func(t *testing.T) {
- thresholds := resourceStaleThresholdsForConfig(nil, true, func() time.Duration {
+ thresholds := resourceStaleThresholdsForConfig(nil, true, mockResourceCadences{supplemental: func() time.Duration {
return 20 * time.Second
- })
+ }})
for _, source := range supplementalSources {
if got := thresholds[source]; got != defaultPlatformResourceStaleThreshold {
t.Fatalf("%s threshold = %s, want default %s", source, got, defaultPlatformResourceStaleThreshold)
@@ -41,9 +42,9 @@ func TestResourceStaleThresholdsDeriveMockSupplementalCadence(t *testing.T) {
// a fixed 120s threshold there flagged every TrueNAS/VMware/availability
// row as a stale source for most of each refresh cycle.
t.Run("slow mock cadence widens supplemental thresholds", func(t *testing.T) {
- thresholds := resourceStaleThresholdsForConfig(nil, true, func() time.Duration {
+ thresholds := resourceStaleThresholdsForConfig(nil, true, mockResourceCadences{supplemental: func() time.Duration {
return 300 * time.Second
- })
+ }})
want := 600 * time.Second
for _, source := range supplementalSources {
if got := thresholds[source]; got != want {
@@ -53,10 +54,10 @@ func TestResourceStaleThresholdsDeriveMockSupplementalCadence(t *testing.T) {
})
t.Run("supplemental derivation leaves polled sources untouched", func(t *testing.T) {
- base := resourceStaleThresholdsForConfig(nil, false, nil)
- derived := resourceStaleThresholdsForConfig(nil, true, func() time.Duration {
+ base := resourceStaleThresholdsForConfig(nil, false, mockResourceCadences{})
+ derived := resourceStaleThresholdsForConfig(nil, true, mockResourceCadences{supplemental: func() time.Duration {
return 300 * time.Second
- })
+ }})
for _, source := range []unifiedresources.DataSource{
unifiedresources.SourceProxmox,
unifiedresources.SourcePBS,
@@ -68,3 +69,44 @@ func TestResourceStaleThresholdsDeriveMockSupplementalCadence(t *testing.T) {
}
})
}
+
+func TestResourceStaleThresholdsDeriveMockProxmoxCadence(t *testing.T) {
+ proxmoxCadence := func(cadence time.Duration) mockResourceCadences {
+ return mockResourceCadences{proxmox: func() time.Duration { return cadence }}
+ }
+ configured := resourceStaleThresholdsForConfig(nil, false, mockResourceCadences{})[unifiedresources.SourceProxmox]
+
+ t.Run("mock disabled ignores the mock Proxmox cadence", func(t *testing.T) {
+ thresholds := resourceStaleThresholdsForConfig(nil, false, mockResourceCadences{proxmox: func() time.Duration {
+ t.Fatal("Proxmox cadence must not be consulted while mock mode is off")
+ return 0
+ }})
+ if got := thresholds[unifiedresources.SourceProxmox]; got != configured {
+ t.Fatalf("Proxmox threshold = %s with mock disabled, want configured %s", got, configured)
+ }
+ })
+
+ // The public demo's 15s tick sights PVE guests every 30s, inside the
+ // configured threshold, so the threshold must not widen.
+ t.Run("cadence inside the budget keeps the configured threshold", func(t *testing.T) {
+ if got := resourceStaleThresholdsForConfig(nil, true, proxmoxCadence(30*time.Second))[unifiedresources.SourceProxmox]; got != configured {
+ t.Fatalf("Proxmox threshold = %s, want configured %s", got, configured)
+ }
+ })
+
+ // A tick slower than the budget sights every guest once per tick; a 60s
+ // tick judged by the 60s threshold flagged every guest stale on every
+ // tick boundary.
+ t.Run("slow tick derives the threshold from the sighting cadence", func(t *testing.T) {
+ if got, want := resourceStaleThresholdsForConfig(nil, true, proxmoxCadence(60*time.Second))[unifiedresources.SourceProxmox], 120*time.Second; got != want {
+ t.Fatalf("Proxmox threshold = %s, want %s", got, want)
+ }
+ })
+
+ t.Run("mock cadence never narrows a slower configured PVE interval", func(t *testing.T) {
+ cfg := &config.Config{PVEPollingInterval: 5 * time.Minute}
+ if got, want := resourceStaleThresholdsForConfig(cfg, true, proxmoxCadence(20*time.Second))[unifiedresources.SourceProxmox], 10*time.Minute; got != want {
+ t.Fatalf("Proxmox threshold = %s, want %s", got, want)
+ }
+ })
+}