Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
fix(mock): keep PVE guest sightings fresh on slow mock ticks
rcourtmanf69860b15337
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.mdindex 664c7bf48c..fdcdcf1aac 100644--- a/docs/release-control/v6/internal/subsystems/monitoring.md+++ b/docs/release-control/v6/internal/subsystems/monitoring.md@@ -3764,6 +3764,14 @@ service-history reads plus denial/recovery without fabricated samples.resource timestamps between cohorts, cover every node within twenty seconds,and refresh provider-backed fixtures only once per full rotation so one demotick cannot manufacture an estate-wide WebSocket delta.+ The twenty-second coverage is a sighting budget, not a side effect of the+ two-second tick: at a slower `PULSE_MOCK_UPDATE_INTERVAL` the PVE estate+ rotates through the largest divisor of the ten cohorts that still sights+ every node and its guests within thirty seconds (`proxmoxCohortCount`), down+ to every node per tick. A tick slower than that budget derives the mock+ Proxmox stale threshold from `mock.ProxmoxRefreshInterval`, never below the+ configured PVE derivation, so healthy running guests on a long-running+ mock stack never read as `telemetry_stale`.Mock replication timing follows that same refresh instead of staying at itsstartup stamp: each job keeps a stable phase inside its schedule window andis rewritten only when that phase passes, so a healthy job always lastdiff --git a/docs/release-control/v6/internal/subsystems/performance-and-scalability.md b/docs/release-control/v6/internal/subsystems/performance-and-scalability.mdindex 3e467ac6d5..5cdb2e7699 100644--- a/docs/release-control/v6/internal/subsystems/performance-and-scalability.md+++ b/docs/release-control/v6/internal/subsystems/performance-and-scalability.md@@ -814,6 +814,10 @@ leaving broadcast headroom to coalesce several sampler invalidations. A cohortmust not rebase unchanged guest, node, storage, or disk evidence, and theprovider-backed fixture set refreshes once per full rotation rather than onevery tick.+A slower mock tick rotates the PVE estate through a divisor of those cohorts+that keeps every node sighted within thirty seconds, down to every node per+tick, so per-second churn never exceeds the two-second default while running+guests stay inside the Proxmox freshness threshold.Mock replication timing is advanced in that same full-rotation refresh, andonly a job whose schedule phase has passed is rewritten, so keeping demoreplication on schedule adds no per-tick change for jobs that did not run.diff --git a/internal/mock/fixture_graph.go b/internal/mock/fixture_graph.goindex 0eb602949b..9cfd529433 100644--- a/internal/mock/fixture_graph.go+++ b/internal/mock/fixture_graph.go@@ -95,9 +95,14 @@ func (g *FixtureGraph) UpdateMetricCohort(}setMockUpdateInterval(cfg.UpdateInterval)- selectedNodes := make(map[string]struct{}, (len(g.State.Nodes)+cohortCount-1)/cohortCount)+ // PVE sightings rotate through a divisor of cohortCount, so a slow tick+ // still sights every node and its guests inside the Proxmox freshness+ // budget instead of once per full (tick x cohortCount) rotation.+ proxmoxCohorts := proxmoxCohortCount(currentMockUpdateInterval(), cohortCount)+ proxmoxCohortIndex := cohortIndex % proxmoxCohorts+ selectedNodes := make(map[string]struct{}, (len(g.State.Nodes)+proxmoxCohorts-1)/proxmoxCohorts)for index := range g.State.Nodes {- if index%cohortCount != cohortIndex {+ if index%proxmoxCohorts != proxmoxCohortIndex {continue}selectedNodes[g.State.Nodes[index].Name] = struct{}{}@@ -119,7 +124,7 @@ func (g *FixtureGraph) UpdateMetricCohort(updateFixtureStateMetricsSelectedAt(&g.State, cfg, now, fixtureMetricSelection{proxmoxNodeNames: selectedNodes,includeSupplemental: includeSupplemental,- uptimeStep: currentMockUpdateStepInt64() * int64(cohortCount),+ uptimeStep: currentMockUpdateStepInt64() * int64(proxmoxCohorts),supplementalUptimeStep: currentMockUpdateStepInt64() * supplementalTicks,})if includeSupplemental {diff --git a/internal/mock/integration.go b/internal/mock/integration.goindex 8ae4c87c58..bf09ea0592 100644--- a/internal/mock/integration.go+++ b/internal/mock/integration.go@@ -43,6 +43,8 @@ var (// seconds while bounding each realtime delta to roughly one tenth of a large// demo estate. The WebSocket publisher may coalesce several sampler ticks into// one broadcast, so this leaves enough headroom for that union to remain small.+// Slower ticks rotate the PVE estate through fewer of them; see+// proxmoxCohortCount.const mockMetricCohortCount = 10func init() {@@ -95,6 +97,39 @@ func currentMockUpdateStepInt64() int64 {// never let supplemental fixtures age beyond this budget between rebases.const supplementalFreshnessBudget = 120 * time.Second+// proxmoxSightingBudget bounds how long a PVE node and its guests go between+// mock sightings. It is half the unified registry's minimum Proxmox stale+// threshold, so a rotation inside it never brushes against that threshold.+const proxmoxSightingBudget = 30 * time.Second++// proxmoxCohortCount returns how many of the update loop's cohorts the PVE+// estate rotates through at a tick interval. The ten two-second cohorts sight+// every node within twenty seconds, but the count is fixed, so a slower tick+// stretches the rotation (150s at the public demo's 15s interval) past the+// 60s Proxmox stale threshold and running guests read as stale sources for+// most of it. The PVE rotation therefore uses the largest divisor of+// cohortCount whose rotation fits proxmoxSightingBudget, down to sighting+// every node each tick. A divisor keeps the loop's cohort index mapping each+// node to exactly one tick of every PVE rotation, and the per-second delta+// stays no larger than the default configuration's.+func proxmoxCohortCount(interval time.Duration, cohortCount int) int {+ for count := cohortCount; count > 1; count-- {+ if cohortCount%count == 0 && interval*time.Duration(count) <= proxmoxSightingBudget {+ return count+ }+ }+ return 1+}++// ProxmoxRefreshInterval returns the wall-clock cadence at which the mock+// update loop sights each PVE node and its guests. Stale-threshold derivation+// uses it the same way the Proxmox poller's interval is used, which matters+// once a tick alone is slower than proxmoxSightingBudget.+func ProxmoxRefreshInterval() time.Duration {+ interval := currentMockUpdateInterval()+ return interval * time.Duration(proxmoxCohortCount(interval, mockMetricCohortCount))+}+// supplementalRefreshEveryTick reports whether the update loop must rebase// provider-backed fixtures on every tick. At fast tick rates one rebase per// cohort rotation keeps them comfortably fresh while bounding per-tick socketdiff --git a/internal/mock/platform_fixtures_test.go b/internal/mock/platform_fixtures_test.goindex 33bb5e9067..ccde16a834 100644--- a/internal/mock/platform_fixtures_test.go+++ b/internal/mock/platform_fixtures_test.go@@ -1032,6 +1032,39 @@ func TestSupplementalRefreshCadenceTracksFreshnessBudget(t *testing.T) {}}+func TestProxmoxRefreshCadenceTracksSightingBudget(t *testing.T) {+ t.Cleanup(func() { setMockUpdateInterval(DefaultConfig.UpdateInterval) })++ cases := []struct {+ name string+ interval time.Duration+ wantCohorts int+ wantInterval time.Duration+ }{+ {"default rotation keeps every cohort", 2 * time.Second, mockMetricCohortCount, 20 * time.Second},+ {"rotation at the budget keeps every cohort", 3 * time.Second, mockMetricCohortCount, 30 * time.Second},+ {"rotation past the budget drops to a divisor", 4 * time.Second, 5, 20 * time.Second},+ {"public demo interval rotates two cohorts", 15 * time.Second, 2, 30 * time.Second},+ {"tick past half the budget sights every node per tick", 16 * time.Second, 1, 16 * time.Second},+ {"tick past the budget sights every node per tick", 60 * time.Second, 1, 60 * time.Second},+ }+ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ got := proxmoxCohortCount(tc.interval, mockMetricCohortCount)+ if got != tc.wantCohorts {+ t.Fatalf("proxmoxCohortCount(%s, %d) = %d, want %d", tc.interval, mockMetricCohortCount, got, tc.wantCohorts)+ }+ if mockMetricCohortCount%got != 0 {+ t.Fatalf("proxmoxCohortCount(%s) = %d does not divide %d", tc.interval, got, mockMetricCohortCount)+ }+ setMockUpdateInterval(tc.interval)+ if got := ProxmoxRefreshInterval(); got != tc.wantInterval {+ t.Fatalf("ProxmoxRefreshInterval() = %s at %s ticks, want %s", got, tc.interval, tc.wantInterval)+ }+ })+ }+}+func TestFixtureGraphSlowTicksKeepSupplementalFixturesFresh(t *testing.T) {t.Cleanup(func() { setMockUpdateInterval(DefaultConfig.UpdateInterval) })@@ -1071,6 +1104,61 @@ func TestFixtureGraphSlowTicksKeepSupplementalFixturesFresh(t *testing.T) {}}+// The public demo runs PULSE_MOCK_UPDATE_INTERVAL=15s. A fixed ten-cohort+// rotation sighted each PVE node's guests only every 150s there, while the+// registry judges Proxmox sightings stale after 60s, so most running guests+// drifted into Attention as telemetry_stale within minutes of startup.+func TestFixtureGraphSlowTicksKeepProxmoxGuestSightingsFresh(t *testing.T) {+ t.Cleanup(func() { setMockUpdateInterval(DefaultConfig.UpdateInterval) })++ for _, interval := range []time.Duration{2 * time.Second, 7 * time.Second, 15 * time.Second, 60 * time.Second} {+ t.Run(interval.String(), func(t *testing.T) {+ cfg := DefaultConfig+ cfg.NodeCount = 8+ cfg.VMsPerNode = 6+ cfg.LXCsPerNode = 4+ cfg.DockerHostCount = 0+ cfg.GenericHostCount = 0+ cfg.K8sClusterCount = 0+ cfg.UpdateInterval = interval++ base := time.Date(2026, 10, 22, 12, 0, 0, 0, time.UTC)+ graph := buildFixtureGraph(cfg, base)+ online := make(map[string]struct{}, len(graph.State.Nodes))+ for _, node := range graph.State.Nodes {+ if node.Status == "online" && node.ConnectionHealth != "offline" && node.Uptime > 0 {+ online[node.Name] = struct{}{}+ }+ }+ if len(online) == 0 {+ t.Fatal("fixture estate has no online PVE node")+ }++ bound := proxmoxSightingBudget+ if interval > bound {+ bound = interval+ }+ // Drive the update loop's own cohort sequence through two full+ // rotations and measure every online guest's sighting age just+ // before each tick lands, when it is oldest.+ for tick := 1; tick <= 2*mockMetricCohortCount; tick++ {+ now := base.Add(time.Duration(tick) * interval)+ for _, vm := range graph.State.VMs {+ if _, ok := online[vm.Node]; ok && now.Sub(vm.LastSeen) > bound {+ t.Fatalf("tick %d: VM %s on %s last sighted %s ago, want within %s", tick, vm.ID, vm.Node, now.Sub(vm.LastSeen), bound)+ }+ }+ for _, container := range graph.State.Containers {+ if _, ok := online[container.Node]; ok && now.Sub(container.LastSeen) > bound {+ t.Fatalf("tick %d: container %s on %s last sighted %s ago, want within %s", tick, container.ID, container.Node, now.Sub(container.LastSeen), bound)+ }+ }+ graph.UpdateMetricCohort(cfg, now, (tick-1)%mockMetricCohortCount, mockMetricCohortCount)+ }+ })+ }+}+func TestFixtureGraphMetricCohortProducesSparseUnifiedResourceChanges(t *testing.T) {cfg := DefaultConfigcfg.NodeCount = 50diff --git a/internal/monitoring/monitor_host_agents_test.go b/internal/monitoring/monitor_host_agents_test.goindex 797f10d78f..5332e2f6b9 100644--- a/internal/monitoring/monitor_host_agents_test.go+++ b/internal/monitoring/monitor_host_agents_test.go@@ -6044,8 +6044,10 @@ func TestMockUnifiedViewAppliesOperatorManualLinks(t *testing.T) {if len(thresholds) != 0 {t.Fatalf("mock seed borrowed live thresholds: %v", thresholds)}- if got := adapter.StaleThresholds()[unifiedresources.SourceProxmox]; got != 4*time.Minute {- t.Fatalf("live adapter threshold = %v, want configured four minutes", got)+ // The held five-minute mock tick sights every PVE node once per tick,+ // so that cadence widens the configured four minutes to ten.+ if got := adapter.StaleThresholds()[unifiedresources.SourceProxmox]; got != 10*time.Minute {+ t.Fatalf("live adapter threshold = %v, want ten minutes from the mock PVE cadence", got)}rest := unifiedresources.NewRegistryWithStaleThresholds(store, thresholds)rest.IngestResources(seed)diff --git a/internal/monitoring/resource_stale_thresholds.go b/internal/monitoring/resource_stale_thresholds.goindex f5dc5cbb0a..7f51b37fe9 100644--- a/internal/monitoring/resource_stale_thresholds.go+++ b/internal/monitoring/resource_stale_thresholds.go@@ -55,21 +55,36 @@ func resourcePollIntervalsForConfig(cfg *config.Config) resourcePollIntervals {// Monitor.resourceStaleThresholds instead, which honours the runtime polling// overrides its scheduler reads.func ResourceStaleThresholdsForConfig(cfg *config.Config) map[unifiedresources.DataSource]time.Duration {- return resourceStaleThresholdsForConfig(cfg, mock.IsMockEnabled(), mock.SupplementalRefreshInterval)+ return resourceStaleThresholdsForConfig(cfg, mock.IsMockEnabled(), currentMockResourceCadences())+}++// mockResourceCadences are the mock update loop's wall-clock refresh+// cadences, which stand in for poll intervals while mock mode is on. A nil+// cadence leaves its sources on the polled derivation.+type mockResourceCadences struct {+ proxmox func() time.Duration+ supplemental func() time.Duration+}++func currentMockResourceCadences() mockResourceCadences {+ return mockResourceCadences{+ proxmox: mock.ProxmoxRefreshInterval,+ supplemental: mock.SupplementalRefreshInterval,+ }}func resourceStaleThresholdsForConfig(cfg *config.Config,mockEnabled bool,- mockSupplementalCadence func() time.Duration,+ mockCadences mockResourceCadences,) map[unifiedresources.DataSource]time.Duration {- return resourceStaleThresholdsForIntervals(resourcePollIntervalsForConfig(cfg), mockEnabled, mockSupplementalCadence)+ return resourceStaleThresholdsForIntervals(resourcePollIntervalsForConfig(cfg), mockEnabled, mockCadences)}func resourceStaleThresholdsForIntervals(intervals resourcePollIntervals,mockEnabled bool,- mockSupplementalCadence func() time.Duration,+ mockCadences mockResourceCadences,) map[unifiedresources.DataSource]time.Duration {thresholds := map[unifiedresources.DataSource]time.Duration{unifiedresources.SourceProxmox: resourceStaleThresholdForPollInterval(@@ -91,15 +106,29 @@ func resourceStaleThresholdsForIntervals(// way the entries above derive theirs, so a slow mock tick (e.g. a large// PULSE_MOCK_UPDATE_INTERVAL on the public demo) does not flag every// provider-owned row as a stale source between refreshes.- if mockEnabled && mockSupplementalCadence != nil {+ if mockEnabled && mockCadences.supplemental != nil {supplemental := resourceStaleThresholdForPollInterval(- mockSupplementalCadence(),+ mockCadences.supplemental(),defaultPlatformResourceStaleThreshold,)thresholds[unifiedresources.SourceTrueNAS] = supplementalthresholds[unifiedresources.SourceVMware] = supplementalthresholds[unifiedresources.SourceAvailability] = supplemental}+ // Mock PVE nodes and guests are sighted on the update loop's Proxmox+ // cohort cadence, not the configured PVE poll interval. That cadence+ // stays inside the default threshold unless a single tick is slower than+ // its budget; then the threshold derives from it, never dropping below+ // the configured derivation.+ if mockEnabled && mockCadences.proxmox != nil {+ proxmox := resourceStaleThresholdForPollInterval(+ mockCadences.proxmox(),+ defaultProxmoxResourceStaleThreshold,+ )+ if proxmox > thresholds[unifiedresources.SourceProxmox] {+ thresholds[unifiedresources.SourceProxmox] = proxmox+ }+ }return thresholds}@@ -166,7 +195,7 @@ func (m *Monitor) resourceStaleThresholds() map[unifiedresources.DataSource]timereturn resourceStaleThresholdsForIntervals(m.resourcePollIntervals(),mock.IsMockEnabled(),- mock.SupplementalRefreshInterval,+ currentMockResourceCadences(),)}diff --git a/internal/monitoring/resource_stale_thresholds_test.go b/internal/monitoring/resource_stale_thresholds_test.goindex 71507dc5cb..3cf53d6d2a 100644--- a/internal/monitoring/resource_stale_thresholds_test.go+++ b/internal/monitoring/resource_stale_thresholds_test.go@@ -4,6 +4,7 @@ import ("testing""time"+ "github.com/rcourtman/pulse-go-rewrite/internal/config""github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources")@@ -15,10 +16,10 @@ func TestResourceStaleThresholdsDeriveMockSupplementalCadence(t *testing.T) {}t.Run("mock disabled leaves supplemental sources on registry defaults", func(t *testing.T) {- thresholds := resourceStaleThresholdsForConfig(nil, false, func() time.Duration {+ thresholds := resourceStaleThresholdsForConfig(nil, false, mockResourceCadences{supplemental: func() time.Duration {t.Fatal("supplemental cadence must not be consulted while mock mode is off")return 0- })+ }})for _, source := range supplementalSources {if _, ok := thresholds[source]; ok {t.Fatalf("unexpected %s threshold override with mock disabled", source)@@ -27,9 +28,9 @@ func TestResourceStaleThresholdsDeriveMockSupplementalCadence(t *testing.T) {})t.Run("fast mock cadence keeps the default platform threshold", func(t *testing.T) {- thresholds := resourceStaleThresholdsForConfig(nil, true, func() time.Duration {+ thresholds := resourceStaleThresholdsForConfig(nil, true, mockResourceCadences{supplemental: func() time.Duration {return 20 * time.Second- })+ }})for _, source := range supplementalSources {if got := thresholds[source]; got != defaultPlatformResourceStaleThreshold {t.Fatalf("%s threshold = %s, want default %s", source, got, defaultPlatformResourceStaleThreshold)@@ -41,9 +42,9 @@ func TestResourceStaleThresholdsDeriveMockSupplementalCadence(t *testing.T) {// a fixed 120s threshold there flagged every TrueNAS/VMware/availability// row as a stale source for most of each refresh cycle.t.Run("slow mock cadence widens supplemental thresholds", func(t *testing.T) {- thresholds := resourceStaleThresholdsForConfig(nil, true, func() time.Duration {+ thresholds := resourceStaleThresholdsForConfig(nil, true, mockResourceCadences{supplemental: func() time.Duration {return 300 * time.Second- })+ }})want := 600 * time.Secondfor _, source := range supplementalSources {if got := thresholds[source]; got != want {@@ -53,10 +54,10 @@ func TestResourceStaleThresholdsDeriveMockSupplementalCadence(t *testing.T) {})t.Run("supplemental derivation leaves polled sources untouched", func(t *testing.T) {- base := resourceStaleThresholdsForConfig(nil, false, nil)- derived := resourceStaleThresholdsForConfig(nil, true, func() time.Duration {+ base := resourceStaleThresholdsForConfig(nil, false, mockResourceCadences{})+ derived := resourceStaleThresholdsForConfig(nil, true, mockResourceCadences{supplemental: func() time.Duration {return 300 * time.Second- })+ }})for _, source := range []unifiedresources.DataSource{unifiedresources.SourceProxmox,unifiedresources.SourcePBS,@@ -68,3 +69,44 @@ func TestResourceStaleThresholdsDeriveMockSupplementalCadence(t *testing.T) {}})}++func TestResourceStaleThresholdsDeriveMockProxmoxCadence(t *testing.T) {+ proxmoxCadence := func(cadence time.Duration) mockResourceCadences {+ return mockResourceCadences{proxmox: func() time.Duration { return cadence }}+ }+ configured := resourceStaleThresholdsForConfig(nil, false, mockResourceCadences{})[unifiedresources.SourceProxmox]++ t.Run("mock disabled ignores the mock Proxmox cadence", func(t *testing.T) {+ thresholds := resourceStaleThresholdsForConfig(nil, false, mockResourceCadences{proxmox: func() time.Duration {+ t.Fatal("Proxmox cadence must not be consulted while mock mode is off")+ return 0+ }})+ if got := thresholds[unifiedresources.SourceProxmox]; got != configured {+ t.Fatalf("Proxmox threshold = %s with mock disabled, want configured %s", got, configured)+ }+ })++ // The public demo's 15s tick sights PVE guests every 30s, inside the+ // configured threshold, so the threshold must not widen.+ t.Run("cadence inside the budget keeps the configured threshold", func(t *testing.T) {+ if got := resourceStaleThresholdsForConfig(nil, true, proxmoxCadence(30*time.Second))[unifiedresources.SourceProxmox]; got != configured {+ t.Fatalf("Proxmox threshold = %s, want configured %s", got, configured)+ }+ })++ // A tick slower than the budget sights every guest once per tick; a 60s+ // tick judged by the 60s threshold flagged every guest stale on every+ // tick boundary.+ t.Run("slow tick derives the threshold from the sighting cadence", func(t *testing.T) {+ if got, want := resourceStaleThresholdsForConfig(nil, true, proxmoxCadence(60*time.Second))[unifiedresources.SourceProxmox], 120*time.Second; got != want {+ t.Fatalf("Proxmox threshold = %s, want %s", got, want)+ }+ })++ t.Run("mock cadence never narrows a slower configured PVE interval", func(t *testing.T) {+ cfg := &config.Config{PVEPollingInterval: 5 * time.Minute}+ if got, want := resourceStaleThresholdsForConfig(cfg, true, proxmoxCadence(20*time.Second))[unifiedresources.SourceProxmox], 10*time.Minute; got != want {+ t.Fatalf("Proxmox threshold = %s, want %s", got, want)+ }+ })+}