Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Judge PBS and PMG freshness by the live polling interval

rcourtman32a4f00c44a1
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md
index d4eb93380d..9a064a4e9b 100644
--- a/docs/release-control/v6/internal/subsystems/monitoring.md
+++ b/docs/release-control/v6/internal/subsystems/monitoring.md
@@ -2182,6 +2182,19 @@ service-history reads plus denial/recovery without fabricated samples.
poll interval and passed into the unified-resource adapter. New pollers or
config paths that change source cadence must update that derivation instead
of hard-coding stale windows inside registry or API code.
+ Non-default tenant monitors poll against a detached config copy (#1619), so
+ a system-settings save reaches them only as PBS and PMG runtime polling
+ overrides. Under fixed-cadence scheduling, `Monitor.resourceStaleThresholds`
+ reads those overrides through the same clamped setting readers as the
+ scheduler, so a saved interval moves the monitor's resource freshness and
+ polling together. An adaptive scheduler selects its own intervals and
+ ignores those overrides, so freshness there keeps deriving from the
+ configured per-platform intervals. A PVE interval save reloads every
+ monitor from saved config instead. `ResourceStaleThresholdsForConfig` stays
+ the config-only derivation for callers without a live monitor, such as
+ adapter construction. Regression coverage:
+ `TestMonitorResourceStaleThresholdsFollowRuntimePollingOverrides` in
+ `internal/monitoring/canonical_guardrails_test.go`.
Periodic out-of-scheduler platform pollers (TrueNAS, VMware) share their
lifecycle and config-resolution scaffold through
`internal/monitoring/platform_poller_shared.go`: `startPollerLoop` owns
diff --git a/internal/monitoring/canonical_guardrails_test.go b/internal/monitoring/canonical_guardrails_test.go
index f723e78dc7..2099e7b53c 100644
--- a/internal/monitoring/canonical_guardrails_test.go
+++ b/internal/monitoring/canonical_guardrails_test.go
@@ -2879,6 +2879,140 @@ func TestResourceStaleThresholdsPreserveDefaultFloors(t *testing.T) {
}
}
+// Non-default tenant monitors poll against a detached config copy (#1619), so
+// a saved PBS or PMG interval reaches them only as a runtime override. The
+// freshness thresholds the monitor publishes must follow the cadence its
+// scheduler polls at, or PBS and PMG rows read stale between real polls.
+func TestMonitorResourceStaleThresholdsFollowRuntimePollingOverrides(t *testing.T) {
+ cases := []struct {
+ name string
+ configured time.Duration
+ override time.Duration
+ wantThreshold time.Duration
+ wantStatus unifiedresources.ResourceStatus
+ }{
+ {
+ name: "raised interval keeps a sighting inside the new cadence online",
+ configured: time.Minute,
+ override: 5 * time.Minute,
+ wantThreshold: 10 * time.Minute,
+ wantStatus: unifiedresources.StatusOnline,
+ },
+ {
+ name: "lowered interval judges the same sighting stale",
+ configured: 5 * time.Minute,
+ override: time.Minute,
+ wantThreshold: 2 * time.Minute,
+ wantStatus: unifiedresources.StatusWarning,
+ },
+ }
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ sighting := time.Now().UTC().Add(-3 * time.Minute)
+ state := models.NewState()
+ state.UpdatePBSInstances([]models.PBSInstance{{
+ ID: "pbs-main",
+ Name: "pbs-main",
+ Host: "https://pbs.lab.local:8007",
+ Status: "online",
+ ConnectionHealth: "healthy",
+ LastSeen: sighting,
+ }})
+ state.UpdatePMGInstances([]models.PMGInstance{{
+ ID: "pmg-main",
+ Name: "pmg-main",
+ Host: "https://pmg.lab.local:8006",
+ Status: "online",
+ ConnectionHealth: "healthy",
+ LastSeen: sighting,
+ LastUpdated: sighting,
+ }})
+
+ base := &config.Config{
+ PVEPollingInterval: 10 * time.Second,
+ PBSPollingInterval: tc.configured,
+ PMGPollingInterval: tc.configured,
+ }
+ monitor := &Monitor{state: state, config: base.DeepCopy()}
+ adapter := unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(nil))
+ monitor.SetResourceStore(adapter)
+ monitor.SetPBSPollingInterval(tc.override)
+ monitor.SetPMGPollingInterval(tc.override)
+
+ if got := monitor.baseIntervalForInstanceType(InstanceTypePBS); got != tc.override {
+ t.Fatalf("PBS scheduler interval = %v, want override %v", got, tc.override)
+ }
+ thresholds := monitor.resourceStaleThresholds()
+ for _, source := range []unifiedresources.DataSource{unifiedresources.SourcePBS, unifiedresources.SourcePMG} {
+ if got := thresholds[source]; got != tc.wantThreshold {
+ t.Errorf("%s threshold = %v, want %v from the %v runtime interval", source, got, tc.wantThreshold, tc.override)
+ }
+ }
+ if got := thresholds[unifiedresources.SourceProxmox]; got != 60*time.Second {
+ t.Errorf("Proxmox threshold = %v, want the unchanged %v floor", got, 60*time.Second)
+ }
+
+ monitor.updateResourceStore(state.GetSnapshot(), monitor.mockModeFence.begin())
+
+ pbs := adapter.PBSInstances()
+ if len(pbs) != 1 {
+ t.Fatalf("PBS instances = %d, want 1", len(pbs))
+ }
+ if got := pbs[0].Status(); got != tc.wantStatus {
+ t.Fatalf("PBS status = %q, want %q for a 3m-old sighting on a %v cadence", got, tc.wantStatus, tc.override)
+ }
+ pmg := adapter.PMGInstances()
+ if len(pmg) != 1 {
+ t.Fatalf("PMG instances = %d, want 1", len(pmg))
+ }
+ if got := pmg[0].Status(); got != tc.wantStatus {
+ t.Fatalf("PMG status = %q, want %q for a 3m-old sighting on a %v cadence", got, tc.wantStatus, tc.override)
+ }
+ })
+ }
+
+ t.Run("freshness reads the scheduler's clamped interval", func(t *testing.T) {
+ for _, override := range []time.Duration{5 * time.Second, 90 * time.Second, 2 * time.Hour} {
+ base := &config.Config{PBSPollingInterval: time.Minute, PMGPollingInterval: time.Minute}
+ monitor := &Monitor{config: base.DeepCopy()}
+ monitor.SetPBSPollingInterval(override)
+ monitor.SetPMGPollingInterval(override)
+ thresholds := monitor.resourceStaleThresholds()
+ for instanceType, source := range map[InstanceType]unifiedresources.DataSource{
+ InstanceTypePBS: unifiedresources.SourcePBS,
+ InstanceTypePMG: unifiedresources.SourcePMG,
+ } {
+ scheduled := monitor.baseIntervalForInstanceType(instanceType)
+ want := resourceStaleThresholdForPollInterval(scheduled, defaultPlatformResourceStaleThreshold)
+ if got := thresholds[source]; got != want {
+ t.Errorf("%s threshold = %v for override %v, want %v from the scheduler's %v", source, got, override, want, scheduled)
+ }
+ }
+ }
+ })
+
+ // The adaptive scheduler picks intervals from its own bounds and never
+ // reads the per-platform overrides, so a save must not move freshness
+ // there; following adaptive cadence is a separate derivation.
+ t.Run("adaptive scheduling ignores per-platform overrides", func(t *testing.T) {
+ base := &config.Config{PBSPollingInterval: 5 * time.Minute, PMGPollingInterval: 5 * time.Minute}
+ monitor := &Monitor{
+ config: base.DeepCopy(),
+ scheduler: NewAdaptiveScheduler(SchedulerConfig{}, nil, nil, nil),
+ }
+ before := monitor.resourceStaleThresholds()
+ monitor.SetPBSPollingInterval(time.Minute)
+ monitor.SetPMGPollingInterval(time.Minute)
+ after := monitor.resourceStaleThresholds()
+ for _, source := range []unifiedresources.DataSource{unifiedresources.SourcePBS, unifiedresources.SourcePMG} {
+ if before[source] != 10*time.Minute || after[source] != before[source] {
+ t.Errorf("%s threshold = %v -> %v across an override the adaptive scheduler ignores, want %v throughout",
+ source, before[source], after[source], 10*time.Minute)
+ }
+ }
+ })
+}
+
// Mock history is seeded for a bounded window, so chart ranges longer than the
// seed fall through to the synthetic generator. That generator must still carry
// the host-relative `memoryused` byte series, or the workloads memory column in
diff --git a/internal/monitoring/resource_stale_thresholds.go b/internal/monitoring/resource_stale_thresholds.go
index be89dcea08..5f32913bc7 100644
--- a/internal/monitoring/resource_stale_thresholds.go
+++ b/internal/monitoring/resource_stale_thresholds.go
@@ -13,9 +13,27 @@ const (
defaultPlatformResourceStaleThreshold = 120 * time.Second
)
+// resourcePollIntervals are the poll cadences canonical resource freshness is
+// derived from.
+type resourcePollIntervals struct {
+ pve time.Duration
+ pbs time.Duration
+ pmg time.Duration
+}
+
+func resourcePollIntervalsForConfig(cfg *config.Config) resourcePollIntervals {
+ return resourcePollIntervals{
+ pve: effectivePVEPollingIntervalForConfig(cfg),
+ pbs: effectivePlatformPollingIntervalForConfig(cfg, "pbs"),
+ pmg: effectivePlatformPollingIntervalForConfig(cfg, "pmg"),
+ }
+}
+
// ResourceStaleThresholdsForConfig derives canonical resource freshness from
-// polling cadence. A source should not be considered stale until it has missed
-// at least one expected poll cycle plus the normal interval.
+// configured polling cadence. A source should not be considered stale until it
+// has missed at least one expected poll cycle plus the normal interval. A
+// running monitor judges by Monitor.resourceStaleThresholds instead, which
+// honours the runtime polling overrides its scheduler reads.
func ResourceStaleThresholdsForConfig(cfg *config.Config) map[unifiedresources.DataSource]time.Duration {
return resourceStaleThresholdsForConfig(cfg, mock.IsMockEnabled(), mock.SupplementalRefreshInterval)
}
@@ -24,18 +42,26 @@ func resourceStaleThresholdsForConfig(
cfg *config.Config,
mockEnabled bool,
mockSupplementalCadence func() time.Duration,
+) map[unifiedresources.DataSource]time.Duration {
+ return resourceStaleThresholdsForIntervals(resourcePollIntervalsForConfig(cfg), mockEnabled, mockSupplementalCadence)
+}
+
+func resourceStaleThresholdsForIntervals(
+ intervals resourcePollIntervals,
+ mockEnabled bool,
+ mockSupplementalCadence func() time.Duration,
) map[unifiedresources.DataSource]time.Duration {
thresholds := map[unifiedresources.DataSource]time.Duration{
unifiedresources.SourceProxmox: resourceStaleThresholdForPollInterval(
- effectivePVEPollingIntervalForConfig(cfg),
+ intervals.pve,
defaultProxmoxResourceStaleThreshold,
),
unifiedresources.SourcePBS: resourceStaleThresholdForPollInterval(
- effectivePlatformPollingIntervalForConfig(cfg, "pbs"),
+ intervals.pbs,
defaultPlatformResourceStaleThreshold,
),
unifiedresources.SourcePMG: resourceStaleThresholdForPollInterval(
- effectivePlatformPollingIntervalForConfig(cfg, "pmg"),
+ intervals.pmg,
defaultPlatformResourceStaleThreshold,
),
}
@@ -90,10 +116,33 @@ func unifiedProjectionResourcesWithStaleThresholds(
}
func (m *Monitor) resourceStaleThresholds() map[unifiedresources.DataSource]time.Duration {
- if m == nil {
+ if m == nil || m.config == nil {
return ResourceStaleThresholdsForConfig(nil)
}
- return ResourceStaleThresholdsForConfig(m.config)
+ return resourceStaleThresholdsForIntervals(
+ m.resourcePollIntervals(),
+ mock.IsMockEnabled(),
+ mock.SupplementalRefreshInterval,
+ )
+}
+
+// resourcePollIntervals reads the per-platform base cadences, honouring the
+// runtime overrides a fixed-cadence scheduler polls at. Non-default tenant
+// monitors poll against a detached config copy (#1619), so the settings API
+// pushes saved PBS and PMG intervals into them as runtime overrides; freshness
+// derived from m.config alone would judge those sources by an interval nothing
+// polls at. A PVE interval change reloads every monitor from saved config
+// instead, so the config value is already the live one. An adaptive scheduler
+// selects its own intervals and never reads the per-platform overrides, so
+// they must not move freshness there either.
+func (m *Monitor) resourcePollIntervals() resourcePollIntervals {
+ intervals := resourcePollIntervalsForConfig(m.config)
+ if m.scheduler != nil {
+ return intervals
+ }
+ intervals.pbs = clampInterval(m.pbsPollingIntervalSetting(), 10*time.Second, time.Hour)
+ intervals.pmg = clampInterval(m.pmgPollingIntervalSetting(), 10*time.Second, time.Hour)
+ return intervals
}
func (m *Monitor) pveNodeOfflineGracePeriod() time.Duration {