Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Judge PBS and PMG freshness by the live polling interval
rcourtman32a4f00c44a1
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.mdindex d4eb93380d..9a064a4e9b 100644--- a/docs/release-control/v6/internal/subsystems/monitoring.md+++ b/docs/release-control/v6/internal/subsystems/monitoring.md@@ -2182,6 +2182,19 @@ service-history reads plus denial/recovery without fabricated samples.poll interval and passed into the unified-resource adapter. New pollers orconfig paths that change source cadence must update that derivation insteadof hard-coding stale windows inside registry or API code.+ Non-default tenant monitors poll against a detached config copy (#1619), so+ a system-settings save reaches them only as PBS and PMG runtime polling+ overrides. Under fixed-cadence scheduling, `Monitor.resourceStaleThresholds`+ reads those overrides through the same clamped setting readers as the+ scheduler, so a saved interval moves the monitor's resource freshness and+ polling together. An adaptive scheduler selects its own intervals and+ ignores those overrides, so freshness there keeps deriving from the+ configured per-platform intervals. A PVE interval save reloads every+ monitor from saved config instead. `ResourceStaleThresholdsForConfig` stays+ the config-only derivation for callers without a live monitor, such as+ adapter construction. Regression coverage:+ `TestMonitorResourceStaleThresholdsFollowRuntimePollingOverrides` in+ `internal/monitoring/canonical_guardrails_test.go`.Periodic out-of-scheduler platform pollers (TrueNAS, VMware) share theirlifecycle and config-resolution scaffold through`internal/monitoring/platform_poller_shared.go`: `startPollerLoop` ownsdiff --git a/internal/monitoring/canonical_guardrails_test.go b/internal/monitoring/canonical_guardrails_test.goindex f723e78dc7..2099e7b53c 100644--- a/internal/monitoring/canonical_guardrails_test.go+++ b/internal/monitoring/canonical_guardrails_test.go@@ -2879,6 +2879,140 @@ func TestResourceStaleThresholdsPreserveDefaultFloors(t *testing.T) {}}+// Non-default tenant monitors poll against a detached config copy (#1619), so+// a saved PBS or PMG interval reaches them only as a runtime override. The+// freshness thresholds the monitor publishes must follow the cadence its+// scheduler polls at, or PBS and PMG rows read stale between real polls.+func TestMonitorResourceStaleThresholdsFollowRuntimePollingOverrides(t *testing.T) {+ cases := []struct {+ name string+ configured time.Duration+ override time.Duration+ wantThreshold time.Duration+ wantStatus unifiedresources.ResourceStatus+ }{+ {+ name: "raised interval keeps a sighting inside the new cadence online",+ configured: time.Minute,+ override: 5 * time.Minute,+ wantThreshold: 10 * time.Minute,+ wantStatus: unifiedresources.StatusOnline,+ },+ {+ name: "lowered interval judges the same sighting stale",+ configured: 5 * time.Minute,+ override: time.Minute,+ wantThreshold: 2 * time.Minute,+ wantStatus: unifiedresources.StatusWarning,+ },+ }+ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ sighting := time.Now().UTC().Add(-3 * time.Minute)+ state := models.NewState()+ state.UpdatePBSInstances([]models.PBSInstance{{+ ID: "pbs-main",+ Name: "pbs-main",+ Host: "https://pbs.lab.local:8007",+ Status: "online",+ ConnectionHealth: "healthy",+ LastSeen: sighting,+ }})+ state.UpdatePMGInstances([]models.PMGInstance{{+ ID: "pmg-main",+ Name: "pmg-main",+ Host: "https://pmg.lab.local:8006",+ Status: "online",+ ConnectionHealth: "healthy",+ LastSeen: sighting,+ LastUpdated: sighting,+ }})++ base := &config.Config{+ PVEPollingInterval: 10 * time.Second,+ PBSPollingInterval: tc.configured,+ PMGPollingInterval: tc.configured,+ }+ monitor := &Monitor{state: state, config: base.DeepCopy()}+ adapter := unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(nil))+ monitor.SetResourceStore(adapter)+ monitor.SetPBSPollingInterval(tc.override)+ monitor.SetPMGPollingInterval(tc.override)++ if got := monitor.baseIntervalForInstanceType(InstanceTypePBS); got != tc.override {+ t.Fatalf("PBS scheduler interval = %v, want override %v", got, tc.override)+ }+ thresholds := monitor.resourceStaleThresholds()+ for _, source := range []unifiedresources.DataSource{unifiedresources.SourcePBS, unifiedresources.SourcePMG} {+ if got := thresholds[source]; got != tc.wantThreshold {+ t.Errorf("%s threshold = %v, want %v from the %v runtime interval", source, got, tc.wantThreshold, tc.override)+ }+ }+ if got := thresholds[unifiedresources.SourceProxmox]; got != 60*time.Second {+ t.Errorf("Proxmox threshold = %v, want the unchanged %v floor", got, 60*time.Second)+ }++ monitor.updateResourceStore(state.GetSnapshot(), monitor.mockModeFence.begin())++ pbs := adapter.PBSInstances()+ if len(pbs) != 1 {+ t.Fatalf("PBS instances = %d, want 1", len(pbs))+ }+ if got := pbs[0].Status(); got != tc.wantStatus {+ t.Fatalf("PBS status = %q, want %q for a 3m-old sighting on a %v cadence", got, tc.wantStatus, tc.override)+ }+ pmg := adapter.PMGInstances()+ if len(pmg) != 1 {+ t.Fatalf("PMG instances = %d, want 1", len(pmg))+ }+ if got := pmg[0].Status(); got != tc.wantStatus {+ t.Fatalf("PMG status = %q, want %q for a 3m-old sighting on a %v cadence", got, tc.wantStatus, tc.override)+ }+ })+ }++ t.Run("freshness reads the scheduler's clamped interval", func(t *testing.T) {+ for _, override := range []time.Duration{5 * time.Second, 90 * time.Second, 2 * time.Hour} {+ base := &config.Config{PBSPollingInterval: time.Minute, PMGPollingInterval: time.Minute}+ monitor := &Monitor{config: base.DeepCopy()}+ monitor.SetPBSPollingInterval(override)+ monitor.SetPMGPollingInterval(override)+ thresholds := monitor.resourceStaleThresholds()+ for instanceType, source := range map[InstanceType]unifiedresources.DataSource{+ InstanceTypePBS: unifiedresources.SourcePBS,+ InstanceTypePMG: unifiedresources.SourcePMG,+ } {+ scheduled := monitor.baseIntervalForInstanceType(instanceType)+ want := resourceStaleThresholdForPollInterval(scheduled, defaultPlatformResourceStaleThreshold)+ if got := thresholds[source]; got != want {+ t.Errorf("%s threshold = %v for override %v, want %v from the scheduler's %v", source, got, override, want, scheduled)+ }+ }+ }+ })++ // The adaptive scheduler picks intervals from its own bounds and never+ // reads the per-platform overrides, so a save must not move freshness+ // there; following adaptive cadence is a separate derivation.+ t.Run("adaptive scheduling ignores per-platform overrides", func(t *testing.T) {+ base := &config.Config{PBSPollingInterval: 5 * time.Minute, PMGPollingInterval: 5 * time.Minute}+ monitor := &Monitor{+ config: base.DeepCopy(),+ scheduler: NewAdaptiveScheduler(SchedulerConfig{}, nil, nil, nil),+ }+ before := monitor.resourceStaleThresholds()+ monitor.SetPBSPollingInterval(time.Minute)+ monitor.SetPMGPollingInterval(time.Minute)+ after := monitor.resourceStaleThresholds()+ for _, source := range []unifiedresources.DataSource{unifiedresources.SourcePBS, unifiedresources.SourcePMG} {+ if before[source] != 10*time.Minute || after[source] != before[source] {+ t.Errorf("%s threshold = %v -> %v across an override the adaptive scheduler ignores, want %v throughout",+ source, before[source], after[source], 10*time.Minute)+ }+ }+ })+}+// Mock history is seeded for a bounded window, so chart ranges longer than the// seed fall through to the synthetic generator. That generator must still carry// the host-relative `memoryused` byte series, or the workloads memory column indiff --git a/internal/monitoring/resource_stale_thresholds.go b/internal/monitoring/resource_stale_thresholds.goindex be89dcea08..5f32913bc7 100644--- a/internal/monitoring/resource_stale_thresholds.go+++ b/internal/monitoring/resource_stale_thresholds.go@@ -13,9 +13,27 @@ const (defaultPlatformResourceStaleThreshold = 120 * time.Second)+// resourcePollIntervals are the poll cadences canonical resource freshness is+// derived from.+type resourcePollIntervals struct {+ pve time.Duration+ pbs time.Duration+ pmg time.Duration+}++func resourcePollIntervalsForConfig(cfg *config.Config) resourcePollIntervals {+ return resourcePollIntervals{+ pve: effectivePVEPollingIntervalForConfig(cfg),+ pbs: effectivePlatformPollingIntervalForConfig(cfg, "pbs"),+ pmg: effectivePlatformPollingIntervalForConfig(cfg, "pmg"),+ }+}+// ResourceStaleThresholdsForConfig derives canonical resource freshness from-// polling cadence. A source should not be considered stale until it has missed-// at least one expected poll cycle plus the normal interval.+// configured polling cadence. A source should not be considered stale until it+// has missed at least one expected poll cycle plus the normal interval. A+// running monitor judges by Monitor.resourceStaleThresholds instead, which+// honours the runtime polling overrides its scheduler reads.func ResourceStaleThresholdsForConfig(cfg *config.Config) map[unifiedresources.DataSource]time.Duration {return resourceStaleThresholdsForConfig(cfg, mock.IsMockEnabled(), mock.SupplementalRefreshInterval)}@@ -24,18 +42,26 @@ func resourceStaleThresholdsForConfig(cfg *config.Config,mockEnabled bool,mockSupplementalCadence func() time.Duration,+) map[unifiedresources.DataSource]time.Duration {+ return resourceStaleThresholdsForIntervals(resourcePollIntervalsForConfig(cfg), mockEnabled, mockSupplementalCadence)+}++func resourceStaleThresholdsForIntervals(+ intervals resourcePollIntervals,+ mockEnabled bool,+ mockSupplementalCadence func() time.Duration,) map[unifiedresources.DataSource]time.Duration {thresholds := map[unifiedresources.DataSource]time.Duration{unifiedresources.SourceProxmox: resourceStaleThresholdForPollInterval(- effectivePVEPollingIntervalForConfig(cfg),+ intervals.pve,defaultProxmoxResourceStaleThreshold,),unifiedresources.SourcePBS: resourceStaleThresholdForPollInterval(- effectivePlatformPollingIntervalForConfig(cfg, "pbs"),+ intervals.pbs,defaultPlatformResourceStaleThreshold,),unifiedresources.SourcePMG: resourceStaleThresholdForPollInterval(- effectivePlatformPollingIntervalForConfig(cfg, "pmg"),+ intervals.pmg,defaultPlatformResourceStaleThreshold,),}@@ -90,10 +116,33 @@ func unifiedProjectionResourcesWithStaleThresholds(}func (m *Monitor) resourceStaleThresholds() map[unifiedresources.DataSource]time.Duration {- if m == nil {+ if m == nil || m.config == nil {return ResourceStaleThresholdsForConfig(nil)}- return ResourceStaleThresholdsForConfig(m.config)+ return resourceStaleThresholdsForIntervals(+ m.resourcePollIntervals(),+ mock.IsMockEnabled(),+ mock.SupplementalRefreshInterval,+ )+}++// resourcePollIntervals reads the per-platform base cadences, honouring the+// runtime overrides a fixed-cadence scheduler polls at. Non-default tenant+// monitors poll against a detached config copy (#1619), so the settings API+// pushes saved PBS and PMG intervals into them as runtime overrides; freshness+// derived from m.config alone would judge those sources by an interval nothing+// polls at. A PVE interval change reloads every monitor from saved config+// instead, so the config value is already the live one. An adaptive scheduler+// selects its own intervals and never reads the per-platform overrides, so+// they must not move freshness there either.+func (m *Monitor) resourcePollIntervals() resourcePollIntervals {+ intervals := resourcePollIntervalsForConfig(m.config)+ if m.scheduler != nil {+ return intervals+ }+ intervals.pbs = clampInterval(m.pbsPollingIntervalSetting(), 10*time.Second, time.Hour)+ intervals.pmg = clampInterval(m.pmgPollingIntervalSetting(), 10*time.Second, time.Hour)+ return intervals}func (m *Monitor) pveNodeOfflineGracePeriod() time.Duration {