Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
package monitoring
import "time"
// Runtime-tunable polling cadence settings.
//
// Each tenant monitor polls against a detached DeepCopy of the base config,
// so mutating the base config after a system-settings save never reaches a
// running monitor (#1619). These setters push the saved values into live
// monitors directly, shadowing the config fields. The overrides live behind
// runtimePollingMu because polling goroutines read them every cycle while
// the settings API writes them.
// runtimePollingOverrides holds the polling-cadence values the settings API
// has pushed into this live monitor, shadowing the corresponding fields of
// the monitor's detached config copy. nil means "use the config value".
type runtimePollingOverrides struct {
backupEnabled *bool
backupInterval *time.Duration
pbsInterval *time.Duration
pmgInterval *time.Duration
}
// backupPollingEnabledSetting returns the effective EnableBackupPolling value.
func (m *Monitor) backupPollingEnabledSetting() bool {
if m == nil || m.config == nil {
return false
}
m.runtimePollingMu.RLock()
override := m.runtimePollingOverride.backupEnabled
m.runtimePollingMu.RUnlock()
if override != nil {
return *override
}
return m.config.EnableBackupPolling
}
// backupPollingIntervalSetting returns the effective BackupPollingInterval.
// Zero means cycle-based scheduling (BackupPollingCycles).
func (m *Monitor) backupPollingIntervalSetting() time.Duration {
if m == nil || m.config == nil {
return 0
}
m.runtimePollingMu.RLock()
override := m.runtimePollingOverride.backupInterval
m.runtimePollingMu.RUnlock()
if override != nil {
return *override
}
return m.config.BackupPollingInterval
}
// pbsPollingIntervalSetting returns the effective PBSPollingInterval before
// clamping.
func (m *Monitor) pbsPollingIntervalSetting() time.Duration {
if m == nil || m.config == nil {
return 0
}
m.runtimePollingMu.RLock()
override := m.runtimePollingOverride.pbsInterval
m.runtimePollingMu.RUnlock()
if override != nil {
return *override
}
return m.config.PBSPollingInterval
}
// pmgPollingIntervalSetting returns the effective PMGPollingInterval before
// clamping.
func (m *Monitor) pmgPollingIntervalSetting() time.Duration {
if m == nil || m.config == nil {
return 0
}
m.runtimePollingMu.RLock()
override := m.runtimePollingOverride.pmgInterval
m.runtimePollingMu.RUnlock()
if override != nil {
return *override
}
return m.config.PMGPollingInterval
}
// BasePollInterval returns the clamped base cadence this monitor polls PVE,
// PBS or PMG at, honouring the runtime polling overrides. For the built-in
// providers these are the values the fixed-cadence scheduler reads
// (baseIntervalForInstanceType); a replacement provider or a task already
// queued at its previous interval can differ until the next planning pass.
// This accessor skips the poll-provider lookup, so it never takes m.mu.
// Non-default tenant monitors poll against a detached config copy (#1619), so
// a saved PBS or PMG interval reaches them only as an override. Code outside
// the monitor that judges poll freshness must read the cadence here, not from
// GetConfig. An adaptive scheduler plans its own intervals independently of
// this base; see PlannedPollInterval. Zero means the cadence is unknown:
// another instance type, or a monitor without config.
func (m *Monitor) BasePollInterval(instanceType InstanceType) time.Duration {
if m == nil || m.config == nil {
return 0
}
switch instanceType {
case InstanceTypePVE:
return m.effectivePVEPollingInterval()
case InstanceTypePBS:
return clampInterval(m.pbsPollingIntervalSetting(), 10*time.Second, time.Hour)
case InstanceTypePMG:
return clampInterval(m.pmgPollingIntervalSetting(), 10*time.Second, time.Hour)
default:
return 0
}
}
// SetBackupPollingEnabled toggles backup polling on the live monitor.
// Re-enabling clears the per-instance last-poll timestamps so the next
// polling cycle runs an immediate catch-up poll, matching the behavior of
// a full monitor reload.
func (m *Monitor) SetBackupPollingEnabled(enabled bool) {
if m == nil {
return
}
m.runtimePollingMu.Lock()
wasEnabled := m.config != nil && m.config.EnableBackupPolling
if m.runtimePollingOverride.backupEnabled != nil {
wasEnabled = *m.runtimePollingOverride.backupEnabled
}
m.runtimePollingOverride.backupEnabled = &enabled
m.runtimePollingMu.Unlock()
if enabled && !wasEnabled {
m.resetBackupPollTimestamps()
}
}
// SetBackupPollingInterval updates the backup polling cadence on the live
// monitor. Lowering the interval (or switching from cycle-based scheduling
// to an interval) clears the per-instance last-poll timestamps so the next
// polling cycle runs an immediate catch-up poll rather than waiting out the
// remainder of the old interval.
func (m *Monitor) SetBackupPollingInterval(interval time.Duration) {
if m == nil {
return
}
if interval < 0 {
interval = 0
}
m.runtimePollingMu.Lock()
previous := time.Duration(0)
if m.config != nil {
previous = m.config.BackupPollingInterval
}
if m.runtimePollingOverride.backupInterval != nil {
previous = *m.runtimePollingOverride.backupInterval
}
m.runtimePollingOverride.backupInterval = &interval
m.runtimePollingMu.Unlock()
if interval > 0 && (previous <= 0 || interval < previous) {
m.resetBackupPollTimestamps()
}
}
// SetPBSPollingInterval updates the PBS polling cadence on the live monitor.
// The scheduler re-reads the base interval every cycle, so no further action
// is needed.
func (m *Monitor) SetPBSPollingInterval(interval time.Duration) {
if m == nil || interval <= 0 {
return
}
m.runtimePollingMu.Lock()
m.runtimePollingOverride.pbsInterval = &interval
m.runtimePollingMu.Unlock()
}
// SetPMGPollingInterval updates the PMG polling cadence on the live monitor.
// The scheduler re-reads the base interval every cycle, so no further action
// is needed.
func (m *Monitor) SetPMGPollingInterval(interval time.Duration) {
if m == nil || interval <= 0 {
return
}
m.runtimePollingMu.Lock()
m.runtimePollingOverride.pmgInterval = &interval
m.runtimePollingMu.Unlock()
}
// ResetSSHFailureBackoff clears the temperature collector's per-host SSH
// backoff and the knownhosts keyscan backoff. A system-settings save is an
// operator touchpoint that often follows repairing SSH access, and the
// on-disk key-change check only notices key file replacement, so the save
// itself also drops the backoff windows rather than making the operator wait
// one out (#1638).
func (m *Monitor) ResetSSHFailureBackoff() {
if m == nil {
return
}
m.tempCollector.ResetSSHFailures()
}
// resetBackupPollTimestamps clears the per-instance backup poll timestamps
// so the next cycle polls immediately.
func (m *Monitor) resetBackupPollTimestamps() {
m.mu.Lock()
m.lastPVEBackupPoll = make(map[string]time.Time)
m.lastPBSBackupPoll = make(map[string]time.Time)
m.mu.Unlock()
}