Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
package api
import (
"context"
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
"github.com/rcourtman/pulse-go-rewrite/internal/config"
"github.com/rcourtman/pulse-go-rewrite/internal/mock"
"github.com/rcourtman/pulse-go-rewrite/internal/models"
"github.com/rcourtman/pulse-go-rewrite/internal/monitoring"
)
type aggregatorRuntimeSources struct {
orgID string
vmwarePoller *monitoring.VMwarePoller
truenasPoller *monitoring.TrueNASPoller
// platformRowsOnly leaves out what only the agent rows and the PBS host
// aliases read: the monitor's hosts and PBS instances. A caller that keeps
// the platform rows alone (the connection-degraded alert feed) would
// otherwise ask the monitor for both, and in mock mode each ask rebuilds
// the whole estate view whenever a metric tick has made it stale.
platformRowsOnly bool
}
func buildAggregatorInputsWithRuntimeSources(
ctx context.Context,
cfg *config.Config,
persistence *config.ConfigPersistence,
monitor *monitoring.Monitor,
runtime aggregatorRuntimeSources,
) aggregatorInputs {
inputs := aggregatorInputs{now: time.Now()}
if cfg != nil {
inputs.pveInstances = cfg.PVEInstances
inputs.pbsInstances = cfg.PBSInstances
inputs.pmgInstances = cfg.PMGInstances
// A tenant org's agent tokens live in the instance inventory; its
// monitor sees them alongside its own config.
if monitor != nil && monitor.GetConfig() == cfg {
inputs.apiTokens = monitor.APITokenSnapshot()
} else {
inputs.apiTokens = append([]config.APITokenRecord(nil), cfg.APITokens...)
}
inputs.agentTokenInventoryKnown = true
inputs.pvePollingInterval = cfg.PVEPollingInterval
inputs.pbsPollingInterval = cfg.PBSPollingInterval
inputs.pmgPollingInterval = cfg.PMGPollingInterval
}
if persistence != nil {
if vmw, err := persistence.LoadVMwareConfig(); err == nil {
inputs.vmwareInstances = vmw
}
if tn, err := persistence.LoadTrueNASConfig(); err == nil {
inputs.truenasInstances = tn
}
if availability, err := persistence.LoadAvailabilityTargets(); err == nil {
inputs.availabilityTargets = availability
}
}
orgID := runtime.orgID
if orgID == "" {
orgID = "default"
}
if runtime.vmwarePoller != nil && len(inputs.vmwareInstances) > 0 {
inputs.vmwareSummaries = runtime.vmwarePoller.ConnectionSummaries(orgID, inputs.vmwareInstances)
}
if runtime.truenasPoller != nil && len(inputs.truenasInstances) > 0 {
inputs.truenasSummaries = runtime.truenasPoller.ConnectionSummaries(orgID, inputs.truenasInstances)
}
if monitor != nil {
if runtime.platformRowsOnly {
inputs.hosts = []models.Host{}
} else {
inputs.hosts = monitor.HostsSnapshot()
inputs.agentDesiredConfigs = connectionAgentDesiredConfigFingerprints(monitor, inputs.hosts, inputs.apiTokens)
inputs.pbsReportedNodeNames = pbsReportedNodeNamesByInstance(monitor.PBSInstancesSnapshot())
}
inputs.instanceHealth = instanceHealthByKey(monitor.SchedulerHealth())
inputs.availabilityStatuses = monitor.AvailabilityStatusSnapshot()
// A non-default org's cfg is its monitor's detached config copy
// (#1619), which a settings save does not update: saved PBS and PMG
// intervals reach the monitor only as runtime overrides. Scale the
// stale cutoff by the cadence the monitor actually polls at.
if interval := monitor.BasePollInterval(monitoring.InstanceTypePVE); interval > 0 {
inputs.pvePollingInterval = interval
}
if interval := monitor.BasePollInterval(monitoring.InstanceTypePBS); interval > 0 {
inputs.pbsPollingInterval = interval
}
if interval := monitor.BasePollInterval(monitoring.InstanceTypePMG); interval > 0 {
inputs.pmgPollingInterval = interval
}
inputs.plannedPollIntervals = plannedPollIntervalsForConfig(monitor, cfg)
} else {
inputs.hosts = []models.Host{}
inputs.instanceHealth = map[string]monitoring.InstanceHealth{}
inputs.availabilityStatuses = map[string]monitoring.AvailabilityProbeStatus{}
}
if mock.IsMockEnabled() {
inputs = applyMockLedgerInputs(inputs)
inputs.agentTokenInventoryKnown = false
}
inputs.expectedAgentVersion = currentAgentTargetVersion()
_ = ctx
return inputs
}
// connectionTypeForAlerts maps the wire ConnectionType to the alerts package's
// narrow platform set. Types that fall outside the platform set return false
// so the caller drops the row before invoking CheckConnection.
func connectionTypeForAlerts(t ConnectionType) (alerts.ConnectionType, bool) {
switch t {
case ConnectionTypePVE:
return alerts.ConnectionTypePVE, true
case ConnectionTypePBS:
return alerts.ConnectionTypePBS, true
case ConnectionTypePMG:
return alerts.ConnectionTypePMG, true
case ConnectionTypeVMware:
return alerts.ConnectionTypeVMware, true
case ConnectionTypeTrueNAS:
return alerts.ConnectionTypeTrueNAS, true
default:
return "", false
}
}
// snapshotConnectionsForAlerts translates the aggregator's rich Connection
// rows into the narrow alerts.ConnectionSnapshot view. Non-platform types
// (agent, availability, docker, kubernetes) are dropped because they have
// their own alert lifecycles.
func snapshotConnectionsForAlerts(connections []Connection) []alerts.ConnectionSnapshot {
out := make([]alerts.ConnectionSnapshot, 0, len(connections))
for _, conn := range connections {
alertType, ok := connectionTypeForAlerts(conn.Type)
if !ok {
continue
}
snap := alerts.ConnectionSnapshot{
ID: conn.ID,
PolicyResourceID: conn.alertPolicyResourceID,
Name: conn.Name,
Type: alertType,
State: alerts.ConnectionState(conn.State),
StateReason: conn.StateReason,
Enabled: conn.Enabled,
LastSeen: conn.LastSeen,
}
if conn.LastError != nil {
snap.LastError = &alerts.ConnectionErrorSnapshot{
At: conn.LastError.At,
Message: conn.LastError.Message,
Category: conn.LastError.Category,
}
}
out = append(out, snap)
}
return out
}
// installConnectionAlertLister gives monitor the closure its poll tick uses to
// raise and resolve connection-degraded alerts, bound to that monitor. Every
// path that attaches a monitor to the router calls it through
// configureMonitorDependencies: startup, SetMonitor after a full reload
// (node auto-registration replaces the default monitor), and the
// multi-tenant initializer that lazily creates the default monitor. A lister
// installed once at startup stays on the discarded monitor, so the
// replacement would never evaluate connection alerts again until restart.
//
// Only the default org's monitor gets a lister: the closure reads the
// default org's connection config, and per-org connection alerts are not
// wired yet. Until the platform pollers exist the monitor keeps whatever
// lister it had, so a running monitor never sees nil runtime owners.
func (r *Router) installConnectionAlertLister(monitor *monitoring.Monitor) {
if r == nil || monitor == nil || r.configHandlers == nil || r.trueNASPoller == nil || r.vmwarePoller == nil {
return
}
if orgID := monitor.GetOrgID(); orgID != "" && orgID != "default" {
return
}
getCfg := r.configHandlers.Config
getPersist := r.configHandlers.Persistence
runtime := aggregatorRuntimeSources{
orgID: "default",
truenasPoller: r.trueNASPoller,
vmwarePoller: r.vmwarePoller,
}
monitor.SetConnectionsSnapshotLister(func() []alerts.ConnectionSnapshot {
ctx := context.Background()
return buildAlertConnectionSnapshotsWithRuntimeSources(ctx, getCfg(ctx), getPersist(ctx), monitor, runtime)
})
}
func buildAlertConnectionSnapshotsWithRuntimeSources(
ctx context.Context,
cfg *config.Config,
persistence *config.ConfigPersistence,
monitor *monitoring.Monitor,
runtime aggregatorRuntimeSources,
) []alerts.ConnectionSnapshot {
// snapshotConnectionsForAlerts keeps the platform rows (PVE, PBS, PMG,
// VMware, TrueNAS) and drops every agent row, and a PBS row's reported node
// name only becomes a host alias that the snapshot does not carry.
runtime.platformRowsOnly = true
inputs := buildAggregatorInputsWithRuntimeSources(ctx, cfg, persistence, monitor, runtime)
return snapshotConnectionsForAlerts(buildConnections(inputs))
}
// plannedPollIntervalsForConfig collects the adaptive scheduler's currently
// planned interval for every configured PVE/PBS/PMG instance, keyed the same
// way as instanceHealth ("pve::<name>"). Instances the scheduler has no plan
// for are omitted so the aggregator falls back to the configured cadence.
func plannedPollIntervalsForConfig(monitor *monitoring.Monitor, cfg *config.Config) map[string]time.Duration {
if monitor == nil || cfg == nil {
return nil
}
out := make(map[string]time.Duration)
record := func(instanceType monitoring.InstanceType, keyPrefix, name string) {
if interval := monitor.PlannedPollInterval(instanceType, name); interval > 0 {
out[keyPrefix+name] = interval
}
}
for _, inst := range cfg.PVEInstances {
record(monitoring.InstanceTypePVE, "pve::", inst.Name)
}
for _, inst := range cfg.PBSInstances {
record(monitoring.InstanceTypePBS, "pbs::", inst.Name)
}
for _, inst := range cfg.PMGInstances {
record(monitoring.InstanceTypePMG, "pmg::", inst.Name)
}
return out
}