Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

9.0 KB
Raw Blame
package api

import (
	"context"
	"time"

	"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
	"github.com/rcourtman/pulse-go-rewrite/internal/config"
	"github.com/rcourtman/pulse-go-rewrite/internal/mock"
	"github.com/rcourtman/pulse-go-rewrite/internal/models"
	"github.com/rcourtman/pulse-go-rewrite/internal/monitoring"
)

type aggregatorRuntimeSources struct {
	orgID         string
	vmwarePoller  *monitoring.VMwarePoller
	truenasPoller *monitoring.TrueNASPoller
	// platformRowsOnly leaves out what only the agent rows and the PBS host
	// aliases read: the monitor's hosts and PBS instances. A caller that keeps
	// the platform rows alone (the connection-degraded alert feed) would
	// otherwise ask the monitor for both, and in mock mode each ask rebuilds
	// the whole estate view whenever a metric tick has made it stale.
	platformRowsOnly bool
}

func buildAggregatorInputsWithRuntimeSources(
	ctx context.Context,
	cfg *config.Config,
	persistence *config.ConfigPersistence,
	monitor *monitoring.Monitor,
	runtime aggregatorRuntimeSources,
) aggregatorInputs {
	inputs := aggregatorInputs{now: time.Now()}

	if cfg != nil {
		inputs.pveInstances = cfg.PVEInstances
		inputs.pbsInstances = cfg.PBSInstances
		inputs.pmgInstances = cfg.PMGInstances
		// A tenant org's agent tokens live in the instance inventory; its
		// monitor sees them alongside its own config.
		if monitor != nil && monitor.GetConfig() == cfg {
			inputs.apiTokens = monitor.APITokenSnapshot()
		} else {
			inputs.apiTokens = append([]config.APITokenRecord(nil), cfg.APITokens...)
		}
		inputs.agentTokenInventoryKnown = true
		inputs.pvePollingInterval = cfg.PVEPollingInterval
		inputs.pbsPollingInterval = cfg.PBSPollingInterval
		inputs.pmgPollingInterval = cfg.PMGPollingInterval
	}

	if persistence != nil {
		if vmw, err := persistence.LoadVMwareConfig(); err == nil {
			inputs.vmwareInstances = vmw
		}
		if tn, err := persistence.LoadTrueNASConfig(); err == nil {
			inputs.truenasInstances = tn
		}
		if availability, err := persistence.LoadAvailabilityTargets(); err == nil {
			inputs.availabilityTargets = availability
		}
	}

	orgID := runtime.orgID
	if orgID == "" {
		orgID = "default"
	}
	if runtime.vmwarePoller != nil && len(inputs.vmwareInstances) > 0 {
		inputs.vmwareSummaries = runtime.vmwarePoller.ConnectionSummaries(orgID, inputs.vmwareInstances)
	}
	if runtime.truenasPoller != nil && len(inputs.truenasInstances) > 0 {
		inputs.truenasSummaries = runtime.truenasPoller.ConnectionSummaries(orgID, inputs.truenasInstances)
	}

	if monitor != nil {
		if runtime.platformRowsOnly {
			inputs.hosts = []models.Host{}
		} else {
			inputs.hosts = monitor.HostsSnapshot()
			inputs.agentDesiredConfigs = connectionAgentDesiredConfigFingerprints(monitor, inputs.hosts, inputs.apiTokens)
			inputs.pbsReportedNodeNames = pbsReportedNodeNamesByInstance(monitor.PBSInstancesSnapshot())
		}
		inputs.instanceHealth = instanceHealthByKey(monitor.SchedulerHealth())
		inputs.availabilityStatuses = monitor.AvailabilityStatusSnapshot()
		// A non-default org's cfg is its monitor's detached config copy
		// (#1619), which a settings save does not update: saved PBS and PMG
		// intervals reach the monitor only as runtime overrides. Scale the
		// stale cutoff by the cadence the monitor actually polls at.
		if interval := monitor.BasePollInterval(monitoring.InstanceTypePVE); interval > 0 {
			inputs.pvePollingInterval = interval
		}
		if interval := monitor.BasePollInterval(monitoring.InstanceTypePBS); interval > 0 {
			inputs.pbsPollingInterval = interval
		}
		if interval := monitor.BasePollInterval(monitoring.InstanceTypePMG); interval > 0 {
			inputs.pmgPollingInterval = interval
		}
		inputs.plannedPollIntervals = plannedPollIntervalsForConfig(monitor, cfg)
	} else {
		inputs.hosts = []models.Host{}
		inputs.instanceHealth = map[string]monitoring.InstanceHealth{}
		inputs.availabilityStatuses = map[string]monitoring.AvailabilityProbeStatus{}
	}
	if mock.IsMockEnabled() {
		inputs = applyMockLedgerInputs(inputs)
		inputs.agentTokenInventoryKnown = false
	}
	inputs.expectedAgentVersion = currentAgentTargetVersion()
	_ = ctx
	return inputs
}

// connectionTypeForAlerts maps the wire ConnectionType to the alerts package's
// narrow platform set. Types that fall outside the platform set return false
// so the caller drops the row before invoking CheckConnection.
func connectionTypeForAlerts(t ConnectionType) (alerts.ConnectionType, bool) {
	switch t {
	case ConnectionTypePVE:
		return alerts.ConnectionTypePVE, true
	case ConnectionTypePBS:
		return alerts.ConnectionTypePBS, true
	case ConnectionTypePMG:
		return alerts.ConnectionTypePMG, true
	case ConnectionTypeVMware:
		return alerts.ConnectionTypeVMware, true
	case ConnectionTypeTrueNAS:
		return alerts.ConnectionTypeTrueNAS, true
	default:
		return "", false
	}
}

// snapshotConnectionsForAlerts translates the aggregator's rich Connection
// rows into the narrow alerts.ConnectionSnapshot view. Non-platform types
// (agent, availability, docker, kubernetes) are dropped because they have
// their own alert lifecycles.
func snapshotConnectionsForAlerts(connections []Connection) []alerts.ConnectionSnapshot {
	out := make([]alerts.ConnectionSnapshot, 0, len(connections))
	for _, conn := range connections {
		alertType, ok := connectionTypeForAlerts(conn.Type)
		if !ok {
			continue
		}

		snap := alerts.ConnectionSnapshot{
			ID:               conn.ID,
			PolicyResourceID: conn.alertPolicyResourceID,
			Name:             conn.Name,
			Type:             alertType,
			State:            alerts.ConnectionState(conn.State),
			StateReason:      conn.StateReason,
			Enabled:          conn.Enabled,
			LastSeen:         conn.LastSeen,
		}
		if conn.LastError != nil {
			snap.LastError = &alerts.ConnectionErrorSnapshot{
				At:       conn.LastError.At,
				Message:  conn.LastError.Message,
				Category: conn.LastError.Category,
			}
		}
		out = append(out, snap)
	}
	return out
}

// installConnectionAlertLister gives monitor the closure its poll tick uses to
// raise and resolve connection-degraded alerts, bound to that monitor. Every
// path that attaches a monitor to the router calls it through
// configureMonitorDependencies: startup, SetMonitor after a full reload
// (node auto-registration replaces the default monitor), and the
// multi-tenant initializer that lazily creates the default monitor. A lister
// installed once at startup stays on the discarded monitor, so the
// replacement would never evaluate connection alerts again until restart.
//
// Only the default org's monitor gets a lister: the closure reads the
// default org's connection config, and per-org connection alerts are not
// wired yet. Until the platform pollers exist the monitor keeps whatever
// lister it had, so a running monitor never sees nil runtime owners.
func (r *Router) installConnectionAlertLister(monitor *monitoring.Monitor) {
	if r == nil || monitor == nil || r.configHandlers == nil || r.trueNASPoller == nil || r.vmwarePoller == nil {
		return
	}
	if orgID := monitor.GetOrgID(); orgID != "" && orgID != "default" {
		return
	}
	getCfg := r.configHandlers.Config
	getPersist := r.configHandlers.Persistence
	runtime := aggregatorRuntimeSources{
		orgID:         "default",
		truenasPoller: r.trueNASPoller,
		vmwarePoller:  r.vmwarePoller,
	}
	monitor.SetConnectionsSnapshotLister(func() []alerts.ConnectionSnapshot {
		ctx := context.Background()
		return buildAlertConnectionSnapshotsWithRuntimeSources(ctx, getCfg(ctx), getPersist(ctx), monitor, runtime)
	})
}

func buildAlertConnectionSnapshotsWithRuntimeSources(
	ctx context.Context,
	cfg *config.Config,
	persistence *config.ConfigPersistence,
	monitor *monitoring.Monitor,
	runtime aggregatorRuntimeSources,
) []alerts.ConnectionSnapshot {
	// snapshotConnectionsForAlerts keeps the platform rows (PVE, PBS, PMG,
	// VMware, TrueNAS) and drops every agent row, and a PBS row's reported node
	// name only becomes a host alias that the snapshot does not carry.
	runtime.platformRowsOnly = true
	inputs := buildAggregatorInputsWithRuntimeSources(ctx, cfg, persistence, monitor, runtime)
	return snapshotConnectionsForAlerts(buildConnections(inputs))
}

// plannedPollIntervalsForConfig collects the adaptive scheduler's currently
// planned interval for every configured PVE/PBS/PMG instance, keyed the same
// way as instanceHealth ("pve::<name>"). Instances the scheduler has no plan
// for are omitted so the aggregator falls back to the configured cadence.
func plannedPollIntervalsForConfig(monitor *monitoring.Monitor, cfg *config.Config) map[string]time.Duration {
	if monitor == nil || cfg == nil {
		return nil
	}
	out := make(map[string]time.Duration)
	record := func(instanceType monitoring.InstanceType, keyPrefix, name string) {
		if interval := monitor.PlannedPollInterval(instanceType, name); interval > 0 {
			out[keyPrefix+name] = interval
		}
	}
	for _, inst := range cfg.PVEInstances {
		record(monitoring.InstanceTypePVE, "pve::", inst.Name)
	}
	for _, inst := range cfg.PBSInstances {
		record(monitoring.InstanceTypePBS, "pbs::", inst.Name)
	}
	for _, inst := range cfg.PMGInstances {
		record(monitoring.InstanceTypePMG, "pmg::", inst.Name)
	}
	return out
}