Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
package monitoring
import (
"context"
stderrors "errors"
"fmt"
"net"
"net/http"
"net/url"
"strings"
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/config"
"github.com/rcourtman/pulse-go-rewrite/internal/models"
"github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"
"github.com/rs/zerolog/log"
)
func resolveNodeConnectionInfo(instanceCfg *config.PVEInstance, discoveryCfg config.DiscoveryConfig, nodeName string) (string, string) {
connectionHost := instanceCfg.Host
guestURL := instanceCfg.GuestURL
if instanceCfg.IsCluster && len(instanceCfg.ClusterEndpoints) > 0 {
hasFingerprint := instanceCfg.Fingerprint != ""
for _, ep := range instanceCfg.ClusterEndpoints {
if strings.EqualFold(ep.NodeName, nodeName) {
if effective := clusterEndpointRuntimeURL(ep, instanceCfg.VerifySSL, hasFingerprint, discoveryCfg); effective != "" {
connectionHost = effective
}
if ep.GuestURL != "" {
guestURL = ep.GuestURL
} else if guestURL == "" {
guestURL = clusterMemberBrowserURL(ep, connectionHost)
}
break
}
}
}
return connectionHost, guestURL
}
// clusterMemberBrowserURL gives a cluster member's web interface link the
// address Proxmox reports for that node when Pulse dials it by a bare
// hostname. Strict TLS without a per-member pin dials by name so the
// certificate can be validated, but a browser often cannot resolve that
// label, and the frontend would otherwise guess an address from the node's
// interfaces, which picks bridges every member shares.
func clusterMemberBrowserURL(endpoint config.ClusterEndpoint, connectionHost string) string {
ip := net.ParseIP(strings.TrimSpace(endpoint.EffectiveIP()))
if ip == nil {
return ""
}
parsed, err := url.Parse(connectionHost)
if err != nil || parsed.Hostname() == "" {
return ""
}
host := parsed.Hostname()
if net.ParseIP(host) != nil || strings.Contains(host, ".") {
return ""
}
port := parsed.Port()
if port == "" {
port = "8006"
}
scheme := parsed.Scheme
if scheme == "" {
scheme = "https"
}
return scheme + "://" + net.JoinHostPort(ip.String(), port)
}
func (m *Monitor) determineNodeIDAndStatus(instanceName string, instanceCfg *config.PVEInstance, node proxmox.Node) (string, string) {
// Apply grace period for node status to prevent flapping
// The configured instance is the canonical provider identity. Normally a
// cluster name remains the stable scope used by earlier Pulse releases,
// but two unrelated clusters are allowed to share that display name. In
// that ambiguous case the instance name prevents cross-cluster resource
// collisions without churning IDs for established unambiguous clusters.
nodeID := m.pveNodeID(instanceName, instanceCfg, node.Node)
effectiveStatus := node.Status
now := time.Now()
gracePeriod := m.pveNodeOfflineGracePeriod()
m.mu.Lock()
if strings.ToLower(node.Status) == "online" {
// Node is online - update last-online timestamp
m.nodeLastOnline[nodeID] = now
} else {
// Node is reported as offline - check grace period
lastOnline, exists := m.nodeLastOnline[nodeID]
if exists && now.Sub(lastOnline) < gracePeriod {
// Still within grace period - preserve online status
effectiveStatus = "online"
log.Debug().
Str("instance", instanceName).
Str("node", node.Node).
Dur("timeSinceOnline", now.Sub(lastOnline)).
Dur("gracePeriod", gracePeriod).
Msg("Node offline but within grace period - preserving online status")
} else {
// Grace period expired or never seen online - mark as offline
if exists {
log.Info().
Str("instance", instanceName).
Str("node", node.Node).
Dur("timeSinceOnline", now.Sub(lastOnline)).
Msg("Node offline and grace period expired - marking as offline")
}
}
}
m.mu.Unlock()
return nodeID, effectiveStatus
}
func (m *Monitor) pveNodeID(instanceName string, instanceCfg *config.PVEInstance, nativeName string) string {
if instanceCfg != nil && instanceCfg.IsCluster {
if identityID := config.PVEClusterNodeIdentityForName(instanceCfg, nativeName); identityID != "" {
return identityID
}
}
return m.pveNodeIdentityScope(instanceName, instanceCfg) + "-" + nativeName
}
// pveNodeTLSFingerprint resolves the TOFU-captured TLS certificate
// fingerprint recorded in configuration for one node's API endpoint. The
// fingerprint is per-machine identity evidence for node state aggregation, so
// a cluster node only inherits the fingerprint of its own named endpoint -
// never the instance-level fingerprint, which pins whichever member the
// connection URL happens to reach. Unknown stays empty and asserts nothing.
func pveNodeTLSFingerprint(instanceCfg *config.PVEInstance, nodeName string) string {
if instanceCfg == nil {
return ""
}
if !instanceCfg.IsCluster {
return strings.TrimSpace(instanceCfg.Fingerprint)
}
nodeName = strings.TrimSpace(nodeName)
match := ""
for _, endpoint := range instanceCfg.ClusterEndpoints {
if !strings.EqualFold(strings.TrimSpace(endpoint.NodeName), nodeName) {
continue
}
fingerprint := strings.TrimSpace(endpoint.Fingerprint)
if fingerprint == "" {
continue
}
if match != "" && !strings.EqualFold(match, fingerprint) {
// Ambiguous duplicate endpoint records fail unknown.
return ""
}
match = fingerprint
}
return match
}
func (m *Monitor) pveNodeUsesProviderScopedIdentity(instanceName string, instanceCfg *config.PVEInstance) bool {
if instanceCfg == nil || !instanceCfg.IsCluster || strings.TrimSpace(instanceCfg.ClusterName) == "" {
return false
}
return !strings.EqualFold(
strings.TrimSpace(m.pveNodeIdentityScope(instanceName, instanceCfg)),
strings.TrimSpace(instanceCfg.ClusterName),
)
}
// nodeTemperatureCarryFloor is the shortest time a node may keep showing its
// last temperature after collection stops returning one, sized to ride out the
// SSH collector's early failure backoff steps (30s, 1m, 2m).
const nodeTemperatureCarryFloor = 5 * time.Minute
// nodeTemperatureCarryWindow bounds how old a carried-over node temperature may
// be. Beyond it the reading no longer describes the node, so it stops being
// presented as available. It follows the PVE cadence the resource freshness
// and node offline grace use, so a node preserved inside that grace keeps a
// reading from its last healthy poll.
func (m *Monitor) nodeTemperatureCarryWindow() time.Duration {
return resourceStaleThresholdForPollInterval(m.resourcePollIntervals().pve, nodeTemperatureCarryFloor)
}
func (m *Monitor) collectNodeTemperatureData(
ctx context.Context,
instanceName string,
instanceCfg *config.PVEInstance,
node proxmox.Node,
modelNode *models.Node,
prevInstanceNodes []models.Node,
effectiveStatus string,
) {
if modelNode == nil {
return
}
// Collect temperature data via SSH (non-blocking, best effort)
// Only attempt for online nodes when temperature monitoring is enabled
// Check per-node setting first, fall back to global setting
tempMonitoringEnabled := m.config.TemperatureMonitoringEnabled
if instanceCfg.TemperatureMonitoringEnabled != nil {
tempMonitoringEnabled = *instanceCfg.TemperatureMonitoringEnabled
}
if effectiveStatus == "online" && tempMonitoringEnabled {
// First, check if there's a matching host agent with temperature data.
// Host agent temperatures are preferred because they don't require SSH access.
// Match by the polled node's identity, not its name, so two connections'
// same-named nodes (e.g., "px1" at two sites) never share an agent.
hostAgentTemp := m.getHostAgentTemperatureForNode(*modelNode)
if hostAgentTemp != nil {
log.Debug().
Str("node", node.Node).
Float64("cpuPackage", hostAgentTemp.CPUPackage).
Float64("cpuMax", hostAgentTemp.CPUMax).
Int("nvmeCount", len(hostAgentTemp.NVMe)).
Msg("Using temperature data from host agent")
}
// If no host agent temp or we need additional data (SMART), try SSH/proxy collection
var sshTemp *models.Temperature
var err error
if m.tempCollector != nil {
// Temperature collection is best-effort - use a short timeout to avoid blocking node polling
// Use context.Background() so the timeout is truly independent of the parent polling context
// If SSH is slow or unresponsive, we'll preserve previous temperature data
tempCtx, tempCancel := context.WithTimeout(context.Background(), 10*time.Second)
defer tempCancel()
// Determine SSH hostname to use (most robust approach):
// Prefer the resolved host for this node, with cluster overrides when available.
sshHost := modelNode.Host
foundNodeEndpoint := false
if modelNode.IsClusterMember && instanceCfg.IsCluster {
// Try to find specific endpoint configuration for this node
if len(instanceCfg.ClusterEndpoints) > 0 {
hasFingerprint := instanceCfg.Fingerprint != ""
for _, ep := range instanceCfg.ClusterEndpoints {
if strings.EqualFold(ep.NodeName, node.Node) {
if effective := clusterEndpointEffectiveURL(ep, instanceCfg.VerifySSL, hasFingerprint); effective != "" {
sshHost = effective
foundNodeEndpoint = true
}
break
}
}
}
// If no specific endpoint found, fall back to node name
if !foundNodeEndpoint {
sshHost = node.Node
log.Debug().
Str("node", node.Node).
Str("instance", instanceCfg.Name).
Msg("Node endpoint not found in cluster metadata - falling back to node name for temperature collection")
}
}
if strings.TrimSpace(sshHost) == "" {
sshHost = node.Node
}
// A recent host agent payload is authoritative for this node.
// Legacy SSH remains only as a fallback when no usable local
// temperature reading has arrived from the linked agent.
skipSSHCollection := shouldSkipTemperatureSSHCollection(hostAgentTemp)
if !skipSSHCollection {
sshTemp, err = m.tempCollector.CollectTemperature(tempCtx, sshHost, node.Node)
if err != nil && hostAgentTemp == nil {
log.Debug().
Str("node", node.Node).
Str("sshHost", sshHost).
Bool("isCluster", modelNode.IsClusterMember).
Int("endpointCount", len(instanceCfg.ClusterEndpoints)).
Msg("Temperature collection failed - check SSH access")
}
}
// Debug: log SSH temp details before merge
if sshTemp != nil {
log.Debug().
Str("node", node.Node).
Bool("sshTempAvailable", sshTemp.Available).
Bool("sshHasSMART", sshTemp.HasSMART).
Int("sshSMARTCount", len(sshTemp.SMART)).
Bool("sshHasNVMe", sshTemp.HasNVMe).
Int("sshNVMeCount", len(sshTemp.NVMe)).
Msg("SSH temperature data before merge")
} else {
log.Debug().
Str("node", node.Node).
Msg("SSH temperature data is nil")
}
}
// Merge host agent and SSH temperatures
temp := mergeTemperatureData(hostAgentTemp, sshTemp)
if temp != nil && temp.Available {
// Track the low and record from the primary reading, so the
// record is never below the headline the node shows.
currentTemp := 0.0
if primary := models.PrimaryCPUTemperatureCelsius(temp); primary != nil {
currentTemp = *primary
}
// Find previous temperature data for this node to preserve min/max
var prevTemp *models.Temperature
for _, prevNode := range prevInstanceNodes {
if prevNode.ID == modelNode.ID && prevNode.Temperature != nil {
prevTemp = prevNode.Temperature
break
}
}
// Initialize or update min/max tracking
if prevTemp != nil && prevTemp.CPUMin > 0 {
// Preserve existing min/max and update if necessary
temp.CPUMin = prevTemp.CPUMin
temp.CPUMaxRecord = prevTemp.CPUMaxRecord
temp.MinRecorded = prevTemp.MinRecorded
temp.MaxRecorded = prevTemp.MaxRecorded
// Update min if current is lower
if currentTemp > 0 && currentTemp < temp.CPUMin {
temp.CPUMin = currentTemp
temp.MinRecorded = time.Now()
}
// Update max if current is higher
if currentTemp > temp.CPUMaxRecord {
temp.CPUMaxRecord = currentTemp
temp.MaxRecorded = time.Now()
}
} else if currentTemp > 0 {
// First reading - initialize min/max to current value
temp.CPUMin = currentTemp
temp.CPUMaxRecord = currentTemp
temp.MinRecorded = time.Now()
temp.MaxRecorded = time.Now()
}
modelNode.Temperature = temp
// Determine source for logging
tempSource := "ssh"
if hostAgentTemp != nil && sshTemp == nil {
tempSource = "agent"
} else if hostAgentTemp != nil && sshTemp != nil {
tempSource = "agent+ssh"
}
log.Debug().
Str("node", node.Node).
Str("source", tempSource).
Float64("cpuPackage", temp.CPUPackage).
Float64("cpuMax", temp.CPUMax).
Float64("cpuMin", temp.CPUMin).
Float64("cpuMaxRecord", temp.CPUMaxRecord).
Int("nvmeCount", len(temp.NVMe)).
Msg("Collected temperature data")
} else {
// Temperature data returned but not available (temp != nil && !temp.Available)
// OR no temperature data from any source - preserve previous temperature if available
// This prevents the temperature column from flickering when collection temporarily fails
var prevTemp *models.Temperature
for _, prevNode := range prevInstanceNodes {
if prevNode.ID == modelNode.ID && prevNode.Temperature != nil && prevNode.Temperature.Available {
prevTemp = prevNode.Temperature
break
}
}
if m.carriedTemperatureOutlivesAgentLease(*modelNode, prevTemp, time.Now()) {
// The reading came from a host agent that has stopped reporting.
// Its reporting lease already rode out transient gaps, so
// carrying it would outlive that lease.
log.Debug().
Str("node", node.Node).
Time("lastUpdate", prevTemp.LastUpdate).
Msg("Dropped temperature data (host agent stopped reporting)")
} else if prevTemp != nil && time.Since(prevTemp.LastUpdate) > m.nodeTemperatureCarryWindow() {
// Collection has stayed broken past the carry window. Re-presenting
// the old reading would let alerts, history and the UI treat it as
// a live measurement forever, so the node has no reading now.
log.Debug().
Str("node", node.Node).
Bool("isCluster", modelNode.IsClusterMember).
Time("lastUpdate", prevTemp.LastUpdate).
Msg("Dropped stale temperature data (collection has not returned a reading within the carry window)")
} else if preserved := carriedNodeTemperature(prevTemp); preserved != nil {
// The carried copy keeps its original LastUpdate to indicate staleness.
modelNode.Temperature = preserved
log.Debug().
Str("node", node.Node).
Bool("isCluster", modelNode.IsClusterMember).
Float64("cpuPackage", preserved.CPUPackage).
Time("lastUpdate", preserved.LastUpdate).
Msg("Preserved previous temperature data (current collection failed or unavailable)")
} else {
log.Debug().
Str("node", node.Node).
Bool("isCluster", modelNode.IsClusterMember).
Msg("No temperature data available (collection failed, no previous data to preserve)")
}
}
}
}
// carriedNodeTemperature returns the part of an earlier reading a node may
// keep presenting while collection fails: its CPU and GPU temperatures, with
// the original LastUpdate. Disk temperatures (SMART, NVMe) are left out. They
// also stamp the node's physical disks, which have their own freshness and
// standby state that a carried copy cannot follow: a disk that spun down would
// be stamped, and recorded, with its pre-sleep reading. Returns nil when the
// earlier reading held nothing else.
func carriedNodeTemperature(prev *models.Temperature) *models.Temperature {
if prev == nil || !prev.Available {
return nil
}
carried := *prev
carried.Cores = append([]models.CoreTemp(nil), prev.Cores...)
carried.GPU = append([]models.GPUTemp(nil), prev.GPU...)
carried.SMART = []models.DiskTemp{}
carried.HasSMART = false
carried.NVMe = []models.NVMeTemp{}
carried.HasNVMe = false
if !hasUsableTemperatureReading(&carried) {
return nil
}
return &carried
}
func (m *Monitor) applyNodePendingUpdates(ctx context.Context, instanceName string, client PVEClientInterface, node proxmox.Node, nodeID string, effectiveStatus string, modelNode *models.Node) {
if modelNode == nil {
return
}
modelNode.PendingUpdatesStatus = "not_checked"
modelNode.PendingUpdatesReason = ""
if effectiveStatus != "online" {
modelNode.PendingUpdatesReason = "node_offline"
return
}
// Successful observations and explicit access denials have separate clocks.
// Reusing a denial must not turn an old count (including zero) current.
now := time.Now()
m.mu.RLock()
cached := m.nodePendingUpdatesCache[nodeID]
m.mu.RUnlock()
if !cached.deniedAt.IsZero() && !cached.deniedAt.After(now) && now.Sub(cached.deniedAt) < pendingUpdatesCacheTTL {
modelNode.PendingUpdatesReason = "permission_denied"
} else if !cached.checkedAt.IsZero() && now.Sub(cached.checkedAt) < pendingUpdatesCacheTTL {
modelNode.PendingUpdates = cached.count
modelNode.PendingUpdatesCheckedAt = cached.checkedAt
modelNode.PendingUpdatesStatus = "checked"
return
} else {
pendingPkgs, err := client.GetNodePendingUpdates(ctx, node.Node)
if err == nil {
modelNode.PendingUpdates = len(pendingPkgs)
modelNode.PendingUpdatesCheckedAt = now
modelNode.PendingUpdatesStatus = "checked"
m.mu.Lock()
if m.nodePendingUpdatesCache == nil {
m.nodePendingUpdatesCache = make(map[string]pendingUpdatesCache)
}
m.nodePendingUpdatesCache[nodeID] = pendingUpdatesCache{count: len(pendingPkgs), checkedAt: now}
m.mu.Unlock()
return
}
log.Debug().Err(err).Str("node", node.Node).Str("instance", instanceName).
Msg("Could not check pending apt updates with configured Proxmox credentials")
modelNode.PendingUpdatesReason = pendingUpdatesFailureReason(err)
if status, known := proxmox.APIErrorStatus(err); known && status == http.StatusForbidden {
// A quoted status in an error body is not access-denial evidence.
// Keep ordinary transient failures on their existing retry path.
cached.deniedAt = now
m.mu.Lock()
if m.nodePendingUpdatesCache == nil {
m.nodePendingUpdatesCache = make(map[string]pendingUpdatesCache)
}
m.nodePendingUpdatesCache[nodeID] = cached
m.mu.Unlock()
}
}
modelNode.PendingUpdates = cached.count
modelNode.PendingUpdatesCheckedAt = cached.checkedAt
modelNode.PendingUpdatesStatus = "unavailable"
if !cached.checkedAt.IsZero() {
modelNode.PendingUpdatesStatus = "stale"
}
}
func pendingUpdatesFailureReason(err error) string {
if err == nil {
return ""
}
if status, known := proxmox.APIErrorStatus(err); known {
switch {
case status == http.StatusForbidden:
return "permission_denied"
case status >= http.StatusInternalServerError:
return "source_unavailable"
default:
return "check_failed"
}
}
message := strings.ToLower(err.Error())
switch {
case strings.Contains(message, "403"), strings.Contains(message, "permission"):
return "permission_denied"
case strings.Contains(message, "no healthy nodes"), strings.Contains(message, "595"),
strings.Contains(message, "offline"), strings.Contains(message, "unavailable"),
strings.Contains(message, "connection"), strings.Contains(message, "timeout"):
return "source_unavailable"
default:
return "check_failed"
}
}
func (m *Monitor) recordNodePollMetrics(instanceName string, node proxmox.Node, modelNode *models.Node, nodeStart time.Time) {
if m.pollMetrics == nil || modelNode == nil {
return
}
nodeNameLabel := strings.TrimSpace(node.Node)
if nodeNameLabel == "" {
nodeNameLabel = strings.TrimSpace(modelNode.DisplayName)
}
if nodeNameLabel == "" {
nodeNameLabel = "unknown-node"
}
success := true
nodeErrReason := ""
health := strings.ToLower(strings.TrimSpace(modelNode.ConnectionHealth))
if health != "" && health != "healthy" {
success = false
nodeErrReason = fmt.Sprintf("connection health %s", health)
}
status := strings.ToLower(strings.TrimSpace(modelNode.Status))
if success && status != "" && status != "online" {
success = false
nodeErrReason = fmt.Sprintf("status %s", status)
}
var nodeErr error
if !success {
if nodeErrReason == "" {
nodeErrReason = "unknown node error"
}
nodeErr = stderrors.New(nodeErrReason)
}
m.pollMetrics.RecordNodeResult(NodePollResult{
InstanceName: instanceName,
InstanceType: "pve",
NodeName: nodeNameLabel,
Success: success,
Error: nodeErr,
StartTime: nodeStart,
EndTime: time.Now(),
})
}