Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Hand a node only the usage metrics its agent evaluates
rcourtman9206486d56e2
diff --git a/docs/release-control/v6/internal/subsystems/alerts.md b/docs/release-control/v6/internal/subsystems/alerts.mdindex c385b63579..aca39a8060 100644--- a/docs/release-control/v6/internal/subsystems/alerts.md+++ b/docs/release-control/v6/internal/subsystems/alerts.md@@ -1803,11 +1803,16 @@ host-agent deduplication bookkeeping, and instance-scoped node display-namecache updates; future Proxmox node alert behavior should extend that resourcechecker owner rather than expanding the central Manager file.When a reporting host agent is linked to a node (`LinkedNodeID`, the monitor's-automatic or operator-set identity decision), the agent resource owns that-machine's CPU, memory and disk usage alerts, and `CheckNode` releases its own-copies through the disabled-threshold path every cycle: the pending run is-dropped and any node alert still open from before the link is resolved, never-left frozen until the agent goes offline. Deduplication keys on that link,+automatic or operator-set identity decision), the agent resource owns each+CPU, memory or disk usage metric it actually evaluates, and `CheckNode` releases+its own copy of those metrics through the disabled-threshold path every cycle:+the pending run is dropped and any node alert still open from before the link is+resolved, never left frozen until the agent goes offline. `CheckHost` registers+the link only after evaluation, recording which metrics had a live threshold+(memory also needs a known reading, disk at least one evaluated filesystem), and+removes it while agent alerts are disabled; a metric the agent does not evaluate+stays with the node, so deduplication never leaves a machine unmonitored.+Deduplication keys on that link,never on a hostname match, so a same-named node in another instance keeps itsalerts, an agent reporting an FQDN still dedups its node, and an operator unlinkhands the alerts back. CPU temperature has no host-agent metric, so the node@@ -1826,7 +1831,8 @@ instance name in `Instance`.`TestCheckNodeReleasesOpenMetricAlertWhenHostAgentRegisters`,`TestCheckNodeMissingTemperatureDoesNotResolveOpenAlert`,`TestCheckNodeMissingTemperatureInterruptsTimingRuns` and-`TestConfigSaveKeepsNodeTemperatureAlertOverTrigger` in+`TestConfigSaveKeepsNodeTemperatureAlertOverTrigger` and+`TestCheckNodeKeepsUsageMetricsTheAgentDoesNotEvaluate` in`internal/alerts/threshold_resolution_shared_test.go`, and`TestHostAgentDeduplicationFollowsNodeLink` in`internal/alerts/host_dedup_test.go`, pin these rules.diff --git a/internal/alerts/host.go b/internal/alerts/host.goindex 8e0e8c5516..718cc75f9a 100644--- a/internal/alerts/host.go+++ b/internal/alerts/host.go@@ -220,10 +220,6 @@ func (m *Manager) CheckHost(host models.Host) {return}- // Record the Proxmox node this agent is linked to, so the node releases the- // usage alerts the agent now owns instead of alerting twice for one machine.- m.registerHostAgentNodeLink(host)-// Cache display name so host alerts show the user-configured name.m.UpdateNodeDisplayName("", host.Hostname, host.DisplayName)@@ -242,11 +238,15 @@ func (m *Manager) CheckHost(host models.Host) {}m.mu.RUnlock()+ // While this agent evaluates nothing, its linked node keeps its own usage+ // alerts; the link is registered below once the evaluated metrics are known.if !alertsEnabled {+ m.unregisterHostAgentNodeLink(host.ID)return}if disableAllAgents {+ m.unregisterHostAgentNodeLink(host.ID)// Clear any existing host alerts when all host alerts are disabledm.clearHostMetricAlerts(host.ID)m.clearHostDiskAlerts(host.ID)@@ -257,6 +257,7 @@ func (m *Manager) CheckHost(host models.Host) {}if thresholds.Disabled {+ m.unregisterHostAgentNodeLink(host.ID)m.clearHostMetricAlerts(host.ID)m.clearHostDiskAlerts(host.ID)m.clearHostRAIDAlerts(host.ID)@@ -386,6 +387,7 @@ func (m *Manager) CheckHost(host models.Host) {}seenDisks := make(map[string]struct{}, len(host.Disks))+ evaluatesDiskUsage := falseif len(host.Sensors.SMART) > 0 {for _, disk := range host.Sensors.SMART {diskResourceID, diskName := hostSMARTDiskResourceID(host, disk)@@ -443,6 +445,7 @@ func (m *Manager) CheckHost(host models.Host) {if effectiveDiskThreshold == nil {continue}+ evaluatesDiskUsage = evaluatesDiskUsage || effectiveDiskThreshold.Trigger > 0diskMetadata := cloneMetadata(baseMetadata)diskMetadata["metric"] = "disk"@@ -490,6 +493,17 @@ func (m *Manager) CheckHost(host models.Host) {m.cleanupHostDiskAlerts(host, seenDisks)+ // The linked node releases exactly the usage metrics evaluated above and+ // keeps the rest, so deduplication never leaves the machine unmonitored.+ m.registerHostAgentNodeLink(hostAgentNodeLink{+ agentID: host.ID,+ agentName: resourceName,+ nodeID: host.LinkedNodeID,+ cpu: thresholds.CPU != nil && thresholds.CPU.Trigger > 0,+ memory: thresholds.Memory != nil && thresholds.Memory.Trigger > 0 && host.Memory.HasKnownUsage(),+ disk: evaluatesDiskUsage,+ })+if host.Unraid != nil {m.syncHostUnraidStorageAlert(host, nodeName, instanceName, resourceName, baseMetadata)} else {diff --git a/internal/alerts/host_dedup_test.go b/internal/alerts/host_dedup_test.goindex a0b4ac3751..aee90c38fa 100644--- a/internal/alerts/host_dedup_test.go+++ b/internal/alerts/host_dedup_test.go@@ -7,7 +7,7 @@ import ()func TestHostAgentDeduplicatesNodeAlerts(t *testing.T) {- agent := models.Host{ID: "host-pi", Hostname: "pi", LinkedNodeID: "node/pi"}+ agent := hostAgentNodeLink{agentID: "host-pi", nodeID: "node/pi", cpu: true, memory: true, disk: true}// Test 1: Without a linked host agent, node metrics ARE checkedt.Run("node_metrics_checked_without_agent", func(t *testing.T) {@@ -81,7 +81,7 @@ func TestHostAgentDeduplicatesNodeAlerts(t *testing.T) {// Register and then unregisterm.registerHostAgentNodeLink(agent)- m.unregisterHostAgentNodeLink(agent.ID)+ m.unregisterHostAgentNodeLink(agent.agentID)// Verify host agent is NOT linkedif m.hasHostAgentForNode("node/pi") {@@ -112,8 +112,9 @@ func TestHostAgentDeduplicatesNodeAlerts(t *testing.T) {// instances can both have a node named "pve", and an agent can report an FQDN.func TestHostAgentDeduplicationFollowsNodeLink(t *testing.T) {m := NewManager()+ m.config.Enabled = true- m.registerHostAgentNodeLink(models.Host{ID: "host-a", Hostname: "pve.example.com", LinkedNodeID: "site-a-pve"})+ m.CheckHost(models.Host{ID: "host-a", Hostname: "pve.example.com", LinkedNodeID: "site-a-pve", CPUUsage: 10})if !m.hasHostAgentForNode("site-a-pve") {t.Error("Expected the linked node to dedup even though the agent hostname is an FQDN")}@@ -121,13 +122,13 @@ func TestHostAgentDeduplicationFollowsNodeLink(t *testing.T) {t.Error("Expected a same-named node in another instance to keep its own alerts")}- m.registerHostAgentNodeLink(models.Host{ID: "host-b", Hostname: "pve"})+ m.CheckHost(models.Host{ID: "host-b", Hostname: "pve", CPUUsage: 10})if m.hasHostAgentForNode("site-b-pve") {t.Error("Expected an unlinked agent with a matching hostname not to suppress node alerts")}// An operator unlinking the agent hands the node's alerts back.- m.registerHostAgentNodeLink(models.Host{ID: "host-a", Hostname: "pve.example.com"})+ m.CheckHost(models.Host{ID: "host-a", Hostname: "pve.example.com", CPUUsage: 10})if m.hasHostAgentForNode("site-a-pve") {t.Error("Expected an unlinked agent to stop deduplicating its former node")}@@ -168,7 +169,7 @@ func TestHandleHostOfflineUnregistersNodeLink(t *testing.T) {}// Register the node link- m.registerHostAgentNodeLink(host)+ m.registerHostAgentNodeLink(hostAgentNodeLink{agentID: host.ID, nodeID: host.LinkedNodeID, cpu: true})if !m.hasHostAgentForNode("pve-offlinehost") {t.Error("Expected host agent registered")diff --git a/internal/alerts/manager.go b/internal/alerts/manager.goindex a511bc3691..603dbfcb76 100644--- a/internal/alerts/manager.go+++ b/internal/alerts/manager.go@@ -102,9 +102,9 @@ type Manager struct {// Cleanup controlcleanupStop chan struct{} // Signal to stop cleanup goroutine// Host agent deduplication: the Proxmox node each reporting host agent is- // linked to. While a link holds, the agent resource owns that machine's- // CPU, memory and disk usage alerts and the node releases its own copies.- hostAgentNodeLinks map[string]string // Host agent ID -> linked node ID+ // linked to and the usage metrics the agent evaluates for it. The node+ // releases only those metrics and keeps evaluating the rest itself.+ hostAgentNodeLinks map[string]hostAgentNodeLink // Host agent ID -> node link// Node display name caches. Proxmox nodes can share the same raw node name// across multiple configured instances, so keep instance-scoped entries in// addition to the legacy raw-name cache used by instance-less resources.@@ -242,7 +242,7 @@ func NewManagerWithDataDir(dataDir string, options ...ManagerOption) *Manager {flappingHistory: make(map[string][]time.Time),flappingActive: make(map[string]bool),cleanupStop: make(chan struct{}),- hostAgentNodeLinks: make(map[string]string),+ hostAgentNodeLinks: make(map[string]hostAgentNodeLink),nodeDisplayNames: make(map[string]string),instanceNodeDisplayNames: make(map[string]string),now: time.Now,diff --git a/internal/alerts/node.go b/internal/alerts/node.goindex 5f9e0500b2..2c310e3d83 100644--- a/internal/alerts/node.go+++ b/internal/alerts/node.go@@ -93,27 +93,37 @@ func (m *Manager) CheckNode(node models.Node) {m.clearNodeOfflineAlert(node)// Check each metric (only if node is online and reachable)- // Check for host agent deduplication: if a host agent is linked to this node,- // CPU, memory and disk usage alerts belong to the agent resource (CheckHost- // evaluates them), so the node releases its own copies instead of alerting twice.- if m.hasHostAgentForNode(node.ID) {- m.releaseNodeMetricAlerts(node, "cpu", "memory", "disk")+ // Check for host agent deduplication: a usage metric that a linked host+ // agent evaluates belongs to the agent resource (CheckHost raises it), so+ // the node releases its own copy instead of alerting twice. A metric the+ // agent does not evaluate (agent alerts disabled, threshold off, usage+ // unknown) stays with the node, so the machine is never left unmonitored.+ link, _ := m.linkedHostAgentForNode(node.ID)+ input := &UnifiedResourceInput{+ ID: node.ID,+ Type: "node",+ Name: node.Name,+ Node: node.Name,+ Instance: node.Instance,+ }+ released := make([]string, 0, 3)+ if link.cpu {+ released = append(released, "cpu")} else {- var memoryMetric *UnifiedResourceMetric- if node.Memory.HasKnownUsage() {- memoryMetric = &UnifiedResourceMetric{Percent: node.Memory.Usage}- }- m.evaluateUnifiedMetrics(&UnifiedResourceInput{- ID: node.ID,- Type: "node",- Name: node.Name,- Node: node.Name,- Instance: node.Instance,- CPU: &UnifiedResourceMetric{Percent: node.CPU * 100},- Memory: memoryMetric,- Disk: &UnifiedResourceMetric{Percent: node.Disk.Usage},- }, thresholds, nil)+ input.CPU = &UnifiedResourceMetric{Percent: node.CPU * 100}+ }+ if link.memory {+ released = append(released, "memory")+ } else if node.Memory.HasKnownUsage() {+ input.Memory = &UnifiedResourceMetric{Percent: node.Memory.Usage}}+ if link.disk {+ released = append(released, "disk")+ } else {+ input.Disk = &UnifiedResourceMetric{Percent: node.Disk.Usage}+ }+ m.releaseNodeMetricAlerts(node, released...)+ m.evaluateUnifiedMetrics(input, thresholds, nil)// CPU temperature stays with the node even when a host agent runs on it:// CheckHost has no CPU temperature metric, and the node poll already@@ -189,32 +199,46 @@ func (m *Manager) releaseNodeMetricAlerts(node models.Node, metrics ...string) {}}-// registerHostAgentNodeLink records which Proxmox node a reporting host agent-// is linked to. While the link holds, the agent resource owns that machine's-// CPU, memory and disk usage alerts. The link is the monitor's identity decision-// (automatic or operator-set), so an agent whose hostname merely matches a node-// name in another instance, or one an operator unlinked, never makes a node-// release its alerts, and an agent reporting an FQDN still dedups its node.-func (m *Manager) registerHostAgentNodeLink(host models.Host) {- hostID := strings.TrimSpace(host.ID)- if hostID == "" {+// hostAgentNodeLink is a reporting host agent's link to a Proxmox node and the+// usage metrics the agent evaluates for that machine. The link is the monitor's+// identity decision (automatic or operator-set), so an agent whose hostname+// merely matches a node name in another instance, or one an operator unlinked,+// never makes a node release its alerts, and an agent reporting an FQDN still+// dedups its node.+type hostAgentNodeLink struct {+ agentID string+ agentName string+ nodeID string+ cpu bool+ memory bool+ disk bool+}++// registerHostAgentNodeLink records a host agent's node link after CheckHost+// has decided which usage metrics it evaluates. A link without a node or+// without any evaluated metric is removed, so the node keeps all its alerts.+func (m *Manager) registerHostAgentNodeLink(link hostAgentNodeLink) {+ link.agentID = strings.TrimSpace(link.agentID)+ link.nodeID = strings.TrimSpace(link.nodeID)+ if link.agentID == "" {return}- linkedNodeID := strings.TrimSpace(host.LinkedNodeID)- m.mu.Lock()- previous, existed := m.hostAgentNodeLinks[hostID]- if linkedNodeID == "" {- delete(m.hostAgentNodeLinks, hostID)- } else {- m.hostAgentNodeLinks[hostID] = linkedNodeID+ if link.nodeID == "" || !(link.cpu || link.memory || link.disk) {+ m.unregisterHostAgentNodeLink(link.agentID)+ return}+ m.mu.Lock()+ previous, existed := m.hostAgentNodeLinks[link.agentID]+ m.hostAgentNodeLinks[link.agentID] = linkm.mu.Unlock()- if !existed || previous != linkedNodeID {+ if !existed || previous != link {log.Debug().- Str("hostID", hostID).- Str("hostname", host.Hostname).- Str("linkedNodeID", linkedNodeID).+ Str("hostID", link.agentID).+ Str("linkedNodeID", link.nodeID).+ Bool("cpu", link.cpu).+ Bool("memory", link.memory).+ Bool("disk", link.disk).Msg("Updated host agent node link for deduplication")}}@@ -227,33 +251,47 @@ func (m *Manager) unregisterHostAgentNodeLink(hostID string) {return}m.mu.Lock()- linkedNodeID, existed := m.hostAgentNodeLinks[hostID]+ link, existed := m.hostAgentNodeLinks[hostID]delete(m.hostAgentNodeLinks, hostID)m.mu.Unlock()if existed {log.Debug().Str("hostID", hostID).- Str("linkedNodeID", linkedNodeID).+ Str("linkedNodeID", link.nodeID).Msg("Removed host agent node link from deduplication")}}-// hasHostAgentForNode reports whether a reporting host agent is linked to the-// given Proxmox node, in which case the node's usage alerts belong to the agent.-func (m *Manager) hasHostAgentForNode(nodeID string) bool {+// linkedHostAgentForNode returns the host agent link that owns usage metrics+// for the given Proxmox node. If several agents are linked to one node, the+// lowest agent ID wins so the choice is stable.+func (m *Manager) linkedHostAgentForNode(nodeID string) (hostAgentNodeLink, bool) {nodeID = strings.TrimSpace(nodeID)if nodeID == "" {- return false+ return hostAgentNodeLink{}, false}m.mu.RLock()defer m.mu.RUnlock()- for _, linkedNodeID := range m.hostAgentNodeLinks {- if linkedNodeID == nodeID {- return true+ var found hostAgentNodeLink+ ok := false+ for _, link := range m.hostAgentNodeLinks {+ if link.nodeID != nodeID {+ continue+ }+ if !ok || link.agentID < found.agentID {+ found = link+ ok = true}}- return false+ return found, ok+}++// hasHostAgentForNode reports whether a reporting host agent is linked to the+// given Proxmox node and owns at least one of its usage metrics.+func (m *Manager) hasHostAgentForNode(nodeID string) bool {+ _, ok := m.linkedHostAgentForNode(nodeID)+ return ok}// UpdateNodeDisplayName caches the display name for a node/host so alertsdiff --git a/internal/alerts/threshold_resolution_shared_test.go b/internal/alerts/threshold_resolution_shared_test.goindex 540ab6aedd..3b3b3f1b44 100644--- a/internal/alerts/threshold_resolution_shared_test.go+++ b/internal/alerts/threshold_resolution_shared_test.go@@ -359,6 +359,67 @@ func TestCheckNodeMissingTemperatureDoesNotResolveOpenAlert(t *testing.T) {}}+// Deduplication only hands the agent the usage metrics it evaluates. With agent+// alerts switched off, or one agent threshold off, the node keeps those alerts+// so the machine is never left unmonitored.+func TestCheckNodeKeepsUsageMetricsTheAgentDoesNotEvaluate(t *testing.T) {+ setup := func(t *testing.T) (*Manager, models.Node, models.Host) {+ m := newTestManager(t)+ m.mu.Lock()+ m.config.Enabled = true+ m.config.TimeThresholds = map[string]int{}+ m.config.NodeDefaults.CPU = &HysteresisThreshold{Trigger: 80, Clear: 75}+ m.config.NodeDefaults.Memory = &HysteresisThreshold{Trigger: 85, Clear: 80}+ m.config.AgentDefaults.CPU = &HysteresisThreshold{Trigger: 80, Clear: 75}+ m.config.AgentDefaults.Memory = &HysteresisThreshold{Trigger: 85, Clear: 80}+ m.mu.Unlock()+ node, host := testNodeWithHostAgent()+ node.CPU = 0.95+ node.Memory = models.Memory{Total: 100, Used: 95, Free: 5, Usage: 95}+ host.CPUUsage = 95+ host.Memory = node.Memory+ return m, node, host+ }+ nodeCPU := func(node models.Node) string { return canonicalMetricStateID(node.ID, "cpu") }+ nodeMemory := func(node models.Node) string { return canonicalMetricStateID(node.ID, "memory") }++ t.Run("agent_alerts_disabled", func(t *testing.T) {+ m, node, host := setup(t)+ m.CheckHost(host)+ m.CheckNode(node)+ if testHasActiveAlert(t, m, nodeCPU(node)) {+ t.Fatalf("expected the agent to own CPU while its alerts are enabled")+ }++ m.mu.Lock()+ m.config.DisableAllAgents = true+ m.mu.Unlock()+ m.CheckHost(host)+ m.CheckNode(node)+ if !testHasActiveAlert(t, m, nodeCPU(node)) || !testHasActiveAlert(t, m, nodeMemory(node)) {+ t.Fatalf("expected the node to alert on CPU and memory while agent alerts are disabled")+ }+ })++ t.Run("agent_cpu_threshold_off", func(t *testing.T) {+ m, node, host := setup(t)+ m.mu.Lock()+ m.config.AgentDefaults.CPU = &HysteresisThreshold{Trigger: 0, Clear: 0}+ m.mu.Unlock()+ m.CheckHost(host)+ m.CheckNode(node)+ if !testHasActiveAlert(t, m, nodeCPU(node)) {+ t.Fatalf("expected the node to keep CPU when the agent CPU threshold is off")+ }+ if testHasActiveAlert(t, m, nodeMemory(node)) {+ t.Fatalf("expected the agent to keep owning memory")+ }+ if got := testActiveAlertIDsOfType(m, "memory"); len(got) != 1 {+ t.Fatalf("expected exactly one memory alert for the machine, got %v", got)+ }+ })+}+// A missing reading keeps the incident but must not let a sustained-for or// recovery delay complete across the gap in evidence.func TestCheckNodeMissingTemperatureInterruptsTimingRuns(t *testing.T) {