Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Hand a node only the usage metrics its agent evaluates

rcourtman9206486d56e2
diff --git a/docs/release-control/v6/internal/subsystems/alerts.md b/docs/release-control/v6/internal/subsystems/alerts.md
index c385b63579..aca39a8060 100644
--- a/docs/release-control/v6/internal/subsystems/alerts.md
+++ b/docs/release-control/v6/internal/subsystems/alerts.md
@@ -1803,11 +1803,16 @@ host-agent deduplication bookkeeping, and instance-scoped node display-name
cache updates; future Proxmox node alert behavior should extend that resource
checker owner rather than expanding the central Manager file.
When a reporting host agent is linked to a node (`LinkedNodeID`, the monitor's
-automatic or operator-set identity decision), the agent resource owns that
-machine's CPU, memory and disk usage alerts, and `CheckNode` releases its own
-copies through the disabled-threshold path every cycle: the pending run is
-dropped and any node alert still open from before the link is resolved, never
-left frozen until the agent goes offline. Deduplication keys on that link,
+automatic or operator-set identity decision), the agent resource owns each
+CPU, memory or disk usage metric it actually evaluates, and `CheckNode` releases
+its own copy of those metrics through the disabled-threshold path every cycle:
+the pending run is dropped and any node alert still open from before the link is
+resolved, never left frozen until the agent goes offline. `CheckHost` registers
+the link only after evaluation, recording which metrics had a live threshold
+(memory also needs a known reading, disk at least one evaluated filesystem), and
+removes it while agent alerts are disabled; a metric the agent does not evaluate
+stays with the node, so deduplication never leaves a machine unmonitored.
+Deduplication keys on that link,
never on a hostname match, so a same-named node in another instance keeps its
alerts, an agent reporting an FQDN still dedups its node, and an operator unlink
hands the alerts back. CPU temperature has no host-agent metric, so the node
@@ -1826,7 +1831,8 @@ instance name in `Instance`.
`TestCheckNodeReleasesOpenMetricAlertWhenHostAgentRegisters`,
`TestCheckNodeMissingTemperatureDoesNotResolveOpenAlert`,
`TestCheckNodeMissingTemperatureInterruptsTimingRuns` and
-`TestConfigSaveKeepsNodeTemperatureAlertOverTrigger` in
+`TestConfigSaveKeepsNodeTemperatureAlertOverTrigger` and
+`TestCheckNodeKeepsUsageMetricsTheAgentDoesNotEvaluate` in
`internal/alerts/threshold_resolution_shared_test.go`, and
`TestHostAgentDeduplicationFollowsNodeLink` in
`internal/alerts/host_dedup_test.go`, pin these rules.
diff --git a/internal/alerts/host.go b/internal/alerts/host.go
index 8e0e8c5516..718cc75f9a 100644
--- a/internal/alerts/host.go
+++ b/internal/alerts/host.go
@@ -220,10 +220,6 @@ func (m *Manager) CheckHost(host models.Host) {
return
}
- // Record the Proxmox node this agent is linked to, so the node releases the
- // usage alerts the agent now owns instead of alerting twice for one machine.
- m.registerHostAgentNodeLink(host)
-
// Cache display name so host alerts show the user-configured name.
m.UpdateNodeDisplayName("", host.Hostname, host.DisplayName)
@@ -242,11 +238,15 @@ func (m *Manager) CheckHost(host models.Host) {
}
m.mu.RUnlock()
+ // While this agent evaluates nothing, its linked node keeps its own usage
+ // alerts; the link is registered below once the evaluated metrics are known.
if !alertsEnabled {
+ m.unregisterHostAgentNodeLink(host.ID)
return
}
if disableAllAgents {
+ m.unregisterHostAgentNodeLink(host.ID)
// Clear any existing host alerts when all host alerts are disabled
m.clearHostMetricAlerts(host.ID)
m.clearHostDiskAlerts(host.ID)
@@ -257,6 +257,7 @@ func (m *Manager) CheckHost(host models.Host) {
}
if thresholds.Disabled {
+ m.unregisterHostAgentNodeLink(host.ID)
m.clearHostMetricAlerts(host.ID)
m.clearHostDiskAlerts(host.ID)
m.clearHostRAIDAlerts(host.ID)
@@ -386,6 +387,7 @@ func (m *Manager) CheckHost(host models.Host) {
}
seenDisks := make(map[string]struct{}, len(host.Disks))
+ evaluatesDiskUsage := false
if len(host.Sensors.SMART) > 0 {
for _, disk := range host.Sensors.SMART {
diskResourceID, diskName := hostSMARTDiskResourceID(host, disk)
@@ -443,6 +445,7 @@ func (m *Manager) CheckHost(host models.Host) {
if effectiveDiskThreshold == nil {
continue
}
+ evaluatesDiskUsage = evaluatesDiskUsage || effectiveDiskThreshold.Trigger > 0
diskMetadata := cloneMetadata(baseMetadata)
diskMetadata["metric"] = "disk"
@@ -490,6 +493,17 @@ func (m *Manager) CheckHost(host models.Host) {
m.cleanupHostDiskAlerts(host, seenDisks)
+ // The linked node releases exactly the usage metrics evaluated above and
+ // keeps the rest, so deduplication never leaves the machine unmonitored.
+ m.registerHostAgentNodeLink(hostAgentNodeLink{
+ agentID: host.ID,
+ agentName: resourceName,
+ nodeID: host.LinkedNodeID,
+ cpu: thresholds.CPU != nil && thresholds.CPU.Trigger > 0,
+ memory: thresholds.Memory != nil && thresholds.Memory.Trigger > 0 && host.Memory.HasKnownUsage(),
+ disk: evaluatesDiskUsage,
+ })
+
if host.Unraid != nil {
m.syncHostUnraidStorageAlert(host, nodeName, instanceName, resourceName, baseMetadata)
} else {
diff --git a/internal/alerts/host_dedup_test.go b/internal/alerts/host_dedup_test.go
index a0b4ac3751..aee90c38fa 100644
--- a/internal/alerts/host_dedup_test.go
+++ b/internal/alerts/host_dedup_test.go
@@ -7,7 +7,7 @@ import (
)
func TestHostAgentDeduplicatesNodeAlerts(t *testing.T) {
- agent := models.Host{ID: "host-pi", Hostname: "pi", LinkedNodeID: "node/pi"}
+ agent := hostAgentNodeLink{agentID: "host-pi", nodeID: "node/pi", cpu: true, memory: true, disk: true}
// Test 1: Without a linked host agent, node metrics ARE checked
t.Run("node_metrics_checked_without_agent", func(t *testing.T) {
@@ -81,7 +81,7 @@ func TestHostAgentDeduplicatesNodeAlerts(t *testing.T) {
// Register and then unregister
m.registerHostAgentNodeLink(agent)
- m.unregisterHostAgentNodeLink(agent.ID)
+ m.unregisterHostAgentNodeLink(agent.agentID)
// Verify host agent is NOT linked
if m.hasHostAgentForNode("node/pi") {
@@ -112,8 +112,9 @@ func TestHostAgentDeduplicatesNodeAlerts(t *testing.T) {
// instances can both have a node named "pve", and an agent can report an FQDN.
func TestHostAgentDeduplicationFollowsNodeLink(t *testing.T) {
m := NewManager()
+ m.config.Enabled = true
- m.registerHostAgentNodeLink(models.Host{ID: "host-a", Hostname: "pve.example.com", LinkedNodeID: "site-a-pve"})
+ m.CheckHost(models.Host{ID: "host-a", Hostname: "pve.example.com", LinkedNodeID: "site-a-pve", CPUUsage: 10})
if !m.hasHostAgentForNode("site-a-pve") {
t.Error("Expected the linked node to dedup even though the agent hostname is an FQDN")
}
@@ -121,13 +122,13 @@ func TestHostAgentDeduplicationFollowsNodeLink(t *testing.T) {
t.Error("Expected a same-named node in another instance to keep its own alerts")
}
- m.registerHostAgentNodeLink(models.Host{ID: "host-b", Hostname: "pve"})
+ m.CheckHost(models.Host{ID: "host-b", Hostname: "pve", CPUUsage: 10})
if m.hasHostAgentForNode("site-b-pve") {
t.Error("Expected an unlinked agent with a matching hostname not to suppress node alerts")
}
// An operator unlinking the agent hands the node's alerts back.
- m.registerHostAgentNodeLink(models.Host{ID: "host-a", Hostname: "pve.example.com"})
+ m.CheckHost(models.Host{ID: "host-a", Hostname: "pve.example.com", CPUUsage: 10})
if m.hasHostAgentForNode("site-a-pve") {
t.Error("Expected an unlinked agent to stop deduplicating its former node")
}
@@ -168,7 +169,7 @@ func TestHandleHostOfflineUnregistersNodeLink(t *testing.T) {
}
// Register the node link
- m.registerHostAgentNodeLink(host)
+ m.registerHostAgentNodeLink(hostAgentNodeLink{agentID: host.ID, nodeID: host.LinkedNodeID, cpu: true})
if !m.hasHostAgentForNode("pve-offlinehost") {
t.Error("Expected host agent registered")
diff --git a/internal/alerts/manager.go b/internal/alerts/manager.go
index a511bc3691..603dbfcb76 100644
--- a/internal/alerts/manager.go
+++ b/internal/alerts/manager.go
@@ -102,9 +102,9 @@ type Manager struct {
// Cleanup control
cleanupStop chan struct{} // Signal to stop cleanup goroutine
// Host agent deduplication: the Proxmox node each reporting host agent is
- // linked to. While a link holds, the agent resource owns that machine's
- // CPU, memory and disk usage alerts and the node releases its own copies.
- hostAgentNodeLinks map[string]string // Host agent ID -> linked node ID
+ // linked to and the usage metrics the agent evaluates for it. The node
+ // releases only those metrics and keeps evaluating the rest itself.
+ hostAgentNodeLinks map[string]hostAgentNodeLink // Host agent ID -> node link
// Node display name caches. Proxmox nodes can share the same raw node name
// across multiple configured instances, so keep instance-scoped entries in
// addition to the legacy raw-name cache used by instance-less resources.
@@ -242,7 +242,7 @@ func NewManagerWithDataDir(dataDir string, options ...ManagerOption) *Manager {
flappingHistory: make(map[string][]time.Time),
flappingActive: make(map[string]bool),
cleanupStop: make(chan struct{}),
- hostAgentNodeLinks: make(map[string]string),
+ hostAgentNodeLinks: make(map[string]hostAgentNodeLink),
nodeDisplayNames: make(map[string]string),
instanceNodeDisplayNames: make(map[string]string),
now: time.Now,
diff --git a/internal/alerts/node.go b/internal/alerts/node.go
index 5f9e0500b2..2c310e3d83 100644
--- a/internal/alerts/node.go
+++ b/internal/alerts/node.go
@@ -93,27 +93,37 @@ func (m *Manager) CheckNode(node models.Node) {
m.clearNodeOfflineAlert(node)
// Check each metric (only if node is online and reachable)
- // Check for host agent deduplication: if a host agent is linked to this node,
- // CPU, memory and disk usage alerts belong to the agent resource (CheckHost
- // evaluates them), so the node releases its own copies instead of alerting twice.
- if m.hasHostAgentForNode(node.ID) {
- m.releaseNodeMetricAlerts(node, "cpu", "memory", "disk")
+ // Check for host agent deduplication: a usage metric that a linked host
+ // agent evaluates belongs to the agent resource (CheckHost raises it), so
+ // the node releases its own copy instead of alerting twice. A metric the
+ // agent does not evaluate (agent alerts disabled, threshold off, usage
+ // unknown) stays with the node, so the machine is never left unmonitored.
+ link, _ := m.linkedHostAgentForNode(node.ID)
+ input := &UnifiedResourceInput{
+ ID: node.ID,
+ Type: "node",
+ Name: node.Name,
+ Node: node.Name,
+ Instance: node.Instance,
+ }
+ released := make([]string, 0, 3)
+ if link.cpu {
+ released = append(released, "cpu")
} else {
- var memoryMetric *UnifiedResourceMetric
- if node.Memory.HasKnownUsage() {
- memoryMetric = &UnifiedResourceMetric{Percent: node.Memory.Usage}
- }
- m.evaluateUnifiedMetrics(&UnifiedResourceInput{
- ID: node.ID,
- Type: "node",
- Name: node.Name,
- Node: node.Name,
- Instance: node.Instance,
- CPU: &UnifiedResourceMetric{Percent: node.CPU * 100},
- Memory: memoryMetric,
- Disk: &UnifiedResourceMetric{Percent: node.Disk.Usage},
- }, thresholds, nil)
+ input.CPU = &UnifiedResourceMetric{Percent: node.CPU * 100}
+ }
+ if link.memory {
+ released = append(released, "memory")
+ } else if node.Memory.HasKnownUsage() {
+ input.Memory = &UnifiedResourceMetric{Percent: node.Memory.Usage}
}
+ if link.disk {
+ released = append(released, "disk")
+ } else {
+ input.Disk = &UnifiedResourceMetric{Percent: node.Disk.Usage}
+ }
+ m.releaseNodeMetricAlerts(node, released...)
+ m.evaluateUnifiedMetrics(input, thresholds, nil)
// CPU temperature stays with the node even when a host agent runs on it:
// CheckHost has no CPU temperature metric, and the node poll already
@@ -189,32 +199,46 @@ func (m *Manager) releaseNodeMetricAlerts(node models.Node, metrics ...string) {
}
}
-// registerHostAgentNodeLink records which Proxmox node a reporting host agent
-// is linked to. While the link holds, the agent resource owns that machine's
-// CPU, memory and disk usage alerts. The link is the monitor's identity decision
-// (automatic or operator-set), so an agent whose hostname merely matches a node
-// name in another instance, or one an operator unlinked, never makes a node
-// release its alerts, and an agent reporting an FQDN still dedups its node.
-func (m *Manager) registerHostAgentNodeLink(host models.Host) {
- hostID := strings.TrimSpace(host.ID)
- if hostID == "" {
+// hostAgentNodeLink is a reporting host agent's link to a Proxmox node and the
+// usage metrics the agent evaluates for that machine. The link is the monitor's
+// identity decision (automatic or operator-set), so an agent whose hostname
+// merely matches a node name in another instance, or one an operator unlinked,
+// never makes a node release its alerts, and an agent reporting an FQDN still
+// dedups its node.
+type hostAgentNodeLink struct {
+ agentID string
+ agentName string
+ nodeID string
+ cpu bool
+ memory bool
+ disk bool
+}
+
+// registerHostAgentNodeLink records a host agent's node link after CheckHost
+// has decided which usage metrics it evaluates. A link without a node or
+// without any evaluated metric is removed, so the node keeps all its alerts.
+func (m *Manager) registerHostAgentNodeLink(link hostAgentNodeLink) {
+ link.agentID = strings.TrimSpace(link.agentID)
+ link.nodeID = strings.TrimSpace(link.nodeID)
+ if link.agentID == "" {
return
}
- linkedNodeID := strings.TrimSpace(host.LinkedNodeID)
- m.mu.Lock()
- previous, existed := m.hostAgentNodeLinks[hostID]
- if linkedNodeID == "" {
- delete(m.hostAgentNodeLinks, hostID)
- } else {
- m.hostAgentNodeLinks[hostID] = linkedNodeID
+ if link.nodeID == "" || !(link.cpu || link.memory || link.disk) {
+ m.unregisterHostAgentNodeLink(link.agentID)
+ return
}
+ m.mu.Lock()
+ previous, existed := m.hostAgentNodeLinks[link.agentID]
+ m.hostAgentNodeLinks[link.agentID] = link
m.mu.Unlock()
- if !existed || previous != linkedNodeID {
+ if !existed || previous != link {
log.Debug().
- Str("hostID", hostID).
- Str("hostname", host.Hostname).
- Str("linkedNodeID", linkedNodeID).
+ Str("hostID", link.agentID).
+ Str("linkedNodeID", link.nodeID).
+ Bool("cpu", link.cpu).
+ Bool("memory", link.memory).
+ Bool("disk", link.disk).
Msg("Updated host agent node link for deduplication")
}
}
@@ -227,33 +251,47 @@ func (m *Manager) unregisterHostAgentNodeLink(hostID string) {
return
}
m.mu.Lock()
- linkedNodeID, existed := m.hostAgentNodeLinks[hostID]
+ link, existed := m.hostAgentNodeLinks[hostID]
delete(m.hostAgentNodeLinks, hostID)
m.mu.Unlock()
if existed {
log.Debug().
Str("hostID", hostID).
- Str("linkedNodeID", linkedNodeID).
+ Str("linkedNodeID", link.nodeID).
Msg("Removed host agent node link from deduplication")
}
}
-// hasHostAgentForNode reports whether a reporting host agent is linked to the
-// given Proxmox node, in which case the node's usage alerts belong to the agent.
-func (m *Manager) hasHostAgentForNode(nodeID string) bool {
+// linkedHostAgentForNode returns the host agent link that owns usage metrics
+// for the given Proxmox node. If several agents are linked to one node, the
+// lowest agent ID wins so the choice is stable.
+func (m *Manager) linkedHostAgentForNode(nodeID string) (hostAgentNodeLink, bool) {
nodeID = strings.TrimSpace(nodeID)
if nodeID == "" {
- return false
+ return hostAgentNodeLink{}, false
}
m.mu.RLock()
defer m.mu.RUnlock()
- for _, linkedNodeID := range m.hostAgentNodeLinks {
- if linkedNodeID == nodeID {
- return true
+ var found hostAgentNodeLink
+ ok := false
+ for _, link := range m.hostAgentNodeLinks {
+ if link.nodeID != nodeID {
+ continue
+ }
+ if !ok || link.agentID < found.agentID {
+ found = link
+ ok = true
}
}
- return false
+ return found, ok
+}
+
+// hasHostAgentForNode reports whether a reporting host agent is linked to the
+// given Proxmox node and owns at least one of its usage metrics.
+func (m *Manager) hasHostAgentForNode(nodeID string) bool {
+ _, ok := m.linkedHostAgentForNode(nodeID)
+ return ok
}
// UpdateNodeDisplayName caches the display name for a node/host so alerts
diff --git a/internal/alerts/threshold_resolution_shared_test.go b/internal/alerts/threshold_resolution_shared_test.go
index 540ab6aedd..3b3b3f1b44 100644
--- a/internal/alerts/threshold_resolution_shared_test.go
+++ b/internal/alerts/threshold_resolution_shared_test.go
@@ -359,6 +359,67 @@ func TestCheckNodeMissingTemperatureDoesNotResolveOpenAlert(t *testing.T) {
}
}
+// Deduplication only hands the agent the usage metrics it evaluates. With agent
+// alerts switched off, or one agent threshold off, the node keeps those alerts
+// so the machine is never left unmonitored.
+func TestCheckNodeKeepsUsageMetricsTheAgentDoesNotEvaluate(t *testing.T) {
+ setup := func(t *testing.T) (*Manager, models.Node, models.Host) {
+ m := newTestManager(t)
+ m.mu.Lock()
+ m.config.Enabled = true
+ m.config.TimeThresholds = map[string]int{}
+ m.config.NodeDefaults.CPU = &HysteresisThreshold{Trigger: 80, Clear: 75}
+ m.config.NodeDefaults.Memory = &HysteresisThreshold{Trigger: 85, Clear: 80}
+ m.config.AgentDefaults.CPU = &HysteresisThreshold{Trigger: 80, Clear: 75}
+ m.config.AgentDefaults.Memory = &HysteresisThreshold{Trigger: 85, Clear: 80}
+ m.mu.Unlock()
+ node, host := testNodeWithHostAgent()
+ node.CPU = 0.95
+ node.Memory = models.Memory{Total: 100, Used: 95, Free: 5, Usage: 95}
+ host.CPUUsage = 95
+ host.Memory = node.Memory
+ return m, node, host
+ }
+ nodeCPU := func(node models.Node) string { return canonicalMetricStateID(node.ID, "cpu") }
+ nodeMemory := func(node models.Node) string { return canonicalMetricStateID(node.ID, "memory") }
+
+ t.Run("agent_alerts_disabled", func(t *testing.T) {
+ m, node, host := setup(t)
+ m.CheckHost(host)
+ m.CheckNode(node)
+ if testHasActiveAlert(t, m, nodeCPU(node)) {
+ t.Fatalf("expected the agent to own CPU while its alerts are enabled")
+ }
+
+ m.mu.Lock()
+ m.config.DisableAllAgents = true
+ m.mu.Unlock()
+ m.CheckHost(host)
+ m.CheckNode(node)
+ if !testHasActiveAlert(t, m, nodeCPU(node)) || !testHasActiveAlert(t, m, nodeMemory(node)) {
+ t.Fatalf("expected the node to alert on CPU and memory while agent alerts are disabled")
+ }
+ })
+
+ t.Run("agent_cpu_threshold_off", func(t *testing.T) {
+ m, node, host := setup(t)
+ m.mu.Lock()
+ m.config.AgentDefaults.CPU = &HysteresisThreshold{Trigger: 0, Clear: 0}
+ m.mu.Unlock()
+ m.CheckHost(host)
+ m.CheckNode(node)
+ if !testHasActiveAlert(t, m, nodeCPU(node)) {
+ t.Fatalf("expected the node to keep CPU when the agent CPU threshold is off")
+ }
+ if testHasActiveAlert(t, m, nodeMemory(node)) {
+ t.Fatalf("expected the agent to keep owning memory")
+ }
+ if got := testActiveAlertIDsOfType(m, "memory"); len(got) != 1 {
+ t.Fatalf("expected exactly one memory alert for the machine, got %v", got)
+ }
+ })
+}
+
// A missing reading keeps the incident but must not let a sustained-for or
// recovery delay complete across the gap in evidence.
func TestCheckNodeMissingTemperatureInterruptsTimingRuns(t *testing.T) {