Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Judge legacy disk temperature alerts by the lowest trigger on save

rcourtman6413cda1712a
diff --git a/docs/release-control/v6/internal/subsystems/alerts.md b/docs/release-control/v6/internal/subsystems/alerts.md
index bbfbba386d..1c5fa9f1ff 100644
--- a/docs/release-control/v6/internal/subsystems/alerts.md
+++ b/docs/release-control/v6/internal/subsystems/alerts.md
@@ -1598,8 +1598,10 @@ Host-agent SMART disk temperature alerts (`diskTemperature`, resource
`agent:<host>/disk_temp:<device>`) follow one effective threshold: the
resolved agent `DiskTemperature`, refined by the disk type's `DiskTempByType`
entry unless an explicit host or linked-resource override set it
-(`hostDiskTemperatureThresholdNoLock` in `internal/alerts/host.go`). When that
-threshold is off, `CheckHost` clears the host's disk temperature alerts. The
+(`hostDiskTemperatureThresholdNoLock` in `internal/alerts/host.go`). When the
+host's resolved `DiskTemperature` is off, `CheckHost` clears every disk
+temperature alert on the host. A disabled `DiskTempByType` entry clears only
+the alerts of disks of that type, through the disabled metric spec. The
all-agents switch, a disabled host override, agent removal and confirmed
agent offline clear them with the host's other disk alerts. Before this,
nothing cleared them once the agent-level threshold was off.
@@ -1607,15 +1609,17 @@ nothing cleared them once the agent-level threshold was off.
A config save judges an open disk temperature alert against the same
threshold (`resolveHostAlertThresholdsNoLock` refines it by the alert's
`diskType` metadata, and `metric_runtime.go` classifies `diskTemperature` as a
-threshold metric), so turning the threshold off resolves the alert
-immediately. Every firing evaluation refreshes `diskType`, so an alert at or
-above its trigger always carries it. Only an alert held in its recovery band
-since before that metadata existed (May 2026) falls back to the agent
-threshold. Its reading is already below its trigger, so the fallback can
-resolve it early or leave it to the evaluator, but it cannot resolve an alert
-the next report raises again.
-
-For every threshold metric, a config save resolves an alert whose reading is
+threshold metric), so turning the host's `DiskTemperature` off resolves the
+alert immediately. An alert persisted before `CheckHost` recorded `diskType` (May
+2026) carries no disk type until its next firing evaluation, so a save judges
+it against the lowest enabled trigger any of the host's disks can use: the
+agent threshold or a `DiskTempByType` entry
+(`lowestHostDiskTemperatureThresholdNoLock`). A save therefore cannot resolve
+an alert its disk type would still fire.
+
+For a metric alert that reaches the threshold comparison in
+`reevaluateActiveAlertsLocked` (docker-host alerts return before it), a config
+save resolves an alert whose reading is
below the new trigger. That includes readings inside a valid recovery band,
which the evaluator would hold. A clear level at or above the trigger is no
recovery band (`buildCanonicalMetricSpec` drops it), and a reading at the
@@ -1623,7 +1627,8 @@ trigger stays firing, as it does in the reducer, so a save no longer resolves
an alert the next evaluation raises again.
`TestCheckHostClearsDiskTemperatureAlertsWhenThresholdTurnsOff`,
`TestHostDiskTemperatureAlertsClearWhenAgentLeaves`,
-`TestConfigSaveResolvesDiskTemperatureAlertAgainstItsDiskTypeThreshold` and
+`TestConfigSaveResolvesDiskTemperatureAlertAgainstItsDiskTypeThreshold`,
+`TestConfigSaveKeepsDiskTemperatureAlertWithoutDiskType` and
`TestConfigSaveKeepsAlertTheEvaluatorKeepsWithoutRecoveryBand` in
`internal/alerts/host_unraid_lifecycle_test.go` pin these paths.
diff --git a/internal/alerts/host.go b/internal/alerts/host.go
index 6084b5c158..5fb2692605 100644
--- a/internal/alerts/host.go
+++ b/internal/alerts/host.go
@@ -143,7 +143,13 @@ func (m *Manager) resolveHostAlertThresholdsNoLock(alert *Alert, resourceID stri
if alert != nil && alert.Type == "diskTemperature" {
override, exists := m.hostThresholdOverrideNoLock(hostID, linkedNodeID, linkedVMID, linkedContainerID)
overridden := exists && override.DiskTemperature != nil
- thresholds.DiskTemperature = m.hostDiskTemperatureThresholdNoLock(thresholds.DiskTemperature, overridden, metadataStringValue(alert.Metadata, "diskType"))
+ if diskType, known := alert.Metadata["diskType"].(string); known || overridden {
+ thresholds.DiskTemperature = m.hostDiskTemperatureThresholdNoLock(thresholds.DiskTemperature, overridden, diskType)
+ } else {
+ // Alerts persisted before CheckHost recorded diskType carry no
+ // disk type until their next firing evaluation.
+ thresholds.DiskTemperature = m.lowestHostDiskTemperatureThresholdNoLock(thresholds.DiskTemperature)
+ }
}
return thresholds
}
@@ -934,6 +940,24 @@ func (m *Manager) hostDiskTemperatureThresholdNoLock(hostThreshold *HysteresisTh
return hostThreshold
}
+// lowestHostDiskTemperatureThresholdNoLock returns the lowest enabled
+// threshold any SMART disk of the host can be evaluated against. An alert
+// whose disk type is unknown is judged against it, so a config save never
+// resolves an alert its disk type would still fire. Callers must hold m.mu.
+func (m *Manager) lowestHostDiskTemperatureThresholdNoLock(hostThreshold *HysteresisThreshold) *HysteresisThreshold {
+ if hostThreshold == nil || hostThreshold.Trigger <= 0 {
+ return hostThreshold
+ }
+ lowest := hostThreshold
+ for _, th := range m.config.DiskTempByType {
+ if th.Trigger > 0 && th.Trigger < lowest.Trigger {
+ t := th
+ lowest = &t
+ }
+ }
+ return lowest
+}
+
// clearHostDiskTemperatureAlerts resolves every SMART disk temperature alert
// for a host.
func (m *Manager) clearHostDiskTemperatureAlerts(hostID string) {
diff --git a/internal/alerts/host_unraid_lifecycle_test.go b/internal/alerts/host_unraid_lifecycle_test.go
index 300f35b26f..afb88c6c8c 100644
--- a/internal/alerts/host_unraid_lifecycle_test.go
+++ b/internal/alerts/host_unraid_lifecycle_test.go
@@ -326,6 +326,41 @@ func TestConfigSaveResolvesDiskTemperatureAlertAgainstItsDiskTypeThreshold(t *te
}
}
+func TestConfigSaveKeepsDiskTemperatureAlertWithoutDiskType(t *testing.T) {
+ m := configureDiskTempTypeHostManager(t)
+
+ m.mu.Lock()
+ m.config.AgentDefaults.DiskTemperature = &HysteresisThreshold{Trigger: 75, Clear: 70}
+ m.mu.Unlock()
+
+ host := hostWithSMARTDiskTemp("host-temp-legacy", "nvme", 72)
+ alertID := hostDiskTempAlertID(host)
+ m.CheckHost(host)
+ alert, exists := testLookupActiveAlert(t, m, alertID)
+ if !exists {
+ t.Fatalf("expected nvme disk temperature alert at 72C, active: %v", alertKeys(m))
+ }
+ // Restored from a release that did not record the disk type.
+ m.mu.Lock()
+ delete(alert.Metadata, "diskType")
+ m.mu.Unlock()
+
+ // 72C is above the nvme trigger (70) but below the agent default (75).
+ cfg := m.GetConfig()
+ cfg.AgentDefaults.CPU = &HysteresisThreshold{Trigger: 85, Clear: 80}
+ m.UpdateConfig(cfg)
+ if _, exists := testLookupActiveAlert(t, m, alertID); !exists {
+ t.Fatalf("config save resolved a disk temperature alert of unknown disk type that its nvme trigger still fires, active: %v", alertKeys(m))
+ }
+
+ cfg = m.GetConfig()
+ cfg.AgentDefaults.DiskTemperature = &HysteresisThreshold{Trigger: 0, Clear: 0}
+ m.UpdateConfig(cfg)
+ if _, exists := testLookupActiveAlert(t, m, alertID); exists {
+ t.Fatalf("disk temperature alert of unknown disk type stayed active after the threshold was turned off, active: %v", alertKeys(m))
+ }
+}
+
func TestConfigSaveKeepsAlertTheEvaluatorKeepsWithoutRecoveryBand(t *testing.T) {
// Normalization keeps a 70/75 entry, and the evaluator treats a clear
// level at or above the trigger as no recovery band, firing from 70C.