Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Judge legacy disk temperature alerts by the lowest trigger on save
rcourtman6413cda1712a
diff --git a/docs/release-control/v6/internal/subsystems/alerts.md b/docs/release-control/v6/internal/subsystems/alerts.mdindex bbfbba386d..1c5fa9f1ff 100644--- a/docs/release-control/v6/internal/subsystems/alerts.md+++ b/docs/release-control/v6/internal/subsystems/alerts.md@@ -1598,8 +1598,10 @@ Host-agent SMART disk temperature alerts (`diskTemperature`, resource`agent:<host>/disk_temp:<device>`) follow one effective threshold: theresolved agent `DiskTemperature`, refined by the disk type's `DiskTempByType`entry unless an explicit host or linked-resource override set it-(`hostDiskTemperatureThresholdNoLock` in `internal/alerts/host.go`). When that-threshold is off, `CheckHost` clears the host's disk temperature alerts. The+(`hostDiskTemperatureThresholdNoLock` in `internal/alerts/host.go`). When the+host's resolved `DiskTemperature` is off, `CheckHost` clears every disk+temperature alert on the host. A disabled `DiskTempByType` entry clears only+the alerts of disks of that type, through the disabled metric spec. Theall-agents switch, a disabled host override, agent removal and confirmedagent offline clear them with the host's other disk alerts. Before this,nothing cleared them once the agent-level threshold was off.@@ -1607,15 +1609,17 @@ nothing cleared them once the agent-level threshold was off.A config save judges an open disk temperature alert against the samethreshold (`resolveHostAlertThresholdsNoLock` refines it by the alert's`diskType` metadata, and `metric_runtime.go` classifies `diskTemperature` as a-threshold metric), so turning the threshold off resolves the alert-immediately. Every firing evaluation refreshes `diskType`, so an alert at or-above its trigger always carries it. Only an alert held in its recovery band-since before that metadata existed (May 2026) falls back to the agent-threshold. Its reading is already below its trigger, so the fallback can-resolve it early or leave it to the evaluator, but it cannot resolve an alert-the next report raises again.--For every threshold metric, a config save resolves an alert whose reading is+threshold metric), so turning the host's `DiskTemperature` off resolves the+alert immediately. An alert persisted before `CheckHost` recorded `diskType` (May+2026) carries no disk type until its next firing evaluation, so a save judges+it against the lowest enabled trigger any of the host's disks can use: the+agent threshold or a `DiskTempByType` entry+(`lowestHostDiskTemperatureThresholdNoLock`). A save therefore cannot resolve+an alert its disk type would still fire.++For a metric alert that reaches the threshold comparison in+`reevaluateActiveAlertsLocked` (docker-host alerts return before it), a config+save resolves an alert whose reading isbelow the new trigger. That includes readings inside a valid recovery band,which the evaluator would hold. A clear level at or above the trigger is norecovery band (`buildCanonicalMetricSpec` drops it), and a reading at the@@ -1623,7 +1627,8 @@ trigger stays firing, as it does in the reducer, so a save no longer resolvesan alert the next evaluation raises again.`TestCheckHostClearsDiskTemperatureAlertsWhenThresholdTurnsOff`,`TestHostDiskTemperatureAlertsClearWhenAgentLeaves`,-`TestConfigSaveResolvesDiskTemperatureAlertAgainstItsDiskTypeThreshold` and+`TestConfigSaveResolvesDiskTemperatureAlertAgainstItsDiskTypeThreshold`,+`TestConfigSaveKeepsDiskTemperatureAlertWithoutDiskType` and`TestConfigSaveKeepsAlertTheEvaluatorKeepsWithoutRecoveryBand` in`internal/alerts/host_unraid_lifecycle_test.go` pin these paths.diff --git a/internal/alerts/host.go b/internal/alerts/host.goindex 6084b5c158..5fb2692605 100644--- a/internal/alerts/host.go+++ b/internal/alerts/host.go@@ -143,7 +143,13 @@ func (m *Manager) resolveHostAlertThresholdsNoLock(alert *Alert, resourceID striif alert != nil && alert.Type == "diskTemperature" {override, exists := m.hostThresholdOverrideNoLock(hostID, linkedNodeID, linkedVMID, linkedContainerID)overridden := exists && override.DiskTemperature != nil- thresholds.DiskTemperature = m.hostDiskTemperatureThresholdNoLock(thresholds.DiskTemperature, overridden, metadataStringValue(alert.Metadata, "diskType"))+ if diskType, known := alert.Metadata["diskType"].(string); known || overridden {+ thresholds.DiskTemperature = m.hostDiskTemperatureThresholdNoLock(thresholds.DiskTemperature, overridden, diskType)+ } else {+ // Alerts persisted before CheckHost recorded diskType carry no+ // disk type until their next firing evaluation.+ thresholds.DiskTemperature = m.lowestHostDiskTemperatureThresholdNoLock(thresholds.DiskTemperature)+ }}return thresholds}@@ -934,6 +940,24 @@ func (m *Manager) hostDiskTemperatureThresholdNoLock(hostThreshold *HysteresisThreturn hostThreshold}+// lowestHostDiskTemperatureThresholdNoLock returns the lowest enabled+// threshold any SMART disk of the host can be evaluated against. An alert+// whose disk type is unknown is judged against it, so a config save never+// resolves an alert its disk type would still fire. Callers must hold m.mu.+func (m *Manager) lowestHostDiskTemperatureThresholdNoLock(hostThreshold *HysteresisThreshold) *HysteresisThreshold {+ if hostThreshold == nil || hostThreshold.Trigger <= 0 {+ return hostThreshold+ }+ lowest := hostThreshold+ for _, th := range m.config.DiskTempByType {+ if th.Trigger > 0 && th.Trigger < lowest.Trigger {+ t := th+ lowest = &t+ }+ }+ return lowest+}+// clearHostDiskTemperatureAlerts resolves every SMART disk temperature alert// for a host.func (m *Manager) clearHostDiskTemperatureAlerts(hostID string) {diff --git a/internal/alerts/host_unraid_lifecycle_test.go b/internal/alerts/host_unraid_lifecycle_test.goindex 300f35b26f..afb88c6c8c 100644--- a/internal/alerts/host_unraid_lifecycle_test.go+++ b/internal/alerts/host_unraid_lifecycle_test.go@@ -326,6 +326,41 @@ func TestConfigSaveResolvesDiskTemperatureAlertAgainstItsDiskTypeThreshold(t *te}}+func TestConfigSaveKeepsDiskTemperatureAlertWithoutDiskType(t *testing.T) {+ m := configureDiskTempTypeHostManager(t)++ m.mu.Lock()+ m.config.AgentDefaults.DiskTemperature = &HysteresisThreshold{Trigger: 75, Clear: 70}+ m.mu.Unlock()++ host := hostWithSMARTDiskTemp("host-temp-legacy", "nvme", 72)+ alertID := hostDiskTempAlertID(host)+ m.CheckHost(host)+ alert, exists := testLookupActiveAlert(t, m, alertID)+ if !exists {+ t.Fatalf("expected nvme disk temperature alert at 72C, active: %v", alertKeys(m))+ }+ // Restored from a release that did not record the disk type.+ m.mu.Lock()+ delete(alert.Metadata, "diskType")+ m.mu.Unlock()++ // 72C is above the nvme trigger (70) but below the agent default (75).+ cfg := m.GetConfig()+ cfg.AgentDefaults.CPU = &HysteresisThreshold{Trigger: 85, Clear: 80}+ m.UpdateConfig(cfg)+ if _, exists := testLookupActiveAlert(t, m, alertID); !exists {+ t.Fatalf("config save resolved a disk temperature alert of unknown disk type that its nvme trigger still fires, active: %v", alertKeys(m))+ }++ cfg = m.GetConfig()+ cfg.AgentDefaults.DiskTemperature = &HysteresisThreshold{Trigger: 0, Clear: 0}+ m.UpdateConfig(cfg)+ if _, exists := testLookupActiveAlert(t, m, alertID); exists {+ t.Fatalf("disk temperature alert of unknown disk type stayed active after the threshold was turned off, active: %v", alertKeys(m))+ }+}+func TestConfigSaveKeepsAlertTheEvaluatorKeepsWithoutRecoveryBand(t *testing.T) {// Normalization keeps a 70/75 entry, and the evaluator treats a clear// level at or above the trigger as no recovery band, firing from 70C.