Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Restart SMART temperature timing after observation or policy gaps

pulse-triage[bot]31797ec37107
diff --git a/docs/release-control/v6/internal/subsystems/alerts.md b/docs/release-control/v6/internal/subsystems/alerts.md
index 0c6c3f1a9f..2083df26fc 100644
--- a/docs/release-control/v6/internal/subsystems/alerts.md
+++ b/docs/release-control/v6/internal/subsystems/alerts.md
@@ -3688,3 +3688,30 @@ display name differs from its hostname was never highlighted, and Machines rows
had no alert highlighting at all. `getAlertStyles` keeps its
id-plus-node-name matching for the platform tables whose alerts are keyed that
way.
+
+### SMART temperature continuity and pending-only policy intervals
+
+Disk temperature activation/grace and recovery require continuous accepted
+observations. Empty SMART inventory, one disk omission, standby, non-positive
+readings, explicit missing/unavailable/unsupported collection and reporting-lease
+expiry interrupt pending/recovery timing without changing an open alert, its
+acknowledgement or lifecycle events. Positive legacy readings without temperature
+provenance remain accepted; an expired last-known positive value is not live
+health. Canonical and shadow reducers plus explicit intent grace are interrupted
+together. A non-empty inventory still resolves a departed disk only after three
+consecutive omissions; empty/expired inventory breaks that confirmation sequence.
+
+Configuration saves reset pending-only runs when global agent policy, host or
+inherited node/guest overrides, or the applicable disk-type threshold disables
+evaluation, even if no report arrives before re-enablement. Only live pending
+identity/link/type context is retained; unrelated saves preserve a known enabled
+run and explicit overrides retain precedence over type defaults. Restored intent
+without that context restarts conservatively on a save rather than inventing
+continuous enablement. Cleared intent grace is checkpointed. Existing active-alert
+per-type/save/offline/removal resolution remains unchanged.
+
+`metric_observation_continuity_test.go` and `host_unraid_lifecycle_test.go` cover
+valid/legacy/provenance controls, missing/rejected/expiry gaps on both timing
+edges, explicit grace, acknowledgement/event preservation, host isolation,
+configuration intervals, override precedence and departure. These deterministic
+source tests do not establish installed temperatures or native guest recovery.
diff --git a/internal/alerts/config_runtime.go b/internal/alerts/config_runtime.go
index 308d134046..ad7711498f 100644
--- a/internal/alerts/config_runtime.go
+++ b/internal/alerts/config_runtime.go
@@ -93,6 +93,12 @@ func (m *Manager) UpdateConfig(config AlertConfig) {
Interface("guestDefaults", config.GuestDefaults).
Msg("Alert configuration updated")
+ // Pending-only temperature runs have no active alert to re-evaluate.
+ // Disabling and re-enabling between reports must not reuse their grace.
+ if m.reevaluateHostDiskTemperaturePendingNoLock() {
+ m.saveActiveAlertsAsync("disk temperature pending policy disabled")
+ }
+
// Re-evaluate active alerts against new thresholds
m.reevaluateActiveAlertsLocked()
}
diff --git a/internal/alerts/host.go b/internal/alerts/host.go
index 979eef1f7b..ce83e6ff94 100644
--- a/internal/alerts/host.go
+++ b/internal/alerts/host.go
@@ -390,7 +390,7 @@ func (m *Manager) CheckHost(host models.Host) {
// reading resolves it instead of clearing and re-raising.
tempResourceID := hostDiskTemperatureResourceID(host.ID, disk.Device)
seenDiskTemps[tempResourceID] = struct{}{}
- if disk.Temperature > 0 && !disk.Standby {
+ if hostDiskTemperatureObserved(disk) {
m.mu.RLock()
effectiveTempThreshold := m.hostDiskTemperatureThresholdNoLock(thresholds.DiskTemperature, diskTempOverridden, disk.Type)
m.mu.RUnlock()
@@ -411,15 +411,21 @@ func (m *Manager) CheckHost(host models.Host) {
Str("host", resourceName).
Str("device", disk.Device).
Msg("Skipping invalid canonical host disk temperature metric spec")
+ m.interruptHostDiskTemperatureRun(tempResourceID)
continue
}
m.checkMetricWithCanonicalSpec(spec, tempResourceName, nodeName, disk.Device, "agent", float64(disk.Temperature), effectiveTempThreshold, &metricOptions{Metadata: diskTempMetadata})
+ m.rememberHostDiskTemperaturePending(host, disk, tempResourceID)
+ } else {
+ m.interruptHostDiskTemperatureRun(tempResourceID)
}
}
// A disk missing from consecutive non-empty reports was removed,
// replaced or renamed, so no later reading will resolve its alert.
m.cleanupHostDiskTemperatureAlerts(host.ID, seenDiskTemps)
+ } else {
+ m.interruptHostDiskTemperatureRuns(host.ID)
}
} else {
// Disk temperature alerting is off for this host, so no later reading
@@ -704,6 +710,9 @@ func (m *Manager) HandleHostTelemetryExpired(host models.Host) {
return
}
+ // Expiry interrupts timing, not the separate connectivity confirmation.
+ m.interruptHostDiskTemperatureRuns(host.ID)
+
if host.Unraid != nil {
unraid := *host.Unraid
unraid.SyncAction = ""
@@ -1054,6 +1063,12 @@ func (m *Manager) cleanupHostDiskTemperatureAlerts(hostID string, seen map[strin
// Absent resources map to the storage keys of their active alerts; a
// resource with only a pending run maps to none.
absent := make(map[string][]string)
+ intentChanged := false
+ defer func() {
+ if intentChanged {
+ m.saveActiveAlertsAsync("disk temperature inventory gap")
+ }
+ }()
for storageKey, alert := range m.activeAlerts {
// Only CheckHost raises alerts under this prefix. Matching on it
// alone also catches alerts raised before the type became
@@ -1066,7 +1081,7 @@ func (m *Manager) cleanupHostDiskTemperatureAlerts(hostID string, seen map[strin
}
absent[alert.ResourceID] = append(absent[alert.ResourceID], storageKey)
}
- for _, resourceID := range m.core.PendingResourceIDs() {
+ for resourceID := range m.hostDiskTemperatureTrackedResourcesNoLock() {
if !strings.HasPrefix(resourceID, resourcePrefix) {
continue
}
@@ -1092,6 +1107,9 @@ func (m *Manager) cleanupHostDiskTemperatureAlerts(hostID string, seen map[strin
}
for resourceID, storageKeys := range absent {
if seen != nil {
+ // One omission breaks sustained/recovery evidence; only confirmed
+ // departure resolves a firing alert.
+ intentChanged = m.interruptHostDiskTemperatureRunNoLock(resourceID) || intentChanged
m.hostDiskTempAbsences[resourceID]++
if m.hostDiskTempAbsences[resourceID] < hostDiskTemperatureAbsenceConfirmations {
continue
@@ -1099,7 +1117,7 @@ func (m *Manager) cleanupHostDiskTemperatureAlerts(hostID string, seen map[strin
}
delete(m.hostDiskTempAbsences, resourceID)
// A departed disk never sends the reading that would close its run.
- m.core.DropPendingForResource(resourceID)
+ intentChanged = m.interruptHostDiskTemperatureRunNoLock(resourceID) || intentChanged
for _, storageKey := range storageKeys {
m.clearAlertNoLock(storageKey)
}
diff --git a/internal/alerts/host_disk_temperature.go b/internal/alerts/host_disk_temperature.go
new file mode 100644
index 0000000000..80d9a397ff
--- /dev/null
+++ b/internal/alerts/host_disk_temperature.go
@@ -0,0 +1,132 @@
+package alerts
+
+import (
+ "strings"
+
+ "github.com/rcourtman/pulse-go-rewrite/internal/alerts/reducer"
+ "github.com/rcourtman/pulse-go-rewrite/internal/models"
+ "github.com/rcourtman/pulse-go-rewrite/pkg/diskinventory"
+)
+
+type hostDiskTempContext struct {
+ hostID, linkedNodeID, linkedVMID, linkedContainerID, diskType string
+}
+
+// A positive legacy reading is still accepted. Explicit failed/unsupported
+// collection, including expired last-known values, is never live evidence.
+func hostDiskTemperatureObserved(disk models.HostDiskSMART) bool {
+ if disk.Temperature <= 0 || disk.Standby {
+ return false
+ }
+ return disk.Collection == nil || disk.Collection.Temperature.State == "" ||
+ disk.Collection.Temperature.State == diskinventory.FieldAvailable
+}
+
+func (m *Manager) hostDiskTemperaturePendingNoLock(resourceID string) bool {
+ for _, state := range m.mirrorStatesNoLock() {
+ if incident, ok := state.Incident(resourceID, canonicalMetricSpecID(resourceID, "diskTemperature")); ok && incident.State == reducer.StatePending {
+ return true
+ }
+ }
+ _, pending := m.intentPending[canonicalMetricStateID(resourceID, "diskTemperature")]
+ return pending
+}
+
+// Keep only the identity of a live pending run, not another telemetry cache.
+func (m *Manager) rememberHostDiskTemperaturePending(host models.Host, disk models.HostDiskSMART, resourceID string) {
+ m.mu.Lock()
+ defer m.mu.Unlock()
+ if !m.hostDiskTemperaturePendingNoLock(resourceID) {
+ delete(m.hostDiskTempPendingContexts, resourceID)
+ return
+ }
+ if m.hostDiskTempPendingContexts == nil {
+ m.hostDiskTempPendingContexts = make(map[string]hostDiskTempContext)
+ }
+ m.hostDiskTempPendingContexts[resourceID] = hostDiskTempContext{host.ID, host.LinkedNodeID, host.LinkedVMID, host.LinkedContainerID, disk.Type}
+}
+
+func (m *Manager) hostDiskTemperatureTrackedResourcesNoLock() map[string]struct{} {
+ resources := make(map[string]struct{})
+ for _, state := range m.mirrorStatesNoLock() {
+ for _, resourceID := range state.PendingResourceIDs() {
+ resources[resourceID] = struct{}{}
+ }
+ }
+ for _, alert := range m.activeAlerts {
+ if alert != nil {
+ resources[alert.ResourceID] = struct{}{}
+ }
+ }
+ for _, pending := range m.intentPending {
+ if pending.Signal == MetricAlertIntentSignal("diskTemperature") {
+ resources[pending.ResourceID] = struct{}{}
+ }
+ }
+ return resources
+}
+
+func (m *Manager) interruptHostDiskTemperatureRunNoLock(resourceID string) bool {
+ delete(m.hostDiskTempPendingContexts, resourceID)
+ return m.interruptMetricRunNoLock(resourceID, canonicalMetricSpecID(resourceID, "diskTemperature"), canonicalMetricStateID(resourceID, "diskTemperature"))
+}
+
+func (m *Manager) interruptHostDiskTemperatureRun(resourceID string) {
+ m.mu.Lock()
+ changed := m.interruptHostDiskTemperatureRunNoLock(resourceID)
+ m.mu.Unlock()
+ if changed {
+ m.saveActiveAlertsAsync("disk temperature observation gap")
+ }
+}
+
+func (m *Manager) interruptHostDiskTemperatureRuns(hostID string) {
+ m.mu.Lock()
+ changed := false
+ for resourceID := range m.hostDiskTemperatureTrackedResourcesNoLock() {
+ if strings.HasPrefix(resourceID, hostDiskTemperatureResourcePrefix(hostID)) {
+ changed = m.interruptHostDiskTemperatureRunNoLock(resourceID) || changed
+ }
+ }
+ // Empty/expired inventory is not another confirmation that a disk left.
+ for resourceID := range m.hostDiskTempAbsences {
+ if strings.HasPrefix(resourceID, hostDiskTemperatureResourcePrefix(hostID)) {
+ delete(m.hostDiskTempAbsences, resourceID)
+ }
+ }
+ m.mu.Unlock()
+ if changed {
+ m.saveActiveAlertsAsync("host disk temperature observation gap")
+ }
+}
+
+// Caller holds m.mu after config normalisation. Active alerts still use the
+// existing per-type threshold re-evaluation; this only drops pending evidence.
+func (m *Manager) reevaluateHostDiskTemperaturePendingNoLock() bool {
+ intentChanged := false
+ for resourceID := range m.hostDiskTemperatureTrackedResourcesNoLock() {
+ if !strings.HasPrefix(resourceID, "agent:") || !strings.Contains(resourceID, "/disk_temp:") || !m.hostDiskTemperaturePendingNoLock(resourceID) {
+ continue
+ }
+ ctx, known := m.hostDiskTempPendingContexts[resourceID]
+ // Restarted intent grace has no retained link/type context. Do not
+ // assume it was continuously enabled across a configuration save.
+ if !known {
+ intentChanged = m.interruptHostDiskTemperatureRunNoLock(resourceID) || intentChanged
+ continue
+ }
+ thresholds := m.resolveHostThresholdsNoLock(ctx.hostID, ctx.linkedNodeID, ctx.linkedVMID, ctx.linkedContainerID)
+ override, exists := m.hostThresholdOverrideNoLock(ctx.hostID, ctx.linkedNodeID, ctx.linkedVMID, ctx.linkedContainerID)
+ threshold := m.hostDiskTemperatureThresholdNoLock(thresholds.DiskTemperature, exists && override.DiskTemperature != nil, ctx.diskType)
+ allDisabled, _ := m.alertPolicyTypeSwitchesNoLock("agent")
+ if !m.config.Enabled || allDisabled || thresholds.Disabled || threshold == nil || threshold.Trigger <= 0 {
+ intentChanged = m.interruptHostDiskTemperatureRunNoLock(resourceID) || intentChanged
+ }
+ }
+ for resourceID := range m.hostDiskTempPendingContexts {
+ if !m.hostDiskTemperaturePendingNoLock(resourceID) {
+ delete(m.hostDiskTempPendingContexts, resourceID)
+ }
+ }
+ return intentChanged
+}
diff --git a/internal/alerts/host_unraid_lifecycle_test.go b/internal/alerts/host_unraid_lifecycle_test.go
index 1dd52d59b6..0ced00d816 100644
--- a/internal/alerts/host_unraid_lifecycle_test.go
+++ b/internal/alerts/host_unraid_lifecycle_test.go
@@ -6,7 +6,9 @@ import (
"testing"
"time"
+ "github.com/rcourtman/pulse-go-rewrite/internal/alerts/reducer"
"github.com/rcourtman/pulse-go-rewrite/internal/models"
+ "github.com/rcourtman/pulse-go-rewrite/pkg/diskinventory"
)
func TestHostCustomSensorAlertLifecycle(t *testing.T) {
@@ -411,7 +413,7 @@ func TestCheckHostClearsDiskTemperatureAlertWhenDiskLeavesSMARTReport(t *testing
host.Sensors.SMART = []models.HostDiskSMART{smartDisk("/dev/sdb", 40)}
for report := 1; report <= hostDiskTemperatureAbsenceConfirmations; report++ {
m.CheckHost(host)
- wantPending := report < hostDiskTemperatureAbsenceConfirmations
+ wantPending := false // A single omission interrupts pending evidence, not firing alerts.
if pending := sdaPending(); pending != wantPending {
t.Fatalf("report %d: pending sda run = %v, want %v", report, pending, wantPending)
}
@@ -891,3 +893,143 @@ func TestHostChildAlertsSurviveProxmoxNodeCleanup(t *testing.T) {
}
}
}
+
+// Disabling between two reports is a real policy interval even when there is
+// no firing alert for UpdateConfig to visit. Linked overrides keep precedence.
+func TestDiskTemperaturePendingConfigDisablement(t *testing.T) {
+ for _, explicit := range []bool{false, true} {
+ for _, scope := range []string{"global", "all-agents", "host", "node", "vm", "container", "type", "unrelated"} {
+ t.Run(fmt.Sprintf("%s/explicit=%v", scope, explicit), func(t *testing.T) {
+ m, elapsed := continuityManager(t, explicit)
+ host := hostWithSMARTDiskTemp("pending-temp", "sata", 85)
+ host.LinkedNodeID, host.LinkedVMID, host.LinkedContainerID = "site:node", "site:node:101", "site:node:102"
+ id := hostDiskTemperatureResourceID(host.ID, host.Sensors.SMART[0].Device)
+ m.CheckHost(host)
+ if incident, ok := continuityIncident(m, id, "diskTemperature"); !ok || incident.State != reducer.StatePending {
+ t.Fatalf("no initial pending run: %+v %v", incident, ok)
+ }
+ cfg := m.GetConfig()
+ switch scope {
+ case "global":
+ cfg.Enabled = false
+ case "all-agents":
+ cfg.DisableAllAgents = true
+ case "host":
+ cfg.Overrides[host.ID] = ThresholdConfig{DiskTemperature: &HysteresisThreshold{Trigger: 0}}
+ case "node":
+ cfg.Overrides[host.LinkedNodeID] = ThresholdConfig{DiskTemperature: &HysteresisThreshold{Trigger: 0}}
+ case "vm":
+ host.LinkedNodeID = "" // The linked node override must not mask the VM.
+ m.CheckHost(host)
+ cfg.Overrides[host.LinkedVMID] = ThresholdConfig{DiskTemperature: &HysteresisThreshold{Trigger: 0}}
+ case "container":
+ host.LinkedNodeID, host.LinkedVMID = "", ""
+ m.CheckHost(host)
+ cfg.Overrides[host.LinkedContainerID] = ThresholdConfig{DiskTemperature: &HysteresisThreshold{Trigger: 0}}
+ case "type":
+ cfg.DiskTempByType = map[string]HysteresisThreshold{"sata": {Trigger: 0}}
+ case "unrelated":
+ cfg.Schedule.QuietHours.Timezone = "UTC"
+ }
+ m.UpdateConfig(cfg)
+ _, pending := continuityIncident(m, id, "diskTemperature")
+ if pending != (scope == "unrelated") {
+ t.Fatalf("after save pending=%v; unrelated saves alone must preserve evidence", pending)
+ }
+ if scope == "unrelated" {
+ return
+ }
+ cfg.Enabled, cfg.DisableAllAgents = true, false
+ cfg.Overrides = map[string]ThresholdConfig{}
+ cfg.DiskTempByType = nil
+ m.UpdateConfig(cfg)
+ elapsed.Store(int64(2 * time.Minute))
+ m.CheckHost(host)
+ if len(m.GetActiveAlerts()) != 0 {
+ t.Fatal("re-enable inherited the pre-disable start")
+ }
+ elapsed.Store(int64(2*time.Minute + 59*time.Second))
+ m.CheckHost(host)
+ if len(m.GetActiveAlerts()) != 0 {
+ t.Fatal("fresh grace fired early")
+ }
+ elapsed.Store(int64(3 * time.Minute))
+ m.CheckHost(host)
+ if len(m.GetActiveAlerts()) != 1 {
+ t.Fatal("fresh continuous breach never fired")
+ }
+ m.mu.RLock()
+ defer m.mu.RUnlock()
+ if len(m.hostDiskTempPendingContexts) != 0 {
+ t.Fatal("context outlived the pending run")
+ }
+ })
+ }
+ }
+}
+
+func TestDiskTemperaturePendingRestartContextIsNotInvented(t *testing.T) {
+ m, elapsed := continuityManager(t, true)
+ host := hostWithSMARTDiskTemp("restored-temp", "sata", 85)
+ id := hostDiskTemperatureResourceID(host.ID, host.Sensors.SMART[0].Device)
+ m.CheckHost(host)
+ m.mu.Lock()
+ // Persisted intent carries grace but not host link/type context. Reducer
+ // pending incidents are not restored; this is the actual restart shape.
+ m.core.Reset()
+ m.hostDiskTempPendingContexts = nil
+ key := canonicalMetricStateID(id, "diskTemperature")
+ _, pending := m.intentPending[key]
+ m.mu.Unlock()
+ if !pending {
+ t.Fatal("explicit intent control did not retain grace")
+ }
+ m.UpdateConfig(m.GetConfig())
+ m.mu.RLock()
+ _, pending = m.intentPending[key]
+ m.mu.RUnlock()
+ if pending {
+ t.Fatal("save retained grace whose link/type enablement is unknown")
+ }
+ elapsed.Store(int64(2 * time.Minute))
+ m.CheckHost(host)
+ if len(m.GetActiveAlerts()) != 0 {
+ t.Fatal("restored grace fired without fresh evidence")
+ }
+}
+
+func TestDiskTemperatureCurrentCollectionAndOverrideControls(t *testing.T) {
+ for _, provenance := range []string{"legacy", "available", "other-field-only"} {
+ t.Run(provenance, func(t *testing.T) {
+ m, elapsed := continuityManager(t, false)
+ host := hostWithSMARTDiskTemp("current-temp", "sata", 85)
+ switch provenance {
+ case "available":
+ host.Sensors.SMART[0].Collection = &diskinventory.CollectionStatus{Temperature: diskinventory.Available("smartctl")}
+ case "other-field-only":
+ host.Sensors.SMART[0].Collection = &diskinventory.CollectionStatus{Serial: diskinventory.Available("smartctl")}
+ }
+ m.CheckHost(host)
+ elapsed.Store(int64(time.Minute))
+ m.CheckHost(host)
+ if len(m.GetActiveAlerts()) != 1 {
+ t.Fatal("continuous accepted temperature did not fire")
+ }
+ })
+ }
+ t.Run("host override wins over disabled type", func(t *testing.T) {
+ m, elapsed := continuityManager(t, false)
+ host := hostWithSMARTDiskTemp("override-temp", "sata", 85)
+ cfg := m.GetConfig()
+ cfg.DiskTempByType = map[string]HysteresisThreshold{"sata": {Trigger: 0}}
+ cfg.Overrides[host.ID] = ThresholdConfig{DiskTemperature: &HysteresisThreshold{Trigger: 80, Clear: 70}}
+ m.UpdateConfig(cfg)
+ m.CheckHost(host)
+ m.UpdateConfig(m.GetConfig())
+ elapsed.Store(int64(time.Minute))
+ m.CheckHost(host)
+ if len(m.GetActiveAlerts()) != 1 {
+ t.Fatal("save discarded an enabled explicit override")
+ }
+ })
+}
diff --git a/internal/alerts/manager.go b/internal/alerts/manager.go
index 1447c03df2..27a4e9fab5 100644
--- a/internal/alerts/manager.go
+++ b/internal/alerts/manager.go
@@ -78,6 +78,7 @@ type Manager struct {
unifiedIncidentConfirmations map[string]int // Track consecutive provider-incident observations before activation
unifiedIncidentFirstSeen map[string]time.Time // Preserve the first confirmed observation as lifecycle start
unifiedIncidentRecoveries map[string]int // Track consecutive healthy observations before provider-incident recovery
+ hostDiskTempPendingContexts map[string]hostDiskTempContext // Only live pending temperature runs; config saves need link/type identity
hostDiskTempAbsences map[string]int // Track consecutive SMART reports missing a disk that holds temperature alert state
dockerRestartTracking map[string]*dockerRestartRecord // Track restart counts and times for restart loop detection
dockerUpdateFirstSeen map[string]time.Time // Track when image updates were first detected for alert delay
diff --git a/internal/alerts/metric_observation_continuity_test.go b/internal/alerts/metric_observation_continuity_test.go
index ee9a372651..c916050b51 100644
--- a/internal/alerts/metric_observation_continuity_test.go
+++ b/internal/alerts/metric_observation_continuity_test.go
@@ -13,17 +13,20 @@ import (
"github.com/rcourtman/pulse-go-rewrite/internal/alerts/reducer"
"github.com/rcourtman/pulse-go-rewrite/internal/models"
"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
+ "github.com/rcourtman/pulse-go-rewrite/pkg/diskinventory"
)
func continuityManager(t *testing.T, explicit bool) (*Manager, *atomic.Int64) {
t.Helper()
m := newEventLogManager(t)
cfg := characterizationBaseConfig()
- cfg.MetricTimeThresholds = map[string]map[string]int{"all": {"cpu": 60, "memory": 60, "disk": 60, "temperature": 60, "diskRead": 60, "diskWrite": 60, "networkIn": 60, "networkOut": 60, "usage": 60}}
+ cfg.MetricTimeThresholds = map[string]map[string]int{"all": {"cpu": 60, "memory": 60, "disk": 60, "temperature": 60, "diskTemperature": 60, "diskRead": 60, "diskWrite": 60, "networkIn": 60, "networkOut": 60, "usage": 60}}
cfg.MetricEvaluationWindows = nil
cfg.GuestDefaults.CPU = &HysteresisThreshold{Trigger: 99, Clear: 98}
cfg.GuestDefaults.Memory = &HysteresisThreshold{Trigger: 80, Clear: 70}
cfg.GuestDefaults.Disk = &HysteresisThreshold{Trigger: 80, Clear: 70}
+ cfg.AgentDefaults.DiskTemperature = &HysteresisThreshold{Trigger: 80, Clear: 70}
+ cfg.DiskTempByType = nil
cfg.AgentDefaults.Memory = &HysteresisThreshold{Trigger: 80, Clear: 70}
m.UpdateConfig(cfg)
elapsed := &atomic.Int64{}
@@ -32,7 +35,7 @@ func continuityManager(t *testing.T, explicit bool) (*Manager, *atomic.Int64) {
m.intentClock = func() time.Duration { return time.Duration(elapsed.Load()) }
if explicit {
doc := NewAlertIntentPolicyDocument()
- for _, metric := range []string{"cpu", "memory", "disk", "temperature", "diskWrite", "networkIn", "networkOut", "diskRead", "usage"} {
+ for _, metric := range []string{"cpu", "memory", "disk", "temperature", "diskTemperature", "diskWrite", "networkIn", "networkOut", "diskRead", "usage"} {
doc.Defaults[MetricAlertIntentSignal(metric)] = AlertIntentRule{GraceSeconds: intPointer(60)}
}
if err := m.LoadIntentPolicies(doc); err != nil {
@@ -114,6 +117,34 @@ func continuityObserver(t *testing.T, m *Manager, route, gap string) (string, st
}
m.CheckHost(host)
}
+ case "host-disk-temperature":
+ id, metric = hostDiskTemperatureResourceID("temp-host", "/dev/sda"), "diskTemperature"
+ return id, metric, func(value float64, missing bool) {
+ host := models.Host{ID: "temp-host", Hostname: "host", Status: "online",
+ Sensors: models.HostSensorSummary{SMART: []models.HostDiskSMART{{Device: "/dev/sda", Temperature: int(value)}}}}
+ if missing {
+ switch gap {
+ case "empty":
+ host.Sensors.SMART = nil
+ case "omitted":
+ host.Sensors.SMART = []models.HostDiskSMART{{Device: "/dev/sdb", Temperature: 10}}
+ case "zero", "legacy":
+ host.Sensors.SMART[0].Temperature = 0
+ case "negative":
+ host.Sensors.SMART[0].Temperature = -1
+ case "standby":
+ host.Sensors.SMART[0].Standby = true
+ case "expired":
+ m.HandleHostTelemetryExpired(host)
+ return
+ default:
+ host.Sensors.SMART[0].Collection = &diskinventory.CollectionStatus{Temperature: diskinventory.FieldStatus{State: diskinventory.FieldState(gap)}}
+ }
+ } else if gap != "legacy" {
+ host.Sensors.SMART[0].Collection = &diskinventory.CollectionStatus{Temperature: diskinventory.Available("smartctl")}
+ }
+ m.CheckHost(host)
+ }
case "guest-memory", "guest-aggregate", "guest-filesystem", "guest-filesystem-expired":
if route != "guest-memory" {
metric = "disk"
@@ -153,11 +184,14 @@ func continuityObserver(t *testing.T, m *Manager, route, gap string) (string, st
func TestMetricObservationGapRestartsActivation(t *testing.T) {
for _, explicit := range []bool{false, true} {
- for _, route := range []string{"legacy", "canonical", "unified", "host", "guest-memory", "guest-aggregate", "guest-filesystem", "guest-filesystem-expired"} {
+ for _, route := range []string{"legacy", "canonical", "unified", "host", "guest-memory", "guest-aggregate", "guest-filesystem", "guest-filesystem-expired", "host-disk-temperature"} {
gaps := []string{"missing"}
if route == "legacy" || route == "canonical" {
gaps = []string{"NaN", "+Inf", "-Inf", "history-error", "history-empty", "history-gap", "history-NaN"}
}
+ if route == "host-disk-temperature" {
+ gaps = []string{"empty", "omitted", "zero", "negative", "standby", "unavailable", "unsupported", "missing", "invalid-state", "expired", "legacy"}
+ }
for _, gap := range gaps {
t.Run(route+"/"+gap+"/explicit="+map[bool]string{false: "false", true: "true"}[explicit], func(t *testing.T) {
m, elapsed := continuityManager(t, explicit)
@@ -203,11 +237,14 @@ func TestMetricObservationGapRestartsActivation(t *testing.T) {
}
func TestMetricObservationGapRestartsRecovery(t *testing.T) {
- for _, route := range []string{"legacy", "canonical", "unified", "host", "guest-memory"} {
+ for _, route := range []string{"legacy", "canonical", "unified", "host", "guest-memory", "host-disk-temperature"} {
gaps := []string{"missing"}
if route == "legacy" || route == "canonical" {
gaps = []string{"NaN", "+Inf", "-Inf", "history-error", "history-empty", "history-gap", "history-NaN"}
}
+ if route == "host-disk-temperature" {
+ gaps = []string{"empty", "omitted", "zero", "negative", "standby", "unavailable", "unsupported", "missing", "invalid-state", "expired", "legacy"}
+ }
for _, gap := range gaps {
t.Run(route+"/"+gap, func(t *testing.T) {
m, elapsed := continuityManager(t, false)