Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Preserve SMART risk alerts across missing agent reports

pulse-triage[bot]4c796995b2c0
diff --git a/docs/release-control/v6/internal/subsystems/alerts.md b/docs/release-control/v6/internal/subsystems/alerts.md
index e409d32d97..a524c531a3 100644
--- a/docs/release-control/v6/internal/subsystems/alerts.md
+++ b/docs/release-control/v6/internal/subsystems/alerts.md
@@ -3849,3 +3849,30 @@ valid/legacy/provenance controls, missing/rejected/expiry gaps on both timing
edges, explicit grace, acknowledgement/event preservation, host isolation,
configuration intervals, override precedence and departure. These deterministic
source tests do not establish installed temperatures or native guest recovery.
+
+### SMART risk inventory gaps and disabled rules
+
+Agent `disk-health` and `disk-wearout` alerts follow the SMART inventory, not
+filesystem usage cleanup. Empty inventory, standby and unobserved risk fields
+hold the existing occurrence and acknowledgement without recovery/refiring.
+Departure requires three consecutive non-empty reports omitting the disk;
+listing it again, an empty inventory or reporting-lease expiry restarts that
+confirmation. Both risk families on the same disk count once per report.
+Filesystem absence still resolves usage on its first missing report. Temperature
+pending/grace/recovery interruption and departure semantics remain unchanged.
+
+Disabling every recorded cause releases that risk alert without a reading,
+both on a configuration save and on an empty report. Partially enabled causes,
+missing legacy cause metadata and unknown codes remain conservative. Re-enabling
+cannot raise an alert from absent evidence; fresh bad evidence starts a new
+occurrence. Node linking retains existing node ownership, including when the
+agent's SMART inventory is empty; host removal and policy disablement release
+agent risk state and stale absence counts. Counts are ephemeral and restart
+conservatively after a process restart.
+
+`TestHostSMARTRisk*` in `host_unraid_lifecycle_test.go` exercises actual `CheckHost`, expiry, configuration
+save, firing/recovery callbacks and event history with synthetic telemetry.
+It covers both families sharing a disk, interruption/reconfirmation, partial
+rule disablement, re-enablement, legacy/unknown causes, filesystem and host
+isolation, removal and node-link ownership. These are source lifecycle controls,
+not collector, smartctl, installed notification or native disk-recovery proof.
diff --git a/internal/alerts/config_runtime.go b/internal/alerts/config_runtime.go
index 6fe14b53e6..d099d13728 100644
--- a/internal/alerts/config_runtime.go
+++ b/internal/alerts/config_runtime.go
@@ -102,6 +102,7 @@ func (m *Manager) UpdateConfig(config AlertConfig) {
// Re-evaluate active alerts against new thresholds
m.reevaluateActiveAlertsLocked()
+ m.pruneHostSMARTDiskAbsencesNoLock()
}
// migrateActivationState handles backward compatibility for activation state.
@@ -435,6 +436,13 @@ func (m *Manager) reevaluateActiveAlertsLocked() {
alertsToResolve = append(alertsToResolve, alertID)
continue
}
+ if isHostSMARTRiskAlertType(metricType) {
+ smartThresholds, crcMinimumDelta := hostSMARTRiskThresholds(thresholds)
+ if hostSMARTRiskRulesOff(alert, smartThresholds, crcMinimumDelta) {
+ alertsToResolve = append(alertsToResolve, alertID)
+ }
+ continue
+ }
threshold = getThresholdForMetric(thresholds, metricType)
// Agent policy is the only policy for agent alerts. Falling
// through would let the storage or guest switches resolve a
diff --git a/internal/alerts/host.go b/internal/alerts/host.go
index b3f592d6c3..3320916881 100644
--- a/internal/alerts/host.go
+++ b/internal/alerts/host.go
@@ -449,17 +449,20 @@ func (m *Manager) CheckHost(host models.Host) {
}
evaluatesSummaryDisk := false
summaryDiskLive := false
- if len(host.Sensors.SMART) > 0 {
- for _, disk := range host.Sensors.SMART {
- diskResourceID, diskName := hostSMARTDiskResourceID(host, disk)
- if host.LinkedNodeID == "" {
- seenDisks[diskResourceID] = struct{}{}
+ if host.LinkedNodeID != "" {
+ // Preserve node ownership even when this report carries no SMART list.
+ m.clearHostSMARTDiskAlerts(host.ID)
+ } else {
+ var seenSMARTDisks map[string]struct{}
+ if len(host.Sensors.SMART) > 0 {
+ seenSMARTDisks = make(map[string]struct{}, len(host.Sensors.SMART))
+ for _, disk := range host.Sensors.SMART {
+ diskResourceID, diskName := hostSMARTDiskResourceID(host, disk)
+ seenSMARTDisks[diskResourceID] = struct{}{}
m.syncHostSMARTDiskRiskAlerts(host, disk, diskResourceID, diskName, nodeName, instanceName, baseMetadata, thresholds)
- continue
}
- m.syncHostSMARTDiskAlert(host, disk, diskResourceID, diskName, nodeName, instanceName, baseMetadata, "disk-health", nil)
- m.syncHostSMARTDiskAlert(host, disk, diskResourceID, diskName, nodeName, instanceName, baseMetadata, "disk-wearout", nil)
}
+ m.cleanupHostSMARTDiskAlerts(host.ID, seenSMARTDisks, thresholds)
}
for _, disk := range host.Disks {
@@ -728,6 +731,7 @@ func (m *Manager) HandleHostTelemetryExpired(host models.Host) {
// Expiry interrupts timing, not the separate connectivity confirmation.
m.interruptHostDiskTemperatureRuns(host.ID)
+ m.resetHostSMARTDiskAbsences(host.ID)
if host.Unraid != nil {
unraid := *host.Unraid
@@ -984,6 +988,7 @@ func (m *Manager) clearHostDiskAlerts(hostID string) {
m.mu.Lock()
defer m.mu.Unlock()
+ m.resetHostSMARTDiskAbsencesNoLock(hostID)
for storageKey, alert := range m.activeAlerts {
alertID := effectiveAlertID(alert, storageKey)
@@ -1400,6 +1405,10 @@ func (m *Manager) cleanupHostDiskAlerts(host models.Host, seen map[string]struct
if !matches {
continue
}
+ // SMART health and wear follow their own inventory, not filesystems.
+ if isHostSMARTRiskAlertType(alert.Type) {
+ continue
+ }
if _, exists := seen[alert.ResourceID]; exists {
continue
}
@@ -1408,25 +1417,15 @@ func (m *Manager) cleanupHostDiskAlerts(host models.Host, seen map[string]struct
}
func (m *Manager) syncHostSMARTDiskRiskAlerts(host models.Host, disk models.HostDiskSMART, resourceID, resourceName, nodeName, instanceName string, baseMetadata map[string]interface{}, thresholds ThresholdConfig) {
- smartThresholds := storagehealth.SMARTThresholds{
- HealthFailure: intValue(thresholds.SMARTHealthFailure) > 0,
- ReallocatedSectors: int64Value(thresholds.SMARTReallocated),
- PendingSectors: int64Value(thresholds.SMARTPending),
- OfflineUncorrectable: int64Value(thresholds.SMARTUncorrectable),
- MediaErrors: int64Value(thresholds.SMARTMediaErrors),
- LifeWarning: intValue(thresholds.SMARTLifeWarning),
- LifeCritical: intValue(thresholds.SMARTLifeCritical),
- AvailableSpareWarn: intValue(thresholds.SMARTSpareWarning),
- AvailableSpareCrit: intValue(thresholds.SMARTSpareCritical),
- }
+ smartThresholds, crcMinimumDelta := hostSMARTRiskThresholds(thresholds)
assessment := storagehealth.AssessHostSMARTDiskWithThresholds(disk, smartThresholds)
- assessment.Reasons = append(assessment.Reasons, m.hostSMARTCounterGrowthReasons(resourceID, disk, int64Value(thresholds.SMARTCRCErrorDelta))...)
+ assessment.Reasons = append(assessment.Reasons, m.hostSMARTCounterGrowthReasons(resourceID, disk, crcMinimumDelta)...)
healthReasons, wearReasons := splitSMARTAlertReasons(assessment.Reasons)
- if m.hostSMARTDiskAlertEvidenceKnown(resourceID, "disk-health", disk, healthReasons, smartThresholds, int64Value(thresholds.SMARTCRCErrorDelta)) {
+ if m.hostSMARTDiskAlertEvidenceKnown(resourceID, "disk-health", disk, healthReasons, smartThresholds, crcMinimumDelta) {
m.syncHostSMARTDiskAlert(host, disk, resourceID, resourceName, nodeName, instanceName, baseMetadata, "disk-health", healthReasons)
}
- if m.hostSMARTDiskAlertEvidenceKnown(resourceID, "disk-wearout", disk, wearReasons, smartThresholds, int64Value(thresholds.SMARTCRCErrorDelta)) {
+ if m.hostSMARTDiskAlertEvidenceKnown(resourceID, "disk-wearout", disk, wearReasons, smartThresholds, crcMinimumDelta) {
m.syncHostSMARTDiskAlert(host, disk, resourceID, resourceName, nodeName, instanceName, baseMetadata, "disk-wearout", wearReasons)
}
}
diff --git a/internal/alerts/host_smart_lifecycle.go b/internal/alerts/host_smart_lifecycle.go
new file mode 100644
index 0000000000..37ed5f8285
--- /dev/null
+++ b/internal/alerts/host_smart_lifecycle.go
@@ -0,0 +1,141 @@
+package alerts
+
+import (
+ "strings"
+
+ "github.com/rcourtman/pulse-go-rewrite/internal/storagehealth"
+)
+
+// Match temperature's existing departure confirmation without sharing its
+// pending-run state or resource IDs. One failed probe can omit a disk.
+const hostSMARTRiskAbsenceConfirmations = hostDiskTemperatureAbsenceConfirmations
+
+func isHostSMARTRiskAlertType(alertType string) bool {
+ return alertType == "disk-health" || alertType == "disk-wearout"
+}
+
+func hostSMARTRiskThresholds(thresholds ThresholdConfig) (storagehealth.SMARTThresholds, int64) {
+ return storagehealth.SMARTThresholds{
+ HealthFailure: intValue(thresholds.SMARTHealthFailure) > 0,
+ ReallocatedSectors: int64Value(thresholds.SMARTReallocated),
+ PendingSectors: int64Value(thresholds.SMARTPending),
+ OfflineUncorrectable: int64Value(thresholds.SMARTUncorrectable),
+ MediaErrors: int64Value(thresholds.SMARTMediaErrors),
+ LifeWarning: intValue(thresholds.SMARTLifeWarning),
+ LifeCritical: intValue(thresholds.SMARTLifeCritical),
+ AvailableSpareWarn: intValue(thresholds.SMARTSpareWarning),
+ AvailableSpareCrit: intValue(thresholds.SMARTSpareCritical),
+ }, int64Value(thresholds.SMARTCRCErrorDelta)
+}
+
+// Missing risk metadata is not evidence that the rules which raised an old
+// alert have been disabled. Unknown codes are conservatively enabled too.
+func hostSMARTRiskRulesOff(alert *Alert, thresholds storagehealth.SMARTThresholds, crcMinimumDelta int64) bool {
+ if alert == nil {
+ return false
+ }
+ codes := hostSMARTRiskCodes(alert.Metadata["riskCodes"])
+ if len(codes) == 0 {
+ return false
+ }
+ for _, code := range codes {
+ if hostSMARTRiskRuleEnabled(code, thresholds, crcMinimumDelta) {
+ return false
+ }
+ }
+ return true
+}
+
+// Caller holds m.mu. Keep counts only for resources still holding risk alerts.
+// A disabled global policy breaks the observation sequence, not the alert.
+func (m *Manager) pruneHostSMARTDiskAbsencesNoLock() {
+ active := make(map[string]struct{})
+ if m.config.Enabled {
+ for _, alert := range m.activeAlerts {
+ if alert != nil && isHostSMARTRiskAlertType(alert.Type) {
+ active[alert.ResourceID] = struct{}{}
+ }
+ }
+ }
+ for resourceID := range m.hostSMARTRiskAbsences {
+ if _, exists := active[resourceID]; !exists {
+ delete(m.hostSMARTRiskAbsences, resourceID)
+ }
+ }
+}
+
+func (m *Manager) resetHostSMARTDiskAbsencesNoLock(hostID string) {
+ prefix := hostResourceID(hostID) + "/disk:"
+ for resourceID := range m.hostSMARTRiskAbsences {
+ if strings.HasPrefix(resourceID, prefix) {
+ delete(m.hostSMARTRiskAbsences, resourceID)
+ }
+ }
+}
+
+func (m *Manager) resetHostSMARTDiskAbsences(hostID string) {
+ m.mu.Lock()
+ defer m.mu.Unlock()
+ m.resetHostSMARTDiskAbsencesNoLock(hostID)
+}
+
+func (m *Manager) clearHostSMARTDiskAlerts(hostID string) {
+ prefix := hostResourceID(hostID) + "/disk:"
+ m.mu.Lock()
+ defer m.mu.Unlock()
+ m.resetHostSMARTDiskAbsencesNoLock(hostID)
+ for storageKey, alert := range m.activeAlerts {
+ if alert != nil && isHostSMARTRiskAlertType(alert.Type) && strings.HasPrefix(alert.ResourceID, prefix) {
+ m.clearAlertNoLock(storageKey)
+ }
+ }
+}
+
+// An empty report holds risk alerts and breaks departure confirmation. A
+// listed disk (including standby or unknown fields) restarts its count; the
+// ordinary evidence evaluator decides whether its risk actually recovered.
+// Only three consecutive non-empty omissions release departed disk alerts.
+// Explicit rule disablement is authoritative even without a disk reading.
+func (m *Manager) cleanupHostSMARTDiskAlerts(hostID string, seen map[string]struct{}, thresholds ThresholdConfig) {
+ prefix := hostResourceID(hostID) + "/disk:"
+ smartThresholds, crcMinimumDelta := hostSMARTRiskThresholds(thresholds)
+ m.mu.Lock()
+ defer m.mu.Unlock()
+ if m.hostSMARTRiskAbsences == nil {
+ m.hostSMARTRiskAbsences = make(map[string]int)
+ }
+ absent := make(map[string][]string)
+ for storageKey, alert := range m.activeAlerts {
+ if alert == nil || !isHostSMARTRiskAlertType(alert.Type) || !strings.HasPrefix(alert.ResourceID, prefix) {
+ continue
+ }
+ if hostSMARTRiskRulesOff(alert, smartThresholds, crcMinimumDelta) {
+ m.clearAlertNoLock(storageKey)
+ continue
+ }
+ if _, exists := seen[alert.ResourceID]; !exists {
+ absent[alert.ResourceID] = append(absent[alert.ResourceID], storageKey)
+ }
+ }
+ for resourceID := range m.hostSMARTRiskAbsences {
+ if !strings.HasPrefix(resourceID, prefix) {
+ continue
+ }
+ if _, missing := absent[resourceID]; !missing || seen == nil {
+ delete(m.hostSMARTRiskAbsences, resourceID)
+ }
+ }
+ if seen == nil {
+ return
+ }
+ for resourceID, storageKeys := range absent {
+ m.hostSMARTRiskAbsences[resourceID]++
+ if m.hostSMARTRiskAbsences[resourceID] < hostSMARTRiskAbsenceConfirmations {
+ continue
+ }
+ delete(m.hostSMARTRiskAbsences, resourceID)
+ for _, storageKey := range storageKeys {
+ m.clearAlertNoLock(storageKey)
+ }
+ }
+}
diff --git a/internal/alerts/host_unraid_lifecycle_test.go b/internal/alerts/host_unraid_lifecycle_test.go
index b340cb7bf1..c6716ece4d 100644
--- a/internal/alerts/host_unraid_lifecycle_test.go
+++ b/internal/alerts/host_unraid_lifecycle_test.go
@@ -3,6 +3,7 @@ package alerts
import (
"fmt"
"strings"
+ "sync/atomic"
"testing"
"time"
@@ -1064,3 +1065,312 @@ func TestDiskTemperaturePendingContextCleanup(t *testing.T) {
})
}
}
+
+// Feed only synthetic telemetry through the real CheckHost, config and callback
+// paths. No collector, smartctl process, native host or notification provider.
+type smartLifecycleFixture struct {
+ m *Manager
+ host models.Host
+ healthID, wearID string
+ initial map[string]Alert
+ fires atomic.Int32
+ recoveries chan *ResolvedAlert
+}
+
+func newSMARTLifecycleFixture(t *testing.T) *smartLifecycleFixture {
+ t.Helper()
+ m := newEventLogManager(t)
+ cfg := m.GetConfig()
+ cfg.FlappingEnabled = false
+ cfg.Schedule.Cooldown = 0
+ cfg.Schedule.MaxAlertsHour = 0
+ cfg.AgentDefaults.Disk = &HysteresisThreshold{Trigger: 80, Clear: 70}
+ cfg.AgentDefaults.DiskTemperature = &HysteresisThreshold{Trigger: 80, Clear: 70}
+ cfg.DiskTempByType = map[string]HysteresisThreshold{"sata": {Trigger: 80, Clear: 70}}
+ m.UpdateConfig(cfg)
+ m.mu.Lock()
+ m.config.TimeThresholds = map[string]int{}
+ m.config.MetricTimeThresholds = map[string]map[string]int{"all": {"disk": 0, "diskTemperature": 0}}
+ m.mu.Unlock()
+ used, pending := 96, int64(2)
+ host := models.Host{ID: "smart-lifecycle", Hostname: "storage", Status: "online",
+ Disks: []models.Disk{{Mountpoint: "/data", Device: "/dev/sda1", Total: 1000, Used: 950, Usage: 95}},
+ Sensors: models.HostSensorSummary{SMART: []models.HostDiskSMART{
+ {Device: "/dev/sda", Type: "sata", Health: "FAILED", Temperature: 95, Attributes: &models.SMARTAttributes{PercentageUsed: &used, PendingSectors: &pending}},
+ {Device: "/dev/sdb", Type: "sata", Health: "PASSED", Temperature: 20},
+ }},
+ }
+ resourceID, _ := hostSMARTDiskResourceID(host, host.Sensors.SMART[0])
+ f := &smartLifecycleFixture{m: m, host: host,
+ healthID: buildCanonicalStateID(resourceID, resourceID+"-disk-health"),
+ wearID: buildCanonicalStateID(resourceID, resourceID+"-disk-wearout"),
+ initial: make(map[string]Alert), recoveries: make(chan *ResolvedAlert, 32),
+ }
+ m.SetAlertCallback(func(a *Alert) {
+ if a.Type == "disk-health" || a.Type == "disk-wearout" {
+ f.fires.Add(1)
+ }
+ })
+ m.SetResolvedAlertCallback(func(r *ResolvedAlert) {
+ if r.Alert.Type == "disk-health" || r.Alert.Type == "disk-wearout" {
+ f.recoveries <- r
+ }
+ })
+ m.CheckHost(host)
+ for _, id := range []string{f.healthID, f.wearID} {
+ f.initial[id] = *testRequireActiveAlert(t, m, id)
+ }
+ if f.fires.Load() != 2 {
+ t.Fatalf("initial risk callbacks = %d, want 2", f.fires.Load())
+ }
+ return f
+}
+
+func (f *smartLifecycleFixture) held(t *testing.T) {
+ t.Helper()
+ for _, id := range []string{f.healthID, f.wearID} {
+ a := testRequireActiveAlert(t, f.m, id)
+ if !a.StartTime.Equal(f.initial[id].StartTime) {
+ t.Fatalf("%s was replaced by a new occurrence", id)
+ }
+ }
+ if f.fires.Load() != 2 {
+ t.Fatalf("missing report refired risk notifications: %d", f.fires.Load())
+ }
+ for _, e := range queryAlertEvents(t, f.m, eventlog.Filter{Types: []string{eventlog.TypeResolved}}) {
+ if e.AlertID == f.healthID || e.AlertID == f.wearID {
+ t.Fatalf("missing evidence published recovery: %+v", e)
+ }
+ }
+}
+
+func (f *smartLifecycleFixture) resolved(t *testing.T, ids ...string) {
+ t.Helper()
+ want := make(map[string]bool)
+ for _, id := range ids {
+ if testHasActiveAlert(t, f.m, id) {
+ t.Fatalf("risk alert remains active: %s", id)
+ }
+ want[id] = true
+ }
+ for range ids {
+ select {
+ case r := <-f.recoveries:
+ if !want[r.Alert.ID] || !r.Alert.StartTime.Equal(f.initial[r.Alert.ID].StartTime) {
+ t.Fatalf("wrong recovery occurrence: %+v", r)
+ }
+ delete(want, r.Alert.ID)
+ case <-time.After(3 * time.Second):
+ t.Fatal("ordinary recovery callback did not complete")
+ }
+ }
+ for _, id := range ids {
+ events := queryAlertEvents(t, f.m, eventlog.Filter{AlertID: id, Types: []string{eventlog.TypeResolved}})
+ if len(events) != 1 {
+ t.Fatalf("%s has %d resolution events, want 1", id, len(events))
+ }
+ }
+}
+
+func TestHostSMARTRiskMissingReportsPreserveOccurrenceAndCallbacks(t *testing.T) {
+ f := newSMARTLifecycleFixture(t)
+ if err := f.m.AcknowledgeAlert(f.healthID, "operator"); err != nil {
+ t.Fatal(err)
+ }
+ present := f.host.Sensors.SMART
+ // Both risk families on one disk must count once per report, not twice.
+ for _, list := range [][]models.HostDiskSMART{nil, {present[1]}, {present[1]}, nil, {present[1]}, present, nil, present} {
+ f.host.Sensors.SMART = list
+ f.m.CheckHost(f.host)
+ f.held(t)
+ if !testRequireActiveAlert(t, f.m, f.healthID).Acknowledged {
+ t.Fatal("acknowledgement lost during observation gap")
+ }
+ }
+ // Standby and unavailable health/wear fields are presence, not recovery.
+ f.host.Sensors.SMART = []models.HostDiskSMART{{Device: "/dev/sda", Standby: true}, present[1]}
+ f.m.CheckHost(f.host)
+ f.held(t)
+ f.host.Sensors.SMART[0].Standby = false
+ f.m.CheckHost(f.host)
+ f.held(t)
+ used, pending := 10, int64(0)
+ f.host.Sensors.SMART[0] = models.HostDiskSMART{Device: "/dev/sda", Health: "PASSED", Temperature: 20, Attributes: &models.SMARTAttributes{PercentageUsed: &used, PendingSectors: &pending}}
+ f.m.CheckHost(f.host)
+ f.resolved(t, f.healthID, f.wearID)
+ if f.fires.Load() != 2 {
+ t.Fatalf("healthy recovery refired: %d", f.fires.Load())
+ }
+ t.Log("empty/partial/standby/unknown reports preserved occurrence and acknowledgement; fresh complete evidence recovered once")
+}
+
+func TestHostSMARTRiskDepartureRequiresFreshConsecutiveReports(t *testing.T) {
+ for _, gap := range []string{"empty", "expiry", "present"} {
+ t.Run(gap, func(t *testing.T) {
+ f := newSMARTLifecycleFixture(t)
+ present := f.host.Sensors.SMART
+ f.host.Sensors.SMART = []models.HostDiskSMART{present[1]}
+ for i := 0; i < 2; i++ {
+ f.m.CheckHost(f.host)
+ f.held(t)
+ }
+ switch gap {
+ case "empty":
+ f.host.Sensors.SMART = nil
+ f.m.CheckHost(f.host)
+ case "expiry":
+ f.m.HandleHostTelemetryExpired(f.host)
+ case "present":
+ f.host.Sensors.SMART = present
+ f.m.CheckHost(f.host)
+ }
+ f.held(t)
+ f.host.Sensors.SMART = []models.HostDiskSMART{present[1]}
+ for i := 1; i <= 3; i++ {
+ f.m.CheckHost(f.host)
+ if i < 3 {
+ f.held(t)
+ }
+ }
+ f.resolved(t, f.healthID, f.wearID)
+ if f.fires.Load() != 2 {
+ t.Fatalf("departure refired %d callbacks", f.fires.Load())
+ }
+ })
+ }
+}
+
+func TestHostSMARTRiskRulesDisableBetweenReports(t *testing.T) {
+ f := newSMARTLifecycleFixture(t)
+ cfg := f.m.GetConfig()
+ off := 0
+ cfg.AgentDefaults.SMARTHealthFailure = &off
+ f.m.UpdateConfig(cfg)
+ f.held(t) // Pending sectors remain enabled; disabling only one cause is not recovery.
+ zero := int64(0)
+ cfg.AgentDefaults.SMARTPending = &zero
+ f.m.UpdateConfig(cfg)
+ f.resolved(t, f.healthID)
+ if !testHasActiveAlert(t, f.m, f.wearID) {
+ t.Fatal("health policy disabled unrelated wearout")
+ }
+ // A missing report cannot resurrect it. Re-enable before another report;
+ // only a newly observed failing disk may start another occurrence.
+ f.host.Sensors.SMART = nil
+ f.m.CheckHost(f.host)
+ if testHasActiveAlert(t, f.m, f.healthID) {
+ t.Fatal("missing report raised disabled health")
+ }
+ on, one := 1, int64(1)
+ cfg.AgentDefaults.SMARTHealthFailure = &on
+ cfg.AgentDefaults.SMARTPending = &one
+ f.m.UpdateConfig(cfg)
+ f.m.CheckHost(f.host)
+ if testHasActiveAlert(t, f.m, f.healthID) {
+ t.Fatal("re-enable raised health without evidence")
+ }
+ used, pending := 96, int64(2)
+ f.host.Sensors.SMART = []models.HostDiskSMART{{Device: "/dev/sda", Health: "FAILED", Attributes: &models.SMARTAttributes{PercentageUsed: &used, PendingSectors: &pending}}}
+ f.m.CheckHost(f.host)
+ a := testRequireActiveAlert(t, f.m, f.healthID)
+ if a.StartTime.Equal(f.initial[f.healthID].StartTime) || f.fires.Load() != 3 {
+ t.Fatal("fresh bad evidence did not start exactly one new health occurrence")
+ }
+}
+
+func TestHostSMARTRiskEmptyReportHonoursRuleOffAndLegacyUnknowns(t *testing.T) {
+ for _, metadata := range []string{"current", "legacy", "unknown"} {
+ t.Run(metadata, func(t *testing.T) {
+ f := newSMARTLifecycleFixture(t)
+ off, zero := 0, int64(0)
+ f.m.mu.Lock()
+ f.m.config.AgentDefaults.SMARTHealthFailure = &off
+ f.m.config.AgentDefaults.SMARTPending = &zero
+ if metadata != "current" {
+ a, _ := f.m.getActiveAlertNoLock(f.healthID)
+ if metadata == "legacy" {
+ delete(a.Metadata, "riskCodes")
+ } else {
+ a.Metadata["riskCodes"] = []interface{}{"future-code"}
+ }
+ }
+ f.m.mu.Unlock()
+ f.host.Sensors.SMART = nil
+ f.m.CheckHost(f.host)
+ if metadata == "current" {
+ f.resolved(t, f.healthID)
+ } else {
+ f.held(t)
+ }
+ if !testHasActiveAlert(t, f.m, f.wearID) {
+ t.Fatal("wearout was disabled with health")
+ }
+ })
+ }
+}
+
+func TestHostSMARTRiskFilesystemAndHostLifecycleStayIndependent(t *testing.T) {
+ for _, action := range []string{"filesystem-absent", "node-link", "removed", "host-disabled", "agents-disabled", "host-isolation"} {
+ t.Run(action, func(t *testing.T) {
+ f := newSMARTLifecycleFixture(t)
+ switch action {
+ case "filesystem-absent":
+ fsID, _ := hostDiskResourceID(f.host, f.host.Disks[0])
+ id := canonicalMetricStateID(fsID, "disk")
+ testRequireActiveAlert(t, f.m, id)
+ f.host.Disks = nil
+ f.host.Sensors.SMART = nil
+ f.m.CheckHost(f.host)
+ if testHasActiveAlert(t, f.m, id) {
+ t.Fatal("filesystem absence incorrectly held usage")
+ }
+ f.held(t)
+ case "host-isolation":
+ other := f.host
+ other.ID = "smart-lifecycle-other"
+ f.m.CheckHost(other)
+ other.Sensors.SMART = []models.HostDiskSMART{other.Sensors.SMART[1]}
+ for i := 0; i < 3; i++ {
+ f.m.CheckHost(other)
+ }
+ f.held(t)
+ default:
+ present := f.host.Sensors.SMART
+ f.host.Sensors.SMART = []models.HostDiskSMART{present[1]}
+ f.m.CheckHost(f.host)
+ f.held(t)
+ switch action {
+ case "node-link":
+ f.host.LinkedNodeID = "pve-node"
+ f.host.Sensors.SMART = nil
+ f.m.CheckHost(f.host)
+ case "removed":
+ f.m.HandleHostRemoved(f.host)
+ case "host-disabled":
+ cfg := f.m.GetConfig()
+ cfg.Overrides["agent:"+f.host.ID] = ThresholdConfig{Disabled: true}
+ f.m.UpdateConfig(cfg)
+ case "agents-disabled":
+ cfg := f.m.GetConfig()
+ cfg.DisableAllAgents = true
+ f.m.UpdateConfig(cfg)
+ }
+ f.resolved(t, f.healthID, f.wearID)
+ // Returning ownership gets a new occurrence and a fresh departure run.
+ f.host.LinkedNodeID = ""
+ cfg := f.m.GetConfig()
+ cfg.DisableAllAgents = false
+ delete(cfg.Overrides, "agent:"+f.host.ID)
+ f.m.UpdateConfig(cfg)
+ f.host.Sensors.SMART = present
+ f.m.CheckHost(f.host)
+ f.host.Sensors.SMART = []models.HostDiskSMART{present[1]}
+ for i := 0; i < 2; i++ {
+ f.m.CheckHost(f.host)
+ testRequireActiveAlert(t, f.m, f.healthID)
+ testRequireActiveAlert(t, f.m, f.wearID)
+ }
+ }
+ })
+ }
+}
diff --git a/internal/alerts/manager.go b/internal/alerts/manager.go
index a4781899c4..a958d1c906 100644
--- a/internal/alerts/manager.go
+++ b/internal/alerts/manager.go
@@ -79,6 +79,7 @@ type Manager struct {
unifiedIncidentFirstSeen map[string]time.Time // Preserve the first confirmed observation as lifecycle start
unifiedIncidentRecoveries map[string]int // Track consecutive healthy observations before provider-incident recovery
hostDiskTempPendingContexts map[string]hostDiskTempContext // Only live pending temperature runs; config saves need link/type identity
+ hostSMARTRiskAbsences map[string]int // Consecutive non-empty reports omitting a SMART risk resource
hostDiskTempAbsences map[string]int // Track consecutive SMART reports missing a disk that holds temperature alert state
dockerRestartTracking map[string]*dockerRestartRecord // Track restart counts and times for restart loop detection
dockerUpdateFirstSeen map[string]time.Time // Track when image updates were first detected for alert delay