Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Preserve SMART risk alerts across missing agent reports
pulse-triage[bot]4c796995b2c0
diff --git a/docs/release-control/v6/internal/subsystems/alerts.md b/docs/release-control/v6/internal/subsystems/alerts.mdindex e409d32d97..a524c531a3 100644--- a/docs/release-control/v6/internal/subsystems/alerts.md+++ b/docs/release-control/v6/internal/subsystems/alerts.md@@ -3849,3 +3849,30 @@ valid/legacy/provenance controls, missing/rejected/expiry gaps on both timingedges, explicit grace, acknowledgement/event preservation, host isolation,configuration intervals, override precedence and departure. These deterministicsource tests do not establish installed temperatures or native guest recovery.++### SMART risk inventory gaps and disabled rules++Agent `disk-health` and `disk-wearout` alerts follow the SMART inventory, not+filesystem usage cleanup. Empty inventory, standby and unobserved risk fields+hold the existing occurrence and acknowledgement without recovery/refiring.+Departure requires three consecutive non-empty reports omitting the disk;+listing it again, an empty inventory or reporting-lease expiry restarts that+confirmation. Both risk families on the same disk count once per report.+Filesystem absence still resolves usage on its first missing report. Temperature+pending/grace/recovery interruption and departure semantics remain unchanged.++Disabling every recorded cause releases that risk alert without a reading,+both on a configuration save and on an empty report. Partially enabled causes,+missing legacy cause metadata and unknown codes remain conservative. Re-enabling+cannot raise an alert from absent evidence; fresh bad evidence starts a new+occurrence. Node linking retains existing node ownership, including when the+agent's SMART inventory is empty; host removal and policy disablement release+agent risk state and stale absence counts. Counts are ephemeral and restart+conservatively after a process restart.++`TestHostSMARTRisk*` in `host_unraid_lifecycle_test.go` exercises actual `CheckHost`, expiry, configuration+save, firing/recovery callbacks and event history with synthetic telemetry.+It covers both families sharing a disk, interruption/reconfirmation, partial+rule disablement, re-enablement, legacy/unknown causes, filesystem and host+isolation, removal and node-link ownership. These are source lifecycle controls,+not collector, smartctl, installed notification or native disk-recovery proof.diff --git a/internal/alerts/config_runtime.go b/internal/alerts/config_runtime.goindex 6fe14b53e6..d099d13728 100644--- a/internal/alerts/config_runtime.go+++ b/internal/alerts/config_runtime.go@@ -102,6 +102,7 @@ func (m *Manager) UpdateConfig(config AlertConfig) {// Re-evaluate active alerts against new thresholdsm.reevaluateActiveAlertsLocked()+ m.pruneHostSMARTDiskAbsencesNoLock()}// migrateActivationState handles backward compatibility for activation state.@@ -435,6 +436,13 @@ func (m *Manager) reevaluateActiveAlertsLocked() {alertsToResolve = append(alertsToResolve, alertID)continue}+ if isHostSMARTRiskAlertType(metricType) {+ smartThresholds, crcMinimumDelta := hostSMARTRiskThresholds(thresholds)+ if hostSMARTRiskRulesOff(alert, smartThresholds, crcMinimumDelta) {+ alertsToResolve = append(alertsToResolve, alertID)+ }+ continue+ }threshold = getThresholdForMetric(thresholds, metricType)// Agent policy is the only policy for agent alerts. Falling// through would let the storage or guest switches resolve adiff --git a/internal/alerts/host.go b/internal/alerts/host.goindex b3f592d6c3..3320916881 100644--- a/internal/alerts/host.go+++ b/internal/alerts/host.go@@ -449,17 +449,20 @@ func (m *Manager) CheckHost(host models.Host) {}evaluatesSummaryDisk := falsesummaryDiskLive := false- if len(host.Sensors.SMART) > 0 {- for _, disk := range host.Sensors.SMART {- diskResourceID, diskName := hostSMARTDiskResourceID(host, disk)- if host.LinkedNodeID == "" {- seenDisks[diskResourceID] = struct{}{}+ if host.LinkedNodeID != "" {+ // Preserve node ownership even when this report carries no SMART list.+ m.clearHostSMARTDiskAlerts(host.ID)+ } else {+ var seenSMARTDisks map[string]struct{}+ if len(host.Sensors.SMART) > 0 {+ seenSMARTDisks = make(map[string]struct{}, len(host.Sensors.SMART))+ for _, disk := range host.Sensors.SMART {+ diskResourceID, diskName := hostSMARTDiskResourceID(host, disk)+ seenSMARTDisks[diskResourceID] = struct{}{}m.syncHostSMARTDiskRiskAlerts(host, disk, diskResourceID, diskName, nodeName, instanceName, baseMetadata, thresholds)- continue}- m.syncHostSMARTDiskAlert(host, disk, diskResourceID, diskName, nodeName, instanceName, baseMetadata, "disk-health", nil)- m.syncHostSMARTDiskAlert(host, disk, diskResourceID, diskName, nodeName, instanceName, baseMetadata, "disk-wearout", nil)}+ m.cleanupHostSMARTDiskAlerts(host.ID, seenSMARTDisks, thresholds)}for _, disk := range host.Disks {@@ -728,6 +731,7 @@ func (m *Manager) HandleHostTelemetryExpired(host models.Host) {// Expiry interrupts timing, not the separate connectivity confirmation.m.interruptHostDiskTemperatureRuns(host.ID)+ m.resetHostSMARTDiskAbsences(host.ID)if host.Unraid != nil {unraid := *host.Unraid@@ -984,6 +988,7 @@ func (m *Manager) clearHostDiskAlerts(hostID string) {m.mu.Lock()defer m.mu.Unlock()+ m.resetHostSMARTDiskAbsencesNoLock(hostID)for storageKey, alert := range m.activeAlerts {alertID := effectiveAlertID(alert, storageKey)@@ -1400,6 +1405,10 @@ func (m *Manager) cleanupHostDiskAlerts(host models.Host, seen map[string]structif !matches {continue}+ // SMART health and wear follow their own inventory, not filesystems.+ if isHostSMARTRiskAlertType(alert.Type) {+ continue+ }if _, exists := seen[alert.ResourceID]; exists {continue}@@ -1408,25 +1417,15 @@ func (m *Manager) cleanupHostDiskAlerts(host models.Host, seen map[string]struct}func (m *Manager) syncHostSMARTDiskRiskAlerts(host models.Host, disk models.HostDiskSMART, resourceID, resourceName, nodeName, instanceName string, baseMetadata map[string]interface{}, thresholds ThresholdConfig) {- smartThresholds := storagehealth.SMARTThresholds{- HealthFailure: intValue(thresholds.SMARTHealthFailure) > 0,- ReallocatedSectors: int64Value(thresholds.SMARTReallocated),- PendingSectors: int64Value(thresholds.SMARTPending),- OfflineUncorrectable: int64Value(thresholds.SMARTUncorrectable),- MediaErrors: int64Value(thresholds.SMARTMediaErrors),- LifeWarning: intValue(thresholds.SMARTLifeWarning),- LifeCritical: intValue(thresholds.SMARTLifeCritical),- AvailableSpareWarn: intValue(thresholds.SMARTSpareWarning),- AvailableSpareCrit: intValue(thresholds.SMARTSpareCritical),- }+ smartThresholds, crcMinimumDelta := hostSMARTRiskThresholds(thresholds)assessment := storagehealth.AssessHostSMARTDiskWithThresholds(disk, smartThresholds)- assessment.Reasons = append(assessment.Reasons, m.hostSMARTCounterGrowthReasons(resourceID, disk, int64Value(thresholds.SMARTCRCErrorDelta))...)+ assessment.Reasons = append(assessment.Reasons, m.hostSMARTCounterGrowthReasons(resourceID, disk, crcMinimumDelta)...)healthReasons, wearReasons := splitSMARTAlertReasons(assessment.Reasons)- if m.hostSMARTDiskAlertEvidenceKnown(resourceID, "disk-health", disk, healthReasons, smartThresholds, int64Value(thresholds.SMARTCRCErrorDelta)) {+ if m.hostSMARTDiskAlertEvidenceKnown(resourceID, "disk-health", disk, healthReasons, smartThresholds, crcMinimumDelta) {m.syncHostSMARTDiskAlert(host, disk, resourceID, resourceName, nodeName, instanceName, baseMetadata, "disk-health", healthReasons)}- if m.hostSMARTDiskAlertEvidenceKnown(resourceID, "disk-wearout", disk, wearReasons, smartThresholds, int64Value(thresholds.SMARTCRCErrorDelta)) {+ if m.hostSMARTDiskAlertEvidenceKnown(resourceID, "disk-wearout", disk, wearReasons, smartThresholds, crcMinimumDelta) {m.syncHostSMARTDiskAlert(host, disk, resourceID, resourceName, nodeName, instanceName, baseMetadata, "disk-wearout", wearReasons)}}diff --git a/internal/alerts/host_smart_lifecycle.go b/internal/alerts/host_smart_lifecycle.gonew file mode 100644index 0000000000..37ed5f8285--- /dev/null+++ b/internal/alerts/host_smart_lifecycle.go@@ -0,0 +1,141 @@+package alerts++import (+ "strings"++ "github.com/rcourtman/pulse-go-rewrite/internal/storagehealth"+)++// Match temperature's existing departure confirmation without sharing its+// pending-run state or resource IDs. One failed probe can omit a disk.+const hostSMARTRiskAbsenceConfirmations = hostDiskTemperatureAbsenceConfirmations++func isHostSMARTRiskAlertType(alertType string) bool {+ return alertType == "disk-health" || alertType == "disk-wearout"+}++func hostSMARTRiskThresholds(thresholds ThresholdConfig) (storagehealth.SMARTThresholds, int64) {+ return storagehealth.SMARTThresholds{+ HealthFailure: intValue(thresholds.SMARTHealthFailure) > 0,+ ReallocatedSectors: int64Value(thresholds.SMARTReallocated),+ PendingSectors: int64Value(thresholds.SMARTPending),+ OfflineUncorrectable: int64Value(thresholds.SMARTUncorrectable),+ MediaErrors: int64Value(thresholds.SMARTMediaErrors),+ LifeWarning: intValue(thresholds.SMARTLifeWarning),+ LifeCritical: intValue(thresholds.SMARTLifeCritical),+ AvailableSpareWarn: intValue(thresholds.SMARTSpareWarning),+ AvailableSpareCrit: intValue(thresholds.SMARTSpareCritical),+ }, int64Value(thresholds.SMARTCRCErrorDelta)+}++// Missing risk metadata is not evidence that the rules which raised an old+// alert have been disabled. Unknown codes are conservatively enabled too.+func hostSMARTRiskRulesOff(alert *Alert, thresholds storagehealth.SMARTThresholds, crcMinimumDelta int64) bool {+ if alert == nil {+ return false+ }+ codes := hostSMARTRiskCodes(alert.Metadata["riskCodes"])+ if len(codes) == 0 {+ return false+ }+ for _, code := range codes {+ if hostSMARTRiskRuleEnabled(code, thresholds, crcMinimumDelta) {+ return false+ }+ }+ return true+}++// Caller holds m.mu. Keep counts only for resources still holding risk alerts.+// A disabled global policy breaks the observation sequence, not the alert.+func (m *Manager) pruneHostSMARTDiskAbsencesNoLock() {+ active := make(map[string]struct{})+ if m.config.Enabled {+ for _, alert := range m.activeAlerts {+ if alert != nil && isHostSMARTRiskAlertType(alert.Type) {+ active[alert.ResourceID] = struct{}{}+ }+ }+ }+ for resourceID := range m.hostSMARTRiskAbsences {+ if _, exists := active[resourceID]; !exists {+ delete(m.hostSMARTRiskAbsences, resourceID)+ }+ }+}++func (m *Manager) resetHostSMARTDiskAbsencesNoLock(hostID string) {+ prefix := hostResourceID(hostID) + "/disk:"+ for resourceID := range m.hostSMARTRiskAbsences {+ if strings.HasPrefix(resourceID, prefix) {+ delete(m.hostSMARTRiskAbsences, resourceID)+ }+ }+}++func (m *Manager) resetHostSMARTDiskAbsences(hostID string) {+ m.mu.Lock()+ defer m.mu.Unlock()+ m.resetHostSMARTDiskAbsencesNoLock(hostID)+}++func (m *Manager) clearHostSMARTDiskAlerts(hostID string) {+ prefix := hostResourceID(hostID) + "/disk:"+ m.mu.Lock()+ defer m.mu.Unlock()+ m.resetHostSMARTDiskAbsencesNoLock(hostID)+ for storageKey, alert := range m.activeAlerts {+ if alert != nil && isHostSMARTRiskAlertType(alert.Type) && strings.HasPrefix(alert.ResourceID, prefix) {+ m.clearAlertNoLock(storageKey)+ }+ }+}++// An empty report holds risk alerts and breaks departure confirmation. A+// listed disk (including standby or unknown fields) restarts its count; the+// ordinary evidence evaluator decides whether its risk actually recovered.+// Only three consecutive non-empty omissions release departed disk alerts.+// Explicit rule disablement is authoritative even without a disk reading.+func (m *Manager) cleanupHostSMARTDiskAlerts(hostID string, seen map[string]struct{}, thresholds ThresholdConfig) {+ prefix := hostResourceID(hostID) + "/disk:"+ smartThresholds, crcMinimumDelta := hostSMARTRiskThresholds(thresholds)+ m.mu.Lock()+ defer m.mu.Unlock()+ if m.hostSMARTRiskAbsences == nil {+ m.hostSMARTRiskAbsences = make(map[string]int)+ }+ absent := make(map[string][]string)+ for storageKey, alert := range m.activeAlerts {+ if alert == nil || !isHostSMARTRiskAlertType(alert.Type) || !strings.HasPrefix(alert.ResourceID, prefix) {+ continue+ }+ if hostSMARTRiskRulesOff(alert, smartThresholds, crcMinimumDelta) {+ m.clearAlertNoLock(storageKey)+ continue+ }+ if _, exists := seen[alert.ResourceID]; !exists {+ absent[alert.ResourceID] = append(absent[alert.ResourceID], storageKey)+ }+ }+ for resourceID := range m.hostSMARTRiskAbsences {+ if !strings.HasPrefix(resourceID, prefix) {+ continue+ }+ if _, missing := absent[resourceID]; !missing || seen == nil {+ delete(m.hostSMARTRiskAbsences, resourceID)+ }+ }+ if seen == nil {+ return+ }+ for resourceID, storageKeys := range absent {+ m.hostSMARTRiskAbsences[resourceID]+++ if m.hostSMARTRiskAbsences[resourceID] < hostSMARTRiskAbsenceConfirmations {+ continue+ }+ delete(m.hostSMARTRiskAbsences, resourceID)+ for _, storageKey := range storageKeys {+ m.clearAlertNoLock(storageKey)+ }+ }+}diff --git a/internal/alerts/host_unraid_lifecycle_test.go b/internal/alerts/host_unraid_lifecycle_test.goindex b340cb7bf1..c6716ece4d 100644--- a/internal/alerts/host_unraid_lifecycle_test.go+++ b/internal/alerts/host_unraid_lifecycle_test.go@@ -3,6 +3,7 @@ package alertsimport ("fmt""strings"+ "sync/atomic""testing""time"@@ -1064,3 +1065,312 @@ func TestDiskTemperaturePendingContextCleanup(t *testing.T) {})}}++// Feed only synthetic telemetry through the real CheckHost, config and callback+// paths. No collector, smartctl process, native host or notification provider.+type smartLifecycleFixture struct {+ m *Manager+ host models.Host+ healthID, wearID string+ initial map[string]Alert+ fires atomic.Int32+ recoveries chan *ResolvedAlert+}++func newSMARTLifecycleFixture(t *testing.T) *smartLifecycleFixture {+ t.Helper()+ m := newEventLogManager(t)+ cfg := m.GetConfig()+ cfg.FlappingEnabled = false+ cfg.Schedule.Cooldown = 0+ cfg.Schedule.MaxAlertsHour = 0+ cfg.AgentDefaults.Disk = &HysteresisThreshold{Trigger: 80, Clear: 70}+ cfg.AgentDefaults.DiskTemperature = &HysteresisThreshold{Trigger: 80, Clear: 70}+ cfg.DiskTempByType = map[string]HysteresisThreshold{"sata": {Trigger: 80, Clear: 70}}+ m.UpdateConfig(cfg)+ m.mu.Lock()+ m.config.TimeThresholds = map[string]int{}+ m.config.MetricTimeThresholds = map[string]map[string]int{"all": {"disk": 0, "diskTemperature": 0}}+ m.mu.Unlock()+ used, pending := 96, int64(2)+ host := models.Host{ID: "smart-lifecycle", Hostname: "storage", Status: "online",+ Disks: []models.Disk{{Mountpoint: "/data", Device: "/dev/sda1", Total: 1000, Used: 950, Usage: 95}},+ Sensors: models.HostSensorSummary{SMART: []models.HostDiskSMART{+ {Device: "/dev/sda", Type: "sata", Health: "FAILED", Temperature: 95, Attributes: &models.SMARTAttributes{PercentageUsed: &used, PendingSectors: &pending}},+ {Device: "/dev/sdb", Type: "sata", Health: "PASSED", Temperature: 20},+ }},+ }+ resourceID, _ := hostSMARTDiskResourceID(host, host.Sensors.SMART[0])+ f := &smartLifecycleFixture{m: m, host: host,+ healthID: buildCanonicalStateID(resourceID, resourceID+"-disk-health"),+ wearID: buildCanonicalStateID(resourceID, resourceID+"-disk-wearout"),+ initial: make(map[string]Alert), recoveries: make(chan *ResolvedAlert, 32),+ }+ m.SetAlertCallback(func(a *Alert) {+ if a.Type == "disk-health" || a.Type == "disk-wearout" {+ f.fires.Add(1)+ }+ })+ m.SetResolvedAlertCallback(func(r *ResolvedAlert) {+ if r.Alert.Type == "disk-health" || r.Alert.Type == "disk-wearout" {+ f.recoveries <- r+ }+ })+ m.CheckHost(host)+ for _, id := range []string{f.healthID, f.wearID} {+ f.initial[id] = *testRequireActiveAlert(t, m, id)+ }+ if f.fires.Load() != 2 {+ t.Fatalf("initial risk callbacks = %d, want 2", f.fires.Load())+ }+ return f+}++func (f *smartLifecycleFixture) held(t *testing.T) {+ t.Helper()+ for _, id := range []string{f.healthID, f.wearID} {+ a := testRequireActiveAlert(t, f.m, id)+ if !a.StartTime.Equal(f.initial[id].StartTime) {+ t.Fatalf("%s was replaced by a new occurrence", id)+ }+ }+ if f.fires.Load() != 2 {+ t.Fatalf("missing report refired risk notifications: %d", f.fires.Load())+ }+ for _, e := range queryAlertEvents(t, f.m, eventlog.Filter{Types: []string{eventlog.TypeResolved}}) {+ if e.AlertID == f.healthID || e.AlertID == f.wearID {+ t.Fatalf("missing evidence published recovery: %+v", e)+ }+ }+}++func (f *smartLifecycleFixture) resolved(t *testing.T, ids ...string) {+ t.Helper()+ want := make(map[string]bool)+ for _, id := range ids {+ if testHasActiveAlert(t, f.m, id) {+ t.Fatalf("risk alert remains active: %s", id)+ }+ want[id] = true+ }+ for range ids {+ select {+ case r := <-f.recoveries:+ if !want[r.Alert.ID] || !r.Alert.StartTime.Equal(f.initial[r.Alert.ID].StartTime) {+ t.Fatalf("wrong recovery occurrence: %+v", r)+ }+ delete(want, r.Alert.ID)+ case <-time.After(3 * time.Second):+ t.Fatal("ordinary recovery callback did not complete")+ }+ }+ for _, id := range ids {+ events := queryAlertEvents(t, f.m, eventlog.Filter{AlertID: id, Types: []string{eventlog.TypeResolved}})+ if len(events) != 1 {+ t.Fatalf("%s has %d resolution events, want 1", id, len(events))+ }+ }+}++func TestHostSMARTRiskMissingReportsPreserveOccurrenceAndCallbacks(t *testing.T) {+ f := newSMARTLifecycleFixture(t)+ if err := f.m.AcknowledgeAlert(f.healthID, "operator"); err != nil {+ t.Fatal(err)+ }+ present := f.host.Sensors.SMART+ // Both risk families on one disk must count once per report, not twice.+ for _, list := range [][]models.HostDiskSMART{nil, {present[1]}, {present[1]}, nil, {present[1]}, present, nil, present} {+ f.host.Sensors.SMART = list+ f.m.CheckHost(f.host)+ f.held(t)+ if !testRequireActiveAlert(t, f.m, f.healthID).Acknowledged {+ t.Fatal("acknowledgement lost during observation gap")+ }+ }+ // Standby and unavailable health/wear fields are presence, not recovery.+ f.host.Sensors.SMART = []models.HostDiskSMART{{Device: "/dev/sda", Standby: true}, present[1]}+ f.m.CheckHost(f.host)+ f.held(t)+ f.host.Sensors.SMART[0].Standby = false+ f.m.CheckHost(f.host)+ f.held(t)+ used, pending := 10, int64(0)+ f.host.Sensors.SMART[0] = models.HostDiskSMART{Device: "/dev/sda", Health: "PASSED", Temperature: 20, Attributes: &models.SMARTAttributes{PercentageUsed: &used, PendingSectors: &pending}}+ f.m.CheckHost(f.host)+ f.resolved(t, f.healthID, f.wearID)+ if f.fires.Load() != 2 {+ t.Fatalf("healthy recovery refired: %d", f.fires.Load())+ }+ t.Log("empty/partial/standby/unknown reports preserved occurrence and acknowledgement; fresh complete evidence recovered once")+}++func TestHostSMARTRiskDepartureRequiresFreshConsecutiveReports(t *testing.T) {+ for _, gap := range []string{"empty", "expiry", "present"} {+ t.Run(gap, func(t *testing.T) {+ f := newSMARTLifecycleFixture(t)+ present := f.host.Sensors.SMART+ f.host.Sensors.SMART = []models.HostDiskSMART{present[1]}+ for i := 0; i < 2; i++ {+ f.m.CheckHost(f.host)+ f.held(t)+ }+ switch gap {+ case "empty":+ f.host.Sensors.SMART = nil+ f.m.CheckHost(f.host)+ case "expiry":+ f.m.HandleHostTelemetryExpired(f.host)+ case "present":+ f.host.Sensors.SMART = present+ f.m.CheckHost(f.host)+ }+ f.held(t)+ f.host.Sensors.SMART = []models.HostDiskSMART{present[1]}+ for i := 1; i <= 3; i++ {+ f.m.CheckHost(f.host)+ if i < 3 {+ f.held(t)+ }+ }+ f.resolved(t, f.healthID, f.wearID)+ if f.fires.Load() != 2 {+ t.Fatalf("departure refired %d callbacks", f.fires.Load())+ }+ })+ }+}++func TestHostSMARTRiskRulesDisableBetweenReports(t *testing.T) {+ f := newSMARTLifecycleFixture(t)+ cfg := f.m.GetConfig()+ off := 0+ cfg.AgentDefaults.SMARTHealthFailure = &off+ f.m.UpdateConfig(cfg)+ f.held(t) // Pending sectors remain enabled; disabling only one cause is not recovery.+ zero := int64(0)+ cfg.AgentDefaults.SMARTPending = &zero+ f.m.UpdateConfig(cfg)+ f.resolved(t, f.healthID)+ if !testHasActiveAlert(t, f.m, f.wearID) {+ t.Fatal("health policy disabled unrelated wearout")+ }+ // A missing report cannot resurrect it. Re-enable before another report;+ // only a newly observed failing disk may start another occurrence.+ f.host.Sensors.SMART = nil+ f.m.CheckHost(f.host)+ if testHasActiveAlert(t, f.m, f.healthID) {+ t.Fatal("missing report raised disabled health")+ }+ on, one := 1, int64(1)+ cfg.AgentDefaults.SMARTHealthFailure = &on+ cfg.AgentDefaults.SMARTPending = &one+ f.m.UpdateConfig(cfg)+ f.m.CheckHost(f.host)+ if testHasActiveAlert(t, f.m, f.healthID) {+ t.Fatal("re-enable raised health without evidence")+ }+ used, pending := 96, int64(2)+ f.host.Sensors.SMART = []models.HostDiskSMART{{Device: "/dev/sda", Health: "FAILED", Attributes: &models.SMARTAttributes{PercentageUsed: &used, PendingSectors: &pending}}}+ f.m.CheckHost(f.host)+ a := testRequireActiveAlert(t, f.m, f.healthID)+ if a.StartTime.Equal(f.initial[f.healthID].StartTime) || f.fires.Load() != 3 {+ t.Fatal("fresh bad evidence did not start exactly one new health occurrence")+ }+}++func TestHostSMARTRiskEmptyReportHonoursRuleOffAndLegacyUnknowns(t *testing.T) {+ for _, metadata := range []string{"current", "legacy", "unknown"} {+ t.Run(metadata, func(t *testing.T) {+ f := newSMARTLifecycleFixture(t)+ off, zero := 0, int64(0)+ f.m.mu.Lock()+ f.m.config.AgentDefaults.SMARTHealthFailure = &off+ f.m.config.AgentDefaults.SMARTPending = &zero+ if metadata != "current" {+ a, _ := f.m.getActiveAlertNoLock(f.healthID)+ if metadata == "legacy" {+ delete(a.Metadata, "riskCodes")+ } else {+ a.Metadata["riskCodes"] = []interface{}{"future-code"}+ }+ }+ f.m.mu.Unlock()+ f.host.Sensors.SMART = nil+ f.m.CheckHost(f.host)+ if metadata == "current" {+ f.resolved(t, f.healthID)+ } else {+ f.held(t)+ }+ if !testHasActiveAlert(t, f.m, f.wearID) {+ t.Fatal("wearout was disabled with health")+ }+ })+ }+}++func TestHostSMARTRiskFilesystemAndHostLifecycleStayIndependent(t *testing.T) {+ for _, action := range []string{"filesystem-absent", "node-link", "removed", "host-disabled", "agents-disabled", "host-isolation"} {+ t.Run(action, func(t *testing.T) {+ f := newSMARTLifecycleFixture(t)+ switch action {+ case "filesystem-absent":+ fsID, _ := hostDiskResourceID(f.host, f.host.Disks[0])+ id := canonicalMetricStateID(fsID, "disk")+ testRequireActiveAlert(t, f.m, id)+ f.host.Disks = nil+ f.host.Sensors.SMART = nil+ f.m.CheckHost(f.host)+ if testHasActiveAlert(t, f.m, id) {+ t.Fatal("filesystem absence incorrectly held usage")+ }+ f.held(t)+ case "host-isolation":+ other := f.host+ other.ID = "smart-lifecycle-other"+ f.m.CheckHost(other)+ other.Sensors.SMART = []models.HostDiskSMART{other.Sensors.SMART[1]}+ for i := 0; i < 3; i++ {+ f.m.CheckHost(other)+ }+ f.held(t)+ default:+ present := f.host.Sensors.SMART+ f.host.Sensors.SMART = []models.HostDiskSMART{present[1]}+ f.m.CheckHost(f.host)+ f.held(t)+ switch action {+ case "node-link":+ f.host.LinkedNodeID = "pve-node"+ f.host.Sensors.SMART = nil+ f.m.CheckHost(f.host)+ case "removed":+ f.m.HandleHostRemoved(f.host)+ case "host-disabled":+ cfg := f.m.GetConfig()+ cfg.Overrides["agent:"+f.host.ID] = ThresholdConfig{Disabled: true}+ f.m.UpdateConfig(cfg)+ case "agents-disabled":+ cfg := f.m.GetConfig()+ cfg.DisableAllAgents = true+ f.m.UpdateConfig(cfg)+ }+ f.resolved(t, f.healthID, f.wearID)+ // Returning ownership gets a new occurrence and a fresh departure run.+ f.host.LinkedNodeID = ""+ cfg := f.m.GetConfig()+ cfg.DisableAllAgents = false+ delete(cfg.Overrides, "agent:"+f.host.ID)+ f.m.UpdateConfig(cfg)+ f.host.Sensors.SMART = present+ f.m.CheckHost(f.host)+ f.host.Sensors.SMART = []models.HostDiskSMART{present[1]}+ for i := 0; i < 2; i++ {+ f.m.CheckHost(f.host)+ testRequireActiveAlert(t, f.m, f.healthID)+ testRequireActiveAlert(t, f.m, f.wearID)+ }+ }+ })+ }+}diff --git a/internal/alerts/manager.go b/internal/alerts/manager.goindex a4781899c4..a958d1c906 100644--- a/internal/alerts/manager.go+++ b/internal/alerts/manager.go@@ -79,6 +79,7 @@ type Manager struct {unifiedIncidentFirstSeen map[string]time.Time // Preserve the first confirmed observation as lifecycle startunifiedIncidentRecoveries map[string]int // Track consecutive healthy observations before provider-incident recoveryhostDiskTempPendingContexts map[string]hostDiskTempContext // Only live pending temperature runs; config saves need link/type identity+ hostSMARTRiskAbsences map[string]int // Consecutive non-empty reports omitting a SMART risk resourcehostDiskTempAbsences map[string]int // Track consecutive SMART reports missing a disk that holds temperature alert statedockerRestartTracking map[string]*dockerRestartRecord // Track restart counts and times for restart loop detectiondockerUpdateFirstSeen map[string]time.Time // Track when image updates were first detected for alert delay