Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
package alerts
import (
"fmt"
"path/filepath"
"strings"
"sync/atomic"
"testing"
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/alerts/eventlog"
"github.com/rcourtman/pulse-go-rewrite/internal/alerts/reducer"
"github.com/rcourtman/pulse-go-rewrite/internal/models"
"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
"github.com/rcourtman/pulse-go-rewrite/pkg/diskinventory"
)
func TestHostCustomSensorAlertLifecycle(t *testing.T) {
manager := newTestManager(t)
manager.ClearActiveAlerts()
value := 23.0
eventAt := time.Now().UTC().Add(-2 * time.Hour)
host := models.Host{
ID: "custom-sensor-host",
Hostname: "edge-1",
DisplayName: "Edge 1",
Sensors: models.HostSensorSummary{
Custom: []models.HostCustomSensorMetric{{
ID: "queue_depth",
Name: "Queue depth",
Group: "Main server",
Subgroup: "Backup",
Kind: "timestamp",
Unit: "items",
Value: &value,
Status: "critical",
ObservedAt: time.Now().UTC(),
EventAt: &eventAt,
AlertOnError: true,
}},
},
}
manager.CheckHost(host)
active := manager.GetActiveAlerts()
if !hasAlertType(active, "custom-sensor") {
t.Fatal("expected custom sensor alert")
}
for _, alert := range active {
if alert.Type != "custom-sensor" {
continue
}
if alert.Metadata["customSensorGroup"] != "Main server" ||
alert.Metadata["customSensorSubgroup"] != "Backup" ||
alert.Metadata["customSensorKind"] != "timestamp" ||
alert.Metadata["customSensorEventAt"] != eventAt.Format(time.RFC3339) {
t.Fatalf("custom sensor alert metadata = %#v", alert.Metadata)
}
}
host.Sensors.Custom[0].Status = "ok"
manager.CheckHost(host)
if hasAlertType(manager.GetActiveAlerts(), "custom-sensor") {
t.Fatal("healthy custom sensor did not resolve its alert")
}
host.Sensors.Custom[0].Status = "error"
host.Sensors.Custom[0].Error = "probe timed out"
manager.CheckHost(host)
if !hasAlertType(manager.GetActiveAlerts(), "custom-sensor") {
t.Fatal("alertOnError custom sensor did not alert")
}
host.Sensors.Custom = nil
manager.CheckHost(host)
if hasAlertType(manager.GetActiveAlerts(), "custom-sensor") {
t.Fatal("removed custom sensor did not clear its alert")
}
}
func TestHostCustomSensorErrorCanBeReportOnly(t *testing.T) {
manager := newTestManager(t)
manager.ClearActiveAlerts()
host := models.Host{
ID: "custom-sensor-report-only",
Hostname: "edge-2",
Sensors: models.HostSensorSummary{
Custom: []models.HostCustomSensorMetric{{
ID: "optional_probe",
Name: "Optional probe",
Status: "error",
ObservedAt: time.Now().UTC(),
Error: "not installed",
}},
},
}
manager.CheckHost(host)
if hasAlertType(manager.GetActiveAlerts(), "custom-sensor") {
t.Fatal("alertOnError=false custom sensor unexpectedly alerted")
}
}
func TestHandleHostOfflineExpiresUnraidOperationAlertBeforeConnectivityConfirmation(t *testing.T) {
manager := newTestManager(t)
manager.ClearActiveAlerts()
host := models.Host{
ID: "unraid-host",
Hostname: "tower",
DisplayName: "Tower",
Platform: "unraid",
Unraid: &models.HostUnraidStorage{
ArrayStarted: true,
ArrayState: "STARTED",
SyncAction: "check",
SyncProgress: 40,
Disks: []models.HostUnraidDisk{
{Name: "parity", Role: "parity", Status: "online", Device: "/dev/sda"},
{Name: "disk1", Role: "data", Status: "online", Device: "/dev/sdb"},
},
},
}
manager.CheckHost(host)
if !hasAlertType(manager.GetActiveAlerts(), "storage-topology") {
t.Fatal("expected active Unraid operation alert")
}
manager.HandleHostOffline(host)
alerts := manager.GetActiveAlerts()
if hasAlertType(alerts, "storage-topology") {
t.Fatal("expired Unraid operation alert remained active")
}
if hasAlertType(alerts, "host-offline") {
t.Fatal("connectivity alert should still require its confirmation window")
}
}
func TestHandleHostTelemetryExpiredPreservesStaticStorageRisk(t *testing.T) {
t.Run("Unraid no-parity risk survives transient check expiry", func(t *testing.T) {
manager := newTestManager(t)
manager.ClearActiveAlerts()
host := models.Host{
ID: "unraid-no-parity",
Hostname: "tower",
DisplayName: "Tower",
Platform: "unraid",
Unraid: &models.HostUnraidStorage{
ArrayStarted: true,
ArrayState: "STARTED",
SyncAction: "check",
SyncProgress: 40,
Disks: []models.HostUnraidDisk{
{Name: "disk1", Role: "data", Status: "online", Device: "/dev/sdb"},
},
},
}
manager.CheckHost(host)
manager.HandleHostTelemetryExpired(host)
var storageAlert *Alert
activeAlerts := manager.GetActiveAlerts()
for i := range activeAlerts {
alert := activeAlerts[i]
if alert.Type == "storage-topology" {
storageAlert = &alert
break
}
}
if storageAlert == nil {
t.Fatal("static no-parity risk was cleared with transient operation state")
}
if strings.Contains(strings.ToLower(storageAlert.Message), "check") {
t.Fatalf("expired operation remained in static alert: %q", storageAlert.Message)
}
})
t.Run("degraded RAID risk survives transient rebuild expiry", func(t *testing.T) {
manager := newTestManager(t)
manager.ClearActiveAlerts()
host := models.Host{
ID: "raid-degraded",
Hostname: "storage-host",
DisplayName: "Storage Host",
Platform: "linux",
RAID: []models.HostRAIDArray{{
Device: "/dev/md0",
Level: "raid1",
State: "degraded",
TotalDevices: 2,
ActiveDevices: 1,
WorkingDevices: 1,
FailedDevices: 1,
Operation: "recovery",
RebuildPercent: 40,
RebuildSpeed: "100M/sec",
}},
}
manager.CheckHost(host)
manager.HandleHostTelemetryExpired(host)
if !hasAlertType(manager.GetActiveAlerts(), "raid") {
t.Fatal("static degraded RAID risk was cleared with transient rebuild state")
}
})
}
func TestCheckHostClearsDiskTemperatureAlertsWhenThresholdTurnsOff(t *testing.T) {
cases := []struct {
name string
turnOff func(m *Manager, hostID string)
}{
{
name: "agent default disabled",
turnOff: func(m *Manager, hostID string) {
m.config.AgentDefaults.DiskTemperature = &HysteresisThreshold{Trigger: 0, Clear: 0}
},
},
{
name: "host override disables disk temperature",
turnOff: func(m *Manager, hostID string) {
m.config.Overrides[hostID] = ThresholdConfig{DiskTemperature: &HysteresisThreshold{Trigger: 0, Clear: 0}}
},
},
{
name: "host override disables the agent",
turnOff: func(m *Manager, hostID string) {
m.config.Overrides[hostID] = ThresholdConfig{Disabled: true}
},
},
{
name: "all agent alerts disabled",
turnOff: func(m *Manager, hostID string) {
m.config.DisableAllAgents = true
},
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
m := configureDiskTempTypeHostManager(t)
host := hostWithSMARTDiskTemp("host-temp-off", "sata", 60)
alertID := hostDiskTempAlertID(host)
m.CheckHost(host)
alert, exists := testLookupActiveAlert(t, m, alertID)
if !exists {
t.Fatalf("expected disk temperature alert at 60C (sata trigger 55), active: %v", alertKeys(m))
}
if alert.Type != "diskTemperature" || !strings.HasPrefix(alert.ResourceID, hostDiskTemperatureResourcePrefix(host.ID)) {
t.Fatalf("disk temperature alert type/resource = %q/%q", alert.Type, alert.ResourceID)
}
// Change config directly so only CheckHost can resolve the alert.
m.mu.Lock()
tc.turnOff(m, host.ID)
m.mu.Unlock()
m.CheckHost(host)
if _, exists := testLookupActiveAlert(t, m, alertID); exists {
t.Fatalf("disk temperature alert stayed active after its threshold was turned off, active: %v", alertKeys(m))
}
})
}
}
func TestHostDiskTemperatureAlertsClearWhenAgentLeaves(t *testing.T) {
t.Run("agent removed", func(t *testing.T) {
m := configureDiskTempTypeHostManager(t)
host := hostWithSMARTDiskTemp("host-temp-removed", "sata", 60)
m.CheckHost(host)
if _, exists := testLookupActiveAlert(t, m, hostDiskTempAlertID(host)); !exists {
t.Fatalf("expected disk temperature alert, active: %v", alertKeys(m))
}
m.HandleHostRemoved(host)
if _, exists := testLookupActiveAlert(t, m, hostDiskTempAlertID(host)); exists {
t.Fatalf("disk temperature alert outlived its removed agent, active: %v", alertKeys(m))
}
})
t.Run("agent confirmed offline", func(t *testing.T) {
m := configureDiskTempTypeHostManager(t)
host := hostWithSMARTDiskTemp("host-temp-offline", "sata", 60)
m.CheckHost(host)
if _, exists := testLookupActiveAlert(t, m, hostDiskTempAlertID(host)); !exists {
t.Fatalf("expected disk temperature alert, active: %v", alertKeys(m))
}
for i := 0; i < 3; i++ {
m.HandleHostOffline(host)
}
if _, exists := testLookupActiveAlert(t, m, canonicalConnectivityStateID(hostResourceID(host.ID))); !exists {
t.Fatalf("expected confirmed offline alert, active: %v", alertKeys(m))
}
if _, exists := testLookupActiveAlert(t, m, hostDiskTempAlertID(host)); exists {
t.Fatalf("disk temperature alert stayed beside the offline alert, active: %v", alertKeys(m))
}
})
}
func TestCheckHostClearsDiskTemperatureAlertWhenDiskLeavesSMARTReport(t *testing.T) {
const hostID = "host-temp-absent"
smartDisk := func(device string, temperature int) models.HostDiskSMART {
return models.HostDiskSMART{Device: device, Model: "test-disk", Type: "sata", Temperature: temperature}
}
alertIDFor := func(device string) string {
return canonicalMetricStateID(hostDiskTemperatureResourceID(hostID, device), "diskTemperature")
}
sdaActive := func(m *Manager) bool {
_, exists := testLookupActiveAlert(t, m, alertIDFor("/dev/sda"))
return exists
}
// sda runs hot (sata trigger 55, clear 50); sdb stays cool.
hotHost := func(t *testing.T) (*Manager, models.Host) {
m := configureDiskTempTypeHostManager(t)
host := hostWithSMARTDiskTemp(hostID, "sata", 60)
host.Sensors.SMART = []models.HostDiskSMART{smartDisk("/dev/sda", 60), smartDisk("/dev/sdb", 40)}
m.CheckHost(host)
if !sdaActive(m) {
t.Fatalf("expected sda disk temperature alert at 60C, active: %v", alertKeys(m))
}
return m, host
}
cases := []struct {
name string
next []models.HostDiskSMART
wantHeld bool
wantRaised string
}{
{name: "disk removed", next: []models.HostDiskSMART{smartDisk("/dev/sdb", 40)}},
{name: "device renamed", next: []models.HostDiskSMART{smartDisk("/dev/sdc", 60), smartDisk("/dev/sdb", 40)}, wantRaised: "/dev/sdc"},
// No fresh reading says the disk cooled or left, so the alert holds.
{name: "SMART collection empty", next: nil, wantHeld: true},
{name: "disk in standby", next: []models.HostDiskSMART{{Device: "/dev/sda", Type: "sata", Temperature: 45, Standby: true}, smartDisk("/dev/sdb", 40)}, wantHeld: true},
{name: "probe returned no temperature", next: []models.HostDiskSMART{{Device: "/dev/sda", Type: "sata"}, smartDisk("/dev/sdb", 40)}, wantHeld: true},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
m, host := hotHost(t)
host.Sensors.SMART = tc.next
for report := 1; report <= hostDiskTemperatureAbsenceConfirmations; report++ {
m.CheckHost(host)
wantActive := tc.wantHeld || report < hostDiskTemperatureAbsenceConfirmations
if active := sdaActive(m); active != wantActive {
t.Fatalf("report %d: sda disk temperature alert active = %v, want %v; active: %v", report, active, wantActive, alertKeys(m))
}
}
if tc.wantRaised != "" {
if _, exists := testLookupActiveAlert(t, m, alertIDFor(tc.wantRaised)); !exists {
t.Fatalf("expected %s disk temperature alert, active: %v", tc.wantRaised, alertKeys(m))
}
}
if !tc.wantHeld {
return
}
// A held alert still resolves on the disk's next cool reading.
host.Sensors.SMART = []models.HostDiskSMART{smartDisk("/dev/sda", 45), smartDisk("/dev/sdb", 40)}
m.CheckHost(host)
if sdaActive(m) {
t.Fatalf("held sda disk temperature alert outlived a 45C reading, active: %v", alertKeys(m))
}
})
}
t.Run("intermittently dropped disk keeps its alert", func(t *testing.T) {
m, host := hotHost(t)
present := host.Sensors.SMART
dropped := []models.HostDiskSMART{smartDisk("/dev/sdb", 40)}
// The run of omissions restarts whenever the disk is reported again.
for _, smart := range [][]models.HostDiskSMART{dropped, dropped, present, dropped, dropped} {
host.Sensors.SMART = smart
m.CheckHost(host)
if !sdaActive(m) {
t.Fatalf("sda disk temperature alert cleared by an omission run shorter than %d reports, active: %v", hostDiskTemperatureAbsenceConfirmations, alertKeys(m))
}
}
})
// pendingHost leaves a hot sda inside a 300s alert delay, so sda holds a
// pending run and no alert.
pendingHost := func(t *testing.T) (*Manager, models.Host, func() bool) {
m := configureDiskTempTypeHostManager(t)
m.mu.Lock()
m.config.TimeThresholds = map[string]int{"agent": 300}
m.mu.Unlock()
sdaResourceID := hostDiskTemperatureResourceID(hostID, "/dev/sda")
sdaPending := func() bool {
m.mu.RLock()
defer m.mu.RUnlock()
for _, resourceID := range m.core.PendingResourceIDs() {
if resourceID == sdaResourceID {
return true
}
}
return false
}
host := hostWithSMARTDiskTemp(hostID, "sata", 60)
host.Sensors.SMART = []models.HostDiskSMART{smartDisk("/dev/sda", 60), smartDisk("/dev/sdb", 40)}
m.CheckHost(host)
if sdaActive(m) || !sdaPending() {
t.Fatalf("expected a pending sda run inside the 300s delay, active: %v", alertKeys(m))
}
return m, host, sdaPending
}
t.Run("departed disk drops its pending run", func(t *testing.T) {
m, host, sdaPending := pendingHost(t)
host.Sensors.SMART = []models.HostDiskSMART{smartDisk("/dev/sdb", 40)}
for report := 1; report <= hostDiskTemperatureAbsenceConfirmations; report++ {
m.CheckHost(host)
wantPending := false // A single omission interrupts pending evidence, not firing alerts.
if pending := sdaPending(); pending != wantPending {
t.Fatalf("report %d: pending sda run = %v, want %v", report, pending, wantPending)
}
}
})
t.Run("threshold off drops pending runs", func(t *testing.T) {
m, host, sdaPending := pendingHost(t)
m.mu.Lock()
m.config.AgentDefaults.DiskTemperature = &HysteresisThreshold{Trigger: 0, Clear: 0}
m.mu.Unlock()
m.CheckHost(host)
if sdaPending() {
t.Fatal("pending sda run survived its threshold being turned off; re-enabling would inherit its old start time")
}
})
t.Run("removed host leaves no pending run or absence count", func(t *testing.T) {
m, host, sdaPending := pendingHost(t)
host.Sensors.SMART = []models.HostDiskSMART{smartDisk("/dev/sdb", 40)}
m.CheckHost(host)
m.HandleHostRemoved(host)
if sdaPending() {
t.Fatal("pending sda run outlived its removed host")
}
m.mu.RLock()
defer m.mu.RUnlock()
if len(m.hostDiskTempAbsences) != 0 {
t.Fatalf("absence counts outlived their removed host: %v", m.hostDiskTempAbsences)
}
})
}
func TestConfigSaveResolvesDiskTemperatureAlertAgainstItsDiskTypeThreshold(t *testing.T) {
m := configureDiskTempTypeHostManager(t)
m.mu.Lock()
m.config.AgentDefaults.DiskTemperature = &HysteresisThreshold{Trigger: 75, Clear: 70}
m.mu.Unlock()
// 72C is above the nvme trigger (70) but below the agent default (75).
host := hostWithSMARTDiskTemp("host-temp-save", "nvme", 72)
alertID := hostDiskTempAlertID(host)
m.CheckHost(host)
if _, exists := testLookupActiveAlert(t, m, alertID); !exists {
t.Fatalf("expected nvme disk temperature alert at 72C, active: %v", alertKeys(m))
}
// An unrelated save keeps the alert CheckHost would raise again.
cfg := m.GetConfig()
cfg.AgentDefaults.CPU = &HysteresisThreshold{Trigger: 85, Clear: 80}
m.UpdateConfig(cfg)
if _, exists := testLookupActiveAlert(t, m, alertID); !exists {
t.Fatalf("unrelated config save resolved a disk temperature alert above its nvme trigger, active: %v", alertKeys(m))
}
// Turning disk temperature off resolves it on save, before any report.
cfg = m.GetConfig()
cfg.AgentDefaults.DiskTemperature = &HysteresisThreshold{Trigger: 0, Clear: 0}
m.UpdateConfig(cfg)
if _, exists := testLookupActiveAlert(t, m, alertID); exists {
t.Fatalf("disk temperature alert stayed active after the threshold was turned off, active: %v", alertKeys(m))
}
}
func TestConfigSaveKeepsDiskTemperatureAlertWithoutDiskType(t *testing.T) {
m := configureDiskTempTypeHostManager(t)
m.mu.Lock()
m.config.AgentDefaults.DiskTemperature = &HysteresisThreshold{Trigger: 75, Clear: 70}
m.mu.Unlock()
host := hostWithSMARTDiskTemp("host-temp-legacy", "nvme", 72)
alertID := hostDiskTempAlertID(host)
m.CheckHost(host)
alert, exists := testLookupActiveAlert(t, m, alertID)
if !exists {
t.Fatalf("expected nvme disk temperature alert at 72C, active: %v", alertKeys(m))
}
// Restored from a release that did not record the disk type.
m.mu.Lock()
delete(alert.Metadata, "diskType")
m.mu.Unlock()
// 72C is above the nvme trigger (70) but below the agent default (75).
cfg := m.GetConfig()
cfg.AgentDefaults.CPU = &HysteresisThreshold{Trigger: 85, Clear: 80}
m.UpdateConfig(cfg)
if _, exists := testLookupActiveAlert(t, m, alertID); !exists {
t.Fatalf("config save resolved a disk temperature alert of unknown disk type that its nvme trigger still fires, active: %v", alertKeys(m))
}
cfg = m.GetConfig()
cfg.AgentDefaults.DiskTemperature = &HysteresisThreshold{Trigger: 0, Clear: 0}
m.UpdateConfig(cfg)
if _, exists := testLookupActiveAlert(t, m, alertID); exists {
t.Fatalf("disk temperature alert of unknown disk type stayed active after the threshold was turned off, active: %v", alertKeys(m))
}
}
func TestConfigSaveKeepsAlertTheEvaluatorKeepsWithoutRecoveryBand(t *testing.T) {
// Normalization keeps a 70/75 entry, and the evaluator treats a clear
// level at or above the trigger as no recovery band, firing from 70C.
for _, reading := range []int{72, 70} {
t.Run(fmt.Sprintf("%dC", reading), func(t *testing.T) {
m := configureDiskTempTypeHostManager(t)
m.mu.Lock()
m.config.DiskTempByType["nvme"] = HysteresisThreshold{Trigger: 70, Clear: 75}
m.mu.Unlock()
host := hostWithSMARTDiskTemp("host-temp-no-band", "nvme", reading)
alertID := hostDiskTempAlertID(host)
m.CheckHost(host)
if _, exists := testLookupActiveAlert(t, m, alertID); !exists {
t.Fatalf("expected nvme disk temperature alert at %dC, active: %v", reading, alertKeys(m))
}
cfg := m.GetConfig()
cfg.AgentDefaults.CPU = &HysteresisThreshold{Trigger: 85, Clear: 80}
m.UpdateConfig(cfg)
if _, exists := testLookupActiveAlert(t, m, alertID); !exists {
t.Fatalf("config save resolved an alert the evaluator keeps firing at %dC, active: %v", reading, alertKeys(m))
}
m.CheckHost(host)
if _, exists := testLookupActiveAlert(t, m, alertID); !exists {
t.Fatalf("expected the next report to keep the alert, active: %v", alertKeys(m))
}
})
}
}
const unraidTempHostID = "host-unraid-temp"
// unraidTempHost is an Unraid agent report with the given SMART rows and
// Unraid inventory rows.
func unraidTempHost(smart []models.HostDiskSMART, unraid ...models.HostUnraidDisk) models.Host {
host := hostWithSMARTDiskTemp(unraidTempHostID, "sata", 40)
host.Sensors.SMART = smart
host.Unraid = &models.HostUnraidStorage{ArrayStarted: true, ArrayState: "STARTED", Disks: unraid}
return host
}
func unraidTempAlertID(device string) string {
return canonicalMetricStateID(hostDiskTemperatureResourceID(unraidTempHostID, device), "diskTemperature")
}
func TestCheckHostJudgesTheUnraidTemperatureADiskShows(t *testing.T) {
cool := models.HostDiskSMART{Device: "sda", Model: "cool-disk", Serial: "COOL-1", Type: "sata", Temperature: 40}
coolUnraid := models.HostUnraidDisk{Name: "disk1", Device: "/dev/sda", Serial: "COOL-1", Transport: "sata", Temperature: 40}
hot := models.HostUnraidDisk{Name: "disk2", Device: "/dev/sdc", Model: "hot-disk", Serial: "HOT-1", Transport: "sata", Temperature: 60}
cases := []struct {
name string
host func() models.Host
configure func(m *Manager)
device string
wantAlert bool
// quiet lists devices that must raise nothing.
quiet []string
}{
{
// smartctl returned no temperature, and the disk shows Unraid's.
name: "SMART row without a reading",
host: func() models.Host {
return unraidTempHost([]models.HostDiskSMART{
{Device: "sdc", Model: "hot-disk", Serial: "HOT-1", Type: "sata"},
}, hot)
},
device: "sdc",
wantAlert: true,
},
{
name: "disk only the Unraid inventory lists",
host: func() models.Host {
return unraidTempHost([]models.HostDiskSMART{cool}, coolUnraid, hot)
},
device: "sdc",
wantAlert: true,
},
{
name: "Unraid disk while the SMART list is empty",
host: func() models.Host {
return unraidTempHost(nil, coolUnraid, hot)
},
device: "sdc",
wantAlert: true,
},
{
// smartctl and Unraid name the device with different serials, so
// two disks show on it; its alert judges the hotter reading.
name: "two disks shown on one device",
host: func() models.Host {
return unraidTempHost([]models.HostDiskSMART{
cool,
{Device: "sdc", Model: "bridge", Serial: "BRIDGE-9", Type: "sata", Temperature: 41},
}, coolUnraid, hot)
},
device: "sdc",
wantAlert: true,
},
{
// 64C is under the sas trigger (65); 60C is over the sata one (55).
name: "two disks of different types on one device",
host: func() models.Host {
return unraidTempHost([]models.HostDiskSMART{
cool,
{Device: "sdc", Model: "bridge", Serial: "BRIDGE-9", Type: "sas", Temperature: 64},
}, coolUnraid, hot)
},
device: "sdc",
wantAlert: true,
},
{
// A controller member and its Unraid device are one registry
// disk, alerted once under the member's own resource.
name: "controller member under another device label",
host: func() models.Host {
member := models.HostDiskSMART{Device: "0 [megaraid,0]", Model: "hot-disk", Serial: "HOT-1", Type: "sata", Controller: "0", Target: "megaraid,0", Temperature: 60}
return unraidTempHost([]models.HostDiskSMART{cool, member}, coolUnraid, hot)
},
device: "0 [megaraid,0]",
wantAlert: true,
quiet: []string{"sdc"},
},
{
name: "spun-down Unraid disk",
host: func() models.Host {
spunDown := hot
spunDown.SpunDown = true
return unraidTempHost([]models.HostDiskSMART{cool}, coolUnraid, spunDown)
},
device: "sdc",
},
{
name: "Unraid disk the agent excludes",
host: func() models.Host {
host := unraidTempHost([]models.HostDiskSMART{cool}, coolUnraid, hot)
host.DiskExclude = []string{"/dev/sdc"}
return host
},
device: "sdc",
},
{
// 65C is under the nvme trigger (70) and over the agent default (55).
name: "Unraid disk judged by its disk type",
host: func() models.Host {
nvme := hot
nvme.Transport = "nvme"
nvme.Temperature = 65
return unraidTempHost([]models.HostDiskSMART{cool}, coolUnraid, nvme)
},
device: "sdc",
},
{
name: "Unraid disk under a host override",
host: func() models.Host {
return unraidTempHost([]models.HostDiskSMART{cool}, coolUnraid, hot)
},
configure: func(m *Manager) {
m.config.Overrides[unraidTempHostID] = ThresholdConfig{DiskTemperature: &HysteresisThreshold{Trigger: 65, Clear: 60}}
},
device: "sdc",
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
m := configureDiskTempTypeHostManager(t)
if tc.configure != nil {
m.mu.Lock()
tc.configure(m)
m.mu.Unlock()
}
m.CheckHost(tc.host())
alert, exists := testLookupActiveAlert(t, m, unraidTempAlertID(tc.device))
if exists != tc.wantAlert {
t.Fatalf("disk temperature alert for %s active = %v, want %v; active: %v", tc.device, exists, tc.wantAlert, alertKeys(m))
}
for _, device := range append([]string{"sda"}, tc.quiet...) {
if _, exists := testLookupActiveAlert(t, m, unraidTempAlertID(device)); exists {
t.Fatalf("%s raised a temperature alert, active: %v", device, alertKeys(m))
}
}
if !tc.wantAlert {
return
}
if alert.Value != 60 || alert.Metadata["temperature"] != 60 || alert.Metadata["diskType"] != "sata" || alert.Metadata["model"] != "hot-disk" {
t.Fatalf("alert value/metadata = %v/%v, want the 60C sata reading of hot-disk", alert.Value, alert.Metadata)
}
})
}
}
func TestCheckHostHoldsUnraidDiskTemperatureAlertUntilAReadingOrDeparture(t *testing.T) {
cool := models.HostDiskSMART{Device: "sda", Model: "cool-disk", Serial: "COOL-1", Type: "sata", Temperature: 40}
hot := models.HostUnraidDisk{Name: "disk2", Device: "/dev/sdc", Model: "hot-disk", Serial: "HOT-1", Transport: "sata", Temperature: 60}
alertID := unraidTempAlertID("sdc")
// CheckHost clears a disk's alert once three non-empty reports in a row
// omit it.
const absentReports = 3
active := func(m *Manager) bool {
_, exists := testLookupActiveAlert(t, m, alertID)
return exists
}
hotHost := func(t *testing.T) *Manager {
m := configureDiskTempTypeHostManager(t)
m.CheckHost(unraidTempHost([]models.HostDiskSMART{cool}, hot))
if !active(m) {
t.Fatalf("expected the Unraid reading to raise a disk temperature alert, active: %v", alertKeys(m))
}
return m
}
t.Run("listed without a reading", func(t *testing.T) {
m := hotHost(t)
spunDown := hot
spunDown.SpunDown = true
for i := 0; i < 4; i++ {
m.CheckHost(unraidTempHost([]models.HostDiskSMART{cool}, spunDown))
}
if !active(m) {
t.Fatalf("a spun-down disk still in the inventory lost its alert, active: %v", alertKeys(m))
}
})
t.Run("cooled reading resolves it", func(t *testing.T) {
m := hotHost(t)
cooled := hot
cooled.Temperature = 45
m.CheckHost(unraidTempHost([]models.HostDiskSMART{cool}, cooled))
if active(m) {
t.Fatalf("a 45C Unraid reading (clear 50) left the alert active: %v", alertKeys(m))
}
})
t.Run("smartctl reading takes over the same alert", func(t *testing.T) {
m := hotHost(t)
withSMART := []models.HostDiskSMART{cool, {Device: "sdc", Model: "hot-disk", Serial: "HOT-1", Type: "sata", Temperature: 62}}
m.CheckHost(unraidTempHost(withSMART, hot))
if !active(m) {
t.Fatalf("the disk's SMART row did not carry its Unraid-raised alert, active: %v", alertKeys(m))
}
withSMART[1].Temperature = 45
m.CheckHost(unraidTempHost(withSMART, hot))
if active(m) {
t.Fatalf("a 45C SMART reading left the alert active: %v", alertKeys(m))
}
})
t.Run("left the inventory", func(t *testing.T) {
m := hotHost(t)
for i := 1; i < absentReports; i++ {
m.CheckHost(unraidTempHost([]models.HostDiskSMART{cool}))
if !active(m) {
t.Fatalf("alert cleared after %d report(s) without the disk, want %d", i, absentReports)
}
}
m.CheckHost(unraidTempHost([]models.HostDiskSMART{cool}))
if active(m) {
t.Fatalf("alert outlived a disk gone from %d reports, active: %v", absentReports, alertKeys(m))
}
})
t.Run("excluded row on a device a SMART row lists", func(t *testing.T) {
m := hotHost(t)
excluded := unraidTempHost([]models.HostDiskSMART{
cool,
{Device: "sdc", Model: "bridge", Serial: "BRIDGE-9", Type: "sata", Temperature: 62},
}, hot)
excluded.DiskExclude = []string{"sdc"}
m.CheckHost(excluded)
if !active(m) {
t.Fatalf("the exclusion cleared an alert the device's SMART row still raises, active: %v", alertKeys(m))
}
})
// The disk is still listed but out of monitoring, so its alert resolves
// on the first report, even one with an empty SMART list.
for name, smart := range map[string][]models.HostDiskSMART{
"excluded after it fired": {cool},
"excluded after it fired, SMART list empty": nil,
} {
t.Run(name, func(t *testing.T) {
m := hotHost(t)
excluded := unraidTempHost(smart, hot)
excluded.DiskExclude = []string{"sdc"}
m.CheckHost(excluded)
if active(m) {
t.Fatalf("alert outlived the disk's exclusion from monitoring, active: %v", alertKeys(m))
}
})
}
}
func TestJudgedHostDiskTemperatureOutranks(t *testing.T) {
sata := &HysteresisThreshold{Trigger: 55, Clear: 50}
sas := &HysteresisThreshold{Trigger: 65, Clear: 60}
noBand := &HysteresisThreshold{Trigger: 55, Clear: 55}
off := &HysteresisThreshold{Trigger: 0}
judged := func(temperature int, threshold *HysteresisThreshold) judgedHostDiskTemperature {
return judgedHostDiskTemperature{
reading: unifiedresources.HostDiskTemperatureReading{Temperature: temperature},
threshold: threshold,
}
}
cases := []struct {
name string
first, second judgedHostDiskTemperature
secondWins bool
}{
{name: "over its trigger beats a hotter reading under its own", first: judged(64, sas), second: judged(56, sata), secondWins: true},
{name: "in its recovery band beats one below its clear", first: judged(49, sata), second: judged(62, sas), secondWins: true},
{name: "no band means under the trigger is below it", first: judged(54, noBand), second: judged(51, sata), secondWins: true},
{name: "further over its trigger wins", first: judged(56, sata), second: judged(70, sas), secondWins: true},
{name: "a threshold that is off never wins", first: judged(40, sata), second: judged(90, off)},
{name: "a tie keeps the first", first: judged(60, sata), second: judged(60, sata)},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
if got := tc.second.outranks(tc.first); got != tc.secondWins {
t.Fatalf("second outranks first = %v, want %v", got, tc.secondWins)
}
})
}
}
func hasAlertType(alerts []Alert, alertType string) bool {
for _, alert := range alerts {
if alert.Type == alertType {
return true
}
}
return false
}
func TestHostCustomSensorEscalationDelivery(t *testing.T) {
for _, policy := range []string{"ready", "acknowledged", "snoozed", "rate-limited", "flapping", "inactive"} {
t.Run(policy, func(t *testing.T) {
m := newTestManager(t)
cfg := m.GetConfig()
cfg.Enabled = true
cfg.ActivationState = ActivationActive
cfg.FlappingEnabled = false
cfg.Schedule.MaxAlertsHour = 0
m.UpdateConfig(cfg)
var delivered []AlertLevel
m.SetAlertCallback(func(a *Alert) {
if a.Type == "custom-sensor" {
delivered = append(delivered, a.Level)
}
})
host := models.Host{ID: "sensor-host", Hostname: "sensor-host", Sensors: models.HostSensorSummary{
Custom: []models.HostCustomSensorMetric{{ID: "probe", Name: "Probe", Status: "warning", ObservedAt: time.Now()}},
}}
m.CheckHost(host)
if len(delivered) != 1 || delivered[0] != AlertLevelWarning {
t.Fatalf("initial delivery = %v", delivered)
}
var id string
for _, a := range m.GetActiveAlerts() {
if a.Type == "custom-sensor" {
id = a.ID
}
}
if id == "" {
t.Fatal("missing custom sensor incident")
}
switch policy {
case "acknowledged":
if err := m.AcknowledgeAlert(id, "tester"); err != nil {
t.Fatal(err)
}
case "snoozed":
if err := m.SnoozeAlert(id, "tester", time.Now().Add(time.Hour)); err != nil {
t.Fatal(err)
}
case "rate-limited":
a := testRequireActiveAlert(t, m, id)
m.mu.Lock()
m.config.Schedule.MaxAlertsHour = 1
m.alertRateLimit[canonicalTrackingKeyForAlert(a)] = []time.Time{time.Now()}
m.mu.Unlock()
case "flapping":
a := testRequireActiveAlert(t, m, id)
m.mu.Lock()
m.config.FlappingEnabled = true
m.suppressedUntil[canonicalTrackingKeyForAlert(a)] = time.Now().Add(time.Hour)
m.mu.Unlock()
case "inactive":
cfg.ActivationState = ActivationPending
m.UpdateConfig(cfg)
}
// Repeated warning observations must not resend.
m.CheckHost(host)
host.Sensors.Custom[0].Status = "critical"
m.CheckHost(host)
want := 1
if policy == "ready" {
want = 2
}
if len(delivered) != want {
t.Fatalf("warning->critical deliveries = %v, want %d callbacks", delivered, want)
}
if policy == "ready" && delivered[1] != AlertLevelCritical {
t.Fatalf("escalation = %v", delivered)
}
active := m.GetActiveAlerts()
found := false
for _, a := range active {
if a.Type == "custom-sensor" {
found = true
if a.ID != id || a.Level != AlertLevelCritical {
t.Fatalf("updated incident = %+v", a)
}
if policy == "acknowledged" && !a.Acknowledged {
t.Fatal("acknowledgement lost")
}
}
}
if !found {
t.Fatal("critical incident missing")
}
m.CheckHost(host)
host.Sensors.Custom[0].Status = "warning"
m.CheckHost(host)
m.CheckHost(host)
if len(delivered) != want {
t.Fatalf("unchanged/downgrade noise: %v", delivered)
}
})
}
}
// An agent reports things besides its own machine: filesystems, disks,
// arrays and custom sensors. Their alerts target a child resource, and the
// alert card's monitoring policy writes to that child, so they must not
// carry the machine's "agent" type: the card would offer the machine's
// retirement copy ("Agent removal remains available from Machines") for a
// policy that never touches the machine. platformType keeps every one of
// them linked to the agent's page.
func TestHostChildAlertsNameTheirOwnResourceType(t *testing.T) {
m := newTestManager(t)
m.ClearActiveAlerts()
m.mu.Lock()
m.config.TimeThresholds = map[string]int{}
m.config.AgentDefaults.CPU = &HysteresisThreshold{Trigger: 80, Clear: 70}
m.config.AgentDefaults.Disk = &HysteresisThreshold{Trigger: 80, Clear: 70}
m.config.AgentDefaults.DiskTemperature = &HysteresisThreshold{Trigger: 50, Clear: 45}
m.mu.Unlock()
pending := int64(2)
sensorValue := 23.0
linuxHost := models.Host{
ID: "child-linux",
Hostname: "storage-host",
DisplayName: "Storage Host",
Platform: "linux",
CPUUsage: 95,
Disks: []models.Disk{{
Mountpoint: "/srv",
Device: "/dev/sdc1",
Usage: 92,
Total: 100,
Used: 92,
Free: 8,
}},
Sensors: models.HostSensorSummary{
SMART: []models.HostDiskSMART{{
Device: "/dev/sda",
Model: "IronWolf",
Serial: "SERIAL-CHILD-1",
Health: "PASSED",
Temperature: 55,
Attributes: &models.SMARTAttributes{PendingSectors: &pending},
}},
Custom: []models.HostCustomSensorMetric{{
ID: "queue_depth",
Name: "Queue depth",
Value: &sensorValue,
Status: "critical",
ObservedAt: time.Now().UTC(),
}},
},
RAID: []models.HostRAIDArray{{
Device: "/dev/md0",
Level: "raid1",
State: "degraded",
TotalDevices: 2,
ActiveDevices: 1,
WorkingDevices: 1,
FailedDevices: 1,
}},
}
unraidHost := models.Host{
ID: "child-unraid",
Hostname: "tower",
DisplayName: "Tower",
Platform: "unraid",
Unraid: &models.HostUnraidStorage{
ArrayStarted: true,
ArrayState: "STARTED",
Disks: []models.HostUnraidDisk{
{Name: "disk1", Role: "data", Status: "online", Device: "/dev/sdb"},
},
},
}
m.CheckHost(linuxHost)
m.CheckHost(unraidHost)
want := map[string]string{
"cpu": "agent",
"disk": "agent-disk",
"diskTemperature": "agent-disk",
"disk-health": "agent-disk",
"raid": "agent-storage",
"storage-topology": "agent-storage",
"custom-sensor": "agent-sensor",
}
seen := make(map[string]bool, len(want))
for _, alert := range m.GetActiveAlerts() {
wantType, ok := want[alert.Type]
if !ok {
continue
}
seen[alert.Type] = true
if got := alert.Metadata["resourceType"]; got != wantType {
t.Errorf("%s alert resourceType = %v, want %s", alert.Type, got, wantType)
}
if got := alert.Metadata["platformType"]; got != "agent" {
t.Errorf("%s alert platformType = %v, want agent", alert.Type, got)
}
hostID, _ := alert.Metadata["hostId"].(string)
if wantType != "agent" && alert.ResourceID == hostResourceID(hostID) {
t.Errorf("%s alert targets the machine itself (%s), so it is not a child alert", alert.Type, alert.ResourceID)
}
}
for alertType := range want {
if !seen[alertType] {
t.Errorf("expected an active %s alert", alertType)
}
}
}
// Child alerts keep the agent policy path. A configuration save that turns
// off storage or guest alerts must leave them alone: CheckHost only consults
// the agent switches, so resolving them here would re-raise and re-notify
// on the next report.
func TestHostChildAlertsIgnoreOtherPlatformSwitchesOnConfigSave(t *testing.T) {
m := newTestManager(t)
m.ClearActiveAlerts()
m.mu.Lock()
m.config.TimeThresholds = map[string]int{}
m.mu.Unlock()
pending := int64(2)
sensorValue := 23.0
host := models.Host{
ID: "child-policy",
Hostname: "storage-host",
Platform: "linux",
Sensors: models.HostSensorSummary{
SMART: []models.HostDiskSMART{{
Device: "/dev/sda",
Serial: "SERIAL-CHILD-2",
Health: "PASSED",
Attributes: &models.SMARTAttributes{PendingSectors: &pending},
}},
Custom: []models.HostCustomSensorMetric{{
ID: "queue_depth",
Name: "Queue depth",
Value: &sensorValue,
Status: "critical",
ObservedAt: time.Now().UTC(),
}},
},
RAID: []models.HostRAIDArray{{
Device: "/dev/md0",
Level: "raid1",
State: "degraded",
TotalDevices: 2,
ActiveDevices: 1,
WorkingDevices: 1,
FailedDevices: 1,
}},
}
m.CheckHost(host)
childTypes := []string{"disk-health", "raid", "custom-sensor"}
for _, alertType := range childTypes {
if !hasAlertType(m.GetActiveAlerts(), alertType) {
t.Fatalf("expected an active %s alert before the config save", alertType)
}
}
m.mu.Lock()
m.config.DisableAllStorage = true
m.config.DisableAllGuests = true
m.reevaluateActiveAlertsLocked()
m.mu.Unlock()
for _, alertType := range childTypes {
if !hasAlertType(m.GetActiveAlerts(), alertType) {
t.Errorf("%s alert was resolved by the storage or guest switch", alertType)
}
}
m.mu.Lock()
m.config.DisableAllAgents = true
m.reevaluateActiveAlertsLocked()
m.mu.Unlock()
for _, alertType := range childTypes {
if hasAlertType(m.GetActiveAlerts(), alertType) {
t.Errorf("%s alert survived disabling agent alerts", alertType)
}
}
}
// The Proxmox node sweep removes alerts whose node is not a Proxmox node
// unless it recognises them as agent alerts, by an agent: resource id or the
// agent type. Child alerts no longer carry the agent type, so they must keep
// the agent: id their canonical spec gives them.
func TestHostChildAlertsSurviveProxmoxNodeCleanup(t *testing.T) {
m := newTestManager(t)
m.ClearActiveAlerts()
pending := int64(2)
sensorValue := 23.0
host := models.Host{
ID: "child-sweep",
Hostname: "storage-host",
Platform: "linux",
Sensors: models.HostSensorSummary{
SMART: []models.HostDiskSMART{{
Device: "/dev/sda",
Serial: "SERIAL-CHILD-3",
Health: "PASSED",
Attributes: &models.SMARTAttributes{PendingSectors: &pending},
}},
Custom: []models.HostCustomSensorMetric{{
ID: "queue_depth",
Name: "Queue depth",
Value: &sensorValue,
Status: "critical",
ObservedAt: time.Now().UTC(),
}},
},
RAID: []models.HostRAIDArray{{
Device: "/dev/md0",
Level: "raid1",
State: "degraded",
TotalDevices: 2,
ActiveDevices: 1,
WorkingDevices: 1,
FailedDevices: 1,
}},
}
m.CheckHost(host)
childTypes := []string{"disk-health", "raid", "custom-sensor"}
for _, alertType := range childTypes {
if !hasAlertType(m.GetActiveAlerts(), alertType) {
t.Fatalf("expected an active %s alert", alertType)
}
}
m.CleanupAlertsForNodes(map[string]bool{"pve1": true})
for _, alertType := range childTypes {
if !hasAlertType(m.GetActiveAlerts(), alertType) {
t.Errorf("Proxmox node cleanup removed an agent %s alert", alertType)
}
}
}
// Disabling between two reports is a real policy interval even when there is
// no firing alert for UpdateConfig to visit. Linked overrides keep precedence.
func TestDiskTemperaturePendingConfigDisablement(t *testing.T) {
for _, explicit := range []bool{false, true} {
for _, scope := range []string{"global", "agent-default", "all-agents", "host", "host-disabled", "node", "vm", "container", "type", "unrelated"} {
t.Run(fmt.Sprintf("%s/explicit=%v", scope, explicit), func(t *testing.T) {
m, elapsed := continuityManager(t, explicit)
host := hostWithSMARTDiskTemp("pending-temp", "custom", 85)
host.LinkedNodeID, host.LinkedVMID, host.LinkedContainerID = "site:node", "site:node:101", "site:node:102"
id := hostDiskTemperatureResourceID(host.ID, host.Sensors.SMART[0].Device)
m.CheckHost(host)
if incident, ok := continuityIncident(m, id, "diskTemperature"); !ok || incident.State != reducer.StatePending {
t.Fatalf("no initial pending run: %+v %v", incident, ok)
}
cfg := m.GetConfig()
switch scope {
case "global":
cfg.Enabled = false
case "agent-default":
cfg.AgentDefaults.DiskTemperature = &HysteresisThreshold{Trigger: 0}
case "host-disabled":
cfg.Overrides[host.ID] = ThresholdConfig{Disabled: true}
case "all-agents":
cfg.DisableAllAgents = true
case "host":
cfg.Overrides[host.ID] = ThresholdConfig{DiskTemperature: &HysteresisThreshold{Trigger: 0}}
case "node":
cfg.Overrides[host.LinkedNodeID] = ThresholdConfig{DiskTemperature: &HysteresisThreshold{Trigger: 0}}
case "vm":
host.LinkedNodeID = "" // The linked node override must not mask the VM.
m.CheckHost(host)
cfg.Overrides[host.LinkedVMID] = ThresholdConfig{DiskTemperature: &HysteresisThreshold{Trigger: 0}}
case "container":
host.LinkedNodeID, host.LinkedVMID = "", ""
m.CheckHost(host)
cfg.Overrides[host.LinkedContainerID] = ThresholdConfig{DiskTemperature: &HysteresisThreshold{Trigger: 0}}
case "type":
cfg.DiskTempByType = map[string]HysteresisThreshold{"custom": {Trigger: 0}}
case "unrelated":
cfg.Schedule.QuietHours.Timezone = "UTC"
}
m.UpdateConfig(cfg)
_, pending := continuityIncident(m, id, "diskTemperature")
if pending != (scope == "unrelated") {
t.Fatalf("after save pending=%v; unrelated saves alone must preserve evidence", pending)
}
if scope == "unrelated" {
return
}
cfg.Enabled, cfg.DisableAllAgents = true, false
cfg.AgentDefaults.DiskTemperature = &HysteresisThreshold{Trigger: 80, Clear: 70}
cfg.Overrides = map[string]ThresholdConfig{}
cfg.DiskTempByType = map[string]HysteresisThreshold{"custom": {Trigger: 80, Clear: 70}}
m.UpdateConfig(cfg)
elapsed.Store(int64(2 * time.Minute))
m.CheckHost(host)
if len(m.GetActiveAlerts()) != 0 {
t.Fatal("re-enable inherited the pre-disable start")
}
elapsed.Store(int64(2*time.Minute + 59*time.Second))
m.CheckHost(host)
if len(m.GetActiveAlerts()) != 0 {
t.Fatal("fresh grace fired early")
}
elapsed.Store(int64(3 * time.Minute))
m.CheckHost(host)
if len(m.GetActiveAlerts()) != 1 {
t.Fatal("fresh continuous breach never fired")
}
m.mu.RLock()
defer m.mu.RUnlock()
if len(m.hostDiskTempPendingContexts) != 0 {
t.Fatal("context outlived the pending run")
}
})
}
}
}
func TestDiskTemperaturePendingRestartContextIsNotInvented(t *testing.T) {
m, elapsed := continuityManager(t, true)
host := hostWithSMARTDiskTemp("restored-temp", "sata", 85)
id := hostDiskTemperatureResourceID(host.ID, host.Sensors.SMART[0].Device)
m.CheckHost(host)
m.mu.Lock()
// Persisted intent carries grace but not host link/type context. Reducer
// pending incidents are not restored; this is the actual restart shape.
m.core.Reset()
m.hostDiskTempPendingContexts = nil
key := canonicalMetricStateID(id, "diskTemperature")
_, pending := m.intentPending[key]
m.mu.Unlock()
if !pending {
t.Fatal("explicit intent control did not retain grace")
}
m.UpdateConfig(m.GetConfig())
m.mu.RLock()
_, pending = m.intentPending[key]
m.mu.RUnlock()
if pending {
t.Fatal("save retained grace whose link/type enablement is unknown")
}
elapsed.Store(int64(2 * time.Minute))
m.CheckHost(host)
if len(m.GetActiveAlerts()) != 0 {
t.Fatal("restored grace fired without fresh evidence")
}
}
func TestDiskTemperatureCurrentCollectionAndOverrideControls(t *testing.T) {
for _, provenance := range []string{"legacy", "available", "other-field-only"} {
t.Run(provenance, func(t *testing.T) {
m, elapsed := continuityManager(t, false)
host := hostWithSMARTDiskTemp("current-temp", "sata", 85)
switch provenance {
case "available":
host.Sensors.SMART[0].Collection = &diskinventory.CollectionStatus{Temperature: diskinventory.Available("smartctl")}
case "other-field-only":
host.Sensors.SMART[0].Collection = &diskinventory.CollectionStatus{Serial: diskinventory.Available("smartctl")}
}
m.CheckHost(host)
elapsed.Store(int64(time.Minute))
m.CheckHost(host)
if len(m.GetActiveAlerts()) != 1 {
t.Fatal("continuous accepted temperature did not fire")
}
})
}
t.Run("host override wins over disabled type", func(t *testing.T) {
m, elapsed := continuityManager(t, false)
host := hostWithSMARTDiskTemp("override-temp", "custom", 85)
cfg := m.GetConfig()
cfg.DiskTempByType = map[string]HysteresisThreshold{"custom": {Trigger: 0}}
cfg.Overrides[host.ID] = ThresholdConfig{DiskTemperature: &HysteresisThreshold{Trigger: 80, Clear: 70}}
m.UpdateConfig(cfg)
m.CheckHost(host)
m.UpdateConfig(m.GetConfig())
elapsed.Store(int64(time.Minute))
m.CheckHost(host)
if len(m.GetActiveAlerts()) != 1 {
t.Fatal("save discarded an enabled explicit override")
}
})
}
func TestDiskTemperaturePendingContextCleanup(t *testing.T) {
for _, removal := range []bool{false, true} {
t.Run(fmt.Sprintf("removed=%v", removal), func(t *testing.T) {
m, _ := continuityManager(t, false)
host := hostWithSMARTDiskTemp("pending-cleanup", "custom", 85)
m.CheckHost(host)
m.mu.RLock()
tracked := len(m.hostDiskTempPendingContexts)
m.mu.RUnlock()
if tracked != 1 || len(m.GetActiveAlerts()) != 0 {
t.Fatal("control must have pending context and no active alert")
}
if removal {
m.HandleHostRemoved(host)
} else {
m.ClearActiveAlerts()
}
m.mu.RLock()
defer m.mu.RUnlock()
if len(m.hostDiskTempPendingContexts) != 0 || m.hostDiskTemperaturePendingNoLock(hostDiskTemperatureResourceID(host.ID, host.Sensors.SMART[0].Device)) {
t.Fatal("pending context/state outlived its host or explicit clear")
}
})
}
}
// Feed only synthetic telemetry through the real CheckHost, config and callback
// paths. No collector, smartctl process, native host or notification provider.
type smartLifecycleFixture struct {
m *Manager
host models.Host
healthID, wearID string
initial map[string]Alert
fires atomic.Int32
recoveries chan *ResolvedAlert
}
func newSMARTLifecycleFixture(t *testing.T) *smartLifecycleFixture {
t.Helper()
m := newEventLogManager(t)
cfg := m.GetConfig()
cfg.FlappingEnabled = false
cfg.Schedule.Cooldown = 0
cfg.Schedule.MaxAlertsHour = 0
cfg.AgentDefaults.Disk = &HysteresisThreshold{Trigger: 80, Clear: 70}
cfg.AgentDefaults.DiskTemperature = &HysteresisThreshold{Trigger: 80, Clear: 70}
cfg.DiskTempByType = map[string]HysteresisThreshold{"sata": {Trigger: 80, Clear: 70}}
m.UpdateConfig(cfg)
m.mu.Lock()
m.config.TimeThresholds = map[string]int{}
m.config.MetricTimeThresholds = map[string]map[string]int{"all": {"disk": 0, "diskTemperature": 0}}
m.mu.Unlock()
used, pending := 96, int64(2)
host := models.Host{ID: "smart-lifecycle", Hostname: "storage", Status: "online",
Disks: []models.Disk{{Mountpoint: "/data", Device: "/dev/sda1", Total: 1000, Used: 950, Usage: 95}},
Sensors: models.HostSensorSummary{SMART: []models.HostDiskSMART{
{Device: "/dev/sda", Type: "sata", Health: "FAILED", Temperature: 95, Attributes: &models.SMARTAttributes{PercentageUsed: &used, PendingSectors: &pending}},
{Device: "/dev/sdb", Type: "sata", Health: "PASSED", Temperature: 20},
}},
}
resourceID, _ := hostSMARTDiskResourceID(host, host.Sensors.SMART[0])
f := &smartLifecycleFixture{m: m, host: host,
healthID: buildCanonicalStateID(resourceID, resourceID+"-disk-health"),
wearID: buildCanonicalStateID(resourceID, resourceID+"-disk-wearout"),
initial: make(map[string]Alert), recoveries: make(chan *ResolvedAlert, 32),
}
m.SetAlertCallback(func(a *Alert) {
if a.ID == f.healthID || a.ID == f.wearID {
f.fires.Add(1)
}
})
m.SetResolvedAlertCallback(func(r *ResolvedAlert) {
if r.Alert.ID == f.healthID || r.Alert.ID == f.wearID {
f.recoveries <- r
}
})
m.CheckHost(host)
for _, id := range []string{f.healthID, f.wearID} {
f.initial[id] = *testRequireActiveAlert(t, m, id)
}
if f.fires.Load() != 2 {
t.Fatalf("initial risk callbacks = %d, want 2", f.fires.Load())
}
return f
}
func (f *smartLifecycleFixture) held(t *testing.T) {
t.Helper()
for _, id := range []string{f.healthID, f.wearID} {
a := testRequireActiveAlert(t, f.m, id)
if !a.StartTime.Equal(f.initial[id].StartTime) {
t.Fatalf("%s was replaced by a new occurrence", id)
}
}
if f.fires.Load() != 2 {
t.Fatalf("missing report refired risk notifications: %d", f.fires.Load())
}
for _, e := range queryAlertEvents(t, f.m, eventlog.Filter{Types: []string{eventlog.TypeResolved}}) {
if e.AlertID == f.healthID || e.AlertID == f.wearID {
t.Fatalf("missing evidence published recovery: %+v", e)
}
}
}
func (f *smartLifecycleFixture) resolved(t *testing.T, ids ...string) {
t.Helper()
want := make(map[string]bool)
for _, id := range ids {
if testHasActiveAlert(t, f.m, id) {
t.Fatalf("risk alert remains active: %s", id)
}
want[id] = true
}
for range ids {
select {
case r := <-f.recoveries:
if !want[r.Alert.ID] || !r.Alert.StartTime.Equal(f.initial[r.Alert.ID].StartTime) {
t.Fatalf("wrong recovery occurrence: %+v", r)
}
delete(want, r.Alert.ID)
case <-time.After(3 * time.Second):
t.Fatal("ordinary recovery callback did not complete")
}
}
for _, id := range ids {
events := queryAlertEvents(t, f.m, eventlog.Filter{AlertID: id, Types: []string{eventlog.TypeResolved}})
if len(events) != 1 {
t.Fatalf("%s has %d resolution events, want 1", id, len(events))
}
}
}
func TestHostSMARTRiskMissingReportsPreserveOccurrenceAndCallbacks(t *testing.T) {
f := newSMARTLifecycleFixture(t)
if err := f.m.AcknowledgeAlert(f.healthID, "operator"); err != nil {
t.Fatal(err)
}
present := f.host.Sensors.SMART
// Both risk families on one disk must count once per report, not twice.
for _, list := range [][]models.HostDiskSMART{nil, {present[1]}, {present[1]}, nil, {present[1]}, present, nil, present} {
f.host.Sensors.SMART = list
f.m.CheckHost(f.host)
f.held(t)
if !testRequireActiveAlert(t, f.m, f.healthID).Acknowledged {
t.Fatal("acknowledgement lost during observation gap")
}
}
// Standby and unavailable health/wear fields are presence, not recovery.
f.host.Sensors.SMART = []models.HostDiskSMART{{Device: "/dev/sda", Standby: true}, present[1]}
f.m.CheckHost(f.host)
f.held(t)
f.host.Sensors.SMART[0].Standby = false
f.m.CheckHost(f.host)
f.held(t)
used, pending := 10, int64(0)
f.host.Sensors.SMART[0] = models.HostDiskSMART{Device: "/dev/sda", Health: "PASSED", Temperature: 20, Attributes: &models.SMARTAttributes{PercentageUsed: &used, PendingSectors: &pending}}
f.m.CheckHost(f.host)
f.resolved(t, f.healthID, f.wearID)
if f.fires.Load() != 2 {
t.Fatalf("healthy recovery refired: %d", f.fires.Load())
}
t.Log("empty/partial/standby/unknown reports preserved occurrence and acknowledgement; fresh complete evidence recovered once")
}
func TestHostSMARTRiskDepartureRequiresFreshConsecutiveReports(t *testing.T) {
for _, gap := range []string{"empty", "expiry", "present"} {
t.Run(gap, func(t *testing.T) {
f := newSMARTLifecycleFixture(t)
present := f.host.Sensors.SMART
f.host.Sensors.SMART = []models.HostDiskSMART{present[1]}
for i := 0; i < 2; i++ {
f.m.CheckHost(f.host)
f.held(t)
}
switch gap {
case "empty":
f.host.Sensors.SMART = nil
f.m.CheckHost(f.host)
case "expiry":
f.m.HandleHostTelemetryExpired(f.host)
case "present":
f.host.Sensors.SMART = present
f.m.CheckHost(f.host)
}
f.held(t)
f.host.Sensors.SMART = []models.HostDiskSMART{present[1]}
for i := 1; i <= 3; i++ {
f.m.CheckHost(f.host)
if i < 3 {
f.held(t)
}
}
f.resolved(t, f.healthID, f.wearID)
if f.fires.Load() != 2 {
t.Fatalf("departure refired %d callbacks", f.fires.Load())
}
})
}
}
func TestHostSMARTRiskRulesDisableBetweenReports(t *testing.T) {
f := newSMARTLifecycleFixture(t)
cfg := f.m.GetConfig()
off := 0
cfg.AgentDefaults.SMARTHealthFailure = &off
f.m.UpdateConfig(cfg)
f.held(t) // Pending sectors remain enabled; disabling only one cause is not recovery.
zero := int64(0)
cfg.AgentDefaults.SMARTPending = &zero
f.m.UpdateConfig(cfg)
f.resolved(t, f.healthID)
if !testHasActiveAlert(t, f.m, f.wearID) {
t.Fatal("health policy disabled unrelated wearout")
}
// A missing report cannot resurrect it. Re-enable before another report;
// only fresh failing evidence may reactivate it, under the unchanged
// stateful refire/cooldown history policy.
f.host.Sensors.SMART = nil
f.m.CheckHost(f.host)
if testHasActiveAlert(t, f.m, f.healthID) {
t.Fatal("missing report raised disabled health")
}
on, one := 1, int64(1)
cfg.AgentDefaults.SMARTHealthFailure = &on
cfg.AgentDefaults.SMARTPending = &one
f.m.UpdateConfig(cfg)
f.m.CheckHost(f.host)
if testHasActiveAlert(t, f.m, f.healthID) {
t.Fatal("re-enable raised health without evidence")
}
used, pending := 96, int64(2)
f.host.Sensors.SMART = []models.HostDiskSMART{{Device: "/dev/sda", Health: "FAILED", Attributes: &models.SMARTAttributes{PercentageUsed: &used, PendingSectors: &pending}}}
f.m.CheckHost(f.host)
a := testRequireActiveAlert(t, f.m, f.healthID)
if !a.StartTime.Equal(f.initial[f.healthID].StartTime) || !a.LastSeen.After(f.initial[f.healthID].LastSeen) || f.fires.Load() != 3 {
t.Fatalf("fresh evidence must refire once within the existing cooldown: start %v, prior %v, seen %v, prior %v, callbacks %d", a.StartTime, f.initial[f.healthID].StartTime, a.LastSeen, f.initial[f.healthID].LastSeen, f.fires.Load())
}
}
func TestHostSMARTRiskEmptyReportHonoursRuleOffAndLegacyUnknowns(t *testing.T) {
for _, metadata := range []string{"current", "legacy", "unknown"} {
t.Run(metadata, func(t *testing.T) {
f := newSMARTLifecycleFixture(t)
off, zero := 0, int64(0)
f.m.mu.Lock()
f.m.config.AgentDefaults.SMARTHealthFailure = &off
f.m.config.AgentDefaults.SMARTPending = &zero
if metadata != "current" {
a, _ := f.m.getActiveAlertNoLock(f.healthID)
if metadata == "legacy" {
delete(a.Metadata, "riskCodes")
} else {
a.Metadata["riskCodes"] = []interface{}{"future-code"}
}
}
f.m.mu.Unlock()
f.host.Sensors.SMART = nil
f.m.CheckHost(f.host)
if metadata == "current" {
f.resolved(t, f.healthID)
} else {
f.held(t)
}
if !testHasActiveAlert(t, f.m, f.wearID) {
t.Fatal("wearout was disabled with health")
}
})
}
}
func TestHostSMARTRiskFilesystemAndHostLifecycleStayIndependent(t *testing.T) {
for _, action := range []string{"filesystem-absent", "node-link", "removed", "host-disabled", "agents-disabled", "host-isolation"} {
t.Run(action, func(t *testing.T) {
f := newSMARTLifecycleFixture(t)
switch action {
case "filesystem-absent":
fsID, _ := hostDiskResourceID(f.host, f.host.Disks[0])
id := canonicalMetricStateID(fsID, "disk")
testRequireActiveAlert(t, f.m, id)
// A non-empty replacement establishes filesystem removal;
// an empty report alone is now an observation gap.
f.host.Disks = []models.Disk{{Mountpoint: "/other", Total: 1000}}
f.host.Sensors.SMART = nil
f.m.CheckHost(f.host)
if testHasActiveAlert(t, f.m, id) {
t.Fatal("filesystem absence incorrectly held usage")
}
f.held(t)
case "host-isolation":
other := f.host
other.ID = "smart-lifecycle-other"
f.m.CheckHost(other)
other.Sensors.SMART = []models.HostDiskSMART{other.Sensors.SMART[1]}
for i := 0; i < 3; i++ {
f.m.CheckHost(other)
}
f.held(t)
default:
present := f.host.Sensors.SMART
f.host.Sensors.SMART = []models.HostDiskSMART{present[1]}
f.m.CheckHost(f.host)
f.held(t)
switch action {
case "node-link":
f.host.LinkedNodeID = "pve-node"
f.host.Sensors.SMART = nil
f.m.CheckHost(f.host)
case "removed":
f.m.HandleHostRemoved(f.host)
case "host-disabled":
cfg := f.m.GetConfig()
cfg.Overrides[f.host.ID] = ThresholdConfig{Disabled: true}
f.m.UpdateConfig(cfg)
case "agents-disabled":
cfg := f.m.GetConfig()
cfg.DisableAllAgents = true
f.m.UpdateConfig(cfg)
}
f.resolved(t, f.healthID, f.wearID)
// Returning ownership gets a new occurrence and a fresh departure run.
f.host.LinkedNodeID = ""
cfg := f.m.GetConfig()
cfg.DisableAllAgents = false
delete(cfg.Overrides, f.host.ID)
f.m.UpdateConfig(cfg)
f.host.Sensors.SMART = present
f.m.CheckHost(f.host)
f.host.Sensors.SMART = []models.HostDiskSMART{present[1]}
for i := 0; i < 2; i++ {
f.m.CheckHost(f.host)
testRequireActiveAlert(t, f.m, f.healthID)
testRequireActiveAlert(t, f.m, f.wearID)
}
}
})
}
}
func TestHostSMARTRiskFilesystemIdentityCannotSubstituteInventory(t *testing.T) {
for _, direction := range []string{"SMART-only", "filesystem-only"} {
t.Run(direction, func(t *testing.T) {
f := newSMARTLifecycleFixture(t)
f.host.Disks[0].Mountpoint = "/sda"
f.m.CheckHost(f.host)
diskResourceID, _ := hostDiskResourceID(f.host, f.host.Disks[0])
riskResourceID, _ := hostSMARTDiskResourceID(f.host, f.host.Sensors.SMART[0])
if diskResourceID != riskResourceID {
t.Fatal("control must share the disk resource ID")
}
usageID := canonicalMetricStateID(diskResourceID, "disk")
testRequireActiveAlert(t, f.m, usageID)
if direction == "SMART-only" {
// A non-empty replacement establishes filesystem removal;
// an empty report alone is now an observation gap.
f.host.Disks = []models.Disk{{Mountpoint: "/other", Total: 1000}}
f.m.CheckHost(f.host)
if testHasActiveAlert(t, f.m, usageID) {
t.Fatal("SMART identity held missing filesystem usage")
}
f.held(t)
} else {
f.host.Sensors.SMART = []models.HostDiskSMART{f.host.Sensors.SMART[1]}
for i := 1; i <= 3; i++ {
f.m.CheckHost(f.host)
testRequireActiveAlert(t, f.m, usageID)
if i < 3 {
f.held(t)
}
}
f.resolved(t, f.healthID, f.wearID)
}
})
}
}
func TestHostSMARTRiskRestartAndGlobalPauseRestartDeparture(t *testing.T) {
for _, gap := range []string{"restart", "global-pause"} {
t.Run(gap, func(t *testing.T) {
f := newSMARTLifecycleFixture(t)
if err := f.m.AcknowledgeAlert(f.healthID, "operator"); err != nil {
t.Fatal(err)
}
f.host.Sensors.SMART = []models.HostDiskSMART{f.host.Sensors.SMART[1]}
for i := 0; i < 2; i++ {
f.m.CheckHost(f.host)
f.held(t)
}
cfg := f.m.GetConfig()
if gap == "restart" {
dir := filepath.Dir(f.m.alertsDir)
f.m.Stop() // Actual checkpoint and ordinary persisted-alert restoration.
f.m = NewManagerWithDataDir(dir)
t.Cleanup(f.m.Stop)
store, err := eventlog.OpenInMemory()
if err != nil {
t.Fatal(err)
}
f.m.SetEventLog(store) // Observe post-restart lifecycle, not nonexistent old in-memory events.
f.m.UpdateConfig(cfg)
f.m.SetAlertCallback(func(a *Alert) {
if a.ID == f.healthID || a.ID == f.wearID {
f.fires.Add(1)
}
})
f.m.SetResolvedAlertCallback(func(r *ResolvedAlert) {
if r.Alert.ID == f.healthID || r.Alert.ID == f.wearID {
f.recoveries <- r
}
})
} else {
cfg.Enabled = false
f.m.UpdateConfig(cfg)
cfg.Enabled = true
f.m.UpdateConfig(cfg)
}
for i := 1; i <= 3; i++ {
f.m.CheckHost(f.host)
if i < 3 {
f.held(t)
if !testRequireActiveAlert(t, f.m, f.healthID).Acknowledged {
t.Fatal("gap lost persisted acknowledgement")
}
}
}
f.resolved(t, f.healthID, f.wearID)
})
}
}
// Upstream's shared Unraid readings must retain reviewed pending-grace
// ownership even when no SMART row exists; genuine gaps still restart grace.
func TestUnraidOnlyTemperatureRetainsPendingContinuity(t *testing.T) {
for _, explicit := range []bool{false, true} {
for _, gap := range []string{"continuous", "spun-down", "omitted", "type-off"} {
t.Run(fmt.Sprintf("%s/explicit=%v", gap, explicit), func(t *testing.T) {
m, elapsed := continuityManager(t, explicit)
disk := models.HostUnraidDisk{Name: "disk1", Device: "sda", Serial: "CONTINUITY-1", Transport: "sata", Temperature: 85}
if gap == "type-off" {
// Canonical SATA zero triggers are restored to defaults on save.
// Custom transport entries support disabling through UpdateConfig.
disk.Transport = "custom"
}
host := unraidTempHost(nil, disk)
id := hostDiskTemperatureResourceID(host.ID, "sda")
// The inventory also has its separate array-health alert.
// Judge only the disk-temperature occurrence under test.
alertID := canonicalMetricStateID(id, "diskTemperature")
m.CheckHost(host)
if incident, ok := continuityIncident(m, id, "diskTemperature"); !ok || incident.State != reducer.StatePending {
t.Fatalf("no initial pending Unraid run: %+v %v", incident, ok)
}
elapsed.Store(int64(30 * time.Second))
switch gap {
case "continuous":
m.CheckHost(host)
case "spun-down":
host.Unraid.Disks[0].Temperature = 0
m.CheckHost(host)
host.Unraid.Disks[0] = disk
case "omitted":
host.Unraid.Disks = nil
m.CheckHost(host)
host.Unraid.Disks = []models.HostUnraidDisk{disk}
case "type-off":
cfg := m.GetConfig()
cfg.DiskTempByType = map[string]HysteresisThreshold{"custom": {Trigger: 0}}
m.UpdateConfig(cfg)
if threshold := m.DiskTemperatureThreshold(disk.Transport); threshold != nil && threshold.Trigger > 0 {
t.Fatalf("type-off fixture did not disable temperature alerts: %+v", threshold)
}
cfg.DiskTempByType = map[string]HysteresisThreshold{"custom": {Trigger: 80, Clear: 70}}
m.UpdateConfig(cfg)
}
elapsed.Store(int64(time.Minute))
m.CheckHost(host)
if gap == "continuous" {
if !testHasActiveAlert(t, m, alertID) {
t.Fatal("live Unraid-only reports lost continuous pending grace")
}
return
}
if testHasActiveAlert(t, m, alertID) {
t.Fatal("Unraid gap inherited earlier pending grace")
}
elapsed.Store(int64(119 * time.Second))
m.CheckHost(host)
if testHasActiveAlert(t, m, alertID) {
t.Fatal("fresh Unraid grace fired early")
}
elapsed.Store(int64(2 * time.Minute))
m.CheckHost(host)
if !testHasActiveAlert(t, m, alertID) {
t.Fatal("fresh continuous Unraid breach did not fire")
}
})
}
}
}