Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
package alerts
import (
"sort"
"strings"
"testing"
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/alerts/reducer"
alertspecs "github.com/rcourtman/pulse-go-rewrite/internal/alerts/specs"
"github.com/rcourtman/pulse-go-rewrite/internal/models"
"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
"github.com/rcourtman/pulse-go-rewrite/pkg/diskinventory"
)
func unifiedEvalBaseConfig() AlertConfig {
return AlertConfig{
Enabled: true,
ActivationState: ActivationActive,
GuestDefaults: ThresholdConfig{
CPU: &HysteresisThreshold{Trigger: 80, Clear: 75},
Memory: &HysteresisThreshold{Trigger: 85, Clear: 80},
Disk: &HysteresisThreshold{Trigger: 90, Clear: 85},
},
NodeDefaults: ThresholdConfig{
CPU: &HysteresisThreshold{Trigger: 80, Clear: 75},
Memory: &HysteresisThreshold{Trigger: 85, Clear: 80},
Disk: &HysteresisThreshold{Trigger: 90, Clear: 85},
},
AgentDefaults: ThresholdConfig{
CPU: &HysteresisThreshold{Trigger: 80, Clear: 75},
Memory: &HysteresisThreshold{Trigger: 85, Clear: 80},
Disk: &HysteresisThreshold{Trigger: 90, Clear: 85},
},
PBSDefaults: ThresholdConfig{
CPU: &HysteresisThreshold{Trigger: 80, Clear: 75},
Memory: &HysteresisThreshold{Trigger: 85, Clear: 80},
},
KubernetesDefaults: ThresholdConfig{
CPU: &HysteresisThreshold{Trigger: 80, Clear: 75},
Memory: &HysteresisThreshold{Trigger: 85, Clear: 80},
Disk: &HysteresisThreshold{Trigger: 90, Clear: 85},
DiskRead: &HysteresisThreshold{Trigger: 0, Clear: 0},
DiskWrite: &HysteresisThreshold{Trigger: 0, Clear: 0},
NetworkIn: &HysteresisThreshold{Trigger: 0, Clear: 0},
NetworkOut: &HysteresisThreshold{Trigger: 0, Clear: 0},
},
TrueNASDefaults: ThresholdConfig{
CPU: &HysteresisThreshold{Trigger: 80, Clear: 75},
Memory: &HysteresisThreshold{Trigger: 85, Clear: 80},
Disk: &HysteresisThreshold{Trigger: 85, Clear: 80},
Usage: &HysteresisThreshold{Trigger: 85, Clear: 80},
Temperature: &HysteresisThreshold{Trigger: 80, Clear: 75},
DiskRead: &HysteresisThreshold{Trigger: 0, Clear: 0},
DiskWrite: &HysteresisThreshold{Trigger: 0, Clear: 0},
NetworkIn: &HysteresisThreshold{Trigger: 0, Clear: 0},
NetworkOut: &HysteresisThreshold{Trigger: 0, Clear: 0},
},
TrueNASDiskDefaults: ThresholdConfig{
Temperature: &HysteresisThreshold{Trigger: 55, Clear: 50},
},
VMwareDefaults: ThresholdConfig{
CPU: &HysteresisThreshold{Trigger: 80, Clear: 75},
Memory: &HysteresisThreshold{Trigger: 85, Clear: 80},
Disk: &HysteresisThreshold{Trigger: 90, Clear: 85},
Usage: &HysteresisThreshold{Trigger: 85, Clear: 80},
DiskRead: &HysteresisThreshold{Trigger: 0, Clear: 0},
DiskWrite: &HysteresisThreshold{Trigger: 0, Clear: 0},
NetworkIn: &HysteresisThreshold{Trigger: 0, Clear: 0},
NetworkOut: &HysteresisThreshold{Trigger: 0, Clear: 0},
},
StorageDefault: HysteresisThreshold{Trigger: 85, Clear: 80},
Overrides: map[string]ThresholdConfig{},
// Keep these explicit to make test intent obvious; final values are forced in configureUnifiedEvalManager.
TimeThresholds: map[string]int{},
SuppressionWindow: 0,
MinimumDelta: 0,
}
}
func configureUnifiedEvalManager(t *testing.T, m *Manager, cfg AlertConfig) {
t.Helper()
m.UpdateConfig(cfg)
// UpdateConfig normalizes zero values back to defaults; force immediate alerting in tests.
m.mu.Lock()
m.config.TimeThresholds = map[string]int{}
m.config.MetricTimeThresholds = nil
m.config.SuppressionWindow = 0
m.config.MinimumDelta = 0
m.mu.Unlock()
m.ClearActiveAlerts()
}
func alertKeys(m *Manager) []string {
m.mu.RLock()
defer m.mu.RUnlock()
keys := make([]string, 0, len(m.activeAlerts))
for storageKey, alert := range m.activeAlerts {
keys = append(keys, effectiveAlertID(alert, storageKey))
}
sort.Strings(keys)
return keys
}
func assertAlertPresent(t *testing.T, m *Manager, alertID string) {
t.Helper()
m.mu.RLock()
_, exists := testLookupActiveAlert(t, m, alertID)
m.mu.RUnlock()
if !exists {
t.Fatalf("expected alert %q to exist, active alerts: %v", alertID, alertKeys(m))
}
}
func assertAlertMissing(t *testing.T, m *Manager, alertID string) {
t.Helper()
m.mu.RLock()
_, exists := testLookupActiveAlert(t, m, alertID)
m.mu.RUnlock()
if exists {
t.Fatalf("expected alert %q to be absent, active alerts: %v", alertID, alertKeys(m))
}
}
func TestUnifiedGuestDiskRenotificationRespectsMaxAlertsHour(t *testing.T) {
m := newTestManager(t)
cfg := unifiedEvalBaseConfig()
cfg.GuestDefaults = ThresholdConfig{
Disk: &HysteresisThreshold{Trigger: 80, Clear: 70},
}
cfg.Schedule.Cooldown = 30
cfg.Schedule.MaxAlertsHour = 1
configureUnifiedEvalManager(t, m, cfg)
dispatched := make(chan string, 4)
m.SetAlertCallback(func(alert *Alert) {
dispatched <- alert.ID
})
container := models.Container{
ID: "ct101",
Name: "smr",
Node: "ryzen5800x",
Status: "running",
Disks: []models.Disk{{
Mountpoint: "/",
Usage: 90.5,
Total: 100,
Used: 90,
Free: 10,
}},
}
m.CheckGuest(container, "pve1")
select {
case <-dispatched:
case <-time.After(time.Second):
t.Fatal("expected initial guest disk alert notification")
}
alertID := canonicalMetricStateID("ct101-disk-root", "disk")
lastNotified := time.Now().Add(-31 * time.Minute)
m.mu.Lock()
alert, exists := m.getActiveAlertNoLock(alertID)
if !exists {
m.mu.Unlock()
t.Fatalf("expected active guest disk alert %s", alertID)
}
alert.LastNotified = &lastNotified
m.mu.Unlock()
m.CheckGuest(container, "pve1")
select {
case id := <-dispatched:
t.Fatalf("expected max-alerts/hour to suppress repeated disk notification, got %s", id)
case <-time.After(250 * time.Millisecond):
}
}
func TestCheckUnifiedResourceMajorFamilies(t *testing.T) {
m := newTestManager(t)
configureUnifiedEvalManager(t, m, unifiedEvalBaseConfig())
tests := []struct {
name string
alertID string
input *UnifiedResourceInput
}{
{
name: "VM CPU above threshold creates alert",
alertID: canonicalMetricStateID("vm-101", "cpu"),
input: &UnifiedResourceInput{
ID: "vm-101",
Type: "vm",
Name: "vm-101",
CPU: &UnifiedResourceMetric{Percent: 90},
},
},
{
name: "System container CPU above threshold creates alert",
alertID: canonicalMetricStateID("lxc-200", "cpu"),
input: &UnifiedResourceInput{
ID: "lxc-200",
Type: "system-container",
Name: "worker-ct",
CPU: &UnifiedResourceMetric{Percent: 91},
},
},
{
name: "Node memory above threshold creates alert",
alertID: canonicalMetricStateID("node-a", "memory"),
input: &UnifiedResourceInput{
ID: "node-a",
Type: "node",
Name: "node-a",
Memory: &UnifiedResourceMetric{Percent: 90},
},
},
{
name: "Agent disk above threshold creates alert",
alertID: canonicalMetricStateID("host-1", "disk"),
input: &UnifiedResourceInput{
ID: "host-1",
Type: "agent",
Name: "host-1",
Disk: &UnifiedResourceMetric{Percent: 95},
},
},
{
name: "Storage usage above threshold creates alert",
alertID: canonicalMetricStateID("storage-1", "usage"),
input: &UnifiedResourceInput{
ID: "storage-1",
Type: "storage",
Name: "storage-1",
Disk: &UnifiedResourceMetric{Percent: 92},
},
},
{
name: "PBS CPU above threshold creates alert",
alertID: canonicalMetricStateID("pbs-1", "cpu"),
input: &UnifiedResourceInput{
ID: "pbs-1",
Type: "pbs",
Name: "pbs-1",
CPU: &UnifiedResourceMetric{Percent: 88},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
m.ClearActiveAlerts()
m.CheckUnifiedResource(tt.input)
assertAlertPresent(t, m, tt.alertID)
})
}
}
func TestCheckUnifiedResourceSupportsKubernetesTrueNASAndVMwareMetricTargets(t *testing.T) {
m := newTestManager(t)
configureUnifiedEvalManager(t, m, unifiedEvalBaseConfig())
tests := []struct {
name string
alertID string
input *UnifiedResourceInput
wantResourceType string
}{
{
name: "Kubernetes cluster CPU",
alertID: canonicalMetricStateID("k8s:prod", "cpu"),
input: &UnifiedResourceInput{
ID: "k8s:prod",
Type: "k8s-cluster",
Name: "prod",
CPU: &UnifiedResourceMetric{Percent: 88},
},
wantResourceType: "Kubernetes Cluster",
},
{
name: "Kubernetes node memory",
alertID: canonicalMetricStateID("k8s:prod/node:worker-1", "memory"),
input: &UnifiedResourceInput{
ID: "k8s:prod/node:worker-1",
Type: "k8s-node",
Name: "worker-1",
Node: "worker-1",
Memory: &UnifiedResourceMetric{Percent: 90},
},
wantResourceType: "Kubernetes Node",
},
{
name: "Kubernetes deployment CPU",
alertID: canonicalMetricStateID("k8s:prod/ns:default/deployment:api", "cpu"),
input: &UnifiedResourceInput{
ID: "k8s:prod/ns:default/deployment:api",
Type: "k8s-deployment",
Name: "api",
Node: "prod",
Instance: "prod",
CPU: &UnifiedResourceMetric{Percent: 83},
},
wantResourceType: "Kubernetes Deployment",
},
{
name: "Kubernetes pod disk",
alertID: canonicalMetricStateID("k8s:prod/ns:default/pod:api-7d9f", "disk"),
input: &UnifiedResourceInput{
ID: "k8s:prod/ns:default/pod:api-7d9f",
Type: "pod",
Name: "api-7d9f",
Node: "worker-1",
Instance: "prod",
Disk: &UnifiedResourceMetric{Percent: 93},
},
wantResourceType: "Kubernetes Pod",
},
{
name: "TrueNAS system temperature",
alertID: canonicalMetricStateID("agent:truenas-main", "temperature"),
input: &UnifiedResourceInput{
ID: "agent:truenas-main",
Type: "truenas-system",
Name: "truenas-main",
Node: "truenas-main",
Temperature: &UnifiedResourceMetric{Value: 82, Percent: 82},
},
wantResourceType: "TrueNAS System",
},
{
name: "TrueNAS pool usage",
alertID: canonicalMetricStateID("storage:truenas-main/pool:tank", "usage"),
input: &UnifiedResourceInput{
ID: "storage:truenas-main/pool:tank",
Type: "truenas-pool",
Name: "tank",
Node: "truenas-main",
Instance: "TrueNAS",
Disk: &UnifiedResourceMetric{Percent: 89},
},
wantResourceType: "TrueNAS Pool",
},
{
name: "TrueNAS dataset usage",
alertID: canonicalMetricStateID("storage:truenas-main/dataset:tank/apps", "usage"),
input: &UnifiedResourceInput{
ID: "storage:truenas-main/dataset:tank/apps",
Type: "truenas-dataset",
Name: "tank/apps",
Node: "truenas-main",
Instance: "TrueNAS",
Disk: &UnifiedResourceMetric{Percent: 91},
},
wantResourceType: "TrueNAS Dataset",
},
{
name: "TrueNAS disk temperature",
alertID: canonicalMetricStateID("physical-disk:truenas-main/ada0", "temperature"),
input: &UnifiedResourceInput{
ID: "physical-disk:truenas-main/ada0",
Type: "truenas-disk",
Name: "ada0",
Node: "truenas-main",
Instance: "TrueNAS",
Temperature: &UnifiedResourceMetric{Value: 58, Percent: 58},
},
wantResourceType: "TrueNAS Disk",
},
{
name: "vSphere host CPU",
alertID: canonicalMetricStateID("vmware:vc-1:host:host-101", "cpu"),
input: &UnifiedResourceInput{
ID: "vmware:vc-1:host:host-101",
Type: "vmware-host",
Name: "esxi-01.lab.local",
Node: "Prod Compute",
Instance: "Lab vCenter",
CPU: &UnifiedResourceMetric{Percent: 88},
},
wantResourceType: "vSphere Host",
},
{
name: "vSphere VM disk",
alertID: canonicalMetricStateID("vmware:vc-1:vm:vm-201", "disk"),
input: &UnifiedResourceInput{
ID: "vmware:vc-1:vm:vm-201",
Type: "vmware-vm",
Name: "app-01",
Node: "esxi-01.lab.local",
Instance: "Lab vCenter",
Disk: &UnifiedResourceMetric{Percent: 94},
},
wantResourceType: "vSphere VM",
},
{
name: "vSphere datastore usage",
alertID: canonicalMetricStateID("vmware:vc-1:datastore:datastore-301", "usage"),
input: &UnifiedResourceInput{
ID: "vmware:vc-1:datastore:datastore-301",
Type: "vmware-datastore",
Name: "nvme-primary",
Node: "Lab Datacenter",
Instance: "Lab vCenter",
Disk: &UnifiedResourceMetric{Percent: 89},
},
wantResourceType: "vSphere Datastore",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
m.ClearActiveAlerts()
m.CheckUnifiedResource(tt.input)
alert := activeAlert(t, m, tt.alertID)
if got := alert.Metadata["resourceType"]; got != tt.wantResourceType {
t.Fatalf("resourceType metadata = %v, want %s", got, tt.wantResourceType)
}
// resourceType is a display label, so the owning platform travels separately.
wantPlatform := map[string]string{
"Kubernetes": "kubernetes",
"TrueNAS": "truenas",
"vSphere": "vmware-vsphere",
}[strings.Fields(tt.wantResourceType)[0]]
if got := alert.Metadata["platformType"]; got != wantPlatform {
t.Fatalf("platformType metadata = %v, want %s", got, wantPlatform)
}
})
}
}
func TestCheckUnifiedResourceUsesKubernetesTrueNASAndVMwareOverrides(t *testing.T) {
m := newTestManager(t)
cfg := unifiedEvalBaseConfig()
cfg.KubernetesDefaults.CPU = &HysteresisThreshold{Trigger: 95, Clear: 90}
cfg.TrueNASDefaults.Usage = &HysteresisThreshold{Trigger: 95, Clear: 90}
cfg.VMwareDefaults.CPU = &HysteresisThreshold{Trigger: 95, Clear: 90}
cfg.Overrides["k8s:prod/ns:default/pod:api-7d9f"] = ThresholdConfig{
CPU: &HysteresisThreshold{Trigger: 60, Clear: 55},
}
cfg.Overrides["storage:truenas-main/pool:tank"] = ThresholdConfig{
Usage: &HysteresisThreshold{Trigger: 60, Clear: 55},
}
cfg.Overrides["vmware:vc-1:host:host-101"] = ThresholdConfig{
CPU: &HysteresisThreshold{Trigger: 60, Clear: 55},
}
configureUnifiedEvalManager(t, m, cfg)
m.CheckUnifiedResource(&UnifiedResourceInput{
ID: "k8s:prod/ns:default/pod:api-7d9f",
Type: "pod",
Name: "api-7d9f",
CPU: &UnifiedResourceMetric{Percent: 65},
})
m.CheckUnifiedResource(&UnifiedResourceInput{
ID: "storage:truenas-main/pool:tank",
Type: "truenas-pool",
Name: "tank",
Disk: &UnifiedResourceMetric{Percent: 65},
})
m.CheckUnifiedResource(&UnifiedResourceInput{
ID: "vmware:vc-1:host:host-101",
Type: "vmware-host",
Name: "esxi-01.lab.local",
CPU: &UnifiedResourceMetric{Percent: 65},
})
k8sAlert := activeAlert(t, m, canonicalMetricStateID("k8s:prod/ns:default/pod:api-7d9f", "cpu"))
if k8sAlert.Threshold != 60 {
t.Fatalf("kubernetes override threshold = %v, want 60", k8sAlert.Threshold)
}
truenasAlert := activeAlert(t, m, canonicalMetricStateID("storage:truenas-main/pool:tank", "usage"))
if truenasAlert.Threshold != 60 {
t.Fatalf("truenas override threshold = %v, want 60", truenasAlert.Threshold)
}
vmwareAlert := activeAlert(t, m, canonicalMetricStateID("vmware:vc-1:host:host-101", "cpu"))
if vmwareAlert.Threshold != 60 {
t.Fatalf("vmware override threshold = %v, want 60", vmwareAlert.Threshold)
}
}
func TestUnifiedResourceInputFromKubernetesTrueNASAndVMwareResources(t *testing.T) {
pod := unifiedresources.Resource{
ID: " k8s:prod/ns:default/pod:api-7d9f ",
Type: unifiedresources.ResourceTypePod,
Name: "api-7d9f",
ParentName: "worker-1",
Tags: []string{"customer:alpha", "critical"},
Sources: []unifiedresources.DataSource{unifiedresources.SourceK8s},
Kubernetes: &unifiedresources.K8sData{
ClusterName: "prod",
NodeName: "worker-1",
Namespace: "default",
},
Metrics: &unifiedresources.ResourceMetrics{
CPU: &unifiedresources.MetricValue{Percent: 82},
Memory: &unifiedresources.MetricValue{Percent: 64},
},
}
podInput, ok := UnifiedResourceInputFromResource(pod)
if !ok {
t.Fatalf("expected Kubernetes pod resource to become unified alert input")
}
if podInput.Type != "pod" || podInput.ID != "k8s:prod/ns:default/pod:api-7d9f" || podInput.Node != "worker-1" || podInput.Instance != "prod" {
t.Fatalf("unexpected pod input: %+v", podInput)
}
if podInput.CPU == nil || podInput.CPU.Percent != 82 {
t.Fatalf("unexpected pod CPU metric: %+v", podInput.CPU)
}
if len(podInput.Tags) != 2 || podInput.Tags[0] != "customer:alpha" || podInput.Tags[1] != "critical" {
t.Fatalf("unexpected pod routing tags: %+v", podInput.Tags)
}
pool := unifiedresources.Resource{
ID: "storage:truenas-main/pool:tank",
Type: unifiedresources.ResourceTypeStorage,
Name: "tank",
ParentName: "truenas-main",
Sources: []unifiedresources.DataSource{unifiedresources.SourceTrueNAS},
Storage: &unifiedresources.StorageMeta{
Platform: string(unifiedresources.SourceTrueNAS),
Topology: "pool",
},
Metrics: &unifiedresources.ResourceMetrics{
Disk: &unifiedresources.MetricValue{Percent: 89},
},
}
poolInput, ok := UnifiedResourceInputFromResource(pool)
if !ok {
t.Fatalf("expected TrueNAS pool resource to become unified alert input")
}
if poolInput.Type != "truenas-pool" || poolInput.ID != "storage:truenas-main/pool:tank" || poolInput.Node != "truenas-main" || poolInput.Instance != "TrueNAS" {
t.Fatalf("unexpected pool input: %+v", poolInput)
}
if poolInput.Disk == nil || poolInput.Disk.Percent != 89 {
t.Fatalf("unexpected pool disk metric: %+v", poolInput.Disk)
}
temp := 58.0
disk := unifiedresources.Resource{
ID: "physical-disk:truenas-main/ada0",
Type: unifiedresources.ResourceTypePhysicalDisk,
Name: "ada0",
ParentName: "truenas-main",
Sources: []unifiedresources.DataSource{unifiedresources.SourceTrueNAS},
Temperature: &temp,
}
diskInput, ok := UnifiedResourceInputFromResource(disk)
if !ok {
t.Fatalf("expected TrueNAS disk resource to become unified alert input")
}
if diskInput.Type != "truenas-disk" || diskInput.Temperature == nil || diskInput.Temperature.Value != 58 {
t.Fatalf("unexpected disk input: %+v", diskInput)
}
vmwareHost := unifiedresources.Resource{
ID: " vmware:vc-1:host:host-101 ",
Type: unifiedresources.ResourceTypeAgent,
Name: "esxi-01.lab.local",
Sources: []unifiedresources.DataSource{unifiedresources.SourceVMware},
VMware: &unifiedresources.VMwareData{
ConnectionName: "Lab vCenter",
ClusterName: "Prod Compute",
VCenterHost: "vcenter.lab.local",
},
Metrics: &unifiedresources.ResourceMetrics{
CPU: &unifiedresources.MetricValue{Percent: 82},
},
}
hostInput, ok := UnifiedResourceInputFromResource(vmwareHost)
if !ok {
t.Fatalf("expected VMware host resource to become unified alert input")
}
if hostInput.Type != "vmware-host" || hostInput.ID != "vmware:vc-1:host:host-101" || hostInput.Node != "Prod Compute" || hostInput.Instance != "Lab vCenter" {
t.Fatalf("unexpected VMware host input: %+v", hostInput)
}
vmwareVM := unifiedresources.Resource{
ID: "vmware:vc-1:vm:vm-201",
Type: unifiedresources.ResourceTypeVM,
Name: "app-01",
ParentName: "esxi-01.lab.local",
Sources: []unifiedresources.DataSource{unifiedresources.SourceVMware},
VMware: &unifiedresources.VMwareData{
ConnectionName: "Lab vCenter",
RuntimeHostName: "esxi-01.lab.local",
},
Metrics: &unifiedresources.ResourceMetrics{
Disk: &unifiedresources.MetricValue{Percent: 94},
},
}
vmInput, ok := UnifiedResourceInputFromResource(vmwareVM)
if !ok {
t.Fatalf("expected VMware VM resource to become unified alert input")
}
if vmInput.Type != "vmware-vm" || vmInput.Node != "esxi-01.lab.local" || vmInput.Instance != "Lab vCenter" || vmInput.Disk == nil || vmInput.Disk.Percent != 94 {
t.Fatalf("unexpected VMware VM input: %+v", vmInput)
}
vmwareDatastore := unifiedresources.Resource{
ID: "vmware:vc-1:datastore:datastore-301",
Type: unifiedresources.ResourceTypeStorage,
Name: "nvme-primary",
Sources: []unifiedresources.DataSource{unifiedresources.SourceVMware},
Storage: &unifiedresources.StorageMeta{
Platform: "vmware-vsphere",
Topology: "datastore",
},
VMware: &unifiedresources.VMwareData{
ConnectionName: "Lab vCenter",
DatacenterName: "Lab Datacenter",
},
Metrics: &unifiedresources.ResourceMetrics{
Disk: &unifiedresources.MetricValue{Percent: 89},
},
}
datastoreInput, ok := UnifiedResourceInputFromResource(vmwareDatastore)
if !ok {
t.Fatalf("expected VMware datastore resource to become unified alert input")
}
if datastoreInput.Type != "vmware-datastore" || datastoreInput.Node != "Lab Datacenter" || datastoreInput.Instance != "Lab vCenter" {
t.Fatalf("unexpected VMware datastore input: %+v", datastoreInput)
}
vmwareNetwork := unifiedresources.Resource{
ID: "vmware:vc-1:network:network-401",
Type: unifiedresources.ResourceTypeNetwork,
Name: "VM Network",
Sources: []unifiedresources.DataSource{unifiedresources.SourceVMware},
VMware: &unifiedresources.VMwareData{
ConnectionName: "Lab vCenter",
DatacenterName: "Lab Datacenter",
EntityType: "network",
},
}
networkInput, ok := UnifiedResourceInputFromResource(vmwareNetwork)
if !ok {
t.Fatalf("expected VMware network resource to become unified alert input")
}
if networkInput.Type != "vmware-network" || networkInput.Node != "Lab Datacenter" || networkInput.Instance != "Lab vCenter" {
t.Fatalf("unexpected VMware network input: %+v", networkInput)
}
}
func TestCheckUnifiedResourceHonorsKubernetesTrueNASAndVMwareGlobalDisables(t *testing.T) {
m := newTestManager(t)
cfg := unifiedEvalBaseConfig()
cfg.DisableAllKubernetes = true
cfg.DisableAllTrueNAS = true
cfg.DisableAllVMware = true
configureUnifiedEvalManager(t, m, cfg)
m.CheckUnifiedResource(&UnifiedResourceInput{
ID: "k8s:prod",
Type: "k8s-cluster",
Name: "prod",
CPU: &UnifiedResourceMetric{Percent: 90},
})
m.CheckUnifiedResource(&UnifiedResourceInput{
ID: "storage:truenas-main/pool:tank",
Type: "truenas-pool",
Name: "tank",
Disk: &UnifiedResourceMetric{Percent: 90},
})
m.CheckUnifiedResource(&UnifiedResourceInput{
ID: "vmware:vc-1:host:host-101",
Type: "vmware-host",
Name: "esxi-01.lab.local",
CPU: &UnifiedResourceMetric{Percent: 90},
})
assertAlertMissing(t, m, canonicalMetricStateID("k8s:prod", "cpu"))
assertAlertMissing(t, m, canonicalMetricStateID("storage:truenas-main/pool:tank", "usage"))
assertAlertMissing(t, m, canonicalMetricStateID("vmware:vc-1:host:host-101", "cpu"))
}
func TestCheckUnifiedResourceRejectsLegacyGuestTypeAlias(t *testing.T) {
m := newTestManager(t)
configureUnifiedEvalManager(t, m, unifiedEvalBaseConfig())
m.CheckUnifiedResource(&UnifiedResourceInput{
ID: "legacy-ct-200",
Type: "lxc",
Name: "legacy-ct",
CPU: &UnifiedResourceMetric{Percent: 95},
})
assertAlertMissing(t, m, canonicalMetricStateID("legacy-ct-200", "cpu"))
}
func TestCheckUnifiedResourceOverrideLowerThresholdCreatesAlert(t *testing.T) {
m := newTestManager(t)
cfg := unifiedEvalBaseConfig()
cfg.Overrides["vm-override"] = ThresholdConfig{
CPU: &HysteresisThreshold{Trigger: 60, Clear: 55},
}
configureUnifiedEvalManager(t, m, cfg)
m.CheckUnifiedResource(&UnifiedResourceInput{
ID: "vm-override",
Type: "vm",
Name: "vm-override",
CPU: &UnifiedResourceMetric{Percent: 65},
})
assertAlertPresent(t, m, canonicalMetricStateID("vm-override", "cpu"))
}
func TestResolveResourceThresholdsUsesCephStorageSourceAlias(t *testing.T) {
m := newTestManager(t)
m.mu.Lock()
m.config.StorageDefault = HysteresisThreshold{Trigger: 95, Clear: 90}
m.config.Overrides = map[string]ThresholdConfig{
"agent:pve5-ceph-pool-data_replication": {
Usage: &HysteresisThreshold{Trigger: 50, Clear: 45},
},
}
thresholds := m.resolveResourceThresholds("storage", "pve5-ceph-pool-data_replication")
m.mu.Unlock()
if thresholds.Usage == nil {
t.Fatalf("expected storage usage threshold")
}
if thresholds.Usage.Trigger != 50 || thresholds.Usage.Clear != 45 {
t.Fatalf("usage threshold = %#v, want legacy agent alias override trigger 50 clear 45", thresholds.Usage)
}
}
func TestCheckUnifiedResourceNilInputNoPanic(t *testing.T) {
m := newTestManager(t)
configureUnifiedEvalManager(t, m, unifiedEvalBaseConfig())
defer func() {
if r := recover(); r != nil {
t.Fatalf("CheckUnifiedResource(nil) panicked: %v", r)
}
}()
m.CheckUnifiedResource(nil)
}
func TestCheckUnifiedResourceDisabledThresholdsNoAlert(t *testing.T) {
m := newTestManager(t)
cfg := unifiedEvalBaseConfig()
cfg.GuestDefaults.Disabled = true
configureUnifiedEvalManager(t, m, cfg)
m.CheckUnifiedResource(&UnifiedResourceInput{
ID: "vm-disabled",
Type: "vm",
Name: "vm-disabled",
CPU: &UnifiedResourceMetric{Percent: 95},
})
assertAlertMissing(t, m, canonicalMetricStateID("vm-disabled", "cpu"))
}
func TestCheckUnifiedResourceAnnotatesMetricAlertsWithCanonicalSpecMetadata(t *testing.T) {
m := newTestManager(t)
configureUnifiedEvalManager(t, m, unifiedEvalBaseConfig())
m.CheckUnifiedResource(&UnifiedResourceInput{
ID: "vm-annotated",
Type: "vm",
Name: "vm-annotated",
CPU: &UnifiedResourceMetric{Percent: 90},
})
m.mu.RLock()
alertID := canonicalMetricStateID("vm-annotated", "cpu")
alert := testRequireActiveAlert(t, m, alertID)
m.mu.RUnlock()
if alert == nil {
t.Fatalf("expected %s alert", alertID)
}
if got := alert.Metadata["canonicalAlertKind"]; got != string(alertspecs.AlertSpecKindMetricThreshold) {
t.Fatalf("canonicalAlertKind = %v, want %s", got, alertspecs.AlertSpecKindMetricThreshold)
}
if got := alert.Metadata["canonicalSpecID"]; got != canonicalMetricSpecID("vm-annotated", "cpu") {
t.Fatalf("canonicalSpecID = %v, want %s", got, canonicalMetricSpecID("vm-annotated", "cpu"))
}
if alert.CanonicalSpecID != canonicalMetricSpecID("vm-annotated", "cpu") {
t.Fatalf("CanonicalSpecID = %q, want %s", alert.CanonicalSpecID, canonicalMetricSpecID("vm-annotated", "cpu"))
}
if alert.CanonicalKind != string(alertspecs.AlertSpecKindMetricThreshold) {
t.Fatalf("CanonicalKind = %q, want %s", alert.CanonicalKind, alertspecs.AlertSpecKindMetricThreshold)
}
if alert.CanonicalState != canonicalMetricStateID("vm-annotated", "cpu") {
t.Fatalf("CanonicalState = %q, want %s", alert.CanonicalState, canonicalMetricStateID("vm-annotated", "cpu"))
}
}
func TestCheckUnifiedResourceKeepsInstanceScopedNodeDisplayNames(t *testing.T) {
m := newTestManager(t)
configureUnifiedEvalManager(t, m, unifiedEvalBaseConfig())
m.UpdateNodeDisplayName("cluster-a", "pve", "Alpha")
m.UpdateNodeDisplayName("cluster-b", "pve", "Beta")
m.CheckUnifiedResource(&UnifiedResourceInput{
ID: "vm-a",
Type: "vm",
Name: "vm-a",
Node: "pve",
Instance: "cluster-a",
CPU: &UnifiedResourceMetric{Percent: 90},
})
m.CheckUnifiedResource(&UnifiedResourceInput{
ID: "vm-b",
Type: "vm",
Name: "vm-b",
Node: "pve",
Instance: "cluster-b",
CPU: &UnifiedResourceMetric{Percent: 91},
})
m.UpdateNodeDisplayName("cluster-a", "pve", "Alpha Updated")
m.CheckUnifiedResource(&UnifiedResourceInput{
ID: "vm-a",
Type: "vm",
Name: "vm-a",
Node: "pve",
Instance: "cluster-a",
CPU: &UnifiedResourceMetric{Percent: 92},
})
m.mu.RLock()
alertA := testRequireActiveAlert(t, m, canonicalMetricStateID("vm-a", "cpu"))
alertB := testRequireActiveAlert(t, m, canonicalMetricStateID("vm-b", "cpu"))
m.mu.RUnlock()
if alertA.NodeDisplayName != "Alpha Updated" {
t.Fatalf("vm-a node display name = %q, want %q", alertA.NodeDisplayName, "Alpha Updated")
}
if alertB.NodeDisplayName != "Beta" {
t.Fatalf("vm-b node display name = %q, want %q", alertB.NodeDisplayName, "Beta")
}
gotByResourceID := make(map[string]Alert)
for _, alert := range m.GetActiveAlerts() {
gotByResourceID[alert.ResourceID] = alert
}
if gotByResourceID["vm-a"].NodeDisplayName != "Alpha Updated" {
t.Fatalf("GetActiveAlerts vm-a node display name = %q, want %q", gotByResourceID["vm-a"].NodeDisplayName, "Alpha Updated")
}
if gotByResourceID["vm-b"].NodeDisplayName != "Beta" {
t.Fatalf("GetActiveAlerts vm-b node display name = %q, want %q", gotByResourceID["vm-b"].NodeDisplayName, "Beta")
}
}
func TestCheckGuestPerDiskAnnotatesCanonicalSpecMetadata(t *testing.T) {
m := newTestManager(t)
configureUnifiedEvalManager(t, m, unifiedEvalBaseConfig())
guestID := BuildGuestKey("pve1", "node1", 101)
m.CheckGuest(models.VM{
ID: guestID,
VMID: 101,
Name: "app01",
Node: "node1",
Instance: "pve1",
Status: "running",
CPU: 0.20,
Memory: models.Memory{Usage: 40},
Disk: models.Disk{Usage: 40},
Disks: []models.Disk{
{
Mountpoint: "/",
Device: "scsi0",
Usage: 95,
Total: 100,
Used: 95,
Free: 5,
},
},
}, "pve1")
resourceID := guestID + "-disk-scsi0"
alertID := canonicalMetricStateID(resourceID, "disk")
m.mu.RLock()
alert := testRequireActiveAlert(t, m, alertID)
m.mu.RUnlock()
if alert == nil {
t.Fatalf("expected guest disk alert %q", alertID)
}
if got := alert.Metadata["canonicalAlertKind"]; got != string(alertspecs.AlertSpecKindMetricThreshold) {
t.Fatalf("canonicalAlertKind = %v, want %s", got, alertspecs.AlertSpecKindMetricThreshold)
}
if got := alert.Metadata["canonicalSpecID"]; got != canonicalMetricSpecID(resourceID, "disk") {
t.Fatalf("canonicalSpecID = %v, want %s", got, canonicalMetricSpecID(resourceID, "disk"))
}
}
// A guest that drops its last tag leaves none on its open alerts. Config-save
// reevaluation reads pulse-relaxed back from alert metadata, so a removed tag
// left behind judged the alert by the relaxed trigger: the save resolved a
// reading still over the guest's own trigger, and the next poll raised it
// again.
func TestCheckGuestDropsRemovedTagsFromOpenAlerts(t *testing.T) {
m := newTestManager(t)
configureUnifiedEvalManager(t, m, unifiedEvalBaseConfig())
guestID := BuildGuestKey("pve1", "node1", 102)
vm := models.VM{
ID: guestID,
VMID: 102,
Name: "app02",
Node: "node1",
Instance: "pve1",
Status: "running",
CPU: 0.97,
Memory: models.Memory{Usage: 40},
Disk: models.Disk{Usage: 40},
Tags: []string{"pulse-relaxed"},
}
alertID := canonicalMetricStateID(guestID, "cpu")
m.CheckGuest(vm, "pve1")
if !testHasActiveAlert(t, m, alertID) {
t.Fatalf("expected CPU alert %q over the relaxed trigger", alertID)
}
vm.Tags = nil
vm.CPU = 0.90
m.CheckGuest(vm, "pve1")
alert := testRequireActiveAlert(t, m, alertID)
m.mu.RLock()
tags, hasTags := alert.Metadata["tags"]
m.mu.RUnlock()
if hasTags {
t.Fatalf("alert metadata tags = %v, want none once the guest dropped them", tags)
}
m.UpdateConfig(m.GetConfig())
if !testHasActiveAlert(t, m, alertID) {
t.Fatalf("expected a config save to keep a CPU alert over the guest's own trigger after pulse-relaxed is removed")
}
}
func TestCheckGuestPerDiskCleansUpRemovedDiskAlerts(t *testing.T) {
m := newTestManager(t)
configureUnifiedEvalManager(t, m, unifiedEvalBaseConfig())
guestID := BuildGuestKey("pve1", "node1", 101)
baseGuest := models.VM{
ID: guestID,
VMID: 101,
Name: "app01",
Node: "node1",
Instance: "pve1",
Status: "running",
CPU: 0.20,
Memory: models.Memory{Usage: 40},
Disk: models.Disk{Usage: 40},
}
withDisk := baseGuest
withDisk.Disks = []models.Disk{
{
Mountpoint: "/",
Device: "scsi0",
Usage: 95,
Total: 100,
Used: 95,
Free: 5,
},
}
m.CheckGuest(withDisk, "pve1")
originalAlertID := canonicalMetricStateID(guestID+"-disk-scsi0", "disk")
if activeAlert(t, m, originalAlertID) == nil {
t.Fatalf("expected guest disk alert %q", originalAlertID)
}
changedDisk := baseGuest
changedDisk.Disks = []models.Disk{
{
Mountpoint: "/data",
Device: "scsi1",
Usage: 96,
Total: 100,
Used: 96,
Free: 4,
},
}
m.CheckGuest(changedDisk, "pve1")
newAlertID := canonicalMetricStateID(guestID+"-disk-data-scsi1", "disk")
m.mu.RLock()
_, oldExists := testLookupActiveAlert(t, m, originalAlertID)
_, newExists := testLookupActiveAlert(t, m, newAlertID)
m.mu.RUnlock()
if oldExists {
t.Fatalf("expected stale guest disk alert %q to be cleared", originalAlertID)
}
if !newExists {
t.Fatalf("expected replacement guest disk alert %q", newAlertID)
}
}
func TestCheckGuestDiskAlertMigrationDoesNotCrossGuestIdentity(t *testing.T) {
m := newTestManager(t)
configureUnifiedEvalManager(t, m, unifiedEvalBaseConfig())
oldResourceID := BuildGuestKey("proxmox2", "proxmox2", 107) + "-disk-local-lvm-vm-107-disk-0"
newResourceID := BuildGuestKey("proxmox2", "proxmox2", 108) + "-disk-local-lvm-vm-108-disk-0"
oldAlertID := canonicalMetricStateID(oldResourceID, "disk")
newAlertID := canonicalMetricStateID(newResourceID, "disk")
start := time.Now().Add(-10 * time.Minute)
alert := &Alert{
ID: oldAlertID,
Type: "disk",
Level: AlertLevelWarning,
ResourceID: oldResourceID,
CanonicalSpecID: canonicalMetricSpecID(oldResourceID, "disk"),
CanonicalKind: string(alertspecs.AlertSpecKindMetricThreshold),
CanonicalState: oldAlertID,
ResourceName: "wireguard",
Node: "proxmox2",
Instance: "proxmox2",
Message: "wireguard disk at 92.5%",
Value: 92.5,
Threshold: 90,
StartTime: start,
LastSeen: start.Add(5 * time.Minute),
}
m.mu.Lock()
m.activeAlerts[oldAlertID] = alert
m.historyManager.AddAlert(*alert)
m.mu.Unlock()
m.checkMetric(newResourceID, "pulse", "proxmox2", "proxmox2", "vm", "disk", 63.6, &HysteresisThreshold{Trigger: 90, Clear: 85}, nil)
m.mu.RLock()
preserved, oldExists := testLookupActiveAlert(t, m, oldAlertID)
_, newExists := testLookupActiveAlert(t, m, newAlertID)
m.mu.RUnlock()
if !oldExists {
t.Fatal("expected guest 107 disk alert to stay on guest 107")
}
if preserved.ResourceID != oldResourceID || preserved.ResourceName != "wireguard" {
t.Fatalf("guest 107 alert was mutated: %#v", preserved)
}
if newExists {
t.Fatal("did not expect a guest 108 disk alert to be created")
}
if resolved := m.GetResolvedAlert(newAlertID); resolved != nil {
t.Fatalf("did not expect guest 108 resolved alert after evaluating guest 108 below threshold: %#v", resolved)
}
}
func TestCheckGuestClearsPerDiskAlertsWhenGuestStops(t *testing.T) {
m := newTestManager(t)
configureUnifiedEvalManager(t, m, unifiedEvalBaseConfig())
guestID := BuildGuestKey("pve1", "node1", 101)
guest := models.VM{
ID: guestID,
VMID: 101,
Name: "app01",
Node: "node1",
Instance: "pve1",
Status: "running",
CPU: 0.20,
Memory: models.Memory{Usage: 40},
Disk: models.Disk{Usage: 40},
Disks: []models.Disk{
{
Mountpoint: "/",
Device: "scsi0",
Usage: 95,
Total: 100,
Used: 95,
Free: 5,
},
},
}
m.CheckGuest(guest, "pve1")
alertID := canonicalMetricStateID(guestID+"-disk-scsi0", "disk")
if activeAlert(t, m, alertID) == nil {
t.Fatalf("expected guest disk alert %q", alertID)
}
guest.Status = "stopped"
guest.Disks = nil
m.CheckGuest(guest, "pve1")
m.mu.RLock()
_, exists := testLookupActiveAlert(t, m, alertID)
m.mu.RUnlock()
if exists {
t.Fatalf("expected guest disk alert %q to clear when guest stops", alertID)
}
}
func TestCheckNodeTemperatureAnnotatesCanonicalSpecMetadata(t *testing.T) {
m := newTestManager(t)
configureUnifiedEvalManager(t, m, unifiedEvalBaseConfig())
m.CheckNode(models.Node{
ID: "node/pve-1",
Name: "pve-1",
Instance: "pve-1",
Status: "online",
CPU: 0.20,
Memory: models.Memory{Usage: 40},
Disk: models.Disk{Usage: 40},
Temperature: &models.Temperature{
Available: true,
CPUPackage: 90,
},
})
m.mu.RLock()
alertID := canonicalMetricStateID("node/pve-1", "temperature")
alert := testRequireActiveAlert(t, m, alertID)
m.mu.RUnlock()
if alert == nil {
t.Fatal("expected node temperature alert")
}
if got := alert.Metadata["canonicalAlertKind"]; got != string(alertspecs.AlertSpecKindMetricThreshold) {
t.Fatalf("canonicalAlertKind = %v, want %s", got, alertspecs.AlertSpecKindMetricThreshold)
}
if got := alert.Metadata["canonicalSpecID"]; got != canonicalMetricSpecID("node/pve-1", "temperature") {
t.Fatalf("canonicalSpecID = %v, want %s", got, canonicalMetricSpecID("node/pve-1", "temperature"))
}
}
func TestCheckGuestPoweredOffAnnotatesCanonicalSpecMetadata(t *testing.T) {
m := newTestManager(t)
configureUnifiedEvalManager(t, m, unifiedEvalBaseConfig())
resourceID := BuildGuestKey("pve1", "node1", 101)
guest := models.VM{
ID: resourceID,
VMID: 101,
Name: "app01",
Node: "node1",
Instance: "pve1",
Status: "stopped",
}
m.CheckGuest(guest, "pve1")
m.CheckGuest(guest, "pve1")
alert := activeAlert(t, m, "guest-powered-off-"+resourceID)
if got := alert.Metadata["canonicalAlertKind"]; got != "powered-state" {
t.Fatalf("canonicalAlertKind = %v, want powered-state", got)
}
if got := alert.Metadata["canonicalSpecID"]; got != resourceID+"-powered-state" {
t.Fatalf("canonicalSpecID = %v, want %s", got, resourceID+"-powered-state")
}
}
func TestCheckNodeOfflineAnnotatesCanonicalSpecMetadata(t *testing.T) {
m := newTestManager(t)
configureUnifiedEvalManager(t, m, unifiedEvalBaseConfig())
m.mu.Lock()
m.mu.Unlock()
// Confirmation runs live in the reducer core: drive the
// preceding observations for real instead of seeding maps.
target := models.Node{
ID: "node/pve-1",
Name: "pve-1",
Instance: "pve1",
Status: "offline",
ConnectionHealth: "failed",
}
m.CheckNode(target)
m.CheckNode(target)
m.CheckNode(target)
alert := activeAlert(t, m, buildCanonicalStateID("node/pve-1", "node/pve-1-connectivity"))
if got := alert.Metadata["canonicalAlertKind"]; got != "connectivity" {
t.Fatalf("canonicalAlertKind = %v, want connectivity", got)
}
if got := alert.Metadata["canonicalSpecID"]; got != "node/pve-1-connectivity" {
t.Fatalf("canonicalSpecID = %v, want %s", got, "node/pve-1-connectivity")
}
}
func TestCheckPBSOfflineAnnotatesCanonicalSpecMetadata(t *testing.T) {
m := newTestManager(t)
configureUnifiedEvalManager(t, m, unifiedEvalBaseConfig())
m.mu.Lock()
m.mu.Unlock()
// Confirmation runs live in the reducer core: drive the
// preceding observations for real instead of seeding maps.
target := models.PBSInstance{
ID: "pbs-1",
Name: "pbs-main",
Host: "pbs-host",
Status: "online",
ConnectionHealth: "unhealthy",
}
m.CheckPBS(target)
m.CheckPBS(target)
m.CheckPBS(target)
alert := activeAlert(t, m, "pbs-offline-pbs-1")
if got := alert.Metadata["canonicalAlertKind"]; got != "connectivity" {
t.Fatalf("canonicalAlertKind = %v, want connectivity", got)
}
if got := alert.Metadata["canonicalSpecID"]; got != "pbs-1-connectivity" {
t.Fatalf("canonicalSpecID = %v, want %s", got, "pbs-1-connectivity")
}
}
func TestCheckStorageOfflineAnnotatesCanonicalSpecMetadata(t *testing.T) {
m := newTestManager(t)
configureUnifiedEvalManager(t, m, unifiedEvalBaseConfig())
m.mu.Lock()
m.mu.Unlock()
// Confirmation runs live in the reducer core: drive the
// preceding observations for real instead of seeding maps.
target := models.Storage{
ID: "storage-1",
Name: "local-lvm",
Node: "pve-1",
Instance: "pve1",
Status: "unavailable",
}
m.CheckStorage(target)
m.CheckStorage(target)
alert := activeAlert(t, m, "storage-offline-storage-1")
if got := alert.Metadata["canonicalAlertKind"]; got != "connectivity" {
t.Fatalf("canonicalAlertKind = %v, want connectivity", got)
}
if got := alert.Metadata["canonicalSpecID"]; got != "storage-1-connectivity" {
t.Fatalf("canonicalSpecID = %v, want %s", got, "storage-1-connectivity")
}
}
func TestCheckPMGOfflineAnnotatesCanonicalSpecMetadata(t *testing.T) {
m := newTestManager(t)
configureUnifiedEvalManager(t, m, unifiedEvalBaseConfig())
m.mu.Lock()
m.mu.Unlock()
// Confirmation runs live in the reducer core: drive the
// preceding observations for real instead of seeding maps.
target := models.PMGInstance{
ID: "pmg-1",
Name: "pmg-main",
Host: "pmg-host",
Status: "online",
ConnectionHealth: "unhealthy",
}
m.CheckPMG(target)
m.CheckPMG(target)
m.CheckPMG(target)
alert := activeAlert(t, m, "pmg-offline-pmg-1")
if got := alert.Metadata["canonicalAlertKind"]; got != "connectivity" {
t.Fatalf("canonicalAlertKind = %v, want connectivity", got)
}
if got := alert.Metadata["canonicalSpecID"]; got != "pmg-1-connectivity" {
t.Fatalf("canonicalSpecID = %v, want %s", got, "pmg-1-connectivity")
}
}
func TestHandleDockerHostOfflineAnnotatesCanonicalSpecMetadata(t *testing.T) {
m := newTestManager(t)
configureUnifiedEvalManager(t, m, unifiedEvalBaseConfig())
m.mu.Lock()
m.mu.Unlock()
// Confirmation runs live in the reducer core: drive the
// preceding observations for real instead of seeding maps.
target := models.DockerHost{
ID: "docker1",
DisplayName: "Docker Host 1",
Hostname: "docker.local",
AgentID: "agent-123",
}
m.HandleDockerHostOffline(target)
m.HandleDockerHostOffline(target)
m.HandleDockerHostOffline(target)
alert := activeAlert(t, m, "docker-host-offline-docker1")
if got := alert.Metadata["canonicalAlertKind"]; got != "connectivity" {
t.Fatalf("canonicalAlertKind = %v, want connectivity", got)
}
if got := alert.Metadata["canonicalSpecID"]; got != "docker:docker1-connectivity" {
t.Fatalf("canonicalSpecID = %v, want %s", got, "docker:docker1-connectivity")
}
}
func TestCheckDockerContainerStateAnnotatesCanonicalSpecMetadata(t *testing.T) {
m := newTestManager(t)
configureUnifiedEvalManager(t, m, unifiedEvalBaseConfig())
host := models.DockerHost{
ID: "host-1",
DisplayName: "Docker Host",
Hostname: "docker.local",
Containers: []models.DockerContainer{
{
ID: "container-1",
Name: "web",
State: "exited",
Status: "Exited (1) seconds ago",
Labels: map[string]string{"customer": "alpha", "critical": ""},
},
},
}
m.CheckDockerHost(host)
m.CheckDockerHost(host)
resourceID := DockerContainerResourceID(host.ID, "container-1", "")
alert := activeAlert(t, m, "docker-container-state-"+resourceID)
if got := alert.Metadata["canonicalAlertKind"]; got != "discrete-state" {
t.Fatalf("canonicalAlertKind = %v, want discrete-state", got)
}
if got := alert.Metadata["canonicalSpecID"]; got != resourceID+"-runtime-state" {
t.Fatalf("canonicalSpecID = %v, want %s", got, resourceID+"-runtime-state")
}
tags, ok := alert.Metadata["tags"].([]string)
if !ok || len(tags) != 2 || tags[0] != "critical" || tags[1] != "customer:alpha" {
t.Fatalf("container routing tags = %#v", alert.Metadata["tags"])
}
}
func TestCheckDockerServiceAnnotatesCanonicalSpecMetadata(t *testing.T) {
m := newTestManager(t)
configureUnifiedEvalManager(t, m, unifiedEvalBaseConfig())
host := models.DockerHost{
ID: "host-1",
DisplayName: "Prod Swarm",
Hostname: "swarm-prod",
Services: []models.DockerService{
{
ID: "svc-1",
Name: "web",
DesiredTasks: 4,
RunningTasks: 2,
Mode: "replicated",
Labels: map[string]string{"customer": "alpha", "critical": ""},
},
},
}
m.CheckDockerHost(host)
resourceID := DockerServiceResourceID(host.ID, "svc-1", "web")
alert := activeAlert(t, m, "docker-service-health-"+resourceID)
if got := alert.Metadata["canonicalAlertKind"]; got != "service-gap" {
t.Fatalf("canonicalAlertKind = %v, want service-gap", got)
}
if got := alert.Metadata["canonicalSpecID"]; got != resourceID+"-service-gap" {
t.Fatalf("canonicalSpecID = %v, want %s", got, resourceID+"-service-gap")
}
tags, ok := alert.Metadata["tags"].([]string)
if !ok || len(tags) != 2 || tags[0] != "critical" || tags[1] != "customer:alpha" {
t.Fatalf("service routing tags = %#v", alert.Metadata["tags"])
}
}
func TestCheckDockerServiceUpdateStateAnnotatesCanonicalSpecMetadata(t *testing.T) {
m := newTestManager(t)
configureUnifiedEvalManager(t, m, unifiedEvalBaseConfig())
now := time.Now()
host := models.DockerHost{
ID: "host-update",
DisplayName: "Swarm",
Hostname: "swarm.local",
Services: []models.DockerService{
{
ID: "svc-update",
Name: "api",
DesiredTasks: 1,
RunningTasks: 1,
UpdateStatus: &models.DockerServiceUpdate{
State: "rollback_failed",
Message: "Rollback failed",
CompletedAt: &now,
},
},
},
}
m.CheckDockerHost(host)
resourceID := DockerServiceResourceID(host.ID, "svc-update", "api")
alert := activeAlert(t, m, "docker-service-health-"+resourceID)
if got := alert.Metadata["canonicalAlertKind"]; got != "discrete-state" {
t.Fatalf("canonicalAlertKind = %v, want discrete-state", got)
}
if got := alert.Metadata["canonicalSpecID"]; got != resourceID+"-update-state" {
t.Fatalf("canonicalSpecID = %v, want %s", got, resourceID+"-update-state")
}
}
func TestCheckPMGQueueDepthAnnotatesCanonicalSpecMetadata(t *testing.T) {
m := newTestManager(t)
pmg := models.PMGInstance{
ID: "pmg-1",
Name: "PMG 1",
Nodes: []models.PMGNodeStatus{
{Name: "node1", QueueStatus: &models.PMGQueueStatus{Total: 300}},
{Name: "node2", QueueStatus: &models.PMGQueueStatus{Total: 250}},
},
}
m.checkPMGQueueDepths(pmg, PMGThresholdConfig{
QueueTotalWarning: 500,
QueueTotalCritical: 1000,
})
alert := activeAlert(t, m, "pmg-1-queue-total")
if got := alert.Metadata["canonicalAlertKind"]; got != "severity-threshold" {
t.Fatalf("canonicalAlertKind = %v, want severity-threshold", got)
}
if got := alert.Metadata["canonicalSpecID"]; got != "pmg-1-queue-total" {
t.Fatalf("canonicalSpecID = %v, want pmg-1-queue-total", got)
}
}
func TestCheckPMGOldestMessageAnnotatesCanonicalSpecMetadata(t *testing.T) {
m := newTestManager(t)
pmg := models.PMGInstance{
ID: "pmg-1",
Name: "PMG 1",
Nodes: []models.PMGNodeStatus{
{Name: "node1", QueueStatus: &models.PMGQueueStatus{OldestAge: 2400}},
},
}
m.checkPMGOldestMessage(pmg, PMGThresholdConfig{
OldestMessageWarnMins: 30,
OldestMessageCritMins: 60,
})
alert := activeAlert(t, m, "pmg-1-oldest-message")
if got := alert.Metadata["canonicalAlertKind"]; got != "severity-threshold" {
t.Fatalf("canonicalAlertKind = %v, want severity-threshold", got)
}
if got := alert.Metadata["canonicalSpecID"]; got != "pmg-1-oldest-message" {
t.Fatalf("canonicalSpecID = %v, want pmg-1-oldest-message", got)
}
if got := alert.Metadata["resourceType"]; got != string(unifiedresources.ResourceTypePMG) {
t.Fatalf("resourceType = %v, want pmg", got)
}
}
func TestCheckPMGNodeQueueAnnotatesCanonicalSpecMetadata(t *testing.T) {
m := newTestManager(t)
pmg := models.PMGInstance{
ID: "pmg-1",
Name: "PMG 1",
Nodes: []models.PMGNodeStatus{
{Name: "node-a", QueueStatus: &models.PMGQueueStatus{Total: 80}},
},
}
m.checkPMGNodeQueues(pmg, PMGThresholdConfig{
QueueTotalWarning: 100,
QueueTotalCritical: 200,
})
alert := activeAlert(t, m, "pmg-1-node-a-queue-total")
if got := alert.Metadata["canonicalAlertKind"]; got != "severity-threshold" {
t.Fatalf("canonicalAlertKind = %v, want severity-threshold", got)
}
if got := alert.Metadata["canonicalSpecID"]; got != "pmg-1-node-a-queue-total" {
t.Fatalf("canonicalSpecID = %v, want pmg-1-node-a-queue-total", got)
}
}
func TestCheckPMGQuarantineAnnotatesCanonicalSpecMetadata(t *testing.T) {
m := newTestManager(t)
pmg := models.PMGInstance{
ID: "pmg-1",
Name: "PMG 1",
Quarantine: &models.PMGQuarantineTotals{
Spam: 2500,
},
}
m.checkPMGQuarantineBacklog(pmg, PMGThresholdConfig{
QuarantineSpamWarn: 2000,
QuarantineSpamCritical: 5000,
})
alert := activeAlert(t, m, "pmg-1-quarantine-spam")
if got := alert.Metadata["canonicalAlertKind"]; got != "change-threshold" {
t.Fatalf("canonicalAlertKind = %v, want change-threshold", got)
}
if got := alert.Metadata["canonicalSpecID"]; got != "pmg-1-quarantine-spam" {
t.Fatalf("canonicalSpecID = %v, want pmg-1-quarantine-spam", got)
}
}
func TestCheckPMGAnomalyAnnotatesCanonicalSpecMetadata(t *testing.T) {
m := newTestManager(t)
base := time.Now().Add(-13 * time.Hour)
tracker := &pmgAnomalyTracker{
Samples: make([]pmgMailMetricSample, 0, 12),
LastSampleTime: base.Add(11 * time.Hour),
SampleCount: 12,
}
for i := 0; i < 12; i++ {
tracker.Samples = append(tracker.Samples, pmgMailMetricSample{
SpamIn: 100,
SpamOut: 10,
VirusIn: 1,
VirusOut: 1,
Timestamp: base.Add(time.Duration(i) * time.Hour),
})
}
m.mu.Lock()
m.pmgAnomalyTrackers["pmg-1"] = tracker
m.mu.Unlock()
pmg := models.PMGInstance{
ID: "pmg-1",
Name: "PMG 1",
MailCount: []models.PMGMailCountPoint{
{Timestamp: base.Add(12 * time.Hour), SpamIn: 420},
},
}
m.checkPMGAnomalies(pmg, PMGThresholdConfig{})
pmg.MailCount = []models.PMGMailCountPoint{
{Timestamp: base.Add(13 * time.Hour), SpamIn: 430},
}
m.checkPMGAnomalies(pmg, PMGThresholdConfig{})
alert := activeAlert(t, m, "pmg-1-anomaly-spamIn")
if got := alert.Metadata["canonicalAlertKind"]; got != "baseline-anomaly" {
t.Fatalf("canonicalAlertKind = %v, want baseline-anomaly", got)
}
if got := alert.Metadata["canonicalSpecID"]; got != "pmg-1-anomaly-spamIn" {
t.Fatalf("canonicalSpecID = %v, want pmg-1-anomaly-spamIn", got)
}
}
func TestUnifiedResourceInputExcludesDockerAppContainers(t *testing.T) {
// Docker containers are evaluated by CheckDockerHost against the
// docker:{host}/{containerName} override key (#1601); unified eval must
// not double-evaluate them under a second identity.
resource := unifiedresources.Resource{
ID: "app-container-1234567890abcdef",
Type: unifiedresources.ResourceTypeAppContainer,
Name: "media-server",
Docker: &unifiedresources.DockerData{
HostSourceID: "host-1",
ContainerID: "aaaaaaaaaaaa",
},
}
if input, ok := UnifiedResourceInputFromResource(resource); ok {
t.Fatalf("docker app-containers must stay owned by CheckDockerHost, got unified input %+v", input)
}
// TrueNAS apps remain unified-eval owned.
resource.Docker = nil
resource.TrueNAS = &unifiedresources.TrueNASData{}
input, ok := UnifiedResourceInputFromResource(resource)
if !ok || input.Type != "truenas-app" {
t.Fatalf("truenas app containers must stay unified-eval owned, got %+v ok=%v", input, ok)
}
}
func TestUnifiedStorageForecastKeepsCanonicalMetricSpecAcrossPlatforms(t *testing.T) {
m := newTestManager(t)
configureUnifiedEvalManager(t, m, unifiedEvalBaseConfig())
input := &UnifiedResourceInput{
ID: "vmware:lab/datastore:archive",
Type: "vmware-datastore",
Name: "archive",
Disk: &UnifiedResourceMetric{Percent: 70},
}
trend := CapacityTrendObservation{
Ready: true, Reason: "increasing", ObservedAt: time.Now(), DailyChange: 7,
Confidence: 0.99, SampleCount: 300, BucketCount: 48, CoverageSpan: 48 * time.Hour,
}
m.CheckUnifiedResourceWithCapacityTrend(input, trend)
m.CheckUnifiedResourceWithCapacityTrend(input, trend)
alert := testRequireActiveAlert(t, m, canonicalMetricStateID(input.ID, "usage"))
if alert.CanonicalSpecID != canonicalMetricSpecID(input.ID, "usage") {
t.Fatalf("CanonicalSpecID = %q, want canonical usage spec", alert.CanonicalSpecID)
}
}
func TestUnifiedMetricNoisyWarningWaitsButCriticalFiresImmediately(t *testing.T) {
m := newTestManager(t)
input := &UnifiedResourceInput{
ID: "vm-noisy-memory",
Type: "vm",
Name: "database",
Memory: &UnifiedResourceMetric{Percent: 90},
}
alertID := canonicalMetricStateID(input.ID, "memory")
m.CheckUnifiedResource(input)
assertAlertMissing(t, m, alertID)
input.Memory.Percent = 96
m.CheckUnifiedResource(input)
alert := testRequireActiveAlert(t, m, alertID)
if alert.Level != AlertLevelCritical {
t.Fatalf("critical memory alert level = %q, want %q", alert.Level, AlertLevelCritical)
}
if got := alert.Metadata["stabilityWindowSeconds"]; got != defaultNoisyGaugeStabilitySeconds {
t.Fatalf("stabilityWindowSeconds = %v, want %d", got, defaultNoisyGaugeStabilitySeconds)
}
}
func TestNodeUsageDispatchPreservesSeparatelyEvaluatedTemperature(t *testing.T) {
m, elapsed := continuityManager(t, false)
m.mu.Lock()
m.config.NodeDefaults.Temperature = &HysteresisThreshold{Trigger: 80, Clear: 70}
m.mu.Unlock()
node, _ := testNodeWithHostAgent()
node.Temperature = &models.Temperature{Available: true, CPUPackage: 85}
m.CheckNode(node)
elapsed.Store(int64(40 * time.Second))
m.CheckNode(node)
incident, ok := continuityIncident(m, node.ID, "temperature")
if !ok || incident.State != reducer.StatePending {
t.Fatal("temperature control did not remain pending")
}
originalStart := incident.PendingSince
elapsed.Store(int64(59 * time.Second))
m.CheckNode(node)
incident, _ = continuityIncident(m, node.ID, "temperature")
if !incident.PendingSince.Equal(originalStart) {
t.Fatal("usage dispatch interrupted a separately observed temperature")
}
elapsed.Store(int64(time.Minute))
m.CheckNode(node)
if !testHasActiveAlert(t, m, canonicalMetricStateID(node.ID, "temperature")) {
t.Fatal("continuous node temperature did not complete its delay")
}
node.Temperature = nil
elapsed.Store(int64(2 * time.Minute))
m.CheckNode(node)
if !testHasActiveAlert(t, m, canonicalMetricStateID(node.ID, "temperature")) {
t.Fatal("actual missing temperature became recovery")
}
}
// metricStatusClock drives a manager's wall and monotonic clocks together so
// recovery runs measure exactly the time the test advances.
type metricStatusClock struct {
now time.Time
tick time.Duration
}
func (c *metricStatusClock) advance(d time.Duration) {
c.now = c.now.Add(d)
c.tick += d
}
func newMetricStatusTestManager(t *testing.T) (*Manager, *metricStatusClock) {
t.Helper()
m := newTestManager(t)
// Recently-resolved retention prunes against the real clock, so start there.
clock := &metricStatusClock{now: time.Now().UTC()}
m.now = func() time.Time { return clock.now }
m.intentClock = func() time.Duration { return clock.tick }
m.mu.Lock()
m.config.NodeDefaults.Temperature = &HysteresisThreshold{Trigger: 80, Clear: 75}
m.mu.Unlock()
return m, clock
}
// fireNodeTemperatureAlert holds a breaching reading until the factory
// stability delay lets the alert open, as a real poll loop would.
func fireNodeTemperatureAlert(t *testing.T, m *Manager, clock *metricStatusClock, celsius float64) *Alert {
t.Helper()
for attempt := 0; attempt < 40; attempt++ {
checkMetricStatusNode(m, celsius)
if alert := activeNodeTemperatureAlert(t, m); alert != nil {
return alert
}
clock.advance(30 * time.Second)
}
t.Fatalf("%v°C never opened the temperature alert", celsius)
return nil
}
func checkMetricStatusNode(m *Manager, celsius float64) {
m.CheckNode(models.Node{
ID: "homelab-minipc",
Name: "minipc",
Instance: "homelab",
Status: "online",
CPU: 0.2,
Memory: models.Memory{Usage: 40},
Disk: models.Disk{Usage: 40},
Temperature: &models.Temperature{
Available: true,
CPUPackage: celsius,
},
})
}
func activeNodeTemperatureAlert(t *testing.T, m *Manager) *Alert {
t.Helper()
m.mu.RLock()
defer m.mu.RUnlock()
alert, ok := m.getActiveAlertNoLock(canonicalMetricStateID("homelab-minipc", "temperature"))
if !ok {
return nil
}
return alert.Clone()
}
func TestOpenTemperatureAlertReportsLiveReadingWhileItHolds(t *testing.T) {
m, clock := newMetricStatusTestManager(t)
// Buffered so a dispatch never blocks the evaluator; len() counts sends.
notified := make(chan struct{}, 64)
m.SetAlertCallback(func(*Alert) { notified <- struct{}{} })
fired := fireNodeTemperatureAlert(t, m, clock, 85)
status := fired.MetricStatus
if status == nil || status.Phase != models.MetricAlertPhaseBreaching || status.Value != 85 ||
status.Trigger != 80 || status.Recovery != 75 || status.Unit != "°C" ||
!metricBreachTime(t, status).Equal(fired.LastSeen) {
t.Fatalf("breaching status = %+v", status)
}
if status.RecoveryDelaySeconds <= 0 {
t.Fatalf("temperature alerts carry a recovery delay, got %+v", status)
}
delay := time.Duration(status.RecoveryDelaySeconds) * time.Second
firedMessage, firedLastSeen, notifications := fired.Message, fired.LastSeen, len(notified)
// Between the clear and trigger levels the alert holds. The legacy
// snapshot keeps the breach; only the live status moves.
clock.advance(30 * time.Second)
checkMetricStatusNode(m, 78)
held := activeNodeTemperatureAlert(t, m)
if held == nil {
t.Fatal("78°C cleared an alert that should hold until 75°C")
}
if held.Value != 85 || held.Message != firedMessage || !held.LastSeen.Equal(firedLastSeen) {
t.Fatalf("hold rewrote the breach snapshot: value=%v message=%q lastSeen=%v", held.Value, held.Message, held.LastSeen)
}
if got := held.MetricStatus; got == nil || got.Phase != models.MetricAlertPhaseLatched || got.Value != 78 ||
!got.ObservedAt.Equal(clock.now) || got.RecoveryStartedAt != nil ||
!metricBreachTime(t, got).Equal(firedLastSeen) {
t.Fatalf("latched status = %+v", got)
}
// At or below the recovery level the recovery run starts.
clock.advance(30 * time.Second)
recoveryStart := clock.now
checkMetricStatusNode(m, 72)
if got := activeNodeTemperatureAlert(t, m).MetricStatus; got == nil ||
got.Phase != models.MetricAlertPhaseRecovering || got.Value != 72 ||
got.RecoveryStartedAt == nil || !got.RecoveryStartedAt.Equal(recoveryStart) || got.RecoveryElapsedSeconds != 0 ||
!metricBreachTime(t, got).Equal(firedLastSeen) {
t.Fatalf("recovery start status = %+v", got)
}
clock.advance(delay / 2)
checkMetricStatusNode(m, 75)
if got := activeNodeTemperatureAlert(t, m).MetricStatus; got == nil ||
got.Phase != models.MetricAlertPhaseRecovering || got.RecoveryElapsedSeconds != int(delay/2/time.Second) ||
!metricBreachTime(t, got).Equal(firedLastSeen) {
t.Fatalf("recovery progress status = %+v", got)
}
// A reading back above the recovery level resets the run.
clock.advance(30 * time.Second)
checkMetricStatusNode(m, 77)
if got := activeNodeTemperatureAlert(t, m).MetricStatus; got == nil ||
got.Phase != models.MetricAlertPhaseLatched || got.RecoveryStartedAt != nil || got.RecoveryElapsedSeconds != 0 ||
!metricBreachTime(t, got).Equal(firedLastSeen) {
t.Fatalf("status after recovery reset = %+v", got)
}
if got := len(notified); got != notifications {
t.Fatalf("holding and recovering sent %d notifications", got-notifications)
}
// A full recovery delay at or below 75°C resolves the alert.
checkMetricStatusNode(m, 72)
clock.advance(delay)
checkMetricStatusNode(m, 72)
if alert := activeNodeTemperatureAlert(t, m); alert != nil {
t.Fatalf("alert still open after a full recovery delay: %+v", alert.MetricStatus)
}
m.resolvedMutex.RLock()
defer m.resolvedMutex.RUnlock()
resolved := 0
for _, entry := range m.recentlyResolved {
if entry == nil || entry.Type != "temperature" {
continue
}
resolved++
if entry.MetricStatus != nil {
t.Fatalf("resolved record kept the live status of an open alert: %+v", entry.MetricStatus)
}
}
if resolved != 1 {
t.Fatalf("resolved temperature records = %d, want 1", resolved)
}
}
func TestBuildMetricAlertStatusCarriesWindowAndMonotonicRecovery(t *testing.T) {
observedAt := time.Date(2026, 10, 6, 10, 49, 0, 0, time.UTC)
status := buildMetricAlertStatus(metricStatusInput{
value: 74,
trigger: 80,
clear: 75,
recoveryDelaySeconds: 300,
unit: "%",
window: metricWindowObservation{Value: 74, CurrentValue: 91, WindowSeconds: 600, Ready: true},
incident: reducer.Incident{
RecoverySince: observedAt.Add(-10 * time.Minute),
RecoveryElapsed: 90 * time.Second,
RecoveryTicksSupplied: true,
},
observedAt: observedAt,
})
if status.Phase != models.MetricAlertPhaseRecovering || status.Value != 74 ||
status.RawValue == nil || *status.RawValue != 91 || status.EvaluationWindowSeconds != 600 {
t.Fatalf("windowed status = %+v", status)
}
// Elapsed time comes from the monotonic run, not wall-clock arithmetic.
if status.RecoveryElapsedSeconds != 90 {
t.Fatalf("recovery elapsed = %d, want 90", status.RecoveryElapsedSeconds)
}
// The reducer clears at the trigger when no clear level is configured.
noClear := buildMetricAlertStatus(metricStatusInput{value: 85, triggered: true, trigger: 80, observedAt: observedAt})
if noClear.Recovery != 80 || noClear.RawValue != nil {
t.Fatalf("status without clear level = %+v", noClear)
}
}
func trueNASTemperatureDisk(device, diskType string, celsius int) unifiedresources.Resource {
return unifiedresources.Resource{
ID: "physical-disk:truenas-main/" + device,
Type: unifiedresources.ResourceTypePhysicalDisk,
Name: device,
ParentName: "truenas-main",
Sources: []unifiedresources.DataSource{unifiedresources.SourceTrueNAS},
PhysicalDisk: &unifiedresources.PhysicalDiskMeta{
DevPath: "/dev/" + device,
DiskType: diskType,
Temperature: celsius,
},
}
}
func trueNASDiskTemperatureAlertID(resource unifiedresources.Resource) string {
return canonicalMetricStateID(resource.ID, "temperature")
}
func checkTrueNASTemperatureDisk(t *testing.T, m *Manager, resource unifiedresources.Resource) {
t.Helper()
input, ok := UnifiedResourceInputFromResource(resource)
if !ok {
t.Fatalf("TrueNAS disk %s did not become alert input", resource.ID)
}
m.CheckUnifiedResource(input)
}
func trueNASDiskTemperatureAlert(t *testing.T, m *Manager, resource unifiedresources.Resource) (*Alert, bool) {
t.Helper()
return testLookupActiveAlert(t, m, trueNASDiskTemperatureAlertID(resource))
}
// TestTrueNASDiskTemperatureAlertsFollowDiskTemperaturePolicy pins TrueNAS
// disk temperature alerts to the disk temperature policy that the TrueNAS
// storage table, Physical Disks and Patrol judge heat by: per disk type,
// unless the user saved a TrueNAS-wide value, with a per-disk override first.
func TestTrueNASDiskTemperatureAlertsFollowDiskTemperaturePolicy(t *testing.T) {
nvme63 := trueNASTemperatureDisk("nvme0n1", "nvme", 63)
nvme72 := trueNASTemperatureDisk("nvme1n1", "nvme", 72)
sata56 := trueNASTemperatureDisk("sda", "sata", 56)
sata60 := trueNASTemperatureDisk("sdb", "sata", 60)
tests := []struct {
name string
mutate func(*AlertConfig)
disks []unifiedresources.Resource
firing []unifiedresources.Resource
trigger map[string]float64
}{
{
name: "factory thresholds judge each disk by its type",
disks: []unifiedresources.Resource{nvme63, nvme72, sata56},
firing: []unifiedresources.Resource{nvme72, sata56},
trigger: map[string]float64{nvme72.ID: 70, sata56.ID: 55},
},
{
name: "a raised NVMe trigger moves NVMe alerts with it",
mutate: func(cfg *AlertConfig) {
cfg.DiskTempByType = map[string]HysteresisThreshold{"nvme": {Trigger: 75, Clear: 70}}
},
disks: []unifiedresources.Resource{nvme72, sata56},
firing: []unifiedresources.Resource{sata56},
},
{
name: "a saved TrueNAS-wide value applies to every TrueNAS disk",
mutate: func(cfg *AlertConfig) {
cfg.TrueNASDiskDefaults.Temperature = &HysteresisThreshold{Trigger: 62, Clear: 57}
},
disks: []unifiedresources.Resource{nvme63, sata60},
firing: []unifiedresources.Resource{nvme63},
trigger: map[string]float64{nvme63.ID: 62},
},
{
name: "a per-disk override beats the TrueNAS-wide value",
mutate: func(cfg *AlertConfig) {
cfg.TrueNASDiskDefaults.Temperature = &HysteresisThreshold{Trigger: 62, Clear: 57}
cfg.Overrides = map[string]ThresholdConfig{
sata56.ID: {Temperature: &HysteresisThreshold{Trigger: 50, Clear: 45}},
}
},
disks: []unifiedresources.Resource{sata56},
firing: []unifiedresources.Resource{sata56},
trigger: map[string]float64{sata56.ID: 50},
},
{
name: "a switched-off agent Disk Temp default switches TrueNAS disks off",
mutate: func(cfg *AlertConfig) {
cfg.AgentDefaults.DiskTemperature = &HysteresisThreshold{Trigger: 0, Clear: 0}
},
disks: []unifiedresources.Resource{nvme72, sata60},
},
{
name: "a saved TrueNAS-wide value still applies when the agent default is off",
mutate: func(cfg *AlertConfig) {
cfg.AgentDefaults.DiskTemperature = &HysteresisThreshold{Trigger: 0, Clear: 0}
cfg.TrueNASDiskDefaults.Temperature = &HysteresisThreshold{Trigger: 58, Clear: 53}
},
disks: []unifiedresources.Resource{sata60},
firing: []unifiedresources.Resource{sata60},
},
{
name: "a saved TrueNAS-wide off switches TrueNAS disks off",
mutate: func(cfg *AlertConfig) {
cfg.TrueNASDiskDefaults.Temperature = &HysteresisThreshold{Trigger: 0, Clear: 0}
},
disks: []unifiedresources.Resource{nvme72, sata60},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
m := newTestManager(t)
cfg := unifiedEvalBaseConfig()
cfg.TrueNASDiskDefaults = ThresholdConfig{}
if tt.mutate != nil {
tt.mutate(&cfg)
}
configureUnifiedEvalManager(t, m, cfg)
for _, disk := range tt.disks {
checkTrueNASTemperatureDisk(t, m, disk)
}
want := make(map[string]bool, len(tt.firing))
for _, disk := range tt.firing {
want[disk.ID] = true
}
for _, disk := range tt.disks {
alert, firing := trueNASDiskTemperatureAlert(t, m, disk)
if firing != want[disk.ID] {
t.Fatalf("%s firing = %v, want %v (active: %v)", disk.ID, firing, want[disk.ID], alertKeys(m))
}
if !firing {
continue
}
if got := alert.Metadata["diskType"]; got != disk.PhysicalDisk.DiskType {
t.Fatalf("%s diskType metadata = %v, want %q", disk.ID, got, disk.PhysicalDisk.DiskType)
}
if trigger, ok := tt.trigger[disk.ID]; ok && alert.Threshold != trigger {
t.Fatalf("%s threshold = %v, want %v", disk.ID, alert.Threshold, trigger)
}
}
})
}
}
// A last-known reading the source no longer collects is not evidence of heat.
func TestTrueNASDiskTemperatureAlertIgnoresRetainedReading(t *testing.T) {
m := newTestManager(t)
cfg := unifiedEvalBaseConfig()
cfg.TrueNASDiskDefaults = ThresholdConfig{}
configureUnifiedEvalManager(t, m, cfg)
disk := trueNASTemperatureDisk("sda", "sata", 80)
disk.PhysicalDisk.Collection = &diskinventory.CollectionStatus{
Temperature: diskinventory.Unavailable("truenas", "disk is in standby"),
}
checkTrueNASTemperatureDisk(t, m, disk)
if _, firing := trueNASDiskTemperatureAlert(t, m, disk); firing {
t.Fatalf("retained 80C reading raised an alert: %v", alertKeys(m))
}
}