Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

65.7 KB
Raw Blame
package alerts

import (
	"sort"
	"strings"
	"testing"
	"time"

	"github.com/rcourtman/pulse-go-rewrite/internal/alerts/reducer"
	alertspecs "github.com/rcourtman/pulse-go-rewrite/internal/alerts/specs"
	"github.com/rcourtman/pulse-go-rewrite/internal/models"
	"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
	"github.com/rcourtman/pulse-go-rewrite/pkg/diskinventory"
)

func unifiedEvalBaseConfig() AlertConfig {
	return AlertConfig{
		Enabled:         true,
		ActivationState: ActivationActive,
		GuestDefaults: ThresholdConfig{
			CPU:    &HysteresisThreshold{Trigger: 80, Clear: 75},
			Memory: &HysteresisThreshold{Trigger: 85, Clear: 80},
			Disk:   &HysteresisThreshold{Trigger: 90, Clear: 85},
		},
		NodeDefaults: ThresholdConfig{
			CPU:    &HysteresisThreshold{Trigger: 80, Clear: 75},
			Memory: &HysteresisThreshold{Trigger: 85, Clear: 80},
			Disk:   &HysteresisThreshold{Trigger: 90, Clear: 85},
		},
		AgentDefaults: ThresholdConfig{
			CPU:    &HysteresisThreshold{Trigger: 80, Clear: 75},
			Memory: &HysteresisThreshold{Trigger: 85, Clear: 80},
			Disk:   &HysteresisThreshold{Trigger: 90, Clear: 85},
		},
		PBSDefaults: ThresholdConfig{
			CPU:    &HysteresisThreshold{Trigger: 80, Clear: 75},
			Memory: &HysteresisThreshold{Trigger: 85, Clear: 80},
		},
		KubernetesDefaults: ThresholdConfig{
			CPU:        &HysteresisThreshold{Trigger: 80, Clear: 75},
			Memory:     &HysteresisThreshold{Trigger: 85, Clear: 80},
			Disk:       &HysteresisThreshold{Trigger: 90, Clear: 85},
			DiskRead:   &HysteresisThreshold{Trigger: 0, Clear: 0},
			DiskWrite:  &HysteresisThreshold{Trigger: 0, Clear: 0},
			NetworkIn:  &HysteresisThreshold{Trigger: 0, Clear: 0},
			NetworkOut: &HysteresisThreshold{Trigger: 0, Clear: 0},
		},
		TrueNASDefaults: ThresholdConfig{
			CPU:         &HysteresisThreshold{Trigger: 80, Clear: 75},
			Memory:      &HysteresisThreshold{Trigger: 85, Clear: 80},
			Disk:        &HysteresisThreshold{Trigger: 85, Clear: 80},
			Usage:       &HysteresisThreshold{Trigger: 85, Clear: 80},
			Temperature: &HysteresisThreshold{Trigger: 80, Clear: 75},
			DiskRead:    &HysteresisThreshold{Trigger: 0, Clear: 0},
			DiskWrite:   &HysteresisThreshold{Trigger: 0, Clear: 0},
			NetworkIn:   &HysteresisThreshold{Trigger: 0, Clear: 0},
			NetworkOut:  &HysteresisThreshold{Trigger: 0, Clear: 0},
		},
		TrueNASDiskDefaults: ThresholdConfig{
			Temperature: &HysteresisThreshold{Trigger: 55, Clear: 50},
		},
		VMwareDefaults: ThresholdConfig{
			CPU:        &HysteresisThreshold{Trigger: 80, Clear: 75},
			Memory:     &HysteresisThreshold{Trigger: 85, Clear: 80},
			Disk:       &HysteresisThreshold{Trigger: 90, Clear: 85},
			Usage:      &HysteresisThreshold{Trigger: 85, Clear: 80},
			DiskRead:   &HysteresisThreshold{Trigger: 0, Clear: 0},
			DiskWrite:  &HysteresisThreshold{Trigger: 0, Clear: 0},
			NetworkIn:  &HysteresisThreshold{Trigger: 0, Clear: 0},
			NetworkOut: &HysteresisThreshold{Trigger: 0, Clear: 0},
		},
		StorageDefault: HysteresisThreshold{Trigger: 85, Clear: 80},
		Overrides:      map[string]ThresholdConfig{},

		// Keep these explicit to make test intent obvious; final values are forced in configureUnifiedEvalManager.
		TimeThresholds:    map[string]int{},
		SuppressionWindow: 0,
		MinimumDelta:      0,
	}
}

func configureUnifiedEvalManager(t *testing.T, m *Manager, cfg AlertConfig) {
	t.Helper()

	m.UpdateConfig(cfg)

	// UpdateConfig normalizes zero values back to defaults; force immediate alerting in tests.
	m.mu.Lock()
	m.config.TimeThresholds = map[string]int{}
	m.config.MetricTimeThresholds = nil
	m.config.SuppressionWindow = 0
	m.config.MinimumDelta = 0
	m.mu.Unlock()

	m.ClearActiveAlerts()
}

func alertKeys(m *Manager) []string {
	m.mu.RLock()
	defer m.mu.RUnlock()

	keys := make([]string, 0, len(m.activeAlerts))
	for storageKey, alert := range m.activeAlerts {
		keys = append(keys, effectiveAlertID(alert, storageKey))
	}
	sort.Strings(keys)
	return keys
}

func assertAlertPresent(t *testing.T, m *Manager, alertID string) {
	t.Helper()

	m.mu.RLock()
	_, exists := testLookupActiveAlert(t, m, alertID)
	m.mu.RUnlock()
	if !exists {
		t.Fatalf("expected alert %q to exist, active alerts: %v", alertID, alertKeys(m))
	}
}

func assertAlertMissing(t *testing.T, m *Manager, alertID string) {
	t.Helper()

	m.mu.RLock()
	_, exists := testLookupActiveAlert(t, m, alertID)
	m.mu.RUnlock()
	if exists {
		t.Fatalf("expected alert %q to be absent, active alerts: %v", alertID, alertKeys(m))
	}
}

func TestUnifiedGuestDiskRenotificationRespectsMaxAlertsHour(t *testing.T) {
	m := newTestManager(t)
	cfg := unifiedEvalBaseConfig()
	cfg.GuestDefaults = ThresholdConfig{
		Disk: &HysteresisThreshold{Trigger: 80, Clear: 70},
	}
	cfg.Schedule.Cooldown = 30
	cfg.Schedule.MaxAlertsHour = 1
	configureUnifiedEvalManager(t, m, cfg)

	dispatched := make(chan string, 4)
	m.SetAlertCallback(func(alert *Alert) {
		dispatched <- alert.ID
	})

	container := models.Container{
		ID:     "ct101",
		Name:   "smr",
		Node:   "ryzen5800x",
		Status: "running",
		Disks: []models.Disk{{
			Mountpoint: "/",
			Usage:      90.5,
			Total:      100,
			Used:       90,
			Free:       10,
		}},
	}

	m.CheckGuest(container, "pve1")

	select {
	case <-dispatched:
	case <-time.After(time.Second):
		t.Fatal("expected initial guest disk alert notification")
	}

	alertID := canonicalMetricStateID("ct101-disk-root", "disk")
	lastNotified := time.Now().Add(-31 * time.Minute)
	m.mu.Lock()
	alert, exists := m.getActiveAlertNoLock(alertID)
	if !exists {
		m.mu.Unlock()
		t.Fatalf("expected active guest disk alert %s", alertID)
	}
	alert.LastNotified = &lastNotified
	m.mu.Unlock()

	m.CheckGuest(container, "pve1")

	select {
	case id := <-dispatched:
		t.Fatalf("expected max-alerts/hour to suppress repeated disk notification, got %s", id)
	case <-time.After(250 * time.Millisecond):
	}
}

func TestCheckUnifiedResourceMajorFamilies(t *testing.T) {
	m := newTestManager(t)
	configureUnifiedEvalManager(t, m, unifiedEvalBaseConfig())

	tests := []struct {
		name    string
		alertID string
		input   *UnifiedResourceInput
	}{
		{
			name:    "VM CPU above threshold creates alert",
			alertID: canonicalMetricStateID("vm-101", "cpu"),
			input: &UnifiedResourceInput{
				ID:   "vm-101",
				Type: "vm",
				Name: "vm-101",
				CPU:  &UnifiedResourceMetric{Percent: 90},
			},
		},
		{
			name:    "System container CPU above threshold creates alert",
			alertID: canonicalMetricStateID("lxc-200", "cpu"),
			input: &UnifiedResourceInput{
				ID:   "lxc-200",
				Type: "system-container",
				Name: "worker-ct",
				CPU:  &UnifiedResourceMetric{Percent: 91},
			},
		},
		{
			name:    "Node memory above threshold creates alert",
			alertID: canonicalMetricStateID("node-a", "memory"),
			input: &UnifiedResourceInput{
				ID:     "node-a",
				Type:   "node",
				Name:   "node-a",
				Memory: &UnifiedResourceMetric{Percent: 90},
			},
		},
		{
			name:    "Agent disk above threshold creates alert",
			alertID: canonicalMetricStateID("host-1", "disk"),
			input: &UnifiedResourceInput{
				ID:   "host-1",
				Type: "agent",
				Name: "host-1",
				Disk: &UnifiedResourceMetric{Percent: 95},
			},
		},
		{
			name:    "Storage usage above threshold creates alert",
			alertID: canonicalMetricStateID("storage-1", "usage"),
			input: &UnifiedResourceInput{
				ID:   "storage-1",
				Type: "storage",
				Name: "storage-1",
				Disk: &UnifiedResourceMetric{Percent: 92},
			},
		},
		{
			name:    "PBS CPU above threshold creates alert",
			alertID: canonicalMetricStateID("pbs-1", "cpu"),
			input: &UnifiedResourceInput{
				ID:   "pbs-1",
				Type: "pbs",
				Name: "pbs-1",
				CPU:  &UnifiedResourceMetric{Percent: 88},
			},
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			m.ClearActiveAlerts()
			m.CheckUnifiedResource(tt.input)
			assertAlertPresent(t, m, tt.alertID)
		})
	}
}

func TestCheckUnifiedResourceSupportsKubernetesTrueNASAndVMwareMetricTargets(t *testing.T) {
	m := newTestManager(t)
	configureUnifiedEvalManager(t, m, unifiedEvalBaseConfig())

	tests := []struct {
		name             string
		alertID          string
		input            *UnifiedResourceInput
		wantResourceType string
	}{
		{
			name:    "Kubernetes cluster CPU",
			alertID: canonicalMetricStateID("k8s:prod", "cpu"),
			input: &UnifiedResourceInput{
				ID:   "k8s:prod",
				Type: "k8s-cluster",
				Name: "prod",
				CPU:  &UnifiedResourceMetric{Percent: 88},
			},
			wantResourceType: "Kubernetes Cluster",
		},
		{
			name:    "Kubernetes node memory",
			alertID: canonicalMetricStateID("k8s:prod/node:worker-1", "memory"),
			input: &UnifiedResourceInput{
				ID:     "k8s:prod/node:worker-1",
				Type:   "k8s-node",
				Name:   "worker-1",
				Node:   "worker-1",
				Memory: &UnifiedResourceMetric{Percent: 90},
			},
			wantResourceType: "Kubernetes Node",
		},
		{
			name:    "Kubernetes deployment CPU",
			alertID: canonicalMetricStateID("k8s:prod/ns:default/deployment:api", "cpu"),
			input: &UnifiedResourceInput{
				ID:       "k8s:prod/ns:default/deployment:api",
				Type:     "k8s-deployment",
				Name:     "api",
				Node:     "prod",
				Instance: "prod",
				CPU:      &UnifiedResourceMetric{Percent: 83},
			},
			wantResourceType: "Kubernetes Deployment",
		},
		{
			name:    "Kubernetes pod disk",
			alertID: canonicalMetricStateID("k8s:prod/ns:default/pod:api-7d9f", "disk"),
			input: &UnifiedResourceInput{
				ID:       "k8s:prod/ns:default/pod:api-7d9f",
				Type:     "pod",
				Name:     "api-7d9f",
				Node:     "worker-1",
				Instance: "prod",
				Disk:     &UnifiedResourceMetric{Percent: 93},
			},
			wantResourceType: "Kubernetes Pod",
		},
		{
			name:    "TrueNAS system temperature",
			alertID: canonicalMetricStateID("agent:truenas-main", "temperature"),
			input: &UnifiedResourceInput{
				ID:          "agent:truenas-main",
				Type:        "truenas-system",
				Name:        "truenas-main",
				Node:        "truenas-main",
				Temperature: &UnifiedResourceMetric{Value: 82, Percent: 82},
			},
			wantResourceType: "TrueNAS System",
		},
		{
			name:    "TrueNAS pool usage",
			alertID: canonicalMetricStateID("storage:truenas-main/pool:tank", "usage"),
			input: &UnifiedResourceInput{
				ID:       "storage:truenas-main/pool:tank",
				Type:     "truenas-pool",
				Name:     "tank",
				Node:     "truenas-main",
				Instance: "TrueNAS",
				Disk:     &UnifiedResourceMetric{Percent: 89},
			},
			wantResourceType: "TrueNAS Pool",
		},
		{
			name:    "TrueNAS dataset usage",
			alertID: canonicalMetricStateID("storage:truenas-main/dataset:tank/apps", "usage"),
			input: &UnifiedResourceInput{
				ID:       "storage:truenas-main/dataset:tank/apps",
				Type:     "truenas-dataset",
				Name:     "tank/apps",
				Node:     "truenas-main",
				Instance: "TrueNAS",
				Disk:     &UnifiedResourceMetric{Percent: 91},
			},
			wantResourceType: "TrueNAS Dataset",
		},
		{
			name:    "TrueNAS disk temperature",
			alertID: canonicalMetricStateID("physical-disk:truenas-main/ada0", "temperature"),
			input: &UnifiedResourceInput{
				ID:          "physical-disk:truenas-main/ada0",
				Type:        "truenas-disk",
				Name:        "ada0",
				Node:        "truenas-main",
				Instance:    "TrueNAS",
				Temperature: &UnifiedResourceMetric{Value: 58, Percent: 58},
			},
			wantResourceType: "TrueNAS Disk",
		},
		{
			name:    "vSphere host CPU",
			alertID: canonicalMetricStateID("vmware:vc-1:host:host-101", "cpu"),
			input: &UnifiedResourceInput{
				ID:       "vmware:vc-1:host:host-101",
				Type:     "vmware-host",
				Name:     "esxi-01.lab.local",
				Node:     "Prod Compute",
				Instance: "Lab vCenter",
				CPU:      &UnifiedResourceMetric{Percent: 88},
			},
			wantResourceType: "vSphere Host",
		},
		{
			name:    "vSphere VM disk",
			alertID: canonicalMetricStateID("vmware:vc-1:vm:vm-201", "disk"),
			input: &UnifiedResourceInput{
				ID:       "vmware:vc-1:vm:vm-201",
				Type:     "vmware-vm",
				Name:     "app-01",
				Node:     "esxi-01.lab.local",
				Instance: "Lab vCenter",
				Disk:     &UnifiedResourceMetric{Percent: 94},
			},
			wantResourceType: "vSphere VM",
		},
		{
			name:    "vSphere datastore usage",
			alertID: canonicalMetricStateID("vmware:vc-1:datastore:datastore-301", "usage"),
			input: &UnifiedResourceInput{
				ID:       "vmware:vc-1:datastore:datastore-301",
				Type:     "vmware-datastore",
				Name:     "nvme-primary",
				Node:     "Lab Datacenter",
				Instance: "Lab vCenter",
				Disk:     &UnifiedResourceMetric{Percent: 89},
			},
			wantResourceType: "vSphere Datastore",
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			m.ClearActiveAlerts()
			m.CheckUnifiedResource(tt.input)

			alert := activeAlert(t, m, tt.alertID)
			if got := alert.Metadata["resourceType"]; got != tt.wantResourceType {
				t.Fatalf("resourceType metadata = %v, want %s", got, tt.wantResourceType)
			}
			// resourceType is a display label, so the owning platform travels separately.
			wantPlatform := map[string]string{
				"Kubernetes": "kubernetes",
				"TrueNAS":    "truenas",
				"vSphere":    "vmware-vsphere",
			}[strings.Fields(tt.wantResourceType)[0]]
			if got := alert.Metadata["platformType"]; got != wantPlatform {
				t.Fatalf("platformType metadata = %v, want %s", got, wantPlatform)
			}
		})
	}
}

func TestCheckUnifiedResourceUsesKubernetesTrueNASAndVMwareOverrides(t *testing.T) {
	m := newTestManager(t)
	cfg := unifiedEvalBaseConfig()
	cfg.KubernetesDefaults.CPU = &HysteresisThreshold{Trigger: 95, Clear: 90}
	cfg.TrueNASDefaults.Usage = &HysteresisThreshold{Trigger: 95, Clear: 90}
	cfg.VMwareDefaults.CPU = &HysteresisThreshold{Trigger: 95, Clear: 90}
	cfg.Overrides["k8s:prod/ns:default/pod:api-7d9f"] = ThresholdConfig{
		CPU: &HysteresisThreshold{Trigger: 60, Clear: 55},
	}
	cfg.Overrides["storage:truenas-main/pool:tank"] = ThresholdConfig{
		Usage: &HysteresisThreshold{Trigger: 60, Clear: 55},
	}
	cfg.Overrides["vmware:vc-1:host:host-101"] = ThresholdConfig{
		CPU: &HysteresisThreshold{Trigger: 60, Clear: 55},
	}
	configureUnifiedEvalManager(t, m, cfg)

	m.CheckUnifiedResource(&UnifiedResourceInput{
		ID:   "k8s:prod/ns:default/pod:api-7d9f",
		Type: "pod",
		Name: "api-7d9f",
		CPU:  &UnifiedResourceMetric{Percent: 65},
	})
	m.CheckUnifiedResource(&UnifiedResourceInput{
		ID:   "storage:truenas-main/pool:tank",
		Type: "truenas-pool",
		Name: "tank",
		Disk: &UnifiedResourceMetric{Percent: 65},
	})
	m.CheckUnifiedResource(&UnifiedResourceInput{
		ID:   "vmware:vc-1:host:host-101",
		Type: "vmware-host",
		Name: "esxi-01.lab.local",
		CPU:  &UnifiedResourceMetric{Percent: 65},
	})

	k8sAlert := activeAlert(t, m, canonicalMetricStateID("k8s:prod/ns:default/pod:api-7d9f", "cpu"))
	if k8sAlert.Threshold != 60 {
		t.Fatalf("kubernetes override threshold = %v, want 60", k8sAlert.Threshold)
	}
	truenasAlert := activeAlert(t, m, canonicalMetricStateID("storage:truenas-main/pool:tank", "usage"))
	if truenasAlert.Threshold != 60 {
		t.Fatalf("truenas override threshold = %v, want 60", truenasAlert.Threshold)
	}
	vmwareAlert := activeAlert(t, m, canonicalMetricStateID("vmware:vc-1:host:host-101", "cpu"))
	if vmwareAlert.Threshold != 60 {
		t.Fatalf("vmware override threshold = %v, want 60", vmwareAlert.Threshold)
	}
}

func TestUnifiedResourceInputFromKubernetesTrueNASAndVMwareResources(t *testing.T) {
	pod := unifiedresources.Resource{
		ID:         " k8s:prod/ns:default/pod:api-7d9f ",
		Type:       unifiedresources.ResourceTypePod,
		Name:       "api-7d9f",
		ParentName: "worker-1",
		Tags:       []string{"customer:alpha", "critical"},
		Sources:    []unifiedresources.DataSource{unifiedresources.SourceK8s},
		Kubernetes: &unifiedresources.K8sData{
			ClusterName: "prod",
			NodeName:    "worker-1",
			Namespace:   "default",
		},
		Metrics: &unifiedresources.ResourceMetrics{
			CPU:    &unifiedresources.MetricValue{Percent: 82},
			Memory: &unifiedresources.MetricValue{Percent: 64},
		},
	}
	podInput, ok := UnifiedResourceInputFromResource(pod)
	if !ok {
		t.Fatalf("expected Kubernetes pod resource to become unified alert input")
	}
	if podInput.Type != "pod" || podInput.ID != "k8s:prod/ns:default/pod:api-7d9f" || podInput.Node != "worker-1" || podInput.Instance != "prod" {
		t.Fatalf("unexpected pod input: %+v", podInput)
	}
	if podInput.CPU == nil || podInput.CPU.Percent != 82 {
		t.Fatalf("unexpected pod CPU metric: %+v", podInput.CPU)
	}
	if len(podInput.Tags) != 2 || podInput.Tags[0] != "customer:alpha" || podInput.Tags[1] != "critical" {
		t.Fatalf("unexpected pod routing tags: %+v", podInput.Tags)
	}

	pool := unifiedresources.Resource{
		ID:         "storage:truenas-main/pool:tank",
		Type:       unifiedresources.ResourceTypeStorage,
		Name:       "tank",
		ParentName: "truenas-main",
		Sources:    []unifiedresources.DataSource{unifiedresources.SourceTrueNAS},
		Storage: &unifiedresources.StorageMeta{
			Platform: string(unifiedresources.SourceTrueNAS),
			Topology: "pool",
		},
		Metrics: &unifiedresources.ResourceMetrics{
			Disk: &unifiedresources.MetricValue{Percent: 89},
		},
	}
	poolInput, ok := UnifiedResourceInputFromResource(pool)
	if !ok {
		t.Fatalf("expected TrueNAS pool resource to become unified alert input")
	}
	if poolInput.Type != "truenas-pool" || poolInput.ID != "storage:truenas-main/pool:tank" || poolInput.Node != "truenas-main" || poolInput.Instance != "TrueNAS" {
		t.Fatalf("unexpected pool input: %+v", poolInput)
	}
	if poolInput.Disk == nil || poolInput.Disk.Percent != 89 {
		t.Fatalf("unexpected pool disk metric: %+v", poolInput.Disk)
	}

	temp := 58.0
	disk := unifiedresources.Resource{
		ID:          "physical-disk:truenas-main/ada0",
		Type:        unifiedresources.ResourceTypePhysicalDisk,
		Name:        "ada0",
		ParentName:  "truenas-main",
		Sources:     []unifiedresources.DataSource{unifiedresources.SourceTrueNAS},
		Temperature: &temp,
	}
	diskInput, ok := UnifiedResourceInputFromResource(disk)
	if !ok {
		t.Fatalf("expected TrueNAS disk resource to become unified alert input")
	}
	if diskInput.Type != "truenas-disk" || diskInput.Temperature == nil || diskInput.Temperature.Value != 58 {
		t.Fatalf("unexpected disk input: %+v", diskInput)
	}

	vmwareHost := unifiedresources.Resource{
		ID:      " vmware:vc-1:host:host-101 ",
		Type:    unifiedresources.ResourceTypeAgent,
		Name:    "esxi-01.lab.local",
		Sources: []unifiedresources.DataSource{unifiedresources.SourceVMware},
		VMware: &unifiedresources.VMwareData{
			ConnectionName: "Lab vCenter",
			ClusterName:    "Prod Compute",
			VCenterHost:    "vcenter.lab.local",
		},
		Metrics: &unifiedresources.ResourceMetrics{
			CPU: &unifiedresources.MetricValue{Percent: 82},
		},
	}
	hostInput, ok := UnifiedResourceInputFromResource(vmwareHost)
	if !ok {
		t.Fatalf("expected VMware host resource to become unified alert input")
	}
	if hostInput.Type != "vmware-host" || hostInput.ID != "vmware:vc-1:host:host-101" || hostInput.Node != "Prod Compute" || hostInput.Instance != "Lab vCenter" {
		t.Fatalf("unexpected VMware host input: %+v", hostInput)
	}

	vmwareVM := unifiedresources.Resource{
		ID:         "vmware:vc-1:vm:vm-201",
		Type:       unifiedresources.ResourceTypeVM,
		Name:       "app-01",
		ParentName: "esxi-01.lab.local",
		Sources:    []unifiedresources.DataSource{unifiedresources.SourceVMware},
		VMware: &unifiedresources.VMwareData{
			ConnectionName:  "Lab vCenter",
			RuntimeHostName: "esxi-01.lab.local",
		},
		Metrics: &unifiedresources.ResourceMetrics{
			Disk: &unifiedresources.MetricValue{Percent: 94},
		},
	}
	vmInput, ok := UnifiedResourceInputFromResource(vmwareVM)
	if !ok {
		t.Fatalf("expected VMware VM resource to become unified alert input")
	}
	if vmInput.Type != "vmware-vm" || vmInput.Node != "esxi-01.lab.local" || vmInput.Instance != "Lab vCenter" || vmInput.Disk == nil || vmInput.Disk.Percent != 94 {
		t.Fatalf("unexpected VMware VM input: %+v", vmInput)
	}

	vmwareDatastore := unifiedresources.Resource{
		ID:      "vmware:vc-1:datastore:datastore-301",
		Type:    unifiedresources.ResourceTypeStorage,
		Name:    "nvme-primary",
		Sources: []unifiedresources.DataSource{unifiedresources.SourceVMware},
		Storage: &unifiedresources.StorageMeta{
			Platform: "vmware-vsphere",
			Topology: "datastore",
		},
		VMware: &unifiedresources.VMwareData{
			ConnectionName: "Lab vCenter",
			DatacenterName: "Lab Datacenter",
		},
		Metrics: &unifiedresources.ResourceMetrics{
			Disk: &unifiedresources.MetricValue{Percent: 89},
		},
	}
	datastoreInput, ok := UnifiedResourceInputFromResource(vmwareDatastore)
	if !ok {
		t.Fatalf("expected VMware datastore resource to become unified alert input")
	}
	if datastoreInput.Type != "vmware-datastore" || datastoreInput.Node != "Lab Datacenter" || datastoreInput.Instance != "Lab vCenter" {
		t.Fatalf("unexpected VMware datastore input: %+v", datastoreInput)
	}

	vmwareNetwork := unifiedresources.Resource{
		ID:      "vmware:vc-1:network:network-401",
		Type:    unifiedresources.ResourceTypeNetwork,
		Name:    "VM Network",
		Sources: []unifiedresources.DataSource{unifiedresources.SourceVMware},
		VMware: &unifiedresources.VMwareData{
			ConnectionName: "Lab vCenter",
			DatacenterName: "Lab Datacenter",
			EntityType:     "network",
		},
	}
	networkInput, ok := UnifiedResourceInputFromResource(vmwareNetwork)
	if !ok {
		t.Fatalf("expected VMware network resource to become unified alert input")
	}
	if networkInput.Type != "vmware-network" || networkInput.Node != "Lab Datacenter" || networkInput.Instance != "Lab vCenter" {
		t.Fatalf("unexpected VMware network input: %+v", networkInput)
	}
}

func TestCheckUnifiedResourceHonorsKubernetesTrueNASAndVMwareGlobalDisables(t *testing.T) {
	m := newTestManager(t)
	cfg := unifiedEvalBaseConfig()
	cfg.DisableAllKubernetes = true
	cfg.DisableAllTrueNAS = true
	cfg.DisableAllVMware = true
	configureUnifiedEvalManager(t, m, cfg)

	m.CheckUnifiedResource(&UnifiedResourceInput{
		ID:   "k8s:prod",
		Type: "k8s-cluster",
		Name: "prod",
		CPU:  &UnifiedResourceMetric{Percent: 90},
	})
	m.CheckUnifiedResource(&UnifiedResourceInput{
		ID:   "storage:truenas-main/pool:tank",
		Type: "truenas-pool",
		Name: "tank",
		Disk: &UnifiedResourceMetric{Percent: 90},
	})
	m.CheckUnifiedResource(&UnifiedResourceInput{
		ID:   "vmware:vc-1:host:host-101",
		Type: "vmware-host",
		Name: "esxi-01.lab.local",
		CPU:  &UnifiedResourceMetric{Percent: 90},
	})

	assertAlertMissing(t, m, canonicalMetricStateID("k8s:prod", "cpu"))
	assertAlertMissing(t, m, canonicalMetricStateID("storage:truenas-main/pool:tank", "usage"))
	assertAlertMissing(t, m, canonicalMetricStateID("vmware:vc-1:host:host-101", "cpu"))
}

func TestCheckUnifiedResourceRejectsLegacyGuestTypeAlias(t *testing.T) {
	m := newTestManager(t)
	configureUnifiedEvalManager(t, m, unifiedEvalBaseConfig())

	m.CheckUnifiedResource(&UnifiedResourceInput{
		ID:   "legacy-ct-200",
		Type: "lxc",
		Name: "legacy-ct",
		CPU:  &UnifiedResourceMetric{Percent: 95},
	})

	assertAlertMissing(t, m, canonicalMetricStateID("legacy-ct-200", "cpu"))
}

func TestCheckUnifiedResourceOverrideLowerThresholdCreatesAlert(t *testing.T) {
	m := newTestManager(t)
	cfg := unifiedEvalBaseConfig()
	cfg.Overrides["vm-override"] = ThresholdConfig{
		CPU: &HysteresisThreshold{Trigger: 60, Clear: 55},
	}
	configureUnifiedEvalManager(t, m, cfg)

	m.CheckUnifiedResource(&UnifiedResourceInput{
		ID:   "vm-override",
		Type: "vm",
		Name: "vm-override",
		CPU:  &UnifiedResourceMetric{Percent: 65},
	})

	assertAlertPresent(t, m, canonicalMetricStateID("vm-override", "cpu"))
}

func TestResolveResourceThresholdsUsesCephStorageSourceAlias(t *testing.T) {
	m := newTestManager(t)

	m.mu.Lock()
	m.config.StorageDefault = HysteresisThreshold{Trigger: 95, Clear: 90}
	m.config.Overrides = map[string]ThresholdConfig{
		"agent:pve5-ceph-pool-data_replication": {
			Usage: &HysteresisThreshold{Trigger: 50, Clear: 45},
		},
	}
	thresholds := m.resolveResourceThresholds("storage", "pve5-ceph-pool-data_replication")
	m.mu.Unlock()

	if thresholds.Usage == nil {
		t.Fatalf("expected storage usage threshold")
	}
	if thresholds.Usage.Trigger != 50 || thresholds.Usage.Clear != 45 {
		t.Fatalf("usage threshold = %#v, want legacy agent alias override trigger 50 clear 45", thresholds.Usage)
	}
}

func TestCheckUnifiedResourceNilInputNoPanic(t *testing.T) {
	m := newTestManager(t)
	configureUnifiedEvalManager(t, m, unifiedEvalBaseConfig())

	defer func() {
		if r := recover(); r != nil {
			t.Fatalf("CheckUnifiedResource(nil) panicked: %v", r)
		}
	}()

	m.CheckUnifiedResource(nil)
}

func TestCheckUnifiedResourceDisabledThresholdsNoAlert(t *testing.T) {
	m := newTestManager(t)
	cfg := unifiedEvalBaseConfig()
	cfg.GuestDefaults.Disabled = true
	configureUnifiedEvalManager(t, m, cfg)

	m.CheckUnifiedResource(&UnifiedResourceInput{
		ID:   "vm-disabled",
		Type: "vm",
		Name: "vm-disabled",
		CPU:  &UnifiedResourceMetric{Percent: 95},
	})

	assertAlertMissing(t, m, canonicalMetricStateID("vm-disabled", "cpu"))
}

func TestCheckUnifiedResourceAnnotatesMetricAlertsWithCanonicalSpecMetadata(t *testing.T) {
	m := newTestManager(t)
	configureUnifiedEvalManager(t, m, unifiedEvalBaseConfig())

	m.CheckUnifiedResource(&UnifiedResourceInput{
		ID:   "vm-annotated",
		Type: "vm",
		Name: "vm-annotated",
		CPU:  &UnifiedResourceMetric{Percent: 90},
	})

	m.mu.RLock()
	alertID := canonicalMetricStateID("vm-annotated", "cpu")
	alert := testRequireActiveAlert(t, m, alertID)
	m.mu.RUnlock()
	if alert == nil {
		t.Fatalf("expected %s alert", alertID)
	}
	if got := alert.Metadata["canonicalAlertKind"]; got != string(alertspecs.AlertSpecKindMetricThreshold) {
		t.Fatalf("canonicalAlertKind = %v, want %s", got, alertspecs.AlertSpecKindMetricThreshold)
	}
	if got := alert.Metadata["canonicalSpecID"]; got != canonicalMetricSpecID("vm-annotated", "cpu") {
		t.Fatalf("canonicalSpecID = %v, want %s", got, canonicalMetricSpecID("vm-annotated", "cpu"))
	}
	if alert.CanonicalSpecID != canonicalMetricSpecID("vm-annotated", "cpu") {
		t.Fatalf("CanonicalSpecID = %q, want %s", alert.CanonicalSpecID, canonicalMetricSpecID("vm-annotated", "cpu"))
	}
	if alert.CanonicalKind != string(alertspecs.AlertSpecKindMetricThreshold) {
		t.Fatalf("CanonicalKind = %q, want %s", alert.CanonicalKind, alertspecs.AlertSpecKindMetricThreshold)
	}
	if alert.CanonicalState != canonicalMetricStateID("vm-annotated", "cpu") {
		t.Fatalf("CanonicalState = %q, want %s", alert.CanonicalState, canonicalMetricStateID("vm-annotated", "cpu"))
	}
}

func TestCheckUnifiedResourceKeepsInstanceScopedNodeDisplayNames(t *testing.T) {
	m := newTestManager(t)
	configureUnifiedEvalManager(t, m, unifiedEvalBaseConfig())

	m.UpdateNodeDisplayName("cluster-a", "pve", "Alpha")
	m.UpdateNodeDisplayName("cluster-b", "pve", "Beta")

	m.CheckUnifiedResource(&UnifiedResourceInput{
		ID:       "vm-a",
		Type:     "vm",
		Name:     "vm-a",
		Node:     "pve",
		Instance: "cluster-a",
		CPU:      &UnifiedResourceMetric{Percent: 90},
	})
	m.CheckUnifiedResource(&UnifiedResourceInput{
		ID:       "vm-b",
		Type:     "vm",
		Name:     "vm-b",
		Node:     "pve",
		Instance: "cluster-b",
		CPU:      &UnifiedResourceMetric{Percent: 91},
	})

	m.UpdateNodeDisplayName("cluster-a", "pve", "Alpha Updated")
	m.CheckUnifiedResource(&UnifiedResourceInput{
		ID:       "vm-a",
		Type:     "vm",
		Name:     "vm-a",
		Node:     "pve",
		Instance: "cluster-a",
		CPU:      &UnifiedResourceMetric{Percent: 92},
	})

	m.mu.RLock()
	alertA := testRequireActiveAlert(t, m, canonicalMetricStateID("vm-a", "cpu"))
	alertB := testRequireActiveAlert(t, m, canonicalMetricStateID("vm-b", "cpu"))
	m.mu.RUnlock()

	if alertA.NodeDisplayName != "Alpha Updated" {
		t.Fatalf("vm-a node display name = %q, want %q", alertA.NodeDisplayName, "Alpha Updated")
	}
	if alertB.NodeDisplayName != "Beta" {
		t.Fatalf("vm-b node display name = %q, want %q", alertB.NodeDisplayName, "Beta")
	}

	gotByResourceID := make(map[string]Alert)
	for _, alert := range m.GetActiveAlerts() {
		gotByResourceID[alert.ResourceID] = alert
	}
	if gotByResourceID["vm-a"].NodeDisplayName != "Alpha Updated" {
		t.Fatalf("GetActiveAlerts vm-a node display name = %q, want %q", gotByResourceID["vm-a"].NodeDisplayName, "Alpha Updated")
	}
	if gotByResourceID["vm-b"].NodeDisplayName != "Beta" {
		t.Fatalf("GetActiveAlerts vm-b node display name = %q, want %q", gotByResourceID["vm-b"].NodeDisplayName, "Beta")
	}
}

func TestCheckGuestPerDiskAnnotatesCanonicalSpecMetadata(t *testing.T) {
	m := newTestManager(t)
	configureUnifiedEvalManager(t, m, unifiedEvalBaseConfig())

	guestID := BuildGuestKey("pve1", "node1", 101)
	m.CheckGuest(models.VM{
		ID:       guestID,
		VMID:     101,
		Name:     "app01",
		Node:     "node1",
		Instance: "pve1",
		Status:   "running",
		CPU:      0.20,
		Memory:   models.Memory{Usage: 40},
		Disk:     models.Disk{Usage: 40},
		Disks: []models.Disk{
			{
				Mountpoint: "/",
				Device:     "scsi0",
				Usage:      95,
				Total:      100,
				Used:       95,
				Free:       5,
			},
		},
	}, "pve1")

	resourceID := guestID + "-disk-scsi0"
	alertID := canonicalMetricStateID(resourceID, "disk")
	m.mu.RLock()
	alert := testRequireActiveAlert(t, m, alertID)
	m.mu.RUnlock()
	if alert == nil {
		t.Fatalf("expected guest disk alert %q", alertID)
	}
	if got := alert.Metadata["canonicalAlertKind"]; got != string(alertspecs.AlertSpecKindMetricThreshold) {
		t.Fatalf("canonicalAlertKind = %v, want %s", got, alertspecs.AlertSpecKindMetricThreshold)
	}
	if got := alert.Metadata["canonicalSpecID"]; got != canonicalMetricSpecID(resourceID, "disk") {
		t.Fatalf("canonicalSpecID = %v, want %s", got, canonicalMetricSpecID(resourceID, "disk"))
	}
}

// A guest that drops its last tag leaves none on its open alerts. Config-save
// reevaluation reads pulse-relaxed back from alert metadata, so a removed tag
// left behind judged the alert by the relaxed trigger: the save resolved a
// reading still over the guest's own trigger, and the next poll raised it
// again.
func TestCheckGuestDropsRemovedTagsFromOpenAlerts(t *testing.T) {
	m := newTestManager(t)
	configureUnifiedEvalManager(t, m, unifiedEvalBaseConfig())

	guestID := BuildGuestKey("pve1", "node1", 102)
	vm := models.VM{
		ID:       guestID,
		VMID:     102,
		Name:     "app02",
		Node:     "node1",
		Instance: "pve1",
		Status:   "running",
		CPU:      0.97,
		Memory:   models.Memory{Usage: 40},
		Disk:     models.Disk{Usage: 40},
		Tags:     []string{"pulse-relaxed"},
	}
	alertID := canonicalMetricStateID(guestID, "cpu")
	m.CheckGuest(vm, "pve1")
	if !testHasActiveAlert(t, m, alertID) {
		t.Fatalf("expected CPU alert %q over the relaxed trigger", alertID)
	}

	vm.Tags = nil
	vm.CPU = 0.90
	m.CheckGuest(vm, "pve1")
	alert := testRequireActiveAlert(t, m, alertID)
	m.mu.RLock()
	tags, hasTags := alert.Metadata["tags"]
	m.mu.RUnlock()
	if hasTags {
		t.Fatalf("alert metadata tags = %v, want none once the guest dropped them", tags)
	}

	m.UpdateConfig(m.GetConfig())
	if !testHasActiveAlert(t, m, alertID) {
		t.Fatalf("expected a config save to keep a CPU alert over the guest's own trigger after pulse-relaxed is removed")
	}
}

func TestCheckGuestPerDiskCleansUpRemovedDiskAlerts(t *testing.T) {
	m := newTestManager(t)
	configureUnifiedEvalManager(t, m, unifiedEvalBaseConfig())

	guestID := BuildGuestKey("pve1", "node1", 101)
	baseGuest := models.VM{
		ID:       guestID,
		VMID:     101,
		Name:     "app01",
		Node:     "node1",
		Instance: "pve1",
		Status:   "running",
		CPU:      0.20,
		Memory:   models.Memory{Usage: 40},
		Disk:     models.Disk{Usage: 40},
	}

	withDisk := baseGuest
	withDisk.Disks = []models.Disk{
		{
			Mountpoint: "/",
			Device:     "scsi0",
			Usage:      95,
			Total:      100,
			Used:       95,
			Free:       5,
		},
	}
	m.CheckGuest(withDisk, "pve1")

	originalAlertID := canonicalMetricStateID(guestID+"-disk-scsi0", "disk")
	if activeAlert(t, m, originalAlertID) == nil {
		t.Fatalf("expected guest disk alert %q", originalAlertID)
	}

	changedDisk := baseGuest
	changedDisk.Disks = []models.Disk{
		{
			Mountpoint: "/data",
			Device:     "scsi1",
			Usage:      96,
			Total:      100,
			Used:       96,
			Free:       4,
		},
	}
	m.CheckGuest(changedDisk, "pve1")

	newAlertID := canonicalMetricStateID(guestID+"-disk-data-scsi1", "disk")
	m.mu.RLock()
	_, oldExists := testLookupActiveAlert(t, m, originalAlertID)
	_, newExists := testLookupActiveAlert(t, m, newAlertID)
	m.mu.RUnlock()

	if oldExists {
		t.Fatalf("expected stale guest disk alert %q to be cleared", originalAlertID)
	}
	if !newExists {
		t.Fatalf("expected replacement guest disk alert %q", newAlertID)
	}
}

func TestCheckGuestDiskAlertMigrationDoesNotCrossGuestIdentity(t *testing.T) {
	m := newTestManager(t)
	configureUnifiedEvalManager(t, m, unifiedEvalBaseConfig())

	oldResourceID := BuildGuestKey("proxmox2", "proxmox2", 107) + "-disk-local-lvm-vm-107-disk-0"
	newResourceID := BuildGuestKey("proxmox2", "proxmox2", 108) + "-disk-local-lvm-vm-108-disk-0"
	oldAlertID := canonicalMetricStateID(oldResourceID, "disk")
	newAlertID := canonicalMetricStateID(newResourceID, "disk")
	start := time.Now().Add(-10 * time.Minute)

	alert := &Alert{
		ID:              oldAlertID,
		Type:            "disk",
		Level:           AlertLevelWarning,
		ResourceID:      oldResourceID,
		CanonicalSpecID: canonicalMetricSpecID(oldResourceID, "disk"),
		CanonicalKind:   string(alertspecs.AlertSpecKindMetricThreshold),
		CanonicalState:  oldAlertID,
		ResourceName:    "wireguard",
		Node:            "proxmox2",
		Instance:        "proxmox2",
		Message:         "wireguard disk at 92.5%",
		Value:           92.5,
		Threshold:       90,
		StartTime:       start,
		LastSeen:        start.Add(5 * time.Minute),
	}

	m.mu.Lock()
	m.activeAlerts[oldAlertID] = alert
	m.historyManager.AddAlert(*alert)
	m.mu.Unlock()

	m.checkMetric(newResourceID, "pulse", "proxmox2", "proxmox2", "vm", "disk", 63.6, &HysteresisThreshold{Trigger: 90, Clear: 85}, nil)

	m.mu.RLock()
	preserved, oldExists := testLookupActiveAlert(t, m, oldAlertID)
	_, newExists := testLookupActiveAlert(t, m, newAlertID)
	m.mu.RUnlock()

	if !oldExists {
		t.Fatal("expected guest 107 disk alert to stay on guest 107")
	}
	if preserved.ResourceID != oldResourceID || preserved.ResourceName != "wireguard" {
		t.Fatalf("guest 107 alert was mutated: %#v", preserved)
	}
	if newExists {
		t.Fatal("did not expect a guest 108 disk alert to be created")
	}
	if resolved := m.GetResolvedAlert(newAlertID); resolved != nil {
		t.Fatalf("did not expect guest 108 resolved alert after evaluating guest 108 below threshold: %#v", resolved)
	}
}

func TestCheckGuestClearsPerDiskAlertsWhenGuestStops(t *testing.T) {
	m := newTestManager(t)
	configureUnifiedEvalManager(t, m, unifiedEvalBaseConfig())

	guestID := BuildGuestKey("pve1", "node1", 101)
	guest := models.VM{
		ID:       guestID,
		VMID:     101,
		Name:     "app01",
		Node:     "node1",
		Instance: "pve1",
		Status:   "running",
		CPU:      0.20,
		Memory:   models.Memory{Usage: 40},
		Disk:     models.Disk{Usage: 40},
		Disks: []models.Disk{
			{
				Mountpoint: "/",
				Device:     "scsi0",
				Usage:      95,
				Total:      100,
				Used:       95,
				Free:       5,
			},
		},
	}

	m.CheckGuest(guest, "pve1")

	alertID := canonicalMetricStateID(guestID+"-disk-scsi0", "disk")
	if activeAlert(t, m, alertID) == nil {
		t.Fatalf("expected guest disk alert %q", alertID)
	}

	guest.Status = "stopped"
	guest.Disks = nil
	m.CheckGuest(guest, "pve1")

	m.mu.RLock()
	_, exists := testLookupActiveAlert(t, m, alertID)
	m.mu.RUnlock()
	if exists {
		t.Fatalf("expected guest disk alert %q to clear when guest stops", alertID)
	}
}

func TestCheckNodeTemperatureAnnotatesCanonicalSpecMetadata(t *testing.T) {
	m := newTestManager(t)
	configureUnifiedEvalManager(t, m, unifiedEvalBaseConfig())

	m.CheckNode(models.Node{
		ID:       "node/pve-1",
		Name:     "pve-1",
		Instance: "pve-1",
		Status:   "online",
		CPU:      0.20,
		Memory:   models.Memory{Usage: 40},
		Disk:     models.Disk{Usage: 40},
		Temperature: &models.Temperature{
			Available:  true,
			CPUPackage: 90,
		},
	})

	m.mu.RLock()
	alertID := canonicalMetricStateID("node/pve-1", "temperature")
	alert := testRequireActiveAlert(t, m, alertID)
	m.mu.RUnlock()
	if alert == nil {
		t.Fatal("expected node temperature alert")
	}
	if got := alert.Metadata["canonicalAlertKind"]; got != string(alertspecs.AlertSpecKindMetricThreshold) {
		t.Fatalf("canonicalAlertKind = %v, want %s", got, alertspecs.AlertSpecKindMetricThreshold)
	}
	if got := alert.Metadata["canonicalSpecID"]; got != canonicalMetricSpecID("node/pve-1", "temperature") {
		t.Fatalf("canonicalSpecID = %v, want %s", got, canonicalMetricSpecID("node/pve-1", "temperature"))
	}
}

func TestCheckGuestPoweredOffAnnotatesCanonicalSpecMetadata(t *testing.T) {
	m := newTestManager(t)
	configureUnifiedEvalManager(t, m, unifiedEvalBaseConfig())

	resourceID := BuildGuestKey("pve1", "node1", 101)
	guest := models.VM{
		ID:       resourceID,
		VMID:     101,
		Name:     "app01",
		Node:     "node1",
		Instance: "pve1",
		Status:   "stopped",
	}

	m.CheckGuest(guest, "pve1")
	m.CheckGuest(guest, "pve1")

	alert := activeAlert(t, m, "guest-powered-off-"+resourceID)
	if got := alert.Metadata["canonicalAlertKind"]; got != "powered-state" {
		t.Fatalf("canonicalAlertKind = %v, want powered-state", got)
	}
	if got := alert.Metadata["canonicalSpecID"]; got != resourceID+"-powered-state" {
		t.Fatalf("canonicalSpecID = %v, want %s", got, resourceID+"-powered-state")
	}
}

func TestCheckNodeOfflineAnnotatesCanonicalSpecMetadata(t *testing.T) {
	m := newTestManager(t)
	configureUnifiedEvalManager(t, m, unifiedEvalBaseConfig())

	m.mu.Lock()
	m.mu.Unlock()

	// Confirmation runs live in the reducer core: drive the
	// preceding observations for real instead of seeding maps.
	target := models.Node{
		ID:               "node/pve-1",
		Name:             "pve-1",
		Instance:         "pve1",
		Status:           "offline",
		ConnectionHealth: "failed",
	}
	m.CheckNode(target)
	m.CheckNode(target)
	m.CheckNode(target)

	alert := activeAlert(t, m, buildCanonicalStateID("node/pve-1", "node/pve-1-connectivity"))
	if got := alert.Metadata["canonicalAlertKind"]; got != "connectivity" {
		t.Fatalf("canonicalAlertKind = %v, want connectivity", got)
	}
	if got := alert.Metadata["canonicalSpecID"]; got != "node/pve-1-connectivity" {
		t.Fatalf("canonicalSpecID = %v, want %s", got, "node/pve-1-connectivity")
	}
}

func TestCheckPBSOfflineAnnotatesCanonicalSpecMetadata(t *testing.T) {
	m := newTestManager(t)
	configureUnifiedEvalManager(t, m, unifiedEvalBaseConfig())

	m.mu.Lock()
	m.mu.Unlock()

	// Confirmation runs live in the reducer core: drive the
	// preceding observations for real instead of seeding maps.
	target := models.PBSInstance{
		ID:               "pbs-1",
		Name:             "pbs-main",
		Host:             "pbs-host",
		Status:           "online",
		ConnectionHealth: "unhealthy",
	}
	m.CheckPBS(target)
	m.CheckPBS(target)
	m.CheckPBS(target)

	alert := activeAlert(t, m, "pbs-offline-pbs-1")
	if got := alert.Metadata["canonicalAlertKind"]; got != "connectivity" {
		t.Fatalf("canonicalAlertKind = %v, want connectivity", got)
	}
	if got := alert.Metadata["canonicalSpecID"]; got != "pbs-1-connectivity" {
		t.Fatalf("canonicalSpecID = %v, want %s", got, "pbs-1-connectivity")
	}
}

func TestCheckStorageOfflineAnnotatesCanonicalSpecMetadata(t *testing.T) {
	m := newTestManager(t)
	configureUnifiedEvalManager(t, m, unifiedEvalBaseConfig())

	m.mu.Lock()
	m.mu.Unlock()

	// Confirmation runs live in the reducer core: drive the
	// preceding observations for real instead of seeding maps.
	target := models.Storage{
		ID:       "storage-1",
		Name:     "local-lvm",
		Node:     "pve-1",
		Instance: "pve1",
		Status:   "unavailable",
	}
	m.CheckStorage(target)
	m.CheckStorage(target)

	alert := activeAlert(t, m, "storage-offline-storage-1")
	if got := alert.Metadata["canonicalAlertKind"]; got != "connectivity" {
		t.Fatalf("canonicalAlertKind = %v, want connectivity", got)
	}
	if got := alert.Metadata["canonicalSpecID"]; got != "storage-1-connectivity" {
		t.Fatalf("canonicalSpecID = %v, want %s", got, "storage-1-connectivity")
	}
}

func TestCheckPMGOfflineAnnotatesCanonicalSpecMetadata(t *testing.T) {
	m := newTestManager(t)
	configureUnifiedEvalManager(t, m, unifiedEvalBaseConfig())

	m.mu.Lock()
	m.mu.Unlock()

	// Confirmation runs live in the reducer core: drive the
	// preceding observations for real instead of seeding maps.
	target := models.PMGInstance{
		ID:               "pmg-1",
		Name:             "pmg-main",
		Host:             "pmg-host",
		Status:           "online",
		ConnectionHealth: "unhealthy",
	}
	m.CheckPMG(target)
	m.CheckPMG(target)
	m.CheckPMG(target)

	alert := activeAlert(t, m, "pmg-offline-pmg-1")
	if got := alert.Metadata["canonicalAlertKind"]; got != "connectivity" {
		t.Fatalf("canonicalAlertKind = %v, want connectivity", got)
	}
	if got := alert.Metadata["canonicalSpecID"]; got != "pmg-1-connectivity" {
		t.Fatalf("canonicalSpecID = %v, want %s", got, "pmg-1-connectivity")
	}
}

func TestHandleDockerHostOfflineAnnotatesCanonicalSpecMetadata(t *testing.T) {
	m := newTestManager(t)
	configureUnifiedEvalManager(t, m, unifiedEvalBaseConfig())

	m.mu.Lock()
	m.mu.Unlock()

	// Confirmation runs live in the reducer core: drive the
	// preceding observations for real instead of seeding maps.
	target := models.DockerHost{
		ID:          "docker1",
		DisplayName: "Docker Host 1",
		Hostname:    "docker.local",
		AgentID:     "agent-123",
	}
	m.HandleDockerHostOffline(target)
	m.HandleDockerHostOffline(target)
	m.HandleDockerHostOffline(target)

	alert := activeAlert(t, m, "docker-host-offline-docker1")
	if got := alert.Metadata["canonicalAlertKind"]; got != "connectivity" {
		t.Fatalf("canonicalAlertKind = %v, want connectivity", got)
	}
	if got := alert.Metadata["canonicalSpecID"]; got != "docker:docker1-connectivity" {
		t.Fatalf("canonicalSpecID = %v, want %s", got, "docker:docker1-connectivity")
	}
}

func TestCheckDockerContainerStateAnnotatesCanonicalSpecMetadata(t *testing.T) {
	m := newTestManager(t)
	configureUnifiedEvalManager(t, m, unifiedEvalBaseConfig())

	host := models.DockerHost{
		ID:          "host-1",
		DisplayName: "Docker Host",
		Hostname:    "docker.local",
		Containers: []models.DockerContainer{
			{
				ID:     "container-1",
				Name:   "web",
				State:  "exited",
				Status: "Exited (1) seconds ago",
				Labels: map[string]string{"customer": "alpha", "critical": ""},
			},
		},
	}

	m.CheckDockerHost(host)
	m.CheckDockerHost(host)

	resourceID := DockerContainerResourceID(host.ID, "container-1", "")
	alert := activeAlert(t, m, "docker-container-state-"+resourceID)
	if got := alert.Metadata["canonicalAlertKind"]; got != "discrete-state" {
		t.Fatalf("canonicalAlertKind = %v, want discrete-state", got)
	}
	if got := alert.Metadata["canonicalSpecID"]; got != resourceID+"-runtime-state" {
		t.Fatalf("canonicalSpecID = %v, want %s", got, resourceID+"-runtime-state")
	}
	tags, ok := alert.Metadata["tags"].([]string)
	if !ok || len(tags) != 2 || tags[0] != "critical" || tags[1] != "customer:alpha" {
		t.Fatalf("container routing tags = %#v", alert.Metadata["tags"])
	}
}

func TestCheckDockerServiceAnnotatesCanonicalSpecMetadata(t *testing.T) {
	m := newTestManager(t)
	configureUnifiedEvalManager(t, m, unifiedEvalBaseConfig())

	host := models.DockerHost{
		ID:          "host-1",
		DisplayName: "Prod Swarm",
		Hostname:    "swarm-prod",
		Services: []models.DockerService{
			{
				ID:           "svc-1",
				Name:         "web",
				DesiredTasks: 4,
				RunningTasks: 2,
				Mode:         "replicated",
				Labels:       map[string]string{"customer": "alpha", "critical": ""},
			},
		},
	}

	m.CheckDockerHost(host)

	resourceID := DockerServiceResourceID(host.ID, "svc-1", "web")
	alert := activeAlert(t, m, "docker-service-health-"+resourceID)
	if got := alert.Metadata["canonicalAlertKind"]; got != "service-gap" {
		t.Fatalf("canonicalAlertKind = %v, want service-gap", got)
	}
	if got := alert.Metadata["canonicalSpecID"]; got != resourceID+"-service-gap" {
		t.Fatalf("canonicalSpecID = %v, want %s", got, resourceID+"-service-gap")
	}
	tags, ok := alert.Metadata["tags"].([]string)
	if !ok || len(tags) != 2 || tags[0] != "critical" || tags[1] != "customer:alpha" {
		t.Fatalf("service routing tags = %#v", alert.Metadata["tags"])
	}
}

func TestCheckDockerServiceUpdateStateAnnotatesCanonicalSpecMetadata(t *testing.T) {
	m := newTestManager(t)
	configureUnifiedEvalManager(t, m, unifiedEvalBaseConfig())

	now := time.Now()
	host := models.DockerHost{
		ID:          "host-update",
		DisplayName: "Swarm",
		Hostname:    "swarm.local",
		Services: []models.DockerService{
			{
				ID:           "svc-update",
				Name:         "api",
				DesiredTasks: 1,
				RunningTasks: 1,
				UpdateStatus: &models.DockerServiceUpdate{
					State:       "rollback_failed",
					Message:     "Rollback failed",
					CompletedAt: &now,
				},
			},
		},
	}

	m.CheckDockerHost(host)

	resourceID := DockerServiceResourceID(host.ID, "svc-update", "api")
	alert := activeAlert(t, m, "docker-service-health-"+resourceID)
	if got := alert.Metadata["canonicalAlertKind"]; got != "discrete-state" {
		t.Fatalf("canonicalAlertKind = %v, want discrete-state", got)
	}
	if got := alert.Metadata["canonicalSpecID"]; got != resourceID+"-update-state" {
		t.Fatalf("canonicalSpecID = %v, want %s", got, resourceID+"-update-state")
	}
}

func TestCheckPMGQueueDepthAnnotatesCanonicalSpecMetadata(t *testing.T) {
	m := newTestManager(t)

	pmg := models.PMGInstance{
		ID:   "pmg-1",
		Name: "PMG 1",
		Nodes: []models.PMGNodeStatus{
			{Name: "node1", QueueStatus: &models.PMGQueueStatus{Total: 300}},
			{Name: "node2", QueueStatus: &models.PMGQueueStatus{Total: 250}},
		},
	}

	m.checkPMGQueueDepths(pmg, PMGThresholdConfig{
		QueueTotalWarning:  500,
		QueueTotalCritical: 1000,
	})

	alert := activeAlert(t, m, "pmg-1-queue-total")
	if got := alert.Metadata["canonicalAlertKind"]; got != "severity-threshold" {
		t.Fatalf("canonicalAlertKind = %v, want severity-threshold", got)
	}
	if got := alert.Metadata["canonicalSpecID"]; got != "pmg-1-queue-total" {
		t.Fatalf("canonicalSpecID = %v, want pmg-1-queue-total", got)
	}
}

func TestCheckPMGOldestMessageAnnotatesCanonicalSpecMetadata(t *testing.T) {
	m := newTestManager(t)

	pmg := models.PMGInstance{
		ID:   "pmg-1",
		Name: "PMG 1",
		Nodes: []models.PMGNodeStatus{
			{Name: "node1", QueueStatus: &models.PMGQueueStatus{OldestAge: 2400}},
		},
	}

	m.checkPMGOldestMessage(pmg, PMGThresholdConfig{
		OldestMessageWarnMins: 30,
		OldestMessageCritMins: 60,
	})

	alert := activeAlert(t, m, "pmg-1-oldest-message")
	if got := alert.Metadata["canonicalAlertKind"]; got != "severity-threshold" {
		t.Fatalf("canonicalAlertKind = %v, want severity-threshold", got)
	}
	if got := alert.Metadata["canonicalSpecID"]; got != "pmg-1-oldest-message" {
		t.Fatalf("canonicalSpecID = %v, want pmg-1-oldest-message", got)
	}
	if got := alert.Metadata["resourceType"]; got != string(unifiedresources.ResourceTypePMG) {
		t.Fatalf("resourceType = %v, want pmg", got)
	}
}

func TestCheckPMGNodeQueueAnnotatesCanonicalSpecMetadata(t *testing.T) {
	m := newTestManager(t)

	pmg := models.PMGInstance{
		ID:   "pmg-1",
		Name: "PMG 1",
		Nodes: []models.PMGNodeStatus{
			{Name: "node-a", QueueStatus: &models.PMGQueueStatus{Total: 80}},
		},
	}

	m.checkPMGNodeQueues(pmg, PMGThresholdConfig{
		QueueTotalWarning:  100,
		QueueTotalCritical: 200,
	})

	alert := activeAlert(t, m, "pmg-1-node-a-queue-total")
	if got := alert.Metadata["canonicalAlertKind"]; got != "severity-threshold" {
		t.Fatalf("canonicalAlertKind = %v, want severity-threshold", got)
	}
	if got := alert.Metadata["canonicalSpecID"]; got != "pmg-1-node-a-queue-total" {
		t.Fatalf("canonicalSpecID = %v, want pmg-1-node-a-queue-total", got)
	}
}

func TestCheckPMGQuarantineAnnotatesCanonicalSpecMetadata(t *testing.T) {
	m := newTestManager(t)

	pmg := models.PMGInstance{
		ID:   "pmg-1",
		Name: "PMG 1",
		Quarantine: &models.PMGQuarantineTotals{
			Spam: 2500,
		},
	}

	m.checkPMGQuarantineBacklog(pmg, PMGThresholdConfig{
		QuarantineSpamWarn:     2000,
		QuarantineSpamCritical: 5000,
	})

	alert := activeAlert(t, m, "pmg-1-quarantine-spam")
	if got := alert.Metadata["canonicalAlertKind"]; got != "change-threshold" {
		t.Fatalf("canonicalAlertKind = %v, want change-threshold", got)
	}
	if got := alert.Metadata["canonicalSpecID"]; got != "pmg-1-quarantine-spam" {
		t.Fatalf("canonicalSpecID = %v, want pmg-1-quarantine-spam", got)
	}
}

func TestCheckPMGAnomalyAnnotatesCanonicalSpecMetadata(t *testing.T) {
	m := newTestManager(t)
	base := time.Now().Add(-13 * time.Hour)

	tracker := &pmgAnomalyTracker{
		Samples:        make([]pmgMailMetricSample, 0, 12),
		LastSampleTime: base.Add(11 * time.Hour),
		SampleCount:    12,
	}
	for i := 0; i < 12; i++ {
		tracker.Samples = append(tracker.Samples, pmgMailMetricSample{
			SpamIn:    100,
			SpamOut:   10,
			VirusIn:   1,
			VirusOut:  1,
			Timestamp: base.Add(time.Duration(i) * time.Hour),
		})
	}

	m.mu.Lock()
	m.pmgAnomalyTrackers["pmg-1"] = tracker
	m.mu.Unlock()

	pmg := models.PMGInstance{
		ID:   "pmg-1",
		Name: "PMG 1",
		MailCount: []models.PMGMailCountPoint{
			{Timestamp: base.Add(12 * time.Hour), SpamIn: 420},
		},
	}
	m.checkPMGAnomalies(pmg, PMGThresholdConfig{})

	pmg.MailCount = []models.PMGMailCountPoint{
		{Timestamp: base.Add(13 * time.Hour), SpamIn: 430},
	}
	m.checkPMGAnomalies(pmg, PMGThresholdConfig{})

	alert := activeAlert(t, m, "pmg-1-anomaly-spamIn")
	if got := alert.Metadata["canonicalAlertKind"]; got != "baseline-anomaly" {
		t.Fatalf("canonicalAlertKind = %v, want baseline-anomaly", got)
	}
	if got := alert.Metadata["canonicalSpecID"]; got != "pmg-1-anomaly-spamIn" {
		t.Fatalf("canonicalSpecID = %v, want pmg-1-anomaly-spamIn", got)
	}
}

func TestUnifiedResourceInputExcludesDockerAppContainers(t *testing.T) {
	// Docker containers are evaluated by CheckDockerHost against the
	// docker:{host}/{containerName} override key (#1601); unified eval must
	// not double-evaluate them under a second identity.
	resource := unifiedresources.Resource{
		ID:   "app-container-1234567890abcdef",
		Type: unifiedresources.ResourceTypeAppContainer,
		Name: "media-server",
		Docker: &unifiedresources.DockerData{
			HostSourceID: "host-1",
			ContainerID:  "aaaaaaaaaaaa",
		},
	}
	if input, ok := UnifiedResourceInputFromResource(resource); ok {
		t.Fatalf("docker app-containers must stay owned by CheckDockerHost, got unified input %+v", input)
	}

	// TrueNAS apps remain unified-eval owned.
	resource.Docker = nil
	resource.TrueNAS = &unifiedresources.TrueNASData{}
	input, ok := UnifiedResourceInputFromResource(resource)
	if !ok || input.Type != "truenas-app" {
		t.Fatalf("truenas app containers must stay unified-eval owned, got %+v ok=%v", input, ok)
	}
}

func TestUnifiedStorageForecastKeepsCanonicalMetricSpecAcrossPlatforms(t *testing.T) {
	m := newTestManager(t)
	configureUnifiedEvalManager(t, m, unifiedEvalBaseConfig())
	input := &UnifiedResourceInput{
		ID:   "vmware:lab/datastore:archive",
		Type: "vmware-datastore",
		Name: "archive",
		Disk: &UnifiedResourceMetric{Percent: 70},
	}
	trend := CapacityTrendObservation{
		Ready: true, Reason: "increasing", ObservedAt: time.Now(), DailyChange: 7,
		Confidence: 0.99, SampleCount: 300, BucketCount: 48, CoverageSpan: 48 * time.Hour,
	}
	m.CheckUnifiedResourceWithCapacityTrend(input, trend)
	m.CheckUnifiedResourceWithCapacityTrend(input, trend)

	alert := testRequireActiveAlert(t, m, canonicalMetricStateID(input.ID, "usage"))
	if alert.CanonicalSpecID != canonicalMetricSpecID(input.ID, "usage") {
		t.Fatalf("CanonicalSpecID = %q, want canonical usage spec", alert.CanonicalSpecID)
	}
}

func TestUnifiedMetricNoisyWarningWaitsButCriticalFiresImmediately(t *testing.T) {
	m := newTestManager(t)
	input := &UnifiedResourceInput{
		ID:     "vm-noisy-memory",
		Type:   "vm",
		Name:   "database",
		Memory: &UnifiedResourceMetric{Percent: 90},
	}
	alertID := canonicalMetricStateID(input.ID, "memory")

	m.CheckUnifiedResource(input)
	assertAlertMissing(t, m, alertID)

	input.Memory.Percent = 96
	m.CheckUnifiedResource(input)
	alert := testRequireActiveAlert(t, m, alertID)
	if alert.Level != AlertLevelCritical {
		t.Fatalf("critical memory alert level = %q, want %q", alert.Level, AlertLevelCritical)
	}
	if got := alert.Metadata["stabilityWindowSeconds"]; got != defaultNoisyGaugeStabilitySeconds {
		t.Fatalf("stabilityWindowSeconds = %v, want %d", got, defaultNoisyGaugeStabilitySeconds)
	}
}

func TestNodeUsageDispatchPreservesSeparatelyEvaluatedTemperature(t *testing.T) {
	m, elapsed := continuityManager(t, false)
	m.mu.Lock()
	m.config.NodeDefaults.Temperature = &HysteresisThreshold{Trigger: 80, Clear: 70}
	m.mu.Unlock()
	node, _ := testNodeWithHostAgent()
	node.Temperature = &models.Temperature{Available: true, CPUPackage: 85}
	m.CheckNode(node)
	elapsed.Store(int64(40 * time.Second))
	m.CheckNode(node)
	incident, ok := continuityIncident(m, node.ID, "temperature")
	if !ok || incident.State != reducer.StatePending {
		t.Fatal("temperature control did not remain pending")
	}
	originalStart := incident.PendingSince
	elapsed.Store(int64(59 * time.Second))
	m.CheckNode(node)
	incident, _ = continuityIncident(m, node.ID, "temperature")
	if !incident.PendingSince.Equal(originalStart) {
		t.Fatal("usage dispatch interrupted a separately observed temperature")
	}
	elapsed.Store(int64(time.Minute))
	m.CheckNode(node)
	if !testHasActiveAlert(t, m, canonicalMetricStateID(node.ID, "temperature")) {
		t.Fatal("continuous node temperature did not complete its delay")
	}
	node.Temperature = nil
	elapsed.Store(int64(2 * time.Minute))
	m.CheckNode(node)
	if !testHasActiveAlert(t, m, canonicalMetricStateID(node.ID, "temperature")) {
		t.Fatal("actual missing temperature became recovery")
	}
}

// metricStatusClock drives a manager's wall and monotonic clocks together so
// recovery runs measure exactly the time the test advances.
type metricStatusClock struct {
	now  time.Time
	tick time.Duration
}

func (c *metricStatusClock) advance(d time.Duration) {
	c.now = c.now.Add(d)
	c.tick += d
}

func newMetricStatusTestManager(t *testing.T) (*Manager, *metricStatusClock) {
	t.Helper()
	m := newTestManager(t)
	// Recently-resolved retention prunes against the real clock, so start there.
	clock := &metricStatusClock{now: time.Now().UTC()}
	m.now = func() time.Time { return clock.now }
	m.intentClock = func() time.Duration { return clock.tick }
	m.mu.Lock()
	m.config.NodeDefaults.Temperature = &HysteresisThreshold{Trigger: 80, Clear: 75}
	m.mu.Unlock()
	return m, clock
}

// fireNodeTemperatureAlert holds a breaching reading until the factory
// stability delay lets the alert open, as a real poll loop would.
func fireNodeTemperatureAlert(t *testing.T, m *Manager, clock *metricStatusClock, celsius float64) *Alert {
	t.Helper()
	for attempt := 0; attempt < 40; attempt++ {
		checkMetricStatusNode(m, celsius)
		if alert := activeNodeTemperatureAlert(t, m); alert != nil {
			return alert
		}
		clock.advance(30 * time.Second)
	}
	t.Fatalf("%v°C never opened the temperature alert", celsius)
	return nil
}

func checkMetricStatusNode(m *Manager, celsius float64) {
	m.CheckNode(models.Node{
		ID:       "homelab-minipc",
		Name:     "minipc",
		Instance: "homelab",
		Status:   "online",
		CPU:      0.2,
		Memory:   models.Memory{Usage: 40},
		Disk:     models.Disk{Usage: 40},
		Temperature: &models.Temperature{
			Available:  true,
			CPUPackage: celsius,
		},
	})
}

func activeNodeTemperatureAlert(t *testing.T, m *Manager) *Alert {
	t.Helper()
	m.mu.RLock()
	defer m.mu.RUnlock()
	alert, ok := m.getActiveAlertNoLock(canonicalMetricStateID("homelab-minipc", "temperature"))
	if !ok {
		return nil
	}
	return alert.Clone()
}

func TestOpenTemperatureAlertReportsLiveReadingWhileItHolds(t *testing.T) {
	m, clock := newMetricStatusTestManager(t)
	// Buffered so a dispatch never blocks the evaluator; len() counts sends.
	notified := make(chan struct{}, 64)
	m.SetAlertCallback(func(*Alert) { notified <- struct{}{} })

	fired := fireNodeTemperatureAlert(t, m, clock, 85)
	status := fired.MetricStatus
	if status == nil || status.Phase != models.MetricAlertPhaseBreaching || status.Value != 85 ||
		status.Trigger != 80 || status.Recovery != 75 || status.Unit != "°C" ||
		!metricBreachTime(t, status).Equal(fired.LastSeen) {
		t.Fatalf("breaching status = %+v", status)
	}
	if status.RecoveryDelaySeconds <= 0 {
		t.Fatalf("temperature alerts carry a recovery delay, got %+v", status)
	}
	delay := time.Duration(status.RecoveryDelaySeconds) * time.Second
	firedMessage, firedLastSeen, notifications := fired.Message, fired.LastSeen, len(notified)

	// Between the clear and trigger levels the alert holds. The legacy
	// snapshot keeps the breach; only the live status moves.
	clock.advance(30 * time.Second)
	checkMetricStatusNode(m, 78)
	held := activeNodeTemperatureAlert(t, m)
	if held == nil {
		t.Fatal("78°C cleared an alert that should hold until 75°C")
	}
	if held.Value != 85 || held.Message != firedMessage || !held.LastSeen.Equal(firedLastSeen) {
		t.Fatalf("hold rewrote the breach snapshot: value=%v message=%q lastSeen=%v", held.Value, held.Message, held.LastSeen)
	}
	if got := held.MetricStatus; got == nil || got.Phase != models.MetricAlertPhaseLatched || got.Value != 78 ||
		!got.ObservedAt.Equal(clock.now) || got.RecoveryStartedAt != nil ||
		!metricBreachTime(t, got).Equal(firedLastSeen) {
		t.Fatalf("latched status = %+v", got)
	}

	// At or below the recovery level the recovery run starts.
	clock.advance(30 * time.Second)
	recoveryStart := clock.now
	checkMetricStatusNode(m, 72)
	if got := activeNodeTemperatureAlert(t, m).MetricStatus; got == nil ||
		got.Phase != models.MetricAlertPhaseRecovering || got.Value != 72 ||
		got.RecoveryStartedAt == nil || !got.RecoveryStartedAt.Equal(recoveryStart) || got.RecoveryElapsedSeconds != 0 ||
		!metricBreachTime(t, got).Equal(firedLastSeen) {
		t.Fatalf("recovery start status = %+v", got)
	}
	clock.advance(delay / 2)
	checkMetricStatusNode(m, 75)
	if got := activeNodeTemperatureAlert(t, m).MetricStatus; got == nil ||
		got.Phase != models.MetricAlertPhaseRecovering || got.RecoveryElapsedSeconds != int(delay/2/time.Second) ||
		!metricBreachTime(t, got).Equal(firedLastSeen) {
		t.Fatalf("recovery progress status = %+v", got)
	}

	// A reading back above the recovery level resets the run.
	clock.advance(30 * time.Second)
	checkMetricStatusNode(m, 77)
	if got := activeNodeTemperatureAlert(t, m).MetricStatus; got == nil ||
		got.Phase != models.MetricAlertPhaseLatched || got.RecoveryStartedAt != nil || got.RecoveryElapsedSeconds != 0 ||
		!metricBreachTime(t, got).Equal(firedLastSeen) {
		t.Fatalf("status after recovery reset = %+v", got)
	}
	if got := len(notified); got != notifications {
		t.Fatalf("holding and recovering sent %d notifications", got-notifications)
	}

	// A full recovery delay at or below 75°C resolves the alert.
	checkMetricStatusNode(m, 72)
	clock.advance(delay)
	checkMetricStatusNode(m, 72)
	if alert := activeNodeTemperatureAlert(t, m); alert != nil {
		t.Fatalf("alert still open after a full recovery delay: %+v", alert.MetricStatus)
	}
	m.resolvedMutex.RLock()
	defer m.resolvedMutex.RUnlock()
	resolved := 0
	for _, entry := range m.recentlyResolved {
		if entry == nil || entry.Type != "temperature" {
			continue
		}
		resolved++
		if entry.MetricStatus != nil {
			t.Fatalf("resolved record kept the live status of an open alert: %+v", entry.MetricStatus)
		}
	}
	if resolved != 1 {
		t.Fatalf("resolved temperature records = %d, want 1", resolved)
	}
}

func TestBuildMetricAlertStatusCarriesWindowAndMonotonicRecovery(t *testing.T) {
	observedAt := time.Date(2026, 10, 6, 10, 49, 0, 0, time.UTC)
	status := buildMetricAlertStatus(metricStatusInput{
		value:                74,
		trigger:              80,
		clear:                75,
		recoveryDelaySeconds: 300,
		unit:                 "%",
		window:               metricWindowObservation{Value: 74, CurrentValue: 91, WindowSeconds: 600, Ready: true},
		incident: reducer.Incident{
			RecoverySince:         observedAt.Add(-10 * time.Minute),
			RecoveryElapsed:       90 * time.Second,
			RecoveryTicksSupplied: true,
		},
		observedAt: observedAt,
	})
	if status.Phase != models.MetricAlertPhaseRecovering || status.Value != 74 ||
		status.RawValue == nil || *status.RawValue != 91 || status.EvaluationWindowSeconds != 600 {
		t.Fatalf("windowed status = %+v", status)
	}
	// Elapsed time comes from the monotonic run, not wall-clock arithmetic.
	if status.RecoveryElapsedSeconds != 90 {
		t.Fatalf("recovery elapsed = %d, want 90", status.RecoveryElapsedSeconds)
	}

	// The reducer clears at the trigger when no clear level is configured.
	noClear := buildMetricAlertStatus(metricStatusInput{value: 85, triggered: true, trigger: 80, observedAt: observedAt})
	if noClear.Recovery != 80 || noClear.RawValue != nil {
		t.Fatalf("status without clear level = %+v", noClear)
	}
}

func trueNASTemperatureDisk(device, diskType string, celsius int) unifiedresources.Resource {
	return unifiedresources.Resource{
		ID:         "physical-disk:truenas-main/" + device,
		Type:       unifiedresources.ResourceTypePhysicalDisk,
		Name:       device,
		ParentName: "truenas-main",
		Sources:    []unifiedresources.DataSource{unifiedresources.SourceTrueNAS},
		PhysicalDisk: &unifiedresources.PhysicalDiskMeta{
			DevPath:     "/dev/" + device,
			DiskType:    diskType,
			Temperature: celsius,
		},
	}
}

func trueNASDiskTemperatureAlertID(resource unifiedresources.Resource) string {
	return canonicalMetricStateID(resource.ID, "temperature")
}

func checkTrueNASTemperatureDisk(t *testing.T, m *Manager, resource unifiedresources.Resource) {
	t.Helper()
	input, ok := UnifiedResourceInputFromResource(resource)
	if !ok {
		t.Fatalf("TrueNAS disk %s did not become alert input", resource.ID)
	}
	m.CheckUnifiedResource(input)
}

func trueNASDiskTemperatureAlert(t *testing.T, m *Manager, resource unifiedresources.Resource) (*Alert, bool) {
	t.Helper()
	return testLookupActiveAlert(t, m, trueNASDiskTemperatureAlertID(resource))
}

// TestTrueNASDiskTemperatureAlertsFollowDiskTemperaturePolicy pins TrueNAS
// disk temperature alerts to the disk temperature policy that the TrueNAS
// storage table, Physical Disks and Patrol judge heat by: per disk type,
// unless the user saved a TrueNAS-wide value, with a per-disk override first.
func TestTrueNASDiskTemperatureAlertsFollowDiskTemperaturePolicy(t *testing.T) {
	nvme63 := trueNASTemperatureDisk("nvme0n1", "nvme", 63)
	nvme72 := trueNASTemperatureDisk("nvme1n1", "nvme", 72)
	sata56 := trueNASTemperatureDisk("sda", "sata", 56)
	sata60 := trueNASTemperatureDisk("sdb", "sata", 60)

	tests := []struct {
		name    string
		mutate  func(*AlertConfig)
		disks   []unifiedresources.Resource
		firing  []unifiedresources.Resource
		trigger map[string]float64
	}{
		{
			name:    "factory thresholds judge each disk by its type",
			disks:   []unifiedresources.Resource{nvme63, nvme72, sata56},
			firing:  []unifiedresources.Resource{nvme72, sata56},
			trigger: map[string]float64{nvme72.ID: 70, sata56.ID: 55},
		},
		{
			name: "a raised NVMe trigger moves NVMe alerts with it",
			mutate: func(cfg *AlertConfig) {
				cfg.DiskTempByType = map[string]HysteresisThreshold{"nvme": {Trigger: 75, Clear: 70}}
			},
			disks:  []unifiedresources.Resource{nvme72, sata56},
			firing: []unifiedresources.Resource{sata56},
		},
		{
			name: "a saved TrueNAS-wide value applies to every TrueNAS disk",
			mutate: func(cfg *AlertConfig) {
				cfg.TrueNASDiskDefaults.Temperature = &HysteresisThreshold{Trigger: 62, Clear: 57}
			},
			disks:   []unifiedresources.Resource{nvme63, sata60},
			firing:  []unifiedresources.Resource{nvme63},
			trigger: map[string]float64{nvme63.ID: 62},
		},
		{
			name: "a per-disk override beats the TrueNAS-wide value",
			mutate: func(cfg *AlertConfig) {
				cfg.TrueNASDiskDefaults.Temperature = &HysteresisThreshold{Trigger: 62, Clear: 57}
				cfg.Overrides = map[string]ThresholdConfig{
					sata56.ID: {Temperature: &HysteresisThreshold{Trigger: 50, Clear: 45}},
				}
			},
			disks:   []unifiedresources.Resource{sata56},
			firing:  []unifiedresources.Resource{sata56},
			trigger: map[string]float64{sata56.ID: 50},
		},
		{
			name: "a switched-off agent Disk Temp default switches TrueNAS disks off",
			mutate: func(cfg *AlertConfig) {
				cfg.AgentDefaults.DiskTemperature = &HysteresisThreshold{Trigger: 0, Clear: 0}
			},
			disks: []unifiedresources.Resource{nvme72, sata60},
		},
		{
			name: "a saved TrueNAS-wide value still applies when the agent default is off",
			mutate: func(cfg *AlertConfig) {
				cfg.AgentDefaults.DiskTemperature = &HysteresisThreshold{Trigger: 0, Clear: 0}
				cfg.TrueNASDiskDefaults.Temperature = &HysteresisThreshold{Trigger: 58, Clear: 53}
			},
			disks:  []unifiedresources.Resource{sata60},
			firing: []unifiedresources.Resource{sata60},
		},
		{
			name: "a saved TrueNAS-wide off switches TrueNAS disks off",
			mutate: func(cfg *AlertConfig) {
				cfg.TrueNASDiskDefaults.Temperature = &HysteresisThreshold{Trigger: 0, Clear: 0}
			},
			disks: []unifiedresources.Resource{nvme72, sata60},
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			m := newTestManager(t)
			cfg := unifiedEvalBaseConfig()
			cfg.TrueNASDiskDefaults = ThresholdConfig{}
			if tt.mutate != nil {
				tt.mutate(&cfg)
			}
			configureUnifiedEvalManager(t, m, cfg)

			for _, disk := range tt.disks {
				checkTrueNASTemperatureDisk(t, m, disk)
			}

			want := make(map[string]bool, len(tt.firing))
			for _, disk := range tt.firing {
				want[disk.ID] = true
			}
			for _, disk := range tt.disks {
				alert, firing := trueNASDiskTemperatureAlert(t, m, disk)
				if firing != want[disk.ID] {
					t.Fatalf("%s firing = %v, want %v (active: %v)", disk.ID, firing, want[disk.ID], alertKeys(m))
				}
				if !firing {
					continue
				}
				if got := alert.Metadata["diskType"]; got != disk.PhysicalDisk.DiskType {
					t.Fatalf("%s diskType metadata = %v, want %q", disk.ID, got, disk.PhysicalDisk.DiskType)
				}
				if trigger, ok := tt.trigger[disk.ID]; ok && alert.Threshold != trigger {
					t.Fatalf("%s threshold = %v, want %v", disk.ID, alert.Threshold, trigger)
				}
			}
		})
	}
}

// A last-known reading the source no longer collects is not evidence of heat.
func TestTrueNASDiskTemperatureAlertIgnoresRetainedReading(t *testing.T) {
	m := newTestManager(t)
	cfg := unifiedEvalBaseConfig()
	cfg.TrueNASDiskDefaults = ThresholdConfig{}
	configureUnifiedEvalManager(t, m, cfg)

	disk := trueNASTemperatureDisk("sda", "sata", 80)
	disk.PhysicalDisk.Collection = &diskinventory.CollectionStatus{
		Temperature: diskinventory.Unavailable("truenas", "disk is in standby"),
	}
	checkTrueNASTemperatureDisk(t, m, disk)
	if _, firing := trueNASDiskTemperatureAlert(t, m, disk); firing {
		t.Fatalf("retained 80C reading raised an alert: %v", alertKeys(m))
	}
}