Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

23.1 KB
Raw Blame
package ai

import (
	"errors"
	"strings"
	"testing"
	"time"

	"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
	"github.com/rcourtman/pulse-go-rewrite/internal/models"
	"github.com/rcourtman/pulse-go-rewrite/internal/truenas"
	"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
	"github.com/rcourtman/pulse-go-rewrite/pkg/aicontracts"
	"github.com/rcourtman/pulse-go-rewrite/pkg/diskinventory"
)

// raisedNVMeThresholds is a user who raised the NVMe disk temperature trigger
// to 75C (clear 70) in Alerts and left every other type at its default.
func raisedNVMeThresholds() *mockThresholdProvider {
	return &mockThresholdProvider{diskTemperature: map[string][2]float64{"nvme": {75, 70}}}
}

func TestDiskTemperatureLimitsFollowTheAlertPolicy(t *testing.T) {
	for _, tc := range []struct {
		name     string
		provider ThresholdProvider
		diskType string
		want     diskTemperatureLimits
	}{
		{name: "factory nvme", diskType: "nvme", want: diskTemperatureLimits{trigger: 70, clear: 65}},
		{name: "factory sata", diskType: "sata", want: diskTemperatureLimits{trigger: 55, clear: 50}},
		{name: "factory untyped", diskType: "", want: diskTemperatureLimits{trigger: 55, clear: 50}},
		{name: "raised nvme", provider: raisedNVMeThresholds(), diskType: "nvme", want: diskTemperatureLimits{trigger: 75, clear: 70}},
		{name: "raised nvme leaves sata", provider: raisedNVMeThresholds(), diskType: "sata", want: diskTemperatureLimits{trigger: 55, clear: 50}},
		{name: "alerting off", provider: &mockThresholdProvider{diskTemperature: map[string][2]float64{"sata": {0, 0}}}, diskType: "sata", want: diskTemperatureLimits{}},
		{name: "clear above trigger", provider: &mockThresholdProvider{diskTemperature: map[string][2]float64{"sas": {60, 64}}}, diskType: "sas", want: diskTemperatureLimits{trigger: 60, clear: 60}},
	} {
		if got := diskTemperatureLimitsFor(tc.provider, alerts.DiskTemperatureHost{}, tc.diskType); got != tc.want {
			t.Errorf("%s: limits = %+v, want %+v", tc.name, got, tc.want)
		}
	}

	off := diskTemperatureLimits{}
	if off.hot(95) || !off.cooled(95) {
		t.Fatal("with disk temperature alerting off no reading is heat")
	}
}

func diskTemperatureTriageState(provider ThresholdProvider) patrolRuntimeState {
	state := newPatrolRuntimeState(models.StateSnapshot{PhysicalDisks: []models.PhysicalDisk{
		{ID: "nvme-63", DevPath: "/dev/nvme0n1", Type: "nvme", Health: "PASSED", Wearout: 90, Temperature: 63},
		{ID: "nvme-66", DevPath: "/dev/nvme1n1", Type: "nvme", Health: "PASSED", Wearout: 90, Temperature: 66},
		{ID: "nvme-72", DevPath: "/dev/nvme2n1", Type: "nvme", Health: "PASSED", Wearout: 90, Temperature: 72},
		{ID: "sata-52", DevPath: "/dev/sda", Type: "sata", Health: "PASSED", Wearout: -1, Temperature: 52},
		{ID: "sata-56", DevPath: "/dev/sdb", Type: "sata", Health: "PASSED", Wearout: -1, Temperature: 56},
	}})
	state.thresholdProvider = provider
	return state
}

// Patrol judges disk heat by the alert disk temperature policy: a warning from
// the per-type trigger and nothing below it, the band under the trigger being
// a Temp cell colour rather than a finding. An NVMe at 63C is not hot; a SATA
// disk at 56C is.
func TestTriageDiskTemperatureFollowsTheAlertPolicy(t *testing.T) {
	temperatureFlag := func(flags []TriageFlag, id string) *TriageFlag {
		return triageFindFlag(flags, func(f TriageFlag) bool {
			return f.ResourceID == id && strings.Contains(f.Reason, "Disk temperature")
		})
	}
	expect := func(flags []TriageFlag, id, severity string, threshold float64) {
		t.Helper()
		flag := temperatureFlag(flags, id)
		if severity == "" {
			if flag != nil {
				t.Errorf("%s: unexpected temperature flag %+v", id, *flag)
			}
			return
		}
		if flag == nil || flag.Severity != severity || flag.Threshold != threshold {
			t.Errorf("%s: temperature flag = %+v, want %s at threshold %.0f", id, flag, severity, threshold)
		}
	}

	flags := triageDiskHealthChecksState(diskTemperatureTriageState(nil), nil)
	expect(flags, "nvme-63", "", 0)
	expect(flags, "nvme-66", "", 0)
	expect(flags, "nvme-72", "warning", 70)
	expect(flags, "sata-52", "", 0)
	expect(flags, "sata-56", "warning", 55)
	if flag := temperatureFlag(flags, "sata-56"); flag == nil || flag.Reason != "Disk temperature 56°C (alert threshold: 55°C)" {
		t.Errorf("sata-56 reason = %+v, want the reading and the alert trigger", flag)
	}

	flags = triageDiskHealthChecksState(diskTemperatureTriageState(raisedNVMeThresholds()), nil)
	expect(flags, "nvme-72", "", 0)
	expect(flags, "sata-56", "warning", 55)

	off := &mockThresholdProvider{diskTemperature: map[string][2]float64{"nvme": {0, 0}, "sata": {0, 0}}}
	if flags := triageDiskHealthChecksState(diskTemperatureTriageState(off), nil); len(flags) != 0 {
		t.Fatalf("disk temperature alerting off should raise no heat flags, got %+v", flags)
	}
}

// The seed's disk issue list and finding verification share the policy: a
// disk is an issue from the trigger and recovers below the clear value.
func TestPatrolDiskHeatIssueAndRecoveryFollowTheAlertPolicy(t *testing.T) {
	issues := map[string]bool{}
	for _, row := range patrolPhysicalDiskRows(diskTemperatureTriageState(nil), nil) {
		issues[row.id] = patrolPhysicalDiskHealthIssue(row)
	}
	for id, want := range map[string]bool{"nvme-63": false, "nvme-66": false, "nvme-72": true, "sata-52": false, "sata-56": true} {
		if issues[id] != want {
			t.Errorf("%s: health issue = %v, want %v", id, issues[id], want)
		}
	}
	for _, row := range patrolPhysicalDiskRows(diskTemperatureTriageState(raisedNVMeThresholds()), nil) {
		if row.id == "nvme-72" && patrolPhysicalDiskHealthIssue(row) {
			t.Error("nvme-72 under a raised 75C trigger should not be an issue")
		}
	}

	state := diskTemperatureTriageState(nil)
	for id, want := range map[string]bool{"nvme-63": true, "nvme-66": false, "nvme-72": false, "sata-52": false} {
		recovered, err := verifyMetricRecoveredState(state, PatrolThresholds{}, "disk-high", id, "physical_disk")
		if err != nil {
			t.Fatalf("%s: verification error %v", id, err)
		}
		if recovered != want {
			t.Errorf("%s: recovered = %v, want %v", id, recovered, want)
		}
	}
}

func TestResourceContextListsDisksHotByTheAlertPolicy(t *testing.T) {
	registry := unifiedresources.NewRegistry(nil)
	now := time.Now()
	registry.IngestSnapshot(models.StateSnapshot{PhysicalDisks: []models.PhysicalDisk{
		{ID: "pve-nvme", Node: "pve", Instance: "lab", DevPath: "/dev/nvme0n1", Model: "Samsung 980", Serial: "N63", Type: "nvme", Health: "PASSED", Wearout: 90, Temperature: 63, LastChecked: now},
		{ID: "pve-nvme-72", Node: "pve", Instance: "lab", DevPath: "/dev/nvme1n1", Model: "Samsung 990", Serial: "N72", Type: "nvme", Health: "PASSED", Wearout: 90, Temperature: 72, LastChecked: now},
		{ID: "pve-sata", Node: "pve", Instance: "lab", DevPath: "/dev/sda", Model: "WDC WD80EFAX", Serial: "S56", Type: "sata", Health: "PASSED", Wearout: -1, Temperature: 56, LastChecked: now},
	}})

	s := &Service{}
	s.SetUnifiedResourceProvider(unifiedresources.NewUnifiedAIAdapter(registry))
	attention := func() string {
		got := s.buildUnifiedResourceContext()
		_, section, found := strings.Cut(got, "Physical Disks Needing Attention")
		if !found {
			return ""
		}
		section, _, _ = strings.Cut(section, "\n\n")
		return section
	}

	// Disk names are redacted in this context, so the readings identify them.
	section := attention()
	if !strings.Contains(section, "Temp: 56C") || !strings.Contains(section, "Temp: 72C") || strings.Contains(section, "Temp: 63C") {
		t.Fatalf("factory policy: attention section = %q, want the SATA 56C and NVMe 72C disks only", section)
	}

	s.SetPatrolThresholdProvider(raisedNVMeThresholds())
	section = attention()
	if !strings.Contains(section, "Temp: 56C") || strings.Contains(section, "Temp: 72C") || strings.Contains(section, "Temp: 63C") {
		t.Fatalf("raised NVMe trigger: attention section = %q, want only the SATA 56C disk", section)
	}
}

// A Patrol run judges disk heat by the thresholds the user configured, through
// both ways a run builds its state.
func TestPatrolRunStateCarriesTheConfiguredDiskTemperaturePolicy(t *testing.T) {
	snapshot := models.StateSnapshot{PhysicalDisks: []models.PhysicalDisk{
		{ID: "nvme-72", DevPath: "/dev/nvme0n1", Type: "nvme", Health: "PASSED", Wearout: 90, Temperature: 72},
	}}
	ps := NewPatrolService(nil, &mockStateProvider{state: snapshot})
	if got := ps.currentPatrolRuntimeState().diskTemperatureLimits(alerts.DiskTemperatureHost{}, "nvme"); got.trigger != 70 {
		t.Fatalf("no provider: nvme trigger = %v, want the factory 70", got.trigger)
	}

	ps.SetThresholdProvider(raisedNVMeThresholds())
	for name, state := range map[string]patrolRuntimeState{
		"current":  ps.currentPatrolRuntimeState(),
		"snapshot": ps.patrolRuntimeStateForSnapshot(snapshot),
	} {
		if got := state.diskTemperatureLimits(alerts.DiskTemperatureHost{}, "nvme"); got.trigger != 75 || got.clear != 70 {
			t.Errorf("%s state: nvme limits = %+v, want the configured 75/70", name, got)
		}
		if flags := triageDiskHealthChecksState(state, nil); len(flags) != 0 {
			t.Errorf("%s state: nvme at 72C under a 75C trigger flagged %+v", name, flags)
		}
	}
}

// A scoped Patrol run filters its state; the filter must keep the configured
// thresholds, or a scoped run would judge heat by the factory triggers.
func TestScopedPatrolStateKeepsTheConfiguredDiskTemperaturePolicy(t *testing.T) {
	state := diskTemperatureTriageState(raisedNVMeThresholds())
	scoped := filterPatrolStateByScopeState(state, PatrolScope{ResourceIDs: []string{"nvme-72"}})
	if got := scoped.diskTemperatureLimits(alerts.DiskTemperatureHost{}, "nvme"); got.trigger != 75 || got.clear != 70 {
		t.Fatalf("scoped state nvme limits = %+v, want the configured 75/70", got)
	}
	if flags := triageDiskHealthChecksState(scoped, nil); len(flags) != 0 {
		t.Fatalf("scoped run flagged nvme at 72C under a 75C trigger: %+v", flags)
	}
}

// The alert threshold provider can arrive before Patrol exists; Patrol must
// still start with it.
func TestPatrolStartsWithAThresholdProviderSetBeforeIt(t *testing.T) {
	s := NewService(nil, nil)
	s.SetPatrolThresholdProvider(raisedNVMeThresholds())
	s.SetStateProvider(&mockStateProvider{})
	patrol := s.GetPatrolService()
	if patrol == nil {
		t.Fatal("expected Patrol to start once a state provider exists")
	}
	if got := patrol.currentPatrolRuntimeState().diskTemperatureLimits(alerts.DiskTemperatureHost{}, "nvme"); got.trigger != 75 {
		t.Fatalf("Patrol nvme trigger = %v, want the provider's 75", got.trigger)
	}
}

// A heat finding recovers at the clear value, where its alert recovers.
func TestDiskHeatRecoversAtTheClearValue(t *testing.T) {
	limits := diskTemperatureLimitsFor(nil, alerts.DiskTemperatureHost{}, "nvme")
	if !limits.cooled(65) || limits.cooled(66) {
		t.Fatalf("nvme limits %+v: want 65C cooled and 66C still warm", limits)
	}
	// With no band below the trigger, a reading at the trigger is still hot.
	noBand := diskTemperatureLimitsFor(&mockThresholdProvider{diskTemperature: map[string][2]float64{"sas": {70, 70}}}, alerts.DiskTemperatureHost{}, "sas")
	if !noBand.hot(70) || noBand.cooled(70) || !noBand.cooled(69) {
		t.Fatalf("no-band limits %+v: want 70C hot and not cooled, 69C cooled", noBand)
	}
}

// With no current reading (standby, a silent agent) a disk last seen hot has
// not shown that it cooled, so heat recovery is unknown rather than proven.
func TestDiskHeatRecoveryNeedsACurrentReading(t *testing.T) {
	retained := &diskinventory.CollectionStatus{Temperature: diskinventory.Unavailable("smartctl", "disk is in standby")}
	state := newPatrolRuntimeState(models.StateSnapshot{PhysicalDisks: []models.PhysicalDisk{
		{ID: "sata-retained-hot", DevPath: "/dev/sda", Type: "sata", Health: "PASSED", Wearout: -1, Temperature: 56, Collection: retained},
		{ID: "sata-retained-cool", DevPath: "/dev/sdb", Type: "sata", Health: "PASSED", Wearout: -1, Temperature: 40, Collection: retained},
	}})

	if _, err := verifyMetricRecoveredState(state, PatrolThresholds{}, "disk-high", "sata-retained-hot", "physical_disk"); !errors.Is(err, aicontracts.ErrVerificationUnknown) {
		t.Fatalf("retained 56C: err = %v, want ErrVerificationUnknown", err)
	}
	recovered, err := verifyMetricRecoveredState(state, PatrolThresholds{}, "disk-high", "sata-retained-cool", "physical_disk")
	if err != nil || !recovered {
		t.Fatalf("retained 40C: recovered = %v, err = %v, want recovered", recovered, err)
	}
}

// overriddenAgentDisksRegistry is two host agents, each reporting one NVMe
// disk hotter than the NVMe trigger, with a Disk Temp override of 80C (clear
// 75) set on agent-cool only. Alerts judge agent-cool's disk by the override.
func overriddenAgentDisksRegistry(t *testing.T) (*unifiedresources.ResourceRegistry, *AlertThresholdAdapter) {
	t.Helper()
	now := time.Now()
	agent := func(id string, temperature int) models.Host {
		return models.Host{
			ID: id, Hostname: id, DisplayName: id, Status: "online", LastSeen: now, IntervalSeconds: 30,
			Sensors: models.HostSensorSummary{SMART: []models.HostDiskSMART{{
				Device: "nvme0n1", Model: "Samsung 990", Serial: id + "-serial", Type: "nvme",
				Health: "PASSED", Temperature: temperature,
			}}},
		}
	}
	registry := unifiedresources.NewRegistry(nil)
	registry.IngestSnapshot(models.StateSnapshot{Hosts: []models.Host{agent("agent-cool", 76), agent("agent-plain", 72)}})

	mgr := alerts.NewManager()
	cfg := mgr.GetConfig()
	cfg.Overrides = map[string]alerts.ThresholdConfig{
		"agent-cool": {DiskTemperature: &alerts.HysteresisThreshold{Trigger: 80, Clear: 75}},
	}
	mgr.UpdateConfig(cfg)
	return registry, NewAlertThresholdAdapter(mgr)
}

// A host agent's Disk Temp override replaces the per-type trigger for every
// disk that agent reports, in Patrol exactly as in its alerts: agent-cool's
// NVMe at 76C is under its 80C override, agent-plain's at 72C is over the
// NVMe trigger of 70C.
func TestPatrolJudgesAgentDisksByTheAgentDiskTemperatureOverride(t *testing.T) {
	registry, provider := overriddenAgentDisksRegistry(t)
	state := newPatrolRuntimeStateWithProviders(models.StateSnapshot{}, nil, unifiedresources.NewUnifiedAIAdapter(registry))
	state.thresholdProvider = provider

	rows := patrolPhysicalDiskRows(state, nil)
	if len(rows) != 2 {
		t.Fatalf("rows = %+v, want one disk per agent", rows)
	}
	for _, row := range rows {
		want, wantIssue := diskTemperatureLimits{trigger: 70, clear: 65}, true
		if row.temperature.Collected == 76 {
			want, wantIssue = diskTemperatureLimits{trigger: 80, clear: 75}, false
		}
		if row.temperatureLimits != want {
			t.Errorf("disk at %dC: limits = %+v, want %+v", row.temperature.Collected, row.temperatureLimits, want)
		}
		if got := patrolPhysicalDiskHealthIssue(row); got != wantIssue {
			t.Errorf("disk at %dC: health issue = %v, want %v", row.temperature.Collected, got, wantIssue)
		}
	}

	s := &Service{}
	s.SetUnifiedResourceProvider(unifiedresources.NewUnifiedAIAdapter(registry))
	s.SetPatrolThresholdProvider(provider)
	_, section, _ := strings.Cut(s.buildUnifiedResourceContext(), "Physical Disks Needing Attention")
	section, _, _ = strings.Cut(section, "\n\n")
	if !strings.Contains(section, "Temp: 72C") || strings.Contains(section, "Temp: 76C") {
		t.Fatalf("attention section = %q, want agent-plain's 72C disk only", section)
	}
}

// A disk reaches its reporting agent through its parent: the agent's machine
// directly, or the storage pool an Unraid array or cache disk hangs off. A
// disk on a machine with no agent gets the hostless policy.
func TestPhysicalDiskTemperatureHostFollowsTheParentChain(t *testing.T) {
	ref := func(id string) *string { return &id }
	owners := map[string]unifiedresources.Resource{
		"agent-1": {ID: "agent-1", Type: unifiedresources.ResourceTypeAgent, Agent: &unifiedresources.AgentData{
			AgentID: "host-1", LinkedNodeID: "lab-pve1",
		}},
		"vm-1": {ID: "vm-1", Type: unifiedresources.ResourceTypeVM, Agent: &unifiedresources.AgentData{
			AgentID: "host-vm", LinkedVMID: "lab-pve1-101",
		}},
		"array-1": {ID: "array-1", Type: unifiedresources.ResourceTypeStorage, ParentID: ref("agent-1")},
		"node-1":  {ID: "node-1", Type: unifiedresources.ResourceTypeAgent},
	}
	for parent, want := range map[string]alerts.DiskTemperatureHost{
		"agent-1": {ID: "host-1", LinkedNodeID: "lab-pve1"},
		"vm-1":    {ID: "host-vm", LinkedVMID: "lab-pve1-101"},
		"array-1": {ID: "host-1", LinkedNodeID: "lab-pve1"},
		"node-1":  {},
		"missing": {},
	} {
		disk := unifiedresources.Resource{ID: "disk-under-" + parent, Type: unifiedresources.ResourceTypePhysicalDisk, ParentID: ref(parent)}
		if got := physicalDiskTemperatureHost(disk, owners); got != want {
			t.Errorf("disk under %s: host = %+v, want %+v", parent, got, want)
		}
	}
	if got := physicalDiskTemperatureHost(unifiedresources.Resource{ID: "orphan"}, owners); got != (alerts.DiskTemperatureHost{}) {
		t.Errorf("parentless disk: host = %+v, want hostless", got)
	}
}

// trueNASHeatRegistry is a TrueNAS system whose pool holds four disks beside
// the two overridden host agents of overriddenAgentDisksRegistry: nvme0n1 at
// 72C, nvme1n1 at 63C, SATA sda at 60C and SATA sdb, whose 80C reading is
// retained from before standby. The user saved a TrueNAS-wide 62C (clear 57),
// raised nvme0n1's own trigger to 75C (clear 70), and left a 50C Disk Temp
// override under the TrueNAS system's synthetic agent ID. It returns the disk
// resources by name.
func trueNASHeatRegistry(t *testing.T) (*unifiedresources.ResourceRegistry, *AlertThresholdAdapter, map[string]unifiedresources.Resource) {
	t.Helper()
	registry, _ := overriddenAgentDisksRegistry(t)
	now := time.Now()
	disk := func(name, transport string, temperature int) truenas.Disk {
		return truenas.Disk{
			ID: "disk-" + name, Name: name, Pool: "tank", Status: "ONLINE", Model: "Model " + name,
			Serial: "SERIAL-" + strings.ToUpper(name), Temperature: temperature, Transport: transport,
		}
	}
	records := truenas.FixtureRecords(truenas.FixtureSnapshot{
		CollectedAt: now,
		System:      truenas.SystemInfo{Hostname: "truenas-main", Healthy: true, CollectedAt: now},
		Pools:       []truenas.Pool{{ID: "pool-tank", Name: "tank", Status: "ONLINE"}},
		Disks: []truenas.Disk{
			disk("nvme0n1", "nvme", 72), disk("nvme1n1", "nvme", 63),
			disk("sda", "sata", 60), disk("sdb", "sata", 80),
		},
	})
	for i := range records {
		if meta := records[i].Resource.PhysicalDisk; meta != nil && records[i].Resource.Name == "sdb" {
			meta.Collection = &diskinventory.CollectionStatus{Temperature: diskinventory.Unavailable("truenas", "disk is in standby")}
		}
	}
	registry.IngestRecords(unifiedresources.SourceTrueNAS, records)

	adapter := unifiedresources.NewUnifiedAIAdapter(registry)
	disks := make(map[string]unifiedresources.Resource)
	for _, r := range adapter.GetByType(unifiedresources.ResourceTypePhysicalDisk) {
		if alerts.IsTrueNASDiskResource(r) {
			disks[r.Name] = r
		}
	}
	systemAgentID := ""
	for _, r := range adapter.GetAll() {
		if r.Agent != nil && r.Agent.Platform == "truenas" {
			systemAgentID = r.Agent.AgentID
		}
	}
	if len(disks) != 4 || systemAgentID == "" {
		t.Fatalf("TrueNAS disks = %v, system agent ID = %q; want four disks under one system", disks, systemAgentID)
	}

	mgr := alerts.NewManager()
	t.Cleanup(mgr.Stop)
	cfg := mgr.GetConfig()
	cfg.TrueNASDiskDefaults.Temperature = &alerts.HysteresisThreshold{Trigger: 62, Clear: 57}
	cfg.Overrides = map[string]alerts.ThresholdConfig{
		"agent-cool":        {DiskTemperature: &alerts.HysteresisThreshold{Trigger: 80, Clear: 75}},
		systemAgentID:       {DiskTemperature: &alerts.HysteresisThreshold{Trigger: 50, Clear: 45}},
		disks["nvme0n1"].ID: {Temperature: &alerts.HysteresisThreshold{Trigger: 75, Clear: 70}},
	}
	mgr.UpdateConfig(cfg)
	return registry, NewAlertThresholdAdapter(mgr), disks
}

// Patrol judges a TrueNAS disk by its own temperature alert's tiers, not as a
// disk of the TrueNAS system's synthetic host agent: nvme0n1's override keeps
// it quiet at 72C, the TrueNAS-wide 62C flags nvme1n1 at 63C (under the NVMe
// trigger of 70C) and spares SATA sda at 60C (over the SATA trigger of 55C),
// and the 50C host override under the system ID reaches none of them. A
// retained reading stays unjudged. Agent disks keep their host's override.
func TestPatrolJudgesTrueNASDisksByTheirAlertTiers(t *testing.T) {
	registry, provider, disks := trueNASHeatRegistry(t)
	state := newPatrolRuntimeStateWithProviders(models.StateSnapshot{}, nil, unifiedresources.NewUnifiedAIAdapter(registry))
	state.thresholdProvider = provider

	type want struct {
		limits diskTemperatureLimits
		issue  bool
	}
	expected := map[string]want{
		disks["nvme0n1"].ID: {limits: diskTemperatureLimits{trigger: 75, clear: 70}},
		disks["nvme1n1"].ID: {limits: diskTemperatureLimits{trigger: 62, clear: 57}, issue: true},
		disks["sda"].ID:     {limits: diskTemperatureLimits{trigger: 62, clear: 57}},
		disks["sdb"].ID:     {limits: diskTemperatureLimits{trigger: 62, clear: 57}},
	}
	rows := patrolPhysicalDiskRows(state, nil)
	if len(rows) != 6 {
		t.Fatalf("rows = %+v, want four TrueNAS disks and two agent disks", rows)
	}
	for _, row := range rows {
		w, trueNAS := expected[row.id]
		if !trueNAS {
			// overriddenAgentDisksRegistry: agent-cool's 76C NVMe under its 80C
			// override, agent-plain's 72C NVMe over the NVMe trigger.
			w = want{limits: diskTemperatureLimits{trigger: 70, clear: 65}, issue: true}
			if row.temperature.Collected == 76 {
				w = want{limits: diskTemperatureLimits{trigger: 80, clear: 75}}
			}
		}
		if row.temperatureLimits != w.limits {
			t.Errorf("disk %s at %dC: limits = %+v, want %+v", row.id, row.temperature.Collected, row.temperatureLimits, w.limits)
		}
		if got := patrolPhysicalDiskHealthIssue(row); got != w.issue {
			t.Errorf("disk %s at %dC: health issue = %v, want %v", row.id, row.temperature.Collected, got, w.issue)
		}
	}

	flags := triageDiskHealthChecksState(state, nil)
	for _, name := range []string{"nvme0n1", "sda", "sdb"} {
		if flag := triageFindFlag(flags, func(f TriageFlag) bool {
			return f.ResourceID == disks[name].ID && strings.Contains(f.Reason, "Disk temperature")
		}); flag != nil {
			t.Errorf("%s: unexpected temperature flag %+v", name, *flag)
		}
	}
	if flag := triageFindFlag(flags, func(f TriageFlag) bool {
		return f.ResourceID == disks["nvme1n1"].ID && strings.Contains(f.Reason, "Disk temperature")
	}); flag == nil || flag.Threshold != 62 {
		t.Errorf("nvme1n1: temperature flag = %+v, want one at the TrueNAS-wide 62C", flag)
	}

	if _, err := verifyMetricRecoveredState(state, PatrolThresholds{}, "disk-high", disks["sdb"].ID, "physical_disk"); !errors.Is(err, aicontracts.ErrVerificationUnknown) {
		t.Errorf("retained 80C: err = %v, want ErrVerificationUnknown", err)
	}

	s := &Service{}
	s.SetUnifiedResourceProvider(unifiedresources.NewUnifiedAIAdapter(registry))
	s.SetPatrolThresholdProvider(provider)
	_, section, _ := strings.Cut(s.buildUnifiedResourceContext(), "Physical Disks Needing Attention")
	section, _, _ = strings.Cut(section, "\n\n")
	for _, reading := range []string{"Temp: 63C", "Temp: 72C"} {
		if !strings.Contains(section, reading) {
			t.Errorf("attention section = %q, want the disk at %s", section, reading)
		}
	}
	for _, reading := range []string{"Temp: 60C", "Temp: 76C", "Temp: 80C"} {
		if strings.Contains(section, reading) {
			t.Errorf("attention section = %q, lists a disk at %s", section, reading)
		}
	}
	if strings.Count(section, "Temp: 72C") != 1 {
		t.Errorf("attention section = %q, want only agent-plain's 72C disk, not TrueNAS nvme0n1 under its 75C override", section)
	}
}