Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

55.8 KB
Raw Blame
package monitoring

import (
	"context"
	"encoding/json"
	"os"
	"path/filepath"
	"strings"
	"testing"
	"time"

	"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
	"github.com/rcourtman/pulse-go-rewrite/internal/config"
	"github.com/rcourtman/pulse-go-rewrite/internal/models"
	"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
	"github.com/rcourtman/pulse-go-rewrite/pkg/diskinventory"
	"github.com/rcourtman/pulse-go-rewrite/pkg/metrics"
	"github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"
)

type issue1595TopologyFixture struct {
	Node        string `json:"node"`
	Instance    string `json:"instance"`
	AgentID     string `json:"agentId"`
	Model       string `json:"model"`
	SizeBytes   int64  `json:"sizeBytes"`
	Controllers []struct {
		ID    string `json:"id"`
		Pool  string `json:"pool"`
		Disks []struct {
			Device         string `json:"device"`
			Target         string `json:"target"`
			Serial         string `json:"serial"`
			ProviderSerial string `json:"providerSerial"`
			Temperature    int    `json:"temperature"`
			ReadBytes      uint64 `json:"readBytes"`
			WriteBytes     uint64 `json:"writeBytes"`
			IOTimeMs       uint64 `json:"ioTimeMs"`
		} `json:"disks"`
	} `json:"controllers"`
}

func loadIssue1595TopologyFixture(t *testing.T) issue1595TopologyFixture {
	t.Helper()
	data, err := os.ReadFile(filepath.Join("..", "..", "testdata", "issue1595_sas_topology.json"))
	if err != nil {
		t.Fatalf("read issue #1595 fixture: %v", err)
	}
	var fixture issue1595TopologyFixture
	if err := json.Unmarshal(data, &fixture); err != nil {
		t.Fatalf("decode issue #1595 fixture: %v", err)
	}
	return fixture
}

func TestIssue1595SASTopologySurvivesMergeRegistryAndReadState(t *testing.T) {
	fixture := loadIssue1595TopologyFixture(t)
	now := time.Date(2026, 7, 20, 12, 0, 0, 0, time.UTC)
	nodeID := fixture.Instance + "-" + fixture.Node

	host := models.Host{
		ID:           fixture.AgentID,
		Hostname:     fixture.Node,
		Status:       "online",
		LastSeen:     now,
		LinkedNodeID: nodeID,
	}
	node := models.Node{
		ID:            nodeID,
		Name:          fixture.Node,
		Instance:      fixture.Instance,
		Status:        "online",
		LastSeen:      now,
		LinkedAgentID: fixture.AgentID,
	}

	var providerDisks []models.PhysicalDisk
	expectedBySerial := make(map[string]models.PhysicalDisk)
	for _, controller := range fixture.Controllers {
		for index, disk := range controller.Disks {
			device := "/dev/" + disk.Device
			io := &models.DiskIO{
				Device:     disk.Device,
				ReadBytes:  disk.ReadBytes,
				WriteBytes: disk.WriteBytes,
				ReadOps:    uint64(100 + index),
				WriteOps:   uint64(200 + index),
				ReadTime:   uint64(300 + index),
				WriteTime:  uint64(400 + index),
				IOTime:     disk.IOTimeMs,
			}
			host.DiskIO = append(host.DiskIO, *io)
			host.Sensors.SMART = append(host.Sensors.SMART, models.HostDiskSMART{
				Device:      device,
				Model:       fixture.Model,
				Serial:      disk.Serial,
				Type:        "sas",
				Controller:  controller.ID,
				Target:      disk.Target,
				SizeBytes:   fixture.SizeBytes,
				Temperature: disk.Temperature,
				Health:      "PASSED",
				Pool:        controller.Pool,
				IO:          io,
				Collection: &diskinventory.CollectionStatus{
					Serial:      diskinventory.Available("smartctl"),
					Temperature: diskinventory.Available("smartctl"),
					IO:          diskinventory.Available("linux-diskstats"),
					Controller:  diskinventory.Available("linux-sysfs"),
					Pool:        diskinventory.Available("zpool-status"),
				},
			})

			providerDisk := models.PhysicalDisk{
				ID:           fixture.Instance + "-" + fixture.Node + "-" + disk.Device,
				Node:         fixture.Node,
				Instance:     fixture.Instance,
				DevPath:      device,
				Model:        fixture.Model,
				Serial:       disk.ProviderSerial,
				Type:         "unknown",
				Size:         fixture.SizeBytes,
				Health:       "UNKNOWN",
				Wearout:      -1,
				Used:         "ZFS",
				StorageGroup: controller.Pool,
				Collection: &diskinventory.CollectionStatus{
					Serial:      diskinventory.Available("pve-api"),
					Temperature: diskinventory.Unsupported("pve-api", "provider does not report disk temperature"),
					IO:          diskinventory.Unsupported("pve-api", "provider does not report per-disk I/O"),
					Controller:  diskinventory.Missing("pve-api", "controller association absent"),
					Pool:        diskinventory.Available("pve-storage"),
				},
				LastChecked: now,
			}
			providerDisk.ExpectedUpdateInterval = 15 * time.Minute
			providerDisks = append(providerDisks, providerDisk)

			want := providerDisk
			want.Serial = disk.Serial
			want.Type = "sas"
			want.Controller = controller.ID
			want.Target = disk.Target
			want.Temperature = disk.Temperature
			want.Health = "PASSED"
			want.IO = io
			expectedBySerial[disk.Serial] = want
		}
	}

	merged := mergeHostAgentSMARTIntoDisks(providerDisks, []models.Node{node}, []models.Host{host}, nil)
	if len(merged) != 24 {
		t.Fatalf("merged disk count = %d, want 24", len(merged))
	}
	for _, disk := range merged {
		want, ok := expectedBySerial[disk.Serial]
		if !ok {
			t.Fatalf("provider SAS address survived as stable serial for %s: %q", disk.DevPath, disk.Serial)
		}
		assertIssue1595PhysicalDisk(t, disk, want)
	}

	registry := unifiedresources.NewRegistry(nil)
	registry.IngestSnapshot(models.StateSnapshot{
		Nodes:         []models.Node{node},
		Hosts:         []models.Host{host},
		PhysicalDisks: providerDisks,
	})

	resources := registry.ListByType(unifiedresources.ResourceTypePhysicalDisk)
	if len(resources) != 24 {
		t.Fatalf("registry physical disk count = %d, want 24 distinct disks", len(resources))
	}
	for _, resource := range resources {
		if resource.PhysicalDisk == nil {
			t.Fatalf("physical disk resource %q has no physicalDisk metadata", resource.ID)
		}
		_, ok := expectedBySerial[resource.PhysicalDisk.Serial]
		if !ok {
			t.Fatalf("registry serial = %q, want a smartctl serial", resource.PhysicalDisk.Serial)
		}
		if !hasIssue1595Source(resource.Sources, unifiedresources.SourceAgent) ||
			!hasIssue1595Source(resource.Sources, unifiedresources.SourceProxmox) {
			t.Fatalf("disk %q sources = %v, want agent and proxmox", resource.PhysicalDisk.Serial, resource.Sources)
		}
	}

	views := registry.PhysicalDisks()
	if len(views) != 24 {
		t.Fatalf("read-state physical disk count = %d, want 24", len(views))
	}
	for _, view := range views {
		want, ok := expectedBySerial[view.Serial()]
		if !ok {
			t.Fatalf("read-state serial = %q, want smartctl serial", view.Serial())
		}
		if view.MetricResourceID() != want.Serial {
			t.Fatalf("disk %q metrics target = %q, want serial-stable target", want.Serial, view.MetricResourceID())
		}
		readBack := physicalDiskFromReadStateView(view)
		if readBack.ExpectedUpdateInterval != want.ExpectedUpdateInterval {
			t.Fatalf("disk cadence lost during canonical roundtrip: %s != %s", readBack.ExpectedUpdateInterval, want.ExpectedUpdateInterval)
		}
		assertIssue1595PhysicalDisk(t, readBack, want)
		if readBack.Collection == nil ||
			readBack.Collection.Serial.State != diskinventory.FieldAvailable ||
			readBack.Collection.Temperature.State != diskinventory.FieldAvailable ||
			readBack.Collection.IO.State != diskinventory.FieldAvailable ||
			readBack.Collection.Controller.State != diskinventory.FieldAvailable ||
			readBack.Collection.Pool.State != diskinventory.FieldAvailable {
			t.Fatalf("disk %q collection status = %+v", readBack.Serial, readBack.Collection)
		}
	}
}

func TestPhysicalDiskUnavailableEvidenceIsRetainedWithoutPretendingItWasCollected(t *testing.T) {
	previous := models.PhysicalDisk{
		Serial:       "ZR5TESTA0001",
		Controller:   "0000:03:00.0",
		Target:       "6:0:0:0",
		Temperature:  30,
		StorageGroup: "tank-a",
		IO:           &models.DiskIO{Device: "sda", ReadBytes: 1000},
	}
	current := models.PhysicalDisk{
		Collection: &diskinventory.CollectionStatus{
			Serial:      diskinventory.Missing("smartctl", "serial absent from successful response"),
			Temperature: diskinventory.Unavailable("smartctl", "collection deadline exceeded"),
			IO:          diskinventory.Unsupported("controller", "per-member counters unavailable"),
			Controller:  diskinventory.Unavailable("linux-sysfs", "topology lookup failed"),
			Pool:        diskinventory.Unavailable("zpool-status", "command failed"),
		},
	}

	got := preserveUnavailablePhysicalDiskEvidence(current, previous)
	if got.Serial != previous.Serial ||
		got.Controller != previous.Controller ||
		got.Target != previous.Target ||
		got.Temperature != previous.Temperature ||
		got.StorageGroup != previous.StorageGroup ||
		got.IO == nil ||
		got.IO.ReadBytes != previous.IO.ReadBytes {
		t.Fatalf("unavailable evidence was discarded: %+v", got)
	}
	if got.Collection.Temperature.State != diskinventory.FieldUnavailable ||
		got.Collection.IO.State != diskinventory.FieldUnsupported ||
		got.Collection.Serial.State != diskinventory.FieldMissing {
		t.Fatalf("retained values concealed current collection state: %+v", got.Collection)
	}

	got.IO.ReadBytes = 2000
	if previous.IO.ReadBytes != 1000 {
		t.Fatal("retained I/O evidence aliases the previous snapshot")
	}
}

func TestTrustedSMARTSerialPromotionDoesNotRewriteSATAOrNVMeIdentity(t *testing.T) {
	for _, diskType := range []string{"sata", "nvme"} {
		t.Run(diskType, func(t *testing.T) {
			device := "/dev/sda"
			if diskType == "nvme" {
				device = "/dev/nvme0n1"
			}
			disks := []models.PhysicalDisk{{
				ID:      "provider-disk",
				Node:    "node",
				DevPath: device,
				Serial:  "PROVIDER-SERIAL",
				Type:    diskType,
			}}
			hosts := []models.Host{{
				ID: "agent",
				Sensors: models.HostSensorSummary{
					SMART: []models.HostDiskSMART{{
						Device: device,
						Serial: "SMARTCTL-SERIAL",
						Type:   diskType,
						Collection: &diskinventory.CollectionStatus{
							Serial: diskinventory.Available("smartctl"),
						},
					}},
				},
			}}

			got := mergeHostAgentSMARTIntoDisks(
				disks,
				[]models.Node{{Name: "node", LinkedAgentID: "agent"}},
				hosts,
				nil,
			)[0]
			if got.Serial != "PROVIDER-SERIAL" || got.Type != diskType {
				t.Fatalf("%s identity changed during SAS remediation: %+v", diskType, got)
			}
		})
	}
}

func assertIssue1595PhysicalDisk(t *testing.T, got, want models.PhysicalDisk) {
	t.Helper()
	if got.Serial != want.Serial ||
		got.Type != "sas" ||
		got.Controller != want.Controller ||
		got.Target != want.Target ||
		got.Temperature != want.Temperature ||
		got.StorageGroup != want.StorageGroup ||
		got.IO == nil ||
		got.IO.ReadBytes != want.IO.ReadBytes ||
		got.IO.WriteBytes != want.IO.WriteBytes ||
		got.IO.ReadOps != want.IO.ReadOps ||
		got.IO.WriteOps != want.IO.WriteOps ||
		got.IO.ReadTime != want.IO.ReadTime ||
		got.IO.WriteTime != want.IO.WriteTime ||
		got.IO.IOTime != want.IO.IOTime {
		t.Fatalf("disk %s lost trusted inventory data:\n got  %+v\n want %+v", got.DevPath, got, want)
	}
}

func hasIssue1595Source(sources []unifiedresources.DataSource, want unifiedresources.DataSource) bool {
	for _, source := range sources {
		if source == want {
			return true
		}
	}
	return false
}

// An agent report's SMART row may carry a temperature its collector did not
// collect this time (retained across a standby or failed probe, or kept after
// the agent's reporting lease expired). Only a temperature
// diskinventory.TemperatureCollected accepts becomes a history sample.
func TestWriteHostSMARTMetricsRecordsOnlyCollectedTemperatures(t *testing.T) {
	storeConfig := metrics.DefaultConfig(t.TempDir())
	storeConfig.FlushInterval = time.Hour
	store, err := metrics.NewStore(storeConfig)
	if err != nil {
		t.Fatal(err)
	}
	t.Cleanup(func() { _ = store.Close() })
	m := &Monitor{metricsStore: store}

	host := models.Host{ID: "agent-1", Hostname: "node1", Sensors: models.HostSensorSummary{SMART: []models.HostDiskSMART{
		{Device: "sda", Serial: "COLLECTED1", Temperature: 41,
			Collection: &diskinventory.CollectionStatus{Temperature: diskinventory.Available("smartctl")}},
		{Device: "sdb", Serial: "LEGACY1", Temperature: 38},
		{Device: "sdc", Serial: "RETAINED1", Temperature: 36,
			Collection: &diskinventory.CollectionStatus{Temperature: diskinventory.Unavailable("smartctl", "disk is in standby")}},
		{Device: "sdd", Serial: "SILENT1", Temperature: 44,
			Collection: &diskinventory.CollectionStatus{Temperature: diskinventory.Unavailable("smartctl", models.HostAgentStoppedReportingReason)}},
		{Device: "sde", Serial: "MISSING1", Temperature: 39,
			Collection: &diskinventory.CollectionStatus{Temperature: diskinventory.Missing("smartctl", "temperature attribute absent")}},
	}}}
	now := time.Now()
	m.writeHostSMARTMetrics(host, now)
	store.Flush()

	for _, tc := range []struct {
		disk models.HostDiskSMART
		want int
	}{
		{host.Sensors.SMART[0], 1},
		{host.Sensors.SMART[1], 1},
		{host.Sensors.SMART[2], 0},
		{host.Sensors.SMART[3], 0},
		{host.Sensors.SMART[4], 0},
	} {
		id := unifiedresources.HostSMARTDiskMetricID(host, tc.disk)
		points, err := store.Query("disk", id, "smart_temp", now.Add(-time.Minute), now.Add(time.Minute), 0)
		if err != nil {
			t.Fatal(err)
		}
		if len(points) != tc.want {
			t.Fatalf("%s: temperature samples = %d, want %d", tc.disk.Serial, len(points), tc.want)
		}
	}
}

// A controller member without a usable serial or WWN keys its history by its
// source ID, which already names the member behind the shared block path: the
// host agent writes under HostSMARTDiskMetricID, Proxmox under
// PhysicalDiskMetricID, which returns ProxmoxPhysicalDiskSourceID. The metrics
// target a chart reads must be that key, in the live registry and in one
// rehydrated from persisted resources, not the key with the member's topology
// appended a second time, or every sample those writers store goes unread. The
// agent's I/O for a lone member is filed under the member's SMART key. A linked
// node's agent files the I/O of a member it reads no SMART for through the
// Proxmox disk's metrics target, so that I/O must share the member's SMART key.
func TestIdentitylessControllerMembersReadTheirWritersHistory(t *testing.T) {
	storeConfig := metrics.DefaultConfig(t.TempDir())
	storeConfig.FlushInterval = time.Hour
	store, err := metrics.NewStore(storeConfig)
	if err != nil {
		t.Fatal(err)
	}
	t.Cleanup(func() { _ = store.Close() })
	state := models.NewState()
	adapter := unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(nil))
	m := &Monitor{state: state, resourceStore: adapter, metricsStore: store, rateTracker: NewRateTracker()}

	now := time.Now().UTC().Truncate(time.Second)
	collected := func() *diskinventory.CollectionStatus {
		return &diskinventory.CollectionStatus{Temperature: diskinventory.Available("smartctl")}
	}
	state.UpdateNodesForInstance("pve", []models.Node{
		{ID: "pve-node1", Name: "node1", Instance: "pve", Status: "online", LastSeen: now},
		{ID: "pve-node2", Name: "node2", Instance: "pve", Status: "online", LastSeen: now, LinkedAgentID: "agent-node2"},
	})
	standalone := models.Host{ID: "host-pve", Hostname: "standalone", MachineID: "machine-standalone", Status: "online", LastSeen: now,
		Sensors: models.HostSensorSummary{SMART: []models.HostDiskSMART{
			{Device: "sdd", Controller: "ctrl0", Target: "megaraid,3", Type: "sas", Temperature: 33, Collection: collected()},
			{Device: "sdd", Controller: "ctrl0", Target: "megaraid,4", Type: "sas", Temperature: 34, Collection: collected()},
			{Device: "/dev/sde", Controller: "ctrl0", Target: "megaraid,5", Type: "sas", Temperature: 35, Collection: collected()},
		}},
		DiskIO: []models.DiskIO{{Device: "sde", ReadBytes: 1 << 20, WriteBytes: 1 << 20, IOTime: 100}},
	}
	linked := models.Host{ID: "agent-node2", Hostname: "node2", MachineID: "machine-node2", LinkedNodeID: "pve-node2", Status: "online", LastSeen: now,
		Sensors: models.HostSensorSummary{SMART: []models.HostDiskSMART{
			{Device: "sdc", Controller: "ctrl1", Target: "megaraid,1", Type: "sas", Standby: true, Collection: &diskinventory.CollectionStatus{
				Temperature: diskinventory.Unavailable("smartctl", "disk is in standby")}},
		}},
		DiskIO: []models.DiskIO{{Device: "sdc", ReadBytes: 1 << 20, WriteBytes: 1 << 20, IOTime: 100}},
	}
	state.UpsertHost(standalone)
	state.UpsertHost(linked)
	pveMember := models.PhysicalDisk{
		ID:       unifiedresources.ProxmoxPhysicalDiskSourceID("pve", "node1", "/dev/sdx", "", "megaraid,0"),
		Instance: "pve", Node: "node1", DevPath: "/dev/sdx", Target: "megaraid,0", Type: "sas", Temperature: 40, LastChecked: now,
	}
	pveBlockPath := models.PhysicalDisk{
		ID:       unifiedresources.ProxmoxPhysicalDiskSourceID("pve", "node2", "/dev/sdc", "", ""),
		Instance: "pve", Node: "node2", DevPath: "/dev/sdc", Serial: "unknown", Type: "sas", LastChecked: now,
	}
	state.UpdatePhysicalDisks("pve", []models.PhysicalDisk{pveMember, pveBlockPath})
	adapter.PopulateFromSnapshot(state.GetSnapshot())

	hosts := []models.Host{standalone, linked}
	for _, host := range hosts {
		m.writeHostPhysicalDiskIOMetrics(host, now.Add(-30*time.Second), counterSampleTime{})
	}
	for _, host := range hosts {
		for i := range host.DiskIO {
			host.DiskIO[i].ReadBytes += 1 << 20
			host.DiskIO[i].WriteBytes += 1 << 20
			host.DiskIO[i].IOTime += 1000
		}
		m.writeHostPhysicalDiskIOMetrics(host, now, counterSampleTime{})
		m.writeHostSMARTMetrics(host, now)
	}
	m.writeSMARTMetrics(pveMember, now)
	store.Flush()

	smartTemp := []string{"smart_temp"}
	diskIO := []string{"diskread", "diskwrite", "disk"}
	want := map[string]struct {
		key     string
		metrics []string
	}{
		"megaraid,3": {unifiedresources.HostSMARTDiskMetricID(standalone, standalone.Sensors.SMART[0]), smartTemp},
		"megaraid,4": {unifiedresources.HostSMARTDiskMetricID(standalone, standalone.Sensors.SMART[1]), smartTemp},
		"megaraid,5": {unifiedresources.HostSMARTDiskMetricID(standalone, standalone.Sensors.SMART[2]), append(smartTemp, diskIO...)},
		"megaraid,0": {unifiedresources.PhysicalDiskMetricID(pveMember), smartTemp},
		"megaraid,1": {unifiedresources.HostSMARTDiskMetricID(linked, linked.Sensors.SMART[0]), diskIO},
	}
	for target, member := range map[string]string{
		"megaraid,3": "host-pve:sdd@ctrl0/megaraid,3",
		"megaraid,5": "host-pve:sde@ctrl0/megaraid,5",
		"megaraid,0": "pve-node1--dev-sdx:sdx@/megaraid,0",
		"megaraid,1": "agent-node2:sdc@ctrl1/megaraid,1",
	} {
		if want[target].key != member {
			t.Fatalf("%s: writer key = %q, want the member scoped once %q", target, want[target].key, member)
		}
	}

	live := unifiedresources.NewRegistry(nil)
	live.IngestSnapshot(state.GetSnapshot())
	payload, err := json.Marshal(live.List())
	if err != nil {
		t.Fatal(err)
	}
	var persisted []unifiedresources.Resource
	if err := json.Unmarshal(payload, &persisted); err != nil {
		t.Fatal(err)
	}
	rehydrated := unifiedresources.NewRegistry(nil)
	rehydrated.IngestResources(persisted)
	for _, tc := range []struct {
		name     string
		registry *unifiedresources.ResourceRegistry
	}{{"live", live}, {"rehydrated", rehydrated}} {
		disks := tc.registry.ListByType(unifiedresources.ResourceTypePhysicalDisk)
		if len(disks) != len(want) {
			t.Fatalf("%s: disks = %d, want %d with the standby member merged into its Proxmox row", tc.name, len(disks), len(want))
		}
		for _, disk := range disks {
			member, ok := want[disk.PhysicalDisk.Target]
			if !ok {
				t.Fatalf("%s: unexpected disk %s at %q target %q", tc.name, disk.ID, disk.PhysicalDisk.DevPath, disk.PhysicalDisk.Target)
			}
			target := tc.registry.MetricsTarget(disk.ID)
			if target == nil || target.ResourceType != "disk" || target.ResourceID != member.key {
				t.Errorf("%s: %s metrics target = %+v, want the writer's disk/%s", tc.name, disk.PhysicalDisk.Target, target, member.key)
				continue
			}
			for _, metric := range member.metrics {
				points, err := store.Query(target.ResourceType, target.ResourceID, metric, now.Add(-time.Minute), now.Add(time.Minute), 0)
				if err != nil {
					t.Fatal(err)
				}
				if len(points) == 0 {
					t.Errorf("%s: %s metrics target %s reads no %s history", tc.name, disk.PhysicalDisk.Target, target.ResourceID, metric)
				}
			}
		}
	}
}

// A dual-ported SAS shelf, cloned VMs with an explicit serial and fixed-serial
// USB bridges report one usable serial on several hosts. The registry keeps a
// disk per host, and each must keep its own agent source target, in the live
// registry and in one rehydrated from its resources, or a disk only the agent
// reports has no metrics target and no chart. Both targets read the series the
// agent's SMART writer files under the serial: disk history follows the drive's
// hardware identity, not the host.
func TestSameSerialAgentDisksOnTwoHostsEachKeepTheirMetricsTarget(t *testing.T) {
	storeConfig := metrics.DefaultConfig(t.TempDir())
	storeConfig.FlushInterval = time.Hour
	store, err := metrics.NewStore(storeConfig)
	if err != nil {
		t.Fatal(err)
	}
	t.Cleanup(func() { _ = store.Close() })
	m := &Monitor{metricsStore: store}

	now := time.Now().UTC().Truncate(time.Second)
	const serial = "SHELF-SERIAL-1"
	hosts := []models.Host{
		{ID: "agent-alpha", Hostname: "alpha", MachineID: "machine-alpha", Status: "online", LastSeen: now},
		{ID: "agent-beta", Hostname: "beta", MachineID: "machine-beta", Status: "online", LastSeen: now},
	}
	wantSourceIDs := make(map[string]bool, len(hosts))
	for i := range hosts {
		hosts[i].Sensors.SMART = []models.HostDiskSMART{{Device: "sda", Serial: serial, Model: "Shelf Disk", Type: "sas",
			Health: "PASSED", Temperature: 40 + i, Collection: &diskinventory.CollectionStatus{Temperature: diskinventory.Available("smartctl")}}}
		m.writeHostSMARTMetrics(hosts[i], now)
		wantSourceIDs[unifiedresources.HostSMARTDiskSourceID(hosts[i], hosts[i].Sensors.SMART[0])] = true
	}
	store.Flush()
	if len(wantSourceIDs) != len(hosts) {
		t.Fatalf("agent disk source IDs = %v, want one per host", wantSourceIDs)
	}

	live := unifiedresources.NewRegistry(nil)
	live.IngestSnapshot(models.StateSnapshot{Hosts: hosts})
	rehydrated := unifiedresources.NewRegistry(nil)
	rehydrated.IngestResources(live.List())
	for _, tc := range []struct {
		name     string
		registry *unifiedresources.ResourceRegistry
	}{{"live", live}, {"rehydrated", rehydrated}} {
		disks := tc.registry.ListByType(unifiedresources.ResourceTypePhysicalDisk)
		if len(disks) != len(hosts) {
			t.Fatalf("%s: disks = %d, want one per host", tc.name, len(disks))
		}
		gotSourceIDs := make(map[string]bool, len(disks))
		for _, disk := range disks {
			for _, target := range tc.registry.SourceTargets(disk.ID) {
				if target.Source == unifiedresources.SourceAgent {
					gotSourceIDs[target.SourceID] = true
				}
			}
			target := tc.registry.MetricsTarget(disk.ID)
			if target == nil {
				t.Fatalf("%s: disk %s has no metrics target", tc.name, disk.ID)
			}
			if target.ResourceType != "disk" || target.ResourceID != serial {
				t.Fatalf("%s: disk %s metrics target = %+v, want the serial's disk series", tc.name, disk.ID, *target)
			}
			points, err := store.Query(target.ResourceType, target.ResourceID, "smart_temp", now.Add(-time.Minute), now.Add(time.Minute), 0)
			if err != nil {
				t.Fatal(err)
			}
			if len(points) == 0 {
				t.Fatalf("%s: disk %s metrics target reads no SMART temperature history", tc.name, disk.ID)
			}
		}
		if len(gotSourceIDs) != len(wantSourceIDs) {
			t.Fatalf("%s: agent source targets = %v, want each host's own %v", tc.name, gotSourceIDs, wantSourceIDs)
		}
		for sourceID := range wantSourceIDs {
			if !gotSourceIDs[sourceID] {
				t.Fatalf("%s: agent source targets = %v, want each host's own %v", tc.name, gotSourceIDs, wantSourceIDs)
			}
		}
	}
}

// An Unraid host reports each array disk twice: as a SMART row and as an
// Unraid inventory row. When the SMART row carries no serial, smartctl's
// standby row among them, the disk takes the serial its Unraid row reports,
// and its metrics target reads that serial. The SMART and disk I/O writers must
// file under it too, as must the I/O writer for a disk with no SMART row,
// or the disk's temperature and I/O history never chart.
func TestAgentDiskHistoryFollowsTheSerialItsUnraidRowReports(t *testing.T) {
	storeConfig := metrics.DefaultConfig(t.TempDir())
	storeConfig.FlushInterval = time.Hour
	store, err := metrics.NewStore(storeConfig)
	if err != nil {
		t.Fatal(err)
	}
	t.Cleanup(func() { _ = store.Close() })
	m := &Monitor{metricsStore: store, rateTracker: NewRateTracker()}

	now := time.Now().UTC().Truncate(time.Second)
	collected := func() *diskinventory.CollectionStatus {
		return &diskinventory.CollectionStatus{Temperature: diskinventory.Available("smartctl")}
	}
	host := models.Host{ID: "host-tower", Hostname: "tower", MachineID: "machine-tower", Status: "online", LastSeen: now}
	host.Sensors.SMART = []models.HostDiskSMART{
		{Device: "sdb", Temperature: 40, Collection: collected()},
		{Device: "/dev/sdc", WWN: "0x5000c500aaaa0001", Temperature: 41, Collection: collected()},
		{Device: "sdd", Standby: true, Collection: &diskinventory.CollectionStatus{
			Temperature: diskinventory.Unavailable("smartctl", "disk is in standby")}},
	}
	host.Unraid = &models.HostUnraidStorage{ArrayStarted: true, Disks: []models.HostUnraidDisk{
		{Device: "sdb", Serial: "UNRAID-SERIAL-B", Name: "disk1", Role: "data", Status: "online"},
		{Device: "sdc", Serial: "UNRAID-SERIAL-C", Name: "disk2", Role: "data", Status: "online"},
		{Device: "sdd", Serial: "UNRAID-SERIAL-D", Name: "disk3", Role: "data", Status: "online", SpunDown: true},
		{Device: "sde", Serial: "UNRAID-SERIAL-E", Name: "disk4", Role: "data", Status: "online"},
		{Device: "sdf", Name: "disk5", Role: "data", Status: "online"},
	}}
	want := map[string][]string{
		"sdb":      {"smart_temp", "diskread", "diskwrite", "disk"},
		"/dev/sdc": {"smart_temp", "diskread", "diskwrite", "disk"},
		"sdd":      {"diskread", "diskwrite", "disk"},
		"sde":      {"diskread", "diskwrite", "disk"},
		"sdf":      {"diskread", "diskwrite", "disk"},
	}
	for _, device := range []string{"sdb", "sdc", "sdd", "sde", "sdf"} {
		host.DiskIO = append(host.DiskIO, models.DiskIO{Device: device, ReadBytes: 1 << 20, WriteBytes: 1 << 20, IOTime: 100})
	}
	m.writeHostPhysicalDiskIOMetrics(host, now.Add(-30*time.Second), counterSampleTime{})
	for i := range host.DiskIO {
		host.DiskIO[i].ReadBytes += 1 << 20
		host.DiskIO[i].WriteBytes += 1 << 20
		host.DiskIO[i].IOTime += 1000
	}
	m.writeHostPhysicalDiskIOMetrics(host, now, counterSampleTime{})
	m.writeHostSMARTMetrics(host, now)
	store.Flush()

	live := unifiedresources.NewRegistry(nil)
	live.IngestSnapshot(models.StateSnapshot{Hosts: []models.Host{host}})
	rehydrated := unifiedresources.NewRegistry(nil)
	rehydrated.IngestResources(live.List())
	for _, tc := range []struct {
		name     string
		registry *unifiedresources.ResourceRegistry
	}{{"live", live}, {"rehydrated", rehydrated}} {
		disks := tc.registry.ListByType(unifiedresources.ResourceTypePhysicalDisk)
		if len(disks) != len(want) {
			t.Fatalf("%s: disks = %d, want %d", tc.name, len(disks), len(want))
		}
		for _, disk := range disks {
			metricNames, ok := want[disk.PhysicalDisk.DevPath]
			if !ok {
				t.Fatalf("%s: unexpected disk %s at %q", tc.name, disk.ID, disk.PhysicalDisk.DevPath)
			}
			target := tc.registry.MetricsTarget(disk.ID)
			if target == nil {
				t.Fatalf("%s: disk %s has no metrics target", tc.name, disk.PhysicalDisk.DevPath)
			}
			for _, metric := range metricNames {
				points, err := store.Query(target.ResourceType, target.ResourceID, metric, now.Add(-time.Minute), now.Add(time.Minute), 0)
				if err != nil {
					t.Fatal(err)
				}
				if len(points) == 0 {
					t.Errorf("%s: disk %s metrics target %+v reads no %s history", tc.name, disk.PhysicalDisk.DevPath, *target, metric)
				}
			}
		}
	}
}

// An Unraid host's disk can show a temperature only its Unraid inventory
// reports: a member the agent's SMART collection skips (--disk-exclude), or
// one Unraid reads with its own per-disk SMART settings while the agent's
// probe returns no temperature. The chart must hold the reading the disk
// shows, under the key its metrics target reads, and nothing for a reading the
// disk does not show as collected (a spun-down disk, a host past its lease) or
// does not show at all (a controller member behind the device Unraid reads).
func TestAgentDiskChartsTheUnraidTemperatureItShows(t *testing.T) {
	storeConfig := metrics.DefaultConfig(t.TempDir())
	storeConfig.FlushInterval = time.Hour
	store, err := metrics.NewStore(storeConfig)
	if err != nil {
		t.Fatal(err)
	}
	t.Cleanup(func() { _ = store.Close() })
	m := &Monitor{metricsStore: store, rateTracker: NewRateTracker()}

	now := time.Now().UTC().Truncate(time.Second)
	host := models.Host{ID: "host-tower", Hostname: "tower", MachineID: "machine-tower", Status: "online", LastSeen: now}
	host.Sensors.SMART = []models.HostDiskSMART{
		// The agent's identity-only row for a disk its probe got nothing from.
		{Device: "sdb", Serial: "SER-B", Health: "UNKNOWN", Collection: &diskinventory.CollectionStatus{
			Temperature: diskinventory.Unavailable("smartctl", "SMART probe returned no usable temperature data")}},
		// A row with its own reading keeps it; the Unraid one is not written too.
		{Device: "sdc", WWN: "0x5000c500aaaa0003", Health: "PASSED", Temperature: 41, Collection: &diskinventory.CollectionStatus{
			Temperature: diskinventory.Available("smartctl")}},
		// The agent's native standby row for a disk Unraid reports spun down.
		{Device: "sdd", Serial: "SER-D", Health: "UNKNOWN", Standby: true, Collection: &diskinventory.CollectionStatus{
			Temperature: diskinventory.Unavailable("unraid", "disk is reported spun down")}},
		// Controller members behind one kernel block device, which the Unraid
		// row describes as a whole: neither member takes its temperature.
		{Device: "sdh", Controller: "sdh", Target: "megaraid,0", WWN: "0x5000c500bbbb0001", Health: "PASSED", Collection: &diskinventory.CollectionStatus{
			Temperature: diskinventory.Unavailable("smartctl", "SMART probe returned no usable temperature data")}},
		{Device: "sdh", Controller: "sdh", Target: "megaraid,1", WWN: "0x5000c500bbbb0002", Health: "PASSED", Collection: &diskinventory.CollectionStatus{
			Temperature: diskinventory.Unavailable("smartctl", "SMART probe returned no usable temperature data")}},
		// A member whose own serial is the Unraid row's is the disk that row
		// describes, though its path is shared.
		{Device: "sdi", Controller: "sdi", Target: "megaraid,0", Serial: "SER-I", Health: "UNKNOWN", Collection: &diskinventory.CollectionStatus{
			Temperature: diskinventory.Unavailable("smartctl", "SMART probe returned no usable temperature data")}},
		{Device: "sdi", Controller: "sdi", Target: "megaraid,1", Serial: "SER-I2", Health: "UNKNOWN", Collection: &diskinventory.CollectionStatus{
			Temperature: diskinventory.Unavailable("smartctl", "SMART probe returned no usable temperature data")}},
		// A placeholder serial is no evidence of which member the row is.
		{Device: "sdn", Controller: "sdn", Target: "megaraid,0", Serial: "UNKNOWN", WWN: "0x5000c500cccc0001", Health: "UNKNOWN", Collection: &diskinventory.CollectionStatus{
			Temperature: diskinventory.Unavailable("smartctl", "SMART probe returned no usable temperature data")}},
		{Device: "sdn", Controller: "sdn", Target: "megaraid,1", Serial: "UNKNOWN", WWN: "0x5000c500cccc0002", Health: "UNKNOWN", Collection: &diskinventory.CollectionStatus{
			Temperature: diskinventory.Unavailable("smartctl", "SMART probe returned no usable temperature data")}},
		// A placeholder serial matches no Unraid row; the row's path finds its own.
		{Device: "sdo", Serial: "0123456789", WWN: "0x5000c500eeee0001", Health: "UNKNOWN", Collection: &diskinventory.CollectionStatus{
			Temperature: diskinventory.Unavailable("smartctl", "SMART probe returned no usable temperature data")}},
		// The only reported member behind a controller path is still a member.
		{Device: "sdq", Controller: "sdq", Target: "megaraid,0", WWN: "0x5000c500ffff0001", Health: "UNKNOWN", Collection: &diskinventory.CollectionStatus{
			Temperature: diskinventory.Unavailable("smartctl", "SMART probe returned no usable temperature data")}},
		// Without a serial or WWN, a member charts its own reading under its
		// topology key.
		{Device: "sdr", Controller: "sdr", Target: "megaraid,0", Health: "PASSED", Temperature: 35, Collection: &diskinventory.CollectionStatus{
			Temperature: diskinventory.Available("smartctl")}},
		// A SMART row's key is its own: the merged disk shows the row's reading,
		// here not collected, and the Unraid one is not charted over it.
		{Device: "sdm", WWN: "0x5000c500dddd0001", Health: "PASSED", Temperature: 40, Collection: &diskinventory.CollectionStatus{
			Temperature: diskinventory.Unavailable("smartctl", "disk is in standby")}},
	}
	host.Unraid = &models.HostUnraidStorage{ArrayStarted: true, Disks: []models.HostUnraidDisk{
		{Device: "sdb", Serial: "SER-B", Name: "disk1", Role: "data", Status: "online", Temperature: 38},
		{Device: "sdc", Serial: "SER-C", Name: "disk2", Role: "data", Status: "online", Temperature: 39},
		{Device: "sdd", Serial: "SER-D", Name: "disk3", Role: "data", Status: "online", Temperature: 35, SpunDown: true},
		// Members with no SMART row at all.
		{Device: "sde", Serial: "SER-E", Name: "disk4", Role: "data", Status: "online", Temperature: 36},
		{Device: "sdf", Serial: "SER-F", Name: "disk5", Role: "data", Status: "online", Temperature: 33, SpunDown: true},
		{Device: "sdg", Name: "disk6", Role: "data", Status: "online", Temperature: 37},
		{Device: "sdh", Serial: "SER-H", Name: "disk7", Role: "data", Status: "online", Temperature: 42},
		{Device: "sdi", Serial: "SER-I", Name: "disk8", Role: "data", Status: "online", Temperature: 44},
		// Rows sharing one history key are one disk, showing the latest reading.
		{Device: "sdj", Name: "disk9", Role: "data", Status: "online"},
		{Device: "sdj", Name: "disk10", Role: "data", Status: "online", Temperature: 43},
		{Device: "sdk", Name: "disk11", Role: "data", Status: "online", Temperature: 38},
		{Device: "sdk", Name: "disk12", Role: "data", Status: "online", Temperature: 39},
		{Device: "sdl", Name: "disk13", Role: "data", Status: "online", Temperature: 40},
		{Device: "sdl", Name: "disk14", Role: "data", Status: "online", Temperature: 32, SpunDown: true},
		{Device: "sdn", Serial: "UNKNOWN", Name: "disk15", Role: "data", Status: "online", Temperature: 45},
		{Device: "sdm", Serial: "SER-M", Name: "disk16", Role: "data", Status: "online", Temperature: 37},
		{Device: "sdp", Serial: "0123456789", Name: "disk17", Role: "data", Status: "online", Temperature: 30},
		{Device: "sdo", Serial: "0123456789", Name: "disk18", Role: "data", Status: "online", Temperature: 46},
		{Device: "sdq", Serial: "SER-Q", Name: "disk19", Role: "data", Status: "online", Temperature: 47},
		{Device: "sdr", Serial: "SER-R", Name: "disk20", Role: "data", Status: "online", Temperature: 36},
	}}
	// Disks by usable serial, else WWN, else device path, and the reading each
	// shows as collected.
	want := map[string]float64{
		"SER-B": 38, "SER-C": 41, "SER-D": 0, "SER-E": 36, "SER-F": 0, "sdg": 37,
		"SER-H": 42, "0x5000c500bbbb0001": 0, "0x5000c500bbbb0002": 0,
		"SER-I": 44, "SER-I2": 0, "sdj": 43, "sdk": 39, "sdl": 0,
		"sdn": 45, "0x5000c500cccc0001": 0, "0x5000c500cccc0002": 0, "SER-M": 0,
		"0x5000c500eeee0001": 46, "sdo": 46, "sdp": 30, "0x5000c500ffff0001": 0, "SER-Q": 47,
		"sdr": 35, "SER-R": 36,
	}

	m.writeHostSMARTMetrics(host, now)
	// Once the host's lease expires its last report is kept, offline, as
	// context; nothing it carries is a current reading.
	reported := host
	reported.Sensors.SMART = append([]models.HostDiskSMART(nil), host.Sensors.SMART...)
	state := &models.State{Hosts: []models.Host{reported}}
	offline, expired := state.ExpireHostTelemetry(host.ID, host.LastSeen)
	if !expired {
		t.Fatal("host telemetry did not expire")
	}
	m.writeHostSMARTMetrics(offline, now.Add(30*time.Second))
	store.Flush()

	live := unifiedresources.NewRegistry(nil)
	live.IngestSnapshot(models.StateSnapshot{Hosts: []models.Host{host}})
	rehydrated := unifiedresources.NewRegistry(nil)
	rehydrated.IngestResources(live.List())
	for _, tc := range []struct {
		name     string
		registry *unifiedresources.ResourceRegistry
	}{{"live", live}, {"rehydrated", rehydrated}} {
		disks := tc.registry.ListByType(unifiedresources.ResourceTypePhysicalDisk)
		if len(disks) != len(want) {
			t.Fatalf("%s: disks = %d, want %d", tc.name, len(disks), len(want))
		}
		for _, disk := range disks {
			device := disk.PhysicalDisk.Serial
			if !diskinventory.IsUsableHardwareID(device) {
				device = ""
			}
			if device == "" {
				device = disk.PhysicalDisk.WWN
			}
			if device == "" {
				device = disk.PhysicalDisk.DevPath
			}
			wantTemp, ok := want[device]
			if !ok {
				t.Fatalf("%s: unexpected disk %s at %q", tc.name, disk.ID, device)
			}
			shown := 0.0
			if diskinventory.TemperatureCollected(disk.PhysicalDisk.Temperature, disk.PhysicalDisk.Collection) {
				shown = float64(disk.PhysicalDisk.Temperature)
			}
			if shown != wantTemp {
				t.Fatalf("%s: disk %s shows %v°C as collected, want %v°C", tc.name, device, shown, wantTemp)
			}
			target := tc.registry.MetricsTarget(disk.ID)
			if target == nil {
				t.Fatalf("%s: disk %s has no metrics target", tc.name, device)
			}
			points, err := store.Query(target.ResourceType, target.ResourceID, "smart_temp", now.Add(-time.Minute), now.Add(time.Minute), 0)
			if err != nil {
				t.Fatal(err)
			}
			switch {
			case wantTemp == 0 && len(points) != 0:
				t.Errorf("%s: disk %s shows no collected temperature but target %+v charts %+v", tc.name, device, *target, points)
			case wantTemp > 0 && (len(points) != 1 || points[0].Value != wantTemp || !points[0].Timestamp.Equal(now)):
				t.Errorf("%s: disk %s shows %v°C but target %+v charts %+v, want one sample at the online report", tc.name, device, wantTemp, *target, points)
			}
		}
	}
}

// A SMART row found only by device path may describe the slot's previous
// occupant. It is refused when its identity contradicts the Proxmox disk's,
// judged only on what both producers report like for like: WWNs for every
// disk, serials only where Proxmox's serial is the drive's own, for NVMe and
// for disks the kernel reaches through libata (vendor "ATA"). A SAS disk's
// Proxmox serial may be a transport address and a USB bridge's is its own.
// Reporter spellings of one identity (udev's underscores, a T10 designator
// ending with the drive serial, the agent's NAA fields, udev's 64-bit NAA 6
// prefix) are not disagreement.
func TestHostAgentSMARTMergeRefusesPathMatchAcrossContradictingIdentity(t *testing.T) {
	type mergeCase struct {
		name  string
		disk  models.PhysicalDisk
		smart models.HostDiskSMART
		// others are further rows on the same path, which make it ambiguous.
		others []models.HostDiskSMART
		merged bool
		serial string
	}
	smartctlSerial := &diskinventory.CollectionStatus{Serial: diskinventory.Available("smartctl")}
	pveDisk := func(device, vendor, serial, wwn, diskType string) models.PhysicalDisk {
		return models.PhysicalDisk{
			ID: "pve1-node1-" + device, Node: "node1", Instance: "pve1", DevPath: "/dev/" + device,
			Vendor: vendor, Serial: serial, WWN: wwn, Type: diskType, Health: "PASSED",
		}
	}
	smartRow := func(device, serial, wwn, diskType string) models.HostDiskSMART {
		return models.HostDiskSMART{
			Device: "/dev/" + device, Serial: serial, WWN: wwn, Type: diskType,
			Health: "FAILED", Temperature: 47, Collection: diskinventory.CloneStatus(smartctlSerial),
		}
	}
	for _, tc := range []mergeCase{
		{
			name:  "SAS replacement beside a stale row with another WWN",
			disk:  pveDisk("sda", "SEAGATE", "5000c500b0000001", "0x5000c500b0000001", "unknown"),
			smart: smartRow("sda", "ZR5OLD0001", "naa.5000c500a0000001", "sas"),
		},
		{
			name:  "SATA replacement beside a stale row with another serial",
			disk:  pveDisk("sdb", "ATA", "WD-NEW0001", "", "hdd"),
			smart: smartRow("sdb", "WD-OLD0001", "", "sata"),
		},
		{
			name:   "same SAS disk whose agent row has no WWN",
			disk:   pveDisk("sdc", "SEAGATE", "5000c500a0000003", "0x5000c500a0000003", "unknown"),
			smart:  smartRow("sdc", "ZR5TESTA0003", "", "sas"),
			merged: true, serial: "ZR5TESTA0003",
		},
		{
			name:   "same SAS disk renamed, agent WWN in smartctl NAA fields",
			disk:   pveDisk("sdd", "SEAGATE", "5000c500a0000004", "0x5000c500a0000004", "unknown"),
			smart:  smartRow("sdx", "ZR5TESTA0004", "5-c50-a0000004", "sas"),
			merged: true, serial: "ZR5TESTA0004",
		},
		{
			name:   "same disk with udev's spelling of the serial",
			disk:   pveDisk("sde", "ATA", "WD-WX_1234", "", "hdd"),
			smart:  smartRow("sde", "WD-WX 1234", "", "sata"),
			merged: true, serial: "WD-WX_1234",
		},
		{
			name:  "NVMe replacement whose serial looks like a SAS address",
			disk:  pveDisk("nvme0n1", "", "50026B7282A0FB69", "", "nvme"),
			smart: smartRow("nvme0n1", "50026B7282A0FB11", "", "nvme"),
		},
		{
			name:  "ambiguous path where only one row contradicts",
			disk:  pveDisk("sdg", "ATA", "WD-NEW0007", "", "hdd"),
			smart: smartRow("sdg", "WD-OLD0007", "", "sata"),
			others: []models.HostDiskSMART{
				smartRow("sdg", "", "", "sata"),
			},
		},
		{
			name:   "same disk in a USB enclosure whose serial udev reports",
			disk:   pveDisk("sdh", "JMicron", "000000000000000F1E2D", "", "usb"),
			smart:  smartRow("sdh", "WD-WX0008", "", "sata"),
			merged: true, serial: "000000000000000F1E2D",
		},
		{
			name:   "same SSD in a USB enclosure Proxmox types ssd",
			disk:   pveDisk("sdj", "ASMT", "00000000000000000009", "", "ssd"),
			smart:  smartRow("sdj", "S3Z9NB0K123456X", "", "sata"),
			merged: true, serial: "00000000000000000009",
		},
		{
			name:   "same disk behind a T10 designator ending with its serial",
			disk:   pveDisk("sdi", "ATA", "ATA_ST4000NM000A-2HZ_ZC1ABCDE", "", "unknown"),
			smart:  smartRow("sdi", "ZC1ABCDE", "", "sata"),
			merged: true, serial: "ATA_ST4000NM000A-2HZ_ZC1ABCDE",
		},
		{
			name:   "same NAA 6 volume with udev's 64-bit WWN",
			disk:   pveDisk("sdf", "HP", "PDNLH0BRH8V0AB", "0x600508b1001c4d5e", "hdd"),
			smart:  smartRow("sdf", "", "naa.600508b1001c4d5e7f8a9b0c1d2e3f40", "scsi"),
			merged: true, serial: "PDNLH0BRH8V0AB",
		},
	} {
		t.Run(tc.name, func(t *testing.T) {
			got := mergeHostAgentSMARTIntoDisks(
				[]models.PhysicalDisk{tc.disk},
				[]models.Node{{Name: "node1", LinkedAgentID: "agent-1"}},
				[]models.Host{{ID: "agent-1", Sensors: models.HostSensorSummary{
					SMART: append([]models.HostDiskSMART{tc.smart}, tc.others...),
				}}},
				nil,
			)[0]
			if !tc.merged {
				if got.Serial != tc.disk.Serial || got.WWN != tc.disk.WWN || got.Type != tc.disk.Type ||
					got.Temperature != 0 || got.Health != "PASSED" {
					t.Fatalf("contradicting row was merged into the disk: %+v", got)
				}
				return
			}
			if got.Serial != tc.serial || got.Temperature != 47 || got.Health != "FAILED" {
				t.Fatalf("same disk's row was not merged: serial %q (want %q), temperature %d, health %q",
					got.Serial, tc.serial, got.Temperature, got.Health)
			}
		})
	}
}

// Proxmox's disk inventory reports a serial or WWN udev cannot read as the
// literal "unknown". The poller records it as unreported rather than as a
// collected serial; the disk's identity keys were already blind to it.
func TestProxmoxUnknownDiskIdentityIsRecordedAsUnreported(t *testing.T) {
	t.Setenv("PULSE_DATA_DIR", t.TempDir())
	state := models.NewState()
	state.UpdateNodesForInstance("pve1", []models.Node{{
		ID: "pve1-node1", Name: "node1", Instance: "pve1", Status: "online", LastSeen: time.Now(),
	}})
	alertManager := alerts.NewManager()
	t.Cleanup(alertManager.Stop)
	m := &Monitor{
		state: state, alertManager: alertManager,
		startTime: time.Now().Add(-time.Hour), lastPhysicalDiskPoll: make(map[string]time.Time),
	}
	// A record from before the placeholder was recorded as unreported must
	// not carry it back in as the disk's last known serial.
	state.UpdatePhysicalDisks("pve1", []models.PhysicalDisk{{
		ID:   unifiedresources.ProxmoxPhysicalDiskSourceID("pve1", "node1", "/dev/sdb", "", ""),
		Node: "node1", Instance: "pve1", DevPath: "/dev/sdb", Serial: "unknown", WWN: "unknown",
		Collection: &diskinventory.CollectionStatus{Serial: diskinventory.Available("proxmox_disks")},
	}})
	client := &slotDiskPVEClient{}
	client.setDisk(proxmox.Disk{
		DevPath: "/dev/sdb", Model: "USB3.0 Bridge", Serial: "unknown", WWN: "unknown",
		Type: "hdd", Health: "PASSED", Wearout: 100, Size: 1000204886016,
	})
	started := time.Now()
	m.maybePollPhysicalDisksAsync(context.Background(), "pve1", &config.PVEInstance{}, client,
		[]proxmox.Node{{Node: "node1", Status: "online"}}, map[string]string{"node1": "online"}, nil)
	deadline := started.Add(3 * time.Second)
	for disks := state.GetSnapshot().PhysicalDisks; len(disks) != 1 || disks[0].LastChecked.Before(started); disks = state.GetSnapshot().PhysicalDisks {
		if time.Now().After(deadline) {
			t.Fatal("physical disk poll did not land in state")
		}
		time.Sleep(10 * time.Millisecond)
	}
	got := state.GetSnapshot().PhysicalDisks[0]
	if got.Serial != "" || got.WWN != "" {
		t.Fatalf("placeholder identity recorded as reported: serial %q, wwn %q", got.Serial, got.WWN)
	}
	if got.Collection == nil || got.Collection.Serial.State != diskinventory.FieldMissing {
		t.Fatalf("serial collection = %+v, want missing", got.Collection)
	}
	placeholder := got
	placeholder.Serial, placeholder.WWN = "unknown", "unknown"
	if unifiedresources.PhysicalDiskMetricID(got) != unifiedresources.PhysicalDiskMetricID(placeholder) {
		t.Fatalf("metric key moved: %q, was %q",
			unifiedresources.PhysicalDiskMetricID(got), unifiedresources.PhysicalDiskMetricID(placeholder))
	}
}

// The registry joins a Proxmox disk to its linked agent's disk on the same
// path when the agent reports SAS, because Proxmox puts the SAS address in
// the serial (#1595). A WWN naming another disk refuses that join: the agent's
// row may be the slot's previous occupant, retained by a silent agent, and the
// join used to hand the replacement that disk's serial, readings and canonical
// resource.
func TestRegistrySASPathJoinRefusesContradictingWWN(t *testing.T) {
	const replacementWWN = "0x5000c500bbbb0002"
	for _, tc := range []struct {
		name     string
		agentWWN string
		joined   bool
	}{
		{name: "retained row of the previous occupant", agentWWN: "5-c50-aaaa0001"},
		{name: "same disk in the agent's WWN spelling", agentWWN: "5-c50-bbbb0002", joined: true},
		{name: "same disk without an agent WWN", agentWWN: "", joined: true},
	} {
		t.Run(tc.name, func(t *testing.T) {
			now := time.Now()
			node := models.Node{
				ID: "pve1-node1", Name: "node1", Instance: "pve1", Status: "online",
				LastSeen: now, LinkedAgentID: "agent-1",
			}
			host := models.Host{
				ID: "agent-1", Hostname: "node1", LinkedNodeID: "pve1-node1", Status: "online", LastSeen: now,
				Sensors: models.HostSensorSummary{SMART: []models.HostDiskSMART{{
					Device: "/dev/sdb", Serial: "ZR5AGENT0001", WWN: tc.agentWWN, Type: "sas",
					Health: "FAILED", Temperature: 52,
					Collection: &diskinventory.CollectionStatus{
						Serial:      diskinventory.Available("smartctl"),
						Temperature: diskinventory.Available("smartctl"),
					},
				}}},
			}
			replacement := models.PhysicalDisk{
				ID: "pve1-node1--dev-sdb", Node: "node1", Instance: "pve1", DevPath: "/dev/sdb",
				Serial: strings.TrimPrefix(replacementWWN, "0x"), WWN: replacementWWN, Type: "unknown",
				Health: "PASSED", Wearout: -1, LastChecked: now,
			}
			registry := unifiedresources.NewRegistry(nil)
			registry.IngestSnapshot(models.StateSnapshot{
				Nodes: []models.Node{node}, Hosts: []models.Host{host}, PhysicalDisks: []models.PhysicalDisk{replacement},
			})

			var proxmoxDisk *unifiedresources.Resource
			disks := registry.ListByType(unifiedresources.ResourceTypePhysicalDisk)
			for index := range disks {
				if hasIssue1595Source(disks[index].Sources, unifiedresources.SourceProxmox) {
					proxmoxDisk = &disks[index]
				}
			}
			if proxmoxDisk == nil || proxmoxDisk.PhysicalDisk == nil {
				t.Fatalf("no canonical Proxmox disk among %d disks", len(disks))
			}
			joined := hasIssue1595Source(proxmoxDisk.Sources, unifiedresources.SourceAgent)
			if joined != tc.joined || (tc.joined && len(disks) != 1) || (!tc.joined && len(disks) != 2) {
				t.Fatalf("joined = %v with %d disks, want joined = %v", joined, len(disks), tc.joined)
			}
			if !tc.joined && (proxmoxDisk.PhysicalDisk.Serial == "ZR5AGENT0001" ||
				proxmoxDisk.PhysicalDisk.Temperature != 0 || proxmoxDisk.PhysicalDisk.Health == "FAILED") {
				t.Fatalf("replacement took the retained row's identity or readings: %+v", proxmoxDisk.PhysicalDisk)
			}
		})
	}
}

// A registry rehydrated from a persisted unified snapshot seeds each source
// key's mapping before the next poll arrives. Proxmox keys a disk by its slot,
// so a disk swapped into the slot arrives under the previous disk's key. The
// mapping is refused when the two carry hardware identities naming different
// disks, and nothing else may merge the replacement back into the old disk,
// so it gets its own canonical resource instead of the old disk's ID, serial,
// temperature and failed health. Identity missing on one side keeps the
// mapping, and so does a serial that may not be the drive's own (a SAS
// address, a SCSI designator, a USB bridge's serial) set against the agent's.
func TestRegistrySeededSlotMappingRefusesReplacedDisk(t *testing.T) {
	const slotSourceID = "pve1-node1--dev-sdb"
	// Each kind is the slot's previous disk as Proxmox and the agent reported
	// it; the registry merged the two reports by WWN or by path, unless the
	// case has Proxmox alone report it.
	kinds := map[string]struct{ agentType, vendor, proxmoxType, proxmoxSerial string }{
		"sata":    {agentType: "sata", vendor: "ATA", proxmoxType: "hdd", proxmoxSerial: "ZR5A0001"},
		"sas":     {agentType: "sas", vendor: "SEAGATE", proxmoxType: "hdd", proxmoxSerial: "5000c500aaaa0003"},
		"usb":     {agentType: "usb", vendor: "JMicron", proxmoxType: "usb", proxmoxSerial: "BRIDGE0001"},
		"scsi":    {agentType: "sata", vendor: "SEAGATE", proxmoxType: "hdd", proxmoxSerial: "Z1Z0VPD0001"},
		"usb-ata": {agentType: "sata", vendor: "ATA", proxmoxType: "usb", proxmoxSerial: "BRIDGE0001"},
	}
	for _, tc := range []struct {
		name          string
		kind          string
		proxmoxFirst  bool
		proxmoxOnly   bool
		legacyAgent   bool
		serial        string
		wwn           string
		replacedDrive bool
	}{
		{name: "replacement disk", kind: "sata", serial: "ZR5B0002", wwn: "0x5000c500bbbb0002", replacedDrive: true},
		{name: "replacement disk without a WWN", kind: "sata", serial: "ZR5B0002", wwn: "unknown", replacedDrive: true},
		{name: "same disk", kind: "sata", serial: "ZR5A0001", wwn: "0x5000c500aaaa0001"},
		{name: "same disk without a serial", kind: "sata", serial: "unknown", wwn: "0x5000c500aaaa0001"},
		{name: "SAS disk whose Proxmox serial is a SAS address", kind: "sas", serial: "5000c500aaaa0003", wwn: "0x5000c500aaaa0001"},
		{name: "SAS disk without a Proxmox WWN", kind: "sas", serial: "5000c500aaaa0003", wwn: "unknown"},
		{name: "USB disk whose Proxmox serial is the bridge's", kind: "usb", serial: "BRIDGE0001", wwn: "unknown"},
		{name: "USB bridge serial under Proxmox vendor ATA", kind: "usb-ata", serial: "BRIDGE0001", wwn: "unknown"},
		{name: "disk whose Proxmox serial is a SCSI designator", kind: "scsi", serial: "Z1Z0VPD0001", wwn: "unknown"},
		{name: "SAS replacement disk", kind: "sas", serial: "5000c500bbbb0003", wwn: "0x5000c500bbbb0002", replacedDrive: true},
		{name: "drive swapped in the same USB enclosure", kind: "usb", proxmoxFirst: true,
			serial: "BRIDGE0001", wwn: "0x5000c500bbbb0002", replacedDrive: true},
		{name: "SCSI designator from an agent without collection status", kind: "scsi", legacyAgent: true,
			serial: "Z1Z0VPD0001", wwn: "unknown"},
		{name: "replacement Proxmox alone reports, without a WWN", kind: "scsi", proxmoxOnly: true,
			serial: "Z1Z0VPD0002", wwn: "unknown", replacedDrive: true},
	} {
		t.Run(tc.name, func(t *testing.T) {
			kind := kinds[tc.kind]
			now := time.Now()
			node := models.Node{
				ID: "pve1-node1", Name: "node1", Instance: "pve1", Status: "online",
				LastSeen: now, LinkedAgentID: "agent-1",
			}
			host := models.Host{
				ID: "agent-1", Hostname: "node1", LinkedNodeID: "pve1-node1", Status: "online", LastSeen: now,
				Sensors: models.HostSensorSummary{SMART: []models.HostDiskSMART{{
					Device: "/dev/sdb", Serial: "ZR5A0001", WWN: "5-c50-aaaa0001", Type: kind.agentType,
					Health: "FAILED", Temperature: 55,
					Collection: &diskinventory.CollectionStatus{
						Serial:      diskinventory.Available("smartctl"),
						Temperature: diskinventory.Available("smartctl"),
					},
				}}},
			}
			// As monitor_pve builds the slot's row from disks/list.
			slotDisk := func(serial, wwn, health string) models.PhysicalDisk {
				disk := models.PhysicalDisk{
					ID: slotSourceID, Node: "node1", Instance: "pve1", DevPath: "/dev/sdb", Vendor: kind.vendor,
					Serial: proxmoxReportedDiskIdentity(serial), WWN: proxmoxReportedDiskIdentity(wwn),
					Type: kind.proxmoxType, Health: health, Wearout: -1, LastChecked: now,
					Collection: &diskinventory.CollectionStatus{
						Serial: diskinventory.Missing("proxmox_disks", "disk serial was not reported"),
					},
				}
				if disk.Serial != "" {
					disk.Collection.Serial = diskinventory.Available("proxmox_disks")
				}
				return disk
			}
			previousDisk := slotDisk(kind.proxmoxSerial, "0x5000c500aaaa0001", "FAILED")
			previous := unifiedresources.NewRegistry(nil)
			if tc.proxmoxFirst {
				previous.IngestSnapshot(models.StateSnapshot{
					Nodes: []models.Node{node}, PhysicalDisks: []models.PhysicalDisk{previousDisk},
				})
			}
			hosts := []models.Host{host}
			if tc.proxmoxOnly {
				hosts = nil
			}
			if tc.legacyAgent {
				host.Sensors.SMART[0].Collection = nil
			}
			previous.IngestSnapshot(models.StateSnapshot{
				Nodes: []models.Node{node}, Hosts: hosts, PhysicalDisks: []models.PhysicalDisk{previousDisk},
			})
			persisted := previous.ListByType(unifiedresources.ResourceTypePhysicalDisk)
			if len(persisted) != 1 || persisted[0].PhysicalDisk == nil ||
				!hasIssue1595Source(persisted[0].Sources, unifiedresources.SourceProxmox) ||
				hasIssue1595Source(persisted[0].Sources, unifiedresources.SourceAgent) == tc.proxmoxOnly {
				t.Fatalf("want one disk merged from Proxmox and the agent (Proxmox alone: %v), got %+v", tc.proxmoxOnly, persisted)
			}
			oldID, oldDisk := persisted[0].ID, *persisted[0].PhysicalDisk

			// Restart: rehydrate from the persisted resources, then poll
			// Proxmox before the agent has reported again. A second restart
			// rehydrates from the result, where both disks may claim the slot.
			current := slotDisk(tc.serial, tc.wwn, "PASSED")
			seed := previous.List()
			slotID := ""
			for restart := 1; restart <= 2; restart++ {
				rehydrated := unifiedresources.NewRegistry(nil)
				rehydrated.IngestResources(seed)
				rehydrated.IngestSnapshot(models.StateSnapshot{
					Nodes: []models.Node{node}, PhysicalDisks: []models.PhysicalDisk{current},
				})
				seed = rehydrated.List()

				var slot *unifiedresources.Resource
				disks := rehydrated.ListByType(unifiedresources.ResourceTypePhysicalDisk)
				for index := range disks {
					for _, target := range rehydrated.SourceTargets(disks[index].ID) {
						if target.Source == unifiedresources.SourceProxmox && target.SourceID == slotSourceID {
							slot = &disks[index]
						}
					}
				}
				if slot == nil || slot.PhysicalDisk == nil {
					t.Fatalf("restart %d: no disk holds the Proxmox slot key among %d disks", restart, len(disks))
				}
				if slotID != "" && slot.ID != slotID {
					t.Fatalf("restart %d: slot moved from %s to %s", restart, slotID, slot.ID)
				}
				slotID = slot.ID
				if !tc.replacedDrive {
					if slot.ID != oldID || len(disks) != 1 {
						t.Fatalf("restart %d: slot disk = %s with %d disks, want the persisted disk %s alone",
							restart, slot.ID, len(disks), oldID)
					}
					continue
				}
				if slot.ID == oldID || len(disks) != 2 {
					t.Fatalf("restart %d: replacement took the persisted disk's canonical ID %s (%d disks)", restart, oldID, len(disks))
				}
				if slot.PhysicalDisk.Serial != current.Serial || slot.PhysicalDisk.WWN != current.WWN ||
					slot.PhysicalDisk.Temperature != 0 || slot.PhysicalDisk.Health != "PASSED" {
					t.Fatalf("restart %d: replacement took the persisted disk's identity or readings: %+v", restart, slot.PhysicalDisk)
				}
				for _, disk := range disks {
					if disk.ID == oldID && (disk.PhysicalDisk == nil ||
						disk.PhysicalDisk.Serial != oldDisk.Serial || disk.PhysicalDisk.WWN != oldDisk.WWN) {
						t.Fatalf("restart %d: persisted disk was rewritten: %+v", restart, disk.PhysicalDisk)
					}
				}
			}
		})
	}
}