Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

72.2 KB
Raw Blame
package models

import (
	"encoding/json"
	"strings"
	"testing"
	"time"
)

func TestUpsertHost(t *testing.T) {
	state := NewState()

	// Test insert new host
	host1 := Host{
		ID:       "host-1",
		Hostname: "server-1",
		Status:   "online",
	}
	state.UpsertHost(host1)

	hosts := state.GetHosts()
	if len(hosts) != 1 {
		t.Fatalf("Expected 1 host, got %d", len(hosts))
	}
	if hosts[0].ID != "host-1" {
		t.Errorf("Expected host ID 'host-1', got %q", hosts[0].ID)
	}

	// Test update existing host
	host1Updated := Host{
		ID:       "host-1",
		Hostname: "server-1",
		Status:   "offline",
	}
	state.UpsertHost(host1Updated)

	hosts = state.GetHosts()
	if len(hosts) != 1 {
		t.Fatalf("Expected 1 host after update, got %d", len(hosts))
	}
	if hosts[0].Status != "offline" {
		t.Errorf("Expected status 'offline', got %q", hosts[0].Status)
	}

	// Test multiple hosts and sorting
	state.UpsertHost(Host{ID: "host-2", Hostname: "alpha-server"})
	state.UpsertHost(Host{ID: "host-3", Hostname: "zulu-server"})

	hosts = state.GetHosts()
	if len(hosts) != 3 {
		t.Fatalf("Expected 3 hosts, got %d", len(hosts))
	}

	// Hosts should be sorted by hostname
	if hosts[0].Hostname != "alpha-server" {
		t.Errorf("First host should be 'alpha-server', got %q", hosts[0].Hostname)
	}
}

func TestUpsertHostPreservesReportedCommandAuthority(t *testing.T) {
	state := NewState()
	state.UpsertHost(Host{
		ID: "host-authority",
		AgentPrivilege: &AgentPrivilegeStatus{
			RunningAsRoot:    true,
			CommandAuthority: "monitoring-only",
		},
	})

	hosts := state.GetHosts()
	if len(hosts) != 1 || hosts[0].AgentPrivilege == nil {
		t.Fatalf("reported privilege missing from state: %+v", hosts)
	}
	if got := hosts[0].AgentPrivilege.CommandAuthority; got != "monitoring-only" {
		t.Fatalf("command authority = %q, want monitoring-only", got)
	}
}

func TestGetHosts_Copy(t *testing.T) {
	state := NewState()

	state.UpsertHost(Host{ID: "host-1", Hostname: "server-1"})

	hosts1 := state.GetHosts()
	hosts2 := state.GetHosts()

	// Modify one slice
	if len(hosts1) > 0 {
		hosts1[0].Hostname = "modified"
	}

	// Other slice should be unchanged
	if len(hosts2) > 0 && hosts2[0].Hostname == "modified" {
		t.Error("GetHosts should return a copy, not the same slice")
	}

	// Original state should be unchanged
	hosts3 := state.GetHosts()
	if hosts3[0].Hostname == "modified" {
		t.Error("State should be unchanged by modifications to returned slice")
	}
}

func TestRemoveHost(t *testing.T) {
	state := NewState()

	// Insert hosts
	state.UpsertHost(Host{ID: "host-1", Hostname: "server1"})
	state.UpsertHost(Host{ID: "host-2", Hostname: "server2"})

	// Remove existing host
	removed, ok := state.RemoveHost("host-1")
	if !ok {
		t.Error("Expected RemoveHost to return true for existing host")
	}
	if removed.ID != "host-1" {
		t.Errorf("Expected removed host ID 'host-1', got %q", removed.ID)
	}

	hosts := state.GetHosts()
	if len(hosts) != 1 {
		t.Fatalf("Expected 1 host after removal, got %d", len(hosts))
	}
	if hosts[0].ID != "host-2" {
		t.Errorf("Remaining host should be 'host-2', got %q", hosts[0].ID)
	}

	// Remove non-existing host
	removed, ok = state.RemoveHost("non-existent")
	if ok {
		t.Error("Expected RemoveHost to return false for non-existent host")
	}
	if removed.ID != "" {
		t.Errorf("Expected empty Host for non-existent removal, got ID %q", removed.ID)
	}
}

func TestSetHostStatus(t *testing.T) {
	state := NewState()

	// Test with no hosts
	changed := state.SetHostStatus("host-1", "online")
	if changed {
		t.Error("SetHostStatus should return false when host doesn't exist")
	}

	// Add host and set status
	state.UpsertHost(Host{ID: "host-1", Hostname: "server1", Status: "offline"})

	changed = state.SetHostStatus("host-1", "online")
	if !changed {
		t.Error("SetHostStatus should return true when host exists")
	}

	hosts := state.GetHosts()
	if hosts[0].Status != "online" {
		t.Errorf("Expected status 'online', got %q", hosts[0].Status)
	}

	// Set same status (no change)
	changed = state.SetHostStatus("host-1", "online")
	if !changed {
		t.Error("SetHostStatus should return true even when status unchanged")
	}
}

func TestTouchHost(t *testing.T) {
	state := NewState()

	now := time.Now()

	// Test with non-existent host
	ok := state.TouchHost("host-1", now)
	if ok {
		t.Error("TouchHost should return false for non-existent host")
	}

	// Add host and touch it
	state.UpsertHost(Host{ID: "host-1", Hostname: "server1"})

	later := now.Add(time.Hour)
	ok = state.TouchHost("host-1", later)
	if !ok {
		t.Error("TouchHost should return true for existing host")
	}

	hosts := state.GetHosts()
	if !hosts[0].LastSeen.Equal(later) {
		t.Errorf("LastSeen should be updated to %v, got %v", later, hosts[0].LastSeen)
	}
}

func TestSetConnectionHealth(t *testing.T) {
	state := NewState()

	// Set connection healthy
	state.SetConnectionHealth("pve-cluster-1", true)

	snapshot := state.GetSnapshot()
	if healthy, ok := snapshot.ConnectionHealth["pve-cluster-1"]; !ok || !healthy {
		t.Error("Expected connection to be healthy")
	}

	// Set connection unhealthy
	state.SetConnectionHealth("pve-cluster-1", false)

	snapshot = state.GetSnapshot()
	if healthy, ok := snapshot.ConnectionHealth["pve-cluster-1"]; !ok || healthy {
		t.Error("Expected connection to be unhealthy")
	}

	// Multiple connections
	state.SetConnectionHealth("pve-cluster-2", true)
	state.SetConnectionHealth("pbs-instance-1", true)

	snapshot = state.GetSnapshot()
	if len(snapshot.ConnectionHealth) != 3 {
		t.Errorf("Expected 3 connection health entries, got %d", len(snapshot.ConnectionHealth))
	}
}

func TestPVETagStyleSnapshotRebuildsAggregateByInstance(t *testing.T) {
	state := NewState()

	state.MergePVETagStyle("pve-a", PVETagStyle{
		Colors: map[string]string{
			"Production": "#00ff00",
		},
		CaseSensitive: true,
	})
	state.MergePVETagStyle("pve-b", PVETagStyle{
		Colors: map[string]string{
			"Backup": "#112233",
		},
	})

	snapshot := state.GetSnapshot()
	if got := snapshot.PVETagStyles["pve-a"].Colors["Production"]; got != "#00ff00" {
		t.Fatalf("expected pve-a tag style to preserve case-sensitive color, got %q", got)
	}
	if !snapshot.PVETagStyles["pve-a"].CaseSensitive {
		t.Fatal("expected pve-a tag style to preserve case-sensitive flag")
	}
	if got := snapshot.PVETagStyles["pve-b"].Colors["backup"]; got != "#112233" {
		t.Fatalf("expected pve-b tag style to normalize non-case-sensitive tag, got %q", got)
	}

	state.MergePVETagStyle("pve-a", PVETagStyle{})
	snapshot = state.GetSnapshot()
	if _, ok := snapshot.PVETagColors["Production"]; ok {
		t.Fatalf("expected stale pve-a aggregate color to be removed, got %#v", snapshot.PVETagColors)
	}
	if got := snapshot.PVETagColors["backup"]; got != "#112233" {
		t.Fatalf("expected pve-b aggregate color to remain, got %q", got)
	}
}

func TestRemoveConnectionHealth(t *testing.T) {
	state := NewState()

	// Set up connection health entries
	state.SetConnectionHealth("pve-cluster-1", true)
	state.SetConnectionHealth("pve-cluster-2", false)

	// Remove one
	state.RemoveConnectionHealth("pve-cluster-1")

	snapshot := state.GetSnapshot()
	if _, ok := snapshot.ConnectionHealth["pve-cluster-1"]; ok {
		t.Error("Expected pve-cluster-1 to be removed")
	}
	if _, ok := snapshot.ConnectionHealth["pve-cluster-2"]; !ok {
		t.Error("Expected pve-cluster-2 to still exist")
	}

	// Remove non-existent (should not panic)
	state.RemoveConnectionHealth("non-existent")

	// Remove remaining
	state.RemoveConnectionHealth("pve-cluster-2")
	snapshot = state.GetSnapshot()
	if len(snapshot.ConnectionHealth) != 0 {
		t.Errorf("Expected empty connection health map, got %d entries", len(snapshot.ConnectionHealth))
	}
}

func TestUpdatePBSBackups(t *testing.T) {
	state := NewState()

	now := time.Now()

	// Add backups from first instance
	backups1 := []PBSBackup{
		{ID: "backup-1", Instance: "pbs-1", BackupTime: now},
		{ID: "backup-2", Instance: "pbs-1", BackupTime: now.Add(-time.Hour)},
	}
	state.UpdatePBSBackups("pbs-1", backups1)

	snapshot := state.GetSnapshot()
	if len(snapshot.PBSBackups) != 2 {
		t.Fatalf("Expected 2 backups, got %d", len(snapshot.PBSBackups))
	}

	// Add backups from second instance
	backups2 := []PBSBackup{
		{ID: "backup-3", Instance: "pbs-2", BackupTime: now.Add(-30 * time.Minute)},
	}
	state.UpdatePBSBackups("pbs-2", backups2)

	snapshot = state.GetSnapshot()
	if len(snapshot.PBSBackups) != 3 {
		t.Fatalf("Expected 3 backups, got %d", len(snapshot.PBSBackups))
	}

	// Update first instance (should replace its backups)
	backups1Updated := []PBSBackup{
		{ID: "backup-4", Instance: "pbs-1", BackupTime: now.Add(time.Hour)},
	}
	state.UpdatePBSBackups("pbs-1", backups1Updated)

	snapshot = state.GetSnapshot()
	if len(snapshot.PBSBackups) != 2 {
		t.Fatalf("Expected 2 backups after update, got %d", len(snapshot.PBSBackups))
	}

	// Verify pbs-1 backups were replaced
	hasOldBackup := false
	for _, b := range snapshot.PBSBackups {
		if b.ID == "backup-1" || b.ID == "backup-2" {
			hasOldBackup = true
			break
		}
	}
	if hasOldBackup {
		t.Error("Old pbs-1 backups should have been replaced")
	}
}

func TestUpdatePMGBackups(t *testing.T) {
	state := NewState()

	now := time.Now()

	// Add backups from first instance
	backups1 := []PMGBackup{
		{Filename: "backup-1.conf", Instance: "pmg-1", BackupTime: now},
		{Filename: "backup-2.conf", Instance: "pmg-1", BackupTime: now.Add(-time.Hour)},
	}
	state.UpdatePMGBackups("pmg-1", backups1)

	snapshot := state.GetSnapshot()
	if len(snapshot.PMGBackups) != 2 {
		t.Fatalf("Expected 2 backups, got %d", len(snapshot.PMGBackups))
	}

	// Backups should be sorted by time (newest first)
	if snapshot.PMGBackups[0].BackupTime.Before(snapshot.PMGBackups[1].BackupTime) {
		t.Error("Backups should be sorted by time descending")
	}

	// Add backups from second instance
	backups2 := []PMGBackup{
		{Filename: "backup-3.conf", Instance: "pmg-2", BackupTime: now.Add(-30 * time.Minute)},
	}
	state.UpdatePMGBackups("pmg-2", backups2)

	snapshot = state.GetSnapshot()
	if len(snapshot.PMGBackups) != 3 {
		t.Fatalf("Expected 3 backups, got %d", len(snapshot.PMGBackups))
	}

	// Update first instance with empty (should remove its backups)
	state.UpdatePMGBackups("pmg-1", []PMGBackup{})

	snapshot = state.GetSnapshot()
	if len(snapshot.PMGBackups) != 1 {
		t.Fatalf("Expected 1 backup after clearing pmg-1, got %d", len(snapshot.PMGBackups))
	}
	if snapshot.PMGBackups[0].Instance != "pmg-2" {
		t.Error("Remaining backup should be from pmg-2")
	}
}

func TestUpdatePhysicalDisks(t *testing.T) {
	state := NewState()

	// Add disks from first instance
	disks1 := []PhysicalDisk{
		{ID: "disk-1", Instance: "pve-1", Node: "node1", DevPath: "/dev/sda"},
		{ID: "disk-2", Instance: "pve-1", Node: "node1", DevPath: "/dev/sdb"},
	}
	state.UpdatePhysicalDisks("pve-1", disks1)

	snapshot := state.GetSnapshot()
	if len(snapshot.PhysicalDisks) != 2 {
		t.Fatalf("Expected 2 disks, got %d", len(snapshot.PhysicalDisks))
	}

	// Add disks from second instance
	disks2 := []PhysicalDisk{
		{ID: "disk-3", Instance: "pve-2", Node: "node2", DevPath: "/dev/sda"},
	}
	state.UpdatePhysicalDisks("pve-2", disks2)

	snapshot = state.GetSnapshot()
	if len(snapshot.PhysicalDisks) != 3 {
		t.Fatalf("Expected 3 disks, got %d", len(snapshot.PhysicalDisks))
	}

	// Update first instance (should replace its disks)
	disks1Updated := []PhysicalDisk{
		{ID: "disk-4", Instance: "pve-1", Node: "node1", DevPath: "/dev/nvme0n1"},
	}
	state.UpdatePhysicalDisks("pve-1", disks1Updated)

	snapshot = state.GetSnapshot()
	if len(snapshot.PhysicalDisks) != 2 {
		t.Fatalf("Expected 2 disks after update, got %d", len(snapshot.PhysicalDisks))
	}

	// Verify sorting by node then devpath
	if snapshot.PhysicalDisks[0].Node > snapshot.PhysicalDisks[1].Node {
		t.Error("Disks should be sorted by node")
	}
}

func TestUpdateStorageForInstance(t *testing.T) {
	state := NewState()

	// Add storage from first instance
	storage1 := []Storage{
		{ID: "storage-1", Instance: "pve-1", Name: "local"},
		{ID: "storage-2", Instance: "pve-1", Name: "ceph-pool"},
	}
	state.UpdateStorageForInstance("pve-1", storage1)

	snapshot := state.GetSnapshot()
	if len(snapshot.Storage) != 2 {
		t.Fatalf("Expected 2 storage entries, got %d", len(snapshot.Storage))
	}

	// Add storage from second instance
	storage2 := []Storage{
		{ID: "storage-3", Instance: "pve-2", Name: "local"},
	}
	state.UpdateStorageForInstance("pve-2", storage2)

	snapshot = state.GetSnapshot()
	if len(snapshot.Storage) != 3 {
		t.Fatalf("Expected 3 storage entries, got %d", len(snapshot.Storage))
	}

	// Update first instance with empty (should remove its storage)
	state.UpdateStorageForInstance("pve-1", []Storage{})

	snapshot = state.GetSnapshot()
	if len(snapshot.Storage) != 1 {
		t.Fatalf("Expected 1 storage after clearing pve-1, got %d", len(snapshot.Storage))
	}
}

func TestUpdatePBSInstances(t *testing.T) {
	state := NewState()

	instances := []PBSInstance{
		{ID: "pbs-1", Name: "Backup Server 1"},
		{ID: "pbs-2", Name: "Backup Server 2"},
	}
	state.UpdatePBSInstances(instances)

	snapshot := state.GetSnapshot()
	if len(snapshot.PBSInstances) != 2 {
		t.Fatalf("Expected 2 PBS instances, got %d", len(snapshot.PBSInstances))
	}

	// Replace with different instances
	newInstances := []PBSInstance{
		{ID: "pbs-3", Name: "New Backup Server"},
	}
	state.UpdatePBSInstances(newInstances)

	snapshot = state.GetSnapshot()
	if len(snapshot.PBSInstances) != 1 {
		t.Fatalf("Expected 1 PBS instance after replacement, got %d", len(snapshot.PBSInstances))
	}
	if snapshot.PBSInstances[0].ID != "pbs-3" {
		t.Error("Expected pbs-3 instance")
	}
}

func TestUpdatePBSInstance(t *testing.T) {
	state := NewState()

	// Add first instance
	state.UpdatePBSInstance(PBSInstance{ID: "pbs-1", Name: "Server 1"})

	snapshot := state.GetSnapshot()
	if len(snapshot.PBSInstances) != 1 {
		t.Fatalf("Expected 1 PBS instance, got %d", len(snapshot.PBSInstances))
	}

	// Add second instance
	state.UpdatePBSInstance(PBSInstance{ID: "pbs-2", Name: "Server 2"})

	snapshot = state.GetSnapshot()
	if len(snapshot.PBSInstances) != 2 {
		t.Fatalf("Expected 2 PBS instances, got %d", len(snapshot.PBSInstances))
	}

	// Update existing instance
	state.UpdatePBSInstance(PBSInstance{ID: "pbs-1", Name: "Server 1 Updated"})

	snapshot = state.GetSnapshot()
	if len(snapshot.PBSInstances) != 2 {
		t.Fatalf("Expected 2 PBS instances after update, got %d", len(snapshot.PBSInstances))
	}

	// Find updated instance
	var found bool
	for _, inst := range snapshot.PBSInstances {
		if inst.ID == "pbs-1" && inst.Name == "Server 1 Updated" {
			found = true
			break
		}
	}
	if !found {
		t.Error("Expected pbs-1 to be updated with new name")
	}
}

func TestPBSInstanceJobHealthEvidenceNormalizeAndClone(t *testing.T) {
	state := NewState()
	observedAt := time.Unix(1700000000, 0).UTC()
	state.UpdatePBSInstance(PBSInstance{
		ID:   "pbs-1",
		Name: "pbs-main",
		JobHealthEvidence: []PBSJobHealthEvidence{{
			ID:             "sync-1",
			Family:         "sync",
			Store:          "fast",
			LastRunState:   "OK",
			LastRunUPID:    "UPID:sync:1",
			LastRunEndtime: 1699999900,
			NextRun:        1700003600,
			Confidence:     "direct-task-match",
			EvidenceSource: "pbs-job-config",
			EvidenceScope:  "configured-job",
			Freshness: PBSJobHealthFreshness{
				ObservedAt: observedAt,
				State:      "observed",
			},
			Posture: "healthy",
		}},
	})

	snapshot := state.GetSnapshot()
	if len(snapshot.PBSInstances) != 1 || len(snapshot.PBSInstances[0].JobHealthEvidence) != 1 {
		t.Fatalf("expected cloned PBS job health evidence, got %+v", snapshot.PBSInstances)
	}
	snapshot.PBSInstances[0].JobHealthEvidence[0].LastRunState = "changed"

	fresh := state.GetSnapshot()
	if got := fresh.PBSInstances[0].JobHealthEvidence[0].LastRunState; got != "OK" {
		t.Fatalf("state clone leaked mutation, got last-run-state %q", got)
	}
	if got := fresh.PBSInstances[0].JobHealthEvidence[0]; got.EvidenceSource != "pbs-job-config" || got.EvidenceScope != "configured-job" {
		t.Fatalf("state clone did not preserve evidence source/scope: %+v", got)
	}

	normalized := (PBSInstance{}).NormalizeCollections()
	if normalized.JobHealthEvidence == nil {
		t.Fatal("JobHealthEvidence should normalize to an initialized empty slice")
	}
}

func TestUpdatePMGInstances(t *testing.T) {
	state := NewState()

	instances := []PMGInstance{
		{ID: "pmg-1", Name: "Mail Gateway 1"},
		{ID: "pmg-2", Name: "Mail Gateway 2"},
	}
	state.UpdatePMGInstances(instances)

	snapshot := state.GetSnapshot()
	if len(snapshot.PMGInstances) != 2 {
		t.Fatalf("Expected 2 PMG instances, got %d", len(snapshot.PMGInstances))
	}

	// Replace with empty
	state.UpdatePMGInstances([]PMGInstance{})

	snapshot = state.GetSnapshot()
	if len(snapshot.PMGInstances) != 0 {
		t.Fatalf("Expected 0 PMG instances after clearing, got %d", len(snapshot.PMGInstances))
	}
}

func TestUpdatePMGInstance(t *testing.T) {
	state := NewState()

	// Add first instance
	state.UpdatePMGInstance(PMGInstance{ID: "pmg-1", Name: "Gateway 1"})

	snapshot := state.GetSnapshot()
	if len(snapshot.PMGInstances) != 1 {
		t.Fatalf("Expected 1 PMG instance, got %d", len(snapshot.PMGInstances))
	}

	// Add second instance
	state.UpdatePMGInstance(PMGInstance{ID: "pmg-2", Name: "Gateway 2"})

	snapshot = state.GetSnapshot()
	if len(snapshot.PMGInstances) != 2 {
		t.Fatalf("Expected 2 PMG instances, got %d", len(snapshot.PMGInstances))
	}

	// Update existing instance
	state.UpdatePMGInstance(PMGInstance{ID: "pmg-1", Name: "Gateway 1 Updated"})

	snapshot = state.GetSnapshot()
	if len(snapshot.PMGInstances) != 2 {
		t.Fatalf("Expected 2 PMG instances after update, got %d", len(snapshot.PMGInstances))
	}

	// Verify update
	var found bool
	for _, inst := range snapshot.PMGInstances {
		if inst.ID == "pmg-1" && inst.Name == "Gateway 1 Updated" {
			found = true
			break
		}
	}
	if !found {
		t.Error("Expected pmg-1 to be updated with new name")
	}
}

func TestUpdateCephClustersForInstance(t *testing.T) {
	state := NewState()

	// Add clusters from first instance
	clusters1 := []CephCluster{
		{ID: "ceph-1", Instance: "pve-1", Name: "ceph-pool-1"},
		{ID: "ceph-2", Instance: "pve-1", Name: "ceph-pool-2"},
	}
	state.UpdateCephClustersForInstance("pve-1", clusters1)

	snapshot := state.GetSnapshot()
	if len(snapshot.CephClusters) != 2 {
		t.Fatalf("Expected 2 clusters, got %d", len(snapshot.CephClusters))
	}

	// Add clusters from second instance
	clusters2 := []CephCluster{
		{ID: "ceph-3", Instance: "pve-2", Name: "ceph-pool-1"},
	}
	state.UpdateCephClustersForInstance("pve-2", clusters2)

	snapshot = state.GetSnapshot()
	if len(snapshot.CephClusters) != 3 {
		t.Fatalf("Expected 3 clusters, got %d", len(snapshot.CephClusters))
	}

	// Update first instance with empty (should remove its clusters)
	state.UpdateCephClustersForInstance("pve-1", []CephCluster{})

	snapshot = state.GetSnapshot()
	if len(snapshot.CephClusters) != 1 {
		t.Fatalf("Expected 1 cluster after clearing pve-1, got %d", len(snapshot.CephClusters))
	}
	if snapshot.CephClusters[0].Instance != "pve-2" {
		t.Error("Remaining cluster should be from pve-2")
	}
}

func TestCephClusterStatePrefersProxmoxAPIByFSID(t *testing.T) {
	state := NewState()

	state.UpsertCephCluster(CephCluster{
		ID:       "fsid-123",
		Instance: "pve5",
		Source:   CephClusterSourceHostAgent,
		Name:     "pve5 Ceph",
		FSID:     "fsid-123",
		Pools: []CephPool{{
			ID:             2,
			Name:           "data_replication",
			StoredBytes:    70,
			AvailableBytes: 30,
			PercentUsed:    70,
		}},
	})

	stored := state.UpdateCephClustersForInstance("prod", []CephCluster{{
		ID:       "prod-fsid-123",
		Instance: "prod",
		Source:   CephClusterSourceProxmoxAPI,
		Name:     "Ceph",
		FSID:     "fsid-123",
		Health:   "HEALTH_OK",
	}})

	snapshot := state.GetSnapshot()
	if len(snapshot.CephClusters) != 1 {
		t.Fatalf("expected API and agent reports for same FSID to reconcile to one cluster, got %#v", snapshot.CephClusters)
	}
	cluster := snapshot.CephClusters[0]
	if cluster.Source != CephClusterSourceProxmoxAPI || cluster.Instance != "prod" || cluster.ID != "prod-fsid-123" {
		t.Fatalf("expected Proxmox API cluster to be canonical, got %+v", cluster)
	}
	if !containsString(cluster.InstanceAliases, "pve5") {
		t.Fatalf("expected agent instance alias to be preserved, got %#v", cluster.InstanceAliases)
	}
	if len(cluster.Pools) != 1 || cluster.Pools[0].Name != "data_replication" {
		t.Fatalf("expected agent pool data to supplement API status-only cluster, got %#v", cluster.Pools)
	}
	if len(stored) != 1 || stored[0].Instance != "prod" || !containsString(stored[0].InstanceAliases, "pve5") {
		t.Fatalf("stored clusters did not return reconciled canonical cluster: %#v", stored)
	}
}

// A reported status and its health details are one observation. In particular,
// healthy reports cannot borrow a previous incident's message or checks.
func TestCephClusterHealthDetailsFollowAuthoritativeStatus(t *testing.T) {
	warning := func(source, instance string) CephCluster {
		return CephCluster{ID: instance + "-fsid", Instance: instance, Source: source, FSID: "fsid-health",
			Health: "WARN", HealthMessage: "No active Ceph manager",
			HealthChecks: []CephHealthCheck{{Code: "MGR_DOWN", Severity: "HEALTH_WARN", Summary: "No active Ceph manager"}}, NumMgrs: 3}
	}
	agentWarning := warning(CephClusterSourceHostAgent, "agent")
	apiWarning := warning(CephClusterSourceProxmoxAPI, "api")
	agentHealthy := CephCluster{ID: "agent-fsid", Instance: "agent", Source: CephClusterSourceHostAgent, FSID: "fsid-health", Health: "OK"}
	apiHealthy := CephCluster{ID: "api-fsid", Instance: "api", Source: CephClusterSourceProxmoxAPI, FSID: "fsid-health", Health: "HEALTH_OK"}
	apiUnknown := apiHealthy
	apiUnknown.Health = ""
	apiBlank := apiUnknown
	apiBlank.Health = "  "
	agentNewWarning := agentHealthy
	agentNewWarning.Health = "WARN"
	cases := []struct {
		name         string
		reports      []CephCluster
		wantStatus   string
		wantFallback bool
	}{
		{"agent recovery clears the previous incident", []CephCluster{agentWarning, agentHealthy}, "OK", false},
		{"API recovery clears the previous incident", []CephCluster{apiWarning, apiHealthy}, "HEALTH_OK", false},
		{"API health takes authority over agent warning", []CephCluster{agentWarning, apiHealthy}, "HEALTH_OK", false},
		{"agent warning cannot fill explicit API health", []CephCluster{apiHealthy, agentWarning}, "HEALTH_OK", false},
		{"agent health cannot clear authoritative API warning", []CephCluster{apiWarning, agentHealthy}, "WARN", true},
		{"missing API status retains health fallback", []CephCluster{agentWarning, apiUnknown}, "WARN", true},
		{"blank API status retains health fallback", []CephCluster{agentWarning, apiBlank}, "WARN", true},
		{"new warning without detail cannot inherit an old cause", []CephCluster{agentWarning, agentNewWarning}, "WARN", false},
	}
	for _, tc := range cases {
		t.Run(tc.name, func(t *testing.T) {
			state := NewState()
			for _, report := range tc.reports {
				state.UpsertCephCluster(report)
			}
			clusters := state.GetSnapshot().CephClusters
			if len(clusters) != 1 {
				t.Fatalf("clusters = %#v, want one FSID", clusters)
			}
			got := clusters[0]
			if got.Health != tc.wantStatus {
				t.Fatalf("health = %q, want %q", got.Health, tc.wantStatus)
			}
			if tc.wantFallback {
				if got.HealthMessage != "No active Ceph manager" || len(got.HealthChecks) != 1 || got.HealthChecks[0].Code != "MGR_DOWN" {
					t.Fatalf("health fallback lost: %#v", got)
				}
			} else if got.HealthMessage != "" || len(got.HealthChecks) != 0 {
				t.Fatalf("explicit health inherited stale details: %#v", got)
			}
			if got.NumMgrs != 3 {
				t.Fatalf("unrelated count fallback changed: managers = %d", got.NumMgrs)
			}
		})
	}
}

func TestUpsertCephClusterKeepsProxmoxAPIWhenAgentRefreshArrives(t *testing.T) {
	state := NewState()

	state.UpdateCephClustersForInstance("prod", []CephCluster{{
		ID:       "prod-fsid-123",
		Instance: "prod",
		Source:   CephClusterSourceProxmoxAPI,
		Name:     "Ceph",
		FSID:     "fsid-123",
		Health:   "HEALTH_OK",
		Pools: []CephPool{{
			ID:             2,
			Name:           "data_replication",
			StoredBytes:    60,
			AvailableBytes: 40,
			PercentUsed:    60,
		}},
	}})

	stored := state.UpsertCephCluster(CephCluster{
		ID:       "fsid-123",
		Instance: "pve5",
		Source:   CephClusterSourceHostAgent,
		Name:     "pve5 Ceph",
		FSID:     "fsid-123",
	})

	snapshot := state.GetSnapshot()
	if len(snapshot.CephClusters) != 1 {
		t.Fatalf("expected agent refresh to merge with API cluster, got %#v", snapshot.CephClusters)
	}
	cluster := snapshot.CephClusters[0]
	if cluster.Source != CephClusterSourceProxmoxAPI || cluster.Instance != "prod" {
		t.Fatalf("expected API cluster to remain canonical, got %+v", cluster)
	}
	if !containsString(cluster.InstanceAliases, "pve5") {
		t.Fatalf("expected agent instance alias to be preserved, got %#v", cluster.InstanceAliases)
	}
	if stored.Source != CephClusterSourceProxmoxAPI || stored.Instance != "prod" {
		t.Fatalf("expected upsert to return canonical API cluster, got %+v", stored)
	}
}

func TestUpdateBackupTasksForInstance(t *testing.T) {
	state := NewState()

	now := time.Now()

	// Add tasks from first instance
	tasks1 := []BackupTask{
		{ID: "pve-1-task-1", Instance: "pve-1", StartTime: now},
		{ID: "pve-1-task-2", Instance: "pve-1", StartTime: now.Add(-time.Hour)},
	}
	state.UpdateBackupTasksForInstance("pve-1", tasks1)

	snapshot := state.GetSnapshot()
	if len(snapshot.PVEBackups.BackupTasks) != 2 {
		t.Fatalf("Expected 2 tasks, got %d", len(snapshot.PVEBackups.BackupTasks))
	}

	// Tasks should be sorted by start time descending
	if snapshot.PVEBackups.BackupTasks[0].StartTime.Before(snapshot.PVEBackups.BackupTasks[1].StartTime) {
		t.Error("Tasks should be sorted by start time descending")
	}

	// Add tasks from second instance
	tasks2 := []BackupTask{
		{ID: "pve-2-task-1", Instance: "pve-2", StartTime: now.Add(-30 * time.Minute)},
	}
	state.UpdateBackupTasksForInstance("pve-2", tasks2)

	snapshot = state.GetSnapshot()
	if len(snapshot.PVEBackups.BackupTasks) != 3 {
		t.Fatalf("Expected 3 tasks, got %d", len(snapshot.PVEBackups.BackupTasks))
	}

	// Update first instance (should replace its tasks)
	tasks1Updated := []BackupTask{
		{ID: "pve-1-task-3", Instance: "pve-1", StartTime: now.Add(time.Hour)},
	}
	state.UpdateBackupTasksForInstance("pve-1", tasks1Updated)

	snapshot = state.GetSnapshot()
	if len(snapshot.PVEBackups.BackupTasks) != 2 {
		t.Fatalf("Expected 2 tasks after update, got %d", len(snapshot.PVEBackups.BackupTasks))
	}
}

func TestUpdateReplicationJobsForInstance(t *testing.T) {
	state := NewState()

	// Add jobs from first instance
	jobs1 := []ReplicationJob{
		{ID: "job-1", Instance: "pve-1", GuestID: 100, JobNumber: 0},
		{ID: "job-2", Instance: "pve-1", GuestID: 101, JobNumber: 0},
	}
	state.UpdateReplicationJobsForInstance("pve-1", jobs1)

	snapshot := state.GetSnapshot()
	if len(snapshot.ReplicationJobs) != 2 {
		t.Fatalf("Expected 2 jobs, got %d", len(snapshot.ReplicationJobs))
	}

	// Add jobs from second instance
	jobs2 := []ReplicationJob{
		{ID: "job-3", Instance: "pve-2", GuestID: 200, JobNumber: 0},
	}
	state.UpdateReplicationJobsForInstance("pve-2", jobs2)

	snapshot = state.GetSnapshot()
	if len(snapshot.ReplicationJobs) != 3 {
		t.Fatalf("Expected 3 jobs, got %d", len(snapshot.ReplicationJobs))
	}

	// Update first instance with empty (should remove its jobs)
	state.UpdateReplicationJobsForInstance("pve-1", []ReplicationJob{})

	snapshot = state.GetSnapshot()
	if len(snapshot.ReplicationJobs) != 1 {
		t.Fatalf("Expected 1 job after clearing pve-1, got %d", len(snapshot.ReplicationJobs))
	}
	if snapshot.ReplicationJobs[0].Instance != "pve-2" {
		t.Error("Remaining job should be from pve-2")
	}
}

func TestUpdateGuestSnapshotsForInstance(t *testing.T) {
	state := NewState()

	now := time.Now()

	// Add snapshots from first instance
	snapshots1 := []GuestSnapshot{
		{ID: "pve-1-snap-1", Instance: "pve-1", VMID: 100, Name: "snapshot1", Time: now},
		{ID: "pve-1-snap-2", Instance: "pve-1", VMID: 100, Name: "snapshot2", Time: now.Add(-time.Hour)},
	}
	state.UpdateGuestSnapshotsForInstance("pve-1", snapshots1)

	snapshot := state.GetSnapshot()
	if len(snapshot.PVEBackups.GuestSnapshots) != 2 {
		t.Fatalf("Expected 2 snapshots, got %d", len(snapshot.PVEBackups.GuestSnapshots))
	}

	// Add snapshots from second instance
	snapshots2 := []GuestSnapshot{
		{ID: "pve-2-snap-1", Instance: "pve-2", VMID: 200, Name: "snapshot1", Time: now.Add(-30 * time.Minute)},
	}
	state.UpdateGuestSnapshotsForInstance("pve-2", snapshots2)

	snapshot = state.GetSnapshot()
	if len(snapshot.PVEBackups.GuestSnapshots) != 3 {
		t.Fatalf("Expected 3 snapshots, got %d", len(snapshot.PVEBackups.GuestSnapshots))
	}

	// Update first instance (should replace its snapshots)
	snapshots1Updated := []GuestSnapshot{
		{ID: "pve-1-snap-3", Instance: "pve-1", VMID: 100, Name: "new-snapshot", Time: now.Add(time.Hour)},
	}
	state.UpdateGuestSnapshotsForInstance("pve-1", snapshots1Updated)

	snapshot = state.GetSnapshot()
	if len(snapshot.PVEBackups.GuestSnapshots) != 2 {
		t.Fatalf("Expected 2 snapshots after update, got %d", len(snapshot.PVEBackups.GuestSnapshots))
	}
}

func TestSyncGuestBackupTimes(t *testing.T) {
	state := NewState()

	now := time.Now()
	oldBackup := now.Add(-24 * time.Hour)
	newBackup := now.Add(-1 * time.Hour)

	// Set up VMs and containers
	state.UpdateVMs([]VM{
		{VMID: 100, Name: "vm-100", Instance: "pve-1"},
		{VMID: 101, Name: "vm-101", Instance: "pve-1"},
	})
	state.UpdateContainers([]Container{
		{VMID: 200, Name: "ct-200", Instance: "pve-1"},
	})

	// Add storage backups for VM 100 (must include Instance for proper matching)
	state.mu.Lock()
	state.PVEBackups.StorageBackups = []StorageBackup{
		{ID: "pve-1-backup-1", VMID: 100, Instance: "pve-1", Time: oldBackup},
		{ID: "pve-1-backup-2", VMID: 100, Instance: "pve-1", Time: newBackup}, // newer
	}
	state.mu.Unlock()

	// Add PBS backup for container 200
	state.UpdatePBSBackups("pbs-1", []PBSBackup{
		{ID: "pbs-backup-1", VMID: "200", BackupType: "ct", BackupTime: newBackup},
	})

	// Sync backup times
	state.SyncGuestBackupTimes()

	snapshot := state.GetSnapshot()

	// VM 100 should have the newer backup time
	var vm100 *VM
	for i := range snapshot.VMs {
		if snapshot.VMs[i].VMID == 100 {
			vm100 = &snapshot.VMs[i]
			break
		}
	}
	if vm100 == nil {
		t.Fatal("VM 100 not found")
	}
	if !vm100.LastBackup.Equal(newBackup) {
		t.Errorf("VM 100 LastBackup = %v, expected %v", vm100.LastBackup, newBackup)
	}

	// VM 101 should have no backup time (zero)
	var vm101 *VM
	for i := range snapshot.VMs {
		if snapshot.VMs[i].VMID == 101 {
			vm101 = &snapshot.VMs[i]
			break
		}
	}
	if vm101 == nil {
		t.Fatal("VM 101 not found")
	}
	if !vm101.LastBackup.IsZero() {
		t.Errorf("VM 101 LastBackup should be zero, got %v", vm101.LastBackup)
	}

	// Container 200 should have backup time from PBS
	var ct200 *Container
	for i := range snapshot.Containers {
		if snapshot.Containers[i].VMID == 200 {
			ct200 = &snapshot.Containers[i]
			break
		}
	}
	if ct200 == nil {
		t.Fatal("Container 200 not found")
	}
	if !ct200.LastBackup.Equal(newBackup) {
		t.Errorf("Container 200 LastBackup = %v, expected %v", ct200.LastBackup, newBackup)
	}
}

// TestSyncGuestBackupTimesCrossInstance verifies that backup matching uses instance+VMID.
// This prevents a newly created container on one instance from incorrectly showing
// backup time from a different container with the same VMID on another instance.
func TestSyncGuestBackupTimesCrossInstance(t *testing.T) {
	state := NewState()

	now := time.Now()
	threeMonthsAgo := now.Add(-90 * 24 * time.Hour)

	// Set up containers with the SAME VMID on DIFFERENT instances
	// This simulates: pve-1 has VMID 100 with old backup, pve-2 has newly created VMID 100
	state.UpdateContainers([]Container{
		{VMID: 100, Name: "old-container", Instance: "pve-1"},
		{VMID: 100, Name: "new-container", Instance: "pve-2"}, // newly created, no backup
	})

	// Add a 3-month-old backup for pve-1's container (VMID 100)
	state.mu.Lock()
	state.PVEBackups.StorageBackups = []StorageBackup{
		{ID: "pve-1-backup-old", VMID: 100, Instance: "pve-1", Time: threeMonthsAgo},
	}
	state.mu.Unlock()

	// Sync backup times
	state.SyncGuestBackupTimes()

	snapshot := state.GetSnapshot()

	// Find both containers
	var oldContainer, newContainer *Container
	for i := range snapshot.Containers {
		if snapshot.Containers[i].Instance == "pve-1" && snapshot.Containers[i].VMID == 100 {
			oldContainer = &snapshot.Containers[i]
		}
		if snapshot.Containers[i].Instance == "pve-2" && snapshot.Containers[i].VMID == 100 {
			newContainer = &snapshot.Containers[i]
		}
	}

	if oldContainer == nil {
		t.Fatal("pve-1 container not found")
	}
	if newContainer == nil {
		t.Fatal("pve-2 container not found")
	}

	// The old container on pve-1 SHOULD have the backup time
	if !oldContainer.LastBackup.Equal(threeMonthsAgo) {
		t.Errorf("pve-1 container LastBackup = %v, expected %v", oldContainer.LastBackup, threeMonthsAgo)
	}

	// The NEW container on pve-2 should NOT have any backup time (it's a different container!)
	if !newContainer.LastBackup.IsZero() {
		t.Errorf("pve-2 container (newly created) should have no backup, got %v", newContainer.LastBackup)
	}
}

// TestSyncGuestBackupTimesNamespaceDisambiguation verifies that PBS namespace is used to
// disambiguate backups when multiple VMs have the same VMID across different PVE instances.
// This addresses issue #1095 where users have multiple PVE instances with overlapping VMIDs.
func TestSyncGuestBackupTimesNamespaceDisambiguation(t *testing.T) {
	state := NewState()

	now := time.Now()
	pveBackupTime := now.Add(-1 * time.Hour)
	pveNatBackupTime := now.Add(-2 * time.Hour)

	// Set up VMs with the SAME VMID on DIFFERENT instances
	state.UpdateVMs([]VM{
		{VMID: 100, Name: "webserver-pve", Instance: "pve", Node: "node1"},
		{VMID: 100, Name: "webserver-nat", Instance: "pve-nat", Node: "node2"},
	})

	// Add PBS backups with namespaces that correspond to the PVE instances
	state.mu.Lock()
	state.PBSBackups = []PBSBackup{
		{
			ID:         "pbs-pve-100",
			VMID:       "100",
			Namespace:  "pve",
			BackupType: "vm",
			BackupTime: pveBackupTime,
			Instance:   "pbs-main",
		},
		{
			ID:         "pbs-nat-100",
			VMID:       "100",
			Namespace:  "nat", // Should match "pve-nat" instance
			BackupType: "vm",
			BackupTime: pveNatBackupTime,
			Instance:   "pbs-main",
		},
	}
	state.mu.Unlock()

	// Sync backup times
	state.SyncGuestBackupTimes()

	snapshot := state.GetSnapshot()

	// Find both VMs
	var vmPVE, vmNAT *VM
	for i := range snapshot.VMs {
		if snapshot.VMs[i].Instance == "pve" && snapshot.VMs[i].VMID == 100 {
			vmPVE = &snapshot.VMs[i]
		}
		if snapshot.VMs[i].Instance == "pve-nat" && snapshot.VMs[i].VMID == 100 {
			vmNAT = &snapshot.VMs[i]
		}
	}

	if vmPVE == nil {
		t.Fatal("pve VM not found")
	}
	if vmNAT == nil {
		t.Fatal("pve-nat VM not found")
	}

	// The pve VM should have the backup with namespace "pve"
	if !vmPVE.LastBackup.Equal(pveBackupTime) {
		t.Errorf("pve VM LastBackup = %v, expected %v (from namespace 'pve')", vmPVE.LastBackup, pveBackupTime)
	}

	// The pve-nat VM should have the backup with namespace "nat"
	if !vmNAT.LastBackup.Equal(pveNatBackupTime) {
		t.Errorf("pve-nat VM LastBackup = %v, expected %v (from namespace 'nat')", vmNAT.LastBackup, pveNatBackupTime)
	}
}

// TestSyncGuestBackupTimesClusterEntrypointUsesGuestNodeNamespace verifies that
// a cluster API entrypoint name does not shadow the guest's actual node namespace.
func TestSyncGuestBackupTimesClusterEntrypointUsesGuestNodeNamespace(t *testing.T) {
	state := NewState()

	now := time.Now()
	oldBackupTime := now.Add(-180 * 24 * time.Hour)
	freshBackupTime := now.Add(-1 * time.Hour)

	state.UpdateContainers([]Container{
		{VMID: 112, Name: "debian-go", Instance: "delly", Node: "minipc"},
	})

	state.mu.Lock()
	state.PBSBackups = []PBSBackup{
		{
			ID:         "pbs-delly-112-old",
			VMID:       "112",
			Namespace:  "delly",
			BackupType: "ct",
			Comment:    "112",
			BackupTime: oldBackupTime,
			Instance:   "pbs-main",
		},
		{
			ID:         "pbs-minipc-112-fresh",
			VMID:       "112",
			Namespace:  "minipc",
			BackupType: "ct",
			Comment:    "debian-go",
			BackupTime: freshBackupTime,
			Instance:   "pbs-main",
		},
	}
	state.mu.Unlock()

	state.SyncGuestBackupTimes()
	snapshot := state.GetSnapshot()

	var found *Container
	for i := range snapshot.Containers {
		if snapshot.Containers[i].VMID == 112 {
			found = &snapshot.Containers[i]
			break
		}
	}
	if found == nil {
		t.Fatal("container 112 not found")
	}
	if !found.LastBackup.Equal(freshBackupTime) {
		t.Errorf("cluster entrypoint guest LastBackup = %v, want fresh node-namespaced backup %v",
			found.LastBackup, freshBackupTime)
	}
}

func TestSyncGuestBackupTimesClearsStaleBackupWhenCurrentEvidenceDisappears(t *testing.T) {
	state := NewState()

	staleBackupTime := time.Now().Add(-30 * 24 * time.Hour)
	state.UpdateVMs([]VM{
		{VMID: 101, Name: "unbacked-vm", Instance: "pve-1", Node: "node1", LastBackup: staleBackupTime},
	})
	state.UpdateContainers([]Container{
		{VMID: 201, Name: "unbacked-ct", Instance: "pve-1", Node: "node1", LastBackup: staleBackupTime},
	})

	state.SyncGuestBackupTimes()
	snapshot := state.GetSnapshot()

	for _, vm := range snapshot.VMs {
		if vm.VMID == 101 && !vm.LastBackup.IsZero() {
			t.Errorf("VM stale LastBackup should be cleared, got %v", vm.LastBackup)
		}
	}
	for _, ct := range snapshot.Containers {
		if ct.VMID == 201 && !ct.LastBackup.IsZero() {
			t.Errorf("container stale LastBackup should be cleared, got %v", ct.LastBackup)
		}
	}
}

// TestSyncGuestBackupTimesVMIDCollisionNonMatchingNamespace verifies that when the same VMID
// exists on multiple PVE instances and a PBS backup namespace matches neither, both guests
// get zero LastBackup instead of a false positive. The backup's submission
// source (owner/datastore) carries no learned attribution and no PVE-side
// storage listing confirms it, so the #1639 evidence paths must not fire.
func TestSyncGuestBackupTimesVMIDCollisionNonMatchingNamespace(t *testing.T) {
	state := NewState()

	now := time.Now()
	backupTime := now.Add(-1 * time.Hour)

	// Two VMs with the same VMID on different instances
	state.UpdateVMs([]VM{
		{VMID: 100, Name: "vm-pve1", Instance: "pve1", Node: "node1"},
		{VMID: 100, Name: "vm-pve2", Instance: "pve2", Node: "node2"},
	})

	// PBS backup with a namespace that matches neither instance
	state.mu.Lock()
	state.PBSBackups = []PBSBackup{
		{
			ID:         "pbs-100",
			VMID:       "100",
			Namespace:  "staging",
			BackupType: "vm",
			BackupTime: backupTime,
			Instance:   "pbs-main",
			Datastore:  "backups",
			Owner:      "unlearned@pbs!token",
		},
	}
	state.mu.Unlock()

	state.SyncGuestBackupTimes()
	snapshot := state.GetSnapshot()

	for _, vm := range snapshot.VMs {
		if vm.VMID == 100 && !vm.LastBackup.IsZero() {
			t.Errorf("VM %q on %s should have zero LastBackup (ambiguous VMID, non-matching namespace), got %v",
				vm.Name, vm.Instance, vm.LastBackup)
		}
	}
}

// TestSyncGuestBackupTimesVMIDCollisionEmptyNamespace verifies that when the same VMID
// exists on multiple PVE instances and a PBS backup has no namespace, both guests
// get zero LastBackup. This is the genuinely unattributable core of #1639:
// with a single snapshot, no learnable submission source, and no PVE-side
// confirmation, guessing either guest would be a false positive. The
// resolvable variants live in issue1639_pbs_collision_test.go.
func TestSyncGuestBackupTimesVMIDCollisionEmptyNamespace(t *testing.T) {
	state := NewState()

	now := time.Now()
	backupTime := now.Add(-1 * time.Hour)

	state.UpdateVMs([]VM{
		{VMID: 100, Name: "vm-pve1", Instance: "pve1", Node: "node1"},
		{VMID: 100, Name: "vm-pve2", Instance: "pve2", Node: "node2"},
	})

	// PBS backup with empty namespace
	state.mu.Lock()
	state.PBSBackups = []PBSBackup{
		{
			ID:         "pbs-100",
			VMID:       "100",
			Namespace:  "",
			BackupType: "vm",
			BackupTime: backupTime,
			Instance:   "pbs-main",
		},
	}
	state.mu.Unlock()

	state.SyncGuestBackupTimes()
	snapshot := state.GetSnapshot()

	for _, vm := range snapshot.VMs {
		if vm.VMID == 100 && !vm.LastBackup.IsZero() {
			t.Errorf("VM %q on %s should have zero LastBackup (ambiguous VMID, empty namespace), got %v",
				vm.Name, vm.Instance, vm.LastBackup)
		}
	}
}

// TestSyncGuestBackupTimesUniqueVMIDFallback verifies that a unique VMID still gets
// the PBS backup via fallback even when the namespace doesn't match.
func TestSyncGuestBackupTimesUniqueVMIDFallback(t *testing.T) {
	state := NewState()

	now := time.Now()
	backupTime := now.Add(-1 * time.Hour)

	// Single VM — VMID is unique
	state.UpdateVMs([]VM{
		{VMID: 100, Name: "my-vm", Instance: "pve1", Node: "node1"},
	})

	// PBS backup with a namespace that does NOT match the instance
	state.mu.Lock()
	state.PBSBackups = []PBSBackup{
		{
			ID:         "pbs-100",
			VMID:       "100",
			Namespace:  "daily",
			BackupType: "vm",
			BackupTime: backupTime,
			Instance:   "pbs-main",
		},
	}
	state.mu.Unlock()

	state.SyncGuestBackupTimes()
	snapshot := state.GetSnapshot()

	var found *VM
	for i := range snapshot.VMs {
		if snapshot.VMs[i].VMID == 100 {
			found = &snapshot.VMs[i]
		}
	}
	if found == nil {
		t.Fatal("VM not found")
	}
	if !found.LastBackup.Equal(backupTime) {
		t.Errorf("unique VMID should fall back to PBS backup; got LastBackup = %v, want %v",
			found.LastBackup, backupTime)
	}
}

// TestSyncGuestBackupTimesVMContainerCollision verifies that PBS vm/ID and ct/ID
// subjects are matched independently even when a VM and container share the same
// numeric ID.
func TestSyncGuestBackupTimesVMContainerCollision(t *testing.T) {
	state := NewState()

	now := time.Now()
	backupTime := now.Add(-1 * time.Hour)

	state.UpdateVMs([]VM{
		{VMID: 100, Name: "vm-pve1", Instance: "pve1", Node: "node1"},
	})
	state.UpdateContainers([]Container{
		{VMID: 100, Name: "ct-pve2", Instance: "pve2", Node: "node2"},
	})

	state.mu.Lock()
	state.PBSBackups = []PBSBackup{
		{
			ID:         "pbs-100",
			VMID:       "100",
			Namespace:  "node1",
			BackupType: "vm",
			BackupTime: backupTime,
			Instance:   "pbs-main",
		},
	}
	state.mu.Unlock()

	state.SyncGuestBackupTimes()
	snapshot := state.GetSnapshot()

	for _, vm := range snapshot.VMs {
		if vm.VMID == 100 && !vm.LastBackup.Equal(backupTime) {
			t.Errorf("VM %q LastBackup = %v, want typed vm/100 backup %v",
				vm.Name, vm.LastBackup, backupTime)
		}
	}
	for _, ct := range snapshot.Containers {
		if ct.VMID == 100 && !ct.LastBackup.IsZero() {
			t.Errorf("Container %q should have zero LastBackup because the PBS subject is vm/100, got %v",
				ct.Name, ct.LastBackup)
		}
	}
}

func TestUpdateGuestsForInstancePublishesCoherentGeneration(t *testing.T) {
	state := NewState()
	previousBackup := time.Now().Add(-2 * time.Hour).UTC()

	state.UpdateVMs([]VM{
		{ID: "lab-a:node-a:101", VMID: 101, Instance: "lab-a", Node: "node-a", LastBackup: previousBackup},
		{ID: "lab-b:node-b:201", VMID: 201, Instance: "lab-b", Node: "node-b"},
	})
	state.UpdateContainers([]Container{
		{ID: "lab-a:node-a:102", VMID: 102, Instance: "lab-a", Node: "node-a", LastBackup: previousBackup},
		{ID: "lab-b:node-b:202", VMID: 202, Instance: "lab-b", Node: "node-b"},
	})
	before := state.GetSnapshot().LastUpdate

	state.UpdateGuestsForInstance(
		"lab-a",
		[]VM{{ID: "lab-a:node-a:101", VMID: 101, Instance: "lab-a", Node: "node-a", Status: "running"}},
		[]Container{{ID: "lab-a:node-a:103", VMID: 103, Instance: "lab-a", Node: "node-a", Status: "running"}},
	)

	snapshot := state.GetSnapshot()
	if !snapshot.LastUpdate.After(before) {
		t.Fatalf("LastUpdate did not advance: before=%v after=%v", before, snapshot.LastUpdate)
	}
	if len(snapshot.VMs) != 2 || len(snapshot.Containers) != 2 {
		t.Fatalf("unexpected coherent generation sizes: vms=%d containers=%d", len(snapshot.VMs), len(snapshot.Containers))
	}
	if snapshot.VMs[0].ID != "lab-a:node-a:101" || !snapshot.VMs[0].LastBackup.Equal(previousBackup) {
		t.Fatalf("updated VM did not retain backup state: %+v", snapshot.VMs[0])
	}
	if snapshot.Containers[0].ID != "lab-a:node-a:103" {
		t.Fatalf("authoritatively deleted container remained in state: %+v", snapshot.Containers)
	}
	if snapshot.VMs[1].Instance != "lab-b" || snapshot.Containers[1].Instance != "lab-b" {
		t.Fatalf("other instance was not isolated: vms=%+v containers=%+v", snapshot.VMs, snapshot.Containers)
	}
}

func TestUpdateStorageBackupsForInstance(t *testing.T) {
	state := NewState()

	now := time.Now()

	// Set up a VM so node normalization has something to work with
	state.UpdateVMsForInstance("pve-1", []VM{
		{VMID: 100, Instance: "pve-1", Node: "node1"},
	})

	// Add backups from first instance
	backups1 := []StorageBackup{
		{ID: "pve-1-backup-1", Instance: "pve-1", VMID: 100, Time: now, Node: "node1"},
		{ID: "pve-1-backup-2", Instance: "pve-1", VMID: 100, Time: now.Add(-time.Hour), Node: "node1"},
	}
	state.UpdateStorageBackupsForInstance("pve-1", backups1)

	snapshot := state.GetSnapshot()
	if len(snapshot.PVEBackups.StorageBackups) != 2 {
		t.Fatalf("Expected 2 backups, got %d", len(snapshot.PVEBackups.StorageBackups))
	}

	// Backups should be sorted by time descending
	if snapshot.PVEBackups.StorageBackups[0].Time.Before(snapshot.PVEBackups.StorageBackups[1].Time) {
		t.Error("Backups should be sorted by time descending")
	}

	// Add backups from second instance
	backups2 := []StorageBackup{
		{ID: "pve-2-backup-1", Instance: "pve-2", VMID: 200, Time: now.Add(-30 * time.Minute), Node: "node2"},
	}
	state.UpdateStorageBackupsForInstance("pve-2", backups2)

	snapshot = state.GetSnapshot()
	if len(snapshot.PVEBackups.StorageBackups) != 3 {
		t.Fatalf("Expected 3 backups, got %d", len(snapshot.PVEBackups.StorageBackups))
	}

	// Update first instance (should replace its backups)
	backups1Updated := []StorageBackup{
		{ID: "pve-1-backup-3", Instance: "pve-1", VMID: 100, Time: now.Add(time.Hour), Node: "node1"},
	}
	state.UpdateStorageBackupsForInstance("pve-1", backups1Updated)

	snapshot = state.GetSnapshot()
	if len(snapshot.PVEBackups.StorageBackups) != 2 {
		t.Fatalf("Expected 2 backups after update, got %d", len(snapshot.PVEBackups.StorageBackups))
	}

	// Verify old pve-1 backups were replaced
	for _, b := range snapshot.PVEBackups.StorageBackups {
		if b.ID == "pve-1-backup-1" || b.ID == "pve-1-backup-2" {
			t.Error("Old pve-1 backups should have been replaced")
		}
	}
}

// Host.IntegrationSource is fabric-only provenance: it must survive host
// upserts, serialize when set, and stay entirely absent from JSON for real
// agent-report hosts so existing payloads do not change shape.
func TestHostIntegrationSourceRoundTrip(t *testing.T) {
	state := NewState()
	state.UpsertHost(Host{ID: "host-esxi", Hostname: "esxi-01", IntegrationSource: "vmware"})
	state.UpsertHost(Host{ID: "host-agent", Hostname: "apollo-114"})

	var esxi, agent *Host
	hosts := state.GetHosts()
	for i := range hosts {
		switch hosts[i].ID {
		case "host-esxi":
			esxi = &hosts[i]
		case "host-agent":
			agent = &hosts[i]
		}
	}
	if esxi == nil || esxi.IntegrationSource != "vmware" {
		t.Fatalf("expected upserted host to preserve IntegrationSource %q, got %+v", "vmware", esxi)
	}
	if agent == nil || agent.IntegrationSource != "" {
		t.Fatalf("expected agent host to keep empty IntegrationSource, got %+v", agent)
	}

	marked, err := json.Marshal(esxi)
	if err != nil {
		t.Fatalf("marshal integration host: %v", err)
	}
	if !strings.Contains(string(marked), `"integrationSource":"vmware"`) {
		t.Fatalf("expected integrationSource in payload, got %s", marked)
	}

	unmarked, err := json.Marshal(agent)
	if err != nil {
		t.Fatalf("marshal agent host: %v", err)
	}
	if strings.Contains(string(unmarked), "integrationSource") {
		t.Fatalf("expected integrationSource omitted for agent hosts, got %s", unmarked)
	}
}

// Two clusters reusing the same node names (pve01/pve02 on different subnets)
// must stay apart: the second cluster's node must not steal the first
// cluster's agent link by bare hostname, and the shared name must not merge
// the two nodes into one slot.
func TestUpdateNodesForInstanceKeepsClustersWithSharedNodeNamesApart(t *testing.T) {
	state := &State{
		Hosts: []Host{
			{
				ID:       "host-enacon-pve01",
				Hostname: "pve01",
				ReportIP: "192.168.16.11",
				NetworkInterfaces: []HostNetworkInterface{
					{Name: "eth0", Addresses: []string{"192.168.16.11/24"}},
				},
			},
			{
				ID:       "host-enacon-pve02",
				Hostname: "pve02",
				ReportIP: "192.168.16.12",
				NetworkInterfaces: []HostNetworkInterface{
					{Name: "eth0", Addresses: []string{"192.168.16.12/24"}},
				},
			},
		},
		Nodes: []Node{
			{
				ID:              "enacon-pve01",
				Name:            "pve01",
				Instance:        "enacon",
				ClusterName:     "enacon",
				IsClusterMember: true,
				Host:            "https://192.168.16.11:8006",
				LinkedAgentID:   "host-enacon-pve01",
				Status:          "online",
			},
			{
				ID:              "enacon-pve02",
				Name:            "pve02",
				Instance:        "enacon",
				ClusterName:     "enacon",
				IsClusterMember: true,
				Host:            "https://192.168.16.12:8006",
				LinkedAgentID:   "host-enacon-pve02",
				Status:          "online",
			},
		},
	}

	state.UpdateNodesForInstance("rewo", []Node{
		{
			ID:              "rewo-pve01",
			Name:            "pve01",
			Instance:        "rewo",
			ClusterName:     "rewo",
			IsClusterMember: true,
			Host:            "https://192.168.1.11:8006",
			Status:          "online",
		},
	})

	if len(state.Nodes) != 3 {
		t.Fatalf("nodes = %#v, want 3 (enacon pve01+pve02 and rewo pve01)", state.Nodes)
	}

	byID := make(map[string]Node)
	for _, node := range state.Nodes {
		byID[node.ID] = node
	}

	enacon, ok := byID["enacon-pve01"]
	if !ok {
		t.Fatalf("enacon-pve01 was clobbered, nodes = %#v", state.Nodes)
	}
	if enacon.ClusterName != "enacon" {
		t.Fatalf("enacon-pve01 ClusterName = %q, want enacon", enacon.ClusterName)
	}
	if enacon.LinkedAgentID != "host-enacon-pve01" {
		t.Fatalf("enacon-pve01 LinkedAgentID = %q, want host-enacon-pve01", enacon.LinkedAgentID)
	}

	rewo, ok := byID["rewo-pve01"]
	if !ok {
		t.Fatalf("rewo-pve01 missing, nodes = %#v", state.Nodes)
	}
	if rewo.ClusterName != "rewo" {
		t.Fatalf("rewo-pve01 ClusterName = %q, want rewo", rewo.ClusterName)
	}
	if rewo.LinkedAgentID != "" {
		t.Fatalf("rewo-pve01 LinkedAgentID = %q, want none (its host runs no agent)", rewo.LinkedAgentID)
	}
}

// Standalone PVE connections have no cluster metadata, so sequential setup is
// the dangerous variant of the shared-name case: while only the first site's
// agent exists, a second pve01 must not borrow that agent through the short
// endpoint alias and collapse both provider instances into one node slot.
// Refs #1753.
func TestUpdateNodesForInstanceKeepsSequentialSameNameStandaloneSitesApart(t *testing.T) {
	state := &State{
		Hosts: []Host{
			{
				ID:       "agent-staging-pve01",
				Hostname: "pve01",
				ReportIP: "192.0.2.11",
				NetworkInterfaces: []HostNetworkInterface{
					{Name: "eth0", Addresses: []string{"192.0.2.11/24"}},
				},
			},
		},
		Nodes: []Node{
			{
				ID:            "staging-pve01",
				Name:          "pve01",
				Instance:      "staging",
				Host:          "https://pve01.staging.example:8006",
				LinkedAgentID: "agent-staging-pve01",
				Status:        "online",
			},
		},
	}

	state.UpdateNodesForInstance("production", []Node{
		{
			ID:       "production-pve01",
			Name:     "pve01",
			Instance: "production",
			Host:     "https://pve01.production.example:8006",
			Status:   "online",
		},
	})

	if len(state.Nodes) != 2 {
		t.Fatalf("nodes = %#v, want 2 standalone sites", state.Nodes)
	}
	byID := make(map[string]Node, len(state.Nodes))
	for _, node := range state.Nodes {
		byID[node.ID] = node
	}
	if byID["staging-pve01"].LinkedAgentID != "agent-staging-pve01" {
		t.Fatalf("staging link = %q, want agent-staging-pve01", byID["staging-pve01"].LinkedAgentID)
	}
	if byID["production-pve01"].LinkedAgentID != "" {
		t.Fatalf("production link = %q, want no guessed cross-site agent", byID["production-pve01"].LinkedAgentID)
	}

	// Existing installations may already hold the bad many-nodes-to-one-agent
	// link in memory. The next provider poll must discard it, not preserve it.
	for i := range state.Nodes {
		if state.Nodes[i].ID == "production-pve01" {
			state.Nodes[i].LinkedAgentID = "agent-staging-pve01"
		}
	}
	state.UpdateNodesForInstance("production", []Node{
		{
			ID:       "production-pve01",
			Name:     "pve01",
			Instance: "production",
			Host:     "https://pve01.production.example:8006",
			Status:   "online",
		},
	})
	if len(state.Nodes) != 2 {
		t.Fatalf("self-heal nodes = %#v, want 2 standalone sites", state.Nodes)
	}
	for _, node := range state.Nodes {
		if node.ID == "production-pve01" && node.LinkedAgentID != "" {
			t.Fatalf("self-healed production link = %q, want empty", node.LinkedAgentID)
		}
	}
}

// When TLS settings degrade node endpoints to the bare node name, the
// endpoint-host merge alias is identical for both clusters. The alias must
// not merge nodes whose named clusters contradict each other.
func TestUpdateNodesForInstanceEndpointHostAliasDoesNotMergeAcrossClusters(t *testing.T) {
	state := &State{
		Nodes: []Node{
			{
				ID:              "enacon-pve01",
				Name:            "pve01",
				Instance:        "enacon",
				ClusterName:     "enacon",
				IsClusterMember: true,
				Host:            "https://pve01:8006",
				Status:          "online",
			},
		},
	}

	state.UpdateNodesForInstance("rewo", []Node{
		{
			ID:              "rewo-pve01",
			Name:            "pve01",
			Instance:        "rewo",
			ClusterName:     "rewo",
			IsClusterMember: true,
			Host:            "https://pve01:8006",
			Status:          "online",
		},
	})

	if len(state.Nodes) != 2 {
		t.Fatalf("nodes = %#v, want 2 (one per cluster)", state.Nodes)
	}
	clusters := map[string]bool{}
	for _, node := range state.Nodes {
		clusters[node.ClusterName] = true
	}
	if !clusters["enacon"] || !clusters["rewo"] {
		t.Fatalf("clusters = %#v, want both enacon and rewo", clusters)
	}
}

// A standalone node whose endpoint IP is absent from a same-named agent's
// reported IPs must not link to that agent: the IP evidence contradicts the
// hostname match.
func TestUpdateNodesForInstanceHostnameMatchRejectedWhenIPContradicts(t *testing.T) {
	state := &State{
		Hosts: []Host{
			{
				ID:       "host-a",
				Hostname: "pve01",
				ReportIP: "192.168.16.11",
				NetworkInterfaces: []HostNetworkInterface{
					{Name: "eth0", Addresses: []string{"192.168.16.11/24"}},
				},
			},
		},
	}

	state.UpdateNodesForInstance("steinboeck", []Node{
		{
			ID:       "steinboeck-pve01",
			Name:     "pve01",
			Instance: "steinboeck",
			Host:     "https://10.20.1.5:8006",
			Status:   "online",
		},
	})

	if len(state.Nodes) != 1 {
		t.Fatalf("nodes = %#v, want 1", state.Nodes)
	}
	if state.Nodes[0].LinkedAgentID != "" {
		t.Fatalf("LinkedAgentID = %q, want none (agent IPs contradict node endpoint)", state.Nodes[0].LinkedAgentID)
	}
}

// Two different clusters at different sites can reuse the same corosync
// cluster name, the same member hostnames, and the same RFC1918 addressing
// (MSP support case: two "enacon" clusters both exposing pve01 on
// 192.168.1.11). The colliding endpoint-IP alias must not merge the two node
// slots, and the address match must not bind the second site's node to the
// first site's host agent when the TLS fingerprint evidence contradicts.
func TestUpdateNodesForInstanceSameNamedClustersWithCollidingIPsStayApart(t *testing.T) {
	state := &State{
		Hosts: []Host{
			{
				ID:       "host-site-a-pve01",
				Hostname: "pve01",
				ReportIP: "192.168.1.11",
				NetworkInterfaces: []HostNetworkInterface{
					{Name: "eth0", Addresses: []string{"192.168.1.11/24"}},
				},
			},
		},
		Nodes: []Node{
			{
				ID:              "site-a-pve01",
				Name:            "pve01",
				Instance:        "site-a",
				ClusterName:     "enacon",
				IsClusterMember: true,
				Host:            "https://192.168.1.11:8006",
				TLSFingerprint:  "AA:AA:AA:AA",
				LinkedAgentID:   "host-site-a-pve01",
				Status:          "online",
			},
		},
	}

	state.UpdateNodesForInstance("site-b", []Node{
		{
			ID:              "site-b-pve01",
			Name:            "pve01",
			Instance:        "site-b",
			ClusterName:     "enacon",
			IsClusterMember: true,
			Host:            "https://192.168.1.11:8006",
			TLSFingerprint:  "BB:BB:BB:BB",
			Status:          "online",
		},
	})

	if len(state.Nodes) != 2 {
		t.Fatalf("nodes = %#v, want 2 (one per site)", state.Nodes)
	}
	byID := make(map[string]Node)
	for _, node := range state.Nodes {
		byID[node.ID] = node
	}
	siteA, ok := byID["site-a-pve01"]
	if !ok {
		t.Fatalf("site-a-pve01 was clobbered, nodes = %#v", state.Nodes)
	}
	if siteA.LinkedAgentID != "host-site-a-pve01" {
		t.Fatalf("site-a-pve01 LinkedAgentID = %q, want host-site-a-pve01", siteA.LinkedAgentID)
	}
	siteB, ok := byID["site-b-pve01"]
	if !ok {
		t.Fatalf("site-b-pve01 missing, nodes = %#v", state.Nodes)
	}
	if siteB.LinkedAgentID != "" {
		t.Fatalf("site-b-pve01 LinkedAgentID = %q, want none (fingerprint contradicts the colliding-IP agent match)", siteB.LinkedAgentID)
	}
}

// The endpoint-IP agent match must also respect the cluster-identity
// contradiction when no fingerprints are known: an agent whose linked nodes
// live in cluster "enacon" must not bind to a "rewo" node that presents the
// same RFC1918 address from another site.
func TestUpdateNodesForInstanceEndpointIPMatchRejectedWhenClusterContradicts(t *testing.T) {
	state := &State{
		Hosts: []Host{
			{
				ID:       "host-enacon-pve01",
				Hostname: "pve01",
				ReportIP: "192.168.1.11",
				NetworkInterfaces: []HostNetworkInterface{
					{Name: "eth0", Addresses: []string{"192.168.1.11/24"}},
				},
			},
		},
		Nodes: []Node{
			{
				ID:              "enacon-pve01",
				Name:            "pve01",
				Instance:        "enacon",
				ClusterName:     "enacon",
				IsClusterMember: true,
				Host:            "https://192.168.1.11:8006",
				LinkedAgentID:   "host-enacon-pve01",
				Status:          "online",
			},
		},
	}

	state.UpdateNodesForInstance("rewo", []Node{
		{
			ID:              "rewo-pve01",
			Name:            "pve01",
			Instance:        "rewo",
			ClusterName:     "rewo",
			IsClusterMember: true,
			Host:            "https://192.168.1.11:8006",
			Status:          "online",
		},
	})

	if len(state.Nodes) != 2 {
		t.Fatalf("nodes = %#v, want 2 (one per cluster)", state.Nodes)
	}
	for _, node := range state.Nodes {
		if node.ClusterName == "rewo" && node.LinkedAgentID != "" {
			t.Fatalf("rewo pve01 LinkedAgentID = %q, want none (agent belongs to the enacon cluster)", node.LinkedAgentID)
		}
	}
}

// The same cluster added twice through different connection instances is the
// legitimate duplicate: when both views carry the same TOFU-captured TLS
// fingerprint for the same-named node, the endpoint alias must still fold
// them into one slot even though the instances differ.
func TestUpdateNodesForInstanceSameClusterAddedTwiceMergesWithMatchingFingerprints(t *testing.T) {
	state := &State{
		Nodes: []Node{
			{
				ID:              "enacon-a-pve01",
				Name:            "pve01",
				Instance:        "enacon-a",
				ClusterName:     "enacon",
				IsClusterMember: true,
				Host:            "https://192.168.1.11:8006",
				TLSFingerprint:  "AA:AA:AA:AA",
				Status:          "online",
			},
		},
	}

	state.UpdateNodesForInstance("enacon-b", []Node{
		{
			ID:              "enacon-b-pve01",
			Name:            "pve01",
			Instance:        "enacon-b",
			ClusterName:     "enacon",
			IsClusterMember: true,
			Host:            "https://192.168.1.11:8006",
			TLSFingerprint:  "aaaaaaaa",
			Status:          "online",
		},
	})

	if len(state.Nodes) != 1 {
		t.Fatalf("nodes = %#v, want 1 (matching fingerprints prove one machine)", state.Nodes)
	}
	if state.Nodes[0].Name != "pve01" {
		t.Fatalf("merged node = %#v, want pve01", state.Nodes[0])
	}
}

// Without fingerprint evidence the same-named cluster from another instance
// stays split: unknown identity must never fold two possibly-distinct
// clusters into one (fail-safe direction).
func TestUpdateNodesForInstanceSameNamedClustersStayApartWithoutFingerprints(t *testing.T) {
	state := &State{
		Nodes: []Node{
			{
				ID:              "enacon-a-pve01",
				Name:            "pve01",
				Instance:        "enacon-a",
				ClusterName:     "enacon",
				IsClusterMember: true,
				Host:            "https://192.168.1.11:8006",
				Status:          "online",
			},
		},
	}

	state.UpdateNodesForInstance("enacon-b", []Node{
		{
			ID:              "enacon-b-pve01",
			Name:            "pve01",
			Instance:        "enacon-b",
			ClusterName:     "enacon",
			IsClusterMember: true,
			Host:            "https://192.168.1.11:8006",
			Status:          "online",
		},
	})

	if len(state.Nodes) != 2 {
		t.Fatalf("nodes = %#v, want 2 (unknown identity must not merge)", state.Nodes)
	}
}

// A shared agent link is the strongest cross-view merge signal, but for
// same-named clusters from different instances it must still be gated on
// fingerprint identity: matching fingerprints merge the duplicate views onto
// one slot, contradicting fingerprints never do.
func TestUpdateNodesForInstanceSharedAgentLinkMergeGatedOnFingerprint(t *testing.T) {
	makeState := func(existingFP string) *State {
		return &State{
			Hosts: []Host{
				{
					ID:       "host-pve01",
					Hostname: "pve01",
					ReportIP: "192.168.1.11",
					NetworkInterfaces: []HostNetworkInterface{
						{Name: "eth0", Addresses: []string{"192.168.1.11/24"}},
					},
				},
			},
			Nodes: []Node{
				{
					ID:              "enacon-a-pve01",
					Name:            "pve01",
					Instance:        "enacon-a",
					ClusterName:     "enacon",
					IsClusterMember: true,
					Host:            "https://pve01.site-a.internal:8006",
					TLSFingerprint:  existingFP,
					LinkedAgentID:   "host-pve01",
					Status:          "online",
				},
			},
		}
	}
	incoming := func(fp string) []Node {
		return []Node{
			{
				ID:              "enacon-b-pve01",
				Name:            "pve01",
				Instance:        "enacon-b",
				ClusterName:     "enacon",
				IsClusterMember: true,
				Host:            "https://192.168.1.11:8006",
				TLSFingerprint:  fp,
				LinkedAgentID:   "host-pve01",
				Status:          "online",
			},
		}
	}

	merged := makeState("AA:AA:AA:AA")
	merged.UpdateNodesForInstance("enacon-b", incoming("AA:AA:AA:AA"))
	if len(merged.Nodes) != 1 {
		t.Fatalf("matching fingerprints: nodes = %#v, want 1", merged.Nodes)
	}

	split := makeState("AA:AA:AA:AA")
	split.UpdateNodesForInstance("enacon-b", incoming("BB:BB:BB:BB"))
	if len(split.Nodes) != 2 {
		t.Fatalf("contradicting fingerprints: nodes = %#v, want 2", split.Nodes)
	}
}

// A cluster connection's first polls can commit nodes before membership
// detection classifies them (ClusterName ""), and cloned template
// deployments reuse /etc/machine-id, collapsing two sites' agents into one
// identity (MSP support case: clusters enacon and rewo, both with pve01, on
// reused RFC1918 ranges). During that window the shared agent identity must
// not fold the unclassified node into the established cluster's slot.
func TestUpdateNodesForInstanceUnclassifiedNodeSharedAgentMustNotStealClusterSlot(t *testing.T) {
	state := &State{
		Hosts: []Host{
			{
				// One host row standing in for two physical pve01 machines
				// whose cloned machine-id collapsed them into one agent
				// identity; it currently reports the second site's address.
				ID:       "host-cloned-machine-id",
				Hostname: "pve01",
				ReportIP: "192.168.1.11",
				NetworkInterfaces: []HostNetworkInterface{
					{Name: "eth0", Addresses: []string{"192.168.1.11/24"}},
				},
			},
		},
		Nodes: []Node{
			{
				ID:              "enacon-pve01",
				Name:            "pve01",
				Instance:        "enacon",
				ClusterName:     "enacon",
				IsClusterMember: true,
				Host:            "https://192.168.16.11:8006",
				LinkedAgentID:   "host-cloned-machine-id",
				Status:          "online",
			},
		},
	}

	state.UpdateNodesForInstance("rewo", []Node{
		{
			ID:       "rewo-pve01",
			Name:     "pve01",
			Instance: "rewo",
			// Membership detection has not classified this connection yet.
			ClusterName: "",
			Host:        "https://192.168.1.11:8006",
			Status:      "online",
		},
	})

	if len(state.Nodes) != 2 {
		t.Fatalf("nodes = %#v, want 2 (unclassified node must not fold into the enacon slot)", state.Nodes)
	}
	byID := make(map[string]Node)
	for _, node := range state.Nodes {
		byID[node.ID] = node
	}
	enacon, ok := byID["enacon-pve01"]
	if !ok {
		t.Fatalf("enacon-pve01 was clobbered, nodes = %#v", state.Nodes)
	}
	if enacon.ClusterName != "enacon" {
		t.Fatalf("enacon-pve01 ClusterName = %q, want enacon", enacon.ClusterName)
	}
	if _, ok := byID["rewo-pve01"]; !ok {
		t.Fatalf("rewo-pve01 missing, nodes = %#v", state.Nodes)
	}
}

// When TLS settings degrade both connections' node endpoints to the bare
// node name, the endpoint-host alias is identical across sites. An
// unclassified node (empty cluster name, e.g. during the first-poll
// membership window) must not ride that weak alias into another instance's
// established cluster slot.
func TestUpdateNodesForInstanceEndpointHostAliasRequiresProofForUnclassifiedNodes(t *testing.T) {
	state := &State{
		Nodes: []Node{
			{
				ID:              "enacon-pve01",
				Name:            "pve01",
				Instance:        "enacon",
				ClusterName:     "enacon",
				IsClusterMember: true,
				Host:            "https://pve01:8006",
				Status:          "online",
			},
		},
	}

	state.UpdateNodesForInstance("rewo", []Node{
		{
			ID:          "rewo-pve01",
			Name:        "pve01",
			Instance:    "rewo",
			ClusterName: "",
			Host:        "https://pve01:8006",
			Status:      "online",
		},
	})

	if len(state.Nodes) != 2 {
		t.Fatalf("nodes = %#v, want 2 (bare-hostname alias must not fold an unclassified cross-instance node)", state.Nodes)
	}
}

// The legitimate standalone-into-cluster fold must survive the weak-evidence
// tightening: a standalone connection view dialing the same address as the
// cluster's endpoint record still folds via the address alias, and a
// fingerprint-proven view still folds via the shared agent link.
func TestUpdateNodesForInstanceStandaloneViewStillFoldsIntoClusterNode(t *testing.T) {
	byAddress := &State{
		Nodes: []Node{
			{
				ID:              "enacon-pve01",
				Name:            "pve01",
				Instance:        "enacon",
				ClusterName:     "enacon",
				IsClusterMember: true,
				Host:            "https://192.168.16.11:8006",
				Status:          "online",
			},
		},
	}
	byAddress.UpdateNodesForInstance("pve01-direct", []Node{
		{
			ID:       "direct-pve01",
			Name:     "pve01",
			Instance: "pve01-direct",
			Host:     "https://192.168.16.11:8006",
			Status:   "online",
		},
	})
	if len(byAddress.Nodes) != 1 {
		t.Fatalf("address alias: nodes = %#v, want 1 (same-address standalone view folds)", byAddress.Nodes)
	}

	byAgent := &State{
		Hosts: []Host{
			{
				ID:       "host-pve01",
				Hostname: "pve01",
				ReportIP: "192.168.16.11",
				NetworkInterfaces: []HostNetworkInterface{
					{Name: "eth0", Addresses: []string{"192.168.16.11/24"}},
				},
			},
		},
		Nodes: []Node{
			{
				ID:              "enacon-pve01",
				Name:            "pve01",
				Instance:        "enacon",
				ClusterName:     "enacon",
				IsClusterMember: true,
				Host:            "https://pve01:8006",
				TLSFingerprint:  "AA:AA:AA:AA",
				LinkedAgentID:   "host-pve01",
				Status:          "online",
			},
		},
	}
	byAgent.UpdateNodesForInstance("pve01-direct", []Node{
		{
			ID:             "direct-pve01",
			Name:           "pve01",
			Instance:       "pve01-direct",
			Host:           "https://192.168.16.11:8006",
			TLSFingerprint: "aaaaaaaa",
			Status:         "online",
		},
	})
	if len(byAgent.Nodes) != 1 {
		t.Fatalf("agent link: nodes = %#v, want 1 (fingerprint-proven view folds via shared agent)", byAgent.Nodes)
	}
}

// splitNodeDecider reports the pairs an operator split, by node and agent ID.
type splitNodeDecider map[[2]string]bool

func (d splitNodeDecider) ProxmoxNodeAgentSplit(node Node, host Host) bool {
	return d[[2]string{node.ID, host.ID}]
}

// A split recorded after one connection's poll is applied to that connection's
// retained node before its link can fold the next connection's view into it.
func TestUpdateNodesForInstanceReadsASplitOfARetainedNodeBeforeItFolds(t *testing.T) {
	agent := Host{ID: "agent-1", Hostname: "pve1.lan", ReportIP: "10.0.0.5", Status: "online"}
	nodeA := Node{ID: "a-pve1", Name: "pve1", Instance: "a", Host: "https://10.0.0.5:8006", Status: "online"}
	nodeB := Node{ID: "b-pve1", Name: "pve1", Instance: "b", Host: "https://pve1.lan:8006", Status: "online"}

	decider := splitNodeDecider{}
	state := NewState()
	state.UpsertHost(agent)
	state.SetNodeAgentSplitDecider(decider)
	state.UpdateNodesForInstance("a", []Node{nodeA})
	if linked := state.GetSnapshot().Nodes; len(linked) != 1 || linked[0].LinkedAgentID != agent.ID {
		t.Fatalf("fixture: the first poll should link its node to the agent: %+v", linked)
	}

	// The operator splits the first node from the agent; nothing has written
	// a link since, so the retained node still names the agent.
	decider[[2]string{nodeA.ID, agent.ID}] = true
	state.UpdateNodesForInstance("b", []Node{nodeB})

	nodes := state.GetSnapshot().Nodes
	if len(nodes) != 2 {
		t.Fatalf("a retained node's split link still folded the next view into its slot: %+v", nodes)
	}
	for _, node := range nodes {
		switch node.ID {
		case nodeA.ID:
			if node.LinkedAgentID != "" {
				t.Fatalf("node %s kept the agent %q the operator split it from", node.ID, node.LinkedAgentID)
			}
		case nodeB.ID:
			if node.LinkedAgentID != agent.ID {
				t.Fatalf("node %s lost an agent link nobody split: %q", node.ID, node.LinkedAgentID)
			}
		default:
			t.Fatalf("unexpected node %+v", node)
		}
	}
}

// One agent corroborates two standalone connections' nodes through the node's
// IP address and its full hostname, which is the proof that folds the second
// view into the first node's slot. When the operator has split the second
// node from that agent, the agent is no evidence for it, so the view is not
// folded, and neither a node nor the agent keeps a link the operator split.
func TestUpdateNodesForInstanceHoldsASplitAgentBackBeforeItFoldsNodeViews(t *testing.T) {
	agent := Host{
		ID:       "agent-1",
		Hostname: "pve1.lan",
		ReportIP: "10.0.0.5",
		Status:   "online",
	}
	nodeA := Node{ID: "a-pve1", Name: "pve1", Instance: "a", Host: "https://10.0.0.5:8006", Status: "online"}
	nodeB := Node{ID: "b-pve1", Name: "pve1", Instance: "b", Host: "https://pve1.lan:8006", Status: "online"}

	poll := func(decider NodeAgentSplitDecider) []Node {
		state := NewState()
		state.UpsertHost(agent)
		state.SetNodeAgentSplitDecider(decider)
		state.UpdateNodesForInstance("a", []Node{nodeA})
		state.UpdateNodesForInstance("b", []Node{nodeB})
		return state.GetSnapshot().Nodes
	}

	folded := poll(nil)
	if len(folded) != 1 || folded[0].LinkedAgentID != agent.ID {
		t.Fatalf("fixture: the agent should fold both views into one linked node, got %+v", folded)
	}

	apart := poll(splitNodeDecider{{nodeB.ID, agent.ID}: true})
	if len(apart) != 2 {
		t.Fatalf("a node split from the agent folded into its slot anyway: %+v", apart)
	}
	for _, node := range apart {
		switch node.ID {
		case nodeA.ID:
			if node.LinkedAgentID != agent.ID {
				t.Fatalf("the split of the other node unlinked %s from the agent", node.ID)
			}
		case nodeB.ID:
			if node.LinkedAgentID != "" {
				t.Fatalf("node %s kept the agent %q the operator split it from", node.ID, node.LinkedAgentID)
			}
		default:
			t.Fatalf("unexpected node %+v", node)
		}
	}
}

func TestUpdateStorageBackupsForInstanceKeepsArchiveGuestKind(t *testing.T) {
	for _, kind := range []string{"qemu", "vm", " QEMU ", "lxc", "ct", " LXC ", "", "unknown", " UNKNOWN ", "unsupported", "host"} {
		t.Run(kind, func(t *testing.T) {
			state := NewState()
			state.UpdateVMs([]VM{{VMID: 100, Instance: "a", Node: "vm-node"}, {VMID: 100, Instance: "b", Node: "other-cluster"}})
			state.UpdateContainers([]Container{{VMID: 101, Instance: "a", Node: "ct-node"}})
			rows := []StorageBackup{{ID: "100", Instance: "a", VMID: 100, Node: "read-node", StorageNode: "read-node", Type: kind}, {ID: "101", Instance: "a", VMID: 101, Node: "read-node", StorageNode: "read-node", Type: kind}}
			state.UpdateStorageBackupsForInstance("a", rows)
			want100, want101 := "vm-node", "ct-node"
			switch kind {
			case "qemu", "vm", " QEMU ":
				want101 = "read-node"
			case "lxc", "ct", " LXC ":
				want100 = "read-node"
			case "unknown", " UNKNOWN ", "unsupported", "host":
				want100, want101 = "read-node", "read-node"
			}
			snapshot := state.GetSnapshot()
			if len(snapshot.PVEBackups.StorageBackups) != 2 {
				t.Fatal("archive count changed")
			}
			for _, b := range snapshot.PVEBackups.StorageBackups {
				want := want100
				if b.VMID == 101 {
					want = want101
				}
				if b.Node != want || b.StorageNode != "read-node" || b.Type != kind {
					t.Errorf("explicit kind borrowed opposite placement or lost original facts: %+v want=%s", b, want)
				}
			}
			if rows[0].Node != "read-node" || rows[1].Node != "read-node" {
				t.Error("state alignment mutated caller archive facts")
			}
		})
	}
}