Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

52.7 KB
Raw Blame
package monitoring

import (
	"context"
	"encoding/json"
	"fmt"
	"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
	"github.com/rcourtman/pulse-go-rewrite/internal/config"
	"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
	"github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"
	"net/http"
	"net/http/httptest"
	"reflect"
	"strings"
	"sync/atomic"
	"testing"
	"time"

	"github.com/rcourtman/pulse-go-rewrite/internal/models"
)

func TestMakeNodeSnapshotKey(t *testing.T) {
	tests := []struct {
		name     string
		instance string
		node     string
		expected string
	}{
		{
			name:     "simple instance and node",
			instance: "pve1",
			node:     "node1",
			expected: "pve1|node1",
		},
		{
			name:     "instance with port",
			instance: "pve.example.com:8006",
			node:     "pve-node",
			expected: "pve.example.com:8006|pve-node",
		},
		{
			name:     "IP address instance",
			instance: "192.168.1.100",
			node:     "server01",
			expected: "192.168.1.100|server01",
		},
		{
			name:     "empty instance",
			instance: "",
			node:     "node1",
			expected: "|node1",
		},
		{
			name:     "empty node",
			instance: "pve1",
			node:     "",
			expected: "pve1|",
		},
		{
			name:     "both empty",
			instance: "",
			node:     "",
			expected: "|",
		},
		{
			name:     "special characters in node name",
			instance: "pve1",
			node:     "node-with-dashes_and_underscores",
			expected: "pve1|node-with-dashes_and_underscores",
		},
		{
			name:     "FQDN instance",
			instance: "proxmox.datacenter.example.com",
			node:     "pve-cluster-node-01",
			expected: "proxmox.datacenter.example.com|pve-cluster-node-01",
		},
	}

	for _, tc := range tests {
		t.Run(tc.name, func(t *testing.T) {
			got := makeNodeSnapshotKey(tc.instance, tc.node)
			if got != tc.expected {
				t.Errorf("makeNodeSnapshotKey(%q, %q) = %q, want %q",
					tc.instance, tc.node, got, tc.expected)
			}
		})
	}
}

func TestMakeGuestSnapshotKey(t *testing.T) {
	tests := []struct {
		name      string
		instance  string
		guestType string
		node      string
		vmid      int
		expected  string
	}{
		{
			name:      "VM guest",
			instance:  "pve1",
			guestType: "qemu",
			node:      "node1",
			vmid:      100,
			expected:  "pve1|qemu|node1|100",
		},
		{
			name:      "LXC container",
			instance:  "pve1",
			guestType: "lxc",
			node:      "node1",
			vmid:      200,
			expected:  "pve1|lxc|node1|200",
		},
		{
			name:      "high VMID",
			instance:  "pve1",
			guestType: "qemu",
			node:      "node1",
			vmid:      999999,
			expected:  "pve1|qemu|node1|999999",
		},
		{
			name:      "zero VMID",
			instance:  "pve1",
			guestType: "qemu",
			node:      "node1",
			vmid:      0,
			expected:  "pve1|qemu|node1|0",
		},
		{
			name:      "negative VMID",
			instance:  "pve1",
			guestType: "qemu",
			node:      "node1",
			vmid:      -1,
			expected:  "pve1|qemu|node1|-1",
		},
		{
			name:      "instance with port",
			instance:  "pve.example.com:8006",
			guestType: "lxc",
			node:      "pve-node",
			vmid:      101,
			expected:  "pve.example.com:8006|lxc|pve-node|101",
		},
		{
			name:      "empty instance",
			instance:  "",
			guestType: "qemu",
			node:      "node1",
			vmid:      100,
			expected:  "|qemu|node1|100",
		},
		{
			name:      "empty guest type",
			instance:  "pve1",
			guestType: "",
			node:      "node1",
			vmid:      100,
			expected:  "pve1||node1|100",
		},
		{
			name:      "empty node",
			instance:  "pve1",
			guestType: "qemu",
			node:      "",
			vmid:      100,
			expected:  "pve1|qemu||100",
		},
		{
			name:      "all empty except VMID",
			instance:  "",
			guestType: "",
			node:      "",
			vmid:      42,
			expected:  "|||42",
		},
		{
			name:      "FQDN with complex node name",
			instance:  "proxmox.datacenter.example.com",
			guestType: "qemu",
			node:      "pve-cluster-node-01",
			vmid:      12345,
			expected:  "proxmox.datacenter.example.com|qemu|pve-cluster-node-01|12345",
		},
		{
			name:      "IP address instance",
			instance:  "192.168.1.100",
			guestType: "lxc",
			node:      "server01",
			vmid:      500,
			expected:  "192.168.1.100|lxc|server01|500",
		},
	}

	for _, tc := range tests {
		t.Run(tc.name, func(t *testing.T) {
			got := makeGuestSnapshotKey(tc.instance, tc.guestType, tc.node, tc.vmid)
			if got != tc.expected {
				t.Errorf("makeGuestSnapshotKey(%q, %q, %q, %d) = %q, want %q",
					tc.instance, tc.guestType, tc.node, tc.vmid, got, tc.expected)
			}
		})
	}
}

func TestMakeSnapshotKey_Uniqueness(t *testing.T) {
	// Test that different inputs produce different keys

	t.Run("node keys with different instances are unique", func(t *testing.T) {
		key1 := makeNodeSnapshotKey("pve1", "node1")
		key2 := makeNodeSnapshotKey("pve2", "node1")
		if key1 == key2 {
			t.Errorf("Keys should be unique: %q == %q", key1, key2)
		}
	})

	t.Run("node keys with different nodes are unique", func(t *testing.T) {
		key1 := makeNodeSnapshotKey("pve1", "node1")
		key2 := makeNodeSnapshotKey("pve1", "node2")
		if key1 == key2 {
			t.Errorf("Keys should be unique: %q == %q", key1, key2)
		}
	})

	t.Run("guest keys with different VMIDs are unique", func(t *testing.T) {
		key1 := makeGuestSnapshotKey("pve1", "qemu", "node1", 100)
		key2 := makeGuestSnapshotKey("pve1", "qemu", "node1", 101)
		if key1 == key2 {
			t.Errorf("Keys should be unique: %q == %q", key1, key2)
		}
	})

	t.Run("guest keys with different guest types are unique", func(t *testing.T) {
		key1 := makeGuestSnapshotKey("pve1", "qemu", "node1", 100)
		key2 := makeGuestSnapshotKey("pve1", "lxc", "node1", 100)
		if key1 == key2 {
			t.Errorf("Keys should be unique: %q == %q", key1, key2)
		}
	})

	t.Run("guest keys with different nodes are unique", func(t *testing.T) {
		key1 := makeGuestSnapshotKey("pve1", "qemu", "node1", 100)
		key2 := makeGuestSnapshotKey("pve1", "qemu", "node2", 100)
		if key1 == key2 {
			t.Errorf("Keys should be unique: %q == %q", key1, key2)
		}
	})

	t.Run("guest keys with different instances are unique", func(t *testing.T) {
		key1 := makeGuestSnapshotKey("pve1", "qemu", "node1", 100)
		key2 := makeGuestSnapshotKey("pve2", "qemu", "node1", 100)
		if key1 == key2 {
			t.Errorf("Keys should be unique: %q == %q", key1, key2)
		}
	})
}

func TestDiagnosticSnapshotSet_Fields(t *testing.T) {
	// Test struct field initialization
	set := DiagnosticSnapshotSet{
		Nodes:  []NodeMemorySnapshot{},
		Guests: []GuestMemorySnapshot{},
	}

	if set.Nodes == nil {
		t.Error("Nodes should not be nil")
	}
	if set.Guests == nil {
		t.Error("Guests should not be nil")
	}
	if len(set.Nodes) != 0 {
		t.Errorf("Nodes length = %d, want 0", len(set.Nodes))
	}
	if len(set.Guests) != 0 {
		t.Errorf("Guests length = %d, want 0", len(set.Guests))
	}
}

func TestNodeMemoryRaw_Fields(t *testing.T) {
	raw := NodeMemoryRaw{
		Total:               16000000000,
		Used:                8000000000,
		Free:                8000000000,
		Available:           10000000000,
		Buffers:             500000000,
		Cached:              2000000000,
		ProxmoxMemorySource: "rrd-available",
	}

	if raw.Total != 16000000000 {
		t.Errorf("Total = %d, want 16000000000", raw.Total)
	}
	if raw.Used != 8000000000 {
		t.Errorf("Used = %d, want 8000000000", raw.Used)
	}
	if raw.Free != 8000000000 {
		t.Errorf("Free = %d, want 8000000000", raw.Free)
	}
	if raw.Available != 10000000000 {
		t.Errorf("Available = %d, want 10000000000", raw.Available)
	}
	if raw.Buffers != 500000000 {
		t.Errorf("Buffers = %d, want 500000000", raw.Buffers)
	}
	if raw.Cached != 2000000000 {
		t.Errorf("Cached = %d, want 2000000000", raw.Cached)
	}
	if raw.ProxmoxMemorySource != "rrd-available" {
		t.Errorf("ProxmoxMemorySource = %q, want %q", raw.ProxmoxMemorySource, "rrd-available")
	}
}

func TestVMMemoryRaw_Fields(t *testing.T) {
	raw := VMMemoryRaw{
		ListingMem:             2000000000,
		ListingMaxMem:          4000000000,
		StatusMem:              2100000000,
		StatusMaxMem:           4000000000,
		Balloon:                3000000000,
		BalloonMin:             1000000000,
		GuestAgentMemAvailable: 1500000000,
		Agent:                  1,
	}

	if raw.ListingMem != 2000000000 {
		t.Errorf("ListingMem = %d, want 2000000000", raw.ListingMem)
	}
	if raw.ListingMaxMem != 4000000000 {
		t.Errorf("ListingMaxMem = %d, want 4000000000", raw.ListingMaxMem)
	}
	if raw.StatusMem != 2100000000 {
		t.Errorf("StatusMem = %d, want 2100000000", raw.StatusMem)
	}
	if raw.StatusMaxMem != 4000000000 {
		t.Errorf("StatusMaxMem = %d, want 4000000000", raw.StatusMaxMem)
	}
	if raw.Balloon != 3000000000 {
		t.Errorf("Balloon = %d, want 3000000000", raw.Balloon)
	}
	if raw.BalloonMin != 1000000000 {
		t.Errorf("BalloonMin = %d, want 1000000000", raw.BalloonMin)
	}
	if raw.GuestAgentMemAvailable != 1500000000 {
		t.Errorf("GuestAgentMemAvailable = %d, want 1500000000", raw.GuestAgentMemAvailable)
	}
	if raw.Agent != 1 {
		t.Errorf("Agent = %d, want 1", raw.Agent)
	}
}

func TestRecordGuestSnapshot(t *testing.T) {
	t.Run("nil Monitor is no-op", func(t *testing.T) {
		var m *Monitor
		// Should not panic
		m.recordGuestSnapshot("instance", "qemu", "node1", 100, GuestMemorySnapshot{})
	})

	t.Run("records single guest snapshot", func(t *testing.T) {
		m := &Monitor{
			nodeSnapshots:  make(map[string]NodeMemorySnapshot),
			guestSnapshots: make(map[string]GuestMemorySnapshot),
		}

		m.recordGuestSnapshot("pve1", "qemu", "node1", 100, GuestMemorySnapshot{
			Name:         "testvm",
			Status:       "running",
			MemorySource: "status-mem",
		})

		if len(m.guestSnapshots) != 1 {
			t.Fatalf("Expected 1 guest snapshot, got %d", len(m.guestSnapshots))
		}

		key := makeGuestSnapshotKey("pve1", "qemu", "node1", 100)
		snapshot, ok := m.guestSnapshots[key]
		if !ok {
			t.Fatalf("Expected snapshot with key %q not found", key)
		}

		if snapshot.Name != "testvm" {
			t.Errorf("Name = %q, want %q", snapshot.Name, "testvm")
		}
		if snapshot.Status != "running" {
			t.Errorf("Status = %q, want %q", snapshot.Status, "running")
		}
		if snapshot.MemorySource != "status-mem" {
			t.Errorf("MemorySource = %q, want %q", snapshot.MemorySource, "status-mem")
		}
	})

	t.Run("canonicalizes guest memory source alias on record", func(t *testing.T) {
		m := &Monitor{
			nodeSnapshots:  make(map[string]NodeMemorySnapshot),
			guestSnapshots: make(map[string]GuestMemorySnapshot),
		}

		m.recordGuestSnapshot("pve1", "qemu", "node1", 100, GuestMemorySnapshot{
			Name:         "testvm",
			Status:       "running",
			MemorySource: "meminfo-total-minus-used",
		})

		key := makeGuestSnapshotKey("pve1", "qemu", "node1", 100)
		snapshot := m.guestSnapshots[key]
		if snapshot.MemorySource != "derived-total-minus-used" {
			t.Errorf("MemorySource = %q, want %q", snapshot.MemorySource, "derived-total-minus-used")
		}
		if snapshot.FallbackReason != "derived-total-minus-used" {
			t.Errorf("FallbackReason = %q, want %q", snapshot.FallbackReason, "derived-total-minus-used")
		}
	})

	t.Run("sets Instance, GuestType, Node, VMID from parameters", func(t *testing.T) {
		m := &Monitor{
			nodeSnapshots:  make(map[string]NodeMemorySnapshot),
			guestSnapshots: make(map[string]GuestMemorySnapshot),
		}

		m.recordGuestSnapshot("my-instance", "lxc", "my-node", 200, GuestMemorySnapshot{
			Name: "testct",
		})

		key := makeGuestSnapshotKey("my-instance", "lxc", "my-node", 200)
		snapshot := m.guestSnapshots[key]

		if snapshot.Instance != "my-instance" {
			t.Errorf("Instance = %q, want %q", snapshot.Instance, "my-instance")
		}
		if snapshot.GuestType != "lxc" {
			t.Errorf("GuestType = %q, want %q", snapshot.GuestType, "lxc")
		}
		if snapshot.Node != "my-node" {
			t.Errorf("Node = %q, want %q", snapshot.Node, "my-node")
		}
		if snapshot.VMID != 200 {
			t.Errorf("VMID = %d, want %d", snapshot.VMID, 200)
		}
	})

	t.Run("sets RetrievedAt to current time when zero", func(t *testing.T) {
		m := &Monitor{
			nodeSnapshots:  make(map[string]NodeMemorySnapshot),
			guestSnapshots: make(map[string]GuestMemorySnapshot),
		}

		beforeRecord := time.Now()
		m.recordGuestSnapshot("pve1", "qemu", "node1", 100, GuestMemorySnapshot{})
		afterRecord := time.Now()

		key := makeGuestSnapshotKey("pve1", "qemu", "node1", 100)
		snapshot := m.guestSnapshots[key]

		if snapshot.RetrievedAt.Before(beforeRecord) || snapshot.RetrievedAt.After(afterRecord) {
			t.Errorf("RetrievedAt = %v, want between %v and %v", snapshot.RetrievedAt, beforeRecord, afterRecord)
		}
	})

	t.Run("preserves non-zero RetrievedAt", func(t *testing.T) {
		m := &Monitor{
			nodeSnapshots:  make(map[string]NodeMemorySnapshot),
			guestSnapshots: make(map[string]GuestMemorySnapshot),
		}

		specificTime := time.Date(2024, 6, 15, 12, 0, 0, 0, time.UTC)
		m.recordGuestSnapshot("pve1", "qemu", "node1", 100, GuestMemorySnapshot{
			RetrievedAt: specificTime,
		})

		key := makeGuestSnapshotKey("pve1", "qemu", "node1", 100)
		snapshot := m.guestSnapshots[key]

		if !snapshot.RetrievedAt.Equal(specificTime) {
			t.Errorf("RetrievedAt = %v, want %v", snapshot.RetrievedAt, specificTime)
		}
	})

	t.Run("records multiple guests with different keys", func(t *testing.T) {
		m := &Monitor{
			nodeSnapshots:  make(map[string]NodeMemorySnapshot),
			guestSnapshots: make(map[string]GuestMemorySnapshot),
		}

		m.recordGuestSnapshot("pve1", "qemu", "node1", 100, GuestMemorySnapshot{Name: "vm1"})
		m.recordGuestSnapshot("pve1", "qemu", "node1", 101, GuestMemorySnapshot{Name: "vm2"})
		m.recordGuestSnapshot("pve1", "lxc", "node1", 200, GuestMemorySnapshot{Name: "ct1"})
		m.recordGuestSnapshot("pve2", "qemu", "node2", 100, GuestMemorySnapshot{Name: "vm3"})

		if len(m.guestSnapshots) != 4 {
			t.Fatalf("Expected 4 guest snapshots, got %d", len(m.guestSnapshots))
		}

		// Verify each one exists
		key1 := makeGuestSnapshotKey("pve1", "qemu", "node1", 100)
		key2 := makeGuestSnapshotKey("pve1", "qemu", "node1", 101)
		key3 := makeGuestSnapshotKey("pve1", "lxc", "node1", 200)
		key4 := makeGuestSnapshotKey("pve2", "qemu", "node2", 100)

		if m.guestSnapshots[key1].Name != "vm1" {
			t.Errorf("Snapshot 1 Name = %q, want %q", m.guestSnapshots[key1].Name, "vm1")
		}
		if m.guestSnapshots[key2].Name != "vm2" {
			t.Errorf("Snapshot 2 Name = %q, want %q", m.guestSnapshots[key2].Name, "vm2")
		}
		if m.guestSnapshots[key3].Name != "ct1" {
			t.Errorf("Snapshot 3 Name = %q, want %q", m.guestSnapshots[key3].Name, "ct1")
		}
		if m.guestSnapshots[key4].Name != "vm3" {
			t.Errorf("Snapshot 4 Name = %q, want %q", m.guestSnapshots[key4].Name, "vm3")
		}
	})

	t.Run("overwrites existing guest snapshot with same key", func(t *testing.T) {
		m := &Monitor{
			nodeSnapshots:  make(map[string]NodeMemorySnapshot),
			guestSnapshots: make(map[string]GuestMemorySnapshot),
		}

		m.recordGuestSnapshot("pve1", "qemu", "node1", 100, GuestMemorySnapshot{
			Name:         "oldname",
			Status:       "stopped",
			MemorySource: "listing",
		})

		m.recordGuestSnapshot("pve1", "qemu", "node1", 100, GuestMemorySnapshot{
			Name:         "newname",
			Status:       "running",
			MemorySource: "status-mem",
		})

		if len(m.guestSnapshots) != 1 {
			t.Fatalf("Expected 1 guest snapshot after overwrite, got %d", len(m.guestSnapshots))
		}

		key := makeGuestSnapshotKey("pve1", "qemu", "node1", 100)
		snapshot := m.guestSnapshots[key]

		if snapshot.Name != "newname" {
			t.Errorf("Name = %q, want %q", snapshot.Name, "newname")
		}
		if snapshot.Status != "running" {
			t.Errorf("Status = %q, want %q", snapshot.Status, "running")
		}
		if snapshot.MemorySource != "status-mem" {
			t.Errorf("MemorySource = %q, want %q", snapshot.MemorySource, "status-mem")
		}
	})

	t.Run("initializes nil guestSnapshots map", func(t *testing.T) {
		m := &Monitor{
			nodeSnapshots: make(map[string]NodeMemorySnapshot),
			// guestSnapshots intentionally nil
		}

		m.recordGuestSnapshot("pve1", "qemu", "node1", 100, GuestMemorySnapshot{Name: "test"})

		if m.guestSnapshots == nil {
			t.Fatal("guestSnapshots should have been initialized")
		}
		if len(m.guestSnapshots) != 1 {
			t.Errorf("Expected 1 guest snapshot, got %d", len(m.guestSnapshots))
		}
	})
}

func TestLogNodeMemorySource(t *testing.T) {
	t.Run("nil Monitor is no-op", func(t *testing.T) {
		var m *Monitor
		// Should not panic
		m.logNodeMemorySource("instance", "node1", NodeMemorySnapshot{
			MemorySource: "rrd-data",
		})
	})

	t.Run("same source as existing snapshot - no log emitted", func(t *testing.T) {
		m := &Monitor{
			nodeSnapshots:  make(map[string]NodeMemorySnapshot),
			guestSnapshots: make(map[string]GuestMemorySnapshot),
		}

		// Pre-populate with existing snapshot
		key := makeNodeSnapshotKey("pve1", "node1")
		m.nodeSnapshots[key] = NodeMemorySnapshot{
			MemorySource: "rrd-data",
		}

		// Should not panic and should return early (same source)
		m.logNodeMemorySource("pve1", "node1", NodeMemorySnapshot{
			MemorySource: "rrd-data",
		})
	})

	t.Run("empty source triggers warn level", func(t *testing.T) {
		m := &Monitor{
			nodeSnapshots:  make(map[string]NodeMemorySnapshot),
			guestSnapshots: make(map[string]GuestMemorySnapshot),
		}

		// Should not panic - empty source triggers Warn
		m.logNodeMemorySource("pve1", "node1", NodeMemorySnapshot{
			MemorySource: "",
		})
	})

	t.Run("nodes-endpoint source triggers warn level", func(t *testing.T) {
		m := &Monitor{
			nodeSnapshots:  make(map[string]NodeMemorySnapshot),
			guestSnapshots: make(map[string]GuestMemorySnapshot),
		}

		// Should not panic - nodes-endpoint triggers Warn
		m.logNodeMemorySource("pve1", "node1", NodeMemorySnapshot{
			MemorySource: "nodes-endpoint",
		})
	})

	t.Run("node-status-used source triggers warn level", func(t *testing.T) {
		m := &Monitor{
			nodeSnapshots:  make(map[string]NodeMemorySnapshot),
			guestSnapshots: make(map[string]GuestMemorySnapshot),
		}

		// Should not panic - node-status-used triggers Warn
		m.logNodeMemorySource("pve1", "node1", NodeMemorySnapshot{
			MemorySource: "node-status-used",
		})
	})

	t.Run("previous-snapshot source triggers warn level", func(t *testing.T) {
		m := &Monitor{
			nodeSnapshots:  make(map[string]NodeMemorySnapshot),
			guestSnapshots: make(map[string]GuestMemorySnapshot),
		}

		// Should not panic - previous-snapshot triggers Warn
		m.logNodeMemorySource("pve1", "node1", NodeMemorySnapshot{
			MemorySource: "previous-snapshot",
		})
	})

	t.Run("rrd-data source triggers debug level", func(t *testing.T) {
		m := &Monitor{
			nodeSnapshots:  make(map[string]NodeMemorySnapshot),
			guestSnapshots: make(map[string]GuestMemorySnapshot),
		}

		// Should not panic - rrd-data triggers Debug
		m.logNodeMemorySource("pve1", "node1", NodeMemorySnapshot{
			MemorySource: "rrd-data",
		})
	})

	t.Run("rrd-available source triggers debug level", func(t *testing.T) {
		m := &Monitor{
			nodeSnapshots:  make(map[string]NodeMemorySnapshot),
			guestSnapshots: make(map[string]GuestMemorySnapshot),
		}

		// Should not panic - rrd-available triggers Debug
		m.logNodeMemorySource("pve1", "node1", NodeMemorySnapshot{
			MemorySource: "rrd-available",
		})
	})

	t.Run("source change from existing triggers log", func(t *testing.T) {
		m := &Monitor{
			nodeSnapshots:  make(map[string]NodeMemorySnapshot),
			guestSnapshots: make(map[string]GuestMemorySnapshot),
		}

		// Pre-populate with existing snapshot
		key := makeNodeSnapshotKey("pve1", "node1")
		m.nodeSnapshots[key] = NodeMemorySnapshot{
			MemorySource: "rrd-data",
		}

		// Different source should trigger logging
		m.logNodeMemorySource("pve1", "node1", NodeMemorySnapshot{
			MemorySource: "rrd-available",
		})
	})

	t.Run("FallbackReason is logged when present", func(t *testing.T) {
		m := &Monitor{
			nodeSnapshots:  make(map[string]NodeMemorySnapshot),
			guestSnapshots: make(map[string]GuestMemorySnapshot),
		}

		// Should not panic - FallbackReason present
		m.logNodeMemorySource("pve1", "node1", NodeMemorySnapshot{
			MemorySource:   "previous-snapshot",
			FallbackReason: "no rrd data available",
		})
	})

	t.Run("Raw fields are logged when > 0", func(t *testing.T) {
		m := &Monitor{
			nodeSnapshots:  make(map[string]NodeMemorySnapshot),
			guestSnapshots: make(map[string]GuestMemorySnapshot),
		}

		// Should not panic - various Raw fields > 0
		m.logNodeMemorySource("pve1", "node1", NodeMemorySnapshot{
			MemorySource: "rrd-data",
			Raw: NodeMemoryRaw{
				Available:           1000000000,
				Buffers:             200000000,
				Cached:              500000000,
				TotalMinusUsed:      800000000,
				RRDAvailable:        900000000,
				RRDUsed:             100000000,
				RRDTotal:            1000000000,
				ProxmoxMemorySource: "rrd",
			},
		})
	})

	t.Run("Memory fields are logged when > 0", func(t *testing.T) {
		m := &Monitor{
			nodeSnapshots:  make(map[string]NodeMemorySnapshot),
			guestSnapshots: make(map[string]GuestMemorySnapshot),
		}

		// Should not panic - Memory fields > 0
		m.logNodeMemorySource("pve1", "node1", NodeMemorySnapshot{
			MemorySource: "rrd-data",
			Memory: models.Memory{
				Total: 16000000000,
				Used:  8000000000,
				Free:  8000000000,
				Usage: 0.5,
			},
		})
	})

	t.Run("all warn sources", func(t *testing.T) {
		warnSources := []string{"", "nodes-endpoint", "node-status-used", "previous-snapshot"}

		for _, source := range warnSources {
			t.Run("source_"+source, func(t *testing.T) {
				m := &Monitor{
					nodeSnapshots:  make(map[string]NodeMemorySnapshot),
					guestSnapshots: make(map[string]GuestMemorySnapshot),
				}

				// Should not panic
				m.logNodeMemorySource("pve1", "node1", NodeMemorySnapshot{
					MemorySource: source,
				})
			})
		}
	})

	t.Run("debug sources", func(t *testing.T) {
		debugSources := []string{"rrd-data", "rrd-available", "node-status-available", "calculated"}

		for _, source := range debugSources {
			t.Run("source_"+source, func(t *testing.T) {
				m := &Monitor{
					nodeSnapshots:  make(map[string]NodeMemorySnapshot),
					guestSnapshots: make(map[string]GuestMemorySnapshot),
				}

				// Should not panic
				m.logNodeMemorySource("pve1", "node1", NodeMemorySnapshot{
					MemorySource: source,
				})
			})
		}
	})

	t.Run("new node with no prior snapshot logs", func(t *testing.T) {
		m := &Monitor{
			nodeSnapshots:  make(map[string]NodeMemorySnapshot),
			guestSnapshots: make(map[string]GuestMemorySnapshot),
		}

		// No existing snapshot, empty prevSource should not match non-empty source
		m.logNodeMemorySource("pve1", "node1", NodeMemorySnapshot{
			MemorySource: "rrd-data",
		})
	})

	t.Run("new node with empty source matches no prior snapshot", func(t *testing.T) {
		m := &Monitor{
			nodeSnapshots:  make(map[string]NodeMemorySnapshot),
			guestSnapshots: make(map[string]GuestMemorySnapshot),
		}

		// No existing snapshot means prevSource is "", matching empty MemorySource
		// This should skip logging due to prevSource == snapshot.MemorySource
		m.logNodeMemorySource("pve1", "node1", NodeMemorySnapshot{
			MemorySource: "",
		})
	})
}

func TestLogGuestMemorySource(t *testing.T) {
	t.Run("nil Monitor is no-op", func(t *testing.T) {
		var m *Monitor
		m.logGuestMemorySource("instance", "qemu", "node1", 100, GuestMemorySnapshot{
			MemorySource: "agent",
		})
	})

	t.Run("same source as existing snapshot - no log emitted", func(t *testing.T) {
		m := &Monitor{
			nodeSnapshots:  make(map[string]NodeMemorySnapshot),
			guestSnapshots: make(map[string]GuestMemorySnapshot),
		}

		key := makeGuestSnapshotKey("pve1", "qemu", "node1", 100)
		m.guestSnapshots[key] = GuestMemorySnapshot{
			MemorySource: "agent",
		}

		m.logGuestMemorySource("pve1", "qemu", "node1", 100, GuestMemorySnapshot{
			MemorySource: "agent",
		})
	})

	t.Run("fallback source triggers warn level", func(t *testing.T) {
		m := &Monitor{
			nodeSnapshots:  make(map[string]NodeMemorySnapshot),
			guestSnapshots: make(map[string]GuestMemorySnapshot),
		}

		m.logGuestMemorySource("pve1", "qemu", "node1", 100, GuestMemorySnapshot{
			Name:           "vm1",
			Status:         "running",
			MemorySource:   "derived-total-minus-used",
			FallbackReason: "derived-total-minus-used",
		})
	})

	t.Run("preferred source triggers debug level", func(t *testing.T) {
		m := &Monitor{
			nodeSnapshots:  make(map[string]NodeMemorySnapshot),
			guestSnapshots: make(map[string]GuestMemorySnapshot),
		}

		m.logGuestMemorySource("pve1", "qemu", "node1", 100, GuestMemorySnapshot{
			Name:         "vm1",
			Status:       "running",
			MemorySource: "available-field",
		})
	})

	t.Run("raw and memory fields are logged when populated", func(t *testing.T) {
		m := &Monitor{
			nodeSnapshots:  make(map[string]NodeMemorySnapshot),
			guestSnapshots: make(map[string]GuestMemorySnapshot),
		}

		m.logGuestMemorySource("pve1", "qemu", "node1", 100, GuestMemorySnapshot{
			Name:           "vm1",
			Status:         "running",
			MemorySource:   "rrd-memavailable",
			FallbackReason: "rrd-memavailable",
			Memory: models.Memory{
				Total: 8_000_000_000,
				Used:  3_000_000_000,
				Free:  5_000_000_000,
				Usage: 0.375,
			},
			Raw: VMMemoryRaw{
				ListingMem:            3_000_000_000,
				ListingMaxMem:         8_000_000_000,
				StatusMem:             3_100_000_000,
				StatusFreeMem:         4_900_000_000,
				StatusMaxMem:          8_000_000_000,
				MemInfoAvailable:      4_800_000_000,
				MemInfoBuffers:        200_000_000,
				MemInfoCached:         700_000_000,
				MemInfoTotalMinusUsed: 4_900_000_000,
				HostAgentTotal:        8_000_000_000,
				HostAgentUsed:         3_200_000_000,
			},
		})
	})
}

func TestRecordNodeSnapshot(t *testing.T) {
	tests := []struct {
		name             string
		monitor          *Monitor
		instance         string
		node             string
		snapshot         NodeMemorySnapshot
		wantPanic        bool
		wantMapLen       int
		wantInstance     string
		wantNode         string
		checkRetrievedAt string // "set" (auto-set), "preserved" (kept from input), "skip" (don't check)
		inputRetrievedAt time.Time
	}{
		{
			name:             "nil Monitor returns early without panic",
			monitor:          nil,
			instance:         "pve1",
			node:             "node1",
			snapshot:         NodeMemorySnapshot{},
			wantPanic:        false,
			checkRetrievedAt: "skip",
		},
		{
			name: "nil nodeSnapshots map gets initialized",
			monitor: &Monitor{
				nodeSnapshots:  nil,
				guestSnapshots: make(map[string]GuestMemorySnapshot),
			},
			instance:         "pve1",
			node:             "node1",
			snapshot:         NodeMemorySnapshot{MemorySource: "test"},
			wantMapLen:       1,
			wantInstance:     "pve1",
			wantNode:         "node1",
			checkRetrievedAt: "set",
		},
		{
			name: "Instance and Node are set from parameters",
			monitor: &Monitor{
				nodeSnapshots:  make(map[string]NodeMemorySnapshot),
				guestSnapshots: make(map[string]GuestMemorySnapshot),
			},
			instance: "my-instance",
			node:     "my-node",
			snapshot: NodeMemorySnapshot{
				Instance: "wrong-instance",
				Node:     "wrong-node",
			},
			wantMapLen:       1,
			wantInstance:     "my-instance",
			wantNode:         "my-node",
			checkRetrievedAt: "set",
		},
		{
			name: "zero RetrievedAt gets set to current time",
			monitor: &Monitor{
				nodeSnapshots:  make(map[string]NodeMemorySnapshot),
				guestSnapshots: make(map[string]GuestMemorySnapshot),
			},
			instance:         "pve1",
			node:             "node1",
			snapshot:         NodeMemorySnapshot{},
			wantMapLen:       1,
			wantInstance:     "pve1",
			wantNode:         "node1",
			checkRetrievedAt: "set",
		},
		{
			name: "non-zero RetrievedAt is preserved",
			monitor: &Monitor{
				nodeSnapshots:  make(map[string]NodeMemorySnapshot),
				guestSnapshots: make(map[string]GuestMemorySnapshot),
			},
			instance:         "pve1",
			node:             "node1",
			snapshot:         NodeMemorySnapshot{RetrievedAt: time.Date(2024, 6, 15, 12, 0, 0, 0, time.UTC)},
			wantMapLen:       1,
			wantInstance:     "pve1",
			wantNode:         "node1",
			checkRetrievedAt: "preserved",
			inputRetrievedAt: time.Date(2024, 6, 15, 12, 0, 0, 0, time.UTC),
		},
		{
			name: "snapshot is stored with correct key",
			monitor: &Monitor{
				nodeSnapshots:  make(map[string]NodeMemorySnapshot),
				guestSnapshots: make(map[string]GuestMemorySnapshot),
			},
			instance:         "pve1",
			node:             "node1",
			snapshot:         NodeMemorySnapshot{MemorySource: "rrd-available"},
			wantMapLen:       1,
			wantInstance:     "pve1",
			wantNode:         "node1",
			checkRetrievedAt: "set",
		},
	}

	for _, tc := range tests {
		t.Run(tc.name, func(t *testing.T) {
			beforeRecord := time.Now()

			// Call recordNodeSnapshot - should not panic
			tc.monitor.recordNodeSnapshot(tc.instance, tc.node, tc.snapshot)

			afterRecord := time.Now()

			// For nil monitor, we just verify no panic occurred
			if tc.monitor == nil {
				return
			}

			// Verify map was initialized and has expected length
			if tc.monitor.nodeSnapshots == nil {
				t.Fatal("nodeSnapshots should have been initialized")
			}
			if len(tc.monitor.nodeSnapshots) != tc.wantMapLen {
				t.Errorf("nodeSnapshots length = %d, want %d", len(tc.monitor.nodeSnapshots), tc.wantMapLen)
			}

			// Verify snapshot was stored with correct key
			key := makeNodeSnapshotKey(tc.instance, tc.node)
			stored, ok := tc.monitor.nodeSnapshots[key]
			if !ok {
				t.Fatalf("snapshot not found with key %q", key)
			}

			// Verify Instance and Node were set from parameters
			if stored.Instance != tc.wantInstance {
				t.Errorf("Instance = %q, want %q", stored.Instance, tc.wantInstance)
			}
			if stored.Node != tc.wantNode {
				t.Errorf("Node = %q, want %q", stored.Node, tc.wantNode)
			}

			// Verify RetrievedAt handling
			switch tc.checkRetrievedAt {
			case "set":
				if stored.RetrievedAt.Before(beforeRecord) || stored.RetrievedAt.After(afterRecord) {
					t.Errorf("RetrievedAt = %v, want between %v and %v", stored.RetrievedAt, beforeRecord, afterRecord)
				}
			case "preserved":
				if !stored.RetrievedAt.Equal(tc.inputRetrievedAt) {
					t.Errorf("RetrievedAt = %v, want %v", stored.RetrievedAt, tc.inputRetrievedAt)
				}
			}
		})
	}
}

func TestRecordNodeSnapshotCanonicalizesMemorySource(t *testing.T) {
	m := &Monitor{
		nodeSnapshots:  make(map[string]NodeMemorySnapshot),
		guestSnapshots: make(map[string]GuestMemorySnapshot),
	}

	m.recordNodeSnapshot("pve1", "node1", NodeMemorySnapshot{
		MemorySource: "avail-field",
	})

	snapshot := m.nodeSnapshots[makeNodeSnapshotKey("pve1", "node1")]
	if snapshot.MemorySource != "available-field" {
		t.Fatalf("MemorySource = %q, want available-field", snapshot.MemorySource)
	}
	if snapshot.FallbackReason != "" {
		t.Fatalf("FallbackReason = %q, want empty", snapshot.FallbackReason)
	}
}

func TestRecordNodeSnapshot_MultipleSnapshots(t *testing.T) {
	t.Run("records multiple nodes with different keys", func(t *testing.T) {
		m := &Monitor{
			nodeSnapshots:  make(map[string]NodeMemorySnapshot),
			guestSnapshots: make(map[string]GuestMemorySnapshot),
		}

		m.recordNodeSnapshot("pve1", "node1", NodeMemorySnapshot{MemorySource: "src1"})
		m.recordNodeSnapshot("pve1", "node2", NodeMemorySnapshot{MemorySource: "src2"})
		m.recordNodeSnapshot("pve2", "node1", NodeMemorySnapshot{MemorySource: "src3"})

		if len(m.nodeSnapshots) != 3 {
			t.Fatalf("Expected 3 node snapshots, got %d", len(m.nodeSnapshots))
		}

		// Verify each one exists with correct data
		key1 := makeNodeSnapshotKey("pve1", "node1")
		key2 := makeNodeSnapshotKey("pve1", "node2")
		key3 := makeNodeSnapshotKey("pve2", "node1")

		if m.nodeSnapshots[key1].MemorySource != "src1" {
			t.Errorf("Snapshot 1 MemorySource = %q, want %q", m.nodeSnapshots[key1].MemorySource, "src1")
		}
		if m.nodeSnapshots[key2].MemorySource != "src2" {
			t.Errorf("Snapshot 2 MemorySource = %q, want %q", m.nodeSnapshots[key2].MemorySource, "src2")
		}
		if m.nodeSnapshots[key3].MemorySource != "src3" {
			t.Errorf("Snapshot 3 MemorySource = %q, want %q", m.nodeSnapshots[key3].MemorySource, "src3")
		}
	})

	t.Run("overwrites existing snapshot with same key", func(t *testing.T) {
		m := &Monitor{
			nodeSnapshots:  make(map[string]NodeMemorySnapshot),
			guestSnapshots: make(map[string]GuestMemorySnapshot),
		}

		m.recordNodeSnapshot("pve1", "node1", NodeMemorySnapshot{
			MemorySource: "old-source",
		})

		m.recordNodeSnapshot("pve1", "node1", NodeMemorySnapshot{
			MemorySource: "new-source",
		})

		if len(m.nodeSnapshots) != 1 {
			t.Fatalf("Expected 1 node snapshot after overwrite, got %d", len(m.nodeSnapshots))
		}

		key := makeNodeSnapshotKey("pve1", "node1")
		if m.nodeSnapshots[key].MemorySource != "new-source" {
			t.Errorf("MemorySource = %q, want %q", m.nodeSnapshots[key].MemorySource, "new-source")
		}
	})
}

func TestGetDiagnosticSnapshots(t *testing.T) {
	t.Run("nil Monitor returns empty set with non-nil slices", func(t *testing.T) {
		var m *Monitor
		result := m.GetDiagnosticSnapshots()

		if result.Nodes == nil {
			t.Error("Nodes should not be nil")
		}
		if result.Guests == nil {
			t.Error("Guests should not be nil")
		}
		if len(result.Nodes) != 0 {
			t.Errorf("Nodes length = %d, want 0", len(result.Nodes))
		}
		if len(result.Guests) != 0 {
			t.Errorf("Guests length = %d, want 0", len(result.Guests))
		}
	})

	t.Run("guest notes normalize to empty array", func(t *testing.T) {
		m := &Monitor{
			nodeSnapshots:  make(map[string]NodeMemorySnapshot),
			guestSnapshots: make(map[string]GuestMemorySnapshot),
		}

		m.guestSnapshots[makeGuestSnapshotKey("pve1", "qemu", "node1", 100)] = GuestMemorySnapshot{
			Instance:  "pve1",
			Node:      "node1",
			GuestType: "qemu",
			VMID:      100,
		}

		result := m.GetDiagnosticSnapshots()
		if len(result.Guests) != 1 {
			t.Fatalf("Guests length = %d, want 1", len(result.Guests))
		}
		if result.Guests[0].Notes == nil {
			t.Fatal("guest notes should normalize to an empty slice")
		}

		payload, err := json.Marshal(result.Guests[0])
		if err != nil {
			t.Fatalf("marshal guest snapshot: %v", err)
		}
		if !strings.Contains(string(payload), `"notes":[]`) {
			t.Fatalf("expected guest snapshot to retain notes array, got %s", payload)
		}
	})

	t.Run("guest carry-forward notes remain visible", func(t *testing.T) {
		m := &Monitor{
			nodeSnapshots:  make(map[string]NodeMemorySnapshot),
			guestSnapshots: make(map[string]GuestMemorySnapshot),
		}

		m.recordGuestSnapshot("pve1", "qemu", "node1", 100, GuestMemorySnapshot{
			Instance:     "pve1",
			Node:         "node1",
			GuestType:    "qemu",
			VMID:         100,
			MemorySource: "previous-snapshot",
			Notes:        []string{"preserved-previous-memory-for-healthy-guest-low-trust-full-usage"},
		})

		result := m.GetDiagnosticSnapshots()
		if len(result.Guests) != 1 {
			t.Fatalf("Guests length = %d, want 1", len(result.Guests))
		}
		if result.Guests[0].MemorySource != "previous-snapshot" {
			t.Fatalf("MemorySource = %q, want previous-snapshot", result.Guests[0].MemorySource)
		}
		if result.Guests[0].FallbackReason != "preserved-previous-snapshot" {
			t.Fatalf("FallbackReason = %q, want preserved-previous-snapshot", result.Guests[0].FallbackReason)
		}
		if len(result.Guests[0].Notes) != 1 || result.Guests[0].Notes[0] != "preserved-previous-memory-for-healthy-guest-low-trust-full-usage" {
			t.Fatalf("Notes = %#v, want carry-forward note", result.Guests[0].Notes)
		}
	})

	t.Run("empty Monitor returns empty set", func(t *testing.T) {
		m := &Monitor{
			nodeSnapshots:  make(map[string]NodeMemorySnapshot),
			guestSnapshots: make(map[string]GuestMemorySnapshot),
		}

		result := m.GetDiagnosticSnapshots()

		if len(result.Nodes) != 0 {
			t.Errorf("Nodes length = %d, want 0", len(result.Nodes))
		}
		if len(result.Guests) != 0 {
			t.Errorf("Guests length = %d, want 0", len(result.Guests))
		}
	})

	t.Run("single node snapshot is returned", func(t *testing.T) {
		m := &Monitor{
			nodeSnapshots:  make(map[string]NodeMemorySnapshot),
			guestSnapshots: make(map[string]GuestMemorySnapshot),
		}

		m.nodeSnapshots[makeNodeSnapshotKey("pve1", "node1")] = NodeMemorySnapshot{
			Instance:     "pve1",
			Node:         "node1",
			MemorySource: "rrd-available",
		}

		result := m.GetDiagnosticSnapshots()

		if len(result.Nodes) != 1 {
			t.Fatalf("Nodes length = %d, want 1", len(result.Nodes))
		}

		if result.Nodes[0].Instance != "pve1" {
			t.Errorf("Instance = %q, want %q", result.Nodes[0].Instance, "pve1")
		}
		if result.Nodes[0].Node != "node1" {
			t.Errorf("Node = %q, want %q", result.Nodes[0].Node, "node1")
		}
		if result.Nodes[0].MemorySource != "rrd-memavailable" {
			t.Errorf("MemorySource = %q, want %q", result.Nodes[0].MemorySource, "rrd-memavailable")
		}
		if result.Nodes[0].FallbackReason != "rrd-memavailable" {
			t.Errorf("FallbackReason = %q, want %q", result.Nodes[0].FallbackReason, "rrd-memavailable")
		}
	})

	t.Run("single guest snapshot is returned", func(t *testing.T) {
		m := &Monitor{
			nodeSnapshots:  make(map[string]NodeMemorySnapshot),
			guestSnapshots: make(map[string]GuestMemorySnapshot),
		}

		m.guestSnapshots[makeGuestSnapshotKey("pve1", "qemu", "node1", 100)] = GuestMemorySnapshot{
			Instance:     "pve1",
			GuestType:    "qemu",
			Node:         "node1",
			VMID:         100,
			Name:         "testvm",
			MemorySource: "listing",
		}

		result := m.GetDiagnosticSnapshots()

		if len(result.Guests) != 1 {
			t.Fatalf("Guests length = %d, want 1", len(result.Guests))
		}

		if result.Guests[0].Instance != "pve1" {
			t.Errorf("Instance = %q, want %q", result.Guests[0].Instance, "pve1")
		}
		if result.Guests[0].GuestType != "qemu" {
			t.Errorf("GuestType = %q, want %q", result.Guests[0].GuestType, "qemu")
		}
		if result.Guests[0].Node != "node1" {
			t.Errorf("Node = %q, want %q", result.Guests[0].Node, "node1")
		}
		if result.Guests[0].VMID != 100 {
			t.Errorf("VMID = %d, want %d", result.Guests[0].VMID, 100)
		}
		if result.Guests[0].Name != "testvm" {
			t.Errorf("Name = %q, want %q", result.Guests[0].Name, "testvm")
		}
		if result.Guests[0].MemorySource != "cluster-resources" {
			t.Errorf("MemorySource = %q, want %q", result.Guests[0].MemorySource, "cluster-resources")
		}
		if result.Guests[0].FallbackReason != "cluster-resources" {
			t.Errorf("FallbackReason = %q, want %q", result.Guests[0].FallbackReason, "cluster-resources")
		}
	})

	t.Run("multiple nodes are sorted by instance then node", func(t *testing.T) {
		m := &Monitor{
			nodeSnapshots:  make(map[string]NodeMemorySnapshot),
			guestSnapshots: make(map[string]GuestMemorySnapshot),
		}

		// Add in unsorted order
		m.nodeSnapshots[makeNodeSnapshotKey("pve2", "node2")] = NodeMemorySnapshot{Instance: "pve2", Node: "node2"}
		m.nodeSnapshots[makeNodeSnapshotKey("pve1", "node2")] = NodeMemorySnapshot{Instance: "pve1", Node: "node2"}
		m.nodeSnapshots[makeNodeSnapshotKey("pve2", "node1")] = NodeMemorySnapshot{Instance: "pve2", Node: "node1"}
		m.nodeSnapshots[makeNodeSnapshotKey("pve1", "node1")] = NodeMemorySnapshot{Instance: "pve1", Node: "node1"}

		result := m.GetDiagnosticSnapshots()

		if len(result.Nodes) != 4 {
			t.Fatalf("Nodes length = %d, want 4", len(result.Nodes))
		}

		// Expected order: pve1|node1, pve1|node2, pve2|node1, pve2|node2
		expectedOrder := []struct {
			instance string
			node     string
		}{
			{"pve1", "node1"},
			{"pve1", "node2"},
			{"pve2", "node1"},
			{"pve2", "node2"},
		}

		for i, expected := range expectedOrder {
			if result.Nodes[i].Instance != expected.instance || result.Nodes[i].Node != expected.node {
				t.Errorf("Nodes[%d] = {%q, %q}, want {%q, %q}",
					i, result.Nodes[i].Instance, result.Nodes[i].Node, expected.instance, expected.node)
			}
		}
	})

	t.Run("multiple guests are sorted by instance then node then guestType then VMID", func(t *testing.T) {
		m := &Monitor{
			nodeSnapshots:  make(map[string]NodeMemorySnapshot),
			guestSnapshots: make(map[string]GuestMemorySnapshot),
		}

		// Add in unsorted order
		m.guestSnapshots[makeGuestSnapshotKey("pve2", "qemu", "node1", 100)] = GuestMemorySnapshot{Instance: "pve2", Node: "node1", GuestType: "qemu", VMID: 100}
		m.guestSnapshots[makeGuestSnapshotKey("pve1", "qemu", "node1", 101)] = GuestMemorySnapshot{Instance: "pve1", Node: "node1", GuestType: "qemu", VMID: 101}
		m.guestSnapshots[makeGuestSnapshotKey("pve1", "lxc", "node1", 200)] = GuestMemorySnapshot{Instance: "pve1", Node: "node1", GuestType: "lxc", VMID: 200}
		m.guestSnapshots[makeGuestSnapshotKey("pve1", "qemu", "node2", 100)] = GuestMemorySnapshot{Instance: "pve1", Node: "node2", GuestType: "qemu", VMID: 100}
		m.guestSnapshots[makeGuestSnapshotKey("pve1", "qemu", "node1", 100)] = GuestMemorySnapshot{Instance: "pve1", Node: "node1", GuestType: "qemu", VMID: 100}

		result := m.GetDiagnosticSnapshots()

		if len(result.Guests) != 5 {
			t.Fatalf("Guests length = %d, want 5", len(result.Guests))
		}

		// Expected order:
		// pve1|node1|lxc|200 (lxc < qemu)
		// pve1|node1|qemu|100
		// pve1|node1|qemu|101
		// pve1|node2|qemu|100
		// pve2|node1|qemu|100
		expectedOrder := []struct {
			instance  string
			node      string
			guestType string
			vmid      int
		}{
			{"pve1", "node1", "lxc", 200},
			{"pve1", "node1", "qemu", 100},
			{"pve1", "node1", "qemu", 101},
			{"pve1", "node2", "qemu", 100},
			{"pve2", "node1", "qemu", 100},
		}

		for i, expected := range expectedOrder {
			got := result.Guests[i]
			if got.Instance != expected.instance || got.Node != expected.node ||
				got.GuestType != expected.guestType || got.VMID != expected.vmid {
				t.Errorf("Guests[%d] = {%q, %q, %q, %d}, want {%q, %q, %q, %d}",
					i, got.Instance, got.Node, got.GuestType, got.VMID,
					expected.instance, expected.node, expected.guestType, expected.vmid)
			}
		}
	})

	t.Run("mix of nodes and guests", func(t *testing.T) {
		m := &Monitor{
			nodeSnapshots:  make(map[string]NodeMemorySnapshot),
			guestSnapshots: make(map[string]GuestMemorySnapshot),
		}

		// Add nodes
		m.nodeSnapshots[makeNodeSnapshotKey("pve1", "node1")] = NodeMemorySnapshot{Instance: "pve1", Node: "node1"}
		m.nodeSnapshots[makeNodeSnapshotKey("pve1", "node2")] = NodeMemorySnapshot{Instance: "pve1", Node: "node2"}

		// Add guests
		m.guestSnapshots[makeGuestSnapshotKey("pve1", "qemu", "node1", 100)] = GuestMemorySnapshot{Instance: "pve1", Node: "node1", GuestType: "qemu", VMID: 100}
		m.guestSnapshots[makeGuestSnapshotKey("pve1", "lxc", "node2", 200)] = GuestMemorySnapshot{Instance: "pve1", Node: "node2", GuestType: "lxc", VMID: 200}

		result := m.GetDiagnosticSnapshots()

		if len(result.Nodes) != 2 {
			t.Errorf("Nodes length = %d, want 2", len(result.Nodes))
		}
		if len(result.Guests) != 2 {
			t.Errorf("Guests length = %d, want 2", len(result.Guests))
		}

		// Verify nodes are sorted
		if result.Nodes[0].Node != "node1" || result.Nodes[1].Node != "node2" {
			t.Errorf("Nodes not in expected order: got [%s, %s], want [node1, node2]",
				result.Nodes[0].Node, result.Nodes[1].Node)
		}

		// Verify guests are sorted
		if result.Guests[0].Node != "node1" || result.Guests[1].Node != "node2" {
			t.Errorf("Guests not in expected order: got [%s, %s], want [node1, node2]",
				result.Guests[0].Node, result.Guests[1].Node)
		}
	})
}

func TestGuestMemorySampleCapacityContract(t *testing.T) {
	t.Run("selection", testGuestMemorySampleCapacitySelection)
	t.Run("cache-binding", testGuestMemorySampleCapacityCacheBinding)
}

func TestGuestPollOwnershipStagesAllReadingEffects(t *testing.T) {
	c := &guestPollOwnershipClient{cpu: .95}
	m := guestPollOwnershipMonitor(t, c)
	ctx, owner := m.beginPVEGuestPoll(context.Background(), "site-a", c)
	if owner == nil {
		t.Fatal("registered cycle not admitted")
	}
	defer m.finishPVEGuestPoll("site-a", owner)
	resources := c.resources()
	vm, ok := m.handleClusterVMResource(ctx, "site-a", resources[0], "site-a:node:105", c, nil, nil)
	if !ok {
		t.Fatal("VM builder lost inventory")
	}
	ct, ok := m.handleClusterContainerResource(ctx, "site-a", resources[1], "site-a:node:106", c, nil, nil)
	if !ok {
		t.Fatal("container builder lost inventory")
	}
	stagePVEGuestEffect(ctx, func() { m.recordGuestMetrics([]models.VM{vm}, []models.Container{ct}, time.Time{}) })
	if len(m.GetDiagnosticSnapshots().Guests) != 0 || len(m.metricsHistory.GetGuestMetrics(vm.ID, "cpu", time.Hour)) != 0 || len(m.state.GetSnapshot().VMs) != 0 {
		t.Fatal("a partial builder published before the complete guest generation")
	}
	m.publishPVEGuests(ctx, "site-a", []models.VM{vm}, []models.Container{ct})
	if len(m.GetDiagnosticSnapshots().Guests) != 2 || len(m.state.GetSnapshot().VMs) != 1 || len(m.metricsHistory.GetGuestMetrics(vm.ID, "cpu", time.Hour)) != 1 || len(guestHistoryStoredPoints(t, m, "vm", vm.ID, "cpu")) != 1 {
		t.Fatal("accepted generation lost diagnostics or in-memory/persistent History")
	}
}

// Walk the actual efficient enumeration -> fallback -> accepted publication ->
// canonical projection. Malformed successes cannot remove guests, refresh their
// last observation, clear a breach or invalidate template backup eligibility.
// All endpoints are synthetic; no backup, guest command or live send occurs.
func TestGuestInventoryFailurePreservesOwnedReadingsAndRecovers(t *testing.T) {
	for _, failure := range []string{"missing-data", "trailing-value", "truncated-prefix", "duplicate-id"} {
		t.Run(failure, func(t *testing.T) {
			t.Setenv("PULSE_DATA_DIR", t.TempDir())
			var phase, commands, clusterReads, listReads atomic.Int32
			corrupt := func(w http.ResponseWriter, prefix string) {
				switch failure {
				case "missing-data":
					fmt.Fprint(w, `{}`)
				case "trailing-value":
					fmt.Fprint(w, prefix+` {"errors":{"message":"fixture-private-detail"}}`)
				case "truncated-prefix":
					w.Header().Set("Content-Length", fmt.Sprint(len(prefix)+100))
					fmt.Fprint(w, prefix)
				case "duplicate-id":
					var payload struct {
						Data []json.RawMessage `json:"data"`
					}
					if err := json.Unmarshal([]byte(prefix), &payload); err != nil {
						t.Error(err)
						return
					}
					payload.Data = append(payload.Data, payload.Data[0])
					_ = json.NewEncoder(w).Encode(payload)
				}
			}
			const vm = `{"vmid":105,"type":"qemu","node":"member","name":"kept-vm","status":"running","lock":"backup","cpu":0.95,"maxmem":1073741824,"maxdisk":2147483648}`
			const extra = `{"vmid":106,"type":"qemu","node":"member","name":"later-deleted-vm","status":"running","lock":"backup","cpu":0.1,"maxmem":1073741824}`
			const ct = `{"vmid":205,"type":"lxc","node":"member","name":"kept-container","status":"running","lock":"backup","cpu":0.15,"mem":104857600,"maxmem":1073741824,"disk":268435456,"maxdisk":2147483648}`
			const template = `{"vmid":999,"type":"qemu","node":"member","template":1}`
			server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
				w.Header().Set("Content-Type", "application/json")
				p := phase.Load()
				switch {
				case strings.HasSuffix(r.URL.Path, "/cluster/resources"):
					clusterReads.Add(1)
					if p == 1 || p == 2 {
						corrupt(w, `{"data":[`+vm+`]}`)
						return
					}
					if p == 4 {
						http.NotFound(w, r)
						return
					}
					rows := vm + `,` + ct
					if p == 0 {
						rows += `,` + extra + `,` + template
					}
					if p == 3 {
						rows += `,{"vmid":108,"type":"qemu","node":"member","name":"new-vm","status":"running","lock":"backup","cpu":0,"maxmem":1073741824}`
					}
					fmt.Fprint(w, `{"data":[`+rows+`]}`)
				case strings.HasSuffix(r.URL.Path, "/member/qemu"):
					listReads.Add(1)
					if p == 1 {
						corrupt(w, `{"data":[`+vm+`]}`)
						return
					}
					if p == 4 {
						fmt.Fprint(w, `{"data":[]}`)
						return
					}
					fmt.Fprint(w, `{"data":[`+vm+`]}`)
				case strings.HasSuffix(r.URL.Path, "/member/lxc"):
					listReads.Add(1)
					if p == 1 {
						corrupt(w, `{"data":[`+ct+`]}`)
						return
					}
					if p == 4 {
						fmt.Fprint(w, `{"data":[]}`)
						return
					}
					// Keep the existing older-PVE string identifier compatibility on fallback.
					fmt.Fprint(w, `{"data":[`+strings.Replace(ct, `"vmid":205`, `"vmid":"205"`, 1)+`]}`)
				case strings.Contains(r.URL.Path, "/agent/"):
					commands.Add(1)
					t.Error("inventory proof dispatched an in-guest command")
					http.Error(w, "unexpected command", 500)
				case strings.HasSuffix(r.URL.Path, "/qemu/105/status/current"), strings.HasSuffix(r.URL.Path, "/qemu/106/status/current"), strings.HasSuffix(r.URL.Path, "/qemu/108/status/current"):
					fmt.Fprint(w, `{"data":{"status":"running","lock":"backup","maxmem":1073741824,"mem":104857600}}`)
				case strings.HasSuffix(r.URL.Path, "/lxc/205/status/current"):
					fmt.Fprint(w, `{"data":{"vmid":205,"status":"running","lock":"backup","maxmem":1073741824,"mem":104857600,"maxdisk":2147483648,"disk":268435456}}`)
				case strings.HasSuffix(r.URL.Path, "/config"):
					fmt.Fprint(w, `{"data":{"lock":"backup"}}`)
				default:
					http.NotFound(w, r)
				}
			}))
			defer server.Close()
			client, err := proxmox.NewClient(proxmox.ClientConfig{Host: server.URL, TokenName: "fixture@pve!pulse", TokenValue: "fixture", Timeout: 10 * time.Second})
			if err != nil {
				t.Fatal(err)
			}
			m := newTestPVEMonitor("inventory")
			m.resourceStore = unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(nil))
			t.Cleanup(m.alertManager.Stop)
			t.Cleanup(m.notificationMgr.Stop)
			m.replacePVEClient("inventory", client)
			alertCfg := m.alertManager.GetConfig()
			alertCfg.Enabled = true
			alertCfg.GuestDefaults.CPU = &alerts.HysteresisThreshold{Trigger: 80, Clear: 75}
			alertCfg.MetricTimeThresholds = map[string]map[string]int{"guest": {"cpu": 0}}
			m.alertManager.UpdateConfig(alertCfg)
			run := func() models.StateSnapshot {
				t.Helper()
				if err := m.pollGuestsWithFallback(context.Background(), "inventory", &config.PVEInstance{MonitorVMs: true, MonitorContainers: true}, client, []proxmox.Node{{Node: "member", Status: "online"}}, map[string]string{"member": "online"}); err != nil {
					t.Fatal(err)
				}
				snapshot := m.state.GetSnapshot()
				m.updateResourceStore(m.currentStateWithScope())
				return snapshot
			}
			// A second registered connection has its own observation and must survive
			// both failed enumerations and actual removals on this one.
			peer := models.VM{ID: "peer:other:305", Instance: "peer", Node: "other", VMID: 305, Name: "independent-peer", Type: "qemu", Status: "running", LastSeen: time.Now()}
			m.state.UpdateVMsForInstance("peer", []models.VM{peer})
			original := run()
			if len(original.VMs) != 3 || len(original.Containers) != 1 {
				t.Fatalf("initial complete inventory not established: %+v", original)
			}
			templateKey := pveBackupTemplateSubjectKey("inventory", "qemu", "member", 999)
			if _, ok := m.pveBackupTemplateSubjects["inventory"][templateKey]; !ok {
				t.Fatal("initial template not established")
			}
			originalDiagnostics := m.GetDiagnosticSnapshots().Guests
			originalHistory := m.metricsHistory.GetGuestMetrics("inventory:member:105", "cpu", time.Hour)
			originalAlerts := m.alertManager.GetActiveAlerts()
			if len(originalHistory) != 1 || len(originalAlerts) != 1 {
				t.Fatalf("initial history/breach not established: %d/%d", len(originalHistory), len(originalAlerts))
			}
			emit := func(label string, snapshot models.StateSnapshot) {
				registry := unifiedresources.NewRegistry(nil)
				registry.IngestSnapshot(snapshot)
				projection, err := json.Marshal(map[string]any{"case": failure, "phase": label, "snapshot": snapshot, "resources": registry.List()})
				if err != nil {
					t.Fatal(err)
				}
				t.Logf("GUEST_INVENTORY_PROJECTION %s", projection)
			}
			emit("initial", original)
			phase.Store(1)
			failed := run()
			if !reflect.DeepEqual(failed.VMs, original.VMs) || !reflect.DeepEqual(failed.Containers, original.Containers) {
				t.Error("incomplete inventory changed retained guests or their observation time")
			}
			if !reflect.DeepEqual(m.metricsHistory.GetGuestMetrics("inventory:member:105", "cpu", time.Hour), originalHistory) {
				t.Error("failed inventory fabricated a new History sample")
			}
			if !reflect.DeepEqual(m.alertManager.GetActiveAlerts(), originalAlerts) {
				t.Error("failed inventory changed an existing breach")
			}
			if _, ok := m.pveBackupTemplateSubjects["inventory"][templateKey]; !ok {
				t.Error("failed inventory erased backup-template eligibility")
			}
			if !reflect.DeepEqual(m.GetDiagnosticSnapshots().Guests, originalDiagnostics) {
				t.Error("failed inventory changed original diagnostic memory/source/time")
			}
			emit("both-inventories-refused", failed)
			phase.Store(2)
			fallback := run()
			if len(fallback.VMs) != 2 || len(fallback.Containers) != 1 || fallback.VMs[0].VMID == 106 {
				t.Error("complete fallback failed to publish a real deletion")
			}
			if _, ok := m.pveBackupTemplateSubjects["inventory"][templateKey]; ok {
				t.Error("complete fallback retained a deleted template")
			}
			if len(m.metricsHistory.GetGuestMetrics("inventory:member:105", "cpu", time.Hour)) != 2 {
				t.Error("complete fallback failed to resume History")
			}
			emit("complete-node-fallback", fallback)
			before := listReads.Load()
			phase.Store(3)
			recovered := run()
			if len(recovered.VMs) != 3 || len(recovered.Containers) != 1 || listReads.Load() != before {
				t.Error("complete cluster recovery did not return to the efficient route")
			}
			emit("complete-cluster-recovery", recovered)
			phase.Store(4)
			empty := run()
			if len(empty.VMs) != 1 || empty.VMs[0].ID != peer.ID || len(empty.Containers) != 0 {
				t.Error("explicit empty node inventories failed to remove guests within their scope")
			}
			if commands.Load() != 0 || clusterReads.Load() != 5 || listReads.Load() != 6 || len(m.pveGuestPolls) != 0 {
				t.Errorf("unexpected command/retry/owner leak: %d/%d/%d/%d", commands.Load(), clusterReads.Load(), listReads.Load(), len(m.pveGuestPolls))
			}
			emit("complete-scoped-empty", empty)
		})
	}
}