Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
package unifiedresources
import (
"encoding/json"
"fmt"
"reflect"
"slices"
"strings"
"testing"
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/models"
"github.com/rcourtman/pulse-go-rewrite/internal/storagehealth"
"github.com/rcourtman/pulse-go-rewrite/pkg/diskinventory"
)
func TestPendingHostContinuityPreservesUnknownDeliveryAcrossSerializedReads(t *testing.T) {
seen := time.Now().Add(-time.Minute).UTC()
record := HostIngestRecord(models.Host{ID: "waiting-agent", Hostname: "waiting-host", Status: "pending", LastSeen: seen})
record.Resource.SourceStatus = map[DataSource]SourceStatus{SourceAgent: {Status: "pending", LastSeen: seen}}
rr := NewRegistry(nil)
id := rr.ingestRecord(SourceAgent, record.SourceID, record.Resource, record.Identity, true)
rr.MarkStale(time.Now(), map[DataSource]time.Duration{SourceAgent: time.Second})
encoded, err := json.Marshal(rr.List())
if err != nil {
t.Fatal(err)
}
var resources []Resource
if err := json.Unmarshal(encoded, &resources); err != nil {
t.Fatal(err)
}
copy := NewRegistry(nil)
copy.IngestResourcesWithStaleThresholds(resources, map[DataSource]time.Duration{SourceAgent: time.Second})
got, ok := copy.Get(id)
if !ok || got.Status != StatusUnknown || got.SourceStatus[SourceAgent].Status != "pending" || !got.LastSeen.Equal(seen) || got.Agent.Stale {
t.Fatalf("waiting enrollment acquired a fresh or offline verdict: %+v", got)
}
// Normal ingest must replace the continuity marker with a real delivery.
rr.IngestSnapshot(models.StateSnapshot{Hosts: []models.Host{{ID: "waiting-agent", Hostname: "waiting-host", Status: "online", LastSeen: time.Now().UTC()}}})
got, ok = rr.Get(id)
if !ok || got.Status != StatusOnline || got.SourceStatus[SourceAgent].Status != "online" {
t.Fatalf("accepted live sighting did not replace pending continuity: %+v", got)
}
if replaced := rr.ingestRecord(SourceAgent, record.SourceID, record.Resource, record.Identity, true); replaced != "" {
t.Fatal("pending continuity replaced an existing live observation")
}
}
// The snapshot entry point shares the registry's journal ownership, without
// mutating either the snapshot or the alert's recorded metadata.
func TestProxmoxDiskAlertSnapshotOwnershipMatchesRegistry(t *testing.T) {
rr := NewRegistry(nil)
rr.resources = map[string]*Resource{
"disk-a": {ID: "disk-a", Type: ResourceTypePhysicalDisk,
PhysicalDisk: &PhysicalDiskMeta{Serial: "unknown", WWN: "naa.5000c500a1b2c3d4", DevPath: "/dev/sdz"}},
"disk-b": {ID: "disk-b", Type: ResourceTypePhysicalDisk,
PhysicalDisk: &PhysicalDiskMeta{Serial: "unknown", WWN: "0x5000c500000000b2", DevPath: "/dev/sda"}},
}
var snapshot []Resource
for _, resource := range rr.resources {
snapshot = append(snapshot, *resource)
}
ref := ProxmoxPhysicalDiskAlertResourceID("lab", "pve1", "/dev/sda")
for _, tc := range []struct{ name, serial, wwn, want string }{
{"moved hardware without PVE facet", "unknown", "wwn-0x5000c500a1b2c3d4", "disk-a"},
{"placeholder is not hardware", "unknown", "", ""},
{"second hardware", "unknown", "5000c500000000b2", "disk-b"},
{"derived absent hardware", "NEW00001", "", MachineIdentityCanonicalID(ResourceTypePhysicalDisk, "NEW00001")},
} {
t.Run(tc.name, func(t *testing.T) {
metadata := map[string]any{MetadataDiskSerial: tc.serial, MetadataDiskWWN: tc.wwn}
before, _ := json.Marshal(snapshot)
if got := ProxmoxPhysicalDiskAlertOwner(ref, metadata, snapshot); got != tc.want || got != rr.proxmoxDiskAlertOwner(ref, metadata) {
t.Fatalf("snapshot owner = %q, registry = %q, want %q", got, rr.proxmoxDiskAlertOwner(ref, metadata), tc.want)
}
after, _ := json.Marshal(snapshot)
if string(before) != string(after) || metadata[MetadataDiskSerial] != tc.serial || metadata[MetadataDiskWWN] != tc.wwn {
t.Fatal("ownership read mutated snapshot or recorded identity")
}
})
}
}
func TestProxmoxGuestReadAdmissionReplacesPreviousFacet(t *testing.T) {
rr := NewRegistry(nil)
vm := models.VM{ID: "pve:node:105", Instance: "pve", Node: "node", VMID: 105, Type: "qemu", Status: "running", Name: "guest", LastSeen: time.Now(), Disk: models.Disk{Total: 100, Used: 30, Usage: 30}}
for _, tc := range []struct {
status, reason string
expected bool
}{
{"available", "", true},
{"deferred", "prev-agent-cooldown", true},
{"available", "", true},
{"not-running", "agent-not-running", false},
{"disabled", "agent-disabled", false},
} {
vm.GuestAgentStatus, vm.DiskStatusReason, vm.GuestAgentExpected = tc.status, tc.reason, tc.expected
rr.IngestSnapshot(models.StateSnapshot{VMs: []models.VM{vm}})
views := rr.VMs()
if len(views) != 1 || views[0].DiskStatusReason() != tc.reason {
t.Fatalf("current disk admission missing from typed view: %q", tc.reason)
}
resource, ok := rr.Get(views[0].ID())
if !ok || resource.Proxmox == nil || resource.Proxmox.GuestAgentStatus != tc.status || resource.Proxmox.GuestAgentExpected != tc.expected {
t.Fatalf("current guest admission missing from resource: %+v", resource)
}
// A detached read and JSON retain the current state, including a cleared
// reason/false expectation. Non-guest partial facets cannot erase it.
encoded, err := json.Marshal(resource)
var served Resource
if err != nil || json.Unmarshal(encoded, &served) != nil || served.Proxmox.DiskStatusReason != tc.reason || served.Proxmox.GuestAgentStatus != tc.status || served.Proxmox.GuestAgentExpected != tc.expected {
t.Fatal("read JSON changed guest admission")
}
partial := mergeProxmoxData(resource.Proxmox, &ProxmoxData{NodeName: "node"})
if partial.DiskStatusReason != tc.reason || partial.GuestAgentStatus != tc.status || partial.GuestAgentExpected != tc.expected {
t.Fatal("partial non-guest facet erased guest-owned admission")
}
}
}
func TestRegistry_CachedReadsUseSharedLock(t *testing.T) {
rr := NewRegistry(nil)
// A clean, empty registry already has a valid empty cache. Holding another
// read lock must not prevent a cached accessor from completing.
rr.mu.RLock()
done := make(chan struct{})
go func() {
_ = rr.VMs()
close(done)
}()
select {
case <-done:
rr.mu.RUnlock()
case <-time.After(5 * time.Second):
rr.mu.RUnlock()
<-done
t.Fatal("cached view read waited for an exclusive registry lock")
}
}
func TestRegistryGenerationComparisonIgnoresUnchangedTelemetry(t *testing.T) {
store := NewMemoryStore()
before := NewRegistry(store)
after := NewRegistry(store)
observedAt := time.Date(2026, 9, 24, 6, 0, 0, 0, time.UTC)
first := IngestRecord{
SourceID: "vm-101",
Resource: Resource{
Type: ResourceTypeVM,
Name: "vm-101",
Status: StatusOnline,
LastSeen: observedAt,
},
}
before.IngestRecords(SourceProxmox, []IngestRecord{first})
updated := first
updated.Resource.LastSeen = observedAt.Add(time.Minute)
after.IngestRecords(SourceProxmox, []IngestRecord{updated})
recordRegistryChangesBetweenGenerations(before, after, observedAt.Add(time.Minute), nil, SourcePulseDiff, "")
if got := len(store.changes); got != 0 {
t.Fatalf("telemetry-only generation emitted %d change records, want none", got)
}
}
func TestRegistryGenerationComparisonTreatsIdentityListsAsSets(t *testing.T) {
store := NewMemoryStore()
now := time.Date(2026, 9, 29, 17, 0, 0, 0, time.UTC)
entry := IngestRecord{
SourceID: "disk-1",
Resource: Resource{Type: ResourceTypePhysicalDisk, Name: "disk", Status: StatusOnline, LastSeen: now},
Identity: ResourceIdentity{MachineID: "serial-1", Hostnames: []string{"node", "node.example"}},
}
before := NewRegistry(store)
before.IngestRecords(SourceAgent, []IngestRecord{entry})
reordered := entry
reordered.Identity = ResourceIdentity{
MachineID: "serial-1", Hostnames: []string{"node.example", "node"},
IPAddresses: []string{}, MACAddresses: []string{},
}
after := NewRegistry(store)
after.IngestRecords(SourceAgent, []IngestRecord{reordered})
recordRegistryChangesBetweenGenerations(before, after, now.Add(time.Minute), nil, SourcePulseDiff, "")
if len(store.changes) != 0 {
t.Fatalf("equivalent identity lists emitted %d history rows: %+v", len(store.changes), store.changes)
}
changed := reordered
changed.Identity.IPAddresses = []string{"192.0.2.10"}
later := NewRegistry(store)
later.IngestRecords(SourceAgent, []IngestRecord{changed})
recordRegistryChangesBetweenGenerations(after, later, now.Add(2*time.Minute), nil, SourcePulseDiff, "")
if len(store.changes) != 1 || !sameStringSet(mustChangedFields(t, &store.changes[0]), []string{"identity"}) {
t.Fatalf("real address change not recorded once: %+v", store.changes)
}
}
// A guest that rebooted between two polls stays "running", so its Proxmox
// uptime reset is the only trace. Ingest stamps the poll time per source, and
// an older poll published late must not read as a reboot.
func TestRegistryGenerationComparisonRecordsProxmoxUptimeResetOnce(t *testing.T) {
store := NewMemoryStore()
polled := time.Date(2026, 10, 29, 3, 0, 0, 0, time.UTC)
vm := models.VM{ID: "pve:node:105", Instance: "pve", Node: "node", VMID: 105, Type: "qemu", Status: "running", Name: "guest", Uptime: 86400, LastSeen: polled}
generation := func(uptime int64, seen time.Time) *ResourceRegistry {
rr := NewRegistry(store)
vm.Uptime, vm.LastSeen = uptime, seen
rr.IngestSnapshot(models.StateSnapshot{VMs: []models.VM{vm}})
return rr
}
before := generation(86400, polled)
rebooted := generation(40, polled.Add(time.Minute))
recordRegistryChangesBetweenGenerations(before, rebooted, polled.Add(time.Minute), nil, SourcePulseDiff, "")
if len(store.changes) != 1 || store.changes[0].Kind != ChangeRestart {
t.Fatalf("reboot between polls: %+v", store.changes)
}
newer := generation(100, polled.Add(2*time.Minute))
stale := generation(70, polled.Add(90*time.Second))
recordRegistryChangesBetweenGenerations(newer, stale, polled.Add(2*time.Minute), nil, SourcePulseDiff, "")
for _, change := range store.changes[1:] {
if change.Kind == ChangeRestart {
t.Fatalf("older poll published late recorded a restart: %+v", store.changes)
}
}
}
// TestMemoryStore_RecordActionAuditAppliesRedaction is an integration check
// at the registry-store boundary. The MemoryStore is the backing store the
// registry uses in tests and contract examples, and operator-authored audit
// fields must be scrubbed of secret shapes uniformly across both SQLite and
// in-memory persistence so test fixtures cannot accidentally exercise an
// unredacted path that production never sees.
func TestMemoryStore_RecordActionAuditAppliesRedaction(t *testing.T) {
store := NewMemoryStore()
now := time.Now().UTC()
record := ActionAuditRecord{
ID: "act-redact",
CreatedAt: now,
UpdatedAt: now,
State: ActionStatePlanned,
Request: ActionRequest{
RequestID: "req-redact",
ResourceID: "vm-100",
CapabilityName: "pulse_control",
Reason: "rotate sk-leakedkey1234567 because it appeared in logs",
RequestedBy: "[email protected]",
Params: map[string]any{
"command": "curl -H 'Authorization: Bearer eyJleak' https://api.example.com",
"targetType": "agent",
},
},
Plan: ActionPlan{
ActionID: "act-redact",
RequestID: "req-redact",
Allowed: true,
PlanHash: "deadbeef",
},
}
if err := store.RecordActionAudit(record); err != nil {
t.Fatalf("RecordActionAudit: %v", err)
}
stored, ok, err := store.GetActionAudit("act-redact")
if err != nil {
t.Fatalf("GetActionAudit: %v", err)
}
if !ok {
t.Fatalf("expected stored audit, got none")
}
if strings.Contains(stored.Request.Reason, "sk-leakedkey1234567") {
t.Fatalf("Reason still contains secret after persistence: %s", stored.Request.Reason)
}
if cmd, _ := stored.Request.Params["command"].(string); strings.Contains(cmd, "eyJleak") {
t.Fatalf("Params[command] still contains Bearer token: %s", cmd)
}
// Plan fields must be untouched so PlanHash-based drift detection
// continues to work.
if stored.Plan.PlanHash != "deadbeef" {
t.Fatalf("Plan.PlanHash mutated by redaction: %s", stored.Plan.PlanHash)
}
}
func TestRegistryProjectsAndDeduplicatesXCPNGPoolVMs(t *testing.T) {
now := time.Now().UTC()
poolUUID := "11111111-1111-1111-1111-111111111111"
hostOneUUID := "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"
hostTwoUUID := "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb"
vm := models.HostXCPNGVM{
UUID: "cccccccc-cccc-cccc-cccc-cccccccccccc",
Name: "database",
PowerState: "running",
VCPUs: 4,
MemoryActual: 4 << 30,
MemoryStaticMax: 8 << 30,
ResidentHostUUID: hostTwoUUID,
}
hosts := []models.Host{
{
ID: "agent-one",
Hostname: "xcp-one",
Platform: "linux",
Status: "online",
LastSeen: now,
XCPNG: &models.HostXCPNGInventory{
PoolUUID: poolUUID,
PoolName: "Lab Pool",
LocalHostUUID: hostOneUUID,
VMs: []models.HostXCPNGVM{vm},
CollectedAt: now,
},
},
{
ID: "agent-two",
Hostname: "xcp-two",
Platform: "linux",
Status: "online",
LastSeen: now.Add(time.Second),
XCPNG: &models.HostXCPNGInventory{
PoolUUID: poolUUID,
PoolName: "Lab Pool",
LocalHostUUID: hostTwoUUID,
VMs: []models.HostXCPNGVM{vm},
CollectedAt: now.Add(time.Second),
},
},
}
registry := NewRegistry(nil)
registry.IngestSnapshot(models.StateSnapshot{Hosts: hosts})
vms := registry.ListByType(ResourceTypeVM)
if len(vms) != 1 {
t.Fatalf("XCP-ng VMs = %d, want one de-duplicated VM: %+v", len(vms), vms)
}
projected := vms[0]
if projected.Technology != "xcp-ng" ||
projected.Status != StatusOnline ||
projected.VirtualMachine == nil ||
projected.VirtualMachine.Hypervisor != "xcp-ng" ||
projected.VirtualMachine.VCPUs != 4 {
t.Fatalf("XCP-ng VM = %+v", projected)
}
if projected.Metrics == nil ||
projected.Metrics.Memory == nil ||
projected.Metrics.Memory.Percent != 50 {
t.Fatalf("XCP-ng VM memory = %+v", projected.Metrics)
}
if projected.ParentID == nil {
t.Fatal("XCP-ng VM has no resident parent")
}
parent, ok := registry.Get(*projected.ParentID)
if !ok || parent.Name != "xcp-two" || parent.Technology != "xcp-ng" {
t.Fatalf("resident parent = %+v, found=%v", parent, ok)
}
if parent.Identity.ClusterName != "Lab Pool" {
t.Fatalf("parent cluster identity = %+v", parent.Identity)
}
if BuildMetricsTargetForRegistry(registry, projected.ID) != nil {
t.Fatal("inventory-only XCP-ng VM must not advertise nonexistent metrics history")
}
}
func TestXCPNGResourceStatusMapping(t *testing.T) {
for state, want := range map[string]ResourceStatus{
"running": StatusOnline,
"halted": StatusOffline,
"paused": StatusWarning,
"suspended": StatusWarning,
"unknown": StatusUnknown,
} {
if got := xcpngResourceStatus(state); got != want {
t.Errorf("xcpngResourceStatus(%q) = %q, want %q", state, got, want)
}
}
}
func TestResourceRegistry_GetByReferenceResolvesSourceIDAndCanonicalAlias(t *testing.T) {
rr := NewRegistry(nil)
rr.IngestRecords(SourceProxmox, []IngestRecord{{
SourceID: "delly:delly:101",
Resource: Resource{
Type: ResourceTypeSystemContainer,
Name: "homeassistant",
},
}})
rr.IngestResources([]Resource{{
ID: "agent-host-1",
Type: ResourceTypeAgent,
Name: "delly",
Agent: &AgentData{
AgentID: "machine-1",
Hostname: "delly.local",
},
}})
resource, resolvedID, ok := rr.GetByReference("delly:delly:101")
if !ok {
t.Fatal("expected Proxmox source ID to resolve")
}
wantID := SourceSpecificID(ResourceTypeSystemContainer, SourceProxmox, "delly:delly:101")
if resolvedID != wantID || resource.ID != wantID {
t.Fatalf("source reference resolved to id=%q resource=%q, want %q", resolvedID, resource.ID, wantID)
}
resource, resolvedID, ok = rr.GetByReference("machine-1")
if !ok {
t.Fatal("expected canonical alias to resolve")
}
if resolvedID != "agent-host-1" || resource.ID != "agent-host-1" {
t.Fatalf("alias reference resolved to id=%q resource=%q, want agent-host-1", resolvedID, resource.ID)
}
resource, resolvedID, ok = rr.GetByReference("agent:machine-1")
if !ok {
t.Fatal("expected prefixed agent reference to resolve")
}
if resolvedID != "agent-host-1" || resource.ID != "agent-host-1" {
t.Fatalf("agent reference resolved to id=%q resource=%q, want agent-host-1", resolvedID, resource.ID)
}
}
// A pulse-agent installed on a Proxmox node merges into one canonical
// resource whose primary ID is "node:{sourceID}", not "agent:{agentID}".
// Alert evaluation still references the host as "agent:{hostID}"
// (alerts.hostResourceID), and per-resource intent policies are keyed by the
// canonical registry ID, so this reference must resolve or grace overrides
// silently fall back to factory (#1497).
func TestResourceRegistry_GetByReferenceResolvesAgentRefOnMergedProxmoxHost(t *testing.T) {
rr := NewRegistry(nil)
rr.IngestResources([]Resource{{
ID: "agent-abcdef123456",
Type: ResourceTypeAgent,
Name: "proxmox2",
Agent: &AgentData{
AgentID: "machine-2",
Hostname: "proxmox2.local",
},
Proxmox: &ProxmoxData{
SourceID: "node/proxmox2",
NodeName: "proxmox2",
},
}})
merged, ok := rr.Get("agent-abcdef123456")
if !ok || merged.Canonical == nil {
t.Fatalf("merged host missing canonical identity: ok=%v resource=%+v", ok, merged)
}
if merged.Canonical.PrimaryID != "node:node/proxmox2" {
t.Fatalf("merged host primary ID = %q, want node:node/proxmox2", merged.Canonical.PrimaryID)
}
resource, resolvedID, ok := rr.GetByReference("agent:machine-2")
if !ok {
t.Fatal("expected agent reference to resolve on merged Proxmox host")
}
if resolvedID != "agent-abcdef123456" || resource.ID != "agent-abcdef123456" {
t.Fatalf("agent reference resolved to id=%q resource=%q, want agent-abcdef123456", resolvedID, resource.ID)
}
}
func TestResourceRegistry_CanonicalAliasIndexPreservesFoldAndAmbiguity(t *testing.T) {
rr := NewRegistry(nil)
rr.IngestResources([]Resource{
{ID: "kelvin-host", Type: ResourceTypeAgent, Agent: &AgentData{AgentID: "Kelvin"}},
{ID: "dotted-host", Type: ResourceTypeAgent, Agent: &AgentData{AgentID: "İmachine"}},
{ID: "first-shared", Type: ResourceTypeAgent, Agent: &AgentData{AgentID: "shared"}},
{ID: "second-shared", Type: ResourceTypeAgent, Agent: &AgentData{AgentID: "shared"}},
})
if rr.canonicalIdentityIndex != nil {
t.Fatal("ingest should defer alias index construction")
}
if id, ok := rr.ResolveReferenceID("first-shared"); !ok || id != "first-shared" {
t.Fatalf("exact ID resolved to %q, %v; want first-shared", id, ok)
}
if rr.canonicalIdentityIndex != nil {
t.Fatal("exact ID read should not build the alias index")
}
if id, ok := rr.ResolveReferenceID(" AGENT:kelvin "); !ok || id != "kelvin-host" {
t.Fatalf("Unicode folded alias resolved to %q, %v; want kelvin-host", id, ok)
}
if rr.canonicalIdentityIndex == nil {
t.Fatal("alias lookup did not build the index")
}
if id, ok := rr.ResolveReferenceID("agent:imachine"); ok {
t.Fatalf("dotted I must not fold into ordinary i, got %q", id)
}
if id, ok := rr.ResolveReferenceID("agent:shared"); ok {
t.Fatalf("ambiguous canonical alias resolved to %q", id)
}
if id, ok := rr.ResolveReferenceID("first-shared"); !ok || id != "first-shared" {
t.Fatalf("exact ID lost priority to ambiguous alias: %q, %v", id, ok)
}
rr.mu.Lock()
rr.canonicalIdentityIndex = nil
rr.mu.Unlock()
rr.mu.RLock()
id := rr.uniqueCanonicalIdentityResourceIDLocked("AGENT:kelvin")
sharedID := rr.uniqueCanonicalIdentityResourceIDLocked("agent:shared")
rr.mu.RUnlock()
if id != "kelvin-host" {
t.Fatalf("in-progress ingest scan resolved to %q; want kelvin-host", id)
}
if sharedID != "" {
t.Fatalf("in-progress ingest scan resolved ambiguous alias to %q", sharedID)
}
}
func TestResourceRegistry_CanonicalAliasIndexRefreshesAfterIngest(t *testing.T) {
rr := NewRegistry(nil)
rr.IngestResources([]Resource{{
ID: "renamed-host", Type: ResourceTypeAgent, Agent: &AgentData{AgentID: "old-machine"},
}})
if _, ok := rr.ResolveReferenceID("agent:old-machine"); !ok {
t.Fatal("initial alias missing")
}
rr.IngestResources([]Resource{{
ID: "renamed-host", Type: ResourceTypeAgent, Agent: &AgentData{AgentID: "new-machine"},
}})
if id, ok := rr.ResolveReferenceID("agent:old-machine"); ok {
t.Fatalf("stale alias resolved after ingest to %q", id)
}
if id, ok := rr.ResolveReferenceID("agent:new-machine"); !ok || id != "renamed-host" {
t.Fatalf("new alias resolved to %q, %v; want renamed-host", id, ok)
}
}
func BenchmarkResourceRegistry_GetByCanonicalAlias(b *testing.B) {
resources := make([]Resource, 1500)
for i := range resources {
resources[i] = Resource{
ID: fmt.Sprintf("agent-%d", i),
Type: ResourceTypeAgent,
Agent: &AgentData{AgentID: fmt.Sprintf("machine-%d", i)},
}
}
rr := NewRegistry(nil)
rr.IngestResources(resources)
for _, test := range []struct {
name string
ref string
}{
{"hit", "agent:machine-750"},
{"miss", "agent:unknown"},
} {
b.Run(test.name, func(b *testing.B) {
b.ReportAllocs()
for i := 0; i < b.N; i++ {
_, _, _ = rr.GetByReference(test.ref)
}
})
}
}
func TestMonitorAdapterKeepsSameNamedProxmoxProvidersDistinct(t *testing.T) {
adapter := NewMonitorAdapter(NewRegistry(NewMemoryStore()))
seen := time.Date(2026, 7, 24, 8, 0, 0, 0, time.UTC)
adapter.PopulateFromSnapshot(models.StateSnapshot{
Nodes: []models.Node{
{
ID: "site-a-pve-1",
Name: "pve-1",
Instance: "site-a",
Status: "online",
Type: "node",
IsClusterMember: true,
ClusterName: "production",
LastSeen: seen,
LinkedAgentID: "shared-agent-id",
},
{
ID: "site-b-pve-1",
Name: "pve-1",
Instance: "site-b",
Status: "offline",
Type: "node",
IsClusterMember: true,
ClusterName: "production",
LastSeen: seen,
LinkedAgentID: "shared-agent-id",
},
},
Hosts: []models.Host{{
ID: "shared-agent-id",
Hostname: "pve-1",
MachineID: "machine-site-a",
LinkedNodeID: "site-a-pve-1",
LastSeen: seen,
}},
LastUpdate: seen,
})
resources := adapter.GetAll()
if len(resources) != 2 {
t.Fatalf("same-name provider resources = %d, want 2: %+v", len(resources), resources)
}
byInstance := make(map[string]Resource, len(resources))
for _, resource := range resources {
if resource.Proxmox != nil {
byInstance[resource.Proxmox.Instance] = resource
}
}
if byInstance["site-a"].ID == "" || byInstance["site-b"].ID == "" ||
byInstance["site-a"].ID == byInstance["site-b"].ID {
t.Fatalf("provider-scoped canonical identities collided: %+v", resources)
}
if got := MonitoredSystemCount(adapter); got != 2 {
t.Fatalf("same-name provider monitored-system count = %d, want 2", got)
}
presented := CoalescePresentationHostResources(resources)
if len(presented) != 2 {
t.Fatalf("repeated cluster label collapsed provider-scoped presentation rows: %+v", presented)
}
}
func TestResourceRegistryAvailabilityLinkedResourceResolvesSourceReference(t *testing.T) {
rr := NewRegistry(nil)
now := time.Now().UTC()
const serviceSourceID = "swarm-1:service:svc-api"
rr.IngestRecords(SourceDocker, []IngestRecord{{
SourceID: serviceSourceID,
Resource: Resource{
Type: ResourceTypeDockerService,
Name: "api",
Status: StatusOnline,
LastSeen: now,
},
Identity: ResourceIdentity{Hostnames: []string{"api"}},
}})
rr.IngestRecords(SourceAvailability, []IngestRecord{
availabilityProbeRecord("probe-service", "api.example.test", &AvailabilityData{
LinkedResourceID: serviceSourceID,
Address: "api.example.test",
Protocol: "tcp",
Port: 8443,
TargetKind: "service",
Enabled: true,
Available: true,
}),
})
if got := rr.ListByType(ResourceTypeNetworkEndpoint); len(got) != 1 {
t.Fatalf("expected source-referenced probe to retain its endpoint row, got %d", len(got))
}
services := rr.ListByType(ResourceTypeDockerService)
if len(services) != 1 {
t.Fatalf("expected 1 docker service, got %d", len(services))
}
service := services[0]
if service.Availability == nil || service.Availability.TargetID != "probe-service" {
t.Fatalf("expected availability facet probe-service on docker service, got %+v", service.Availability)
}
if service.Availability.LinkedResourceID != serviceSourceID {
t.Fatalf("linkedResourceId = %q, want %q", service.Availability.LinkedResourceID, serviceSourceID)
}
if !hasDataSource(service.Sources, SourceAvailability) {
t.Fatalf("expected docker service sources to include availability, got %v", service.Sources)
}
}
func TestResourceRegistry_ListByType(t *testing.T) {
rr := NewRegistry(nil)
now := time.Date(2026, 2, 10, 0, 0, 0, 0, time.UTC)
rr.IngestRecords(SourceAgent, []IngestRecord{
{
SourceID: "host-1",
Resource: Resource{
Type: ResourceTypeAgent,
Name: "host-1",
Status: StatusOnline,
LastSeen: now,
},
Identity: ResourceIdentity{MachineID: "machine-1"},
},
})
rr.IngestRecords(SourceProxmox, []IngestRecord{
{
SourceID: "vm-100",
Resource: Resource{
Type: ResourceTypeVM,
Name: "vm-100",
Status: StatusOnline,
LastSeen: now,
},
Identity: ResourceIdentity{Hostnames: []string{"vm-100"}},
},
{
SourceID: "vm-101",
Resource: Resource{
Type: ResourceTypeVM,
Name: "vm-101",
Status: StatusOnline,
LastSeen: now,
},
Identity: ResourceIdentity{Hostnames: []string{"vm-101"}},
},
{
SourceID: "ct-200",
Resource: Resource{
Type: ResourceTypeSystemContainer,
Name: "ct-200",
Status: StatusOnline,
LastSeen: now,
},
Identity: ResourceIdentity{Hostnames: []string{"ct-200"}},
},
})
got := rr.ListByType(ResourceTypeVM)
if len(got) != 2 {
t.Fatalf("expected 2 VMs, got %d", len(got))
}
for _, r := range got {
if r.Type != ResourceTypeVM {
t.Fatalf("expected all resources to be type %q, got %q", ResourceTypeVM, r.Type)
}
}
// Deterministic ordering (sorted by ID).
wantIDs := []string{
rr.sourceSpecificID(ResourceTypeVM, SourceProxmox, "vm-100"),
rr.sourceSpecificID(ResourceTypeVM, SourceProxmox, "vm-101"),
}
// Hash order isn't meaningful; the contract is lexicographic by ID.
if wantIDs[0] > wantIDs[1] {
wantIDs[0], wantIDs[1] = wantIDs[1], wantIDs[0]
}
if got[0].ID != wantIDs[0] || got[1].ID != wantIDs[1] {
t.Fatalf("expected IDs %v, got [%s %s]", wantIDs, got[0].ID, got[1].ID)
}
// Returned resources should be copies (mutating the result does not mutate the registry).
origName := got[0].Name
got[0].Name = "mutated"
if r, ok := rr.Get(got[0].ID); !ok || r == nil {
t.Fatalf("expected Get(%q) to succeed", got[0].ID)
} else if r.Name != origName {
t.Fatalf("expected registry resource name %q, got %q", origName, r.Name)
}
}
func TestResourceRegistry_ListByType_Empty(t *testing.T) {
rr := NewRegistry(nil)
if got := rr.ListByType(ResourceTypeVM); len(got) != 0 {
t.Fatalf("expected empty result, got %d", len(got))
}
}
func TestResourceRegistry_IngestRecordsPreservesAvailabilityEndpoints(t *testing.T) {
rr := NewRegistry(nil)
now := time.Date(2026, 5, 6, 12, 0, 0, 0, time.UTC)
rr.IngestRecords(SourceAvailability, []IngestRecord{
{
SourceID: "energy-meter",
Resource: Resource{
Type: ResourceTypeNetworkEndpoint,
Name: "Energy meter",
Status: StatusOffline,
LastSeen: now,
Sources: []DataSource{SourceAvailability},
Availability: &AvailabilityData{
TargetID: "energy-meter",
Address: "192.0.2.44",
Protocol: "icmp",
Enabled: true,
Available: false,
ConsecutiveFailures: 2,
FailureThreshold: 2,
},
Incidents: []ResourceIncident{{
Provider: "availability",
NativeID: "energy-meter",
Code: "availability_unreachable",
Severity: storagehealth.RiskCritical,
Source: "availability",
Summary: "Energy meter is unreachable by ICMP probe",
}},
},
Identity: ResourceIdentity{IPAddresses: []string{"192.0.2.44"}},
},
})
got := rr.ListByType(ResourceTypeNetworkEndpoint)
if len(got) != 1 {
t.Fatalf("expected 1 network endpoint, got %d", len(got))
}
resource := got[0]
if resource.Availability == nil {
t.Fatal("expected availability metadata")
}
if resource.Availability.TargetID != "energy-meter" {
t.Fatalf("target id = %q, want energy-meter", resource.Availability.TargetID)
}
if resource.Status != StatusOffline {
t.Fatalf("status = %q, want %q", resource.Status, StatusOffline)
}
if len(resource.Incidents) != 1 || resource.Incidents[0].Code != "availability_unreachable" {
t.Fatalf("expected availability incident, got %+v", resource.Incidents)
}
if resource.Canonical == nil || resource.Canonical.PrimaryID != "availability:energy-meter" {
t.Fatalf("canonical identity = %+v, want primary availability:energy-meter", resource.Canonical)
}
}
func TestResourceRegistry_ListUsesDeterministicNameTieBreakers(t *testing.T) {
rr := NewRegistry(nil)
now := time.Date(2026, 4, 11, 0, 0, 0, 0, time.UTC)
rr.IngestResources([]Resource{
{ID: "storage-z", Type: ResourceTypeStorage, Name: "backup-vault-a", Status: StatusOnline, LastSeen: now},
{ID: "storage-b", Type: ResourceTypeStorage, Name: "backup-vault-a", Status: StatusOnline, LastSeen: now},
{ID: "agent-1", Type: ResourceTypeAgent, Name: "alpha-host", Status: StatusOnline, LastSeen: now},
{ID: "storage-a", Type: ResourceTypeStorage, Name: "backup-vault-a", Status: StatusOnline, LastSeen: now},
})
got := rr.List()
gotIDs := make([]string, 0, len(got))
for _, resource := range got {
gotIDs = append(gotIDs, resource.ID)
}
assertStringSlice(t, gotIDs, []string{"agent-1", "storage-a", "storage-b", "storage-z"})
}
func TestResourceRegistry_IngestResourcesSeedsMatcherForOverlayRecords(t *testing.T) {
rr := NewRegistry(nil)
now := time.Date(2026, 4, 15, 12, 0, 0, 0, time.UTC)
rr.IngestResources([]Resource{
{
ID: "node-1",
Type: ResourceTypeAgent,
Name: "tower",
Status: StatusOnline,
LastSeen: now,
Sources: []DataSource{SourceProxmox},
Identity: ResourceIdentity{
MachineID: "machine-1",
Hostnames: []string{"tower"},
},
Proxmox: &ProxmoxData{
NodeName: "tower",
Instance: "lab",
},
},
})
rr.IngestRecords(SourceAgent, []IngestRecord{
{
SourceID: "host-1",
Resource: Resource{
Type: ResourceTypeAgent,
Name: "tower",
Status: StatusOnline,
LastSeen: now.Add(time.Minute),
Agent: &AgentData{
AgentID: "agent-1",
},
},
Identity: ResourceIdentity{
MachineID: "machine-1",
Hostnames: []string{"tower"},
},
},
})
resources := rr.List()
if len(resources) != 1 {
t.Fatalf("expected overlay record to merge into seeded resource, got %d resources", len(resources))
}
if resources[0].Proxmox == nil || resources[0].Agent == nil {
t.Fatalf("expected merged resource to keep seeded and overlay metadata, got %+v", resources[0])
}
if got := len(resources[0].Sources); got != 2 {
t.Fatalf("expected merged sources from seed and overlay, got %d (%+v)", got, resources[0].Sources)
}
}
func TestResourceRegistry_IngestResourcesRebuildsSourceMappingsForMetricsTargets(t *testing.T) {
rr := NewRegistry(nil)
now := time.Date(2026, 4, 21, 9, 0, 0, 0, time.UTC)
rr.IngestResources([]Resource{
{
ID: "agent-dashboard-1",
Type: ResourceTypeAgent,
Name: "delly",
Status: StatusOnline,
LastSeen: now,
Sources: []DataSource{SourceProxmox, SourceAgent},
Identity: ResourceIdentity{
MachineID: "machine-delly",
Hostnames: []string{"delly"},
},
Proxmox: &ProxmoxData{
SourceID: "homelab-delly",
NodeName: "delly",
Instance: "homelab-entry",
},
Agent: &AgentData{
AgentID: "agent-source-delly",
Hostname: "delly",
},
},
})
target := BuildMetricsTargetForRegistry(rr, "agent-dashboard-1")
if target == nil {
t.Fatal("expected seeded unified agent to rebuild a metrics target")
}
// The agent source ID wins over the proxmox node ID: it is the host.ID
// key the agent metrics writer stores rows under.
if target.ResourceType != "agent" || target.ResourceID != "agent-source-delly" {
t.Fatalf("metrics target = %+v, want agent/agent-source-delly", target)
}
targets := rr.SourceTargets("agent-dashboard-1")
if len(targets) != 2 {
t.Fatalf("expected 2 source targets, got %d", len(targets))
}
foundAgent := false
foundProxmox := false
for _, sourceTarget := range targets {
switch sourceTarget.Source {
case SourceAgent:
foundAgent = sourceTarget.SourceID == "agent-source-delly"
case SourceProxmox:
foundProxmox = sourceTarget.SourceID == "homelab-delly"
}
}
if !foundAgent || !foundProxmox {
t.Fatalf("expected rebuilt agent+proxmox source targets, got %+v", targets)
}
}
func TestResourceRegistry_IngestResourcesSeedsHostScopedDockerContainerMappings(t *testing.T) {
rr := NewRegistry(nil)
now := time.Date(2026, 5, 19, 14, 0, 0, 0, time.UTC)
host105 := "proxmox-lxc-docker:pve-a:node-a:105"
host141 := "proxmox-lxc-docker:pve-a:node-a:141"
rr.IngestResources([]Resource{
{
ID: "docker-container-frigate-105",
Type: ResourceTypeAppContainer,
Name: "frigate",
Status: StatusOnline,
LastSeen: now,
Sources: []DataSource{SourceDocker},
Docker: &DockerData{
HostSourceID: host105,
ContainerID: "frigate",
DisplayName: "frigate",
Runtime: "docker",
ContainerState: "running",
},
},
{
ID: "docker-container-frigate-141",
Type: ResourceTypeAppContainer,
Name: "frigate",
Status: StatusOnline,
LastSeen: now,
Sources: []DataSource{SourceDocker},
Docker: &DockerData{
HostSourceID: host141,
ContainerID: "frigate",
DisplayName: "frigate",
Runtime: "docker",
ContainerState: "running",
},
},
})
resources := rr.ListByType(ResourceTypeAppContainer)
if len(resources) != 2 {
t.Fatalf("expected seeded resources to preserve two host-scoped containers, got %d", len(resources))
}
assertSourceTarget := func(resourceID, wantSourceID string) {
t.Helper()
for _, target := range rr.SourceTargets(resourceID) {
if target.Source == SourceDocker && target.SourceID == wantSourceID {
wantCandidateID := SourceSpecificID(ResourceTypeAppContainer, SourceDocker, wantSourceID)
if target.CandidateID != wantCandidateID {
t.Fatalf("resource %q docker candidate ID = %q, want %q", resourceID, target.CandidateID, wantCandidateID)
}
return
}
}
t.Fatalf("resource %q missing docker source target %q; got %+v", resourceID, wantSourceID, rr.SourceTargets(resourceID))
}
assertSourceTarget("docker-container-frigate-105", host105+"/container/frigate")
assertSourceTarget("docker-container-frigate-141", host141+"/container/frigate")
scopesByHost := make(map[string][]string, len(resources))
for _, resource := range resources {
if resource.Docker == nil {
t.Fatalf("docker container resource missing Docker payload: %#v", resource)
}
scopesByHost[resource.Docker.HostSourceID] = resource.PlatformScopes
}
for _, host := range []string{host105, host141} {
if got := scopesByHost[host]; !reflect.DeepEqual(got, []string{"proxmox-pve", "docker"}) {
t.Fatalf("host %q platform scopes = %#v, want proxmox-pve + docker", host, got)
}
}
}
func TestResourceRegistry_IngestResourcesDerivesClusterWorkloadParentFromSeededProxmoxNode(t *testing.T) {
rr := NewRegistry(nil)
now := time.Date(2026, 5, 14, 10, 0, 0, 0, time.UTC)
rr.IngestResources([]Resource{
{
ID: "agent-delly",
Type: ResourceTypeAgent,
Name: "delly",
Status: StatusOnline,
LastSeen: now,
Sources: []DataSource{SourceProxmox, SourceAgent},
Identity: ResourceIdentity{
MachineID: "machine-delly",
Hostnames: []string{"delly"},
},
Proxmox: &ProxmoxData{
SourceID: "homelab-delly",
NodeName: "delly",
ClusterName: "homelab",
Instance: "delly",
},
Agent: &AgentData{
AgentID: "agent-source-delly",
Hostname: "delly",
},
},
{
ID: "system-container-cloudflared",
Type: ResourceTypeSystemContainer,
Name: "cloudflared",
Status: StatusOnline,
LastSeen: now,
Sources: []DataSource{SourceProxmox},
Identity: ResourceIdentity{Hostnames: []string{"cloudflared"}},
Proxmox: &ProxmoxData{
SourceID: "delly:delly:104",
NodeName: "delly",
ClusterName: "homelab",
Instance: "delly",
VMID: 104,
},
},
})
child, ok := rr.Get("system-container-cloudflared")
if !ok {
t.Fatal("expected cloudflared resource")
}
if child.ParentID == nil || *child.ParentID != "agent-delly" {
t.Fatalf("expected cloudflared parent agent-delly, got %+v", child.ParentID)
}
if child.ParentName != "delly" {
t.Fatalf("expected cloudflared parent name delly, got %q", child.ParentName)
}
parent, ok := rr.Get("agent-delly")
if !ok {
t.Fatal("expected delly parent resource")
}
if parent.ChildCount != 1 {
t.Fatalf("expected delly child count 1, got %d", parent.ChildCount)
}
}
func TestStorageRiskSemanticsPrefersUnraidParitySummaryOverGenericDiskCounts(t *testing.T) {
risk := &StorageRisk{
Reasons: []StorageRiskReason{
{Code: "unraid_disabled_disks", Severity: storagehealth.RiskCritical, Summary: "Unraid array reports 1 disabled disk(s)"},
{Code: "unraid_parity_unavailable", Severity: storagehealth.RiskCritical, Summary: "Unraid parity protection is unavailable"},
},
}
_, protectionReduced, _, protectionSummary, _ := StorageRiskSemantics(risk)
if !protectionReduced {
t.Fatal("expected protectionReduced=true")
}
if protectionSummary != "Unraid parity protection is unavailable" {
t.Fatalf("protectionSummary = %q, want %q", protectionSummary, "Unraid parity protection is unavailable")
}
}
func TestResourceRegistry_MergesPhysicalDiskTemperatureAggregate(t *testing.T) {
rr := NewRegistry(nil)
now := time.Date(2026, 3, 29, 12, 0, 0, 0, time.UTC)
rr.IngestRecords(SourceTrueNAS, []IngestRecord{
{
SourceID: "disk-sda",
Resource: Resource{
Type: ResourceTypePhysicalDisk,
Name: "sda",
Status: StatusOnline,
LastSeen: now,
PhysicalDisk: &PhysicalDiskMeta{
DevPath: "/dev/sda",
Model: "Seagate Exos X18",
Serial: "SER-A",
DiskType: "sata",
SizeBytes: 1_000_000,
Health: "PASSED",
Temperature: 34,
Wearout: -1,
TemperatureAggregate: &TemperatureAggregateMeta{
WindowDays: 7,
MinCelsius: 29.0,
AvgCelsius: 32.7,
MaxCelsius: 38.0,
},
},
},
Identity: ResourceIdentity{MachineID: "SER-A"},
},
})
disks := rr.ListByType(ResourceTypePhysicalDisk)
if len(disks) != 1 {
t.Fatalf("expected 1 physical disk, got %d", len(disks))
}
aggregate := disks[0].PhysicalDisk.TemperatureAggregate
if aggregate == nil {
t.Fatalf("expected temperature aggregate on merged disk record: %+v", disks[0].PhysicalDisk)
}
if aggregate.WindowDays != 7 || aggregate.MinCelsius != 29.0 || aggregate.AvgCelsius != 32.7 || aggregate.MaxCelsius != 38.0 {
t.Fatalf("unexpected merged temperature aggregate: %+v", aggregate)
}
aggregate.MaxCelsius = 99.0
got, ok := rr.Get(disks[0].ID)
if !ok || got == nil || got.PhysicalDisk == nil || got.PhysicalDisk.TemperatureAggregate == nil {
t.Fatalf("expected stored physical disk record, got %+v", got)
}
if got.PhysicalDisk.TemperatureAggregate.MaxCelsius != 38.0 {
t.Fatalf("expected registry clone isolation for temperature aggregate, got %+v", got.PhysicalDisk.TemperatureAggregate)
}
}
func TestResourceRegistry_IngestsVMwareSourceAsCanonicalResources(t *testing.T) {
rr := NewRegistry(nil)
now := time.Date(2026, 3, 30, 18, 30, 0, 0, time.UTC)
rr.IngestRecords(SourceVMware, []IngestRecord{
{
SourceID: "vc-1:host:host-101",
Resource: Resource{
Type: ResourceTypeAgent,
Name: "esxi-01.lab.local",
Status: StatusOnline,
LastSeen: now,
VMware: &VMwareData{
ConnectionID: "vc-1",
ConnectionName: "Lab VC",
VCenterHost: "vc.lab.local",
ManagedObjectID: "host-101",
EntityType: "host",
HostUUID: "uuid-host-1",
},
},
Identity: ResourceIdentity{
DMIUUID: "uuid-host-1",
Hostnames: []string{"esxi-01.lab.local"},
},
},
})
agents := rr.ListByType(ResourceTypeAgent)
if len(agents) != 1 {
t.Fatalf("expected 1 VMware agent resource, got %d", len(agents))
}
agent := agents[0]
if len(agent.Sources) != 1 || agent.Sources[0] != SourceVMware {
t.Fatalf("expected VMware source ownership, got %+v", agent.Sources)
}
if agent.VMware == nil {
t.Fatalf("expected VMware metadata on merged resource, got %+v", agent)
}
if got := agent.VMware.ConnectionID; got != "vc-1" {
t.Fatalf("vmware connection id = %q, want vc-1", got)
}
if got := agent.Canonical.PrimaryID; got != "vmware:vc-1:host:host-101" {
t.Fatalf("canonical primary id = %q, want vmware:vc-1:host:host-101", got)
}
}
func TestResourceRegistry_PreservesVMwareSourceOwnedDatastoreConsumers(t *testing.T) {
rr := NewRegistry(nil)
now := time.Date(2026, 3, 30, 18, 45, 0, 0, time.UTC)
rr.IngestRecords(SourceVMware, []IngestRecord{
{
SourceID: "vc-1:datastore:datastore-201",
Resource: Resource{
Type: ResourceTypeStorage,
Name: "nvme-primary",
Status: StatusOnline,
LastSeen: now,
Storage: &StorageMeta{
Type: "vmfs",
Platform: "vmware-vsphere",
Topology: "datastore",
ConsumerCount: 2,
ConsumerTypes: []string{string(ResourceTypeVM)},
TopConsumers: []StorageConsumerMeta{
{ResourceType: ResourceTypeVM, Name: "warehouse-api-01"},
{ResourceType: ResourceTypeVM, Name: "etl-batch-01"},
},
},
VMware: &VMwareData{
ConnectionID: "vc-1",
ManagedObjectID: "datastore-201",
EntityType: "datastore",
},
},
},
})
datastores := rr.ListByType(ResourceTypeStorage)
if len(datastores) != 1 {
t.Fatalf("expected 1 VMware datastore, got %d", len(datastores))
}
datastore := datastores[0]
if datastore.Storage == nil {
t.Fatalf("expected VMware datastore storage metadata, got %+v", datastore)
}
if got := datastore.Storage.ConsumerCount; got != 2 {
t.Fatalf("consumer count = %d, want 2", got)
}
if got := datastore.Storage.ConsumerTypes; len(got) != 1 || got[0] != string(ResourceTypeVM) {
t.Fatalf("consumer types = %#v, want [vm]", got)
}
if len(datastore.Storage.TopConsumers) != 2 {
t.Fatalf("top consumers = %#v, want 2 VMware VM consumers", datastore.Storage.TopConsumers)
}
if datastore.Storage.TopConsumers[0].Name != "warehouse-api-01" {
t.Fatalf("first top consumer = %#v, want warehouse-api-01", datastore.Storage.TopConsumers[0])
}
}
func TestBuildMetricsTargetForRegistryFallsBackToStoredMetricsTarget(t *testing.T) {
rr := NewRegistry(nil)
now := time.Date(2026, 3, 31, 10, 0, 0, 0, time.UTC)
resourceID := CanonicalResourceID("app-demo")
rr.resources[resourceID] = &Resource{
ID: resourceID,
Type: ResourceTypeAppContainer,
Name: "Nextcloud",
Status: StatusOnline,
LastSeen: now,
MetricsTarget: &MetricsTarget{
ResourceType: "dockerContainer",
ResourceID: "nextcloud-web-1",
},
}
target := BuildMetricsTargetForRegistry(rr, resourceID)
if target == nil {
t.Fatal("expected stored metrics target fallback")
}
if target.ResourceType != "dockerContainer" || target.ResourceID != "nextcloud-web-1" {
t.Fatalf("unexpected fallback metrics target %+v", target)
}
target.ResourceID = "mutated"
stored, ok := rr.Get(resourceID)
if !ok || stored == nil || stored.MetricsTarget == nil {
t.Fatalf("expected stored app resource with metrics target, got %+v", stored)
}
if stored.MetricsTarget.ResourceID != "nextcloud-web-1" {
t.Fatalf("expected fallback target clone isolation, got %+v", stored.MetricsTarget)
}
}
func TestResourceRegistryStoragePoolViewsCarryResolvedMetricsTarget(t *testing.T) {
rr := NewRegistry(nil)
now := time.Date(2026, 4, 1, 11, 0, 0, 0, time.UTC)
rr.IngestRecords(SourceVMware, []IngestRecord{
{
SourceID: "vc-1:datastore:datastore-202",
Resource: Resource{
ID: "storage-vmware-1",
Type: ResourceTypeStorage,
Name: "archive-tier",
Status: StatusOnline,
LastSeen: now,
Storage: &StorageMeta{
Type: "datastore",
Platform: "vmware",
Nodes: []string{"esxi-01.lab.local"},
},
VMware: &VMwareData{
ConnectionID: "vc-1",
EntityType: "datastore",
ManagedObjectID: "datastore-202",
RuntimeHostName: "esxi-01.lab.local",
},
},
},
})
pools := rr.StoragePools()
if len(pools) != 1 {
t.Fatalf("expected 1 storage pool view, got %d", len(pools))
}
if got := pools[0].SourceID(); got != "vc-1:datastore:datastore-202" {
t.Fatalf("expected storage pool view to expose resolved metrics target, got %q", got)
}
if got := pools[0].Node(); got != "esxi-01.lab.local" {
t.Fatalf("expected storage pool view node hint to remain VMware runtime host, got %q", got)
}
stored, ok := rr.Get(pools[0].ID())
if !ok || stored == nil {
t.Fatalf("expected stored storage resource, got %+v", stored)
}
if stored.MetricsTarget != nil {
t.Fatalf("expected registry storage record to remain raw without persisted metrics target, got %+v", stored.MetricsTarget)
}
}
func TestMergeVMwareDataMergesSignalFieldsWithoutDroppingExistingIdentity(t *testing.T) {
existing := &VMwareData{
ConnectionID: "vc-1",
ConnectionName: "Lab VC",
VCenterHost: "vc.lab.local",
ManagedObjectID: "vm-101",
EntityType: "vm",
HostUUID: "uuid-host-1",
DatacenterName: "DC1",
ClusterName: "Cluster A",
DatastoreNames: []string{"primary-vmfs"},
NetworkHostNames: []string{"esxi-01.lab.local"},
ConnectionState: "connected",
PowerState: "poweredOn",
OverallStatus: "green",
GuestHostname: "app-01.internal",
ActiveAlarmCount: 1,
ActiveAlarmSummary: "old alarm",
RecentTaskCount: 1,
RecentTaskSummary: "old task",
SnapshotCount: 1,
}
accessible := false
clusterHA := true
clusterDRS := false
bootRetry := true
incoming := &VMwareData{
ClusterName: "Cluster A",
ClusterHAEnabled: &clusterHA,
ClusterDRSEnabled: &clusterDRS,
RuntimeHostName: "esxi-01.lab.local",
DatastoreNames: []string{"backup-nfs"},
NetworkType: "STANDARD_PORTGROUP",
NetworkHostNames: []string{"esxi-02.lab.local"},
NetworkVMNames: []string{"app-01"},
DatastoreAccessible: &accessible,
GuestIPAddresses: []string{"10.0.0.21"},
OverallStatus: "yellow",
ActiveAlarmCount: 3,
ActiveAlarmSummary: "Host connection lost; datastore latency elevated",
RecentTaskCount: 2,
RecentTaskSummary: "vMotion task completed",
SnapshotCount: 4,
CurrentSnapshotID: "snapshot-202",
SnapshotTree: []VMwareSnapshotData{{Snapshot: "snapshot-202", Name: "pre-maintenance"}},
NetworkAdapters: []VMwareNetworkAdapterData{{NIC: "4000", NetworkName: "VM Network"}},
VirtualDisks: []VMwareVirtualDiskData{{Disk: "2000", Label: "Hard disk 1"}},
Tools: &VMwareToolsData{RunState: "RUNNING", VersionStatus: "CURRENT"},
Hardware: &VMwareVMHardwareData{
Version: "VMX_20",
BootRetry: &bootRetry,
BootDevices: []VMwareBootDeviceData{{
Type: "DISK",
Disks: []string{"2000"},
}},
},
}
merged := mergeVMwareData(existing, incoming)
if merged == nil {
t.Fatal("expected merged VMware metadata")
}
if got := merged.ConnectionID; got != "vc-1" {
t.Fatalf("connection id = %q, want vc-1", got)
}
if got := merged.ManagedObjectID; got != "vm-101" {
t.Fatalf("managed object id = %q, want vm-101", got)
}
if got := merged.OverallStatus; got != "yellow" {
t.Fatalf("overall status = %q, want yellow", got)
}
if got := merged.ActiveAlarmCount; got != 3 {
t.Fatalf("active alarm count = %d, want 3", got)
}
if got := merged.ActiveAlarmSummary; got != "Host connection lost; datastore latency elevated" {
t.Fatalf("active alarm summary = %q", got)
}
if got := merged.RecentTaskCount; got != 2 {
t.Fatalf("recent task count = %d, want 2", got)
}
if got := merged.RecentTaskSummary; got != "vMotion task completed" {
t.Fatalf("recent task summary = %q", got)
}
if got := merged.SnapshotCount; got != 4 {
t.Fatalf("snapshot count = %d, want 4", got)
}
if got := merged.RuntimeHostName; got != "esxi-01.lab.local" {
t.Fatalf("runtime host name = %q, want esxi-01.lab.local", got)
}
if merged.ClusterHAEnabled == nil || !*merged.ClusterHAEnabled {
t.Fatalf("cluster HA enabled = %#v, want true", merged.ClusterHAEnabled)
}
if merged.ClusterDRSEnabled == nil || *merged.ClusterDRSEnabled {
t.Fatalf("cluster DRS enabled = %#v, want false", merged.ClusterDRSEnabled)
}
if got := merged.DatastoreNames; !reflect.DeepEqual(got, []string{"primary-vmfs", "backup-nfs"}) {
t.Fatalf("datastore names = %#v", got)
}
if got := merged.NetworkType; got != "STANDARD_PORTGROUP" {
t.Fatalf("network type = %q, want STANDARD_PORTGROUP", got)
}
if got := merged.NetworkHostNames; !reflect.DeepEqual(got, []string{"esxi-01.lab.local", "esxi-02.lab.local"}) {
t.Fatalf("network host names = %#v", got)
}
if got := merged.NetworkVMNames; !reflect.DeepEqual(got, []string{"app-01"}) {
t.Fatalf("network VM names = %#v", got)
}
if merged.DatastoreAccessible == nil || *merged.DatastoreAccessible {
t.Fatalf("datastore accessible = %#v, want false", merged.DatastoreAccessible)
}
if got := merged.GuestHostname; got != "app-01.internal" {
t.Fatalf("guest hostname = %q, want app-01.internal", got)
}
if got := merged.GuestIPAddresses; !reflect.DeepEqual(got, []string{"10.0.0.21"}) {
t.Fatalf("guest ip addresses = %#v", got)
}
if got := merged.CurrentSnapshotID; got != "snapshot-202" {
t.Fatalf("current snapshot id = %q, want snapshot-202", got)
}
if got := merged.SnapshotTree; len(got) != 1 || got[0].Name != "pre-maintenance" {
t.Fatalf("snapshot tree = %#v", got)
}
if got := merged.NetworkAdapters; len(got) != 1 || got[0].NetworkName != "VM Network" {
t.Fatalf("network adapters = %#v", got)
}
if got := merged.VirtualDisks; len(got) != 1 || got[0].Label != "Hard disk 1" {
t.Fatalf("virtual disks = %#v", got)
}
if got := merged.Tools; got == nil || got.RunState != "RUNNING" {
t.Fatalf("tools = %#v", got)
}
if got := merged.Hardware; got == nil || got.Version != "VMX_20" || got.BootDevices[0].Disks[0] != "2000" {
t.Fatalf("hardware = %#v", got)
}
}
func TestResourceRegistryClonesCarryPolicyMetadata(t *testing.T) {
rr := NewRegistry(nil)
now := time.Date(2026, 3, 17, 12, 0, 0, 0, time.UTC)
rr.IngestRecords(SourceProxmox, []IngestRecord{
{
SourceID: "vm-900",
Resource: Resource{
ID: "vm-900",
Type: ResourceTypeVM,
Name: "customer-vm",
Status: StatusOnline,
LastSeen: now,
Tags: []string{"customer-data"},
},
Identity: ResourceIdentity{
Hostnames: []string{"customer.internal"},
IPAddresses: []string{"10.0.0.90"},
},
},
})
listed := rr.ListByType(ResourceTypeVM)
if len(listed) != 1 {
t.Fatalf("expected 1 VM, got %d", len(listed))
}
if listed[0].Policy == nil {
t.Fatal("expected list result to include policy metadata")
}
if got := listed[0].Policy.Sensitivity; got != ResourceSensitivityRestricted {
t.Fatalf("policy sensitivity = %q, want %q", got, ResourceSensitivityRestricted)
}
if listed[0].AISafeSummary == "" {
t.Fatal("expected aiSafeSummary on list result")
}
got, ok := rr.Get(listed[0].ID)
if !ok || got == nil {
t.Fatalf("expected Get(%q) to succeed", listed[0].ID)
}
if got.Policy == nil {
t.Fatal("expected get result to include policy metadata")
}
if got.Policy == listed[0].Policy {
t.Fatal("expected policy metadata to be cloned, not shared by pointer")
}
}
func TestResourceRegistry_MonitoredSystemsSummarizeCanonicalTopLevelViews(t *testing.T) {
rr := NewRegistry(nil)
now := time.Date(2026, 3, 17, 12, 0, 0, 0, time.UTC)
rr.IngestRecords(SourceAgent, []IngestRecord{
{
SourceID: "host-1",
Resource: Resource{
ID: "host-1",
Type: ResourceTypeAgent,
Name: "lab-a",
Status: StatusOnline,
LastSeen: now.Add(-2 * time.Minute),
Agent: &AgentData{
AgentID: "agent-1",
Hostname: "lab-a",
MachineID: "machine-1",
},
},
Identity: ResourceIdentity{
MachineID: "machine-1",
Hostnames: []string{"lab-a"},
},
},
})
rr.IngestRecords(SourcePBS, []IngestRecord{
{
SourceID: "pbs-1",
Resource: Resource{
ID: "pbs-1",
Type: ResourceTypePBS,
Name: "pbs-a",
Status: StatusOnline,
LastSeen: now,
PBS: &PBSData{
InstanceID: "pbs-1",
Hostname: "lab-a",
HostURL: "https://lab-a:8007",
},
},
},
})
rr.IngestRecords(SourcePMG, []IngestRecord{
{
SourceID: "pmg-1",
Resource: Resource{
ID: "pmg-1",
Type: ResourceTypePMG,
Name: "pmg-b",
Status: StatusOffline,
LastSeen: now.Add(-10 * time.Minute),
PMG: &PMGData{
InstanceID: "pmg-1",
Hostname: "mail-b",
},
},
},
})
rr.IngestRecords(SourceK8s, []IngestRecord{
{
SourceID: "k8s-1",
Resource: Resource{
ID: "k8s-1",
Type: ResourceTypeK8sCluster,
Name: "cluster-a",
Status: StatusOnline,
LastSeen: now.Add(-30 * time.Second),
Kubernetes: &K8sData{
ClusterID: "cluster-a",
AgentID: "k8s-agent-1",
Server: "https://cluster-a.example:6443",
},
},
},
})
systems := MonitoredSystems(rr)
if len(systems) != 3 {
t.Fatalf("MonitoredSystems() returned %d systems, want 3", len(systems))
}
byName := make(map[string]MonitoredSystemRecord, len(systems))
for _, system := range systems {
byName[system.Name] = system
}
hostSummary, ok := byName["lab-a"]
if !ok {
t.Fatalf("expected monitored systems to include lab-a, got %+v", systems)
}
if hostSummary.Type != "host" || hostSummary.Source != "multiple" {
t.Fatalf("expected deduped host summary for lab-a, got %+v", hostSummary)
}
if !hostSummary.LastSeen.Equal(now) {
t.Fatalf("expected deduped host last_seen %s, got %s", now, hostSummary.LastSeen)
}
k8sSummary, ok := byName["cluster-a"]
if !ok {
t.Fatalf("expected monitored systems to include cluster-a, got %+v", systems)
}
if k8sSummary.Type != "kubernetes-cluster" || k8sSummary.Source != "kubernetes" {
t.Fatalf("expected kubernetes summary for cluster-a, got %+v", k8sSummary)
}
pmgSummary, ok := byName["pmg-b"]
if !ok {
t.Fatalf("expected monitored systems to include pmg-b, got %+v", systems)
}
if pmgSummary.Type != "pmg-server" || pmgSummary.Status != StatusOffline {
t.Fatalf("expected PMG summary for pmg-b, got %+v", pmgSummary)
}
}
func TestMonitoredSystemsExplainsStaleGroupedSourceWhileLastSeenStaysFresh(t *testing.T) {
rr := NewRegistry(nil)
now := time.Date(2026, 3, 23, 12, 0, 0, 0, time.UTC)
agentResource := topLevelTestAgent("agent-host", "tower.local", "machine-1", "agent-1")
agentResource.LastSeen = now.Add(-5 * time.Minute)
dockerResource := topLevelTestDockerHost("docker-host", "tower.local", "docker-runtime-1", "agent-1")
dockerResource.LastSeen = now.Add(-10 * time.Second)
rr.IngestRecords(SourceAgent, []IngestRecord{
{
SourceID: "agent-host",
Resource: agentResource,
},
})
rr.IngestRecords(SourceDocker, []IngestRecord{
{
SourceID: "docker-host",
Resource: dockerResource,
},
})
rr.MarkStale(now, map[DataSource]time.Duration{
SourceAgent: 60 * time.Second,
SourceDocker: 60 * time.Second,
})
systems := MonitoredSystems(rr)
if len(systems) != 1 {
t.Fatalf("MonitoredSystems() returned %d systems, want 1", len(systems))
}
system := systems[0]
if system.Status != StatusWarning {
t.Fatalf("expected grouped monitored system status warning, got %+v", system)
}
if !system.LastSeen.Equal(dockerResource.LastSeen) {
t.Fatalf("expected grouped last_seen %s, got %s", dockerResource.LastSeen, system.LastSeen)
}
if system.LatestIncludedSignal.Source != string(SourceDocker) {
t.Fatalf("expected latest included signal source docker, got %+v", system)
}
if system.LatestIncludedSignal.Name != "tower.local" {
t.Fatalf("expected latest included signal name tower.local, got %+v", system)
}
if system.LatestIncludedSignal.Type != "docker-host" {
t.Fatalf("expected latest included signal type docker-host, got %+v", system)
}
if !system.LatestIncludedSignal.At.Equal(dockerResource.LastSeen) {
t.Fatalf("expected latest included signal at %s, got %+v", dockerResource.LastSeen, system)
}
if system.StatusExplanation.Summary == "" {
t.Fatal("expected grouped monitored system status explanation summary")
}
if system.StatusExplanation.Summary != "Pulse most recently heard from Docker for tower.local at 2026-03-23T11:59:50Z, but Agent data for tower.local is stale (last reported 2026-03-23T11:55:00Z), so this monitored system is warning." {
t.Fatalf("unexpected grouped monitored system summary: %q", system.StatusExplanation.Summary)
}
if len(system.StatusExplanation.Reasons) != 1 {
t.Fatalf("expected one stale grouped-source reason, got %+v", system.StatusExplanation.Reasons)
}
reason := system.StatusExplanation.Reasons[0]
if reason.Kind != "source-stale" {
t.Fatalf("expected stale source reason kind, got %+v", reason)
}
if reason.Source != string(SourceAgent) {
t.Fatalf("expected agent source reason, got %+v", reason)
}
if reason.Status != "stale" {
t.Fatalf("expected stale reason status, got %+v", reason)
}
if !reason.ReportedAt.Equal(agentResource.LastSeen) {
t.Fatalf("expected stale reason reported_at %s, got %s", agentResource.LastSeen, reason.ReportedAt)
}
if reason.Summary == "" {
t.Fatalf("expected stale reason summary, got %+v", reason)
}
}
func TestMonitoredSystemsMergeShortAndFQDNHostnamesForOneHost(t *testing.T) {
rr := NewRegistry(nil)
now := time.Date(2026, 4, 21, 21, 30, 0, 0, time.UTC)
rr.IngestRecords(SourceAgent, []IngestRecord{
{
SourceID: "agent-qnap",
Resource: Resource{
ID: "agent-qnap",
Type: ResourceTypeAgent,
Name: "qnap.local",
Status: StatusOnline,
LastSeen: now.Add(-2 * time.Minute),
Agent: &AgentData{
AgentID: "agent-qnap",
Hostname: "qnap.local",
},
},
Identity: ResourceIdentity{
Hostnames: []string{"qnap.local"},
},
},
})
rr.IngestRecords(SourceDocker, []IngestRecord{
{
SourceID: "docker-qnap",
Resource: Resource{
ID: "docker-qnap",
Type: ResourceTypeAgent,
Name: "qnap",
Status: StatusOnline,
LastSeen: now,
Docker: &DockerData{
HostSourceID: "docker-qnap",
Hostname: "qnap",
},
},
Identity: ResourceIdentity{
Hostnames: []string{"qnap"},
},
},
})
systems := MonitoredSystems(rr)
if len(systems) != 1 {
t.Fatalf("MonitoredSystems() returned %d systems, want 1", len(systems))
}
if systems[0].Source != "multiple" {
t.Fatalf("MonitoredSystems()[0].Source = %q, want %q", systems[0].Source, "multiple")
}
if systems[0].Explanation.Summary == "" {
t.Fatal("expected grouped monitored-system explanation summary")
}
if !systems[0].LastSeen.Equal(now) {
t.Fatalf("MonitoredSystems()[0].LastSeen = %s, want %s", systems[0].LastSeen, now)
}
}
func TestMonitoredSystemsReportOnlineStatusWithoutUnknownBaselineBias(t *testing.T) {
rr := NewRegistry(nil)
proxmoxNode := topLevelTestProxmoxNode("proxmox-node", "tower", "proxmox-1", "https://tower.local:8006")
proxmoxNode.LastSeen = time.Date(2026, 3, 24, 10, 0, 0, 0, time.UTC)
rr.IngestRecords(SourceProxmox, []IngestRecord{
{
SourceID: "proxmox-node",
Resource: proxmoxNode,
},
})
systems := MonitoredSystems(rr)
if len(systems) != 1 {
t.Fatalf("MonitoredSystems() returned %d systems, want 1", len(systems))
}
if systems[0].Status != StatusOnline {
t.Fatalf("MonitoredSystems()[0].Status = %q, want %q", systems[0].Status, StatusOnline)
}
if systems[0].StatusExplanation.Summary != "All included top-level collection paths currently report online status." {
t.Fatalf("unexpected online status summary: %q", systems[0].StatusExplanation.Summary)
}
}
func TestMonitoredSystemsGroupedOnlineSourcesStayOnline(t *testing.T) {
rr := NewRegistry(nil)
now := time.Date(2026, 3, 24, 10, 0, 0, 0, time.UTC)
agentResource := topLevelTestAgent("agent-host", "tower.local", "machine-1", "agent-1")
agentResource.LastSeen = now.Add(-1 * time.Minute)
dockerResource := topLevelTestDockerHost("docker-host", "tower.local", "docker-runtime-1", "agent-1")
dockerResource.LastSeen = now
rr.IngestRecords(SourceAgent, []IngestRecord{
{
SourceID: "agent-host",
Resource: agentResource,
},
})
rr.IngestRecords(SourceDocker, []IngestRecord{
{
SourceID: "docker-host",
Resource: dockerResource,
},
})
systems := MonitoredSystems(rr)
if len(systems) != 1 {
t.Fatalf("MonitoredSystems() returned %d systems, want 1", len(systems))
}
if systems[0].Status != StatusOnline {
t.Fatalf("MonitoredSystems()[0].Status = %q, want %q", systems[0].Status, StatusOnline)
}
}
func TestMonitoredSystemsPreferOfflineOverWarning(t *testing.T) {
rr := NewRegistry(nil)
now := time.Date(2026, 3, 24, 10, 0, 0, 0, time.UTC)
agentResource := topLevelTestAgent("agent-host", "tower.local", "machine-1", "agent-1")
agentResource.Status = StatusWarning
agentResource.LastSeen = now.Add(-1 * time.Minute)
dockerResource := topLevelTestDockerHost("docker-host", "tower.local", "docker-runtime-1", "agent-1")
dockerResource.Status = StatusOffline
dockerResource.LastSeen = now
rr.IngestRecords(SourceAgent, []IngestRecord{
{
SourceID: "agent-host",
Resource: agentResource,
},
})
rr.IngestRecords(SourceDocker, []IngestRecord{
{
SourceID: "docker-host",
Resource: dockerResource,
},
})
systems := MonitoredSystems(rr)
if len(systems) != 1 {
t.Fatalf("MonitoredSystems() returned %d systems, want 1", len(systems))
}
if systems[0].Status != StatusOffline {
t.Fatalf("MonitoredSystems()[0].Status = %q, want %q", systems[0].Status, StatusOffline)
}
}
func TestResourceRegistry_IngestRecords_UnknownSource(t *testing.T) {
rr := NewRegistry(nil)
now := time.Date(2026, 2, 20, 12, 0, 0, 0, time.UTC)
customSource := DataSource("xcp")
rr.IngestRecords(customSource, []IngestRecord{
{
SourceID: "host-1",
Resource: Resource{
Type: ResourceTypeAgent,
Name: "xcp-host-1",
Status: StatusOnline,
LastSeen: now,
},
Identity: ResourceIdentity{Hostnames: []string{"xcp-host-1"}},
},
})
hosts := rr.ListByType(ResourceTypeAgent)
if len(hosts) != 1 {
t.Fatalf("expected 1 host for custom source, got %d", len(hosts))
}
if hosts[0].Name != "xcp-host-1" {
t.Fatalf("expected host name xcp-host-1, got %q", hosts[0].Name)
}
targets := rr.SourceTargets(hosts[0].ID)
if len(targets) != 1 {
t.Fatalf("expected 1 source target, got %d", len(targets))
}
if targets[0].Source != customSource {
t.Fatalf("expected custom source %q, got %q", customSource, targets[0].Source)
}
if targets[0].SourceID != "host-1" {
t.Fatalf("expected source ID host-1, got %q", targets[0].SourceID)
}
}
func TestResourceRegistry_IngestRecords_CanonicalizesSourceIDWhitespace(t *testing.T) {
rr := NewRegistry(nil)
now := time.Date(2026, 2, 20, 12, 0, 0, 0, time.UTC)
rr.IngestRecords(SourceProxmox, []IngestRecord{
{
SourceID: " vm-100 ",
Resource: Resource{
Type: ResourceTypeVM,
Name: "vm-100",
Status: StatusOnline,
LastSeen: now,
},
Identity: ResourceIdentity{Hostnames: []string{"vm-100"}},
},
})
got := rr.sourceResourceID(SourceProxmox, "vm-100")
if got == "" {
t.Fatal("expected trimmed source ID to resolve canonical resource ID")
}
gotTrimmedVariant := rr.sourceResourceID(SourceProxmox, " vm-100 ")
if gotTrimmedVariant != got {
t.Fatalf("expected whitespace variants to resolve same resource ID, got %q want %q", gotTrimmedVariant, got)
}
targets := rr.SourceTargets(got)
if len(targets) != 1 {
t.Fatalf("expected 1 source target, got %d", len(targets))
}
if targets[0].SourceID != "vm-100" {
t.Fatalf("expected canonical trimmed source ID, got %q", targets[0].SourceID)
}
if targets[0].CandidateID != rr.sourceSpecificID(ResourceTypeVM, SourceProxmox, "vm-100") {
t.Fatalf("expected candidate ID to use canonical trimmed source ID, got %q", targets[0].CandidateID)
}
}
func TestResourceRegistry_IngestSnapshotUnifiesLinkedProxmoxNodeViewsByHostIdentity(t *testing.T) {
rr := NewRegistry(nil)
now := time.Date(2026, 3, 7, 12, 0, 0, 0, time.UTC)
rr.IngestSnapshot(models.StateSnapshot{
Nodes: []models.Node{
{
ID: "homelab-minipc",
NodeIdentity: "homelab-minipc",
Name: "minipc",
DisplayName: "Render East",
Instance: "homelab-entry",
ClusterName: "homelab",
IsClusterMember: true,
Host: "https://10.0.0.5:8006",
LinkedAgentID: "host-1",
Status: "online",
LastSeen: now,
},
{
ID: "standalone-minipc",
Name: "minipc",
Instance: "minipc-standalone",
Host: "https://10.0.0.5:8006",
LinkedAgentID: "host-1",
Status: "online",
LastSeen: now,
},
},
Hosts: []models.Host{
{
ID: "host-1",
Hostname: "minipc.local",
MachineID: "machine-1",
ReportIP: "10.0.0.5",
Status: "online",
LastSeen: now,
NetworkInterfaces: []models.HostNetworkInterface{
{Name: "eth0", MAC: "00:11:22:33:44:55", Addresses: []string{"10.0.0.5/24"}},
},
},
},
})
agents := rr.ListByType(ResourceTypeAgent)
if len(agents) != 1 {
t.Fatalf("expected 1 unified agent resource, got %d", len(agents))
}
resource := agents[0]
if resource.Identity.MachineID != "machine-1" {
t.Fatalf("MachineID = %q, want machine-1", resource.Identity.MachineID)
}
if !containsDataSource(resource.Sources, SourceAgent) || !containsDataSource(resource.Sources, SourceProxmox) {
t.Fatalf("expected merged agent+proxmox sources, got %+v", resource.Sources)
}
if resource.Proxmox == nil || resource.Proxmox.NodeName != "minipc" {
t.Fatalf("expected proxmox node metadata for minipc, got %+v", resource.Proxmox)
}
if resource.Name != "Render East" || resource.Proxmox.NodeIdentity != "homelab-minipc" ||
resource.Proxmox.NodeDisplayName != "Render East" {
t.Fatalf("linked agent did not retain Proxmox presentation identity: %+v", resource)
}
}
func TestResourceRegistry_IngestSnapshotUnifiesAsymmetricLinkedProxmoxNodeViewsByHostIdentity(t *testing.T) {
rr := NewRegistry(nil)
now := time.Date(2026, 3, 7, 12, 0, 0, 0, time.UTC)
rr.IngestSnapshot(models.StateSnapshot{
Nodes: []models.Node{
{
ID: "homelab-minipc",
Name: "minipc",
Instance: "homelab-entry",
ClusterName: "homelab",
IsClusterMember: true,
Host: "https://10.0.0.5:8006",
LinkedAgentID: "host-1",
Status: "online",
LastSeen: now,
},
{
ID: "homelab-minipc-shadow",
Name: "minipc",
Instance: "homelab-shadow",
ClusterName: "homelab",
IsClusterMember: true,
Host: "https://10.0.0.5:8006",
Status: "online",
LastSeen: now.Add(-time.Minute),
},
},
Hosts: []models.Host{
{
ID: "host-1",
Hostname: "minipc.local",
MachineID: "machine-1",
ReportIP: "10.0.0.5",
Status: "online",
LastSeen: now,
LinkedNodeID: "homelab-minipc",
NetworkInterfaces: []models.HostNetworkInterface{
{Name: "eth0", MAC: "00:11:22:33:44:55", Addresses: []string{"10.0.0.5/24"}},
},
},
},
})
agents := rr.ListByType(ResourceTypeAgent)
if len(agents) != 1 {
t.Fatalf("expected 1 unified agent resource, got %d", len(agents))
}
resource := agents[0]
if resource.Identity.MachineID != "machine-1" {
t.Fatalf("MachineID = %q, want machine-1", resource.Identity.MachineID)
}
if resource.Proxmox == nil {
t.Fatalf("expected proxmox metadata")
}
if got := resource.Proxmox.LinkedAgentID; got != "host-1" {
t.Fatalf("LinkedAgentID = %q, want host-1", got)
}
targets := rr.SourceTargets(resource.ID)
if len(targets) != 3 {
t.Fatalf("expected 3 source targets (2 proxmox + 1 agent), got %d", len(targets))
}
}
func TestResourceRegistry_IngestSnapshotUnifiesHostLinkedProxmoxNodeViewsByHostIdentity(t *testing.T) {
rr := NewRegistry(nil)
now := time.Date(2026, 3, 7, 12, 0, 0, 0, time.UTC)
rr.IngestSnapshot(models.StateSnapshot{
Nodes: []models.Node{
{
ID: "homelab-delly",
Name: "delly",
Instance: "homelab-entry",
ClusterName: "homelab",
IsClusterMember: true,
Host: "https://10.0.0.9:8006",
Status: "online",
LastSeen: now,
},
{
ID: "homelab-delly-shadow",
Name: "delly",
Instance: "homelab-shadow",
ClusterName: "homelab",
IsClusterMember: true,
Host: "https://10.0.0.9:8006",
Status: "online",
LastSeen: now.Add(-time.Minute),
},
},
Hosts: []models.Host{
{
ID: "host-1",
Hostname: "delly.local",
MachineID: "machine-delly",
ReportIP: "10.0.0.9",
Status: "online",
LastSeen: now,
LinkedNodeID: "homelab-delly",
NetworkInterfaces: []models.HostNetworkInterface{
{Name: "eth0", MAC: "00:11:22:33:44:66", Addresses: []string{"10.0.0.9/24"}},
},
},
},
})
agents := rr.ListByType(ResourceTypeAgent)
if len(agents) != 1 {
t.Fatalf("expected 1 unified agent resource, got %d", len(agents))
}
resource := agents[0]
if resource.Identity.MachineID != "machine-delly" {
t.Fatalf("MachineID = %q, want machine-delly", resource.Identity.MachineID)
}
if resource.Proxmox == nil {
t.Fatalf("expected proxmox metadata")
}
if got := resource.Proxmox.LinkedAgentID; got != "host-1" {
t.Fatalf("LinkedAgentID = %q, want host-1", got)
}
targets := rr.SourceTargets(resource.ID)
if len(targets) != 3 {
t.Fatalf("expected 3 source targets (2 proxmox + 1 agent), got %d", len(targets))
}
}
func TestResourceRegistry_IngestSnapshotParentsClusterNamedProxmoxGuestsToMergedNode(t *testing.T) {
rr := NewRegistry(nil)
now := time.Date(2026, 5, 14, 10, 0, 0, 0, time.UTC)
rr.IngestSnapshot(models.StateSnapshot{
Nodes: []models.Node{
{
ID: "homelab-delly",
Name: "delly",
Instance: "delly",
ClusterName: "homelab",
IsClusterMember: true,
Host: "https://10.0.0.9:8006",
LinkedAgentID: "host-1",
Status: "online",
LastSeen: now,
},
},
Hosts: []models.Host{
{
ID: "host-1",
Hostname: "delly.local",
MachineID: "machine-delly",
ReportIP: "10.0.0.9",
Status: "online",
LastSeen: now,
LinkedNodeID: "homelab-delly",
NetworkInterfaces: []models.HostNetworkInterface{
{Name: "eth0", MAC: "00:11:22:33:44:66", Addresses: []string{"10.0.0.9/24"}},
},
},
},
VMs: []models.VM{
{
ID: "delly:delly:100",
VMID: 100,
Name: "docker-vm",
Node: "delly",
Instance: "delly",
Status: "running",
LastSeen: now,
},
},
Containers: []models.Container{
{
ID: "delly:delly:104",
VMID: 104,
Name: "cloudflared",
Node: "delly",
Instance: "delly",
Status: "running",
LastSeen: now,
},
},
})
agents := rr.ListByType(ResourceTypeAgent)
if len(agents) != 1 {
t.Fatalf("expected 1 merged delly resource, got %d", len(agents))
}
parentID := agents[0].ID
vms := rr.ListByType(ResourceTypeVM)
if len(vms) != 1 {
t.Fatalf("expected 1 vm, got %d", len(vms))
}
if vms[0].ParentID == nil || *vms[0].ParentID != parentID {
t.Fatalf("expected vm parent %q, got %+v", parentID, vms[0].ParentID)
}
if vms[0].ParentName == "" {
t.Fatal("expected vm parent name to be derived")
}
if vms[0].Proxmox == nil || vms[0].Proxmox.LinkedAgentID != "host-1" {
t.Fatalf("expected vm linked agent host-1, got %+v", vms[0].Proxmox)
}
containers := rr.ListByType(ResourceTypeSystemContainer)
if len(containers) != 1 {
t.Fatalf("expected 1 container, got %d", len(containers))
}
if containers[0].ParentID == nil || *containers[0].ParentID != parentID {
t.Fatalf("expected container parent %q, got %+v", parentID, containers[0].ParentID)
}
if containers[0].Proxmox == nil || containers[0].Proxmox.LinkedAgentID != "host-1" {
t.Fatalf("expected container linked agent host-1, got %+v", containers[0].Proxmox)
}
parent, ok := rr.Get(parentID)
if !ok {
t.Fatalf("expected parent %q", parentID)
}
if parent.ChildCount != 2 {
t.Fatalf("expected parent child count 2, got %d", parent.ChildCount)
}
}
func TestResourceRegistry_MergedNodeAgentMetricsTargetMatchesAgentStoreWriteKey(t *testing.T) {
// Real-mode agent metrics land in the metrics store keyed by host.ID
// (monitor_agents.go: m.metricsStore.Write("agent", host.ID, ...), the
// write key is pinned by a canonical guardrail in internal/monitoring).
// The registry's metrics target for the merged node+agent resource must
// resolve to that same key, or every store reader that trusts the
// target — performance reports via MetricsResourceID, the drawer
// history endpoint — queries an ID with zero rows.
rr := NewRegistry(nil)
now := time.Date(2026, 6, 10, 10, 0, 0, 0, time.UTC)
rr.IngestSnapshot(models.StateSnapshot{
Nodes: []models.Node{
{
ID: "homelab-delly",
Name: "delly",
Instance: "delly",
ClusterName: "homelab",
IsClusterMember: true,
Host: "https://10.0.0.9:8006",
LinkedAgentID: "host-1",
Status: "online",
LastSeen: now,
},
},
Hosts: []models.Host{
{
ID: "host-1",
Hostname: "delly.local",
MachineID: "machine-delly",
ReportIP: "10.0.0.9",
Status: "online",
LastSeen: now,
LinkedNodeID: "homelab-delly",
NetworkInterfaces: []models.HostNetworkInterface{
{Name: "eth0", MAC: "00:11:22:33:44:66", Addresses: []string{"10.0.0.9/24"}},
},
},
},
})
agents := rr.ListByType(ResourceTypeAgent)
if len(agents) != 1 {
t.Fatalf("expected 1 merged delly resource, got %d", len(agents))
}
resource := agents[0]
targets := rr.SourceTargets(resource.ID)
if len(targets) != 2 {
t.Fatalf("expected merged proxmox+agent source targets, got %+v", targets)
}
target := BuildMetricsTargetForRegistry(rr, resource.ID)
if target == nil {
t.Fatal("BuildMetricsTargetForRegistry() returned nil")
}
if target.ResourceType != "agent" {
t.Fatalf("ResourceType = %q, want agent", target.ResourceType)
}
if target.ResourceID != "host-1" {
t.Fatalf("ResourceID = %q, want host-1 (the host.ID key the agent metrics writer uses)", target.ResourceID)
}
}
func TestResourceRegistry_ManualNodeMergeRewritesProxmoxGuestParentAndActionAgent(t *testing.T) {
store := NewMemoryStore()
if err := store.AddLink(ResourceLink{
ResourceA: "agent-old-proxmox-node",
ResourceB: "agent-current",
PrimaryID: "agent-current",
}); err != nil {
t.Fatalf("add link: %v", err)
}
rr := NewRegistry(store)
now := time.Date(2026, 5, 14, 10, 0, 0, 0, time.UTC)
staleParentID := "agent-old-proxmox-node"
rr.IngestResources([]Resource{
{
ID: staleParentID,
Type: ResourceTypeAgent,
Name: "delly",
Status: "online",
LastSeen: now,
Sources: []DataSource{SourceProxmox},
SourceStatus: map[DataSource]SourceStatus{
SourceProxmox: {Status: "online", LastSeen: now},
},
Proxmox: &ProxmoxData{
SourceID: "homelab-delly",
NodeName: "delly",
ClusterName: "homelab",
Instance: "delly",
},
},
{
ID: "agent-current",
Type: ResourceTypeAgent,
Name: "delly",
Status: "online",
LastSeen: now,
Sources: []DataSource{SourceAgent},
SourceStatus: map[DataSource]SourceStatus{
SourceAgent: {Status: "online", LastSeen: now},
},
Agent: &AgentData{
AgentID: "agent-delly",
Hostname: "delly",
},
},
{
ID: "system-container-grafana",
Type: ResourceTypeSystemContainer,
Name: "grafana",
Status: "online",
LastSeen: now,
ParentID: &staleParentID,
Sources: []DataSource{SourceProxmox},
SourceStatus: map[DataSource]SourceStatus{
SourceProxmox: {Status: "online", LastSeen: now},
},
Proxmox: &ProxmoxData{
SourceID: "delly:delly:124",
NodeName: "delly",
ClusterName: "homelab",
Instance: "delly",
VMID: 124,
},
},
})
containers := rr.ListByType(ResourceTypeSystemContainer)
if len(containers) != 1 {
t.Fatalf("expected 1 container, got %d", len(containers))
}
if containers[0].ParentID == nil || *containers[0].ParentID != "agent-current" {
t.Fatalf("expected container parent agent-current, got %+v", containers[0].ParentID)
}
if containers[0].ParentName != "delly" {
t.Fatalf("expected container parent name delly, got %q", containers[0].ParentName)
}
if containers[0].Proxmox == nil || containers[0].Proxmox.LinkedAgentID != "agent-delly" {
t.Fatalf("expected container linked agent agent-delly, got %+v", containers[0].Proxmox)
}
}
// TestRegistryWithManualLinksFoldsLikeTheStoreBackedRegistry pins the
// store-less registry the mock-mode unified view seeds: given the links an
// adapter generation loaded, it folds already-unified resources exactly as a
// registry backed by the link store does. The adapter reports the links of its
// current generation and picks up a new one on its next rebuild.
func TestRegistryWithManualLinksFoldsLikeTheStoreBackedRegistry(t *testing.T) {
now := time.Date(2026, 10, 7, 12, 0, 0, 0, time.UTC)
resources := []Resource{
{
ID: "vm-checkout",
Type: ResourceTypeVM,
Name: "checkout-web-01",
Status: StatusOnline,
LastSeen: now,
Sources: []DataSource{SourceProxmox},
Proxmox: &ProxmoxData{SourceID: "pve:node1:101", NodeName: "node1", Instance: "pve", VMID: 101},
},
{
ID: "agent-apollo",
Type: ResourceTypeAgent,
Name: "apollo-114",
Status: StatusOnline,
LastSeen: now,
Sources: []DataSource{SourceAgent},
Agent: &AgentData{AgentID: "agent-apollo", Hostname: "apollo-114"},
},
}
link := ResourceLink{ResourceA: "vm-checkout", ResourceB: "agent-apollo", PrimaryID: "vm-checkout"}
store := NewMemoryStore()
adapter := NewMonitorAdapter(NewRegistry(store))
if err := store.AddLink(link); err != nil {
t.Fatalf("AddLink: %v", err)
}
if got := adapter.ManualLinks(); len(got) != 0 {
t.Fatalf("ManualLinks before a rebuild = %+v, want the generation's empty set", got)
}
adapter.PopulateFromSnapshot(models.StateSnapshot{})
links := adapter.ManualLinks()
if len(links) != 1 || links[0].ResourceA != link.ResourceA || links[0].ResourceB != link.ResourceB || links[0].PrimaryID != link.PrimaryID {
t.Fatalf("ManualLinks after a rebuild = %+v, want %+v", links, link)
}
links[0].PrimaryID = "agent-apollo"
if got := adapter.ManualLinks(); got[0].PrimaryID != "vm-checkout" {
t.Fatalf("ManualLinks handed out the registry's own slice: %+v", got)
}
links[0].PrimaryID = "vm-checkout"
storeBacked := NewRegistry(store)
storeBacked.IngestResources(resources)
storeLess := NewRegistryWithManualLinks(links)
// The registry keeps its own copy of the caller's links.
links[0].ResourceB = "agent-other"
storeLess.IngestResources(resources)
for name, rr := range map[string]*ResourceRegistry{"store-backed": storeBacked, "store-less": storeLess} {
got := rr.List()
if len(got) != 1 || got[0].ID != "vm-checkout" || got[0].Agent == nil || got[0].Proxmox == nil {
t.Fatalf("%s registry listed %v, want the VM carrying the linked agent", name, resourceIDs(got))
}
}
}
func TestResourceRegistry_ProxmoxGuestParentPrefersAgentBackedNodeDuplicate(t *testing.T) {
rr := NewRegistry(nil)
now := time.Date(2026, 5, 14, 10, 0, 0, 0, time.UTC)
rr.IngestResources([]Resource{
{
ID: "agent-current",
Type: ResourceTypeAgent,
Name: "delly",
Status: "online",
LastSeen: now,
Sources: []DataSource{SourceAgent, SourceProxmox},
SourceStatus: map[DataSource]SourceStatus{
SourceAgent: {Status: "online", LastSeen: now},
SourceProxmox: {Status: "online", LastSeen: now},
},
Agent: &AgentData{
AgentID: "agent-delly",
Hostname: "delly",
},
Proxmox: &ProxmoxData{
SourceID: "homelab-delly",
NodeName: "delly",
ClusterName: "homelab",
Instance: "delly",
},
},
{
ID: "agent-old-proxmox-node",
Type: ResourceTypeAgent,
Name: "delly",
Status: "online",
LastSeen: now,
Sources: []DataSource{SourceProxmox},
SourceStatus: map[DataSource]SourceStatus{
SourceProxmox: {Status: "online", LastSeen: now},
},
Proxmox: &ProxmoxData{
SourceID: "homelab-delly",
NodeName: "delly",
ClusterName: "homelab",
Instance: "delly",
},
},
})
rr.IngestRecords(SourceProxmox, []IngestRecord{
{
SourceID: "delly:delly:124",
ParentSourceID: "homelab-delly",
Resource: Resource{
Type: ResourceTypeSystemContainer,
Name: "grafana",
Status: "online",
LastSeen: now,
Proxmox: &ProxmoxData{
SourceID: "delly:delly:124",
NodeName: "delly",
ClusterName: "homelab",
Instance: "delly",
VMID: 124,
},
},
Identity: ResourceIdentity{
Hostnames: []string{"grafana"},
},
},
})
containers := rr.ListByType(ResourceTypeSystemContainer)
if len(containers) != 1 {
t.Fatalf("expected 1 container, got %d", len(containers))
}
if containers[0].ParentID == nil || *containers[0].ParentID != "agent-current" {
t.Fatalf("expected container parent agent-current, got %+v", containers[0].ParentID)
}
if containers[0].Proxmox == nil || containers[0].Proxmox.LinkedAgentID != "agent-delly" {
t.Fatalf("expected container linked agent agent-delly, got %+v", containers[0].Proxmox)
}
}
func TestResourceRegistry_IngestSnapshotDerivesProxmoxWorkloadParentWhenNodeSourceIDUsesPreviousClusterAlias(t *testing.T) {
rr := NewRegistry(nil)
now := time.Date(2026, 5, 14, 10, 0, 0, 0, time.UTC)
rr.IngestSnapshot(models.StateSnapshot{
Nodes: []models.Node{
{
ID: "mock-cluster-pve1",
NodeIdentity: "mock-cluster-pve1",
Name: "pve1",
DisplayName: "West Production A",
Instance: "Core Fabric",
ClusterName: "Core Fabric",
IsClusterMember: true,
Status: "online",
LastSeen: now,
},
},
VMs: []models.VM{
{
ID: "Core Fabric:pve1:100",
VMID: 100,
Name: "checkout-web-01",
Node: "pve1",
Instance: "Core Fabric",
Status: "running",
LastSeen: now,
},
},
Containers: []models.Container{
{
ID: "Core Fabric:pve1:104",
VMID: 104,
Name: "auth-service-01",
Node: "pve1",
Instance: "Core Fabric",
Status: "running",
LastSeen: now,
},
},
})
agents := rr.ListByType(ResourceTypeAgent)
if len(agents) != 1 {
t.Fatalf("expected 1 Proxmox parent resource, got %d", len(agents))
}
parentID := agents[0].ID
vms := rr.ListByType(ResourceTypeVM)
if len(vms) != 1 {
t.Fatalf("expected 1 vm, got %d", len(vms))
}
if vms[0].ParentID == nil || *vms[0].ParentID != parentID {
t.Fatalf("expected vm parent %q from node metadata fallback, got %+v", parentID, vms[0].ParentID)
}
if vms[0].ParentName != "West Production A" {
t.Fatalf("expected vm parent name West Production A, got %q", vms[0].ParentName)
}
if vms[0].Proxmox == nil || vms[0].Proxmox.NodeIdentity != "mock-cluster-pve1" ||
vms[0].Proxmox.NodeDisplayName != "West Production A" ||
vms[0].Proxmox.NodeName != "pve1" {
t.Fatalf("expected VM presentation metadata without changing native name, got %+v", vms[0].Proxmox)
}
containers := rr.ListByType(ResourceTypeSystemContainer)
if len(containers) != 1 {
t.Fatalf("expected 1 container, got %d", len(containers))
}
if containers[0].ParentID == nil || *containers[0].ParentID != parentID {
t.Fatalf("expected container parent %q from node metadata fallback, got %+v", parentID, containers[0].ParentID)
}
parent, ok := rr.Get(parentID)
if !ok {
t.Fatalf("expected parent %q", parentID)
}
if parent.ChildCount != 2 {
t.Fatalf("expected parent child count 2, got %d", parent.ChildCount)
}
}
func TestResourceRegistry_IngestSnapshotUnifiesHostLinkedProxmoxNodeViewsAcrossEndpointForms(t *testing.T) {
rr := NewRegistry(nil)
now := time.Date(2026, 3, 7, 12, 0, 0, 0, time.UTC)
rr.IngestSnapshot(models.StateSnapshot{
Nodes: []models.Node{
{
ID: "minipc-ip-view",
Name: "minipc",
Instance: "standalone-ip",
Host: "https://10.0.0.5:8006",
Status: "online",
LastSeen: now,
},
{
ID: "minipc-hostname-view",
Name: "minipc",
Instance: "standalone-hostname",
Host: "https://minipc.local:8006",
Status: "online",
LastSeen: now.Add(-time.Minute),
},
},
Hosts: []models.Host{
{
ID: "host-1",
Hostname: "minipc.local",
MachineID: "machine-minipc",
ReportIP: "10.0.0.5",
Status: "online",
LastSeen: now,
LinkedNodeID: "minipc-ip-view",
NetworkInterfaces: []models.HostNetworkInterface{
{Name: "eth0", MAC: "00:11:22:33:44:55", Addresses: []string{"10.0.0.5/24"}},
},
},
},
})
agents := rr.ListByType(ResourceTypeAgent)
if len(agents) != 1 {
t.Fatalf("expected 1 unified agent resource, got %d", len(agents))
}
resource := agents[0]
if resource.Identity.MachineID != "machine-minipc" {
t.Fatalf("MachineID = %q, want machine-minipc", resource.Identity.MachineID)
}
if resource.Proxmox == nil {
t.Fatalf("expected proxmox metadata")
}
targets := rr.SourceTargets(resource.ID)
if len(targets) != 3 {
t.Fatalf("expected 3 source targets (2 proxmox + 1 agent), got %d", len(targets))
}
}
func TestResourceRegistry_IngestSnapshotCreatesPhysicalDisksFromHostSMART(t *testing.T) {
rr := NewRegistry(nil)
now := time.Date(2026, 3, 7, 12, 0, 0, 0, time.UTC)
rr.IngestSnapshot(models.StateSnapshot{
Hosts: []models.Host{
{
ID: "host-tower",
Hostname: "tower",
Status: "online",
LastSeen: now,
Disks: []models.Disk{
{Device: "/dev/sdb", Total: 12 * 1024, Mountpoint: "/mnt/disk1"},
},
Sensors: models.HostSensorSummary{
SMART: []models.HostDiskSMART{
{
Device: "/dev/sdb",
Model: "Seagate IronWolf",
Serial: "SERIAL-TOWER-1",
Type: "sata",
Temperature: 37,
Health: "PASSED",
},
},
},
},
},
})
disks := rr.ListByType(ResourceTypePhysicalDisk)
if len(disks) != 1 {
t.Fatalf("expected 1 physical disk resource, got %d", len(disks))
}
disk := disks[0]
if !containsDataSource(disk.Sources, SourceAgent) {
t.Fatalf("expected agent-backed physical disk source, got %+v", disk.Sources)
}
if disk.ParentID == nil {
t.Fatalf("expected host parent on physical disk")
}
parent, ok := rr.Get(*disk.ParentID)
if !ok || parent == nil || parent.Name != "tower" {
t.Fatalf("expected disk parent to resolve to tower host, got %+v", parent)
}
if disk.PhysicalDisk == nil || disk.PhysicalDisk.Serial != "SERIAL-TOWER-1" {
t.Fatalf("expected SMART-backed disk metadata, got %+v", disk.PhysicalDisk)
}
if disk.Identity.MachineID != "SERIAL-TOWER-1" {
t.Fatalf("MachineID = %q, want SERIAL-TOWER-1", disk.Identity.MachineID)
}
if disk.PhysicalDisk.Risk != nil {
t.Fatalf("expected healthy disk to have no risk payload, got %+v", disk.PhysicalDisk.Risk)
}
}
func TestResourceRegistry_IngestSnapshotSkipsVirtualBlockDevicesFromHostSMART(t *testing.T) {
rr := NewRegistry(nil)
now := time.Date(2026, 4, 23, 12, 0, 0, 0, time.UTC)
rr.IngestSnapshot(models.StateSnapshot{
Hosts: []models.Host{
{
ID: "host-minipc",
Hostname: "minipc",
Status: "online",
LastSeen: now,
Sensors: models.HostSensorSummary{
SMART: []models.HostDiskSMART{
{Device: "/dev/nvme0n1", Model: "Samsung 970 EVO", Serial: "REAL-DISK", Type: "nvme", Health: "PASSED"},
{Device: "/dev/zd0", Model: "", Serial: "", Type: ""},
{Device: "/dev/zd16", Model: "", Serial: "", Type: ""},
{Device: "zram0", Model: "", Serial: "", Type: ""},
{Device: "/dev/loop3", Model: "", Serial: "", Type: ""},
{Device: "/dev/dm-1", Model: "", Serial: "", Type: ""},
},
},
},
},
})
disks := rr.ListByType(ResourceTypePhysicalDisk)
if len(disks) != 1 {
t.Fatalf("expected 1 physical disk resource (virtual devices filtered), got %d", len(disks))
}
if disks[0].PhysicalDisk == nil || disks[0].PhysicalDisk.Serial != "REAL-DISK" {
t.Fatalf("expected only the real nvme disk, got %+v", disks[0].PhysicalDisk)
}
}
func TestResourceRegistry_IngestSnapshotSkipsOpticalProxmoxDisks(t *testing.T) {
rr := NewRegistry(nil)
rr.IngestSnapshot(models.StateSnapshot{
PhysicalDisks: []models.PhysicalDisk{
{ID: "pve-disk", Instance: "pve-main", Node: "pve-1", DevPath: "/dev/sda", Serial: "REAL-DISK"},
{ID: "pve-optical", Instance: "pve-main", Node: "pve-1", DevPath: "/dev/sr0", Model: "DVD-ROM", Type: "sata"},
},
})
disks := rr.ListByType(ResourceTypePhysicalDisk)
if len(disks) != 1 {
t.Fatalf("expected only the real Proxmox disk, got %d: %+v", len(disks), disks)
}
if disks[0].PhysicalDisk == nil || disks[0].PhysicalDisk.Serial != "REAL-DISK" {
t.Fatalf("expected only the real Proxmox disk, got %+v", disks[0].PhysicalDisk)
}
}
func TestResourceRegistry_IngestSnapshotMergesAgentAndProxmoxPhysicalDisksByIdentity(t *testing.T) {
rr := NewRegistry(nil)
now := time.Date(2026, 3, 7, 12, 0, 0, 0, time.UTC)
rr.IngestSnapshot(models.StateSnapshot{
PhysicalDisks: []models.PhysicalDisk{
{
ID: "pve-disk-1",
Node: "tower",
Instance: "pve-tower",
DevPath: "/dev/sdb",
Model: "Seagate IronWolf",
Serial: "SERIAL-TOWER-1",
Type: "sata",
Health: "PASSED",
Temperature: 34,
LastChecked: now,
},
},
Hosts: []models.Host{
{
ID: "host-tower",
Hostname: "tower",
Status: "online",
LastSeen: now,
Sensors: models.HostSensorSummary{
SMART: []models.HostDiskSMART{
{
Device: "/dev/sdb",
Model: "Seagate IronWolf",
Serial: "SERIAL-TOWER-1",
Type: "sata",
Temperature: 37,
Health: "PASSED",
},
},
},
},
},
})
disks := rr.ListByType(ResourceTypePhysicalDisk)
if len(disks) != 1 {
t.Fatalf("expected 1 merged physical disk resource, got %d", len(disks))
}
disk := disks[0]
if !containsDataSource(disk.Sources, SourceAgent) || !containsDataSource(disk.Sources, SourceProxmox) {
t.Fatalf("expected merged proxmox+agent disk sources, got %+v", disk.Sources)
}
if disk.PhysicalDisk == nil || disk.PhysicalDisk.Temperature != 37 {
t.Fatalf("expected merged SMART temperature from agent disk, got %+v", disk.PhysicalDisk)
}
if disk.Proxmox == nil || disk.Proxmox.NodeName != "tower" || disk.Proxmox.Instance != "pve-tower" {
t.Fatalf("expected merged disk to retain Proxmox node identity, got %+v", disk.Proxmox)
}
}
// Regression for issue #1483: a legacy host agent reports an NVMe disk keyed by
// its controller ("nvme0 [nvme]") with no size, while the Proxmox disks/list poll
// reports the canonical namespace ("/dev/nvme0n1") with the real capacity. The
// merge must keep the authoritative Proxmox devPath and size, and only enrich
// with the agent's SMART temperature.
func TestResourceRegistry_AgentDiskDoesNotClobberProxmoxDevPath(t *testing.T) {
rr := NewRegistry(nil)
now := time.Date(2026, 5, 29, 12, 0, 0, 0, time.UTC)
const nvmeSize = int64(2000398934016)
rr.IngestSnapshot(models.StateSnapshot{
PhysicalDisks: []models.PhysicalDisk{
{
ID: "pve-disk-1",
Node: "pve",
Instance: "pve",
DevPath: "/dev/nvme0n1",
Model: "KINGSTON SNV3S2000G",
Serial: "SERIAL-NVME-0",
Type: "nvme",
Size: nvmeSize,
Health: "PASSED",
LastChecked: now,
},
},
Hosts: []models.Host{
{
ID: "host-pve",
Hostname: "pve",
Status: "online",
LastSeen: now,
Sensors: models.HostSensorSummary{
SMART: []models.HostDiskSMART{
{
Device: "nvme0 [nvme]", // legacy controller label, no size
Model: "KINGSTON SNV3S2000G",
Serial: "SERIAL-NVME-0",
Type: "nvme",
Temperature: 37,
Health: "PASSED",
},
},
},
},
},
})
disks := rr.ListByType(ResourceTypePhysicalDisk)
if len(disks) != 1 {
t.Fatalf("expected 1 merged physical disk resource, got %d", len(disks))
}
disk := disks[0]
if disk.PhysicalDisk == nil {
t.Fatal("expected physical disk metadata")
}
if disk.PhysicalDisk.DevPath != "/dev/nvme0n1" {
t.Fatalf("devPath = %q, want canonical /dev/nvme0n1 (agent label must not clobber)", disk.PhysicalDisk.DevPath)
}
if disk.PhysicalDisk.SizeBytes != nvmeSize {
t.Fatalf("sizeBytes = %d, want authoritative Proxmox capacity %d", disk.PhysicalDisk.SizeBytes, nvmeSize)
}
if disk.PhysicalDisk.Temperature != 37 {
t.Fatalf("temperature = %d, want enriched agent SMART value 37", disk.PhysicalDisk.Temperature)
}
}
func TestShouldReplacePhysicalDiskDevPath(t *testing.T) {
cases := []struct {
name string
existing string
incoming string
want bool
}{
{"canonical not downgraded by scan label", "/dev/nvme0n1", "nvme0 [nvme]", false},
{"canonical not downgraded by bare token", "/dev/nvme0n1", "nvme0n1", false},
{"scan label upgraded to canonical", "nvme0 [nvme]", "/dev/nvme0n1", true},
{"empty existing always replaced", "", "nvme0 [nvme]", true},
{"empty incoming never replaces", "/dev/nvme0n1", "", false},
{"both canonical last writer wins", "/dev/sda", "/dev/sdb", true},
}
for _, tc := range cases {
if got := shouldReplacePhysicalDiskDevPath(tc.existing, tc.incoming); got != tc.want {
t.Errorf("%s: shouldReplacePhysicalDiskDevPath(%q, %q) = %v, want %v", tc.name, tc.existing, tc.incoming, got, tc.want)
}
}
}
func TestResourceRegistry_IngestSnapshotPropagatesUnraidDiskRole(t *testing.T) {
rr := NewRegistry(nil)
now := time.Date(2026, 3, 7, 12, 0, 0, 0, time.UTC)
rr.IngestSnapshot(models.StateSnapshot{
Hosts: []models.Host{
{
ID: "host-tower",
Hostname: "tower",
Status: "online",
LastSeen: now,
Unraid: &models.HostUnraidStorage{
ArrayStarted: true,
Disks: []models.HostUnraidDisk{
{Name: "parity", Device: "/dev/sdb", Role: "parity", Status: "online", Serial: "SERIAL-TOWER-1"},
},
},
Sensors: models.HostSensorSummary{
SMART: []models.HostDiskSMART{
{
Device: "/dev/sdb",
Model: "Seagate IronWolf",
Serial: "SERIAL-TOWER-1",
Type: "sata",
Temperature: 37,
Health: "PASSED",
},
},
},
},
},
})
disks := rr.ListByType(ResourceTypePhysicalDisk)
if len(disks) != 1 {
t.Fatalf("expected 1 physical disk resource, got %d", len(disks))
}
if disks[0].PhysicalDisk == nil {
t.Fatal("expected physical disk metadata")
}
if got := disks[0].PhysicalDisk.StorageRole; got != "parity" {
t.Fatalf("storageRole = %q, want parity", got)
}
if got := disks[0].PhysicalDisk.StorageGroup; got != "unraid-array" {
t.Fatalf("storageGroup = %q, want unraid-array", got)
}
}
func TestResourceRegistry_IngestSnapshotCreatesUnraidDisksWithoutSMART(t *testing.T) {
rr := NewRegistry(nil)
now := time.Date(2026, 3, 7, 12, 0, 0, 0, time.UTC)
rr.IngestSnapshot(models.StateSnapshot{
Hosts: []models.Host{
{
ID: "host-tower",
Hostname: "tower",
Status: "online",
LastSeen: now,
Disks: []models.Disk{
{Mountpoint: "/mnt/user", Total: 10_000, Used: 7_000, Free: 3_000, Usage: 70},
},
Unraid: &models.HostUnraidStorage{
ArrayStarted: true,
ArrayState: "STARTED",
Disks: []models.HostUnraidDisk{
{
Name: "disk1",
Device: "/dev/sdb",
Role: "data",
Status: "online",
Model: "WDC WD60EFRX",
Serial: "DATA-1",
Filesystem: "xfs",
Transport: "sata",
SizeBytes: 6_000_000_000_000,
UsedBytes: 4_000,
FreeBytes: 2_000,
Temperature: 31,
SpunDown: true,
ReadCount: 11,
WriteCount: 12,
ErrorCount: 16,
},
{
Name: "cachepool",
Device: "/dev/sdc",
Role: "cache",
Status: "online",
Model: "SSD 2000GB",
Serial: "CACHE-1",
Filesystem: "btrfs",
Transport: "sata",
SizeBytes: 2_000_000_000_000,
UsedBytes: 200,
FreeBytes: 800,
},
},
},
},
},
})
disks := rr.ListByType(ResourceTypePhysicalDisk)
if len(disks) != 2 {
t.Fatalf("expected 2 Unraid physical disks, got %d: %+v", len(disks), disks)
}
bySerial := map[string]Resource{}
for _, disk := range disks {
if disk.PhysicalDisk != nil {
bySerial[disk.PhysicalDisk.Serial] = disk
}
}
data := bySerial["DATA-1"]
if data.PhysicalDisk == nil || data.PhysicalDisk.StorageRole != "data" || data.PhysicalDisk.StorageGroup != "unraid-array" {
t.Fatalf("expected data disk to belong to unraid array, got %+v", data.PhysicalDisk)
}
if data.PhysicalDisk.SizeBytes != 6_000_000_000_000 || data.PhysicalDisk.Temperature != 31 {
t.Fatalf("expected Unraid size/temp on data disk, got %+v", data.PhysicalDisk)
}
if !data.PhysicalDisk.SpunDown || data.PhysicalDisk.ReadCount != 11 || data.PhysicalDisk.WriteCount != 12 || data.PhysicalDisk.ErrorCount != 16 {
t.Fatalf("expected Unraid disk counters on data disk, got %+v", data.PhysicalDisk)
}
cache := bySerial["CACHE-1"]
if cache.PhysicalDisk == nil || cache.PhysicalDisk.StorageRole != "cache" || cache.PhysicalDisk.StorageGroup != "cachepool" {
t.Fatalf("expected cache disk to belong to cachepool, got %+v", cache.PhysicalDisk)
}
storage := rr.ListByType(ResourceTypeStorage)
var sawArray, sawCache bool
for _, resource := range storage {
if resource.Storage == nil {
continue
}
switch resource.Storage.Type {
case "unraid-array":
sawArray = true
case "unraid-cache-pool":
sawCache = true
if resource.Name != "cachepool" || resource.Metrics == nil || resource.Metrics.Disk == nil || resource.Metrics.Disk.Percent != 20 {
t.Fatalf("unexpected cache storage resource: %+v", resource)
}
}
}
if !sawArray || !sawCache {
t.Fatalf("expected array and cache storage resources, got %+v", storage)
}
}
func TestHostDiskTemperatureReadingsMatchTheDisksTheRegistryShows(t *testing.T) {
now := time.Date(2026, 10, 7, 12, 0, 0, 0, time.UTC)
host := models.Host{
ID: "host-tower",
Hostname: "tower",
Status: "online",
LastSeen: now,
Sensors: models.HostSensorSummary{
SMART: []models.HostDiskSMART{
// smartctl read this disk itself.
{Device: "sda", Model: "WDC A", Serial: "SER-A", Type: "sata", Temperature: 38},
// smartctl returned no temperature; Unraid read one.
{Device: "sdb", Model: "WDC B", Serial: "SER-B", Type: "sata"},
// smartctl and Unraid name this device with different serials.
{Device: "sde", Model: "Bridge E", Serial: "SER-E1", Type: "sata", Temperature: 41},
// The registry ingests no virtual device.
{Device: "zram0", Temperature: 70},
// A controller member Unraid lists under its block device.
{Device: "0 [megaraid,0]", Model: "Seagate F", Serial: "SER-F", Type: "sata", Controller: "0", Target: "megaraid,0", Temperature: 44},
},
},
Unraid: &models.HostUnraidStorage{
ArrayStarted: true,
ArrayState: "STARTED",
Disks: []models.HostUnraidDisk{
{Name: "disk1", Device: "/dev/sda", Role: "data", Status: "online", Serial: "SER-A", Transport: "sata", Temperature: 36},
{Name: "disk2", Device: "/dev/sdb", Role: "data", Status: "online", Serial: "SER-B", Transport: "sata", Temperature: 47},
// No SMART row, as for a --disk-exclude match.
{Name: "disk3", Device: "/dev/nvme0n1", Role: "data", Status: "online", Model: "Samsung C", Serial: "SER-C", Transport: "nvme", Temperature: 52},
{Name: "disk4", Device: "/dev/sdd", Role: "data", Status: "online", Model: "WDC D", Serial: "SER-D", Transport: "sata", Temperature: 30, SpunDown: true},
{Name: "disk5", Device: "/dev/sde", Role: "data", Status: "online", Model: "WDC E", Serial: "SER-E2", Transport: "sata", Temperature: 43},
{Name: "disk7", Device: "/dev/sdf", Role: "data", Status: "online", Model: "Seagate F", Serial: "SER-F", Transport: "sata", Temperature: 44},
// An empty slot: the registry ingests no disk for it.
{Name: "disk6", Role: "data", Status: "missing"},
},
},
}
type want struct {
device string
diskType string
temperature int
collected bool
unraidOnly bool
}
wants := map[string]want{
"SER-A": {device: "sda", diskType: "sata", temperature: 38, collected: true},
"SER-B": {device: "sdb", diskType: "sata", temperature: 47, collected: true},
"SER-E1": {device: "sde", diskType: "sata", temperature: 41, collected: true},
"SER-F": {device: "0 [megaraid,0]", diskType: "sata", temperature: 44, collected: true},
"SER-C": {device: "nvme0n1", diskType: "nvme", temperature: 52, collected: true, unraidOnly: true},
"SER-D": {device: "sdd", diskType: "sata", temperature: 30, unraidOnly: true},
"SER-E2": {device: "sde", diskType: "sata", temperature: 43, collected: true, unraidOnly: true},
}
readings := HostDiskTemperatureReadings(host)
if len(readings) != len(wants) {
t.Fatalf("expected %d readings, got %d: %+v", len(wants), len(readings), readings)
}
byKey := make(map[string]HostDiskTemperatureReading, len(readings))
for _, reading := range readings {
w, ok := wants[reading.MetricID]
if !ok {
t.Fatalf("unexpected reading %+v", reading)
}
if reading.Device != w.device || reading.DiskType != w.diskType || reading.Temperature != w.temperature ||
reading.Collected() != w.collected || reading.UnraidOnly != w.unraidOnly {
t.Fatalf("reading for %s = %+v (collected %v), want %+v", reading.MetricID, reading, reading.Collected(), w)
}
byKey[reading.MetricID] = reading
}
// The readings are the ones the disks the registry builds show, one each.
rr := NewRegistry(nil)
rr.IngestSnapshot(models.StateSnapshot{Hosts: []models.Host{host}})
disks := rr.ListByType(ResourceTypePhysicalDisk)
if len(disks) != len(wants) {
t.Fatalf("expected %d physical disks, got %d: %+v", len(wants), len(disks), disks)
}
for _, disk := range disks {
meta := disk.PhysicalDisk
if meta == nil {
t.Fatalf("physical disk without meta: %+v", disk)
}
reading, ok := byKey[meta.Serial]
if !ok {
t.Fatalf("registry disk %q has no reading", meta.Serial)
}
if meta.Temperature != reading.Temperature || meta.DiskType != reading.DiskType ||
diskinventory.TemperatureCollected(meta.Temperature, meta.Collection) != diskinventory.TemperatureCollected(reading.Temperature, reading.Collection) {
t.Fatalf("registry disk %q shows %dC %q (collection %+v), reading %+v", meta.Serial, meta.Temperature, meta.DiskType, meta.Collection, reading)
}
}
}
func TestResourceRegistry_IngestSnapshotCreatesUnraidStorageResource(t *testing.T) {
rr := NewRegistry(nil)
now := time.Date(2026, 3, 7, 12, 0, 0, 0, time.UTC)
rr.IngestSnapshot(models.StateSnapshot{
Hosts: []models.Host{
{
ID: "host-tower",
Hostname: "tower",
Status: "online",
LastSeen: now,
MachineID: "machine-tower",
Disks: []models.Disk{
{Mountpoint: "/mnt/user", Total: 1000, Used: 400, Free: 600, Usage: 40},
},
Unraid: &models.HostUnraidStorage{
ArrayStarted: true,
ArrayState: "STARTED",
NumProtected: 1,
Disks: []models.HostUnraidDisk{
{Name: "parity", Role: "parity", Status: "online"},
{Name: "disk1", Role: "data", Status: "online"},
},
},
},
},
})
storage := rr.ListByType(ResourceTypeStorage)
if len(storage) != 1 {
t.Fatalf("expected 1 unraid storage resource, got %d", len(storage))
}
resource := storage[0]
if !containsDataSource(resource.Sources, SourceAgent) {
t.Fatalf("expected agent-backed storage source, got %+v", resource.Sources)
}
if resource.ParentID == nil {
t.Fatalf("expected host parent on unraid storage")
}
if resource.Storage == nil || resource.Storage.Type != "unraid-array" || resource.Storage.Platform != "unraid" {
t.Fatalf("expected unraid storage metadata, got %+v", resource.Storage)
}
if resource.Metrics == nil || resource.Metrics.Disk == nil || resource.Metrics.Disk.Percent != 40 {
t.Fatalf("expected disk metrics from unraid storage, got %+v", resource.Metrics)
}
}
func TestResourceRegistry_IngestSnapshotParentsUnraidArrayDisksUnderStorage(t *testing.T) {
rr := NewRegistry(nil)
now := time.Date(2026, 3, 7, 12, 0, 0, 0, time.UTC)
rr.IngestSnapshot(models.StateSnapshot{
Hosts: []models.Host{
{
ID: "host-tower",
Hostname: "tower",
Status: "online",
LastSeen: now,
MachineID: "machine-tower",
Unraid: &models.HostUnraidStorage{
ArrayStarted: true,
NumProtected: 1,
Disks: []models.HostUnraidDisk{
{Name: "parity", Device: "/dev/sdb", Role: "parity", Status: "online", Serial: "SERIAL-TOWER-1"},
},
},
Sensors: models.HostSensorSummary{
SMART: []models.HostDiskSMART{
{
Device: "/dev/sdb",
Model: "Seagate IronWolf",
Serial: "SERIAL-TOWER-1",
Type: "sata",
Temperature: 37,
Health: "PASSED",
},
},
},
},
},
})
storage := rr.ListByType(ResourceTypeStorage)
disks := rr.ListByType(ResourceTypePhysicalDisk)
if len(storage) != 1 || len(disks) != 1 {
t.Fatalf("expected 1 storage and 1 disk resource, got storage=%d disk=%d", len(storage), len(disks))
}
if disks[0].ParentID == nil || *disks[0].ParentID != storage[0].ID {
t.Fatalf("expected unraid disk parent to be storage resource %q, got %+v", storage[0].ID, disks[0].ParentID)
}
if storage[0].ChildCount != 1 {
t.Fatalf("expected storage child count 1, got %d", storage[0].ChildCount)
}
}
func TestResourceRegistry_IngestSnapshotParentsInferredUnraidArrayDisksUnderStorage(t *testing.T) {
rr := NewRegistry(nil)
now := time.Date(2026, 3, 7, 12, 0, 0, 0, time.UTC)
rr.IngestSnapshot(models.StateSnapshot{
Hosts: []models.Host{
{
ID: "host-tower",
Hostname: "tower",
Status: "online",
LastSeen: now,
MachineID: "machine-tower",
Unraid: &models.HostUnraidStorage{
ArrayStarted: true,
Disks: []models.HostUnraidDisk{
{Name: "md1p1", Device: "/dev/sde", Status: "online", Slot: 1},
},
},
Sensors: models.HostSensorSummary{
SMART: []models.HostDiskSMART{
{
Device: "sde [sat]",
Model: "Array Disk",
Type: "sata",
Health: "UNKNOWN",
},
},
},
},
},
})
storage := rr.ListByType(ResourceTypeStorage)
disks := rr.ListByType(ResourceTypePhysicalDisk)
if len(storage) != 1 || len(disks) != 1 {
t.Fatalf("expected 1 storage and 1 disk resource, got storage=%d disk=%d", len(storage), len(disks))
}
if disks[0].ParentID == nil || *disks[0].ParentID != storage[0].ID {
t.Fatalf("expected inferred unraid disk parent to be storage resource %q, got %+v", storage[0].ID, disks[0].ParentID)
}
if disks[0].PhysicalDisk == nil || disks[0].PhysicalDisk.StorageRole != "data" || disks[0].PhysicalDisk.StorageGroup != "unraid-array" {
t.Fatalf("expected inferred unraid data disk metadata, got %+v", disks[0].PhysicalDisk)
}
if storage[0].ChildCount != 1 {
t.Fatalf("expected storage child count 1, got %d", storage[0].ChildCount)
}
}
func TestResourceRegistry_IngestSnapshotDerivesStorageConsumers(t *testing.T) {
rr := NewRegistry(nil)
now := time.Date(2026, 3, 8, 12, 0, 0, 0, time.UTC)
rr.IngestSnapshot(models.StateSnapshot{
Storage: []models.Storage{
{
ID: "cluster-a-pve1-local-lvm",
Name: "local-lvm",
Node: "pve1",
Instance: "cluster-a",
Type: "lvmthin",
Status: "available",
Enabled: true,
Active: true,
},
{
ID: "cluster-a-pve2-local-lvm",
Name: "local-lvm",
Node: "pve2",
Instance: "cluster-a",
Type: "lvmthin",
Status: "available",
Enabled: true,
Active: true,
},
{
ID: "cluster-a-cluster-ceph",
Name: "ceph",
Node: "cluster",
Instance: "cluster-a",
Type: "rbd",
Status: "available",
Enabled: true,
Active: true,
Shared: true,
Nodes: []string{"pve1", "pve2"},
},
{
ID: "cluster-a-pve1-media",
Name: "media",
Node: "pve1",
Instance: "cluster-a",
Type: "dir",
Status: "available",
Enabled: true,
Active: true,
Path: "/mnt/pve/media",
},
},
VMs: []models.VM{
{
ID: "vm-100",
Name: "app01",
Node: "pve1",
Instance: "cluster-a",
Status: "running",
LastSeen: now,
Disks: []models.Disk{
{Device: "local-lvm:vm-100-disk-0"},
{Device: "ceph:vm-100-disk-1"},
},
},
},
Containers: []models.Container{
{
ID: "ct-200",
Name: "media01",
Node: "pve1",
Instance: "cluster-a",
Status: "running",
LastSeen: now,
Disks: []models.Disk{
{Device: "/mnt/pve/media/subvol-200-disk-1"},
{Device: "local-lvm:vm-200-disk-0"},
},
},
},
})
storageResources := rr.ListByType(ResourceTypeStorage)
local := findStorageResource(storageResources, "local-lvm", "pve1")
if local.Storage == nil {
t.Fatalf("expected local-lvm storage metadata, got %+v", local)
}
if got := local.Storage.ConsumerCount; got != 2 {
t.Fatalf("local-lvm consumerCount = %d, want 2", got)
}
if got := local.Storage.ConsumerTypes; len(got) != 2 || got[0] != "system-container" || got[1] != "vm" {
t.Fatalf("local-lvm consumerTypes = %v, want [system-container vm]", got)
}
if len(local.Storage.TopConsumers) != 2 {
t.Fatalf("local-lvm topConsumers length = %d, want 2", len(local.Storage.TopConsumers))
}
if !hasStorageConsumer(local.Storage.TopConsumers, "app01", ResourceTypeVM, 1) {
t.Fatalf("expected vm consumer on local-lvm, got %+v", local.Storage.TopConsumers)
}
if !hasStorageConsumer(local.Storage.TopConsumers, "media01", ResourceTypeSystemContainer, 1) {
t.Fatalf("expected container consumer on local-lvm, got %+v", local.Storage.TopConsumers)
}
if got := local.Storage.ConsumerImpactSummary; got != "Affects 2 dependent resources: media01, app01" {
t.Fatalf("local-lvm consumerImpactSummary = %q", got)
}
ceph := findStorageResource(storageResources, "ceph", "cluster")
if ceph.Storage == nil || ceph.Storage.ConsumerCount != 1 {
t.Fatalf("ceph consumerCount = %+v, want 1", ceph.Storage)
}
if !hasStorageConsumer(ceph.Storage.TopConsumers, "app01", ResourceTypeVM, 1) {
t.Fatalf("expected vm consumer on ceph, got %+v", ceph.Storage.TopConsumers)
}
media := findStorageResource(storageResources, "media", "pve1")
if media.Storage == nil || media.Storage.ConsumerCount != 1 {
t.Fatalf("media consumerCount = %+v, want 1", media.Storage)
}
if !hasStorageConsumer(media.Storage.TopConsumers, "media01", ResourceTypeSystemContainer, 1) {
t.Fatalf("expected container consumer on media, got %+v", media.Storage.TopConsumers)
}
otherLocal := findStorageResource(storageResources, "local-lvm", "pve2")
if otherLocal.Storage == nil {
t.Fatalf("expected second local-lvm storage metadata")
}
if otherLocal.Storage.ConsumerCount != 0 {
t.Fatalf("expected pve2 local-lvm to remain without consumers, got %+v", otherLocal.Storage)
}
}
func TestResourceRegistry_IngestSnapshotProjectsCephPoolsAsStorage(t *testing.T) {
now := time.Date(2026, 3, 8, 12, 0, 0, 0, time.UTC)
rr := NewRegistry(nil)
rr.IngestSnapshot(models.StateSnapshot{
CephClusters: []models.CephCluster{
{
ID: "Main-fsid123",
Instance: "Main",
Name: "Main Ceph",
FSID: "fsid123",
Health: "HEALTH_OK",
LastUpdated: now,
Pools: []models.CephPool{
{
ID: 1,
Name: "data_replication",
StoredBytes: 70,
AvailableBytes: 30,
PercentUsed: 70,
},
},
},
},
})
storageResources := rr.ListByType(ResourceTypeStorage)
pool := findStorageResource(storageResources, "data_replication", "cluster")
if pool.Storage == nil {
t.Fatalf("expected Ceph pool storage metadata, got %+v", pool)
}
if !pool.Storage.IsCeph || !pool.Storage.Shared || pool.Storage.Type != "ceph" {
t.Fatalf("unexpected Ceph pool storage metadata: %+v", pool.Storage)
}
if pool.Metrics == nil || pool.Metrics.Disk == nil || pool.Metrics.Disk.Percent != 70 {
t.Fatalf("expected Ceph pool disk metrics, got %+v", pool.Metrics)
}
target := BuildMetricsTargetForRegistry(rr, pool.ID)
if target == nil {
t.Fatalf("expected metrics target for Ceph pool storage")
}
wantID := models.CephPoolStorageID("Main", "data_replication")
if target.ResourceType != "storage" || target.ResourceID != wantID {
t.Fatalf("metrics target = %+v, want storage/%s", target, wantID)
}
}
func TestResourceRegistry_IngestSnapshotDerivesPBSDatastoreConsumers(t *testing.T) {
rr := NewRegistry(nil)
now := time.Date(2026, 3, 8, 12, 0, 0, 0, time.UTC)
rr.IngestSnapshot(models.StateSnapshot{
PBSInstances: []models.PBSInstance{
{
ID: "pbs-1",
Name: "pbs-main",
Host: "https://pbs-main.local:8007",
Status: "online",
LastSeen: now,
Datastores: []models.PBSDatastore{
{Name: "backup-store", Status: "online"},
},
},
},
PBSBackups: []models.PBSBackup{
{
ID: "pbs-1/backup-store/vm/100",
Instance: "pbs-main",
Datastore: "backup-store",
Namespace: "pve",
BackupType: "vm",
VMID: "100",
BackupTime: now,
},
{
ID: "pbs-1/backup-store/ct/200",
Instance: "pbs-main",
Datastore: "backup-store",
Namespace: "nat",
BackupType: "ct",
VMID: "200",
BackupTime: now,
},
},
VMs: []models.VM{
{
ID: "vm-100",
Name: "app01",
Node: "pve-1",
Instance: "pve",
Status: "running",
LastSeen: now,
VMID: 100,
},
},
Containers: []models.Container{
{
ID: "ct-200",
Name: "media01",
Node: "pve-2",
Instance: "pve-nat",
Status: "running",
LastSeen: now,
VMID: 200,
},
},
})
storageResources := rr.ListByType(ResourceTypeStorage)
datastore := findStorageResourceByPlatform(storageResources, "backup-store", "pbs", "datastore")
if datastore.Storage == nil {
t.Fatalf("expected PBS datastore storage metadata, got %+v", datastore)
}
if got := datastore.Storage.ConsumerCount; got != 2 {
t.Fatalf("backup-store consumerCount = %d, want 2", got)
}
if got := datastore.Storage.ConsumerTypes; len(got) != 2 || got[0] != "system-container" || got[1] != "vm" {
t.Fatalf("backup-store consumerTypes = %v, want [system-container vm]", got)
}
if len(datastore.Storage.TopConsumers) != 2 {
t.Fatalf("backup-store topConsumers length = %d, want 2", len(datastore.Storage.TopConsumers))
}
if !hasStorageConsumer(datastore.Storage.TopConsumers, "app01", ResourceTypeVM, 1) {
t.Fatalf("expected vm consumer on backup-store, got %+v", datastore.Storage.TopConsumers)
}
if !hasStorageConsumer(datastore.Storage.TopConsumers, "media01", ResourceTypeSystemContainer, 1) {
t.Fatalf("expected container consumer on backup-store, got %+v", datastore.Storage.TopConsumers)
}
if got := datastore.Storage.ConsumerImpactSummary; got != "Puts backups for 2 protected workloads at risk: media01, app01" {
t.Fatalf("backup-store consumerImpactSummary = %q", got)
}
pbsResources := rr.ListByType(ResourceTypePBS)
if len(pbsResources) != 1 {
t.Fatalf("expected 1 PBS resource, got %d", len(pbsResources))
}
pbs := pbsResources[0]
if pbs.PBS == nil {
t.Fatalf("expected PBS payload, got %+v", pbs)
}
if got := pbs.PBS.ProtectedWorkloadCount; got != 2 {
t.Fatalf("protectedWorkloadCount = %d, want 2", got)
}
if got := pbs.PBS.ProtectedWorkloadTypes; len(got) != 2 || got[0] != "system-container" || got[1] != "vm" {
t.Fatalf("protectedWorkloadTypes = %v, want [system-container vm]", got)
}
if got := pbs.PBS.ProtectedWorkloadNames; len(got) != 2 || got[0] != "media01" || got[1] != "app01" {
t.Fatalf("protectedWorkloadNames = %v, want [media01 app01] in sorted rollup order", got)
}
if got := pbs.PBS.AffectedDatastoreCount; got != 0 {
t.Fatalf("affectedDatastoreCount = %d, want 0 without datastore risk", got)
}
if got := pbs.PBS.ProtectedWorkloadSummary; got != "Puts backups for 2 protected workloads at risk: media01, app01" {
t.Fatalf("protectedWorkloadSummary = %q", got)
}
if got := pbs.PBS.PostureSummary; got != "Puts backups for 2 protected workloads at risk: media01, app01" {
t.Fatalf("postureSummary = %q", got)
}
}
func TestResourceRegistry_IngestResourcesDerivesPrimaryIncidentRollups(t *testing.T) {
rr := NewRegistry(nil)
rr.IngestResources([]Resource{
{
ID: "storage:tank",
Type: ResourceTypeStorage,
Name: "tank",
Status: StatusWarning,
Storage: &StorageMeta{
Platform: "truenas",
Topology: "pool",
Protection: "zfs",
Risk: &StorageRisk{
Level: storagehealth.RiskWarning,
Reasons: []StorageRiskReason{
{Code: "capacity_runway_low", Severity: storagehealth.RiskWarning, Summary: "Storage tank is 92% full"},
{Code: "zfs_pool_state", Severity: storagehealth.RiskWarning, Summary: "ZFS pool tank is DEGRADED"},
},
},
},
Incidents: []ResourceIncident{
{Code: "capacity_runway_low", Severity: storagehealth.RiskWarning, Summary: "Storage tank is 92% full"},
{Code: "zfs_pool_state", Severity: storagehealth.RiskWarning, Summary: "ZFS pool tank is DEGRADED"},
},
},
{
ID: "pbs:main",
Type: ResourceTypePBS,
Name: "pbs-main",
Status: StatusWarning,
PBS: &PBSData{
StorageRisk: &StorageRisk{
Level: storagehealth.RiskWarning,
Reasons: []StorageRiskReason{
{Code: "pbs_datastore_state", Severity: storagehealth.RiskWarning, Summary: "PBS datastore archive is READ_ONLY"},
{Code: "capacity_runway_low", Severity: storagehealth.RiskWarning, Summary: "PBS datastore fast is 96% full"},
},
},
AffectedDatastoreSummary: "Affects 1 backup datastore: fast",
ProtectedWorkloadSummary: "Puts backups for 2 protected workloads at risk: media01, app01",
PostureSummary: "Affects 1 backup datastore: fast. Puts backups for 2 protected workloads at risk: media01, app01",
},
Incidents: []ResourceIncident{
{Code: "pbs_datastore_state", Severity: storagehealth.RiskWarning, Summary: "PBS datastore archive is READ_ONLY"},
{Code: "capacity_runway_low", Severity: storagehealth.RiskWarning, Summary: "PBS datastore fast is 96% full"},
},
},
})
storage, ok := rr.Get("storage:tank")
if !ok {
t.Fatal("expected storage resource")
}
if storage.IncidentCount != 2 || storage.IncidentCode != "zfs_pool_state" || storage.IncidentSeverity != storagehealth.RiskWarning || storage.IncidentSummary != "ZFS pool tank is DEGRADED" || storage.IncidentCategory != IncidentCategoryProtection || storage.IncidentLabel != "Protection Reduced" || storage.IncidentPriority != 3400 || storage.IncidentImpactSummary != "" || storage.IncidentUrgency != IncidentUrgencyToday || storage.IncidentAction != "Inspect native pool and vdev status and plan maintenance to restore ONLINE state" {
t.Fatalf("unexpected storage incident rollup %+v", storage)
}
pbs, ok := rr.Get("pbs:main")
if !ok {
t.Fatal("expected pbs resource")
}
if pbs.IncidentCount != 2 || pbs.IncidentCode != "pbs_datastore_state" || pbs.IncidentSeverity != storagehealth.RiskWarning || pbs.IncidentSummary != "PBS datastore archive is READ_ONLY" || pbs.IncidentCategory != IncidentCategoryRecoverability || pbs.IncidentLabel != "Backup Coverage At Risk" || pbs.IncidentPriority != 3500 || pbs.IncidentImpactSummary != "" || pbs.IncidentUrgency != IncidentUrgencyToday || pbs.IncidentAction != "Investigate backup target health and preserve backup coverage" {
t.Fatalf("unexpected pbs incident rollup %+v", pbs)
}
}
func TestResourceRegistry_IngestSnapshotDerivesPhysicalDiskRisk(t *testing.T) {
rr := NewRegistry(nil)
now := time.Date(2026, 3, 7, 12, 0, 0, 0, time.UTC)
pending := int64(2)
rr.IngestSnapshot(models.StateSnapshot{
Hosts: []models.Host{
{
ID: "host-risky",
Hostname: "tower",
Status: "online",
LastSeen: now,
Sensors: models.HostSensorSummary{
SMART: []models.HostDiskSMART{
{
Device: "/dev/sdc",
Model: "Seagate IronWolf",
Serial: "SERIAL-RISK-1",
Type: "sata",
Temperature: 64,
Health: "PASSED",
Attributes: &models.SMARTAttributes{
PendingSectors: &pending,
},
},
},
},
},
},
})
disks := rr.ListByType(ResourceTypePhysicalDisk)
if len(disks) != 1 {
t.Fatalf("expected 1 physical disk resource, got %d", len(disks))
}
disk := disks[0]
if disk.Status != StatusWarning {
t.Fatalf("Status = %q, want %q", disk.Status, StatusWarning)
}
if disk.PhysicalDisk == nil || disk.PhysicalDisk.Risk == nil {
t.Fatalf("expected disk risk payload, got %+v", disk.PhysicalDisk)
}
if disk.PhysicalDisk.Risk.Level != storagehealth.RiskCritical {
t.Fatalf("risk level = %q, want %q", disk.PhysicalDisk.Risk.Level, storagehealth.RiskCritical)
}
if len(disk.PhysicalDisk.Risk.Reasons) == 0 || disk.PhysicalDisk.Risk.Reasons[0].Code != "pending_sectors" {
t.Fatalf("expected pending sectors reason, got %+v", disk.PhysicalDisk.Risk.Reasons)
}
}
func TestResourceRegistry_MergePhysicalDiskKeepsIncidentBackedRisk(t *testing.T) {
rr := NewRegistry(nil)
now := time.Date(2026, 3, 30, 12, 0, 0, 0, time.UTC)
rr.IngestRecords(SourceAgent, []IngestRecord{
{
SourceID: "agent-disk-sdc",
Resource: Resource{
Type: ResourceTypePhysicalDisk,
Name: "sdc",
Status: StatusOnline,
LastSeen: now,
PhysicalDisk: &PhysicalDiskMeta{
DevPath: "/dev/sdc",
Model: "Seagate IronWolf",
Serial: "SERIAL-RISK-2",
DiskType: "sata",
SizeBytes: 4_000_000_000_000,
Health: "PASSED",
Temperature: 38,
Wearout: -1,
},
},
Identity: ResourceIdentity{
MachineID: "SERIAL-RISK-2",
Hostnames: []string{"truenas-main"},
},
},
})
rr.IngestRecords(SourceTrueNAS, []IngestRecord{
{
SourceID: "truenas-disk-sdc",
Resource: Resource{
Type: ResourceTypePhysicalDisk,
Name: "sdc",
Status: StatusWarning,
LastSeen: now,
PhysicalDisk: &PhysicalDiskMeta{
DevPath: "/dev/sdc",
Model: "Seagate IronWolf",
Serial: "SERIAL-RISK-2",
DiskType: "sata",
SizeBytes: 4_000_000_000_000,
Health: "UNKNOWN",
Temperature: 63,
Wearout: -1,
},
Incidents: []ResourceIncident{
{
Code: "truenas_smart",
Severity: storagehealth.RiskCritical,
Summary: "Device /dev/sdc has SMART test failures.",
},
},
},
Identity: ResourceIdentity{
MachineID: "SERIAL-RISK-2",
Hostnames: []string{"truenas-main"},
},
},
})
disks := rr.ListByType(ResourceTypePhysicalDisk)
if len(disks) != 1 {
t.Fatalf("expected 1 merged physical disk, got %d", len(disks))
}
disk := disks[0]
if disk.PhysicalDisk == nil || disk.PhysicalDisk.Risk == nil {
t.Fatalf("expected merged disk risk, got %+v", disk.PhysicalDisk)
}
if disk.PhysicalDisk.Risk.Level != storagehealth.RiskCritical {
t.Fatalf("risk level = %q, want %q", disk.PhysicalDisk.Risk.Level, storagehealth.RiskCritical)
}
foundSmartReason := false
for _, reason := range disk.PhysicalDisk.Risk.Reasons {
if reason.Code == "truenas_smart" {
foundSmartReason = true
break
}
}
if !foundSmartReason {
t.Fatalf("expected SMART-backed risk reason after merge, got %+v", disk.PhysicalDisk.Risk.Reasons)
}
}
func TestResourceRegistry_MergeTrueNASPayloadOnIdentityMatch(t *testing.T) {
rr := NewRegistry(nil)
now := time.Date(2026, 5, 8, 12, 0, 0, 0, time.UTC)
rr.IngestRecords(SourceAgent, []IngestRecord{
{
SourceID: "agent-archive",
Resource: Resource{
Type: ResourceTypeAgent,
Name: "archive",
Status: StatusOnline,
LastSeen: now,
Agent: &AgentData{
Hostname: "archive.local",
OSName: "Debian GNU/Linux",
},
},
Identity: ResourceIdentity{
MachineID: "archive-machine",
Hostnames: []string{"archive.local"},
},
},
})
rr.IngestRecords(SourceTrueNAS, []IngestRecord{
{
SourceID: "truenas-archive",
Resource: Resource{
Type: ResourceTypeAgent,
Name: "archive.local",
Status: StatusWarning,
LastSeen: now.Add(time.Minute),
TrueNAS: &TrueNASData{
Hostname: "archive.local",
Version: "24.10",
UptimeSeconds: 86400,
StorageRisk: &StorageRisk{Level: storagehealth.RiskWarning},
StorageRiskSummary: "Pool tank is DEGRADED",
StoragePostureSummary: "Pool tank is DEGRADED",
ProtectionReduced: true,
ProtectionSummary: "Pool redundancy is reduced",
},
Agent: &AgentData{
Hostname: "archive.local",
OSName: "TrueNAS SCALE",
},
},
Identity: ResourceIdentity{
MachineID: "archive-machine",
Hostnames: []string{"archive.local"},
},
},
})
resources := rr.ListByType(ResourceTypeAgent)
if len(resources) != 1 {
t.Fatalf("ListByType(ResourceTypeAgent) returned %d resources, want 1", len(resources))
}
resource := resources[0]
if !hasDataSource(resource.Sources, SourceAgent) || !hasDataSource(resource.Sources, SourceTrueNAS) {
t.Fatalf("sources = %#v, want agent and truenas", resource.Sources)
}
if resource.TrueNAS == nil {
t.Fatal("expected TrueNAS payload to merge onto matched agent resource")
}
if resource.TrueNAS.Version != "24.10" || resource.TrueNAS.UptimeSeconds != 86400 {
t.Fatalf("TrueNAS payload not refreshed: %+v", resource.TrueNAS)
}
if resource.TrueNAS.StorageRisk == nil || resource.TrueNAS.StorageRisk.Level != storagehealth.RiskWarning {
t.Fatalf("TrueNAS storage risk not merged: %+v", resource.TrueNAS.StorageRisk)
}
if resource.Agent == nil || resource.Agent.OSName != "Debian GNU/Linux" {
t.Fatalf("SourceAgent facet should remain authoritative, got %+v", resource.Agent)
}
}
func hasStorageConsumer(consumers []StorageConsumerMeta, name string, resourceType ResourceType, diskCount int) bool {
for _, consumer := range consumers {
if consumer.Name == name && consumer.ResourceType == resourceType && consumer.DiskCount == diskCount {
return true
}
}
return false
}
func findStorageResource(resources []Resource, name, node string) Resource {
for _, resource := range resources {
if resource.Name != name || resource.Proxmox == nil || resource.Proxmox.NodeName != node {
continue
}
return resource
}
return Resource{}
}
func findStorageResourceByPlatform(resources []Resource, name, platform, topology string) Resource {
for _, resource := range resources {
if resource.Name != name || resource.Storage == nil {
continue
}
if resource.Storage.Platform != platform || resource.Storage.Topology != topology {
continue
}
return resource
}
return Resource{}
}
func TestResourceRegistry_BuildChildCounts_ReparentClearsOldParentCount(t *testing.T) {
rr := NewRegistry(nil)
now := time.Date(2026, 2, 12, 1, 0, 0, 0, time.UTC)
rr.IngestRecords(SourceProxmox, []IngestRecord{
{
SourceID: "node-a",
Resource: Resource{
Type: ResourceTypeAgent,
Name: "node-a",
Status: StatusOnline,
LastSeen: now,
},
Identity: ResourceIdentity{Hostnames: []string{"node-a"}},
},
{
SourceID: "node-b",
Resource: Resource{
Type: ResourceTypeAgent,
Name: "node-b",
Status: StatusOnline,
LastSeen: now,
},
Identity: ResourceIdentity{Hostnames: []string{"node-b"}},
},
{
SourceID: "vm-100",
ParentSourceID: "node-a",
Resource: Resource{
Type: ResourceTypeVM,
Name: "vm-100",
Status: StatusOnline,
LastSeen: now,
},
Identity: ResourceIdentity{Hostnames: []string{"vm-100"}},
},
})
parentAID := rr.sourceSpecificID(ResourceTypeAgent, SourceProxmox, "node-a")
parentBID := rr.sourceSpecificID(ResourceTypeAgent, SourceProxmox, "node-b")
vmID := rr.sourceSpecificID(ResourceTypeVM, SourceProxmox, "vm-100")
parentA, ok := rr.Get(parentAID)
if !ok {
t.Fatalf("expected parent A %q to exist", parentAID)
}
parentB, ok := rr.Get(parentBID)
if !ok {
t.Fatalf("expected parent B %q to exist", parentBID)
}
vm, ok := rr.Get(vmID)
if !ok {
t.Fatalf("expected vm %q to exist", vmID)
}
if parentA.ChildCount != 1 || parentB.ChildCount != 0 {
t.Fatalf("expected initial child counts parentA=1 parentB=0, got parentA=%d parentB=%d", parentA.ChildCount, parentB.ChildCount)
}
if vm.ParentName != "node-a" {
t.Fatalf("expected vm parent name %q, got %q", "node-a", vm.ParentName)
}
rr.IngestRecords(SourceProxmox, []IngestRecord{
{
SourceID: "node-a",
Resource: Resource{
Type: ResourceTypeAgent,
Name: "node-a",
Status: StatusOnline,
LastSeen: now.Add(30 * time.Second),
},
Identity: ResourceIdentity{Hostnames: []string{"node-a"}},
},
{
SourceID: "node-b",
Resource: Resource{
Type: ResourceTypeAgent,
Name: "node-b",
Status: StatusOnline,
LastSeen: now.Add(30 * time.Second),
},
Identity: ResourceIdentity{Hostnames: []string{"node-b"}},
},
{
SourceID: "vm-100",
ParentSourceID: "node-b",
Resource: Resource{
Type: ResourceTypeVM,
Name: "vm-100",
Status: StatusOnline,
LastSeen: now.Add(30 * time.Second),
},
Identity: ResourceIdentity{Hostnames: []string{"vm-100"}},
},
})
parentA, ok = rr.Get(parentAID)
if !ok {
t.Fatalf("expected parent A %q to exist after reparent", parentAID)
}
parentB, ok = rr.Get(parentBID)
if !ok {
t.Fatalf("expected parent B %q to exist after reparent", parentBID)
}
vm, ok = rr.Get(vmID)
if !ok {
t.Fatalf("expected vm %q to exist after reparent", vmID)
}
if parentA.ChildCount != 0 || parentB.ChildCount != 1 {
t.Fatalf("expected child counts parentA=0 parentB=1 after reparent, got parentA=%d parentB=%d", parentA.ChildCount, parentB.ChildCount)
}
if vm.ParentName != "node-b" {
t.Fatalf("expected vm parent name %q after reparent, got %q", "node-b", vm.ParentName)
}
}
func TestResourceRegistry_BuildChildCounts_SourceUpdateClearsRemovedParent(t *testing.T) {
rr := NewRegistry(nil)
now := time.Date(2026, 2, 12, 1, 0, 0, 0, time.UTC)
rr.IngestRecords(SourceProxmox, []IngestRecord{
{
SourceID: "node-a",
Resource: Resource{
Type: ResourceTypeAgent,
Name: "node-a",
Status: StatusOnline,
LastSeen: now,
},
Identity: ResourceIdentity{Hostnames: []string{"node-a"}},
},
{
SourceID: "vm-100",
ParentSourceID: "node-a",
Resource: Resource{
Type: ResourceTypeVM,
Name: "vm-100",
Status: StatusOnline,
LastSeen: now,
},
Identity: ResourceIdentity{Hostnames: []string{"vm-100"}},
},
})
parentID := rr.sourceSpecificID(ResourceTypeAgent, SourceProxmox, "node-a")
vmID := rr.sourceSpecificID(ResourceTypeVM, SourceProxmox, "vm-100")
parent, ok := rr.Get(parentID)
if !ok {
t.Fatalf("expected parent %q to exist", parentID)
}
vm, ok := rr.Get(vmID)
if !ok {
t.Fatalf("expected vm %q to exist", vmID)
}
if parent.ChildCount != 1 {
t.Fatalf("expected parent child count 1 before clearing parent, got %d", parent.ChildCount)
}
if vm.ParentID == nil || *vm.ParentID != parentID {
t.Fatalf("expected vm parent id %q before clearing parent, got %+v", parentID, vm.ParentID)
}
rr.IngestRecords(SourceProxmox, []IngestRecord{
{
SourceID: "node-a",
Resource: Resource{
Type: ResourceTypeAgent,
Name: "node-a",
Status: StatusOnline,
LastSeen: now.Add(30 * time.Second),
},
Identity: ResourceIdentity{Hostnames: []string{"node-a"}},
},
{
SourceID: "vm-100",
Resource: Resource{
Type: ResourceTypeVM,
Name: "vm-100",
Status: StatusOnline,
LastSeen: now.Add(30 * time.Second),
},
Identity: ResourceIdentity{Hostnames: []string{"vm-100"}},
},
})
parent, ok = rr.Get(parentID)
if !ok {
t.Fatalf("expected parent %q to exist after clearing parent", parentID)
}
vm, ok = rr.Get(vmID)
if !ok {
t.Fatalf("expected vm %q to exist after clearing parent", vmID)
}
if parent.ChildCount != 0 {
t.Fatalf("expected parent child count 0 after clearing parent, got %d", parent.ChildCount)
}
if vm.ParentID != nil {
t.Fatalf("expected vm parent id to clear after source update, got %+v", vm.ParentID)
}
if vm.ParentName != "" {
t.Fatalf("expected vm parent name to clear after source update, got %q", vm.ParentName)
}
}
func TestRegistryMonitoredSystemCountDedupesAcrossTopLevelViews(t *testing.T) {
rr := NewRegistry(nil)
rr.IngestRecords(SourceAgent, []IngestRecord{
{
SourceID: "host-1",
Resource: Resource{
ID: "host-1",
Type: ResourceTypeAgent,
Name: "lab-a",
Status: StatusOnline,
Agent: &AgentData{
AgentID: "agent-1",
Hostname: "lab-a",
MachineID: "machine-1",
},
Identity: ResourceIdentity{
MachineID: "machine-1",
Hostnames: []string{"lab-a"},
},
},
},
})
rr.IngestRecords(SourcePBS, []IngestRecord{
{
SourceID: "pbs-1",
Resource: Resource{
ID: "pbs-1",
Type: ResourceTypePBS,
Name: "pbs-a",
Status: StatusOnline,
PBS: &PBSData{
InstanceID: "pbs-1",
Hostname: "lab-a",
HostURL: "https://lab-a:8007",
},
},
},
})
rr.IngestRecords(SourcePMG, []IngestRecord{
{
SourceID: "pmg-1",
Resource: Resource{
ID: "pmg-1",
Type: ResourceTypePMG,
Name: "pmg-b",
Status: StatusOnline,
PMG: &PMGData{
InstanceID: "pmg-1",
Hostname: "mail-b",
},
},
},
})
rr.IngestRecords(SourceK8s, []IngestRecord{
{
SourceID: "k8s-1",
Resource: Resource{
ID: "k8s-1",
Type: ResourceTypeK8sCluster,
Name: "cluster-a",
Status: StatusOnline,
Kubernetes: &K8sData{
ClusterID: "cluster-a",
AgentID: "k8s-agent-1",
Server: "https://cluster-a.example:6443",
},
},
},
})
if got := MonitoredSystemCount(rr); got != 3 {
t.Fatalf("MonitoredSystemCount() = %d, want 3", got)
}
}
func TestRegistryHasMatchingMonitoredSystemUsesCanonicalHostIdentity(t *testing.T) {
rr := NewRegistry(nil)
rr.IngestRecords(SourceTrueNAS, []IngestRecord{
{
SourceID: "truenas-1",
Resource: Resource{
ID: "truenas-1",
Type: ResourceTypeAgent,
Name: "archive",
Status: StatusOnline,
TrueNAS: &TrueNASData{
Hostname: "archive.local",
},
Identity: ResourceIdentity{
Hostnames: []string{"archive.local"},
},
},
},
})
if !HasMatchingMonitoredSystem(rr, MonitoredSystemCandidate{
Type: ResourceTypeAgent,
Hostname: "archive.local",
HostURL: "https://archive.local",
}) {
t.Fatal("expected candidate to match existing counted system")
}
if HasMatchingMonitoredSystem(rr, MonitoredSystemCandidate{
Type: ResourceTypePBS,
Hostname: "other.local",
HostURL: "https://other.local:8007",
}) {
t.Fatal("expected unrelated candidate not to match existing counted system")
}
}
func TestRegistryIngestSnapshotPublishesCanonicalKubernetesMetricsTargets(t *testing.T) {
rr := NewRegistry(nil)
rr.IngestSnapshot(models.StateSnapshot{
KubernetesClusters: []models.KubernetesCluster{
{
ID: "cluster-1",
Name: "cluster-a",
DisplayName: "Cluster A",
AgentID: "agent-cluster-1",
Context: "cluster-a",
Nodes: []models.KubernetesNode{
{
UID: "node-1",
Name: "worker-1",
},
},
Pods: []models.KubernetesPod{
{
UID: "pod-1",
Namespace: "default",
Name: "api-7dd5f6c7f9-jx2ql",
},
},
Deployments: []models.KubernetesDeployment{
{
UID: "deploy-1",
Namespace: "default",
Name: "api",
},
},
},
},
})
assertMetricsTarget := func(resourceType ResourceType, want string) {
t.Helper()
resources := rr.ListByType(resourceType)
if len(resources) != 1 {
t.Fatalf("ListByType(%q) len = %d, want 1", resourceType, len(resources))
}
target := BuildMetricsTargetForRegistry(rr, resources[0].ID)
if target == nil {
t.Fatalf("BuildMetricsTargetForRegistry(%q) returned nil MetricsTarget", resources[0].ID)
}
if target.ResourceID != want {
t.Fatalf("ListByType(%q) metrics target = %q, want %q", resourceType, target.ResourceID, want)
}
}
assertMetricsTarget(ResourceTypeK8sCluster, "cluster-1")
assertMetricsTarget(ResourceTypeK8sNode, "cluster-1:node:node-1")
assertMetricsTarget(ResourceTypePod, "k8s:cluster-1:pod:pod-1")
assertMetricsTarget(ResourceTypeK8sDeployment, "cluster-1:deployment:deploy-1")
}
func TestRegistryIngestSnapshotMergesLinkedKubernetesNodeIntoAgentResource(t *testing.T) {
rr := NewRegistry(nil)
rr.IngestSnapshot(models.StateSnapshot{
Hosts: []models.Host{
{
ID: "host-1",
Hostname: "prod-euw1-k8s-01",
MachineID: "machine-k8s-1",
Status: "online",
},
},
KubernetesClusters: []models.KubernetesCluster{
{
ID: "cluster-1",
Name: "production",
AgentID: "agent-cluster-1",
Nodes: []models.KubernetesNode{
{
UID: "node-1",
Name: "prod-euw1-k8s-01",
Ready: true,
},
},
},
},
})
resources := rr.List()
if got := len(resources); got != 2 {
t.Fatalf("expected cluster + merged agent resource, got %d resources", got)
}
var cluster *Resource
var host *Resource
for i := range resources {
switch resources[i].Type {
case ResourceTypeK8sCluster:
cluster = &resources[i]
case ResourceTypeAgent:
host = &resources[i]
case ResourceTypeK8sNode:
t.Fatalf("expected linked kubernetes node to merge into backing agent, got standalone node %+v", resources[i])
}
}
if cluster == nil {
t.Fatal("expected kubernetes cluster resource")
}
if host == nil {
t.Fatal("expected merged agent resource")
}
if !hasDataSource(host.Sources, SourceAgent) || !hasDataSource(host.Sources, SourceK8s) {
t.Fatalf("expected merged host sources to include agent+kubernetes, got %+v", host.Sources)
}
if host.Kubernetes == nil {
t.Fatal("expected merged host to retain kubernetes node payload")
}
if got := strings.TrimSpace(host.Kubernetes.NodeName); got != "prod-euw1-k8s-01" {
t.Fatalf("kubernetes node name = %q, want prod-euw1-k8s-01", got)
}
if host.ParentID == nil || strings.TrimSpace(*host.ParentID) != cluster.ID {
t.Fatalf("expected merged host parent to point at kubernetes cluster %q, got %+v", cluster.ID, host.ParentID)
}
}
func TestRegistryHasMatchingMonitoredSystemRejectsInvalidCandidate(t *testing.T) {
rr := NewRegistry(nil)
rr.IngestRecords(SourceAgent, []IngestRecord{
{
SourceID: "host-1",
Resource: Resource{
ID: "host-1",
Type: ResourceTypeAgent,
Name: "lab-a",
Status: StatusOnline,
Agent: &AgentData{
AgentID: "agent-1",
Hostname: "lab-a",
MachineID: "machine-1",
},
Identity: ResourceIdentity{
MachineID: "machine-1",
Hostnames: []string{"lab-a"},
},
},
},
})
if HasMatchingMonitoredSystem(rr, MonitoredSystemCandidate{}) {
t.Fatal("expected empty candidate not to match an existing counted system")
}
if HasMatchingMonitoredSystem(rr, MonitoredSystemCandidate{
Source: SourceAgent,
Name: " ",
Hostname: " ",
HostURL: " ",
}) {
t.Fatal("expected whitespace-only candidate not to match an existing counted system")
}
}
func TestProjectMonitoredSystemRecords_DedupesVMwareHostAgainstExistingAgentCount(t *testing.T) {
rr := NewRegistry(nil)
now := time.Date(2026, 4, 7, 12, 0, 0, 0, time.UTC)
rr.IngestRecords(SourceAgent, []IngestRecord{
{
SourceID: "agent-host-1",
Resource: Resource{
Type: ResourceTypeAgent,
Name: "esxi-01.lab.local",
Status: StatusOnline,
LastSeen: now,
},
Identity: ResourceIdentity{
DMIUUID: "uuid-host-1",
Hostnames: []string{"esxi-01.lab.local"},
},
},
})
projection := ProjectMonitoredSystemRecords(rr, map[DataSource][]IngestRecord{
SourceVMware: {
{
SourceID: "vc-1:host:host-101",
Resource: Resource{
Type: ResourceTypeAgent,
Name: "esxi-01.lab.local",
Status: StatusOnline,
LastSeen: now,
VMware: &VMwareData{
ConnectionID: "vc-1",
ConnectionName: "Lab VC",
ManagedObjectID: "host-101",
EntityType: "host",
HostUUID: "uuid-host-1",
},
},
Identity: ResourceIdentity{
DMIUUID: "uuid-host-1",
Hostnames: []string{"esxi-01.lab.local"},
},
},
},
})
if projection.CurrentCount != 1 {
t.Fatalf("CurrentCount = %d, want 1", projection.CurrentCount)
}
if projection.ProjectedCount != 1 {
t.Fatalf("ProjectedCount = %d, want 1", projection.ProjectedCount)
}
if projection.AdditionalCount != 0 {
t.Fatalf("AdditionalCount = %d, want 0", projection.AdditionalCount)
}
}
func TestPreviewMonitoredSystemRecords_ReturnsCurrentAndProjectedSystemsForVMwareAdmission(t *testing.T) {
rr := NewRegistry(nil)
now := time.Date(2026, 4, 7, 12, 0, 0, 0, time.UTC)
rr.IngestRecords(SourceAgent, []IngestRecord{
{
SourceID: "agent-host-1",
Resource: Resource{
Type: ResourceTypeAgent,
Name: "esxi-01.lab.local",
Status: StatusOnline,
LastSeen: now,
},
Identity: ResourceIdentity{
DMIUUID: "uuid-host-1",
Hostnames: []string{"esxi-01.lab.local"},
},
},
})
preview := PreviewMonitoredSystemRecords(rr, map[DataSource][]IngestRecord{
SourceVMware: {
{
SourceID: "vc-1:host:host-101",
Resource: Resource{
Type: ResourceTypeAgent,
Name: "esxi-01.lab.local",
Status: StatusOnline,
LastSeen: now,
VMware: &VMwareData{
ConnectionID: "vc-1",
ConnectionName: "Lab VC",
ManagedObjectID: "host-101",
EntityType: "host",
HostUUID: "uuid-host-1",
},
},
Identity: ResourceIdentity{
DMIUUID: "uuid-host-1",
Hostnames: []string{"esxi-01.lab.local"},
},
},
{
SourceID: "vc-1:host:host-102",
Resource: Resource{
Type: ResourceTypeAgent,
Name: "esxi-02.lab.local",
Status: StatusOnline,
LastSeen: now,
VMware: &VMwareData{
ConnectionID: "vc-1",
ConnectionName: "Lab VC",
ManagedObjectID: "host-102",
EntityType: "host",
HostUUID: "uuid-host-2",
},
},
Identity: ResourceIdentity{
DMIUUID: "uuid-host-2",
Hostnames: []string{"esxi-02.lab.local"},
},
},
},
})
if preview.CurrentCount != 1 {
t.Fatalf("CurrentCount = %d, want 1", preview.CurrentCount)
}
if preview.ProjectedCount != 2 {
t.Fatalf("ProjectedCount = %d, want 2", preview.ProjectedCount)
}
if preview.AdditionalCount != 1 {
t.Fatalf("AdditionalCount = %d, want 1", preview.AdditionalCount)
}
if len(preview.CurrentSystems) != 1 {
t.Fatalf("len(CurrentSystems) = %d, want 1", len(preview.CurrentSystems))
}
if len(preview.ProjectedSystems) != 2 {
t.Fatalf("len(ProjectedSystems) = %d, want 2", len(preview.ProjectedSystems))
}
if preview.CurrentSystems[0].Source != "agent" {
t.Fatalf("CurrentSystems[0].Source = %q, want agent", preview.CurrentSystems[0].Source)
}
}
func TestPreviewMonitoredSystemRecordsReplacementOffsetsReplacedVMwareConnection(t *testing.T) {
rr := NewRegistry(nil)
now := time.Date(2026, 4, 7, 12, 0, 0, 0, time.UTC)
rr.IngestRecords(SourceAgent, []IngestRecord{
{
SourceID: "agent-host-1",
Resource: Resource{
Type: ResourceTypeAgent,
Name: "esxi-01.lab.local",
Status: StatusOnline,
LastSeen: now,
},
Identity: ResourceIdentity{
DMIUUID: "uuid-host-1",
Hostnames: []string{"esxi-01.lab.local"},
},
},
})
rr.IngestRecords(SourceVMware, []IngestRecord{
{
SourceID: "vc-1:host:host-101",
Resource: Resource{
Type: ResourceTypeAgent,
Name: "esxi-01.lab.local",
Status: StatusOnline,
LastSeen: now,
VMware: &VMwareData{
ConnectionID: "vc-1",
ConnectionName: "Lab VC",
ManagedObjectID: "host-101",
EntityType: "host",
HostUUID: "uuid-host-1",
},
},
Identity: ResourceIdentity{
DMIUUID: "uuid-host-1",
Hostnames: []string{"esxi-01.lab.local"},
},
},
{
SourceID: "vc-1:host:host-102",
Resource: Resource{
Type: ResourceTypeAgent,
Name: "esxi-02.lab.local",
Status: StatusOnline,
LastSeen: now,
VMware: &VMwareData{
ConnectionID: "vc-1",
ConnectionName: "Lab VC",
ManagedObjectID: "host-102",
EntityType: "host",
HostUUID: "uuid-host-2",
},
},
Identity: ResourceIdentity{
DMIUUID: "uuid-host-2",
Hostnames: []string{"esxi-02.lab.local"},
},
},
})
preview := PreviewMonitoredSystemRecordsReplacement(rr, MonitoredSystemReplacement{
Source: SourceVMware,
Selector: MonitoredSystemReplacementSelector{
ResourceID: "vc-1",
},
}, map[DataSource][]IngestRecord{
SourceVMware: {
{
SourceID: "vc-1:host:host-101",
Resource: Resource{
Type: ResourceTypeAgent,
Name: "esxi-01.lab.local",
Status: StatusOnline,
LastSeen: now,
VMware: &VMwareData{
ConnectionID: "vc-1",
ConnectionName: "Lab VC",
ManagedObjectID: "host-101",
EntityType: "host",
HostUUID: "uuid-host-1",
},
},
Identity: ResourceIdentity{
DMIUUID: "uuid-host-1",
Hostnames: []string{"esxi-01.lab.local"},
},
},
{
SourceID: "vc-1:host:host-103",
Resource: Resource{
Type: ResourceTypeAgent,
Name: "esxi-03.lab.local",
Status: StatusOnline,
LastSeen: now,
VMware: &VMwareData{
ConnectionID: "vc-1",
ConnectionName: "Lab VC",
ManagedObjectID: "host-103",
EntityType: "host",
HostUUID: "uuid-host-3",
},
},
Identity: ResourceIdentity{
DMIUUID: "uuid-host-3",
Hostnames: []string{"esxi-03.lab.local"},
},
},
},
})
if preview.CurrentCount != 2 {
t.Fatalf("CurrentCount = %d, want 2", preview.CurrentCount)
}
if preview.ProjectedCount != 2 {
t.Fatalf("ProjectedCount = %d, want 2", preview.ProjectedCount)
}
if preview.AdditionalCount != 0 {
t.Fatalf("AdditionalCount = %d, want 0", preview.AdditionalCount)
}
if len(preview.CurrentSystems) != 2 {
t.Fatalf("len(CurrentSystems) = %d, want 2", len(preview.CurrentSystems))
}
if len(preview.ProjectedSystems) != 2 {
t.Fatalf("len(ProjectedSystems) = %d, want 2", len(preview.ProjectedSystems))
}
if preview.CurrentSystem != nil {
t.Fatalf("CurrentSystem = %+v, want nil for multi-system replacement", preview.CurrentSystem)
}
if preview.ProjectedSystem != nil {
t.Fatalf("ProjectedSystem = %+v, want nil for multi-system replacement", preview.ProjectedSystem)
}
}
func TestPreviewMonitoredSystemRecordsReplacementScopesVMwareResourceIDToConnection(t *testing.T) {
rr := NewRegistry(nil)
now := time.Date(2026, 4, 7, 12, 0, 0, 0, time.UTC)
rr.IngestRecords(SourceVMware, []IngestRecord{
{
SourceID: "vc-edit:host:host-101",
Resource: Resource{
Type: ResourceTypeAgent,
Name: "esxi-01.lab.local",
Status: StatusOnline,
LastSeen: now,
VMware: &VMwareData{
ConnectionID: "vc-edit",
ConnectionName: "Edited VC",
ManagedObjectID: "host-101",
EntityType: "host",
HostUUID: "uuid-host-1",
},
},
Identity: ResourceIdentity{
DMIUUID: "uuid-host-1",
Hostnames: []string{"esxi-01.lab.local"},
},
},
{
SourceID: "vc-other:host:host-201",
Resource: Resource{
Type: ResourceTypeAgent,
Name: "esxi-collision.lab.local",
Status: StatusOnline,
LastSeen: now,
VMware: &VMwareData{
ConnectionID: "vc-other",
ConnectionName: "Other VC",
ManagedObjectID: "host-201",
EntityType: "host",
HostUUID: "vc-edit",
},
},
Identity: ResourceIdentity{
DMIUUID: "vc-edit",
Hostnames: []string{"esxi-collision.lab.local"},
},
},
})
preview := PreviewMonitoredSystemRecordsReplacement(rr, MonitoredSystemReplacement{
Source: SourceVMware,
Selector: MonitoredSystemReplacementSelector{
ResourceID: "vc-edit",
},
}, map[DataSource][]IngestRecord{
SourceVMware: {
{
SourceID: "vc-edit:host:host-101",
Resource: Resource{
Type: ResourceTypeAgent,
Name: "esxi-01.lab.local",
Status: StatusOnline,
LastSeen: now,
VMware: &VMwareData{
ConnectionID: "vc-edit",
ConnectionName: "Edited VC",
ManagedObjectID: "host-101",
EntityType: "host",
HostUUID: "uuid-host-1",
},
},
Identity: ResourceIdentity{
DMIUUID: "uuid-host-1",
Hostnames: []string{"esxi-01.lab.local"},
},
},
},
})
if preview.CurrentCount != 2 {
t.Fatalf("CurrentCount = %d, want 2", preview.CurrentCount)
}
if preview.ProjectedCount != 2 {
t.Fatalf("ProjectedCount = %d, want 2", preview.ProjectedCount)
}
if preview.AdditionalCount != 0 {
t.Fatalf("AdditionalCount = %d, want 0", preview.AdditionalCount)
}
if len(preview.CurrentSystems) != 1 {
t.Fatalf("len(CurrentSystems) = %d, want 1", len(preview.CurrentSystems))
}
if preview.CurrentSystems[0].Name != "esxi-01.lab.local" {
t.Fatalf("CurrentSystems[0].Name = %q, want esxi-01.lab.local", preview.CurrentSystems[0].Name)
}
if len(preview.ProjectedSystems) != 1 {
t.Fatalf("len(ProjectedSystems) = %d, want 1", len(preview.ProjectedSystems))
}
if preview.ProjectedSystems[0].Name != "esxi-01.lab.local" {
t.Fatalf("ProjectedSystems[0].Name = %q, want esxi-01.lab.local", preview.ProjectedSystems[0].Name)
}
}
func TestPreviewMonitoredSystemCandidateInactiveKeepsRegistryCountUnchanged(t *testing.T) {
rr := NewRegistry(nil)
rr.IngestRecords(SourceAgent, []IngestRecord{
{
SourceID: "host-1",
Resource: Resource{
ID: "host-1",
Type: ResourceTypeAgent,
Name: "tower.local",
Status: StatusOnline,
Agent: &AgentData{
AgentID: "agent-1",
Hostname: "tower.local",
MachineID: "machine-1",
},
Identity: ResourceIdentity{
MachineID: "machine-1",
Hostnames: []string{"tower.local"},
},
},
},
})
preview := PreviewMonitoredSystemCandidate(rr, MonitoredSystemCandidate{
Source: SourceTrueNAS,
Type: ResourceTypeAgent,
Name: "tower",
Hostname: "tower.local",
HostURL: "https://tower.local",
State: MonitoredSystemCandidateStateInactive,
})
if preview.CurrentCount != 1 || preview.ProjectedCount != 1 || preview.AdditionalCount != 0 {
t.Fatalf("unexpected inactive candidate counts: %+v", preview)
}
if len(preview.CurrentSystems) != 0 || len(preview.ProjectedSystems) != 0 {
t.Fatalf("inactive candidate should not materialize preview systems: %+v", preview)
}
}
func TestPreviewMonitoredSystemCandidateReplacementInactiveRemovesVMwareConnectionFromRegistry(t *testing.T) {
rr := NewRegistry(nil)
rr.IngestRecords(SourceVMware, []IngestRecord{
{
SourceID: "vc-1:host:host-101",
Resource: Resource{
ID: "vmware-host-101",
Type: ResourceTypeAgent,
Name: "esxi-01.lab.local",
Status: StatusOnline,
VMware: &VMwareData{
ConnectionID: "vc-1",
ConnectionName: "Lab VC",
VCenterHost: "vcsa.lab.local",
ManagedObjectID: "host-101",
EntityType: "host",
HostUUID: "host-uuid-101",
},
},
Identity: ResourceIdentity{
DMIUUID: "host-uuid-101",
Hostnames: []string{"esxi-01.lab.local"},
},
},
})
preview := PreviewMonitoredSystemCandidateReplacement(rr, MonitoredSystemReplacement{
Source: SourceVMware,
Selector: MonitoredSystemReplacementSelector{
ResourceID: "vc-1",
},
}, MonitoredSystemCandidate{
Source: SourceVMware,
Type: ResourceTypeAgent,
Name: "Lab VC",
Hostname: "vcsa.lab.local",
HostURL: "https://vcsa.lab.local",
State: MonitoredSystemCandidateStateInactive,
})
if preview.CurrentCount != 1 || preview.ProjectedCount != 0 || preview.AdditionalCount != 0 {
t.Fatalf("unexpected inactive replacement counts: %+v", preview)
}
if preview.CurrentSystem == nil {
t.Fatal("expected current monitored system preview")
}
if preview.ProjectedSystem != nil {
t.Fatalf("inactive replacement should not leave a projected system: %+v", preview.ProjectedSystem)
}
}
func TestResourceRegistry_WorkloadsIncludeCanonicalAppContainers(t *testing.T) {
rr := NewRegistry(nil)
now := time.Date(2026, 3, 29, 12, 0, 0, 0, time.UTC)
rr.IngestRecords(SourceProxmox, []IngestRecord{
{
SourceID: "vm-101",
Resource: Resource{
Type: ResourceTypeVM,
Name: "db-01",
Status: StatusOnline,
LastSeen: now,
},
},
})
rr.IngestRecords(SourceDocker, []IngestRecord{
{
SourceID: "ctr-201",
Resource: Resource{
Type: ResourceTypeAppContainer,
Name: "metrics-sidecar",
Status: StatusOnline,
LastSeen: now,
},
},
})
rr.IngestRecords(SourceTrueNAS, []IngestRecord{
{
SourceID: "app:nextcloud",
Resource: Resource{
Type: ResourceTypeAppContainer,
Name: "Nextcloud",
Status: StatusOnline,
LastSeen: now,
TrueNAS: &TrueNASData{Hostname: "truenas-main"},
Docker: &DockerData{
ContainerID: "nextcloud",
DisplayName: "Nextcloud",
Image: "library/nextcloud:latest",
Runtime: "docker",
ContainerState: "running",
},
},
},
})
workloads := rr.Workloads()
if len(workloads) != 3 {
t.Fatalf("Workloads() returned %d records, want 3", len(workloads))
}
typesByName := make(map[string]ResourceType, len(workloads))
for _, workload := range workloads {
typesByName[workload.Name()] = workload.Type()
}
if got := typesByName["db-01"]; got != ResourceTypeVM {
t.Fatalf("expected VM workload for db-01, got %q", got)
}
if got := typesByName["metrics-sidecar"]; got != ResourceTypeAppContainer {
t.Fatalf("expected app-container workload for metrics-sidecar, got %q", got)
}
if got := typesByName["Nextcloud"]; got != ResourceTypeAppContainer {
t.Fatalf("expected app-container workload for Nextcloud, got %q", got)
}
}
// TestRegistryIngestKeepsDockerContainersScopedToTheirHost pins the fix for the
// "frigate@141" host re-attribution flicker on the Docker page. The bug was
// that app-container source keys did not include the docker host, so any
// scenario producing a colliding container source ID (a docker ps that
// briefly returns an empty ID and parseDockerInventoryContainerLine falls
// back to the container name, a future short-ID truncation, daemon-side
// identifier reset across recreate cycles, ...) routed two containers
// reported by different hosts to the same registry resource. mergeInto
// then overwrote one host's Docker payload with the other's, flipping
// HostSourceID and ParentID on every projection rebuild until the colliding
// inputs diverged again.
func TestRegistryIngestKeepsDockerContainersScopedToTheirHost(t *testing.T) {
rr := NewRegistry(nil)
// Two distinct docker hosts each reporting a container whose docker
// source ID happens to collide (here: both fell back to the "frigate"
// name because their docker ps lines were missing an ID).
rr.IngestSnapshot(models.StateSnapshot{
DockerHosts: []models.DockerHost{
{
ID: "proxmox-lxc-docker:pve-a:node-a:105",
Hostname: "frigate.lab",
Status: "online",
Containers: []models.DockerContainer{{
ID: "frigate",
Name: "frigate",
State: "running",
}},
},
{
ID: "proxmox-lxc-docker:pve-a:node-a:141",
Hostname: "homepage-docker.lab",
Status: "online",
Containers: []models.DockerContainer{{
ID: "frigate",
Name: "frigate",
State: "running",
}},
},
},
})
resources := rr.ListByType(ResourceTypeAppContainer)
if len(resources) != 2 {
t.Fatalf("expected two distinct docker container resources (one per host); got %d", len(resources))
}
hostsSeen := make(map[string]struct{}, len(resources))
for _, resource := range resources {
if resource.Docker == nil {
t.Fatalf("docker container resource missing Docker payload: %#v", resource)
}
if resource.Name != "frigate" {
t.Fatalf("unexpected container name: %q", resource.Name)
}
if !reflect.DeepEqual(resource.PlatformScopes, []string{"proxmox-pve", "docker"}) {
t.Fatalf("platform scopes = %#v, want proxmox-pve + docker", resource.PlatformScopes)
}
hostsSeen[strings.TrimSpace(resource.Docker.HostSourceID)] = struct{}{}
}
for _, want := range []string{
"proxmox-lxc-docker:pve-a:node-a:105",
"proxmox-lxc-docker:pve-a:node-a:141",
} {
if _, ok := hostsSeen[want]; !ok {
t.Fatalf("expected a container attributed to host %q; got hosts %v", want, hostsSeen)
}
}
}
func TestRegistryIngestSnapshotPublishesDockerNativeInventory(t *testing.T) {
now := time.Date(2026, 5, 24, 8, 0, 0, 0, time.UTC)
rr := NewRegistry(NewMemoryStore())
rr.IngestSnapshot(models.StateSnapshot{
DockerHosts: []models.DockerHost{{
ID: "docker-host-1",
Hostname: "edge",
Runtime: "docker",
Status: "online",
LastSeen: now,
Images: []models.DockerImage{{
ID: "sha256:image1", RepoTags: []string{"repo/app:latest"}, SizeBytes: 1024,
}},
Volumes: []models.DockerVolume{{Name: "app-data", Driver: "local", SizeBytes: 2048}},
Networks: []models.DockerNetwork{{
ID: "net1", Name: "app-net", Driver: "bridge", Subnets: []models.DockerNetworkSubnet{{Subnet: "10.88.0.0/24", Gateway: "10.88.0.1"}},
}},
Tasks: []models.DockerTask{{
ID: "task-1", ServiceID: "svc-1", ServiceName: "api", Slot: 1, DesiredState: "running", CurrentState: "running",
}},
Secrets: []models.DockerSecret{{ID: "secret-1", Name: "db-password", DriverName: "builtin"}},
Configs: []models.DockerConfig{{ID: "config-1", Name: "nginx-conf", TemplatingDriver: "golang"}},
Swarm: &models.DockerSwarmInfo{
NodeID: "node-1",
NodeRole: "manager",
LocalState: "active",
},
Nodes: []models.DockerNode{{
ID: "node-1", Hostname: "manager-1", Role: "manager", State: "ready",
}},
}},
})
counts := map[ResourceType]int{}
for _, resource := range rr.List() {
counts[resource.Type]++
}
for _, resourceType := range []ResourceType{
ResourceTypeDockerImage,
ResourceTypeDockerVolume,
ResourceTypeDockerNetwork,
ResourceTypeDockerTask,
ResourceTypeDockerSwarmNode,
ResourceTypeDockerSecret,
ResourceTypeDockerConfig,
} {
if counts[resourceType] != 1 {
t.Fatalf("expected one %s resource, got counts %#v", resourceType, counts)
}
}
}
func TestRegistryIngestSnapshotLinksDockerContainersToNetworks(t *testing.T) {
now := time.Date(2026, 6, 3, 9, 0, 0, 0, time.UTC)
rr := NewRegistry(NewMemoryStore())
rr.IngestSnapshot(models.StateSnapshot{
DockerHosts: []models.DockerHost{{
ID: "docker-host-1",
Hostname: "edge",
Runtime: "docker",
Status: "online",
LastSeen: now,
Containers: []models.DockerContainer{{
ID: "container-1",
Name: "api",
Image: "repo/api:latest",
State: "running",
Networks: []models.DockerContainerNetworkLink{{
Name: "frontend",
IPv4: "10.88.0.12",
}},
}},
Networks: []models.DockerNetwork{{
ID: "network-1",
Name: "frontend",
Driver: "bridge",
}},
}},
})
containers := rr.ListByType(ResourceTypeAppContainer)
networks := rr.ListByType(ResourceTypeDockerNetwork)
if len(containers) != 1 || len(networks) != 1 {
t.Fatalf("expected one container and one network, got containers=%d networks=%d", len(containers), len(networks))
}
container := containers[0]
network := networks[0]
assertDockerNetworkAttachment := func(resource Resource) {
t.Helper()
for _, relationship := range resource.Relationships {
if relationship.Type != RelAttachedTo {
continue
}
if relationship.SourceID != container.ID || relationship.TargetID != network.ID {
t.Fatalf("unexpected attachment relationship endpoints on %s: %+v", resource.ID, relationship)
}
if relationship.Discoverer != dockerAdapterRelationshipDiscoverer {
t.Fatalf("relationship discoverer = %q, want %q", relationship.Discoverer, dockerAdapterRelationshipDiscoverer)
}
if relationship.Metadata["networkName"] != "frontend" || relationship.Metadata["ipv4"] != "10.88.0.12" {
t.Fatalf("relationship metadata = %#v", relationship.Metadata)
}
return
}
t.Fatalf("expected docker network attachment relationship on %s, got %+v", resource.ID, resource.Relationships)
}
assertDockerNetworkAttachment(container)
assertDockerNetworkAttachment(network)
}
// A live source must override a metric held by a stale higher-priority source.
func TestMergeMetric_LiveSourceOverridesStaleHigherPriority(t *testing.T) {
now := time.Now().UTC()
status := map[DataSource]SourceStatus{
SourceAgent: {Status: "stale", LastSeen: now.Add(-2 * time.Hour)},
SourceProxmox: {Status: "online", LastSeen: now},
}
existing := &MetricValue{Value: 0, Percent: 0, Source: SourceAgent}
incoming := &MetricValue{Value: 6.2, Percent: 6.2, Source: SourceProxmox}
got := mergeMetric(&Resource{Type: ResourceTypeAgent}, existing, incoming, SourceProxmox, now, status, nil)
if got == nil || got.Percent != 6.2 || got.Source != SourceProxmox {
t.Fatalf("expected live proxmox CPU to override stale agent CPU, got %+v", got)
}
}
// When both sources are fresh, static priority still decides: the agent wins
// on a host-shaped resource (its own kernel, its own numbers).
func TestMergeMetric_FreshHigherPriorityStillWins(t *testing.T) {
now := time.Now().UTC()
status := map[DataSource]SourceStatus{
SourceAgent: {Status: "online", LastSeen: now},
SourceProxmox: {Status: "online", LastSeen: now},
}
existing := &MetricValue{Percent: 6.2, Source: SourceProxmox}
incoming := &MetricValue{Percent: 12.5, Source: SourceAgent}
got := mergeMetric(&Resource{Type: ResourceTypeAgent}, existing, incoming, SourceAgent, now, status, nil)
if got == nil || got.Percent != 12.5 || got.Source != SourceAgent {
t.Fatalf("expected fresh agent CPU to win by priority, got %+v", got)
}
}
// Issue #1597: an agent inside an LXC measures through the shared kernel and
// reports the node's CPU, not the container's. On a hypervisor-managed guest
// the platform metric must survive an agent merge even though the agent
// outranks Proxmox on host-shaped resources.
func TestMergeCPUMetric_GuestKeepsPlatformCPUOverInGuestAgent(t *testing.T) {
now := time.Now().UTC()
status := map[DataSource]SourceStatus{
SourceAgent: {Status: "online", LastSeen: now},
SourceProxmox: {Status: "online", LastSeen: now},
}
guest := &Resource{
Type: ResourceTypeSystemContainer,
Proxmox: &ProxmoxData{VMID: 301, ContainerType: "lxc"},
}
existing := &MetricValue{Percent: 0.58, Source: SourceProxmox}
incoming := &MetricValue{Percent: 58, Source: SourceAgent}
got := mergeCPUMetric(guest, existing, incoming, SourceAgent, now, status, nil)
if got == nil || got.Percent != 0.58 || got.Source != SourceProxmox {
t.Fatalf("expected platform CPU to survive in-guest agent merge, got %+v", got)
}
}
// Reverse ingest order for the same guest: the platform metric arrives while
// the agent's value is already in place. The platform must take over.
func TestMergeCPUMetric_GuestPlatformOverridesEarlierAgentValue(t *testing.T) {
now := time.Now().UTC()
status := map[DataSource]SourceStatus{
SourceAgent: {Status: "online", LastSeen: now},
SourceProxmox: {Status: "online", LastSeen: now},
}
guest := &Resource{
Type: ResourceTypeSystemContainer,
Proxmox: &ProxmoxData{VMID: 301, ContainerType: "lxc"},
}
existing := &MetricValue{Percent: 58, Source: SourceAgent}
incoming := &MetricValue{Percent: 0.58, Source: SourceProxmox}
got := mergeCPUMetric(guest, existing, incoming, SourceProxmox, now, status, nil)
if got == nil || got.Percent != 0.58 || got.Source != SourceProxmox {
t.Fatalf("expected platform CPU to override earlier agent value on guest, got %+v", got)
}
}
// Source freshness is reported independently from semantic authority. A live
// in-guest agent must not replace a stale platform CPU observation because the
// two values use different accounting and history remains platform-keyed.
func TestMergeCPUMetric_GuestRetainsStalePlatformObservation(t *testing.T) {
now := time.Now().UTC()
status := map[DataSource]SourceStatus{
SourceAgent: {Status: "online", LastSeen: now},
SourceProxmox: {Status: "stale", LastSeen: now.Add(-2 * time.Hour)},
}
guest := &Resource{
Type: ResourceTypeSystemContainer,
Proxmox: &ProxmoxData{VMID: 301, ContainerType: "lxc"},
}
existing := &MetricValue{Percent: 0.58, Source: SourceProxmox}
incoming := &MetricValue{Percent: 12.5, Source: SourceAgent}
got := mergeCPUMetric(guest, existing, incoming, SourceAgent, now, status, nil)
if got == nil || got.Percent != 0.58 || got.Source != SourceProxmox {
t.Fatalf("expected stale platform CPU to remain authoritative, got %+v", got)
}
}
// A stale source must not clobber a metric currently held by a live source.
func TestMergeMetric_StaleSourceDoesNotClobberLive(t *testing.T) {
now := time.Now().UTC()
status := map[DataSource]SourceStatus{
SourceAgent: {Status: "stale", LastSeen: now.Add(-2 * time.Hour)},
SourceProxmox: {Status: "online", LastSeen: now},
}
existing := &MetricValue{Percent: 6.2, Source: SourceProxmox}
incoming := &MetricValue{Percent: 0, Source: SourceAgent}
got := mergeMetric(&Resource{Type: ResourceTypeAgent}, existing, incoming, SourceAgent, now, status, nil)
if got == nil || got.Percent != 6.2 || got.Source != SourceProxmox {
t.Fatalf("expected live proxmox CPU to survive stale agent merge, got %+v", got)
}
}
func TestResourceRegistry_ManualGuestAgentLinksPreserveMetricAuthority(t *testing.T) {
now := time.Now().UTC()
type guestShape struct {
name string
resourceType ResourceType
technology string
targetType string
containerType string
guestID string
guestSourceID string
guestSourceCPU float64
}
guestShapes := []guestShape{
{
name: "lxc",
resourceType: ResourceTypeSystemContainer,
technology: "lxc",
targetType: "system-container",
containerType: "lxc",
guestID: "system-container-cluster-a-node-1-301",
guestSourceID: "cluster-a-node-1-301",
guestSourceCPU: 0.58,
},
{
name: "qemu",
resourceType: ResourceTypeVM,
technology: "qemu",
targetType: "vm",
guestID: "vm-cluster-a-node-1-302",
guestSourceID: "cluster-a-node-1-302",
guestSourceCPU: 37.5,
},
}
freshnessCases := []struct {
name string
platformLastSeen time.Time
agentLastSeen time.Time
thresholds map[DataSource]time.Duration
wantMemorySource DataSource
}{
{
name: "both-fresh",
platformLastSeen: now,
agentLastSeen: now,
wantMemorySource: SourceProxmox,
},
{
name: "platform-stale-agent-fresh",
platformLastSeen: now.Add(-2 * time.Hour),
agentLastSeen: now,
wantMemorySource: SourceAgent,
},
{
name: "platform-fresh-agent-stale",
platformLastSeen: now,
agentLastSeen: now.Add(-2 * time.Hour),
wantMemorySource: SourceProxmox,
},
{
name: "both-stale",
platformLastSeen: now.Add(-2 * time.Hour),
agentLastSeen: now.Add(-2 * time.Hour),
wantMemorySource: SourceProxmox,
},
{
name: "caller-threshold-keeps-platform-fresh",
platformLastSeen: now.Add(-2 * time.Minute),
agentLastSeen: now,
thresholds: map[DataSource]time.Duration{SourceProxmox: 5 * time.Minute},
wantMemorySource: SourceProxmox,
},
}
for _, shape := range guestShapes {
for _, freshness := range freshnessCases {
for _, primaryIsAgent := range []bool{false, true} {
direction := "guest-primary"
if primaryIsAgent {
direction = "agent-primary"
}
t.Run(shape.name+"/"+freshness.name+"/"+direction, func(t *testing.T) {
const agentID = "agent-inside-guest"
primaryID := shape.guestID
if primaryIsAgent {
primaryID = agentID
}
store := NewMemoryStore()
if err := store.AddLink(ResourceLink{
ResourceA: shape.guestID,
ResourceB: agentID,
PrimaryID: primaryID,
Reason: "same guest",
}); err != nil {
t.Fatalf("add link: %v", err)
}
rr := NewRegistry(store)
rr.IngestResourcesWithStaleThresholds([]Resource{
{
ID: shape.guestID,
Type: shape.resourceType,
Technology: shape.technology,
Name: shape.name + "-guest",
Status: StatusOnline,
LastSeen: freshness.platformLastSeen,
Sources: []DataSource{SourceProxmox},
SourceStatus: map[DataSource]SourceStatus{
SourceProxmox: {Status: "online", LastSeen: freshness.platformLastSeen},
},
Proxmox: &ProxmoxData{
SourceID: shape.guestSourceID,
Instance: "pve-a",
ClusterName: "cluster-a",
NodeName: "node-1",
VMID: 301,
ContainerType: shape.containerType,
CPUs: 8,
},
Metrics: &ResourceMetrics{
CPU: &MetricValue{Percent: shape.guestSourceCPU, Source: SourceProxmox},
Memory: &MetricValue{Percent: 40, Source: SourceProxmox},
},
},
{
ID: agentID,
Type: ResourceTypeAgent,
Name: shape.name + "-guest",
Status: StatusOnline,
LastSeen: freshness.agentLastSeen,
Sources: []DataSource{SourceAgent},
SourceStatus: map[DataSource]SourceStatus{
SourceAgent: {Status: "online", LastSeen: freshness.agentLastSeen},
},
Agent: &AgentData{AgentID: agentID, Hostname: shape.name + "-guest", CPUCount: 8},
Metrics: &ResourceMetrics{
CPU: &MetricValue{Percent: 94, Source: SourceAgent},
Memory: &MetricValue{Percent: 70, Source: SourceAgent},
NetOut: &MetricValue{Value: 4096, Unit: "bytes/s", Source: SourceAgent},
},
},
}, freshness.thresholds)
resources := rr.List()
if len(resources) != 1 {
t.Fatalf("resources = %d, want one merged guest: %+v", len(resources), resources)
}
got := resources[0]
if got.ID != shape.guestID || got.Type != shape.resourceType {
t.Fatalf("merged resource = %s/%s, want guest %s/%s", got.ID, got.Type, shape.guestID, shape.resourceType)
}
if got.Proxmox == nil || got.Agent == nil {
t.Fatalf("merged guest lost provider payloads: proxmox=%+v agent=%+v", got.Proxmox, got.Agent)
}
if got.Metrics == nil || got.Metrics.CPU == nil {
t.Fatalf("merged guest lost CPU metrics: %+v", got.Metrics)
}
if got.Metrics.CPU.Percent != shape.guestSourceCPU || got.Metrics.CPU.Source != SourceProxmox {
t.Fatalf("CPU = %+v, want authoritative proxmox %.2f", got.Metrics.CPU, shape.guestSourceCPU)
}
if got.Metrics.Memory == nil || got.Metrics.Memory.Source != freshness.wantMemorySource {
t.Fatalf("memory = %+v, want source %s", got.Metrics.Memory, freshness.wantMemorySource)
}
if got.Metrics.NetOut == nil || got.Metrics.NetOut.Value != 4096 || got.Metrics.NetOut.Source != SourceAgent {
t.Fatalf("agent-only network metric was not preserved: %+v", got.Metrics.NetOut)
}
target := BuildMetricsTargetForRegistry(rr, got.ID)
if target == nil || target.ResourceType != shape.targetType || target.ResourceID != shape.guestSourceID {
t.Fatalf("metrics target = %+v, want %s/%s", target, shape.targetType, shape.guestSourceID)
}
})
}
}
}
}
func TestResourceRegistry_ManualGuestAgentLinkUsesAgentCPUOnlyWhenPlatformHasNoCPU(t *testing.T) {
now := time.Now().UTC()
const (
guestID = "vm-cluster-a-node-1-401"
agentID = "agent-inside-vm-401"
)
store := NewMemoryStore()
if err := store.AddLink(ResourceLink{
ResourceA: guestID,
ResourceB: agentID,
PrimaryID: agentID,
}); err != nil {
t.Fatalf("add link: %v", err)
}
rr := NewRegistry(store)
rr.IngestResources([]Resource{
{
ID: guestID,
Type: ResourceTypeVM,
Name: "vm-401",
Status: StatusOnline,
LastSeen: now,
Sources: []DataSource{SourceProxmox},
SourceStatus: map[DataSource]SourceStatus{
SourceProxmox: {Status: "online", LastSeen: now},
},
Proxmox: &ProxmoxData{SourceID: "cluster-a-node-1-401", VMID: 401},
Metrics: &ResourceMetrics{
Memory: &MetricValue{Percent: 50, Source: SourceProxmox},
},
},
{
ID: agentID,
Type: ResourceTypeAgent,
Name: "vm-401",
Status: StatusOnline,
LastSeen: now,
Sources: []DataSource{SourceAgent},
SourceStatus: map[DataSource]SourceStatus{
SourceAgent: {Status: "online", LastSeen: now},
},
Agent: &AgentData{AgentID: agentID},
Metrics: &ResourceMetrics{
CPU: &MetricValue{Percent: 33, Source: SourceAgent},
},
},
})
got, ok := rr.Get(guestID)
if !ok || got.Metrics == nil || got.Metrics.CPU == nil {
t.Fatalf("merged guest missing fallback CPU: %+v", got)
}
if got.Metrics.CPU.Percent != 33 || got.Metrics.CPU.Source != SourceAgent {
t.Fatalf("CPU = %+v, want agent fallback when platform observation is absent", got.Metrics.CPU)
}
}
// The workload table, guest drawer and History judge a guest's disk reading by
// the linked agent's freshness when the agent's filesystems fill in for
// Proxmox's (useWorkloads disksFromAgent). That needs the registry to keep the
// agent's disk metric while Proxmox has no guest disk usage, also once the
// agent is silent, and Proxmox's own fresh reading otherwise.
func TestResourceRegistry_ManualGuestAgentLinkDiskMetricFollowsProxmoxGuestFilesystems(t *testing.T) {
now := time.Now().UTC()
agentDisks := []models.Disk{{Total: 1000, Used: 400, Free: 600, Usage: 40, Mountpoint: "/"}}
cases := []struct {
name string
vmDisk models.Disk
vmDisks []models.Disk
reason string
agentStatus string
agentLastSeen time.Time
wantSource DataSource
wantPercent float64
wantStale bool
}{
{
name: "no proxmox filesystems, live agent",
vmDisk: models.Disk{Total: 32 << 30, Usage: -1},
reason: "agent-not-running",
agentStatus: "online",
agentLastSeen: now,
wantSource: SourceAgent,
wantPercent: 40,
},
{
name: "no proxmox filesystems, silent agent keeps its last disk",
vmDisk: models.Disk{Total: 32 << 30, Usage: -1},
reason: "agent-not-running",
agentStatus: "offline",
agentLastSeen: now.Add(-10 * time.Minute),
wantSource: SourceAgent,
wantPercent: 40,
wantStale: true,
},
{
name: "proxmox guest filesystems keep proxmox disk",
vmDisk: models.Disk{Total: 1000, Used: 700, Free: 300, Usage: 70},
vmDisks: []models.Disk{{Total: 1000, Used: 700, Free: 300, Usage: 70, Mountpoint: "/"}},
agentStatus: "online",
agentLastSeen: now,
wantSource: SourceProxmox,
wantPercent: 70,
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
snapshot := models.StateSnapshot{
VMs: []models.VM{{
ID: "pve-a:node-1:402",
VMID: 402,
Name: "vm-402",
Node: "node-1",
Instance: "pve-a",
Type: "qemu",
Status: "running",
LastSeen: now,
Disk: tc.vmDisk,
Disks: tc.vmDisks,
DiskStatusReason: tc.reason,
}},
Hosts: []models.Host{{
ID: "agent-402",
Hostname: "guest-402",
Status: tc.agentStatus,
LastSeen: tc.agentLastSeen,
Disks: agentDisks,
}},
}
store := NewMemoryStore()
unlinked := NewRegistry(store)
unlinked.IngestSnapshot(snapshot)
var vmID, agentID string
for _, resource := range unlinked.List() {
switch resource.Type {
case ResourceTypeVM:
vmID = resource.ID
case ResourceTypeAgent:
agentID = resource.ID
}
}
if vmID == "" || agentID == "" {
t.Fatalf("vm %q / agent %q not ingested", vmID, agentID)
}
if err := store.AddLink(ResourceLink{ResourceA: vmID, ResourceB: agentID, PrimaryID: vmID}); err != nil {
t.Fatalf("add link: %v", err)
}
rr := NewRegistry(store)
rr.IngestSnapshot(snapshot)
got, ok := rr.Get(vmID)
if !ok || got.Metrics == nil || got.Metrics.Disk == nil {
t.Fatalf("linked guest lost its disk metric: %+v", got)
}
if got.Metrics.Disk.Source != tc.wantSource || got.Metrics.Disk.Percent != tc.wantPercent {
t.Fatalf("disk = %+v, want %s at %.0f%%", got.Metrics.Disk, tc.wantSource, tc.wantPercent)
}
agentFilesystemsFillIn := got.Proxmox != nil && len(got.Proxmox.Disks) == 0 &&
got.Agent != nil && len(got.Agent.Disks) > 0
if agentFilesystemsFillIn != (tc.wantSource == SourceAgent) {
t.Fatalf("agent filesystems fill in = %v, disk metric source %s", agentFilesystemsFillIn, tc.wantSource)
}
if got.Agent == nil || got.Agent.Stale != tc.wantStale {
t.Fatalf("agent stale = %+v, want %v", got.Agent, tc.wantStale)
}
})
}
}
func TestResourceRegistry_ManualGuestAgentLinkReconnectRestoresFreshPlatformCPU(t *testing.T) {
now := time.Now().UTC()
const (
guestID = "system-container-cluster-a-node-1-451"
guestSourceID = "cluster-a-node-1-451"
agentID = "agent-inside-lxc-451"
)
store := NewMemoryStore()
if err := store.AddLink(ResourceLink{
ResourceA: guestID,
ResourceB: agentID,
PrimaryID: agentID,
}); err != nil {
t.Fatalf("add link: %v", err)
}
rr := NewRegistry(store)
ingest := func(platformCPU float64, platformLastSeen time.Time) {
t.Helper()
rr.IngestResources([]Resource{
{
ID: guestID,
Type: ResourceTypeSystemContainer,
Name: "lxc-451",
Status: StatusOnline,
LastSeen: platformLastSeen,
Sources: []DataSource{SourceProxmox},
SourceStatus: map[DataSource]SourceStatus{
SourceProxmox: {Status: "online", LastSeen: platformLastSeen},
},
Proxmox: &ProxmoxData{SourceID: guestSourceID, VMID: 451, ContainerType: "lxc"},
Metrics: &ResourceMetrics{
CPU: &MetricValue{Percent: platformCPU, Source: SourceProxmox},
},
},
{
ID: agentID,
Type: ResourceTypeAgent,
Name: "lxc-451",
Status: StatusOnline,
LastSeen: now,
Sources: []DataSource{SourceAgent},
SourceStatus: map[DataSource]SourceStatus{
SourceAgent: {Status: "online", LastSeen: now},
},
Agent: &AgentData{AgentID: agentID},
Metrics: &ResourceMetrics{
CPU: &MetricValue{Percent: 94, Source: SourceAgent},
},
},
})
}
ingest(0.58, now.Add(-2*time.Hour))
first, ok := rr.Get(guestID)
if !ok || first.Metrics == nil || first.Metrics.CPU == nil {
t.Fatalf("first merged guest missing CPU: %+v", first)
}
if first.Metrics.CPU.Percent != 0.58 || first.Metrics.CPU.Source != SourceProxmox {
t.Fatalf("stale platform CPU = %+v, want proxmox 0.58", first.Metrics.CPU)
}
ingest(12.5, now.Add(time.Second))
reconnected, ok := rr.Get(guestID)
if !ok || reconnected.Metrics == nil || reconnected.Metrics.CPU == nil {
t.Fatalf("reconnected guest missing CPU: %+v", reconnected)
}
if reconnected.Metrics.CPU.Percent != 12.5 || reconnected.Metrics.CPU.Source != SourceProxmox {
t.Fatalf("reconnected CPU = %+v, want fresh proxmox 12.5", reconnected.Metrics.CPU)
}
target := BuildMetricsTargetForRegistry(rr, guestID)
if target == nil || target.ResourceType != "system-container" || target.ResourceID != guestSourceID {
t.Fatalf("reconnected metrics target = %+v", target)
}
}
func TestResourceRegistry_LinkHintsKeepClusterGuestsDistinctAndPlatformAuthoritative(t *testing.T) {
now := time.Now().UTC()
rr := NewRegistry(nil)
rr.IngestSnapshot(models.StateSnapshot{
Nodes: []models.Node{
{ID: "cluster-a-node-1", Name: "node-1", Instance: "pve-a", ClusterName: "cluster-a", IsClusterMember: true, Status: "online", LastSeen: now},
{ID: "cluster-b-node-1", Name: "node-1", Instance: "pve-b", ClusterName: "cluster-b", IsClusterMember: true, Status: "online", LastSeen: now},
},
VMs: []models.VM{
{ID: "cluster-a-node-1-501", VMID: 501, Name: "duplicate-vmid", Node: "node-1", Instance: "pve-a", Status: "running", CPU: 0.0058, CPUs: 1, LastSeen: now},
{ID: "cluster-b-node-1-501", VMID: 501, Name: "duplicate-vmid", Node: "node-1", Instance: "pve-b", Status: "running", CPU: 0.375, CPUs: 8, LastSeen: now},
},
Containers: []models.Container{
{ID: "cluster-a-node-1-601", VMID: 601, Name: "lxc-with-agent", Node: "node-1", Instance: "pve-a", Type: "lxc", Status: "running", CPU: 0.0058, CPUs: 4, LastSeen: now},
},
Hosts: []models.Host{
{ID: "agent-vm-501", Hostname: "duplicate-vmid", LinkedVMID: "cluster-a-node-1-501", CPUUsage: 94, Status: "online", LastSeen: now},
{ID: "agent-lxc-601", Hostname: "lxc-with-agent", LinkedContainerID: "cluster-a-node-1-601", CPUUsage: 88, Status: "online", LastSeen: now},
},
})
vms := rr.ListByType(ResourceTypeVM)
if len(vms) != 2 {
t.Fatalf("VMs = %d, want two distinct cluster-scoped identities: %+v", len(vms), vms)
}
wantVMCPU := map[string]float64{
"cluster-a-node-1-501": 0.58,
"cluster-b-node-1-501": 37.5,
}
for _, vm := range vms {
if vm.Proxmox == nil || vm.Metrics == nil || vm.Metrics.CPU == nil {
t.Fatalf("VM missing proxmox CPU: %+v", vm)
}
want, ok := wantVMCPU[vm.Proxmox.SourceID]
if !ok {
t.Fatalf("unexpected VM source identity: %+v", vm.Proxmox)
}
if vm.Metrics.CPU.Percent != want || vm.Metrics.CPU.Source != SourceProxmox {
t.Fatalf("VM %s CPU = %+v, want proxmox %.2f", vm.Proxmox.SourceID, vm.Metrics.CPU, want)
}
}
containers := rr.ListByType(ResourceTypeSystemContainer)
if len(containers) != 1 || containers[0].Metrics == nil || containers[0].Metrics.CPU == nil {
t.Fatalf("LXC resources = %+v, want one with CPU", containers)
}
if containers[0].Metrics.CPU.Percent != 0.58 || containers[0].Metrics.CPU.Source != SourceProxmox {
t.Fatalf("LXC CPU = %+v, want proxmox 0.58", containers[0].Metrics.CPU)
}
hostAgentCount := 0
for _, resource := range rr.ListByType(ResourceTypeAgent) {
if resource.Agent != nil {
hostAgentCount++
}
}
if hostAgentCount != 2 {
t.Fatalf("link hints must not implicitly merge guest identities; host agents = %d", hostAgentCount)
}
}
// The merged-source host shape from the homelab canonical-ID bug: a PVE node
// record that only knows cluster+hostname, and a pulse-agent record that
// knows the machine ID. Whichever record mints the canonical resource decides
// which identity key gets hashed, so without durable identity pins the
// canonical ID flips between boot windows (node-only) and steady state
// (agent present), fragmenting the change journal into per-boot eras.
func mergedHostNodeResource() Resource {
return Resource{
Type: ResourceTypeAgent,
Name: "delly",
Status: StatusOnline,
Proxmox: &ProxmoxData{SourceID: "homelab-delly", NodeName: "delly", ClusterName: "homelab"},
}
}
func mergedHostAgentResource(machineID string) Resource {
return Resource{
Type: ResourceTypeAgent,
Name: "delly",
Status: StatusOnline,
Agent: &AgentData{AgentID: machineID, Hostname: "delly", MachineID: machineID},
}
}
func TestMergedHostCanonicalIDStableAcrossRestartsAndIngestOrders(t *testing.T) {
store := NewMemoryStore()
const machineID = "7d465a78-test-machine-id"
nodeIdentity := ResourceIdentity{Hostnames: []string{"delly"}, ClusterName: "homelab"}
agentIdentity := ResourceIdentity{MachineID: machineID, Hostnames: []string{"delly"}}
// IngestSnapshot merges the linked host's identity into the node record
// when the agent is present in the snapshot.
steadyIdentity := mergeIdentity(nodeIdentity, agentIdentity)
steadyID := buildHashID(ResourceTypeAgent, "machine:"+machineID)
bootEraID := buildHashID(ResourceTypeAgent, "cluster:homelab:delly")
if steadyID == bootEraID {
t.Fatalf("test setup broken: era IDs must differ")
}
// Steady-state boot: node record carries the merged identity and mints
// the machine-keyed canonical ID; the agent record merges into it.
steady := NewRegistry(store)
if id := steady.ingest(SourceProxmox, "homelab-delly", mergedHostNodeResource(), steadyIdentity); id != steadyID {
t.Fatalf("steady-state node ingest minted %q, want machine-keyed %q", id, steadyID)
}
if id := steady.ingest(SourceAgent, machineID, mergedHostAgentResource(machineID), agentIdentity); id != steadyID {
t.Fatalf("steady-state agent ingest resolved %q, want %q", id, steadyID)
}
steady.PersistIdentityPins()
// Restart into a boot window: the agent has not checked in yet, so the
// node record only knows cluster+hostname. Before identity pins this
// minted bootEraID and the change journal fragmented into a second era.
bootWindow := NewRegistry(store)
if id := bootWindow.ingest(SourceProxmox, "homelab-delly", mergedHostNodeResource(), nodeIdentity); id != steadyID {
t.Fatalf("boot-window node ingest minted %q, want pinned %q", id, steadyID)
}
// Restart with the opposite ingest order: agent record first into an
// empty registry, node record after.
reversed := NewRegistry(store)
if id := reversed.ingest(SourceAgent, machineID, mergedHostAgentResource(machineID), agentIdentity); id != steadyID {
t.Fatalf("agent-first ingest minted %q, want %q", id, steadyID)
}
if id := reversed.ingest(SourceProxmox, "homelab-delly", mergedHostNodeResource(), nodeIdentity); id != steadyID {
t.Fatalf("node-after-agent ingest resolved %q, want %q", id, steadyID)
}
if got := len(reversed.List()); got != 1 {
t.Fatalf("expected one merged resource after reversed-order ingest, got %d", got)
}
}
// The end-to-end #1559 scenario: multiple standalone agents with dotted
// hostnames (cloud.rnd-lax1, cloud.gce-or1, cloud.dmi-lax1) must mint
// distinct canonical resources, persist full dotted hostnames in their
// identity pins instead of collapsing them all to "cloud", and boot-window
// records that only know the hostname must resolve to the right machine
// instead of the first pinned one.
func TestStandaloneDottedHostnameAgentsStayDistinct(t *testing.T) {
store := NewMemoryStore()
hosts := []struct {
hostname string
machineID string
}{
{"cloud.rnd-lax1", "machine-rnd"},
{"cloud.gce-or1", "machine-gce"},
{"cloud.dmi-lax1", "machine-dmi"},
}
agentResource := func(hostname, machineID string) Resource {
return Resource{
Type: ResourceTypeAgent,
Name: hostname,
Status: StatusOnline,
Agent: &AgentData{AgentID: machineID, Hostname: hostname, MachineID: machineID},
}
}
steady := NewRegistry(store)
idByHostname := make(map[string]string, len(hosts))
for _, host := range hosts {
identity := ResourceIdentity{MachineID: host.machineID, Hostnames: []string{host.hostname}}
id := steady.ingest(SourceAgent, host.machineID, agentResource(host.hostname, host.machineID), identity)
if id == "" {
t.Fatalf("agent ingest for %q returned no ID", host.hostname)
}
for hostname, existingID := range idByHostname {
if existingID == id {
t.Fatalf("hosts %q and %q collapsed to canonical ID %q", hostname, host.hostname, id)
}
}
idByHostname[host.hostname] = id
}
if got := len(steady.List()); got != len(hosts) {
t.Fatalf("expected %d distinct resources, got %d", len(hosts), got)
}
steady.PersistIdentityPins()
pins, err := store.ListResourceIdentityPins()
if err != nil {
t.Fatalf("ListResourceIdentityPins: %v", err)
}
pinnedHostnames := make(map[string]struct{}, len(pins))
for _, pin := range pins {
pinnedHostnames[pin.Hostname] = struct{}{}
}
for _, host := range hosts {
if _, ok := pinnedHostnames[host.hostname]; !ok {
t.Fatalf("expected pinned primary hostname %q, got %v", host.hostname, pinnedHostnames)
}
}
// Boot window: runtime records that only know the hostname (no machine
// ID yet) arrive before the agents. Each must complete from its own pin
// and land on its own canonical ID.
boot := NewRegistry(store)
for _, host := range hosts {
record := Resource{
Type: ResourceTypeAgent,
Name: host.hostname,
Status: StatusOnline,
Docker: &DockerData{HostSourceID: "docker:" + host.hostname, Hostname: host.hostname},
}
identity := ResourceIdentity{Hostnames: []string{host.hostname}}
id := boot.ingest(SourceDocker, "docker:"+host.hostname, record, identity)
if want := idByHostname[host.hostname]; id != want {
t.Fatalf("boot-window record for %q resolved %q, want its own canonical ID %q", host.hostname, id, want)
}
}
for _, host := range hosts {
identity := ResourceIdentity{MachineID: host.machineID, Hostnames: []string{host.hostname}}
id := boot.ingest(SourceAgent, host.machineID, agentResource(host.hostname, host.machineID), identity)
if want := idByHostname[host.hostname]; id != want {
t.Fatalf("agent ingest for %q resolved %q, want %q", host.hostname, id, want)
}
}
if got := len(boot.List()); got != len(hosts) {
t.Fatalf("expected %d merged resources after boot-window ingest, got %d", len(hosts), got)
}
}
// A synthesized placeholder (e.g. an offline PVE node for an instance that
// has never completed a poll) carries a zero LastSeen. Ingest must preserve
// that zero instead of fabricating a fresh sighting, and the per-source
// delivery status must be "unknown", not "online" — an online stamp with a
// zero LastSeen is permanently exempt from stale-marking and masks the
// source's real delivery state (the loop behind commit 8372a22c5).
func TestIngestPreservesNeverSeenSightingAndStampsUnknownSourceStatus(t *testing.T) {
rr := NewRegistry(nil)
placeholder := topLevelTestProxmoxNode("proxmox-node", "tower", "proxmox-1", "https://tower.local:8006")
placeholder.Status = StatusOffline
placeholder.LastSeen = time.Time{}
rr.IngestRecords(SourceProxmox, []IngestRecord{
{SourceID: "proxmox-node", Resource: placeholder},
})
resources := rr.List()
if len(resources) != 1 {
t.Fatalf("List() returned %d resources, want 1", len(resources))
}
resource := resources[0]
if !resource.LastSeen.IsZero() {
t.Fatalf("resource.LastSeen = %s, want zero: never-seen resources must not get a fabricated sighting", resource.LastSeen)
}
status, ok := resource.SourceStatus[SourceProxmox]
if !ok {
t.Fatal("expected a proxmox source status entry")
}
if status.Status != "unknown" {
t.Fatalf("SourceStatus[proxmox].Status = %q, want %q: a source that never delivered the resource must not be stamped online", status.Status, "unknown")
}
if !status.LastSeen.IsZero() {
t.Fatalf("SourceStatus[proxmox].LastSeen = %s, want zero", status.LastSeen)
}
// Stale marking must leave a never-seen source alone in both directions.
rr.MarkStale(time.Now().UTC(), nil)
if got := rr.List()[0].SourceStatus[SourceProxmox].Status; got != "unknown" {
t.Fatalf("SourceStatus[proxmox].Status after MarkStale = %q, want %q", got, "unknown")
}
}
// A real sighting keeps the online stamp and the genuine timestamp on both
// the resource and the per-source delivery status.
func TestIngestStampsOnlineSourceStatusForRealSighting(t *testing.T) {
rr := NewRegistry(nil)
seen := time.Now().UTC().Add(-5 * time.Second).Truncate(time.Millisecond)
node := topLevelTestProxmoxNode("proxmox-node", "tower", "proxmox-1", "https://tower.local:8006")
node.LastSeen = seen
rr.IngestRecords(SourceProxmox, []IngestRecord{
{SourceID: "proxmox-node", Resource: node},
})
resource := rr.List()[0]
if !resource.LastSeen.Equal(seen) {
t.Fatalf("resource.LastSeen = %s, want %s", resource.LastSeen, seen)
}
status := resource.SourceStatus[SourceProxmox]
if status.Status != "online" {
t.Fatalf("SourceStatus[proxmox].Status = %q, want online", status.Status)
}
if !status.LastSeen.Equal(seen) {
t.Fatalf("SourceStatus[proxmox].LastSeen = %s, want %s", status.LastSeen, seen)
}
}
// Datastore-derived storage entries written by the PBS poller (instance
// "pbs-<name>", type "pbs") are delivered on the PBS cadence, so they must be
// keyed to SourcePBS — judging them against the faster Proxmox stale
// threshold would flap healthy datastores stale between PBS polls. PVE also
// reports pbs-typed storage.cfg backends, and those stay SourceProxmox: they
// really are delivered by the PVE storage poll.
func TestIngestStorageRoutesPBSPollerEntriesToPBSSource(t *testing.T) {
seen := time.Now().UTC().Add(-30 * time.Second).Truncate(time.Millisecond)
snapshot := models.StateSnapshot{
PBSInstances: []models.PBSInstance{{
ID: "pbs-backup",
Name: "backup",
Host: "https://pbs.local:8007",
Status: "online",
LastSeen: seen,
}},
Storage: []models.Storage{
{
ID: "pbs-backup-main",
Name: "main",
Node: "backup",
Instance: "pbs-backup",
Type: "pbs",
Status: "available",
LastSeen: seen,
},
{
ID: "home-pve1-pbs-remote",
Name: "pbs-remote",
Node: "pve1",
Instance: "home",
Type: "pbs",
Status: "available",
LastSeen: seen,
},
},
}
rr := NewRegistry(nil)
rr.IngestSnapshot(snapshot)
var pbsInstanceID string
var pollerStorage, pveStorage *Resource
for _, resource := range rr.List() {
resource := resource
switch {
case resource.Type == ResourceTypePBS:
pbsInstanceID = resource.ID
case resource.Type == ResourceTypeStorage && resource.Proxmox != nil && resource.Proxmox.SourceID == "pbs-backup-main":
pollerStorage = &resource
case resource.Type == ResourceTypeStorage && resource.Proxmox != nil && resource.Proxmox.SourceID == "home-pve1-pbs-remote":
pveStorage = &resource
}
}
if pbsInstanceID == "" {
t.Fatal("expected a PBS instance resource")
}
if pollerStorage == nil || pveStorage == nil {
t.Fatalf("expected both storage resources (pollerStorage=%v, pveStorage=%v)", pollerStorage != nil, pveStorage != nil)
}
if len(pollerStorage.Sources) != 1 || pollerStorage.Sources[0] != SourcePBS {
t.Fatalf("PBS poller storage Sources = %v, want [%s]", pollerStorage.Sources, SourcePBS)
}
status, ok := pollerStorage.SourceStatus[SourcePBS]
if !ok {
t.Fatal("expected a pbs source status entry on the PBS poller storage")
}
if !status.LastSeen.Equal(seen) {
t.Fatalf("SourceStatus[pbs].LastSeen = %s, want %s", status.LastSeen, seen)
}
if pollerStorage.ParentID == nil || *pollerStorage.ParentID != pbsInstanceID {
t.Fatalf("PBS poller storage ParentID = %v, want the PBS instance %q", pollerStorage.ParentID, pbsInstanceID)
}
if len(pveStorage.Sources) != 1 || pveStorage.Sources[0] != SourceProxmox {
t.Fatalf("PVE-reported pbs-typed storage Sources = %v, want [%s]", pveStorage.Sources, SourceProxmox)
}
}
func TestRegistryAvailabilityLinkAttachesFacetToKnownResource(t *testing.T) {
rr := NewRegistry(nil)
now := time.Now().UTC()
rr.IngestRecords(SourceAgent, []IngestRecord{{
SourceID: "host-1",
Resource: Resource{Type: ResourceTypeAgent, Name: "host-1", Status: StatusOnline, LastSeen: now},
Identity: ResourceIdentity{MachineID: "machine-1"},
}})
hostID := rr.ListByType(ResourceTypeAgent)[0].ID
rr.IngestRecords(SourceAvailability, []IngestRecord{{
SourceID: "probe-1",
Resource: Resource{
Type: ResourceTypeNetworkEndpoint, Name: "probe-1", Status: StatusOnline, LastSeen: now,
Sources: []DataSource{SourceAvailability},
Availability: &AvailabilityData{
TargetID: "probe-1", LinkedResourceID: hostID, Address: "192.0.2.10",
Protocol: "icmp", Enabled: true, Available: true,
},
},
Identity: ResourceIdentity{IPAddresses: []string{"192.0.2.10"}},
}})
if got := rr.ListByType(ResourceTypeNetworkEndpoint); len(got) != 1 {
t.Fatalf("expected attached availability check to retain its endpoint row, got %d", len(got))
}
host, ok := rr.Get(hostID)
if !ok || host.Availability == nil || host.Availability.TargetID != "probe-1" {
t.Fatalf("expected availability facet probe-1 on host, got %+v", host.Availability)
}
}
func TestChooseStatusAvailabilityDoesNotOverrideHigherPriority(t *testing.T) {
rr := NewRegistry(nil)
now := time.Now().UTC()
rr.IngestRecords(SourceProxmox, []IngestRecord{{
SourceID: "delly:minipc:135",
Resource: Resource{
Type: ResourceTypeSystemContainer, Name: "grafana",
Status: StatusOnline, LastSeen: now,
},
Identity: ResourceIdentity{IPAddresses: []string{"192.0.2.130"}},
}})
container := rr.ListByType(ResourceTypeSystemContainer)[0]
rr.IngestRecords(SourceAvailability, []IngestRecord{{
SourceID: "probe-grafana",
Resource: Resource{
Type: ResourceTypeNetworkEndpoint, Name: "probe-grafana",
Status: StatusOffline, LastSeen: now,
Sources: []DataSource{SourceAvailability},
Availability: &AvailabilityData{
TargetID: "probe-grafana", LinkedResourceID: container.ID,
Address: "192.0.2.130", Protocol: "http", Port: 3000,
Enabled: true, Available: false, ConsecutiveFailures: 3,
FailureThreshold: 2,
},
},
Identity: ResourceIdentity{IPAddresses: []string{"192.0.2.130"}},
}})
got, ok := rr.Get(container.ID)
if !ok {
t.Fatal("resource not found after availability ingest")
}
if got.Status != StatusOnline {
t.Fatalf("availability source (offline) should not override proxmox (online); got %s", got.Status)
}
if got.Availability == nil || got.Availability.Available != false {
t.Fatalf("availability facet should record unavailable, got %+v", got.Availability)
}
}
func TestMarkStaleRecomputesFromRemainingFreshSources(t *testing.T) {
rr := NewRegistry(nil)
oldNow := time.Now().UTC().Add(-5 * time.Minute)
recentNow := time.Now().UTC()
rr.IngestRecords(SourceProxmox, []IngestRecord{{
SourceID: "delly:delly",
Resource: Resource{
Type: ResourceTypeAgent, Name: "delly",
Status: StatusOnline, LastSeen: oldNow,
},
Identity: ResourceIdentity{MachineID: "delly"},
}})
host := rr.ListByType(ResourceTypeAgent)[0]
rr.IngestRecords(SourceAvailability, []IngestRecord{{
SourceID: "probe-delly",
Resource: Resource{
Type: ResourceTypeNetworkEndpoint, Name: "probe-delly",
Status: StatusOnline, LastSeen: recentNow,
Sources: []DataSource{SourceAvailability},
Availability: &AvailabilityData{
TargetID: "probe-delly", LinkedResourceID: host.ID,
Address: "192.0.2.5", Protocol: "tcp", Port: 8006,
Enabled: true, Available: true, LastChecked: &recentNow,
Evidence: availabilityProbeEvidence(t, "probe-delly", recentNow),
},
},
Identity: ResourceIdentity{IPAddresses: []string{"192.0.2.5"}},
}})
check, ok := rr.Get(host.ID)
if !ok || check.Status != StatusOnline {
t.Fatalf("expected online before stale, got %s", check.Status)
}
rr.MarkStale(recentNow, nil)
got, ok := rr.Get(host.ID)
if !ok {
t.Fatal("resource not found after MarkStale")
}
proxmoxStatus := got.SourceStatus[SourceProxmox]
if proxmoxStatus.Status != "stale" {
t.Fatalf("expected proxmox source to be stale, got %s", proxmoxStatus.Status)
}
availStatus := got.SourceStatus[SourceAvailability]
if availStatus.Status != "online" {
t.Fatalf("expected availability source to still be fresh/online, got %s", availStatus.Status)
}
if got.Status != StatusOnline {
t.Fatalf("status should be recomputed from remaining fresh availability source; got %s, want online", got.Status)
}
}
func TestMarkStaleKeepsProxmoxRuntimeStateIndependentOfAvailabilityFacet(t *testing.T) {
rr := NewRegistry(nil)
staleSeen := time.Now().UTC().Add(-5 * time.Minute)
freshSeen := time.Now().UTC()
for _, sourceID := range []string{"lab:node-a:101", "lab:node-a:102"} {
rr.IngestRecords(SourceProxmox, []IngestRecord{{
SourceID: sourceID,
Resource: Resource{
Type: ResourceTypeSystemContainer,
Name: sourceID,
Status: StatusOnline,
LastSeen: staleSeen,
Proxmox: &ProxmoxData{RuntimeStatus: "running", NodeName: "node-a"},
},
}})
}
resources := rr.ListByType(ResourceTypeSystemContainer)
if len(resources) != 2 {
t.Fatalf("expected two Proxmox containers, got %d", len(resources))
}
var checkedID string
for _, resource := range resources {
if resource.Name == "lab:node-a:102" {
checkedID = resource.ID
break
}
}
if checkedID == "" {
t.Fatal("availability target container not found")
}
rr.IngestRecords(SourceAvailability, []IngestRecord{{
SourceID: "probe-102",
Resource: Resource{
Type: ResourceTypeNetworkEndpoint,
Name: "probe-102",
Status: StatusOnline,
LastSeen: freshSeen,
Availability: &AvailabilityData{
TargetID: "probe-102", LinkedResourceID: checkedID,
Address: "192.0.2.102", Protocol: "icmp", Enabled: true, Available: true,
LastChecked: &freshSeen, Evidence: availabilityProbeEvidence(t, "probe-102", freshSeen),
},
},
}})
rr.MarkStale(freshSeen, nil)
resources = rr.ListByType(ResourceTypeSystemContainer)
statuses := make(map[string]ResourceStatus, len(resources))
for _, resource := range resources {
statuses[resource.Name] = resource.Status
if resource.Proxmox == nil || resource.Proxmox.RuntimeStatus != "running" {
t.Fatalf("runtime status changed with source freshness for %s: %+v", resource.Name, resource.Proxmox)
}
}
if got := statuses["lab:node-a:101"]; got != StatusWarning {
t.Fatalf("unfaceted stale container status = %q, want warning", got)
}
if got := statuses["lab:node-a:102"]; got != StatusOnline {
t.Fatalf("availability-faceted stale container status = %q, want online", got)
}
}
// availabilityCheckRecord is a check the poller ran at checkedAt against the
// resource targetID names. Its evidence stays current for two minutes, the
// two poll intervals of a local check; Available stays as the last run left
// it, as a local check that misses its cadence does.
func availabilityCheckRecord(t *testing.T, checkID, targetID string, passing bool, checkedAt time.Time) IngestRecord {
t.Helper()
status, failures := StatusOnline, 0
if !passing {
status, failures = StatusOffline, 3
}
return IngestRecord{
SourceID: checkID,
Resource: Resource{
Type: ResourceTypeNetworkEndpoint, Name: checkID, Status: status, LastSeen: checkedAt,
Sources: []DataSource{SourceAvailability},
Availability: &AvailabilityData{
TargetID: checkID, LinkedResourceID: targetID,
Address: "192.0.2.80", Protocol: "http", Port: 443,
Enabled: true, Available: passing, LastChecked: &checkedAt,
ConsecutiveFailures: failures, FailureThreshold: 2, PollIntervalSeconds: 60,
Evidence: availabilityProbeEvidence(t, checkID, checkedAt),
},
},
}
}
func ingestProxmoxGuestSeenAt(t *testing.T, rr *ResourceRegistry, sourceID string, seen time.Time) string {
t.Helper()
rr.IngestRecords(SourceProxmox, []IngestRecord{{
SourceID: sourceID,
Resource: Resource{
Type: ResourceTypeSystemContainer, Name: sourceID, Status: StatusOnline, LastSeen: seen,
Proxmox: &ProxmoxData{RuntimeStatus: "running", NodeName: "node-a"},
},
}})
for _, resource := range rr.ListByType(ResourceTypeSystemContainer) {
if resource.Name == sourceID {
return resource.ID
}
}
t.Fatalf("guest %s not ingested", sourceID)
return ""
}
// A failing availability check proves only that one port or service does not
// answer, so it never decides its target's status: a node the poller expired
// stays offline, a guest whose poll went quiet stays a warning instead of
// counting as stopped, and neither reads online. Only a passing check with
// current evidence proves the target answers. The stale pass used to count
// any current check as online, so a failing probe kept an expired node up.
func TestStalePassReadsAvailabilityCheckVerdicts(t *testing.T) {
now := time.Now().UTC()
quietSeen := now.Add(-5 * time.Minute)
lapsed := now.Add(-10 * time.Minute)
expiredNode := models.StateSnapshot{Nodes: []models.Node{{
ID: "homelab-pve9", Name: "pve9", Instance: "homelab", ClusterName: "homelab",
Status: "offline", LastSeen: quietSeen,
}}}
quietGuest := models.StateSnapshot{VMs: []models.VM{{
ID: "homelab:pve9:101", Name: "web", Node: "pve9", Instance: "homelab",
VMID: 101, Status: "running", Type: "qemu", LastSeen: quietSeen,
}}}
for _, tc := range []struct {
name string
snapshot models.StateSnapshot
targetType ResourceType
passing bool
checkedAt time.Time
want ResourceStatus
}{
{"expired node, current failing check", expiredNode, ResourceTypeAgent, false, now, StatusOffline},
{"expired node, lapsed failing check", expiredNode, ResourceTypeAgent, false, lapsed, StatusOffline},
{"expired node, lapsed passing check", expiredNode, ResourceTypeAgent, true, lapsed, StatusOffline},
{"expired node, current passing check", expiredNode, ResourceTypeAgent, true, now, StatusOnline},
{"quiet guest, current failing check", quietGuest, ResourceTypeVM, false, now, StatusWarning},
{"quiet guest, lapsed failing check", quietGuest, ResourceTypeVM, false, lapsed, StatusWarning},
{"quiet guest, current passing check", quietGuest, ResourceTypeVM, true, now, StatusOnline},
} {
t.Run(tc.name, func(t *testing.T) {
rr := NewRegistry(nil)
rr.IngestSnapshot(tc.snapshot)
target := onlyResourceOfType(t, rr, tc.targetType)
rr.IngestRecords(SourceAvailability, []IngestRecord{
availabilityCheckRecord(t, "probe-1", target.ID, tc.passing, tc.checkedAt),
})
rr.MarkStale(now, nil)
got, ok := rr.Get(target.ID)
if !ok || len(AvailabilityChecksForResource(*got)) != 1 {
t.Fatalf("check not projected onto %s: %+v", target.ID, got.AvailabilityChecks)
}
if got.Status != tc.want {
t.Fatalf("status = %q, want %q", got.Status, tc.want)
}
})
}
}
// One availability sighting stands for every check on a target, and it
// carries whichever check was projected last. Each check is judged by its own
// evidence instead, so a failing check that just ran cannot lend its
// freshness to another check's old pass, in either projection order.
func TestAvailabilityChecksAreJudgedByTheirOwnEvidence(t *testing.T) {
now := time.Now().UTC()
lapsed := now.Add(-10 * time.Minute)
for _, tc := range []struct {
name string
passedAt, failedAt time.Time
want ResourceStatus
}{
{name: "lapsed pass, current failure", passedAt: lapsed, failedAt: now, want: StatusWarning},
{name: "current pass, lapsed failure", passedAt: now, failedAt: lapsed, want: StatusOnline},
} {
for _, order := range [][]string{{"pass", "fail"}, {"fail", "pass"}} {
t.Run(tc.name+", "+strings.Join(order, " then "), func(t *testing.T) {
rr := NewRegistry(nil)
target := ingestProxmoxGuestSeenAt(t, rr, "lab:node-a:101", now.Add(-5*time.Minute))
records := map[string]IngestRecord{
"pass": availabilityCheckRecord(t, "probe-pass", target, true, tc.passedAt),
"fail": availabilityCheckRecord(t, "probe-fail", target, false, tc.failedAt),
}
for _, key := range order {
rr.IngestRecords(SourceAvailability, []IngestRecord{records[key]})
}
rr.MarkStale(now, nil)
got, ok := rr.Get(target)
if !ok {
t.Fatal("target missing")
}
if checks := AvailabilityChecksForResource(*got); len(checks) != 2 {
t.Fatalf("expected both checks on the target, got %+v", checks)
}
if got.Status != tc.want {
t.Fatalf("status = %q, want %q", got.Status, tc.want)
}
})
}
}
}
// A check's verdict moves with the check. Projecting, re-running or
// retargeting a check is not a delivery from the resource's own sources, so a
// target whose status the stale pass owns is judged again at once rather than
// keeping the old checks' verdict until the next pass; the resources API
// replays checks after its stale pass and never runs another.
func TestChangedAvailabilityChecksJudgeTheirTargetsAgain(t *testing.T) {
now := time.Now().UTC()
quietSeen := now.Add(-5 * time.Minute)
rr := NewRegistry(nil)
first := ingestProxmoxGuestSeenAt(t, rr, "lab:node-a:101", quietSeen)
second := ingestProxmoxGuestSeenAt(t, rr, "lab:node-a:102", quietSeen)
rr.MarkStale(now, nil)
status := func(id string) ResourceStatus {
t.Helper()
resource, ok := rr.Get(id)
if !ok {
t.Fatalf("resource %s missing", id)
}
return resource.Status
}
rr.IngestRecords(SourceAvailability, []IngestRecord{
availabilityCheckRecord(t, "probe-moving", first, true, now),
availabilityCheckRecord(t, "probe-failing", first, false, now),
})
if got := status(first); got != StatusOnline {
t.Fatalf("first status = %q once a passing check lands, want online", got)
}
rr.IngestRecords(SourceAvailability, []IngestRecord{
availabilityCheckRecord(t, "probe-moving", second, true, now),
})
left, _ := rr.Get(first)
if checks := AvailabilityChecksForResource(*left); len(checks) != 1 || checks[0].TargetID != "probe-failing" {
t.Fatalf("first checks = %+v, want only probe-failing", checks)
}
if got := status(first); got != StatusWarning {
t.Fatalf("first status = %q after its passing check moved away, want warning", got)
}
if got := status(second); got != StatusOnline {
t.Fatalf("second status = %q after the passing check moved onto it, want online", got)
}
rr.IngestRecords(SourceAvailability, []IngestRecord{
availabilityCheckRecord(t, "probe-moving", second, false, now),
})
if got := status(second); got != StatusWarning {
t.Fatalf("second status = %q once its only check fails, want warning", got)
}
rr.MarkStale(now, nil)
if got := status(first); got != StatusWarning {
t.Fatalf("first status = %q after the next pass, want warning", got)
}
if got := status(second); got != StatusWarning {
t.Fatalf("second status = %q after the next pass, want warning", got)
}
}
// A projected check's sighting is not one of its target's own sources. Its
// going quiet hands the stale pass nothing to decide, and replacing the check
// takes no ownership away, so a target whose own source has never delivered
// keeps the status that source gave it whatever its checks say.
func TestAvailabilitySightingNeverHandsTheStalePassItsTarget(t *testing.T) {
now := time.Now().UTC()
quickChecks := map[DataSource]time.Duration{SourceAvailability: 30 * time.Second}
for _, tc := range []struct {
name string
ingested ResourceStatus
passing bool
}{
{name: "online guest, quiet failing check", ingested: StatusOnline, passing: false},
{name: "offline placeholder, quiet passing check", ingested: StatusOffline, passing: true},
} {
t.Run(tc.name, func(t *testing.T) {
rr := NewRegistry(nil)
rr.IngestRecords(SourceProxmox, []IngestRecord{{
SourceID: "lab:node-a:101",
Resource: Resource{
Type: ResourceTypeSystemContainer, Name: "lab:node-a:101", Status: tc.ingested,
Proxmox: &ProxmoxData{RuntimeStatus: "running", NodeName: "node-a"},
},
}})
target := onlyResourceOfType(t, rr, ResourceTypeSystemContainer).ID
rr.IngestRecords(SourceAvailability, []IngestRecord{
availabilityCheckRecord(t, "probe-1", target, tc.passing, now.Add(-time.Minute)),
})
rr.MarkStale(now, quickChecks)
got, _ := rr.Get(target)
if sighting := got.SourceStatus[SourceAvailability]; sighting.Status != "stale" {
t.Fatalf("availability sighting = %+v, want stale", sighting)
}
if got.Status != tc.ingested {
t.Fatalf("status = %q with a quiet check, want %q from the target's own source", got.Status, tc.ingested)
}
rr.IngestRecords(SourceAvailability, []IngestRecord{
availabilityCheckRecord(t, "probe-1", target, !tc.passing, now),
})
rr.MarkStale(now, quickChecks)
if got, _ := rr.Get(target); got.Status != tc.ingested {
t.Fatalf("status = %q after the check flipped, want %q", got.Status, tc.ingested)
}
})
}
}
// A manual link keeps the primary's own availability checks and not the
// linked resource's, so the merged resource is judged by the checks it
// carries. The linked resource's fresher availability sighting survives the
// merge as delivery bookkeeping; it must not become a verdict for checks the
// merged resource does not show.
func TestManualLinkJudgesOnlyTheChecksTheMergedResourceCarries(t *testing.T) {
now := time.Now().UTC()
quietSeen := now.Add(-5 * time.Minute)
store := NewMemoryStore()
if err := store.AddLink(ResourceLink{ResourceA: "vm-web", ResourceB: "agent-web", PrimaryID: "vm-web"}); err != nil {
t.Fatalf("add link: %v", err)
}
rr := NewRegistry(store)
rr.IngestResources([]Resource{
{
ID: "vm-web", Type: ResourceTypeVM, Name: "web", Status: StatusWarning, LastSeen: quietSeen,
Sources: []DataSource{SourceProxmox},
SourceStatus: map[DataSource]SourceStatus{
SourceProxmox: {Status: "stale", LastSeen: quietSeen, reported: StatusOnline},
},
},
{
ID: "agent-web", Type: ResourceTypeAgent, Name: "web", Status: StatusWarning, LastSeen: now,
Sources: []DataSource{SourceAgent, SourceAvailability},
SourceStatus: map[DataSource]SourceStatus{
SourceAgent: {Status: "stale", LastSeen: quietSeen, reported: StatusOnline},
SourceAvailability: {Status: "online", LastSeen: now},
},
AvailabilityChecks: []AvailabilityData{{
TargetID: "probe-web", Enabled: true, Available: false, LastChecked: &now,
ConsecutiveFailures: 3, FailureThreshold: 2,
Evidence: availabilityProbeEvidence(t, "probe-web", now),
}},
Agent: &AgentData{AgentID: "host-web", Hostname: "web"},
},
})
merged, ok := rr.Get("vm-web")
if !ok {
t.Fatal("linked resource missing")
}
if checks := AvailabilityChecksForResource(*merged); len(checks) != 0 {
t.Fatalf("merged resource exposes the linked resource's checks: %+v", checks)
}
if merged.Status != StatusWarning {
t.Fatalf("status = %q, want warning from its own quiet sources", merged.Status)
}
}
func TestResourceRegistryUsesConfiguredProxmoxStaleThresholds(t *testing.T) {
seen := time.Now().UTC().Add(-90 * time.Second).Truncate(time.Millisecond)
snapshot := models.StateSnapshot{
VMs: []models.VM{{
ID: "cluster-a:pve-a:101",
Name: "db",
Node: "pve-a",
Instance: "cluster-a",
VMID: 101,
Status: "running",
Type: "qemu",
LastSeen: seen,
}},
}
defaultRegistry := NewRegistry(nil)
defaultRegistry.IngestSnapshot(snapshot)
defaultVMs := defaultRegistry.ListByType(ResourceTypeVM)
if len(defaultVMs) != 1 {
t.Fatalf("default VM count = %d, want 1", len(defaultVMs))
}
if defaultVMs[0].Status != StatusWarning {
t.Fatalf("default VM status = %q, want warning", defaultVMs[0].Status)
}
thresholds := map[DataSource]time.Duration{
SourceProxmox: 10 * time.Minute,
}
configuredRegistry := NewRegistry(nil)
configuredRegistry.IngestSnapshotWithStaleThresholds(snapshot, thresholds)
configuredVMs := configuredRegistry.ListByType(ResourceTypeVM)
if len(configuredVMs) != 1 {
t.Fatalf("configured VM count = %d, want 1", len(configuredVMs))
}
if configuredVMs[0].Status != StatusOnline {
t.Fatalf("configured VM status = %q, want online", configuredVMs[0].Status)
}
clonedRegistry := NewRegistry(nil)
clonedRegistry.IngestResourcesWithStaleThresholds(configuredRegistry.List(), thresholds)
clonedVMs := clonedRegistry.ListByType(ResourceTypeVM)
if len(clonedVMs) != 1 {
t.Fatalf("cloned VM count = %d, want 1", len(clonedVMs))
}
if clonedVMs[0].Status != StatusOnline {
t.Fatalf("cloned VM status = %q, want online", clonedVMs[0].Status)
}
if status := clonedVMs[0].SourceStatus[SourceProxmox]; status.Status != "online" {
t.Fatalf("cloned SourceStatus[proxmox] = %+v, want online", status)
}
}
// Two Docker Swarm members with FQDN hostnames and no machine keys
// (containerized agents without /etc/machine-id) must mint distinct
// cluster-scoped canonical IDs. The historical derivation hashed the short
// hostname, so cloud.a and cloud.b both derived cluster:<swarm>:cloud and
// fully merged in the registry (sibling of #1559).
func TestSwarmFQDNHostsWithoutMachineIDsStayDistinct(t *testing.T) {
store := NewMemoryStore()
registry := NewRegistry(store)
swarmHostIdentity := func(hostname string) ResourceIdentity {
return ResourceIdentity{
ClusterName: "prod-swarm",
Hostnames: []string{hostname, "prod-swarm:" + hostname},
}
}
swarmHostResource := func(hostname string) Resource {
return Resource{
Type: ResourceTypeAgent,
Name: hostname,
Status: StatusOnline,
Docker: &DockerData{HostSourceID: "docker:" + hostname, Hostname: hostname},
}
}
idA := registry.ingest(SourceDocker, "docker:cloud.a", swarmHostResource("cloud.a"), swarmHostIdentity("cloud.a"))
idB := registry.ingest(SourceDocker, "docker:cloud.b", swarmHostResource("cloud.b"), swarmHostIdentity("cloud.b"))
if idA == "" || idB == "" {
t.Fatalf("swarm host ingest returned empty IDs: %q, %q", idA, idB)
}
if idA == idB {
t.Fatalf("swarm members cloud.a and cloud.b collapsed to canonical ID %q", idA)
}
if got := len(registry.List()); got != 2 {
t.Fatalf("expected 2 distinct swarm host resources, got %d", got)
}
// Journal rows recorded under the collapsed short-hostname era must stay
// part of the timeline read through the new full-hostname ID.
shortEraID := buildHashID(ResourceTypeAgent, "cluster:prod-swarm:cloud")
if shortEraID == idA || shortEraID == idB {
t.Fatalf("full-hostname derivation still matches the short era ID %q", shortEraID)
}
if err := store.RecordChange(ResourceChange{
ID: "change-short-era",
ResourceID: shortEraID,
ObservedAt: time.Now().Add(-time.Hour),
Kind: ChangeStateTransition,
}); err != nil {
t.Fatalf("RecordChange: %v", err)
}
registry.PersistIdentityPins()
changes, err := store.GetRecentChanges(idA, time.Time{}, 10)
if err != nil {
t.Fatalf("GetRecentChanges: %v", err)
}
found := false
for _, change := range changes {
if change.ID == "change-short-era" {
found = true
}
}
if !found {
t.Fatalf("short-era journal row missing from timeline of %q; got %d changes", idA, len(changes))
}
}
// A machine-keyless swarm host whose canonical ID was minted under the
// collapsed short-hostname derivation must carry its operator-owned rows to
// the full-hostname ID: the superseded pin row goes, operator state
// (never-auto-remediate) is re-keyed, and the new pin preserves the full
// dotted hostname.
func TestCanonicalIDSuccessionMovesOperatorState(t *testing.T) {
store := NewMemoryStore()
shortEraID := buildHashID(ResourceTypeAgent, "cluster:prod-swarm:cloud")
if err := store.UpsertResourceIdentityPins([]ResourceIdentityPin{{
CanonicalID: shortEraID,
ResourceType: ResourceTypeAgent,
ClusterName: "prod-swarm",
Hostname: "cloud",
}}); err != nil {
t.Fatalf("seed identity pin: %v", err)
}
if err := store.SetResourceOperatorState(ResourceOperatorState{
CanonicalID: shortEraID,
NeverAutoRemediate: true,
}); err != nil {
t.Fatalf("seed operator state: %v", err)
}
registry := NewRegistry(store)
identity := ResourceIdentity{
ClusterName: "prod-swarm",
Hostnames: []string{"cloud.a", "prod-swarm:cloud.a"},
}
resource := Resource{
Type: ResourceTypeAgent,
Name: "cloud.a",
Status: StatusOnline,
Docker: &DockerData{HostSourceID: "docker:cloud.a", Hostname: "cloud.a"},
}
newID := registry.ingest(SourceDocker, "docker:cloud.a", resource, identity)
if newID == "" || newID == shortEraID {
t.Fatalf("expected a fresh full-hostname canonical ID, got %q (short era %q)", newID, shortEraID)
}
registry.PersistIdentityPins()
if _, found := operatorStateRow(t, store, shortEraID); found {
t.Fatalf("operator state still keyed by superseded ID")
}
state, found, err := store.GetResourceOperatorState(newID)
if err != nil || !found {
t.Fatalf("operator state missing under successor ID (found=%v, err=%v)", found, err)
}
if !state.NeverAutoRemediate {
t.Fatalf("never-auto-remediate flag lost across canonical ID succession")
}
pins, err := store.ListResourceIdentityPins()
if err != nil {
t.Fatalf("ListResourceIdentityPins: %v", err)
}
byID := make(map[string]ResourceIdentityPin, len(pins))
for _, pin := range pins {
byID[pin.CanonicalID] = pin
}
if _, stale := byID[shortEraID]; stale {
t.Fatalf("superseded short-era pin row still present")
}
successor, ok := byID[newID]
if !ok {
t.Fatalf("successor pin missing; pins: %+v", pins)
}
if successor.Hostname != "cloud.a" {
t.Fatalf("successor pin hostname = %q, want full dotted cloud.a", successor.Hostname)
}
}
// A genuinely short-named host that is alive in the registry owns its
// canonical ID; an FQDN sibling sharing the short name must not steal its
// operator rows through succession.
func TestCanonicalIDSuccessionSkipsLiveShortNamedHost(t *testing.T) {
store := NewMemoryStore()
registry := NewRegistry(store)
ingestSwarmHost := func(hostname string) string {
return registry.ingest(SourceDocker, "docker:"+hostname, Resource{
Type: ResourceTypeAgent,
Name: hostname,
Status: StatusOnline,
Docker: &DockerData{HostSourceID: "docker:" + hostname, Hostname: hostname},
}, ResourceIdentity{
ClusterName: "prod-swarm",
Hostnames: []string{hostname, "prod-swarm:" + hostname},
})
}
shortID := ingestSwarmHost("cloud")
registry.PersistIdentityPins()
if err := store.SetResourceOperatorState(ResourceOperatorState{
CanonicalID: shortID,
NeverAutoRemediate: true,
}); err != nil {
t.Fatalf("seed operator state: %v", err)
}
fqdnID := ingestSwarmHost("cloud.a")
if fqdnID == shortID {
t.Fatalf("cloud and cloud.a collapsed to canonical ID %q", fqdnID)
}
registry.PersistIdentityPins()
state, found, err := store.GetResourceOperatorState(shortID)
if err != nil || !found || !state.NeverAutoRemediate {
t.Fatalf("live short-named host lost its operator state (found=%v, err=%v)", found, err)
}
if _, stolen, _ := store.GetResourceOperatorState(fqdnID); stolen {
t.Fatalf("FQDN sibling stole the live short-named host's operator state")
}
}
// Record-declared successions (IngestRecord.SupersededCanonicalIDs) must
// re-key operator rows to the record's minted canonical ID, but never while
// the superseded ID still belongs to a live resource in the registry: a
// distinct sibling's rows must not be stolen.
func TestIngestRecordsRecordDeclaredSuccessions(t *testing.T) {
store := NewMemoryStore()
oldID := SourceSpecificID(ResourceTypeStorage, SourceTrueNAS, "system:legacy-host/pool:tank")
if err := store.SetResourceOperatorState(ResourceOperatorState{
CanonicalID: oldID,
NeverAutoRemediate: true,
}); err != nil {
t.Fatalf("seed operator state: %v", err)
}
registry := NewRegistry(store)
registry.IngestRecords(SourceTrueNAS, []IngestRecord{{
SourceID: "system:conn-a/pool:tank",
Resource: Resource{
Type: ResourceTypeStorage,
Name: "tank",
LastSeen: time.Now().UTC(),
},
SupersededCanonicalIDs: []string{oldID},
}})
newID := SourceSpecificID(ResourceTypeStorage, SourceTrueNAS, "system:conn-a/pool:tank")
if _, found := operatorStateRow(t, store, oldID); found {
t.Fatalf("expected operator state to leave superseded ID")
}
state, found, err := store.GetResourceOperatorState(newID)
if err != nil || !found || !state.NeverAutoRemediate {
t.Fatalf("expected operator state re-keyed to %s (found=%v err=%v state=%+v)", newID, found, err, state)
}
}
func TestIngestRecordsSkipRecordDeclaredSuccessionForLiveOldID(t *testing.T) {
store := NewMemoryStore()
oldID := SourceSpecificID(ResourceTypeStorage, SourceTrueNAS, "system:legacy-host/pool:tank")
if err := store.SetResourceOperatorState(ResourceOperatorState{
CanonicalID: oldID,
NeverAutoRemediate: true,
}); err != nil {
t.Fatalf("seed operator state: %v", err)
}
registry := NewRegistry(store)
// The old derivation is still alive in this registry: another record
// legitimately owns it, so the succession must not run.
registry.IngestRecords(SourceTrueNAS, []IngestRecord{
{
SourceID: "system:legacy-host/pool:tank",
Resource: Resource{
Type: ResourceTypeStorage,
Name: "tank",
LastSeen: time.Now().UTC(),
},
},
{
SourceID: "system:conn-a/pool:tank",
Resource: Resource{
Type: ResourceTypeStorage,
Name: "tank",
LastSeen: time.Now().UTC(),
},
SupersededCanonicalIDs: []string{oldID},
},
})
if _, found, err := store.GetResourceOperatorState(oldID); err != nil || !found {
t.Fatalf("expected operator state to stay under live old ID (found=%v err=%v)", found, err)
}
}
func TestResourceRegistryClampsExhaustedSMARTWearout(t *testing.T) {
percentageUsed := 143
registry := NewRegistry(nil)
registry.IngestSnapshot(models.StateSnapshot{
Hosts: []models.Host{
{
ID: "host-exhausted",
Hostname: "pve-exhausted",
Status: "online",
Sensors: models.HostSensorSummary{
SMART: []models.HostDiskSMART{
{
Device: "/dev/nvme0n1",
Model: "Example NVMe",
Serial: "EXHAUSTED-NVME",
Type: "nvme",
Health: "PASSED",
Attributes: &models.SMARTAttributes{
PercentageUsed: &percentageUsed,
},
},
},
},
},
},
})
disks := registry.ListByType(ResourceTypePhysicalDisk)
if len(disks) != 1 || disks[0].PhysicalDisk == nil {
t.Fatalf("expected one physical disk, got %+v", disks)
}
if disks[0].PhysicalDisk.Wearout != 0 {
t.Fatalf("wearout remaining = %d, want 0", disks[0].PhysicalDisk.Wearout)
}
if risk := disks[0].PhysicalDisk.Risk; risk == nil || risk.Level != storagehealth.RiskCritical {
t.Fatalf("exhausted media risk = %+v, want critical", risk)
}
}
// The unified-resource risk projection and the server-side alert path must
// reach the same verdict about the same disk. Both now gate wearout on
// storagehealth.WearoutReported instead of each carrying its own inline
// boundary, which is how a spent SSD came to read critical in the UI while the
// alert path stayed silent.
func TestPhysicalDiskRiskProjectionUsesSharedWearoutPredicate(t *testing.T) {
tests := []struct {
name string
diskType string
wearout int
wantCritical bool
}{
{"spent ssd", "ssd", 0, true},
{"spent nvme", "nvme", 0, true},
{"rotational zero reports no endurance", "hdd", 0, false},
{"untyped zero reports no endurance", "", 0, false},
{"unreported sentinel", "ssd", -1, false},
{"healthy ssd", "ssd", 90, false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
meta := &PhysicalDiskMeta{Health: "PASSED", DiskType: tt.diskType, Wearout: tt.wearout}
assessment := physicalDiskAssessmentFromMeta(meta)
gotCritical := assessment.Level == storagehealth.RiskCritical
if gotCritical != tt.wantCritical {
t.Fatalf("risk level = %q, want critical=%v", assessment.Level, tt.wantCritical)
}
// The projection must agree with the predicate the alert path uses.
if storagehealth.WearoutReported(meta.Wearout, meta.DiskType) != (tt.wearout >= 0 && (tt.wearout > 0 || tt.diskType == "ssd" || tt.diskType == "nvme")) {
t.Fatal("projection and shared predicate disagree about whether this reading is evidence")
}
})
}
}
func TestRegistryIngestsAgentLibvirtDomainUnderReportingHost(t *testing.T) {
now := time.Now().UTC()
registry := NewRegistry(nil)
registry.IngestSnapshot(models.StateSnapshot{Hosts: []models.Host{{
ID: "host-1",
Hostname: "kvm-host",
Status: "online",
LastSeen: now,
Libvirt: &models.HostLibvirtInventory{
CollectedAt: now,
Domains: []models.HostLibvirtDomain{{
ID: "domain-a",
Name: "app",
State: "running",
VCPUs: 4,
}},
},
}}})
vms := registry.ListByType(ResourceTypeVM)
if len(vms) != 1 {
t.Fatalf("libvirt VM resources = %+v", vms)
}
vm := vms[0]
if vm.Technology != "libvirt" ||
vm.VirtualMachine == nil ||
vm.VirtualMachine.VCPUs != 4 ||
vm.ParentID == nil {
t.Fatalf("libvirt VM resource = %+v", vm)
}
parent, ok := registry.Get(*vm.ParentID)
if !ok || parent.Type != ResourceTypeAgent || parent.Name != "kvm-host" {
t.Fatalf("libvirt VM parent = %+v, found=%v", parent, ok)
}
}
func proxmoxGuestMigrationSnapshot(now time.Time, node string) models.StateSnapshot {
return models.StateSnapshot{
Nodes: []models.Node{
{ID: "delly-pve1", Name: "pve1", Instance: "delly", ClusterName: "delly", Status: "online", LastSeen: now},
{ID: "delly-pve2", Name: "pve2", Instance: "delly", ClusterName: "delly", Status: "online", LastSeen: now},
},
VMs: []models.VM{
{
ID: "delly:" + node + ":100",
VMID: 100,
Name: "web-vm",
Node: node,
Instance: "delly",
Status: "running",
LastSeen: now,
},
},
Containers: []models.Container{
{
ID: "delly:" + node + ":112",
VMID: 112,
Name: "debian-ct",
Node: node,
Instance: "delly",
Status: "running",
LastSeen: now,
},
},
}
}
// A live migration changes the guest's node and therefore its node-scoped
// source ID, but the canonical resource ID must stay put so operator-owned
// rows (availability links, overrides, operator state) keep following the
// guest (#1669).
func TestProxmoxGuestCanonicalIDSurvivesNodeMigration(t *testing.T) {
now := time.Now().UTC()
before := NewRegistry(nil)
before.IngestSnapshot(proxmoxGuestMigrationSnapshot(now, "pve1"))
after := NewRegistry(nil)
after.IngestSnapshot(proxmoxGuestMigrationSnapshot(now, "pve2"))
for _, resourceType := range []ResourceType{ResourceTypeVM, ResourceTypeSystemContainer} {
beforeList := before.ListByType(resourceType)
afterList := after.ListByType(resourceType)
if len(beforeList) != 1 || len(afterList) != 1 {
t.Fatalf("%s: expected exactly one guest before and after migration, got %d and %d", resourceType, len(beforeList), len(afterList))
}
if beforeList[0].ID != afterList[0].ID {
t.Fatalf("%s: canonical ID changed across node migration: %q -> %q", resourceType, beforeList[0].ID, afterList[0].ID)
}
}
vms := before.ListByType(ResourceTypeVM)
if want := ProxmoxGuestCanonicalID(ResourceTypeVM, "delly", 100); vms[0].ID != want {
t.Fatalf("VM canonical ID = %q, want node-independent %q", vms[0].ID, want)
}
cts := before.ListByType(ResourceTypeSystemContainer)
if want := ProxmoxGuestCanonicalID(ResourceTypeSystemContainer, "delly", 112); cts[0].ID != want {
t.Fatalf("container canonical ID = %q, want node-independent %q", cts[0].ID, want)
}
}
// The retired node-scoped canonical IDs must be declared superseded for every
// node the instance knows, so rows orphaned by a migration that happened
// before the upgrade still succeed into the new ID.
func TestProxmoxGuestDeclaresNodeScopedSupersededEras(t *testing.T) {
now := time.Now().UTC()
rr := NewRegistry(nil)
rr.IngestSnapshot(proxmoxGuestMigrationSnapshot(now, "pve2"))
vms := rr.ListByType(ResourceTypeVM)
if len(vms) != 1 {
t.Fatalf("expected one VM, got %d", len(vms))
}
superseded := make(map[string]struct{}, len(vms[0].SupersededCanonicalIDs))
for _, id := range vms[0].SupersededCanonicalIDs {
superseded[id] = struct{}{}
}
for _, node := range []string{"pve1", "pve2"} {
legacy := SourceSpecificID(ResourceTypeVM, SourceProxmox, "delly:"+node+":100")
if _, ok := superseded[legacy]; !ok {
t.Fatalf("superseded IDs %v missing legacy era for node %s (%s)", vms[0].SupersededCanonicalIDs, node, legacy)
}
}
}
// Operator-owned rows persisted under a retired node-scoped ID re-key to the
// node-independent ID during snapshot ingest.
func TestProxmoxGuestSuccessionRekeysOperatorState(t *testing.T) {
now := time.Now().UTC()
store := NewMemoryStore()
legacyID := SourceSpecificID(ResourceTypeVM, SourceProxmox, "delly:pve1:100")
if err := store.SetResourceOperatorState(ResourceOperatorState{
CanonicalID: legacyID,
NeverAutoRemediate: true,
IntentionallyOffline: false,
}); err != nil {
t.Fatalf("seed operator state: %v", err)
}
rr := NewRegistry(store)
// Guest already migrated to pve2 before the upgrade: the legacy row keyed
// by its pve1-era ID must still follow it.
rr.IngestSnapshot(proxmoxGuestMigrationSnapshot(now, "pve2"))
newID := ProxmoxGuestCanonicalID(ResourceTypeVM, "delly", 100)
state, ok, err := store.GetResourceOperatorState(newID)
if err != nil || !ok {
t.Fatalf("operator state missing under new canonical ID %q (ok=%v err=%v)", newID, ok, err)
}
if !state.NeverAutoRemediate {
t.Fatalf("re-keyed operator state lost its fields: %+v", state)
}
if _, stale := operatorStateRow(t, store, legacyID); stale {
t.Fatalf("operator state still present under retired ID %q", legacyID)
}
}
// Guests that cannot derive an instance+VMID identity keep the
// source-specific derivation, as do VM resources from other providers.
func TestProxmoxGuestKeylessFallsBackToSourceSpecificID(t *testing.T) {
now := time.Now().UTC()
rr := NewRegistry(nil)
rr.IngestSnapshot(models.StateSnapshot{
VMs: []models.VM{
{ID: "legacy-opaque-id", Name: "old-vm", Status: "running", LastSeen: now},
},
})
vms := rr.ListByType(ResourceTypeVM)
if len(vms) != 1 {
t.Fatalf("expected one VM, got %d", len(vms))
}
if want := SourceSpecificID(ResourceTypeVM, SourceProxmox, "legacy-opaque-id"); vms[0].ID != want {
t.Fatalf("keyless guest ID = %q, want source-specific %q", vms[0].ID, want)
}
if len(vms[0].SupersededCanonicalIDs) != 0 {
t.Fatalf("keyless guest must not declare superseded eras, got %v", vms[0].SupersededCanonicalIDs)
}
}
func TestRegistryResourcesReportPlatformAdmission(t *testing.T) {
// Admission is derived from the same registry resources the API serves, so
// it must hold for resources that reached the registry through ingest
// rather than being hand-built in a test.
registry := NewRegistry(nil)
registry.IngestResources([]Resource{
{
ID: "pulse-host",
Type: ResourceTypeAgent,
Name: "workshop",
Sources: []DataSource{SourceAgent},
Agent: &AgentData{},
},
{
ID: "truenas-host",
Type: ResourceTypeAgent,
Name: "storage",
Sources: []DataSource{SourceAgent, SourceTrueNAS},
Agent: &AgentData{},
TrueNAS: &TrueNASData{},
},
})
admission := BuildPlatformAdmission(registry.List())
if !admission.Standalone {
t.Fatalf("genuine pulse agent must admit standalone, got %+v", admission)
}
if !admission.TrueNAS {
t.Fatalf("truenas host must admit truenas, got %+v", admission)
}
if admission.Proxmox || admission.Docker || admission.Kubernetes || admission.VMware {
t.Fatalf("no other platform should be admitted, got %+v", admission)
}
}
func TestIncidentActionFallbackKeepsNetworkAndStorageGuidanceDistinct(t *testing.T) {
warning := ResourceIncident{Code: "generic", Severity: storagehealth.RiskWarning}
_, networkAction := IncidentActionForResource(
&Resource{ID: "network-1", Type: ResourceTypeNetwork},
warning,
IncidentCategoryHealth,
)
if networkAction != "Review resource health and plan corrective action" {
t.Fatalf("network action = %q, want neutral resource guidance", networkAction)
}
_, storageAction := IncidentActionForResource(
&Resource{ID: "storage-1", Type: ResourceTypeStorage},
warning,
IncidentCategoryHealth,
)
if storageAction != "Review storage health and plan corrective action" {
t.Fatalf("storage action = %q, want storage guidance", storageAction)
}
}
func TestIssue1720AgentControllerMembersInheritOwningProxmoxScope(t *testing.T) {
now := time.Date(2026, 8, 25, 12, 0, 0, 0, time.UTC)
registry := NewRegistry(nil)
registry.IngestSnapshot(models.StateSnapshot{
Nodes: []models.Node{{
ID: "cluster-a-pve1",
Name: "pve1",
Instance: "cluster-a",
ClusterName: "cluster-a",
LinkedAgentID: "host-1",
Status: "online",
LastSeen: now,
}},
Hosts: []models.Host{{
ID: "host-1",
Hostname: "pve1",
LinkedNodeID: "cluster-a-pve1",
Status: "online",
LastSeen: now,
Sensors: models.HostSensorSummary{SMART: []models.HostDiskSMART{
{
Device: "0 [megaraid,0]",
Model: "TOSHIBA MG03SCA300",
Serial: "9410A0FWFVL9",
Type: "sas",
Controller: "0",
Target: "megaraid,0",
SizeBytes: 3_000_592_982_016,
Temperature: 35,
Health: "PASSED",
},
{
Device: "0 [megaraid,1]",
Model: "TOSHIBA MG03SCA300",
Serial: "35C0A39YFVL9",
Type: "sas",
Controller: "0",
Target: "megaraid,1",
SizeBytes: 3_000_592_982_016,
Temperature: 37,
Health: "PASSED",
},
}},
}},
})
disks := registry.ListByType(ResourceTypePhysicalDisk)
if len(disks) != 2 {
t.Fatalf("controller member resources = %d, want 2: %+v", len(disks), disks)
}
targets := make(map[string]bool, len(disks))
for _, disk := range disks {
if disk.PhysicalDisk == nil {
t.Fatalf("disk %q has no physical-disk facet", disk.ID)
}
targets[disk.PhysicalDisk.Target] = true
if !reflect.DeepEqual(disk.Sources, []DataSource{SourceAgent}) {
t.Fatalf("disk %q sources = %v, want agent telemetry only", disk.ID, disk.Sources)
}
if !reflect.DeepEqual(disk.PlatformScopes, []string{"agent", "proxmox-pve"}) {
t.Fatalf("disk %q platform scopes = %v, want agent + proxmox-pve", disk.ID, disk.PlatformScopes)
}
if disk.Proxmox == nil || disk.Proxmox.NodeName != "pve1" || disk.Proxmox.Instance != "cluster-a" {
t.Fatalf("disk %q Proxmox ownership = %+v", disk.ID, disk.Proxmox)
}
if disk.ParentID == nil {
t.Fatalf("disk %q has no owning node parent", disk.ID)
}
parent, ok := registry.Get(*disk.ParentID)
if !ok || parent.Proxmox == nil || parent.Proxmox.Instance != "cluster-a" {
t.Fatalf("disk %q parent = %+v, found=%v", disk.ID, parent, ok)
}
}
if !targets["megaraid,0"] || !targets["megaraid,1"] {
t.Fatalf("controller members collapsed or disappeared: %v", targets)
}
}
// Follow-up regression on issue #1720: the PVE disks/list poll and the agent
// SMART report both observe the same RAID array volume, but PVE surfaces the
// NAA identifier as a bare-hex serial while smartctl reports it as a
// naa.-prefixed WWN with no serial at all. The two observations must join into
// one resource instead of rendering the volume twice with split histories.
// Sibling volumes on one controller share their truncated udev ID_WWN
// (0x + leading 16 hex), so the join must never match on truncated forms.
func TestIssue1720ArrayVolumeMergesBareSerialWithPrefixedWWN(t *testing.T) {
now := time.Date(2026, 8, 27, 10, 0, 0, 0, time.UTC)
registry := NewRegistry(nil)
registry.IngestSnapshot(models.StateSnapshot{
Nodes: []models.Node{{
ID: "cluster-a-indus",
Name: "indus",
Instance: "cluster-a",
ClusterName: "cluster-a",
LinkedAgentID: "host-1",
Status: "online",
LastSeen: now,
}},
PhysicalDisks: []models.PhysicalDisk{
{
ID: "cluster-a-indus--dev-sda",
Node: "indus",
Instance: "cluster-a",
DevPath: "/dev/sda",
Model: "PERC_H730_Mini",
Serial: "61866da053481f002f58a43b22f964a7",
WWN: "0x61866da053481f00",
Type: "sata",
Health: "UNKNOWN",
Size: 479_559_942_144,
LastChecked: now,
},
{
ID: "cluster-a-indus--dev-sdb",
Node: "indus",
Instance: "cluster-a",
DevPath: "/dev/sdb",
Model: "PERC_H730_Mini",
Serial: "61866da053481f0030543ecb1d3b4cca",
WWN: "0x61866da053481f00",
Type: "sata",
Health: "UNKNOWN",
Size: 3_000_034_656_256,
LastChecked: now,
},
},
Hosts: []models.Host{{
ID: "host-1",
Hostname: "indus",
LinkedNodeID: "cluster-a-indus",
Status: "online",
LastSeen: now,
Sensors: models.HostSensorSummary{SMART: []models.HostDiskSMART{
{
Device: "sda",
Model: "PERC H730 Mini",
WWN: "naa.61866da053481f002f58a43b22f964a7",
Type: "sata",
Controller: "0000:02:00.0",
Target: "0:2:0:0",
SizeBytes: 479_559_942_144,
Health: "UNKNOWN",
Standby: true,
},
{
Device: "sdb",
Model: "PERC H730 Mini",
WWN: "naa.61866da053481f0030543ecb1d3b4cca",
Type: "sata",
Controller: "0000:02:00.0",
Target: "0:2:1:0",
SizeBytes: 3_000_034_656_256,
Health: "UNKNOWN",
Standby: true,
},
}},
}},
})
disks := registry.ListByType(ResourceTypePhysicalDisk)
if len(disks) != 2 {
devs := make([]string, 0, len(disks))
for _, disk := range disks {
if disk.PhysicalDisk != nil {
devs = append(devs, disk.PhysicalDisk.DevPath)
}
}
t.Fatalf("physical disk resources = %d (%v), want 2 merged volumes", len(disks), devs)
}
byPath := make(map[string]Resource, len(disks))
for _, disk := range disks {
if disk.PhysicalDisk == nil {
t.Fatalf("disk %q has no physical-disk facet", disk.ID)
}
byPath[disk.PhysicalDisk.DevPath] = disk
}
for _, devPath := range []string{"/dev/sda", "/dev/sdb"} {
disk, ok := byPath[devPath]
if !ok {
t.Fatalf("no merged resource kept canonical devPath %q: %v", devPath, byPath)
}
if !containsDataSource(disk.Sources, SourceAgent) || !containsDataSource(disk.Sources, SourceProxmox) {
t.Fatalf("disk %q sources = %v, want merged proxmox+agent", devPath, disk.Sources)
}
}
if byPath["/dev/sda"].PhysicalDisk.SizeBytes == byPath["/dev/sdb"].PhysicalDisk.SizeBytes {
t.Fatalf("sibling volumes collapsed into one identity: %+v", byPath)
}
}
func TestPhysicalDiskFreshnessUsesCollectorCadenceAndRecovers(t *testing.T) {
now := time.Now().UTC()
for _, cadence := range []time.Duration{5 * time.Minute, 15 * time.Minute} {
t.Run(cadence.String(), func(t *testing.T) {
rr := NewRegistry(nil)
disk := models.PhysicalDisk{ID: "disk-source", Instance: "pve", Node: "node", DevPath: "/dev/nvme0n1", Serial: "serial-1", Health: "PASSED", Wearout: 63, LastChecked: now, ExpectedUpdateInterval: cadence}
rr.ingestPhysicalDisk(disk)
check := func(wantStatus ResourceStatus, wantFreshness string) Resource {
t.Helper()
resources := rr.ListByType(ResourceTypePhysicalDisk)
if len(resources) != 1 {
t.Fatalf("disks: %+v", resources)
}
got := resources[0]
if got.Status != wantStatus || got.SourceStatus[SourceProxmox].Status != wantFreshness {
t.Fatalf("status=%s freshness=%+v, want %s/%s", got.Status, got.SourceStatus, wantStatus, wantFreshness)
}
if got.SourceStatus[SourceProxmox].ExpectedUpdateIntervalSeconds != int64(cadence/time.Second) {
t.Fatalf("cadence lost: %+v", got.SourceStatus)
}
return got
}
rr.MarkStale(now.Add(cadence), map[DataSource]time.Duration{SourceProxmox: time.Minute})
original := check(StatusOnline, "online")
rr.MarkStale(now.Add(2*cadence+time.Second), map[DataSource]time.Duration{SourceProxmox: time.Minute})
check(StatusWarning, "stale")
disk.LastChecked = now.Add(2*cadence + 2*time.Second)
rr.ingestPhysicalDisk(disk)
recovered := check(StatusOnline, "online")
if recovered.ID != original.ID {
t.Fatal("freshness recovery changed identity")
}
disk.Wearout = 8
disk.LastChecked = disk.LastChecked.Add(cadence)
rr.ingestPhysicalDisk(disk)
warning := check(StatusWarning, "online")
if warning.PhysicalDisk.Risk == nil {
t.Fatal("fresh source hid measured disk risk")
}
})
}
}
func TestPlatformOnlyResourceStatusRecoversWithoutOverridingAgent(t *testing.T) {
now := time.Now().UTC()
rr := NewRegistry(nil)
identity := ResourceIdentity{MachineID: "0123456789abcdef0123456789abcdef", Hostnames: []string{"node-one"}}
source := Resource{Type: ResourceTypeAgent, Name: "node-one", Status: StatusOnline, LastSeen: now}
id := rr.ingest(SourceProxmox, "pve-node", source, identity)
rr.MarkStale(now.Add(3*time.Minute), nil)
source.LastSeen = now.Add(4 * time.Minute)
rr.ingest(SourceProxmox, "pve-node", source, identity)
got, _ := rr.Get(id)
if got.Status != StatusOnline {
t.Fatalf("fresh platform status remains %s", got.Status)
}
source.Status = StatusWarning
if mergedID := rr.ingest(SourceAgent, "agent-node", source, identity); mergedID != id {
t.Fatalf("fixture did not merge: %s != %s", mergedID, id)
}
source.Status = StatusOnline
rr.ingest(SourceProxmox, "pve-node", source, identity)
got, _ = rr.Get(id)
if got.Status != StatusWarning {
t.Fatalf("platform overwrote higher-priority agent status: %s", got.Status)
}
}
func usbAliasSnapshot(agentSerial, pveSerial string) models.StateSnapshot {
now := time.Now()
return models.StateSnapshot{
Nodes: []models.Node{{ID: "pve-node", Name: "node", Instance: "pve", LinkedAgentID: "agent", Status: "online", LastSeen: now}},
Hosts: []models.Host{{ID: "agent", Hostname: "node", LinkedNodeID: "pve-node", Status: "online", LastSeen: now, Sensors: models.HostSensorSummary{SMART: []models.HostDiskSMART{{Device: "sdx", Serial: agentSerial, Type: "usb", SizeBytes: 239_000_000_000, Health: "PASSED", Temperature: 31}}}}},
PhysicalDisks: []models.PhysicalDisk{{ID: ProxmoxPhysicalDiskSourceID("pve", "node", "/dev/sdx", "", ""), Instance: "pve", Node: "node", DevPath: "/dev/sdx", Serial: pveSerial, Type: "usb", Size: 239_000_000_000, Health: "PASSED", LastChecked: now}},
}
}
// Exercise ordinary host/PVE snapshot ingestion, canonical views and their JSON
// projection repeatedly, not only a correlation helper or an invented identity.
func TestIssue2076USBMixedSourceSnapshot(t *testing.T) {
for _, serials := range [][2]string{{"", "USB-SERIAL"}, {"USB-SERIAL", ""}, {"", ""}, {"UNKNOWN", "USB-SERIAL"}, {"USB-SERIAL", "UNKNOWN"}} {
t.Run(fmt.Sprintf("%s/%s", serials[0], serials[1]), func(t *testing.T) {
snapshot := usbAliasSnapshot(serials[0], serials[1])
adapter := NewMonitorAdapter(NewRegistry(nil))
var id string
for cycle := 0; cycle < 4; cycle++ {
adapter.PopulateFromSnapshot(snapshot)
views := adapter.PhysicalDisks()
if len(views) != 1 {
t.Fatalf("cycle %d: disks = %d, want one linked USB device", cycle, len(views))
}
if cycle == 0 {
id = views[0].ID()
} else if views[0].ID() != id {
t.Fatal("canonical ID changed")
}
payload, err := json.Marshal(adapter.GetAll())
if err != nil {
t.Fatal(err)
}
var resources []Resource
if err := json.Unmarshal(payload, &resources); err != nil {
t.Fatal(err)
}
count := 0
for _, r := range resources {
if r.Type != ResourceTypePhysicalDisk {
continue
}
count++
if !hasDataSource(r.Sources, SourceAgent) || !hasDataSource(r.Sources, SourceProxmox) || r.ParentID == nil {
t.Fatalf("missing source/parent: %+v", r)
}
if r.PhysicalDisk == nil || r.PhysicalDisk.SizeBytes != 239_000_000_000 || r.PhysicalDisk.Temperature != 31 {
t.Fatalf("lost disk facts: %+v", r.PhysicalDisk)
}
if serials[0] == "USB-SERIAL" || serials[1] == "USB-SERIAL" {
if r.PhysicalDisk.Serial != "USB-SERIAL" {
t.Fatalf("lost serial: %+v", r.PhysicalDisk)
}
}
}
if count != 1 {
t.Fatalf("JSON disks = %d", count)
}
}
})
}
}
func TestIssue2076USBDoesNotMergeUnsafeAliases(t *testing.T) {
for _, tc := range []struct {
name string
change func(*models.StateSnapshot)
}{
{"conflicting serials", func(s *models.StateSnapshot) { s.Hosts[0].Sensors.SMART[0].Serial = "OTHER-SERIAL" }},
{"different device same size", func(s *models.StateSnapshot) { s.Hosts[0].Sensors.SMART[0].Device = "sdy" }},
{"controller member", func(s *models.StateSnapshot) { s.Hosts[0].Sensors.SMART[0].Target = "megaraid,0" }},
{"conflicting controllers", func(s *models.StateSnapshot) {
s.Hosts[0].Sensors.SMART[0].Controller = "controller-a"
s.PhysicalDisks[0].Controller = "controller-b"
}},
{"different hosts", func(s *models.StateSnapshot) {
s.Hosts[0].LinkedNodeID = ""
s.Hosts[0].Hostname = "other"
s.Nodes[0].LinkedAgentID = ""
}},
} {
t.Run(tc.name, func(t *testing.T) {
s := usbAliasSnapshot("", "USB-SERIAL")
tc.change(&s)
rr := NewRegistry(nil)
rr.IngestSnapshot(s)
if got := len(rr.ListByType(ResourceTypePhysicalDisk)); got != 2 {
t.Fatalf("disks=%d, want separate observations", got)
}
})
}
}
func TestIssue2076USBLinkedDiskAmbiguity(t *testing.T) {
for _, source := range []DataSource{SourceAgent, SourceProxmox} {
t.Run(string(source), func(t *testing.T) {
other := SourceAgent
if source == SourceAgent {
other = SourceProxmox
}
rr := NewRegistry(nil)
parent := "host:linked"
incoming := Resource{Type: ResourceTypePhysicalDisk, ParentID: &parent, PhysicalDisk: &PhysicalDiskMeta{DevPath: "sdx", DiskType: "usb"}}
for _, id := range []string{"a", "b"} {
rr.resources[id] = &Resource{ID: id, Type: ResourceTypePhysicalDisk, ParentID: &parent, Sources: []DataSource{other}, PhysicalDisk: &PhysicalDiskMeta{DevPath: "/dev/sdx", Serial: "SERIAL-" + id, DiskType: "usb"}}
}
if got := rr.resolveLinkedPhysicalDisk(source, incoming); got != "" {
t.Fatalf("ambiguous path matched %q", got)
}
delete(rr.resources, "b")
if got := rr.resolveLinkedPhysicalDisk(source, incoming); got != "a" {
t.Fatalf("unique missing-identity alias matched %q", got)
}
incoming.PhysicalDisk.WWN = "DIFFERENT-WWN"
if got := rr.resolveLinkedPhysicalDisk(source, incoming); got != "" {
t.Fatalf("conflicting hardware identity matched %q", got)
}
})
}
}
// Guest ingest resolves each guest's parent node, which can fall back to a
// scan of every agent. The guest pass answers that fallback from
// agentNodeScanIndex, so the index must resolve exactly what the scan would.
// agentNodeScanFixture spans three clusters that reuse node names, so a
// by-name bucket holds candidates from every cluster and scope scoring has to
// pick the right one. Every other node runs a linked host agent. A fourth
// cluster's nodes are keyed by their endpoint instance while its guests report
// the cluster name, so no direct source mapping reaches their parent and only
// the agent scan resolves it.
func agentNodeScanFixture(now time.Time) models.StateSnapshot {
snapshot := models.StateSnapshot{LastUpdate: now}
for n := 0; n < 2; n++ {
nodeName := fmt.Sprintf("pve%d", n)
snapshot.Nodes = append(snapshot.Nodes, models.Node{
ID: "ep-" + nodeName, Name: nodeName, Instance: "ep", ClusterName: "prod",
Status: "online", Type: "node", LastSeen: now,
})
vmid := 900 + n
snapshot.VMs = append(snapshot.VMs, models.VM{
ID: fmt.Sprintf("prod:%s:%d", nodeName, vmid), VMID: vmid,
Name: fmt.Sprintf("vm-%d", vmid), Node: nodeName, Instance: "prod",
Status: "running", Type: "qemu", LastSeen: now,
})
}
for cluster := 0; cluster < 3; cluster++ {
instance := fmt.Sprintf("lab%d", cluster)
for n := 0; n < 4; n++ {
nodeName := fmt.Sprintf("pve%d", n)
nodeID := instance + "-" + nodeName
node := models.Node{
ID: nodeID,
Name: nodeName,
Instance: instance,
Status: "online",
Type: "node",
LastSeen: now,
}
if n%2 == 0 {
hostID := fmt.Sprintf("agent-%s-%s", instance, nodeName)
node.LinkedAgentID = hostID
snapshot.Hosts = append(snapshot.Hosts, models.Host{
ID: hostID,
Hostname: fmt.Sprintf("%s.%s.example", nodeName, instance),
Status: "online",
LastSeen: now,
LinkedNodeID: nodeID,
})
}
snapshot.Nodes = append(snapshot.Nodes, node)
for g := 0; g < 3; g++ {
vmid := 100 + cluster*100 + n*10 + g
snapshot.VMs = append(snapshot.VMs, models.VM{
ID: fmt.Sprintf("%s:%s:%d", instance, nodeName, vmid), VMID: vmid,
Name: fmt.Sprintf("vm-%d", vmid), Node: nodeName, Instance: instance,
Status: "running", Type: "qemu", LastSeen: now,
})
ctid := 5000 + vmid
snapshot.Containers = append(snapshot.Containers, models.Container{
ID: fmt.Sprintf("%s:%s:%d", instance, nodeName, ctid), VMID: ctid,
Name: fmt.Sprintf("ct-%d", ctid), Node: nodeName, Instance: instance,
Status: "running", Type: "lxc", LastSeen: now,
})
}
}
}
return snapshot
}
func TestGuestIngestParentsMatchFullAgentScan(t *testing.T) {
snapshot := agentNodeScanFixture(time.Now().UTC())
rr := NewRegistry(nil)
rr.IngestSnapshot(snapshot)
rr.mu.RLock()
defer rr.mu.RUnlock()
if rr.agentNodeScanIndex != nil {
t.Fatal("agent node index outlived the ingest pass")
}
checked := 0
for id, resource := range rr.resources {
if resource.Type != ResourceTypeVM && resource.Type != ResourceTypeSystemContainer {
continue
}
if resource.Proxmox == nil {
t.Fatalf("guest %s has no Proxmox data", id)
}
// With the index nil this resolves through the full scan.
want := rr.proxmoxNodeParentIDLocked(resource.Proxmox.Instance, resource.Proxmox.ClusterName, resource.Proxmox.NodeName, "")
if want == "" {
t.Fatalf("full scan found no parent for guest %s on %s/%s", id, resource.Proxmox.Instance, resource.Proxmox.NodeName)
}
if got := CanonicalResourceID(resource.parentBySource[SourceProxmox]); got != want {
t.Fatalf("guest %s on %s/%s: ingest parent %q, full scan %q", id, resource.Proxmox.Instance, resource.Proxmox.NodeName, got, want)
}
checked++
}
if want := len(snapshot.VMs) + len(snapshot.Containers); checked != want {
t.Fatalf("checked %d guests, want %d", checked, want)
}
}
func TestAgentNodeScanIndexTracksAgentsIngestedWhileLive(t *testing.T) {
now := time.Now().UTC()
rr := NewRegistry(nil)
rr.IngestSnapshot(models.StateSnapshot{
LastUpdate: now,
Nodes: []models.Node{{ID: "lab-pve1", Name: "pve1", Instance: "lab", Status: "online", Type: "node", LastSeen: now}},
})
rr.mu.Lock()
rr.agentNodeScanIndex = rr.buildAgentNodeScanIndexLocked()
rr.mu.Unlock()
defer func() {
rr.mu.Lock()
rr.agentNodeScanIndex = nil
rr.mu.Unlock()
}()
rr.ingestProxmoxNode(models.Node{ID: "lab-pve2", Name: "pve2", Instance: "lab", Status: "online", Type: "node", LastSeen: now}, nil)
rr.mu.RLock()
defer rr.mu.RUnlock()
candidates := rr.agentNodeScanIndex["pve2"]
if len(candidates) != 1 || rr.resources[candidates[0].id] != candidates[0].resource {
t.Fatalf("index bucket for an agent ingested while live = %+v, want its registry entry", candidates)
}
}
func TestRegistryBulkMetadataInvalidatesAtRecordBoundary(t *testing.T) {
rr := NewRegistry(nil)
rr.IngestRecords(SourceAgent, []IngestRecord{{SourceID: "host", Resource: Resource{Type: ResourceTypeAgent, Name: "before", Status: StatusOnline, LastSeen: time.Now().UTC(), Agent: &AgentData{AgentID: "host", Hostname: "before.example.test"}}}})
rr.List()
previous := rr.Hosts()[0]
id := rr.ingestRecord(SourceAgent, "host", Resource{Type: ResourceTypeAgent, Name: "after", Status: StatusOnline, LastSeen: time.Now().UTC(), Agent: &AgentData{AgentID: "host", Hostname: "after.example.test"}}, ResourceIdentity{}, false)
// This is the real record boundary, before parent/rollup batch finalization.
actual := assertRegistryMetadataOracle(t, rr)
if actual[0].Canonical.Hostname != "after.example.test" || rr.Hosts()[0].Name() != "after" {
t.Fatal("record mutation reused pre-batch metadata or views")
}
if previous.Name() != "before" {
t.Fatal("record mutation rewrote an already returned view")
}
rr.retainSupersededCanonicalIDs(id, []string{"agent:prior-era"})
actual = assertRegistryMetadataOracle(t, rr)
if len(actual[0].Canonical.SupersededIDs) != 1 || actual[0].Canonical.SupersededIDs[0] != "agent:prior-era" {
t.Fatal("superseded identity retention failed to invalidate bulk metadata")
}
}
func TestEvaluateResourceHealthKeepsStaleTelemetryBehindAWinningAlert(t *testing.T) {
now := time.Date(2026, 10, 5, 1, 0, 0, 0, time.UTC)
silent := Resource{
ID: "docker-host-1",
Type: ResourceTypeAgent,
Status: StatusOnline,
LastSeen: now.Add(-70 * time.Minute),
SourceStatus: map[DataSource]SourceStatus{
SourceDocker: {Status: "stale", LastSeen: now.Add(-70 * time.Minute)},
},
}
// The alert still explains the verdict first, and the silence follows it
// so tables can stop presenting the host's last readings as live.
health := EvaluateResourceHealth(silent, []ResourceHealthAlert{{ResourceID: "docker-host-1", Level: "critical", Type: "docker-host-offline"}}, now)
if health.Verdict != HealthCritical || len(health.Reasons) != 2 {
t.Fatalf("expected critical with two reasons, got %+v", health)
}
if health.Reasons[0].Code != "critical_alert" {
t.Fatalf("expected the alert to stay the first reason, got %+v", health.Reasons)
}
if health.Reasons[1] != (ResourceHealthReason{Code: "telemetry_stale", Detail: "1h"}) {
t.Fatalf("expected a trailing telemetry_stale reason with the age, got %+v", health.Reasons[1])
}
// Without an alert the stale verdict carries the single reason as before.
health = EvaluateResourceHealth(silent, nil, now)
if health.Verdict != HealthStale || len(health.Reasons) != 1 || health.Reasons[0].Code != "telemetry_stale" {
t.Fatalf("expected a lone stale reason, got %+v", health)
}
// A fresh source adds nothing behind an alert.
fresh := silent
fresh.SourceStatus = map[DataSource]SourceStatus{SourceDocker: {Status: "online", LastSeen: now}}
health = EvaluateResourceHealth(fresh, []ResourceHealthAlert{{ResourceID: "docker-host-1", Level: "warning", Type: "cpu"}}, now)
if len(health.Reasons) != 1 {
t.Fatalf("expected only the alert reason for a fresh source, got %+v", health.Reasons)
}
}
// A List-only broadcast must capture history coordinates without constructing
// every typed view, and without trusting freshness timestamps as mutation keys.
func TestRegistryProjectionSnapshotDetachesAndTracksMappings(t *testing.T) {
rr := NewRegistry(nil)
rr.IngestSnapshot(benchmarkVMState(3))
resources, targets := rr.ListWithMetricsTargets()
if !reflect.DeepEqual(resources, rr.List()) {
t.Fatal("bulk read changed resource content")
}
if !rr.viewsDirty || rr.cachedVMs != nil {
t.Fatal("projection forced unrelated typed views to materialize")
}
if len(targets) != len(resources) {
t.Fatalf("targets=%d resources=%d", len(targets), len(resources))
}
for _, r := range resources {
if want := rr.MetricsTarget(r.ID); want == nil || *want != targets[r.ID] {
t.Fatalf("target for %s differs from point resolver", r.ID)
}
}
id := resources[0].ID
observedAt := resources[0].LastSeen
rr.mu.Lock()
for key, mapped := range rr.bySource[SourceProxmox] {
if mapped == id {
delete(rr.bySource[SourceProxmox], key)
}
}
rr.bySource[SourceProxmox]["new-history-coordinate"] = id
rr.resources[id].Tags = []string{"customer-data"}
rr.invalidateViewsLocked()
rr.mu.Unlock()
changed, updatedTargets := rr.ListWithMetricsTargets()
if updatedTargets[id].ResourceID != "new-history-coordinate" {
t.Fatal("same-time mapping change left an old target")
}
if targets[id].ResourceID == "new-history-coordinate" {
t.Fatal("later mapping edit mutated a prior target snapshot")
}
for _, r := range changed {
if r.ID == id {
if !r.LastSeen.Equal(observedAt) || r.Policy.Sensitivity != ResourceSensitivityRestricted {
t.Fatal("same-time metadata edit was stale")
}
r.Canonical.Aliases[0] = "caller-change"
r.Tags[0] = "caller-change"
r.Policy.Routing.Redact = append(r.Policy.Routing.Redact, ResourceRedactionPath)
}
}
again, againTargets := rr.ListWithMetricsTargets()
if !reflect.DeepEqual(againTargets, updatedTargets) {
t.Fatal("detached map mutation escaped")
}
for _, r := range again {
if r.ID == id && (r.Tags[0] != "customer-data" || r.Canonical.Aliases[0] == "caller-change") {
t.Fatal("caller changed live registry")
}
}
empty := NewRegistry(nil)
rows, emptyTargets := empty.ListWithMetricsTargets()
if len(rows) != 0 || len(emptyTargets) != 0 {
t.Fatal("empty projection invented resources or targets")
}
}
// Disk risk (and the warning status it drives) never judges temperature,
// collected or retained. Heat belongs to the alert disk temperature policy,
// which users tune per disk type and per host and which no registry can see.
// The reading itself stays on the resource for that policy to judge.
func TestPhysicalDiskRiskNeverJudgesTemperature(t *testing.T) {
// A fresh check time keeps the registry's source-staleness sweep out of
// the status this test reads.
now := time.Now()
retained := diskinventory.Unavailable("host_agent", "host agent stopped reporting")
cases := []struct {
name string
collection *diskinventory.CollectionStatus
}{
{name: "collected now", collection: &diskinventory.CollectionStatus{Temperature: diskinventory.Available("proxmox_node_smart")}},
{name: "source predates collection state", collection: nil},
{name: "state left empty", collection: &diskinventory.CollectionStatus{}},
{name: "retained after the agent went silent", collection: &diskinventory.CollectionStatus{Temperature: retained}},
{name: "retained while in standby", collection: &diskinventory.CollectionStatus{Temperature: diskinventory.Unavailable("smartctl", "disk is in standby")}},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
adapter := NewMonitorAdapter(NewRegistry(nil))
adapter.PopulateFromSnapshot(models.StateSnapshot{
PhysicalDisks: []models.PhysicalDisk{{
ID: "homelab-pve--dev-sda",
Node: "pve",
Instance: "homelab",
DevPath: "/dev/sda",
Model: "WDC WD80EFAX",
Serial: "SERIAL-SDA",
Type: "sata",
Health: "PASSED",
Wearout: -1,
Temperature: 72,
Collection: tc.collection,
LastChecked: now,
}},
})
var disks []Resource
for _, resource := range adapter.GetAll() {
if resource.Type == ResourceTypePhysicalDisk {
disks = append(disks, resource)
}
}
if len(disks) != 1 || disks[0].PhysicalDisk == nil {
t.Fatalf("physical disks = %+v, want one", disks)
}
disk := disks[0]
if disk.PhysicalDisk.Temperature != 72 {
t.Fatalf("temperature = %d, want the reading kept at 72", disk.PhysicalDisk.Temperature)
}
if disk.PhysicalDisk.Risk != nil || disk.Status != StatusOnline {
t.Fatalf("72C disk: status = %q risk = %+v, want online with no risk", disk.Status, disk.PhysicalDisk.Risk)
}
})
}
// A host agent's SMART row merged with the Proxmox inventory row for the
// same disk goes through the registry merge and its risk recompute.
adapter := NewMonitorAdapter(NewRegistry(nil))
adapter.PopulateFromSnapshot(models.StateSnapshot{
Hosts: []models.Host{{
ID: "host-pve",
Hostname: "pve",
Status: "online",
LastSeen: now,
Sensors: models.HostSensorSummary{SMART: []models.HostDiskSMART{{
Device: "/dev/sda",
Model: "WDC WD80EFAX",
Serial: "SERIAL-SDA",
Type: "sata",
Temperature: 72,
Health: "PASSED",
Collection: &diskinventory.CollectionStatus{Temperature: diskinventory.Available("host_agent")},
}}},
}},
PhysicalDisks: []models.PhysicalDisk{{
ID: "homelab-pve--dev-sda",
Node: "pve",
Instance: "homelab",
DevPath: "/dev/sda",
Model: "WDC WD80EFAX",
Serial: "SERIAL-SDA",
Type: "sata",
Health: "PASSED",
Wearout: -1,
Collection: &diskinventory.CollectionStatus{Temperature: diskinventory.Unsupported("proxmox_disks", "Proxmox disk inventory does not expose temperature")},
LastChecked: now,
}},
})
var merged []Resource
for _, resource := range adapter.GetAll() {
if resource.Type == ResourceTypePhysicalDisk {
merged = append(merged, resource)
}
}
if len(merged) != 1 || merged[0].PhysicalDisk == nil {
t.Fatalf("physical disks = %d, want the agent and Proxmox rows merged into one", len(merged))
}
if disk := merged[0].PhysicalDisk; disk.Temperature != 72 || disk.Risk != nil {
t.Fatalf("merged disk temperature = %d risk = %+v, want 72 with no risk", disk.Temperature, disk.Risk)
}
meta := &PhysicalDiskMeta{Health: "PASSED", Temperature: 72, Collection: &diskinventory.CollectionStatus{Temperature: diskinventory.Available("host_agent")}}
if risk := physicalDiskRiskFromMeta(meta, nil); risk != nil {
t.Fatalf("meta recompute risk = %+v, want none for a collected 72C", risk)
}
}
// The Proxmox page's Overview lists nodes without their disk inventory, so the
// outdated sensor setup verdict has to arrive on the node itself. The registry
// derives it from the node's legacy-format temperature payload and the disks
// it parents, with the same reading rule every disk temperature consumer uses.
func TestProxmoxNodeSensorSetupOutdatedFromItsDisks(t *testing.T) {
now := time.Now()
legacy := &models.Temperature{Available: true, LegacySensorsFormat: true}
inventoryOnly := &diskinventory.CollectionStatus{
Temperature: diskinventory.Unsupported("proxmox_disks", "Proxmox disk inventory does not expose temperature"),
}
node := func(name string, temperature *models.Temperature) models.Node {
return models.Node{
ID: "homelab-" + name,
Name: name,
Instance: "homelab",
Status: "online",
LastSeen: now,
Temperature: temperature,
}
}
disk := func(nodeName, devPath, diskType string, temperature int, collection *diskinventory.CollectionStatus) models.PhysicalDisk {
return models.PhysicalDisk{
ID: ProxmoxPhysicalDiskSourceID("homelab", nodeName, devPath, "", ""),
Node: nodeName,
Instance: "homelab",
DevPath: devPath,
Model: "Disk " + devPath,
Serial: "SERIAL-" + nodeName + devPath,
Type: diskType,
Health: "PASSED",
Wearout: -1,
Temperature: temperature,
Collection: collection,
LastChecked: now,
}
}
cases := []struct {
name string
nodes []models.Node
disks []models.PhysicalDisk
want bool
}{
{
name: "SATA disk without a temperature",
nodes: []models.Node{node("pve", legacy)},
disks: []models.PhysicalDisk{disk("pve", "/dev/sda", "sata", 0, inventoryOnly)},
want: true,
},
{
// Proxmox's own disk inventory types these disks by form factor.
name: "Proxmox hdd and ssd inventory types",
nodes: []models.Node{node("pve", legacy)},
disks: []models.PhysicalDisk{disk("pve", "/dev/sda", "hdd", 0, inventoryOnly), disk("pve", "/dev/sdb", "ssd", 0, inventoryOnly)},
want: true,
},
{
name: "SAS disk whose reading is retained, not current",
nodes: []models.Node{node("pve", legacy)},
disks: []models.PhysicalDisk{disk("pve", "/dev/sda", "sas", 41, &diskinventory.CollectionStatus{
Temperature: diskinventory.Unavailable("host_agent", "host agent stopped reporting"),
})},
want: true,
},
{
name: "SATA disk whose reading the source marks missing",
nodes: []models.Node{node("pve", legacy)},
disks: []models.PhysicalDisk{disk("pve", "/dev/sda", "sata", 38, &diskinventory.CollectionStatus{
Temperature: diskinventory.Missing("proxmox_node_smart", "temperature was not reported"),
})},
want: true,
},
{
name: "SATA disk with a current reading",
nodes: []models.Node{node("pve", legacy)},
disks: []models.PhysicalDisk{disk("pve", "/dev/sda", "sata", 36, &diskinventory.CollectionStatus{
Temperature: diskinventory.Available("proxmox_node_smart"),
})},
want: false,
},
{
name: "reading from a source that predates collection state",
nodes: []models.Node{node("pve", legacy)},
disks: []models.PhysicalDisk{disk("pve", "/dev/sda", "sata", 36, nil)},
want: false,
},
{
// NVMe temperatures arrive through kernel hwmon even on a legacy setup.
name: "NVMe disk without a temperature",
nodes: []models.Node{node("pve", legacy)},
disks: []models.PhysicalDisk{disk("pve", "/dev/nvme0n1", "nvme", 0, inventoryOnly)},
want: false,
},
{
name: "current setup payload",
nodes: []models.Node{node("pve", &models.Temperature{Available: true})},
disks: []models.PhysicalDisk{disk("pve", "/dev/sda", "sata", 0, inventoryOnly)},
want: false,
},
{
name: "failed temperature collection",
nodes: []models.Node{node("pve", &models.Temperature{Available: false, LegacySensorsFormat: true})},
disks: []models.PhysicalDisk{disk("pve", "/dev/sda", "sata", 0, inventoryOnly)},
want: false,
},
{
name: "waiting disk belongs to another node",
nodes: []models.Node{node("pve", legacy), node("pve2", &models.Temperature{Available: true})},
disks: []models.PhysicalDisk{disk("pve2", "/dev/sda", "sata", 0, inventoryOnly)},
want: false,
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
registry := NewRegistry(nil)
registry.IngestSnapshot(models.StateSnapshot{Nodes: tc.nodes, PhysicalDisks: tc.disks})
got := proxmoxNodeSensorSetupOutdated(t, registry.List(), "pve")
if got != tc.want {
t.Fatalf("pve sensorSetupOutdated = %v, want %v", got, tc.want)
}
})
}
}
// A linked host agent that reads the disk's SMART temperature itself answers
// what the legacy SSH setup cannot, so the node is not flagged; the verdict is
// derived on every ingest, so a later snapshot clears an earlier flag.
func TestProxmoxNodeSensorSetupOutdatedFollowsWhereTheReadingArrives(t *testing.T) {
now := time.Now()
nodes := []models.Node{{
ID: "homelab-pve",
Name: "pve",
Instance: "homelab",
LinkedAgentID: "host-pve",
Status: "online",
LastSeen: now,
Temperature: &models.Temperature{Available: true, LegacySensorsFormat: true},
}}
inventory := []models.PhysicalDisk{{
ID: ProxmoxPhysicalDiskSourceID("homelab", "pve", "/dev/sda", "", ""),
Node: "pve",
Instance: "homelab",
DevPath: "/dev/sda",
Model: "WDC WD80EFAX",
Serial: "SERIAL-SDA",
Type: "hdd",
Health: "PASSED",
Wearout: -1,
Collection: &diskinventory.CollectionStatus{Temperature: diskinventory.Unsupported("proxmox_disks", "Proxmox disk inventory does not expose temperature")},
LastChecked: now,
}}
agent := func(collection *diskinventory.CollectionStatus) []models.Host {
return []models.Host{{
ID: "host-pve",
Hostname: "pve",
LinkedNodeID: "homelab-pve",
Status: "online",
LastSeen: now,
Sensors: models.HostSensorSummary{SMART: []models.HostDiskSMART{{
Device: "/dev/sda",
Model: "WDC WD80EFAX",
Serial: "SERIAL-SDA",
Type: "sata",
Temperature: 34,
Health: "PASSED",
Collection: collection,
}}},
}}
}
registry := NewRegistry(nil)
registry.IngestSnapshot(models.StateSnapshot{Nodes: nodes, PhysicalDisks: inventory})
if !proxmoxNodeSensorSetupOutdated(t, registry.List(), "pve") {
t.Fatal("legacy node with an unread hdd: sensorSetupOutdated = false, want true")
}
registry.IngestSnapshot(models.StateSnapshot{
Nodes: nodes,
PhysicalDisks: inventory,
Hosts: agent(&diskinventory.CollectionStatus{Temperature: diskinventory.Available("host_agent")}),
})
if proxmoxNodeSensorSetupOutdated(t, registry.List(), "pve") {
t.Fatal("linked agent reads the disk now: sensorSetupOutdated = true, want the earlier flag cleared")
}
// Clients merge a facet field by field, so the cleared verdict must be on
// the wire as false, not omitted.
assertProxmoxNodeSensorSetupJSON(t, registry.ListForPresentation(), "pve", `"sensorSetupOutdated":false`)
registry.IngestSnapshot(models.StateSnapshot{
Nodes: nodes,
PhysicalDisks: inventory,
Hosts: agent(&diskinventory.CollectionStatus{Temperature: diskinventory.Unavailable("host_agent", "host agent stopped reporting")}),
})
presented := registry.ListForPresentation()
if !proxmoxNodeSensorSetupOutdated(t, presented, "pve") {
t.Fatal("agent reading retained, not current: sensorSetupOutdated = false, want true")
}
assertProxmoxNodeSensorSetupJSON(t, presented, "pve", `"sensorSetupOutdated":true`)
}
func assertProxmoxNodeSensorSetupJSON(t *testing.T, resources []Resource, nodeName, want string) {
t.Helper()
for _, resource := range resources {
if resource.Proxmox == nil || resource.Proxmox.NodeName != nodeName || CanonicalResourceType(resource.Type) != ResourceTypeAgent {
continue
}
payload, err := json.Marshal(resource.Proxmox)
if err != nil {
t.Fatalf("marshal node proxmox facet: %v", err)
}
if !strings.Contains(string(payload), want) {
t.Fatalf("node proxmox payload = %s, want %s for list readers", payload, want)
}
return
}
t.Fatalf("node %q not presented", nodeName)
}
func proxmoxNodeSensorSetupOutdated(t *testing.T, resources []Resource, nodeName string) bool {
t.Helper()
var found *Resource
for i := range resources {
resource := &resources[i]
if CanonicalResourceType(resource.Type) != ResourceTypeAgent || resource.Proxmox == nil || resource.Proxmox.NodeName != nodeName {
continue
}
if found != nil {
t.Fatalf("node %q presented twice: %s and %s", nodeName, found.ID, resource.ID)
}
found = resource
}
if found == nil {
t.Fatalf("node %q not found among %d resources", nodeName, len(resources))
}
for _, resource := range resources {
if CanonicalResourceType(resource.Type) != ResourceTypeAgent && resource.Proxmox != nil && resource.Proxmox.SensorSetupOutdated != nil {
t.Fatalf("%s %s carries the node verdict", resource.Type, resource.ID)
}
}
if found.Proxmox.SensorSetupOutdated == nil {
t.Fatalf("node %q has no explicit sensor setup verdict", nodeName)
}
return *found.Proxmox.SensorSetupOutdated
}
// An operator's remediation lock is a safety property of the thing they
// locked, not of one canonical ID. Linking a locked resource into another
// folds it out of the registry, and the surviving resource is the only ID the
// planner and the dispatch gate ever read, so the lock has to be readable
// there. Lock state is judged the way the gates judge it: the stored row of
// the ID the registry lists.
func TestResourceRegistry_ManualLinkFoldKeepsRemediationLock(t *testing.T) {
now := time.Now().UTC()
const (
guestID = "vm-cluster-a-node-1-501"
agentID = "agent-inside-vm-501"
)
ingest := func(rr *ResourceRegistry) {
rr.IngestResources([]Resource{
{
ID: guestID, Type: ResourceTypeVM, Name: "vm-501", Status: StatusOnline, LastSeen: now,
Sources: []DataSource{SourceProxmox},
SourceStatus: map[DataSource]SourceStatus{SourceProxmox: {Status: "online", LastSeen: now}},
Proxmox: &ProxmoxData{SourceID: "cluster-a-node-1-501", VMID: 501},
},
{
ID: agentID, Type: ResourceTypeAgent, Name: "vm-501", Status: StatusOnline, LastSeen: now,
Sources: []DataSource{SourceAgent},
SourceStatus: map[DataSource]SourceStatus{SourceAgent: {Status: "online", LastSeen: now}},
Agent: &AgentData{AgentID: agentID},
},
})
}
stores := map[string]func(t *testing.T) ResourceStore{
"memory": func(t *testing.T) ResourceStore { return NewMemoryStore() },
"sqlite": func(t *testing.T) ResourceStore {
store, err := NewSQLiteResourceStore(t.TempDir(), "default")
if err != nil {
t.Fatalf("NewSQLiteResourceStore: %v", err)
}
t.Cleanup(func() { _ = store.Close() })
return store
},
}
locks := map[string]ResourceOperatorState{
"never auto-remediate": {NeverAutoRemediate: true},
"retired": {LifecycleState: LifecycleStateRetired},
}
for storeName, newStore := range stores {
for lockName, lock := range locks {
for _, tc := range []struct {
name string
locked string
// primary is the side the link names as its primary. The registry
// keeps a guest over an agent running inside it whichever way the
// link points, so a lock on the guest-folded agent has to survive
// a link that names the agent the primary and a link that names the
// guest.
primary string
}{
{name: "locked agent folded into the guest, agent named primary", locked: agentID, primary: agentID},
{name: "locked agent folded into the guest, guest named primary", locked: agentID, primary: guestID},
{name: "control: locked guest absorbing the agent", locked: guestID, primary: agentID},
} {
t.Run(storeName+"/"+lockName+"/"+tc.name, func(t *testing.T) {
store := newStore(t)
state := lock
state.CanonicalID = tc.locked
state.SetAt = now.Add(-time.Hour)
state.SetBy = "operator"
if err := store.SetResourceOperatorState(state); err != nil {
t.Fatalf("lock %s: %v", tc.locked, err)
}
if err := store.AddLink(ResourceLink{ResourceA: guestID, ResourceB: agentID, PrimaryID: tc.primary}); err != nil {
t.Fatalf("add link: %v", err)
}
rr := NewRegistry(store)
ingest(rr)
survivors := rr.List()
if len(survivors) != 1 {
t.Fatalf("link did not fold the pair into one resource: %d resources", len(survivors))
}
survivor := survivors[0].ID
got, found, err := store.GetResourceOperatorState(survivor)
if err != nil {
t.Fatalf("read operator state of survivor %s: %v", survivor, err)
}
if !found || !got.BlocksRemediation() {
t.Fatalf("surviving resource %s lost the lock on %s: found=%v state=%+v", survivor, tc.locked, found, got)
}
if got.SuppressesAllAttention() != (lockName == "retired" && tc.locked == survivor) {
t.Fatalf("lock on %s changed the survivor's alert attention: %+v", tc.locked, got)
}
})
}
}
}
}
// Carrying the lock must not rewrite anything else the survivor's operator
// said: its note, priority and monitoring posture stay as they were.
func TestResourceRegistry_ManualLinkFoldLockLeavesSurvivorSettings(t *testing.T) {
now := time.Now().UTC()
store := NewMemoryStore()
const (
keeperID = "vm-cluster-a-node-1-502"
foldedID = "agent-inside-vm-502"
)
if err := store.SetResourceOperatorState(ResourceOperatorState{
CanonicalID: keeperID,
MonitoringMode: MonitoringModeExpectedOffline,
Criticality: CriticalityHigh,
Note: "database host",
SetAt: now.Add(-time.Hour),
SetBy: "alice",
}); err != nil {
t.Fatalf("seed survivor state: %v", err)
}
if err := store.SetResourceOperatorState(ResourceOperatorState{
CanonicalID: foldedID, NeverAutoRemediate: true, SetAt: now.Add(-2 * time.Hour), SetBy: "bob",
}); err != nil {
t.Fatalf("seed folded state: %v", err)
}
if err := store.AddLink(ResourceLink{ResourceA: keeperID, ResourceB: foldedID, PrimaryID: keeperID}); err != nil {
t.Fatalf("add link: %v", err)
}
got, found, err := store.GetResourceOperatorState(keeperID)
if err != nil || !found {
t.Fatalf("survivor state missing: found=%v err=%v", found, err)
}
if !got.NeverAutoRemediate {
t.Fatalf("survivor did not take the folded resource's lock: %+v", got)
}
if got.MonitoringMode != MonitoringModeExpectedOffline || got.Criticality != CriticalityHigh || got.Note != "database host" || got.LifecycleState != LifecycleStateActive {
t.Fatalf("carrying the lock rewrote the survivor's own settings: %+v", got)
}
if got.SetBy != RemediationLockCarriedBy {
t.Fatalf("carried lock is attributed to %q, not to the identity change", got.SetBy)
}
// Unlinking splits the pair again; the lock was the operator's on both, so
// neither side loses it.
if err := store.AddExclusion(ResourceExclusion{ResourceA: keeperID, ResourceB: foldedID}); err != nil {
t.Fatalf("unlink: %v", err)
}
for _, id := range []string{keeperID, foldedID} {
if state, found, err := store.GetResourceOperatorState(id); err != nil || !found || !state.NeverAutoRemediate {
t.Fatalf("unlinking dropped the lock on %s: found=%v err=%v state=%+v", id, found, err, state)
}
}
}
// Links recorded before locks travelled with them already folded a locked
// resource out from under its lock, and a succession that landed on an existing
// successor row left its predecessor's lock behind. Opening a store that has
// not yet run the carry restores those once; an operator who then lifts the
// survivor's lock is not overruled by the dormant row on every later start.
func TestSQLiteStoreRestoresLocksEarlierIdentityChangesDropped(t *testing.T) {
dir := t.TempDir()
const (
lockedID = "agent-legacy-locked"
survivorID = "vm-legacy-survivor"
chainEndID = "vm-legacy-chain-end"
unlinkedA = "agent-legacy-unlinked-locked"
unlinkedB = "vm-legacy-unlinked-open"
oldEraID = "agent-legacy-old-era"
newEraID = "agent-legacy-new-era"
linkedAt = "2026-09-01T10:00:00Z"
unlinkedAt = "2026-09-02T10:00:00Z"
operator = "operator"
legacyReason = "recorded before locks travelled"
)
open := func() *SQLiteResourceStore {
t.Helper()
store, err := NewSQLiteResourceStore(dir, "default")
if err != nil {
t.Fatalf("NewSQLiteResourceStore: %v", err)
}
return store
}
now := time.Now().UTC()
store := open()
for _, id := range []string{lockedID, unlinkedA, oldEraID} {
if err := store.SetResourceOperatorState(ResourceOperatorState{CanonicalID: id, NeverAutoRemediate: true, SetAt: now.Add(-48 * time.Hour), SetBy: operator}); err != nil {
t.Fatalf("lock %s: %v", id, err)
}
}
// A successor row written before the succession was declared, as the
// succession that shadowed the predecessor's lock found it.
if err := store.SetResourceOperatorState(ResourceOperatorState{CanonicalID: newEraID, Note: "set under the new ID", SetAt: now.Add(-24 * time.Hour), SetBy: operator}); err != nil {
t.Fatalf("seed successor: %v", err)
}
// Rows exactly as a store that did not carry locks wrote them: a live link,
// a chain extension of it, one a later unlink superseded (its exclusion is
// newer), and a succession already recorded with the predecessor's row left
// behind.
for _, row := range []struct{ a, b, primary, at string }{
{lockedID, survivorID, survivorID, linkedAt},
{survivorID, chainEndID, chainEndID, linkedAt},
{unlinkedA, unlinkedB, unlinkedB, linkedAt},
} {
if _, err := store.db.Exec(`INSERT INTO resource_links (resource_a, resource_b, primary_id, reason, created_by, created_at) VALUES (?, ?, ?, ?, ?, ?)`,
row.a, row.b, row.primary, legacyReason, operator, row.at); err != nil {
t.Fatalf("seed legacy link: %v", err)
}
}
if _, err := store.db.Exec(`INSERT INTO resource_exclusions (resource_a, resource_b, reason, created_by, created_at) VALUES (?, ?, ?, ?, ?)`,
unlinkedA, unlinkedB, "unlinked", operator, unlinkedAt); err != nil {
t.Fatalf("seed legacy exclusion: %v", err)
}
if _, err := store.db.Exec(`INSERT INTO canonical_id_successions (old_canonical_id, new_canonical_id) VALUES (?, ?)`, oldEraID, newEraID); err != nil {
t.Fatalf("seed recorded succession: %v", err)
}
// This store has never run the carry.
if _, err := store.db.Exec(`DELETE FROM resource_store_migrations`); err != nil {
t.Fatalf("forget the carry marker: %v", err)
}
if err := store.Close(); err != nil {
t.Fatalf("close: %v", err)
}
store = open()
for _, id := range []string{survivorID, chainEndID, newEraID} {
got, found, err := store.GetResourceOperatorState(id)
if err != nil || !found || !got.BlocksRemediation() {
t.Fatalf("%s did not get the lock an earlier identity change dropped: found=%v err=%v state=%+v", id, found, err, got)
}
}
if got, _, _ := store.GetResourceOperatorState(newEraID); got.Note != "set under the new ID" {
t.Fatalf("restoring the successor's lock rewrote its own settings: %+v", got)
}
if other, found, err := store.GetResourceOperatorState(unlinkedB); err != nil || found {
t.Fatalf("a pair the operator unlinked was carried: found=%v err=%v state=%+v", found, err, other)
}
for _, id := range []string{survivorID, newEraID} {
if err := store.ClearResourceOperatorState(id); err != nil {
t.Fatalf("operator clears %s: %v", id, err)
}
}
if err := store.Close(); err != nil {
t.Fatalf("close: %v", err)
}
store = open()
defer store.Close()
for _, id := range []string{survivorID, newEraID} {
if got, found, err := store.GetResourceOperatorState(id); err != nil || found {
t.Fatalf("a restart re-locked %s, which the operator cleared: found=%v err=%v state=%+v", id, found, err, got)
}
}
}
// The registry keeps one member of a link and acts only on that ID, so a lock
// has to be on every member of the linked component, not just the two IDs of
// the newest link, and it has to stay on each member the unlink splits off.
func TestLinkedComponentSharesRemediationLockThroughLinksAndUnlinks(t *testing.T) {
stores := map[string]func(t *testing.T) ResourceStore{
"memory": func(t *testing.T) ResourceStore { return NewMemoryStore() },
"sqlite": func(t *testing.T) ResourceStore {
store, err := NewSQLiteResourceStore(t.TempDir(), "default")
if err != nil {
t.Fatalf("NewSQLiteResourceStore: %v", err)
}
t.Cleanup(func() { _ = store.Close() })
return store
},
}
locked := func(t *testing.T, store ResourceStore, id string) bool {
t.Helper()
state, found, err := store.GetResourceOperatorState(id)
if err != nil {
t.Fatalf("read operator state of %s: %v", id, err)
}
return found && state.BlocksRemediation()
}
now := time.Now().UTC()
for name, newStore := range stores {
t.Run(name+"/chain", func(t *testing.T) {
store := newStore(t)
if err := store.AddLink(ResourceLink{ResourceA: "host-a", ResourceB: "host-b", PrimaryID: "host-a"}); err != nil {
t.Fatalf("link a-b: %v", err)
}
if err := store.SetResourceOperatorState(ResourceOperatorState{CanonicalID: "host-c", NeverAutoRemediate: true, SetAt: now}); err != nil {
t.Fatalf("lock c: %v", err)
}
if err := store.AddLink(ResourceLink{ResourceA: "host-b", ResourceB: "host-c", PrimaryID: "host-c"}); err != nil {
t.Fatalf("link b-c: %v", err)
}
for _, id := range []string{"host-a", "host-b", "host-c"} {
if !locked(t, store, id) {
t.Fatalf("%s is outside the lock of its linked component", id)
}
}
})
t.Run(name+"/lock set on the survivor after the link, then unlinked", func(t *testing.T) {
store := newStore(t)
if err := store.AddLink(ResourceLink{ResourceA: "vm-1", ResourceB: "agent-1", PrimaryID: "vm-1"}); err != nil {
t.Fatalf("link: %v", err)
}
if err := store.SetResourceOperatorState(ResourceOperatorState{CanonicalID: "vm-1", NeverAutoRemediate: true, SetAt: now}); err != nil {
t.Fatalf("operator locks the merged resource: %v", err)
}
if locked(t, store, "agent-1") {
t.Fatalf("the lock reached the folded member before any identity change")
}
if err := store.AddExclusion(ResourceExclusion{ResourceA: "vm-1", ResourceB: "agent-1"}); err != nil {
t.Fatalf("unlink: %v", err)
}
for _, id := range []string{"vm-1", "agent-1"} {
if !locked(t, store, id) {
t.Fatalf("unlinking left %s without the merged resource's lock", id)
}
}
})
t.Run(name+"/lock on one end of a chain, unlinking the other end", func(t *testing.T) {
store := newStore(t)
for _, link := range []ResourceLink{
{ResourceA: "chain-a", ResourceB: "chain-b", PrimaryID: "chain-a"},
{ResourceA: "chain-b", ResourceB: "chain-c", PrimaryID: "chain-b"},
} {
if err := store.AddLink(link); err != nil {
t.Fatalf("link %s-%s: %v", link.ResourceA, link.ResourceB, err)
}
}
if err := store.SetResourceOperatorState(ResourceOperatorState{CanonicalID: "chain-a", NeverAutoRemediate: true, SetAt: now}); err != nil {
t.Fatalf("operator locks the merged resource: %v", err)
}
if err := store.AddExclusion(ResourceExclusion{ResourceA: "chain-b", ResourceB: "chain-c"}); err != nil {
t.Fatalf("unlink b-c: %v", err)
}
for _, id := range []string{"chain-a", "chain-b", "chain-c"} {
if !locked(t, store, id) {
t.Fatalf("%s left the chain without the lock the whole chain carried", id)
}
}
})
t.Run(name+"/unlinking a pair that was never linked carries nothing", func(t *testing.T) {
store := newStore(t)
if err := store.SetResourceOperatorState(ResourceOperatorState{CanonicalID: "vm-2", NeverAutoRemediate: true, SetAt: now}); err != nil {
t.Fatalf("lock: %v", err)
}
if err := store.AddExclusion(ResourceExclusion{ResourceA: "vm-2", ResourceB: "agent-2"}); err != nil {
t.Fatalf("exclude: %v", err)
}
if locked(t, store, "agent-2") {
t.Fatalf("an exclusion between unlinked resources locked one of them")
}
})
}
}
// Recorded successions can chain, and an intermediate row may have moved on to
// its own successor, so the upgrade carry follows the chain instead of
// stepping one succession at a time; a lock it restores then spreads over the
// links of the ID it lands on.
func TestSQLiteStoreRestoresLocksAcrossRecordedSuccessionChains(t *testing.T) {
dir := t.TempDir()
open := func() *SQLiteResourceStore {
t.Helper()
store, err := NewSQLiteResourceStore(dir, "default")
if err != nil {
t.Fatalf("NewSQLiteResourceStore: %v", err)
}
return store
}
now := time.Now().UTC()
store := open()
defer func() { _ = store.Close() }()
// a1 -> b1 -> c1: b1's settings row moved to c1, so b1 has no row and the
// shadowed lock on a1 has to reach c1 across the gap.
// a2 -> b2 -> c2, recorded in the opposite order from the chain: both
// intermediate rows still exist.
// a3 -> b3 with a live link b3 - c3: the restored lock must reach c3.
seed := func(id string, state ResourceOperatorState) {
t.Helper()
state.CanonicalID, state.SetAt = id, now.Add(-72*time.Hour)
if err := store.SetResourceOperatorState(state); err != nil {
t.Fatalf("seed %s: %v", id, err)
}
}
seed("chain-a1", ResourceOperatorState{NeverAutoRemediate: true})
seed("chain-c1", ResourceOperatorState{Note: "moved here from b1"})
seed("chain-a2", ResourceOperatorState{NeverAutoRemediate: true})
seed("chain-b2", ResourceOperatorState{Note: "unlocked"})
seed("chain-c2", ResourceOperatorState{Note: "unlocked"})
seed("chain-a3", ResourceOperatorState{LifecycleState: LifecycleStateRetired})
seed("chain-b3", ResourceOperatorState{Note: "unlocked"})
// a4 -> b4 and c4 -> d4 with a live link b4 - c4: the lock restored onto b4
// spreads over the link to c4, and from c4 on to d4, which the first pass
// over the successions had already passed.
seed("chain-a4", ResourceOperatorState{NeverAutoRemediate: true})
seed("chain-b4", ResourceOperatorState{Note: "unlocked"})
seed("chain-c4", ResourceOperatorState{Note: "unlocked"})
seed("chain-d4", ResourceOperatorState{Note: "unlocked"})
for _, succession := range [][2]string{
{"chain-b2", "chain-c2"}, {"chain-a2", "chain-b2"},
{"chain-a1", "chain-b1"}, {"chain-b1", "chain-c1"},
{"chain-a3", "chain-b3"},
{"chain-a4", "chain-b4"}, {"chain-c4", "chain-d4"},
} {
if _, err := store.db.Exec(`INSERT INTO canonical_id_successions (old_canonical_id, new_canonical_id) VALUES (?, ?)`, succession[0], succession[1]); err != nil {
t.Fatalf("record succession %v: %v", succession, err)
}
}
for _, link := range [][2]string{{"chain-b3", "chain-c3"}, {"chain-b4", "chain-c4"}} {
if _, err := store.db.Exec(`INSERT INTO resource_links (resource_a, resource_b, primary_id, reason, created_by, created_at) VALUES (?, ?, ?, ?, ?, ?)`,
link[0], link[1], link[0], "linked before locks travelled", "operator", "2026-09-01T10:00:00Z"); err != nil {
t.Fatalf("seed link: %v", err)
}
}
if _, err := store.db.Exec(`DELETE FROM resource_store_migrations`); err != nil {
t.Fatalf("forget the carry marker: %v", err)
}
if err := store.Close(); err != nil {
t.Fatalf("close: %v", err)
}
store = open()
for _, id := range []string{"chain-c1", "chain-b2", "chain-c2", "chain-b3", "chain-c3", "chain-b4", "chain-c4", "chain-d4"} {
got, found, err := store.GetResourceOperatorState(id)
if err != nil || !found || !got.BlocksRemediation() {
t.Fatalf("%s did not get the lock its succession chain dropped: found=%v err=%v state=%+v", id, found, err, got)
}
}
if got, _, _ := store.GetResourceOperatorState("chain-c1"); got.Note != "moved here from b1" {
t.Fatalf("restoring the lock rewrote the successor's own settings: %+v", got)
}
}
// A host agent that reports no strong key is keyed by its source-specific ID
// and holds no identity pin (a pin needs a machine ID, a DMI UUID or a cluster
// slot), so when it starts reporting one the pin-driven succession has no
// predecessor to name and the operator's rows stayed on the ID the host no
// longer has. The retired ID is recomputable from the report itself, as a
// Proxmox guest's is, so the registry declares it. Each rebuild is a fresh
// registry on the durable store followed by the pin persist, as the monitor
// runs it.
func TestReporterGainingAStrongKeyKeepsItsRemediationLock(t *testing.T) {
stores := map[string]func(t *testing.T) ResourceStore{
"memory": func(t *testing.T) ResourceStore { return NewMemoryStore() },
"sqlite": func(t *testing.T) ResourceStore {
store, err := NewSQLiteResourceStore(t.TempDir(), "default")
if err != nil {
t.Fatalf("NewSQLiteResourceStore: %v", err)
}
t.Cleanup(func() { _ = store.Close() })
return store
},
}
locks := map[string]ResourceOperatorState{
"never auto-remediate": {NeverAutoRemediate: true},
"retired": {LifecycleState: LifecycleStateRetired},
}
now := time.Now().UTC()
host := models.Host{ID: "agent-standalone-01", Hostname: "standalone-01", Status: "online", LastSeen: now}
dockerHost := models.DockerHost{ID: "docker-standalone-02", Hostname: "standalone-02", Status: "online", LastSeen: now}
// Both reporters own their identity, so both are keyed by the key they
// report: the host agent by /etc/machine-id, the Docker host by the engine's
// machine ID, or by the Swarm cluster it joins (a cluster slot).
swarmHost := models.DockerHost{ID: "docker-standalone-05", Hostname: "standalone-05", Status: "online", LastSeen: now}
reporters := map[string]struct {
snapshot func(key string) models.StateSnapshot
hostname string
}{
"host agent": {hostname: host.Hostname, snapshot: func(machineID string) models.StateSnapshot {
reporter := host
reporter.MachineID = machineID
return models.StateSnapshot{Hosts: []models.Host{reporter}}
}},
"docker host": {hostname: dockerHost.Hostname, snapshot: func(machineID string) models.StateSnapshot {
reporter := dockerHost
reporter.MachineID = machineID
return models.StateSnapshot{DockerHosts: []models.DockerHost{reporter}}
}},
"docker host joining a swarm": {hostname: swarmHost.Hostname, snapshot: func(key string) models.StateSnapshot {
reporter := swarmHost
if key != "" {
reporter.Swarm = &models.DockerSwarmInfo{ClusterID: "swarm-a", ClusterName: "swarm-a"}
}
return models.StateSnapshot{DockerHosts: []models.DockerHost{reporter}}
}},
}
rebuild := func(t *testing.T, store ResourceStore, snapshot models.StateSnapshot) string {
t.Helper()
rr := NewRegistry(store)
rr.IngestSnapshot(snapshot)
rr.PersistIdentityPins()
agents := rr.ListByType(ResourceTypeAgent)
if len(agents) != 1 {
t.Fatalf("rebuild listed %d agents, want the one reporter", len(agents))
}
return agents[0].ID
}
for reporterName, reporter := range reporters {
for storeName, newStore := range stores {
for lockName, lock := range locks {
t.Run(reporterName+"/"+storeName+"/"+lockName, func(t *testing.T) {
store := newStore(t)
keyless := rebuild(t, store, reporter.snapshot(""))
state := lock
state.CanonicalID, state.SetAt, state.SetBy = keyless, now.Add(-time.Hour), "operator"
state.Note = "set before the reporter had a machine ID"
if err := store.SetResourceOperatorState(state); err != nil {
t.Fatalf("lock %s: %v", keyless, err)
}
keyed := rebuild(t, store, reporter.snapshot("machine-"+reporter.hostname))
if keyed == keyless {
t.Fatalf("the reporter kept ID %s after gaining a machine ID; the fixture no longer changes its era", keyless)
}
got, found, err := store.GetResourceOperatorState(keyed)
if err != nil {
t.Fatalf("read operator state of %s: %v", keyed, err)
}
if !found || !got.BlocksRemediation() {
t.Fatalf("the reporter's new ID %s lost the lock on %s: found=%v state=%+v", keyed, keyless, found, got)
}
if got.Note != state.Note {
t.Fatalf("the reporter's operator row did not move with it: %+v", got)
}
// The operator owns the lock from here: clearing it must hold
// across later rebuilds, which declare the same era again.
if err := store.ClearResourceOperatorState(keyed); err != nil {
t.Fatalf("operator clears %s: %v", keyed, err)
}
if again := rebuild(t, store, reporter.snapshot("machine-"+reporter.hostname)); again != keyed {
t.Fatalf("a steady-state rebuild moved the reporter from %s to %s", keyed, again)
}
if restored, found, err := store.GetResourceOperatorState(keyed); err != nil || found {
t.Fatalf("a later rebuild restored a row the operator cleared: found=%v err=%v state=%+v", found, err, restored)
}
})
}
}
}
}
// The era a reporter leaves is declared only for the reporters that own their
// identity and only when the report carries a key: a reporter that stays
// keyless keeps its ID, saved-host continuity (which may only introduce absent
// resources) moves no operator row, and a Proxmox node that borrows its linked
// agent's key is not an era of the node's own source-specific ID.
func TestReporterEraIsDeclaredOnlyForAKeyTheReporterGains(t *testing.T) {
stores := map[string]func(t *testing.T) ResourceStore{
"memory": func(t *testing.T) ResourceStore { return NewMemoryStore() },
"sqlite": func(t *testing.T) ResourceStore {
store, err := NewSQLiteResourceStore(t.TempDir(), "default")
if err != nil {
t.Fatalf("NewSQLiteResourceStore: %v", err)
}
t.Cleanup(func() { _ = store.Close() })
return store
},
}
recorded := func(t *testing.T, store ResourceStore) map[string]string {
t.Helper()
switch s := store.(type) {
case *SQLiteResourceStore:
return s.successionMap()
case *MemoryStore:
return s.canonicalSuccessions
}
t.Fatalf("%T records no successions", store)
return nil
}
lock := func(t *testing.T, store ResourceStore, id string) {
t.Helper()
if err := store.SetResourceOperatorState(ResourceOperatorState{CanonicalID: id, NeverAutoRemediate: true, SetAt: time.Now().UTC(), SetBy: "operator"}); err != nil {
t.Fatalf("lock %s: %v", id, err)
}
}
locked := func(t *testing.T, store ResourceStore, id string) bool {
t.Helper()
state, found, err := store.GetResourceOperatorState(id)
if err != nil {
t.Fatalf("read operator state of %s: %v", id, err)
}
return found && state.BlocksRemediation()
}
now := time.Now().UTC()
host := func(machineID string) models.Host {
return models.Host{ID: "agent-standalone-03", Hostname: "standalone-03", MachineID: machineID, Status: "online", LastSeen: now}
}
hostEra := SourceSpecificID(ResourceTypeAgent, SourceAgent, "agent-standalone-03")
for storeName, newStore := range stores {
t.Run(storeName+"/a reporter that stays keyless declares nothing", func(t *testing.T) {
store := newStore(t)
lock(t, store, hostEra)
for range 2 {
rr := NewRegistry(store)
rr.IngestSnapshot(models.StateSnapshot{Hosts: []models.Host{host("")}})
rr.PersistIdentityPins()
if ids := rr.ListByType(ResourceTypeAgent); len(ids) != 1 || ids[0].ID != hostEra {
t.Fatalf("a keyless host is listed as %+v, want its source-specific ID %s", ids, hostEra)
}
}
if got := recorded(t, store); len(got) != 0 {
t.Fatalf("a keyless reporter declared successions %v", got)
}
if !locked(t, store, hostEra) {
t.Fatalf("the keyless host lost the lock on its own ID")
}
})
// A keyless agent linked to a Proxmox node lands on the node's row. That
// is a merge by link, not a key the reporter gained, so its
// source-specific ID is not declared retired onto the node's.
t.Run(storeName+"/a keyless agent absorbed by a linked node declares nothing", func(t *testing.T) {
store := newStore(t)
snapshot := sharedSerialDiskSnapshot(now, true, "pve1")
snapshot.Hosts[0].MachineID = ""
linkedHostEra := SourceSpecificID(ResourceTypeAgent, SourceAgent, snapshot.Hosts[0].ID)
lock(t, store, linkedHostEra)
rr := NewRegistry(store)
rr.IngestSnapshot(snapshot)
rr.PersistIdentityPins()
if agents := rr.ListByType(ResourceTypeAgent); len(agents) != 1 || agents[0].ID == linkedHostEra || !containsDataSource(agents[0].Sources, SourceAgent) {
t.Fatalf("the keyless agent is listed as %+v, want it merged into the node's row", agents)
}
if got := recorded(t, store); len(got) != 0 {
t.Fatalf("a keyless agent absorbed by a node declared successions %v", got)
}
if !locked(t, store, linkedHostEra) {
t.Fatalf("the agent's lock moved off %s", linkedHostEra)
}
})
t.Run(storeName+"/saved-host continuity moves no operator row", func(t *testing.T) {
store := newStore(t)
lock(t, store, hostEra)
rr := NewRegistry(store)
rr.ingestRecords(SourceAgent, []IngestRecord{HostIngestRecord(host("machine-standalone-03"))}, true, nil)
if len(rr.ListByType(ResourceTypeAgent)) != 1 {
t.Fatalf("continuity did not introduce the saved host")
}
if got := recorded(t, store); len(got) != 0 {
t.Fatalf("continuity hydration declared successions %v", got)
}
if !locked(t, store, hostEra) {
t.Fatalf("continuity hydration moved the lock off %s", hostEra)
}
})
t.Run(storeName+"/record ingest declares the era like a snapshot", func(t *testing.T) {
store := newStore(t)
lock(t, store, hostEra)
rr := NewRegistry(store)
rr.IngestRecords(SourceAgent, []IngestRecord{HostIngestRecord(host("machine-standalone-03"))})
agents := rr.ListByType(ResourceTypeAgent)
if len(agents) != 1 || agents[0].ID == hostEra {
t.Fatalf("a keyed host record is listed as %+v, want an ID other than its source-specific %s", agents, hostEra)
}
if !locked(t, store, agents[0].ID) {
t.Fatalf("the record-fed reporter's new ID %s lost the lock on %s", agents[0].ID, hostEra)
}
})
// An operator split names the split source by its source-specific ID.
// While the other side is present the source lands on that ID and
// declares nothing; the generation the other side is absent it lands on
// the ID it was split from, and declaring its source-specific ID retired
// would rewrite the pair into an exclusion of the resource from itself,
// so the split would be gone when the other side returned. The pins
// decide whether pin succession also reaches the pair (the split source
// and its sibling both pin the shared machine key, and the store keeps
// one); this holds the key on the sibling, where only a reporter era
// could rewrite the pair.
t.Run(storeName+"/a split source keeps its exclusion while its sibling is absent", func(t *testing.T) {
store := newStore(t)
machineID := "machine-shared-04"
agentHost := models.Host{ID: "agent-shared-04", Hostname: "shared-04", MachineID: machineID, Status: "online", LastSeen: now}
dockerHost := models.DockerHost{ID: "docker-shared-04", Hostname: "shared-04", MachineID: machineID, Status: "online", LastSeen: now}
merged := MachineIdentityCanonicalID(ResourceTypeAgent, machineID)
candidate := SourceSpecificID(ResourceTypeAgent, SourceDocker, dockerHost.ID)
if err := store.AddExclusion(ResourceExclusion{ResourceA: merged, ResourceB: candidate}); err != nil {
t.Fatalf("split the Docker source from %s: %v", merged, err)
}
agents := func(snapshot models.StateSnapshot) int {
t.Helper()
rr := NewRegistry(store)
rr.IngestSnapshot(snapshot)
rr.PersistIdentityPins()
return len(rr.ListByType(ResourceTypeAgent))
}
both := models.StateSnapshot{Hosts: []models.Host{agentHost}, DockerHosts: []models.DockerHost{dockerHost}}
if got := agents(both); got != 2 {
t.Fatalf("with both reporters present the split left %d agents, want the two kept apart", got)
}
if err := store.UpsertResourceIdentityPins([]ResourceIdentityPin{{
CanonicalID: merged, ResourceType: ResourceTypeAgent, MachineID: machineID, Hostname: "shared-04",
}}); err != nil {
t.Fatalf("hold the machine key on %s: %v", merged, err)
}
if pins, err := store.ListResourceIdentityPins(); err != nil || len(pins) != 1 || pins[0].CanonicalID != merged {
t.Fatalf("pins after holding the key on %s = %+v (err %v), want that one", merged, pins, err)
}
if got := agents(models.StateSnapshot{DockerHosts: []models.DockerHost{dockerHost}}); got != 1 {
t.Fatalf("with the sibling absent %d agents were listed, want the Docker host alone", got)
}
if got := recorded(t, store); got[candidate] != "" {
t.Fatalf("the split source's ID was declared superseded by %s", got[candidate])
}
if got := agents(both); got != 2 {
t.Fatalf("the split did not survive its sibling's absence: %d agents, want the two kept apart", got)
}
})
// A rebuild that cannot read its decisions carries the previous
// generation's exclusions (carryOverridesFrom) instead of loading them,
// and the split is still the operator's: the guard reads the carried
// decisions, not only the ones a store read returned. The control is the
// same reporter on a store with no split, which does note its era.
t.Run(storeName+"/a rebuild that cannot read its decisions still guards the split", func(t *testing.T) {
dockerHost := models.DockerHost{ID: "docker-shared-06", Hostname: "shared-06", MachineID: "machine-shared-06", Status: "online", LastSeen: now}
merged := MachineIdentityCanonicalID(ResourceTypeAgent, dockerHost.MachineID)
candidate := SourceSpecificID(ResourceTypeAgent, SourceDocker, dockerHost.ID)
noted := func(rr *ResourceRegistry) bool {
t.Helper()
resource, identity := resourceFromDockerHost(dockerHost)
rr.ingest(SourceDocker, dockerHost.ID, resource, identity)
return slices.Contains(rr.reporterEras, CanonicalIDSuccession{OldCanonicalID: candidate, NewCanonicalID: merged})
}
if !noted(NewRegistry(newStore(t))) {
t.Fatalf("control: a reporter with no split did not note %s -> %s", candidate, merged)
}
flaky := &unreadableDecisionStore{ResourceStore: newStore(t)}
if err := flaky.AddExclusion(ResourceExclusion{ResourceA: merged, ResourceB: candidate}); err != nil {
t.Fatalf("split the Docker source from %s: %v", merged, err)
}
previous := NewRegistry(flaky)
flaky.failAll(true)
rebuilt := newRegistryFrom(previous)
if !rebuilt.overridesUnreadable || len(rebuilt.exclusions) != 1 {
t.Fatalf("the rebuild carried %d exclusions (unreadable=%v), want the previous generation's one", len(rebuilt.exclusions), rebuilt.overridesUnreadable)
}
if noted(rebuilt) {
t.Fatalf("a rebuild holding the carried split noted %s superseded", candidate)
}
})
// A pin on the source-specific ID means the ID was a keyed machine's.
// Pin succession decides whether another key is the same host, and a
// different machine mints fresh instead of taking the rows over.
t.Run(storeName+"/a pinned source-specific ID is another machine's", func(t *testing.T) {
store := newStore(t)
lock(t, store, hostEra)
if err := store.UpsertResourceIdentityPins([]ResourceIdentityPin{{
CanonicalID: hostEra, ResourceType: ResourceTypeAgent, MachineID: "machine-before-reinstall", Hostname: "standalone-03",
}}); err != nil {
t.Fatalf("pin %s: %v", hostEra, err)
}
rr := NewRegistry(store)
rr.IngestSnapshot(models.StateSnapshot{Hosts: []models.Host{host("machine-after-reinstall")}})
rr.PersistIdentityPins()
agents := rr.ListByType(ResourceTypeAgent)
if len(agents) != 1 || agents[0].ID == hostEra {
t.Fatalf("a different machine under the same report ID is listed as %+v, want a fresh ID", agents)
}
if locked(t, store, agents[0].ID) {
t.Fatalf("the different machine %s inherited the pinned machine's lock", agents[0].ID)
}
if !locked(t, store, hostEra) || recorded(t, store)[hostEra] != "" {
t.Fatalf("the pinned machine's row was succeeded onto another machine")
}
})
t.Run(storeName+"/a node borrowing its agent's key is not the node's era", func(t *testing.T) {
store := newStore(t)
nodeEra := SourceSpecificID(ResourceTypeAgent, SourceProxmox, proxmoxNodeSourceID("pve", "pve1"))
lock(t, store, nodeEra)
rr := NewRegistry(store)
rr.IngestSnapshot(sharedSerialDiskSnapshot(now, true, "pve1"))
rr.PersistIdentityPins()
if got := recorded(t, store); got[nodeEra] != "" {
t.Fatalf("the Proxmox node declared its source-specific ID %s superseded by %s", nodeEra, got[nodeEra])
}
if !locked(t, store, nodeEra) {
t.Fatalf("the node's row moved off %s onto the agent's ID", nodeEra)
}
})
}
}
// successionForwardingStore gives a store wrapper that embeds the ResourceStore
// interface the optional succession capability of the store beneath it, which
// the interface does not carry: without it the registry's assertion to
// canonicalIDSuccessor fails and no succession ever reaches the store.
type successionForwardingStore struct{ *unreadableDecisionStore }
func (s successionForwardingStore) ApplyCanonicalIDSuccessions(successions []CanonicalIDSuccession) error {
return s.ResourceStore.(canonicalIDSuccessor).ApplyCanonicalIDSuccessions(successions)
}
// An operator split of one reporter from another that reports the same machine
// survives the other reporter's absence whichever pin holds the shared machine
// key. The store lets one pin hold a strong key, so while both reporters are
// present the split source's pin (on its source-specific ID) or its sibling's
// ends up holding it. Pin succession reads the holder as the same host's
// earlier era, and in a generation where the sibling is absent the split source
// lands on the sibling's ID: declaring the one superseded by the other would
// rewrite the exclusion of the pair into one of the resource from itself, and
// the sibling's return would merge the two. An operator decision that the old
// and the new ID are different resources outranks the key that suggests they
// are one, and so does a batch that would retire both sides of a split onto one
// successor. A split of an ID that is not paired with the succession's
// successor still follows the succession.
func TestPinSuccessionKeepsASplitSourceApart(t *testing.T) {
stores := map[string]func(t *testing.T) ResourceStore{
"memory": func(t *testing.T) ResourceStore { return NewMemoryStore() },
"sqlite": func(t *testing.T) ResourceStore {
store, err := NewSQLiteResourceStore(t.TempDir(), "default")
if err != nil {
t.Fatalf("NewSQLiteResourceStore: %v", err)
}
t.Cleanup(func() { _ = store.Close() })
return store
},
}
recorded := func(t *testing.T, store ResourceStore) map[string]string {
t.Helper()
switch s := store.(type) {
case *SQLiteResourceStore:
return s.successionMap()
case *MemoryStore:
return s.canonicalSuccessions
}
t.Fatalf("%T records no successions", store)
return nil
}
now := time.Now().UTC()
machineID := "machine-shared-07"
agentHost := models.Host{ID: "agent-shared-07", Hostname: "shared-07", MachineID: machineID, Status: "online", LastSeen: now}
dockerHost := models.DockerHost{ID: "docker-shared-07", Hostname: "shared-07", MachineID: machineID, Status: "online", LastSeen: now}
merged := MachineIdentityCanonicalID(ResourceTypeAgent, machineID)
candidate := SourceSpecificID(ResourceTypeAgent, SourceDocker, dockerHost.ID)
both := models.StateSnapshot{Hosts: []models.Host{agentHost}, DockerHosts: []models.DockerHost{dockerHost}}
dockerOnly := models.StateSnapshot{DockerHosts: []models.DockerHost{dockerHost}}
for storeName, newStore := range stores {
for holderName, holder := range map[string]string{
"the split source's pin holds the machine key": candidate,
"the sibling's pin holds the machine key": merged,
} {
t.Run(storeName+"/"+holderName, func(t *testing.T) {
store := newStore(t)
if err := store.AddExclusion(ResourceExclusion{ResourceA: merged, ResourceB: candidate}); err != nil {
t.Fatalf("split the Docker source from %s: %v", merged, err)
}
agents := func(snapshot models.StateSnapshot) int {
t.Helper()
rr := NewRegistry(store)
rr.IngestSnapshot(snapshot)
rr.PersistIdentityPins()
return len(rr.ListByType(ResourceTypeAgent))
}
if got := agents(both); got != 2 {
t.Fatalf("with both reporters present the split left %d agents, want the two kept apart", got)
}
// The store keeps one pin per strong key and hands it to the last
// writer, so which pin holds it is the write order's outcome; the
// fixture decides it.
if err := store.UpsertResourceIdentityPins([]ResourceIdentityPin{{
CanonicalID: holder, ResourceType: ResourceTypeAgent, MachineID: machineID, Hostname: "shared-07",
}}); err != nil {
t.Fatalf("hold the machine key on %s: %v", holder, err)
}
pins, err := store.ListResourceIdentityPins()
if err != nil {
t.Fatalf("ListResourceIdentityPins: %v", err)
}
if i := slices.IndexFunc(pins, func(pin ResourceIdentityPin) bool { return pin.MachineID == machineID }); i < 0 || pins[i].CanonicalID != holder {
t.Fatalf("pins = %+v, want the machine key held by %s", pins, holder)
}
if got := agents(dockerOnly); got != 1 {
t.Fatalf("with the sibling absent %d agents were listed, want the Docker host alone", got)
}
if got := recorded(t, store); got[candidate] != "" || got[merged] != "" {
t.Fatalf("a split pair was declared a succession: %v", got)
}
exclusions, err := store.GetExclusions()
if err != nil {
t.Fatalf("GetExclusions: %v", err)
}
if len(exclusions) != 1 || exclusionKey(exclusions[0].ResourceA, exclusions[0].ResourceB) != exclusionKey(merged, candidate) {
t.Fatalf("exclusions = %+v, want the one split of %s from %s", exclusions, candidate, merged)
}
if got := agents(both); got != 2 {
t.Fatalf("the split did not survive its sibling's absence: %d agents, want the two kept apart", got)
}
})
}
// A rebuild that cannot read its decisions carries the previous
// generation's exclusions (carryOverridesFrom), and the check reads those.
// The control is the same outage with no split, which declares the
// succession through the same store, so the refusal is the check's.
t.Run(storeName+"/a rebuild that cannot read its decisions still refuses the pair", func(t *testing.T) {
outage := func(t *testing.T, split bool) *unreadableDecisionStore {
t.Helper()
flaky := &unreadableDecisionStore{ResourceStore: newStore(t)}
store := successionForwardingStore{flaky}
// Both runs carry a split, so only the pair check can tell them apart.
if err := store.AddExclusion(ResourceExclusion{ResourceA: "agent-unrelated-a", ResourceB: "agent-unrelated-b"}); err != nil {
t.Fatalf("split two unrelated resources: %v", err)
}
if split {
if err := store.AddExclusion(ResourceExclusion{ResourceA: merged, ResourceB: candidate}); err != nil {
t.Fatalf("split the Docker source from %s: %v", merged, err)
}
}
// The previous generation ingests nothing: a both-present ingest
// declares the candidate's reporter era itself, which would record
// the succession before the outage and hide whether the outage's
// pin succession does.
previous := NewRegistry(store)
if err := store.UpsertResourceIdentityPins([]ResourceIdentityPin{{
CanonicalID: candidate, ResourceType: ResourceTypeAgent, MachineID: machineID, Hostname: "shared-07",
}}); err != nil {
t.Fatalf("hold the machine key on %s: %v", candidate, err)
}
if got := recorded(t, flaky.ResourceStore); got[candidate] != "" {
t.Fatalf("%s was declared superseded before the outage: %v", candidate, got)
}
flaky.failAll(true)
rebuilt := newRegistryFrom(previous)
if wantCarried := map[bool]int{true: 2, false: 1}[split]; !rebuilt.overridesUnreadable || len(rebuilt.exclusions) != wantCarried {
t.Fatalf("the rebuild carried %d exclusions (unreadable=%v), want %d", len(rebuilt.exclusions), rebuilt.overridesUnreadable, wantCarried)
}
rebuilt.IngestSnapshot(dockerOnly)
rebuilt.PersistIdentityPins()
flaky.failAll(false)
return flaky
}
if got := recorded(t, outage(t, false).ResourceStore); got[candidate] != merged {
t.Fatalf("control: with no split the rebuild declared %v, want %s superseded by %s", got, candidate, merged)
}
flaky := outage(t, true)
if got := recorded(t, flaky.ResourceStore); got[candidate] != "" {
t.Fatalf("a rebuild holding the carried split declared %s superseded by %s", candidate, got[candidate])
}
exclusions, err := flaky.GetExclusions()
if err != nil {
t.Fatalf("GetExclusions: %v", err)
}
if !slices.ContainsFunc(exclusions, func(e ResourceExclusion) bool {
return exclusionKey(e.ResourceA, e.ResourceB) == exclusionKey(merged, candidate)
}) {
t.Fatalf("exclusions = %+v, want the split of %s from %s kept", exclusions, candidate, merged)
}
})
// One pin can succeed two predecessors, a machine-keyed pin through its
// key and a machine-keyless cluster-slot pin through its slot. Each
// passes the pair check against the successor, but the operator split
// the two from each other, and the successor cannot be both. The control
// is the same pins with no split, which retires both.
t.Run(storeName+"/two split IDs are not retired onto one successor", func(t *testing.T) {
keyedID, slotID := "agent-fanin-keyed", "agent-fanin-slot"
run := func(t *testing.T, exclusions ...ResourceExclusion) (ResourceStore, string) {
t.Helper()
store := newStore(t)
if err := store.UpsertResourceIdentityPins([]ResourceIdentityPin{
{CanonicalID: keyedID, ResourceType: ResourceTypeAgent, MachineID: "machine-fanin", Hostname: "fanin-keyed"},
{CanonicalID: slotID, ResourceType: ResourceTypeAgent, ClusterName: "fanin-swarm", Hostname: "fanin-host"},
}); err != nil {
t.Fatalf("seed identity pins: %v", err)
}
for _, exclusion := range exclusions {
if err := store.AddExclusion(exclusion); err != nil {
t.Fatalf("exclude %s from %s: %v", exclusion.ResourceA, exclusion.ResourceB, err)
}
}
rr := NewRegistry(store)
successorID := rr.ingest(SourceDocker, "docker:fanin-host", Resource{
Type: ResourceTypeAgent, Name: "fanin-host", Status: StatusOnline,
Docker: &DockerData{HostSourceID: "docker:fanin-host", Hostname: "fanin-host"},
}, ResourceIdentity{MachineID: "machine-fanin", ClusterName: "fanin-swarm", Hostnames: []string{"fanin-host", "fanin-swarm:fanin-host"}})
if successorID == "" || successorID == keyedID || successorID == slotID {
t.Fatalf("the host was listed as %q, want a successor distinct from %s and %s", successorID, keyedID, slotID)
}
rr.PersistIdentityPins()
return store, successorID
}
store, successorID := run(t)
if got := recorded(t, store); got[keyedID] != successorID || got[slotID] != successorID {
t.Fatalf("control: with no split the pins declared %v, want %s and %s retired onto %s", got, keyedID, slotID, successorID)
}
// An exclusion of an ID from itself is what the defect left behind in
// a store it already reached, and it separates nothing from the other
// predecessor.
store, successorID = run(t, ResourceExclusion{ResourceA: keyedID, ResourceB: keyedID})
if got := recorded(t, store); got[keyedID] != successorID || got[slotID] != successorID {
t.Fatalf("a self-pair exclusion of %s stopped the pins declaring %v, want %s and %s retired onto %s", keyedID, got, keyedID, slotID, successorID)
}
store, successorID = run(t, ResourceExclusion{ResourceA: keyedID, ResourceB: slotID})
if got := recorded(t, store); got[keyedID] != "" || got[slotID] != "" {
t.Fatalf("two split IDs were retired onto %s: %v", successorID, got)
}
exclusions, err := store.GetExclusions()
if err != nil {
t.Fatalf("GetExclusions: %v", err)
}
if len(exclusions) != 1 || exclusionKey(exclusions[0].ResourceA, exclusions[0].ResourceB) != exclusionKey(keyedID, slotID) {
t.Fatalf("exclusions = %+v, want the one split of %s from %s", exclusions, keyedID, slotID)
}
})
// The control for the rule's reach: an operator split names an ID whose
// host later gains a stronger key, and the succession carries the split
// with the host, since the ID it supersedes is not paired with its
// successor.
t.Run(storeName+"/a split of a superseded ID that is not paired with its successor follows it", func(t *testing.T) {
store := newStore(t)
shortEraID := buildHashID(ResourceTypeAgent, "cluster:prod-swarm:cloud")
if err := store.UpsertResourceIdentityPins([]ResourceIdentityPin{{
CanonicalID: shortEraID, ResourceType: ResourceTypeAgent, ClusterName: "prod-swarm", Hostname: "cloud",
}}); err != nil {
t.Fatalf("seed identity pin: %v", err)
}
if err := store.AddExclusion(ResourceExclusion{ResourceA: shortEraID, ResourceB: "other-resource"}); err != nil {
t.Fatalf("split %s from another resource: %v", shortEraID, err)
}
rr := NewRegistry(store)
newID := rr.ingest(SourceDocker, "docker:cloud.a", Resource{
Type: ResourceTypeAgent, Name: "cloud.a", Status: StatusOnline,
Docker: &DockerData{HostSourceID: "docker:cloud.a", Hostname: "cloud.a"},
}, ResourceIdentity{ClusterName: "prod-swarm", Hostnames: []string{"cloud.a", "prod-swarm:cloud.a"}})
if newID == "" || newID == shortEraID {
t.Fatalf("expected a full-hostname canonical ID, got %q (short era %q)", newID, shortEraID)
}
rr.PersistIdentityPins()
if got := recorded(t, store); got[shortEraID] != newID {
t.Fatalf("the short era was not succeeded by %s: %v", newID, got)
}
exclusions, err := store.GetExclusions()
if err != nil {
t.Fatalf("GetExclusions: %v", err)
}
if len(exclusions) != 1 || exclusionKey(exclusions[0].ResourceA, exclusions[0].ResourceB) != exclusionKey(newID, "other-resource") {
t.Fatalf("exclusions = %+v, want the split re-keyed onto %s", exclusions, newID)
}
})
}
// The fan-in filter weighs each predecessor's first declaration (the stores
// re-key a predecessor once), and only the successions that retire two split
// IDs onto one successor are dropped.
t.Run("the fan-in filter drops only successions that retire two split IDs onto one", func(t *testing.T) {
succession := func(oldID, newID string) CanonicalIDSuccession {
return CanonicalIDSuccession{OldCanonicalID: oldID, NewCanonicalID: newID}
}
rr := NewRegistry(nil)
rr.exclusions[exclusionKey("fan-a", "fan-b")] = now
rr.exclusions[exclusionKey("fan-p", "fan-q")] = now
for name, tc := range map[string]struct{ batch, want []CanonicalIDSuccession }{
"two split IDs onto one successor": {
[]CanonicalIDSuccession{succession("fan-a", "fan-w"), succession("fan-b", "fan-w")}, nil,
},
"an unrelated third predecessor still moves": {
[]CanonicalIDSuccession{succession("fan-a", "fan-w"), succession("fan-b", "fan-w"), succession("fan-c", "fan-w")},
[]CanonicalIDSuccession{succession("fan-c", "fan-w")},
},
"a split predecessor with an ID outside its successor's group": {
[]CanonicalIDSuccession{succession("fan-a", "fan-w"), succession("fan-c", "fan-w"), succession("fan-b", "fan-x")},
[]CanonicalIDSuccession{succession("fan-a", "fan-w"), succession("fan-c", "fan-w"), succession("fan-b", "fan-x")},
},
"an unrelated split does not stop a star": {
[]CanonicalIDSuccession{succession("fan-c", "fan-w"), succession("fan-d", "fan-w")},
[]CanonicalIDSuccession{succession("fan-c", "fan-w"), succession("fan-d", "fan-w")},
},
"a later declaration for a predecessor retires nothing": {
[]CanonicalIDSuccession{succession("fan-a", "fan-x"), succession("fan-b", "fan-y"), succession("fan-a", "fan-y")},
[]CanonicalIDSuccession{succession("fan-a", "fan-x"), succession("fan-b", "fan-y")},
},
} {
if got := rr.withoutSplitFanInLocked(tc.batch); !slices.Equal(got, tc.want) {
t.Errorf("%s: kept %v, want %v", name, got, tc.want)
}
}
})
}
// A physical disk keyed by a serial that two machines report is re-keyed in
// place to a machine-scoped ID (rekeyPhysicalDiskLocked), and that moves no
// operator-state row: the registry keeps no durable record of a disk's IDs, so
// the stores cannot be told which row to carry. The remediation lock is not
// carried across that re-key because it guards nothing there: the planner
// plans only a capability the resource advertises, and no producer gives a
// physical disk one, so no dispatch can reach a disk's ID. This pins that
// premise for the producers this package builds disks from (a Proxmox disk,
// an agent's SMART disk, an Unraid array disk) and for the registry's merge of
// a record-fed disk; the TrueNAS provider's own records are pinned beside the
// producer (internal/truenas TestProviderDiskRecordsAdvertiseNoActionCapability).
// A change that gives a disk a capability must first make the lock follow the
// disk through the re-key (see the remediation-lock paragraph of the
// unified-resources contract), or it lets Pulse act on a disk the operator
// locked.
func TestPhysicalDiskResourcesAdvertiseNoActionCapability(t *testing.T) {
now := time.Date(2026, 10, 7, 12, 0, 0, 0, time.UTC)
rr := NewRegistry(nil)
snapshot := sharedSerialDiskSnapshot(now, true, "pve1", "pve2")
snapshot.Hosts = append(snapshot.Hosts, models.Host{
ID: "agent-tower", Hostname: "tower", MachineID: "machine-tower", Status: "online", LastSeen: now,
Unraid: &models.HostUnraidStorage{Disks: []models.HostUnraidDisk{
{Name: "disk1", Device: "sdc", Role: "data", Status: "online", Serial: "UNRAID-LOCK1", Temperature: 37},
}},
})
rr.IngestSnapshot(snapshot)
rr.IngestRecords(SourceTrueNAS, trueNASDiskRecords(now, "a", "archive-a", "tank", "TRUENAS-LOCK1"))
disks := rr.ListByType(ResourceTypePhysicalDisk)
for _, disk := range disks {
if len(disk.Capabilities) != 0 {
t.Errorf("physical disk %s (%v) advertises capabilities %+v; make the remediation lock follow a disk through its in-place re-key first", disk.ID, disk.Sources, disk.Capabilities)
}
}
// Each producer's disk is in the registry under its serial, so a fixture
// that stops producing one cannot leave the check above passing for it.
for _, want := range []struct {
serial string
source DataSource
}{
{sharedDiskSerial, SourceProxmox},
{sharedDiskSerial, SourceAgent},
{"UNIQUE-SERIAL-1", SourceProxmox},
{"UNRAID-LOCK1", SourceAgent},
{"TRUENAS-LOCK1", SourceTrueNAS},
} {
if !slices.ContainsFunc(disks, func(disk Resource) bool {
return disk.PhysicalDisk != nil && disk.PhysicalDisk.Serial == want.serial && containsDataSource(disk.Sources, want.source)
}) {
t.Errorf("no physical disk with serial %s came from %s, so the fixture no longer covers that producer: %d disks", want.serial, want.source, len(disks))
}
}
}
// A physical disk's operator-state row is keyed by the canonical ID the disk
// holds, and that ID changes: a disk keyed by a serial one machine reports takes
// the unscoped ID, a second machine reporting the serial re-keys every copy to a
// machine-scoped ID, and the unscoped ID returns on the rebuild after the other
// copy goes away. No row moves with it, by decision. The registry cannot tell
// which copy a row was set for (one disk seen from two machines, or two disks
// behind one fixed serial), and a retirement or mute carried onto a disk the
// operator never silenced can hide that disk's failure alerts, where a row left
// behind only makes the disk eligible to alert again at the split. The quiet
// outcome is when the other machine goes away: the original row applies again,
// over anything the operator wrote on the scoped ID since, which the contract
// accepts.
//
// This drives the monitor adapter's rebuild on both stores and reads the way
// alert intent does: ResolveCanonicalResourceID on the alias a PVE disk alert
// carries, then GetResourceOperatorState on the result. It does not run the API
// write path, Patrol's provider or the other readers, so a carry added there
// would not fail here. Rows are compared as JSON, so a carry of any one field
// (a note, a window, SetBy) fails as well as a carry of the row. A change that
// carries disk rows has to reconsider the trade in the disk paragraph of the
// unified-resources contract, not discover it here.
func TestPhysicalDiskOperatorStateIsNotCarriedAcrossASerialSplit(t *testing.T) {
stores := map[string]func(t *testing.T) ResourceStore{
"memory": func(t *testing.T) ResourceStore { return NewMemoryStore() },
"sqlite": func(t *testing.T) ResourceStore {
store, err := NewSQLiteResourceStore(t.TempDir(), "default")
if err != nil {
t.Fatalf("NewSQLiteResourceStore: %v", err)
}
t.Cleanup(func() { _ = store.Close() })
return store
},
}
now := time.Date(2026, 10, 10, 12, 0, 0, 0, time.UTC)
// A window is active against the wall clock, not the fixture's, so a carry
// that acts only on an active window cannot go inert as the date passes.
windowStart := time.Now().UTC().Add(-time.Hour).Truncate(time.Second)
windowEnd := windowStart.Add(48 * time.Hour)
alertRef := ProxmoxPhysicalDiskAlertResourceID("pve", "pve2", "/dev/sda")
// canon renders every field the row persists, normalized as the stores
// normalize it, so two rows compare equal only when nothing differs.
canon := func(state ResourceOperatorState) string {
state = NormalizeResourceOperatorState(state)
state.SetAt = state.SetAt.UTC()
for _, at := range []**time.Time{&state.MaintenanceStartAt, &state.MaintenanceEndAt} {
if *at != nil {
utc := (*at).UTC()
*at = &utc
}
}
encoded, err := json.Marshal(state)
if err != nil {
t.Fatalf("encode operator state: %v", err)
}
return string(encoded)
}
for name, newStore := range stores {
t.Run(name, func(t *testing.T) {
store := newStore(t)
// One adapter, rebuilt by every step as the monitor rebuilds it.
adapter := NewMonitorAdapter(NewRegistry(store))
ingest := func(at time.Time, nodes ...string) map[string]Resource {
t.Helper()
adapter.PopulateFromSnapshot(sharedSerialDiskSnapshot(at, false, nodes...))
return sharedSerialDisksByNode(t, adapter.currentRegistry())
}
row := func(id string) (ResourceOperatorState, bool) {
t.Helper()
state, found, err := store.GetResourceOperatorState(id)
if err != nil {
t.Fatalf("read operator state of %s: %v", id, err)
}
return state, found
}
// What alert intent reads for pve2's disk: the alert reference
// resolved to a canonical ID, then that ID's row.
seen := func() (string, ResourceOperatorState, bool) {
t.Helper()
id, ok := adapter.ResolveCanonicalResourceID(alertRef)
if !ok {
t.Fatalf("alert reference %s does not resolve", alertRef)
}
state, found, err := adapter.GetResourceOperatorState(id)
if err != nil {
t.Fatalf("read operator state of %s: %v", id, err)
}
return id, state, found
}
isRow := func(label string, got ResourceOperatorState, found bool, want ResourceOperatorState) {
t.Helper()
if !found {
t.Errorf("%s: no row, want %s", label, canon(want))
return
}
if canon(got) != canon(want) {
t.Errorf("%s: row is %s, want %s", label, canon(got), canon(want))
}
}
unscoped := ingest(now, "pve2")["pve2"].ID
retired := ResourceOperatorState{
CanonicalID: unscoped, LifecycleState: LifecycleStateRetired, MonitoringMode: MonitoringModeMuted,
NeverAutoRemediate: true, Criticality: CriticalityHigh, Note: "retired after the RMA",
AutoRemediationPolicy: AutoRemediationPolicy{
Enabled: true, CapabilityNames: []string{"restart"},
Window: &AutoRemediationWindow{Timezone: "UTC", StartMinute: 60, EndMinute: 120},
},
MaintenanceStartAt: &windowStart, MaintenanceEndAt: &windowEnd, MaintenanceReason: "shelf swap",
SetAt: now, SetBy: "operator",
}
if err := store.SetResourceOperatorState(retired); err != nil {
t.Fatalf("retire the disk: %v", err)
}
id, state, found := seen()
if id != unscoped {
t.Fatalf("one machine: the alert reference reads %s, want %s", id, unscoped)
}
isRow("one machine: the alert reference's row", state, found, retired)
copies := ingest(now.Add(time.Minute), "pve2", "pve3")
if len(copies) != 2 || copies["pve2"].ID == unscoped || copies["pve3"].ID == unscoped || copies["pve2"].ID == copies["pve3"].ID {
t.Fatalf("second machine: copies %s and %s, want two machine-scoped IDs apart from %s", copies["pve2"].ID, copies["pve3"].ID, unscoped)
}
for node, disk := range copies {
if _, found := row(disk.ID); found {
t.Errorf("second machine: the %s copy %s holds a row set on the unscoped ID", node, disk.ID)
}
}
if id, _, found := seen(); id != copies["pve2"].ID || found {
t.Errorf("second machine: the alert reference reads %s (row %v), want %s with no row", id, found, copies["pve2"].ID)
}
state, found = row(unscoped)
isRow("second machine: the unscoped ID's row", state, found, retired)
// The operator writes a row on pve2's copy: a note and a recurring
// window, which also puts the copy back to active monitoring.
scoped := copies["pve2"].ID
restored := ResourceOperatorState{
CanonicalID: scoped, Note: "pve2 copy restored",
MaintenanceRecurrence: &RecurringMaintenanceWindow{Timezone: "UTC", Weekdays: []string{"monday"}, StartMinute: 60, EndMinute: 120},
SetAt: now.Add(2 * time.Minute), SetBy: "second-operator",
}
if err := store.SetResourceOperatorState(restored); err != nil {
t.Fatalf("write the pve2 copy's row: %v", err)
}
reordered := ingest(now.Add(3*time.Minute), "pve3", "pve2")
if reordered["pve2"].ID != scoped || reordered["pve3"].ID != copies["pve3"].ID {
t.Fatalf("ingest order moved the copies: pve2 %s -> %s, pve3 %s -> %s", scoped, reordered["pve2"].ID, copies["pve3"].ID, reordered["pve3"].ID)
}
if _, found := row(copies["pve3"].ID); found {
t.Errorf("the pve3 copy %s holds a row though none was set on it", copies["pve3"].ID)
}
state, found = row(scoped)
isRow("after the ingest order changed: the pve2 copy's row", state, found, restored)
if disk := ingest(now.Add(4*time.Minute), "pve2")["pve2"]; disk.ID != unscoped {
t.Fatalf("other machine gone: the disk holds %s, want the unscoped %s back", disk.ID, unscoped)
}
id, state, found = seen()
if id != unscoped {
t.Fatalf("other machine gone: the alert reference reads %s, want %s", id, unscoped)
}
isRow("other machine gone: the alert reference's row (the original, over the later write)", state, found, retired)
state, found = row(scoped)
isRow("other machine gone: the row written on the scoped ID", state, found, restored)
if _, found := row(copies["pve3"].ID); found {
t.Errorf("other machine gone: the pve3 copy's ID %s holds a row though none was set on it", copies["pve3"].ID)
}
// A row already on a scoped ID applies again whenever a copy with the
// same serial and machine scope is scoped again.
again := ingest(now.Add(5*time.Minute), "pve2", "pve3")
if again["pve2"].ID != scoped || again["pve3"].ID != copies["pve3"].ID {
t.Fatalf("second split: copies %s and %s, want the earlier %s and %s", again["pve2"].ID, again["pve3"].ID, scoped, copies["pve3"].ID)
}
id, state, found = seen()
if id != scoped {
t.Fatalf("second split: the alert reference reads %s, want %s", id, scoped)
}
isRow("second split: the alert reference's row", state, found, restored)
state, found = row(unscoped)
isRow("second split: the unscoped ID's row", state, found, retired)
if _, found := row(copies["pve3"].ID); found {
t.Errorf("second split: the pve3 copy %s holds a row though none was set on it", copies["pve3"].ID)
}
})
}
}
// A succession batch is applied in the order it is declared, and each group of
// linked IDs it saved follows only the re-keys applied after it. A batch
// declared against its direction, and one that returns an ID to its start, both
// leave the lock on the IDs that survive and give no row to an ID re-keyed away.
func TestSuccessionBatchOrderKeepsLinkedRemediationLock(t *testing.T) {
stores := map[string]func(t *testing.T) ResourceStore{
"memory": func(t *testing.T) ResourceStore { return NewMemoryStore() },
"sqlite": func(t *testing.T) ResourceStore {
store, err := NewSQLiteResourceStore(t.TempDir(), "default")
if err != nil {
t.Fatalf("NewSQLiteResourceStore: %v", err)
}
t.Cleanup(func() { _ = store.Close() })
return store
},
}
now := time.Now().UTC()
locked := func(t *testing.T, store ResourceStore, id string) bool {
t.Helper()
state, found, err := store.GetResourceOperatorState(id)
if err != nil {
t.Fatalf("read operator state of %s: %v", id, err)
}
return found && state.BlocksRemediation()
}
for name, newStore := range stores {
t.Run(name+"/declared against its direction", func(t *testing.T) {
store := newStore(t)
if err := store.AddLink(ResourceLink{ResourceA: "rev-a", ResourceB: "rev-b", PrimaryID: "rev-b"}); err != nil {
t.Fatalf("link: %v", err)
}
if err := store.SetResourceOperatorState(ResourceOperatorState{CanonicalID: "rev-a", NeverAutoRemediate: true, SetAt: now}); err != nil {
t.Fatalf("lock: %v", err)
}
if err := store.(canonicalIDSuccessor).ApplyCanonicalIDSuccessions([]CanonicalIDSuccession{
{OldCanonicalID: "rev-b", NewCanonicalID: "rev-c"},
{OldCanonicalID: "rev-a", NewCanonicalID: "rev-b"},
}); err != nil {
t.Fatalf("ApplyCanonicalIDSuccessions: %v", err)
}
for _, id := range []string{"rev-b", "rev-c"} {
if !locked(t, store, id) {
t.Fatalf("%s lost the lock when the batch was declared against its direction", id)
}
}
})
t.Run(name+"/returns an ID to its start", func(t *testing.T) {
store := newStore(t)
if err := store.AddLink(ResourceLink{ResourceA: "loop-a", ResourceB: "loop-x", PrimaryID: "loop-x"}); err != nil {
t.Fatalf("link: %v", err)
}
if err := store.SetResourceOperatorState(ResourceOperatorState{CanonicalID: "loop-a", NeverAutoRemediate: true, SetAt: now}); err != nil {
t.Fatalf("lock: %v", err)
}
if err := store.(canonicalIDSuccessor).ApplyCanonicalIDSuccessions([]CanonicalIDSuccession{
{OldCanonicalID: "loop-a", NewCanonicalID: "loop-b"},
{OldCanonicalID: "loop-b", NewCanonicalID: "loop-a"},
}); err != nil {
t.Fatalf("ApplyCanonicalIDSuccessions: %v", err)
}
for _, id := range []string{"loop-a", "loop-x"} {
if !locked(t, store, id) {
t.Fatalf("%s lost the lock across a batch that returned to its start", id)
}
}
if state, found := operatorStateRow(t, store, "loop-b"); found {
t.Fatalf("the re-keyed intermediate ID was given a row: state=%+v", state)
}
})
}
}
// chainLinkEstate arrives the way the monitor ingests it: two agents, a
// Proxmox VM and its storage in the snapshot, then two TrueNAS pools and a
// TrueNAS VM, then a vSphere VM and a vSphere datastore, each record source
// in its own pass.
type chainLinkEstate struct {
snapshot models.StateSnapshot
records map[DataSource][]IngestRecord
// saved is an unlinked host that only saved-host continuity lists.
saved models.Host
ids map[string]string
}
func newChainLinkEstate(t *testing.T) chainLinkEstate {
t.Helper()
now := time.Now().UTC()
pool := func(name string) IngestRecord {
return IngestRecord{
SourceID: "system:tn-1:pool:" + name,
Resource: Resource{
Type: ResourceTypeStorage, Name: name, Status: StatusOnline, LastSeen: now,
Storage: &StorageMeta{Type: "zfs-pool", Platform: "truenas", Topology: "pool"},
},
}
}
estate := chainLinkEstate{
snapshot: models.StateSnapshot{
LastUpdate: now,
VMs: []models.VM{
{ID: "lab:pve1:101", VMID: 101, Name: "web", Node: "pve1", Instance: "lab", Status: "running", Type: "qemu", LastSeen: now},
{ID: "lab:pve1:111", VMID: 111, Name: "app-r", Node: "pve1", Instance: "lab", Status: "running", Type: "qemu", LastSeen: now},
{ID: "lab:pve1:112", VMID: 112, Name: "app-u", Node: "pve1", Instance: "lab", Status: "running", Type: "qemu", LastSeen: now},
{ID: "lab:pve1:113", VMID: 113, Name: "app-x", Node: "pve1", Instance: "lab", Status: "running", Type: "qemu", LastSeen: now},
{ID: "lab:pve1:114", VMID: 114, Name: "app-y", Node: "pve1", Instance: "lab", Status: "running", Type: "qemu", LastSeen: now},
},
Storage: []models.Storage{{
ID: "lab-pve1-local-zfs", Name: "local-zfs", Node: "pve1", Instance: "lab", Type: "zfspool", Status: "available",
Total: 1000, Used: 400, Free: 600, Usage: 40, Enabled: true, Active: true, LastSeen: now,
}},
Hosts: []models.Host{
{ID: "host-app", Hostname: "app-agent", MachineID: "0123456789abcdef", Status: "online", LastSeen: now},
{ID: "host-ops", Hostname: "ops-agent", MachineID: "fedcba9876543210", Status: "online", LastSeen: now},
},
},
records: map[DataSource][]IngestRecord{
SourceTrueNAS: {
pool("tank"), pool("vault"),
{
SourceID: "system:tn-1:vm:nas-vm",
Resource: Resource{Type: ResourceTypeVM, Name: "nas-vm", Status: StatusOnline, LastSeen: now},
},
},
SourceVMware: {
{
SourceID: "vc-1:vm:vm-42",
Resource: Resource{
Type: ResourceTypeVM, Technology: "vmware", Name: "app-guest", Status: StatusOnline, LastSeen: now,
VMware: &VMwareData{ConnectionID: "vc-1", ManagedObjectID: "vm-42", EntityType: "vm"},
},
Identity: ResourceIdentity{Hostnames: []string{"app-guest"}},
},
{
SourceID: "vc-1:datastore:datastore-7",
Resource: Resource{
Type: ResourceTypeStorage, Technology: "vmware", Name: "ds-7", Status: StatusOnline, LastSeen: now,
VMware: &VMwareData{ConnectionID: "vc-1", ManagedObjectID: "datastore-7", EntityType: "datastore"},
},
},
},
},
saved: models.Host{ID: "host-saved", Hostname: "saved-agent", MachineID: "00112233445566aa", Status: "offline", LastSeen: now.Add(-time.Hour)},
ids: map[string]string{},
}
unlinked := NewMonitorAdapter(NewRegistry(nil))
unlinked.PopulateSnapshotAndSupplemental(estate.snapshot, estate.records)
roles := map[string]string{
"app-agent": "agent", "ops-agent": "ops-agent", "web": "pve-vm", "local-zfs": "pve-storage",
"tank": "tank", "vault": "vault", "nas-vm": "nas-vm", "app-guest": "vsphere-vm", "ds-7": "datastore",
"app-r": "vm-r", "app-u": "vm-u", "app-x": "vm-x", "app-y": "vm-y",
}
for _, resource := range unlinked.GetAll() {
if role, ok := roles[resource.Name]; ok {
estate.ids[role] = resource.ID
}
}
if len(estate.ids) != len(roles) || len(unlinked.GetAll()) != len(roles) {
t.Fatalf("unlinked estate = %+v, want one row per role %v", unlinked.GetAll(), roles)
}
return estate
}
// linkOrders lists every order of n links.
func linkOrders(n int) [][]int {
if n == 0 {
return [][]int{{}}
}
var orders [][]int
for _, rest := range linkOrders(n - 1) {
for i := 0; i <= len(rest); i++ {
order := append(append(append([]int{}, rest[:i]...), n-1), rest[i:]...)
orders = append(orders, order)
}
}
return orders
}
// Links that share a member name one identity, so a chain folds into one row
// chosen by precedence, never by the order the store lists the links in or
// the order its members' sources arrive. Each estate is rebuilt in every link
// order and checked after each of two rebuilds (identity pins persist on the
// first), after a live refresh of every record source, in the read state that overlays a saved host, and in
// a registry seeded from that read state, as the resources API seeds when the
// listing already carries every source: each view lists the same row,
// resolves every member to it and records every link's pair, with the same
// fold records in every order, refreshes included. Applied one link at a
// time, a link naming a member an earlier link had folded away found nothing,
// so one order folded the chain and another left a member standing.
func TestManualLinkChainsFoldTheSameWayInAnyLinkOrder(t *testing.T) {
estate := newChainLinkEstate(t)
created := time.Date(2026, 10, 1, 9, 0, 0, 0, time.UTC)
link := func(a, b, primary string, minute int) ResourceLink {
return ResourceLink{
ResourceA: estate.ids[a], ResourceB: estate.ids[b], PrimaryID: estate.ids[primary],
CreatedAt: created.Add(time.Duration(minute) * time.Minute),
}
}
// The cycle case below needs its two tie-broken VMs in canonical-ID order.
vmB, vmC := "vm-x", "vm-y"
if estate.ids[vmC] > estate.ids[vmB] {
vmB, vmC = vmC, vmB
}
for _, tc := range []struct {
name string
links []ResourceLink
root string
rootType ResourceType
}{
{
// The agent linked from its own page still folds into its guest, and
// the guest's own link takes both into the storage it names, all in
// the snapshot pass.
name: "agent primary over its guest, guest into storage",
links: []ResourceLink{link("agent", "pve-vm", "agent", 0), link("pve-vm", "pve-storage", "pve-storage", 1)},
root: "pve-storage",
rootType: ResourceTypeStorage,
},
{
// The same chain with record-sourced storage: the rebuild judges
// links once every source is in.
name: "agent primary over its guest, guest into record-sourced storage",
links: []ResourceLink{link("agent", "pve-vm", "agent", 0), link("pve-vm", "tank", "tank", 1)},
root: "tank",
rootType: ResourceTypeStorage,
},
{
// Both links name a primary over the agent, and the guest outranks
// the storage, though the pools' source sorts before the VM's.
name: "guest outranks storage as competing primary",
links: []ResourceLink{link("agent", "tank", "tank", 0), link("agent", "vsphere-vm", "vsphere-vm", 1)},
root: "vsphere-vm",
rootType: ResourceTypeVM,
},
{
name: "competing storage primaries keep the earliest link's",
links: []ResourceLink{link("agent", "tank", "tank", 0), link("pve-vm", "vault", "vault", 1), link("agent", "pve-vm", "agent", 2)},
root: "tank",
rootType: ResourceTypeStorage,
},
{
name: "competing storage primaries follow creation time, not names",
links: []ResourceLink{link("agent", "tank", "tank", 2), link("pve-vm", "vault", "vault", 1), link("agent", "pve-vm", "agent", 0)},
root: "vault",
rootType: ResourceTypeStorage,
},
{
// Every candidate is an agent, so the guest takes its place.
name: "an agent never holds a guest",
links: []ResourceLink{link("pve-vm", "tank", "tank", 0), link("tank", "ops-agent", "ops-agent", 1)},
root: "pve-vm",
rootType: ResourceTypeVM,
},
{
name: "a cycle folds into its best member",
links: []ResourceLink{link("agent", "tank", "tank", 0), link("tank", "vault", "vault", 1), link("vault", "agent", "agent", 2)},
root: "tank",
rootType: ResourceTypeStorage,
},
{
// A cycle across the snapshot and two record sources. Folding as the
// sources arrived put the TrueNAS VM inside the Proxmox storage
// before the vSphere VM's link closed the cycle, so the storage or
// the vSphere VM survived; the rebuild judges the whole cycle and
// keeps the guest whose link is oldest.
name: "a cycle across record sources keeps its best guest",
links: []ResourceLink{
link("nas-vm", "pve-storage", "pve-storage", 1),
link("pve-storage", "vsphere-vm", "vsphere-vm", 2),
link("vsphere-vm", "nas-vm", "nas-vm", 0),
},
root: "nas-vm",
rootType: ResourceTypeVM,
},
{
// The pool folds in through one VM's link while a second VM's link
// to it closes a cycle. A TrueNAS refresh recreates the pool's row
// (its source maps to a VM); it folds back through the link that
// took it in, not the cycle's closing link, so no record turns.
name: "a recreated cycle member folds back through its own link",
links: []ResourceLink{
link(vmB, "vm-r", "vm-r", 0),
link("vm-u", "vm-r", "vm-r", 1),
link(vmC, "vm-u", "vm-u", 2),
link(vmC, "tank", "tank", 3),
link("tank", vmB, vmB, 4),
},
root: "vm-r",
rootType: ResourceTypeVM,
},
{
// Two interior members of the chain are recreated by one refresh.
// Each folds back along its record; reaching the second through
// the VM the first holds would turn that VM's record round and
// leave its ID without a row.
name: "two recreated interior members fold back along their records",
links: []ResourceLink{
link("vm-u", "vault", "vault", 0),
link("vault", "tank", "tank", 1),
link("tank", "vm-r", "vm-r", 2),
},
root: "vm-r",
rootType: ResourceTypeVM,
},
{
name: "two recreated interior members with a closing link",
links: []ResourceLink{
link("vm-u", "vault", "vault", 0),
link("vault", "tank", "tank", 1),
link("tank", "vm-r", "vm-r", 2),
link("vm-u", "vm-r", "vm-r", 3),
},
root: "vm-r",
rootType: ResourceTypeVM,
},
{
// A TrueNAS refresh recreates the VM's row, since its source maps to
// a row of another type; the row folds back through the same pair.
name: "a member a refresh recreates folds back into its root",
links: []ResourceLink{link("nas-vm", "pve-storage", "pve-storage", 0)},
root: "pve-storage",
rootType: ResourceTypeStorage,
},
} {
t.Run(tc.name, func(t *testing.T) {
rootID := estate.ids[tc.root]
var reference []ManualLinkFold
check := func(order []int, view string, rr *ResourceRegistry) {
t.Helper()
root, ok := rr.Get(rootID)
if !ok || root.Type != tc.rootType {
t.Fatalf("order %v, %s: root %s = %+v (listed=%v), want a listed %s", order, view, rootID, root, ok, tc.rootType)
}
pairs := make(map[string]bool, len(tc.links))
for _, l := range tc.links {
pairs[exclusionKey(l.ResourceA, l.ResourceB)] = true
for _, member := range []string{l.ResourceA, l.ResourceB} {
if _, listed := rr.Get(member); listed && member != rootID {
t.Fatalf("order %v, %s: member %s still listed beside %s", order, view, member, rootID)
}
if got, ok := rr.ResolveReferenceID(member); !ok || got != rootID {
t.Fatalf("order %v, %s: %s resolved to %q (%v), want %s", order, view, member, got, ok, rootID)
}
}
}
folds := rr.ManualLinkFolds(rootID)
slices.SortFunc(folds, func(a, b ManualLinkFold) int {
return strings.Compare(a.HolderID+"|"+a.FoldedID, b.HolderID+"|"+b.FoldedID)
})
for _, fold := range folds {
delete(pairs, exclusionKey(fold.HolderID, fold.FoldedID))
}
if len(folds) != len(tc.links) || len(pairs) != 0 {
t.Fatalf("order %v, %s: folds %+v, want one per link pair %+v", order, view, folds, tc.links)
}
if reference == nil {
reference = folds
} else if !reflect.DeepEqual(folds, reference) {
t.Fatalf("order %v, %s: folds %+v, want %+v as in the first order", order, view, folds, reference)
}
}
for _, order := range linkOrders(len(tc.links)) {
store := NewMemoryStore()
for _, i := range order {
if err := store.AddLink(tc.links[i]); err != nil {
t.Fatalf("add link: %v", err)
}
}
adapter := NewMonitorAdapter(NewRegistry(store))
adapter.PopulateSnapshotAndSupplemental(estate.snapshot, estate.records)
check(order, "first rebuild", adapter.currentRegistry())
adapter.PopulateSnapshotAndSupplemental(estate.snapshot, estate.records)
check(order, "second rebuild", adapter.currentRegistry())
readState := ReadStateWithHostContinuity(adapter, []IngestRecord{HostIngestRecord(estate.saved)}).(*MonitorAdapter)
check(order, "read state", readState.registry)
seeded := NewRegistry(store)
seeded.IngestResources(readState.GetAll())
check(order, "resources API", seeded)
for _, source := range []DataSource{SourceTrueNAS, SourceVMware} {
adapter.PopulateSupplementalRecords(source, estate.records[source])
check(order, "refresh of "+string(source), adapter.currentRegistry())
}
}
})
}
}
// A live refresh can bring a member the last rebuild had not seen, and it
// cannot undo the folds the rebuild made. The TrueNAS VM is linked into the
// Proxmox storage, and the storage into a vSphere host that only the refresh
// reports. The host is the chain's only candidate, but an agent never holds a
// guest, so the storage that already holds the VM keeps the chain; the next
// rebuild judges the whole chain and keeps the VM itself.
func TestManualLinkChainRefreshNeverLeavesAnAgentHoldingAGuest(t *testing.T) {
estate := newChainLinkEstate(t)
now := time.Now().UTC()
host := IngestRecord{
SourceID: "vc-1:host:host-9",
Resource: Resource{
Type: ResourceTypeAgent, Technology: "vmware", Name: "esx-9", Status: StatusOnline, LastSeen: now,
VMware: &VMwareData{ConnectionID: "vc-1", ManagedObjectID: "host-9", EntityType: "host"},
},
Identity: ResourceIdentity{Hostnames: []string{"esx-9"}},
}
withHost := make(map[DataSource][]IngestRecord, len(estate.records))
for source, records := range estate.records {
withHost[source] = records
}
withHost[SourceVMware] = append(slices.Clone(estate.records[SourceVMware]), host)
probe := NewMonitorAdapter(NewRegistry(nil))
probe.PopulateSnapshotAndSupplemental(estate.snapshot, withHost)
var hostID string
for _, resource := range probe.GetAll() {
if resource.Name == "esx-9" {
hostID = resource.ID
}
}
if hostID == "" {
t.Fatalf("probe estate %+v has no vSphere host", probe.GetAll())
}
vmID, storageID := estate.ids["nas-vm"], estate.ids["pve-storage"]
created := time.Date(2026, 10, 1, 9, 0, 0, 0, time.UTC)
store := NewMemoryStore()
for _, link := range []ResourceLink{
{ResourceA: vmID, ResourceB: storageID, PrimaryID: storageID, CreatedAt: created},
{ResourceA: storageID, ResourceB: hostID, PrimaryID: hostID, CreatedAt: created.Add(time.Minute)},
} {
if err := store.AddLink(link); err != nil {
t.Fatalf("add link: %v", err)
}
}
adapter := NewMonitorAdapter(NewRegistry(store))
adapter.PopulateSnapshotAndSupplemental(estate.snapshot, estate.records)
assertChainRoot(t, "rebuild without the host", adapter.currentRegistry(), storageID, ResourceTypeStorage, vmID, storageID)
adapter.PopulateSupplementalRecords(SourceVMware, withHost[SourceVMware])
assertChainRoot(t, "refresh bringing the host", adapter.currentRegistry(), storageID, ResourceTypeStorage, vmID, storageID, hostID)
adapter.PopulateSnapshotAndSupplemental(estate.snapshot, withHost)
assertChainRoot(t, "rebuild with the host", adapter.currentRegistry(), vmID, ResourceTypeVM, vmID, storageID, hostID)
}
func assertChainRoot(t *testing.T, step string, rr *ResourceRegistry, rootID string, rootType ResourceType, members ...string) {
t.Helper()
root, ok := rr.Get(rootID)
if !ok || root.Type != rootType {
t.Fatalf("%s: root %s = %+v (listed=%v), want a listed %s", step, rootID, root, ok, rootType)
}
for _, member := range members {
if _, listed := rr.Get(member); listed && member != rootID {
t.Fatalf("%s: member %s still listed beside %s", step, member, rootID)
}
if got, ok := rr.ResolveReferenceID(member); !ok || got != rootID {
t.Fatalf("%s: %s resolved to %q (%v), want %s", step, member, got, ok, rootID)
}
}
}
// A relink of a Proxmox node and its agent names the node's own ID, which the
// inferred join consumed before any link applies. The join records that fold
// before the links resolve their members, so a second link naming the node
// joins the chain whichever link the store lists first: applied in store
// order, a link naming the node ahead of the relink found no node and was
// skipped for good. The folded node keeps the agent shape it had, so a guest
// linked to it still outranks storage as the chain's root.
func TestManualLinkToAJoinedNodeFoldsTheSameWayInAnyLinkOrder(t *testing.T) {
now := time.Now().UTC()
node := models.Node{ID: "lab-pve1", Name: "pve1", Instance: "lab", Host: "https://10.0.0.5:8006", Status: "online", LastSeen: now, LinkedAgentID: "host-pve1"}
host := models.Host{
ID: "host-pve1", Hostname: "pve1", MachineID: "0123456789abcdef", Status: "online", LastSeen: now, LinkedNodeID: node.ID,
NetworkInterfaces: []models.HostNetworkInterface{{Name: "vmbr0", MAC: "aa:bb:cc:dd:ee:01", Addresses: []string{"10.0.0.5/24"}}},
}
pool := IngestRecord{
SourceID: "system:tn-1:pool:tank",
Resource: Resource{
Type: ResourceTypeStorage, Name: "tank", Status: StatusOnline, LastSeen: now,
Storage: &StorageMeta{Type: "zfs-pool", Platform: "truenas", Topology: "pool"},
},
}
snapshot := models.StateSnapshot{
LastUpdate: now, Nodes: []models.Node{node}, Hosts: []models.Host{host},
VMs: []models.VM{{ID: "lab:pve1:101", VMID: 101, Name: "web", Node: "pve1", Instance: "lab", Status: "running", Type: "qemu", LastSeen: now}},
}
records := map[DataSource][]IngestRecord{SourceTrueNAS: {pool}}
alone := NewRegistry(nil)
alone.IngestSnapshot(models.StateSnapshot{LastUpdate: now, Nodes: []models.Node{node}})
nodeID := hostRowsByFacet(t, alone.List()).node
probe := NewMonitorAdapter(NewRegistry(nil))
probe.PopulateSnapshotAndSupplemental(snapshot, records)
joinedID := hostRowsByFacet(t, probe.GetAll()).joined
var poolID, vmID string
for _, resource := range probe.GetAll() {
switch resource.Type {
case ResourceTypeStorage:
poolID = resource.ID
case ResourceTypeVM:
vmID = resource.ID
}
}
if nodeID == "" || joinedID == "" || poolID == "" || vmID == "" || nodeID == joinedID {
t.Fatalf("fixture joined node %q and agent into %q beside pool %q and VM %q", nodeID, joinedID, poolID, vmID)
}
created := time.Date(2026, 10, 1, 9, 0, 0, 0, time.UTC)
relink := ResourceLink{ResourceA: joinedID, ResourceB: nodeID, PrimaryID: joinedID, CreatedAt: created}
for _, tc := range []struct {
name string
links []ResourceLink
root string
rootType ResourceType
}{
{
name: "a pool linked to the node",
links: []ResourceLink{relink, {ResourceA: nodeID, ResourceB: poolID, PrimaryID: poolID, CreatedAt: created.Add(time.Minute)}},
root: poolID,
rootType: ResourceTypeStorage,
},
{
// The node is an agent-type row whichever way it was folded, so a
// guest linked to it folds the chain, not the pool.
name: "a guest and a pool linked to the node",
links: []ResourceLink{
relink,
{ResourceA: nodeID, ResourceB: vmID, PrimaryID: nodeID, CreatedAt: created.Add(time.Minute)},
{ResourceA: nodeID, ResourceB: poolID, PrimaryID: poolID, CreatedAt: created.Add(2 * time.Minute)},
},
root: vmID,
rootType: ResourceTypeVM,
},
} {
t.Run(tc.name, func(t *testing.T) {
var reference []ManualLinkFold
for _, order := range linkOrders(len(tc.links)) {
store := NewMemoryStore()
for _, i := range order {
if err := store.AddLink(tc.links[i]); err != nil {
t.Fatalf("add link: %v", err)
}
}
adapter := NewMonitorAdapter(NewRegistry(store))
adapter.PopulateSnapshotAndSupplemental(snapshot, records)
seeded := NewRegistry(store)
seeded.IngestResources(adapter.GetAll())
for _, view := range []struct {
name string
rr *ResourceRegistry
}{{"monitor", adapter.currentRegistry()}, {"resources API", seeded}} {
assertChainRoot(t, fmt.Sprintf("order %v, %s", order, view.name), view.rr, tc.root, tc.rootType, joinedID, nodeID, poolID)
folds := view.rr.ManualLinkFolds(tc.root)
slices.SortFunc(folds, func(a, b ManualLinkFold) int {
return strings.Compare(a.HolderID+"|"+a.FoldedID, b.HolderID+"|"+b.FoldedID)
})
if reference == nil {
reference = folds
} else if !reflect.DeepEqual(folds, reference) {
t.Fatalf("order %v, %s: folds %+v, want %+v as in the first order", order, view.name, folds, reference)
}
}
}
})
}
}
// A registry seeded from a listing keeps the folds that listing's links made,
// whatever the store holds now: the links changed after the monitor published
// it. A record that recreates a folded member then folds it under the new
// links, and every ID the listing's fold records fold in keeps answering to
// the row that survives. Turning a record the surviving row carries to match
// the new link would leave the ID it folded in without a row, and so would
// hanging the folded row's records from the linking member when the surviving
// row's own ID lies between that member and the folded row.
func TestManualLinkChangedAfterSeedingKeepsFoldedIDsResolving(t *testing.T) {
estate := newChainLinkEstate(t)
created := time.Date(2026, 10, 1, 9, 0, 0, 0, time.UTC)
vmR, vmT, pool, otherPool := estate.ids["vm-r"], estate.ids["vm-u"], estate.ids["tank"], estate.ids["vault"]
chain := []ResourceLink{
{ResourceA: vmT, ResourceB: pool, PrimaryID: pool},
{ResourceA: pool, ResourceB: vmR, PrimaryID: vmR},
}
// The VM, the pool and the second pool linked in a cycle, rooted at the VM.
cycle := []ResourceLink{
{ResourceA: pool, ResourceB: vmR, PrimaryID: vmR},
{ResourceA: otherPool, ResourceB: vmR, PrimaryID: vmR},
{ResourceA: pool, ResourceB: otherPool, PrimaryID: otherPool},
}
for _, tc := range []struct {
name string
published []ResourceLink
changed []ResourceLink
// root and rootType name the row left after the replay.
root string
rootType ResourceType
members []string
}{
{
// The pool now folds into the VM inside the published root.
name: "the pool folds into a member of the root",
published: chain,
changed: []ResourceLink{{ResourceA: pool, ResourceB: vmT, PrimaryID: vmT}},
root: vmR,
rootType: ResourceTypeVM,
members: []string{vmR, vmT, pool},
},
{
// Only the link to the pool's folded VM stays, so the recreated pool
// is the chain's root and the published root folds into it through
// that VM, a member its records reach only through the pool.
name: "the recreated pool becomes the root through a member",
published: chain,
changed: []ResourceLink{{ResourceA: vmT, ResourceB: pool, PrimaryID: pool}},
root: pool,
rootType: ResourceTypeStorage,
members: []string{vmR, vmT, pool},
},
{
// Only the link between the pool and the published root stays, with
// the pool primary: the root folds into the recreated pool through
// the very ID the pool's records fold it under.
name: "the recreated pool becomes the root over the published root",
published: chain,
changed: []ResourceLink{{ResourceA: pool, ResourceB: vmR, PrimaryID: pool}},
root: pool,
rootType: ResourceTypeStorage,
members: []string{vmR, vmT, pool},
},
{
// The published records form a cycle, and the links now root it at a
// recreated pool. Every record that named the pool as folded turns
// away from it, so the new root is never a folded ID.
name: "a recreated pool roots a cycle of records",
published: cycle,
changed: []ResourceLink{
{ResourceA: vmR, ResourceB: pool, PrimaryID: pool},
{ResourceA: otherPool, ResourceB: pool, PrimaryID: pool},
{ResourceA: vmR, ResourceB: otherPool, PrimaryID: otherPool},
},
root: pool,
rootType: ResourceTypeStorage,
members: []string{vmR, pool, otherPool},
},
} {
t.Run(tc.name, func(t *testing.T) {
published := NewMemoryStore()
for i, link := range tc.published {
link.CreatedAt = created.Add(time.Duration(i) * time.Minute)
if err := published.AddLink(link); err != nil {
t.Fatalf("add link: %v", err)
}
}
adapter := NewMonitorAdapter(NewRegistry(published))
adapter.PopulateSnapshotAndSupplemental(estate.snapshot, estate.records)
assertChainRoot(t, "published", adapter.currentRegistry(), vmR, ResourceTypeVM, tc.members...)
changed := NewMemoryStore()
for i, link := range tc.changed {
link.CreatedAt = created.Add(time.Duration(10+i) * time.Minute)
if err := changed.AddLink(link); err != nil {
t.Fatalf("add link: %v", err)
}
}
seeded := NewRegistry(changed)
seeded.IngestResources(adapter.GetAll())
assertChainRoot(t, "seeded", seeded, vmR, ResourceTypeVM, tc.members...)
seeded.IngestRecords(SourceTrueNAS, estate.records[SourceTrueNAS])
assertChainRoot(t, "seeded then replayed", seeded, tc.root, tc.rootType, tc.members...)
// Each side keeps what it is on its own through the turned and the
// carried records, which report-merge selects links by, and the
// surviving row is never recorded as a folded ID, which report-merge
// infers its root from when the row is re-keyed.
own := map[string][]DataSource{vmR: {SourceProxmox}, vmT: {SourceProxmox}, pool: {SourceTrueNAS}, otherPool: {SourceTrueNAS}}
for _, fold := range seeded.ManualLinkFolds(tc.root) {
if !reflect.DeepEqual(fold.HolderOwn, own[fold.HolderID]) || !reflect.DeepEqual(fold.FoldedOwn, own[fold.FoldedID]) {
t.Fatalf("fold %+v: own sources not %v for holder and %v for folded", fold, own[fold.HolderID], own[fold.FoldedID])
}
if fold.FoldedID == tc.root {
t.Fatalf("fold %+v records the surviving row %s as folded", fold, tc.root)
}
}
})
}
}
// A cycle of links is recorded link by link, so a report-merge naming one
// member's source cuts every link that member holds: the pair set the chain
// pass records is what ReportedManualLinkFolds selects from, and the link that
// closes the cycle is in it with its sides' own sources. A fold recorded for
// only the links that folded rows in left the reported member joined through
// the link that closed the cycle.
func TestManualLinkChainCycleLinksAreAllCutByAReportedMember(t *testing.T) {
estate := newChainLinkEstate(t)
created := time.Date(2026, 10, 1, 9, 0, 0, 0, time.UTC)
nasVM, storage, guest := estate.ids["nas-vm"], estate.ids["pve-storage"], estate.ids["vsphere-vm"]
links := []ResourceLink{
{ResourceA: nasVM, ResourceB: storage, PrimaryID: storage, CreatedAt: created.Add(time.Minute)},
{ResourceA: storage, ResourceB: guest, PrimaryID: guest, CreatedAt: created.Add(2 * time.Minute)},
{ResourceA: guest, ResourceB: nasVM, PrimaryID: nasVM, CreatedAt: created},
}
for _, order := range linkOrders(len(links)) {
store := NewMemoryStore()
for _, i := range order {
if err := store.AddLink(links[i]); err != nil {
t.Fatalf("add link: %v", err)
}
}
adapter := NewMonitorAdapter(NewRegistry(store))
adapter.PopulateSnapshotAndSupplemental(estate.snapshot, estate.records)
seeded := NewRegistry(store)
seeded.IngestResources(adapter.GetAll())
for name, rr := range map[string]*ResourceRegistry{"monitor": adapter.currentRegistry(), "resources API": seeded} {
folds := rr.ManualLinkFolds(nasVM)
cut := ReportedManualLinkFolds(nasVM, folds, func(sources ...DataSource) bool {
return slices.Contains(sources, SourceVMware)
})
pairs := make(map[string]bool, len(cut))
for _, fold := range cut {
pairs[exclusionKey(fold.HolderID, fold.FoldedID)] = true
}
for _, link := range links {
if link.ResourceA != guest && link.ResourceB != guest {
continue
}
if !pairs[exclusionKey(link.ResourceA, link.ResourceB)] {
t.Fatalf("order %v, %s: reporting vmware leaves the link %s - %s joined; cut %+v of %+v", order, name, link.ResourceA, link.ResourceB, cut, folds)
}
}
if len(cut) != 2 {
t.Fatalf("order %v, %s: reporting vmware cuts %d links, want the two the vSphere VM holds: %+v", order, name, len(cut), cut)
}
}
}
}
// successionHookStore runs a callback from inside a rebuild, right after the
// store applied a canonical-ID succession and before the rebuilt registry is
// published: the moment the previous generation is still served while the
// store already answers for the successor.
type successionHookStore struct {
ResourceStore
afterSuccession func()
}
func (s *successionHookStore) ApplyCanonicalIDSuccessions(successions []CanonicalIDSuccession) error {
err := s.ResourceStore.(canonicalIDSuccessor).ApplyCanonicalIDSuccessions(successions)
if err == nil && s.afterSuccession != nil {
s.afterSuccession()
}
return err
}
// A rebuild applies a succession to the store (the operator's rows move to the
// successor) before the generation that lists the successor is published, and
// the previous generation keeps serving until then. The dispatch gates and the
// planner read the operator state of the ID the generation they consult lists,
// so in that window they ask for an ID whose row has gone. A lock the operator
// set must still be found there: the old ID is the resource the previous
// generation would act on.
func TestRemediationLockHoldsOnTheRetiredIDWhileThePreviousGenerationServes(t *testing.T) {
now := time.Now().UTC()
type scenario struct {
before, after models.StateSnapshot
// kind is the resource type that changes ID.
kind ResourceType
}
scenarios := map[string]scenario{
// A record-declared era (the reporter's source-specific ID), applied
// while the snapshot is ingested.
"host agent gaining a machine ID": {
kind: ResourceTypeAgent,
before: models.StateSnapshot{Hosts: []models.Host{{ID: "agent-window-01", Hostname: "window-01", Status: "online", LastSeen: now}}},
after: models.StateSnapshot{Hosts: []models.Host{{ID: "agent-window-01", Hostname: "window-01", MachineID: "machine-window-01", Status: "online", LastSeen: now}}},
},
// A pin-driven era, applied once the rebuilt registry persists its pins.
"swarm host gaining a full hostname": {
kind: ResourceTypeAgent,
before: models.StateSnapshot{DockerHosts: []models.DockerHost{{
ID: "docker-window-02", Hostname: "cloud", Status: "online", LastSeen: now,
Swarm: &models.DockerSwarmInfo{ClusterID: "swarm-window", ClusterName: "prod-swarm"},
}}},
after: models.StateSnapshot{DockerHosts: []models.DockerHost{{
ID: "docker-window-02", Hostname: "cloud.a", Status: "online", LastSeen: now,
Swarm: &models.DockerSwarmInfo{ClusterID: "swarm-window", ClusterName: "prod-swarm"},
}}},
},
}
stores := map[string]func(t *testing.T) ResourceStore{
"memory": func(t *testing.T) ResourceStore { return NewMemoryStore() },
"sqlite": func(t *testing.T) ResourceStore {
store, err := NewSQLiteResourceStore(t.TempDir(), "default")
if err != nil {
t.Fatalf("NewSQLiteResourceStore: %v", err)
}
t.Cleanup(func() { _ = store.Close() })
return store
},
}
locks := map[string]ResourceOperatorState{
"never auto-remediate": {NeverAutoRemediate: true},
"retired": {LifecycleState: LifecycleStateRetired},
}
idsOf := func(adapter *MonitorAdapter, kind ResourceType) []string {
var ids []string
for _, resource := range adapter.GetByType(kind) {
ids = append(ids, resource.ID)
}
return ids
}
for scenarioName, sc := range scenarios {
for storeName, newStore := range stores {
for lockName, lock := range locks {
t.Run(scenarioName+"/"+storeName+"/"+lockName, func(t *testing.T) {
hooked := &successionHookStore{ResourceStore: newStore(t)}
adapter := NewMonitorAdapter(NewRegistry(hooked))
adapter.PopulateFromSnapshot(sc.before)
before := idsOf(adapter, sc.kind)
if len(before) != 1 {
t.Fatalf("first generation lists %v, want the one resource", before)
}
oldID := before[0]
state := lock
state.CanonicalID, state.SetAt, state.SetBy = oldID, now.Add(-time.Hour), "operator"
if err := hooked.SetResourceOperatorState(state); err != nil {
t.Fatalf("lock %s: %v", oldID, err)
}
var (
applied int
listedDuring []string
foundDuring bool
lockedDuring bool
readErrDuring error
)
hooked.afterSuccession = func() {
applied++
listedDuring = idsOf(adapter, sc.kind)
var got ResourceOperatorState
got, foundDuring, readErrDuring = adapter.GetResourceOperatorState(oldID)
lockedDuring = foundDuring && got.BlocksRemediation()
}
adapter.PopulateFromSnapshot(sc.after)
if applied == 0 {
t.Fatalf("the rebuild applied no succession; the fixture no longer changes %s's era", oldID)
}
if !slices.Equal(listedDuring, []string{oldID}) {
t.Fatalf("during the rebuild the served generation lists %v, want the previous generation's %s", listedDuring, oldID)
}
if readErrDuring != nil {
t.Fatalf("read operator state of %s during the rebuild: %v", oldID, readErrDuring)
}
if !lockedDuring {
t.Fatalf("while the previous generation still lists %s the lock the operator set on it reads as missing (found=%v)", oldID, foundDuring)
}
after := idsOf(adapter, sc.kind)
if len(after) != 1 || after[0] == oldID {
t.Fatalf("the published generation lists %v, want the resource under a new ID", after)
}
if got, found, err := adapter.GetResourceOperatorState(after[0]); err != nil || !found || !got.BlocksRemediation() {
t.Fatalf("the published ID %s does not hold the lock: found=%v err=%v state=%+v", after[0], found, err, got)
}
})
}
}
}
}