Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

38.9 KB
Raw Blame
package monitoring

import (
	"encoding/json"
	"testing"
	"time"

	"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
	"github.com/rcourtman/pulse-go-rewrite/internal/config"
	"github.com/rcourtman/pulse-go-rewrite/internal/mock"
	"github.com/rcourtman/pulse-go-rewrite/internal/models"
	"github.com/rcourtman/pulse-go-rewrite/internal/storagehealth"
	"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
)

type resourceOnlyStore struct {
	resources []unifiedresources.Resource
}

func (s *resourceOnlyStore) ShouldSkipAPIPolling(string) bool { return false }

func (s *resourceOnlyStore) GetPollingRecommendations() map[string]float64 { return nil }

func (s *resourceOnlyStore) GetAll() []unifiedresources.Resource {
	out := make([]unifiedresources.Resource, len(s.resources))
	copy(out, s.resources)
	return out
}

func (s *resourceOnlyStore) PopulateFromSnapshot(models.StateSnapshot) {}

type freshnessResourceStore struct {
	resourceOnlyStore
	freshness time.Time
}

func (s *freshnessResourceStore) UnifiedResourceFreshness() time.Time {
	return s.freshness
}

type testSupplementalInventoryReadinessProvider struct {
	source  unifiedresources.DataSource
	records []unifiedresources.IngestRecord
	settled bool
	readyAt time.Time
}

func (p *testSupplementalInventoryReadinessProvider) SupplementalRecords(_ *Monitor, _ string) []unifiedresources.IngestRecord {
	out := make([]unifiedresources.IngestRecord, len(p.records))
	copy(out, p.records)
	return out
}

func (p *testSupplementalInventoryReadinessProvider) SnapshotOwnedSources() []unifiedresources.DataSource {
	if p.source == "" {
		return nil
	}
	return []unifiedresources.DataSource{p.source}
}

func (p *testSupplementalInventoryReadinessProvider) SupplementalInventoryReadyAt(*Monitor, string) (time.Time, bool) {
	return p.readyAt, p.settled
}

type unifiedResourceGetter interface {
	GetAll() []unifiedresources.Resource
}

func hasUnifiedResource(resources []unifiedresources.Resource, id string) bool {
	for _, resource := range resources {
		if resource.ID == id {
			return true
		}
	}
	return false
}

func hasUnifiedResourceName(resources []unifiedresources.Resource, name string) bool {
	for _, resource := range resources {
		if resource.Name == name {
			return true
		}
	}
	return false
}

func TestConvertResourcesForBroadcastCoalescesSplitHostResources(t *testing.T) {
	now := time.Date(2026, 5, 14, 10, 30, 0, 0, time.UTC)
	resources := []unifiedresources.Resource{
		{
			ID:       "agent-proxmox-delly",
			Type:     unifiedresources.ResourceTypeAgent,
			Name:     "delly",
			Status:   unifiedresources.StatusOnline,
			LastSeen: now.Add(-1 * time.Second),
			Sources:  []unifiedresources.DataSource{unifiedresources.SourceProxmox},
			Identity: unifiedresources.ResourceIdentity{Hostnames: []string{"delly"}},
			Proxmox: &unifiedresources.ProxmoxData{
				NodeName:    "delly",
				ClusterName: "homelab",
				PVEVersion:  "9.1.9",
			},
		},
		{
			ID:       "agent-runtime-delly",
			Type:     unifiedresources.ResourceTypeAgent,
			Name:     "delly",
			Status:   unifiedresources.StatusOnline,
			LastSeen: now,
			Sources:  []unifiedresources.DataSource{unifiedresources.SourceAgent},
			Identity: unifiedresources.ResourceIdentity{
				MachineID: "agent-machine-delly",
				Hostnames: []string{
					"delly",
				},
			},
			Agent: &unifiedresources.AgentData{
				AgentID:  "agent-machine-delly",
				Hostname: "delly",
				OSName:   "Proxmox VE",
			},
			Metrics: &unifiedresources.ResourceMetrics{
				DiskRead: &unifiedresources.MetricValue{
					Value:  4096,
					Unit:   "bytes/s",
					Source: unifiedresources.SourceAgent,
				},
				DiskWrite: &unifiedresources.MetricValue{
					Value:  8192,
					Unit:   "bytes/s",
					Source: unifiedresources.SourceAgent,
				},
			},
		},
	}

	frontend, _ := convertResourcesForBroadcast(resources)
	if len(frontend) != 1 {
		t.Fatalf("expected split host resources to coalesce into 1 frontend resource, got %d: %#v", len(frontend), frontend)
	}

	resource := frontend[0]
	if resource.ID != "agent-runtime-delly" {
		t.Fatalf("expected agent-backed resource ID, got %q", resource.ID)
	}
	if resource.SourceType != "hybrid" {
		t.Fatalf("expected hybrid source type, got %q", resource.SourceType)
	}
	if resource.PlatformType != "proxmox-pve" {
		t.Fatalf("expected proxmox-pve platform type, got %q", resource.PlatformType)
	}
	sourceSet := make(map[string]bool, len(resource.Sources))
	for _, source := range resource.Sources {
		sourceSet[source] = true
	}
	if !sourceSet["agent"] || !sourceSet["proxmox"] {
		t.Fatalf("expected agent and proxmox sources, got %#v", resource.Sources)
	}
	if len(resource.Proxmox) == 0 {
		t.Fatal("expected merged Proxmox facet")
	}
	if len(resource.Agent) == 0 {
		t.Fatal("expected merged agent facet")
	}
	if resource.DiskIO == nil {
		t.Fatal("expected aggregate disk I/O rates in broadcast resource")
	}
	if resource.DiskIO.ReadRate == nil || resource.DiskIO.WriteRate == nil || *resource.DiskIO.ReadRate != 4096 || *resource.DiskIO.WriteRate != 8192 {
		t.Fatalf("unexpected aggregate disk I/O rates: %+v", resource.DiskIO)
	}
}

func TestConvertResourcesForBroadcastAttachesResolvedStorageMetricsTarget(t *testing.T) {
	now := time.Date(2026, 5, 14, 10, 45, 0, 0, time.UTC)
	agentID := "de6d3fee-2595-6c2b-6b08-43db6b0ab427"
	storageID := "storage-2642e9fca16b2f87"
	parentID := agentID
	total := int64(24_000_000_000_000)
	used := int64(16_900_000_000_000)

	registry := unifiedresources.NewRegistry(nil)
	registry.IngestResources([]unifiedresources.Resource{
		{
			ID:       agentID,
			Type:     unifiedresources.ResourceTypeAgent,
			Name:     "Tower",
			Status:   unifiedresources.StatusOnline,
			LastSeen: now,
			Sources:  []unifiedresources.DataSource{unifiedresources.SourceAgent},
			Agent: &unifiedresources.AgentData{
				AgentID:  agentID,
				Hostname: "Tower",
			},
		},
		{
			ID:       storageID,
			Type:     unifiedresources.ResourceTypeStorage,
			Name:     "Tower Array",
			Status:   unifiedresources.StatusWarning,
			LastSeen: now,
			Sources:  []unifiedresources.DataSource{unifiedresources.SourceAgent},
			ParentID: &parentID,
			Metrics: &unifiedresources.ResourceMetrics{
				Disk: &unifiedresources.MetricValue{
					Used:    &used,
					Total:   &total,
					Percent: 70.4,
					Unit:    "bytes",
					Source:  unifiedresources.SourceAgent,
				},
			},
			Storage: &unifiedresources.StorageMeta{
				Type:     "unraid-array",
				Platform: "unraid",
				Topology: "array",
			},
		},
	})
	adapter := unifiedresources.NewMonitorAdapter(registry)

	frontend, _ := convertResourcesForBroadcast(adapter.GetAll(), adapter)

	var storagePayload struct {
		ID            string `json:"id"`
		MetricsTarget struct {
			ResourceType string `json:"resourceType"`
			ResourceID   string `json:"resourceId"`
		} `json:"metricsTarget"`
	}
	for _, resource := range frontend {
		if resource.ID != storageID {
			continue
		}
		encoded, err := json.Marshal(resource)
		if err != nil {
			t.Fatalf("marshal storage frontend resource: %v", err)
		}
		if err := json.Unmarshal(encoded, &storagePayload); err != nil {
			t.Fatalf("unmarshal storage frontend resource: %v", err)
		}
		break
	}

	if storagePayload.ID != storageID {
		t.Fatalf("expected storage resource %q in frontend payload, got %q", storageID, storagePayload.ID)
	}
	if storagePayload.MetricsTarget.ResourceType != "storage" ||
		storagePayload.MetricsTarget.ResourceID != agentID+"/storage:unraid-array" {
		t.Fatalf("metrics target = %+v", storagePayload.MetricsTarget)
	}
}

func TestConvertResourcesForBroadcastKeepsLinkedGuestCPUAndHistoryTargetAligned(t *testing.T) {
	now := time.Now().UTC()
	tests := []struct {
		name          string
		resourceType  unifiedresources.ResourceType
		technology    string
		targetType    string
		containerType string
	}{
		{
			name:          "lxc",
			resourceType:  unifiedresources.ResourceTypeSystemContainer,
			technology:    "lxc",
			targetType:    "system-container",
			containerType: "lxc",
		},
		{
			name:         "qemu",
			resourceType: unifiedresources.ResourceTypeVM,
			technology:   "qemu",
			targetType:   "vm",
		},
	}

	for _, tc := range tests {
		t.Run(tc.name, func(t *testing.T) {
			guestID := string(tc.resourceType) + "-guest-301"
			guestSourceID := "cluster-a-node-1-" + tc.name + "-301"
			const agentID = "agent-inside-guest-301"
			store := unifiedresources.NewMemoryStore()
			if err := store.AddLink(unifiedresources.ResourceLink{
				ResourceA: guestID,
				ResourceB: agentID,
				PrimaryID: agentID,
			}); err != nil {
				t.Fatalf("add link: %v", err)
			}
			registry := unifiedresources.NewRegistry(store)
			registry.IngestResources([]unifiedresources.Resource{
				{
					ID:         guestID,
					Type:       tc.resourceType,
					Technology: tc.technology,
					Name:       tc.name + "-301",
					Status:     unifiedresources.StatusOnline,
					LastSeen:   now,
					Sources:    []unifiedresources.DataSource{unifiedresources.SourceProxmox},
					SourceStatus: map[unifiedresources.DataSource]unifiedresources.SourceStatus{
						unifiedresources.SourceProxmox: {Status: "online", LastSeen: now},
					},
					Proxmox: &unifiedresources.ProxmoxData{
						SourceID:      guestSourceID,
						VMID:          301,
						ContainerType: tc.containerType,
						CPUs:          8,
					},
					Metrics: &unifiedresources.ResourceMetrics{
						CPU: &unifiedresources.MetricValue{
							Value:   0.58,
							Percent: 0.58,
							Unit:    "percent",
							Source:  unifiedresources.SourceProxmox,
						},
					},
				},
				{
					ID:       agentID,
					Type:     unifiedresources.ResourceTypeAgent,
					Name:     tc.name + "-301",
					Status:   unifiedresources.StatusOnline,
					LastSeen: now,
					Sources:  []unifiedresources.DataSource{unifiedresources.SourceAgent},
					SourceStatus: map[unifiedresources.DataSource]unifiedresources.SourceStatus{
						unifiedresources.SourceAgent: {Status: "online", LastSeen: now},
					},
					Agent: &unifiedresources.AgentData{AgentID: agentID, CPUCount: 8},
					Metrics: &unifiedresources.ResourceMetrics{
						CPU: &unifiedresources.MetricValue{
							Value:   94,
							Percent: 94,
							Unit:    "percent",
							Source:  unifiedresources.SourceAgent,
						},
					},
				},
			})

			adapter := unifiedresources.NewMonitorAdapter(registry)
			frontend, _ := convertResourcesForBroadcast(adapter.GetAll(), adapter)
			if len(frontend) != 1 {
				t.Fatalf("broadcast resources = %d, want one merged guest: %+v", len(frontend), frontend)
			}
			got := frontend[0]
			if got.ID != guestID || got.Type != string(tc.resourceType) || got.Technology != tc.technology {
				t.Fatalf("broadcast resource = %s/%s/%s, want %s/%s/%s", got.ID, got.Type, got.Technology, guestID, tc.resourceType, tc.technology)
			}
			if got.CPU == nil || got.CPU.Current != 0.58 {
				t.Fatalf("broadcast CPU = %+v, want proxmox 0.58", got.CPU)
			}

			var target unifiedresources.MetricsTarget
			if err := json.Unmarshal(got.MetricsTarget, &target); err != nil {
				t.Fatalf("decode metrics target: %v", err)
			}
			if target.ResourceType != tc.targetType || target.ResourceID != guestSourceID {
				t.Fatalf("broadcast metrics target = %+v, want %s/%s", target, tc.targetType, guestSourceID)
			}
		})
	}
}

func hasFrontendResourceName(resources []models.ResourceFrontend, name string) bool {
	for _, resource := range resources {
		if resource.Name == name {
			return true
		}
	}
	return false
}

// hasFrontendResourceNameForHostType returns true only when a docker-host or
// proxmox-node-shaped resource carries the literal Name. Docker Swarm nodes
// added in 89abed099 legitimately publish the swarm-node hostname (e.g.
// "edge-apps-01") as their Name because that is how Docker Swarm identifies
// members; the canonical legacy-label rejection rules below apply only to
// the host-type resources that previously surfaced the lowercase-hyphenated
// host hostname instead of the canonical DisplayName.
func hasFrontendResourceNameForHostType(resources []models.ResourceFrontend, name string) bool {
	for _, resource := range resources {
		if resource.Name != name {
			continue
		}
		switch resource.Type {
		case "docker-host", "agent", "node":
			return true
		}
	}
	return false
}

func TestMonitorGetUnifiedReadStateOrSnapshotUsesStoreResourcesWithoutSnapshotFallback(t *testing.T) {
	now := time.Date(2026, 3, 6, 12, 0, 0, 0, time.UTC)
	store := &resourceOnlyStore{
		resources: []unifiedresources.Resource{
			{
				ID:       "agent-store-1",
				Type:     unifiedresources.ResourceTypeAgent,
				Name:     "agent-store-1",
				Status:   unifiedresources.StatusOnline,
				LastSeen: now,
				Sources:  []unifiedresources.DataSource{unifiedresources.SourceAgent},
				Identity: unifiedresources.ResourceIdentity{Hostnames: []string{"agent-store-1"}},
			},
		},
	}

	m := &Monitor{
		state:         models.NewState(),
		resourceStore: store,
	}

	readState := m.GetUnifiedReadStateOrSnapshot()
	getter, ok := readState.(unifiedResourceGetter)
	if !ok {
		t.Fatalf("expected read-state adapter with GetAll, got %T", readState)
	}

	resources := getter.GetAll()
	if !hasUnifiedResource(resources, "agent-store-1") {
		t.Fatalf("expected store-backed unified resource in read-state, got %#v", resources)
	}
}

func TestMonitorGetUnifiedReadStateOrSnapshotFallsBackToSnapshotWhenStoreEmpty(t *testing.T) {
	now := time.Date(2026, 3, 6, 12, 30, 0, 0, time.UTC)
	state := models.NewState()
	state.UpsertHost(models.Host{
		ID:       "host-snapshot-1",
		Hostname: "host-snapshot-1",
		Status:   "online",
		LastSeen: now,
	})

	m := &Monitor{
		state:         state,
		resourceStore: &resourceOnlyStore{},
	}

	readState := m.GetUnifiedReadStateOrSnapshot()
	getter, ok := readState.(unifiedResourceGetter)
	if !ok {
		t.Fatalf("expected read-state adapter with GetAll, got %T", readState)
	}

	resources := getter.GetAll()
	if !hasUnifiedResourceName(resources, "host-snapshot-1") {
		t.Fatalf("expected snapshot-backed unified resource in read-state, got %#v", resources)
	}
}

func TestMonitorUnifiedResourceSnapshotFallsBackToSnapshotWhenStoreEmpty(t *testing.T) {
	now := time.Date(2026, 3, 6, 13, 0, 0, 0, time.UTC)
	state := models.NewState()
	state.UpsertHost(models.Host{
		ID:       "host-snapshot-2",
		Hostname: "host-snapshot-2",
		Status:   "online",
		LastSeen: now,
	})

	m := &Monitor{
		state:         state,
		resourceStore: &resourceOnlyStore{},
	}

	resources, freshness := m.UnifiedResourceSnapshot()
	if !hasUnifiedResourceName(resources, "host-snapshot-2") {
		t.Fatalf("expected snapshot-backed unified resource, got %#v", resources)
	}
	if freshness.IsZero() {
		t.Fatal("expected non-zero freshness from snapshot fallback")
	}
}

func TestMonitorUnifiedResourceSnapshotIncludesRecentStandaloneHostContinuity(t *testing.T) {
	now := time.Now().UTC().Truncate(time.Second)
	store := config.NewHostContinuityStore(t.TempDir(), nil)
	if err := store.Upsert(config.HostContinuityEntry{
		HostID:       "host-1",
		ReportHostID: "machine-1",
		Hostname:     "host-1.local",
		DisplayName:  "Host One",
		MachineID:    "machine-1",
		AgentVersion: "6.0.0-rc.5",
		LastSeen:     now,
	}); err != nil {
		t.Fatalf("Upsert continuity: %v", err)
	}

	m := &Monitor{
		state:               models.NewState(),
		resourceStore:       unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(nil)),
		hostContinuityStore: store,
	}

	resources, freshness := m.UnifiedResourceSnapshot()
	if !hasUnifiedResourceName(resources, "Host One") {
		t.Fatalf("expected continuity-backed resource in unified snapshot, got %#v", resources)
	}
	if freshness.IsZero() {
		t.Fatal("expected continuity-backed snapshot to carry non-zero freshness")
	}
}

func TestMonitorUnifiedResourceSnapshotPrefersStoreFreshness(t *testing.T) {
	state := models.NewState()
	state.UpsertHost(models.Host{
		ID:       "host-state-1",
		Hostname: "host-state-1",
		Status:   "online",
		LastSeen: time.Date(2026, 3, 6, 13, 0, 0, 0, time.UTC),
	})

	storeFreshness := time.Date(2026, 3, 7, 9, 30, 0, 0, time.UTC)
	m := &Monitor{
		state: state,
		resourceStore: &freshnessResourceStore{
			resourceOnlyStore: resourceOnlyStore{
				resources: []unifiedresources.Resource{
					{
						ID:       "agent-store-1",
						Type:     unifiedresources.ResourceTypeAgent,
						Name:     "agent-store-1",
						Status:   unifiedresources.StatusOnline,
						LastSeen: storeFreshness,
						Sources:  []unifiedresources.DataSource{unifiedresources.SourceAgent},
						Identity: unifiedresources.ResourceIdentity{Hostnames: []string{"agent-store-1"}},
					},
				},
			},
			freshness: storeFreshness,
		},
	}

	_, freshness := m.UnifiedResourceSnapshot()
	if !freshness.Equal(storeFreshness) {
		t.Fatalf("expected store freshness %v, got %v", storeFreshness, freshness)
	}
}

func TestMonitorGetUnifiedReadStateOrSnapshotUsesCanonicalMockUnifiedResources(t *testing.T) {
	mustSetMockEnabled(t, true)
	t.Cleanup(func() { mustSetMockEnabled(t, false) })

	graph := mock.CurrentFixtureGraph()
	legacyName := ""
	if len(graph.State.VMs) > 0 {
		legacyName = graph.State.VMs[0].Name
	} else if len(graph.State.Containers) > 0 {
		legacyName = graph.State.Containers[0].Name
	}
	if legacyName == "" {
		t.Fatal("expected canonical mock graph to include at least one legacy resource")
	}

	store := &resourceOnlyStore{
		resources: []unifiedresources.Resource{
			{
				ID:       "store-only-resource",
				Type:     unifiedresources.ResourceTypeAgent,
				Name:     "store-only-resource",
				Status:   unifiedresources.StatusOnline,
				LastSeen: time.Date(2026, 3, 6, 14, 0, 0, 0, time.UTC),
				Sources:  []unifiedresources.DataSource{unifiedresources.SourceAgent},
				Identity: unifiedresources.ResourceIdentity{Hostnames: []string{"store-only-resource"}},
			},
		},
	}

	m := &Monitor{
		state:         models.NewState(),
		resourceStore: store,
		alertManager:  alerts.NewManager(),
	}
	defer m.alertManager.Stop()

	readState := m.GetUnifiedReadStateOrSnapshot()
	getter, ok := readState.(unifiedResourceGetter)
	if !ok {
		t.Fatalf("expected read-state adapter with GetAll, got %T", readState)
	}

	resources := getter.GetAll()
	if hasUnifiedResource(resources, "store-only-resource") {
		t.Fatal("expected mock-mode read-state to ignore live resource store data")
	}
	if !hasUnifiedResourceName(resources, "truenas-main") {
		t.Fatalf("expected mock-mode read-state to include TrueNAS mock resources, got %#v", resources)
	}
	if !hasUnifiedResourceName(resources, "esxi-01.lab.local") {
		t.Fatalf("expected mock-mode read-state to include VMware mock resources, got %#v", resources)
	}
	if !hasUnifiedResourceName(resources, legacyName) {
		t.Fatalf("expected mock-mode read-state to include legacy mock resource %q, got %#v", legacyName, resources)
	}
}

func TestMonitorBuildBroadcastFrontendStateUsesCanonicalMockUnifiedResources(t *testing.T) {
	previousEnabled := mock.IsMockEnabled()
	previousConfig := mock.GetConfig()
	t.Cleanup(func() {
		mustSetMockEnabled(t, false)
		mock.SetMockConfig(previousConfig)
		if previousEnabled {
			mustSetMockEnabled(t, true)
		}
	})

	stableConfig := mock.DefaultConfig
	stableConfig.UpdateInterval = 5 * time.Minute
	mustSetMockEnabled(t, false)
	mock.SetMockConfig(stableConfig)
	mustSetMockEnabled(t, true)

	graph := mock.CurrentFixtureGraph()
	legacyName := ""
	if len(graph.State.VMs) > 0 {
		legacyName = graph.State.VMs[0].Name
	} else if len(graph.State.Containers) > 0 {
		legacyName = graph.State.Containers[0].Name
	}
	if legacyName == "" {
		t.Fatal("expected canonical mock graph to include at least one legacy resource")
	}

	store := &resourceOnlyStore{
		resources: []unifiedresources.Resource{
			{
				ID:       "store-only-resource",
				Type:     unifiedresources.ResourceTypeAgent,
				Name:     "store-only-resource",
				Status:   unifiedresources.StatusOnline,
				LastSeen: time.Date(2026, 3, 6, 14, 0, 0, 0, time.UTC),
				Sources:  []unifiedresources.DataSource{unifiedresources.SourceAgent},
				Identity: unifiedresources.ResourceIdentity{Hostnames: []string{"store-only-resource"}},
			},
		},
	}

	m := &Monitor{
		state:         models.NewState(),
		resourceStore: store,
		alertManager:  alerts.NewManager(),
	}
	defer m.alertManager.Stop()

	expectedResources, freshness := m.UnifiedResourceSnapshot()
	frontend := m.BuildBroadcastFrontendState()

	if hasFrontendResourceName(frontend.Resources, "store-only-resource") {
		t.Fatal("expected mock-mode broadcast state to ignore live resource store data")
	}
	if !hasFrontendResourceName(frontend.Resources, "truenas-main") {
		t.Fatalf("expected mock-mode broadcast state to include TrueNAS mock resources, got %#v", frontend.Resources)
	}
	if hasFrontendResourceNameForHostType(frontend.Resources, "pve1") {
		t.Fatalf("expected mock-mode broadcast state to stop publishing legacy proxmox node labels, got %#v", frontend.Resources)
	}
	if hasFrontendResourceNameForHostType(frontend.Resources, "edge-apps-01") {
		t.Fatalf("expected mock-mode broadcast state to stop publishing legacy docker host labels, got %#v", frontend.Resources)
	}
	if !hasFrontendResourceName(frontend.Resources, "esxi-01.lab.local") {
		t.Fatalf("expected mock-mode broadcast state to include VMware mock resources, got %#v", frontend.Resources)
	}
	if !hasFrontendResourceName(frontend.Resources, "West Production A") {
		t.Fatalf("expected mock-mode broadcast state to include canonical proxmox host label, got %#v", frontend.Resources)
	}
	if !hasFrontendResourceName(frontend.Resources, "Edge Apps 01") {
		t.Fatalf("expected mock-mode broadcast state to include canonical docker host label, got %#v", frontend.Resources)
	}
	if !hasFrontendResourceName(frontend.Resources, legacyName) {
		t.Fatalf("expected mock-mode broadcast state to include legacy mock resource %q, got %#v", legacyName, frontend.Resources)
	}
	// The broadcast path coalesces resources that share a canonical host
	// merge key (for example, a Kubernetes node that the registry already
	// linked to its host agent), reshapes identity through
	// `attachBroadcastMetricsTargets`, and runs a second coalesce pass
	// inside convertResourcesForBroadcast. Larger fixture sizes
	// legitimately produce more merge candidates and the two coalesce
	// passes can drop a handful of additional rows beyond what a single
	// coalesce on the raw snapshot would; this test's purpose is to
	// confirm the broadcast publishes canonical mock unified resources
	// (not legacy node/docker labels) at SMB-fixture density, not to
	// pin an exact merge count. Require the broadcast count to fall
	// within a 5% tolerance of the convert-pipeline count so future
	// fixture bumps stay green as long as the merge contract is honored.
	expectedBroadcastResources, _ := convertResourcesForBroadcast(expectedResources)
	expectedBroadcastCount := len(expectedBroadcastResources)
	tolerance := expectedBroadcastCount / 20
	if tolerance < 5 {
		tolerance = 5
	}
	drift := expectedBroadcastCount - len(frontend.Resources)
	if drift < 0 {
		drift = -drift
	}
	if drift > tolerance {
		t.Fatalf("expected mock-mode broadcast state within ±%d of canonical unified resource count %d, got %d (raw snapshot count %d)", tolerance, expectedBroadcastCount, len(frontend.Resources), len(expectedResources))
	}
	if freshness.IsZero() {
		t.Fatal("expected canonical mock unified snapshot freshness")
	}
	if frontend.LastUpdate != freshness.UnixMilli() {
		t.Fatalf("expected broadcast lastUpdate %d from canonical mock freshness, got %d", freshness.UnixMilli(), frontend.LastUpdate)
	}
	for _, resource := range frontend.Resources {
		if resource.Type == "node" {
			t.Fatalf("expected canonical broadcast resource types only, got legacy node resource %#v", resource)
		}
	}
}

func TestConvertResourcesForBroadcastIncludesCanonicalHealthContext(t *testing.T) {
	resources := []unifiedresources.Resource{
		{
			ID:               "agent:tower",
			Type:             unifiedresources.ResourceTypeAgent,
			Name:             "Tower",
			Status:           unifiedresources.StatusWarning,
			LastSeen:         time.Date(2026, 5, 8, 11, 30, 0, 0, time.UTC),
			Sources:          []unifiedresources.DataSource{unifiedresources.SourceAgent},
			IncidentCount:    1,
			IncidentSeverity: storagehealth.RiskWarning,
			IncidentSummary:  "Unraid array is running without parity protection",
			Agent: &unifiedresources.AgentData{
				AgentID:               "agent-1",
				Hostname:              "tower",
				OSName:                "Unraid",
				StoragePostureSummary: "Unraid array is running without parity protection",
				StorageRisk: &unifiedresources.StorageRisk{
					Level: storagehealth.RiskWarning,
					Reasons: []unifiedresources.StorageRiskReason{{
						Code:     "unraid_no_parity",
						Severity: storagehealth.RiskWarning,
						Summary:  "Unraid array is running without parity protection",
					}},
				},
				Unraid: &unifiedresources.HostUnraidMeta{
					ArrayStarted:   true,
					ArrayState:     "STARTED",
					PostureSummary: "Unraid array is running without parity protection",
					Risk: &unifiedresources.StorageRisk{
						Level: storagehealth.RiskWarning,
						Reasons: []unifiedresources.StorageRiskReason{{
							Code:     "unraid_no_parity",
							Severity: storagehealth.RiskWarning,
							Summary:  "Unraid array is running without parity protection",
						}},
					},
				},
			},
			Docker: &unifiedresources.DockerData{
				AgentID:      "agent-1",
				HostSourceID: "agent-1",
				Hostname:     "tower",
				Runtime:      "docker",
			},
		},
		{
			ID:       "agent:tower/storage:unraid-array",
			Type:     unifiedresources.ResourceTypeStorage,
			Name:     "Tower Array",
			Status:   unifiedresources.StatusWarning,
			LastSeen: time.Date(2026, 5, 8, 11, 30, 0, 0, time.UTC),
			Sources:  []unifiedresources.DataSource{unifiedresources.SourceAgent},
			Storage: &unifiedresources.StorageMeta{
				Type:           "unraid-array",
				Platform:       "unraid",
				Topology:       "array",
				RiskSummary:    "Unraid array is running without parity protection",
				PostureSummary: "Unraid array is running without parity protection",
				SyncAction:     "check",
				Risk: &unifiedresources.StorageRisk{
					Level: storagehealth.RiskWarning,
					Reasons: []unifiedresources.StorageRiskReason{{
						Code:     "unraid_no_parity",
						Severity: storagehealth.RiskWarning,
						Summary:  "Unraid array is running without parity protection",
					}},
				},
			},
		},
	}

	frontend, _ := convertResourcesForBroadcast(resources)
	if len(frontend) != 2 {
		t.Fatalf("expected 2 frontend resources, got %d", len(frontend))
	}

	var agentPayload struct {
		IncidentSummary string `json:"incidentSummary"`
		Canonical       struct {
			PrimaryID string `json:"primaryId"`
		} `json:"canonicalIdentity"`
		DiscoveryTarget struct {
			ResourceType string `json:"resourceType"`
			AgentID      string `json:"agentId"`
		} `json:"discoveryTarget"`
		MetricsTarget struct {
			ResourceType string `json:"resourceType"`
			ResourceID   string `json:"resourceId"`
		} `json:"metricsTarget"`
		Agent struct {
			StoragePostureSummary string `json:"storagePostureSummary"`
			Unraid                struct {
				PostureSummary string `json:"postureSummary"`
				Risk           struct {
					Reasons []struct {
						Code    string `json:"code"`
						Summary string `json:"summary"`
					} `json:"reasons"`
				} `json:"risk"`
			} `json:"unraid"`
		} `json:"agent"`
	}
	encoded, err := json.Marshal(frontend[0])
	if err != nil {
		t.Fatalf("marshal agent frontend resource: %v", err)
	}
	if err := json.Unmarshal(encoded, &agentPayload); err != nil {
		t.Fatalf("unmarshal agent frontend resource: %v", err)
	}
	if agentPayload.IncidentSummary != "Unraid array is running without parity protection" {
		t.Fatalf("incident summary = %q", agentPayload.IncidentSummary)
	}
	if agentPayload.Canonical.PrimaryID != "agent:agent-1" {
		t.Fatalf("canonical primary ID = %q", agentPayload.Canonical.PrimaryID)
	}
	if agentPayload.DiscoveryTarget.ResourceType != "agent" || agentPayload.DiscoveryTarget.AgentID != "agent-1" {
		t.Fatalf("discovery target = %+v", agentPayload.DiscoveryTarget)
	}
	if agentPayload.MetricsTarget.ResourceType != "agent" || agentPayload.MetricsTarget.ResourceID != "agent-1" {
		t.Fatalf("metrics target = %+v", agentPayload.MetricsTarget)
	}
	if agentPayload.Agent.StoragePostureSummary != "Unraid array is running without parity protection" {
		t.Fatalf("agent storage posture summary = %q", agentPayload.Agent.StoragePostureSummary)
	}
	if agentPayload.Agent.Unraid.PostureSummary != "Unraid array is running without parity protection" {
		t.Fatalf("unraid posture summary = %q", agentPayload.Agent.Unraid.PostureSummary)
	}
	if len(agentPayload.Agent.Unraid.Risk.Reasons) != 1 || agentPayload.Agent.Unraid.Risk.Reasons[0].Code != "unraid_no_parity" {
		t.Fatalf("unraid risk reasons = %#v", agentPayload.Agent.Unraid.Risk.Reasons)
	}

	var storagePayload struct {
		PlatformType string   `json:"platformType"`
		SourceType   string   `json:"sourceType"`
		Sources      []string `json:"sources"`
		PlatformData struct {
			Platform       string `json:"platform"`
			Type           string `json:"type"`
			Topology       string `json:"topology"`
			PostureSummary string `json:"postureSummary"`
			SyncAction     string `json:"syncAction"`
		} `json:"platformData"`
		Storage struct {
			PostureSummary string `json:"postureSummary"`
			Risk           struct {
				Reasons []struct {
					Code string `json:"code"`
				} `json:"reasons"`
			} `json:"risk"`
		} `json:"storage"`
	}
	encoded, err = json.Marshal(frontend[1])
	if err != nil {
		t.Fatalf("marshal storage frontend resource: %v", err)
	}
	if err := json.Unmarshal(encoded, &storagePayload); err != nil {
		t.Fatalf("unmarshal storage frontend resource: %v", err)
	}
	if storagePayload.PlatformType != "agent" || storagePayload.SourceType != "agent" {
		t.Fatalf("expected unraid storage to remain agent-backed, got platform=%q source=%q", storagePayload.PlatformType, storagePayload.SourceType)
	}
	if len(storagePayload.Sources) != 1 || storagePayload.Sources[0] != "agent" {
		t.Fatalf("storage sources = %#v, want [agent]", storagePayload.Sources)
	}
	if storagePayload.PlatformData.Platform != "unraid" || storagePayload.PlatformData.Type != "unraid-array" || storagePayload.PlatformData.Topology != "array" {
		t.Fatalf("expected unraid storage platform data, got %+v", storagePayload.PlatformData)
	}
	if storagePayload.PlatformData.PostureSummary != "Unraid array is running without parity protection" {
		t.Fatalf("platformData posture summary = %q", storagePayload.PlatformData.PostureSummary)
	}
	if storagePayload.Storage.PostureSummary != "Unraid array is running without parity protection" {
		t.Fatalf("storage posture summary = %q", storagePayload.Storage.PostureSummary)
	}
	if len(storagePayload.Storage.Risk.Reasons) != 1 || storagePayload.Storage.Risk.Reasons[0].Code != "unraid_no_parity" {
		t.Fatalf("storage risk reasons = %#v", storagePayload.Storage.Risk.Reasons)
	}
}

func TestMonitorMonitoredSystemUsageWaitsForSupplementalReadinessAndStoreFreshness(t *testing.T) {
	monitor := &Monitor{}
	monitor.SetResourceStore(unifiedresources.NewMonitorAdapter(nil))

	provider := &testSupplementalInventoryReadinessProvider{
		source: unifiedresources.SourceTrueNAS,
	}
	monitor.SetSupplementalRecordsProvider(unifiedresources.SourceTrueNAS, provider)

	if usage := monitor.MonitoredSystemUsage(); usage.Available {
		t.Fatalf("expected unsettled supplemental inventory to be unavailable, got %+v", usage)
	}

	readyAt := time.Now().UTC()
	provider.records = []unifiedresources.IngestRecord{{
		SourceID: "system:tn-main",
		Resource: unifiedresources.Resource{
			ID:       "tn-main",
			Type:     unifiedresources.ResourceTypeAgent,
			Name:     "tn-main",
			Status:   unifiedresources.StatusOnline,
			LastSeen: readyAt,
			Identity: unifiedresources.ResourceIdentity{
				Hostnames: []string{"tn-main"},
			},
			TrueNAS: &unifiedresources.TrueNASData{
				Hostname: "tn-main",
			},
		},
	}}
	provider.readyAt = readyAt
	provider.settled = true

	if usage := monitor.MonitoredSystemUsage(); usage.Available {
		t.Fatalf("expected stale store freshness to keep usage unavailable, got %+v", usage)
	}

	monitor.SetSupplementalRecordsProvider(unifiedresources.SourceTrueNAS, provider)

	usage := monitor.MonitoredSystemUsage()
	if !usage.Available {
		t.Fatalf("expected usage to become available after rebuild, got %+v", usage)
	}
	if usage.Count != 1 {
		t.Fatalf("MonitoredSystemUsage().Count = %d, want 1", usage.Count)
	}
}

func TestMonitorMonitoredSystemUsageIncludesRecentStandaloneHostContinuity(t *testing.T) {
	now := time.Now().UTC()
	store := config.NewHostContinuityStore(t.TempDir(), nil)
	if err := store.Upsert(config.HostContinuityEntry{
		HostID:       "host-1",
		ReportHostID: "machine-1",
		Hostname:     "host-1.local",
		MachineID:    "machine-1",
		LastSeen:     now,
	}); err != nil {
		t.Fatalf("Upsert continuity: %v", err)
	}

	monitor := &Monitor{
		state:               models.NewState(),
		hostContinuityStore: store,
	}

	usage := monitor.MonitoredSystemUsage()
	if !usage.Available {
		t.Fatalf("expected continuity-backed usage to be available, got %+v", usage)
	}
	if usage.Count != 1 {
		t.Fatalf("MonitoredSystemUsage().Count = %d, want 1", usage.Count)
	}
}

func TestMonitorMonitoredSystemUsageDoesNotDoubleCountLinkedProxmoxHostContinuity(t *testing.T) {
	now := time.Now().UTC()
	store := config.NewHostContinuityStore(t.TempDir(), nil)
	if err := store.Upsert(config.HostContinuityEntry{
		HostID:       "host-1",
		ReportHostID: "machine-1",
		Hostname:     "pve-1",
		MachineID:    "machine-1",
		LinkedNodeID: "node-1",
		LastSeen:     now,
	}); err != nil {
		t.Fatalf("Upsert continuity: %v", err)
	}

	state := models.NewState()
	state.UpdateNodes([]models.Node{{
		ID:       "node-1",
		Name:     "pve-1",
		Status:   "online",
		LastSeen: now,
	}})

	monitor := &Monitor{
		state:               state,
		hostContinuityStore: store,
	}

	usage := monitor.MonitoredSystemUsage()
	if !usage.Available {
		t.Fatalf("expected continuity-backed usage to be available, got %+v", usage)
	}
	if usage.Count != 1 {
		t.Fatalf("MonitoredSystemUsage().Count = %d, want 1", usage.Count)
	}
}

func TestBroadcastPayloadSlimsStaticMetadata(t *testing.T) {
	sharedCapabilities := []unifiedresources.ResourceCapability{{
		Name:                 "restart",
		Type:                 unifiedresources.CapabilityTypeCommon,
		Description:          "Restart the guest",
		MinimumApprovalLevel: unifiedresources.ApprovalAdmin,
	}}
	resources := []unifiedresources.Resource{
		{
			ID:                     "vm-100",
			Type:                   unifiedresources.ResourceTypeVM,
			Name:                   "web-01",
			Status:                 unifiedresources.StatusOnline,
			LastSeen:               time.Date(2026, 8, 25, 10, 0, 0, 0, time.UTC),
			Sources:                []unifiedresources.DataSource{unifiedresources.SourceProxmox},
			SupersededCanonicalIDs: []string{"vm-superseded-hash-1", "vm-superseded-hash-2"},
			Capabilities:           sharedCapabilities,
		},
		{
			ID:           "vm-101",
			Type:         unifiedresources.ResourceTypeVM,
			Name:         "web-02",
			Status:       unifiedresources.StatusOnline,
			LastSeen:     time.Date(2026, 8, 25, 10, 0, 0, 0, time.UTC),
			Sources:      []unifiedresources.DataSource{unifiedresources.SourceProxmox},
			Capabilities: sharedCapabilities,
		},
		{
			ID:       "storage-1",
			Type:     unifiedresources.ResourceTypeStorage,
			Name:     "tank",
			Status:   unifiedresources.StatusOnline,
			LastSeen: time.Date(2026, 8, 25, 10, 0, 0, 0, time.UTC),
			Sources:  []unifiedresources.DataSource{unifiedresources.SourceProxmox},
		},
	}

	frontend, catalogs := convertResourcesForBroadcast(resources)
	catalog := catalogs.capabilities
	if len(frontend) != 3 {
		t.Fatalf("expected 3 frontend resources, got %d", len(frontend))
	}
	byID := make(map[string]map[string]json.RawMessage, len(frontend))
	refs := make(map[string]string, len(frontend))
	for _, resource := range frontend {
		encoded, err := json.Marshal(resource)
		if err != nil {
			t.Fatalf("marshal %s: %v", resource.ID, err)
		}
		fields := map[string]json.RawMessage{}
		if err := json.Unmarshal(encoded, &fields); err != nil {
			t.Fatalf("unmarshal %s: %v", resource.ID, err)
		}
		byID[resource.ID] = fields
		refs[resource.ID] = resource.CapabilitiesRef
	}

	// Default-posture workloads must not re-ship policy or AI-safe prose.
	vm := byID["vm-100"]
	if _, ok := vm["policy"]; ok {
		t.Fatalf("default-posture vm must omit policy, got %s", vm["policy"])
	}
	if _, ok := vm["aiSafeSummary"]; ok {
		t.Fatalf("default-posture vm must omit aiSafeSummary, got %s", vm["aiSafeSummary"])
	}
	// Sensitive storage keeps its governed posture, deduped through the
	// policy catalog instead of inline duplication.
	storage := byID["storage-1"]
	if _, ok := storage["policy"]; ok {
		t.Fatalf("catalog broadcast must not inline non-default policy, got %s", storage["policy"])
	}
	var storagePolicyRef string
	if err := json.Unmarshal(storage["policyRef"], &storagePolicyRef); err != nil || storagePolicyRef == "" {
		t.Fatalf("sensitive storage must carry policyRef: %v (payload %s)", err, storage["policyRef"])
	}
	policyEntry, ok := catalogs.policies[storagePolicyRef]
	if !ok {
		t.Fatalf("policy catalog missing ref %q (catalog %v)", storagePolicyRef, catalogs.policies)
	}
	var storagePolicy struct {
		Sensitivity string `json:"sensitivity"`
	}
	if err := json.Unmarshal(policyEntry, &storagePolicy); err != nil {
		t.Fatalf("decode policy catalog entry: %v", err)
	}
	if storagePolicy.Sensitivity != "sensitive" {
		t.Fatalf("storage sensitivity = %q", storagePolicy.Sensitivity)
	}
	if summaryRefRaw, ok := storage["aiSafeSummaryRef"]; ok {
		var summaryRef string
		if err := json.Unmarshal(summaryRefRaw, &summaryRef); err != nil || summaryRef == "" {
			t.Fatalf("aiSafeSummaryRef must be a non-empty string: %v", err)
		}
		if _, ok := catalogs.aiSafeSummaries[summaryRef]; !ok {
			t.Fatalf("summary catalog missing ref %q", summaryRef)
		}
	}
	if _, ok := storage["aiSafeSummary"]; ok {
		t.Fatalf("catalog broadcast must not inline aiSafeSummary, got %s", storage["aiSafeSummary"])
	}

	// Superseded ids stay resolvable via supersededIds but are no longer
	// duplicated into the alias list.
	var canonical struct {
		Aliases       []string `json:"aliases"`
		SupersededIDs []string `json:"supersededIds"`
	}
	if err := json.Unmarshal(vm["canonicalIdentity"], &canonical); err != nil {
		t.Fatalf("unmarshal canonicalIdentity: %v", err)
	}
	if len(canonical.SupersededIDs) != 2 {
		t.Fatalf("supersededIds = %v", canonical.SupersededIDs)
	}
	for _, alias := range canonical.Aliases {
		if alias == "vm-superseded-hash-1" || alias == "vm-superseded-hash-2" {
			t.Fatalf("aliases must not duplicate superseded ids, got %v", canonical.Aliases)
		}
	}

	// Identical capability sets collapse onto one catalog entry.
	if refs["vm-100"] == "" || refs["vm-100"] != refs["vm-101"] {
		t.Fatalf("expected shared capabilitiesRef, got %q vs %q", refs["vm-100"], refs["vm-101"])
	}
	if _, ok := vm["capabilities"]; ok {
		t.Fatalf("catalog broadcast must not inline capabilities, got %s", vm["capabilities"])
	}
	entry, ok := catalog[refs["vm-100"]]
	if !ok {
		t.Fatalf("catalog missing ref %q (catalog %v)", refs["vm-100"], catalog)
	}
	var decoded []unifiedresources.ResourceCapability
	if err := json.Unmarshal(entry, &decoded); err != nil {
		t.Fatalf("decode catalog entry: %v", err)
	}
	if len(decoded) != 1 || decoded[0].Name != "restart" {
		t.Fatalf("catalog entry = %+v", decoded)
	}
}