Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

61.4 KB
Raw Blame
package mock

import (
	"bytes"
	"encoding/json"
	"fmt"
	"go/ast"
	"go/parser"
	"go/token"
	"maps"
	"math"
	"os"
	"path/filepath"
	"reflect"
	"slices"
	"strings"
	"testing"
	"time"

	"github.com/rcourtman/pulse-go-rewrite/internal/models"
	"github.com/rcourtman/pulse-go-rewrite/internal/operationaltrust"
	"github.com/rcourtman/pulse-go-rewrite/internal/truenas"
	"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
)

func TestUnifiedResourceSnapshotIncludesPlatformFixtures(t *testing.T) {
	previous := IsMockEnabled()
	mustSetEnabled(t, true)
	t.Cleanup(func() { mustSetEnabled(t, previous) })

	graph := CurrentFixtureGraph()
	legacyName := ""
	if len(graph.State.VMs) > 0 {
		legacyName = graph.State.VMs[0].Name
	} else if len(graph.State.Containers) > 0 {
		legacyName = graph.State.Containers[0].Name
	}
	if legacyName == "" {
		t.Fatal("expected canonical mock graph to include at least one legacy resource name")
	}
	if len(graph.PlatformFixtures.VMware.Hosts) == 0 {
		t.Fatal("expected canonical mock graph to include VMware host fixtures")
	}
	if len(graph.PlatformFixtures.VMware.Networks) == 0 {
		t.Fatal("expected canonical mock graph to include VMware network fixtures")
	}

	resources, freshness := UnifiedResourceSnapshot()
	if len(resources) == 0 {
		t.Fatal("expected unified resources in mock mode")
	}
	if freshness.IsZero() {
		t.Fatal("expected non-zero freshness for mock unified resources")
	}

	wantNames := map[string]bool{
		graph.PlatformFixtures.TrueNAS.System.Hostname: false,
		graph.PlatformFixtures.VMware.Hosts[0].Name:    false,
		graph.PlatformFixtures.VMware.Networks[0].Name: false,
		legacyName: false,
	}
	vmwareNetworkProjected := false
	for _, resource := range resources {
		if _, ok := wantNames[resource.Name]; ok {
			wantNames[resource.Name] = true
		}
		if resource.Name == graph.PlatformFixtures.VMware.Networks[0].Name &&
			resource.Type == unifiedresources.ResourceTypeNetwork &&
			slices.Contains(resource.Sources, unifiedresources.SourceVMware) {
			vmwareNetworkProjected = true
		}
	}
	for name, found := range wantNames {
		if !found {
			t.Fatalf("expected mock unified resources to include %q", name)
		}
	}
	if !vmwareNetworkProjected {
		t.Fatalf(
			"expected VMware fixture network %q to project as a canonical VMware network resource",
			graph.PlatformFixtures.VMware.Networks[0].Name,
		)
	}
}

func TestGenerateDisksForNodeUsesStableRealisticHealthEvidence(t *testing.T) {
	totals := map[string]int{}
	reportedWearout := 0
	unreportedWearout := 0
	lowWearout := 0
	rotationalModels := map[string]struct{}{
		"Seagate BarraCuda 4TB": {},
		"WD Red Pro 8TB":        {},
		"Toshiba X300 6TB":      {},
	}

	for i := 0; i < 300; i++ {
		node := models.Node{
			ID:       fmt.Sprintf("disk-health-node-%d", i),
			Name:     fmt.Sprintf("pve-%d", i),
			Instance: "mock-pve",
		}
		first := generateDisksForNode(node)
		second := generateDisksForNode(node)
		if len(first) != len(second) {
			t.Fatalf("disk count changed for stable node %q: %d then %d", node.ID, len(first), len(second))
		}

		for diskIndex := range first {
			disk := first[diskIndex]
			repeated := second[diskIndex]
			if disk.Health != repeated.Health || disk.Wearout != repeated.Wearout {
				t.Fatalf(
					"health evidence changed for %q disk %d: health=%q wearout=%d, then health=%q wearout=%d",
					node.ID,
					diskIndex,
					disk.Health,
					disk.Wearout,
					repeated.Health,
					repeated.Wearout,
				)
			}

			totals[disk.Health]++
			if disk.Wearout == 0 {
				t.Fatalf("mock disk %q encoded unreported endurance as spent 0%% life", disk.ID)
			}
			if _, rotational := rotationalModels[disk.Model]; rotational && disk.Wearout != -1 {
				t.Fatalf("rotational mock disk %q reports SSD life %d%%", disk.Model, disk.Wearout)
			}
			if disk.Wearout < 0 {
				unreportedWearout++
			} else {
				reportedWearout++
				if disk.Wearout <= 9 {
					lowWearout++
				}
			}
		}
	}

	if totals["PASSED"] <= totals["FAILED"]+totals["UNKNOWN"] {
		t.Fatalf("mock disk health is not predominantly healthy: %+v", totals)
	}
	for _, health := range []string{"PASSED", "UNKNOWN", "FAILED"} {
		if totals[health] == 0 {
			t.Fatalf("mock disk health does not exercise %q: %+v", health, totals)
		}
	}
	if reportedWearout == 0 || unreportedWearout == 0 || lowWearout == 0 {
		t.Fatalf(
			"mock wearout mix missing a required state: reported=%d unreported=%d low=%d",
			reportedWearout,
			unreportedWearout,
			lowWearout,
		)
	}
}

func TestUnifiedResourceSnapshotIncludesRuntimeNativeTabFixtures(t *testing.T) {
	previous := IsMockEnabled()
	mustSetEnabled(t, true)
	t.Cleanup(func() { mustSetEnabled(t, previous) })

	resources, _ := UnifiedResourceSnapshot()
	counts := make(map[unifiedresources.ResourceType]int)
	dockerStorageHosts := 0
	for _, resource := range resources {
		counts[resource.Type]++
		if resource.Docker != nil &&
			(resource.Docker.ImagesUsage != nil ||
				resource.Docker.ContainersUsage != nil ||
				resource.Docker.VolumesUsage != nil ||
				resource.Docker.BuildCacheUsage != nil) {
			dockerStorageHosts++
		}
	}

	requiredTypes := []unifiedresources.ResourceType{
		unifiedresources.ResourceTypeDockerImage,
		unifiedresources.ResourceTypeDockerVolume,
		unifiedresources.ResourceTypeDockerNetwork,
		unifiedresources.ResourceTypeDockerSwarmNode,
		unifiedresources.ResourceTypeK8sStorageClass,
		unifiedresources.ResourceTypeK8sPV,
		unifiedresources.ResourceTypeK8sPVC,
	}
	for _, resourceType := range requiredTypes {
		if counts[resourceType] == 0 {
			t.Fatalf("expected mock unified resources to include %s rows", resourceType)
		}
	}
	if dockerStorageHosts == 0 {
		t.Fatal("expected mock unified resources to include Docker / Podman storage usage on host rows")
	}
}

func TestUnifiedResourceSnapshotParentsDemoProxmoxWorkloads(t *testing.T) {
	previous := IsMockEnabled()
	mustSetEnabled(t, true)
	t.Cleanup(func() { mustSetEnabled(t, previous) })

	resources, _ := UnifiedResourceSnapshot()
	agentsByID := make(map[string]unifiedresources.Resource)
	for _, resource := range resources {
		if resource.Type == unifiedresources.ResourceTypeAgent {
			agentsByID[resource.ID] = resource
		}
	}

	proxmoxWorkloadCount := 0
	for _, resource := range resources {
		if resource.Type != unifiedresources.ResourceTypeVM && resource.Type != unifiedresources.ResourceTypeSystemContainer {
			continue
		}
		if !slices.Contains(resource.Sources, unifiedresources.SourceProxmox) {
			continue
		}
		proxmoxWorkloadCount++
		if resource.ParentID == nil || *resource.ParentID == "" {
			t.Fatalf("expected mock Proxmox workload %q (%s) to have a parent", resource.Name, resource.ID)
		}
		parent, ok := agentsByID[*resource.ParentID]
		if !ok {
			t.Fatalf("expected mock Proxmox workload %q parent %q to resolve to an agent", resource.Name, *resource.ParentID)
		}
		if resource.Proxmox != nil && parent.Proxmox != nil && resource.Proxmox.NodeName != parent.Proxmox.NodeName {
			t.Fatalf(
				"expected mock Proxmox workload %q parent node %q, got %q",
				resource.Name,
				resource.Proxmox.NodeName,
				parent.Proxmox.NodeName,
			)
		}
	}

	if proxmoxWorkloadCount == 0 {
		t.Fatal("expected mock unified resources to include Proxmox workloads")
	}
}

func TestSupplementalRecordsNormalizesVMwareAlias(t *testing.T) {
	previous := IsMockEnabled()
	mustSetEnabled(t, true)
	t.Cleanup(func() { mustSetEnabled(t, previous) })

	records := SupplementalRecords(unifiedresources.DataSource("vmware-vsphere"))
	if len(records) == 0 {
		t.Fatal("expected records for vmware-vsphere alias")
	}
}

// The router's mock adapters outlive mock mode briefly while it swaps the real
// pollers back in; a live refresh that reaches them must find no fixtures.
func TestSupplementalFixturesServeNothingOutsideMockMode(t *testing.T) {
	previous := IsMockEnabled()
	mustSetEnabled(t, false)
	t.Cleanup(func() { mustSetEnabled(t, previous) })

	for _, source := range []unifiedresources.DataSource{
		unifiedresources.SourceTrueNAS,
		unifiedresources.SourceVMware,
		unifiedresources.SourceAvailability,
	} {
		if records := SupplementalRecords(source); len(records) != 0 {
			t.Errorf("SupplementalRecords(%s) served %d fixture records outside mock mode", source, len(records))
		}
		if changes := SupplementalChanges(source); len(changes) != 0 {
			t.Errorf("SupplementalChanges(%s) served %d fixture changes outside mock mode", source, len(changes))
		}
	}
}

func TestSupplementalChangesNormalizesVMwareAlias(t *testing.T) {
	previous := IsMockEnabled()
	mustSetEnabled(t, true)
	t.Cleanup(func() { mustSetEnabled(t, previous) })

	changes := SupplementalChanges(unifiedresources.DataSource("vmware-vsphere"))
	if len(changes) == 0 {
		t.Fatal("expected activity changes for vmware-vsphere alias")
	}
	if changes[0].Kind != unifiedresources.ChangeActivity || changes[0].SourceAdapter != unifiedresources.AdapterVMware {
		t.Fatalf("unexpected VMware activity change: %#v", changes[0])
	}
}

func TestFixtureGraphProjectsAvailabilityFixturesAsNetworkEndpoints(t *testing.T) {
	now := time.Date(2026, time.May, 6, 12, 0, 0, 0, time.UTC)

	graph := buildFixtureGraph(DefaultConfig, now)
	resources, freshness := graph.UnifiedResourceSnapshot()
	if freshness.IsZero() {
		t.Fatal("expected non-zero unified resource freshness")
	}

	var mqtt *unifiedresources.Resource
	var esphome *unifiedresources.Resource
	var door *unifiedresources.Resource
	for i := range resources {
		switch resources[i].Name {
		case "MQTT power meter":
			mqtt = &resources[i]
		case "ESPHome greenhouse sensor":
			esphome = &resources[i]
		case "Workshop door controller":
			door = &resources[i]
		}
	}
	if mqtt == nil {
		t.Fatal("expected MQTT power meter availability endpoint")
	}
	if mqtt.Type != unifiedresources.ResourceTypeNetworkEndpoint {
		t.Fatalf("MQTT resource type = %q, want network-endpoint", mqtt.Type)
	}
	if mqtt.Availability == nil || mqtt.Availability.Protocol != "tcp" || mqtt.Availability.Port != 1883 {
		t.Fatalf("unexpected MQTT availability metadata: %+v", mqtt.Availability)
	}
	if esphome == nil {
		t.Fatal("expected ESPHome greenhouse sensor availability endpoint")
	}
	if esphome.Availability == nil || esphome.Availability.Protocol != "tcp" || esphome.Availability.Port != 6053 {
		t.Fatalf("unexpected ESPHome availability metadata: %+v", esphome.Availability)
	}
	if !slices.Contains(esphome.Tags, "esphome") {
		t.Fatalf("expected ESPHome availability tag, got %+v", esphome.Tags)
	}
	if door == nil {
		t.Fatal("expected workshop door controller availability endpoint")
	}
	if door.Status != unifiedresources.StatusOffline {
		t.Fatalf("door controller status = %q, want offline", door.Status)
	}
	if door.Availability == nil || door.Availability.TargetID != "mock-availability-door-controller" {
		t.Fatalf("unexpected door availability metadata: %+v", door.Availability)
	}
	if len(door.Incidents) != 1 || door.Incidents[0].Code != "availability_unreachable" {
		t.Fatalf("expected availability incident, got %+v", door.Incidents)
	}
	if door.Canonical == nil || door.Canonical.PrimaryID != "availability:mock-availability-door-controller" {
		t.Fatalf("unexpected door canonical identity: %+v", door.Canonical)
	}
}

func TestAvailabilityFixtureRecordOmitsUnknownProbeTimes(t *testing.T) {
	now := time.Date(2026, time.May, 6, 12, 0, 0, 0, time.UTC)
	record, ok := availabilityFixtureRecord(AvailabilityFixture{
		Target: AvailabilityTargetFixture{
			ID:       "not-checked",
			Name:     "Not checked",
			Address:  "not-checked.example",
			Protocol: mockAvailabilityProbeICMP,
			Enabled:  true,
		},
	}, now)
	if !ok || record.Resource.Availability == nil {
		t.Fatalf("availabilityFixtureRecord() = (%+v, %t), want availability record", record, ok)
	}
	if record.Resource.Availability.LastChecked != nil {
		t.Fatalf("last checked = %v, want nil before the first probe", record.Resource.Availability.LastChecked)
	}
	if record.Resource.Availability.LastSuccess != nil {
		t.Fatalf("last success = %v, want nil before the first successful probe", record.Resource.Availability.LastSuccess)
	}
	if record.Resource.Availability.Evidence == nil ||
		record.Resource.Availability.Evidence.Completeness != operationaltrust.EvidencePartial {
		t.Fatalf("initial evidence = %+v, want partial envelope", record.Resource.Availability.Evidence)
	}
}

func TestFixtureGraphAttachesServiceAvailabilityFixturesToServiceResources(t *testing.T) {
	now := time.Date(2026, time.May, 6, 12, 0, 0, 0, time.UTC)

	graph := buildFixtureGraph(DefaultConfig, now)
	resources, _ := graph.UnifiedResourceSnapshot()

	// A configured check does not collapse into the resource it matches. It
	// keeps its own source-owned network-endpoint row, which owns probe status,
	// incidents and history, and additively projects a facet onto the matched
	// resource. Both therefore carry the same TargetID, so the matched service
	// has to be selected by resource type rather than by target alone.
	var dockerService *unifiedresources.Resource
	var kubernetesService *unifiedresources.Resource
	var dockerEndpoint *unifiedresources.Resource
	var kubernetesEndpoint *unifiedresources.Resource
	for i := range resources {
		availability := resources[i].Availability
		if availability == nil {
			continue
		}
		isEndpoint := resources[i].Type == unifiedresources.ResourceTypeNetworkEndpoint
		switch availability.TargetID {
		case "mock-availability-docker-frontend-service":
			if isEndpoint {
				dockerEndpoint = &resources[i]
			} else {
				dockerService = &resources[i]
			}
		case "mock-availability-k8s-checkout-api":
			if isEndpoint {
				kubernetesEndpoint = &resources[i]
			} else {
				kubernetesService = &resources[i]
			}
		}
	}

	if dockerEndpoint == nil {
		t.Fatal("Docker service check lost its source-owned network-endpoint row")
	}
	if kubernetesEndpoint == nil {
		t.Fatal("Kubernetes service check lost its source-owned network-endpoint row")
	}

	if dockerService == nil {
		t.Fatal("expected Docker service availability facet on curated mock service")
	}
	if dockerService.Type != unifiedresources.ResourceTypeDockerService {
		t.Fatalf("Docker service availability attached to %q, want docker-service", dockerService.Type)
	}
	if dockerService.Docker == nil || dockerService.Docker.ServiceID != "svc-frontend-0" {
		t.Fatalf("Docker service metadata = %+v, want service id svc-frontend-0", dockerService.Docker)
	}
	if dockerService.Availability.TargetKind != "service" ||
		dockerService.Availability.Protocol != "https" ||
		dockerService.Availability.Path != "/health" ||
		dockerService.Availability.LatencyMillis != 9 {
		t.Fatalf("unexpected Docker service availability facet: %+v", dockerService.Availability)
	}
	if !slices.Contains(dockerService.Sources, unifiedresources.SourceAvailability) {
		t.Fatalf("expected Docker service sources to include availability, got %+v", dockerService.Sources)
	}
	if dockerService.Availability.Certificate == nil ||
		dockerService.Availability.Certificate.TrustStatus != "trusted" ||
		dockerService.Availability.CertificateExpiryWarningDays != 30 {
		t.Fatalf("Docker service certificate posture = %+v", dockerService.Availability)
	}
	if dockerService.Availability.CorrelationState != unifiedresources.AvailabilityCorrelationAttached ||
		dockerService.Availability.Evidence == nil ||
		dockerService.Availability.Evidence.Subject.ResourceID != dockerService.ID {
		t.Fatalf("Docker service availability trust contract = %+v", dockerService.Availability)
	}
	if !hasChecksEdgeTo(dockerEndpoint, dockerService.ID) {
		t.Fatalf("Docker check relationships = %+v, want outgoing checks edge to %s", dockerEndpoint.Relationships, dockerService.ID)
	}

	if kubernetesService == nil {
		t.Fatal("expected Kubernetes service availability facet on curated mock service")
	}
	if kubernetesService.Type != unifiedresources.ResourceTypeK8sService {
		t.Fatalf("Kubernetes service availability attached to %q, want k8s-service", kubernetesService.Type)
	}
	if kubernetesService.Name != "checkout-api" {
		t.Fatalf("Kubernetes service name = %q, want checkout-api", kubernetesService.Name)
	}
	if kubernetesService.Availability.TargetKind != "service" ||
		kubernetesService.Availability.Protocol != "tcp" ||
		kubernetesService.Availability.Port != 8080 ||
		kubernetesService.Availability.LatencyMillis != 5 {
		t.Fatalf("unexpected Kubernetes service availability facet: %+v", kubernetesService.Availability)
	}
	if !slices.Contains(kubernetesService.Sources, unifiedresources.SourceAvailability) {
		t.Fatalf("expected Kubernetes service sources to include availability, got %+v", kubernetesService.Sources)
	}
	if !hasChecksEdgeTo(kubernetesEndpoint, kubernetesService.ID) {
		t.Fatalf("Kubernetes check relationships = %+v, want outgoing checks edge to %s", kubernetesEndpoint.Relationships, kubernetesService.ID)
	}
}

// TestFixtureGraphAppliesManualLinksAtTheRebuildsBoundary pins the linked
// build behind the mock-mode unified view to the live rebuild's link boundary
// (MonitorAdapter.replaceRegistryLocked): the links apply once the snapshot
// and every supplemental source but availability are in, and before
// availability. Two things hang on it. Availability goes in after the pass, so
// folding the curated Docker service into a VM moves its availability check
// onto the VM, as the live rebuild's availability ingest resolves the check's
// source reference through the fold; folding the finished list dropped the
// facet with the folded row. And a chain of links is judged over the assembled
// estate, so mock mode keeps the row the live rebuild keeps instead of the one
// a pass per source happens to leave.
func TestFixtureGraphAppliesManualLinksAtTheRebuildsBoundary(t *testing.T) {
	now := time.Date(2026, time.May, 6, 12, 0, 0, 0, time.UTC)
	graph := buildFixtureGraph(DefaultConfig, now)
	unlinked, _ := graph.UnifiedResourceSnapshot()

	t.Run("an availability check follows the fold onto the primary", func(t *testing.T) {
		const checkID = "mock-availability-docker-frontend-service"
		var serviceID, vmID string
		for _, resource := range unlinked {
			switch {
			case resource.Type == unifiedresources.ResourceTypeDockerService && resource.Availability != nil &&
				resource.Availability.TargetID == checkID:
				serviceID = resource.ID
			case vmID == "" && resource.Type == unifiedresources.ResourceTypeVM && resource.Proxmox != nil:
				vmID = resource.ID
			}
		}
		if serviceID == "" || vmID == "" {
			t.Fatalf("fixture graph needs the checked Docker service and a Proxmox VM, got service=%q vm=%q", serviceID, vmID)
		}

		links := []unifiedresources.ResourceLink{{ResourceA: vmID, ResourceB: serviceID, PrimaryID: vmID}}
		linked, _ := graph.unifiedResourceSnapshot(unifiedresources.NewRegistryWithManualLinks(links))
		var vm, endpoint *unifiedresources.Resource
		for i := range linked {
			switch {
			case linked[i].ID == serviceID:
				t.Fatalf("linked build still lists the folded Docker service %s", serviceID)
			case linked[i].ID == vmID:
				vm = &linked[i]
			case linked[i].Type == unifiedresources.ResourceTypeNetworkEndpoint && linked[i].Availability != nil &&
				linked[i].Availability.TargetID == checkID:
				endpoint = &linked[i]
			}
		}
		if vm == nil || vm.Docker == nil {
			t.Fatalf("linked VM %s = %+v, want it to carry the folded service's Docker facet", vmID, vm)
		}
		if vm.Availability == nil || vm.Availability.TargetID != checkID {
			t.Fatalf("linked VM availability = %+v, want the folded service's check projected onto it", vm.Availability)
		}
		if !hasChecksEdgeTo(endpoint, vmID) {
			t.Fatalf("check relationships = %+v, want the checks edge to follow the fold to %s", endpoint, vmID)
		}
	})

	// A Proxmox storage, a TrueNAS VM and a vSphere VM linked in a cycle keep
	// the TrueNAS VM, whose link is the oldest. A pass after the TrueNAS
	// records has already folded that VM into the storage when the vSphere
	// records bring a guest that outranks the storage, so folding as the
	// sources arrive keeps the vSphere VM instead.
	t.Run("a chain of links is judged over the assembled estate", func(t *testing.T) {
		soleSource := func(resource unifiedresources.Resource, source unifiedresources.DataSource) bool {
			return len(resource.Sources) == 1 && resource.Sources[0] == source
		}
		var storageID, nasVMID, guestID string
		for _, resource := range unlinked {
			switch {
			case storageID == "" && resource.Type == unifiedresources.ResourceTypeStorage &&
				soleSource(resource, unifiedresources.SourceProxmox):
				storageID = resource.ID
			case nasVMID == "" && resource.Type == unifiedresources.ResourceTypeVM &&
				soleSource(resource, unifiedresources.SourceTrueNAS):
				nasVMID = resource.ID
			case guestID == "" && resource.Type == unifiedresources.ResourceTypeVM &&
				soleSource(resource, unifiedresources.SourceVMware):
				guestID = resource.ID
			}
		}
		if storageID == "" || nasVMID == "" || guestID == "" {
			t.Fatalf("fixture graph needs a Proxmox storage, a TrueNAS VM and a vSphere VM, got storage=%q nas=%q guest=%q",
				storageID, nasVMID, guestID)
		}
		created := time.Date(2026, time.October, 1, 9, 0, 0, 0, time.UTC)
		links := []unifiedresources.ResourceLink{
			{ResourceA: nasVMID, ResourceB: storageID, PrimaryID: storageID, CreatedAt: created.Add(time.Minute)},
			{ResourceA: storageID, ResourceB: guestID, PrimaryID: guestID, CreatedAt: created.Add(2 * time.Minute)},
			{ResourceA: guestID, ResourceB: nasVMID, PrimaryID: nasVMID, CreatedAt: created},
		}
		// cycleRows is what is left of the three members: type, ID and sources.
		cycleRows := func(resources []unifiedresources.Resource) []string {
			var rows []string
			for _, resource := range resources {
				if resource.ID != storageID && resource.ID != nasVMID && resource.ID != guestID {
					continue
				}
				sources := make([]string, 0, len(resource.Sources))
				for _, source := range resource.Sources {
					sources = append(sources, string(source))
				}
				slices.Sort(sources)
				rows = append(rows, fmt.Sprintf("%s %s %v", resource.Type, resource.ID, sources))
			}
			return rows
		}
		owned := SupplementalOwnedSources()
		snapshot := unifiedresources.SnapshotWithoutSources(graph.State, owned)
		records := make(map[unifiedresources.DataSource][]unifiedresources.IngestRecord, len(owned))
		for _, source := range owned {
			records[source] = graph.SupplementalRecords(source)
		}

		// The monitor's rebuild is the reference the mock build must match.
		store := unifiedresources.NewMemoryStore()
		for _, link := range links {
			if err := store.AddLink(link); err != nil {
				t.Fatalf("add link: %v", err)
			}
		}
		adapter := unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(store))
		adapter.PopulateSnapshotAndSupplemental(snapshot, records)
		live := cycleRows(adapter.GetAll())
		if len(live) != 1 || !strings.HasPrefix(live[0], "vm "+nasVMID+" ") {
			t.Fatalf("live rebuild kept %v, want the TrueNAS VM %s alone", live, nasVMID)
		}

		// A registry that ingests the same sources one call at a time judges
		// the chain at each call. It must settle elsewhere, or this fixture no
		// longer separates the boundary from per-call passes.
		perCall := unifiedresources.NewRegistryWithManualLinks(links)
		perCall.IngestSnapshot(snapshot)
		for _, source := range owned {
			if len(records[source]) > 0 {
				perCall.IngestRecords(source, records[source])
			}
		}
		if got := cycleRows(perCall.List()); slices.Equal(got, live) {
			t.Fatalf("a pass per source also kept %v, so this fixture no longer separates the rebuild's boundary from per-call passes", got)
		}

		linked, _ := graph.unifiedResourceSnapshot(unifiedresources.NewRegistryWithManualLinks(links))
		if got := cycleRows(linked); !slices.Equal(got, live) {
			t.Fatalf("mock build kept %v, want what the live rebuild keeps: %v", got, live)
		}
	})
}

// TestUnifiedResourceSnapshotWithLinksLeavesTheSharedSnapshotUnlinked pins the
// memo boundary: a linked build belongs to its caller (one tenant's links),
// and the memoized snapshot every other reader shares stays unlinked.
func TestUnifiedResourceSnapshotWithLinksLeavesTheSharedSnapshotUnlinked(t *testing.T) {
	previous := IsMockEnabled()
	previousConfig := GetConfig()
	if previous {
		mustSetEnabled(t, false)
	}
	testConfig := previousConfig
	testConfig.UpdateInterval = 5 * time.Minute
	SetMockConfig(testConfig)
	mustSetEnabled(t, true)
	t.Cleanup(func() {
		mustSetEnabled(t, false)
		SetMockConfig(previousConfig)
		mustSetEnabled(t, previous)
	})

	shared, _ := UnifiedResourceSnapshot()
	if same, _ := UnifiedResourceSnapshotWithLinks(nil); len(same) == 0 || &same[0] != &shared[0] {
		t.Fatal("UnifiedResourceSnapshotWithLinks without links must serve the memoized snapshot")
	}
	var vmID, agentID string
	for _, resource := range shared {
		switch {
		case vmID == "" && resource.Type == unifiedresources.ResourceTypeVM && resource.Agent == nil:
			vmID = resource.ID
		case agentID == "" && resource.Type == unifiedresources.ResourceTypeAgent && resource.Agent != nil &&
			len(resource.Sources) == 1 && resource.Sources[0] == unifiedresources.SourceAgent:
			agentID = resource.ID
		}
	}
	if vmID == "" || agentID == "" {
		t.Fatalf("fixture graph needs an agentless VM and a standalone agent, got vm=%q agent=%q", vmID, agentID)
	}

	linked, _ := UnifiedResourceSnapshotWithLinks([]unifiedresources.ResourceLink{{ResourceA: vmID, ResourceB: agentID, PrimaryID: vmID}})
	if slices.ContainsFunc(linked, func(r unifiedresources.Resource) bool { return r.ID == agentID }) {
		t.Fatalf("linked build still lists the folded agent %s", agentID)
	}
	again, _ := UnifiedResourceSnapshot()
	if !slices.ContainsFunc(again, func(r unifiedresources.Resource) bool { return r.ID == agentID }) {
		t.Fatalf("the shared snapshot lost the agent %s after a linked build", agentID)
	}
}

// hasChecksEdgeTo reports whether the source-owned check row carries the
// outgoing checks relationship to the resource it matched. The check row owns
// that edge; the matched resource carries only the projected facet.
func hasChecksEdgeTo(check *unifiedresources.Resource, matchedID string) bool {
	if check == nil {
		return false
	}
	for _, relationship := range check.Relationships {
		if relationship.Type == unifiedresources.RelChecks && relationship.TargetID == matchedID {
			return true
		}
	}
	return false
}

func TestBuildFixtureGraphRebasesPlatformFixtureTimestampsForDemoRuntime(t *testing.T) {
	now := time.Date(2026, time.March, 31, 17, 30, 0, 0, time.UTC)

	graph := buildFixtureGraph(DefaultConfig, now)

	if got := trueNASCollectedAt(graph.PlatformFixtures.TrueNAS); !got.Equal(now) {
		t.Fatalf("expected TrueNAS collectedAt %s, got %s", now, got)
	}
	if got := graph.PlatformFixtures.VMware.CollectedAt; !got.Equal(now) {
		t.Fatalf("expected VMware collectedAt %s, got %s", now, got)
	}
	if got := availabilityFixturesFreshness(graph.AvailabilityFixtures); got.IsZero() || got.Before(now.Add(-2*time.Minute)) || got.After(now) {
		t.Fatalf("expected availability fixture freshness near %s, got %s", now, got)
	}
	if got := graph.PlatformFixtures.TrueNAS.System.CollectedAt; got.IsZero() || got.Before(now.Add(-2*time.Minute)) || got.After(now) {
		t.Fatalf("expected rebased TrueNAS system collectedAt near %s, got %s", now, got)
	}
	if len(graph.PlatformFixtures.VMware.Hosts) == 0 || len(graph.PlatformFixtures.VMware.Hosts[0].RecentEvents) == 0 {
		t.Fatal("expected canonical VMware fixtures with recent events")
	}
	if got := graph.PlatformFixtures.VMware.Hosts[0].RecentEvents[0].CreatedAt; got.IsZero() || got.Before(now.Add(-2*time.Hour)) || got.After(now) {
		t.Fatalf("expected rebased VMware event timestamp near %s, got %s", now, got)
	}
}

func TestFixtureGraphUpdateMetricsKeepsPlatformFixtureFreshnessCurrent(t *testing.T) {
	cfg := DefaultConfig
	cfg.RandomMetrics = false

	start := time.Date(2026, time.March, 31, 17, 30, 0, 0, time.UTC)
	later := start.Add(12 * time.Minute)

	graph := buildFixtureGraph(cfg, start)
	graph.UpdateMetrics(cfg, later)

	if got := trueNASCollectedAt(graph.PlatformFixtures.TrueNAS); !got.Equal(later) {
		t.Fatalf("expected rebased TrueNAS collectedAt %s, got %s", later, got)
	}
	if got := graph.PlatformFixtures.VMware.CollectedAt; !got.Equal(later) {
		t.Fatalf("expected rebased VMware collectedAt %s, got %s", later, got)
	}
	if got := availabilityFixturesFreshness(graph.AvailabilityFixtures); got.IsZero() || got.Before(later.Add(-2*time.Minute)) || got.After(later) {
		t.Fatalf("expected availability fixture freshness near %s, got %s", later, got)
	}
	if len(graph.PlatformFixtures.VMware.Hosts) == 0 || len(graph.PlatformFixtures.VMware.Hosts[0].RecentEvents) == 0 {
		t.Fatal("expected canonical VMware fixtures with host events")
	}
	if got := graph.PlatformFixtures.VMware.Hosts[0].RecentEvents[0].CreatedAt; got.Before(later.Add(-2*time.Hour)) || got.After(later) {
		t.Fatalf("expected VMware event timestamp to remain fresh near %s, got %s", later, got)
	}
}

func TestBuildFixtureGraphRefreshesPlatformFixtureMetricsFromCanonicalModel(t *testing.T) {
	now := time.Date(2026, time.March, 31, 17, 30, 45, 0, time.UTC)

	graph := buildFixtureGraph(DefaultConfig, now)

	system := graph.PlatformFixtures.TrueNAS.System
	if got, want := system.CPUPercent, SampleMetric("agent", system.Hostname, "cpu", now); math.Abs(got-want) > 1e-9 {
		t.Fatalf("expected refreshed TrueNAS system cpu %.6f, got %.6f", want, got)
	}

	pool := graph.PlatformFixtures.TrueNAS.Pools[0]
	poolID := TrueNASPoolMetricID(system.Hostname, pool.Name)
	if got, want := pool.UsedBytes, bytesFromPercent(pool.TotalBytes, SampleMetric("storage", poolID, "usage", now)); got != want {
		t.Fatalf("expected refreshed TrueNAS pool used bytes %d, got %d", want, got)
	}

	dataset := graph.PlatformFixtures.TrueNAS.Datasets[0]
	datasetID := TrueNASDatasetMetricID(system.Hostname, dataset.Name)
	datasetTotal := dataset.UsedBytes + dataset.AvailBytes
	if got, want := dataset.UsedBytes, bytesFromPercent(datasetTotal, SampleMetric("storage", datasetID, "usage", now)); got != want {
		t.Fatalf("expected refreshed TrueNAS dataset used bytes %d, got %d", want, got)
	}

	app := graph.PlatformFixtures.TrueNAS.Apps[1]
	if app.Stats == nil {
		t.Fatal("expected refreshed TrueNAS app stats")
	}
	appID := TrueNASAppMetricID(system.Hostname, app)
	if got, want := app.Stats.CPUPercent, SampleMetric("dockerContainer", appID, "cpu", now); math.Abs(got-want) > 1e-9 {
		t.Fatalf("expected refreshed TrueNAS app cpu %.6f, got %.6f", want, got)
	}
	if got, want := app.Stats.MemoryBytes, bytesFromPercent(system.MemoryTotalBytes, SampleMetric("dockerContainer", appID, "memory", now)); got != want {
		t.Fatalf("expected refreshed TrueNAS app memory bytes %d, got %d", want, got)
	}

	disk := graph.PlatformFixtures.TrueNAS.Disks[0]
	if got, want := disk.Temperature, int(math.Round(SampleMetric("disk", disk.Serial, "smart_temp", now))); got != want {
		t.Fatalf("expected refreshed TrueNAS disk temperature %d, got %d", want, got)
	}

	host := graph.PlatformFixtures.VMware.Hosts[0]
	if host.Metrics == nil || host.Metrics.CPUPercent == nil {
		t.Fatal("expected refreshed VMware host metrics")
	}
	if got, want := *host.Metrics.CPUPercent, SampleMetric("agent", "vc-mock-1:host:host-101", "cpu", now); math.Abs(got-want) > 1e-9 {
		t.Fatalf("expected refreshed VMware host cpu %.6f, got %.6f", want, got)
	}
	// Mock fixture must also synthesize uptime so the workloads table
	// renders real "N days" cells for vSphere hosts and VMs instead of
	// blank "0s" placeholders. Hosts get sys.uptime-style uptime only;
	// guest disk usage is VM-only because ESXi exposes no `guest` shape.
	if host.Metrics.UptimeSeconds == nil || *host.Metrics.UptimeSeconds <= 0 {
		t.Fatalf("expected refreshed VMware host uptime seconds, got %+v", host.Metrics.UptimeSeconds)
	}
	if host.Metrics.DiskTotalBytes != nil || host.Metrics.DiskUsedBytes != nil || host.Metrics.DiskPercent != nil {
		t.Fatalf("expected VMware host guest filesystem fields to stay nil, got total=%+v used=%+v percent=%+v", host.Metrics.DiskTotalBytes, host.Metrics.DiskUsedBytes, host.Metrics.DiskPercent)
	}

	var poweredOnVM *struct{}
	for _, vm := range graph.PlatformFixtures.VMware.VMs {
		if vm.PowerState != "POWERED_ON" || vm.Metrics == nil {
			continue
		}
		if vm.Metrics.UptimeSeconds == nil || *vm.Metrics.UptimeSeconds <= 0 {
			t.Fatalf("expected powered-on VMware VM %q to surface uptime, got %+v", vm.Name, vm.Metrics.UptimeSeconds)
		}
		if vm.Metrics.DiskTotalBytes == nil || *vm.Metrics.DiskTotalBytes <= 0 {
			t.Fatalf("expected powered-on VMware VM %q to surface guest disk total, got %+v", vm.Name, vm.Metrics.DiskTotalBytes)
		}
		if vm.Metrics.DiskUsedBytes == nil || *vm.Metrics.DiskUsedBytes < 0 {
			t.Fatalf("expected powered-on VMware VM %q to surface guest disk used, got %+v", vm.Name, vm.Metrics.DiskUsedBytes)
		}
		if vm.Metrics.DiskPercent == nil {
			t.Fatalf("expected powered-on VMware VM %q to surface guest disk percent", vm.Name)
		}
		marker := struct{}{}
		poweredOnVM = &marker
		break
	}
	if poweredOnVM == nil {
		t.Fatal("expected at least one powered-on VMware VM fixture to assert uptime + guest disk projection")
	}

	for _, vm := range graph.PlatformFixtures.VMware.VMs {
		if vm.PowerState != "POWERED_OFF" || vm.Metrics == nil {
			continue
		}
		if vm.Metrics.UptimeSeconds != nil {
			t.Fatalf("expected powered-off VMware VM %q to drop uptime, got %+v", vm.Name, vm.Metrics.UptimeSeconds)
		}
		if vm.Metrics.DiskTotalBytes != nil || vm.Metrics.DiskUsedBytes != nil || vm.Metrics.DiskPercent != nil {
			t.Fatalf("expected powered-off VMware VM %q to drop guest disk fields, got total=%+v used=%+v percent=%+v", vm.Name, vm.Metrics.DiskTotalBytes, vm.Metrics.DiskUsedBytes, vm.Metrics.DiskPercent)
		}
		break
	}

	datastore := graph.PlatformFixtures.VMware.Datastores[0]
	wantFree := datastore.Capacity - bytesFromPercent(datastore.Capacity, SampleMetric("storage", "vc-mock-1:datastore:"+datastore.Datastore, "usage", now))
	if datastore.FreeSpace != wantFree {
		t.Fatalf("expected refreshed VMware datastore free space %d, got %d", wantFree, datastore.FreeSpace)
	}
}

func TestBuildFixtureGraphRefreshesStateMetricsFromCanonicalModel(t *testing.T) {
	now := time.Date(2026, time.March, 31, 17, 30, 45, 0, time.UTC)

	graph := buildFixtureGraph(DefaultConfig, now)

	node := graph.State.Nodes[0]
	if got, want := node.CPU*100, SampleMetric("node", node.ID, "cpu", now); math.Abs(got-want) > 1e-9 {
		t.Fatalf("expected refreshed node cpu %.6f, got %.6f", want, got)
	}
	if got, want := node.Memory.Usage, SampleMetric("node", node.ID, "memory", now); math.Abs(got-want) > 1e-9 {
		t.Fatalf("expected refreshed node memory %.6f, got %.6f", want, got)
	}

	var runningVMID string
	var runningVMCPU float64
	var runningVMMemory float64
	for _, vm := range graph.State.VMs {
		if vm.Status != "running" {
			continue
		}
		runningVMID = vm.ID
		runningVMCPU = vm.CPU * 100
		runningVMMemory = vm.Memory.Usage
		break
	}
	if runningVMID == "" {
		t.Fatal("expected at least one running VM in canonical fixture graph")
	}
	if got, want := runningVMCPU, SampleMetric("vm", runningVMID, "cpu", now); math.Abs(got-want) > 1e-9 {
		t.Fatalf("expected refreshed vm cpu %.6f, got %.6f", want, got)
	}
	if got, want := runningVMMemory, SampleMetric("vm", runningVMID, "memory", now); math.Abs(got-want) > 1e-9 {
		t.Fatalf("expected refreshed vm memory %.6f, got %.6f", want, got)
	}

	var onlineHostID string
	var onlineHostCPU float64
	var onlineHostMemory float64
	for _, host := range graph.State.Hosts {
		if host.Status == "offline" {
			continue
		}
		onlineHostID = host.ID
		onlineHostCPU = host.CPUUsage
		onlineHostMemory = host.Memory.Usage
		break
	}
	if onlineHostID == "" {
		t.Fatal("expected at least one online host in canonical fixture graph")
	}
	if got, want := onlineHostCPU, SampleMetric("agent", onlineHostID, "cpu", now); math.Abs(got-want) > 1e-9 {
		t.Fatalf("expected refreshed host cpu %.6f, got %.6f", want, got)
	}
	if got, want := onlineHostMemory, SampleMetric("agent", onlineHostID, "memory", now); math.Abs(got-want) > 1e-9 {
		t.Fatalf("expected refreshed host memory %.6f, got %.6f", want, got)
	}

	storage := graph.State.Storage[0]
	if got, want := storage.Usage, SampleMetric("storage", storage.ID, "usage", now); math.Abs(got-want) > 1e-9 {
		t.Fatalf("expected refreshed storage usage %.6f, got %.6f", want, got)
	}

	disk := graph.State.PhysicalDisks[0]
	resourceID := disk.Serial
	if resourceID == "" {
		resourceID = disk.ID
	}
	if resourceID == "" {
		resourceID = fmt.Sprintf("%s-%s-%s", disk.Instance, disk.Node, disk.DevPath)
	}
	if got, want := disk.Temperature, int(math.Round(SampleMetric("disk", resourceID, "smart_temp", now))); got != want {
		t.Fatalf("expected refreshed physical disk temperature %d, got %d", want, got)
	}
}

func TestCloneTrueNASFixtureSnapshotIsolatesPoolHealthEvidence(t *testing.T) {
	original := truenas.FixtureSnapshot{Pools: []truenas.Pool{{
		Name: "tank",
		Scan: &truenas.PoolScan{
			Function: "RESILVER",
			State:    "SCANNING",
		},
		VDevs: []truenas.PoolVDev{{
			GUID:   "leaf-guid",
			Status: "UNAVAIL",
		}},
		DiskMembers: []truenas.PoolDiskMember{{
			GUID:   "leaf-guid",
			Status: "UNAVAIL",
		}},
	}}}

	cloned := cloneTrueNASFixtureSnapshot(original)
	cloned.Pools[0].Scan.State = "FINISHED"
	cloned.Pools[0].VDevs[0].Status = "ONLINE"
	cloned.Pools[0].DiskMembers[0].Status = "ONLINE"

	if original.Pools[0].Scan.State != "SCANNING" ||
		original.Pools[0].VDevs[0].Status != "UNAVAIL" ||
		original.Pools[0].DiskMembers[0].Status != "UNAVAIL" {
		t.Fatalf("clone mutated original pool evidence: %+v", original.Pools[0])
	}
}

func TestFixtureGraphMetricCohortsBoundPVEChurnAndCoverTheEstate(t *testing.T) {
	cfg := DefaultConfig
	cfg.NodeCount = 10
	cfg.VMsPerNode = 2
	cfg.LXCsPerNode = 2
	cfg.DockerHostCount = 0
	cfg.GenericHostCount = 0
	cfg.K8sClusterCount = 0
	cfg.RandomMetrics = true

	base := time.Date(2026, 8, 23, 12, 0, 0, 0, time.UTC)
	graph := buildFixtureGraph(cfg, base)
	initialVMSeen := make(map[string]time.Time, len(graph.State.VMs))
	initialContainerSeen := make(map[string]time.Time, len(graph.State.Containers))
	for _, vm := range graph.State.VMs {
		initialVMSeen[vm.ID] = vm.LastSeen
	}
	for _, container := range graph.State.Containers {
		initialContainerSeen[container.ID] = container.LastSeen
	}

	graph.UpdateMetricCohort(cfg, base.Add(cfg.UpdateInterval), 0, 5)
	selectedNodes := map[string]struct{}{
		graph.State.Nodes[0].Name: {},
		graph.State.Nodes[5].Name: {},
	}
	changedGuests := 0
	for _, vm := range graph.State.VMs {
		_, selected := selectedNodes[vm.Node]
		changed := !vm.LastSeen.Equal(initialVMSeen[vm.ID])
		if changed {
			changedGuests++
		}
		if changed != selected {
			t.Fatalf("VM %s on %s changed=%v, selected=%v", vm.ID, vm.Node, changed, selected)
		}
	}
	for _, container := range graph.State.Containers {
		_, selected := selectedNodes[container.Node]
		changed := !container.LastSeen.Equal(initialContainerSeen[container.ID])
		if changed {
			changedGuests++
		}
		if changed != selected {
			t.Fatalf("container %s on %s changed=%v, selected=%v", container.ID, container.Node, changed, selected)
		}
	}
	if total := len(graph.State.VMs) + len(graph.State.Containers); changedGuests <= 0 || changedGuests >= total/2 {
		t.Fatalf("expected a bounded non-empty guest cohort, changed=%d total=%d", changedGuests, total)
	}

	for cohort := 1; cohort < 5; cohort++ {
		graph.UpdateMetricCohort(cfg, base.Add(time.Duration(cohort+1)*cfg.UpdateInterval), cohort, 5)
	}
	for _, vm := range graph.State.VMs {
		if !vm.LastSeen.After(initialVMSeen[vm.ID]) {
			t.Fatalf("VM %s was not refreshed during a full cohort rotation", vm.ID)
		}
	}
	for _, container := range graph.State.Containers {
		if !container.LastSeen.After(initialContainerSeen[container.ID]) {
			t.Fatalf("container %s was not refreshed during a full cohort rotation", container.ID)
		}
	}
}

func TestSupplementalRefreshCadenceTracksFreshnessBudget(t *testing.T) {
	t.Cleanup(func() { setMockUpdateInterval(DefaultConfig.UpdateInterval) })

	cases := []struct {
		name         string
		interval     time.Duration
		cohortCount  int
		everyTick    bool
		wantInterval time.Duration
	}{
		{"default rotation stays within budget", 2 * time.Second, mockMetricCohortCount, false, 20 * time.Second},
		{"rotation at the budget keeps the cohort cadence", 12 * time.Second, mockMetricCohortCount, false, 120 * time.Second},
		{"rotation past the budget refreshes per tick", 13 * time.Second, mockMetricCohortCount, true, 13 * time.Second},
		{"public demo interval refreshes per tick", 60 * time.Second, mockMetricCohortCount, true, 60 * time.Second},
		{"degenerate cohort count refreshes per tick", 2 * time.Second, 1, true, 2 * time.Second},
	}
	for _, tc := range cases {
		t.Run(tc.name, func(t *testing.T) {
			if got := supplementalRefreshEveryTick(tc.interval, tc.cohortCount); got != tc.everyTick {
				t.Fatalf("supplementalRefreshEveryTick(%s, %d) = %v, want %v", tc.interval, tc.cohortCount, got, tc.everyTick)
			}
			if tc.cohortCount == mockMetricCohortCount {
				setMockUpdateInterval(tc.interval)
				if got := SupplementalRefreshInterval(); got != tc.wantInterval {
					t.Fatalf("SupplementalRefreshInterval() = %s at %s ticks, want %s", got, tc.interval, tc.wantInterval)
				}
			}
		})
	}
}

func TestProxmoxRefreshCadenceTracksSightingBudget(t *testing.T) {
	t.Cleanup(func() { setMockUpdateInterval(DefaultConfig.UpdateInterval) })

	cases := []struct {
		name         string
		interval     time.Duration
		wantCohorts  int
		wantInterval time.Duration
	}{
		{"default rotation keeps every cohort", 2 * time.Second, mockMetricCohortCount, 20 * time.Second},
		{"rotation at the budget keeps every cohort", 3 * time.Second, mockMetricCohortCount, 30 * time.Second},
		{"rotation past the budget drops to a divisor", 4 * time.Second, 5, 20 * time.Second},
		{"public demo interval rotates two cohorts", 15 * time.Second, 2, 30 * time.Second},
		{"tick past half the budget sights every node per tick", 16 * time.Second, 1, 16 * time.Second},
		{"tick past the budget sights every node per tick", 60 * time.Second, 1, 60 * time.Second},
	}
	for _, tc := range cases {
		t.Run(tc.name, func(t *testing.T) {
			got := proxmoxCohortCount(tc.interval, mockMetricCohortCount)
			if got != tc.wantCohorts {
				t.Fatalf("proxmoxCohortCount(%s, %d) = %d, want %d", tc.interval, mockMetricCohortCount, got, tc.wantCohorts)
			}
			if mockMetricCohortCount%got != 0 {
				t.Fatalf("proxmoxCohortCount(%s) = %d does not divide %d", tc.interval, got, mockMetricCohortCount)
			}
			setMockUpdateInterval(tc.interval)
			if got := ProxmoxRefreshInterval(); got != tc.wantInterval {
				t.Fatalf("ProxmoxRefreshInterval() = %s at %s ticks, want %s", got, tc.interval, tc.wantInterval)
			}
		})
	}
}

func TestFixtureGraphSlowTicksKeepSupplementalFixturesFresh(t *testing.T) {
	t.Cleanup(func() { setMockUpdateInterval(DefaultConfig.UpdateInterval) })

	cfg := DefaultConfig
	cfg.NodeCount = 4
	cfg.VMsPerNode = 1
	cfg.LXCsPerNode = 1
	cfg.DockerHostCount = 0
	cfg.GenericHostCount = 0
	cfg.K8sClusterCount = 0

	base := time.Date(2026, 8, 23, 12, 0, 0, 0, time.UTC)

	// Fast ticks: a non-zero cohort must keep the once-per-rotation cadence.
	graph := buildFixtureGraph(cfg, base)
	collectedAt := trueNASCollectedAt(graph.PlatformFixtures.TrueNAS)
	graph.UpdateMetricCohort(cfg, base.Add(cfg.UpdateInterval), 3, mockMetricCohortCount)
	if got := trueNASCollectedAt(graph.PlatformFixtures.TrueNAS); !got.Equal(collectedAt) {
		t.Fatalf("fast ticks rebased supplemental fixtures on a non-zero cohort: %s -> %s", collectedAt, got)
	}

	// Slow ticks (the public demo's PULSE_MOCK_UPDATE_INTERVAL shape): the
	// rotation outlives the registry's freshness budget, so every cohort must
	// rebase provider-backed fixtures or their rows read as stale sources.
	cfg.UpdateInterval = 60 * time.Second
	graph = buildFixtureGraph(cfg, base)
	tick := base.Add(cfg.UpdateInterval)
	graph.UpdateMetricCohort(cfg, tick, 3, mockMetricCohortCount)
	if got := trueNASCollectedAt(graph.PlatformFixtures.TrueNAS); !got.Equal(tick) {
		t.Fatalf("slow ticks left TrueNAS fixtures at %s, want rebase to %s", got, tick)
	}
	if got := graph.PlatformFixtures.VMware.CollectedAt; !got.Equal(tick) {
		t.Fatalf("slow ticks left VMware fixtures at %s, want rebase to %s", got, tick)
	}
	if got := availabilityFixturesFreshness(graph.AvailabilityFixtures); !got.Equal(tick) {
		t.Fatalf("slow ticks left availability fixtures at %s, want rebase to %s", got, tick)
	}
}

// The public demo runs PULSE_MOCK_UPDATE_INTERVAL=15s. A fixed ten-cohort
// rotation sighted each PVE node's guests only every 150s there, while the
// registry judges Proxmox sightings stale after 60s, so most running guests
// drifted into Attention as telemetry_stale within minutes of startup.
func TestFixtureGraphSlowTicksKeepProxmoxGuestSightingsFresh(t *testing.T) {
	t.Cleanup(func() { setMockUpdateInterval(DefaultConfig.UpdateInterval) })

	for _, interval := range []time.Duration{2 * time.Second, 7 * time.Second, 15 * time.Second, 60 * time.Second} {
		t.Run(interval.String(), func(t *testing.T) {
			cfg := DefaultConfig
			cfg.NodeCount = 8
			cfg.VMsPerNode = 6
			cfg.LXCsPerNode = 4
			cfg.DockerHostCount = 0
			cfg.GenericHostCount = 0
			cfg.K8sClusterCount = 0
			cfg.UpdateInterval = interval

			base := time.Date(2026, 10, 22, 12, 0, 0, 0, time.UTC)
			graph := buildFixtureGraph(cfg, base)
			online := make(map[string]struct{}, len(graph.State.Nodes))
			for _, node := range graph.State.Nodes {
				if node.Status == "online" && node.ConnectionHealth != "offline" && node.Uptime > 0 {
					online[node.Name] = struct{}{}
				}
			}
			if len(online) == 0 {
				t.Fatal("fixture estate has no online PVE node")
			}

			bound := proxmoxSightingBudget
			if interval > bound {
				bound = interval
			}
			// Drive the update loop's own cohort sequence through two full
			// rotations and measure every online guest's sighting age just
			// before each tick lands, when it is oldest.
			for tick := 1; tick <= 2*mockMetricCohortCount; tick++ {
				now := base.Add(time.Duration(tick) * interval)
				for _, vm := range graph.State.VMs {
					if _, ok := online[vm.Node]; ok && now.Sub(vm.LastSeen) > bound {
						t.Fatalf("tick %d: VM %s on %s last sighted %s ago, want within %s", tick, vm.ID, vm.Node, now.Sub(vm.LastSeen), bound)
					}
				}
				for _, container := range graph.State.Containers {
					if _, ok := online[container.Node]; ok && now.Sub(container.LastSeen) > bound {
						t.Fatalf("tick %d: container %s on %s last sighted %s ago, want within %s", tick, container.ID, container.Node, now.Sub(container.LastSeen), bound)
					}
				}
				graph.UpdateMetricCohort(cfg, now, (tick-1)%mockMetricCohortCount, mockMetricCohortCount)
			}
		})
	}
}

// Shared storage (ceph-rbd, cephfs-data, the cluster PBS stores) and the
// Ceph cluster belong to no single PVE node, so the node-keyed cohort
// selection never re-sighted them: they kept their startup stamp and read as
// stale Proxmox sources on every mock tick rate, including the default.
func TestFixtureGraphSlowTicksKeepProxmoxClusterSightingsFresh(t *testing.T) {
	t.Cleanup(func() { setMockUpdateInterval(DefaultConfig.UpdateInterval) })

	for _, interval := range []time.Duration{2 * time.Second, 7 * time.Second, 15 * time.Second, 60 * time.Second} {
		t.Run(interval.String(), func(t *testing.T) {
			cfg := DefaultConfig
			cfg.NodeCount = 8
			cfg.VMsPerNode = 6
			cfg.LXCsPerNode = 4
			cfg.DockerHostCount = 0
			cfg.GenericHostCount = 0
			cfg.K8sClusterCount = 0
			cfg.UpdateInterval = interval

			base := time.Date(2026, 10, 22, 12, 0, 0, 0, time.UTC)
			graph := buildFixtureGraph(cfg, base)
			nodeNames := make(map[string]struct{}, len(graph.State.Nodes))
			for _, node := range graph.State.Nodes {
				nodeNames[node.Name] = struct{}{}
			}
			var shared []string
			for _, storage := range graph.State.Storage {
				if _, owned := nodeNames[storage.Node]; !owned && storage.Status == "available" {
					shared = append(shared, storage.ID)
				}
			}
			if !slices.ContainsFunc(shared, func(id string) bool { return strings.HasSuffix(id, "-ceph-rbd") }) {
				t.Fatalf("fixture estate has no available shared ceph-rbd storage: %v", shared)
			}
			if len(graph.State.CephClusters) == 0 {
				t.Fatal("fixture estate has no Ceph cluster")
			}

			bound := proxmoxSightingBudget
			if interval > bound {
				bound = interval
			}
			// Measure just before each tick lands, when sightings are oldest,
			// and keep each resource's worst age over two full rotations.
			oldest := make(map[string]time.Duration)
			observe := func(key string, age time.Duration) {
				if age > oldest[key] {
					oldest[key] = age
				}
			}
			for tick := 1; tick <= 2*mockMetricCohortCount; tick++ {
				now := base.Add(time.Duration(tick) * interval)
				for _, storage := range graph.State.Storage {
					if slices.Contains(shared, storage.ID) {
						observe("shared storage "+storage.ID+" (node "+storage.Node+")", now.Sub(storage.LastSeen))
					}
				}
				for _, cluster := range graph.State.CephClusters {
					observe("Ceph cluster "+cluster.ID, now.Sub(cluster.LastUpdated))
				}
				graph.UpdateMetricCohort(cfg, now, (tick-1)%mockMetricCohortCount, mockMetricCohortCount)
			}
			keys := slices.Sorted(maps.Keys(oldest))
			for _, key := range keys {
				if oldest[key] > bound {
					t.Errorf("%s last sighted up to %s ago, want within %s", key, oldest[key], bound)
				}
			}
		})
	}
}

func TestFixtureGraphMetricCohortProducesSparseUnifiedResourceChanges(t *testing.T) {
	cfg := DefaultConfig
	cfg.NodeCount = 50
	cfg.VMsPerNode = 10
	cfg.LXCsPerNode = 8
	cfg.RandomMetrics = true

	base := time.Date(2026, 8, 23, 12, 0, 0, 0, time.UTC)
	graph := buildFixtureGraph(cfg, base)
	before, _ := graph.UnifiedResourceSnapshot()

	graph.UpdateMetricCohort(cfg, base.Add(cfg.UpdateInterval), 0, mockMetricCohortCount)
	after, _ := graph.UnifiedResourceSnapshot()

	encodeBroadcastRelevant := func(resource unifiedresources.Resource) []byte {
		encoded, err := json.Marshal(resource)
		if err != nil {
			t.Fatalf("marshal resource %s: %v", resource.ID, err)
		}
		var payload map[string]json.RawMessage
		if err := json.Unmarshal(encoded, &payload); err != nil {
			t.Fatalf("decode resource %s: %v", resource.ID, err)
		}
		// ResourceFrontend intentionally omits the registry's ingestion timestamp.
		// It is regenerated whenever a throwaway mock registry is built and is not
		// part of the WebSocket resource payload.
		delete(payload, "updatedAt")
		encoded, err = json.Marshal(payload)
		if err != nil {
			t.Fatalf("re-marshal resource %s: %v", resource.ID, err)
		}
		return encoded
	}

	encodedBefore := make(map[string][]byte, len(before))
	for _, resource := range before {
		encodedBefore[resource.ID] = encodeBroadcastRelevant(resource)
	}

	changed := 0
	for _, resource := range after {
		encoded := encodeBroadcastRelevant(resource)
		if !bytes.Equal(encodedBefore[resource.ID], encoded) {
			changed++
		}
	}

	if changed == 0 || changed >= len(after)/2 {
		t.Fatalf("expected sparse unified resource churn, changed=%d total=%d", changed, len(after))
	}
}

func fixtureMetricsTargets(t *testing.T) map[string]unifiedresources.MetricsTarget {
	t.Helper()
	resources, _ := UnifiedResourceSnapshot()
	if len(resources) == 0 {
		t.Fatal("expected mock resources")
	}
	registry := unifiedresources.NewRegistry(nil)
	registry.IngestResources(resources)
	_, targets := registry.ListWithMetricsTargets()
	if len(targets) == 0 {
		t.Fatal("expected the mock estate to resolve metrics targets")
	}
	return targets
}

func fixtureResourceIDs(t *testing.T) []string {
	t.Helper()
	resources, _ := UnifiedResourceSnapshot()
	ids := make([]string, 0, len(resources))
	for _, resource := range resources {
		ids = append(ids, resource.ID)
	}
	slices.Sort(ids)
	return ids
}

// smallMockEstateForTest replaces the ambient estate with a fixed small one,
// so a test neither depends on the process's PULSE_MOCK_* settings (a node
// count at the supported maximum clamps back when raised) nor pays for a
// 929-guest snapshot per assertion. The ambient configuration returns when
// the test ends.
func smallMockEstateForTest(t *testing.T, nodes int) MockConfig {
	t.Helper()
	previous := GetConfig()
	t.Cleanup(func() { SetMockConfig(previous) })
	small := previous
	small.NodeCount = nodes
	SetMockConfig(small)
	return small
}

// Callers cache a resource's metrics target for as long as the structure
// revision holds (see Monitor.MetricsTargetForResource), so a metric tick must
// leave the revision, the resource set and every target exactly as they were.
// Checked after every tick of two full cohort rotations, not only at the end,
// so a change that a later tick reverts is still caught.
func TestMetricTicksNeverChangeTheFixtureStructure(t *testing.T) {
	withMockEnabledForTest(t)
	stopUpdateLoop()
	t.Cleanup(func() {
		if IsMockEnabled() {
			startUpdateLoop()
		}
	})
	smallMockEstateForTest(t, 6)
	stopUpdateLoop()

	revision := FixtureStructureRevision()
	ids := fixtureResourceIDs(t)
	targets := fixtureMetricsTargets(t)
	version := FixtureDataVersion()

	for tick := 1; tick <= mockMetricCohortCount*2+1; tick++ {
		updateMetrics(GetConfig())

		if got := FixtureDataVersion(); got <= version {
			t.Fatalf("tick %d did not advance the data version (%d)", tick, got)
		} else {
			version = got
		}
		if got := FixtureStructureRevision(); got != revision {
			t.Fatalf("tick %d moved the fixture structure revision from %d to %d", tick, revision, got)
		}
		if got := fixtureResourceIDs(t); !slices.Equal(got, ids) {
			t.Fatalf("tick %d changed the resource set: %d resources before, %d after", tick, len(ids), len(got))
		}
		if got := fixtureMetricsTargets(t); !reflect.DeepEqual(got, targets) {
			t.Fatalf("tick %d changed the resolved metrics targets", tick)
		}
	}
}

func TestFixtureStructureRevisionAdvancesOnStructuralChange(t *testing.T) {
	withMockEnabledForTest(t)
	stopUpdateLoop()
	t.Cleanup(func() {
		if IsMockEnabled() {
			startUpdateLoop()
		}
	})
	small := smallMockEstateForTest(t, 3)
	stopUpdateLoop()

	revision := FixtureStructureRevision()
	grown := small
	grown.NodeCount = 4
	SetMockConfig(grown)
	if got := FixtureStructureRevision(); got <= revision {
		t.Fatalf("reconfiguring the estate left the structure revision at %d (was %d)", got, revision)
	}

	revision = FixtureStructureRevision()
	if err := SetEnabled(false); err != nil {
		t.Fatalf("disable mock mode: %v", err)
	}
	if got := FixtureStructureRevision(); got <= revision {
		t.Fatalf("disabling mock mode left the structure revision at %d (was %d)", got, revision)
	}
	revision = FixtureStructureRevision()
	if err := SetEnabled(true); err != nil {
		t.Fatalf("enable mock mode: %v", err)
	}
	if got := FixtureStructureRevision(); got <= revision {
		t.Fatalf("enabling mock mode left the structure revision at %d (was %d)", got, revision)
	}
}

// A cache fill reads the structure revision before the data version, and the
// fixture advances its data version before its revision, so a reader that
// sees a new revision is guaranteed the new data version after it and is
// never handed the previous estate's memoized snapshot under the new
// revision. The interleaving cannot be forced from outside, so the order of
// the two writes and of the two reads is pinned on the syntax tree: inside
// the named functions only, so comments, local names and the position of the
// functions in their package do not matter.
func TestFixtureStructureRevisionOrderingContract(t *testing.T) {
	mockDecls := parsePackageFuncs(t, ".")
	advance := packageFunc(mockDecls["advanceFixtureStructure"])
	if advance == nil {
		t.Fatal("the mock package lost advanceFixtureStructure")
	}
	version := synchronousCallOffsets(advance, "fixtureDataVersion", "Add")
	revision := synchronousCallOffsets(advance, "fixtureRevision", "Add")
	if len(version) != 1 || len(revision) != 1 || version[0] > revision[0] {
		t.Fatal("advanceFixtureStructure must advance the data version once, before the structure revision once, as plain statements")
	}
	if got := countCalls(advance, "fixtureRevision", "Add") + countCalls(advance, "fixtureDataVersion", "Add"); got != 2 {
		t.Fatalf("advanceFixtureStructure must not defer or spawn its counter writes; found %d writes in all", got)
	}

	// Every declaration counts for the bypass scan, including methods and the
	// build-tagged duplicates of a function.
	for name, decls := range mockDecls {
		if name == "advanceFixtureStructure" {
			continue
		}
		for _, decl := range decls {
			if got := countCalls(decl, "fixtureRevision", "Add"); got != 0 {
				t.Fatalf("%s advances the structure revision directly; it must go through advanceFixtureStructure", name)
			}
		}
	}
	for _, name := range []string{"enableMockMode", "disableMockMode", "SetMockConfig"} {
		decl := packageFunc(mockDecls[name])
		if decl == nil {
			t.Fatalf("the mock package lost %s", name)
		}
		if got := len(synchronousIdentCallOffsets(decl, "advanceFixtureStructure")); got != 1 {
			t.Fatalf("%s must advance the fixture structure through advanceFixtureStructure exactly once, as a plain statement; found %d", name, got)
		}
	}

	view := packageFunc(parsePackageFuncs(t, "../monitoring")["currentUnifiedStateView"])
	if view == nil {
		t.Fatal("the monitoring package lost currentUnifiedStateView")
	}
	structureRead := synchronousCallOffsets(view, "mock", "FixtureStructureRevision")
	versionRead := synchronousCallOffsets(view, "mock", "FixtureDataVersion")
	if len(structureRead) != 1 || len(versionRead) != 1 || structureRead[0] > versionRead[0] {
		t.Fatal("currentUnifiedStateView must read the structure revision once, before it reads the data version")
	}
}

// parsePackageFuncs returns every non-test function and method declaration of
// the package in dir, keyed by name. Names repeat across build-tagged files
// and between functions and methods, so each key holds all of them.
func parsePackageFuncs(t *testing.T, dir string) map[string][]*ast.FuncDecl {
	t.Helper()
	entries, err := os.ReadDir(dir)
	if err != nil {
		t.Fatalf("read %s: %v", dir, err)
	}
	funcs := make(map[string][]*ast.FuncDecl)
	fset := token.NewFileSet()
	for _, entry := range entries {
		name := entry.Name()
		if entry.IsDir() || !strings.HasSuffix(name, ".go") || strings.HasSuffix(name, "_test.go") {
			continue
		}
		file, err := parser.ParseFile(fset, filepath.Join(dir, name), nil, 0)
		if err != nil {
			t.Fatalf("parse %s: %v", name, err)
		}
		for _, decl := range file.Decls {
			if fn, ok := decl.(*ast.FuncDecl); ok {
				funcs[fn.Name.Name] = append(funcs[fn.Name.Name], fn)
			}
		}
	}
	return funcs
}

// packageFunc returns the declaration when it is the only one of its name in
// the package (function or method), and nil otherwise.
func packageFunc(decls []*ast.FuncDecl) *ast.FuncDecl {
	if len(decls) != 1 {
		return nil
	}
	return decls[0]
}

// walkSynchronous visits the nodes of fn that run in the function's own
// control flow: it does not descend into function literals, deferred calls or
// goroutine starts, whose source position says nothing about when they run.
func walkSynchronous(fn *ast.FuncDecl, visit func(ast.Node)) {
	ast.Inspect(fn, func(node ast.Node) bool {
		switch node.(type) {
		case *ast.FuncLit, *ast.DeferStmt, *ast.GoStmt:
			return false
		}
		visit(node)
		return true
	})
}

func isSelectorCall(node ast.Node, receiver, method string) bool {
	call, ok := node.(*ast.CallExpr)
	if !ok {
		return false
	}
	selector, ok := call.Fun.(*ast.SelectorExpr)
	if !ok || selector.Sel.Name != method {
		return false
	}
	ident, ok := selector.X.(*ast.Ident)
	return ok && ident.Name == receiver
}

// synchronousCallOffsets lists the positions of receiver.method calls that run
// in fn's own control flow, in source order.
func synchronousCallOffsets(fn *ast.FuncDecl, receiver, method string) []token.Pos {
	var offsets []token.Pos
	walkSynchronous(fn, func(node ast.Node) {
		if isSelectorCall(node, receiver, method) {
			offsets = append(offsets, node.Pos())
		}
	})
	return offsets
}

// synchronousIdentCallOffsets lists the positions of calls to the package-level
// function name that run in fn's own control flow.
func synchronousIdentCallOffsets(fn *ast.FuncDecl, name string) []token.Pos {
	var offsets []token.Pos
	walkSynchronous(fn, func(node ast.Node) {
		if call, ok := node.(*ast.CallExpr); ok {
			if ident, ok := call.Fun.(*ast.Ident); ok && ident.Name == name {
				offsets = append(offsets, call.Pos())
			}
		}
	})
	return offsets
}

// countCalls counts every receiver.method call anywhere in fn, including
// deferred calls, goroutines and function literals.
func countCalls(fn *ast.FuncDecl, receiver, method string) int {
	count := 0
	ast.Inspect(fn, func(node ast.Node) bool {
		if isSelectorCall(node, receiver, method) {
			count++
		}
		return true
	})
	return count
}

func TestFixtureGraphMatchesMonitorSupplementalFreshness(t *testing.T) {
	now := time.Now().UTC()
	for _, age := range []time.Duration{0, 2 * time.Hour} {
		t.Run(age.String(), func(t *testing.T) {
			graph := buildFixtureGraph(DefaultConfig, now.Add(-age))
			owned := SupplementalOwnedSources()
			records := make(map[unifiedresources.DataSource][]unifiedresources.IngestRecord, len(owned))
			for _, source := range owned {
				records[source] = graph.SupplementalRecords(source)
			}
			adapter := unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(nil))
			adapter.PopulateSnapshotAndSupplemental(unifiedresources.SnapshotWithoutSources(graph.State, owned), records)
			demo, _ := graph.UnifiedResourceSnapshot()
			byID := make(map[string]unifiedresources.Resource, len(demo))
			for _, resource := range demo {
				byID[resource.ID] = resource
			}
			checked := 0
			for _, live := range adapter.GetAll() {
				if live.Type != unifiedresources.ResourceTypeVM || len(live.Sources) != 1 || live.Sources[0] != unifiedresources.SourceTrueNAS {
					continue
				}
				checked++
				wantStatus := "online"
				if age > 0 {
					wantStatus = "stale"
				}
				if live.SourceStatus[unifiedresources.SourceTrueNAS].Status != wantStatus {
					t.Fatalf("live reference has no %s TrueNAS sighting: %+v", wantStatus, live)
				}
				got, ok := byID[live.ID]
				if !ok || got.Status != live.Status || got.SourceStatus[unifiedresources.SourceTrueNAS].Status != wantStatus {
					t.Fatalf("demo freshness differs from live for %s: demo=%+v live=%+v", live.ID, got, live)
				}
			}
			if checked == 0 {
				t.Fatal("fixture has no TrueNAS-only VM to compare")
			}
		})
	}
}