Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
package mock
import (
"fmt"
"math"
"sort"
"strings"
"testing"
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/models"
"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
)
func TestBuildFixtureStateIncludesDockerHosts(t *testing.T) {
cfg := DefaultConfig
cfg.DockerHostCount = 2
cfg.DockerContainersPerHost = 5
data := buildFixtureState(cfg)
// Nested Docker-in-LXC hosts are appended on top of the configured
// agent-reported hosts and deliberately carry no native engine
// inventory, so the native-host expectations exclude them.
nativeHosts := 0
for _, host := range data.DockerHosts {
if host.ID == "" {
t.Fatalf("docker host missing id: %+v", host)
}
if strings.HasPrefix(host.ID, mockProxmoxLXCDockerHostSourcePrefix) {
continue
}
nativeHosts++
if len(host.Containers) == 0 {
t.Fatalf("docker host %s has no containers", host.Hostname)
}
if len(host.Images) == 0 {
t.Fatalf("docker host %s has no image inventory", host.Hostname)
}
if len(host.Volumes) == 0 {
t.Fatalf("docker host %s has no volume inventory", host.Hostname)
}
if len(host.Networks) == 0 {
t.Fatalf("docker host %s has no network inventory", host.Hostname)
}
if host.StorageUsage == nil || host.StorageUsage.Images.TotalCount == 0 || host.StorageUsage.Volumes.TotalCount == 0 {
t.Fatalf("docker host %s has no engine storage usage inventory: %+v", host.Hostname, host.StorageUsage)
}
}
if nativeHosts != cfg.DockerHostCount {
t.Fatalf("expected %d native docker hosts, got %d", cfg.DockerHostCount, nativeHosts)
}
}
func TestComputeGuestCountsHandlesZeroBaselines(t *testing.T) {
cfg := MockConfig{
VMsPerNode: 0,
LXCsPerNode: 0,
}
roles := []string{"vm-heavy", "container-heavy", "light", "mixed"}
for _, role := range roles {
vmCount, lxcCount := computeGuestCounts(cfg, role)
if vmCount < 0 || lxcCount < 0 {
t.Fatalf("expected non-negative counts for role %q, got vm=%d lxc=%d", role, vmCount, lxcCount)
}
}
}
func TestBuildFixtureStateWithZeroGuestBaselinesDoesNotPanic(t *testing.T) {
cfg := DefaultConfig
cfg.NodeCount = 4
cfg.VMsPerNode = 0
cfg.LXCsPerNode = 0
cfg.DockerHostCount = 0
cfg.GenericHostCount = 0
cfg.K8sClusterCount = 0
for i := 0; i < 20; i++ {
func() {
defer func() {
if r := recover(); r != nil {
t.Fatalf("buildFixtureState panicked on iteration %d: %v", i, r)
}
}()
_ = buildFixtureState(cfg)
}()
}
}
func TestBuildFixtureStateIncludesSwarmServices(t *testing.T) {
cfg := DefaultConfig
// Two hosts is the smallest mixed Docker/Podman estate generated by the
// mock runtime and must still include a Docker manager inventory.
cfg.DockerHostCount = 2
cfg.DockerContainersPerHost = 6
cfg.RandomMetrics = false
data := buildFixtureState(cfg)
found := false
for _, host := range data.DockerHosts {
if len(host.Services) == 0 {
continue
}
if host.Swarm == nil {
t.Fatalf("expected swarm metadata for host %s", host.ID)
}
if len(host.Tasks) == 0 {
t.Fatalf("expected tasks for service host %s", host.ID)
}
if len(host.Secrets) == 0 {
t.Fatalf("expected secrets for service host %s", host.ID)
}
if len(host.Configs) == 0 {
t.Fatalf("expected configs for service host %s", host.ID)
}
if len(host.Nodes) == 0 {
t.Fatalf("expected swarm nodes for service host %s", host.ID)
}
found = true
break
}
if !found {
t.Fatalf("expected at least one docker host with swarm services")
}
}
func TestBuildFixtureStateIncludesHostAgents(t *testing.T) {
cfg := DefaultConfig
cfg.GenericHostCount = 5
cfg.K8sClusterCount = 0
cfg.RandomMetrics = false
data := buildFixtureState(cfg)
expectedMin := cfg.GenericHostCount + cfg.NodeCount
if len(data.Hosts) < expectedMin {
t.Fatalf("expected at least %d host agents, got %d", expectedMin, len(data.Hosts))
}
for _, host := range data.Hosts {
if host.ID == "" {
t.Fatalf("host agent missing id: %+v", host)
}
if host.Hostname == "" {
t.Fatalf("host agent missing hostname: %+v", host)
}
if host.Status == "" {
t.Fatalf("host agent missing status: %+v", host)
}
if host.Memory.Total > 0 {
if host.Memory.Cache <= 0 {
t.Fatalf("host agent %s should report reclaimable cache so the memory split is exercisable: %+v", host.ID, host.Memory)
}
if sum := host.Memory.Used + host.Memory.Cache + host.Memory.Free; sum > host.Memory.Total {
t.Fatalf("host agent %s memory used+cache+free %d exceeds total %d", host.ID, sum, host.Memory.Total)
}
}
}
}
func TestBuildFixtureStatePreservesStandaloneHostAgentsWhenLinkingNodes(t *testing.T) {
cfg := DefaultConfig
cfg.NodeCount = 3
cfg.GenericHostCount = 4
cfg.K8sClusterCount = 0
cfg.RandomMetrics = false
data := buildFixtureState(cfg)
standalone := 0
proxmoxLinked := 0
for _, host := range data.Hosts {
if host.LinkedNodeID != "" {
proxmoxLinked++
} else {
standalone++
}
}
if standalone != cfg.GenericHostCount {
t.Fatalf("expected %d standalone agent hosts, got %d", cfg.GenericHostCount, standalone)
}
if proxmoxLinked != cfg.NodeCount {
t.Fatalf("expected %d Proxmox-linked agent hosts, got %d", cfg.NodeCount, proxmoxLinked)
}
}
func TestGenerateVMStoppedPreservesConfiguredCapacity(t *testing.T) {
cfg := DefaultConfig
cfg.StoppedPercent = 1
vm := generateVM("node-01", "mock-cluster", 1001, cfg)
if vm.Status != "stopped" {
t.Fatalf("expected stopped VM, got %q", vm.Status)
}
if vm.Memory.Total <= 0 {
t.Fatalf("expected stopped VM to keep configured memory total, got %d", vm.Memory.Total)
}
if vm.Memory.Used != 0 || vm.Memory.Usage != 0 {
t.Fatalf("expected stopped VM memory usage to be zero, got used=%d usage=%f", vm.Memory.Used, vm.Memory.Usage)
}
if vm.Memory.Free != vm.Memory.Total {
t.Fatalf("expected stopped VM free memory %d to equal total %d", vm.Memory.Free, vm.Memory.Total)
}
}
func TestGenerateContainerStoppedPreservesConfiguredCapacity(t *testing.T) {
cfg := DefaultConfig
cfg.StoppedPercent = 1
ct := generateContainer("node-01", "mock-cluster", 2001, cfg)
if ct.Status != "stopped" {
t.Fatalf("expected stopped container, got %q", ct.Status)
}
if ct.Memory.Total <= 0 {
t.Fatalf("expected stopped container to keep configured memory total, got %d", ct.Memory.Total)
}
if ct.Memory.Used != 0 || ct.Memory.Usage != 0 {
t.Fatalf("expected stopped container memory usage to be zero, got used=%d usage=%f", ct.Memory.Used, ct.Memory.Usage)
}
if ct.Memory.Free != ct.Memory.Total {
t.Fatalf("expected stopped container free memory %d to equal total %d", ct.Memory.Free, ct.Memory.Total)
}
}
func TestNormalizeMockBlendWeight_ComposesAcrossUpdateInterval(t *testing.T) {
perMinuteWeight := 0.22
perTickWeight := normalizeMockBlendWeight(perMinuteWeight, defaultMockUpdateInterval, time.Minute)
compounded := 1 - math.Pow(1-perTickWeight, float64(time.Minute/defaultMockUpdateInterval))
if perTickWeight >= perMinuteWeight {
t.Fatalf("expected per-tick weight %.6f to be less than per-minute weight %.6f", perTickWeight, perMinuteWeight)
}
if math.Abs(compounded-perMinuteWeight) > 0.01 {
t.Fatalf("expected compounded weight %.6f to stay close to %.6f", compounded, perMinuteWeight)
}
}
func TestBuildFixtureStateLinksAllNodesToHostAgents(t *testing.T) {
cfg := DefaultConfig
cfg.NodeCount = 7
cfg.GenericHostCount = 2
cfg.RandomMetrics = false
data := buildFixtureState(cfg)
if len(data.Hosts) < cfg.NodeCount {
t.Fatalf("expected enough hosts to link all nodes, got hosts=%d nodes=%d", len(data.Hosts), cfg.NodeCount)
}
hostsByID := make(map[string]models.Host, len(data.Hosts))
for _, host := range data.Hosts {
hostsByID[host.ID] = host
}
for _, node := range data.Nodes {
if node.LinkedAgentID == "" {
t.Fatalf("node %s is missing linked host agent id", node.ID)
}
host, ok := hostsByID[node.LinkedAgentID]
if !ok {
t.Fatalf("node %s linked host %s not found", node.ID, node.LinkedAgentID)
}
if host.LinkedNodeID != node.ID {
t.Fatalf("host %s linkedNodeID=%q, want %q", host.ID, host.LinkedNodeID, node.ID)
}
}
}
// A node and its linked agent carry one CPU reading under the agent's real
// keys, taken from the series that seeds the agent's history, so the node
// drawer's headline continues the line drawn beneath it.
func TestFixtureNodeAndLinkedAgentShareOnePrimaryCPUTemperature(t *testing.T) {
cfg := DefaultConfig
cfg.NodeCount = 7
cfg.GenericHostCount = 2
cfg.RandomMetrics = true
data := buildFixtureState(cfg)
check := func(stage string, at time.Time, sampled bool) {
t.Helper()
hostsByID := make(map[string]models.Host, len(data.Hosts))
for _, host := range data.Hosts {
hostsByID[host.ID] = host
if _, legacy := host.Sensors.TemperatureCelsius["cpu.package"]; legacy {
t.Fatalf("%s: host %s carries a second package reading under cpu.package", stage, host.ID)
}
}
compared := 0
for _, node := range data.Nodes {
nodeReading := models.PrimaryCPUTemperatureCelsius(node.Temperature)
if nodeReading == nil || node.Status != "online" {
continue
}
host, ok := hostsByID[node.LinkedAgentID]
if !ok || host.Status == "offline" {
continue
}
hostReading := models.HostSensorPrimaryCPUTemperatureCelsius(host.Sensors)
if hostReading == nil || *hostReading != *nodeReading {
t.Fatalf("%s: node %s reads %v but its agent %s reads %v", stage, node.ID, *nodeReading, host.ID, hostReading)
}
if want := SampleMetric("agent", host.ID, "temperature", at); sampled && *nodeReading != want {
t.Fatalf("%s: node %s reads %v, want the agent series sample %v", stage, node.ID, *nodeReading, want)
}
compared++
}
if compared == 0 {
t.Fatalf("%s: no online node with a CPU reading to compare", stage)
}
}
check("build", time.Time{}, false)
refreshAt := time.Now().Add(time.Minute)
updateFixtureStateMetricsAt(&data, cfg, refreshAt)
check("refresh", refreshAt, true)
}
func TestBuildFixtureStatePopulatesHostIORates(t *testing.T) {
cfg := DefaultConfig
cfg.NodeCount = 6
cfg.GenericHostCount = 1
cfg.RandomMetrics = true
data := buildFixtureState(cfg)
for _, host := range data.Hosts {
if host.Status == "offline" {
if host.NetInRate != 0 || host.NetOutRate != 0 || host.DiskReadRate != 0 || host.DiskWriteRate != 0 {
t.Fatalf("offline host %s should have zero I/O rates", host.ID)
}
continue
}
if host.NetInRate <= 0 || host.NetOutRate <= 0 {
t.Fatalf("host %s missing network rates: in=%f out=%f", host.ID, host.NetInRate, host.NetOutRate)
}
if host.DiskReadRate <= 0 || host.DiskWriteRate <= 0 {
t.Fatalf("host %s missing disk rates: read=%f write=%f", host.ID, host.DiskReadRate, host.DiskWriteRate)
}
}
}
func TestBuildFixtureStatePopulatesDockerHostIORates(t *testing.T) {
cfg := DefaultConfig
cfg.DockerHostCount = 3
cfg.RandomMetrics = true
data := buildFixtureState(cfg)
if len(data.DockerHosts) == 0 {
t.Fatal("expected docker hosts in mock data")
}
for _, host := range data.DockerHosts {
if strings.HasPrefix(host.ID, mockProxmoxLXCDockerHostSourcePrefix) {
// Docker-in-LXC hosts mirror the production pct exec inventory,
// which reports no host-level I/O rates or temperature.
continue
}
if host.Status == "offline" {
if host.NetInRate != 0 || host.NetOutRate != 0 || host.DiskReadRate != 0 || host.DiskWriteRate != 0 {
t.Fatalf("offline docker host %s should have zero I/O rates", host.ID)
}
if host.Temperature != nil {
t.Fatalf("offline docker host %s should not report temperature", host.ID)
}
continue
}
if host.NetInRate <= 0 || host.NetOutRate <= 0 {
t.Fatalf("docker host %s missing network rates: in=%f out=%f", host.ID, host.NetInRate, host.NetOutRate)
}
if host.DiskReadRate <= 0 || host.DiskWriteRate <= 0 {
t.Fatalf("docker host %s missing disk rates: read=%f write=%f", host.ID, host.DiskReadRate, host.DiskWriteRate)
}
if host.Temperature == nil {
t.Fatalf("docker host %s missing temperature", host.ID)
}
}
}
func TestBuildFixtureStatePopulatesKubernetesUsageMetrics(t *testing.T) {
cfg := DefaultConfig
cfg.K8sClusterCount = 1
cfg.K8sNodesPerCluster = 4
cfg.K8sPodsPerCluster = 24
cfg.RandomMetrics = false
data := buildFixtureState(cfg)
if len(data.KubernetesClusters) != 1 {
t.Fatalf("expected exactly one kubernetes cluster, got %d", len(data.KubernetesClusters))
}
cluster := data.KubernetesClusters[0]
readyNodeNames := make(map[string]struct{}, len(cluster.Nodes))
for _, node := range cluster.Nodes {
if node.Ready {
readyNodeNames[strings.TrimSpace(node.Name)] = struct{}{}
}
}
runningPodsWithUsage := 0
for _, pod := range cluster.Pods {
if !strings.EqualFold(pod.Phase, "running") {
continue
}
if _, ok := readyNodeNames[strings.TrimSpace(pod.NodeName)]; !ok {
continue
}
if pod.UsageCPUMilliCores <= 0 || pod.UsageMemoryBytes <= 0 {
t.Fatalf("running pod %s missing cpu/memory usage: %+v", pod.Name, pod)
}
if pod.NetInRate <= 0 || pod.NetOutRate <= 0 {
t.Fatalf("running pod %s missing network rates: in=%f out=%f", pod.Name, pod.NetInRate, pod.NetOutRate)
}
if pod.EphemeralStorageCapacityBytes <= 0 || pod.EphemeralStorageUsedBytes <= 0 {
t.Fatalf("running pod %s missing ephemeral storage usage: %+v", pod.Name, pod)
}
if pod.DiskUsagePercent <= 0 {
t.Fatalf("running pod %s missing disk usage percent: %+v", pod.Name, pod)
}
runningPodsWithUsage++
}
if runningPodsWithUsage == 0 {
t.Fatal("expected at least one running kubernetes pod with usage metrics")
}
readyNodesWithUsage := 0
for _, node := range cluster.Nodes {
if !node.Ready {
continue
}
if node.UsageCPUMilliCores <= 0 || node.UsageMemoryBytes <= 0 {
t.Fatalf("ready node %s missing usage metrics: %+v", node.Name, node)
}
readyNodesWithUsage++
}
if readyNodesWithUsage == 0 {
t.Fatal("expected at least one ready kubernetes node with usage metrics")
}
}
func TestGenerateKubernetesNodesAlwaysRetainsSchedulableReadyNode(t *testing.T) {
for count := 1; count <= 8; count++ {
for iteration := 0; iteration < 100; iteration++ {
nodes := generateKubernetesNodes("fixture-cluster", count)
hasSchedulableReadyNode := false
for _, node := range nodes {
if node.Ready && !node.Unschedulable {
hasSchedulableReadyNode = true
break
}
}
if !hasSchedulableReadyNode {
t.Fatalf("node count %d iteration %d produced no schedulable ready node: %+v", count, iteration, nodes)
}
}
}
}
func TestBuildFixtureStateIncludesKubernetesDeploymentAPIMetadata(t *testing.T) {
cfg := DefaultConfig
cfg.K8sClusterCount = 1
cfg.K8sNodesPerCluster = 3
cfg.K8sPodsPerCluster = 8
cfg.K8sDeploymentsPerCluster = 6
cfg.RandomMetrics = false
data := buildFixtureState(cfg)
if len(data.KubernetesClusters) != 1 {
t.Fatalf("expected exactly one kubernetes cluster, got %d", len(data.KubernetesClusters))
}
cluster := data.KubernetesClusters[0]
if len(cluster.Deployments) == 0 {
t.Fatal("expected kubernetes deployment inventory")
}
for _, deployment := range cluster.Deployments {
if deployment.UID == "" {
t.Fatalf("deployment missing uid: %+v", deployment)
}
if deployment.CreatedAt.IsZero() {
t.Fatalf("deployment missing creation metadata: %+v", deployment)
}
if deployment.ObservedGeneration == 0 {
t.Fatalf("deployment missing observed generation: %+v", deployment)
}
}
}
func TestBuildFixtureStateCreatesHostEntriesForKubernetesNodes(t *testing.T) {
cfg := DefaultConfig
cfg.K8sClusterCount = 1
cfg.K8sNodesPerCluster = 3
cfg.K8sPodsPerCluster = 8
cfg.NodeCount = 0
cfg.RandomMetrics = false
data := buildFixtureState(cfg)
if len(data.KubernetesClusters) == 0 {
t.Fatal("expected kubernetes clusters in mock data")
}
hostsByName := make(map[string]models.Host, len(data.Hosts))
for _, host := range data.Hosts {
hostsByName[strings.ToLower(strings.TrimSpace(host.Hostname))] = host
}
for _, node := range data.KubernetesClusters[0].Nodes {
host, ok := hostsByName[strings.ToLower(strings.TrimSpace(node.Name))]
if !ok {
t.Fatalf("expected host entry for kubernetes node %s", node.Name)
}
if host.NetInRate <= 0 || host.NetOutRate <= 0 {
t.Fatalf("kubernetes node host %s missing network rates: in=%f out=%f", host.Hostname, host.NetInRate, host.NetOutRate)
}
if host.DiskReadRate <= 0 || host.DiskWriteRate <= 0 {
t.Fatalf("kubernetes node host %s missing disk rates: read=%f write=%f", host.Hostname, host.DiskReadRate, host.DiskWriteRate)
}
if len(host.Sensors.TemperatureCelsius) == 0 {
t.Fatalf("kubernetes node host %s missing temperature sensors", host.Hostname)
}
}
}
// newMockKubernetesTestCluster builds ready, schedulable nodes with a fixed
// number of running pods each, so placement does not depend on the random
// readiness the fixture generator rolls.
func newMockKubernetesTestCluster(nodeCount, podsPerNode int) models.KubernetesCluster {
cluster := models.KubernetesCluster{ID: "usage-test", Name: "usage-test", Status: "online"}
for n := 0; n < nodeCount; n++ {
nodeName := fmt.Sprintf("usage-test-%d", n)
cluster.Nodes = append(cluster.Nodes, models.KubernetesNode{
UID: nodeName + "-uid",
Name: nodeName,
Ready: true,
CapacityCPU: 8,
AllocCPU: 8,
CapacityMemoryBytes: 32 << 30,
AllocMemoryBytes: 32 << 30,
})
for p := 0; p < podsPerNode; p++ {
cluster.Pods = append(cluster.Pods, models.KubernetesPod{
UID: fmt.Sprintf("%s-pod-%d-uid", nodeName, p),
Name: fmt.Sprintf("%s-pod-%d", nodeName, p),
Namespace: "apps",
NodeName: nodeName,
Phase: "Running",
OwnerKind: "Deployment",
})
}
}
return cluster
}
func TestMockKubernetesDenseNodesStayBelowAllocatableMemory(t *testing.T) {
cluster := newMockKubernetesTestCluster(3, 13)
data := models.StateSnapshot{KubernetesClusters: []models.KubernetesCluster{cluster}}
for _, node := range cluster.Nodes {
data.Hosts = append(data.Hosts, models.Host{ID: "host-" + node.Name, Hostname: node.Name, Status: "online"})
}
start := time.Date(2026, 10, 6, 12, 0, 0, 0, time.UTC)
for tick := 0; tick < 40; tick++ {
initializeMockKubernetesClusterUsage(&data.KubernetesClusters[0], start.Add(time.Duration(tick)*15*time.Second), false)
// The mock tick re-derives each linked host's memory from its node.
syncMockKubernetesNodeHosts(&data)
podMemory := make(map[string]float64, len(cluster.Nodes))
for _, pod := range data.KubernetesClusters[0].Pods {
podMemory[pod.NodeName] += pod.UsageMemoryPercent
}
for _, node := range data.KubernetesClusters[0].Nodes {
if node.UsageMemoryBytes >= node.AllocMemoryBytes || node.UsageMemoryPercent >= 100 {
t.Fatalf("tick %d: node %s with 13 running pods is pinned at allocatable memory (%.1f%%)", tick, node.Name, node.UsageMemoryPercent)
}
// Steady-state shares stay under the ceiling, which clips to it
// exactly; a total at the ceiling means it bound without a transient.
if podMemory[node.Name] >= mockKubernetesNodePodMemoryCeiling*100-0.01 {
t.Fatalf("tick %d: node %s pods hold %.2f%%, the ceiling bound in steady state", tick, node.Name, podMemory[node.Name])
}
}
for _, host := range data.Hosts {
if host.Memory.Usage >= 100 {
t.Fatalf("tick %d: linked host %s memory = %.1f%%, want the node's unsaturated reading", tick, host.Hostname, host.Memory.Usage)
}
}
}
}
func TestMockKubernetesRescheduledPodsDoNotSaturateReceivingNode(t *testing.T) {
cluster := newMockKubernetesTestCluster(3, 8)
start := time.Date(2026, 10, 6, 12, 0, 0, 0, time.UTC)
for tick := 0; tick < 40; tick++ {
initializeMockKubernetesClusterUsage(&cluster, start.Add(time.Duration(tick)*15*time.Second), false)
}
// Two nodes fail: their sixteen pods move to the survivor carrying
// readings sized for nodes that each ran a third as many pods.
cluster.Nodes[0].Ready = false
cluster.Nodes[1].Ready = false
initializeMockKubernetesClusterUsage(&cluster, start.Add(40*15*time.Second), false)
survivor := cluster.Nodes[2]
running := 0
for _, pod := range cluster.Pods {
if pod.NodeName == survivor.Name {
running++
}
}
if running != 24 {
t.Fatalf("survivor runs %d pods after the failures, want all 24", running)
}
// Pods hold at most the ceiling and system overhead tops out at 26%.
if limit := (mockKubernetesNodePodMemoryCeiling + 0.26) * 100; survivor.UsageMemoryPercent > limit+1e-9 {
t.Fatalf("survivor memory = %.1f%% right after rescheduling, want at most %.1f%%", survivor.UsageMemoryPercent, limit)
}
}
func TestMockKubernetesPodMemoryShareKeepsNodeWithinBudget(t *testing.T) {
for pods := 0; pods <= 64; pods++ {
share := mockKubernetesPodMemoryShare(pods)
if share <= 0 || share > 1 {
t.Fatalf("share(%d) = %f, want (0, 1]", pods, share)
}
if committed := float64(pods) * share; committed > mockKubernetesNodePodMemoryBudget+1e-9 {
t.Fatalf("%d pods commit %.2f single-pod footprints, want at most %.2f", pods, committed, mockKubernetesNodePodMemoryBudget)
}
}
}
func TestMockKubernetesLoneHeavyPodStaysUnderNodeCeiling(t *testing.T) {
// A five-minute cadence lets smoothing track each target closely, so the
// pod reaches its real peaks.
setMockUpdateInterval(5 * time.Minute)
t.Cleanup(func() { setMockUpdateInterval(DefaultConfig.UpdateInterval) })
// The heaviest footprint: the highest base and burst seeds on a
// Guaranteed StatefulSet pod.
uid, best := "", uint64(0)
for i := 0; i < 2000; i++ {
candidate := fmt.Sprintf("lone-pod-%d", i)
weight := mockStableHash64(candidate, "mem-base")%20 + mockStableHash64(candidate, "mem-burst")%36
if uid == "" || weight > best {
uid, best = candidate, weight
}
}
cluster := newMockKubernetesTestCluster(1, 1)
cluster.Pods[0].UID = uid
cluster.Pods[0].OwnerKind = "StatefulSet"
cluster.Pods[0].QoSClass = "Guaranteed"
start := time.Date(2026, 10, 6, 0, 0, 0, 0, time.UTC)
peak := 0.0
for tick := 0; tick < 288; tick++ {
initializeMockKubernetesClusterUsage(&cluster, start.Add(time.Duration(tick)*5*time.Minute), false)
peak = math.Max(peak, cluster.Pods[0].UsageMemoryPercent)
}
if peak >= mockKubernetesNodePodMemoryCeiling*100-0.01 {
t.Fatalf("lone pod %s peaked at %.2f%%: the ceiling clipped a steady-state reading", uid, peak)
}
}
func TestGenerateNodesBoundsDirectConfiguration(t *testing.T) {
cfg := DefaultConfig
cfg.NodeCount = 1 << 30
if got := len(generateNodes(cfg)); got != maxMockNodeCount {
t.Fatalf("generateNodes() count = %d, want max %d", got, maxMockNodeCount)
}
cfg.NodeCount = -1
if got := len(generateNodes(cfg)); got != minMockNodeCount {
t.Fatalf("generateNodes() count = %d, want min %d", got, minMockNodeCount)
}
}
func TestGenerateNodeIncludesProxmoxNetworkInventory(t *testing.T) {
node := generateNode("pve1", false, DefaultConfig)
if len(node.NetworkInterfaces) != 2 {
t.Fatalf("network interface count = %d, want 2", len(node.NetworkInterfaces))
}
if node.NetworkInterfaces[0].Name != "eno1" || node.NetworkInterfaces[1].Name != "vmbr0" {
t.Fatalf("unexpected network interfaces: %+v", node.NetworkInterfaces)
}
if len(node.NetworkInterfaces[1].Addresses) != 2 {
t.Fatalf("bridge addresses = %v, want IPv4 and IPv6", node.NetworkInterfaces[1].Addresses)
}
}
func TestBuildFixtureStateIncludesKubernetesStorageInventory(t *testing.T) {
cfg := DefaultConfig
cfg.K8sClusterCount = 1
cfg.K8sNodesPerCluster = 3
cfg.K8sPodsPerCluster = 8
cfg.RandomMetrics = false
data := buildFixtureState(cfg)
if len(data.KubernetesClusters) != 1 {
t.Fatalf("expected exactly one kubernetes cluster, got %d", len(data.KubernetesClusters))
}
cluster := data.KubernetesClusters[0]
if len(cluster.StorageClasses) == 0 {
t.Fatal("expected kubernetes storage class inventory")
}
if len(cluster.PersistentVolumes) == 0 {
t.Fatal("expected kubernetes persistent volume inventory")
}
if len(cluster.PersistentVolumeClaims) == 0 {
t.Fatal("expected kubernetes persistent volume claim inventory")
}
}
func TestBuildFixtureStateIncludesKubernetesNetworkingInventory(t *testing.T) {
cfg := DefaultConfig
cfg.K8sClusterCount = 1
cfg.K8sNodesPerCluster = 3
cfg.K8sPodsPerCluster = 8
cfg.RandomMetrics = false
data := buildFixtureState(cfg)
if len(data.KubernetesClusters) != 1 {
t.Fatalf("expected exactly one kubernetes cluster, got %d", len(data.KubernetesClusters))
}
cluster := data.KubernetesClusters[0]
if len(cluster.Services) == 0 {
t.Fatal("expected kubernetes service inventory")
}
foundPublishedService := false
for _, service := range cluster.Services {
if service.ServiceType != "NodePort" || len(service.ExternalIPs) == 0 {
continue
}
for _, port := range service.Ports {
if port.NodePort > 0 {
foundPublishedService = true
}
}
}
if !foundPublishedService {
t.Fatal("expected kubernetes service inventory to include node-port/external service metadata")
}
if len(cluster.Ingresses) == 0 {
t.Fatal("expected kubernetes ingress inventory")
}
if len(cluster.EndpointSlices) == 0 {
t.Fatal("expected kubernetes endpoint slice inventory")
}
}
func TestMockStateIncludesHostAgents(t *testing.T) {
mustSetEnabled(t, true)
t.Cleanup(func() {
mustSetEnabled(t, false)
})
state := CurrentFixtureGraph().State
if len(state.Hosts) == 0 {
t.Fatalf("expected hosts in mock state, got %d", len(state.Hosts))
}
frontend := state.ToFrontend()
if frontend.Resources == nil {
t.Fatal("expected canonical frontend resources slice to be initialized")
}
if frontend.ConnectedInfrastructure == nil {
t.Fatal("expected canonical connectedInfrastructure slice to be initialized")
}
}
func TestUpdateMetricsMaintainsServiceHealth(t *testing.T) {
cfg := DefaultConfig
cfg.DockerHostCount = 3
cfg.DockerContainersPerHost = 6
data := buildFixtureState(cfg)
updateFixtureStateMetrics(&data, cfg)
for _, host := range data.DockerHosts {
if len(host.Services) == 0 {
continue
}
if host.Swarm == nil {
t.Fatalf("expected swarm metadata for host %s after update", host.ID)
}
for _, svc := range host.Services {
if svc.DesiredTasks < 0 {
t.Fatalf("service %s has negative desired tasks", svc.Name)
}
if svc.RunningTasks < 0 {
t.Fatalf("service %s has negative running tasks", svc.Name)
}
if svc.RunningTasks > svc.DesiredTasks && svc.DesiredTasks > 0 {
t.Fatalf("service %s has running (%d) > desired (%d)", svc.Name, svc.RunningTasks, svc.DesiredTasks)
}
}
}
}
func TestBuildFixtureStateIncludesPMGInstances(t *testing.T) {
cfg := DefaultConfig
data := buildFixtureState(cfg)
if len(data.PMGInstances) == 0 {
t.Fatalf("expected PMG instances in mock data")
}
for _, inst := range data.PMGInstances {
if inst.Name == "" {
t.Fatalf("PMG instance missing name: %+v", inst)
}
if inst.Status == "" {
t.Fatalf("PMG instance missing status: %+v", inst)
}
}
}
func TestCloneStateCopiesPMGInstances(t *testing.T) {
state := models.StateSnapshot{
PMGInstances: []models.PMGInstance{
{ID: "pmg-test", Name: "pmg-test", Status: "online"},
},
}
cloned := cloneState(state)
if len(cloned.PMGInstances) != 1 {
t.Fatalf("expected cloned state to include PMG instances, got %d", len(cloned.PMGInstances))
}
cloned.PMGInstances[0].Name = "modified"
if state.PMGInstances[0].Name == "modified" {
t.Fatal("expected PMG instances to be deep-copied")
}
}
func TestCloneStatePreservesIgnoredInfrastructureEntries(t *testing.T) {
state := models.StateSnapshot{
RemovedDockerHosts: []models.RemovedDockerHost{
{ID: "docker-1", Hostname: "docker.local"},
},
RemovedHostAgents: []models.RemovedHostAgent{
{ID: "agent-1", Hostname: "host.local"},
},
RemovedKubernetesClusters: []models.RemovedKubernetesCluster{
{ID: "cluster-1", Name: "cluster.local"},
},
}
cloned := cloneState(state)
if len(cloned.RemovedDockerHosts) != 1 {
t.Fatalf("expected cloned state to include removed docker hosts, got %d", len(cloned.RemovedDockerHosts))
}
if len(cloned.RemovedHostAgents) != 1 {
t.Fatalf("expected cloned state to include removed host agents, got %d", len(cloned.RemovedHostAgents))
}
if len(cloned.RemovedKubernetesClusters) != 1 {
t.Fatalf("expected cloned state to include removed kubernetes clusters, got %d", len(cloned.RemovedKubernetesClusters))
}
cloned.RemovedDockerHosts[0].Hostname = "mutated-docker.local"
cloned.RemovedHostAgents[0].Hostname = "mutated-host.local"
cloned.RemovedKubernetesClusters[0].Name = "mutated-cluster.local"
if state.RemovedDockerHosts[0].Hostname == "mutated-docker.local" {
t.Fatal("expected removed docker hosts to be copied")
}
if state.RemovedHostAgents[0].Hostname == "mutated-host.local" {
t.Fatal("expected removed host agents to be copied")
}
if state.RemovedKubernetesClusters[0].Name == "mutated-cluster.local" {
t.Fatal("expected removed kubernetes clusters to be copied")
}
}
func TestBuildFixtureStateInitializesIgnoredInfrastructureSlices(t *testing.T) {
data := buildFixtureState(DefaultConfig)
if data.RemovedDockerHosts == nil {
t.Fatal("expected buildFixtureState to initialize RemovedDockerHosts")
}
if data.RemovedHostAgents == nil {
t.Fatal("expected buildFixtureState to initialize RemovedHostAgents")
}
if data.RemovedKubernetesClusters == nil {
t.Fatal("expected buildFixtureState to initialize RemovedKubernetesClusters")
}
}
// collectFixtureIdentities gathers every identity-bearing ID the fixture
// graph produces, keyed by category. Metric series resource IDs in the mock
// metrics store derive from these, so any per-boot churn here orphans stored
// history and breaks resource continuity across backend restarts.
func collectFixtureIdentities(graph FixtureGraph) map[string][]string {
ids := map[string][]string{}
add := func(category, id string) {
ids[category] = append(ids[category], id)
}
state := graph.State
for _, node := range state.Nodes {
add("node", node.ID)
}
for _, vm := range state.VMs {
add("vm", vm.ID)
}
for _, ct := range state.Containers {
add("container", ct.ID)
}
for _, host := range state.Hosts {
add("host", host.ID)
add("host-hostname", host.Hostname)
add("host-machine-id", host.ID+"="+host.MachineID)
}
for _, dockerHost := range state.DockerHosts {
add("docker-host", dockerHost.ID)
add("docker-host-machine-id", dockerHost.ID+"="+dockerHost.MachineID)
add("docker-agent-id", dockerHost.ID+"="+dockerHost.AgentID)
for _, c := range dockerHost.Containers {
add("docker-container", c.ID)
}
}
for _, cluster := range state.KubernetesClusters {
add("k8s-cluster", cluster.ID)
for _, node := range cluster.Nodes {
add("k8s-node", node.UID)
}
for _, pod := range cluster.Pods {
add("k8s-pod", pod.UID)
}
for _, deployment := range cluster.Deployments {
add("k8s-deployment", deployment.UID)
}
for _, svc := range cluster.Services {
add("k8s-service", svc.UID)
}
}
for _, storage := range state.Storage {
add("storage", storage.ID)
}
for _, ceph := range state.CephClusters {
add("ceph-cluster", ceph.ID)
add("ceph-fsid", ceph.FSID)
}
for _, disk := range state.PhysicalDisks {
add("physical-disk", disk.ID)
}
resources, _ := graph.UnifiedResourceSnapshot()
for _, resource := range resources {
if resource.Type == unifiedresources.ResourceTypeAgent {
add("canonical-agent", resource.ID)
}
}
for category := range ids {
sort.Strings(ids[category])
}
return ids
}
// TestFixtureIdentityStableAcrossBoots simulates two backend boots (the
// package-global rand source is in a different state for each build, exactly
// as two freshly seeded processes would be) and requires every identity set
// to match. Guards the generator's "stable IDs" contract end to end.
func TestFixtureIdentityStableAcrossBoots(t *testing.T) {
now := time.Date(2026, time.June, 10, 12, 0, 0, 0, time.UTC)
first := collectFixtureIdentities(buildFixtureGraph(DefaultConfig, now))
second := collectFixtureIdentities(buildFixtureGraph(DefaultConfig, now))
if len(first["canonical-agent"]) == 0 || len(second["canonical-agent"]) == 0 {
t.Fatal("fixture graph must project canonical agents before comparing their identities")
}
categories := map[string]struct{}{}
for category := range first {
categories[category] = struct{}{}
}
for category := range second {
categories[category] = struct{}{}
}
for category := range categories {
diff := diffIdentitySets(first[category], second[category])
if diff != "" {
t.Errorf("%s identities churned across boots:\n%s", category, diff)
}
}
}
func diffIdentitySets(first, second []string) string {
firstSet := make(map[string]struct{}, len(first))
for _, id := range first {
firstSet[id] = struct{}{}
}
secondSet := make(map[string]struct{}, len(second))
for _, id := range second {
secondSet[id] = struct{}{}
}
var lines []string
for _, id := range first {
if _, ok := secondSet[id]; !ok {
lines = append(lines, fmt.Sprintf(" only in first boot: %s", id))
}
}
for _, id := range second {
if _, ok := firstSet[id]; !ok {
lines = append(lines, fmt.Sprintf(" only in second boot: %s", id))
}
}
if len(lines) == 0 {
return ""
}
sort.Strings(lines)
out := ""
for _, line := range lines {
out += line + "\n"
}
return out
}
// Host-relative guest memory history is a byte series derived from fixture
// capacity, so the fixture registries must expose that capacity per guest
// without a caller cloning the whole graph on every lookup.
func TestMemoryTotalForResourceTracksFixtureCapacity(t *testing.T) {
mustSetEnabled(t, true)
t.Cleanup(func() {
mustSetEnabled(t, false)
})
graph := CurrentFixtureGraph()
checked := 0
for _, vm := range graph.State.VMs {
if vm.ID == "" || vm.Memory.Total <= 0 {
continue
}
if got := MemoryTotalForResource("vm", vm.ID); got != float64(vm.Memory.Total) {
t.Fatalf("MemoryTotalForResource(vm, %s) = %f, want %d", vm.ID, got, vm.Memory.Total)
}
checked++
if checked >= 3 {
break
}
}
if checked == 0 {
t.Fatal("fixture graph has no VM with a memory total")
}
for _, ct := range graph.State.Containers {
if ct.ID == "" || ct.Memory.Total <= 0 {
continue
}
if got := MemoryTotalForResource("container", ct.ID); got != float64(ct.Memory.Total) {
t.Fatalf("MemoryTotalForResource(container, %s) = %f, want %d", ct.ID, got, ct.Memory.Total)
}
break
}
if got := MemoryTotalForResource("vm", "definitely-not-a-fixture-guest"); got != 0 {
t.Fatalf("unknown guest capacity = %f, want 0", got)
}
if got := MemoryTotalForResource("vm", " "); got != 0 {
t.Fatalf("blank guest capacity = %f, want 0", got)
}
}
func TestEnsureConfigOnlyMountFixtureSeedsUnknownUsageMount(t *testing.T) {
data := buildFixtureState(DefaultConfig)
var fixtureContainer *models.Container
for i := range data.Containers {
for _, disk := range data.Containers[i].Disks {
if disk.Mountpoint == "/srv/archive" {
fixtureContainer = &data.Containers[i]
}
}
}
if fixtureContainer == nil {
t.Fatal("no container carries the config-only mount fixture (#1477)")
}
if fixtureContainer.Status != "running" {
t.Fatalf("fixture container %s is %s, want running", fixtureContainer.Name, fixtureContainer.Status)
}
var rootfs, archive *models.Disk
for i := range fixtureContainer.Disks {
disk := &fixtureContainer.Disks[i]
switch disk.Mountpoint {
case "/":
rootfs = disk
case "/srv/archive":
archive = disk
}
}
if rootfs == nil || rootfs.Total <= 0 || rootfs.Usage < 0 {
t.Fatalf("fixture container must keep a measured rootfs row, got %+v", rootfs)
}
if archive == nil {
t.Fatal("config-only mount row missing")
}
if archive.Usage != -1 {
t.Fatalf("config-only mount usage = %f, want the -1 unknown sentinel", archive.Usage)
}
if archive.Total != int64(20)*1024*1024*1024 {
t.Fatalf("config-only mount total = %d, want the configured 20 GiB", archive.Total)
}
if archive.Type != "mp0" || archive.Device == "" {
t.Fatalf("config-only mount must keep its mp key and device, got %+v", archive)
}
}
func assertMockDockerInLXCFixture(t *testing.T, data *models.StateSnapshot) []models.DockerHost {
t.Helper()
var nested []models.DockerHost
for _, host := range data.DockerHosts {
if strings.HasPrefix(host.ID, mockProxmoxLXCDockerHostSourcePrefix) {
nested = append(nested, host)
}
}
if len(nested) != 2 {
t.Fatalf("expected 2 nested Docker-in-LXC hosts, got %d", len(nested))
}
guestsBySourceID := make(map[string]models.Container, len(data.Containers))
for _, ct := range data.Containers {
key := fmt.Sprintf("%s%s:%s:%d", mockProxmoxLXCDockerHostSourcePrefix, ct.Instance, ct.Node, ct.VMID)
guestsBySourceID[key] = ct
}
containerCounts := make(map[int]bool)
for _, host := range nested {
guest, ok := guestsBySourceID[host.ID]
if !ok {
t.Fatalf("nested docker host %s has no matching LXC guest", host.ID)
}
if guest.Status != "running" {
t.Fatalf("nested docker host %s bound to non-running guest %s", host.ID, guest.ID)
}
if !guest.HasDocker || guest.DockerCheckedAt.IsZero() {
t.Fatalf("parent LXC %s not marked as probed Docker host", guest.ID)
}
if host.AgentID != host.ID {
t.Fatalf("nested docker host agent id %q must equal its source id %q", host.AgentID, host.ID)
}
if host.Hostname != guest.Name {
t.Fatalf("nested docker host hostname %q should be the guest name %q", host.Hostname, guest.Name)
}
if host.Status != "online" {
t.Fatalf("nested docker host %s bound to a running guest must be online, got %q", host.ID, host.Status)
}
if len(host.Containers) == 0 {
t.Fatalf("nested docker host %s has no containers", host.ID)
}
containerCounts[len(host.Containers)] = true
if len(host.Images) != 0 || len(host.Volumes) != 0 || len(host.Networks) != 0 || host.StorageUsage != nil {
t.Fatalf("nested docker host %s must not fabricate native engine inventory (production pct exec inventory has none)", host.ID)
}
if host.Swarm != nil {
t.Fatalf("nested docker host %s must not join the mock Swarm", host.ID)
}
if !data.ConnectionHealth[dockerConnectionPrefix+host.ID] {
t.Fatalf("nested docker host %s missing connection health entry", host.ID)
}
}
if !containerCounts[1] {
t.Fatalf("expected one single-container nested host so the row cue renders its singular form, got counts %v", containerCounts)
}
return nested
}
func TestFixtureGraphIncludesDockerInLXCFixture(t *testing.T) {
cfg := DefaultConfig
now := time.Now()
graph := buildFixtureGraph(cfg, now)
first := assertMockDockerInLXCFixture(t, &graph.State)
// The demo scenario layer reapplies on every metric tick; the fixture
// must stay idempotent and the scenario's Docker fleet profiles must
// keep their hands off the nested hosts.
graph.UpdateMetrics(cfg, now.Add(time.Minute))
second := assertMockDockerInLXCFixture(t, &graph.State)
firstIDs := map[string]bool{}
for _, host := range first {
firstIDs[host.ID] = true
}
for _, host := range second {
if !firstIDs[host.ID] {
t.Fatalf("nested docker host identity changed across ticks: %s not in first pass", host.ID)
}
}
// Disabling Docker in the mock estate disables the nested fixture too.
cfgNoDocker := DefaultConfig
cfgNoDocker.DockerHostCount = 0
graphNoDocker := buildFixtureGraph(cfgNoDocker, now)
for _, host := range graphNoDocker.State.DockerHosts {
if strings.HasPrefix(host.ID, mockProxmoxLXCDockerHostSourcePrefix) {
t.Fatalf("nested docker host %s generated despite DockerHostCount=0", host.ID)
}
}
for _, ct := range graphNoDocker.State.Containers {
if ct.HasDocker {
t.Fatalf("LXC %s marked HasDocker despite DockerHostCount=0", ct.ID)
}
}
}
// assertMockBackupRunningFixture verifies the running-backup estate invariants
// and returns the set of flagged guest keys ("vm-<vmid>" / "ct-<vmid>").
func assertMockBackupRunningFixture(t *testing.T, state *models.StateSnapshot, now time.Time) map[string]bool {
t.Helper()
flagged := map[string]bool{}
neverBackedUp := false
backedUp := false
for _, vm := range state.VMs {
if !vm.BackupInProgress {
continue
}
flagged[fmt.Sprintf("vm-%d", vm.VMID)] = true
if vm.LastBackup.IsZero() {
neverBackedUp = true
} else {
backedUp = true
}
}
for _, ct := range state.Containers {
if !ct.BackupInProgress {
continue
}
flagged[fmt.Sprintf("ct-%d", ct.VMID)] = true
if ct.LastBackup.IsZero() {
neverBackedUp = true
} else {
backedUp = true
}
}
if len(flagged) == 0 {
t.Fatal("no guest flagged BackupInProgress")
}
if !neverBackedUp || !backedUp {
t.Fatalf("running backups must spread across the age distribution; never=%v backed-up=%v", neverBackedUp, backedUp)
}
guests := len(state.VMs) + len(state.Containers)
if len(flagged) > guests/10 {
t.Fatalf("%d of %d guests flagged running; expected a small fraction", len(flagged), guests)
}
// The first-cycle showcase guests are pinned to specific protection
// stories (failed run, no evidence at all); a running backup would mask
// those states, so the fixture must never pick them.
showcase := map[string]bool{}
for _, profile := range demoVMProfiles {
showcase[profile.Name] = true
}
for _, profile := range demoContainerProfiles {
showcase[profile.Name] = true
}
for _, vm := range state.VMs {
if vm.BackupInProgress && showcase[vm.Name] {
t.Fatalf("showcase guest %s flagged BackupInProgress", vm.Name)
}
}
for _, ct := range state.Containers {
if ct.BackupInProgress && showcase[ct.Name] {
t.Fatalf("showcase guest %s flagged BackupInProgress", ct.Name)
}
}
inflight := map[string]bool{}
for _, backup := range state.PBSBackups {
if !strings.HasPrefix(backup.ID, mockBackupRunningIDPrefix) {
if backup.InProgress {
t.Fatalf("non-fixture PBS backup %s marked InProgress", backup.ID)
}
continue
}
key := fmt.Sprintf("%s-%s", backup.BackupType, backup.VMID)
if inflight[key] {
t.Fatalf("duplicate in-flight PBS backup for %s", key)
}
inflight[key] = true
if !flagged[key] {
t.Fatalf("in-flight PBS backup %s has no flagged guest", backup.ID)
}
if !backup.InProgress || backup.Size != 0 || backup.Verified {
t.Fatalf("in-flight PBS backup %s must be InProgress with no size and no verification: %+v", backup.ID, backup)
}
wantBlob := "qemu-server.conf.blob"
if backup.BackupType == "ct" {
wantBlob = "pct.conf.blob"
}
if len(backup.Files) != 1 || backup.Files[0] != wantBlob {
t.Fatalf("in-flight PBS backup %s files = %v, want only %s", backup.ID, backup.Files, wantBlob)
}
if age := now.Sub(backup.BackupTime); age <= 0 || age > time.Hour {
t.Fatalf("in-flight PBS backup %s should have started minutes ago, started %s before now", backup.ID, age)
}
}
if len(inflight) != len(flagged) {
t.Fatalf("flagged guests = %d but in-flight PBS backups = %d", len(flagged), len(inflight))
}
runningTasks := map[string]bool{}
for _, task := range state.PVEBackups.BackupTasks {
if !strings.HasPrefix(task.ID, mockBackupRunningIDPrefix) {
continue
}
kind := "vm"
if task.Type == "lxc" {
kind = "ct"
}
key := fmt.Sprintf("%s-%d", kind, task.VMID)
if runningTasks[key] {
t.Fatalf("duplicate running task for %s", key)
}
runningTasks[key] = true
if !flagged[key] {
t.Fatalf("running task %s has no flagged guest", task.ID)
}
if task.Status != "running" || !task.EndTime.IsZero() {
t.Fatalf("fixture task %s must be running with no end time: %+v", task.ID, task)
}
if task.ObservedAt.Before(now.Add(-time.Minute)) {
t.Fatalf("fixture task %s ObservedAt %s is stale relative to %s", task.ID, task.ObservedAt, now)
}
}
if len(runningTasks) != len(flagged) {
t.Fatalf("flagged guests = %d but running tasks = %d", len(flagged), len(runningTasks))
}
// The canonical Backups projection must mirror the fixture artifacts.
mirrored := 0
for _, backup := range state.Backups.PBS {
if strings.HasPrefix(backup.ID, mockBackupRunningIDPrefix) {
mirrored++
}
}
if mirrored != len(inflight) {
t.Fatalf("state.Backups.PBS mirrors %d in-flight backups, want %d", mirrored, len(inflight))
}
return flagged
}
func TestFixtureGraphIncludesBackupRunningFixture(t *testing.T) {
cfg := DefaultConfig
now := time.Now()
graph := buildFixtureGraph(cfg, now)
first := assertMockBackupRunningFixture(t, &graph.State, now)
// The demo scenario layer reapplies on every metric tick; the fixture
// must re-select the same guests and replace - not duplicate - its
// in-flight artifacts.
tick := now.Add(time.Minute)
graph.UpdateMetrics(cfg, tick)
second := assertMockBackupRunningFixture(t, &graph.State, tick)
if len(first) != len(second) {
t.Fatalf("running-guest selection changed size across ticks: %d then %d", len(first), len(second))
}
for key := range second {
if !first[key] {
t.Fatalf("running-guest selection changed across ticks: %s not in first pass", key)
}
}
}
func replicationFixtureEstate() ([]models.Node, []models.VM) {
nodes := []models.Node{{Name: "pve1"}, {Name: "pve2"}, {Name: "pve3"}}
vms := make([]models.VM, 0, 9)
for i := 0; i < 9; i++ {
// Every guest lives on pve1, so the old (i+1) modulo target picked the
// source node for every third job.
vms = append(vms, models.VM{VMID: 100 + i, Name: "guest", Node: "pve1", Instance: "cluster"})
}
return nodes, vms
}
func TestGenerateReplicationJobsNeverTargetsTheSourceNode(t *testing.T) {
nodes, vms := replicationFixtureEstate()
jobs := generateReplicationJobs(nodes, vms)
if len(jobs) == 0 {
t.Fatal("expected replication jobs for a three-node estate")
}
for _, job := range jobs {
if job.TargetNode == job.SourceNode {
t.Fatalf("job %s replicates %s to itself", job.JobID, job.SourceNode)
}
}
}
func TestGenerateReplicationJobsNeedsASecondNode(t *testing.T) {
_, vms := replicationFixtureEstate()
if jobs := generateReplicationJobs([]models.Node{{Name: "pve1"}}, vms); len(jobs) != 0 {
t.Fatalf("expected no replication jobs without a target node, got %d", len(jobs))
}
}
func TestRebaseMockReplicationJobsKeepsJobsOnSchedule(t *testing.T) {
nodes, vms := replicationFixtureEstate()
jobs := generateReplicationJobs(nodes, vms)
jobs[0].FailCount = 2
jobs[0].LastSyncStatus = "error"
for i := 1; i < len(jobs); i++ {
jobs[i].FailCount = 0
jobs[i].LastSyncStatus = "ok"
}
start := time.Date(2026, 10, 2, 9, 7, 0, 0, time.UTC)
// A long-running demo used to leave every job hours overdue; walk the
// fixture clock well past one schedule window.
for _, now := range []time.Time{start, start.Add(47 * time.Minute), start.Add(26 * time.Hour)} {
rebaseMockReplicationJobs(jobs, now)
for i, job := range jobs {
if job.LastSyncTime == nil || job.NextSyncTime == nil {
t.Fatalf("job %s lost its sync times", job.JobID)
}
if job.LastSyncUnix != job.LastSyncTime.Unix() || job.NextSyncUnix != job.NextSyncTime.Unix() {
t.Fatalf("job %s unix companions drifted from its sync times", job.JobID)
}
if !job.NextSyncTime.After(now) || job.NextSyncTime.Sub(now) > mockReplicationInterval {
t.Fatalf("job %s next sync %s is not within one interval after %s", job.JobID, job.NextSyncTime, now)
}
age := now.Sub(*job.LastSyncTime)
if i == 0 {
if age < 2*mockReplicationInterval {
t.Fatalf("failing job last synced %s ago, want at least two missed intervals", age)
}
continue
}
if age < 0 || age >= mockReplicationInterval {
t.Fatalf("healthy job %s last synced %s ago, want inside one interval", job.JobID, age)
}
}
}
}
func TestRebaseMockReplicationJobsLeavesUnchangedJobsAlone(t *testing.T) {
nodes, vms := replicationFixtureEstate()
jobs := generateReplicationJobs(nodes, vms)
start := time.Date(2026, 10, 2, 9, 0, 0, 0, time.UTC)
rebaseMockReplicationJobs(jobs, start)
before := append([]models.ReplicationJob(nil), jobs...)
// One second later no job's phase can have passed for all of them, and a
// job whose phase has not passed must keep its polled stamp and pointers so
// the refresh does not manufacture a change for every job on every tick.
rebaseMockReplicationJobs(jobs, start.Add(time.Second))
unchanged := 0
for i := range jobs {
if jobs[i].LastSyncUnix != before[i].LastSyncUnix {
continue
}
unchanged++
if !jobs[i].LastPolled.Equal(before[i].LastPolled) || jobs[i].LastSyncTime != before[i].LastSyncTime {
t.Fatalf("job %s was rewritten although its schedule phase had not passed", jobs[i].JobID)
}
}
if unchanged < len(jobs)-1 {
t.Fatalf("expected at most one job to roll over in one second, %d of %d stayed put", unchanged, len(jobs))
}
}
func TestBuildFixtureStateMergesSharedPBSStorageLikeThePoller(t *testing.T) {
cfg := DefaultConfig
cfg.NodeCount = 5
data := buildFixtureState(cfg)
// Issue 1049: the poller folds a shared storage reported by every node
// into one cluster-scoped row. The fixture must not fan it back out into
// one row per node, each with its own usage figure.
seen := map[string]models.Storage{}
for _, storage := range data.Storage {
if storage.Type != "pbs" {
continue
}
key := storage.Instance + "/" + storage.Name
if _, dup := seen[key]; dup {
t.Fatalf("shared PBS storage %s appears more than once", key)
}
seen[key] = storage
if storage.Node != "cluster" || storage.NodeCount < 2 || len(storage.Nodes) != storage.NodeCount {
t.Fatalf("shared PBS storage %s is not cluster-scoped: node=%q nodes=%v count=%d", key, storage.Node, storage.Nodes, storage.NodeCount)
}
if !strings.HasSuffix(storage.ID, "-cluster-"+storage.Name) {
t.Fatalf("shared PBS storage %s keeps a node-scoped id %q", key, storage.ID)
}
}
if len(seen) == 0 {
t.Fatal("expected shared PBS storage in the fixture")
}
}
// Mock guests report where their memory reading came from, the way a live
// estate does, so the UI qualifies mock readings instead of treating every
// guest as unannotated ("Freshness unknown").
func TestGeneratedGuestMemoryProvenanceMatchesPowerStateAndOS(t *testing.T) {
cfg := DefaultConfig
cfg.StoppedPercent = 0.3
now := time.Now()
sources := map[string]int{}
for i := 0; i < 300; i++ {
vm := generateVM("node-01", "mock-cluster", 1000+i, cfg)
obs := vm.Memory.Observation
if vm.Status != "running" {
if obs.State != "unavailable" || obs.Source != "powered-off" {
t.Fatalf("stopped VM observation = %+v, want unavailable powered-off", obs)
}
continue
}
sources[obs.Source]++
if obs.State != "current" || obs.ObservedAt.IsZero() || obs.ObservedAt.After(now.Add(time.Minute)) {
t.Fatalf("running VM observation = %+v, want a current reading observed now", obs)
}
switch obs.Source {
case "status-mem":
// Proxmox's value carries no used | cache | free split.
if vm.Memory.Cache != 0 {
t.Fatalf("cache-inclusive VM reported a measured cache split: %+v", vm.Memory)
}
case "guest-agent-meminfo":
// The QEMU guest agent reads Linux /proc/meminfo only.
if strings.Contains(strings.ToLower(vm.OSName), "windows") {
t.Fatalf("Windows VM %q claims Linux meminfo provenance", vm.OSName)
}
case "agent":
default:
t.Fatalf("running VM has unexpected memory source %q", obs.Source)
}
}
if sources["status-mem"] == 0 || sources["guest-agent-meminfo"] == 0 || sources["agent"] == 0 {
t.Fatalf("expected cache-inclusive, guest-agent and Pulse-agent VMs in the estate, got %v", sources)
}
sources = map[string]int{}
for i := 0; i < 300; i++ {
ct := generateContainer("node-01", "mock-cluster", 2000+i, cfg)
obs := ct.Memory.Observation
if ct.Status != "running" {
if obs.State != "unavailable" || obs.Source != "powered-off" {
t.Fatalf("stopped container observation = %+v, want unavailable powered-off", obs)
}
continue
}
sources[obs.Source]++
if obs.State != "current" || obs.ObservedAt.IsZero() {
t.Fatalf("running container observation = %+v, want a current reading", obs)
}
if obs.Source != "cluster-resources" && obs.Source != "agent" {
t.Fatalf("running container has unexpected memory source %q", obs.Source)
}
}
if sources["cluster-resources"] == 0 || sources["agent"] == 0 {
t.Fatalf("expected cgroup and Pulse-agent containers in the estate, got %v", sources)
}
}
func TestMockGuestMemoryObservationFollowsScenarioPowerState(t *testing.T) {
cases := []struct {
name string
before models.MemoryObservation
running bool
want models.MemoryObservation
}{
{
name: "forced stopped loses its reading",
before: models.MemoryObservation{State: "current", Source: "status-mem", ObservedAt: time.Now()},
running: false,
want: models.MemoryObservation{State: "unavailable", Source: "powered-off"},
},
{
name: "forced running after generation gets a current source",
before: models.MemoryObservation{State: "unavailable", Source: "powered-off"},
running: true,
want: models.MemoryObservation{State: "current", Source: "agent"},
},
{
name: "a running guest keeps its own source",
before: models.MemoryObservation{State: "current", Source: "status-mem", ObservedAt: time.Now()},
running: true,
want: models.MemoryObservation{State: "current", Source: "status-mem"},
},
{
name: "a retained reading on a running guest is kept",
before: models.MemoryObservation{State: "last-known", Source: "status-mem", ObservedAt: time.Now().Add(-time.Hour)},
running: true,
want: models.MemoryObservation{State: "last-known", Source: "status-mem"},
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
mem := models.Memory{Observation: tc.before}
mockSettleGuestMemoryObservation(&mem, tc.running, "agent")
got := mem.Observation
if got.State != tc.want.State || got.Source != tc.want.Source {
t.Fatalf("observation = %+v, want state %q source %q", got, tc.want.State, tc.want.Source)
}
if tc.want.State == "current" && got.ObservedAt.IsZero() {
t.Fatalf("a current observation needs its observation time: %+v", got)
}
})
}
}
func TestFixtureRefreshMovesCurrentGuestMemoryObservationTime(t *testing.T) {
for _, randomMetrics := range []bool{true, false} {
name := "random metrics"
if !randomMetrics {
name = "static metrics"
}
t.Run(name, func(t *testing.T) {
cfg := DefaultConfig
cfg.RandomMetrics = randomMetrics
stale := time.Now().Add(-time.Hour)
refreshAt := time.Now().Truncate(time.Second)
data := &models.StateSnapshot{
VMs: []models.VM{
{ID: "vm-current", Node: "node-01", Status: "running", Memory: models.Memory{
Total: 100, Observation: models.MemoryObservation{State: "current", Source: "status-mem", ObservedAt: stale},
}},
{ID: "vm-retained", Node: "node-01", Status: "running", Memory: models.Memory{
Total: 100, Observation: models.MemoryObservation{State: "last-known", Source: "status-mem", ObservedAt: stale},
}},
{ID: "vm-stopped", Node: "node-01", Status: "stopped", Memory: models.Memory{
Total: 100, Observation: models.MemoryObservation{State: "unavailable", Source: "powered-off"},
}},
},
Containers: []models.Container{
{ID: "ct-current", Node: "node-01", Status: "running", Memory: models.Memory{
Total: 100, Observation: models.MemoryObservation{State: "current", Source: "cluster-resources", ObservedAt: stale},
}},
},
}
updateFixtureStateMetricsAt(data, cfg, refreshAt)
if got := data.VMs[0].Memory.Observation; !got.ObservedAt.Equal(refreshAt) || got.Source != "status-mem" {
t.Fatalf("current VM observation = %+v, want source kept and time %v", got, refreshAt)
}
if got := data.VMs[1].Memory.Observation; !got.ObservedAt.Equal(stale) || got.State != "last-known" {
t.Fatalf("a retained observation keeps its own state and time, got %+v", got)
}
if got := data.VMs[2].Memory.Observation; !got.ObservedAt.IsZero() || got.Source != "powered-off" {
t.Fatalf("a stopped guest has no observation time to move, got %+v", got)
}
if got := data.Containers[0].Memory.Observation; !got.ObservedAt.Equal(refreshAt) || got.Source != "cluster-resources" {
t.Fatalf("current container observation = %+v, want source kept and time %v", got, refreshAt)
}
})
}
}
// The mock update states mirror what the agent reports. A local image has no
// RepoDigest, so its status carries the sentinel and no digests, and is never
// an available update.
func TestGenerateDockerContainerUpdateStatusLocalImageShape(t *testing.T) {
now := time.Date(2026, 10, 21, 12, 0, 0, 0, time.UTC)
seen := false
for i := 0; i < 5000 && !seen; i++ {
status := generateDockerContainerUpdateStatus(now)
if status == nil || status.Error != "local image" {
continue
}
seen = true
if status.UpdateAvailable || status.CurrentDigest != "" || status.LatestDigest != "" {
t.Fatalf("local image status = %+v, want no update and no digests", status)
}
}
if !seen {
t.Fatal("generator never produced the local image state")
}
}
// The mock native inventory must look like an ingested agent report: ids are
// "<chip>/<channel>", sorted and unique, status matches the reading, and only
// some machines are Intel Macs with applesmc sensors (an invalid one among
// them) so the Thresholds sensor section has realistic rows to show.
func TestMockHostNativeSensorsMatchIngestedInventoryShape(t *testing.T) {
now := time.Date(2026, 11, 1, 10, 0, 0, 0, time.UTC)
appleSMCHosts := 0
for i := 1; i <= 40; i++ {
hostID := fmt.Sprintf("host-linux-%d", i)
sensors := mockHostNativeSensors(hostID, now)
if len(sensors) == 0 {
t.Fatalf("%s: empty native inventory", hostID)
}
seen := make(map[string]bool, len(sensors))
hasAppleSMC := false
for idx, sensor := range sensors {
if sensor.ID != sensor.Chip+"/"+sensor.Channel {
t.Fatalf("%s: id %q is not chip/channel", hostID, sensor.ID)
}
if seen[sensor.ID] {
t.Fatalf("%s: duplicate id %q", hostID, sensor.ID)
}
seen[sensor.ID] = true
if idx > 0 && sensors[idx-1].ID >= sensor.ID {
t.Fatalf("%s: inventory not sorted at %q", hostID, sensor.ID)
}
if sensor.Value == nil {
t.Fatalf("%s: %s has no value", hostID, sensor.ID)
}
switch sensor.Kind {
case "temperature":
valid := *sensor.Value >= -55 && *sensor.Value <= 150
if valid != (sensor.Status == "ok") {
t.Fatalf("%s: %s reads %v with status %q", hostID, sensor.ID, *sensor.Value, sensor.Status)
}
if sensor.Unit != "celsius" {
t.Fatalf("%s: %s unit %q", hostID, sensor.ID, sensor.Unit)
}
case "fan":
if sensor.Unit != "rpm" || sensor.Status != "ok" {
t.Fatalf("%s: fan %s unit %q status %q", hostID, sensor.ID, sensor.Unit, sensor.Status)
}
default:
t.Fatalf("%s: %s kind %q", hostID, sensor.ID, sensor.Kind)
}
if strings.HasPrefix(sensor.Chip, "applesmc-") {
hasAppleSMC = true
}
}
if hasAppleSMC {
appleSMCHosts++
if !seen["applesmc-isa-0300/temp1"] || !seen["applesmc-isa-0300/temp2"] {
t.Fatalf("%s: applesmc inventory missing TB0T or the invalid TB1T", hostID)
}
}
if again := mockHostNativeSensors(hostID, now); len(again) != len(sensors) {
t.Fatalf("%s: inventory size changed between calls: %d then %d", hostID, len(sensors), len(again))
}
}
if appleSMCHosts == 0 || appleSMCHosts == 40 {
t.Fatalf("applesmc hosts = %d of 40, want some but not all", appleSMCHosts)
}
}
// The ordinary host update refreshes a native inventory with its CPU sensors,
// and leaves hosts without one alone.
func TestUpdateHostsRefreshesNativeSensorInventory(t *testing.T) {
start := time.Date(2026, 11, 1, 10, 0, 0, 0, time.UTC)
later := start.Add(37 * time.Minute)
data := &models.StateSnapshot{Hosts: []models.Host{
{ID: "host-linux-1", Status: "online", Sensors: models.HostSensorSummary{
TemperatureCelsius: map[string]float64{"cpu_package": 40},
Native: mockHostNativeSensors("host-linux-1", start),
}},
{ID: "host-windows-1", Status: "online"},
}}
updateHosts(data, MockConfig{RandomMetrics: true}, later)
want := mockHostNativeSensors("host-linux-1", later)
got := data.Hosts[0].Sensors.Native
if len(got) != len(want) {
t.Fatalf("native inventory has %d entries, want %d", len(got), len(want))
}
for i := range want {
if got[i].ID != want[i].ID || *got[i].Value != *want[i].Value || got[i].Status != want[i].Status {
t.Fatalf("entry %d = %s %v %s, want %s %v %s", i, got[i].ID, *got[i].Value, got[i].Status, want[i].ID, *want[i].Value, want[i].Status)
}
}
if data.Hosts[1].Sensors.Native != nil {
t.Fatalf("host without an inventory gained one: %+v", data.Hosts[1].Sensors.Native)
}
}
// Mock PBS servers report job results so the Jobs column and datastore job
// alerts have data: every job succeeded except the secondary's sync (#1815).
func TestGeneratePBSInstancesReportsJobResultsWithOneFailedSync(t *testing.T) {
instances := generatePBSInstances()
failed := 0
for _, instance := range instances {
stores := make(map[string]bool, len(instance.Datastores))
for _, datastore := range instance.Datastores {
stores[datastore.Name] = true
}
if len(instance.JobHealthEvidence) == 0 {
t.Fatalf("%s reports no job results", instance.Name)
}
for _, job := range instance.JobHealthEvidence {
if !stores[job.Store] {
t.Fatalf("%s job %s names unknown datastore %q", instance.Name, job.ID, job.Store)
}
if job.Posture == "critical" {
failed++
if job.Family != "sync" || job.PostureReason != "last-run-failed" || job.Remote == "" {
t.Fatalf("unexpected failed job %+v", job)
}
}
}
}
if failed != 1 {
t.Fatalf("failed jobs = %d, want exactly one", failed)
}
}