Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

7.2 KB
Raw Blame
package models

import (
	"fmt"
	"sync"
	"testing"
	"time"
)

func TestStateConcurrentSnapshots(t *testing.T) {
	state := NewState()

	const iterations = 1000

	var wg sync.WaitGroup
	wg.Add(3)

	go func() {
		defer wg.Done()
		for i := 0; i < iterations; i++ {
			idx := i
			nodes := []Node{{
				ID:       fmt.Sprintf("node-%d", idx),
				Name:     fmt.Sprintf("node-%d", idx),
				Instance: "test-instance",
				Status:   "online",
				CPU:      float64(idx % 100),
				Memory: Memory{
					Total: 1024 * 1024,
					Used:  int64(idx % 1024),
				},
				Disk: Disk{
					Total: 1024 * 1024 * 10,
					Used:  int64(idx % (1024 * 10)),
				},
				LastSeen: time.Now(),
			}}
			vms := []VM{{
				ID:         fmt.Sprintf("vm-%d", idx),
				VMID:       idx,
				Name:       fmt.Sprintf("vm-%d", idx),
				Node:       nodes[0].Name,
				Instance:   "test-instance",
				CPU:        float64(idx % 100),
				Memory:     Memory{Total: 512 * 1024, Used: int64(idx % 512)},
				Disk:       Disk{Total: 1024 * 1024, Used: int64(idx % 1024)},
				NetworkIn:  int64(idx),
				NetworkOut: int64(idx),
				DiskRead:   int64(idx),
				DiskWrite:  int64(idx),
				LastSeen:   time.Now(),
			}}
			containers := []Container{{
				ID:         fmt.Sprintf("ct-%d", idx),
				VMID:       idx,
				Name:       fmt.Sprintf("ct-%d", idx),
				Node:       nodes[0].Name,
				Instance:   "test-instance",
				CPU:        float64(idx % 100),
				Memory:     Memory{Total: 256 * 1024, Used: int64(idx % 256)},
				Disk:       Disk{Total: 512 * 1024, Used: int64(idx % 512)},
				NetworkIn:  int64(idx),
				NetworkOut: int64(idx),
				DiskRead:   int64(idx),
				DiskWrite:  int64(idx),
				LastSeen:   time.Now(),
			}}
			storage := []Storage{{
				ID:       fmt.Sprintf("storage-%d", idx),
				Name:     fmt.Sprintf("storage-%d", idx),
				Node:     nodes[0].Name,
				Instance: "test-instance",
				Type:     "lvm",
				Status:   "available",
				Total:    1024 * 1024 * 1024,
				Used:     int64(idx % (1024 * 1024)),
				Free:     1024*1024*1024 - int64(idx%(1024*1024)),
				Usage:    float64(idx % 100),
			}}
			alerts := []Alert{{
				ID:           fmt.Sprintf("alert-%d", idx),
				Type:         "cpu",
				Level:        "warning",
				ResourceID:   vms[0].ID,
				ResourceName: vms[0].Name,
				Node:         nodes[0].Name,
				Instance:     "test-instance",
				Message:      "CPU usage high",
				Value:        float64(idx % 100),
				Threshold:    80,
				StartTime:    time.Now(),
			}}

			state.UpdateNodes(nodes)
			state.UpdateVMs(vms)
			state.UpdateContainers(containers)
			state.UpdateStorage(storage)
			state.UpdateActiveAlerts(alerts)
		}
	}()

	go func() {
		defer wg.Done()
		for i := 0; i < iterations; i++ {
			snapshot := state.GetSnapshot()
			_ = snapshot.ToFrontend()
		}
	}()

	go func() {
		defer wg.Done()
		for i := 0; i < iterations; i++ {
			_ = state.ToFrontend()
		}
	}()

	wg.Wait()
}

// TestStateMultiInstanceConcurrentUpdates simulates the real-world scenario where
// multiple PVE instances (like delly.lan cluster + pi standalone) update nodes concurrently.
// This is the pattern that caused the node count to alternate between 1 and 3.
func TestStateMultiInstanceConcurrentUpdates(t *testing.T) {
	state := NewState()

	const iterations = 1000
	const numInstances = 3

	var wg sync.WaitGroup

	// Simulate multiple PVE instances updating concurrently (like delly.lan + pi)
	for instance := 0; instance < numInstances; instance++ {
		wg.Add(1)
		go func(instanceID int) {
			defer wg.Done()
			instanceName := fmt.Sprintf("instance-%d", instanceID)

			for i := 0; i < iterations; i++ {
				// Each instance updates its own nodes
				nodesPerInstance := instanceID + 1 // instance-0 has 1 node, instance-1 has 2 nodes, etc.
				nodes := make([]Node, nodesPerInstance)
				for n := 0; n < nodesPerInstance; n++ {
					nodes[n] = Node{
						ID:       fmt.Sprintf("%s-node-%d", instanceName, n),
						Name:     fmt.Sprintf("node-%d", n),
						Instance: instanceName,
						Status:   "online",
						CPU:      float64(i % 100),
						Memory: Memory{
							Total: 1024 * 1024,
							Used:  int64(i % 1024),
						},
						Disk: Disk{
							Total: 1024 * 1024 * 10,
							Used:  int64(i % (1024 * 10)),
						},
						LastSeen: time.Now(),
					}
				}

				// Update nodes for this specific instance (merges with other instances)
				state.UpdateNodesForInstance(instanceName, nodes)

				// Also update VMs and containers for this instance
				vms := make([]VM, nodesPerInstance)
				for v := 0; v < nodesPerInstance; v++ {
					vms[v] = VM{
						ID:       fmt.Sprintf("%s-vm-%d", instanceName, v),
						VMID:     v,
						Name:     fmt.Sprintf("vm-%d", v),
						Node:     nodes[0].Name,
						Instance: instanceName,
						Status:   "running",
						CPU:      float64(i % 100),
						Memory:   Memory{Total: 512 * 1024, Used: int64(i % 512)},
						Disk:     Disk{Total: 1024 * 1024, Used: int64(i % 1024)},
						LastSeen: time.Now(),
					}
				}
				state.UpdateVMsForInstance(instanceName, vms)

				containers := make([]Container, nodesPerInstance)
				for c := 0; c < nodesPerInstance; c++ {
					containers[c] = Container{
						ID:       fmt.Sprintf("%s-ct-%d", instanceName, c),
						VMID:     c,
						Name:     fmt.Sprintf("ct-%d", c),
						Node:     nodes[0].Name,
						Instance: instanceName,
						Status:   "running",
						CPU:      float64(i % 100),
						Memory:   Memory{Total: 256 * 1024, Used: int64(i % 256)},
						Disk:     Disk{Total: 512 * 1024, Used: int64(i % 512)},
						LastSeen: time.Now(),
					}
				}
				state.UpdateContainersForInstance(instanceName, containers)
			}
		}(instance)
	}

	// Simulate concurrent reads (like GetState() calls from API handlers and WebSocket broadcasts)
	for reader := 0; reader < 3; reader++ {
		wg.Add(1)
		go func() {
			defer wg.Done()
			for i := 0; i < iterations; i++ {
				snapshot := state.GetSnapshot()

				// Verify state consistency - all instances' data should be present
				// After initial updates, we should have 1 + 2 + 3 = 6 total nodes
				if i > 100 { // Give time for initial updates
					nodeCount := len(snapshot.Nodes)
					if nodeCount != 0 && nodeCount != 6 {
						// It's OK to have 0 nodes initially, or partial counts during ramp-up,
						// but once stable we should have all 6 nodes
						// Don't fail here - just track for race detector
					}
				}
			}
		}()
	}

	wg.Wait()

	// Final verification - all instances' nodes should be present
	finalSnapshot := state.GetSnapshot()
	if len(finalSnapshot.Nodes) != 6 {
		t.Errorf("Expected 6 nodes (1+2+3), got %d", len(finalSnapshot.Nodes))
	}

	// Verify each instance's nodes are present
	nodesByInstance := make(map[string]int)
	for _, node := range finalSnapshot.Nodes {
		nodesByInstance[node.Instance]++
	}

	for instance := 0; instance < numInstances; instance++ {
		instanceName := fmt.Sprintf("instance-%d", instance)
		expectedCount := instance + 1
		actualCount := nodesByInstance[instanceName]
		if actualCount != expectedCount {
			t.Errorf("Instance %s: expected %d nodes, got %d", instanceName, expectedCount, actualCount)
		}
	}

	// Verify VMs and containers too
	if len(finalSnapshot.VMs) != 6 {
		t.Errorf("Expected 6 VMs, got %d", len(finalSnapshot.VMs))
	}
	if len(finalSnapshot.Containers) != 6 {
		t.Errorf("Expected 6 containers, got %d", len(finalSnapshot.Containers))
	}
}