Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

18.0 KB
Raw Blame
package monitoring

import (
	"context"
	"fmt"
	"math"
	"testing"
	"time"

	"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
	"github.com/rcourtman/pulse-go-rewrite/internal/config"
	"github.com/rcourtman/pulse-go-rewrite/internal/models"
	"github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"
)

// fakeStorageClient provides minimal PVE responses needed by the optimized storage poller.
type fakeStorageClient struct {
	allStorage     []proxmox.Storage
	storageByNode  map[string][]proxmox.Storage
	zfsPoolsByNode map[string][]proxmox.ZFSPoolInfo
}

func (f *fakeStorageClient) GetNodes(ctx context.Context) ([]proxmox.Node, error) {
	return nil, nil
}

func (f *fakeStorageClient) GetNodeStatus(ctx context.Context, node string) (*proxmox.NodeStatus, error) {
	return nil, nil
}

func (f *fakeStorageClient) GetNodeRRDData(ctx context.Context, node string, timeframe string, cf string, ds []string) ([]proxmox.NodeRRDPoint, error) {
	return nil, nil
}

func (f *fakeStorageClient) GetLXCRRDData(ctx context.Context, node string, vmid int, timeframe string, cf string, ds []string) ([]proxmox.GuestRRDPoint, error) {
	return nil, nil
}

func (f *fakeStorageClient) GetVMRRDData(ctx context.Context, node string, vmid int, timeframe string, cf string, ds []string) ([]proxmox.GuestRRDPoint, error) {
	return nil, nil
}

func (f *fakeStorageClient) GetVMs(ctx context.Context, node string) ([]proxmox.VM, error) {
	return nil, nil
}

func (f *fakeStorageClient) GetContainers(ctx context.Context, node string) ([]proxmox.Container, error) {
	return nil, nil
}

func (f *fakeStorageClient) GetStorage(ctx context.Context, node string) ([]proxmox.Storage, error) {
	if storages, ok := f.storageByNode[node]; ok {
		return storages, nil
	}
	return nil, fmt.Errorf("unexpected node: %s", node)
}

func (f *fakeStorageClient) GetAllStorage(ctx context.Context) ([]proxmox.Storage, error) {
	return f.allStorage, nil
}

func (f *fakeStorageClient) GetTaskLog(ctx context.Context, node, upid string) ([]proxmox.TaskLogLine, error) {
	return nil, nil
}

func (f *fakeStorageClient) GetBackupTasks(ctx context.Context) ([]proxmox.Task, error) {
	return nil, nil
}

func (f *fakeStorageClient) GetReplicationStatus(ctx context.Context) ([]proxmox.ReplicationJob, error) {
	return nil, nil
}

func (f *fakeStorageClient) GetStorageContent(ctx context.Context, node, storage string) ([]proxmox.StorageContent, error) {
	return nil, nil
}

func (f *fakeStorageClient) GetVMSnapshots(ctx context.Context, node string, vmid int) ([]proxmox.Snapshot, error) {
	return nil, nil
}

func (f *fakeStorageClient) GetContainerSnapshots(ctx context.Context, node string, vmid int) ([]proxmox.Snapshot, error) {
	return nil, nil
}

func (f *fakeStorageClient) GetVMStatus(ctx context.Context, node string, vmid int) (*proxmox.VMStatus, error) {
	return nil, nil
}

func (f *fakeStorageClient) GetContainerStatus(ctx context.Context, node string, vmid int) (*proxmox.Container, error) {
	return nil, nil
}

func (f *fakeStorageClient) GetContainerConfig(ctx context.Context, node string, vmid int) (map[string]interface{}, error) {
	return nil, nil
}
func (f *fakeStorageClient) GetContainerInterfaces(ctx context.Context, node string, vmid int) ([]proxmox.ContainerInterface, error) {
	return nil, nil
}

func (f *fakeStorageClient) GetClusterResources(ctx context.Context, resourceType string) ([]proxmox.ClusterResource, error) {
	return nil, nil
}

func (f *fakeStorageClient) IsClusterMember(ctx context.Context) (bool, error) {
	return false, nil
}

func (f *fakeStorageClient) GetVMFSInfo(ctx context.Context, node string, vmid int) ([]proxmox.VMFileSystem, error) {
	return nil, nil
}

func (f *fakeStorageClient) GetVMNetworkInterfaces(ctx context.Context, node string, vmid int) ([]proxmox.VMNetworkInterface, error) {
	return nil, nil
}

func (f *fakeStorageClient) GetVMAgentInfo(ctx context.Context, node string, vmid int) (map[string]interface{}, error) {
	return nil, nil
}

func (f *fakeStorageClient) GetVMAgentVersion(ctx context.Context, node string, vmid int) (string, error) {
	return "", nil
}

func (f *fakeStorageClient) GetZFSPoolStatus(ctx context.Context, node string) ([]proxmox.ZFSPoolStatus, error) {
	return nil, nil
}

func (f *fakeStorageClient) GetZFSPoolsWithDetails(ctx context.Context, node string) ([]proxmox.ZFSPoolInfo, error) {
	if pools, ok := f.zfsPoolsByNode[node]; ok {
		return pools, nil
	}
	return nil, nil
}

func (f *fakeStorageClient) GetDisks(ctx context.Context, node string) ([]proxmox.Disk, error) {
	return nil, nil
}

func (f *fakeStorageClient) GetCephStatus(ctx context.Context) (*proxmox.CephStatus, error) {
	return nil, nil
}

func (f *fakeStorageClient) GetCephDF(ctx context.Context) (*proxmox.CephDF, error) {
	return nil, nil
}

func (f *fakeStorageClient) GetNodePendingUpdates(ctx context.Context, node string) ([]proxmox.AptPackage, error) {
	return nil, nil
}

func TestPollStorageWithNodesOptimizedRecordsMetricsAndAlerts(t *testing.T) {
	t.Setenv("PULSE_DATA_DIR", t.TempDir())

	monitor := &Monitor{
		state:          &models.State{},
		metricsHistory: NewMetricsHistory(16, time.Hour),
		alertManager:   alerts.NewManager(),
	}
	t.Cleanup(func() {
		monitor.alertManager.Stop()
	})

	// Ensure storage alerts trigger immediately for the test.
	cfg := monitor.alertManager.GetConfig()
	cfg.MinimumDelta = 0
	if cfg.TimeThresholds == nil {
		cfg.TimeThresholds = make(map[string]int)
	}
	cfg.TimeThresholds["storage"] = 0
	monitor.alertManager.UpdateConfig(cfg)

	storage := proxmox.Storage{
		Storage:   "local",
		Type:      "dir",
		Content:   "images",
		Active:    1,
		Enabled:   1,
		Shared:    0,
		Total:     1000,
		Used:      900,
		Available: 100,
	}

	client := &fakeStorageClient{
		allStorage: []proxmox.Storage{storage},
		storageByNode: map[string][]proxmox.Storage{
			"node1": {storage},
		},
	}

	nodes := []proxmox.Node{
		{Node: "node1", Status: "online"},
	}

	monitor.pollStorageWithNodes(context.Background(), "inst1", client, nodes)

	metrics := monitor.metricsHistory.GetAllStorageMetrics("inst1-node1-local", time.Minute)
	if len(metrics["usage"]) != 1 {
		t.Fatalf("expected one usage metric entry, got %d", len(metrics["usage"]))
	}
	if len(metrics["used"]) != 1 {
		t.Fatalf("expected one used metric entry, got %d", len(metrics["used"]))
	}
	if len(metrics["total"]) != 1 {
		t.Fatalf("expected one total metric entry, got %d", len(metrics["total"]))
	}
	if len(metrics["avail"]) != 1 {
		t.Fatalf("expected one avail metric entry, got %d", len(metrics["avail"]))
	}

	if diff := math.Abs(metrics["usage"][0].Value - 90); diff > 0.001 {
		t.Fatalf("expected usage metric 90, diff %.4f", diff)
	}
	if diff := math.Abs(metrics["used"][0].Value - 900); diff > 0.001 {
		t.Fatalf("expected used metric 900, diff %.4f", diff)
	}
	if diff := math.Abs(metrics["total"][0].Value - 1000); diff > 0.001 {
		t.Fatalf("expected total metric 1000, diff %.4f", diff)
	}
	if diff := math.Abs(metrics["avail"][0].Value - 100); diff > 0.001 {
		t.Fatalf("expected avail metric 100, diff %.4f", diff)
	}

	alerts := monitor.alertManager.GetActiveAlerts()
	found := false
	for _, alert := range alerts {
		if alert.Instance == "inst1" &&
			alert.Node == "node1" &&
			(alert.ResourceID == "inst1-node1-local" ||
				alert.ResourceName == "local" ||
				alert.ID == "inst1-node1-local::inst1-node1-local-usage") {
			found = true
			break
		}
	}
	if !found {
		t.Fatalf("expected storage usage alert to be active, alerts=%+v", alerts)
	}
}

func TestPollStorageWithNodesOptimizedClearsStaleStorageAlertsWhenIdentityChanges(t *testing.T) {
	t.Setenv("PULSE_DATA_DIR", t.TempDir())

	monitor := &Monitor{
		state:          &models.State{},
		metricsHistory: NewMetricsHistory(16, time.Hour),
		alertManager:   alerts.NewManager(),
	}
	t.Cleanup(func() {
		monitor.alertManager.Stop()
	})

	cfg := monitor.alertManager.GetConfig()
	cfg.MinimumDelta = 0
	if cfg.TimeThresholds == nil {
		cfg.TimeThresholds = make(map[string]int)
	}
	cfg.TimeThresholds["storage"] = 0
	cfg.StorageDefault = alerts.HysteresisThreshold{Trigger: 80, Clear: 70}
	monitor.alertManager.UpdateConfig(cfg)

	nodes := []proxmox.Node{{Node: "node1", Status: "online"}}

	hasStorageAlert := func(resourceID string) bool {
		for _, alert := range monitor.alertManager.GetActiveAlerts() {
			if alert.ResourceID == resourceID {
				return true
			}
		}
		return false
	}

	oldStorage := proxmox.Storage{
		Storage: "local", Type: "dir", Content: "images",
		Active: 1, Enabled: 1, Shared: 0,
		Total: 1000, Used: 900, Available: 100,
	}
	oldClient := &fakeStorageClient{
		allStorage:    []proxmox.Storage{oldStorage},
		storageByNode: map[string][]proxmox.Storage{"node1": {oldStorage}},
	}
	monitor.pollStorageWithNodes(context.Background(), "inst1", oldClient, nodes)

	if !hasStorageAlert("inst1-node1-local") {
		t.Fatal("expected initial storage usage alert to be active")
	}

	// Next poll: "local" is gone and replaced by a below-threshold "rootfs".
	newStorage := proxmox.Storage{
		Storage: "rootfs", Type: "dir", Content: "images",
		Active: 1, Enabled: 1, Shared: 0,
		Total: 1000, Used: 200, Available: 800,
	}
	newClient := &fakeStorageClient{
		allStorage:    []proxmox.Storage{newStorage},
		storageByNode: map[string][]proxmox.Storage{"node1": {newStorage}},
	}
	monitor.pollStorageWithNodes(context.Background(), "inst1", newClient, nodes)

	if hasStorageAlert("inst1-node1-local") {
		t.Fatal("expected stale storage usage alert to be cleared after storage identity changed")
	}
	if hasStorageAlert("inst1-node1-rootfs") {
		t.Fatal("expected no usage alert for replacement storage below threshold")
	}
}

func TestPollStorageWithNodesSynthesizesSharedClusterOnlyStorage(t *testing.T) {
	monitor := &Monitor{
		state: models.NewState(),
		config: &config.Config{
			PVEInstances: []config.PVEInstance{
				{
					Name:        "inst1",
					IsCluster:   true,
					ClusterName: "cluster-a",
				},
			},
		},
	}

	client := &fakeStorageClient{
		allStorage: []proxmox.Storage{
			{
				Storage:   "cephfs",
				Type:      "cephfs",
				Content:   "images,backup",
				Shared:    1,
				Total:     1000,
				Used:      100,
				Available: 900,
				Nodes:     "node1,node2",
			},
		},
		storageByNode: map[string][]proxmox.Storage{
			"node1": {
				{Storage: "local", Type: "dir", Content: "images", Active: 1, Enabled: 1, Total: 100, Used: 10, Available: 90},
			},
			"node2": {
				{Storage: "local", Type: "dir", Content: "images", Active: 1, Enabled: 1, Total: 100, Used: 20, Available: 80},
			},
		},
	}

	nodes := []proxmox.Node{
		{Node: "node1", Status: "online"},
		{Node: "node2", Status: "online"},
	}

	monitor.pollStorageWithNodes(context.Background(), "inst1", client, nodes)

	var shared *models.Storage
	for _, storage := range monitor.state.GetSnapshot().Storage {
		if storage.Name == "cephfs" {
			storageCopy := storage
			shared = &storageCopy
			break
		}
	}
	if shared == nil {
		t.Fatalf("expected synthesized shared storage in state, got %+v", monitor.state.GetSnapshot().Storage)
	}
	if shared.ID != "cluster-a-cluster-cephfs" || shared.Node != "cluster" || !shared.Shared {
		t.Fatalf("unexpected synthesized shared storage identity: %+v", *shared)
	}
	if shared.NodeCount != 2 || len(shared.Nodes) != 2 {
		t.Fatalf("expected cluster storage node affinity, got %+v", *shared)
	}
	if shared.Total != 1000 || shared.Used != 100 || shared.Free != 900 {
		t.Fatalf("expected cluster storage capacity from config, got %+v", *shared)
	}
}

func TestPollStorageWithNodesKeepsSharedStorageAvailableWhenAnyNodeIsActive(t *testing.T) {
	monitor := &Monitor{
		state: models.NewState(),
		config: &config.Config{
			PVEInstances: []config.PVEInstance{
				{
					Name:        "inst1",
					IsCluster:   true,
					ClusterName: "cluster-a",
				},
			},
		},
	}

	client := &fakeStorageClient{
		allStorage: []proxmox.Storage{
			{
				Storage:   "shared-nfs",
				Type:      "nfs",
				Content:   "images,backup",
				Shared:    1,
				Total:     1000,
				Used:      900,
				Available: 100,
				Nodes:     "node1,node2",
			},
		},
		storageByNode: map[string][]proxmox.Storage{
			"node1": {
				{
					Storage:   "shared-nfs",
					Type:      "nfs",
					Content:   "images,backup",
					Shared:    1,
					Enabled:   1,
					Active:    0,
					Total:     1000,
					Used:      900,
					Available: 100,
				},
			},
			"node2": {
				{
					Storage:   "shared-nfs",
					Type:      "nfs",
					Content:   "images,backup",
					Shared:    1,
					Enabled:   1,
					Active:    1,
					Total:     1000,
					Used:      800,
					Available: 200,
				},
			},
		},
	}

	nodes := []proxmox.Node{
		{Node: "node1", Status: "online"},
		{Node: "node2", Status: "online"},
	}

	monitor.pollStorageWithNodes(context.Background(), "inst1", client, nodes)

	var shared *models.Storage
	for _, storage := range monitor.state.GetSnapshot().Storage {
		if storage.Name == "shared-nfs" {
			storageCopy := storage
			shared = &storageCopy
			break
		}
	}
	if shared == nil {
		t.Fatalf("expected shared storage in state, got %+v", monitor.state.GetSnapshot().Storage)
	}
	if shared.Node != "cluster" || !shared.Shared {
		t.Fatalf("expected cluster-scoped shared storage, got %+v", *shared)
	}
	if !shared.Enabled || !shared.Active {
		t.Fatalf("expected shared storage to remain enabled and active when any node is active, got %+v", *shared)
	}
	if shared.Status != "available" {
		t.Fatalf("expected shared storage status to stay available when any node is active, got %+v", *shared)
	}
	if shared.Total != 1000 || shared.Used != 900 || shared.Free != 100 {
		t.Fatalf("expected shared storage to retain best capacity sample, got %+v", *shared)
	}
}

func TestPollStorageWithNodesAttachesZFSPoolFromExplicitPoolField(t *testing.T) {
	t.Setenv("PULSE_DATA_DIR", t.TempDir())

	monitor := &Monitor{
		state:          &models.State{},
		metricsHistory: NewMetricsHistory(16, time.Hour),
		alertManager:   alerts.NewManager(),
	}
	t.Cleanup(func() {
		monitor.alertManager.Stop()
	})

	storage := proxmox.Storage{
		Storage:   "local-zfs",
		Type:      "zfspool",
		Pool:      "rpool/data",
		Content:   "images,rootdir",
		Active:    1,
		Enabled:   1,
		Shared:    0,
		Total:     1000,
		Used:      250,
		Available: 750,
	}

	client := &fakeStorageClient{
		allStorage: []proxmox.Storage{storage},
		storageByNode: map[string][]proxmox.Storage{
			"node1": {storage},
		},
		zfsPoolsByNode: map[string][]proxmox.ZFSPoolInfo{
			"node1": {
				{Name: "rpool", Size: 1000, Alloc: 250, Free: 750, Frag: 1, Dedup: 1.0, Health: "ONLINE"},
			},
		},
	}

	nodes := []proxmox.Node{{Node: "node1", Status: "online"}}
	monitor.pollStorageWithNodes(context.Background(), "inst1", client, nodes)

	if len(monitor.state.Storage) != 1 {
		t.Fatalf("expected 1 storage entry, got %d", len(monitor.state.Storage))
	}
	if got := monitor.state.Storage[0].Pool; got != "rpool/data" {
		t.Fatalf("storage pool = %q, want %q", got, "rpool/data")
	}
	if monitor.state.Storage[0].ZFSPool == nil {
		t.Fatal("expected explicit pool field to attach ZFS pool details")
	}
	if monitor.state.Storage[0].ZFSPool.Name != "rpool" {
		t.Fatalf("ZFS pool name = %q, want rpool", monitor.state.Storage[0].ZFSPool.Name)
	}
}

func TestPollStorageWithNodesUsesClusterStoragePoolFallback(t *testing.T) {
	t.Setenv("PULSE_DATA_DIR", t.TempDir())

	monitor := &Monitor{
		state: models.NewState(),
		config: &config.Config{
			PVEInstances: []config.PVEInstance{
				{
					Name:        "inst1",
					IsCluster:   true,
					ClusterName: "cluster-a",
				},
			},
		},
		metricsHistory: NewMetricsHistory(16, time.Hour),
		alertManager:   alerts.NewManager(),
	}
	t.Cleanup(func() {
		monitor.alertManager.Stop()
	})

	clusterStorage := proxmox.Storage{
		Storage:   "local-zfs",
		Type:      "zfspool",
		Pool:      "rpool/data",
		Content:   "images,rootdir",
		Shared:    0,
		Total:     1000,
		Used:      250,
		Available: 750,
	}
	nodeStorage := clusterStorage
	nodeStorage.Pool = ""
	nodeStorage.Active = 1
	nodeStorage.Enabled = 1

	client := &fakeStorageClient{
		allStorage: []proxmox.Storage{clusterStorage},
		storageByNode: map[string][]proxmox.Storage{
			"node1": {nodeStorage},
		},
		zfsPoolsByNode: map[string][]proxmox.ZFSPoolInfo{
			"node1": {
				{Name: "rpool", Size: 1000, Alloc: 250, Free: 750, Frag: 1, Dedup: 1.0, Health: "ONLINE"},
			},
		},
	}

	nodes := []proxmox.Node{{Node: "node1", Status: "online"}}
	monitor.pollStorageWithNodes(context.Background(), "inst1", client, nodes)

	if len(monitor.state.Storage) != 1 {
		t.Fatalf("expected 1 storage entry, got %d", len(monitor.state.Storage))
	}
	if got := monitor.state.Storage[0].Pool; got != "rpool/data" {
		t.Fatalf("storage pool = %q, want %q", got, "rpool/data")
	}
	if monitor.state.Storage[0].ZFSPool == nil || monitor.state.Storage[0].ZFSPool.Name != "rpool" {
		t.Fatalf("expected cluster pool fallback to attach rpool, got %#v", monitor.state.Storage[0].ZFSPool)
	}
}

func TestPollStorageWithNodesOptimizedAttachesZFSPoolForDirStorageOnDatasetPath(t *testing.T) {
	t.Setenv("PULSE_DATA_DIR", t.TempDir())

	monitor := &Monitor{
		state:          &models.State{},
		metricsHistory: NewMetricsHistory(16, time.Hour),
		alertManager:   alerts.NewManager(),
	}
	t.Cleanup(func() {
		monitor.alertManager.Stop()
	})

	storage := proxmox.Storage{
		Storage:   "local",
		Type:      "dir",
		Path:      "/rpool/data",
		Content:   "images",
		Active:    1,
		Enabled:   1,
		Shared:    0,
		Total:     1000,
		Used:      250,
		Available: 750,
	}

	client := &fakeStorageClient{
		allStorage: []proxmox.Storage{storage},
		storageByNode: map[string][]proxmox.Storage{
			"node1": {storage},
		},
		zfsPoolsByNode: map[string][]proxmox.ZFSPoolInfo{
			"node1": {
				{Name: "rpool", Size: 1000, Alloc: 250, Free: 750, Frag: 1, Dedup: 1.0, Health: "ONLINE"},
			},
		},
	}

	nodes := []proxmox.Node{{Node: "node1", Status: "online"}}
	monitor.pollStorageWithNodes(context.Background(), "inst1", client, nodes)

	if len(monitor.state.Storage) != 1 {
		t.Fatalf("expected 1 storage entry, got %d", len(monitor.state.Storage))
	}
	if monitor.state.Storage[0].ZFSPool == nil {
		t.Fatal("expected dir storage on ZFS dataset path to have ZFS pool attached")
	}
	if monitor.state.Storage[0].ZFSPool.Name != "rpool" {
		t.Fatalf("ZFS pool name = %q, want rpool", monitor.state.Storage[0].ZFSPool.Name)
	}
}