Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

15.5 KB
Raw Blame
package monitoring

import (
	"context"
	"fmt"
	"testing"
	"time"

	"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
	"github.com/rcourtman/pulse-go-rewrite/internal/config"
	"github.com/rcourtman/pulse-go-rewrite/internal/models"
	"github.com/rcourtman/pulse-go-rewrite/internal/notifications"
	"github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"
)

func TestPreserveOrExpireNodes(t *testing.T) {
	m := &Monitor{
		state:          models.NewState(),
		nodeLastOnline: make(map[string]time.Time),
	}

	t.Run("within grace keeps node online with degraded health", func(t *testing.T) {
		preserved := m.preserveOrExpireNodes([]models.Node{{
			ID:               "inst-node1",
			Name:             "node1",
			Status:           "online",
			ConnectionHealth: "error",
			Uptime:           1234,
			LastSeen:         time.Now().Add(-10 * time.Second),
		}})
		if len(preserved) != 1 {
			t.Fatalf("preserved = %d nodes, want 1", len(preserved))
		}
		if preserved[0].Status != "online" {
			t.Fatalf("Status = %q, want online within grace", preserved[0].Status)
		}
		if preserved[0].ConnectionHealth != "degraded" {
			t.Fatalf("ConnectionHealth = %q, want degraded within grace", preserved[0].ConnectionHealth)
		}
	})

	t.Run("offline node with stamped LastSeen is not resurrected to online", func(t *testing.T) {
		// Simulates the unified-registry round-trip: ingest stamps zero
		// LastSeen with the ingest time, so a synthesized offline
		// placeholder comes back from prev state looking "recently seen"
		// even though no poll ever reached the node. Grace must not flip
		// it to online — that loop made never-seen hosts permanently
		// report online/degraded.
		preserved := m.preserveOrExpireNodes([]models.Node{{
			ID:               "inst-placeholder",
			Name:             "placeholder",
			Status:           "offline",
			ConnectionHealth: "error",
			LastSeen:         time.Now().Add(-5 * time.Second),
		}})
		if len(preserved) != 1 {
			t.Fatalf("preserved = %d nodes, want 1", len(preserved))
		}
		if preserved[0].Status != "offline" {
			t.Fatalf("Status = %q, want offline (no online sighting to justify grace)", preserved[0].Status)
		}
		if preserved[0].ConnectionHealth != "error" {
			t.Fatalf("ConnectionHealth = %q, want error", preserved[0].ConnectionHealth)
		}
	})

	t.Run("past grace marks node offline", func(t *testing.T) {
		preserved := m.preserveOrExpireNodes([]models.Node{{
			ID:               "inst-node1",
			Name:             "node1",
			Status:           "online",
			ConnectionHealth: "healthy",
			Uptime:           1234,
			LastSeen:         time.Now().Add(-2 * nodeOfflineGracePeriod),
		}})
		if len(preserved) != 1 {
			t.Fatalf("preserved = %d nodes, want 1", len(preserved))
		}
		if preserved[0].Status != "offline" {
			t.Fatalf("Status = %q, want offline past grace", preserved[0].Status)
		}
		if preserved[0].ConnectionHealth != "error" {
			t.Fatalf("ConnectionHealth = %q, want error past grace", preserved[0].ConnectionHealth)
		}
		if preserved[0].Uptime != 0 {
			t.Fatalf("Uptime = %d, want 0 past grace", preserved[0].Uptime)
		}
	})
}

type unreachablePVEClient struct {
	mockPVEClientExtended
}

func (c *unreachablePVEClient) GetNodes(ctx context.Context) ([]proxmox.Node, error) {
	return nil, fmt.Errorf("connection refused")
}

type membershipPVEClient struct {
	stubPVEClient
	statuses  []proxmox.ClusterStatus
	statusErr error
}

func (c *membershipPVEClient) GetClusterStatus(context.Context) ([]proxmox.ClusterStatus, error) {
	return c.statuses, c.statusErr
}

func newUnreachableTestMonitor(t *testing.T, cfg *config.Config) *Monitor {
	t.Helper()
	m := &Monitor{
		config:                  cfg,
		state:                   models.NewState(),
		pveClients:              make(map[string]PVEClientInterface),
		nodeLastOnline:          make(map[string]time.Time),
		nodeSnapshots:           make(map[string]NodeMemorySnapshot),
		guestSnapshots:          make(map[string]GuestMemorySnapshot),
		metricsHistory:          NewMetricsHistory(32, time.Hour),
		lastClusterCheck:        make(map[string]time.Time),
		lastPhysicalDiskPoll:    make(map[string]time.Time),
		lastPVEBackupPoll:       make(map[string]time.Time),
		authFailures:            make(map[string]int),
		lastAuthAttempt:         make(map[string]time.Time),
		pollStatusMap:           make(map[string]*pollStatus),
		nodePendingUpdatesCache: make(map[string]pendingUpdatesCache),
		instanceInfoCache:       make(map[string]*instanceInfo),
		lastOutcome:             make(map[string]taskOutcome),
		failureCounts:           make(map[string]int),
		alertManager:            alerts.NewManager(),
		notificationMgr:         notifications.NewNotificationManager(""),
	}
	t.Cleanup(m.alertManager.Stop)
	t.Cleanup(m.notificationMgr.Stop)
	return m
}

// Regression test for #1441: when the whole instance stops answering (host
// shut down), the poll error path must still run the offline grace policy
// instead of freezing the last online snapshot in state forever.
func TestPollPVEInstanceMarksNodesOfflineWhenUnreachable(t *testing.T) {
	m := newUnreachableTestMonitor(t, &config.Config{
		PVEInstances: []config.PVEInstance{
			// Non-default port so the portless fallback path stays out of the way.
			{Name: "pve-test", Host: "https://localhost:9999"},
		},
	})

	// Seed state as if a previous poll saw the node online, longer ago than
	// the offline grace period.
	m.state.UpdateNodesForInstance("pve-test", []models.Node{{
		ID:       "pve-test-node1",
		Name:     "node1",
		Instance: "pve-test",
		Status:   "online",
		Type:     "node",
		Uptime:   4242,
		LastSeen: time.Now().Add(-2 * nodeOfflineGracePeriod),
	}})

	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
	defer cancel()
	m.pollPVEInstance(ctx, "pve-test", &unreachablePVEClient{})

	nodes := m.state.GetSnapshot().Nodes
	if len(nodes) != 1 {
		t.Fatalf("state nodes = %d, want 1", len(nodes))
	}
	if nodes[0].Status != "offline" {
		t.Fatalf("node Status = %q after unreachable poll past grace, want offline", nodes[0].Status)
	}
	if nodes[0].ConnectionHealth != "error" {
		t.Fatalf("node ConnectionHealth = %q, want error", nodes[0].ConnectionHealth)
	}
}

// Regression test for #1433: an instance that is already down when Pulse
// starts has no previous node state, so the unreachable path must synthesize
// an offline placeholder from config instead of leaving the configured
// instance invisible until its first successful poll.
func TestPollPVEInstanceSynthesizesPlaceholderWhenNeverSeen(t *testing.T) {
	m := newUnreachableTestMonitor(t, &config.Config{
		PVEInstances: []config.PVEInstance{
			// Non-default port so the portless fallback path stays out of the way.
			{Name: "pve-test", Host: "https://localhost:9999"},
		},
	})

	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
	defer cancel()
	m.pollPVEInstance(ctx, "pve-test", &unreachablePVEClient{})

	nodes := m.state.GetSnapshot().Nodes
	if len(nodes) != 1 {
		t.Fatalf("state nodes = %d, want 1 synthesized placeholder", len(nodes))
	}
	if nodes[0].ID != "pve-test-pve-test" {
		t.Fatalf("node ID = %q, want pve-test-pve-test", nodes[0].ID)
	}
	if nodes[0].Name != "pve-test" {
		t.Fatalf("node Name = %q, want pve-test", nodes[0].Name)
	}
	if nodes[0].Status != "offline" {
		t.Fatalf("node Status = %q, want offline", nodes[0].Status)
	}
	if nodes[0].ConnectionHealth != "error" {
		t.Fatalf("node ConnectionHealth = %q, want error", nodes[0].ConnectionHealth)
	}
	if nodes[0].Host != "https://localhost:9999" {
		t.Fatalf("node Host = %q, want configured endpoint", nodes[0].Host)
	}
}

func TestPlaceholderNodesForInstanceCluster(t *testing.T) {
	m := &Monitor{
		state: models.NewState(),
		config: &config.Config{
			PVEInstances: []config.PVEInstance{{
				Name:        "cluster-inst",
				Host:        "https://10.0.0.1:8006",
				IsCluster:   true,
				ClusterName: "homelab",
				ClusterEndpoints: []config.ClusterEndpoint{
					{NodeName: "pve1", Host: "https://10.0.0.1:8006"},
					{NodeName: "pve2", Host: "https://10.0.0.2:8006"},
					{NodeName: ""},     // skipped: no node name
					{NodeName: "pve1"}, // skipped: duplicate
				},
			}},
		},
	}

	nodes := m.placeholderNodesForInstance("cluster-inst")
	if len(nodes) != 2 {
		t.Fatalf("placeholders = %d nodes, want 2 (deduped, blank skipped)", len(nodes))
	}
	for i, want := range []string{"pve1", "pve2"} {
		if nodes[i].Name != want {
			t.Fatalf("nodes[%d].Name = %q, want %q", i, nodes[i].Name, want)
		}
		if wantID := "homelab-" + want; nodes[i].ID != wantID {
			t.Fatalf("nodes[%d].ID = %q, want %q (cluster ID convention)", i, nodes[i].ID, wantID)
		}
		if nodes[i].Status != "offline" {
			t.Fatalf("nodes[%d].Status = %q, want offline", i, nodes[i].Status)
		}
		if !nodes[i].IsClusterMember {
			t.Fatalf("nodes[%d].IsClusterMember = false, want true", i)
		}
		if nodes[i].ClusterName != "homelab" {
			t.Fatalf("nodes[%d].ClusterName = %q, want homelab", i, nodes[i].ClusterName)
		}
	}
}

func TestReconcilePVENodeInventoryPreservesCanonicalMembership(t *testing.T) {
	lastSeen := time.Now().Add(-2 * time.Hour).UTC()
	cfg := &config.Config{
		PVEInstances: []config.PVEInstance{{
			Name:        "cluster-api",
			Host:        "https://pve-a:8006",
			IsCluster:   true,
			ClusterName: "production",
			ClusterEndpoints: []config.ClusterEndpoint{
				{NodeID: "node/a", NodeName: "pve-a", Host: "https://pve-a:8006", Online: true, LastSeen: lastSeen},
				{NodeID: "node/b", NodeName: "pve-b", Host: "https://pve-b:8006", Online: true, LastSeen: lastSeen},
			},
		}},
	}
	m := newUnreachableTestMonitor(t, cfg)
	instanceCfg := &cfg.PVEInstances[0]
	client := &membershipPVEClient{statuses: []proxmox.ClusterStatus{
		{Type: "cluster", Name: "production", Quorate: 1},
		{Type: "node", ID: "node/a", Name: "pve-a", Online: 1},
		{Type: "node", ID: "node/b", Name: "pve-b", Online: 0},
	}}
	previous := []models.Node{
		{
			ID:               "production-pve-a",
			Name:             "pve-a",
			Instance:         "cluster-api",
			ClusterName:      "production",
			Status:           "online",
			ConnectionHealth: "healthy",
			LastSeen:         lastSeen,
		},
		{
			ID:               "production-pve-b",
			Name:             "pve-b",
			Instance:         "cluster-api",
			ClusterName:      "production",
			Host:             "https://pve-b:8006",
			Status:           "online",
			ConnectionHealth: "healthy",
			CPU:              0.8,
			Uptime:           3600,
			LastSeen:         lastSeen,
			LinkedAgentID:    "agent-pve-b",
		},
	}
	current := []models.Node{previous[0]}

	got := m.reconcilePVENodeInventory(
		context.Background(),
		"cluster-api",
		instanceCfg,
		client,
		current,
		previous,
	)
	if len(got) != 2 {
		t.Fatalf("reconciled nodes = %d, want 2: %+v", len(got), got)
	}
	byName := pveNodeByName(got)
	offline := byName["pve-b"]
	if offline.ID != "production-pve-b" {
		t.Fatalf("offline node ID = %q, want stable canonical ID", offline.ID)
	}
	if offline.Status != "offline" || offline.ConnectionHealth != "error" {
		t.Fatalf("offline node state = (%q, %q), want offline/error", offline.Status, offline.ConnectionHealth)
	}
	if offline.CPU != 0 || offline.Uptime != 0 {
		t.Fatalf("offline node live metrics = (cpu=%v uptime=%d), want zeroed", offline.CPU, offline.Uptime)
	}
	if !offline.LastSeen.Equal(lastSeen) || offline.LinkedAgentID != "agent-pve-b" {
		t.Fatalf("offline node lost last-known identity evidence: %+v", offline)
	}
	if len(instanceCfg.ClusterEndpoints) != 2 {
		t.Fatalf("cluster endpoints = %d, want 2 canonical members", len(instanceCfg.ClusterEndpoints))
	}
}

func TestReconcilePVENodeInventoryTreatsPartialAndNonQuorateReadsAsUncertain(t *testing.T) {
	tests := []struct {
		name      string
		statuses  []proxmox.ClusterStatus
		statusErr error
	}{
		{
			name:      "membership API unavailable",
			statusErr: fmt.Errorf("temporary connection failure"),
		},
		{
			name: "membership response incomplete",
			statuses: []proxmox.ClusterStatus{
				{Type: "cluster", Name: "production", Quorate: 1},
			},
		},
		{
			name: "cluster has no quorum",
			statuses: []proxmox.ClusterStatus{
				{Type: "cluster", Name: "production", Quorate: 0},
				{Type: "node", Name: "pve-a", Online: 1},
			},
		},
		{
			name: "cluster identity changed",
			statuses: []proxmox.ClusterStatus{
				{Type: "cluster", Name: "different-cluster", Quorate: 1},
				{Type: "node", Name: "pve-a", Online: 1},
			},
		},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			cfg := &config.Config{PVEInstances: []config.PVEInstance{{
				Name:        "cluster-api",
				IsCluster:   true,
				ClusterName: "production",
				ClusterEndpoints: []config.ClusterEndpoint{
					{NodeName: "pve-a"},
					{NodeName: "pve-b"},
				},
			}}}
			m := newUnreachableTestMonitor(t, cfg)
			previous := []models.Node{
				{ID: "production-pve-a", Name: "pve-a", Instance: "cluster-api", Status: "online", LastSeen: time.Now()},
				{ID: "production-pve-b", Name: "pve-b", Instance: "cluster-api", Status: "offline"},
			}
			got := m.reconcilePVENodeInventory(
				context.Background(),
				"cluster-api",
				&cfg.PVEInstances[0],
				&membershipPVEClient{statuses: tt.statuses, statusErr: tt.statusErr},
				previous[:1],
				previous,
			)
			if len(got) != 2 {
				t.Fatalf("uncertain membership returned %d nodes, want last-known union: %+v", len(got), got)
			}
			if got[1].Name != "pve-b" || got[1].Status != "offline" {
				t.Fatalf("unobserved member = %+v, want pve-b retained offline", got[1])
			}
			if len(cfg.PVEInstances[0].ClusterEndpoints) != 2 {
				t.Fatalf("uncertain membership rewrote endpoints: %+v", cfg.PVEInstances[0].ClusterEndpoints)
			}
		})
	}
}

func TestReconcilePVENodeInventoryAddsNewAuthoritativeMemberImmediately(t *testing.T) {
	cfg := &config.Config{PVEInstances: []config.PVEInstance{{
		Name:        "cluster-api",
		Host:        "https://pve-a:8006",
		IsCluster:   true,
		ClusterName: "production",
		ClusterEndpoints: []config.ClusterEndpoint{
			{NodeName: "pve-a", Host: "https://pve-a:8006"},
		},
	}}}
	m := newUnreachableTestMonitor(t, cfg)
	previous := []models.Node{{
		ID:          "production-pve-a",
		Name:        "pve-a",
		Instance:    "cluster-api",
		ClusterName: "production",
		Status:      "online",
		LastSeen:    time.Now(),
	}}
	client := &membershipPVEClient{statuses: []proxmox.ClusterStatus{
		{Type: "cluster", Name: "production", Quorate: 1},
		{Type: "node", ID: "node/a", Name: "pve-a", Online: 1},
		{Type: "node", ID: "node/c", Name: "pve-c", IP: "10.0.0.3", Online: 1},
	}}

	got := m.reconcilePVENodeInventory(
		context.Background(),
		"cluster-api",
		&cfg.PVEInstances[0],
		client,
		previous,
		previous,
	)
	if len(got) != 2 {
		t.Fatalf("membership addition returned %d nodes, want 2: %+v", len(got), got)
	}
	added := pveNodeByName(got)["pve-c"]
	if added.ID != "production-pve-c" || added.Status != "unknown" || added.ConnectionHealth != "degraded" {
		t.Fatalf("new member = %+v, want immediate stable unknown/degraded placeholder", added)
	}
	if len(cfg.PVEInstances[0].ClusterEndpoints) != 2 {
		t.Fatalf("new member was not persisted to endpoints: %+v", cfg.PVEInstances[0].ClusterEndpoints)
	}
}

func TestPVENodeIdentityScopeSeparatesSameNameClusters(t *testing.T) {
	cfg := &config.Config{PVEInstances: []config.PVEInstance{
		{Name: "site-a", IsCluster: true, ClusterName: "production"},
		{Name: "site-b", IsCluster: true, ClusterName: "production"},
	}}
	m := &Monitor{config: cfg, state: models.NewState()}

	a := m.placeholderNodeForInstance("site-a", &cfg.PVEInstances[0], "pve-1")
	b := m.placeholderNodeForInstance("site-b", &cfg.PVEInstances[1], "pve-1")
	if a.ID != "site-a-pve-1" || b.ID != "site-b-pve-1" || a.ID == b.ID {
		t.Fatalf("same-name cluster node IDs collided: a=%q b=%q", a.ID, b.ID)
	}
}