Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

8.7 KB
Raw Blame
package alerts

import (
	"sync"
	"sync/atomic"
	"testing"
	"time"
)

type poweredOffTestClock struct {
	wall time.Time
	tick time.Duration
}

func newPoweredOffTestManager(t *testing.T, start time.Time) (*Manager, *poweredOffTestClock) {
	t.Helper()
	manager := NewManagerWithDataDir(t.TempDir())
	t.Cleanup(manager.Stop)
	clock := &poweredOffTestClock{wall: start}
	manager.now = func() time.Time { return clock.wall }
	manager.intentClock = func() time.Duration { return clock.tick }
	return manager, clock
}

func installPoweredOffPolicy(t *testing.T, manager *Manager, graceSeconds int, backup *BackupOfflineIntentPolicy) {
	t.Helper()
	document := NewAlertIntentPolicyDocument()
	document.ResourceTypes["guest"] = map[string]AlertIntentRule{
		string(AlertIntentSignalOffline): {
			GraceSeconds:  intPointer(graceSeconds),
			BackupOffline: backup,
		},
	}
	if err := manager.LoadIntentPolicies(document); err != nil {
		t.Fatalf("LoadIntentPolicies() error = %v", err)
	}
}

func hasPoweredOffAlert(manager *Manager, resourceID string) bool {
	manager.mu.RLock()
	defer manager.mu.RUnlock()
	_, exists := manager.getActiveAlertNoLock(canonicalPoweredStateStateID(resourceID))
	return exists
}

func TestPoweredOffToleranceUsesElapsedTimeForVMAndLXC(t *testing.T) {
	for _, test := range []struct {
		name         string
		resourceID   string
		resourceType string
	}{
		{name: "vm", resourceID: "vm:101", resourceType: "VM"},
		{name: "lxc", resourceID: "lxc:202", resourceType: "Container"},
	} {
		t.Run(test.name, func(t *testing.T) {
			start := time.Date(2026, 7, 24, 8, 0, 0, 0, time.UTC)
			manager, clock := newPoweredOffTestManager(t, start)
			installPoweredOffPolicy(t, manager, 300, nil)

			for range 20 {
				manager.checkGuestPoweredOff(test.resourceID, test.name, "node-a", "pve-a", test.resourceType, false)
			}
			if hasPoweredOffAlert(manager, test.resourceID) {
				t.Fatal("duplicate reports activated the alert before the duration elapsed")
			}
			if testCoreConfirmations(manager, test.resourceID, canonicalPoweredStateSpecID(test.resourceID)) > 1 {
				t.Fatal("duration-based powered-off policy must not accumulate poll confirmations")
			}

			clock.wall, clock.tick = start.Add(299*time.Second), 299*time.Second
			manager.checkGuestPoweredOff(test.resourceID, test.name, "node-a", "pve-a", test.resourceType, false)
			if hasPoweredOffAlert(manager, test.resourceID) {
				t.Fatal("alert activated before the configured duration")
			}

			clock.wall, clock.tick = start.Add(300*time.Second), 300*time.Second
			manager.checkGuestPoweredOff(test.resourceID, test.name, "node-a", "pve-a", test.resourceType, false)
			if !hasPoweredOffAlert(manager, test.resourceID) {
				t.Fatal("alert did not activate at the configured duration")
			}
		})
	}
}

func TestPoweredOffToleranceZeroAndInheritance(t *testing.T) {
	start := time.Date(2026, 7, 24, 9, 0, 0, 0, time.UTC)
	manager, _ := newPoweredOffTestManager(t, start)
	document := NewAlertIntentPolicyDocument()
	document.ResourceTypes["guest"] = map[string]AlertIntentRule{
		string(AlertIntentSignalOffline): {GraceSeconds: intPointer(300)},
	}
	document.Resources["vm:zero"] = map[string]AlertIntentRule{
		string(AlertIntentSignalOffline): {GraceSeconds: intPointer(0)},
	}
	if err := manager.LoadIntentPolicies(document); err != nil {
		t.Fatal(err)
	}

	manager.checkGuestPoweredOff("vm:zero", "zero", "node-a", "pve-a", "VM", false)
	if !hasPoweredOffAlert(manager, "vm:zero") {
		t.Fatal("explicit zero did not activate on the first stopped observation")
	}
	manager.checkGuestPoweredOff("vm:inherit", "inherit", "node-a", "pve-a", "VM", false)
	if hasPoweredOffAlert(manager, "vm:inherit") {
		t.Fatal("inherited guest tolerance was not applied")
	}
}

func TestPoweredOffToleranceIgnoresWallClockChanges(t *testing.T) {
	start := time.Date(2026, 7, 24, 10, 0, 0, 0, time.UTC)
	manager, clock := newPoweredOffTestManager(t, start)
	installPoweredOffPolicy(t, manager, 300, nil)

	manager.checkGuestPoweredOff("vm:clock", "clock", "node-a", "pve-a", "VM", false)
	clock.wall = start.Add(24 * time.Hour)
	manager.checkGuestPoweredOff("vm:clock", "clock", "node-a", "pve-a", "VM", false)
	if hasPoweredOffAlert(manager, "vm:clock") {
		t.Fatal("forward wall-clock jump activated the alert")
	}

	clock.wall = start.Add(-24 * time.Hour)
	clock.tick = 300 * time.Second
	manager.checkGuestPoweredOff("vm:clock", "clock", "node-a", "pve-a", "VM", false)
	if !hasPoweredOffAlert(manager, "vm:clock") {
		t.Fatal("monotonic elapsed time did not activate after a backward wall-clock jump")
	}
}

func TestPoweredOffToleranceClearsOnRecoveryAndSuppression(t *testing.T) {
	start := time.Date(2026, 7, 24, 11, 0, 0, 0, time.UTC)
	manager, clock := newPoweredOffTestManager(t, start)
	installPoweredOffPolicy(t, manager, 300, nil)

	manager.checkGuestPoweredOff("vm:flap", "flap", "node-a", "pve-a", "VM", false)
	clock.wall, clock.tick = start.Add(200*time.Second), 200*time.Second
	manager.clearGuestPoweredOffAlert("vm:flap", "flap")
	clock.wall, clock.tick = start.Add(300*time.Second), 300*time.Second
	manager.checkGuestPoweredOff("vm:flap", "flap", "node-a", "pve-a", "VM", false)
	if hasPoweredOffAlert(manager, "vm:flap") {
		t.Fatal("recovery did not reset the powered-off duration")
	}

	if !manager.suppressGuestAlerts("vm:flap") {
		t.Fatal("suppression did not report clearing pending intent state")
	}
	manager.mu.RLock()
	pending := len(manager.intentPending)
	ticks := len(manager.intentRuntimeTicks)
	manager.mu.RUnlock()
	if pending != 0 || ticks != 0 {
		t.Fatalf("suppression left pending state: pending=%d ticks=%d", pending, ticks)
	}
}

func TestPoweredOffToleranceRecoveryNotifiesAfterActivation(t *testing.T) {
	start := time.Date(2026, 7, 24, 11, 30, 0, 0, time.UTC)
	manager, clock := newPoweredOffTestManager(t, start)
	installPoweredOffPolicy(t, manager, 60, nil)
	resolved := make(chan string, 1)
	manager.SetResolvedCallback(func(alertID string) {
		resolved <- alertID
	})

	manager.checkGuestPoweredOff("vm:recover", "recover", "node-a", "pve-a", "VM", false)
	clock.wall, clock.tick = start.Add(60*time.Second), 60*time.Second
	manager.checkGuestPoweredOff("vm:recover", "recover", "node-a", "pve-a", "VM", false)
	if !hasPoweredOffAlert(manager, "vm:recover") {
		t.Fatal("powered-off alert did not activate before recovery")
	}

	manager.clearGuestPoweredOffAlert("vm:recover", "recover")
	select {
	case alertID := <-resolved:
		if alertID == "" {
			t.Fatal("recovery callback omitted the alert identity")
		}
	case <-time.After(time.Second):
		t.Fatal("activated powered-off alert did not emit a recovery callback")
	}
	if hasPoweredOffAlert(manager, "vm:recover") {
		t.Fatal("powered-off alert remained active after recovery")
	}
}

func TestPoweredOffBackupDeferralHasHardCap(t *testing.T) {
	start := time.Date(2026, 7, 24, 12, 0, 0, 0, time.UTC)
	manager, clock := newPoweredOffTestManager(t, start)
	installPoweredOffPolicy(t, manager, 300, &BackupOfflineIntentPolicy{
		Enabled: true, PostGraceSeconds: 60, MaxDeferralSeconds: 600,
	})
	backup := BackupIntentContext{Active: true, Evidence: "fresh_task"}

	manager.checkGuestPoweredOffWithThresholdsAndIntent("vm:backup", "backup", "node-a", "pve-a", "VM", manager.config.GuestDefaults, false, backup)
	clock.wall, clock.tick = start.Add(599*time.Second), 599*time.Second
	manager.checkGuestPoweredOffWithThresholdsAndIntent("vm:backup", "backup", "node-a", "pve-a", "VM", manager.config.GuestDefaults, false, backup)
	if hasPoweredOffAlert(manager, "vm:backup") {
		t.Fatal("backup deferral activated before its hard cap")
	}

	clock.wall, clock.tick = start.Add(600*time.Second), 600*time.Second
	manager.checkGuestPoweredOffWithThresholdsAndIntent("vm:backup", "backup", "node-a", "pve-a", "VM", manager.config.GuestDefaults, false, backup)
	if !hasPoweredOffAlert(manager, "vm:backup") {
		t.Fatal("backup evidence hid an outage beyond the hard cap")
	}
}

func TestPoweredOffToleranceConcurrentReportsActivateOnce(t *testing.T) {
	start := time.Date(2026, 7, 24, 13, 0, 0, 0, time.UTC)
	manager, _ := newPoweredOffTestManager(t, start)
	installPoweredOffPolicy(t, manager, 0, nil)
	manager.config.ActivationState = ActivationActive
	var notifications atomic.Int32
	manager.SetAlertCallback(func(*Alert) {
		notifications.Add(1)
	})

	var workers sync.WaitGroup
	for range 32 {
		workers.Add(1)
		go func() {
			defer workers.Done()
			manager.checkGuestPoweredOff("vm:race", "race", "node-a", "pve-a", "VM", false)
		}()
	}
	workers.Wait()

	if !hasPoweredOffAlert(manager, "vm:race") {
		t.Fatal("concurrent reports did not activate the alert")
	}
	if got := notifications.Load(); got != 1 {
		t.Fatalf("notification count = %d, want 1", got)
	}
	history := manager.GetAlertHistory(10)
	if len(history) != 1 {
		t.Fatalf("history entries = %d, want 1", len(history))
	}
}