Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

16.3 KB
Raw Blame
package alerts

import (
	"testing"
	"time"
)

// platformConnectionSnapshot is a small helper that produces a baseline
// ConnectionSnapshot for a pve connection so test cases can override one
// field at a time without restating the boilerplate.
func platformConnectionSnapshot(id, name string, state ConnectionState) ConnectionSnapshot {
	return ConnectionSnapshot{
		ID:      id,
		Name:    name,
		Type:    ConnectionTypePVE,
		State:   state,
		Enabled: true,
	}
}

func TestCheckConnection(t *testing.T) {
	t.Run("active state never fires", func(t *testing.T) {
		m := newTestManager(t)
		snap := platformConnectionSnapshot("conn-1", "Delly PVE", ConnectionStateActive)

		for i := 0; i < 5; i++ {
			m.CheckConnection(snap)
		}

		alertID := canonicalDiscreteStateStateID(snap.ID, connectionDegradedStateKey)
		if testHasActiveAlert(t, m, alertID) {
			t.Fatal("expected no alert for active connection")
		}
		m.mu.RLock()
		count := testCoreConfirmations(m, snap.ID, canonicalDiscreteStateSpecID(snap.ID, connectionDegradedStateKey))
		m.mu.RUnlock()
		if count != 0 {
			t.Errorf("expected degraded count 0, got %d", count)
		}
	})

	t.Run("three consecutive stale observations fire a warning", func(t *testing.T) {
		m := newTestManager(t)
		snap := platformConnectionSnapshot("conn-stale", "Delly PVE", ConnectionStateStale)
		snap.StateReason = "no successful poll in 5m"

		alertID := canonicalDiscreteStateStateID(snap.ID, connectionDegradedStateKey)

		m.CheckConnection(snap)
		if testHasActiveAlert(t, m, alertID) {
			t.Fatal("expected no alert after 1 stale observation")
		}
		m.CheckConnection(snap)
		if testHasActiveAlert(t, m, alertID) {
			t.Fatal("expected no alert after 2 stale observations")
		}
		m.CheckConnection(snap)

		alert := testRequireActiveAlert(t, m, alertID)
		if alert.Level != AlertLevelWarning {
			t.Errorf("expected warning level for stale, got %s", alert.Level)
		}
		if alert.Type != connectionDegradedAlertType {
			t.Errorf("expected alert type %s, got %s", connectionDegradedAlertType, alert.Type)
		}
		if alert.ResourceID != snap.ID {
			t.Errorf("expected resourceID %s, got %s", snap.ID, alert.ResourceID)
		}
		if got, _ := alert.Metadata["connectionType"].(string); got != string(ConnectionTypePVE) {
			t.Errorf("expected connectionType=pve in metadata, got %q", got)
		}
		if got, _ := alert.Metadata["state"].(string); got != string(ConnectionStateStale) {
			t.Errorf("expected state=stale in metadata, got %q", got)
		}
	})

	t.Run("unreachable escalates an already-firing warning to critical", func(t *testing.T) {
		m := newTestManager(t)
		snap := platformConnectionSnapshot("conn-esc", "Delly PVE", ConnectionStateStale)
		alertID := canonicalDiscreteStateStateID(snap.ID, connectionDegradedStateKey)

		// Build up to a firing warning first.
		m.CheckConnection(snap)
		m.CheckConnection(snap)
		m.CheckConnection(snap)
		warningAlert := testRequireActiveAlert(t, m, alertID)
		if warningAlert.Level != AlertLevelWarning {
			t.Fatalf("expected warning alert before escalation, got %s", warningAlert.Level)
		}

		// Now the connection goes fully unreachable.
		snap.State = ConnectionStateUnreachable
		snap.StateReason = "context deadline exceeded"
		m.CheckConnection(snap)

		escalated := testRequireActiveAlert(t, m, alertID)
		if escalated.Level != AlertLevelCritical {
			t.Errorf("expected critical level after escalation, got %s", escalated.Level)
		}
		if got, _ := escalated.Metadata["state"].(string); got != string(ConnectionStateUnreachable) {
			t.Errorf("expected state=unreachable in metadata, got %q", got)
		}
	})

	t.Run("unauthorized fires critical from cold start", func(t *testing.T) {
		m := newTestManager(t)
		snap := platformConnectionSnapshot("conn-auth", "Delly PVE", ConnectionStateUnauthorized)
		snap.StateReason = "invalid api key"

		alertID := canonicalDiscreteStateStateID(snap.ID, connectionDegradedStateKey)

		m.CheckConnection(snap)
		m.CheckConnection(snap)
		m.CheckConnection(snap)

		alert := testRequireActiveAlert(t, m, alertID)
		if alert.Level != AlertLevelCritical {
			t.Errorf("expected critical level for unauthorized, got %s", alert.Level)
		}
	})

	t.Run("paused connection never fires", func(t *testing.T) {
		m := newTestManager(t)
		snap := platformConnectionSnapshot("conn-paused", "Delly PVE", ConnectionStatePaused)

		for i := 0; i < 5; i++ {
			m.CheckConnection(snap)
		}

		alertID := canonicalDiscreteStateStateID(snap.ID, connectionDegradedStateKey)
		if testHasActiveAlert(t, m, alertID) {
			t.Fatal("expected no alert for paused connection")
		}
	})

	t.Run("disabled connection never fires even when unreachable", func(t *testing.T) {
		m := newTestManager(t)
		snap := platformConnectionSnapshot("conn-disabled", "Delly PVE", ConnectionStateUnreachable)
		snap.Enabled = false

		for i := 0; i < 5; i++ {
			m.CheckConnection(snap)
		}

		alertID := canonicalDiscreteStateStateID(snap.ID, connectionDegradedStateKey)
		if testHasActiveAlert(t, m, alertID) {
			t.Fatal("expected no alert for disabled connection")
		}
	})

	t.Run("agent connection is dropped before reaching the alert path", func(t *testing.T) {
		m := newTestManager(t)
		// Synthesize a non-platform snapshot directly — agents and
		// availability targets never reach CheckConnection in production
		// because the aggregator drops them, but a defense-in-depth test
		// here guards the type guard.
		snap := ConnectionSnapshot{
			ID:      "agent-1",
			Name:    "Host agent",
			Type:    ConnectionType("agent"),
			State:   ConnectionStateUnreachable,
			Enabled: true,
		}

		for i := 0; i < 5; i++ {
			m.CheckConnection(snap)
		}

		alertID := canonicalDiscreteStateStateID(snap.ID, connectionDegradedStateKey)
		if testHasActiveAlert(t, m, alertID) {
			t.Fatal("expected no alert for non-platform connection type")
		}
		m.mu.RLock()
		count := testCoreConfirmations(m, snap.ID, canonicalDiscreteStateSpecID(snap.ID, connectionDegradedStateKey))
		m.mu.RUnlock()
		if count != 0 {
			t.Errorf("expected degraded count untouched for non-platform type, got %d", count)
		}
	})
}

func TestCheckConnectionHonorsOwningResourceAvailabilityPolicy(t *testing.T) {
	pbs, adapter, canonicalID := newPBSOfflinePolicyFixture(t)
	connectionID := "pbs:" + pbs.Name
	resolvedPolicyID, ok := adapter.ResolveCanonicalResourceID(pbs.ID)
	if !ok || resolvedPolicyID != canonicalID {
		t.Fatalf("PBS policy ID %q resolved to %q, %t; want %q", pbs.ID, resolvedPolicyID, ok, canonicalID)
	}

	newPBSConnectionManager := func(t *testing.T) (*Manager, chan *Alert) {
		t.Helper()
		m := newTestManager(t)
		m.SetResourceIntentIdentityResolver(adapter.ResolveCanonicalResourceID)
		cfg := m.GetConfig()
		cfg.ActivationState = ActivationActive
		m.UpdateConfig(cfg)
		fired := make(chan *Alert, 4)
		m.SetAlertCallback(func(alert *Alert) { fired <- alert })
		return m, fired
	}
	snap := ConnectionSnapshot{
		ID:               connectionID,
		PolicyResourceID: pbs.ID,
		Name:             pbs.Name,
		Type:             ConnectionTypePBS,
		State:            ConnectionStateUnreachable,
		Enabled:          true,
	}
	alertID := canonicalDiscreteStateStateID(snap.ID, connectionDegradedStateKey)

	t.Run("canonical per-resource offline toggle blocks alert and notification", func(t *testing.T) {
		m, fired := newPBSConnectionManager(t)
		cfg := m.GetConfig()
		cfg.Overrides = map[string]ThresholdConfig{
			canonicalID: {DisableConnectivity: true},
		}
		m.UpdateConfig(cfg)

		for range 5 {
			m.CheckConnection(snap)
		}

		if testHasActiveAlert(t, m, alertID) {
			t.Fatal("offline-disabled PBS created a connection-degraded alert")
		}
		m.mu.RLock()
		tracked := testCoreHasIncident(m, snap.ID, canonicalDiscreteStateSpecID(snap.ID, connectionDegradedStateKey))
		m.mu.RUnlock()
		if tracked {
			t.Fatal("offline-disabled PBS retained connection-degraded confirmation state")
		}
		select {
		case alert := <-fired:
			t.Fatalf("offline-disabled PBS dispatched %q", alert.ID)
		default:
		}
	})

	t.Run("global PBS offline toggle blocks the parallel detector", func(t *testing.T) {
		m, fired := newPBSConnectionManager(t)
		cfg := m.GetConfig()
		cfg.DisableAllPBSOffline = true
		m.UpdateConfig(cfg)

		for range 5 {
			m.CheckConnection(snap)
		}

		if testHasActiveAlert(t, m, alertID) {
			t.Fatal("global PBS offline toggle left connection-degraded active")
		}
		select {
		case alert := <-fired:
			t.Fatalf("global PBS offline toggle dispatched %q", alert.ID)
		default:
		}
	})

	t.Run("enabling the toggle immediately resolves a standing alert", func(t *testing.T) {
		m, fired := newPBSConnectionManager(t)
		for range 3 {
			m.CheckConnection(snap)
		}
		testRequireActiveAlert(t, m, alertID)
		select {
		case <-fired:
		default:
			t.Fatal("enabled PBS connection did not dispatch its firing alert")
		}

		cfg := m.GetConfig()
		cfg.Overrides = map[string]ThresholdConfig{
			canonicalID: {DisableConnectivity: true},
		}
		m.UpdateConfig(cfg)

		if testHasActiveAlert(t, m, alertID) {
			t.Fatal("policy update did not immediately resolve connection-degraded")
		}
		m.mu.RLock()
		tracked := testCoreHasIncident(m, snap.ID, canonicalDiscreteStateSpecID(snap.ID, connectionDegradedStateKey))
		m.mu.RUnlock()
		if tracked {
			t.Fatal("policy update left connection-degraded confirmation state")
		}
	})

	t.Run("expected-offline intent blocks both state and notification", func(t *testing.T) {
		m, fired := newPBSConnectionManager(t)
		m.SetOperatorIntentContextResolver(func(resourceID string, observedAt time.Time) (OperatorIntentContext, bool) {
			if resourceID != connectionID {
				return OperatorIntentContext{}, false
			}
			return OperatorIntentContext{MonitoringMode: "expected_offline", LifecycleState: "active"}, true
		})

		for range 5 {
			m.CheckConnection(snap)
		}

		if testHasActiveAlert(t, m, alertID) {
			t.Fatal("expected-offline PBS created a connection-degraded alert")
		}
		select {
		case alert := <-fired:
			t.Fatalf("expected-offline PBS dispatched %q", alert.ID)
		default:
		}
	})
}

func TestConnectionDegradedUsesOfflineNotificationPolicy(t *testing.T) {
	alert := &Alert{Type: connectionDegradedAlertType}
	if got := quietHoursCategoryForAlert(alert); got != "offline" {
		t.Fatalf("connection-degraded quiet-hours category = %q, want offline", got)
	}
}

func TestConnectionDegradedPolicyUsesEveryPlatformResourceOverride(t *testing.T) {
	tests := []struct {
		name           string
		connectionType ConnectionType
		resourceID     string
	}{
		{name: "PVE", connectionType: ConnectionTypePVE, resourceID: "pve:lab"},
		{name: "PBS", connectionType: ConnectionTypePBS, resourceID: "pbs:backup"},
		{name: "PMG", connectionType: ConnectionTypePMG, resourceID: "pmg:mail"},
		{name: "VMware", connectionType: ConnectionTypeVMware, resourceID: "vmware:vcenter"},
		{name: "TrueNAS", connectionType: ConnectionTypeTrueNAS, resourceID: "truenas:nas"},
	}

	for _, test := range tests {
		t.Run(test.name, func(t *testing.T) {
			m := newTestManager(t)
			cfg := m.GetConfig()
			cfg.Overrides = map[string]ThresholdConfig{
				test.resourceID: {DisableConnectivity: true},
			}
			m.UpdateConfig(cfg)

			snap := ConnectionSnapshot{
				ID:      test.resourceID,
				Name:    test.name,
				Type:    test.connectionType,
				State:   ConnectionStateUnreachable,
				Enabled: true,
			}
			for range 5 {
				m.CheckConnection(snap)
			}

			alertID := canonicalDiscreteStateStateID(snap.ID, connectionDegradedStateKey)
			if testHasActiveAlert(t, m, alertID) {
				t.Fatalf("%s connectivity-disabled override created connection-degraded", test.name)
			}
		})
	}
}

func TestClearConnectionDegradedAlert(t *testing.T) {
	t.Run("clears an active alert after the recovery confirmation gate", func(t *testing.T) {
		m := newTestManager(t)
		resolvedCh := make(chan struct{}, 1)
		m.SetResolvedCallback(func(alertID string) {
			resolvedCh <- struct{}{}
		})

		snap := platformConnectionSnapshot("conn-recover", "Delly PVE", ConnectionStateStale)
		alertID := canonicalDiscreteStateStateID(snap.ID, connectionDegradedStateKey)

		// Drive the alert to firing.
		m.CheckConnection(snap)
		m.CheckConnection(snap)
		m.CheckConnection(snap)
		testRequireActiveAlert(t, m, alertID)

		// State returns to active — first observation does NOT clear yet
		// because the recovery confirmation gate is 3.
		snap.State = ConnectionStateActive
		m.CheckConnection(snap)
		if !testHasActiveAlert(t, m, alertID) {
			t.Fatal("expected alert to remain active until recovery confirmations are met")
		}
		m.mu.RLock()
		count1 := testCoreRecoveryCount(m, snap.ID, canonicalDiscreteStateSpecID(snap.ID, connectionDegradedStateKey))
		m.mu.RUnlock()
		if count1 != 1 {
			t.Errorf("expected recovery confirmation 1, got %d", count1)
		}

		m.CheckConnection(snap)
		if !testHasActiveAlert(t, m, alertID) {
			t.Fatal("expected alert to remain active until recovery confirmations are met")
		}

		m.CheckConnection(snap)
		if testHasActiveAlert(t, m, alertID) {
			t.Fatal("expected alert to clear after 3 recovery confirmations")
		}

		// Confirm the resolved callback fires (handled via the safeCall path).
		select {
		case <-resolvedCh:
		case <-time.After(2 * time.Second):
			t.Fatal("expected resolved callback to fire")
		}
	})

	t.Run("no-op when nothing was ever firing", func(t *testing.T) {
		m := newTestManager(t)
		snap := platformConnectionSnapshot("conn-quiet", "Delly PVE", ConnectionStateActive)

		m.CheckConnection(snap)
		m.CheckConnection(snap)

		m.mu.RLock()
		alertCount := len(m.activeAlerts)
		recovery := testCoreRecoveryCount(m, snap.ID, canonicalDiscreteStateSpecID(snap.ID, connectionDegradedStateKey))
		m.mu.RUnlock()

		if alertCount != 0 {
			t.Errorf("expected no active alerts, got %d", alertCount)
		}
		if recovery != 0 {
			t.Errorf("expected no recovery confirmations tracked, got %d", recovery)
		}
	})

	t.Run("degraded observation resets in-flight recovery confirmations", func(t *testing.T) {
		m := newTestManager(t)
		snap := platformConnectionSnapshot("conn-flap", "Delly PVE", ConnectionStateStale)
		alertID := canonicalDiscreteStateStateID(snap.ID, connectionDegradedStateKey)

		// Drive to firing.
		m.CheckConnection(snap)
		m.CheckConnection(snap)
		m.CheckConnection(snap)
		testRequireActiveAlert(t, m, alertID)

		// One healthy observation builds up a recovery confirmation.
		snap.State = ConnectionStateActive
		m.CheckConnection(snap)
		m.mu.RLock()
		recoveryBefore := testCoreRecoveryCount(m, snap.ID, canonicalDiscreteStateSpecID(snap.ID, connectionDegradedStateKey))
		m.mu.RUnlock()
		if recoveryBefore != 1 {
			t.Fatalf("expected 1 recovery confirmation, got %d", recoveryBefore)
		}

		// A degraded blip should wipe the recovery progress so a single
		// flap doesn't pretend the outage is healed.
		snap.State = ConnectionStateStale
		m.CheckConnection(snap)
		m.mu.RLock()
		recoveryAfter := testCoreRecoveryCount(m, snap.ID, canonicalDiscreteStateSpecID(snap.ID, connectionDegradedStateKey))
		m.mu.RUnlock()
		if recoveryAfter != 0 {
			t.Errorf("expected recovery confirmations reset to 0, got %d", recoveryAfter)
		}
		if !testHasActiveAlert(t, m, alertID) {
			t.Fatal("expected alert to stay active after flap")
		}
	})
}

func TestConnectionAlertReFireCooldown(t *testing.T) {
	t.Run("re-fire within cooldown does not create duplicate history entry", func(t *testing.T) {
		m := newTestManager(t)
		snap := platformConnectionSnapshot("conn-cooldown", "Delly PVE", ConnectionStateUnreachable)
		snap.StateReason = "context deadline exceeded"
		alertID := canonicalDiscreteStateStateID(snap.ID, connectionDegradedStateKey)

		m.CheckConnection(snap)
		m.CheckConnection(snap)
		m.CheckConnection(snap)
		originalAlert := testRequireActiveAlert(t, m, alertID)
		originalStart := originalAlert.StartTime

		historyAfterFire := len(m.historyManager.GetAllHistory(1000))

		snap.State = ConnectionStateActive
		m.CheckConnection(snap)
		m.CheckConnection(snap)
		m.CheckConnection(snap)
		if testHasActiveAlert(t, m, alertID) {
			t.Fatal("expected alert to clear after recovery")
		}

		snap.State = ConnectionStateUnreachable
		m.CheckConnection(snap)
		m.CheckConnection(snap)
		m.CheckConnection(snap)
		reactivated := testRequireActiveAlert(t, m, alertID)

		historyAfterReFire := len(m.historyManager.GetAllHistory(1000))
		if historyAfterReFire != historyAfterFire {
			t.Errorf("expected %d history entries after re-fire (same as after initial fire), got %d", historyAfterFire, historyAfterReFire)
		}

		if !reactivated.StartTime.Equal(originalStart) {
			t.Errorf("expected reactivated alert to preserve original StartTime %v, got %v", originalStart, reactivated.StartTime)
		}
	})
}