Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

17.4 KB
Raw Blame
package alerts

import (
	"testing"
	"time"
)

func TestSetLicenseCheckerStoresChecker(t *testing.T) {
	m := newTestManager(t)

	m.SetLicenseChecker(func(feature string) bool {
		return feature == "updates"
	})

	m.mu.RLock()
	checker := m.hasProFeature
	m.mu.RUnlock()

	if checker == nil {
		t.Fatal("expected license checker to be stored")
	}
	if !checker("updates") {
		t.Fatal("expected checker to allow configured feature")
	}
	if checker("other") {
		t.Fatal("expected checker to reject other features")
	}
}

func TestAcknowledgeAlertInvokesCallback(t *testing.T) {
	m := newTestManager(t)
	alertID := "ack-callback-alert"

	m.mu.Lock()
	m.activeAlerts[alertID] = &Alert{ID: alertID, Type: "cpu", ResourceID: "guest-1"}
	m.mu.Unlock()

	done := make(chan struct{})
	var gotAlert *Alert
	var gotUser string
	m.SetAcknowledgedCallback(func(alert *Alert, user string) {
		gotAlert = alert
		gotUser = user
		close(done)
	})

	if err := m.AcknowledgeAlert(alertID, "alice"); err != nil {
		t.Fatalf("AcknowledgeAlert returned error: %v", err)
	}

	select {
	case <-done:
	case <-time.After(time.Second):
		t.Fatal("acknowledged callback not invoked")
	}

	if gotAlert == nil {
		t.Fatal("expected callback alert payload")
	}
	if gotUser != "alice" {
		t.Fatalf("expected callback user alice, got %q", gotUser)
	}
	if !gotAlert.Acknowledged || gotAlert.AckTime == nil || gotAlert.AckUser != "alice" {
		t.Fatalf("callback alert did not include acknowledged state: %+v", gotAlert)
	}

	m.mu.RLock()
	stored := testRequireActiveAlert(t, m, alertID)
	m.mu.RUnlock()
	if gotAlert == stored {
		t.Fatal("expected callback to receive a cloned alert")
	}
}

func TestUnacknowledgeAlertInvokesCallback(t *testing.T) {
	m := newTestManager(t)
	alertID := "unack-callback-alert"
	now := time.Now().Add(-time.Minute)

	m.mu.Lock()
	m.activeAlerts[alertID] = &Alert{
		ID:           alertID,
		Type:         "cpu",
		ResourceID:   "guest-1",
		Acknowledged: true,
		AckUser:      "alice",
		AckTime:      &now,
	}
	m.ackState[alertID] = ackRecord{acknowledged: true, user: "alice", time: now}
	m.mu.Unlock()

	done := make(chan struct{})
	var gotAlert *Alert
	var gotUser string
	m.SetUnacknowledgedCallback(func(alert *Alert, user string) {
		gotAlert = alert
		gotUser = user
		close(done)
	})

	if err := m.UnacknowledgeAlert(alertID); err != nil {
		t.Fatalf("UnacknowledgeAlert returned error: %v", err)
	}

	select {
	case <-done:
	case <-time.After(time.Second):
		t.Fatal("unacknowledged callback not invoked")
	}

	if gotAlert == nil {
		t.Fatal("expected callback alert payload")
	}
	if gotUser != "" {
		t.Fatalf("expected empty callback user, got %q", gotUser)
	}
	if gotAlert.Acknowledged || gotAlert.AckTime != nil || gotAlert.AckUser != "" {
		t.Fatalf("callback alert did not include unacknowledged state: %+v", gotAlert)
	}

	m.mu.RLock()
	stored := testRequireActiveAlert(t, m, alertID)
	m.mu.RUnlock()
	if gotAlert == stored {
		t.Fatal("expected callback to receive a cloned alert")
	}
}

func TestSafeCallAcknowledgedCallbackRecoversPanic(t *testing.T) {
	m := newTestManager(t)

	done := make(chan struct{})
	m.SetAcknowledgedCallback(func(alert *Alert, user string) {
		defer close(done)
		panic("ack callback panic")
	})

	m.safeCallAcknowledgedCallback(&Alert{ID: "ack-panic-alert"}, "alice")

	select {
	case <-done:
	case <-time.After(time.Second):
		t.Fatal("panicing acknowledged callback was not invoked")
	}
}

func TestSafeCallUnacknowledgedCallbackRecoversPanic(t *testing.T) {
	m := newTestManager(t)

	done := make(chan struct{})
	m.SetUnacknowledgedCallback(func(alert *Alert, user string) {
		defer close(done)
		panic("unack callback panic")
	})

	m.safeCallUnacknowledgedCallback(&Alert{ID: "unack-panic-alert"}, "")

	select {
	case <-done:
	case <-time.After(time.Second):
		t.Fatal("panicing unacknowledged callback was not invoked")
	}
}

func TestCheckMetricInvokesAICallbackWhenNotificationsSuppressed(t *testing.T) {
	m := newTestManager(t)

	aiDone := make(chan *Alert, 1)
	notifyDone := make(chan struct{}, 1)

	m.SetAlertForAICallback(func(alert *Alert) {
		aiDone <- alert
	})
	m.SetAlertCallback(func(alert *Alert) {
		notifyDone <- struct{}{}
	})

	m.mu.Lock()
	m.config.ActivationState = ActivationPending
	m.config.TimeThresholds = map[string]int{}
	m.config.SuppressionWindow = 0
	m.config.MinimumDelta = 0
	m.mu.Unlock()

	m.checkMetric("ai-resource", "AI Resource", "node-1", "node-1/qemu/100", "guest", "cpu", 95, &HysteresisThreshold{Trigger: 80, Clear: 75}, nil)

	var aiAlert *Alert
	select {
	case aiAlert = <-aiDone:
	case <-time.After(time.Second):
		t.Fatal("AI callback was not invoked")
	}

	select {
	case <-notifyDone:
		t.Fatal("expected regular notification callback to be suppressed while activation is pending")
	case <-time.After(100 * time.Millisecond):
	}

	m.mu.RLock()
	stored := testRequireActiveAlert(t, m, canonicalMetricStateID("ai-resource", "cpu"))
	m.mu.RUnlock()

	if stored == nil {
		t.Fatal("expected metric alert to be stored")
	}
	if aiAlert == stored {
		t.Fatal("expected AI callback to receive a cloned alert")
	}
	expectedID := canonicalMetricStateID("ai-resource", "cpu")
	if aiAlert.ID != expectedID {
		t.Fatalf("expected AI callback alert ID %q, got %q", expectedID, aiAlert.ID)
	}
}

func TestDispatchAlertInvokesSubscribedAlertCallbacks(t *testing.T) {
	m := newTestManager(t)

	alert := &Alert{
		ID:           "subscribed-alert",
		Type:         "cpu",
		Level:        AlertLevelWarning,
		ResourceID:   "vm-100",
		ResourceName: "web-1",
		StartTime:    time.Now().Add(-time.Minute),
		LastSeen:     time.Now(),
	}

	firstDone := make(chan string, 1)
	secondDone := make(chan string, 1)
	m.SubscribeAlertCallback(func(alert *Alert) {
		firstDone <- alert.ID
	})
	m.SubscribeAlertCallback(func(alert *Alert) {
		secondDone <- alert.ID
	})

	m.mu.Lock()
	m.config.ActivationState = ActivationActive
	dispatched := m.dispatchAlert(alert, false)
	m.mu.Unlock()

	if !dispatched {
		t.Fatal("expected dispatchAlert to report a subscribed callback dispatch")
	}

	select {
	case got := <-firstDone:
		if got != alert.ID {
			t.Fatalf("first callback ID = %q, want %q", got, alert.ID)
		}
	case <-time.After(time.Second):
		t.Fatal("expected first subscribed callback to fire")
	}

	select {
	case got := <-secondDone:
		if got != alert.ID {
			t.Fatalf("second callback ID = %q, want %q", got, alert.ID)
		}
	case <-time.After(time.Second):
		t.Fatal("expected second subscribed callback to fire")
	}
}

func TestSubscribeAlertCallbackUnsubscribeKeepsOtherSubscribers(t *testing.T) {
	m := newTestManager(t)

	alert := &Alert{
		ID:           "unsubscribed-alert",
		Type:         "cpu",
		Level:        AlertLevelWarning,
		ResourceID:   "vm-101",
		ResourceName: "api-1",
		StartTime:    time.Now().Add(-time.Minute),
		LastSeen:     time.Now(),
	}

	firstDone := make(chan string, 1)
	secondDone := make(chan string, 1)
	unsubscribeFirst := m.SubscribeAlertCallback(func(alert *Alert) {
		firstDone <- alert.ID
	})
	m.SubscribeAlertCallback(func(alert *Alert) {
		secondDone <- alert.ID
	})
	unsubscribeFirst()

	m.mu.Lock()
	m.config.ActivationState = ActivationActive
	dispatched := m.dispatchAlert(alert, false)
	m.mu.Unlock()

	if !dispatched {
		t.Fatal("expected dispatchAlert to report a subscribed callback dispatch")
	}

	select {
	case got := <-firstDone:
		t.Fatalf("unsubscribed callback fired with alert ID %q", got)
	case <-time.After(100 * time.Millisecond):
	}

	select {
	case got := <-secondDone:
		if got != alert.ID {
			t.Fatalf("second callback ID = %q, want %q", got, alert.ID)
		}
	case <-time.After(time.Second):
		t.Fatal("expected remaining subscribed callback to fire")
	}
}

func TestDispatchAlertSuppressesMonitorOnlyAlert(t *testing.T) {
	m := newTestManager(t)

	alert := &Alert{
		ID:           "monitor-only-alert",
		Type:         "cpu",
		Level:        AlertLevelWarning,
		ResourceID:   "vm-102",
		ResourceName: "worker-1",
		StartTime:    time.Now().Add(-time.Minute),
		LastSeen:     time.Now(),
		Metadata: map[string]interface{}{
			"monitorOnly": true,
		},
	}

	notifyDone := make(chan string, 1)
	m.SubscribeAlertCallback(func(alert *Alert) {
		notifyDone <- alert.ID
	})

	m.mu.Lock()
	m.config.ActivationState = ActivationActive
	dispatched := m.dispatchAlert(alert, false)
	m.mu.Unlock()

	if dispatched {
		t.Fatal("expected monitor-only alert dispatch to be suppressed")
	}

	select {
	case got := <-notifyDone:
		t.Fatalf("monitor-only alert fired callback with alert ID %q", got)
	case <-time.After(100 * time.Millisecond):
	}
}

func TestOnAlertHistoryRegistersCallback(t *testing.T) {
	m := newTestManager(t)

	done := make(chan string, 1)
	m.OnAlertHistory(func(alert Alert) {
		done <- alert.ID
	})

	m.historyManager.AddAlert(Alert{ID: "history-wrapper-callback"})

	select {
	case got := <-done:
		if got != "history-wrapper-callback" {
			t.Fatalf("expected callback ID history-wrapper-callback, got %q", got)
		}
	case <-time.After(time.Second):
		t.Fatal("history callback was not invoked")
	}
}

func TestMigrateActivationState(t *testing.T) {
	t.Run("existing install with active alerts migrates to active", func(t *testing.T) {
		m := &Manager{
			activeAlerts: map[string]*Alert{
				"existing-alert": {ID: "existing-alert"},
			},
		}
		cfg := AlertConfig{Overrides: map[string]ThresholdConfig{}}

		m.migrateActivationState(&cfg)

		if cfg.ActivationState != ActivationActive {
			t.Fatalf("expected activation state %q, got %q", ActivationActive, cfg.ActivationState)
		}
		if cfg.ActivationTime == nil {
			t.Fatal("expected activation time to be set during migration")
		}
	})

	t.Run("existing install with overrides migrates to active", func(t *testing.T) {
		m := &Manager{}
		cfg := AlertConfig{
			Overrides: map[string]ThresholdConfig{
				"qemu-100": {},
			},
		}

		m.migrateActivationState(&cfg)

		if cfg.ActivationState != ActivationActive {
			t.Fatalf("expected activation state %q, got %q", ActivationActive, cfg.ActivationState)
		}
		if cfg.ActivationTime == nil {
			t.Fatal("expected activation time to be set during migration")
		}
	})

	t.Run("new install defaults to pending", func(t *testing.T) {
		m := &Manager{}
		cfg := AlertConfig{}

		m.migrateActivationState(&cfg)

		if cfg.ActivationState != ActivationPending {
			t.Fatalf("expected activation state %q, got %q", ActivationPending, cfg.ActivationState)
		}
		if cfg.ActivationTime != nil {
			t.Fatal("expected activation time to remain nil for pending state")
		}
	})

	t.Run("pre-set state is preserved", func(t *testing.T) {
		m := &Manager{
			activeAlerts: map[string]*Alert{
				"existing-alert": {ID: "existing-alert"},
			},
		}
		now := time.Now().Add(-time.Hour)
		cfg := AlertConfig{
			ActivationState: ActivationSnoozed,
			ActivationTime:  &now,
		}

		m.migrateActivationState(&cfg)

		if cfg.ActivationState != ActivationSnoozed {
			t.Fatalf("expected pre-set activation state to be preserved, got %q", cfg.ActivationState)
		}
		if cfg.ActivationTime == nil || !cfg.ActivationTime.Equal(now) {
			t.Fatal("expected pre-set activation time to be preserved")
		}
	})
}

func TestUpdateConfigPreservesExistingActivationStateWhenOmitted(t *testing.T) {
	m := newTestManager(t)
	activatedAt := time.Now().Add(-time.Hour)

	m.mu.Lock()
	m.config.ActivationState = ActivationActive
	m.config.ActivationTime = &activatedAt
	cfg := m.config
	m.mu.Unlock()

	cfg.ActivationState = ""
	cfg.ActivationTime = nil

	m.UpdateConfig(cfg)

	got := m.GetConfig()
	if got.ActivationState != ActivationActive {
		t.Fatalf("expected activation state %q, got %q", ActivationActive, got.ActivationState)
	}
	if got.ActivationTime == nil {
		t.Fatal("expected activation time to be preserved")
	}
	if !got.ActivationTime.Equal(activatedAt) {
		t.Fatalf("expected activation time %s, got %s", activatedAt, got.ActivationTime)
	}
}

func TestValidateQuietHoursTimezone(t *testing.T) {
	t.Run("invalid timezone disables quiet hours", func(t *testing.T) {
		cfg := AlertConfig{
			Schedule: ScheduleConfig{
				QuietHours: QuietHours{
					Enabled:  true,
					Timezone: "Invalid/Timezone",
				},
			},
		}

		validateQuietHoursTimezone(&cfg)

		if cfg.Schedule.QuietHours.Enabled {
			t.Fatal("expected quiet hours to be disabled for invalid timezone")
		}
	})

	t.Run("valid timezone keeps quiet hours enabled", func(t *testing.T) {
		cfg := AlertConfig{
			Schedule: ScheduleConfig{
				QuietHours: QuietHours{
					Enabled:  true,
					Timezone: "America/New_York",
				},
			},
		}

		validateQuietHoursTimezone(&cfg)

		if !cfg.Schedule.QuietHours.Enabled {
			t.Fatal("expected quiet hours to remain enabled for valid timezone")
		}
	})

	t.Run("empty timezone leaves quiet hours enabled", func(t *testing.T) {
		cfg := AlertConfig{
			Schedule: ScheduleConfig{
				QuietHours: QuietHours{
					Enabled:  true,
					Timezone: "",
				},
			},
		}

		validateQuietHoursTimezone(&cfg)

		if !cfg.Schedule.QuietHours.Enabled {
			t.Fatal("expected quiet hours to remain enabled when timezone is empty")
		}
	})

	t.Run("disabled quiet hours stay disabled", func(t *testing.T) {
		cfg := AlertConfig{
			Schedule: ScheduleConfig{
				QuietHours: QuietHours{
					Enabled:  false,
					Timezone: "Invalid/Timezone",
				},
			},
		}

		validateQuietHoursTimezone(&cfg)

		if cfg.Schedule.QuietHours.Enabled {
			t.Fatal("expected quiet hours to remain disabled")
		}
	})
}
func TestMonitorOnlyAlertNeverEscalates(t *testing.T) {
	m := newTestManager(t)
	now := time.Now().UTC()
	m.mu.Lock()
	m.config.Enabled = true
	m.config.ActivationState = ActivationActive
	m.config.Schedule.Escalation = EscalationConfig{
		Enabled:        true,
		Levels:         []EscalationLevel{{After: 30, Notify: "email"}},
		RepeatCritical: true,
		RepeatEvery:    30,
	}
	monitorOnly := &Alert{
		ID:        "vm-monitor-only",
		Type:      "cpu",
		StartTime: now.Add(-time.Hour),
		Metadata:  map[string]interface{}{"monitorOnly": true},
	}
	repeatingMonitorOnly := &Alert{
		ID:              "vm-monitor-only-critical",
		Type:            "cpu",
		Level:           AlertLevelCritical,
		StartTime:       now.Add(-2 * time.Hour),
		LastEscalation:  1,
		EscalationTimes: []time.Time{now.Add(-time.Hour)},
		Metadata:        map[string]interface{}{"monitorOnly": true},
	}
	ordinary := &Alert{ID: "vm-ordinary", Type: "cpu", StartTime: now.Add(-time.Hour)}
	m.setActiveAlertNoLock(monitorOnly.ID, monitorOnly)
	m.setActiveAlertNoLock(repeatingMonitorOnly.ID, repeatingMonitorOnly)
	m.setActiveAlertNoLock(ordinary.ID, ordinary)
	m.mu.Unlock()

	m.checkEscalations()
	if monitorOnly.LastEscalation != 0 || len(monitorOnly.EscalationTimes) != 0 {
		t.Fatalf("monitor-only alert scheduled escalation: %+v", monitorOnly)
	}
	if repeatingMonitorOnly.LastEscalation != 1 || len(repeatingMonitorOnly.EscalationTimes) != 1 {
		t.Fatalf("monitor-only critical alert repeated escalation: %+v", repeatingMonitorOnly)
	}
	if ordinary.LastEscalation != 1 {
		t.Fatalf("ordinary alert did not exercise scheduler: %+v", ordinary)
	}
}

func TestMonitorOnlyPolicyChangeRejectsPendingEscalation(t *testing.T) {
	m := newTestManager(t)
	now := time.Now().UTC()
	m.mu.Lock()
	m.config.Enabled = true
	m.config.ActivationState = ActivationActive
	m.config.Schedule.Escalation = EscalationConfig{
		Enabled: true,
		Levels:  []EscalationLevel{{After: 30, Notify: "email"}},
	}
	alert := &Alert{ID: "vm-becomes-monitor-only", Type: "cpu", StartTime: now.Add(-time.Hour)}
	m.setActiveAlertNoLock(alert.ID, alert)
	snapshot := cloneAlertForOutput(alert)
	alert.Metadata = map[string]interface{}{"monitorOnly": true}
	m.mu.Unlock()

	if _, _, eligible := m.PrepareEscalationNotification(snapshot, 1); eligible {
		t.Fatal("queued escalation remained eligible after alert became monitor-only")
	}
}

func TestMonitorOnlyAlertCannotAcquireQuietHoursReplay(t *testing.T) {
	m := newTestManager(t)
	now := time.Date(2026, time.April, 12, 12, 0, 0, 0, time.UTC)
	m.now = func() time.Time { return now }
	m.config.ActivationState = ActivationActive
	m.config.Schedule.QuietHours = QuietHours{
		Enabled:  true,
		Start:    "00:00",
		End:      "23:59",
		Timezone: "UTC",
		Days:     map[string]bool{"sunday": true},
	}
	m.SetAlertCallback(func(*Alert) { t.Error("monitor-only firing alert reached callback") })
	alert := &Alert{
		ID:        "monitor-only-quiet-hours",
		Type:      "cpu",
		Level:     AlertLevelWarning,
		StartTime: m.policyNow().Add(-time.Hour),
		Metadata: map[string]interface{}{
			"monitorOnly":              true,
			MetadataQuietHoursReplayAt: m.policyNow().Add(time.Hour).Format(time.RFC3339),
		},
	}
	if suppressed, _ := m.shouldSuppressNotification(alert); !suppressed {
		t.Fatal("fixture did not place alert in quiet hours")
	}
	m.mu.Lock()
	m.setActiveAlertNoLock(alert.ID, alert)
	if m.dispatchAlert(alert, false) {
		m.mu.Unlock()
		t.Fatal("monitor-only firing alert was dispatched")
	}
	m.mu.Unlock()
	if hasQuietHoursNotificationReplay(alert) {
		t.Fatalf("monitor-only alert retained replay metadata: %+v", alert.Metadata)
	}
	if alert.LastNotified != nil {
		t.Fatalf("monitor-only alert was marked notified at %s", alert.LastNotified)
	}
	if !m.ShouldSuppressNotification(alert) || hasQuietHoursNotificationReplay(alert) {
		t.Fatalf("public suppression helper deferred monitor-only alert: %+v", alert.Metadata)
	}
	lastNotified := m.policyNow().Add(-time.Minute)
	alert.LastNotified = &lastNotified
	if !m.ShouldSuppressResolvedNotification(alert) {
		t.Fatal("monitor-only recovery notification was admitted")
	}
	diagnosis, exists := m.DiagnoseAlertDelivery(alert.ID)
	if !exists || diagnosis.Status != AlertDeliveryStatusSuppressed || diagnosis.Reason != AlertDeliveryReasonMonitorOnly || diagnosis.QuietHoursReplayAt != nil {
		t.Fatalf("monitor-only delivery diagnosis = %+v, exists=%t", diagnosis, exists)
	}
}