Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

15.2 KB
Raw Blame
package alerts

import (
	"fmt"
	"sync/atomic"
	"testing"
	"time"

	"github.com/rcourtman/pulse-go-rewrite/internal/alerts/reducer"
	alertspecs "github.com/rcourtman/pulse-go-rewrite/internal/alerts/specs"
	"github.com/rcourtman/pulse-go-rewrite/internal/storagehealth"
	"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
)

// Ack-lifecycle parity: acknowledge suppression state must survive alert
// rebuilds (preserveAlertState's existing branch), survive short
// resolve/re-fire cycles via the canonical ack records
// (preserveAlertState's restore branch), be removed by unacknowledge, and
// expire once cleanup prunes an inactive record (one-hour TTL). The
// manager's restore has no age check of its own — expiry is cleanup — so
// the expiry scenario backdates the record and runs Cleanup explicitly,
// mirroring how the hour boundary is enforced in production.

type ackParityStep struct {
	action  string // "observe", "ack", "unack", "cleanup"
	matched bool
	advance time.Duration

	wantFiring bool
	wantAcked  bool
}

type ackParityScenario struct {
	name  string
	steps []ackParityStep
}

const ackParityResourceID = "parity-ack-1"

type managerAckParityEngine struct {
	manager *Manager
	clock   time.Time
}

func (e *managerAckParityEngine) alertID() string {
	return canonicalDiscreteStateStateID(ackParityResourceID, "parity-state")
}

func (e *managerAckParityEngine) shift(advance time.Duration) {
	if advance <= 0 {
		return
	}
	e.manager.resolvedMutex.Lock()
	for _, resolved := range e.manager.recentlyResolved {
		if resolved != nil {
			resolved.ResolvedTime = resolved.ResolvedTime.Add(-advance)
		}
	}
	e.manager.resolvedMutex.Unlock()
	e.manager.mu.Lock()
	for key, record := range e.manager.ackState {
		record.time = record.time.Add(-advance)
		if !record.inactiveAt.IsZero() {
			record.inactiveAt = record.inactiveAt.Add(-advance)
		}
		e.manager.ackState[key] = record
	}
	for key, record := range e.manager.ackStateByCanonical {
		record.time = record.time.Add(-advance)
		if !record.inactiveAt.IsZero() {
			record.inactiveAt = record.inactiveAt.Add(-advance)
		}
		e.manager.ackStateByCanonical[key] = record
	}
	e.manager.mu.Unlock()
}

func (e *managerAckParityEngine) step(t *testing.T, s ackParityStep) {
	t.Helper()
	e.clock = e.clock.Add(s.advance)
	e.shift(s.advance)

	switch s.action {
	case "ack":
		if err := e.manager.AcknowledgeAlert(e.alertID(), "richard"); err != nil {
			t.Fatalf("AcknowledgeAlert: %v", err)
		}
		return
	case "unack":
		if err := e.manager.UnacknowledgeAlert(e.alertID()); err != nil {
			t.Fatalf("UnacknowledgeAlert: %v", err)
		}
		return
	case "cleanup":
		e.manager.Cleanup(24 * time.Hour)
		return
	}

	spec, err := buildCanonicalDiscreteStateSpec(
		ackParityResourceID, "parity-ack", unifiedresources.ResourceTypeAgent,
		AlertLevelWarning, 1, false, "parity-state", []string{"bad"},
	)
	if err != nil {
		t.Fatalf("build spec: %v", err)
	}
	observed := "ok"
	if s.matched {
		observed = "bad"
	}
	if _, ok := e.manager.evaluateCanonicalLifecycleAlert(canonicalLifecycleAlertParams{
		Spec: spec,
		Evidence: alertspecs.AlertEvidence{
			ObservedAt:    e.clock,
			DiscreteState: &alertspecs.DiscreteStateEvidence{StateKey: "parity-state", Observed: observed},
		},
		AlertID:      e.alertID(),
		AlertType:    "parity-state",
		ResourceID:   ackParityResourceID,
		ResourceName: "parity-ack",
		Message:      "parity ack condition",
	}); !ok {
		t.Fatal("evaluateCanonicalLifecycleAlert rejected the parity spec")
	}
}

func ackParityScenarios() []ackParityScenario {
	return []ackParityScenario{
		{
			name: "ack survives rebuilds and unack clears it",
			steps: []ackParityStep{
				{action: "observe", matched: true, wantFiring: true, wantAcked: false},
				{action: "ack", wantFiring: true, wantAcked: true},
				{action: "observe", matched: true, advance: 30 * time.Second, wantFiring: true, wantAcked: true},
				{action: "unack", wantFiring: true, wantAcked: false},
				{action: "observe", matched: true, advance: 30 * time.Second, wantFiring: true, wantAcked: false},
			},
		},
		{
			name: "ack survives a short resolve and re-fire",
			steps: []ackParityStep{
				{action: "observe", matched: true, wantFiring: true, wantAcked: false},
				{action: "ack", wantFiring: true, wantAcked: true},
				{action: "observe", matched: false, advance: 30 * time.Second, wantFiring: false},
				{action: "observe", matched: true, advance: 30 * time.Second, wantFiring: true, wantAcked: true},
			},
		},
		{
			name: "ack expires after the inactive TTL",
			steps: []ackParityStep{
				{action: "observe", matched: true, wantFiring: true, wantAcked: false},
				{action: "ack", wantFiring: true, wantAcked: true},
				{action: "observe", matched: false, advance: 30 * time.Second, wantFiring: false},
				{action: "cleanup", advance: 70 * time.Minute, wantFiring: false},
				{action: "observe", matched: true, wantFiring: true, wantAcked: false},
			},
		},
	}
}

func TestReducerParityWithManagerAckLifecycle(t *testing.T) {
	for _, scenario := range ackParityScenarios() {
		scenario := scenario
		t.Run(scenario.name, func(t *testing.T) {
			epoch := time.Now().UTC().Truncate(time.Second)

			manager := NewManagerWithDataDir(t.TempDir(), WithoutPersistedAlertRestore())
			t.Cleanup(manager.Stop)
			manager.mu.Lock()
			manager.config.Enabled = true
			manager.config.FlappingEnabled = false
			manager.mu.Unlock()

			managerEngine := &managerAckParityEngine{manager: manager, clock: epoch}
			reducerState := reducer.NewState()
			reducerClock := epoch
			rule := reducer.DiscreteRule{Confirmations: 1}

			for i, step := range scenario.steps {
				managerEngine.step(t, step)
				reducerClock = reducerClock.Add(step.advance)
				switch step.action {
				case "ack":
					if !reducerState.Acknowledge(ackParityResourceID, "parity-state", "richard", reducerClock) {
						t.Fatalf("step %d: reducer Acknowledge failed", i)
					}
				case "unack":
					if !reducerState.Unacknowledge(ackParityResourceID, "parity-state") {
						t.Fatalf("step %d: reducer Unacknowledge failed", i)
					}
				case "cleanup":
					// The reducer needs no cleanup pass: AckRetention is
					// enforced deterministically at restore time.
				case "observe":
					reducerState.ApplyDiscrete(reducer.DiscreteSignal{
						ResourceID: ackParityResourceID,
						Key:        "parity-state",
						Matched:    step.matched,
						Severity:   reducer.SeverityWarning,
						ObservedAt: reducerClock,
					}, rule)
				}

				manager.mu.Lock()
				alert, exists := manager.getActiveAlertNoLock(managerEngine.alertID())
				manager.mu.Unlock()
				managerFiring := exists && alert != nil
				managerAcked := managerFiring && alert.Acknowledged
				incident, ok := reducerState.Incident(ackParityResourceID, "parity-state")
				reducerFiring := ok && incident.State == reducer.StateFiring
				reducerAcked := reducerFiring && incident.Acknowledged

				label := fmt.Sprintf("step %d (%s matched=%v advance=%s)", i, step.action, step.matched, step.advance)

				if managerFiring != step.wantFiring {
					t.Fatalf("%s: manager firing = %v, scenario expects %v (characterization drift)",
						label, managerFiring, step.wantFiring)
				}
				if managerFiring && managerAcked != step.wantAcked {
					t.Fatalf("%s: manager acked = %v, scenario expects %v", label, managerAcked, step.wantAcked)
				}

				if reducerFiring != managerFiring {
					t.Fatalf("%s: PARITY DIVERGENCE — reducer firing = %v, manager = %v",
						label, reducerFiring, managerFiring)
				}
				if managerFiring && reducerAcked != managerAcked {
					t.Fatalf("%s: PARITY DIVERGENCE — reducer acked = %v, manager = %v",
						label, reducerAcked, managerAcked)
				}
			}
		})
	}
}

func TestAutoAcknowledgementSurvivesMetricEvaluation(t *testing.T) {
	m := newTestManager(t)
	m.mu.Lock()
	m.config.TimeThresholds = map[string]int{}
	m.config.SuppressionWindow = 0
	m.config.MinimumDelta = 0
	m.config.AutoAcknowledgeAfterHours = 2
	m.mu.Unlock()
	threshold := &HysteresisThreshold{Trigger: 80, Clear: 70}
	evaluate := func(value float64) {
		m.checkMetric("auto-ack-resource", "Resource", "node", "instance", "guest", "usage", value, threshold, nil)
	}
	evaluate(90)
	key := buildCanonicalStateID("auto-ack-resource", "metric-threshold:usage")
	m.mu.Lock()
	m.activeAlerts[key].StartTime = time.Now().Add(-3 * time.Hour)
	m.mu.Unlock()
	m.Cleanup(time.Hour)
	assertAcknowledged := func(stage string) {
		t.Helper()
		alerts := m.GetActiveAlerts()
		if len(alerts) != 1 || !alerts[0].Acknowledged || alerts[0].AckUser != "system-auto" || alerts[0].AckTime == nil {
			t.Fatalf("%s: automatic acknowledgement lost: %+v", stage, alerts)
		}
	}
	assertAcknowledged("cleanup")
	evaluate(85)
	assertAcknowledged("next evaluation")
	evaluate(60)
	evaluate(90)
	assertAcknowledged("short recovery and refire")
}

// Age only the acknowledgement timestamps, keeping detector observations fresh.
// The hourly sweep must not discard a long-running operator decision before a
// real recovery can start the existing inactive-retention window.
func TestTrackingCleanupPreservesAcknowledgedProviderRecurrence(t *testing.T) {
	for _, durable := range []bool{false, true} {
		for _, automatic := range []bool{false, true} {
			name := "JSON"
			if durable {
				name = "durable"
			}
			if automatic {
				name += "/automatic"
			} else {
				name += "/manual"
			}
			t.Run(name, func(t *testing.T) {
				dir := t.TempDir()
				newManager := func() *Manager {
					return newCleanupRetentionManager(t, dir, durable)
				}
				m := newManager()
				resource := unifiedresources.Resource{
					ID: "storage:ack-tank", Type: unifiedresources.ResourceTypeStorage,
					Name: "ack-tank", ParentName: "truenas-main",
					Sources: []unifiedresources.DataSource{unifiedresources.SourceTrueNAS},
					Storage: &unifiedresources.StorageMeta{Platform: "truenas", Topology: "pool", Protection: "zfs", IsZFS: true},
					Incidents: []unifiedresources.ResourceIncident{{
						Provider: "truenas", NativeID: "ack-pool-alert", Code: "truenas_volume_status",
						Severity: storagehealth.RiskWarning, Summary: "Pool ack-tank is DEGRADED",
						StartedAt: time.Now().Add(-26 * time.Hour),
					}},
				}
				healthy := resource
				healthy.Incidents = nil
				evaluate := func(m *Manager, observed unifiedresources.Resource) {
					m.SyncUnifiedResourceIncidents([]unifiedresources.Resource{observed})
				}
				var deliveries atomic.Int32
				m.SetAlertCallback(func(*Alert) { deliveries.Add(1) })
				evaluate(m, resource)
				initial := m.GetActiveAlerts()
				if len(initial) != 1 || deliveries.Load() != 1 {
					t.Fatalf("initial provider condition: %d alerts / %d dispatches, want 1 / 1", len(initial), deliveries.Load())
				}
				id := initial[0].ID
				user := "operator"
				if automatic {
					user = "system-auto"
					m.mu.Lock()
					m.config.AutoAcknowledgeAfterHours = 1
					m.mu.Unlock()
					m.Cleanup(time.Hour)
				} else if err := m.AcknowledgeAlert(id, user); err != nil {
					t.Fatal(err)
				}
				ackAt := time.Now().Add(-25 * time.Hour)
				m.mu.Lock()
				active, ok := m.getActiveAlertNoLock(id)
				if !ok || !active.Acknowledged {
					m.mu.Unlock()
					t.Fatal("detector acknowledgement was not established")
				}
				active.AckTime = &ackAt
				record := m.ackStateByCanonical[id]
				record.time = ackAt
				m.ackStateByCanonical[id] = record
				m.mirrorAcknowledgeNoLock(active, user, ackAt)
				m.setActiveAlertNoLock(id, active)
				m.mu.Unlock()

				checkCycle := func(m *Manager, count *atomic.Int32, stage string) {
					t.Helper()
					before := count.Load()
					m.cleanupStaleMaps()
					m.mu.RLock()
					record, retained := m.ackStateByCanonical[id]
					m.mu.RUnlock()
					if !retained || record.user != user || !record.time.Equal(ackAt) {
						t.Errorf("%s: hourly sweep lost the active acknowledgement: %+v, retained=%v", stage, record, retained)
					}
					evaluate(m, healthy) // Observed pool with no incident: affirmative recovery.
					if got := len(m.GetActiveAlerts()); got != 0 {
						t.Fatalf("%s: real recovery left %d active alerts", stage, got)
					}
					m.Cleanup(time.Hour) // Old acknowledgement, newly inactive: retain it.
					evaluate(m, resource)
					alerts := m.GetActiveAlerts()
					if len(alerts) != 1 || alerts[0].ID != id || !alerts[0].Acknowledged || alerts[0].AckUser != user || alerts[0].AckTime == nil || !alerts[0].AckTime.Equal(ackAt) {
						t.Errorf("%s: short recurrence lost acknowledgement identity: %+v", stage, alerts)
					}
					if got := count.Load() - before; got != 0 {
						t.Errorf("%s: acknowledged recovery/recurrence dispatched %d duplicate notifications", stage, got)
					}
					t.Logf("%s: recovery/recurrence dispatch delta %d", stage, count.Load()-before)
				}
				checkCycle(m, &deliveries, "before restart")
				m.Stop() // The actual final checkpoint, not a hand-built JSON fixture.
				restarted := newManager()
				var afterRestart atomic.Int32
				restarted.SetAlertCallback(func(*Alert) { afterRestart.Add(1) })
				checkCycle(restarted, &afterRestart, "after restart")
				if err := restarted.UnacknowledgeAlert(id); err != nil {
					t.Fatal(err)
				}
				before := afterRestart.Load()
				evaluate(restarted, healthy)
				evaluate(restarted, resource)
				if got := afterRestart.Load() - before; got != 1 {
					t.Errorf("explicit unacknowledge/recovery/recurrence dispatched %d notifications, want 1", got)
				}
			})
		}
	}
}

func TestTrackingCleanupCanonicalAckRetentionBounds(t *testing.T) {
	for _, tc := range []struct {
		name       string
		active     bool
		legacy     bool
		inactive   time.Duration
		wantHourly bool
		wantNormal bool
	}{
		{name: "active canonical", active: true, wantHourly: true, wantNormal: true},
		{name: "active legacy identity", active: true, legacy: true, wantHourly: true, wantNormal: true},
		{name: "active after earlier recurrence", active: true, inactive: 25 * time.Hour, wantHourly: true, wantNormal: true},
		{name: "recently inactive old acknowledgement", inactive: 30 * time.Minute, wantHourly: true, wantNormal: true},
		{name: "existing one-hour expiry", inactive: 2 * time.Hour, wantHourly: true},
		{name: "stale inactive", inactive: 25 * time.Hour},
		{name: "legacy missing inactive timestamp"},
	} {
		t.Run(tc.name, func(t *testing.T) {
			m := newTestManager(t)
			now := time.Now()
			old := now.Add(-25 * time.Hour)
			id, alert := testNewCanonicalAlert("ack-bounds", "metric-threshold:cpu", "metric-threshold", "cpu")
			alert.StartTime, alert.LastSeen = old, now
			alert.Acknowledged, alert.AckUser, alert.AckTime = true, "operator", &old
			record := ackRecord{acknowledged: true, user: "operator", time: old}
			if tc.inactive > 0 {
				record.inactiveAt = now.Add(-tc.inactive)
			}
			m.mu.Lock()
			m.ackStateByCanonical[id] = record
			if tc.active {
				storageKey := id
				if tc.legacy {
					storageKey = "legacy-cpu-ack-bounds"
				}
				m.activeAlerts[storageKey] = alert
			}
			m.mu.Unlock()
			assertRetained := func(stage string, want bool) {
				t.Helper()
				m.mu.RLock()
				got, retained := m.ackStateByCanonical[id]
				m.mu.RUnlock()
				if retained != want || (retained && got != record) {
					t.Errorf("%s: acknowledgement retained=%v (%+v), want %v with unchanged record", stage, retained, got, want)
				}
			}
			m.cleanupStaleMaps()
			assertRetained("hourly sweep", tc.wantHourly)
			m.Cleanup(time.Hour)
			assertRetained("normal one-hour inactive expiry", tc.wantNormal)
		})
	}
}