Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

13.9 KB
Raw Blame
package reducer

import (
	"testing"
	"time"
)

func TestInterruptDiscreteRun(t *testing.T) {
	t.Run("pending confirmation and grace restart for the exact key", func(t *testing.T) {
		state := NewState()
		rule := DiscreteRule{Confirmations: 3, Intent: &DiscreteIntent{Explicit: true, GraceSeconds: 120}}
		signal := discreteSignalAt(true, SeverityWarning, 0)
		state.ApplyDiscrete(signal, rule)
		signal.ObservedAt = signal.ObservedAt.Add(time.Minute)
		state.ApplyDiscrete(signal, rule)
		other := signal
		other.Key = "other-condition"
		state.ApplyDiscrete(other, rule)
		other.ResourceID = "node-10"
		state.ApplyDiscrete(other, rule)
		if !state.InterruptDiscreteRun("node-1", "connectivity") {
			t.Fatal("pending run was not interrupted")
		}
		if _, ok := state.Incident("node-1", "connectivity"); ok {
			t.Fatal("pending run survived interruption")
		}
		for _, id := range []string{"node-1", "node-10"} {
			if _, ok := state.Incident(id, "other-condition"); !ok {
				t.Fatalf("interruption removed unrelated key for %s", id)
			}
		}
		if state.InterruptDiscreteRun("node-1", "connectivity") {
			t.Fatal("absent run reported an interruption")
		}
		signal.ObservedAt = signal.ObservedAt.Add(time.Hour)
		events := state.ApplyDiscrete(signal, rule)
		incident, ok := state.Incident(signal.ResourceID, signal.Key)
		if len(events) != 1 || events[0].Type != EventPending || !ok || incident.Confirmations != 1 || !incident.PendingSince.Equal(signal.ObservedAt) {
			t.Fatalf("activation did not restart: events=%+v incident=%+v", events, incident)
		}
	})

	t.Run("firing occurrence and acknowledgement survive a recovery gap", func(t *testing.T) {
		state := NewState()
		rule := DiscreteRule{Confirmations: 1, RecoveryConfirmations: 3}
		signal := discreteSignalAt(true, SeverityCritical, 0)
		state.ApplyDiscrete(signal, rule)
		if !state.Acknowledge(signal.ResourceID, signal.Key, "test-operator", signal.ObservedAt) {
			t.Fatal("acknowledgement failed")
		}
		signal.Matched = false
		state.ApplyDiscrete(signal, rule)
		state.ApplyDiscrete(signal, rule)
		before, _ := state.Incident(signal.ResourceID, signal.Key)
		if before.RecoveryCount != 2 || !state.InterruptDiscreteRun(signal.ResourceID, signal.Key) {
			t.Fatal("recovery run was not interrupted")
		}
		want := before
		want.RecoveryCount = 0
		got, ok := state.Incident(signal.ResourceID, signal.Key)
		if !ok || got != want {
			t.Fatalf("interruption changed the firing occurrence: got=%+v want=%+v", got, want)
		}
		if len(state.resolved) != 0 || len(state.acks) != 1 || state.InterruptDiscreteRun(signal.ResourceID, signal.Key) {
			t.Fatal("interruption manufactured resolution, lost acknowledgement or was not idempotent")
		}
		for range 2 {
			if events := state.ApplyDiscrete(signal, rule); len(events) != 0 {
				t.Fatalf("non-consecutive recovery emitted %+v", events)
			}
		}
		if events := state.ApplyDiscrete(signal, rule); len(events) != 1 || events[0].Type != EventResolved {
			t.Fatalf("fresh recovery did not resolve: %+v", events)
		}
	})
}

func discreteSignalAt(matched bool, severity Severity, offset time.Duration) DiscreteSignal {
	return DiscreteSignal{
		ResourceID: "node-1",
		Key:        "connectivity",
		Matched:    matched,
		Severity:   severity,
		ObservedAt: t0.Add(offset),
	}
}

func TestDiscreteFiresAfterConfirmations(t *testing.T) {
	state := NewState()
	rule := DiscreteRule{Confirmations: 3}

	events := state.ApplyDiscrete(discreteSignalAt(true, SeverityCritical, 0), rule)
	if len(events) != 1 || events[0].Type != EventPending {
		t.Fatalf("events = %+v, want pending on first match", events)
	}
	if events := state.ApplyDiscrete(discreteSignalAt(true, SeverityCritical, time.Minute), rule); len(events) != 0 {
		t.Fatalf("second match emitted %+v", events)
	}

	events = state.ApplyDiscrete(discreteSignalAt(true, SeverityCritical, 2*time.Minute), rule)
	if len(events) != 1 || events[0].Type != EventFired || events[0].Severity != SeverityCritical {
		t.Fatalf("events = %+v, want critical fired on third match", events)
	}

	incident, ok := state.Incident("node-1", "connectivity")
	if !ok || incident.State != StateFiring {
		t.Fatalf("incident = %+v, want firing", incident)
	}
	if !incident.StartedAt.Equal(t0) {
		t.Fatalf("StartedAt = %v, want first matched observation %v", incident.StartedAt, t0)
	}
	if incident.Confirmations != 3 {
		t.Fatalf("Confirmations = %d, want clamped at 3", incident.Confirmations)
	}
}

func TestDiscreteNonMatchResetsPendingCount(t *testing.T) {
	state := NewState()
	rule := DiscreteRule{Confirmations: 3}

	state.ApplyDiscrete(discreteSignalAt(true, SeverityWarning, 0), rule)
	state.ApplyDiscrete(discreteSignalAt(true, SeverityWarning, time.Minute), rule)

	events := state.ApplyDiscrete(discreteSignalAt(false, "", 2*time.Minute), rule)
	if len(events) != 1 || events[0].Type != EventPendingCleared {
		t.Fatalf("events = %+v, want pending cleared", events)
	}

	// The count starts over: two more matches must not fire.
	state.ApplyDiscrete(discreteSignalAt(true, SeverityWarning, 3*time.Minute), rule)
	if events := state.ApplyDiscrete(discreteSignalAt(true, SeverityWarning, 4*time.Minute), rule); len(events) != 0 {
		t.Fatalf("post-reset second match emitted %+v", events)
	}
	events = state.ApplyDiscrete(discreteSignalAt(true, SeverityWarning, 5*time.Minute), rule)
	if len(events) != 1 || events[0].Type != EventFired {
		t.Fatalf("events = %+v, want fired on third consecutive match", events)
	}
	incident, _ := state.Incident("node-1", "connectivity")
	if !incident.StartedAt.Equal(t0.Add(3 * time.Minute)) {
		t.Fatalf("StartedAt = %v, want restart of the pending run", incident.StartedAt)
	}
}

func TestDiscreteSingleConfirmationFiresImmediately(t *testing.T) {
	state := NewState()
	rule := DiscreteRule{Confirmations: 1}

	events := state.ApplyDiscrete(discreteSignalAt(true, SeverityWarning, 0), rule)
	if len(events) != 1 || events[0].Type != EventFired {
		t.Fatalf("events = %+v, want immediate fire", events)
	}
	events = state.ApplyDiscrete(discreteSignalAt(false, "", time.Minute), rule)
	if len(events) != 1 || events[0].Type != EventResolved {
		t.Fatalf("events = %+v, want resolved on single recovery", events)
	}
}

func TestDiscreteSeverityFollowsRuleWhileFiring(t *testing.T) {
	state := NewState()
	rule := DiscreteRule{Confirmations: 1}

	state.ApplyDiscrete(discreteSignalAt(true, SeverityWarning, 0), rule)
	events := state.ApplyDiscrete(discreteSignalAt(true, SeverityCritical, time.Minute), rule)
	if len(events) != 1 || events[0].Type != EventSeverityChanged || events[0].Severity != SeverityCritical {
		t.Fatalf("events = %+v, want severity change to critical", events)
	}
	if events := state.ApplyDiscrete(discreteSignalAt(true, SeverityCritical, 2*time.Minute), rule); len(events) != 0 {
		t.Fatalf("unchanged severity emitted %+v", events)
	}
}

func TestDiscreteDisabledClearsPendingAndFiring(t *testing.T) {
	pending := NewState()
	rule := DiscreteRule{Confirmations: 3}
	pending.ApplyDiscrete(discreteSignalAt(true, SeverityWarning, 0), rule)
	events := pending.ApplyDiscrete(discreteSignalAt(true, SeverityWarning, time.Minute), DiscreteRule{Confirmations: 3, Disabled: true})
	if len(events) != 1 || events[0].Type != EventPendingCleared {
		t.Fatalf("events = %+v, want pending cleared on disable", events)
	}

	firing := NewState()
	firing.ApplyDiscrete(discreteSignalAt(true, SeverityWarning, 0), DiscreteRule{Confirmations: 1})
	events = firing.ApplyDiscrete(discreteSignalAt(true, SeverityWarning, time.Minute), DiscreteRule{Confirmations: 1, Disabled: true})
	if len(events) != 1 || events[0].Type != EventResolved {
		t.Fatalf("events = %+v, want resolved on disable", events)
	}
}

func TestDiscreteRefireRequiresFullConfirmations(t *testing.T) {
	state := NewState()
	rule := DiscreteRule{Confirmations: 2}

	state.ApplyDiscrete(discreteSignalAt(true, SeverityWarning, 0), rule)
	state.ApplyDiscrete(discreteSignalAt(true, SeverityWarning, time.Minute), rule)
	state.ApplyDiscrete(discreteSignalAt(false, "", 2*time.Minute), rule)

	// After recovery the confirmation run restarts from zero.
	events := state.ApplyDiscrete(discreteSignalAt(true, SeverityWarning, 3*time.Minute), rule)
	if len(events) != 1 || events[0].Type != EventPending {
		t.Fatalf("events = %+v, want pending after recovery", events)
	}
	events = state.ApplyDiscrete(discreteSignalAt(true, SeverityWarning, 4*time.Minute), rule)
	if len(events) != 1 || events[0].Type != EventRefired {
		t.Fatalf("events = %+v, want refired after full confirmations within retention", events)
	}
	// Within RefireRetention the original occurrence's start is restored.
	incident, _ := state.Incident("node-1", "connectivity")
	if !incident.StartedAt.Equal(t0) {
		t.Fatalf("StartedAt = %v, want restored original start %v", incident.StartedAt, t0)
	}
}

func TestDiscreteRecoveryGateHoldsUntilConsecutiveConfirmations(t *testing.T) {
	state := NewState()
	rule := DiscreteRule{Confirmations: 1, RecoveryConfirmations: 3}

	state.ApplyDiscrete(discreteSignalAt(true, SeverityCritical, 0), rule)

	// Two healthy observations: still firing.
	if events := state.ApplyDiscrete(discreteSignalAt(false, "", time.Minute), rule); len(events) != 0 {
		t.Fatalf("first healthy emitted %+v", events)
	}
	if events := state.ApplyDiscrete(discreteSignalAt(false, "", 2*time.Minute), rule); len(events) != 0 {
		t.Fatalf("second healthy emitted %+v", events)
	}
	incident, ok := state.Incident("node-1", "connectivity")
	if !ok || incident.State != StateFiring || incident.RecoveryCount != 2 {
		t.Fatalf("incident = %+v, want firing with recovery count 2", incident)
	}

	events := state.ApplyDiscrete(discreteSignalAt(false, "", 3*time.Minute), rule)
	if len(events) != 1 || events[0].Type != EventResolved {
		t.Fatalf("events = %+v, want resolved on third consecutive healthy", events)
	}
	if _, ok := state.Incident("node-1", "connectivity"); ok {
		t.Fatal("incident should be gone after gated resolve")
	}
}

func TestDiscreteMatchedObservationResetsRecoveryRun(t *testing.T) {
	state := NewState()
	rule := DiscreteRule{Confirmations: 1, RecoveryConfirmations: 3}

	state.ApplyDiscrete(discreteSignalAt(true, SeverityWarning, 0), rule)
	state.ApplyDiscrete(discreteSignalAt(false, "", time.Minute), rule)
	state.ApplyDiscrete(discreteSignalAt(false, "", 2*time.Minute), rule)

	// Back offline: recovery run resets.
	state.ApplyDiscrete(discreteSignalAt(true, SeverityWarning, 3*time.Minute), rule)
	incident, _ := state.Incident("node-1", "connectivity")
	if incident.RecoveryCount != 0 {
		t.Fatalf("RecoveryCount = %d, want reset to 0", incident.RecoveryCount)
	}

	// Two healthy observations are again not enough.
	state.ApplyDiscrete(discreteSignalAt(false, "", 4*time.Minute), rule)
	if events := state.ApplyDiscrete(discreteSignalAt(false, "", 5*time.Minute), rule); len(events) != 0 {
		t.Fatalf("post-reset second healthy emitted %+v", events)
	}
	events := state.ApplyDiscrete(discreteSignalAt(false, "", 6*time.Minute), rule)
	if len(events) != 1 || events[0].Type != EventResolved {
		t.Fatalf("events = %+v, want resolved after full recovery run", events)
	}
}

func TestDiscretePendingClearsImmediatelyDespiteRecoveryGate(t *testing.T) {
	state := NewState()
	rule := DiscreteRule{Confirmations: 3, RecoveryConfirmations: 3}

	state.ApplyDiscrete(discreteSignalAt(true, SeverityWarning, 0), rule)
	events := state.ApplyDiscrete(discreteSignalAt(false, "", time.Minute), rule)
	if len(events) != 1 || events[0].Type != EventPendingCleared {
		t.Fatalf("events = %+v, want pending cleared on single healthy observation", events)
	}
}

func TestDiscreteDisableBypassesRecoveryGate(t *testing.T) {
	state := NewState()
	rule := DiscreteRule{Confirmations: 1, RecoveryConfirmations: 3}

	state.ApplyDiscrete(discreteSignalAt(true, SeverityWarning, 0), rule)
	events := state.ApplyDiscrete(discreteSignalAt(true, SeverityWarning, time.Minute), DiscreteRule{Confirmations: 1, RecoveryConfirmations: 3, Disabled: true})
	if len(events) != 1 || events[0].Type != EventResolved {
		t.Fatalf("events = %+v, want immediate resolve on disable", events)
	}
}

func TestDiscreteRefireOutsideRetentionStartsFresh(t *testing.T) {
	state := NewState()
	rule := DiscreteRule{Confirmations: 1}

	state.ApplyDiscrete(discreteSignalAt(true, SeverityWarning, 0), rule)
	state.ApplyDiscrete(discreteSignalAt(false, "", time.Minute), rule)

	// Beyond RefireRetention the resolved occurrence is no longer
	// consumable: a re-fire is a new occurrence with a fresh start.
	refireAt := time.Minute + RefireRetention + time.Second
	events := state.ApplyDiscrete(discreteSignalAt(true, SeverityWarning, refireAt), rule)
	if len(events) != 1 || events[0].Type != EventFired {
		t.Fatalf("events = %+v, want plain fired outside retention", events)
	}
	incident, _ := state.Incident("node-1", "connectivity")
	if !incident.StartedAt.Equal(t0.Add(refireAt)) {
		t.Fatalf("StartedAt = %v, want fresh start %v", incident.StartedAt, t0.Add(refireAt))
	}
}

func TestDiscreteRefireRecordConsumedOnce(t *testing.T) {
	state := NewState()
	rule := DiscreteRule{Confirmations: 1}

	state.ApplyDiscrete(discreteSignalAt(true, SeverityWarning, 0), rule)
	state.ApplyDiscrete(discreteSignalAt(false, "", time.Minute), rule)

	// First re-fire restores; its own later resolve records a NEW
	// occurrence whose start is the restored (original) start.
	events := state.ApplyDiscrete(discreteSignalAt(true, SeverityWarning, 2*time.Minute), rule)
	if len(events) != 1 || events[0].Type != EventRefired {
		t.Fatalf("events = %+v, want refired", events)
	}
	state.ApplyDiscrete(discreteSignalAt(false, "", 3*time.Minute), rule)

	// Second re-fire within retention restores again — from the second
	// occurrence's record, which carries the same original start.
	events = state.ApplyDiscrete(discreteSignalAt(true, SeverityWarning, 4*time.Minute), rule)
	if len(events) != 1 || events[0].Type != EventRefired {
		t.Fatalf("events = %+v, want refired from the newer record", events)
	}
	incident, _ := state.Incident("node-1", "connectivity")
	if !incident.StartedAt.Equal(t0) {
		t.Fatalf("StartedAt = %v, want original start carried through %v", incident.StartedAt, t0)
	}
}