Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

5.6 KB
Raw Blame
package reducer

import (
	"testing"
	"time"
)

func TestAcknowledgeRequiresFiringIncident(t *testing.T) {
	state := NewState()
	if state.Acknowledge("node-1", "connectivity", "richard", t0) {
		t.Fatal("ack of a missing incident should fail")
	}

	rule := DiscreteRule{Confirmations: 2}
	state.ApplyDiscrete(discreteSignalAt(true, SeverityWarning, 0), rule)
	if state.Acknowledge("node-1", "connectivity", "richard", t0) {
		t.Fatal("ack of a pending incident should fail")
	}

	state.ApplyDiscrete(discreteSignalAt(true, SeverityWarning, time.Minute), rule)
	if !state.Acknowledge("node-1", "connectivity", "richard", t0.Add(2*time.Minute)) {
		t.Fatal("ack of a firing incident should succeed")
	}
	incident, _ := state.Incident("node-1", "connectivity")
	if !incident.Acknowledged || incident.AckUser != "richard" {
		t.Fatalf("incident = %+v, want acknowledged by richard", incident)
	}
}

func TestUnacknowledgeClearsFlagAndRecord(t *testing.T) {
	state := NewState()
	rule := DiscreteRule{Confirmations: 1}
	state.ApplyDiscrete(discreteSignalAt(true, SeverityWarning, 0), rule)
	state.Acknowledge("node-1", "connectivity", "richard", t0.Add(time.Minute))

	if !state.Unacknowledge("node-1", "connectivity") {
		t.Fatal("unack of a firing incident should succeed")
	}
	incident, _ := state.Incident("node-1", "connectivity")
	if incident.Acknowledged {
		t.Fatal("incident should no longer be acknowledged")
	}

	// After unack, a resolve/re-fire cycle must NOT restore the ack.
	state.ApplyDiscrete(discreteSignalAt(false, "", 2*time.Minute), rule)
	state.ApplyDiscrete(discreteSignalAt(true, SeverityWarning, 3*time.Minute), rule)
	incident, _ = state.Incident("node-1", "connectivity")
	if incident.Acknowledged {
		t.Fatal("unacknowledged record must not be restored on re-fire")
	}
}

func TestAckSurvivesResolveAndRefireWithinRetention(t *testing.T) {
	state := NewState()
	rule := DiscreteRule{Confirmations: 1}
	state.ApplyDiscrete(discreteSignalAt(true, SeverityWarning, 0), rule)
	state.Acknowledge("node-1", "connectivity", "richard", t0.Add(time.Minute))

	state.ApplyDiscrete(discreteSignalAt(false, "", 2*time.Minute), rule)
	state.ApplyDiscrete(discreteSignalAt(true, SeverityWarning, 3*time.Minute), rule)

	incident, _ := state.Incident("node-1", "connectivity")
	if !incident.Acknowledged || incident.AckUser != "richard" {
		t.Fatalf("incident = %+v, want ack restored on re-fire", incident)
	}
	if !incident.AckAt.Equal(t0.Add(time.Minute)) {
		t.Fatalf("AckAt = %v, want original ack time preserved", incident.AckAt)
	}
}

func TestAckExpiresAfterRetention(t *testing.T) {
	state := NewState()
	rule := DiscreteRule{Confirmations: 1}
	state.ApplyDiscrete(discreteSignalAt(true, SeverityWarning, 0), rule)
	state.Acknowledge("node-1", "connectivity", "richard", t0.Add(time.Minute))
	state.ApplyDiscrete(discreteSignalAt(false, "", 2*time.Minute), rule)

	// Re-fire more than AckRetention after the resolve: ack must not return.
	refireAt := 2*time.Minute + AckRetention + time.Minute
	state.ApplyDiscrete(discreteSignalAt(true, SeverityWarning, refireAt), rule)
	incident, _ := state.Incident("node-1", "connectivity")
	if incident.Acknowledged {
		t.Fatal("expired ack record must not be restored")
	}
}

func TestAckAppliesToMetricFamilyToo(t *testing.T) {
	state := NewState()
	rule := MetricRule{Trigger: 80, Clear: 75}
	state.ApplyMetric(MetricSignal{ResourceID: "vm-1", Metric: "cpu", Value: 85, ObservedAt: t0}, rule)
	if !state.Acknowledge("vm-1", "cpu", "richard", t0.Add(time.Minute)) {
		t.Fatal("ack of a firing metric incident should succeed")
	}

	state.ApplyMetric(MetricSignal{ResourceID: "vm-1", Metric: "cpu", Value: 70, ObservedAt: t0.Add(2 * time.Minute)}, rule)
	state.ApplyMetric(MetricSignal{ResourceID: "vm-1", Metric: "cpu", Value: 85, ObservedAt: t0.Add(3 * time.Minute)}, rule)
	incident, _ := state.Incident("vm-1", "cpu")
	if !incident.Acknowledged {
		t.Fatal("metric re-fire within retention should restore the ack")
	}
}

// A short reactivation retains acknowledgement by design; expiry is measured
// from resolution, not acknowledgement or the previous occurrence's start.
func TestMetricAcknowledgementRetentionBoundary(t *testing.T) {
	for _, tc := range []struct {
		name         string
		afterResolve time.Duration
		wantAck      bool
	}{
		{"just before expiry", AckRetention - time.Nanosecond, true},
		{"at expiry", AckRetention, false},
		{"next day", 24 * time.Hour, false},
	} {
		t.Run(tc.name, func(t *testing.T) {
			state := NewState()
			rule := MetricRule{Trigger: 80, Clear: 75}
			observe := func(value float64, at time.Time) {
				state.ApplyMetric(MetricSignal{ResourceID: "vm-1", Metric: "cpu", Value: value, ObservedAt: at}, rule)
			}
			observe(95, t0)
			ackedAt := t0.Add(time.Minute)
			if !state.Acknowledge("vm-1", "cpu", "previous-operator", ackedAt) {
				t.Fatal("expected acknowledgement of firing metric")
			}
			resolvedAt := t0.Add(2 * time.Hour)
			observe(10, resolvedAt)
			refiredAt := resolvedAt.Add(tc.afterResolve)
			observe(95, refiredAt)
			incident, ok := state.Incident("vm-1", "cpu")
			if !ok || incident.State != StateFiring {
				t.Fatal("expected a firing metric recurrence")
			}
			if !incident.StartedAt.Equal(refiredAt) {
				t.Fatalf("start = %v, want new occurrence at %v", incident.StartedAt, refiredAt)
			}
			if incident.Acknowledged != tc.wantAck {
				t.Fatalf("acknowledged = %v, want %v", incident.Acknowledged, tc.wantAck)
			}
			if tc.wantAck {
				if incident.AckUser != "previous-operator" || !incident.AckAt.Equal(ackedAt) {
					t.Fatal("retained acknowledgement lost its original attribution")
				}
			} else if incident.AckUser != "" || !incident.AckAt.IsZero() {
				t.Fatal("expired acknowledgement left stale attribution")
			}
		})
	}
}