Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

5.3 KB
Raw Blame
package ai

import (
	"testing"
	"time"
)

// flapFinding builds a fresh warning finding for the flap tests.
func flapFinding() *Finding {
	return &Finding{
		ID:           "docker:h/web:exited",
		Key:          "container-exited",
		Severity:     FindingSeverityWarning,
		Category:     FindingCategoryReliability,
		ResourceID:   "docker:h/web",
		ResourceName: "web",
		Title:        `Container "web" exited`,
	}
}

func lifecycleTypes(f *Finding) []string {
	types := make([]string, 0, len(f.Lifecycle))
	for _, event := range f.Lifecycle {
		types = append(types, event.Type)
	}
	return types
}

func TestFindingsStore_CollapsesFlappingIntoOneLifecycleRow(t *testing.T) {
	store := NewFindingsStore()
	store.SetStormThrottler(newFindingStormThrottler())

	store.Add(flapFinding())
	// Resolve and re-detect repeatedly. Each resolve and each regression is
	// one open/resolved transition.
	for i := 0; i < 6; i++ {
		if !store.ResolveWithReason("docker:h/web:exited", "container running again") {
			t.Fatalf("resolve %d failed", i)
		}
		if store.Add(flapFinding()) {
			t.Fatalf("re-detection %d was treated as a new finding", i)
		}
	}

	f := store.Get("docker:h/web:exited")
	if f == nil {
		t.Fatal("finding disappeared")
	}
	if f.Flapping == nil {
		t.Fatalf("twelve transitions in a minute were not labelled flapping: %v", lifecycleTypes(f))
	}
	if f.Flapping.TransitionCount != 12 {
		t.Fatalf("TransitionCount = %d, want 12", f.Flapping.TransitionCount)
	}
	if f.RegressionCount != 6 {
		t.Fatalf("RegressionCount = %d, want 6 (regressions still counted while collapsed)", f.RegressionCount)
	}

	// Transitions 1-3 were recorded individually; from the fourth onward the
	// store maintains a single flapping row instead of appending nine more.
	types := lifecycleTypes(f)
	flappingRows := 0
	individualTransitions := 0
	for _, typ := range types {
		switch typ {
		case FindingLifecycleFlapping:
			flappingRows++
		case "regressed", "auto_resolved", "resolved":
			individualTransitions++
		}
	}
	if flappingRows != 1 {
		t.Fatalf("flapping rows = %d, want exactly one collapsed row: %v", flappingRows, types)
	}
	if individualTransitions != findingFlapThreshold-1 {
		t.Fatalf("individual transition rows = %d, want %d pre-label rows: %v", individualTransitions, findingFlapThreshold-1, types)
	}
	last := f.Lifecycle[len(f.Lifecycle)-1]
	if last.Type != FindingLifecycleFlapping {
		t.Fatalf("last lifecycle row = %q, want the collapsed flapping row", last.Type)
	}
	if last.Metadata["transition_count"] != "12" {
		t.Fatalf("collapsed row transition_count = %q, want 12", last.Metadata["transition_count"])
	}
	if last.Metadata["last_transition"] != "regressed" {
		t.Fatalf("collapsed row last_transition = %q, want regressed", last.Metadata["last_transition"])
	}
}

func TestFindingsStore_WithoutThrottlerKeepsIndividualRows(t *testing.T) {
	store := NewFindingsStore()
	store.Add(flapFinding())
	for i := 0; i < 3; i++ {
		store.ResolveWithReason("docker:h/web:exited", "running again")
		store.Add(flapFinding())
	}
	f := store.Get("docker:h/web:exited")
	if f.Flapping != nil {
		t.Fatal("no throttler installed but the finding was labelled flapping")
	}
	for _, typ := range lifecycleTypes(f) {
		if typ == FindingLifecycleFlapping {
			t.Fatalf("collapsed row appeared without a throttler: %v", lifecycleTypes(f))
		}
	}
}

func TestFindingsStore_FlappingLabelClearsWhenChurnStops(t *testing.T) {
	store := NewFindingsStore()
	throttler := newFindingStormThrottler()
	store.SetStormThrottler(throttler)
	store.Add(flapFinding())
	for i := 0; i < 3; i++ {
		store.ResolveWithReason("docker:h/web:exited", "running again")
		store.Add(flapFinding())
	}
	if store.Get("docker:h/web:exited").Flapping == nil {
		t.Fatal("expected flapping after six transitions")
	}

	// Age the tracked window out, then one more transition: no longer flapping.
	throttler.mu.Lock()
	tracker := throttler.flaps["docker:h/web:exited"]
	for i := range tracker.transitions {
		tracker.transitions[i] = tracker.transitions[i].Add(-findingFlapWindow - time.Hour)
	}
	throttler.mu.Unlock()
	store.ResolveWithReason("docker:h/web:exited", "running again")

	f := store.Get("docker:h/web:exited")
	if f.Flapping != nil {
		t.Fatalf("label persisted after the window emptied: %+v", f.Flapping)
	}
	if last := f.Lifecycle[len(f.Lifecycle)-1]; last.Type != "auto_resolved" {
		t.Fatalf("post-flap transition row = %q, want an ordinary auto_resolved row", last.Type)
	}
}

func TestFindingJSONRoundTripsFlappingAndMirror(t *testing.T) {
	now := time.Date(2026, 8, 10, 12, 0, 0, 0, time.UTC)
	f := flapFinding()
	f.DetectedAt = now
	f.LastSeenAt = now
	f.Flapping = &FindingFlapping{TransitionCount: 7, WindowHours: 24, FirstTransitionAt: now.Add(-20 * time.Hour), LastTransitionAt: now}
	f.MirrorsAlertID = "docker-container-state-docker:h/web"
	f.MirrorsAlertType = "docker-container-state"

	data, err := f.MarshalJSON()
	if err != nil {
		t.Fatal(err)
	}
	var decoded Finding
	if err := decoded.UnmarshalJSON(data); err != nil {
		t.Fatal(err)
	}
	if decoded.Flapping == nil || decoded.Flapping.TransitionCount != 7 || decoded.Flapping.WindowHours != 24 {
		t.Fatalf("flapping did not round-trip: %+v", decoded.Flapping)
	}
	if decoded.MirrorsAlertID != f.MirrorsAlertID || decoded.MirrorsAlertType != f.MirrorsAlertType {
		t.Fatalf("mirror fields did not round-trip: %q %q", decoded.MirrorsAlertID, decoded.MirrorsAlertType)
	}
}