Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

13.8 KB
Raw Blame
package alerts

import (
	"encoding/json"
	"reflect"
	"sync"
	"testing"
	"time"

	"github.com/rcourtman/pulse-go-rewrite/internal/models"
)

func TestUpdateConfigNormalizesBackupIndicatorHours(t *testing.T) {
	t.Run("non-positive values default", func(t *testing.T) {
		m := newTestManager(t)

		cfg := AlertConfig{
			Enabled: true,
			BackupDefaults: BackupAlertConfig{
				WarningDays:  2,
				CriticalDays: 4,
				FreshHours:   -1,
				StaleHours:   0,
			},
		}

		m.UpdateConfig(cfg)

		m.mu.RLock()
		defer m.mu.RUnlock()

		if m.config.BackupDefaults.FreshHours != 24 {
			t.Fatalf("expected FreshHours to default to 24, got %d", m.config.BackupDefaults.FreshHours)
		}
		if m.config.BackupDefaults.StaleHours != 72 {
			t.Fatalf("expected StaleHours to default to 72 when invalid, got %d", m.config.BackupDefaults.StaleHours)
		}
	})

	t.Run("stale is clamped to fresh when lower", func(t *testing.T) {
		m := newTestManager(t)

		cfg := AlertConfig{
			Enabled: true,
			BackupDefaults: BackupAlertConfig{
				WarningDays:  2,
				CriticalDays: 4,
				FreshHours:   48,
				StaleHours:   24,
			},
		}

		m.UpdateConfig(cfg)

		m.mu.RLock()
		defer m.mu.RUnlock()

		if m.config.BackupDefaults.FreshHours != 48 {
			t.Fatalf("expected FreshHours to remain 48, got %d", m.config.BackupDefaults.FreshHours)
		}
		if m.config.BackupDefaults.StaleHours != 48 {
			t.Fatalf("expected StaleHours to clamp to FreshHours (48), got %d", m.config.BackupDefaults.StaleHours)
		}
	})
}

func TestUpdateConfigNormalizesFlappingSettings(t *testing.T) {
	m := newTestManager(t)

	cfg := AlertConfig{
		Enabled:                 true,
		FlappingEnabled:         true,
		FlappingWindowSeconds:   0,
		FlappingThreshold:       -3,
		FlappingCooldownMinutes: 0,
	}

	m.UpdateConfig(cfg)

	m.mu.RLock()
	defer m.mu.RUnlock()

	if m.config.FlappingWindowSeconds != 300 {
		t.Fatalf("expected FlappingWindowSeconds to default to 300, got %d", m.config.FlappingWindowSeconds)
	}
	if m.config.FlappingThreshold != 5 {
		t.Fatalf("expected FlappingThreshold to default to 5, got %d", m.config.FlappingThreshold)
	}
	if m.config.FlappingCooldownMinutes != 15 {
		t.Fatalf("expected FlappingCooldownMinutes to default to 15, got %d", m.config.FlappingCooldownMinutes)
	}
}

// The thresholds page sends a positive trigger with clear max(0, trigger-5),
// so a 1-5% default arrives with clear 0. Agent defaults used to fall back to
// the factory clear, storing {trigger: 1, clear: 75}; the evaluator ignored
// that clear and cleared at the trigger, but firing alerts reported 75 as
// their clear level.
func TestUpdateConfigKeepsLowTriggerClearBelowTrigger(t *testing.T) {
	m := newTestManager(t)
	m.UpdateConfig(AlertConfig{
		Enabled: true,
		AgentDefaults: ThresholdConfig{
			CPU:    &HysteresisThreshold{Trigger: 1, Clear: 0},
			Memory: &HysteresisThreshold{Trigger: 3, Clear: 0},
			Disk:   &HysteresisThreshold{Trigger: 5, Clear: 0},
		},
		NodeDefaults: ThresholdConfig{
			Temperature: &HysteresisThreshold{Trigger: 2, Clear: 0},
		},
		PBSDefaults: ThresholdConfig{
			CPU:    &HysteresisThreshold{Trigger: 4, Clear: 0},
			Memory: &HysteresisThreshold{Trigger: 5, Clear: 0},
		},
	})

	cfg := m.GetConfig()
	for name, tc := range map[string]struct {
		got  *HysteresisThreshold
		want HysteresisThreshold
	}{
		"agent cpu":        {cfg.AgentDefaults.CPU, HysteresisThreshold{Trigger: 1, Clear: 0}},
		"agent memory":     {cfg.AgentDefaults.Memory, HysteresisThreshold{Trigger: 3, Clear: 0}},
		"agent disk":       {cfg.AgentDefaults.Disk, HysteresisThreshold{Trigger: 5, Clear: 0}},
		"node temperature": {cfg.NodeDefaults.Temperature, HysteresisThreshold{Trigger: 2, Clear: 0}},
		"pbs cpu":          {cfg.PBSDefaults.CPU, HysteresisThreshold{Trigger: 4, Clear: 0}},
		"pbs memory":       {cfg.PBSDefaults.Memory, HysteresisThreshold{Trigger: 5, Clear: 0}},
	} {
		if tc.got == nil || *tc.got != tc.want {
			t.Errorf("%s = %+v, want %+v", name, tc.got, tc.want)
		}
	}

	m.mu.Lock()
	m.config.TimeThresholds = map[string]int{}
	m.config.ActivationState = ActivationActive
	m.mu.Unlock()
	host := models.Host{
		ID:              "host-low-trigger",
		DisplayName:     "Low Trigger Host",
		Hostname:        "low-trigger.example",
		Memory:          models.Memory{Usage: 68.9, Total: 16384, Used: 11288, Free: 5096},
		Status:          "online",
		IntervalSeconds: 30,
		LastSeen:        time.Now(),
	}
	m.CheckHost(host)
	memAlert, exists := testLookupActiveAlert(t, m, canonicalMetricStateID(hostResourceID(host.ID), "memory"))
	if !exists {
		t.Fatalf("expected memory alert at a 3%% trigger")
	}
	// No band below a 3% trigger: the alert clears at the trigger itself.
	if got := memAlert.Metadata["clearThreshold"]; got != 3.0 {
		t.Fatalf("memory alert clearThreshold = %v, want 3", got)
	}

	// A config saved before the fix is repaired on its next load.
	m.UpdateConfig(AlertConfig{
		Enabled: true,
		AgentDefaults: ThresholdConfig{
			CPU: &HysteresisThreshold{Trigger: 1, Clear: 75},
		},
	})
	if got := m.GetConfig().AgentDefaults.CPU; got == nil || *got != (HysteresisThreshold{Trigger: 1, Clear: 0}) {
		t.Fatalf("stored agent cpu {1 75} normalized to %+v, want {1 0}", got)
	}
}

func TestUpdateConfigNormalizesExactEscalationRoutingAndRepeatPolicy(t *testing.T) {
	m := newTestManager(t)
	cfg := m.GetConfig()
	cfg.Schedule.Escalation = EscalationConfig{
		Enabled:        true,
		RepeatCritical: true,
		RepeatEvery:    1,
		Levels: []EscalationLevel{
			{
				After:          -1,
				Notify:         "WEBHOOKS",
				DestinationIDs: []string{" webhook:pager ", "email", "webhook:pager", "https://secret.example"},
			},
			{After: 181, Notify: "email"},
		},
	}

	m.UpdateConfig(cfg)

	got := m.GetConfig().Schedule.Escalation
	if got.RepeatEvery != DefaultEscalationRepeatMinutes {
		t.Fatalf("repeat interval = %d, want %d", got.RepeatEvery, DefaultEscalationRepeatMinutes)
	}
	if got.Levels[0].Notify != "webhook" {
		t.Fatalf("legacy target = %q, want webhook", got.Levels[0].Notify)
	}
	if got.Levels[0].After != MinEscalationDelayMinutes {
		t.Fatalf("minimum escalation delay = %d, want %d", got.Levels[0].After, MinEscalationDelayMinutes)
	}
	if got.Levels[1].After != MaxEscalationDelayMinutes {
		t.Fatalf("maximum escalation delay = %d, want %d", got.Levels[1].After, MaxEscalationDelayMinutes)
	}
	wantIDs := []string{"webhook:pager", "email"}
	if len(got.Levels[0].DestinationIDs) != len(wantIDs) {
		t.Fatalf("destination IDs = %v, want %v", got.Levels[0].DestinationIDs, wantIDs)
	}
	for index, want := range wantIDs {
		if got.Levels[0].DestinationIDs[index] != want {
			t.Fatalf("destination IDs = %v, want %v", got.Levels[0].DestinationIDs, wantIDs)
		}
	}
}

// The thresholds page sends no flapping, alert TTL or custom-rule keys. A save
// through ApplyConfigUpdate must leave those settings as stored, also after
// the merged config is normalized in place.
func TestUpdateConfigKeepsSettingsAThresholdsSaveDidNotSend(t *testing.T) {
	m := newTestManager(t)
	stored := m.GetConfig()
	stored.FlappingEnabled = true
	stored.FlappingWindowSeconds = 600
	stored.MaxAlertAgeDays = 7
	stored.MaxAcknowledgedAgeDays = 1
	stored.AutoAcknowledgeAfterHours = 24
	stored.CustomRules = []CustomAlertRule{{ID: "rule-1", Name: "Busy web", Enabled: true}}
	m.UpdateConfig(stored)

	snapshot, err := m.ApplyConfigUpdate([]byte(`{
		"enabled": true,
		"nodeDefaults": {"cpu": {"trigger": 82, "clear": 77}},
		"overrides": {}
	}`))
	if err != nil {
		t.Fatalf("ApplyConfigUpdate: %v", err)
	}
	snapshot.TimeThresholds["guest"] = 999
	if m.GetConfig().TimeThresholds["guest"] == 999 {
		t.Fatal("the returned snapshot shares its maps with the live config")
	}

	got := m.GetConfig()
	if got.NodeDefaults.CPU == nil || got.NodeDefaults.CPU.Trigger != 82 {
		t.Fatalf("node CPU default = %+v, want the saved trigger 82", got.NodeDefaults.CPU)
	}
	if !got.FlappingEnabled || got.FlappingWindowSeconds != 600 {
		t.Fatalf("flapping = %v/%d after a thresholds save, want true/600", got.FlappingEnabled, got.FlappingWindowSeconds)
	}
	if got.MaxAlertAgeDays != 7 || got.MaxAcknowledgedAgeDays != 1 || got.AutoAcknowledgeAfterHours != 24 {
		t.Fatalf("TTL cleanup = %d/%d/%d after a thresholds save, want 7/1/24",
			got.MaxAlertAgeDays, got.MaxAcknowledgedAgeDays, got.AutoAcknowledgeAfterHours)
	}
	if len(got.CustomRules) != 1 || got.CustomRules[0].ID != "rule-1" {
		t.Fatalf("custom rules = %+v after a thresholds save, want the stored rule", got.CustomRules)
	}
}

// Each partial update reads, merges and applies under the manager lock, so
// two saves that touch different settings cannot revert each other.
func TestApplyConfigUpdateKeepsConcurrentPartialUpdates(t *testing.T) {
	m := newTestManager(t)

	for round := 0; round < 50; round++ {
		base := m.GetConfig()
		base.FlappingEnabled = false
		base.MaxAlertAgeDays = 0
		m.UpdateConfig(base)

		bodies := []string{`{"flappingEnabled": true}`, `{"maxAlertAgeDays": 9}`}
		done := make(chan struct{}, len(bodies))
		for _, body := range bodies {
			go func(body string) {
				defer func() { done <- struct{}{} }()
				if _, err := m.ApplyConfigUpdate([]byte(body)); err != nil {
					t.Errorf("ApplyConfigUpdate(%s): %v", body, err)
				}
			}(body)
		}
		for range bodies {
			<-done
		}

		got := m.GetConfig()
		if !got.FlappingEnabled || got.MaxAlertAgeDays != 9 {
			t.Fatalf("round %d: flapping=%v maxAlertAgeDays=%d, want both concurrent updates kept",
				round, got.FlappingEnabled, got.MaxAlertAgeDays)
		}
	}
}

func inputOwnershipConfig(m *Manager) AlertConfig {
	cfg := m.GetConfig()
	cfg.Enabled = true
	cfg.ActivationState = ActivationActive
	cfg.GuestDefaults.CPU = &HysteresisThreshold{Trigger: 80, Clear: 75}
	cfg.TimeThresholds = map[string]int{"guest": 0}
	cfg.Overrides = map[string]ThresholdConfig{"ownership-guest": {CPU: &HysteresisThreshold{Trigger: 80, Clear: 75}}}
	cfg.MetricTimeThresholds = map[string]map[string]int{"guest": {"cpu": 0}}
	cfg.MetricEvaluationWindows = map[string]map[string]int{"guest": {"cpu": 0}}
	cfg.IgnoredGuestPrefixes = []string{"ignored-"}
	cfg.Schedule.QuietHours.Days = map[string]bool{"monday": true}
	cfg.Schedule.Escalation.Levels = []EscalationLevel{{After: 10, Notify: "all", DestinationIDs: []string{"webhook:original"}}}
	return cfg
}

func TestUpdateConfigDoesNotNormalizeCallerInput(t *testing.T) {
	m := NewManagerWithDataDir(t.TempDir())
	t.Cleanup(m.Stop)
	cfg := inputOwnershipConfig(m)
	// Invalid hysteresis and delays must still be normalised in the manager,
	// without rewriting the caller's pointers or maps.
	cfg.GuestDefaults.CPU.Clear = 90
	cfg.TimeThresholds["guest"] = -1
	before := cfg.Clone()
	m.UpdateConfig(cfg)
	if !reflect.DeepEqual(cfg, before) {
		t.Error("applying a config rewrote caller-owned configuration")
	}
	live := m.GetConfig()
	if live.GuestDefaults.CPU.Clear != 75 || live.Overrides["ownership-guest"].CPU.Clear != 75 || live.TimeThresholds["guest"] < 0 {
		t.Fatalf("owned config did not retain normalisation: guest=%+v override=%+v delay=%d", live.GuestDefaults.CPU, live.Overrides["ownership-guest"].CPU, live.TimeThresholds["guest"])
	}
}

func TestUpdateConfigCallerEditsDoNotChangeAppliedPolicy(t *testing.T) {
	cases := map[string]func(*AlertConfig){
		"threshold-pointer": func(c *AlertConfig) { c.GuestDefaults.CPU.Trigger = 0 },
		"override-map":      func(c *AlertConfig) { c.Overrides["ownership-guest"] = ThresholdConfig{Disabled: true} },
		"override-pointer":  func(c *AlertConfig) { c.Overrides["ownership-guest"].CPU.Trigger = 0 },
		"delay-map":         func(c *AlertConfig) { c.TimeThresholds["guest"] = 3600 },
		"nested-delay-map":  func(c *AlertConfig) { c.MetricTimeThresholds["guest"]["cpu"] = 3600 },
		"window-map":        func(c *AlertConfig) { c.MetricEvaluationWindows["guest"]["cpu"] = 3600 },
		"prefix-slice":      func(c *AlertConfig) { c.IgnoredGuestPrefixes[0] = "ownership-" },
		"schedule-map":      func(c *AlertConfig) { c.Schedule.QuietHours.Days["monday"] = false },
		"destination-slice": func(c *AlertConfig) { c.Schedule.Escalation.Levels[0].DestinationIDs[0] = "webhook:changed" },
	}
	for name, edit := range cases {
		t.Run(name, func(t *testing.T) {
			m := NewManagerWithDataDir(t.TempDir())
			t.Cleanup(m.Stop)
			cfg := inputOwnershipConfig(m)
			m.UpdateConfig(cfg)
			applied := m.GetConfig()
			edit(&cfg)
			if got := m.GetConfig(); !reflect.DeepEqual(got, applied) {
				t.Error("editing the submitted config changed live policy without an update")
			}
			guest := models.VM{ID: "ownership-guest", Name: "ownership-guest", Status: "running", CPU: .95}
			m.CheckGuest(guest, "ownership")
			found := false
			for _, alert := range m.GetActiveAlerts() {
				if alert.ResourceID == guest.ID && alert.Type == "cpu" {
					found = true
				}
			}
			if !found {
				t.Error("submitted-config edit prevented the ordinary CPU alert")
			}
			// An intentional resubmission still applies the changed setting.
			m.UpdateConfig(cfg)
			expected := m.GetConfig()
			if reflect.DeepEqual(expected, applied) {
				t.Error("explicit resubmission did not apply the edit")
			}
		})
	}
}

func TestUpdateConfigInputCanBeEncodedDuringLaterUpdates(t *testing.T) {
	m := NewManagerWithDataDir(t.TempDir())
	t.Cleanup(m.Stop)
	cfg := inputOwnershipConfig(m)
	m.UpdateConfig(cfg)
	before := cfg.Clone()
	// A caller may retain an applied snapshot for a response or persistence.
	// Later policy reconciliation must not write its override map while it is
	// being encoded. Exercise this under -race, using actual manager updates.
	var wg sync.WaitGroup
	wg.Add(2)
	go func() {
		defer wg.Done()
		for i := 0; i < 100; i++ {
			if _, err := json.Marshal(cfg); err != nil {
				t.Error(err)
			}
		}
	}()
	go func() {
		defer wg.Done()
		for i := 0; i < 100; i++ {
			// An alias of an already applied config is legal sequential input. The
			// manager owns normalisation; neither call transfers caller ownership.
			m.UpdateConfig(cfg)
			m.CheckGuest(models.VM{ID: "ownership-guest", Status: "running", CPU: .2, LastSeen: time.Now()}, "ownership")
		}
	}()
	wg.Wait()
	if !reflect.DeepEqual(cfg, before) {
		t.Error("later updates modified retained input")
	}
}