Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

9.2 KB
Raw Blame
package monitoring

import (
	"testing"
	"time"

	"github.com/rcourtman/pulse-go-rewrite/internal/config"
	"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
	"github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"
)

// simulateAdaptiveSuccessGaps drives the real adaptive scheduler, interval
// selector and staleness scoring for one always-healthy instance on a
// simulated clock, the way the monitor does: a worker runs the queued task
// once it is due and reschedules it, and a planning pass re-plans every poll
// tick. It returns the gaps between successive successful polls after warmup.
// The worker is instantaneous and uncontended, so real gaps can run longer
// when workers are busy; the factor of two in the threshold is headroom for
// that, not a bound.
func simulateAdaptiveSuccessGaps(t *testing.T, cfg SchedulerConfig, instanceType InstanceType, tick, warmup, horizon time.Duration) []time.Duration {
	t.Helper()
	const name = "instance-1"
	tracker := NewStalenessTracker(nil)
	tracker.SetBounds(cfg.BaseInterval, cfg.MaxInterval)
	scheduler := NewAdaptiveScheduler(cfg, tracker, nil, nil)

	start := time.Unix(1_800_000_000, 0)
	simNow := start
	var lastSuccess time.Time
	plan := func(desc InstanceDescriptor) ScheduledTask {
		// StalenessTracker scores against the wall clock, so stamp the
		// simulated age of the last success just before planning.
		if !lastSuccess.IsZero() {
			tracker.setSnapshot(FreshnessSnapshot{
				InstanceType: instanceType,
				Instance:     name,
				LastSuccess:  time.Now().Add(-simNow.Sub(lastSuccess)),
			})
		}
		tasks := scheduler.BuildPlan(simNow, []InstanceDescriptor{desc}, 0)
		if len(tasks) != 1 {
			t.Fatalf("BuildPlan returned %d tasks, want 1", len(tasks))
		}
		return tasks[0]
	}
	planningPass := func() ScheduledTask {
		desc := InstanceDescriptor{Name: name, Type: instanceType, LastSuccess: lastSuccess}
		if last, ok := scheduler.LastScheduled(instanceType, name); ok {
			desc.LastScheduled = last.NextRun
			desc.LastInterval = last.Interval
		}
		return plan(desc)
	}

	queued := planningPass()
	var gaps []time.Duration
	for step := time.Duration(0); step <= horizon; step += time.Second {
		simNow = start.Add(step)
		if !simNow.Before(queued.NextRun) {
			if !lastSuccess.IsZero() && step >= warmup {
				gaps = append(gaps, simNow.Sub(lastSuccess))
			}
			lastSuccess = simNow
			queued = plan(InstanceDescriptor{
				Name:          name,
				Type:          instanceType,
				LastInterval:  queued.Interval,
				LastScheduled: queued.NextRun,
				LastSuccess:   lastSuccess,
			})
		}
		if step > 0 && step%tick == 0 {
			queued = planningPass()
		}
	}
	if len(gaps) == 0 {
		t.Fatalf("%s: no polls after warmup", instanceType)
	}
	return gaps
}

func longestGap(gaps []time.Duration) time.Duration {
	var longest time.Duration
	for _, gap := range gaps {
		longest = max(longest, gap)
	}
	return longest
}

// With adaptive polling enabled, a healthy instance's staleness score is near
// zero after every success, so the scheduler stretches its cadence toward
// AdaptivePollingMaxInterval and never reads the per-platform intervals.
// Freshness, node offline grace and the temperature carry window must cover
// that cadence, or healthy Proxmox, PBS and PMG rows read stale (and nodes
// lose their grace) for most of every cycle.
func TestMonitorResourceFreshnessFollowsAdaptiveCadence(t *testing.T) {
	sources := map[InstanceType]unifiedresources.DataSource{
		InstanceTypePVE: unifiedresources.SourceProxmox,
		InstanceTypePBS: unifiedresources.SourcePBS,
		InstanceTypePMG: unifiedresources.SourcePMG,
	}
	for _, bounds := range []SchedulerConfig{
		{BaseInterval: 10 * time.Second, MinInterval: 5 * time.Second, MaxInterval: 5 * time.Minute},
		{BaseInterval: 30 * time.Second, MinInterval: 10 * time.Second, MaxInterval: 15 * time.Minute},
	} {
		monitor := &Monitor{
			config: &config.Config{
				PVEPollingInterval: 10 * time.Second,
				PBSPollingInterval: time.Minute,
				PMGPollingInterval: time.Minute,
			},
			scheduler: NewAdaptiveScheduler(bounds, nil, nil, nil),
		}
		thresholds := monitor.resourceStaleThresholds()
		for instanceType, source := range sources {
			gaps := simulateAdaptiveSuccessGaps(t, bounds, instanceType, 10*time.Second, time.Hour, 8*time.Hour)
			longest := longestGap(gaps)
			if longest > bounds.MaxInterval+time.Second {
				t.Fatalf("%s: healthy polls %v apart exceed the scheduler's %v maximum", instanceType, longest, bounds.MaxInterval)
			}
			if got, want := thresholds[source], 2*bounds.MaxInterval; got != want {
				t.Errorf("max %v: %s threshold = %v, want %v from the adaptive cadence (healthy polls up to %v apart)",
					bounds.MaxInterval, source, got, want, longest)
			}
			if instanceType != InstanceTypePVE {
				continue
			}
			if grace := monitor.pveNodeOfflineGracePeriod(); grace <= longest {
				t.Errorf("max %v: node offline grace %v expires before the next healthy poll (%v apart)", bounds.MaxInterval, grace, longest)
			}
			if carry, want := monitor.nodeTemperatureCarryWindow(), max(2*bounds.MaxInterval, nodeTemperatureCarryFloor); carry != want || carry <= longest {
				t.Errorf("max %v: temperature carry window = %v, want %v from the adaptive cadence (healthy polls up to %v apart)", bounds.MaxInterval, carry, want, longest)
			}
		}
	}
}

// A node the API reports offline is only believed once it has been away
// longer than one healthy adaptive cycle; past twice the cadence the grace
// lapses as it does under fixed-cadence polling.
func TestAdaptiveNodeOfflineGraceSpansTheAdaptiveCycle(t *testing.T) {
	instance := &config.PVEInstance{Name: "site-a"}
	monitor := &Monitor{
		config:         &config.Config{PVEPollingInterval: 10 * time.Second},
		scheduler:      NewAdaptiveScheduler(SchedulerConfig{BaseInterval: 10 * time.Second, MinInterval: 5 * time.Second, MaxInterval: 5 * time.Minute}, nil, nil, nil),
		nodeLastOnline: map[string]time.Time{"site-a-pve-a": time.Now().Add(-4 * time.Minute)},
	}
	offline := proxmox.Node{Node: "pve-a", Status: "offline"}
	if _, status := monitor.determineNodeIDAndStatus("site-a", instance, offline); status != "online" {
		t.Fatalf("node status = %q one adaptive cycle after its last online poll, want online inside the grace", status)
	}
	monitor.nodeLastOnline["site-a-pve-a"] = time.Now().Add(-11 * time.Minute)
	if _, status := monitor.determineNodeIDAndStatus("site-a", instance, offline); status != "offline" {
		t.Fatalf("node status = %q past twice the adaptive cadence, want offline", status)
	}
}

// Callers without a live monitor (adapter construction) derive freshness from
// config alone. It must name the same cadence the monitor's scheduler will
// run, including the defaults the scheduler fills in for unset or inconsistent
// adaptive bounds.
func TestResourceStaleThresholdsForConfigFollowAdaptiveBounds(t *testing.T) {
	cases := []struct {
		name string
		cfg  config.Config
		want time.Duration
	}{
		{
			name: "adaptive off keeps the per-platform intervals",
			cfg: config.Config{
				PVEPollingInterval: 10 * time.Second, PBSPollingInterval: 3 * time.Minute, PMGPollingInterval: 3 * time.Minute,
				AdaptivePollingMaxInterval: 15 * time.Minute,
			},
		},
		{
			name: "configured maximum",
			cfg: config.Config{
				AdaptivePollingEnabled:     true,
				PBSPollingInterval:         3 * time.Minute,
				AdaptivePollingMinInterval: 5 * time.Second, AdaptivePollingMaxInterval: 15 * time.Minute,
			},
			want: 30 * time.Minute,
		},
		{
			name: "unset bounds use the scheduler default",
			cfg:  config.Config{AdaptivePollingEnabled: true},
			want: 2 * DefaultSchedulerConfig().MaxInterval,
		},
		{
			name: "maximum below the minimum uses the scheduler default",
			cfg: config.Config{
				AdaptivePollingEnabled:     true,
				AdaptivePollingMinInterval: time.Minute, AdaptivePollingMaxInterval: 30 * time.Second,
			},
			want: 2 * DefaultSchedulerConfig().MaxInterval,
		},
	}
	for _, tc := range cases {
		t.Run(tc.name, func(t *testing.T) {
			cfg := tc.cfg
			thresholds := ResourceStaleThresholdsForConfig(&cfg)
			for _, source := range []unifiedresources.DataSource{unifiedresources.SourceProxmox, unifiedresources.SourcePBS, unifiedresources.SourcePMG} {
				want := tc.want
				if !cfg.AdaptivePollingEnabled {
					want = map[unifiedresources.DataSource]time.Duration{
						unifiedresources.SourceProxmox: defaultProxmoxResourceStaleThreshold,
						unifiedresources.SourcePBS:     6 * time.Minute,
						unifiedresources.SourcePMG:     6 * time.Minute,
					}[source]
				}
				if got := thresholds[source]; got != want {
					t.Errorf("%s threshold = %v, want %v", source, got, want)
				}
			}
			if !cfg.AdaptivePollingEnabled {
				return
			}
			scheduler := NewAdaptiveScheduler(SchedulerConfig{
				BaseInterval: cfg.AdaptivePollingBaseInterval,
				MinInterval:  cfg.AdaptivePollingMinInterval,
				MaxInterval:  cfg.AdaptivePollingMaxInterval,
			}, nil, nil, nil)
			monitor := &Monitor{config: &cfg, scheduler: scheduler}
			for source, want := range thresholds {
				if source != unifiedresources.SourceProxmox && source != unifiedresources.SourcePBS && source != unifiedresources.SourcePMG {
					continue
				}
				if got := monitor.resourceStaleThresholds()[source]; got != want {
					t.Errorf("%s: monitor threshold %v differs from the config-only %v", source, got, want)
				}
			}
		})
	}
}