Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

10.2 KB
Raw Blame
package monitoring

import (
	"fmt"
	"strings"
	"sync"
	"testing"
	"time"

	"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
	"github.com/rcourtman/pulse-go-rewrite/internal/mock"
	"github.com/rcourtman/pulse-go-rewrite/internal/models"
	"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
)

// This resolver blocks only after metricWindowPoints has captured its history.
// It lets the test replace that history at an exact point without scheduling
// sleeps or exposing a production-only hook.
type blockingMetricTargetStore struct {
	entered chan struct{}
	resume  <-chan struct{}
}

func (*blockingMetricTargetStore) ShouldSkipAPIPolling(string) bool { return false }
func (*blockingMetricTargetStore) GetPollingRecommendations() map[string]float64 {
	return nil
}
func (*blockingMetricTargetStore) GetAll() []unifiedresources.Resource       { return nil }
func (*blockingMetricTargetStore) PopulateFromSnapshot(models.StateSnapshot) {}
func (s *blockingMetricTargetStore) MetricsTargetForResource(string) *unifiedresources.MetricsTarget {
	s.entered <- struct{}{}
	<-s.resume
	return nil
}

func TestMetricWindowPointsUsesInMemoryMetricAlias(t *testing.T) {
	now := time.Now().UTC()
	history := NewMetricsHistory(32, time.Hour)
	history.AddGuestMetric("vm-1", "netin", 12, now.Add(-4*time.Minute))
	history.AddGuestMetric("vm-1", "netin", 18, now.Add(-2*time.Minute))
	monitor := &Monitor{metricsHistory: history}

	points, err := monitor.metricWindowPoints(alerts.MetricWindowRequest{
		ResourceID:   "vm-1",
		ResourceType: "vm",
		Metric:       "networkIn",
		Start:        now.Add(-5 * time.Minute),
		End:          now,
	})
	if err != nil {
		t.Fatalf("metricWindowPoints returned error: %v", err)
	}
	if len(points) != 2 || points[0].Value != 12 || points[1].Value != 18 {
		t.Fatalf("metricWindowPoints = %+v, want canonical netin history", points)
	}
}

func TestUsedMemoryHistoryCleanupThroughMonitorAndReaders(t *testing.T) {
	previous := mock.IsMockEnabled()
	mustSetMockEnabled(t, false)
	defer mustSetMockEnabled(t, previous)
	for _, resourceType := range []string{"vm", "container"} {
		t.Run(resourceType, func(t *testing.T) {
			now := time.Now()
			history := NewMetricsHistory(32, 4*time.Hour)
			monitor := &Monitor{metricsHistory: history}
			const id = "instance:node:105"
			monitor.recordGuestMetric(resourceType, id, 20, 60, 4096, -1, -1, -1, -1, -1, now.Add(-2*time.Hour))
			monitor.recordGuestMetric(resourceType, id, 25, -1, -1, -1, -1, -1, -1, -1, now.Add(-time.Minute))
			monitor.cleanupMetricsHistory()
			if got := monitor.GetGuestMetrics(id, 4*time.Hour)["memoryused"]; len(got) != 1 || got[0].Value != 4096 {
				t.Fatalf("in-window last-known bytes were lost or refreshed: %+v", got)
			}
			history.retentionTime = time.Hour
			monitor.cleanupMetricsHistory()
			if got := monitor.GetGuestMetrics(id, 4*time.Hour)["memoryused"]; len(got) != 0 {
				t.Fatalf("ongoing CPU retained expired byte samples: %+v", got)
			}
			monitor.recordGuestMetric(resourceType, id, 30, 70, 8192, -1, -1, -1, -1, -1, now)
			monitor.cleanupMetricsHistory()
			chart := monitor.GetGuestMetricsForChart(id, resourceType, id, 4*time.Hour)["memoryused"]
			if len(chart) != 1 || chart[0].Value != 8192 || !chart[0].Timestamp.Equal(now) {
				t.Fatalf("chart did not retain the fresh byte observation: %+v", chart)
			}
			points, err := monitor.metricWindowPoints(alerts.MetricWindowRequest{
				ResourceID: id, ResourceType: resourceType, Metric: "memoryused",
				Start: now.Add(-4 * time.Hour), End: now,
			})
			if err != nil || len(points) != 1 || points[0].Value != 8192 || !points[0].Timestamp.Equal(now) {
				t.Fatalf("metric window diverged from fresh chart bytes: points=%+v err=%v", points, err)
			}
		})
	}
}

func TestMetricWindowMergePrefersFreshInMemoryDuplicate(t *testing.T) {
	now := time.Now().UTC()
	stored := []MetricPoint{
		{Timestamp: now.Add(-4 * time.Minute), Value: 40},
		{Timestamp: now.Add(-2 * time.Minute), Value: 99},
	}
	inMemory := []MetricPoint{
		{Timestamp: now.Add(-2 * time.Minute), Value: 55},
		{Timestamp: now.Add(-time.Minute), Value: 60},
	}

	got := mergeMetricWindowPoints(stored, inMemory, now.Add(-5*time.Minute), now)
	if len(got) != 3 {
		t.Fatalf("mergeMetricWindowPoints returned %d points, want 3: %+v", len(got), got)
	}
	if got[1].Value != 55 {
		t.Fatalf("duplicate timestamp value = %.1f, want fresh in-memory value 55", got[1].Value)
	}
}

func TestMetricsHistoryResetClearsMetricWindowCache(t *testing.T) {
	history := NewMetricsHistory(32, time.Hour)
	now := time.Now().UTC()
	history.cacheMetricWindow("vm\x00vm-1\x00cpu\x00300", []MetricPoint{{Timestamp: now, Value: 42}}, now)

	history.Reset()
	if history.metricWindowCache != nil || history.metricWindowBytes != 0 || !history.metricWindowSweepAt.IsZero() {
		t.Fatalf("metric window cache survived reset: %+v", history.metricWindowCache)
	}
}

func TestMetricWindowPointsKeepsHistorySnapshotDuringReplacement(t *testing.T) {
	now := time.Now().UTC()
	monitor := newChartFallbackTestMonitor(t)
	history := monitor.metricsHistory
	cacheKey := strings.Join([]string{"vm", "vm-1", "cpu", "300"}, "\x00")
	history.cacheMetricWindow(cacheKey, []MetricPoint{{Timestamp: now.Add(-time.Minute), Value: 42}}, now)

	resume := make(chan struct{})
	var resumeOnce sync.Once
	resumeRequest := func() { resumeOnce.Do(func() { close(resume) }) }
	defer resumeRequest()
	entered := make(chan struct{}, 1)
	monitor.resourceStore = &blockingMetricTargetStore{entered: entered, resume: resume}
	type requestResult struct {
		points []alerts.MetricWindowPoint
		err    error
	}
	result := make(chan requestResult, 1)
	go func() {
		points, err := monitor.metricWindowPoints(alerts.MetricWindowRequest{
			ResourceID:   "vm-1",
			ResourceType: "vm",
			Metric:       "cpu",
			Start:        now.Add(-5 * time.Minute),
			End:          now,
		})
		result <- requestResult{points: points, err: err}
	}()

	select {
	case <-entered:
	case <-time.After(10 * time.Second):
		t.Fatal("metric request did not reach target resolution after snapshot")
	}
	monitor.mu.Lock()
	monitor.metricsHistory = NewMetricsHistory(32, time.Hour)
	monitor.mu.Unlock()
	resumeRequest()

	select {
	case got := <-result:
		if got.err != nil {
			t.Fatalf("metricWindowPoints returned error: %v", got.err)
		}
		if len(got.points) != 1 || got.points[0].Value != 42 {
			t.Fatalf("metricWindowPoints = %+v, want cached point from the request snapshot", got.points)
		}
	case <-time.After(10 * time.Second):
		t.Fatal("metricWindowPoints did not finish after history replacement")
	}
}

func TestMetricWindowPointsPreservesCompleteWindowAtCacheCapacity(t *testing.T) {
	for _, pressure := range []string{"none", "entries", "bytes"} {
		t.Run(pressure, func(t *testing.T) {
			monitor := newChartFallbackTestMonitor(t)
			history := monitor.metricsHistory
			now := time.Now().UTC().Truncate(time.Second)
			writeRawMetricBatch(t, monitor.metricsStore, "vm", "vm-1", "cpu", []MetricPoint{
				{Timestamp: now.Add(-4 * time.Minute), Value: 0},
				{Timestamp: now.Add(-2 * time.Minute), Value: 99},
				{Timestamp: now.Add(-time.Minute), Value: 30},
			})
			history.AddGuestMetric("vm-1", "cpu", 55, now.Add(-2*time.Minute))
			history.AddGuestMetric("vm-1", "cpu", 60, now.Add(-30*time.Second))

			// Fill the optional cache, not the query source or live history.
			// Its complete source result must still reach alert evaluation.
			cacheNow := time.Now()
			switch pressure {
			case "entries":
				for i := 0; i < metricWindowCacheMaxEntries; i++ {
					history.cacheMetricWindow(fmt.Sprintf("other-%04d", i), nil, cacheNow)
				}
			case "bytes":
				points := make([]MetricPoint, 4096)
				for i := 0; i < 63; i++ {
					history.cacheMetricWindow(fmt.Sprintf("other-%04d", i), points, cacheNow)
				}
				remaining := metricWindowCacheMaxBytes - history.metricWindowBytes
				if remaining <= metricWindowCacheEntryBytes {
					t.Fatal("fixture has no room for the final exact-budget entry")
				}
				history.cacheMetricWindow(strings.Repeat("x", remaining-metricWindowCacheEntryBytes), nil, cacheNow)
				if history.metricWindowBytes != metricWindowCacheMaxBytes {
					t.Fatalf("fixture filled %d bytes, want %d", history.metricWindowBytes, metricWindowCacheMaxBytes)
				}
			}
			retainedEntries, retainedBytes := len(history.metricWindowCache), history.metricWindowBytes
			request := alerts.MetricWindowRequest{
				ResourceID: "vm-1", ResourceType: "vm", Metric: "cpu",
				Start: now.Add(-5 * time.Minute), End: now,
			}
			want := []alerts.MetricWindowPoint{
				{Timestamp: now.Add(-4 * time.Minute), Value: 0},
				{Timestamp: now.Add(-2 * time.Minute), Value: 55},
				{Timestamp: now.Add(-time.Minute), Value: 30},
				{Timestamp: now.Add(-30 * time.Second), Value: 60},
			}
			assertWindow := func() {
				t.Helper()
				got, err := monitor.metricWindowPoints(request)
				if err != nil || len(got) != len(want) {
					t.Fatalf("window lost source coverage: got=%+v err=%v want=%+v", got, err, want)
				}
				for i := range want {
					if !got[i].Timestamp.Equal(want[i].Timestamp) || got[i].Value != want[i].Value {
						t.Fatalf("window changed source time/value or live-tail authority: got=%+v want=%+v", got, want)
					}
				}
				// A returned result never owns the cache or next query's data.
				got[0].Value = -1
			}
			assertWindow()
			assertWindow()
			cacheKey := strings.Join([]string{"vm", "vm-1", "cpu", "300"}, "\x00")
			_, cached := history.cachedMetricWindow(cacheKey, time.Now())
			if cached != (pressure == "none") {
				t.Fatalf("optional copy admitted=%v under %q pressure", cached, pressure)
			}
			if pressure != "none" && (len(history.metricWindowCache) != retainedEntries || history.metricWindowBytes != retainedBytes) {
				t.Fatal("refused optional copy changed the retained working set")
			}
			assertMetricWindowCacheAccounting(t, history)

			// Reset releases capacity and live history, not the persistent source.
			// The ordinary reader can cache again and still returns every point.
			history.Reset()
			history.AddGuestMetric("vm-1", "cpu", 55, now.Add(-2*time.Minute))
			history.AddGuestMetric("vm-1", "cpu", 60, now.Add(-30*time.Second))
			assertWindow()
			if _, ok := history.cachedMetricWindow(cacheKey, time.Now()); !ok {
				t.Fatal("reset did not restore ordinary cache admission")
			}
			assertMetricWindowCacheAccounting(t, history)
		})
	}
}