Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

14.3 KB
Raw Blame
package metrics

import (
	"fmt"
	pdb "github.com/rcourtman/pulse-go-rewrite/pkg/db"
	"reflect"
	"testing"
	"time"
)

func TestStoreRetainedTierCoverage(t *testing.T) {
	db := newPlanTestDB(t)
	store := &Store{db: pdb.Wrap(db, "tier-coverage-test")}
	base := time.Date(2026, 9, 5, 12, 0, 0, 0, time.UTC)
	insert := func(id, metric string, tier Tier, offset time.Duration, value, low, high float64) {
		t.Helper()
		_, err := db.Exec(`INSERT INTO metrics(resource_type,resource_id,metric_type,tier,timestamp,value,min_value,max_value) VALUES ('node',?,?,?,?,?,?,?)`, id, metric, tier, base.Add(offset).Unix(), value, low, high)
		if err != nil {
			t.Fatal(err)
		}
	}
	// Raw observations before, within and after aggregate buckets. The raw
	// overlap deliberately differs so double-counting is visible in the mean.
	insert("a", "cpu", TierMinute, 0, 10, 2, 90)
	insert("a", "cpu", TierMinute, 2*time.Minute, 30, 25, 40)
	insert("a", "cpu", TierRaw, -time.Minute, 5, 5, 5)
	insert("a", "cpu", TierRaw, 20*time.Second, 999, 999, 999)
	insert("a", "cpu", TierRaw, time.Minute, 20, 20, 20)
	insert("a", "cpu", TierRaw, 3*time.Minute, 50, 50, 50)
	insert("a", "memory", TierRaw, 3*time.Minute, 88, 88, 88)
	// The older hourly fallback remains useful, but the overlapping hour is
	// indivisible and must not be mixed with the preferred minute/raw evidence.
	insert("a", "cpu", TierHourly, -2*time.Hour, 7, 1, 9)
	insert("a", "cpu", TierHourly, 0, 777, 777, 777)
	insert("b", "cpu", TierRaw, 20*time.Second, 60, 60, 60)
	insert("b", "memory", TierMinute, 0, 70, 65, 75)
	insert("b", "memory", TierRaw, 3*time.Minute, 80, 80, 80)
	// Same ID in another resource family cannot shadow or contribute evidence.
	_, err := db.Exec(`INSERT INTO metrics(resource_type,resource_id,metric_type,tier,timestamp,value) VALUES ('vm','a','cpu','minute',?,1234)`, base.Add(time.Minute).Unix())
	if err != nil {
		t.Fatal(err)
	}
	start, end := base.Add(-23*time.Hour), base.Add(4*time.Minute)
	want := []MetricPoint{
		{base.Add(-2 * time.Hour), 7, 1, 9}, {base.Add(-time.Minute), 5, 5, 5},
		{base, 10, 2, 90}, {base.Add(time.Minute), 20, 20, 20},
		{base.Add(2 * time.Minute), 30, 25, 40}, {base.Add(3 * time.Minute), 50, 50, 50},
	}
	for _, step := range []int64{0, 120} {
		t.Run(fmt.Sprintf("step_%d", step), func(t *testing.T) {
			batch, err := store.QueryAllBatch("node", []string{"a", "b", "a", "missing"}, start, end, step)
			if err != nil {
				t.Fatal(err)
			}
			filtered, err := store.QueryMetricTypesBatch("node", []string{"a", "b"}, []string{"cpu"}, start, end, step)
			if err != nil {
				t.Fatal(err)
			}
			filteredMany, err := store.QueryMetricTypesBatch("node", []string{"a", "b"}, []string{"memory", "cpu", "cpu"}, start, end, step)
			if err != nil {
				t.Fatal(err)
			}
			for _, id := range []string{"a", "b"} {
				all, err := store.QueryAll("node", id, start, end, step)
				if err != nil {
					t.Fatal(err)
				}
				if !reflect.DeepEqual(all, batch[id]) {
					t.Fatalf("batch differs for %s: %v != %v", id, batch[id], all)
				}
				if !reflect.DeepEqual(filteredMany[id], all) {
					t.Fatalf("multi-metric filter changed ordering or retained values for %s: %v != %v", id, filteredMany[id], all)
				}
				for _, metric := range []string{"cpu", "memory"} {
					single, err := store.Query("node", id, metric, start, end, step)
					if err != nil {
						t.Fatal(err)
					}
					if !reflect.DeepEqual(single, all[metric]) {
						t.Fatalf("single differs for %s/%s", id, metric)
					}
				}
				if len(filtered[id]) != 1 || !reflect.DeepEqual(filtered[id]["cpu"], all["cpu"]) {
					t.Fatalf("filtered mismatch: %+v", filtered)
				}
			}
			got := batch["a"]["cpu"]
			expected := want
			if step == 120 {
				expected = []MetricPoint{{base.Add(-119 * time.Minute), 7, 1, 9}, {base.Add(-time.Minute), 5, 5, 5}, {base.Add(time.Minute), 15, 2, 90}, {base.Add(3 * time.Minute), 40, 25, 50}}
			}
			if len(got) != len(expected) {
				t.Fatalf("got %+v\nwant %+v", got, expected)
			}
			for i, p := range got {
				want := expected[i]
				if !p.Timestamp.Equal(want.Timestamp) || p.Value != want.Value || p.Min != want.Min || p.Max != want.Max {
					t.Fatalf("point %d: got %+v, want %+v", i, p, want)
				}
			}
			if _, ok := batch["missing"]; ok {
				t.Fatal("invented missing resource")
			}
		})
	}
	// On a short raw-preferred window the real raw sample wins, rather than
	// its encompassing minute/hour buckets. With no raw in a minute, retain it.
	short, err := store.Query("node", "a", "cpu", base, base.Add(4*time.Minute), 0)
	if err != nil {
		t.Fatal(err)
	}
	if len(short) != 4 || short[0].Value != 999 || short[2].Value != 30 {
		t.Fatalf("raw precedence: %+v", short)
	}
	// An aggregate outside the requested timestamp window cannot suppress a
	// raw point inside it, even when they share a nominal retention bucket.
	clipped, err := store.Query("node", "a", "cpu", base.Add(10*time.Second), base.Add(20*time.Second), 0)
	if err != nil || len(clipped) != 1 || clipped[0].Value != 999 {
		t.Fatalf("clipped boundary: %+v, %v", clipped, err)
	}
}

func TestStoreRetainedDailyAndHourlyCoverage(t *testing.T) {
	db := newPlanTestDB(t)
	store := &Store{db: pdb.Wrap(db, "tier-coverage-test")}
	base := time.Date(2026, 9, 5, 0, 0, 0, 0, time.UTC)
	for _, row := range []struct {
		tier   Tier
		offset time.Duration
		value  float64
	}{
		{TierDaily, -24 * time.Hour, 10}, {TierHourly, -23 * time.Hour, 999},
		{TierHourly, 0, 20}, {TierMinute, 30 * time.Minute, 999},
		{TierMinute, time.Hour, 30}, {TierRaw, time.Hour + 20*time.Second, 999},
		{TierRaw, time.Hour + time.Minute, 40},
	} {
		_, err := db.Exec(`INSERT INTO metrics(resource_type,resource_id,metric_type,tier,timestamp,value) VALUES ('node','a','cpu',?,?,?)`, row.tier, base.Add(row.offset).Unix(), row.value)
		if err != nil {
			t.Fatal(err)
		}
	}
	got, err := store.Query("node", "a", "cpu", base.Add(-30*24*time.Hour), base.Add(2*time.Hour), 0)
	if err != nil {
		t.Fatal(err)
	}
	if len(got) != 4 {
		t.Fatalf("coverage: %+v", got)
	}
	for i, p := range got {
		if p.Value != float64((i+1)*10) {
			t.Fatalf("precedence: %+v", got)
		}
	}
}

func TestStoreRollupPreservesRetainedExtrema(t *testing.T) {
	for _, legacy := range []bool{false, true} {
		t.Run(fmt.Sprintf("legacy_%v", legacy), func(t *testing.T) {
			db := newPlanTestDB(t)
			store := &Store{db: pdb.Wrap(db, "tier-coverage-test")}
			base := time.Date(2026, 9, 5, 0, 0, 0, 0, time.UTC)
			_, err := db.Exec(`INSERT INTO metrics(resource_type,resource_id,metric_type,tier,timestamp,value,min_value,max_value) VALUES ('node','a','cpu','minute',?,10,1,99),('node','a','cpu','minute',?,20,5,35)`, base.Unix(), base.Add(time.Minute).Unix())
			if err != nil {
				t.Fatal(err)
			}
			if legacy {
				store.rollupCandidate("node", "a", "cpu", TierMinute, TierHourly, 3600, base.Unix(), base.Add(time.Hour).Unix())
			} else if !store.rollupTierWindow(TierMinute, TierHourly, 3600, base.Unix(), base.Add(time.Hour).Unix()) {
				t.Fatal("rollup failed")
			}
			var value, low, high float64
			if err := db.QueryRow(`SELECT value,min_value,max_value FROM metrics WHERE tier='hourly'`).Scan(&value, &low, &high); err != nil {
				t.Fatal(err)
			}
			if value != 15 || low != 1 || high != 99 {
				t.Fatalf("lost retained extrema: %v %v %v", value, low, high)
			}
		})
	}
}

func TestRetainedReadStatementsReadCurrentSnapshot(t *testing.T) {
	for _, step := range []int64{0, 60} {
		t.Run(fmt.Sprint(step), func(t *testing.T) {
			db := newPlanTestDB(t)
			db.SetMaxOpenConns(1)
			store := &Store{db: pdb.Wrap(db, "presence-snapshot")}
			end := time.Unix(2000000040, 0)
			start := end.Add(-3 * time.Hour)
			query := func(id string) []MetricPoint {
				t.Helper()
				points, err := store.Query("node", id, "cpu", start, end, step)
				if err != nil {
					t.Fatal(err)
				}
				return points
			}
			if points := query("a"); len(points) != 0 {
				t.Fatalf("empty inventory: %+v", points)
			}
			// Warm the fallback query while the preferred minute tier is absent.
			// A later preferred bucket must replace its overlapping raw point,
			// even when the same SQL template and statement are reused.
			if _, err := db.Exec(`INSERT INTO metrics(resource_type,resource_id,metric_type,tier,timestamp,value) VALUES ('node','a','cpu','raw',?,23)`, end.Add(-40*time.Second).Unix()); err != nil {
				t.Fatal(err)
			}
			if points := query("a"); len(points) != 1 || points[0].Value != 23 {
				t.Fatalf("raw fallback was hidden: %+v", points)
			}
			if _, err := db.Exec(`INSERT INTO metrics(resource_type,resource_id,metric_type,tier,timestamp,value) VALUES ('node','a','cpu','minute',?,17)`, end.Add(-time.Minute).Unix()); err != nil {
				t.Fatal(err)
			}
			if points := query("a"); len(points) != 1 || points[0].Value != 17 {
				t.Fatalf("newly present tier was hidden: %+v", points)
			}
			if points := query("b"); len(points) != 0 {
				t.Fatalf("statement reused another resource's result: %+v", points)
			}
			if _, err := db.Exec("DELETE FROM metrics"); err != nil {
				t.Fatal(err)
			}
			if points := query("a"); len(points) != 0 {
				t.Fatalf("removed history remained present: %+v", points)
			}
			// Vary the parameter-count shape beyond the bound. Uncached shapes must
			// remain functional without expanding the retained statement set.
			for n := 1; n <= maxRetainedReadStatements+2; n++ {
				ids := make([]string, n)
				for i := range ids {
					ids[i] = fmt.Sprintf("missing-%d", i)
				}
				got, err := store.QueryAllBatch("node", ids, start, end, step)
				if err != nil || len(got) != 0 {
					t.Fatalf("shape %d: %+v, %v", n, got, err)
				}
			}
			if len(store.readStatements) > maxRetainedReadStatements {
				t.Fatalf("unbounded compiled SQL retention: %d", len(store.readStatements))
			}
			if len(store.readQueryShapes) > maxRetainedReadStatements {
				t.Fatalf("unbounded SQL template retention: %d", len(store.readQueryShapes))
			}
		})
	}
}

func TestRetainedQueryTemplatesBindCurrentWindowAndStep(t *testing.T) {
	db := newPlanTestDB(t)
	db.SetMaxOpenConns(1)
	store := &Store{db: pdb.Wrap(db, "query-template-bindings")}
	base := time.Date(2026, 9, 6, 0, 0, 0, 0, time.UTC)
	for _, row := range []struct {
		family, metric string
		offset         time.Duration
		value          float64
	}{{"node", "cpu", 10 * time.Second, 5}, {"node", "cpu", 70 * time.Second, 15}, {"vm", "cpu", 70 * time.Second, 91}, {"node", "memory", 70 * time.Second, 99}} {
		if _, err := db.Exec(`INSERT INTO metrics(resource_type,resource_id,metric_type,tier,timestamp,value) VALUES (?,'a',?,'raw',?,?)`, row.family, row.metric, base.Add(row.offset).Unix(), row.value); err != nil {
			t.Fatal(err)
		}
	}
	for _, tc := range []struct {
		family, metric string
		start          time.Duration
		step           int64
		values         []float64
	}{{"node", "cpu", 0, 60, []float64{5, 15}}, {"node", "cpu", 0, 120, []float64{10}}, {"node", "cpu", time.Minute, 120, []float64{15}}, {"vm", "cpu", 0, 120, []float64{91}}, {"node", "memory", 0, 120, []float64{99}}} {
		points, err := store.Query(tc.family, "a", tc.metric, base.Add(tc.start), base.Add(3*time.Minute), tc.step)
		if err != nil {
			t.Fatal(err)
		}
		var values []float64
		for _, point := range points {
			values = append(values, point.Value)
		}
		if !reflect.DeepEqual(values, tc.values) {
			t.Fatalf("%+v returned stale bindings: %+v", tc, points)
		}
	}
}

// Extrema can be missing independently, including after a row with explicit
// zero bounds. Every retained read shape must apply the fallback per point.
func TestStoreRetainedNullableExtrema(t *testing.T) {
	db := newPlanTestDB(t)
	store := &Store{db: pdb.Wrap(db, "nullable-extrema-test")}
	base := time.Date(2026, 9, 5, 12, 0, 0, 0, time.UTC)
	type observation struct {
		value             float64
		low, high         interface{}
		wantLow, wantHigh float64
	}
	observations := []observation{
		{10, -2.0, 99.0, -2, 99},
		{20, nil, nil, 20, 20},
		{30, 0.0, 0.0, 0, 0},
		{-10, nil, -3.0, -10, -3},
		{50, 2.0, nil, 2, 50},
		{60, nil, nil, 60, 60},
	}
	for _, id := range []string{"a", "b"} {
		for _, metric := range []string{"cpu", "memory"} {
			for i, row := range observations {
				// Alternate non-overlapping tiers so single-resource buckets must
				// also normalize NULL bounds across the reconciliation UNION.
				tier := TierMinute
				if i%2 != 0 {
					tier = TierRaw
				}
				if _, err := db.Exec(`INSERT INTO metrics(resource_type,resource_id,metric_type,tier,timestamp,value,min_value,max_value) VALUES ('node',?,?,?,?,?,?,?)`, id, metric, tier, base.Add(time.Duration(i)*time.Minute).Unix(), row.value, row.low, row.high); err != nil {
					t.Fatal(err)
				}
			}
		}
	}
	for _, step := range []int64{0, 120} {
		for _, ids := range [][]string{{"a"}, {"a", "b"}} {
			for _, filter := range [][]string{nil, {"cpu"}, {"cpu", "memory"}} {
				t.Run(fmt.Sprintf("step=%d/ids=%d/filter=%d", step, len(ids), len(filter)), func(t *testing.T) {
					got, err := store.QueryMetricTypesBatch("node", ids, filter, base.Add(-2*time.Hour), base.Add(6*time.Minute), step)
					if err != nil {
						t.Fatal(err)
					}
					values, err := store.QueryMetricValuesBatch("node", ids, filter, base.Add(-2*time.Hour), base.Add(6*time.Minute), step)
					if err != nil {
						t.Fatal(err)
					}
					assertMetricValueProjection(t, got, values)
					for _, id := range ids {
						metrics := filter
						if len(metrics) == 0 {
							metrics = []string{"cpu", "memory"}
						}
						for _, metric := range metrics {
							want := make([]MetricPoint, 0, len(observations))
							for i := 0; i < len(observations); {
								row := observations[i]
								point := MetricPoint{Timestamp: base.Add(time.Duration(i) * time.Minute), Value: row.value, Min: row.wantLow, Max: row.wantHigh}
								i++
								if step > 1 {
									next := observations[i]
									point.Timestamp = point.Timestamp.Add(time.Minute)
									point.Value = (point.Value + next.value) / 2
									point.Min = min(point.Min, next.wantLow)
									point.Max = max(point.Max, next.wantHigh)
									i++
								}
								want = append(want, point)
							}
							points := got[id][metric]
							if len(points) != len(want) {
								t.Fatalf("%s/%s: got %+v, want %+v", id, metric, points, want)
							}
							for i, point := range points {
								expected := want[i]
								if !point.Timestamp.Equal(expected.Timestamp) || point.Value != expected.Value || point.Min != expected.Min || point.Max != expected.Max {
									t.Fatalf("%s/%s point %d: got %+v, want %+v", id, metric, i, point, expected)
								}
							}
						}
					}
				})
			}
		}
	}
}