Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
package metrics
import (
"fmt"
pdb "github.com/rcourtman/pulse-go-rewrite/pkg/db"
"reflect"
"testing"
"time"
)
func TestStoreRetainedTierCoverage(t *testing.T) {
db := newPlanTestDB(t)
store := &Store{db: pdb.Wrap(db, "tier-coverage-test")}
base := time.Date(2026, 9, 5, 12, 0, 0, 0, time.UTC)
insert := func(id, metric string, tier Tier, offset time.Duration, value, low, high float64) {
t.Helper()
_, err := db.Exec(`INSERT INTO metrics(resource_type,resource_id,metric_type,tier,timestamp,value,min_value,max_value) VALUES ('node',?,?,?,?,?,?,?)`, id, metric, tier, base.Add(offset).Unix(), value, low, high)
if err != nil {
t.Fatal(err)
}
}
// Raw observations before, within and after aggregate buckets. The raw
// overlap deliberately differs so double-counting is visible in the mean.
insert("a", "cpu", TierMinute, 0, 10, 2, 90)
insert("a", "cpu", TierMinute, 2*time.Minute, 30, 25, 40)
insert("a", "cpu", TierRaw, -time.Minute, 5, 5, 5)
insert("a", "cpu", TierRaw, 20*time.Second, 999, 999, 999)
insert("a", "cpu", TierRaw, time.Minute, 20, 20, 20)
insert("a", "cpu", TierRaw, 3*time.Minute, 50, 50, 50)
insert("a", "memory", TierRaw, 3*time.Minute, 88, 88, 88)
// The older hourly fallback remains useful, but the overlapping hour is
// indivisible and must not be mixed with the preferred minute/raw evidence.
insert("a", "cpu", TierHourly, -2*time.Hour, 7, 1, 9)
insert("a", "cpu", TierHourly, 0, 777, 777, 777)
insert("b", "cpu", TierRaw, 20*time.Second, 60, 60, 60)
insert("b", "memory", TierMinute, 0, 70, 65, 75)
insert("b", "memory", TierRaw, 3*time.Minute, 80, 80, 80)
// Same ID in another resource family cannot shadow or contribute evidence.
_, err := db.Exec(`INSERT INTO metrics(resource_type,resource_id,metric_type,tier,timestamp,value) VALUES ('vm','a','cpu','minute',?,1234)`, base.Add(time.Minute).Unix())
if err != nil {
t.Fatal(err)
}
start, end := base.Add(-23*time.Hour), base.Add(4*time.Minute)
want := []MetricPoint{
{base.Add(-2 * time.Hour), 7, 1, 9}, {base.Add(-time.Minute), 5, 5, 5},
{base, 10, 2, 90}, {base.Add(time.Minute), 20, 20, 20},
{base.Add(2 * time.Minute), 30, 25, 40}, {base.Add(3 * time.Minute), 50, 50, 50},
}
for _, step := range []int64{0, 120} {
t.Run(fmt.Sprintf("step_%d", step), func(t *testing.T) {
batch, err := store.QueryAllBatch("node", []string{"a", "b", "a", "missing"}, start, end, step)
if err != nil {
t.Fatal(err)
}
filtered, err := store.QueryMetricTypesBatch("node", []string{"a", "b"}, []string{"cpu"}, start, end, step)
if err != nil {
t.Fatal(err)
}
filteredMany, err := store.QueryMetricTypesBatch("node", []string{"a", "b"}, []string{"memory", "cpu", "cpu"}, start, end, step)
if err != nil {
t.Fatal(err)
}
for _, id := range []string{"a", "b"} {
all, err := store.QueryAll("node", id, start, end, step)
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(all, batch[id]) {
t.Fatalf("batch differs for %s: %v != %v", id, batch[id], all)
}
if !reflect.DeepEqual(filteredMany[id], all) {
t.Fatalf("multi-metric filter changed ordering or retained values for %s: %v != %v", id, filteredMany[id], all)
}
for _, metric := range []string{"cpu", "memory"} {
single, err := store.Query("node", id, metric, start, end, step)
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(single, all[metric]) {
t.Fatalf("single differs for %s/%s", id, metric)
}
}
if len(filtered[id]) != 1 || !reflect.DeepEqual(filtered[id]["cpu"], all["cpu"]) {
t.Fatalf("filtered mismatch: %+v", filtered)
}
}
got := batch["a"]["cpu"]
expected := want
if step == 120 {
expected = []MetricPoint{{base.Add(-119 * time.Minute), 7, 1, 9}, {base.Add(-time.Minute), 5, 5, 5}, {base.Add(time.Minute), 15, 2, 90}, {base.Add(3 * time.Minute), 40, 25, 50}}
}
if len(got) != len(expected) {
t.Fatalf("got %+v\nwant %+v", got, expected)
}
for i, p := range got {
want := expected[i]
if !p.Timestamp.Equal(want.Timestamp) || p.Value != want.Value || p.Min != want.Min || p.Max != want.Max {
t.Fatalf("point %d: got %+v, want %+v", i, p, want)
}
}
if _, ok := batch["missing"]; ok {
t.Fatal("invented missing resource")
}
})
}
// On a short raw-preferred window the real raw sample wins, rather than
// its encompassing minute/hour buckets. With no raw in a minute, retain it.
short, err := store.Query("node", "a", "cpu", base, base.Add(4*time.Minute), 0)
if err != nil {
t.Fatal(err)
}
if len(short) != 4 || short[0].Value != 999 || short[2].Value != 30 {
t.Fatalf("raw precedence: %+v", short)
}
// An aggregate outside the requested timestamp window cannot suppress a
// raw point inside it, even when they share a nominal retention bucket.
clipped, err := store.Query("node", "a", "cpu", base.Add(10*time.Second), base.Add(20*time.Second), 0)
if err != nil || len(clipped) != 1 || clipped[0].Value != 999 {
t.Fatalf("clipped boundary: %+v, %v", clipped, err)
}
}
func TestStoreRetainedDailyAndHourlyCoverage(t *testing.T) {
db := newPlanTestDB(t)
store := &Store{db: pdb.Wrap(db, "tier-coverage-test")}
base := time.Date(2026, 9, 5, 0, 0, 0, 0, time.UTC)
for _, row := range []struct {
tier Tier
offset time.Duration
value float64
}{
{TierDaily, -24 * time.Hour, 10}, {TierHourly, -23 * time.Hour, 999},
{TierHourly, 0, 20}, {TierMinute, 30 * time.Minute, 999},
{TierMinute, time.Hour, 30}, {TierRaw, time.Hour + 20*time.Second, 999},
{TierRaw, time.Hour + time.Minute, 40},
} {
_, err := db.Exec(`INSERT INTO metrics(resource_type,resource_id,metric_type,tier,timestamp,value) VALUES ('node','a','cpu',?,?,?)`, row.tier, base.Add(row.offset).Unix(), row.value)
if err != nil {
t.Fatal(err)
}
}
got, err := store.Query("node", "a", "cpu", base.Add(-30*24*time.Hour), base.Add(2*time.Hour), 0)
if err != nil {
t.Fatal(err)
}
if len(got) != 4 {
t.Fatalf("coverage: %+v", got)
}
for i, p := range got {
if p.Value != float64((i+1)*10) {
t.Fatalf("precedence: %+v", got)
}
}
}
func TestStoreRollupPreservesRetainedExtrema(t *testing.T) {
for _, legacy := range []bool{false, true} {
t.Run(fmt.Sprintf("legacy_%v", legacy), func(t *testing.T) {
db := newPlanTestDB(t)
store := &Store{db: pdb.Wrap(db, "tier-coverage-test")}
base := time.Date(2026, 9, 5, 0, 0, 0, 0, time.UTC)
_, err := db.Exec(`INSERT INTO metrics(resource_type,resource_id,metric_type,tier,timestamp,value,min_value,max_value) VALUES ('node','a','cpu','minute',?,10,1,99),('node','a','cpu','minute',?,20,5,35)`, base.Unix(), base.Add(time.Minute).Unix())
if err != nil {
t.Fatal(err)
}
if legacy {
store.rollupCandidate("node", "a", "cpu", TierMinute, TierHourly, 3600, base.Unix(), base.Add(time.Hour).Unix())
} else if !store.rollupTierWindow(TierMinute, TierHourly, 3600, base.Unix(), base.Add(time.Hour).Unix()) {
t.Fatal("rollup failed")
}
var value, low, high float64
if err := db.QueryRow(`SELECT value,min_value,max_value FROM metrics WHERE tier='hourly'`).Scan(&value, &low, &high); err != nil {
t.Fatal(err)
}
if value != 15 || low != 1 || high != 99 {
t.Fatalf("lost retained extrema: %v %v %v", value, low, high)
}
})
}
}
func TestRetainedReadStatementsReadCurrentSnapshot(t *testing.T) {
for _, step := range []int64{0, 60} {
t.Run(fmt.Sprint(step), func(t *testing.T) {
db := newPlanTestDB(t)
db.SetMaxOpenConns(1)
store := &Store{db: pdb.Wrap(db, "presence-snapshot")}
end := time.Unix(2000000040, 0)
start := end.Add(-3 * time.Hour)
query := func(id string) []MetricPoint {
t.Helper()
points, err := store.Query("node", id, "cpu", start, end, step)
if err != nil {
t.Fatal(err)
}
return points
}
if points := query("a"); len(points) != 0 {
t.Fatalf("empty inventory: %+v", points)
}
// Warm the fallback query while the preferred minute tier is absent.
// A later preferred bucket must replace its overlapping raw point,
// even when the same SQL template and statement are reused.
if _, err := db.Exec(`INSERT INTO metrics(resource_type,resource_id,metric_type,tier,timestamp,value) VALUES ('node','a','cpu','raw',?,23)`, end.Add(-40*time.Second).Unix()); err != nil {
t.Fatal(err)
}
if points := query("a"); len(points) != 1 || points[0].Value != 23 {
t.Fatalf("raw fallback was hidden: %+v", points)
}
if _, err := db.Exec(`INSERT INTO metrics(resource_type,resource_id,metric_type,tier,timestamp,value) VALUES ('node','a','cpu','minute',?,17)`, end.Add(-time.Minute).Unix()); err != nil {
t.Fatal(err)
}
if points := query("a"); len(points) != 1 || points[0].Value != 17 {
t.Fatalf("newly present tier was hidden: %+v", points)
}
if points := query("b"); len(points) != 0 {
t.Fatalf("statement reused another resource's result: %+v", points)
}
if _, err := db.Exec("DELETE FROM metrics"); err != nil {
t.Fatal(err)
}
if points := query("a"); len(points) != 0 {
t.Fatalf("removed history remained present: %+v", points)
}
// Vary the parameter-count shape beyond the bound. Uncached shapes must
// remain functional without expanding the retained statement set.
for n := 1; n <= maxRetainedReadStatements+2; n++ {
ids := make([]string, n)
for i := range ids {
ids[i] = fmt.Sprintf("missing-%d", i)
}
got, err := store.QueryAllBatch("node", ids, start, end, step)
if err != nil || len(got) != 0 {
t.Fatalf("shape %d: %+v, %v", n, got, err)
}
}
if len(store.readStatements) > maxRetainedReadStatements {
t.Fatalf("unbounded compiled SQL retention: %d", len(store.readStatements))
}
if len(store.readQueryShapes) > maxRetainedReadStatements {
t.Fatalf("unbounded SQL template retention: %d", len(store.readQueryShapes))
}
})
}
}
func TestRetainedQueryTemplatesBindCurrentWindowAndStep(t *testing.T) {
db := newPlanTestDB(t)
db.SetMaxOpenConns(1)
store := &Store{db: pdb.Wrap(db, "query-template-bindings")}
base := time.Date(2026, 9, 6, 0, 0, 0, 0, time.UTC)
for _, row := range []struct {
family, metric string
offset time.Duration
value float64
}{{"node", "cpu", 10 * time.Second, 5}, {"node", "cpu", 70 * time.Second, 15}, {"vm", "cpu", 70 * time.Second, 91}, {"node", "memory", 70 * time.Second, 99}} {
if _, err := db.Exec(`INSERT INTO metrics(resource_type,resource_id,metric_type,tier,timestamp,value) VALUES (?,'a',?,'raw',?,?)`, row.family, row.metric, base.Add(row.offset).Unix(), row.value); err != nil {
t.Fatal(err)
}
}
for _, tc := range []struct {
family, metric string
start time.Duration
step int64
values []float64
}{{"node", "cpu", 0, 60, []float64{5, 15}}, {"node", "cpu", 0, 120, []float64{10}}, {"node", "cpu", time.Minute, 120, []float64{15}}, {"vm", "cpu", 0, 120, []float64{91}}, {"node", "memory", 0, 120, []float64{99}}} {
points, err := store.Query(tc.family, "a", tc.metric, base.Add(tc.start), base.Add(3*time.Minute), tc.step)
if err != nil {
t.Fatal(err)
}
var values []float64
for _, point := range points {
values = append(values, point.Value)
}
if !reflect.DeepEqual(values, tc.values) {
t.Fatalf("%+v returned stale bindings: %+v", tc, points)
}
}
}
// Extrema can be missing independently, including after a row with explicit
// zero bounds. Every retained read shape must apply the fallback per point.
func TestStoreRetainedNullableExtrema(t *testing.T) {
db := newPlanTestDB(t)
store := &Store{db: pdb.Wrap(db, "nullable-extrema-test")}
base := time.Date(2026, 9, 5, 12, 0, 0, 0, time.UTC)
type observation struct {
value float64
low, high interface{}
wantLow, wantHigh float64
}
observations := []observation{
{10, -2.0, 99.0, -2, 99},
{20, nil, nil, 20, 20},
{30, 0.0, 0.0, 0, 0},
{-10, nil, -3.0, -10, -3},
{50, 2.0, nil, 2, 50},
{60, nil, nil, 60, 60},
}
for _, id := range []string{"a", "b"} {
for _, metric := range []string{"cpu", "memory"} {
for i, row := range observations {
// Alternate non-overlapping tiers so single-resource buckets must
// also normalize NULL bounds across the reconciliation UNION.
tier := TierMinute
if i%2 != 0 {
tier = TierRaw
}
if _, err := db.Exec(`INSERT INTO metrics(resource_type,resource_id,metric_type,tier,timestamp,value,min_value,max_value) VALUES ('node',?,?,?,?,?,?,?)`, id, metric, tier, base.Add(time.Duration(i)*time.Minute).Unix(), row.value, row.low, row.high); err != nil {
t.Fatal(err)
}
}
}
}
for _, step := range []int64{0, 120} {
for _, ids := range [][]string{{"a"}, {"a", "b"}} {
for _, filter := range [][]string{nil, {"cpu"}, {"cpu", "memory"}} {
t.Run(fmt.Sprintf("step=%d/ids=%d/filter=%d", step, len(ids), len(filter)), func(t *testing.T) {
got, err := store.QueryMetricTypesBatch("node", ids, filter, base.Add(-2*time.Hour), base.Add(6*time.Minute), step)
if err != nil {
t.Fatal(err)
}
values, err := store.QueryMetricValuesBatch("node", ids, filter, base.Add(-2*time.Hour), base.Add(6*time.Minute), step)
if err != nil {
t.Fatal(err)
}
assertMetricValueProjection(t, got, values)
for _, id := range ids {
metrics := filter
if len(metrics) == 0 {
metrics = []string{"cpu", "memory"}
}
for _, metric := range metrics {
want := make([]MetricPoint, 0, len(observations))
for i := 0; i < len(observations); {
row := observations[i]
point := MetricPoint{Timestamp: base.Add(time.Duration(i) * time.Minute), Value: row.value, Min: row.wantLow, Max: row.wantHigh}
i++
if step > 1 {
next := observations[i]
point.Timestamp = point.Timestamp.Add(time.Minute)
point.Value = (point.Value + next.value) / 2
point.Min = min(point.Min, next.wantLow)
point.Max = max(point.Max, next.wantHigh)
i++
}
want = append(want, point)
}
points := got[id][metric]
if len(points) != len(want) {
t.Fatalf("%s/%s: got %+v, want %+v", id, metric, points, want)
}
for i, point := range points {
expected := want[i]
if !point.Timestamp.Equal(expected.Timestamp) || point.Value != expected.Value || point.Min != expected.Min || point.Max != expected.Max {
t.Fatalf("%s/%s point %d: got %+v, want %+v", id, metric, i, point, expected)
}
}
}
}
})
}
}
}
}