Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

19.0 KB
Raw Blame
package metrics

import (
	"context"
	"database/sql"
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"reflect"
	"strings"
	"sync"
	"testing"
	"time"

	pdb "github.com/rcourtman/pulse-go-rewrite/pkg/db"
)

// Pause deferred maintenance, not the constructor, so tests can exercise the
// installed legacy schema and the cutover without racing a background upgrade.
func clusteredLegacyStore(t *testing.T) (*Store, func()) {
	t.Helper()
	suppressTestLogs(t)
	dbPath := filepath.Join(t.TempDir(), "metrics.db")
	issue1124CreateLegacyStore(t, dbPath, 100, false)
	entered, release := make(chan struct{}), make(chan struct{})
	previous := startupMaintenanceHook
	startupMaintenanceHook = func() { close(entered); <-release }
	cfg := DefaultConfig(filepath.Dir(dbPath))
	cfg.DBPath = dbPath
	cfg.FlushInterval, cfg.RollupInterval = time.Hour, time.Hour
	cfg.RetentionRaw = 10 * 365 * 24 * time.Hour
	store, err := NewStore(cfg)
	startupMaintenanceHook = previous
	var once sync.Once
	unblock := func() { once.Do(func() { close(release) }) }
	if err != nil {
		unblock()
		t.Fatal(err)
	}
	t.Cleanup(func() { unblock(); _ = store.Close() })
	select {
	case <-entered:
	case <-time.After(5 * time.Second):
		t.Fatal("deferred maintenance did not start")
	}
	return store, unblock
}

func assertClusteredStore(t *testing.T, store *Store) {
	t.Helper()
	clustered, err := metricsTableIsClustered(context.Background(), store.db.DB)
	if err != nil || !clustered {
		t.Fatalf("clustered schema=%t, error=%v", clustered, err)
	}
	issue1124AssertConsolidatedIndexes(t, store)
	var integrity string
	if err := store.db.QueryRow(`PRAGMA integrity_check`).Scan(&integrity); err != nil || integrity != "ok" {
		t.Fatalf("integrity=%q, error=%v", integrity, err)
	}
}

func TestClusteredMetricsRollupPlanUsesBoundedPrimaryRanges(t *testing.T) {
	store, err := NewStore(DefaultConfig(t.TempDir()))
	if err != nil {
		t.Fatal(err)
	}
	t.Cleanup(func() { _ = store.Close() })
	rows, err := store.db.Query("EXPLAIN QUERY PLAN "+clusteredMetricsRollupSQL,
		int64(60), int64(60), "minute", "raw", int64(1_700_000_000), int64(1_700_000_120))
	if err != nil {
		t.Fatal(err)
	}
	defer rows.Close()
	var plan []string
	for rows.Next() {
		var id, parent, unused int
		var detail string
		if err := rows.Scan(&id, &parent, &unused, &detail); err != nil {
			t.Fatal(err)
		}
		plan = append(plan, detail)
	}
	if err := rows.Err(); err != nil {
		t.Fatal(err)
	}
	joined := strings.Join(plan, "\n")
	for _, want := range []string{
		"SEARCH metrics USING COVERING INDEX idx_metrics_tier_time (tier=? AND timestamp>? AND timestamp<?)",
		"SEARCH m USING PRIMARY KEY (resource_type=? AND resource_id=? AND tier=? AND timestamp>? AND timestamp<?)",
	} {
		if !strings.Contains(joined, want) {
			t.Fatalf("rollup lost its bounded covering discovery/primary range: missing %q in\n%s", want, joined)
		}
	}
	if strings.Contains(joined, "SCAN metrics") || strings.Contains(joined, "SCAN m ") {
		t.Fatalf("rollup scans observations outside its indexed window:\n%s", joined)
	}
}

func TestClusteredMetricsRollupsMatchLegacyAggregation(t *testing.T) {
	suppressLogs(t)
	stores := make([]*Store, 2)
	for i := range stores {
		cfg := DefaultConfig(t.TempDir())
		cfg.FlushInterval, cfg.RollupInterval = time.Hour, time.Hour
		store, err := NewStore(cfg)
		if err != nil {
			t.Fatal(err)
		}
		t.Cleanup(func() { _ = store.Close() })
		if err := store.WaitForMaintenance(30 * time.Second); err != nil {
			t.Fatal(err)
		}
		if i == 0 {
			issue1124InstallRowidSchema(t, store)
		}
		stores[i] = store
	}
	base := int64(1_700_006_400) // Aligned to a day, hour and minute.
	for _, store := range stores {
		for _, resourceType := range []string{"vm", "node", "dockercontainer"} {
			for _, resourceID := range []string{"same-id", "same-id:child", "z"} {
				for _, metric := range []string{"cpu", "memory", "bytes_total"} {
					for i, offset := range []int64{-1, 0, 5, 59, 60, 65, 119, 120} {
						var low, high any
						if i%2 == 0 {
							low = 0.0 // Explicit zero must remain a bound.
						}
						if i%3 == 0 {
							high = 60.0
						}
						if _, err := store.db.Exec(`INSERT INTO metrics
							(resource_type,resource_id,metric_type,value,min_value,max_value,timestamp,tier)
							VALUES (?,?,?,?,?,?,?,'raw')`, resourceType, resourceID, metric,
							float64(i*6), low, high, base+offset); err != nil {
							t.Fatal(err)
						}
					}
				}
			}
		}
		// A completed output is authoritative. Re-running a window must retain it.
		if _, err := store.db.Exec(`INSERT INTO metrics
			(resource_type,resource_id,metric_type,value,min_value,max_value,timestamp,tier)
			VALUES ('vm','same-id','cpu',900,0,1000,?,'minute')`, base); err != nil {
			t.Fatal(err)
		}
	}
	read := func(store *Store) []string {
		t.Helper()
		rows, err := store.db.Query(`SELECT resource_type,resource_id,metric_type,
			quote(value),quote(min_value),quote(max_value),timestamp,tier
			FROM metrics ORDER BY resource_type,resource_id,tier,timestamp,metric_type`)
		if err != nil {
			t.Fatal(err)
		}
		defer rows.Close()
		var result []string
		for rows.Next() {
			var rt, id, metric, value, low, high, tier string
			var ts int64
			if err := rows.Scan(&rt, &id, &metric, &value, &low, &high, &ts, &tier); err != nil {
				t.Fatal(err)
			}
			result = append(result, fmt.Sprint(rt, id, metric, value, low, high, ts, tier))
		}
		if err := rows.Err(); err != nil {
			t.Fatal(err)
		}
		return result
	}
	for _, stage := range []struct {
		from, to Tier
		bucket   int64
		end      int64
	}{{TierRaw, TierMinute, 60, base + 120}, {TierMinute, TierHourly, 3600, base + 3600}, {TierHourly, TierDaily, 86400, base + 86400}} {
		for repeat := 0; repeat < 2; repeat++ {
			for _, store := range stores {
				if !store.rollupTierWindow(stage.from, stage.to, stage.bucket, base, stage.end) {
					t.Fatalf("%s -> %s rollup failed", stage.from, stage.to)
				}
			}
			if old, current := read(stores[0]), read(stores[1]); !reflect.DeepEqual(old, current) {
				t.Fatalf("%s -> %s changed nullable bounds, identity, bucket edges or idempotence\nlegacy=%v\nclustered=%v", stage.from, stage.to, old, current)
			}
		}
	}
}

func TestClusteredMetricsFreshAndRestart(t *testing.T) {
	cfg := DefaultConfig(t.TempDir())
	for i := 0; i < 2; i++ {
		store, err := NewStore(cfg)
		if err != nil {
			t.Fatal(err)
		}
		if err := store.WaitForMaintenance(30 * time.Second); err != nil {
			t.Fatal(err)
		}
		assertClusteredStore(t, store)
		if err := store.Close(); err != nil {
			t.Fatal(err)
		}
	}
}

func TestClusteredMetricsPreservesObservationsAndMetadata(t *testing.T) {
	store, _ := clusteredLegacyStore(t)
	base := time.Unix(1_700_000_000, 0)
	for i, tier := range []Tier{TierRaw, TierMinute, TierHourly, TierDaily} {
		for j, resource := range []string{"vm", "dockercontainer"} {
			for k, bounds := range [][2]any{{nil, nil}, {0.0, nil}, {nil, 0.0}, {-4.0, 12.0}} {
				_, err := store.db.Exec(`INSERT INTO metrics
					(resource_type, resource_id, metric_type, value, min_value, max_value, timestamp, tier)
					VALUES (?, ?, 'cpu', ?, ?, ?, ?, ?)`, resource, fmt.Sprintf("case-%d", j),
					float64(k+1), bounds[0], bounds[1], base.Add(time.Duration(i*86400+k*60)*time.Second).Unix(), tier)
				if err != nil {
					t.Fatal(err)
				}
			}
		}
	}
	if _, err := store.db.Exec(`INSERT INTO metrics_meta VALUES ('rollup_sentinel', '1700000000')`); err != nil {
		t.Fatal(err)
	}
	// The exact stored nullable bounds, observation names, tiers and values
	// must survive, independently of any reader's normalization or fallback.
	read := func() []string {
		rows, err := store.db.Query(`SELECT resource_type, resource_id, metric_type,
			quote(value), quote(min_value), quote(max_value), timestamp, tier
			FROM metrics ORDER BY resource_type, resource_id, tier, timestamp, metric_type`)
		if err != nil {
			t.Fatal(err)
		}
		defer rows.Close()
		var result []string
		for rows.Next() {
			var resource, id, metric, value, low, high, tier string
			var ts int64
			if err := rows.Scan(&resource, &id, &metric, &value, &low, &high, &ts, &tier); err != nil {
				t.Fatal(err)
			}
			result = append(result, fmt.Sprint(resource, id, metric, value, low, high, ts, tier))
		}
		if err := rows.Err(); err != nil {
			t.Fatal(err)
		}
		return result
	}
	before := read()
	if err := store.migrateClusteredMetrics(context.Background()); err != nil {
		t.Fatal(err)
	}
	if after := read(); !reflect.DeepEqual(before, after) {
		t.Fatalf("stored observations changed across clustered migration")
	}
	var sentinel string
	if err := store.db.QueryRow(`SELECT value FROM metrics_meta WHERE key='rollup_sentinel'`).Scan(&sentinel); err != nil || sentinel != "1700000000" {
		t.Fatalf("rollup metadata=%q, error=%v", sentinel, err)
	}
	assertClusteredStore(t, store)
}

func TestClusteredMetricsReadersAndQueuedWritesSurviveCutover(t *testing.T) {
	store, _ := clusteredLegacyStore(t)
	entered, release := make(chan struct{}), make(chan struct{})
	var once sync.Once
	unblock := func() { once.Do(func() { close(release) }) }
	defer unblock()
	store.clusteredMigrationHook = func(stage string) {
		if stage == "after-copy" {
			close(entered)
			<-release
		}
	}
	// An already open SQLite snapshot must retain its original schema while
	// the other connection swaps and drops the physical table.
	tx, err := store.db.BeginTx(context.Background(), &sql.TxOptions{ReadOnly: true})
	if err != nil {
		t.Fatal(err)
	}
	defer tx.Rollback()
	var count int
	if err := tx.QueryRow(`SELECT COUNT(*) FROM metrics INDEXED BY idx_metrics_query_all`).Scan(&count); err != nil {
		t.Fatal(err)
	}
	migrated := make(chan error, 1)
	go func() { migrated <- store.migrateClusteredMetrics(context.Background()) }()
	select {
	case <-entered:
	case <-time.After(5 * time.Second):
		t.Fatal("migration did not copy")
	}
	points, err := store.Query("vm", "legacy-0001", "cpu", time.Unix(1_699_999_999, 0), time.Unix(1_700_001_000, 0), 0)
	if err != nil || len(points) != 1 {
		t.Fatalf("reader during copy points=%d, error=%v", len(points), err)
	}
	written := make(chan struct{})
	go func() {
		store.WriteBatchSync([]WriteMetric{{ResourceType: "vm", ResourceID: "queued", MetricType: "cpu", Value: 42, Timestamp: time.Unix(1_700_020_000, 0), Tier: TierRaw}})
		close(written)
	}()
	unblock()
	if err := <-migrated; err != nil {
		t.Fatal(err)
	}
	if err := tx.QueryRow(`SELECT COUNT(*) FROM metrics INDEXED BY idx_metrics_query_all`).Scan(&count); err != nil || count != 100 {
		t.Fatalf("old read snapshot count=%d, error=%v", count, err)
	}
	_ = tx.Rollback()
	select {
	case <-written:
	case <-time.After(35 * time.Second):
		t.Fatal("queued write did not resume")
	}
	var value float64
	if err := store.db.QueryRow(`SELECT value FROM metrics WHERE resource_id='queued'`).Scan(&value); err != nil || value != 42 {
		t.Fatalf("queued write=%v, error=%v", value, err)
	}
	assertClusteredStore(t, store)
}

func TestClusteredMetricsCancellationRollsBackSwap(t *testing.T) {
	store, _ := clusteredLegacyStore(t)
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()
	store.clusteredMigrationHook = func(stage string) {
		if stage == "before-commit" {
			cancel()
		}
	}
	if err := store.migrateClusteredMetrics(ctx); err == nil {
		t.Fatal("canceled migration succeeded")
	}
	clustered, err := metricsTableIsClustered(context.Background(), store.db.DB)
	if err != nil || clustered {
		t.Fatalf("canceled swap changed layout=%t, error=%v", clustered, err)
	}
	var count int
	if err := store.db.QueryRow(`SELECT COUNT(*) FROM metrics`).Scan(&count); err != nil || count != 100 {
		t.Fatalf("rollback rows=%d, error=%v", count, err)
	}
	if err := store.db.QueryRow(`SELECT COUNT(*) FROM sqlite_schema WHERE name='metrics_clustered_upgrade'`).Scan(&count); err != nil || count != 0 {
		t.Fatalf("rollback replacement tables=%d, error=%v", count, err)
	}
	store.clusteredMigrationHook = nil
	if err := store.migrateClusteredMetrics(context.Background()); err != nil {
		t.Fatal(err)
	}
	assertClusteredStore(t, store)
}

func TestClusteredMetricsPreservesConflictingReplacementTable(t *testing.T) {
	store, _ := clusteredLegacyStore(t)
	if _, err := store.db.Exec(`CREATE TABLE metrics_clustered_upgrade(value TEXT);
		INSERT INTO metrics_clustered_upgrade VALUES ('preserve')`); err != nil {
		t.Fatal(err)
	}
	if err := store.migrateClusteredMetrics(context.Background()); err == nil {
		t.Fatal("conflicting table was overwritten")
	}
	var value string
	if err := store.db.QueryRow(`SELECT value FROM metrics_clustered_upgrade`).Scan(&value); err != nil || value != "preserve" {
		t.Fatalf("conflicting table value=%q, error=%v", value, err)
	}
	var count int
	if err := store.db.QueryRow(`SELECT COUNT(*) FROM metrics`).Scan(&count); err != nil || count != 100 {
		t.Fatalf("original rows=%d, error=%v", count, err)
	}
}

func TestClusteredMetricsCrashRollsBackSwap(t *testing.T) {
	if dbPath := os.Getenv("PULSE_CLUSTERED_CRASH_DB"); dbPath != "" {
		metricsClusteredMigrationHook = func(stage string) {
			if stage == "before-commit" {
				os.Exit(88)
			}
		}
		cfg := DefaultConfig(filepath.Dir(dbPath))
		cfg.DBPath = dbPath
		cfg.RetentionRaw = 10 * 365 * 24 * time.Hour
		store, err := NewStore(cfg)
		if err != nil {
			os.Exit(90)
		}
		_ = store.WaitForMaintenance(30 * time.Second)
		os.Exit(89)
	}
	suppressTestLogs(t)
	dbPath := filepath.Join(t.TempDir(), "metrics.db")
	issue1124CreateLegacyStore(t, dbPath, 10_000, false)
	cmd := exec.Command(os.Args[0], "-test.run=^TestClusteredMetricsCrashRollsBackSwap$")
	cmd.Env = append(os.Environ(), "PULSE_CLUSTERED_CRASH_DB="+dbPath)
	err := cmd.Run()
	if exit, ok := err.(*exec.ExitError); !ok || exit.ExitCode() != 88 {
		t.Fatalf("crash child=%v, want exit 88", err)
	}
	cfg := DefaultConfig(filepath.Dir(dbPath))
	cfg.DBPath = dbPath
	cfg.RetentionRaw = 10 * 365 * 24 * time.Hour
	store, err := NewStore(cfg)
	if err != nil {
		t.Fatal(err)
	}
	defer store.Close()
	if err := store.WaitForMaintenance(30 * time.Second); err != nil {
		t.Fatal(err)
	}
	var count int
	if err := store.db.QueryRow(`SELECT COUNT(*) FROM metrics`).Scan(&count); err != nil || count != 10_000 {
		t.Fatalf("recovered rows=%d, error=%v", count, err)
	}
	assertClusteredStore(t, store)
}

func TestClusteredMetricsRefreshesAnotherStoreAndCachedShapes(t *testing.T) {
	store, _ := clusteredLegacyStore(t)
	// A second handle has its own physical-schema bit and bounded SQL caches.
	release := make(chan struct{})
	previous := startupMaintenanceHook
	startupMaintenanceHook = func() { <-release }
	other, err := NewStore(store.config)
	startupMaintenanceHook = previous
	if err != nil {
		close(release)
		t.Fatal(err)
	}
	t.Cleanup(func() { close(release); _ = other.Close() })
	start, end := time.Unix(1_699_999_999, 0), time.Unix(1_700_001_000, 0)
	for _, step := range []int64{0, 60} {
		if _, err := other.QueryAllBatch("vm", []string{"legacy-0001", "legacy-0002"}, start, end, step); err != nil {
			t.Fatal(err)
		}
	}
	if err := store.migrateClusteredMetrics(context.Background()); err != nil {
		t.Fatal(err)
	}
	for _, step := range []int64{0, 60} {
		// Force the other handle's stale layout, including a cached presence
		// statement, so both snapshot paths exercise the real missing-index error.
		other.clusteredMetrics.Store(false)
		points, err := other.QueryAllBatch("vm", []string{"legacy-0001", "legacy-0002"}, start, end, step)
		if err != nil || len(points) != 2 || !other.clusteredMetrics.Load() {
			t.Fatalf("stale full reader step=%d: resources=%d, refreshed=%t, error=%v", step, len(points), other.clusteredMetrics.Load(), err)
		}
		other.clusteredMetrics.Store(false)
		values, err := other.QueryMetricValuesBatch("vm", []string{"legacy-0001", "legacy-0002"}, nil, start, end, step)
		if err != nil || len(values) != 2 || !other.clusteredMetrics.Load() {
			t.Fatalf("stale value reader step=%d: resources=%d, refreshed=%t, error=%v", step, len(values), other.clusteredMetrics.Load(), err)
		}
	}
}

func TestClusteredMetricsReopeningLegacyExecutableIndexes(t *testing.T) {
	cfg := DefaultConfig(t.TempDir())
	store, err := NewStore(cfg)
	if err != nil {
		t.Fatal(err)
	}
	if err := store.WaitForMaintenance(30 * time.Second); err != nil {
		t.Fatal(err)
	}
	// Reproduce the previous runtime's initSchema/unique-index behavior on
	// the new table. Its unchanged upsert conflict target remains functional.
	if _, err := store.db.Exec(`CREATE UNIQUE INDEX idx_metrics_query_all
		ON metrics(resource_type, resource_id, tier, timestamp, metric_type);
		INSERT INTO metrics(resource_type, resource_id, metric_type, value, timestamp, tier)
		VALUES ('vm', 'downgrade', 'cpu', 1, 2000000000, 'raw')
		ON CONFLICT(resource_type, resource_id, metric_type, tier, timestamp) DO UPDATE SET value=excluded.value;
		INSERT INTO metrics(resource_type, resource_id, metric_type, value, timestamp, tier)
		VALUES ('vm', 'downgrade', 'cpu', 2, 2000000000, 'raw')
		ON CONFLICT(resource_type, resource_id, metric_type, tier, timestamp) DO UPDATE SET value=excluded.value;`); err != nil {
		t.Fatal(err)
	}
	if err := store.Close(); err != nil {
		t.Fatal(err)
	}
	reopened, err := NewStore(cfg)
	if err != nil {
		t.Fatal(err)
	}
	defer reopened.Close()
	if err := reopened.WaitForMaintenance(30 * time.Second); err != nil {
		t.Fatal(err)
	}
	assertClusteredStore(t, reopened)
	var value float64
	if err := reopened.db.QueryRow(`SELECT value FROM metrics WHERE resource_id='downgrade'`).Scan(&value); err != nil || value != 2 {
		t.Fatalf("legacy-compatible upsert value=%v, error=%v", value, err)
	}
}

func TestClusteredRetainedPlansUsePrimaryTable(t *testing.T) {
	db := newPlanTestDB(t)
	store := &Store{db: pdb.Wrap(db, "metrics"), config: DefaultConfig(t.TempDir())}
	if err := store.migrateClusteredMetrics(context.Background()); err != nil {
		t.Fatal(err)
	}
	end := time.Unix(2_000_000_040, 0)
	for _, valuesOnly := range []bool{false, true} {
		for _, window := range []time.Duration{time.Hour, 24 * time.Hour, 7 * 24 * time.Hour, 30 * 24 * time.Hour} {
			for _, ids := range [][]string{{"vm-1"}, {"vm-1", "vm-2"}} {
				for _, metrics := range [][]string{nil, {"cpu"}, {"cpu", "memory"}} {
					for _, step := range []int64{0, 60} {
						t.Run(fmt.Sprintf("values_%t/%s/resources_%d/metrics_%d/step_%d", valuesOnly, window, len(ids), len(metrics), step), func(t *testing.T) {
							query, args := store.retainedQuerySQL("vm", ids, metrics, end.Add(-window), end, step, store.tierFallbacks(window), false, valuesOnly)
							plan := explainQueryPlan(t, db, query, args)
							for _, line := range strings.Split(plan, "\n") {
								if strings.Contains(line, "SCAN m ") || strings.Contains(line, "SCAN h ") {
									t.Fatalf("unbounded clustered read: %s", plan)
								}
								if strings.Contains(line, "SEARCH m ") || strings.Contains(line, "SEARCH h ") {
									for _, constraint := range []string{"USING PRIMARY KEY", "resource_type=?", "resource_id=?", "tier=?", "timestamp>?", "timestamp<?"} {
										if !strings.Contains(line, constraint) {
											t.Fatalf("clustered lookup missing %s: %s", constraint, plan)
										}
									}
								}
								if step == 0 && strings.Contains(line, "TEMP B-TREE") {
									t.Fatalf("clustered unaggregated read sorts: %s", plan)
								}
							}
							if !strings.Contains(plan, "SEARCH m USING PRIMARY KEY") {
								t.Fatalf("missing primary table lookup: %s", plan)
							}
						})
					}
				}
			}
		}
	}
}