Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
package metrics
import (
"fmt"
"sync"
"sync/atomic"
"testing"
"time"
)
// =============================================================================
// 500-Node Deployment Simulation Load Tests
//
// These tests simulate a production deployment with 500 Proxmox nodes, each
// reporting 4 metric types (cpu, memory, disk_read, disk_write). The goal is
// to validate that the metrics store can handle ingestion, query, and rollup
// workloads at this scale without unacceptable latency or contention.
//
// Scale parameters:
// - 500 nodes × 4 metrics = 2000 metric series
// - Ingestion: 100 metrics/batch, simulating 5-second recording intervals
// - Query: concurrent dashboard loads (QueryAll for multiple resources)
// - Rollup: minute-tier aggregation of raw data
// =============================================================================
const (
loadTestNodes = 500
loadTestMetrics = 4 // cpu, memory, disk_read, disk_write
loadTestSeries = loadTestNodes * loadTestMetrics
loadTestSeedPoints = 20 // per metric series (simulates ~100s of data at 5s intervals)
)
var loadTestMetricTypes = [loadTestMetrics]string{"cpu", "memory", "disk_read", "disk_write"}
// seedLoadTestData populates the store with realistic 500-node data.
// Returns the time range of the seeded data.
func seedLoadTestData(b *testing.B, store *Store) (start, end time.Time) {
b.Helper()
base := time.Now().Add(-30 * time.Minute)
// Batch all seed data in a single WriteBatchSync for efficiency.
batch := make([]WriteMetric, 0, loadTestSeries*loadTestSeedPoints)
for n := 0; n < loadTestNodes; n++ {
nodeID := fmt.Sprintf("node-%d", n)
for _, mt := range loadTestMetricTypes {
for p := 0; p < loadTestSeedPoints; p++ {
batch = append(batch, WriteMetric{
ResourceType: "node",
ResourceID: nodeID,
MetricType: mt,
Value: float64((n + p) % 100),
Timestamp: base.Add(time.Duration(p) * 5 * time.Second),
Tier: TierRaw,
})
}
}
}
store.WriteBatchSync(batch)
return base.Add(-time.Second), base.Add(time.Duration(loadTestSeedPoints) * 5 * time.Second)
}
// BenchmarkLoadTest500Nodes_WriteBatch measures batch-write throughput at
// 500-node scale. Each batch contains 100 metrics from mixed resources,
// mirroring the production recording tick that collects all node metrics
// and writes them in one batch.
func BenchmarkLoadTest500Nodes_WriteBatch(b *testing.B) {
suppressLogs(b)
store := newBenchStore(b)
base := time.Now()
// Precompute resource IDs and batches.
nodeIDs := make([]string, loadTestNodes)
for n := range nodeIDs {
nodeIDs[n] = fmt.Sprintf("node-%d", n)
}
// Each batch covers 25 nodes × 4 metrics = 100 entries, cycling through
// all 500 nodes across successive batches (20 batches = full coverage).
const nodesPerBatch = 25
const batchSize = nodesPerBatch * loadTestMetrics
batches := make([][]WriteMetric, b.N)
for n := range batches {
batch := make([]WriteMetric, batchSize)
nodeOffset := (n * nodesPerBatch) % loadTestNodes
ts := base.Add(time.Duration(n) * 5 * time.Second)
for i := 0; i < nodesPerBatch; i++ {
for m := 0; m < loadTestMetrics; m++ {
batch[i*loadTestMetrics+m] = WriteMetric{
ResourceType: "node",
ResourceID: nodeIDs[(nodeOffset+i)%loadTestNodes],
MetricType: loadTestMetricTypes[m],
Value: float64((nodeOffset + i + m) % 100),
Timestamp: ts,
Tier: TierRaw,
}
}
}
batches[n] = batch
}
b.ResetTimer()
b.ReportAllocs()
for i := 0; i < b.N; i++ {
store.WriteBatchSync(batches[i])
}
}
// BenchmarkLoadTest500Nodes_QuerySingle measures single-resource query latency
// when the table contains data for 500 nodes. Validates that index isolation
// prevents full table scans at scale.
func BenchmarkLoadTest500Nodes_QuerySingle(b *testing.B) {
suppressLogs(b)
store := newBenchStore(b)
start, end := seedLoadTestData(b, store)
// Precompute resource IDs to avoid fmt.Sprintf in the timed loop.
nodeIDs := make([]string, loadTestNodes)
for n := range nodeIDs {
nodeIDs[n] = fmt.Sprintf("node-%d", n)
}
b.ResetTimer()
b.ReportAllocs()
for i := 0; i < b.N; i++ {
pts, err := store.Query("node", nodeIDs[i%loadTestNodes], "cpu", start, end, 0)
if err != nil {
b.Fatalf("Query: %v", err)
}
if len(pts) != loadTestSeedPoints {
b.Fatalf("expected %d points, got %d", loadTestSeedPoints, len(pts))
}
}
}
// BenchmarkLoadTest500Nodes_QueryAllDashboard measures QueryAll latency
// (all 4 metric types for one resource) at 500-node scale. This is the
// hot path when a user opens a node's detail view.
func BenchmarkLoadTest500Nodes_QueryAllDashboard(b *testing.B) {
suppressLogs(b)
store := newBenchStore(b)
start, end := seedLoadTestData(b, store)
nodeIDs := make([]string, loadTestNodes)
for n := range nodeIDs {
nodeIDs[n] = fmt.Sprintf("node-%d", n)
}
b.ResetTimer()
b.ReportAllocs()
for i := 0; i < b.N; i++ {
result, err := store.QueryAll("node", nodeIDs[i%loadTestNodes], start, end, 0)
if err != nil {
b.Fatalf("QueryAll: %v", err)
}
if len(result) != loadTestMetrics {
b.Fatalf("expected %d metric types, got %d", loadTestMetrics, len(result))
}
for _, mt := range loadTestMetricTypes {
if len(result[mt]) != loadTestSeedPoints {
b.Fatalf("expected %d points for %s, got %d", loadTestSeedPoints, mt, len(result[mt]))
}
}
}
}
// BenchmarkLoadTest500Nodes_ConcurrentDashboardLoad simulates 10 concurrent
// users each loading a different node's dashboard (QueryAll) while a background
// writer saturates the store with unbounded write pressure. This measures
// worst-case query latency under maximum write contention at 500-node scale
// (same stress pattern as BenchmarkConcurrentReadWrite).
func BenchmarkLoadTest500Nodes_ConcurrentDashboardLoad(b *testing.B) {
suppressLogs(b)
store := newBenchStore(b)
start, end := seedLoadTestData(b, store)
nodeIDs := make([]string, loadTestNodes)
for n := range nodeIDs {
nodeIDs[n] = fmt.Sprintf("node-%d", n)
}
// Background writer simulating continuous ingestion: 5 nodes × 4 metrics
// per batch, cycling through all 500 nodes across ticks.
const writerNodesPerBatch = 5
const writerBatchSize = writerNodesPerBatch * loadTestMetrics
stop := make(chan struct{})
writerDone := make(chan struct{})
started := make(chan struct{})
go func() {
defer close(writerDone)
writeBase := end
tick := 0
for {
select {
case <-stop:
return
default:
}
batch := make([]WriteMetric, writerBatchSize)
nodeOffset := (tick * writerNodesPerBatch) % loadTestNodes
ts := writeBase.Add(time.Duration(tick) * 5 * time.Second)
for n := 0; n < writerNodesPerBatch; n++ {
for m := 0; m < loadTestMetrics; m++ {
batch[n*loadTestMetrics+m] = WriteMetric{
ResourceType: "node",
ResourceID: nodeIDs[(nodeOffset+n)%loadTestNodes],
MetricType: loadTestMetricTypes[m],
Value: float64((tick + n + m) % 100),
Timestamp: ts,
Tier: TierRaw,
}
}
}
store.WriteBatchSync(batch)
if tick == 0 {
close(started)
}
tick++
}
}()
<-started
b.Cleanup(func() {
close(stop)
<-writerDone
})
b.ResetTimer()
b.ReportAllocs()
for i := 0; i < b.N; i++ {
// Simulate 10 concurrent dashboard loads.
var wg sync.WaitGroup
var queryErrors atomic.Int32
for u := 0; u < 10; u++ {
wg.Add(1)
go func(userIdx int) {
defer wg.Done()
nodeIdx := (i*10 + userIdx) % loadTestNodes
_, err := store.QueryAll("node", nodeIDs[nodeIdx], start, end, 0)
if err != nil {
queryErrors.Add(1)
}
}(u)
}
wg.Wait()
if errs := queryErrors.Load(); errs > 0 {
b.Fatalf("%d query errors in iteration %d", errs, i)
}
}
}
// BenchmarkLoadTest500Nodes_RollupAtScale measures rollup aggregation
// throughput with 500-node data volume. Exercises the rollupCandidate
// path that runs once per resource/metric pair every 5 minutes in production.
func BenchmarkLoadTest500Nodes_RollupAtScale(b *testing.B) {
suppressLogs(b)
store := newBenchStore(b)
// Seed data with minute-aligned timestamps for clean rollup boundaries.
rawBase := time.Now().Add(-30 * time.Minute).Unix()
base := time.Unix((rawBase/60)*60, 0)
batch := make([]WriteMetric, 0, loadTestNodes*loadTestSeedPoints)
for n := 0; n < loadTestNodes; n++ {
nodeID := fmt.Sprintf("node-%d", n)
for p := 0; p < loadTestSeedPoints; p++ {
batch = append(batch, WriteMetric{
ResourceType: "node",
ResourceID: nodeID,
MetricType: "cpu",
Value: float64((n + p) % 100),
Timestamp: base.Add(time.Duration(p) * 5 * time.Second),
Tier: TierRaw,
})
}
}
store.WriteBatchSync(batch)
bucketSecs := int64(60)
startTs := (base.Unix() / bucketSecs) * bucketSecs
rawEnd := base.Add(time.Duration(loadTestSeedPoints) * 5 * time.Second).Unix()
endTs := ((rawEnd + bucketSecs - 1) / bucketSecs) * bucketSecs
// Precompute a random subset of nodes to rollup (benchmarking one node
// per iteration at 500-node scale, cycling through all nodes).
nodeIDs := make([]string, loadTestNodes)
for n := range nodeIDs {
nodeIDs[n] = fmt.Sprintf("node-%d", n)
}
b.ResetTimer()
b.ReportAllocs()
for i := 0; i < b.N; i++ {
store.rollupCandidate("node", nodeIDs[i%loadTestNodes], "cpu", TierRaw, TierMinute, bucketSecs, startTs, endTs)
}
}
// BenchmarkLoadTest500Nodes_RollupTierBatched measures the production batched
// rollupTier path at 500-node scale. This is the real aggregation path used in
// the store: bounded INSERT...SELECT...GROUP BY chunks across node/metric series.
func BenchmarkLoadTest500Nodes_RollupTierBatched(b *testing.B) {
suppressLogs(b)
store := newBenchStore(b)
rawBase := time.Now().Add(-30 * time.Minute).Unix()
base := time.Unix((rawBase/60)*60, 0)
batch := make([]WriteMetric, 0, loadTestSeries*loadTestSeedPoints)
for n := 0; n < loadTestNodes; n++ {
nodeID := fmt.Sprintf("node-%d", n)
for _, mt := range loadTestMetricTypes {
for p := 0; p < loadTestSeedPoints; p++ {
batch = append(batch, WriteMetric{
ResourceType: "node",
ResourceID: nodeID,
MetricType: mt,
Value: float64((n + p) % 100),
Timestamp: base.Add(time.Duration(p) * 5 * time.Second),
Tier: TierRaw,
})
}
}
}
store.WriteBatchSync(batch)
metaKey := "rollup:raw:minute"
b.ResetTimer()
b.ReportAllocs()
for i := 0; i < b.N; i++ {
_ = store.setMetaInt(metaKey, 0)
store.rollupTier(TierRaw, TierMinute, time.Minute, 0)
}
}
// BenchmarkLoadTest500Nodes_QueryAllBatchDashboard measures QueryAllBatch
// latency at 500-node scale — the production hot path when the dashboard
// loads charts for all nodes in a single batched SQL call. Contrasts with
// BenchmarkLoadTest500Nodes_QueryAllDashboard which queries one node at a time.
func BenchmarkLoadTest500Nodes_QueryAllBatchDashboard(b *testing.B) {
suppressLogs(b)
store := newBenchStore(b)
start, end := seedLoadTestData(b, store)
// Build full ID list (all 500 nodes) and smaller subsets.
allIDs := make([]string, loadTestNodes)
for n := range allIDs {
allIDs[n] = fmt.Sprintf("node-%d", n)
}
for _, count := range []int{10, 50, 100, 500} {
ids := allIDs[:count]
b.Run(fmt.Sprintf("%d-nodes", count), func(b *testing.B) {
b.ReportAllocs()
for i := 0; i < b.N; i++ {
result, err := store.QueryAllBatch("node", ids, start, end, 0)
if err != nil {
b.Fatalf("QueryAllBatch: %v", err)
}
if len(result) != count {
b.Fatalf("expected %d resources, got %d", count, len(result))
}
// Spot-check one resource per iteration (rotating) for metric completeness.
checkID := ids[i%count]
resMetrics := result[checkID]
if len(resMetrics) != loadTestMetrics {
b.Fatalf("resource %s: expected %d metric types, got %d", checkID, loadTestMetrics, len(resMetrics))
}
for _, mt := range loadTestMetricTypes {
if len(resMetrics[mt]) != loadTestSeedPoints {
b.Fatalf("resource %s metric %s: expected %d points, got %d", checkID, mt, loadTestSeedPoints, len(resMetrics[mt]))
}
}
}
})
}
}
// TestLoadTest500Nodes_WriteAndQueryIntegration is a non-benchmark integration
// test that verifies correctness at 500-node scale: seeds data, queries every
// resource, and validates result counts. Uses t.Parallel() for CI efficiency.
func TestLoadTest500Nodes_WriteAndQueryIntegration(t *testing.T) {
if testing.Short() {
t.Skip("skipping 500-node load test in short mode")
}
t.Parallel()
suppressLogs(t)
store := func() *Store {
t.Helper()
dir := t.TempDir()
cfg := DefaultConfig(dir)
cfg.FlushInterval = time.Hour
cfg.WriteBufferSize = 50_000
s, err := NewStore(cfg)
if err != nil {
t.Fatalf("NewStore: %v", err)
}
t.Cleanup(func() { s.Close() })
return s
}()
base := time.Now().Add(-30 * time.Minute)
const pointsPerSeries = 20
// Seed all 500 nodes × 4 metrics.
batch := make([]WriteMetric, 0, loadTestSeries*pointsPerSeries)
for n := 0; n < loadTestNodes; n++ {
nodeID := fmt.Sprintf("node-%d", n)
for _, mt := range loadTestMetricTypes {
for p := 0; p < pointsPerSeries; p++ {
batch = append(batch, WriteMetric{
ResourceType: "node",
ResourceID: nodeID,
MetricType: mt,
Value: float64((n + p) % 100),
Timestamp: base.Add(time.Duration(p) * 5 * time.Second),
Tier: TierRaw,
})
}
}
}
store.WriteBatchSync(batch)
start := base.Add(-time.Second)
end := base.Add(time.Duration(pointsPerSeries) * 5 * time.Second)
// Verify all 500 nodes to ensure complete correctness at scale.
for idx := 0; idx < loadTestNodes; idx++ {
nodeID := fmt.Sprintf("node-%d", idx)
result, err := store.QueryAll("node", nodeID, start, end, 0)
if err != nil {
t.Fatalf("QueryAll(%s): %v", nodeID, err)
}
if len(result) != loadTestMetrics {
t.Fatalf("QueryAll(%s): expected %d metric types, got %d", nodeID, loadTestMetrics, len(result))
}
for _, mt := range loadTestMetricTypes {
pts := result[mt]
if len(pts) != pointsPerSeries {
t.Fatalf("QueryAll(%s)[%s]: expected %d points, got %d", nodeID, mt, pointsPerSeries, len(pts))
}
}
}
}