Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
package monitoring
import (
"context"
"fmt"
"math"
"os"
"reflect"
"sort"
"testing"
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/mock"
"github.com/rcourtman/pulse-go-rewrite/internal/models"
"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
"github.com/rcourtman/pulse-go-rewrite/pkg/diskinventory"
"github.com/rcourtman/pulse-go-rewrite/pkg/metrics"
)
// Monitoring-layer SLO targets for batched chart retrieval.
//
// These protect the chart aggregation layer above the raw metrics store. The
// store benchmarks already validate SQL performance; these tests cover the
// additional alias resolution, gap-fill retry, conversion, and downsampling
// work that powers /api/charts.
//
// Baseline measurements:
// - GetGuestMetricsForChartBatch(50 guests × 5 metrics × 240 points): ~42ms on the March 2026 Apple M4 baseline;
// ~105ms p95 on the April 11, 2026 serial local run after the long-range store-backed chart path was stabilized
// - GetNodeMetricsForChartBatch(20 nodes × 5 metrics × 240 points): ~16ms
// - GitHub-hosted runners on the April 9, 2026 RC dry run reached ~337ms
// and ~153ms p95 respectively, so CI keeps separate hosted-runner budgets
// while preserving strict local thresholds.
const (
SLOGuestChartBatchP95 = 120 * time.Millisecond
SLONodeChartBatchP95 = 35 * time.Millisecond
SLOGuestChartBatchGitHubActionsP95 = 400 * time.Millisecond
SLONodeChartBatchGitHubActionsP95 = 180 * time.Millisecond
SLOPhysicalDiskChartFallbackP95 = 30 * time.Millisecond
SLOPhysicalDiskChartFallbackGHA = 120 * time.Millisecond
SLOGuestChartFallbackP95 = 30 * time.Millisecond
SLOGuestChartFallbackGHA = 120 * time.Millisecond
monitoringSLOIterations = 120
)
func skipMonitoringSLOUnderRace(t *testing.T) {
t.Helper()
if raceEnabled {
t.Skip("skipping monitoring SLO latency test under -race")
}
}
func suppressMonitoringTestLogs(t *testing.T) {
t.Helper()
}
func TestStartMockMetricsSampler_PrewarmsDefaultWorkloadChartCaches(t *testing.T) {
setMockSamplerTestEnv(t, time.Hour, 5*time.Minute)
previousEnabled := mock.IsMockEnabled()
previousConfig := mock.GetConfig()
t.Cleanup(func() {
mustSetMockEnabled(t, false)
mock.SetMockConfig(previousConfig)
if previousEnabled {
mustSetMockEnabled(t, true)
mock.SetMockConfig(previousConfig)
}
})
cfg := compactMockFixtureConfig()
// The fixture intentionally varies per-node guest density. Keep the compact
// test floor above that variation so every workload family is guaranteed to
// exist regardless of whether the single node is clustered or standalone.
cfg.VMsPerNode = 4
cfg.LXCsPerNode = 4
cfg.DockerHostCount = 1
cfg.DockerContainersPerHost = 1
cfg.K8sClusterCount = 1
cfg.K8sNodesPerCluster = 1
cfg.K8sPodsPerCluster = 1
mustSetMockEnabled(t, false)
mock.SetMockConfig(cfg)
mustSetMockEnabled(t, true)
monitor := &Monitor{
metricsHistory: NewMetricsHistory(128, 24*time.Hour),
}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
monitor.startMockMetricsSampler(ctx)
defer monitor.stopMockMetricsSampler()
requests := mockWorkloadChartRequestsForReadState(monitor.GetUnifiedReadStateOrSnapshot())
assertPrewarmed := func(resourceType string, requests []GuestChartRequest) {
t.Helper()
if len(requests) == 0 {
t.Fatalf("expected %s workload chart prewarm requests", resourceType)
}
request := requests[0]
key := mockChartMetricMapCacheKey{
kind: "guest",
resourceType: resourceType,
resourceID: request.SQLResourceID,
aux: request.InMemoryKey,
duration: mockDashboardWorkloadPrewarmDuration,
}
cached, ok := monitor.readMockChartMetricMapCache(key)
if !ok {
t.Fatalf("expected prewarmed %s cache entry for %+v", resourceType, key)
}
if len(cached["cpu"]) == 0 {
t.Fatalf("expected prewarmed %s cache entry to include cpu points", resourceType)
}
}
assertPrewarmed("vm", requests.vms)
assertPrewarmed("container", requests.containers)
assertPrewarmed("k8s", requests.pods)
assertPrewarmed("dockerContainer", requests.dockerContainers)
}
func TestBoundMockWorkloadChartPrewarmRequestsCapsEveryWorkloadFamily(t *testing.T) {
makeRequests := func(count int) []GuestChartRequest {
requests := make([]GuestChartRequest, count)
for index := range requests {
id := fmt.Sprintf("guest-%d", index)
requests[index] = GuestChartRequest{InMemoryKey: id, SQLResourceID: id}
}
return requests
}
bounded := boundMockWorkloadChartPrewarmRequests(mockWorkloadChartPrewarmRequests{
vms: makeRequests(100),
containers: makeRequests(90),
pods: makeRequests(80),
dockerContainers: makeRequests(70),
})
for family, requests := range map[string][]GuestChartRequest{
"vms": bounded.vms, "containers": bounded.containers,
"pods": bounded.pods, "dockerContainers": bounded.dockerContainers,
} {
if len(requests) != mockDashboardWorkloadPrewarmLimitPerType {
t.Fatalf("%s prewarm requests=%d, want=%d", family, len(requests), mockDashboardWorkloadPrewarmLimitPerType)
}
}
}
func newChartBatchSLOMonitor(t *testing.T) *Monitor {
t.Helper()
cfg := metrics.DefaultConfig(t.TempDir())
cfg.RetentionRaw = 90 * 24 * time.Hour
cfg.RetentionMinute = 90 * 24 * time.Hour
cfg.RetentionHourly = 90 * 24 * time.Hour
cfg.RetentionDaily = 90 * 24 * time.Hour
cfg.WriteBufferSize = 10_000
cfg.FlushInterval = time.Hour
store, err := metrics.NewStore(cfg)
if err != nil {
t.Fatalf("failed to create metrics store: %v", err)
}
if err := store.WaitForMaintenance(5 * time.Second); err != nil {
t.Fatalf("WaitForMaintenance: %v", err)
}
t.Cleanup(func() { _ = store.Close() })
return &Monitor{
metricsHistory: NewMetricsHistory(4096, 24*time.Hour),
metricsStore: store,
}
}
func measureMonitoringLatencies(t *testing.T, fn func()) []time.Duration {
t.Helper()
for i := 0; i < 20; i++ {
fn()
}
latencies := make([]time.Duration, monitoringSLOIterations)
for i := 0; i < monitoringSLOIterations; i++ {
start := time.Now()
fn()
latencies[i] = time.Since(start)
}
return latencies
}
func monitoringPercentile(durations []time.Duration, pct float64) time.Duration {
if len(durations) == 0 {
return 0
}
sorted := make([]time.Duration, len(durations))
copy(sorted, durations)
sort.Slice(sorted, func(i, j int) bool { return sorted[i] < sorted[j] })
idx := int(float64(len(sorted)-1) * pct)
return sorted[idx]
}
// assertLatencySLO logs the measured latency distribution and enforces the
// SLO target. The budgets assume a controlled host. On GitHub Actions runners
// (which already get their own envelopes via effectiveMonitoringSLOTarget)
// that holds, so an overrun fails. On a local dev machine it does not:
// parallel builds and other agents inflate wall-clock latency without bound
// (observed up to ~4x on the median during vite builds), so a local overrun
// cannot be attributed to a code regression and the test skips with the full
// distribution, the same way skipMonitoringSLOUnderRace treats race-detector
// overhead. A local pass still means the budget was genuinely met.
func assertLatencySLO(t *testing.T, label string, latencies []time.Duration, target time.Duration) {
t.Helper()
p50 := monitoringPercentile(latencies, 0.50)
p95 := monitoringPercentile(latencies, 0.95)
t.Logf("%s p50=%v p95=%v p99=%v SLO=%v", label, p50, p95, monitoringPercentile(latencies, 0.99), target)
if p95 <= target {
return
}
if latencyBudgetEnforced() {
t.Errorf("SLO VIOLATION: p95=%v exceeds target %v", p95, target)
return
}
t.Skipf("p95=%v exceeds target %v (median=%v): host CPU contention from parallel builds inflates wall-clock latency, so this overrun cannot be attributed to a regression; re-run on a quiet machine for a strict check (CI enforces the budget unconditionally)", p95, target, p50)
}
// latencyBudgetEnforced reports whether an SLO overrun should fail the run.
// Only a controlled GitHub-hosted runner qualifies. The shared release
// preflight worker exports GITHUB_ACTIONS=true for its isolated
// single-repository checkout (scripts/release-preflight-worker.sh), but it is a
// contended shared host where an overrun cannot be attributed to a regression,
// so the documented local-contention skip applies there. GITHUB_RUN_ID is set
// only by a real GitHub Actions run, never by the worker.
func latencyBudgetEnforced() bool {
return os.Getenv("GITHUB_ACTIONS") == "true" && os.Getenv("GITHUB_RUN_ID") != ""
}
func effectiveMonitoringSLOTarget(localTarget, githubActionsTarget time.Duration) time.Duration {
if githubActionsTarget > 0 && latencyBudgetEnforced() {
return githubActionsTarget
}
return localTarget
}
func seedSLOGuestMetrics(t *testing.T, store *metrics.Store, resourceType string, numResources, numPoints int) []string {
t.Helper()
metricTypes := []string{"cpu", "memory", "disk", "netin", "netout"}
now := time.Now().UTC().Truncate(time.Second)
ids := make([]string, numResources)
batch := make([]metrics.WriteMetric, 0, numResources*numPoints*len(metricTypes))
for r := 0; r < numResources; r++ {
id := fmt.Sprintf("%s-slo-%d", resourceType, r)
ids[r] = id
for _, mt := range metricTypes {
for p := 0; p < numPoints; p++ {
batch = append(batch, metrics.WriteMetric{
ResourceType: resourceType,
ResourceID: id,
MetricType: mt,
Value: float64(p%100) + float64(r%10),
Timestamp: now.Add(-time.Duration(numPoints-p) * time.Minute),
Tier: metrics.TierMinute,
})
}
}
}
store.WriteBatchSync(batch)
return ids
}
func seedSLONodeMetrics(t *testing.T, store *metrics.Store, numNodes, numPoints int) []string {
t.Helper()
metricTypes := []string{"cpu", "memory", "disk", "netin", "netout"}
now := time.Now().UTC().Truncate(time.Second)
ids := make([]string, numNodes)
batch := make([]metrics.WriteMetric, 0, numNodes*numPoints*len(metricTypes))
for n := 0; n < numNodes; n++ {
id := fmt.Sprintf("node-slo-%d", n)
ids[n] = id
for _, mt := range metricTypes {
for p := 0; p < numPoints; p++ {
batch = append(batch, metrics.WriteMetric{
ResourceType: "node",
ResourceID: id,
MetricType: mt,
Value: float64(p%100) + float64(n%10),
Timestamp: now.Add(-time.Duration(numPoints-p) * time.Minute),
Tier: metrics.TierMinute,
})
}
}
}
store.WriteBatchSync(batch)
return ids
}
func seedInMemorySLOGuestMetrics(t *testing.T, history *MetricsHistory, resourceType string, numResources, numPoints int, duration time.Duration) []GuestChartRequest {
t.Helper()
metricTypes := []string{"cpu", "memory", "disk", "netin", "netout"}
now := time.Now().UTC().Truncate(time.Second)
step := duration / time.Duration(numPoints-1)
if step <= 0 {
step = time.Minute
}
requests := make([]GuestChartRequest, numResources)
for r := 0; r < numResources; r++ {
id := fmt.Sprintf("%s-inmem-slo-%d", resourceType, r)
requests[r] = GuestChartRequest{InMemoryKey: id, SQLResourceID: id}
for _, mt := range metricTypes {
for p := 0; p < numPoints; p++ {
history.AddGuestMetric(
id,
mt,
float64((p%100)+(r%10)),
now.Add(-duration).Add(time.Duration(p)*step),
)
}
}
}
return requests
}
func seedInMemorySLONodeMetrics(t *testing.T, history *MetricsHistory, numNodes, numPoints int, duration time.Duration) []string {
t.Helper()
metricTypes := []string{"cpu", "memory", "disk", "netin", "netout"}
now := time.Now().UTC().Truncate(time.Second)
step := duration / time.Duration(numPoints-1)
if step <= 0 {
step = time.Minute
}
ids := make([]string, numNodes)
for n := 0; n < numNodes; n++ {
id := fmt.Sprintf("node-inmem-slo-%d", n)
ids[n] = id
for _, mt := range metricTypes {
for p := 0; p < numPoints; p++ {
history.AddNodeMetric(
id,
mt,
float64((p%100)+(n%10)),
now.Add(-duration).Add(time.Duration(p)*step),
)
}
}
}
return ids
}
func TestSLO_GetGuestMetricsForChartBatch(t *testing.T) {
skipMonitoringSLOUnderRace(t)
suppressMonitoringTestLogs(t)
monitor := newChartBatchSLOMonitor(t)
duration := 4 * time.Hour
ids := seedSLOGuestMetrics(t, monitor.metricsStore, "container", 50, 240)
requests := make([]GuestChartRequest, len(ids))
for i, id := range ids {
requests[i] = GuestChartRequest{InMemoryKey: id, SQLResourceID: id}
}
sanity := monitor.GetGuestMetricsForChartBatch("container", requests, duration)
if len(sanity) != len(ids) {
t.Fatalf("sanity: expected %d guests, got %d", len(ids), len(sanity))
}
for _, id := range ids {
if cpuPts := sanity[id]["cpu"]; len(cpuPts) == 0 {
t.Fatalf("sanity: guest %s has no cpu points", id)
}
}
latencies := measureMonitoringLatencies(t, func() {
result := monitor.GetGuestMetricsForChartBatch("container", requests, duration)
if len(result) != len(ids) {
t.Fatalf("expected %d guests, got %d", len(ids), len(result))
}
})
target := effectiveMonitoringSLOTarget(SLOGuestChartBatchP95, SLOGuestChartBatchGitHubActionsP95)
assertLatencySLO(t, "GetGuestMetricsForChartBatch(50×5×240)", latencies, target)
}
func TestSLO_GetGuestMetricsForChartBatch_LongRangeCoveredInMemory(t *testing.T) {
skipMonitoringSLOUnderRace(t)
suppressMonitoringTestLogs(t)
duration := 7 * 24 * time.Hour
metricTypes := []string{"cpu", "memory", "disk", "netin", "netout"}
monitor := &Monitor{
metricsHistory: NewMetricsHistory(4096, 30*24*time.Hour),
}
requests := seedInMemorySLOGuestMetrics(t, monitor.metricsHistory, "container", 24, 240, duration)
sanity := monitor.GetGuestMetricsForChartBatch("container", requests, duration, metricTypes...)
if len(sanity) != len(requests) {
t.Fatalf("sanity: expected %d guests, got %d", len(requests), len(sanity))
}
for _, req := range requests {
if cpuPts := sanity[req.SQLResourceID]["cpu"]; len(cpuPts) == 0 {
t.Fatalf("sanity: guest %s has no cpu points", req.SQLResourceID)
}
}
latencies := measureMonitoringLatencies(t, func() {
result := monitor.GetGuestMetricsForChartBatch("container", requests, duration, metricTypes...)
if len(result) != len(requests) {
t.Fatalf("expected %d guests, got %d", len(requests), len(result))
}
})
target := effectiveMonitoringSLOTarget(SLOGuestChartBatchP95, SLOGuestChartBatchGitHubActionsP95)
assertLatencySLO(t, "GetGuestMetricsForChartBatch(24×5×240, 7d in-memory)", latencies, target)
}
func TestSLO_GetNodeMetricsForChartBatch(t *testing.T) {
skipMonitoringSLOUnderRace(t)
suppressMonitoringTestLogs(t)
monitor := newChartBatchSLOMonitor(t)
duration := 4 * time.Hour
metricTypes := []string{"cpu", "memory", "disk", "netin", "netout"}
ids := seedSLONodeMetrics(t, monitor.metricsStore, 20, 240)
sanity := monitor.GetNodeMetricsForChartBatch(ids, metricTypes, duration)
if len(sanity) != len(ids) {
t.Fatalf("sanity: expected %d nodes, got %d", len(ids), len(sanity))
}
for _, id := range ids {
if cpuPts := sanity[id]["cpu"]; len(cpuPts) == 0 {
t.Fatalf("sanity: node %s has no cpu points", id)
}
}
latencies := measureMonitoringLatencies(t, func() {
result := monitor.GetNodeMetricsForChartBatch(ids, metricTypes, duration)
if len(result) != len(ids) {
t.Fatalf("expected %d nodes, got %d", len(ids), len(result))
}
})
target := effectiveMonitoringSLOTarget(SLONodeChartBatchP95, SLONodeChartBatchGitHubActionsP95)
assertLatencySLO(t, "GetNodeMetricsForChartBatch(20×5×240)", latencies, target)
}
func TestSLO_GetNodeMetricsForChartBatch_LongRangeCoveredInMemory(t *testing.T) {
skipMonitoringSLOUnderRace(t)
suppressMonitoringTestLogs(t)
duration := 7 * 24 * time.Hour
metricTypes := []string{"cpu", "memory", "disk", "netin", "netout"}
monitor := &Monitor{
metricsHistory: NewMetricsHistory(4096, 30*24*time.Hour),
}
ids := seedInMemorySLONodeMetrics(t, monitor.metricsHistory, 20, 240, duration)
sanity := monitor.GetNodeMetricsForChartBatch(ids, metricTypes, duration)
if len(sanity) != len(ids) {
t.Fatalf("sanity: expected %d nodes, got %d", len(ids), len(sanity))
}
for _, id := range ids {
if cpuPts := sanity[id]["cpu"]; len(cpuPts) == 0 {
t.Fatalf("sanity: node %s has no cpu points", id)
}
}
latencies := measureMonitoringLatencies(t, func() {
result := monitor.GetNodeMetricsForChartBatch(ids, metricTypes, duration)
if len(result) != len(ids) {
t.Fatalf("expected %d nodes, got %d", len(ids), len(result))
}
})
target := effectiveMonitoringSLOTarget(SLONodeChartBatchP95, SLONodeChartBatchGitHubActionsP95)
assertLatencySLO(t, "GetNodeMetricsForChartBatch(20×5×240, 7d in-memory)", latencies, target)
}
func TestGetNodeMetricsForChartBatch_FiltersStoreReadsToRequestedMetricTypes(t *testing.T) {
suppressMonitoringTestLogs(t)
monitor := newChartBatchSLOMonitor(t)
now := time.Now().UTC().Truncate(time.Second)
duration := 4 * time.Hour
writeBatch := []metrics.WriteMetric{
{ResourceType: "node", ResourceID: "node-filter-1", MetricType: "cpu", Value: 41, Timestamp: now.Add(-2 * time.Hour), Tier: metrics.TierMinute},
{ResourceType: "node", ResourceID: "node-filter-1", MetricType: "cpu", Value: 43, Timestamp: now.Add(-1 * time.Hour), Tier: metrics.TierMinute},
{ResourceType: "node", ResourceID: "node-filter-1", MetricType: "memory", Value: 62, Timestamp: now.Add(-2 * time.Hour), Tier: metrics.TierMinute},
{ResourceType: "node", ResourceID: "node-filter-1", MetricType: "memory", Value: 64, Timestamp: now.Add(-1 * time.Hour), Tier: metrics.TierMinute},
{ResourceType: "node", ResourceID: "node-filter-1", MetricType: "disk", Value: 83, Timestamp: now.Add(-2 * time.Hour), Tier: metrics.TierMinute},
{ResourceType: "node", ResourceID: "node-filter-1", MetricType: "disk", Value: 84, Timestamp: now.Add(-1 * time.Hour), Tier: metrics.TierMinute},
}
monitor.metricsStore.WriteBatchSync(writeBatch)
result := monitor.GetNodeMetricsForChartBatch([]string{"node-filter-1"}, []string{"cpu", "memory"}, duration)
if got := len(result["node-filter-1"]["cpu"]); got == 0 {
t.Fatalf("expected cpu series, got %+v", result["node-filter-1"])
}
if got := len(result["node-filter-1"]["memory"]); got == 0 {
t.Fatalf("expected memory series, got %+v", result["node-filter-1"])
}
if _, ok := result["node-filter-1"]["disk"]; ok {
t.Fatalf("expected filtered batch query to omit disk series, got %+v", result["node-filter-1"])
}
}
func TestGetGuestMetricsForChartBatch_FiltersStoreReadsToRequestedMetricTypes(t *testing.T) {
suppressMonitoringTestLogs(t)
monitor := newChartBatchSLOMonitor(t)
now := time.Now().UTC().Truncate(time.Second)
duration := 4 * time.Hour
writeBatch := []metrics.WriteMetric{
{ResourceType: "vm", ResourceID: "vm-filter-1", MetricType: "cpu", Value: 41, Timestamp: now.Add(-2 * time.Hour), Tier: metrics.TierMinute},
{ResourceType: "vm", ResourceID: "vm-filter-1", MetricType: "cpu", Value: 43, Timestamp: now.Add(-1 * time.Hour), Tier: metrics.TierMinute},
{ResourceType: "vm", ResourceID: "vm-filter-1", MetricType: "memory", Value: 62, Timestamp: now.Add(-2 * time.Hour), Tier: metrics.TierMinute},
{ResourceType: "vm", ResourceID: "vm-filter-1", MetricType: "memory", Value: 64, Timestamp: now.Add(-1 * time.Hour), Tier: metrics.TierMinute},
{ResourceType: "vm", ResourceID: "vm-filter-1", MetricType: "disk", Value: 83, Timestamp: now.Add(-2 * time.Hour), Tier: metrics.TierMinute},
{ResourceType: "vm", ResourceID: "vm-filter-1", MetricType: "disk", Value: 84, Timestamp: now.Add(-1 * time.Hour), Tier: metrics.TierMinute},
}
monitor.metricsStore.WriteBatchSync(writeBatch)
result := monitor.GetGuestMetricsForChartBatch(
"vm",
[]GuestChartRequest{{InMemoryKey: "vm-filter-1", SQLResourceID: "vm-filter-1"}},
duration,
"cpu",
"memory",
)
if got := len(result["vm-filter-1"]["cpu"]); got == 0 {
t.Fatalf("expected cpu series, got %+v", result["vm-filter-1"])
}
if got := len(result["vm-filter-1"]["memory"]); got == 0 {
t.Fatalf("expected memory series, got %+v", result["vm-filter-1"])
}
if _, ok := result["vm-filter-1"]["disk"]; ok {
t.Fatalf("expected filtered batch query to omit disk series, got %+v", result["vm-filter-1"])
}
}
func TestSLO_GetPhysicalDiskTemperatureCharts_WithNativeHistoryFallback(t *testing.T) {
skipMonitoringSLOUnderRace(t)
suppressMonitoringTestLogs(t)
monitor := newChartFallbackTestMonitor(t)
now := time.Now().UTC().Truncate(time.Second)
registry := unifiedresources.NewRegistry(nil)
registry.IngestSnapshot(models.StateSnapshot{
PhysicalDisks: []models.PhysicalDisk{
{
ID: "disk-1",
Node: "truenas-main",
DevPath: "/dev/sda",
Model: "Seagate Exos X18",
Serial: "SERIAL-DISK-1",
Temperature: 34,
LastChecked: now,
},
},
})
monitor.resourceStore = unifiedresources.NewMonitorAdapter(registry)
monitor.supplementalProviders = map[unifiedresources.DataSource]MonitorSupplementalRecordsProvider{
unifiedresources.SourceTrueNAS: &stubDiskTemperatureHistoryProvider{
history: map[string][]MetricPoint{
"SERIAL-DISK-1": {
{Timestamp: now.Add(-2 * time.Hour), Value: 29},
{Timestamp: now.Add(-1 * time.Hour), Value: 31},
{Timestamp: now, Value: 34},
},
},
},
}
sanity := monitor.GetPhysicalDiskTemperatureCharts(4 * time.Hour)
entry, ok := sanity["SERIAL-DISK-1"]
if !ok {
t.Fatalf("sanity: expected chart entry for canonical disk metric id, got %#v", sanity)
}
if len(entry.Temperature) != 3 {
t.Fatalf("sanity: expected native history points instead of padded fallback, got %+v", entry.Temperature)
}
latencies := measureMonitoringLatencies(t, func() {
result := monitor.GetPhysicalDiskTemperatureCharts(4 * time.Hour)
entry, ok := result["SERIAL-DISK-1"]
if !ok {
t.Fatalf("expected chart entry for canonical disk metric id, got %#v", result)
}
if len(entry.Temperature) != 3 {
t.Fatalf("expected native history points instead of padded fallback, got %+v", entry.Temperature)
}
})
target := effectiveMonitoringSLOTarget(SLOPhysicalDiskChartFallbackP95, SLOPhysicalDiskChartFallbackGHA)
assertLatencySLO(t, "GetPhysicalDiskTemperatureCharts(native-history fallback)", latencies, target)
}
// A temperature, current or retained, cannot supply historical timestamps.
// Keep only observed samples, including single-point histories. Collected
// readings retain disk metadata even without history; retained-only readings do
// not add empty entries. Neither may restore synthetic two-point padding.
func TestDiskTemperatureChartsRetainOnlyObservedSamples(t *testing.T) {
now := time.Now().UTC()
standby := &diskinventory.CollectionStatus{Temperature: diskinventory.Unavailable("smartctl", "disk is in standby")}
silent := &diskinventory.CollectionStatus{Temperature: diskinventory.Unavailable("smartctl", models.HostAgentStoppedReportingReason)}
collected := &diskinventory.CollectionStatus{Temperature: diskinventory.Available("smartctl")}
registry := unifiedresources.NewRegistry(nil)
registry.IngestSnapshot(models.StateSnapshot{PhysicalDisks: []models.PhysicalDisk{
{ID: "disk-standby", Node: "node-a", DevPath: "/dev/sda", Model: "WDC WD80EFAX", Serial: "STANDBY-NO-HISTORY", Temperature: 41, Collection: standby, LastChecked: now},
{ID: "disk-silent", Node: "node-a", DevPath: "/dev/sdb", Model: "WDC WD80EFAX", Serial: "SILENT-ONE-SAMPLE", Temperature: 43, Collection: silent, LastChecked: now},
{ID: "disk-live", Node: "node-a", DevPath: "/dev/sdc", Model: "WDC WD80EFAX", Serial: "LIVE-NO-HISTORY", Temperature: 38, Collection: collected, LastChecked: now},
}})
m := &Monitor{
metricsHistory: NewMetricsHistory(1024, 24*time.Hour),
state: models.NewState(),
resourceStore: unifiedresources.NewMonitorAdapter(registry),
}
sampledAt := now.Add(-20 * time.Minute)
m.metricsHistory.AddDiskMetric("SILENT-ONE-SAMPLE", "smart_temp", 44, sampledAt)
charts := m.GetPhysicalDiskTemperatureCharts(time.Hour)
if entry, ok := charts["STANDBY-NO-HISTORY"]; ok {
t.Fatalf("a standby disk with no stored samples got a series from its retained reading: %+v", entry.Temperature)
}
silentEntry, ok := charts["SILENT-ONE-SAMPLE"]
if !ok || len(silentEntry.Temperature) != 1 || silentEntry.Temperature[0].Value != 44 ||
!silentEntry.Temperature[0].Timestamp.Equal(sampledAt) {
t.Fatalf("silent agent disk series = %+v (found %v), want only its stored 44C sample at %s", silentEntry.Temperature, ok, sampledAt)
}
live, ok := charts["LIVE-NO-HISTORY"]
if !ok || len(live.Temperature) != 0 || live.Name != "WDC WD80EFAX" || live.Node != "node-a" {
t.Fatalf("live disk = %+v (found %v), want its metadata with no fabricated history", live, ok)
}
m.metricsHistory.AddDiskMetric("LIVE-NO-HISTORY", "smart_temp", 37, sampledAt)
live, ok = m.GetPhysicalDiskTemperatureCharts(time.Hour)["LIVE-NO-HISTORY"]
if !ok || len(live.Temperature) != 1 || live.Temperature[0].Value != 37 ||
!live.Temperature[0].Timestamp.Equal(sampledAt) {
t.Fatalf("live disk series = %+v (found %v), want only its observed 37C sample at %s", live.Temperature, ok, sampledAt)
}
}
func TestSLO_GetDiskMetricsForChart_WithNativeStoreFallback(t *testing.T) {
skipMonitoringSLOUnderRace(t)
suppressMonitoringTestLogs(t)
monitor := newChartFallbackTestMonitor(t)
now := time.Now().UTC().Truncate(time.Second)
duration := 4 * time.Hour
writeRawMetricBatch(t, monitor.metricsStore, "disk", "SERIAL-DISK-IO-1", "diskread", []MetricPoint{
{Timestamp: now.Add(-2 * time.Hour), Value: 1.25 * 1024 * 1024},
{Timestamp: now.Add(-1 * time.Hour), Value: 2.5 * 1024 * 1024},
{Timestamp: now, Value: 3.75 * 1024 * 1024},
})
sanity := monitor.GetDiskMetricsForChart("SERIAL-DISK-IO-1", "diskread", duration)
if got := len(sanity); got != 3 {
t.Fatalf("sanity: expected native diskread history, got %+v", sanity)
}
latencies := measureMonitoringLatencies(t, func() {
result := monitor.GetDiskMetricsForChart("SERIAL-DISK-IO-1", "diskread", duration)
if got := len(result); got != 3 {
t.Fatalf("expected native diskread history, got %+v", result)
}
})
target := effectiveMonitoringSLOTarget(SLOPhysicalDiskChartFallbackP95, SLOPhysicalDiskChartFallbackGHA)
assertLatencySLO(t, "GetDiskMetricsForChart(native-store fallback)", latencies, target)
}
func TestSLO_GetGuestMetricsForChart_WithNativeHistoryFallback(t *testing.T) {
skipMonitoringSLOUnderRace(t)
suppressMonitoringTestLogs(t)
monitor := newChartFallbackTestMonitor(t)
now := time.Now().UTC().Truncate(time.Second)
duration := 4 * time.Hour
writeRawMetricBatch(t, monitor.metricsStore, "agent", "truenas-main", "cpu", []MetricPoint{
{Timestamp: now.Add(-5 * time.Minute), Value: 41},
})
monitor.supplementalProviders = map[unifiedresources.DataSource]MonitorSupplementalRecordsProvider{
unifiedresources.SourceTrueNAS: &stubGuestMetricHistoryProvider{
history: map[string]map[string][]MetricPoint{
"truenas-main": {
"cpu": {
{Timestamp: now.Add(-2 * time.Hour), Value: 20},
{Timestamp: now.Add(-1 * time.Hour), Value: 28},
{Timestamp: now, Value: 34},
},
"memory": {
{Timestamp: now.Add(-2 * time.Hour), Value: 45},
{Timestamp: now, Value: 62},
},
},
},
},
}
sanity := monitor.GetGuestMetricsForChart("agent:truenas-main", "agent", "truenas-main", duration)
if got := len(sanity["cpu"]); got != 3 {
t.Fatalf("sanity: expected native cpu history, got %+v", sanity["cpu"])
}
latencies := measureMonitoringLatencies(t, func() {
result := monitor.GetGuestMetricsForChart("agent:truenas-main", "agent", "truenas-main", duration)
if got := len(result["cpu"]); got != 3 {
t.Fatalf("expected native cpu history, got %+v", result["cpu"])
}
if got := len(result["memory"]); got != 2 {
t.Fatalf("expected native memory history, got %+v", result["memory"])
}
})
target := effectiveMonitoringSLOTarget(SLOGuestChartFallbackP95, SLOGuestChartFallbackGHA)
assertLatencySLO(t, "GetGuestMetricsForChart(native-history fallback)", latencies, target)
}
func TestSLO_GetGuestMetricsForChartBatch_DoesNotStitchSparseStoreTailOntoCoveredInMemorySeries(t *testing.T) {
previous := mock.IsMockEnabled()
mustSetMockEnabled(t, true)
t.Cleanup(func() { mustSetMockEnabled(t, previous) })
monitor := newChartFallbackTestMonitor(t)
now := time.Now().UTC().Truncate(time.Second)
duration := time.Hour
inMemorySeries := []MetricPoint{
{Timestamp: now.Add(-58 * time.Minute), Value: 74},
{Timestamp: now.Add(-36 * time.Minute), Value: 75},
{Timestamp: now.Add(-14 * time.Minute), Value: 77},
{Timestamp: now.Add(-2 * time.Minute), Value: 78},
}
for _, point := range inMemorySeries {
monitor.metricsHistory.AddGuestMetric("vm-1", "memory", point.Value, point.Timestamp)
}
// This sparse late store point models the old stitched-tail behavior. In
// mock mode the batch chart path must ignore persisted history entirely and
// reuse the seeded in-memory mock history instead of synthesizing or
// stitching a different tail.
writeRawMetricBatch(t, monitor.metricsStore, "vm", "vm-1", "memory", []MetricPoint{
{Timestamp: now.Add(-1 * time.Minute), Value: 21},
})
result := monitor.GetGuestMetricsForChartBatch("vm", []GuestChartRequest{{
InMemoryKey: "vm-1",
SQLResourceID: "vm-1",
}}, duration)
memoryPoints := result["vm-1"]["memory"]
if len(memoryPoints) != len(inMemorySeries) {
t.Fatalf("expected seeded mock memory series, got %+v", memoryPoints)
}
for idx, want := range inMemorySeries {
if memoryPoints[idx].Timestamp != want.Timestamp || memoryPoints[idx].Value != want.Value {
t.Fatalf("memoryPoints[%d] = %+v, want seeded %+v", idx, memoryPoints[idx], want)
}
}
}
func TestMockChartCacheInvalidatesAfterMockHistoryRefresh(t *testing.T) {
previous := mock.IsMockEnabled()
mustSetMockEnabled(t, true)
t.Cleanup(func() { mustSetMockEnabled(t, previous) })
monitor := newChartFallbackTestMonitor(t)
now := time.Now().UTC().Truncate(time.Second)
duration := time.Hour
for _, point := range []MetricPoint{
{Timestamp: now.Add(-58 * time.Minute), Value: 41},
{Timestamp: now.Add(-36 * time.Minute), Value: 43},
{Timestamp: now.Add(-14 * time.Minute), Value: 47},
{Timestamp: now.Add(-2 * time.Minute), Value: 49},
} {
monitor.metricsHistory.AddGuestMetric("vm-1", "cpu", point.Value, point.Timestamp)
}
first := monitor.GetGuestMetricsForChartBatch("vm", []GuestChartRequest{{
InMemoryKey: "vm-1",
SQLResourceID: "vm-1",
}}, duration)
firstCPU := first["vm-1"]["cpu"]
if len(firstCPU) == 0 {
t.Fatal("expected cached mock guest chart series")
}
if got := len(monitor.mockChartMapCache); got == 0 {
t.Fatal("expected mock chart cache to populate after chart read")
}
newestTimestamp := now.Add(-30 * time.Second)
monitor.metricsHistory.AddGuestMetric("vm-1", "cpu", 99, newestTimestamp)
monitor.invalidateMockChartCaches()
if got := len(monitor.mockChartMapCache); got != 0 {
t.Fatalf("expected mock chart cache to clear after invalidation, got %d entries", got)
}
refreshed := monitor.GetGuestMetricsForChartBatch("vm", []GuestChartRequest{{
InMemoryKey: "vm-1",
SQLResourceID: "vm-1",
}}, duration)
refreshedCPU := refreshed["vm-1"]["cpu"]
if len(refreshedCPU) == 0 {
t.Fatal("expected refreshed mock guest chart series")
}
lastPoint := refreshedCPU[len(refreshedCPU)-1]
if !lastPoint.Timestamp.Equal(newestTimestamp) || lastPoint.Value != 99 {
t.Fatalf("expected refreshed mock chart tail %+v, got %+v", MetricPoint{
Timestamp: newestTimestamp,
Value: 99,
}, lastPoint)
}
}
// A storage chart request totals each pool whose chart is not cached, and a
// linked pool takes its capacity from whichever source was seen last, which
// moves without a structure change, so the totals read the data-version view.
// A batch hands every pool one shared read (GetStorageMetricsForChartBatch):
// the view is rebuilt after a tick, and on a starved process a read per pool
// rebuilt it for each. A pool whose chart is cached reads nothing.
func TestMockStorageChartReadsTheEstateOnlyForAnUncachedChart(t *testing.T) {
useMockEstate(t, 4, 5*time.Minute)
m := newMockEstateMonitor()
pools := m.currentUnifiedStateView().readState.StoragePools()
if len(pools) == 0 {
t.Fatal("the mock estate lists no storage pool")
}
id := pools[0].SourceID()
reads := 0
readState := func() unifiedresources.ReadState {
reads++
return m.GetUnifiedReadStateOrSnapshot()
}
first := m.mockStorageMetricsForChartCached(id, time.Hour, nil, readState)
if reads != 1 || len(first["total"]) == 0 {
t.Fatalf("an uncached chart read the estate %d times and returned %d total points, want one read and a total series", reads, len(first["total"]))
}
m.mockStorageMetricsForChartCached(id, time.Hour, nil, readState)
if reads != 1 {
t.Fatalf("a cached chart read the estate again: %d reads", reads)
}
}
// The poller writes every metric of a resource at the same timestamps, and
// the drawer history panels break a line wherever another series in the panel
// has a reading this one lacks. Mock history must therefore keep one shared
// set of timestamps per resource, or every mock chart draws as dots.
var (
mockGuestSharedTimestampMetrics = []string{"cpu", "memory", "memoryused", "disk", "diskread", "diskwrite", "netin", "netout"}
mockNodeSharedTimestampMetrics = []string{"cpu", "memory", "disk", "netin", "netout"}
mockSharedTimestampDurations = []time.Duration{time.Hour, 4 * time.Hour, 24 * time.Hour, 7 * 24 * time.Hour}
)
func TestSampleMetricSeriesOnMockChartGridKeepsTimestampsIndependentOfValues(t *testing.T) {
start := time.Date(2026, time.October, 13, 0, 0, 0, 0, time.UTC)
flat := make([]MetricPoint, 0, 721)
spiky := make([]MetricPoint, 0, 721)
for i := 0; i <= 720; i++ {
ts := start.Add(time.Duration(i) * 2 * time.Minute)
flat = append(flat, MetricPoint{Timestamp: ts, Value: 20 + math.Sin(float64(i)/40)})
value := 5.0
if i%37 == 0 || i%53 == 0 {
value = 90
}
spiky = append(spiky, MetricPoint{Timestamp: ts, Value: value})
}
// Control: LTTB picks points by value, so these two series keep
// different timestamps. This is the behaviour that broke mock charts.
if sameMetricTimestamps(lttb(cloneMetricSeries(flat), 48), lttb(cloneMetricSeries(spiky), 48)) {
t.Fatal("control: expected LTTB to keep different timestamps for differently shaped series")
}
interval := mockChartGridInterval(24 * time.Hour)
gridFlat := sampleMetricSeriesOnMockChartGrid(flat, interval)
gridSpiky := sampleMetricSeriesOnMockChartGrid(spiky, interval)
if !sameMetricTimestamps(gridFlat, gridSpiky) {
t.Fatalf("grid sampling kept different timestamps: %v vs %v", metricTimestamps(gridFlat), metricTimestamps(gridSpiky))
}
if got, limit := len(gridFlat), mockChartPointTarget(24*time.Hour); got > limit || got < limit-2 {
t.Fatalf("expected grid sampling to stay within two points of %d, got %d", limit, got)
}
if first := gridSpiky[0]; first != spiky[0] {
t.Fatalf("expected the first reading to be kept as-is, got %+v want %+v", first, spiky[0])
}
if last, newest := gridSpiky[len(gridSpiky)-1], spiky[len(spiky)-1]; last != newest {
t.Fatalf("expected the newest reading to be kept as-is, got %+v want %+v", last, newest)
}
readings := make(map[int64]float64, len(spiky))
for _, point := range spiky {
readings[point.Timestamp.UnixNano()] = point.Value
}
for i, point := range gridSpiky {
if i > 0 && !point.Timestamp.After(gridSpiky[i-1].Timestamp) {
t.Fatalf("grid timestamps not strictly ascending at %d", i)
}
if value, ok := readings[point.Timestamp.UnixNano()]; !ok || value != point.Value {
t.Fatalf("grid point %+v is not a reading from the input series", point)
}
}
}
func TestMockChartHistorySharesTimestampsAcrossMetrics(t *testing.T) {
previousEnabled := mock.IsMockEnabled()
previousConfig := mock.GetConfig()
t.Cleanup(func() {
mustSetMockEnabled(t, false)
mock.SetMockConfig(previousConfig)
if previousEnabled {
mustSetMockEnabled(t, true)
mock.SetMockConfig(previousConfig)
}
})
// The first mock node is "mixed": at least base-2 VMs and base-3 LXCs.
cfg := compactMockChartFixtureConfig()
cfg.VMsPerNode = 3
cfg.LXCsPerNode = 4
mustSetMockEnabled(t, false)
mock.SetMockConfig(cfg)
mustSetMockEnabled(t, true)
graph := mock.CurrentFixtureGraph()
if len(graph.State.Containers) == 0 || len(graph.State.VMs) == 0 || len(graph.State.Nodes) == 0 {
t.Fatalf(
"expected mock nodes, VMs and containers in the fixture graph, got %d nodes, %d VMs, %d containers",
len(graph.State.Nodes), len(graph.State.VMs), len(graph.State.Containers),
)
}
container := graph.State.Containers[0].ID
vm := graph.State.VMs[0].ID
node := graph.State.Nodes[0].ID
newMonitor := func() *Monitor {
resources, _ := mock.UnifiedResourceSnapshot()
registry := unifiedresources.NewRegistry(nil)
registry.IngestResources(resources)
return &Monitor{
metricsHistory: NewMetricsHistory(1000, 24*time.Hour),
state: models.NewState(),
resourceStore: unifiedresources.NewMonitorAdapter(registry),
}
}
check := func(t *testing.T, monitor *Monitor) {
for _, duration := range mockSharedTimestampDurations {
// The newest reading sits on the current minute; rerun if the
// minute turned over between two per-metric node reads.
for attempt := 0; ; attempt++ {
minute := time.Now().Truncate(time.Minute)
monitor.invalidateMockChartCaches()
guests := map[string]map[string][]MetricPoint{
"container " + container: monitor.GetGuestMetricsForChart(container, "container", container, duration),
"vm " + vm: monitor.GetGuestMetricsForChart(vm, "vm", vm, duration),
}
nodeSeries := make(map[string][]MetricPoint, len(mockNodeSharedTimestampMetrics))
for _, metric := range mockNodeSharedTimestampMetrics {
nodeSeries[metric] = monitor.GetNodeMetricsForChart(node, metric, duration)
}
if !time.Now().Truncate(time.Minute).Equal(minute) && attempt < 2 {
continue
}
for name, series := range guests {
assertMockSeriesShareTimestamps(t, name, duration, series, mockGuestSharedTimestampMetrics)
}
assertMockSeriesShareTimestamps(t, "node "+node, duration, nodeSeries, mockNodeSharedTimestampMetrics)
break
}
}
}
t.Run("synthetic history", func(t *testing.T) {
check(t, newMonitor())
})
t.Run("seeded history", func(t *testing.T) {
monitor := newMonitor()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
monitor.startMockMetricsSampler(ctx)
t.Cleanup(func() { monitor.stopMockMetricsSampler() })
check(t, monitor)
})
}
func assertMockSeriesShareTimestamps(t *testing.T, resource string, duration time.Duration, series map[string][]MetricPoint, metrics []string) {
t.Helper()
reference := series["cpu"]
if len(reference) < 2 {
t.Fatalf("%s %s: expected a cpu history, got %d points", resource, duration, len(reference))
}
minValue, maxValue := reference[0].Value, reference[0].Value
for _, point := range reference {
minValue = math.Min(minValue, point.Value)
maxValue = math.Max(maxValue, point.Value)
}
if minValue == maxValue {
t.Fatalf("%s %s: cpu history is flat at %v; mock values should still vary", resource, duration, minValue)
}
for _, metric := range metrics {
points := series[metric]
if len(points) == 0 {
t.Fatalf("%s %s: expected a %s history", resource, duration, metric)
}
if !sameMetricTimestamps(reference, points) {
t.Fatalf(
"%s %s: %s timestamps diverge from cpu (%d vs %d points, %d shared)",
resource, duration, metric, len(points), len(reference), sharedMetricTimestampCount(reference, points),
)
}
}
}
func metricTimestamps(points []MetricPoint) []int64 {
out := make([]int64, len(points))
for i, point := range points {
out[i] = point.Timestamp.UnixNano()
}
return out
}
func sameMetricTimestamps(a, b []MetricPoint) bool {
if len(a) != len(b) {
return false
}
for i := range a {
if !a[i].Timestamp.Equal(b[i].Timestamp) {
return false
}
}
return true
}
func sharedMetricTimestampCount(a, b []MetricPoint) int {
seen := make(map[int64]struct{}, len(a))
for _, point := range a {
seen[point.Timestamp.UnixNano()] = struct{}{}
}
shared := 0
for _, point := range b {
if _, ok := seen[point.Timestamp.UnixNano()]; ok {
shared++
}
}
return shared
}
func TestStoreValueProjectionPreservesChartSampling(t *testing.T) {
store, err := metrics.NewStore(metrics.DefaultConfig(t.TempDir()))
if err != nil {
t.Fatal(err)
}
defer store.Close()
end := time.Now().Truncate(time.Minute)
start := end.Add(-24 * time.Hour)
batch := make([]metrics.WriteMetric, 0, 4000)
ids := []string{"a", "b"}
names := []string{"cpu", "memory"}
for _, id := range ids {
for _, metric := range names {
for i := 0; i < 1000; i++ {
batch = append(batch, metrics.WriteMetric{ResourceType: "vm", ResourceID: id, MetricType: metric, Value: float64((i * 37) % 97), Timestamp: end.Add(-18 * time.Hour).Add(time.Duration(i) * time.Minute), Tier: metrics.TierMinute})
}
}
}
store.WriteBatchSync(batch)
monitor := &Monitor{metricsStore: store}
for _, filter := range [][]string{nil, {"cpu"}, {"cpu", "memory"}} {
t.Run(fmt.Sprint(filter), func(t *testing.T) {
full, err := store.QueryMetricTypesBatch("vm", ids, filter, start, end, 0)
if err != nil {
t.Fatal(err)
}
want := make(map[string]map[string][]MetricPoint, len(full))
for id, series := range full {
want[id] = make(map[string][]MetricPoint, len(series))
for metric, points := range series {
if len(points) != 1000 {
t.Fatalf("fixture %s/%s: got %d points", id, metric, len(points))
}
want[id][metric] = convertSeriesAndDownsample(points, chartDownsampleTarget)
}
}
got := monitor.queryStoreBatchMetricMapWithGapFill("vm", ids, 24*time.Hour, filter)
if !reflect.DeepEqual(got, want) {
t.Fatal("value projection changed the sampled chart observations")
}
for _, series := range got {
for _, points := range series {
points[0].Value = -999
points[0].Timestamp = time.Time{}
}
}
again := monitor.queryStoreBatchMetricMapWithGapFill("vm", ids, 24*time.Hour, filter)
if !reflect.DeepEqual(again, want) {
t.Fatal("mutating a chart result changed a subsequent read")
}
})
}
}