Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

41.5 KB
Raw Blame
package monitoring

import (
	"context"
	"fmt"
	"math"
	"os"
	"reflect"
	"sort"
	"testing"
	"time"

	"github.com/rcourtman/pulse-go-rewrite/internal/mock"
	"github.com/rcourtman/pulse-go-rewrite/internal/models"
	"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
	"github.com/rcourtman/pulse-go-rewrite/pkg/diskinventory"
	"github.com/rcourtman/pulse-go-rewrite/pkg/metrics"
)

// Monitoring-layer SLO targets for batched chart retrieval.
//
// These protect the chart aggregation layer above the raw metrics store. The
// store benchmarks already validate SQL performance; these tests cover the
// additional alias resolution, gap-fill retry, conversion, and downsampling
// work that powers /api/charts.
//
// Baseline measurements:
//   - GetGuestMetricsForChartBatch(50 guests × 5 metrics × 240 points): ~42ms on the March 2026 Apple M4 baseline;
//     ~105ms p95 on the April 11, 2026 serial local run after the long-range store-backed chart path was stabilized
//   - GetNodeMetricsForChartBatch(20 nodes × 5 metrics × 240 points):  ~16ms
//   - GitHub-hosted runners on the April 9, 2026 RC dry run reached ~337ms
//     and ~153ms p95 respectively, so CI keeps separate hosted-runner budgets
//     while preserving strict local thresholds.
const (
	SLOGuestChartBatchP95              = 120 * time.Millisecond
	SLONodeChartBatchP95               = 35 * time.Millisecond
	SLOGuestChartBatchGitHubActionsP95 = 400 * time.Millisecond
	SLONodeChartBatchGitHubActionsP95  = 180 * time.Millisecond
	SLOPhysicalDiskChartFallbackP95    = 30 * time.Millisecond
	SLOPhysicalDiskChartFallbackGHA    = 120 * time.Millisecond
	SLOGuestChartFallbackP95           = 30 * time.Millisecond
	SLOGuestChartFallbackGHA           = 120 * time.Millisecond

	monitoringSLOIterations = 120
)

func skipMonitoringSLOUnderRace(t *testing.T) {
	t.Helper()
	if raceEnabled {
		t.Skip("skipping monitoring SLO latency test under -race")
	}
}

func suppressMonitoringTestLogs(t *testing.T) {
	t.Helper()
}

func TestStartMockMetricsSampler_PrewarmsDefaultWorkloadChartCaches(t *testing.T) {
	setMockSamplerTestEnv(t, time.Hour, 5*time.Minute)

	previousEnabled := mock.IsMockEnabled()
	previousConfig := mock.GetConfig()
	t.Cleanup(func() {
		mustSetMockEnabled(t, false)
		mock.SetMockConfig(previousConfig)
		if previousEnabled {
			mustSetMockEnabled(t, true)
			mock.SetMockConfig(previousConfig)
		}
	})

	cfg := compactMockFixtureConfig()
	// The fixture intentionally varies per-node guest density. Keep the compact
	// test floor above that variation so every workload family is guaranteed to
	// exist regardless of whether the single node is clustered or standalone.
	cfg.VMsPerNode = 4
	cfg.LXCsPerNode = 4
	cfg.DockerHostCount = 1
	cfg.DockerContainersPerHost = 1
	cfg.K8sClusterCount = 1
	cfg.K8sNodesPerCluster = 1
	cfg.K8sPodsPerCluster = 1

	mustSetMockEnabled(t, false)
	mock.SetMockConfig(cfg)
	mustSetMockEnabled(t, true)

	monitor := &Monitor{
		metricsHistory: NewMetricsHistory(128, 24*time.Hour),
	}

	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	monitor.startMockMetricsSampler(ctx)
	defer monitor.stopMockMetricsSampler()

	requests := mockWorkloadChartRequestsForReadState(monitor.GetUnifiedReadStateOrSnapshot())
	assertPrewarmed := func(resourceType string, requests []GuestChartRequest) {
		t.Helper()
		if len(requests) == 0 {
			t.Fatalf("expected %s workload chart prewarm requests", resourceType)
		}

		request := requests[0]
		key := mockChartMetricMapCacheKey{
			kind:         "guest",
			resourceType: resourceType,
			resourceID:   request.SQLResourceID,
			aux:          request.InMemoryKey,
			duration:     mockDashboardWorkloadPrewarmDuration,
		}
		cached, ok := monitor.readMockChartMetricMapCache(key)
		if !ok {
			t.Fatalf("expected prewarmed %s cache entry for %+v", resourceType, key)
		}
		if len(cached["cpu"]) == 0 {
			t.Fatalf("expected prewarmed %s cache entry to include cpu points", resourceType)
		}
	}

	assertPrewarmed("vm", requests.vms)
	assertPrewarmed("container", requests.containers)
	assertPrewarmed("k8s", requests.pods)
	assertPrewarmed("dockerContainer", requests.dockerContainers)
}

func TestBoundMockWorkloadChartPrewarmRequestsCapsEveryWorkloadFamily(t *testing.T) {
	makeRequests := func(count int) []GuestChartRequest {
		requests := make([]GuestChartRequest, count)
		for index := range requests {
			id := fmt.Sprintf("guest-%d", index)
			requests[index] = GuestChartRequest{InMemoryKey: id, SQLResourceID: id}
		}
		return requests
	}

	bounded := boundMockWorkloadChartPrewarmRequests(mockWorkloadChartPrewarmRequests{
		vms:              makeRequests(100),
		containers:       makeRequests(90),
		pods:             makeRequests(80),
		dockerContainers: makeRequests(70),
	})
	for family, requests := range map[string][]GuestChartRequest{
		"vms": bounded.vms, "containers": bounded.containers,
		"pods": bounded.pods, "dockerContainers": bounded.dockerContainers,
	} {
		if len(requests) != mockDashboardWorkloadPrewarmLimitPerType {
			t.Fatalf("%s prewarm requests=%d, want=%d", family, len(requests), mockDashboardWorkloadPrewarmLimitPerType)
		}
	}
}

func newChartBatchSLOMonitor(t *testing.T) *Monitor {
	t.Helper()

	cfg := metrics.DefaultConfig(t.TempDir())
	cfg.RetentionRaw = 90 * 24 * time.Hour
	cfg.RetentionMinute = 90 * 24 * time.Hour
	cfg.RetentionHourly = 90 * 24 * time.Hour
	cfg.RetentionDaily = 90 * 24 * time.Hour
	cfg.WriteBufferSize = 10_000
	cfg.FlushInterval = time.Hour

	store, err := metrics.NewStore(cfg)
	if err != nil {
		t.Fatalf("failed to create metrics store: %v", err)
	}
	if err := store.WaitForMaintenance(5 * time.Second); err != nil {
		t.Fatalf("WaitForMaintenance: %v", err)
	}
	t.Cleanup(func() { _ = store.Close() })

	return &Monitor{
		metricsHistory: NewMetricsHistory(4096, 24*time.Hour),
		metricsStore:   store,
	}
}

func measureMonitoringLatencies(t *testing.T, fn func()) []time.Duration {
	t.Helper()
	for i := 0; i < 20; i++ {
		fn()
	}

	latencies := make([]time.Duration, monitoringSLOIterations)
	for i := 0; i < monitoringSLOIterations; i++ {
		start := time.Now()
		fn()
		latencies[i] = time.Since(start)
	}
	return latencies
}

func monitoringPercentile(durations []time.Duration, pct float64) time.Duration {
	if len(durations) == 0 {
		return 0
	}
	sorted := make([]time.Duration, len(durations))
	copy(sorted, durations)
	sort.Slice(sorted, func(i, j int) bool { return sorted[i] < sorted[j] })
	idx := int(float64(len(sorted)-1) * pct)
	return sorted[idx]
}

// assertLatencySLO logs the measured latency distribution and enforces the
// SLO target. The budgets assume a controlled host. On GitHub Actions runners
// (which already get their own envelopes via effectiveMonitoringSLOTarget)
// that holds, so an overrun fails. On a local dev machine it does not:
// parallel builds and other agents inflate wall-clock latency without bound
// (observed up to ~4x on the median during vite builds), so a local overrun
// cannot be attributed to a code regression and the test skips with the full
// distribution, the same way skipMonitoringSLOUnderRace treats race-detector
// overhead. A local pass still means the budget was genuinely met.
func assertLatencySLO(t *testing.T, label string, latencies []time.Duration, target time.Duration) {
	t.Helper()
	p50 := monitoringPercentile(latencies, 0.50)
	p95 := monitoringPercentile(latencies, 0.95)
	t.Logf("%s p50=%v p95=%v p99=%v SLO=%v", label, p50, p95, monitoringPercentile(latencies, 0.99), target)
	if p95 <= target {
		return
	}
	if latencyBudgetEnforced() {
		t.Errorf("SLO VIOLATION: p95=%v exceeds target %v", p95, target)
		return
	}
	t.Skipf("p95=%v exceeds target %v (median=%v): host CPU contention from parallel builds inflates wall-clock latency, so this overrun cannot be attributed to a regression; re-run on a quiet machine for a strict check (CI enforces the budget unconditionally)", p95, target, p50)
}

// latencyBudgetEnforced reports whether an SLO overrun should fail the run.
// Only a controlled GitHub-hosted runner qualifies. The shared release
// preflight worker exports GITHUB_ACTIONS=true for its isolated
// single-repository checkout (scripts/release-preflight-worker.sh), but it is a
// contended shared host where an overrun cannot be attributed to a regression,
// so the documented local-contention skip applies there. GITHUB_RUN_ID is set
// only by a real GitHub Actions run, never by the worker.
func latencyBudgetEnforced() bool {
	return os.Getenv("GITHUB_ACTIONS") == "true" && os.Getenv("GITHUB_RUN_ID") != ""
}

func effectiveMonitoringSLOTarget(localTarget, githubActionsTarget time.Duration) time.Duration {
	if githubActionsTarget > 0 && latencyBudgetEnforced() {
		return githubActionsTarget
	}
	return localTarget
}

func seedSLOGuestMetrics(t *testing.T, store *metrics.Store, resourceType string, numResources, numPoints int) []string {
	t.Helper()

	metricTypes := []string{"cpu", "memory", "disk", "netin", "netout"}
	now := time.Now().UTC().Truncate(time.Second)
	ids := make([]string, numResources)

	batch := make([]metrics.WriteMetric, 0, numResources*numPoints*len(metricTypes))
	for r := 0; r < numResources; r++ {
		id := fmt.Sprintf("%s-slo-%d", resourceType, r)
		ids[r] = id
		for _, mt := range metricTypes {
			for p := 0; p < numPoints; p++ {
				batch = append(batch, metrics.WriteMetric{
					ResourceType: resourceType,
					ResourceID:   id,
					MetricType:   mt,
					Value:        float64(p%100) + float64(r%10),
					Timestamp:    now.Add(-time.Duration(numPoints-p) * time.Minute),
					Tier:         metrics.TierMinute,
				})
			}
		}
	}
	store.WriteBatchSync(batch)
	return ids
}

func seedSLONodeMetrics(t *testing.T, store *metrics.Store, numNodes, numPoints int) []string {
	t.Helper()

	metricTypes := []string{"cpu", "memory", "disk", "netin", "netout"}
	now := time.Now().UTC().Truncate(time.Second)
	ids := make([]string, numNodes)

	batch := make([]metrics.WriteMetric, 0, numNodes*numPoints*len(metricTypes))
	for n := 0; n < numNodes; n++ {
		id := fmt.Sprintf("node-slo-%d", n)
		ids[n] = id
		for _, mt := range metricTypes {
			for p := 0; p < numPoints; p++ {
				batch = append(batch, metrics.WriteMetric{
					ResourceType: "node",
					ResourceID:   id,
					MetricType:   mt,
					Value:        float64(p%100) + float64(n%10),
					Timestamp:    now.Add(-time.Duration(numPoints-p) * time.Minute),
					Tier:         metrics.TierMinute,
				})
			}
		}
	}
	store.WriteBatchSync(batch)
	return ids
}

func seedInMemorySLOGuestMetrics(t *testing.T, history *MetricsHistory, resourceType string, numResources, numPoints int, duration time.Duration) []GuestChartRequest {
	t.Helper()

	metricTypes := []string{"cpu", "memory", "disk", "netin", "netout"}
	now := time.Now().UTC().Truncate(time.Second)
	step := duration / time.Duration(numPoints-1)
	if step <= 0 {
		step = time.Minute
	}

	requests := make([]GuestChartRequest, numResources)
	for r := 0; r < numResources; r++ {
		id := fmt.Sprintf("%s-inmem-slo-%d", resourceType, r)
		requests[r] = GuestChartRequest{InMemoryKey: id, SQLResourceID: id}
		for _, mt := range metricTypes {
			for p := 0; p < numPoints; p++ {
				history.AddGuestMetric(
					id,
					mt,
					float64((p%100)+(r%10)),
					now.Add(-duration).Add(time.Duration(p)*step),
				)
			}
		}
	}

	return requests
}

func seedInMemorySLONodeMetrics(t *testing.T, history *MetricsHistory, numNodes, numPoints int, duration time.Duration) []string {
	t.Helper()

	metricTypes := []string{"cpu", "memory", "disk", "netin", "netout"}
	now := time.Now().UTC().Truncate(time.Second)
	step := duration / time.Duration(numPoints-1)
	if step <= 0 {
		step = time.Minute
	}

	ids := make([]string, numNodes)
	for n := 0; n < numNodes; n++ {
		id := fmt.Sprintf("node-inmem-slo-%d", n)
		ids[n] = id
		for _, mt := range metricTypes {
			for p := 0; p < numPoints; p++ {
				history.AddNodeMetric(
					id,
					mt,
					float64((p%100)+(n%10)),
					now.Add(-duration).Add(time.Duration(p)*step),
				)
			}
		}
	}

	return ids
}

func TestSLO_GetGuestMetricsForChartBatch(t *testing.T) {
	skipMonitoringSLOUnderRace(t)
	suppressMonitoringTestLogs(t)

	monitor := newChartBatchSLOMonitor(t)
	duration := 4 * time.Hour
	ids := seedSLOGuestMetrics(t, monitor.metricsStore, "container", 50, 240)
	requests := make([]GuestChartRequest, len(ids))
	for i, id := range ids {
		requests[i] = GuestChartRequest{InMemoryKey: id, SQLResourceID: id}
	}

	sanity := monitor.GetGuestMetricsForChartBatch("container", requests, duration)
	if len(sanity) != len(ids) {
		t.Fatalf("sanity: expected %d guests, got %d", len(ids), len(sanity))
	}
	for _, id := range ids {
		if cpuPts := sanity[id]["cpu"]; len(cpuPts) == 0 {
			t.Fatalf("sanity: guest %s has no cpu points", id)
		}
	}

	latencies := measureMonitoringLatencies(t, func() {
		result := monitor.GetGuestMetricsForChartBatch("container", requests, duration)
		if len(result) != len(ids) {
			t.Fatalf("expected %d guests, got %d", len(ids), len(result))
		}
	})

	target := effectiveMonitoringSLOTarget(SLOGuestChartBatchP95, SLOGuestChartBatchGitHubActionsP95)
	assertLatencySLO(t, "GetGuestMetricsForChartBatch(50×5×240)", latencies, target)
}

func TestSLO_GetGuestMetricsForChartBatch_LongRangeCoveredInMemory(t *testing.T) {
	skipMonitoringSLOUnderRace(t)
	suppressMonitoringTestLogs(t)

	duration := 7 * 24 * time.Hour
	metricTypes := []string{"cpu", "memory", "disk", "netin", "netout"}
	monitor := &Monitor{
		metricsHistory: NewMetricsHistory(4096, 30*24*time.Hour),
	}
	requests := seedInMemorySLOGuestMetrics(t, monitor.metricsHistory, "container", 24, 240, duration)

	sanity := monitor.GetGuestMetricsForChartBatch("container", requests, duration, metricTypes...)
	if len(sanity) != len(requests) {
		t.Fatalf("sanity: expected %d guests, got %d", len(requests), len(sanity))
	}
	for _, req := range requests {
		if cpuPts := sanity[req.SQLResourceID]["cpu"]; len(cpuPts) == 0 {
			t.Fatalf("sanity: guest %s has no cpu points", req.SQLResourceID)
		}
	}

	latencies := measureMonitoringLatencies(t, func() {
		result := monitor.GetGuestMetricsForChartBatch("container", requests, duration, metricTypes...)
		if len(result) != len(requests) {
			t.Fatalf("expected %d guests, got %d", len(requests), len(result))
		}
	})

	target := effectiveMonitoringSLOTarget(SLOGuestChartBatchP95, SLOGuestChartBatchGitHubActionsP95)
	assertLatencySLO(t, "GetGuestMetricsForChartBatch(24×5×240, 7d in-memory)", latencies, target)
}

func TestSLO_GetNodeMetricsForChartBatch(t *testing.T) {
	skipMonitoringSLOUnderRace(t)
	suppressMonitoringTestLogs(t)

	monitor := newChartBatchSLOMonitor(t)
	duration := 4 * time.Hour
	metricTypes := []string{"cpu", "memory", "disk", "netin", "netout"}
	ids := seedSLONodeMetrics(t, monitor.metricsStore, 20, 240)

	sanity := monitor.GetNodeMetricsForChartBatch(ids, metricTypes, duration)
	if len(sanity) != len(ids) {
		t.Fatalf("sanity: expected %d nodes, got %d", len(ids), len(sanity))
	}
	for _, id := range ids {
		if cpuPts := sanity[id]["cpu"]; len(cpuPts) == 0 {
			t.Fatalf("sanity: node %s has no cpu points", id)
		}
	}

	latencies := measureMonitoringLatencies(t, func() {
		result := monitor.GetNodeMetricsForChartBatch(ids, metricTypes, duration)
		if len(result) != len(ids) {
			t.Fatalf("expected %d nodes, got %d", len(ids), len(result))
		}
	})

	target := effectiveMonitoringSLOTarget(SLONodeChartBatchP95, SLONodeChartBatchGitHubActionsP95)
	assertLatencySLO(t, "GetNodeMetricsForChartBatch(20×5×240)", latencies, target)
}

func TestSLO_GetNodeMetricsForChartBatch_LongRangeCoveredInMemory(t *testing.T) {
	skipMonitoringSLOUnderRace(t)
	suppressMonitoringTestLogs(t)

	duration := 7 * 24 * time.Hour
	metricTypes := []string{"cpu", "memory", "disk", "netin", "netout"}
	monitor := &Monitor{
		metricsHistory: NewMetricsHistory(4096, 30*24*time.Hour),
	}
	ids := seedInMemorySLONodeMetrics(t, monitor.metricsHistory, 20, 240, duration)

	sanity := monitor.GetNodeMetricsForChartBatch(ids, metricTypes, duration)
	if len(sanity) != len(ids) {
		t.Fatalf("sanity: expected %d nodes, got %d", len(ids), len(sanity))
	}
	for _, id := range ids {
		if cpuPts := sanity[id]["cpu"]; len(cpuPts) == 0 {
			t.Fatalf("sanity: node %s has no cpu points", id)
		}
	}

	latencies := measureMonitoringLatencies(t, func() {
		result := monitor.GetNodeMetricsForChartBatch(ids, metricTypes, duration)
		if len(result) != len(ids) {
			t.Fatalf("expected %d nodes, got %d", len(ids), len(result))
		}
	})

	target := effectiveMonitoringSLOTarget(SLONodeChartBatchP95, SLONodeChartBatchGitHubActionsP95)
	assertLatencySLO(t, "GetNodeMetricsForChartBatch(20×5×240, 7d in-memory)", latencies, target)
}

func TestGetNodeMetricsForChartBatch_FiltersStoreReadsToRequestedMetricTypes(t *testing.T) {
	suppressMonitoringTestLogs(t)

	monitor := newChartBatchSLOMonitor(t)
	now := time.Now().UTC().Truncate(time.Second)
	duration := 4 * time.Hour
	writeBatch := []metrics.WriteMetric{
		{ResourceType: "node", ResourceID: "node-filter-1", MetricType: "cpu", Value: 41, Timestamp: now.Add(-2 * time.Hour), Tier: metrics.TierMinute},
		{ResourceType: "node", ResourceID: "node-filter-1", MetricType: "cpu", Value: 43, Timestamp: now.Add(-1 * time.Hour), Tier: metrics.TierMinute},
		{ResourceType: "node", ResourceID: "node-filter-1", MetricType: "memory", Value: 62, Timestamp: now.Add(-2 * time.Hour), Tier: metrics.TierMinute},
		{ResourceType: "node", ResourceID: "node-filter-1", MetricType: "memory", Value: 64, Timestamp: now.Add(-1 * time.Hour), Tier: metrics.TierMinute},
		{ResourceType: "node", ResourceID: "node-filter-1", MetricType: "disk", Value: 83, Timestamp: now.Add(-2 * time.Hour), Tier: metrics.TierMinute},
		{ResourceType: "node", ResourceID: "node-filter-1", MetricType: "disk", Value: 84, Timestamp: now.Add(-1 * time.Hour), Tier: metrics.TierMinute},
	}
	monitor.metricsStore.WriteBatchSync(writeBatch)

	result := monitor.GetNodeMetricsForChartBatch([]string{"node-filter-1"}, []string{"cpu", "memory"}, duration)
	if got := len(result["node-filter-1"]["cpu"]); got == 0 {
		t.Fatalf("expected cpu series, got %+v", result["node-filter-1"])
	}
	if got := len(result["node-filter-1"]["memory"]); got == 0 {
		t.Fatalf("expected memory series, got %+v", result["node-filter-1"])
	}
	if _, ok := result["node-filter-1"]["disk"]; ok {
		t.Fatalf("expected filtered batch query to omit disk series, got %+v", result["node-filter-1"])
	}
}

func TestGetGuestMetricsForChartBatch_FiltersStoreReadsToRequestedMetricTypes(t *testing.T) {
	suppressMonitoringTestLogs(t)

	monitor := newChartBatchSLOMonitor(t)
	now := time.Now().UTC().Truncate(time.Second)
	duration := 4 * time.Hour
	writeBatch := []metrics.WriteMetric{
		{ResourceType: "vm", ResourceID: "vm-filter-1", MetricType: "cpu", Value: 41, Timestamp: now.Add(-2 * time.Hour), Tier: metrics.TierMinute},
		{ResourceType: "vm", ResourceID: "vm-filter-1", MetricType: "cpu", Value: 43, Timestamp: now.Add(-1 * time.Hour), Tier: metrics.TierMinute},
		{ResourceType: "vm", ResourceID: "vm-filter-1", MetricType: "memory", Value: 62, Timestamp: now.Add(-2 * time.Hour), Tier: metrics.TierMinute},
		{ResourceType: "vm", ResourceID: "vm-filter-1", MetricType: "memory", Value: 64, Timestamp: now.Add(-1 * time.Hour), Tier: metrics.TierMinute},
		{ResourceType: "vm", ResourceID: "vm-filter-1", MetricType: "disk", Value: 83, Timestamp: now.Add(-2 * time.Hour), Tier: metrics.TierMinute},
		{ResourceType: "vm", ResourceID: "vm-filter-1", MetricType: "disk", Value: 84, Timestamp: now.Add(-1 * time.Hour), Tier: metrics.TierMinute},
	}
	monitor.metricsStore.WriteBatchSync(writeBatch)

	result := monitor.GetGuestMetricsForChartBatch(
		"vm",
		[]GuestChartRequest{{InMemoryKey: "vm-filter-1", SQLResourceID: "vm-filter-1"}},
		duration,
		"cpu",
		"memory",
	)
	if got := len(result["vm-filter-1"]["cpu"]); got == 0 {
		t.Fatalf("expected cpu series, got %+v", result["vm-filter-1"])
	}
	if got := len(result["vm-filter-1"]["memory"]); got == 0 {
		t.Fatalf("expected memory series, got %+v", result["vm-filter-1"])
	}
	if _, ok := result["vm-filter-1"]["disk"]; ok {
		t.Fatalf("expected filtered batch query to omit disk series, got %+v", result["vm-filter-1"])
	}
}

func TestSLO_GetPhysicalDiskTemperatureCharts_WithNativeHistoryFallback(t *testing.T) {
	skipMonitoringSLOUnderRace(t)
	suppressMonitoringTestLogs(t)

	monitor := newChartFallbackTestMonitor(t)
	now := time.Now().UTC().Truncate(time.Second)
	registry := unifiedresources.NewRegistry(nil)
	registry.IngestSnapshot(models.StateSnapshot{
		PhysicalDisks: []models.PhysicalDisk{
			{
				ID:          "disk-1",
				Node:        "truenas-main",
				DevPath:     "/dev/sda",
				Model:       "Seagate Exos X18",
				Serial:      "SERIAL-DISK-1",
				Temperature: 34,
				LastChecked: now,
			},
		},
	})
	monitor.resourceStore = unifiedresources.NewMonitorAdapter(registry)
	monitor.supplementalProviders = map[unifiedresources.DataSource]MonitorSupplementalRecordsProvider{
		unifiedresources.SourceTrueNAS: &stubDiskTemperatureHistoryProvider{
			history: map[string][]MetricPoint{
				"SERIAL-DISK-1": {
					{Timestamp: now.Add(-2 * time.Hour), Value: 29},
					{Timestamp: now.Add(-1 * time.Hour), Value: 31},
					{Timestamp: now, Value: 34},
				},
			},
		},
	}

	sanity := monitor.GetPhysicalDiskTemperatureCharts(4 * time.Hour)
	entry, ok := sanity["SERIAL-DISK-1"]
	if !ok {
		t.Fatalf("sanity: expected chart entry for canonical disk metric id, got %#v", sanity)
	}
	if len(entry.Temperature) != 3 {
		t.Fatalf("sanity: expected native history points instead of padded fallback, got %+v", entry.Temperature)
	}

	latencies := measureMonitoringLatencies(t, func() {
		result := monitor.GetPhysicalDiskTemperatureCharts(4 * time.Hour)
		entry, ok := result["SERIAL-DISK-1"]
		if !ok {
			t.Fatalf("expected chart entry for canonical disk metric id, got %#v", result)
		}
		if len(entry.Temperature) != 3 {
			t.Fatalf("expected native history points instead of padded fallback, got %+v", entry.Temperature)
		}
	})

	target := effectiveMonitoringSLOTarget(SLOPhysicalDiskChartFallbackP95, SLOPhysicalDiskChartFallbackGHA)
	assertLatencySLO(t, "GetPhysicalDiskTemperatureCharts(native-history fallback)", latencies, target)
}

// A temperature, current or retained, cannot supply historical timestamps.
// Keep only observed samples, including single-point histories. Collected
// readings retain disk metadata even without history; retained-only readings do
// not add empty entries. Neither may restore synthetic two-point padding.
func TestDiskTemperatureChartsRetainOnlyObservedSamples(t *testing.T) {
	now := time.Now().UTC()
	standby := &diskinventory.CollectionStatus{Temperature: diskinventory.Unavailable("smartctl", "disk is in standby")}
	silent := &diskinventory.CollectionStatus{Temperature: diskinventory.Unavailable("smartctl", models.HostAgentStoppedReportingReason)}
	collected := &diskinventory.CollectionStatus{Temperature: diskinventory.Available("smartctl")}
	registry := unifiedresources.NewRegistry(nil)
	registry.IngestSnapshot(models.StateSnapshot{PhysicalDisks: []models.PhysicalDisk{
		{ID: "disk-standby", Node: "node-a", DevPath: "/dev/sda", Model: "WDC WD80EFAX", Serial: "STANDBY-NO-HISTORY", Temperature: 41, Collection: standby, LastChecked: now},
		{ID: "disk-silent", Node: "node-a", DevPath: "/dev/sdb", Model: "WDC WD80EFAX", Serial: "SILENT-ONE-SAMPLE", Temperature: 43, Collection: silent, LastChecked: now},
		{ID: "disk-live", Node: "node-a", DevPath: "/dev/sdc", Model: "WDC WD80EFAX", Serial: "LIVE-NO-HISTORY", Temperature: 38, Collection: collected, LastChecked: now},
	}})
	m := &Monitor{
		metricsHistory: NewMetricsHistory(1024, 24*time.Hour),
		state:          models.NewState(),
		resourceStore:  unifiedresources.NewMonitorAdapter(registry),
	}
	sampledAt := now.Add(-20 * time.Minute)
	m.metricsHistory.AddDiskMetric("SILENT-ONE-SAMPLE", "smart_temp", 44, sampledAt)

	charts := m.GetPhysicalDiskTemperatureCharts(time.Hour)
	if entry, ok := charts["STANDBY-NO-HISTORY"]; ok {
		t.Fatalf("a standby disk with no stored samples got a series from its retained reading: %+v", entry.Temperature)
	}
	silentEntry, ok := charts["SILENT-ONE-SAMPLE"]
	if !ok || len(silentEntry.Temperature) != 1 || silentEntry.Temperature[0].Value != 44 ||
		!silentEntry.Temperature[0].Timestamp.Equal(sampledAt) {
		t.Fatalf("silent agent disk series = %+v (found %v), want only its stored 44C sample at %s", silentEntry.Temperature, ok, sampledAt)
	}
	live, ok := charts["LIVE-NO-HISTORY"]
	if !ok || len(live.Temperature) != 0 || live.Name != "WDC WD80EFAX" || live.Node != "node-a" {
		t.Fatalf("live disk = %+v (found %v), want its metadata with no fabricated history", live, ok)
	}
	m.metricsHistory.AddDiskMetric("LIVE-NO-HISTORY", "smart_temp", 37, sampledAt)
	live, ok = m.GetPhysicalDiskTemperatureCharts(time.Hour)["LIVE-NO-HISTORY"]
	if !ok || len(live.Temperature) != 1 || live.Temperature[0].Value != 37 ||
		!live.Temperature[0].Timestamp.Equal(sampledAt) {
		t.Fatalf("live disk series = %+v (found %v), want only its observed 37C sample at %s", live.Temperature, ok, sampledAt)
	}
}

func TestSLO_GetDiskMetricsForChart_WithNativeStoreFallback(t *testing.T) {
	skipMonitoringSLOUnderRace(t)
	suppressMonitoringTestLogs(t)

	monitor := newChartFallbackTestMonitor(t)
	now := time.Now().UTC().Truncate(time.Second)
	duration := 4 * time.Hour

	writeRawMetricBatch(t, monitor.metricsStore, "disk", "SERIAL-DISK-IO-1", "diskread", []MetricPoint{
		{Timestamp: now.Add(-2 * time.Hour), Value: 1.25 * 1024 * 1024},
		{Timestamp: now.Add(-1 * time.Hour), Value: 2.5 * 1024 * 1024},
		{Timestamp: now, Value: 3.75 * 1024 * 1024},
	})

	sanity := monitor.GetDiskMetricsForChart("SERIAL-DISK-IO-1", "diskread", duration)
	if got := len(sanity); got != 3 {
		t.Fatalf("sanity: expected native diskread history, got %+v", sanity)
	}

	latencies := measureMonitoringLatencies(t, func() {
		result := monitor.GetDiskMetricsForChart("SERIAL-DISK-IO-1", "diskread", duration)
		if got := len(result); got != 3 {
			t.Fatalf("expected native diskread history, got %+v", result)
		}
	})

	target := effectiveMonitoringSLOTarget(SLOPhysicalDiskChartFallbackP95, SLOPhysicalDiskChartFallbackGHA)
	assertLatencySLO(t, "GetDiskMetricsForChart(native-store fallback)", latencies, target)
}

func TestSLO_GetGuestMetricsForChart_WithNativeHistoryFallback(t *testing.T) {
	skipMonitoringSLOUnderRace(t)
	suppressMonitoringTestLogs(t)

	monitor := newChartFallbackTestMonitor(t)
	now := time.Now().UTC().Truncate(time.Second)
	duration := 4 * time.Hour

	writeRawMetricBatch(t, monitor.metricsStore, "agent", "truenas-main", "cpu", []MetricPoint{
		{Timestamp: now.Add(-5 * time.Minute), Value: 41},
	})
	monitor.supplementalProviders = map[unifiedresources.DataSource]MonitorSupplementalRecordsProvider{
		unifiedresources.SourceTrueNAS: &stubGuestMetricHistoryProvider{
			history: map[string]map[string][]MetricPoint{
				"truenas-main": {
					"cpu": {
						{Timestamp: now.Add(-2 * time.Hour), Value: 20},
						{Timestamp: now.Add(-1 * time.Hour), Value: 28},
						{Timestamp: now, Value: 34},
					},
					"memory": {
						{Timestamp: now.Add(-2 * time.Hour), Value: 45},
						{Timestamp: now, Value: 62},
					},
				},
			},
		},
	}

	sanity := monitor.GetGuestMetricsForChart("agent:truenas-main", "agent", "truenas-main", duration)
	if got := len(sanity["cpu"]); got != 3 {
		t.Fatalf("sanity: expected native cpu history, got %+v", sanity["cpu"])
	}

	latencies := measureMonitoringLatencies(t, func() {
		result := monitor.GetGuestMetricsForChart("agent:truenas-main", "agent", "truenas-main", duration)
		if got := len(result["cpu"]); got != 3 {
			t.Fatalf("expected native cpu history, got %+v", result["cpu"])
		}
		if got := len(result["memory"]); got != 2 {
			t.Fatalf("expected native memory history, got %+v", result["memory"])
		}
	})

	target := effectiveMonitoringSLOTarget(SLOGuestChartFallbackP95, SLOGuestChartFallbackGHA)
	assertLatencySLO(t, "GetGuestMetricsForChart(native-history fallback)", latencies, target)
}

func TestSLO_GetGuestMetricsForChartBatch_DoesNotStitchSparseStoreTailOntoCoveredInMemorySeries(t *testing.T) {
	previous := mock.IsMockEnabled()
	mustSetMockEnabled(t, true)
	t.Cleanup(func() { mustSetMockEnabled(t, previous) })

	monitor := newChartFallbackTestMonitor(t)
	now := time.Now().UTC().Truncate(time.Second)
	duration := time.Hour

	inMemorySeries := []MetricPoint{
		{Timestamp: now.Add(-58 * time.Minute), Value: 74},
		{Timestamp: now.Add(-36 * time.Minute), Value: 75},
		{Timestamp: now.Add(-14 * time.Minute), Value: 77},
		{Timestamp: now.Add(-2 * time.Minute), Value: 78},
	}
	for _, point := range inMemorySeries {
		monitor.metricsHistory.AddGuestMetric("vm-1", "memory", point.Value, point.Timestamp)
	}

	// This sparse late store point models the old stitched-tail behavior. In
	// mock mode the batch chart path must ignore persisted history entirely and
	// reuse the seeded in-memory mock history instead of synthesizing or
	// stitching a different tail.
	writeRawMetricBatch(t, monitor.metricsStore, "vm", "vm-1", "memory", []MetricPoint{
		{Timestamp: now.Add(-1 * time.Minute), Value: 21},
	})

	result := monitor.GetGuestMetricsForChartBatch("vm", []GuestChartRequest{{
		InMemoryKey:   "vm-1",
		SQLResourceID: "vm-1",
	}}, duration)

	memoryPoints := result["vm-1"]["memory"]
	if len(memoryPoints) != len(inMemorySeries) {
		t.Fatalf("expected seeded mock memory series, got %+v", memoryPoints)
	}
	for idx, want := range inMemorySeries {
		if memoryPoints[idx].Timestamp != want.Timestamp || memoryPoints[idx].Value != want.Value {
			t.Fatalf("memoryPoints[%d] = %+v, want seeded %+v", idx, memoryPoints[idx], want)
		}
	}
}

func TestMockChartCacheInvalidatesAfterMockHistoryRefresh(t *testing.T) {
	previous := mock.IsMockEnabled()
	mustSetMockEnabled(t, true)
	t.Cleanup(func() { mustSetMockEnabled(t, previous) })

	monitor := newChartFallbackTestMonitor(t)
	now := time.Now().UTC().Truncate(time.Second)
	duration := time.Hour

	for _, point := range []MetricPoint{
		{Timestamp: now.Add(-58 * time.Minute), Value: 41},
		{Timestamp: now.Add(-36 * time.Minute), Value: 43},
		{Timestamp: now.Add(-14 * time.Minute), Value: 47},
		{Timestamp: now.Add(-2 * time.Minute), Value: 49},
	} {
		monitor.metricsHistory.AddGuestMetric("vm-1", "cpu", point.Value, point.Timestamp)
	}

	first := monitor.GetGuestMetricsForChartBatch("vm", []GuestChartRequest{{
		InMemoryKey:   "vm-1",
		SQLResourceID: "vm-1",
	}}, duration)
	firstCPU := first["vm-1"]["cpu"]
	if len(firstCPU) == 0 {
		t.Fatal("expected cached mock guest chart series")
	}
	if got := len(monitor.mockChartMapCache); got == 0 {
		t.Fatal("expected mock chart cache to populate after chart read")
	}

	newestTimestamp := now.Add(-30 * time.Second)
	monitor.metricsHistory.AddGuestMetric("vm-1", "cpu", 99, newestTimestamp)

	monitor.invalidateMockChartCaches()
	if got := len(monitor.mockChartMapCache); got != 0 {
		t.Fatalf("expected mock chart cache to clear after invalidation, got %d entries", got)
	}

	refreshed := monitor.GetGuestMetricsForChartBatch("vm", []GuestChartRequest{{
		InMemoryKey:   "vm-1",
		SQLResourceID: "vm-1",
	}}, duration)
	refreshedCPU := refreshed["vm-1"]["cpu"]
	if len(refreshedCPU) == 0 {
		t.Fatal("expected refreshed mock guest chart series")
	}

	lastPoint := refreshedCPU[len(refreshedCPU)-1]
	if !lastPoint.Timestamp.Equal(newestTimestamp) || lastPoint.Value != 99 {
		t.Fatalf("expected refreshed mock chart tail %+v, got %+v", MetricPoint{
			Timestamp: newestTimestamp,
			Value:     99,
		}, lastPoint)
	}
}

// A storage chart request totals each pool whose chart is not cached, and a
// linked pool takes its capacity from whichever source was seen last, which
// moves without a structure change, so the totals read the data-version view.
// A batch hands every pool one shared read (GetStorageMetricsForChartBatch):
// the view is rebuilt after a tick, and on a starved process a read per pool
// rebuilt it for each. A pool whose chart is cached reads nothing.
func TestMockStorageChartReadsTheEstateOnlyForAnUncachedChart(t *testing.T) {
	useMockEstate(t, 4, 5*time.Minute)
	m := newMockEstateMonitor()
	pools := m.currentUnifiedStateView().readState.StoragePools()
	if len(pools) == 0 {
		t.Fatal("the mock estate lists no storage pool")
	}
	id := pools[0].SourceID()

	reads := 0
	readState := func() unifiedresources.ReadState {
		reads++
		return m.GetUnifiedReadStateOrSnapshot()
	}
	first := m.mockStorageMetricsForChartCached(id, time.Hour, nil, readState)
	if reads != 1 || len(first["total"]) == 0 {
		t.Fatalf("an uncached chart read the estate %d times and returned %d total points, want one read and a total series", reads, len(first["total"]))
	}
	m.mockStorageMetricsForChartCached(id, time.Hour, nil, readState)
	if reads != 1 {
		t.Fatalf("a cached chart read the estate again: %d reads", reads)
	}
}

// The poller writes every metric of a resource at the same timestamps, and
// the drawer history panels break a line wherever another series in the panel
// has a reading this one lacks. Mock history must therefore keep one shared
// set of timestamps per resource, or every mock chart draws as dots.
var (
	mockGuestSharedTimestampMetrics = []string{"cpu", "memory", "memoryused", "disk", "diskread", "diskwrite", "netin", "netout"}
	mockNodeSharedTimestampMetrics  = []string{"cpu", "memory", "disk", "netin", "netout"}
	mockSharedTimestampDurations    = []time.Duration{time.Hour, 4 * time.Hour, 24 * time.Hour, 7 * 24 * time.Hour}
)

func TestSampleMetricSeriesOnMockChartGridKeepsTimestampsIndependentOfValues(t *testing.T) {
	start := time.Date(2026, time.October, 13, 0, 0, 0, 0, time.UTC)
	flat := make([]MetricPoint, 0, 721)
	spiky := make([]MetricPoint, 0, 721)
	for i := 0; i <= 720; i++ {
		ts := start.Add(time.Duration(i) * 2 * time.Minute)
		flat = append(flat, MetricPoint{Timestamp: ts, Value: 20 + math.Sin(float64(i)/40)})
		value := 5.0
		if i%37 == 0 || i%53 == 0 {
			value = 90
		}
		spiky = append(spiky, MetricPoint{Timestamp: ts, Value: value})
	}

	// Control: LTTB picks points by value, so these two series keep
	// different timestamps. This is the behaviour that broke mock charts.
	if sameMetricTimestamps(lttb(cloneMetricSeries(flat), 48), lttb(cloneMetricSeries(spiky), 48)) {
		t.Fatal("control: expected LTTB to keep different timestamps for differently shaped series")
	}

	interval := mockChartGridInterval(24 * time.Hour)
	gridFlat := sampleMetricSeriesOnMockChartGrid(flat, interval)
	gridSpiky := sampleMetricSeriesOnMockChartGrid(spiky, interval)
	if !sameMetricTimestamps(gridFlat, gridSpiky) {
		t.Fatalf("grid sampling kept different timestamps: %v vs %v", metricTimestamps(gridFlat), metricTimestamps(gridSpiky))
	}
	if got, limit := len(gridFlat), mockChartPointTarget(24*time.Hour); got > limit || got < limit-2 {
		t.Fatalf("expected grid sampling to stay within two points of %d, got %d", limit, got)
	}
	if first := gridSpiky[0]; first != spiky[0] {
		t.Fatalf("expected the first reading to be kept as-is, got %+v want %+v", first, spiky[0])
	}
	if last, newest := gridSpiky[len(gridSpiky)-1], spiky[len(spiky)-1]; last != newest {
		t.Fatalf("expected the newest reading to be kept as-is, got %+v want %+v", last, newest)
	}
	readings := make(map[int64]float64, len(spiky))
	for _, point := range spiky {
		readings[point.Timestamp.UnixNano()] = point.Value
	}
	for i, point := range gridSpiky {
		if i > 0 && !point.Timestamp.After(gridSpiky[i-1].Timestamp) {
			t.Fatalf("grid timestamps not strictly ascending at %d", i)
		}
		if value, ok := readings[point.Timestamp.UnixNano()]; !ok || value != point.Value {
			t.Fatalf("grid point %+v is not a reading from the input series", point)
		}
	}
}

func TestMockChartHistorySharesTimestampsAcrossMetrics(t *testing.T) {
	previousEnabled := mock.IsMockEnabled()
	previousConfig := mock.GetConfig()
	t.Cleanup(func() {
		mustSetMockEnabled(t, false)
		mock.SetMockConfig(previousConfig)
		if previousEnabled {
			mustSetMockEnabled(t, true)
			mock.SetMockConfig(previousConfig)
		}
	})

	// The first mock node is "mixed": at least base-2 VMs and base-3 LXCs.
	cfg := compactMockChartFixtureConfig()
	cfg.VMsPerNode = 3
	cfg.LXCsPerNode = 4
	mustSetMockEnabled(t, false)
	mock.SetMockConfig(cfg)
	mustSetMockEnabled(t, true)

	graph := mock.CurrentFixtureGraph()
	if len(graph.State.Containers) == 0 || len(graph.State.VMs) == 0 || len(graph.State.Nodes) == 0 {
		t.Fatalf(
			"expected mock nodes, VMs and containers in the fixture graph, got %d nodes, %d VMs, %d containers",
			len(graph.State.Nodes), len(graph.State.VMs), len(graph.State.Containers),
		)
	}
	container := graph.State.Containers[0].ID
	vm := graph.State.VMs[0].ID
	node := graph.State.Nodes[0].ID

	newMonitor := func() *Monitor {
		resources, _ := mock.UnifiedResourceSnapshot()
		registry := unifiedresources.NewRegistry(nil)
		registry.IngestResources(resources)
		return &Monitor{
			metricsHistory: NewMetricsHistory(1000, 24*time.Hour),
			state:          models.NewState(),
			resourceStore:  unifiedresources.NewMonitorAdapter(registry),
		}
	}

	check := func(t *testing.T, monitor *Monitor) {
		for _, duration := range mockSharedTimestampDurations {
			// The newest reading sits on the current minute; rerun if the
			// minute turned over between two per-metric node reads.
			for attempt := 0; ; attempt++ {
				minute := time.Now().Truncate(time.Minute)
				monitor.invalidateMockChartCaches()
				guests := map[string]map[string][]MetricPoint{
					"container " + container: monitor.GetGuestMetricsForChart(container, "container", container, duration),
					"vm " + vm:               monitor.GetGuestMetricsForChart(vm, "vm", vm, duration),
				}
				nodeSeries := make(map[string][]MetricPoint, len(mockNodeSharedTimestampMetrics))
				for _, metric := range mockNodeSharedTimestampMetrics {
					nodeSeries[metric] = monitor.GetNodeMetricsForChart(node, metric, duration)
				}
				if !time.Now().Truncate(time.Minute).Equal(minute) && attempt < 2 {
					continue
				}
				for name, series := range guests {
					assertMockSeriesShareTimestamps(t, name, duration, series, mockGuestSharedTimestampMetrics)
				}
				assertMockSeriesShareTimestamps(t, "node "+node, duration, nodeSeries, mockNodeSharedTimestampMetrics)
				break
			}
		}
	}

	t.Run("synthetic history", func(t *testing.T) {
		check(t, newMonitor())
	})

	t.Run("seeded history", func(t *testing.T) {
		monitor := newMonitor()
		ctx, cancel := context.WithCancel(context.Background())
		defer cancel()
		monitor.startMockMetricsSampler(ctx)
		t.Cleanup(func() { monitor.stopMockMetricsSampler() })
		check(t, monitor)
	})
}

func assertMockSeriesShareTimestamps(t *testing.T, resource string, duration time.Duration, series map[string][]MetricPoint, metrics []string) {
	t.Helper()
	reference := series["cpu"]
	if len(reference) < 2 {
		t.Fatalf("%s %s: expected a cpu history, got %d points", resource, duration, len(reference))
	}
	minValue, maxValue := reference[0].Value, reference[0].Value
	for _, point := range reference {
		minValue = math.Min(minValue, point.Value)
		maxValue = math.Max(maxValue, point.Value)
	}
	if minValue == maxValue {
		t.Fatalf("%s %s: cpu history is flat at %v; mock values should still vary", resource, duration, minValue)
	}
	for _, metric := range metrics {
		points := series[metric]
		if len(points) == 0 {
			t.Fatalf("%s %s: expected a %s history", resource, duration, metric)
		}
		if !sameMetricTimestamps(reference, points) {
			t.Fatalf(
				"%s %s: %s timestamps diverge from cpu (%d vs %d points, %d shared)",
				resource, duration, metric, len(points), len(reference), sharedMetricTimestampCount(reference, points),
			)
		}
	}
}

func metricTimestamps(points []MetricPoint) []int64 {
	out := make([]int64, len(points))
	for i, point := range points {
		out[i] = point.Timestamp.UnixNano()
	}
	return out
}

func sameMetricTimestamps(a, b []MetricPoint) bool {
	if len(a) != len(b) {
		return false
	}
	for i := range a {
		if !a[i].Timestamp.Equal(b[i].Timestamp) {
			return false
		}
	}
	return true
}

func sharedMetricTimestampCount(a, b []MetricPoint) int {
	seen := make(map[int64]struct{}, len(a))
	for _, point := range a {
		seen[point.Timestamp.UnixNano()] = struct{}{}
	}
	shared := 0
	for _, point := range b {
		if _, ok := seen[point.Timestamp.UnixNano()]; ok {
			shared++
		}
	}
	return shared
}

func TestStoreValueProjectionPreservesChartSampling(t *testing.T) {
	store, err := metrics.NewStore(metrics.DefaultConfig(t.TempDir()))
	if err != nil {
		t.Fatal(err)
	}
	defer store.Close()
	end := time.Now().Truncate(time.Minute)
	start := end.Add(-24 * time.Hour)
	batch := make([]metrics.WriteMetric, 0, 4000)
	ids := []string{"a", "b"}
	names := []string{"cpu", "memory"}
	for _, id := range ids {
		for _, metric := range names {
			for i := 0; i < 1000; i++ {
				batch = append(batch, metrics.WriteMetric{ResourceType: "vm", ResourceID: id, MetricType: metric, Value: float64((i * 37) % 97), Timestamp: end.Add(-18 * time.Hour).Add(time.Duration(i) * time.Minute), Tier: metrics.TierMinute})
			}
		}
	}
	store.WriteBatchSync(batch)
	monitor := &Monitor{metricsStore: store}
	for _, filter := range [][]string{nil, {"cpu"}, {"cpu", "memory"}} {
		t.Run(fmt.Sprint(filter), func(t *testing.T) {
			full, err := store.QueryMetricTypesBatch("vm", ids, filter, start, end, 0)
			if err != nil {
				t.Fatal(err)
			}
			want := make(map[string]map[string][]MetricPoint, len(full))
			for id, series := range full {
				want[id] = make(map[string][]MetricPoint, len(series))
				for metric, points := range series {
					if len(points) != 1000 {
						t.Fatalf("fixture %s/%s: got %d points", id, metric, len(points))
					}
					want[id][metric] = convertSeriesAndDownsample(points, chartDownsampleTarget)
				}
			}
			got := monitor.queryStoreBatchMetricMapWithGapFill("vm", ids, 24*time.Hour, filter)
			if !reflect.DeepEqual(got, want) {
				t.Fatal("value projection changed the sampled chart observations")
			}
			for _, series := range got {
				for _, points := range series {
					points[0].Value = -999
					points[0].Timestamp = time.Time{}
				}
			}
			again := monitor.queryStoreBatchMetricMapWithGapFill("vm", ids, 24*time.Hour, filter)
			if !reflect.DeepEqual(again, want) {
				t.Fatal("mutating a chart result changed a subsequent read")
			}
		})
	}
}