Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

32.3 KB
Raw Blame
package chartapi

import (
	"context"
	"encoding/json"
	"net/http"
	"net/http/httptest"
	"reflect"
	"sort"
	"strings"
	"testing"
	"time"
	"unsafe"

	"github.com/rcourtman/pulse-go-rewrite/internal/config"
	"github.com/rcourtman/pulse-go-rewrite/internal/models"
	"github.com/rcourtman/pulse-go-rewrite/internal/monitoring"
	"github.com/rcourtman/pulse-go-rewrite/internal/testutil"
	"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
	"github.com/rcourtman/pulse-go-rewrite/internal/vmware"
)

type fixedMonitorResolver struct {
	monitor *monitoring.Monitor
}

func (r fixedMonitorResolver) MonitorForContext(context.Context) *monitoring.Monitor {
	return r.monitor
}

func setUnexportedField(t *testing.T, target any, fieldName string, value any) {
	t.Helper()
	field := reflect.ValueOf(target).Elem().FieldByName(fieldName)
	if !field.IsValid() {
		t.Fatalf("field %q not found", fieldName)
	}
	reflect.NewAt(field.Type(), unsafe.Pointer(field.UnsafeAddr())).Elem().Set(reflect.ValueOf(value))
}

func setMockModeForTest(t *testing.T, enabled bool) {
	t.Helper()
	testutil.SetMockMode(t, enabled)
}

func newTestMonitor(t *testing.T) (*monitoring.Monitor, *models.State, *monitoring.MetricsHistory) {
	t.Helper()

	monitor := &monitoring.Monitor{}
	state := models.NewState()
	metricsHistory := monitoring.NewMetricsHistory(10, time.Hour)

	setUnexportedField(t, monitor, "state", state)
	setUnexportedField(t, monitor, "metricsHistory", metricsHistory)

	return monitor, state, metricsHistory
}

// syncTestResourceStore populates a MonitorAdapter (ResourceRegistry) from the
// legacy state so that GetUnifiedReadState() returns a valid ReadState.
// Call this after setting state.VMs, state.Nodes, etc.
func syncTestResourceStore(t *testing.T, monitor *monitoring.Monitor, state *models.State) {
	t.Helper()
	adapter := unifiedresources.NewMonitorAdapter(nil)
	adapter.PopulateFromSnapshot(state.GetSnapshot())
	setUnexportedField(t, monitor, "resourceStore", monitoring.ResourceStoreInterface(adapter))
}

func mapKeys[V any](values map[string]V) []string {
	keys := make([]string, 0, len(values))
	for key := range values {
		keys = append(keys, key)
	}
	sort.Strings(keys)
	return keys
}

func TestHandleCharts_Success(t *testing.T) {
	monitor, state, _ := newTestMonitor(t)
	state.VMs = []models.VM{{ID: "vm-1", Name: "vm-one", CPU: 0.2}}
	syncTestResourceStore(t, monitor, state)
	service := NewService(fixedMonitorResolver{monitor: monitor})

	req := httptest.NewRequest(http.MethodGet, "/api/charts?range=5m", nil)
	rec := httptest.NewRecorder()

	service.HandleCharts(rec, req)

	if rec.Code != http.StatusOK {
		t.Fatalf("expected status %d, got %d", http.StatusOK, rec.Code)
	}
	if ct := rec.Header().Get("Content-Type"); ct != "application/json" {
		t.Fatalf("expected application/json, got %q", ct)
	}
}

func TestHandleCharts_StatsDebugMetadata(t *testing.T) {
	monitor, state, _ := newTestMonitor(t)
	state.VMs = []models.VM{{ID: "vm-1", Name: "vm-one", CPU: 0.2}}
	state.Nodes = []models.Node{{ID: "node-1", Name: "node-one", CPU: 0.1}}
	state.Storage = []models.Storage{{ID: "store-1", Name: "Store One", Used: 50, Total: 100}}
	syncTestResourceStore(t, monitor, state)
	service := NewService(fixedMonitorResolver{monitor: monitor})

	req := httptest.NewRequest(http.MethodGet, "/api/charts?range=5m", nil)
	rec := httptest.NewRecorder()

	service.HandleCharts(rec, req)

	if rec.Code != http.StatusOK {
		t.Fatalf("expected status %d, got %d", http.StatusOK, rec.Code)
	}

	body := rec.Body.Bytes()

	var decoded ChartResponse
	if err := json.Unmarshal(body, &decoded); err != nil {
		t.Fatalf("unmarshal ChartResponse: %v", err)
	}

	if decoded.Stats.Range != "5m" {
		t.Fatalf("expected stats.range=5m, got %q", decoded.Stats.Range)
	}
	if decoded.Stats.RangeSeconds != 300 {
		t.Fatalf("expected stats.rangeSeconds=300, got %d", decoded.Stats.RangeSeconds)
	}
	if decoded.Stats.MetricsStoreEnabled {
		t.Fatalf("expected stats.metricsStoreEnabled=false in test monitor, got true")
	}
	if decoded.Stats.PrimarySourceHint != "memory" {
		t.Fatalf("expected stats.primarySourceHint=memory, got %q", decoded.Stats.PrimarySourceHint)
	}
	if decoded.Stats.InMemoryThresholdSecs != 7200 {
		t.Fatalf("expected stats.inMemoryThresholdSecs=7200, got %d", decoded.Stats.InMemoryThresholdSecs)
	}
	if decoded.Stats.OldestDataTimestamp <= 0 {
		t.Fatalf("expected stats.oldestDataTimestamp to be set, got %d", decoded.Stats.OldestDataTimestamp)
	}
	if decoded.Stats.OldestDataTimestamp > decoded.Timestamp {
		t.Fatalf(
			"expected stats.oldestDataTimestamp <= timestamp, got oldest=%d timestamp=%d",
			decoded.Stats.OldestDataTimestamp,
			decoded.Timestamp,
		)
	}

	// With no history in the test monitor, handleCharts falls back to synthetic points:
	// guests: cpu/memory/memoryused/disk/netin/netout (6 — diskread/diskwrite excluded from sparkline payloads)
	// nodes: cpu/memory/disk (3)
	// storage: disk (1)
	if decoded.Stats.PointCounts.Guests != 6 {
		t.Fatalf("expected stats.pointCounts.guests=6, got %d", decoded.Stats.PointCounts.Guests)
	}
	if points := decoded.ChartData["vm-1"]["memoryused"]; len(points) != 1 || points[0].Value != 0 {
		t.Fatalf("expected additive raw memory-used fallback point, got %+v", points)
	}
	if decoded.Stats.PointCounts.Nodes != 3 {
		t.Fatalf("expected stats.pointCounts.nodes=3, got %d", decoded.Stats.PointCounts.Nodes)
	}
	if decoded.Stats.PointCounts.Storage != 1 {
		t.Fatalf("expected stats.pointCounts.storage=1, got %d", decoded.Stats.PointCounts.Storage)
	}
	if decoded.Stats.PointCounts.DockerContainers != 0 || decoded.Stats.PointCounts.DockerHosts != 0 || decoded.Stats.PointCounts.Agents != 0 {
		t.Fatalf(
			"expected dockerContainers/dockerHosts/agents all 0, got dc=%d dh=%d agents=%d",
			decoded.Stats.PointCounts.DockerContainers,
			decoded.Stats.PointCounts.DockerHosts,
			decoded.Stats.PointCounts.Agents,
		)
	}

	sum := decoded.Stats.PointCounts.Guests +
		decoded.Stats.PointCounts.Nodes +
		decoded.Stats.PointCounts.Storage +
		decoded.Stats.PointCounts.DockerContainers +
		decoded.Stats.PointCounts.DockerHosts +
		decoded.Stats.PointCounts.Agents
	if decoded.Stats.PointCounts.Total != sum {
		t.Fatalf("expected stats.pointCounts.total=%d, got %d", sum, decoded.Stats.PointCounts.Total)
	}

	var raw map[string]interface{}
	if err := json.Unmarshal(body, &raw); err != nil {
		t.Fatalf("unmarshal raw JSON: %v", err)
	}
	stats, ok := raw["stats"].(map[string]interface{})
	if !ok {
		t.Fatalf("expected stats object in JSON response")
	}
	if _, ok := stats["pointCounts"]; !ok {
		t.Fatalf("expected stats.pointCounts to be present in JSON response")
	}
}

func TestHandleCharts_UsesCanonicalMockUnifiedReadStateForVMwareHosts(t *testing.T) {
	setMockModeForTest(t, true)

	fixtures := vmware.DefaultFixtures()
	if len(fixtures.Hosts) == 0 {
		t.Fatal("expected default VMware fixtures to include at least one host")
	}
	expectedHostID := vmware.SourceID(fixtures.ConnectionID, "host", fixtures.Hosts[0].Host)

	monitor, state, _ := newTestMonitor(t)
	state.Hosts = []models.Host{{
		ID:       "live-store-host-1",
		Hostname: "live-store-host-1",
		CPUUsage: 11.0,
		Memory:   models.Memory{Usage: 22.0},
		Disks:    []models.Disk{{Total: 100, Used: 33, Usage: 33.0}},
		Status:   "online",
	}}
	syncTestResourceStore(t, monitor, state)

	service := NewService(fixedMonitorResolver{monitor: monitor})
	req := httptest.NewRequest(http.MethodGet, "/api/charts?range=5m", nil)
	rec := httptest.NewRecorder()

	service.HandleCharts(rec, req)

	if rec.Code != http.StatusOK {
		t.Fatalf("expected status %d, got %d: %s", http.StatusOK, rec.Code, rec.Body.String())
	}

	var decoded ChartResponse
	if err := json.Unmarshal(rec.Body.Bytes(), &decoded); err != nil {
		t.Fatalf("unmarshal ChartResponse: %v", err)
	}

	vmwareSeries, ok := decoded.AgentData[expectedHostID]
	if !ok {
		t.Fatalf("expected VMware host %q in full charts payload, got keys=%v", expectedHostID, mapKeys(decoded.AgentData))
	}
	if len(vmwareSeries["cpu"]) == 0 {
		t.Fatalf("expected VMware host %q cpu series", expectedHostID)
	}
	if _, ok := decoded.AgentData["live-store-host-1"]; ok {
		t.Fatalf("expected mock-aware full charts to ignore live store-only host, got keys=%v", mapKeys(decoded.AgentData))
	}
}

func TestChartsDoNotSynthesizeCurrentMetricsForPendingEnrollment(t *testing.T) {
	for _, age := range []time.Duration{time.Second, 4 * time.Minute} {
		t.Run(age.String(), func(t *testing.T) {
			monitor, state, history := newTestMonitor(t)
			syncTestResourceStore(t, monitor, state)
			continuity := config.NewHostContinuityStore(t.TempDir(), nil)
			if err := continuity.Upsert(config.HostContinuityEntry{HostID: "waiting-agent", Hostname: "waiting", LastSeen: time.Now().Add(-age), IntervalSeconds: 30}); err != nil {
				t.Fatal(err)
			}
			setUnexportedField(t, monitor, "hostContinuityStore", continuity)
			for _, stored := range []bool{false, true} {
				if stored {
					history.AddGuestMetric("agent:waiting-agent", "cpu", 37, time.Now().Add(-4*time.Minute))
				}
				for _, path := range []string{"/api/charts?range=5m", "/api/charts/infrastructure?range=5m"} {
					service := NewService(fixedMonitorResolver{monitor: monitor})
					rec := httptest.NewRecorder()
					req := httptest.NewRequest(http.MethodGet, path, nil)
					if strings.HasPrefix(path, "/api/charts/infrastructure") {
						service.HandleInfrastructureCharts(rec, req)
					} else {
						service.HandleCharts(rec, req)
					}
					var response struct {
						AgentData map[string]VMChartData `json:"agentData"`
					}
					if rec.Code != http.StatusOK {
						t.Fatalf("%s: status %d: %s", path, rec.Code, rec.Body.String())
					}
					if err := json.Unmarshal(rec.Body.Bytes(), &response); err != nil {
						t.Fatal(err)
					}
					series := response.AgentData["waiting-agent"]
					if _, ok := response.AgentData["waiting-agent"]; !ok {
						t.Fatalf("%s lost the pending enrollment", path)
					}
					if stored {
						if len(series["cpu"]) != 1 || series["cpu"][0].Value != 37 {
							t.Fatalf("%s lost stored history: %+v", path, series)
						}
					} else if len(series) != 0 {
						t.Fatalf("%s invented current readings: %+v", path, series)
					}
				}
			}
		})
	}
}

func TestHandleInfrastructureCharts_Lightweight(t *testing.T) {
	monitor, state, _ := newTestMonitor(t)
	state.Nodes = []models.Node{{
		ID:     "node-1",
		Name:   "node-one",
		Status: "online",
		CPU:    0.1,
		Memory: models.Memory{Usage: 12.0},
		Disk:   models.Disk{Usage: 34.0},
	}}
	state.DockerHosts = []models.DockerHost{{
		ID:       "docker-host-1",
		Runtime:  "docker",
		CPUUsage: 23.0,
		Memory:   models.Memory{Usage: 45.0},
		Disks:    []models.Disk{{Usage: 67.0}},
		Status:   "online",
	}}
	state.Hosts = []models.Host{{
		ID:       "host-1",
		Hostname: "host-one",
		CPUUsage: 11.0,
		Memory:   models.Memory{Usage: 22.0},
		Disks:    []models.Disk{{Total: 100, Used: 33, Usage: 33.0}},
		Status:   "online",
	}}
	syncTestResourceStore(t, monitor, state)
	service := NewService(fixedMonitorResolver{monitor: monitor})

	req := httptest.NewRequest(http.MethodGet, "/api/charts/infrastructure?range=5m", nil)
	rec := httptest.NewRecorder()

	service.HandleInfrastructureCharts(rec, req)

	if rec.Code != http.StatusOK {
		t.Fatalf("expected status %d, got %d", http.StatusOK, rec.Code)
	}
	if ct := rec.Header().Get("Content-Type"); ct != "application/json" {
		t.Fatalf("expected application/json, got %q", ct)
	}

	body := rec.Body.Bytes()

	var decoded InfrastructureChartsResponse
	if err := json.Unmarshal(body, &decoded); err != nil {
		t.Fatalf("unmarshal InfrastructureChartsResponse: %v", err)
	}
	if decoded.Stats.Range != "5m" {
		t.Fatalf("expected stats.range=5m, got %q", decoded.Stats.Range)
	}
	if decoded.Stats.RangeSeconds != 300 {
		t.Fatalf("expected stats.rangeSeconds=300, got %d", decoded.Stats.RangeSeconds)
	}

	// With no history in the test monitor, handler falls back to synthetic points:
	// nodes: cpu/memory/disk (3)
	// dockerHosts: cpu/memory/disk (3)
	// agents: cpu/memory/disk (3)
	if decoded.Stats.PointCounts.Nodes != 3 {
		t.Fatalf("expected stats.pointCounts.nodes=3, got %d", decoded.Stats.PointCounts.Nodes)
	}
	if decoded.Stats.PointCounts.DockerHosts != 3 {
		t.Fatalf("expected stats.pointCounts.dockerHosts=3, got %d", decoded.Stats.PointCounts.DockerHosts)
	}
	if decoded.Stats.PointCounts.Agents != 3 {
		t.Fatalf("expected stats.pointCounts.agents=3, got %d", decoded.Stats.PointCounts.Agents)
	}
	sum := decoded.Stats.PointCounts.Nodes + decoded.Stats.PointCounts.DockerHosts + decoded.Stats.PointCounts.Agents
	if decoded.Stats.PointCounts.Total != sum {
		t.Fatalf("expected stats.pointCounts.total=%d, got %d", sum, decoded.Stats.PointCounts.Total)
	}

	var raw map[string]interface{}
	if err := json.Unmarshal(body, &raw); err != nil {
		t.Fatalf("unmarshal raw JSON: %v", err)
	}
	for _, forbidden := range []string{"data", "storageData", "dockerData", "guestTypes"} {
		if _, ok := raw[forbidden]; ok {
			t.Fatalf("expected %q to be absent from infra summary response", forbidden)
		}
	}
}

func TestHandleInfrastructureCharts_MetricFilter(t *testing.T) {
	monitor, state, _ := newTestMonitor(t)
	state.Nodes = []models.Node{{
		ID:     "node-1",
		Name:   "node-one",
		Status: "online",
		CPU:    0.1,
		Memory: models.Memory{Usage: 12.0},
		Disk:   models.Disk{Usage: 34.0},
	}}
	state.DockerHosts = []models.DockerHost{{
		ID:       "docker-host-1",
		Runtime:  "docker",
		CPUUsage: 23.0,
		Memory:   models.Memory{Usage: 45.0},
		Disks:    []models.Disk{{Usage: 67.0}},
		Status:   "online",
	}}
	state.Hosts = []models.Host{{
		ID:       "host-1",
		Hostname: "host-one",
		CPUUsage: 11.0,
		Memory:   models.Memory{Usage: 22.0},
		Disks:    []models.Disk{{Total: 100, Used: 33, Usage: 33.0}},
		Status:   "online",
	}}
	syncTestResourceStore(t, monitor, state)
	service := NewService(fixedMonitorResolver{monitor: monitor})

	req := httptest.NewRequest(http.MethodGet, "/api/charts/infrastructure?range=5m&metrics=cpu,memory", nil)
	rec := httptest.NewRecorder()

	service.HandleInfrastructureCharts(rec, req)

	if rec.Code != http.StatusOK {
		t.Fatalf("expected status %d, got %d", http.StatusOK, rec.Code)
	}

	var decoded InfrastructureChartsResponse
	if err := json.Unmarshal(rec.Body.Bytes(), &decoded); err != nil {
		t.Fatalf("unmarshal InfrastructureChartsResponse: %v", err)
	}

	if decoded.Stats.PointCounts.Nodes != 2 {
		t.Fatalf("expected stats.pointCounts.nodes=2, got %d", decoded.Stats.PointCounts.Nodes)
	}
	if decoded.Stats.PointCounts.DockerHosts != 2 {
		t.Fatalf("expected stats.pointCounts.dockerHosts=2, got %d", decoded.Stats.PointCounts.DockerHosts)
	}
	if decoded.Stats.PointCounts.Agents != 2 {
		t.Fatalf("expected stats.pointCounts.agents=2, got %d", decoded.Stats.PointCounts.Agents)
	}
	if _, ok := decoded.NodeData["node-1"]["disk"]; ok {
		t.Fatalf("expected disk series to be filtered out of node payload")
	}
	if _, ok := decoded.DockerHostData["docker-host-1"]["disk"]; ok {
		t.Fatalf("expected disk series to be filtered out of docker host payload")
	}
	if _, ok := decoded.AgentData["host-1"]["disk"]; ok {
		t.Fatalf("expected disk series to be filtered out of agent payload")
	}
}

func TestHandleInfrastructureCharts_UsesCanonicalMockUnifiedReadStateForVMwareHosts(t *testing.T) {
	setMockModeForTest(t, true)

	fixtures := vmware.DefaultFixtures()
	if len(fixtures.Hosts) == 0 {
		t.Fatal("expected default VMware fixtures to include at least one host")
	}
	expectedHostID := vmware.SourceID(fixtures.ConnectionID, "host", fixtures.Hosts[0].Host)

	monitor, state, _ := newTestMonitor(t)
	state.Hosts = []models.Host{{
		ID:       "live-store-host-1",
		Hostname: "live-store-host-1",
		CPUUsage: 11.0,
		Memory:   models.Memory{Usage: 22.0},
		Disks:    []models.Disk{{Total: 100, Used: 33, Usage: 33.0}},
		Status:   "online",
	}}
	syncTestResourceStore(t, monitor, state)

	service := NewService(fixedMonitorResolver{monitor: monitor})
	req := httptest.NewRequest(http.MethodGet, "/api/charts/infrastructure?range=5m", nil)
	rec := httptest.NewRecorder()

	service.HandleInfrastructureCharts(rec, req)

	if rec.Code != http.StatusOK {
		t.Fatalf("expected status %d, got %d: %s", http.StatusOK, rec.Code, rec.Body.String())
	}

	var decoded InfrastructureChartsResponse
	if err := json.Unmarshal(rec.Body.Bytes(), &decoded); err != nil {
		t.Fatalf("unmarshal InfrastructureChartsResponse: %v", err)
	}

	vmwareSeries, ok := decoded.AgentData[expectedHostID]
	if !ok {
		t.Fatalf("expected VMware host %q in infrastructure charts, got keys=%v", expectedHostID, mapKeys(decoded.AgentData))
	}
	if len(vmwareSeries["cpu"]) == 0 {
		t.Fatalf("expected VMware host %q cpu series", expectedHostID)
	}
	if _, ok := decoded.AgentData["live-store-host-1"]; ok {
		t.Fatalf("expected mock-aware infrastructure charts to ignore live store-only host, got keys=%v", mapKeys(decoded.AgentData))
	}
}

func TestHandleWorkloadCharts_WorkloadOnlyPayloadAndNodeFilter(t *testing.T) {
	monitor, state, _ := newTestMonitor(t)
	state.Nodes = []models.Node{{
		ID:       "node-pve-1",
		Name:     "pve-1",
		Instance: "pve",
	}}
	state.VMs = []models.VM{
		{
			ID:       "vm-101",
			Name:     "vm-101",
			Node:     "pve-1",
			Instance: "pve",
			CPU:      0.2,
			Memory:   models.Memory{Usage: 55},
			Disk:     models.Disk{Usage: 40},
		},
		{
			ID:       "vm-202",
			Name:     "vm-202",
			Node:     "other",
			Instance: "other",
			CPU:      0.4,
			Memory:   models.Memory{Usage: 65},
			Disk:     models.Disk{Usage: 70},
		},
	}
	state.Containers = []models.Container{
		{
			ID:       "ct-301",
			Name:     "ct-301",
			Node:     "pve-1",
			Instance: "pve",
			CPU:      0.3,
			Memory:   models.Memory{Usage: 40},
			Disk:     models.Disk{Usage: 25},
		},
	}
	state.DockerHosts = []models.DockerHost{{
		ID:          "docker-host-1",
		Hostname:    "pve-1",
		DisplayName: "pve-1",
		Containers: []models.DockerContainer{{
			ID:            "docker-401",
			Name:          "docker-401",
			CPUPercent:    22,
			MemoryPercent: 30,
		}},
	}}

	readState := monitor.GetUnifiedReadStateOrSnapshot()
	if readState == nil {
		t.Fatal("expected unified read state")
	}

	vmResponseKey := ""
	containerResponseKey := ""
	excludedResponseKey := ""
	for _, vm := range readState.VMs() {
		if vm == nil {
			continue
		}
		switch vm.Name() {
		case "vm-101":
			vmResponseKey, _, _ = VMChartRequest(vm)
		case "vm-202":
			excludedResponseKey, _, _ = VMChartRequest(vm)
		}
	}
	for _, ct := range readState.Containers() {
		if ct == nil {
			continue
		}
		if ct.Name() == "ct-301" {
			containerResponseKey, _, _ = SystemContainerChartRequest(ct)
		}
	}
	if vmResponseKey == "" || containerResponseKey == "" || excludedResponseKey == "" {
		t.Fatalf(
			"expected canonical response keys for vm/container test fixtures, got vm=%q ct=%q excluded=%q",
			vmResponseKey,
			containerResponseKey,
			excludedResponseKey,
		)
	}

	service := NewService(fixedMonitorResolver{monitor: monitor})

	req := httptest.NewRequest(http.MethodGet, "/api/charts/workloads?range=5m", nil)
	rec := httptest.NewRecorder()
	service.HandleWorkloadCharts(rec, req)

	if rec.Code != http.StatusOK {
		t.Fatalf("expected status %d, got %d", http.StatusOK, rec.Code)
	}
	if ct := rec.Header().Get("Content-Type"); ct != "application/json" {
		t.Fatalf("expected application/json, got %q", ct)
	}

	var decoded WorkloadChartsResponse
	if err := json.Unmarshal(rec.Body.Bytes(), &decoded); err != nil {
		t.Fatalf("unmarshal WorkloadChartsResponse: %v", err)
	}

	if decoded.Stats.Range != "5m" {
		t.Fatalf("expected stats.range=5m, got %q", decoded.Stats.Range)
	}
	if decoded.Stats.RangeSeconds != 300 {
		t.Fatalf("expected stats.rangeSeconds=300, got %d", decoded.Stats.RangeSeconds)
	}
	if len(decoded.ChartData) != 3 {
		t.Fatalf("expected 3 workload chart entries, got %d", len(decoded.ChartData))
	}
	if len(decoded.DockerData) != 1 {
		t.Fatalf("expected 1 docker chart entry, got %d", len(decoded.DockerData))
	}
	if decoded.GuestTypes[vmResponseKey] != "vm" {
		t.Fatalf("expected vm guest type for %s, got %q", vmResponseKey, decoded.GuestTypes[vmResponseKey])
	}
	if decoded.GuestTypes[containerResponseKey] != "system-container" {
		t.Fatalf(
			"expected system-container guest type for %s, got %q",
			containerResponseKey,
			decoded.GuestTypes[containerResponseKey],
		)
	}
	if decoded.Stats.PointCounts.Total <= 0 {
		t.Fatalf("expected stats.pointCounts.total > 0, got %d", decoded.Stats.PointCounts.Total)
	}

	// Node-scoped request should only include workloads linked to node-pve-1
	// plus docker containers running on matching docker hosts.
	nodeReq := httptest.NewRequest(
		http.MethodGet,
		"/api/charts/workloads?range=5m&node=node-pve-1",
		nil,
	)
	nodeRec := httptest.NewRecorder()
	service.HandleWorkloadCharts(nodeRec, nodeReq)

	if nodeRec.Code != http.StatusOK {
		t.Fatalf("node-scoped expected status %d, got %d", http.StatusOK, nodeRec.Code)
	}

	var scoped WorkloadChartsResponse
	if err := json.Unmarshal(nodeRec.Body.Bytes(), &scoped); err != nil {
		t.Fatalf("unmarshal node-scoped WorkloadChartsResponse: %v", err)
	}
	if len(scoped.ChartData) != 2 {
		t.Fatalf("expected 2 scoped workload chart entries, got %d", len(scoped.ChartData))
	}
	if _, ok := scoped.ChartData[excludedResponseKey]; ok {
		t.Fatalf("expected %s to be excluded by node scope", excludedResponseKey)
	}
	if len(scoped.DockerData) != 1 {
		t.Fatalf("expected 1 scoped docker chart entry, got %d", len(scoped.DockerData))
	}
}

func TestHandleWorkloadCharts_UsesCanonicalWorkloadIDsForVMwareVMs(t *testing.T) {
	monitor, state, history := newTestMonitor(t)
	now := time.Now().UTC().Add(-10 * time.Minute).Truncate(time.Minute)
	metricID := "vc-1:vm:vm-201"
	resourceID := "vm-vmware-1"

	history.AddGuestMetric(metricID, "cpu", 37, now.Add(-10*time.Minute))
	history.AddGuestMetric(metricID, "memory", 62, now.Add(-5*time.Minute))

	adapter := unifiedresources.NewMonitorAdapter(nil)
	adapter.PopulateSnapshotAndSupplemental(state.GetSnapshot(), map[unifiedresources.DataSource][]unifiedresources.IngestRecord{
		unifiedresources.SourceVMware: {
			{
				SourceID: metricID,
				Resource: unifiedresources.Resource{
					ID:       resourceID,
					Type:     unifiedresources.ResourceTypeVM,
					Name:     "warehouse-api-01",
					Status:   unifiedresources.StatusOnline,
					LastSeen: now,
					MetricsTarget: &unifiedresources.MetricsTarget{
						ResourceType: "vm",
						ResourceID:   metricID,
					},
					VMware: &unifiedresources.VMwareData{
						ConnectionID:    "vc-1",
						EntityType:      "vm",
						ManagedObjectID: "vm-201",
					},
				},
			},
		},
	})
	setUnexportedField(t, monitor, "resourceStore", monitoring.ResourceStoreInterface(adapter))

	readState := monitor.GetUnifiedReadStateOrSnapshot()
	if readState == nil || len(readState.VMs()) != 1 || readState.VMs()[0] == nil {
		t.Fatalf("expected one VMware VM in unified read state, got %+v", readState)
	}
	resourceID, _, ok := VMChartRequest(readState.VMs()[0])
	if !ok {
		t.Fatal("expected canonical VMware vm chart request")
	}

	service := NewService(fixedMonitorResolver{monitor: monitor})
	req := httptest.NewRequest(http.MethodGet, "/api/charts/workloads?range=1h", nil)
	rec := httptest.NewRecorder()
	service.HandleWorkloadCharts(rec, req)

	if rec.Code != http.StatusOK {
		t.Fatalf("expected status %d, got %d: %s", http.StatusOK, rec.Code, rec.Body.String())
	}

	var decoded WorkloadChartsResponse
	if err := json.Unmarshal(rec.Body.Bytes(), &decoded); err != nil {
		t.Fatalf("unmarshal WorkloadChartsResponse: %v", err)
	}

	series, ok := decoded.ChartData[resourceID]
	if !ok {
		t.Fatalf("expected VMware workload chart keyed by canonical workload id %q, got %v", resourceID, decoded.ChartData)
	}
	if _, ok := decoded.ChartData[metricID]; ok {
		t.Fatalf("expected VMware metrics target id %q to stay out of workload chart response keys", metricID)
	}
	if decoded.GuestTypes[resourceID] != "vm" {
		t.Fatalf("expected guest type vm for %q, got %q", resourceID, decoded.GuestTypes[resourceID])
	}
	if len(series["cpu"]) == 0 {
		t.Fatalf("expected VMware cpu series for %q", resourceID)
	}
	if got := series["cpu"][len(series["cpu"])-1].Value; got != 37 {
		t.Fatalf("expected latest VMware cpu value 37, got %v", got)
	}
}

func TestHandleWorkloadCharts_IncludesKubernetesPods(t *testing.T) {
	setMockModeForTest(t, false)

	monitor, state, _ := newTestMonitor(t)
	state.Nodes = []models.Node{{
		ID:       "node-pve-1",
		Name:     "pve-1",
		Instance: "pve",
	}}
	state.KubernetesClusters = []models.KubernetesCluster{{
		ID:     "cluster-k8s-summary-test",
		Name:   "cluster-k8s-summary-test",
		Status: "online",
		Pods: []models.KubernetesPod{
			{
				UID:       "pod-001",
				Name:      "api-0",
				Namespace: "default",
				NodeName:  "pve-1",
				Phase:     "Running",
				Containers: []models.KubernetesPodContainer{
					{Name: "api", Ready: true},
				},
			},
		},
	}}

	service := NewService(fixedMonitorResolver{monitor: monitor})
	req := httptest.NewRequest(http.MethodGet, "/api/charts/workloads?range=5m", nil)
	rec := httptest.NewRecorder()
	service.HandleWorkloadCharts(rec, req)

	if rec.Code != http.StatusOK {
		t.Fatalf("expected status %d, got %d", http.StatusOK, rec.Code)
	}

	var decoded WorkloadChartsResponse
	if err := json.Unmarshal(rec.Body.Bytes(), &decoded); err != nil {
		t.Fatalf("unmarshal WorkloadChartsResponse: %v", err)
	}

	var metricID string
	for id, typ := range decoded.GuestTypes {
		if typ == "k8s" {
			metricID = id
			break
		}
	}
	if metricID == "" {
		t.Fatal("expected at least one kubernetes pod series")
	}

	series, ok := decoded.ChartData[metricID]
	if !ok {
		t.Fatalf("expected kubernetes pod series for %s", metricID)
	}
	if len(series["cpu"]) == 0 {
		t.Fatalf("expected kubernetes cpu points for %s", metricID)
	}
	if got := len(series["disk"]); got != 1 || series["disk"][0].Value != 0 {
		t.Fatalf("expected unsupported kubernetes pod disk metric fallback to single zero point; got len=%d value=%v", got, series["disk"])
	}
	if got := len(series["netin"]); got != 1 || series["netin"][0].Value != 0 {
		t.Fatalf("expected unsupported kubernetes pod network metric fallback to single zero point; got len=%d value=%v", got, series["netin"])
	}
}

func TestHandleWorkloadCharts_UnknownNodeFilterFallsBackToGlobalScope(t *testing.T) {
	monitor, state, _ := newTestMonitor(t)
	state.Nodes = []models.Node{{
		ID:       "node-pve-1",
		Name:     "pve-1",
		Instance: "pve",
	}}
	state.VMs = []models.VM{{
		ID:       "vm-101",
		Name:     "vm-101",
		Node:     "pve-1",
		Instance: "pve",
		CPU:      0.3,
		Memory:   models.Memory{Total: 100, Used: 42, Free: 58, Usage: 42},
		Disk:     models.Disk{Usage: 55},
	}}

	service := NewService(fixedMonitorResolver{monitor: monitor})
	req := httptest.NewRequest(http.MethodGet, "/api/charts/workloads?range=5m&node=missing-node-id", nil)
	rec := httptest.NewRecorder()

	service.HandleWorkloadCharts(rec, req)

	if rec.Code != http.StatusOK {
		t.Fatalf("expected status %d, got %d", http.StatusOK, rec.Code)
	}

	var decoded WorkloadChartsResponse
	if err := json.Unmarshal(rec.Body.Bytes(), &decoded); err != nil {
		t.Fatalf("unmarshal WorkloadChartsResponse: %v", err)
	}
	if len(decoded.ChartData) == 0 {
		t.Fatalf("expected fallback to global scope when node filter is stale")
	}
	for _, series := range decoded.ChartData {
		if points := series["memoryused"]; len(points) != 1 || points[0].Value != 42 {
			t.Fatalf("expected raw memory-used workload fallback point, got %+v", points)
		}
	}
}

func TestHandleStorageCharts_Success(t *testing.T) {
	monitor, state, metricsHistory := newTestMonitor(t)
	state.Storage = []models.Storage{{ID: "store-1", Name: "Store One"}}
	metricsHistory.AddStorageMetric("store-1", "usage", 0.4, time.Now())
	syncTestResourceStore(t, monitor, state)

	service := NewService(fixedMonitorResolver{monitor: monitor})
	req := httptest.NewRequest(http.MethodGet, "/api/storage/charts?range=30", nil)
	rec := httptest.NewRecorder()

	service.HandleStorageCharts(rec, req)

	if rec.Code != http.StatusOK {
		t.Fatalf("expected status %d, got %d", http.StatusOK, rec.Code)
	}
	if ct := rec.Header().Get("Content-Type"); ct != "application/json" {
		t.Fatalf("expected application/json, got %q", ct)
	}
}

func TestHandleStorageCharts_IncludesSupplementalStorageAndResolvesUnifiedNodeFilter(t *testing.T) {
	monitor, state, metricsHistory := newTestMonitor(t)
	now := time.Now()
	metricsHistory.AddStorageMetric("vc-1:datastore:datastore-202", "usage", 0.25, now)
	metricsHistory.AddStorageMetric("vc-1:datastore:datastore-202", "used", 3.57*1024*1024*1024*1024, now)
	metricsHistory.AddStorageMetric("vc-1:datastore:datastore-202", "avail", 11.03*1024*1024*1024*1024, now)
	metricsHistory.AddStorageMetric("pool:archive", "usage", 0.36, now)

	adapter := unifiedresources.NewMonitorAdapter(nil)
	adapter.PopulateSnapshotAndSupplemental(state.GetSnapshot(), map[unifiedresources.DataSource][]unifiedresources.IngestRecord{
		unifiedresources.SourceVMware: {
			{
				SourceID: "vc-1:host:host-101",
				Resource: unifiedresources.Resource{
					ID:   "agent-vmware-1",
					Type: unifiedresources.ResourceTypeAgent,
					Name: "esxi-01.lab.local",
					Identity: unifiedresources.ResourceIdentity{
						Hostnames: []string{"esxi-01.lab.local"},
					},
					MetricsTarget: &unifiedresources.MetricsTarget{
						ResourceType: "agent",
						ResourceID:   "vc-1:host:host-101",
					},
					VMware: &unifiedresources.VMwareData{
						ConnectionID:    "vc-1",
						EntityType:      "host",
						ManagedObjectID: "host-101",
					},
				},
			},
			{
				SourceID:       "vc-1:datastore:datastore-202",
				ParentSourceID: "vc-1:host:host-101",
				Resource: unifiedresources.Resource{
					ID:         "storage-vmware-1",
					Type:       unifiedresources.ResourceTypeStorage,
					Name:       "archive-tier",
					ParentName: "esxi-01.lab.local",
					MetricsTarget: &unifiedresources.MetricsTarget{
						ResourceType: "storage",
						ResourceID:   "vc-1:datastore:datastore-202",
					},
					Storage: &unifiedresources.StorageMeta{
						Type:     "datastore",
						Platform: "vmware",
						Nodes:    []string{"esxi-01.lab.local", "esxi-02.lab.local"},
					},
					VMware: &unifiedresources.VMwareData{
						ConnectionID:    "vc-1",
						EntityType:      "datastore",
						ManagedObjectID: "datastore-202",
						RuntimeHostName: "esxi-01.lab.local",
					},
				},
			},
		},
		unifiedresources.SourceTrueNAS: {
			{
				SourceID: "system:truenas-main",
				Resource: unifiedresources.Resource{
					ID:   "agent-truenas-1",
					Type: unifiedresources.ResourceTypeAgent,
					Name: "truenas-main",
					Identity: unifiedresources.ResourceIdentity{
						Hostnames: []string{"truenas-main"},
					},
					TrueNAS: &unifiedresources.TrueNASData{
						Hostname: "truenas-main",
					},
				},
			},
			{
				SourceID:       "pool:archive",
				ParentSourceID: "system:truenas-main",
				Resource: unifiedresources.Resource{
					ID:         "storage-truenas-1",
					Type:       unifiedresources.ResourceTypeStorage,
					Name:       "archive",
					ParentName: "truenas-main",
					MetricsTarget: &unifiedresources.MetricsTarget{
						ResourceType: "storage",
						ResourceID:   "pool:archive",
					},
					Storage: &unifiedresources.StorageMeta{
						Type:     "zfs-pool",
						Platform: "truenas",
					},
					TrueNAS: &unifiedresources.TrueNASData{
						Hostname: "truenas-main",
					},
				},
			},
		},
	})
	selectedNodeID := ""
	for _, resource := range adapter.GetAll() {
		if resource.Type == unifiedresources.ResourceTypeAgent && resource.Name == "esxi-01.lab.local" {
			selectedNodeID = resource.ID
			break
		}
	}
	if selectedNodeID == "" {
		t.Fatal("expected canonical VMware host resource id in adapter")
	}
	setUnexportedField(t, monitor, "resourceStore", monitoring.ResourceStoreInterface(adapter))

	service := NewService(fixedMonitorResolver{monitor: monitor})
	req := httptest.NewRequest(http.MethodGet, "/api/storage-charts?range=30&node="+selectedNodeID, nil)
	rec := httptest.NewRecorder()

	service.HandleStorageCharts(rec, req)

	if rec.Code != http.StatusOK {
		t.Fatalf("expected status %d, got %d", http.StatusOK, rec.Code)
	}

	var decoded StorageChartsResponse
	if err := json.Unmarshal(rec.Body.Bytes(), &decoded); err != nil {
		t.Fatalf("unmarshal StorageChartsResponse: %v", err)
	}
	if len(decoded.Pools) != 1 {
		t.Fatalf("expected unified node filter to keep 1 VMware pool, got %d (%v)", len(decoded.Pools), decoded.Pools)
	}
	if _, ok := decoded.Pools["vc-1:datastore:datastore-202"]; !ok {
		t.Fatalf("expected VMware datastore chart keyed by canonical metrics target, got %v", decoded.Pools)
	}
	if _, ok := decoded.Pools["pool:archive"]; ok {
		t.Fatalf("expected TrueNAS pool to be filtered out by VMware host selection, got %v", decoded.Pools)
	}
}