Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

29.2 KB
Raw Blame
package monitoring

import (
	"sort"
	"sync"
	"time"

	"github.com/rcourtman/pulse-go-rewrite/internal/models"
	"github.com/rs/zerolog/log"
)

// MetricPoint is an alias for models.MetricPoint
type MetricPoint = models.MetricPoint

// GuestMetrics holds historical metrics for a single guest
type GuestMetrics struct {
	CPU            []MetricPoint `json:"cpu"`
	Memory         []MetricPoint `json:"memory"`
	MemoryUsed     []MetricPoint `json:"memoryused"`
	Disk           []MetricPoint `json:"disk"`
	GPU            []MetricPoint `json:"gpu"`
	GPUMemory      []MetricPoint `json:"gpu_memory"`
	GPUTemperature []MetricPoint `json:"gpu_temperature"`
	DiskRead       []MetricPoint `json:"diskread"`
	DiskWrite      []MetricPoint `json:"diskwrite"`
	NetworkIn      []MetricPoint `json:"netin"`
	NetworkOut     []MetricPoint `json:"netout"`
	Temperature    []MetricPoint `json:"temperature"`
}

// StorageMetrics holds historical metrics for a single storage
type StorageMetrics struct {
	Usage []MetricPoint `json:"usage"`
	Used  []MetricPoint `json:"used"`
	Total []MetricPoint `json:"total"`
	Avail []MetricPoint `json:"avail"`
}

// DiskMetrics holds historical metrics for a single physical disk.
type DiskMetrics struct {
	Temperature []MetricPoint `json:"smart_temp"`
	Utilization []MetricPoint `json:"disk"`
	DiskRead    []MetricPoint `json:"diskread"`
	DiskWrite   []MetricPoint `json:"diskwrite"`
}

// MetricsHistory maintains historical metrics for guests, nodes, storage, and disks.
type MetricsHistory struct {
	mu             sync.RWMutex
	guestMetrics   map[string]*GuestMetrics   // key: guestID
	nodeMetrics    map[string]*GuestMetrics   // key: nodeID
	storageMetrics map[string]*StorageMetrics // key: storageID
	diskMetrics    map[string]*DiskMetrics    // key: disk metrics resource ID
	maxDataPoints  int
	retentionTime  time.Duration

	capacityForecastMu    sync.Mutex
	capacityForecastCache map[string]storageCapacityForecastCacheEntry
	metricWindowMu        sync.Mutex
	metricWindowCache     map[string]metricWindowCacheEntry
	metricWindowBytes     int
	metricWindowSweepAt   time.Time
}

// NewMetricsHistory creates a new metrics history tracker
func NewMetricsHistory(maxDataPoints int, retentionTime time.Duration) *MetricsHistory {
	return &MetricsHistory{
		guestMetrics:   make(map[string]*GuestMetrics),
		nodeMetrics:    make(map[string]*GuestMetrics),
		storageMetrics: make(map[string]*StorageMetrics),
		diskMetrics:    make(map[string]*DiskMetrics),
		maxDataPoints:  maxDataPoints,
		retentionTime:  retentionTime,
	}
}

func (mh *MetricsHistory) clone() *MetricsHistory {
	if mh == nil {
		return nil
	}

	mh.mu.RLock()
	defer mh.mu.RUnlock()

	cloned := NewMetricsHistory(mh.maxDataPoints, mh.retentionTime)
	for id, metrics := range mh.guestMetrics {
		cloned.guestMetrics[id] = cloneGuestMetrics(metrics)
	}
	for id, metrics := range mh.nodeMetrics {
		cloned.nodeMetrics[id] = cloneGuestMetrics(metrics)
	}
	for id, metrics := range mh.storageMetrics {
		if metrics == nil {
			cloned.storageMetrics[id] = nil
			continue
		}
		cloned.storageMetrics[id] = &StorageMetrics{
			Usage: append([]MetricPoint(nil), metrics.Usage...),
			Used:  append([]MetricPoint(nil), metrics.Used...),
			Total: append([]MetricPoint(nil), metrics.Total...),
			Avail: append([]MetricPoint(nil), metrics.Avail...),
		}
	}
	for id, metrics := range mh.diskMetrics {
		if metrics == nil {
			cloned.diskMetrics[id] = nil
			continue
		}
		cloned.diskMetrics[id] = &DiskMetrics{
			Temperature: append([]MetricPoint(nil), metrics.Temperature...),
			Utilization: append([]MetricPoint(nil), metrics.Utilization...),
			DiskRead:    append([]MetricPoint(nil), metrics.DiskRead...),
			DiskWrite:   append([]MetricPoint(nil), metrics.DiskWrite...),
		}
	}
	return cloned
}

func cloneGuestMetrics(metrics *GuestMetrics) *GuestMetrics {
	if metrics == nil {
		return nil
	}
	return &GuestMetrics{
		CPU:            append([]MetricPoint(nil), metrics.CPU...),
		Memory:         append([]MetricPoint(nil), metrics.Memory...),
		MemoryUsed:     append([]MetricPoint(nil), metrics.MemoryUsed...),
		Disk:           append([]MetricPoint(nil), metrics.Disk...),
		GPU:            append([]MetricPoint(nil), metrics.GPU...),
		GPUMemory:      append([]MetricPoint(nil), metrics.GPUMemory...),
		GPUTemperature: append([]MetricPoint(nil), metrics.GPUTemperature...),
		DiskRead:       append([]MetricPoint(nil), metrics.DiskRead...),
		DiskWrite:      append([]MetricPoint(nil), metrics.DiskWrite...),
		NetworkIn:      append([]MetricPoint(nil), metrics.NetworkIn...),
		NetworkOut:     append([]MetricPoint(nil), metrics.NetworkOut...),
		Temperature:    append([]MetricPoint(nil), metrics.Temperature...),
	}
}

// Reset clears all historical metrics data.
func (mh *MetricsHistory) Reset() {
	mh.mu.Lock()
	mh.guestMetrics = make(map[string]*GuestMetrics)
	mh.nodeMetrics = make(map[string]*GuestMetrics)
	mh.storageMetrics = make(map[string]*StorageMetrics)
	mh.diskMetrics = make(map[string]*DiskMetrics)
	mh.mu.Unlock()

	mh.metricWindowMu.Lock()
	mh.metricWindowCache = nil
	mh.metricWindowBytes = 0
	mh.metricWindowSweepAt = time.Time{}
	mh.metricWindowMu.Unlock()
}

// AddGuestMetric adds a metric value for a guest
func (mh *MetricsHistory) AddGuestMetric(guestID string, metricType string, value float64, timestamp time.Time) {
	mh.mu.Lock()
	defer mh.mu.Unlock()

	// Initialize guest metrics if not exists
	if _, exists := mh.guestMetrics[guestID]; !exists {
		mh.guestMetrics[guestID] = &GuestMetrics{}
	}

	metrics := mh.guestMetrics[guestID]
	point := MetricPoint{Value: value, Timestamp: timestamp}

	// Add metric based on type
	switch metricType {
	case "cpu":
		metrics.CPU = mh.appendMetric(metrics.CPU, point)
	case "memory":
		metrics.Memory = mh.appendMetric(metrics.Memory, point)
	case "memoryused":
		metrics.MemoryUsed = mh.appendMetric(metrics.MemoryUsed, point)
	case "disk":
		metrics.Disk = mh.appendMetric(metrics.Disk, point)
	case "gpu":
		metrics.GPU = mh.appendMetric(metrics.GPU, point)
	case "gpu_memory":
		metrics.GPUMemory = mh.appendMetric(metrics.GPUMemory, point)
	case "gpu_temperature":
		metrics.GPUTemperature = mh.appendMetric(metrics.GPUTemperature, point)
	case "diskread":
		metrics.DiskRead = mh.appendMetric(metrics.DiskRead, point)
	case "diskwrite":
		metrics.DiskWrite = mh.appendMetric(metrics.DiskWrite, point)
	case "netin":
		metrics.NetworkIn = mh.appendMetric(metrics.NetworkIn, point)
	case "netout":
		metrics.NetworkOut = mh.appendMetric(metrics.NetworkOut, point)
	case "temperature":
		metrics.Temperature = mh.appendMetric(metrics.Temperature, point)
	}
}

// AddNodeMetric adds a metric value for a node
func (mh *MetricsHistory) AddNodeMetric(nodeID string, metricType string, value float64, timestamp time.Time) {
	mh.mu.Lock()
	defer mh.mu.Unlock()

	// Initialize node metrics if not exists
	if _, exists := mh.nodeMetrics[nodeID]; !exists {
		mh.nodeMetrics[nodeID] = &GuestMetrics{}
	}

	metrics := mh.nodeMetrics[nodeID]
	point := MetricPoint{Value: value, Timestamp: timestamp}

	// Add metric based on type
	switch metricType {
	case "cpu":
		metrics.CPU = mh.appendMetric(metrics.CPU, point)
	case "memory":
		metrics.Memory = mh.appendMetric(metrics.Memory, point)
	case "memoryused":
		metrics.MemoryUsed = mh.appendMetric(metrics.MemoryUsed, point)
	case "disk":
		metrics.Disk = mh.appendMetric(metrics.Disk, point)
	case "netin":
		metrics.NetworkIn = mh.appendMetric(metrics.NetworkIn, point)
	case "netout":
		metrics.NetworkOut = mh.appendMetric(metrics.NetworkOut, point)
	case "temperature":
		metrics.Temperature = mh.appendMetric(metrics.Temperature, point)
	}
}

// appendMetric appends a metric point and maintains max data points and retention
func (mh *MetricsHistory) appendMetric(metrics []MetricPoint, point MetricPoint) []MetricPoint {
	// Keep a single canonical value per timestamp so chart consumers never
	// have to guess which duplicate tail point to render.
	if len(metrics) > 0 && metrics[len(metrics)-1].Timestamp.Equal(point.Timestamp) {
		metrics[len(metrics)-1] = point
	} else {
		metrics = append(metrics, point)
	}

	// Remove old points beyond retention time. Compact in place instead of
	// re-slicing the front: `metrics[i:]` hides the trimmed prefix from
	// cap() while keeping the whole backing array reachable, so seed-sized
	// arrays stayed pinned long after their points aged out.
	cutoffTime := time.Now().Add(-mh.retentionTime)
	firstRetained := len(metrics)
	for i, p := range metrics {
		if p.Timestamp.After(cutoffTime) {
			firstRetained = i
			break
		}
	}
	if firstRetained == len(metrics) {
		metrics = metrics[:0]
	} else if firstRetained > 0 {
		metrics = append(metrics[:0], metrics[firstRetained:]...)
	}

	// Ensure we don't exceed max data points, keeping the most recent.
	if over := len(metrics) - mh.maxDataPoints; over > 0 {
		metrics = append(metrics[:0], metrics[over:]...)
	}

	return releaseTrimmedCapacity(metrics)
}

// releaseTrimmedCapacity copies the retained window into a right-sized
// backing array when the window occupies a small fraction of it. After a
// dense historical seed (~2,800 points/series) ages out of retention, every
// series would otherwise keep its seed-sized array while holding a few
// hundred live points — across thousands of series that pinned ~700MB on
// the public demo and wedged it against its memory cgroup. The trims above
// compact in place (never re-slice the front) so cap() here reflects the
// full backing array. The 4x threshold keeps the copy amortized: a healthy
// sliding window sits between 1x and ~2x capacity and never triggers it.
func releaseTrimmedCapacity(metrics []MetricPoint) []MetricPoint {
	if cap(metrics) <= 64 || cap(metrics) <= 4*len(metrics) {
		return metrics
	}
	resized := make([]MetricPoint, len(metrics), len(metrics)+len(metrics)/4+8)
	copy(resized, metrics)
	return resized
}

// appendMetricSeries applies the same retention, duplicate-tail, and capacity
// rules as appendMetric while amortizing them across a historical backfill.
// Seed timestamps are ordered oldest-first, matching the live append contract.
func (mh *MetricsHistory) appendMetricSeries(metrics []MetricPoint, values []float64, timestamps []time.Time) []MetricPoint {
	count := len(values)
	if len(timestamps) < count {
		count = len(timestamps)
	}
	if count == 0 {
		return metrics
	}

	for i := 0; i < count; i++ {
		point := MetricPoint{Value: values[i], Timestamp: timestamps[i]}
		if len(metrics) > 0 && metrics[len(metrics)-1].Timestamp.Equal(point.Timestamp) {
			metrics[len(metrics)-1] = point
		} else {
			metrics = append(metrics, point)
		}
	}

	cutoffTime := time.Now().Add(-mh.retentionTime)
	firstRetained := sort.Search(len(metrics), func(i int) bool {
		return metrics[i].Timestamp.After(cutoffTime)
	})
	if firstRetained == len(metrics) {
		metrics = metrics[:0]
	} else if firstRetained > 0 {
		metrics = append(metrics[:0], metrics[firstRetained:]...)
	}

	if over := len(metrics) - mh.maxDataPoints; over > 0 {
		metrics = append(metrics[:0], metrics[over:]...)
	}
	return releaseTrimmedCapacity(metrics)
}

func (mh *MetricsHistory) addGuestMetricSeries(guestID, metricType string, values []float64, timestamps []time.Time) {
	mh.mu.Lock()
	defer mh.mu.Unlock()

	if _, exists := mh.guestMetrics[guestID]; !exists {
		mh.guestMetrics[guestID] = &GuestMetrics{}
	}
	metrics := mh.guestMetrics[guestID]
	switch metricType {
	case "cpu":
		metrics.CPU = mh.appendMetricSeries(metrics.CPU, values, timestamps)
	case "memory":
		metrics.Memory = mh.appendMetricSeries(metrics.Memory, values, timestamps)
	case "memoryused":
		metrics.MemoryUsed = mh.appendMetricSeries(metrics.MemoryUsed, values, timestamps)
	case "disk":
		metrics.Disk = mh.appendMetricSeries(metrics.Disk, values, timestamps)
	case "gpu":
		metrics.GPU = mh.appendMetricSeries(metrics.GPU, values, timestamps)
	case "gpu_memory":
		metrics.GPUMemory = mh.appendMetricSeries(metrics.GPUMemory, values, timestamps)
	case "gpu_temperature":
		metrics.GPUTemperature = mh.appendMetricSeries(metrics.GPUTemperature, values, timestamps)
	case "diskread":
		metrics.DiskRead = mh.appendMetricSeries(metrics.DiskRead, values, timestamps)
	case "diskwrite":
		metrics.DiskWrite = mh.appendMetricSeries(metrics.DiskWrite, values, timestamps)
	case "netin":
		metrics.NetworkIn = mh.appendMetricSeries(metrics.NetworkIn, values, timestamps)
	case "netout":
		metrics.NetworkOut = mh.appendMetricSeries(metrics.NetworkOut, values, timestamps)
	case "temperature":
		metrics.Temperature = mh.appendMetricSeries(metrics.Temperature, values, timestamps)
	}
}

func (mh *MetricsHistory) addNodeMetricSeries(nodeID, metricType string, values []float64, timestamps []time.Time) {
	mh.mu.Lock()
	defer mh.mu.Unlock()

	if _, exists := mh.nodeMetrics[nodeID]; !exists {
		mh.nodeMetrics[nodeID] = &GuestMetrics{}
	}
	metrics := mh.nodeMetrics[nodeID]
	switch metricType {
	case "cpu":
		metrics.CPU = mh.appendMetricSeries(metrics.CPU, values, timestamps)
	case "memory":
		metrics.Memory = mh.appendMetricSeries(metrics.Memory, values, timestamps)
	case "memoryused":
		metrics.MemoryUsed = mh.appendMetricSeries(metrics.MemoryUsed, values, timestamps)
	case "disk":
		metrics.Disk = mh.appendMetricSeries(metrics.Disk, values, timestamps)
	case "netin":
		metrics.NetworkIn = mh.appendMetricSeries(metrics.NetworkIn, values, timestamps)
	case "netout":
		metrics.NetworkOut = mh.appendMetricSeries(metrics.NetworkOut, values, timestamps)
	case "temperature":
		metrics.Temperature = mh.appendMetricSeries(metrics.Temperature, values, timestamps)
	}
}

func (mh *MetricsHistory) addStorageMetricSeries(storageID, metricType string, values []float64, timestamps []time.Time) {
	mh.mu.Lock()
	defer mh.mu.Unlock()

	if _, exists := mh.storageMetrics[storageID]; !exists {
		mh.storageMetrics[storageID] = &StorageMetrics{}
	}
	metrics := mh.storageMetrics[storageID]
	switch metricType {
	case "usage":
		metrics.Usage = mh.appendMetricSeries(metrics.Usage, values, timestamps)
	case "used":
		metrics.Used = mh.appendMetricSeries(metrics.Used, values, timestamps)
	case "total":
		metrics.Total = mh.appendMetricSeries(metrics.Total, values, timestamps)
	case "avail":
		metrics.Avail = mh.appendMetricSeries(metrics.Avail, values, timestamps)
	}
}

func (mh *MetricsHistory) addDiskMetricSeries(resourceID, metricType string, values []float64, timestamps []time.Time) {
	mh.mu.Lock()
	defer mh.mu.Unlock()

	if _, exists := mh.diskMetrics[resourceID]; !exists {
		mh.diskMetrics[resourceID] = &DiskMetrics{}
	}
	metrics := mh.diskMetrics[resourceID]
	switch metricType {
	case "disk":
		metrics.Utilization = mh.appendMetricSeries(metrics.Utilization, values, timestamps)
	case "diskread":
		metrics.DiskRead = mh.appendMetricSeries(metrics.DiskRead, values, timestamps)
	case "diskwrite":
		metrics.DiskWrite = mh.appendMetricSeries(metrics.DiskWrite, values, timestamps)
	case "smart_temp":
		metrics.Temperature = mh.appendMetricSeries(metrics.Temperature, values, timestamps)
	}
}

// GetGuestMetrics returns historical metrics for a guest
func (mh *MetricsHistory) GetGuestMetrics(guestID string, metricType string, duration time.Duration) []MetricPoint {
	mh.mu.RLock()
	defer mh.mu.RUnlock()

	metrics, exists := mh.guestMetrics[guestID]
	if !exists {
		return []MetricPoint{}
	}

	cutoffTime := time.Now().Add(-duration)
	var data []MetricPoint

	switch metricType {
	case "cpu":
		data = metrics.CPU
	case "memory":
		data = metrics.Memory
	case "memoryused":
		data = metrics.MemoryUsed
	case "disk":
		data = metrics.Disk
	case "gpu":
		data = metrics.GPU
	case "gpu_memory":
		data = metrics.GPUMemory
	case "gpu_temperature":
		data = metrics.GPUTemperature
	case "diskread":
		data = metrics.DiskRead
	case "diskwrite":
		data = metrics.DiskWrite
	case "netin":
		data = metrics.NetworkIn
	case "netout":
		data = metrics.NetworkOut
	case "temperature":
		data = metrics.Temperature
	default:
		return []MetricPoint{}
	}

	return filterMetricsByTime(data, cutoffTime)
}

// GetNodeMetrics returns historical metrics for a node
func (mh *MetricsHistory) GetNodeMetrics(nodeID string, metricType string, duration time.Duration) []MetricPoint {
	mh.mu.RLock()
	defer mh.mu.RUnlock()

	metrics, exists := mh.nodeMetrics[nodeID]
	if !exists {
		return []MetricPoint{}
	}

	cutoffTime := time.Now().Add(-duration)
	var data []MetricPoint

	switch metricType {
	case "cpu":
		data = metrics.CPU
	case "memory":
		data = metrics.Memory
	case "memoryused":
		data = metrics.MemoryUsed
	case "disk":
		data = metrics.Disk
	case "netin":
		data = metrics.NetworkIn
	case "netout":
		data = metrics.NetworkOut
	case "temperature":
		data = metrics.Temperature
	default:
		return []MetricPoint{}
	}

	return filterMetricsByTime(data, cutoffTime)
}

// filterMetricsByTime returns only the points whose timestamp is after cutoffTime.
func filterMetricsByTime(data []MetricPoint, cutoffTime time.Time) []MetricPoint {
	// Skip the expired prefix, then count the survivors before allocating.
	// Dense chart reads otherwise repeatedly grow and copy their result under
	// the History read lock. Allocate only the requested window, not all stored
	// points, and keep caller-owned snapshots rather than aliasing live History.
	first := 0
	for first < len(data) && !data[first].Timestamp.After(cutoffTime) {
		first++
	}
	data = data[first:]
	count := 0
	for _, point := range data {
		if point.Timestamp.After(cutoffTime) {
			count++
		}
	}
	filtered := make([]MetricPoint, count) // Preserve [] (not null) for empty series.
	if count == len(data) {
		copy(filtered, data)
		return filtered
	}
	// Usually the retained window is a contiguous suffix. Preserve the original
	// per-point filtering and order even when an input is not chronological.
	next := 0
	for _, point := range data {
		if point.Timestamp.After(cutoffTime) {
			filtered[next] = point
			next++
		}
	}
	return filtered
}

func metricsSeriesCoverageSpanAfter(data []MetricPoint, cutoffTime time.Time) time.Duration {
	if len(data) < 2 {
		return 0
	}

	first := sort.Search(len(data), func(i int) bool {
		return data[i].Timestamp.After(cutoffTime)
	})
	if first >= len(data)-1 {
		return 0
	}

	oldest := data[first].Timestamp
	newest := data[len(data)-1].Timestamp
	if !newest.After(oldest) {
		return 0
	}
	return newest.Sub(oldest)
}

func guestMetricSeries(metrics *GuestMetrics, metricType string) []MetricPoint {
	if metrics == nil {
		return nil
	}

	switch metricType {
	case "cpu":
		return metrics.CPU
	case "memory":
		return metrics.Memory
	case "memoryused":
		return metrics.MemoryUsed
	case "disk":
		return metrics.Disk
	case "gpu":
		return metrics.GPU
	case "gpu_memory":
		return metrics.GPUMemory
	case "gpu_temperature":
		return metrics.GPUTemperature
	case "diskread":
		return metrics.DiskRead
	case "diskwrite":
		return metrics.DiskWrite
	case "netin":
		return metrics.NetworkIn
	case "netout":
		return metrics.NetworkOut
	case "temperature":
		return metrics.Temperature
	default:
		return nil
	}
}

func guestMetricsCoverageSpan(metrics *GuestMetrics, metricTypes []string, cutoffTime time.Time) time.Duration {
	if metrics == nil {
		return 0
	}

	if len(metricTypes) == 0 {
		metricTypes = []string{"cpu", "memory", "memoryused", "disk", "gpu", "gpu_memory", "gpu_temperature", "diskread", "diskwrite", "netin", "netout", "temperature"}
	}

	var best time.Duration
	for _, metricType := range metricTypes {
		if span := metricsSeriesCoverageSpanAfter(guestMetricSeries(metrics, metricType), cutoffTime); span > best {
			best = span
		}
	}
	return best
}

// GuestMetricCoverageSpan returns the best in-memory coverage span across the
// requested guest metric types after applying the requested duration window.
func (mh *MetricsHistory) GuestMetricCoverageSpan(guestID string, metricTypes []string, duration time.Duration) time.Duration {
	if mh == nil {
		return 0
	}

	mh.mu.RLock()
	defer mh.mu.RUnlock()

	metrics, exists := mh.guestMetrics[guestID]
	if !exists {
		return 0
	}

	return guestMetricsCoverageSpan(metrics, metricTypes, time.Now().Add(-duration))
}

// NodeMetricCoverageSpan returns the best in-memory coverage span across the
// requested node metric types after applying the requested duration window.
func (mh *MetricsHistory) NodeMetricCoverageSpan(nodeID string, metricTypes []string, duration time.Duration) time.Duration {
	if mh == nil {
		return 0
	}

	mh.mu.RLock()
	defer mh.mu.RUnlock()

	metrics, exists := mh.nodeMetrics[nodeID]
	if !exists {
		return 0
	}

	return guestMetricsCoverageSpan(metrics, metricTypes, time.Now().Add(-duration))
}

// GetAllGuestMetrics returns all metrics for a guest within a duration
func (mh *MetricsHistory) GetAllGuestMetrics(guestID string, duration time.Duration) map[string][]MetricPoint {
	mh.mu.RLock()
	defer mh.mu.RUnlock()

	result := make(map[string][]MetricPoint)
	cutoffTime := time.Now().Add(-duration)

	metrics, exists := mh.guestMetrics[guestID]
	if !exists {
		return result
	}

	result["cpu"] = filterMetricsByTime(metrics.CPU, cutoffTime)
	result["memory"] = filterMetricsByTime(metrics.Memory, cutoffTime)
	result["memoryused"] = filterMetricsByTime(metrics.MemoryUsed, cutoffTime)
	result["disk"] = filterMetricsByTime(metrics.Disk, cutoffTime)
	result["gpu"] = filterMetricsByTime(metrics.GPU, cutoffTime)
	result["gpu_memory"] = filterMetricsByTime(metrics.GPUMemory, cutoffTime)
	result["gpu_temperature"] = filterMetricsByTime(metrics.GPUTemperature, cutoffTime)
	result["diskread"] = filterMetricsByTime(metrics.DiskRead, cutoffTime)
	result["diskwrite"] = filterMetricsByTime(metrics.DiskWrite, cutoffTime)
	result["netin"] = filterMetricsByTime(metrics.NetworkIn, cutoffTime)
	result["netout"] = filterMetricsByTime(metrics.NetworkOut, cutoffTime)
	result["temperature"] = filterMetricsByTime(metrics.Temperature, cutoffTime)

	return result
}

// AddStorageMetric adds a metric value for storage
func (mh *MetricsHistory) AddStorageMetric(storageID string, metricType string, value float64, timestamp time.Time) {
	mh.mu.Lock()
	defer mh.mu.Unlock()

	// Initialize storage metrics if not exists
	if _, exists := mh.storageMetrics[storageID]; !exists {
		mh.storageMetrics[storageID] = &StorageMetrics{}
	}

	metrics := mh.storageMetrics[storageID]
	point := MetricPoint{Value: value, Timestamp: timestamp}

	// Add metric based on type
	switch metricType {
	case "usage":
		metrics.Usage = mh.appendMetric(metrics.Usage, point)
	case "used":
		metrics.Used = mh.appendMetric(metrics.Used, point)
	case "total":
		metrics.Total = mh.appendMetric(metrics.Total, point)
	case "avail":
		metrics.Avail = mh.appendMetric(metrics.Avail, point)
	}
}

// GetAllStorageMetrics returns all metrics for storage within a duration
func (mh *MetricsHistory) GetAllStorageMetrics(storageID string, duration time.Duration) map[string][]MetricPoint {
	mh.mu.RLock()
	defer mh.mu.RUnlock()

	result := make(map[string][]MetricPoint)
	cutoffTime := time.Now().Add(-duration)

	metrics, exists := mh.storageMetrics[storageID]
	if !exists {
		return result
	}

	result["usage"] = filterMetricsByTime(metrics.Usage, cutoffTime)
	result["used"] = filterMetricsByTime(metrics.Used, cutoffTime)
	result["total"] = filterMetricsByTime(metrics.Total, cutoffTime)
	result["avail"] = filterMetricsByTime(metrics.Avail, cutoffTime)

	return result
}

// AddDiskMetric adds a metric value for a physical disk.
func (mh *MetricsHistory) AddDiskMetric(resourceID string, metricType string, value float64, timestamp time.Time) {
	mh.mu.Lock()
	defer mh.mu.Unlock()

	if _, exists := mh.diskMetrics[resourceID]; !exists {
		mh.diskMetrics[resourceID] = &DiskMetrics{}
	}

	metrics := mh.diskMetrics[resourceID]
	point := MetricPoint{Value: value, Timestamp: timestamp}

	switch metricType {
	case "disk":
		metrics.Utilization = mh.appendMetric(metrics.Utilization, point)
	case "diskread":
		metrics.DiskRead = mh.appendMetric(metrics.DiskRead, point)
	case "diskwrite":
		metrics.DiskWrite = mh.appendMetric(metrics.DiskWrite, point)
	case "smart_temp":
		metrics.Temperature = mh.appendMetric(metrics.Temperature, point)
	}
}

// GetDiskMetrics returns historical metrics for a physical disk.
func (mh *MetricsHistory) GetDiskMetrics(resourceID string, metricType string, duration time.Duration) []MetricPoint {
	mh.mu.RLock()
	defer mh.mu.RUnlock()

	metrics, exists := mh.diskMetrics[resourceID]
	if !exists {
		return []MetricPoint{}
	}

	cutoffTime := time.Now().Add(-duration)

	switch metricType {
	case "disk":
		return filterMetricsByTime(metrics.Utilization, cutoffTime)
	case "diskread":
		return filterMetricsByTime(metrics.DiskRead, cutoffTime)
	case "diskwrite":
		return filterMetricsByTime(metrics.DiskWrite, cutoffTime)
	case "smart_temp":
		return filterMetricsByTime(metrics.Temperature, cutoffTime)
	default:
		return []MetricPoint{}
	}
}

// Cleanup removes old data points beyond retention time and deletes
// map entries for resources that have no remaining data points.
// This prevents unbounded memory growth when containers/VMs are deleted.
func (mh *MetricsHistory) Cleanup() {
	mh.mu.Lock()
	defer mh.mu.Unlock()

	cutoffTime := time.Now().Add(-mh.retentionTime)
	var guestsRemoved, nodesRemoved, storageRemoved, disksRemoved int

	// Cleanup guest metrics and remove empty entries
	for key, metrics := range mh.guestMetrics {
		metrics.CPU = mh.cleanupMetrics(metrics.CPU, cutoffTime)
		metrics.Memory = mh.cleanupMetrics(metrics.Memory, cutoffTime)
		metrics.MemoryUsed = mh.cleanupMetrics(metrics.MemoryUsed, cutoffTime)
		metrics.Disk = mh.cleanupMetrics(metrics.Disk, cutoffTime)
		metrics.GPU = mh.cleanupMetrics(metrics.GPU, cutoffTime)
		metrics.GPUMemory = mh.cleanupMetrics(metrics.GPUMemory, cutoffTime)
		metrics.GPUTemperature = mh.cleanupMetrics(metrics.GPUTemperature, cutoffTime)
		metrics.DiskRead = mh.cleanupMetrics(metrics.DiskRead, cutoffTime)
		metrics.DiskWrite = mh.cleanupMetrics(metrics.DiskWrite, cutoffTime)
		metrics.NetworkIn = mh.cleanupMetrics(metrics.NetworkIn, cutoffTime)
		metrics.NetworkOut = mh.cleanupMetrics(metrics.NetworkOut, cutoffTime)
		metrics.Temperature = mh.cleanupMetrics(metrics.Temperature, cutoffTime)

		// If all slices are empty, remove the map entry entirely to free memory
		if len(metrics.CPU) == 0 && len(metrics.Memory) == 0 && len(metrics.MemoryUsed) == 0 && len(metrics.Disk) == 0 &&
			len(metrics.GPU) == 0 && len(metrics.GPUMemory) == 0 && len(metrics.GPUTemperature) == 0 &&
			len(metrics.DiskRead) == 0 && len(metrics.DiskWrite) == 0 &&
			len(metrics.NetworkIn) == 0 && len(metrics.NetworkOut) == 0 &&
			len(metrics.Temperature) == 0 {
			delete(mh.guestMetrics, key)
			guestsRemoved++
		}
	}

	// Cleanup node metrics and remove empty entries
	for key, metrics := range mh.nodeMetrics {
		metrics.CPU = mh.cleanupMetrics(metrics.CPU, cutoffTime)
		metrics.Memory = mh.cleanupMetrics(metrics.Memory, cutoffTime)
		metrics.MemoryUsed = mh.cleanupMetrics(metrics.MemoryUsed, cutoffTime)
		metrics.Disk = mh.cleanupMetrics(metrics.Disk, cutoffTime)
		metrics.NetworkIn = mh.cleanupMetrics(metrics.NetworkIn, cutoffTime)
		metrics.NetworkOut = mh.cleanupMetrics(metrics.NetworkOut, cutoffTime)
		metrics.Temperature = mh.cleanupMetrics(metrics.Temperature, cutoffTime)

		if len(metrics.CPU) == 0 && len(metrics.Memory) == 0 && len(metrics.MemoryUsed) == 0 && len(metrics.Disk) == 0 &&
			len(metrics.NetworkIn) == 0 && len(metrics.NetworkOut) == 0 &&
			len(metrics.Temperature) == 0 {
			delete(mh.nodeMetrics, key)
			nodesRemoved++
		}
	}

	// Cleanup storage metrics and remove empty entries
	for key, metrics := range mh.storageMetrics {
		metrics.Usage = mh.cleanupMetrics(metrics.Usage, cutoffTime)
		metrics.Used = mh.cleanupMetrics(metrics.Used, cutoffTime)
		metrics.Total = mh.cleanupMetrics(metrics.Total, cutoffTime)
		metrics.Avail = mh.cleanupMetrics(metrics.Avail, cutoffTime)

		if len(metrics.Usage) == 0 && len(metrics.Used) == 0 &&
			len(metrics.Total) == 0 && len(metrics.Avail) == 0 {
			delete(mh.storageMetrics, key)
			storageRemoved++
		}
	}

	// Cleanup physical disk metrics and remove empty entries
	for key, metrics := range mh.diskMetrics {
		metrics.Temperature = mh.cleanupMetrics(metrics.Temperature, cutoffTime)
		metrics.Utilization = mh.cleanupMetrics(metrics.Utilization, cutoffTime)
		metrics.DiskRead = mh.cleanupMetrics(metrics.DiskRead, cutoffTime)
		metrics.DiskWrite = mh.cleanupMetrics(metrics.DiskWrite, cutoffTime)

		if len(metrics.Temperature) == 0 && len(metrics.Utilization) == 0 &&
			len(metrics.DiskRead) == 0 && len(metrics.DiskWrite) == 0 {
			delete(mh.diskMetrics, key)
			disksRemoved++
		}
	}

	// Log cleanup activity at debug level
	if guestsRemoved > 0 || nodesRemoved > 0 || storageRemoved > 0 || disksRemoved > 0 {
		log.Debug().
			Int("guestsRemoved", guestsRemoved).
			Int("nodesRemoved", nodesRemoved).
			Int("storageRemoved", storageRemoved).
			Int("disksRemoved", disksRemoved).
			Int("guestsRemaining", len(mh.guestMetrics)).
			Int("nodesRemaining", len(mh.nodeMetrics)).
			Int("storageRemaining", len(mh.storageMetrics)).
			Int("disksRemaining", len(mh.diskMetrics)).
			Msg("Cleaned up stale metrics history entries")
	}
}

// cleanupMetrics removes points older than cutoff time.
// Returns nil (not empty slice) when all points are expired to release backing array memory.
func (mh *MetricsHistory) cleanupMetrics(metrics []MetricPoint, cutoffTime time.Time) []MetricPoint {
	for i, p := range metrics {
		if p.Timestamp.After(cutoffTime) {
			if i > 0 {
				// Compact rather than re-slice so the backing array's true
				// size stays visible to the release check below.
				metrics = append(metrics[:0], metrics[i:]...)
			}
			return releaseTrimmedCapacity(metrics)
		}
	}
	// Return nil instead of metrics[:0] to release the backing array
	return nil
}