Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

29.7 KB
Raw Blame
package alerts

import (
	"math"
	"strings"

	alertspecs "github.com/rcourtman/pulse-go-rewrite/internal/alerts/specs"
	"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
	"github.com/rcourtman/pulse-go-rewrite/pkg/diskinventory"
	"github.com/rs/zerolog/log"
)

// Override Key Format by Resource Type
//
// The config.Overrides map is keyed by resource ID. Each resource type
// uses the following canonical key format:
//
//	VM / Container:    monitoring resource ID, e.g. "pve1:node1:100"
//	Node:              node.ID             e.g. "node/pve-1"
//	Agent:             host.ID             e.g. "host1" (without "agent:" prefix)
//	Agent Disk:        "agent:{hostID}/disk:{mountpoint}" e.g. "agent:host1/disk:root"
//	Storage:           storage.ID          e.g. "local-lvm"
//	PBS:               pbs.ID              e.g. "pbs-1"
//	PMG:               pmg.ID              e.g. "pmg-1"
//	Docker Container:  "docker:{hostID}/{containerName}" (stable across
//	                   container recreates; legacy "docker:{hostID}/{containerID}"
//	                   entries are still honoured and migrated, #1601)
//	Kubernetes:        canonical unified resource ID for cluster/node/namespace/deployment/pod
//	TrueNAS System:    canonical unified agent resource ID
//	TrueNAS Storage:   canonical unified storage resource ID for pool/dataset
//	TrueNAS Disk:      canonical unified physical_disk resource ID
//	VMware vSphere:    canonical unified resource ID for host/vm/datastore/network
//
// For guests, CheckUnifiedResource expects the same canonical resource ID the
// monitoring pipeline already uses. It does not translate IDs.
// UnifiedResourceMetric holds a single metric value for unified evaluation.
type UnifiedResourceMetric struct {
	Value   float64
	Percent float64
}

// UnifiedResourceInput is the data needed to evaluate alerts for a unified resource.
// This avoids importing unifiedresources (which would cause an import cycle).
type UnifiedResourceInput struct {
	ID          string
	Type        string // lowercase: "vm", "system-container", "app-container", "agent", "pbs", "storage", "pmg"
	Name        string
	Node        string
	Instance    string
	Tags        []string
	CPU         *UnifiedResourceMetric
	Memory      *UnifiedResourceMetric
	Disk        *UnifiedResourceMetric
	DiskRead    *UnifiedResourceMetric
	DiskWrite   *UnifiedResourceMetric
	NetworkIn   *UnifiedResourceMetric
	NetworkOut  *UnifiedResourceMetric
	Temperature *UnifiedResourceMetric

	StorageAliases []string // Durable policy lookup identities for storage alerts.
	// DiskType is a physical disk's type ("nvme", "sas", "sata"). It picks
	// the disk temperature threshold of a TrueNAS disk.
	DiskType string
}

type unifiedMetricCandidate struct {
	Spec      alertspecs.ResourceAlertSpec
	Value     float64
	Threshold *HysteresisThreshold
}

// unifiedAlertType maps a resource type key to the alert system's display type.
func unifiedAlertType(typeKey string) string {
	switch typeKey {
	case "vm":
		return "VM"
	case "system-container":
		return "Container"
	case "app-container":
		return "Docker"
	case "agent":
		return "Agent"
	case "node":
		return "Node"
	case "pbs":
		return "PBS"
	case "storage":
		return "Storage"
	case "pmg":
		return "PMG"
	case "k8s-cluster":
		return "Kubernetes Cluster"
	case "k8s-node":
		return "Kubernetes Node"
	case "k8s-deployment":
		return "Kubernetes Deployment"
	case "k8s-namespace":
		return "Kubernetes Namespace"
	case "pod":
		return "Kubernetes Pod"
	case "truenas-system":
		return "TrueNAS System"
	case "truenas-pool":
		return "TrueNAS Pool"
	case "truenas-dataset":
		return "TrueNAS Dataset"
	case "truenas-disk":
		return "TrueNAS Disk"
	case "vmware-host":
		return "vSphere Host"
	case "vmware-vm":
		return "vSphere VM"
	case "vmware-datastore":
		return "vSphere Datastore"
	case "vmware-network":
		return "vSphere Network"
	default:
		return typeKey
	}
}

func supportsUnifiedIOMetrics(typeKey string) bool {
	switch typeKey {
	case "vm", "system-container", "app-container", "k8s-cluster", "k8s-deployment", "pod", "truenas-system", "vmware-host", "vmware-vm":
		return true
	default:
		return false
	}
}

func isUnifiedKubernetesAlertType(typeKey string) bool {
	switch typeKey {
	case "k8s-cluster", "k8s-node", "k8s-deployment", "k8s-namespace", "pod":
		return true
	default:
		return false
	}
}

func isUnifiedTrueNASAlertType(typeKey string) bool {
	switch typeKey {
	case "truenas-system", "truenas-pool", "truenas-dataset", "truenas-disk":
		return true
	default:
		return false
	}
}

func isUnifiedVMwareAlertType(typeKey string) bool {
	switch typeKey {
	case "vmware-host", "vmware-vm", "vmware-datastore", "vmware-network":
		return true
	default:
		return false
	}
}

func isUnifiedModernPlatformAlertType(typeKey string) bool {
	return isUnifiedKubernetesAlertType(typeKey) || isUnifiedTrueNASAlertType(typeKey) || isUnifiedVMwareAlertType(typeKey)
}

// alertPlatformTypeKey carries the canonical platform id of the resource an
// alert is about. Metric alerts put a display label such as "vSphere VM" in
// resourceType, so consumers that link or name the owning platform read this.
const alertPlatformTypeKey = "platformType"

// unifiedAlertPlatformType returns the canonical platform id for a unified
// alert type key, or "" when the key does not name one.
func unifiedAlertPlatformType(typeKey string) string {
	switch {
	case isUnifiedKubernetesAlertType(typeKey):
		return "kubernetes"
	case strings.HasPrefix(typeKey, "truenas-"):
		return "truenas"
	case strings.HasPrefix(typeKey, "vmware-"):
		return "vmware-vsphere"
	default:
		return ""
	}
}

func (m *Manager) unifiedPlatformAlertsDisabledNoLock(typeKey string) bool {
	switch {
	case isUnifiedKubernetesAlertType(typeKey):
		return m.config.DisableAllKubernetes
	case isUnifiedTrueNASAlertType(typeKey):
		return m.config.DisableAllTrueNAS
	case isUnifiedVMwareAlertType(typeKey):
		return m.config.DisableAllVMware
	default:
		return false
	}
}

// defaultThresholdsForResourceType returns the default ThresholdConfig for a resource type key.
func (m *Manager) defaultThresholdsForResourceType(typeKey string) ThresholdConfig {
	switch typeKey {
	case "vm", "system-container", "app-container":
		return cloneThresholdConfig(m.config.GuestDefaults)
	case "agent":
		return cloneThresholdConfig(m.config.AgentDefaults)
	case "node":
		return cloneThresholdConfig(m.config.NodeDefaults)
	case "pbs":
		return cloneThresholdConfig(m.config.PBSDefaults)
	case "storage":
		return ThresholdConfig{Usage: cloneThreshold(&m.config.StorageDefault)}
	case "k8s-cluster", "k8s-node", "k8s-deployment", "pod":
		return cloneThresholdConfig(m.config.KubernetesDefaults)
	case "k8s-namespace":
		return ThresholdConfig{}
	case "truenas-system":
		return cloneThresholdConfig(m.config.TrueNASDefaults)
	case "truenas-pool", "truenas-dataset":
		defaults := cloneThresholdConfig(m.config.TrueNASDefaults)
		return ThresholdConfig{Usage: cloneThreshold(defaults.Usage)}
	case "truenas-disk":
		return cloneThresholdConfig(m.config.TrueNASDiskDefaults)
	case "vmware-host", "vmware-vm":
		return cloneThresholdConfig(m.config.VMwareDefaults)
	case "vmware-datastore":
		defaults := cloneThresholdConfig(m.config.VMwareDefaults)
		return ThresholdConfig{Usage: cloneThreshold(defaults.Usage)}
	case "vmware-network":
		return ThresholdConfig{}
	default:
		return ThresholdConfig{}
	}
}

// resolveThresholdOverride applies an exact resource-ID override onto an existing base config.
// Callers must hold m.mu when reading config through this helper.
func (m *Manager) resolveThresholdOverride(base ThresholdConfig, resourceID string) ThresholdConfig {
	if override, exists := m.thresholdOverrideForResourceNoLock(resourceID); exists {
		return m.applyThresholdOverride(base, override)
	}

	return base
}

// thresholdOverrideForResourceNoLock looks up a stored threshold override by
// exact resource ID first, then under the canonical unified registry identity
// for that ID. The thresholds UI stores overrides keyed by registry resource
// IDs while the engine check paths look up legacy monitor IDs, so without the
// translation a per-resource override on an API-only PVE node saves, displays
// as Custom, and silently never applies (#1738). Callers must hold m.mu.
func (m *Manager) thresholdOverrideForResourceNoLock(resourceID string) (ThresholdConfig, bool) {
	if override, exists := m.config.Overrides[resourceID]; exists {
		return override, true
	}
	if m.resourceIntentResolver == nil {
		return ThresholdConfig{}, false
	}
	canonicalID, ok := m.resourceIntentResolver(resourceID)
	if !ok {
		return ThresholdConfig{}, false
	}
	canonicalID = strings.TrimSpace(canonicalID)
	if canonicalID == "" || canonicalID == resourceID {
		return ThresholdConfig{}, false
	}
	override, exists := m.config.Overrides[canonicalID]
	return override, exists
}

// resolveGuestThresholdOverride applies the shared guest override lookup onto an existing base config.
// Callers must hold m.mu when reading config through this helper.
func (m *Manager) resolveGuestThresholdOverride(base ThresholdConfig, guest any, guestID string) ThresholdConfig {
	if override, exists := lookupGuestOverride(m.config.Overrides, guest, guestID); exists {
		return m.applyThresholdOverride(base, override)
	}
	return base
}

// resolveResourceThresholds builds the effective thresholds for a resource type and ID.
// Callers must hold m.mu when reading config through this helper.
func (m *Manager) resolveResourceThresholds(typeKey, resourceID string) ThresholdConfig {
	return m.effectiveAlertPolicyNoLock(alertPolicyQuery{TypeKey: typeKey, ResourceID: resourceID}).Thresholds
}

// evaluateUnifiedMetrics runs the common metric dispatch path for unified resources.
func (m *Manager) evaluateUnifiedMetrics(input *UnifiedResourceInput, thresholds ThresholdConfig, opts *metricOptions, separatelyEvaluated ...string) {
	if input == nil {
		return
	}

	opts = metricOptionsWithTags(opts, input.Tags)
	platformType := unifiedAlertPlatformType(input.Type)
	if len(input.StorageAliases) > 0 || platformType != "" {
		merged := metricOptions{}
		if opts != nil {
			merged = *opts
		}
		metadata := make(map[string]interface{}, len(merged.Metadata)+2)
		for k, v := range merged.Metadata {
			metadata[k] = v
		}
		if len(input.StorageAliases) > 0 {
			metadata[storagePolicyAliasesKey] = append([]string(nil), input.StorageAliases...)
		}
		if platformType != "" {
			metadata[alertPlatformTypeKey] = platformType
		}
		if input.Type == "truenas-disk" {
			// A config save re-judges the open temperature alert against
			// the threshold for this disk type.
			metadata["diskType"] = input.DiskType
		}
		merged.Metadata = metadata
		opts = &merged
	}
	candidates := buildUnifiedMetricCandidates(input, thresholds)
	m.interruptUnobservedUnifiedMetrics(input, candidates, separatelyEvaluated)
	for _, candidate := range candidates {
		m.checkMetricWithCanonicalSpec(candidate.Spec, input.Name, input.Node, input.Instance, unifiedAlertType(input.Type), candidate.Value, candidate.Threshold, opts)
	}
}

// An omitted/rejected metric is unknown, not a pause in its sustained-for
// clock. Build candidates stays pure; dispatch owns interruption for metrics
// this resource supports but cannot currently observe.
func (m *Manager) interruptUnobservedUnifiedMetrics(input *UnifiedResourceInput, candidates []unifiedMetricCandidate, separatelyEvaluated []string) {
	if _, ok := unifiedMetricResourceType(input.Type); !ok {
		return
	}
	metrics := [...]string{"cpu", "memory", "disk", "temperature", "diskRead", "diskWrite", "networkIn", "networkOut", "usage"}
	m.mu.Lock()
	intentChanged := false
	for _, metric := range metrics {
		separateOwner := false
		for _, separateMetric := range separatelyEvaluated {
			if metric == separateMetric {
				separateOwner = true
				break
			}
		}
		if separateOwner {
			continue
		}
		switch metric {
		case "diskRead", "diskWrite", "networkIn", "networkOut":
			if !supportsUnifiedIOMetrics(input.Type) {
				continue
			}
		case "usage":
			if !unifiedStorageUsageResourceType(input.Type) {
				continue
			}
		}
		observed := false
		for _, candidate := range candidates {
			if candidate.Spec.MetricThreshold.Metric == metric {
				observed = true
				break
			}
		}
		if !observed {
			if m.interruptMetricRunNoLock(input.ID, canonicalMetricSpecID(input.ID, metric), canonicalMetricStateID(input.ID, metric)) {
				intentChanged = true
			}
		}
	}
	m.mu.Unlock()
	if intentChanged {
		m.saveActiveAlertsAsync("unified metric observation gap")
	}
}

// metricOptionsWithTags records a resource's tags on its metric alerts. With
// no tags it drops any an open alert still carries: config-save reevaluation
// reads pulse-relaxed back from them, and a removed tag left behind judged
// the alert by thresholds the evaluator no longer applies.
func metricOptionsWithTags(opts *metricOptions, tags []string) *metricOptions {
	merged := metricOptions{}
	if opts != nil {
		merged = *opts
	}
	if len(tags) == 0 {
		merged.RemoveMetadata = append(append([]string(nil), merged.RemoveMetadata...), "tags")
		return &merged
	}
	metadata := make(map[string]interface{}, len(merged.Metadata)+1)
	for key, value := range merged.Metadata {
		metadata[key] = value
	}
	metadata["tags"] = append([]string(nil), tags...)
	merged.Metadata = metadata
	return &merged
}

func buildUnifiedMetricCandidates(input *UnifiedResourceInput, thresholds ThresholdConfig) []unifiedMetricCandidate {
	if input == nil {
		return nil
	}

	resourceType, ok := unifiedMetricResourceType(input.Type)
	if !ok {
		return nil
	}

	candidates := make([]unifiedMetricCandidate, 0, 8)
	appendCandidate := func(metricType string, metric *UnifiedResourceMetric, value float64, threshold *HysteresisThreshold) {
		if metric == nil {
			return
		}
		if math.IsNaN(value) || math.IsInf(value, 0) {
			return
		}
		if isPercentageMetric(metricType) && (value < 0 || value > 100) {
			return
		}
		spec, err := buildCanonicalMetricSpec(input.ID, input.Name, resourceType, metricType, threshold)
		if err != nil {
			log.Warn().
				Err(err).
				Str("resourceID", input.ID).
				Str("resourceType", input.Type).
				Str("metricType", metricType).
				Msg("Skipping invalid canonical unified metric spec")
			return
		}
		candidates = append(candidates, unifiedMetricCandidate{
			Spec:      spec,
			Value:     value,
			Threshold: threshold,
		})
	}

	appendCandidate("cpu", input.CPU, input.CPUValue(), thresholds.CPU)
	appendCandidate("memory", input.Memory, input.MemoryValue(), thresholds.Memory)
	appendCandidate("disk", input.Disk, input.DiskValue(), thresholds.Disk)

	if supportsUnifiedIOMetrics(input.Type) {
		appendCandidate("diskRead", input.DiskRead, input.DiskReadValue(), thresholds.DiskRead)
		appendCandidate("diskWrite", input.DiskWrite, input.DiskWriteValue(), thresholds.DiskWrite)
		appendCandidate("networkIn", input.NetworkIn, input.NetworkInValue(), thresholds.NetworkIn)
		appendCandidate("networkOut", input.NetworkOut, input.NetworkOutValue(), thresholds.NetworkOut)
	}

	if unifiedStorageUsageResourceType(input.Type) && input.Disk != nil {
		appendCandidate("usage", input.Disk, input.DiskValue(), thresholds.Usage)
	}

	appendCandidate("temperature", input.Temperature, input.TemperatureValue(), thresholds.Temperature)

	return candidates
}

func unifiedStorageUsageResourceType(typeKey string) bool {
	switch typeKey {
	case "storage", "truenas-pool", "truenas-dataset", "vmware-datastore":
		return true
	default:
		return false
	}
}

// isPercentageMetric returns true for metrics whose values are bounded 0–100.
// The critical threshold for these must stay below 100 or escalation will never fire.
func isPercentageMetric(metricType string) bool {
	switch metricType {
	case "cpu", "memory", "disk", "usage":
		return true
	default:
		return false
	}
}

// computeCriticalThreshold derives the critical (severity-escalation) threshold
// from the trigger value. For percentage metrics (0–100 range) the result is
// capped at 99 so that critical escalation remains reachable when the trigger
// is set high (e.g. trigger 95 → critical 99, not 105).
func computeCriticalThreshold(trigger float64, metricType string) float64 {
	critical := trigger + 10
	if isPercentageMetric(metricType) && critical > 99 {
		return 99
	}
	return critical
}

func buildCanonicalMetricSpec(resourceID, title string, resourceType unifiedresources.ResourceType, metricType string, threshold *HysteresisThreshold) (alertspecs.ResourceAlertSpec, error) {
	spec := alertspecs.ResourceAlertSpec{
		ID:           canonicalMetricSpecID(resourceID, metricType),
		ResourceID:   resourceID,
		ResourceType: resourceType,
		Kind:         alertspecs.AlertSpecKindMetricThreshold,
		Severity:     alertspecs.AlertSeverityWarning,
		Title:        title,
		Disabled:     threshold == nil || threshold.Trigger <= 0,
		MetricThreshold: &alertspecs.MetricThresholdSpec{
			Metric:    metricType,
			Direction: alertspecs.ThresholdDirectionAbove,
			Trigger:   0,
		},
	}

	if threshold != nil {
		spec.MetricThreshold.Trigger = threshold.Trigger
		if threshold.Clear > 0 && threshold.Clear < threshold.Trigger {
			recovery := threshold.Clear
			spec.MetricThreshold.Recovery = &recovery
		}
		// A percentage trigger of 99+ collides with the 99 critical cap; a
		// critical at or below the trigger fails spec validation and used to
		// drop the whole metric from evaluation (silently unmonitored, plus a
		// warn-log every cycle). Omit escalation instead so the warning-level
		// threshold keeps working.
		if critical := computeCriticalThreshold(threshold.Trigger, metricType); critical > threshold.Trigger {
			spec.MetricThreshold.Critical = &critical
		}
	}

	return spec, spec.Validate()
}

func (m *Manager) checkMetricWithCanonicalSpec(spec alertspecs.ResourceAlertSpec, resourceName, node, instance, resourceType string, value float64, threshold *HysteresisThreshold, opts *metricOptions) bool {
	if spec.MetricThreshold == nil {
		return false
	}
	return m.evaluateCanonicalMetricAlert(spec, resourceName, node, instance, resourceType, value, threshold, opts)
}

func unifiedMetricResourceType(typeKey string) (unifiedresources.ResourceType, bool) {
	switch typeKey {
	case "vm":
		return unifiedresources.ResourceTypeVM, true
	case "system-container":
		return unifiedresources.ResourceTypeSystemContainer, true
	case "app-container":
		return unifiedresources.ResourceTypeAppContainer, true
	case "agent":
		return unifiedresources.ResourceTypeAgent, true
	case "node":
		return unifiedresources.ResourceType("node"), true
	case "pbs":
		return unifiedresources.ResourceTypePBS, true
	case "storage":
		return unifiedresources.ResourceTypeStorage, true
	case "pmg":
		return unifiedresources.ResourceTypePMG, true
	case "k8s-cluster":
		return unifiedresources.ResourceTypeK8sCluster, true
	case "k8s-node":
		return unifiedresources.ResourceTypeK8sNode, true
	case "k8s-deployment":
		return unifiedresources.ResourceTypeK8sDeployment, true
	case "k8s-namespace":
		return unifiedresources.ResourceTypeK8sNamespace, true
	case "pod":
		return unifiedresources.ResourceTypePod, true
	case "truenas-system":
		return unifiedresources.ResourceTypeAgent, true
	case "truenas-pool", "truenas-dataset":
		return unifiedresources.ResourceTypeStorage, true
	case "truenas-disk":
		return unifiedresources.ResourceTypePhysicalDisk, true
	case "vmware-host":
		return unifiedresources.ResourceTypeAgent, true
	case "vmware-vm":
		return unifiedresources.ResourceTypeVM, true
	case "vmware-datastore":
		return unifiedresources.ResourceTypeStorage, true
	case "vmware-network":
		return unifiedresources.ResourceTypeNetwork, true
	default:
		return "", false
	}
}

func (m *Manager) CheckUnifiedResourceMetrics(resources []unifiedresources.Resource) {
	m.CheckUnifiedResourceMetricsWithCapacityTrends(resources, nil)
}

// CheckUnifiedResourceMetricsWithCapacityTrends evaluates canonical resources
// with optional alert-grade capacity evidence keyed by canonical resource ID.
func (m *Manager) CheckUnifiedResourceMetricsWithCapacityTrends(resources []unifiedresources.Resource, capacityTrends map[string]CapacityTrendObservation) {
	if m == nil {
		return
	}
	for _, resource := range resources {
		input, ok := UnifiedResourceInputFromResource(resource)
		if !ok {
			continue
		}
		m.CheckUnifiedResourceWithCapacityTrend(input, capacityTrends[input.ID])
	}
}

func UnifiedResourceInputFromResource(resource unifiedresources.Resource) (*UnifiedResourceInput, bool) {
	typeKey, ok := unifiedAlertResourceType(resource)
	if !ok {
		return nil, false
	}
	resourceID := unifiedresources.CanonicalResourceID(resource.ID)
	if strings.TrimSpace(resourceID) == "" {
		return nil, false
	}

	input := &UnifiedResourceInput{
		ID:       resourceID,
		Type:     typeKey,
		Name:     unifiedResourceAlertName(resource),
		Node:     unifiedResourceAlertNode(resource),
		Instance: unifiedResourceAlertInstance(resource, typeKey),
		Tags:     append([]string(nil), resource.Tags...),
	}

	if metrics := resource.Metrics; metrics != nil {
		input.CPU = unifiedResourceMetric(metrics.CPU)
		input.Memory = unifiedResourceMetric(metrics.Memory)
		input.Disk = unifiedResourceMetric(metrics.Disk)
		input.DiskRead = unifiedResourceMetric(metrics.DiskRead)
		input.DiskWrite = unifiedResourceMetric(metrics.DiskWrite)
		input.NetworkIn = unifiedResourceMetric(metrics.NetIn)
		input.NetworkOut = unifiedResourceMetric(metrics.NetOut)
	}
	if resource.Temperature != nil {
		input.Temperature = &UnifiedResourceMetric{Value: *resource.Temperature, Percent: *resource.Temperature}
	} else if resource.PhysicalDisk != nil &&
		diskinventory.TemperatureCollected(resource.PhysicalDisk.Temperature, resource.PhysicalDisk.Collection) {
		// A retained last-known reading is not current evidence of heat.
		value := float64(resource.PhysicalDisk.Temperature)
		input.Temperature = &UnifiedResourceMetric{Value: value, Percent: value}
	}
	if resource.PhysicalDisk != nil {
		input.DiskType = strings.ToLower(strings.TrimSpace(resource.PhysicalDisk.DiskType))
	}

	return input, true
}

func unifiedAlertResourceType(resource unifiedresources.Resource) (string, bool) {
	resourceType := unifiedresources.CanonicalResourceType(resource.Type)
	switch resourceType {
	case unifiedresources.ResourceTypeK8sCluster,
		unifiedresources.ResourceTypeK8sNode,
		unifiedresources.ResourceTypeK8sDeployment,
		unifiedresources.ResourceTypeK8sNamespace,
		unifiedresources.ResourceTypePod:
		return string(resourceType), true
	case unifiedresources.ResourceTypeAgent:
		if resource.TrueNAS != nil || resourceHasSource(resource, unifiedresources.SourceTrueNAS) {
			return "truenas-system", true
		}
		if resource.VMware != nil || resourceHasSource(resource, unifiedresources.SourceVMware) {
			return "vmware-host", true
		}
	case unifiedresources.ResourceTypeVM:
		if resource.VMware != nil || resourceHasSource(resource, unifiedresources.SourceVMware) {
			return "vmware-vm", true
		}
	case unifiedresources.ResourceTypeAppContainer:
		if resource.TrueNAS != nil || resourceHasSource(resource, unifiedresources.SourceTrueNAS) {
			return "truenas-app", true
		}
	case unifiedresources.ResourceTypeStorage:
		if isTrueNASStorageResource(resource) {
			switch strings.ToLower(strings.TrimSpace(resource.Storage.Topology)) {
			case "dataset":
				return "truenas-dataset", true
			default:
				return "truenas-pool", true
			}
		}
		if isVMwareDatastoreResource(resource) {
			return "vmware-datastore", true
		}
	case unifiedresources.ResourceTypePhysicalDisk:
		if resourceHasSource(resource, unifiedresources.SourceTrueNAS) || resource.TrueNAS != nil {
			return "truenas-disk", true
		}
	case unifiedresources.ResourceTypeNetwork:
		if resource.VMware != nil || resourceHasSource(resource, unifiedresources.SourceVMware) {
			return "vmware-network", true
		}
	}
	return "", false
}

func isTrueNASStorageResource(resource unifiedresources.Resource) bool {
	if resource.TrueNAS != nil || resourceHasSource(resource, unifiedresources.SourceTrueNAS) {
		return true
	}
	if resource.Storage == nil {
		return false
	}
	return strings.EqualFold(strings.TrimSpace(resource.Storage.Platform), string(unifiedresources.SourceTrueNAS))
}

func isVMwareDatastoreResource(resource unifiedresources.Resource) bool {
	if resource.VMware != nil || resourceHasSource(resource, unifiedresources.SourceVMware) {
		return true
	}
	if resource.Storage == nil {
		return false
	}
	platform := strings.TrimSpace(resource.Storage.Platform)
	return strings.EqualFold(platform, "vmware-vsphere") || strings.EqualFold(platform, string(unifiedresources.SourceVMware))
}

func resourceHasSource(resource unifiedresources.Resource, source unifiedresources.DataSource) bool {
	for _, candidate := range resource.Sources {
		if candidate == source {
			return true
		}
	}
	if _, ok := resource.SourceStatus[source]; ok {
		return true
	}
	return false
}

func unifiedResourceMetric(metric *unifiedresources.MetricValue) *UnifiedResourceMetric {
	if metric == nil {
		return nil
	}
	return &UnifiedResourceMetric{Value: metric.Value, Percent: metric.Percent}
}

func unifiedResourceAlertName(resource unifiedresources.Resource) string {
	if resource.Canonical != nil && strings.TrimSpace(resource.Canonical.DisplayName) != "" {
		return strings.TrimSpace(resource.Canonical.DisplayName)
	}
	if strings.TrimSpace(resource.Name) != "" {
		return strings.TrimSpace(resource.Name)
	}
	return strings.TrimSpace(resource.ID)
}

func unifiedResourceAlertNode(resource unifiedresources.Resource) string {
	if strings.TrimSpace(resource.ParentName) != "" {
		return strings.TrimSpace(resource.ParentName)
	}
	if resource.Canonical != nil && strings.TrimSpace(resource.Canonical.Hostname) != "" {
		return strings.TrimSpace(resource.Canonical.Hostname)
	}
	if resource.Kubernetes != nil {
		if strings.TrimSpace(resource.Kubernetes.NodeName) != "" {
			return strings.TrimSpace(resource.Kubernetes.NodeName)
		}
		if strings.TrimSpace(resource.Kubernetes.ClusterName) != "" {
			return strings.TrimSpace(resource.Kubernetes.ClusterName)
		}
	}
	if resource.TrueNAS != nil && strings.TrimSpace(resource.TrueNAS.Hostname) != "" {
		return strings.TrimSpace(resource.TrueNAS.Hostname)
	}
	if resource.VMware != nil {
		if strings.TrimSpace(resource.VMware.RuntimeHostName) != "" {
			return strings.TrimSpace(resource.VMware.RuntimeHostName)
		}
		if strings.TrimSpace(resource.VMware.ClusterName) != "" {
			return strings.TrimSpace(resource.VMware.ClusterName)
		}
		if strings.TrimSpace(resource.VMware.DatacenterName) != "" {
			return strings.TrimSpace(resource.VMware.DatacenterName)
		}
		if strings.TrimSpace(resource.VMware.VCenterHost) != "" {
			return strings.TrimSpace(resource.VMware.VCenterHost)
		}
	}
	return unifiedResourceAlertName(resource)
}

func unifiedResourceAlertInstance(resource unifiedresources.Resource, typeKey string) string {
	if resource.Kubernetes != nil {
		if strings.TrimSpace(resource.Kubernetes.ClusterName) != "" {
			return strings.TrimSpace(resource.Kubernetes.ClusterName)
		}
		return "Kubernetes"
	}
	if strings.HasPrefix(typeKey, "truenas-") {
		return "TrueNAS"
	}
	if strings.HasPrefix(typeKey, "vmware-") {
		if resource.VMware != nil {
			if strings.TrimSpace(resource.VMware.ConnectionName) != "" {
				return strings.TrimSpace(resource.VMware.ConnectionName)
			}
			if strings.TrimSpace(resource.VMware.VCenterHost) != "" {
				return strings.TrimSpace(resource.VMware.VCenterHost)
			}
		}
		return "vSphere"
	}
	return ""
}

func (i *UnifiedResourceInput) CPUValue() float64 {
	if i == nil || i.CPU == nil {
		return 0
	}
	return i.CPU.Percent
}

func (i *UnifiedResourceInput) MemoryValue() float64 {
	if i == nil || i.Memory == nil {
		return 0
	}
	return i.Memory.Percent
}

func (i *UnifiedResourceInput) DiskValue() float64 {
	if i == nil || i.Disk == nil {
		return 0
	}
	return i.Disk.Percent
}

func (i *UnifiedResourceInput) DiskReadValue() float64 {
	if i == nil || i.DiskRead == nil {
		return 0
	}
	return i.DiskRead.Value
}

func (i *UnifiedResourceInput) DiskWriteValue() float64 {
	if i == nil || i.DiskWrite == nil {
		return 0
	}
	return i.DiskWrite.Value
}

func (i *UnifiedResourceInput) NetworkInValue() float64 {
	if i == nil || i.NetworkIn == nil {
		return 0
	}
	return i.NetworkIn.Value
}

func (i *UnifiedResourceInput) NetworkOutValue() float64 {
	if i == nil || i.NetworkOut == nil {
		return 0
	}
	return i.NetworkOut.Value
}

func (i *UnifiedResourceInput) TemperatureValue() float64 {
	if i == nil || i.Temperature == nil {
		return 0
	}
	return i.Temperature.Value
}

// CheckUnifiedResource evaluates threshold-based metric alerts for a unified resource.
// It resolves thresholds (defaults + overrides) and calls checkMetric for each
// available metric. Discrete event alerts (offline, RAID, backup age, etc.)
// are NOT evaluated here — they remain in the typed Check* methods.
func (m *Manager) CheckUnifiedResource(input *UnifiedResourceInput) {
	m.CheckUnifiedResourceWithCapacityTrend(input, CapacityTrendObservation{})
}

// CheckUnifiedResourceWithCapacityTrend keeps predictive and static capacity
// evaluation on the same canonical lifecycle for every admitted storage
// platform.
func (m *Manager) CheckUnifiedResourceWithCapacityTrend(input *UnifiedResourceInput, trend CapacityTrendObservation) {
	if input == nil {
		return
	}

	m.mu.RLock()
	if !m.config.Enabled {
		m.mu.RUnlock()
		return
	}
	if m.unifiedPlatformAlertsDisabledNoLock(input.Type) {
		m.mu.RUnlock()
		return
	}

	thresholds := m.effectiveAlertPolicyNoLock(alertPolicyQuery{TypeKey: input.Type, ResourceID: input.ID, DiskType: input.DiskType}).Thresholds
	m.mu.RUnlock()

	if thresholds.Disabled {
		return
	}

	log.Debug().
		Str("resourceID", input.ID).
		Str("resourceName", input.Name).
		Str("resourceType", unifiedAlertType(input.Type)).
		Msg("Evaluating unified resource metrics")

	if unifiedStorageUsageResourceType(input.Type) && input.Disk != nil {
		m.evaluateUnifiedCapacity(input, thresholds, trend, func() bool {
			return !m.config.Enabled || m.unifiedPlatformAlertsDisabledNoLock(input.Type) || m.resolveResourceThresholds(input.Type, input.ID).Disabled
		})
		return
	}

	m.evaluateUnifiedMetrics(input, thresholds, nil)
}