Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
package alerts
import (
"math"
"strings"
alertspecs "github.com/rcourtman/pulse-go-rewrite/internal/alerts/specs"
"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
"github.com/rcourtman/pulse-go-rewrite/pkg/diskinventory"
"github.com/rs/zerolog/log"
)
// Override Key Format by Resource Type
//
// The config.Overrides map is keyed by resource ID. Each resource type
// uses the following canonical key format:
//
// VM / Container: monitoring resource ID, e.g. "pve1:node1:100"
// Node: node.ID e.g. "node/pve-1"
// Agent: host.ID e.g. "host1" (without "agent:" prefix)
// Agent Disk: "agent:{hostID}/disk:{mountpoint}" e.g. "agent:host1/disk:root"
// Storage: storage.ID e.g. "local-lvm"
// PBS: pbs.ID e.g. "pbs-1"
// PMG: pmg.ID e.g. "pmg-1"
// Docker Container: "docker:{hostID}/{containerName}" (stable across
// container recreates; legacy "docker:{hostID}/{containerID}"
// entries are still honoured and migrated, #1601)
// Kubernetes: canonical unified resource ID for cluster/node/namespace/deployment/pod
// TrueNAS System: canonical unified agent resource ID
// TrueNAS Storage: canonical unified storage resource ID for pool/dataset
// TrueNAS Disk: canonical unified physical_disk resource ID
// VMware vSphere: canonical unified resource ID for host/vm/datastore/network
//
// For guests, CheckUnifiedResource expects the same canonical resource ID the
// monitoring pipeline already uses. It does not translate IDs.
// UnifiedResourceMetric holds a single metric value for unified evaluation.
type UnifiedResourceMetric struct {
Value float64
Percent float64
}
// UnifiedResourceInput is the data needed to evaluate alerts for a unified resource.
// This avoids importing unifiedresources (which would cause an import cycle).
type UnifiedResourceInput struct {
ID string
Type string // lowercase: "vm", "system-container", "app-container", "agent", "pbs", "storage", "pmg"
Name string
Node string
Instance string
Tags []string
CPU *UnifiedResourceMetric
Memory *UnifiedResourceMetric
Disk *UnifiedResourceMetric
DiskRead *UnifiedResourceMetric
DiskWrite *UnifiedResourceMetric
NetworkIn *UnifiedResourceMetric
NetworkOut *UnifiedResourceMetric
Temperature *UnifiedResourceMetric
StorageAliases []string // Durable policy lookup identities for storage alerts.
// DiskType is a physical disk's type ("nvme", "sas", "sata"). It picks
// the disk temperature threshold of a TrueNAS disk.
DiskType string
}
type unifiedMetricCandidate struct {
Spec alertspecs.ResourceAlertSpec
Value float64
Threshold *HysteresisThreshold
}
// unifiedAlertType maps a resource type key to the alert system's display type.
func unifiedAlertType(typeKey string) string {
switch typeKey {
case "vm":
return "VM"
case "system-container":
return "Container"
case "app-container":
return "Docker"
case "agent":
return "Agent"
case "node":
return "Node"
case "pbs":
return "PBS"
case "storage":
return "Storage"
case "pmg":
return "PMG"
case "k8s-cluster":
return "Kubernetes Cluster"
case "k8s-node":
return "Kubernetes Node"
case "k8s-deployment":
return "Kubernetes Deployment"
case "k8s-namespace":
return "Kubernetes Namespace"
case "pod":
return "Kubernetes Pod"
case "truenas-system":
return "TrueNAS System"
case "truenas-pool":
return "TrueNAS Pool"
case "truenas-dataset":
return "TrueNAS Dataset"
case "truenas-disk":
return "TrueNAS Disk"
case "vmware-host":
return "vSphere Host"
case "vmware-vm":
return "vSphere VM"
case "vmware-datastore":
return "vSphere Datastore"
case "vmware-network":
return "vSphere Network"
default:
return typeKey
}
}
func supportsUnifiedIOMetrics(typeKey string) bool {
switch typeKey {
case "vm", "system-container", "app-container", "k8s-cluster", "k8s-deployment", "pod", "truenas-system", "vmware-host", "vmware-vm":
return true
default:
return false
}
}
func isUnifiedKubernetesAlertType(typeKey string) bool {
switch typeKey {
case "k8s-cluster", "k8s-node", "k8s-deployment", "k8s-namespace", "pod":
return true
default:
return false
}
}
func isUnifiedTrueNASAlertType(typeKey string) bool {
switch typeKey {
case "truenas-system", "truenas-pool", "truenas-dataset", "truenas-disk":
return true
default:
return false
}
}
func isUnifiedVMwareAlertType(typeKey string) bool {
switch typeKey {
case "vmware-host", "vmware-vm", "vmware-datastore", "vmware-network":
return true
default:
return false
}
}
func isUnifiedModernPlatformAlertType(typeKey string) bool {
return isUnifiedKubernetesAlertType(typeKey) || isUnifiedTrueNASAlertType(typeKey) || isUnifiedVMwareAlertType(typeKey)
}
// alertPlatformTypeKey carries the canonical platform id of the resource an
// alert is about. Metric alerts put a display label such as "vSphere VM" in
// resourceType, so consumers that link or name the owning platform read this.
const alertPlatformTypeKey = "platformType"
// unifiedAlertPlatformType returns the canonical platform id for a unified
// alert type key, or "" when the key does not name one.
func unifiedAlertPlatformType(typeKey string) string {
switch {
case isUnifiedKubernetesAlertType(typeKey):
return "kubernetes"
case strings.HasPrefix(typeKey, "truenas-"):
return "truenas"
case strings.HasPrefix(typeKey, "vmware-"):
return "vmware-vsphere"
default:
return ""
}
}
func (m *Manager) unifiedPlatformAlertsDisabledNoLock(typeKey string) bool {
switch {
case isUnifiedKubernetesAlertType(typeKey):
return m.config.DisableAllKubernetes
case isUnifiedTrueNASAlertType(typeKey):
return m.config.DisableAllTrueNAS
case isUnifiedVMwareAlertType(typeKey):
return m.config.DisableAllVMware
default:
return false
}
}
// defaultThresholdsForResourceType returns the default ThresholdConfig for a resource type key.
func (m *Manager) defaultThresholdsForResourceType(typeKey string) ThresholdConfig {
switch typeKey {
case "vm", "system-container", "app-container":
return cloneThresholdConfig(m.config.GuestDefaults)
case "agent":
return cloneThresholdConfig(m.config.AgentDefaults)
case "node":
return cloneThresholdConfig(m.config.NodeDefaults)
case "pbs":
return cloneThresholdConfig(m.config.PBSDefaults)
case "storage":
return ThresholdConfig{Usage: cloneThreshold(&m.config.StorageDefault)}
case "k8s-cluster", "k8s-node", "k8s-deployment", "pod":
return cloneThresholdConfig(m.config.KubernetesDefaults)
case "k8s-namespace":
return ThresholdConfig{}
case "truenas-system":
return cloneThresholdConfig(m.config.TrueNASDefaults)
case "truenas-pool", "truenas-dataset":
defaults := cloneThresholdConfig(m.config.TrueNASDefaults)
return ThresholdConfig{Usage: cloneThreshold(defaults.Usage)}
case "truenas-disk":
return cloneThresholdConfig(m.config.TrueNASDiskDefaults)
case "vmware-host", "vmware-vm":
return cloneThresholdConfig(m.config.VMwareDefaults)
case "vmware-datastore":
defaults := cloneThresholdConfig(m.config.VMwareDefaults)
return ThresholdConfig{Usage: cloneThreshold(defaults.Usage)}
case "vmware-network":
return ThresholdConfig{}
default:
return ThresholdConfig{}
}
}
// resolveThresholdOverride applies an exact resource-ID override onto an existing base config.
// Callers must hold m.mu when reading config through this helper.
func (m *Manager) resolveThresholdOverride(base ThresholdConfig, resourceID string) ThresholdConfig {
if override, exists := m.thresholdOverrideForResourceNoLock(resourceID); exists {
return m.applyThresholdOverride(base, override)
}
return base
}
// thresholdOverrideForResourceNoLock looks up a stored threshold override by
// exact resource ID first, then under the canonical unified registry identity
// for that ID. The thresholds UI stores overrides keyed by registry resource
// IDs while the engine check paths look up legacy monitor IDs, so without the
// translation a per-resource override on an API-only PVE node saves, displays
// as Custom, and silently never applies (#1738). Callers must hold m.mu.
func (m *Manager) thresholdOverrideForResourceNoLock(resourceID string) (ThresholdConfig, bool) {
if override, exists := m.config.Overrides[resourceID]; exists {
return override, true
}
if m.resourceIntentResolver == nil {
return ThresholdConfig{}, false
}
canonicalID, ok := m.resourceIntentResolver(resourceID)
if !ok {
return ThresholdConfig{}, false
}
canonicalID = strings.TrimSpace(canonicalID)
if canonicalID == "" || canonicalID == resourceID {
return ThresholdConfig{}, false
}
override, exists := m.config.Overrides[canonicalID]
return override, exists
}
// resolveGuestThresholdOverride applies the shared guest override lookup onto an existing base config.
// Callers must hold m.mu when reading config through this helper.
func (m *Manager) resolveGuestThresholdOverride(base ThresholdConfig, guest any, guestID string) ThresholdConfig {
if override, exists := lookupGuestOverride(m.config.Overrides, guest, guestID); exists {
return m.applyThresholdOverride(base, override)
}
return base
}
// resolveResourceThresholds builds the effective thresholds for a resource type and ID.
// Callers must hold m.mu when reading config through this helper.
func (m *Manager) resolveResourceThresholds(typeKey, resourceID string) ThresholdConfig {
return m.effectiveAlertPolicyNoLock(alertPolicyQuery{TypeKey: typeKey, ResourceID: resourceID}).Thresholds
}
// evaluateUnifiedMetrics runs the common metric dispatch path for unified resources.
func (m *Manager) evaluateUnifiedMetrics(input *UnifiedResourceInput, thresholds ThresholdConfig, opts *metricOptions, separatelyEvaluated ...string) {
if input == nil {
return
}
opts = metricOptionsWithTags(opts, input.Tags)
platformType := unifiedAlertPlatformType(input.Type)
if len(input.StorageAliases) > 0 || platformType != "" {
merged := metricOptions{}
if opts != nil {
merged = *opts
}
metadata := make(map[string]interface{}, len(merged.Metadata)+2)
for k, v := range merged.Metadata {
metadata[k] = v
}
if len(input.StorageAliases) > 0 {
metadata[storagePolicyAliasesKey] = append([]string(nil), input.StorageAliases...)
}
if platformType != "" {
metadata[alertPlatformTypeKey] = platformType
}
if input.Type == "truenas-disk" {
// A config save re-judges the open temperature alert against
// the threshold for this disk type.
metadata["diskType"] = input.DiskType
}
merged.Metadata = metadata
opts = &merged
}
candidates := buildUnifiedMetricCandidates(input, thresholds)
m.interruptUnobservedUnifiedMetrics(input, candidates, separatelyEvaluated)
for _, candidate := range candidates {
m.checkMetricWithCanonicalSpec(candidate.Spec, input.Name, input.Node, input.Instance, unifiedAlertType(input.Type), candidate.Value, candidate.Threshold, opts)
}
}
// An omitted/rejected metric is unknown, not a pause in its sustained-for
// clock. Build candidates stays pure; dispatch owns interruption for metrics
// this resource supports but cannot currently observe.
func (m *Manager) interruptUnobservedUnifiedMetrics(input *UnifiedResourceInput, candidates []unifiedMetricCandidate, separatelyEvaluated []string) {
if _, ok := unifiedMetricResourceType(input.Type); !ok {
return
}
metrics := [...]string{"cpu", "memory", "disk", "temperature", "diskRead", "diskWrite", "networkIn", "networkOut", "usage"}
m.mu.Lock()
intentChanged := false
for _, metric := range metrics {
separateOwner := false
for _, separateMetric := range separatelyEvaluated {
if metric == separateMetric {
separateOwner = true
break
}
}
if separateOwner {
continue
}
switch metric {
case "diskRead", "diskWrite", "networkIn", "networkOut":
if !supportsUnifiedIOMetrics(input.Type) {
continue
}
case "usage":
if !unifiedStorageUsageResourceType(input.Type) {
continue
}
}
observed := false
for _, candidate := range candidates {
if candidate.Spec.MetricThreshold.Metric == metric {
observed = true
break
}
}
if !observed {
if m.interruptMetricRunNoLock(input.ID, canonicalMetricSpecID(input.ID, metric), canonicalMetricStateID(input.ID, metric)) {
intentChanged = true
}
}
}
m.mu.Unlock()
if intentChanged {
m.saveActiveAlertsAsync("unified metric observation gap")
}
}
// metricOptionsWithTags records a resource's tags on its metric alerts. With
// no tags it drops any an open alert still carries: config-save reevaluation
// reads pulse-relaxed back from them, and a removed tag left behind judged
// the alert by thresholds the evaluator no longer applies.
func metricOptionsWithTags(opts *metricOptions, tags []string) *metricOptions {
merged := metricOptions{}
if opts != nil {
merged = *opts
}
if len(tags) == 0 {
merged.RemoveMetadata = append(append([]string(nil), merged.RemoveMetadata...), "tags")
return &merged
}
metadata := make(map[string]interface{}, len(merged.Metadata)+1)
for key, value := range merged.Metadata {
metadata[key] = value
}
metadata["tags"] = append([]string(nil), tags...)
merged.Metadata = metadata
return &merged
}
func buildUnifiedMetricCandidates(input *UnifiedResourceInput, thresholds ThresholdConfig) []unifiedMetricCandidate {
if input == nil {
return nil
}
resourceType, ok := unifiedMetricResourceType(input.Type)
if !ok {
return nil
}
candidates := make([]unifiedMetricCandidate, 0, 8)
appendCandidate := func(metricType string, metric *UnifiedResourceMetric, value float64, threshold *HysteresisThreshold) {
if metric == nil {
return
}
if math.IsNaN(value) || math.IsInf(value, 0) {
return
}
if isPercentageMetric(metricType) && (value < 0 || value > 100) {
return
}
spec, err := buildCanonicalMetricSpec(input.ID, input.Name, resourceType, metricType, threshold)
if err != nil {
log.Warn().
Err(err).
Str("resourceID", input.ID).
Str("resourceType", input.Type).
Str("metricType", metricType).
Msg("Skipping invalid canonical unified metric spec")
return
}
candidates = append(candidates, unifiedMetricCandidate{
Spec: spec,
Value: value,
Threshold: threshold,
})
}
appendCandidate("cpu", input.CPU, input.CPUValue(), thresholds.CPU)
appendCandidate("memory", input.Memory, input.MemoryValue(), thresholds.Memory)
appendCandidate("disk", input.Disk, input.DiskValue(), thresholds.Disk)
if supportsUnifiedIOMetrics(input.Type) {
appendCandidate("diskRead", input.DiskRead, input.DiskReadValue(), thresholds.DiskRead)
appendCandidate("diskWrite", input.DiskWrite, input.DiskWriteValue(), thresholds.DiskWrite)
appendCandidate("networkIn", input.NetworkIn, input.NetworkInValue(), thresholds.NetworkIn)
appendCandidate("networkOut", input.NetworkOut, input.NetworkOutValue(), thresholds.NetworkOut)
}
if unifiedStorageUsageResourceType(input.Type) && input.Disk != nil {
appendCandidate("usage", input.Disk, input.DiskValue(), thresholds.Usage)
}
appendCandidate("temperature", input.Temperature, input.TemperatureValue(), thresholds.Temperature)
return candidates
}
func unifiedStorageUsageResourceType(typeKey string) bool {
switch typeKey {
case "storage", "truenas-pool", "truenas-dataset", "vmware-datastore":
return true
default:
return false
}
}
// isPercentageMetric returns true for metrics whose values are bounded 0–100.
// The critical threshold for these must stay below 100 or escalation will never fire.
func isPercentageMetric(metricType string) bool {
switch metricType {
case "cpu", "memory", "disk", "usage":
return true
default:
return false
}
}
// computeCriticalThreshold derives the critical (severity-escalation) threshold
// from the trigger value. For percentage metrics (0–100 range) the result is
// capped at 99 so that critical escalation remains reachable when the trigger
// is set high (e.g. trigger 95 → critical 99, not 105).
func computeCriticalThreshold(trigger float64, metricType string) float64 {
critical := trigger + 10
if isPercentageMetric(metricType) && critical > 99 {
return 99
}
return critical
}
func buildCanonicalMetricSpec(resourceID, title string, resourceType unifiedresources.ResourceType, metricType string, threshold *HysteresisThreshold) (alertspecs.ResourceAlertSpec, error) {
spec := alertspecs.ResourceAlertSpec{
ID: canonicalMetricSpecID(resourceID, metricType),
ResourceID: resourceID,
ResourceType: resourceType,
Kind: alertspecs.AlertSpecKindMetricThreshold,
Severity: alertspecs.AlertSeverityWarning,
Title: title,
Disabled: threshold == nil || threshold.Trigger <= 0,
MetricThreshold: &alertspecs.MetricThresholdSpec{
Metric: metricType,
Direction: alertspecs.ThresholdDirectionAbove,
Trigger: 0,
},
}
if threshold != nil {
spec.MetricThreshold.Trigger = threshold.Trigger
if threshold.Clear > 0 && threshold.Clear < threshold.Trigger {
recovery := threshold.Clear
spec.MetricThreshold.Recovery = &recovery
}
// A percentage trigger of 99+ collides with the 99 critical cap; a
// critical at or below the trigger fails spec validation and used to
// drop the whole metric from evaluation (silently unmonitored, plus a
// warn-log every cycle). Omit escalation instead so the warning-level
// threshold keeps working.
if critical := computeCriticalThreshold(threshold.Trigger, metricType); critical > threshold.Trigger {
spec.MetricThreshold.Critical = &critical
}
}
return spec, spec.Validate()
}
func (m *Manager) checkMetricWithCanonicalSpec(spec alertspecs.ResourceAlertSpec, resourceName, node, instance, resourceType string, value float64, threshold *HysteresisThreshold, opts *metricOptions) bool {
if spec.MetricThreshold == nil {
return false
}
return m.evaluateCanonicalMetricAlert(spec, resourceName, node, instance, resourceType, value, threshold, opts)
}
func unifiedMetricResourceType(typeKey string) (unifiedresources.ResourceType, bool) {
switch typeKey {
case "vm":
return unifiedresources.ResourceTypeVM, true
case "system-container":
return unifiedresources.ResourceTypeSystemContainer, true
case "app-container":
return unifiedresources.ResourceTypeAppContainer, true
case "agent":
return unifiedresources.ResourceTypeAgent, true
case "node":
return unifiedresources.ResourceType("node"), true
case "pbs":
return unifiedresources.ResourceTypePBS, true
case "storage":
return unifiedresources.ResourceTypeStorage, true
case "pmg":
return unifiedresources.ResourceTypePMG, true
case "k8s-cluster":
return unifiedresources.ResourceTypeK8sCluster, true
case "k8s-node":
return unifiedresources.ResourceTypeK8sNode, true
case "k8s-deployment":
return unifiedresources.ResourceTypeK8sDeployment, true
case "k8s-namespace":
return unifiedresources.ResourceTypeK8sNamespace, true
case "pod":
return unifiedresources.ResourceTypePod, true
case "truenas-system":
return unifiedresources.ResourceTypeAgent, true
case "truenas-pool", "truenas-dataset":
return unifiedresources.ResourceTypeStorage, true
case "truenas-disk":
return unifiedresources.ResourceTypePhysicalDisk, true
case "vmware-host":
return unifiedresources.ResourceTypeAgent, true
case "vmware-vm":
return unifiedresources.ResourceTypeVM, true
case "vmware-datastore":
return unifiedresources.ResourceTypeStorage, true
case "vmware-network":
return unifiedresources.ResourceTypeNetwork, true
default:
return "", false
}
}
func (m *Manager) CheckUnifiedResourceMetrics(resources []unifiedresources.Resource) {
m.CheckUnifiedResourceMetricsWithCapacityTrends(resources, nil)
}
// CheckUnifiedResourceMetricsWithCapacityTrends evaluates canonical resources
// with optional alert-grade capacity evidence keyed by canonical resource ID.
func (m *Manager) CheckUnifiedResourceMetricsWithCapacityTrends(resources []unifiedresources.Resource, capacityTrends map[string]CapacityTrendObservation) {
if m == nil {
return
}
for _, resource := range resources {
input, ok := UnifiedResourceInputFromResource(resource)
if !ok {
continue
}
m.CheckUnifiedResourceWithCapacityTrend(input, capacityTrends[input.ID])
}
}
func UnifiedResourceInputFromResource(resource unifiedresources.Resource) (*UnifiedResourceInput, bool) {
typeKey, ok := unifiedAlertResourceType(resource)
if !ok {
return nil, false
}
resourceID := unifiedresources.CanonicalResourceID(resource.ID)
if strings.TrimSpace(resourceID) == "" {
return nil, false
}
input := &UnifiedResourceInput{
ID: resourceID,
Type: typeKey,
Name: unifiedResourceAlertName(resource),
Node: unifiedResourceAlertNode(resource),
Instance: unifiedResourceAlertInstance(resource, typeKey),
Tags: append([]string(nil), resource.Tags...),
}
if metrics := resource.Metrics; metrics != nil {
input.CPU = unifiedResourceMetric(metrics.CPU)
input.Memory = unifiedResourceMetric(metrics.Memory)
input.Disk = unifiedResourceMetric(metrics.Disk)
input.DiskRead = unifiedResourceMetric(metrics.DiskRead)
input.DiskWrite = unifiedResourceMetric(metrics.DiskWrite)
input.NetworkIn = unifiedResourceMetric(metrics.NetIn)
input.NetworkOut = unifiedResourceMetric(metrics.NetOut)
}
if resource.Temperature != nil {
input.Temperature = &UnifiedResourceMetric{Value: *resource.Temperature, Percent: *resource.Temperature}
} else if resource.PhysicalDisk != nil &&
diskinventory.TemperatureCollected(resource.PhysicalDisk.Temperature, resource.PhysicalDisk.Collection) {
// A retained last-known reading is not current evidence of heat.
value := float64(resource.PhysicalDisk.Temperature)
input.Temperature = &UnifiedResourceMetric{Value: value, Percent: value}
}
if resource.PhysicalDisk != nil {
input.DiskType = strings.ToLower(strings.TrimSpace(resource.PhysicalDisk.DiskType))
}
return input, true
}
func unifiedAlertResourceType(resource unifiedresources.Resource) (string, bool) {
resourceType := unifiedresources.CanonicalResourceType(resource.Type)
switch resourceType {
case unifiedresources.ResourceTypeK8sCluster,
unifiedresources.ResourceTypeK8sNode,
unifiedresources.ResourceTypeK8sDeployment,
unifiedresources.ResourceTypeK8sNamespace,
unifiedresources.ResourceTypePod:
return string(resourceType), true
case unifiedresources.ResourceTypeAgent:
if resource.TrueNAS != nil || resourceHasSource(resource, unifiedresources.SourceTrueNAS) {
return "truenas-system", true
}
if resource.VMware != nil || resourceHasSource(resource, unifiedresources.SourceVMware) {
return "vmware-host", true
}
case unifiedresources.ResourceTypeVM:
if resource.VMware != nil || resourceHasSource(resource, unifiedresources.SourceVMware) {
return "vmware-vm", true
}
case unifiedresources.ResourceTypeAppContainer:
if resource.TrueNAS != nil || resourceHasSource(resource, unifiedresources.SourceTrueNAS) {
return "truenas-app", true
}
case unifiedresources.ResourceTypeStorage:
if isTrueNASStorageResource(resource) {
switch strings.ToLower(strings.TrimSpace(resource.Storage.Topology)) {
case "dataset":
return "truenas-dataset", true
default:
return "truenas-pool", true
}
}
if isVMwareDatastoreResource(resource) {
return "vmware-datastore", true
}
case unifiedresources.ResourceTypePhysicalDisk:
if resourceHasSource(resource, unifiedresources.SourceTrueNAS) || resource.TrueNAS != nil {
return "truenas-disk", true
}
case unifiedresources.ResourceTypeNetwork:
if resource.VMware != nil || resourceHasSource(resource, unifiedresources.SourceVMware) {
return "vmware-network", true
}
}
return "", false
}
func isTrueNASStorageResource(resource unifiedresources.Resource) bool {
if resource.TrueNAS != nil || resourceHasSource(resource, unifiedresources.SourceTrueNAS) {
return true
}
if resource.Storage == nil {
return false
}
return strings.EqualFold(strings.TrimSpace(resource.Storage.Platform), string(unifiedresources.SourceTrueNAS))
}
func isVMwareDatastoreResource(resource unifiedresources.Resource) bool {
if resource.VMware != nil || resourceHasSource(resource, unifiedresources.SourceVMware) {
return true
}
if resource.Storage == nil {
return false
}
platform := strings.TrimSpace(resource.Storage.Platform)
return strings.EqualFold(platform, "vmware-vsphere") || strings.EqualFold(platform, string(unifiedresources.SourceVMware))
}
func resourceHasSource(resource unifiedresources.Resource, source unifiedresources.DataSource) bool {
for _, candidate := range resource.Sources {
if candidate == source {
return true
}
}
if _, ok := resource.SourceStatus[source]; ok {
return true
}
return false
}
func unifiedResourceMetric(metric *unifiedresources.MetricValue) *UnifiedResourceMetric {
if metric == nil {
return nil
}
return &UnifiedResourceMetric{Value: metric.Value, Percent: metric.Percent}
}
func unifiedResourceAlertName(resource unifiedresources.Resource) string {
if resource.Canonical != nil && strings.TrimSpace(resource.Canonical.DisplayName) != "" {
return strings.TrimSpace(resource.Canonical.DisplayName)
}
if strings.TrimSpace(resource.Name) != "" {
return strings.TrimSpace(resource.Name)
}
return strings.TrimSpace(resource.ID)
}
func unifiedResourceAlertNode(resource unifiedresources.Resource) string {
if strings.TrimSpace(resource.ParentName) != "" {
return strings.TrimSpace(resource.ParentName)
}
if resource.Canonical != nil && strings.TrimSpace(resource.Canonical.Hostname) != "" {
return strings.TrimSpace(resource.Canonical.Hostname)
}
if resource.Kubernetes != nil {
if strings.TrimSpace(resource.Kubernetes.NodeName) != "" {
return strings.TrimSpace(resource.Kubernetes.NodeName)
}
if strings.TrimSpace(resource.Kubernetes.ClusterName) != "" {
return strings.TrimSpace(resource.Kubernetes.ClusterName)
}
}
if resource.TrueNAS != nil && strings.TrimSpace(resource.TrueNAS.Hostname) != "" {
return strings.TrimSpace(resource.TrueNAS.Hostname)
}
if resource.VMware != nil {
if strings.TrimSpace(resource.VMware.RuntimeHostName) != "" {
return strings.TrimSpace(resource.VMware.RuntimeHostName)
}
if strings.TrimSpace(resource.VMware.ClusterName) != "" {
return strings.TrimSpace(resource.VMware.ClusterName)
}
if strings.TrimSpace(resource.VMware.DatacenterName) != "" {
return strings.TrimSpace(resource.VMware.DatacenterName)
}
if strings.TrimSpace(resource.VMware.VCenterHost) != "" {
return strings.TrimSpace(resource.VMware.VCenterHost)
}
}
return unifiedResourceAlertName(resource)
}
func unifiedResourceAlertInstance(resource unifiedresources.Resource, typeKey string) string {
if resource.Kubernetes != nil {
if strings.TrimSpace(resource.Kubernetes.ClusterName) != "" {
return strings.TrimSpace(resource.Kubernetes.ClusterName)
}
return "Kubernetes"
}
if strings.HasPrefix(typeKey, "truenas-") {
return "TrueNAS"
}
if strings.HasPrefix(typeKey, "vmware-") {
if resource.VMware != nil {
if strings.TrimSpace(resource.VMware.ConnectionName) != "" {
return strings.TrimSpace(resource.VMware.ConnectionName)
}
if strings.TrimSpace(resource.VMware.VCenterHost) != "" {
return strings.TrimSpace(resource.VMware.VCenterHost)
}
}
return "vSphere"
}
return ""
}
func (i *UnifiedResourceInput) CPUValue() float64 {
if i == nil || i.CPU == nil {
return 0
}
return i.CPU.Percent
}
func (i *UnifiedResourceInput) MemoryValue() float64 {
if i == nil || i.Memory == nil {
return 0
}
return i.Memory.Percent
}
func (i *UnifiedResourceInput) DiskValue() float64 {
if i == nil || i.Disk == nil {
return 0
}
return i.Disk.Percent
}
func (i *UnifiedResourceInput) DiskReadValue() float64 {
if i == nil || i.DiskRead == nil {
return 0
}
return i.DiskRead.Value
}
func (i *UnifiedResourceInput) DiskWriteValue() float64 {
if i == nil || i.DiskWrite == nil {
return 0
}
return i.DiskWrite.Value
}
func (i *UnifiedResourceInput) NetworkInValue() float64 {
if i == nil || i.NetworkIn == nil {
return 0
}
return i.NetworkIn.Value
}
func (i *UnifiedResourceInput) NetworkOutValue() float64 {
if i == nil || i.NetworkOut == nil {
return 0
}
return i.NetworkOut.Value
}
func (i *UnifiedResourceInput) TemperatureValue() float64 {
if i == nil || i.Temperature == nil {
return 0
}
return i.Temperature.Value
}
// CheckUnifiedResource evaluates threshold-based metric alerts for a unified resource.
// It resolves thresholds (defaults + overrides) and calls checkMetric for each
// available metric. Discrete event alerts (offline, RAID, backup age, etc.)
// are NOT evaluated here — they remain in the typed Check* methods.
func (m *Manager) CheckUnifiedResource(input *UnifiedResourceInput) {
m.CheckUnifiedResourceWithCapacityTrend(input, CapacityTrendObservation{})
}
// CheckUnifiedResourceWithCapacityTrend keeps predictive and static capacity
// evaluation on the same canonical lifecycle for every admitted storage
// platform.
func (m *Manager) CheckUnifiedResourceWithCapacityTrend(input *UnifiedResourceInput, trend CapacityTrendObservation) {
if input == nil {
return
}
m.mu.RLock()
if !m.config.Enabled {
m.mu.RUnlock()
return
}
if m.unifiedPlatformAlertsDisabledNoLock(input.Type) {
m.mu.RUnlock()
return
}
thresholds := m.effectiveAlertPolicyNoLock(alertPolicyQuery{TypeKey: input.Type, ResourceID: input.ID, DiskType: input.DiskType}).Thresholds
m.mu.RUnlock()
if thresholds.Disabled {
return
}
log.Debug().
Str("resourceID", input.ID).
Str("resourceName", input.Name).
Str("resourceType", unifiedAlertType(input.Type)).
Msg("Evaluating unified resource metrics")
if unifiedStorageUsageResourceType(input.Type) && input.Disk != nil {
m.evaluateUnifiedCapacity(input, thresholds, trend, func() bool {
return !m.config.Enabled || m.unifiedPlatformAlertsDisabledNoLock(input.Type) || m.resolveResourceThresholds(input.Type, input.ID).Disabled
})
return
}
m.evaluateUnifiedMetrics(input, thresholds, nil)
}