Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
package alerts
import (
"errors"
"fmt"
"strings"
"time"
alertconfig "github.com/rcourtman/pulse-go-rewrite/internal/alerts/config"
alertspecs "github.com/rcourtman/pulse-go-rewrite/internal/alerts/specs"
"github.com/rs/zerolog/log"
)
// UpdateConfig applies an owned copy of the alert configuration. The caller
// retains its maps, slices and thresholds for encoding or further edits; only
// another explicit update may change the running alert policy.
func (m *Manager) UpdateConfig(config AlertConfig) {
m.mu.Lock()
defer m.mu.Unlock()
m.updateConfigLocked(config.Clone())
}
// ErrConfigSnapshot reports an update that was applied but could not be copied
// for the caller.
var ErrConfigSnapshot = errors.New("snapshot applied alert configuration")
// ApplyConfigUpdate applies a client's JSON update to the stored
// configuration: keys the update carries replace their settings and the rest
// keep their stored values (see alertconfig.ApplyAlertConfigUpdate). Reading,
// merging and applying under one lock keeps two partial updates from
// reverting each other's settings. It returns a snapshot of the applied
// config that the caller owns, so persisting it cannot write into the live
// config's maps.
func (m *Manager) ApplyConfigUpdate(update []byte) (AlertConfig, error) {
m.mu.Lock()
defer m.mu.Unlock()
config, err := alertconfig.ApplyAlertConfigUpdate(m.config, update)
if err != nil {
return AlertConfig{}, err
}
m.updateConfigLocked(config)
snapshot, err := alertconfig.CloneAlertConfig(m.config)
if err != nil {
return AlertConfig{}, fmt.Errorf("%w: %w", ErrConfigSnapshot, err)
}
return snapshot, nil
}
func (m *Manager) updateConfigLocked(config AlertConfig) {
// Clients and rollback-era config writers may not know about the additive
// identity schema marker. Never lower a version already held by this
// process; an older binary can still omit it on disk and the migration will
// safely rerun after a later upgrade.
if config.IdentitySchemaVersion < m.config.IdentitySchemaVersion {
config.IdentitySchemaVersion = m.config.IdentitySchemaVersion
}
// Preserve activation state/time when clients update the config without including it.
// This avoids unintentionally resetting alerts to pending review when saving thresholds.
if config.ActivationState == "" && m.config.ActivationState != "" {
config.ActivationState = m.config.ActivationState
if config.ActivationTime == nil && m.config.ActivationTime != nil {
config.ActivationTime = m.config.ActivationTime
}
}
// Normalize all config sections
alertconfig.NormalizeAlertConfigAliases(&config)
alertconfig.NormalizeStorageDefaults(&config)
alertconfig.NormalizeDockerDefaults(&config)
alertconfig.NormalizePMGDefaults(&config)
alertconfig.NormalizePBSDefaults(&config)
alertconfig.NormalizeSnapshotDefaults(&config)
alertconfig.NormalizeBackupDefaults(&config)
alertconfig.NormalizeRecoveryOverrides(&config)
alertconfig.NormalizeNodeDefaults(&config)
alertconfig.NormalizeAgentDefaults(&config)
alertconfig.NormalizeKubernetesDefaults(&config)
alertconfig.NormalizeTrueNASDefaults(&config)
alertconfig.NormalizeVMwareDefaults(&config)
alertconfig.NormalizeGeneralSettings(&config)
alertconfig.NormalizeTimeThresholds(&config)
config.MetricEvaluationWindows = alertconfig.NormalizeMetricEvaluationWindows(config.MetricEvaluationWindows)
config.GuestDefaults.PoweredOffSeverity = alertconfig.NormalizePoweredOffSeverity(config.GuestDefaults.PoweredOffSeverity)
config.NodeDefaults.PoweredOffSeverity = alertconfig.NormalizePoweredOffSeverity(config.NodeDefaults.PoweredOffSeverity)
config.DockerIgnoredContainerPrefixes = alertconfig.NormalizeDockerIgnoredPrefixes(config.DockerIgnoredContainerPrefixes)
config.Schedule.InitialNotify = alertconfig.NormalizeNotificationDeliveryTarget(config.Schedule.InitialNotify)
alertconfig.NormalizeEscalationConfig(&config.Schedule.Escalation)
// Migration logic for activation state (backward compatibility)
m.migrateActivationState(&config)
// Validate hysteresis thresholds to prevent stuck alerts
alertconfig.ValidateHysteresisThresholds(&config)
// Validate timezone if quiet hours are enabled
alertconfig.ValidateQuietHoursTimezone(&config)
m.config = config
normalizeOverrides(m.config.Overrides)
m.reconcileHostAgentNodeLinksNoLock()
// Update cached quiet hours location
if m.config.Schedule.QuietHours.Enabled && m.config.Schedule.QuietHours.Timezone != "" {
loc, err := time.LoadLocation(m.config.Schedule.QuietHours.Timezone)
if err == nil {
m.quietHoursLoc = loc
} else {
m.quietHoursLoc = time.Local
}
} else {
m.quietHoursLoc = time.Local
}
if !m.config.SnapshotDefaults.Enabled {
m.clearSnapshotAlertsForInstanceLocked("")
}
if !m.config.BackupDefaults.Enabled {
m.clearBackupAlertsLocked()
}
m.applyGlobalOfflineSettingsLocked()
log.Info().
Bool("enabled", config.Enabled).
Interface("guestDefaults", config.GuestDefaults).
Msg("Alert configuration updated")
// Pending-only temperature runs have no active alert to re-evaluate.
// Disabling and re-enabling between reports must not reuse their grace.
if m.reevaluateHostDiskTemperaturePendingNoLock() {
m.saveActiveAlertsAsync("disk temperature pending policy disabled")
}
if m.reevaluateHostNativeSensorPendingNoLock() {
m.saveActiveAlertsAsync("native sensor pending policy removed")
}
// Re-evaluate active alerts against new thresholds
m.reevaluateActiveAlertsLocked()
m.pruneHostSMARTDiskAbsencesNoLock()
}
// migrateActivationState handles backward compatibility for activation state.
func (m *Manager) migrateActivationState(config *AlertConfig) {
if config.ActivationState == "" {
// Determine if this is an existing installation or new.
// Existing installations have active alerts already.
isExistingInstall := len(m.activeAlerts) > 0 || len(config.Overrides) > 0
if isExistingInstall {
// Existing install: auto-activate to preserve behavior.
config.ActivationState = ActivationActive
now := time.Now()
config.ActivationTime = &now
log.Info().Msg("migrating existing installation to active alert state")
} else {
// New install: start in pending review.
config.ActivationState = ActivationPending
log.Info().Msg("new installation: alerts pending activation")
}
}
}
// normalizeOverrides normalizes all threshold overrides.
func normalizeOverrides(overrides map[string]ThresholdConfig) {
normalized := make(map[string]ThresholdConfig, len(overrides))
priorityByKey := make(map[string]int, len(overrides))
for id, override := range overrides {
// An unset powered-off severity means "follow the global default".
// Normalizing it here would stamp every override with an explicit
// "warning" and silently downgrade guests whose global default is
// critical, so only normalize values the user actually set.
if override.PoweredOffSeverity != "" {
override.PoweredOffSeverity = NormalizePoweredOffSeverity(override.PoweredOffSeverity)
}
if override.Usage != nil {
override.Usage = ensureHysteresisThreshold(override.Usage)
}
normalizedKey := id
if ident, ok := parseCanonicalGuestKey(id); ok {
if stableKey := clusteredGuestOverrideKey(ident); stableKey != "" {
normalizedKey = stableKey
}
}
priority := 0
if normalizedKey == id {
priority = 1
}
if existingPriority, exists := priorityByKey[normalizedKey]; exists && existingPriority > priority {
continue
}
priorityByKey[normalizedKey] = priority
normalized[normalizedKey] = override
}
for key := range overrides {
delete(overrides, key)
}
for key, override := range normalized {
overrides[key] = override
}
}
// applyGlobalOfflineSettingsLocked clears tracking and active alerts for globally disabled offline detectors.
// Caller must hold m.mu.
func (m *Manager) applyGlobalOfflineSettingsLocked() {
_, nodesOfflineDisabled := m.alertPolicyTypeSwitchesNoLock("node")
_, pbsOfflineDisabled := m.alertPolicyTypeSwitchesNoLock("pbs")
_, guestsOfflineDisabled := m.alertPolicyTypeSwitchesNoLock("vm")
_, dockerHostsOfflineDisabled := m.alertPolicyTypeSwitchesNoLock("docker-host")
containersDisabled, _ := m.alertPolicyTypeSwitchesNoLock("app-container")
servicesDisabled, _ := m.alertPolicyTypeSwitchesNoLock("docker-service")
kubernetesDisabled, _ := m.alertPolicyTypeSwitchesNoLock("k8s-node")
truenasDisabled, _ := m.alertPolicyTypeSwitchesNoLock("truenas-system")
vmwareDisabled, _ := m.alertPolicyTypeSwitchesNoLock("vmware-host")
if nodesOfflineDisabled {
var nodeAlerts []string
for storageKey, alert := range m.activeAlerts {
if alert == nil {
continue
}
resourceType, _ := alert.Metadata["resourceType"].(string)
isCanonicalNodeConnectivity := alert.CanonicalKind == string(alertspecs.AlertSpecKindConnectivity) && resourceType == "node"
isLegacyNodeConnectivity := alert.Type == "connectivity" && (strings.HasPrefix(alert.ID, "node-offline-") || strings.HasPrefix(storageKey, "node-offline-"))
if isCanonicalNodeConnectivity || isLegacyNodeConnectivity {
nodeAlerts = append(nodeAlerts, storageKey)
}
}
for _, alertID := range nodeAlerts {
m.clearAlertNoLock(alertID)
}
}
if pbsOfflineDisabled {
var pbsAlerts []string
for storageKey, alert := range m.activeAlerts {
if alert != nil && alert.CanonicalKind == string(alertspecs.AlertSpecKindConnectivity) {
if resourceType, _ := alert.Metadata["resourceType"].(string); resourceType == "pbs" {
pbsAlerts = append(pbsAlerts, storageKey)
}
}
}
for _, alertID := range pbsAlerts {
m.clearAlertNoLock(alertID)
}
}
if guestsOfflineDisabled {
var guestAlerts []string
for storageKey, alert := range m.activeAlerts {
if alert != nil && alert.CanonicalKind == string(alertspecs.AlertSpecKindPoweredState) {
guestAlerts = append(guestAlerts, storageKey)
}
}
for _, alertID := range guestAlerts {
m.clearAlertNoLock(alertID)
}
}
if dockerHostsOfflineDisabled {
var hostAlerts []string
for storageKey, alert := range m.activeAlerts {
if alert != nil && alert.CanonicalKind == string(alertspecs.AlertSpecKindConnectivity) {
if resourceType, _ := alert.Metadata["resourceType"].(string); resourceType == "docker-host" {
hostAlerts = append(hostAlerts, storageKey)
}
}
}
for _, alertID := range hostAlerts {
m.clearAlertNoLock(alertID)
}
}
if containersDisabled {
var containerAlerts []string
for storageKey, alert := range m.activeAlerts {
id := effectiveAlertID(alert, storageKey)
if strings.HasPrefix(id, "docker-container-") || isDockerContainerAlert(alert) {
containerAlerts = append(containerAlerts, id)
}
}
for _, alertID := range containerAlerts {
m.clearAlertNoLock(alertID)
}
m.dockerRestartTracking = make(map[string]*dockerRestartRecord)
m.dockerUpdateFirstSeen = make(map[string]time.Time)
m.dockerUpdateFirstSeenByIdentity = make(map[string]time.Time)
m.dockerUpdateLastObserved = make(map[string]time.Time)
}
if m.config.DockerDefaults.UpdateAlertDelayHours < 0 && !containersDisabled {
m.clearDockerContainerUpdateAlertsLocked()
m.dockerUpdateFirstSeen = make(map[string]time.Time)
m.dockerUpdateFirstSeenByIdentity = make(map[string]time.Time)
m.dockerUpdateLastObserved = make(map[string]time.Time)
}
if servicesDisabled {
var serviceAlerts []string
for storageKey, alert := range m.activeAlerts {
id := effectiveAlertID(alert, storageKey)
if strings.HasPrefix(id, "docker-service-") || isDockerServiceAlert(alert) {
serviceAlerts = append(serviceAlerts, id)
}
}
for _, alertID := range serviceAlerts {
m.clearAlertNoLock(alertID)
}
}
if kubernetesDisabled || truenasDisabled || vmwareDisabled {
var platformAlerts []string
for storageKey, alert := range m.activeAlerts {
primaryType := alertPrimaryResourceType(alert)
if primaryType == "" {
continue
}
switch {
case kubernetesDisabled && isUnifiedKubernetesAlertType(primaryType):
platformAlerts = append(platformAlerts, effectiveAlertID(alert, storageKey))
case truenasDisabled && isUnifiedTrueNASAlertType(primaryType):
platformAlerts = append(platformAlerts, effectiveAlertID(alert, storageKey))
case vmwareDisabled && isUnifiedVMwareAlertType(primaryType):
platformAlerts = append(platformAlerts, effectiveAlertID(alert, storageKey))
}
}
for _, alertID := range platformAlerts {
m.clearAlertNoLock(alertID)
}
}
}
// dockerAlertResourcePath splits the alert's resource ID after the "docker:"
// scheme into the host and the child path, matching the IDs built by
// DockerHostResourceID ("<host>"), DockerContainerResourceID
// ("<host>/<container ID>" or "<host>/name:<name>") and
// DockerServiceResourceID ("<host>/service/<service ID>" or
// "<host>/service/name:<name>"). child is empty for a host's own alert. A
// name can contain anything, so callers classify by the first segment of
// child, never by searching the whole path. Canonical alerts are stored under
// "<resourceID>::<specID>" state IDs, so the legacy "docker-container-" /
// "docker-service-" alert-ID prefixes never match them.
func dockerAlertResourcePath(alert *Alert) (host, child string, ok bool) {
if alert == nil {
return "", "", false
}
path, found := strings.CutPrefix(strings.TrimSpace(alert.ResourceID), "docker:")
if !found {
return "", "", false
}
host, child, _ = strings.Cut(path, "/")
return host, child, true
}
func isDockerContainerAlert(alert *Alert) bool {
_, child, ok := dockerAlertResourcePath(alert)
return ok && child != "" && !strings.HasPrefix(child, "service/")
}
func isDockerServiceAlert(alert *Alert) bool {
if alert != nil && strings.HasPrefix(strings.TrimSpace(alert.ResourceID), "docker-service:") {
return true
}
_, child, ok := dockerAlertResourcePath(alert)
return ok && strings.HasPrefix(child, "service/")
}
func alertPrimaryResourceType(alert *Alert) string {
if alert == nil || alert.Metadata == nil {
return ""
}
metaType, ok := alert.Metadata["resourceType"].(string)
if !ok {
return ""
}
keys := CanonicalResourceTypeKeys(metaType)
if len(keys) == 0 {
return ""
}
return keys[0]
}
func alertResourceTypeKeysContain(keys []string, target string) bool {
for _, key := range keys {
if key == target {
return true
}
}
return false
}
// reevaluateActiveAlertsLocked re-evaluates all active alerts against the current configuration.
// This should only be called with m.mu already locked.
func (m *Manager) reevaluateActiveAlertsLocked() {
if len(m.activeAlerts) == 0 {
return
}
// Track alerts that should be resolved.
alertsToResolve := make([]string, 0)
for storageKey, alert := range m.activeAlerts {
alertID := effectiveAlertID(alert, storageKey)
backfillCanonicalIdentity(alert)
if alert.Type == "docker-container-update" || strings.HasPrefix(alertID, "docker-container-update-") {
if m.shouldResolveDockerContainerUpdateAlertLocked(alert) {
alertsToResolve = append(alertsToResolve, alertID)
}
continue
}
resourceID := alert.ResourceID
metricType := alert.Type
if resourceID == "" || metricType == "" {
parts := strings.Split(alertID, "-")
if len(parts) < 2 {
continue
}
metricType = parts[len(parts)-1]
resourceID = strings.Join(parts[:len(parts)-1], "-")
}
var threshold *HysteresisThreshold
resourceTypeMeta := ""
if alert.Metadata != nil {
if metaType, ok := alert.Metadata["resourceType"].(string); ok {
resourceTypeMeta = alertconfig.CanonicalAlertResourceType(metaType)
}
}
resourceTypeKeys := CanonicalResourceTypeKeys(resourceTypeMeta)
primaryResourceType := ""
if len(resourceTypeKeys) > 0 {
primaryResourceType = resourceTypeKeys[0]
}
if connectionType, ok := connectionTypeFromAlert(alert); ok {
policyResourceID := metadataStringValue(alert.Metadata, "policyResourceID")
if m.connectionDegradedPolicyDisabledNoLock(resourceID, policyResourceID, connectionType) {
alertsToResolve = append(alertsToResolve, alertID)
}
continue
}
if alert.Type == "queue-depth" || alert.Type == "queue-deferred" || alert.Type == "queue-hold" || alert.Type == "message-age" {
if allDisabled, _ := m.alertPolicyTypeSwitchesNoLock("pmg"); allDisabled {
alertsToResolve = append(alertsToResolve, alertID)
continue
}
}
handledModernPlatformType := false
if isUnifiedModernPlatformAlertType(primaryResourceType) {
handledModernPlatformType = true
if m.unifiedPlatformAlertsDisabledNoLock(primaryResourceType) {
alertsToResolve = append(alertsToResolve, alertID)
continue
}
query := alertPolicyQuery{TypeKey: primaryResourceType, ResourceID: resourceID}
if primaryResourceType == "truenas-disk" {
// Judge the alert against its disk type's threshold, as the
// next evaluation will. An alert raised before it recorded a
// disk type is held to the lowest per-type trigger, so a save
// never resolves an alert its type would raise again.
diskType, known := alert.Metadata["diskType"].(string)
query.DiskType = diskType
query.DiskTypeUnknown = !known
}
thresholds := m.effectiveAlertPolicyNoLock(query).Thresholds
if thresholds.Disabled {
alertsToResolve = append(alertsToResolve, alertID)
continue
}
threshold = getThresholdForMetric(thresholds, metricType)
}
// A Proxmox disk temperature alert is judged by the disk temperature
// policy for its disk type, as CheckProxmoxDiskTemperature judges it.
// The node and guest policies below have no disk temperature, so a
// save would otherwise resolve an alert the next poll raises again.
if primaryResourceType == proxmoxDiskResourceType && metricType == proxmoxDiskTemperatureMetric {
diskType, known := alert.Metadata["diskType"].(string)
threshold = m.proxmoxDiskTemperatureThresholdNoLock(diskType, !known)
if threshold == nil || threshold.Trigger <= 0 {
alertsToResolve = append(alertsToResolve, alertID)
continue
}
}
isAgentResource := alertResourceTypeKeysContain(resourceTypeKeys, "agent")
if !handledModernPlatformType && isAgentResource {
if allDisabled, _ := m.alertPolicyTypeSwitchesNoLock("agent"); allDisabled {
alertsToResolve = append(alertsToResolve, alertID)
continue
}
thresholds := m.resolveHostAlertThresholdsNoLock(alert, resourceID)
if thresholds.Disabled {
alertsToResolve = append(alertsToResolve, alertID)
continue
}
if isHostSMARTRiskAlertType(metricType) {
smartThresholds, crcMinimumDelta := hostSMARTRiskThresholds(thresholds)
if hostSMARTRiskRulesOff(alert, smartThresholds, crcMinimumDelta) {
alertsToResolve = append(alertsToResolve, alertID)
}
continue
}
threshold = getThresholdForMetric(thresholds, metricType)
// Agent policy is the only policy for agent alerts. Falling
// through would let the storage or guest switches resolve a
// SMART, RAID or sensor alert that CheckHost re-raises on the
// next report.
if threshold == nil {
if isMetricThresholdAlertType(metricType) {
alertsToResolve = append(alertsToResolve, alertID)
}
continue
}
}
if alert.Type == "docker-host-offline" ||
alert.Type == "docker-container-health" ||
alert.Type == "docker-container-state" ||
alert.Type == "docker-container-restart-loop" ||
alert.Type == "docker-container-oom-kill" ||
alert.Type == "docker-container-memory-limit" {
continue
}
if resourceTypeMeta == "docker-host" {
if allDisabled, _ := m.alertPolicyTypeSwitchesNoLock("docker-host"); allDisabled {
alertsToResolve = append(alertsToResolve, alertID)
continue
}
continue
}
if resourceTypeMeta == "app-container" {
if allDisabled, _ := m.alertPolicyTypeSwitchesNoLock("app-container"); allDisabled {
alertsToResolve = append(alertsToResolve, alertID)
continue
}
containerName := strings.TrimSpace(alert.ResourceName)
containerID := ""
hostID := ""
if alert.Metadata != nil {
if val, ok := alert.Metadata["containerId"].(string); ok {
containerID = strings.ToLower(strings.TrimSpace(val))
}
if val, ok := alert.Metadata["containerName"].(string); ok && containerName == "" {
containerName = strings.TrimSpace(val)
}
if val, ok := alert.Metadata["hostId"].(string); ok {
hostID = val
}
}
if matchesDockerIgnoredPrefix(strings.ToLower(containerName), containerID, m.config.DockerIgnoredContainerPrefixes) {
alertsToResolve = append(alertsToResolve, alertID)
continue
}
thresholds := ThresholdConfig{
CPU: cloneThreshold(&m.config.DockerDefaults.CPU),
Memory: cloneThreshold(&m.config.DockerDefaults.Memory),
Disk: cloneThreshold(&m.config.DockerDefaults.Disk),
}
if override, exists := m.lookupDockerContainerOverrideNoLock(hostID, containerName, resourceID); exists {
if override.Disabled {
alertsToResolve = append(alertsToResolve, alertID)
continue
}
thresholds = m.applyThresholdOverride(thresholds, override)
}
threshold = getThresholdForMetric(thresholds, metricType)
}
// Node alerts carry resourceType "node" in metadata and keep the PVE
// instance name in Instance, so the metadata decides. The Instance
// heuristic only classifies legacy alerts without a resource type;
// applied to current node alerts it fell through to guest thresholds,
// which have no temperature threshold, and a config save resolved
// temperature alerts that were still over their trigger.
isNodeResource := primaryResourceType == "node" ||
(primaryResourceType == "" && !strings.Contains(resourceID, ":") && (alert.Instance == "Node" || alert.Instance == alert.Node))
isStorageResource := alertResourceTypeKeysContain(resourceTypeKeys, "storage")
if threshold == nil && !handledModernPlatformType && isNodeResource {
if allDisabled, _ := m.alertPolicyTypeSwitchesNoLock("node"); allDisabled {
alertsToResolve = append(alertsToResolve, alertID)
continue
}
thresholds := m.resolveResourceThresholds("node", resourceID)
if thresholds.Disabled {
alertsToResolve = append(alertsToResolve, alertID)
continue
}
threshold = getThresholdForMetric(thresholds, metricType)
} else if threshold == nil && !handledModernPlatformType && (isStorageResource || alert.Instance == "Storage" || strings.Contains(alert.ResourceID, ":storage/")) {
if allDisabled, _ := m.alertPolicyTypeSwitchesNoLock("storage"); allDisabled {
alertsToResolve = append(alertsToResolve, alertID)
continue
}
thresholds := m.effectiveAlertPolicyNoLock(alertPolicyQuery{TypeKey: "storage", ResourceID: resourceID, StorageAliases: storagePolicyAliases(alert)}).Thresholds
if thresholds.Disabled {
alertsToResolve = append(alertsToResolve, alertID)
continue
}
threshold = getThresholdForMetric(thresholds, metricType)
} else if threshold == nil && !handledModernPlatformType && (resourceTypeMeta == "pbs" || alert.Instance == "PBS") {
if allDisabled, _ := m.alertPolicyTypeSwitchesNoLock("pbs"); allDisabled {
alertsToResolve = append(alertsToResolve, alertID)
continue
}
thresholds := m.resolveResourceThresholds("pbs", resourceID)
if thresholds.Disabled {
alertsToResolve = append(alertsToResolve, alertID)
continue
}
threshold = getThresholdForMetric(thresholds, metricType)
}
if threshold == nil && !handledModernPlatformType {
if allDisabled, _ := m.alertPolicyTypeSwitchesNoLock("vm"); allDisabled {
alertsToResolve = append(alertsToResolve, alertID)
continue
}
guestThresholds := m.resolveGuestAlertThresholdsNoLock(alert, resourceID)
if guestThresholds.Disabled {
alertsToResolve = append(alertsToResolve, alertID)
continue
}
switch alert.Type {
case "snapshot-age":
if !snapshotAlertStillTriggered(alert, m.resolvedSnapshotAlertConfigNoLock(guestThresholds)) {
alertsToResolve = append(alertsToResolve, alertID)
}
continue
case "backup-age":
if !backupAlertStillTriggered(alert, m.resolvedBackupAlertConfigNoLock(guestThresholds)) {
alertsToResolve = append(alertsToResolve, alertID)
}
continue
case "powered-off":
if guestThresholds.DisableConnectivity {
alertsToResolve = append(alertsToResolve, alertID)
continue
}
alert.Level = NormalizePoweredOffSeverity(guestThresholds.PoweredOffSeverity)
continue
}
threshold = getThresholdForMetric(guestThresholds, metricType)
}
// Provider-owned incidents and other non-metric alerts are not threshold
// observations. An unrelated config save must leave their lifecycle and
// acknowledgement state intact; only their evaluator (or an explicit
// resource-disable policy handled above) can resolve them.
if !isMetricThresholdAlertType(metricType) {
continue
}
if threshold == nil || threshold.Trigger <= 0 {
alertsToResolve = append(alertsToResolve, alertID)
continue
}
// buildCanonicalMetricSpec keeps a clear level only below the trigger,
// and without one the evaluator keeps firing at the trigger itself.
// A clear level at or above the trigger must not resolve a reading
// the next evaluation raises again.
clearThreshold := threshold.Clear
hasRecoveryBand := clearThreshold > 0 && clearThreshold < threshold.Trigger
if hasRecoveryBand && alert.Value <= clearThreshold {
alertsToResolve = append(alertsToResolve, alertID)
log.Info().
Str("alertID", alertID).
Float64("value", alert.Value).
Float64("oldThreshold", alert.Threshold).
Float64("newClearThreshold", clearThreshold).
Msg("Resolving alert due to threshold change")
} else if alert.Value < threshold.Trigger {
alertsToResolve = append(alertsToResolve, alertID)
log.Info().
Str("alertID", alertID).
Float64("value", alert.Value).
Float64("newTrigger", threshold.Trigger).
Float64("newClear", clearThreshold).
Msg("Resolving alert - value now below trigger threshold after config change")
}
}
for _, alertID := range alertsToResolve {
if alert, exists := m.getActiveAlertNoLock(alertID); exists {
resolvedAlert := m.newResolvedAlert(alert, time.Now(), nil)
m.mirrorForgetAlertNoLock(alert)
m.removeActiveAlertNoLock(alertID)
m.addRecentlyResolvedWithPrimaryLock(resolvedAlert)
log.Info().
Str("alertID", alertID).
Msg("Alert auto-resolved after configuration change")
m.safeCallResolvedAlertCallback(resolvedAlert, alertID, true)
}
}
if len(alertsToResolve) > 0 {
m.saveActiveAlertsAsync("config update")
}
}
// GetConfig returns a copy of the alert configuration that the caller owns.
// Callers encode, persist and edit it outside m.mu, and UpdateConfig
// normalizes the maps it is handed in place, so a copy that shared the live
// config's maps let a save or a re-apply write them while evaluation read
// them, which Go aborts as a concurrent map write.
func (m *Manager) GetConfig() AlertConfig {
m.mu.RLock()
defer m.mu.RUnlock()
return m.config.Clone()
}
func cloneThreshold(threshold *HysteresisThreshold) *HysteresisThreshold {
if threshold == nil {
return nil
}
clone := *threshold
return &clone
}
func cloneStringPtr(value *string) *string {
if value == nil {
return nil
}
v := *value
return &v
}
func cloneInt(value *int) *int {
if value == nil {
return nil
}
clone := *value
return &clone
}
func cloneInt64(value *int64) *int64 {
if value == nil {
return nil
}
clone := *value
return &clone
}
func cloneSnapshotConfig(cfg *SnapshotAlertConfig) *SnapshotAlertConfig {
if cfg == nil {
return nil
}
clone := *cfg
return &clone
}
func cloneBackupConfig(cfg *BackupAlertConfig) *BackupAlertConfig {
if cfg == nil {
return nil
}
clone := *cfg
if cfg.AlertOrphaned != nil {
value := *cfg.AlertOrphaned
clone.AlertOrphaned = &value
}
if len(cfg.IgnoreVMIDs) > 0 {
clone.IgnoreVMIDs = append([]string(nil), cfg.IgnoreVMIDs...)
}
return &clone
}
func cloneThresholdConfig(cfg ThresholdConfig) ThresholdConfig {
clone := cfg
clone.CPU = cloneThreshold(cfg.CPU)
clone.Memory = cloneThreshold(cfg.Memory)
clone.Disk = cloneThreshold(cfg.Disk)
clone.DiskRead = cloneThreshold(cfg.DiskRead)
clone.DiskWrite = cloneThreshold(cfg.DiskWrite)
clone.NetworkIn = cloneThreshold(cfg.NetworkIn)
clone.NetworkOut = cloneThreshold(cfg.NetworkOut)
clone.Temperature = cloneThreshold(cfg.Temperature)
clone.DiskTemperature = cloneThreshold(cfg.DiskTemperature)
clone.SMARTHealthFailure = cloneInt(cfg.SMARTHealthFailure)
clone.SMARTReallocated = cloneInt64(cfg.SMARTReallocated)
clone.SMARTPending = cloneInt64(cfg.SMARTPending)
clone.SMARTUncorrectable = cloneInt64(cfg.SMARTUncorrectable)
clone.SMARTMediaErrors = cloneInt64(cfg.SMARTMediaErrors)
clone.SMARTCRCErrorDelta = cloneInt64(cfg.SMARTCRCErrorDelta)
clone.SMARTLifeWarning = cloneInt(cfg.SMARTLifeWarning)
clone.SMARTLifeCritical = cloneInt(cfg.SMARTLifeCritical)
clone.SMARTSpareWarning = cloneInt(cfg.SMARTSpareWarning)
clone.SMARTSpareCritical = cloneInt(cfg.SMARTSpareCritical)
clone.Usage = cloneThreshold(cfg.Usage)
clone.Backup = cloneBackupConfig(cfg.Backup)
clone.Snapshot = cloneSnapshotConfig(cfg.Snapshot)
clone.Note = cloneStringPtr(cfg.Note)
return clone
}
func (m *Manager) applyThresholdOverride(base ThresholdConfig, override ThresholdConfig) ThresholdConfig {
result := base
if override.Disabled {
result.Disabled = true
}
if override.DisableConnectivity {
result.DisableConnectivity = true
} else if override.PoweredOffSeverity != "" {
// An override carrying an explicit powered-off severity is that row's
// offline control set to warning/critical, so it re-enables
// connectivity alerts over a disabled default. A per-row "off" is
// stored as DisableConnectivity=true and takes the branch above.
result.DisableConnectivity = false
}
if override.PoweredOffSeverity != "" {
result.PoweredOffSeverity = NormalizePoweredOffSeverity(override.PoweredOffSeverity)
}
if override.CPU != nil {
result.CPU = ensureHysteresisThreshold(cloneThreshold(override.CPU))
}
if override.Memory != nil {
result.Memory = ensureHysteresisThreshold(cloneThreshold(override.Memory))
}
if override.Disk != nil {
result.Disk = ensureHysteresisThreshold(cloneThreshold(override.Disk))
}
if override.DiskRead != nil {
result.DiskRead = ensureHysteresisThreshold(cloneThreshold(override.DiskRead))
}
if override.DiskWrite != nil {
result.DiskWrite = ensureHysteresisThreshold(cloneThreshold(override.DiskWrite))
}
if override.NetworkIn != nil {
result.NetworkIn = ensureHysteresisThreshold(cloneThreshold(override.NetworkIn))
}
if override.NetworkOut != nil {
result.NetworkOut = ensureHysteresisThreshold(cloneThreshold(override.NetworkOut))
}
if override.Temperature != nil {
result.Temperature = ensureHysteresisThreshold(cloneThreshold(override.Temperature))
}
if override.DiskTemperature != nil {
result.DiskTemperature = ensureHysteresisThreshold(cloneThreshold(override.DiskTemperature))
}
if override.SMARTHealthFailure != nil {
result.SMARTHealthFailure = cloneInt(override.SMARTHealthFailure)
}
if override.SMARTReallocated != nil {
result.SMARTReallocated = cloneInt64(override.SMARTReallocated)
}
if override.SMARTPending != nil {
result.SMARTPending = cloneInt64(override.SMARTPending)
}
if override.SMARTUncorrectable != nil {
result.SMARTUncorrectable = cloneInt64(override.SMARTUncorrectable)
}
if override.SMARTMediaErrors != nil {
result.SMARTMediaErrors = cloneInt64(override.SMARTMediaErrors)
}
if override.SMARTCRCErrorDelta != nil {
result.SMARTCRCErrorDelta = cloneInt64(override.SMARTCRCErrorDelta)
}
if override.SMARTLifeWarning != nil {
result.SMARTLifeWarning = cloneInt(override.SMARTLifeWarning)
}
if override.SMARTLifeCritical != nil {
result.SMARTLifeCritical = cloneInt(override.SMARTLifeCritical)
}
if override.SMARTSpareWarning != nil {
result.SMARTSpareWarning = cloneInt(override.SMARTSpareWarning)
}
if override.SMARTSpareCritical != nil {
result.SMARTSpareCritical = cloneInt(override.SMARTSpareCritical)
}
if override.Usage != nil {
result.Usage = ensureHysteresisThreshold(cloneThreshold(override.Usage))
}
if override.Backup != nil {
result.Backup = cloneBackupConfig(override.Backup)
}
if override.Snapshot != nil {
result.Snapshot = cloneSnapshotConfig(override.Snapshot)
}
if override.Note != nil {
note := strings.TrimSpace(*override.Note)
if note == "" {
result.Note = nil
} else {
noteCopy := note
result.Note = ¬eCopy
}
}
return result
}
// ensureHysteresisThreshold ensures a threshold has hysteresis configured.
func ensureHysteresisThreshold(threshold *HysteresisThreshold) *HysteresisThreshold {
if threshold == nil {
return nil
}
if threshold.Clear <= 0 {
threshold.Clear = threshold.Trigger - 5.0 // Default 5% margin
}
return threshold
}