Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
package alerts
import (
"fmt"
"strings"
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/alerts/reducer"
alertspecs "github.com/rcourtman/pulse-go-rewrite/internal/alerts/specs"
"github.com/rcourtman/pulse-go-rewrite/internal/models"
"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
"github.com/rs/zerolog/log"
)
// ReevaluateGuestAlert reevaluates a specific guest's alerts with full threshold resolution including custom rules.
// This should be called by the monitor with the current guest state.
func (m *Manager) ReevaluateGuestAlert(guest any, guestID string) {
m.mu.Lock()
defer m.mu.Unlock()
// Get the correct thresholds for this guest (includes custom rules evaluation)
thresholds := m.getGuestThresholds(guest, guestID)
// Check all metric types for this guest
metricTypes := []string{"cpu", "memory", "disk", "diskRead", "diskWrite", "networkIn", "networkOut"}
for _, metricType := range metricTypes {
alertID := canonicalMetricStateID(guestID, metricType)
alert, exists := m.getActiveAlertNoLock(alertID)
if !exists {
alert, exists = m.getActiveAlertNoLock(fmt.Sprintf("%s-%s", guestID, metricType))
}
if !exists {
continue
}
trackingKey := canonicalTrackingKeyForAlert(alert)
if trackingKey == "" {
trackingKey = alertID
}
// Get the threshold for this metric
var threshold *HysteresisThreshold
switch metricType {
case "cpu":
threshold = thresholds.CPU
case "memory":
threshold = thresholds.Memory
case "disk":
threshold = thresholds.Disk
case "diskRead":
threshold = thresholds.DiskRead
case "diskWrite":
threshold = thresholds.DiskWrite
case "networkIn":
threshold = thresholds.NetworkIn
case "networkOut":
threshold = thresholds.NetworkOut
}
// If threshold is disabled or doesn't exist, clear the alert
if threshold == nil || threshold.Trigger <= 0 {
m.clearAlertNoLock(trackingKey)
log.Info().
Str("alertID", alertID).
Str("metric", metricType).
Msg("Cleared alert - threshold disabled")
continue
}
// Check if alert should be cleared based on new threshold
clearThreshold := threshold.Clear
if clearThreshold <= 0 {
clearThreshold = threshold.Trigger
}
if alert.Value <= clearThreshold || alert.Value < threshold.Trigger {
m.clearAlertNoLock(trackingKey)
log.Info().
Str("alertID", alertID).
Str("metric", metricType).
Float64("value", alert.Value).
Float64("trigger", threshold.Trigger).
Float64("clear", clearThreshold).
Msg("Cleared alert - value now below threshold after config change")
}
}
}
// CheckGuest checks a guest (VM or container) against thresholds.
func (m *Manager) CheckGuest(guest any, instanceName string) {
m.mu.RLock()
enabled := m.config.Enabled
disableAllGuests, disableAllGuestsOffline := m.alertPolicyTypeSwitchesNoLock("vm")
m.mu.RUnlock()
if !enabled {
log.Debug().Msg("checkGuest: alerts disabled globally")
return
}
if disableAllGuests {
log.Debug().Msg("checkGuest: all guest alerts disabled")
return
}
snapshot, ok := extractGuestSnapshot(guest)
if !ok {
log.Debug().
Str("type", fmt.Sprintf("%T", guest)).
Msg("CheckGuest: unsupported guest type")
return
}
guestID := snapshot.ID
name := snapshot.Name
node := snapshot.Node
guestType := snapshot.displayType()
status := snapshot.Status
cpu := snapshot.CPUPercent
memUsage := snapshot.MemUsage
diskUsage := snapshot.DiskUsage
disks := snapshot.Disks
// Debug logging for high memory VMs
if snapshot.Kind == guestKindVM && memUsage > 85 {
log.Debug().
Str("vm", name).
Float64("memUsage", memUsage).
Str("status", status).
Msg("VM with high memory detected in CheckGuest")
}
policy := m.resolveGuestAlertPolicy(snapshot)
if policy.SuppressionReason != "" {
if cleared := m.suppressGuestAlerts(guestID); cleared {
m.saveActiveAlertsAsync(policy.SuppressionReason)
}
log.Debug().
Str("guestID", guestID).
Str("reason", policy.SuppressionReason).
Msg("Guest alert policy suppressed alerts")
return
}
monitorOnly := policy.TagSettings.MonitorOnly
if monitorOnly || m.guestHasMonitorOnlyAlerts(guestID) {
log.Debug().
Str("guest", name).
Bool("monitorOnly", monitorOnly).
Msg("Pulse monitor-only status applied")
}
// Handle non-running guests
// Proxmox VM states: running, stopped, paused, suspended
if status != "running" {
// Check for powered-off state and generate alert if configured
if status == "stopped" {
if disableAllGuestsOffline {
// Clear any pending powered-off tracking and alerts when globally disabled
m.mu.Lock()
intentCleared := m.clearIntentPendingNoLock(canonicalPoweredStateStateID(guestID))
m.mu.Unlock()
if intentCleared {
m.saveActiveAlertsAsync("guest offline intent disabled")
}
m.rehomeStrandedGuestAlert(canonicalPoweredStateStateID(guestID), canonicalPoweredStateSpecID(guestID), string(alertspecs.AlertSpecKindPoweredState), guestID, name, node, instanceName, "guest")
m.clearAlert(canonicalPoweredStateStateID(guestID))
} else if snapshot.OnBoot != nil && !*snapshot.OnBoot {
// Guest is explicitly not configured to autostart (onboot=0).
// Being stopped is the expected state — suppress the alert.
m.mu.Lock()
m.mu.Unlock()
m.clearAlert(canonicalPoweredStateStateID(guestID))
m.clearGuestPoweredOffAlert(guestID, name)
} else {
m.mu.RLock()
thresholds := m.getGuestThresholds(guest, guestID)
m.mu.RUnlock()
observedAt := m.policyNow().UTC()
backupContext := BackupIntentContext{
Active: strings.EqualFold(strings.TrimSpace(snapshot.Lock), "backup"),
ObservedAt: observedAt,
Evidence: "guest_lock",
}
if !backupContext.Active {
m.mu.RLock()
resolver := m.backupIntentResolver
m.mu.RUnlock()
if resolver != nil {
if resolved, ok := resolver(guestID, instanceName, node, snapshot.VMID, observedAt); ok {
backupContext = resolved
}
}
}
m.checkGuestPoweredOffWithThresholdsAndIntentAndTags(guestID, name, node, instanceName, guestType, thresholds, monitorOnly, backupContext, snapshot.Tags)
}
} else {
// For paused/suspended, clear powered-off alert
m.clearGuestPoweredOffAlert(guestID, name)
}
alertsCleared := m.clearGuestMetricAlerts(guestID)
if alertsCleared > 0 {
log.Debug().
Str("guest", name).
Str("status", status).
Int("alertsCleared", alertsCleared).
Msg("Cleared metric alerts for non-running guest")
m.saveActiveAlertsAsync("guest-not-running")
}
return
}
// If guest is running, clear any powered-off alert
m.clearGuestPoweredOffAlert(guestID, name)
// Get thresholds (defaults, custom rules, overrides, then the
// pulse-relaxed floor)
m.mu.RLock()
thresholds := m.getGuestThresholds(guest, guestID)
m.mu.RUnlock()
// If alerts are disabled for this guest, clear any existing alerts and return
if thresholds.Disabled {
if alertsCleared := m.clearGuestMetricAlerts(guestID); alertsCleared > 0 {
log.Info().
Str("guest", name).
Int("alertsCleared", alertsCleared).
Msg("Cleared guest metric alerts because alerts are disabled")
m.saveActiveAlertsAsync("guest-alerts-disabled")
}
return
}
// Check each metric
log.Debug().
Str("guest", name).
Float64("cpu", cpu).
Float64("memory", memUsage).
Float64("disk", diskUsage).
Interface("thresholds", thresholds).
Msg("Checking guest thresholds")
// Evaluate standard metrics through unified path
var evalOpts *metricOptions
if monitorOnly {
evalOpts = &metricOptions{MonitorOnly: true}
}
var memoryMetric *UnifiedResourceMetric
if !snapshot.MemoryUnavailable {
memoryMetric = &UnifiedResourceMetric{Percent: memUsage}
}
diskReadMetric, diskWriteMetric, networkInMetric, networkOutMetric := guestIORateMetrics(snapshot)
var diskMetric *UnifiedResourceMetric
if !snapshot.DiskUnavailable {
diskMetric = &UnifiedResourceMetric{Percent: diskUsage}
}
if !snapshot.DiskUnavailable && len(disks) > 0 {
seenAggregateDiskKeys := make(map[string]struct{})
for idx, disk := range disks {
if disk.Total <= 0 || disk.Usage < 0 {
continue
}
_, _, sanitizedKey := guestDiskIdentity(disk, idx)
if _, exists := seenAggregateDiskKeys[sanitizedKey]; exists {
continue
}
seenAggregateDiskKeys[sanitizedKey] = struct{}{}
}
if len(seenAggregateDiskKeys) > 0 {
// Filesystem evidence is more actionable than the guest-wide rollup.
// Evaluating both creates two incidents for the same full filesystem.
diskMetric = nil
m.clearAlert(canonicalMetricStateID(guestID, "disk"))
}
}
m.evaluateUnifiedMetrics(&UnifiedResourceInput{
ID: guestID,
Type: snapshot.resourceType(),
Name: name,
Node: node,
Instance: instanceName,
Tags: snapshot.Tags,
CPU: &UnifiedResourceMetric{Percent: cpu},
Memory: memoryMetric,
Disk: diskMetric,
DiskRead: diskReadMetric,
DiskWrite: diskWriteMetric,
NetworkIn: networkInMetric,
NetworkOut: networkOutMetric,
}, thresholds, evalOpts)
// Retained/failed filesystem evidence is useful to display, but neither
// its values nor an empty result can establish a breach, recovery or disk
// removal. CPU/I/O and independent current memory were evaluated above.
// Still apply explicit per-filesystem disablement to retained identities;
// this is operator policy, not recovery inferred from an old value.
if snapshot.DiskUnavailable {
m.interruptGuestFilesystemMetricRuns(guestID)
}
if len(disks) > 0 {
seenDiskKeys := make(map[string]struct{})
seenDiskResources := make(map[string]struct{})
for idx, disk := range disks {
if disk.Total <= 0 {
continue
}
if disk.Usage < 0 {
continue
}
label, keySource, sanitizedKey := guestDiskIdentity(disk, idx)
// Avoid duplicate checks if two disks resolve to the same key
if _, exists := seenDiskKeys[sanitizedKey]; exists {
continue
}
seenDiskKeys[sanitizedKey] = struct{}{}
perDiskResourceID := fmt.Sprintf("%s-disk-%s", guestID, sanitizedKey)
seenDiskResources[perDiskResourceID] = struct{}{}
message := fmt.Sprintf("%s disk (%s) at %.1f%%", guestType, label, disk.Usage)
m.mu.RLock()
effectiveDiskThreshold, diskDisabled := m.guestDiskUsageThresholdNoLock(guest, guestID, keySource, thresholds.Disk)
m.mu.RUnlock()
if diskDisabled {
m.clearAlert(canonicalMetricStateID(perDiskResourceID, "disk"))
continue
}
if snapshot.DiskUnavailable {
if effectiveDiskThreshold == nil || effectiveDiskThreshold.Trigger <= 0 {
m.clearAlert(canonicalMetricStateID(perDiskResourceID, "disk"))
}
continue
}
log.Debug().
Str("guest", name).
Str("node", node).
Str("instance", instanceName).
Str("diskLabel", label).
Float64("usage", disk.Usage).
Msg("Evaluating individual disk for alert thresholds")
metadata := map[string]interface{}{
"mountpoint": disk.Mountpoint,
"device": disk.Device,
"diskType": disk.Type,
"totalBytes": disk.Total,
"usedBytes": disk.Used,
"freeBytes": disk.Free,
"diskIndex": idx,
"label": label,
"tags": append([]string(nil), snapshot.Tags...),
// The filesystem ID names no registry resource; this is the
// guest the alert belongs to, as its guest-level alerts name it.
"guestResourceId": guestID,
}
resourceType, ok := unifiedMetricResourceType(snapshot.resourceType())
if !ok {
m.checkMetric(perDiskResourceID, name, node, instanceName, snapshot.resourceType(), "disk", disk.Usage, effectiveDiskThreshold, &metricOptions{
Metadata: metadata,
Message: message,
MonitorOnly: monitorOnly,
})
continue
}
spec, err := buildCanonicalMetricSpec(perDiskResourceID, name, resourceType, "disk", effectiveDiskThreshold)
if err != nil {
log.Warn().
Err(err).
Str("resourceID", perDiskResourceID).
Str("guest", name).
Msg("Skipping invalid canonical guest disk metric spec")
continue
}
m.checkMetricWithCanonicalSpec(spec, name, node, instanceName, snapshot.resourceType(), disk.Usage, effectiveDiskThreshold, &metricOptions{
Metadata: metadata,
Message: message,
MonitorOnly: monitorOnly,
})
}
if !snapshot.DiskUnavailable {
if cleared := m.cleanupGuestDiskAlerts(guestID, seenDiskResources); cleared > 0 {
m.saveActiveAlertsAsync("guest-disk-set-changed")
}
}
} else if !snapshot.DiskUnavailable {
if cleared := m.cleanupGuestDiskAlerts(guestID, nil); cleared > 0 {
m.saveActiveAlertsAsync("guest-disk-alerts-cleared")
}
}
}
func guestDiskIdentity(disk models.Disk, idx int) (label, keySource, sanitizedKey string) {
label = strings.TrimSpace(disk.Mountpoint)
if label == "" {
label = strings.TrimSpace(disk.Device)
}
if label == "" {
label = fmt.Sprintf("Disk %d", idx+1)
}
keySource = label
if disk.Device != "" && !strings.EqualFold(disk.Device, label) {
keySource = fmt.Sprintf("%s-%s", label, disk.Device)
}
sanitizedKey = sanitizeAlertKey(keySource)
if sanitizedKey == "" {
sanitizedKey = fmt.Sprintf("disk-%d", idx+1)
}
return label, keySource, sanitizedKey
}
// guestDiskUsageThresholdNoLock resolves the usage threshold one guest
// filesystem is judged by: the filesystem's own override, else the guest's disk
// threshold. disabled reports an override that switches the filesystem's
// alerts off, and then the threshold is nil. CheckGuest and config-save
// reevaluation both resolve through it. An override threshold is returned as
// an owned copy, so normalizing it never writes into the live config. Callers
// must hold m.mu.
func (m *Manager) guestDiskUsageThresholdNoLock(guest any, guestID, diskKey string, guestDisk *HysteresisThreshold) (threshold *HysteresisThreshold, disabled bool) {
if override, ok := lookupGuestDiskOverride(m.config.Overrides, guest, guestID, diskKey); ok {
if override.Disabled {
return nil, true
}
if override.Disk != nil {
return ensureHysteresisThreshold(cloneThreshold(override.Disk)), false
}
}
return guestDisk, false
}
// resolveGuestAlertThresholdsNoLock resolves thresholds for a persisted guest
// alert. A filesystem alert ("<guestID>-disk-<key>") resolves its guest's
// thresholds by the guest ID, and its Disk by the filesystem's own override, as
// CheckGuest does. Resolved by the filesystem's resource ID, which names no
// guest, it missed both overrides, and a config save resolved an alert the next
// poll raised again. Callers must hold m.mu.
func (m *Manager) resolveGuestAlertThresholdsNoLock(alert *Alert, resourceID string) ThresholdConfig {
resourceID = strings.TrimSpace(resourceID)
if identity, ok := parseGuestAlertIdentity(resourceID); ok {
if diskKey, isFilesystem := strings.CutPrefix(identity.resourceSuffix, "-disk-"); isFilesystem {
guestID := strings.TrimSuffix(resourceID, identity.resourceSuffix)
guest := guestSnapshotFromAlert(alert, guestID)
thresholds := m.getGuestThresholds(guest, guestID)
thresholds.Disk, _ = m.guestDiskUsageThresholdNoLock(guest, guestID, diskKey, thresholds.Disk)
return thresholds
}
}
return m.getGuestThresholds(guestSnapshotFromAlert(alert, resourceID), resourceID)
}
func guestIORateMetrics(snapshot guestSnapshot) (diskRead, diskWrite, networkIn, networkOut *UnifiedResourceMetric) {
validity := snapshot.IORateValidity.EffectiveForRates(
snapshot.DiskRead,
snapshot.DiskWrite,
snapshot.NetworkIn,
snapshot.NetworkOut,
)
if validity.DiskRead {
diskRead = &UnifiedResourceMetric{Value: float64(snapshot.DiskRead) / 1024 / 1024}
}
if validity.DiskWrite {
diskWrite = &UnifiedResourceMetric{Value: float64(snapshot.DiskWrite) / 1024 / 1024}
}
if validity.NetworkIn {
networkIn = &UnifiedResourceMetric{Value: float64(snapshot.NetworkIn) / 1024 / 1024}
}
if validity.NetworkOut {
networkOut = &UnifiedResourceMetric{Value: float64(snapshot.NetworkOut) / 1024 / 1024}
}
return
}
// checkGuestPoweredOff creates an alert for powered-off guests.
func (m *Manager) checkGuestPoweredOff(guestID, name, node, instanceName, guestType string, monitorOnly bool) {
m.mu.RLock()
thresholds := m.getGuestThresholds(guestSnapshotFromIdentity(guestID, name, node, instanceName, guestType, "stopped"), guestID)
m.mu.RUnlock()
m.checkGuestPoweredOffWithThresholds(guestID, name, node, instanceName, guestType, thresholds, monitorOnly)
}
func (m *Manager) checkGuestPoweredOffWithThresholds(guestID, name, node, instanceName, guestType string, thresholds ThresholdConfig, monitorOnly bool) {
m.checkGuestPoweredOffWithThresholdsAndIntent(guestID, name, node, instanceName, guestType, thresholds, monitorOnly, BackupIntentContext{})
}
func (m *Manager) checkGuestPoweredOffWithThresholdsAndIntent(guestID, name, node, instanceName, guestType string, thresholds ThresholdConfig, monitorOnly bool, backupContext BackupIntentContext) {
m.checkGuestPoweredOffWithThresholdsAndIntentAndTags(guestID, name, node, instanceName, guestType, thresholds, monitorOnly, backupContext, nil)
}
func (m *Manager) checkGuestPoweredOffWithThresholdsAndIntentAndTags(guestID, name, node, instanceName, guestType string, thresholds ThresholdConfig, monitorOnly bool, backupContext BackupIntentContext, tags []string) {
alertID := fmt.Sprintf("guest-powered-off-%s", guestID)
severity := NormalizePoweredOffSeverity(thresholds.PoweredOffSeverity)
resourceType := unifiedresources.ResourceTypeVM
if strings.EqualFold(guestType, "container") {
resourceType = unifiedresources.ResourceTypeSystemContainer
}
m.mu.RLock()
effectiveIntent := m.resolveEffectiveIntentPolicyNoLock(guestID, string(resourceType), string(AlertIntentSignalOffline))
m.mu.RUnlock()
confirmations := 2
durationAuthoritative := effectiveIntent.Sources["graceSeconds"] != "factory" ||
(effectiveIntent.BackupOffline != nil && effectiveIntent.BackupOffline.Enabled)
if durationAuthoritative {
// Explicit powered-state policies are duration authoritative. A zero
// grace therefore fires on the first stopped observation, while a
// positive grace is independent of polling cadence.
confirmations = 1
}
spec, err := buildCanonicalPoweredStateSpec(guestID, name, resourceType, severity, confirmations, thresholds.Disabled || thresholds.DisableConnectivity)
if err != nil {
log.Warn().
Err(err).
Str("guest", name).
Str("guestID", guestID).
Msg("Skipping invalid canonical guest powered-state spec")
return
}
metadata := map[string]interface{}{
"monitorOnly": monitorOnly,
"resourceType": strings.ToLower(guestType),
}
if len(tags) > 0 {
metadata["tags"] = append([]string(nil), tags...)
}
m.evaluateCanonicalLifecycleAlert(canonicalLifecycleAlertParams{
Spec: spec,
Evidence: alertspecs.AlertEvidence{
ObservedAt: m.policyNow().UTC(),
PoweredState: &alertspecs.PoweredStateEvidence{
Expected: alertspecs.PowerStateOn,
Observed: alertspecs.PowerStateOff,
},
},
AlertID: alertID,
AlertType: "powered-off",
ResourceID: guestID,
ResourceName: name,
Node: node,
Instance: instanceName,
Message: fmt.Sprintf("%s '%s' is powered off", guestType, name),
Summary: "Powered off",
Metadata: metadata,
AddToRecent: true,
AddToHistory: true,
DispatchAsync: false,
IntentBackup: backupContext,
})
}
// clearGuestPoweredOffAlert removes powered-off alert when guest starts running.
func (m *Manager) clearGuestPoweredOffAlert(guestID, name string) {
alertID := canonicalPoweredStateStateID(guestID)
m.mu.Lock()
migratedAlertIdentity := false
defer func() {
if migratedAlertIdentity {
m.saveActiveAlertsAsync("guest powered-off node move resolve")
}
}()
defer m.mu.Unlock()
// Healthy and intentionally stopped observations end the shadow's run as
// well, even when the live alert is only pending (and the lookup below
// returns early). Otherwise a later stopped run can inherit old grace or
// confirmation evidence and falsely disagree with the live manager.
defer m.shadowObserveRecoveryNoLock(guestID, canonicalPoweredStateSpecID(guestID), alertID, 1)
// Reset confirmation count when guest comes back online
if decision := m.evaluateIntentNoLock(guestID, "", string(AlertIntentSignalOffline), alertID, m.policyNow().UTC(), false, BackupIntentContext{}); decision.StateChanged {
m.saveActiveAlertsAsync("guest offline intent cleared")
}
// A running guest is a healthy observation for the core's powered-state
// run: pending clears, and a firing incident resolves immediately (the
// powered-off family has no recovery gate).
m.core.ApplyDiscrete(reducer.DiscreteSignal{
ResourceID: guestID,
Key: canonicalPoweredStateSpecID(guestID),
Matched: false,
ObservedAt: m.policyNow(),
}, reducer.DiscreteRule{})
// Check if powered-off alert exists. A guest that moved nodes may still
// hold the alert under its old node-scoped identity; re-home it so it
// resolves instead of stranding.
alert, exists := m.getActiveAlertNoLock(alertID)
if !exists {
alert = m.migrateGuestAlertNoLock(alertID, canonicalPoweredStateSpecID(guestID), string(alertspecs.AlertSpecKindPoweredState), guestID, name, "", "", "guest")
if alert == nil {
return
}
migratedAlertIdentity = true
}
// Remove from active alerts
m.removeActiveAlertNoLock(alertID)
downtime := time.Since(alert.StartTime)
resolvedAlert := m.newResolvedAlert(alert, time.Now(), nil)
m.addRecentlyResolvedWithPrimaryLock(resolvedAlert)
// Send recovery notification (async to avoid deadlock because callback acquires m.mu.RLock
// via ShouldSuppressResolvedNotification, and we currently hold m.mu.Lock)
m.safeCallResolvedAlertCallback(resolvedAlert, effectiveAlertID(alert, alertID), true)
// Log recovery
log.Info().
Str("guest", name).
Dur("downtime", downtime).
Msg("Guest is now running")
}
type pulseTagSettings struct {
Suppress bool
MonitorOnly bool
Relaxed bool
}
type guestAlertPolicy struct {
TagSettings pulseTagSettings
SuppressionReason string
}
func (m *Manager) resolveGuestAlertPolicy(snapshot guestSnapshot) guestAlertPolicy {
m.mu.RLock()
ignoredPrefixes := append([]string(nil), m.config.IgnoredGuestPrefixes...)
tagWhitelist := append([]string(nil), m.config.GuestTagWhitelist...)
tagBlacklist := append([]string(nil), m.config.GuestTagBlacklist...)
m.mu.RUnlock()
policy := guestAlertPolicy{TagSettings: parsePulseTags(snapshot.Tags)}
for _, prefix := range ignoredPrefixes {
if prefix != "" && strings.HasPrefix(snapshot.Name, prefix) {
policy.SuppressionReason = "ignored-prefix"
return policy
}
}
if policy.TagSettings.Suppress {
policy.SuppressionReason = "pulse-no-alerts"
return policy
}
if len(tagBlacklist) == 0 && len(tagWhitelist) == 0 {
return policy
}
normalizedTags := make(map[string]struct{}, len(snapshot.Tags))
for _, tag := range snapshot.Tags {
normalizedTags[strings.ToLower(strings.TrimSpace(tag))] = struct{}{}
}
for _, blockedTag := range tagBlacklist {
if _, blocked := normalizedTags[strings.ToLower(strings.TrimSpace(blockedTag))]; blocked {
policy.SuppressionReason = "tag-blacklist"
return policy
}
}
if len(tagWhitelist) == 0 {
return policy
}
for _, allowedTag := range tagWhitelist {
if _, allowed := normalizedTags[strings.ToLower(strings.TrimSpace(allowedTag))]; allowed {
return policy
}
}
policy.SuppressionReason = "tag-whitelist"
return policy
}
func parsePulseTags(tags []string) pulseTagSettings {
settings := pulseTagSettings{}
for _, raw := range tags {
tag := strings.TrimSpace(strings.ToLower(raw))
switch tag {
case "pulse-no-alerts":
settings.Suppress = true
case "pulse-monitor-only":
settings.MonitorOnly = true
case "pulse-relaxed":
settings.Relaxed = true
}
}
return settings
}
// applyRelaxedGuestThresholds lifts a pulse-relaxed guest's CPU, memory and
// disk triggers to a 95/92/95 floor. An unset threshold gets the floor; one
// switched Off (trigger <= 0) stays Off, since relaxing a guest must never
// raise an alert its config turned off.
func applyRelaxedGuestThresholds(cfg ThresholdConfig) ThresholdConfig {
relaxed := cloneThresholdConfig(cfg)
adjust := func(th **HysteresisThreshold, minTrigger float64) {
if *th == nil {
*th = &HysteresisThreshold{Trigger: minTrigger, Clear: minTrigger - 5}
return
}
if (*th).Trigger <= 0 {
return
}
ensureHysteresisThreshold(*th)
if (*th).Trigger < minTrigger {
(*th).Trigger = minTrigger
}
if (*th).Clear >= (*th).Trigger {
(*th).Clear = (*th).Trigger - 5
}
if (*th).Clear < 0 {
(*th).Clear = 0
}
}
adjust(&relaxed.CPU, 95)
adjust(&relaxed.Memory, 92)
adjust(&relaxed.Disk, 95)
return relaxed
}
func (m *Manager) suppressGuestAlerts(guestID string) bool {
m.mu.Lock()
defer m.mu.Unlock()
cleared := false
for storageKey, alert := range m.activeAlerts {
alertID := effectiveAlertID(alert, storageKey)
trackingKey := canonicalTrackingKeyForAlert(alert)
if alert == nil {
continue
}
if alert.ResourceID == guestID || strings.HasPrefix(alert.ResourceID, guestID+"/") || strings.HasPrefix(alertID, guestID) ||
guestAlertBelongsToGuest(alert.ResourceID, guestID) {
m.clearAlertNoLock(alertID)
delete(m.recentAlerts, trackingKey)
delete(m.suppressedUntil, trackingKey)
delete(m.alertRateLimit, trackingKey)
cleared = true
}
}
m.core.DropPendingForResource(guestID)
for key := range m.recentAlerts {
if strings.HasPrefix(key, guestID) {
delete(m.recentAlerts, key)
}
}
for key := range m.suppressedUntil {
if strings.HasPrefix(key, guestID) {
delete(m.suppressedUntil, key)
}
}
for key := range m.alertRateLimit {
if strings.HasPrefix(key, guestID) {
delete(m.alertRateLimit, key)
}
}
for key, pending := range m.intentPending {
if pending.ResourceID == guestID || strings.HasPrefix(pending.ResourceID, guestID+"/") {
if m.clearIntentPendingNoLock(key) {
cleared = true
}
}
}
return cleared
}
func (m *Manager) guestHasMonitorOnlyAlerts(guestID string) bool {
m.mu.RLock()
defer m.mu.RUnlock()
for _, alert := range m.activeAlerts {
if alert == nil {
continue
}
if alert.ResourceID != guestID {
continue
}
if isMonitorOnlyAlert(alert) {
return true
}
}
return false
}