Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

6.3 KB
Raw Blame
package alerts

import (
	"fmt"
	"strings"
	"time"

	alertspecs "github.com/rcourtman/pulse-go-rewrite/internal/alerts/specs"
	"github.com/rcourtman/pulse-go-rewrite/internal/models"
	"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
	"github.com/rs/zerolog/log"
)

func isPBSOffline(pbs models.PBSInstance) bool {
	status := strings.ToLower(strings.TrimSpace(pbs.Status))
	health := strings.ToLower(strings.TrimSpace(pbs.ConnectionHealth))
	return status == "offline" || health == "error" || health == "failed" || health == "unhealthy"
}

func (m *Manager) clearPBSMetricAlerts(pbsID string) {
	if strings.TrimSpace(pbsID) == "" {
		return
	}

	m.clearAlert(canonicalMetricStateID(pbsID, "cpu"))
	m.clearAlert(canonicalMetricStateID(pbsID, "memory"))
}

// CheckPBS checks PBS instance metrics against thresholds
func (m *Manager) CheckPBS(pbs models.PBSInstance) {
	m.mu.RLock()
	if !m.config.Enabled {
		m.mu.RUnlock()
		return
	}
	if allDisabled, _ := m.alertPolicyTypeSwitchesNoLock("pbs"); allDisabled {
		m.mu.RUnlock()
		// Clear any existing PBS alerts when all PBS alerts are disabled
		m.mu.Lock()
		// Reset offline confirmation tracking
		// Clear CPU alert
		cpuAlertID := canonicalMetricStateID(pbs.ID, "cpu")
		if m.clearActiveAlertIfPresentNoLock(cpuAlertID) {
			log.Info().
				Str("alertID", cpuAlertID).
				Str("pbs", pbs.Name).
				Msg("Cleared CPU alert - all PBS alerts disabled")
		}
		// Clear Memory alert
		memAlertID := canonicalMetricStateID(pbs.ID, "memory")
		if m.clearActiveAlertIfPresentNoLock(memAlertID) {
			log.Info().
				Str("alertID", memAlertID).
				Str("pbs", pbs.Name).
				Msg("Cleared Memory alert - all PBS alerts disabled")
		}
		// Clear offline alert
		offlineAlertID := canonicalConnectivityStateID(pbs.ID)
		if m.clearActiveAlertIfPresentNoLock(offlineAlertID) {
			log.Info().
				Str("alertID", offlineAlertID).
				Str("pbs", pbs.Name).
				Msg("Cleared offline alert - all PBS alerts disabled")
		}
		m.mu.Unlock()
		return
	}

	thresholds := m.resolveResourceThresholds("pbs", pbs.ID)
	_, disablePBSOffline := m.alertPolicyTypeSwitchesNoLock("pbs")
	m.mu.RUnlock()

	// Check override disable BEFORE offline detection to prevent spurious notifications
	if thresholds.Disabled {
		m.mu.Lock()
		// Reset offline confirmation tracking
		// Clear CPU alert
		cpuAlertID := canonicalMetricStateID(pbs.ID, "cpu")
		if m.clearActiveAlertIfPresentNoLock(cpuAlertID) {
			log.Debug().
				Str("alertID", cpuAlertID).
				Str("pbs", pbs.Name).
				Msg("Cleared CPU alert - PBS has alerts disabled")
		}
		// Clear Memory alert
		memAlertID := canonicalMetricStateID(pbs.ID, "memory")
		if m.clearActiveAlertIfPresentNoLock(memAlertID) {
			log.Debug().
				Str("alertID", memAlertID).
				Str("pbs", pbs.Name).
				Msg("Cleared Memory alert - PBS has alerts disabled")
		}
		// Clear offline alert
		offlineAlertID := canonicalConnectivityStateID(pbs.ID)
		if m.clearActiveAlertIfPresentNoLock(offlineAlertID) {
			log.Debug().
				Str("alertID", offlineAlertID).
				Str("pbs", pbs.Name).
				Msg("Cleared offline alert - PBS has alerts disabled")
		}
		m.mu.Unlock()
		return
	}

	pbsOffline := isPBSOffline(pbs)

	if disablePBSOffline || thresholds.DisableConnectivity {
		// Clear tracking and any existing offline alerts when globally disabled
		m.mu.Lock()
		m.mu.Unlock()
		m.clearAlert(canonicalConnectivityStateID(pbs.ID))
	} else {
		// Check if PBS is offline first (similar to nodes)
		if pbsOffline {
			m.checkPBSOfflineWithThresholds(pbs, thresholds)
		} else {
			// Clear any existing offline alert if PBS is back online
			m.clearPBSOfflineAlert(pbs)
		}
	}

	// When PBS is offline/unhealthy, clear stale metric alerts immediately.
	if pbsOffline {
		m.clearPBSMetricAlerts(pbs.ID)
		return
	}

	input := &UnifiedResourceInput{
		ID:       pbs.ID,
		Type:     "pbs",
		Name:     pbs.Name,
		Node:     pbs.Host,
		Instance: pbs.Name,
	}
	// Endpoint failure is a missing observation, not merely a pause in metric
	// evaluation. Let the shared evaluator interrupt activation/recovery runs
	// and intent grace without changing an open alert's trusted evidence.
	// Connectivity above and full-outage precedence remain independent.
	if !pbs.NodeMetricsUnavailable {
		input.CPU = &UnifiedResourceMetric{Percent: pbs.CPU}
		input.Memory = &UnifiedResourceMetric{Percent: pbs.Memory}
	}
	m.evaluateUnifiedMetrics(input, thresholds, nil)
}

// checkPBSOffline creates an alert for offline PBS instances
func (m *Manager) checkPBSOffline(pbs models.PBSInstance) {
	m.mu.RLock()
	thresholds := m.resolveResourceThresholds("pbs", pbs.ID)
	m.mu.RUnlock()
	m.checkPBSOfflineWithThresholds(pbs, thresholds)
}

// checkPBSOfflineWithThresholds evaluates one observation against the policy
// snapshot CheckPBS resolved when that observation began. A concurrent config
// update applies to the next observation instead of changing eligibility
// between this observation's pre-dispatch gate and lifecycle evaluation.
func (m *Manager) checkPBSOfflineWithThresholds(pbs models.PBSInstance, thresholds ThresholdConfig) {
	m.mu.Lock()
	m.mu.Unlock()

	spec, err := buildCanonicalConnectivitySpec(pbs.ID, pbs.Name, unifiedresources.ResourceTypePBS, AlertLevelCritical, 3, thresholds.Disabled || thresholds.DisableConnectivity)
	if err != nil {
		log.Warn().
			Err(err).
			Str("pbs", pbs.Name).
			Str("pbsID", pbs.ID).
			Msg("Skipping invalid canonical PBS connectivity spec")
		return
	}

	_, _ = m.evaluateCanonicalLifecycleAlert(canonicalLifecycleAlertParams{
		Spec:         spec,
		Evidence:     alertspecs.AlertEvidence{ObservedAt: time.Now(), Connectivity: &alertspecs.ConnectivityEvidence{Signal: "status", Connected: false}},
		AlertID:      fmt.Sprintf("pbs-offline-%s", pbs.ID),
		AlertType:    "offline",
		ResourceID:   pbs.ID,
		ResourceName: pbs.Name,
		Node:         pbs.Host,
		Instance:     pbs.Name,
		Message:      fmt.Sprintf("PBS instance %s is offline", pbs.Name),
		Summary:      "Offline",
		Metadata: map[string]interface{}{
			"resourceType":     "pbs",
			"status":           pbs.Status,
			"connectionHealth": pbs.ConnectionHealth,
		},
		RateLimit:     true,
		DispatchAsync: true,
	})
}

// clearPBSOfflineAlert removes offline alert when PBS comes back online
func (m *Manager) clearPBSOfflineAlert(pbs models.PBSInstance) {
	m.clearResourceOfflineAlert(pbs.ID, pbs.Name, pbs.Host, "PBS", offlineRecoveryConfirmationsDefault)
}