Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
package alerts
import (
"fmt"
"strings"
"time"
alertspecs "github.com/rcourtman/pulse-go-rewrite/internal/alerts/specs"
"github.com/rcourtman/pulse-go-rewrite/internal/models"
"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
"github.com/rs/zerolog/log"
)
func isPBSOffline(pbs models.PBSInstance) bool {
status := strings.ToLower(strings.TrimSpace(pbs.Status))
health := strings.ToLower(strings.TrimSpace(pbs.ConnectionHealth))
return status == "offline" || health == "error" || health == "failed" || health == "unhealthy"
}
func (m *Manager) clearPBSMetricAlerts(pbsID string) {
if strings.TrimSpace(pbsID) == "" {
return
}
m.clearAlert(canonicalMetricStateID(pbsID, "cpu"))
m.clearAlert(canonicalMetricStateID(pbsID, "memory"))
}
// CheckPBS checks PBS instance metrics against thresholds
func (m *Manager) CheckPBS(pbs models.PBSInstance) {
m.mu.RLock()
if !m.config.Enabled {
m.mu.RUnlock()
return
}
if allDisabled, _ := m.alertPolicyTypeSwitchesNoLock("pbs"); allDisabled {
m.mu.RUnlock()
// Clear any existing PBS alerts when all PBS alerts are disabled
m.mu.Lock()
// Reset offline confirmation tracking
// Clear CPU alert
cpuAlertID := canonicalMetricStateID(pbs.ID, "cpu")
if m.clearActiveAlertIfPresentNoLock(cpuAlertID) {
log.Info().
Str("alertID", cpuAlertID).
Str("pbs", pbs.Name).
Msg("Cleared CPU alert - all PBS alerts disabled")
}
// Clear Memory alert
memAlertID := canonicalMetricStateID(pbs.ID, "memory")
if m.clearActiveAlertIfPresentNoLock(memAlertID) {
log.Info().
Str("alertID", memAlertID).
Str("pbs", pbs.Name).
Msg("Cleared Memory alert - all PBS alerts disabled")
}
// Clear offline alert
offlineAlertID := canonicalConnectivityStateID(pbs.ID)
if m.clearActiveAlertIfPresentNoLock(offlineAlertID) {
log.Info().
Str("alertID", offlineAlertID).
Str("pbs", pbs.Name).
Msg("Cleared offline alert - all PBS alerts disabled")
}
m.mu.Unlock()
return
}
thresholds := m.resolveResourceThresholds("pbs", pbs.ID)
_, disablePBSOffline := m.alertPolicyTypeSwitchesNoLock("pbs")
m.mu.RUnlock()
// Check override disable BEFORE offline detection to prevent spurious notifications
if thresholds.Disabled {
m.mu.Lock()
// Reset offline confirmation tracking
// Clear CPU alert
cpuAlertID := canonicalMetricStateID(pbs.ID, "cpu")
if m.clearActiveAlertIfPresentNoLock(cpuAlertID) {
log.Debug().
Str("alertID", cpuAlertID).
Str("pbs", pbs.Name).
Msg("Cleared CPU alert - PBS has alerts disabled")
}
// Clear Memory alert
memAlertID := canonicalMetricStateID(pbs.ID, "memory")
if m.clearActiveAlertIfPresentNoLock(memAlertID) {
log.Debug().
Str("alertID", memAlertID).
Str("pbs", pbs.Name).
Msg("Cleared Memory alert - PBS has alerts disabled")
}
// Clear offline alert
offlineAlertID := canonicalConnectivityStateID(pbs.ID)
if m.clearActiveAlertIfPresentNoLock(offlineAlertID) {
log.Debug().
Str("alertID", offlineAlertID).
Str("pbs", pbs.Name).
Msg("Cleared offline alert - PBS has alerts disabled")
}
m.mu.Unlock()
return
}
pbsOffline := isPBSOffline(pbs)
if disablePBSOffline || thresholds.DisableConnectivity {
// Clear tracking and any existing offline alerts when globally disabled
m.mu.Lock()
m.mu.Unlock()
m.clearAlert(canonicalConnectivityStateID(pbs.ID))
} else {
// Check if PBS is offline first (similar to nodes)
if pbsOffline {
m.checkPBSOfflineWithThresholds(pbs, thresholds)
} else {
// Clear any existing offline alert if PBS is back online
m.clearPBSOfflineAlert(pbs)
}
}
// When PBS is offline/unhealthy, clear stale metric alerts immediately.
if pbsOffline {
m.clearPBSMetricAlerts(pbs.ID)
return
}
input := &UnifiedResourceInput{
ID: pbs.ID,
Type: "pbs",
Name: pbs.Name,
Node: pbs.Host,
Instance: pbs.Name,
}
// Endpoint failure is a missing observation, not merely a pause in metric
// evaluation. Let the shared evaluator interrupt activation/recovery runs
// and intent grace without changing an open alert's trusted evidence.
// Connectivity above and full-outage precedence remain independent.
if !pbs.NodeMetricsUnavailable {
input.CPU = &UnifiedResourceMetric{Percent: pbs.CPU}
input.Memory = &UnifiedResourceMetric{Percent: pbs.Memory}
}
m.evaluateUnifiedMetrics(input, thresholds, nil)
}
// checkPBSOffline creates an alert for offline PBS instances
func (m *Manager) checkPBSOffline(pbs models.PBSInstance) {
m.mu.RLock()
thresholds := m.resolveResourceThresholds("pbs", pbs.ID)
m.mu.RUnlock()
m.checkPBSOfflineWithThresholds(pbs, thresholds)
}
// checkPBSOfflineWithThresholds evaluates one observation against the policy
// snapshot CheckPBS resolved when that observation began. A concurrent config
// update applies to the next observation instead of changing eligibility
// between this observation's pre-dispatch gate and lifecycle evaluation.
func (m *Manager) checkPBSOfflineWithThresholds(pbs models.PBSInstance, thresholds ThresholdConfig) {
m.mu.Lock()
m.mu.Unlock()
spec, err := buildCanonicalConnectivitySpec(pbs.ID, pbs.Name, unifiedresources.ResourceTypePBS, AlertLevelCritical, 3, thresholds.Disabled || thresholds.DisableConnectivity)
if err != nil {
log.Warn().
Err(err).
Str("pbs", pbs.Name).
Str("pbsID", pbs.ID).
Msg("Skipping invalid canonical PBS connectivity spec")
return
}
_, _ = m.evaluateCanonicalLifecycleAlert(canonicalLifecycleAlertParams{
Spec: spec,
Evidence: alertspecs.AlertEvidence{ObservedAt: time.Now(), Connectivity: &alertspecs.ConnectivityEvidence{Signal: "status", Connected: false}},
AlertID: fmt.Sprintf("pbs-offline-%s", pbs.ID),
AlertType: "offline",
ResourceID: pbs.ID,
ResourceName: pbs.Name,
Node: pbs.Host,
Instance: pbs.Name,
Message: fmt.Sprintf("PBS instance %s is offline", pbs.Name),
Summary: "Offline",
Metadata: map[string]interface{}{
"resourceType": "pbs",
"status": pbs.Status,
"connectionHealth": pbs.ConnectionHealth,
},
RateLimit: true,
DispatchAsync: true,
})
}
// clearPBSOfflineAlert removes offline alert when PBS comes back online
func (m *Manager) clearPBSOfflineAlert(pbs models.PBSInstance) {
m.clearResourceOfflineAlert(pbs.ID, pbs.Name, pbs.Host, "PBS", offlineRecoveryConfirmationsDefault)
}