Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
package reporting
import (
"fmt"
"strings"
)
// AlertResolutionMovedToAgent is the reason code of an alert that closed
// because a linked Pulse agent took its metric over
// (alerts.AlertResolutionMovedToAgent).
const AlertResolutionMovedToAgent = "moved_to_agent"
// Recovered reports whether the alert closed because its condition cleared.
// An open alert, or one that closed without recovering, is not recovered.
func (a AlertInfo) Recovered() bool {
return a.ResolvedTime != nil && a.Resolution == nil
}
// ClosedWithoutRecovery reports whether the alert closed while its condition
// may still hold, such as a node alert handed to its Pulse agent.
func (a AlertInfo) ClosedWithoutRecovery() bool {
return a.ResolvedTime != nil && a.Resolution != nil
}
// ResolutionOutcome is the short form of a close that was not a recovery,
// such as "moved to pve1 (Host Agent)". It is empty for an open alert and for
// an ordinary recovery.
func (a AlertInfo) ResolutionOutcome() string {
if !a.ClosedWithoutRecovery() {
return ""
}
if a.Resolution.Reason == AlertResolutionMovedToAgent {
return "moved to " + a.successorLabel()
}
return "closed without recovering"
}
// ResolutionSummary is the alert engine's account of a close that was not a
// recovery, falling back to the short form when the engine supplied none. It
// is empty for an open alert and for an ordinary recovery.
func (a AlertInfo) ResolutionSummary() string {
if !a.ClosedWithoutRecovery() {
return ""
}
if summary := strings.TrimSpace(a.Resolution.Summary); summary != "" {
return summary
}
return fmt.Sprintf("Alert %s. This is not a recovery.", a.ResolutionOutcome())
}
func (a AlertInfo) successorLabel() string {
if a.Resolution != nil {
if name := strings.TrimSpace(a.Resolution.SuccessorName); name != "" {
return name
}
}
return "its Pulse agent"
}
// unresolvedAlerts returns the alerts a report must treat as live: every open
// alert, and every alert that closed without recovering unless the report
// also shows its successor's own reading for that metric. That alert carries
// the condition's current state, so one condition is never counted twice and
// a handover never reads as a recovery.
func unresolvedAlerts(alerts []AlertInfo) []AlertInfo {
return unresolvedAlertsAmong(alerts, alerts)
}
// unresolvedAlertsAmong is unresolvedAlerts with successors looked up in
// pool, so a fleet report matches a handover against every resource's alerts.
func unresolvedAlertsAmong(alerts, pool []AlertInfo) []AlertInfo {
out := make([]AlertInfo, 0, len(alerts))
for _, alert := range alerts {
switch {
case alert.ResolvedTime == nil:
out = append(out, alert)
case alert.ClosedWithoutRecovery() && successorAlertFor(alert, pool) == nil:
out = append(out, alert)
}
}
return out
}
// successorAlertFor returns the alert in pool that carries a moved alert's
// condition: one of the same type on its successor or one of the successor's
// children (an agent's filesystem alerts sit under "<agent>/disk:<mount>"),
// still open or recovered at or after the handover. A successor recovery from
// before the handover says nothing about the reading since, so it does not
// count.
func successorAlertFor(moved AlertInfo, pool []AlertInfo) *AlertInfo {
if !moved.ClosedWithoutRecovery() {
return nil
}
successor := strings.TrimSpace(moved.Resolution.SuccessorResourceID)
if successor == "" {
return nil
}
for i := range pool {
candidate := &pool[i]
if candidate.Type != moved.Type || candidate.ClosedWithoutRecovery() {
continue
}
if candidate.ResolvedTime != nil && candidate.ResolvedTime.Before(*moved.ResolvedTime) {
continue
}
resourceID := strings.TrimSpace(candidate.ResourceID)
if resourceID == successor || strings.HasPrefix(resourceID, successor+"/") {
return candidate
}
}
return nil
}
// SuccessorAlertListed reports whether alerts shows the reading of the
// resource this alert moved to: an alert of the same type on the successor,
// open or recovered since the handover. The report then counts the condition
// once, through that alert. It is false for an open alert or a recovery.
func (a AlertInfo) SuccessorAlertListed(alerts []AlertInfo) bool {
return successorAlertFor(a, alerts) != nil
}
// fleetAlerts is every alert listed across a fleet report's resources.
func (d *MultiReportData) fleetAlerts() []AlertInfo {
if d == nil {
return nil
}
var pool []AlertInfo
for _, rd := range d.Resources {
if rd != nil {
pool = append(pool, rd.Alerts...)
}
}
return pool
}
// alertHealth is the health verdict a report's alerts support. The PDF health
// card and the heuristic narrator both read it, so they never disagree.
type alertHealth struct {
Status string // HEALTHY, WARNING or CRITICAL
Message string
Critical int
Warning int
// Unresolved is every alert the report treats as live (unresolvedAlerts).
Unresolved []AlertInfo
}
func assessAlertHealth(alerts []AlertInfo) alertHealth {
health := alertHealth{Unresolved: unresolvedAlerts(alerts)}
open := 0
var moved []AlertInfo
for _, alert := range health.Unresolved {
if alert.Level == "critical" {
health.Critical++
} else {
health.Warning++
}
if alert.ClosedWithoutRecovery() {
moved = append(moved, alert)
} else {
open++
}
}
switch {
case health.Critical > 0:
health.Status = "CRITICAL"
case health.Warning > 0:
health.Status = "WARNING"
default:
health.Status = "HEALTHY"
}
switch {
case open == 0 && len(moved) > 0:
health.Message = movedAlertsHealthMessage(moved)
case health.Critical == 1:
health.Message = "1 critical issue requires immediate attention"
case health.Critical > 1:
health.Message = fmt.Sprintf("%d critical issues require immediate attention", health.Critical)
case health.Warning == 1:
health.Message = "1 warning detected - review recommended"
case health.Warning > 1:
health.Message = fmt.Sprintf("%d warnings detected - review recommended", health.Warning)
default:
health.Message = "All systems operating normally"
}
return health
}
// movedAlertsHealthMessage says where alerts that closed without recovering
// went, so the report points at the resource that now holds the reading
// instead of calling this one healthy.
func movedAlertsHealthMessage(moved []AlertInfo) string {
first := moved[0]
for _, alert := range moved[1:] {
if alert.ResolutionOutcome() != first.ResolutionOutcome() {
return fmt.Sprintf("%d alerts closed without recovering - review recommended", len(moved))
}
}
if first.Resolution.Reason != AlertResolutionMovedToAgent {
if len(moved) == 1 {
return "1 alert closed without recovering - review recommended"
}
return fmt.Sprintf("%d alerts closed without recovering - review recommended", len(moved))
}
if len(moved) == 1 {
return fmt.Sprintf("Alert %s - check the agent for the current reading", first.ResolutionOutcome())
}
return fmt.Sprintf("%d alerts %s - check the agent for the current reading", len(moved), first.ResolutionOutcome())
}
// movedAlertSuccessors names, once each and in report order, the Pulse agents
// that unresolved alerts moved to.
func movedAlertSuccessors(unresolved []AlertInfo) []string {
var successors []string
seen := make(map[string]struct{})
for _, alert := range unresolved {
if !alert.ClosedWithoutRecovery() || alert.Resolution.Reason != AlertResolutionMovedToAgent {
continue
}
successor := alert.successorLabel()
if _, ok := seen[successor]; ok {
continue
}
seen[successor] = struct{}{}
successors = append(successors, successor)
}
return successors
}