Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
package monitoring
import (
"strings"
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
"github.com/rcourtman/pulse-go-rewrite/internal/logging"
"github.com/rcourtman/pulse-go-rewrite/internal/models"
"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
"github.com/rs/zerolog"
"github.com/rs/zerolog/log"
)
// syncAlertsToState copies the latest alert manager data into the shared state snapshot.
// This keeps WebSocket broadcasts aligned with in-memory acknowledgement updates.
func (m *Monitor) syncAlertsToState() {
if m.pruneStaleDockerAlerts() {
if logging.IsLevelEnabled(zerolog.DebugLevel) {
log.Debug().Msg("pruned stale docker alerts during sync")
}
}
modelAlerts := m.activeAlertsSnapshot()
for _, alert := range modelAlerts {
if alert.Acknowledged && logging.IsLevelEnabled(zerolog.DebugLevel) {
log.Debug().Str("alertID", alert.ID).Interface("ackTime", alert.AckTime).Msg("syncing acknowledged alert")
}
}
m.state.UpdateActiveAlerts(modelAlerts)
recentlyResolved := m.alertManager.GetRecentlyResolved()
if len(recentlyResolved) > 0 && logging.IsLevelEnabled(zerolog.DebugLevel) {
log.Debug().Int("count", len(recentlyResolved)).Msg("syncing recently resolved alerts")
}
m.state.UpdateRecentlyResolved(recentlyResolved)
}
// SyncAlertState is the exported wrapper used by APIs that mutate alerts outside the poll loop.
func (m *Monitor) SyncAlertState() {
m.syncAlertsToState()
}
func (m *Monitor) activeAlertsSnapshot() []models.Alert {
if m == nil {
return nil
}
if m.alertManager == nil {
if m.state == nil {
return nil
}
return m.state.GetSnapshot().ActiveAlerts
}
activeAlerts := m.alertManager.GetActiveAlerts()
modelAlerts := make([]models.Alert, 0, len(activeAlerts))
for _, alert := range activeAlerts {
modelAlerts = append(modelAlerts, models.Alert{
ID: alert.ID,
Type: alert.Type,
Level: string(alert.Level),
ResourceID: alert.ResourceID,
ResourceName: alert.ResourceName,
Node: alert.Node,
NodeDisplayName: alert.NodeDisplayName,
Instance: alert.Instance,
Message: alert.Message,
Summary: alert.Summary,
Value: alert.Value,
Threshold: alert.Threshold,
StartTime: alert.StartTime,
Acknowledged: alert.Acknowledged,
AckTime: alert.AckTime,
AckUser: alert.AckUser,
// GetActiveAlerts returns deep clones, so the map and the live
// metric status are already private.
Metadata: alert.Metadata,
MetricStatus: alert.MetricStatus,
})
}
return modelAlerts
}
func (m *Monitor) recentlyResolvedAlertsSnapshot() []models.ResolvedAlert {
if m == nil {
return nil
}
if m.alertManager == nil {
if m.state == nil {
return nil
}
return m.state.GetSnapshot().RecentlyResolved
}
return m.alertManager.GetRecentlyResolved()
}
// syncUnifiedResourceAlertsToState evaluates resources read under scope. A
// mock-mode flip since that read refuses every remaining step, so a snapshot
// of the mode the monitor left can neither open or resolve alerts nor rewrite
// persisted identities and links in the mode it entered.
func (m *Monitor) syncUnifiedResourceAlertsToState(resources []unifiedresources.Resource, scope mockModeScope) {
if m == nil || m.alertManager == nil {
return
}
// Alert intent resolves through the published registry with saved hosts
// overlaid, under the alert manager's lock. Build that overlay for the new
// generation here, outside the lock, so its lookups need not.
m.operatorIntentIdentity.Load().refresh()
// Resource publication can make persisted canonical policy resolvable for
// restored native-ID alerts that preceded the first observation at startup.
scope.run(func() { m.alertManager.ReconcileOperatorIntentState() })
scope.run(func() { m.migrateAlertIdentities(resources) })
scope.run(func() { m.migrateAvailabilityLinksToCanonicalIDs(resources) })
// Metrics go one resource per fenced call, as
// CheckUnifiedResourceMetricsWithCapacityTrends would iterate them, so a
// mode change waits for one resource's evaluation rather than the estate's.
capacityTrends := m.unifiedStorageCapacityTrends(resources, time.Now())
for _, resource := range resources {
input, ok := alerts.UnifiedResourceInputFromResource(resource)
if !ok {
continue
}
scope.run(func() { m.alertManager.CheckUnifiedResourceWithCapacityTrend(input, capacityTrends[input.ID]) })
}
scope.run(func() { m.alertManager.SyncUnifiedResourceIncidents(resources) })
m.syncAlertsToState()
}
// migrateAlertIdentities persists the alert identity migration that resources
// make resolvable.
func (m *Monitor) migrateAlertIdentities(resources []unifiedresources.Resource) {
config := m.alertManager.GetConfig()
migrationPlan := alerts.PlanAlertIdentityMigration(config, resources)
if migrationPlan.UnsupportedVersion {
log.Warn().
Int("configVersion", migrationPlan.FromVersion).
Int("supportedVersion", alerts.CurrentAlertIdentitySchemaVersion).
Msg("alert identity config uses a newer schema; leaving persisted identities untouched")
} else if alerts.ApplyAlertIdentityMigration(&config, migrationPlan) {
if m.configPersist == nil {
log.Warn().
Int("fromVersion", migrationPlan.FromVersion).
Int("toVersion", migrationPlan.ToVersion).
Int("removedOverrides", len(migrationPlan.RemovedOverrideKeys)).
Int("addedOverrides", len(migrationPlan.AddedOverrideKeys)).
Int("deferredOverrides", len(migrationPlan.Deferred)).
Msg("cannot persist planned alert identity migration without config persistence")
} else if err := m.configPersist.SaveAlertConfig(config); err != nil {
log.Error().
Err(err).
Int("fromVersion", migrationPlan.FromVersion).
Int("toVersion", migrationPlan.ToVersion).
Msg("failed to persist planned alert identity migration")
} else {
m.alertManager.UpdateConfig(config)
log.Info().
Int("fromVersion", migrationPlan.FromVersion).
Int("toVersion", migrationPlan.ToVersion).
Int("removedOverrides", len(migrationPlan.RemovedOverrideKeys)).
Int("addedOverrides", len(migrationPlan.AddedOverrideKeys)).
Int("deferredOverrides", len(migrationPlan.Deferred)).
Msg("persisted alert identity migration plan")
}
}
}
// pruneStaleDockerAlerts removes docker alerts that reference hosts no longer present in state.
func (m *Monitor) pruneStaleDockerAlerts() bool {
if m.alertManager == nil {
return false
}
// The inventory is the fixture read view in mock mode. A prune that read
// it must not remove Docker alerts the other mode raised after a switch.
// Only the hosts' IDs matter, so the view may predate the last metric ticks.
scope := m.mockModeFence.begin()
readState := m.currentModeStructureReadState()
if readState == nil {
return false
}
hosts := readState.DockerHosts()
knownHosts := make(map[string]struct{}, len(hosts)*2)
for _, host := range hosts {
hostID := strings.TrimSpace(host.ID())
if hostID != "" {
knownHosts[hostID] = struct{}{}
}
if sourceID := strings.TrimSpace(host.HostSourceID()); sourceID != "" {
knownHosts[sourceID] = struct{}{}
}
}
if len(knownHosts) == 0 {
// Still allow stale entries to be cleared if no hosts remain.
}
active := m.alertManager.GetActiveAlerts()
processed := make(map[string]struct{})
cleared := false
for _, alert := range active {
var hostID string
switch {
case alert.Type == "docker-host-offline":
hostID = strings.TrimPrefix(strings.TrimSpace(alert.ResourceID), "docker:")
case strings.HasPrefix(alert.ResourceID, "docker:"):
resource := strings.TrimPrefix(alert.ResourceID, "docker:")
if idx := strings.Index(resource, "/"); idx >= 0 {
hostID = resource[:idx]
} else {
hostID = resource
}
default:
continue
}
hostID = strings.TrimSpace(hostID)
if hostID == "" {
continue
}
if _, known := knownHosts[hostID]; known {
continue
}
if _, alreadyCleared := processed[hostID]; alreadyCleared {
continue
}
host := models.DockerHost{
ID: hostID,
DisplayName: alert.ResourceName,
Hostname: alert.Node,
}
if host.DisplayName == "" {
host.DisplayName = hostID
}
if host.Hostname == "" {
host.Hostname = hostID
}
if !scope.run(func() { m.alertManager.HandleDockerHostRemoved(host) }) {
return cleared
}
processed[hostID] = struct{}{}
cleared = true
}
return cleared
}