Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
package alerts
// Alert event emission: the manager records lifecycle transitions and
// notification decisions — including suppressions, with reasons — into the
// append-only event log (docs/ALERT_ENGINE_EVOLUTION.md, Phase 0). Emission
// is additive and non-blocking: a nil or failed store records nothing and
// changes no lifecycle behavior.
import (
"encoding/json"
"os"
"github.com/rcourtman/pulse-go-rewrite/internal/alerts/eventlog"
"github.com/rs/zerolog/log"
)
// EnableEventLog opens the persistent alert event log under the manager's
// alerts directory. The monitoring bootstrap calls this once per manager;
// without it (tests, ephemeral managers) no events are recorded.
func (m *Manager) EnableEventLog() {
if m == nil {
return
}
if err := os.MkdirAll(m.alertsDir, 0700); err != nil {
log.Warn().Err(err).Str("dir", m.alertsDir).Msg("alert event log disabled: cannot create alerts directory")
return
}
store, err := eventlog.Open(m.alertsDir)
if err != nil {
log.Warn().Err(err).Msg("alert event log disabled")
return
}
m.SetEventLog(store)
m.eventHistoryAuthoritative.Store(m.importLegacyHistoryIntoEventLog(store))
m.activeStateAuthoritative.Store(m.bootstrapActiveState(store))
}
// SetEventLog installs an event log store. Passing nil disables recording.
// The previous store, if any, is closed.
func (m *Manager) SetEventLog(store *eventlog.Store) {
if m == nil {
return
}
previous := m.eventLog.Swap(store)
m.historyProjectionMu.Lock()
m.historyProjection = nil
m.historyProjectionMu.Unlock()
authoritative := store != nil && m.historyManager != nil &&
!m.historyManager.StorageFileExists() &&
!m.historyManager.ImportedStorageFileExists() &&
m.historyManager.StorageLoadError() == nil
m.eventHistoryAuthoritative.Store(authoritative)
if previous != nil && previous != store {
previous.Close()
}
}
func (m *Manager) eventLogStore() *eventlog.Store {
if m == nil {
return nil
}
return m.eventLog.Load()
}
// AlertEvents returns matching events from the persistent event log, newest
// first. It flushes buffered appends first so a caller reading right after a
// transition observes it.
func (m *Manager) AlertEvents(filter eventlog.Filter) ([]eventlog.Event, error) {
store := m.eventLogStore()
if store == nil {
return nil, nil
}
if err := store.Flush(); err != nil {
return nil, err
}
return store.Query(filter)
}
// ReplayLifecycleEvents visits durable lifecycle transitions and migrated
// history snapshots with ids above afterID, oldest occurrence first. It is the
// projection-repair seam for consumers such as incident and resource
// timelines: delivery callbacks are deliberately not involved. Migrated
// snapshots carry a final occurrence state rather than a transition; consumers
// must expand only the facts present in that snapshot. afterID is the
// consumer's projection watermark (zero replays the full log); the visitor
// receives each event's durable id so it can advance that watermark once the
// event's projections are applied.
func (m *Manager) ReplayLifecycleEvents(afterID int64, visit func(eventID int64, event LifecycleEvent) error) error {
if m == nil || visit == nil {
return nil
}
store := m.eventLogStore()
if store == nil {
return nil
}
return store.WalkOldest(eventlog.Filter{AfterID: afterID, Types: []string{
eventlog.TypeFired,
eventlog.TypeRefired,
eventlog.TypeResolved,
eventlog.TypeAcknowledged,
eventlog.TypeUnacknowledged,
eventlog.TypeSnoozed,
eventlog.TypeUnsnoozed,
eventlog.TypeHistoryImported,
}}, func(event eventlog.Event) error {
if len(event.Snapshot) == 0 {
return nil
}
var snapshot Alert
if err := json.Unmarshal(event.Snapshot, &snapshot); err != nil {
log.Warn().Err(err).
Int64("eventID", event.ID).
Str("alertID", event.AlertID).
Msg("skipping invalid alert lifecycle snapshot during projection replay")
return nil
}
return visit(event.ID, LifecycleEvent{
Type: event.Type,
OccurredAt: event.OccurredAt,
Alert: &snapshot,
Details: cloneStringMap(event.Details),
Persisted: true,
})
})
}
// LifecycleProjectionWatermark returns the durable replay watermark for a
// named projection consumer, or zero when no event log is enabled.
func (m *Manager) LifecycleProjectionWatermark(name string) int64 {
if m == nil {
return 0
}
return m.eventLogStore().ProjectionWatermark(name)
}
// StoreLifecycleProjectionWatermark durably advances (or resets) the replay
// watermark for a named projection consumer. Failures are logged, not fatal:
// a stale watermark only means the next replay revisits already-idempotent
// projections.
func (m *Manager) StoreLifecycleProjectionWatermark(name string, eventID int64) {
if m == nil {
return
}
if err := m.eventLogStore().SetProjectionWatermark(name, eventID); err != nil {
log.Warn().Err(err).
Str("consumer", name).
Int64("eventID", eventID).
Msg("failed to persist lifecycle projection watermark")
}
}
// recordAlertEvent emits one canonical lifecycle transition and appends it to
// the durable event log. alertID is the identity fallback for callers whose
// *Alert may be nil (some resolve paths); when the alert is present its
// exported ID wins. Lifecycle subscribers run even when the diagnostic event
// log is unavailable, so incident history never depends on notification or
// event-log availability.
func (m *Manager) recordAlertEvent(eventType string, alert *Alert, alertID, reason, message string, details map[string]string) {
event := eventlog.Event{
OccurredAt: m.policyNow(),
Type: eventType,
AlertID: alertID,
Reason: reason,
Message: message,
Details: details,
}
if alert != nil {
if id := exportedAlertID(alert, alertID); id != "" {
event.AlertID = id
}
event.ResourceID = alert.ResourceID
event.ResourceName = alert.ResourceName
event.AlertType = alert.Type
event.Level = string(alert.Level)
if eventCarriesAlertSnapshot(eventType) {
if snapshot, err := json.Marshal(cloneAlertForOutput(alert)); err == nil {
event.Snapshot = snapshot
}
}
}
if event.AlertID == "" {
return
}
if eventCarriesAlertSnapshot(eventType) {
if eventType == eventlog.TypeResolved {
m.removeActiveRecoveryAlert(alert, event.AlertID)
} else if alert != nil {
m.setActiveRecoveryAlert(alert, event.AlertID)
}
}
persisted := false
if store := m.eventLogStore(); store != nil {
if eventCarriesAlertSnapshot(eventType) {
if err := store.AppendDurable(event); err != nil {
m.eventHistoryAuthoritative.Store(false)
m.activeStateFailureEpoch.Add(1)
if mirrorErr := m.checkpointActiveRecoveryAfterDurableFailure(err); mirrorErr != nil {
log.Error().Err(mirrorErr).
Str("alertID", event.AlertID).
Str("eventType", eventType).
Msg("failed to checkpoint active alert recovery after lifecycle write failure")
}
log.Error().Err(err).
Str("alertID", event.AlertID).
Str("eventType", eventType).
Msg("durable alert lifecycle event write failed")
} else {
persisted = true
}
} else {
store.Append(event)
}
}
if eventCarriesAlertSnapshot(eventType) && alert != nil {
lifecycleEvent := LifecycleEvent{
Type: eventType,
OccurredAt: event.OccurredAt,
Alert: cloneAlertForOutput(alert),
Details: cloneStringMap(details),
Persisted: persisted,
}
for _, callback := range m.getLifecycleCallbacks() {
func(cb func(LifecycleEvent)) {
defer func() {
if recovered := recover(); recovered != nil {
log.Error().
Interface("panic", recovered).
Str("alertID", event.AlertID).
Str("eventType", eventType).
Msg("panic in alert lifecycle callback")
}
}()
cb(lifecycleEvent)
}(callback)
}
}
}
func cloneStringMap(input map[string]string) map[string]string {
if len(input) == 0 {
return nil
}
output := make(map[string]string, len(input))
for key, value := range input {
output[key] = value
}
return output
}
// eventCarriesAlertSnapshot reports whether an event type records the full
// alert state alongside the transition. Lifecycle transitions do — they are
// what alert history is projected from — while high-frequency notification
// decisions stay lean.
func eventCarriesAlertSnapshot(eventType string) bool {
switch eventType {
case eventlog.TypeFired, eventlog.TypeRefired, eventlog.TypeResolved,
eventlog.TypeAcknowledged, eventlog.TypeUnacknowledged, eventlog.TypeSnoozed,
eventlog.TypeUnsnoozed, eventlog.TypeEscalated:
return true
default:
return false
}
}