Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
package alerts
import (
"encoding/json"
"errors"
"fmt"
"os"
"path/filepath"
"sync"
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/operationaltrust"
"github.com/rcourtman/pulse-go-rewrite/internal/securityutil"
"github.com/rcourtman/pulse-go-rewrite/internal/utils"
"github.com/rs/zerolog/log"
)
const (
// MaxHistoryDays is the maximum number of days to keep alert history
MaxHistoryDays = 30
// HistoryFileName is the name of the history file
HistoryFileName = "alert-history.json"
// HistoryBackupFileName is the name of the backup history file
HistoryBackupFileName = "alert-history.backup.json"
)
// HistoryEntry represents a historical alert entry
type HistoryEntry struct {
Alert Alert `json:"alert"`
Timestamp time.Time `json:"timestamp"`
}
// HistoryStats represents typed alert history statistics.
type HistoryStats struct {
TotalEntries int
OldestEntry time.Time
NewestEntry time.Time
DataDir string
FileSize int64
}
// AlertCallback is called when an alert is added to history
// This enables external systems to track alerts (e.g., pattern detection)
type AlertCallback func(alert Alert)
// HistoryManager manages persistent alert history
type HistoryManager struct {
mu sync.RWMutex
saveMu sync.Mutex // Serializes disk writes to prevent save race condition
stopOnce sync.Once
dataDir string
historyFile string
backupFile string
history []HistoryEntry
saveInterval time.Duration
stopChan chan struct{}
saveTicker *time.Ticker
workerWG sync.WaitGroup
callbacks []AlertCallback // Called when alerts are added
// storageRetired marks the JSON files migrated into the event log:
// in-memory history continues, disk writes stop.
storageRetired bool
// storageLoadErr keeps an unreadable or malformed legacy source from being
// overwritten or retired as an empty history before an operator can recover
// it. It is immutable after construction.
storageLoadErr error
}
func historyIdentityKey(alert *Alert) string {
if alert == nil {
return ""
}
backfillCanonicalIdentity(alert)
if alert.CanonicalState != "" {
return alert.CanonicalState
}
return alert.ID
}
func resolveHistoryStoragePaths(dataDir string) (resolvedDataDir string, historyFile string, backupFile string, err error) {
resolvedDataDir, err = securityutil.NormalizeStorageDir(dataDir)
if err != nil {
return "", "", "", fmt.Errorf("resolve history data directory: %w", err)
}
historyFile, err = securityutil.JoinStorageLeaf(resolvedDataDir, HistoryFileName)
if err != nil {
return "", "", "", fmt.Errorf("resolve history file path: %w", err)
}
backupFile, err = securityutil.JoinStorageLeaf(resolvedDataDir, HistoryBackupFileName)
if err != nil {
return "", "", "", fmt.Errorf("resolve history backup file path: %w", err)
}
return resolvedDataDir, historyFile, backupFile, nil
}
// NewHistoryManager creates a new history manager
func NewHistoryManager(dataDir string) *HistoryManager {
resolvedDataDir, historyFile, backupFile, err := resolveHistoryStoragePaths(utils.ResolveDataDir(dataDir))
if err != nil {
panic(fmt.Sprintf("invalid alert history storage paths for %q: %v", dataDir, err))
}
hm := &HistoryManager{
dataDir: resolvedDataDir,
historyFile: historyFile,
backupFile: backupFile,
history: make([]HistoryEntry, 0),
saveInterval: 5 * time.Minute,
stopChan: make(chan struct{}),
}
// Ensure data directory exists
if err := os.MkdirAll(resolvedDataDir, alertsDirPerm); err != nil {
log.Error().Err(err).Str("dir", resolvedDataDir).Msg("Failed to create data directory")
} else if err := os.Chmod(resolvedDataDir, alertsDirPerm); err != nil {
log.Warn().Err(err).Str("dir", resolvedDataDir).Msg("Failed to harden history directory permissions")
}
// Load existing history
hm.storageLoadErr = hm.loadHistory()
if hm.storageLoadErr != nil {
log.Error().Err(hm.storageLoadErr).Msg("failed to load alert history")
}
// Start periodic save routine
hm.startPeriodicSave()
// Start cleanup routine
hm.workerWG.Add(1)
go func() {
defer hm.workerWG.Done()
hm.cleanupRoutine()
}()
return hm
}
// OnAlert registers a callback to be called when alerts are added
func (hm *HistoryManager) OnAlert(cb AlertCallback) {
hm.mu.Lock()
defer hm.mu.Unlock()
hm.callbacks = append(hm.callbacks, cb)
}
// AddAlert records a newly observed alert occurrence. Repeated attempts to add
// the same still-open occurrence are folded into its existing history row.
// This makes the history write path enforce the same incident semantics as the
// runtime alert lifecycle instead of relying on the daily cleanup pass.
func (hm *HistoryManager) AddAlert(alert Alert) {
hm.addAlert(alert, false)
}
// AddAlertTransition records an explicit transition snapshot even when the
// alert occurrence is still open. Callers should use this only when the extra
// row is intentional, such as a configured severity-change history event.
func (hm *HistoryManager) AddAlertTransition(alert Alert) {
hm.addAlert(alert, true)
}
func (hm *HistoryManager) addAlert(alert Alert, forceAppend bool) {
hm.mu.Lock()
entry := HistoryEntry{
Alert: *alert.Clone(),
Timestamp: time.Now(),
}
if !forceAppend {
for i := len(hm.history) - 1; i >= 0; i-- {
if historyIdentityKey(&hm.history[i].Alert) != historyIdentityKey(&entry.Alert) {
continue
}
if sameHistoryIncident(hm.history[i], entry) {
hm.history[i].Alert = mergeHistoryAlertSnapshots(hm.history[i].Alert, entry.Alert)
hm.mu.Unlock()
log.Debug().Str("alertID", alert.ID).Msg("updated existing alert history occurrence")
return
}
break
}
}
hm.history = append(hm.history, entry)
callbacks := append([]AlertCallback(nil), hm.callbacks...)
hm.mu.Unlock()
log.Debug().Str("alertID", alert.ID).Msg("added alert to history")
// Call callbacks outside the lock
for _, cb := range callbacks {
cb(alert)
}
}
func sameHistoryIncident(existing, incoming HistoryEntry) bool {
existingKey := historyIdentityKey(&existing.Alert)
if existingKey == "" || existingKey != historyIdentityKey(&incoming.Alert) {
return false
}
if !existing.Alert.StartTime.IsZero() &&
!incoming.Alert.StartTime.IsZero() &&
existing.Alert.StartTime.Equal(incoming.Alert.StartTime) {
return true
}
existingResolved := existing.Alert.OperationalRecord != nil &&
existing.Alert.OperationalRecord.State == operationaltrust.OperationalResolved
incomingResolved := incoming.Alert.OperationalRecord != nil &&
incoming.Alert.OperationalRecord.State == operationaltrust.OperationalResolved
if existingResolved || incomingResolved {
return false
}
// Two open records do not by themselves make one incident: an operational
// record can survive across occurrences (or be stranded open), and treating
// that as identity merged every recurrence into its first row forever. The
// observation-gap window below decides instead.
lastObservation := existing.Timestamp
if existing.Alert.LastSeen.After(lastObservation) {
lastObservation = existing.Alert.LastSeen
}
gap := incoming.Timestamp.Sub(lastObservation)
return gap >= 0 && gap <= historyDedupWindow
}
func mergeHistoryAlertSnapshots(existing, incoming Alert) Alert {
merged := *incoming.Clone()
if !existing.StartTime.IsZero() &&
(merged.StartTime.IsZero() || existing.StartTime.Before(merged.StartTime)) {
merged.StartTime = existing.StartTime
}
if existing.LastSeen.After(merged.LastSeen) {
merged.LastSeen = existing.LastSeen
}
if existing.Acknowledged && !merged.Acknowledged {
merged.Acknowledged = true
merged.AckTime = existing.AckTime
merged.AckUser = existing.AckUser
}
return merged
}
// SnapshotEntries returns a copy of every history entry, oldest first, for
// the one-time migration into the event log.
func (hm *HistoryManager) SnapshotEntries() []HistoryEntry {
hm.mu.RLock()
defer hm.mu.RUnlock()
entries := make([]HistoryEntry, 0, len(hm.history))
for _, entry := range hm.history {
entries = append(entries, HistoryEntry{Alert: *entry.Alert.Clone(), Timestamp: entry.Timestamp})
}
return entries
}
// RetireStorage renames the JSON history files out of the load path after a
// successful migration into the event log, and stops further disk writes.
// The in-memory entries stay available as the fallback read model for the
// rest of the process lifetime.
func (hm *HistoryManager) RetireStorage() error {
hm.saveMu.Lock()
defer hm.saveMu.Unlock()
hm.mu.Lock()
hm.storageRetired = true
hm.mu.Unlock()
for _, path := range []string{hm.historyFile, hm.backupFile} {
if path == "" {
continue
}
if _, err := os.Stat(path); err != nil {
continue
}
if err := os.Rename(path, path+".imported"); err != nil {
return fmt.Errorf("retire history file %s: %w", path, err)
}
}
return nil
}
// StorageFileExists reports whether either JSON history source is still
// present. The backup can be the only surviving retry source after an
// interrupted save or partial retirement, so it must keep migration active
// until both fixed legacy leaves have retired.
func (hm *HistoryManager) StorageFileExists() bool {
for _, path := range []string{hm.historyFile, hm.backupFile} {
if path == "" {
continue
}
if _, err := os.Stat(path); err == nil {
return true
}
}
return false
}
// ImportedStorageFileExists reports whether a retired JSON source remains as
// a recovery copy. It is replayed idempotently if events.db is recreated and
// remains the read fallback when the event store is unavailable.
func (hm *HistoryManager) ImportedStorageFileExists() bool {
for _, path := range []string{hm.historyFile + ".imported", hm.backupFile + ".imported"} {
if _, err := os.Stat(path); err == nil {
return true
}
}
return false
}
// StorageLoadError reports why a present legacy source could not be loaded.
// Migration and periodic persistence must leave that source untouched.
func (hm *HistoryManager) StorageLoadError() error {
if hm == nil {
return nil
}
return hm.storageLoadErr
}
// UpdateAlertLastSeen updates the LastSeen timestamp on the most recent
// history entry matching the given alert ID. This is called when an alert is
// resolved so that the stored history reflects the true duration of the alert,
// not just the snapshot captured at creation time.
func (hm *HistoryManager) UpdateAlertLastSeen(alertID string, lastSeen time.Time) {
hm.UpdateAlertLastSeenForAlert(&Alert{ID: alertID}, lastSeen)
}
// UpdateAlertLastSeenForAlert updates the LastSeen timestamp on the most recent
// history entry matching the given alert identity. Canonical alerts match by
// canonical state first, then fall back to legacy alert ID for compatibility.
func (hm *HistoryManager) UpdateAlertLastSeenForAlert(alert *Alert, lastSeen time.Time) {
if alert == nil {
return
}
matchKey := historyIdentityKey(alert)
matchID := alert.ID
hm.mu.Lock()
defer hm.mu.Unlock()
// Iterate from newest to oldest to find the most recent entry for this alert.
for i := len(hm.history) - 1; i >= 0; i-- {
entry := hm.history[i].Alert.Clone()
entryKey := historyIdentityKey(entry)
if matchKey != "" && entryKey == matchKey {
hm.history[i].Alert.LastSeen = lastSeen
return
}
if matchID != "" && hm.history[i].Alert.ID == matchID {
hm.history[i].Alert.LastSeen = lastSeen
return
}
}
}
// UpdateAlertOperationalContractForAlert updates the most recent history
// entry with the durable evidence and lifecycle state from the active alert.
// This keeps incident history and notification linkage on the same record and
// transition identities as the runtime alert.
func (hm *HistoryManager) UpdateAlertOperationalContractForAlert(alert *Alert) {
if alert == nil {
return
}
snapshot := alert.Clone()
if snapshot == nil {
return
}
matchKey := historyIdentityKey(snapshot)
matchID := snapshot.ID
hm.mu.Lock()
defer hm.mu.Unlock()
for i := len(hm.history) - 1; i >= 0; i-- {
entry := hm.history[i].Alert.Clone()
entryKey := historyIdentityKey(entry)
if (matchKey != "" && entryKey == matchKey) ||
(matchID != "" && hm.history[i].Alert.ID == matchID) {
// A resolved row is this occurrence's final state. A later
// occurrence of the same identity (different StartTime) gets its
// own row from AddAlert, so its record must not overwrite the
// resolved one here.
entryResolved := entry.OperationalRecord != nil &&
entry.OperationalRecord.State == operationaltrust.OperationalResolved
snapshotResolved := snapshot.OperationalRecord != nil &&
snapshot.OperationalRecord.State == operationaltrust.OperationalResolved
if entryResolved && !snapshotResolved &&
!entry.StartTime.IsZero() && !snapshot.StartTime.IsZero() &&
!entry.StartTime.Equal(snapshot.StartTime) {
return
}
hm.history[i].Alert.OperationalRecord = snapshot.OperationalRecord
hm.history[i].Alert.LatestTransition = snapshot.LatestTransition
hm.history[i].Alert.Transitions = snapshot.Transitions
hm.history[i].Alert.Evidence = snapshot.Evidence
hm.history[i].Alert.Resolution = snapshot.Resolution
// The live alert is authoritative for acknowledgement; a row
// that keeps its fire-time ack state shows an acknowledged
// alert as unacknowledged for its whole active life.
hm.history[i].Alert.Acknowledged = snapshot.Acknowledged
hm.history[i].Alert.AckTime = snapshot.AckTime
hm.history[i].Alert.AckUser = snapshot.AckUser
return
}
}
}
// LatestAlertForAlert returns a clone of the newest history entry matching
// the alert's canonical identity, with legacy alert ID as the compatibility
// fallback.
func (hm *HistoryManager) LatestAlertForAlert(alert *Alert) *Alert {
if alert == nil {
return nil
}
matchKey := historyIdentityKey(alert)
matchID := alert.ID
hm.mu.RLock()
defer hm.mu.RUnlock()
for i := len(hm.history) - 1; i >= 0; i-- {
entry := hm.history[i].Alert.Clone()
entryKey := historyIdentityKey(entry)
if (matchKey != "" && entryKey == matchKey) ||
(matchID != "" && hm.history[i].Alert.ID == matchID) {
return entry
}
}
return nil
}
// MigrateActiveAlert updates the most recent history entry for an in-flight
// alert when its canonical runtime identity changes, such as a guest metric
// alert moving from one node-scoped resource key to another after a VM move.
func (hm *HistoryManager) MigrateActiveAlert(oldTrackingKey string, updated Alert) {
if oldTrackingKey == "" {
return
}
updatedAlert := updated.Clone()
if updatedAlert == nil {
return
}
backfillCanonicalIdentity(updatedAlert)
hm.mu.Lock()
defer hm.mu.Unlock()
for i := len(hm.history) - 1; i >= 0; i-- {
entry := hm.history[i].Alert.Clone()
if historyIdentityKey(entry) == oldTrackingKey || hm.history[i].Alert.ID == oldTrackingKey {
hm.history[i].Alert = *updatedAlert
return
}
}
}
// GetHistory returns alert history within the specified time range
func (hm *HistoryManager) GetHistory(since time.Time, limit int) []Alert {
hm.mu.RLock()
defer hm.mu.RUnlock()
var results []Alert
count := 0
// Iterate from newest to oldest
for i := len(hm.history) - 1; i >= 0 && (limit <= 0 || count < limit); i-- {
entry := hm.history[i]
if entry.Timestamp.After(since) {
results = append(results, entry.Alert)
count++
}
}
return results
}
// GetAllHistory returns all alert history (up to limit)
func (hm *HistoryManager) GetAllHistory(limit int) []Alert {
hm.mu.RLock()
defer hm.mu.RUnlock()
if limit <= 0 || limit > len(hm.history) {
limit = len(hm.history)
}
results := make([]Alert, 0, limit)
start := len(hm.history) - limit
for i := len(hm.history) - 1; i >= start; i-- {
results = append(results, hm.history[i].Alert)
}
return results
}
// loadHistory loads history from disk
func (hm *HistoryManager) loadHistory() error {
// Try loading from main file first
data, mainErr := readLimitedRegularFile(hm.historyFile, maxAlertHistoryFileSizeBytes)
if mainErr != nil {
if !os.IsNotExist(mainErr) {
log.Warn().Err(mainErr).Str("file", hm.historyFile).Msg("Failed to read history file")
}
// Try backup file
var backupErr error
loadedRetired := false
data, backupErr = readLimitedRegularFile(hm.backupFile, maxAlertHistoryFileSizeBytes)
if backupErr != nil {
if os.IsNotExist(backupErr) && os.IsNotExist(mainErr) {
// Retired JSON remains a recovery source if events.db is lost or
// unavailable. Prefer the primary retired leaf, then its backup.
var importedErr error
data, importedErr = readLimitedRegularFile(hm.historyFile+".imported", maxAlertHistoryFileSizeBytes)
if importedErr != nil {
data, importedErr = readLimitedRegularFile(hm.backupFile+".imported", maxAlertHistoryFileSizeBytes)
}
if importedErr != nil {
if os.IsNotExist(importedErr) {
log.Debug().Msg("No alert history files found, starting fresh")
return nil
}
return fmt.Errorf("failed to read retired alert history recovery source: %w", importedErr)
}
hm.storageRetired = true
loadedRetired = true
log.Info().Msg("loaded retired alert history recovery source")
}
if !loadedRetired && os.IsPermission(backupErr) {
log.Warn().
Err(backupErr).
Str("file", hm.backupFile).
Msg("Permission denied reading backup history file - check file ownership")
}
if loadedRetired {
backupErr = nil
} else if !os.IsNotExist(mainErr) {
return fmt.Errorf("failed to read history file %q (%v); failed to read history backup file %q: %w",
hm.historyFile, mainErr, hm.backupFile, backupErr)
} else {
return fmt.Errorf("failed to read history backup file %q: %w", hm.backupFile, backupErr)
}
}
if !loadedRetired {
log.Info().Msg("loaded alert history from backup file")
}
}
var history []HistoryEntry
if err := json.Unmarshal(data, &history); err != nil {
return fmt.Errorf("failed to unmarshal history: %w", err)
}
hm.history = history
log.Info().Int("count", len(history)).Msg("Loaded alert history")
// Clean old entries immediately
hm.cleanOldEntries()
hm.deduplicateHistory()
return nil
}
// saveHistory saves history to disk with retry logic
func (hm *HistoryManager) saveHistory() error {
return hm.saveHistoryWithRetry(3)
}
// saveHistoryWithRetry saves history with exponential backoff retry
func (hm *HistoryManager) saveHistoryWithRetry(maxRetries int) error {
hm.mu.RLock()
retired := hm.storageRetired
hm.mu.RUnlock()
if retired {
return nil
}
if hm.storageLoadErr != nil {
return fmt.Errorf("alert history storage remains read-only after load failure: %w", hm.storageLoadErr)
}
if maxRetries < 1 {
maxRetries = 1
}
// Serialize all disk writes to prevent concurrent saves from overwriting each other
hm.saveMu.Lock()
defer hm.saveMu.Unlock()
hm.mu.RLock()
snapshot := make([]HistoryEntry, len(hm.history))
copy(snapshot, hm.history)
hm.mu.RUnlock()
data, err := json.Marshal(snapshot)
if err != nil {
return fmt.Errorf("historyManager.saveHistoryWithRetry: marshal history snapshot: %w", err)
}
historyFile := hm.historyFile
backupFile := hm.backupFile
// Create backup of existing file once before any write attempts.
// This ensures we don't lose data if all retries fail.
backupCreated := false
if _, err := os.Stat(historyFile); err == nil {
if err := os.Remove(backupFile); err != nil && !os.IsNotExist(err) {
log.Warn().Err(err).Str("file", backupFile).Msg("Failed to remove existing backup file before save")
}
if err := os.Rename(historyFile, backupFile); err != nil {
log.Warn().
Err(err).
Str("source", historyFile).
Str("backup", backupFile).
Msg("Failed to create backup file")
} else {
backupCreated = true
}
}
var lastErr error
for attempt := 1; attempt <= maxRetries; attempt++ {
tempFile := fmt.Sprintf("%s.tmp-%d-%d", historyFile, os.Getpid(), time.Now().UnixNano())
if err := writeFileSynced(tempFile, data, 0644); err != nil {
lastErr = fmt.Errorf("failed to write temp history file: %w", err)
_ = os.Remove(tempFile)
log.Warn().
Err(lastErr).
Int("attempt", attempt).
Int("maxRetries", maxRetries).
Msg("failed to write history file, will retry")
// Exponential backoff: 100ms, 200ms, 400ms
if attempt < maxRetries {
backoff := time.Duration(100*(1<<uint(attempt-1))) * time.Millisecond
time.Sleep(backoff)
}
continue
}
if err := os.Rename(tempFile, historyFile); err != nil {
lastErr = fmt.Errorf("failed to replace history file with temp file: %w", err)
_ = os.Remove(tempFile)
log.Warn().
Err(lastErr).
Int("attempt", attempt).
Int("maxRetries", maxRetries).
Msg("Failed to write history file, will retry")
// Exponential backoff: 100ms, 200ms, 400ms
if attempt < maxRetries {
backoff := time.Duration(100*(1<<uint(attempt-1))) * time.Millisecond
time.Sleep(backoff)
}
continue
}
if err := os.Chmod(historyFile, alertsFilePerm); err != nil {
lastErr = err
log.Warn().
Err(err).
Int("attempt", attempt).
Int("maxRetries", maxRetries).
Msg("Failed to harden history file permissions, will retry")
if attempt < maxRetries {
backoff := time.Duration(100*(1<<uint(attempt-1))) * time.Millisecond
time.Sleep(backoff)
}
continue
}
if err := syncDir(filepath.Dir(historyFile)); err != nil {
log.Warn().Err(err).Str("dir", filepath.Dir(historyFile)).Msg("Failed to sync history directory")
}
// Success - remove backup file now that we've successfully written
if backupCreated {
if err := os.Remove(backupFile); err != nil && !os.IsNotExist(err) {
log.Warn().Err(err).Str("file", backupFile).Msg("failed to remove backup file after successful save")
}
}
log.Debug().Int("entries", len(snapshot)).Msg("saved alert history")
return nil
}
if lastErr == nil {
lastErr = fmt.Errorf("historyManager.saveHistoryWithRetry: no write attempts executed for %s", historyFile)
}
// All retries failed - restore backup if we have one
if backupCreated {
if restoreErr := os.Rename(backupFile, historyFile); restoreErr != nil {
restoreErr = fmt.Errorf("restore backup file %s to %s: %w", backupFile, historyFile, restoreErr)
log.Error().Err(restoreErr).Msg("failed to restore backup after all write attempts failed")
} else {
if err := syncDir(filepath.Dir(historyFile)); err != nil {
log.Warn().Err(err).Str("dir", filepath.Dir(historyFile)).Msg("Failed to sync history directory after restore")
}
log.Info().Msg("restored backup after history save failure")
}
log.Info().Msg("Restored backup after history save failure")
}
return fmt.Errorf("historyManager.saveHistoryWithRetry: failed to write history file after %d attempts: %w", maxRetries, lastErr)
}
func writeFileSynced(path string, data []byte, perm os.FileMode) error {
file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, perm)
if err != nil {
return err
}
if _, err := file.Write(data); err != nil {
_ = file.Close()
return err
}
if err := file.Sync(); err != nil {
_ = file.Close()
return err
}
return file.Close()
}
func syncDir(dirPath string) error {
dir, err := os.Open(dirPath)
if err != nil {
return err
}
defer dir.Close()
return dir.Sync()
}
// startPeriodicSave starts the periodic save routine
func (hm *HistoryManager) startPeriodicSave() {
hm.saveTicker = time.NewTicker(hm.saveInterval)
hm.workerWG.Add(1)
go func() {
defer hm.workerWG.Done()
for {
select {
case <-hm.saveTicker.C:
if err := hm.saveHistory(); err != nil {
log.Error().Err(err).Msg("failed to save alert history")
}
case <-hm.stopChan:
return
}
}
}()
}
// cleanupRoutine runs periodically to clean old entries
func (hm *HistoryManager) cleanupRoutine() {
// Run cleanup daily
ticker := time.NewTicker(24 * time.Hour)
defer ticker.Stop()
// Also run cleanup on startup after a delay
startupDelay := time.NewTimer(1 * time.Minute)
defer func() {
if !startupDelay.Stop() {
select {
case <-startupDelay.C:
default:
}
}
}()
select {
case <-startupDelay.C:
hm.cleanOldEntries()
hm.deduplicateHistory()
case <-hm.stopChan:
return
}
for {
select {
case <-ticker.C:
hm.cleanOldEntries()
hm.deduplicateHistory()
case <-hm.stopChan:
return
}
}
}
// cleanOldEntries removes entries older than MaxHistoryDays
func (hm *HistoryManager) cleanOldEntries() {
hm.mu.Lock()
defer hm.mu.Unlock()
cutoff := time.Now().AddDate(0, 0, -MaxHistoryDays)
removed := 0
for _, entry := range hm.history {
if !entry.Timestamp.After(cutoff) {
removed++
}
}
if removed == 0 {
return
}
// Allocate for survivors, not the loaded file's original row count. A
// mostly-expired history otherwise keeps its old-sized Alert array after
// startup cleanup, even though no expired rows remain visible.
newHistory := make([]HistoryEntry, 0, len(hm.history)-removed)
for _, entry := range hm.history {
if entry.Timestamp.After(cutoff) {
newHistory = append(newHistory, entry)
}
}
hm.history = newHistory
log.Info().
Int("removed", removed).
Int("remaining", len(newHistory)).
Msg("cleaned old alert history entries")
}
// historyDedupWindow is the maximum gap between consecutive same-alert entries
// for them to be considered duplicates of one continuous incident.
const historyDedupWindow = 5 * time.Minute
// deduplicateHistory collapses consecutive entries for the same alert identity
// where the gap between entries is shorter than the dedup window. This removes
// flapping/re-fire noise where a bug or polling pattern caused the same alert
// to be recorded many times in quick succession. The surviving entry keeps the
// earliest timestamp and the latest LastSeen.
func (hm *HistoryManager) deduplicateHistory() {
hm.mu.Lock()
defer hm.mu.Unlock()
if len(hm.history) < 2 {
return
}
lastTimeByKey := make(map[string]time.Time)
removed := 0
for _, entry := range hm.history {
key := historyIdentityKey(&entry.Alert)
if lastTime, ok := lastTimeByKey[key]; ok && entry.Timestamp.Sub(lastTime) <= historyDedupWindow {
removed++
}
lastTimeByKey[key] = entry.Timestamp
}
if removed == 0 {
return
}
// Count first so the new backing array holds only surviving occurrences.
// Clipping a preallocated slice's capacity would keep its original array;
// copying twice would instead add another full array on sparse reductions.
deduped := make([]HistoryEntry, 0, len(hm.history)-removed)
lastIdxByKey := make(map[string]int)
clear(lastTimeByKey)
for _, entry := range hm.history {
key := historyIdentityKey(&entry.Alert)
if lastTime, ok := lastTimeByKey[key]; ok {
if entry.Timestamp.Sub(lastTime) <= historyDedupWindow {
idx := lastIdxByKey[key]
deduped[idx].Alert = mergeHistoryAlertSnapshots(deduped[idx].Alert, entry.Alert)
lastTimeByKey[key] = entry.Timestamp
continue
}
}
deduped = append(deduped, entry)
lastIdxByKey[key] = len(deduped) - 1
lastTimeByKey[key] = entry.Timestamp
}
hm.history = deduped
log.Info().
Int("removed", removed).
Int("remaining", len(deduped)).
Msg("deduplicated alert history entries")
}
// RemoveAlert removes a specific alert from history by ID
func (hm *HistoryManager) RemoveAlert(alertID string) {
hm.mu.Lock()
defer hm.mu.Unlock()
removed := 0
for _, entry := range hm.history {
if entry.Alert.ID == alertID {
removed++
}
}
if removed == 0 {
return
}
newHistory := make([]HistoryEntry, 0, len(hm.history)-removed)
for _, entry := range hm.history {
if entry.Alert.ID != alertID {
newHistory = append(newHistory, entry)
}
}
hm.history = newHistory
log.Debug().Str("alertID", alertID).Msg("removed alert from history")
}
// ClearAllHistory clears all alert history
func (hm *HistoryManager) ClearAllHistory() error {
hm.mu.Lock()
defer hm.mu.Unlock()
// Clear the in-memory history
hm.history = make([]HistoryEntry, 0)
// Remove the history files
var removeErrs []error
if err := os.Remove(hm.historyFile); err != nil && !os.IsNotExist(err) {
removeErrs = append(removeErrs, fmt.Errorf("remove history file %s: %w", hm.historyFile, err))
}
if err := os.Remove(hm.backupFile); err != nil && !os.IsNotExist(err) {
removeErrs = append(removeErrs, fmt.Errorf("remove backup file %s: %w", hm.backupFile, err))
}
if len(removeErrs) > 0 {
return fmt.Errorf("clear alert history files: %w", errors.Join(removeErrs...))
}
log.Info().Msg("cleared all alert history")
return nil
}
// Stop stops the history manager
func (hm *HistoryManager) Stop() {
hm.stopOnce.Do(func() {
close(hm.stopChan)
if hm.saveTicker != nil {
hm.saveTicker.Stop()
}
hm.workerWG.Wait()
// Save one final time
if err := hm.saveHistory(); err != nil {
log.Error().Err(err).Msg("Failed to save alert history on shutdown")
}
})
}
// GetStats returns statistics about the alert history
func (hm *HistoryManager) GetStats() map[string]any {
stats := hm.Stats()
return map[string]any{
"totalEntries": stats.TotalEntries,
"oldestEntry": stats.OldestEntry,
"newestEntry": stats.NewestEntry,
"dataDir": stats.DataDir,
"fileSize": stats.FileSize,
}
}
// Stats returns typed statistics about the alert history.
func (hm *HistoryManager) Stats() HistoryStats {
hm.mu.RLock()
defer hm.mu.RUnlock()
oldest := time.Now()
newest := time.Time{}
if len(hm.history) > 0 {
oldest = hm.history[0].Timestamp
newest = hm.history[len(hm.history)-1].Timestamp
}
return HistoryStats{
TotalEntries: len(hm.history),
OldestEntry: oldest,
NewestEntry: newest,
DataDir: hm.dataDir,
FileSize: hm.getFileSize(),
}
}
// getFileSize returns the size of the history file
func (hm *HistoryManager) getFileSize() int64 {
info, err := os.Stat(hm.historyFile)
if err != nil {
return 0
}
return info.Size()
}