Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

29.3 KB
Raw Blame
package alerts

import (
	"encoding/json"
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"sync"
	"time"

	"github.com/rcourtman/pulse-go-rewrite/internal/operationaltrust"
	"github.com/rcourtman/pulse-go-rewrite/internal/securityutil"
	"github.com/rcourtman/pulse-go-rewrite/internal/utils"
	"github.com/rs/zerolog/log"
)

const (
	// MaxHistoryDays is the maximum number of days to keep alert history
	MaxHistoryDays = 30
	// HistoryFileName is the name of the history file
	HistoryFileName = "alert-history.json"
	// HistoryBackupFileName is the name of the backup history file
	HistoryBackupFileName = "alert-history.backup.json"
)

// HistoryEntry represents a historical alert entry
type HistoryEntry struct {
	Alert     Alert     `json:"alert"`
	Timestamp time.Time `json:"timestamp"`
}

// HistoryStats represents typed alert history statistics.
type HistoryStats struct {
	TotalEntries int
	OldestEntry  time.Time
	NewestEntry  time.Time
	DataDir      string
	FileSize     int64
}

// AlertCallback is called when an alert is added to history
// This enables external systems to track alerts (e.g., pattern detection)
type AlertCallback func(alert Alert)

// HistoryManager manages persistent alert history
type HistoryManager struct {
	mu           sync.RWMutex
	saveMu       sync.Mutex // Serializes disk writes to prevent save race condition
	stopOnce     sync.Once
	dataDir      string
	historyFile  string
	backupFile   string
	history      []HistoryEntry
	saveInterval time.Duration
	stopChan     chan struct{}
	saveTicker   *time.Ticker
	workerWG     sync.WaitGroup
	callbacks    []AlertCallback // Called when alerts are added
	// storageRetired marks the JSON files migrated into the event log:
	// in-memory history continues, disk writes stop.
	storageRetired bool
	// storageLoadErr keeps an unreadable or malformed legacy source from being
	// overwritten or retired as an empty history before an operator can recover
	// it. It is immutable after construction.
	storageLoadErr error
}

func historyIdentityKey(alert *Alert) string {
	if alert == nil {
		return ""
	}
	backfillCanonicalIdentity(alert)
	if alert.CanonicalState != "" {
		return alert.CanonicalState
	}
	return alert.ID
}

func resolveHistoryStoragePaths(dataDir string) (resolvedDataDir string, historyFile string, backupFile string, err error) {
	resolvedDataDir, err = securityutil.NormalizeStorageDir(dataDir)
	if err != nil {
		return "", "", "", fmt.Errorf("resolve history data directory: %w", err)
	}

	historyFile, err = securityutil.JoinStorageLeaf(resolvedDataDir, HistoryFileName)
	if err != nil {
		return "", "", "", fmt.Errorf("resolve history file path: %w", err)
	}
	backupFile, err = securityutil.JoinStorageLeaf(resolvedDataDir, HistoryBackupFileName)
	if err != nil {
		return "", "", "", fmt.Errorf("resolve history backup file path: %w", err)
	}

	return resolvedDataDir, historyFile, backupFile, nil
}

// NewHistoryManager creates a new history manager
func NewHistoryManager(dataDir string) *HistoryManager {
	resolvedDataDir, historyFile, backupFile, err := resolveHistoryStoragePaths(utils.ResolveDataDir(dataDir))
	if err != nil {
		panic(fmt.Sprintf("invalid alert history storage paths for %q: %v", dataDir, err))
	}

	hm := &HistoryManager{
		dataDir:      resolvedDataDir,
		historyFile:  historyFile,
		backupFile:   backupFile,
		history:      make([]HistoryEntry, 0),
		saveInterval: 5 * time.Minute,
		stopChan:     make(chan struct{}),
	}

	// Ensure data directory exists
	if err := os.MkdirAll(resolvedDataDir, alertsDirPerm); err != nil {
		log.Error().Err(err).Str("dir", resolvedDataDir).Msg("Failed to create data directory")
	} else if err := os.Chmod(resolvedDataDir, alertsDirPerm); err != nil {
		log.Warn().Err(err).Str("dir", resolvedDataDir).Msg("Failed to harden history directory permissions")
	}

	// Load existing history
	hm.storageLoadErr = hm.loadHistory()
	if hm.storageLoadErr != nil {
		log.Error().Err(hm.storageLoadErr).Msg("failed to load alert history")
	}

	// Start periodic save routine
	hm.startPeriodicSave()

	// Start cleanup routine
	hm.workerWG.Add(1)
	go func() {
		defer hm.workerWG.Done()
		hm.cleanupRoutine()
	}()

	return hm
}

// OnAlert registers a callback to be called when alerts are added
func (hm *HistoryManager) OnAlert(cb AlertCallback) {
	hm.mu.Lock()
	defer hm.mu.Unlock()
	hm.callbacks = append(hm.callbacks, cb)
}

// AddAlert records a newly observed alert occurrence. Repeated attempts to add
// the same still-open occurrence are folded into its existing history row.
// This makes the history write path enforce the same incident semantics as the
// runtime alert lifecycle instead of relying on the daily cleanup pass.
func (hm *HistoryManager) AddAlert(alert Alert) {
	hm.addAlert(alert, false)
}

// AddAlertTransition records an explicit transition snapshot even when the
// alert occurrence is still open. Callers should use this only when the extra
// row is intentional, such as a configured severity-change history event.
func (hm *HistoryManager) AddAlertTransition(alert Alert) {
	hm.addAlert(alert, true)
}

func (hm *HistoryManager) addAlert(alert Alert, forceAppend bool) {
	hm.mu.Lock()

	entry := HistoryEntry{
		Alert:     *alert.Clone(),
		Timestamp: time.Now(),
	}

	if !forceAppend {
		for i := len(hm.history) - 1; i >= 0; i-- {
			if historyIdentityKey(&hm.history[i].Alert) != historyIdentityKey(&entry.Alert) {
				continue
			}
			if sameHistoryIncident(hm.history[i], entry) {
				hm.history[i].Alert = mergeHistoryAlertSnapshots(hm.history[i].Alert, entry.Alert)
				hm.mu.Unlock()
				log.Debug().Str("alertID", alert.ID).Msg("updated existing alert history occurrence")
				return
			}
			break
		}
	}

	hm.history = append(hm.history, entry)
	callbacks := append([]AlertCallback(nil), hm.callbacks...)
	hm.mu.Unlock()

	log.Debug().Str("alertID", alert.ID).Msg("added alert to history")

	// Call callbacks outside the lock
	for _, cb := range callbacks {
		cb(alert)
	}
}

func sameHistoryIncident(existing, incoming HistoryEntry) bool {
	existingKey := historyIdentityKey(&existing.Alert)
	if existingKey == "" || existingKey != historyIdentityKey(&incoming.Alert) {
		return false
	}

	if !existing.Alert.StartTime.IsZero() &&
		!incoming.Alert.StartTime.IsZero() &&
		existing.Alert.StartTime.Equal(incoming.Alert.StartTime) {
		return true
	}

	existingResolved := existing.Alert.OperationalRecord != nil &&
		existing.Alert.OperationalRecord.State == operationaltrust.OperationalResolved
	incomingResolved := incoming.Alert.OperationalRecord != nil &&
		incoming.Alert.OperationalRecord.State == operationaltrust.OperationalResolved
	if existingResolved || incomingResolved {
		return false
	}

	// Two open records do not by themselves make one incident: an operational
	// record can survive across occurrences (or be stranded open), and treating
	// that as identity merged every recurrence into its first row forever. The
	// observation-gap window below decides instead.
	lastObservation := existing.Timestamp
	if existing.Alert.LastSeen.After(lastObservation) {
		lastObservation = existing.Alert.LastSeen
	}
	gap := incoming.Timestamp.Sub(lastObservation)
	return gap >= 0 && gap <= historyDedupWindow
}

func mergeHistoryAlertSnapshots(existing, incoming Alert) Alert {
	merged := *incoming.Clone()
	if !existing.StartTime.IsZero() &&
		(merged.StartTime.IsZero() || existing.StartTime.Before(merged.StartTime)) {
		merged.StartTime = existing.StartTime
	}
	if existing.LastSeen.After(merged.LastSeen) {
		merged.LastSeen = existing.LastSeen
	}
	if existing.Acknowledged && !merged.Acknowledged {
		merged.Acknowledged = true
		merged.AckTime = existing.AckTime
		merged.AckUser = existing.AckUser
	}
	return merged
}

// SnapshotEntries returns a copy of every history entry, oldest first, for
// the one-time migration into the event log.
func (hm *HistoryManager) SnapshotEntries() []HistoryEntry {
	hm.mu.RLock()
	defer hm.mu.RUnlock()
	entries := make([]HistoryEntry, 0, len(hm.history))
	for _, entry := range hm.history {
		entries = append(entries, HistoryEntry{Alert: *entry.Alert.Clone(), Timestamp: entry.Timestamp})
	}
	return entries
}

// RetireStorage renames the JSON history files out of the load path after a
// successful migration into the event log, and stops further disk writes.
// The in-memory entries stay available as the fallback read model for the
// rest of the process lifetime.
func (hm *HistoryManager) RetireStorage() error {
	hm.saveMu.Lock()
	defer hm.saveMu.Unlock()
	hm.mu.Lock()
	hm.storageRetired = true
	hm.mu.Unlock()
	for _, path := range []string{hm.historyFile, hm.backupFile} {
		if path == "" {
			continue
		}
		if _, err := os.Stat(path); err != nil {
			continue
		}
		if err := os.Rename(path, path+".imported"); err != nil {
			return fmt.Errorf("retire history file %s: %w", path, err)
		}
	}
	return nil
}

// StorageFileExists reports whether either JSON history source is still
// present. The backup can be the only surviving retry source after an
// interrupted save or partial retirement, so it must keep migration active
// until both fixed legacy leaves have retired.
func (hm *HistoryManager) StorageFileExists() bool {
	for _, path := range []string{hm.historyFile, hm.backupFile} {
		if path == "" {
			continue
		}
		if _, err := os.Stat(path); err == nil {
			return true
		}
	}
	return false
}

// ImportedStorageFileExists reports whether a retired JSON source remains as
// a recovery copy. It is replayed idempotently if events.db is recreated and
// remains the read fallback when the event store is unavailable.
func (hm *HistoryManager) ImportedStorageFileExists() bool {
	for _, path := range []string{hm.historyFile + ".imported", hm.backupFile + ".imported"} {
		if _, err := os.Stat(path); err == nil {
			return true
		}
	}
	return false
}

// StorageLoadError reports why a present legacy source could not be loaded.
// Migration and periodic persistence must leave that source untouched.
func (hm *HistoryManager) StorageLoadError() error {
	if hm == nil {
		return nil
	}
	return hm.storageLoadErr
}

// UpdateAlertLastSeen updates the LastSeen timestamp on the most recent
// history entry matching the given alert ID. This is called when an alert is
// resolved so that the stored history reflects the true duration of the alert,
// not just the snapshot captured at creation time.
func (hm *HistoryManager) UpdateAlertLastSeen(alertID string, lastSeen time.Time) {
	hm.UpdateAlertLastSeenForAlert(&Alert{ID: alertID}, lastSeen)
}

// UpdateAlertLastSeenForAlert updates the LastSeen timestamp on the most recent
// history entry matching the given alert identity. Canonical alerts match by
// canonical state first, then fall back to legacy alert ID for compatibility.
func (hm *HistoryManager) UpdateAlertLastSeenForAlert(alert *Alert, lastSeen time.Time) {
	if alert == nil {
		return
	}

	matchKey := historyIdentityKey(alert)
	matchID := alert.ID

	hm.mu.Lock()
	defer hm.mu.Unlock()

	// Iterate from newest to oldest to find the most recent entry for this alert.
	for i := len(hm.history) - 1; i >= 0; i-- {
		entry := hm.history[i].Alert.Clone()
		entryKey := historyIdentityKey(entry)
		if matchKey != "" && entryKey == matchKey {
			hm.history[i].Alert.LastSeen = lastSeen
			return
		}
		if matchID != "" && hm.history[i].Alert.ID == matchID {
			hm.history[i].Alert.LastSeen = lastSeen
			return
		}
	}
}

// UpdateAlertOperationalContractForAlert updates the most recent history
// entry with the durable evidence and lifecycle state from the active alert.
// This keeps incident history and notification linkage on the same record and
// transition identities as the runtime alert.
func (hm *HistoryManager) UpdateAlertOperationalContractForAlert(alert *Alert) {
	if alert == nil {
		return
	}

	snapshot := alert.Clone()
	if snapshot == nil {
		return
	}
	matchKey := historyIdentityKey(snapshot)
	matchID := snapshot.ID

	hm.mu.Lock()
	defer hm.mu.Unlock()

	for i := len(hm.history) - 1; i >= 0; i-- {
		entry := hm.history[i].Alert.Clone()
		entryKey := historyIdentityKey(entry)
		if (matchKey != "" && entryKey == matchKey) ||
			(matchID != "" && hm.history[i].Alert.ID == matchID) {
			// A resolved row is this occurrence's final state. A later
			// occurrence of the same identity (different StartTime) gets its
			// own row from AddAlert, so its record must not overwrite the
			// resolved one here.
			entryResolved := entry.OperationalRecord != nil &&
				entry.OperationalRecord.State == operationaltrust.OperationalResolved
			snapshotResolved := snapshot.OperationalRecord != nil &&
				snapshot.OperationalRecord.State == operationaltrust.OperationalResolved
			if entryResolved && !snapshotResolved &&
				!entry.StartTime.IsZero() && !snapshot.StartTime.IsZero() &&
				!entry.StartTime.Equal(snapshot.StartTime) {
				return
			}
			hm.history[i].Alert.OperationalRecord = snapshot.OperationalRecord
			hm.history[i].Alert.LatestTransition = snapshot.LatestTransition
			hm.history[i].Alert.Transitions = snapshot.Transitions
			hm.history[i].Alert.Evidence = snapshot.Evidence
			hm.history[i].Alert.Resolution = snapshot.Resolution
			// The live alert is authoritative for acknowledgement; a row
			// that keeps its fire-time ack state shows an acknowledged
			// alert as unacknowledged for its whole active life.
			hm.history[i].Alert.Acknowledged = snapshot.Acknowledged
			hm.history[i].Alert.AckTime = snapshot.AckTime
			hm.history[i].Alert.AckUser = snapshot.AckUser
			return
		}
	}
}

// LatestAlertForAlert returns a clone of the newest history entry matching
// the alert's canonical identity, with legacy alert ID as the compatibility
// fallback.
func (hm *HistoryManager) LatestAlertForAlert(alert *Alert) *Alert {
	if alert == nil {
		return nil
	}
	matchKey := historyIdentityKey(alert)
	matchID := alert.ID

	hm.mu.RLock()
	defer hm.mu.RUnlock()

	for i := len(hm.history) - 1; i >= 0; i-- {
		entry := hm.history[i].Alert.Clone()
		entryKey := historyIdentityKey(entry)
		if (matchKey != "" && entryKey == matchKey) ||
			(matchID != "" && hm.history[i].Alert.ID == matchID) {
			return entry
		}
	}
	return nil
}

// MigrateActiveAlert updates the most recent history entry for an in-flight
// alert when its canonical runtime identity changes, such as a guest metric
// alert moving from one node-scoped resource key to another after a VM move.
func (hm *HistoryManager) MigrateActiveAlert(oldTrackingKey string, updated Alert) {
	if oldTrackingKey == "" {
		return
	}

	updatedAlert := updated.Clone()
	if updatedAlert == nil {
		return
	}
	backfillCanonicalIdentity(updatedAlert)

	hm.mu.Lock()
	defer hm.mu.Unlock()

	for i := len(hm.history) - 1; i >= 0; i-- {
		entry := hm.history[i].Alert.Clone()
		if historyIdentityKey(entry) == oldTrackingKey || hm.history[i].Alert.ID == oldTrackingKey {
			hm.history[i].Alert = *updatedAlert
			return
		}
	}
}

// GetHistory returns alert history within the specified time range
func (hm *HistoryManager) GetHistory(since time.Time, limit int) []Alert {
	hm.mu.RLock()
	defer hm.mu.RUnlock()

	var results []Alert
	count := 0

	// Iterate from newest to oldest
	for i := len(hm.history) - 1; i >= 0 && (limit <= 0 || count < limit); i-- {
		entry := hm.history[i]
		if entry.Timestamp.After(since) {
			results = append(results, entry.Alert)
			count++
		}
	}

	return results
}

// GetAllHistory returns all alert history (up to limit)
func (hm *HistoryManager) GetAllHistory(limit int) []Alert {
	hm.mu.RLock()
	defer hm.mu.RUnlock()

	if limit <= 0 || limit > len(hm.history) {
		limit = len(hm.history)
	}

	results := make([]Alert, 0, limit)
	start := len(hm.history) - limit

	for i := len(hm.history) - 1; i >= start; i-- {
		results = append(results, hm.history[i].Alert)
	}

	return results
}

// loadHistory loads history from disk
func (hm *HistoryManager) loadHistory() error {
	// Try loading from main file first
	data, mainErr := readLimitedRegularFile(hm.historyFile, maxAlertHistoryFileSizeBytes)
	if mainErr != nil {
		if !os.IsNotExist(mainErr) {
			log.Warn().Err(mainErr).Str("file", hm.historyFile).Msg("Failed to read history file")
		}

		// Try backup file
		var backupErr error
		loadedRetired := false
		data, backupErr = readLimitedRegularFile(hm.backupFile, maxAlertHistoryFileSizeBytes)
		if backupErr != nil {
			if os.IsNotExist(backupErr) && os.IsNotExist(mainErr) {
				// Retired JSON remains a recovery source if events.db is lost or
				// unavailable. Prefer the primary retired leaf, then its backup.
				var importedErr error
				data, importedErr = readLimitedRegularFile(hm.historyFile+".imported", maxAlertHistoryFileSizeBytes)
				if importedErr != nil {
					data, importedErr = readLimitedRegularFile(hm.backupFile+".imported", maxAlertHistoryFileSizeBytes)
				}
				if importedErr != nil {
					if os.IsNotExist(importedErr) {
						log.Debug().Msg("No alert history files found, starting fresh")
						return nil
					}
					return fmt.Errorf("failed to read retired alert history recovery source: %w", importedErr)
				}
				hm.storageRetired = true
				loadedRetired = true
				log.Info().Msg("loaded retired alert history recovery source")
			}
			if !loadedRetired && os.IsPermission(backupErr) {
				log.Warn().
					Err(backupErr).
					Str("file", hm.backupFile).
					Msg("Permission denied reading backup history file - check file ownership")
			}
			if loadedRetired {
				backupErr = nil
			} else if !os.IsNotExist(mainErr) {
				return fmt.Errorf("failed to read history file %q (%v); failed to read history backup file %q: %w",
					hm.historyFile, mainErr, hm.backupFile, backupErr)
			} else {
				return fmt.Errorf("failed to read history backup file %q: %w", hm.backupFile, backupErr)
			}
		}
		if !loadedRetired {
			log.Info().Msg("loaded alert history from backup file")
		}
	}

	var history []HistoryEntry
	if err := json.Unmarshal(data, &history); err != nil {
		return fmt.Errorf("failed to unmarshal history: %w", err)
	}

	hm.history = history
	log.Info().Int("count", len(history)).Msg("Loaded alert history")

	// Clean old entries immediately
	hm.cleanOldEntries()
	hm.deduplicateHistory()
	return nil
}

// saveHistory saves history to disk with retry logic
func (hm *HistoryManager) saveHistory() error {
	return hm.saveHistoryWithRetry(3)
}

// saveHistoryWithRetry saves history with exponential backoff retry
func (hm *HistoryManager) saveHistoryWithRetry(maxRetries int) error {
	hm.mu.RLock()
	retired := hm.storageRetired
	hm.mu.RUnlock()
	if retired {
		return nil
	}
	if hm.storageLoadErr != nil {
		return fmt.Errorf("alert history storage remains read-only after load failure: %w", hm.storageLoadErr)
	}
	if maxRetries < 1 {
		maxRetries = 1
	}

	// Serialize all disk writes to prevent concurrent saves from overwriting each other
	hm.saveMu.Lock()
	defer hm.saveMu.Unlock()

	hm.mu.RLock()
	snapshot := make([]HistoryEntry, len(hm.history))
	copy(snapshot, hm.history)
	hm.mu.RUnlock()

	data, err := json.Marshal(snapshot)
	if err != nil {
		return fmt.Errorf("historyManager.saveHistoryWithRetry: marshal history snapshot: %w", err)
	}

	historyFile := hm.historyFile
	backupFile := hm.backupFile

	// Create backup of existing file once before any write attempts.
	// This ensures we don't lose data if all retries fail.
	backupCreated := false
	if _, err := os.Stat(historyFile); err == nil {
		if err := os.Remove(backupFile); err != nil && !os.IsNotExist(err) {
			log.Warn().Err(err).Str("file", backupFile).Msg("Failed to remove existing backup file before save")
		}
		if err := os.Rename(historyFile, backupFile); err != nil {
			log.Warn().
				Err(err).
				Str("source", historyFile).
				Str("backup", backupFile).
				Msg("Failed to create backup file")
		} else {
			backupCreated = true
		}
	}

	var lastErr error
	for attempt := 1; attempt <= maxRetries; attempt++ {
		tempFile := fmt.Sprintf("%s.tmp-%d-%d", historyFile, os.Getpid(), time.Now().UnixNano())

		if err := writeFileSynced(tempFile, data, 0644); err != nil {
			lastErr = fmt.Errorf("failed to write temp history file: %w", err)
			_ = os.Remove(tempFile)
			log.Warn().
				Err(lastErr).
				Int("attempt", attempt).
				Int("maxRetries", maxRetries).
				Msg("failed to write history file, will retry")

			// Exponential backoff: 100ms, 200ms, 400ms
			if attempt < maxRetries {
				backoff := time.Duration(100*(1<<uint(attempt-1))) * time.Millisecond
				time.Sleep(backoff)
			}
			continue
		}

		if err := os.Rename(tempFile, historyFile); err != nil {
			lastErr = fmt.Errorf("failed to replace history file with temp file: %w", err)
			_ = os.Remove(tempFile)
			log.Warn().
				Err(lastErr).
				Int("attempt", attempt).
				Int("maxRetries", maxRetries).
				Msg("Failed to write history file, will retry")

			// Exponential backoff: 100ms, 200ms, 400ms
			if attempt < maxRetries {
				backoff := time.Duration(100*(1<<uint(attempt-1))) * time.Millisecond
				time.Sleep(backoff)
			}
			continue
		}
		if err := os.Chmod(historyFile, alertsFilePerm); err != nil {
			lastErr = err
			log.Warn().
				Err(err).
				Int("attempt", attempt).
				Int("maxRetries", maxRetries).
				Msg("Failed to harden history file permissions, will retry")

			if attempt < maxRetries {
				backoff := time.Duration(100*(1<<uint(attempt-1))) * time.Millisecond
				time.Sleep(backoff)
			}
			continue
		}

		if err := syncDir(filepath.Dir(historyFile)); err != nil {
			log.Warn().Err(err).Str("dir", filepath.Dir(historyFile)).Msg("Failed to sync history directory")
		}

		// Success - remove backup file now that we've successfully written
		if backupCreated {
			if err := os.Remove(backupFile); err != nil && !os.IsNotExist(err) {
				log.Warn().Err(err).Str("file", backupFile).Msg("failed to remove backup file after successful save")
			}
		}
		log.Debug().Int("entries", len(snapshot)).Msg("saved alert history")
		return nil
	}

	if lastErr == nil {
		lastErr = fmt.Errorf("historyManager.saveHistoryWithRetry: no write attempts executed for %s", historyFile)
	}

	// All retries failed - restore backup if we have one
	if backupCreated {
		if restoreErr := os.Rename(backupFile, historyFile); restoreErr != nil {
			restoreErr = fmt.Errorf("restore backup file %s to %s: %w", backupFile, historyFile, restoreErr)
			log.Error().Err(restoreErr).Msg("failed to restore backup after all write attempts failed")
		} else {
			if err := syncDir(filepath.Dir(historyFile)); err != nil {
				log.Warn().Err(err).Str("dir", filepath.Dir(historyFile)).Msg("Failed to sync history directory after restore")
			}
			log.Info().Msg("restored backup after history save failure")
		}
		log.Info().Msg("Restored backup after history save failure")
	}

	return fmt.Errorf("historyManager.saveHistoryWithRetry: failed to write history file after %d attempts: %w", maxRetries, lastErr)
}

func writeFileSynced(path string, data []byte, perm os.FileMode) error {
	file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, perm)
	if err != nil {
		return err
	}

	if _, err := file.Write(data); err != nil {
		_ = file.Close()
		return err
	}

	if err := file.Sync(); err != nil {
		_ = file.Close()
		return err
	}

	return file.Close()
}

func syncDir(dirPath string) error {
	dir, err := os.Open(dirPath)
	if err != nil {
		return err
	}
	defer dir.Close()

	return dir.Sync()
}

// startPeriodicSave starts the periodic save routine
func (hm *HistoryManager) startPeriodicSave() {
	hm.saveTicker = time.NewTicker(hm.saveInterval)

	hm.workerWG.Add(1)
	go func() {
		defer hm.workerWG.Done()
		for {
			select {
			case <-hm.saveTicker.C:
				if err := hm.saveHistory(); err != nil {
					log.Error().Err(err).Msg("failed to save alert history")
				}
			case <-hm.stopChan:
				return
			}
		}
	}()
}

// cleanupRoutine runs periodically to clean old entries
func (hm *HistoryManager) cleanupRoutine() {
	// Run cleanup daily
	ticker := time.NewTicker(24 * time.Hour)
	defer ticker.Stop()

	// Also run cleanup on startup after a delay
	startupDelay := time.NewTimer(1 * time.Minute)
	defer func() {
		if !startupDelay.Stop() {
			select {
			case <-startupDelay.C:
			default:
			}
		}
	}()

	select {
	case <-startupDelay.C:
		hm.cleanOldEntries()
		hm.deduplicateHistory()
	case <-hm.stopChan:
		return
	}

	for {
		select {
		case <-ticker.C:
			hm.cleanOldEntries()
			hm.deduplicateHistory()
		case <-hm.stopChan:
			return
		}
	}
}

// cleanOldEntries removes entries older than MaxHistoryDays
func (hm *HistoryManager) cleanOldEntries() {
	hm.mu.Lock()
	defer hm.mu.Unlock()

	cutoff := time.Now().AddDate(0, 0, -MaxHistoryDays)
	removed := 0
	for _, entry := range hm.history {
		if !entry.Timestamp.After(cutoff) {
			removed++
		}
	}
	if removed == 0 {
		return
	}

	// Allocate for survivors, not the loaded file's original row count. A
	// mostly-expired history otherwise keeps its old-sized Alert array after
	// startup cleanup, even though no expired rows remain visible.
	newHistory := make([]HistoryEntry, 0, len(hm.history)-removed)
	for _, entry := range hm.history {
		if entry.Timestamp.After(cutoff) {
			newHistory = append(newHistory, entry)
		}
	}

	hm.history = newHistory
	log.Info().
		Int("removed", removed).
		Int("remaining", len(newHistory)).
		Msg("cleaned old alert history entries")
}

// historyDedupWindow is the maximum gap between consecutive same-alert entries
// for them to be considered duplicates of one continuous incident.
const historyDedupWindow = 5 * time.Minute

// deduplicateHistory collapses consecutive entries for the same alert identity
// where the gap between entries is shorter than the dedup window. This removes
// flapping/re-fire noise where a bug or polling pattern caused the same alert
// to be recorded many times in quick succession. The surviving entry keeps the
// earliest timestamp and the latest LastSeen.
func (hm *HistoryManager) deduplicateHistory() {
	hm.mu.Lock()
	defer hm.mu.Unlock()

	if len(hm.history) < 2 {
		return
	}

	lastTimeByKey := make(map[string]time.Time)
	removed := 0
	for _, entry := range hm.history {
		key := historyIdentityKey(&entry.Alert)
		if lastTime, ok := lastTimeByKey[key]; ok && entry.Timestamp.Sub(lastTime) <= historyDedupWindow {
			removed++
		}
		lastTimeByKey[key] = entry.Timestamp
	}
	if removed == 0 {
		return
	}

	// Count first so the new backing array holds only surviving occurrences.
	// Clipping a preallocated slice's capacity would keep its original array;
	// copying twice would instead add another full array on sparse reductions.
	deduped := make([]HistoryEntry, 0, len(hm.history)-removed)
	lastIdxByKey := make(map[string]int)
	clear(lastTimeByKey)

	for _, entry := range hm.history {
		key := historyIdentityKey(&entry.Alert)

		if lastTime, ok := lastTimeByKey[key]; ok {
			if entry.Timestamp.Sub(lastTime) <= historyDedupWindow {
				idx := lastIdxByKey[key]
				deduped[idx].Alert = mergeHistoryAlertSnapshots(deduped[idx].Alert, entry.Alert)
				lastTimeByKey[key] = entry.Timestamp
				continue
			}
		}

		deduped = append(deduped, entry)
		lastIdxByKey[key] = len(deduped) - 1
		lastTimeByKey[key] = entry.Timestamp
	}

	hm.history = deduped
	log.Info().
		Int("removed", removed).
		Int("remaining", len(deduped)).
		Msg("deduplicated alert history entries")
}

// RemoveAlert removes a specific alert from history by ID
func (hm *HistoryManager) RemoveAlert(alertID string) {
	hm.mu.Lock()
	defer hm.mu.Unlock()

	removed := 0
	for _, entry := range hm.history {
		if entry.Alert.ID == alertID {
			removed++
		}
	}
	if removed == 0 {
		return
	}

	newHistory := make([]HistoryEntry, 0, len(hm.history)-removed)
	for _, entry := range hm.history {
		if entry.Alert.ID != alertID {
			newHistory = append(newHistory, entry)
		}
	}

	hm.history = newHistory
	log.Debug().Str("alertID", alertID).Msg("removed alert from history")
}

// ClearAllHistory clears all alert history
func (hm *HistoryManager) ClearAllHistory() error {
	hm.mu.Lock()
	defer hm.mu.Unlock()

	// Clear the in-memory history
	hm.history = make([]HistoryEntry, 0)

	// Remove the history files
	var removeErrs []error
	if err := os.Remove(hm.historyFile); err != nil && !os.IsNotExist(err) {
		removeErrs = append(removeErrs, fmt.Errorf("remove history file %s: %w", hm.historyFile, err))
	}
	if err := os.Remove(hm.backupFile); err != nil && !os.IsNotExist(err) {
		removeErrs = append(removeErrs, fmt.Errorf("remove backup file %s: %w", hm.backupFile, err))
	}
	if len(removeErrs) > 0 {
		return fmt.Errorf("clear alert history files: %w", errors.Join(removeErrs...))
	}

	log.Info().Msg("cleared all alert history")
	return nil
}

// Stop stops the history manager
func (hm *HistoryManager) Stop() {
	hm.stopOnce.Do(func() {
		close(hm.stopChan)
		if hm.saveTicker != nil {
			hm.saveTicker.Stop()
		}

		hm.workerWG.Wait()

		// Save one final time
		if err := hm.saveHistory(); err != nil {
			log.Error().Err(err).Msg("Failed to save alert history on shutdown")
		}
	})
}

// GetStats returns statistics about the alert history
func (hm *HistoryManager) GetStats() map[string]any {
	stats := hm.Stats()
	return map[string]any{
		"totalEntries": stats.TotalEntries,
		"oldestEntry":  stats.OldestEntry,
		"newestEntry":  stats.NewestEntry,
		"dataDir":      stats.DataDir,
		"fileSize":     stats.FileSize,
	}
}

// Stats returns typed statistics about the alert history.
func (hm *HistoryManager) Stats() HistoryStats {
	hm.mu.RLock()
	defer hm.mu.RUnlock()

	oldest := time.Now()
	newest := time.Time{}

	if len(hm.history) > 0 {
		oldest = hm.history[0].Timestamp
		newest = hm.history[len(hm.history)-1].Timestamp
	}

	return HistoryStats{
		TotalEntries: len(hm.history),
		OldestEntry:  oldest,
		NewestEntry:  newest,
		DataDir:      hm.dataDir,
		FileSize:     hm.getFileSize(),
	}
}

// getFileSize returns the size of the history file
func (hm *HistoryManager) getFileSize() int64 {
	info, err := os.Stat(hm.historyFile)
	if err != nil {
		return 0
	}
	return info.Size()
}