Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
package memory
import (
"bytes"
"encoding/json"
"fmt"
"io"
"os"
"sort"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/google/uuid"
"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
"github.com/rs/zerolog/log"
)
// IncidentStatus represents the current state of an incident.
type IncidentStatus string
const (
IncidentStatusOpen IncidentStatus = "open"
IncidentStatusUnknown IncidentStatus = "unknown"
IncidentStatusResolved IncidentStatus = "resolved"
)
// IncidentEventType describes a timeline event type.
type IncidentEventType string
const (
IncidentEventAlertFired IncidentEventType = "alert_fired"
IncidentEventAlertAcknowledged IncidentEventType = "alert_acknowledged"
IncidentEventAlertUnacknowledged IncidentEventType = "alert_unacknowledged"
IncidentEventAlertSnoozed IncidentEventType = "alert_snoozed"
IncidentEventAlertUnsnoozed IncidentEventType = "alert_unsnoozed"
IncidentEventAlertResolved IncidentEventType = "alert_resolved"
IncidentEventAnalysis IncidentEventType = "ai_analysis"
IncidentEventCommand IncidentEventType = "command"
IncidentEventRunbook IncidentEventType = "runbook"
IncidentEventNote IncidentEventType = "note"
)
// IncidentEvent represents a single timeline entry for an incident.
type IncidentEvent struct {
Source string `json:"source,omitempty"`
Evidence *unifiedresources.ResourceChange `json:"evidence,omitempty"`
ID string `json:"id"`
Type IncidentEventType `json:"type"`
Timestamp time.Time `json:"timestamp"`
Summary string `json:"summary"`
Details map[string]interface{} `json:"details,omitempty"`
}
// Incident captures an alert occurrence and its investigation timeline.
// It is an alert-scoped memory/projection for investigation support rather than
// the canonical durable resource-change history.
type Incident struct {
History *IncidentHistoryCoverage `json:"history,omitempty"`
ID string `json:"id"`
AlertIdentifier string `json:"alertIdentifier"`
AlertType string `json:"alertType"`
Level string `json:"level"`
ResourceID string `json:"resourceId"`
ResourceName string `json:"resourceName"`
ResourceType string `json:"resourceType,omitempty"`
Node string `json:"node,omitempty"`
Instance string `json:"instance,omitempty"`
Message string `json:"message,omitempty"`
Status IncidentStatus `json:"status"`
OpenedAt time.Time `json:"openedAt"`
ClosedAt *time.Time `json:"closedAt,omitempty"`
Acknowledged bool `json:"acknowledged"`
AckUser string `json:"ackUser,omitempty"`
AckTime *time.Time `json:"ackTime,omitempty"`
Events []IncidentEvent `json:"events,omitempty"`
}
type incidentShell struct {
ID string
AlertIdentifier string
AlertType string
Level string
ResourceID string
ResourceName string
ResourceType string
Node string
Instance string
Message string
OpenedAt time.Time
OccurrenceClosedAt *time.Time
OccurrenceRefiredAt *time.Time
Events []IncidentEvent
}
type incidentJSON struct {
History *IncidentHistoryCoverage `json:"history,omitempty"`
ID string `json:"id"`
AlertIdentifier string `json:"alertIdentifier"`
AlertType string `json:"alertType"`
Level string `json:"level"`
ResourceID string `json:"resourceId"`
ResourceName string `json:"resourceName"`
ResourceType string `json:"resourceType,omitempty"`
Node string `json:"node,omitempty"`
Instance string `json:"instance,omitempty"`
Message string `json:"message,omitempty"`
Status IncidentStatus `json:"status"`
OpenedAt time.Time `json:"openedAt"`
ClosedAt *time.Time `json:"closedAt,omitempty"`
Acknowledged bool `json:"acknowledged"`
AckUser string `json:"ackUser,omitempty"`
AckTime *time.Time `json:"ackTime,omitempty"`
Events []IncidentEvent `json:"events,omitempty"`
}
type incidentShellJSON struct {
ID string `json:"id"`
AlertIdentifier string `json:"alertIdentifier"`
AlertType string `json:"alertType"`
Level string `json:"level"`
ResourceID string `json:"resourceId"`
ResourceName string `json:"resourceName"`
ResourceType string `json:"resourceType,omitempty"`
Node string `json:"node,omitempty"`
Instance string `json:"instance,omitempty"`
Message string `json:"message,omitempty"`
OpenedAt time.Time `json:"openedAt"`
OccurrenceClosedAt *time.Time `json:"occurrenceClosedAt,omitempty"`
OccurrenceRefiredAt *time.Time `json:"occurrenceRefiredAt,omitempty"`
Events []IncidentEvent `json:"events,omitempty"`
Status IncidentStatus `json:"status,omitempty"`
ClosedAt *time.Time `json:"closedAt,omitempty"`
Acknowledged bool `json:"acknowledged,omitempty"`
AckUser string `json:"ackUser,omitempty"`
AckTime *time.Time `json:"ackTime,omitempty"`
}
func (i Incident) MarshalJSON() ([]byte, error) {
alertIdentifier := strings.TrimSpace(i.AlertIdentifier)
return json.Marshal(incidentJSON{
History: i.History,
ID: i.ID,
AlertIdentifier: alertIdentifier,
AlertType: i.AlertType,
Level: i.Level,
ResourceID: i.ResourceID,
ResourceName: i.ResourceName,
ResourceType: i.ResourceType,
Node: i.Node,
Instance: i.Instance,
Message: i.Message,
Status: i.Status,
OpenedAt: i.OpenedAt,
ClosedAt: i.ClosedAt,
Acknowledged: i.Acknowledged,
AckUser: i.AckUser,
AckTime: i.AckTime,
Events: i.Events,
})
}
func (i *Incident) UnmarshalJSON(data []byte) error {
if i == nil {
return nil
}
var payload incidentJSON
if err := json.Unmarshal(data, &payload); err != nil {
return err
}
*i = Incident{
History: payload.History,
ID: payload.ID,
AlertIdentifier: strings.TrimSpace(payload.AlertIdentifier),
AlertType: payload.AlertType,
Level: payload.Level,
ResourceID: payload.ResourceID,
ResourceName: payload.ResourceName,
ResourceType: payload.ResourceType,
Node: payload.Node,
Instance: payload.Instance,
Message: payload.Message,
Status: payload.Status,
OpenedAt: payload.OpenedAt,
ClosedAt: payload.ClosedAt,
Acknowledged: payload.Acknowledged,
AckUser: payload.AckUser,
AckTime: payload.AckTime,
Events: payload.Events,
}
return nil
}
func (s incidentShell) MarshalJSON() ([]byte, error) {
return json.Marshal(incidentShellJSON{
ID: s.ID,
AlertIdentifier: strings.TrimSpace(s.AlertIdentifier),
AlertType: s.AlertType,
Level: s.Level,
ResourceID: s.ResourceID,
ResourceName: s.ResourceName,
ResourceType: s.ResourceType,
Node: s.Node,
Instance: s.Instance,
Message: s.Message,
OpenedAt: s.OpenedAt,
OccurrenceClosedAt: s.OccurrenceClosedAt,
OccurrenceRefiredAt: s.OccurrenceRefiredAt,
Events: s.Events,
})
}
func (s *incidentShell) UnmarshalJSON(data []byte) error {
if s == nil {
return nil
}
var payload incidentShellJSON
if err := json.Unmarshal(data, &payload); err != nil {
return err
}
*s = incidentShell{
ID: payload.ID,
AlertIdentifier: strings.TrimSpace(payload.AlertIdentifier),
AlertType: payload.AlertType,
Level: payload.Level,
ResourceID: payload.ResourceID,
ResourceName: payload.ResourceName,
ResourceType: payload.ResourceType,
Node: payload.Node,
Instance: payload.Instance,
Message: payload.Message,
OpenedAt: payload.OpenedAt,
OccurrenceClosedAt: payload.OccurrenceClosedAt,
OccurrenceRefiredAt: payload.OccurrenceRefiredAt,
Events: cloneIncidentEvents(payload.Events),
}
if s.OccurrenceClosedAt == nil && payload.ClosedAt != nil {
s.OccurrenceClosedAt = cloneTime(*payload.ClosedAt)
}
if payload.Acknowledged && !hasIncidentEventType(s.Events, IncidentEventAlertAcknowledged, IncidentEventAlertUnacknowledged) {
ackAt := payload.OpenedAt
if payload.AckTime != nil && !payload.AckTime.IsZero() {
ackAt = *payload.AckTime
}
s.Events = append(s.Events, IncidentEvent{
ID: generateIncidentEventID(),
Type: IncidentEventAlertAcknowledged,
Timestamp: ackAt,
Summary: "Alert acknowledged",
Details: map[string]interface{}{
"user": strings.TrimSpace(payload.AckUser),
},
})
}
if (payload.Status == IncidentStatusResolved || payload.ClosedAt != nil) && !hasIncidentEventType(s.Events, IncidentEventAlertResolved) {
resolvedAt := payload.OpenedAt
if payload.ClosedAt != nil && !payload.ClosedAt.IsZero() {
resolvedAt = *payload.ClosedAt
}
s.Events = append(s.Events, IncidentEvent{
ID: generateIncidentEventID(),
Type: IncidentEventAlertResolved,
Timestamp: resolvedAt,
Summary: "Alert resolved",
Details: map[string]interface{}{
"resolved_at": resolvedAt.Format(time.RFC3339),
},
})
}
sortIncidentEvents(s.Events)
return nil
}
// IncidentStoreConfig configures incident retention and persistence.
type IncidentStoreConfig struct {
DataDir string
MaxIncidents int
MaxEventsPerIncident int
MaxAgeDays int
}
// IncidentStore maintains alert-scoped incident timelines and persistence for
// investigation memory. Durable resource history belongs to the canonical
// unified-resource change model.
type IncidentStore struct {
mu sync.RWMutex
saveMu sync.Mutex
savePending atomic.Bool // a queued save will capture the latest state; further requests coalesce
saveWG sync.WaitGroup // asynchronous saves still running or queued; joined by flush
savesCompleted atomic.Int64 // completed JSON replacements, observable by tests
incidents []*incidentShell
maxIncidents int
maxEvents int
maxAge time.Duration
dataDir string
filePath string
resourceTimelineStore IncidentTimelineStore
}
const (
defaultIncidentMaxIncidents = 500
defaultIncidentMaxEvents = 120
defaultIncidentMaxAgeDays = 90
maxIncidentFileSize = 20 * 1024 * 1024 // 20MB
incidentStartMatchTolerance = time.Second
incidentSnapshotSource = "alert_history_snapshot"
)
// IncidentTimelineStore exposes the canonical resource timeline used to derive
// incident lifecycle and remediation history.
type IncidentTimelineStore interface {
GetRecentChangesFiltered(string, time.Time, int, unifiedresources.ResourceChangeFilters) ([]unifiedresources.ResourceChange, error)
ResourceHistoryIDs(string) ([]string, error)
GetRecentChanges(canonicalID string, since time.Time, limit int) ([]unifiedresources.ResourceChange, error)
}
// NewIncidentStore creates a new incident store with persistence.
func NewIncidentStore(cfg IncidentStoreConfig) *IncidentStore {
maxIncidents := cfg.MaxIncidents
if maxIncidents <= 0 {
maxIncidents = defaultIncidentMaxIncidents
}
maxEvents := cfg.MaxEventsPerIncident
if maxEvents <= 0 {
maxEvents = defaultIncidentMaxEvents
}
maxAgeDays := cfg.MaxAgeDays
if maxAgeDays <= 0 {
maxAgeDays = defaultIncidentMaxAgeDays
}
store := &IncidentStore{
incidents: make([]*incidentShell, 0),
maxIncidents: maxIncidents,
maxEvents: maxEvents,
maxAge: time.Duration(maxAgeDays) * 24 * time.Hour,
dataDir: normalizeOptionalMemoryDataDir(cfg.DataDir),
}
if store.dataDir != "" {
filePath, err := memoryPersistencePath(store.dataDir, incidentHistoryFileName)
if err != nil {
panic(fmt.Sprintf("invalid AI incident persistence path for %q: %v", store.dataDir, err))
}
store.filePath = filePath
if err := store.loadFromDisk(); err != nil {
log.Warn().Err(err).Msg("failed to load incident history from disk")
} else if len(store.incidents) > 0 {
log.Info().Int("count", len(store.incidents)).Msg("loaded incident history from disk")
}
}
return store
}
// SetResourceTimelineStore attaches the canonical resource timeline used to
// project durable incident lifecycle and remediation history.
func (s *IncidentStore) SetResourceTimelineStore(store IncidentTimelineStore) {
if s == nil {
return
}
s.mu.Lock()
s.resourceTimelineStore = store
s.mu.Unlock()
}
// RecordAlertFired opens or updates an incident for a fired alert.
func (s *IncidentStore) RecordAlertFired(alert *alerts.Alert) {
if alert == nil {
return
}
s.mu.Lock()
defer s.mu.Unlock()
shell := s.findLifecycleOccurrenceLocked(alert)
if alert.StartTime.IsZero() {
shell = s.findOpenIncidentByAlertIdentifierLocked(alert.ID)
}
if shell == nil {
shell = newIncidentShellFromAlert(alert)
s.incidents = append(s.incidents, shell)
if !s.projectsFromCanonicalLocked() {
s.addEventAtLocked(shell, IncidentEventAlertFired, alert.StartTime, formatAlertSummary(alert), map[string]interface{}{
"type": alert.Type,
"level": string(alert.Level),
"value": alert.Value,
"threshold": alert.Threshold,
})
}
} else if incidentOccurrenceClosedAt(shell) != nil {
// A replayed fired event must not reopen a completed occurrence.
return
} else {
updateIncidentShellFromAlert(shell, alert)
}
s.trimLocked()
s.saveAsync()
}
// RecordAlertRefired reopens a retained occurrence only for a newer explicit
// refire. Its timestamp survives checkpoints so delayed lifecycle replay cannot
// close it with the previous resolution or reopen it after a later resolution.
func (s *IncidentStore) RecordAlertRefired(alert *alerts.Alert, occurredAt time.Time) {
if alert == nil || occurredAt.IsZero() {
return
}
s.mu.Lock()
defer s.mu.Unlock()
shell := s.findLifecycleOccurrenceLocked(alert)
if shell == nil {
shell = newIncidentShellFromAlert(alert)
s.incidents = append(s.incidents, shell)
}
if shell.OccurrenceRefiredAt != nil && !occurredAt.After(*shell.OccurrenceRefiredAt) {
return
}
if shell.OccurrenceClosedAt != nil && !occurredAt.After(*shell.OccurrenceClosedAt) {
return
}
shell.OccurrenceRefiredAt = cloneTime(occurredAt)
shell.OccurrenceClosedAt = nil
updateIncidentShellFromAlert(shell, alert)
if !s.projectsFromCanonicalLocked() {
s.addEventAtLocked(shell, IncidentEventAlertFired, occurredAt, formatAlertSummary(alert), nil)
}
s.trimLocked()
s.saveAsync()
}
// RecordAlertAcknowledged records an acknowledgement event for an alert.
func (s *IncidentStore) RecordAlertAcknowledged(alert *alerts.Alert, user string) {
if alert == nil {
return
}
s.mu.Lock()
defer s.mu.Unlock()
shell := s.ensureIncidentForAlertLocked(alert)
if !s.projectsFromCanonicalLocked() {
ackAt := time.Now()
if alert.AckTime != nil && !alert.AckTime.IsZero() {
ackAt = *alert.AckTime
}
s.addEventAtLocked(shell, IncidentEventAlertAcknowledged, ackAt, "Alert acknowledged", map[string]interface{}{
"user": user,
})
}
s.trimLocked()
s.saveAsync()
}
// RecordAlertUnacknowledged records an unacknowledge event for an alert.
func (s *IncidentStore) RecordAlertUnacknowledged(alert *alerts.Alert, user string) {
if alert == nil {
return
}
s.mu.Lock()
defer s.mu.Unlock()
shell := s.ensureIncidentForAlertLocked(alert)
if !s.projectsFromCanonicalLocked() {
s.addEventAtLocked(shell, IncidentEventAlertUnacknowledged, time.Now(), "Alert unacknowledged", map[string]interface{}{
"user": user,
})
}
s.trimLocked()
s.saveAsync()
}
// alertResolvedEventSummary is the timeline line for an alert's close: the
// handover account when it closed without recovering, else "Alert resolved".
func alertResolvedEventSummary(alert *alerts.Alert) string {
if alert != nil {
if summary := alert.Resolution.Summary(); summary != "" {
return summary
}
}
return "Alert resolved"
}
// RecordAlertResolved records a resolved event and closes the incident.
func (s *IncidentStore) RecordAlertResolved(alert *alerts.Alert, resolvedAt time.Time) {
if alert == nil {
return
}
s.mu.Lock()
defer s.mu.Unlock()
shell := s.findLifecycleOccurrenceLocked(alert)
if alert.StartTime.IsZero() {
shell = s.findOpenIncidentByAlertIdentifierLocked(alert.ID)
}
if shell == nil {
shell = newIncidentShellFromAlert(alert)
s.incidents = append(s.incidents, shell)
}
// Resolution replay is idempotent, including after a checkpoint reload.
if incidentOccurrenceClosedAt(shell) != nil {
return
}
if resolvedAt.IsZero() {
now := time.Now()
resolvedAt = now
}
if shell.OccurrenceRefiredAt != nil && resolvedAt.Before(*shell.OccurrenceRefiredAt) {
return
}
shell.OccurrenceClosedAt = cloneTime(resolvedAt)
if !s.projectsFromCanonicalLocked() {
s.addEventAtLocked(shell, IncidentEventAlertResolved, resolvedAt, alertResolvedEventSummary(alert), map[string]interface{}{
"resolved_at": resolvedAt.Format(time.RFC3339),
})
}
s.trimLocked()
s.saveAsync()
}
// EnsureAlertOccurrence materializes the minimum honest timeline carried by
// an alert snapshot. It is the read-repair boundary for active alerts and
// legacy history entries that predate the canonical resource timeline. When
// canonical events exist, QueryIncidents replaces these snapshot-derived
// lifecycle breadcrumbs with the durable projection.
func (s *IncidentStore) EnsureAlertOccurrence(alert *alerts.Alert, resolvedAt *time.Time) *Incident {
if !s.materializeAlertOccurrence(alert, resolvedAt) {
return nil
}
return s.GetTimelineByAlertAt(alert.ID, alert.StartTime)
}
// MaterializeAlertOccurrence repairs the same snapshot-derived occurrence as
// EnsureAlertOccurrence without reading its projected timeline. Lifecycle
// replay writes canonical changes separately and discards the returned
// timeline, so hydrating it for every imported snapshot only adds history
// reads and copies. Callers serving a timeline must use EnsureAlertOccurrence
// or QueryIncidents, retaining canonical evidence and read-error semantics.
func (s *IncidentStore) MaterializeAlertOccurrence(alert *alerts.Alert, resolvedAt *time.Time) {
s.materializeAlertOccurrence(alert, resolvedAt)
}
func (s *IncidentStore) materializeAlertOccurrence(alert *alerts.Alert, resolvedAt *time.Time) bool {
if s == nil || alert == nil || strings.TrimSpace(alert.ID) == "" {
return false
}
s.mu.Lock()
shell := s.findIncidentByAlertAtLocked(alert.ID, alert.StartTime)
changed := false
if shell == nil {
shell = newIncidentShellFromAlert(alert)
s.incidents = append(s.incidents, shell)
changed = true
} else {
updateIncidentShellFromAlert(shell, alert)
}
if !hasIncidentEventType(shell.Events, IncidentEventAlertFired) {
s.addEventAtLocked(shell, IncidentEventAlertFired, shell.OpenedAt, formatAlertSummary(alert), map[string]interface{}{
"type": alert.Type,
"level": string(alert.Level),
"value": alert.Value,
"threshold": alert.Threshold,
"projection_source": incidentSnapshotSource,
})
changed = true
}
if alert.Acknowledged && !hasIncidentEventType(shell.Events, IncidentEventAlertAcknowledged) {
ackAt := shell.OpenedAt
if alert.AckTime != nil && !alert.AckTime.IsZero() {
ackAt = *alert.AckTime
}
s.addEventAtLocked(shell, IncidentEventAlertAcknowledged, ackAt, "Alert acknowledged", map[string]interface{}{
"user": strings.TrimSpace(alert.AckUser),
"projection_source": incidentSnapshotSource,
})
changed = true
}
if resolvedAt != nil && !resolvedAt.IsZero() && (shell.OccurrenceRefiredAt == nil || !resolvedAt.Before(*shell.OccurrenceRefiredAt)) {
shell.OccurrenceClosedAt = cloneTime(*resolvedAt)
if !hasIncidentEventType(shell.Events, IncidentEventAlertResolved) {
s.addEventAtLocked(shell, IncidentEventAlertResolved, *resolvedAt, alertResolvedEventSummary(alert), map[string]interface{}{
"resolved_at": resolvedAt.Format(time.RFC3339),
"projection_source": incidentSnapshotSource,
})
changed = true
}
}
if changed {
s.trimLockedPreserving(shell)
s.saveAsync()
}
s.mu.Unlock()
return true
}
// RecordAnalysis adds an AI analysis event to the incident for an alert.
func (s *IncidentStore) RecordAnalysis(alertIdentifier, summary string, details map[string]interface{}) {
if alertIdentifier == "" {
return
}
s.mu.Lock()
defer s.mu.Unlock()
shell := s.findLatestIncidentByAlertIdentifierLocked(alertIdentifier)
if shell == nil {
shell = &incidentShell{
ID: generateIncidentID(),
AlertIdentifier: alertIdentifier,
OpenedAt: time.Now(),
}
s.incidents = append(s.incidents, shell)
}
if summary == "" {
summary = "Pulse Patrol analysis completed"
}
s.addEventLocked(shell, IncidentEventAnalysis, summary, details)
s.trimLocked()
s.saveAsync()
}
// RecordCommand adds a command execution event to the incident for an alert.
func (s *IncidentStore) RecordCommand(alertIdentifier, command string, success bool, output string, details map[string]interface{}) {
if alertIdentifier == "" || command == "" {
return
}
s.mu.Lock()
defer s.mu.Unlock()
shell := s.findLatestIncidentByAlertIdentifierLocked(alertIdentifier)
if shell == nil {
shell = &incidentShell{
ID: generateIncidentID(),
AlertIdentifier: alertIdentifier,
OpenedAt: time.Now(),
}
s.incidents = append(s.incidents, shell)
}
if details == nil {
details = make(map[string]interface{})
}
if shell.ResourceID == "" {
if resourceID, ok := details["resource_id"].(string); ok {
shell.ResourceID = strings.TrimSpace(resourceID)
}
}
details["command"] = command
details["success"] = success
if output != "" {
details["output_excerpt"] = truncateOutput(output, 500)
}
status := "failed"
if success {
status = "succeeded"
}
summary := fmt.Sprintf("Command %s: %s", status, command)
if !s.projectsFromCanonicalLocked() {
s.addEventLocked(shell, IncidentEventCommand, summary, details)
}
s.trimLocked()
s.saveAsync()
}
// RecordRunbook adds a runbook execution event to the incident for an alert.
func (s *IncidentStore) RecordRunbook(alertIdentifier, runbookID, title string, outcome string, automatic bool, message string) {
if alertIdentifier == "" || runbookID == "" {
return
}
s.mu.Lock()
defer s.mu.Unlock()
shell := s.findLatestIncidentByAlertIdentifierLocked(alertIdentifier)
if shell == nil {
shell = &incidentShell{
ID: generateIncidentID(),
AlertIdentifier: alertIdentifier,
OpenedAt: time.Now(),
}
s.incidents = append(s.incidents, shell)
}
summary := fmt.Sprintf("Runbook %s (%s)", title, outcome)
details := map[string]interface{}{
"runbook_id": runbookID,
"outcome": outcome,
"automatic": automatic,
}
if message != "" {
details["message"] = message
}
if !s.projectsFromCanonicalLocked() {
s.addEventLocked(shell, IncidentEventRunbook, summary, details)
}
s.trimLocked()
s.saveAsync()
}
// RecordNote appends a user note to an incident identified by canonical alert identifier or incident ID.
func (s *IncidentStore) RecordNote(alertIdentifier, incidentID, note, user string) bool {
note = strings.TrimSpace(note)
if note == "" {
return false
}
var projected *Incident
if strings.HasPrefix(incidentID, "projected-") {
page, err := s.QueryIncidents(IncidentQuery{AlertIdentifier: alertIdentifier, IncidentID: incidentID, Limit: 1})
if err != nil || len(page.Incidents) != 1 {
return false
}
projected = page.Incidents[0]
}
s.mu.Lock()
defer s.mu.Unlock()
var shell *incidentShell
if incidentID != "" {
shell = s.findIncidentByIDLocked(incidentID)
} else if alertIdentifier != "" {
shell = s.findLatestIncidentByAlertIdentifierLocked(alertIdentifier)
}
if shell == nil && projected != nil {
shell = &incidentShell{ID: projected.ID, AlertIdentifier: projected.AlertIdentifier, AlertType: projected.AlertType, Level: projected.Level, ResourceID: projected.ResourceID, ResourceName: projected.ResourceName, ResourceType: projected.ResourceType, Node: projected.Node, Instance: projected.Instance, Message: projected.Message, OpenedAt: projected.OpenedAt}
if projected.ClosedAt != nil {
closed := *projected.ClosedAt
shell.OccurrenceClosedAt = &closed
}
s.incidents = append(s.incidents, shell)
}
if shell == nil {
return false
}
summary := "Note added"
if user != "" {
summary = fmt.Sprintf("Note added by %s", user)
}
s.addEventLocked(shell, IncidentEventNote, summary, map[string]interface{}{
"note": note,
"user": user,
})
s.trimLocked()
s.saveAsync()
return true
}
// GetTimelineByAlertIdentifier returns the most recent incident for the alert.
func (s *IncidentStore) GetTimelineByAlertIdentifier(alertIdentifier string) *Incident {
return s.GetTimelineByAlertAt(alertIdentifier, time.Time{})
}
// GetTimelineByAlertAt is retained for local writer callers. Runtime readers
// use QueryIncidents to distinguish an unavailable store from absent history.
func (s *IncidentStore) GetTimelineByAlertAt(alertIdentifier string, startedAt time.Time) *Incident {
if strings.TrimSpace(alertIdentifier) == "" {
return nil
}
page, err := s.QueryIncidents(IncidentQuery{AlertIdentifier: alertIdentifier, StartedAt: startedAt, Limit: 1})
if err != nil {
log.Warn().Err(err).Msg("incident history unavailable")
return nil
}
if len(page.Incidents) == 0 {
return nil
}
return page.Incidents[0]
}
func (s *IncidentStore) findIncidentByAlertAtLocked(alertIdentifier string, startedAt time.Time) *incidentShell {
if startedAt.IsZero() {
return s.findLatestIncidentByAlertIdentifierLocked(alertIdentifier)
}
best, delta := s.findClosestIncidentByAlertAtLocked(alertIdentifier, startedAt)
if best == nil || delta > incidentStartMatchTolerance {
return nil
}
return best
}
func (s *IncidentStore) findClosestIncidentByAlertAtLocked(alertIdentifier string, startedAt time.Time) (*incidentShell, time.Duration) {
var best *incidentShell
var bestDelta time.Duration
for _, incident := range s.incidents {
if incident == nil || incident.AlertIdentifier != alertIdentifier {
continue
}
delta := incident.OpenedAt.Sub(startedAt)
if delta < 0 {
delta = -delta
}
if best == nil || delta < bestDelta {
best = incident
bestDelta = delta
}
}
return best, bestDelta
}
// ListIncidentsByResource returns recent incidents for a resource.
func (s *IncidentStore) ListIncidentsByResource(resourceID string, limit int) []*Incident {
if strings.TrimSpace(resourceID) == "" {
return nil
}
page, err := s.QueryIncidents(IncidentQuery{ResourceID: resourceID, Limit: limit})
if err != nil {
log.Warn().Err(err).Msg("incident history unavailable")
return nil
}
return page.Incidents
}
// FormatForAlert returns a condensed incident timeline for prompt injection.
func (s *IncidentStore) FormatForAlert(alertIdentifier string, maxEvents int) string {
page, err := s.QueryIncidents(IncidentQuery{AlertIdentifier: alertIdentifier, Limit: 1})
if err != nil {
return "\n\nIncident history unavailable: canonical evidence could not be read.\n"
}
if len(page.Incidents) == 0 {
return formatIncidentCoverage(page)
}
incident := page.Incidents[0]
var b strings.Builder
b.WriteString("\n\n## Incident Memory\n")
b.WriteString(formatIncidentCoverage(page))
b.WriteString(fmt.Sprintf("Alert incident for %s (%s, %s)\n",
incident.ResourceName, incident.AlertType, incident.Level))
b.WriteString(fmt.Sprintf("Status: %s\n", incident.Status))
if incident.History != nil {
b.WriteString("Occurrence evidence source: " + incident.History.Source + "\n")
}
events := incident.Events
if maxEvents > 0 && len(events) > maxEvents {
events = events[len(events)-maxEvents:]
b.WriteString(fmt.Sprintf("Showing latest %d of %d recorded events.\n", len(events), len(incident.Events)))
}
for _, evt := range events {
b.WriteString("- ")
b.WriteString(formatIncidentTime(evt.Timestamp))
b.WriteString(": ")
b.WriteString(evt.Summary)
if evt.Type == IncidentEventNote {
if note, ok := stringMetadata(evt.Details, "note"); ok {
b.WriteString(": " + note)
}
}
if evt.Evidence != nil {
b.WriteString(fmt.Sprintf(" [record=%s observed=%s occurred=%s source=%s actor=%s]", evt.Evidence.ID, formatIncidentTime(evt.Evidence.ObservedAt), formatIncidentOccurredTime(evt.Evidence.OccurredAt), evt.Evidence.SourceAdapter, evt.Evidence.Actor))
} else if evt.Source != "" {
b.WriteString(" [source=" + evt.Source + "]")
}
b.WriteString("\n")
}
return b.String()
}
// FormatForResource returns a condensed incident summary for a resource.
func (s *IncidentStore) FormatForResource(resourceID string, limit int) string {
page, err := s.QueryIncidents(IncidentQuery{ResourceID: resourceID, Limit: limit})
if err != nil {
return "\n\nIncident history unavailable: canonical evidence could not be read.\n"
}
return FormatIncidentPageForResource(page)
}
// FormatIncidentPageForResource formats the same snapshot returned to an API
// caller, so a second read cannot silently describe a different history.
func FormatIncidentPageForResource(page IncidentPage) string {
incidents := page.Incidents
if len(incidents) == 0 {
return formatIncidentCoverage(page)
}
var b strings.Builder
b.WriteString("\n\n## Incident Memory\n")
b.WriteString(formatIncidentCoverage(page))
b.WriteString("Recent incidents:\n")
for _, incident := range incidents {
status := string(incident.Status)
if incident.Acknowledged && incident.Status == IncidentStatusOpen {
status = "acknowledged"
}
b.WriteString("- ")
b.WriteString(formatIncidentTime(incident.OpenedAt))
b.WriteString(": ")
b.WriteString(incident.AlertType)
if incident.Level != "" {
b.WriteString(" (")
b.WriteString(incident.Level)
b.WriteString(")")
}
b.WriteString(" - ")
b.WriteString(status)
if incident.History != nil {
b.WriteString(" [source=" + incident.History.Source + "]")
}
b.WriteString("\n")
}
return b.String()
}
// FormatForPatrol returns a condensed incident summary for infrastructure-wide patrol analysis.
func (s *IncidentStore) FormatForPatrol(limit int) string {
if limit <= 0 {
limit = 8
}
page, err := s.QueryIncidents(IncidentQuery{Limit: limit})
if err != nil {
return "\n\nIncident history unavailable: canonical evidence could not be read.\n"
}
snapshot := page.Incidents
if len(snapshot) == 0 {
return formatIncidentCoverage(page)
}
var b strings.Builder
b.WriteString("\n\n## Incident Memory\n")
b.WriteString(formatIncidentCoverage(page))
b.WriteString("Recent incidents across infrastructure:\n")
for _, incident := range snapshot {
status := string(incident.Status)
if incident.Acknowledged && incident.Status == IncidentStatusOpen {
status = "acknowledged"
}
lastSummary := ""
if len(incident.Events) > 0 {
lastSummary = incident.Events[len(incident.Events)-1].Summary
}
b.WriteString("- ")
b.WriteString(formatIncidentTime(incident.OpenedAt))
b.WriteString(": ")
if incident.ResourceName != "" {
b.WriteString(incident.ResourceName)
b.WriteString(" - ")
}
if incident.AlertType != "" {
b.WriteString(incident.AlertType)
}
if incident.Level != "" {
b.WriteString(" (")
b.WriteString(incident.Level)
b.WriteString(")")
}
b.WriteString(" - ")
b.WriteString(status)
if incident.History != nil {
b.WriteString(" [source=" + incident.History.Source + "]")
}
if lastSummary != "" {
b.WriteString(" - last: ")
b.WriteString(truncateOutput(lastSummary, 80))
} else if incident.Message != "" {
b.WriteString(" - ")
b.WriteString(truncateOutput(incident.Message, 80))
}
b.WriteString("\n")
}
return b.String()
}
func (s *IncidentStore) projectsFromCanonicalLocked() bool {
return s.resourceTimelineStore != nil
}
func hydrateIncidentFromCanonicalChange(incident *Incident, change unifiedresources.ResourceChange) {
if incident == nil {
return
}
// Alert evidence owns its target and risk context. A command may execute on
// a related host, so its target must not rename the alert's resource.
alertEvidence := false
switch change.Kind {
case unifiedresources.ChangeAlertFired, unifiedresources.ChangeAlertResolved,
unifiedresources.ChangeAlertAcknowledged, unifiedresources.ChangeAlertUnacknowledged,
unifiedresources.ChangeAlertSnoozed, unifiedresources.ChangeAlertUnsnoozed:
alertEvidence = true
}
if resourceID := strings.TrimSpace(change.ResourceID); resourceID != "" && (incident.ResourceID == "" || alertEvidence) {
incident.ResourceID = resourceID
}
if alertType, ok := stringMetadata(change.Metadata, unifiedresources.MetadataAlertType); ok && (incident.AlertType == "" || alertEvidence) {
incident.AlertType = alertType
}
if level, ok := stringMetadata(change.Metadata, unifiedresources.MetadataAlertLevel); ok && (incident.Level == "" || alertEvidence) {
incident.Level = level
}
if message, ok := stringMetadata(change.Metadata, unifiedresources.MetadataAlertMessage); ok && (incident.Message == "" || alertEvidence) {
incident.Message = message
}
if resourceType, ok := stringMetadata(change.Metadata, "resourceType"); ok && (incident.ResourceType == "" || alertEvidence) {
incident.ResourceType = resourceType
}
if incident.OpenedAt.IsZero() {
incident.OpenedAt = incidentEventTimestamp(change)
}
}
func applyProjectedIncidentState(incident *Incident, events []IncidentEvent) {
if incident == nil || len(events) == 0 {
return
}
for _, event := range events {
if incident.OpenedAt.IsZero() || (!event.Timestamp.IsZero() && event.Timestamp.Before(incident.OpenedAt)) {
incident.OpenedAt = event.Timestamp
}
switch event.Type {
case IncidentEventAlertFired:
incident.Status = IncidentStatusOpen
incident.ClosedAt = nil
case IncidentEventAlertAcknowledged:
incident.Acknowledged = true
timestamp := event.Timestamp
incident.AckTime = ×tamp
incident.AckUser = eventActor(event)
case IncidentEventAlertUnacknowledged:
incident.Acknowledged = false
incident.AckTime = nil
incident.AckUser = ""
case IncidentEventAlertResolved:
incident.Status = IncidentStatusResolved
timestamp := event.Timestamp
incident.ClosedAt = ×tamp
}
}
}
func incidentEventFromResourceChange(change unifiedresources.ResourceChange) (IncidentEvent, bool) {
eventType, ok := incidentEventTypeFromChangeKind(change.Kind)
if !ok {
return IncidentEvent{}, false
}
details := cloneIncidentEventDetails(change.Metadata)
if user := strings.TrimSpace(change.Actor); user != "" {
switch eventType {
case IncidentEventAlertAcknowledged, IncidentEventAlertUnacknowledged,
IncidentEventAlertSnoozed, IncidentEventAlertUnsnoozed:
details["user"] = user
}
}
evidence := change
evidence.Metadata = cloneIncidentEventDetails(change.Metadata)
evidence.RelatedResources = append([]string(nil), change.RelatedResources...)
if change.OccurredAt != nil {
occurred := *change.OccurredAt
evidence.OccurredAt = &occurred
}
return IncidentEvent{
Source: "canonical_resource_history",
Evidence: &evidence,
ID: strings.TrimSpace(change.ID),
Type: eventType,
Timestamp: incidentEventTimestamp(change),
Summary: incidentEventSummaryFromChange(change, eventType),
Details: details,
}, true
}
func incidentEventTypeFromChangeKind(kind unifiedresources.ChangeKind) (IncidentEventType, bool) {
switch kind {
case unifiedresources.ChangeAlertFired:
return IncidentEventAlertFired, true
case unifiedresources.ChangeAlertAcknowledged:
return IncidentEventAlertAcknowledged, true
case unifiedresources.ChangeAlertUnacknowledged:
return IncidentEventAlertUnacknowledged, true
case unifiedresources.ChangeAlertSnoozed:
return IncidentEventAlertSnoozed, true
case unifiedresources.ChangeAlertUnsnoozed:
return IncidentEventAlertUnsnoozed, true
case unifiedresources.ChangeAlertResolved:
return IncidentEventAlertResolved, true
case unifiedresources.ChangeCommandExecuted:
return IncidentEventCommand, true
case unifiedresources.ChangeRunbookExecuted:
return IncidentEventRunbook, true
default:
return "", false
}
}
func incidentEventSummaryFromChange(change unifiedresources.ResourceChange, eventType IncidentEventType) string {
switch eventType {
case IncidentEventAlertFired:
// The source owns the alert condition and comparison direction. Numeric
// placeholders cannot establish either, including for legacy records.
if message, ok := stringMetadata(change.Metadata, unifiedresources.MetadataAlertMessage); ok {
return message
}
if reason := strings.TrimSpace(change.Reason); reason != "" {
return reason
}
if alertType, ok := stringMetadata(change.Metadata, unifiedresources.MetadataAlertType); ok {
if level, ok := stringMetadata(change.Metadata, unifiedresources.MetadataAlertLevel); ok {
return fmt.Sprintf("Alert triggered: %s (%s)", alertType, level)
}
return "Alert triggered: " + alertType
}
return "Alert triggered"
case IncidentEventAlertAcknowledged:
return "Alert acknowledged"
case IncidentEventAlertUnacknowledged:
return "Alert unacknowledged"
case IncidentEventAlertSnoozed:
return "Alert snoozed"
case IncidentEventAlertUnsnoozed:
return "Alert notifications resumed"
case IncidentEventAlertResolved:
// A handover closes the alert without recovering; its change reason
// says so and where the condition went.
if _, ok := stringMetadata(change.Metadata, unifiedresources.MetadataAlertResolution); ok {
if reason := strings.TrimSpace(change.Reason); reason != "" {
return reason
}
}
return "Alert resolved"
default:
if summary := strings.TrimSpace(change.Reason); summary != "" {
return summary
}
return unifiedresources.ChangeKindLabel(change.Kind)
}
}
func incidentEventTimestamp(change unifiedresources.ResourceChange) time.Time {
if change.OccurredAt != nil && !change.OccurredAt.IsZero() {
return change.OccurredAt.UTC()
}
if !change.ObservedAt.IsZero() {
return change.ObservedAt.UTC()
}
return time.Time{}
}
func projectedAlertIdentifier(change unifiedresources.ResourceChange) string {
value, _ := stringMetadata(change.Metadata, unifiedresources.MetadataAlertIdentifier)
return value
}
func stringMetadata(metadata map[string]any, key string) (string, bool) {
if len(metadata) == 0 {
return "", false
}
value, ok := metadata[key]
if !ok {
return "", false
}
str, ok := value.(string)
if !ok {
return "", false
}
str = strings.TrimSpace(str)
if str == "" {
return "", false
}
return str, true
}
func eventActor(event IncidentEvent) string {
if event.Details == nil {
return ""
}
value, ok := event.Details["user"].(string)
if !ok {
return ""
}
return strings.TrimSpace(value)
}
func isCanonicalProjectedIncidentEventType(eventType IncidentEventType) bool {
switch eventType {
case IncidentEventAlertFired,
IncidentEventAlertAcknowledged,
IncidentEventAlertUnacknowledged,
IncidentEventAlertSnoozed,
IncidentEventAlertUnsnoozed,
IncidentEventAlertResolved,
IncidentEventCommand,
IncidentEventRunbook:
return true
default:
return false
}
}
func isSnapshotProjectionEvent(event IncidentEvent) bool {
if event.Details == nil {
return false
}
source, ok := event.Details["projection_source"].(string)
return ok && source == incidentSnapshotSource
}
func sortIncidentEvents(events []IncidentEvent) {
sort.Slice(events, func(i, j int) bool {
if events[i].Timestamp.Equal(events[j].Timestamp) {
iRank := incidentEventTimestampRank(events[i].Type)
jRank := incidentEventTimestampRank(events[j].Type)
if iRank != jRank {
return iRank < jRank
}
return events[i].ID < events[j].ID
}
return events[i].Timestamp.Before(events[j].Timestamp)
})
}
func incidentEventTimestampRank(eventType IncidentEventType) int {
switch eventType {
case IncidentEventAlertFired:
return 0
case IncidentEventAlertResolved:
return 2
default:
return 1
}
}
func cloneIncidentEvent(event IncidentEvent) IncidentEvent {
cloned := event
cloned.Details = cloneIncidentEventDetails(event.Details)
if event.Evidence != nil {
evidence := *event.Evidence
evidence.Metadata = cloneIncidentEventDetails(evidence.Metadata)
evidence.RelatedResources = append([]string(nil), evidence.RelatedResources...)
if evidence.OccurredAt != nil {
occurred := *evidence.OccurredAt
evidence.OccurredAt = &occurred
}
cloned.Evidence = &evidence
}
return cloned
}
func cloneIncidentEventDetails(details map[string]any) map[string]interface{} {
if len(details) == 0 {
return map[string]interface{}{}
}
cloned := make(map[string]interface{}, len(details))
for key, value := range details {
cloned[key] = cloneIncidentDetailValue(value)
}
return cloned
}
func cloneIncidentDetailValue(value any) any {
switch v := value.(type) {
case map[string]any:
return cloneIncidentEventDetails(v)
case []any:
out := make([]any, len(v))
for i, item := range v {
out[i] = cloneIncidentDetailValue(item)
}
return out
case []string:
return append([]string(nil), v...)
default:
return value
}
}
func newIncidentShellFromAlert(alert *alerts.Alert) *incidentShell {
openedAt := alert.StartTime
if openedAt.IsZero() {
openedAt = time.Now()
}
return &incidentShell{
ID: generateIncidentID(),
AlertIdentifier: alert.ID,
AlertType: alert.Type,
Level: string(alert.Level),
ResourceID: alert.ResourceID,
ResourceName: alert.ResourceName,
Node: alert.Node,
Instance: alert.Instance,
Message: alert.Message,
OpenedAt: openedAt,
Events: make([]IncidentEvent, 0),
}
}
func updateIncidentShellFromAlert(shell *incidentShell, alert *alerts.Alert) {
if shell == nil || alert == nil {
return
}
shell.AlertType = alert.Type
shell.Level = string(alert.Level)
shell.ResourceID = alert.ResourceID
shell.ResourceName = alert.ResourceName
shell.Node = alert.Node
shell.Instance = alert.Instance
shell.Message = alert.Message
}
// Lifecycle snapshots carry the exact occurrence start. Unlike legacy read
// repair, they must not use a time tolerance or select an unrelated open/latest
// incident: delayed transitions and replay can arrive after a genuine recurrence.
func (s *IncidentStore) findLifecycleOccurrenceLocked(alert *alerts.Alert) *incidentShell {
if alert.ID == "" {
return nil
}
if alert.StartTime.IsZero() {
// Legacy callers without an occurrence key retain best-effort matching.
return s.findLatestIncidentByAlertIdentifierLocked(alert.ID)
}
for i := len(s.incidents) - 1; i >= 0; i-- {
shell := s.incidents[i]
if shell != nil && shell.AlertIdentifier == alert.ID && shell.OpenedAt.Equal(alert.StartTime) {
return shell
}
}
return nil
}
func (s *IncidentStore) ensureIncidentForAlertLocked(alert *alerts.Alert) *incidentShell {
shell := s.findLifecycleOccurrenceLocked(alert)
if shell == nil {
shell = newIncidentShellFromAlert(alert)
s.incidents = append(s.incidents, shell)
}
updateIncidentShellFromAlert(shell, alert)
return shell
}
func resetDerivedIncidentState(incident *Incident) {
if incident == nil {
return
}
incident.Status = IncidentStatusOpen
incident.ClosedAt = nil
incident.Acknowledged = false
incident.AckUser = ""
incident.AckTime = nil
}
func (s *IncidentStore) addEventLocked(shell *incidentShell, eventType IncidentEventType, summary string, details map[string]interface{}) {
s.addEventAtLocked(shell, eventType, time.Now(), summary, details)
}
func (s *IncidentStore) addEventAtLocked(shell *incidentShell, eventType IncidentEventType, occurredAt time.Time, summary string, details map[string]interface{}) {
if shell == nil {
return
}
if summary == "" {
summary = string(eventType)
}
if occurredAt.IsZero() {
occurredAt = time.Now()
}
event := IncidentEvent{
ID: generateIncidentEventID(),
Type: eventType,
Timestamp: occurredAt,
Summary: summary,
Details: details,
}
if eventType == IncidentEventNote {
event.Source = "operator_note"
}
if isSnapshotProjectionEvent(event) {
event.Source = incidentSnapshotSource
}
shell.Events = append(shell.Events, event)
if s.maxEvents > 0 && len(shell.Events) > s.maxEvents {
shell.Events = shell.Events[len(shell.Events)-s.maxEvents:]
}
}
func (s *IncidentStore) findOpenIncidentByAlertIdentifierLocked(alertIdentifier string) *incidentShell {
if alertIdentifier == "" {
return nil
}
for i := len(s.incidents) - 1; i >= 0; i-- {
shell := s.incidents[i]
if shell != nil && shell.AlertIdentifier == alertIdentifier && incidentOccurrenceClosedAt(shell) == nil {
return shell
}
}
return nil
}
func (s *IncidentStore) findLatestIncidentByAlertIdentifierLocked(alertIdentifier string) *incidentShell {
if alertIdentifier == "" {
return nil
}
for i := len(s.incidents) - 1; i >= 0; i-- {
shell := s.incidents[i]
if shell != nil && shell.AlertIdentifier == alertIdentifier {
return shell
}
}
return nil
}
func (s *IncidentStore) findIncidentByIDLocked(incidentID string) *incidentShell {
if incidentID == "" {
return nil
}
for i := len(s.incidents) - 1; i >= 0; i-- {
shell := s.incidents[i]
if shell != nil && shell.ID == incidentID {
return shell
}
}
return nil
}
func (s *IncidentStore) trimLocked() {
if s.maxAge > 0 {
cutoff := time.Now().Add(-s.maxAge)
filtered := make([]*incidentShell, 0, len(s.incidents))
for _, shell := range s.incidents {
if shell == nil {
continue
}
compareTime := shell.OpenedAt
if closedAt := incidentOccurrenceClosedAt(shell); closedAt != nil {
compareTime = *closedAt
}
if compareTime.After(cutoff) {
filtered = append(filtered, shell)
}
}
s.incidents = filtered
}
if s.maxIncidents > 0 && len(s.incidents) > s.maxIncidents {
sort.Slice(s.incidents, func(i, j int) bool {
return s.incidents[i].OpenedAt.Before(s.incidents[j].OpenedAt)
})
if len(s.incidents) > s.maxIncidents {
s.incidents = s.incidents[len(s.incidents)-s.maxIncidents:]
}
}
}
// trimLockedPreserving applies the ordinary retention policy while keeping the
// occurrence currently being read-repaired. This lets a user open an older
// retained alert-history row even when the bounded incident cache is already
// full; another least-recent occurrence becomes repairable on demand instead.
func (s *IncidentStore) trimLockedPreserving(protected *incidentShell) {
s.trimLocked()
if protected == nil {
return
}
for _, shell := range s.incidents {
if shell == protected {
return
}
}
if s.maxAge > 0 {
compareTime := protected.OpenedAt
if closedAt := incidentOccurrenceClosedAt(protected); closedAt != nil {
compareTime = *closedAt
}
if compareTime.Before(time.Now().Add(-s.maxAge)) {
return
}
}
s.incidents = append(s.incidents, protected)
if s.maxIncidents <= 0 || len(s.incidents) <= s.maxIncidents {
return
}
sort.Slice(s.incidents, func(i, j int) bool {
return s.incidents[i].OpenedAt.Before(s.incidents[j].OpenedAt)
})
for len(s.incidents) > s.maxIncidents {
removeAt := -1
for index, shell := range s.incidents {
if shell != protected {
removeAt = index
break
}
}
if removeAt < 0 {
break
}
s.incidents = append(s.incidents[:removeAt], s.incidents[removeAt+1:]...)
}
}
func (s *IncidentStore) saveAsync() {
if s.dataDir == "" || s.filePath == "" {
return
}
// Coalesce: each save marshals the whole store, so a burst of mutations
// (lifecycle replay, alert storms) must not queue one full serialization
// per call. One pending save is enough — it snapshots the store when it
// runs, so it covers every mutation made before it started, and any
// mutation after the pending flag clears queues exactly one more save.
if !s.savePending.CompareAndSwap(false, true) {
return
}
s.saveWG.Add(1)
go func() {
defer s.saveWG.Done()
if err := s.saveToDisk(); err != nil {
log.Warn().Err(err).Msg("failed to save incident history")
}
}()
}
// flush blocks until every asynchronous save queued by a mutation has
// finished. A queued save still creates the data directory and writes its
// temp file after the mutation returns, so callers that remove or inspect the
// data directory (including tests) join it first. Stop mutating the store
// before calling flush: WaitGroup forbids a new save being queued while a
// previous Wait is still returning.
func (s *IncidentStore) flush() {
s.saveWG.Wait()
}
func (s *IncidentStore) saveToDisk() error {
s.saveMu.Lock()
defer s.saveMu.Unlock()
// Clear the coalescing flag only once this save actually starts: the
// snapshot below covers every mutation made before this point, and a
// mutation after it queues exactly one follow-up save.
s.savePending.Store(false)
if s.dataDir == "" || s.filePath == "" {
return nil
}
if err := os.MkdirAll(s.dataDir, 0700); err != nil {
return err
}
s.mu.RLock()
snapshot := make([]*incidentShell, 0, len(s.incidents))
for _, shell := range s.incidents {
snapshot = append(snapshot, cloneIncidentShell(shell))
}
s.mu.RUnlock()
data, err := json.Marshal(snapshot)
if err != nil {
return err
}
// Coalescing handles concurrent bursts, but sequential lifecycle replays
// can also produce identical snapshots. Compare the actual recovery file
// rather than caching a hash: restarts, failed writes and missing files
// must not prevent the next checkpoint from restoring durable state.
if file, err := os.Open(s.filePath); err == nil {
// One extra byte detects a longer file without an unbounded read.
existing, readErr := io.ReadAll(io.LimitReader(file, int64(len(data))+1))
file.Close()
if readErr == nil && bytes.Equal(existing, data) {
return nil
}
}
tmpFile := s.filePath + ".tmp"
if err := os.WriteFile(tmpFile, data, 0600); err != nil {
return err
}
if err := os.Rename(tmpFile, s.filePath); err != nil {
return err
}
s.savesCompleted.Add(1)
return nil
}
func (s *IncidentStore) loadFromDisk() error {
if s.filePath == "" {
return nil
}
info, err := os.Stat(s.filePath)
if err != nil {
if os.IsNotExist(err) {
return nil
}
return err
}
if info.Size() > maxIncidentFileSize {
return fmt.Errorf("incident history file too large (%d bytes)", info.Size())
}
data, err := os.ReadFile(s.filePath)
if err != nil {
return err
}
var incidents []*incidentShell
if err := json.Unmarshal(data, &incidents); err != nil {
return err
}
for _, shell := range incidents {
normalizeIncidentShellState(shell)
}
s.incidents = incidents
s.trimLocked()
return nil
}
func incidentFromShell(shell *incidentShell) *Incident {
if shell == nil {
return nil
}
incident := &Incident{
ID: shell.ID,
AlertIdentifier: shell.AlertIdentifier,
AlertType: shell.AlertType,
Level: shell.Level,
ResourceID: shell.ResourceID,
ResourceName: shell.ResourceName,
ResourceType: shell.ResourceType,
Node: shell.Node,
Instance: shell.Instance,
Message: shell.Message,
Status: IncidentStatusOpen,
OpenedAt: shell.OpenedAt,
Events: cloneIncidentEvents(shell.Events),
}
if shell.OpenedAt.IsZero() {
incident.Status = IncidentStatusUnknown
}
if shell.OccurrenceClosedAt != nil {
closed := *shell.OccurrenceClosedAt
incident.ClosedAt = &closed
incident.Status = IncidentStatusResolved
}
applyProjectedIncidentState(incident, incident.Events)
incident.OpenedAt = shell.OpenedAt
return incident
}
func cloneIncident(src *Incident) *Incident {
if src == nil {
return nil
}
clone := *src
if src.AckTime != nil {
t := *src.AckTime
clone.AckTime = &t
}
if src.ClosedAt != nil {
t := *src.ClosedAt
clone.ClosedAt = &t
}
clone.Events = cloneIncidentEvents(src.Events)
if src.History != nil {
history := *src.History
clone.History = &history
}
return &clone
}
func cloneIncidentShell(src *incidentShell) *incidentShell {
if src == nil {
return nil
}
clone := *src
if src.OccurrenceClosedAt != nil {
t := *src.OccurrenceClosedAt
clone.OccurrenceClosedAt = &t
}
if src.OccurrenceRefiredAt != nil {
clone.OccurrenceRefiredAt = cloneTime(*src.OccurrenceRefiredAt)
}
clone.Events = cloneIncidentEvents(src.Events)
return &clone
}
func cloneIncidentEvents(src []IncidentEvent) []IncidentEvent {
if len(src) == 0 {
return nil
}
cloned := make([]IncidentEvent, len(src))
for i, event := range src {
cloned[i] = cloneIncidentEvent(event)
}
return cloned
}
func incidentOccurrenceClosedAt(shell *incidentShell) *time.Time {
if shell == nil {
return nil
}
return shell.OccurrenceClosedAt
}
func normalizeIncidentShellState(shell *incidentShell) {
if shell == nil {
return
}
sortIncidentEvents(shell.Events)
if shell.OccurrenceClosedAt == nil {
for i := len(shell.Events) - 1; i >= 0; i-- {
if shell.Events[i].Type == IncidentEventAlertResolved && (shell.OccurrenceRefiredAt == nil || !shell.Events[i].Timestamp.Before(*shell.OccurrenceRefiredAt)) {
shell.OccurrenceClosedAt = cloneTime(shell.Events[i].Timestamp)
break
}
}
}
}
func hasIncidentEventType(events []IncidentEvent, types ...IncidentEventType) bool {
if len(events) == 0 || len(types) == 0 {
return false
}
wanted := make(map[IncidentEventType]struct{}, len(types))
for _, eventType := range types {
wanted[eventType] = struct{}{}
}
for _, event := range events {
if _, ok := wanted[event.Type]; ok {
return true
}
}
return false
}
func cloneTime(value time.Time) *time.Time {
if value.IsZero() {
return nil
}
cloned := value
return &cloned
}
// IDs are opaque persisted references. Store-local locks cannot protect a
// process-wide counter across tenant monitors, and a wrapping counter can
// collide within a burst or after a restart. Random UUIDs need no shared state
// and leave previously persisted IDs usable without a migration.
func generateIncidentID() string {
return "inc-" + uuid.NewString()
}
func generateIncidentEventID() string {
return "inc-evt-" + uuid.NewString()
}
func formatAlertSummary(alert *alerts.Alert) string {
if alert == nil {
return "Alert triggered"
}
summary := fmt.Sprintf("Alert triggered: %s (%s)", alert.Type, alert.Level)
if alert.Value > 0 || alert.Threshold > 0 {
summary = fmt.Sprintf("Alert triggered: %s (%s %.1f >= %.1f)", alert.Type, alert.Level, alert.Value, alert.Threshold)
}
return summary
}