Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Join and checkpoint alert-history replay before monitor shutdown
pulse-triage[bot]d32f493de986
diff --git a/internal/monitoring/alert_projection_replay_test.go b/internal/monitoring/alert_projection_replay_test.goindex d4e0a736a6..40511f68f4 100644--- a/internal/monitoring/alert_projection_replay_test.go+++ b/internal/monitoring/alert_projection_replay_test.go@@ -1,6 +1,7 @@package monitoringimport (+ "context""encoding/json""fmt""sync/atomic"@@ -73,8 +74,8 @@ func TestImportedTimelineReplayDoesNotReadDiscardedProjections(t *testing.T) {incidents := memory.NewIncidentStore(memory.IncidentStoreConfig{})incidents.SetResourceTimelineStore(history)m := &Monitor{alertManager: manager, resourceStore: adapter, incidentStore: incidents}- m.replayAlertLifecycleProjections()- m.replayAlertLifecycleProjections() // the watermark must make this a no-op+ m.replayAlertLifecycleProjections(context.Background())+ m.replayAlertLifecycleProjections(context.Background()) // the watermark must make this a no-opreads := history.filteredReads.Load() + history.identityReads.Load() + history.recentReads.Load()t.Logf("imported=%d discarded-projection reads: filtered=%d identities=%d recent=%d", len(snapshots),history.filteredReads.Load(), history.identityReads.Load(), history.recentReads.Load())@@ -136,7 +137,7 @@ func TestImportedTimelineReplayCheckpointAndCacheRecovery(t *testing.T) {incidents := memory.NewIncidentStore(memory.IncidentStoreConfig{MaxIncidents: 2})incidents.SetResourceTimelineStore(adapter)m := &Monitor{alertManager: manager, resourceStore: recorder, incidentStore: incidents}- m.replayAlertLifecycleProjections()+ m.replayAlertLifecycleProjections(context.Background())require.Equal(t, len(snapshots), recorder.imports)require.Positive(t, recorder.checkpoint, "checkpoint must persist before the next import")complete := manager.LifecycleProjectionWatermark(alertLifecycleProjectionConsumer)@@ -149,7 +150,7 @@ func TestImportedTimelineReplayCheckpointAndCacheRecovery(t *testing.T) {repaired.SetResourceTimelineStore(adapter)resume := &Monitor{alertManager: manager, resourceStore: recorder, incidentStore: repaired}before := recorder.imports- resume.replayAlertLifecycleProjections()+ resume.replayAlertLifecycleProjections(context.Background())require.Equal(t, before+1, recorder.imports)require.Equal(t, complete, manager.LifecycleProjectionWatermark(alertLifecycleProjectionConsumer))changes, err := store.GetRecentChanges("shared-resource", time.Time{}, 2*len(snapshots)+1)diff --git a/internal/monitoring/alert_projection_shutdown_test.go b/internal/monitoring/alert_projection_shutdown_test.goindex ac75109905..5119db70dd 100644--- a/internal/monitoring/alert_projection_shutdown_test.go+++ b/internal/monitoring/alert_projection_shutdown_test.go@@ -2,14 +2,22 @@ package monitoringimport ("context"+ "encoding/json"+ "fmt"+ "os"+ "path/filepath""sync""sync/atomic""testing""time"+ "github.com/rcourtman/pulse-go-rewrite/internal/ai/memory""github.com/rcourtman/pulse-go-rewrite/internal/alerts"+ "github.com/rcourtman/pulse-go-rewrite/internal/alerts/eventlog""github.com/rcourtman/pulse-go-rewrite/internal/config"+ "github.com/rcourtman/pulse-go-rewrite/internal/models""github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"+ "github.com/stretchr/testify/require")// Hold a real replay at the exported canonical writer boundary. No database@@ -113,3 +121,208 @@ func TestMonitorStartJoinsAlertProjectionBeforeReturning(t *testing.T) {t.Fatalf("admitted snapshot wrote %d changes, want its complete fired/acknowledged/resolved occurrence", got)}}++func projectionCancellation(m *Monitor) <-chan struct{} {+ m.alertProjectionMu.Lock()+ defer m.alertProjectionMu.Unlock()+ return m.alertProjectionCtx.Done()+}++func TestMonitorStopCheckpointsWholeAlertOccurrenceAndResumesTail(t *testing.T) {+ manager := alerts.NewManagerWithDataDir(t.TempDir(), alerts.WithoutPersistedAlertRestore())+ t.Cleanup(manager.Stop)+ eventDir := t.TempDir()+ logStore, err := eventlog.Open(eventDir)+ require.NoError(t, err)+ manager.SetEventLog(logStore)+ first := shutdownHistorySnapshots()[0]+ snapshots := make([]alerts.Alert, 3)+ for i := range snapshots {+ snapshot := first.Clone()+ snapshot.ID = fmt.Sprintf("shutdown-occurrence-%d", i)+ snapshot.StartTime = first.StartTime.Add(time.Duration(i) * time.Minute)+ ack := snapshot.StartTime.Add(time.Second)+ snapshot.AckTime, snapshot.LastSeen = &ack, snapshot.StartTime.Add(2*time.Second)+ snapshots[i] = *snapshot+ payload, err := json.Marshal(snapshot)+ require.NoError(t, err)+ require.NoError(t, logStore.ImportEvents([]eventlog.Event{{Type: eventlog.TypeHistoryImported,+ AlertID: snapshot.ID, ResourceID: snapshot.ResourceID, OccurredAt: snapshot.LastSeen, Snapshot: payload}}))+ }+ firstEvents, err := manager.AlertEvents(eventlog.Filter{AlertID: snapshots[0].ID})+ require.NoError(t, err)+ require.Len(t, firstEvents, 1)++ recorder, unblock := blockedProjectionRecorder(t)+ incidents := memory.NewIncidentStore(memory.IncidentStoreConfig{})+ incidents.SetResourceTimelineStore(recorder)+ m := &Monitor{alertManager: manager, incidentStore: incidents, resourceStore: recorder}+ m.scheduleAlertProjectionCatchUp()+ t.Cleanup(func() { unblock(); m.alertProjectionWG.Wait() })+ select {+ case <-recorder.entered:+ case <-time.After(5 * time.Second):+ t.Fatal("replay did not reach its first canonical write")+ }+ cancelled := projectionCancellation(m)+ stopped := make(chan struct{})+ go func() { m.Stop(); close(stopped) }()+ select {+ case <-cancelled:+ case <-time.After(5 * time.Second):+ t.Fatal("Stop did not close replay admission")+ }+ select {+ case <-stopped:+ t.Fatal("Stop closed history while an occurrence was only partly applied")+ case <-time.After(100 * time.Millisecond):+ }+ unblock()+ select {+ case <-stopped:+ case <-time.After(5 * time.Second):+ t.Fatal("Stop did not join its completed occurrence")+ }+ require.EqualValues(t, 3, recorder.writes.Load(), "cancel only between occurrences, not between their three projections")++ // Stop really closed the log. Reopen through the public interface, then+ // attach a fresh monitor and cold incident cache to the retained evidence.+ resumedLog, err := eventlog.Open(eventDir)+ require.NoError(t, err)+ resumedManager := alerts.NewManagerWithDataDir(t.TempDir(), alerts.WithoutPersistedAlertRestore())+ resumedManager.SetEventLog(resumedLog)+ t.Cleanup(resumedManager.Stop)+ checkpoint := resumedManager.LifecycleProjectionWatermark(alertLifecycleProjectionConsumer)+ require.Equal(t, firstEvents[0].ID, checkpoint, "checkpoint only the fully applied first occurrence")+ resumedIncidents := memory.NewIncidentStore(memory.IncidentStoreConfig{})+ resumedIncidents.SetResourceTimelineStore(recorder)+ resumed := &Monitor{alertManager: resumedManager, incidentStore: resumedIncidents, resourceStore: recorder}+ resumed.scheduleAlertProjectionCatchUp()+ resumed.alertProjectionWG.Wait()+ require.EqualValues(t, 9, recorder.writes.Load(), "resume only the two unapplied occurrences")+ resumed.scheduleAlertProjectionCatchUp()+ resumed.alertProjectionWG.Wait()+ require.EqualValues(t, 9, recorder.writes.Load(), "completed replay must remain a no-op")+ for _, snapshot := range snapshots {+ page, err := resumedIncidents.QueryIncidents(memory.IncidentQuery{AlertIdentifier: snapshot.ID, StartedAt: snapshot.StartTime})+ require.NoError(t, err)+ require.Len(t, page.Incidents, 1)+ incident := page.Incidents[0]+ require.Equal(t, memory.IncidentStatusResolved, incident.Status)+ require.Equal(t, &snapshot.LastSeen, incident.ClosedAt)+ require.True(t, incident.Acknowledged)+ require.Equal(t, snapshot.AckUser, incident.AckUser)+ require.Len(t, incident.Events, 3)+ }+ t.Logf("interrupted watermark=%d; resumed watermark=%d; complete canonical writes=%d", checkpoint,+ resumedManager.LifecycleProjectionWatermark(alertLifecycleProjectionConsumer), recorder.writes.Load())+ resumed.Stop()+}++func TestAlertProjectionShutdownRefusesQueuedAndLateAdmissions(t *testing.T) {+ manager := alerts.NewManagerWithDataDir(t.TempDir(), alerts.WithoutPersistedAlertRestore())+ t.Cleanup(manager.Stop)+ importReplaySnapshots(t, manager, shutdownHistorySnapshots())+ recorder, unblock := blockedProjectionRecorder(t)+ unblock() // Only the replay serialization gate below holds these requests.+ m := &Monitor{alertManager: manager, incidentStore: memory.NewIncidentStore(memory.IncidentStoreConfig{}), resourceStore: recorder}+ m.alertProjectionReplayMu.Lock()+ m.scheduleAlertProjectionCatchUp()+ cancelled := projectionCancellation(m)+ stopped := make(chan struct{})+ go func() { m.stopAlertProjectionCatchUp(); close(stopped) }()+ var callers sync.WaitGroup+ for i := 0; i < 64; i++ {+ callers.Go(m.scheduleAlertProjectionCatchUp)+ }+ select {+ case <-cancelled:+ case <-time.After(5 * time.Second):+ m.alertProjectionReplayMu.Unlock()+ t.Fatal("shutdown did not cancel queued projection work")+ }+ callers.Wait()+ m.alertProjectionReplayMu.Unlock()+ select {+ case <-stopped:+ case <-time.After(5 * time.Second):+ t.Fatal("cancelled queued work did not join")+ }+ for i := 0; i < 64; i++ {+ m.scheduleAlertProjectionCatchUp()+ }+ m.alertProjectionWG.Wait()+ require.Zero(t, recorder.writes.Load(), "queued or late catch-up cannot revive a stopped writer")+ require.Zero(t, manager.LifecycleProjectionWatermark(alertLifecycleProjectionConsumer), "unapplied evidence must remain replayable")+}++func TestCancelledAlertReconciliationDoesNotReadOrWriteHistory(t *testing.T) {+ manager := alerts.NewManagerWithDataDir(t.TempDir(), alerts.WithoutPersistedAlertRestore())+ t.Cleanup(manager.Stop)+ cfg := manager.GetConfig()+ cfg.Enabled, cfg.ActivationState = true, alerts.ActivationPending+ cfg.TimeThresholds = map[string]int{}+ manager.UpdateConfig(cfg)+ vm := models.VM{ID: "cancelled-reconciliation", Name: "Synthetic VM", Node: "node", Instance: "pve", Status: "stopped"}+ manager.CheckGuest(vm, vm.Instance)+ manager.CheckGuest(vm, vm.Instance)+ require.Len(t, manager.GetActiveAlerts(), 1)+ recorder, unblock := blockedProjectionRecorder(t)+ unblock()+ history := &replayHistoryReadCounter{IncidentTimelineStore: recorder}+ incidents := memory.NewIncidentStore(memory.IncidentStoreConfig{})+ incidents.SetResourceTimelineStore(history)+ m := &Monitor{alertManager: manager, incidentStore: incidents, resourceStore: recorder}+ ctx, cancel := context.WithCancel(context.Background())+ cancel()+ m.reconcileActiveAlertTimelines(ctx)+ require.Zero(t, history.filteredReads.Load()+history.identityReads.Load()+history.recentReads.Load())+ changes, err := recorder.GetRecentChanges(vm.ID, time.Time{}, 10)+ require.NoError(t, err)+ require.Empty(t, changes)+}++func TestTenantDeletionRetainsBlockedAlertProjectionThenRemovesCompletedRuntime(t *testing.T) {+ configureAlertShutdownFixture(t)+ shortenTenantShutdownBudgets(t, 100*time.Millisecond, 100*time.Millisecond)+ root := t.TempDir()+ persistence := config.NewMultiTenantPersistence(root)+ require.NoError(t, persistence.SaveOrganization(&models.Organization{ID: "history-owner", DisplayName: "History owner"}))+ manager := NewMultiTenantMonitor(&config.Config{DataPath: root}, persistence, nil)+ recorder, unblock := blockedProjectionRecorder(t)+ manager.SetMonitorInitializer(func(m *Monitor) {+ importReplaySnapshots(t, m.GetAlertManager(), shutdownHistorySnapshots())+ m.SetResourceStore(recorder)+ })+ monitor, err := manager.GetMonitor("history-owner")+ require.NoError(t, err)+ t.Cleanup(func() { unblock(); manager.Stop() })+ select {+ case <-recorder.entered:+ case <-time.After(5 * time.Second):+ t.Fatal("tenant replay did not reach its canonical writer")+ }+ // The existing deletion budget must refuse a live replay writer without+ // closing its stores, dropping its owner or permitting lazy recreation.+ require.Error(t, manager.BeginTenantDeletion("history-owner"))+ retained, ok := manager.PeekMonitor("history-owner")+ require.True(t, ok)+ require.Same(t, monitor, retained)+ manager.FinishTenantDeletion("history-owner")+ _, err = manager.GetMonitor("history-owner")+ require.Error(t, err)+ orgPath := filepath.Join(root, "orgs", "history-owner")+ _, err = os.Stat(filepath.Join(orgPath, "org.json"))+ require.NoError(t, err, "blocked writer must retain organization bytes")+ unblock()+ require.NoError(t, manager.BeginTenantDeletion("history-owner"))+ require.NoError(t, persistence.DeleteOrganization("history-owner"))+ manager.FinishTenantDeletion("history-owner")+ _, ok = manager.PeekMonitor("history-owner")+ require.False(t, ok)+ _, err = manager.GetMonitor("history-owner")+ require.Error(t, err)+ _, err = os.Stat(orgPath)+ require.True(t, os.IsNotExist(err), "completed history writer must permit safe removal: %v", err)+ require.EqualValues(t, 3, recorder.writes.Load())+}diff --git a/internal/monitoring/monitor.go b/internal/monitoring/monitor.goindex 7cb9626f15..17da6faf26 100644--- a/internal/monitoring/monitor.go+++ b/internal/monitoring/monitor.go@@ -1344,7 +1344,11 @@ type Monitor struct {alertPushCallback func(*alerts.Alert)connectionsSnapshotLister func() []alerts.ConnectionSnapshot // returns platform connection snapshots for the connection-degraded checkincidentStore *memory.IncidentStore- alertProjectionReplayMu sync.Mutex // serializes lifecycle projection replay passes+ alertProjectionReplayMu sync.Mutex // serializes lifecycle projection replay passes+ alertProjectionMu sync.Mutex // closes admission before joining background replay+ alertProjectionCtx context.Context+ alertProjectionCancel context.CancelFunc+ alertProjectionStopped boolalertProjectionWG sync.WaitGroup // tracks scheduled background catch-up runsnotificationMgr *notifications.NotificationManagerdeadMan *deadManRuntime@@ -2254,6 +2258,10 @@ func (m *Monitor) Start(ctx context.Context, wsHub *websocket.Hub) {Msg("Starting monitoring loop")m.setRuntimeContext(ctx, wsHub)+ // Router attachment can start history replay before Start. Tenant teardown+ // must not observe this runtime as finished while that writer still owns+ // the canonical history or its durable replay watermark.+ defer m.stopAlertProjectionCatchUp()defer m.stopMockMetricsSampler()// Best-effort startup cleanup: when direct PBS is configured, remove legacy@@ -8282,6 +8290,10 @@ func (m *Monitor) Stop() {func (m *Monitor) stop() error {log.Info().Msg("stopping monitor")+ // Also cover monitors stopped before Start. Refuse new replay admissions,+ // finish the current occurrence, and checkpoint before closing any stores.+ m.stopAlertProjectionCatchUp()+// A trailing agent-report refresh must not run against stores closed below.m.stopAgentReportRefresh()diff --git a/internal/monitoring/monitor_alert_handling_test.go b/internal/monitoring/monitor_alert_handling_test.goindex d1d88aac64..d49bfc1b7f 100644--- a/internal/monitoring/monitor_alert_handling_test.go+++ b/internal/monitoring/monitor_alert_handling_test.go@@ -506,8 +506,8 @@ func TestActiveTimelineReconciliationIsIdempotent(t *testing.T) {resourceStore: unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(resourceStore)),}incidentStore.SetResourceTimelineStore(monitor.resourceStore.(memory.IncidentTimelineStore))- monitor.reconcileActiveAlertTimelines()- monitor.reconcileActiveAlertTimelines()+ monitor.reconcileActiveAlertTimelines(context.Background())+ monitor.reconcileActiveAlertTimelines(context.Background())active := manager.GetActiveAlerts()if len(active) != 1 {@@ -560,8 +560,8 @@ func TestLifecycleReplayRepairsResolvedIncidentTimeline(t *testing.T) {resourceStore: unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(resourceStore)),}incidentStore.SetResourceTimelineStore(monitor.resourceStore.(memory.IncidentTimelineStore))- monitor.replayAlertLifecycleProjections()- monitor.replayAlertLifecycleProjections()+ monitor.replayAlertLifecycleProjections(context.Background())+ monitor.replayAlertLifecycleProjections(context.Background())timeline := incidentStore.GetTimelineByAlertAt(alertID, startedAt)if timeline == nil || len(timeline.Events) != 2 {@@ -682,7 +682,7 @@ func TestLifecycleReplayWatermarkBoundsSubsequentPasses(t *testing.T) {// not advance the durable watermark, or resource-timeline projections for// those events would never materialize.partial := &Monitor{alertManager: manager, incidentStore: memory.NewIncidentStore(memory.IncidentStoreConfig{})}- partial.replayAlertLifecycleProjections()+ partial.replayAlertLifecycleProjections(context.Background())if got := manager.LifecycleProjectionWatermark(alertLifecycleProjectionConsumer); got != 0 {t.Fatalf("watermark after partial-surface replay = %d, want 0", got)}@@ -693,7 +693,7 @@ func TestLifecycleReplayWatermarkBoundsSubsequentPasses(t *testing.T) {incidentStore: memory.NewIncidentStore(memory.IncidentStoreConfig{}),resourceStore: unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(resourceStore)),}- monitor.replayAlertLifecycleProjections()+ monitor.replayAlertLifecycleProjections(context.Background())watermark := manager.LifecycleProjectionWatermark(alertLifecycleProjectionConsumer)if watermark == 0 {t.Fatal("watermark did not advance after full-surface replay")@@ -707,14 +707,14 @@ func TestLifecycleReplayWatermarkBoundsSubsequentPasses(t *testing.T) {incidentStore: memory.NewIncidentStore(memory.IncidentStoreConfig{}),resourceStore: unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(freshResources)),}- rerun.replayAlertLifecycleProjections()+ rerun.replayAlertLifecycleProjections(context.Background())if changes, err := freshResources.GetRecentChanges(vm.ID, time.Time{}, 10); err != nil || len(changes) != 0 {t.Fatalf("bounded pass wrote %d changes (err %v), want none", len(changes), err)}// Resetting the watermark forces a full repair replay for rebuilt stores.manager.StoreLifecycleProjectionWatermark(alertLifecycleProjectionConsumer, 0)- rerun.replayAlertLifecycleProjections()+ rerun.replayAlertLifecycleProjections(context.Background())changes, err := freshResources.GetRecentChanges(vm.ID, time.Time{}, 10)if err != nil {t.Fatalf("GetRecentChanges after watermark reset: %v", err)diff --git a/internal/monitoring/monitor_alerts.go b/internal/monitoring/monitor_alerts.goindex 1b19697fc4..bdf77b0fc1 100644--- a/internal/monitoring/monitor_alerts.go+++ b/internal/monitoring/monitor_alerts.go@@ -2,6 +2,7 @@ package monitoringimport ("context"+ "errors""fmt""strings""time"@@ -470,17 +471,43 @@ func (m *Monitor) scheduleAlertProjectionCatchUp() {if m == nil {return}+ m.alertProjectionMu.Lock()+ if m.alertProjectionStopped {+ m.alertProjectionMu.Unlock()+ return+ }+ if m.alertProjectionCtx == nil {+ m.alertProjectionCtx, m.alertProjectionCancel = context.WithCancel(context.Background())+ }+ ctx := m.alertProjectionCtx+ // Add and closing admission share a lock: shutdown must never race a new+ // Add against Wait on an empty group, including attachment before Start.m.alertProjectionWG.Add(1)+ m.alertProjectionMu.Unlock()go func() {defer m.alertProjectionWG.Done()defer recoverFromPanic("alertProjectionCatchUp")- m.replayAlertLifecycleProjections()- m.reconcileActiveAlertTimelines()+ m.replayAlertLifecycleProjections(ctx)+ m.reconcileActiveAlertTimelines(ctx)}()}-func (m *Monitor) replayAlertLifecycleProjections() {- if m == nil || m.alertManager == nil {+func (m *Monitor) stopAlertProjectionCatchUp() {+ m.alertProjectionMu.Lock()+ m.alertProjectionStopped = true+ if m.alertProjectionCancel != nil {+ m.alertProjectionCancel()+ }+ m.alertProjectionMu.Unlock()+ // An admitted canonical write cannot be interrupted safely. Start stays+ // live until it finishes; the existing tenant deletion budget then retains+ // the runtime and its data if that writer is blocked, rather than removing+ // its directory underneath it.+ m.alertProjectionWG.Wait()+}++func (m *Monitor) replayAlertLifecycleProjections(ctx context.Context) {+ if m == nil || m.alertManager == nil || ctx.Err() != nil {return}// Serialize passes instead of skipping: a second trigger waits for the@@ -488,6 +515,9 @@ func (m *Monitor) replayAlertLifecycleProjections() {// that need replay-complete semantics can rely on a finished call.m.alertProjectionReplayMu.Lock()defer m.alertProjectionReplayMu.Unlock()+ if ctx.Err() != nil {+ return+ }m.mu.RLock()_, hasRecorder := m.resourceStore.(canonicalResourceChangeRecorder)@@ -503,6 +533,12 @@ func (m *Monitor) replayAlertLifecycleProjections() {maxApplied := afterIDvisited := 0err := m.alertManager.ReplayLifecycleEvents(afterID, func(eventID int64, event alerts.LifecycleEvent) error {+ if err := ctx.Err(); err != nil {+ return err+ }+ // Apply the whole occurrence (including imported acknowledgement and+ // resolution) before advancing its watermark, even if shutdown arrives+ // during a canonical write. Cancellation only stops the next event.m.handleAlertLifecycleEvent(event)if eventID > maxApplied {maxApplied = eventID@@ -513,13 +549,17 @@ func (m *Monitor) replayAlertLifecycleProjections() {}return nil})- if err != nil {+ cancelled := err != nil && errors.Is(err, ctx.Err())+ if err != nil && !cancelled {log.Error().Err(err).Msg("failed to replay canonical alert lifecycle projections")return}if advance && maxApplied > afterID {m.alertManager.StoreLifecycleProjectionWatermark(alertLifecycleProjectionConsumer, maxApplied)}+ if cancelled {+ return // The next runtime resumes the unapplied tail, not the full log.+ }if visited > 0 {log.Info().Int("events", visited).@@ -529,18 +569,24 @@ func (m *Monitor) replayAlertLifecycleProjections() {}}-func (m *Monitor) reconcileActiveAlertTimelines() {- if m == nil || m.alertManager == nil || m.incidentStore == nil {+func (m *Monitor) reconcileActiveAlertTimelines(ctx context.Context) {+ if m == nil || m.alertManager == nil || m.incidentStore == nil || ctx.Err() != nil {return}activeAlerts := m.alertManager.GetActiveAlerts()for i := range activeAlerts {+ if ctx.Err() != nil {+ return+ }alert := &activeAlerts[i]page, err := m.incidentStore.QueryIncidents(memory.IncidentQuery{AlertIdentifier: alert.ID, StartedAt: alert.StartTime, Limit: 1})if err != nil {log.Warn().Err(err).Msg("Skipping alert timeline reconciliation because canonical history is unavailable")continue}+ if ctx.Err() != nil {+ return+ }var timeline *memory.Incidentif len(page.Incidents) > 0 {timeline = page.Incidents[0]