Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Join and checkpoint alert-history replay before monitor shutdown

pulse-triage[bot]d32f493de986
diff --git a/internal/monitoring/alert_projection_replay_test.go b/internal/monitoring/alert_projection_replay_test.go
index d4e0a736a6..40511f68f4 100644
--- a/internal/monitoring/alert_projection_replay_test.go
+++ b/internal/monitoring/alert_projection_replay_test.go
@@ -1,6 +1,7 @@
package monitoring
import (
+ "context"
"encoding/json"
"fmt"
"sync/atomic"
@@ -73,8 +74,8 @@ func TestImportedTimelineReplayDoesNotReadDiscardedProjections(t *testing.T) {
incidents := memory.NewIncidentStore(memory.IncidentStoreConfig{})
incidents.SetResourceTimelineStore(history)
m := &Monitor{alertManager: manager, resourceStore: adapter, incidentStore: incidents}
- m.replayAlertLifecycleProjections()
- m.replayAlertLifecycleProjections() // the watermark must make this a no-op
+ m.replayAlertLifecycleProjections(context.Background())
+ m.replayAlertLifecycleProjections(context.Background()) // the watermark must make this a no-op
reads := history.filteredReads.Load() + history.identityReads.Load() + history.recentReads.Load()
t.Logf("imported=%d discarded-projection reads: filtered=%d identities=%d recent=%d", len(snapshots),
history.filteredReads.Load(), history.identityReads.Load(), history.recentReads.Load())
@@ -136,7 +137,7 @@ func TestImportedTimelineReplayCheckpointAndCacheRecovery(t *testing.T) {
incidents := memory.NewIncidentStore(memory.IncidentStoreConfig{MaxIncidents: 2})
incidents.SetResourceTimelineStore(adapter)
m := &Monitor{alertManager: manager, resourceStore: recorder, incidentStore: incidents}
- m.replayAlertLifecycleProjections()
+ m.replayAlertLifecycleProjections(context.Background())
require.Equal(t, len(snapshots), recorder.imports)
require.Positive(t, recorder.checkpoint, "checkpoint must persist before the next import")
complete := manager.LifecycleProjectionWatermark(alertLifecycleProjectionConsumer)
@@ -149,7 +150,7 @@ func TestImportedTimelineReplayCheckpointAndCacheRecovery(t *testing.T) {
repaired.SetResourceTimelineStore(adapter)
resume := &Monitor{alertManager: manager, resourceStore: recorder, incidentStore: repaired}
before := recorder.imports
- resume.replayAlertLifecycleProjections()
+ resume.replayAlertLifecycleProjections(context.Background())
require.Equal(t, before+1, recorder.imports)
require.Equal(t, complete, manager.LifecycleProjectionWatermark(alertLifecycleProjectionConsumer))
changes, err := store.GetRecentChanges("shared-resource", time.Time{}, 2*len(snapshots)+1)
diff --git a/internal/monitoring/alert_projection_shutdown_test.go b/internal/monitoring/alert_projection_shutdown_test.go
index ac75109905..5119db70dd 100644
--- a/internal/monitoring/alert_projection_shutdown_test.go
+++ b/internal/monitoring/alert_projection_shutdown_test.go
@@ -2,14 +2,22 @@ package monitoring
import (
"context"
+ "encoding/json"
+ "fmt"
+ "os"
+ "path/filepath"
"sync"
"sync/atomic"
"testing"
"time"
+ "github.com/rcourtman/pulse-go-rewrite/internal/ai/memory"
"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
+ "github.com/rcourtman/pulse-go-rewrite/internal/alerts/eventlog"
"github.com/rcourtman/pulse-go-rewrite/internal/config"
+ "github.com/rcourtman/pulse-go-rewrite/internal/models"
"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
+ "github.com/stretchr/testify/require"
)
// Hold a real replay at the exported canonical writer boundary. No database
@@ -113,3 +121,208 @@ func TestMonitorStartJoinsAlertProjectionBeforeReturning(t *testing.T) {
t.Fatalf("admitted snapshot wrote %d changes, want its complete fired/acknowledged/resolved occurrence", got)
}
}
+
+func projectionCancellation(m *Monitor) <-chan struct{} {
+ m.alertProjectionMu.Lock()
+ defer m.alertProjectionMu.Unlock()
+ return m.alertProjectionCtx.Done()
+}
+
+func TestMonitorStopCheckpointsWholeAlertOccurrenceAndResumesTail(t *testing.T) {
+ manager := alerts.NewManagerWithDataDir(t.TempDir(), alerts.WithoutPersistedAlertRestore())
+ t.Cleanup(manager.Stop)
+ eventDir := t.TempDir()
+ logStore, err := eventlog.Open(eventDir)
+ require.NoError(t, err)
+ manager.SetEventLog(logStore)
+ first := shutdownHistorySnapshots()[0]
+ snapshots := make([]alerts.Alert, 3)
+ for i := range snapshots {
+ snapshot := first.Clone()
+ snapshot.ID = fmt.Sprintf("shutdown-occurrence-%d", i)
+ snapshot.StartTime = first.StartTime.Add(time.Duration(i) * time.Minute)
+ ack := snapshot.StartTime.Add(time.Second)
+ snapshot.AckTime, snapshot.LastSeen = &ack, snapshot.StartTime.Add(2*time.Second)
+ snapshots[i] = *snapshot
+ payload, err := json.Marshal(snapshot)
+ require.NoError(t, err)
+ require.NoError(t, logStore.ImportEvents([]eventlog.Event{{Type: eventlog.TypeHistoryImported,
+ AlertID: snapshot.ID, ResourceID: snapshot.ResourceID, OccurredAt: snapshot.LastSeen, Snapshot: payload}}))
+ }
+ firstEvents, err := manager.AlertEvents(eventlog.Filter{AlertID: snapshots[0].ID})
+ require.NoError(t, err)
+ require.Len(t, firstEvents, 1)
+
+ recorder, unblock := blockedProjectionRecorder(t)
+ incidents := memory.NewIncidentStore(memory.IncidentStoreConfig{})
+ incidents.SetResourceTimelineStore(recorder)
+ m := &Monitor{alertManager: manager, incidentStore: incidents, resourceStore: recorder}
+ m.scheduleAlertProjectionCatchUp()
+ t.Cleanup(func() { unblock(); m.alertProjectionWG.Wait() })
+ select {
+ case <-recorder.entered:
+ case <-time.After(5 * time.Second):
+ t.Fatal("replay did not reach its first canonical write")
+ }
+ cancelled := projectionCancellation(m)
+ stopped := make(chan struct{})
+ go func() { m.Stop(); close(stopped) }()
+ select {
+ case <-cancelled:
+ case <-time.After(5 * time.Second):
+ t.Fatal("Stop did not close replay admission")
+ }
+ select {
+ case <-stopped:
+ t.Fatal("Stop closed history while an occurrence was only partly applied")
+ case <-time.After(100 * time.Millisecond):
+ }
+ unblock()
+ select {
+ case <-stopped:
+ case <-time.After(5 * time.Second):
+ t.Fatal("Stop did not join its completed occurrence")
+ }
+ require.EqualValues(t, 3, recorder.writes.Load(), "cancel only between occurrences, not between their three projections")
+
+ // Stop really closed the log. Reopen through the public interface, then
+ // attach a fresh monitor and cold incident cache to the retained evidence.
+ resumedLog, err := eventlog.Open(eventDir)
+ require.NoError(t, err)
+ resumedManager := alerts.NewManagerWithDataDir(t.TempDir(), alerts.WithoutPersistedAlertRestore())
+ resumedManager.SetEventLog(resumedLog)
+ t.Cleanup(resumedManager.Stop)
+ checkpoint := resumedManager.LifecycleProjectionWatermark(alertLifecycleProjectionConsumer)
+ require.Equal(t, firstEvents[0].ID, checkpoint, "checkpoint only the fully applied first occurrence")
+ resumedIncidents := memory.NewIncidentStore(memory.IncidentStoreConfig{})
+ resumedIncidents.SetResourceTimelineStore(recorder)
+ resumed := &Monitor{alertManager: resumedManager, incidentStore: resumedIncidents, resourceStore: recorder}
+ resumed.scheduleAlertProjectionCatchUp()
+ resumed.alertProjectionWG.Wait()
+ require.EqualValues(t, 9, recorder.writes.Load(), "resume only the two unapplied occurrences")
+ resumed.scheduleAlertProjectionCatchUp()
+ resumed.alertProjectionWG.Wait()
+ require.EqualValues(t, 9, recorder.writes.Load(), "completed replay must remain a no-op")
+ for _, snapshot := range snapshots {
+ page, err := resumedIncidents.QueryIncidents(memory.IncidentQuery{AlertIdentifier: snapshot.ID, StartedAt: snapshot.StartTime})
+ require.NoError(t, err)
+ require.Len(t, page.Incidents, 1)
+ incident := page.Incidents[0]
+ require.Equal(t, memory.IncidentStatusResolved, incident.Status)
+ require.Equal(t, &snapshot.LastSeen, incident.ClosedAt)
+ require.True(t, incident.Acknowledged)
+ require.Equal(t, snapshot.AckUser, incident.AckUser)
+ require.Len(t, incident.Events, 3)
+ }
+ t.Logf("interrupted watermark=%d; resumed watermark=%d; complete canonical writes=%d", checkpoint,
+ resumedManager.LifecycleProjectionWatermark(alertLifecycleProjectionConsumer), recorder.writes.Load())
+ resumed.Stop()
+}
+
+func TestAlertProjectionShutdownRefusesQueuedAndLateAdmissions(t *testing.T) {
+ manager := alerts.NewManagerWithDataDir(t.TempDir(), alerts.WithoutPersistedAlertRestore())
+ t.Cleanup(manager.Stop)
+ importReplaySnapshots(t, manager, shutdownHistorySnapshots())
+ recorder, unblock := blockedProjectionRecorder(t)
+ unblock() // Only the replay serialization gate below holds these requests.
+ m := &Monitor{alertManager: manager, incidentStore: memory.NewIncidentStore(memory.IncidentStoreConfig{}), resourceStore: recorder}
+ m.alertProjectionReplayMu.Lock()
+ m.scheduleAlertProjectionCatchUp()
+ cancelled := projectionCancellation(m)
+ stopped := make(chan struct{})
+ go func() { m.stopAlertProjectionCatchUp(); close(stopped) }()
+ var callers sync.WaitGroup
+ for i := 0; i < 64; i++ {
+ callers.Go(m.scheduleAlertProjectionCatchUp)
+ }
+ select {
+ case <-cancelled:
+ case <-time.After(5 * time.Second):
+ m.alertProjectionReplayMu.Unlock()
+ t.Fatal("shutdown did not cancel queued projection work")
+ }
+ callers.Wait()
+ m.alertProjectionReplayMu.Unlock()
+ select {
+ case <-stopped:
+ case <-time.After(5 * time.Second):
+ t.Fatal("cancelled queued work did not join")
+ }
+ for i := 0; i < 64; i++ {
+ m.scheduleAlertProjectionCatchUp()
+ }
+ m.alertProjectionWG.Wait()
+ require.Zero(t, recorder.writes.Load(), "queued or late catch-up cannot revive a stopped writer")
+ require.Zero(t, manager.LifecycleProjectionWatermark(alertLifecycleProjectionConsumer), "unapplied evidence must remain replayable")
+}
+
+func TestCancelledAlertReconciliationDoesNotReadOrWriteHistory(t *testing.T) {
+ manager := alerts.NewManagerWithDataDir(t.TempDir(), alerts.WithoutPersistedAlertRestore())
+ t.Cleanup(manager.Stop)
+ cfg := manager.GetConfig()
+ cfg.Enabled, cfg.ActivationState = true, alerts.ActivationPending
+ cfg.TimeThresholds = map[string]int{}
+ manager.UpdateConfig(cfg)
+ vm := models.VM{ID: "cancelled-reconciliation", Name: "Synthetic VM", Node: "node", Instance: "pve", Status: "stopped"}
+ manager.CheckGuest(vm, vm.Instance)
+ manager.CheckGuest(vm, vm.Instance)
+ require.Len(t, manager.GetActiveAlerts(), 1)
+ recorder, unblock := blockedProjectionRecorder(t)
+ unblock()
+ history := &replayHistoryReadCounter{IncidentTimelineStore: recorder}
+ incidents := memory.NewIncidentStore(memory.IncidentStoreConfig{})
+ incidents.SetResourceTimelineStore(history)
+ m := &Monitor{alertManager: manager, incidentStore: incidents, resourceStore: recorder}
+ ctx, cancel := context.WithCancel(context.Background())
+ cancel()
+ m.reconcileActiveAlertTimelines(ctx)
+ require.Zero(t, history.filteredReads.Load()+history.identityReads.Load()+history.recentReads.Load())
+ changes, err := recorder.GetRecentChanges(vm.ID, time.Time{}, 10)
+ require.NoError(t, err)
+ require.Empty(t, changes)
+}
+
+func TestTenantDeletionRetainsBlockedAlertProjectionThenRemovesCompletedRuntime(t *testing.T) {
+ configureAlertShutdownFixture(t)
+ shortenTenantShutdownBudgets(t, 100*time.Millisecond, 100*time.Millisecond)
+ root := t.TempDir()
+ persistence := config.NewMultiTenantPersistence(root)
+ require.NoError(t, persistence.SaveOrganization(&models.Organization{ID: "history-owner", DisplayName: "History owner"}))
+ manager := NewMultiTenantMonitor(&config.Config{DataPath: root}, persistence, nil)
+ recorder, unblock := blockedProjectionRecorder(t)
+ manager.SetMonitorInitializer(func(m *Monitor) {
+ importReplaySnapshots(t, m.GetAlertManager(), shutdownHistorySnapshots())
+ m.SetResourceStore(recorder)
+ })
+ monitor, err := manager.GetMonitor("history-owner")
+ require.NoError(t, err)
+ t.Cleanup(func() { unblock(); manager.Stop() })
+ select {
+ case <-recorder.entered:
+ case <-time.After(5 * time.Second):
+ t.Fatal("tenant replay did not reach its canonical writer")
+ }
+ // The existing deletion budget must refuse a live replay writer without
+ // closing its stores, dropping its owner or permitting lazy recreation.
+ require.Error(t, manager.BeginTenantDeletion("history-owner"))
+ retained, ok := manager.PeekMonitor("history-owner")
+ require.True(t, ok)
+ require.Same(t, monitor, retained)
+ manager.FinishTenantDeletion("history-owner")
+ _, err = manager.GetMonitor("history-owner")
+ require.Error(t, err)
+ orgPath := filepath.Join(root, "orgs", "history-owner")
+ _, err = os.Stat(filepath.Join(orgPath, "org.json"))
+ require.NoError(t, err, "blocked writer must retain organization bytes")
+ unblock()
+ require.NoError(t, manager.BeginTenantDeletion("history-owner"))
+ require.NoError(t, persistence.DeleteOrganization("history-owner"))
+ manager.FinishTenantDeletion("history-owner")
+ _, ok = manager.PeekMonitor("history-owner")
+ require.False(t, ok)
+ _, err = manager.GetMonitor("history-owner")
+ require.Error(t, err)
+ _, err = os.Stat(orgPath)
+ require.True(t, os.IsNotExist(err), "completed history writer must permit safe removal: %v", err)
+ require.EqualValues(t, 3, recorder.writes.Load())
+}
diff --git a/internal/monitoring/monitor.go b/internal/monitoring/monitor.go
index 7cb9626f15..17da6faf26 100644
--- a/internal/monitoring/monitor.go
+++ b/internal/monitoring/monitor.go
@@ -1344,7 +1344,11 @@ type Monitor struct {
alertPushCallback func(*alerts.Alert)
connectionsSnapshotLister func() []alerts.ConnectionSnapshot // returns platform connection snapshots for the connection-degraded check
incidentStore *memory.IncidentStore
- alertProjectionReplayMu sync.Mutex // serializes lifecycle projection replay passes
+ alertProjectionReplayMu sync.Mutex // serializes lifecycle projection replay passes
+ alertProjectionMu sync.Mutex // closes admission before joining background replay
+ alertProjectionCtx context.Context
+ alertProjectionCancel context.CancelFunc
+ alertProjectionStopped bool
alertProjectionWG sync.WaitGroup // tracks scheduled background catch-up runs
notificationMgr *notifications.NotificationManager
deadMan *deadManRuntime
@@ -2254,6 +2258,10 @@ func (m *Monitor) Start(ctx context.Context, wsHub *websocket.Hub) {
Msg("Starting monitoring loop")
m.setRuntimeContext(ctx, wsHub)
+ // Router attachment can start history replay before Start. Tenant teardown
+ // must not observe this runtime as finished while that writer still owns
+ // the canonical history or its durable replay watermark.
+ defer m.stopAlertProjectionCatchUp()
defer m.stopMockMetricsSampler()
// Best-effort startup cleanup: when direct PBS is configured, remove legacy
@@ -8282,6 +8290,10 @@ func (m *Monitor) Stop() {
func (m *Monitor) stop() error {
log.Info().Msg("stopping monitor")
+ // Also cover monitors stopped before Start. Refuse new replay admissions,
+ // finish the current occurrence, and checkpoint before closing any stores.
+ m.stopAlertProjectionCatchUp()
+
// A trailing agent-report refresh must not run against stores closed below.
m.stopAgentReportRefresh()
diff --git a/internal/monitoring/monitor_alert_handling_test.go b/internal/monitoring/monitor_alert_handling_test.go
index d1d88aac64..d49bfc1b7f 100644
--- a/internal/monitoring/monitor_alert_handling_test.go
+++ b/internal/monitoring/monitor_alert_handling_test.go
@@ -506,8 +506,8 @@ func TestActiveTimelineReconciliationIsIdempotent(t *testing.T) {
resourceStore: unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(resourceStore)),
}
incidentStore.SetResourceTimelineStore(monitor.resourceStore.(memory.IncidentTimelineStore))
- monitor.reconcileActiveAlertTimelines()
- monitor.reconcileActiveAlertTimelines()
+ monitor.reconcileActiveAlertTimelines(context.Background())
+ monitor.reconcileActiveAlertTimelines(context.Background())
active := manager.GetActiveAlerts()
if len(active) != 1 {
@@ -560,8 +560,8 @@ func TestLifecycleReplayRepairsResolvedIncidentTimeline(t *testing.T) {
resourceStore: unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(resourceStore)),
}
incidentStore.SetResourceTimelineStore(monitor.resourceStore.(memory.IncidentTimelineStore))
- monitor.replayAlertLifecycleProjections()
- monitor.replayAlertLifecycleProjections()
+ monitor.replayAlertLifecycleProjections(context.Background())
+ monitor.replayAlertLifecycleProjections(context.Background())
timeline := incidentStore.GetTimelineByAlertAt(alertID, startedAt)
if timeline == nil || len(timeline.Events) != 2 {
@@ -682,7 +682,7 @@ func TestLifecycleReplayWatermarkBoundsSubsequentPasses(t *testing.T) {
// not advance the durable watermark, or resource-timeline projections for
// those events would never materialize.
partial := &Monitor{alertManager: manager, incidentStore: memory.NewIncidentStore(memory.IncidentStoreConfig{})}
- partial.replayAlertLifecycleProjections()
+ partial.replayAlertLifecycleProjections(context.Background())
if got := manager.LifecycleProjectionWatermark(alertLifecycleProjectionConsumer); got != 0 {
t.Fatalf("watermark after partial-surface replay = %d, want 0", got)
}
@@ -693,7 +693,7 @@ func TestLifecycleReplayWatermarkBoundsSubsequentPasses(t *testing.T) {
incidentStore: memory.NewIncidentStore(memory.IncidentStoreConfig{}),
resourceStore: unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(resourceStore)),
}
- monitor.replayAlertLifecycleProjections()
+ monitor.replayAlertLifecycleProjections(context.Background())
watermark := manager.LifecycleProjectionWatermark(alertLifecycleProjectionConsumer)
if watermark == 0 {
t.Fatal("watermark did not advance after full-surface replay")
@@ -707,14 +707,14 @@ func TestLifecycleReplayWatermarkBoundsSubsequentPasses(t *testing.T) {
incidentStore: memory.NewIncidentStore(memory.IncidentStoreConfig{}),
resourceStore: unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(freshResources)),
}
- rerun.replayAlertLifecycleProjections()
+ rerun.replayAlertLifecycleProjections(context.Background())
if changes, err := freshResources.GetRecentChanges(vm.ID, time.Time{}, 10); err != nil || len(changes) != 0 {
t.Fatalf("bounded pass wrote %d changes (err %v), want none", len(changes), err)
}
// Resetting the watermark forces a full repair replay for rebuilt stores.
manager.StoreLifecycleProjectionWatermark(alertLifecycleProjectionConsumer, 0)
- rerun.replayAlertLifecycleProjections()
+ rerun.replayAlertLifecycleProjections(context.Background())
changes, err := freshResources.GetRecentChanges(vm.ID, time.Time{}, 10)
if err != nil {
t.Fatalf("GetRecentChanges after watermark reset: %v", err)
diff --git a/internal/monitoring/monitor_alerts.go b/internal/monitoring/monitor_alerts.go
index 1b19697fc4..bdf77b0fc1 100644
--- a/internal/monitoring/monitor_alerts.go
+++ b/internal/monitoring/monitor_alerts.go
@@ -2,6 +2,7 @@ package monitoring
import (
"context"
+ "errors"
"fmt"
"strings"
"time"
@@ -470,17 +471,43 @@ func (m *Monitor) scheduleAlertProjectionCatchUp() {
if m == nil {
return
}
+ m.alertProjectionMu.Lock()
+ if m.alertProjectionStopped {
+ m.alertProjectionMu.Unlock()
+ return
+ }
+ if m.alertProjectionCtx == nil {
+ m.alertProjectionCtx, m.alertProjectionCancel = context.WithCancel(context.Background())
+ }
+ ctx := m.alertProjectionCtx
+ // Add and closing admission share a lock: shutdown must never race a new
+ // Add against Wait on an empty group, including attachment before Start.
m.alertProjectionWG.Add(1)
+ m.alertProjectionMu.Unlock()
go func() {
defer m.alertProjectionWG.Done()
defer recoverFromPanic("alertProjectionCatchUp")
- m.replayAlertLifecycleProjections()
- m.reconcileActiveAlertTimelines()
+ m.replayAlertLifecycleProjections(ctx)
+ m.reconcileActiveAlertTimelines(ctx)
}()
}
-func (m *Monitor) replayAlertLifecycleProjections() {
- if m == nil || m.alertManager == nil {
+func (m *Monitor) stopAlertProjectionCatchUp() {
+ m.alertProjectionMu.Lock()
+ m.alertProjectionStopped = true
+ if m.alertProjectionCancel != nil {
+ m.alertProjectionCancel()
+ }
+ m.alertProjectionMu.Unlock()
+ // An admitted canonical write cannot be interrupted safely. Start stays
+ // live until it finishes; the existing tenant deletion budget then retains
+ // the runtime and its data if that writer is blocked, rather than removing
+ // its directory underneath it.
+ m.alertProjectionWG.Wait()
+}
+
+func (m *Monitor) replayAlertLifecycleProjections(ctx context.Context) {
+ if m == nil || m.alertManager == nil || ctx.Err() != nil {
return
}
// Serialize passes instead of skipping: a second trigger waits for the
@@ -488,6 +515,9 @@ func (m *Monitor) replayAlertLifecycleProjections() {
// that need replay-complete semantics can rely on a finished call.
m.alertProjectionReplayMu.Lock()
defer m.alertProjectionReplayMu.Unlock()
+ if ctx.Err() != nil {
+ return
+ }
m.mu.RLock()
_, hasRecorder := m.resourceStore.(canonicalResourceChangeRecorder)
@@ -503,6 +533,12 @@ func (m *Monitor) replayAlertLifecycleProjections() {
maxApplied := afterID
visited := 0
err := m.alertManager.ReplayLifecycleEvents(afterID, func(eventID int64, event alerts.LifecycleEvent) error {
+ if err := ctx.Err(); err != nil {
+ return err
+ }
+ // Apply the whole occurrence (including imported acknowledgement and
+ // resolution) before advancing its watermark, even if shutdown arrives
+ // during a canonical write. Cancellation only stops the next event.
m.handleAlertLifecycleEvent(event)
if eventID > maxApplied {
maxApplied = eventID
@@ -513,13 +549,17 @@ func (m *Monitor) replayAlertLifecycleProjections() {
}
return nil
})
- if err != nil {
+ cancelled := err != nil && errors.Is(err, ctx.Err())
+ if err != nil && !cancelled {
log.Error().Err(err).Msg("failed to replay canonical alert lifecycle projections")
return
}
if advance && maxApplied > afterID {
m.alertManager.StoreLifecycleProjectionWatermark(alertLifecycleProjectionConsumer, maxApplied)
}
+ if cancelled {
+ return // The next runtime resumes the unapplied tail, not the full log.
+ }
if visited > 0 {
log.Info().
Int("events", visited).
@@ -529,18 +569,24 @@ func (m *Monitor) replayAlertLifecycleProjections() {
}
}
-func (m *Monitor) reconcileActiveAlertTimelines() {
- if m == nil || m.alertManager == nil || m.incidentStore == nil {
+func (m *Monitor) reconcileActiveAlertTimelines(ctx context.Context) {
+ if m == nil || m.alertManager == nil || m.incidentStore == nil || ctx.Err() != nil {
return
}
activeAlerts := m.alertManager.GetActiveAlerts()
for i := range activeAlerts {
+ if ctx.Err() != nil {
+ return
+ }
alert := &activeAlerts[i]
page, err := m.incidentStore.QueryIncidents(memory.IncidentQuery{AlertIdentifier: alert.ID, StartedAt: alert.StartTime, Limit: 1})
if err != nil {
log.Warn().Err(err).Msg("Skipping alert timeline reconciliation because canonical history is unavailable")
continue
}
+ if ctx.Err() != nil {
+ return
+ }
var timeline *memory.Incident
if len(page.Incidents) > 0 {
timeline = page.Incidents[0]