Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
test(monitoring): expose alert replay surviving runtime shutdown
pulse-triage[bot]1419d61c4b33
diff --git a/internal/monitoring/alert_projection_shutdown_test.go b/internal/monitoring/alert_projection_shutdown_test.gonew file mode 100644index 0000000000..ac75109905--- /dev/null+++ b/internal/monitoring/alert_projection_shutdown_test.go@@ -0,0 +1,115 @@+package monitoring++import (+ "context"+ "sync"+ "sync/atomic"+ "testing"+ "time"++ "github.com/rcourtman/pulse-go-rewrite/internal/alerts"+ "github.com/rcourtman/pulse-go-rewrite/internal/config"+ "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"+)++// Hold a real replay at the exported canonical writer boundary. No database+// implementation, workload or timing assumption chooses the overlap.+type projectionShutdownRecorder struct {+ *unifiedresources.MonitorAdapter+ resourceID string+ entered chan struct{}+ release chan struct{}+ enterOnce sync.Once+ writes atomic.Int64+}++func (s *projectionShutdownRecorder) RecordChange(change unifiedresources.ResourceChange) error {+ if change.ResourceID == s.resourceID {+ s.enterOnce.Do(func() {+ close(s.entered)+ <-s.release+ })+ s.writes.Add(1)+ }+ return s.MonitorAdapter.RecordChange(change)+}++func blockedProjectionRecorder(t *testing.T) (*projectionShutdownRecorder, func()) {+ t.Helper()+ store := unifiedresources.NewMemoryStore()+ s := &projectionShutdownRecorder{+ MonitorAdapter: unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(store)),+ resourceID: "shutdown-history-resource",+ entered: make(chan struct{}),+ release: make(chan struct{}),+ }+ var once sync.Once+ unblock := func() { once.Do(func() { close(s.release) }) }+ t.Cleanup(unblock)+ return s, unblock+}++func shutdownHistorySnapshots() []alerts.Alert {+ start := time.Now().UTC().Add(-time.Hour).Truncate(time.Second)+ ack := start.Add(time.Second)+ return []alerts.Alert{{+ ID: "shutdown-history-alert", ResourceID: "shutdown-history-resource",+ Type: "cpu", Level: alerts.AlertLevelWarning,+ StartTime: start, LastSeen: start.Add(2 * time.Second),+ Acknowledged: true, AckTime: &ack, AckUser: "synthetic-operator",+ }}+}++func TestMonitorStartJoinsAlertProjectionBeforeReturning(t *testing.T) {+ configureAlertShutdownFixture(t)+ monitor, err := New(&config.Config{DataPath: t.TempDir()})+ if err != nil {+ t.Fatal(err)+ }+ recorder, unblock := blockedProjectionRecorder(t)+ importReplaySnapshots(t, monitor.GetAlertManager(), shutdownHistorySnapshots())+ monitor.SetResourceStore(recorder)+ select {+ case <-recorder.entered:+ case <-time.After(5 * time.Second):+ unblock()+ monitor.alertProjectionWG.Wait()+ monitor.Stop()+ t.Fatal("history replay did not reach the canonical writer")+ }++ // Replay starts during router attachment, before Start. Even a runtime+ // cancelled before startup must join that already-admitted writer.+ ctx, cancel := context.WithCancel(context.Background())+ cancel()+ done := make(chan struct{})+ go func() {+ defer close(done)+ monitor.Start(ctx, nil)+ }()+ t.Cleanup(func() {+ unblock()+ select {+ case <-done:+ monitor.alertProjectionWG.Wait()+ monitor.Stop()+ case <-time.After(5 * time.Second):+ t.Error("monitor did not finish after releasing its history writer")+ }+ })+ select {+ case <-done:+ t.Error("monitor reported completion while alert-history replay could still write")+ case <-time.After(100 * time.Millisecond):+ }+ unblock()+ select {+ case <-done:+ case <-time.After(5 * time.Second):+ t.Fatal("monitor failed to join its completed history writer")+ }+ monitor.alertProjectionWG.Wait()+ if got := recorder.writes.Load(); got != 3 {+ t.Fatalf("admitted snapshot wrote %d changes, want its complete fired/acknowledged/resolved occurrence", got)+ }+}