Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

11.9 KB
Raw Blame
package alerts

import (
	"encoding/json"
	"fmt"
	"runtime"
	"sync"
	"testing"
	"time"

	alertspecs "github.com/rcourtman/pulse-go-rewrite/internal/alerts/specs"
	"github.com/rcourtman/pulse-go-rewrite/internal/models"
	"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
	"github.com/rs/zerolog"
)

// Regression for issue #1590. The canonical alert evaluation paths used to
// read and mutate recentlyResolved/resolvedAlias while holding only m.mu,
// while the broadcast and recovery paths guarded them with resolvedMutex, so
// the two lock domains did not exclude each other and Go's runtime aborted
// the process with a concurrent map access fault on flapping resources.
//
// Run with -race. Unlike the original reproducer, this proof drives both
// production canonical evaluators through fire -> recovery -> cooldown refire
// while public broadcast/lookup, alias repair, cleanup, and shutdown paths run
// concurrently.
func TestRecentlyResolvedProductionPathsAreRaceAndDeadlockFree(t *testing.T) {
	originalLogLevel := zerolog.GlobalLevel()
	zerolog.SetGlobalLevel(zerolog.Disabled)
	t.Cleanup(func() {
		zerolog.SetGlobalLevel(originalLogLevel)
	})

	m := NewManagerWithDataDir(t.TempDir())
	t.Cleanup(m.Stop)

	lifecycleResourceID := "agent:issue-1590-lifecycle"
	lifecycleSpec, err := buildCanonicalConnectivitySpec(
		lifecycleResourceID,
		"Issue 1590 lifecycle",
		unifiedresources.ResourceTypeAgent,
		AlertLevelWarning,
		1,
		false,
	)
	if err != nil {
		t.Fatal(err)
	}
	lifecycleParams := canonicalLifecycleAlertParams{
		Spec:         lifecycleSpec,
		AlertID:      "host-offline-issue-1590-lifecycle",
		AlertType:    "connectivity",
		ResourceID:   lifecycleResourceID,
		ResourceName: "Issue 1590 lifecycle",
		Instance:     "issue-1590",
		Message:      "lifecycle test alert",
		Metadata:     map[string]interface{}{"resourceType": string(unifiedresources.ResourceTypeAgent)},
		AddToHistory: true,
	}
	lifecycleStateID := canonicalTrackingKeyForSpec(lifecycleSpec, lifecycleParams.AlertID)

	statefulResourceID := "storage:issue-1590-stateful"
	statefulSpec, err := buildCanonicalHealthAssessmentSpec(
		"issue-1590-health",
		statefulResourceID,
		"Issue 1590 stateful",
		unifiedresources.ResourceTypeStorage,
		"pool-health",
		[]string{"degraded"},
		false,
	)
	if err != nil {
		t.Fatal(err)
	}
	statefulParams := canonicalStatefulAlertParams{
		Spec:         statefulSpec,
		AlertID:      "issue-1590-stateful-health",
		AlertType:    "storage-health",
		ResourceID:   statefulResourceID,
		ResourceName: "Issue 1590 stateful",
		Instance:     "issue-1590",
		Message:      "stateful test alert",
		Metadata:     map[string]interface{}{"resourceType": string(unifiedresources.ResourceTypeStorage)},
		AddToHistory: true,
	}
	statefulStateID := canonicalTrackingKeyForSpec(statefulSpec, statefulParams.AlertID)

	fireLifecycle := func(observedAt time.Time) alertspecs.AlertState {
		lifecycleParams.Evidence = alertspecs.AlertEvidence{
			ObservedAt: observedAt,
			Connectivity: &alertspecs.ConnectivityEvidence{
				Signal:    "status",
				Connected: false,
			},
		}
		result, _ := m.evaluateCanonicalLifecycleAlert(lifecycleParams)
		return result.State.State
	}
	recoverLifecycle := func(observedAt time.Time) alertspecs.AlertState {
		lifecycleParams.Evidence = alertspecs.AlertEvidence{
			ObservedAt: observedAt,
			Connectivity: &alertspecs.ConnectivityEvidence{
				Signal:    "status",
				Connected: true,
			},
		}
		result, _ := m.evaluateCanonicalLifecycleAlert(lifecycleParams)
		return result.State.State
	}
	fireStateful := func(observedAt time.Time) alertspecs.AlertState {
		statefulParams.Evidence = alertspecs.AlertEvidence{
			ObservedAt: observedAt,
			HealthAssessment: &alertspecs.HealthAssessmentEvidence{
				Signal:   "pool-health",
				Severity: alertspecs.AlertSeverityWarning,
				Codes:    []string{"degraded"},
			},
		}
		result, _ := m.evaluateCanonicalStatefulAlert(statefulParams)
		return result.State.State
	}
	recoverStateful := func(observedAt time.Time) alertspecs.AlertState {
		statefulParams.Evidence = alertspecs.AlertEvidence{
			ObservedAt:       observedAt,
			HealthAssessment: &alertspecs.HealthAssessmentEvidence{Signal: "pool-health"},
		}
		result, _ := m.evaluateCanonicalStatefulAlert(statefulParams)
		return result.State.State
	}

	base := time.Now().Add(-time.Minute)
	if got := fireLifecycle(base); got != alertspecs.AlertStateFiring {
		t.Fatalf("initial lifecycle state = %q, want firing", got)
	}
	lifecycleStart := testRequireActiveAlert(t, m, lifecycleStateID).StartTime
	if got := fireStateful(base); got != alertspecs.AlertStateFiring {
		t.Fatalf("initial stateful state = %q, want firing", got)
	}
	statefulStart := testRequireActiveAlert(t, m, statefulStateID).StartTime
	if got := recoverLifecycle(base.Add(time.Millisecond)); got != alertspecs.AlertStateClear {
		t.Fatalf("initial lifecycle recovery state = %q, want clear", got)
	}
	if got := recoverStateful(base.Add(time.Millisecond)); got != alertspecs.AlertStateClear {
		t.Fatalf("initial stateful recovery state = %q, want clear", got)
	}

	// Force the canonical-alias fallback path. GetResolvedAlert repairs this
	// alias under the resolved write lock while broadcasts iterate the map.
	aliasStateID := buildCanonicalStateID("agent:issue-1590-alias", "issue-1590-alias")
	m.resolvedMutex.Lock()
	m.recentlyResolved["issue-1590-legacy-storage-key"] = &ResolvedAlert{
		Alert: &Alert{
			ID:              "issue-1590-legacy-alert",
			ResourceID:      "agent:issue-1590-alias",
			CanonicalSpecID: "issue-1590-alias",
			CanonicalState:  aliasStateID,
			StartTime:       base,
		},
		ResolvedTime: time.Now(),
	}
	m.resolvedMutex.Unlock()

	start := make(chan struct{})
	errs := make(chan error, 4)
	var wg sync.WaitGroup
	wg.Add(4)

	// Canonical lifecycle recovery and cooldown refire path.
	go func() {
		defer wg.Done()
		<-start
		for i := 0; i < 100; i++ {
			observedAt := base.Add(time.Duration(2*i+2) * time.Millisecond)
			if got := fireLifecycle(observedAt); got != alertspecs.AlertStateFiring {
				errs <- fmt.Errorf("lifecycle iteration %d fire state = %q", i, got)
				return
			}
			if got := recoverLifecycle(observedAt.Add(time.Microsecond)); got != alertspecs.AlertStateClear {
				errs <- fmt.Errorf("lifecycle iteration %d recovery state = %q", i, got)
				return
			}
			runtime.Gosched()
		}
	}()

	// Canonical stateful recovery and cooldown refire path.
	go func() {
		defer wg.Done()
		<-start
		for i := 0; i < 100; i++ {
			observedAt := base.Add(time.Duration(2*i+2) * time.Millisecond)
			if got := fireStateful(observedAt); got != alertspecs.AlertStateFiring {
				errs <- fmt.Errorf("stateful iteration %d fire state = %q", i, got)
				return
			}
			if got := recoverStateful(observedAt.Add(time.Microsecond)); got != alertspecs.AlertStateClear {
				errs <- fmt.Errorf("stateful iteration %d recovery state = %q", i, got)
				return
			}
			runtime.Gosched()
		}
	}()

	// Production broadcast and point-lookup paths.
	go func() {
		defer wg.Done()
		<-start
		for i := 0; i < 800; i++ {
			m.GetRecentlyResolved()
			m.GetResolvedAlert(aliasStateID)
			m.GetResolvedAlert(lifecycleStateID)
			m.GetResolvedAlert(statefulStateID)
			runtime.Gosched()
		}
	}()

	// Cleanup is the other production path that holds m.mu before acquiring
	// resolvedMutex.
	go func() {
		defer wg.Done()
		<-start
		for i := 0; i < 100; i++ {
			m.Cleanup(time.Hour)
			runtime.Gosched()
		}
	}()

	close(start)
	waitForResolvedConcurrencyGroup(t, &wg)
	close(errs)
	for err := range errs {
		t.Error(err)
	}
	if t.Failed() {
		return
	}

	finalObservedAt := base.Add(2 * time.Second)
	if got := fireLifecycle(finalObservedAt); got != alertspecs.AlertStateFiring {
		t.Fatalf("final lifecycle state = %q, want firing", got)
	}
	if got := fireStateful(finalObservedAt); got != alertspecs.AlertStateFiring {
		t.Fatalf("final stateful state = %q, want firing", got)
	}
	if got := testRequireActiveAlert(t, m, lifecycleStateID).StartTime; !got.Equal(lifecycleStart) {
		t.Fatalf("lifecycle refire start = %v, want original %v", got, lifecycleStart)
	}
	if got := testRequireActiveAlert(t, m, statefulStateID).StartTime; !got.Equal(statefulStart) {
		t.Fatalf("stateful refire start = %v, want original %v", got, statefulStart)
	}
	if got := len(m.historyManager.GetAllHistory(1000)); got != 2 {
		t.Fatalf("history entries after repeated cooldown refires = %d, want 2", got)
	}
	if got := m.GetResolvedAlert(lifecycleStateID); got != nil {
		t.Fatal("lifecycle resolved entry survived cooldown refire")
	}
	if got := m.GetResolvedAlert(statefulStateID); got != nil {
		t.Fatal("stateful resolved entry survived cooldown refire")
	}
	if got := m.GetResolvedAlert(aliasStateID); got == nil || got.Alert == nil {
		t.Fatal("canonical alias lookup did not survive concurrent repair and broadcasts")
	}

	// Shutdown must not introduce a reverse-order wait against concurrent
	// resolved reads or cleanup.
	var shutdownWG sync.WaitGroup
	shutdownWG.Add(6)
	for i := 0; i < 2; i++ {
		go func() {
			defer shutdownWG.Done()
			for j := 0; j < 100; j++ {
				m.GetRecentlyResolved()
				m.GetResolvedAlert(aliasStateID)
			}
		}()
	}
	for i := 0; i < 2; i++ {
		go func() {
			defer shutdownWG.Done()
			for j := 0; j < 30; j++ {
				m.Cleanup(time.Hour)
			}
		}()
	}
	for i := 0; i < 2; i++ {
		go func() {
			defer shutdownWG.Done()
			m.Stop()
		}()
	}
	waitForResolvedConcurrencyGroup(t, &shutdownWG)
}

func waitForResolvedConcurrencyGroup(t *testing.T, wg *sync.WaitGroup) {
	t.Helper()

	done := make(chan struct{})
	go func() {
		wg.Wait()
		close(done)
	}()

	select {
	case <-done:
	case <-time.After(30 * time.Second):
		t.Fatal("resolved alert concurrency paths deadlocked")
	}
}

// GetConfig snapshots are encoded, persisted and edited outside m.mu, and
// UpdateConfig normalizes the maps it is handed in place. While a snapshot
// shared the live config's maps, re-applying one rewrote the override map
// that the GET /api/alerts/config handler was encoding from another snapshot,
// and Go aborted the process with a concurrent map write. Run with -race.
func TestConfigSnapshotsShareNothingWithTheLiveConfig(t *testing.T) {
	originalLogLevel := zerolog.GlobalLevel()
	zerolog.SetGlobalLevel(zerolog.Disabled)
	t.Cleanup(func() {
		zerolog.SetGlobalLevel(originalLogLevel)
	})

	m := NewManagerWithDataDir(t.TempDir())
	t.Cleanup(m.Stop)

	seed := m.GetConfig()
	seed.Enabled = true
	seed.ActivationState = ActivationActive
	seed.Overrides = make(map[string]ThresholdConfig)
	for i := 0; i < 64; i++ {
		seed.Overrides[fmt.Sprintf("agent:snapshot-%d", i)] = ThresholdConfig{
			CPU: &HysteresisThreshold{Trigger: 95, Clear: 90},
		}
	}
	m.UpdateConfig(seed)

	snapshot := m.GetConfig()
	snapshot.Overrides["agent:snapshot-0"].CPU.Trigger = 1
	delete(snapshot.Overrides, "agent:snapshot-1")
	snapshot.TimeThresholds["guest"] = 99
	snapshot.AgentDefaults.CPU.Trigger = 1

	live := m.GetConfig()
	if got := live.Overrides["agent:snapshot-0"].CPU.Trigger; got != 95 {
		t.Fatalf("editing a snapshot changed the live override trigger to %v", got)
	}
	if _, ok := live.Overrides["agent:snapshot-1"]; !ok {
		t.Fatal("deleting from a snapshot removed the live override")
	}
	if live.TimeThresholds["guest"] == 99 {
		t.Fatal("editing a snapshot changed the live guest delay")
	}
	if live.AgentDefaults.CPU.Trigger == 1 {
		t.Fatal("editing a snapshot changed the live agent CPU trigger")
	}

	var wg sync.WaitGroup
	wg.Add(3)
	go func() {
		defer wg.Done()
		for i := 0; i < 200; i++ {
			m.UpdateConfig(m.GetConfig())
		}
	}()
	go func() {
		defer wg.Done()
		for i := 0; i < 200; i++ {
			if _, err := json.Marshal(m.GetConfig()); err != nil {
				t.Errorf("encode snapshot: %v", err)
				return
			}
		}
	}()
	go func() {
		defer wg.Done()
		for i := 0; i < 200; i++ {
			id := fmt.Sprintf("snapshot-%d", i%64)
			m.CheckHost(models.Host{ID: id, Hostname: id, Status: "online", CPUUsage: 50})
		}
	}()
	wg.Wait()

	if got := len(m.GetConfig().Overrides); got != 64 {
		t.Fatalf("live overrides = %d after re-applying snapshots, want 64", got)
	}
}