Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Move the handover tests off the end of the contract test file

rcourtman602e2d04cd51
diff --git a/internal/alerts/operational_contract_test.go b/internal/alerts/operational_contract_test.go
index a4d0dc43bc..51922249de 100644
--- a/internal/alerts/operational_contract_test.go
+++ b/internal/alerts/operational_contract_test.go
@@ -413,6 +413,162 @@ func TestResolvedOperationalContractUpdatesHistorySnapshot(t *testing.T) {
}
}
+func TestReleasedMetricAlertCarriesResolutionInsteadOfRecovery(t *testing.T) {
+ manager := newUnifiedEvalParityManager(t)
+ store, err := eventlog.OpenInMemory()
+ if err != nil {
+ t.Fatalf("open in-memory event log: %v", err)
+ }
+ manager.SetEventLog(store)
+ manager.UpdateConfig(AlertConfig{
+ Enabled: true,
+ NodeDefaults: ThresholdConfig{
+ CPU: &HysteresisThreshold{Trigger: 80, Clear: 75},
+ Memory: &HysteresisThreshold{Trigger: 85, Clear: 80},
+ },
+ })
+ disableTestTimeThresholds(manager)
+ node := models.Node{
+ ID: "pve-1",
+ Name: "pve-1",
+ Instance: "test",
+ Status: "online",
+ CPU: 0.95,
+ Memory: models.Memory{Total: 8 << 30, Used: 7 << 30, Usage: 95},
+ }
+ manager.CheckNode(node)
+ if active := manager.GetActiveAlerts(); len(active) != 2 {
+ t.Fatalf("active alerts = %#v, want cpu and memory", active)
+ }
+
+ resolvedByType := map[string]*ResolvedAlert{}
+ manager.SetResolvedAlertCallback(func(resolved *ResolvedAlert) {
+ resolvedByType[resolved.Alert.Type] = resolved
+ })
+ release := func(metric string, resolution *AlertResolution) {
+ t.Helper()
+ spec, err := buildCanonicalMetricSpec(node.ID, node.Name, unifiedresources.ResourceType("node"), metric, nil)
+ if err != nil {
+ t.Fatalf("build %s spec: %v", metric, err)
+ }
+ manager.releaseCanonicalMetricAlert(spec, node.Name, node.Name, node.Instance, "node", 0, resolution)
+ }
+
+ handover := &AlertResolution{
+ Reason: AlertResolutionMovedToAgent,
+ SuccessorResourceID: "agent-pve-1",
+ SuccessorName: "pve-1 (Host Agent)",
+ }
+ release("memory", handover)
+ // The close is recorded as a copy: the caller's value cannot rewrite it.
+ handover.SuccessorName = "changed"
+ release("cpu", nil)
+
+ if active := manager.GetActiveAlerts(); len(active) != 0 {
+ t.Fatalf("active alerts after release = %#v, want none: the close must still happen", active)
+ }
+
+ moved := resolvedByType["memory"]
+ if moved == nil {
+ t.Fatal("memory release did not reach the resolved consumer")
+ }
+ resolution := moved.Alert.Resolution
+ if resolution == nil ||
+ resolution.Reason != AlertResolutionMovedToAgent ||
+ resolution.SuccessorResourceID != "agent-pve-1" ||
+ resolution.SuccessorName != "pve-1 (Host Agent)" {
+ t.Fatalf("resolution = %+v, want moved to pve-1 (Host Agent)", resolution)
+ }
+ wantSummary := "Alert moved to pve-1 (Host Agent). This is not a recovery: check the agent for the current reading."
+ if got := resolution.Summary(); got != wantSummary {
+ t.Fatalf("Summary() = %q, want %q", got, wantSummary)
+ }
+ if got := resolution.Outcome(); got != "moved to pve-1 (Host Agent)" {
+ t.Fatalf("Outcome() = %q", got)
+ }
+ if got := resolution.Describe("Memory alert"); got != "Memory alert moved to pve-1 (Host Agent). This is not a recovery: check the agent for the current reading." {
+ t.Fatalf("Describe() = %q", got)
+ }
+
+ record := moved.Alert.OperationalRecord
+ if record == nil || record.State != operationaltrust.OperationalResolved {
+ t.Fatalf("operational record = %+v, want resolved", record)
+ }
+ if err := record.Validate(); err != nil {
+ t.Fatalf("operational record Validate() error = %v", err)
+ }
+ transition := moved.Alert.LatestTransition
+ if transition == nil || transition.Reason != wantSummary {
+ t.Fatalf("resolve transition = %+v, want reason %q", transition, wantSummary)
+ }
+ if err := transition.Validate(); err != nil {
+ t.Fatalf("resolve transition Validate() error = %v", err)
+ }
+ closing := moved.Alert.Evidence[len(moved.Alert.Evidence)-1]
+ if closing.ID != transition.EvidenceIDs[0] ||
+ closing.Reason == nil ||
+ closing.Reason.Code != string(AlertResolutionMovedToAgent) ||
+ closing.Reason.Message != wantSummary {
+ t.Fatalf("closing evidence = %+v, want the handover, not a recovery projection", closing)
+ }
+
+ recovered := resolvedByType["cpu"]
+ if recovered == nil || recovered.Alert.Resolution != nil {
+ t.Fatalf("plain disabled release = %+v, want an ordinary resolve with no resolution", recovered)
+ }
+ if last := recovered.Alert.Evidence[len(recovered.Alert.Evidence)-1]; last.Reason == nil ||
+ last.Reason.Code != legacyAlertRecoveryEvidenceReason {
+ t.Fatalf("plain release evidence = %+v, want legacy recovery projection", last.Reason)
+ }
+
+ var historyResolution *AlertResolution
+ for _, entry := range manager.GetAlertHistory(10) {
+ if entry.Type == "memory" {
+ historyResolution = entry.Resolution
+ }
+ }
+ if historyResolution == nil || historyResolution.Reason != AlertResolutionMovedToAgent {
+ t.Fatalf("history resolution = %+v, want moved_to_agent", historyResolution)
+ }
+
+ events, err := manager.AlertEvents(eventlog.Filter{AlertID: moved.Alert.ID, Types: []string{eventlog.TypeResolved}})
+ if err != nil {
+ t.Fatalf("AlertEvents: %v", err)
+ }
+ if len(events) != 1 ||
+ events[0].Reason != string(AlertResolutionMovedToAgent) ||
+ events[0].Message != wantSummary {
+ t.Fatalf("resolved events = %+v, want one handover event", events)
+ }
+}
+
+func TestResolvedHistoryRowKeepsResolutionWithoutEventLog(t *testing.T) {
+ firstObservedAt := time.Date(2026, 10, 6, 9, 0, 0, 0, time.UTC)
+ alert := &Alert{
+ ID: "pve-1-memory",
+ Type: "memory",
+ Level: AlertLevelWarning,
+ ResourceID: "pve-1",
+ CanonicalSpecID: "metric-threshold:memory",
+ CanonicalState: "pve-1::metric-threshold:memory",
+ StartTime: firstObservedAt,
+ LastSeen: firstObservedAt,
+ }
+ ensureOperationalContract(alert, firstObservedAt)
+ history := newTestHistoryManager(t)
+ history.AddAlert(*alert)
+ manager := &Manager{historyManager: history}
+
+ alert.Resolution = &AlertResolution{Reason: AlertResolutionMovedToAgent, SuccessorName: "pve-1 (Host Agent)"}
+ manager.newResolvedAlert(alert, firstObservedAt.Add(time.Minute), nil)
+
+ entries := history.GetAllHistory(1)
+ if len(entries) != 1 || entries[0].Resolution == nil ||
+ entries[0].Resolution.Reason != AlertResolutionMovedToAgent {
+ t.Fatalf("history entries = %+v, want the row to keep the moved_to_agent resolution", entries)
+ }
+}
+
func TestMergeOperationalRecurrenceReopensStableCause(t *testing.T) {
firstObservedAt := time.Date(2026, 7, 18, 20, 0, 0, 0, time.UTC)
resolvedAt := firstObservedAt.Add(5 * time.Minute)
@@ -745,159 +901,3 @@ func TestResolvedOccurrenceCallbackSurvivesLegacyPanic(t *testing.T) {
t.Fatal("legacy panic prevented occurrence-qualified recovery")
}
}
-
-func TestReleasedMetricAlertCarriesResolutionInsteadOfRecovery(t *testing.T) {
- manager := newUnifiedEvalParityManager(t)
- store, err := eventlog.OpenInMemory()
- if err != nil {
- t.Fatalf("open in-memory event log: %v", err)
- }
- manager.SetEventLog(store)
- manager.UpdateConfig(AlertConfig{
- Enabled: true,
- NodeDefaults: ThresholdConfig{
- CPU: &HysteresisThreshold{Trigger: 80, Clear: 75},
- Memory: &HysteresisThreshold{Trigger: 85, Clear: 80},
- },
- })
- disableTestTimeThresholds(manager)
- node := models.Node{
- ID: "pve-1",
- Name: "pve-1",
- Instance: "test",
- Status: "online",
- CPU: 0.95,
- Memory: models.Memory{Total: 8 << 30, Used: 7 << 30, Usage: 95},
- }
- manager.CheckNode(node)
- if active := manager.GetActiveAlerts(); len(active) != 2 {
- t.Fatalf("active alerts = %#v, want cpu and memory", active)
- }
-
- resolvedByType := map[string]*ResolvedAlert{}
- manager.SetResolvedAlertCallback(func(resolved *ResolvedAlert) {
- resolvedByType[resolved.Alert.Type] = resolved
- })
- release := func(metric string, resolution *AlertResolution) {
- t.Helper()
- spec, err := buildCanonicalMetricSpec(node.ID, node.Name, unifiedresources.ResourceType("node"), metric, nil)
- if err != nil {
- t.Fatalf("build %s spec: %v", metric, err)
- }
- manager.releaseCanonicalMetricAlert(spec, node.Name, node.Name, node.Instance, "node", 0, resolution)
- }
-
- handover := &AlertResolution{
- Reason: AlertResolutionMovedToAgent,
- SuccessorResourceID: "agent-pve-1",
- SuccessorName: "pve-1 (Host Agent)",
- }
- release("memory", handover)
- // The close is recorded as a copy: the caller's value cannot rewrite it.
- handover.SuccessorName = "changed"
- release("cpu", nil)
-
- if active := manager.GetActiveAlerts(); len(active) != 0 {
- t.Fatalf("active alerts after release = %#v, want none: the close must still happen", active)
- }
-
- moved := resolvedByType["memory"]
- if moved == nil {
- t.Fatal("memory release did not reach the resolved consumer")
- }
- resolution := moved.Alert.Resolution
- if resolution == nil ||
- resolution.Reason != AlertResolutionMovedToAgent ||
- resolution.SuccessorResourceID != "agent-pve-1" ||
- resolution.SuccessorName != "pve-1 (Host Agent)" {
- t.Fatalf("resolution = %+v, want moved to pve-1 (Host Agent)", resolution)
- }
- wantSummary := "Alert moved to pve-1 (Host Agent). This is not a recovery: check the agent for the current reading."
- if got := resolution.Summary(); got != wantSummary {
- t.Fatalf("Summary() = %q, want %q", got, wantSummary)
- }
- if got := resolution.Outcome(); got != "moved to pve-1 (Host Agent)" {
- t.Fatalf("Outcome() = %q", got)
- }
- if got := resolution.Describe("Memory alert"); got != "Memory alert moved to pve-1 (Host Agent). This is not a recovery: check the agent for the current reading." {
- t.Fatalf("Describe() = %q", got)
- }
-
- record := moved.Alert.OperationalRecord
- if record == nil || record.State != operationaltrust.OperationalResolved {
- t.Fatalf("operational record = %+v, want resolved", record)
- }
- if err := record.Validate(); err != nil {
- t.Fatalf("operational record Validate() error = %v", err)
- }
- transition := moved.Alert.LatestTransition
- if transition == nil || transition.Reason != wantSummary {
- t.Fatalf("resolve transition = %+v, want reason %q", transition, wantSummary)
- }
- if err := transition.Validate(); err != nil {
- t.Fatalf("resolve transition Validate() error = %v", err)
- }
- closing := moved.Alert.Evidence[len(moved.Alert.Evidence)-1]
- if closing.ID != transition.EvidenceIDs[0] ||
- closing.Reason == nil ||
- closing.Reason.Code != string(AlertResolutionMovedToAgent) ||
- closing.Reason.Message != wantSummary {
- t.Fatalf("closing evidence = %+v, want the handover, not a recovery projection", closing)
- }
-
- recovered := resolvedByType["cpu"]
- if recovered == nil || recovered.Alert.Resolution != nil {
- t.Fatalf("plain disabled release = %+v, want an ordinary resolve with no resolution", recovered)
- }
- if last := recovered.Alert.Evidence[len(recovered.Alert.Evidence)-1]; last.Reason == nil ||
- last.Reason.Code != legacyAlertRecoveryEvidenceReason {
- t.Fatalf("plain release evidence = %+v, want legacy recovery projection", last.Reason)
- }
-
- var historyResolution *AlertResolution
- for _, entry := range manager.GetAlertHistory(10) {
- if entry.Type == "memory" {
- historyResolution = entry.Resolution
- }
- }
- if historyResolution == nil || historyResolution.Reason != AlertResolutionMovedToAgent {
- t.Fatalf("history resolution = %+v, want moved_to_agent", historyResolution)
- }
-
- events, err := manager.AlertEvents(eventlog.Filter{AlertID: moved.Alert.ID, Types: []string{eventlog.TypeResolved}})
- if err != nil {
- t.Fatalf("AlertEvents: %v", err)
- }
- if len(events) != 1 ||
- events[0].Reason != string(AlertResolutionMovedToAgent) ||
- events[0].Message != wantSummary {
- t.Fatalf("resolved events = %+v, want one handover event", events)
- }
-}
-
-func TestResolvedHistoryRowKeepsResolutionWithoutEventLog(t *testing.T) {
- firstObservedAt := time.Date(2026, 10, 6, 9, 0, 0, 0, time.UTC)
- alert := &Alert{
- ID: "pve-1-memory",
- Type: "memory",
- Level: AlertLevelWarning,
- ResourceID: "pve-1",
- CanonicalSpecID: "metric-threshold:memory",
- CanonicalState: "pve-1::metric-threshold:memory",
- StartTime: firstObservedAt,
- LastSeen: firstObservedAt,
- }
- ensureOperationalContract(alert, firstObservedAt)
- history := newTestHistoryManager(t)
- history.AddAlert(*alert)
- manager := &Manager{historyManager: history}
-
- alert.Resolution = &AlertResolution{Reason: AlertResolutionMovedToAgent, SuccessorName: "pve-1 (Host Agent)"}
- manager.newResolvedAlert(alert, firstObservedAt.Add(time.Minute), nil)
-
- entries := history.GetAllHistory(1)
- if len(entries) != 1 || entries[0].Resolution == nil ||
- entries[0].Resolution.Reason != AlertResolutionMovedToAgent {
- t.Fatalf("history entries = %+v, want the row to keep the moved_to_agent resolution", entries)
- }
-}