Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

48.1 KB
Raw Blame
package alerts

import (
	"encoding/json"
	"testing"
	"time"

	"github.com/rcourtman/pulse-go-rewrite/internal/alerts/eventlog"
	alertspecs "github.com/rcourtman/pulse-go-rewrite/internal/alerts/specs"
	"github.com/rcourtman/pulse-go-rewrite/internal/models"
	"github.com/rcourtman/pulse-go-rewrite/internal/operationaltrust"
	"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
)

func TestEnsureOperationalContractBackfillsLegacyAlertHonestly(t *testing.T) {
	firstObservedAt := time.Date(2026, 7, 18, 20, 0, 0, 0, time.UTC)
	lastObservedAt := firstObservedAt.Add(time.Minute)
	alert := &Alert{
		ID:              "legacy-alert",
		Type:            "cpu",
		Level:           AlertLevelWarning,
		ResourceID:      "resource-1",
		CanonicalSpecID: "metric-threshold:cpu",
		CanonicalKind:   "metric-threshold",
		CanonicalState:  "resource-1::metric-threshold:cpu",
		Message:         "CPU usage exceeded threshold",
		StartTime:       firstObservedAt,
		LastSeen:        lastObservedAt,
	}

	ensureOperationalContract(alert, lastObservedAt.Add(time.Second))

	if len(alert.Evidence) != 1 {
		t.Fatalf("evidence count = %d, want 1", len(alert.Evidence))
	}
	if got := alert.Evidence[0].Reason; got == nil || got.Code != legacyAlertEvidenceReason {
		t.Fatalf("evidence reason = %+v, want %q", got, legacyAlertEvidenceReason)
	}
	if got := alert.Evidence[0].Completeness; got != operationaltrust.EvidencePartial {
		t.Fatalf("evidence completeness = %q, want partial", got)
	}
	if err := alert.Evidence[0].Validate(); err != nil {
		t.Fatalf("evidence Validate() error = %v", err)
	}
	if alert.OperationalRecord == nil {
		t.Fatal("operational record is nil")
	}
	if got := alert.OperationalRecord.State; got != operationaltrust.OperationalOpen {
		t.Fatalf("operational state = %q, want open", got)
	}
	if len(alert.OperationalRecord.EvidenceIDs) != 1 ||
		alert.OperationalRecord.EvidenceIDs[0] != alert.Evidence[0].ID {
		t.Fatalf(
			"operational evidence ids = %v, want %q",
			alert.OperationalRecord.EvidenceIDs,
			alert.Evidence[0].ID,
		)
	}
	if err := alert.OperationalRecord.Validate(); err != nil {
		t.Fatalf("operational record Validate() error = %v", err)
	}
	if alert.LatestTransition == nil {
		t.Fatal("latest transition is nil")
	}
	if got := alert.LatestTransition.Cause; got != operationaltrust.TransitionDetectorDecision {
		t.Fatalf("transition cause = %q, want detector decision", got)
	}
	if err := alert.LatestTransition.Validate(); err != nil {
		t.Fatalf("transition Validate() error = %v", err)
	}
	if len(alert.Transitions) != 1 ||
		alert.Transitions[0].ID != alert.LatestTransition.ID {
		t.Fatalf("transition timeline = %+v, want initial transition", alert.Transitions)
	}
	if got, want := alert.OperationalRecord.RecommendedNextStep, "Open the affected resource and verify its current state before making changes."; got != want {
		t.Fatalf("RecommendedNextStep = %q, want %q", got, want)
	}
}

func TestIndefiniteOperationalSuppressionRemainsDeliverySafe(t *testing.T) {
	m := newTestManager(t)
	now := time.Now().UTC()
	cfg := m.GetConfig()
	cfg.Enabled = true
	cfg.ActivationState = ActivationActive
	m.UpdateConfig(cfg)
	alert := &Alert{ID: "maintenance:vm/100", CanonicalState: "maintenance:vm/100", CanonicalSpecID: "maintenance", Type: "maintenance", Level: AlertLevelWarning, ResourceID: "vm/100", ResourceName: "vm-100", StartTime: now, LastSeen: now}
	m.mu.Lock()
	m.setActiveAlertNoLock(alert.ID, alert)
	m.mu.Unlock()
	if err := m.SuppressOperationalAlert(alert.ID, "[email protected]", "maintenance", nil); err != nil {
		t.Fatal(err)
	}
	if m.dispatchAlert(activeAlert(t, m, alert.ID), false) {
		t.Fatal("indefinitely suppressed alert was dispatched")
	}
	diagnosis, ok := m.DiagnoseAlertDelivery(alert.ID)
	if !ok || diagnosis.Reason != AlertDeliveryReasonSnoozed || diagnosis.SuppressedUntil != nil {
		t.Fatalf("diagnosis = %+v, exists=%v", diagnosis, ok)
	}
}

func TestEnsureOperationalContractUsesCanonicalIncidentAction(t *testing.T) {
	now := time.Date(2026, 7, 19, 1, 0, 0, 0, time.UTC)
	alert := &Alert{
		ID:           "storage:pool-1",
		Type:         "storage-incident",
		Level:        AlertLevelCritical,
		ResourceID:   "pool-1",
		ResourceName: "Pool 1",
		Message:      "Pool health is degraded",
		StartTime:    now.Add(-time.Minute),
		LastSeen:     now,
		Metadata: map[string]interface{}{
			"incidentAction": "Inspect the degraded devices and restore redundancy",
		},
	}

	ensureOperationalContract(alert, now)

	if alert.OperationalRecord == nil {
		t.Fatal("expected operational record")
	}
	if got, want := alert.OperationalRecord.RecommendedNextStep, "Inspect the degraded devices and restore redundancy"; got != want {
		t.Fatalf("RecommendedNextStep = %q, want %q", got, want)
	}
}

func TestCanonicalAlertEvidenceBuildsConfirmedEnvelope(t *testing.T) {
	observedAt := time.Date(2026, 7, 18, 20, 0, 0, 0, time.UTC)
	spec := alertspecs.ResourceAlertSpec{
		ID:           "provider-incident:disk-1",
		ResourceID:   "disk-1",
		ResourceType: unifiedresources.ResourceTypeStorage,
		Kind:         alertspecs.AlertSpecKindProviderIncident,
	}
	evidence := alertspecs.AlertEvidence{
		ObservedAt: observedAt,
		ProviderIncident: &alertspecs.ProviderIncidentEvidence{
			Provider: "truenas",
			NativeID: "provider-alert-7",
			Source:   "truenas-alert-feed",
		},
	}

	first, ok := canonicalAlertEvidenceEnvelope(spec, evidence, "nas-a", observedAt.Add(time.Second))
	if !ok {
		t.Fatal("canonicalAlertEvidenceEnvelope() rejected valid evidence")
	}
	second, ok := canonicalAlertEvidenceEnvelope(spec, evidence, "nas-a", observedAt.Add(2*time.Second))
	if !ok {
		t.Fatal("canonicalAlertEvidenceEnvelope() retry rejected valid evidence")
	}
	if first.ID != second.ID {
		t.Fatalf("canonical evidence retry id = %q, want %q", second.ID, first.ID)
	}
	if first.Source.Provider != "truenas" ||
		first.Source.Collector != "truenas-alert-feed" ||
		first.Completeness != operationaltrust.EvidenceComplete ||
		first.Confidence != operationaltrust.EvidenceConfirmed {
		t.Fatalf("canonical evidence = %+v", first)
	}
	if first.PayloadRef == nil || first.PayloadRef.Kind != "alert-evidence-digest" {
		t.Fatalf("payload reference = %+v, want bounded digest", first.PayloadRef)
	}
}

func TestCanonicalAlertEvidencePreservesTypedLimitations(t *testing.T) {
	observedAt := time.Date(2026, 7, 18, 20, 0, 0, 0, time.UTC)
	spec := alertspecs.ResourceAlertSpec{
		ID:           "connectivity:resource-1",
		ResourceID:   "resource-1",
		ResourceType: unifiedresources.ResourceTypeAgent,
		Kind:         alertspecs.AlertSpecKindConnectivity,
	}
	source := operationaltrust.EvidenceSource{
		Provider:  "proxmox",
		Collector: "connectivity-check",
	}
	subject := operationaltrust.EvidenceSubject{ResourceID: spec.ResourceID}
	id, err := operationaltrust.NewEvidenceID(source, subject, observedAt, "check-7")
	if err != nil {
		t.Fatalf("NewEvidenceID() error = %v", err)
	}
	envelope := operationaltrust.EvidenceEnvelope{
		ID:           id,
		Source:       source,
		Subject:      subject,
		ObservedAt:   observedAt,
		IngestedAt:   observedAt,
		Completeness: operationaltrust.EvidencePartial,
		Confidence:   operationaltrust.EvidenceUnknown,
		Permissions:  operationaltrust.EvidencePermissionsDenied,
		Reason:       &operationaltrust.EvidenceReason{Code: "permission_denied"},
	}
	evidence := alertspecs.AlertEvidence{
		ObservedAt: observedAt,
		Envelope:   &envelope,
		Connectivity: &alertspecs.ConnectivityEvidence{
			Signal:    "heartbeat",
			Connected: false,
		},
	}

	got, ok := canonicalAlertEvidenceEnvelope(spec, evidence, "pve-a", observedAt)
	if !ok {
		t.Fatal("canonicalAlertEvidenceEnvelope() rejected typed limited evidence")
	}
	if got.Permissions != operationaltrust.EvidencePermissionsDenied ||
		got.Reason == nil ||
		got.Reason.Code != "permission_denied" {
		t.Fatalf("limited evidence = %+v", got)
	}

	mismatched := envelope.Clone()
	mismatched.Subject.ResourceID = "other-resource"
	evidence.Envelope = &mismatched
	if _, ok := canonicalAlertEvidenceEnvelope(spec, evidence, "pve-a", observedAt); ok {
		t.Fatal("canonicalAlertEvidenceEnvelope() accepted mismatched subject")
	}
}

func TestEnsureOperationalContractTracksAcknowledgementWithoutResolution(t *testing.T) {
	firstObservedAt := time.Date(2026, 7, 18, 20, 0, 0, 0, time.UTC)
	acknowledgedAt := firstObservedAt.Add(time.Minute)
	alert := &Alert{
		ID:              "alert-1",
		Type:            "offline",
		Level:           AlertLevelCritical,
		ResourceID:      "resource-1",
		CanonicalSpecID: "connectivity:offline",
		CanonicalState:  "resource-1::connectivity:offline",
		StartTime:       firstObservedAt,
		LastSeen:        acknowledgedAt,
		Acknowledged:    true,
		AckTime:         &acknowledgedAt,
		AckUser:         "operator-1",
	}

	ensureOperationalContract(alert, acknowledgedAt)

	record := alert.OperationalRecord
	if record == nil {
		t.Fatal("operational record is nil")
	}
	if record.State != operationaltrust.OperationalAcknowledged {
		t.Fatalf("state = %q, want acknowledged", record.State)
	}
	if record.ResolvedAt != nil {
		t.Fatalf("acknowledged record resolvedAt = %v, want nil", record.ResolvedAt)
	}
	if record.Acknowledgement == nil || record.Acknowledgement.By != "operator-1" {
		t.Fatalf("acknowledgement = %+v, want operator-1", record.Acknowledgement)
	}
	if alert.LatestTransition == nil ||
		alert.LatestTransition.Cause != operationaltrust.TransitionAcknowledgement {
		t.Fatalf("latest transition = %+v, want acknowledgement", alert.LatestTransition)
	}
	if alert.LatestTransition.To != operationaltrust.OperationalAcknowledged {
		t.Fatalf("transition to = %q, want acknowledged", alert.LatestTransition.To)
	}
}

func TestEnsureOperationalContractTracksUnacknowledgement(t *testing.T) {
	firstObservedAt := time.Date(2026, 7, 18, 20, 0, 0, 0, time.UTC)
	acknowledgedAt := firstObservedAt.Add(time.Minute)
	unacknowledgedAt := acknowledgedAt.Add(time.Minute)
	alert := &Alert{
		ID:              "alert-1",
		Type:            "offline",
		Level:           AlertLevelCritical,
		ResourceID:      "resource-1",
		CanonicalSpecID: "connectivity:offline",
		CanonicalState:  "resource-1::connectivity:offline",
		StartTime:       firstObservedAt,
		LastSeen:        firstObservedAt,
	}
	ensureOperationalContract(alert, firstObservedAt)

	alert.Acknowledged = true
	alert.AckTime = &acknowledgedAt
	alert.AckUser = "operator-1"
	alert.LastSeen = acknowledgedAt
	ensureOperationalContract(alert, acknowledgedAt)

	alert.Acknowledged = false
	alert.AckTime = nil
	alert.AckUser = ""
	alert.LastSeen = unacknowledgedAt
	ensureOperationalContract(alert, unacknowledgedAt)

	if alert.OperationalRecord == nil ||
		alert.OperationalRecord.State != operationaltrust.OperationalOpen {
		t.Fatalf("record = %+v, want open", alert.OperationalRecord)
	}
	if alert.LatestTransition == nil ||
		alert.LatestTransition.Cause != operationaltrust.TransitionUnacknowledgement {
		t.Fatalf("latest transition = %+v, want unacknowledgement", alert.LatestTransition)
	}
	if len(alert.Transitions) != 3 {
		t.Fatalf("transition count = %d, want open, acknowledged, open", len(alert.Transitions))
	}
	for _, transition := range alert.Transitions {
		if err := transition.Validate(); err != nil {
			t.Fatalf("transition %q Validate() error = %v", transition.ID, err)
		}
	}
}

func TestNewResolvedAlertAddsDistinctRecoveryEvidence(t *testing.T) {
	firstObservedAt := time.Date(2026, 7, 18, 20, 0, 0, 0, time.UTC)
	resolvedAt := firstObservedAt.Add(5 * time.Minute)
	alert := &Alert{
		ID:              "alert-1",
		Type:            "cpu",
		Level:           AlertLevelWarning,
		ResourceID:      "resource-1",
		CanonicalSpecID: "metric-threshold:cpu",
		CanonicalState:  "resource-1::metric-threshold:cpu",
		StartTime:       firstObservedAt,
		LastSeen:        firstObservedAt,
	}
	ensureOperationalContract(alert, firstObservedAt)
	triggerEvidenceID := alert.Evidence[0].ID

	resolved := newResolvedAlert(alert, resolvedAt, nil)

	if resolved == nil || resolved.OperationalRecord == nil {
		t.Fatal("resolved operational record is nil")
	}
	if resolved.OperationalRecord.State != operationaltrust.OperationalResolved {
		t.Fatalf("resolved state = %q, want resolved", resolved.OperationalRecord.State)
	}
	if len(resolved.Evidence) != 2 {
		t.Fatalf("resolved evidence count = %d, want trigger and recovery", len(resolved.Evidence))
	}
	recoveryEvidence := resolved.Evidence[1]
	if recoveryEvidence.ID == triggerEvidenceID {
		t.Fatal("recovery evidence reused trigger evidence id")
	}
	if recoveryEvidence.Reason == nil ||
		recoveryEvidence.Reason.Code != legacyAlertRecoveryEvidenceReason {
		t.Fatalf("recovery reason = %+v, want legacy recovery projection", recoveryEvidence.Reason)
	}
	if resolved.LatestTransition == nil ||
		resolved.LatestTransition.Cause != operationaltrust.TransitionRecoveryEvidence {
		t.Fatalf("resolved transition = %+v, want recovery evidence", resolved.LatestTransition)
	}
	if len(resolved.LatestTransition.EvidenceIDs) != 1 ||
		resolved.LatestTransition.EvidenceIDs[0] != recoveryEvidence.ID {
		t.Fatalf(
			"resolution transition evidence = %v, want %q",
			resolved.LatestTransition.EvidenceIDs,
			recoveryEvidence.ID,
		)
	}
	if err := resolved.OperationalRecord.Validate(); err != nil {
		t.Fatalf("resolved operational record Validate() error = %v", err)
	}
	if err := resolved.LatestTransition.Validate(); err != nil {
		t.Fatalf("resolved transition Validate() error = %v", err)
	}
	if len(resolved.Transitions) != 2 {
		t.Fatalf("resolved transition count = %d, want trigger and resolution", len(resolved.Transitions))
	}
	if resolved.Transitions[0].ID == resolved.Transitions[1].ID ||
		resolved.Transitions[1].ID != resolved.LatestTransition.ID {
		t.Fatalf("resolved transitions = %+v, want distinct ordered timeline", resolved.Transitions)
	}
}

func TestResolvedOperationalContractUpdatesHistorySnapshot(t *testing.T) {
	firstObservedAt := time.Date(2026, 7, 18, 20, 0, 0, 0, time.UTC)
	resolvedAt := firstObservedAt.Add(5 * time.Minute)
	alert := &Alert{
		ID:              "alert-1",
		Type:            "cpu",
		Level:           AlertLevelWarning,
		ResourceID:      "resource-1",
		CanonicalSpecID: "metric-threshold:cpu",
		CanonicalState:  "resource-1::metric-threshold:cpu",
		StartTime:       firstObservedAt,
		LastSeen:        firstObservedAt,
	}
	ensureOperationalContract(alert, firstObservedAt)

	history := newTestHistoryManager(t)
	history.AddAlert(*alert)
	manager := &Manager{historyManager: history}
	if resolved := manager.newResolvedAlert(alert, resolvedAt, nil); resolved == nil {
		t.Fatal("newResolvedAlert() returned nil")
	}

	entries := history.GetAllHistory(1)
	if len(entries) != 1 {
		t.Fatalf("history entries = %d, want 1", len(entries))
	}
	entry := entries[0]
	if entry.OperationalRecord == nil ||
		entry.OperationalRecord.State != operationaltrust.OperationalResolved {
		t.Fatalf("history operational record = %+v, want resolved", entry.OperationalRecord)
	}
	if entry.LatestTransition == nil ||
		entry.LatestTransition.ID != alert.LatestTransition.ID {
		t.Fatalf("history latest transition = %+v, want %q", entry.LatestTransition, alert.LatestTransition.ID)
	}
	if len(entry.Transitions) != 2 || len(entry.Evidence) != 2 {
		t.Fatalf(
			"history contract transitions/evidence = %d/%d, want 2/2",
			len(entry.Transitions),
			len(entry.Evidence),
		)
	}
}

func TestReleasedMetricAlertCarriesResolutionInsteadOfRecovery(t *testing.T) {
	manager := newUnifiedEvalParityManager(t)
	store, err := eventlog.OpenInMemory()
	if err != nil {
		t.Fatalf("open in-memory event log: %v", err)
	}
	manager.SetEventLog(store)
	manager.UpdateConfig(AlertConfig{
		Enabled: true,
		NodeDefaults: ThresholdConfig{
			CPU:    &HysteresisThreshold{Trigger: 80, Clear: 75},
			Memory: &HysteresisThreshold{Trigger: 85, Clear: 80},
		},
	})
	disableTestTimeThresholds(manager)
	node := models.Node{
		ID:       "pve-1",
		Name:     "pve-1",
		Instance: "test",
		Status:   "online",
		CPU:      0.95,
		Memory:   models.Memory{Total: 8 << 30, Used: 7 << 30, Usage: 95},
	}
	manager.CheckNode(node)
	if active := manager.GetActiveAlerts(); len(active) != 2 {
		t.Fatalf("active alerts = %#v, want cpu and memory", active)
	}

	resolvedByType := map[string]*ResolvedAlert{}
	manager.SetResolvedAlertCallback(func(resolved *ResolvedAlert) {
		resolvedByType[resolved.Alert.Type] = resolved
	})
	release := func(metric string, resolution *AlertResolution) {
		t.Helper()
		spec, err := buildCanonicalMetricSpec(node.ID, node.Name, unifiedresources.ResourceType("node"), metric, nil)
		if err != nil {
			t.Fatalf("build %s spec: %v", metric, err)
		}
		manager.releaseCanonicalMetricAlert(spec, node.Name, node.Name, node.Instance, "node", 0, resolution)
	}

	handover := &AlertResolution{
		Reason:              AlertResolutionMovedToAgent,
		SuccessorResourceID: "agent-pve-1",
		SuccessorName:       "pve-1 (Host Agent)",
	}
	release("memory", handover)
	// The close is recorded as a copy: the caller's value cannot rewrite it.
	handover.SuccessorName = "changed"
	release("cpu", nil)

	if active := manager.GetActiveAlerts(); len(active) != 0 {
		t.Fatalf("active alerts after release = %#v, want none: the close must still happen", active)
	}

	moved := resolvedByType["memory"]
	if moved == nil {
		t.Fatal("memory release did not reach the resolved consumer")
	}
	resolution := moved.Alert.Resolution
	if resolution == nil ||
		resolution.Reason != AlertResolutionMovedToAgent ||
		resolution.SuccessorResourceID != "agent-pve-1" ||
		resolution.SuccessorName != "pve-1 (Host Agent)" {
		t.Fatalf("resolution = %+v, want moved to pve-1 (Host Agent)", resolution)
	}
	wantSummary := "Alert moved to pve-1 (Host Agent). This is not a recovery: check the agent for the current reading."
	if got := resolution.Summary(); got != wantSummary {
		t.Fatalf("Summary() = %q, want %q", got, wantSummary)
	}
	if got := resolution.Outcome(); got != "moved to pve-1 (Host Agent)" {
		t.Fatalf("Outcome() = %q", got)
	}
	if got := resolution.Describe("Memory alert"); got != "Memory alert moved to pve-1 (Host Agent). This is not a recovery: check the agent for the current reading." {
		t.Fatalf("Describe() = %q", got)
	}

	record := moved.Alert.OperationalRecord
	if record == nil || record.State != operationaltrust.OperationalResolved {
		t.Fatalf("operational record = %+v, want resolved", record)
	}
	if err := record.Validate(); err != nil {
		t.Fatalf("operational record Validate() error = %v", err)
	}
	transition := moved.Alert.LatestTransition
	if transition == nil || transition.Reason != wantSummary {
		t.Fatalf("resolve transition = %+v, want reason %q", transition, wantSummary)
	}
	if err := transition.Validate(); err != nil {
		t.Fatalf("resolve transition Validate() error = %v", err)
	}
	// Patrol's timeline prints the cause above the reason, so a handover
	// must not be labelled as recovery evidence.
	if transition.Cause != operationaltrust.TransitionOwnershipTransferred {
		t.Fatalf("resolve transition cause = %q, want %q", transition.Cause, operationaltrust.TransitionOwnershipTransferred)
	}
	if recorded := moved.Alert.Transitions[len(moved.Alert.Transitions)-1]; recorded.ID != transition.ID ||
		recorded.Cause != operationaltrust.TransitionOwnershipTransferred {
		t.Fatalf("recorded timeline transition = %+v, want the ownership transfer", recorded)
	}
	closing := moved.Alert.Evidence[len(moved.Alert.Evidence)-1]
	if closing.ID != transition.EvidenceIDs[0] ||
		closing.Reason == nil ||
		closing.Reason.Code != string(AlertResolutionMovedToAgent) ||
		closing.Reason.Message != wantSummary {
		t.Fatalf("closing evidence = %+v, want the handover, not a recovery projection", closing)
	}

	recovered := resolvedByType["cpu"]
	if recovered == nil || recovered.Alert.Resolution != nil {
		t.Fatalf("plain disabled release = %+v, want an ordinary resolve with no resolution", recovered)
	}
	if last := recovered.Alert.Evidence[len(recovered.Alert.Evidence)-1]; last.Reason == nil ||
		last.Reason.Code != legacyAlertRecoveryEvidenceReason {
		t.Fatalf("plain release evidence = %+v, want legacy recovery projection", last.Reason)
	}
	if recovered.Alert.LatestTransition == nil ||
		recovered.Alert.LatestTransition.Cause != operationaltrust.TransitionRecoveryEvidence {
		t.Fatalf("plain release transition = %+v, want recovery_evidence", recovered.Alert.LatestTransition)
	}

	var historyResolution *AlertResolution
	for _, entry := range manager.GetAlertHistory(10) {
		if entry.Type == "memory" {
			historyResolution = entry.Resolution
		}
	}
	if historyResolution == nil || historyResolution.Reason != AlertResolutionMovedToAgent {
		t.Fatalf("history resolution = %+v, want moved_to_agent", historyResolution)
	}

	events, err := manager.AlertEvents(eventlog.Filter{AlertID: moved.Alert.ID, Types: []string{eventlog.TypeResolved}})
	if err != nil {
		t.Fatalf("AlertEvents: %v", err)
	}
	if len(events) != 1 ||
		events[0].Reason != string(AlertResolutionMovedToAgent) ||
		events[0].Message != wantSummary {
		t.Fatalf("resolved events = %+v, want one handover event", events)
	}
}

func TestResolvedHistoryRowKeepsResolutionWithoutEventLog(t *testing.T) {
	firstObservedAt := time.Date(2026, 10, 6, 9, 0, 0, 0, time.UTC)
	alert := &Alert{
		ID:              "pve-1-memory",
		Type:            "memory",
		Level:           AlertLevelWarning,
		ResourceID:      "pve-1",
		CanonicalSpecID: "metric-threshold:memory",
		CanonicalState:  "pve-1::metric-threshold:memory",
		StartTime:       firstObservedAt,
		LastSeen:        firstObservedAt,
	}
	ensureOperationalContract(alert, firstObservedAt)
	history := newTestHistoryManager(t)
	history.AddAlert(*alert)
	manager := &Manager{historyManager: history}

	alert.Resolution = &AlertResolution{Reason: AlertResolutionMovedToAgent, SuccessorName: "pve-1 (Host Agent)"}
	manager.newResolvedAlert(alert, firstObservedAt.Add(time.Minute), nil)

	entries := history.GetAllHistory(1)
	if len(entries) != 1 || entries[0].Resolution == nil ||
		entries[0].Resolution.Reason != AlertResolutionMovedToAgent {
		t.Fatalf("history entries = %+v, want the row to keep the moved_to_agent resolution", entries)
	}
}

func TestMergeOperationalRecurrenceReopensStableCause(t *testing.T) {
	firstObservedAt := time.Date(2026, 7, 18, 20, 0, 0, 0, time.UTC)
	resolvedAt := firstObservedAt.Add(5 * time.Minute)
	recurredAt := resolvedAt.Add(time.Minute)
	previous := &Alert{
		ID:              "alert-1",
		Type:            "cpu",
		Level:           AlertLevelWarning,
		ResourceID:      "resource-1",
		CanonicalSpecID: "metric-threshold:cpu",
		CanonicalState:  "resource-1::metric-threshold:cpu",
		StartTime:       firstObservedAt,
		LastSeen:        firstObservedAt,
	}
	ensureOperationalContract(previous, firstObservedAt)
	newResolvedAlert(previous, resolvedAt, nil)

	recurrence := &Alert{
		ID:              previous.ID,
		Type:            previous.Type,
		Level:           previous.Level,
		ResourceID:      previous.ResourceID,
		CanonicalSpecID: previous.CanonicalSpecID,
		CanonicalState:  previous.CanonicalState,
		StartTime:       previous.StartTime,
		LastSeen:        recurredAt,
	}
	ensureOperationalContract(recurrence, recurredAt)
	recurrenceEvidenceID := recurrence.Evidence[0].ID

	if !mergeOperationalRecurrence(recurrence, previous, recurredAt) {
		t.Fatal("mergeOperationalRecurrence() did not reopen resolved cause")
	}
	if recurrence.OperationalRecord == nil ||
		recurrence.OperationalRecord.ID != previous.OperationalRecord.ID ||
		recurrence.OperationalRecord.State != operationaltrust.OperationalOpen ||
		recurrence.OperationalRecord.ResolvedAt != nil {
		t.Fatalf("recurrence record = %+v", recurrence.OperationalRecord)
	}
	if recurrence.LatestTransition == nil ||
		recurrence.LatestTransition.From != operationaltrust.OperationalResolved ||
		recurrence.LatestTransition.To != operationaltrust.OperationalOpen ||
		recurrence.LatestTransition.Cause != operationaltrust.TransitionDetectorDecision {
		t.Fatalf("recurrence transition = %+v", recurrence.LatestTransition)
	}
	if len(recurrence.Transitions) != 3 {
		t.Fatalf("recurrence transition count = %d, want open, resolved, reopened", len(recurrence.Transitions))
	}
	if len(recurrence.Evidence) != 3 {
		t.Fatalf("recurrence evidence count = %d, want trigger, recovery, recurrence", len(recurrence.Evidence))
	}
	if recurrence.Evidence[2].ID != recurrenceEvidenceID {
		t.Fatalf("recurrence evidence id = %q, want %q", recurrence.Evidence[2].ID, recurrenceEvidenceID)
	}
	for _, transition := range recurrence.Transitions {
		if err := transition.Validate(); err != nil {
			t.Fatalf("transition %q Validate() error = %v", transition.ID, err)
		}
	}
}

func TestEnsureOperationalContractDoesNotInventCanonicalSubject(t *testing.T) {
	alert := &Alert{
		ID:        "provider-only-alert",
		Type:      "provider-incident",
		Instance:  "provider-a",
		StartTime: time.Date(2026, 7, 18, 20, 0, 0, 0, time.UTC),
	}

	ensureOperationalContract(alert, alert.StartTime)

	if alert.OperationalRecord != nil {
		t.Fatalf("operational record = %+v, want nil without canonical subject", alert.OperationalRecord)
	}
	if len(alert.Evidence) != 1 || alert.Evidence[0].Subject.ProviderRef == "" {
		t.Fatalf("unresolved evidence = %+v, want scoped provider reference", alert.Evidence)
	}
}

func TestAlertCloneDoesNotShareOperationalContractState(t *testing.T) {
	now := time.Date(2026, 7, 18, 20, 0, 0, 0, time.UTC)
	alert := &Alert{
		ID:              "alert-1",
		Type:            "cpu",
		Level:           AlertLevelWarning,
		ResourceID:      "resource-1",
		CanonicalSpecID: "metric-threshold:cpu",
		CanonicalState:  "resource-1::metric-threshold:cpu",
		StartTime:       now,
		LastSeen:        now,
	}
	ensureOperationalContract(alert, now)

	clone := alert.Clone()
	clone.OperationalRecord.EvidenceIDs[0] = "mutated"
	clone.LatestTransition.EvidenceIDs[0] = "mutated"
	clone.Transitions[0].EvidenceIDs[0] = "mutated"
	clone.Evidence[0].Reason.Code = "mutated"

	if alert.OperationalRecord.EvidenceIDs[0] == "mutated" ||
		alert.LatestTransition.EvidenceIDs[0] == "mutated" ||
		alert.Transitions[0].EvidenceIDs[0] == "mutated" ||
		alert.Evidence[0].Reason.Code == "mutated" {
		t.Fatal("Alert.Clone() shared operational contract state")
	}
}

func TestAlertOperationalContractJSONRemainsAdditive(t *testing.T) {
	legacyPayload := []byte(`{
		"id":"alert-1",
		"type":"cpu",
		"level":"warning",
		"resourceId":"resource-1",
		"resourceName":"vm-1",
		"node":"node-1",
		"instance":"pve-1",
		"message":"CPU high",
		"value":95,
		"threshold":90,
		"startTime":"2026-07-18T20:00:00Z",
		"lastSeen":"2026-07-18T20:00:00Z",
		"acknowledged":false
	}`)
	var alert Alert
	if err := json.Unmarshal(legacyPayload, &alert); err != nil {
		t.Fatalf("legacy alert unmarshal error = %v", err)
	}
	ensureOperationalContract(&alert, alert.LastSeen)
	payload, err := json.Marshal(alert)
	if err != nil {
		t.Fatalf("operational alert marshal error = %v", err)
	}
	var roundTrip Alert
	if err := json.Unmarshal(payload, &roundTrip); err != nil {
		t.Fatalf("operational alert unmarshal error = %v", err)
	}
	if roundTrip.ID != "alert-1" ||
		roundTrip.OperationalRecord == nil ||
		len(roundTrip.Transitions) != 1 ||
		len(roundTrip.Evidence) != 1 {
		t.Fatalf("round-trip alert = %+v", roundTrip)
	}
}

func TestEnsureOperationalContractKeepsExplicitUncertainStateAheadOfLegacyAcknowledgement(t *testing.T) {
	observedAt := time.Date(2026, 7, 19, 6, 0, 0, 0, time.UTC)
	alert := &Alert{
		ID:              "alert-uncertain",
		Type:            "availability",
		ResourceID:      "app-container:web",
		CanonicalSpecID: "availability",
		CanonicalState:  "app-container:web::availability",
		StartTime:       observedAt.Add(-time.Hour),
		LastSeen:        observedAt.Add(-time.Minute),
		Acknowledged:    true,
		OperationalRecord: &operationaltrust.OperationalRecord{
			ID:                "alert-uncertain",
			CanonicalSpecID:   "availability",
			SubjectResourceID: "app-container:web",
			State:             operationaltrust.OperationalStale,
			Severity:          operationaltrust.SeverityWarning,
			FirstObservedAt:   observedAt.Add(-time.Hour),
			LastObservedAt:    observedAt,
			StateChangedAt:    observedAt.Add(-time.Minute),
			CauseKey:          "availability",
			EvidenceIDs:       []string{},
		},
	}

	ensureOperationalContract(alert, observedAt.Add(time.Minute))

	if got := alert.OperationalRecord.State; got != operationaltrust.OperationalStale {
		t.Fatalf("operational state = %q, want stale", got)
	}
	if got := alert.OperationalRecord.LastObservedAt; !got.Equal(observedAt) {
		t.Fatalf("LastObservedAt = %s, want %s", got, observedAt)
	}
}

func TestMemoryUnavailablePreservesActiveAlertsUntilTrustedRecovery(t *testing.T) {
	tests := []struct {
		name  string
		setup func(*Manager)
		check func(*Manager, models.Memory)
	}{
		{
			name: "guest",
			setup: func(manager *Manager) {
				manager.UpdateConfig(AlertConfig{
					Enabled: true,
					GuestDefaults: ThresholdConfig{
						Memory: &HysteresisThreshold{Trigger: 85, Clear: 80},
					},
				})
			},
			check: func(manager *Manager, memory models.Memory) {
				manager.CheckGuest(models.VM{
					ID:       "qemu/1501",
					Name:     "linux-guest",
					Node:     "pve-1",
					Instance: "test",
					Status:   "running",
					Memory:   memory,
				}, "test")
			},
		},
		{
			name: "node",
			setup: func(manager *Manager) {
				manager.UpdateConfig(AlertConfig{
					Enabled: true,
					NodeDefaults: ThresholdConfig{
						Memory: &HysteresisThreshold{Trigger: 85, Clear: 80},
					},
				})
			},
			check: func(manager *Manager, memory models.Memory) {
				manager.CheckNode(models.Node{
					ID:       "pve-1",
					Name:     "pve-1",
					Instance: "test",
					Status:   "online",
					Memory:   memory,
				})
			},
		},
		{
			name: "agent",
			setup: func(manager *Manager) {
				manager.UpdateConfig(AlertConfig{
					Enabled: true,
					AgentDefaults: ThresholdConfig{
						Memory: &HysteresisThreshold{Trigger: 85, Clear: 80},
					},
				})
			},
			check: func(manager *Manager, memory models.Memory) {
				manager.CheckHost(models.Host{
					ID:       "agent-1501",
					Hostname: "linux-host",
					Status:   "online",
					Memory:   memory,
				})
			},
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			manager := newUnifiedEvalParityManager(t)
			tt.setup(manager)
			disableTestTimeThresholds(manager)

			tt.check(manager, models.Memory{
				Total: 8 << 30,
				Used:  7 << 30,
				Usage: 90,
			})
			active := manager.GetActiveAlerts()
			if len(active) != 1 || active[0].Type != "memory" {
				t.Fatalf("active alerts after trusted high sample = %#v, want one memory alert", active)
			}

			tt.check(manager, models.UnavailableMemory(8<<30))
			active = manager.GetActiveAlerts()
			if len(active) != 1 || active[0].Type != "memory" || active[0].Value != 90 {
				t.Fatalf("active alerts after unavailable sample = %#v, want prior memory alert", active)
			}

			tt.check(manager, models.Memory{
				Total: 8 << 30,
				Used:  5 << 30,
				Usage: 70,
			})
			if active = manager.GetActiveAlerts(); len(active) != 0 {
				t.Fatalf("active alerts after trusted recovery = %#v, want cleared", active)
			}
		})
	}
}

func TestResolvedOccurrenceCallbackCapturesBeforeDispatch(t *testing.T) {
	m := newTestManager(t)
	start := time.Now().Add(-time.Hour)
	end := start.Add(time.Minute)
	resolved := &ResolvedAlert{Alert: &Alert{ID: "reusable", StartTime: start,
		Metadata: map[string]interface{}{"generation": "old"}}, ResolvedTime: end}
	entered, release := make(chan struct{}), make(chan struct{})
	defer close(release)
	m.SetResolvedCallback(func(id string) {
		if id != "reusable" {
			t.Errorf("legacy ID = %s", id)
		}
		close(entered)
		<-release
	})
	got := make(chan *ResolvedAlert, 1)
	m.SetResolvedAlertCallback(func(r *ResolvedAlert) { got <- r })
	m.safeCallResolvedAlertCallback(resolved, resolved.Alert.ID, true)
	select {
	case <-entered:
	case <-time.After(time.Second):
		t.Fatal("legacy callback did not start")
	}
	// Both the source object and the ID-indexed history can change while the
	// consumer waits. Neither is a source for the already-admitted snapshot.
	resolved.Alert.StartTime = start.Add(time.Hour)
	resolved.Alert.Metadata["generation"] = "new"
	resolved.ResolvedTime = end.Add(time.Hour)
	m.addRecentlyResolvedUnlocked(resolved)
	// Unblock without closing twice in cleanup.
	release <- struct{}{}
	select {
	case r := <-got:
		if !r.Alert.StartTime.Equal(start) || !r.ResolvedTime.Equal(end) || r.Alert.Metadata["generation"] != "old" {
			t.Fatalf("callback reconstructed the wrong occurrence: %+v", r)
		}
	case <-time.After(time.Second):
		t.Fatal("no occurrence callback")
	}
}

func TestResolvedOccurrenceCallbackSurvivesLegacyPanic(t *testing.T) {
	m := newTestManager(t)
	m.SetResolvedCallback(func(string) { panic("legacy consumer failure") })
	called := false
	m.SetResolvedAlertCallback(func(r *ResolvedAlert) { called = r.Alert.ID == "reusable" })
	m.safeCallResolvedAlertCallback(&ResolvedAlert{Alert: &Alert{ID: "reusable"}}, "reusable", false)
	if !called {
		t.Fatal("legacy panic prevented occurrence-qualified recovery")
	}
}

func TestConnectionDegradedAlertNamesItsPlatform(t *testing.T) {
	tests := []struct {
		connectionType ConnectionType
		wantPlatform   string
	}{
		{connectionType: ConnectionTypePVE, wantPlatform: "proxmox-pve"},
		{connectionType: ConnectionTypePBS, wantPlatform: "proxmox-pbs"},
		{connectionType: ConnectionTypePMG, wantPlatform: "proxmox-pmg"},
		{connectionType: ConnectionTypeVMware, wantPlatform: "vmware-vsphere"},
		{connectionType: ConnectionTypeTrueNAS, wantPlatform: "truenas"},
	}

	for _, test := range tests {
		t.Run(string(test.connectionType), func(t *testing.T) {
			m := newTestManager(t)
			snap := ConnectionSnapshot{
				ID:      string(test.connectionType) + ":lab",
				Name:    "Lab",
				Type:    test.connectionType,
				State:   ConnectionStateUnreachable,
				Enabled: true,
			}
			for range 5 {
				m.CheckConnection(snap)
			}

			alert := testRequireActiveAlert(t, m, canonicalDiscreteStateStateID(snap.ID, connectionDegradedStateKey))
			if got, _ := alert.Metadata["platformType"].(string); got != test.wantPlatform {
				t.Fatalf("platformType metadata = %q, want %q", got, test.wantPlatform)
			}
		})
	}
}

// A partial platform read (the connection answered, one reading failed) is a
// warning-level degraded observation, never an outage (#2960).
func TestConnectionDegradedPartialReadAlertsAtWarning(t *testing.T) {
	t.Run("degraded fires a warning, never critical", func(t *testing.T) {
		m := newTestManager(t)
		snap := platformConnectionSnapshot("conn-partial", "Mail gateway", ConnectionStateDegraded)
		snap.Type = ConnectionTypePMG
		snap.StateReason = "pmg_queue_status: api error 500"
		snap.LastError = &ConnectionErrorSnapshot{At: time.Now(), Message: snap.StateReason, Category: "partial"}
		alertID := canonicalDiscreteStateStateID(snap.ID, connectionDegradedStateKey)

		m.CheckConnection(snap)
		m.CheckConnection(snap)
		if testHasActiveAlert(t, m, alertID) {
			t.Fatal("expected no alert before three degraded observations")
		}
		m.CheckConnection(snap)

		alert := testRequireActiveAlert(t, m, alertID)
		if alert.Level != AlertLevelWarning {
			t.Errorf("expected warning level for degraded, got %s", alert.Level)
		}
		if got, _ := alert.Metadata["state"].(string); got != string(ConnectionStateDegraded) {
			t.Errorf("expected state=degraded in metadata, got %q", got)
		}
		if got, _ := alert.Metadata["lastErrorCategory"].(string); got != "partial" {
			t.Errorf("expected lastErrorCategory=partial in metadata, got %q", got)
		}
	})

	t.Run("degraded keeps an unreachable outage open at warning", func(t *testing.T) {
		m := newTestManager(t)
		snap := platformConnectionSnapshot("conn-partial-recover", "Mail gateway", ConnectionStateUnreachable)
		snap.Type = ConnectionTypePMG
		alertID := canonicalDiscreteStateStateID(snap.ID, connectionDegradedStateKey)

		m.CheckConnection(snap)
		m.CheckConnection(snap)
		m.CheckConnection(snap)
		if alert := testRequireActiveAlert(t, m, alertID); alert.Level != AlertLevelCritical {
			t.Fatalf("expected critical outage before the partial poll, got %s", alert.Level)
		}

		// The connection answers again but one reading still fails: a
		// missing reading is not recovery evidence.
		snap.State = ConnectionStateDegraded
		snap.StateReason = "pmg_queue_status: api error 500"
		for i := 0; i < 5; i++ {
			m.CheckConnection(snap)
		}
		alert := testRequireActiveAlert(t, m, alertID)
		if alert.Level != AlertLevelWarning {
			t.Errorf("expected the open alert to drop to warning, got %s", alert.Level)
		}
		if got, _ := alert.Metadata["state"].(string); got != string(ConnectionStateDegraded) {
			t.Errorf("expected state=degraded in metadata, got %q", got)
		}
	})
}

func TestCapacityForecastAlertNamesItsPlatform(t *testing.T) {
	m := newTestManager(t)
	m.ClearActiveAlerts()
	m.mu.Lock()
	m.config.TimeThresholds = map[string]int{}
	m.config.VMwareDefaults.Usage = &HysteresisThreshold{Trigger: 85, Clear: 75}
	m.mu.Unlock()

	input := &UnifiedResourceInput{
		ID:       "vmware:vc-1/datastore:ds-ssd",
		Type:     "vmware-datastore",
		Name:     "ds-ssd",
		Node:     "Lab DC",
		Instance: "Lab vCenter",
		Disk:     &UnifiedResourceMetric{Percent: 70},
	}
	trend := CapacityTrendObservation{
		Ready: true, Reason: "increasing", ObservedAt: time.Now(), DailyChange: 7,
		Confidence: 0.99, SampleCount: 300, BucketCount: 48, CoverageSpan: 48 * time.Hour,
	}
	m.CheckUnifiedResourceWithCapacityTrend(input, trend)
	m.CheckUnifiedResourceWithCapacityTrend(input, trend)

	alert := testRequireActiveAlert(t, m, canonicalMetricStateID(input.ID, "usage"))
	if got := alert.Metadata[capacityAlertOriginKey]; got != capacityAlertOriginForecast {
		t.Fatalf("capacity origin = %v, want forecast", got)
	}
	if got := alert.Metadata[alertPlatformTypeKey]; got != "vmware-vsphere" {
		t.Fatalf("platformType metadata = %v, want vmware-vsphere", got)
	}
}

// History output keeps an alert's operational record and transitions whole but
// only its newest evidence envelope, and never changes the alert it copies
// (#1803).
func TestHistoryOutputKeepsTheRecordAndOnlyTheNewestEvidence(t *testing.T) {
	observed := time.Date(2026, 9, 1, 12, 0, 0, 0, time.UTC)
	alert := Alert{
		ID:         "disk-1",
		ResourceID: "vm-1",
		Level:      AlertLevelWarning,
		StartTime:  observed,
		LastSeen:   observed,
		Evidence: []operationaltrust.EvidenceEnvelope{
			{ID: "ev-new", ObservedAt: observed.Add(time.Hour)},
			{ID: "ev-old", ObservedAt: observed},
		},
		OperationalRecord: &operationaltrust.OperationalRecord{ID: "record-1", EvidenceIDs: []string{"ev-old", "ev-new"}},
		Transitions:       []operationaltrust.LifecycleTransition{{ID: "transition-1", EvidenceIDs: []string{"ev-old"}}},
	}
	output := canonicalizeAlertHistoryForOutput([]Alert{alert})
	if len(output) != 1 {
		t.Fatalf("output = %d entries", len(output))
	}
	if got := output[0].Evidence; len(got) != 1 || got[0].ID != "ev-new" {
		t.Fatalf("history evidence = %+v, want only ev-new", got)
	}
	if got := output[0].OperationalRecord; got == nil || len(got.EvidenceIDs) != 2 {
		t.Fatalf("the operational record must keep both evidence IDs, got %+v", got)
	}
	if got := output[0].Transitions; len(got) != 1 || len(got[0].EvidenceIDs) != 1 {
		t.Fatalf("transitions must be kept whole, got %+v", got)
	}
	if len(alert.Evidence) != 2 {
		t.Fatalf("shaping history output changed its input alert to %d envelopes", len(alert.Evidence))
	}
}

func nativeSensorManager(t *testing.T) *Manager {
	t.Helper()
	m, _ := continuityManager(t, false)
	m.mu.Lock()
	m.config.MetricTimeThresholds = map[string]map[string]int{"all": {"temperature": 0}}
	m.mu.Unlock()
	return m
}

func nativeSensorHost(value *float64, status string) models.Host {
	return models.Host{ID: "host-mbp", Hostname: "macbook-pve", Status: "online", LinkedNodeID: "node-mbp",
		Sensors: models.HostSensorSummary{Native: []models.HostNativeSensor{
			{ID: "applesmc-isa-0300/temp1", Chip: "applesmc-isa-0300", Channel: "temp1", Label: "TB0T", Kind: "temperature", Unit: "celsius", Value: value, Status: status},
			// Differs from TB0T's ID only by case after the channel; must stay its own resource.
			{ID: "applesmc-isa-0300/temp2", Chip: "applesmc-isa-0300", Channel: "temp2", Label: "TC1f", Kind: "temperature", Unit: "celsius", Value: value, Status: status},
		}}}
}

func nativeSensorAlerts(m *Manager) []Alert {
	var out []Alert
	for _, alert := range m.GetActiveAlerts() {
		if isHostNativeSensorResourceID(alert.ResourceID) {
			out = append(out, alert)
		}
	}
	return out
}

func TestHostNativeSensorResourceIDIsCollisionFree(t *testing.T) {
	// Lower-case hex of the UTF-8 bytes: case, punctuation and non-ASCII
	// differences survive, and the key carries no '/' or ':'.
	for sensorID, want := range map[string]string{
		"applesmc-isa-0300/temp1": "agent:h/sensor:6170706c65736d632d6973612d303330302f74656d7031",
		"A/temp1":                 "agent:h/sensor:412f74656d7031",
		"a/temp1":                 "agent:h/sensor:612f74656d7031",
		"é/temp1":                 "agent:h/sensor:c3a92f74656d7031",
	} {
		if got := HostNativeSensorResourceID("h", sensorID); got != want {
			t.Errorf("HostNativeSensorResourceID(%q) = %q, want %q", sensorID, got, want)
		}
	}
	if !isHostNativeSensorResourceID("agent:h/sensor:00") || isHostNativeSensorResourceID("agent:h/custom:fan") || isHostNativeSensorResourceID("agent:h") {
		t.Fatal("native sensor resource ID classification is wrong")
	}
}

func TestHostNativeSensorThresholdRaisesOnlyFromItsOwnOverride(t *testing.T) {
	m := nativeSensorManager(t)
	hot := 47.0
	host := nativeSensorHost(&hot, "ok")
	battery := HostNativeSensorResourceID(host.ID, "applesmc-isa-0300/temp1")

	// No override: no default, and neither the host's nor the linked node's
	// CPU temperature threshold is inherited.
	m.mu.Lock()
	m.config.AgentDefaults.Temperature = &HysteresisThreshold{Trigger: 30, Clear: 25}
	m.config.Overrides[host.LinkedNodeID] = ThresholdConfig{Temperature: &HysteresisThreshold{Trigger: 30, Clear: 25}}
	m.mu.Unlock()
	m.CheckHost(host)
	if alerts := nativeSensorAlerts(m); len(alerts) != 0 {
		t.Fatalf("sensor without its own override raised %+v", alerts)
	}

	m.mu.Lock()
	m.config.Overrides[battery] = ThresholdConfig{Temperature: &HysteresisThreshold{Trigger: 45, Clear: 40}}
	m.mu.Unlock()
	m.CheckHost(host)
	alerts := nativeSensorAlerts(m)
	if len(alerts) != 1 {
		t.Fatalf("battery override raised %d sensor alerts, want exactly the battery's", len(alerts))
	}
	alert := alerts[0]
	if alert.ResourceID != battery || alert.Type != "temperature" || alert.ID != battery+"::metric-threshold:temperature" ||
		alert.Value != hot || alert.Threshold != 45 || alert.ResourceName != "macbook-pve (Host Agent) (TB0T)" ||
		alert.Message != "TB0T temperature at 47.0°C" {
		t.Fatalf("battery alert = %+v", alert)
	}
	if alert.Metadata["sensorId"] != "applesmc-isa-0300/temp1" || alert.Metadata["sensorLabel"] != "TB0T" || alert.Metadata["resourceType"] != "agent-sensor" {
		t.Fatalf("battery alert metadata = %+v", alert.Metadata)
	}

	cool := 38.0
	m.CheckHost(nativeSensorHost(&cool, "ok"))
	if alerts := nativeSensorAlerts(m); len(alerts) != 0 {
		t.Fatalf("reading below the clear value kept %+v", alerts)
	}
}

func TestHostNativeSensorGapsHoldAndPolicyResolves(t *testing.T) {
	hot := 47.0
	for _, gap := range []string{"invalid", "no-value", "missing", "empty-inventory", "expired"} {
		for _, policy := range []string{"override-removed", "override-disabled", "override-zero", "config-save", "host-disabled", "agents-disabled", "host-removed"} {
			t.Run(gap+"/"+policy, func(t *testing.T) {
				m := nativeSensorManager(t)
				host := nativeSensorHost(&hot, "ok")
				battery := HostNativeSensorResourceID(host.ID, "applesmc-isa-0300/temp1")
				m.mu.Lock()
				m.config.Overrides[battery] = ThresholdConfig{Temperature: &HysteresisThreshold{Trigger: 45, Clear: 40}}
				m.mu.Unlock()
				m.CheckHost(host)
				if len(nativeSensorAlerts(m)) != 1 {
					t.Fatal("control must fire")
				}

				gapped := nativeSensorHost(&hot, "ok")
				switch gap {
				case "invalid":
					unattached := -127.0
					gapped = nativeSensorHost(&unattached, "invalid")
				case "no-value":
					gapped = nativeSensorHost(nil, "invalid")
				case "missing":
					gapped.Sensors.Native = gapped.Sensors.Native[1:]
				case "empty-inventory":
					gapped.Sensors.Native = nil
				}
				if gap == "expired" {
					m.HandleHostTelemetryExpired(host)
				} else {
					m.CheckHost(gapped)
				}
				if alerts := nativeSensorAlerts(m); len(alerts) != 1 || alerts[0].ResourceID != battery {
					t.Fatalf("a %s reading resolved or duplicated the held alert: %+v", gap, alerts)
				}

				switch policy {
				case "config-save":
					cfg := m.GetConfig()
					delete(cfg.Overrides, battery)
					m.UpdateConfig(cfg)
				case "host-removed":
					m.HandleHostRemoved(gapped)
				default:
					m.mu.Lock()
					switch policy {
					case "override-removed":
						delete(m.config.Overrides, battery)
					case "override-disabled":
						m.config.Overrides[battery] = ThresholdConfig{Disabled: true, Temperature: &HysteresisThreshold{Trigger: 45, Clear: 40}}
					case "override-zero":
						m.config.Overrides[battery] = ThresholdConfig{Temperature: &HysteresisThreshold{}}
					case "host-disabled":
						m.config.Overrides[host.ID] = ThresholdConfig{Disabled: true}
					case "agents-disabled":
						m.config.DisableAllAgents = true
					}
					m.mu.Unlock()
					m.CheckHost(gapped)
				}
				if alerts := nativeSensorAlerts(m); len(alerts) != 0 {
					t.Fatalf("%s left %+v", policy, alerts)
				}
			})
		}
	}
}

func TestHostNativeSensorConfigSaveKeepsAlertItsOverrideStillHolds(t *testing.T) {
	m := nativeSensorManager(t)
	hot := 47.0
	host := nativeSensorHost(&hot, "ok")
	battery := HostNativeSensorResourceID(host.ID, "applesmc-isa-0300/temp1")
	m.mu.Lock()
	m.config.Overrides[battery] = ThresholdConfig{Temperature: &HysteresisThreshold{Trigger: 45, Clear: 40}}
	m.mu.Unlock()
	m.CheckHost(host)
	// Any save re-judges active alerts. The host has no CPU temperature
	// threshold, so a resolver that read the host's would resolve this.
	cfg := m.GetConfig()
	cfg.AgentDefaults.Temperature = nil
	m.UpdateConfig(cfg)
	if alerts := nativeSensorAlerts(m); len(alerts) != 1 {
		t.Fatalf("config save resolved a sensor alert its own override still holds: %+v", alerts)
	}
	if got := m.GetConfig().Overrides[battery].Temperature; got == nil || got.Trigger != 45 {
		t.Fatalf("saved sensor override = %+v, want trigger 45 preserved", got)
	}
}

func TestHostNativeSensorPendingRunRestartsAfterPolicyOff(t *testing.T) {
	for _, policy := range []string{"override-removed", "host-disabled", "agents-disabled", "alerts-disabled", "alerts-cleared"} {
		t.Run(policy, func(t *testing.T) {
			m, elapsed := continuityManager(t, false)
			hot := 47.0
			host := nativeSensorHost(&hot, "ok")
			battery := HostNativeSensorResourceID(host.ID, "applesmc-isa-0300/temp1")
			setPolicy := func(on bool) {
				cfg := m.GetConfig()
				cfg.Overrides[battery] = ThresholdConfig{Temperature: &HysteresisThreshold{Trigger: 45, Clear: 40}}
				switch policy {
				case "override-removed":
					if !on {
						delete(cfg.Overrides, battery)
					}
				case "host-disabled":
					if on {
						delete(cfg.Overrides, host.ID)
					} else {
						cfg.Overrides[host.ID] = ThresholdConfig{Disabled: true}
					}
				case "agents-disabled":
					cfg.DisableAllAgents = !on
				case "alerts-disabled":
					cfg.Enabled = on
				}
				m.UpdateConfig(cfg)
			}
			setPolicy(true)
			m.CheckHost(host) // starts the 60s temperature grace
			elapsed.Store(int64(40 * time.Second))
			if policy == "alerts-cleared" {
				m.ClearActiveAlerts()
			} else {
				setPolicy(false)
				setPolicy(true)
			}
			elapsed.Store(int64(70 * time.Second))
			m.CheckHost(host)
			if alerts := nativeSensorAlerts(m); len(alerts) != 0 {
				t.Fatalf("re-enabled sensor reused grace from before %s: %+v", policy, alerts)
			}
			elapsed.Store(int64(131 * time.Second))
			m.CheckHost(host)
			if alerts := nativeSensorAlerts(m); len(alerts) != 1 {
				t.Fatalf("sustained reading after re-enable did not fire: %+v", alerts)
			}
		})
	}
}

func TestHostNativeSensorUsesSensorDelayPolicy(t *testing.T) {
	m, _ := continuityManager(t, false)
	hot := 47.0
	host := nativeSensorHost(&hot, "ok")
	battery := HostNativeSensorResourceID(host.ID, "applesmc-isa-0300/temp1")
	cfg := m.GetConfig()
	cfg.Overrides[battery] = ThresholdConfig{Temperature: &HysteresisThreshold{Trigger: 45, Clear: 40}}
	// The agent-wide temperature delay stays 60s; a sensor-specific delay of
	// zero must win for sensor alerts.
	cfg.MetricTimeThresholds = map[string]map[string]int{"all": {"temperature": 60}, "agent-sensor": {"temperature": 0}}
	m.UpdateConfig(cfg)
	m.CheckHost(host)
	if alerts := nativeSensorAlerts(m); len(alerts) != 1 {
		t.Fatalf("sensor-specific zero delay was ignored: %+v", alerts)
	}
}