Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

80.1 KB
Raw Blame
package monitoring

import (
	"context"
	"encoding/json"
	"io"
	"net/http"
	"net/http/httptest"
	"strings"
	"sync"
	"sync/atomic"
	"testing"
	"time"

	"github.com/rcourtman/pulse-go-rewrite/internal/ai/memory"
	"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
	"github.com/rcourtman/pulse-go-rewrite/internal/alerts/eventlog"
	"github.com/rcourtman/pulse-go-rewrite/internal/config"
	"github.com/rcourtman/pulse-go-rewrite/internal/models"
	"github.com/rcourtman/pulse-go-rewrite/internal/notifications"
	unifiedresources "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
	"github.com/rcourtman/pulse-go-rewrite/internal/websocket"
	"github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"
	"github.com/stretchr/testify/require"
)

func TestDockerAlertTimelineUsesCanonicalHistoryIdentity(t *testing.T) {
	dir := t.TempDir()
	store, err := unifiedresources.NewSQLiteResourceStore(dir, "default")
	require.NoError(t, err)
	t.Cleanup(func() { require.NoError(t, store.Close()) })
	manager := alerts.NewManagerWithDataDir(t.TempDir())
	t.Cleanup(manager.Stop)
	config := manager.GetConfig()
	config.Enabled = true
	config.ActivationState = alerts.ActivationPending
	config.TimeThresholds = map[string]int{}
	config.SuppressionWindow = 0
	manager.UpdateConfig(config)
	containerID := strings.Repeat("f", 64)
	host := models.DockerHost{ID: "history-host", Hostname: "history-host", LastSeen: time.Now(), Containers: []models.DockerContainer{
		{ID: containerID, Name: "worker", State: "running", Health: "unhealthy"},
		{ID: strings.Repeat("a", 64), Name: "worker", State: "running", Health: "healthy"},
	}}
	registry := unifiedresources.NewRegistry(store)
	registry.IngestSnapshot(models.StateSnapshot{DockerHosts: []models.DockerHost{host}})
	adapter := unifiedresources.NewMonitorAdapter(registry)
	monitor := &Monitor{alertManager: manager, resourceStore: adapter}
	manager.SubscribeLifecycleCallback(monitor.handleAlertLifecycleEvent)
	manager.CheckDockerHost(host)
	canonicalID := unifiedresources.SourceSpecificID(unifiedresources.ResourceTypeAppContainer, unifiedresources.SourceDocker, host.ID+"/container/"+containerID)
	filters := unifiedresources.ResourceChangeFilters{Kinds: []unifiedresources.ChangeKind{unifiedresources.ChangeAlertFired, unifiedresources.ChangeAlertResolved}}
	changes, err := store.GetRecentChangesFiltered(canonicalID, time.Time{}, 10, filters)
	require.NoError(t, err)
	require.Len(t, changes, 1)
	require.Equal(t, unifiedresources.ChangeAlertFired, changes[0].Kind)
	require.Equal(t, canonicalID, changes[0].ResourceID)
	var fired alerts.Alert
	for _, alert := range manager.GetActiveAlerts() {
		if alert.Type == "docker-container-health" {
			fired = alert
		}
	}
	require.NotEmpty(t, fired.ID)
	// Recovery is emitted after the monitored container has left the registry.
	// Its exact retained source binding must still select the original resource.
	monitor.resourceStore = unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(store))
	host.Containers[0].Health = "healthy"
	manager.CheckDockerHost(host)
	changes, err = store.GetRecentChangesFiltered(canonicalID, time.Time{}, 10, filters)
	require.NoError(t, err)
	require.Len(t, changes, 2)
	require.Equal(t, unifiedresources.ChangeAlertResolved, changes[0].Kind)
	require.Equal(t, canonicalID, changes[0].ResourceID)
	monitor.recordAlertTimelineChange(&fired, unifiedresources.ChangeAlertFired, fired.StartTime, "")
	monitor.recordAlertTimelineChange(&fired, unifiedresources.ChangeAlertResolved, *changes[0].OccurredAt, "")
	again, err := store.GetRecentChangesFiltered(canonicalID, time.Time{}, 10, filters)
	require.NoError(t, err)
	require.Equal(t, changes, again)
	controlID := unifiedresources.SourceSpecificID(unifiedresources.ResourceTypeAppContainer, unifiedresources.SourceDocker, host.ID+"/container/"+host.Containers[1].ID)
	control, err := store.GetRecentChangesFiltered(controlID, time.Time{}, 10, filters)
	require.NoError(t, err)
	require.Empty(t, control)
	require.NoError(t, store.Close())
	restarted, err := unifiedresources.NewSQLiteResourceStore(dir, "default")
	require.NoError(t, err)
	defer restarted.Close()
	afterRestart, err := restarted.GetRecentChangesFiltered(canonicalID, time.Time{}, 10, filters)
	require.NoError(t, err)
	require.Equal(t, changes, afterRestart)
	encoded, err := json.Marshal(afterRestart)
	require.NoError(t, err)
	t.Logf("DOCKER_HISTORY_LIFECYCLE %s", encoded)
}

// Proxmox node and guest alerts carry source-native IDs. Their lifecycle must
// reach the canonical resource history that facets, the drawer and the
// assistant read, including events emitted after the resource left inventory.
func TestProxmoxAlertTimelineUsesCanonicalHistoryIdentity(t *testing.T) {
	store, err := unifiedresources.NewSQLiteResourceStore(t.TempDir(), "default")
	require.NoError(t, err)
	t.Cleanup(func() { require.NoError(t, store.Close()) })
	manager := alerts.NewManagerWithDataDir(t.TempDir())
	t.Cleanup(manager.Stop)
	config := manager.GetConfig()
	config.Enabled = true
	config.ActivationState = alerts.ActivationPending
	config.TimeThresholds = map[string]int{"node": 0, "guest": 0}
	config.SuppressionWindow = 0
	manager.UpdateConfig(config)
	now := time.Now()
	node := models.Node{ID: "lab-pve1", Name: "pve1", Instance: "lab", Host: "https://pve1.lab:8006", Status: "online", CPU: 0.99, LastSeen: now}
	vm := models.VM{ID: "lab:pve1:101", VMID: 101, Name: "web", Node: "pve1", Instance: "lab", Status: "stopped", Type: "qemu", LastSeen: now}
	registry := unifiedresources.NewRegistry(store)
	registry.IngestSnapshot(models.StateSnapshot{Nodes: []models.Node{node}, VMs: []models.VM{vm}})
	monitor := &Monitor{alertManager: manager, resourceStore: unifiedresources.NewMonitorAdapter(registry)}
	manager.SubscribeLifecycleCallback(monitor.handleAlertLifecycleEvent)
	manager.CheckNode(node)
	manager.CheckGuest(vm, vm.Instance)
	manager.CheckGuest(vm, vm.Instance)

	canonical := map[string]string{}
	for _, resource := range registry.List() {
		canonical[resource.Name] = resource.ID
	}
	require.NotEqual(t, node.ID, canonical["pve1"])
	require.NotEqual(t, vm.ID, canonical["web"])
	filters := unifiedresources.ResourceChangeFilters{Kinds: []unifiedresources.ChangeKind{unifiedresources.ChangeAlertFired, unifiedresources.ChangeAlertResolved}}
	for name, sourceID := range map[string]string{"pve1": node.ID, "web": vm.ID} {
		changes, err := store.GetRecentChangesFiltered(canonical[name], time.Time{}, 10, filters)
		require.NoError(t, err)
		require.Len(t, changes, 1, name)
		require.Equal(t, unifiedresources.ChangeAlertFired, changes[0].Kind)
		require.Equal(t, canonical[name], changes[0].ResourceID)
		kinds, err := store.CountRecentChangesByKind(canonical[name], time.Time{})
		require.NoError(t, err)
		require.Equal(t, 1, kinds[unifiedresources.ChangeAlertFired], name)
		legacy, err := store.GetRecentChangesFiltered(sourceID, time.Time{}, 10, filters)
		require.NoError(t, err)
		require.Equal(t, changes, legacy, "the source reference reads the same history")
	}

	// The node recovers after it left inventory; its retained binding holds.
	monitor.resourceStore = unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(store))
	node.CPU = 0.05
	manager.CheckNode(node)
	changes, err := store.GetRecentChangesFiltered(canonical["pve1"], time.Time{}, 10, filters)
	require.NoError(t, err)
	require.Len(t, changes, 2)
	require.Equal(t, unifiedresources.ChangeAlertResolved, changes[0].Kind)
	require.Equal(t, canonical["pve1"], changes[0].ResourceID)
}

// Docker host, Swarm service and sub-resource alerts (a ZFS pool and device,
// a host filesystem) carry references the registry has no resource for. Their
// lifecycle must reach the owner's canonical history, so the producers' ref
// shapes are pinned here against the real alert manager. A container or
// service reported without an ID alerts under its own name reference, so it
// joins neither its host's history nor that of the service whose ID it is
// named like.
func TestOwnerAlertTimelinesUseCanonicalHistoryIdentity(t *testing.T) {
	store, err := unifiedresources.NewSQLiteResourceStore(t.TempDir(), "default")
	require.NoError(t, err)
	t.Cleanup(func() { require.NoError(t, store.Close()) })
	manager := alerts.NewManagerWithDataDir(t.TempDir())
	t.Cleanup(manager.Stop)
	config := manager.GetConfig()
	config.Enabled = true
	config.ActivationState = alerts.ActivationPending
	config.TimeThresholds = map[string]int{"all": 0}
	config.MetricTimeThresholds = map[string]map[string]int{"all": {"disk": 0}}
	config.SuppressionWindow = 0
	manager.UpdateConfig(config)
	now := time.Now()
	dockerHost := models.DockerHost{ID: "dh-7f3a", Hostname: "tower", DisplayName: "tower", Status: "online", LastSeen: now,
		Swarm:      &models.DockerSwarmInfo{NodeID: "node-a", NodeRole: "manager", LocalState: "active", ControlAvailable: true, ClusterID: "swarm-1"},
		Containers: []models.DockerContainer{{Name: "sidecar", State: "running", Health: "unhealthy"}},
		Services: []models.DockerService{{ID: "x7k2m9q4w1e8r5t3y6u0i2o4p", Name: "web", DesiredTasks: 2, RunningTasks: 0},
			{Name: "X7K2M9Q4W1E8R5T3Y6U0I2O4P", DesiredTasks: 2, RunningTasks: 0}}}
	storage := models.Storage{ID: "lab-pve1-local-zfs", Name: "local-zfs", Node: "pve1", Instance: "lab", Type: "zfspool", Status: "available", Total: 100, Used: 10, Usage: 10, Enabled: true, Active: true, LastSeen: now,
		ZFSPool: &models.ZFSPool{Name: "rpool", State: "DEGRADED", Status: "Degraded", Devices: []models.ZFSDevice{{Name: "sda2", Type: "disk", State: "FAULTED"}}}}
	agentHost := models.Host{ID: "host-nas", Hostname: "nas", DisplayName: "nas", MachineID: "fedcba9876543210", Status: "online", LastSeen: now,
		Disks: []models.Disk{{Mountpoint: "/var", Device: "/dev/sdb1", Type: "ext4", Total: 100, Used: 99, Free: 1, Usage: 99}}}
	registry := unifiedresources.NewRegistry(store)
	registry.IngestSnapshot(models.StateSnapshot{DockerHosts: []models.DockerHost{dockerHost}, Storage: []models.Storage{storage}, Hosts: []models.Host{agentHost}})
	monitor := &Monitor{alertManager: manager, resourceStore: unifiedresources.NewMonitorAdapter(registry)}
	manager.SubscribeLifecycleCallback(monitor.handleAlertLifecycleEvent)
	manager.CheckDockerHost(dockerHost)
	manager.CheckStorage(storage)
	manager.CheckHost(agentHost)
	for range 3 {
		manager.HandleDockerHostOffline(dockerHost)
	}

	owner := map[string]string{}
	for _, resource := range registry.List() {
		owner[string(resource.Type)+"/"+resource.Name] = resource.ID
	}
	filters := unifiedresources.ResourceChangeFilters{Kinds: []unifiedresources.ChangeKind{unifiedresources.ChangeAlertFired}}
	want := map[string]string{
		"docker-host-offline":   owner["agent/tower"],
		"docker-service-health": owner["docker-service/web"],
		"zfs-pool-state":        owner["storage/local-zfs"],
		"zfs-device":            owner["storage/local-zfs"],
		"disk":                  owner["agent/nas"],
	}
	for alertType, canonicalID := range want {
		require.NotEmpty(t, canonicalID, alertType)
		changes, err := store.GetRecentChangesFiltered(canonicalID, time.Time{}, 10, filters)
		require.NoError(t, err)
		found := false
		for _, change := range changes {
			require.Equal(t, canonicalID, change.ResourceID)
			found = found || change.Metadata["alert_type"] == alertType
		}
		require.True(t, found, "%s alert missing from %s history: %+v", alertType, canonicalID, changes)
	}
	alertTypes := func(ref string) []any {
		changes, err := store.GetRecentChangesFiltered(ref, time.Time{}, 10, filters)
		require.NoError(t, err)
		types := make([]any, 0, len(changes))
		for _, change := range changes {
			types = append(types, change.Metadata["alert_type"])
		}
		return types
	}
	require.ElementsMatch(t, []any{"docker-host-offline"}, alertTypes(owner["agent/tower"]))
	require.ElementsMatch(t, []any{"docker-service-health"}, alertTypes(owner["docker-service/web"]))
	require.ElementsMatch(t, []any{"docker-container-health"}, alertTypes("docker:dh-7f3a/name:sidecar"))
	require.ElementsMatch(t, []any{"docker-service-health"}, alertTypes("docker:dh-7f3a/service/name:x7k2m9q4w1e8r5t3y6u0i2o4p"))
	for _, name := range []string{"app-container/sidecar", "docker-service/X7K2M9Q4W1E8R5T3Y6U0I2O4P"} {
		require.NotEmpty(t, owner[name], name)
		require.Empty(t, alertTypes(owner[name]), "a name binds nothing: %s", name)
	}
}

func TestMonitor_HandleAlertFired_Extra(t *testing.T) {
	// 1. Alert is nil
	m1 := &Monitor{}
	m1.handleAlertFired(nil) // Should return safely

	// 2. Alert is not nil, with Hub and NotificationMgr
	hub := websocket.NewHub(nil)
	notifMgr := notifications.NewNotificationManager("dummy")

	// mock incidentStore - but it is an interface or struct?
	// In monitor.go: func (m *Monitor) GetIncidentStore() *incidents.Store
	// It's a pointer to struct, so hard to mock unless we set it to nil or real store.
	// We can set it to nil for this test to avoid disk I/O.

	m2 := &Monitor{
		wsHub:           hub,
		notificationMgr: notifMgr,
		incidentStore:   nil,
	}

	alert := &alerts.Alert{
		ID:    "test-alert",
		Level: alerts.AlertLevelWarning,
	}

	var pushed atomic.Bool
	m2.SetAlertPushCallback(func(got *alerts.Alert) {
		if got == alert {
			pushed.Store(true)
		}
	})
	m2.handleAlertFired(alert)
	if !pushed.Load() {
		t.Fatal("alert push callback was not invoked")
	}
	// We are just verifying it doesn't crash and calls methods.
	// Hub doesn't expose way to check broadcasts easily without client.
	// NotificationMgr might spin up goroutine.
}

func TestMonitor_HandleAlertFired_RecoversFromPushCallbackPanic(t *testing.T) {
	m := &Monitor{}
	m.SetAlertPushCallback(func(*alerts.Alert) {
		panic("push transport failure")
	})

	// A transport adapter must not be able to abort the canonical alert
	// lifecycle or its remaining persistence callbacks.
	m.handleAlertFired(&alerts.Alert{ID: "alert-push-panic"})
}

func TestDeliveryCallbackDoesNotDuplicateCanonicalAITrigger(t *testing.T) {
	called := make(chan struct{}, 1)
	monitor := &Monitor{
		alertTriggeredAICallback: func(*alerts.Alert) { called <- struct{}{} },
	}

	monitor.handleAlertFired(&alerts.Alert{ID: "single-ai-trigger"})
	select {
	case <-called:
		t.Fatal("delivery callback invoked AI analysis; the manager-owned unconditional AI callback is the sole trigger")
	case <-time.After(100 * time.Millisecond):
	}
}

func TestMonitor_HandleAlertLifecycle_WritesCanonicalChanges(t *testing.T) {
	store := unifiedresources.NewMemoryStore()
	m := &Monitor{
		resourceStore: unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(store)),
	}

	startedAt := time.Date(2026, 3, 20, 9, 0, 0, 0, time.UTC)
	ackAt := startedAt.Add(2 * time.Minute)
	alert := &alerts.Alert{
		ID:         "alert-canonical-1",
		Type:       "cpu",
		Level:      alerts.AlertLevelCritical,
		ResourceID: "vm-1",
		Message:    "CPU threshold exceeded",
		Value:      93.4,
		Threshold:  80,
		StartTime:  startedAt,
		AckTime:    &ackAt,
		Metadata: map[string]interface{}{
			"incidentCategory":   "health",
			"vmwareConnectionId": "vc-1",
		},
	}

	m.handleAlertLifecycleEvent(alerts.LifecycleEvent{Type: eventlog.TypeFired, OccurredAt: startedAt, Alert: alert})
	m.handleAlertLifecycleEvent(alerts.LifecycleEvent{
		Type:       eventlog.TypeAcknowledged,
		OccurredAt: ackAt,
		Alert:      alert,
		Details:    map[string]string{"user": "admin"},
	})
	m.handleAlertLifecycleEvent(alerts.LifecycleEvent{
		Type:       eventlog.TypeSnoozed,
		OccurredAt: ackAt.Add(2 * time.Minute),
		Alert:      alert,
		Details: map[string]string{
			"actor": "admin",
			"until": ackAt.Add(2 * time.Hour).Format(time.RFC3339),
		},
	})
	m.handleAlertLifecycleEvent(alerts.LifecycleEvent{
		Type:       eventlog.TypeUnsnoozed,
		OccurredAt: ackAt.Add(3 * time.Minute),
		Alert:      alert,
		Details:    map[string]string{"actor": "admin"},
	})
	m.handleAlertLifecycleEvent(alerts.LifecycleEvent{
		Type:       eventlog.TypeUnacknowledged,
		OccurredAt: ackAt.Add(time.Minute),
		Alert:      alert,
		Details:    map[string]string{"user": "admin"},
	})

	changes, err := store.GetRecentChanges("vm-1", time.Time{}, 10)
	if err != nil {
		t.Fatalf("GetRecentChanges: %v", err)
	}
	if len(changes) != 5 {
		t.Fatalf("expected 5 canonical changes, got %d", len(changes))
	}
	// Arrival order can differ from observation order during lifecycle replay.
	wantKinds := []unifiedresources.ChangeKind{
		unifiedresources.ChangeAlertUnsnoozed,
		unifiedresources.ChangeAlertSnoozed,
		unifiedresources.ChangeAlertUnacknowledged,
		unifiedresources.ChangeAlertAcknowledged,
		unifiedresources.ChangeAlertFired,
	}
	for idx, want := range wantKinds {
		if changes[idx].Kind != want {
			t.Fatalf("changes[%d].Kind = %q, want %q", idx, changes[idx].Kind, want)
		}
	}
	if got := changes[4].Metadata["alert_identifier"]; got != "alert-canonical-1" {
		t.Fatalf("alert_identifier = %#v, want alert-canonical-1", got)
	}
	if got := changes[4].Metadata["incidentCategory"]; got != "health" {
		t.Fatalf("incidentCategory = %#v, want health", got)
	}
	if got := changes[2].Metadata["vmwareConnectionId"]; got != "vc-1" {
		t.Fatalf("vmwareConnectionId = %#v, want vc-1", got)
	}
}

// A node alert handed to its Pulse agent closes without recovering. The
// resource history and the alert's incident timeline (the Alerts history row
// expansion) must say where it went, not "Alert resolved: Memory usage at 95%".
func TestMonitor_HandleAlertLifecycle_HandoverCloseIsNotARecovery(t *testing.T) {
	resourceStore := unifiedresources.NewMemoryStore()
	incidentStore := memory.NewIncidentStore(memory.IncidentStoreConfig{})
	m := &Monitor{
		incidentStore: incidentStore,
		resourceStore: unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(resourceStore)),
	}
	incidentStore.SetResourceTimelineStore(m.resourceStore.(memory.IncidentTimelineStore))

	startedAt := time.Date(2026, 10, 6, 9, 0, 0, 0, time.UTC)
	movedAt := startedAt.Add(2 * time.Hour)
	alert := &alerts.Alert{
		ID:         "pve1-memory",
		Type:       "memory",
		Level:      alerts.AlertLevelWarning,
		ResourceID: "pve1",
		Message:    "Memory usage at 95%",
		Value:      95,
		Threshold:  85,
		StartTime:  startedAt,
	}
	m.handleAlertLifecycleEvent(alerts.LifecycleEvent{Type: eventlog.TypeFired, OccurredAt: startedAt, Alert: alert})
	closed := alert.Clone()
	closed.Resolution = &alerts.AlertResolution{
		Reason:              alerts.AlertResolutionMovedToAgent,
		SuccessorResourceID: "agent-pve1",
		SuccessorName:       "pve1 (Host Agent)",
	}
	m.handleAlertLifecycleEvent(alerts.LifecycleEvent{Type: eventlog.TypeResolved, OccurredAt: movedAt, Alert: closed})

	summary := "Alert moved to pve1 (Host Agent). This is not a recovery: check the agent for the current reading."
	changes, err := resourceStore.GetRecentChanges("pve1", time.Time{}, 10)
	require.NoError(t, err)
	require.Len(t, changes, 2)
	require.Equal(t, unifiedresources.ChangeAlertResolved, changes[0].Kind)
	require.Equal(t, summary, changes[0].Reason)
	require.Equal(t, "moved_to_agent", changes[0].Metadata[unifiedresources.MetadataAlertResolution])
	require.Equal(t, unifiedresources.ChangeAlertFired, changes[1].Kind)
	require.NotContains(t, changes[1].Metadata, unifiedresources.MetadataAlertResolution)

	timeline := incidentStore.GetTimelineByAlertAt(alert.ID, startedAt)
	require.NotNil(t, timeline)
	require.Len(t, timeline.Events, 2)
	require.Equal(t, memory.IncidentEventAlertResolved, timeline.Events[1].Type)
	require.Equal(t, summary, timeline.Events[1].Summary)

	// An ordinary recovery keeps its existing wording.
	recovered := alert.Clone()
	recovered.ID = "pve1-cpu"
	recovered.Type = "cpu"
	recovered.Message = "CPU usage at 90%"
	m.handleAlertLifecycleEvent(alerts.LifecycleEvent{Type: eventlog.TypeFired, OccurredAt: startedAt, Alert: recovered})
	m.handleAlertLifecycleEvent(alerts.LifecycleEvent{Type: eventlog.TypeResolved, OccurredAt: movedAt.Add(time.Minute), Alert: recovered})
	recoveredTimeline := incidentStore.GetTimelineByAlertAt(recovered.ID, startedAt)
	require.NotNil(t, recoveredTimeline)
	require.Len(t, recoveredTimeline.Events, 2)
	require.Equal(t, "Alert resolved", recoveredTimeline.Events[1].Summary)
}

func TestPausedDeliveryStillBuildsTimelineThroughRealAlertLifecycle(t *testing.T) {
	manager := alerts.NewManagerWithDataDir(t.TempDir())
	t.Cleanup(manager.Stop)
	config := manager.GetConfig()
	config.Enabled = true
	config.ActivationState = alerts.ActivationPending
	config.TimeThresholds = map[string]int{}
	config.SuppressionWindow = 0
	manager.UpdateConfig(config)

	resourceStore := unifiedresources.NewMemoryStore()
	incidentStore := memory.NewIncidentStore(memory.IncidentStoreConfig{})
	monitor := &Monitor{
		alertManager:  manager,
		incidentStore: incidentStore,
		resourceStore: unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(resourceStore)),
	}
	incidentStore.SetResourceTimelineStore(monitor.resourceStore.(memory.IncidentTimelineStore))
	manager.SubscribeLifecycleCallback(monitor.handleAlertLifecycleEvent)
	delivered := make(chan *alerts.Alert, 1)
	manager.SetAlertCallback(func(alert *alerts.Alert) { delivered <- alert })

	vm := models.VM{ID: "paused-vm", Name: "Paused VM", Node: "node-1", Instance: "pve-1", Status: "stopped"}
	manager.CheckGuest(vm, vm.Instance)
	manager.CheckGuest(vm, vm.Instance)

	active := manager.GetActiveAlerts()
	if len(active) != 1 {
		t.Fatalf("active alerts = %d, want 1", len(active))
	}
	timeline := incidentStore.GetTimelineByAlertAt(active[0].ID, active[0].StartTime)
	if timeline == nil || len(timeline.Events) == 0 || timeline.Events[0].Type != memory.IncidentEventAlertFired {
		t.Fatalf("paused-delivery lifecycle did not produce an incident timeline: %#v", timeline)
	}
	select {
	case alert := <-delivered:
		t.Fatalf("pending-review alert reached delivery callback: %s", alert.ID)
	default:
	}
}

func TestActiveTimelineReconciliationIsIdempotent(t *testing.T) {
	manager := alerts.NewManagerWithDataDir(t.TempDir())
	t.Cleanup(manager.Stop)
	config := manager.GetConfig()
	config.Enabled = true
	config.ActivationState = alerts.ActivationPending
	config.TimeThresholds = map[string]int{}
	manager.UpdateConfig(config)

	vm := models.VM{ID: "restored-vm", Name: "Restored VM", Node: "node-1", Instance: "pve-1", Status: "stopped"}
	manager.CheckGuest(vm, vm.Instance)
	manager.CheckGuest(vm, vm.Instance)

	resourceStore := unifiedresources.NewMemoryStore()
	incidentStore := memory.NewIncidentStore(memory.IncidentStoreConfig{})
	monitor := &Monitor{
		alertManager:  manager,
		incidentStore: incidentStore,
		resourceStore: unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(resourceStore)),
	}
	incidentStore.SetResourceTimelineStore(monitor.resourceStore.(memory.IncidentTimelineStore))
	monitor.reconcileActiveAlertTimelines(context.Background())
	monitor.reconcileActiveAlertTimelines(context.Background())

	active := manager.GetActiveAlerts()
	if len(active) != 1 {
		t.Fatalf("active alerts = %d, want 1", len(active))
	}
	timeline := incidentStore.GetTimelineByAlertAt(active[0].ID, active[0].StartTime)
	if timeline == nil || len(timeline.Events) != 1 || timeline.Events[0].Type != memory.IncidentEventAlertFired {
		t.Fatalf("reconciled timeline = %#v, want one fired event", timeline)
	}
	changes, err := resourceStore.GetRecentChanges(active[0].ResourceID, time.Time{}, 10)
	if err != nil {
		t.Fatalf("GetRecentChanges: %v", err)
	}
	if len(changes) != 1 {
		t.Fatalf("reconciliation wrote %d canonical changes, want 1", len(changes))
	}
}

func TestLifecycleReplayRepairsResolvedIncidentTimeline(t *testing.T) {
	manager := alerts.NewManagerWithDataDir(t.TempDir(), alerts.WithoutPersistedAlertRestore())
	t.Cleanup(manager.Stop)
	manager.EnableEventLog()
	config := manager.GetConfig()
	config.Enabled = true
	config.ActivationState = alerts.ActivationPending
	config.TimeThresholds = map[string]int{}
	config.SuppressionWindow = 0
	manager.UpdateConfig(config)

	vm := models.VM{ID: "historical-vm", Name: "Historical VM", Node: "node-1", Instance: "pve-1", Status: "stopped"}
	manager.CheckGuest(vm, vm.Instance)
	manager.CheckGuest(vm, vm.Instance)
	active := manager.GetActiveAlerts()
	if len(active) != 1 {
		t.Fatalf("active alerts = %d, want 1", len(active))
	}
	alertID, startedAt := active[0].ID, active[0].StartTime

	vm.Status = "running"
	manager.CheckGuest(vm, vm.Instance)
	if active := manager.GetActiveAlerts(); len(active) != 0 {
		t.Fatalf("active alerts after recovery = %d, want 0", len(active))
	}

	resourceStore := unifiedresources.NewMemoryStore()
	incidentStore := memory.NewIncidentStore(memory.IncidentStoreConfig{})
	monitor := &Monitor{
		alertManager:  manager,
		incidentStore: incidentStore,
		resourceStore: unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(resourceStore)),
	}
	incidentStore.SetResourceTimelineStore(monitor.resourceStore.(memory.IncidentTimelineStore))
	monitor.replayAlertLifecycleProjections(context.Background())
	monitor.replayAlertLifecycleProjections(context.Background())

	timeline := incidentStore.GetTimelineByAlertAt(alertID, startedAt)
	if timeline == nil || len(timeline.Events) != 2 {
		t.Fatalf("replayed timeline = %#v, want fired and resolved events", timeline)
	}
	if timeline.Events[0].Type != memory.IncidentEventAlertFired || timeline.Events[1].Type != memory.IncidentEventAlertResolved {
		t.Fatalf("replayed event types = %#v, want fired then resolved", timeline.Events)
	}
	changes, err := resourceStore.GetRecentChanges(vm.ID, time.Time{}, 10)
	if err != nil {
		t.Fatalf("GetRecentChanges: %v", err)
	}
	if len(changes) != 2 {
		t.Fatalf("replay wrote %d canonical changes, want 2", len(changes))
	}
}

func TestLifecycleReplayMaterializesImportedHistoryTimeline(t *testing.T) {
	manager := alerts.NewManagerWithDataDir(t.TempDir(), alerts.WithoutPersistedAlertRestore())
	t.Cleanup(manager.Stop)
	store, err := eventlog.Open(t.TempDir())
	if err != nil {
		t.Fatalf("open event log: %v", err)
	}
	manager.SetEventLog(store)

	startedAt := time.Now().UTC().Add(-30 * time.Minute).Truncate(time.Second)
	ackAt := startedAt.Add(5 * time.Minute)
	resolvedAt := startedAt.Add(20 * time.Minute)
	snapshot := alerts.Alert{
		ID:           "imported-alert-1",
		Type:         "cpu",
		Level:        alerts.AlertLevelWarning,
		ResourceID:   "imported-resource-1",
		ResourceName: "Imported VM",
		StartTime:    startedAt,
		LastSeen:     resolvedAt,
		Acknowledged: true,
		AckTime:      &ackAt,
		AckUser:      "operator",
	}
	payload, err := json.Marshal(snapshot)
	if err != nil {
		t.Fatalf("marshal imported snapshot: %v", err)
	}
	if err := store.ImportEvents([]eventlog.Event{{
		OccurredAt: resolvedAt,
		Type:       eventlog.TypeHistoryImported,
		AlertID:    snapshot.ID,
		ResourceID: snapshot.ResourceID,
		Snapshot:   payload,
	}}); err != nil {
		t.Fatalf("import history event: %v", err)
	}

	incidentStore := memory.NewIncidentStore(memory.IncidentStoreConfig{})
	monitor := &Monitor{alertManager: manager, incidentStore: incidentStore}
	resourceStore := unifiedresources.NewMemoryStore()
	adapter := unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(resourceStore))
	// Hold replay at its serialization boundary. Router construction attaches
	// this store, so attachment must return even while history repair cannot
	// make progress. Eventual timeline assertions alone miss a synchronous
	// replay regression that stalls startup on an upgrade backlog.
	monitor.alertProjectionReplayMu.Lock()
	attached := make(chan struct{})
	go func() {
		monitor.SetResourceStore(adapter)
		monitor.SetResourceStore(adapter)
		close(attached)
	}()
	select {
	case <-attached:
		monitor.alertProjectionReplayMu.Unlock()
	case <-time.After(2 * time.Second):
		// Release the probe before failing, including for a synchronous-replay
		// negative control, so no goroutine retains the test's stores.
		monitor.alertProjectionReplayMu.Unlock()
		<-attached
		monitor.alertProjectionWG.Wait()
		t.Fatal("resource-store attachment waited for lifecycle replay")
	}
	monitor.alertProjectionWG.Wait()

	timeline := incidentStore.GetTimelineByAlertAt(snapshot.ID, snapshot.StartTime)
	if timeline == nil || timeline.Status != memory.IncidentStatusResolved || !timeline.Acknowledged {
		t.Fatalf("imported timeline state = %#v", timeline)
	}
	if len(timeline.Events) != 3 {
		t.Fatalf("imported timeline events = %d, want three idempotent snapshot events", len(timeline.Events))
	}
	changes, err := resourceStore.GetRecentChanges(snapshot.ResourceID, time.Time{}, 10)
	if err != nil {
		t.Fatalf("read imported canonical changes: %v", err)
	}
	if len(changes) != 3 {
		t.Fatalf("imported canonical changes = %d, want fired, acknowledged, and resolved", len(changes))
	}
}

func TestLifecycleReplayWatermarkBoundsSubsequentPasses(t *testing.T) {
	manager := alerts.NewManagerWithDataDir(t.TempDir(), alerts.WithoutPersistedAlertRestore())
	t.Cleanup(manager.Stop)
	manager.EnableEventLog()
	config := manager.GetConfig()
	config.Enabled = true
	config.ActivationState = alerts.ActivationPending
	config.TimeThresholds = map[string]int{}
	config.SuppressionWindow = 0
	manager.UpdateConfig(config)

	vm := models.VM{ID: "watermark-vm", Name: "Watermark VM", Node: "node-1", Instance: "pve-1", Status: "stopped"}
	manager.CheckGuest(vm, vm.Instance)
	manager.CheckGuest(vm, vm.Instance)
	vm.Status = "running"
	manager.CheckGuest(vm, vm.Instance)

	// A pass without the canonical resource store repairs incidents but must
	// not advance the durable watermark, or resource-timeline projections for
	// those events would never materialize.
	partial := &Monitor{alertManager: manager, incidentStore: memory.NewIncidentStore(memory.IncidentStoreConfig{})}
	partial.replayAlertLifecycleProjections(context.Background())
	if got := manager.LifecycleProjectionWatermark(alertLifecycleProjectionConsumer); got != 0 {
		t.Fatalf("watermark after partial-surface replay = %d, want 0", got)
	}

	resourceStore := unifiedresources.NewMemoryStore()
	monitor := &Monitor{
		alertManager:  manager,
		incidentStore: memory.NewIncidentStore(memory.IncidentStoreConfig{}),
		resourceStore: unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(resourceStore)),
	}
	monitor.replayAlertLifecycleProjections(context.Background())
	watermark := manager.LifecycleProjectionWatermark(alertLifecycleProjectionConsumer)
	if watermark == 0 {
		t.Fatal("watermark did not advance after full-surface replay")
	}

	// A later pass walks only events beyond the watermark, so fresh projection
	// stores stay empty: nothing is left to replay.
	freshResources := unifiedresources.NewMemoryStore()
	rerun := &Monitor{
		alertManager:  manager,
		incidentStore: memory.NewIncidentStore(memory.IncidentStoreConfig{}),
		resourceStore: unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(freshResources)),
	}
	rerun.replayAlertLifecycleProjections(context.Background())
	if changes, err := freshResources.GetRecentChanges(vm.ID, time.Time{}, 10); err != nil || len(changes) != 0 {
		t.Fatalf("bounded pass wrote %d changes (err %v), want none", len(changes), err)
	}

	// Resetting the watermark forces a full repair replay for rebuilt stores.
	manager.StoreLifecycleProjectionWatermark(alertLifecycleProjectionConsumer, 0)
	rerun.replayAlertLifecycleProjections(context.Background())
	changes, err := freshResources.GetRecentChanges(vm.ID, time.Time{}, 10)
	if err != nil {
		t.Fatalf("GetRecentChanges after watermark reset: %v", err)
	}
	if len(changes) != 2 {
		t.Fatalf("reset replay wrote %d canonical changes, want fired and resolved", len(changes))
	}
	if got := manager.LifecycleProjectionWatermark(alertLifecycleProjectionConsumer); got != watermark {
		t.Fatalf("watermark after reset replay = %d, want %d", got, watermark)
	}
}

func TestSystemAlertTimelineUsesCanonicalPulseResource(t *testing.T) {
	resourceStore := unifiedresources.NewMemoryStore()
	incidentStore := memory.NewIncidentStore(memory.IncidentStoreConfig{})
	monitor := &Monitor{
		incidentStore: incidentStore,
		resourceStore: unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(resourceStore)),
	}
	incidentStore.SetResourceTimelineStore(monitor.resourceStore.(memory.IncidentTimelineStore))
	alert := &alerts.Alert{
		ID:           alerts.SystemAlertID("event-store-health"),
		Type:         "event-store-health",
		Level:        alerts.AlertLevelCritical,
		ResourceName: alerts.SystemAlertResourceName,
		Message:      "Alert history storage is unavailable",
		StartTime:    time.Now().UTC(),
	}
	monitor.handleAlertLifecycleEvent(alerts.LifecycleEvent{Type: eventlog.TypeFired, OccurredAt: alert.StartTime, Alert: alert})
	timeline := incidentStore.GetTimelineByAlertAt(alert.ID, alert.StartTime)
	if timeline == nil || timeline.ResourceID != "pulse-system" || len(timeline.Events) != 1 {
		t.Fatalf("system alert timeline = %#v", timeline)
	}
}

func TestMonitor_HandleAlertResolved_Detailed_Extra(t *testing.T) {
	// 1. With Hub and NotificationMgr and Resolve Notify ON
	hub := websocket.NewHub(nil)
	notifMgr := notifications.NewNotificationManager("dummy")

	// Enable resolve notifications
	// Notifications config needs to be updated?
	// notificationMgr.GetNotifyOnResolve() reads config.
	// But NotificationManager struct doesn't export Config update easily without SetConfig?
	// The constructor initializes defaults.

	m := &Monitor{
		wsHub:           hub,
		notificationMgr: notifMgr,
		alertManager:    alerts.NewManager(),
	}

	// This should run safely
	m.handleAlertResolved("alert-id")
}

func TestMonitor_HandleAlertResolved_QuietHoursSuppressesRecovery(t *testing.T) {
	// Verify that resolved notifications are suppressed during quiet hours (#1068).
	// We seed a resolved alert in the manager, configure quiet hours to suppress all
	// (00:00–23:59 every day), then verify SendResolvedAlert is NOT called.

	hub := websocket.NewHub(nil)
	notifMgr := notifications.NewNotificationManager("dummy")
	notifMgr.SetNotifyOnResolve(true)

	mgr := alerts.NewManager()
	// Configure quiet hours to be always active, alerts enabled, no time delay
	mgr.UpdateConfig(alerts.AlertConfig{
		Enabled: true,
		TimeThresholds: map[string]int{
			"guest": 0, "node": 0, "storage": 0, "pbs": 0, "host": 0,
		},
		GuestDefaults: alerts.ThresholdConfig{
			CPU:    &alerts.HysteresisThreshold{Trigger: 80, Clear: 75},
			Memory: &alerts.HysteresisThreshold{Trigger: 85, Clear: 80},
		},
		Schedule: alerts.ScheduleConfig{
			NotifyOnResolve: true,
			QuietHours: alerts.QuietHours{
				Enabled:  true,
				Start:    "00:00",
				End:      "23:59",
				Timezone: "UTC",
				Days: map[string]bool{
					"monday": true, "tuesday": true, "wednesday": true,
					"thursday": true, "friday": true, "saturday": true, "sunday": true,
				},
				Suppress: alerts.QuietHoursSuppression{
					Performance: true,
					Storage:     true,
					Offline:     true,
				},
			},
		},
	})

	m := &Monitor{
		wsHub:           hub,
		notificationMgr: notifMgr,
		alertManager:    mgr,
	}

	// Seed a resolved alert in the alert manager.
	// Fire a warning alert via CheckGuest and then resolve it.
	guest := models.VM{
		ID:       "100",
		VMID:     100,
		Name:     "test-vm",
		Node:     "pve1",
		Status:   "running",
		Type:     "qemu",
		CPU:      0.95, // 95% — above default threshold
		CPUs:     1,
		Memory:   models.Memory{Usage: 50},
		Instance: "https://pve.local:8006",
	}

	// Fire the alert (CPU > threshold)
	mgr.CheckGuest(guest, "pve1")

	activeAlerts := mgr.GetActiveAlerts()
	if len(activeAlerts) == 0 {
		t.Skip("no alert fired — threshold may differ from defaults, skipping integration test")
	}

	alertID := activeAlerts[0].ID

	// Now resolve: bring CPU below threshold
	guest.CPU = 0.10
	mgr.CheckGuest(guest, "pve1")

	// The alert should now be in recently resolved
	resolved := mgr.GetResolvedAlert(alertID)
	if resolved == nil {
		t.Skip("alert was not resolved by CheckGuest — skipping integration test")
	}

	// Verify quiet hours suppression directly
	if !mgr.ShouldSuppressResolvedNotification(resolved.Alert) {
		t.Fatal("expected ShouldSuppressResolvedNotification to return true during quiet hours")
	}

	// Track whether resolved AI callback fires (it should, even during quiet hours)
	var aiCallbackCalled atomic.Int32
	m.alertResolvedAICallback = func(a *alerts.Alert) {
		aiCallbackCalled.Add(1)
	}

	// Call handleAlertResolved — quiet hours should suppress the notification
	m.handleAlertResolved(alertID)

	// Give goroutine time to execute
	time.Sleep(50 * time.Millisecond)

	// AI callback should always fire regardless of quiet hours
	if aiCallbackCalled.Load() == 0 {
		t.Error("expected AI resolved callback to fire even during quiet hours")
	}
}

func TestMonitor_HandleAlertResolved_NoQuietHoursSendsNotification(t *testing.T) {
	// Verify that resolved notifications are sent when quiet hours are NOT active.
	hub := websocket.NewHub(nil)
	notifMgr := notifications.NewNotificationManager("dummy")
	notifMgr.SetNotifyOnResolve(true)

	mgr := alerts.NewManager()
	// No quiet hours, but alerts enabled with no time delay
	mgr.UpdateConfig(alerts.AlertConfig{
		Enabled: true,
		TimeThresholds: map[string]int{
			"guest": 0, "node": 0, "storage": 0, "pbs": 0, "host": 0,
		},
		GuestDefaults: alerts.ThresholdConfig{
			CPU:    &alerts.HysteresisThreshold{Trigger: 80, Clear: 75},
			Memory: &alerts.HysteresisThreshold{Trigger: 85, Clear: 80},
		},
	})

	m := &Monitor{
		wsHub:           hub,
		notificationMgr: notifMgr,
		alertManager:    mgr,
	}

	// Seed a resolved alert
	guest := models.VM{
		ID:       "200",
		VMID:     200,
		Name:     "test-vm-2",
		Node:     "pve2",
		Status:   "running",
		Type:     "qemu",
		CPU:      0.95,
		CPUs:     1,
		Memory:   models.Memory{Usage: 50},
		Instance: "https://pve.local:8006",
	}

	mgr.CheckGuest(guest, "pve2")
	activeAlerts := mgr.GetActiveAlerts()
	if len(activeAlerts) == 0 {
		t.Skip("no alert fired — threshold may differ from defaults, skipping integration test")
	}

	alertID := activeAlerts[0].ID
	guest.CPU = 0.10
	mgr.CheckGuest(guest, "pve2")

	// Should not crash, and notification should be dispatched (not suppressed)
	m.handleAlertResolved(alertID)
}

func TestMonitor_HandleAlertResolved_SendsRecoveryForGuestPoweredOffState(t *testing.T) {
	received := make(chan []byte, 2)
	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		defer r.Body.Close()
		body, _ := io.ReadAll(r.Body)
		select {
		case received <- body:
		default:
		}
		w.WriteHeader(http.StatusOK)
	}))
	t.Cleanup(srv.Close)

	notifMgr := notifications.NewNotificationManagerWithDataDir("http://pulse.example", t.TempDir())
	t.Cleanup(notifMgr.Stop)
	if err := notifMgr.UpdateAllowedPrivateCIDRs("127.0.0.1/32,::1/128"); err != nil {
		t.Fatalf("UpdateAllowedPrivateCIDRs: %v", err)
	}
	notifMgr.AddWebhook(notifications.WebhookConfig{
		ID:      "test-webhook",
		Name:    "test-webhook",
		URL:     srv.URL,
		Enabled: true,
		Service: "generic",
	})
	notifMgr.SetNotifyOnResolve(true)
	notifMgr.SetGroupingWindow(0)

	alertMgr := alerts.NewManagerWithDataDir(t.TempDir())
	t.Cleanup(alertMgr.Stop)
	cfg := alertMgr.GetConfig()
	cfg.Enabled = true
	cfg.ActivationState = alerts.ActivationActive
	cfg.Schedule.QuietHours.Enabled = false
	alertMgr.UpdateConfig(cfg)

	m := &Monitor{
		alertManager:    alertMgr,
		notificationMgr: notifMgr,
	}
	alertMgr.SetAlertCallback(m.handleAlertFired)
	alertMgr.SetResolvedCallback(m.handleAlertResolved)

	vm := models.VM{
		ID:       "vm-powered-off",
		Name:     "powered-off-vm",
		Node:     "node-1",
		Instance: "inst-1",
		Status:   "stopped",
	}

	alertMgr.CheckGuest(vm, vm.Instance)
	alertMgr.CheckGuest(vm, vm.Instance)

	// Firing uses the grouped envelope even with a zero grouping window.
	var firingPayload struct {
		Grouped bool `json:"grouped"`
		Alerts  []struct {
			ID string `json:"id"`
		} `json:"alerts"`
	}
	select {
	case body := <-received:
		if err := json.Unmarshal(body, &firingPayload); err != nil {
			t.Fatalf("failed to parse firing webhook payload: %v", err)
		}
		if !firingPayload.Grouped || len(firingPayload.Alerts) != 1 {
			t.Fatalf("expected one grouped firing alert, got %+v", firingPayload)
		}
	case <-time.After(5 * time.Second):
		t.Fatalf("timed out waiting for initial powered-off notification webhook")
	}

	activeAlerts := alertMgr.GetActiveAlerts()
	if len(activeAlerts) != 1 {
		t.Fatalf("expected one active powered-off alert, got %#v", activeAlerts)
	}
	alertID := activeAlerts[0].ID
	if firingPayload.Alerts[0].ID != alertID {
		t.Fatalf("expected firing webhook alert ID=%q, got %q", alertID, firingPayload.Alerts[0].ID)
	}
	if activeAlerts[0].LastNotified == nil {
		t.Fatalf("expected powered-off alert %q to record firing notification time", alertID)
	}

	if resolved := alertMgr.GetResolvedAlert(alertID); resolved != nil {
		t.Fatalf("did not expect powered-off alert %q to be resolved before the VM starts", alertID)
	}

	vm.Status = "running"
	alertMgr.CheckGuest(vm, vm.Instance)

	select {
	case body := <-received:
		var payload map[string]interface{}
		if err := json.Unmarshal(body, &payload); err != nil {
			t.Fatalf("failed to parse webhook payload: %v", err)
		}
		if payload["event"] != "resolved" {
			t.Fatalf("expected webhook event=resolved, got %v", payload["event"])
		}
		if payload["alertIdentifier"] != alertID {
			t.Fatalf("expected webhook alertIdentifier=%q, got %v", alertID, payload["alertIdentifier"])
		}
	case <-time.After(5 * time.Second):
		t.Fatalf("timed out waiting for powered-off recovery notification webhook")
	}
}

func TestMonitor_HandleAlertResolved_SuppressesRecoveryWhenFiringNeverDelivered(t *testing.T) {
	// Regression test for #1553: an alert that resolves while its firing
	// notification is still waiting in the grouping window must not produce a
	// recovery-only notification.
	t.Setenv("PULSE_DATA_DIR", t.TempDir())
	received := make(chan []byte, 2)
	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		defer r.Body.Close()
		body, _ := io.ReadAll(r.Body)
		select {
		case received <- body:
		default:
		}
		w.WriteHeader(http.StatusOK)
	}))
	t.Cleanup(srv.Close)

	notifMgr := notifications.NewNotificationManagerWithDataDir("http://pulse.example", t.TempDir())
	if err := notifMgr.UpdateAllowedPrivateCIDRs("127.0.0.1/32,::1/128"); err != nil {
		t.Fatalf("UpdateAllowedPrivateCIDRs: %v", err)
	}
	notifMgr.AddWebhook(notifications.WebhookConfig{
		ID:      "test-webhook",
		Name:    "test-webhook",
		URL:     srv.URL,
		Enabled: true,
		Service: "generic",
	})
	notifMgr.SetNotifyOnResolve(true)
	// Large grouping window keeps the firing notification undelivered until
	// the alert resolves.
	notifMgr.SetGroupingWindow(120)

	alertMgr := alerts.NewManager()
	t.Cleanup(alertMgr.Stop)
	cfg := alertMgr.GetConfig()
	cfg.Enabled = true
	cfg.ActivationState = alerts.ActivationActive
	cfg.Schedule.QuietHours.Enabled = false
	alertMgr.UpdateConfig(cfg)

	m := &Monitor{
		alertManager:    alertMgr,
		notificationMgr: notifMgr,
	}

	vm := models.VM{
		ID:       "vm-transient-spike",
		Name:     "transient-spike-vm",
		Node:     "node-1",
		Instance: "inst-1",
		Status:   "stopped",
	}

	// Fire the alert without wiring the async fired-callback, then hand the
	// firing notification to the manager synchronously so it is deterministic
	// that it sits in the grouping window when the alert resolves.
	alertMgr.CheckGuest(vm, vm.Instance)
	alertMgr.CheckGuest(vm, vm.Instance)

	activeAlerts := alertMgr.GetActiveAlerts()
	if len(activeAlerts) != 1 {
		t.Fatalf("expected one active powered-off alert, got %#v", activeAlerts)
	}
	firing := activeAlerts[0]
	notifMgr.SendAlert(&firing)

	vm.Status = "running"
	alertMgr.CheckGuest(vm, vm.Instance)
	if resolved := alertMgr.GetResolvedAlert(firing.ID); resolved == nil {
		t.Fatalf("expected alert %q to be resolved", firing.ID)
	}

	m.handleAlertResolved(firing.ID)

	select {
	case body := <-received:
		t.Fatalf("expected no notification for a firing that never left the grouping window, got %s", string(body))
	case <-time.After(1500 * time.Millisecond):
	}
}

func TestMonitor_HandleAlertEscalated_QuietHoursSuppressesNotification(t *testing.T) {
	t.Setenv("PULSE_DATA_DIR", t.TempDir())

	requests := make(chan struct{}, 1)
	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		requests <- struct{}{}
		w.WriteHeader(http.StatusOK)
	}))
	defer server.Close()

	notifMgr := notifications.NewNotificationManager("https://pulse.local")
	defer notifMgr.Stop()
	notifMgr.SetGroupingWindow(0)
	notifMgr.SetCooldown(0)
	if err := notifMgr.UpdateAllowedPrivateCIDRs("127.0.0.1/32"); err != nil {
		t.Fatalf("UpdateAllowedPrivateCIDRs: %v", err)
	}
	notifMgr.AddWebhook(notifications.WebhookConfig{
		ID:      "quiet-hours-hook",
		Name:    "quiet-hours",
		URL:     server.URL,
		Enabled: true,
	})

	mgr := alerts.NewManager()
	cfg := mgr.GetConfig()
	cfg.Schedule.Escalation.Levels = []alerts.EscalationLevel{{After: 1, Notify: "webhook"}}
	cfg.Schedule.QuietHours = alerts.QuietHours{
		Enabled:  true,
		Start:    "00:00",
		End:      "23:59",
		Timezone: "UTC",
		Days: map[string]bool{
			"monday": true, "tuesday": true, "wednesday": true,
			"thursday": true, "friday": true, "saturday": true, "sunday": true,
		},
		Suppress: alerts.QuietHoursSuppression{Offline: true},
	}
	mgr.UpdateConfig(cfg)

	m := &Monitor{
		notificationMgr: notifMgr,
		alertManager:    mgr,
	}

	alert := activeEscalationFixture(t, mgr, "connectivity", alerts.AlertLevelCritical)

	m.handleAlertEscalated(nil, alert, 1)

	select {
	case <-requests:
		t.Fatal("expected quiet hours to suppress escalated notification delivery")
	case <-time.After(500 * time.Millisecond):
	}
}

func TestMonitor_HandleAlertEscalated_SendsNotificationWhenNotSuppressed(t *testing.T) {
	t.Setenv("PULSE_DATA_DIR", t.TempDir())

	requests := make(chan struct{}, 1)
	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		requests <- struct{}{}
		w.WriteHeader(http.StatusOK)
	}))
	defer server.Close()

	notifMgr := notifications.NewNotificationManager("https://pulse.local")
	defer notifMgr.Stop()
	notifMgr.SetGroupingWindow(0)
	notifMgr.SetCooldown(0)
	if err := notifMgr.UpdateAllowedPrivateCIDRs("127.0.0.1/32"); err != nil {
		t.Fatalf("UpdateAllowedPrivateCIDRs: %v", err)
	}
	notifMgr.AddWebhook(notifications.WebhookConfig{
		ID:      "normal-hook",
		Name:    "normal",
		URL:     server.URL,
		Enabled: true,
	})

	mgr := alerts.NewManager()
	cfg := mgr.GetConfig()
	cfg.Schedule.Escalation.Levels = []alerts.EscalationLevel{{After: 1, Notify: "webhook"}}
	cfg.Schedule.QuietHours.Enabled = false
	mgr.UpdateConfig(cfg)

	m := &Monitor{
		notificationMgr: notifMgr,
		alertManager:    mgr,
	}

	alert := activeEscalationFixture(t, mgr, "connectivity", alerts.AlertLevelCritical)

	m.handleAlertEscalated(nil, alert, 1)

	select {
	case <-requests:
	case <-time.After(2 * time.Second):
		t.Fatal("expected escalated notification delivery")
	}
}

func TestMonitor_HandleAlertEscalated_BypassesDeliveryCooldown(t *testing.T) {
	t.Setenv("PULSE_DATA_DIR", t.TempDir())

	requests := make(chan struct{}, 2)
	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		requests <- struct{}{}
		w.WriteHeader(http.StatusOK)
	}))
	defer server.Close()

	notifMgr := notifications.NewNotificationManager("https://pulse.local")
	defer notifMgr.Stop()
	notifMgr.SetGroupingWindow(0)
	notifMgr.SetCooldown(30)
	if err := notifMgr.UpdateAllowedPrivateCIDRs("127.0.0.1/32"); err != nil {
		t.Fatalf("UpdateAllowedPrivateCIDRs: %v", err)
	}
	notifMgr.AddWebhook(notifications.WebhookConfig{
		ID:      "cooldown-hook",
		Name:    "cooldown",
		URL:     server.URL,
		Enabled: true,
	})

	mgr := alerts.NewManager()
	cfg := mgr.GetConfig()
	cfg.Schedule.Escalation.Levels = []alerts.EscalationLevel{{After: 1, Notify: "webhook"}}
	cfg.Schedule.QuietHours.Enabled = false
	mgr.UpdateConfig(cfg)

	m := &Monitor{
		notificationMgr: notifMgr,
		alertManager:    mgr,
	}

	alert := activeEscalationFixture(t, mgr, "memory", alerts.AlertLevelWarning)

	notifMgr.SendAlert(alert)
	select {
	case <-requests:
	case <-time.After(2 * time.Second):
		t.Fatal("expected initial notification delivery")
	}

	m.handleAlertEscalated(nil, alert, 1)

	select {
	case <-requests:
	case <-time.After(2 * time.Second):
		t.Fatal("expected escalated notification delivery despite active cooldown")
	}
}

// Exercise the dispatcher used by lifecycle replay with both projections
// attached. Store-only tests cannot detect cross-occurrence canonical history.
func TestMonitorLifecycleReplayPreservesOccurrenceTimelines(t *testing.T) {
	for _, gap := range []time.Duration{2 * time.Minute, 500 * time.Millisecond} {
		t.Run(gap.String(), func(t *testing.T) {
			store := unifiedresources.NewMemoryStore()
			incidents := memory.NewIncidentStore(memory.IncidentStoreConfig{})
			m := &Monitor{
				incidentStore: incidents,
				resourceStore: unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(store)),
			}
			incidents.SetResourceTimelineStore(m.resourceStore.(memory.IncidentTimelineStore))
			start := time.Now().UTC().Add(-time.Hour).Truncate(time.Second)
			old := &alerts.Alert{ID: "pbs-connectivity", ResourceID: "pbs", StartTime: start}
			end := start.Add(gap / 2)
			fired := alerts.LifecycleEvent{Type: eventlog.TypeFired, OccurredAt: start, Alert: old}
			resolved := alerts.LifecycleEvent{Type: eventlog.TypeResolved, OccurredAt: end, Alert: old}
			m.handleAlertLifecycleEvent(fired)
			m.handleAlertLifecycleEvent(resolved)
			first := incidents.GetTimelineByAlertAt(old.ID, start)
			require.NotNil(t, first)
			for i := 0; i < 10; i++ {
				m.handleAlertLifecycleEvent(fired)
				m.handleAlertLifecycleEvent(resolved)
			}
			require.Len(t, incidents.ListIncidentsByResource(old.ResourceID, 0), 1)
			replayed := incidents.GetTimelineByAlertAt(old.ID, start)
			require.Equal(t, first.ID, replayed.ID)
			require.Equal(t, memory.IncidentStatusResolved, replayed.Status)
			require.Len(t, replayed.Events, 2)

			next := old.Clone()
			next.StartTime = start.Add(gap)
			m.handleAlertLifecycleEvent(alerts.LifecycleEvent{Type: eventlog.TypeFired, OccurredAt: next.StartTime, Alert: next})
			// A historical resolution arriving after recurrence must not close it;
			// conversely, the recurrence must not reopen the historical timeline.
			m.handleAlertLifecycleEvent(resolved)
			require.Len(t, incidents.ListIncidentsByResource(old.ResourceID, 0), 2)
			historical := incidents.GetTimelineByAlertAt(old.ID, start)
			current := incidents.GetTimelineByAlertAt(next.ID, next.StartTime)
			require.NotNil(t, historical)
			require.NotNil(t, current)
			require.Equal(t, first.ID, historical.ID)
			require.NotEqual(t, historical.ID, current.ID)
			require.Equal(t, memory.IncidentStatusResolved, historical.Status)
			require.Equal(t, &end, historical.ClosedAt)
			require.Len(t, historical.Events, 2)
			require.Equal(t, memory.IncidentStatusOpen, current.Status)
			require.Nil(t, current.ClosedAt)
			require.Len(t, current.Events, 1)
			changes, err := store.GetRecentChanges(old.ResourceID, time.Time{}, 100)
			require.NoError(t, err)
			require.Len(t, changes, 3)

			ackAt := start.Add(gap / 4)
			old.AckTime = &ackAt
			m.handleAlertLifecycleEvent(alerts.LifecycleEvent{
				Type: eventlog.TypeAcknowledged, OccurredAt: ackAt, Alert: old,
				Details: map[string]string{"user": "historical-operator"},
			})
			historical = incidents.GetTimelineByAlertAt(old.ID, start)
			current = incidents.GetTimelineByAlertAt(next.ID, next.StartTime)
			require.Equal(t, memory.IncidentStatusResolved, historical.Status)
			require.True(t, historical.Acknowledged)
			require.Equal(t, "historical-operator", historical.AckUser)
			require.Len(t, historical.Events, 3)
			require.False(t, current.Acknowledged)
			require.Equal(t, memory.IncidentStatusOpen, current.Status)
			require.Len(t, current.Events, 1)
		})
	}
}

func TestMonitorLifecycleRefireReopensRetainedOccurrence(t *testing.T) {
	for _, canonical := range []bool{false, true} {
		name := "local"
		if canonical {
			name = "canonical"
		}
		t.Run(name, func(t *testing.T) {
			store := unifiedresources.NewMemoryStore()
			incidents := memory.NewIncidentStore(memory.IncidentStoreConfig{})
			m := &Monitor{incidentStore: incidents, resourceStore: unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(store))}
			if canonical {
				incidents.SetResourceTimelineStore(m.resourceStore.(memory.IncidentTimelineStore))
			}
			start := time.Now().UTC().Add(-time.Hour).Truncate(time.Second)
			alert := &alerts.Alert{ID: "node-connectivity", ResourceID: "node", StartTime: start}
			fire := alerts.LifecycleEvent{Type: eventlog.TypeFired, OccurredAt: start, Alert: alert}
			resolve := alerts.LifecycleEvent{Type: eventlog.TypeResolved, OccurredAt: start.Add(time.Minute), Alert: alert}
			refire := alerts.LifecycleEvent{Type: eventlog.TypeRefired, OccurredAt: start.Add(2 * time.Minute), Alert: alert}
			m.handleAlertLifecycleEvent(fire)
			m.handleAlertLifecycleEvent(resolve)
			original := incidents.GetTimelineByAlertAt(alert.ID, start)
			require.Equal(t, memory.IncidentStatusResolved, original.Status)
			m.handleAlertLifecycleEvent(refire)
			assertOpen := func() {
				t.Helper()
				current := incidents.GetTimelineByAlertAt(alert.ID, start)
				require.Equal(t, original.ID, current.ID)
				require.Equal(t, memory.IncidentStatusOpen, current.Status)
				require.Nil(t, current.ClosedAt)
				require.Len(t, current.Events, 3)
				require.Len(t, incidents.ListIncidentsByResource(alert.ResourceID, 0), 1)
			}
			assertOpen()
			if canonical {
				// Canonical history must recover the retained occurrence even when
				// no incident checkpoint survives.
				recovered := memory.NewIncidentStore(memory.IncidentStoreConfig{})
				recovered.SetResourceTimelineStore(m.resourceStore.(memory.IncidentTimelineStore))
				page, err := recovered.QueryIncidents(memory.IncidentQuery{ResourceID: alert.ResourceID})
				require.NoError(t, err)
				require.Len(t, page.Incidents, 1)
				require.Equal(t, memory.IncidentStatusOpen, page.Incidents[0].Status)
				require.Equal(t, start, page.Incidents[0].OpenedAt)
				require.Len(t, page.Incidents[0].Events, 3)
			}
			for i := 0; i < 10; i++ {
				m.handleAlertLifecycleEvent(fire)
				m.handleAlertLifecycleEvent(resolve)
				m.handleAlertLifecycleEvent(refire)
			}
			assertOpen()
			// Historical read repair must not close an occurrence that re-fired.
			incidents.EnsureAlertOccurrence(alert, &resolve.OccurredAt)
			assertOpen()
			finalResolve := resolve
			finalResolve.OccurredAt = start.Add(3 * time.Minute)
			m.handleAlertLifecycleEvent(finalResolve)
			m.handleAlertLifecycleEvent(refire)
			current := incidents.GetTimelineByAlertAt(alert.ID, start)
			require.Equal(t, memory.IncidentStatusResolved, current.Status)
			require.Equal(t, &finalResolve.OccurredAt, current.ClosedAt)
			require.Len(t, current.Events, 4)
		})
	}
}

type occurrenceHTTPReceipt struct {
	Event  string          `json:"event"`
	Alerts []*alerts.Alert `json:"alerts"`
}

func occurrenceNotifier(t *testing.T, dir string, endpoint string) *notifications.NotificationManager {
	t.Helper()
	n := notifications.NewNotificationManagerWithDeferredQueue("", dir)
	t.Cleanup(n.Stop)
	if err := n.UpdateAllowedPrivateCIDRs("127.0.0.1/32,::1/128"); err != nil {
		t.Fatal(err)
	}
	n.AddWebhook(notifications.WebhookConfig{ID: "ops", Enabled: true, URL: endpoint, Service: "generic"})
	n.SetGroupingWindow(0)
	n.SetNotifyOnResolve(false)
	return n
}

func occurrenceManager(t *testing.T) (*alerts.Manager, models.PBSInstance) {
	t.Helper()
	a := alerts.NewManagerWithDataDir(t.TempDir())
	t.Cleanup(a.Stop)
	cfg := a.GetConfig()
	cfg.Enabled, cfg.ActivationState, cfg.FlappingEnabled = true, alerts.ActivationActive, false
	cfg.Schedule.QuietHours.Enabled = false
	cfg.TimeThresholds["pbs"] = 0
	cfg.PBSDefaults.CPU = &alerts.HysteresisThreshold{Trigger: 80, Clear: 75}
	a.UpdateConfig(cfg)
	return a, models.PBSInstance{ID: "recurring-pbs", Name: "backup", Host: "pbs.invalid", Status: "online", ConnectionHealth: "healthy", CPU: 99}
}

func occurrenceEndpoint(t *testing.T) (string, <-chan occurrenceHTTPReceipt) {
	t.Helper()
	receipts := make(chan occurrenceHTTPReceipt, 8)
	s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		defer r.Body.Close()
		var receipt occurrenceHTTPReceipt
		if err := json.NewDecoder(r.Body).Decode(&receipt); err != nil {
			t.Errorf("decode HTTP acceptance: %v", err)
			w.WriteHeader(http.StatusBadRequest)
			return
		}
		receipts <- receipt
		w.WriteHeader(http.StatusOK)
	}))
	t.Cleanup(s.Close)
	return s.URL, receipts
}

func awaitOccurrenceRows(t *testing.T, n *notifications.NotificationManager, count int) []*notifications.QueuedNotification {
	t.Helper()
	deadline := time.Now().Add(3 * time.Second)
	for {
		rows, err := n.GetQueue().GetPending(10)
		if err != nil {
			t.Fatal(err)
		}
		if len(rows) == count {
			return rows
		}
		if time.Now().After(deadline) {
			t.Fatalf("pending rows = %d, want %d", len(rows), count)
		}
		time.Sleep(time.Millisecond)
	}
}

func assertOccurrenceHTTP(t *testing.T, n *notifications.NotificationManager, receipts <-chan occurrenceHTTPReceipt, current *alerts.Alert) {
	t.Helper()
	n.StartQueueProcessing()
	select {
	case got := <-receipts:
		if len(got.Alerts) != 1 || got.Alerts[0].ID != current.ID || !got.Alerts[0].StartTime.Equal(current.StartTime) {
			t.Fatalf("wrong HTTP occurrence: %+v", got)
		}
		t.Logf("HTTP 200 accepted current occurrence %s at %s", current.ID, current.StartTime.Format(time.RFC3339Nano))
	case <-time.After(12 * time.Second):
		t.Fatal("current firing did not reach the local HTTP destination")
	}
	deadline := time.Now().Add(3 * time.Second)
	for {
		stats, err := n.GetQueueStats()
		if err != nil {
			t.Fatal(err)
		}
		if stats["sent"] == 1 && stats["cancelled"] == 1 && stats["pending"] == 0 && stats["sending"] == 0 {
			t.Logf("persistent queue completion: %v", stats)
			break
		}
		if time.Now().After(deadline) {
			t.Fatalf("queue completion = %v", stats)
		}
		time.Sleep(time.Millisecond)
	}
	select {
	case extra := <-receipts:
		t.Fatalf("obsolete or duplicate delivery: %+v", extra)
	case <-time.After(200 * time.Millisecond):
	}
}

// This existing-interface control also runs against the exact predecessor.
// It models a delayed ID callback after the next occurrence has entered the
// real persistent queue. It does not assert native timing or cause a refire.
func TestMonitorDelayedIDResolutionPreservesNewQueuedOccurrence(t *testing.T) {
	endpoint, receipts := occurrenceEndpoint(t)
	dir := t.TempDir()
	n := occurrenceNotifier(t, dir, endpoint)
	a, pbs := occurrenceManager(t)
	a.CheckPBS(pbs)
	active := a.GetActiveAlerts()
	if len(active) != 1 {
		t.Fatalf("PBS firing = %+v", active)
	}
	old := active[0].Clone()
	pbs.CPU = 0
	a.CheckPBS(pbs)
	if r := a.GetResolvedAlert(old.ID); r == nil || !r.Alert.StartTime.Equal(old.StartTime) {
		t.Fatal("missing resolved PBS snapshot")
	}
	current := old.Clone()
	current.StartTime = current.StartTime.Add(time.Nanosecond)
	n.SendAlert(old)
	n.SendAlert(current)
	awaitOccurrenceRows(t, n, 2)
	m := &Monitor{alertManager: a, notificationMgr: n}
	m.handleAlertResolved(old.ID)
	rows, err := n.GetQueue().GetPending(10)
	if err != nil || len(rows) != 1 || len(rows[0].Alerts) != 1 || !rows[0].Alerts[0].StartTime.Equal(current.StartTime) {
		t.Fatalf("delayed recovery cancelled the replacement occurrence: rows=%+v err=%v", rows, err)
	}
	// Resume the autonomous processor only after reopening the same disk state.
	n.Stop()
	n = occurrenceNotifier(t, dir, endpoint)
	assertOccurrenceHTTP(t, n, receipts, current)
}

// One destination accepted the old firing and another still has a retry. After
// restart, that pending work (and the lost RAM cooldown marker) is not evidence
// that no recipient saw it. Recovery must use exact persisted receipts.
func TestMonitorDelayedPartialResolutionKeepsDestinationRecovery(t *testing.T) {
	endpoint, receipts := occurrenceEndpoint(t)
	var failedFirings, unwantedRecoveries atomic.Int32
	broken := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		defer r.Body.Close()
		var receipt occurrenceHTTPReceipt
		if err := json.NewDecoder(r.Body).Decode(&receipt); err != nil {
			t.Errorf("decode failing destination: %v", err)
		}
		if receipt.Event == "resolved" {
			unwantedRecoveries.Add(1)
		} else {
			failedFirings.Add(1)
		}
		w.WriteHeader(http.StatusServiceUnavailable)
	}))
	t.Cleanup(broken.Close)
	dir := t.TempDir()
	open := func() *notifications.NotificationManager {
		n := occurrenceNotifier(t, dir, endpoint)
		n.SetNotifyOnResolve(true)
		n.AddWebhook(notifications.WebhookConfig{ID: "unannounced", Enabled: true, URL: broken.URL, Service: "generic"})
		return n
	}
	n := open()
	a, pbs := occurrenceManager(t)
	m := &Monitor{alertManager: a, notificationMgr: n}
	// Use the live firing callback so the detector records notification policy
	// state. Direct notifier calls would not establish LastNotified, and existing
	// recovery policy would correctly suppress that unannounced detector alert.
	a.SetAlertCallback(m.handleAlertFired)
	a.CheckPBS(pbs)
	old := a.GetActiveAlerts()[0].Clone()
	if old.LastNotified == nil {
		t.Fatal("detector did not admit the firing notification")
	}
	awaitOccurrenceRows(t, n, 2)
	n.StartQueueProcessing()
	select {
	case got := <-receipts:
		if got.Event != "" || len(got.Alerts) != 1 || !got.Alerts[0].StartTime.Equal(old.StartTime) {
			t.Fatalf("wrong initial firing: %+v", got)
		}
	case <-time.After(12 * time.Second):
		t.Fatal("initial firing did not reach accepting destination")
	}
	deadline := time.Now().Add(3 * time.Second)
	for {
		stats, err := n.GetQueueStats()
		if err != nil {
			t.Fatal(err)
		}
		if stats["sent"] == 1 && stats["pending"] == 1 && failedFirings.Load() > 0 {
			break
		}
		if time.Now().After(deadline) {
			t.Fatalf("partial delivery not retained: %v", stats)
		}
		time.Sleep(time.Millisecond)
	}
	n.Stop()
	pbs.CPU = 0
	a.CheckPBS(pbs)
	n = open() // deferred processor; prior destination receipts are now on disk only
	m = &Monitor{alertManager: a, notificationMgr: n}
	a.SetAlertCallback(m.handleAlertFired)
	pbs.CPU = 95
	a.CheckPBS(pbs)
	active := a.GetActiveAlerts()
	if len(active) != 1 || active[0].ID != old.ID || active[0].StartTime.Equal(old.StartTime) {
		t.Fatalf("not a new occurrence: %+v", active)
	}
	current := active[0].Clone()
	awaitOccurrenceRows(t, n, 2)
	m.handleAlertResolved(old.ID)
	n.StartQueueProcessing()
	seenFiring, seenRecovery := false, false
	for i := 0; i < 2; i++ {
		select {
		case got := <-receipts:
			if len(got.Alerts) != 1 || got.Alerts[0].ID != old.ID {
				t.Fatalf("wrong delivery: %+v", got)
			}
			switch got.Event {
			case "resolved":
				if seenRecovery || !got.Alerts[0].StartTime.Equal(old.StartTime) {
					t.Fatalf("wrong recovery occurrence: %+v", got)
				}
				seenRecovery = true
			case "":
				if seenFiring || !got.Alerts[0].StartTime.Equal(current.StartTime) {
					t.Fatalf("wrong firing occurrence: %+v", got)
				}
				seenFiring = true
			default:
				t.Fatalf("unknown event: %+v", got)
			}
		case <-time.After(12 * time.Second):
			t.Fatal("lost current firing or old accepted destination's recovery after restart")
		}
	}
	deadline = time.Now().Add(3 * time.Second)
	for {
		stats, err := n.GetQueueStats()
		if err != nil {
			t.Fatal(err)
		}
		if stats["sent"] == 3 && stats["cancelled"] == 1 && stats["pending"]+stats["sending"]+stats["failed"]+stats["dlq"] == 1 {
			t.Logf("partial destination recovery retained exact old receipt and new firing: %v", stats)
			break
		}
		if time.Now().After(deadline) {
			t.Fatalf("partial recovery queue completion: %v", stats)
		}
		time.Sleep(time.Millisecond)
	}
	if unwantedRecoveries.Load() != 0 {
		t.Fatal("unannounced destination received a recovery")
	}
}

// A host agent linked to a Proxmox node merges into one read-state row that
// every PVE poll keeps fresh. When the agent stops reporting, its retained
// sensors must stop feeding the node's temperature: otherwise the frozen
// reading is re-observed every poll, an open temperature alert's LastSeen keeps
// moving, and stale-alert cleanup never reaches it.
func TestSilentLinkedAgentStopsRefreshingNodeTemperatureAlert(t *testing.T) {
	manager := alerts.NewManagerWithDataDir(t.TempDir(), alerts.WithoutPersistedAlertRestore())
	t.Cleanup(manager.Stop)
	alertConfig := manager.GetConfig()
	alertConfig.Enabled = true
	alertConfig.TimeThresholds = map[string]int{}
	alertConfig.SuppressionWindow = 0
	alertConfig.NodeDefaults.Temperature = &alerts.HysteresisThreshold{Trigger: 80, Clear: 75}
	manager.UpdateConfig(alertConfig)

	adapter := unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(nil))
	monitor := &Monitor{
		config:        &config.Config{TemperatureMonitoringEnabled: true},
		state:         models.NewState(),
		resourceStore: adapter,
	}
	node := models.Node{
		ID: "pve1-node1", Name: "node1", Instance: "pve1", Status: "online",
		ConnectionHealth: "healthy", LinkedAgentID: "agent-1",
	}
	// ingest rebuilds the store, as each monitor refresh does, from the last PVE
	// poll's node (with the temperature it published, which the next poll reads
	// back as its previous node) plus the agent as of its last report.
	lastPolled := node
	ingest := func(agentLastReport time.Time, agentStatus string) {
		polledNode := lastPolled
		polledNode.LastSeen = time.Now()
		adapter.PopulateFromSnapshot(models.StateSnapshot{
			Nodes: []models.Node{polledNode},
			Hosts: []models.Host{{
				ID: "agent-1", Hostname: "node1", Status: agentStatus, LinkedNodeID: node.ID,
				IntervalSeconds: 30, LastSeen: agentLastReport,
				Sensors: models.HostSensorSummary{TemperatureCelsius: map[string]float64{"cpu_package": 92}},
			}},
		})
	}
	// poll runs the node temperature step of a PVE poll (no SSH collector) and
	// evaluates the node's alerts with the result.
	poll := func() *models.Temperature {
		prevNodes := monitor.snapshotPrevNodes("pve1")
		polled := node
		monitor.collectNodeTemperatureData(
			context.Background(), "pve1", &config.PVEInstance{Name: "pve1"}, proxmox.Node{Node: node.Name},
			&polled, prevNodes, "online",
		)
		manager.CheckNode(polled)
		lastPolled = polled
		return polled.Temperature
	}
	temperatureAlert := func() alerts.Alert {
		t.Helper()
		for _, alert := range manager.GetActiveAlerts() {
			if alert.Type == "temperature" && alert.ResourceID == node.ID {
				return alert
			}
		}
		t.Fatalf("expected an open temperature alert for %s", node.ID)
		return alerts.Alert{}
	}

	// While the agent reports, its hot reading feeds the node, stamped with the
	// agent's report time, and opens the alert.
	agentReport := time.Now().Add(-10 * time.Second)
	ingest(agentReport, "online")
	reading := poll()
	require.NotNil(t, reading)
	require.Equal(t, 92.0, reading.CPUPackage)
	require.True(t, reading.LastUpdate.Equal(agentReport), "the reading carries the agent's own report time")
	opened := temperatureAlert()

	// The agent goes silent while PVE polling continues: the merged row stays
	// fresh, but the agent's retained sensors no longer count as a reading, and
	// the poller does not carry the agent's last reading past its lease. With no
	// clock to advance, age the agent's last report and the reading it produced
	// by the same amount.
	silence := hostAgentHealthWindow(30) + time.Minute
	agedReading := *lastPolled.Temperature
	agedReading.LastUpdate = agedReading.LastUpdate.Add(-silence)
	lastPolled.Temperature = &agedReading
	ingest(agentReport.Add(-silence), "offline")
	time.Sleep(5 * time.Millisecond)
	require.Nil(t, poll(), "a silent agent's retained sensors must not be presented as a current reading")
	held := temperatureAlert()
	require.True(t, held.LastSeen.Equal(opened.LastSeen), "a frozen agent reading must not keep the alert fresh")
	require.Nil(t, poll())
	require.True(t, temperatureAlert().LastSeen.Equal(opened.LastSeen))

	// The agent reports again and its reading is evaluated.
	ingest(time.Now(), "online")
	time.Sleep(5 * time.Millisecond)
	require.NotNil(t, poll())
	require.True(t, temperatureAlert().LastSeen.After(held.LastSeen), "a fresh agent reading is evaluated")
}

// Drive the actual node/agent evaluators into the monitor's lifecycle handler,
// rather than fabricating a pre-stamped handover event.
func TestMonitorReportedAgentHandoverUsesActualMetricOwner(t *testing.T) {
	manager := alerts.NewManagerWithDataDir(t.TempDir(), alerts.WithoutPersistedAlertRestore())
	t.Cleanup(manager.Stop)
	// Match the monitoring bootstrap: ephemeral managers otherwise have no
	// durable event log, so AlertEvents cannot prove the served History route.
	manager.EnableEventLog()
	cfg := manager.GetConfig()
	cfg.Enabled = true
	cfg.ActivationState = alerts.ActivationPending
	cfg.TimeThresholds = map[string]int{}
	// Pin zero delay deliberately; config normalization restores factory type
	// delays, and CPU (unlike critical memory) does not bypass that delay.
	cfg.MetricTimeThresholds = map[string]map[string]int{"all": {"cpu": 0, "memory": 0}}
	cfg.SuppressionWindow = 0
	cfg.NodeDefaults.CPU = &alerts.HysteresisThreshold{Trigger: 80, Clear: 75}
	cfg.NodeDefaults.Memory = &alerts.HysteresisThreshold{Trigger: 85, Clear: 80}
	cfg.AgentDefaults.CPU = &alerts.HysteresisThreshold{Trigger: 80, Clear: 75}
	cfg.AgentDefaults.Memory = &alerts.HysteresisThreshold{Trigger: 85, Clear: 80}
	cfg.Overrides = map[string]alerts.ThresholdConfig{
		"agent-a": {Memory: &alerts.HysteresisThreshold{Trigger: 0}},
		"agent-b": {CPU: &alerts.HysteresisThreshold{Trigger: 0}},
	}
	manager.UpdateConfig(cfg)
	store := unifiedresources.NewMemoryStore()
	incidentStore := memory.NewIncidentStore(memory.IncidentStoreConfig{})
	m := &Monitor{alertManager: manager, incidentStore: incidentStore, resourceStore: unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(store))}
	incidentStore.SetResourceTimelineStore(m.resourceStore.(memory.IncidentTimelineStore))
	manager.SubscribeLifecycleCallback(m.handleAlertLifecycleEvent)
	node := models.Node{ID: "instance/node", Name: "node", Instance: "instance", Status: "online", ConnectionHealth: "healthy", CPU: 0.99, Memory: models.Memory{Total: 100, Used: 99, Free: 1, Usage: 99}}
	manager.CheckNode(node)
	opened := manager.GetActiveAlerts()
	require.Len(t, opened, 2)
	fired, err := manager.AlertEvents(eventlog.Filter{Types: []string{eventlog.TypeFired}})
	require.NoError(t, err)
	require.Len(t, fired, 2, "control must record both original node occurrences")
	cpuAgent := models.Host{ID: "agent-a", Hostname: "node", DisplayName: "CPU reader", Status: "online", LinkedNodeID: node.ID, CPUUsage: 50}
	memoryAgent := models.Host{ID: "agent-b", Hostname: "node", DisplayName: "Memory reader", Status: "online", LinkedNodeID: node.ID, Memory: node.Memory}
	manager.CheckHost(cpuAgent)
	manager.CheckHost(memoryAgent)
	manager.CheckNode(node)
	for _, a := range opened {
		timeline := incidentStore.GetTimelineByAlertAt(a.ID, a.StartTime)
		require.NotNil(t, timeline)
		require.Len(t, timeline.Events, 2)
		wantName := "CPU reader (Host Agent)"
		wantID := "agent:agent-a"
		if a.Type == "memory" {
			wantName = "Memory reader (Host Agent)"
			wantID = "agent:agent-b"
		}
		summary := "Alert moved to " + wantName + ". This is not a recovery: check the agent for the current reading."
		require.Equal(t, summary, timeline.Events[1].Summary)
		events, err := manager.AlertEvents(eventlog.Filter{AlertID: a.ID, Types: []string{eventlog.TypeResolved}})
		require.NoError(t, err)
		require.Len(t, events, 1)
		require.Equal(t, "moved_to_agent", events[0].Reason)
		require.Equal(t, summary, events[0].Message)
		found := false
		for _, row := range manager.GetAlertHistory(10) {
			if row.ID == a.ID && row.Resolution != nil {
				found = true
				require.NotNil(t, row.Resolution)
				require.Equal(t, wantID, row.Resolution.SuccessorResourceID)
			}
		}
		require.True(t, found, "no resolved History row for %s", a.ID)
	}
	changes, err := store.GetRecentChanges(node.ID, time.Time{}, 10)
	require.NoError(t, err)
	require.Len(t, changes, 4)
	for _, c := range changes {
		if c.Kind == unifiedresources.ChangeAlertResolved {
			require.Equal(t, "moved_to_agent", c.Metadata[unifiedresources.MetadataAlertResolution])
			require.Contains(t, c.Reason, "This is not a recovery")
		}
	}
}

// PVE disk alerts reference the disk by device path and record the evaluated
// disk's serial and WWN. Through the real alert manager, each lifecycle row
// joins the disk that identity names, even when a reboot has reordered the
// devices and the registry still places the other disk at the path. The
// Alerts history Resource action, which reads incidents by the alert's own
// reference, still lists that reference's occurrences.
func TestProxmoxDiskAlertTimelineFollowsRecordedHardware(t *testing.T) {
	store, err := unifiedresources.NewSQLiteResourceStore(t.TempDir(), "default")
	require.NoError(t, err)
	t.Cleanup(func() { require.NoError(t, store.Close()) })
	manager := alerts.NewManagerWithDataDir(t.TempDir())
	t.Cleanup(manager.Stop)
	config := manager.GetConfig()
	config.Enabled = true
	config.ActivationState = alerts.ActivationPending
	config.SuppressionWindow = 0
	manager.UpdateConfig(config)
	now := time.Now()
	disk := func(device, serial, health string) models.PhysicalDisk {
		return models.PhysicalDisk{ID: unifiedresources.ProxmoxPhysicalDiskSourceID("lab", "pve1", device, "", ""), Node: "pve1", Instance: "lab",
			DevPath: device, Model: "Disk " + serial, Serial: serial, WWN: "0x5000c500" + serial, Type: "sata", Health: health, Wearout: -1, LastChecked: now}
	}
	registry := unifiedresources.NewRegistry(store)
	registry.IngestSnapshot(models.StateSnapshot{PhysicalDisks: []models.PhysicalDisk{disk("/dev/sda", "ZA1A2B3C", "FAILED"), disk("/dev/sdb", "ZA4D5E6F", "PASSED")}})
	incidents := memory.NewIncidentStore(memory.IncidentStoreConfig{})
	incidents.SetResourceTimelineStore(store)
	monitor := &Monitor{alertManager: manager, resourceStore: unifiedresources.NewMonitorAdapter(registry), incidentStore: incidents}
	manager.SubscribeLifecycleCallback(monitor.handleAlertLifecycleEvent)

	manager.CheckDiskHealth("lab", "pve1", proxmoxDiskFromPhysicalDisk(disk("/dev/sda", "ZA1A2B3C", "FAILED")))
	// A reboot swaps the devices before the registry catches up: the failing
	// disk now raises its alert at /dev/sdb, and the healthy one at /dev/sda
	// resolves the alert the failing disk raised there.
	manager.CheckDiskHealth("lab", "pve1", proxmoxDiskFromPhysicalDisk(disk("/dev/sdb", "ZA1A2B3C", "FAILED")))
	manager.CheckDiskHealth("lab", "pve1", proxmoxDiskFromPhysicalDisk(disk("/dev/sda", "ZA4D5E6F", "PASSED")))

	diskIDs := map[string]string{}
	for _, resource := range registry.List() {
		diskIDs[resource.Name] = resource.ID
	}
	failing, healthy := diskIDs["Disk ZA1A2B3C"], diskIDs["Disk ZA4D5E6F"]
	filters := unifiedresources.ResourceChangeFilters{Kinds: []unifiedresources.ChangeKind{unifiedresources.ChangeAlertFired, unifiedresources.ChangeAlertResolved}}
	changes, err := store.GetRecentChangesFiltered(failing, time.Time{}, 10, filters)
	require.NoError(t, err)
	require.Len(t, changes, 3)
	sda := unifiedresources.ProxmoxPhysicalDiskAlertResourceID("lab", "pve1", "/dev/sda")
	sdb := unifiedresources.ProxmoxPhysicalDiskAlertResourceID("lab", "pve1", "/dev/sdb")
	refs := map[string]int{}
	for _, change := range changes {
		require.Equal(t, failing, change.ResourceID)
		refs[string(change.Kind)+" "+unifiedresources.OwnedAlertReference(change)]++
	}
	require.Equal(t, map[string]int{"alert_fired " + sda: 1, "alert_resolved " + sda: 1, "alert_fired " + sdb: 1}, refs)
	changes, err = store.GetRecentChangesFiltered(healthy, time.Time{}, 10, filters)
	require.NoError(t, err)
	require.Empty(t, changes)

	page, err := incidents.QueryIncidents(memory.IncidentQuery{ResourceID: sda, Limit: 10})
	require.NoError(t, err)
	require.Len(t, page.Incidents, 1)
	require.Equal(t, memory.IncidentStatusResolved, page.Incidents[0].Status)
	require.Equal(t, sda, page.Incidents[0].ResourceID)
	page, err = incidents.QueryIncidents(memory.IncidentQuery{ResourceID: failing, Limit: 10})
	require.NoError(t, err)
	require.Len(t, page.Incidents, 2)
	page, err = incidents.QueryIncidents(memory.IncidentQuery{ResourceID: healthy, Limit: 10})
	require.NoError(t, err)
	require.Empty(t, page.Incidents)
}

// The state snapshot the frontend reads carries the alert's subject-free
// summary beside its full message.
func TestActiveAlertsSnapshotCarriesSubjectFreeSummary(t *testing.T) {
	manager := alerts.NewManagerWithDataDir(t.TempDir())
	t.Cleanup(manager.Stop)
	config := manager.GetConfig()
	config.Enabled = true
	config.ActivationState = alerts.ActivationPending
	config.TimeThresholds = map[string]int{}
	config.SuppressionWindow = 0
	manager.UpdateConfig(config)
	host := models.DockerHost{ID: "summary-host", Hostname: "summary-host", LastSeen: time.Now(), Containers: []models.DockerContainer{
		{ID: strings.Repeat("b", 64), Name: "worker", State: "running", Health: "unhealthy"},
	}}
	manager.CheckDockerHost(host)
	monitor := &Monitor{alertManager: manager}
	found := false
	for _, alert := range monitor.activeAlertsSnapshot() {
		if alert.Type != "docker-container-health" {
			continue
		}
		found = true
		require.Equal(t, "Docker container 'worker' health is unhealthy", alert.Message)
		require.Equal(t, "Health is unhealthy", alert.Summary)
	}
	require.True(t, found, "expected the container health alert in the state snapshot")
}

type observedAlertChanges struct {
	mu      sync.Mutex
	changes []unifiedresources.ResourceChange
}

func (o *observedAlertChanges) observe(change unifiedresources.ResourceChange) {
	o.mu.Lock()
	defer o.mu.Unlock()
	if change.Kind == unifiedresources.ChangeAlertFired || change.Kind == unifiedresources.ChangeAlertResolved {
		o.changes = append(o.changes, change)
	}
}

func (o *observedAlertChanges) take() []unifiedresources.ResourceChange {
	o.mu.Lock()
	defer o.mu.Unlock()
	out := o.changes
	o.changes = nil
	return out
}

// Live alert lifecycle events reach the adapter's change observer (and so the
// agent event stream) with the canonical resource ID they were stored under.
// Replay, reconciliation and imported history record the same rows without
// announcing them again.
func TestAlertTimelineAnnouncesOnlyLiveLifecycleChanges(t *testing.T) {
	store, err := unifiedresources.NewSQLiteResourceStore(t.TempDir(), "default")
	require.NoError(t, err)
	t.Cleanup(func() { require.NoError(t, store.Close()) })
	manager := alerts.NewManagerWithDataDir(t.TempDir())
	t.Cleanup(manager.Stop)
	config := manager.GetConfig()
	config.Enabled = true
	config.ActivationState = alerts.ActivationPending
	config.TimeThresholds = map[string]int{}
	config.SuppressionWindow = 0
	manager.UpdateConfig(config)

	containerID := strings.Repeat("e", 64)
	host := models.DockerHost{ID: "live-host", Hostname: "live-host", LastSeen: time.Now(), Containers: []models.DockerContainer{
		{ID: containerID, Name: "api", State: "running", Health: "unhealthy"},
	}}
	registry := unifiedresources.NewRegistry(store)
	registry.IngestSnapshot(models.StateSnapshot{DockerHosts: []models.DockerHost{host}})
	adapter := unifiedresources.NewMonitorAdapter(registry)
	observed := &observedAlertChanges{}
	adapter.SetChangeObserver(observed.observe)
	monitor := &Monitor{alertManager: manager, resourceStore: adapter}
	manager.SubscribeLifecycleCallback(monitor.handleAlertLifecycleEvent)
	canonicalID := unifiedresources.SourceSpecificID(unifiedresources.ResourceTypeAppContainer, unifiedresources.SourceDocker, host.ID+"/container/"+containerID)

	manager.CheckDockerHost(host)
	fired := observed.take()
	require.Len(t, fired, 1)
	require.Equal(t, unifiedresources.ChangeAlertFired, fired[0].Kind)
	require.Equal(t, canonicalID, fired[0].ResourceID)
	require.Equal(t, "docker-container-health", fired[0].Metadata[unifiedresources.MetadataAlertType])

	host.Containers[0].Health = "healthy"
	manager.CheckDockerHost(host)
	resolved := observed.take()
	require.Len(t, resolved, 1)
	require.Equal(t, unifiedresources.ChangeAlertResolved, resolved[0].Kind)
	require.Equal(t, canonicalID, resolved[0].ResourceID)

	// The announced changes are the stored rows, so agents can look them up.
	stored, err := store.GetRecentChangesFiltered(canonicalID, time.Time{}, 10, unifiedresources.ResourceChangeFilters{
		Kinds: []unifiedresources.ChangeKind{unifiedresources.ChangeAlertFired, unifiedresources.ChangeAlertResolved},
	})
	require.NoError(t, err)
	require.Len(t, stored, 2)
	require.ElementsMatch(t, []string{fired[0].ID, resolved[0].ID}, []string{stored[0].ID, stored[1].ID})

	// Replaying the whole lifecycle log projects the same events again; none
	// of them may be announced as new.
	monitor.replayAlertLifecycleProjections(context.Background())
	require.Empty(t, observed.take(), "replayed alert history must not reach the live change observer")

	alert := &alerts.Alert{ID: "imported-alert", Type: "cpu", Level: alerts.AlertLevelWarning, ResourceID: canonicalID, StartTime: time.Now().Add(-time.Hour)}
	monitor.recordAlertTimelineChange(alert, unifiedresources.ChangeAlertFired, alert.StartTime, "")
	require.Empty(t, observed.take(), "history projected outside the live callback must not be announced")
	recorded, err := store.GetRecentChangesFiltered(canonicalID, time.Time{}, 10, unifiedresources.ResourceChangeFilters{AlertIdentifiers: []string{alert.ID}})
	require.NoError(t, err)
	require.Len(t, recorded, 1, "the non-live projection still records the row")
}