Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

53.8 KB
Raw Blame
package alerting

import (
	"bytes"
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"net/http"
	"net/http/httptest"
	"net/url"
	"strings"
	"sync"
	"testing"
	"time"

	"github.com/rcourtman/pulse-go-rewrite/internal/ai/memory"
	"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
	"github.com/rcourtman/pulse-go-rewrite/internal/alerts/eventlog"
	pulsemock "github.com/rcourtman/pulse-go-rewrite/internal/mock"
	"github.com/rcourtman/pulse-go-rewrite/internal/models"
	"github.com/rcourtman/pulse-go-rewrite/internal/monitoring"
	"github.com/rcourtman/pulse-go-rewrite/internal/notifications"
	"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/mock"
	testifymock "github.com/stretchr/testify/mock"
)

// Mock implementations for testing
type MockAlertManager struct {
	testifymock.Mock
}

func (m *MockAlertManager) GetConfig() alerts.AlertConfig {
	args := m.Called()
	return args.Get(0).(alerts.AlertConfig)
}

func TestExportIncident_UsesCanonicalEmptyCollections(t *testing.T) {
	openedAt := time.Unix(1700000000, 0).UTC()
	view := exportIncident(&memory.Incident{
		ID:              "inc-1",
		AlertIdentifier: "canonical:inc-1",
		AlertType:       "cpu",
		Level:           "warning",
		ResourceID:      "node/pve-1",
		ResourceName:    "pve-1",
		Status:          memory.IncidentStatusOpen,
		OpenedAt:        openedAt,
		Events: []memory.IncidentEvent{
			{
				ID:        "evt-1",
				Type:      memory.IncidentEventAlertFired,
				Timestamp: openedAt,
				Summary:   "Alert fired",
			},
		},
	})

	payload, err := json.Marshal(view)
	if err != nil {
		t.Fatalf("marshal exported incident: %v", err)
	}
	if !strings.Contains(string(payload), `"events":[`) {
		t.Fatalf("expected exported incident to retain events array, got %s", payload)
	}
	if !strings.Contains(string(payload), `"details":{}`) {
		t.Fatalf("expected exported incident event to retain empty details map, got %s", payload)
	}
}

func (m *MockAlertManager) UpdateConfig(cfg alerts.AlertConfig) {
	m.Called(cfg)
}

func (m *MockAlertManager) ApplyConfigUpdate(update []byte) (alerts.AlertConfig, error) {
	args := m.Called(update)
	return args.Get(0).(alerts.AlertConfig), args.Error(1)
}

func (m *MockAlertManager) GetActiveAlerts() []alerts.Alert {
	args := m.Called()
	return args.Get(0).([]alerts.Alert)
}

func (m *MockAlertManager) DiagnoseAlertDelivery(alertIdentifier string) (alerts.AlertDeliveryDiagnosis, bool) {
	args := m.Called(alertIdentifier)
	return args.Get(0).(alerts.AlertDeliveryDiagnosis), args.Bool(1)
}

func (m *MockAlertManager) DiagnoseActiveAlertDeliveries() []alerts.AlertDeliveryDiagnosis {
	args := m.Called()
	return args.Get(0).([]alerts.AlertDeliveryDiagnosis)
}

func (m *MockAlertManager) AlertEvents(filter eventlog.Filter) ([]eventlog.Event, error) {
	args := m.Called(filter)
	events, _ := args.Get(0).([]eventlog.Event)
	return events, args.Error(1)
}

func (m *MockAlertManager) NotifyExistingAlert(id string) {
	m.Called(id)
}

func (m *MockAlertManager) ClearAlertHistory() error {
	args := m.Called()
	return args.Error(0)
}

func (m *MockAlertManager) UnacknowledgeAlert(id string) error {
	args := m.Called(id)
	return args.Error(0)
}

func (m *MockAlertManager) AcknowledgeAlert(id, user string) error {
	args := m.Called(id, user)
	return args.Error(0)
}

func (m *MockAlertManager) SnoozeAlert(id, user string, until time.Time) error {
	return m.Called(id, user, until).Error(0)
}

func (m *MockAlertManager) UnsnoozeAlert(id, user string) error {
	return m.Called(id, user).Error(0)
}

func (m *MockAlertManager) ClearAlert(id string) bool {
	args := m.Called(id)
	return args.Bool(0)
}

func (m *MockAlertManager) GetAlertHistory(limit int) []alerts.Alert {
	args := m.Called(limit)
	return args.Get(0).([]alerts.Alert)
}

func (m *MockAlertManager) GetAlertHistorySince(since time.Time, limit int) []alerts.Alert {
	args := m.Called(since, limit)
	return args.Get(0).([]alerts.Alert)
}

type MockAlertMonitor struct {
	testifymock.Mock
}

func (m *MockAlertMonitor) GetAlertManager() AlertManager {
	args := m.Called()
	return args.Get(0).(AlertManager)
}

func (m *MockAlertMonitor) GetConfigPersistence() ConfigPersistence {
	args := m.Called()
	return args.Get(0).(ConfigPersistence)
}

func (m *MockAlertMonitor) GetIncidentStore() *memory.IncidentStore {
	args := m.Called()
	if store := args.Get(0); store != nil {
		return store.(*memory.IncidentStore)
	}
	return nil
}

func (m *MockAlertMonitor) GetNotificationManager() *notifications.NotificationManager {
	args := m.Called()
	return args.Get(0).(*notifications.NotificationManager)
}

func (m *MockAlertMonitor) DeadManStatus() monitoring.DeadManStatus {
	args := m.Called()
	if value := args.Get(0); value != nil {
		return value.(monitoring.DeadManStatus)
	}
	return monitoring.DeadManStatus{}
}

func (m *MockAlertMonitor) DeadManConfig() notifications.DeadManConfig {
	args := m.Called()
	if value := args.Get(0); value != nil {
		return value.(notifications.DeadManConfig)
	}
	return notifications.DeadManConfig{}
}

func (m *MockAlertMonitor) UpdateDeadManConfig(config notifications.DeadManConfig) error {
	return m.Called(config).Error(0)
}

func (m *MockAlertMonitor) SyncAlertState() {
	m.Called()
}

func (m *MockAlertMonitor) BuildFrontendState() models.StateFrontend {
	args := m.Called()
	return args.Get(0).(models.StateFrontend)
}

type MockConfigPersistence struct {
	testifymock.Mock
}

func (m *MockConfigPersistence) SaveAlertConfig(cfg alerts.AlertConfig) error {
	args := m.Called(cfg)
	return args.Error(0)
}

// Tests
func TestGetAlertConfig(t *testing.T) {
	mockMonitor := new(MockAlertMonitor)
	mockManager := new(MockAlertManager)
	mockMonitor.On("GetAlertManager").Return(mockManager)

	h := NewAlertHandlers(nil, mockMonitor, nil)

	cfg := alerts.AlertConfig{Enabled: true}
	mockManager.On("GetConfig").Return(cfg)

	req := httptest.NewRequest("GET", "/api/alerts/config", nil)
	w := httptest.NewRecorder()

	h.GetAlertConfig(w, req)

	assert.Equal(t, 200, w.Code)
	var resp alerts.AlertConfig
	_ = json.NewDecoder(w.Body).Decode(&resp)
	assert.True(t, resp.Enabled)
}

func TestUpdateAlertConfig(t *testing.T) {
	mockMonitor := new(MockAlertMonitor)
	mockManager := new(MockAlertManager)
	mockPersist := new(MockConfigPersistence)
	notificationMgr := notifications.NewNotificationManagerWithDataDir("", t.TempDir())
	defer notificationMgr.Stop()

	mockMonitor.On("GetAlertManager").Return(mockManager)
	mockMonitor.On("GetConfigPersistence").Return(mockPersist)
	mockMonitor.On("GetNotificationManager").Return(notificationMgr)

	h := NewAlertHandlers(nil, mockMonitor, nil)

	cfg := alerts.AlertConfig{Enabled: true, ActivationState: alerts.ActivationPending}
	mockManager.On("ApplyConfigUpdate", testifymock.Anything).Return(cfg, nil)
	mockManager.On("GetConfig").Return(cfg)
	mockPersist.On("SaveAlertConfig", testifymock.Anything).Return(nil)

	body, _ := json.Marshal(cfg)
	req := httptest.NewRequest("POST", "/api/alerts/config", bytes.NewReader(body))
	w := httptest.NewRecorder()

	h.UpdateAlertConfig(w, req)

	assert.Equal(t, 200, w.Code)
	assert.False(t, notificationMgr.IsEnabled())
}

// The Thresholds page has no control for flapping detection or alert TTL
// cleanup and never sends those keys; decoding its body into a zero config
// turned both off on every save.
func TestUpdateAlertConfig_KeepsStoredValuesForUnsentKeys(t *testing.T) {
	manager := alerts.NewManagerWithDataDir(t.TempDir())
	defer manager.Stop()
	stored := manager.GetConfig()
	stored.MaxAlertAgeDays = 7
	stored.MaxAcknowledgedAgeDays = 1
	stored.AutoAcknowledgeAfterHours = 24
	stored.FlappingEnabled = true
	manager.UpdateConfig(stored)

	mockMonitor := new(MockAlertMonitor)
	mockPersist := new(MockConfigPersistence)
	notificationMgr := notifications.NewNotificationManagerWithDataDir("", t.TempDir())
	defer notificationMgr.Stop()
	mockMonitor.On("GetAlertManager").Return(manager)
	mockMonitor.On("GetConfigPersistence").Return(mockPersist)
	mockMonitor.On("GetNotificationManager").Return(notificationMgr)
	var persisted alerts.AlertConfig
	mockPersist.On("SaveAlertConfig", testifymock.Anything).Run(func(args testifymock.Arguments) {
		persisted = args.Get(0).(alerts.AlertConfig)
	}).Return(nil)

	h := NewAlertHandlers(nil, mockMonitor, nil)

	body := `{"enabled":true,"guestDefaults":{"cpu":{"trigger":90,"clear":85}},"overrides":{},` +
		`"flapping":{"enabled":true,"threshold":5,"window":10,"suppressionTime":30,"minStability":0.8}}`
	req := httptest.NewRequest(http.MethodPut, "/api/alerts/config", strings.NewReader(body))
	w := httptest.NewRecorder()

	h.UpdateAlertConfig(w, req)

	if w.Code != http.StatusOK {
		t.Fatalf("status = %d, want 200: %s", w.Code, w.Body.String())
	}
	// Persistence normalizes the config it is handed in place, so it must get
	// a snapshot rather than the live config's maps.
	persisted.TimeThresholds["guest"] = 999
	if manager.GetConfig().TimeThresholds["guest"] == 999 {
		t.Fatal("the persisted config shares its maps with the live alert config")
	}
	persisted.TimeThresholds["guest"] = manager.GetConfig().TimeThresholds["guest"]
	for name, cfg := range map[string]alerts.AlertConfig{"live": manager.GetConfig(), "persisted": persisted} {
		if cfg.GuestDefaults.CPU == nil || cfg.GuestDefaults.CPU.Trigger != 90 {
			t.Fatalf("%s guest CPU default = %+v, want the sent trigger 90", name, cfg.GuestDefaults.CPU)
		}
		if !cfg.FlappingEnabled {
			t.Fatalf("thresholds save turned %s flapping detection off", name)
		}
		if cfg.MaxAlertAgeDays != 7 || cfg.MaxAcknowledgedAgeDays != 1 || cfg.AutoAcknowledgeAfterHours != 24 {
			t.Fatalf("%s TTL cleanup = %d/%d/%d after a thresholds save, want 7/1/24",
				name, cfg.MaxAlertAgeDays, cfg.MaxAcknowledgedAgeDays, cfg.AutoAcknowledgeAfterHours)
		}
	}
}

func TestUpdateAlertConfig_RejectsNonObjectBody(t *testing.T) {
	manager := alerts.NewManagerWithDataDir(t.TempDir())
	defer manager.Stop()
	before := manager.GetConfig()

	mockMonitor := new(MockAlertMonitor)
	mockMonitor.On("GetAlertManager").Return(manager)
	h := NewAlertHandlers(nil, mockMonitor, nil)

	for _, body := range []string{`null`, `[]`, `"config"`} {
		req := httptest.NewRequest(http.MethodPut, "/api/alerts/config", strings.NewReader(body))
		w := httptest.NewRecorder()

		h.UpdateAlertConfig(w, req)

		if w.Code != http.StatusBadRequest {
			t.Fatalf("status = %d for body %s, want 400", w.Code, body)
		}
	}
	if after := manager.GetConfig(); after.Enabled != before.Enabled || after.ActivationState != before.ActivationState {
		t.Fatalf("a rejected body changed the config: %+v", after)
	}
}

// Regression for #1341: persistence failures used to be silently swallowed
// (logged but reported as "saved successfully"), so a user setting a custom
// override would see the value stick in memory and revert on reload with
// no clue why. Surface the error so the frontend can show a real
// save-failed signal.
func TestUpdateAlertConfig_PersistenceFailureSurfacesAsError(t *testing.T) {
	mockMonitor := new(MockAlertMonitor)
	mockManager := new(MockAlertManager)
	mockPersist := new(MockConfigPersistence)
	notificationMgr := notifications.NewNotificationManagerWithDataDir("", t.TempDir())
	defer notificationMgr.Stop()

	mockMonitor.On("GetAlertManager").Return(mockManager)
	mockMonitor.On("GetConfigPersistence").Return(mockPersist)
	mockMonitor.On("GetNotificationManager").Return(notificationMgr)

	h := NewAlertHandlers(nil, mockMonitor, nil)

	cfg := alerts.AlertConfig{Enabled: true, ActivationState: alerts.ActivationPending}
	mockManager.On("ApplyConfigUpdate", testifymock.Anything).Return(cfg, nil)
	mockManager.On("GetConfig").Return(cfg)
	mockPersist.On("SaveAlertConfig", testifymock.Anything).Return(errors.New("permission denied"))

	body, _ := json.Marshal(cfg)
	req := httptest.NewRequest("POST", "/api/alerts/config", bytes.NewReader(body))
	w := httptest.NewRecorder()

	h.UpdateAlertConfig(w, req)

	if w.Code != http.StatusInternalServerError {
		t.Fatalf("status = %d, want 500 on persistence failure", w.Code)
	}
	if !strings.Contains(w.Body.String(), "permission denied") {
		t.Fatalf("response body should expose persistence error, got %q", w.Body.String())
	}
}

func TestActivateAlerts_EnablesNotificationManager(t *testing.T) {
	mockMonitor := new(MockAlertMonitor)
	mockManager := new(MockAlertManager)
	mockPersist := new(MockConfigPersistence)
	notificationMgr := notifications.NewNotificationManagerWithDataDir("", t.TempDir())
	defer notificationMgr.Stop()

	mockMonitor.On("GetAlertManager").Return(mockManager)
	mockMonitor.On("GetConfigPersistence").Return(mockPersist)
	mockMonitor.On("GetNotificationManager").Return(notificationMgr)

	initialConfig := alerts.AlertConfig{Enabled: true, ActivationState: alerts.ActivationPending}
	updatedConfig := alerts.AlertConfig{Enabled: true, ActivationState: alerts.ActivationActive}
	mockManager.On("GetConfig").Return(initialConfig).Once()
	mockManager.On("UpdateConfig", testifymock.AnythingOfType("config.AlertConfig")).Run(func(args testifymock.Arguments) {
		cfg := args.Get(0).(alerts.AlertConfig)
		if cfg.ActivationState != alerts.ActivationActive {
			t.Fatalf("expected activation state to be active, got %q", cfg.ActivationState)
		}
	}).Return()
	mockPersist.On("SaveAlertConfig", testifymock.Anything).Return(nil)
	mockManager.On("GetActiveAlerts").Return([]alerts.Alert{}).Maybe()
	mockManager.On("GetConfig").Return(updatedConfig)

	h := NewAlertHandlers(nil, mockMonitor, nil)

	req := httptest.NewRequest("POST", "/api/alerts/activate", nil)
	w := httptest.NewRecorder()

	h.ActivateAlerts(w, req)

	assert.Equal(t, 200, w.Code)
	assert.True(t, notificationMgr.IsEnabled())
}

func TestGetActiveAlerts(t *testing.T) {
	mockMonitor := new(MockAlertMonitor)
	mockManager := new(MockAlertManager)
	mockMonitor.On("GetAlertManager").Return(mockManager)

	h := NewAlertHandlers(nil, mockMonitor, nil)

	mockManager.On("GetActiveAlerts").Return([]alerts.Alert{{ID: "a1"}})

	req := httptest.NewRequest("GET", "/api/alerts/active", nil)
	w := httptest.NewRecorder()

	h.GetActiveAlerts(w, req)

	assert.Equal(t, 200, w.Code)
	var resp []alerts.Alert
	_ = json.NewDecoder(w.Body).Decode(&resp)
	assert.Len(t, resp, 1)
	assert.Equal(t, "a1", resp[0].ID)
}

func TestGetAlertDeliveryDiagnosis(t *testing.T) {
	mockMonitor := new(MockAlertMonitor)
	mockManager := new(MockAlertManager)
	mockMonitor.On("GetAlertManager").Return(mockManager)

	h := NewAlertHandlers(nil, mockMonitor, nil)

	expected := alerts.AlertDeliveryDiagnosis{
		AlertIdentifier:      "canonical:a1",
		AlertID:              "a1",
		TrackingKey:          "node/a1/cpu",
		Status:               alerts.AlertDeliveryStatusDeferred,
		Reason:               alerts.AlertDeliveryReasonQuietHours + ":performance",
		Message:              "Alert delivery is deferred by quiet-hours policy and will be replayed later.",
		NotificationsEnabled: true,
		ActivationState:      string(alerts.ActivationActive),
	}
	mockManager.On("DiagnoseAlertDelivery", "canonical:a1").Return(expected, true)

	req := httptest.NewRequest(http.MethodGet, "/api/alerts/delivery-diagnosis?alertIdentifier=canonical:a1", nil)
	w := httptest.NewRecorder()

	h.GetAlertDeliveryDiagnosis(w, req)

	assert.Equal(t, http.StatusOK, w.Code)
	var resp alerts.AlertDeliveryDiagnosis
	if err := json.NewDecoder(w.Body).Decode(&resp); err != nil {
		t.Fatalf("decode response: %v", err)
	}
	assert.Equal(t, expected.AlertIdentifier, resp.AlertIdentifier)
	assert.Equal(t, expected.Status, resp.Status)
	assert.Equal(t, expected.Reason, resp.Reason)
	assert.Equal(t, expected.TrackingKey, resp.TrackingKey)
}

func TestGetAlertEventsReturnsFilteredEvents(t *testing.T) {
	mockMonitor := new(MockAlertMonitor)
	mockManager := new(MockAlertManager)
	mockMonitor.On("GetAlertManager").Return(mockManager)

	expectedFilter := eventlog.Filter{
		AlertID: "a1",
		Types:   []string{eventlog.TypeNotificationSuppressed, eventlog.TypeResolved},
		Limit:   10,
	}
	mockManager.On("AlertEvents", expectedFilter).Return([]eventlog.Event{
		{ID: 2, Type: eventlog.TypeNotificationSuppressed, AlertID: "a1", Reason: "flapping"},
	}, nil)

	h := NewAlertHandlers(nil, mockMonitor, nil)

	req := httptest.NewRequest(http.MethodGet,
		"/api/alerts/events?alertIdentifier=a1&type=notification_suppressed,resolved&limit=10", nil)
	w := httptest.NewRecorder()

	h.GetAlertEvents(w, req)

	assert.Equal(t, http.StatusOK, w.Code)
	var resp []eventlog.Event
	if err := json.NewDecoder(w.Body).Decode(&resp); err != nil {
		t.Fatalf("decode response: %v", err)
	}
	assert.Len(t, resp, 1)
	assert.Equal(t, "flapping", resp[0].Reason)
}

func TestGetAlertEventsWritesEmptyArrayWithoutEventLog(t *testing.T) {
	mockMonitor := new(MockAlertMonitor)
	mockManager := new(MockAlertManager)
	mockMonitor.On("GetAlertManager").Return(mockManager)
	mockManager.On("AlertEvents", eventlog.Filter{}).Return([]eventlog.Event(nil), nil)

	h := NewAlertHandlers(nil, mockMonitor, nil)

	req := httptest.NewRequest(http.MethodGet, "/api/alerts/events", nil)
	w := httptest.NewRecorder()

	h.GetAlertEvents(w, req)

	assert.Equal(t, http.StatusOK, w.Code)
	assert.Equal(t, "[]", strings.TrimSpace(w.Body.String()))
}

func TestGetAlertEventsRejectsBadSince(t *testing.T) {
	h := NewAlertHandlers(nil, new(MockAlertMonitor), nil)

	req := httptest.NewRequest(http.MethodGet, "/api/alerts/events?since=yesterday", nil)
	w := httptest.NewRecorder()

	h.GetAlertEvents(w, req)

	assert.Equal(t, http.StatusBadRequest, w.Code)
}

func TestGetAlertDeliveryDiagnosisWithoutIdentifierReturnsAllActive(t *testing.T) {
	mockMonitor := new(MockAlertMonitor)
	mockManager := new(MockAlertManager)
	mockMonitor.On("GetAlertManager").Return(mockManager)

	expected := []alerts.AlertDeliveryDiagnosis{
		{
			AlertIdentifier: "a1",
			AlertID:         "a1",
			Status:          alerts.AlertDeliveryStatusWouldSend,
			Reason:          alerts.AlertDeliveryReasonReady,
		},
		{
			AlertIdentifier: "a2",
			AlertID:         "a2",
			Status:          alerts.AlertDeliveryStatusSuppressed,
			Reason:          alerts.AlertDeliveryReasonCooldown,
		},
	}
	mockManager.On("DiagnoseActiveAlertDeliveries").Return(expected)

	h := NewAlertHandlers(nil, mockMonitor, nil)

	req := httptest.NewRequest(http.MethodGet, "/api/alerts/delivery-diagnosis", nil)
	w := httptest.NewRecorder()

	h.GetAlertDeliveryDiagnosis(w, req)

	assert.Equal(t, http.StatusOK, w.Code)
	var resp []alerts.AlertDeliveryDiagnosis
	if err := json.NewDecoder(w.Body).Decode(&resp); err != nil {
		t.Fatalf("decode response: %v", err)
	}
	assert.Len(t, resp, 2)
	assert.Equal(t, expected[0].AlertIdentifier, resp[0].AlertIdentifier)
	assert.Equal(t, expected[1].Reason, resp[1].Reason)
}

func TestGetAlertDeliveryDiagnosisWithoutIdentifierWritesEmptyArray(t *testing.T) {
	mockMonitor := new(MockAlertMonitor)
	mockManager := new(MockAlertManager)
	mockMonitor.On("GetAlertManager").Return(mockManager)
	mockManager.On("DiagnoseActiveAlertDeliveries").Return([]alerts.AlertDeliveryDiagnosis(nil))

	h := NewAlertHandlers(nil, mockMonitor, nil)

	req := httptest.NewRequest(http.MethodGet, "/api/alerts/delivery-diagnosis", nil)
	w := httptest.NewRecorder()

	h.GetAlertDeliveryDiagnosis(w, req)

	assert.Equal(t, http.StatusOK, w.Code)
	assert.Equal(t, "[]", strings.TrimSpace(w.Body.String()))
}

func TestGetAlertDeliveryDiagnosisNotFound(t *testing.T) {
	mockMonitor := new(MockAlertMonitor)
	mockManager := new(MockAlertManager)
	mockMonitor.On("GetAlertManager").Return(mockManager)
	mockManager.On("DiagnoseAlertDelivery", "missing").Return(alerts.AlertDeliveryDiagnosis{}, false)

	h := NewAlertHandlers(nil, mockMonitor, nil)

	req := httptest.NewRequest(http.MethodGet, "/api/alerts/delivery-diagnosis?alertIdentifier=missing", nil)
	w := httptest.NewRecorder()

	h.GetAlertDeliveryDiagnosis(w, req)

	assert.Equal(t, http.StatusNotFound, w.Code)
}

func TestValidateAlertIdentifier(t *testing.T) {
	testCases := []struct {
		name  string
		id    string
		valid bool
	}{
		{name: "basic", id: "guest-powered-off-pve-101", valid: true},
		{name: "with spaces", id: "cluster one-node-101-cpu", valid: true},
		{name: "with slash and colon", id: "pve1:qemu/101-cpu", valid: true},
		{name: "empty", id: "", valid: false},
		{name: "too long", id: string(make([]byte, 501)), valid: false},
		{name: "control char", id: "bad\nvalue", valid: false},
		{name: "path traversal", id: "../etc/passwd", valid: false},
		{name: "path traversal middle", id: "pve/../secret", valid: false},
	}

	for _, tc := range testCases {
		if got := validateAlertIdentifier(tc.id); got != tc.valid {
			t.Errorf("validateAlertIdentifier(%s) = %v, want %v", tc.name, got, tc.valid)
		}
	}
}
func TestAlertHandlers_SetMonitor(t *testing.T) {
	mockMonitor1 := new(MockAlertMonitor)
	mockMonitor2 := new(MockAlertMonitor)
	h := NewAlertHandlers(nil, mockMonitor1, nil)
	assert.Equal(t, mockMonitor1, h.defaultMonitor)
	h.SetMonitor(mockMonitor2)
	assert.Equal(t, mockMonitor2, h.defaultMonitor)
}

func TestAlertHandlers_SetMonitorConcurrentAccess(t *testing.T) {
	mockMonitor1 := new(MockAlertMonitor)
	mockMonitor2 := new(MockAlertMonitor)
	h := NewAlertHandlers(nil, mockMonitor1, nil)

	var wg sync.WaitGroup
	for i := 0; i < 8; i++ {
		wg.Add(1)
		go func(worker int) {
			defer wg.Done()
			for j := 0; j < 500; j++ {
				if (worker+j)%2 == 0 {
					h.SetMonitor(mockMonitor1)
				} else {
					h.SetMonitor(mockMonitor2)
				}
				_ = h.getMonitor(context.Background())
			}
		}(i)
	}

	wg.Wait()
	assert.NotNil(t, h.getMonitor(context.Background()))
}

func TestGetAlertHistory(t *testing.T) {
	mockMonitor := new(MockAlertMonitor)
	mockManager := new(MockAlertManager)
	mockMonitor.On("GetAlertManager").Return(mockManager)
	h := NewAlertHandlers(nil, mockMonitor, nil)

	mockManager.On("GetAlertHistory", testifymock.Anything).Return([]alerts.Alert{{ID: "h1"}})

	req := httptest.NewRequest("GET", "/api/alerts/history?limit=10", nil)
	w := httptest.NewRecorder()
	h.GetAlertHistory(w, req)

	assert.Equal(t, 200, w.Code)
	var resp []alerts.Alert
	_ = json.NewDecoder(w.Body).Decode(&resp)
	assert.Len(t, resp, 1)
}

func TestGetAlertHistoryFiltersInformationalSeverity(t *testing.T) {
	mockMonitor := new(MockAlertMonitor)
	mockManager := new(MockAlertManager)
	mockMonitor.On("GetAlertManager").Return(mockManager)
	h := NewAlertHandlers(nil, mockMonitor, nil)

	history := []alerts.Alert{
		{ID: "info", Level: alerts.AlertLevelInfo},
		{ID: "warning", Level: alerts.AlertLevelWarning},
		{ID: "critical", Level: alerts.AlertLevelCritical},
	}
	mockManager.On("GetAlertHistory", 10).Return(history).Once()

	req := httptest.NewRequest(http.MethodGet, "/api/alerts/history?limit=10&severity=info", nil)
	w := httptest.NewRecorder()
	h.GetAlertHistory(w, req)

	assert.Equal(t, http.StatusOK, w.Code)
	var response []alerts.Alert
	assert.NoError(t, json.NewDecoder(w.Body).Decode(&response))
	if assert.Len(t, response, 1) {
		assert.Equal(t, "info", response[0].ID)
	}
	mockManager.AssertExpectations(t)
}

func TestClearAlertHistory(t *testing.T) {
	mockMonitor := new(MockAlertMonitor)
	mockManager := new(MockAlertManager)
	mockMonitor.On("GetAlertManager").Return(mockManager)
	h := NewAlertHandlers(nil, mockMonitor, nil)

	mockManager.On("ClearAlertHistory").Return(nil).Once()

	req := httptest.NewRequest("POST", "/api/alerts/history/clear", nil)
	w := httptest.NewRecorder()
	h.ClearAlertHistory(w, req)

	assert.Equal(t, 200, w.Code)
}

func TestSaveAlertIncidentNote(t *testing.T) {
	mockMonitor := new(MockAlertMonitor)
	mockStore := memory.NewIncidentStore(memory.IncidentStoreConfig{})
	mockMonitor.On("GetIncidentStore").Return(mockStore)
	h := NewAlertHandlers(nil, mockMonitor, nil)

	// Create an incident first so RecordNote has something to attach to
	alert := &alerts.Alert{ID: "a1", Type: "test"}
	mockStore.RecordAlertFired(alert)

	body := `{"alertIdentifier": "a1", "note": "test note", "user": "admin"}`
	req := httptest.NewRequest("POST", "/api/alerts/note", strings.NewReader(body))
	w := httptest.NewRecorder()
	h.SaveAlertIncidentNote(w, req)

	assert.Equal(t, 200, w.Code)
}

func TestGetAlertIncidentTimeline_ExportsCanonicalAlertIdentifier(t *testing.T) {
	mockMonitor := new(MockAlertMonitor)
	mockStore := memory.NewIncidentStore(memory.IncidentStoreConfig{})
	mockMonitor.On("GetIncidentStore").Return(mockStore)
	h := NewAlertHandlers(nil, mockMonitor, nil)

	alert := &alerts.Alert{
		ID:           "canonical:a1",
		Type:         "cpu",
		Level:        alerts.AlertLevelWarning,
		ResourceID:   "resource-1",
		ResourceName: "resource-1",
		Message:      "test",
		StartTime:    time.Now(),
	}
	mockStore.RecordAlertFired(alert)

	req := httptest.NewRequest("GET", "/api/alerts/incidents?alertIdentifier=canonical:a1", nil)
	w := httptest.NewRecorder()
	h.GetAlertIncidentTimeline(w, req)

	assert.Equal(t, 200, w.Code)
	var incident map[string]interface{}
	_ = json.NewDecoder(w.Body).Decode(&incident)
	assert.Equal(t, "canonical:a1", incident["alertIdentifier"])
}

func TestGetAlertIncidentTimeline_AcceptsCamelCaseAlertIdentifierQuery(t *testing.T) {
	mockMonitor := new(MockAlertMonitor)
	mockStore := memory.NewIncidentStore(memory.IncidentStoreConfig{})
	mockMonitor.On("GetIncidentStore").Return(mockStore)
	h := NewAlertHandlers(nil, mockMonitor, nil)

	alert := &alerts.Alert{
		ID:           "canonical:a1",
		Type:         "cpu",
		Level:        alerts.AlertLevelWarning,
		ResourceID:   "resource-1",
		ResourceName: "resource-1",
		Message:      "test",
		StartTime:    time.Now(),
	}
	mockStore.RecordAlertFired(alert)

	req := httptest.NewRequest("GET", "/api/alerts/incidents?alertIdentifier=canonical:a1", nil)
	w := httptest.NewRecorder()
	h.GetAlertIncidentTimeline(w, req)

	assert.Equal(t, 200, w.Code)
	var incident map[string]interface{}
	_ = json.NewDecoder(w.Body).Decode(&incident)
	assert.Equal(t, "canonical:a1", incident["alertIdentifier"])
}

func TestGetAlertIncidentTimeline_RepairsActiveAlertWithoutIncidentShell(t *testing.T) {
	mockMonitor := new(MockAlertMonitor)
	mockManager := new(MockAlertManager)
	incidentStore := memory.NewIncidentStore(memory.IncidentStoreConfig{})
	incidentStore.SetResourceTimelineStore(unifiedresources.NewMemoryStore())
	startedAt := time.Now().UTC().Add(-5 * time.Minute).Truncate(time.Second)
	alert := alerts.Alert{
		ID:           "active-without-timeline",
		Type:         "cpu",
		Level:        alerts.AlertLevelWarning,
		ResourceID:   "resource-active",
		ResourceName: "Active VM",
		StartTime:    startedAt,
		LastSeen:     startedAt.Add(4 * time.Minute),
	}
	mockMonitor.On("GetIncidentStore").Return(incidentStore)
	mockMonitor.On("GetAlertManager").Return(mockManager)
	mockManager.On("GetActiveAlerts").Return([]alerts.Alert{alert})
	h := NewAlertHandlers(nil, mockMonitor, nil)

	req := httptest.NewRequest("GET", "/api/alerts/incidents?alertIdentifier=active-without-timeline&started_at="+startedAt.Format(time.RFC3339), nil)
	w := httptest.NewRecorder()
	h.GetAlertIncidentTimeline(w, req)

	assert.Equal(t, http.StatusOK, w.Code)
	var incident incidentView
	assert.NoError(t, json.NewDecoder(w.Body).Decode(&incident))
	assert.Equal(t, memory.IncidentStatusOpen, incident.Status)
	assert.Len(t, incident.Events, 1)
	assert.Equal(t, memory.IncidentEventAlertFired, incident.Events[0].Type)
}

func TestGetAlertIncidentTimeline_RepairsResolvedHistoryOccurrence(t *testing.T) {
	mockMonitor := new(MockAlertMonitor)
	mockManager := new(MockAlertManager)
	incidentStore := memory.NewIncidentStore(memory.IncidentStoreConfig{})
	incidentStore.SetResourceTimelineStore(unifiedresources.NewMemoryStore())
	startedAt := time.Now().UTC().Add(-20 * time.Minute).Truncate(time.Second)
	resolvedAt := startedAt.Add(10 * time.Minute)
	alert := alerts.Alert{
		ID:           "history-without-timeline",
		Type:         "memory",
		Level:        alerts.AlertLevelCritical,
		ResourceID:   "resource-history",
		ResourceName: "Historical VM",
		StartTime:    startedAt,
		LastSeen:     resolvedAt,
	}
	mockMonitor.On("GetIncidentStore").Return(incidentStore)
	mockMonitor.On("GetAlertManager").Return(mockManager)
	mockManager.On("GetActiveAlerts").Return([]alerts.Alert{})
	mockManager.On("GetAlertHistory", 0).Return([]alerts.Alert{alert})
	h := NewAlertHandlers(nil, mockMonitor, nil)

	req := httptest.NewRequest("GET", "/api/alerts/incidents?alertIdentifier=history-without-timeline&started_at="+startedAt.Format(time.RFC3339), nil)
	w := httptest.NewRecorder()
	h.GetAlertIncidentTimeline(w, req)

	assert.Equal(t, http.StatusOK, w.Code)
	var incident incidentView
	assert.NoError(t, json.NewDecoder(w.Body).Decode(&incident))
	assert.Equal(t, memory.IncidentStatusResolved, incident.Status)
	assert.Len(t, incident.Events, 2)
	assert.Equal(t, memory.IncidentEventAlertFired, incident.Events[0].Type)
	assert.Equal(t, memory.IncidentEventAlertResolved, incident.Events[1].Type)
}

func TestGetAlertIncidentTimeline_ListExportsCanonicalAlertIdentifier(t *testing.T) {
	mockMonitor := new(MockAlertMonitor)
	mockStore := memory.NewIncidentStore(memory.IncidentStoreConfig{})
	mockMonitor.On("GetIncidentStore").Return(mockStore)
	h := NewAlertHandlers(nil, mockMonitor, nil)

	alert := &alerts.Alert{
		ID:           "canonical:a1",
		Type:         "cpu",
		Level:        alerts.AlertLevelWarning,
		ResourceID:   "resource-1",
		ResourceName: "resource-1",
		Message:      "test",
		StartTime:    time.Now(),
	}
	mockStore.RecordAlertFired(alert)

	req := httptest.NewRequest("GET", "/api/alerts/incidents?resource_id=resource-1", nil)
	w := httptest.NewRecorder()
	h.GetAlertIncidentTimeline(w, req)

	assert.Equal(t, 200, w.Code)
	var incidents []map[string]interface{}
	_ = json.NewDecoder(w.Body).Decode(&incidents)
	assert.Len(t, incidents, 1)
	assert.Equal(t, "canonical:a1", incidents[0]["alertIdentifier"])
}

func TestMockAlertIncidentTimelineSupportsAlertResourceAndNoteWorkflows(t *testing.T) {
	previous := pulsemock.IsMockEnabled()
	if err := pulsemock.SetEnabled(true); err != nil {
		t.Fatalf("enable mock fixtures: %v", err)
	}
	t.Cleanup(func() {
		if err := pulsemock.SetEnabled(previous); err != nil {
			t.Errorf("restore mock fixtures: %v", err)
		}
	})

	history := pulsemock.GetMockAlertHistory(1)
	if len(history) != 1 {
		t.Fatalf("mock alert history returned %d rows, want 1", len(history))
	}
	alert := history[0]
	if alert.LastSeen == nil {
		t.Fatalf("mock history row %q has no resolution timestamp", alert.ID)
	}

	mockMonitor := new(MockAlertMonitor)
	h := NewAlertHandlers(nil, mockMonitor, nil)
	alertURL := "/api/alerts/incidents?alertIdentifier=" + url.QueryEscape(alert.ID) + "&started_at=" + url.QueryEscape(alert.StartTime.Format(time.RFC3339))
	w := httptest.NewRecorder()
	h.GetAlertIncidentTimeline(w, httptest.NewRequest(http.MethodGet, alertURL, nil))
	if w.Code != http.StatusOK {
		t.Fatalf("mock alert timeline status = %d, body = %q", w.Code, w.Body.String())
	}
	var timeline incidentView
	assert.NoError(t, json.NewDecoder(w.Body).Decode(&timeline))
	assert.Equal(t, alert.ID, timeline.AlertIdentifier)
	assert.NotEmpty(t, timeline.Events)
	if timeline.Status != memory.IncidentStatusResolved || timeline.ClosedAt == nil || !timeline.ClosedAt.Equal(*alert.LastSeen) {
		t.Fatalf("mock timeline status = %q, closedAt = %v, history lastSeen = %v", timeline.Status, timeline.ClosedAt, alert.LastSeen)
	}

	resourceURL := "/api/alerts/incidents?resource_id=" + url.QueryEscape(alert.ResourceID) + "&limit=5"
	w = httptest.NewRecorder()
	h.GetAlertIncidentTimeline(w, httptest.NewRequest(http.MethodGet, resourceURL, nil))
	assert.Equal(t, http.StatusOK, w.Code)
	var resourceTimeline []incidentView
	assert.NoError(t, json.NewDecoder(w.Body).Decode(&resourceTimeline))
	assert.NotEmpty(t, resourceTimeline)

	noteBody := fmt.Sprintf(`{"alertIdentifier":%q,"note":"Checking the demo timeline","user":"operator"}`, alert.ID)
	w = httptest.NewRecorder()
	h.SaveAlertIncidentNote(w, httptest.NewRequest(http.MethodPost, "/api/alerts/incidents/note", strings.NewReader(noteBody)))
	assert.Equal(t, http.StatusOK, w.Code)

	w = httptest.NewRecorder()
	h.GetAlertIncidentTimeline(w, httptest.NewRequest(http.MethodGet, alertURL, nil))
	if w.Code != http.StatusOK {
		t.Fatalf("updated mock alert timeline status = %d, body = %q", w.Code, w.Body.String())
	}
	assert.NoError(t, json.NewDecoder(w.Body).Decode(&timeline))
	if len(timeline.Events) == 0 {
		t.Fatal("updated mock alert timeline has no events")
	}
	assert.Equal(t, memory.IncidentEventNote, timeline.Events[len(timeline.Events)-1].Type)
	assert.Equal(t, "Checking the demo timeline", timeline.Events[len(timeline.Events)-1].Details["note"])
}

func TestGetAlertIncidentTimeline_ProjectsCanonicalLifecycleAndRemediation(t *testing.T) {
	mockMonitor := new(MockAlertMonitor)
	incidentStore := memory.NewIncidentStore(memory.IncidentStoreConfig{})
	canonicalStore := unifiedresources.NewMemoryStore()
	incidentStore.SetResourceTimelineStore(canonicalStore)
	mockMonitor.On("GetIncidentStore").Return(incidentStore)
	h := NewAlertHandlers(nil, mockMonitor, nil)

	alertStartedAt := time.Now().UTC().Add(-20 * time.Minute).Truncate(time.Second)
	alert := &alerts.Alert{
		ID:           "canonical:projected-a1",
		Type:         "cpu",
		Level:        alerts.AlertLevelCritical,
		ResourceID:   "resource-1",
		ResourceName: "resource-1",
		Message:      "CPU high",
		StartTime:    alertStartedAt,
		Value:        95,
		Threshold:    80,
	}

	incidentStore.RecordAlertFired(alert)
	incidentStore.RecordAnalysis(alert.ID, "Pulse Patrol analysis completed", map[string]interface{}{"source": "test"})

	changes := []*unifiedresources.ResourceChange{
		unifiedresources.BuildAlertTimelineChange(alert.ResourceID, unifiedresources.ChangeAlertFired, alert.StartTime, "", unifiedresources.AlertTimelineChange{
			AlertIdentifier: alert.ID,
			AlertType:       alert.Type,
			AlertLevel:      string(alert.Level),
			AlertMessage:    alert.Message,
			AlertValue:      alert.Value,
			AlertThreshold:  alert.Threshold,
		}),
		unifiedresources.BuildAlertTimelineChange(alert.ResourceID, unifiedresources.ChangeAlertAcknowledged, alert.StartTime.Add(2*time.Minute), "operator", unifiedresources.AlertTimelineChange{
			AlertIdentifier: alert.ID,
			AlertType:       alert.Type,
			AlertLevel:      string(alert.Level),
			AlertMessage:    alert.Message,
		}),
		unifiedresources.BuildRunbookExecutionChange(alert.ResourceID, alert.ID, "agent:pulse-patrol", "rb-1", "Restart service", "resolved", true, "Recovered", nil),
		unifiedresources.BuildAlertTimelineChange(alert.ResourceID, unifiedresources.ChangeAlertResolved, alert.StartTime.Add(5*time.Minute), "", unifiedresources.AlertTimelineChange{
			AlertIdentifier: alert.ID,
			AlertType:       alert.Type,
			AlertLevel:      string(alert.Level),
			AlertMessage:    "CPU normalized",
		}),
	}
	for _, change := range changes {
		if change == nil {
			t.Fatal("expected canonical change to be built")
		}
		if err := canonicalStore.RecordChange(*change); err != nil {
			t.Fatalf("RecordChange(%s): %v", change.ID, err)
		}
	}

	req := httptest.NewRequest("GET", "/api/alerts/incidents?alertIdentifier=canonical:projected-a1", nil)
	w := httptest.NewRecorder()
	h.GetAlertIncidentTimeline(w, req)

	assert.Equal(t, 200, w.Code)
	var incident map[string]interface{}
	if err := json.NewDecoder(w.Body).Decode(&incident); err != nil {
		t.Fatalf("decode incident response: %v", err)
	}

	assert.Equal(t, "canonical:projected-a1", incident["alertIdentifier"])
	assert.Equal(t, string(memory.IncidentStatusResolved), incident["status"])
	assert.Equal(t, true, incident["acknowledged"])
	assert.Equal(t, "operator", incident["ackUser"])

	events, ok := incident["events"].([]interface{})
	if !ok {
		t.Fatalf("expected events array, got %#v", incident["events"])
	}

	foundAnalysis := false
	foundAck := false
	foundRunbook := false
	foundResolved := false
	for _, rawEvent := range events {
		event, ok := rawEvent.(map[string]interface{})
		if !ok {
			t.Fatalf("expected event object, got %#v", rawEvent)
		}
		switch event["type"] {
		case string(memory.IncidentEventAnalysis):
			foundAnalysis = true
		case string(memory.IncidentEventAlertAcknowledged):
			foundAck = true
		case string(memory.IncidentEventRunbook):
			foundRunbook = true
		case string(memory.IncidentEventAlertResolved):
			foundResolved = true
		}
	}

	assert.True(t, foundAnalysis, "expected analysis annotation in exported timeline")
	assert.True(t, foundAck, "expected canonical acknowledgement in exported timeline")
	assert.True(t, foundRunbook, "expected canonical runbook in exported timeline")
	assert.True(t, foundResolved, "expected canonical resolution in exported timeline")
}

func TestBulkAcknowledgeAlerts(t *testing.T) {
	mockMonitor := new(MockAlertMonitor)
	mockManager := new(MockAlertManager)
	mockMonitor.On("GetAlertManager").Return(mockManager)
	mockMonitor.On("SyncAlertState").Return()
	h := NewAlertHandlers(nil, mockMonitor, nil)

	mockManager.On("AcknowledgeAlert", "a1", testifymock.Anything).Return(nil)
	mockManager.On("AcknowledgeAlert", "a2", testifymock.Anything).Return(fmt.Errorf("error"))

	body := `{"alertIdentifiers": ["a1", "a2"], "user": "admin"}`
	req := httptest.NewRequest("POST", "/api/alerts/bulk/acknowledge", strings.NewReader(body))
	w := httptest.NewRecorder()
	h.BulkAcknowledgeAlerts(w, req)

	assert.Equal(t, 200, w.Code)
	var resp struct {
		Results []map[string]interface{} `json:"results"`
	}
	_ = json.NewDecoder(w.Body).Decode(&resp)
	assert.Len(t, resp.Results, 2)
}

func TestHandleAlerts(t *testing.T) {
	mockMonitor := new(MockAlertMonitor)
	mockManager := new(MockAlertManager)
	mockMonitor.On("GetAlertManager").Return(mockManager)
	mockMonitor.On("GetConfigPersistence").Return(new(MockConfigPersistence))
	mockMonitor.On("GetNotificationManager").Return(&notifications.NotificationManager{})
	mockMonitor.On("SyncAlertState").Return()
	h := NewAlertHandlers(nil, mockMonitor, nil)

	type route struct {
		method string
		path   string
		setup  func()
	}

	routes := []route{
		{"GET", "/api/alerts/active", func() { mockManager.On("GetActiveAlerts").Return([]alerts.Alert{}).Once() }},
		{"GET", "/api/alerts/delivery-diagnosis?alertIdentifier=a1", func() {
			mockManager.On("DiagnoseAlertDelivery", "a1").Return(alerts.AlertDeliveryDiagnosis{
				AlertIdentifier: "a1",
				Status:          alerts.AlertDeliveryStatusWouldSend,
				Reason:          alerts.AlertDeliveryReasonReady,
			}, true).Once()
		}},
		{"GET", "/api/alerts/history", func() {
			mockManager.On("GetAlertHistory", mock.MatchedBy(func(int) bool { return true })).Return([]alerts.Alert{}).Once()
		}},
		{"GET", "/api/alerts/incidents?alertIdentifier=a1", func() {
			mockMonitor.On("GetIncidentStore").Return(memory.NewIncidentStore(memory.IncidentStoreConfig{})).Once()
		}},
		{"POST", "/api/alerts/incidents/note", func() {
			store := memory.NewIncidentStore(memory.IncidentStoreConfig{})
			store.RecordAlertFired(&alerts.Alert{ID: "a1", Type: "test"})
			mockMonitor.On("GetIncidentStore").Return(store).Once()
		}},
		{"DELETE", "/api/alerts/history", func() { mockManager.On("ClearAlertHistory").Return(nil).Once() }},
		{"POST", "/api/alerts/bulk/acknowledge", func() {
			mockManager.On("AcknowledgeAlert", mock.Anything, mock.Anything).Return(nil)
			mockMonitor.On("SyncAlertState").Return()
		}},
		{"POST", "/api/alerts/bulk/clear", func() {
			mockManager.On("ClearAlert", mock.Anything).Return(true)
			mockMonitor.On("SyncAlertState").Return()
		}},
		{"POST", "/api/alerts/acknowledge", func() {
			mockManager.On("AcknowledgeAlert", "a1", testifymock.Anything).Return(nil).Once()
			mockMonitor.On("SyncAlertState").Return()
		}},
		{"POST", "/api/alerts/unacknowledge", func() {
			mockManager.On("UnacknowledgeAlert", "a1").Return(nil).Once()
			mockMonitor.On("SyncAlertState").Return()
		}},
		{"POST", "/api/alerts/clear", func() {
			mockManager.On("ClearAlert", "a1").Return(true).Once()
			mockMonitor.On("SyncAlertState").Return()
		}},
	}

	for _, rt := range routes {
		t.Run(rt.method+"_"+rt.path, func(t *testing.T) {
			rt.setup()
			var body []byte
			if rt.method == "POST" || rt.method == "PUT" || rt.method == "DELETE" {
				if strings.Contains(rt.path, "bulk") {
					body = []byte(`{"alertIdentifiers": ["a1"]}`)
				} else if strings.Contains(rt.path, "note") {
					body = []byte(`{"alertIdentifier": "a1", "note": "test"}`)
				} else {
					body = []byte(`{"alertIdentifier": "a1", "user": "admin"}`)
				}
			}
			req := httptest.NewRequest(rt.method, rt.path, bytes.NewReader(body))
			w := httptest.NewRecorder()
			h.HandleAlerts(w, req)
			assert.Equal(t, 200, w.Code)
		})
	}

	// Test NotFound
	req := httptest.NewRequest("GET", "/api/alerts/unknown", nil)
	w := httptest.NewRecorder()
	h.HandleAlerts(w, req)
	assert.Equal(t, 404, w.Code)
}

func TestBulkClearAlerts(t *testing.T) {
	mockMonitor := new(MockAlertMonitor)
	mockManager := new(MockAlertManager)
	mockMonitor.On("GetAlertManager").Return(mockManager)
	mockMonitor.On("SyncAlertState").Return()
	h := NewAlertHandlers(nil, mockMonitor, nil)

	mockManager.On("ClearAlert", "a1").Return(true)
	mockManager.On("ClearAlert", "a2").Return(false)

	body := `{"alertIdentifiers": ["a1", "a2"]}`
	req := httptest.NewRequest("POST", "/api/alerts/bulk/clear", strings.NewReader(body))
	w := httptest.NewRecorder()
	h.BulkClearAlerts(w, req)

	assert.Equal(t, 200, w.Code)
}

func TestAcknowledgeAlertByBody_Success(t *testing.T) {
	mockMonitor := new(MockAlertMonitor)
	mockManager := new(MockAlertManager)
	mockMonitor.On("GetAlertManager").Return(mockManager)
	mockMonitor.On("SyncAlertState").Return()
	h := NewAlertHandlers(nil, mockMonitor, nil)

	mockManager.On("AcknowledgeAlert", "a1", testifymock.Anything).Return(nil)

	body := `{"alertIdentifier": "a1", "user": "admin"}`
	req := httptest.NewRequest("POST", "/api/alerts/acknowledge", strings.NewReader(body))
	w := httptest.NewRecorder()
	h.AcknowledgeAlertByBody(w, req)

	assert.Equal(t, 200, w.Code)
}

func TestAcknowledgeAlertByBody_CanonicalIdentifierSuccess(t *testing.T) {
	mockMonitor := new(MockAlertMonitor)
	mockManager := new(MockAlertManager)
	mockMonitor.On("GetAlertManager").Return(mockManager)
	mockMonitor.On("SyncAlertState").Return()
	h := NewAlertHandlers(nil, mockMonitor, nil)

	mockManager.On("AcknowledgeAlert", "canonical:a1", testifymock.Anything).Return(nil)

	body := `{"alertIdentifier": "canonical:a1"}`
	req := httptest.NewRequest("POST", "/api/alerts/acknowledge", strings.NewReader(body))
	w := httptest.NewRecorder()
	h.AcknowledgeAlertByBody(w, req)

	assert.Equal(t, 200, w.Code)
}

func TestUnacknowledgeAlertByBody_Success(t *testing.T) {
	mockMonitor := new(MockAlertMonitor)
	mockManager := new(MockAlertManager)
	mockMonitor.On("GetAlertManager").Return(mockManager)
	mockMonitor.On("SyncAlertState").Return()
	h := NewAlertHandlers(nil, mockMonitor, nil)

	mockManager.On("UnacknowledgeAlert", "a1").Return(nil)

	body := `{"alertIdentifier": "a1"}`
	req := httptest.NewRequest("POST", "/api/alerts/unacknowledge", strings.NewReader(body))
	w := httptest.NewRecorder()
	h.UnacknowledgeAlertByBody(w, req)

	assert.Equal(t, 200, w.Code)
}

func TestUnacknowledgeAlertByBody_CanonicalIdentifierSuccess(t *testing.T) {
	mockMonitor := new(MockAlertMonitor)
	mockManager := new(MockAlertManager)
	mockMonitor.On("GetAlertManager").Return(mockManager)
	mockMonitor.On("SyncAlertState").Return()
	h := NewAlertHandlers(nil, mockMonitor, nil)

	mockManager.On("UnacknowledgeAlert", "canonical:a1").Return(nil)

	body := `{"alertIdentifier": "canonical:a1"}`
	req := httptest.NewRequest("POST", "/api/alerts/unacknowledge", strings.NewReader(body))
	w := httptest.NewRecorder()
	h.UnacknowledgeAlertByBody(w, req)

	assert.Equal(t, 200, w.Code)
}

func TestSnoozeAlertByBody_Success(t *testing.T) {
	mockMonitor := new(MockAlertMonitor)
	mockManager := new(MockAlertManager)
	mockMonitor.On("GetAlertManager").Return(mockManager)
	mockMonitor.On("SyncAlertState").Return()
	h := NewAlertHandlers(nil, mockMonitor, nil)
	until := time.Now().UTC().Add(2 * time.Hour).Truncate(time.Second)
	mockManager.On("SnoozeAlert", "canonical:a1", "[email protected]", until).Return(nil)
	req := httptest.NewRequest(http.MethodPost, "/api/alerts/snooze", strings.NewReader(fmt.Sprintf(`{"alertIdentifier":"canonical:a1","until":%q}`, until.Format(time.RFC3339))))
	w := httptest.NewRecorder()
	w.Header().Set("X-Authenticated-User", "[email protected]")
	h.SnoozeAlertByBody(w, req)
	assert.Equal(t, http.StatusOK, w.Code)
	mockManager.AssertExpectations(t)
}

func TestSnoozeAlertByBody_RejectsInvalidExpiry(t *testing.T) {
	h := NewAlertHandlers(nil, new(MockAlertMonitor), nil)
	req := httptest.NewRequest(http.MethodPost, "/api/alerts/snooze", strings.NewReader(`{"alertIdentifier":"a1","until":"not-a-time"}`))
	w := httptest.NewRecorder()
	h.SnoozeAlertByBody(w, req)
	assert.Equal(t, http.StatusBadRequest, w.Code)
}

func TestUnsnoozeAlertByBody_Success(t *testing.T) {
	mockMonitor := new(MockAlertMonitor)
	mockManager := new(MockAlertManager)
	mockMonitor.On("GetAlertManager").Return(mockManager)
	mockMonitor.On("SyncAlertState").Return()
	mockManager.On("UnsnoozeAlert", "a1", "[email protected]").Return(nil)
	h := NewAlertHandlers(nil, mockMonitor, nil)
	req := httptest.NewRequest(http.MethodPost, "/api/alerts/unsnooze", strings.NewReader(`{"alertIdentifier":"a1"}`))
	w := httptest.NewRecorder()
	w.Header().Set("X-Authenticated-User", "[email protected]")
	h.UnsnoozeAlertByBody(w, req)
	assert.Equal(t, http.StatusOK, w.Code)
	mockManager.AssertExpectations(t)
}

func TestClearAlertByBody_Success(t *testing.T) {
	mockMonitor := new(MockAlertMonitor)
	mockManager := new(MockAlertManager)
	mockMonitor.On("GetAlertManager").Return(mockManager)
	mockMonitor.On("SyncAlertState").Return()
	h := NewAlertHandlers(nil, mockMonitor, nil)

	mockManager.On("ClearAlert", "a1").Return(true)

	body := `{"alertIdentifier": "a1"}`
	req := httptest.NewRequest("POST", "/api/alerts/clear", strings.NewReader(body))
	w := httptest.NewRecorder()
	h.ClearAlertByBody(w, req)

	assert.Equal(t, 200, w.Code)
}

func TestClearAlertByBody_CanonicalIdentifierSuccess(t *testing.T) {
	mockMonitor := new(MockAlertMonitor)
	mockManager := new(MockAlertManager)
	mockMonitor.On("GetAlertManager").Return(mockManager)
	mockMonitor.On("SyncAlertState").Return()
	h := NewAlertHandlers(nil, mockMonitor, nil)

	mockManager.On("ClearAlert", "canonical:a1").Return(true)

	body := `{"alertIdentifier": "canonical:a1"}`
	req := httptest.NewRequest("POST", "/api/alerts/clear", strings.NewReader(body))
	w := httptest.NewRecorder()
	h.ClearAlertByBody(w, req)

	assert.Equal(t, 200, w.Code)
}

func TestAlertHandlers_ErrorCases(t *testing.T) {
	mockMonitor := new(MockAlertMonitor)
	mockManager := new(MockAlertManager)
	mockMonitor.On("GetAlertManager").Return(mockManager)
	h := NewAlertHandlers(nil, mockMonitor, nil)

	t.Run("AcknowledgeAlertByBody_InvalidJSON", func(t *testing.T) {
		req := httptest.NewRequest("POST", "/api/alerts/acknowledge", strings.NewReader(`{invalid`))
		w := httptest.NewRecorder()
		h.AcknowledgeAlertByBody(w, req)
		assert.Equal(t, 400, w.Code)
	})

	t.Run("AcknowledgeAlertByBody_MissingID", func(t *testing.T) {
		req := httptest.NewRequest("POST", "/api/alerts/acknowledge", strings.NewReader(`{"alertIdentifier": ""}`))
		w := httptest.NewRecorder()
		h.AcknowledgeAlertByBody(w, req)
		assert.Equal(t, 400, w.Code)
	})

	t.Run("AcknowledgeAlertByBody_InvalidID", func(t *testing.T) {
		req := httptest.NewRequest("POST", "/api/alerts/acknowledge", strings.NewReader(`{"alertIdentifier": "bad\x01"}`))
		w := httptest.NewRecorder()
		h.AcknowledgeAlertByBody(w, req)
		assert.Equal(t, 400, w.Code)
	})

	t.Run("AcknowledgeAlertByBody_ManagerError", func(t *testing.T) {
		mockManager.On("AcknowledgeAlert", "a1", testifymock.Anything).Return(fmt.Errorf("error")).Once()
		req := httptest.NewRequest("POST", "/api/alerts/acknowledge", strings.NewReader(`{"alertIdentifier": "a1", "user": "admin"}`))
		w := httptest.NewRecorder()
		h.AcknowledgeAlertByBody(w, req)
		assert.Equal(t, 404, w.Code)
	})

	t.Run("UnacknowledgeAlertByBody_MissingID", func(t *testing.T) {
		req := httptest.NewRequest("POST", "/api/alerts/unacknowledge", strings.NewReader(`{"alertIdentifier": ""}`))
		w := httptest.NewRecorder()
		h.UnacknowledgeAlertByBody(w, req)
		assert.Equal(t, 400, w.Code)
	})

	t.Run("UnacknowledgeAlertByBody_InvalidID", func(t *testing.T) {
		req := httptest.NewRequest("POST", "/api/alerts/unacknowledge", strings.NewReader(`{"alertIdentifier": "bad\x01"}`))
		w := httptest.NewRecorder()
		h.UnacknowledgeAlertByBody(w, req)
		assert.Equal(t, 400, w.Code)
	})

	t.Run("ClearAlertByBody_MissingID", func(t *testing.T) {
		req := httptest.NewRequest("POST", "/api/alerts/clear", strings.NewReader(`{"alertIdentifier": ""}`))
		w := httptest.NewRecorder()
		h.ClearAlertByBody(w, req)
		assert.Equal(t, 400, w.Code)
	})

	t.Run("ClearAlertByBody_InvalidID", func(t *testing.T) {
		req := httptest.NewRequest("POST", "/api/alerts/clear", strings.NewReader(`{"alertIdentifier": "bad\x01"}`))
		w := httptest.NewRecorder()
		h.ClearAlertByBody(w, req)
		assert.Equal(t, 400, w.Code)
	})

	t.Run("ClearAlertByBody_NotFound", func(t *testing.T) {
		mockManager.On("ClearAlert", "unknown").Return(false).Once()
		req := httptest.NewRequest("POST", "/api/alerts/clear", strings.NewReader(`{"alertIdentifier": "unknown"}`))
		w := httptest.NewRecorder()
		h.ClearAlertByBody(w, req)
		assert.Equal(t, 404, w.Code)
	})

	t.Run("BulkAcknowledgeAlerts_InvalidJSON", func(t *testing.T) {
		req := httptest.NewRequest("POST", "/api/alerts/bulk/acknowledge", strings.NewReader(`{invalid`))
		w := httptest.NewRecorder()
		h.BulkAcknowledgeAlerts(w, req)
		assert.Equal(t, 400, w.Code)
	})

	t.Run("BulkAcknowledgeAlerts_NoIDs", func(t *testing.T) {
		req := httptest.NewRequest("POST", "/api/alerts/bulk/acknowledge", strings.NewReader(`{"alertIdentifiers": []}`))
		w := httptest.NewRecorder()
		h.BulkAcknowledgeAlerts(w, req)
		assert.Equal(t, 400, w.Code)
	})

	t.Run("BulkAcknowledgeAlerts_CanonicalIdentifiers", func(t *testing.T) {
		mockManager.On("AcknowledgeAlert", "canonical:a1", testifymock.Anything).Return(nil).Once()
		mockMonitor.On("SyncAlertState").Return().Once()
		req := httptest.NewRequest("POST", "/api/alerts/bulk/acknowledge", strings.NewReader(`{"alertIdentifiers": ["canonical:a1"]}`))
		w := httptest.NewRecorder()
		h.BulkAcknowledgeAlerts(w, req)
		assert.Equal(t, 200, w.Code)
	})

	t.Run("UnacknowledgeAlertByBody_InvalidJSON", func(t *testing.T) {
		req := httptest.NewRequest("POST", "/api/alerts/unacknowledge", strings.NewReader(`{invalid`))
		w := httptest.NewRecorder()
		h.UnacknowledgeAlertByBody(w, req)
		assert.Equal(t, 400, w.Code)
	})

	t.Run("ClearAlertByBody_InvalidJSON", func(t *testing.T) {
		req := httptest.NewRequest("POST", "/api/alerts/clear", strings.NewReader(`{invalid`))
		w := httptest.NewRecorder()
		h.ClearAlertByBody(w, req)
		assert.Equal(t, 400, w.Code)
	})

	t.Run("SaveAlertIncidentNote_NoStore", func(t *testing.T) {
		mockMonitor2 := new(MockAlertMonitor)
		mockMonitor2.On("GetIncidentStore").Return(nil)
		h2 := NewAlertHandlers(nil, mockMonitor2, nil)
		req := httptest.NewRequest("POST", "/api/alerts/note", strings.NewReader(`{}`))
		w := httptest.NewRecorder()
		h2.SaveAlertIncidentNote(w, req)
		assert.Equal(t, 400, w.Code)
	})

	t.Run("SaveAlertIncidentNote_ValidBodyNoStore", func(t *testing.T) {
		mockMonitor2 := new(MockAlertMonitor)
		mockMonitor2.On("GetIncidentStore").Return(nil)
		h2 := NewAlertHandlers(nil, mockMonitor2, nil)
		req := httptest.NewRequest("POST", "/api/alerts/note", strings.NewReader(`{"alertIdentifier":"a1","note":"investigating"}`))
		w := httptest.NewRecorder()
		h2.SaveAlertIncidentNote(w, req)
		assert.Equal(t, 503, w.Code)
	})

	t.Run("SaveAlertIncidentNote_InvalidBody", func(t *testing.T) {
		mockStore := memory.NewIncidentStore(memory.IncidentStoreConfig{})
		mockMonitor.On("GetIncidentStore").Return(mockStore)
		req := httptest.NewRequest("POST", "/api/alerts/note", strings.NewReader(`{invalid`))
		w := httptest.NewRecorder()
		h.SaveAlertIncidentNote(w, req)
		assert.Equal(t, 400, w.Code)
	})

	t.Run("SaveAlertIncidentNote_MissingIDs", func(t *testing.T) {
		mockStore := memory.NewIncidentStore(memory.IncidentStoreConfig{})
		mockMonitor.On("GetIncidentStore").Return(mockStore)
		req := httptest.NewRequest("POST", "/api/alerts/note", strings.NewReader(`{"note": "test"}`))
		w := httptest.NewRecorder()
		h.SaveAlertIncidentNote(w, req)
		assert.Equal(t, 400, w.Code)
	})

	t.Run("SaveAlertIncidentNote_CanonicalIdentifier", func(t *testing.T) {
		mockMonitor3 := new(MockAlertMonitor)
		mockStore := memory.NewIncidentStore(memory.IncidentStoreConfig{})
		mockMonitor3.On("GetIncidentStore").Return(mockStore)
		h3 := NewAlertHandlers(nil, mockMonitor3, nil)
		incidentAlert := &alerts.Alert{
			ID:           "canonical:a1",
			Type:         "cpu",
			Level:        alerts.AlertLevelWarning,
			ResourceID:   "resource-1",
			ResourceName: "resource-1",
			Message:      "test",
			StartTime:    time.Now(),
		}
		mockStore.RecordAlertFired(incidentAlert)
		req := httptest.NewRequest("POST", "/api/alerts/note", strings.NewReader(`{"alertIdentifier": "canonical:a1", "note": "test"}`))
		w := httptest.NewRecorder()
		h3.SaveAlertIncidentNote(w, req)
		assert.Equal(t, 200, w.Code)
	})

	t.Run("SaveAlertIncidentNote_InvalidAlertIdentifier", func(t *testing.T) {
		mockStore := memory.NewIncidentStore(memory.IncidentStoreConfig{})
		mockMonitor.On("GetIncidentStore").Return(mockStore)
		req := httptest.NewRequest("POST", "/api/alerts/note", strings.NewReader(`{"alertIdentifier": "bad\x01", "note": "test"}`))
		w := httptest.NewRecorder()
		h.SaveAlertIncidentNote(w, req)
		assert.Equal(t, 400, w.Code)
	})

	t.Run("SaveAlertIncidentNote_MissingNote", func(t *testing.T) {
		mockStore := memory.NewIncidentStore(memory.IncidentStoreConfig{})
		mockMonitor.On("GetIncidentStore").Return(mockStore)
		req := httptest.NewRequest("POST", "/api/alerts/note", strings.NewReader(`{"alertIdentifier": "a1", "note": ""}`))
		w := httptest.NewRecorder()
		h.SaveAlertIncidentNote(w, req)
		assert.Equal(t, 400, w.Code)
	})

	t.Run("SaveAlertIncidentNote_NotFound", func(t *testing.T) {
		mockStore := memory.NewIncidentStore(memory.IncidentStoreConfig{})
		mockMonitor.On("GetIncidentStore").Return(mockStore)
		req := httptest.NewRequest("POST", "/api/alerts/note", strings.NewReader(`{"alertIdentifier": "none", "note": "test"}`))
		w := httptest.NewRecorder()
		h.SaveAlertIncidentNote(w, req)
		assert.Equal(t, 400, w.Code)
	})

	t.Run("ClearAlertHistory_Error", func(t *testing.T) {
		mockManager.On("ClearAlertHistory").Return(errors.New("failed")).Once()
		req := httptest.NewRequest("POST", "/api/alerts/history/clear", nil)
		w := httptest.NewRecorder()
		h.ClearAlertHistory(w, req)
		assert.Equal(t, 500, w.Code)
	})
}