Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

14.3 KB
Raw Blame
package alerting

import (
	"encoding/json"
	"errors"
	"net/http"
	"net/http/httptest"
	"strings"
	"testing"
	"time"

	"github.com/rcourtman/pulse-go-rewrite/internal/notifications"
	"github.com/stretchr/testify/mock"
)

func TestGetNotificationHealthReportsRetainedTerminalFailures(t *testing.T) {
	mockMonitor := new(MockNotificationMonitor)
	mockManager := new(MockNotificationManager)
	mockPersistence := new(MockNotificationConfigPersistence)
	mockMonitor.On("GetNotificationManager").Return(mockManager)
	mockMonitor.On("GetConfigPersistence").Return(mockPersistence)
	mockManager.On("GetQueueStats").Return(map[string]int{
		"pending": 4,
		"sending": 1,
		"sent":    9,
		"failed":  2,
		"dlq":     3,
	}, nil).Once()
	mockManager.On("GetTelemetryStats", mock.Anything).Return(notifications.TelemetryStats{
		Failures: 5,
		FailureClasses: notifications.NotificationFailureClassCounts{
			Authentication: 3,
			Connectivity:   2,
		},
	}, nil).Once()
	mockManager.On("GetEmailConfig").Return(notifications.EmailConfig{}).Once()
	mockManager.On("GetWebhooks").Return([]notifications.WebhookConfig{}).Once()
	mockPersistence.On("IsEncryptionEnabled").Return(true).Once()

	rec := httptest.NewRecorder()
	NewNotificationHandlers(nil, mockMonitor).GetNotificationHealth(
		rec,
		httptest.NewRequest(http.MethodGet, "/api/notifications/health", nil),
	)

	if rec.Code != http.StatusOK {
		t.Fatalf("status = %d, want %d", rec.Code, http.StatusOK)
	}
	var response struct {
		OverallHealthy bool `json:"overall_healthy"`
		Queue          struct {
			Healthy                      bool           `json:"healthy"`
			Status                       string         `json:"status"`
			AttentionRequired            int            `json:"attention_required"`
			ReasonCodes                  []string       `json:"reason_codes"`
			CompletedRetentionDays       int            `json:"completed_retention_days"`
			DeadLetterRetentionDays      int            `json:"dead_letter_retention_days"`
			CountsAreRetentionBounded    bool           `json:"counts_are_retention_bounded"`
			RetryAttemptsAffectHealth    bool           `json:"retry_attempts_affect_health"`
			TerminalFailuresAffectHealth bool           `json:"terminal_failures_affect_health"`
			FailureClasses7d             map[string]int `json:"failure_classes_7d"`
			FailureClassesAvailable      bool           `json:"failure_classes_available"`
			FailureClassWindowDays       int            `json:"failure_class_window_days"`
		} `json:"queue"`
	}
	if err := json.Unmarshal(rec.Body.Bytes(), &response); err != nil {
		t.Fatalf("decode health response: %v", err)
	}
	if response.OverallHealthy || response.Queue.Healthy || response.Queue.Status != "degraded" {
		t.Fatalf("health response = %#v, want degraded and unhealthy", response)
	}
	if response.Queue.AttentionRequired != 5 {
		t.Fatalf("attention_required = %d, want 5", response.Queue.AttentionRequired)
	}
	if len(response.Queue.ReasonCodes) != 2 ||
		response.Queue.ReasonCodes[0] != "retained_failed_deliveries" ||
		response.Queue.ReasonCodes[1] != "retained_dead_letter_deliveries" {
		t.Fatalf("reason_codes = %#v", response.Queue.ReasonCodes)
	}
	if response.Queue.CompletedRetentionDays != 7 ||
		response.Queue.DeadLetterRetentionDays != 30 ||
		!response.Queue.CountsAreRetentionBounded ||
		response.Queue.RetryAttemptsAffectHealth ||
		!response.Queue.TerminalFailuresAffectHealth {
		t.Fatalf("queue semantics = %#v", response.Queue)
	}
	if !response.Queue.FailureClassesAvailable ||
		response.Queue.FailureClassWindowDays != 7 ||
		response.Queue.FailureClasses7d["authentication"] != 3 ||
		response.Queue.FailureClasses7d["connectivity"] != 2 {
		t.Fatalf("failure classes = %#v", response.Queue)
	}
}

func TestGetNotificationHealthFailsClosedWhenQueueStatsAreUnavailable(t *testing.T) {
	mockMonitor := new(MockNotificationMonitor)
	mockManager := new(MockNotificationManager)
	mockPersistence := new(MockNotificationConfigPersistence)
	mockMonitor.On("GetNotificationManager").Return(mockManager)
	mockMonitor.On("GetConfigPersistence").Return(mockPersistence)
	mockManager.On("GetQueueStats").Return(nil, errors.New("database path /secret unavailable")).Once()
	mockManager.On("GetTelemetryStats", mock.Anything).Return(
		notifications.TelemetryStats{},
		errors.New("database unavailable"),
	).Once()
	mockManager.On("GetEmailConfig").Return(notifications.EmailConfig{}).Once()
	mockManager.On("GetWebhooks").Return([]notifications.WebhookConfig{}).Once()
	mockPersistence.On("IsEncryptionEnabled").Return(false).Once()

	rec := httptest.NewRecorder()
	NewNotificationHandlers(nil, mockMonitor).GetNotificationHealth(
		rec,
		httptest.NewRequest(http.MethodGet, "/api/notifications/health", nil),
	)

	if rec.Code != http.StatusOK {
		t.Fatalf("status = %d, want %d", rec.Code, http.StatusOK)
	}
	var response struct {
		OverallHealthy bool `json:"overall_healthy"`
		Queue          struct {
			Healthy     bool     `json:"healthy"`
			Status      string   `json:"status"`
			ReasonCodes []string `json:"reason_codes"`
		} `json:"queue"`
	}
	if err := json.Unmarshal(rec.Body.Bytes(), &response); err != nil {
		t.Fatalf("decode health response: %v", err)
	}
	if response.OverallHealthy || response.Queue.Healthy || response.Queue.Status != "unavailable" {
		t.Fatalf("health response = %#v, want unavailable and unhealthy", response)
	}
	if len(response.Queue.ReasonCodes) != 1 ||
		response.Queue.ReasonCodes[0] != "queue_stats_unavailable" {
		t.Fatalf("reason_codes = %#v", response.Queue.ReasonCodes)
	}
	if body := rec.Body.String(); body == "" ||
		containsAny(body, "database path", "/secret") {
		t.Fatalf("health response exposed internal queue error: %s", body)
	}
}

func TestGetDeliveryLogReturnsEntriesWithRedactedErrors(t *testing.T) {
	mockMonitor := new(MockNotificationMonitor)
	mockManager := new(MockNotificationManager)
	mockMonitor.On("GetNotificationManager").Return(mockManager)
	mockManager.On("GetDeliveryLog", mock.MatchedBy(func(since time.Time) bool {
		age := time.Since(since)
		return age >= 29*24*time.Hour && age <= 31*24*time.Hour
	}), 25).Return([]notifications.DeliveryLogEntry{
		{
			NotificationID: "webhook-1",
			Type:           "webhook",
			DestinationID:  "wh-ops",
			Outcome:        notifications.DeliveryOutcomeFailed,
			AlertIDs:       []string{"vm-offline-101"},
			AlertCount:     1,
			Attempts:       3,
			ErrorMessage:   "post https://hooks.example.test/notify?token=supersecret returned 401",
			FailureClass:   "authentication",
		},
		{
			NotificationID: "email-1",
			Type:           "email",
			Outcome:        notifications.DeliveryOutcomeSent,
			AlertIDs:       []string{"disk-critical-1"},
			AlertCount:     1,
			Attempts:       1,
			Success:        true,
		},
	}, nil).Once()

	rec := httptest.NewRecorder()
	NewNotificationHandlers(nil, mockMonitor).GetDeliveryLog(
		rec,
		httptest.NewRequest(http.MethodGet, "/api/notifications/delivery-log?limit=25", nil),
	)

	if rec.Code != http.StatusOK {
		t.Fatalf("status = %d, want %d", rec.Code, http.StatusOK)
	}
	var response struct {
		Entries []struct {
			NotificationID string   `json:"notificationId"`
			Outcome        string   `json:"outcome"`
			DestinationID  string   `json:"destinationId"`
			AlertIDs       []string `json:"alertIds"`
			ErrorMessage   string   `json:"errorMessage"`
			FailureClass   string   `json:"failureClass"`
		} `json:"entries"`
		WindowDays                 int  `json:"window_days"`
		CompletedRetentionDays     int  `json:"completed_retention_days"`
		DeadLetterRetentionDays    int  `json:"dead_letter_retention_days"`
		EntriesAreRetentionBounded bool `json:"entries_are_retention_bounded"`
	}
	if err := json.Unmarshal(rec.Body.Bytes(), &response); err != nil {
		t.Fatalf("decode delivery log response: %v", err)
	}
	if len(response.Entries) != 2 {
		t.Fatalf("entries = %#v, want 2", response.Entries)
	}
	if response.Entries[0].Outcome != "failed" ||
		response.Entries[0].DestinationID != "wh-ops" ||
		response.Entries[0].FailureClass != "authentication" ||
		len(response.Entries[0].AlertIDs) != 1 ||
		response.Entries[0].AlertIDs[0] != "vm-offline-101" {
		t.Fatalf("failed entry = %#v", response.Entries[0])
	}
	if response.Entries[0].ErrorMessage != "post https://hooks.example.test/notify?token=REDACTED returned 401" {
		t.Fatalf("error message not redacted: %q", response.Entries[0].ErrorMessage)
	}
	if response.WindowDays != 30 ||
		response.CompletedRetentionDays != 7 ||
		response.DeadLetterRetentionDays != 30 ||
		!response.EntriesAreRetentionBounded {
		t.Fatalf("retention context = %#v", response)
	}
}

func TestGetDeliveryLogRejectsInvalidLimit(t *testing.T) {
	mockMonitor := new(MockNotificationMonitor)
	mockManager := new(MockNotificationManager)
	mockMonitor.On("GetNotificationManager").Return(mockManager)

	for _, limit := range []string{"abc", "-1", "0"} {
		rec := httptest.NewRecorder()
		NewNotificationHandlers(nil, mockMonitor).GetDeliveryLog(
			rec,
			httptest.NewRequest(http.MethodGet, "/api/notifications/delivery-log?limit="+limit, nil),
		)
		if rec.Code != http.StatusBadRequest {
			t.Fatalf("limit=%q status = %d, want %d", limit, rec.Code, http.StatusBadRequest)
		}
	}
	mockManager.AssertNotCalled(t, "GetDeliveryLog", mock.Anything, mock.Anything)
}

func TestGetDeliveryLogFailsClosedWhenQueueUnavailable(t *testing.T) {
	mockMonitor := new(MockNotificationMonitor)
	mockManager := new(MockNotificationManager)
	mockMonitor.On("GetNotificationManager").Return(mockManager)
	mockManager.On("GetDeliveryLog", mock.Anything, 0).Return(
		nil, errors.New("database path /secret unavailable"),
	).Once()

	rec := httptest.NewRecorder()
	NewNotificationHandlers(nil, mockMonitor).GetDeliveryLog(
		rec,
		httptest.NewRequest(http.MethodGet, "/api/notifications/delivery-log", nil),
	)

	if rec.Code != http.StatusServiceUnavailable {
		t.Fatalf("status = %d, want %d", rec.Code, http.StatusServiceUnavailable)
	}
	if body := rec.Body.String(); containsAny(body, "database path", "/secret") {
		t.Fatalf("delivery log response exposed internal queue error: %s", body)
	}
}

// A test send bypasses the activation gate real alerts honor, so its result
// must say when real delivery is paused instead of reporting bare success.
func TestTestNotificationReportsPausedDeliveryInResult(t *testing.T) {
	mockMonitor := new(MockNotificationMonitor)
	mockManager := new(MockNotificationManager)
	mockMonitor.On("GetNotificationManager").Return(mockManager)
	mockManager.On("SendTestNotification", "email").Return(nil).Once()
	mockManager.On("IsEnabled").Return(false).Once()

	rec := httptest.NewRecorder()
	NewNotificationHandlers(nil, mockMonitor).TestNotification(
		rec,
		httptest.NewRequest(
			http.MethodPost,
			"/api/notifications/test",
			strings.NewReader(`{"method":"email"}`),
		),
	)

	if rec.Code != http.StatusOK {
		t.Fatalf("status = %d, want %d", rec.Code, http.StatusOK)
	}
	var response struct {
		Status         string `json:"status"`
		Message        string `json:"message"`
		DeliveryPaused bool   `json:"deliveryPaused"`
	}
	if err := json.Unmarshal(rec.Body.Bytes(), &response); err != nil {
		t.Fatalf("decode test notification response: %v", err)
	}
	if response.Status != "success" || !response.DeliveryPaused {
		t.Fatalf("response = %#v, want success with deliveryPaused", response)
	}
	if !strings.Contains(response.Message, "paused") {
		t.Fatalf("message %q does not warn about paused delivery", response.Message)
	}
}

func TestTestNotificationOmitsPausedFlagWhenDeliveryActive(t *testing.T) {
	mockMonitor := new(MockNotificationMonitor)
	mockManager := new(MockNotificationManager)
	mockMonitor.On("GetNotificationManager").Return(mockManager)
	mockManager.On("SendTestNotification", "email").Return(nil).Once()
	mockManager.On("IsEnabled").Return(true).Once()

	rec := httptest.NewRecorder()
	NewNotificationHandlers(nil, mockMonitor).TestNotification(
		rec,
		httptest.NewRequest(
			http.MethodPost,
			"/api/notifications/test",
			strings.NewReader(`{"method":"email"}`),
		),
	)

	if rec.Code != http.StatusOK {
		t.Fatalf("status = %d, want %d", rec.Code, http.StatusOK)
	}
	if strings.Contains(rec.Body.String(), "deliveryPaused") {
		t.Fatalf("active delivery response carries paused flag: %s", rec.Body.String())
	}
}

func containsAny(value string, needles ...string) bool {
	for _, needle := range needles {
		if needle != "" && strings.Contains(value, needle) {
			return true
		}
	}
	return false
}

// These are response-boundary checks, not just URL-helper checks.
func TestGetDeliveryLogDiagnosticContext(t *testing.T) {
	for _, tc := range []struct{ name, input, want string }{
		{"plain", "connection refused", "connection refused"},
		{"apprise CLI", `process queued apprise notification "apprise-1" (alert): apprise CLI send failed: execute apprise CLI: executable file not found in PATH`, `process queued apprise notification "apprise-1" (alert): apprise CLI send failed: execute apprise CLI: executable file not found in PATH`},
		{"encoded repeated query", "Post https://example.test/hook?%74oken=secret&token=secret&channel=ops failed", "Post https://example.test/hook?%74oken=REDACTED&token=REDACTED&channel=ops failed"},
		{"userinfo", "Post https://user:[email protected]/hook: timeout", "Post https://[email protected]/hook: timeout"},
		{"malformed", "Post https://user:[email protected]/%zz: timeout", "[invalid webhook URL]"},
	} {
		t.Run(tc.name, func(t *testing.T) {
			manager := new(MockNotificationManager)
			monitor := new(MockNotificationMonitor)
			monitor.On("GetNotificationManager").Return(manager).Once()
			manager.On("GetDeliveryLog", mock.Anything, 0).Return([]notifications.DeliveryLogEntry{
				{NotificationID: "attempt-1", ErrorMessage: tc.input, FailureClass: "transport"},
			}, nil).Once()
			rec := httptest.NewRecorder()
			NewNotificationHandlers(nil, monitor).GetDeliveryLog(rec, httptest.NewRequest(http.MethodGet, "/api/notifications/delivery-log", nil))
			if rec.Code != http.StatusOK {
				t.Fatalf("status = %d", rec.Code)
			}
			var response struct {
				Entries []notifications.DeliveryLogEntry `json:"entries"`
			}
			if err := json.Unmarshal(rec.Body.Bytes(), &response); err != nil {
				t.Fatal(err)
			}
			if len(response.Entries) != 1 || response.Entries[0].ErrorMessage != tc.want ||
				response.Entries[0].NotificationID != "attempt-1" || response.Entries[0].FailureClass != "transport" {
				t.Fatalf("unexpected delivery projection: %#v", response)
			}
			manager.AssertExpectations(t)
			monitor.AssertExpectations(t)
		})
	}
}