Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

19.9 KB
Raw Blame
package notifications

import (
	"context"
	"encoding/json"
	"io"
	"net/http"
	"strings"
	"testing"
	"time"

	"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
)

const legacyCamelAlertIdentifierField = "alertId"

func TestSendResolvedAppriseCLI(t *testing.T) {
	manager := NewNotificationManager("")
	defer manager.Stop()

	var called bool
	manager.appriseExec = func(ctx context.Context, args []string) ([]byte, error) {
		called = true
		if len(args) == 0 || args[len(args)-1] != "target-1" {
			t.Fatalf("expected target to be passed, got %v", args)
		}
		if !containsArg(args, "-t") || !containsArg(args, "-b") {
			t.Fatalf("expected title/body args, got %v", args)
		}
		return nil, nil
	}

	config := AppriseConfig{
		Enabled:        true,
		Mode:           AppriseModeCLI,
		Targets:        []string{"target-1"},
		CLIPath:        "apprise",
		TimeoutSeconds: 1,
	}

	alertList := []*alerts.Alert{
		{
			ID:           "a1",
			Type:         "cpu",
			Level:        alerts.AlertLevelWarning,
			ResourceID:   "r1",
			ResourceName: "db-1",
			Message:      "cpu high",
			Value:        91,
			Threshold:    80,
			StartTime:    time.Now().Add(-time.Minute),
		},
	}

	if err := manager.sendResolvedApprise(config, alertList, time.Now()); err != nil {
		t.Fatalf("sendResolvedApprise error: %v", err)
	}
	if !called {
		t.Fatalf("expected apprise exec to be called")
	}
}

func TestSendGroupedWebhookGeneric(t *testing.T) {
	var gotMethod string
	var gotBody []byte

	server := newIPv4HTTPServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		gotMethod = r.Method
		body, _ := io.ReadAll(r.Body)
		gotBody = body
		w.WriteHeader(http.StatusOK)
	}))
	defer server.Close()

	manager := NewNotificationManager("")
	defer manager.Stop()
	manager.webhookClient = server.Client()
	if err := manager.UpdateAllowedPrivateCIDRs("127.0.0.1/32"); err != nil {
		t.Fatalf("allowlist: %v", err)
	}

	alertsList := []*alerts.Alert{
		{
			ID:           "a1",
			Type:         "cpu",
			Level:        alerts.AlertLevelCritical,
			ResourceID:   "r1",
			ResourceName: "db-1",
			Message:      "cpu critical",
			Value:        99,
			Threshold:    90,
			StartTime:    time.Now().Add(-2 * time.Minute),
		},
		{
			ID:           "a2",
			Type:         "mem",
			Level:        alerts.AlertLevelWarning,
			ResourceID:   "r2",
			ResourceName: "cache-1",
			Message:      "memory high",
			Value:        85,
			Threshold:    80,
			StartTime:    time.Now().Add(-time.Minute),
		},
	}

	webhook := WebhookConfig{
		Name:    "generic",
		URL:     server.URL + "/hook",
		Enabled: true,
	}

	if err := manager.sendGroupedWebhook(webhook, alertsList); err != nil {
		t.Fatalf("sendGroupedWebhook error: %v", err)
	}
	if gotMethod != http.MethodPost {
		t.Fatalf("expected POST, got %s", gotMethod)
	}

	var payload map[string]any
	if err := json.Unmarshal(gotBody, &payload); err != nil {
		t.Fatalf("unmarshal payload: %v", err)
	}
	if grouped, ok := payload["grouped"].(bool); !ok || !grouped {
		t.Fatalf("expected grouped payload, got %v", payload["grouped"])
	}
	if count, ok := payload["count"].(float64); !ok || int(count) != len(alertsList) {
		t.Fatalf("expected count %d, got %v", len(alertsList), payload["count"])
	}
}

func TestSendGroupedWebhookNtfyIncludesLiveMetadataAndEveryAlert(t *testing.T) {
	var gotTitle string
	var gotPriority string
	var gotTags string
	var gotAuthorization string
	var gotBody string

	server := newIPv4HTTPServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		gotTitle = r.Header.Get("Title")
		gotPriority = r.Header.Get("Priority")
		gotTags = r.Header.Get("Tags")
		gotAuthorization = r.Header.Get("Authorization")
		body, _ := io.ReadAll(r.Body)
		gotBody = string(body)
		w.WriteHeader(http.StatusOK)
	}))
	defer server.Close()

	manager := NewNotificationManager("https://pulse.example")
	defer manager.Stop()
	manager.webhookClient = server.Client()
	if err := manager.UpdateAllowedPrivateCIDRs("127.0.0.1/32"); err != nil {
		t.Fatalf("allowlist: %v", err)
	}

	alertList := []*alerts.Alert{
		{
			ID:           "alert-critical",
			Type:         "cpu",
			Level:        alerts.AlertLevelCritical,
			ResourceID:   "vm-101",
			ResourceName: "database",
			Node:         "node-a",
			Message:      "CPU is critical",
			Value:        99,
			Threshold:    90,
			StartTime:    time.Now().Add(-2 * time.Minute),
		},
		{
			ID:           "alert-warning",
			Type:         "memory",
			Level:        alerts.AlertLevelWarning,
			ResourceID:   "vm-102",
			ResourceName: "cache",
			Node:         "node-b",
			Message:      "Memory is high",
			Value:        88,
			Threshold:    80,
			StartTime:    time.Now().Add(-time.Minute),
		},
	}

	webhook := WebhookConfig{
		Name:    "ntfy-live",
		URL:     server.URL + "/topic",
		Enabled: true,
		Service: "ntfy",
		Headers: map[string]string{"Authorization": "Bearer secret"},
	}

	if err := manager.sendGroupedWebhook(webhook, alertList); err != nil {
		t.Fatalf("sendGroupedWebhook error: %v", err)
	}

	if gotTitle != "CRITICAL: 2 alerts" {
		t.Fatalf("Title = %q, want %q", gotTitle, "CRITICAL: 2 alerts")
	}
	if gotPriority != "urgent" {
		t.Fatalf("Priority = %q, want urgent", gotPriority)
	}
	if !strings.Contains(gotTags, "rotating_light") || !strings.Contains(gotTags, "pulse") {
		t.Fatalf("Tags = %q, want critical Pulse tags", gotTags)
	}
	if gotAuthorization != "Bearer secret" {
		t.Fatalf("Authorization = %q, want configured header preserved", gotAuthorization)
	}
	for _, resourceName := range []string{"database", "cache"} {
		if !strings.Contains(gotBody, resourceName) {
			t.Fatalf("ntfy body %q does not include alert resource %q", gotBody, resourceName)
		}
	}
	if !strings.Contains(gotBody, "Memory is high") {
		t.Fatalf("ntfy body %q does not include the second alert message", gotBody)
	}
}

func TestSendGroupedWebhookDiscordEscapesSpecialCharacters(t *testing.T) {
	var gotBody []byte

	server := newIPv4HTTPServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		body, _ := io.ReadAll(r.Body)
		gotBody = body
		w.WriteHeader(http.StatusOK)
	}))
	defer server.Close()

	manager := NewNotificationManager("https://pulse.example")
	defer manager.Stop()
	manager.webhookClient = server.Client()
	if err := manager.UpdateAllowedPrivateCIDRs("127.0.0.1/32"); err != nil {
		t.Fatalf("failed to allowlist localhost: %v", err)
	}

	alert := &alerts.Alert{
		ID:           "alert-1",
		Type:         "cpu",
		Level:        alerts.AlertLevelWarning,
		ResourceID:   "vm-100",
		ResourceName: `db "primary"`,
		Node:         `node\01`,
		Message:      "CPU spike on \"db\"\nPath C:\\temp",
		Value:        92.3,
		Threshold:    80,
		StartTime:    time.Now().Add(-5 * time.Minute),
	}

	webhook := WebhookConfig{
		Name:    "discord-test",
		URL:     server.URL,
		Enabled: true,
		Service: "discord",
	}

	if err := manager.sendGroupedWebhook(webhook, []*alerts.Alert{alert}); err != nil {
		t.Fatalf("sendGroupedWebhook error: %v", err)
	}

	var payload map[string]interface{}
	if err := json.Unmarshal(gotBody, &payload); err != nil {
		t.Fatalf("unmarshal payload: %v", err)
	}

	embeds, ok := payload["embeds"].([]interface{})
	if !ok || len(embeds) == 0 {
		t.Fatalf("expected embeds in payload, got: %v", payload)
	}

	embed, ok := embeds[0].(map[string]interface{})
	if !ok {
		t.Fatalf("expected embed object, got %T", embeds[0])
	}

	description, _ := embed["description"].(string)
	if description != alert.Message {
		t.Fatalf("expected description %q, got %q", alert.Message, description)
	}

	fields, ok := embed["fields"].([]interface{})
	if !ok || len(fields) == 0 {
		t.Fatalf("expected fields in embed, got: %v", embed)
	}

	resourceField, ok := fields[0].(map[string]interface{})
	if !ok {
		t.Fatalf("expected first field object, got %T", fields[0])
	}
	if resourceField["value"] != alert.ResourceName {
		t.Fatalf("expected resource field %q, got %v", alert.ResourceName, resourceField["value"])
	}
}

func TestSendResolvedWebhookHTTP(t *testing.T) {
	var gotBody []byte
	server := newIPv4HTTPServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		body, _ := io.ReadAll(r.Body)
		gotBody = body
		w.WriteHeader(http.StatusNoContent)
	}))
	defer server.Close()

	manager := NewNotificationManager("")
	defer manager.Stop()
	manager.webhookClient = server.Client()
	if err := manager.UpdateAllowedPrivateCIDRs("127.0.0.1/32"); err != nil {
		t.Fatalf("allowlist: %v", err)
	}

	alertList := []*alerts.Alert{
		{
			ID:           "a1",
			Type:         "disk",
			Level:        alerts.AlertLevelWarning,
			ResourceID:   "r1",
			ResourceName: "storage-1",
			Message:      "disk high",
			Value:        92,
			Threshold:    90,
			StartTime:    time.Now().Add(-time.Minute),
		},
	}

	webhook := WebhookConfig{
		Name:    "resolved",
		URL:     server.URL + "/resolved",
		Enabled: true,
	}

	if err := manager.sendResolvedWebhook(webhook, alertList, time.Now()); err != nil {
		t.Fatalf("sendResolvedWebhook error: %v", err)
	}

	var payload map[string]any
	if err := json.Unmarshal(gotBody, &payload); err != nil {
		t.Fatalf("unmarshal payload: %v", err)
	}
	if payload["event"] != "resolved" {
		t.Fatalf("expected event resolved, got %v", payload["event"])
	}
	if payload["alertIdentifier"] != "a1" {
		t.Fatalf("expected alertIdentifier a1, got %v", payload["alertIdentifier"])
	}
	if _, ok := payload[legacyCamelAlertIdentifierField]; ok {
		t.Fatalf(
			"did not expect legacy %s in resolved payload, got %v",
			legacyCamelAlertIdentifierField,
			payload[legacyCamelAlertIdentifierField],
		)
	}
}

func TestSendResolvedWebhookIncludesMention(t *testing.T) {
	cases := []struct {
		service  string
		assertFn func(t *testing.T, payload map[string]any)
	}{
		{
			service: "discord",
			assertFn: func(t *testing.T, payload map[string]any) {
				if payload["content"] != "@everyone" {
					t.Fatalf("expected discord resolved mention in content, got %v", payload["content"])
				}
			},
		},
		{
			service: "slack",
			assertFn: func(t *testing.T, payload map[string]any) {
				text, _ := payload["text"].(string)
				if !strings.HasPrefix(text, "@everyone ") {
					t.Fatalf("expected slack resolved text to start with mention, got %q", text)
				}
			},
		},
		{
			service: "mattermost",
			assertFn: func(t *testing.T, payload map[string]any) {
				text, _ := payload["text"].(string)
				if !strings.HasPrefix(text, "@everyone\n\n") {
					t.Fatalf("expected mattermost resolved text to start with mention, got %q", text)
				}
			},
		},
	}

	for _, tc := range cases {
		t.Run(tc.service, func(t *testing.T) {
			var gotBody []byte
			server := newIPv4HTTPServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
				body, _ := io.ReadAll(r.Body)
				gotBody = body
				w.WriteHeader(http.StatusNoContent)
			}))
			defer server.Close()

			manager := NewNotificationManager("")
			defer manager.Stop()
			manager.webhookClient = server.Client()
			if err := manager.UpdateAllowedPrivateCIDRs("127.0.0.1/32"); err != nil {
				t.Fatalf("allowlist: %v", err)
			}

			alertList := []*alerts.Alert{{
				ID:           "a1",
				Type:         "cpu",
				Level:        alerts.AlertLevelWarning,
				ResourceID:   "vm100",
				ResourceName: "web-server",
				Node:         "pve1",
				Message:      "CPU usage high",
				Value:        95,
				Threshold:    90,
				StartTime:    time.Now().Add(-5 * time.Minute),
			}}

			webhook := WebhookConfig{
				Name:    tc.service + "-test",
				URL:     server.URL + "/" + tc.service,
				Enabled: true,
				Service: tc.service,
				Mention: "@everyone",
			}

			if err := manager.sendResolvedWebhook(webhook, alertList, time.Now()); err != nil {
				t.Fatalf("sendResolvedWebhook error: %v", err)
			}

			var payload map[string]any
			if err := json.Unmarshal(gotBody, &payload); err != nil {
				t.Fatalf("unmarshal payload: %v (body=%s)", err, gotBody)
			}
			tc.assertFn(t, payload)
		})
	}
}

func TestSendResolvedWebhookNtfySingleAlert(t *testing.T) {
	var gotMethod string
	var gotBody string
	var gotHeaders http.Header

	server := newIPv4HTTPServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		gotMethod = r.Method
		gotHeaders = r.Header.Clone()
		body, _ := io.ReadAll(r.Body)
		gotBody = string(body)
		w.WriteHeader(http.StatusAccepted)
	}))
	defer server.Close()

	manager := NewNotificationManager("")
	defer manager.Stop()
	manager.webhookClient = server.Client()
	if err := manager.UpdateAllowedPrivateCIDRs("127.0.0.1/32"); err != nil {
		t.Fatalf("allowlist: %v", err)
	}

	resolvedAt := time.Date(2026, 2, 11, 12, 30, 0, 0, time.UTC)
	alertList := []*alerts.Alert{
		{
			ID:           "a1",
			Type:         "cpu",
			Level:        alerts.AlertLevelWarning,
			ResourceID:   "r1",
			ResourceName: "db-1",
			Node:         "node-1",
			Message:      "cpu high",
			Value:        91,
			Threshold:    80,
			StartTime:    time.Now().Add(-time.Minute),
		},
	}

	webhook := WebhookConfig{
		Name:    "resolved-ntfy",
		URL:     server.URL + "/ntfy",
		Enabled: true,
		Service: "ntfy",
		Headers: map[string]string{
			"X-Static":   "keep-me",
			"X-Template": "{{.ResourceName}}",
		},
	}

	if err := manager.sendResolvedWebhookNtfy(webhook, alertList, resolvedAt); err != nil {
		t.Fatalf("sendResolvedWebhookNtfy error: %v", err)
	}

	if gotMethod != http.MethodPost {
		t.Fatalf("expected POST method by default, got %q", gotMethod)
	}
	if gotHeaders.Get("Content-Type") != "text/plain" {
		t.Fatalf("expected text/plain content type, got %q", gotHeaders.Get("Content-Type"))
	}
	if gotHeaders.Get("Title") != "RESOLVED: db-1" {
		t.Fatalf("expected resolved title, got %q", gotHeaders.Get("Title"))
	}
	if gotHeaders.Get("X-Static") != "keep-me" {
		t.Fatalf("expected static header to be forwarded")
	}
	if gotHeaders.Get("X-Template") != "" {
		t.Fatalf("expected templated header value to be skipped, got %q", gotHeaders.Get("X-Template"))
	}
	if !strings.Contains(gotBody, "Resolved: db-1 on node-1 is now healthy") {
		t.Fatalf("unexpected body: %q", gotBody)
	}
}

func TestSendResolvedWebhookNtfyMultipleAlertsHTTPError(t *testing.T) {
	var gotMethod string
	var gotTitle string
	var gotBody string

	server := newIPv4HTTPServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		gotMethod = r.Method
		gotTitle = r.Header.Get("Title")
		body, _ := io.ReadAll(r.Body)
		gotBody = string(body)
		w.WriteHeader(http.StatusServiceUnavailable)
		_, _ = w.Write([]byte("temporarily unavailable"))
	}))
	defer server.Close()

	manager := NewNotificationManager("")
	defer manager.Stop()
	manager.webhookClient = server.Client()
	if err := manager.UpdateAllowedPrivateCIDRs("127.0.0.1/32"); err != nil {
		t.Fatalf("allowlist: %v", err)
	}

	webhook := WebhookConfig{
		Name:    "resolved-ntfy",
		URL:     server.URL + "/ntfy",
		Method:  http.MethodPut,
		Enabled: true,
		Service: "ntfy",
	}

	alertList := []*alerts.Alert{
		{ResourceName: "db-1", Node: "node-1"},
		{ResourceName: "db-2", Node: "node-2"},
	}

	err := manager.sendResolvedWebhookNtfy(webhook, alertList, time.Date(2026, 2, 11, 12, 0, 0, 0, time.UTC))
	if err == nil {
		t.Fatalf("expected non-2xx response error")
	}
	if !strings.Contains(err.Error(), "ntfy webhook returned HTTP 503") {
		t.Fatalf("expected HTTP status in error, got %v", err)
	}
	if strings.Contains(err.Error(), "temporarily unavailable") || !strings.Contains(err.Error(), "response body withheld") {
		t.Fatalf("provider text must be withheld while retaining the HTTP verdict: %v", err)
	}

	if gotMethod != http.MethodPut {
		t.Fatalf("expected configured method PUT, got %q", gotMethod)
	}
	if gotTitle != "RESOLVED: 2 alerts" {
		t.Fatalf("expected grouped resolved title, got %q", gotTitle)
	}
	if !strings.Contains(gotBody, "2 alerts resolved") {
		t.Fatalf("expected grouped resolved body, got %q", gotBody)
	}
}

func TestSendResolvedWebhookNtfyValidationAndRateLimit(t *testing.T) {
	manager := NewNotificationManager("")
	defer manager.Stop()

	validAlert := []*alerts.Alert{{ResourceName: "vm-1", Node: "node-1"}}

	err := manager.sendResolvedWebhookNtfy(
		WebhookConfig{Name: "invalid-url", URL: "://bad-url", Enabled: true, Service: "ntfy"},
		validAlert,
		time.Now(),
	)
	if err == nil || !strings.Contains(err.Error(), "webhook URL validation failed") {
		t.Fatalf("expected URL validation failure, got %v", err)
	}

	rateLimitedURL := "http://127.0.0.1:1/ntfy"
	if err := manager.UpdateAllowedPrivateCIDRs("127.0.0.1/32"); err != nil {
		t.Fatalf("allowlist: %v", err)
	}
	manager.webhookRateLimits[rateLimitedURL] = &webhookRateLimit{
		lastSent:  time.Now(),
		sentCount: WebhookRateLimitMax,
	}

	err = manager.sendResolvedWebhookNtfy(
		WebhookConfig{Name: "rate-limited", URL: rateLimitedURL, Enabled: true, Service: "ntfy"},
		validAlert,
		time.Now(),
	)
	if err == nil || !strings.Contains(err.Error(), "rate limit exceeded") {
		t.Fatalf("expected rate limit failure, got %v", err)
	}
}

func TestSendResolvedNotificationsDirectDispatchesEnabledTargets(t *testing.T) {
	receipts := make(chan isolatedSMTPReceipt, 1)
	stubSMTPDialSuccess(t, nil, func(receipt isolatedSMTPReceipt) { receipts <- receipt })
	origSpawn := spawnAsync
	spawnAsync = func(f func()) { f() }
	t.Cleanup(func() { spawnAsync = origSpawn })

	webhookHits := 0
	server := newIPv4HTTPServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		webhookHits++
		w.WriteHeader(http.StatusOK)
	}))
	defer server.Close()

	manager := NewNotificationManager("")
	defer manager.Stop()
	manager.webhookClient = server.Client()
	if err := manager.UpdateAllowedPrivateCIDRs("127.0.0.1/32"); err != nil {
		t.Fatalf("allowlist: %v", err)
	}

	manager.emailManager = NewEnhancedEmailManager(EmailProviderConfig{
		EmailConfig: EmailConfig{
			From:     "[email protected]",
			To:       []string{"[email protected]"},
			SMTPHost: "invalid.localhost.test",
			SMTPPort: 25,
		},
		MaxRetries: 0,
		RetryDelay: 0,
		RateLimit:  0,
	})

	appriseCalled := false
	manager.appriseExec = func(ctx context.Context, args []string) ([]byte, error) {
		appriseCalled = true
		return nil, nil
	}

	alertList := []*alerts.Alert{
		{
			ID:           "a1",
			Type:         "cpu",
			Level:        alerts.AlertLevelWarning,
			ResourceID:   "r1",
			ResourceName: "db-1",
			Node:         "node-1",
			Message:      "cpu warning",
			Value:        87,
			Threshold:    80,
			StartTime:    time.Now().Add(-2 * time.Minute),
		},
	}

	manager.sendResolvedNotificationsDirect(
		EmailConfig{
			Enabled:  true,
			From:     "[email protected]",
			To:       []string{"[email protected]"},
			SMTPHost: "invalid.localhost.test",
			SMTPPort: 25,
		},
		[]WebhookConfig{
			{Name: "enabled", URL: server.URL + "/ok", Enabled: true, Service: "ntfy"},
			{Name: "disabled", URL: server.URL + "/skip", Enabled: false, Service: "ntfy"},
		},
		AppriseConfig{
			Enabled:        true,
			Mode:           AppriseModeCLI,
			Targets:        []string{"discord://token"},
			CLIPath:        "apprise",
			TimeoutSeconds: 1,
		},
		alertList,
		time.Now(),
	)

	if webhookHits != 1 {
		t.Fatalf("expected exactly one resolved webhook request, got %d", webhookHits)
	}
	if !appriseCalled {
		t.Fatalf("expected resolved Apprise branch to be called")
	}
	receipt := awaitIsolatedSMTPReceipt(t, receipts)
	if receipt.from != "FROM:<[email protected]>" || len(receipt.recipients) != 1 || receipt.recipients[0] != "TO:<[email protected]>" {
		t.Fatalf("resolved email did not use its submitted SMTP envelope: %+v", receipt)
	}
}

func TestSendResolvedNotificationsDirectNoopForEmptyAlerts(t *testing.T) {
	origSpawn := spawnAsync
	spawned := 0
	spawnAsync = func(f func()) { spawned++ }
	t.Cleanup(func() { spawnAsync = origSpawn })

	manager := NewNotificationManager("")
	defer manager.Stop()

	manager.sendResolvedNotificationsDirect(
		EmailConfig{Enabled: true},
		[]WebhookConfig{{Enabled: true}},
		AppriseConfig{Enabled: true},
		nil,
		time.Now(),
	)

	if spawned != 0 {
		t.Fatalf("expected no async dispatch for empty resolved alert batch, got %d", spawned)
	}
}

func containsArg(args []string, value string) bool {
	for _, arg := range args {
		if strings.TrimSpace(arg) == value {
			return true
		}
	}
	return false
}

func TestNotificationManagerStop_Idempotent(t *testing.T) {
	manager := NewNotificationManager("")

	manager.Stop()
	manager.Stop()
}