Skip to content

SigNoz is an open-source, OpenTelemetry-native observability platform for your team and their AI agents. Get logs, metrics, and traces in one tool with features like APM, distributed tracing, log management, infra monitoring, etc. Combined with SigNoz MCP and a native AI teammate (in SigNoz Cloud) it helps you build more resilient apps.

10.2 KB
Raw Blame
package alertmanagerserver

import (
	"context"
	"log/slog"
	"net/http"
	"testing"
	"time"

	"github.com/SigNoz/signoz/pkg/types/alertmanagertypes/alertmanagertypestest"
	"github.com/prometheus/alertmanager/dispatch"

	"github.com/SigNoz/signoz/pkg/alertmanager/nfmanager"
	"github.com/SigNoz/signoz/pkg/alertmanager/nfmanager/nfroutingstore/nfroutingstoretest"
	"github.com/SigNoz/signoz/pkg/alertmanager/nfmanager/rulebasednotification"
	"github.com/SigNoz/signoz/pkg/instrumentation/instrumentationtest"
	"github.com/SigNoz/signoz/pkg/types"
	"github.com/SigNoz/signoz/pkg/types/alertmanagertypes"
	"github.com/SigNoz/signoz/pkg/valuer"

	"github.com/go-openapi/strfmt"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/common/model"

	"github.com/stretchr/testify/mock"
	"github.com/stretchr/testify/require"
)

func TestEndToEndAlertManagerFlow(t *testing.T) {
	ctx := context.Background()
	providerSettings := instrumentationtest.New().ToProviderSettings()

	store := nfroutingstoretest.NewMockSQLRouteStore()
	store.MatchExpectationsInOrder(false)
	notificationManager, err := rulebasednotification.New(ctx, providerSettings, nfmanager.Config{}, store)
	require.NoError(t, err)
	orgID := "test-org"

	routes := []*alertmanagertypes.RoutePolicy{
		{
			Identifiable: types.Identifiable{
				ID: valuer.GenerateUUID(),
			},
			Expression:     `ruleId = "high-cpu-usage" AND severity = "critical"`,
			ExpressionKind: alertmanagertypes.RuleBasedExpression,
			Name:           "high-cpu-usage",
			Description:    "High CPU critical alerts to webhook",
			Enabled:        true,
			OrgID:          orgID,
			Channels:       []string{"webhook"},
		},
		{
			Identifiable: types.Identifiable{
				ID: valuer.GenerateUUID(),
			},
			Expression:     `ruleId = "high-cpu-usage" AND severity = "warning"`,
			ExpressionKind: alertmanagertypes.RuleBasedExpression,
			Name:           "high-cpu-usage",
			Description:    "High CPU warning alerts to webhook",
			Enabled:        true,
			OrgID:          orgID,
			Channels:       []string{"webhook"},
		},
	}

	store.ExpectCreateBatch(routes)
	err = notificationManager.CreateRoutePolicies(ctx, orgID, routes)
	require.NoError(t, err)

	for range routes {
		ruleID := "high-cpu-usage"
		store.ExpectGetAllByName(orgID, ruleID, routes)
		store.ExpectGetAllByName(orgID, ruleID, routes)
	}

	notifConfig := alertmanagertypes.NotificationConfig{
		NotificationGroup: map[model.LabelName]struct{}{
			model.LabelName("cluster"):  {},
			model.LabelName("instance"): {},
		},
		Renotify: alertmanagertypes.ReNotificationConfig{
			RenotifyInterval: 5 * time.Minute,
		},
		UsePolicy: false,
	}

	err = notificationManager.SetNotificationConfig(orgID, "high-cpu-usage", &notifConfig)
	require.NoError(t, err)

	activeSchedule := &alertmanagertypes.Schedule{
		Timezone:  "UTC",
		StartTime: time.Now().Add(-time.Hour),
		EndTime:   time.Now().Add(time.Hour),
	}
	// mwRuleIDAndScope: only critical high-cpu-usage alerts.
	mwRuleIDAndScope := valuer.GenerateUUID()
	// mwRuleIDOnly: all high-cpu-usage alerts regardless of severity.
	mwRuleIDOnly := valuer.GenerateUUID()
	// mwScopeOnly: all critical alerts regardless of rule ID.
	mwScopeOnly := valuer.GenerateUUID()

	maintenanceStore := alertmanagertypestest.NewMockMaintenanceStore(t)
	maintenanceStore.On("ListPlannedMaintenance", mock.Anything, orgID).Return(
		[]*alertmanagertypes.PlannedMaintenance{
			{ID: mwRuleIDAndScope, Schedule: activeSchedule, RuleIDs: []string{"high-cpu-usage"}, Scope: `severity = "critical"`},
			{ID: mwRuleIDOnly, Schedule: activeSchedule, RuleIDs: []string{"high-cpu-usage"}},
			{ID: mwScopeOnly, Schedule: activeSchedule, Scope: `severity = "critical"`},
		}, nil,
	)

	srvCfg := NewConfig()
	stateStore := alertmanagertypestest.NewStateStore()
	registry := prometheus.NewRegistry()
	logger := slog.New(slog.DiscardHandler)
	server, err := New(context.Background(), logger, registry, srvCfg, orgID, stateStore, notificationManager, maintenanceStore)
	require.NoError(t, err)
	amConfig, err := alertmanagertypes.NewDefaultConfig(srvCfg.Global, srvCfg.Route, orgID)
	require.NoError(t, err)
	err = server.SetConfig(ctx, amConfig)
	require.NoError(t, err)

	// Create test alerts
	now := time.Now()
	testAlerts := []*alertmanagertypes.PostableAlert{
		{
			Alert: alertmanagertypes.AlertModel{
				Labels: map[string]string{
					"ruleId":    "high-cpu-usage",
					"severity":  "critical",
					"cluster":   "prod-cluster",
					"instance":  "server-01",
					"alertname": "HighCPUUsage",
				},
			},
			Annotations: map[string]string{
				"summary":     "High CPU usage detected",
				"description": "CPU usage is above 90% for 5 minutes",
			},
			StartsAt: strfmt.DateTime(now.Add(-5 * time.Minute)),
			EndsAt:   strfmt.DateTime(time.Time{}), // Active alert
		},
		{
			Alert: alertmanagertypes.AlertModel{
				Labels: map[string]string{
					"ruleId":    "high-cpu-usage",
					"severity":  "warning",
					"cluster":   "prod-cluster",
					"instance":  "server-02",
					"alertname": "HighCPUUsage",
				},
			},
			Annotations: map[string]string{
				"summary":     "Moderate CPU usage detected",
				"description": "CPU usage is above 70% for 10 minutes",
			},
			StartsAt: strfmt.DateTime(now.Add(-10 * time.Minute)),
			EndsAt:   strfmt.DateTime(time.Time{}), // Active alert
		},
		{
			Alert: alertmanagertypes.AlertModel{
				Labels: map[string]string{
					"ruleId":    "high-cpu-usage",
					"severity":  "critical",
					"cluster":   "prod-cluster",
					"instance":  "server-03",
					"alertname": "HighCPUUsage",
				},
			},
			Annotations: map[string]string{
				"summary":     "High CPU usage detected on server-03",
				"description": "CPU usage is above 95% for 3 minutes",
			},
			StartsAt: strfmt.DateTime(now.Add(-3 * time.Minute)),
			EndsAt:   strfmt.DateTime(time.Time{}), // Active alert
		},
		{
			Alert: alertmanagertypes.AlertModel{
				Labels: map[string]string{
					"ruleId":    "other-rule",
					"alertname": "OtherAlert",
				},
			},
			StartsAt: strfmt.DateTime(now.Add(-time.Minute)),
			EndsAt:   strfmt.DateTime(time.Time{}), // Active alert
		},
	}

	err = server.PutAlerts(ctx, testAlerts)
	require.NoError(t, err)

	time.Sleep(2 * time.Second)

	t.Run("verify_alerts_processed", func(t *testing.T) {
		dummyRequest, err := http.NewRequest(http.MethodGet, "/alerts", nil)
		require.NoError(t, err)

		params, err := alertmanagertypes.NewGettableAlertsParams(dummyRequest)
		require.NoError(t, err)
		alerts, err := server.GetAlerts(context.Background(), params)
		require.NoError(t, err)
		require.Len(t, alerts, 4, "Expected 4 active alerts")

		for _, alert := range alerts {
			if alert.Labels["ruleId"] != "high-cpu-usage" {
				continue
			}
			require.NotEmpty(t, alert.Labels["severity"])
			require.Contains(t, []string{"critical", "warning"}, alert.Labels["severity"])
			require.Equal(t, "prod-cluster", alert.Labels["cluster"])
			require.NotEmpty(t, alert.Labels["instance"])
		}

		criticalAlerts := 0
		warningAlerts := 0
		for _, alert := range alerts {
			switch alert.Labels["severity"] {
			case "critical":
				criticalAlerts++
			case "warning":
				warningAlerts++
			}
		}
		require.Equal(t, 2, criticalAlerts, "Expected 2 critical alerts")
		require.Equal(t, 1, warningAlerts, "Expected 1 warning alert")
	})

	t.Run("verify_notification_routing", func(t *testing.T) {

		notifConfig, err := notificationManager.GetNotificationConfig(orgID, "high-cpu-usage")
		require.NoError(t, err)
		require.NotNil(t, notifConfig)
		require.Equal(t, 5*time.Minute, notifConfig.Renotify.RenotifyInterval)
		require.Contains(t, notifConfig.NotificationGroup, model.LabelName("ruleId"))
		require.Contains(t, notifConfig.NotificationGroup, model.LabelName("cluster"))
		require.Contains(t, notifConfig.NotificationGroup, model.LabelName("instance"))
	})

	t.Run("verify_alert_groups_and_stages", func(t *testing.T) {
		time.Sleep(2 * time.Second)

		alertGroups, _ := server.dispatcher.Groups(
			func(route *dispatch.Route) bool { return true },                         // Accept all routes
			func(alert *alertmanagertypes.Alert, now time.Time) bool { return true }, // Accept all alerts
		)
		require.Len(t, alertGroups, 3)

		require.NotEmpty(t, alertGroups, "Should have alert groups created by dispatcher")

		totalAlerts := 0
		for _, group := range alertGroups {
			totalAlerts += len(group.Alerts)
		}
		require.Equal(t, 3, totalAlerts, "Should have 3 alerts total across all groups")
		require.Equal(t, "{__receiver__=\"webhook\"}:{cluster=\"prod-cluster\", instance=\"server-01\", ruleId=\"high-cpu-usage\"}", alertGroups[0].GroupKey)
		require.Equal(t, "{__receiver__=\"webhook\"}:{cluster=\"prod-cluster\", instance=\"server-02\", ruleId=\"high-cpu-usage\"}", alertGroups[1].GroupKey)
		require.Equal(t, "{__receiver__=\"webhook\"}:{cluster=\"prod-cluster\", instance=\"server-03\", ruleId=\"high-cpu-usage\"}", alertGroups[2].GroupKey)
	})

	req, err := http.NewRequest(http.MethodGet, "/alerts", nil)
	require.NoError(t, err)
	params, err := alertmanagertypes.NewGettableAlertsParams(req)
	require.NoError(t, err)
	alerts, err := server.GetAlerts(ctx, params)
	require.NoError(t, err)

	t.Run("verify_muting_ruleid_and_scope", func(t *testing.T) {
		// Window with ruleID + scope mutes only alerts matching both.
		for _, alert := range alerts {
			if alert.Labels["ruleId"] == "high-cpu-usage" && alert.Labels["severity"] == "critical" {
				require.Contains(t, alert.Status.MutedBy, mwRuleIDAndScope.String())
			} else {
				require.NotContains(t, alert.Status.MutedBy, mwRuleIDAndScope.String())
			}
		}
	})

	t.Run("verify_muting_ruleid_only", func(t *testing.T) {
		// Window with ruleID but no scope mutes all severities for that rule.
		for _, alert := range alerts {
			if alert.Labels["ruleId"] == "high-cpu-usage" {
				require.Contains(t, alert.Status.MutedBy, mwRuleIDOnly.String())
			} else {
				require.NotContains(t, alert.Status.MutedBy, mwRuleIDOnly.String())
			}
		}
	})

	t.Run("verify_muting_scope_only", func(t *testing.T) {
		// Window with scope but no ruleIDs mutes all critical alerts regardless of rule.
		for _, alert := range alerts {
			if alert.Labels["severity"] == "critical" {
				require.Contains(t, alert.Status.MutedBy, mwScopeOnly.String())
			} else {
				require.NotContains(t, alert.Status.MutedBy, mwScopeOnly.String())
			}
		}
	})
}