SigNoz is an open-source, OpenTelemetry-native observability platform for your team and their AI agents. Get logs, metrics, and traces in one tool with features like APM, distributed tracing, log management, infra monitoring, etc. Combined with SigNoz MCP and a native AI teammate (in SigNoz Cloud) it helps you build more resilient apps.
TypeScript apmapplication-monitoringdistributed-tracinggogood-first-issuejaegerloglogsmetricsmonitoringnextjsobservabilityopen-sourceopentelemetryprometheusreactreactjsself-hostedtracingtypescript
fix(alertmanager): resolve email SMTP settings from env (#12281)
Srikanth Chekuri957580ec7c8e
diff --git a/.github/workflows/integrationci.yaml b/.github/workflows/integrationci.yamlindex c9707dce0e..f74e63e8a3 100644--- a/.github/workflows/integrationci.yaml+++ b/.github/workflows/integrationci.yaml@@ -39,6 +39,8 @@ jobs:matrix:suite:- alerts+ - alertmanager+ - alertmanagerrotation- basepath- callbackauthn- cloudintegrationsdiff --git a/pkg/alertmanager/alertmanagerserver/server.go b/pkg/alertmanager/alertmanagerserver/server.goindex b8c5f22979..70e858b70a 100644--- a/pkg/alertmanager/alertmanagerserver/server.go+++ b/pkg/alertmanager/alertmanagerserver/server.go@@ -241,9 +241,12 @@ func (server *Server) PutAlerts(ctx context.Context, postableAlerts alertmanager}func (server *Server) SetConfig(ctx context.Context, alertmanagerConfig *alertmanagertypes.Config) error {- config := alertmanagerConfig.AlertmanagerConfig()+ resolved, err := alertmanagerConfig.Resolved()+ if err != nil {+ return err+ }+ config := resolved.AlertmanagerConfig()- var err error// Load SigNoz's alertmanager notification templates from the configured// globs. The upstream default templates (default.tmpl, email.tmpl) are// always loaded from the embedded alertmanager assets inside FromGlobs, so@@ -275,7 +278,7 @@ func (server *Server) SetConfig(ctx context.Context, alertmanagerConfig *alertmaserver.logger.InfoContext(ctx, "skipping creation of receiver not referenced by any route", slog.String("receiver", rcv.Name))continue}- extendedRcv, err := alertmanagerConfig.GetReceiver(rcv.Name)+ extendedRcv, err := resolved.GetReceiver(rcv.Name)if err != nil {return err}@@ -350,7 +353,7 @@ func (server *Server) SetConfig(ctx context.Context, alertmanagerConfig *alertmago server.dispatcher.Run()go server.inhibitor.Run()- server.alertmanagerConfig = alertmanagerConfig+ server.alertmanagerConfig = resolvedreturn nil}diff --git a/pkg/alertmanager/config.go b/pkg/alertmanager/config.goindex d21bfb1851..c4cdef4b17 100644--- a/pkg/alertmanager/config.go+++ b/pkg/alertmanager/config.go@@ -1,7 +1,6 @@package alertmanagerimport (- "net/url""strings""time"@@ -26,11 +25,6 @@ type Signoz struct {alertmanagerserver.Config `mapstructure:",squash" yaml:",squash"`}-type Legacy struct {- // ApiURL is the URL of the legacy signoz alertmanager.- ApiURL *url.URL `mapstructure:"api_url"`-}-func NewConfigFactory() factory.ConfigFactory {return factory.NewConfigFactory(factory.MustNewName("alertmanager"), newConfig)}diff --git a/pkg/alertmanager/signozalertmanager/provider.go b/pkg/alertmanager/signozalertmanager/provider.goindex f04f47cd71..a783239d78 100644--- a/pkg/alertmanager/signozalertmanager/provider.go+++ b/pkg/alertmanager/signozalertmanager/provider.go@@ -167,6 +167,10 @@ func (provider *provider) UpdateChannelByReceiverAndID(ctx context.Context, orgIreturn err}+ if err := config.SetGlobalConfig(provider.config.Signoz.Global); err != nil {+ return err+ }+if err := config.UpdateReceiver(receiver); err != nil {return err}@@ -217,6 +221,10 @@ func (provider *provider) CreateChannel(ctx context.Context, orgID string, receireturn nil, err}+ if err := config.SetGlobalConfig(provider.config.Signoz.Global); err != nil {+ return nil, err+ }+if err := config.CreateReceiver(receiver); err != nil {return nil, err}diff --git a/pkg/signoz/provider.go b/pkg/signoz/provider.goindex ccccaeb274..9c26aac7fc 100644--- a/pkg/signoz/provider.go+++ b/pkg/signoz/provider.go@@ -233,6 +233,7 @@ func NewSQLMigrationProviderFactories(sqlmigration.NewFillDashboardMeterSourceFactory(sqlstore, dashboardStore),sqlmigration.NewUpdateRoleTransactionGroupsFactory(),sqlmigration.NewFillDashboardSpecCollectionsFactory(sqlstore, dashboardStore),+ sqlmigration.NewScrubEmailChannelTransportFactory(sqlstore),)}diff --git a/pkg/sqlmigration/107_scrub_email_channel_transport.go b/pkg/sqlmigration/107_scrub_email_channel_transport.gonew file mode 100644index 0000000000..9187688b62--- /dev/null+++ b/pkg/sqlmigration/107_scrub_email_channel_transport.go@@ -0,0 +1,274 @@+package sqlmigration++import (+ "context"+ "crypto/md5"+ "encoding/json"+ "fmt"+ "log/slog"++ "github.com/SigNoz/signoz/pkg/errors"+ "github.com/SigNoz/signoz/pkg/factory"+ "github.com/SigNoz/signoz/pkg/sqlstore"+ "github.com/uptrace/bun"+ "github.com/uptrace/bun/migrate"+)++type scrubEmailChannelTransport struct {+ sqlstore sqlstore.SQLStore+ logger *slog.Logger+}++type alertmanagerConfigScrubRow struct {+ bun.BaseModel `bun:"table:alertmanager_config"`++ ID string `bun:"id"`+ Config string `bun:"config"`+}++type notificationChannelScrubRow struct {+ bun.BaseModel `bun:"table:notification_channel"`++ ID string `bun:"id"`+ Data string `bun:"data"`+}++var emailTransportKeys = []string{+ "from",+ "hello",+ "smarthost",+ "auth_username",+ "auth_password",+ "auth_password_file",+ "auth_secret",+ "auth_secret_file",+ "auth_identity",+ "require_tls",+ "tls_config",+ "force_implicit_tls",+}++var globalSMTPKeys = []string{+ "smtp_from",+ "smtp_hello",+ "smtp_smarthost",+ "smtp_auth_username",+ "smtp_auth_password",+ "smtp_auth_password_file",+ "smtp_auth_secret",+ "smtp_auth_secret_file",+ "smtp_auth_identity",+ "smtp_require_tls",+ "smtp_tls_config",+ "smtp_force_implicit_tls",+}++func NewScrubEmailChannelTransportFactory(sqlstore sqlstore.SQLStore) factory.ProviderFactory[SQLMigration, Config] {+ return factory.NewProviderFactory(+ factory.MustNewName("scrub_email_channel_transport"),+ func(ctx context.Context, ps factory.ProviderSettings, c Config) (SQLMigration, error) {+ return &scrubEmailChannelTransport{sqlstore: sqlstore, logger: ps.Logger}, nil+ },+ )+}++func (migration *scrubEmailChannelTransport) Register(migrations *migrate.Migrations) error {+ if err := migrations.Register(migration.Up, migration.Down); err != nil {+ return err+ }+ return nil+}++func (migration *scrubEmailChannelTransport) Up(ctx context.Context, db *bun.DB) error {+ tx, err := db.BeginTx(ctx, nil)+ if err != nil {+ return err+ }++ defer func() {+ _ = tx.Rollback()+ }()++ if err := migration.scrubConfigs(ctx, tx); err != nil {+ return err+ }++ if err := migration.scrubChannels(ctx, tx); err != nil {+ return err+ }++ return tx.Commit()+}++func (migration *scrubEmailChannelTransport) scrubConfigs(ctx context.Context, tx bun.Tx) error {+ rows := make([]*alertmanagerConfigScrubRow, 0)+ if err := tx.NewSelect().Model(&rows).Scan(ctx); err != nil {+ return err+ }++ for _, row := range rows {+ cfg := make(map[string]json.RawMessage)+ if err := json.Unmarshal([]byte(row.Config), &cfg); err != nil {+ migration.logger.WarnContext(ctx, "skipping alertmanager config with unreadable config", slog.String("config_id", row.ID), errors.Attr(err))+ continue+ }++ changed := false++ if globalRaw, ok := cfg["global"]; ok && string(globalRaw) != "null" {+ global := make(map[string]json.RawMessage)+ if err := json.Unmarshal(globalRaw, &global); err != nil {+ migration.logger.WarnContext(ctx, "skipping alertmanager config with unreadable global", slog.String("config_id", row.ID), errors.Attr(err))+ continue+ }+ if deleteKeys(global, globalSMTPKeys) {+ newGlobal, err := json.Marshal(global)+ if err != nil {+ migration.logger.WarnContext(ctx, "skipping alertmanager config, cannot marshal global", slog.String("config_id", row.ID), errors.Attr(err))+ continue+ }+ cfg["global"] = newGlobal+ changed = true+ }+ }++ if receiversRaw, ok := cfg["receivers"]; ok && string(receiversRaw) != "null" {+ receivers := make([]map[string]json.RawMessage, 0)+ if err := json.Unmarshal(receiversRaw, &receivers); err != nil {+ migration.logger.WarnContext(ctx, "skipping alertmanager config with unreadable receivers", slog.String("config_id", row.ID), errors.Attr(err))+ continue+ }++ receiversChanged := false+ unreadable := false+ for _, receiver := range receivers {+ scrubbed, err := scrubEmailConfigs(receiver)+ if err != nil {+ migration.logger.WarnContext(ctx, "skipping alertmanager config with unreadable email configs", slog.String("config_id", row.ID), errors.Attr(err))+ unreadable = true+ break+ }+ receiversChanged = receiversChanged || scrubbed+ }+ if unreadable {+ continue+ }++ if receiversChanged {+ newReceivers, err := json.Marshal(receivers)+ if err != nil {+ migration.logger.WarnContext(ctx, "skipping alertmanager config, cannot marshal receivers", slog.String("config_id", row.ID), errors.Attr(err))+ continue+ }+ cfg["receivers"] = newReceivers+ changed = true+ }+ }++ if !changed {+ continue+ }++ newConfig, err := json.Marshal(cfg)+ if err != nil {+ migration.logger.WarnContext(ctx, "skipping alertmanager config, cannot marshal config", slog.String("config_id", row.ID), errors.Attr(err))+ continue+ }++ if _, err := tx.NewUpdate().+ Model((*alertmanagerConfigScrubRow)(nil)).+ Set("config = ?", string(newConfig)).+ Set("hash = ?", fmt.Sprintf("%x", md5.Sum(newConfig))).+ Where("id = ?", row.ID).+ Exec(ctx); err != nil {+ return err+ }+ }++ return nil+}++func (migration *scrubEmailChannelTransport) scrubChannels(ctx context.Context, tx bun.Tx) error {+ rows := make([]*notificationChannelScrubRow, 0)+ if err := tx.NewSelect().Model(&rows).Where("type = ?", "email").Scan(ctx); err != nil {+ return err+ }++ for _, row := range rows {+ receiver := make(map[string]json.RawMessage)+ if err := json.Unmarshal([]byte(row.Data), &receiver); err != nil {+ migration.logger.WarnContext(ctx, "skipping notification channel with unreadable data", slog.String("channel_id", row.ID), errors.Attr(err))+ continue+ }++ scrubbed, err := scrubEmailConfigs(receiver)+ if err != nil {+ migration.logger.WarnContext(ctx, "skipping notification channel with unreadable email configs", slog.String("channel_id", row.ID), errors.Attr(err))+ continue+ }+ if !scrubbed {+ continue+ }++ newData, err := json.Marshal(receiver)+ if err != nil {+ migration.logger.WarnContext(ctx, "skipping notification channel, cannot marshal data", slog.String("channel_id", row.ID), errors.Attr(err))+ continue+ }++ if _, err := tx.NewUpdate().+ Model((*notificationChannelScrubRow)(nil)).+ Set("data = ?", string(newData)).+ Where("id = ?", row.ID).+ Exec(ctx); err != nil {+ return err+ }+ }++ return nil+}++func scrubEmailConfigs(receiver map[string]json.RawMessage) (bool, error) {+ emailConfigsRaw, ok := receiver["email_configs"]+ if !ok || string(emailConfigsRaw) == "null" {+ return false, nil+ }++ emailConfigs := make([]map[string]json.RawMessage, 0)+ if err := json.Unmarshal(emailConfigsRaw, &emailConfigs); err != nil {+ return false, err+ }++ changed := false+ for _, emailConfig := range emailConfigs {+ changed = deleteKeys(emailConfig, emailTransportKeys) || changed+ }++ if !changed {+ return false, nil+ }++ newEmailConfigs, err := json.Marshal(emailConfigs)+ if err != nil {+ return false, err+ }+ receiver["email_configs"] = newEmailConfigs++ return true, nil+}++func deleteKeys(m map[string]json.RawMessage, keys []string) bool {+ deleted := false+ for _, key := range keys {+ if _, ok := m[key]; ok {+ delete(m, key)+ deleted = true+ }+ }++ return deleted+}++func (migration *scrubEmailChannelTransport) Down(context.Context, *bun.DB) error {+ return nil+}diff --git a/pkg/types/alertmanagertypes/channel_test.go b/pkg/types/alertmanagertypes/channel_test.goindex c633d73a04..5f2d71a0f2 100644--- a/pkg/types/alertmanagertypes/channel_test.go+++ b/pkg/types/alertmanagertypes/channel_test.go@@ -35,12 +35,8 @@ func TestNewConfigFromChannels(t *testing.T) {"email_configs": []any{map[string]any{"send_resolved": false,"to": "[email protected]",- "from": "[email protected]",- "hello": "localhost",- "smarthost": "smtp.example.com:587",- "require_tls": true,+ "smarthost": "","html": "{{ template \"email.default.html\" . }}",- "tls_config": map[string]any{"insecure_skip_verify": false},"threading": map[string]any{},}},},@@ -63,7 +59,6 @@ func TestNewConfigFromChannels(t *testing.T) {"slack_configs": []any{map[string]any{"send_resolved": true,"api_url": "https://slack.com/api/test",- "app_url": "https://slack.com/api/chat.postMessage","channel": "#alerts","callback_id": "{{ template \"slack.default.callbackid\" . }}","color": "{{ if eq .Status \"firing\" }}danger{{ else }}good{{ end }}",@@ -77,12 +72,6 @@ func TestNewConfigFromChannels(t *testing.T) {"title": "{{ template \"slack.default.title\" . }}","title_link": "{{ template \"slack.default.titlelink\" . }}","username": "{{ template \"slack.default.username\" . }}",- "http_config": map[string]any{- "tls_config": map[string]any{"insecure_skip_verify": false},- "follow_redirects": true,- "enable_http2": true,- "proxy_url": nil,- },}},},},@@ -104,7 +93,6 @@ func TestNewConfigFromChannels(t *testing.T) {"pagerduty_configs": []any{map[string]any{"send_resolved": false,"service_key": "test",- "url": "https://events.pagerduty.com/v2/enqueue","client": "{{ template \"pagerduty.default.client\" . }}","client_url": "{{ template \"pagerduty.default.clientURL\" . }}","description": "{{ template \"pagerduty.default.description\" .}}",@@ -116,12 +104,6 @@ func TestNewConfigFromChannels(t *testing.T) {"num_resolved": "{{ .Alerts.Resolved | len }}","resolved": "{{ .Alerts.Resolved | toJson }}",},- "http_config": map[string]any{- "tls_config": map[string]any{"insecure_skip_verify": false},- "follow_redirects": true,- "enable_http2": true,- "proxy_url": nil,- },}},},},@@ -148,7 +130,6 @@ func TestNewConfigFromChannels(t *testing.T) {"pagerduty_configs": []any{map[string]any{"send_resolved": false,"service_key": "test",- "url": "https://events.pagerduty.com/v2/enqueue","client": "{{ template \"pagerduty.default.client\" . }}","client_url": "{{ template \"pagerduty.default.clientURL\" . }}","description": "{{ template \"pagerduty.default.description\" .}}",@@ -160,12 +141,6 @@ func TestNewConfigFromChannels(t *testing.T) {"num_resolved": "{{ .Alerts.Resolved | len }}","resolved": "{{ .Alerts.Resolved | toJson }}",},- "http_config": map[string]any{- "tls_config": map[string]any{"insecure_skip_verify": false},- "follow_redirects": true,- "enable_http2": true,- "proxy_url": nil,- },}},},{@@ -173,7 +148,6 @@ func TestNewConfigFromChannels(t *testing.T) {"slack_configs": []any{map[string]any{"send_resolved": true,"api_url": "https://slack.com/api/test",- "app_url": "https://slack.com/api/chat.postMessage","channel": "#alerts","callback_id": "{{ template \"slack.default.callbackid\" . }}","color": "{{ if eq .Status \"firing\" }}danger{{ else }}good{{ end }}",@@ -187,12 +161,6 @@ func TestNewConfigFromChannels(t *testing.T) {"title": "{{ template \"slack.default.title\" . }}","title_link": "{{ template \"slack.default.titlelink\" . }}","username": "{{ template \"slack.default.username\" . }}",- "http_config": map[string]any{- "tls_config": map[string]any{"insecure_skip_verify": false},- "follow_redirects": true,- "enable_http2": true,- "proxy_url": nil,- },}},},},diff --git a/pkg/types/alertmanagertypes/config.go b/pkg/types/alertmanagertypes/config.goindex c75b94d9eb..70b67030ee 100644--- a/pkg/types/alertmanagertypes/config.go+++ b/pkg/types/alertmanagertypes/config.go@@ -117,6 +117,11 @@ func NewConfigFromStoreableConfig(sc *StoreableConfig) (*Config, error) {return nil, err}+ // It must be replaced with an empty, non-nil global, upstream swaps nil for+ // DefaultGlobalConfig, which would let a path that skips SetGlobalConfig pass+ // validation and fail silently at delivery instead of failing fast here.+ alertmanagerConfig.Global = &config.GlobalConfig{}+return &Config{alertmanagerConfig: alertmanagerConfig,customConfigs: customConfigs,@@ -174,7 +179,7 @@ func newConfigFromString(s string) (*config.Config, map[string]customReceiverConreturn amConfig, customConfigs, nil}-func newRawFromConfig(c *config.Config, customConfigs map[string]customReceiverConfigs) []byte {+func extendedReceivers(c *config.Config, customConfigs map[string]customReceiverConfigs) []*Receiver {receivers := make([]*Receiver, len(c.Receivers))for i := range c.Receivers {base := c.Receivers[i]@@ -185,7 +190,14 @@ func newRawFromConfig(c *config.Config, customConfigs map[string]customReceiverC}}- b, err := json.Marshal(storedConfig{Config: c, Receivers: receivers})+ return receivers+}++func newRawFromConfig(c *config.Config, customConfigs map[string]customReceiverConfigs) []byte {+ persistable := *c+ persistable.Global = nil++ b, err := json.Marshal(storedConfig{Config: &persistable, Receivers: extendedReceivers(c, customConfigs)})if err != nil {// Taking inspiration from the upstream. This is never expected to happen.return []byte(fmt.Sprintf("<error creating config string: %s>", err))@@ -206,6 +218,37 @@ func (c *Config) flush() {c.storeableConfig.UpdatedAt = time.Now()}+func (c *Config) Resolved() (*Config, error) {+ raw, err := json.Marshal(storedConfig{Config: c.alertmanagerConfig, Receivers: extendedReceivers(c.alertmanagerConfig, c.customConfigs)})+ if err != nil {+ return nil, err+ }++ alertmanagerConfig, customConfigs, err := newConfigFromString(string(raw))+ if err != nil {+ return nil, err+ }++ if err := alertmanagerConfig.UnmarshalYAML(func(i interface{}) error { return nil }); err != nil {+ return nil, err+ }++ storeableConfig := *c.storeableConfig+ resolved := &Config{+ alertmanagerConfig: alertmanagerConfig,+ customConfigs: customConfigs,+ storeableConfig: &storeableConfig,+ }+ resolved.applyNativeDefaults()++ return resolved, nil+}++func (c *Config) validate() error {+ _, err := c.Resolved()+ return err+}+func (c *Config) CopyWithReset() (*Config, error) {newConfig, err := NewDefaultConfig(*c.alertmanagerConfig.Global,@@ -271,6 +314,15 @@ func (c *Config) StoreableConfig() *StoreableConfig {return c.storeableConfig}+func cloneReceiver(receiver *Receiver) (*Receiver, error) {+ raw, err := json.Marshal(receiver)+ if err != nil {+ return nil, err+ }++ return NewReceiver(string(raw))+}+func (c *Config) CreateReceiver(receiver *Receiver) error {// check that receiver name is not already usedfor _, existingReceiver := range c.alertmanagerConfig.Receivers {@@ -279,16 +331,21 @@ func (c *Config) CreateReceiver(receiver *Receiver) error {}}- route, err := NewRouteFromReceiver(receiver)+ owned, err := cloneReceiver(receiver)+ if err != nil {+ return err+ }++ route, err := NewRouteFromReceiver(owned)if err != nil {return err}c.alertmanagerConfig.Route.Routes = append(c.alertmanagerConfig.Route.Routes, route)- c.alertmanagerConfig.Receivers = append(c.alertmanagerConfig.Receivers, *receiver.Receiver)- c.setCustomConfigs(receiver)+ c.alertmanagerConfig.Receivers = append(c.alertmanagerConfig.Receivers, *owned.Receiver)+ c.setCustomConfigs(owned)- if err := c.alertmanagerConfig.UnmarshalYAML(func(i interface{}) error { return nil }); err != nil {+ if err := c.validate(); err != nil {return err}c.applyNativeDefaults()@@ -313,16 +370,21 @@ func (c *Config) GetReceiver(name string) (*Receiver, error) {}func (c *Config) UpdateReceiver(receiver *Receiver) error {+ owned, err := cloneReceiver(receiver)+ if err != nil {+ return err+ }+// find and update receiverfor i, existingReceiver := range c.alertmanagerConfig.Receivers {- if existingReceiver.Name == receiver.Name {- c.alertmanagerConfig.Receivers[i] = *receiver.Receiver- c.setCustomConfigs(receiver)+ if existingReceiver.Name == owned.Name {+ c.alertmanagerConfig.Receivers[i] = *owned.Receiver+ c.setCustomConfigs(owned)break}}- if err := c.alertmanagerConfig.UnmarshalYAML(func(i interface{}) error { return nil }); err != nil {+ if err := c.validate(); err != nil {return err}c.applyNativeDefaults()diff --git a/pkg/types/alertmanagertypes/config_test.go b/pkg/types/alertmanagertypes/config_test.goindex cb671b5117..df01fb37e4 100644--- a/pkg/types/alertmanagertypes/config_test.go+++ b/pkg/types/alertmanagertypes/config_test.go@@ -330,6 +330,150 @@ func TestSetGlobalConfigPreservesSMTPRequireTLS(t *testing.T) {}}+func newSMTPGlobalConfig() GlobalConfig {+ return GlobalConfig{+ SMTPFrom: "[email protected]",+ SMTPHello: "example.com",+ SMTPSmarthost: config.HostPort{Host: "smtp.sendgrid.net", Port: "587"},+ SMTPAuthUsername: "apikey",+ SMTPAuthPassword: "operator-secret",+ SMTPRequireTLS: true,+ }+}++func newEmailTestConfig(t *testing.T) *Config {+ t.Helper()++ cfg, err := NewDefaultConfig(+ newSMTPGlobalConfig(),+ RouteConfig{GroupInterval: time.Minute, GroupWait: time.Minute, RepeatInterval: time.Minute},+ "1",+ )+ require.NoError(t, err)++ receiver, err := NewReceiver(`{"name":"email-receiver","email_configs":[{"to":"[email protected]"}]}`)+ require.NoError(t, err)+ require.NoError(t, cfg.CreateReceiver(receiver))++ return cfg+}++func TestStoreableConfigCarriesNoSMTPSettings(t *testing.T) {+ cfg := newEmailTestConfig(t)++ raw := cfg.StoreableConfig().Config+ assert.NotContains(t, raw, "operator-secret")+ assert.NotContains(t, raw, "smtp.sendgrid.net")+ assert.NotContains(t, raw, "apikey")+ assert.NotContains(t, raw, "[email protected]")++ assert.Equal(t, "operator-secret", string(cfg.alertmanagerConfig.Global.SMTPAuthPassword))+}++func TestStoreableConfigCarriesNoGlobal(t *testing.T) {+ cfg := newEmailTestConfig(t)++ stored := map[string]json.RawMessage{}+ require.NoError(t, json.Unmarshal([]byte(cfg.StoreableConfig().Config), &stored))+ assert.NotContains(t, stored, "global")+}++func TestSetGlobalConfigDoesNotChangeStoreableHash(t *testing.T) {+ cfg := newEmailTestConfig(t)+ hash := cfg.StoreableConfig().Hash++ require.NoError(t, cfg.SetGlobalConfig(GlobalConfig{SMTPSmarthost: config.HostPort{Host: "smtp.other.net", Port: "2525"}, SMTPAuthPassword: "rotated-secret"}))++ assert.Equal(t, hash, cfg.StoreableConfig().Hash)+ assert.NotContains(t, cfg.StoreableConfig().Config, "rotated-secret")+}++func TestNewConfigFromStoreableConfigDiscardsStoredGlobal(t *testing.T) {+ stored := &StoreableConfig{+ Config: `{"global":{"resolve_timeout":"5m","smtp_smarthost":"email-smtp.us-east-1.amazonaws.com:587","smtp_auth_password":"old-secret","slack_api_url":"https://hooks.slack.com/services/T/B/X"},"route":{"receiver":"default-receiver"},"receivers":[{"name":"default-receiver"}]}`,+ OrgID: "1",+ }++ cfg, err := NewConfigFromStoreableConfig(stored)+ require.NoError(t, err)++ assert.Equal(t, &config.GlobalConfig{}, cfg.alertmanagerConfig.Global)+}++func TestResolvedFillsEmailTransportFromGlobal(t *testing.T) {+ cfg := newEmailTestConfig(t)++ resolved, err := cfg.Resolved()+ require.NoError(t, err)++ receiver, err := resolved.GetReceiver("email-receiver")+ require.NoError(t, err)+ require.Len(t, receiver.EmailConfigs, 1)++ got := receiver.EmailConfigs[0]+ assert.Equal(t, "[email protected]", got.To)+ assert.Equal(t, "smtp.sendgrid.net:587", got.Smarthost.String())+ assert.Equal(t, "[email protected]", got.From)+ assert.Equal(t, "apikey", got.AuthUsername)+ assert.Equal(t, "operator-secret", string(got.AuthPassword))+ require.NotNil(t, got.RequireTLS)+ assert.True(t, *got.RequireTLS)++ stored, err := cfg.GetReceiver("email-receiver")+ require.NoError(t, err)+ assert.Empty(t, stored.EmailConfigs[0].Smarthost.String())+ assert.NotContains(t, cfg.StoreableConfig().Config, "operator-secret")+}++func TestStaleStoredSMTPSettingsAreReplacedOnLoad(t *testing.T) {+ stored := &StoreableConfig{+ Config: `{"global":{"resolve_timeout":"5m","smtp_from":"[email protected]","smtp_hello":"localhost","smtp_smarthost":"email-smtp.us-east-1.amazonaws.com:587","smtp_auth_username":"old-user","smtp_auth_password":"old-secret","smtp_require_tls":true},"route":{"receiver":"default-receiver","group_by":["ruleId"],"routes":[{"receiver":"email-receiver","continue":true,"matchers":["ruleId=~\"-1\""]}],"group_wait":"30s","group_interval":"5m","repeat_interval":"4h"},"receivers":[{"name":"default-receiver"},{"name":"email-receiver","email_configs":[{"send_resolved":false,"to":"[email protected]","from":"[email protected]","hello":"localhost","smarthost":"email-smtp.us-east-1.amazonaws.com:587","auth_username":"old-user","auth_password":"old-secret","require_tls":true}]}]}`,+ OrgID: "1",+ }++ cfg, err := NewConfigFromStoreableConfig(stored)+ require.NoError(t, err)++ loaded, err := cfg.GetReceiver("email-receiver")+ require.NoError(t, err)+ require.Len(t, loaded.EmailConfigs, 1)+ assert.Empty(t, loaded.EmailConfigs[0].Smarthost.String())+ assert.Empty(t, string(loaded.EmailConfigs[0].AuthPassword))++ require.NoError(t, cfg.SetGlobalConfig(newSMTPGlobalConfig()))++ resolved, err := cfg.Resolved()+ require.NoError(t, err)++ receiver, err := resolved.GetReceiver("email-receiver")+ require.NoError(t, err)+ got := receiver.EmailConfigs[0]+ assert.Equal(t, "smtp.sendgrid.net:587", got.Smarthost.String())+ assert.Equal(t, "operator-secret", string(got.AuthPassword))+ assert.Equal(t, "[email protected]", got.From)++ assert.NotContains(t, cfg.StoreableConfig().Config, "old-secret")+ assert.NotContains(t, cfg.StoreableConfig().Config, "amazonaws.com")+ assert.NotContains(t, cfg.StoreableConfig().Config, "operator-secret")+}++func TestCreateReceiverDoesNotMutateCaller(t *testing.T) {+ cfg := newEmailTestConfig(t)++ resolved, err := cfg.Resolved()+ require.NoError(t, err)+ receiver, err := resolved.GetReceiver("email-receiver")+ require.NoError(t, err)+ require.Equal(t, "smtp.sendgrid.net:587", receiver.EmailConfigs[0].Smarthost.String())++ throwaway, err := cfg.CopyWithReset()+ require.NoError(t, err)+ require.NoError(t, throwaway.CreateReceiver(receiver))++ assert.Equal(t, "smtp.sendgrid.net:587", receiver.EmailConfigs[0].Smarthost.String())+ assert.Equal(t, "operator-secret", string(receiver.EmailConfigs[0].AuthPassword))+}+// Round-trip: create → serialize → reload → GetReceiver still has the configs.func TestConfigPreservesGoogleChatConfigs(t *testing.T) {webhookURL, err := url.Parse("https://chat.googleapis.com/v1/spaces/test/messages")diff --git a/pkg/types/alertmanagertypes/receiver.go b/pkg/types/alertmanagertypes/receiver.goindex 474612dad6..695545b900 100644--- a/pkg/types/alertmanagertypes/receiver.go+++ b/pkg/types/alertmanagertypes/receiver.go@@ -37,6 +37,7 @@ func NewReceiver(input string) (*Receiver, error) {if err != nil {return nil, err}+ stripEmailTransport(withDefaults)receiver.Receiver = withDefaults// Extend this block when adding another native notifier type.@@ -53,6 +54,23 @@ func NewReceiver(input string) (*Receiver, error) {return receiver, nil}+func stripEmailTransport(base *config.Receiver) {+ for _, ec := range base.EmailConfigs {+ ec.From = ""+ ec.Hello = ""+ ec.Smarthost = config.HostPort{}+ ec.AuthUsername = ""+ ec.AuthPassword = ""+ ec.AuthPasswordFile = ""+ ec.AuthSecret = ""+ ec.AuthSecretFile = ""+ ec.AuthIdentity = ""+ ec.RequireTLS = nil+ ec.TLSConfig = nil+ ec.ForceImplicitTLS = nil+ }+}+func defaultedBaseReceiver(base *config.Receiver) (*config.Receiver, error) {bytes, err := yaml.Marshal(base)if err != nil {@@ -102,7 +120,12 @@ func TestReceiver(ctx context.Context, receiver *Receiver, receiverIntegrationsFreturn err}- defaultedReceiver, err := testConfig.GetReceiver(receiver.Name)+ resolvedConfig, err := testConfig.Resolved()+ if err != nil {+ return err+ }++ defaultedReceiver, err := resolvedConfig.GetReceiver(receiver.Name)if err != nil {return err}diff --git a/pkg/types/alertmanagertypes/receiver_test.go b/pkg/types/alertmanagertypes/receiver_test.goindex 49f3772a4c..2169abd752 100644--- a/pkg/types/alertmanagertypes/receiver_test.go+++ b/pkg/types/alertmanagertypes/receiver_test.go@@ -46,6 +46,31 @@ func TestNewReceiver(t *testing.T) {}}+func TestNewReceiverStripsEmailTransport(t *testing.T) {+ receiver, err := NewReceiver(`{"name":"email","email_configs":[{"to":"[email protected]","from":"[email protected]","hello":"example.com","smarthost":"smtp.example.com:587","auth_username":"user","auth_password":"supersecret","auth_secret":"alsosecret","auth_identity":"id","require_tls":false,"headers":{"Subject":"custom"}}]}`)+ require.NoError(t, err)+ require.Len(t, receiver.EmailConfigs, 1)++ got := receiver.EmailConfigs[0]+ assert.Equal(t, "[email protected]", got.To)+ assert.Equal(t, map[string]string{"Subject": "custom"}, got.Headers)++ assert.Empty(t, got.From)+ assert.Empty(t, got.Hello)+ assert.Empty(t, got.Smarthost.String())+ assert.Empty(t, got.AuthUsername)+ assert.Empty(t, string(got.AuthPassword))+ assert.Empty(t, string(got.AuthSecret))+ assert.Empty(t, got.AuthIdentity)+ assert.Nil(t, got.RequireTLS)+ assert.Nil(t, got.TLSConfig)++ bytes, err := json.Marshal(receiver)+ require.NoError(t, err)+ assert.NotContains(t, string(bytes), "supersecret")+ assert.NotContains(t, string(bytes), "smtp.example.com")+}+// Omitted fields fall back to DefaultGoogleChatReceiverConfig.func TestNewReceiverGoogleChatAppliesDefaults(t *testing.T) {receiver, err := NewReceiver(`{"name":"googlechat","googlechat_configs":[{"webhook_url":"https://chat.googleapis.com/v1/spaces/test/messages"}]}`)diff --git a/tests/conftest.py b/tests/conftest.pyindex 10fc475d8a..eda6c63f10 100644--- a/tests/conftest.py+++ b/tests/conftest.py@@ -24,6 +24,7 @@ pytest_plugins = ["fixtures.keycloak","fixtures.idp","fixtures.notification_channel",+ "fixtures.maildev","fixtures.alerts","fixtures.cloudintegrations","fixtures.jsontypes",diff --git a/tests/fixtures/alerts.py b/tests/fixtures/alerts.pyindex e8ac1c5991..5a38188e3e 100644--- a/tests/fixtures/alerts.py+++ b/tests/fixtures/alerts.py@@ -1,11 +1,13 @@import base64import json+import reimport timeimport urllib.parseimport uuidfrom collections.abc import Callablefrom datetime import UTC, datetime, timedeltafrom http import HTTPStatus+from urllib.parse import urlparseimport pytestimport requests@@ -15,6 +17,7 @@ from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORDfrom fixtures.fs import get_testdata_file_pathfrom fixtures.logger import setup_loggerfrom fixtures.logs import Logs+from fixtures.maildev import get_all_mails, verify_email_receivedfrom fixtures.metrics import Metricsfrom fixtures.traces import Traces@@ -311,3 +314,145 @@ def update_rule_channel_name(rule_data: dict, channel_name: str):# loop over all the sepcs and update the channelsfor spec in thresholds["spec"]:spec["channels"] = [channel_name]+++def _is_json_subset(subset, superset) -> bool:+ """Check if subset is contained within superset recursively.+ - For dicts: all keys in subset must exist in superset with matching values+ - For lists: all items in subset must be present in superset+ - For scalars: exact equality+ """+ if isinstance(subset, dict):+ if not isinstance(superset, dict):+ return False+ return all(key in superset and _is_json_subset(value, superset[key]) for key, value in subset.items())+ if isinstance(subset, list):+ if not isinstance(superset, list):+ return False+ return all(any(_is_json_subset(sub_item, sup_item) for sup_item in superset) for sub_item in subset)+ if isinstance(subset, re.Pattern):+ return isinstance(superset, str) and subset.search(superset) is not None+ return subset == superset+++def verify_webhook_notification_expectation(+ notification_channel: types.TestContainerDocker,+ validation_data: dict,+) -> bool:+ """Check if wiremock received a request at the given path+ whose JSON body is a superset of the expected json_body."""+ path = validation_data["path"]+ json_body = validation_data["json_body"]++ url = notification_channel.host_configs["8080"].get("__admin/requests/find")+ try:+ res = requests.post(url, json={"method": "POST", "url": path}, timeout=10)+ except requests.exceptions.RequestException:+ return False+ if res.status_code != HTTPStatus.OK:+ return False++ for req in res.json()["requests"]:+ body = json.loads(base64.b64decode(req["bodyAsBase64"]).decode("utf-8"))+ if _is_json_subset(json_body, body):+ return True+ return False+++def _check_notification_validation(+ validation: types.NotificationValidation,+ notification_channel: types.TestContainerDocker,+ maildev: types.TestContainerDocker,+) -> bool:+ """Dispatch a single validation check to the appropriate verifier."""+ if validation.destination_type == "webhook":+ return verify_webhook_notification_expectation(notification_channel, validation.validation_data)+ if validation.destination_type == "email":+ return verify_email_received(maildev, validation.validation_data)+ raise ValueError(f"Invalid destination type: {validation.destination_type}")+++def verify_notification_expectation(+ notification_channel: types.TestContainerDocker,+ maildev: types.TestContainerDocker,+ expected_notification: types.AMNotificationExpectation,+) -> bool:+ """Poll for expected notifications across webhook and email channels."""+ time_to_wait = datetime.now() + timedelta(seconds=expected_notification.wait_time_seconds)++ while datetime.now() < time_to_wait:+ all_found = all(_check_notification_validation(v, notification_channel, maildev) for v in expected_notification.notification_validations)++ if expected_notification.should_notify and all_found:+ logger.info("All expected notifications found")+ return True++ time.sleep(1)++ # Timeout reached+ if not expected_notification.should_notify:+ # Verify no notifications were received+ for validation in expected_notification.notification_validations:+ found = _check_notification_validation(validation, notification_channel, maildev)+ assert not found, f"Expected no notification but found one for {validation.destination_type} with data {validation.validation_data}"+ logger.info("No notifications found, as expected")+ return True++ missing = [v for v in expected_notification.notification_validations if not _check_notification_validation(v, notification_channel, maildev)]+ assert len(missing) == 0, f"Expected all notifications to be found but missing: {missing}, received: {_received_notifications(notification_channel, maildev, missing)}"+ return True+++def _received_notifications(+ notification_channel: types.TestContainerDocker,+ maildev: types.TestContainerDocker,+ missing: list[types.NotificationValidation],+) -> dict:+ received = {}+ if any(v.destination_type == "webhook" for v in missing):+ webhook_bodies = []+ for validation in missing:+ if validation.destination_type != "webhook":+ continue+ url = notification_channel.host_configs["8080"].get("__admin/requests/find")+ try:+ res = requests.post(url, json={"method": "POST", "url": validation.validation_data["path"]}, timeout=10)+ webhook_bodies.extend(json.loads(base64.b64decode(req["bodyAsBase64"]).decode("utf-8")) for req in res.json()["requests"])+ except requests.exceptions.RequestException as exc:+ webhook_bodies.append(f"<failed to fetch wiremock journal: {exc}>")+ received["webhook"] = webhook_bodies+ if any(v.destination_type == "email" for v in missing):+ received["email"] = get_all_mails(maildev)+ return received+++def update_raw_channel_config(+ channel_config: dict,+ channel_name: str,+ notification_channel: types.TestContainerDocker,+) -> dict:+ """+ Updates the channel config to point to the given wiremock+ notification_channel container to receive notifications.+ """+ config = channel_config.copy()++ config["name"] = channel_name++ url_field_map = {+ "slack_configs": "api_url",+ "msteamsv2_configs": "webhook_url",+ "webhook_configs": "url",+ "pagerduty_configs": "url",+ "opsgenie_configs": "api_url",+ }++ for config_key, url_field in url_field_map.items():+ if config_key in config:+ for entry in config[config_key]:+ if url_field in entry:+ original_url = entry[url_field]+ path = urlparse(original_url).path+ entry[url_field] = notification_channel.container_configs["8080"].get(path)++ return configdiff --git a/tests/fixtures/auth.py b/tests/fixtures/auth.pyindex c99595069f..7684fa2d38 100644--- a/tests/fixtures/auth.py+++ b/tests/fixtures/auth.py@@ -77,7 +77,7 @@ def register_admin(timeout=5,)- assert response.status_code == HTTPStatus.OK+ assert response.status_code == HTTPStatus.OK, f"failed to register admin: {response.status_code} {response.text}"return types.Operation(name="create_user_admin")diff --git a/tests/fixtures/http.py b/tests/fixtures/http.pyindex 6df7f206a2..65e7048b19 100644--- a/tests/fixtures/http.py+++ b/tests/fixtures/http.py@@ -125,13 +125,19 @@ def gateway(@pytest.fixture(name="make_http_mocks", scope="function")def make_http_mocks() -> Callable[[types.TestContainerDocker, list[Mapping]], None]:+ mocked_containers = []+def _make_http_mocks(container: types.TestContainerDocker, mappings: list[Mapping]) -> None:Config.base_url = container.host_configs["8080"].get("/__admin")for mapping in mappings:Mappings.create_mapping(mapping=mapping)+ mocked_containers.append(container)+yield _make_http_mocks- Mappings.delete_all_mappings()- Requests.reset_request_journal()+ for container in mocked_containers:+ Config.base_url = container.host_configs["8080"].get("/__admin")+ Mappings.delete_all_mappings()+ Requests.reset_request_journal()diff --git a/tests/fixtures/maildev.py b/tests/fixtures/maildev.pynew file mode 100644index 0000000000..d53f56f376--- /dev/null+++ b/tests/fixtures/maildev.py@@ -0,0 +1,143 @@+import re+from http import HTTPStatus++import docker+import docker.errors+import pytest+import requests+from testcontainers.core.container import DockerContainer, Network++from fixtures import reuse, types+from fixtures.logger import setup_logger++logger = setup_logger(__name__)++MAILDEV_INCOMING_USER = "apikey"+MAILDEV_INCOMING_PASS = "integration-smtp-secret"++SMTP_TEST_FROM = "[email protected]"++OLD_PROVIDER_SMTP_PASS = "old-provider-smtp-secret"+NEW_PROVIDER_SMTP_PASS = "new-provider-smtp-secret"+++def signoz_smtp_env(maildev: "types.TestContainerDocker", password: str = MAILDEV_INCOMING_PASS) -> dict:+ return {+ "SIGNOZ_ALERTMANAGER_SIGNOZ_GLOBAL_SMTP__SMARTHOST": f"{maildev.container_configs['1025'].address}:{maildev.container_configs['1025'].port}",+ "SIGNOZ_ALERTMANAGER_SIGNOZ_GLOBAL_SMTP__FROM": SMTP_TEST_FROM,+ "SIGNOZ_ALERTMANAGER_SIGNOZ_GLOBAL_SMTP__AUTH__USERNAME": MAILDEV_INCOMING_USER,+ "SIGNOZ_ALERTMANAGER_SIGNOZ_GLOBAL_SMTP__AUTH__PASSWORD": password,+ "SIGNOZ_ALERTMANAGER_SIGNOZ_GLOBAL_SMTP__REQUIRE__TLS": "false",+ }+++def create_maildev(+ network: Network,+ request: pytest.FixtureRequest,+ pytestconfig: pytest.Config,+ cache_key: str = "maildev",+ incoming_user: str = MAILDEV_INCOMING_USER,+ incoming_pass: str = MAILDEV_INCOMING_PASS,+) -> types.TestContainerDocker:+ def create() -> types.TestContainerDocker:+ container = DockerContainer(image="maildev/maildev:2.2.1")+ container.with_env("MAILDEV_INCOMING_USER", incoming_user)+ container.with_env("MAILDEV_INCOMING_PASS", incoming_pass)+ container.with_exposed_ports(1025, 1080)+ container.with_network(network=network)+ container.start()++ return types.TestContainerDocker(+ id=container.get_wrapped_container().id,+ host_configs={+ "1025": types.TestContainerUrlConfig(+ scheme="smtp",+ address=container.get_container_host_ip(),+ port=container.get_exposed_port(1025),+ ),+ "1080": types.TestContainerUrlConfig(+ scheme="http",+ address=container.get_container_host_ip(),+ port=container.get_exposed_port(1080),+ ),+ },+ container_configs={+ "1025": types.TestContainerUrlConfig(+ scheme="smtp",+ address=container.get_wrapped_container().name,+ port=1025,+ ),+ "1080": types.TestContainerUrlConfig(+ scheme="http",+ address=container.get_wrapped_container().name,+ port=1080,+ ),+ },+ )++ def delete(container: types.TestContainerDocker):+ client = docker.from_env()+ try:+ client.containers.get(container_id=container.id).stop()+ client.containers.get(container_id=container.id).remove(v=True)+ except docker.errors.NotFound:+ logger.info(+ "Skipping removal of MailDev, MailDev(%s) not found. Maybe it was manually removed?",+ {"id": container.id},+ )++ def restore(cache: dict) -> types.TestContainerDocker:+ return types.TestContainerDocker.from_cache(cache)++ return reuse.wrap(+ request,+ pytestconfig,+ cache_key,+ lambda: types.TestContainerDocker(id="", host_configs={}, container_configs={}),+ create,+ delete,+ restore,+ )++[email protected](name="maildev", scope="package")+def maildev(network: Network, request: pytest.FixtureRequest, pytestconfig: pytest.Config) -> types.TestContainerDocker:+ return create_maildev(network, request, pytestconfig)+++def get_all_mails(_maildev: types.TestContainerDocker) -> list[dict]:+ url = _maildev.host_configs["1080"].get("/email")+ response = requests.get(url, timeout=5)+ assert response.status_code == HTTPStatus.OK, f"Failed to fetch emails from MailDev, status code: {response.status_code}, response: {response.text}"++ def addresses(entries: list[dict]) -> str:+ return ",".join(sorted(entry.get("address", "") for entry in entries))++ return [+ {+ "subject": email.get("subject", ""),+ "html": email.get("html", ""),+ "text": email.get("text", ""),+ "from": addresses(email.get("from", [])),+ "to": addresses(email.get("to", [])),+ }+ for email in response.json()+ ]+++def verify_email_received(_maildev: types.TestContainerDocker, filters: dict) -> bool:+ def matches(expected, actual: str) -> bool:+ if isinstance(expected, re.Pattern):+ return expected.search(actual) is not None+ return expected == actual++ for email in get_all_mails(_maildev):+ if all(key in email and matches(filter_value, email[key]) for key, filter_value in filters.items()):+ return True+ return False+++def delete_all_mails(_maildev: types.TestContainerDocker) -> None:+ url = _maildev.host_configs["1080"].get("/email/all")+ response = requests.delete(url, timeout=5)+ assert response.status_code == HTTPStatus.OK, f"Failed to delete emails from MailDev, status code: {response.status_code}, response: {response.text}"diff --git a/tests/fixtures/notification_channel.py b/tests/fixtures/notification_channel.pyindex bf730f56a2..c0547e2f40 100644--- a/tests/fixtures/notification_channel.py+++ b/tests/fixtures/notification_channel.py@@ -1,3 +1,6 @@+# pylint: disable=line-too-long+import json+import timefrom collections.abc import Callablefrom http import HTTPStatus@@ -11,10 +14,116 @@ from wiremock.testing.testcontainer import WireMockContainerfrom fixtures import reuse, typesfrom fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORDfrom fixtures.logger import setup_logger+from fixtures.maildev import MAILDEV_INCOMING_PASS, SMTP_TEST_FROMlogger = setup_logger(__name__)+EMAIL_TRANSPORT_KEYS = [+ "from",+ "hello",+ "smarthost",+ "auth_username",+ "auth_password",+ "auth_password_file",+ "auth_secret",+ "auth_secret_file",+ "auth_identity",+ "require_tls",+ "tls_config",+ "force_implicit_tls",+]+++def assert_email_channel_payload_clean(payload: str) -> None:+ receiver = json.loads(payload)+ for email_config in receiver["email_configs"]:+ transport_keys = set(email_config.keys()) & set(EMAIL_TRANSPORT_KEYS)+ transport_keys -= {"smarthost"} if email_config.get("smarthost", "") == "" else set()+ assert not transport_keys, f"email channel payload carries transport keys {transport_keys}: {payload}"++ assert MAILDEV_INCOMING_PASS not in payload+ assert SMTP_TEST_FROM not in payload+++"""+Default notification channel configs shared across alertmanager tests.+"""+slack_default_config = {+ # channel name configured on runtime+ "slack_configs": [+ {+ "api_url": "services/TEAM_ID/BOT_ID/TOKEN_ID", # base_url configured on runtime+ "title": '[{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}] {{ .CommonLabels.alertname }} for {{ .CommonLabels.job }}\n {{- if gt (len .CommonLabels) (len .GroupLabels) -}}\n {{" "}}(\n {{- with .CommonLabels.Remove .GroupLabels.Names }}\n {{- range $index, $label := .SortedPairs -}}\n {{ if $index }}, {{ end }}\n {{- $label.Name }}="{{ $label.Value -}}"\n {{- end }}\n {{- end -}}\n )\n {{- end }}',+ "text": '{{ range .Alerts -}}\r\n *Alert:* {{ .Labels.alertname }}{{ if .Labels.severity }} - {{ .Labels.severity }}{{ end }}\r\n\r\n *Summary:* {{ .Annotations.summary }}\r\n *Description:* {{ .Annotations.description }}\r\n *RelatedLogs:* {{ if gt (len .Annotations.related_logs) 0 -}} View in <{{ .Annotations.related_logs }}|logs explorer> {{- end}}\r\n *RelatedTraces:* {{ if gt (len .Annotations.related_traces) 0 -}} View in <{{ .Annotations.related_traces }}|traces explorer> {{- end}}\r\n\r\n *Details:*\r\n {{ range .Labels.SortedPairs -}}\r\n {{- if ne .Name "ruleId" -}}\r\n \u2022 *{{ .Name }}:* {{ .Value }}\r\n {{ end -}}\r\n {{ end -}}\r\n{{ end }}',+ }+ ],+}++msteams_default_config = {+ "msteamsv2_configs": [+ {+ "webhook_url": "msteams/webhook_url", # base_url configured on runtime+ "title": '[{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}] {{ .CommonLabels.alertname }} for {{ .CommonLabels.job }}\n {{- if gt (len .CommonLabels) (len .GroupLabels) -}}\n {{" "}}(\n {{- with .CommonLabels.Remove .GroupLabels.Names }}\n {{- range $index, $label := .SortedPairs -}}\n {{ if $index }}, {{ end }}\n {{- $label.Name }}="{{ $label.Value -}}"\n {{- end }}\n {{- end -}}\n )\n {{- end }}',+ "text": '{{ range .Alerts -}}\r\n *Alert:* {{ .Labels.alertname }}{{ if .Labels.severity }} - {{ .Labels.severity }}{{ end }}\r\n\r\n *Summary:* {{ .Annotations.summary }}\r\n *Description:* {{ .Annotations.description }}\r\n *RelatedLogs:* {{ if gt (len .Annotations.related_logs) 0 -}} View in <{{ .Annotations.related_logs }}|logs explorer> {{- end}}\r\n *RelatedTraces:* {{ if gt (len .Annotations.related_traces) 0 -}} View in <{{ .Annotations.related_traces }}|traces explorer> {{- end}}\r\n\r\n *Details:*\r\n {{ range .Labels.SortedPairs -}}\r\n {{- if ne .Name "ruleId" -}}\r\n \u2022 *{{ .Name }}:* {{ .Value }}\r\n {{ end -}}\r\n {{ end -}}\r\n{{ end }}',+ }+ ],+}++pagerduty_default_config = {+ "pagerduty_configs": [+ {+ "routing_key": "PagerDutyRoutingKey",+ "url": "v2/enqueue", # base_url configured on runtime+ "client": "SigNoz Alert Manager",+ "client_url": "https://enter-signoz-host-n-port-here/alerts",+ "description": '[{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}] {{ .CommonLabels.alertname }} for {{ .CommonLabels.job }}\n\t{{- if gt (len .CommonLabels) (len .GroupLabels) -}}\n\t {{" "}}(\n\t {{- with .CommonLabels.Remove .GroupLabels.Names }}\n\t\t{{- range $index, $label := .SortedPairs -}}\n\t\t {{ if $index }}, {{ end }}\n\t\t {{- $label.Name }}="{{ $label.Value -}}"\n\t\t{{- end }}\n\t {{- end -}}\n\t )\n\t{{- end }}',+ "details": {+ "firing": '{{ template "pagerduty.default.instances" .Alerts.Firing }}',+ "num_firing": "{{ .Alerts.Firing | len }}",+ "num_resolved": "{{ .Alerts.Resolved | len }}",+ "resolved": '{{ template "pagerduty.default.instances" .Alerts.Resolved }}',+ },+ "source": "SigNoz Alert Manager",+ "severity": "{{ (index .Alerts 0).Labels.severity }}",+ }+ ],+}++opsgenie_default_config = {+ "opsgenie_configs": [+ {+ "api_key": "OpsGenieAPIKey",+ "api_url": "/", # base_url configured on runtime+ "description": '{{ if gt (len .Alerts.Firing) 0 -}}\r\n\tAlerts Firing:\r\n\t{{ range .Alerts.Firing }}\r\n\t - Message: {{ .Annotations.description }}\r\n\tLabels:\r\n\t{{ range .Labels.SortedPairs -}}\r\n\t\t{{- if ne .Name "ruleId" }} - {{ .Name }} = {{ .Value }}\r\n\t{{ end -}}\r\n\t{{- end }} Annotations:\r\n\t{{ range .Annotations.SortedPairs }} - {{ .Name }} = {{ .Value }}\r\n\t{{ end }} Source: {{ .GeneratorURL }}\r\n\t{{ end }}\r\n{{- end }}\r\n{{ if gt (len .Alerts.Resolved) 0 -}}\r\n\tAlerts Resolved:\r\n\t{{ range .Alerts.Resolved }}\r\n\t - Message: {{ .Annotations.description }}\r\n\tLabels:\r\n\t{{ range .Labels.SortedPairs -}}\r\n\t\t{{- if ne .Name "ruleId" }} - {{ .Name }} = {{ .Value }}\r\n\t{{ end -}}\r\n\t{{- end }} Annotations:\r\n\t{{ range .Annotations.SortedPairs }} - {{ .Name }} = {{ .Value }}\r\n\t{{ end }} Source: {{ .GeneratorURL }}\r\n\t{{ end }}\r\n{{- end }}',+ "priority": '{{ if eq (index .Alerts 0).Labels.severity "critical" }}P1{{ else if eq (index .Alerts 0).Labels.severity "warning" }}P2{{ else if eq (index .Alerts 0).Labels.severity "info" }}P3{{ else }}P4{{ end }}',+ "message": "{{ .CommonLabels.alertname }}",+ "details": {},+ }+ ],+}++webhook_default_config = {+ "webhook_configs": [+ {+ "url": "webhook/webhook_url", # base_url configured on runtime+ }+ ],+}++email_default_config = {+ "email_configs": [+ {+ "to": "[email protected]",+ "html": '<html><body>{{ range .Alerts -}}\r\n *Alert:* {{ .Labels.alertname }}{{ if .Labels.severity }} - {{ .Labels.severity }}{{ end }}\r\n\r\n *Summary:* {{ .Annotations.summary }}\r\n *Description:* {{ .Annotations.description }}\r\n *RelatedLogs:* {{ if gt (len .Annotations.related_logs) 0 -}} View in <{{ .Annotations.related_logs }}|logs explorer> {{- end}}\r\n *RelatedTraces:* {{ if gt (len .Annotations.related_traces) 0 -}} View in <{{ .Annotations.related_traces }}|traces explorer> {{- end}}\r\n\r\n *Details:*\r\n {{ range .Labels.SortedPairs -}}\r\n {{- if ne .Name "ruleId" -}}\r\n \u2022 *{{ .Name }}:* {{ .Value }}\r\n {{ end -}}\r\n {{ end -}}\r\n{{ end }}</body></html>',+ "headers": {+ "Subject": '[{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}] {{ .CommonLabels.alertname }} for {{ .CommonLabels.job }}\n {{- if gt (len .CommonLabels) (len .GroupLabels) -}}\n {{" "}}(\n {{- with .CommonLabels.Remove .GroupLabels.Names }}\n {{- range $index, $label := .SortedPairs -}}\n {{ if $index }}, {{ end }}\n {{- $label.Name }}="{{ $label.Value -}}"\n {{- end }}\n {{- end -}}\n )\n {{- end }}'+ },+ }+ ],+}++@pytest.fixture(name="notification_channel", scope="package")def notification_channel(network: Network,@@ -67,6 +176,40 @@ def notification_channel()[email protected](name="create_notification_channel", scope="function")+def create_notification_channel(+ signoz: types.SigNoz,+ create_user_admin: None, # pylint: disable=unused-argument+ get_token: Callable[[str, str], str],+) -> Callable[[dict], str]:+ admin_token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)++ channel_ids = []++ def _create_notification_channel(channel_config: dict) -> str:+ response = requests.post(+ signoz.self.host_configs["8080"].get("/api/v1/channels"),+ json=channel_config,+ headers={"Authorization": f"Bearer {admin_token}"},+ timeout=5,+ )+ assert response.status_code == HTTPStatus.CREATED, f"Failed to create channel, Response: {response.text} Response status: {response.status_code}"+ channel_id = response.json()["data"]["id"]+ channel_ids.append(channel_id)+ return channel_id++ yield _create_notification_channel++ for channel_id in channel_ids:+ response = requests.delete(+ signoz.self.host_configs["8080"].get(f"/api/v1/channels/{channel_id}"),+ headers={"Authorization": f"Bearer {admin_token}"},+ timeout=5,+ )+ if response.status_code != HTTPStatus.NO_CONTENT:+ logger.error("Failed to delete channel: %s", {"channel_id": channel_id, "status": response.status_code, "response": response.text})++@pytest.fixture(name="create_webhook_notification_channel", scope="function")def create_webhook_notification_channel(signoz: types.SigNoz,@@ -103,3 +246,19 @@ def create_webhook_notification_channel(return channel_idreturn _create_webhook_notification_channel+++def send_test_notification(signoz: types.SigNoz, token: str, receiver: dict, wait_seconds: int = 90) -> None:+ deadline = time.time() + wait_seconds+ last = None+ while time.time() < deadline:+ last = requests.post(+ signoz.self.host_configs["8080"].get("/api/v1/channels/test"),+ json=receiver,+ headers={"Authorization": f"Bearer {token}"},+ timeout=30,+ )+ if last.status_code == HTTPStatus.NO_CONTENT:+ return+ time.sleep(2)+ raise AssertionError(f"test notification did not succeed within {wait_seconds}s, last response: {last.status_code} {last.text}")diff --git a/tests/fixtures/types.py b/tests/fixtures/types.pyindex 4bc0510b65..a070b1b659 100644--- a/tests/fixtures/types.py+++ b/tests/fixtures/types.py@@ -197,3 +197,40 @@ class AlertTestCase:alert_data: list[AlertData]# list of alert expectations for the test casealert_expectation: AlertExpectation+++@dataclass(frozen=True)+class NotificationValidation:+ # destination type of the notification, either webhook or email+ # slack, msteams, pagerduty, opsgenie, webhook channels send notifications through webhook+ # email channels send notifications through email+ destination_type: Literal["webhook", "email"]+ # validation data for validating the received notification payload+ validation_data: dict[str, any]+++@dataclass(frozen=True)+class AMNotificationExpectation:+ # whether we expect any notifications to be fired or not, false when testing downtime scenarios+ # or don't expect any notifications to be fired in given time period+ should_notify: bool+ # seconds to wait for the notifications to be fired, if no+ # notifications are fired in the expected time, the test will fail+ wait_time_seconds: int+ # list of notifications to expect, as a single rule can trigger multiple notifications+ # spanning across different notifiers+ notification_validations: list[NotificationValidation]+++@dataclass(frozen=True)+class AlertManagerNotificationTestCase:+ # name of the test case+ name: str+ # path to the rule file in testdata directory+ rule_path: str+ # list of alert data that will be inserted into the database for the rule to be triggered+ alert_data: list[AlertData]+ # configuration for the notification channel+ channel_config: dict[str, any]+ # notification expectations for the test case+ notification_expectation: AMNotificationExpectationdiff --git a/tests/integration/bootstrap/setup.py b/tests/integration/bootstrap/setup.pyindex 838553da9f..33b34655c3 100644--- a/tests/integration/bootstrap/setup.py+++ b/tests/integration/bootstrap/setup.py@@ -39,5 +39,7 @@ def test_teardown(idp: types.TestContainerIDP, # pylint: disable=unused-argumentcreate_user_admin: types.Operation, # pylint: disable=unused-argumentmigrator: types.Operation, # pylint: disable=unused-argument+ maildev: types.TestContainerDocker, # pylint: disable=unused-argument+ notification_channel: types.TestContainerDocker, # pylint: disable=unused-argument) -> None:passdiff --git a/tests/integration/testdata/alertmanager/content_templating/logs_data.jsonl b/tests/integration/testdata/alertmanager/content_templating/logs_data.jsonlnew file mode 100644index 0000000000..3c00101e04--- /dev/null+++ b/tests/integration/testdata/alertmanager/content_templating/logs_data.jsonl@@ -0,0 +1,20 @@+{ "timestamp": "2026-01-29T10:00:00.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "User login successful", "severity_text": "INFO" }+{ "timestamp": "2026-01-29T10:00:30.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: gateway timeout", "severity_text": "ERROR" }+{ "timestamp": "2026-01-29T10:01:00.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: card declined", "severity_text": "ERROR" }+{ "timestamp": "2026-01-29T10:01:30.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "Database connection established", "severity_text": "INFO" }+{ "timestamp": "2026-01-29T10:02:00.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: insufficient funds", "severity_text": "ERROR" }+{ "timestamp": "2026-01-29T10:02:30.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: invalid token", "severity_text": "ERROR" }+{ "timestamp": "2026-01-29T10:03:00.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "API request received", "severity_text": "INFO" }+{ "timestamp": "2026-01-29T10:03:30.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: gateway timeout", "severity_text": "ERROR" }+{ "timestamp": "2026-01-29T10:04:00.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: card declined", "severity_text": "ERROR" }+{ "timestamp": "2026-01-29T10:04:30.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: invalid token", "severity_text": "ERROR" }+{ "timestamp": "2026-01-29T10:05:00.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: gateway timeout", "severity_text": "ERROR" }+{ "timestamp": "2026-01-29T10:05:30.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: card declined", "severity_text": "ERROR" }+{ "timestamp": "2026-01-29T10:06:00.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: gateway timeout", "severity_text": "ERROR" }+{ "timestamp": "2026-01-29T10:06:30.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: insufficient funds", "severity_text": "ERROR" }+{ "timestamp": "2026-01-29T10:07:00.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: card declined", "severity_text": "ERROR" }+{ "timestamp": "2026-01-29T10:07:30.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: gateway timeout", "severity_text": "ERROR" }+{ "timestamp": "2026-01-29T10:08:00.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "Response sent to client", "severity_text": "INFO" }+{ "timestamp": "2026-01-29T10:08:30.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: invalid token", "severity_text": "ERROR" }+{ "timestamp": "2026-01-29T10:09:00.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: card declined", "severity_text": "ERROR" }+{ "timestamp": "2026-01-29T10:10:00.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: gateway timeout", "severity_text": "ERROR" }diff --git a/tests/integration/testdata/alertmanager/content_templating/logs_rule.json b/tests/integration/testdata/alertmanager/content_templating/logs_rule.jsonnew file mode 100644index 0000000000..13990c69ea--- /dev/null+++ b/tests/integration/testdata/alertmanager/content_templating/logs_rule.json@@ -0,0 +1,69 @@+{+ "alert": "content_templating_logs",+ "ruleType": "threshold_rule",+ "alertType": "LOGS_BASED_ALERT",+ "condition": {+ "thresholds": {+ "kind": "basic",+ "spec": [+ {+ "name": "critical",+ "target": 0,+ "matchType": "1",+ "op": "1",+ "channels": [+ "test channel"+ ]+ }+ ]+ },+ "compositeQuery": {+ "queryType": "builder",+ "panelType": "graph",+ "queries": [+ {+ "type": "builder_query",+ "spec": {+ "name": "A",+ "signal": "logs",+ "filter": {+ "expression": "body CONTAINS 'payment failure'"+ },+ "aggregations": [+ {+ "expression": "count()"+ }+ ],+ "groupBy": [+ {"name": "service.name", "fieldContext": "resource"}+ ]+ }+ }+ ]+ },+ "selectedQueryName": "A"+ },+ "evaluation": {+ "kind": "rolling",+ "spec": {+ "evalWindow": "5m0s",+ "frequency": "15s"+ }+ },+ "labels": {},+ "annotations": {+ "description": "Payment failure spike detected on $service_name",+ "summary": "Payment failures elevated on $service_name"+ },+ "notificationSettings": {+ "groupBy": [],+ "usePolicy": false,+ "renotify": {+ "enabled": false,+ "interval": "30m",+ "alertStates": []+ }+ },+ "version": "v5",+ "schemaVersion": "v2alpha1"+}diff --git a/tests/integration/testdata/alertmanager/content_templating/metrics_data.jsonl b/tests/integration/testdata/alertmanager/content_templating/metrics_data.jsonlnew file mode 100644index 0000000000..e6da266aac--- /dev/null+++ b/tests/integration/testdata/alertmanager/content_templating/metrics_data.jsonl@@ -0,0 +1,12 @@+{"metric_name":"container_memory_bytes_content_templating","labels":{"namespace":"production","pod":"checkout-7d9c8b5f4-x2k9p","container":"checkout","node":"ip-10-0-1-23","severity":"critical","service":"checkout"},"timestamp":"2026-01-29T10:01:00+00:00","value":80,"temporality":"Unspecified","type_":"Gauge","is_monotonic":false,"flags":0,"description":"","unit":"bytes","env":"default","resource_attrs":{},"scope_attrs":{}}+{"metric_name":"container_memory_bytes_content_templating","labels":{"namespace":"production","pod":"checkout-7d9c8b5f4-x2k9p","container":"checkout","node":"ip-10-0-1-23","severity":"critical","service":"checkout"},"timestamp":"2026-01-29T10:02:00+00:00","value":95,"temporality":"Unspecified","type_":"Gauge","is_monotonic":false,"flags":0,"description":"","unit":"bytes","env":"default","resource_attrs":{},"scope_attrs":{}}+{"metric_name":"container_memory_bytes_content_templating","labels":{"namespace":"production","pod":"checkout-7d9c8b5f4-x2k9p","container":"checkout","node":"ip-10-0-1-23","severity":"critical","service":"checkout"},"timestamp":"2026-01-29T10:03:00+00:00","value":110,"temporality":"Unspecified","type_":"Gauge","is_monotonic":false,"flags":0,"description":"","unit":"bytes","env":"default","resource_attrs":{},"scope_attrs":{}}+{"metric_name":"container_memory_bytes_content_templating","labels":{"namespace":"production","pod":"checkout-7d9c8b5f4-x2k9p","container":"checkout","node":"ip-10-0-1-23","severity":"critical","service":"checkout"},"timestamp":"2026-01-29T10:04:00+00:00","value":120,"temporality":"Unspecified","type_":"Gauge","is_monotonic":false,"flags":0,"description":"","unit":"bytes","env":"default","resource_attrs":{},"scope_attrs":{}}+{"metric_name":"container_memory_bytes_content_templating","labels":{"namespace":"production","pod":"checkout-7d9c8b5f4-x2k9p","container":"checkout","node":"ip-10-0-1-23","severity":"critical","service":"checkout"},"timestamp":"2026-01-29T10:05:00+00:00","value":125,"temporality":"Unspecified","type_":"Gauge","is_monotonic":false,"flags":0,"description":"","unit":"bytes","env":"default","resource_attrs":{},"scope_attrs":{}}+{"metric_name":"container_memory_bytes_content_templating","labels":{"namespace":"production","pod":"checkout-7d9c8b5f4-x2k9p","container":"checkout","node":"ip-10-0-1-23","severity":"critical","service":"checkout"},"timestamp":"2026-01-29T10:06:00+00:00","value":130,"temporality":"Unspecified","type_":"Gauge","is_monotonic":false,"flags":0,"description":"","unit":"bytes","env":"default","resource_attrs":{},"scope_attrs":{}}+{"metric_name":"container_memory_bytes_content_templating","labels":{"namespace":"production","pod":"checkout-7d9c8b5f4-x2k9p","container":"checkout","node":"ip-10-0-1-23","severity":"critical","service":"checkout"},"timestamp":"2026-01-29T10:07:00+00:00","value":135,"temporality":"Unspecified","type_":"Gauge","is_monotonic":false,"flags":0,"description":"","unit":"bytes","env":"default","resource_attrs":{},"scope_attrs":{}}+{"metric_name":"container_memory_bytes_content_templating","labels":{"namespace":"production","pod":"checkout-7d9c8b5f4-x2k9p","container":"checkout","node":"ip-10-0-1-23","severity":"critical","service":"checkout"},"timestamp":"2026-01-29T10:08:00+00:00","value":140,"temporality":"Unspecified","type_":"Gauge","is_monotonic":false,"flags":0,"description":"","unit":"bytes","env":"default","resource_attrs":{},"scope_attrs":{}}+{"metric_name":"container_memory_bytes_content_templating","labels":{"namespace":"production","pod":"checkout-7d9c8b5f4-x2k9p","container":"checkout","node":"ip-10-0-1-23","severity":"critical","service":"checkout"},"timestamp":"2026-01-29T10:09:00+00:00","value":145,"temporality":"Unspecified","type_":"Gauge","is_monotonic":false,"flags":0,"description":"","unit":"bytes","env":"default","resource_attrs":{},"scope_attrs":{}}+{"metric_name":"container_memory_bytes_content_templating","labels":{"namespace":"production","pod":"checkout-7d9c8b5f4-x2k9p","container":"checkout","node":"ip-10-0-1-23","severity":"critical","service":"checkout"},"timestamp":"2026-01-29T10:10:00+00:00","value":150,"temporality":"Unspecified","type_":"Gauge","is_monotonic":false,"flags":0,"description":"","unit":"bytes","env":"default","resource_attrs":{},"scope_attrs":{}}+{"metric_name":"container_memory_bytes_content_templating","labels":{"namespace":"production","pod":"checkout-7d9c8b5f4-x2k9p","container":"checkout","node":"ip-10-0-1-23","severity":"critical","service":"checkout"},"timestamp":"2026-01-29T10:11:00+00:00","value":155,"temporality":"Unspecified","type_":"Gauge","is_monotonic":false,"flags":0,"description":"","unit":"bytes","env":"default","resource_attrs":{},"scope_attrs":{}}+{"metric_name":"container_memory_bytes_content_templating","labels":{"namespace":"production","pod":"checkout-7d9c8b5f4-x2k9p","container":"checkout","node":"ip-10-0-1-23","severity":"critical","service":"checkout"},"timestamp":"2026-01-29T10:12:00+00:00","value":160,"temporality":"Unspecified","type_":"Gauge","is_monotonic":false,"flags":0,"description":"","unit":"bytes","env":"default","resource_attrs":{},"scope_attrs":{}}diff --git a/tests/integration/testdata/alertmanager/content_templating/metrics_rule.json b/tests/integration/testdata/alertmanager/content_templating/metrics_rule.jsonnew file mode 100644index 0000000000..da1ee50016--- /dev/null+++ b/tests/integration/testdata/alertmanager/content_templating/metrics_rule.json@@ -0,0 +1,72 @@+{+ "alert": "content_templating_metrics",+ "ruleType": "threshold_rule",+ "alertType": "METRIC_BASED_ALERT",+ "condition": {+ "thresholds": {+ "kind": "basic",+ "spec": [+ {+ "name": "critical",+ "target": 100,+ "matchType": "1",+ "op": "1",+ "channels": [+ "test channel"+ ]+ }+ ]+ },+ "compositeQuery": {+ "queryType": "builder",+ "panelType": "graph",+ "queries": [+ {+ "type": "builder_query",+ "spec": {+ "name": "A",+ "signal": "metrics",+ "aggregations": [+ {+ "metricName": "container_memory_bytes_content_templating",+ "timeAggregation": "avg",+ "spaceAggregation": "max"+ }+ ],+ "groupBy": [+ {"name": "namespace", "fieldContext": "attribute", "fieldDataType": "string"},+ {"name": "pod", "fieldContext": "attribute", "fieldDataType": "string"},+ {"name": "container", "fieldContext": "attribute", "fieldDataType": "string"},+ {"name": "node", "fieldContext": "attribute", "fieldDataType": "string"},+ {"name": "severity", "fieldContext": "attribute", "fieldDataType": "string"}+ ]+ }+ }+ ]+ },+ "selectedQueryName": "A"+ },+ "evaluation": {+ "kind": "rolling",+ "spec": {+ "evalWindow": "5m0s",+ "frequency": "15s"+ }+ },+ "labels": {},+ "annotations": {+ "description": "Container $container in pod $pod ($namespace) exceeded memory threshold",+ "summary": "High container memory in $namespace/$pod"+ },+ "notificationSettings": {+ "groupBy": [],+ "usePolicy": false,+ "renotify": {+ "enabled": false,+ "interval": "30m",+ "alertStates": []+ }+ },+ "version": "v5",+ "schemaVersion": "v2alpha1"+}diff --git a/tests/integration/testdata/alertmanager/content_templating/traces_data.jsonl b/tests/integration/testdata/alertmanager/content_templating/traces_data.jsonlnew file mode 100644index 0000000000..1b29314822--- /dev/null+++ b/tests/integration/testdata/alertmanager/content_templating/traces_data.jsonl@@ -0,0 +1,20 @@+{ "timestamp": "2026-01-29T10:00:00.000000Z", "duration": "PT1.2S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6a1", "span_id": "c1b2c3d4e5f6a7b8", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }+{ "timestamp": "2026-01-29T10:00:30.000000Z", "duration": "PT1.4S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6a2", "span_id": "c2b3c4d5e6f7a8b9", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }+{ "timestamp": "2026-01-29T10:01:00.000000Z", "duration": "PT1.6S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6a3", "span_id": "c3b4c5d6e7f8a9b0", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }+{ "timestamp": "2026-01-29T10:01:30.000000Z", "duration": "PT1.8S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6a4", "span_id": "c4b5c6d7e8f9a0b1", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }+{ "timestamp": "2026-01-29T10:02:00.000000Z", "duration": "PT2.1S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6a5", "span_id": "c5b6c7d8e9f0a1b2", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }+{ "timestamp": "2026-01-29T10:02:30.000000Z", "duration": "PT2.3S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6a6", "span_id": "c6b7c8d9e0f1a2b3", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }+{ "timestamp": "2026-01-29T10:03:00.000000Z", "duration": "PT2.5S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6a7", "span_id": "c7b8c9d0e1f2a3b4", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }+{ "timestamp": "2026-01-29T10:03:30.000000Z", "duration": "PT2.7S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6a8", "span_id": "c8b9c0d1e2f3a4b5", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }+{ "timestamp": "2026-01-29T10:04:00.000000Z", "duration": "PT2.9S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6a9", "span_id": "c9b0c1d2e3f4a5b6", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }+{ "timestamp": "2026-01-29T10:04:30.000000Z", "duration": "PT3.1S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6b1", "span_id": "d1c2d3e4f5a6b7c8", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }+{ "timestamp": "2026-01-29T10:05:00.000000Z", "duration": "PT3.3S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6b2", "span_id": "d2c3d4e5f6a7b8c9", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }+{ "timestamp": "2026-01-29T10:05:30.000000Z", "duration": "PT3.5S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6b3", "span_id": "d3c4d5e6f7a8b9c0", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }+{ "timestamp": "2026-01-29T10:06:00.000000Z", "duration": "PT3.7S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6b4", "span_id": "d4c5d6e7f8a9b0c1", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }+{ "timestamp": "2026-01-29T10:06:30.000000Z", "duration": "PT3.9S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6b5", "span_id": "d5c6d7e8f9a0b1c2", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }+{ "timestamp": "2026-01-29T10:07:00.000000Z", "duration": "PT4.1S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6b6", "span_id": "d6c7d8e9f0a1b2c3", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }+{ "timestamp": "2026-01-29T10:07:30.000000Z", "duration": "PT4.3S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6b7", "span_id": "d7c8d9e0f1a2b3c4", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }+{ "timestamp": "2026-01-29T10:08:00.000000Z", "duration": "PT4.5S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6b8", "span_id": "d8c9d0e1f2a3b4c5", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }+{ "timestamp": "2026-01-29T10:08:30.000000Z", "duration": "PT4.7S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6b9", "span_id": "d9c0d1e2f3a4b5c6", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }+{ "timestamp": "2026-01-29T10:09:00.000000Z", "duration": "PT4.9S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6c1", "span_id": "e1d2e3f4a5b6c7d8", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }+{ "timestamp": "2026-01-29T10:10:00.000000Z", "duration": "PT5.1S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6c2", "span_id": "e2d3e4f5a6b7c8d9", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }diff --git a/tests/integration/testdata/alertmanager/content_templating/traces_rule.json b/tests/integration/testdata/alertmanager/content_templating/traces_rule.jsonnew file mode 100644index 0000000000..4fd28f6647--- /dev/null+++ b/tests/integration/testdata/alertmanager/content_templating/traces_rule.json@@ -0,0 +1,71 @@+{+ "alert": "content_templating_traces",+ "ruleType": "threshold_rule",+ "alertType": "TRACES_BASED_ALERT",+ "condition": {+ "thresholds": {+ "kind": "basic",+ "spec": [+ {+ "name": "critical",+ "target": 1,+ "matchType": "1",+ "op": "1",+ "channels": [+ "test channel"+ ],+ "targetUnit": "s"+ }+ ]+ },+ "compositeQuery": {+ "queryType": "builder",+ "unit": "ns",+ "panelType": "graph",+ "queries": [+ {+ "type": "builder_query",+ "spec": {+ "name": "A",+ "signal": "traces",+ "filter": {+ "expression": "http.request.path = '/checkout'"+ },+ "aggregations": [+ {+ "expression": "p90(duration_nano)"+ }+ ],+ "groupBy": [+ {"name": "service.name", "fieldContext": "resource"}+ ]+ }+ }+ ]+ },+ "selectedQueryName": "A"+ },+ "evaluation": {+ "kind": "rolling",+ "spec": {+ "evalWindow": "5m0s",+ "frequency": "15s"+ }+ },+ "labels": {},+ "annotations": {+ "description": "p90 latency high on $service_name",+ "summary": "p90 latency exceeded threshold on $service_name"+ },+ "notificationSettings": {+ "groupBy": [],+ "usePolicy": false,+ "renotify": {+ "enabled": false,+ "interval": "30m",+ "alertStates": []+ }+ },+ "version": "v5",+ "schemaVersion": "v2alpha1"+}diff --git a/tests/integration/tests/alertmanager/01_channels.py b/tests/integration/tests/alertmanager/01_channels.pynew file mode 100644index 0000000000..2dba0c88a8--- /dev/null+++ b/tests/integration/tests/alertmanager/01_channels.py@@ -0,0 +1,268 @@+import time+import uuid+from collections.abc import Callable+from http import HTTPStatus++import pytest+import requests+from sqlalchemy import text++from fixtures import types+from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD+from fixtures.maildev import (+ MAILDEV_INCOMING_PASS,+ SMTP_TEST_FROM,+ delete_all_mails,+ verify_email_received,+)+from fixtures.notification_channel import assert_email_channel_payload_clean, send_test_notification++TIMEOUT = 10+++CHANNEL_TYPE_CASES = [+ (+ "webhook",+ lambda sink: {"webhook_configs": [{"url": sink.container_configs["8080"].get("/webhook/crud-original"), "send_resolved": True}]},+ lambda sink: {"webhook_configs": [{"url": sink.container_configs["8080"].get("/webhook/crud-updated"), "send_resolved": True}]},+ "crud-original",+ "crud-updated",+ ),+ (+ "slack",+ lambda sink: {"slack_configs": [{"api_url": sink.container_configs["8080"].get("/services/T/B/X"), "channel": "#crud-original"}]},+ lambda sink: {"slack_configs": [{"api_url": sink.container_configs["8080"].get("/services/T/B/X"), "channel": "#crud-updated"}]},+ "#crud-original",+ "#crud-updated",+ ),+ (+ "pagerduty",+ lambda sink: {"pagerduty_configs": [{"routing_key": "crud-original-routing-key"}]},+ lambda sink: {"pagerduty_configs": [{"routing_key": "crud-updated-routing-key"}]},+ "crud-original-routing-key",+ "crud-updated-routing-key",+ ),+ (+ "opsgenie",+ lambda sink: {"opsgenie_configs": [{"api_key": "crud-original-api-key", "message": "{{ .CommonLabels.alertname }}"}]},+ lambda sink: {"opsgenie_configs": [{"api_key": "crud-updated-api-key", "message": "{{ .CommonLabels.alertname }}"}]},+ "crud-original-api-key",+ "crud-updated-api-key",+ ),+ (+ "msteamsv2",+ lambda sink: {"msteamsv2_configs": [{"webhook_url": sink.container_configs["8080"].get("/msteams/crud-original")}]},+ lambda sink: {"msteamsv2_configs": [{"webhook_url": sink.container_configs["8080"].get("/msteams/crud-updated")}]},+ "crud-original",+ "crud-updated",+ ),+ (+ "email",+ lambda sink: {"email_configs": [{"to": "[email protected]"}]},+ lambda sink: {"email_configs": [{"to": "[email protected]"}]},+ "[email protected]",+ "[email protected]",+ ),+]+++ "channel_type,make_config,make_updated_config,created_marker,updated_marker",+ CHANNEL_TYPE_CASES,+ ids=[case[0] for case in CHANNEL_TYPE_CASES],+)+def test_channel_crud( # pylint: disable=too-many-arguments,too-many-positional-arguments+ signoz: types.SigNoz,+ create_user_admin: None, # pylint: disable=unused-argument+ get_token: Callable[[str, str], str],+ notification_channel: types.TestContainerDocker,+ channel_type: str,+ make_config: Callable[[types.TestContainerDocker], dict],+ make_updated_config: Callable[[types.TestContainerDocker], dict],+ created_marker: str,+ updated_marker: str,+) -> None:+ token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)+ name = f"crud-{channel_type}-{uuid.uuid4()}"++ config = {"name": name, **make_config(notification_channel)}+ response = requests.post(signoz.self.host_configs["8080"].get("/api/v1/channels"), json=config, headers={"Authorization": f"Bearer {token}"}, timeout=TIMEOUT)+ assert response.status_code == HTTPStatus.CREATED, response.text+ created = response.json()["data"]+ channel_id = created["id"]+ assert created["name"] == name+ assert created["type"] == channel_type++ response = requests.get(signoz.self.host_configs["8080"].get("/api/v1/channels"), headers={"Authorization": f"Bearer {token}"}, timeout=TIMEOUT)+ assert response.status_code == HTTPStatus.OK, response.text+ listed = {channel["name"]: channel for channel in response.json()["data"]}+ assert name in listed+ assert listed[name]["type"] == channel_type++ response = requests.get(signoz.self.host_configs["8080"].get(f"/api/v1/channels/{channel_id}"), headers={"Authorization": f"Bearer {token}"}, timeout=TIMEOUT)+ assert response.status_code == HTTPStatus.OK, response.text+ assert created_marker in response.json()["data"]["data"]++ updated_config = {"name": name, **make_updated_config(notification_channel)}+ response = requests.put(signoz.self.host_configs["8080"].get(f"/api/v1/channels/{channel_id}"), json=updated_config, headers={"Authorization": f"Bearer {token}"}, timeout=TIMEOUT)+ assert response.status_code == HTTPStatus.NO_CONTENT, response.text++ response = requests.get(signoz.self.host_configs["8080"].get(f"/api/v1/channels/{channel_id}"), headers={"Authorization": f"Bearer {token}"}, timeout=TIMEOUT)+ assert response.status_code == HTTPStatus.OK, response.text+ data = response.json()["data"]["data"]+ assert updated_marker in data+ assert created_marker not in data++ response = requests.delete(signoz.self.host_configs["8080"].get(f"/api/v1/channels/{channel_id}"), headers={"Authorization": f"Bearer {token}"}, timeout=TIMEOUT)+ assert response.status_code == HTTPStatus.NO_CONTENT, response.text++ response = requests.get(signoz.self.host_configs["8080"].get(f"/api/v1/channels/{channel_id}"), headers={"Authorization": f"Bearer {token}"}, timeout=TIMEOUT)+ assert response.status_code == HTTPStatus.NOT_FOUND, response.text+++def test_create_rejects_duplicate_name(+ signoz: types.SigNoz,+ create_user_admin: None, # pylint: disable=unused-argument+ get_token: Callable[[str, str], str],+ create_notification_channel: Callable[[dict], str],+) -> None:+ token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)+ name = f"duplicate-{uuid.uuid4()}"++ create_notification_channel({"name": name, "email_configs": [{"to": "[email protected]"}]})++ response = requests.post(+ signoz.self.host_configs["8080"].get("/api/v1/channels"),+ json={"name": name, "email_configs": [{"to": "[email protected]"}]},+ headers={"Authorization": f"Bearer {token}"},+ timeout=TIMEOUT,+ )+ assert response.status_code == HTTPStatus.BAD_REQUEST, response.text+ assert "unique" in response.text+++def test_create_rejects_channel_without_configs(+ signoz: types.SigNoz,+ create_user_admin: None, # pylint: disable=unused-argument+ get_token: Callable[[str, str], str],+) -> None:+ token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)++ response = requests.post(+ signoz.self.host_configs["8080"].get("/api/v1/channels"),+ json={"name": f"empty-{uuid.uuid4()}"},+ headers={"Authorization": f"Bearer {token}"},+ timeout=TIMEOUT,+ )+ assert response.status_code == HTTPStatus.BAD_REQUEST, response.text+ assert "notification configuration" in response.text+++def test_update_rejects_name_change(+ signoz: types.SigNoz,+ create_user_admin: None, # pylint: disable=unused-argument+ get_token: Callable[[str, str], str],+ create_notification_channel: Callable[[dict], str],+) -> None:+ token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)+ name = f"rename-{uuid.uuid4()}"+ channel_id = create_notification_channel({"name": name, "email_configs": [{"to": "[email protected]"}]})++ response = requests.put(+ signoz.self.host_configs["8080"].get(f"/api/v1/channels/{channel_id}"),+ json={"name": f"{name}-renamed", "email_configs": [{"to": "[email protected]"}]},+ headers={"Authorization": f"Bearer {token}"},+ timeout=TIMEOUT,+ )+ assert response.status_code == HTTPStatus.BAD_REQUEST, response.text+ assert "cannot update channel name" in response.text+++def test_channels_require_authentication(signoz: types.SigNoz) -> None:+ response = requests.get(signoz.self.host_configs["8080"].get("/api/v1/channels"), timeout=TIMEOUT)+ assert response.status_code == HTTPStatus.UNAUTHORIZED, response.text+++def test_email_channel_never_stores_or_serves_smtp_settings(+ signoz: types.SigNoz,+ create_user_admin: None, # pylint: disable=unused-argument+ get_token: Callable[[str, str], str],+) -> None:+ token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)++ hostile_name = f"hostile-email-{uuid.uuid4()}"+ hostile_config = {+ "name": hostile_name,+ "email_configs": [+ {+ "to": "[email protected]",+ "from": "[email protected]",+ "hello": "attacker.test",+ "smarthost": "smtp.attacker.test:2525",+ "auth_username": "attacker",+ "auth_password": "tenant-posted-secret",+ "require_tls": False,+ "headers": {"Subject": "hostile subject"},+ }+ ],+ }++ response = requests.post(signoz.self.host_configs["8080"].get("/api/v1/channels"), json=hostile_config, headers={"Authorization": f"Bearer {token}"}, timeout=TIMEOUT)+ assert response.status_code == HTTPStatus.CREATED, response.text+ created = response.json()["data"]+ assert_email_channel_payload_clean(created["data"])++ response = requests.get(signoz.self.host_configs["8080"].get(f"/api/v1/channels/{created['id']}"), headers={"Authorization": f"Bearer {token}"}, timeout=TIMEOUT)+ assert response.status_code == HTTPStatus.OK, response.text+ served = response.json()["data"]["data"]+ assert_email_channel_payload_clean(served)+ assert "[email protected]" in served+ assert "hostile subject" in served+ assert "smtp.attacker.test" not in served+ assert "tenant-posted-secret" not in served++ response = requests.get(signoz.self.host_configs["8080"].get("/api/v1/channels"), headers={"Authorization": f"Bearer {token}"}, timeout=TIMEOUT)+ assert response.status_code == HTTPStatus.OK, response.text+ assert "tenant-posted-secret" not in response.text+ assert MAILDEV_INCOMING_PASS not in response.text++ with signoz.sqlstore.conn.connect() as conn:+ stored = conn.execute(+ text("SELECT data FROM notification_channel WHERE name = :name"),+ {"name": hostile_name},+ ).fetchone()+ assert stored is not None+ assert_email_channel_payload_clean(stored[0])+ assert "tenant-posted-secret" not in stored[0]++ configs = conn.execute(text("SELECT config FROM alertmanager_config")).fetchall()+ assert len(configs) > 0+ for (config_raw,) in configs:+ assert MAILDEV_INCOMING_PASS not in config_raw+ assert "tenant-posted-secret" not in config_raw+ assert '"smtp_auth_password"' not in config_raw+ assert '"auth_password"' not in config_raw+++def test_email_test_channel_delivers_via_env_transport(+ signoz: types.SigNoz,+ create_user_admin: None, # pylint: disable=unused-argument+ get_token: Callable[[str, str], str],+ maildev: types.TestContainerDocker,+) -> None:+ token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)+ delete_all_mails(maildev)++ recipient = f"delivery-{uuid.uuid4()}@integration.test"+ send_test_notification(+ signoz,+ token,+ {"name": f"delivery-{uuid.uuid4()}", "email_configs": [{"to": recipient}]},+ )++ deadline = time.time() + 30+ while time.time() < deadline:+ if verify_email_received(maildev, {"to": recipient, "from": SMTP_TEST_FROM}):+ return+ time.sleep(1)+ raise AssertionError(f"no email delivered to {recipient} from {SMTP_TEST_FROM}")diff --git a/tests/integration/tests/alertmanager/02_notifiers.py b/tests/integration/tests/alertmanager/02_notifiers.pynew file mode 100644index 0000000000..4b15029728--- /dev/null+++ b/tests/integration/tests/alertmanager/02_notifiers.py@@ -0,0 +1,360 @@+import json+import re+import time+import uuid+from collections.abc import Callable+from datetime import UTC, datetime, timedelta++import pytest+from wiremock.client import HttpMethods, Mapping, MappingRequest, MappingResponse++from fixtures import types+from fixtures.alerts import (+ get_testdata_file_path,+ update_raw_channel_config,+ update_rule_channel_name,+ verify_notification_expectation,+)+from fixtures.logger import setup_logger+from fixtures.maildev import delete_all_mails+from fixtures.notification_channel import (+ email_default_config,+ msteams_default_config,+ opsgenie_default_config,+ pagerduty_default_config,+ slack_default_config,+ webhook_default_config,+)++logger = setup_logger(__name__)++NOTIFIERS_TEST = [+ types.AlertManagerNotificationTestCase(+ name="slack_notifier_default_templating",+ rule_path="alerts/test_scenarios/threshold_above_at_least_once/rule.json",+ alert_data=[+ types.AlertData(+ type="metrics",+ data_path="alerts/test_scenarios/threshold_above_at_least_once/alert_data.jsonl",+ ),+ ],+ channel_config=slack_default_config,+ notification_expectation=types.AMNotificationExpectation(+ should_notify=True,+ # extra wait for alertmanager server setup+ wait_time_seconds=120,+ notification_validations=[+ types.NotificationValidation(+ destination_type="webhook",+ validation_data={+ "path": "/services/TEAM_ID/BOT_ID/TOKEN_ID",+ "json_body": {+ "username": "Alertmanager",+ "attachments": [+ {+ "color": "danger",+ "mrkdwn_in": ["fallback", "pretext", "text"],+ }+ ],+ },+ },+ ),+ ],+ ),+ ),+ types.AlertManagerNotificationTestCase(+ name="msteams_notifier_default_templating",+ rule_path="alerts/test_scenarios/threshold_above_at_least_once/rule.json",+ alert_data=[+ types.AlertData(+ type="metrics",+ data_path="alerts/test_scenarios/threshold_above_at_least_once/alert_data.jsonl",+ ),+ ],+ channel_config=msteams_default_config,+ notification_expectation=types.AMNotificationExpectation(+ should_notify=True,+ wait_time_seconds=120,+ notification_validations=[+ types.NotificationValidation(+ destination_type="webhook",+ validation_data={+ "path": "/msteams/webhook_url",+ "json_body": {+ "type": "message",+ "attachments": [+ {+ "contentType": "application/vnd.microsoft.card.adaptive",+ "content": {+ "$schema": "http://adaptivecards.io/schemas/adaptive-card.json",+ "type": "AdaptiveCard",+ "version": "1.2",+ "body": [+ {+ "type": "TextBlock",+ "text": "Alerts",+ "weight": "Bolder",+ "size": "Medium",+ "wrap": True,+ "color": "Attention",+ },+ {+ "type": "TextBlock",+ "text": "Labels",+ "weight": "Bolder",+ "size": "Medium",+ },+ {+ "type": "FactSet",+ "text": "",+ "facts": [+ {+ "title": "threshold.name",+ "value": "critical",+ }+ ],+ },+ {+ "type": "TextBlock",+ "text": "Annotations",+ "weight": "Bolder",+ "size": "Medium",+ },+ {+ "type": "FactSet",+ "text": "",+ "facts": [+ {+ "title": "description",+ "value": "This alert is fired when the defined metric (current value: 15) crosses the threshold (10)",+ },+ ],+ },+ ],+ "msteams": {"width": "full"},+ "actions": [+ {+ "type": "Action.OpenUrl",+ "title": "View Alert",+ }+ ],+ },+ }+ ],+ },+ },+ ),+ ],+ ),+ ),+ types.AlertManagerNotificationTestCase(+ name="pagerduty_notifier_default_templating",+ rule_path="alerts/test_scenarios/threshold_above_at_least_once/rule.json",+ alert_data=[+ types.AlertData(+ type="metrics",+ data_path="alerts/test_scenarios/threshold_above_at_least_once/alert_data.jsonl",+ ),+ ],+ channel_config=pagerduty_default_config,+ notification_expectation=types.AMNotificationExpectation(+ should_notify=True,+ wait_time_seconds=120,+ notification_validations=[+ types.NotificationValidation(+ destination_type="webhook",+ validation_data={+ "path": "/v2/enqueue",+ "json_body": {+ "routing_key": "PagerDutyRoutingKey",+ "event_action": "trigger",+ "payload": {+ "source": "SigNoz Alert Manager",+ "severity": "critical",+ "custom_details": {+ "firing": {+ "Annotations": [+ {"description = This alert is fired when the defined metric (current value": "15) crosses the threshold (10)"},+ ],+ "Labels": [+ "alertname = threshold_above_at_least_once",+ "severity = critical",+ "threshold.name = critical",+ ],+ }+ },+ },+ "client": "SigNoz Alert Manager",+ "client_url": "https://enter-signoz-host-n-port-here/alerts",+ },+ },+ ),+ ],+ ),+ ),+ types.AlertManagerNotificationTestCase(+ name="opsgenie_notifier_default_templating",+ rule_path="alerts/test_scenarios/threshold_above_at_least_once/rule.json",+ alert_data=[+ types.AlertData(+ type="metrics",+ data_path="alerts/test_scenarios/threshold_above_at_least_once/alert_data.jsonl",+ ),+ ],+ channel_config=opsgenie_default_config,+ notification_expectation=types.AMNotificationExpectation(+ should_notify=True,+ wait_time_seconds=120,+ notification_validations=[+ types.NotificationValidation(+ destination_type="webhook",+ validation_data={+ "path": "/v2/alerts",+ "json_body": {+ "message": "threshold_above_at_least_once",+ "details": {+ "alertname": "threshold_above_at_least_once",+ "severity": "critical",+ "threshold.name": "critical",+ },+ "priority": "P1",+ },+ },+ ),+ ],+ ),+ ),+ types.AlertManagerNotificationTestCase(+ name="webhook_notifier_default_templating",+ rule_path="alerts/test_scenarios/threshold_above_at_least_once/rule.json",+ alert_data=[+ types.AlertData(+ type="metrics",+ data_path="alerts/test_scenarios/threshold_above_at_least_once/alert_data.jsonl",+ ),+ ],+ channel_config=webhook_default_config,+ notification_expectation=types.AMNotificationExpectation(+ should_notify=True,+ wait_time_seconds=120,+ notification_validations=[+ types.NotificationValidation(+ destination_type="webhook",+ validation_data={+ "path": "/webhook/webhook_url",+ "json_body": {+ "status": "firing",+ "alerts": [+ {+ "status": "firing",+ "labels": {+ "alertname": "threshold_above_at_least_once",+ "severity": "critical",+ "threshold.name": "critical",+ },+ "annotations": {+ "description": "This alert is fired when the defined metric (current value: 15) crosses the threshold (10)",+ "summary": "This alert is fired when the defined metric (current value: 15) crosses the threshold (10)",+ },+ }+ ],+ "commonLabels": {+ "alertname": "threshold_above_at_least_once",+ "severity": "critical",+ "threshold.name": "critical",+ },+ "commonAnnotations": {+ "description": "This alert is fired when the defined metric (current value: 15) crosses the threshold (10)",+ "summary": "This alert is fired when the defined metric (current value: 15) crosses the threshold (10)",+ },+ },+ },+ ),+ ],+ ),+ ),+ types.AlertManagerNotificationTestCase(+ name="email_notifier_default_templating",+ rule_path="alerts/test_scenarios/threshold_above_at_least_once/rule.json",+ alert_data=[+ types.AlertData(+ type="metrics",+ data_path="alerts/test_scenarios/threshold_above_at_least_once/alert_data.jsonl",+ ),+ ],+ channel_config=email_default_config,+ notification_expectation=types.AMNotificationExpectation(+ should_notify=True,+ wait_time_seconds=120,+ notification_validations=[+ types.NotificationValidation(+ destination_type="email",+ validation_data={+ "subject": re.compile(r'\[FIRING:1\] threshold_above_at_least_once for \(alertname="threshold_above_at_least_once", ruleSource="http://localhost:8080/alerts/overview\?ruleId=[0-9a-f-]+", severity="critical", threshold\.name="critical"\)'),+ },+ ),+ ],+ ),+ ),+]+++ "notifier_test_case",+ NOTIFIERS_TEST,+ ids=lambda notifier_test_case: notifier_test_case.name,+)+def test_notifier_templating( # pylint: disable=too-many-arguments,too-many-positional-arguments+ notification_channel: types.TestContainerDocker,+ make_http_mocks: Callable[[types.TestContainerDocker, list[Mapping]], None],+ create_notification_channel: Callable[[dict], str],+ create_alert_rule: Callable[[dict], str],+ insert_alert_data: Callable[[list[types.AlertData], datetime], None],+ maildev: types.TestContainerDocker,+ notifier_test_case: types.AlertManagerNotificationTestCase,+):+ channel_name = str(uuid.uuid4())++ channel_config = update_raw_channel_config(notifier_test_case.channel_config, channel_name, notification_channel)+ logger.info("Channel config: %s", {"channel_config": channel_config})++ webhook_validations = [v for v in notifier_test_case.notification_expectation.notification_validations if v.destination_type == "webhook"]+ if len(webhook_validations) > 0:+ mock_mappings = [+ Mapping(+ request=MappingRequest(method=HttpMethods.POST, url=v.validation_data["path"]),+ response=MappingResponse(status=200, json_body={}),+ persistent=False,+ )+ for v in webhook_validations+ ]++ make_http_mocks(notification_channel, mock_mappings)+ logger.info("Mock mappings created")++ if any(v.destination_type == "email" for v in notifier_test_case.notification_expectation.notification_validations):+ delete_all_mails(maildev)+ logger.info("Mails deleted")++ create_notification_channel(channel_config)+ logger.info("Channel created with name: %s", {"channel_name": channel_name})++ time.sleep(12)++ insert_alert_data(+ notifier_test_case.alert_data,+ base_time=datetime.now(tz=UTC) - timedelta(minutes=5),+ )++ rule_path = get_testdata_file_path(notifier_test_case.rule_path)+ with open(rule_path, encoding="utf-8") as f:+ rule_data = json.loads(f.read())+ update_rule_channel_name(rule_data, channel_name)+ rule_id = create_alert_rule(rule_data)+ logger.info("rule created: %s", {"rule_id": rule_id, "rule_name": rule_data["alert"]})++ verify_notification_expectation(+ notification_channel,+ maildev,+ notifier_test_case.notification_expectation,+ )diff --git a/tests/integration/tests/alertmanager/03_content_templating.py b/tests/integration/tests/alertmanager/03_content_templating.pynew file mode 100644index 0000000000..feeef400cc--- /dev/null+++ b/tests/integration/tests/alertmanager/03_content_templating.py@@ -0,0 +1,332 @@+import json+import re+import time+import uuid+from collections.abc import Callable+from datetime import UTC, datetime, timedelta++import pytest+from wiremock.client import HttpMethods, Mapping, MappingRequest, MappingResponse++from fixtures import types+from fixtures.alerts import (+ get_testdata_file_path,+ update_raw_channel_config,+ update_rule_channel_name,+ verify_notification_expectation,+)+from fixtures.logger import setup_logger+from fixtures.maildev import delete_all_mails+from fixtures.notification_channel import (+ msteams_default_config,+ opsgenie_default_config,+ pagerduty_default_config,+ slack_default_config,+ webhook_default_config,+)++logger = setup_logger(__name__)+++CONTENT_TEMPLATING_TEST = [+ types.AlertManagerNotificationTestCase(+ name="msteams_metrics_default_templating",+ rule_path="alertmanager/content_templating/metrics_rule.json",+ alert_data=[+ types.AlertData(+ type="metrics",+ data_path="alertmanager/content_templating/metrics_data.jsonl",+ ),+ ],+ channel_config=msteams_default_config,+ notification_expectation=types.AMNotificationExpectation(+ should_notify=True,+ wait_time_seconds=120,+ notification_validations=[+ types.NotificationValidation(+ destination_type="webhook",+ validation_data={+ "path": "/msteams/webhook_url",+ "json_body": {+ "type": "message",+ "attachments": [+ {+ "contentType": "application/vnd.microsoft.card.adaptive",+ "content": {+ "type": "AdaptiveCard",+ "body": [+ {+ "type": "TextBlock",+ "text": re.compile(+ r'\[FIRING:1\] content_templating_metrics for \(alertname="content_templating_metrics", container="checkout", namespace="production", node="ip-10-0-1-23", pod="checkout-7d9c8b5f4-x2k9p", ruleSource="http://localhost:8080/alerts/overview\?ruleId=[0-9a-f-]+", severity="critical", threshold\.name="critical"\)'+ ),+ },+ ],+ },+ }+ ],+ },+ },+ ),+ ],+ ),+ ),+ types.AlertManagerNotificationTestCase(+ name="opsgenie_metrics_default_templating",+ rule_path="alertmanager/content_templating/metrics_rule.json",+ alert_data=[+ types.AlertData(+ type="metrics",+ data_path="alertmanager/content_templating/metrics_data.jsonl",+ ),+ ],+ channel_config=opsgenie_default_config,+ notification_expectation=types.AMNotificationExpectation(+ should_notify=True,+ wait_time_seconds=120,+ notification_validations=[+ types.NotificationValidation(+ destination_type="webhook",+ validation_data={+ "path": "/v2/alerts",+ "json_body": {+ "message": "content_templating_metrics",+ "details": {+ "alertname": "content_templating_metrics",+ "container": "checkout",+ "namespace": "production",+ "node": "ip-10-0-1-23",+ "pod": "checkout-7d9c8b5f4-x2k9p",+ "severity": "critical",+ "threshold.name": "critical",+ },+ "priority": "P1",+ },+ },+ ),+ ],+ ),+ ),+ types.AlertManagerNotificationTestCase(+ name="pagerduty_metrics_default_templating",+ rule_path="alertmanager/content_templating/metrics_rule.json",+ alert_data=[+ types.AlertData(+ type="metrics",+ data_path="alertmanager/content_templating/metrics_data.jsonl",+ ),+ ],+ channel_config=pagerduty_default_config,+ notification_expectation=types.AMNotificationExpectation(+ should_notify=True,+ wait_time_seconds=120,+ notification_validations=[+ types.NotificationValidation(+ destination_type="webhook",+ validation_data={+ "path": "/v2/enqueue",+ "json_body": {+ "routing_key": "PagerDutyRoutingKey",+ "payload": {+ "severity": "critical",+ "custom_details": {+ "firing": {+ "Labels": [+ "alertname = content_templating_metrics",+ "container = checkout",+ "namespace = production",+ "node = ip-10-0-1-23",+ "pod = checkout-7d9c8b5f4-x2k9p",+ "severity = critical",+ "threshold.name = critical",+ ],+ }+ },+ },+ "client": "SigNoz Alert Manager",+ "client_url": "https://enter-signoz-host-n-port-here/alerts",+ },+ },+ ),+ ],+ ),+ ),+ types.AlertManagerNotificationTestCase(+ name="slack_logs_default_templating",+ rule_path="alertmanager/content_templating/logs_rule.json",+ alert_data=[+ types.AlertData(+ type="logs",+ data_path="alertmanager/content_templating/logs_data.jsonl",+ ),+ ],+ channel_config=slack_default_config,+ notification_expectation=types.AMNotificationExpectation(+ should_notify=True,+ wait_time_seconds=120,+ notification_validations=[+ types.NotificationValidation(+ destination_type="webhook",+ validation_data={+ "path": "/services/TEAM_ID/BOT_ID/TOKEN_ID",+ "json_body": {+ "username": "Alertmanager",+ "attachments": [+ {+ "color": "danger",+ "mrkdwn_in": ["fallback", "pretext", "text"],+ }+ ],+ },+ },+ ),+ ],+ ),+ ),+ types.AlertManagerNotificationTestCase(+ name="slack_metrics_default_templating",+ rule_path="alertmanager/content_templating/metrics_rule.json",+ alert_data=[+ types.AlertData(+ type="metrics",+ data_path="alertmanager/content_templating/metrics_data.jsonl",+ ),+ ],+ channel_config=slack_default_config,+ notification_expectation=types.AMNotificationExpectation(+ should_notify=True,+ wait_time_seconds=120,+ notification_validations=[+ types.NotificationValidation(+ destination_type="webhook",+ validation_data={+ "path": "/services/TEAM_ID/BOT_ID/TOKEN_ID",+ "json_body": {+ "username": "Alertmanager",+ "attachments": [+ {+ "color": "danger",+ "mrkdwn_in": ["fallback", "pretext", "text"],+ }+ ],+ },+ },+ ),+ ],+ ),+ ),+ types.AlertManagerNotificationTestCase(+ name="webhook_metrics_default_templating",+ rule_path="alertmanager/content_templating/metrics_rule.json",+ alert_data=[+ types.AlertData(+ type="metrics",+ data_path="alertmanager/content_templating/metrics_data.jsonl",+ ),+ ],+ channel_config=webhook_default_config,+ notification_expectation=types.AMNotificationExpectation(+ should_notify=True,+ wait_time_seconds=120,+ notification_validations=[+ types.NotificationValidation(+ destination_type="webhook",+ validation_data={+ "path": "/webhook/webhook_url",+ "json_body": {+ "status": "firing",+ "alerts": [+ {+ "status": "firing",+ "labels": {+ "alertname": "content_templating_metrics",+ "container": "checkout",+ "namespace": "production",+ "node": "ip-10-0-1-23",+ "pod": "checkout-7d9c8b5f4-x2k9p",+ "severity": "critical",+ "threshold.name": "critical",+ },+ "annotations": {+ "description": "Container checkout in pod checkout-7d9c8b5f4-x2k9p (production) exceeded memory threshold",+ "summary": "High container memory in production/checkout-7d9c8b5f4-x2k9p",+ },+ }+ ],+ "commonLabels": {+ "alertname": "content_templating_metrics",+ "container": "checkout",+ "namespace": "production",+ "node": "ip-10-0-1-23",+ "pod": "checkout-7d9c8b5f4-x2k9p",+ "severity": "critical",+ "threshold.name": "critical",+ },+ },+ },+ ),+ ],+ ),+ ),+]+++ "content_templating_test_case",+ CONTENT_TEMPLATING_TEST,+ ids=lambda content_templating_test_case: content_templating_test_case.name,+)+def test_content_templating( # pylint: disable=too-many-arguments,too-many-positional-arguments+ notification_channel: types.TestContainerDocker,+ make_http_mocks: Callable[[types.TestContainerDocker, list[Mapping]], None],+ create_notification_channel: Callable[[dict], str],+ create_alert_rule: Callable[[dict], str],+ insert_alert_data: Callable[[list[types.AlertData], datetime], None],+ maildev: types.TestContainerDocker,+ content_templating_test_case: types.AlertManagerNotificationTestCase,+):+ channel_name = str(uuid.uuid4())++ channel_config = update_raw_channel_config(content_templating_test_case.channel_config, channel_name, notification_channel)+ logger.info("Channel config: %s", {"channel_config": channel_config})++ webhook_validations = [v for v in content_templating_test_case.notification_expectation.notification_validations if v.destination_type == "webhook"]+ if len(webhook_validations) > 0:+ mock_mappings = [+ Mapping(+ request=MappingRequest(method=HttpMethods.POST, url=v.validation_data["path"]),+ response=MappingResponse(status=200, json_body={}),+ persistent=False,+ )+ for v in webhook_validations+ ]++ make_http_mocks(notification_channel, mock_mappings)+ logger.info("Mock mappings created")++ if any(v.destination_type == "email" for v in content_templating_test_case.notification_expectation.notification_validations):+ delete_all_mails(maildev)+ logger.info("Mails deleted")++ create_notification_channel(channel_config)+ logger.info("Channel created with name: %s", {"channel_name": channel_name})++ time.sleep(12)++ insert_alert_data(+ content_templating_test_case.alert_data,+ base_time=datetime.now(tz=UTC) - timedelta(minutes=10),+ )++ rule_path = get_testdata_file_path(content_templating_test_case.rule_path)+ with open(rule_path, encoding="utf-8") as f:+ rule_data = json.loads(f.read())+ update_rule_channel_name(rule_data, channel_name)+ rule_id = create_alert_rule(rule_data)+ logger.info("rule created: %s", {"rule_id": rule_id, "rule_name": rule_data["alert"]})++ verify_notification_expectation(+ notification_channel,+ maildev,+ content_templating_test_case.notification_expectation,+ )diff --git a/tests/integration/tests/alertmanager/__init__.py b/tests/integration/tests/alertmanager/__init__.pynew file mode 100644index 0000000000..e69de29bb2diff --git a/tests/integration/tests/alertmanager/conftest.py b/tests/integration/tests/alertmanager/conftest.pynew file mode 100644index 0000000000..1cd44c577e--- /dev/null+++ b/tests/integration/tests/alertmanager/conftest.py@@ -0,0 +1,35 @@+import pytest+from testcontainers.core.container import Network++from fixtures import types+from fixtures.maildev import signoz_smtp_env+from fixtures.signoz import create_signoz++[email protected](name="signoz", scope="package")+def signoz( # pylint: disable=too-many-arguments,too-many-positional-arguments+ network: Network,+ zeus: types.TestContainerDocker,+ gateway: types.TestContainerDocker,+ sqlstore: types.TestContainerSQL,+ clickhouse: types.TestContainerClickhouse,+ request: pytest.FixtureRequest,+ pytestconfig: pytest.Config,+ maildev: types.TestContainerDocker,+ notification_channel: types.TestContainerDocker,+) -> types.SigNoz:+ return create_signoz(+ network=network,+ zeus=zeus,+ gateway=gateway,+ sqlstore=sqlstore,+ clickhouse=clickhouse,+ request=request,+ pytestconfig=pytestconfig,+ cache_key="signoz_alertmanager",+ env_overrides={+ **signoz_smtp_env(maildev),+ "SIGNOZ_ALERTMANAGER_SIGNOZ_GLOBAL_PAGERDUTY__URL": notification_channel.container_configs["8080"].get("/v2/enqueue"),+ "SIGNOZ_ALERTMANAGER_SIGNOZ_GLOBAL_OPSGENIE__API__URL": notification_channel.container_configs["8080"].get("/"),+ },+ )diff --git a/tests/integration/tests/alertmanagerrotation/01_smtp_rotation.py b/tests/integration/tests/alertmanagerrotation/01_smtp_rotation.pynew file mode 100644index 0000000000..54bbf0f145--- /dev/null+++ b/tests/integration/tests/alertmanagerrotation/01_smtp_rotation.py@@ -0,0 +1,99 @@+import time+import uuid+from collections.abc import Callable+from http import HTTPStatus++import docker+import pytest+import requests+from testcontainers.core.container import Network++from fixtures import types+from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD, token_getter+from fixtures.logger import setup_logger+from fixtures.maildev import (+ NEW_PROVIDER_SMTP_PASS,+ SMTP_TEST_FROM,+ delete_all_mails,+ get_all_mails,+ signoz_smtp_env,+ verify_email_received,+)+from fixtures.notification_channel import send_test_notification+from fixtures.signoz import create_signoz++logger = setup_logger(__name__)+++def wait_for_email(maildev: types.TestContainerDocker, filters: dict, wait_seconds: int = 30) -> None:+ deadline = time.time() + wait_seconds+ while time.time() < deadline:+ if verify_email_received(maildev, filters):+ return+ time.sleep(1)+ raise AssertionError(f"no email matching {filters} within {wait_seconds}s, inbox: {get_all_mails(maildev)}")+++def test_smtp_rotation_applies_to_existing_channels( # pylint: disable=too-many-arguments,too-many-positional-arguments,too-many-locals+ network: Network,+ zeus: types.TestContainerDocker,+ gateway: types.TestContainerDocker,+ sqlstore: types.TestContainerSQL,+ clickhouse: types.TestContainerClickhouse,+ request: pytest.FixtureRequest,+ pytestconfig: pytest.Config,+ signoz: types.SigNoz,+ create_user_admin: None, # pylint: disable=unused-argument+ get_token: Callable[[str, str], str],+ maildev_old: types.TestContainerDocker,+ maildev_new: types.TestContainerDocker,+) -> None:+ token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)+ channel_name = f"rotation-{uuid.uuid4()}"+ recipient = f"rotation-{uuid.uuid4()}@integration.test"++ response = requests.post(+ signoz.self.host_configs["8080"].get("/api/v1/channels"),+ json={"name": channel_name, "email_configs": [{"to": recipient}]},+ headers={"Authorization": f"Bearer {token}"},+ timeout=10,+ )+ assert response.status_code == HTTPStatus.CREATED, response.text++ delete_all_mails(maildev_old)+ recipient_old_probe = f"probe-old-{uuid.uuid4()}@integration.test"+ send_test_notification(signoz, token, {"name": f"probe-{uuid.uuid4()}", "email_configs": [{"to": recipient_old_probe}]})+ wait_for_email(maildev_old, {"to": recipient_old_probe, "from": SMTP_TEST_FROM})+ logger.info("Delivery through the old provider verified")++ docker.from_env().containers.get(signoz.self.id).stop()+ logger.info("Stopped signoz running against the old provider")++ signoz_new = create_signoz(+ network=network,+ zeus=zeus,+ gateway=gateway,+ sqlstore=sqlstore,+ clickhouse=clickhouse,+ request=request,+ pytestconfig=pytestconfig,+ cache_key="signoz_smtp_rotation_new",+ env_overrides=signoz_smtp_env(maildev_new, password=NEW_PROVIDER_SMTP_PASS),+ )+ token_new = token_getter(signoz_new)(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)++ response = requests.get(+ signoz_new.self.host_configs["8080"].get("/api/v1/channels"),+ headers={"Authorization": f"Bearer {token_new}"},+ timeout=10,+ )+ assert response.status_code == HTTPStatus.OK, response.text+ listed = {channel["name"]: channel for channel in response.json()["data"]}+ assert channel_name in listed++ delete_all_mails(maildev_new)+ mails_at_old_provider = len(get_all_mails(maildev_old))+ recipient_new_probe = f"probe-new-{uuid.uuid4()}@integration.test"+ send_test_notification(signoz_new, token_new, {"name": f"probe-{uuid.uuid4()}", "email_configs": [{"to": recipient_new_probe}]})+ wait_for_email(maildev_new, {"to": recipient_new_probe, "from": SMTP_TEST_FROM})+ assert len(get_all_mails(maildev_old)) == mails_at_old_provider, "old provider must receive nothing after rotation"diff --git a/tests/integration/tests/alertmanagerrotation/__init__.py b/tests/integration/tests/alertmanagerrotation/__init__.pynew file mode 100644index 0000000000..e69de29bb2diff --git a/tests/integration/tests/alertmanagerrotation/conftest.py b/tests/integration/tests/alertmanagerrotation/conftest.pynew file mode 100644index 0000000000..d600bf8fe1--- /dev/null+++ b/tests/integration/tests/alertmanagerrotation/conftest.py@@ -0,0 +1,40 @@+import pytest+from testcontainers.core.container import Network++from fixtures import types+from fixtures.maildev import NEW_PROVIDER_SMTP_PASS, OLD_PROVIDER_SMTP_PASS, create_maildev, signoz_smtp_env+from fixtures.signoz import create_signoz++[email protected](name="maildev_old", scope="package")+def maildev_old(network: Network, request: pytest.FixtureRequest, pytestconfig: pytest.Config) -> types.TestContainerDocker:+ return create_maildev(network, request, pytestconfig, cache_key="maildev_smtp_old", incoming_pass=OLD_PROVIDER_SMTP_PASS)++[email protected](name="maildev_new", scope="package")+def maildev_new(network: Network, request: pytest.FixtureRequest, pytestconfig: pytest.Config) -> types.TestContainerDocker:+ return create_maildev(network, request, pytestconfig, cache_key="maildev_smtp_new", incoming_pass=NEW_PROVIDER_SMTP_PASS)++[email protected](name="signoz", scope="package")+def signoz( # pylint: disable=too-many-arguments,too-many-positional-arguments+ network: Network,+ zeus: types.TestContainerDocker,+ gateway: types.TestContainerDocker,+ sqlstore: types.TestContainerSQL,+ clickhouse: types.TestContainerClickhouse,+ request: pytest.FixtureRequest,+ pytestconfig: pytest.Config,+ maildev_old: types.TestContainerDocker,+) -> types.SigNoz:+ return create_signoz(+ network=network,+ zeus=zeus,+ gateway=gateway,+ sqlstore=sqlstore,+ clickhouse=clickhouse,+ request=request,+ pytestconfig=pytestconfig,+ cache_key="signoz_smtp_rotation",+ env_overrides=signoz_smtp_env(maildev_old, password=OLD_PROVIDER_SMTP_PASS),+ )diff --git a/tests/pyproject.toml b/tests/pyproject.tomlindex 1d6abfd946..277290d2d1 100644--- a/tests/pyproject.toml+++ b/tests/pyproject.toml@@ -75,3 +75,7 @@ ignore = [[tool.ruff.format]# Defaults align with black (double quotes, 4-space indent).++[tool.ruff.lint.per-file-ignores]+"fixtures/notification_channel.py" = ["E501"]+"integration/tests/alertmanager/*" = ["E501"]