Modern Docker Management, Designed for Everyone
Go composecontainer-managementcontainersdockerdocker-composedocker-managementgoself-hostedsveltekittypescriptweb-ui
perf: dispatch notification providers concuurently (#3820)
Kyle Mendell767992dc34a4
diff --git a/backend/internal/notification/notification.go b/backend/internal/notification/notification.goindex ff588a2c1..b1cf859d7 100644--- a/backend/internal/notification/notification.go+++ b/backend/internal/notification/notification.go@@ -22,6 +22,7 @@ import ("github.com/getarcaneapp/arcane/backend/v2/internal/environment""github.com/getarcaneapp/arcane/backend/v2/internal/event""github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/edge"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils""github.com/getarcaneapp/arcane/backend/v2/pkg/utils/httpx""github.com/getarcaneapp/arcane/backend/v2/pkg/utils/notifications""github.com/getarcaneapp/arcane/backend/v2/pkg/utils/validation"@@ -29,6 +30,7 @@ import (notificationdto "github.com/getarcaneapp/arcane/types/v2/notification""github.com/getarcaneapp/arcane/types/v2/system""go.getarcane.app/sys/crypto"+ "golang.org/x/sync/errgroup")var notificationCredentialFieldsByProviderInternal = map[notifications.NotificationProvider][]string{@@ -518,9 +520,9 @@ func (s *NotificationService) isEventEnabled(config database.JSON, eventType notreturn enabled}-// logNotification records a delivery attempt in the event log so sends and+// logNotificationInternal records a delivery attempt in the event log so sends and// failures are visible alongside every other Arcane event.-func (s *NotificationService) logNotification(ctx context.Context, environmentID string, provider notifications.NotificationProvider, subject, status string, errMsg *string, metadata database.JSON) {+func (s *NotificationService) logNotificationInternal(ctx context.Context, environmentID string, provider notifications.NotificationProvider, subject, status string, errMsg *string, metadata database.JSON) {if s.eventSvc == nil {return}@@ -540,7 +542,7 @@ func (s *NotificationService) logNotification(ctx context.Context, environmentIDresourceType := "notification"providerName := string(provider)- if _, err := s.eventSvc.CreateEvent(ctx, event.CreateEventRequest{+ req := event.CreateEventRequest{Type: event.EventTypeNotificationSend,Severity: severity,Title: title,@@ -549,8 +551,23 @@ func (s *NotificationService) logNotification(ctx context.Context, environmentIDResourceName: &providerName,EnvironmentID: &environmentID,Metadata: eventMetadata,- }); err != nil {- slog.WarnContext(ctx, "Failed to log notification event", "provider", providerName, "error", err.Error())+ }++ done := make(chan struct{})+ go func() {+ defer close(done)+ defer utils.RecoverToError(nil, "notification event logging", "provider", providerName)+ logCtx, cancelLog := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)+ defer cancelLog()+ if _, err := s.eventSvc.CreateEvent(logCtx, req); err != nil {+ slog.WarnContext(logCtx, "Failed to log notification event", "provider", providerName, "error", err.Error())+ }+ }()++ // Cancellation ends the caller's wait, not persistence of a completed attempt.+ select {+ case <-done:+ case <-ctx.Done():}}@@ -575,28 +592,41 @@ func (s *NotificationService) notifyEnabledProvidersInternal(return 0, errors.WrapIf(err, "failed to get notification settings")}- delivered := 0- var errs []string+ eligible := make([]NotificationSettings, 0, len(settings))for _, setting := range settings {- if !setting.Enabled {- continue- }- if !s.isEventEnabled(setting.Config, eventType) {- continue+ if setting.Enabled && s.isEventEnabled(setting.Config, eventType) {+ eligible = append(eligible, setting)}+ }- handled, sendErr := dispatch(ctx, setting.Provider, setting.Config)- if !handled {+ handled := make([]bool, len(eligible))+ sendErrs := make([]error, len(eligible))+ var g errgroup.Group+ g.SetLimit(notificationDispatchConcurrencyInternal)+ for i, setting := range eligible {+ g.Go(func() error {+ defer utils.RecoverToError(&sendErrs[i], "notification dispatch", "provider", setting.Provider)+ handled[i] = true+ handled[i], sendErrs[i] = dispatch(ctx, setting.Provider, setting.Config)+ return nil+ })+ }+ var errs []string+ if err := g.Wait(); err != nil {+ errs = append(errs, fmt.Sprintf("notification dispatch: %v", err))+ }++ delivered := 0+ for i, setting := range eligible {+ if !handled[i] {slog.WarnContext(ctx, "Unknown notification provider", "provider", setting.Provider)continue}-- if sendErr == nil {+ if sendErrs[i] == nil {delivered++}-- status, errMsg := collectNotificationSendResultInternal(&errs, setting.Provider, sendErr)- s.logNotification(ctx, target.EnvironmentID, setting.Provider, logRef, status, errMsg, metadata)+ status, errMsg := collectNotificationSendResultInternal(&errs, setting.Provider, sendErrs[i])+ s.logNotificationInternal(ctx, target.EnvironmentID, setting.Provider, logRef, status, errMsg, metadata)}if s.apnsSvc != nil {@@ -611,6 +641,8 @@ func (s *NotificationService) notifyEnabledProvidersInternal(return delivered, nil}+const notificationDispatchConcurrencyInternal = 4+func collectNotificationSendResultInternal(errors *[]string, provider notifications.NotificationProvider, sendErr error) (string, *string) {if sendErr == nil {return "success", nildiff --git a/backend/internal/notification/service_test.go b/backend/internal/notification/service_test.goindex 9f92dbd95..4eb8cba0a 100644--- a/backend/internal/notification/service_test.go+++ b/backend/internal/notification/service_test.go@@ -1,8 +1,6 @@package notificationimport (- "github.com/getarcaneapp/arcane/backend/v2/internal/settings"-"bytes""context""encoding/base64"@@ -12,6 +10,7 @@ import ("net/http""net/http/httptest""strings"+ "sync""sync/atomic""testing""time"@@ -25,6 +24,7 @@ import ("github.com/getarcaneapp/arcane/backend/v2/internal/database""github.com/getarcaneapp/arcane/backend/v2/internal/environment""github.com/getarcaneapp/arcane/backend/v2/internal/event"+ "github.com/getarcaneapp/arcane/backend/v2/internal/settings""github.com/getarcaneapp/arcane/backend/v2/pkg/utils/notifications""github.com/getarcaneapp/arcane/types/v2/imageupdate"notificationdto "github.com/getarcaneapp/arcane/types/v2/notification"@@ -745,11 +745,14 @@ func TestNotificationService_NotifyEnabledProvidersInternal_SkipsFiltersAndAggrerequire.NoError(t, db.WithContext(ctx).Create(&rows[i]).Error)}+ var dispatchedMu sync.Mutexvar dispatched []notifications.NotificationProvidertarget := NotificationTarget{EnvironmentID: "0", EnvironmentName: "Local Docker"}delivered, err := svc.notifyEnabledProvidersInternal(ctx, target, notifications.NotificationEventPruneReport, "loop-test", database.JSON{"eventType": "prune_report"},func(_ context.Context, provider notifications.NotificationProvider, _ database.JSON) (bool, error) {+ dispatchedMu.Lock()dispatched = append(dispatched, provider)+ dispatchedMu.Unlock()if provider == notifications.NotificationProviderNtfy {return true, errors.New("boom")}@@ -757,7 +760,7 @@ func TestNotificationService_NotifyEnabledProvidersInternal_SkipsFiltersAndAggre})// Disabled and event-disabled rows are never dispatched.- require.Equal(t, []notifications.NotificationProvider{notifications.NotificationProviderGotify, notifications.NotificationProviderNtfy}, dispatched)+ require.ElementsMatch(t, []notifications.NotificationProvider{notifications.NotificationProviderGotify, notifications.NotificationProviderNtfy}, dispatched)require.Equal(t, 1, delivered)require.Error(t, err)require.Contains(t, err.Error(), "notification errors: ntfy: boom")@@ -772,6 +775,40 @@ func TestNotificationService_NotifyEnabledProvidersInternal_SkipsFiltersAndAggrerequire.Equal(t, event.EventSeverityError, events[1].Severity)require.Equal(t, "Notification failed via ntfy", events[1].Title)require.Contains(t, events[1].Description, "boom")++ // Completed attempts persist even when cancellation ends the caller's wait.+ sqlDB, err := db.DB.DB()+ require.NoError(t, err)+ sqlDB.SetMaxOpenConns(1)+ releaseLogging := make(chan struct{})+ var releaseOnce sync.Once+ defer releaseOnce.Do(func() { close(releaseLogging) })+ require.NoError(t, db.Callback().Create().Before("gorm:create").Register("block_notification_events", func(tx *gorm.DB) {+ if tx.Statement.Table == "events" {+ <-releaseLogging+ }+ }))+ canceledCtx, cancel := context.WithCancel(ctx)+ defer cancel()+ returned := make(chan struct{})+ go func() {+ defer close(returned)+ _, _ = svc.notifyEnabledProvidersInternal(canceledCtx, target, notifications.NotificationEventPruneReport, "canceled-loop-test", nil,+ func(context.Context, notifications.NotificationProvider, database.JSON) (bool, error) {+ cancel()+ return true, nil+ })+ }()+ select {+ case <-returned:+ case <-time.After(time.Second):+ t.Fatal("notification call waited for event persistence after cancellation")+ }+ releaseOnce.Do(func() { close(releaseLogging) })+ require.Eventually(t, func() bool {+ var count int64+ return db.WithContext(ctx).Model(&event.Event{}).Where("description = ?", "canceled-loop-test").Count(&count).Error == nil && count == 2+ }, time.Second, 10*time.Millisecond)}func TestSupportedNotificationTestTypes_IncludesAutoHeal(t *testing.T) {