Skip to content

Modern Docker Management, Designed for Everyone

perf: dispatch notification providers concuurently (#3820)

Kyle Mendell767992dc34a4
diff --git a/backend/internal/notification/notification.go b/backend/internal/notification/notification.go
index ff588a2c1..b1cf859d7 100644
--- a/backend/internal/notification/notification.go
+++ b/backend/internal/notification/notification.go
@@ -22,6 +22,7 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/internal/environment"
"github.com/getarcaneapp/arcane/backend/v2/internal/event"
"github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/edge"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/httpx"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/notifications"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/validation"
@@ -29,6 +30,7 @@ import (
notificationdto "github.com/getarcaneapp/arcane/types/v2/notification"
"github.com/getarcaneapp/arcane/types/v2/system"
"go.getarcane.app/sys/crypto"
+ "golang.org/x/sync/errgroup"
)
var notificationCredentialFieldsByProviderInternal = map[notifications.NotificationProvider][]string{
@@ -518,9 +520,9 @@ func (s *NotificationService) isEventEnabled(config database.JSON, eventType not
return enabled
}
-// logNotification records a delivery attempt in the event log so sends and
+// logNotificationInternal records a delivery attempt in the event log so sends and
// failures are visible alongside every other Arcane event.
-func (s *NotificationService) logNotification(ctx context.Context, environmentID string, provider notifications.NotificationProvider, subject, status string, errMsg *string, metadata database.JSON) {
+func (s *NotificationService) logNotificationInternal(ctx context.Context, environmentID string, provider notifications.NotificationProvider, subject, status string, errMsg *string, metadata database.JSON) {
if s.eventSvc == nil {
return
}
@@ -540,7 +542,7 @@ func (s *NotificationService) logNotification(ctx context.Context, environmentID
resourceType := "notification"
providerName := string(provider)
- if _, err := s.eventSvc.CreateEvent(ctx, event.CreateEventRequest{
+ req := event.CreateEventRequest{
Type: event.EventTypeNotificationSend,
Severity: severity,
Title: title,
@@ -549,8 +551,23 @@ func (s *NotificationService) logNotification(ctx context.Context, environmentID
ResourceName: &providerName,
EnvironmentID: &environmentID,
Metadata: eventMetadata,
- }); err != nil {
- slog.WarnContext(ctx, "Failed to log notification event", "provider", providerName, "error", err.Error())
+ }
+
+ done := make(chan struct{})
+ go func() {
+ defer close(done)
+ defer utils.RecoverToError(nil, "notification event logging", "provider", providerName)
+ logCtx, cancelLog := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
+ defer cancelLog()
+ if _, err := s.eventSvc.CreateEvent(logCtx, req); err != nil {
+ slog.WarnContext(logCtx, "Failed to log notification event", "provider", providerName, "error", err.Error())
+ }
+ }()
+
+ // Cancellation ends the caller's wait, not persistence of a completed attempt.
+ select {
+ case <-done:
+ case <-ctx.Done():
}
}
@@ -575,28 +592,41 @@ func (s *NotificationService) notifyEnabledProvidersInternal(
return 0, errors.WrapIf(err, "failed to get notification settings")
}
- delivered := 0
- var errs []string
+ eligible := make([]NotificationSettings, 0, len(settings))
for _, setting := range settings {
- if !setting.Enabled {
- continue
- }
- if !s.isEventEnabled(setting.Config, eventType) {
- continue
+ if setting.Enabled && s.isEventEnabled(setting.Config, eventType) {
+ eligible = append(eligible, setting)
}
+ }
- handled, sendErr := dispatch(ctx, setting.Provider, setting.Config)
- if !handled {
+ handled := make([]bool, len(eligible))
+ sendErrs := make([]error, len(eligible))
+ var g errgroup.Group
+ g.SetLimit(notificationDispatchConcurrencyInternal)
+ for i, setting := range eligible {
+ g.Go(func() error {
+ defer utils.RecoverToError(&sendErrs[i], "notification dispatch", "provider", setting.Provider)
+ handled[i] = true
+ handled[i], sendErrs[i] = dispatch(ctx, setting.Provider, setting.Config)
+ return nil
+ })
+ }
+ var errs []string
+ if err := g.Wait(); err != nil {
+ errs = append(errs, fmt.Sprintf("notification dispatch: %v", err))
+ }
+
+ delivered := 0
+ for i, setting := range eligible {
+ if !handled[i] {
slog.WarnContext(ctx, "Unknown notification provider", "provider", setting.Provider)
continue
}
-
- if sendErr == nil {
+ if sendErrs[i] == nil {
delivered++
}
-
- status, errMsg := collectNotificationSendResultInternal(&errs, setting.Provider, sendErr)
- s.logNotification(ctx, target.EnvironmentID, setting.Provider, logRef, status, errMsg, metadata)
+ status, errMsg := collectNotificationSendResultInternal(&errs, setting.Provider, sendErrs[i])
+ s.logNotificationInternal(ctx, target.EnvironmentID, setting.Provider, logRef, status, errMsg, metadata)
}
if s.apnsSvc != nil {
@@ -611,6 +641,8 @@ func (s *NotificationService) notifyEnabledProvidersInternal(
return delivered, nil
}
+const notificationDispatchConcurrencyInternal = 4
+
func collectNotificationSendResultInternal(errors *[]string, provider notifications.NotificationProvider, sendErr error) (string, *string) {
if sendErr == nil {
return "success", nil
diff --git a/backend/internal/notification/service_test.go b/backend/internal/notification/service_test.go
index 9f92dbd95..4eb8cba0a 100644
--- a/backend/internal/notification/service_test.go
+++ b/backend/internal/notification/service_test.go
@@ -1,8 +1,6 @@
package notification
import (
- "github.com/getarcaneapp/arcane/backend/v2/internal/settings"
-
"bytes"
"context"
"encoding/base64"
@@ -12,6 +10,7 @@ import (
"net/http"
"net/http/httptest"
"strings"
+ "sync"
"sync/atomic"
"testing"
"time"
@@ -25,6 +24,7 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
"github.com/getarcaneapp/arcane/backend/v2/internal/environment"
"github.com/getarcaneapp/arcane/backend/v2/internal/event"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/settings"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/notifications"
"github.com/getarcaneapp/arcane/types/v2/imageupdate"
notificationdto "github.com/getarcaneapp/arcane/types/v2/notification"
@@ -745,11 +745,14 @@ func TestNotificationService_NotifyEnabledProvidersInternal_SkipsFiltersAndAggre
require.NoError(t, db.WithContext(ctx).Create(&rows[i]).Error)
}
+ var dispatchedMu sync.Mutex
var dispatched []notifications.NotificationProvider
target := NotificationTarget{EnvironmentID: "0", EnvironmentName: "Local Docker"}
delivered, err := svc.notifyEnabledProvidersInternal(ctx, target, notifications.NotificationEventPruneReport, "loop-test", database.JSON{"eventType": "prune_report"},
func(_ context.Context, provider notifications.NotificationProvider, _ database.JSON) (bool, error) {
+ dispatchedMu.Lock()
dispatched = append(dispatched, provider)
+ dispatchedMu.Unlock()
if provider == notifications.NotificationProviderNtfy {
return true, errors.New("boom")
}
@@ -757,7 +760,7 @@ func TestNotificationService_NotifyEnabledProvidersInternal_SkipsFiltersAndAggre
})
// Disabled and event-disabled rows are never dispatched.
- require.Equal(t, []notifications.NotificationProvider{notifications.NotificationProviderGotify, notifications.NotificationProviderNtfy}, dispatched)
+ require.ElementsMatch(t, []notifications.NotificationProvider{notifications.NotificationProviderGotify, notifications.NotificationProviderNtfy}, dispatched)
require.Equal(t, 1, delivered)
require.Error(t, err)
require.Contains(t, err.Error(), "notification errors: ntfy: boom")
@@ -772,6 +775,40 @@ func TestNotificationService_NotifyEnabledProvidersInternal_SkipsFiltersAndAggre
require.Equal(t, event.EventSeverityError, events[1].Severity)
require.Equal(t, "Notification failed via ntfy", events[1].Title)
require.Contains(t, events[1].Description, "boom")
+
+ // Completed attempts persist even when cancellation ends the caller's wait.
+ sqlDB, err := db.DB.DB()
+ require.NoError(t, err)
+ sqlDB.SetMaxOpenConns(1)
+ releaseLogging := make(chan struct{})
+ var releaseOnce sync.Once
+ defer releaseOnce.Do(func() { close(releaseLogging) })
+ require.NoError(t, db.Callback().Create().Before("gorm:create").Register("block_notification_events", func(tx *gorm.DB) {
+ if tx.Statement.Table == "events" {
+ <-releaseLogging
+ }
+ }))
+ canceledCtx, cancel := context.WithCancel(ctx)
+ defer cancel()
+ returned := make(chan struct{})
+ go func() {
+ defer close(returned)
+ _, _ = svc.notifyEnabledProvidersInternal(canceledCtx, target, notifications.NotificationEventPruneReport, "canceled-loop-test", nil,
+ func(context.Context, notifications.NotificationProvider, database.JSON) (bool, error) {
+ cancel()
+ return true, nil
+ })
+ }()
+ select {
+ case <-returned:
+ case <-time.After(time.Second):
+ t.Fatal("notification call waited for event persistence after cancellation")
+ }
+ releaseOnce.Do(func() { close(releaseLogging) })
+ require.Eventually(t, func() bool {
+ var count int64
+ return db.WithContext(ctx).Model(&event.Event{}).Where("description = ?", "canceled-loop-test").Count(&count).Error == nil && count == 2
+ }, time.Second, 10*time.Millisecond)
}
func TestSupportedNotificationTestTypes_IncludesAutoHeal(t *testing.T) {