Modern Docker Management, Designed for Everyone
Go composecontainer-managementcontainersdockerdocker-composedocker-managementgoself-hostedsveltekittypescriptweb-ui
fix: webhook trigger endpoint responds 202 immediately instead of blocking until action completes (#3558)
Kyle Mendellb1e2c9d76046
diff --git a/backend/api/webhooks_trigger.go b/backend/api/webhooks_trigger.goindex 96857bff6..65f24ca0a 100644--- a/backend/api/webhooks_trigger.go+++ b/backend/api/webhooks_trigger.go@@ -21,7 +21,7 @@ func RegisterWebhookTrigger(g *echo.Group, webhookService *webhook.WebhookServic}token := c.Param("token")- result, err := webhookService.TriggerByToken(utils.ActivityRuntimeContext(c.Request().Context(), appCtx.Context()), token)+ err := webhookService.TriggerByToken(utils.ActivityRuntimeContext(c.Request().Context(), appCtx.Context()), token)if err != nil {status := http.StatusInternalServerErrorif errors.Is(err, webhook.ErrWebhookNotFound) || errors.Is(err, webhook.ErrWebhookInvalid) {@@ -36,6 +36,7 @@ func RegisterWebhookTrigger(g *echo.Group, webhookService *webhook.WebhookServicreturn c.JSON(status, map[string]any{"success": false, "error": msg})}- return c.JSON(http.StatusOK, map[string]any{"success": true, "data": result})+ // The action runs in the background; its outcome lands in the event log.+ return c.JSON(http.StatusAccepted, map[string]any{"success": true, "data": map[string]string{"status": "accepted"}})})}diff --git a/backend/internal/di/providers.go b/backend/internal/di/providers.goindex 715c05ae3..da551fae6 100644--- a/backend/internal/di/providers.go+++ b/backend/internal/di/providers.go@@ -405,8 +405,8 @@ func provideSystemModuleInternal(db *database.DB, cfg *config.Config, docker *do})}-func provideWebhookModuleInternal(db *database.DB, containerModule *container.Module, updaterModule *updater.Module, project *project.ProjectService, gitOpsSync *gitops.GitOpsSyncService, event *event.EventService, environment *environment.EnvironmentService) *webhook.Module {- return webhook.New(webhook.Dependencies{+func provideWebhookModuleInternal(lc fx.Lifecycle, db *database.DB, containerModule *container.Module, updaterModule *updater.Module, project *project.ProjectService, gitOpsSync *gitops.GitOpsSyncService, event *event.EventService, environment *environment.EnvironmentService) *webhook.Module {+ module := webhook.New(webhook.Dependencies{DB: db,Container: containerModule.Service(),Updater: updaterModule.Service(),@@ -415,4 +415,16 @@ func provideWebhookModuleInternal(db *database.DB, containerModule *container.MoEvent: event,Environment: environment,})+ lc.Append(fx.Hook{+ OnStop: func(ctx context.Context) error {+ // Drain fails only when the stop context expires; fx then skips all+ // remaining teardown regardless of what is returned here, so an+ // error would only mark an expected long action as a failed stop.+ if err := module.Service().DrainActions(ctx); err != nil {+ slog.WarnContext(ctx, "shutdown proceeding with webhook actions still in flight", "error", err)+ }+ return nil+ },+ })+ return module}diff --git a/backend/internal/webhook/webhook.go b/backend/internal/webhook/webhook.goindex df865edaa..e293047c6 100644--- a/backend/internal/webhook/webhook.go+++ b/backend/internal/webhook/webhook.go@@ -2,27 +2,30 @@ package webhookimport ("context"- crand "crypto/rand"+ "crypto/rand""crypto/sha256""encoding/base64""encoding/hex""fmt"+ "log/slog""net/http""net/url""strings"+ "sync""time"- "github.com/getarcaneapp/arcane/backend/v2/internal/environment"- "github.com/getarcaneapp/arcane/backend/v2/internal/event"- "github.com/getarcaneapp/arcane/backend/v2/internal/gitops"- "github.com/getarcaneapp/arcane/backend/v2/internal/project"- "github.com/getarcaneapp/arcane/backend/v2/internal/updater"-+ "emperror.dev/emperror""emperror.dev/errors""github.com/getarcaneapp/arcane/backend/v2/internal/container""github.com/getarcaneapp/arcane/backend/v2/internal/database"+ "github.com/getarcaneapp/arcane/backend/v2/internal/environment"+ "github.com/getarcaneapp/arcane/backend/v2/internal/event"+ "github.com/getarcaneapp/arcane/backend/v2/internal/gitops""github.com/getarcaneapp/arcane/backend/v2/internal/models"+ "github.com/getarcaneapp/arcane/backend/v2/internal/project"+ "github.com/getarcaneapp/arcane/backend/v2/internal/updater"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils""github.com/getarcaneapp/arcane/types/v2""github.com/getarcaneapp/arcane/types/v2/base"updatertypes "github.com/getarcaneapp/arcane/types/v2/updater"@@ -47,6 +50,9 @@ const ()type WebhookService struct {+ // actions tracks in-flight background webhook actions accepted with a 202+ // so shutdown can drain them instead of dropping acknowledged work.+ actions sync.WaitGroupdb *database.DBcontainerService *container.ContainerServiceupdaterService *updater.UpdaterService@@ -78,7 +84,7 @@ func isRemoteWebhookEnvironmentInternal(environmentID string) bool {// (to be shown to the user once), its SHA-256 hash, and the lookup prefix.func generateWebhookTokenInternal() (raw, hash, prefix string, err error) {b := make([]byte, webhookTokenLength)- if _, err = crand.Read(b); err != nil {+ if _, err = rand.Read(b); err != nil {return "", "", "", errors.WrapIf(err, "failed to generate webhook token")}secretHex := hex.EncodeToString(b)@@ -415,12 +421,14 @@ func (s *WebhookService) UpdateWebhook(ctx context.Context, id, environmentID streturn &wh, nil}-// TriggerByToken looks up a webhook by its raw token and executes the configured action.-// Returns an updater result for "updater" webhooks; nil for "project" and "gitops".-func (s *WebhookService) TriggerByToken(ctx context.Context, rawToken string) (*updatertypes.Result, error) {+// TriggerByToken looks up a webhook by its raw token, validates it, and starts+// the configured action in the background so the HTTP caller gets an immediate+// acknowledgement instead of holding the connection for the full action+// duration (#3469). The action outcome is recorded in the event log.+func (s *WebhookService) TriggerByToken(ctx context.Context, rawToken string) error {prefix, err := parseWebhookPrefixInternal(rawToken)if err != nil {- return nil, ErrWebhookInvalid+ return ErrWebhookInvalid}// Narrow by prefix first (indexed), then verify hash@@ -428,7 +436,7 @@ func (s *WebhookService) TriggerByToken(ctx context.Context, rawToken string) (*if err := s.db.WithContext(ctx).Where("token_prefix = ?", prefix).Find(&candidates).Error; err != nil {- return nil, errors.WrapIf(err, "failed to look up webhook")+ return errors.WrapIf(err, "failed to look up webhook")}hash := hashWebhookTokenInternal(rawToken)@@ -440,30 +448,43 @@ func (s *WebhookService) TriggerByToken(ctx context.Context, rawToken string) (*}}if wh == nil {- return nil, ErrWebhookNotFound+ return ErrWebhookNotFound}if !wh.Enabled {- return nil, ErrWebhookDisabled+ return ErrWebhookDisabled}- var result *updatertypes.ResultactionType, err := resolveWebhookActionTypeInternal(wh.TargetType, wh.ActionType)if err != nil {- return nil, err- }-- result, err = s.executeWebhookActionInternal(ctx, wh, actionType)- if err != nil {- return nil, err+ return err}- // Record trigger time — best-effort, do not fail the request if this update fails.+ // Record trigger time on accept — best-effort, do not fail the request if this update fails.now := time.Now()_ = s.db.WithContext(ctx).Model(wh).Update("last_triggered_at", now).Error- s.logWebhookEventInternal(ctx, wh, actionType, models.EventSeveritySuccess, "")+ execCtx := context.WithoutCancel(ctx)+ s.actions.Go(func() {+ defer func() {+ if panicErr := emperror.Recover(recover()); panicErr != nil {+ slog.ErrorContext(execCtx, "webhook action panicked", "webhookID", wh.ID, "webhookName", wh.Name, "actionType", actionType, "error", panicErr)+ }+ }()+ if _, err := s.executeWebhookActionInternal(execCtx, wh, actionType); err != nil {+ // Action failures are recorded as error events by wrapWebhookActionErrorInternal.+ slog.ErrorContext(execCtx, "webhook action failed", "webhookID", wh.ID, "webhookName", wh.Name, "actionType", actionType, "error", err)+ return+ }+ s.logWebhookEventInternal(execCtx, wh, actionType, models.EventSeveritySuccess, "")+ })- return result, nil+ return nil+}++// DrainActions blocks until all accepted background webhook actions finish or+// ctx expires, so shutdown does not silently drop acknowledged work.+func (s *WebhookService) DrainActions(ctx context.Context) error {+ return utils.WaitGroup(ctx, &s.actions)}func (s *WebhookService) executeWebhookActionInternal(ctx context.Context, wh *models.Webhook, actionType string) (*updatertypes.Result, error) {diff --git a/backend/internal/webhook/webhook_test.go b/backend/internal/webhook/webhook_test.goindex c0a60d7c9..2433a4ef5 100644--- a/backend/internal/webhook/webhook_test.go+++ b/backend/internal/webhook/webhook_test.go@@ -491,7 +491,7 @@ func TestTriggerByToken_InvalidFormat(t *testing.T) {db := setupWebhookServiceTestDB(t)svc := newTestWebhookService(db)- _, err := svc.TriggerByToken(ctx, "not-a-webhook-token")+ err := svc.TriggerByToken(ctx, "not-a-webhook-token")assert.ErrorIs(t, err, ErrWebhookInvalid)}@@ -502,7 +502,7 @@ func TestTriggerByToken_NotFound(t *testing.T) {// Well-formatted token but not in the DBraw := "arc_wh_0102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f20"- _, err := svc.TriggerByToken(ctx, raw)+ err := svc.TriggerByToken(ctx, raw)assert.ErrorIs(t, err, ErrWebhookNotFound)}@@ -522,7 +522,7 @@ func TestTriggerByToken_WrongHash_NotFound(t *testing.T) {tampered += "a"}- _, err = svc.TriggerByToken(ctx, tampered)+ err = svc.TriggerByToken(ctx, tampered)assert.ErrorIs(t, err, ErrWebhookNotFound)}@@ -536,7 +536,7 @@ func TestTriggerByToken_DisabledWebhook(t *testing.T) {require.NoError(t, db.WithContext(ctx).Model(&models.Webhook{}).Where("id = ?", created.ID).Update("enabled", false).Error)- _, err = svc.TriggerByToken(ctx, rawToken)+ err = svc.TriggerByToken(ctx, rawToken)assert.ErrorIs(t, err, ErrWebhookDisabled)}@@ -561,7 +561,7 @@ func TestTriggerByToken_UnknownTargetType_ReturnsInvalidType(t *testing.T) {}require.NoError(t, db.WithContext(ctx).Create(wh).Error)- _, err := svc.TriggerByToken(ctx, rawToken)+ err := svc.TriggerByToken(ctx, rawToken)assert.ErrorIs(t, err, ErrWebhookInvalidType)}@@ -585,45 +585,20 @@ func insertWebhookDirect(t *testing.T, ctx context.Context, db *database.DB, rawreturn wh}-func TestTriggerByToken_ContainerType_NilServiceReturnsError(t *testing.T) {+func TestTriggerByToken_AcceptsImmediatelyAndRecordsTriggerTime(t *testing.T) {ctx := context.Background()db := setupWebhookServiceTestDB(t)- svc := newTestWebhookService(db) // updaterService is nil+ svc := newTestWebhookService(db) // action services are nil; the async action fails, acceptance must notrawToken := "arc_wh_ccddeeff01020304aabbccdd0102030405060708090a0b0c0d0e0f1011121314"- insertWebhookDirect(t, ctx, db, rawToken, models.WebhookTargetTypeContainer, models.WebhookActionTypeUpdate, "container-id", types.LocalDockerEnvironmentID)+ wh := insertWebhookDirect(t, ctx, db, rawToken, models.WebhookTargetTypeContainer, models.WebhookActionTypeUpdate, "container-id", types.LocalDockerEnvironmentID)- assert.Panics(t, func() {- _, _ = svc.TriggerByToken(ctx, rawToken) //nolint:errcheck- })-}--func TestTriggerByToken_UpdaterType_NilServiceReturnsError(t *testing.T) {- ctx := context.Background()- db := setupWebhookServiceTestDB(t)- svc := newTestWebhookService(db) // updaterService is nil-- rawToken := "arc_wh_1122334401020304aabbccdd0102030405060708090a0b0c0d0e0f1011121314"- insertWebhookDirect(t, ctx, db, rawToken, models.WebhookTargetTypeUpdater, models.WebhookActionTypeRun, "", types.LocalDockerEnvironmentID)+ // Valid token is accepted synchronously; the action itself runs (and here+ // fails against nil services) in the background without surfacing.+ require.NoError(t, svc.TriggerByToken(ctx, rawToken))- // nil updaterService causes a panic, which we verify the dispatch path is reached- // by recovering — in production the service is always non-nil- assert.Panics(t, func() {- _, _ = svc.TriggerByToken(ctx, rawToken) //nolint:errcheck- })-}--func TestTriggerByToken_GitOpsType_NilServiceReturnsError(t *testing.T) {- ctx := context.Background()- db := setupWebhookServiceTestDB(t)- svc := newTestWebhookService(db) // gitOpsSyncService is nil-- rawToken := "arc_wh_aabbccdd11223344aabbccdd0102030405060708090a0b0c0d0e0f1011121314"- insertWebhookDirect(t, ctx, db, rawToken, models.WebhookTargetTypeGitOps, models.WebhookActionTypeSync, "sync-id", types.LocalDockerEnvironmentID)-- assert.Panics(t, func() {- _, _ = svc.TriggerByToken(ctx, rawToken) //nolint:errcheck- })+ stored := fetchWebhook(t, db, wh.ID)+ assert.NotNil(t, stored.LastTriggeredAt, "last_triggered_at must be set once the trigger is accepted")}func TestTriggerByToken_DoesNotUpdateLastTriggeredAtOnError(t *testing.T) {@@ -634,7 +609,7 @@ func TestTriggerByToken_DoesNotUpdateLastTriggeredAtOnError(t *testing.T) {rawToken := "arc_wh_1122334401020304aabbccdd0102030405060708090a0b0c0d0e0f1011121315"wh := insertWebhookDirect(t, ctx, db, rawToken, "unknown-type", models.WebhookActionTypeUpdate, "some-id", "env-1")- _, err := svc.TriggerByToken(ctx, rawToken)+ err := svc.TriggerByToken(ctx, rawToken)require.Error(t, err)stored := fetchWebhook(t, db, wh.ID)@@ -649,7 +624,7 @@ func TestTriggerByToken_UnknownActionType_ReturnsInvalidAction(t *testing.T) {rawToken := "arc_wh_0011223344556677aabbccdd0102030405060708090a0b0c0d0e0f1011121314"insertWebhookDirect(t, ctx, db, rawToken, models.WebhookTargetTypeProject, "bogus", "project-id", "env-1")- _, err := svc.TriggerByToken(ctx, rawToken)+ err := svc.TriggerByToken(ctx, rawToken)assert.ErrorIs(t, err, ErrWebhookInvalidAction)}