Skip to content

Modern Docker Management, Designed for Everyone

fix: webhook trigger endpoint responds 202 immediately instead of blocking until action completes (#3558)

Kyle Mendellb1e2c9d76046
diff --git a/backend/api/webhooks_trigger.go b/backend/api/webhooks_trigger.go
index 96857bff6..65f24ca0a 100644
--- a/backend/api/webhooks_trigger.go
+++ b/backend/api/webhooks_trigger.go
@@ -21,7 +21,7 @@ func RegisterWebhookTrigger(g *echo.Group, webhookService *webhook.WebhookServic
}
token := c.Param("token")
- result, err := webhookService.TriggerByToken(utils.ActivityRuntimeContext(c.Request().Context(), appCtx.Context()), token)
+ err := webhookService.TriggerByToken(utils.ActivityRuntimeContext(c.Request().Context(), appCtx.Context()), token)
if err != nil {
status := http.StatusInternalServerError
if errors.Is(err, webhook.ErrWebhookNotFound) || errors.Is(err, webhook.ErrWebhookInvalid) {
@@ -36,6 +36,7 @@ func RegisterWebhookTrigger(g *echo.Group, webhookService *webhook.WebhookServic
return c.JSON(status, map[string]any{"success": false, "error": msg})
}
- return c.JSON(http.StatusOK, map[string]any{"success": true, "data": result})
+ // The action runs in the background; its outcome lands in the event log.
+ return c.JSON(http.StatusAccepted, map[string]any{"success": true, "data": map[string]string{"status": "accepted"}})
})
}
diff --git a/backend/internal/di/providers.go b/backend/internal/di/providers.go
index 715c05ae3..da551fae6 100644
--- a/backend/internal/di/providers.go
+++ b/backend/internal/di/providers.go
@@ -405,8 +405,8 @@ func provideSystemModuleInternal(db *database.DB, cfg *config.Config, docker *do
})
}
-func provideWebhookModuleInternal(db *database.DB, containerModule *container.Module, updaterModule *updater.Module, project *project.ProjectService, gitOpsSync *gitops.GitOpsSyncService, event *event.EventService, environment *environment.EnvironmentService) *webhook.Module {
- return webhook.New(webhook.Dependencies{
+func provideWebhookModuleInternal(lc fx.Lifecycle, db *database.DB, containerModule *container.Module, updaterModule *updater.Module, project *project.ProjectService, gitOpsSync *gitops.GitOpsSyncService, event *event.EventService, environment *environment.EnvironmentService) *webhook.Module {
+ module := webhook.New(webhook.Dependencies{
DB: db,
Container: containerModule.Service(),
Updater: updaterModule.Service(),
@@ -415,4 +415,16 @@ func provideWebhookModuleInternal(db *database.DB, containerModule *container.Mo
Event: event,
Environment: environment,
})
+ lc.Append(fx.Hook{
+ OnStop: func(ctx context.Context) error {
+ // Drain fails only when the stop context expires; fx then skips all
+ // remaining teardown regardless of what is returned here, so an
+ // error would only mark an expected long action as a failed stop.
+ if err := module.Service().DrainActions(ctx); err != nil {
+ slog.WarnContext(ctx, "shutdown proceeding with webhook actions still in flight", "error", err)
+ }
+ return nil
+ },
+ })
+ return module
}
diff --git a/backend/internal/webhook/webhook.go b/backend/internal/webhook/webhook.go
index df865edaa..e293047c6 100644
--- a/backend/internal/webhook/webhook.go
+++ b/backend/internal/webhook/webhook.go
@@ -2,27 +2,30 @@ package webhook
import (
"context"
- crand "crypto/rand"
+ "crypto/rand"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"fmt"
+ "log/slog"
"net/http"
"net/url"
"strings"
+ "sync"
"time"
- "github.com/getarcaneapp/arcane/backend/v2/internal/environment"
- "github.com/getarcaneapp/arcane/backend/v2/internal/event"
- "github.com/getarcaneapp/arcane/backend/v2/internal/gitops"
- "github.com/getarcaneapp/arcane/backend/v2/internal/project"
- "github.com/getarcaneapp/arcane/backend/v2/internal/updater"
-
+ "emperror.dev/emperror"
"emperror.dev/errors"
"github.com/getarcaneapp/arcane/backend/v2/internal/container"
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/environment"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/event"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/gitops"
"github.com/getarcaneapp/arcane/backend/v2/internal/models"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/project"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/updater"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils"
"github.com/getarcaneapp/arcane/types/v2"
"github.com/getarcaneapp/arcane/types/v2/base"
updatertypes "github.com/getarcaneapp/arcane/types/v2/updater"
@@ -47,6 +50,9 @@ const (
)
type WebhookService struct {
+ // actions tracks in-flight background webhook actions accepted with a 202
+ // so shutdown can drain them instead of dropping acknowledged work.
+ actions sync.WaitGroup
db *database.DB
containerService *container.ContainerService
updaterService *updater.UpdaterService
@@ -78,7 +84,7 @@ func isRemoteWebhookEnvironmentInternal(environmentID string) bool {
// (to be shown to the user once), its SHA-256 hash, and the lookup prefix.
func generateWebhookTokenInternal() (raw, hash, prefix string, err error) {
b := make([]byte, webhookTokenLength)
- if _, err = crand.Read(b); err != nil {
+ if _, err = rand.Read(b); err != nil {
return "", "", "", errors.WrapIf(err, "failed to generate webhook token")
}
secretHex := hex.EncodeToString(b)
@@ -415,12 +421,14 @@ func (s *WebhookService) UpdateWebhook(ctx context.Context, id, environmentID st
return &wh, nil
}
-// TriggerByToken looks up a webhook by its raw token and executes the configured action.
-// Returns an updater result for "updater" webhooks; nil for "project" and "gitops".
-func (s *WebhookService) TriggerByToken(ctx context.Context, rawToken string) (*updatertypes.Result, error) {
+// TriggerByToken looks up a webhook by its raw token, validates it, and starts
+// the configured action in the background so the HTTP caller gets an immediate
+// acknowledgement instead of holding the connection for the full action
+// duration (#3469). The action outcome is recorded in the event log.
+func (s *WebhookService) TriggerByToken(ctx context.Context, rawToken string) error {
prefix, err := parseWebhookPrefixInternal(rawToken)
if err != nil {
- return nil, ErrWebhookInvalid
+ return ErrWebhookInvalid
}
// Narrow by prefix first (indexed), then verify hash
@@ -428,7 +436,7 @@ func (s *WebhookService) TriggerByToken(ctx context.Context, rawToken string) (*
if err := s.db.WithContext(ctx).
Where("token_prefix = ?", prefix).
Find(&candidates).Error; err != nil {
- return nil, errors.WrapIf(err, "failed to look up webhook")
+ return errors.WrapIf(err, "failed to look up webhook")
}
hash := hashWebhookTokenInternal(rawToken)
@@ -440,30 +448,43 @@ func (s *WebhookService) TriggerByToken(ctx context.Context, rawToken string) (*
}
}
if wh == nil {
- return nil, ErrWebhookNotFound
+ return ErrWebhookNotFound
}
if !wh.Enabled {
- return nil, ErrWebhookDisabled
+ return ErrWebhookDisabled
}
- var result *updatertypes.Result
actionType, err := resolveWebhookActionTypeInternal(wh.TargetType, wh.ActionType)
if err != nil {
- return nil, err
- }
-
- result, err = s.executeWebhookActionInternal(ctx, wh, actionType)
- if err != nil {
- return nil, err
+ return err
}
- // Record trigger time — best-effort, do not fail the request if this update fails.
+ // Record trigger time on accept — best-effort, do not fail the request if this update fails.
now := time.Now()
_ = s.db.WithContext(ctx).Model(wh).Update("last_triggered_at", now).Error
- s.logWebhookEventInternal(ctx, wh, actionType, models.EventSeveritySuccess, "")
+ execCtx := context.WithoutCancel(ctx)
+ s.actions.Go(func() {
+ defer func() {
+ if panicErr := emperror.Recover(recover()); panicErr != nil {
+ slog.ErrorContext(execCtx, "webhook action panicked", "webhookID", wh.ID, "webhookName", wh.Name, "actionType", actionType, "error", panicErr)
+ }
+ }()
+ if _, err := s.executeWebhookActionInternal(execCtx, wh, actionType); err != nil {
+ // Action failures are recorded as error events by wrapWebhookActionErrorInternal.
+ slog.ErrorContext(execCtx, "webhook action failed", "webhookID", wh.ID, "webhookName", wh.Name, "actionType", actionType, "error", err)
+ return
+ }
+ s.logWebhookEventInternal(execCtx, wh, actionType, models.EventSeveritySuccess, "")
+ })
- return result, nil
+ return nil
+}
+
+// DrainActions blocks until all accepted background webhook actions finish or
+// ctx expires, so shutdown does not silently drop acknowledged work.
+func (s *WebhookService) DrainActions(ctx context.Context) error {
+ return utils.WaitGroup(ctx, &s.actions)
}
func (s *WebhookService) executeWebhookActionInternal(ctx context.Context, wh *models.Webhook, actionType string) (*updatertypes.Result, error) {
diff --git a/backend/internal/webhook/webhook_test.go b/backend/internal/webhook/webhook_test.go
index c0a60d7c9..2433a4ef5 100644
--- a/backend/internal/webhook/webhook_test.go
+++ b/backend/internal/webhook/webhook_test.go
@@ -491,7 +491,7 @@ func TestTriggerByToken_InvalidFormat(t *testing.T) {
db := setupWebhookServiceTestDB(t)
svc := newTestWebhookService(db)
- _, err := svc.TriggerByToken(ctx, "not-a-webhook-token")
+ err := svc.TriggerByToken(ctx, "not-a-webhook-token")
assert.ErrorIs(t, err, ErrWebhookInvalid)
}
@@ -502,7 +502,7 @@ func TestTriggerByToken_NotFound(t *testing.T) {
// Well-formatted token but not in the DB
raw := "arc_wh_0102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f20"
- _, err := svc.TriggerByToken(ctx, raw)
+ err := svc.TriggerByToken(ctx, raw)
assert.ErrorIs(t, err, ErrWebhookNotFound)
}
@@ -522,7 +522,7 @@ func TestTriggerByToken_WrongHash_NotFound(t *testing.T) {
tampered += "a"
}
- _, err = svc.TriggerByToken(ctx, tampered)
+ err = svc.TriggerByToken(ctx, tampered)
assert.ErrorIs(t, err, ErrWebhookNotFound)
}
@@ -536,7 +536,7 @@ func TestTriggerByToken_DisabledWebhook(t *testing.T) {
require.NoError(t, db.WithContext(ctx).Model(&models.Webhook{}).Where("id = ?", created.ID).Update("enabled", false).Error)
- _, err = svc.TriggerByToken(ctx, rawToken)
+ err = svc.TriggerByToken(ctx, rawToken)
assert.ErrorIs(t, err, ErrWebhookDisabled)
}
@@ -561,7 +561,7 @@ func TestTriggerByToken_UnknownTargetType_ReturnsInvalidType(t *testing.T) {
}
require.NoError(t, db.WithContext(ctx).Create(wh).Error)
- _, err := svc.TriggerByToken(ctx, rawToken)
+ err := svc.TriggerByToken(ctx, rawToken)
assert.ErrorIs(t, err, ErrWebhookInvalidType)
}
@@ -585,45 +585,20 @@ func insertWebhookDirect(t *testing.T, ctx context.Context, db *database.DB, raw
return wh
}
-func TestTriggerByToken_ContainerType_NilServiceReturnsError(t *testing.T) {
+func TestTriggerByToken_AcceptsImmediatelyAndRecordsTriggerTime(t *testing.T) {
ctx := context.Background()
db := setupWebhookServiceTestDB(t)
- svc := newTestWebhookService(db) // updaterService is nil
+ svc := newTestWebhookService(db) // action services are nil; the async action fails, acceptance must not
rawToken := "arc_wh_ccddeeff01020304aabbccdd0102030405060708090a0b0c0d0e0f1011121314"
- insertWebhookDirect(t, ctx, db, rawToken, models.WebhookTargetTypeContainer, models.WebhookActionTypeUpdate, "container-id", types.LocalDockerEnvironmentID)
+ wh := insertWebhookDirect(t, ctx, db, rawToken, models.WebhookTargetTypeContainer, models.WebhookActionTypeUpdate, "container-id", types.LocalDockerEnvironmentID)
- assert.Panics(t, func() {
- _, _ = svc.TriggerByToken(ctx, rawToken) //nolint:errcheck
- })
-}
-
-func TestTriggerByToken_UpdaterType_NilServiceReturnsError(t *testing.T) {
- ctx := context.Background()
- db := setupWebhookServiceTestDB(t)
- svc := newTestWebhookService(db) // updaterService is nil
-
- rawToken := "arc_wh_1122334401020304aabbccdd0102030405060708090a0b0c0d0e0f1011121314"
- insertWebhookDirect(t, ctx, db, rawToken, models.WebhookTargetTypeUpdater, models.WebhookActionTypeRun, "", types.LocalDockerEnvironmentID)
+ // Valid token is accepted synchronously; the action itself runs (and here
+ // fails against nil services) in the background without surfacing.
+ require.NoError(t, svc.TriggerByToken(ctx, rawToken))
- // nil updaterService causes a panic, which we verify the dispatch path is reached
- // by recovering — in production the service is always non-nil
- assert.Panics(t, func() {
- _, _ = svc.TriggerByToken(ctx, rawToken) //nolint:errcheck
- })
-}
-
-func TestTriggerByToken_GitOpsType_NilServiceReturnsError(t *testing.T) {
- ctx := context.Background()
- db := setupWebhookServiceTestDB(t)
- svc := newTestWebhookService(db) // gitOpsSyncService is nil
-
- rawToken := "arc_wh_aabbccdd11223344aabbccdd0102030405060708090a0b0c0d0e0f1011121314"
- insertWebhookDirect(t, ctx, db, rawToken, models.WebhookTargetTypeGitOps, models.WebhookActionTypeSync, "sync-id", types.LocalDockerEnvironmentID)
-
- assert.Panics(t, func() {
- _, _ = svc.TriggerByToken(ctx, rawToken) //nolint:errcheck
- })
+ stored := fetchWebhook(t, db, wh.ID)
+ assert.NotNil(t, stored.LastTriggeredAt, "last_triggered_at must be set once the trigger is accepted")
}
func TestTriggerByToken_DoesNotUpdateLastTriggeredAtOnError(t *testing.T) {
@@ -634,7 +609,7 @@ func TestTriggerByToken_DoesNotUpdateLastTriggeredAtOnError(t *testing.T) {
rawToken := "arc_wh_1122334401020304aabbccdd0102030405060708090a0b0c0d0e0f1011121315"
wh := insertWebhookDirect(t, ctx, db, rawToken, "unknown-type", models.WebhookActionTypeUpdate, "some-id", "env-1")
- _, err := svc.TriggerByToken(ctx, rawToken)
+ err := svc.TriggerByToken(ctx, rawToken)
require.Error(t, err)
stored := fetchWebhook(t, db, wh.ID)
@@ -649,7 +624,7 @@ func TestTriggerByToken_UnknownActionType_ReturnsInvalidAction(t *testing.T) {
rawToken := "arc_wh_0011223344556677aabbccdd0102030405060708090a0b0c0d0e0f1011121314"
insertWebhookDirect(t, ctx, db, rawToken, models.WebhookTargetTypeProject, "bogus", "project-id", "env-1")
- _, err := svc.TriggerByToken(ctx, rawToken)
+ err := svc.TriggerByToken(ctx, rawToken)
assert.ErrorIs(t, err, ErrWebhookInvalidAction)
}