Skip to content

Modern Docker Management, Designed for Everyone

fix: rate-limit webhook triggers by token instead of client IP (#3742)

Ligode1e978810a4
diff --git a/backend/api/webhooks_trigger.go b/backend/api/webhooks_trigger.go
index 65f24ca0a..157f34c48 100644
--- a/backend/api/webhooks_trigger.go
+++ b/backend/api/webhooks_trigger.go
@@ -13,7 +13,8 @@ import (
// RegisterWebhookTrigger registers the public (unauthenticated) trigger endpoint.
// The token in the URL is the sole authentication mechanism.
-// Rate-limited via PerIPRateLimitForPaths in router_bootstrap.go.
+// Rate-limited via PerTokenRateLimitForPaths in router_bootstrap.go: authentic tokens
+// get per-token buckets, everything else shares a per-source-IP budget.
func RegisterWebhookTrigger(g *echo.Group, webhookService *webhook.WebhookService, appCtx handlerutil.ActivityAppContext) {
g.POST("/webhooks/trigger/:token", func(c *echo.Context) error {
if webhookService == nil {
diff --git a/backend/internal/bootstrap/router_bootstrap.go b/backend/internal/bootstrap/router_bootstrap.go
index 8cbae4f9f..198ed3957 100644
--- a/backend/internal/bootstrap/router_bootstrap.go
+++ b/backend/internal/bootstrap/router_bootstrap.go
@@ -239,8 +239,9 @@ func newRouter(p RouterParams) (*echo.Echo, *edge.TunnelServer) {
apiGroup.Use(middleware.PerIPRateLimitForPaths(
[]string{"/api/auth/federated/token"}, 10, 10,
))
- apiGroup.Use(middleware.PerIPRateLimitForPaths(
- []string{"/api/webhooks/trigger/:token"}, 60, 10,
+ // Keyed per token for stored tokens, per source IP for everything else.
+ apiGroup.Use(middleware.PerTokenRateLimitForPaths(
+ []string{"/api/webhooks/trigger/:token"}, 60, 10, deps.Webhook.Service().IsKnownToken,
))
// Agent event ingestion authenticates on the agent token alone and sits
// outside the auth middleware, so it needs its own brute-force ceiling.
diff --git a/backend/internal/di/providers.go b/backend/internal/di/providers.go
index 7eac7610c..273a5b4fb 100644
--- a/backend/internal/di/providers.go
+++ b/backend/internal/di/providers.go
@@ -451,6 +451,7 @@ func provideWebhookModuleInternal(lc fx.Lifecycle, db *database.DB, containerMod
Environment: environment,
})
lc.Append(fx.Hook{
+ OnStart: module.Service().LoadTokenHashes,
OnStop: func(ctx context.Context) error {
// Drain fails only when the stop context expires; fx then skips all
// remaining teardown regardless of what is returned here, so an
diff --git a/backend/internal/middleware/rate_limit_middleware.go b/backend/internal/middleware/rate_limit_middleware.go
index cfcad5037..3d83e4ac2 100644
--- a/backend/internal/middleware/rate_limit_middleware.go
+++ b/backend/internal/middleware/rate_limit_middleware.go
@@ -165,6 +165,53 @@ func agentTokenRateLimitKeyInternal(token string) string {
return hex.EncodeToString(sum[:])
}
+// PerTokenRateLimitForPaths gives known tokens independent budgets and limits others per IP.
+// isKnown must be an in-memory lookup with bounded input, without request-driven cache fills.
+// A nil lookup falls back to the shared IP limiter.
+func PerTokenRateLimitForPaths(
+ paths []string,
+ perMinute int,
+ burst int,
+ isKnown func(token string) bool,
+) echo.MiddlewareFunc {
+ if perMinute <= 0 {
+ perMinute = 10
+ }
+ if burst <= 0 {
+ burst = perMinute
+ }
+ ipLimiter := newIPRateLimiterInternal(rate.Every(time.Minute/time.Duration(perMinute)), burst)
+ tokenLimiter := newIPRateLimiterInternal(rate.Every(time.Minute/time.Duration(perMinute)), burst)
+
+ pathSet := make(map[string]struct{}, len(paths))
+ for _, p := range paths {
+ pathSet[p] = struct{}{}
+ }
+
+ return func(next echo.HandlerFunc) echo.HandlerFunc {
+ return func(c *echo.Context) error {
+ if _, ok := pathSet[c.Path()]; !ok {
+ return next(c)
+ }
+
+ token := c.Param("token")
+ if token != "" && isKnown != nil && isKnown(token) {
+ if !tokenLimiter.allow(agentTokenRateLimitKeyInternal(token)) {
+ c.Response().Header().Set("Retry-After", "60")
+ return c.JSON(http.StatusTooManyRequests, map[string]any{"error": "rate limit exceeded"})
+ }
+ return next(c)
+ }
+
+ if !ipLimiter.allow(clientIPForRateLimitInternal(c)) {
+ c.Response().Header().Set("Retry-After", "60")
+ return c.JSON(http.StatusTooManyRequests, map[string]any{"error": "rate limit exceeded"})
+ }
+ return next(c)
+ }
+ }
+}
+
// PerIPRateLimitForPaths returns an Echo middleware that applies a per-IP
// rate limit only when c.Path() (the registered route pattern) is in paths.
// Each path gets its own independent token bucket, so traffic on one path
diff --git a/backend/internal/middleware/rate_limit_middleware_test.go b/backend/internal/middleware/rate_limit_middleware_test.go
index e1541d368..75c0f1137 100644
--- a/backend/internal/middleware/rate_limit_middleware_test.go
+++ b/backend/internal/middleware/rate_limit_middleware_test.go
@@ -3,6 +3,7 @@ package middleware
import (
"net/http"
"net/http/httptest"
+ "strconv"
"testing"
"time"
@@ -162,6 +163,137 @@ func TestPerIPRateLimitForPaths_RouteParamsDoNotEscapeFilter(t *testing.T) {
require.Equal(t, http.StatusTooManyRequests, doReq("token-bbb"))
}
+// newTokenRateLimitTestRouterInternal wires PerTokenRateLimitForPaths ahead
+// of handler and returns a doReq helper posting from a fixed source IP.
+func newTokenRateLimitTestRouterInternal(
+ t *testing.T,
+ perMinute, burst int,
+ isAuthentic func(string) bool,
+ handler echo.HandlerFunc,
+) (doReq func(token string) int) {
+ t.Helper()
+
+ router := echo.New()
+ router.IPExtractor = echo.ExtractIPDirect()
+ router.Use(PerTokenRateLimitForPaths([]string{"/webhooks/trigger/:token"}, perMinute, burst, isAuthentic))
+ router.POST("/webhooks/trigger/:token", handler)
+
+ return func(token string) int {
+ req := httptest.NewRequest(http.MethodPost, "/webhooks/trigger/"+token, nil)
+ req.RemoteAddr = "192.0.2.10:4000"
+ rec := httptest.NewRecorder()
+ router.ServeHTTP(rec, req)
+ return rec.Code
+ }
+}
+
+func okHandlerInternal(c *echo.Context) error {
+ return c.NoContent(http.StatusOK)
+}
+
+func TestPerTokenRateLimitForPaths_TracksDistinctTokensNotIP(t *testing.T) {
+ acceptAll := func(string) bool { return true }
+ doReq := newTokenRateLimitTestRouterInternal(t, 60, 1, acceptAll, okHandlerInternal)
+
+ require.Equal(t, http.StatusOK, doReq("token-a"))
+ require.Equal(t, http.StatusOK, doReq("token-b"))
+ require.Equal(t, http.StatusOK, doReq("token-c"))
+ require.Equal(t, http.StatusTooManyRequests, doReq("token-a"))
+}
+
+// TestPerTokenRateLimitForPaths_NilAuthenticatorFailsClosedToSharedIPBucket proves
+// a nil isAuthentic does not hand out independent per-token buckets: all nonempty
+// tokens contend for one shared IP bucket and trip 429 together.
+func TestPerTokenRateLimitForPaths_NilAuthenticatorFailsClosedToSharedIPBucket(t *testing.T) {
+ doReq := newTokenRateLimitTestRouterInternal(t, 60, 1, nil, okHandlerInternal)
+
+ require.Equal(t, http.StatusOK, doReq("token-a"))
+ require.Equal(t, http.StatusTooManyRequests, doReq("token-b"),
+ "a nil authenticator must fail closed: distinct tokens share the IP bucket")
+ require.Equal(t, http.StatusTooManyRequests, doReq("token-a"))
+}
+
+func TestPerTokenRateLimitForPaths_IPCeilingBoundsUnauthenticTokenCycling(t *testing.T) {
+ rejectAll := func(string) bool { return false }
+ const perMinute, burst = 1, 3
+ handlerHits := 0
+ doReq := newTokenRateLimitTestRouterInternal(t, perMinute, burst, rejectAll, func(c *echo.Context) error {
+ handlerHits++
+ return c.NoContent(http.StatusNotFound)
+ })
+
+ rejections := 0
+ for i := range 200 {
+ if doReq(strconv.Itoa(i)) == http.StatusTooManyRequests {
+ rejections++
+ }
+ }
+
+ require.Positive(t, rejections, "expected the IP ceiling to reject some requests once its burst is exhausted")
+ require.Equal(t, burst, handlerHits,
+ "distinct unauthenticated tokens from one IP must reach the handler only up to the configured burst; more would mean the old 10x IP allowance came back")
+}
+
+func TestPerTokenRateLimitForPaths_MalformedTokensGetNoBucketButStillHitIPCeiling(t *testing.T) {
+ isAuthentic := func(token string) bool {
+ return token == "well-formed"
+ }
+ const perMinute, burst = 2, 2
+
+ doReq := newTokenRateLimitTestRouterInternal(t, perMinute, burst, isAuthentic, okHandlerInternal)
+
+ successes := 0
+ for i := range burst + 10 {
+ if doReq("not-well-formed-"+strconv.Itoa(i)) == http.StatusOK {
+ successes++
+ }
+ }
+
+ require.LessOrEqual(t, successes, burst,
+ "malformed tokens must be bounded by the IP ceiling, not given their own per-token bucket")
+}
+
+func TestPerTokenRateLimitForPaths_AuthenticTokenNotStarvedByGarbageFromSameIP(t *testing.T) {
+ isAuthentic := func(token string) bool {
+ return token == "authentic"
+ }
+ doReq := newTokenRateLimitTestRouterInternal(t, 2, 2, isAuthentic, okHandlerInternal)
+
+ for i := range 100 {
+ doReq("garbage-" + strconv.Itoa(i))
+ }
+ require.Equal(t, http.StatusTooManyRequests, doReq("garbage-final"),
+ "IP ceiling must be exhausted by the garbage traffic")
+
+ require.Equal(t, http.StatusOK, doReq("authentic"),
+ "an authentic token must not be starved by garbage traffic from its source IP")
+}
+
+func TestPerTokenRateLimitForPaths_AppliesOnlyToConfiguredPaths(t *testing.T) {
+ router := echo.New()
+ router.IPExtractor = echo.ExtractIPDirect()
+
+ router.Use(PerTokenRateLimitForPaths([]string{"/webhooks/trigger/:token"}, 60, 1, nil))
+ router.POST("/webhooks/trigger/:token", func(c *echo.Context) error {
+ return c.NoContent(http.StatusOK)
+ })
+ router.POST("/unlimited/:token", func(c *echo.Context) error {
+ return c.NoContent(http.StatusOK)
+ })
+
+ doReq := func(path string) int {
+ req := httptest.NewRequest(http.MethodPost, path, nil)
+ req.RemoteAddr = "192.0.2.10:4000"
+ rec := httptest.NewRecorder()
+ router.ServeHTTP(rec, req)
+ return rec.Code
+ }
+
+ for range 10 {
+ require.Equal(t, http.StatusOK, doReq("/unlimited/same-token"))
+ }
+}
+
func TestIPRateLimiter_EnforcesMaxEntriesForRecentClients(t *testing.T) {
limiter := newIPRateLimiterInternal(1, 1)
limiter.maxEntries = 3
diff --git a/backend/internal/webhook/webhook.go b/backend/internal/webhook/webhook.go
index cbafd6298..7926ef1a5 100644
--- a/backend/internal/webhook/webhook.go
+++ b/backend/internal/webhook/webhook.go
@@ -48,9 +48,15 @@ const (
webhookTokenPrefix = "arc_wh_"
webhookTokenLength = 32 // raw bytes → 64 hex chars
webhookTokenPrefixLen = 8 // chars of the hex portion used as lookup prefix
+ // hex length of a generated token's ciphertext: GCM nonce (12) + encrypted secretHex + GCM tag (16)
+ webhookTokenHexLen = 2 * (12 + webhookTokenLength*2 + 16)
)
type WebhookService struct {
+ tokenWriteMu sync.Mutex
+ tokenMu sync.RWMutex
+ tokenHashes map[string]struct{}
+
// actions tracks in-flight background webhook actions accepted with a 202
// so shutdown can drain them instead of dropping acknowledged work.
actions sync.WaitGroup
@@ -119,6 +125,37 @@ func parseWebhookPrefixInternal(raw string) (string, error) {
return webhookTokenPrefix + hexPart[:webhookTokenPrefixLen], nil
}
+// LoadTokenHashes loads the rate-limit index before the server accepts requests.
+func (s *WebhookService) LoadTokenHashes(ctx context.Context) error {
+ s.tokenWriteMu.Lock()
+ defer s.tokenWriteMu.Unlock()
+
+ var hashes []string
+ if err := s.db.WithContext(ctx).Model(&Webhook{}).Pluck("token_hash", &hashes).Error; err != nil {
+ return errors.WrapIf(err, "failed to load webhook token hashes")
+ }
+ s.tokenMu.Lock()
+ defer s.tokenMu.Unlock()
+ s.tokenHashes = make(map[string]struct{}, len(hashes))
+ for _, hash := range hashes {
+ s.tokenHashes[hash] = struct{}{}
+ }
+ return nil
+}
+
+// IsKnownToken checks stored token identity without decrypting or querying the database.
+// TriggerByToken still checks existence and enabled state before accepting an action.
+func (s *WebhookService) IsKnownToken(raw string) bool {
+ if s == nil || len(raw) > len(webhookTokenPrefix)+webhookTokenHexLen || !strings.HasPrefix(raw, webhookTokenPrefix) {
+ return false
+ }
+ hash := hashWebhookTokenInternal(raw)
+ s.tokenMu.RLock()
+ defer s.tokenMu.RUnlock()
+ _, ok := s.tokenHashes[hash]
+ return ok
+}
+
func defaultWebhookActionTypeInternal(targetType string) (string, error) {
switch targetType {
case WebhookTargetTypeContainer, WebhookTargetTypeProject:
@@ -223,10 +260,19 @@ func (s *WebhookService) CreateWebhook(ctx context.Context, name, targetType, ac
Enabled: true,
}
+ s.tokenWriteMu.Lock()
+ defer s.tokenWriteMu.Unlock()
if err := s.db.WithContext(ctx).Create(wh).Error; err != nil {
return nil, "", errors.WrapIf(err, "failed to create webhook")
}
+ s.tokenMu.Lock()
+ if s.tokenHashes == nil {
+ s.tokenHashes = make(map[string]struct{})
+ }
+ s.tokenHashes[hash] = struct{}{}
+ s.tokenMu.Unlock()
+
if s.eventService != nil {
_, _ = s.eventService.CreateEvent(ctx, event.CreateEventRequest{
Type: event.EventTypeWebhookCreate,
@@ -359,6 +405,8 @@ func (s *WebhookService) DeleteWebhook(ctx context.Context, id, environmentID st
return err
}
+ s.tokenWriteMu.Lock()
+ defer s.tokenWriteMu.Unlock()
result := s.db.WithContext(ctx).
Where("id = ? AND environment_id = ?", id, environmentID).
Delete(&Webhook{})
@@ -369,6 +417,10 @@ func (s *WebhookService) DeleteWebhook(ctx context.Context, id, environmentID st
return ErrWebhookNotFound
}
+ s.tokenMu.Lock()
+ delete(s.tokenHashes, wh.TokenHash)
+ s.tokenMu.Unlock()
+
if s.eventService != nil {
_, _ = s.eventService.CreateEvent(ctx, event.CreateEventRequest{
Type: event.EventTypeWebhookDelete,
diff --git a/backend/internal/webhook/webhook_test.go b/backend/internal/webhook/webhook_test.go
index 4f556ec50..2c59fd2cf 100644
--- a/backend/internal/webhook/webhook_test.go
+++ b/backend/internal/webhook/webhook_test.go
@@ -8,10 +8,15 @@ import (
"encoding/base64"
"encoding/hex"
"fmt"
+ "net/http"
+ "net/http/httptest"
"strings"
+ "sync"
"testing"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/middleware"
"github.com/getarcaneapp/arcane/types/v2"
+ "github.com/labstack/echo/v5"
"github.com/libtnb/sqlite"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@@ -128,6 +133,109 @@ func TestParseWebhookPrefix_LeadingWhitespaceStripped(t *testing.T) {
assert.Equal(t, "arc_wh_01020304", prefix)
}
+func TestIsKnownToken_StoredIdentity(t *testing.T) {
+ ctx := context.Background()
+ db := setupWebhookServiceTestDB(t)
+ svc := newTestWebhookService(db)
+ wh, raw, err := svc.CreateWebhook(ctx, "known", WebhookTargetTypeProject, WebhookActionTypeUpdate, "p1", "env-1", common.User{})
+ require.NoError(t, err)
+ require.True(t, svc.IsKnownToken(raw))
+
+ unstored, _, _, err := generateWebhookTokenInternal()
+ require.NoError(t, err)
+ for _, token := range []string{unstored, "", "arc_wh_bad", " " + raw, raw + " ", strings.ToUpper(raw), raw[:len(raw)-1], webhookTokenPrefix + strings.Repeat("ab", 512*1024)} {
+ require.False(t, svc.IsKnownToken(token))
+ }
+ var unavailable *WebhookService
+ require.False(t, unavailable.IsKnownToken(raw))
+
+ restarted := newTestWebhookService(db)
+ require.NoError(t, restarted.LoadTokenHashes(ctx))
+ require.True(t, restarted.IsKnownToken(raw), "existing tokens must be recognized before their first request after restart")
+ _, err = restarted.UpdateWebhook(ctx, wh.ID, "env-1", false, common.User{})
+ require.NoError(t, err)
+ require.True(t, restarted.IsKnownToken(raw), "disabled tokens retain their independent abuse budget")
+ require.ErrorIs(t, restarted.TriggerByToken(ctx, raw), ErrWebhookDisabled)
+ require.ErrorIs(t, restarted.DeleteWebhook(ctx, wh.ID, "wrong-environment", common.User{}), ErrWebhookNotFound)
+ require.True(t, restarted.IsKnownToken(raw))
+ require.NoError(t, restarted.DeleteWebhook(ctx, wh.ID, "env-1", common.User{}))
+ require.False(t, restarted.IsKnownToken(raw))
+ require.ErrorIs(t, restarted.TriggerByToken(ctx, raw), ErrWebhookNotFound)
+}
+
+func TestKnownTokenRateLimit_ExhaustedIPDoesNotBlockFirstUse(t *testing.T) {
+ ctx := context.Background()
+ db := setupWebhookServiceTestDB(t)
+ svc := newTestWebhookService(db)
+ _, first, err := svc.CreateWebhook(ctx, "first", WebhookTargetTypeProject, WebhookActionTypeUpdate, "p1", "env-1", common.User{})
+ require.NoError(t, err)
+ _, second, err := svc.CreateWebhook(ctx, "second", WebhookTargetTypeProject, WebhookActionTypeUpdate, "p2", "env-1", common.User{})
+ require.NoError(t, err)
+ svc = newTestWebhookService(db)
+ require.NoError(t, svc.LoadTokenHashes(ctx))
+
+ // Closing the database proves admission never queries it, even for unknown tokens.
+ sqlDB, err := db.DB.DB()
+ require.NoError(t, err)
+ require.NoError(t, sqlDB.Close())
+ router := echo.New()
+ router.IPExtractor = echo.ExtractIPDirect()
+ router.Use(middleware.PerTokenRateLimitForPaths([]string{"/trigger/:token"}, 1, 1, svc.IsKnownToken))
+ router.POST("/trigger/:token", func(c *echo.Context) error { return c.NoContent(http.StatusAccepted) })
+ request := func(token string) int {
+ req := httptest.NewRequest(http.MethodPost, "/trigger/"+token, nil)
+ req.RemoteAddr = "192.0.2.10:4000"
+ rec := httptest.NewRecorder()
+ router.ServeHTTP(rec, req)
+ if rec.Code == http.StatusTooManyRequests {
+ require.Equal(t, "60", rec.Header().Get("Retry-After"))
+ }
+ return rec.Code
+ }
+ require.Equal(t, http.StatusAccepted, request(webhookTokenPrefix+strings.Repeat("0", webhookTokenHexLen)))
+ for i := range 200 {
+ require.Equal(t, http.StatusTooManyRequests, request(fmt.Sprintf("%s%0184x", webhookTokenPrefix, i+1)))
+ }
+ require.Equal(t, http.StatusAccepted, request(first))
+ require.Equal(t, http.StatusTooManyRequests, request(first))
+ require.Equal(t, http.StatusAccepted, request(second))
+}
+
+func TestLoadTokenHashes_FailurePreservesIndex(t *testing.T) {
+ ctx := context.Background()
+ db := setupWebhookServiceTestDB(t)
+ svc := newTestWebhookService(db)
+ _, raw, err := svc.CreateWebhook(ctx, "known", WebhookTargetTypeProject, WebhookActionTypeUpdate, "p1", "env-1", common.User{})
+ require.NoError(t, err)
+ require.NoError(t, db.Migrator().DropTable(&Webhook{}))
+ require.Error(t, svc.LoadTokenHashes(ctx))
+ require.True(t, svc.IsKnownToken(raw))
+ _, failed, err := svc.CreateWebhook(ctx, "failed", WebhookTargetTypeProject, WebhookActionTypeUpdate, "p1", "env-1", common.User{})
+ require.Error(t, err)
+ require.Empty(t, failed)
+ require.Len(t, svc.tokenHashes, 1)
+}
+
+func TestKnownTokenIndex_ConcurrentReloadAndCRUD(t *testing.T) {
+ ctx := context.Background()
+ svc := newTestWebhookService(setupWebhookServiceTestDB(t))
+ var workers sync.WaitGroup
+ workers.Go(func() {
+ for range 30 {
+ assert.NoError(t, svc.LoadTokenHashes(ctx))
+ svc.IsKnownToken("arc_wh_unknown")
+ }
+ })
+ for range 30 {
+ wh, raw, err := svc.CreateWebhook(ctx, "known", WebhookTargetTypeProject, WebhookActionTypeUpdate, "p1", "env-1", common.User{})
+ require.NoError(t, err)
+ require.True(t, svc.IsKnownToken(raw))
+ require.NoError(t, svc.DeleteWebhook(ctx, wh.ID, "env-1", common.User{}))
+ require.False(t, svc.IsKnownToken(raw))
+ }
+ workers.Wait()
+}
+
// --- CreateWebhook ---
func TestCreateWebhook_TokenNotStoredInPlaintext(t *testing.T) {