Skip to content

The open alternative to Salesforce, designed for AI.

feat(usage-limit): cap webhook calls per workspace at enqueue time (#25673)

Etienne9b3e4d074506
diff --git a/packages/twenty-client-sdk/src/metadata/generated/schema.graphql b/packages/twenty-client-sdk/src/metadata/generated/schema.graphql
index 9199edf39dc..f40c399b8e0 100644
--- a/packages/twenty-client-sdk/src/metadata/generated/schema.graphql
+++ b/packages/twenty-client-sdk/src/metadata/generated/schema.graphql
@@ -1804,6 +1804,7 @@ enum UsageResourceType {
API
LOGIC_FUNCTION
EMAIL
+ WEBHOOK
}
enum UsageOperationType {
@@ -1816,6 +1817,7 @@ enum UsageOperationType {
CALL_RECORDING
EMAIL_SEND
API_REQUEST
+ WEBHOOK_CALL
SUBSCRIPTION
}
@@ -1923,6 +1925,7 @@ enum FeatureFlagKey {
IS_QUOTA_ENGINE_CREDIT_BOUND_ENABLED
IS_RECORD_CREATION_FORM_ENABLED
IS_RECORD_SHARING_ENABLED
+ IS_WEBHOOK_RATE_LIMIT_ENABLED
}
type WorkspaceUrls {
diff --git a/packages/twenty-client-sdk/src/metadata/generated/schema.ts b/packages/twenty-client-sdk/src/metadata/generated/schema.ts
index ba0210f8961..09bcc02adae 100644
--- a/packages/twenty-client-sdk/src/metadata/generated/schema.ts
+++ b/packages/twenty-client-sdk/src/metadata/generated/schema.ts
@@ -1445,9 +1445,9 @@ export interface UsageLimit {
__typename: 'UsageLimit'
}
-export type UsageResourceType = 'AI' | 'WORKFLOW' | 'APP' | 'STORAGE' | 'API' | 'LOGIC_FUNCTION' | 'EMAIL'
+export type UsageResourceType = 'AI' | 'WORKFLOW' | 'APP' | 'STORAGE' | 'API' | 'LOGIC_FUNCTION' | 'EMAIL' | 'WEBHOOK'
-export type UsageOperationType = 'ALL' | 'AI_CHAT_TOKEN' | 'AI_WORKFLOW_TOKEN' | 'WORKFLOW_EXECUTION' | 'CODE_EXECUTION' | 'WEB_SEARCH' | 'CALL_RECORDING' | 'EMAIL_SEND' | 'API_REQUEST' | 'SUBSCRIPTION'
+export type UsageOperationType = 'ALL' | 'AI_CHAT_TOKEN' | 'AI_WORKFLOW_TOKEN' | 'WORKFLOW_EXECUTION' | 'CODE_EXECUTION' | 'WEB_SEARCH' | 'CALL_RECORDING' | 'EMAIL_SEND' | 'API_REQUEST' | 'WEBHOOK_CALL' | 'SUBSCRIPTION'
export interface ApprovedAccessDomain {
id: Scalars['UUID']
@@ -1538,7 +1538,7 @@ export interface FeatureFlag {
__typename: 'FeatureFlag'
}
-export type FeatureFlagKey = 'IS_APP_CLAIMING_ENABLED' | 'IS_UNIQUE_INDEXES_ENABLED' | 'IS_CONFIGURABLE_SEARCH_FIELDS_ENABLED' | 'IS_JSON_FILTER_ENABLED' | 'IS_EMAIL_GROUP_ENABLED' | 'IS_JUNCTION_RELATIONS_ENABLED' | 'IS_REST_METADATA_API_NEW_FORMAT_DIRECT' | 'IS_LOGIC_FUNCTION_PREBUILT_MODE_ENABLED' | 'IS_WORKFLOW_CORE_INDEX_PAGE_ENABLED' | 'IS_API_RATE_LIMIT_V2_ENABLED' | 'IS_MESSAGE_CALENDAR_TARGET_READ_ENABLED' | 'IS_QUOTA_ENGINE_CREDIT_BOUND_ENABLED' | 'IS_RECORD_CREATION_FORM_ENABLED' | 'IS_RECORD_SHARING_ENABLED'
+export type FeatureFlagKey = 'IS_APP_CLAIMING_ENABLED' | 'IS_UNIQUE_INDEXES_ENABLED' | 'IS_CONFIGURABLE_SEARCH_FIELDS_ENABLED' | 'IS_JSON_FILTER_ENABLED' | 'IS_EMAIL_GROUP_ENABLED' | 'IS_JUNCTION_RELATIONS_ENABLED' | 'IS_REST_METADATA_API_NEW_FORMAT_DIRECT' | 'IS_LOGIC_FUNCTION_PREBUILT_MODE_ENABLED' | 'IS_WORKFLOW_CORE_INDEX_PAGE_ENABLED' | 'IS_API_RATE_LIMIT_V2_ENABLED' | 'IS_MESSAGE_CALENDAR_TARGET_READ_ENABLED' | 'IS_QUOTA_ENGINE_CREDIT_BOUND_ENABLED' | 'IS_RECORD_CREATION_FORM_ENABLED' | 'IS_RECORD_SHARING_ENABLED' | 'IS_WEBHOOK_RATE_LIMIT_ENABLED'
export interface WorkspaceUrls {
customUrl?: Scalars['String']
@@ -10381,7 +10381,8 @@ export const enumUsageResourceType = {
STORAGE: 'STORAGE' as const,
API: 'API' as const,
LOGIC_FUNCTION: 'LOGIC_FUNCTION' as const,
- EMAIL: 'EMAIL' as const
+ EMAIL: 'EMAIL' as const,
+ WEBHOOK: 'WEBHOOK' as const
}
export const enumUsageOperationType = {
@@ -10394,6 +10395,7 @@ export const enumUsageOperationType = {
CALL_RECORDING: 'CALL_RECORDING' as const,
EMAIL_SEND: 'EMAIL_SEND' as const,
API_REQUEST: 'API_REQUEST' as const,
+ WEBHOOK_CALL: 'WEBHOOK_CALL' as const,
SUBSCRIPTION: 'SUBSCRIPTION' as const
}
@@ -10417,7 +10419,8 @@ export const enumFeatureFlagKey = {
IS_MESSAGE_CALENDAR_TARGET_READ_ENABLED: 'IS_MESSAGE_CALENDAR_TARGET_READ_ENABLED' as const,
IS_QUOTA_ENGINE_CREDIT_BOUND_ENABLED: 'IS_QUOTA_ENGINE_CREDIT_BOUND_ENABLED' as const,
IS_RECORD_CREATION_FORM_ENABLED: 'IS_RECORD_CREATION_FORM_ENABLED' as const,
- IS_RECORD_SHARING_ENABLED: 'IS_RECORD_SHARING_ENABLED' as const
+ IS_RECORD_SHARING_ENABLED: 'IS_RECORD_SHARING_ENABLED' as const,
+ IS_WEBHOOK_RATE_LIMIT_ENABLED: 'IS_WEBHOOK_RATE_LIMIT_ENABLED' as const
}
export const enumIdentityProviderType = {
diff --git a/packages/twenty-front/src/generated-admin/graphql.ts b/packages/twenty-front/src/generated-admin/graphql.ts
index e39e9b130ec..699996d7552 100644
--- a/packages/twenty-front/src/generated-admin/graphql.ts
+++ b/packages/twenty-front/src/generated-admin/graphql.ts
@@ -414,6 +414,7 @@ export enum FeatureFlagKey {
IS_RECORD_SHARING_ENABLED = 'IS_RECORD_SHARING_ENABLED',
IS_REST_METADATA_API_NEW_FORMAT_DIRECT = 'IS_REST_METADATA_API_NEW_FORMAT_DIRECT',
IS_UNIQUE_INDEXES_ENABLED = 'IS_UNIQUE_INDEXES_ENABLED',
+ IS_WEBHOOK_RATE_LIMIT_ENABLED = 'IS_WEBHOOK_RATE_LIMIT_ENABLED',
IS_WORKFLOW_CORE_INDEX_PAGE_ENABLED = 'IS_WORKFLOW_CORE_INDEX_PAGE_ENABLED'
}
diff --git a/packages/twenty-front/src/generated-metadata/graphql.ts b/packages/twenty-front/src/generated-metadata/graphql.ts
index 4ce39c4c7db..a3eaead18bb 100644
--- a/packages/twenty-front/src/generated-metadata/graphql.ts
+++ b/packages/twenty-front/src/generated-metadata/graphql.ts
@@ -1966,6 +1966,7 @@ export enum FeatureFlagKey {
IS_RECORD_SHARING_ENABLED = 'IS_RECORD_SHARING_ENABLED',
IS_REST_METADATA_API_NEW_FORMAT_DIRECT = 'IS_REST_METADATA_API_NEW_FORMAT_DIRECT',
IS_UNIQUE_INDEXES_ENABLED = 'IS_UNIQUE_INDEXES_ENABLED',
+ IS_WEBHOOK_RATE_LIMIT_ENABLED = 'IS_WEBHOOK_RATE_LIMIT_ENABLED',
IS_WORKFLOW_CORE_INDEX_PAGE_ENABLED = 'IS_WORKFLOW_CORE_INDEX_PAGE_ENABLED'
}
@@ -6654,6 +6655,7 @@ export enum UsageOperationType {
CODE_EXECUTION = 'CODE_EXECUTION',
EMAIL_SEND = 'EMAIL_SEND',
SUBSCRIPTION = 'SUBSCRIPTION',
+ WEBHOOK_CALL = 'WEBHOOK_CALL',
WEB_SEARCH = 'WEB_SEARCH',
WORKFLOW_EXECUTION = 'WORKFLOW_EXECUTION'
}
@@ -6665,6 +6667,7 @@ export enum UsageResourceType {
EMAIL = 'EMAIL',
LOGIC_FUNCTION = 'LOGIC_FUNCTION',
STORAGE = 'STORAGE',
+ WEBHOOK = 'WEBHOOK',
WORKFLOW = 'WORKFLOW'
}
diff --git a/packages/twenty-front/src/modules/settings/usage/constants/UsageOperationTypeLabels.ts b/packages/twenty-front/src/modules/settings/usage/constants/UsageOperationTypeLabels.ts
index be2a0d5a352..66dea4ea704 100644
--- a/packages/twenty-front/src/modules/settings/usage/constants/UsageOperationTypeLabels.ts
+++ b/packages/twenty-front/src/modules/settings/usage/constants/UsageOperationTypeLabels.ts
@@ -18,5 +18,6 @@ export const USAGE_OPERATION_TYPE_LABELS: Record<
[UsageOperationType.CALL_RECORDING]: msg`Call Recording`,
[UsageOperationType.EMAIL_SEND]: msg`Email Send`,
[UsageOperationType.API_REQUEST]: msg`API Request`,
+ [UsageOperationType.WEBHOOK_CALL]: msg`Webhook Call`,
[UsageOperationType.SUBSCRIPTION]: msg`Subscription`,
};
diff --git a/packages/twenty-server/src/engine/core-modules/billing/app-billing/app-billing.service.ts b/packages/twenty-server/src/engine/core-modules/billing/app-billing/app-billing.service.ts
index 26b88d0e477..67d1f490740 100644
--- a/packages/twenty-server/src/engine/core-modules/billing/app-billing/app-billing.service.ts
+++ b/packages/twenty-server/src/engine/core-modules/billing/app-billing/app-billing.service.ts
@@ -30,6 +30,7 @@ const USAGE_UNIT_BY_OPERATION_TYPE: Record<
[UsageOperationType.CALL_RECORDING]: UsageUnit.MINUTE,
[UsageOperationType.EMAIL_SEND]: UsageUnit.INVOCATION,
[UsageOperationType.API_REQUEST]: UsageUnit.REQUEST,
+ [UsageOperationType.WEBHOOK_CALL]: UsageUnit.REQUEST,
// Platform-raised only; ApplicationRecurringChargeService sets its own unit.
[UsageOperationType.SUBSCRIPTION]: UsageUnit.CREDIT,
};
diff --git a/packages/twenty-server/src/engine/core-modules/metrics/types/metrics-keys.type.ts b/packages/twenty-server/src/engine/core-modules/metrics/types/metrics-keys.type.ts
index 2a3016c897f..517d406e849 100644
--- a/packages/twenty-server/src/engine/core-modules/metrics/types/metrics-keys.type.ts
+++ b/packages/twenty-server/src/engine/core-modules/metrics/types/metrics-keys.type.ts
@@ -56,6 +56,7 @@ export enum MetricsKeys {
AppRegistrationVersionPublished = 'app-registration/version-published',
CronJobDeletedWorkspace = 'cron-job/deleted-workspace',
JobWebhookCallCompleted = 'job/webhook-call-completed',
+ JobWebhookCallThrottled = 'job/webhook-call-throttled',
SignUpSuccess = 'sign-up/success',
CommonApiQueryRateLimited = 'common-api-query/rate-limited',
CommonApiApplicationQueryRateLimited = 'common-api-query/application-rate-limited',
diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/config-variables.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/config-variables.ts
index 6e77e7f5396..03e0be60fe3 100644
--- a/packages/twenty-server/src/engine/core-modules/twenty-config/config-variables.ts
+++ b/packages/twenty-server/src/engine/core-modules/twenty-config/config-variables.ts
@@ -1693,6 +1693,24 @@ export class ConfigVariables {
@CastToPositiveNumber()
EMAIL_SEND_WORKSPACE_RATE_LIMITING_LIMIT = 100;
+ @ConfigVariablesMetadata({
+ group: ConfigVariablesGroup.RATE_LIMITING,
+ description:
+ 'Time-to-live for the outbound webhook call rate limiting window, in milliseconds',
+ type: ConfigVariableType.NUMBER,
+ })
+ @CastToPositiveNumber()
+ WEBHOOK_CALL_RATE_LIMITING_TTL_IN_MS = 60_000;
+
+ @ConfigVariablesMetadata({
+ group: ConfigVariablesGroup.RATE_LIMITING,
+ description:
+ 'Maximum number of outbound webhook calls a single workspace may make in the rate limiting window. Calls above it are never enqueued and are recorded as a failed webhook response',
+ type: ConfigVariableType.NUMBER,
+ })
+ @CastToPositiveNumber()
+ WEBHOOK_CALL_RATE_LIMITING_LIMIT = 60;
+
@ConfigVariablesMetadata({
group: ConfigVariablesGroup.RATE_LIMITING,
description:
diff --git a/packages/twenty-server/src/engine/core-modules/usage-limit/constants/try-consume-token-buckets-script.constant.ts b/packages/twenty-server/src/engine/core-modules/usage-limit/constants/try-consume-token-buckets-script.constant.ts
index d0f6499853d..6920abc70be 100644
--- a/packages/twenty-server/src/engine/core-modules/usage-limit/constants/try-consume-token-buckets-script.constant.ts
+++ b/packages/twenty-server/src/engine/core-modules/usage-limit/constants/try-consume-token-buckets-script.constant.ts
@@ -1,10 +1,14 @@
import { type CacheScript } from 'src/engine/core-modules/cache-storage/types/cache-script.type';
+export const TOKEN_BUCKETS_ALLOW_PARTIAL_ARG = '1';
+export const TOKEN_BUCKETS_DENY_PARTIAL_ARG = '0';
+
export const TRY_CONSUME_TOKEN_BUCKETS_SCRIPT: CacheScript = {
name: 'usage-limit:try-consume-token-buckets',
source: `
-local cost = tonumber(ARGV[1])
+local requestedCost = tonumber(ARGV[1])
local buckets = cjson.decode(ARGV[2])
+local allowPartial = ARGV[3] == '1'
if #buckets ~= #KEYS then
return redis.error_reply(
@@ -17,6 +21,7 @@ local nowMs = tonumber(now[1]) * 1000 + math.floor(tonumber(now[2]) / 1000)
local available = {}
local exhaustedIndex = 0
local retryAfterMs = 0
+local admittedCost = requestedCost
for i = 1, #KEYS do
local bucket = buckets[i]
@@ -36,8 +41,12 @@ for i = 1, #KEYS do
if refilled > bucket.burst then refilled = bucket.burst end
available[i] = refilled
- if refilled < cost then
- local bucketRetryAfterMs = math.ceil((cost - refilled) * bucket.windowMs / bucket.refill)
+ local bucketAdmittedCost = math.floor(refilled)
+ if bucketAdmittedCost > requestedCost then bucketAdmittedCost = requestedCost end
+ if bucketAdmittedCost < admittedCost then admittedCost = bucketAdmittedCost end
+
+ if bucketAdmittedCost < requestedCost then
+ local bucketRetryAfterMs = math.ceil((requestedCost - refilled) * bucket.windowMs / bucket.refill)
-- Buckets arrive narrowest first, so the first exhausted one names the
-- scope, while the caller has to wait for the slowest of them to refill.
@@ -51,18 +60,22 @@ for i = 1, #KEYS do
end
end
-if exhaustedIndex > 0 then
+if not allowPartial and exhaustedIndex > 0 then
+ admittedCost = 0
+end
+
+if admittedCost <= 0 then
return { 0, exhaustedIndex, retryAfterMs }
end
for i = 1, #KEYS do
local bucket = buckets[i]
- redis.call('HSET', KEYS[i], 't', tostring(available[i] - cost), 'ts', tostring(nowMs))
+ redis.call('HSET', KEYS[i], 't', tostring(available[i] - admittedCost), 'ts', tostring(nowMs))
-- Never expire before an empty bucket would have refilled, or the key
-- vanishing hands back a full burst the elapsed time had not yet earned.
redis.call('PEXPIRE', KEYS[i], math.ceil(bucket.burst / bucket.refill * bucket.windowMs))
end
-return { 1, 0, 0 }
+return { admittedCost, exhaustedIndex, retryAfterMs }
`,
};
diff --git a/packages/twenty-server/src/engine/core-modules/usage-limit/constants/usage-limit-definitions.constant.ts b/packages/twenty-server/src/engine/core-modules/usage-limit/constants/usage-limit-definitions.constant.ts
index 01a3b15bee7..e78bfb4f08c 100644
--- a/packages/twenty-server/src/engine/core-modules/usage-limit/constants/usage-limit-definitions.constant.ts
+++ b/packages/twenty-server/src/engine/core-modules/usage-limit/constants/usage-limit-definitions.constant.ts
@@ -84,4 +84,19 @@ export const USAGE_LIMIT_DEFINITIONS: Record<
],
},
},
+ [UsageResourceType.WEBHOOK]: {
+ speed: {
+ allowedOperationTypes: [UsageOperationType.WEBHOOK_CALL],
+ allowedSpenderTypes: ['workspace'],
+ defaults: [
+ {
+ spenderType: 'workspace',
+ counterScope: 'perWorkspace',
+ limitValueConfigVariable: 'WEBHOOK_CALL_RATE_LIMITING_LIMIT',
+ windowMsConfigVariable: 'WEBHOOK_CALL_RATE_LIMITING_TTL_IN_MS',
+ isOverridable: true,
+ },
+ ],
+ },
+ },
};
diff --git a/packages/twenty-server/src/engine/core-modules/usage-limit/services/__tests__/usage-limit-speed.service.spec.ts b/packages/twenty-server/src/engine/core-modules/usage-limit/services/__tests__/usage-limit-speed.service.spec.ts
index 5c8ed119616..727b71a8982 100644
--- a/packages/twenty-server/src/engine/core-modules/usage-limit/services/__tests__/usage-limit-speed.service.spec.ts
+++ b/packages/twenty-server/src/engine/core-modules/usage-limit/services/__tests__/usage-limit-speed.service.spec.ts
@@ -72,6 +72,22 @@ describe('UsageLimitSpeedService', () => {
operationType: UsageOperationType.API_REQUEST,
});
+ const consumeCost = (cost: number) =>
+ service.tryConsume({
+ resourceType: UsageResourceType.API,
+ authContext: apiKeyContext,
+ operationType: UsageOperationType.API_REQUEST,
+ cost,
+ });
+
+ const consumeUpTo = (maxCost: number) =>
+ service.tryConsumeUpTo({
+ resourceType: UsageResourceType.API,
+ authContext: apiKeyContext,
+ operationType: UsageOperationType.API_REQUEST,
+ maxCost,
+ });
+
beforeEach(async () => {
jest.clearAllMocks();
entitlementProvider = new TestUsageLimitEntitlementProvider();
@@ -139,6 +155,25 @@ describe('UsageLimitSpeedService', () => {
await expect(consume()).rejects.toThrow(/Rate limit exceeded/);
});
+ it('admits a multi-cost request when the buckets cover all of it', async () => {
+ cacheStorage.runScript.mockResolvedValueOnce([5, 0, 0]);
+
+ await expect(consumeCost(5)).resolves.toEqual({
+ admitted: true,
+ admittedCount: 5,
+ });
+ });
+
+ it('admits part of a batch when the buckets cannot cover all of it', async () => {
+ cacheStorage.runScript.mockResolvedValueOnce([2, 1, 1500]);
+
+ await expect(consumeUpTo(5)).resolves.toMatchObject({
+ admitted: false,
+ admittedCount: 2,
+ retryAfterMs: 1500,
+ });
+ });
+
it('enforces a stored apiKey speed override only when the workspace is entitled', async () => {
workspaceCacheService.getOrRecompute.mockResolvedValue({
usageLimits: {
diff --git a/packages/twenty-server/src/engine/core-modules/usage-limit/services/usage-limit-speed.service.ts b/packages/twenty-server/src/engine/core-modules/usage-limit/services/usage-limit-speed.service.ts
index 023dde1e149..0d469844a6e 100644
--- a/packages/twenty-server/src/engine/core-modules/usage-limit/services/usage-limit-speed.service.ts
+++ b/packages/twenty-server/src/engine/core-modules/usage-limit/services/usage-limit-speed.service.ts
@@ -8,7 +8,11 @@ import { CacheStorageException } from 'src/engine/core-modules/cache-storage/exc
import { CacheStorageService } from 'src/engine/core-modules/cache-storage/services/cache-storage.service';
import { CacheStorageNamespace } from 'src/engine/core-modules/cache-storage/types/cache-storage-namespace.enum';
import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';
-import { TRY_CONSUME_TOKEN_BUCKETS_SCRIPT } from 'src/engine/core-modules/usage-limit/constants/try-consume-token-buckets-script.constant';
+import {
+ TOKEN_BUCKETS_ALLOW_PARTIAL_ARG,
+ TOKEN_BUCKETS_DENY_PARTIAL_ARG,
+ TRY_CONSUME_TOKEN_BUCKETS_SCRIPT,
+} from 'src/engine/core-modules/usage-limit/constants/try-consume-token-buckets-script.constant';
import {
UsageLimitException,
UsageLimitExceptionCode,
@@ -25,8 +29,6 @@ import { type UsageResourceType } from 'src/engine/core-modules/usage/enums/usag
import { WorkspaceCacheException } from 'src/engine/workspace-cache/exceptions/workspace-cache.exception';
import { WorkspaceCacheService } from 'src/engine/workspace-cache/services/workspace-cache.service';
-const ADMITTED: SpeedBucketOutcome = { admitted: true };
-
@Injectable()
export class UsageLimitSpeedService {
private readonly logger = new Logger(UsageLimitSpeedService.name);
@@ -50,7 +52,7 @@ export class UsageLimitSpeedService {
operationType: UsageOperationType;
cost?: number;
}): Promise<void> {
- const outcome = await this.consumeAdmittingOnFailure({
+ const outcome = await this.tryConsume({
resourceType,
authContext,
operationType,
@@ -84,17 +86,64 @@ export class UsageLimitSpeedService {
);
}
+ async tryConsume({
+ resourceType,
+ authContext,
+ operationType,
+ cost = 1,
+ }: {
+ resourceType: UsageResourceType;
+ authContext: WorkspaceAuthContext;
+ operationType: UsageOperationType;
+ cost?: number;
+ }): Promise<SpeedBucketOutcome> {
+ return await this.consumeAdmittingOnFailure({
+ resourceType,
+ authContext,
+ operationType,
+ cost,
+ allowPartial: false,
+ });
+ }
+
+ async tryConsumeUpTo({
+ resourceType,
+ authContext,
+ operationType,
+ maxCost,
+ }: {
+ resourceType: UsageResourceType;
+ authContext: WorkspaceAuthContext;
+ operationType: UsageOperationType;
+ maxCost: number;
+ }): Promise<SpeedBucketOutcome> {
+ return await this.consumeAdmittingOnFailure({
+ resourceType,
+ authContext,
+ operationType,
+ cost: maxCost,
+ allowPartial: true,
+ });
+ }
+
private async consumeAdmittingOnFailure({
resourceType,
authContext,
operationType,
cost,
+ allowPartial,
}: {
resourceType: UsageResourceType;
authContext: WorkspaceAuthContext;
operationType: UsageOperationType;
cost: number;
+ allowPartial: boolean;
}): Promise<SpeedBucketOutcome> {
+ const fullAdmission: SpeedBucketOutcome = {
+ admitted: true,
+ admittedCount: cost,
+ };
+
const buckets = await this.buildBuckets({
resourceType,
authContext,
@@ -102,11 +151,11 @@ export class UsageLimitSpeedService {
});
if (buckets.length === 0) {
- return ADMITTED;
+ return fullAdmission;
}
try {
- return await this.consumeTokens({ buckets, cost });
+ return await this.consumeTokens({ buckets, cost, allowPartial });
} catch (error) {
if (!(error instanceof CacheStorageException)) {
throw error;
@@ -114,16 +163,18 @@ export class UsageLimitSpeedService {
this.logger.error(`Usage limit enforcement degraded: ${error.message}`);
- return ADMITTED;
+ return fullAdmission;
}
}
private async consumeTokens({
buckets,
cost,
+ allowPartial,
}: {
buckets: SpeedBucketRequest[];
cost: number;
+ allowPartial: boolean;
}): Promise<SpeedBucketOutcome> {
const bucketConfigs = buckets.map((bucket) => ({
burst: bucket.burst,
@@ -131,28 +182,34 @@ export class UsageLimitSpeedService {
windowMs: bucket.windowMs,
}));
- const [admitted, failedIndex, retryAfterMs] =
+ const [admittedCount, exhaustedIndex, retryAfterMs] =
await this.cacheStorage.runScript<number[]>({
script: TRY_CONSUME_TOKEN_BUCKETS_SCRIPT,
keys: buckets.map((bucket) => bucket.key),
- args: [String(cost), JSON.stringify(bucketConfigs)],
+ args: [
+ String(cost),
+ JSON.stringify(bucketConfigs),
+ allowPartial
+ ? TOKEN_BUCKETS_ALLOW_PARTIAL_ARG
+ : TOKEN_BUCKETS_DENY_PARTIAL_ARG,
+ ],
});
- if (admitted === 1) {
- return ADMITTED;
+ if (admittedCount === cost) {
+ return { admitted: true, admittedCount };
}
- const exhausted = buckets[failedIndex - 1];
+ const exhausted = buckets[exhaustedIndex - 1];
if (!isDefined(exhausted)) {
this.logger.warn(
- `try-consume-token-buckets returned an out-of-range index ${failedIndex}`,
+ `try-consume-token-buckets returned an out-of-range index ${exhaustedIndex}`,
);
- return ADMITTED;
+ return { admitted: true, admittedCount: cost };
}
- return { admitted: false, exhausted, retryAfterMs };
+ return { admitted: false, admittedCount, exhausted, retryAfterMs };
}
private async buildBuckets({
diff --git a/packages/twenty-server/src/engine/core-modules/usage-limit/types/speed-bucket-outcome.type.ts b/packages/twenty-server/src/engine/core-modules/usage-limit/types/speed-bucket-outcome.type.ts
index 86f0e9c459a..da21a20d395 100644
--- a/packages/twenty-server/src/engine/core-modules/usage-limit/types/speed-bucket-outcome.type.ts
+++ b/packages/twenty-server/src/engine/core-modules/usage-limit/types/speed-bucket-outcome.type.ts
@@ -1,9 +1,10 @@
import { type SpeedBucketRequest } from 'src/engine/core-modules/usage-limit/types/speed-bucket-request.type';
export type SpeedBucketOutcome =
- | { admitted: true }
+ | { admitted: true; admittedCount: number }
| {
admitted: false;
+ admittedCount: number;
exhausted: SpeedBucketRequest;
retryAfterMs: number;
};
diff --git a/packages/twenty-server/src/engine/core-modules/usage/enums/usage-operation-type.enum.ts b/packages/twenty-server/src/engine/core-modules/usage/enums/usage-operation-type.enum.ts
index e2ae48fb319..951b86ee197 100644
--- a/packages/twenty-server/src/engine/core-modules/usage/enums/usage-operation-type.enum.ts
+++ b/packages/twenty-server/src/engine/core-modules/usage/enums/usage-operation-type.enum.ts
@@ -15,6 +15,7 @@ export enum UsageOperationType {
// out of twenty-shared's USAGE_OPERATION_TYPES: an app billing under it
// would spend the workspace's request budget.
API_REQUEST = 'API_REQUEST',
+ WEBHOOK_CALL = 'WEBHOOK_CALL',
// Platform-raised, once per billing period. Also out of
// USAGE_OPERATION_TYPES: an app declares the amount, it never charges this.
SUBSCRIPTION = 'SUBSCRIPTION',
diff --git a/packages/twenty-server/src/engine/core-modules/usage/enums/usage-resource-type.enum.ts b/packages/twenty-server/src/engine/core-modules/usage/enums/usage-resource-type.enum.ts
index 5212fa167e4..21c8727e39a 100644
--- a/packages/twenty-server/src/engine/core-modules/usage/enums/usage-resource-type.enum.ts
+++ b/packages/twenty-server/src/engine/core-modules/usage/enums/usage-resource-type.enum.ts
@@ -10,6 +10,7 @@ export enum UsageResourceType {
API = 'API',
LOGIC_FUNCTION = 'LOGIC_FUNCTION',
EMAIL = 'EMAIL',
+ WEBHOOK = 'WEBHOOK',
}
registerEnumType(UsageResourceType, {
diff --git a/packages/twenty-server/src/engine/metadata-modules/webhook/jobs/call-webhook-jobs-for-metadata.job.ts b/packages/twenty-server/src/engine/metadata-modules/webhook/jobs/call-webhook-jobs-for-metadata.job.ts
index cfbf4a8d2c0..46f8057570a 100644
--- a/packages/twenty-server/src/engine/metadata-modules/webhook/jobs/call-webhook-jobs-for-metadata.job.ts
+++ b/packages/twenty-server/src/engine/metadata-modules/webhook/jobs/call-webhook-jobs-for-metadata.job.ts
@@ -10,6 +10,7 @@ import { type MetadataEventBatch } from 'src/engine/subscriptions/metadata-event
import { type FlatWebhook } from 'src/engine/metadata-modules/flat-webhook/types/flat-webhook.type';
import { CallWebhookJob } from 'src/engine/metadata-modules/webhook/jobs/call-webhook.job';
import { type CallMetadataWebhookJobData } from 'src/engine/metadata-modules/webhook/types/webhook-job-data.type';
+import { WebhookRateLimitService } from 'src/engine/metadata-modules/webhook/jobs/webhook-rate-limit.service';
import { WorkspaceCacheService } from 'src/engine/workspace-cache/services/workspace-cache.service';
const WEBHOOK_JOBS_CHUNK_SIZE = 20;
@@ -20,6 +21,7 @@ export class CallWebhookJobsForMetadataJob {
@InjectMessageQueue(MessageQueue.webhookQueue)
private readonly messageQueueService: MessageQueueService,
private readonly workspaceCacheService: WorkspaceCacheService,
+ private readonly webhookRateLimitService: WebhookRateLimitService,
) {}
@Process(CallWebhookJobsForMetadataJob.name)
@@ -58,7 +60,16 @@ export class CallWebhookJobsForMetadataJob {
webhooks,
});
- const webhookEventsChunks = chunk(webhookEvents, WEBHOOK_JOBS_CHUNK_SIZE);
+ const admittedWebhookEvents =
+ await this.webhookRateLimitService.admitWebhookEventsWithinRateLimit({
+ workspaceId: metadataEventBatch.workspaceId,
+ webhookEvents,
+ });
+
+ const webhookEventsChunks = chunk(
+ admittedWebhookEvents,
+ WEBHOOK_JOBS_CHUNK_SIZE,
+ );
for (const webhookEventsChunk of webhookEventsChunks) {
await this.messageQueueService.add<CallMetadataWebhookJobData[]>(
diff --git a/packages/twenty-server/src/engine/metadata-modules/webhook/jobs/call-webhook-jobs.job.ts b/packages/twenty-server/src/engine/metadata-modules/webhook/jobs/call-webhook-jobs.job.ts
index dab93cc77de..999ed864d09 100644
--- a/packages/twenty-server/src/engine/metadata-modules/webhook/jobs/call-webhook-jobs.job.ts
+++ b/packages/twenty-server/src/engine/metadata-modules/webhook/jobs/call-webhook-jobs.job.ts
@@ -11,6 +11,7 @@ import { Processor } from 'src/engine/core-modules/message-queue/decorators/proc
import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
import { MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service';
import { CallWebhookJob } from 'src/engine/metadata-modules/webhook/jobs/call-webhook.job';
+import { WebhookRateLimitService } from 'src/engine/metadata-modules/webhook/jobs/webhook-rate-limit.service';
import { type CallWebhookJobData } from 'src/engine/metadata-modules/webhook/types/webhook-job-data.type';
import { computeWebhookOperationsToMatch } from 'src/engine/metadata-modules/webhook/utils/compute-webhook-operations-to-match.util';
import { transformEventBatchToWebhookEvents } from 'src/engine/metadata-modules/webhook/utils/transform-event-batch-to-webhook-events';
@@ -26,6 +27,7 @@ export class CallWebhookJobsJob {
@InjectMessageQueue(MessageQueue.webhookQueue)
private readonly messageQueueService: MessageQueueService,
private readonly workspaceCacheService: WorkspaceCacheService,
+ private readonly webhookRateLimitService: WebhookRateLimitService,
) {}
@Process(CallWebhookJobsJob.name)
@@ -62,7 +64,16 @@ export class CallWebhookJobsJob {
webhooks,
});
- const webhookEventsChunks = chunk(webhookEvents, WEBHOOK_JOBS_CHUNK_SIZE);
+ const admittedWebhookEvents =
+ await this.webhookRateLimitService.admitWebhookEventsWithinRateLimit({
+ workspaceId: workspaceEventBatch.workspaceId,
+ webhookEvents,
+ });
+
+ const webhookEventsChunks = chunk(
+ admittedWebhookEvents,
+ WEBHOOK_JOBS_CHUNK_SIZE,
+ );
for (const webhookEventsChunk of webhookEventsChunks) {
await this.messageQueueService.add<CallWebhookJobData[]>(
diff --git a/packages/twenty-server/src/engine/metadata-modules/webhook/jobs/webhook-job.module.ts b/packages/twenty-server/src/engine/metadata-modules/webhook/jobs/webhook-job.module.ts
index 4a148d12289..56a67685fde 100644
--- a/packages/twenty-server/src/engine/metadata-modules/webhook/jobs/webhook-job.module.ts
+++ b/packages/twenty-server/src/engine/metadata-modules/webhook/jobs/webhook-job.module.ts
@@ -1,26 +1,32 @@
import { Module } from '@nestjs/common';
import { EventLogEmitterModule } from 'src/engine/core-modules/event-logs/emit/event-log-emitter.module';
+import { FeatureFlagModule } from 'src/engine/core-modules/feature-flag/feature-flag.module';
import { MetricsModule } from 'src/engine/core-modules/metrics/metrics.module';
import { SecureHttpClientModule } from 'src/engine/core-modules/secure-http-client/secure-http-client.module';
+import { UsageLimitModule } from 'src/engine/core-modules/usage-limit/usage-limit.module';
import { FlatWebhookModule } from 'src/engine/metadata-modules/flat-webhook/flat-webhook.module';
import { CallWebhookJobsForMetadataJob } from 'src/engine/metadata-modules/webhook/jobs/call-webhook-jobs-for-metadata.job';
import { CallWebhookJobsJob } from 'src/engine/metadata-modules/webhook/jobs/call-webhook-jobs.job';
import { CallWebhookJob } from 'src/engine/metadata-modules/webhook/jobs/call-webhook.job';
+import { WebhookRateLimitService } from 'src/engine/metadata-modules/webhook/jobs/webhook-rate-limit.service';
import { WorkspaceCacheModule } from 'src/engine/workspace-cache/workspace-cache.module';
@Module({
imports: [
EventLogEmitterModule,
+ FeatureFlagModule,
FlatWebhookModule,
MetricsModule,
SecureHttpClientModule,
+ UsageLimitModule,
WorkspaceCacheModule,
],
providers: [
CallWebhookJobsJob,
CallWebhookJobsForMetadataJob,
CallWebhookJob,
+ WebhookRateLimitService,
],
})
export class WebhookJobModule {}
diff --git a/packages/twenty-server/src/engine/metadata-modules/webhook/jobs/webhook-rate-limit.service.ts b/packages/twenty-server/src/engine/metadata-modules/webhook/jobs/webhook-rate-limit.service.ts
new file mode 100644
index 00000000000..9819c77277f
--- /dev/null
+++ b/packages/twenty-server/src/engine/metadata-modules/webhook/jobs/webhook-rate-limit.service.ts
@@ -0,0 +1,84 @@
+import { Injectable, Logger } from '@nestjs/common';
+
+import { FeatureFlagKey } from 'twenty-shared/types';
+
+import { FeatureFlagService } from 'src/engine/core-modules/feature-flag/services/feature-flag.service';
+import { MetricsService } from 'src/engine/core-modules/metrics/metrics.service';
+import { MetricsKeys } from 'src/engine/core-modules/metrics/types/metrics-keys.type';
+import { UsageLimitSpeedService } from 'src/engine/core-modules/usage-limit/services/usage-limit-speed.service';
+import { type SpeedBucketRequest } from 'src/engine/core-modules/usage-limit/types/speed-bucket-request.type';
+import { UsageOperationType } from 'src/engine/core-modules/usage/enums/usage-operation-type.enum';
+import { UsageResourceType } from 'src/engine/core-modules/usage/enums/usage-resource-type.enum';
+import { type WebhookJobData } from 'src/engine/metadata-modules/webhook/types/webhook-job-data.type';
+import { buildSystemAuthContext } from 'src/engine/twenty-orm/utils/build-system-auth-context.util';
+
+@Injectable()
+export class WebhookRateLimitService {
+ private readonly logger = new Logger(WebhookRateLimitService.name);
+
+ constructor(
+ private readonly usageLimitSpeedService: UsageLimitSpeedService,
+ private readonly featureFlagService: FeatureFlagService,
+ private readonly metricsService: MetricsService,
+ ) {}
+
+ async admitWebhookEventsWithinRateLimit<
+ TWebhookEvent extends WebhookJobData,
+ >({
+ workspaceId,
+ webhookEvents,
+ }: {
+ workspaceId: string;
+ webhookEvents: TWebhookEvent[];
+ }): Promise<TWebhookEvent[]> {
+ if (webhookEvents.length === 0) {
+ return webhookEvents;
+ }
+
+ const isWebhookRateLimitEnabled =
+ await this.featureFlagService.isFeatureEnabled(
+ FeatureFlagKey.IS_WEBHOOK_RATE_LIMIT_ENABLED,
+ workspaceId,
+ );
+
+ if (!isWebhookRateLimitEnabled) {
+ return webhookEvents;
+ }
+
+ const outcome = await this.usageLimitSpeedService.tryConsumeUpTo({
+ resourceType: UsageResourceType.WEBHOOK,
+ operationType: UsageOperationType.WEBHOOK_CALL,
+ authContext: buildSystemAuthContext(workspaceId),
+ maxCost: webhookEvents.length,
+ });
+
+ if (!outcome.admitted) {
+ this.dropThrottledWebhookCalls({
+ workspaceId,
+ droppedCallCount: webhookEvents.length - outcome.admittedCount,
+ exhaustedBucket: outcome.exhausted,
+ });
+ }
+
+ return webhookEvents.slice(0, outcome.admittedCount);
+ }
+
+ private dropThrottledWebhookCalls({
+ workspaceId,
+ droppedCallCount,
+ exhaustedBucket,
+ }: {
+ workspaceId: string;
+ droppedCallCount: number;
+ exhaustedBucket: SpeedBucketRequest;
+ }): void {
+ this.logger.warn(
+ `Dropped ${droppedCallCount} webhook calls for workspace ${workspaceId}, which exceeded ${exhaustedBucket.refillPerWindow} webhook calls per ${exhaustedBucket.windowMs / 1000}s`,
+ );
+
+ this.metricsService.incrementCounterBy({
+ key: MetricsKeys.JobWebhookCallThrottled,
+ amount: droppedCallCount,
+ });
+ }
+}
diff --git a/packages/twenty-server/test/integration/usage-limit/webhook-rate-limit.integration-spec.ts b/packages/twenty-server/test/integration/usage-limit/webhook-rate-limit.integration-spec.ts
new file mode 100644
index 00000000000..a90dcd01b6b
--- /dev/null
+++ b/packages/twenty-server/test/integration/usage-limit/webhook-rate-limit.integration-spec.ts
@@ -0,0 +1,194 @@
+import { makeGraphqlAPIRequest } from 'test/integration/graphql/utils/make-graphql-api-request.util';
+import {
+ createWebhook,
+ createWebhookReceiver,
+ deleteWebhook,
+} from 'test/integration/metadata/suites/utils/webhook-test.util';
+import { makeAdminPanelAPIRequest } from 'test/integration/twenty-config/utils/make-admin-panel-api-request.util';
+import { expectEventually } from 'test/integration/utils/expect-eventually.util';
+import { getCoreRepository } from 'test/integration/utils/get-core-repository.util';
+
+import { gql } from 'graphql-tag';
+import { createClient } from 'redis';
+import { FeatureFlagKey } from 'twenty-shared/types';
+import { type Repository } from 'typeorm';
+import { v4 as uuidv4 } from 'uuid';
+
+import { FeatureFlagEntity } from 'src/engine/core-modules/feature-flag/feature-flag.entity';
+import { UsageLimitEntity } from 'src/engine/core-modules/usage-limit/usage-limit.entity';
+import { UsageOperationType } from 'src/engine/core-modules/usage/enums/usage-operation-type.enum';
+import { UsageResourceType } from 'src/engine/core-modules/usage/enums/usage-resource-type.enum';
+import { SEED_APPLE_WORKSPACE_ID } from 'src/engine/workspace-manager/dev-seeder/core/constants/seeder-workspaces.constant';
+
+const WEBHOOK_RECEIVER_PORT = 4318;
+const WINDOW_SECONDS = 60;
+const LIMIT_VALUE = 2;
+const PERSON_COUNT = 5;
+
+const CREATE_CONFIG_VARIABLE_MUTATION = gql`
+ mutation CreateDatabaseConfigVariable($key: String!, $value: JSON!) {
+ createDatabaseConfigVariable(key: $key, value: $value)
+ }
+`;
+
+const DELETE_CONFIG_VARIABLE_MUTATION = gql`
+ mutation DeleteDatabaseConfigVariable($key: String!) {
+ deleteDatabaseConfigVariable(key: $key)
+ }
+`;
+
+const CREATE_PERSON_MUTATION = gql`
+ mutation CreatePerson($data: PersonCreateInput!) {
+ createPerson(data: $data) {
+ id
+ }
+ }
+`;
+
+const DESTROY_PERSON_MUTATION = gql`
+ mutation DestroyPerson($id: ID!) {
+ destroyPerson(id: $id) {
+ id
+ }
+ }
+`;
+
+describe('Webhook rate limiting', () => {
+ let usageLimitRepository: Repository<UsageLimitEntity>;
+ let featureFlagRepository: Repository<FeatureFlagEntity>;
+ let redis: Awaited<ReturnType<typeof createClient>>;
+ let usageLimitId: string;
+ let webhookId: string;
+ let receiver: Awaited<ReturnType<typeof createWebhookReceiver>>;
+
+ const createdPersonIds: string[] = [];
+
+ const invalidateWorkspaceCaches = async () => {
+ const keys = [
+ ...(await redis.keys(`*featureFlagsMap:${SEED_APPLE_WORKSPACE_ID}*`)),
+ ...(await redis.keys(`*usageLimits:${SEED_APPLE_WORKSPACE_ID}*`)),
+ ];
+
+ if (keys.length > 0) {
+ await redis.del(keys);
+ }
+
+ await new Promise((resolve) => setTimeout(resolve, 300));
+ };
+
+ const createPerson = async (lastName: string) => {
+ const response = await makeGraphqlAPIRequest({
+ query: CREATE_PERSON_MUTATION,
+ variables: { data: { name: { firstName: 'WebhookThrottle', lastName } } },
+ });
+
+ expect(response.body.errors).toBeUndefined();
+ createdPersonIds.push(response.body.data.createPerson.id);
+ };
+
+ beforeAll(async () => {
+ jest.useRealTimers();
+
+ usageLimitRepository =
+ getCoreRepository<UsageLimitEntity>(UsageLimitEntity);
+ featureFlagRepository =
+ getCoreRepository<FeatureFlagEntity>(FeatureFlagEntity);
+ redis = await createClient({ url: process.env.REDIS_URL }).connect();
+
+ await makeAdminPanelAPIRequest({
+ query: CREATE_CONFIG_VARIABLE_MUTATION,
+ variables: { key: 'OUTBOUND_HTTP_SAFE_MODE_ENABLED', value: false },
+ });
+
+ await featureFlagRepository.delete({
+ key: FeatureFlagKey.IS_WEBHOOK_RATE_LIMIT_ENABLED,
+ workspaceId: SEED_APPLE_WORKSPACE_ID,
+ });
+ await featureFlagRepository.save({
+ key: FeatureFlagKey.IS_WEBHOOK_RATE_LIMIT_ENABLED,
+ value: true,
+ workspaceId: SEED_APPLE_WORKSPACE_ID,
+ });
+
+ const [usageLimit] = await usageLimitRepository.save([
+ {
+ workspaceId: SEED_APPLE_WORKSPACE_ID,
+ resourceType: UsageResourceType.WEBHOOK,
+ operationType: UsageOperationType.WEBHOOK_CALL,
+ spenderType: 'workspace',
+ spenderId: '',
+ limitKind: 'speed',
+ periodCount: WINDOW_SECONDS,
+ periodUnit: 'second',
+ meter: 'quantity',
+ limitValue: LIMIT_VALUE,
+ burstValue: LIMIT_VALUE,
+ },
+ ]);
+
+ usageLimitId = usageLimit.id;
+
+ await invalidateWorkspaceCaches();
+
+ receiver = await createWebhookReceiver(WEBHOOK_RECEIVER_PORT);
+
+ const createWebhookResponse = await createWebhook({
+ targetUrl: `http://127.0.0.1:${WEBHOOK_RECEIVER_PORT}/webhook`,
+ operations: ['person.created'],
+ description: 'Webhook rate limiting test webhook',
+ secret: 'test-secret',
+ });
+
+ expect(createWebhookResponse.body.errors).toBeUndefined();
+ webhookId = createWebhookResponse.body.data.createWebhook.id;
+ });
+
+ afterAll(async () => {
+ await receiver.close();
+ await deleteWebhook(webhookId).catch(() => {});
+
+ for (const personId of createdPersonIds) {
+ await makeGraphqlAPIRequest({
+ query: DESTROY_PERSON_MUTATION,
+ variables: { id: personId },
+ }).catch(() => {});
+ }
+
+ await usageLimitRepository.delete({ id: usageLimitId });
+ await featureFlagRepository.delete({
+ key: FeatureFlagKey.IS_WEBHOOK_RATE_LIMIT_ENABLED,
+ workspaceId: SEED_APPLE_WORKSPACE_ID,
+ });
+ await makeAdminPanelAPIRequest({
+ query: DELETE_CONFIG_VARIABLE_MUTATION,
+ variables: { key: 'OUTBOUND_HTTP_SAFE_MODE_ENABLED' },
+ }).catch(() => {});
+
+ await invalidateWorkspaceCaches();
+ await redis.quit();
+ jest.useFakeTimers();
+ });
+
+ it('delivers what the bucket admits and drops the rest of the burst', async () => {
+ const testId = uuidv4().slice(0, 8);
+
+ for (let personIndex = 0; personIndex < PERSON_COUNT; personIndex++) {
+ await createPerson(`Test-${testId}-${personIndex}`);
+ }
+
+ await expectEventually(
+ () => {
+ expect(receiver.receivedPayloads.length).toBe(LIMIT_VALUE);
+ },
+ { timeoutMs: 30_000, intervalMs: 100 },
+ );
+
+ await new Promise((resolve) => setTimeout(resolve, 2_000));
+
+ expect(receiver.receivedPayloads.length).toBe(LIMIT_VALUE);
+ expect(receiver.receivedPayloads[0]).toMatchObject({
+ targetUrl: `http://127.0.0.1:${WEBHOOK_RECEIVER_PORT}/webhook`,
+ eventName: 'person.created',
+ });
+ }, 60_000);
+});
diff --git a/packages/twenty-shared/src/types/FeatureFlagKey.ts b/packages/twenty-shared/src/types/FeatureFlagKey.ts
index 21150d74962..ba25811594e 100644
--- a/packages/twenty-shared/src/types/FeatureFlagKey.ts
+++ b/packages/twenty-shared/src/types/FeatureFlagKey.ts
@@ -13,4 +13,5 @@ export enum FeatureFlagKey {
IS_QUOTA_ENGINE_CREDIT_BOUND_ENABLED = 'IS_QUOTA_ENGINE_CREDIT_BOUND_ENABLED',
IS_RECORD_CREATION_FORM_ENABLED = 'IS_RECORD_CREATION_FORM_ENABLED',
IS_RECORD_SHARING_ENABLED = 'IS_RECORD_SHARING_ENABLED',
+ IS_WEBHOOK_RATE_LIMIT_ENABLED = 'IS_WEBHOOK_RATE_LIMIT_ENABLED',
}