The open alternative to Salesforce, designed for AI.
TypeScript crmcrm-systemcustomergood-first-issuegraphqlhacktoberfestjavascriptmarketingmonoreponestjsopen-sourcepostgresqlreactreactjssalestypescriptweb
feat(usage-limit): cap webhook calls per workspace at enqueue time (#25673)
Etienne9b3e4d074506
diff --git a/packages/twenty-client-sdk/src/metadata/generated/schema.graphql b/packages/twenty-client-sdk/src/metadata/generated/schema.graphqlindex 9199edf39dc..f40c399b8e0 100644--- a/packages/twenty-client-sdk/src/metadata/generated/schema.graphql+++ b/packages/twenty-client-sdk/src/metadata/generated/schema.graphql@@ -1804,6 +1804,7 @@ enum UsageResourceType {APILOGIC_FUNCTION+ WEBHOOK}enum UsageOperationType {@@ -1816,6 +1817,7 @@ enum UsageOperationType {CALL_RECORDINGEMAIL_SENDAPI_REQUEST+ WEBHOOK_CALLSUBSCRIPTION}@@ -1923,6 +1925,7 @@ enum FeatureFlagKey {IS_QUOTA_ENGINE_CREDIT_BOUND_ENABLEDIS_RECORD_CREATION_FORM_ENABLEDIS_RECORD_SHARING_ENABLED+ IS_WEBHOOK_RATE_LIMIT_ENABLED}type WorkspaceUrls {diff --git a/packages/twenty-client-sdk/src/metadata/generated/schema.ts b/packages/twenty-client-sdk/src/metadata/generated/schema.tsindex ba0210f8961..09bcc02adae 100644--- a/packages/twenty-client-sdk/src/metadata/generated/schema.ts+++ b/packages/twenty-client-sdk/src/metadata/generated/schema.ts@@ -1445,9 +1445,9 @@ export interface UsageLimit {__typename: 'UsageLimit'}-export type UsageResourceType = 'AI' | 'WORKFLOW' | 'APP' | 'STORAGE' | 'API' | 'LOGIC_FUNCTION' | 'EMAIL'+export type UsageResourceType = 'AI' | 'WORKFLOW' | 'APP' | 'STORAGE' | 'API' | 'LOGIC_FUNCTION' | 'EMAIL' | 'WEBHOOK'-export type UsageOperationType = 'ALL' | 'AI_CHAT_TOKEN' | 'AI_WORKFLOW_TOKEN' | 'WORKFLOW_EXECUTION' | 'CODE_EXECUTION' | 'WEB_SEARCH' | 'CALL_RECORDING' | 'EMAIL_SEND' | 'API_REQUEST' | 'SUBSCRIPTION'+export type UsageOperationType = 'ALL' | 'AI_CHAT_TOKEN' | 'AI_WORKFLOW_TOKEN' | 'WORKFLOW_EXECUTION' | 'CODE_EXECUTION' | 'WEB_SEARCH' | 'CALL_RECORDING' | 'EMAIL_SEND' | 'API_REQUEST' | 'WEBHOOK_CALL' | 'SUBSCRIPTION'export interface ApprovedAccessDomain {id: Scalars['UUID']@@ -1538,7 +1538,7 @@ export interface FeatureFlag {__typename: 'FeatureFlag'}-export type FeatureFlagKey = 'IS_APP_CLAIMING_ENABLED' | 'IS_UNIQUE_INDEXES_ENABLED' | 'IS_CONFIGURABLE_SEARCH_FIELDS_ENABLED' | 'IS_JSON_FILTER_ENABLED' | 'IS_EMAIL_GROUP_ENABLED' | 'IS_JUNCTION_RELATIONS_ENABLED' | 'IS_REST_METADATA_API_NEW_FORMAT_DIRECT' | 'IS_LOGIC_FUNCTION_PREBUILT_MODE_ENABLED' | 'IS_WORKFLOW_CORE_INDEX_PAGE_ENABLED' | 'IS_API_RATE_LIMIT_V2_ENABLED' | 'IS_MESSAGE_CALENDAR_TARGET_READ_ENABLED' | 'IS_QUOTA_ENGINE_CREDIT_BOUND_ENABLED' | 'IS_RECORD_CREATION_FORM_ENABLED' | 'IS_RECORD_SHARING_ENABLED'+export type FeatureFlagKey = 'IS_APP_CLAIMING_ENABLED' | 'IS_UNIQUE_INDEXES_ENABLED' | 'IS_CONFIGURABLE_SEARCH_FIELDS_ENABLED' | 'IS_JSON_FILTER_ENABLED' | 'IS_EMAIL_GROUP_ENABLED' | 'IS_JUNCTION_RELATIONS_ENABLED' | 'IS_REST_METADATA_API_NEW_FORMAT_DIRECT' | 'IS_LOGIC_FUNCTION_PREBUILT_MODE_ENABLED' | 'IS_WORKFLOW_CORE_INDEX_PAGE_ENABLED' | 'IS_API_RATE_LIMIT_V2_ENABLED' | 'IS_MESSAGE_CALENDAR_TARGET_READ_ENABLED' | 'IS_QUOTA_ENGINE_CREDIT_BOUND_ENABLED' | 'IS_RECORD_CREATION_FORM_ENABLED' | 'IS_RECORD_SHARING_ENABLED' | 'IS_WEBHOOK_RATE_LIMIT_ENABLED'export interface WorkspaceUrls {customUrl?: Scalars['String']@@ -10381,7 +10381,8 @@ export const enumUsageResourceType = {STORAGE: 'STORAGE' as const,API: 'API' as const,LOGIC_FUNCTION: 'LOGIC_FUNCTION' as const,- EMAIL: 'EMAIL' as const+ EMAIL: 'EMAIL' as const,+ WEBHOOK: 'WEBHOOK' as const}export const enumUsageOperationType = {@@ -10394,6 +10395,7 @@ export const enumUsageOperationType = {CALL_RECORDING: 'CALL_RECORDING' as const,EMAIL_SEND: 'EMAIL_SEND' as const,API_REQUEST: 'API_REQUEST' as const,+ WEBHOOK_CALL: 'WEBHOOK_CALL' as const,SUBSCRIPTION: 'SUBSCRIPTION' as const}@@ -10417,7 +10419,8 @@ export const enumFeatureFlagKey = {IS_MESSAGE_CALENDAR_TARGET_READ_ENABLED: 'IS_MESSAGE_CALENDAR_TARGET_READ_ENABLED' as const,IS_QUOTA_ENGINE_CREDIT_BOUND_ENABLED: 'IS_QUOTA_ENGINE_CREDIT_BOUND_ENABLED' as const,IS_RECORD_CREATION_FORM_ENABLED: 'IS_RECORD_CREATION_FORM_ENABLED' as const,- IS_RECORD_SHARING_ENABLED: 'IS_RECORD_SHARING_ENABLED' as const+ IS_RECORD_SHARING_ENABLED: 'IS_RECORD_SHARING_ENABLED' as const,+ IS_WEBHOOK_RATE_LIMIT_ENABLED: 'IS_WEBHOOK_RATE_LIMIT_ENABLED' as const}export const enumIdentityProviderType = {diff --git a/packages/twenty-front/src/generated-admin/graphql.ts b/packages/twenty-front/src/generated-admin/graphql.tsindex e39e9b130ec..699996d7552 100644--- a/packages/twenty-front/src/generated-admin/graphql.ts+++ b/packages/twenty-front/src/generated-admin/graphql.ts@@ -414,6 +414,7 @@ export enum FeatureFlagKey {IS_RECORD_SHARING_ENABLED = 'IS_RECORD_SHARING_ENABLED',IS_REST_METADATA_API_NEW_FORMAT_DIRECT = 'IS_REST_METADATA_API_NEW_FORMAT_DIRECT',IS_UNIQUE_INDEXES_ENABLED = 'IS_UNIQUE_INDEXES_ENABLED',+ IS_WEBHOOK_RATE_LIMIT_ENABLED = 'IS_WEBHOOK_RATE_LIMIT_ENABLED',IS_WORKFLOW_CORE_INDEX_PAGE_ENABLED = 'IS_WORKFLOW_CORE_INDEX_PAGE_ENABLED'}diff --git a/packages/twenty-front/src/generated-metadata/graphql.ts b/packages/twenty-front/src/generated-metadata/graphql.tsindex 4ce39c4c7db..a3eaead18bb 100644--- a/packages/twenty-front/src/generated-metadata/graphql.ts+++ b/packages/twenty-front/src/generated-metadata/graphql.ts@@ -1966,6 +1966,7 @@ export enum FeatureFlagKey {IS_RECORD_SHARING_ENABLED = 'IS_RECORD_SHARING_ENABLED',IS_REST_METADATA_API_NEW_FORMAT_DIRECT = 'IS_REST_METADATA_API_NEW_FORMAT_DIRECT',IS_UNIQUE_INDEXES_ENABLED = 'IS_UNIQUE_INDEXES_ENABLED',+ IS_WEBHOOK_RATE_LIMIT_ENABLED = 'IS_WEBHOOK_RATE_LIMIT_ENABLED',IS_WORKFLOW_CORE_INDEX_PAGE_ENABLED = 'IS_WORKFLOW_CORE_INDEX_PAGE_ENABLED'}@@ -6654,6 +6655,7 @@ export enum UsageOperationType {CODE_EXECUTION = 'CODE_EXECUTION',EMAIL_SEND = 'EMAIL_SEND',SUBSCRIPTION = 'SUBSCRIPTION',+ WEBHOOK_CALL = 'WEBHOOK_CALL',WEB_SEARCH = 'WEB_SEARCH',WORKFLOW_EXECUTION = 'WORKFLOW_EXECUTION'}@@ -6665,6 +6667,7 @@ export enum UsageResourceType {EMAIL = 'EMAIL',LOGIC_FUNCTION = 'LOGIC_FUNCTION',STORAGE = 'STORAGE',+ WEBHOOK = 'WEBHOOK',WORKFLOW = 'WORKFLOW'}diff --git a/packages/twenty-front/src/modules/settings/usage/constants/UsageOperationTypeLabels.ts b/packages/twenty-front/src/modules/settings/usage/constants/UsageOperationTypeLabels.tsindex be2a0d5a352..66dea4ea704 100644--- a/packages/twenty-front/src/modules/settings/usage/constants/UsageOperationTypeLabels.ts+++ b/packages/twenty-front/src/modules/settings/usage/constants/UsageOperationTypeLabels.ts@@ -18,5 +18,6 @@ export const USAGE_OPERATION_TYPE_LABELS: Record<[UsageOperationType.CALL_RECORDING]: msg`Call Recording`,[UsageOperationType.EMAIL_SEND]: msg`Email Send`,[UsageOperationType.API_REQUEST]: msg`API Request`,+ [UsageOperationType.WEBHOOK_CALL]: msg`Webhook Call`,[UsageOperationType.SUBSCRIPTION]: msg`Subscription`,};diff --git a/packages/twenty-server/src/engine/core-modules/billing/app-billing/app-billing.service.ts b/packages/twenty-server/src/engine/core-modules/billing/app-billing/app-billing.service.tsindex 26b88d0e477..67d1f490740 100644--- a/packages/twenty-server/src/engine/core-modules/billing/app-billing/app-billing.service.ts+++ b/packages/twenty-server/src/engine/core-modules/billing/app-billing/app-billing.service.ts@@ -30,6 +30,7 @@ const USAGE_UNIT_BY_OPERATION_TYPE: Record<[UsageOperationType.CALL_RECORDING]: UsageUnit.MINUTE,[UsageOperationType.EMAIL_SEND]: UsageUnit.INVOCATION,[UsageOperationType.API_REQUEST]: UsageUnit.REQUEST,+ [UsageOperationType.WEBHOOK_CALL]: UsageUnit.REQUEST,// Platform-raised only; ApplicationRecurringChargeService sets its own unit.[UsageOperationType.SUBSCRIPTION]: UsageUnit.CREDIT,};diff --git a/packages/twenty-server/src/engine/core-modules/metrics/types/metrics-keys.type.ts b/packages/twenty-server/src/engine/core-modules/metrics/types/metrics-keys.type.tsindex 2a3016c897f..517d406e849 100644--- a/packages/twenty-server/src/engine/core-modules/metrics/types/metrics-keys.type.ts+++ b/packages/twenty-server/src/engine/core-modules/metrics/types/metrics-keys.type.ts@@ -56,6 +56,7 @@ export enum MetricsKeys {AppRegistrationVersionPublished = 'app-registration/version-published',CronJobDeletedWorkspace = 'cron-job/deleted-workspace',JobWebhookCallCompleted = 'job/webhook-call-completed',+ JobWebhookCallThrottled = 'job/webhook-call-throttled',SignUpSuccess = 'sign-up/success',CommonApiQueryRateLimited = 'common-api-query/rate-limited',CommonApiApplicationQueryRateLimited = 'common-api-query/application-rate-limited',diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/config-variables.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/config-variables.tsindex 6e77e7f5396..03e0be60fe3 100644--- a/packages/twenty-server/src/engine/core-modules/twenty-config/config-variables.ts+++ b/packages/twenty-server/src/engine/core-modules/twenty-config/config-variables.ts@@ -1693,6 +1693,24 @@ export class ConfigVariables {@CastToPositiveNumber()EMAIL_SEND_WORKSPACE_RATE_LIMITING_LIMIT = 100;+ @ConfigVariablesMetadata({+ group: ConfigVariablesGroup.RATE_LIMITING,+ description:+ 'Time-to-live for the outbound webhook call rate limiting window, in milliseconds',+ type: ConfigVariableType.NUMBER,+ })+ @CastToPositiveNumber()+ WEBHOOK_CALL_RATE_LIMITING_TTL_IN_MS = 60_000;++ @ConfigVariablesMetadata({+ group: ConfigVariablesGroup.RATE_LIMITING,+ description:+ 'Maximum number of outbound webhook calls a single workspace may make in the rate limiting window. Calls above it are never enqueued and are recorded as a failed webhook response',+ type: ConfigVariableType.NUMBER,+ })+ @CastToPositiveNumber()+ WEBHOOK_CALL_RATE_LIMITING_LIMIT = 60;+@ConfigVariablesMetadata({group: ConfigVariablesGroup.RATE_LIMITING,description:diff --git a/packages/twenty-server/src/engine/core-modules/usage-limit/constants/try-consume-token-buckets-script.constant.ts b/packages/twenty-server/src/engine/core-modules/usage-limit/constants/try-consume-token-buckets-script.constant.tsindex d0f6499853d..6920abc70be 100644--- a/packages/twenty-server/src/engine/core-modules/usage-limit/constants/try-consume-token-buckets-script.constant.ts+++ b/packages/twenty-server/src/engine/core-modules/usage-limit/constants/try-consume-token-buckets-script.constant.ts@@ -1,10 +1,14 @@import { type CacheScript } from 'src/engine/core-modules/cache-storage/types/cache-script.type';+export const TOKEN_BUCKETS_ALLOW_PARTIAL_ARG = '1';+export const TOKEN_BUCKETS_DENY_PARTIAL_ARG = '0';+export const TRY_CONSUME_TOKEN_BUCKETS_SCRIPT: CacheScript = {name: 'usage-limit:try-consume-token-buckets',source: `-local cost = tonumber(ARGV[1])+local requestedCost = tonumber(ARGV[1])local buckets = cjson.decode(ARGV[2])+local allowPartial = ARGV[3] == '1'if #buckets ~= #KEYS thenreturn redis.error_reply(@@ -17,6 +21,7 @@ local nowMs = tonumber(now[1]) * 1000 + math.floor(tonumber(now[2]) / 1000)local available = {}local exhaustedIndex = 0local retryAfterMs = 0+local admittedCost = requestedCostfor i = 1, #KEYS dolocal bucket = buckets[i]@@ -36,8 +41,12 @@ for i = 1, #KEYS doif refilled > bucket.burst then refilled = bucket.burst endavailable[i] = refilled- if refilled < cost then- local bucketRetryAfterMs = math.ceil((cost - refilled) * bucket.windowMs / bucket.refill)+ local bucketAdmittedCost = math.floor(refilled)+ if bucketAdmittedCost > requestedCost then bucketAdmittedCost = requestedCost end+ if bucketAdmittedCost < admittedCost then admittedCost = bucketAdmittedCost end++ if bucketAdmittedCost < requestedCost then+ local bucketRetryAfterMs = math.ceil((requestedCost - refilled) * bucket.windowMs / bucket.refill)-- Buckets arrive narrowest first, so the first exhausted one names the-- scope, while the caller has to wait for the slowest of them to refill.@@ -51,18 +60,22 @@ for i = 1, #KEYS doendend-if exhaustedIndex > 0 then+if not allowPartial and exhaustedIndex > 0 then+ admittedCost = 0+end++if admittedCost <= 0 thenreturn { 0, exhaustedIndex, retryAfterMs }endfor i = 1, #KEYS dolocal bucket = buckets[i]- redis.call('HSET', KEYS[i], 't', tostring(available[i] - cost), 'ts', tostring(nowMs))+ redis.call('HSET', KEYS[i], 't', tostring(available[i] - admittedCost), 'ts', tostring(nowMs))-- Never expire before an empty bucket would have refilled, or the key-- vanishing hands back a full burst the elapsed time had not yet earned.redis.call('PEXPIRE', KEYS[i], math.ceil(bucket.burst / bucket.refill * bucket.windowMs))end-return { 1, 0, 0 }+return { admittedCost, exhaustedIndex, retryAfterMs }`,};diff --git a/packages/twenty-server/src/engine/core-modules/usage-limit/constants/usage-limit-definitions.constant.ts b/packages/twenty-server/src/engine/core-modules/usage-limit/constants/usage-limit-definitions.constant.tsindex 01a3b15bee7..e78bfb4f08c 100644--- a/packages/twenty-server/src/engine/core-modules/usage-limit/constants/usage-limit-definitions.constant.ts+++ b/packages/twenty-server/src/engine/core-modules/usage-limit/constants/usage-limit-definitions.constant.ts@@ -84,4 +84,19 @@ export const USAGE_LIMIT_DEFINITIONS: Record<],},},+ [UsageResourceType.WEBHOOK]: {+ speed: {+ allowedOperationTypes: [UsageOperationType.WEBHOOK_CALL],+ allowedSpenderTypes: ['workspace'],+ defaults: [+ {+ spenderType: 'workspace',+ counterScope: 'perWorkspace',+ limitValueConfigVariable: 'WEBHOOK_CALL_RATE_LIMITING_LIMIT',+ windowMsConfigVariable: 'WEBHOOK_CALL_RATE_LIMITING_TTL_IN_MS',+ isOverridable: true,+ },+ ],+ },+ },};diff --git a/packages/twenty-server/src/engine/core-modules/usage-limit/services/__tests__/usage-limit-speed.service.spec.ts b/packages/twenty-server/src/engine/core-modules/usage-limit/services/__tests__/usage-limit-speed.service.spec.tsindex 5c8ed119616..727b71a8982 100644--- a/packages/twenty-server/src/engine/core-modules/usage-limit/services/__tests__/usage-limit-speed.service.spec.ts+++ b/packages/twenty-server/src/engine/core-modules/usage-limit/services/__tests__/usage-limit-speed.service.spec.ts@@ -72,6 +72,22 @@ describe('UsageLimitSpeedService', () => {operationType: UsageOperationType.API_REQUEST,});+ const consumeCost = (cost: number) =>+ service.tryConsume({+ resourceType: UsageResourceType.API,+ authContext: apiKeyContext,+ operationType: UsageOperationType.API_REQUEST,+ cost,+ });++ const consumeUpTo = (maxCost: number) =>+ service.tryConsumeUpTo({+ resourceType: UsageResourceType.API,+ authContext: apiKeyContext,+ operationType: UsageOperationType.API_REQUEST,+ maxCost,+ });+beforeEach(async () => {jest.clearAllMocks();entitlementProvider = new TestUsageLimitEntitlementProvider();@@ -139,6 +155,25 @@ describe('UsageLimitSpeedService', () => {await expect(consume()).rejects.toThrow(/Rate limit exceeded/);});+ it('admits a multi-cost request when the buckets cover all of it', async () => {+ cacheStorage.runScript.mockResolvedValueOnce([5, 0, 0]);++ await expect(consumeCost(5)).resolves.toEqual({+ admitted: true,+ admittedCount: 5,+ });+ });++ it('admits part of a batch when the buckets cannot cover all of it', async () => {+ cacheStorage.runScript.mockResolvedValueOnce([2, 1, 1500]);++ await expect(consumeUpTo(5)).resolves.toMatchObject({+ admitted: false,+ admittedCount: 2,+ retryAfterMs: 1500,+ });+ });+it('enforces a stored apiKey speed override only when the workspace is entitled', async () => {workspaceCacheService.getOrRecompute.mockResolvedValue({usageLimits: {diff --git a/packages/twenty-server/src/engine/core-modules/usage-limit/services/usage-limit-speed.service.ts b/packages/twenty-server/src/engine/core-modules/usage-limit/services/usage-limit-speed.service.tsindex 023dde1e149..0d469844a6e 100644--- a/packages/twenty-server/src/engine/core-modules/usage-limit/services/usage-limit-speed.service.ts+++ b/packages/twenty-server/src/engine/core-modules/usage-limit/services/usage-limit-speed.service.ts@@ -8,7 +8,11 @@ import { CacheStorageException } from 'src/engine/core-modules/cache-storage/excimport { CacheStorageService } from 'src/engine/core-modules/cache-storage/services/cache-storage.service';import { CacheStorageNamespace } from 'src/engine/core-modules/cache-storage/types/cache-storage-namespace.enum';import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';-import { TRY_CONSUME_TOKEN_BUCKETS_SCRIPT } from 'src/engine/core-modules/usage-limit/constants/try-consume-token-buckets-script.constant';+import {+ TOKEN_BUCKETS_ALLOW_PARTIAL_ARG,+ TOKEN_BUCKETS_DENY_PARTIAL_ARG,+ TRY_CONSUME_TOKEN_BUCKETS_SCRIPT,+} from 'src/engine/core-modules/usage-limit/constants/try-consume-token-buckets-script.constant';import {UsageLimitException,UsageLimitExceptionCode,@@ -25,8 +29,6 @@ import { type UsageResourceType } from 'src/engine/core-modules/usage/enums/usagimport { WorkspaceCacheException } from 'src/engine/workspace-cache/exceptions/workspace-cache.exception';import { WorkspaceCacheService } from 'src/engine/workspace-cache/services/workspace-cache.service';-const ADMITTED: SpeedBucketOutcome = { admitted: true };-@Injectable()export class UsageLimitSpeedService {private readonly logger = new Logger(UsageLimitSpeedService.name);@@ -50,7 +52,7 @@ export class UsageLimitSpeedService {operationType: UsageOperationType;cost?: number;}): Promise<void> {- const outcome = await this.consumeAdmittingOnFailure({+ const outcome = await this.tryConsume({resourceType,authContext,operationType,@@ -84,17 +86,64 @@ export class UsageLimitSpeedService {);}+ async tryConsume({+ resourceType,+ authContext,+ operationType,+ cost = 1,+ }: {+ resourceType: UsageResourceType;+ authContext: WorkspaceAuthContext;+ operationType: UsageOperationType;+ cost?: number;+ }): Promise<SpeedBucketOutcome> {+ return await this.consumeAdmittingOnFailure({+ resourceType,+ authContext,+ operationType,+ cost,+ allowPartial: false,+ });+ }++ async tryConsumeUpTo({+ resourceType,+ authContext,+ operationType,+ maxCost,+ }: {+ resourceType: UsageResourceType;+ authContext: WorkspaceAuthContext;+ operationType: UsageOperationType;+ maxCost: number;+ }): Promise<SpeedBucketOutcome> {+ return await this.consumeAdmittingOnFailure({+ resourceType,+ authContext,+ operationType,+ cost: maxCost,+ allowPartial: true,+ });+ }+private async consumeAdmittingOnFailure({resourceType,authContext,operationType,cost,+ allowPartial,}: {resourceType: UsageResourceType;authContext: WorkspaceAuthContext;operationType: UsageOperationType;cost: number;+ allowPartial: boolean;}): Promise<SpeedBucketOutcome> {+ const fullAdmission: SpeedBucketOutcome = {+ admitted: true,+ admittedCount: cost,+ };+const buckets = await this.buildBuckets({resourceType,authContext,@@ -102,11 +151,11 @@ export class UsageLimitSpeedService {});if (buckets.length === 0) {- return ADMITTED;+ return fullAdmission;}try {- return await this.consumeTokens({ buckets, cost });+ return await this.consumeTokens({ buckets, cost, allowPartial });} catch (error) {if (!(error instanceof CacheStorageException)) {throw error;@@ -114,16 +163,18 @@ export class UsageLimitSpeedService {this.logger.error(`Usage limit enforcement degraded: ${error.message}`);- return ADMITTED;+ return fullAdmission;}}private async consumeTokens({buckets,cost,+ allowPartial,}: {buckets: SpeedBucketRequest[];cost: number;+ allowPartial: boolean;}): Promise<SpeedBucketOutcome> {const bucketConfigs = buckets.map((bucket) => ({burst: bucket.burst,@@ -131,28 +182,34 @@ export class UsageLimitSpeedService {windowMs: bucket.windowMs,}));- const [admitted, failedIndex, retryAfterMs] =+ const [admittedCount, exhaustedIndex, retryAfterMs] =await this.cacheStorage.runScript<number[]>({script: TRY_CONSUME_TOKEN_BUCKETS_SCRIPT,keys: buckets.map((bucket) => bucket.key),- args: [String(cost), JSON.stringify(bucketConfigs)],+ args: [+ String(cost),+ JSON.stringify(bucketConfigs),+ allowPartial+ ? TOKEN_BUCKETS_ALLOW_PARTIAL_ARG+ : TOKEN_BUCKETS_DENY_PARTIAL_ARG,+ ],});- if (admitted === 1) {- return ADMITTED;+ if (admittedCount === cost) {+ return { admitted: true, admittedCount };}- const exhausted = buckets[failedIndex - 1];+ const exhausted = buckets[exhaustedIndex - 1];if (!isDefined(exhausted)) {this.logger.warn(- `try-consume-token-buckets returned an out-of-range index ${failedIndex}`,+ `try-consume-token-buckets returned an out-of-range index ${exhaustedIndex}`,);- return ADMITTED;+ return { admitted: true, admittedCount: cost };}- return { admitted: false, exhausted, retryAfterMs };+ return { admitted: false, admittedCount, exhausted, retryAfterMs };}private async buildBuckets({diff --git a/packages/twenty-server/src/engine/core-modules/usage-limit/types/speed-bucket-outcome.type.ts b/packages/twenty-server/src/engine/core-modules/usage-limit/types/speed-bucket-outcome.type.tsindex 86f0e9c459a..da21a20d395 100644--- a/packages/twenty-server/src/engine/core-modules/usage-limit/types/speed-bucket-outcome.type.ts+++ b/packages/twenty-server/src/engine/core-modules/usage-limit/types/speed-bucket-outcome.type.ts@@ -1,9 +1,10 @@import { type SpeedBucketRequest } from 'src/engine/core-modules/usage-limit/types/speed-bucket-request.type';export type SpeedBucketOutcome =- | { admitted: true }+ | { admitted: true; admittedCount: number }| {admitted: false;+ admittedCount: number;exhausted: SpeedBucketRequest;retryAfterMs: number;};diff --git a/packages/twenty-server/src/engine/core-modules/usage/enums/usage-operation-type.enum.ts b/packages/twenty-server/src/engine/core-modules/usage/enums/usage-operation-type.enum.tsindex e2ae48fb319..951b86ee197 100644--- a/packages/twenty-server/src/engine/core-modules/usage/enums/usage-operation-type.enum.ts+++ b/packages/twenty-server/src/engine/core-modules/usage/enums/usage-operation-type.enum.ts@@ -15,6 +15,7 @@ export enum UsageOperationType {// out of twenty-shared's USAGE_OPERATION_TYPES: an app billing under it// would spend the workspace's request budget.API_REQUEST = 'API_REQUEST',+ WEBHOOK_CALL = 'WEBHOOK_CALL',// Platform-raised, once per billing period. Also out of// USAGE_OPERATION_TYPES: an app declares the amount, it never charges this.SUBSCRIPTION = 'SUBSCRIPTION',diff --git a/packages/twenty-server/src/engine/core-modules/usage/enums/usage-resource-type.enum.ts b/packages/twenty-server/src/engine/core-modules/usage/enums/usage-resource-type.enum.tsindex 5212fa167e4..21c8727e39a 100644--- a/packages/twenty-server/src/engine/core-modules/usage/enums/usage-resource-type.enum.ts+++ b/packages/twenty-server/src/engine/core-modules/usage/enums/usage-resource-type.enum.ts@@ -10,6 +10,7 @@ export enum UsageResourceType {API = 'API',LOGIC_FUNCTION = 'LOGIC_FUNCTION',EMAIL = 'EMAIL',+ WEBHOOK = 'WEBHOOK',}registerEnumType(UsageResourceType, {diff --git a/packages/twenty-server/src/engine/metadata-modules/webhook/jobs/call-webhook-jobs-for-metadata.job.ts b/packages/twenty-server/src/engine/metadata-modules/webhook/jobs/call-webhook-jobs-for-metadata.job.tsindex cfbf4a8d2c0..46f8057570a 100644--- a/packages/twenty-server/src/engine/metadata-modules/webhook/jobs/call-webhook-jobs-for-metadata.job.ts+++ b/packages/twenty-server/src/engine/metadata-modules/webhook/jobs/call-webhook-jobs-for-metadata.job.ts@@ -10,6 +10,7 @@ import { type MetadataEventBatch } from 'src/engine/subscriptions/metadata-eventimport { type FlatWebhook } from 'src/engine/metadata-modules/flat-webhook/types/flat-webhook.type';import { CallWebhookJob } from 'src/engine/metadata-modules/webhook/jobs/call-webhook.job';import { type CallMetadataWebhookJobData } from 'src/engine/metadata-modules/webhook/types/webhook-job-data.type';+import { WebhookRateLimitService } from 'src/engine/metadata-modules/webhook/jobs/webhook-rate-limit.service';import { WorkspaceCacheService } from 'src/engine/workspace-cache/services/workspace-cache.service';const WEBHOOK_JOBS_CHUNK_SIZE = 20;@@ -20,6 +21,7 @@ export class CallWebhookJobsForMetadataJob {@InjectMessageQueue(MessageQueue.webhookQueue)private readonly messageQueueService: MessageQueueService,private readonly workspaceCacheService: WorkspaceCacheService,+ private readonly webhookRateLimitService: WebhookRateLimitService,) {}@Process(CallWebhookJobsForMetadataJob.name)@@ -58,7 +60,16 @@ export class CallWebhookJobsForMetadataJob {webhooks,});- const webhookEventsChunks = chunk(webhookEvents, WEBHOOK_JOBS_CHUNK_SIZE);+ const admittedWebhookEvents =+ await this.webhookRateLimitService.admitWebhookEventsWithinRateLimit({+ workspaceId: metadataEventBatch.workspaceId,+ webhookEvents,+ });++ const webhookEventsChunks = chunk(+ admittedWebhookEvents,+ WEBHOOK_JOBS_CHUNK_SIZE,+ );for (const webhookEventsChunk of webhookEventsChunks) {await this.messageQueueService.add<CallMetadataWebhookJobData[]>(diff --git a/packages/twenty-server/src/engine/metadata-modules/webhook/jobs/call-webhook-jobs.job.ts b/packages/twenty-server/src/engine/metadata-modules/webhook/jobs/call-webhook-jobs.job.tsindex dab93cc77de..999ed864d09 100644--- a/packages/twenty-server/src/engine/metadata-modules/webhook/jobs/call-webhook-jobs.job.ts+++ b/packages/twenty-server/src/engine/metadata-modules/webhook/jobs/call-webhook-jobs.job.ts@@ -11,6 +11,7 @@ import { Processor } from 'src/engine/core-modules/message-queue/decorators/procimport { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';import { MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service';import { CallWebhookJob } from 'src/engine/metadata-modules/webhook/jobs/call-webhook.job';+import { WebhookRateLimitService } from 'src/engine/metadata-modules/webhook/jobs/webhook-rate-limit.service';import { type CallWebhookJobData } from 'src/engine/metadata-modules/webhook/types/webhook-job-data.type';import { computeWebhookOperationsToMatch } from 'src/engine/metadata-modules/webhook/utils/compute-webhook-operations-to-match.util';import { transformEventBatchToWebhookEvents } from 'src/engine/metadata-modules/webhook/utils/transform-event-batch-to-webhook-events';@@ -26,6 +27,7 @@ export class CallWebhookJobsJob {@InjectMessageQueue(MessageQueue.webhookQueue)private readonly messageQueueService: MessageQueueService,private readonly workspaceCacheService: WorkspaceCacheService,+ private readonly webhookRateLimitService: WebhookRateLimitService,) {}@Process(CallWebhookJobsJob.name)@@ -62,7 +64,16 @@ export class CallWebhookJobsJob {webhooks,});- const webhookEventsChunks = chunk(webhookEvents, WEBHOOK_JOBS_CHUNK_SIZE);+ const admittedWebhookEvents =+ await this.webhookRateLimitService.admitWebhookEventsWithinRateLimit({+ workspaceId: workspaceEventBatch.workspaceId,+ webhookEvents,+ });++ const webhookEventsChunks = chunk(+ admittedWebhookEvents,+ WEBHOOK_JOBS_CHUNK_SIZE,+ );for (const webhookEventsChunk of webhookEventsChunks) {await this.messageQueueService.add<CallWebhookJobData[]>(diff --git a/packages/twenty-server/src/engine/metadata-modules/webhook/jobs/webhook-job.module.ts b/packages/twenty-server/src/engine/metadata-modules/webhook/jobs/webhook-job.module.tsindex 4a148d12289..56a67685fde 100644--- a/packages/twenty-server/src/engine/metadata-modules/webhook/jobs/webhook-job.module.ts+++ b/packages/twenty-server/src/engine/metadata-modules/webhook/jobs/webhook-job.module.ts@@ -1,26 +1,32 @@import { Module } from '@nestjs/common';import { EventLogEmitterModule } from 'src/engine/core-modules/event-logs/emit/event-log-emitter.module';+import { FeatureFlagModule } from 'src/engine/core-modules/feature-flag/feature-flag.module';import { MetricsModule } from 'src/engine/core-modules/metrics/metrics.module';import { SecureHttpClientModule } from 'src/engine/core-modules/secure-http-client/secure-http-client.module';+import { UsageLimitModule } from 'src/engine/core-modules/usage-limit/usage-limit.module';import { FlatWebhookModule } from 'src/engine/metadata-modules/flat-webhook/flat-webhook.module';import { CallWebhookJobsForMetadataJob } from 'src/engine/metadata-modules/webhook/jobs/call-webhook-jobs-for-metadata.job';import { CallWebhookJobsJob } from 'src/engine/metadata-modules/webhook/jobs/call-webhook-jobs.job';import { CallWebhookJob } from 'src/engine/metadata-modules/webhook/jobs/call-webhook.job';+import { WebhookRateLimitService } from 'src/engine/metadata-modules/webhook/jobs/webhook-rate-limit.service';import { WorkspaceCacheModule } from 'src/engine/workspace-cache/workspace-cache.module';@Module({imports: [EventLogEmitterModule,+ FeatureFlagModule,FlatWebhookModule,MetricsModule,SecureHttpClientModule,+ UsageLimitModule,WorkspaceCacheModule,],providers: [CallWebhookJobsJob,CallWebhookJobsForMetadataJob,CallWebhookJob,+ WebhookRateLimitService,],})export class WebhookJobModule {}diff --git a/packages/twenty-server/src/engine/metadata-modules/webhook/jobs/webhook-rate-limit.service.ts b/packages/twenty-server/src/engine/metadata-modules/webhook/jobs/webhook-rate-limit.service.tsnew file mode 100644index 00000000000..9819c77277f--- /dev/null+++ b/packages/twenty-server/src/engine/metadata-modules/webhook/jobs/webhook-rate-limit.service.ts@@ -0,0 +1,84 @@+import { Injectable, Logger } from '@nestjs/common';++import { FeatureFlagKey } from 'twenty-shared/types';++import { FeatureFlagService } from 'src/engine/core-modules/feature-flag/services/feature-flag.service';+import { MetricsService } from 'src/engine/core-modules/metrics/metrics.service';+import { MetricsKeys } from 'src/engine/core-modules/metrics/types/metrics-keys.type';+import { UsageLimitSpeedService } from 'src/engine/core-modules/usage-limit/services/usage-limit-speed.service';+import { type SpeedBucketRequest } from 'src/engine/core-modules/usage-limit/types/speed-bucket-request.type';+import { UsageOperationType } from 'src/engine/core-modules/usage/enums/usage-operation-type.enum';+import { UsageResourceType } from 'src/engine/core-modules/usage/enums/usage-resource-type.enum';+import { type WebhookJobData } from 'src/engine/metadata-modules/webhook/types/webhook-job-data.type';+import { buildSystemAuthContext } from 'src/engine/twenty-orm/utils/build-system-auth-context.util';++@Injectable()+export class WebhookRateLimitService {+ private readonly logger = new Logger(WebhookRateLimitService.name);++ constructor(+ private readonly usageLimitSpeedService: UsageLimitSpeedService,+ private readonly featureFlagService: FeatureFlagService,+ private readonly metricsService: MetricsService,+ ) {}++ async admitWebhookEventsWithinRateLimit<+ TWebhookEvent extends WebhookJobData,+ >({+ workspaceId,+ webhookEvents,+ }: {+ workspaceId: string;+ webhookEvents: TWebhookEvent[];+ }): Promise<TWebhookEvent[]> {+ if (webhookEvents.length === 0) {+ return webhookEvents;+ }++ const isWebhookRateLimitEnabled =+ await this.featureFlagService.isFeatureEnabled(+ FeatureFlagKey.IS_WEBHOOK_RATE_LIMIT_ENABLED,+ workspaceId,+ );++ if (!isWebhookRateLimitEnabled) {+ return webhookEvents;+ }++ const outcome = await this.usageLimitSpeedService.tryConsumeUpTo({+ resourceType: UsageResourceType.WEBHOOK,+ operationType: UsageOperationType.WEBHOOK_CALL,+ authContext: buildSystemAuthContext(workspaceId),+ maxCost: webhookEvents.length,+ });++ if (!outcome.admitted) {+ this.dropThrottledWebhookCalls({+ workspaceId,+ droppedCallCount: webhookEvents.length - outcome.admittedCount,+ exhaustedBucket: outcome.exhausted,+ });+ }++ return webhookEvents.slice(0, outcome.admittedCount);+ }++ private dropThrottledWebhookCalls({+ workspaceId,+ droppedCallCount,+ exhaustedBucket,+ }: {+ workspaceId: string;+ droppedCallCount: number;+ exhaustedBucket: SpeedBucketRequest;+ }): void {+ this.logger.warn(+ `Dropped ${droppedCallCount} webhook calls for workspace ${workspaceId}, which exceeded ${exhaustedBucket.refillPerWindow} webhook calls per ${exhaustedBucket.windowMs / 1000}s`,+ );++ this.metricsService.incrementCounterBy({+ key: MetricsKeys.JobWebhookCallThrottled,+ amount: droppedCallCount,+ });+ }+}diff --git a/packages/twenty-server/test/integration/usage-limit/webhook-rate-limit.integration-spec.ts b/packages/twenty-server/test/integration/usage-limit/webhook-rate-limit.integration-spec.tsnew file mode 100644index 00000000000..a90dcd01b6b--- /dev/null+++ b/packages/twenty-server/test/integration/usage-limit/webhook-rate-limit.integration-spec.ts@@ -0,0 +1,194 @@+import { makeGraphqlAPIRequest } from 'test/integration/graphql/utils/make-graphql-api-request.util';+import {+ createWebhook,+ createWebhookReceiver,+ deleteWebhook,+} from 'test/integration/metadata/suites/utils/webhook-test.util';+import { makeAdminPanelAPIRequest } from 'test/integration/twenty-config/utils/make-admin-panel-api-request.util';+import { expectEventually } from 'test/integration/utils/expect-eventually.util';+import { getCoreRepository } from 'test/integration/utils/get-core-repository.util';++import { gql } from 'graphql-tag';+import { createClient } from 'redis';+import { FeatureFlagKey } from 'twenty-shared/types';+import { type Repository } from 'typeorm';+import { v4 as uuidv4 } from 'uuid';++import { FeatureFlagEntity } from 'src/engine/core-modules/feature-flag/feature-flag.entity';+import { UsageLimitEntity } from 'src/engine/core-modules/usage-limit/usage-limit.entity';+import { UsageOperationType } from 'src/engine/core-modules/usage/enums/usage-operation-type.enum';+import { UsageResourceType } from 'src/engine/core-modules/usage/enums/usage-resource-type.enum';+import { SEED_APPLE_WORKSPACE_ID } from 'src/engine/workspace-manager/dev-seeder/core/constants/seeder-workspaces.constant';++const WEBHOOK_RECEIVER_PORT = 4318;+const WINDOW_SECONDS = 60;+const LIMIT_VALUE = 2;+const PERSON_COUNT = 5;++const CREATE_CONFIG_VARIABLE_MUTATION = gql`+ mutation CreateDatabaseConfigVariable($key: String!, $value: JSON!) {+ createDatabaseConfigVariable(key: $key, value: $value)+ }+`;++const DELETE_CONFIG_VARIABLE_MUTATION = gql`+ mutation DeleteDatabaseConfigVariable($key: String!) {+ deleteDatabaseConfigVariable(key: $key)+ }+`;++const CREATE_PERSON_MUTATION = gql`+ mutation CreatePerson($data: PersonCreateInput!) {+ createPerson(data: $data) {+ id+ }+ }+`;++const DESTROY_PERSON_MUTATION = gql`+ mutation DestroyPerson($id: ID!) {+ destroyPerson(id: $id) {+ id+ }+ }+`;++describe('Webhook rate limiting', () => {+ let usageLimitRepository: Repository<UsageLimitEntity>;+ let featureFlagRepository: Repository<FeatureFlagEntity>;+ let redis: Awaited<ReturnType<typeof createClient>>;+ let usageLimitId: string;+ let webhookId: string;+ let receiver: Awaited<ReturnType<typeof createWebhookReceiver>>;++ const createdPersonIds: string[] = [];++ const invalidateWorkspaceCaches = async () => {+ const keys = [+ ...(await redis.keys(`*featureFlagsMap:${SEED_APPLE_WORKSPACE_ID}*`)),+ ...(await redis.keys(`*usageLimits:${SEED_APPLE_WORKSPACE_ID}*`)),+ ];++ if (keys.length > 0) {+ await redis.del(keys);+ }++ await new Promise((resolve) => setTimeout(resolve, 300));+ };++ const createPerson = async (lastName: string) => {+ const response = await makeGraphqlAPIRequest({+ query: CREATE_PERSON_MUTATION,+ variables: { data: { name: { firstName: 'WebhookThrottle', lastName } } },+ });++ expect(response.body.errors).toBeUndefined();+ createdPersonIds.push(response.body.data.createPerson.id);+ };++ beforeAll(async () => {+ jest.useRealTimers();++ usageLimitRepository =+ getCoreRepository<UsageLimitEntity>(UsageLimitEntity);+ featureFlagRepository =+ getCoreRepository<FeatureFlagEntity>(FeatureFlagEntity);+ redis = await createClient({ url: process.env.REDIS_URL }).connect();++ await makeAdminPanelAPIRequest({+ query: CREATE_CONFIG_VARIABLE_MUTATION,+ variables: { key: 'OUTBOUND_HTTP_SAFE_MODE_ENABLED', value: false },+ });++ await featureFlagRepository.delete({+ key: FeatureFlagKey.IS_WEBHOOK_RATE_LIMIT_ENABLED,+ workspaceId: SEED_APPLE_WORKSPACE_ID,+ });+ await featureFlagRepository.save({+ key: FeatureFlagKey.IS_WEBHOOK_RATE_LIMIT_ENABLED,+ value: true,+ workspaceId: SEED_APPLE_WORKSPACE_ID,+ });++ const [usageLimit] = await usageLimitRepository.save([+ {+ workspaceId: SEED_APPLE_WORKSPACE_ID,+ resourceType: UsageResourceType.WEBHOOK,+ operationType: UsageOperationType.WEBHOOK_CALL,+ spenderType: 'workspace',+ spenderId: '',+ limitKind: 'speed',+ periodCount: WINDOW_SECONDS,+ periodUnit: 'second',+ meter: 'quantity',+ limitValue: LIMIT_VALUE,+ burstValue: LIMIT_VALUE,+ },+ ]);++ usageLimitId = usageLimit.id;++ await invalidateWorkspaceCaches();++ receiver = await createWebhookReceiver(WEBHOOK_RECEIVER_PORT);++ const createWebhookResponse = await createWebhook({+ targetUrl: `http://127.0.0.1:${WEBHOOK_RECEIVER_PORT}/webhook`,+ operations: ['person.created'],+ description: 'Webhook rate limiting test webhook',+ secret: 'test-secret',+ });++ expect(createWebhookResponse.body.errors).toBeUndefined();+ webhookId = createWebhookResponse.body.data.createWebhook.id;+ });++ afterAll(async () => {+ await receiver.close();+ await deleteWebhook(webhookId).catch(() => {});++ for (const personId of createdPersonIds) {+ await makeGraphqlAPIRequest({+ query: DESTROY_PERSON_MUTATION,+ variables: { id: personId },+ }).catch(() => {});+ }++ await usageLimitRepository.delete({ id: usageLimitId });+ await featureFlagRepository.delete({+ key: FeatureFlagKey.IS_WEBHOOK_RATE_LIMIT_ENABLED,+ workspaceId: SEED_APPLE_WORKSPACE_ID,+ });+ await makeAdminPanelAPIRequest({+ query: DELETE_CONFIG_VARIABLE_MUTATION,+ variables: { key: 'OUTBOUND_HTTP_SAFE_MODE_ENABLED' },+ }).catch(() => {});++ await invalidateWorkspaceCaches();+ await redis.quit();+ jest.useFakeTimers();+ });++ it('delivers what the bucket admits and drops the rest of the burst', async () => {+ const testId = uuidv4().slice(0, 8);++ for (let personIndex = 0; personIndex < PERSON_COUNT; personIndex++) {+ await createPerson(`Test-${testId}-${personIndex}`);+ }++ await expectEventually(+ () => {+ expect(receiver.receivedPayloads.length).toBe(LIMIT_VALUE);+ },+ { timeoutMs: 30_000, intervalMs: 100 },+ );++ await new Promise((resolve) => setTimeout(resolve, 2_000));++ expect(receiver.receivedPayloads.length).toBe(LIMIT_VALUE);+ expect(receiver.receivedPayloads[0]).toMatchObject({+ targetUrl: `http://127.0.0.1:${WEBHOOK_RECEIVER_PORT}/webhook`,+ eventName: 'person.created',+ });+ }, 60_000);+});diff --git a/packages/twenty-shared/src/types/FeatureFlagKey.ts b/packages/twenty-shared/src/types/FeatureFlagKey.tsindex 21150d74962..ba25811594e 100644--- a/packages/twenty-shared/src/types/FeatureFlagKey.ts+++ b/packages/twenty-shared/src/types/FeatureFlagKey.ts@@ -13,4 +13,5 @@ export enum FeatureFlagKey {IS_QUOTA_ENGINE_CREDIT_BOUND_ENABLED = 'IS_QUOTA_ENGINE_CREDIT_BOUND_ENABLED',IS_RECORD_CREATION_FORM_ENABLED = 'IS_RECORD_CREATION_FORM_ENABLED',IS_RECORD_SHARING_ENABLED = 'IS_RECORD_SHARING_ENABLED',+ IS_WEBHOOK_RATE_LIMIT_ENABLED = 'IS_WEBHOOK_RATE_LIMIT_ENABLED',}