The open alternative to Salesforce, designed for AI.
TypeScript crmcrm-systemcustomergood-first-issuegraphqlhacktoberfestjavascriptmarketingmonoreponestjsopen-sourcepostgresqlreactreactjssalestypescriptweb
feat(usage-limit): count quota admits on failure and the credits they let through (#25546)
Etienneec1578abe5bc
diff --git a/packages/twenty-server/src/engine/core-modules/metrics/types/metrics-keys.type.ts b/packages/twenty-server/src/engine/core-modules/metrics/types/metrics-keys.type.tsindex 97a6ed35809..2a3016c897f 100644--- a/packages/twenty-server/src/engine/core-modules/metrics/types/metrics-keys.type.ts+++ b/packages/twenty-server/src/engine/core-modules/metrics/types/metrics-keys.type.ts@@ -60,6 +60,8 @@ export enum MetricsKeys {CommonApiQueryRateLimited = 'common-api-query/rate-limited',CommonApiApplicationQueryRateLimited = 'common-api-query/application-rate-limited',JobEnqueueApplicationRateLimited = 'job/enqueue-application-rate-limited',+ UsageLimitQuotaAdmittedOnFailure = 'usage-limit-quota/admitted-on-failure',+ UsageLimitQuotaAdmittedOnFailureCreditsMicro = 'usage-limit-quota/admitted-on-failure-credits-micro',JobCompleted = 'job/completed',JobFailed = 'job/failed',JobStalled = 'job/stalled',diff --git a/packages/twenty-server/src/engine/core-modules/usage-limit/services/__tests__/usage-limit-quota.service.spec.ts b/packages/twenty-server/src/engine/core-modules/usage-limit/services/__tests__/usage-limit-quota.service.spec.tsindex 97382ad95a0..bf6b52cab5f 100644--- a/packages/twenty-server/src/engine/core-modules/usage-limit/services/__tests__/usage-limit-quota.service.spec.ts+++ b/packages/twenty-server/src/engine/core-modules/usage-limit/services/__tests__/usage-limit-quota.service.spec.ts@@ -8,6 +8,8 @@ import {CacheLockExceptionCode,} from 'src/engine/core-modules/cache-lock/exceptions/cache-lock.exception';import { CacheStorageNamespace } from 'src/engine/core-modules/cache-storage/types/cache-storage-namespace.enum';+import { MetricsService } from 'src/engine/core-modules/metrics/metrics.service';+import { MetricsKeys } from 'src/engine/core-modules/metrics/types/metrics-keys.type';import {UsageLimitException,UsageLimitExceptionCode,@@ -113,6 +115,10 @@ describe('UsageLimitQuotaService', () => {findCurrentPeriodsByUnit: jest.fn(),};+ const metricsService = {+ incrementCounterBy: jest.fn(),+ };+let periodByUnit: Partial<Record<PeriodUnit, UsagePeriod>>;const setLimits = (limits: FlatUsageLimit[]) => {@@ -184,6 +190,7 @@ describe('UsageLimitQuotaService', () => {{ provide: CacheLockService, useValue: cacheLockService },{ provide: ClickHouseService, useValue: clickHouseService },{ provide: UsagePeriodService, useValue: usagePeriodService },+ { provide: MetricsService, useValue: metricsService },{provide: DiscoveryService,useValue: {@@ -370,6 +377,37 @@ describe('UsageLimitQuotaService', () => {cacheStorage.mget.mockRejectedValue(new Error('Socket closed'));await expect(assertQuotaNotExhausted()).resolves.toBeUndefined();+ expect(metricsService.incrementCounterBy).toHaveBeenCalledTimes(1);+ expect(metricsService.incrementCounterBy).toHaveBeenCalledWith({+ key: MetricsKeys.UsageLimitQuotaAdmittedOnFailure,+ amount: 1,+ attributes: { operation: 'assert', resourceType: UsageResourceType.AI },+ });+ });++ it('counts the credits admitted when a consume fails', async () => {+ setLimits([buildLimit({})]);+ cacheStorage.mget.mockRejectedValue(new Error('Socket closed'));++ await expect(consumeQuota(50, 7)).resolves.toEqual({ exhausted: [] });+ expect(metricsService.incrementCounterBy).toHaveBeenCalledWith({+ key: MetricsKeys.UsageLimitQuotaAdmittedOnFailure,+ amount: 1,+ attributes: { operation: 'consume', resourceType: UsageResourceType.AI },+ });+ expect(metricsService.incrementCounterBy).toHaveBeenCalledWith({+ key: MetricsKeys.UsageLimitQuotaAdmittedOnFailureCreditsMicro,+ amount: 50,+ attributes: { operation: 'consume', resourceType: UsageResourceType.AI },+ });+ });++ it('does not count an admit on a successful consume', async () => {+ setLimits([buildLimit({})]);+ cacheStorage.runScript.mockResolvedValue([1, 500]);++ await expect(consumeQuota(50)).resolves.toEqual({ exhausted: [] });+ expect(metricsService.incrementCounterBy).not.toHaveBeenCalled();});it('admits when the warm query fails instead of granting a fresh budget', async () => {@@ -644,5 +682,15 @@ describe('UsageLimitQuotaService', () => {).resolves.toBeNull();expect(cacheStorage.mget).not.toHaveBeenCalled();});++ it('does not count a failed read as an admit', async () => {+ setAllowance(2_000_000);+ cacheStorage.mget.mockRejectedValue(new Error('Socket closed'));++ await expect(+ service.getAllowanceRemainingMicro('workspace-1'),+ ).resolves.toBeNull();+ expect(metricsService.incrementCounterBy).not.toHaveBeenCalled();+ });});});diff --git a/packages/twenty-server/src/engine/core-modules/usage-limit/services/usage-limit-quota.service.ts b/packages/twenty-server/src/engine/core-modules/usage-limit/services/usage-limit-quota.service.tsindex 00a5c2f35a8..ca8f5607186 100644--- a/packages/twenty-server/src/engine/core-modules/usage-limit/services/usage-limit-quota.service.ts+++ b/packages/twenty-server/src/engine/core-modules/usage-limit/services/usage-limit-quota.service.ts@@ -14,6 +14,8 @@ import {import { InjectCacheStorage } from 'src/engine/core-modules/cache-storage/decorators/cache-storage.decorator';import { CacheStorageService } from 'src/engine/core-modules/cache-storage/services/cache-storage.service';import { CacheStorageNamespace } from 'src/engine/core-modules/cache-storage/types/cache-storage-namespace.enum';+import { MetricsService } from 'src/engine/core-modules/metrics/metrics.service';+import { MetricsKeys } from 'src/engine/core-modules/metrics/types/metrics-keys.type';import { CONSUME_QUOTA_COUNTERS_SCRIPT } from 'src/engine/core-modules/usage-limit/constants/consume-quota-counters-script.constant';import {UsageLimitException,@@ -56,6 +58,12 @@ type AllowanceSumRow = {total: string | number | null;};+type QuotaEnforcement = {+ operation: 'assert' | 'consume';+ resourceType: UsageResourceType;+ cost?: QuotaCost;+};+type QuotaConsumeArgs = {workspaceId: string;resourceType: UsageResourceType;@@ -78,6 +86,7 @@ export class UsageLimitQuotaService implements OnModuleInit {private readonly usagePeriodService: UsagePeriodService,private readonly discoveryService: DiscoveryService,private readonly usageLimitEntitlementService: UsageLimitEntitlementService,+ private readonly metricsService: MetricsService,) {}onModuleInit() {@@ -283,6 +292,7 @@ export class UsageLimitQuotaService implements OnModuleInit {return this.admitOnFailure({error,workspaceId: args.workspaceId,+ enforcement: { operation: 'assert', resourceType: args.resourceType },admitted: [],});}@@ -321,6 +331,11 @@ export class UsageLimitQuotaService implements OnModuleInit {return this.admitOnFailure({error,workspaceId: args.workspaceId,+ enforcement: {+ operation: 'consume',+ resourceType: args.resourceType,+ cost,+ },admitted: [],});}@@ -378,10 +393,12 @@ export class UsageLimitQuotaService implements OnModuleInit {private admitOnFailure<TAdmitted>({error,workspaceId,+ enforcement,admitted,}: {error: unknown;workspaceId: string;+ enforcement?: QuotaEnforcement;admitted: TAdmitted;}): TAdmitted {if (@@ -395,9 +412,35 @@ export class UsageLimitQuotaService implements OnModuleInit {`Usage quota enforcement degraded for workspace ${workspaceId}: ${error instanceof Error ? error.message : 'unknown error'}`,);+ if (isDefined(enforcement)) {+ this.countAdmitOnFailure(enforcement);+ }+return admitted;}+ private countAdmitOnFailure({+ operation,+ resourceType,+ cost,+ }: QuotaEnforcement): void {+ const attributes = { operation, resourceType };++ this.metricsService.incrementCounterBy({+ key: MetricsKeys.UsageLimitQuotaAdmittedOnFailure,+ amount: 1,+ attributes,+ });++ if (isDefined(cost) && cost.creditsUsedMicro > 0) {+ this.metricsService.incrementCounterBy({+ key: MetricsKeys.UsageLimitQuotaAdmittedOnFailureCreditsMicro,+ amount: cost.creditsUsedMicro,+ attributes,+ });+ }+ }+private async buildCounters(args: QuotaConsumeArgs): Promise<QuotaCounter[]> {const [limitCounters, allowanceCounter] = await Promise.all([this.buildLimitCounters(args),diff --git a/packages/twenty-server/src/engine/core-modules/usage-limit/usage-limit.module.ts b/packages/twenty-server/src/engine/core-modules/usage-limit/usage-limit.module.tsindex d0decb24947..40ea9bb1e5d 100644--- a/packages/twenty-server/src/engine/core-modules/usage-limit/usage-limit.module.ts+++ b/packages/twenty-server/src/engine/core-modules/usage-limit/usage-limit.module.ts@@ -5,6 +5,7 @@ import { TypeOrmModule } from '@nestjs/typeorm';import { ClickHouseModule } from 'src/database/clickhouse/clickhouse.module';import { ApiKeyEntity } from 'src/engine/core-modules/api-key/api-key.entity';import { CacheLockModule } from 'src/engine/core-modules/cache-lock/cache-lock.module';+import { MetricsModule } from 'src/engine/core-modules/metrics/metrics.module';import { UsageLimitQuotaService } from 'src/engine/core-modules/usage-limit/services/usage-limit-quota.service';import { UsagePeriodService } from 'src/engine/core-modules/usage-limit/services/usage-period.service';import { UsageModule } from 'src/engine/core-modules/usage/usage.module';@@ -37,6 +38,7 @@ import { WorkspaceCacheModule } from 'src/engine/workspace-cache/workspace-cacheDiscoveryModule,CacheLockModule,ClickHouseModule,+ MetricsModule,UsageModule,],providers: [