Skip to content

The open alternative to Salesforce, designed for AI.

feat(usage-limit): count quota admits on failure and the credits they let through (#25546)

Etienneec1578abe5bc
diff --git a/packages/twenty-server/src/engine/core-modules/metrics/types/metrics-keys.type.ts b/packages/twenty-server/src/engine/core-modules/metrics/types/metrics-keys.type.ts
index 97a6ed35809..2a3016c897f 100644
--- a/packages/twenty-server/src/engine/core-modules/metrics/types/metrics-keys.type.ts
+++ b/packages/twenty-server/src/engine/core-modules/metrics/types/metrics-keys.type.ts
@@ -60,6 +60,8 @@ export enum MetricsKeys {
CommonApiQueryRateLimited = 'common-api-query/rate-limited',
CommonApiApplicationQueryRateLimited = 'common-api-query/application-rate-limited',
JobEnqueueApplicationRateLimited = 'job/enqueue-application-rate-limited',
+ UsageLimitQuotaAdmittedOnFailure = 'usage-limit-quota/admitted-on-failure',
+ UsageLimitQuotaAdmittedOnFailureCreditsMicro = 'usage-limit-quota/admitted-on-failure-credits-micro',
JobCompleted = 'job/completed',
JobFailed = 'job/failed',
JobStalled = 'job/stalled',
diff --git a/packages/twenty-server/src/engine/core-modules/usage-limit/services/__tests__/usage-limit-quota.service.spec.ts b/packages/twenty-server/src/engine/core-modules/usage-limit/services/__tests__/usage-limit-quota.service.spec.ts
index 97382ad95a0..bf6b52cab5f 100644
--- a/packages/twenty-server/src/engine/core-modules/usage-limit/services/__tests__/usage-limit-quota.service.spec.ts
+++ b/packages/twenty-server/src/engine/core-modules/usage-limit/services/__tests__/usage-limit-quota.service.spec.ts
@@ -8,6 +8,8 @@ import {
CacheLockExceptionCode,
} from 'src/engine/core-modules/cache-lock/exceptions/cache-lock.exception';
import { CacheStorageNamespace } from 'src/engine/core-modules/cache-storage/types/cache-storage-namespace.enum';
+import { MetricsService } from 'src/engine/core-modules/metrics/metrics.service';
+import { MetricsKeys } from 'src/engine/core-modules/metrics/types/metrics-keys.type';
import {
UsageLimitException,
UsageLimitExceptionCode,
@@ -113,6 +115,10 @@ describe('UsageLimitQuotaService', () => {
findCurrentPeriodsByUnit: jest.fn(),
};
+ const metricsService = {
+ incrementCounterBy: jest.fn(),
+ };
+
let periodByUnit: Partial<Record<PeriodUnit, UsagePeriod>>;
const setLimits = (limits: FlatUsageLimit[]) => {
@@ -184,6 +190,7 @@ describe('UsageLimitQuotaService', () => {
{ provide: CacheLockService, useValue: cacheLockService },
{ provide: ClickHouseService, useValue: clickHouseService },
{ provide: UsagePeriodService, useValue: usagePeriodService },
+ { provide: MetricsService, useValue: metricsService },
{
provide: DiscoveryService,
useValue: {
@@ -370,6 +377,37 @@ describe('UsageLimitQuotaService', () => {
cacheStorage.mget.mockRejectedValue(new Error('Socket closed'));
await expect(assertQuotaNotExhausted()).resolves.toBeUndefined();
+ expect(metricsService.incrementCounterBy).toHaveBeenCalledTimes(1);
+ expect(metricsService.incrementCounterBy).toHaveBeenCalledWith({
+ key: MetricsKeys.UsageLimitQuotaAdmittedOnFailure,
+ amount: 1,
+ attributes: { operation: 'assert', resourceType: UsageResourceType.AI },
+ });
+ });
+
+ it('counts the credits admitted when a consume fails', async () => {
+ setLimits([buildLimit({})]);
+ cacheStorage.mget.mockRejectedValue(new Error('Socket closed'));
+
+ await expect(consumeQuota(50, 7)).resolves.toEqual({ exhausted: [] });
+ expect(metricsService.incrementCounterBy).toHaveBeenCalledWith({
+ key: MetricsKeys.UsageLimitQuotaAdmittedOnFailure,
+ amount: 1,
+ attributes: { operation: 'consume', resourceType: UsageResourceType.AI },
+ });
+ expect(metricsService.incrementCounterBy).toHaveBeenCalledWith({
+ key: MetricsKeys.UsageLimitQuotaAdmittedOnFailureCreditsMicro,
+ amount: 50,
+ attributes: { operation: 'consume', resourceType: UsageResourceType.AI },
+ });
+ });
+
+ it('does not count an admit on a successful consume', async () => {
+ setLimits([buildLimit({})]);
+ cacheStorage.runScript.mockResolvedValue([1, 500]);
+
+ await expect(consumeQuota(50)).resolves.toEqual({ exhausted: [] });
+ expect(metricsService.incrementCounterBy).not.toHaveBeenCalled();
});
it('admits when the warm query fails instead of granting a fresh budget', async () => {
@@ -644,5 +682,15 @@ describe('UsageLimitQuotaService', () => {
).resolves.toBeNull();
expect(cacheStorage.mget).not.toHaveBeenCalled();
});
+
+ it('does not count a failed read as an admit', async () => {
+ setAllowance(2_000_000);
+ cacheStorage.mget.mockRejectedValue(new Error('Socket closed'));
+
+ await expect(
+ service.getAllowanceRemainingMicro('workspace-1'),
+ ).resolves.toBeNull();
+ expect(metricsService.incrementCounterBy).not.toHaveBeenCalled();
+ });
});
});
diff --git a/packages/twenty-server/src/engine/core-modules/usage-limit/services/usage-limit-quota.service.ts b/packages/twenty-server/src/engine/core-modules/usage-limit/services/usage-limit-quota.service.ts
index 00a5c2f35a8..ca8f5607186 100644
--- a/packages/twenty-server/src/engine/core-modules/usage-limit/services/usage-limit-quota.service.ts
+++ b/packages/twenty-server/src/engine/core-modules/usage-limit/services/usage-limit-quota.service.ts
@@ -14,6 +14,8 @@ import {
import { InjectCacheStorage } from 'src/engine/core-modules/cache-storage/decorators/cache-storage.decorator';
import { CacheStorageService } from 'src/engine/core-modules/cache-storage/services/cache-storage.service';
import { CacheStorageNamespace } from 'src/engine/core-modules/cache-storage/types/cache-storage-namespace.enum';
+import { MetricsService } from 'src/engine/core-modules/metrics/metrics.service';
+import { MetricsKeys } from 'src/engine/core-modules/metrics/types/metrics-keys.type';
import { CONSUME_QUOTA_COUNTERS_SCRIPT } from 'src/engine/core-modules/usage-limit/constants/consume-quota-counters-script.constant';
import {
UsageLimitException,
@@ -56,6 +58,12 @@ type AllowanceSumRow = {
total: string | number | null;
};
+type QuotaEnforcement = {
+ operation: 'assert' | 'consume';
+ resourceType: UsageResourceType;
+ cost?: QuotaCost;
+};
+
type QuotaConsumeArgs = {
workspaceId: string;
resourceType: UsageResourceType;
@@ -78,6 +86,7 @@ export class UsageLimitQuotaService implements OnModuleInit {
private readonly usagePeriodService: UsagePeriodService,
private readonly discoveryService: DiscoveryService,
private readonly usageLimitEntitlementService: UsageLimitEntitlementService,
+ private readonly metricsService: MetricsService,
) {}
onModuleInit() {
@@ -283,6 +292,7 @@ export class UsageLimitQuotaService implements OnModuleInit {
return this.admitOnFailure({
error,
workspaceId: args.workspaceId,
+ enforcement: { operation: 'assert', resourceType: args.resourceType },
admitted: [],
});
}
@@ -321,6 +331,11 @@ export class UsageLimitQuotaService implements OnModuleInit {
return this.admitOnFailure({
error,
workspaceId: args.workspaceId,
+ enforcement: {
+ operation: 'consume',
+ resourceType: args.resourceType,
+ cost,
+ },
admitted: [],
});
}
@@ -378,10 +393,12 @@ export class UsageLimitQuotaService implements OnModuleInit {
private admitOnFailure<TAdmitted>({
error,
workspaceId,
+ enforcement,
admitted,
}: {
error: unknown;
workspaceId: string;
+ enforcement?: QuotaEnforcement;
admitted: TAdmitted;
}): TAdmitted {
if (
@@ -395,9 +412,35 @@ export class UsageLimitQuotaService implements OnModuleInit {
`Usage quota enforcement degraded for workspace ${workspaceId}: ${error instanceof Error ? error.message : 'unknown error'}`,
);
+ if (isDefined(enforcement)) {
+ this.countAdmitOnFailure(enforcement);
+ }
+
return admitted;
}
+ private countAdmitOnFailure({
+ operation,
+ resourceType,
+ cost,
+ }: QuotaEnforcement): void {
+ const attributes = { operation, resourceType };
+
+ this.metricsService.incrementCounterBy({
+ key: MetricsKeys.UsageLimitQuotaAdmittedOnFailure,
+ amount: 1,
+ attributes,
+ });
+
+ if (isDefined(cost) && cost.creditsUsedMicro > 0) {
+ this.metricsService.incrementCounterBy({
+ key: MetricsKeys.UsageLimitQuotaAdmittedOnFailureCreditsMicro,
+ amount: cost.creditsUsedMicro,
+ attributes,
+ });
+ }
+ }
+
private async buildCounters(args: QuotaConsumeArgs): Promise<QuotaCounter[]> {
const [limitCounters, allowanceCounter] = await Promise.all([
this.buildLimitCounters(args),
diff --git a/packages/twenty-server/src/engine/core-modules/usage-limit/usage-limit.module.ts b/packages/twenty-server/src/engine/core-modules/usage-limit/usage-limit.module.ts
index d0decb24947..40ea9bb1e5d 100644
--- a/packages/twenty-server/src/engine/core-modules/usage-limit/usage-limit.module.ts
+++ b/packages/twenty-server/src/engine/core-modules/usage-limit/usage-limit.module.ts
@@ -5,6 +5,7 @@ import { TypeOrmModule } from '@nestjs/typeorm';
import { ClickHouseModule } from 'src/database/clickhouse/clickhouse.module';
import { ApiKeyEntity } from 'src/engine/core-modules/api-key/api-key.entity';
import { CacheLockModule } from 'src/engine/core-modules/cache-lock/cache-lock.module';
+import { MetricsModule } from 'src/engine/core-modules/metrics/metrics.module';
import { UsageLimitQuotaService } from 'src/engine/core-modules/usage-limit/services/usage-limit-quota.service';
import { UsagePeriodService } from 'src/engine/core-modules/usage-limit/services/usage-period.service';
import { UsageModule } from 'src/engine/core-modules/usage/usage.module';
@@ -37,6 +38,7 @@ import { WorkspaceCacheModule } from 'src/engine/workspace-cache/workspace-cache
DiscoveryModule,
CacheLockModule,
ClickHouseModule,
+ MetricsModule,
UsageModule,
],
providers: [