Skip to content

The open alternative to Salesforce, designed for AI.

refactor(usage-limit) - quota counters hold consumption; allowance cap from the workspace cache (#27479)

Etienne7a6880d75fd9
diff --git a/packages/twenty-server/src/engine/core-modules/billing-webhook/services/billing-webhook-subscription.service.ts b/packages/twenty-server/src/engine/core-modules/billing-webhook/services/billing-webhook-subscription.service.ts
index d7dbbe67c70..0df578e0195 100644
--- a/packages/twenty-server/src/engine/core-modules/billing-webhook/services/billing-webhook-subscription.service.ts
+++ b/packages/twenty-server/src/engine/core-modules/billing-webhook/services/billing-webhook-subscription.service.ts
@@ -26,7 +26,6 @@ import { BillingSubscriptionItemEntity } from 'src/engine/core-modules/billing/e
import { BillingSubscriptionEntity } from 'src/engine/core-modules/billing/entities/billing-subscription.entity';
import { SubscriptionStatus } from 'src/engine/core-modules/billing/enums/billing-subscription-status.enum';
import { BillingWebhookEvent } from 'src/engine/core-modules/billing/enums/billing-webhook-events.enum';
-import { UsageLimitQuotaService } from 'src/engine/core-modules/usage-limit/services/usage-limit-quota.service';
import { StripeCustomerService } from 'src/engine/core-modules/billing/stripe/services/stripe-customer.service';
import { StripeSubscriptionScheduleService } from 'src/engine/core-modules/billing/stripe/services/stripe-subscription-schedule.service';
import { StripeSubscriptionService } from 'src/engine/core-modules/billing/stripe/services/stripe-subscription.service';
@@ -70,7 +69,6 @@ export class BillingWebhookSubscriptionService {
private readonly workspaceService: WorkspaceService,
private readonly stripeSubscriptionScheduleService: StripeSubscriptionScheduleService,
private readonly stripeSubscriptionService: StripeSubscriptionService,
- private readonly usageLimitQuotaService: UsageLimitQuotaService,
private readonly workspaceCacheService: WorkspaceCacheService,
private readonly cacheLockService: CacheLockService,
) {}
@@ -209,8 +207,6 @@ export class BillingWebhookSubscriptionService {
'currentBillingSubscription',
]);
- await this.usageLimitQuotaService.dropAllowanceCounter(workspace.id);
-
const shouldSuspendWorkspace = allLiveSubscriptions.every(
(customerSubscription) =>
this.shouldSuspendWorkspace(customerSubscription),
diff --git a/packages/twenty-server/src/engine/core-modules/billing/services/billing-credit-allowance-provider.service.ts b/packages/twenty-server/src/engine/core-modules/billing/services/billing-credit-allowance-provider.service.ts
index fddaa412018..e1aed76ae4f 100644
--- a/packages/twenty-server/src/engine/core-modules/billing/services/billing-credit-allowance-provider.service.ts
+++ b/packages/twenty-server/src/engine/core-modules/billing/services/billing-credit-allowance-provider.service.ts
@@ -1,13 +1,11 @@
/* @license Enterprise */
-import { Injectable, Logger } from '@nestjs/common';
+import { Injectable } from '@nestjs/common';
import { isDefined } from 'twenty-shared/utils';
import { NO_BILLING_SUBSCRIPTION } from 'src/engine/core-modules/billing/constants/no-billing-subscription.constant';
-import { BillingCreditGrantService } from 'src/engine/core-modules/billing/services/billing-credit-grant.service';
-import { BillingSubscriptionService } from 'src/engine/core-modules/billing/services/billing-subscription.service';
-import { BillingUsageService } from 'src/engine/core-modules/billing/services/billing-usage.service';
+import { type FlatBillingSubscription } from 'src/engine/core-modules/billing/types/flat-billing-subscription.type';
import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';
import { CreditAllowanceProvider } from 'src/engine/core-modules/usage-limit/interfaces/credit-allowance-provider.service';
import { type CreditAllowance } from 'src/engine/core-modules/usage-limit/types/credit-allowance.type';
@@ -16,14 +14,9 @@ import { WorkspaceCacheService } from 'src/engine/workspace-cache/services/works
@Injectable()
export class BillingCreditAllowanceProvider extends CreditAllowanceProvider {
- private readonly logger = new Logger(BillingCreditAllowanceProvider.name);
-
constructor(
private readonly twentyConfigService: TwentyConfigService,
private readonly workspaceCacheService: WorkspaceCacheService,
- private readonly billingSubscriptionService: BillingSubscriptionService,
- private readonly billingCreditGrantService: BillingCreditGrantService,
- private readonly billingUsageService: BillingUsageService,
) {
super();
}
@@ -35,62 +28,52 @@ export class BillingCreditAllowanceProvider extends CreditAllowanceProvider {
async getCreditAllowancePeriod(
workspaceId: string,
): Promise<UsagePeriod | null> {
- if (!this.twentyConfigService.get('IS_BILLING_ENABLED')) {
- return null;
- }
+ const subscription = await this.findCurrentBillingSubscription(workspaceId);
- const { currentBillingSubscription } =
- await this.workspaceCacheService.getOrRecompute(workspaceId, [
- 'currentBillingSubscription',
- ]);
-
- if (currentBillingSubscription === NO_BILLING_SUBSCRIPTION) {
+ if (!isDefined(subscription)) {
return null;
}
return {
- periodStart: new Date(currentBillingSubscription.currentPeriodStart),
- periodEnd: new Date(currentBillingSubscription.currentPeriodEnd),
+ periodStart: new Date(subscription.currentPeriodStart),
+ periodEnd: new Date(subscription.currentPeriodEnd),
};
}
async getCreditAllowance(
workspaceId: string,
): Promise<CreditAllowance | null> {
- if (!this.twentyConfigService.get('IS_BILLING_ENABLED')) {
+ const subscription = await this.findCurrentBillingSubscription(workspaceId);
+
+ if (
+ !isDefined(subscription) ||
+ !isDefined(subscription.planAllowanceMicro)
+ ) {
return null;
}
- try {
- const subscription =
- await this.billingSubscriptionService.getCurrentBillingSubscription({
- workspaceId,
- });
-
- if (!isDefined(subscription)) {
- return null;
- }
-
- const { balanceMicro, earliestExpiryBefore } =
- await this.billingCreditGrantService.getActiveCreditBalance({
- workspaceId,
- boundary: subscription.currentPeriodEnd,
- });
-
- return {
- periodStart: subscription.currentPeriodStart,
- periodEnd: subscription.currentPeriodEnd,
- allowanceMicro:
- this.billingUsageService.getResourceUsageCap(subscription) +
- balanceMicro,
- validUntil: earliestExpiryBefore ?? subscription.currentPeriodEnd,
- };
- } catch (error) {
- this.logger.error(
- `Could not compute allowance for workspace ${workspaceId}: ${error instanceof Error ? error.message : 'unknown error'}`,
- );
+ return {
+ periodStart: new Date(subscription.currentPeriodStart),
+ periodEnd: new Date(subscription.currentPeriodEnd),
+ planAllowanceMicro: subscription.planAllowanceMicro,
+ grants: subscription.creditGrants,
+ };
+ }
+ private async findCurrentBillingSubscription(
+ workspaceId: string,
+ ): Promise<FlatBillingSubscription | null> {
+ if (!this.twentyConfigService.get('IS_BILLING_ENABLED')) {
return null;
}
+
+ const { currentBillingSubscription } =
+ await this.workspaceCacheService.getOrRecompute(workspaceId, [
+ 'currentBillingSubscription',
+ ]);
+
+ return currentBillingSubscription === NO_BILLING_SUBSCRIPTION
+ ? null
+ : currentBillingSubscription;
}
}
diff --git a/packages/twenty-server/src/engine/core-modules/billing/services/billing-credit-grant.service.ts b/packages/twenty-server/src/engine/core-modules/billing/services/billing-credit-grant.service.ts
index 8b96b31a4eb..cf3e00530b7 100644
--- a/packages/twenty-server/src/engine/core-modules/billing/services/billing-credit-grant.service.ts
+++ b/packages/twenty-server/src/engine/core-modules/billing/services/billing-credit-grant.service.ts
@@ -99,31 +99,9 @@ export class BillingCreditGrantService {
}
async getActiveCreditsMicro(workspaceId: string): Promise<number> {
- const { balanceMicro } = await this.getActiveCreditBalance({
- workspaceId,
- boundary: null,
- });
-
- return balanceMicro;
- }
-
- // One statement so the balance and its bounding expiry come from the same snapshot
- async getActiveCreditBalance({
- workspaceId,
- boundary,
- }: {
- workspaceId: string;
- boundary: Date | null;
- }): Promise<{ balanceMicro: number; earliestExpiryBefore: Date | null }> {
const result = await this.billingCreditGrantRepository
.createQueryBuilder('billingCreditGrant')
.select('COALESCE(SUM("billingCreditGrant"."amountMicro"), 0)', 'total')
- .addSelect(
- isDefined(boundary)
- ? 'MIN("billingCreditGrant"."expiresAt") FILTER (WHERE "billingCreditGrant"."expiresAt" < :boundary)'
- : 'NULL',
- 'earliestExpiry',
- )
.where('"billingCreditGrant"."workspaceId" = :workspaceId', {
workspaceId,
})
@@ -132,11 +110,7 @@ export class BillingCreditGrantService {
.andWhere(
'("billingCreditGrant"."expiresAt" IS NULL OR "billingCreditGrant"."expiresAt" > now())',
)
- .setParameters(isDefined(boundary) ? { boundary } : {})
- .getRawOne<{
- total: string | number | null;
- earliestExpiry: Date | string | null;
- }>();
+ .getRawOne<{ total: string | number | null }>();
const balanceMicro = Number(result?.total ?? 0);
@@ -148,12 +122,18 @@ export class BillingCreditGrantService {
);
}
- return {
- balanceMicro,
- earliestExpiryBefore: isDefined(result?.earliestExpiry)
- ? new Date(result.earliestExpiry)
- : null,
- };
+ return balanceMicro;
+ }
+
+ async findUnexpiredGrants(
+ workspaceId: string,
+ ): Promise<BillingCreditGrantEntity[]> {
+ return this.billingCreditGrantRepository.find(workspaceId, {
+ where: [
+ { revokedAt: IsNull(), expiresAt: IsNull() },
+ { revokedAt: IsNull(), expiresAt: MoreThan(new Date()) },
+ ],
+ });
}
async findGrantsLiveDuringPeriod(
diff --git a/packages/twenty-server/src/engine/core-modules/billing/services/billing-credit.service.ts b/packages/twenty-server/src/engine/core-modules/billing/services/billing-credit.service.ts
index 12b3df4b785..093aac38fc6 100644
--- a/packages/twenty-server/src/engine/core-modules/billing/services/billing-credit.service.ts
+++ b/packages/twenty-server/src/engine/core-modules/billing/services/billing-credit.service.ts
@@ -17,7 +17,6 @@ import { BillingService } from 'src/engine/core-modules/billing/services/billing
import { alignGrantExpiryToPeriodEnd } from 'src/engine/core-modules/billing/utils/align-grant-expiry-to-period-end.util';
import { buildBillingCreditStateLockKey } from 'src/engine/core-modules/billing/utils/build-billing-credit-state-lock-key.util';
import { CacheLockService } from 'src/engine/core-modules/cache-lock/cache-lock.service';
-import { UsageLimitQuotaService } from 'src/engine/core-modules/usage-limit/services/usage-limit-quota.service';
import { WorkspaceCacheService } from 'src/engine/workspace-cache/services/workspace-cache.service';
type GrantCreditsParams = {
@@ -43,7 +42,6 @@ export class BillingCreditService {
private readonly billingSubscriptionService: BillingSubscriptionService,
private readonly cacheLockService: CacheLockService,
private readonly workspaceCacheService: WorkspaceCacheService,
- private readonly usageLimitQuotaService: UsageLimitQuotaService,
) {}
async grantCredits(
@@ -166,8 +164,6 @@ export class BillingCreditService {
await this.workspaceCacheService.invalidateAndRecompute(workspaceId, [
'currentBillingSubscription',
]);
-
- await this.usageLimitQuotaService.dropAllowanceCounter(workspaceId);
}
}
diff --git a/packages/twenty-server/src/engine/core-modules/billing/services/billing-subscription.service.ts b/packages/twenty-server/src/engine/core-modules/billing/services/billing-subscription.service.ts
index ade70f1194e..a342eae31b0 100644
--- a/packages/twenty-server/src/engine/core-modules/billing/services/billing-subscription.service.ts
+++ b/packages/twenty-server/src/engine/core-modules/billing/services/billing-subscription.service.ts
@@ -29,7 +29,6 @@ import { BillingEntitlementKey } from 'src/engine/core-modules/billing/enums/bil
import { WORKSPACE_ACTIVATING_SUBSCRIPTION_STATUSES } from 'src/engine/core-modules/billing/constants/workspace-activating-subscription-statuses.constant';
import { SubscriptionStatus } from 'src/engine/core-modules/billing/enums/billing-subscription-status.enum';
import { BillingPriceService } from 'src/engine/core-modules/billing/services/billing-price.service';
-import { UsageLimitQuotaService } from 'src/engine/core-modules/usage-limit/services/usage-limit-quota.service';
import { StripeCustomerService } from 'src/engine/core-modules/billing/stripe/services/stripe-customer.service';
import { StripeSubscriptionScheduleService } from 'src/engine/core-modules/billing/stripe/services/stripe-subscription-schedule.service';
import { StripeSubscriptionService } from 'src/engine/core-modules/billing/stripe/services/stripe-subscription.service';
@@ -68,7 +67,6 @@ export class BillingSubscriptionService {
private readonly billingCustomerRepository: WorkspaceScopedRepository<BillingCustomerEntity>,
private readonly enterprisePlanService: EnterprisePlanService,
private readonly workspaceCacheService: WorkspaceCacheService,
- private readonly usageLimitQuotaService: UsageLimitQuotaService,
) {}
async getBillingSubscriptions(workspaceId: string) {
@@ -323,12 +321,6 @@ export class BillingSubscriptionService {
updatedSubscription.id,
);
- await this.workspaceCacheService.invalidateAndRecompute(workspace.id, [
- 'currentBillingSubscription',
- ]);
-
- await this.usageLimitQuotaService.dropAllowanceCounter(workspace.id);
-
return {
status: getSubscriptionStatus(updatedSubscription.status),
hasPaymentMethod: true,
@@ -471,6 +463,10 @@ export class BillingSubscriptionService {
}
}
+ await this.workspaceCacheService.invalidateAndRecompute(workspaceId, [
+ 'currentBillingSubscription',
+ ]);
+
return currentBillingSubscription;
}
diff --git a/packages/twenty-server/src/engine/core-modules/billing/services/billing-usage.service.ts b/packages/twenty-server/src/engine/core-modules/billing/services/billing-usage.service.ts
index b3325db8ad7..301ea276120 100644
--- a/packages/twenty-server/src/engine/core-modules/billing/services/billing-usage.service.ts
+++ b/packages/twenty-server/src/engine/core-modules/billing/services/billing-usage.service.ts
@@ -241,7 +241,7 @@ export class BillingUsageService {
);
}
- getResourceUsageCap(subscription: BillingSubscriptionEntity): number {
+ findResourceUsageCap(subscription: BillingSubscriptionEntity): number | null {
const isInFreeTrial = subscription.status === SubscriptionStatus.Trialing;
if (isInFreeTrial) {
@@ -260,10 +260,7 @@ export class BillingUsageService {
);
if (!isDefined(resourceCreditPrice)) {
- throw new BillingException(
- `Resource credit price not found for workspace ${subscription.workspaceId}`,
- BillingExceptionCode.BILLING_PRICE_NOT_FOUND,
- );
+ return null;
}
return Number(resourceCreditPrice.metadata?.credit_amount ?? 0);
diff --git a/packages/twenty-server/src/engine/core-modules/billing/services/workspace-current-billing-subscription-cache.service.ts b/packages/twenty-server/src/engine/core-modules/billing/services/workspace-current-billing-subscription-cache.service.ts
index 186b9051c5a..85647b6a8b6 100644
--- a/packages/twenty-server/src/engine/core-modules/billing/services/workspace-current-billing-subscription-cache.service.ts
+++ b/packages/twenty-server/src/engine/core-modules/billing/services/workspace-current-billing-subscription-cache.service.ts
@@ -4,7 +4,9 @@ import { Injectable } from '@nestjs/common';
import { isDefined } from 'twenty-shared/utils';
+import { BillingCreditGrantService } from 'src/engine/core-modules/billing/services/billing-credit-grant.service';
import { BillingSubscriptionService } from 'src/engine/core-modules/billing/services/billing-subscription.service';
+import { BillingUsageService } from 'src/engine/core-modules/billing/services/billing-usage.service';
import { NO_BILLING_SUBSCRIPTION } from 'src/engine/core-modules/billing/constants/no-billing-subscription.constant';
import { type CurrentBillingSubscription } from 'src/engine/core-modules/billing/types/flat-billing-subscription.type';
@@ -17,6 +19,8 @@ import { type WorkspaceCacheProviderContext } from 'src/engine/workspace-cache/t
export class WorkspaceCurrentBillingSubscriptionCacheService extends WorkspaceCacheProvider<CurrentBillingSubscription> {
constructor(
private readonly billingSubscriptionService: BillingSubscriptionService,
+ private readonly billingCreditGrantService: BillingCreditGrantService,
+ private readonly billingUsageService: BillingUsageService,
) {
super();
}
@@ -33,6 +37,9 @@ export class WorkspaceCurrentBillingSubscriptionCacheService extends WorkspaceCa
return NO_BILLING_SUBSCRIPTION;
}
+ const unexpiredGrants =
+ await this.billingCreditGrantService.findUnexpiredGrants(workspaceId);
+
return {
id: subscription.id,
workspaceId: subscription.workspaceId,
@@ -50,6 +57,15 @@ export class WorkspaceCurrentBillingSubscriptionCacheService extends WorkspaceCa
trialStart: subscription.trialStart,
trialEnd: subscription.trialEnd,
collectionMethod: subscription.collectionMethod,
+ planAllowanceMicro:
+ this.billingUsageService.findResourceUsageCap(subscription),
+ creditGrants: unexpiredGrants.map(
+ ({ amountMicro, effectiveAt, expiresAt }) => ({
+ amountMicro,
+ effectiveAtMs: effectiveAt.getTime(),
+ expiresAtMs: expiresAt?.getTime() ?? null,
+ }),
+ ),
};
}
}
diff --git a/packages/twenty-server/src/engine/core-modules/billing/types/flat-billing-subscription.type.ts b/packages/twenty-server/src/engine/core-modules/billing/types/flat-billing-subscription.type.ts
index a130a971d91..c98ceff99f1 100644
--- a/packages/twenty-server/src/engine/core-modules/billing/types/flat-billing-subscription.type.ts
+++ b/packages/twenty-server/src/engine/core-modules/billing/types/flat-billing-subscription.type.ts
@@ -4,6 +4,7 @@ import { NO_BILLING_SUBSCRIPTION } from 'src/engine/core-modules/billing/constan
import { type BillingSubscriptionCollectionMethod } from 'src/engine/core-modules/billing/enums/billing-subscription-collection-method.enum';
import { type SubscriptionInterval } from 'src/engine/core-modules/billing/enums/billing-subscription-interval.enum';
import { type SubscriptionStatus } from 'src/engine/core-modules/billing/enums/billing-subscription-status.enum';
+import { type CreditAllowanceGrant } from 'src/engine/core-modules/usage-limit/types/credit-allowance-grant.type';
export type FlatBillingSubscription = {
id: string;
@@ -22,6 +23,8 @@ export type FlatBillingSubscription = {
trialStart: Date | null;
trialEnd: Date | null;
collectionMethod: BillingSubscriptionCollectionMethod;
+ planAllowanceMicro: number | null;
+ creditGrants: CreditAllowanceGrant[];
};
export type CurrentBillingSubscription =
diff --git a/packages/twenty-server/src/engine/core-modules/usage-limit/constants/warm-and-consume-quota-counters-script.constant.ts b/packages/twenty-server/src/engine/core-modules/usage-limit/constants/warm-and-consume-quota-counters-script.constant.ts
new file mode 100644
index 00000000000..5a1da4acd73
--- /dev/null
+++ b/packages/twenty-server/src/engine/core-modules/usage-limit/constants/warm-and-consume-quota-counters-script.constant.ts
@@ -0,0 +1,25 @@
+import { type CacheScript } from 'src/engine/core-modules/cache-storage/types/cache-script.type';
+
+export const WARM_AND_CONSUME_QUOTA_COUNTERS_SCRIPT: CacheScript = {
+ name: 'usage-limit:warm-and-consume-quota-counters',
+ source: `
+local amounts = cjson.decode(ARGV[1])
+local warmValues = cjson.decode(ARGV[2])
+local warmTtls = cjson.decode(ARGV[3])
+local result = {}
+
+for i = 1, #KEYS do
+ if warmValues[i] then
+ redis.call('SET', KEYS[i], warmValues[i], 'NX', 'PX', warmTtls[i])
+ end
+
+ if redis.call('EXISTS', KEYS[i]) == 1 then
+ result[i] = redis.call('INCRBY', KEYS[i], amounts[i])
+ else
+ result[i] = false
+ end
+end
+
+return result
+`,
+};
diff --git a/packages/twenty-server/src/engine/core-modules/usage-limit/services/usage-limit-quota.service.ts b/packages/twenty-server/src/engine/core-modules/usage-limit/services/usage-limit-quota.service.ts
index 80ef659e4aa..c0a82bc35de 100644
--- a/packages/twenty-server/src/engine/core-modules/usage-limit/services/usage-limit-quota.service.ts
+++ b/packages/twenty-server/src/engine/core-modules/usage-limit/services/usage-limit-quota.service.ts
@@ -2,20 +2,15 @@ import { Injectable, Logger, type OnModuleInit } from '@nestjs/common';
import { DiscoveryService } from '@nestjs/core';
import uniqBy from 'lodash.uniqby';
-import { isDefined } from 'twenty-shared/utils';
+import { isDefined, isNonEmptyArray } from 'twenty-shared/utils';
-import { CacheLockService } from 'src/engine/core-modules/cache-lock/cache-lock.service';
-import {
- CacheLockException,
- CacheLockExceptionCode,
-} from 'src/engine/core-modules/cache-lock/exceptions/cache-lock.exception';
import { InjectCacheStorage } from 'src/engine/core-modules/cache-storage/decorators/cache-storage.decorator';
import { CacheStorageService } from 'src/engine/core-modules/cache-storage/services/cache-storage.service';
import { CacheStorageNamespace } from 'src/engine/core-modules/cache-storage/types/cache-storage-namespace.enum';
import { MetricsService } from 'src/engine/core-modules/metrics/metrics.service';
import { MetricsKeys } from 'src/engine/core-modules/metrics/types/metrics-keys.type';
import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';
-import { CONSUME_QUOTA_COUNTERS_SCRIPT } from 'src/engine/core-modules/usage-limit/constants/consume-quota-counters-script.constant';
+import { WARM_AND_CONSUME_QUOTA_COUNTERS_SCRIPT } from 'src/engine/core-modules/usage-limit/constants/warm-and-consume-quota-counters-script.constant';
import { UsageLimitException } from 'src/engine/core-modules/usage-limit/exceptions/usage-limit.exception';
import { CreditAllowanceProvider } from 'src/engine/core-modules/usage-limit/interfaces/credit-allowance-provider.service';
import { UsageLimitEntitlementService } from 'src/engine/core-modules/usage-limit/services/usage-limit-entitlement.service';
@@ -30,26 +25,22 @@ import { type LimitQuotaCounter } from 'src/engine/core-modules/usage-limit/type
import { type QuotaCost } from 'src/engine/core-modules/usage-limit/types/quota-cost.type';
import { type QuotaCounter } from 'src/engine/core-modules/usage-limit/types/quota-counter.type';
import { type QuotaLimitDefault } from 'src/engine/core-modules/usage-limit/types/quota-limit-default.type';
-import { type UsageLimitCounterScope } from 'src/engine/core-modules/usage-limit/types/usage-limit-counter-scope.type';
import { buildAllowanceCounterKey } from 'src/engine/core-modules/usage-limit/utils/build-allowance-counter-key.util';
+import { buildConsumeScriptArguments } from 'src/engine/core-modules/usage-limit/utils/build-consume-script-arguments.util';
import { buildIntraWorkspaceLimitCounterKeys } from 'src/engine/core-modules/usage-limit/utils/build-intra-workspace-limit-counter-keys.util';
import { buildLimitQuotaCounter } from 'src/engine/core-modules/usage-limit/utils/build-limit-quota-counter.util';
-import { buildLimitWarmedEntries } from 'src/engine/core-modules/usage-limit/utils/build-limit-warmed-entries.util';
-import { buildOverriddenQuotaDefaultCounterKeys } from 'src/engine/core-modules/usage-limit/utils/build-overridden-quota-default-counter-keys.util';
import { buildPeriodGroupKey } from 'src/engine/core-modules/usage-limit/utils/build-period-group-key.util';
-import { buildQuotaCounterKey } from 'src/engine/core-modules/usage-limit/utils/build-quota-counter-key.util';
import { buildQuotaCounters } from 'src/engine/core-modules/usage-limit/utils/build-quota-counters.util';
import { buildQuotaDebits } from 'src/engine/core-modules/usage-limit/utils/build-quota-debits.util';
import { buildQuotaExhaustedScope } from 'src/engine/core-modules/usage-limit/utils/build-quota-exhausted-scope.util';
-import { buildQuotaWarmLockKey } from 'src/engine/core-modules/usage-limit/utils/build-quota-warm-lock-key.util';
import { buildSpendersFromUsageSpenders } from 'src/engine/core-modules/usage-limit/utils/build-spenders-from-usage-spenders.util';
+import { computeCreditAllowanceMicro } from 'src/engine/core-modules/usage-limit/utils/compute-credit-allowance-micro.util';
import { computeQuotaConsumed } from 'src/engine/core-modules/usage-limit/utils/compute-quota-consumed.util';
import { findCreditAllowanceProvider } from 'src/engine/core-modules/usage-limit/utils/find-credit-allowance-provider.util';
import { findExhaustedCounters } from 'src/engine/core-modules/usage-limit/utils/find-exhausted-counters.util';
+import { findSuppressedUsageLimitDefaults } from 'src/engine/core-modules/usage-limit/utils/find-suppressed-usage-limit-defaults.util';
import { findUsageLimitDefinition } from 'src/engine/core-modules/usage-limit/utils/find-usage-limit-definition.util';
-import { isAnchoredPeriodUnit } from 'src/engine/core-modules/usage-limit/utils/is-anchored-period-unit.util';
import { isQuotaLimit } from 'src/engine/core-modules/usage-limit/utils/is-quota-limit.util';
-import { fromConsumeResultsToRemainings } from 'src/engine/core-modules/usage-limit/utils/from-consume-results-to-remainings.util';
import { getPeriodAnchor } from 'src/engine/core-modules/usage-limit/utils/get-period-anchor.util';
import { type UsageOperationType } from 'src/engine/core-modules/usage/enums/usage-operation-type.enum';
import { type UsageResourceType } from 'src/engine/core-modules/usage/enums/usage-resource-type.enum';
@@ -63,8 +54,6 @@ import { fromRecordUsageInputToUsageConsumptionRow } from 'src/engine/core-modul
import { WorkspaceCacheException } from 'src/engine/workspace-cache/exceptions/workspace-cache.exception';
import { WorkspaceCacheService } from 'src/engine/workspace-cache/services/workspace-cache.service';
-const QUOTA_WARM_LOCK_OPTIONS = { ms: 50, maxRetries: 20, ttl: 10_000 };
-
type QuotaEnforcement = {
operation: 'assert' | 'consume';
resourceType: UsageResourceType;
@@ -78,6 +67,11 @@ type QuotaConsumeArgs = {
spenders: UsageSpenders;
};
+type QuotaConsumeResult = {
+ consumedValues: (number | null)[];
+ isDegraded: boolean;
+};
+
@Injectable()
export class UsageLimitQuotaService implements OnModuleInit {
private readonly logger = new Logger(UsageLimitQuotaService.name);
@@ -88,7 +82,6 @@ export class UsageLimitQuotaService implements OnModuleInit {
@InjectCacheStorage(CacheStorageNamespace.EngineUsageLimit)
private readonly cacheStorage: CacheStorageService,
private readonly workspaceCacheService: WorkspaceCacheService,
- private readonly cacheLockService: CacheLockService,
private readonly usagePeriodService: UsagePeriodService,
private readonly usageAnalyticsService: UsageAnalyticsService,
private readonly usageRecorderService: UsageRecorderService,
@@ -153,27 +146,6 @@ export class UsageLimitQuotaService implements OnModuleInit {
};
}
- async dropAllowanceCounter(workspaceId: string): Promise<void> {
- const period =
- await this.creditAllowanceProvider.getCreditAllowancePeriod(workspaceId);
-
- if (!isDefined(period)) {
- return;
- }
-
- await this.delUnderWarmLock({
- workspaceId,
- keys: [
- buildAllowanceCounterKey({
- workspaceId,
- periodStart: period.periodStart,
- }),
- ],
- });
- }
-
- // Intra-workspace counters are not debited while the entitlement is off, so
- // a warm balance misses that usage; dropping them forces a ClickHouse rewarm.
async dropIntraWorkspaceLimitCounters(workspaceId: string): Promise<void> {
const limits = await this.findAllLimits(workspaceId);
@@ -190,66 +162,39 @@ export class UsageLimitQuotaService implements OnModuleInit {
return;
}
- await this.delUnderWarmLock({ workspaceId, keys });
+ await this.cacheStorage.mdel(keys);
}
- async dropLimitCounter(usageLimit: UsageLimitCounterScope): Promise<void> {
+ async dropLimitCounter({
+ workspaceId,
+ usageLimit,
+ }: {
+ workspaceId: string;
+ usageLimit: FlatUsageLimit;
+ }): Promise<void> {
if (
- usageLimit.limitKind !== 'quota' ||
- !isAnchoredPeriodUnit(usageLimit.periodUnit)
+ !isQuotaLimit(usageLimit) ||
+ isNonEmptyArray(findSuppressedUsageLimitDefaults(usageLimit))
) {
return;
}
- const quotaLimitDefaults = this.buildQuotaLimitDefaults(
- usageLimit.resourceType,
- );
-
const periodByUnit = await this.usagePeriodService.findCurrentPeriodsByUnit(
{
- workspaceId: usageLimit.workspaceId,
- periodUnits: [
- usageLimit.periodUnit,
- ...quotaLimitDefaults.map(
- (quotaLimitDefault) => quotaLimitDefault.periodUnit,
- ),
- ],
+ workspaceId,
+ periodUnits: [usageLimit.periodUnit],
},
);
const period = periodByUnit[usageLimit.periodUnit];
- const keys = [
- ...(isDefined(period)
- ? [
- buildQuotaCounterKey({
- workspaceId: usageLimit.workspaceId,
- resourceType: usageLimit.resourceType,
- operationType: usageLimit.operationType,
- spenderType: usageLimit.spenderType,
- spenderId: usageLimit.spenderId,
- unit: usageLimit.unit,
- periodUnit: usageLimit.periodUnit,
- periodStart: period.periodStart,
- limitValue: usageLimit.limitValue,
- }),
- ]
- : []),
- ...buildOverriddenQuotaDefaultCounterKeys({
- usageLimit,
- quotaLimitDefaults,
- periodByUnit,
- }),
- ];
-
- if (keys.length === 0) {
+ if (!isDefined(period)) {
return;
}
- await this.delUnderWarmLock({
- workspaceId: usageLimit.workspaceId,
- keys,
- });
+ await this.cacheStorage.del(
+ buildLimitQuotaCounter({ workspaceId, limit: usageLimit, period }).key,
+ );
}
async getAllowanceUsage(workspaceId: string): Promise<{
@@ -263,40 +208,25 @@ export class UsageLimitQuotaService implements OnModuleInit {
return null;
}
- const allowance =
- await this.creditAllowanceProvider.getCreditAllowance(workspaceId);
-
- if (!isDefined(allowance)) {
- return null;
- }
-
- if (allowance.periodStart.getTime() !== counter.periodStart.getTime()) {
- return {
- limitValue: allowance.allowanceMicro,
- consumedValue: null,
- periodEnd: allowance.periodEnd,
- };
- }
-
let consumedValue: number | null = null;
try {
- const [remaining] = await this.readRemainings({
- workspaceId,
- counters: [counter],
- });
+ const [storedValue] = await this.cacheStorage.mget<number>([counter.key]);
- consumedValue = isDefined(remaining)
- ? Math.max(0, allowance.allowanceMicro - remaining)
- : null;
+ consumedValue =
+ storedValue ??
+ (await this.usageAnalyticsService.getCreditsUsedMicroForBillingPeriod({
+ workspaceId,
+ periodStart: counter.periodStart,
+ }));
} catch (error) {
this.admitOnFailure({ error, workspaceId, admitted: null });
}
return {
- limitValue: allowance.allowanceMicro,
+ limitValue: counter.limitValue,
consumedValue,
- periodEnd: allowance.periodEnd,
+ periodEnd: counter.periodEnd,
};
}
@@ -304,18 +234,23 @@ export class UsageLimitQuotaService implements OnModuleInit {
workspaceId: string,
): Promise<number | null> {
try {
- const allowanceCounter = await this.buildAllowanceCounter(workspaceId);
+ const counter = await this.buildAllowanceCounter(workspaceId);
- if (!isDefined(allowanceCounter)) {
+ if (!isDefined(counter)) {
return null;
}
- const [remaining] = await this.readRemainings({
+ const {
+ consumedValues: [consumedValue],
+ } = await this.consumeCounters({
workspaceId,
- counters: [allowanceCounter],
+ counters: [counter],
+ amounts: [0],
});
- return remaining;
+ return isDefined(consumedValue)
+ ? counter.limitValue - consumedValue
+ : null;
} catch (error) {
return this.admitOnFailure({ error, workspaceId, admitted: null });
}
@@ -377,35 +312,6 @@ export class UsageLimitQuotaService implements OnModuleInit {
);
}
- private async delUnderWarmLock({
- workspaceId,
- keys,
- }: {
- workspaceId: string;
- keys: string[];
- }): Promise<void> {
- try {
- await this.cacheLockService.withLock(
- () => this.cacheStorage.mdel(keys),
- buildQuotaWarmLockKey(workspaceId),
- QUOTA_WARM_LOCK_OPTIONS,
- );
- } catch (error) {
- if (
- !(error instanceof CacheLockException) ||
- error.code !== CacheLockExceptionCode.LOCK_ACQUISITION_TIMEOUT
- ) {
- throw error;
- }
-
- this.logger.warn(
- `Dropping quota counters ${keys.join(', ')} without the warm lock: ${error.message}`,
- );
-
- await this.cacheStorage.mdel(keys);
- }
- }
-
private async findAllLimits(workspaceId: string): Promise<FlatUsageLimit[]> {
const { usageLimits } = await this.workspaceCacheService.getOrRecompute(
workspaceId,
@@ -421,29 +327,45 @@ export class UsageLimitQuotaService implements OnModuleInit {
cost,
...args
}: QuotaConsumeArgs & { cost?: QuotaCost }): Promise<ExhaustedScope[]> {
- try {
- const counters = await this.buildCounters(args);
-
- if (counters.length === 0) {
- return [];
- }
+ const enforcement: QuotaEnforcement = {
+ operation: 'assert',
+ resourceType: args.resourceType,
+ };
- const remainings = await this.readRemainings({
+ try {
+ const [limitCounters, { allowanceCounter, isAllowanceSkipped }] =
+ await Promise.all([
+ this.buildLimitCounters(args),
+ this.buildAllowanceCounterAdmittingOnFailure(args.workspaceId),
+ ]);
+
+ const counters: QuotaCounter[] = [
+ ...limitCounters,
+ ...(isDefined(allowanceCounter) ? [allowanceCounter] : []),
+ ];
+
+ const { consumedValues, isDegraded } = await this.consumeCounters({
workspaceId: args.workspaceId,
counters,
+ amounts: counters.map(() => 0),
});
- return await this.buildExhaustedScopes({
- args,
- counters,
- remainings,
- cost,
- });
+ if (isDegraded || isAllowanceSkipped) {
+ this.countAdmitOnFailure(enforcement);
+ }
+
+ return findExhaustedCounters({ counters, consumedValues, cost }).map(
+ (counter) =>
+ buildQuotaExhaustedScope({
+ resourceType: args.resourceType,
+ counter,
+ }),
+ );
} catch (error) {
return this.admitOnFailure({
error,
workspaceId: args.workspaceId,
- enforcement: { operation: 'assert', resourceType: args.resourceType },
+ enforcement,
admitted: [],
});
}
@@ -457,31 +379,45 @@ export class UsageLimitQuotaService implements OnModuleInit {
events: RecordUsageInput[];
}): Promise<ExhaustedKind | null> {
try {
+ const [limitCounters, { allowanceCounter, isAllowanceSkipped }] =
+ await Promise.all([
+ this.buildLimitCountersForEvents({ workspaceId, events }),
+ this.buildAllowanceCounterAdmittingOnFailure(workspaceId),
+ ]);
+
const debits = buildQuotaDebits({
- counters: await this.buildCountersForEvents({ workspaceId, events }),
+ counters: [
+ ...limitCounters,
+ ...(isDefined(allowanceCounter) ? [allowanceCounter] : []),
+ ],
events,
});
- if (debits.length === 0) {
- return null;
- }
-
const counters = debits.map((debit) => debit.counter);
- // The consume script only debits keys that exist: warm cold counters
- // first so a consume-only caller (workflow, logic function) is metered
- // from its first call instead of waiting for an assert to warm the key.
- await this.readRemainings({ workspaceId, counters });
-
- const consumeResults = await this.cacheStorage.runScript<number[]>({
- script: CONSUME_QUOTA_COUNTERS_SCRIPT,
- keys: counters.map((counter) => counter.key),
- args: [JSON.stringify(debits.map((debit) => debit.amount))],
+ const { consumedValues, isDegraded } = await this.consumeCounters({
+ workspaceId,
+ counters,
+ amounts: debits.map((debit) => debit.amount),
});
+ if (isDegraded || isAllowanceSkipped) {
+ this.countConsumeAdmittedOnFailure({
+ events,
+ isAllowanceUnmetered:
+ isAllowanceSkipped ||
+ counters.some(
+ (counter, index) =>
+ counter.kind === 'allowance' &&
+ !isDefined(consumedValues[index]) &&
+ counter.periodEnd.getTime() > Date.now(),
+ ),
+ });
+ }
+
const exhaustedCounters = findExhaustedCounters({
counters,
- remainings: fromConsumeResultsToRemainings(consumeResults),
+ consumedValues,
});
if (exhaustedCounters.some((counter) => counter.kind === 'allowance')) {
@@ -493,38 +429,22 @@ export class UsageLimitQuotaService implements OnModuleInit {
this.logger.error(
`Usage quota debit degraded for workspace ${workspaceId}: ${error instanceof Error ? error.message : 'unknown error'}`,
);
-
- for (const resourceType of new Set(
- events.map((event) => event.resourceType),
- )) {
- this.countAdmitOnFailure({
- operation: 'consume',
- resourceType,
- creditsUsedMicro: events
- .filter((event) => event.resourceType === resourceType)
- .reduce(
- (total, event) =>
- total +
- Number(
- fromRecordUsageInputToUsageConsumptionRow(event)
- .creditsUsedMicro,
- ),
- 0,
- ),
- });
- }
+ this.countConsumeAdmittedOnFailure({
+ events,
+ isAllowanceUnmetered: true,
+ });
return null;
}
}
- private async buildCountersForEvents({
+ private async buildLimitCountersForEvents({
workspaceId,
events,
}: {
workspaceId: string;
events: RecordUsageInput[];
- }): Promise<QuotaCounter[]> {
+ }): Promise<LimitQuotaCounter[]> {
const scopeEvents = uniqBy(events, (event) =>
JSON.stringify([
event.resourceType,
@@ -535,7 +455,7 @@ export class UsageLimitQuotaService implements OnModuleInit {
const counters = await Promise.all(
scopeEvents.map(({ resourceType, operationType, spenders }) =>
- this.buildCounters({
+ this.buildLimitCounters({
workspaceId,
resourceType,
operationType,
@@ -547,40 +467,34 @@ export class UsageLimitQuotaService implements OnModuleInit {
return uniqBy(counters.flat(), (counter) => counter.key);
}
- private async buildExhaustedScopes({
- args,
- counters,
- remainings,
- cost,
+ private countConsumeAdmittedOnFailure({
+ events,
+ isAllowanceUnmetered,
}: {
- args: QuotaConsumeArgs;
- counters: QuotaCounter[];
- remainings: (number | null)[];
- cost?: QuotaCost;
- }): Promise<ExhaustedScope[]> {
- const exhaustedCounters = findExhaustedCounters({
- counters,
- remainings,
- cost,
- });
-
- if (exhaustedCounters.length === 0) {
- return [];
+ events: RecordUsageInput[];
+ isAllowanceUnmetered: boolean;
+ }): void {
+ for (const resourceType of new Set(
+ events.map((event) => event.resourceType),
+ )) {
+ this.countAdmitOnFailure({
+ operation: 'consume',
+ resourceType,
+ creditsUsedMicro: isAllowanceUnmetered
+ ? events
+ .filter((event) => event.resourceType === resourceType)
+ .reduce(
+ (total, event) =>
+ total +
+ Number(
+ fromRecordUsageInputToUsageConsumptionRow(event)
+ .creditsUsedMicro,
+ ),
+ 0,
+ )
+ : undefined,
+ });
}
-
- const allowance = exhaustedCounters.some(
- (counter) => counter.kind === 'allowance',
- )
- ? await this.creditAllowanceProvider.getCreditAllowance(args.workspaceId)
- : null;
-
- return exhaustedCounters.map((counter) =>
- buildQuotaExhaustedScope({
- resourceType: args.resourceType,
- counter,
- allowance,
- }),
- );
}
private admitOnFailure<TAdmitted>({
@@ -634,17 +548,6 @@ export class UsageLimitQuotaService implements OnModuleInit {
}
}
- private async buildCounters(args: QuotaConsumeArgs): Promise<QuotaCounter[]> {
- const [limitCounters, allowanceCounter] = await Promise.all([
- this.buildLimitCounters(args),
- this.buildAllowanceCounter(args.workspaceId),
- ]);
-
- return isDefined(allowanceCounter)
- ? [...limitCounters, allowanceCounter]
- : limitCounters;
- }
-
private async buildLimitCounters({
workspaceId,
resourceType,
@@ -709,6 +612,30 @@ export class UsageLimitQuotaService implements OnModuleInit {
);
}
+ private async buildAllowanceCounterAdmittingOnFailure(
+ workspaceId: string,
+ ): Promise<{
+ allowanceCounter: AllowanceQuotaCounter | null;
+ isAllowanceSkipped: boolean;
+ }> {
+ try {
+ return {
+ allowanceCounter: await this.buildAllowanceCounter(workspaceId),
+ isAllowanceSkipped: false,
+ };
+ } catch (error) {
+ if (error instanceof WorkspaceCacheException) {
+ throw error;
+ }
+
+ this.logger.error(
+ `Usage quota allowance skipped for workspace ${workspaceId}: ${error instanceof Error ? error.message : 'unknown error'}`,
+ );
+
+ return { allowanceCounter: null, isAllowanceSkipped: true };
+ }
+ }
+
private async buildAllowanceCounter(
workspaceId: string,
): Promise<AllowanceQuotaCounter | null> {
@@ -716,12 +643,10 @@ export class UsageLimitQuotaService implements OnModuleInit {
return null;
}
- const period = await this.usagePeriodService.findCurrentPeriod({
- workspaceId,
- periodUnit: 'allowancePeriod',
- });
+ const allowance =
+ await this.creditAllowanceProvider.getCreditAllowance(workspaceId);
- if (!isDefined(period)) {
+ if (!isDefined(allowance)) {
return null;
}
@@ -729,11 +654,16 @@ export class UsageLimitQuotaService implements OnModuleInit {
kind: 'allowance',
key: buildAllowanceCounterKey({
workspaceId,
- periodStart: period.periodStart,
+ periodStart: allowance.periodStart,
+ }),
+ limitValue: computeCreditAllowanceMicro({
+ planAllowanceMicro: allowance.planAllowanceMicro,
+ grants: allowance.grants,
+ nowMs: Date.now(),
}),
unit: UsageUnit.CREDIT,
- periodStart: period.periodStart,
- periodEnd: period.periodEnd,
+ periodStart: allowance.periodStart,
+ periodEnd: allowance.periodEnd,
};
}
@@ -784,207 +714,147 @@ export class UsageLimitQuotaService implements OnModuleInit {
workspaceId: string;
counters: LimitQuotaCounter[];
}): Promise<(number | null)[]> {
- const remainings = await this.cacheStorage.mget<number>(
+ const storedValues = await this.cacheStorage.mget<number>(
counters.map((counter) => counter.key),
);
- const coldLimitCounters = counters.filter(
- (_, index) => !isDefined(remainings[index]),
+ const coldCounters = counters.filter(
+ (_, index) => !isDefined(storedValues[index]),
);
- const rowsByPeriod =
- coldLimitCounters.length > 0
- ? await this.fetchConsumptionRowsByPeriod({
- workspaceId,
- coldLimitCounters,
- })
- : new Map<string, UsageConsumptionRow[]>();
-
- return counters.map((counter, index) => {
- const remaining = remainings[index];
-
- if (isDefined(remaining)) {
- return counter.limitValue - remaining;
- }
-
- const rows = rowsByPeriod.get(buildPeriodGroupKey(counter));
-
- return isDefined(rows)
- ? computeQuotaConsumed({ rows, scope: counter })
- : null;
+ const warmValues = await this.fetchWarmValues({
+ workspaceId,
+ counters: coldCounters,
});
- }
- private async readRemainings({
- workspaceId,
- counters,
- }: {
- workspaceId: string;
- counters: QuotaCounter[];
- }): Promise<(number | null)[]> {
- const remainings = await this.cacheStorage.mget<number>(
- counters.map((counter) => counter.key),
+ const warmedValueByKey = new Map(
+ coldCounters.map((counter, index) => [counter.key, warmValues[index]]),
);
- const warmRemainings = remainings.filter(isDefined);
-
- if (warmRemainings.length === remainings.length) {
- return warmRemainings;
- }
-
- return this.warmColdCounters({ workspaceId, counters });
+ return counters.map(
+ (counter, index) =>
+ storedValues[index] ?? warmedValueByKey.get(counter.key) ?? null,
+ );
}
- private async warmColdCounters({
+ private async consumeCounters({
workspaceId,
counters,
+ amounts,
}: {
workspaceId: string;
counters: QuotaCounter[];
- }): Promise<(number | null)[]> {
- return this.cacheLockService.withLock(
- async () => {
- const remainings = await this.cacheStorage.mget<number>(
- counters.map((counter) => counter.key),
- );
-
- const warmRemainings = remainings.filter(isDefined);
+ amounts: number[];
+ }): Promise<QuotaConsumeResult> {
+ const consumedValues = await this.runWarmAndConsumeScript({
+ counters,
+ entries: amounts.map((amount) => ({ amount, warm: null })),
+ });
- if (warmRemainings.length === remainings.length) {
- return warmRemainings;
- }
+ const now = Date.now();
- const coldCounters = counters.filter(
- (_, index) => !isDefined(remainings[index]),
- );
+ const coldIndexes = consumedValues.flatMap((consumedValue, index) =>
+ !isDefined(consumedValue) && counters[index].periodEnd.getTime() > now
+ ? [index]
+ : [],
+ );
- const warmedEntries = await this.buildWarmedEntries({
- workspaceId,
- coldCounters,
- });
+ if (coldIndexes.length === 0) {
+ return { consumedValues, isDegraded: false };
+ }
- await this.cacheStorage.mset(warmedEntries);
+ try {
+ const warmValues = await this.fetchWarmValues({
+ workspaceId,
+ counters: coldIndexes.map((index) => counters[index]),
+ });
- const warmedValueByKey = new Map(
- warmedEntries.map((entry) => [entry.key, entry.value]),
- );
+ const warmedAt = Date.now();
- return counters.map(
- (counter, index) =>
- remainings[index] ?? warmedValueByKey.get(counter.key) ?? null,
- );
- },
- buildQuotaWarmLockKey(workspaceId),
- QUOTA_WARM_LOCK_OPTIONS,
- );
- }
+ const warmedEntries = coldIndexes.flatMap((index, position) =>
+ counters[index].periodEnd.getTime() > warmedAt
+ ? [{ index, warmValue: warmValues[position] }]
+ : [],
+ );
- private async buildWarmedEntries({
- workspaceId,
- coldCounters,
- }: {
- workspaceId: string;
- coldCounters: QuotaCounter[];
- }): Promise<{ key: string; value: number; ttl: number }[]> {
- const coldLimitCounters = coldCounters.filter(
- (counter): counter is LimitQuotaCounter => counter.kind === 'limit',
- );
- const coldAllowanceCounter = coldCounters.find(
- (counter): counter is AllowanceQuotaCounter =>
- counter.kind === 'allowance',
- );
+ const warmedValues = await this.runWarmAndConsumeScript({
+ counters: warmedEntries.map(({ index }) => counters[index]),
+ entries: warmedEntries.map(({ index, warmValue }) => ({
+ amount: amounts[index],
+ warm: {
+ value: warmValue,
+ ttl: counters[index].periodEnd.getTime() - warmedAt,
+ },
+ })),
+ });
- const now = Date.now();
+ warmedEntries.forEach(({ index }, position) => {
+ consumedValues[index] = warmedValues[position];
+ });
- const [rowsByPeriod, allowanceEntries] = await Promise.all([
- this.fetchConsumptionRowsByPeriod({
- workspaceId,
- coldLimitCounters,
- }),
- this.buildAllowanceWarmedEntry({
- workspaceId,
- counter: coldAllowanceCounter,
- now,
- }),
- ]);
+ return { consumedValues, isDegraded: false };
+ } catch (error) {
+ this.logger.error(
+ `Usage quota warm degraded for workspace ${workspaceId}: ${error instanceof Error ? error.message : 'unknown error'}`,
+ );
- return [
- ...buildLimitWarmedEntries({ coldLimitCounters, rowsByPeriod, now }),
- ...allowanceEntries,
- ];
+ return { consumedValues, isDegraded: true };
+ }
}
- private async buildAllowanceWarmedEntry({
- workspaceId,
- counter,
- now,
+ private async runWarmAndConsumeScript({
+ counters,
+ entries,
}: {
- workspaceId: string;
- counter: AllowanceQuotaCounter | undefined;
- now: number;
- }): Promise<{ key: string; value: number; ttl: number }[]> {
- if (!isDefined(counter) || counter.periodEnd.getTime() <= now) {
- return [];
- }
-
- const allowance =
- await this.creditAllowanceProvider.getCreditAllowance(workspaceId);
-
- if (
- !isDefined(allowance) ||
- allowance.periodStart.getTime() !== counter.periodStart.getTime()
- ) {
- return [];
- }
-
- const ttl =
- Math.min(counter.periodEnd.getTime(), allowance.validUntil.getTime()) -
- now;
-
- if (ttl <= 0) {
+ counters: QuotaCounter[];
+ entries: Parameters<typeof buildConsumeScriptArguments>[0];
+ }): Promise<(number | null)[]> {
+ if (counters.length === 0) {
return [];
}
- const consumedMicro =
- await this.usageAnalyticsService.getCreditsUsedMicroForBillingPeriod({
- workspaceId,
- periodStart: counter.periodStart,
- });
+ const results = await this.cacheStorage.runScript<unknown[]>({
+ script: WARM_AND_CONSUME_QUOTA_COUNTERS_SCRIPT,
+ keys: counters.map((counter) => counter.key),
+ args: buildConsumeScriptArguments(entries),
+ });
- return [
- {
- key: counter.key,
- value: allowance.allowanceMicro - consumedMicro,
- ttl,
- },
- ];
+ return results.map((result) =>
+ typeof result === 'number' ? result : null,
+ );
}
- private async fetchConsumptionRowsByPeriod({
+ private async fetchWarmValues({
workspaceId,
- coldLimitCounters,
+ counters,
}: {
workspaceId: string;
- coldLimitCounters: LimitQuotaCounter[];
- }): Promise<Map<string, UsageConsumptionRow[]>> {
- const countersByPeriod = new Map<string, LimitQuotaCounter>();
+ counters: QuotaCounter[];
+ }): Promise<number[]> {
+ const rowsReadsByPeriodGroup = new Map<
+ string,
+ Promise<UsageConsumptionRow[]>
+ >();
+
+ return Promise.all(
+ counters.map(async (counter) => {
+ if (counter.kind === 'allowance') {
+ return this.usageAnalyticsService.getCreditsUsedMicroForBillingPeriod(
+ { workspaceId, periodStart: counter.periodStart },
+ );
+ }
- for (const counter of coldLimitCounters) {
- countersByPeriod.set(buildPeriodGroupKey(counter), counter);
- }
+ const periodGroupKey = buildPeriodGroupKey(counter);
+
+ const rowsRead =
+ rowsReadsByPeriodGroup.get(periodGroupKey) ??
+ this.fetchConsumptionRows({ workspaceId, counter });
- const rowsByPeriod = new Map<string, UsageConsumptionRow[]>();
+ rowsReadsByPeriodGroup.set(periodGroupKey, rowsRead);
- await Promise.all(
- [...countersByPeriod.entries()].map(async ([periodGroupKey, counter]) => {
- rowsByPeriod.set(
- periodGroupKey,
- await this.fetchConsumptionRows({ workspaceId, counter }),
- );
+ return computeQuotaConsumed({ rows: await rowsRead, scope: counter });
}),
);
-
- return rowsByPeriod;
}
private fetchConsumptionRows({
diff --git a/packages/twenty-server/src/engine/core-modules/usage-limit/services/usage-limit.service.ts b/packages/twenty-server/src/engine/core-modules/usage-limit/services/usage-limit.service.ts
index 0cbf35d8bf1..e299989e481 100644
--- a/packages/twenty-server/src/engine/core-modules/usage-limit/services/usage-limit.service.ts
+++ b/packages/twenty-server/src/engine/core-modules/usage-limit/services/usage-limit.service.ts
@@ -1,5 +1,6 @@
import { Injectable } from '@nestjs/common';
import { isNonEmptyString } from '@sniptt/guards';
+import isEqual from 'lodash.isequal';
import { isDefined } from 'twenty-shared/utils';
import { ApiKeyEntity } from 'src/engine/core-modules/api-key/api-key.entity';
@@ -170,9 +171,14 @@ export class UsageLimitService {
{ where: { id: usageLimit.id } },
);
- // counters are keyed by scope and value, so an edit leaves two of them behind
- await this.dropCounter(usageLimit);
- await this.dropCounter(updatedUsageLimit);
+ if (
+ isStockLimit(usageLimit) ||
+ usageLimit.isInstanceOverride !== isOperator ||
+ !isEqual(authorizedScope, scope)
+ ) {
+ await this.dropCounter(usageLimit);
+ await this.dropCounter(updatedUsageLimit);
+ }
return updatedUsageLimit;
}
@@ -305,7 +311,10 @@ export class UsageLimitService {
});
}
- return this.usageLimitQuotaService.dropLimitCounter(usageLimit);
+ return this.usageLimitQuotaService.dropLimitCounter({
+ workspaceId: usageLimit.workspaceId,
+ usageLimit,
+ });
}
private async validateSpenderBelongsToWorkspace({
diff --git a/packages/twenty-server/src/engine/core-modules/usage-limit/types/allowance-quota-counter.type.ts b/packages/twenty-server/src/engine/core-modules/usage-limit/types/allowance-quota-counter.type.ts
index e742a68364a..4de1bcf0b8c 100644
--- a/packages/twenty-server/src/engine/core-modules/usage-limit/types/allowance-quota-counter.type.ts
+++ b/packages/twenty-server/src/engine/core-modules/usage-limit/types/allowance-quota-counter.type.ts
@@ -3,6 +3,7 @@ import { type UsageUnit } from 'src/engine/core-modules/usage/enums/usage-unit.e
export type AllowanceQuotaCounter = {
kind: 'allowance';
key: string;
+ limitValue: number;
unit: UsageUnit.CREDIT;
periodStart: Date;
periodEnd: Date;
diff --git a/packages/twenty-server/src/engine/core-modules/usage-limit/types/credit-allowance-grant.type.ts b/packages/twenty-server/src/engine/core-modules/usage-limit/types/credit-allowance-grant.type.ts
new file mode 100644
index 00000000000..c37851b3555
--- /dev/null
+++ b/packages/twenty-server/src/engine/core-modules/usage-limit/types/credit-allowance-grant.type.ts
@@ -0,0 +1,5 @@
+export type CreditAllowanceGrant = {
+ amountMicro: number;
+ effectiveAtMs: number;
+ expiresAtMs: number | null;
+};
diff --git a/packages/twenty-server/src/engine/core-modules/usage-limit/types/credit-allowance.type.ts b/packages/twenty-server/src/engine/core-modules/usage-limit/types/credit-allowance.type.ts
index 6168f850977..b9ab06a88b6 100644
--- a/packages/twenty-server/src/engine/core-modules/usage-limit/types/credit-allowance.type.ts
+++ b/packages/twenty-server/src/engine/core-modules/usage-limit/types/credit-allowance.type.ts
@@ -1,8 +1,7 @@
+import { type CreditAllowanceGrant } from 'src/engine/core-modules/usage-limit/types/credit-allowance-grant.type';
import { type UsagePeriod } from 'src/engine/core-modules/usage-limit/types/usage-period.type';
export type CreditAllowance = UsagePeriod & {
- allowanceMicro: number;
- // When the figure stops being right before the period ends: a grant lapsing
- // inside the period lowers it with no ledger write to drop the counter.
- validUntil: Date;
+ planAllowanceMicro: number;
+ grants: CreditAllowanceGrant[];
};
diff --git a/packages/twenty-server/src/engine/core-modules/usage-limit/types/quota-default-key-scope.type.ts b/packages/twenty-server/src/engine/core-modules/usage-limit/types/quota-default-key-scope.type.ts
deleted file mode 100644
index 38ecbbac438..00000000000
--- a/packages/twenty-server/src/engine/core-modules/usage-limit/types/quota-default-key-scope.type.ts
+++ /dev/null
@@ -1,16 +0,0 @@
-import { type PeriodUnit } from 'src/engine/core-modules/usage-limit/types/period-unit.type';
-import { type SpenderType } from 'src/engine/core-modules/usage-limit/types/spender-type.type';
-import { type UsageOperationType } from 'src/engine/core-modules/usage/enums/usage-operation-type.enum';
-import { type UsageResourceType } from 'src/engine/core-modules/usage/enums/usage-resource-type.enum';
-import { type UsageUnit } from 'src/engine/core-modules/usage/enums/usage-unit.enum';
-
-export type QuotaDefaultKeyScope = {
- workspaceId: string;
- resourceType: UsageResourceType;
- operationType: UsageOperationType;
- spenderType: SpenderType;
- spenderId?: string | null;
- unit: UsageUnit;
- periodUnit: PeriodUnit;
- periodStart: Date;
-};
diff --git a/packages/twenty-server/src/engine/core-modules/usage-limit/types/quota-limit-default-definition.type.ts b/packages/twenty-server/src/engine/core-modules/usage-limit/types/quota-limit-default-definition.type.ts
index a856515d529..69f9cad064f 100644
--- a/packages/twenty-server/src/engine/core-modules/usage-limit/types/quota-limit-default-definition.type.ts
+++ b/packages/twenty-server/src/engine/core-modules/usage-limit/types/quota-limit-default-definition.type.ts
@@ -17,5 +17,5 @@ export type QuotaLimitDefaultDefinition<
periodUnit: AnchoredPeriodUnit;
periodCount: 1;
limitValueConfigVariable: NumericConfigVariableKey;
- isOverridable: boolean;
+ isOverridable: true;
};
diff --git a/packages/twenty-server/src/engine/core-modules/usage-limit/types/usage-limit-counter-scope.type.ts b/packages/twenty-server/src/engine/core-modules/usage-limit/types/usage-limit-counter-scope.type.ts
deleted file mode 100644
index d4398ddfce1..00000000000
--- a/packages/twenty-server/src/engine/core-modules/usage-limit/types/usage-limit-counter-scope.type.ts
+++ /dev/null
@@ -1,14 +0,0 @@
-import { type UsageLimitEntity } from 'src/engine/core-modules/usage-limit/usage-limit.entity';
-
-export type UsageLimitCounterScope = Pick<
- UsageLimitEntity,
- | 'workspaceId'
- | 'resourceType'
- | 'operationType'
- | 'spenderType'
- | 'spenderId'
- | 'limitKind'
- | 'periodUnit'
- | 'unit'
- | 'limitValue'
->;
diff --git a/packages/twenty-server/src/engine/core-modules/usage-limit/utils/__tests__/build-consume-script-arguments.util.spec.ts b/packages/twenty-server/src/engine/core-modules/usage-limit/utils/__tests__/build-consume-script-arguments.util.spec.ts
new file mode 100644
index 00000000000..2fb97e620cf
--- /dev/null
+++ b/packages/twenty-server/src/engine/core-modules/usage-limit/utils/__tests__/build-consume-script-arguments.util.spec.ts
@@ -0,0 +1,24 @@
+import { buildConsumeScriptArguments } from 'src/engine/core-modules/usage-limit/utils/build-consume-script-arguments.util';
+
+describe('buildConsumeScriptArguments', () => {
+ it.each([-1, 1.5, Number.MAX_SAFE_INTEGER + 1, Number.NaN])(
+ 'refuses the amount %s',
+ (amount) => {
+ expect(() =>
+ buildConsumeScriptArguments([{ amount, warm: null }]),
+ ).toThrow(/^Counter amount .* is not a non-negative safe integer$/);
+ },
+ );
+
+ it.each([
+ { value: -1, ttl: 60_000 },
+ { value: 0.5, ttl: 60_000 },
+ { value: 10, ttl: 0 },
+ { value: 10, ttl: -1 },
+ { value: 10, ttl: 1.5 },
+ ])('refuses the warm value $value with TTL $ttl', (warm) => {
+ expect(() => buildConsumeScriptArguments([{ amount: 1, warm }])).toThrow(
+ /^Counter warm value .* is not a non-negative safe integer with a positive safe TTL$/,
+ );
+ });
+});
diff --git a/packages/twenty-server/src/engine/core-modules/usage-limit/utils/__tests__/build-intra-workspace-limit-counter-keys.util.spec.ts b/packages/twenty-server/src/engine/core-modules/usage-limit/utils/__tests__/build-intra-workspace-limit-counter-keys.util.spec.ts
index a75bede69e8..56a2882ada4 100644
--- a/packages/twenty-server/src/engine/core-modules/usage-limit/utils/__tests__/build-intra-workspace-limit-counter-keys.util.spec.ts
+++ b/packages/twenty-server/src/engine/core-modules/usage-limit/utils/__tests__/build-intra-workspace-limit-counter-keys.util.spec.ts
@@ -47,18 +47,20 @@ describe('buildIntraWorkspaceLimitCounterKeys', () => {
periodByUnit: { month: MONTH_PERIOD, week: WEEK_PERIOD },
}),
).toEqual([
- `{workspace-1}:quota:AI:ALL:userWorkspace:-:CREDIT:month:${MONTH_PERIOD.periodStart.getTime()}:1000`,
- `{workspace-1}:quota:AI:ALL:agent:agent-1:CREDIT:week:${WEEK_PERIOD.periodStart.getTime()}:1000`,
+ `{workspace-1}:quota-consumed:AI:ALL:userWorkspace:-:CREDIT:month:${MONTH_PERIOD.periodStart.getTime()}`,
+ `{workspace-1}:quota-consumed:AI:ALL:agent:agent-1:CREDIT:week:${WEEK_PERIOD.periodStart.getTime()}`,
]);
});
- it('leaves workspace-scoped quotas and speed limits alone', () => {
+ it('leaves workspace-scoped quotas, operator overrides and speed limits alone', () => {
expect(
buildIntraWorkspaceLimitCounterKeys({
workspaceId: 'workspace-1',
limits: [
buildLimit({ spenderType: 'workspace' }),
+ buildLimit({ id: 'limit-2', isInstanceOverride: true }),
buildLimit({
+ id: 'limit-3',
limitKind: 'speed',
periodUnit: 'second',
unit: UsageUnit.TOKEN,
diff --git a/packages/twenty-server/src/engine/core-modules/usage-limit/utils/__tests__/build-limit-warmed-entries.util.spec.ts b/packages/twenty-server/src/engine/core-modules/usage-limit/utils/__tests__/build-limit-warmed-entries.util.spec.ts
deleted file mode 100644
index c8b44a6b8a9..00000000000
--- a/packages/twenty-server/src/engine/core-modules/usage-limit/utils/__tests__/build-limit-warmed-entries.util.spec.ts
+++ /dev/null
@@ -1,113 +0,0 @@
-import { type LimitQuotaCounter } from 'src/engine/core-modules/usage-limit/types/limit-quota-counter.type';
-import { type UsageConsumptionRow } from 'src/engine/core-modules/usage/types/usage-consumption-row.type';
-import { buildLimitWarmedEntries } from 'src/engine/core-modules/usage-limit/utils/build-limit-warmed-entries.util';
-import { buildPeriodGroupKey } from 'src/engine/core-modules/usage-limit/utils/build-period-group-key.util';
-import { UsageOperationType } from 'src/engine/core-modules/usage/enums/usage-operation-type.enum';
-import { UsageResourceType } from 'src/engine/core-modules/usage/enums/usage-resource-type.enum';
-import { UsageUnit } from 'src/engine/core-modules/usage/enums/usage-unit.enum';
-
-const NOW = new Date('2026-08-20T00:00:00.000Z').getTime();
-
-const buildCounter = (
- overrides: Partial<LimitQuotaCounter> = {},
-): LimitQuotaCounter => ({
- kind: 'limit',
- isDefault: false,
- key: 'counter-key',
- limitValue: 1_000,
- unit: UsageUnit.CREDIT,
- resourceType: UsageResourceType.AI,
- periodUnit: 'month',
- periodStart: new Date('2026-08-01T00:00:00.000Z'),
- periodEnd: new Date('2026-09-01T00:00:00.000Z'),
- spenderType: 'workspace',
- spenderId: null,
- operationType: UsageOperationType.AI_CHAT_TOKEN,
- ...overrides,
-});
-
-const buildRow = (
- overrides: Partial<UsageConsumptionRow> = {},
-): UsageConsumptionRow => ({
- operationType: UsageOperationType.AI_CHAT_TOKEN,
- unit: UsageUnit.TOKEN,
- userWorkspaceId: 'user-1',
- apiKeyId: '',
- applicationId: '',
- agentId: '',
- workflowId: '',
- logicFunctionId: '',
- creditsUsedMicro: '0',
- quantity: '0',
- ...overrides,
-});
-
-const buildRowsByPeriod = (
- counter: LimitQuotaCounter,
- rows: UsageConsumptionRow[],
-) => new Map([[buildPeriodGroupKey(counter), rows]]);
-
-describe('buildLimitWarmedEntries', () => {
- it('seeds the counter with its budget minus the stamped consumption', () => {
- const counter = buildCounter();
-
- expect(
- buildLimitWarmedEntries({
- coldLimitCounters: [counter],
- rowsByPeriod: buildRowsByPeriod(counter, [
- buildRow({ creditsUsedMicro: '150' }),
- buildRow({ userWorkspaceId: 'user-2', creditsUsedMicro: '100' }),
- ]),
- now: NOW,
- }),
- ).toEqual([
- {
- key: 'counter-key',
- value: 750,
- ttl: counter.periodEnd.getTime() - NOW,
- },
- ]);
- });
-
- it('counts only the rows the counter spender stamped', () => {
- const counter = buildCounter({
- spenderType: 'userWorkspace',
- spenderId: 'user-1',
- });
-
- expect(
- buildLimitWarmedEntries({
- coldLimitCounters: [counter],
- rowsByPeriod: buildRowsByPeriod(counter, [
- buildRow({ creditsUsedMicro: '150' }),
- buildRow({ userWorkspaceId: 'user-2', creditsUsedMicro: '100' }),
- ]),
- now: NOW,
- }),
- ).toMatchObject([{ value: 850 }]);
- });
-
- it('skips a counter whose period already ended', () => {
- const counter = buildCounter({
- periodEnd: new Date('2026-08-10T00:00:00.000Z'),
- });
-
- expect(
- buildLimitWarmedEntries({
- coldLimitCounters: [counter],
- rowsByPeriod: buildRowsByPeriod(counter, [buildRow({})]),
- now: NOW,
- }),
- ).toEqual([]);
- });
-
- it('skips a counter whose consumption was not fetched', () => {
- expect(
- buildLimitWarmedEntries({
- coldLimitCounters: [buildCounter()],
- rowsByPeriod: new Map(),
- now: NOW,
- }),
- ).toEqual([]);
- });
-});
diff --git a/packages/twenty-server/src/engine/core-modules/usage-limit/utils/__tests__/build-overridden-quota-default-counter-keys.util.spec.ts b/packages/twenty-server/src/engine/core-modules/usage-limit/utils/__tests__/build-overridden-quota-default-counter-keys.util.spec.ts
deleted file mode 100644
index d9e22763c31..00000000000
--- a/packages/twenty-server/src/engine/core-modules/usage-limit/utils/__tests__/build-overridden-quota-default-counter-keys.util.spec.ts
+++ /dev/null
@@ -1,77 +0,0 @@
-import { type PeriodUnit } from 'src/engine/core-modules/usage-limit/types/period-unit.type';
-import { type QuotaLimitDefault } from 'src/engine/core-modules/usage-limit/types/quota-limit-default.type';
-import { type UsageLimitCounterScope } from 'src/engine/core-modules/usage-limit/types/usage-limit-counter-scope.type';
-import { type UsagePeriod } from 'src/engine/core-modules/usage-limit/types/usage-period.type';
-import { buildOverriddenQuotaDefaultCounterKeys } from 'src/engine/core-modules/usage-limit/utils/build-overridden-quota-default-counter-keys.util';
-import { UsageOperationType } from 'src/engine/core-modules/usage/enums/usage-operation-type.enum';
-import { UsageResourceType } from 'src/engine/core-modules/usage/enums/usage-resource-type.enum';
-import { UsageUnit } from 'src/engine/core-modules/usage/enums/usage-unit.enum';
-
-const DAY_PERIOD = {
- periodStart: new Date('2026-08-20T00:00:00.000Z'),
- periodEnd: new Date('2026-08-21T00:00:00.000Z'),
-};
-
-const DEFAULT_COUNTER_KEY = `{workspace-1}:quota:EMAIL:EMAIL_SEND:workspace:-:INVOCATION:day:${DAY_PERIOD.periodStart.getTime()}:1000:default`;
-
-const buildUsageLimit = (
- overrides: Partial<UsageLimitCounterScope> = {},
-): UsageLimitCounterScope => ({
- workspaceId: 'workspace-1',
- resourceType: UsageResourceType.EMAIL,
- operationType: UsageOperationType.EMAIL_SEND,
- spenderType: 'workspace',
- spenderId: '',
- limitKind: 'quota',
- periodUnit: 'day',
- unit: UsageUnit.INVOCATION,
- limitValue: 500,
- ...overrides,
-});
-
-const buildDefault = (
- overrides: Partial<QuotaLimitDefault> = {},
-): QuotaLimitDefault => ({
- resourceType: UsageResourceType.EMAIL,
- operationType: UsageOperationType.EMAIL_SEND,
- limitKind: 'quota',
- spenderType: 'workspace',
- spenderId: '',
- unit: UsageUnit.INVOCATION,
- periodUnit: 'day',
- periodCount: 1,
- isOverridable: true,
- limitValue: 1_000,
- ...overrides,
-});
-
-const buildKeys = ({
- usageLimit = buildUsageLimit(),
- quotaLimitDefaults = [buildDefault()],
- periodByUnit = { day: DAY_PERIOD },
-}: {
- usageLimit?: UsageLimitCounterScope;
- quotaLimitDefaults?: QuotaLimitDefault[];
- periodByUnit?: Partial<Record<PeriodUnit, UsagePeriod>>;
-} = {}) =>
- buildOverriddenQuotaDefaultCounterKeys({
- usageLimit,
- quotaLimitDefaults,
- periodByUnit,
- });
-
-describe('buildOverriddenQuotaDefaultCounterKeys', () => {
- it('keys the default counter the row suppresses', () => {
- expect(buildKeys()).toEqual([DEFAULT_COUNTER_KEY]);
- });
-
- it('leaves alone a default the row does not suppress', () => {
- expect(
- buildKeys({ usageLimit: buildUsageLimit({ periodUnit: 'month' }) }),
- ).toEqual([]);
- });
-
- it('skips a default whose period was not resolved', () => {
- expect(buildKeys({ periodByUnit: {} })).toEqual([]);
- });
-});
diff --git a/packages/twenty-server/src/engine/core-modules/usage-limit/utils/__tests__/build-quota-counter-key.util.spec.ts b/packages/twenty-server/src/engine/core-modules/usage-limit/utils/__tests__/build-quota-counter-key.util.spec.ts
index 87423a66999..b6166a03fa8 100644
--- a/packages/twenty-server/src/engine/core-modules/usage-limit/utils/__tests__/build-quota-counter-key.util.spec.ts
+++ b/packages/twenty-server/src/engine/core-modules/usage-limit/utils/__tests__/build-quota-counter-key.util.spec.ts
@@ -6,26 +6,27 @@ import { UsageUnit } from 'src/engine/core-modules/usage/enums/usage-unit.enum';
const WORKSPACE_ID = 'workspace-1';
const PERIOD_START = new Date('2026-09-01T00:00:00.000Z');
-const buildKey = (
- overrides: Partial<Parameters<typeof buildQuotaCounterKey>[0]> = {},
-) =>
+type CounterIdentity = Parameters<typeof buildQuotaCounterKey>[0]['counter'];
+
+const buildKey = (overrides: Partial<CounterIdentity> = {}) =>
buildQuotaCounterKey({
workspaceId: WORKSPACE_ID,
- resourceType: UsageResourceType.AI,
- operationType: UsageOperationType.AI_CHAT_TOKEN,
- spenderType: 'workspace',
- spenderId: null,
- unit: UsageUnit.CREDIT,
- periodUnit: 'month',
- periodStart: PERIOD_START,
- limitValue: 1000,
- ...overrides,
+ counter: {
+ resourceType: UsageResourceType.AI,
+ operationType: UsageOperationType.AI_CHAT_TOKEN,
+ spenderType: 'workspace',
+ spenderId: null,
+ unit: UsageUnit.CREDIT,
+ periodUnit: 'month',
+ periodStart: PERIOD_START,
+ ...overrides,
+ },
});
describe('buildQuotaCounterKey', () => {
- it('hashes on the workspace so every counter of a workspace shares a slot', () => {
+ it('hashes on the workspace and keys a counter by its scope and period', () => {
expect(buildKey()).toBe(
- `{${WORKSPACE_ID}}:quota:AI:AI_CHAT_TOKEN:workspace:-:CREDIT:month:${PERIOD_START.getTime()}:1000`,
+ `{${WORKSPACE_ID}}:quota-consumed:AI:AI_CHAT_TOKEN:workspace:-:CREDIT:month:${PERIOD_START.getTime()}`,
);
});
@@ -35,11 +36,7 @@ describe('buildQuotaCounterKey', () => {
['spenderId', { spenderId: 'user-1' }],
['periodUnit', { periodUnit: 'week' as const }],
['periodStart', { periodStart: new Date('2026-10-01T00:00:00.000Z') }],
- ['limitValue', { limitValue: 2000 }],
- ])(
- 'gives two limits differing only by %s their own counter',
- (_, differs) => {
- expect(buildKey(differs)).not.toBe(buildKey());
- },
- );
+ ])('gives two counters differing only by %s their own key', (_, differs) => {
+ expect(buildKey(differs)).not.toBe(buildKey());
+ });
});
diff --git a/packages/twenty-server/src/engine/core-modules/usage-limit/utils/__tests__/build-quota-counters.util.spec.ts b/packages/twenty-server/src/engine/core-modules/usage-limit/utils/__tests__/build-quota-counters.util.spec.ts
index ba0c9397966..6bd24f291be 100644
--- a/packages/twenty-server/src/engine/core-modules/usage-limit/utils/__tests__/build-quota-counters.util.spec.ts
+++ b/packages/twenty-server/src/engine/core-modules/usage-limit/utils/__tests__/build-quota-counters.util.spec.ts
@@ -73,7 +73,7 @@ describe('buildQuotaCounters', () => {
{
kind: 'limit',
isDefault: false,
- key: `{workspace-1}:quota:AI:AI_CHAT_TOKEN:workspace:-:CREDIT:month:${MONTH_PERIOD.periodStart.getTime()}:1000000`,
+ key: `{workspace-1}:quota-consumed:AI:AI_CHAT_TOKEN:workspace:-:CREDIT:month:${MONTH_PERIOD.periodStart.getTime()}`,
limitValue: 1_000_000,
unit: UsageUnit.CREDIT,
resourceType: UsageResourceType.AI,
@@ -168,7 +168,7 @@ describe('buildQuotaCounters', () => {
{
kind: 'limit',
isDefault: true,
- key: `{workspace-1}:quota:AI:AI_CHAT_TOKEN:workspace:-:CREDIT:month:${MONTH_PERIOD.periodStart.getTime()}:5000:default`,
+ key: `{workspace-1}:quota-consumed:AI:AI_CHAT_TOKEN:workspace:-:CREDIT:month:${MONTH_PERIOD.periodStart.getTime()}`,
limitValue: 5_000,
unit: UsageUnit.CREDIT,
resourceType: UsageResourceType.AI,
@@ -183,15 +183,23 @@ describe('buildQuotaCounters', () => {
});
it.each([500, 50_000])(
- 'overrides the default with a workspace limit of %i',
+ 'overrides the default with a workspace limit of %i on the same counter',
(limitValue) => {
+ const [defaultCounter] = buildCounters({
+ quotaLimitDefaults: [buildDefault()],
+ });
+
const counters = buildCounters({
limits: [buildLimit({ limitValue })],
quotaLimitDefaults: [buildDefault()],
});
expect(counters).toEqual([
- expect.objectContaining({ isDefault: false, limitValue }),
+ expect.objectContaining({
+ isDefault: false,
+ limitValue,
+ key: defaultCounter.key,
+ }),
]);
},
);
diff --git a/packages/twenty-server/src/engine/core-modules/usage-limit/utils/__tests__/build-quota-debits.util.spec.ts b/packages/twenty-server/src/engine/core-modules/usage-limit/utils/__tests__/build-quota-debits.util.spec.ts
index 8f98c745890..0146373592a 100644
--- a/packages/twenty-server/src/engine/core-modules/usage-limit/utils/__tests__/build-quota-debits.util.spec.ts
+++ b/packages/twenty-server/src/engine/core-modules/usage-limit/utils/__tests__/build-quota-debits.util.spec.ts
@@ -12,6 +12,7 @@ const PERIOD_END = new Date('2026-09-01T00:00:00.000Z');
const allowanceCounter: AllowanceQuotaCounter = {
kind: 'allowance',
key: 'allowance',
+ limitValue: 1_000_000,
unit: UsageUnit.CREDIT,
periodStart: PERIOD_START,
periodEnd: PERIOD_END,
diff --git a/packages/twenty-server/src/engine/core-modules/usage-limit/utils/__tests__/build-quota-exhausted-scope.util.spec.ts b/packages/twenty-server/src/engine/core-modules/usage-limit/utils/__tests__/build-quota-exhausted-scope.util.spec.ts
index 031c50263f5..9c77200bf31 100644
--- a/packages/twenty-server/src/engine/core-modules/usage-limit/utils/__tests__/build-quota-exhausted-scope.util.spec.ts
+++ b/packages/twenty-server/src/engine/core-modules/usage-limit/utils/__tests__/build-quota-exhausted-scope.util.spec.ts
@@ -30,6 +30,7 @@ const buildLimitCounter = (
const allowanceCounter: AllowanceQuotaCounter = {
kind: 'allowance',
key: 'allowance-key',
+ limitValue: 2_000_000,
unit: UsageUnit.CREDIT,
periodStart: PERIOD_START,
periodEnd: PERIOD_END,
@@ -49,7 +50,6 @@ describe('buildQuotaExhaustedScope', () => {
buildQuotaExhaustedScope({
resourceType: UsageResourceType.AI,
counter: buildLimitCounter(),
- allowance: null,
}),
).toEqual({
resourceType: UsageResourceType.AI,
@@ -73,7 +73,6 @@ describe('buildQuotaExhaustedScope', () => {
buildQuotaExhaustedScope({
resourceType: UsageResourceType.AI,
counter: buildLimitCounter({ isDefault: true }),
- allowance: null,
}),
).toMatchObject({ exhaustedKind: 'limit', isDefault: true });
});
@@ -83,12 +82,6 @@ describe('buildQuotaExhaustedScope', () => {
buildQuotaExhaustedScope({
resourceType: UsageResourceType.AI,
counter: allowanceCounter,
- allowance: {
- periodStart: PERIOD_START,
- periodEnd: PERIOD_END,
- allowanceMicro: 2_000_000,
- validUntil: PERIOD_END,
- },
}),
).toEqual({
resourceType: UsageResourceType.AI,
@@ -106,16 +99,6 @@ describe('buildQuotaExhaustedScope', () => {
});
});
- it('reports a zero allowance when it is gone since the counter was read', () => {
- expect(
- buildQuotaExhaustedScope({
- resourceType: UsageResourceType.AI,
- counter: allowanceCounter,
- allowance: null,
- }),
- ).toMatchObject({ exhaustedKind: 'allowance', limitValue: 0 });
- });
-
it('clamps the retry delay to zero once the period has ended', () => {
expect(
buildQuotaExhaustedScope({
@@ -123,7 +106,6 @@ describe('buildQuotaExhaustedScope', () => {
counter: buildLimitCounter({
periodEnd: new Date('2026-08-10T00:00:00.000Z'),
}),
- allowance: null,
}),
).toMatchObject({ retryAfterMs: 0 });
});
diff --git a/packages/twenty-server/src/engine/core-modules/usage-limit/utils/__tests__/compute-credit-allowance-micro.util.spec.ts b/packages/twenty-server/src/engine/core-modules/usage-limit/utils/__tests__/compute-credit-allowance-micro.util.spec.ts
new file mode 100644
index 00000000000..7ce4f6a5751
--- /dev/null
+++ b/packages/twenty-server/src/engine/core-modules/usage-limit/utils/__tests__/compute-credit-allowance-micro.util.spec.ts
@@ -0,0 +1,59 @@
+import { type CreditAllowanceGrant } from 'src/engine/core-modules/usage-limit/types/credit-allowance-grant.type';
+import { computeCreditAllowanceMicro } from 'src/engine/core-modules/usage-limit/utils/compute-credit-allowance-micro.util';
+
+const NOW_MS = new Date('2026-08-15T00:00:00.000Z').getTime();
+const DAY_MS = 24 * 60 * 60 * 1000;
+
+const buildGrant = (
+ overrides: Partial<CreditAllowanceGrant>,
+): CreditAllowanceGrant => ({
+ amountMicro: 500,
+ effectiveAtMs: NOW_MS - DAY_MS,
+ expiresAtMs: null,
+ ...overrides,
+});
+
+describe('computeCreditAllowanceMicro', () => {
+ it('adds every grant in effect', () => {
+ expect(
+ computeCreditAllowanceMicro({
+ planAllowanceMicro: 1_000,
+ grants: [
+ buildGrant({ amountMicro: 500 }),
+ buildGrant({ amountMicro: 300, expiresAtMs: NOW_MS + DAY_MS }),
+ ],
+ nowMs: NOW_MS,
+ }),
+ ).toBe(1_800);
+ });
+
+ it('leaves out a grant that is not effective yet', () => {
+ expect(
+ computeCreditAllowanceMicro({
+ planAllowanceMicro: 1_000,
+ grants: [buildGrant({ effectiveAtMs: NOW_MS + 1 })],
+ nowMs: NOW_MS,
+ }),
+ ).toBe(1_000);
+ });
+
+ it('counts a grant from the instant it becomes effective', () => {
+ expect(
+ computeCreditAllowanceMicro({
+ planAllowanceMicro: 1_000,
+ grants: [buildGrant({ effectiveAtMs: NOW_MS })],
+ nowMs: NOW_MS,
+ }),
+ ).toBe(1_500);
+ });
+
+ it('leaves out a grant from the instant it expires', () => {
+ expect(
+ computeCreditAllowanceMicro({
+ planAllowanceMicro: 1_000,
+ grants: [buildGrant({ expiresAtMs: NOW_MS })],
+ nowMs: NOW_MS,
+ }),
+ ).toBe(1_000);
+ });
+});
diff --git a/packages/twenty-server/src/engine/core-modules/usage-limit/utils/__tests__/find-exhausted-counters.util.spec.ts b/packages/twenty-server/src/engine/core-modules/usage-limit/utils/__tests__/find-exhausted-counters.util.spec.ts
index 0529e5a7cd1..d82389729ca 100644
--- a/packages/twenty-server/src/engine/core-modules/usage-limit/utils/__tests__/find-exhausted-counters.util.spec.ts
+++ b/packages/twenty-server/src/engine/core-modules/usage-limit/utils/__tests__/find-exhausted-counters.util.spec.ts
@@ -11,6 +11,7 @@ const PERIOD_END = new Date('2026-09-01T00:00:00.000Z');
const buildAllowanceCounter = (key: string): AllowanceQuotaCounter => ({
kind: 'allowance',
key,
+ limitValue: 5_000,
unit: UsageUnit.CREDIT,
periodStart: PERIOD_START,
periodEnd: PERIOD_END,
@@ -36,77 +37,24 @@ const buildLimitCounter = (
});
describe('findExhaustedCounters', () => {
- it('answers empty when every counter has budget left', () => {
+ it('picks every counter at or past its cap', () => {
expect(
findExhaustedCounters({
counters: [
- buildAllowanceCounter('first'),
- buildAllowanceCounter('second'),
+ buildLimitCounter('under'),
+ buildLimitCounter('at'),
+ buildAllowanceCounter('past'),
],
- remainings: [250, 1],
+ consumedValues: [999, 1_000, 6_000],
}),
- ).toEqual([]);
- });
-
- it('picks every counter whose budget is gone', () => {
- expect(
- findExhaustedCounters({
- counters: [
- buildAllowanceCounter('first'),
- buildAllowanceCounter('second'),
- buildAllowanceCounter('third'),
- ],
- remainings: [0, 100, 0],
- }),
- ).toMatchObject([{ key: 'first' }, { key: 'third' }]);
- });
-
- it('counts an overdrawn counter as exhausted', () => {
- expect(
- findExhaustedCounters({
- counters: [buildAllowanceCounter('first')],
- remainings: [-10],
- }),
- ).toMatchObject([{ key: 'first' }]);
+ ).toMatchObject([{ key: 'at' }, { key: 'past' }]);
});
it('skips cold counters', () => {
expect(
findExhaustedCounters({
- counters: [
- buildAllowanceCounter('first'),
- buildAllowanceCounter('second'),
- ],
- remainings: [null, 0],
- }),
- ).toMatchObject([{ key: 'second' }]);
- });
-
- it('answers empty when every counter is cold', () => {
- expect(
- findExhaustedCounters({
- counters: [buildAllowanceCounter('first')],
- remainings: [null],
- }),
- ).toEqual([]);
- });
-
- it('refuses a limit that cannot cover the named cost', () => {
- expect(
- findExhaustedCounters({
- counters: [buildLimitCounter('first')],
- remainings: [100],
- cost: { [UsageUnit.CREDIT]: 101, [UsageUnit.TOKEN]: 0 },
- }),
- ).toMatchObject([{ key: 'first' }]);
- });
-
- it('admits a cost that fits the remaining exactly', () => {
- expect(
- findExhaustedCounters({
- counters: [buildLimitCounter('first')],
- remainings: [100],
- cost: { [UsageUnit.CREDIT]: 100, [UsageUnit.TOKEN]: 0 },
+ counters: [buildLimitCounter('cold'), buildAllowanceCounter('cold')],
+ consumedValues: [null, null],
}),
).toEqual([]);
});
@@ -114,60 +62,23 @@ describe('findExhaustedCounters', () => {
it('charges a limit only for the unit it counts', () => {
expect(
findExhaustedCounters({
- counters: [buildLimitCounter('first')],
- remainings: [10],
+ counters: [
+ buildLimitCounter('credits'),
+ buildLimitCounter('tokens', { unit: UsageUnit.TOKEN }),
+ ],
+ consumedValues: [990, 990],
cost: { [UsageUnit.CREDIT]: 0, [UsageUnit.TOKEN]: 500 },
}),
- ).toEqual([]);
+ ).toMatchObject([{ key: 'tokens' }]);
});
it('counts a unit missing from the cost as zero', () => {
expect(
findExhaustedCounters({
- counters: [buildLimitCounter('first', { unit: UsageUnit.TOKEN })],
- remainings: [10],
+ counters: [buildLimitCounter('tokens', { unit: UsageUnit.TOKEN })],
+ consumedValues: [990],
cost: { [UsageUnit.CREDIT]: 5_000 },
}),
).toEqual([]);
});
-
- it('still refuses an emptied limit whatever the cost', () => {
- expect(
- findExhaustedCounters({
- counters: [buildLimitCounter('first')],
- remainings: [0],
- cost: { [UsageUnit.CREDIT]: 0, [UsageUnit.TOKEN]: 0 },
- }),
- ).toMatchObject([{ key: 'first' }]);
- });
-
- it('refuses a credit allowance that cannot cover the whole cost', () => {
- expect(
- findExhaustedCounters({
- counters: [buildAllowanceCounter('allowance')],
- remainings: [10],
- cost: { [UsageUnit.CREDIT]: 5_000, [UsageUnit.TOKEN]: 100 },
- }),
- ).toMatchObject([{ key: 'allowance' }]);
- });
-
- it('admits a credit allowance holding exactly the whole cost', () => {
- expect(
- findExhaustedCounters({
- counters: [buildAllowanceCounter('allowance')],
- remainings: [5_000],
- cost: { [UsageUnit.CREDIT]: 5_000, [UsageUnit.TOKEN]: 100 },
- }),
- ).toEqual([]);
- });
-
- it('still refuses an emptied credit allowance under a named cost', () => {
- expect(
- findExhaustedCounters({
- counters: [buildAllowanceCounter('allowance')],
- remainings: [0],
- cost: { [UsageUnit.CREDIT]: 5_000, [UsageUnit.TOKEN]: 100 },
- }),
- ).toMatchObject([{ key: 'allowance' }]);
- });
});
diff --git a/packages/twenty-server/src/engine/core-modules/usage-limit/utils/__tests__/from-consume-results-to-remainings.util.spec.ts b/packages/twenty-server/src/engine/core-modules/usage-limit/utils/__tests__/from-consume-results-to-remainings.util.spec.ts
deleted file mode 100644
index 6bf2cc1a552..00000000000
--- a/packages/twenty-server/src/engine/core-modules/usage-limit/utils/__tests__/from-consume-results-to-remainings.util.spec.ts
+++ /dev/null
@@ -1,17 +0,0 @@
-import { fromConsumeResultsToRemainings } from 'src/engine/core-modules/usage-limit/utils/from-consume-results-to-remainings.util';
-
-describe('fromConsumeResultsToRemainings', () => {
- it('reads one remaining per debited counter', () => {
- expect(fromConsumeResultsToRemainings([1, 500, 1, -10])).toEqual([
- 500, -10,
- ]);
- });
-
- it('leaves a counter the script did not find cold', () => {
- expect(fromConsumeResultsToRemainings([1, 500, 0, 0])).toEqual([500, null]);
- });
-
- it('answers nothing for no results', () => {
- expect(fromConsumeResultsToRemainings([])).toEqual([]);
- });
-});
diff --git a/packages/twenty-server/src/engine/core-modules/usage-limit/utils/__tests__/is-limit-exhausted.util.spec.ts b/packages/twenty-server/src/engine/core-modules/usage-limit/utils/__tests__/is-limit-exhausted.util.spec.ts
new file mode 100644
index 00000000000..626d5cfe7c8
--- /dev/null
+++ b/packages/twenty-server/src/engine/core-modules/usage-limit/utils/__tests__/is-limit-exhausted.util.spec.ts
@@ -0,0 +1,21 @@
+import { isLimitExhausted } from 'src/engine/core-modules/usage-limit/utils/is-limit-exhausted.util';
+
+describe('isLimitExhausted', () => {
+ it('admits a cost that fits the cap exactly', () => {
+ expect(
+ isLimitExhausted({ consumed: 900, cost: 100, limitValue: 1_000 }),
+ ).toBe(false);
+ });
+
+ it('refuses a cost one unit over the cap', () => {
+ expect(
+ isLimitExhausted({ consumed: 900, cost: 101, limitValue: 1_000 }),
+ ).toBe(true);
+ });
+
+ it('refuses at the cap even with no cost', () => {
+ expect(
+ isLimitExhausted({ consumed: 1_000, cost: 0, limitValue: 1_000 }),
+ ).toBe(true);
+ });
+});
diff --git a/packages/twenty-server/src/engine/core-modules/usage-limit/utils/build-allowance-counter-key.util.ts b/packages/twenty-server/src/engine/core-modules/usage-limit/utils/build-allowance-counter-key.util.ts
index 29d8dcda3cb..80c27c4dc43 100644
--- a/packages/twenty-server/src/engine/core-modules/usage-limit/utils/build-allowance-counter-key.util.ts
+++ b/packages/twenty-server/src/engine/core-modules/usage-limit/utils/build-allowance-counter-key.util.ts
@@ -4,4 +4,5 @@ export const buildAllowanceCounterKey = ({
}: {
workspaceId: string;
periodStart: Date;
-}): string => `{${workspaceId}}:quota:allowance:${periodStart.getTime()}`;
+}): string =>
+ `{${workspaceId}}:quota-consumed:allowance:${periodStart.getTime()}`;
diff --git a/packages/twenty-server/src/engine/core-modules/usage-limit/utils/build-consume-script-arguments.util.ts b/packages/twenty-server/src/engine/core-modules/usage-limit/utils/build-consume-script-arguments.util.ts
new file mode 100644
index 00000000000..9e6da1a4ffe
--- /dev/null
+++ b/packages/twenty-server/src/engine/core-modules/usage-limit/utils/build-consume-script-arguments.util.ts
@@ -0,0 +1,33 @@
+import { isDefined } from 'twenty-shared/utils';
+
+export const buildConsumeScriptArguments = (
+ entries: { amount: number; warm: { value: number; ttl: number } | null }[],
+): string[] => {
+ for (const { amount, warm } of entries) {
+ if (!(Number.isSafeInteger(amount) && amount >= 0)) {
+ throw new Error(
+ `Counter amount ${amount} is not a non-negative safe integer`,
+ );
+ }
+
+ if (
+ isDefined(warm) &&
+ !(
+ Number.isSafeInteger(warm.value) &&
+ warm.value >= 0 &&
+ Number.isSafeInteger(warm.ttl) &&
+ warm.ttl > 0
+ )
+ ) {
+ throw new Error(
+ `Counter warm value ${warm.value} with TTL ${warm.ttl} is not a non-negative safe integer with a positive safe TTL`,
+ );
+ }
+ }
+
+ return [
+ JSON.stringify(entries.map(({ amount }) => amount)),
+ JSON.stringify(entries.map(({ warm }) => warm?.value ?? false)),
+ JSON.stringify(entries.map(({ warm }) => warm?.ttl ?? false)),
+ ];
+};
diff --git a/packages/twenty-server/src/engine/core-modules/usage-limit/utils/build-intra-workspace-limit-counter-keys.util.ts b/packages/twenty-server/src/engine/core-modules/usage-limit/utils/build-intra-workspace-limit-counter-keys.util.ts
index 6d37f7729f7..96dbe1e5053 100644
--- a/packages/twenty-server/src/engine/core-modules/usage-limit/utils/build-intra-workspace-limit-counter-keys.util.ts
+++ b/packages/twenty-server/src/engine/core-modules/usage-limit/utils/build-intra-workspace-limit-counter-keys.util.ts
@@ -3,8 +3,9 @@ import { isDefined } from 'twenty-shared/utils';
import { type FlatUsageLimit } from 'src/engine/core-modules/usage-limit/types/flat-usage-limit.type';
import { type PeriodUnit } from 'src/engine/core-modules/usage-limit/types/period-unit.type';
import { type UsagePeriod } from 'src/engine/core-modules/usage-limit/types/usage-period.type';
-import { buildQuotaCounterKey } from 'src/engine/core-modules/usage-limit/utils/build-quota-counter-key.util';
-import { isIntraWorkspaceScoped } from 'src/engine/core-modules/usage-limit/utils/is-intra-workspace-scoped.util';
+import { buildLimitQuotaCounter } from 'src/engine/core-modules/usage-limit/utils/build-limit-quota-counter.util';
+import { findEnforceableLimits } from 'src/engine/core-modules/usage-limit/utils/find-enforceable-limits.util';
+import { isQuotaLimit } from 'src/engine/core-modules/usage-limit/utils/is-quota-limit.util';
export const buildIntraWorkspaceLimitCounterKeys = ({
workspaceId,
@@ -14,31 +15,23 @@ export const buildIntraWorkspaceLimitCounterKeys = ({
workspaceId: string;
limits: FlatUsageLimit[];
periodByUnit: Partial<Record<PeriodUnit, UsagePeriod>>;
-}): string[] =>
- limits
- .filter(
- (limit) =>
- limit.limitKind === 'quota' &&
- isIntraWorkspaceScoped(limit.spenderType),
- )
+}): string[] => {
+ const quotaLimits = limits.filter(isQuotaLimit);
+
+ const limitIdsEnforcedWithoutEntitlement = new Set(
+ findEnforceableLimits({
+ limits: quotaLimits,
+ isIntraWorkspaceLimitEntitled: false,
+ }).map((limit) => limit.id),
+ );
+
+ return quotaLimits
+ .filter((limit) => !limitIdsEnforcedWithoutEntitlement.has(limit.id))
.flatMap((limit) => {
const period = periodByUnit[limit.periodUnit];
- if (!isDefined(period)) {
- return [];
- }
-
- return [
- buildQuotaCounterKey({
- workspaceId,
- resourceType: limit.resourceType,
- operationType: limit.operationType,
- spenderType: limit.spenderType,
- spenderId: limit.spenderId,
- unit: limit.unit,
- periodUnit: limit.periodUnit,
- periodStart: period.periodStart,
- limitValue: limit.limitValue,
- }),
- ];
+ return isDefined(period)
+ ? [buildLimitQuotaCounter({ workspaceId, limit, period }).key]
+ : [];
});
+};
diff --git a/packages/twenty-server/src/engine/core-modules/usage-limit/utils/build-limit-quota-counter.util.ts b/packages/twenty-server/src/engine/core-modules/usage-limit/utils/build-limit-quota-counter.util.ts
index 288c84d6c4c..17eb7e0e49a 100644
--- a/packages/twenty-server/src/engine/core-modules/usage-limit/utils/build-limit-quota-counter.util.ts
+++ b/packages/twenty-server/src/engine/core-modules/usage-limit/utils/build-limit-quota-counter.util.ts
@@ -10,32 +10,22 @@ export const buildLimitQuotaCounter = ({
period,
}: {
workspaceId: string;
- limit: Omit<
- FlatQuotaLimit,
- 'id' | 'limitKind' | 'periodCount' | 'burstValue'
- >;
+ limit: Omit<FlatQuotaLimit, 'limitKind' | 'periodCount' | 'burstValue'>;
period: UsagePeriod;
-}): LimitQuotaCounter => ({
- kind: 'limit',
- isDefault: false,
- key: buildQuotaCounterKey({
- workspaceId,
- resourceType: limit.resourceType,
- operationType: limit.operationType,
- spenderType: limit.spenderType,
- spenderId: limit.spenderId,
+}): LimitQuotaCounter => {
+ const counter = {
+ kind: 'limit' as const,
+ isDefault: false,
+ limitValue: limit.limitValue,
unit: limit.unit,
+ resourceType: limit.resourceType,
periodUnit: limit.periodUnit,
periodStart: period.periodStart,
- limitValue: limit.limitValue,
- }),
- limitValue: limit.limitValue,
- unit: limit.unit,
- resourceType: limit.resourceType,
- periodUnit: limit.periodUnit,
- periodStart: period.periodStart,
- periodEnd: period.periodEnd,
- spenderType: limit.spenderType,
- spenderId: normalizeSpenderId(limit.spenderId),
- operationType: limit.operationType,
-});
+ periodEnd: period.periodEnd,
+ spenderType: limit.spenderType,
+ spenderId: normalizeSpenderId(limit.spenderId),
+ operationType: limit.operationType,
+ };
+
+ return { ...counter, key: buildQuotaCounterKey({ workspaceId, counter }) };
+};
diff --git a/packages/twenty-server/src/engine/core-modules/usage-limit/utils/build-limit-warmed-entries.util.ts b/packages/twenty-server/src/engine/core-modules/usage-limit/utils/build-limit-warmed-entries.util.ts
deleted file mode 100644
index 14e13f82bc1..00000000000
--- a/packages/twenty-server/src/engine/core-modules/usage-limit/utils/build-limit-warmed-entries.util.ts
+++ /dev/null
@@ -1,33 +0,0 @@
-import { isDefined } from 'twenty-shared/utils';
-
-import { type LimitQuotaCounter } from 'src/engine/core-modules/usage-limit/types/limit-quota-counter.type';
-import { type UsageConsumptionRow } from 'src/engine/core-modules/usage/types/usage-consumption-row.type';
-import { buildPeriodGroupKey } from 'src/engine/core-modules/usage-limit/utils/build-period-group-key.util';
-import { computeQuotaConsumed } from 'src/engine/core-modules/usage-limit/utils/compute-quota-consumed.util';
-
-export const buildLimitWarmedEntries = ({
- coldLimitCounters,
- rowsByPeriod,
- now,
-}: {
- coldLimitCounters: LimitQuotaCounter[];
- rowsByPeriod: Map<string, UsageConsumptionRow[]>;
- now: number;
-}): { key: string; value: number; ttl: number }[] =>
- coldLimitCounters.flatMap((counter) => {
- const ttl = counter.periodEnd.getTime() - now;
- const rows = rowsByPeriod.get(buildPeriodGroupKey(counter));
-
- if (ttl <= 0 || !isDefined(rows)) {
- return [];
- }
-
- return [
- {
- key: counter.key,
- value:
- counter.limitValue - computeQuotaConsumed({ rows, scope: counter }),
- ttl,
- },
- ];
- });
diff --git a/packages/twenty-server/src/engine/core-modules/usage-limit/utils/build-overridden-quota-default-counter-keys.util.ts b/packages/twenty-server/src/engine/core-modules/usage-limit/utils/build-overridden-quota-default-counter-keys.util.ts
deleted file mode 100644
index 3da622d6d41..00000000000
--- a/packages/twenty-server/src/engine/core-modules/usage-limit/utils/build-overridden-quota-default-counter-keys.util.ts
+++ /dev/null
@@ -1,46 +0,0 @@
-import { isDefined } from 'twenty-shared/utils';
-
-import { type PeriodUnit } from 'src/engine/core-modules/usage-limit/types/period-unit.type';
-import { type QuotaLimitDefault } from 'src/engine/core-modules/usage-limit/types/quota-limit-default.type';
-import { type UsageLimitCounterScope } from 'src/engine/core-modules/usage-limit/types/usage-limit-counter-scope.type';
-import { type UsagePeriod } from 'src/engine/core-modules/usage-limit/types/usage-period.type';
-import { buildQuotaDefaultCounterKey } from 'src/engine/core-modules/usage-limit/utils/build-quota-default-counter-key.util';
-import { doesUsageLimitRowSuppressDefault } from 'src/engine/core-modules/usage-limit/utils/does-usage-limit-row-suppress-default.util';
-
-export const buildOverriddenQuotaDefaultCounterKeys = ({
- usageLimit,
- quotaLimitDefaults,
- periodByUnit,
-}: {
- usageLimit: UsageLimitCounterScope;
- quotaLimitDefaults: QuotaLimitDefault[];
- periodByUnit: Partial<Record<PeriodUnit, UsagePeriod>>;
-}): string[] =>
- quotaLimitDefaults
- .filter((quotaLimitDefault) =>
- doesUsageLimitRowSuppressDefault({
- scope: usageLimit,
- usageLimitDefault: quotaLimitDefault,
- }),
- )
- .flatMap((quotaLimitDefault) => {
- const period = periodByUnit[quotaLimitDefault.periodUnit];
-
- if (!isDefined(period)) {
- return [];
- }
-
- return [
- buildQuotaDefaultCounterKey({
- workspaceId: usageLimit.workspaceId,
- resourceType: quotaLimitDefault.resourceType,
- operationType: quotaLimitDefault.operationType,
- spenderType: quotaLimitDefault.spenderType,
- spenderId: quotaLimitDefault.spenderId,
- unit: quotaLimitDefault.unit,
- periodUnit: quotaLimitDefault.periodUnit,
- periodStart: period.periodStart,
- limitValue: quotaLimitDefault.limitValue,
- }),
- ];
- });
diff --git a/packages/twenty-server/src/engine/core-modules/usage-limit/utils/build-quota-counter-key.util.ts b/packages/twenty-server/src/engine/core-modules/usage-limit/utils/build-quota-counter-key.util.ts
index f1ffc408845..88e19f3da8c 100644
--- a/packages/twenty-server/src/engine/core-modules/usage-limit/utils/build-quota-counter-key.util.ts
+++ b/packages/twenty-server/src/engine/core-modules/usage-limit/utils/build-quota-counter-key.util.ts
@@ -1,32 +1,23 @@
import { isNonEmptyString } from '@sniptt/guards';
-import { type PeriodUnit } from 'src/engine/core-modules/usage-limit/types/period-unit.type';
-import { type SpenderType } from 'src/engine/core-modules/usage-limit/types/spender-type.type';
-import { type UsageOperationType } from 'src/engine/core-modules/usage/enums/usage-operation-type.enum';
-import { type UsageResourceType } from 'src/engine/core-modules/usage/enums/usage-resource-type.enum';
-import { type UsageUnit } from 'src/engine/core-modules/usage/enums/usage-unit.enum';
+import { type LimitQuotaCounter } from 'src/engine/core-modules/usage-limit/types/limit-quota-counter.type';
const ABSENT = '-';
export const buildQuotaCounterKey = ({
workspaceId,
- resourceType,
- operationType,
- spenderType,
- spenderId,
- unit,
- periodUnit,
- periodStart,
- limitValue,
+ counter,
}: {
workspaceId: string;
- resourceType: UsageResourceType;
- operationType: UsageOperationType;
- spenderType: SpenderType;
- spenderId?: string | null;
- unit: UsageUnit;
- periodUnit: PeriodUnit;
- periodStart: Date;
- limitValue: number;
+ counter: Pick<
+ LimitQuotaCounter,
+ | 'resourceType'
+ | 'operationType'
+ | 'spenderType'
+ | 'spenderId'
+ | 'unit'
+ | 'periodUnit'
+ | 'periodStart'
+ >;
}): string =>
- `{${workspaceId}}:quota:${resourceType}:${operationType}:${spenderType}:${isNonEmptyString(spenderId) ? spenderId : ABSENT}:${unit}:${periodUnit}:${periodStart.getTime()}:${limitValue}`;
+ `{${workspaceId}}:quota-consumed:${counter.resourceType}:${counter.operationType}:${counter.spenderType}:${isNonEmptyString(counter.spenderId) ? counter.spenderId : ABSENT}:${counter.unit}:${counter.periodUnit}:${counter.periodStart.getTime()}`;
diff --git a/packages/twenty-server/src/engine/core-modules/usage-limit/utils/build-quota-default-counter-key.util.ts b/packages/twenty-server/src/engine/core-modules/usage-limit/utils/build-quota-default-counter-key.util.ts
deleted file mode 100644
index bb961bcaccf..00000000000
--- a/packages/twenty-server/src/engine/core-modules/usage-limit/utils/build-quota-default-counter-key.util.ts
+++ /dev/null
@@ -1,6 +0,0 @@
-import { type QuotaDefaultKeyScope } from 'src/engine/core-modules/usage-limit/types/quota-default-key-scope.type';
-import { buildQuotaCounterKey } from 'src/engine/core-modules/usage-limit/utils/build-quota-counter-key.util';
-
-export const buildQuotaDefaultCounterKey = (
- scope: QuotaDefaultKeyScope & { limitValue: number },
-): string => `${buildQuotaCounterKey(scope)}:default`;
diff --git a/packages/twenty-server/src/engine/core-modules/usage-limit/utils/build-quota-default-counter.util.ts b/packages/twenty-server/src/engine/core-modules/usage-limit/utils/build-quota-default-counter.util.ts
index 2ac730948b0..03089989af3 100644
--- a/packages/twenty-server/src/engine/core-modules/usage-limit/utils/build-quota-default-counter.util.ts
+++ b/packages/twenty-server/src/engine/core-modules/usage-limit/utils/build-quota-default-counter.util.ts
@@ -1,7 +1,7 @@
import { type LimitQuotaCounter } from 'src/engine/core-modules/usage-limit/types/limit-quota-counter.type';
import { type QuotaLimitDefault } from 'src/engine/core-modules/usage-limit/types/quota-limit-default.type';
import { type UsagePeriod } from 'src/engine/core-modules/usage-limit/types/usage-period.type';
-import { buildQuotaDefaultCounterKey } from 'src/engine/core-modules/usage-limit/utils/build-quota-default-counter-key.util';
+import { buildQuotaCounterKey } from 'src/engine/core-modules/usage-limit/utils/build-quota-counter-key.util';
export const buildQuotaDefaultCounter = ({
workspaceId,
@@ -11,27 +11,20 @@ export const buildQuotaDefaultCounter = ({
workspaceId: string;
quotaLimitDefault: QuotaLimitDefault;
period: UsagePeriod;
-}): LimitQuotaCounter => ({
- kind: 'limit',
- isDefault: true,
- key: buildQuotaDefaultCounterKey({
- workspaceId,
- resourceType: quotaLimitDefault.resourceType,
- operationType: quotaLimitDefault.operationType,
- spenderType: quotaLimitDefault.spenderType,
- spenderId: quotaLimitDefault.spenderId,
+}): LimitQuotaCounter => {
+ const counter = {
+ kind: 'limit' as const,
+ isDefault: true,
+ limitValue: quotaLimitDefault.limitValue,
unit: quotaLimitDefault.unit,
+ resourceType: quotaLimitDefault.resourceType,
periodUnit: quotaLimitDefault.periodUnit,
periodStart: period.periodStart,
- limitValue: quotaLimitDefault.limitValue,
- }),
- limitValue: quotaLimitDefault.limitValue,
- unit: quotaLimitDefault.unit,
- resourceType: quotaLimitDefault.resourceType,
- periodUnit: quotaLimitDefault.periodUnit,
- periodStart: period.periodStart,
- periodEnd: period.periodEnd,
- spenderType: quotaLimitDefault.spenderType,
- spenderId: null,
- operationType: quotaLimitDefault.operationType,
-});
+ periodEnd: period.periodEnd,
+ spenderType: quotaLimitDefault.spenderType,
+ spenderId: null,
+ operationType: quotaLimitDefault.operationType,
+ };
+
+ return { ...counter, key: buildQuotaCounterKey({ workspaceId, counter }) };
+};
diff --git a/packages/twenty-server/src/engine/core-modules/usage-limit/utils/build-quota-exhausted-scope.util.ts b/packages/twenty-server/src/engine/core-modules/usage-limit/utils/build-quota-exhausted-scope.util.ts
index 2b37977b0c8..2b07753dc9d 100644
--- a/packages/twenty-server/src/engine/core-modules/usage-limit/utils/build-quota-exhausted-scope.util.ts
+++ b/packages/twenty-server/src/engine/core-modules/usage-limit/utils/build-quota-exhausted-scope.util.ts
@@ -1,4 +1,3 @@
-import { type CreditAllowance } from 'src/engine/core-modules/usage-limit/types/credit-allowance.type';
import { type ExhaustedScope } from 'src/engine/core-modules/usage-limit/types/exhausted-scope.type';
import { type QuotaCounter } from 'src/engine/core-modules/usage-limit/types/quota-counter.type';
import { UsageOperationType } from 'src/engine/core-modules/usage/enums/usage-operation-type.enum';
@@ -7,11 +6,9 @@ import { type UsageResourceType } from 'src/engine/core-modules/usage/enums/usag
export const buildQuotaExhaustedScope = ({
resourceType,
counter,
- allowance,
}: {
resourceType: UsageResourceType;
counter: QuotaCounter;
- allowance: CreditAllowance | null;
}): ExhaustedScope => {
const retryAfterMs = Math.max(counter.periodEnd.getTime() - Date.now(), 0);
@@ -41,7 +38,7 @@ export const buildQuotaExhaustedScope = ({
spenderId: null,
operationType: UsageOperationType.ALL,
unit: counter.unit,
- limitValue: allowance?.allowanceMicro ?? 0,
+ limitValue: counter.limitValue,
remaining: 0,
periodCount: null,
periodUnit: null,
diff --git a/packages/twenty-server/src/engine/core-modules/usage-limit/utils/build-quota-warm-lock-key.util.ts b/packages/twenty-server/src/engine/core-modules/usage-limit/utils/build-quota-warm-lock-key.util.ts
deleted file mode 100644
index f36d476de10..00000000000
--- a/packages/twenty-server/src/engine/core-modules/usage-limit/utils/build-quota-warm-lock-key.util.ts
+++ /dev/null
@@ -1,2 +0,0 @@
-export const buildQuotaWarmLockKey = (workspaceId: string): string =>
- `usage-quota-warm:${workspaceId}`;
diff --git a/packages/twenty-server/src/engine/core-modules/usage-limit/utils/compute-credit-allowance-micro.util.ts b/packages/twenty-server/src/engine/core-modules/usage-limit/utils/compute-credit-allowance-micro.util.ts
new file mode 100644
index 00000000000..b469b0340ef
--- /dev/null
+++ b/packages/twenty-server/src/engine/core-modules/usage-limit/utils/compute-credit-allowance-micro.util.ts
@@ -0,0 +1,23 @@
+import { isDefined } from 'twenty-shared/utils';
+
+import { type CreditAllowanceGrant } from 'src/engine/core-modules/usage-limit/types/credit-allowance-grant.type';
+
+export const computeCreditAllowanceMicro = ({
+ planAllowanceMicro,
+ grants,
+ nowMs,
+}: {
+ planAllowanceMicro: number;
+ grants: CreditAllowanceGrant[];
+ nowMs: number;
+}): number =>
+ grants
+ .filter(
+ ({ effectiveAtMs, expiresAtMs }) =>
+ effectiveAtMs <= nowMs &&
+ (!isDefined(expiresAtMs) || expiresAtMs > nowMs),
+ )
+ .reduce(
+ (allowanceMicro, { amountMicro }) => allowanceMicro + amountMicro,
+ planAllowanceMicro,
+ );
diff --git a/packages/twenty-server/src/engine/core-modules/usage-limit/utils/find-exhausted-counters.util.ts b/packages/twenty-server/src/engine/core-modules/usage-limit/utils/find-exhausted-counters.util.ts
index 0c566e01d4a..e35166ded43 100644
--- a/packages/twenty-server/src/engine/core-modules/usage-limit/utils/find-exhausted-counters.util.ts
+++ b/packages/twenty-server/src/engine/core-modules/usage-limit/utils/find-exhausted-counters.util.ts
@@ -2,26 +2,27 @@ import { isDefined } from 'twenty-shared/utils';
import { type QuotaCost } from 'src/engine/core-modules/usage-limit/types/quota-cost.type';
import { type QuotaCounter } from 'src/engine/core-modules/usage-limit/types/quota-counter.type';
+import { isLimitExhausted } from 'src/engine/core-modules/usage-limit/utils/is-limit-exhausted.util';
export const findExhaustedCounters = ({
counters,
- remainings,
+ consumedValues,
cost,
}: {
counters: QuotaCounter[];
- remainings: (number | null)[];
+ consumedValues: (number | null)[];
cost?: QuotaCost;
}): QuotaCounter[] =>
counters.filter((counter, index) => {
- const remaining = remainings[index];
+ const consumed = consumedValues[index];
- if (!isDefined(remaining)) {
+ if (!isDefined(consumed)) {
return false;
}
- if (remaining <= 0) {
- return true;
- }
-
- return remaining < (cost?.[counter.unit] ?? 0);
+ return isLimitExhausted({
+ consumed,
+ cost: cost?.[counter.unit] ?? 0,
+ limitValue: counter.limitValue,
+ });
});
diff --git a/packages/twenty-server/src/engine/core-modules/usage-limit/utils/from-consume-results-to-remainings.util.ts b/packages/twenty-server/src/engine/core-modules/usage-limit/utils/from-consume-results-to-remainings.util.ts
deleted file mode 100644
index d0608170191..00000000000
--- a/packages/twenty-server/src/engine/core-modules/usage-limit/utils/from-consume-results-to-remainings.util.ts
+++ /dev/null
@@ -1,8 +0,0 @@
-// The consume script answers [existed, remaining] pairs, one per counter key;
-// a counter that did not exist was not debited and stays cold (null).
-export const fromConsumeResultsToRemainings = (
- consumeResults: number[],
-): (number | null)[] =>
- Array.from({ length: consumeResults.length / 2 }, (_, index) =>
- consumeResults[2 * index] === 1 ? consumeResults[2 * index + 1] : null,
- );
diff --git a/packages/twenty-server/src/engine/core-modules/usage-limit/utils/is-limit-exhausted.util.ts b/packages/twenty-server/src/engine/core-modules/usage-limit/utils/is-limit-exhausted.util.ts
new file mode 100644
index 00000000000..bf48bd37437
--- /dev/null
+++ b/packages/twenty-server/src/engine/core-modules/usage-limit/utils/is-limit-exhausted.util.ts
@@ -0,0 +1,9 @@
+export const isLimitExhausted = ({
+ consumed,
+ cost,
+ limitValue,
+}: {
+ consumed: number;
+ cost: number;
+ limitValue: number;
+}): boolean => consumed >= limitValue || consumed + cost > limitValue;
diff --git a/packages/twenty-server/test/integration/billing/suites/billing-credit-grant-admin.integration-spec.ts b/packages/twenty-server/test/integration/billing/suites/billing-credit-grant-admin.integration-spec.ts
index 7bdd55e8462..5e7a29d69e6 100644
--- a/packages/twenty-server/test/integration/billing/suites/billing-credit-grant-admin.integration-spec.ts
+++ b/packages/twenty-server/test/integration/billing/suites/billing-credit-grant-admin.integration-spec.ts
@@ -4,13 +4,14 @@ import { randomUUID } from 'node:crypto';
import { addDays, addMonths, startOfMonth } from 'date-fns';
import request from 'supertest';
import {
+ debitInFlightCredits,
getSeededBillingWorkspaceId,
listCreditGrants,
quitBillingFixtureRedis,
- readAllowanceCounter,
+ readAllowanceConsumedMicro,
+ refreshCurrentBillingSubscription,
resetBillingCreditState,
setupResourceCreditSubscription,
- warmAllowanceCounter,
setSubscriptionStatus,
} from 'test/integration/billing/utils/billing-credit-fixtures.util';
@@ -24,6 +25,7 @@ const client = request(`http://localhost:${APP_PORT}`);
const PERIOD_START = startOfMonth(new Date());
const PERIOD_END = addMonths(PERIOD_START, 1);
const ALLOWANCE_MICRO = 1_000_000;
+const IN_FLIGHT_CREDITS_MICRO = 120_000;
const GRANT_MUTATION = `
mutation GrantWorkspaceCredits(
@@ -181,9 +183,12 @@ describe('Admin credit grant and revoke (integration)', () => {
expect(await listCreditGrants(workspaceId)).toHaveLength(0);
});
- it('drops the warm allowance counter when a grant lands', async () => {
- await warmAllowanceCounter(workspaceId, PERIOD_START, 500_000);
+ it('keeps in-flight usage on the allowance when a grant lands', async () => {
+ await refreshCurrentBillingSubscription(workspaceId);
+ const consumedBeforeMicro = await readAllowanceConsumedMicro(workspaceId);
+
+ await debitInFlightCredits(workspaceId, IN_FLIGHT_CREDITS_MICRO);
await grantCredits({
workspaceId,
amount: 2,
@@ -191,10 +196,12 @@ describe('Admin credit grant and revoke (integration)', () => {
reason: null,
});
- expect(await readAllowanceCounter(workspaceId, PERIOD_START)).toBeNull();
+ expect(await readAllowanceConsumedMicro(workspaceId)).toBe(
+ consumedBeforeMicro + IN_FLIGHT_CREDITS_MICRO,
+ );
});
- it('takes a revoked grant off the ledger and drops the counter', async () => {
+ it('takes a revoked grant off the ledger and keeps in-flight usage', async () => {
const granted = await grantCredits({
workspaceId,
amount: 2,
@@ -202,8 +209,9 @@ describe('Admin credit grant and revoke (integration)', () => {
reason: null,
});
const creditGrantId = granted.body.data.grantWorkspaceCredits.id;
+ const consumedBeforeMicro = await readAllowanceConsumedMicro(workspaceId);
- await warmAllowanceCounter(workspaceId, PERIOD_START, 500_000);
+ await debitInFlightCredits(workspaceId, IN_FLIGHT_CREDITS_MICRO);
const revoked = await callAdminGraphql(REVOKE_MUTATION, {
workspaceId,
@@ -218,10 +226,12 @@ describe('Admin credit grant and revoke (integration)', () => {
const grants = await listCreditGrants(workspaceId);
expect(grants[0].revokedAt).not.toBeNull();
- expect(await readAllowanceCounter(workspaceId, PERIOD_START)).toBeNull();
+ expect(await readAllowanceConsumedMicro(workspaceId)).toBe(
+ consumedBeforeMicro + IN_FLIGHT_CREDITS_MICRO,
+ );
});
- it('drops the counter again when a revocation is retried', async () => {
+ it('keeps in-flight usage when a revocation is retried', async () => {
const granted = await grantCredits({
workspaceId,
amount: 2,
@@ -231,7 +241,10 @@ describe('Admin credit grant and revoke (integration)', () => {
const creditGrantId = granted.body.data.grantWorkspaceCredits.id;
await callAdminGraphql(REVOKE_MUTATION, { workspaceId, creditGrantId });
- await warmAllowanceCounter(workspaceId, PERIOD_START, 500_000);
+
+ const consumedBeforeMicro = await readAllowanceConsumedMicro(workspaceId);
+
+ await debitInFlightCredits(workspaceId, IN_FLIGHT_CREDITS_MICRO);
const retried = await callAdminGraphql(REVOKE_MUTATION, {
workspaceId,
@@ -239,7 +252,9 @@ describe('Admin credit grant and revoke (integration)', () => {
});
expect(retried.body.errors).toBeUndefined();
- expect(await readAllowanceCounter(workspaceId, PERIOD_START)).toBeNull();
+ expect(await readAllowanceConsumedMicro(workspaceId)).toBe(
+ consumedBeforeMicro + IN_FLIGHT_CREDITS_MICRO,
+ );
});
it.each([
@@ -286,14 +301,12 @@ describe('Admin credit grant and revoke (integration)', () => {
};
const first = await callAdminGraphql(GRANT_MUTATION, variables);
- await warmAllowanceCounter(workspaceId, PERIOD_START, 500_000);
const second = await callAdminGraphql(GRANT_MUTATION, variables);
expect(second.body.data.grantWorkspaceCredits.id).toBe(
first.body.data.grantWorkspaceCredits.id,
);
expect(await listCreditGrants(workspaceId)).toHaveLength(1);
- expect(await readAllowanceCounter(workspaceId, PERIOD_START)).toBeNull();
});
// The anchoring subscription can be gone by retry time, and the operation already succeeded
diff --git a/packages/twenty-server/test/integration/billing/suites/billing-credit-rollover.integration-spec.ts b/packages/twenty-server/test/integration/billing/suites/billing-credit-rollover.integration-spec.ts
index 379e17e7581..7146745cd0f 100644
--- a/packages/twenty-server/test/integration/billing/suites/billing-credit-rollover.integration-spec.ts
+++ b/packages/twenty-server/test/integration/billing/suites/billing-credit-rollover.integration-spec.ts
@@ -2,15 +2,16 @@ import { addMonths, startOfMonth, subMonths } from 'date-fns';
import request from 'supertest';
import { createMockStripeInvoiceFinalizedData } from 'test/integration/billing/utils/create-mock-stripe-invoice-finalized-data.util';
import {
+ debitInFlightCredits,
getSeededBillingWorkspaceId,
insertCreditGrant,
listCreditGrants,
quitBillingFixtureRedis,
- readAllowanceCounter,
+ readAllowanceConsumedMicro,
+ refreshCurrentBillingSubscription,
resetBillingCreditState,
setupResourceCreditSubscription,
TEST_STRIPE_CUSTOMER_ID,
- warmAllowanceCounter,
} from 'test/integration/billing/utils/billing-credit-fixtures.util';
import { getAppProviderByClassName } from 'test/integration/utils/get-app-provider-by-class-name.util';
@@ -26,6 +27,7 @@ const CLOSING_PERIOD_END = PERIOD_BOUNDARY;
const NEXT_PERIOD_END = addMonths(PERIOD_BOUNDARY, 1);
const ALLOWANCE_MICRO = 1_000_000;
+const IN_FLIGHT_CREDITS_MICRO = 120_000;
const postInvoiceFinalized = (
invoiceId = 'in_test_default',
@@ -200,30 +202,41 @@ describe('Billing credit rollover (integration)', () => {
expect(await listCreditGrants(workspaceId)).toHaveLength(0);
});
- it('drops the allowance counter on the period transition', async () => {
- usageSpy.mockResolvedValue(300_000);
+ describe('once Stripe has advanced the subscription', () => {
+ beforeEach(async () => {
+ usageSpy.mockResolvedValue(300_000);
+ await setupResourceCreditSubscription({
+ workspaceId,
+ periodStart: CLOSING_PERIOD_END,
+ periodEnd: NEXT_PERIOD_END,
+ creditAmountMicro: ALLOWANCE_MICRO,
+ });
+ await refreshCurrentBillingSubscription(workspaceId);
+ });
- // Stripe advances the subscription in a separate event, so the counter is keyed by the closing period
- await warmAllowanceCounter(workspaceId, CLOSING_PERIOD_START, 120_000);
+ it('keeps in-flight usage of the opened period through the transition', async () => {
+ const consumedBeforeMicro = await readAllowanceConsumedMicro(workspaceId);
- await postInvoiceFinalized().expect(200);
+ await debitInFlightCredits(workspaceId, IN_FLIGHT_CREDITS_MICRO);
+ await postInvoiceFinalized().expect(200);
- expect(
- await readAllowanceCounter(workspaceId, CLOSING_PERIOD_START),
- ).toBeNull();
- });
+ expect(await readAllowanceConsumedMicro(workspaceId)).toBe(
+ consumedBeforeMicro + IN_FLIGHT_CREDITS_MICRO,
+ );
+ });
- it('drops the allowance counter again when a successful delivery is repeated', async () => {
- usageSpy.mockResolvedValue(300_000);
+ it('keeps in-flight usage when a successful delivery is repeated', async () => {
+ await postInvoiceFinalized().expect(200);
- await postInvoiceFinalized().expect(200);
- await warmAllowanceCounter(workspaceId, CLOSING_PERIOD_START, 120_000);
+ const consumedBeforeMicro = await readAllowanceConsumedMicro(workspaceId);
- await postInvoiceFinalized().expect(200);
+ await debitInFlightCredits(workspaceId, IN_FLIGHT_CREDITS_MICRO);
+ await postInvoiceFinalized().expect(200);
- expect(
- await readAllowanceCounter(workspaceId, CLOSING_PERIOD_START),
- ).toBeNull();
+ expect(await readAllowanceConsumedMicro(workspaceId)).toBe(
+ consumedBeforeMicro + IN_FLIGHT_CREDITS_MICRO,
+ );
+ });
});
// A 31st anchor runs Jan 31 to Feb 28; once advanced, calendar arithmetic clamps Feb 28 back to Jan 28 and swallows three days
diff --git a/packages/twenty-server/test/integration/billing/suites/billing-credit-top-up.integration-spec.ts b/packages/twenty-server/test/integration/billing/suites/billing-credit-top-up.integration-spec.ts
index 8c03c49f521..151b7945dc9 100644
--- a/packages/twenty-server/test/integration/billing/suites/billing-credit-top-up.integration-spec.ts
+++ b/packages/twenty-server/test/integration/billing/suites/billing-credit-top-up.integration-spec.ts
@@ -3,14 +3,16 @@ import { randomUUID } from 'node:crypto';
import { addMonths, startOfMonth } from 'date-fns';
import request from 'supertest';
import {
+ debitInFlightCredits,
getSeededBillingWorkspaceId,
listCreditGrants,
quitBillingFixtureRedis,
- readAllowanceCounter,
+ readAllowanceConsumedMicro,
+ readAllowanceLimitMicro,
+ refreshCurrentBillingSubscription,
resetBillingCreditState,
setupResourceCreditSubscription,
TEST_STRIPE_CUSTOMER_ID,
- warmAllowanceCounter,
} from 'test/integration/billing/utils/billing-credit-fixtures.util';
import { createMockStripeCreditTopUpInvoicePaidData } from 'test/integration/billing/utils/create-mock-stripe-credit-top-up-invoice-paid-data.util';
@@ -23,6 +25,7 @@ const PERIOD_END = addMonths(PERIOD_START, 1);
const ALLOWANCE_MICRO = 1_000_000;
const TOP_UP_MICRO = 2_000_000;
+const IN_FLIGHT_CREDITS_MICRO = 120_000;
const postCreditTopUpInvoicePaid = ({
invoiceId,
@@ -99,15 +102,24 @@ describe('Billing credit top-up webhook (integration)', () => {
});
});
- it('drops the allowance counter so a blocked workspace resumes at once', async () => {
- await warmAllowanceCounter(workspaceId, PERIOD_START, 0);
+ it('raises the allowance cap at once and keeps in-flight usage', async () => {
+ await refreshCurrentBillingSubscription(workspaceId);
+ const limitBeforeMicro = await readAllowanceLimitMicro(workspaceId);
+ const consumedBeforeMicro = await readAllowanceConsumedMicro(workspaceId);
+
+ await debitInFlightCredits(workspaceId, IN_FLIGHT_CREDITS_MICRO);
await postCreditTopUpInvoicePaid({
invoiceId: `in_test_top_up_${randomUUID()}`,
workspaceId,
});
- expect(await readAllowanceCounter(workspaceId, PERIOD_START)).toBeNull();
+ expect(await readAllowanceLimitMicro(workspaceId)).toBe(
+ limitBeforeMicro + TOP_UP_MICRO,
+ );
+ expect(await readAllowanceConsumedMicro(workspaceId)).toBe(
+ consumedBeforeMicro + IN_FLIGHT_CREDITS_MICRO,
+ );
});
it('grants nothing more when Stripe redelivers the event', async () => {
diff --git a/packages/twenty-server/test/integration/billing/utils/billing-credit-fixtures.util.ts b/packages/twenty-server/test/integration/billing/utils/billing-credit-fixtures.util.ts
index 370c483526f..c24336026e5 100644
--- a/packages/twenty-server/test/integration/billing/utils/billing-credit-fixtures.util.ts
+++ b/packages/twenty-server/test/integration/billing/utils/billing-credit-fixtures.util.ts
@@ -1,10 +1,14 @@
+import { getAppProviderByClassName } from 'test/integration/utils/get-app-provider-by-class-name.util';
import { createClient, type RedisClientType } from 'redis';
import { isDefined, isNonEmptyArray } from 'twenty-shared/utils';
import { type BillingCreditGrantType } from 'src/engine/core-modules/billing/enums/billing-credit-grant-type.enum';
import { type SubscriptionStatus } from 'src/engine/core-modules/billing/enums/billing-subscription-status.enum';
-import { CacheStorageNamespace } from 'src/engine/core-modules/cache-storage/types/cache-storage-namespace.enum';
-import { buildAllowanceCounterKey } from 'src/engine/core-modules/usage-limit/utils/build-allowance-counter-key.util';
+import { type UsageLimitQuotaService } from 'src/engine/core-modules/usage-limit/services/usage-limit-quota.service';
+import { UsageOperationType } from 'src/engine/core-modules/usage/enums/usage-operation-type.enum';
+import { UsageResourceType } from 'src/engine/core-modules/usage/enums/usage-resource-type.enum';
+import { UsageUnit } from 'src/engine/core-modules/usage/enums/usage-unit.enum';
+import { type WorkspaceCacheService } from 'src/engine/workspace-cache/services/workspace-cache.service';
// The dev seeder creates no subscription item or period; rollover needs subscription -> item -> product -> price
export const TEST_STRIPE_CUSTOMER_ID = 'cus_default0';
@@ -182,35 +186,57 @@ export const quitBillingFixtureRedis = async (): Promise<void> => {
}
};
-const buildTestAllowanceCounterKey = (
+const getUsageLimitQuotaService = () =>
+ getAppProviderByClassName<UsageLimitQuotaService>('UsageLimitQuotaService');
+
+export const refreshCurrentBillingSubscription = (
workspaceId: string,
- periodStart: Date,
-): string =>
- `${CacheStorageNamespace.IntegrationTests}:${CacheStorageNamespace.EngineUsageLimit}:${buildAllowanceCounterKey({ workspaceId, periodStart })}`;
+): Promise<void> =>
+ getAppProviderByClassName<WorkspaceCacheService>(
+ 'WorkspaceCacheService',
+ ).invalidateAndRecompute(workspaceId, ['currentBillingSubscription']);
-export const warmAllowanceCounter = async (
+export const debitInFlightCredits = async (
workspaceId: string,
- periodStart: Date,
- valueMicro: number,
+ creditsUsedMicro: number,
): Promise<void> => {
- const redis = await getRedisClient();
+ await getUsageLimitQuotaService().debitAheadOfRecord({
+ workspaceId,
+ event: {
+ resourceType: UsageResourceType.AI,
+ operationType: UsageOperationType.AI_CHAT_TOKEN,
+ unit: UsageUnit.TOKEN,
+ quantity: 1,
+ creditsUsedMicro,
+ spenders: {},
+ },
+ });
+};
- await redis.set(
- buildTestAllowanceCounterKey(workspaceId, periodStart),
- String(valueMicro),
- );
+export const readAllowanceConsumedMicro = async (
+ workspaceId: string,
+): Promise<number> => {
+ const allowanceUsage =
+ await getUsageLimitQuotaService().getAllowanceUsage(workspaceId);
+
+ if (!isDefined(allowanceUsage?.consumedValue)) {
+ throw new Error(`No allowance consumption for workspace ${workspaceId}`);
+ }
+
+ return allowanceUsage.consumedValue;
};
-export const readAllowanceCounter = async (
+export const readAllowanceLimitMicro = async (
workspaceId: string,
- periodStart: Date,
-): Promise<number | null> => {
- const redis = await getRedisClient();
- const value = await redis.get(
- buildTestAllowanceCounterKey(workspaceId, periodStart),
- );
+): Promise<number> => {
+ const allowanceUsage =
+ await getUsageLimitQuotaService().getAllowanceUsage(workspaceId);
+
+ if (!isDefined(allowanceUsage)) {
+ throw new Error(`No allowance for workspace ${workspaceId}`);
+ }
- return isDefined(value) ? Number(value) : null;
+ return allowanceUsage.limitValue;
};
export const resetBillingCreditState = async (
@@ -223,7 +249,7 @@ export const resetBillingCreditState = async (
const redis = await getRedisClient();
const staleKeys = [
- ...(await redis.keys(`*{${workspaceId}}:quota:allowance:*`)),
+ ...(await redis.keys(`*{${workspaceId}}:quota*`)),
...(await redis.keys(`*currentBillingSubscription:${workspaceId}*`)),
];
diff --git a/packages/twenty-server/test/integration/usage-limit/quota-consumption.integration-spec.ts b/packages/twenty-server/test/integration/usage-limit/quota-consumption.integration-spec.ts
index e86bd1c01b8..32eb0521174 100644
--- a/packages/twenty-server/test/integration/usage-limit/quota-consumption.integration-spec.ts
+++ b/packages/twenty-server/test/integration/usage-limit/quota-consumption.integration-spec.ts
@@ -6,6 +6,7 @@ import { expectEventually } from 'test/integration/utils/expect-eventually.util'
import { getAppProviderByClassName } from 'test/integration/utils/get-app-provider-by-class-name.util';
import { getCoreRepository } from 'test/integration/utils/get-core-repository.util';
import { jestExpectToBeDefined } from 'test/utils/jest-expect-to-be-defined.util.test';
+import { randomUUID } from 'node:crypto';
import {
type ClickHouseClient,
@@ -18,10 +19,22 @@ import { isNonEmptyArray } from 'twenty-shared/utils';
import { type Repository } from 'typeorm';
import { formatDateTimeForClickHouse } from 'src/database/clickhouse/utils/format-date-time-for-clickhouse.util';
+import { type CacheStorageService } from 'src/engine/core-modules/cache-storage/services/cache-storage.service';
+import { type MetricsService } from 'src/engine/core-modules/metrics/metrics.service';
+import { MetricsKeys } from 'src/engine/core-modules/metrics/types/metrics-keys.type';
+import { type TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';
+import { type CreateUsageLimitInput } from 'src/engine/core-modules/usage-limit/dtos/create-usage-limit.input';
+import { type UsageLimitEntitlementService } from 'src/engine/core-modules/usage-limit/services/usage-limit-entitlement.service';
+import { type UsageLimitQuotaService } from 'src/engine/core-modules/usage-limit/services/usage-limit-quota.service';
+import { type UsageLimitService } from 'src/engine/core-modules/usage-limit/services/usage-limit.service';
+import { type PeriodUnit } from 'src/engine/core-modules/usage-limit/types/period-unit.type';
import { UsageLimitEntity } from 'src/engine/core-modules/usage-limit/usage-limit.entity';
+import { getCalendarMonthPeriod } from 'src/engine/core-modules/usage-limit/utils/get-calendar-month-period.util';
import { UsageOperationType } from 'src/engine/core-modules/usage/enums/usage-operation-type.enum';
import { UsageResourceType } from 'src/engine/core-modules/usage/enums/usage-resource-type.enum';
import { UsageUnit } from 'src/engine/core-modules/usage/enums/usage-unit.enum';
+import { type UsageAnalyticsService } from 'src/engine/core-modules/usage/services/usage-analytics.service';
+import { type RecordUsageInput } from 'src/engine/core-modules/usage/types/record-usage-input.type';
import { LogicFunctionExecutionStatus } from 'src/engine/metadata-modules/logic-function/dtos/logic-function-execution-result.dto';
import { type WorkspaceCacheService } from 'src/engine/workspace-cache/services/workspace-cache.service';
import { SEED_APPLE_WORKSPACE_ID } from 'src/engine/workspace-manager/dev-seeder/core/constants/seeder-workspaces.constant';
@@ -57,6 +70,8 @@ const QUOTA_UNITS = [UsageUnit.INVOCATION, UsageUnit.CREDIT] as const;
const EXECUTED_RUN_COUNT = 2;
const UNREACHABLE_LIMIT_VALUE = 1_000_000_000_000;
+const CREDIT_LIMIT_MICRO = 1_000;
+const CONCURRENT_CHARGE_COUNT = 8;
const CLICKHOUSE_FLUSH_TIMEOUT_MS = 30_000;
const EXECUTION_TIMEOUT_MS = 120_000;
@@ -81,6 +96,22 @@ const USAGE_QUOTA_SCOPE_CONSUMPTION = gql`
type QuotaUnit = (typeof QUOTA_UNITS)[number];
+const buildRunEvent = ({
+ spenderId,
+ creditsUsedMicro,
+}: {
+ spenderId: string;
+ creditsUsedMicro: number;
+}): RecordUsageInput => ({
+ resourceType: UsageResourceType.LOGIC_FUNCTION,
+ operationType: UsageOperationType.CODE_EXECUTION,
+ unit: UsageUnit.INVOCATION,
+ quantity: 1,
+ creditsUsedMicro,
+ resourceId: spenderId,
+ spenders: { logicFunctionId: spenderId },
+});
+
describe('Usage quota consumption', () => {
let clickHouseClient: ClickHouseClient;
let redis: Awaited<ReturnType<typeof createRedisClient>>;
@@ -88,24 +119,89 @@ describe('Usage quota consumption', () => {
let logicFunctionId: string;
let usageLimitIdByUnit: Record<QuotaUnit, string>;
const savedUsageLimitIds: string[] = [];
+ const chargedSpenderIds: string[] = [];
+
+ const getQuotaService = () =>
+ getAppProviderByClassName<UsageLimitQuotaService>('UsageLimitQuotaService');
+
+ const getUsageLimitService = () =>
+ getAppProviderByClassName<UsageLimitService>('UsageLimitService');
const refreshUsageLimitsCache = () =>
getAppProviderByClassName<WorkspaceCacheService>(
'WorkspaceCacheService',
).invalidateAndRecompute(SEED_APPLE_WORKSPACE_ID, ['usageLimits']);
- const findQuotaCounterKeys = () =>
- redis.keys(
- `*{${SEED_APPLE_WORKSPACE_ID}}:quota:${UsageResourceType.LOGIC_FUNCTION}:${UsageOperationType.CODE_EXECUTION}:logicFunction:${logicFunctionId}:*`,
+ const findSpenderCounterKeys = (spenderId: string) =>
+ redis.keys(`*{${SEED_APPLE_WORKSPACE_ID}}:*:logicFunction:${spenderId}:*`);
+
+ const findQuotaCounterKeys = () => findSpenderCounterKeys(logicFunctionId);
+
+ const flushQuotaCounters = async () => {
+ const counterKeys = await redis.keys(
+ `*{${SEED_APPLE_WORKSPACE_ID}}:quota*`,
);
- const countUsageEventRows = async (): Promise<number> => {
+ if (counterKeys.length > 0) {
+ await redis.del(counterKeys);
+ }
+ };
+
+ const spyOnMetrics = () =>
+ jest.spyOn(
+ getAppProviderByClassName<MetricsService>('MetricsService'),
+ 'incrementCounterBy',
+ );
+
+ const findAdmittedOnFailureAttributes = (
+ incrementCounterBySpy: ReturnType<typeof spyOnMetrics>,
+ ) =>
+ incrementCounterBySpy.mock.calls
+ .map(([metric]) => metric)
+ .filter(
+ (metric) => metric.key === MetricsKeys.UsageLimitQuotaAdmittedOnFailure,
+ )
+ .map((metric) => metric.attributes);
+
+ const chargeRun = (event: Parameters<typeof buildRunEvent>[0]) =>
+ getQuotaService().charge({
+ workspaceId: SEED_APPLE_WORKSPACE_ID,
+ events: [buildRunEvent(event)],
+ });
+
+ const debitRunAheadOfRecord = (event: Parameters<typeof buildRunEvent>[0]) =>
+ getQuotaService().debitAheadOfRecord({
+ workspaceId: SEED_APPLE_WORKSPACE_ID,
+ event: buildRunEvent(event),
+ });
+
+ const findRunRefusal = ({
+ spenderId,
+ costMicro,
+ }: {
+ spenderId: string;
+ costMicro: number;
+ }) =>
+ getQuotaService().findExhaustedScope({
+ workspaceId: SEED_APPLE_WORKSPACE_ID,
+ resourceType: UsageResourceType.LOGIC_FUNCTION,
+ operationType: UsageOperationType.CODE_EXECUTION,
+ spenders: { logicFunctionId: spenderId },
+ cost: { [UsageUnit.CREDIT]: costMicro },
+ });
+
+ const countUsageEventRows = async (
+ spenderId: string = logicFunctionId,
+ ): Promise<number> => {
const result = await clickHouseClient.query({
query: `SELECT count() AS rowCount
FROM usageEvent
WHERE workspaceId = {workspaceId:String}
AND logicFunctionId = {logicFunctionId:String}`,
- query_params: { workspaceId: SEED_APPLE_WORKSPACE_ID, logicFunctionId },
+ query_params: {
+ workspaceId: SEED_APPLE_WORKSPACE_ID,
+ logicFunctionId: spenderId,
+ },
format: 'JSONEachRow',
});
@@ -156,18 +252,26 @@ describe('Usage quota consumption', () => {
});
};
- const saveCodeExecutionQuota = async (unit: QuotaUnit): Promise<string> => {
+ const saveCodeExecutionQuota = async ({
+ spenderId,
+ unit,
+ limitValue,
+ }: {
+ spenderId: string;
+ unit: QuotaUnit;
+ limitValue: number;
+ }): Promise<string> => {
const usageLimit = await usageLimitRepository.save({
workspaceId: SEED_APPLE_WORKSPACE_ID,
resourceType: UsageResourceType.LOGIC_FUNCTION,
operationType: UsageOperationType.CODE_EXECUTION,
spenderType: 'logicFunction',
- spenderId: logicFunctionId,
+ spenderId,
limitKind: 'quota',
periodCount: 1,
periodUnit: 'month',
unit,
- limitValue: UNREACHABLE_LIMIT_VALUE,
+ limitValue,
burstValue: null,
isInstanceOverride: true,
});
@@ -177,6 +281,79 @@ describe('Usage quota consumption', () => {
return usageLimit.id;
};
+ const saveCreditQuotaForNewSpender = async ({
+ limitValue = CREDIT_LIMIT_MICRO,
+ }: {
+ limitValue?: number;
+ } = {}): Promise<{ spenderId: string }> => {
+ const spenderId = randomUUID();
+
+ chargedSpenderIds.push(spenderId);
+
+ await saveCodeExecutionQuota({
+ spenderId,
+ unit: UsageUnit.CREDIT,
+ limitValue,
+ });
+
+ await refreshUsageLimitsCache();
+
+ return { spenderId };
+ };
+
+ const buildWeekCreditQuotaInput = ({
+ periodUnit = 'week',
+ limitValue = UNREACHABLE_LIMIT_VALUE,
+ }: {
+ periodUnit?: PeriodUnit;
+ limitValue?: number;
+ } = {}): CreateUsageLimitInput => ({
+ resourceType: UsageResourceType.LOGIC_FUNCTION,
+ operationType: UsageOperationType.CODE_EXECUTION,
+ spenderType: 'logicFunction',
+ spenderId: logicFunctionId,
+ limitKind: 'quota',
+ periodCount: 1,
+ periodUnit,
+ unit: UsageUnit.CREDIT,
+ limitValue,
+ burstValue: null,
+ });
+
+ const createWeekCreditQuota = ({ isOperator = true } = {}) =>
+ getUsageLimitService().create({
+ workspaceId: SEED_APPLE_WORKSPACE_ID,
+ input: buildWeekCreditQuotaInput(),
+ isOperator,
+ });
+
+ const updateWeekCreditQuota = (
+ usageLimitId: string,
+ payload: Parameters<typeof buildWeekCreditQuotaInput>[0],
+ ) =>
+ getUsageLimitService().update({
+ workspaceId: SEED_APPLE_WORKSPACE_ID,
+ input: { id: usageLimitId, payload: buildWeekCreditQuotaInput(payload) },
+ isOperator: true,
+ });
+
+ const deleteWeekCreditQuota = (usageLimitId: string) =>
+ getUsageLimitService().delete({
+ workspaceId: SEED_APPLE_WORKSPACE_ID,
+ usageLimitId,
+ isOperator: true,
+ });
+
+ const readWeekCreditCounter = async () => {
+ const [counterKey] = await redis.keys(
+ `*{${SEED_APPLE_WORKSPACE_ID}}:quota-consumed:*:logicFunction:${logicFunctionId}:${UsageUnit.CREDIT}:week:*`,
+ );
+
+ jestExpectToBeDefined(counterKey);
+
+ return Number(await redis.get(counterKey));
+ };
+
const findQuotaConsumedValue = async (unit: QuotaUnit) => {
const response = await makeMetadataApiRequest({
query: USAGE_QUOTAS_WITH_CONSUMPTION,
@@ -250,10 +427,16 @@ describe('Usage quota consumption', () => {
await waitForUsageEventRows(SEEDED_RUNS.length * 2);
usageLimitIdByUnit = {
- [UsageUnit.INVOCATION]: await saveCodeExecutionQuota(
- UsageUnit.INVOCATION,
- ),
- [UsageUnit.CREDIT]: await saveCodeExecutionQuota(UsageUnit.CREDIT),
+ [UsageUnit.INVOCATION]: await saveCodeExecutionQuota({
+ spenderId: logicFunctionId,
+ unit: UsageUnit.INVOCATION,
+ limitValue: UNREACHABLE_LIMIT_VALUE,
+ }),
+ [UsageUnit.CREDIT]: await saveCodeExecutionQuota({
+ spenderId: logicFunctionId,
+ unit: UsageUnit.CREDIT,
+ limitValue: UNREACHABLE_LIMIT_VALUE,
+ }),
};
await refreshUsageLimitsCache();
@@ -273,7 +456,11 @@ describe('Usage quota consumption', () => {
}
await clickHouseClient.command({
- query: `ALTER TABLE usageEvent DELETE WHERE workspaceId = '${SEED_APPLE_WORKSPACE_ID}' AND logicFunctionId = '${logicFunctionId}'`,
+ query: `ALTER TABLE usageEvent DELETE WHERE workspaceId = {workspaceId:String} AND logicFunctionId IN {logicFunctionIds:Array(String)}`,
+ query_params: {
+ workspaceId: SEED_APPLE_WORKSPACE_ID,
+ logicFunctionIds: [logicFunctionId, ...chargedSpenderIds],
+ },
clickhouse_settings: { mutations_sync: '2' },
});
@@ -323,9 +510,6 @@ describe('Usage quota consumption', () => {
const scopeConsumedValue = await findScopeConsumedValue(unit);
expect(quotaConsumedValue).toBe(scopeConsumedValue);
- expect(quotaConsumedValue).toBeGreaterThanOrEqual(
- SEEDED_CONSUMPTION_BY_UNIT[unit],
- );
}
expect(await findQuotaConsumedValue(UsageUnit.INVOCATION)).toBe(
@@ -369,7 +553,407 @@ describe('Usage quota consumption', () => {
});
await waitForUsageEventRows(usageEventRowCount + 2);
+
+ await usageLimitRepository.update(
+ { id: usageLimitIdByUnit[UsageUnit.INVOCATION] },
+ { limitValue: UNREACHABLE_LIMIT_VALUE },
+ );
+ await refreshUsageLimitsCache();
},
EXECUTION_TIMEOUT_MS,
);
+
+ describe('on a credit quota', () => {
+ it('counts every concurrent charge on a cold counter once', async () => {
+ const { spenderId } = await saveCreditQuotaForNewSpender();
+
+ await Promise.all(
+ Array.from({ length: CONCURRENT_CHARGE_COUNT }, () =>
+ chargeRun({ spenderId, creditsUsedMicro: 100 }),
+ ),
+ );
+
+ expect(await findRunRefusal({ spenderId, costMicro: 200 })).toBeNull();
+ expect(await findRunRefusal({ spenderId, costMicro: 201 })).toMatchObject(
+ { exhaustedKind: 'limit' },
+ );
+
+ const counterKeys = await findSpenderCounterKeys(spenderId);
+
+ expect(counterKeys).toHaveLength(1);
+ expect(
+ Math.abs(
+ (await redis.pTTL(counterKeys[0])) -
+ (getCalendarMonthPeriod(new Date()).periodEnd.getTime() -
+ Date.now()),
+ ),
+ ).toBeLessThan(60_000);
+ });
+
+ it(
+ 'keeps in-flight usage when the cap is edited',
+ async () => {
+ const usageLimit = await createWeekCreditQuota();
+ const usageEventRowCount = await countUsageEventRows();
+
+ try {
+ await chargeRun({
+ spenderId: logicFunctionId,
+ creditsUsedMicro: 100,
+ });
+
+ const editedLimitValue = (await readWeekCreditCounter()) + 700;
+
+ await debitRunAheadOfRecord({
+ spenderId: logicFunctionId,
+ creditsUsedMicro: 600,
+ });
+ await updateWeekCreditQuota(usageLimit.id, {
+ limitValue: editedLimitValue,
+ });
+
+ expect(
+ await findRunRefusal({
+ spenderId: logicFunctionId,
+ costMicro: 100,
+ }),
+ ).toBeNull();
+ expect(
+ await findRunRefusal({
+ spenderId: logicFunctionId,
+ costMicro: 101,
+ }),
+ ).toMatchObject({
+ exhaustedKind: 'limit',
+ limitValue: editedLimitValue,
+ });
+
+ await waitForUsageEventRows(usageEventRowCount + 1);
+ } finally {
+ await deleteWeekCreditQuota(usageLimit.id);
+ }
+ },
+ EXECUTION_TIMEOUT_MS,
+ );
+
+ it(
+ 're-warms a limit from ClickHouse once it moves back to a period it left',
+ async () => {
+ const usageLimit = await createWeekCreditQuota();
+ const usageEventRowCount = await countUsageEventRows();
+
+ try {
+ await chargeRun({
+ spenderId: logicFunctionId,
+ creditsUsedMicro: 100,
+ });
+
+ const enforcedLimitValue = (await readWeekCreditCounter()) + 300;
+
+ await updateWeekCreditQuota(usageLimit.id, { periodUnit: 'day' });
+ await chargeRun({
+ spenderId: logicFunctionId,
+ creditsUsedMicro: 600,
+ });
+ await waitForUsageEventRows(usageEventRowCount + 2);
+
+ await updateWeekCreditQuota(usageLimit.id, {
+ limitValue: enforcedLimitValue,
+ });
+
+ expect(
+ await findRunRefusal({ spenderId: logicFunctionId, costMicro: 0 }),
+ ).toMatchObject({
+ exhaustedKind: 'limit',
+ limitValue: enforcedLimitValue,
+ });
+ } finally {
+ await deleteWeekCreditQuota(usageLimit.id);
+ }
+ },
+ EXECUTION_TIMEOUT_MS,
+ );
+
+ it(
+ 're-warms a limit from ClickHouse once an operator edit makes it enforced again',
+ async () => {
+ const entitlementSpy = jest
+ .spyOn(
+ getAppProviderByClassName<UsageLimitEntitlementService>(
+ 'UsageLimitEntitlementService',
+ ),
+ 'isIntraWorkspaceLimitEntitled',
+ )
+ .mockResolvedValue(true);
+ const usageLimit = await createWeekCreditQuota({ isOperator: false });
+ const usageEventRowCount = await countUsageEventRows();
+
+ try {
+ await chargeRun({
+ spenderId: logicFunctionId,
+ creditsUsedMicro: 100,
+ });
+
+ const enforcedLimitValue = (await readWeekCreditCounter()) + 300;
+
+ entitlementSpy.mockResolvedValue(false);
+ await chargeRun({
+ spenderId: logicFunctionId,
+ creditsUsedMicro: 600,
+ });
+
+ expect(await readWeekCreditCounter()).toBe(enforcedLimitValue - 300);
+
+ await waitForUsageEventRows(usageEventRowCount + 2);
+
+ await updateWeekCreditQuota(usageLimit.id, {
+ limitValue: enforcedLimitValue,
+ });
+
+ expect(
+ await findRunRefusal({ spenderId: logicFunctionId, costMicro: 0 }),
+ ).toMatchObject({
+ exhaustedKind: 'limit',
+ limitValue: enforcedLimitValue,
+ });
+ } finally {
+ entitlementSpy.mockRestore();
+ await deleteWeekCreditQuota(usageLimit.id);
+ }
+ },
+ EXECUTION_TIMEOUT_MS,
+ );
+ });
+
+ describe('on the daily email default', () => {
+ const emailResourceId = randomUUID();
+ let emailDailyLimit: number;
+ let chargedEmailRowCount = 0;
+
+ const buildEmailEvent = (quantity: number): RecordUsageInput => ({
+ resourceType: UsageResourceType.EMAIL,
+ operationType: UsageOperationType.EMAIL_SEND,
+ unit: UsageUnit.INVOCATION,
+ quantity,
+ creditsUsedMicro: 0,
+ resourceId: emailResourceId,
+ });
+
+ const findEmailRefusal = (invocationCost: number) =>
+ getQuotaService().findExhaustedScope({
+ workspaceId: SEED_APPLE_WORKSPACE_ID,
+ resourceType: UsageResourceType.EMAIL,
+ operationType: UsageOperationType.EMAIL_SEND,
+ spenders: {},
+ cost: {
+ [UsageUnit.CREDIT]: 0,
+ [UsageUnit.INVOCATION]: invocationCost,
+ },
+ });
+
+ const debitEmailAheadOfRecord = (quantity: number) =>
+ getQuotaService().debitAheadOfRecord({
+ workspaceId: SEED_APPLE_WORKSPACE_ID,
+ event: buildEmailEvent(quantity),
+ });
+
+ const chargeEmail = async (quantity: number) => {
+ await getQuotaService().charge({
+ workspaceId: SEED_APPLE_WORKSPACE_ID,
+ events: [buildEmailEvent(quantity)],
+ });
+ chargedEmailRowCount++;
+ };
+
+ const saveEmailOverride = async () => {
+ const usageLimit = await usageLimitRepository.save({
+ workspaceId: SEED_APPLE_WORKSPACE_ID,
+ resourceType: UsageResourceType.EMAIL,
+ operationType: UsageOperationType.EMAIL_SEND,
+ spenderType: 'workspace',
+ spenderId: '',
+ limitKind: 'quota',
+ periodCount: 1,
+ periodUnit: 'day',
+ unit: UsageUnit.INVOCATION,
+ limitValue: emailDailyLimit + 1_000,
+ burstValue: null,
+ isInstanceOverride: true,
+ });
+
+ savedUsageLimitIds.push(usageLimit.id);
+ await refreshUsageLimitsCache();
+
+ return usageLimit.id;
+ };
+
+ const countEmailRows = async (): Promise<number> => {
+ const result = await clickHouseClient.query({
+ query: `SELECT count() AS rowCount
+ FROM usageEvent
+ WHERE workspaceId = {workspaceId:String}
+ AND resourceId = {resourceId:String}`,
+ query_params: {
+ workspaceId: SEED_APPLE_WORKSPACE_ID,
+ resourceId: emailResourceId,
+ },
+ format: 'JSONEachRow',
+ });
+
+ const [row] = await result.json<{ rowCount: string }>();
+
+ return Number(row?.rowCount ?? 0);
+ };
+
+ const deleteEmailRows = () =>
+ clickHouseClient.command({
+ query: `ALTER TABLE usageEvent DELETE WHERE workspaceId = {workspaceId:String} AND resourceType = {resourceType:String}`,
+ query_params: {
+ workspaceId: SEED_APPLE_WORKSPACE_ID,
+ resourceType: UsageResourceType.EMAIL,
+ },
+ clickhouse_settings: { mutations_sync: '2' },
+ });
+
+ beforeAll(async () => {
+ emailDailyLimit = Number(
+ getAppProviderByClassName<TwentyConfigService>(
+ 'TwentyConfigService',
+ ).get('EMAIL_SEND_WORKSPACE_DAILY_LIMIT'),
+ );
+
+ await deleteEmailRows();
+ await flushQuotaCounters();
+ });
+
+ const waitForChargedEmailRows = () =>
+ expectEventually(
+ async () => {
+ expect(await countEmailRows()).toBe(chargedEmailRowCount);
+ },
+ { timeoutMs: CLICKHOUSE_FLUSH_TIMEOUT_MS, intervalMs: 250 },
+ );
+
+ afterAll(async () => {
+ await waitForChargedEmailRows();
+ await deleteEmailRows();
+ await flushQuotaCounters();
+ }, EXECUTION_TIMEOUT_MS);
+
+ it('refuses at the default and keeps what a deleted override counted on it', async () => {
+ await chargeEmail(emailDailyLimit - 1);
+
+ expect(await findEmailRefusal(1)).toBeNull();
+ expect(await findEmailRefusal(2)).toMatchObject({
+ exhaustedKind: 'limit',
+ isDefault: true,
+ unit: UsageUnit.INVOCATION,
+ limitValue: emailDailyLimit,
+ });
+
+ const overrideUsageLimitId = await saveEmailOverride();
+
+ expect(await findEmailRefusal(2)).toBeNull();
+
+ await debitEmailAheadOfRecord(1);
+
+ await getUsageLimitService().delete({
+ workspaceId: SEED_APPLE_WORKSPACE_ID,
+ usageLimitId: overrideUsageLimitId,
+ isOperator: true,
+ });
+
+ expect(await findEmailRefusal(1)).toMatchObject({
+ exhaustedKind: 'limit',
+ isDefault: true,
+ });
+ });
+ });
+
+ it(
+ 'admits the assert and the charge when Redis fails, counts both and still records the charge',
+ async () => {
+ const { spenderId } = await saveCreditQuotaForNewSpender({
+ limitValue: 0,
+ });
+ const cacheStorage: CacheStorageService =
+ getQuotaService()['cacheStorage'];
+ const incrementCounterBySpy = spyOnMetrics();
+ const runScriptSpy = jest
+ .spyOn(cacheStorage, 'runScript')
+ .mockRejectedValue(new Error('Redis is unreachable'));
+
+ try {
+ expect(await findRunRefusal({ spenderId, costMicro: 1 })).toBeNull();
+ expect(await chargeRun({ spenderId, creditsUsedMicro: 100 })).toEqual({
+ exhaustedKind: null,
+ });
+ expect(findAdmittedOnFailureAttributes(incrementCounterBySpy)).toEqual([
+ {
+ operation: 'assert',
+ resourceType: UsageResourceType.LOGIC_FUNCTION,
+ },
+ {
+ operation: 'consume',
+ resourceType: UsageResourceType.LOGIC_FUNCTION,
+ },
+ ]);
+ } finally {
+ runScriptSpy.mockRestore();
+ incrementCounterBySpy.mockRestore();
+ }
+
+ await expectEventually(
+ async () => {
+ expect(await countUsageEventRows(spenderId)).toBe(1);
+ },
+ { timeoutMs: CLICKHOUSE_FLUSH_TIMEOUT_MS, intervalMs: 250 },
+ );
+ },
+ EXECUTION_TIMEOUT_MS,
+ );
+
+ it('still debits the warm counter and leaves the cold one cold when its ClickHouse read fails', async () => {
+ const { spenderId } = await saveCreditQuotaForNewSpender();
+
+ await chargeRun({ spenderId, creditsUsedMicro: 600 });
+ await saveCodeExecutionQuota({
+ spenderId,
+ unit: UsageUnit.INVOCATION,
+ limitValue: UNREACHABLE_LIMIT_VALUE,
+ });
+ await refreshUsageLimitsCache();
+
+ const incrementCounterBySpy = spyOnMetrics();
+ const clickHouseReadSpy = jest
+ .spyOn(
+ getAppProviderByClassName<UsageAnalyticsService>(
+ 'UsageAnalyticsService',
+ ),
+ 'getConsumptionRowsForAllScopes',
+ )
+ .mockRejectedValue(new Error('ClickHouse is unreachable'));
+
+ try {
+ expect(await chargeRun({ spenderId, creditsUsedMicro: 100 })).toEqual({
+ exhaustedKind: null,
+ });
+ expect(await findSpenderCounterKeys(spenderId)).toHaveLength(1);
+ expect(findAdmittedOnFailureAttributes(incrementCounterBySpy)).toEqual([
+ {
+ operation: 'consume',
+ resourceType: UsageResourceType.LOGIC_FUNCTION,
+ },
+ ]);
+ } finally {
+ clickHouseReadSpy.mockRestore();
+ incrementCounterBySpy.mockRestore();
+ }
+
+ expect(await findRunRefusal({ spenderId, costMicro: 300 })).toBeNull();
+ expect(await findRunRefusal({ spenderId, costMicro: 301 })).toMatchObject({
+ exhaustedKind: 'limit',
+ unit: UsageUnit.CREDIT,
+ });
+ });
});