The open alternative to Salesforce, designed for AI.
TypeScript crmcrm-systemcustomergood-first-issuegraphqlhacktoberfestjavascriptmarketingmonoreponestjsopen-sourcepostgresqlreactreactjssalestypescriptweb
refactor(usage-limit) - quota counters hold consumption; allowance cap from the workspace cache (#27479)
Etienne7a6880d75fd9
diff --git a/packages/twenty-server/src/engine/core-modules/billing-webhook/services/billing-webhook-subscription.service.ts b/packages/twenty-server/src/engine/core-modules/billing-webhook/services/billing-webhook-subscription.service.tsindex d7dbbe67c70..0df578e0195 100644--- a/packages/twenty-server/src/engine/core-modules/billing-webhook/services/billing-webhook-subscription.service.ts+++ b/packages/twenty-server/src/engine/core-modules/billing-webhook/services/billing-webhook-subscription.service.ts@@ -26,7 +26,6 @@ import { BillingSubscriptionItemEntity } from 'src/engine/core-modules/billing/eimport { BillingSubscriptionEntity } from 'src/engine/core-modules/billing/entities/billing-subscription.entity';import { SubscriptionStatus } from 'src/engine/core-modules/billing/enums/billing-subscription-status.enum';import { BillingWebhookEvent } from 'src/engine/core-modules/billing/enums/billing-webhook-events.enum';-import { UsageLimitQuotaService } from 'src/engine/core-modules/usage-limit/services/usage-limit-quota.service';import { StripeCustomerService } from 'src/engine/core-modules/billing/stripe/services/stripe-customer.service';import { StripeSubscriptionScheduleService } from 'src/engine/core-modules/billing/stripe/services/stripe-subscription-schedule.service';import { StripeSubscriptionService } from 'src/engine/core-modules/billing/stripe/services/stripe-subscription.service';@@ -70,7 +69,6 @@ export class BillingWebhookSubscriptionService {private readonly workspaceService: WorkspaceService,private readonly stripeSubscriptionScheduleService: StripeSubscriptionScheduleService,private readonly stripeSubscriptionService: StripeSubscriptionService,- private readonly usageLimitQuotaService: UsageLimitQuotaService,private readonly workspaceCacheService: WorkspaceCacheService,private readonly cacheLockService: CacheLockService,) {}@@ -209,8 +207,6 @@ export class BillingWebhookSubscriptionService {'currentBillingSubscription',]);- await this.usageLimitQuotaService.dropAllowanceCounter(workspace.id);-const shouldSuspendWorkspace = allLiveSubscriptions.every((customerSubscription) =>this.shouldSuspendWorkspace(customerSubscription),diff --git a/packages/twenty-server/src/engine/core-modules/billing/services/billing-credit-allowance-provider.service.ts b/packages/twenty-server/src/engine/core-modules/billing/services/billing-credit-allowance-provider.service.tsindex fddaa412018..e1aed76ae4f 100644--- a/packages/twenty-server/src/engine/core-modules/billing/services/billing-credit-allowance-provider.service.ts+++ b/packages/twenty-server/src/engine/core-modules/billing/services/billing-credit-allowance-provider.service.ts@@ -1,13 +1,11 @@/* @license Enterprise */-import { Injectable, Logger } from '@nestjs/common';+import { Injectable } from '@nestjs/common';import { isDefined } from 'twenty-shared/utils';import { NO_BILLING_SUBSCRIPTION } from 'src/engine/core-modules/billing/constants/no-billing-subscription.constant';-import { BillingCreditGrantService } from 'src/engine/core-modules/billing/services/billing-credit-grant.service';-import { BillingSubscriptionService } from 'src/engine/core-modules/billing/services/billing-subscription.service';-import { BillingUsageService } from 'src/engine/core-modules/billing/services/billing-usage.service';+import { type FlatBillingSubscription } from 'src/engine/core-modules/billing/types/flat-billing-subscription.type';import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';import { CreditAllowanceProvider } from 'src/engine/core-modules/usage-limit/interfaces/credit-allowance-provider.service';import { type CreditAllowance } from 'src/engine/core-modules/usage-limit/types/credit-allowance.type';@@ -16,14 +14,9 @@ import { WorkspaceCacheService } from 'src/engine/workspace-cache/services/works@Injectable()export class BillingCreditAllowanceProvider extends CreditAllowanceProvider {- private readonly logger = new Logger(BillingCreditAllowanceProvider.name);-constructor(private readonly twentyConfigService: TwentyConfigService,private readonly workspaceCacheService: WorkspaceCacheService,- private readonly billingSubscriptionService: BillingSubscriptionService,- private readonly billingCreditGrantService: BillingCreditGrantService,- private readonly billingUsageService: BillingUsageService,) {super();}@@ -35,62 +28,52 @@ export class BillingCreditAllowanceProvider extends CreditAllowanceProvider {async getCreditAllowancePeriod(workspaceId: string,): Promise<UsagePeriod | null> {- if (!this.twentyConfigService.get('IS_BILLING_ENABLED')) {- return null;- }+ const subscription = await this.findCurrentBillingSubscription(workspaceId);- const { currentBillingSubscription } =- await this.workspaceCacheService.getOrRecompute(workspaceId, [- 'currentBillingSubscription',- ]);-- if (currentBillingSubscription === NO_BILLING_SUBSCRIPTION) {+ if (!isDefined(subscription)) {return null;}return {- periodStart: new Date(currentBillingSubscription.currentPeriodStart),- periodEnd: new Date(currentBillingSubscription.currentPeriodEnd),+ periodStart: new Date(subscription.currentPeriodStart),+ periodEnd: new Date(subscription.currentPeriodEnd),};}async getCreditAllowance(workspaceId: string,): Promise<CreditAllowance | null> {- if (!this.twentyConfigService.get('IS_BILLING_ENABLED')) {+ const subscription = await this.findCurrentBillingSubscription(workspaceId);++ if (+ !isDefined(subscription) ||+ !isDefined(subscription.planAllowanceMicro)+ ) {return null;}- try {- const subscription =- await this.billingSubscriptionService.getCurrentBillingSubscription({- workspaceId,- });-- if (!isDefined(subscription)) {- return null;- }-- const { balanceMicro, earliestExpiryBefore } =- await this.billingCreditGrantService.getActiveCreditBalance({- workspaceId,- boundary: subscription.currentPeriodEnd,- });-- return {- periodStart: subscription.currentPeriodStart,- periodEnd: subscription.currentPeriodEnd,- allowanceMicro:- this.billingUsageService.getResourceUsageCap(subscription) +- balanceMicro,- validUntil: earliestExpiryBefore ?? subscription.currentPeriodEnd,- };- } catch (error) {- this.logger.error(- `Could not compute allowance for workspace ${workspaceId}: ${error instanceof Error ? error.message : 'unknown error'}`,- );+ return {+ periodStart: new Date(subscription.currentPeriodStart),+ periodEnd: new Date(subscription.currentPeriodEnd),+ planAllowanceMicro: subscription.planAllowanceMicro,+ grants: subscription.creditGrants,+ };+ }+ private async findCurrentBillingSubscription(+ workspaceId: string,+ ): Promise<FlatBillingSubscription | null> {+ if (!this.twentyConfigService.get('IS_BILLING_ENABLED')) {return null;}++ const { currentBillingSubscription } =+ await this.workspaceCacheService.getOrRecompute(workspaceId, [+ 'currentBillingSubscription',+ ]);++ return currentBillingSubscription === NO_BILLING_SUBSCRIPTION+ ? null+ : currentBillingSubscription;}}diff --git a/packages/twenty-server/src/engine/core-modules/billing/services/billing-credit-grant.service.ts b/packages/twenty-server/src/engine/core-modules/billing/services/billing-credit-grant.service.tsindex 8b96b31a4eb..cf3e00530b7 100644--- a/packages/twenty-server/src/engine/core-modules/billing/services/billing-credit-grant.service.ts+++ b/packages/twenty-server/src/engine/core-modules/billing/services/billing-credit-grant.service.ts@@ -99,31 +99,9 @@ export class BillingCreditGrantService {}async getActiveCreditsMicro(workspaceId: string): Promise<number> {- const { balanceMicro } = await this.getActiveCreditBalance({- workspaceId,- boundary: null,- });-- return balanceMicro;- }-- // One statement so the balance and its bounding expiry come from the same snapshot- async getActiveCreditBalance({- workspaceId,- boundary,- }: {- workspaceId: string;- boundary: Date | null;- }): Promise<{ balanceMicro: number; earliestExpiryBefore: Date | null }> {const result = await this.billingCreditGrantRepository.createQueryBuilder('billingCreditGrant').select('COALESCE(SUM("billingCreditGrant"."amountMicro"), 0)', 'total')- .addSelect(- isDefined(boundary)- ? 'MIN("billingCreditGrant"."expiresAt") FILTER (WHERE "billingCreditGrant"."expiresAt" < :boundary)'- : 'NULL',- 'earliestExpiry',- ).where('"billingCreditGrant"."workspaceId" = :workspaceId', {workspaceId,})@@ -132,11 +110,7 @@ export class BillingCreditGrantService {.andWhere('("billingCreditGrant"."expiresAt" IS NULL OR "billingCreditGrant"."expiresAt" > now())',)- .setParameters(isDefined(boundary) ? { boundary } : {})- .getRawOne<{- total: string | number | null;- earliestExpiry: Date | string | null;- }>();+ .getRawOne<{ total: string | number | null }>();const balanceMicro = Number(result?.total ?? 0);@@ -148,12 +122,18 @@ export class BillingCreditGrantService {);}- return {- balanceMicro,- earliestExpiryBefore: isDefined(result?.earliestExpiry)- ? new Date(result.earliestExpiry)- : null,- };+ return balanceMicro;+ }++ async findUnexpiredGrants(+ workspaceId: string,+ ): Promise<BillingCreditGrantEntity[]> {+ return this.billingCreditGrantRepository.find(workspaceId, {+ where: [+ { revokedAt: IsNull(), expiresAt: IsNull() },+ { revokedAt: IsNull(), expiresAt: MoreThan(new Date()) },+ ],+ });}async findGrantsLiveDuringPeriod(diff --git a/packages/twenty-server/src/engine/core-modules/billing/services/billing-credit.service.ts b/packages/twenty-server/src/engine/core-modules/billing/services/billing-credit.service.tsindex 12b3df4b785..093aac38fc6 100644--- a/packages/twenty-server/src/engine/core-modules/billing/services/billing-credit.service.ts+++ b/packages/twenty-server/src/engine/core-modules/billing/services/billing-credit.service.ts@@ -17,7 +17,6 @@ import { BillingService } from 'src/engine/core-modules/billing/services/billingimport { alignGrantExpiryToPeriodEnd } from 'src/engine/core-modules/billing/utils/align-grant-expiry-to-period-end.util';import { buildBillingCreditStateLockKey } from 'src/engine/core-modules/billing/utils/build-billing-credit-state-lock-key.util';import { CacheLockService } from 'src/engine/core-modules/cache-lock/cache-lock.service';-import { UsageLimitQuotaService } from 'src/engine/core-modules/usage-limit/services/usage-limit-quota.service';import { WorkspaceCacheService } from 'src/engine/workspace-cache/services/workspace-cache.service';type GrantCreditsParams = {@@ -43,7 +42,6 @@ export class BillingCreditService {private readonly billingSubscriptionService: BillingSubscriptionService,private readonly cacheLockService: CacheLockService,private readonly workspaceCacheService: WorkspaceCacheService,- private readonly usageLimitQuotaService: UsageLimitQuotaService,) {}async grantCredits(@@ -166,8 +164,6 @@ export class BillingCreditService {await this.workspaceCacheService.invalidateAndRecompute(workspaceId, ['currentBillingSubscription',]);-- await this.usageLimitQuotaService.dropAllowanceCounter(workspaceId);}}diff --git a/packages/twenty-server/src/engine/core-modules/billing/services/billing-subscription.service.ts b/packages/twenty-server/src/engine/core-modules/billing/services/billing-subscription.service.tsindex ade70f1194e..a342eae31b0 100644--- a/packages/twenty-server/src/engine/core-modules/billing/services/billing-subscription.service.ts+++ b/packages/twenty-server/src/engine/core-modules/billing/services/billing-subscription.service.ts@@ -29,7 +29,6 @@ import { BillingEntitlementKey } from 'src/engine/core-modules/billing/enums/bilimport { WORKSPACE_ACTIVATING_SUBSCRIPTION_STATUSES } from 'src/engine/core-modules/billing/constants/workspace-activating-subscription-statuses.constant';import { SubscriptionStatus } from 'src/engine/core-modules/billing/enums/billing-subscription-status.enum';import { BillingPriceService } from 'src/engine/core-modules/billing/services/billing-price.service';-import { UsageLimitQuotaService } from 'src/engine/core-modules/usage-limit/services/usage-limit-quota.service';import { StripeCustomerService } from 'src/engine/core-modules/billing/stripe/services/stripe-customer.service';import { StripeSubscriptionScheduleService } from 'src/engine/core-modules/billing/stripe/services/stripe-subscription-schedule.service';import { StripeSubscriptionService } from 'src/engine/core-modules/billing/stripe/services/stripe-subscription.service';@@ -68,7 +67,6 @@ export class BillingSubscriptionService {private readonly billingCustomerRepository: WorkspaceScopedRepository<BillingCustomerEntity>,private readonly enterprisePlanService: EnterprisePlanService,private readonly workspaceCacheService: WorkspaceCacheService,- private readonly usageLimitQuotaService: UsageLimitQuotaService,) {}async getBillingSubscriptions(workspaceId: string) {@@ -323,12 +321,6 @@ export class BillingSubscriptionService {updatedSubscription.id,);- await this.workspaceCacheService.invalidateAndRecompute(workspace.id, [- 'currentBillingSubscription',- ]);-- await this.usageLimitQuotaService.dropAllowanceCounter(workspace.id);-return {status: getSubscriptionStatus(updatedSubscription.status),hasPaymentMethod: true,@@ -471,6 +463,10 @@ export class BillingSubscriptionService {}}+ await this.workspaceCacheService.invalidateAndRecompute(workspaceId, [+ 'currentBillingSubscription',+ ]);+return currentBillingSubscription;}diff --git a/packages/twenty-server/src/engine/core-modules/billing/services/billing-usage.service.ts b/packages/twenty-server/src/engine/core-modules/billing/services/billing-usage.service.tsindex b3325db8ad7..301ea276120 100644--- a/packages/twenty-server/src/engine/core-modules/billing/services/billing-usage.service.ts+++ b/packages/twenty-server/src/engine/core-modules/billing/services/billing-usage.service.ts@@ -241,7 +241,7 @@ export class BillingUsageService {);}- getResourceUsageCap(subscription: BillingSubscriptionEntity): number {+ findResourceUsageCap(subscription: BillingSubscriptionEntity): number | null {const isInFreeTrial = subscription.status === SubscriptionStatus.Trialing;if (isInFreeTrial) {@@ -260,10 +260,7 @@ export class BillingUsageService {);if (!isDefined(resourceCreditPrice)) {- throw new BillingException(- `Resource credit price not found for workspace ${subscription.workspaceId}`,- BillingExceptionCode.BILLING_PRICE_NOT_FOUND,- );+ return null;}return Number(resourceCreditPrice.metadata?.credit_amount ?? 0);diff --git a/packages/twenty-server/src/engine/core-modules/billing/services/workspace-current-billing-subscription-cache.service.ts b/packages/twenty-server/src/engine/core-modules/billing/services/workspace-current-billing-subscription-cache.service.tsindex 186b9051c5a..85647b6a8b6 100644--- a/packages/twenty-server/src/engine/core-modules/billing/services/workspace-current-billing-subscription-cache.service.ts+++ b/packages/twenty-server/src/engine/core-modules/billing/services/workspace-current-billing-subscription-cache.service.ts@@ -4,7 +4,9 @@ import { Injectable } from '@nestjs/common';import { isDefined } from 'twenty-shared/utils';+import { BillingCreditGrantService } from 'src/engine/core-modules/billing/services/billing-credit-grant.service';import { BillingSubscriptionService } from 'src/engine/core-modules/billing/services/billing-subscription.service';+import { BillingUsageService } from 'src/engine/core-modules/billing/services/billing-usage.service';import { NO_BILLING_SUBSCRIPTION } from 'src/engine/core-modules/billing/constants/no-billing-subscription.constant';import { type CurrentBillingSubscription } from 'src/engine/core-modules/billing/types/flat-billing-subscription.type';@@ -17,6 +19,8 @@ import { type WorkspaceCacheProviderContext } from 'src/engine/workspace-cache/texport class WorkspaceCurrentBillingSubscriptionCacheService extends WorkspaceCacheProvider<CurrentBillingSubscription> {constructor(private readonly billingSubscriptionService: BillingSubscriptionService,+ private readonly billingCreditGrantService: BillingCreditGrantService,+ private readonly billingUsageService: BillingUsageService,) {super();}@@ -33,6 +37,9 @@ export class WorkspaceCurrentBillingSubscriptionCacheService extends WorkspaceCareturn NO_BILLING_SUBSCRIPTION;}+ const unexpiredGrants =+ await this.billingCreditGrantService.findUnexpiredGrants(workspaceId);+return {id: subscription.id,workspaceId: subscription.workspaceId,@@ -50,6 +57,15 @@ export class WorkspaceCurrentBillingSubscriptionCacheService extends WorkspaceCatrialStart: subscription.trialStart,trialEnd: subscription.trialEnd,collectionMethod: subscription.collectionMethod,+ planAllowanceMicro:+ this.billingUsageService.findResourceUsageCap(subscription),+ creditGrants: unexpiredGrants.map(+ ({ amountMicro, effectiveAt, expiresAt }) => ({+ amountMicro,+ effectiveAtMs: effectiveAt.getTime(),+ expiresAtMs: expiresAt?.getTime() ?? null,+ }),+ ),};}}diff --git a/packages/twenty-server/src/engine/core-modules/billing/types/flat-billing-subscription.type.ts b/packages/twenty-server/src/engine/core-modules/billing/types/flat-billing-subscription.type.tsindex a130a971d91..c98ceff99f1 100644--- a/packages/twenty-server/src/engine/core-modules/billing/types/flat-billing-subscription.type.ts+++ b/packages/twenty-server/src/engine/core-modules/billing/types/flat-billing-subscription.type.ts@@ -4,6 +4,7 @@ import { NO_BILLING_SUBSCRIPTION } from 'src/engine/core-modules/billing/constanimport { type BillingSubscriptionCollectionMethod } from 'src/engine/core-modules/billing/enums/billing-subscription-collection-method.enum';import { type SubscriptionInterval } from 'src/engine/core-modules/billing/enums/billing-subscription-interval.enum';import { type SubscriptionStatus } from 'src/engine/core-modules/billing/enums/billing-subscription-status.enum';+import { type CreditAllowanceGrant } from 'src/engine/core-modules/usage-limit/types/credit-allowance-grant.type';export type FlatBillingSubscription = {id: string;@@ -22,6 +23,8 @@ export type FlatBillingSubscription = {trialStart: Date | null;trialEnd: Date | null;collectionMethod: BillingSubscriptionCollectionMethod;+ planAllowanceMicro: number | null;+ creditGrants: CreditAllowanceGrant[];};export type CurrentBillingSubscription =diff --git a/packages/twenty-server/src/engine/core-modules/usage-limit/constants/warm-and-consume-quota-counters-script.constant.ts b/packages/twenty-server/src/engine/core-modules/usage-limit/constants/warm-and-consume-quota-counters-script.constant.tsnew file mode 100644index 00000000000..5a1da4acd73--- /dev/null+++ b/packages/twenty-server/src/engine/core-modules/usage-limit/constants/warm-and-consume-quota-counters-script.constant.ts@@ -0,0 +1,25 @@+import { type CacheScript } from 'src/engine/core-modules/cache-storage/types/cache-script.type';++export const WARM_AND_CONSUME_QUOTA_COUNTERS_SCRIPT: CacheScript = {+ name: 'usage-limit:warm-and-consume-quota-counters',+ source: `+local amounts = cjson.decode(ARGV[1])+local warmValues = cjson.decode(ARGV[2])+local warmTtls = cjson.decode(ARGV[3])+local result = {}++for i = 1, #KEYS do+ if warmValues[i] then+ redis.call('SET', KEYS[i], warmValues[i], 'NX', 'PX', warmTtls[i])+ end++ if redis.call('EXISTS', KEYS[i]) == 1 then+ result[i] = redis.call('INCRBY', KEYS[i], amounts[i])+ else+ result[i] = false+ end+end++return result+`,+};diff --git a/packages/twenty-server/src/engine/core-modules/usage-limit/services/usage-limit-quota.service.ts b/packages/twenty-server/src/engine/core-modules/usage-limit/services/usage-limit-quota.service.tsindex 80ef659e4aa..c0a82bc35de 100644--- a/packages/twenty-server/src/engine/core-modules/usage-limit/services/usage-limit-quota.service.ts+++ b/packages/twenty-server/src/engine/core-modules/usage-limit/services/usage-limit-quota.service.ts@@ -2,20 +2,15 @@ import { Injectable, Logger, type OnModuleInit } from '@nestjs/common';import { DiscoveryService } from '@nestjs/core';import uniqBy from 'lodash.uniqby';-import { isDefined } from 'twenty-shared/utils';+import { isDefined, isNonEmptyArray } from 'twenty-shared/utils';-import { CacheLockService } from 'src/engine/core-modules/cache-lock/cache-lock.service';-import {- CacheLockException,- CacheLockExceptionCode,-} from 'src/engine/core-modules/cache-lock/exceptions/cache-lock.exception';import { InjectCacheStorage } from 'src/engine/core-modules/cache-storage/decorators/cache-storage.decorator';import { CacheStorageService } from 'src/engine/core-modules/cache-storage/services/cache-storage.service';import { CacheStorageNamespace } from 'src/engine/core-modules/cache-storage/types/cache-storage-namespace.enum';import { MetricsService } from 'src/engine/core-modules/metrics/metrics.service';import { MetricsKeys } from 'src/engine/core-modules/metrics/types/metrics-keys.type';import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';-import { CONSUME_QUOTA_COUNTERS_SCRIPT } from 'src/engine/core-modules/usage-limit/constants/consume-quota-counters-script.constant';+import { WARM_AND_CONSUME_QUOTA_COUNTERS_SCRIPT } from 'src/engine/core-modules/usage-limit/constants/warm-and-consume-quota-counters-script.constant';import { UsageLimitException } from 'src/engine/core-modules/usage-limit/exceptions/usage-limit.exception';import { CreditAllowanceProvider } from 'src/engine/core-modules/usage-limit/interfaces/credit-allowance-provider.service';import { UsageLimitEntitlementService } from 'src/engine/core-modules/usage-limit/services/usage-limit-entitlement.service';@@ -30,26 +25,22 @@ import { type LimitQuotaCounter } from 'src/engine/core-modules/usage-limit/typeimport { type QuotaCost } from 'src/engine/core-modules/usage-limit/types/quota-cost.type';import { type QuotaCounter } from 'src/engine/core-modules/usage-limit/types/quota-counter.type';import { type QuotaLimitDefault } from 'src/engine/core-modules/usage-limit/types/quota-limit-default.type';-import { type UsageLimitCounterScope } from 'src/engine/core-modules/usage-limit/types/usage-limit-counter-scope.type';import { buildAllowanceCounterKey } from 'src/engine/core-modules/usage-limit/utils/build-allowance-counter-key.util';+import { buildConsumeScriptArguments } from 'src/engine/core-modules/usage-limit/utils/build-consume-script-arguments.util';import { buildIntraWorkspaceLimitCounterKeys } from 'src/engine/core-modules/usage-limit/utils/build-intra-workspace-limit-counter-keys.util';import { buildLimitQuotaCounter } from 'src/engine/core-modules/usage-limit/utils/build-limit-quota-counter.util';-import { buildLimitWarmedEntries } from 'src/engine/core-modules/usage-limit/utils/build-limit-warmed-entries.util';-import { buildOverriddenQuotaDefaultCounterKeys } from 'src/engine/core-modules/usage-limit/utils/build-overridden-quota-default-counter-keys.util';import { buildPeriodGroupKey } from 'src/engine/core-modules/usage-limit/utils/build-period-group-key.util';-import { buildQuotaCounterKey } from 'src/engine/core-modules/usage-limit/utils/build-quota-counter-key.util';import { buildQuotaCounters } from 'src/engine/core-modules/usage-limit/utils/build-quota-counters.util';import { buildQuotaDebits } from 'src/engine/core-modules/usage-limit/utils/build-quota-debits.util';import { buildQuotaExhaustedScope } from 'src/engine/core-modules/usage-limit/utils/build-quota-exhausted-scope.util';-import { buildQuotaWarmLockKey } from 'src/engine/core-modules/usage-limit/utils/build-quota-warm-lock-key.util';import { buildSpendersFromUsageSpenders } from 'src/engine/core-modules/usage-limit/utils/build-spenders-from-usage-spenders.util';+import { computeCreditAllowanceMicro } from 'src/engine/core-modules/usage-limit/utils/compute-credit-allowance-micro.util';import { computeQuotaConsumed } from 'src/engine/core-modules/usage-limit/utils/compute-quota-consumed.util';import { findCreditAllowanceProvider } from 'src/engine/core-modules/usage-limit/utils/find-credit-allowance-provider.util';import { findExhaustedCounters } from 'src/engine/core-modules/usage-limit/utils/find-exhausted-counters.util';+import { findSuppressedUsageLimitDefaults } from 'src/engine/core-modules/usage-limit/utils/find-suppressed-usage-limit-defaults.util';import { findUsageLimitDefinition } from 'src/engine/core-modules/usage-limit/utils/find-usage-limit-definition.util';-import { isAnchoredPeriodUnit } from 'src/engine/core-modules/usage-limit/utils/is-anchored-period-unit.util';import { isQuotaLimit } from 'src/engine/core-modules/usage-limit/utils/is-quota-limit.util';-import { fromConsumeResultsToRemainings } from 'src/engine/core-modules/usage-limit/utils/from-consume-results-to-remainings.util';import { getPeriodAnchor } from 'src/engine/core-modules/usage-limit/utils/get-period-anchor.util';import { type UsageOperationType } from 'src/engine/core-modules/usage/enums/usage-operation-type.enum';import { type UsageResourceType } from 'src/engine/core-modules/usage/enums/usage-resource-type.enum';@@ -63,8 +54,6 @@ import { fromRecordUsageInputToUsageConsumptionRow } from 'src/engine/core-modulimport { WorkspaceCacheException } from 'src/engine/workspace-cache/exceptions/workspace-cache.exception';import { WorkspaceCacheService } from 'src/engine/workspace-cache/services/workspace-cache.service';-const QUOTA_WARM_LOCK_OPTIONS = { ms: 50, maxRetries: 20, ttl: 10_000 };-type QuotaEnforcement = {operation: 'assert' | 'consume';resourceType: UsageResourceType;@@ -78,6 +67,11 @@ type QuotaConsumeArgs = {spenders: UsageSpenders;};+type QuotaConsumeResult = {+ consumedValues: (number | null)[];+ isDegraded: boolean;+};+@Injectable()export class UsageLimitQuotaService implements OnModuleInit {private readonly logger = new Logger(UsageLimitQuotaService.name);@@ -88,7 +82,6 @@ export class UsageLimitQuotaService implements OnModuleInit {@InjectCacheStorage(CacheStorageNamespace.EngineUsageLimit)private readonly cacheStorage: CacheStorageService,private readonly workspaceCacheService: WorkspaceCacheService,- private readonly cacheLockService: CacheLockService,private readonly usagePeriodService: UsagePeriodService,private readonly usageAnalyticsService: UsageAnalyticsService,private readonly usageRecorderService: UsageRecorderService,@@ -153,27 +146,6 @@ export class UsageLimitQuotaService implements OnModuleInit {};}- async dropAllowanceCounter(workspaceId: string): Promise<void> {- const period =- await this.creditAllowanceProvider.getCreditAllowancePeriod(workspaceId);-- if (!isDefined(period)) {- return;- }-- await this.delUnderWarmLock({- workspaceId,- keys: [- buildAllowanceCounterKey({- workspaceId,- periodStart: period.periodStart,- }),- ],- });- }-- // Intra-workspace counters are not debited while the entitlement is off, so- // a warm balance misses that usage; dropping them forces a ClickHouse rewarm.async dropIntraWorkspaceLimitCounters(workspaceId: string): Promise<void> {const limits = await this.findAllLimits(workspaceId);@@ -190,66 +162,39 @@ export class UsageLimitQuotaService implements OnModuleInit {return;}- await this.delUnderWarmLock({ workspaceId, keys });+ await this.cacheStorage.mdel(keys);}- async dropLimitCounter(usageLimit: UsageLimitCounterScope): Promise<void> {+ async dropLimitCounter({+ workspaceId,+ usageLimit,+ }: {+ workspaceId: string;+ usageLimit: FlatUsageLimit;+ }): Promise<void> {if (- usageLimit.limitKind !== 'quota' ||- !isAnchoredPeriodUnit(usageLimit.periodUnit)+ !isQuotaLimit(usageLimit) ||+ isNonEmptyArray(findSuppressedUsageLimitDefaults(usageLimit))) {return;}- const quotaLimitDefaults = this.buildQuotaLimitDefaults(- usageLimit.resourceType,- );-const periodByUnit = await this.usagePeriodService.findCurrentPeriodsByUnit({- workspaceId: usageLimit.workspaceId,- periodUnits: [- usageLimit.periodUnit,- ...quotaLimitDefaults.map(- (quotaLimitDefault) => quotaLimitDefault.periodUnit,- ),- ],+ workspaceId,+ periodUnits: [usageLimit.periodUnit],},);const period = periodByUnit[usageLimit.periodUnit];- const keys = [- ...(isDefined(period)- ? [- buildQuotaCounterKey({- workspaceId: usageLimit.workspaceId,- resourceType: usageLimit.resourceType,- operationType: usageLimit.operationType,- spenderType: usageLimit.spenderType,- spenderId: usageLimit.spenderId,- unit: usageLimit.unit,- periodUnit: usageLimit.periodUnit,- periodStart: period.periodStart,- limitValue: usageLimit.limitValue,- }),- ]- : []),- ...buildOverriddenQuotaDefaultCounterKeys({- usageLimit,- quotaLimitDefaults,- periodByUnit,- }),- ];-- if (keys.length === 0) {+ if (!isDefined(period)) {return;}- await this.delUnderWarmLock({- workspaceId: usageLimit.workspaceId,- keys,- });+ await this.cacheStorage.del(+ buildLimitQuotaCounter({ workspaceId, limit: usageLimit, period }).key,+ );}async getAllowanceUsage(workspaceId: string): Promise<{@@ -263,40 +208,25 @@ export class UsageLimitQuotaService implements OnModuleInit {return null;}- const allowance =- await this.creditAllowanceProvider.getCreditAllowance(workspaceId);-- if (!isDefined(allowance)) {- return null;- }-- if (allowance.periodStart.getTime() !== counter.periodStart.getTime()) {- return {- limitValue: allowance.allowanceMicro,- consumedValue: null,- periodEnd: allowance.periodEnd,- };- }-let consumedValue: number | null = null;try {- const [remaining] = await this.readRemainings({- workspaceId,- counters: [counter],- });+ const [storedValue] = await this.cacheStorage.mget<number>([counter.key]);- consumedValue = isDefined(remaining)- ? Math.max(0, allowance.allowanceMicro - remaining)- : null;+ consumedValue =+ storedValue ??+ (await this.usageAnalyticsService.getCreditsUsedMicroForBillingPeriod({+ workspaceId,+ periodStart: counter.periodStart,+ }));} catch (error) {this.admitOnFailure({ error, workspaceId, admitted: null });}return {- limitValue: allowance.allowanceMicro,+ limitValue: counter.limitValue,consumedValue,- periodEnd: allowance.periodEnd,+ periodEnd: counter.periodEnd,};}@@ -304,18 +234,23 @@ export class UsageLimitQuotaService implements OnModuleInit {workspaceId: string,): Promise<number | null> {try {- const allowanceCounter = await this.buildAllowanceCounter(workspaceId);+ const counter = await this.buildAllowanceCounter(workspaceId);- if (!isDefined(allowanceCounter)) {+ if (!isDefined(counter)) {return null;}- const [remaining] = await this.readRemainings({+ const {+ consumedValues: [consumedValue],+ } = await this.consumeCounters({workspaceId,- counters: [allowanceCounter],+ counters: [counter],+ amounts: [0],});- return remaining;+ return isDefined(consumedValue)+ ? counter.limitValue - consumedValue+ : null;} catch (error) {return this.admitOnFailure({ error, workspaceId, admitted: null });}@@ -377,35 +312,6 @@ export class UsageLimitQuotaService implements OnModuleInit {);}- private async delUnderWarmLock({- workspaceId,- keys,- }: {- workspaceId: string;- keys: string[];- }): Promise<void> {- try {- await this.cacheLockService.withLock(- () => this.cacheStorage.mdel(keys),- buildQuotaWarmLockKey(workspaceId),- QUOTA_WARM_LOCK_OPTIONS,- );- } catch (error) {- if (- !(error instanceof CacheLockException) ||- error.code !== CacheLockExceptionCode.LOCK_ACQUISITION_TIMEOUT- ) {- throw error;- }-- this.logger.warn(- `Dropping quota counters ${keys.join(', ')} without the warm lock: ${error.message}`,- );-- await this.cacheStorage.mdel(keys);- }- }-private async findAllLimits(workspaceId: string): Promise<FlatUsageLimit[]> {const { usageLimits } = await this.workspaceCacheService.getOrRecompute(workspaceId,@@ -421,29 +327,45 @@ export class UsageLimitQuotaService implements OnModuleInit {cost,...args}: QuotaConsumeArgs & { cost?: QuotaCost }): Promise<ExhaustedScope[]> {- try {- const counters = await this.buildCounters(args);-- if (counters.length === 0) {- return [];- }+ const enforcement: QuotaEnforcement = {+ operation: 'assert',+ resourceType: args.resourceType,+ };- const remainings = await this.readRemainings({+ try {+ const [limitCounters, { allowanceCounter, isAllowanceSkipped }] =+ await Promise.all([+ this.buildLimitCounters(args),+ this.buildAllowanceCounterAdmittingOnFailure(args.workspaceId),+ ]);++ const counters: QuotaCounter[] = [+ ...limitCounters,+ ...(isDefined(allowanceCounter) ? [allowanceCounter] : []),+ ];++ const { consumedValues, isDegraded } = await this.consumeCounters({workspaceId: args.workspaceId,counters,+ amounts: counters.map(() => 0),});- return await this.buildExhaustedScopes({- args,- counters,- remainings,- cost,- });+ if (isDegraded || isAllowanceSkipped) {+ this.countAdmitOnFailure(enforcement);+ }++ return findExhaustedCounters({ counters, consumedValues, cost }).map(+ (counter) =>+ buildQuotaExhaustedScope({+ resourceType: args.resourceType,+ counter,+ }),+ );} catch (error) {return this.admitOnFailure({error,workspaceId: args.workspaceId,- enforcement: { operation: 'assert', resourceType: args.resourceType },+ enforcement,admitted: [],});}@@ -457,31 +379,45 @@ export class UsageLimitQuotaService implements OnModuleInit {events: RecordUsageInput[];}): Promise<ExhaustedKind | null> {try {+ const [limitCounters, { allowanceCounter, isAllowanceSkipped }] =+ await Promise.all([+ this.buildLimitCountersForEvents({ workspaceId, events }),+ this.buildAllowanceCounterAdmittingOnFailure(workspaceId),+ ]);+const debits = buildQuotaDebits({- counters: await this.buildCountersForEvents({ workspaceId, events }),+ counters: [+ ...limitCounters,+ ...(isDefined(allowanceCounter) ? [allowanceCounter] : []),+ ],events,});- if (debits.length === 0) {- return null;- }-const counters = debits.map((debit) => debit.counter);- // The consume script only debits keys that exist: warm cold counters- // first so a consume-only caller (workflow, logic function) is metered- // from its first call instead of waiting for an assert to warm the key.- await this.readRemainings({ workspaceId, counters });-- const consumeResults = await this.cacheStorage.runScript<number[]>({- script: CONSUME_QUOTA_COUNTERS_SCRIPT,- keys: counters.map((counter) => counter.key),- args: [JSON.stringify(debits.map((debit) => debit.amount))],+ const { consumedValues, isDegraded } = await this.consumeCounters({+ workspaceId,+ counters,+ amounts: debits.map((debit) => debit.amount),});+ if (isDegraded || isAllowanceSkipped) {+ this.countConsumeAdmittedOnFailure({+ events,+ isAllowanceUnmetered:+ isAllowanceSkipped ||+ counters.some(+ (counter, index) =>+ counter.kind === 'allowance' &&+ !isDefined(consumedValues[index]) &&+ counter.periodEnd.getTime() > Date.now(),+ ),+ });+ }+const exhaustedCounters = findExhaustedCounters({counters,- remainings: fromConsumeResultsToRemainings(consumeResults),+ consumedValues,});if (exhaustedCounters.some((counter) => counter.kind === 'allowance')) {@@ -493,38 +429,22 @@ export class UsageLimitQuotaService implements OnModuleInit {this.logger.error(`Usage quota debit degraded for workspace ${workspaceId}: ${error instanceof Error ? error.message : 'unknown error'}`,);-- for (const resourceType of new Set(- events.map((event) => event.resourceType),- )) {- this.countAdmitOnFailure({- operation: 'consume',- resourceType,- creditsUsedMicro: events- .filter((event) => event.resourceType === resourceType)- .reduce(- (total, event) =>- total +- Number(- fromRecordUsageInputToUsageConsumptionRow(event)- .creditsUsedMicro,- ),- 0,- ),- });- }+ this.countConsumeAdmittedOnFailure({+ events,+ isAllowanceUnmetered: true,+ });return null;}}- private async buildCountersForEvents({+ private async buildLimitCountersForEvents({workspaceId,events,}: {workspaceId: string;events: RecordUsageInput[];- }): Promise<QuotaCounter[]> {+ }): Promise<LimitQuotaCounter[]> {const scopeEvents = uniqBy(events, (event) =>JSON.stringify([event.resourceType,@@ -535,7 +455,7 @@ export class UsageLimitQuotaService implements OnModuleInit {const counters = await Promise.all(scopeEvents.map(({ resourceType, operationType, spenders }) =>- this.buildCounters({+ this.buildLimitCounters({workspaceId,resourceType,operationType,@@ -547,40 +467,34 @@ export class UsageLimitQuotaService implements OnModuleInit {return uniqBy(counters.flat(), (counter) => counter.key);}- private async buildExhaustedScopes({- args,- counters,- remainings,- cost,+ private countConsumeAdmittedOnFailure({+ events,+ isAllowanceUnmetered,}: {- args: QuotaConsumeArgs;- counters: QuotaCounter[];- remainings: (number | null)[];- cost?: QuotaCost;- }): Promise<ExhaustedScope[]> {- const exhaustedCounters = findExhaustedCounters({- counters,- remainings,- cost,- });-- if (exhaustedCounters.length === 0) {- return [];+ events: RecordUsageInput[];+ isAllowanceUnmetered: boolean;+ }): void {+ for (const resourceType of new Set(+ events.map((event) => event.resourceType),+ )) {+ this.countAdmitOnFailure({+ operation: 'consume',+ resourceType,+ creditsUsedMicro: isAllowanceUnmetered+ ? events+ .filter((event) => event.resourceType === resourceType)+ .reduce(+ (total, event) =>+ total ++ Number(+ fromRecordUsageInputToUsageConsumptionRow(event)+ .creditsUsedMicro,+ ),+ 0,+ )+ : undefined,+ });}-- const allowance = exhaustedCounters.some(- (counter) => counter.kind === 'allowance',- )- ? await this.creditAllowanceProvider.getCreditAllowance(args.workspaceId)- : null;-- return exhaustedCounters.map((counter) =>- buildQuotaExhaustedScope({- resourceType: args.resourceType,- counter,- allowance,- }),- );}private admitOnFailure<TAdmitted>({@@ -634,17 +548,6 @@ export class UsageLimitQuotaService implements OnModuleInit {}}- private async buildCounters(args: QuotaConsumeArgs): Promise<QuotaCounter[]> {- const [limitCounters, allowanceCounter] = await Promise.all([- this.buildLimitCounters(args),- this.buildAllowanceCounter(args.workspaceId),- ]);-- return isDefined(allowanceCounter)- ? [...limitCounters, allowanceCounter]- : limitCounters;- }-private async buildLimitCounters({workspaceId,resourceType,@@ -709,6 +612,30 @@ export class UsageLimitQuotaService implements OnModuleInit {);}+ private async buildAllowanceCounterAdmittingOnFailure(+ workspaceId: string,+ ): Promise<{+ allowanceCounter: AllowanceQuotaCounter | null;+ isAllowanceSkipped: boolean;+ }> {+ try {+ return {+ allowanceCounter: await this.buildAllowanceCounter(workspaceId),+ isAllowanceSkipped: false,+ };+ } catch (error) {+ if (error instanceof WorkspaceCacheException) {+ throw error;+ }++ this.logger.error(+ `Usage quota allowance skipped for workspace ${workspaceId}: ${error instanceof Error ? error.message : 'unknown error'}`,+ );++ return { allowanceCounter: null, isAllowanceSkipped: true };+ }+ }+private async buildAllowanceCounter(workspaceId: string,): Promise<AllowanceQuotaCounter | null> {@@ -716,12 +643,10 @@ export class UsageLimitQuotaService implements OnModuleInit {return null;}- const period = await this.usagePeriodService.findCurrentPeriod({- workspaceId,- periodUnit: 'allowancePeriod',- });+ const allowance =+ await this.creditAllowanceProvider.getCreditAllowance(workspaceId);- if (!isDefined(period)) {+ if (!isDefined(allowance)) {return null;}@@ -729,11 +654,16 @@ export class UsageLimitQuotaService implements OnModuleInit {kind: 'allowance',key: buildAllowanceCounterKey({workspaceId,- periodStart: period.periodStart,+ periodStart: allowance.periodStart,+ }),+ limitValue: computeCreditAllowanceMicro({+ planAllowanceMicro: allowance.planAllowanceMicro,+ grants: allowance.grants,+ nowMs: Date.now(),}),unit: UsageUnit.CREDIT,- periodStart: period.periodStart,- periodEnd: period.periodEnd,+ periodStart: allowance.periodStart,+ periodEnd: allowance.periodEnd,};}@@ -784,207 +714,147 @@ export class UsageLimitQuotaService implements OnModuleInit {workspaceId: string;counters: LimitQuotaCounter[];}): Promise<(number | null)[]> {- const remainings = await this.cacheStorage.mget<number>(+ const storedValues = await this.cacheStorage.mget<number>(counters.map((counter) => counter.key),);- const coldLimitCounters = counters.filter(- (_, index) => !isDefined(remainings[index]),+ const coldCounters = counters.filter(+ (_, index) => !isDefined(storedValues[index]),);- const rowsByPeriod =- coldLimitCounters.length > 0- ? await this.fetchConsumptionRowsByPeriod({- workspaceId,- coldLimitCounters,- })- : new Map<string, UsageConsumptionRow[]>();-- return counters.map((counter, index) => {- const remaining = remainings[index];-- if (isDefined(remaining)) {- return counter.limitValue - remaining;- }-- const rows = rowsByPeriod.get(buildPeriodGroupKey(counter));-- return isDefined(rows)- ? computeQuotaConsumed({ rows, scope: counter })- : null;+ const warmValues = await this.fetchWarmValues({+ workspaceId,+ counters: coldCounters,});- }- private async readRemainings({- workspaceId,- counters,- }: {- workspaceId: string;- counters: QuotaCounter[];- }): Promise<(number | null)[]> {- const remainings = await this.cacheStorage.mget<number>(- counters.map((counter) => counter.key),+ const warmedValueByKey = new Map(+ coldCounters.map((counter, index) => [counter.key, warmValues[index]]),);- const warmRemainings = remainings.filter(isDefined);-- if (warmRemainings.length === remainings.length) {- return warmRemainings;- }-- return this.warmColdCounters({ workspaceId, counters });+ return counters.map(+ (counter, index) =>+ storedValues[index] ?? warmedValueByKey.get(counter.key) ?? null,+ );}- private async warmColdCounters({+ private async consumeCounters({workspaceId,counters,+ amounts,}: {workspaceId: string;counters: QuotaCounter[];- }): Promise<(number | null)[]> {- return this.cacheLockService.withLock(- async () => {- const remainings = await this.cacheStorage.mget<number>(- counters.map((counter) => counter.key),- );-- const warmRemainings = remainings.filter(isDefined);+ amounts: number[];+ }): Promise<QuotaConsumeResult> {+ const consumedValues = await this.runWarmAndConsumeScript({+ counters,+ entries: amounts.map((amount) => ({ amount, warm: null })),+ });- if (warmRemainings.length === remainings.length) {- return warmRemainings;- }+ const now = Date.now();- const coldCounters = counters.filter(- (_, index) => !isDefined(remainings[index]),- );+ const coldIndexes = consumedValues.flatMap((consumedValue, index) =>+ !isDefined(consumedValue) && counters[index].periodEnd.getTime() > now+ ? [index]+ : [],+ );- const warmedEntries = await this.buildWarmedEntries({- workspaceId,- coldCounters,- });+ if (coldIndexes.length === 0) {+ return { consumedValues, isDegraded: false };+ }- await this.cacheStorage.mset(warmedEntries);+ try {+ const warmValues = await this.fetchWarmValues({+ workspaceId,+ counters: coldIndexes.map((index) => counters[index]),+ });- const warmedValueByKey = new Map(- warmedEntries.map((entry) => [entry.key, entry.value]),- );+ const warmedAt = Date.now();- return counters.map(- (counter, index) =>- remainings[index] ?? warmedValueByKey.get(counter.key) ?? null,- );- },- buildQuotaWarmLockKey(workspaceId),- QUOTA_WARM_LOCK_OPTIONS,- );- }+ const warmedEntries = coldIndexes.flatMap((index, position) =>+ counters[index].periodEnd.getTime() > warmedAt+ ? [{ index, warmValue: warmValues[position] }]+ : [],+ );- private async buildWarmedEntries({- workspaceId,- coldCounters,- }: {- workspaceId: string;- coldCounters: QuotaCounter[];- }): Promise<{ key: string; value: number; ttl: number }[]> {- const coldLimitCounters = coldCounters.filter(- (counter): counter is LimitQuotaCounter => counter.kind === 'limit',- );- const coldAllowanceCounter = coldCounters.find(- (counter): counter is AllowanceQuotaCounter =>- counter.kind === 'allowance',- );+ const warmedValues = await this.runWarmAndConsumeScript({+ counters: warmedEntries.map(({ index }) => counters[index]),+ entries: warmedEntries.map(({ index, warmValue }) => ({+ amount: amounts[index],+ warm: {+ value: warmValue,+ ttl: counters[index].periodEnd.getTime() - warmedAt,+ },+ })),+ });- const now = Date.now();+ warmedEntries.forEach(({ index }, position) => {+ consumedValues[index] = warmedValues[position];+ });- const [rowsByPeriod, allowanceEntries] = await Promise.all([- this.fetchConsumptionRowsByPeriod({- workspaceId,- coldLimitCounters,- }),- this.buildAllowanceWarmedEntry({- workspaceId,- counter: coldAllowanceCounter,- now,- }),- ]);+ return { consumedValues, isDegraded: false };+ } catch (error) {+ this.logger.error(+ `Usage quota warm degraded for workspace ${workspaceId}: ${error instanceof Error ? error.message : 'unknown error'}`,+ );- return [- ...buildLimitWarmedEntries({ coldLimitCounters, rowsByPeriod, now }),- ...allowanceEntries,- ];+ return { consumedValues, isDegraded: true };+ }}- private async buildAllowanceWarmedEntry({- workspaceId,- counter,- now,+ private async runWarmAndConsumeScript({+ counters,+ entries,}: {- workspaceId: string;- counter: AllowanceQuotaCounter | undefined;- now: number;- }): Promise<{ key: string; value: number; ttl: number }[]> {- if (!isDefined(counter) || counter.periodEnd.getTime() <= now) {- return [];- }-- const allowance =- await this.creditAllowanceProvider.getCreditAllowance(workspaceId);-- if (- !isDefined(allowance) ||- allowance.periodStart.getTime() !== counter.periodStart.getTime()- ) {- return [];- }-- const ttl =- Math.min(counter.periodEnd.getTime(), allowance.validUntil.getTime()) -- now;-- if (ttl <= 0) {+ counters: QuotaCounter[];+ entries: Parameters<typeof buildConsumeScriptArguments>[0];+ }): Promise<(number | null)[]> {+ if (counters.length === 0) {return [];}- const consumedMicro =- await this.usageAnalyticsService.getCreditsUsedMicroForBillingPeriod({- workspaceId,- periodStart: counter.periodStart,- });+ const results = await this.cacheStorage.runScript<unknown[]>({+ script: WARM_AND_CONSUME_QUOTA_COUNTERS_SCRIPT,+ keys: counters.map((counter) => counter.key),+ args: buildConsumeScriptArguments(entries),+ });- return [- {- key: counter.key,- value: allowance.allowanceMicro - consumedMicro,- ttl,- },- ];+ return results.map((result) =>+ typeof result === 'number' ? result : null,+ );}- private async fetchConsumptionRowsByPeriod({+ private async fetchWarmValues({workspaceId,- coldLimitCounters,+ counters,}: {workspaceId: string;- coldLimitCounters: LimitQuotaCounter[];- }): Promise<Map<string, UsageConsumptionRow[]>> {- const countersByPeriod = new Map<string, LimitQuotaCounter>();+ counters: QuotaCounter[];+ }): Promise<number[]> {+ const rowsReadsByPeriodGroup = new Map<+ string,+ Promise<UsageConsumptionRow[]>+ >();++ return Promise.all(+ counters.map(async (counter) => {+ if (counter.kind === 'allowance') {+ return this.usageAnalyticsService.getCreditsUsedMicroForBillingPeriod(+ { workspaceId, periodStart: counter.periodStart },+ );+ }- for (const counter of coldLimitCounters) {- countersByPeriod.set(buildPeriodGroupKey(counter), counter);- }+ const periodGroupKey = buildPeriodGroupKey(counter);++ const rowsRead =+ rowsReadsByPeriodGroup.get(periodGroupKey) ??+ this.fetchConsumptionRows({ workspaceId, counter });- const rowsByPeriod = new Map<string, UsageConsumptionRow[]>();+ rowsReadsByPeriodGroup.set(periodGroupKey, rowsRead);- await Promise.all(- [...countersByPeriod.entries()].map(async ([periodGroupKey, counter]) => {- rowsByPeriod.set(- periodGroupKey,- await this.fetchConsumptionRows({ workspaceId, counter }),- );+ return computeQuotaConsumed({ rows: await rowsRead, scope: counter });}),);-- return rowsByPeriod;}private fetchConsumptionRows({diff --git a/packages/twenty-server/src/engine/core-modules/usage-limit/services/usage-limit.service.ts b/packages/twenty-server/src/engine/core-modules/usage-limit/services/usage-limit.service.tsindex 0cbf35d8bf1..e299989e481 100644--- a/packages/twenty-server/src/engine/core-modules/usage-limit/services/usage-limit.service.ts+++ b/packages/twenty-server/src/engine/core-modules/usage-limit/services/usage-limit.service.ts@@ -1,5 +1,6 @@import { Injectable } from '@nestjs/common';import { isNonEmptyString } from '@sniptt/guards';+import isEqual from 'lodash.isequal';import { isDefined } from 'twenty-shared/utils';import { ApiKeyEntity } from 'src/engine/core-modules/api-key/api-key.entity';@@ -170,9 +171,14 @@ export class UsageLimitService {{ where: { id: usageLimit.id } },);- // counters are keyed by scope and value, so an edit leaves two of them behind- await this.dropCounter(usageLimit);- await this.dropCounter(updatedUsageLimit);+ if (+ isStockLimit(usageLimit) ||+ usageLimit.isInstanceOverride !== isOperator ||+ !isEqual(authorizedScope, scope)+ ) {+ await this.dropCounter(usageLimit);+ await this.dropCounter(updatedUsageLimit);+ }return updatedUsageLimit;}@@ -305,7 +311,10 @@ export class UsageLimitService {});}- return this.usageLimitQuotaService.dropLimitCounter(usageLimit);+ return this.usageLimitQuotaService.dropLimitCounter({+ workspaceId: usageLimit.workspaceId,+ usageLimit,+ });}private async validateSpenderBelongsToWorkspace({diff --git a/packages/twenty-server/src/engine/core-modules/usage-limit/types/allowance-quota-counter.type.ts b/packages/twenty-server/src/engine/core-modules/usage-limit/types/allowance-quota-counter.type.tsindex e742a68364a..4de1bcf0b8c 100644--- a/packages/twenty-server/src/engine/core-modules/usage-limit/types/allowance-quota-counter.type.ts+++ b/packages/twenty-server/src/engine/core-modules/usage-limit/types/allowance-quota-counter.type.ts@@ -3,6 +3,7 @@ import { type UsageUnit } from 'src/engine/core-modules/usage/enums/usage-unit.eexport type AllowanceQuotaCounter = {kind: 'allowance';key: string;+ limitValue: number;unit: UsageUnit.CREDIT;periodStart: Date;periodEnd: Date;diff --git a/packages/twenty-server/src/engine/core-modules/usage-limit/types/credit-allowance-grant.type.ts b/packages/twenty-server/src/engine/core-modules/usage-limit/types/credit-allowance-grant.type.tsnew file mode 100644index 00000000000..c37851b3555--- /dev/null+++ b/packages/twenty-server/src/engine/core-modules/usage-limit/types/credit-allowance-grant.type.ts@@ -0,0 +1,5 @@+export type CreditAllowanceGrant = {+ amountMicro: number;+ effectiveAtMs: number;+ expiresAtMs: number | null;+};diff --git a/packages/twenty-server/src/engine/core-modules/usage-limit/types/credit-allowance.type.ts b/packages/twenty-server/src/engine/core-modules/usage-limit/types/credit-allowance.type.tsindex 6168f850977..b9ab06a88b6 100644--- a/packages/twenty-server/src/engine/core-modules/usage-limit/types/credit-allowance.type.ts+++ b/packages/twenty-server/src/engine/core-modules/usage-limit/types/credit-allowance.type.ts@@ -1,8 +1,7 @@+import { type CreditAllowanceGrant } from 'src/engine/core-modules/usage-limit/types/credit-allowance-grant.type';import { type UsagePeriod } from 'src/engine/core-modules/usage-limit/types/usage-period.type';export type CreditAllowance = UsagePeriod & {- allowanceMicro: number;- // When the figure stops being right before the period ends: a grant lapsing- // inside the period lowers it with no ledger write to drop the counter.- validUntil: Date;+ planAllowanceMicro: number;+ grants: CreditAllowanceGrant[];};diff --git a/packages/twenty-server/src/engine/core-modules/usage-limit/types/quota-default-key-scope.type.ts b/packages/twenty-server/src/engine/core-modules/usage-limit/types/quota-default-key-scope.type.tsdeleted file mode 100644index 38ecbbac438..00000000000--- a/packages/twenty-server/src/engine/core-modules/usage-limit/types/quota-default-key-scope.type.ts+++ /dev/null@@ -1,16 +0,0 @@-import { type PeriodUnit } from 'src/engine/core-modules/usage-limit/types/period-unit.type';-import { type SpenderType } from 'src/engine/core-modules/usage-limit/types/spender-type.type';-import { type UsageOperationType } from 'src/engine/core-modules/usage/enums/usage-operation-type.enum';-import { type UsageResourceType } from 'src/engine/core-modules/usage/enums/usage-resource-type.enum';-import { type UsageUnit } from 'src/engine/core-modules/usage/enums/usage-unit.enum';--export type QuotaDefaultKeyScope = {- workspaceId: string;- resourceType: UsageResourceType;- operationType: UsageOperationType;- spenderType: SpenderType;- spenderId?: string | null;- unit: UsageUnit;- periodUnit: PeriodUnit;- periodStart: Date;-};diff --git a/packages/twenty-server/src/engine/core-modules/usage-limit/types/quota-limit-default-definition.type.ts b/packages/twenty-server/src/engine/core-modules/usage-limit/types/quota-limit-default-definition.type.tsindex a856515d529..69f9cad064f 100644--- a/packages/twenty-server/src/engine/core-modules/usage-limit/types/quota-limit-default-definition.type.ts+++ b/packages/twenty-server/src/engine/core-modules/usage-limit/types/quota-limit-default-definition.type.ts@@ -17,5 +17,5 @@ export type QuotaLimitDefaultDefinition<periodUnit: AnchoredPeriodUnit;periodCount: 1;limitValueConfigVariable: NumericConfigVariableKey;- isOverridable: boolean;+ isOverridable: true;};diff --git a/packages/twenty-server/src/engine/core-modules/usage-limit/types/usage-limit-counter-scope.type.ts b/packages/twenty-server/src/engine/core-modules/usage-limit/types/usage-limit-counter-scope.type.tsdeleted file mode 100644index d4398ddfce1..00000000000--- a/packages/twenty-server/src/engine/core-modules/usage-limit/types/usage-limit-counter-scope.type.ts+++ /dev/null@@ -1,14 +0,0 @@-import { type UsageLimitEntity } from 'src/engine/core-modules/usage-limit/usage-limit.entity';--export type UsageLimitCounterScope = Pick<- UsageLimitEntity,- | 'workspaceId'- | 'resourceType'- | 'operationType'- | 'spenderType'- | 'spenderId'- | 'limitKind'- | 'periodUnit'- | 'unit'- | 'limitValue'->;diff --git a/packages/twenty-server/src/engine/core-modules/usage-limit/utils/__tests__/build-consume-script-arguments.util.spec.ts b/packages/twenty-server/src/engine/core-modules/usage-limit/utils/__tests__/build-consume-script-arguments.util.spec.tsnew file mode 100644index 00000000000..2fb97e620cf--- /dev/null+++ b/packages/twenty-server/src/engine/core-modules/usage-limit/utils/__tests__/build-consume-script-arguments.util.spec.ts@@ -0,0 +1,24 @@+import { buildConsumeScriptArguments } from 'src/engine/core-modules/usage-limit/utils/build-consume-script-arguments.util';++describe('buildConsumeScriptArguments', () => {+ it.each([-1, 1.5, Number.MAX_SAFE_INTEGER + 1, Number.NaN])(+ 'refuses the amount %s',+ (amount) => {+ expect(() =>+ buildConsumeScriptArguments([{ amount, warm: null }]),+ ).toThrow(/^Counter amount .* is not a non-negative safe integer$/);+ },+ );++ it.each([+ { value: -1, ttl: 60_000 },+ { value: 0.5, ttl: 60_000 },+ { value: 10, ttl: 0 },+ { value: 10, ttl: -1 },+ { value: 10, ttl: 1.5 },+ ])('refuses the warm value $value with TTL $ttl', (warm) => {+ expect(() => buildConsumeScriptArguments([{ amount: 1, warm }])).toThrow(+ /^Counter warm value .* is not a non-negative safe integer with a positive safe TTL$/,+ );+ });+});diff --git a/packages/twenty-server/src/engine/core-modules/usage-limit/utils/__tests__/build-intra-workspace-limit-counter-keys.util.spec.ts b/packages/twenty-server/src/engine/core-modules/usage-limit/utils/__tests__/build-intra-workspace-limit-counter-keys.util.spec.tsindex a75bede69e8..56a2882ada4 100644--- a/packages/twenty-server/src/engine/core-modules/usage-limit/utils/__tests__/build-intra-workspace-limit-counter-keys.util.spec.ts+++ b/packages/twenty-server/src/engine/core-modules/usage-limit/utils/__tests__/build-intra-workspace-limit-counter-keys.util.spec.ts@@ -47,18 +47,20 @@ describe('buildIntraWorkspaceLimitCounterKeys', () => {periodByUnit: { month: MONTH_PERIOD, week: WEEK_PERIOD },}),).toEqual([- `{workspace-1}:quota:AI:ALL:userWorkspace:-:CREDIT:month:${MONTH_PERIOD.periodStart.getTime()}:1000`,- `{workspace-1}:quota:AI:ALL:agent:agent-1:CREDIT:week:${WEEK_PERIOD.periodStart.getTime()}:1000`,+ `{workspace-1}:quota-consumed:AI:ALL:userWorkspace:-:CREDIT:month:${MONTH_PERIOD.periodStart.getTime()}`,+ `{workspace-1}:quota-consumed:AI:ALL:agent:agent-1:CREDIT:week:${WEEK_PERIOD.periodStart.getTime()}`,]);});- it('leaves workspace-scoped quotas and speed limits alone', () => {+ it('leaves workspace-scoped quotas, operator overrides and speed limits alone', () => {expect(buildIntraWorkspaceLimitCounterKeys({workspaceId: 'workspace-1',limits: [buildLimit({ spenderType: 'workspace' }),+ buildLimit({ id: 'limit-2', isInstanceOverride: true }),buildLimit({+ id: 'limit-3',limitKind: 'speed',periodUnit: 'second',unit: UsageUnit.TOKEN,diff --git a/packages/twenty-server/src/engine/core-modules/usage-limit/utils/__tests__/build-limit-warmed-entries.util.spec.ts b/packages/twenty-server/src/engine/core-modules/usage-limit/utils/__tests__/build-limit-warmed-entries.util.spec.tsdeleted file mode 100644index c8b44a6b8a9..00000000000--- a/packages/twenty-server/src/engine/core-modules/usage-limit/utils/__tests__/build-limit-warmed-entries.util.spec.ts+++ /dev/null@@ -1,113 +0,0 @@-import { type LimitQuotaCounter } from 'src/engine/core-modules/usage-limit/types/limit-quota-counter.type';-import { type UsageConsumptionRow } from 'src/engine/core-modules/usage/types/usage-consumption-row.type';-import { buildLimitWarmedEntries } from 'src/engine/core-modules/usage-limit/utils/build-limit-warmed-entries.util';-import { buildPeriodGroupKey } from 'src/engine/core-modules/usage-limit/utils/build-period-group-key.util';-import { UsageOperationType } from 'src/engine/core-modules/usage/enums/usage-operation-type.enum';-import { UsageResourceType } from 'src/engine/core-modules/usage/enums/usage-resource-type.enum';-import { UsageUnit } from 'src/engine/core-modules/usage/enums/usage-unit.enum';--const NOW = new Date('2026-08-20T00:00:00.000Z').getTime();--const buildCounter = (- overrides: Partial<LimitQuotaCounter> = {},-): LimitQuotaCounter => ({- kind: 'limit',- isDefault: false,- key: 'counter-key',- limitValue: 1_000,- unit: UsageUnit.CREDIT,- resourceType: UsageResourceType.AI,- periodUnit: 'month',- periodStart: new Date('2026-08-01T00:00:00.000Z'),- periodEnd: new Date('2026-09-01T00:00:00.000Z'),- spenderType: 'workspace',- spenderId: null,- operationType: UsageOperationType.AI_CHAT_TOKEN,- ...overrides,-});--const buildRow = (- overrides: Partial<UsageConsumptionRow> = {},-): UsageConsumptionRow => ({- operationType: UsageOperationType.AI_CHAT_TOKEN,- unit: UsageUnit.TOKEN,- userWorkspaceId: 'user-1',- apiKeyId: '',- applicationId: '',- agentId: '',- workflowId: '',- logicFunctionId: '',- creditsUsedMicro: '0',- quantity: '0',- ...overrides,-});--const buildRowsByPeriod = (- counter: LimitQuotaCounter,- rows: UsageConsumptionRow[],-) => new Map([[buildPeriodGroupKey(counter), rows]]);--describe('buildLimitWarmedEntries', () => {- it('seeds the counter with its budget minus the stamped consumption', () => {- const counter = buildCounter();-- expect(- buildLimitWarmedEntries({- coldLimitCounters: [counter],- rowsByPeriod: buildRowsByPeriod(counter, [- buildRow({ creditsUsedMicro: '150' }),- buildRow({ userWorkspaceId: 'user-2', creditsUsedMicro: '100' }),- ]),- now: NOW,- }),- ).toEqual([- {- key: 'counter-key',- value: 750,- ttl: counter.periodEnd.getTime() - NOW,- },- ]);- });-- it('counts only the rows the counter spender stamped', () => {- const counter = buildCounter({- spenderType: 'userWorkspace',- spenderId: 'user-1',- });-- expect(- buildLimitWarmedEntries({- coldLimitCounters: [counter],- rowsByPeriod: buildRowsByPeriod(counter, [- buildRow({ creditsUsedMicro: '150' }),- buildRow({ userWorkspaceId: 'user-2', creditsUsedMicro: '100' }),- ]),- now: NOW,- }),- ).toMatchObject([{ value: 850 }]);- });-- it('skips a counter whose period already ended', () => {- const counter = buildCounter({- periodEnd: new Date('2026-08-10T00:00:00.000Z'),- });-- expect(- buildLimitWarmedEntries({- coldLimitCounters: [counter],- rowsByPeriod: buildRowsByPeriod(counter, [buildRow({})]),- now: NOW,- }),- ).toEqual([]);- });-- it('skips a counter whose consumption was not fetched', () => {- expect(- buildLimitWarmedEntries({- coldLimitCounters: [buildCounter()],- rowsByPeriod: new Map(),- now: NOW,- }),- ).toEqual([]);- });-});diff --git a/packages/twenty-server/src/engine/core-modules/usage-limit/utils/__tests__/build-overridden-quota-default-counter-keys.util.spec.ts b/packages/twenty-server/src/engine/core-modules/usage-limit/utils/__tests__/build-overridden-quota-default-counter-keys.util.spec.tsdeleted file mode 100644index d9e22763c31..00000000000--- a/packages/twenty-server/src/engine/core-modules/usage-limit/utils/__tests__/build-overridden-quota-default-counter-keys.util.spec.ts+++ /dev/null@@ -1,77 +0,0 @@-import { type PeriodUnit } from 'src/engine/core-modules/usage-limit/types/period-unit.type';-import { type QuotaLimitDefault } from 'src/engine/core-modules/usage-limit/types/quota-limit-default.type';-import { type UsageLimitCounterScope } from 'src/engine/core-modules/usage-limit/types/usage-limit-counter-scope.type';-import { type UsagePeriod } from 'src/engine/core-modules/usage-limit/types/usage-period.type';-import { buildOverriddenQuotaDefaultCounterKeys } from 'src/engine/core-modules/usage-limit/utils/build-overridden-quota-default-counter-keys.util';-import { UsageOperationType } from 'src/engine/core-modules/usage/enums/usage-operation-type.enum';-import { UsageResourceType } from 'src/engine/core-modules/usage/enums/usage-resource-type.enum';-import { UsageUnit } from 'src/engine/core-modules/usage/enums/usage-unit.enum';--const DAY_PERIOD = {- periodStart: new Date('2026-08-20T00:00:00.000Z'),- periodEnd: new Date('2026-08-21T00:00:00.000Z'),-};--const DEFAULT_COUNTER_KEY = `{workspace-1}:quota:EMAIL:EMAIL_SEND:workspace:-:INVOCATION:day:${DAY_PERIOD.periodStart.getTime()}:1000:default`;--const buildUsageLimit = (- overrides: Partial<UsageLimitCounterScope> = {},-): UsageLimitCounterScope => ({- workspaceId: 'workspace-1',- resourceType: UsageResourceType.EMAIL,- operationType: UsageOperationType.EMAIL_SEND,- spenderType: 'workspace',- spenderId: '',- limitKind: 'quota',- periodUnit: 'day',- unit: UsageUnit.INVOCATION,- limitValue: 500,- ...overrides,-});--const buildDefault = (- overrides: Partial<QuotaLimitDefault> = {},-): QuotaLimitDefault => ({- resourceType: UsageResourceType.EMAIL,- operationType: UsageOperationType.EMAIL_SEND,- limitKind: 'quota',- spenderType: 'workspace',- spenderId: '',- unit: UsageUnit.INVOCATION,- periodUnit: 'day',- periodCount: 1,- isOverridable: true,- limitValue: 1_000,- ...overrides,-});--const buildKeys = ({- usageLimit = buildUsageLimit(),- quotaLimitDefaults = [buildDefault()],- periodByUnit = { day: DAY_PERIOD },-}: {- usageLimit?: UsageLimitCounterScope;- quotaLimitDefaults?: QuotaLimitDefault[];- periodByUnit?: Partial<Record<PeriodUnit, UsagePeriod>>;-} = {}) =>- buildOverriddenQuotaDefaultCounterKeys({- usageLimit,- quotaLimitDefaults,- periodByUnit,- });--describe('buildOverriddenQuotaDefaultCounterKeys', () => {- it('keys the default counter the row suppresses', () => {- expect(buildKeys()).toEqual([DEFAULT_COUNTER_KEY]);- });-- it('leaves alone a default the row does not suppress', () => {- expect(- buildKeys({ usageLimit: buildUsageLimit({ periodUnit: 'month' }) }),- ).toEqual([]);- });-- it('skips a default whose period was not resolved', () => {- expect(buildKeys({ periodByUnit: {} })).toEqual([]);- });-});diff --git a/packages/twenty-server/src/engine/core-modules/usage-limit/utils/__tests__/build-quota-counter-key.util.spec.ts b/packages/twenty-server/src/engine/core-modules/usage-limit/utils/__tests__/build-quota-counter-key.util.spec.tsindex 87423a66999..b6166a03fa8 100644--- a/packages/twenty-server/src/engine/core-modules/usage-limit/utils/__tests__/build-quota-counter-key.util.spec.ts+++ b/packages/twenty-server/src/engine/core-modules/usage-limit/utils/__tests__/build-quota-counter-key.util.spec.ts@@ -6,26 +6,27 @@ import { UsageUnit } from 'src/engine/core-modules/usage/enums/usage-unit.enum';const WORKSPACE_ID = 'workspace-1';const PERIOD_START = new Date('2026-09-01T00:00:00.000Z');-const buildKey = (- overrides: Partial<Parameters<typeof buildQuotaCounterKey>[0]> = {},-) =>+type CounterIdentity = Parameters<typeof buildQuotaCounterKey>[0]['counter'];++const buildKey = (overrides: Partial<CounterIdentity> = {}) =>buildQuotaCounterKey({workspaceId: WORKSPACE_ID,- resourceType: UsageResourceType.AI,- operationType: UsageOperationType.AI_CHAT_TOKEN,- spenderType: 'workspace',- spenderId: null,- unit: UsageUnit.CREDIT,- periodUnit: 'month',- periodStart: PERIOD_START,- limitValue: 1000,- ...overrides,+ counter: {+ resourceType: UsageResourceType.AI,+ operationType: UsageOperationType.AI_CHAT_TOKEN,+ spenderType: 'workspace',+ spenderId: null,+ unit: UsageUnit.CREDIT,+ periodUnit: 'month',+ periodStart: PERIOD_START,+ ...overrides,+ },});describe('buildQuotaCounterKey', () => {- it('hashes on the workspace so every counter of a workspace shares a slot', () => {+ it('hashes on the workspace and keys a counter by its scope and period', () => {expect(buildKey()).toBe(- `{${WORKSPACE_ID}}:quota:AI:AI_CHAT_TOKEN:workspace:-:CREDIT:month:${PERIOD_START.getTime()}:1000`,+ `{${WORKSPACE_ID}}:quota-consumed:AI:AI_CHAT_TOKEN:workspace:-:CREDIT:month:${PERIOD_START.getTime()}`,);});@@ -35,11 +36,7 @@ describe('buildQuotaCounterKey', () => {['spenderId', { spenderId: 'user-1' }],['periodUnit', { periodUnit: 'week' as const }],['periodStart', { periodStart: new Date('2026-10-01T00:00:00.000Z') }],- ['limitValue', { limitValue: 2000 }],- ])(- 'gives two limits differing only by %s their own counter',- (_, differs) => {- expect(buildKey(differs)).not.toBe(buildKey());- },- );+ ])('gives two counters differing only by %s their own key', (_, differs) => {+ expect(buildKey(differs)).not.toBe(buildKey());+ });});diff --git a/packages/twenty-server/src/engine/core-modules/usage-limit/utils/__tests__/build-quota-counters.util.spec.ts b/packages/twenty-server/src/engine/core-modules/usage-limit/utils/__tests__/build-quota-counters.util.spec.tsindex ba0c9397966..6bd24f291be 100644--- a/packages/twenty-server/src/engine/core-modules/usage-limit/utils/__tests__/build-quota-counters.util.spec.ts+++ b/packages/twenty-server/src/engine/core-modules/usage-limit/utils/__tests__/build-quota-counters.util.spec.ts@@ -73,7 +73,7 @@ describe('buildQuotaCounters', () => {{kind: 'limit',isDefault: false,- key: `{workspace-1}:quota:AI:AI_CHAT_TOKEN:workspace:-:CREDIT:month:${MONTH_PERIOD.periodStart.getTime()}:1000000`,+ key: `{workspace-1}:quota-consumed:AI:AI_CHAT_TOKEN:workspace:-:CREDIT:month:${MONTH_PERIOD.periodStart.getTime()}`,limitValue: 1_000_000,unit: UsageUnit.CREDIT,resourceType: UsageResourceType.AI,@@ -168,7 +168,7 @@ describe('buildQuotaCounters', () => {{kind: 'limit',isDefault: true,- key: `{workspace-1}:quota:AI:AI_CHAT_TOKEN:workspace:-:CREDIT:month:${MONTH_PERIOD.periodStart.getTime()}:5000:default`,+ key: `{workspace-1}:quota-consumed:AI:AI_CHAT_TOKEN:workspace:-:CREDIT:month:${MONTH_PERIOD.periodStart.getTime()}`,limitValue: 5_000,unit: UsageUnit.CREDIT,resourceType: UsageResourceType.AI,@@ -183,15 +183,23 @@ describe('buildQuotaCounters', () => {});it.each([500, 50_000])(- 'overrides the default with a workspace limit of %i',+ 'overrides the default with a workspace limit of %i on the same counter',(limitValue) => {+ const [defaultCounter] = buildCounters({+ quotaLimitDefaults: [buildDefault()],+ });+const counters = buildCounters({limits: [buildLimit({ limitValue })],quotaLimitDefaults: [buildDefault()],});expect(counters).toEqual([- expect.objectContaining({ isDefault: false, limitValue }),+ expect.objectContaining({+ isDefault: false,+ limitValue,+ key: defaultCounter.key,+ }),]);},);diff --git a/packages/twenty-server/src/engine/core-modules/usage-limit/utils/__tests__/build-quota-debits.util.spec.ts b/packages/twenty-server/src/engine/core-modules/usage-limit/utils/__tests__/build-quota-debits.util.spec.tsindex 8f98c745890..0146373592a 100644--- a/packages/twenty-server/src/engine/core-modules/usage-limit/utils/__tests__/build-quota-debits.util.spec.ts+++ b/packages/twenty-server/src/engine/core-modules/usage-limit/utils/__tests__/build-quota-debits.util.spec.ts@@ -12,6 +12,7 @@ const PERIOD_END = new Date('2026-09-01T00:00:00.000Z');const allowanceCounter: AllowanceQuotaCounter = {kind: 'allowance',key: 'allowance',+ limitValue: 1_000_000,unit: UsageUnit.CREDIT,periodStart: PERIOD_START,periodEnd: PERIOD_END,diff --git a/packages/twenty-server/src/engine/core-modules/usage-limit/utils/__tests__/build-quota-exhausted-scope.util.spec.ts b/packages/twenty-server/src/engine/core-modules/usage-limit/utils/__tests__/build-quota-exhausted-scope.util.spec.tsindex 031c50263f5..9c77200bf31 100644--- a/packages/twenty-server/src/engine/core-modules/usage-limit/utils/__tests__/build-quota-exhausted-scope.util.spec.ts+++ b/packages/twenty-server/src/engine/core-modules/usage-limit/utils/__tests__/build-quota-exhausted-scope.util.spec.ts@@ -30,6 +30,7 @@ const buildLimitCounter = (const allowanceCounter: AllowanceQuotaCounter = {kind: 'allowance',key: 'allowance-key',+ limitValue: 2_000_000,unit: UsageUnit.CREDIT,periodStart: PERIOD_START,periodEnd: PERIOD_END,@@ -49,7 +50,6 @@ describe('buildQuotaExhaustedScope', () => {buildQuotaExhaustedScope({resourceType: UsageResourceType.AI,counter: buildLimitCounter(),- allowance: null,}),).toEqual({resourceType: UsageResourceType.AI,@@ -73,7 +73,6 @@ describe('buildQuotaExhaustedScope', () => {buildQuotaExhaustedScope({resourceType: UsageResourceType.AI,counter: buildLimitCounter({ isDefault: true }),- allowance: null,}),).toMatchObject({ exhaustedKind: 'limit', isDefault: true });});@@ -83,12 +82,6 @@ describe('buildQuotaExhaustedScope', () => {buildQuotaExhaustedScope({resourceType: UsageResourceType.AI,counter: allowanceCounter,- allowance: {- periodStart: PERIOD_START,- periodEnd: PERIOD_END,- allowanceMicro: 2_000_000,- validUntil: PERIOD_END,- },}),).toEqual({resourceType: UsageResourceType.AI,@@ -106,16 +99,6 @@ describe('buildQuotaExhaustedScope', () => {});});- it('reports a zero allowance when it is gone since the counter was read', () => {- expect(- buildQuotaExhaustedScope({- resourceType: UsageResourceType.AI,- counter: allowanceCounter,- allowance: null,- }),- ).toMatchObject({ exhaustedKind: 'allowance', limitValue: 0 });- });-it('clamps the retry delay to zero once the period has ended', () => {expect(buildQuotaExhaustedScope({@@ -123,7 +106,6 @@ describe('buildQuotaExhaustedScope', () => {counter: buildLimitCounter({periodEnd: new Date('2026-08-10T00:00:00.000Z'),}),- allowance: null,}),).toMatchObject({ retryAfterMs: 0 });});diff --git a/packages/twenty-server/src/engine/core-modules/usage-limit/utils/__tests__/compute-credit-allowance-micro.util.spec.ts b/packages/twenty-server/src/engine/core-modules/usage-limit/utils/__tests__/compute-credit-allowance-micro.util.spec.tsnew file mode 100644index 00000000000..7ce4f6a5751--- /dev/null+++ b/packages/twenty-server/src/engine/core-modules/usage-limit/utils/__tests__/compute-credit-allowance-micro.util.spec.ts@@ -0,0 +1,59 @@+import { type CreditAllowanceGrant } from 'src/engine/core-modules/usage-limit/types/credit-allowance-grant.type';+import { computeCreditAllowanceMicro } from 'src/engine/core-modules/usage-limit/utils/compute-credit-allowance-micro.util';++const NOW_MS = new Date('2026-08-15T00:00:00.000Z').getTime();+const DAY_MS = 24 * 60 * 60 * 1000;++const buildGrant = (+ overrides: Partial<CreditAllowanceGrant>,+): CreditAllowanceGrant => ({+ amountMicro: 500,+ effectiveAtMs: NOW_MS - DAY_MS,+ expiresAtMs: null,+ ...overrides,+});++describe('computeCreditAllowanceMicro', () => {+ it('adds every grant in effect', () => {+ expect(+ computeCreditAllowanceMicro({+ planAllowanceMicro: 1_000,+ grants: [+ buildGrant({ amountMicro: 500 }),+ buildGrant({ amountMicro: 300, expiresAtMs: NOW_MS + DAY_MS }),+ ],+ nowMs: NOW_MS,+ }),+ ).toBe(1_800);+ });++ it('leaves out a grant that is not effective yet', () => {+ expect(+ computeCreditAllowanceMicro({+ planAllowanceMicro: 1_000,+ grants: [buildGrant({ effectiveAtMs: NOW_MS + 1 })],+ nowMs: NOW_MS,+ }),+ ).toBe(1_000);+ });++ it('counts a grant from the instant it becomes effective', () => {+ expect(+ computeCreditAllowanceMicro({+ planAllowanceMicro: 1_000,+ grants: [buildGrant({ effectiveAtMs: NOW_MS })],+ nowMs: NOW_MS,+ }),+ ).toBe(1_500);+ });++ it('leaves out a grant from the instant it expires', () => {+ expect(+ computeCreditAllowanceMicro({+ planAllowanceMicro: 1_000,+ grants: [buildGrant({ expiresAtMs: NOW_MS })],+ nowMs: NOW_MS,+ }),+ ).toBe(1_000);+ });+});diff --git a/packages/twenty-server/src/engine/core-modules/usage-limit/utils/__tests__/find-exhausted-counters.util.spec.ts b/packages/twenty-server/src/engine/core-modules/usage-limit/utils/__tests__/find-exhausted-counters.util.spec.tsindex 0529e5a7cd1..d82389729ca 100644--- a/packages/twenty-server/src/engine/core-modules/usage-limit/utils/__tests__/find-exhausted-counters.util.spec.ts+++ b/packages/twenty-server/src/engine/core-modules/usage-limit/utils/__tests__/find-exhausted-counters.util.spec.ts@@ -11,6 +11,7 @@ const PERIOD_END = new Date('2026-09-01T00:00:00.000Z');const buildAllowanceCounter = (key: string): AllowanceQuotaCounter => ({kind: 'allowance',key,+ limitValue: 5_000,unit: UsageUnit.CREDIT,periodStart: PERIOD_START,periodEnd: PERIOD_END,@@ -36,77 +37,24 @@ const buildLimitCounter = (});describe('findExhaustedCounters', () => {- it('answers empty when every counter has budget left', () => {+ it('picks every counter at or past its cap', () => {expect(findExhaustedCounters({counters: [- buildAllowanceCounter('first'),- buildAllowanceCounter('second'),+ buildLimitCounter('under'),+ buildLimitCounter('at'),+ buildAllowanceCounter('past'),],- remainings: [250, 1],+ consumedValues: [999, 1_000, 6_000],}),- ).toEqual([]);- });-- it('picks every counter whose budget is gone', () => {- expect(- findExhaustedCounters({- counters: [- buildAllowanceCounter('first'),- buildAllowanceCounter('second'),- buildAllowanceCounter('third'),- ],- remainings: [0, 100, 0],- }),- ).toMatchObject([{ key: 'first' }, { key: 'third' }]);- });-- it('counts an overdrawn counter as exhausted', () => {- expect(- findExhaustedCounters({- counters: [buildAllowanceCounter('first')],- remainings: [-10],- }),- ).toMatchObject([{ key: 'first' }]);+ ).toMatchObject([{ key: 'at' }, { key: 'past' }]);});it('skips cold counters', () => {expect(findExhaustedCounters({- counters: [- buildAllowanceCounter('first'),- buildAllowanceCounter('second'),- ],- remainings: [null, 0],- }),- ).toMatchObject([{ key: 'second' }]);- });-- it('answers empty when every counter is cold', () => {- expect(- findExhaustedCounters({- counters: [buildAllowanceCounter('first')],- remainings: [null],- }),- ).toEqual([]);- });-- it('refuses a limit that cannot cover the named cost', () => {- expect(- findExhaustedCounters({- counters: [buildLimitCounter('first')],- remainings: [100],- cost: { [UsageUnit.CREDIT]: 101, [UsageUnit.TOKEN]: 0 },- }),- ).toMatchObject([{ key: 'first' }]);- });-- it('admits a cost that fits the remaining exactly', () => {- expect(- findExhaustedCounters({- counters: [buildLimitCounter('first')],- remainings: [100],- cost: { [UsageUnit.CREDIT]: 100, [UsageUnit.TOKEN]: 0 },+ counters: [buildLimitCounter('cold'), buildAllowanceCounter('cold')],+ consumedValues: [null, null],}),).toEqual([]);});@@ -114,60 +62,23 @@ describe('findExhaustedCounters', () => {it('charges a limit only for the unit it counts', () => {expect(findExhaustedCounters({- counters: [buildLimitCounter('first')],- remainings: [10],+ counters: [+ buildLimitCounter('credits'),+ buildLimitCounter('tokens', { unit: UsageUnit.TOKEN }),+ ],+ consumedValues: [990, 990],cost: { [UsageUnit.CREDIT]: 0, [UsageUnit.TOKEN]: 500 },}),- ).toEqual([]);+ ).toMatchObject([{ key: 'tokens' }]);});it('counts a unit missing from the cost as zero', () => {expect(findExhaustedCounters({- counters: [buildLimitCounter('first', { unit: UsageUnit.TOKEN })],- remainings: [10],+ counters: [buildLimitCounter('tokens', { unit: UsageUnit.TOKEN })],+ consumedValues: [990],cost: { [UsageUnit.CREDIT]: 5_000 },}),).toEqual([]);});-- it('still refuses an emptied limit whatever the cost', () => {- expect(- findExhaustedCounters({- counters: [buildLimitCounter('first')],- remainings: [0],- cost: { [UsageUnit.CREDIT]: 0, [UsageUnit.TOKEN]: 0 },- }),- ).toMatchObject([{ key: 'first' }]);- });-- it('refuses a credit allowance that cannot cover the whole cost', () => {- expect(- findExhaustedCounters({- counters: [buildAllowanceCounter('allowance')],- remainings: [10],- cost: { [UsageUnit.CREDIT]: 5_000, [UsageUnit.TOKEN]: 100 },- }),- ).toMatchObject([{ key: 'allowance' }]);- });-- it('admits a credit allowance holding exactly the whole cost', () => {- expect(- findExhaustedCounters({- counters: [buildAllowanceCounter('allowance')],- remainings: [5_000],- cost: { [UsageUnit.CREDIT]: 5_000, [UsageUnit.TOKEN]: 100 },- }),- ).toEqual([]);- });-- it('still refuses an emptied credit allowance under a named cost', () => {- expect(- findExhaustedCounters({- counters: [buildAllowanceCounter('allowance')],- remainings: [0],- cost: { [UsageUnit.CREDIT]: 5_000, [UsageUnit.TOKEN]: 100 },- }),- ).toMatchObject([{ key: 'allowance' }]);- });});diff --git a/packages/twenty-server/src/engine/core-modules/usage-limit/utils/__tests__/from-consume-results-to-remainings.util.spec.ts b/packages/twenty-server/src/engine/core-modules/usage-limit/utils/__tests__/from-consume-results-to-remainings.util.spec.tsdeleted file mode 100644index 6bf2cc1a552..00000000000--- a/packages/twenty-server/src/engine/core-modules/usage-limit/utils/__tests__/from-consume-results-to-remainings.util.spec.ts+++ /dev/null@@ -1,17 +0,0 @@-import { fromConsumeResultsToRemainings } from 'src/engine/core-modules/usage-limit/utils/from-consume-results-to-remainings.util';--describe('fromConsumeResultsToRemainings', () => {- it('reads one remaining per debited counter', () => {- expect(fromConsumeResultsToRemainings([1, 500, 1, -10])).toEqual([- 500, -10,- ]);- });-- it('leaves a counter the script did not find cold', () => {- expect(fromConsumeResultsToRemainings([1, 500, 0, 0])).toEqual([500, null]);- });-- it('answers nothing for no results', () => {- expect(fromConsumeResultsToRemainings([])).toEqual([]);- });-});diff --git a/packages/twenty-server/src/engine/core-modules/usage-limit/utils/__tests__/is-limit-exhausted.util.spec.ts b/packages/twenty-server/src/engine/core-modules/usage-limit/utils/__tests__/is-limit-exhausted.util.spec.tsnew file mode 100644index 00000000000..626d5cfe7c8--- /dev/null+++ b/packages/twenty-server/src/engine/core-modules/usage-limit/utils/__tests__/is-limit-exhausted.util.spec.ts@@ -0,0 +1,21 @@+import { isLimitExhausted } from 'src/engine/core-modules/usage-limit/utils/is-limit-exhausted.util';++describe('isLimitExhausted', () => {+ it('admits a cost that fits the cap exactly', () => {+ expect(+ isLimitExhausted({ consumed: 900, cost: 100, limitValue: 1_000 }),+ ).toBe(false);+ });++ it('refuses a cost one unit over the cap', () => {+ expect(+ isLimitExhausted({ consumed: 900, cost: 101, limitValue: 1_000 }),+ ).toBe(true);+ });++ it('refuses at the cap even with no cost', () => {+ expect(+ isLimitExhausted({ consumed: 1_000, cost: 0, limitValue: 1_000 }),+ ).toBe(true);+ });+});diff --git a/packages/twenty-server/src/engine/core-modules/usage-limit/utils/build-allowance-counter-key.util.ts b/packages/twenty-server/src/engine/core-modules/usage-limit/utils/build-allowance-counter-key.util.tsindex 29d8dcda3cb..80c27c4dc43 100644--- a/packages/twenty-server/src/engine/core-modules/usage-limit/utils/build-allowance-counter-key.util.ts+++ b/packages/twenty-server/src/engine/core-modules/usage-limit/utils/build-allowance-counter-key.util.ts@@ -4,4 +4,5 @@ export const buildAllowanceCounterKey = ({}: {workspaceId: string;periodStart: Date;-}): string => `{${workspaceId}}:quota:allowance:${periodStart.getTime()}`;+}): string =>+ `{${workspaceId}}:quota-consumed:allowance:${periodStart.getTime()}`;diff --git a/packages/twenty-server/src/engine/core-modules/usage-limit/utils/build-consume-script-arguments.util.ts b/packages/twenty-server/src/engine/core-modules/usage-limit/utils/build-consume-script-arguments.util.tsnew file mode 100644index 00000000000..9e6da1a4ffe--- /dev/null+++ b/packages/twenty-server/src/engine/core-modules/usage-limit/utils/build-consume-script-arguments.util.ts@@ -0,0 +1,33 @@+import { isDefined } from 'twenty-shared/utils';++export const buildConsumeScriptArguments = (+ entries: { amount: number; warm: { value: number; ttl: number } | null }[],+): string[] => {+ for (const { amount, warm } of entries) {+ if (!(Number.isSafeInteger(amount) && amount >= 0)) {+ throw new Error(+ `Counter amount ${amount} is not a non-negative safe integer`,+ );+ }++ if (+ isDefined(warm) &&+ !(+ Number.isSafeInteger(warm.value) &&+ warm.value >= 0 &&+ Number.isSafeInteger(warm.ttl) &&+ warm.ttl > 0+ )+ ) {+ throw new Error(+ `Counter warm value ${warm.value} with TTL ${warm.ttl} is not a non-negative safe integer with a positive safe TTL`,+ );+ }+ }++ return [+ JSON.stringify(entries.map(({ amount }) => amount)),+ JSON.stringify(entries.map(({ warm }) => warm?.value ?? false)),+ JSON.stringify(entries.map(({ warm }) => warm?.ttl ?? false)),+ ];+};diff --git a/packages/twenty-server/src/engine/core-modules/usage-limit/utils/build-intra-workspace-limit-counter-keys.util.ts b/packages/twenty-server/src/engine/core-modules/usage-limit/utils/build-intra-workspace-limit-counter-keys.util.tsindex 6d37f7729f7..96dbe1e5053 100644--- a/packages/twenty-server/src/engine/core-modules/usage-limit/utils/build-intra-workspace-limit-counter-keys.util.ts+++ b/packages/twenty-server/src/engine/core-modules/usage-limit/utils/build-intra-workspace-limit-counter-keys.util.ts@@ -3,8 +3,9 @@ import { isDefined } from 'twenty-shared/utils';import { type FlatUsageLimit } from 'src/engine/core-modules/usage-limit/types/flat-usage-limit.type';import { type PeriodUnit } from 'src/engine/core-modules/usage-limit/types/period-unit.type';import { type UsagePeriod } from 'src/engine/core-modules/usage-limit/types/usage-period.type';-import { buildQuotaCounterKey } from 'src/engine/core-modules/usage-limit/utils/build-quota-counter-key.util';-import { isIntraWorkspaceScoped } from 'src/engine/core-modules/usage-limit/utils/is-intra-workspace-scoped.util';+import { buildLimitQuotaCounter } from 'src/engine/core-modules/usage-limit/utils/build-limit-quota-counter.util';+import { findEnforceableLimits } from 'src/engine/core-modules/usage-limit/utils/find-enforceable-limits.util';+import { isQuotaLimit } from 'src/engine/core-modules/usage-limit/utils/is-quota-limit.util';export const buildIntraWorkspaceLimitCounterKeys = ({workspaceId,@@ -14,31 +15,23 @@ export const buildIntraWorkspaceLimitCounterKeys = ({workspaceId: string;limits: FlatUsageLimit[];periodByUnit: Partial<Record<PeriodUnit, UsagePeriod>>;-}): string[] =>- limits- .filter(- (limit) =>- limit.limitKind === 'quota' &&- isIntraWorkspaceScoped(limit.spenderType),- )+}): string[] => {+ const quotaLimits = limits.filter(isQuotaLimit);++ const limitIdsEnforcedWithoutEntitlement = new Set(+ findEnforceableLimits({+ limits: quotaLimits,+ isIntraWorkspaceLimitEntitled: false,+ }).map((limit) => limit.id),+ );++ return quotaLimits+ .filter((limit) => !limitIdsEnforcedWithoutEntitlement.has(limit.id)).flatMap((limit) => {const period = periodByUnit[limit.periodUnit];- if (!isDefined(period)) {- return [];- }-- return [- buildQuotaCounterKey({- workspaceId,- resourceType: limit.resourceType,- operationType: limit.operationType,- spenderType: limit.spenderType,- spenderId: limit.spenderId,- unit: limit.unit,- periodUnit: limit.periodUnit,- periodStart: period.periodStart,- limitValue: limit.limitValue,- }),- ];+ return isDefined(period)+ ? [buildLimitQuotaCounter({ workspaceId, limit, period }).key]+ : [];});+};diff --git a/packages/twenty-server/src/engine/core-modules/usage-limit/utils/build-limit-quota-counter.util.ts b/packages/twenty-server/src/engine/core-modules/usage-limit/utils/build-limit-quota-counter.util.tsindex 288c84d6c4c..17eb7e0e49a 100644--- a/packages/twenty-server/src/engine/core-modules/usage-limit/utils/build-limit-quota-counter.util.ts+++ b/packages/twenty-server/src/engine/core-modules/usage-limit/utils/build-limit-quota-counter.util.ts@@ -10,32 +10,22 @@ export const buildLimitQuotaCounter = ({period,}: {workspaceId: string;- limit: Omit<- FlatQuotaLimit,- 'id' | 'limitKind' | 'periodCount' | 'burstValue'- >;+ limit: Omit<FlatQuotaLimit, 'limitKind' | 'periodCount' | 'burstValue'>;period: UsagePeriod;-}): LimitQuotaCounter => ({- kind: 'limit',- isDefault: false,- key: buildQuotaCounterKey({- workspaceId,- resourceType: limit.resourceType,- operationType: limit.operationType,- spenderType: limit.spenderType,- spenderId: limit.spenderId,+}): LimitQuotaCounter => {+ const counter = {+ kind: 'limit' as const,+ isDefault: false,+ limitValue: limit.limitValue,unit: limit.unit,+ resourceType: limit.resourceType,periodUnit: limit.periodUnit,periodStart: period.periodStart,- limitValue: limit.limitValue,- }),- limitValue: limit.limitValue,- unit: limit.unit,- resourceType: limit.resourceType,- periodUnit: limit.periodUnit,- periodStart: period.periodStart,- periodEnd: period.periodEnd,- spenderType: limit.spenderType,- spenderId: normalizeSpenderId(limit.spenderId),- operationType: limit.operationType,-});+ periodEnd: period.periodEnd,+ spenderType: limit.spenderType,+ spenderId: normalizeSpenderId(limit.spenderId),+ operationType: limit.operationType,+ };++ return { ...counter, key: buildQuotaCounterKey({ workspaceId, counter }) };+};diff --git a/packages/twenty-server/src/engine/core-modules/usage-limit/utils/build-limit-warmed-entries.util.ts b/packages/twenty-server/src/engine/core-modules/usage-limit/utils/build-limit-warmed-entries.util.tsdeleted file mode 100644index 14e13f82bc1..00000000000--- a/packages/twenty-server/src/engine/core-modules/usage-limit/utils/build-limit-warmed-entries.util.ts+++ /dev/null@@ -1,33 +0,0 @@-import { isDefined } from 'twenty-shared/utils';--import { type LimitQuotaCounter } from 'src/engine/core-modules/usage-limit/types/limit-quota-counter.type';-import { type UsageConsumptionRow } from 'src/engine/core-modules/usage/types/usage-consumption-row.type';-import { buildPeriodGroupKey } from 'src/engine/core-modules/usage-limit/utils/build-period-group-key.util';-import { computeQuotaConsumed } from 'src/engine/core-modules/usage-limit/utils/compute-quota-consumed.util';--export const buildLimitWarmedEntries = ({- coldLimitCounters,- rowsByPeriod,- now,-}: {- coldLimitCounters: LimitQuotaCounter[];- rowsByPeriod: Map<string, UsageConsumptionRow[]>;- now: number;-}): { key: string; value: number; ttl: number }[] =>- coldLimitCounters.flatMap((counter) => {- const ttl = counter.periodEnd.getTime() - now;- const rows = rowsByPeriod.get(buildPeriodGroupKey(counter));-- if (ttl <= 0 || !isDefined(rows)) {- return [];- }-- return [- {- key: counter.key,- value:- counter.limitValue - computeQuotaConsumed({ rows, scope: counter }),- ttl,- },- ];- });diff --git a/packages/twenty-server/src/engine/core-modules/usage-limit/utils/build-overridden-quota-default-counter-keys.util.ts b/packages/twenty-server/src/engine/core-modules/usage-limit/utils/build-overridden-quota-default-counter-keys.util.tsdeleted file mode 100644index 3da622d6d41..00000000000--- a/packages/twenty-server/src/engine/core-modules/usage-limit/utils/build-overridden-quota-default-counter-keys.util.ts+++ /dev/null@@ -1,46 +0,0 @@-import { isDefined } from 'twenty-shared/utils';--import { type PeriodUnit } from 'src/engine/core-modules/usage-limit/types/period-unit.type';-import { type QuotaLimitDefault } from 'src/engine/core-modules/usage-limit/types/quota-limit-default.type';-import { type UsageLimitCounterScope } from 'src/engine/core-modules/usage-limit/types/usage-limit-counter-scope.type';-import { type UsagePeriod } from 'src/engine/core-modules/usage-limit/types/usage-period.type';-import { buildQuotaDefaultCounterKey } from 'src/engine/core-modules/usage-limit/utils/build-quota-default-counter-key.util';-import { doesUsageLimitRowSuppressDefault } from 'src/engine/core-modules/usage-limit/utils/does-usage-limit-row-suppress-default.util';--export const buildOverriddenQuotaDefaultCounterKeys = ({- usageLimit,- quotaLimitDefaults,- periodByUnit,-}: {- usageLimit: UsageLimitCounterScope;- quotaLimitDefaults: QuotaLimitDefault[];- periodByUnit: Partial<Record<PeriodUnit, UsagePeriod>>;-}): string[] =>- quotaLimitDefaults- .filter((quotaLimitDefault) =>- doesUsageLimitRowSuppressDefault({- scope: usageLimit,- usageLimitDefault: quotaLimitDefault,- }),- )- .flatMap((quotaLimitDefault) => {- const period = periodByUnit[quotaLimitDefault.periodUnit];-- if (!isDefined(period)) {- return [];- }-- return [- buildQuotaDefaultCounterKey({- workspaceId: usageLimit.workspaceId,- resourceType: quotaLimitDefault.resourceType,- operationType: quotaLimitDefault.operationType,- spenderType: quotaLimitDefault.spenderType,- spenderId: quotaLimitDefault.spenderId,- unit: quotaLimitDefault.unit,- periodUnit: quotaLimitDefault.periodUnit,- periodStart: period.periodStart,- limitValue: quotaLimitDefault.limitValue,- }),- ];- });diff --git a/packages/twenty-server/src/engine/core-modules/usage-limit/utils/build-quota-counter-key.util.ts b/packages/twenty-server/src/engine/core-modules/usage-limit/utils/build-quota-counter-key.util.tsindex f1ffc408845..88e19f3da8c 100644--- a/packages/twenty-server/src/engine/core-modules/usage-limit/utils/build-quota-counter-key.util.ts+++ b/packages/twenty-server/src/engine/core-modules/usage-limit/utils/build-quota-counter-key.util.ts@@ -1,32 +1,23 @@import { isNonEmptyString } from '@sniptt/guards';-import { type PeriodUnit } from 'src/engine/core-modules/usage-limit/types/period-unit.type';-import { type SpenderType } from 'src/engine/core-modules/usage-limit/types/spender-type.type';-import { type UsageOperationType } from 'src/engine/core-modules/usage/enums/usage-operation-type.enum';-import { type UsageResourceType } from 'src/engine/core-modules/usage/enums/usage-resource-type.enum';-import { type UsageUnit } from 'src/engine/core-modules/usage/enums/usage-unit.enum';+import { type LimitQuotaCounter } from 'src/engine/core-modules/usage-limit/types/limit-quota-counter.type';const ABSENT = '-';export const buildQuotaCounterKey = ({workspaceId,- resourceType,- operationType,- spenderType,- spenderId,- unit,- periodUnit,- periodStart,- limitValue,+ counter,}: {workspaceId: string;- resourceType: UsageResourceType;- operationType: UsageOperationType;- spenderType: SpenderType;- spenderId?: string | null;- unit: UsageUnit;- periodUnit: PeriodUnit;- periodStart: Date;- limitValue: number;+ counter: Pick<+ LimitQuotaCounter,+ | 'resourceType'+ | 'operationType'+ | 'spenderType'+ | 'spenderId'+ | 'unit'+ | 'periodUnit'+ | 'periodStart'+ >;}): string =>- `{${workspaceId}}:quota:${resourceType}:${operationType}:${spenderType}:${isNonEmptyString(spenderId) ? spenderId : ABSENT}:${unit}:${periodUnit}:${periodStart.getTime()}:${limitValue}`;+ `{${workspaceId}}:quota-consumed:${counter.resourceType}:${counter.operationType}:${counter.spenderType}:${isNonEmptyString(counter.spenderId) ? counter.spenderId : ABSENT}:${counter.unit}:${counter.periodUnit}:${counter.periodStart.getTime()}`;diff --git a/packages/twenty-server/src/engine/core-modules/usage-limit/utils/build-quota-default-counter-key.util.ts b/packages/twenty-server/src/engine/core-modules/usage-limit/utils/build-quota-default-counter-key.util.tsdeleted file mode 100644index bb961bcaccf..00000000000--- a/packages/twenty-server/src/engine/core-modules/usage-limit/utils/build-quota-default-counter-key.util.ts+++ /dev/null@@ -1,6 +0,0 @@-import { type QuotaDefaultKeyScope } from 'src/engine/core-modules/usage-limit/types/quota-default-key-scope.type';-import { buildQuotaCounterKey } from 'src/engine/core-modules/usage-limit/utils/build-quota-counter-key.util';--export const buildQuotaDefaultCounterKey = (- scope: QuotaDefaultKeyScope & { limitValue: number },-): string => `${buildQuotaCounterKey(scope)}:default`;diff --git a/packages/twenty-server/src/engine/core-modules/usage-limit/utils/build-quota-default-counter.util.ts b/packages/twenty-server/src/engine/core-modules/usage-limit/utils/build-quota-default-counter.util.tsindex 2ac730948b0..03089989af3 100644--- a/packages/twenty-server/src/engine/core-modules/usage-limit/utils/build-quota-default-counter.util.ts+++ b/packages/twenty-server/src/engine/core-modules/usage-limit/utils/build-quota-default-counter.util.ts@@ -1,7 +1,7 @@import { type LimitQuotaCounter } from 'src/engine/core-modules/usage-limit/types/limit-quota-counter.type';import { type QuotaLimitDefault } from 'src/engine/core-modules/usage-limit/types/quota-limit-default.type';import { type UsagePeriod } from 'src/engine/core-modules/usage-limit/types/usage-period.type';-import { buildQuotaDefaultCounterKey } from 'src/engine/core-modules/usage-limit/utils/build-quota-default-counter-key.util';+import { buildQuotaCounterKey } from 'src/engine/core-modules/usage-limit/utils/build-quota-counter-key.util';export const buildQuotaDefaultCounter = ({workspaceId,@@ -11,27 +11,20 @@ export const buildQuotaDefaultCounter = ({workspaceId: string;quotaLimitDefault: QuotaLimitDefault;period: UsagePeriod;-}): LimitQuotaCounter => ({- kind: 'limit',- isDefault: true,- key: buildQuotaDefaultCounterKey({- workspaceId,- resourceType: quotaLimitDefault.resourceType,- operationType: quotaLimitDefault.operationType,- spenderType: quotaLimitDefault.spenderType,- spenderId: quotaLimitDefault.spenderId,+}): LimitQuotaCounter => {+ const counter = {+ kind: 'limit' as const,+ isDefault: true,+ limitValue: quotaLimitDefault.limitValue,unit: quotaLimitDefault.unit,+ resourceType: quotaLimitDefault.resourceType,periodUnit: quotaLimitDefault.periodUnit,periodStart: period.periodStart,- limitValue: quotaLimitDefault.limitValue,- }),- limitValue: quotaLimitDefault.limitValue,- unit: quotaLimitDefault.unit,- resourceType: quotaLimitDefault.resourceType,- periodUnit: quotaLimitDefault.periodUnit,- periodStart: period.periodStart,- periodEnd: period.periodEnd,- spenderType: quotaLimitDefault.spenderType,- spenderId: null,- operationType: quotaLimitDefault.operationType,-});+ periodEnd: period.periodEnd,+ spenderType: quotaLimitDefault.spenderType,+ spenderId: null,+ operationType: quotaLimitDefault.operationType,+ };++ return { ...counter, key: buildQuotaCounterKey({ workspaceId, counter }) };+};diff --git a/packages/twenty-server/src/engine/core-modules/usage-limit/utils/build-quota-exhausted-scope.util.ts b/packages/twenty-server/src/engine/core-modules/usage-limit/utils/build-quota-exhausted-scope.util.tsindex 2b37977b0c8..2b07753dc9d 100644--- a/packages/twenty-server/src/engine/core-modules/usage-limit/utils/build-quota-exhausted-scope.util.ts+++ b/packages/twenty-server/src/engine/core-modules/usage-limit/utils/build-quota-exhausted-scope.util.ts@@ -1,4 +1,3 @@-import { type CreditAllowance } from 'src/engine/core-modules/usage-limit/types/credit-allowance.type';import { type ExhaustedScope } from 'src/engine/core-modules/usage-limit/types/exhausted-scope.type';import { type QuotaCounter } from 'src/engine/core-modules/usage-limit/types/quota-counter.type';import { UsageOperationType } from 'src/engine/core-modules/usage/enums/usage-operation-type.enum';@@ -7,11 +6,9 @@ import { type UsageResourceType } from 'src/engine/core-modules/usage/enums/usagexport const buildQuotaExhaustedScope = ({resourceType,counter,- allowance,}: {resourceType: UsageResourceType;counter: QuotaCounter;- allowance: CreditAllowance | null;}): ExhaustedScope => {const retryAfterMs = Math.max(counter.periodEnd.getTime() - Date.now(), 0);@@ -41,7 +38,7 @@ export const buildQuotaExhaustedScope = ({spenderId: null,operationType: UsageOperationType.ALL,unit: counter.unit,- limitValue: allowance?.allowanceMicro ?? 0,+ limitValue: counter.limitValue,remaining: 0,periodCount: null,periodUnit: null,diff --git a/packages/twenty-server/src/engine/core-modules/usage-limit/utils/build-quota-warm-lock-key.util.ts b/packages/twenty-server/src/engine/core-modules/usage-limit/utils/build-quota-warm-lock-key.util.tsdeleted file mode 100644index f36d476de10..00000000000--- a/packages/twenty-server/src/engine/core-modules/usage-limit/utils/build-quota-warm-lock-key.util.ts+++ /dev/null@@ -1,2 +0,0 @@-export const buildQuotaWarmLockKey = (workspaceId: string): string =>- `usage-quota-warm:${workspaceId}`;diff --git a/packages/twenty-server/src/engine/core-modules/usage-limit/utils/compute-credit-allowance-micro.util.ts b/packages/twenty-server/src/engine/core-modules/usage-limit/utils/compute-credit-allowance-micro.util.tsnew file mode 100644index 00000000000..b469b0340ef--- /dev/null+++ b/packages/twenty-server/src/engine/core-modules/usage-limit/utils/compute-credit-allowance-micro.util.ts@@ -0,0 +1,23 @@+import { isDefined } from 'twenty-shared/utils';++import { type CreditAllowanceGrant } from 'src/engine/core-modules/usage-limit/types/credit-allowance-grant.type';++export const computeCreditAllowanceMicro = ({+ planAllowanceMicro,+ grants,+ nowMs,+}: {+ planAllowanceMicro: number;+ grants: CreditAllowanceGrant[];+ nowMs: number;+}): number =>+ grants+ .filter(+ ({ effectiveAtMs, expiresAtMs }) =>+ effectiveAtMs <= nowMs &&+ (!isDefined(expiresAtMs) || expiresAtMs > nowMs),+ )+ .reduce(+ (allowanceMicro, { amountMicro }) => allowanceMicro + amountMicro,+ planAllowanceMicro,+ );diff --git a/packages/twenty-server/src/engine/core-modules/usage-limit/utils/find-exhausted-counters.util.ts b/packages/twenty-server/src/engine/core-modules/usage-limit/utils/find-exhausted-counters.util.tsindex 0c566e01d4a..e35166ded43 100644--- a/packages/twenty-server/src/engine/core-modules/usage-limit/utils/find-exhausted-counters.util.ts+++ b/packages/twenty-server/src/engine/core-modules/usage-limit/utils/find-exhausted-counters.util.ts@@ -2,26 +2,27 @@ import { isDefined } from 'twenty-shared/utils';import { type QuotaCost } from 'src/engine/core-modules/usage-limit/types/quota-cost.type';import { type QuotaCounter } from 'src/engine/core-modules/usage-limit/types/quota-counter.type';+import { isLimitExhausted } from 'src/engine/core-modules/usage-limit/utils/is-limit-exhausted.util';export const findExhaustedCounters = ({counters,- remainings,+ consumedValues,cost,}: {counters: QuotaCounter[];- remainings: (number | null)[];+ consumedValues: (number | null)[];cost?: QuotaCost;}): QuotaCounter[] =>counters.filter((counter, index) => {- const remaining = remainings[index];+ const consumed = consumedValues[index];- if (!isDefined(remaining)) {+ if (!isDefined(consumed)) {return false;}- if (remaining <= 0) {- return true;- }-- return remaining < (cost?.[counter.unit] ?? 0);+ return isLimitExhausted({+ consumed,+ cost: cost?.[counter.unit] ?? 0,+ limitValue: counter.limitValue,+ });});diff --git a/packages/twenty-server/src/engine/core-modules/usage-limit/utils/from-consume-results-to-remainings.util.ts b/packages/twenty-server/src/engine/core-modules/usage-limit/utils/from-consume-results-to-remainings.util.tsdeleted file mode 100644index d0608170191..00000000000--- a/packages/twenty-server/src/engine/core-modules/usage-limit/utils/from-consume-results-to-remainings.util.ts+++ /dev/null@@ -1,8 +0,0 @@-// The consume script answers [existed, remaining] pairs, one per counter key;-// a counter that did not exist was not debited and stays cold (null).-export const fromConsumeResultsToRemainings = (- consumeResults: number[],-): (number | null)[] =>- Array.from({ length: consumeResults.length / 2 }, (_, index) =>- consumeResults[2 * index] === 1 ? consumeResults[2 * index + 1] : null,- );diff --git a/packages/twenty-server/src/engine/core-modules/usage-limit/utils/is-limit-exhausted.util.ts b/packages/twenty-server/src/engine/core-modules/usage-limit/utils/is-limit-exhausted.util.tsnew file mode 100644index 00000000000..bf48bd37437--- /dev/null+++ b/packages/twenty-server/src/engine/core-modules/usage-limit/utils/is-limit-exhausted.util.ts@@ -0,0 +1,9 @@+export const isLimitExhausted = ({+ consumed,+ cost,+ limitValue,+}: {+ consumed: number;+ cost: number;+ limitValue: number;+}): boolean => consumed >= limitValue || consumed + cost > limitValue;diff --git a/packages/twenty-server/test/integration/billing/suites/billing-credit-grant-admin.integration-spec.ts b/packages/twenty-server/test/integration/billing/suites/billing-credit-grant-admin.integration-spec.tsindex 7bdd55e8462..5e7a29d69e6 100644--- a/packages/twenty-server/test/integration/billing/suites/billing-credit-grant-admin.integration-spec.ts+++ b/packages/twenty-server/test/integration/billing/suites/billing-credit-grant-admin.integration-spec.ts@@ -4,13 +4,14 @@ import { randomUUID } from 'node:crypto';import { addDays, addMonths, startOfMonth } from 'date-fns';import request from 'supertest';import {+ debitInFlightCredits,getSeededBillingWorkspaceId,listCreditGrants,quitBillingFixtureRedis,- readAllowanceCounter,+ readAllowanceConsumedMicro,+ refreshCurrentBillingSubscription,resetBillingCreditState,setupResourceCreditSubscription,- warmAllowanceCounter,setSubscriptionStatus,} from 'test/integration/billing/utils/billing-credit-fixtures.util';@@ -24,6 +25,7 @@ const client = request(`http://localhost:${APP_PORT}`);const PERIOD_START = startOfMonth(new Date());const PERIOD_END = addMonths(PERIOD_START, 1);const ALLOWANCE_MICRO = 1_000_000;+const IN_FLIGHT_CREDITS_MICRO = 120_000;const GRANT_MUTATION = `mutation GrantWorkspaceCredits(@@ -181,9 +183,12 @@ describe('Admin credit grant and revoke (integration)', () => {expect(await listCreditGrants(workspaceId)).toHaveLength(0);});- it('drops the warm allowance counter when a grant lands', async () => {- await warmAllowanceCounter(workspaceId, PERIOD_START, 500_000);+ it('keeps in-flight usage on the allowance when a grant lands', async () => {+ await refreshCurrentBillingSubscription(workspaceId);+ const consumedBeforeMicro = await readAllowanceConsumedMicro(workspaceId);++ await debitInFlightCredits(workspaceId, IN_FLIGHT_CREDITS_MICRO);await grantCredits({workspaceId,amount: 2,@@ -191,10 +196,12 @@ describe('Admin credit grant and revoke (integration)', () => {reason: null,});- expect(await readAllowanceCounter(workspaceId, PERIOD_START)).toBeNull();+ expect(await readAllowanceConsumedMicro(workspaceId)).toBe(+ consumedBeforeMicro + IN_FLIGHT_CREDITS_MICRO,+ );});- it('takes a revoked grant off the ledger and drops the counter', async () => {+ it('takes a revoked grant off the ledger and keeps in-flight usage', async () => {const granted = await grantCredits({workspaceId,amount: 2,@@ -202,8 +209,9 @@ describe('Admin credit grant and revoke (integration)', () => {reason: null,});const creditGrantId = granted.body.data.grantWorkspaceCredits.id;+ const consumedBeforeMicro = await readAllowanceConsumedMicro(workspaceId);- await warmAllowanceCounter(workspaceId, PERIOD_START, 500_000);+ await debitInFlightCredits(workspaceId, IN_FLIGHT_CREDITS_MICRO);const revoked = await callAdminGraphql(REVOKE_MUTATION, {workspaceId,@@ -218,10 +226,12 @@ describe('Admin credit grant and revoke (integration)', () => {const grants = await listCreditGrants(workspaceId);expect(grants[0].revokedAt).not.toBeNull();- expect(await readAllowanceCounter(workspaceId, PERIOD_START)).toBeNull();+ expect(await readAllowanceConsumedMicro(workspaceId)).toBe(+ consumedBeforeMicro + IN_FLIGHT_CREDITS_MICRO,+ );});- it('drops the counter again when a revocation is retried', async () => {+ it('keeps in-flight usage when a revocation is retried', async () => {const granted = await grantCredits({workspaceId,amount: 2,@@ -231,7 +241,10 @@ describe('Admin credit grant and revoke (integration)', () => {const creditGrantId = granted.body.data.grantWorkspaceCredits.id;await callAdminGraphql(REVOKE_MUTATION, { workspaceId, creditGrantId });- await warmAllowanceCounter(workspaceId, PERIOD_START, 500_000);++ const consumedBeforeMicro = await readAllowanceConsumedMicro(workspaceId);++ await debitInFlightCredits(workspaceId, IN_FLIGHT_CREDITS_MICRO);const retried = await callAdminGraphql(REVOKE_MUTATION, {workspaceId,@@ -239,7 +252,9 @@ describe('Admin credit grant and revoke (integration)', () => {});expect(retried.body.errors).toBeUndefined();- expect(await readAllowanceCounter(workspaceId, PERIOD_START)).toBeNull();+ expect(await readAllowanceConsumedMicro(workspaceId)).toBe(+ consumedBeforeMicro + IN_FLIGHT_CREDITS_MICRO,+ );});it.each([@@ -286,14 +301,12 @@ describe('Admin credit grant and revoke (integration)', () => {};const first = await callAdminGraphql(GRANT_MUTATION, variables);- await warmAllowanceCounter(workspaceId, PERIOD_START, 500_000);const second = await callAdminGraphql(GRANT_MUTATION, variables);expect(second.body.data.grantWorkspaceCredits.id).toBe(first.body.data.grantWorkspaceCredits.id,);expect(await listCreditGrants(workspaceId)).toHaveLength(1);- expect(await readAllowanceCounter(workspaceId, PERIOD_START)).toBeNull();});// The anchoring subscription can be gone by retry time, and the operation already succeededdiff --git a/packages/twenty-server/test/integration/billing/suites/billing-credit-rollover.integration-spec.ts b/packages/twenty-server/test/integration/billing/suites/billing-credit-rollover.integration-spec.tsindex 379e17e7581..7146745cd0f 100644--- a/packages/twenty-server/test/integration/billing/suites/billing-credit-rollover.integration-spec.ts+++ b/packages/twenty-server/test/integration/billing/suites/billing-credit-rollover.integration-spec.ts@@ -2,15 +2,16 @@ import { addMonths, startOfMonth, subMonths } from 'date-fns';import request from 'supertest';import { createMockStripeInvoiceFinalizedData } from 'test/integration/billing/utils/create-mock-stripe-invoice-finalized-data.util';import {+ debitInFlightCredits,getSeededBillingWorkspaceId,insertCreditGrant,listCreditGrants,quitBillingFixtureRedis,- readAllowanceCounter,+ readAllowanceConsumedMicro,+ refreshCurrentBillingSubscription,resetBillingCreditState,setupResourceCreditSubscription,TEST_STRIPE_CUSTOMER_ID,- warmAllowanceCounter,} from 'test/integration/billing/utils/billing-credit-fixtures.util';import { getAppProviderByClassName } from 'test/integration/utils/get-app-provider-by-class-name.util';@@ -26,6 +27,7 @@ const CLOSING_PERIOD_END = PERIOD_BOUNDARY;const NEXT_PERIOD_END = addMonths(PERIOD_BOUNDARY, 1);const ALLOWANCE_MICRO = 1_000_000;+const IN_FLIGHT_CREDITS_MICRO = 120_000;const postInvoiceFinalized = (invoiceId = 'in_test_default',@@ -200,30 +202,41 @@ describe('Billing credit rollover (integration)', () => {expect(await listCreditGrants(workspaceId)).toHaveLength(0);});- it('drops the allowance counter on the period transition', async () => {- usageSpy.mockResolvedValue(300_000);+ describe('once Stripe has advanced the subscription', () => {+ beforeEach(async () => {+ usageSpy.mockResolvedValue(300_000);+ await setupResourceCreditSubscription({+ workspaceId,+ periodStart: CLOSING_PERIOD_END,+ periodEnd: NEXT_PERIOD_END,+ creditAmountMicro: ALLOWANCE_MICRO,+ });+ await refreshCurrentBillingSubscription(workspaceId);+ });- // Stripe advances the subscription in a separate event, so the counter is keyed by the closing period- await warmAllowanceCounter(workspaceId, CLOSING_PERIOD_START, 120_000);+ it('keeps in-flight usage of the opened period through the transition', async () => {+ const consumedBeforeMicro = await readAllowanceConsumedMicro(workspaceId);- await postInvoiceFinalized().expect(200);+ await debitInFlightCredits(workspaceId, IN_FLIGHT_CREDITS_MICRO);+ await postInvoiceFinalized().expect(200);- expect(- await readAllowanceCounter(workspaceId, CLOSING_PERIOD_START),- ).toBeNull();- });+ expect(await readAllowanceConsumedMicro(workspaceId)).toBe(+ consumedBeforeMicro + IN_FLIGHT_CREDITS_MICRO,+ );+ });- it('drops the allowance counter again when a successful delivery is repeated', async () => {- usageSpy.mockResolvedValue(300_000);+ it('keeps in-flight usage when a successful delivery is repeated', async () => {+ await postInvoiceFinalized().expect(200);- await postInvoiceFinalized().expect(200);- await warmAllowanceCounter(workspaceId, CLOSING_PERIOD_START, 120_000);+ const consumedBeforeMicro = await readAllowanceConsumedMicro(workspaceId);- await postInvoiceFinalized().expect(200);+ await debitInFlightCredits(workspaceId, IN_FLIGHT_CREDITS_MICRO);+ await postInvoiceFinalized().expect(200);- expect(- await readAllowanceCounter(workspaceId, CLOSING_PERIOD_START),- ).toBeNull();+ expect(await readAllowanceConsumedMicro(workspaceId)).toBe(+ consumedBeforeMicro + IN_FLIGHT_CREDITS_MICRO,+ );+ });});// A 31st anchor runs Jan 31 to Feb 28; once advanced, calendar arithmetic clamps Feb 28 back to Jan 28 and swallows three daysdiff --git a/packages/twenty-server/test/integration/billing/suites/billing-credit-top-up.integration-spec.ts b/packages/twenty-server/test/integration/billing/suites/billing-credit-top-up.integration-spec.tsindex 8c03c49f521..151b7945dc9 100644--- a/packages/twenty-server/test/integration/billing/suites/billing-credit-top-up.integration-spec.ts+++ b/packages/twenty-server/test/integration/billing/suites/billing-credit-top-up.integration-spec.ts@@ -3,14 +3,16 @@ import { randomUUID } from 'node:crypto';import { addMonths, startOfMonth } from 'date-fns';import request from 'supertest';import {+ debitInFlightCredits,getSeededBillingWorkspaceId,listCreditGrants,quitBillingFixtureRedis,- readAllowanceCounter,+ readAllowanceConsumedMicro,+ readAllowanceLimitMicro,+ refreshCurrentBillingSubscription,resetBillingCreditState,setupResourceCreditSubscription,TEST_STRIPE_CUSTOMER_ID,- warmAllowanceCounter,} from 'test/integration/billing/utils/billing-credit-fixtures.util';import { createMockStripeCreditTopUpInvoicePaidData } from 'test/integration/billing/utils/create-mock-stripe-credit-top-up-invoice-paid-data.util';@@ -23,6 +25,7 @@ const PERIOD_END = addMonths(PERIOD_START, 1);const ALLOWANCE_MICRO = 1_000_000;const TOP_UP_MICRO = 2_000_000;+const IN_FLIGHT_CREDITS_MICRO = 120_000;const postCreditTopUpInvoicePaid = ({invoiceId,@@ -99,15 +102,24 @@ describe('Billing credit top-up webhook (integration)', () => {});});- it('drops the allowance counter so a blocked workspace resumes at once', async () => {- await warmAllowanceCounter(workspaceId, PERIOD_START, 0);+ it('raises the allowance cap at once and keeps in-flight usage', async () => {+ await refreshCurrentBillingSubscription(workspaceId);+ const limitBeforeMicro = await readAllowanceLimitMicro(workspaceId);+ const consumedBeforeMicro = await readAllowanceConsumedMicro(workspaceId);++ await debitInFlightCredits(workspaceId, IN_FLIGHT_CREDITS_MICRO);await postCreditTopUpInvoicePaid({invoiceId: `in_test_top_up_${randomUUID()}`,workspaceId,});- expect(await readAllowanceCounter(workspaceId, PERIOD_START)).toBeNull();+ expect(await readAllowanceLimitMicro(workspaceId)).toBe(+ limitBeforeMicro + TOP_UP_MICRO,+ );+ expect(await readAllowanceConsumedMicro(workspaceId)).toBe(+ consumedBeforeMicro + IN_FLIGHT_CREDITS_MICRO,+ );});it('grants nothing more when Stripe redelivers the event', async () => {diff --git a/packages/twenty-server/test/integration/billing/utils/billing-credit-fixtures.util.ts b/packages/twenty-server/test/integration/billing/utils/billing-credit-fixtures.util.tsindex 370c483526f..c24336026e5 100644--- a/packages/twenty-server/test/integration/billing/utils/billing-credit-fixtures.util.ts+++ b/packages/twenty-server/test/integration/billing/utils/billing-credit-fixtures.util.ts@@ -1,10 +1,14 @@+import { getAppProviderByClassName } from 'test/integration/utils/get-app-provider-by-class-name.util';import { createClient, type RedisClientType } from 'redis';import { isDefined, isNonEmptyArray } from 'twenty-shared/utils';import { type BillingCreditGrantType } from 'src/engine/core-modules/billing/enums/billing-credit-grant-type.enum';import { type SubscriptionStatus } from 'src/engine/core-modules/billing/enums/billing-subscription-status.enum';-import { CacheStorageNamespace } from 'src/engine/core-modules/cache-storage/types/cache-storage-namespace.enum';-import { buildAllowanceCounterKey } from 'src/engine/core-modules/usage-limit/utils/build-allowance-counter-key.util';+import { type UsageLimitQuotaService } from 'src/engine/core-modules/usage-limit/services/usage-limit-quota.service';+import { UsageOperationType } from 'src/engine/core-modules/usage/enums/usage-operation-type.enum';+import { UsageResourceType } from 'src/engine/core-modules/usage/enums/usage-resource-type.enum';+import { UsageUnit } from 'src/engine/core-modules/usage/enums/usage-unit.enum';+import { type WorkspaceCacheService } from 'src/engine/workspace-cache/services/workspace-cache.service';// The dev seeder creates no subscription item or period; rollover needs subscription -> item -> product -> priceexport const TEST_STRIPE_CUSTOMER_ID = 'cus_default0';@@ -182,35 +186,57 @@ export const quitBillingFixtureRedis = async (): Promise<void> => {}};-const buildTestAllowanceCounterKey = (+const getUsageLimitQuotaService = () =>+ getAppProviderByClassName<UsageLimitQuotaService>('UsageLimitQuotaService');++export const refreshCurrentBillingSubscription = (workspaceId: string,- periodStart: Date,-): string =>- `${CacheStorageNamespace.IntegrationTests}:${CacheStorageNamespace.EngineUsageLimit}:${buildAllowanceCounterKey({ workspaceId, periodStart })}`;+): Promise<void> =>+ getAppProviderByClassName<WorkspaceCacheService>(+ 'WorkspaceCacheService',+ ).invalidateAndRecompute(workspaceId, ['currentBillingSubscription']);-export const warmAllowanceCounter = async (+export const debitInFlightCredits = async (workspaceId: string,- periodStart: Date,- valueMicro: number,+ creditsUsedMicro: number,): Promise<void> => {- const redis = await getRedisClient();+ await getUsageLimitQuotaService().debitAheadOfRecord({+ workspaceId,+ event: {+ resourceType: UsageResourceType.AI,+ operationType: UsageOperationType.AI_CHAT_TOKEN,+ unit: UsageUnit.TOKEN,+ quantity: 1,+ creditsUsedMicro,+ spenders: {},+ },+ });+};- await redis.set(- buildTestAllowanceCounterKey(workspaceId, periodStart),- String(valueMicro),- );+export const readAllowanceConsumedMicro = async (+ workspaceId: string,+): Promise<number> => {+ const allowanceUsage =+ await getUsageLimitQuotaService().getAllowanceUsage(workspaceId);++ if (!isDefined(allowanceUsage?.consumedValue)) {+ throw new Error(`No allowance consumption for workspace ${workspaceId}`);+ }++ return allowanceUsage.consumedValue;};-export const readAllowanceCounter = async (+export const readAllowanceLimitMicro = async (workspaceId: string,- periodStart: Date,-): Promise<number | null> => {- const redis = await getRedisClient();- const value = await redis.get(- buildTestAllowanceCounterKey(workspaceId, periodStart),- );+): Promise<number> => {+ const allowanceUsage =+ await getUsageLimitQuotaService().getAllowanceUsage(workspaceId);++ if (!isDefined(allowanceUsage)) {+ throw new Error(`No allowance for workspace ${workspaceId}`);+ }- return isDefined(value) ? Number(value) : null;+ return allowanceUsage.limitValue;};export const resetBillingCreditState = async (@@ -223,7 +249,7 @@ export const resetBillingCreditState = async (const redis = await getRedisClient();const staleKeys = [- ...(await redis.keys(`*{${workspaceId}}:quota:allowance:*`)),+ ...(await redis.keys(`*{${workspaceId}}:quota*`)),...(await redis.keys(`*currentBillingSubscription:${workspaceId}*`)),];diff --git a/packages/twenty-server/test/integration/usage-limit/quota-consumption.integration-spec.ts b/packages/twenty-server/test/integration/usage-limit/quota-consumption.integration-spec.tsindex e86bd1c01b8..32eb0521174 100644--- a/packages/twenty-server/test/integration/usage-limit/quota-consumption.integration-spec.ts+++ b/packages/twenty-server/test/integration/usage-limit/quota-consumption.integration-spec.ts@@ -6,6 +6,7 @@ import { expectEventually } from 'test/integration/utils/expect-eventually.util'import { getAppProviderByClassName } from 'test/integration/utils/get-app-provider-by-class-name.util';import { getCoreRepository } from 'test/integration/utils/get-core-repository.util';import { jestExpectToBeDefined } from 'test/utils/jest-expect-to-be-defined.util.test';+import { randomUUID } from 'node:crypto';import {type ClickHouseClient,@@ -18,10 +19,22 @@ import { isNonEmptyArray } from 'twenty-shared/utils';import { type Repository } from 'typeorm';import { formatDateTimeForClickHouse } from 'src/database/clickhouse/utils/format-date-time-for-clickhouse.util';+import { type CacheStorageService } from 'src/engine/core-modules/cache-storage/services/cache-storage.service';+import { type MetricsService } from 'src/engine/core-modules/metrics/metrics.service';+import { MetricsKeys } from 'src/engine/core-modules/metrics/types/metrics-keys.type';+import { type TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';+import { type CreateUsageLimitInput } from 'src/engine/core-modules/usage-limit/dtos/create-usage-limit.input';+import { type UsageLimitEntitlementService } from 'src/engine/core-modules/usage-limit/services/usage-limit-entitlement.service';+import { type UsageLimitQuotaService } from 'src/engine/core-modules/usage-limit/services/usage-limit-quota.service';+import { type UsageLimitService } from 'src/engine/core-modules/usage-limit/services/usage-limit.service';+import { type PeriodUnit } from 'src/engine/core-modules/usage-limit/types/period-unit.type';import { UsageLimitEntity } from 'src/engine/core-modules/usage-limit/usage-limit.entity';+import { getCalendarMonthPeriod } from 'src/engine/core-modules/usage-limit/utils/get-calendar-month-period.util';import { UsageOperationType } from 'src/engine/core-modules/usage/enums/usage-operation-type.enum';import { UsageResourceType } from 'src/engine/core-modules/usage/enums/usage-resource-type.enum';import { UsageUnit } from 'src/engine/core-modules/usage/enums/usage-unit.enum';+import { type UsageAnalyticsService } from 'src/engine/core-modules/usage/services/usage-analytics.service';+import { type RecordUsageInput } from 'src/engine/core-modules/usage/types/record-usage-input.type';import { LogicFunctionExecutionStatus } from 'src/engine/metadata-modules/logic-function/dtos/logic-function-execution-result.dto';import { type WorkspaceCacheService } from 'src/engine/workspace-cache/services/workspace-cache.service';import { SEED_APPLE_WORKSPACE_ID } from 'src/engine/workspace-manager/dev-seeder/core/constants/seeder-workspaces.constant';@@ -57,6 +70,8 @@ const QUOTA_UNITS = [UsageUnit.INVOCATION, UsageUnit.CREDIT] as const;const EXECUTED_RUN_COUNT = 2;const UNREACHABLE_LIMIT_VALUE = 1_000_000_000_000;+const CREDIT_LIMIT_MICRO = 1_000;+const CONCURRENT_CHARGE_COUNT = 8;const CLICKHOUSE_FLUSH_TIMEOUT_MS = 30_000;const EXECUTION_TIMEOUT_MS = 120_000;@@ -81,6 +96,22 @@ const USAGE_QUOTA_SCOPE_CONSUMPTION = gql`type QuotaUnit = (typeof QUOTA_UNITS)[number];+const buildRunEvent = ({+ spenderId,+ creditsUsedMicro,+}: {+ spenderId: string;+ creditsUsedMicro: number;+}): RecordUsageInput => ({+ resourceType: UsageResourceType.LOGIC_FUNCTION,+ operationType: UsageOperationType.CODE_EXECUTION,+ unit: UsageUnit.INVOCATION,+ quantity: 1,+ creditsUsedMicro,+ resourceId: spenderId,+ spenders: { logicFunctionId: spenderId },+});+describe('Usage quota consumption', () => {let clickHouseClient: ClickHouseClient;let redis: Awaited<ReturnType<typeof createRedisClient>>;@@ -88,24 +119,89 @@ describe('Usage quota consumption', () => {let logicFunctionId: string;let usageLimitIdByUnit: Record<QuotaUnit, string>;const savedUsageLimitIds: string[] = [];+ const chargedSpenderIds: string[] = [];++ const getQuotaService = () =>+ getAppProviderByClassName<UsageLimitQuotaService>('UsageLimitQuotaService');++ const getUsageLimitService = () =>+ getAppProviderByClassName<UsageLimitService>('UsageLimitService');const refreshUsageLimitsCache = () =>getAppProviderByClassName<WorkspaceCacheService>('WorkspaceCacheService',).invalidateAndRecompute(SEED_APPLE_WORKSPACE_ID, ['usageLimits']);- const findQuotaCounterKeys = () =>- redis.keys(- `*{${SEED_APPLE_WORKSPACE_ID}}:quota:${UsageResourceType.LOGIC_FUNCTION}:${UsageOperationType.CODE_EXECUTION}:logicFunction:${logicFunctionId}:*`,+ const findSpenderCounterKeys = (spenderId: string) =>+ redis.keys(`*{${SEED_APPLE_WORKSPACE_ID}}:*:logicFunction:${spenderId}:*`);++ const findQuotaCounterKeys = () => findSpenderCounterKeys(logicFunctionId);++ const flushQuotaCounters = async () => {+ const counterKeys = await redis.keys(+ `*{${SEED_APPLE_WORKSPACE_ID}}:quota*`,);- const countUsageEventRows = async (): Promise<number> => {+ if (counterKeys.length > 0) {+ await redis.del(counterKeys);+ }+ };++ const spyOnMetrics = () =>+ jest.spyOn(+ getAppProviderByClassName<MetricsService>('MetricsService'),+ 'incrementCounterBy',+ );++ const findAdmittedOnFailureAttributes = (+ incrementCounterBySpy: ReturnType<typeof spyOnMetrics>,+ ) =>+ incrementCounterBySpy.mock.calls+ .map(([metric]) => metric)+ .filter(+ (metric) => metric.key === MetricsKeys.UsageLimitQuotaAdmittedOnFailure,+ )+ .map((metric) => metric.attributes);++ const chargeRun = (event: Parameters<typeof buildRunEvent>[0]) =>+ getQuotaService().charge({+ workspaceId: SEED_APPLE_WORKSPACE_ID,+ events: [buildRunEvent(event)],+ });++ const debitRunAheadOfRecord = (event: Parameters<typeof buildRunEvent>[0]) =>+ getQuotaService().debitAheadOfRecord({+ workspaceId: SEED_APPLE_WORKSPACE_ID,+ event: buildRunEvent(event),+ });++ const findRunRefusal = ({+ spenderId,+ costMicro,+ }: {+ spenderId: string;+ costMicro: number;+ }) =>+ getQuotaService().findExhaustedScope({+ workspaceId: SEED_APPLE_WORKSPACE_ID,+ resourceType: UsageResourceType.LOGIC_FUNCTION,+ operationType: UsageOperationType.CODE_EXECUTION,+ spenders: { logicFunctionId: spenderId },+ cost: { [UsageUnit.CREDIT]: costMicro },+ });++ const countUsageEventRows = async (+ spenderId: string = logicFunctionId,+ ): Promise<number> => {const result = await clickHouseClient.query({query: `SELECT count() AS rowCountFROM usageEventWHERE workspaceId = {workspaceId:String}AND logicFunctionId = {logicFunctionId:String}`,- query_params: { workspaceId: SEED_APPLE_WORKSPACE_ID, logicFunctionId },+ query_params: {+ workspaceId: SEED_APPLE_WORKSPACE_ID,+ logicFunctionId: spenderId,+ },format: 'JSONEachRow',});@@ -156,18 +252,26 @@ describe('Usage quota consumption', () => {});};- const saveCodeExecutionQuota = async (unit: QuotaUnit): Promise<string> => {+ const saveCodeExecutionQuota = async ({+ spenderId,+ unit,+ limitValue,+ }: {+ spenderId: string;+ unit: QuotaUnit;+ limitValue: number;+ }): Promise<string> => {const usageLimit = await usageLimitRepository.save({workspaceId: SEED_APPLE_WORKSPACE_ID,resourceType: UsageResourceType.LOGIC_FUNCTION,operationType: UsageOperationType.CODE_EXECUTION,spenderType: 'logicFunction',- spenderId: logicFunctionId,+ spenderId,limitKind: 'quota',periodCount: 1,periodUnit: 'month',unit,- limitValue: UNREACHABLE_LIMIT_VALUE,+ limitValue,burstValue: null,isInstanceOverride: true,});@@ -177,6 +281,79 @@ describe('Usage quota consumption', () => {return usageLimit.id;};+ const saveCreditQuotaForNewSpender = async ({+ limitValue = CREDIT_LIMIT_MICRO,+ }: {+ limitValue?: number;+ } = {}): Promise<{ spenderId: string }> => {+ const spenderId = randomUUID();++ chargedSpenderIds.push(spenderId);++ await saveCodeExecutionQuota({+ spenderId,+ unit: UsageUnit.CREDIT,+ limitValue,+ });++ await refreshUsageLimitsCache();++ return { spenderId };+ };++ const buildWeekCreditQuotaInput = ({+ periodUnit = 'week',+ limitValue = UNREACHABLE_LIMIT_VALUE,+ }: {+ periodUnit?: PeriodUnit;+ limitValue?: number;+ } = {}): CreateUsageLimitInput => ({+ resourceType: UsageResourceType.LOGIC_FUNCTION,+ operationType: UsageOperationType.CODE_EXECUTION,+ spenderType: 'logicFunction',+ spenderId: logicFunctionId,+ limitKind: 'quota',+ periodCount: 1,+ periodUnit,+ unit: UsageUnit.CREDIT,+ limitValue,+ burstValue: null,+ });++ const createWeekCreditQuota = ({ isOperator = true } = {}) =>+ getUsageLimitService().create({+ workspaceId: SEED_APPLE_WORKSPACE_ID,+ input: buildWeekCreditQuotaInput(),+ isOperator,+ });++ const updateWeekCreditQuota = (+ usageLimitId: string,+ payload: Parameters<typeof buildWeekCreditQuotaInput>[0],+ ) =>+ getUsageLimitService().update({+ workspaceId: SEED_APPLE_WORKSPACE_ID,+ input: { id: usageLimitId, payload: buildWeekCreditQuotaInput(payload) },+ isOperator: true,+ });++ const deleteWeekCreditQuota = (usageLimitId: string) =>+ getUsageLimitService().delete({+ workspaceId: SEED_APPLE_WORKSPACE_ID,+ usageLimitId,+ isOperator: true,+ });++ const readWeekCreditCounter = async () => {+ const [counterKey] = await redis.keys(+ `*{${SEED_APPLE_WORKSPACE_ID}}:quota-consumed:*:logicFunction:${logicFunctionId}:${UsageUnit.CREDIT}:week:*`,+ );++ jestExpectToBeDefined(counterKey);++ return Number(await redis.get(counterKey));+ };+const findQuotaConsumedValue = async (unit: QuotaUnit) => {const response = await makeMetadataApiRequest({query: USAGE_QUOTAS_WITH_CONSUMPTION,@@ -250,10 +427,16 @@ describe('Usage quota consumption', () => {await waitForUsageEventRows(SEEDED_RUNS.length * 2);usageLimitIdByUnit = {- [UsageUnit.INVOCATION]: await saveCodeExecutionQuota(- UsageUnit.INVOCATION,- ),- [UsageUnit.CREDIT]: await saveCodeExecutionQuota(UsageUnit.CREDIT),+ [UsageUnit.INVOCATION]: await saveCodeExecutionQuota({+ spenderId: logicFunctionId,+ unit: UsageUnit.INVOCATION,+ limitValue: UNREACHABLE_LIMIT_VALUE,+ }),+ [UsageUnit.CREDIT]: await saveCodeExecutionQuota({+ spenderId: logicFunctionId,+ unit: UsageUnit.CREDIT,+ limitValue: UNREACHABLE_LIMIT_VALUE,+ }),};await refreshUsageLimitsCache();@@ -273,7 +456,11 @@ describe('Usage quota consumption', () => {}await clickHouseClient.command({- query: `ALTER TABLE usageEvent DELETE WHERE workspaceId = '${SEED_APPLE_WORKSPACE_ID}' AND logicFunctionId = '${logicFunctionId}'`,+ query: `ALTER TABLE usageEvent DELETE WHERE workspaceId = {workspaceId:String} AND logicFunctionId IN {logicFunctionIds:Array(String)}`,+ query_params: {+ workspaceId: SEED_APPLE_WORKSPACE_ID,+ logicFunctionIds: [logicFunctionId, ...chargedSpenderIds],+ },clickhouse_settings: { mutations_sync: '2' },});@@ -323,9 +510,6 @@ describe('Usage quota consumption', () => {const scopeConsumedValue = await findScopeConsumedValue(unit);expect(quotaConsumedValue).toBe(scopeConsumedValue);- expect(quotaConsumedValue).toBeGreaterThanOrEqual(- SEEDED_CONSUMPTION_BY_UNIT[unit],- );}expect(await findQuotaConsumedValue(UsageUnit.INVOCATION)).toBe(@@ -369,7 +553,407 @@ describe('Usage quota consumption', () => {});await waitForUsageEventRows(usageEventRowCount + 2);++ await usageLimitRepository.update(+ { id: usageLimitIdByUnit[UsageUnit.INVOCATION] },+ { limitValue: UNREACHABLE_LIMIT_VALUE },+ );+ await refreshUsageLimitsCache();},EXECUTION_TIMEOUT_MS,);++ describe('on a credit quota', () => {+ it('counts every concurrent charge on a cold counter once', async () => {+ const { spenderId } = await saveCreditQuotaForNewSpender();++ await Promise.all(+ Array.from({ length: CONCURRENT_CHARGE_COUNT }, () =>+ chargeRun({ spenderId, creditsUsedMicro: 100 }),+ ),+ );++ expect(await findRunRefusal({ spenderId, costMicro: 200 })).toBeNull();+ expect(await findRunRefusal({ spenderId, costMicro: 201 })).toMatchObject(+ { exhaustedKind: 'limit' },+ );++ const counterKeys = await findSpenderCounterKeys(spenderId);++ expect(counterKeys).toHaveLength(1);+ expect(+ Math.abs(+ (await redis.pTTL(counterKeys[0])) -+ (getCalendarMonthPeriod(new Date()).periodEnd.getTime() -+ Date.now()),+ ),+ ).toBeLessThan(60_000);+ });++ it(+ 'keeps in-flight usage when the cap is edited',+ async () => {+ const usageLimit = await createWeekCreditQuota();+ const usageEventRowCount = await countUsageEventRows();++ try {+ await chargeRun({+ spenderId: logicFunctionId,+ creditsUsedMicro: 100,+ });++ const editedLimitValue = (await readWeekCreditCounter()) + 700;++ await debitRunAheadOfRecord({+ spenderId: logicFunctionId,+ creditsUsedMicro: 600,+ });+ await updateWeekCreditQuota(usageLimit.id, {+ limitValue: editedLimitValue,+ });++ expect(+ await findRunRefusal({+ spenderId: logicFunctionId,+ costMicro: 100,+ }),+ ).toBeNull();+ expect(+ await findRunRefusal({+ spenderId: logicFunctionId,+ costMicro: 101,+ }),+ ).toMatchObject({+ exhaustedKind: 'limit',+ limitValue: editedLimitValue,+ });++ await waitForUsageEventRows(usageEventRowCount + 1);+ } finally {+ await deleteWeekCreditQuota(usageLimit.id);+ }+ },+ EXECUTION_TIMEOUT_MS,+ );++ it(+ 're-warms a limit from ClickHouse once it moves back to a period it left',+ async () => {+ const usageLimit = await createWeekCreditQuota();+ const usageEventRowCount = await countUsageEventRows();++ try {+ await chargeRun({+ spenderId: logicFunctionId,+ creditsUsedMicro: 100,+ });++ const enforcedLimitValue = (await readWeekCreditCounter()) + 300;++ await updateWeekCreditQuota(usageLimit.id, { periodUnit: 'day' });+ await chargeRun({+ spenderId: logicFunctionId,+ creditsUsedMicro: 600,+ });+ await waitForUsageEventRows(usageEventRowCount + 2);++ await updateWeekCreditQuota(usageLimit.id, {+ limitValue: enforcedLimitValue,+ });++ expect(+ await findRunRefusal({ spenderId: logicFunctionId, costMicro: 0 }),+ ).toMatchObject({+ exhaustedKind: 'limit',+ limitValue: enforcedLimitValue,+ });+ } finally {+ await deleteWeekCreditQuota(usageLimit.id);+ }+ },+ EXECUTION_TIMEOUT_MS,+ );++ it(+ 're-warms a limit from ClickHouse once an operator edit makes it enforced again',+ async () => {+ const entitlementSpy = jest+ .spyOn(+ getAppProviderByClassName<UsageLimitEntitlementService>(+ 'UsageLimitEntitlementService',+ ),+ 'isIntraWorkspaceLimitEntitled',+ )+ .mockResolvedValue(true);+ const usageLimit = await createWeekCreditQuota({ isOperator: false });+ const usageEventRowCount = await countUsageEventRows();++ try {+ await chargeRun({+ spenderId: logicFunctionId,+ creditsUsedMicro: 100,+ });++ const enforcedLimitValue = (await readWeekCreditCounter()) + 300;++ entitlementSpy.mockResolvedValue(false);+ await chargeRun({+ spenderId: logicFunctionId,+ creditsUsedMicro: 600,+ });++ expect(await readWeekCreditCounter()).toBe(enforcedLimitValue - 300);++ await waitForUsageEventRows(usageEventRowCount + 2);++ await updateWeekCreditQuota(usageLimit.id, {+ limitValue: enforcedLimitValue,+ });++ expect(+ await findRunRefusal({ spenderId: logicFunctionId, costMicro: 0 }),+ ).toMatchObject({+ exhaustedKind: 'limit',+ limitValue: enforcedLimitValue,+ });+ } finally {+ entitlementSpy.mockRestore();+ await deleteWeekCreditQuota(usageLimit.id);+ }+ },+ EXECUTION_TIMEOUT_MS,+ );+ });++ describe('on the daily email default', () => {+ const emailResourceId = randomUUID();+ let emailDailyLimit: number;+ let chargedEmailRowCount = 0;++ const buildEmailEvent = (quantity: number): RecordUsageInput => ({+ resourceType: UsageResourceType.EMAIL,+ operationType: UsageOperationType.EMAIL_SEND,+ unit: UsageUnit.INVOCATION,+ quantity,+ creditsUsedMicro: 0,+ resourceId: emailResourceId,+ });++ const findEmailRefusal = (invocationCost: number) =>+ getQuotaService().findExhaustedScope({+ workspaceId: SEED_APPLE_WORKSPACE_ID,+ resourceType: UsageResourceType.EMAIL,+ operationType: UsageOperationType.EMAIL_SEND,+ spenders: {},+ cost: {+ [UsageUnit.CREDIT]: 0,+ [UsageUnit.INVOCATION]: invocationCost,+ },+ });++ const debitEmailAheadOfRecord = (quantity: number) =>+ getQuotaService().debitAheadOfRecord({+ workspaceId: SEED_APPLE_WORKSPACE_ID,+ event: buildEmailEvent(quantity),+ });++ const chargeEmail = async (quantity: number) => {+ await getQuotaService().charge({+ workspaceId: SEED_APPLE_WORKSPACE_ID,+ events: [buildEmailEvent(quantity)],+ });+ chargedEmailRowCount++;+ };++ const saveEmailOverride = async () => {+ const usageLimit = await usageLimitRepository.save({+ workspaceId: SEED_APPLE_WORKSPACE_ID,+ resourceType: UsageResourceType.EMAIL,+ operationType: UsageOperationType.EMAIL_SEND,+ spenderType: 'workspace',+ spenderId: '',+ limitKind: 'quota',+ periodCount: 1,+ periodUnit: 'day',+ unit: UsageUnit.INVOCATION,+ limitValue: emailDailyLimit + 1_000,+ burstValue: null,+ isInstanceOverride: true,+ });++ savedUsageLimitIds.push(usageLimit.id);+ await refreshUsageLimitsCache();++ return usageLimit.id;+ };++ const countEmailRows = async (): Promise<number> => {+ const result = await clickHouseClient.query({+ query: `SELECT count() AS rowCount+ FROM usageEvent+ WHERE workspaceId = {workspaceId:String}+ AND resourceId = {resourceId:String}`,+ query_params: {+ workspaceId: SEED_APPLE_WORKSPACE_ID,+ resourceId: emailResourceId,+ },+ format: 'JSONEachRow',+ });++ const [row] = await result.json<{ rowCount: string }>();++ return Number(row?.rowCount ?? 0);+ };++ const deleteEmailRows = () =>+ clickHouseClient.command({+ query: `ALTER TABLE usageEvent DELETE WHERE workspaceId = {workspaceId:String} AND resourceType = {resourceType:String}`,+ query_params: {+ workspaceId: SEED_APPLE_WORKSPACE_ID,+ resourceType: UsageResourceType.EMAIL,+ },+ clickhouse_settings: { mutations_sync: '2' },+ });++ beforeAll(async () => {+ emailDailyLimit = Number(+ getAppProviderByClassName<TwentyConfigService>(+ 'TwentyConfigService',+ ).get('EMAIL_SEND_WORKSPACE_DAILY_LIMIT'),+ );++ await deleteEmailRows();+ await flushQuotaCounters();+ });++ const waitForChargedEmailRows = () =>+ expectEventually(+ async () => {+ expect(await countEmailRows()).toBe(chargedEmailRowCount);+ },+ { timeoutMs: CLICKHOUSE_FLUSH_TIMEOUT_MS, intervalMs: 250 },+ );++ afterAll(async () => {+ await waitForChargedEmailRows();+ await deleteEmailRows();+ await flushQuotaCounters();+ }, EXECUTION_TIMEOUT_MS);++ it('refuses at the default and keeps what a deleted override counted on it', async () => {+ await chargeEmail(emailDailyLimit - 1);++ expect(await findEmailRefusal(1)).toBeNull();+ expect(await findEmailRefusal(2)).toMatchObject({+ exhaustedKind: 'limit',+ isDefault: true,+ unit: UsageUnit.INVOCATION,+ limitValue: emailDailyLimit,+ });++ const overrideUsageLimitId = await saveEmailOverride();++ expect(await findEmailRefusal(2)).toBeNull();++ await debitEmailAheadOfRecord(1);++ await getUsageLimitService().delete({+ workspaceId: SEED_APPLE_WORKSPACE_ID,+ usageLimitId: overrideUsageLimitId,+ isOperator: true,+ });++ expect(await findEmailRefusal(1)).toMatchObject({+ exhaustedKind: 'limit',+ isDefault: true,+ });+ });+ });++ it(+ 'admits the assert and the charge when Redis fails, counts both and still records the charge',+ async () => {+ const { spenderId } = await saveCreditQuotaForNewSpender({+ limitValue: 0,+ });+ const cacheStorage: CacheStorageService =+ getQuotaService()['cacheStorage'];+ const incrementCounterBySpy = spyOnMetrics();+ const runScriptSpy = jest+ .spyOn(cacheStorage, 'runScript')+ .mockRejectedValue(new Error('Redis is unreachable'));++ try {+ expect(await findRunRefusal({ spenderId, costMicro: 1 })).toBeNull();+ expect(await chargeRun({ spenderId, creditsUsedMicro: 100 })).toEqual({+ exhaustedKind: null,+ });+ expect(findAdmittedOnFailureAttributes(incrementCounterBySpy)).toEqual([+ {+ operation: 'assert',+ resourceType: UsageResourceType.LOGIC_FUNCTION,+ },+ {+ operation: 'consume',+ resourceType: UsageResourceType.LOGIC_FUNCTION,+ },+ ]);+ } finally {+ runScriptSpy.mockRestore();+ incrementCounterBySpy.mockRestore();+ }++ await expectEventually(+ async () => {+ expect(await countUsageEventRows(spenderId)).toBe(1);+ },+ { timeoutMs: CLICKHOUSE_FLUSH_TIMEOUT_MS, intervalMs: 250 },+ );+ },+ EXECUTION_TIMEOUT_MS,+ );++ it('still debits the warm counter and leaves the cold one cold when its ClickHouse read fails', async () => {+ const { spenderId } = await saveCreditQuotaForNewSpender();++ await chargeRun({ spenderId, creditsUsedMicro: 600 });+ await saveCodeExecutionQuota({+ spenderId,+ unit: UsageUnit.INVOCATION,+ limitValue: UNREACHABLE_LIMIT_VALUE,+ });+ await refreshUsageLimitsCache();++ const incrementCounterBySpy = spyOnMetrics();+ const clickHouseReadSpy = jest+ .spyOn(+ getAppProviderByClassName<UsageAnalyticsService>(+ 'UsageAnalyticsService',+ ),+ 'getConsumptionRowsForAllScopes',+ )+ .mockRejectedValue(new Error('ClickHouse is unreachable'));++ try {+ expect(await chargeRun({ spenderId, creditsUsedMicro: 100 })).toEqual({+ exhaustedKind: null,+ });+ expect(await findSpenderCounterKeys(spenderId)).toHaveLength(1);+ expect(findAdmittedOnFailureAttributes(incrementCounterBySpy)).toEqual([+ {+ operation: 'consume',+ resourceType: UsageResourceType.LOGIC_FUNCTION,+ },+ ]);+ } finally {+ clickHouseReadSpy.mockRestore();+ incrementCounterBySpy.mockRestore();+ }++ expect(await findRunRefusal({ spenderId, costMicro: 300 })).toBeNull();+ expect(await findRunRefusal({ spenderId, costMicro: 301 })).toMatchObject({+ exhaustedKind: 'limit',+ unit: UsageUnit.CREDIT,+ });+ });});