Skip to content

The open alternative to Salesforce, designed for AI.

Serialize a workspace's entitlement transition (#25660)

Félix Malfait6e4b3041e56b
diff --git a/packages/twenty-server/src/engine/core-modules/billing-webhook/billing-webhook.module.ts b/packages/twenty-server/src/engine/core-modules/billing-webhook/billing-webhook.module.ts
index 206e985ceec..5e5c75b8014 100644
--- a/packages/twenty-server/src/engine/core-modules/billing-webhook/billing-webhook.module.ts
+++ b/packages/twenty-server/src/engine/core-modules/billing-webhook/billing-webhook.module.ts
@@ -13,6 +13,7 @@ import { BillingWebhookProductService } from 'src/engine/core-modules/billing-we
import { BillingWebhookSubscriptionScheduleService } from 'src/engine/core-modules/billing-webhook/services/billing-webhook-subscription-schedule.service';
import { BillingWebhookSubscriptionService } from 'src/engine/core-modules/billing-webhook/services/billing-webhook-subscription.service';
import { BillingModule } from 'src/engine/core-modules/billing/billing.module';
+import { CacheLockModule } from 'src/engine/core-modules/cache-lock/cache-lock.module';
import { UsageLimitModule } from 'src/engine/core-modules/usage-limit/usage-limit.module';
import { WorkspaceIteratorModule } from 'src/database/commands/command-runners/workspace-iterator.module';
import { BillingCustomerEntity } from 'src/engine/core-modules/billing/entities/billing-customer.entity';
@@ -44,6 +45,7 @@ import { WorkspaceCacheModule } from 'src/engine/workspace-cache/workspace-cache
WorkspaceModule,
BillingModule,
UsageLimitModule,
+ CacheLockModule,
WorkspaceIteratorModule,
WorkspaceCacheModule,
TypeOrmModule.forFeature([
diff --git a/packages/twenty-server/src/engine/core-modules/billing-webhook/services/__tests__/billing-entitlement-sync.service.spec.ts b/packages/twenty-server/src/engine/core-modules/billing-webhook/services/__tests__/billing-entitlement-sync.service.spec.ts
new file mode 100644
index 00000000000..5917cb3db8f
--- /dev/null
+++ b/packages/twenty-server/src/engine/core-modules/billing-webhook/services/__tests__/billing-entitlement-sync.service.spec.ts
@@ -0,0 +1,233 @@
+/* @license Enterprise */
+
+import { Test, type TestingModule } from '@nestjs/testing';
+
+import { BillingEntitlementSyncService } from 'src/engine/core-modules/billing-webhook/services/billing-entitlement-sync.service';
+import { BillingEntitlementEntity } from 'src/engine/core-modules/billing/entities/billing-entitlement.entity';
+import { BillingEntitlementKey } from 'src/engine/core-modules/billing/enums/billing-entitlement-key.enum';
+import { CacheLockService } from 'src/engine/core-modules/cache-lock/cache-lock.service';
+import { UsageLimitQuotaService } from 'src/engine/core-modules/usage-limit/services/usage-limit-quota.service';
+import { RowLevelPermissionPredicateGroupService } from 'src/engine/metadata-modules/row-level-permission-predicate/services/row-level-permission-predicate-group.service';
+import { getWorkspaceScopedRepositoryToken } from 'src/engine/twenty-orm/workspace-scoped-repository/get-workspace-scoped-repository-token.util';
+
+const WORKSPACE_ID = '20202020-1c25-4d02-bf25-6aeccf7ea419';
+const STRIPE_CUSTOMER_ID = 'cus_test';
+const RLS_LOOKUP_KEY = 'RLS';
+
+describe('BillingEntitlementSyncService', () => {
+ let service: BillingEntitlementSyncService;
+
+ const billingEntitlementRepository = {
+ find: jest.fn(),
+ upsert: jest.fn(),
+ };
+
+ const rowLevelPermissionPredicateGroupService = {
+ deleteAllRowLevelPermissionPredicateGroups: jest.fn(),
+ };
+
+ const usageLimitQuotaService = {
+ dropIntraWorkspaceLimitCounters: jest.fn(),
+ };
+
+ // A real single-holder lock rather than a pass-through, so a test that runs
+ // two syncs concurrently exercises the serialization instead of asserting
+ // that a stub was called. Queued rather than spin-waiting so ordering is
+ // deterministic.
+ const heldLockKeys = new Set<string>();
+ const lockQueueByKey = new Map<string, Promise<unknown>>();
+ const cacheLockService = {
+ withLock: jest.fn(<TResult>(fn: () => Promise<TResult>, key: string) => {
+ const runWhenFree = (lockQueueByKey.get(key) ?? Promise.resolve()).then(
+ async () => {
+ heldLockKeys.add(key);
+ try {
+ return await fn();
+ } finally {
+ heldLockKeys.delete(key);
+ }
+ },
+ );
+
+ lockQueueByKey.set(
+ key,
+ runWhenFree.catch(() => undefined),
+ );
+
+ return runWhenFree;
+ }),
+ };
+
+ const givenStoredEntitlements = (
+ entitlements: { key: BillingEntitlementKey; value: boolean }[],
+ ) => billingEntitlementRepository.find.mockResolvedValue(entitlements);
+
+ beforeEach(async () => {
+ jest.clearAllMocks();
+ heldLockKeys.clear();
+ lockQueueByKey.clear();
+ billingEntitlementRepository.upsert.mockResolvedValue(undefined);
+ usageLimitQuotaService.dropIntraWorkspaceLimitCounters.mockResolvedValue(
+ undefined,
+ );
+ rowLevelPermissionPredicateGroupService.deleteAllRowLevelPermissionPredicateGroups.mockResolvedValue(
+ undefined,
+ );
+
+ const module: TestingModule = await Test.createTestingModule({
+ providers: [
+ BillingEntitlementSyncService,
+ {
+ provide: getWorkspaceScopedRepositoryToken(BillingEntitlementEntity),
+ useValue: billingEntitlementRepository,
+ },
+ {
+ provide: RowLevelPermissionPredicateGroupService,
+ useValue: rowLevelPermissionPredicateGroupService,
+ },
+ {
+ provide: UsageLimitQuotaService,
+ useValue: usageLimitQuotaService,
+ },
+ {
+ provide: CacheLockService,
+ useValue: cacheLockService,
+ },
+ ],
+ }).compile();
+
+ service = module.get<BillingEntitlementSyncService>(
+ BillingEntitlementSyncService,
+ );
+ });
+
+ const syncEntitlements = (activeLookupKeys: string[]) =>
+ service.syncEntitlements({
+ workspaceId: WORKSPACE_ID,
+ stripeCustomerId: STRIPE_CUSTOMER_ID,
+ activeLookupKeys,
+ });
+
+ it('drops usage-limit counters once when two syncs observe the same grant', async () => {
+ givenStoredEntitlements([
+ { key: BillingEntitlementKey.USAGE_LIMIT, value: false },
+ ]);
+
+ // Once the first sync commits, the stored rows read as granted, so the
+ // second sync sees no transition. Without the lock both would read the
+ // pre-commit state and both would drop, and the later drop would discard
+ // usage already recorded under enforcement.
+ billingEntitlementRepository.upsert.mockImplementation(async () => {
+ givenStoredEntitlements([
+ { key: BillingEntitlementKey.USAGE_LIMIT, value: true },
+ ]);
+ });
+
+ await Promise.all([
+ syncEntitlements([BillingEntitlementKey.USAGE_LIMIT]),
+ syncEntitlements([BillingEntitlementKey.USAGE_LIMIT]),
+ ]);
+
+ expect(
+ usageLimitQuotaService.dropIntraWorkspaceLimitCounters,
+ ).toHaveBeenCalledTimes(1);
+ });
+
+ it('never deletes predicates after a concurrent grant has committed', async () => {
+ let isRlsGrantedInStore = true;
+
+ billingEntitlementRepository.find.mockImplementation(async () => [
+ { key: BillingEntitlementKey.RLS, value: isRlsGrantedInStore },
+ ]);
+
+ billingEntitlementRepository.upsert.mockImplementation(
+ async (
+ _workspaceId: string,
+ entitlements: { key: BillingEntitlementKey; value: boolean }[],
+ ) => {
+ isRlsGrantedInStore =
+ entitlements.find(
+ (entitlement) => entitlement.key === BillingEntitlementKey.RLS,
+ )?.value === true;
+ },
+ );
+
+ // Unserialized, both syncs read the same granted state, then each await
+ // hands over: the revoke's delete lands after the grant has committed and
+ // strips the predicates of a workspace whose feature is back on.
+ await Promise.all([
+ syncEntitlements([]),
+ syncEntitlements([RLS_LOOKUP_KEY]),
+ ]);
+
+ const grantCommitOrder =
+ billingEntitlementRepository.upsert.mock.calls.flatMap((call, index) =>
+ call[1].some(
+ (entitlement: { key: BillingEntitlementKey; value: boolean }) =>
+ entitlement.key === BillingEntitlementKey.RLS && entitlement.value,
+ )
+ ? [
+ billingEntitlementRepository.upsert.mock.invocationCallOrder[
+ index
+ ],
+ ]
+ : [],
+ );
+
+ const deleteOrders =
+ rowLevelPermissionPredicateGroupService
+ .deleteAllRowLevelPermissionPredicateGroups.mock.invocationCallOrder;
+
+ expect(isRlsGrantedInStore).toBe(true);
+ expect(grantCommitOrder).toHaveLength(1);
+ expect(
+ deleteOrders.filter((order) => order > grantCommitOrder[0]),
+ ).toHaveLength(0);
+ });
+
+ it('locks on a key scoped to the workspace', async () => {
+ givenStoredEntitlements([]);
+
+ const otherWorkspaceId = '20202020-1c25-4d02-bf25-6aeccf7ea420';
+ const heldKeysDuringOtherWorkspaceUpsert: string[] = [];
+
+ billingEntitlementRepository.upsert.mockImplementation(async () => {
+ heldKeysDuringOtherWorkspaceUpsert.push(...heldLockKeys);
+ });
+
+ await Promise.all([
+ syncEntitlements([]),
+ service.syncEntitlements({
+ workspaceId: otherWorkspaceId,
+ stripeCustomerId: STRIPE_CUSTOMER_ID,
+ activeLookupKeys: [],
+ }),
+ ]);
+
+ expect(cacheLockService.withLock.mock.calls.map((call) => call[1])).toEqual(
+ [
+ `billing-entitlement-state:${WORKSPACE_ID}`,
+ `billing-entitlement-state:${otherWorkspaceId}`,
+ ],
+ );
+
+ // Two workspaces hold their own keys at once. A constant key would
+ // serialize them and never show both held together.
+ expect(new Set(heldKeysDuringOtherWorkspaceUpsert).size).toBe(2);
+ });
+
+ it('holds the lock for the whole transition', async () => {
+ givenStoredEntitlements([]);
+
+ let lockHeldDuringUpsert = false;
+
+ billingEntitlementRepository.upsert.mockImplementation(async () => {
+ lockHeldDuringUpsert = heldLockKeys.size === 1;
+ });
+
+ await syncEntitlements([]);
+
+ expect(lockHeldDuringUpsert).toBe(true);
+ expect(heldLockKeys.size).toBe(0);
+ });
+});
diff --git a/packages/twenty-server/src/engine/core-modules/billing-webhook/services/billing-entitlement-sync.service.ts b/packages/twenty-server/src/engine/core-modules/billing-webhook/services/billing-entitlement-sync.service.ts
index a37abc825df..92da414ee37 100644
--- a/packages/twenty-server/src/engine/core-modules/billing-webhook/services/billing-entitlement-sync.service.ts
+++ b/packages/twenty-server/src/engine/core-modules/billing-webhook/services/billing-entitlement-sync.service.ts
@@ -2,14 +2,25 @@
import { Injectable } from '@nestjs/common';
+import { CacheLockService } from 'src/engine/core-modules/cache-lock/cache-lock.service';
import { BillingEntitlementEntity } from 'src/engine/core-modules/billing/entities/billing-entitlement.entity';
import { BillingEntitlementKey } from 'src/engine/core-modules/billing/enums/billing-entitlement-key.enum';
+import { BILLING_ENTITLEMENT_STATE_LOCK_OPTIONS } from 'src/engine/core-modules/billing/constants/billing-entitlement-state-lock-options.constant';
+import { buildBillingEntitlementStateLockKey } from 'src/engine/core-modules/billing/utils/build-billing-entitlement-state-lock-key.util';
import { buildBillingEntitlementsFromLookupKeys } from 'src/engine/core-modules/billing/utils/build-billing-entitlements-from-lookup-keys.util';
import { UsageLimitQuotaService } from 'src/engine/core-modules/usage-limit/services/usage-limit-quota.service';
import { RowLevelPermissionPredicateGroupService } from 'src/engine/metadata-modules/row-level-permission-predicate/services/row-level-permission-predicate-group.service';
import { InjectWorkspaceScopedRepository } from 'src/engine/twenty-orm/workspace-scoped-repository/inject-workspace-scoped-repository.decorator';
import { WorkspaceScopedRepository } from 'src/engine/twenty-orm/workspace-scoped-repository/workspace-scoped-repository';
+type EntitlementTransitionArgs = {
+ workspaceId: string;
+ stripeCustomerId: string;
+ activeLookupKeys: string[];
+};
+
+type SyncedEntitlement = { key: BillingEntitlementKey; value: boolean };
+
// Shared by the Stripe webhook and the reconciliation command so a repaired
// entitlement carries the same consequences as one that arrived on time.
// Transitions are read from the stored rows rather than from Stripe's
@@ -21,17 +32,35 @@ export class BillingEntitlementSyncService {
private readonly billingEntitlementRepository: WorkspaceScopedRepository<BillingEntitlementEntity>,
private readonly rowLevelPermissionPredicateGroupService: RowLevelPermissionPredicateGroupService,
private readonly usageLimitQuotaService: UsageLimitQuotaService,
+ private readonly cacheLockService: CacheLockService,
) {}
async syncEntitlements({
workspaceId,
stripeCustomerId,
activeLookupKeys,
- }: {
- workspaceId: string;
- stripeCustomerId: string;
- activeLookupKeys: string[];
- }): Promise<{ key: BillingEntitlementKey; value: boolean }[]> {
+ }: EntitlementTransitionArgs): Promise<SyncedEntitlement[]> {
+ // The whole transition is one unit. Reading the stored rows, acting on the
+ // difference and committing it are three steps, and this service is the
+ // only writer of those rows, so serializing here is what makes the
+ // difference a transition rather than a guess about one.
+ return await this.cacheLockService.withLock(
+ () =>
+ this.applyEntitlementTransition({
+ workspaceId,
+ stripeCustomerId,
+ activeLookupKeys,
+ }),
+ buildBillingEntitlementStateLockKey(workspaceId),
+ BILLING_ENTITLEMENT_STATE_LOCK_OPTIONS,
+ );
+ }
+
+ private async applyEntitlementTransition({
+ workspaceId,
+ stripeCustomerId,
+ activeLookupKeys,
+ }: EntitlementTransitionArgs): Promise<SyncedEntitlement[]> {
const storedEntitlements =
await this.billingEntitlementRepository.find(workspaceId);
@@ -78,27 +107,12 @@ export class BillingEntitlementSyncService {
// the predicate cache and never the entitlement, so committing the revoke
// first is the direction that fails closed: a failure here leaves rows
// filtered by predicates that outlived the feature, not unfiltered.
- // Asked on every pass rather than on the revoke transition, so a failure
- // is retried once the stored row already reads as revoked. Re-read rather
- // than trust the snapshot this sync computed: a concurrent grant that
- // committed after our upsert would otherwise have its predicates deleted
- // here, which is the one direction that leaves the feature on with nothing
- // to filter by.
+ // Asked on every pass rather than on the revoke transition, so a cleanup
+ // that failed after the revoke committed is retried by the next sync.
if (!isGranted(BillingEntitlementKey.RLS)) {
- const storedEntitlementsAfterUpsert =
- await this.billingEntitlementRepository.find(workspaceId);
-
- const isStillRevoked = !storedEntitlementsAfterUpsert.some(
- (entitlement) =>
- entitlement.key === BillingEntitlementKey.RLS &&
- entitlement.value === true,
+ await this.rowLevelPermissionPredicateGroupService.deleteAllRowLevelPermissionPredicateGroups(
+ workspaceId,
);
-
- if (isStillRevoked) {
- await this.rowLevelPermissionPredicateGroupService.deleteAllRowLevelPermissionPredicateGroups(
- workspaceId,
- );
- }
}
return billingEntitlements.map(({ key, value }) => ({ key, value }));
diff --git a/packages/twenty-server/src/engine/core-modules/billing/constants/__tests__/billing-entitlement-state-lock-options.constant.spec.ts b/packages/twenty-server/src/engine/core-modules/billing/constants/__tests__/billing-entitlement-state-lock-options.constant.spec.ts
new file mode 100644
index 00000000000..a834b1da2d5
--- /dev/null
+++ b/packages/twenty-server/src/engine/core-modules/billing/constants/__tests__/billing-entitlement-state-lock-options.constant.spec.ts
@@ -0,0 +1,16 @@
+/* @license Enterprise */
+
+import { BILLING_ENTITLEMENT_STATE_LOCK_OPTIONS } from 'src/engine/core-modules/billing/constants/billing-entitlement-state-lock-options.constant';
+
+describe('BILLING_ENTITLEMENT_STATE_LOCK_OPTIONS', () => {
+ it('waits past the expiry so a waiter can take a key its holder abandoned', () => {
+ const { ttl, ms, maxRetries } = BILLING_ENTITLEMENT_STATE_LOCK_OPTIONS;
+
+ // withLock attempts at 0, ms, 2*ms ... (maxRetries - 1) * ms, so a budget
+ // that only equals the expiry makes its last attempt just before the key
+ // frees. Tuning either number without the other reintroduces that.
+ const lastAttemptAt = (maxRetries - 1) * ms;
+
+ expect(lastAttemptAt).toBeGreaterThan(ttl);
+ });
+});
diff --git a/packages/twenty-server/src/engine/core-modules/billing/constants/billing-entitlement-state-lock-options.constant.ts b/packages/twenty-server/src/engine/core-modules/billing/constants/billing-entitlement-state-lock-options.constant.ts
new file mode 100644
index 00000000000..b3a6822222d
--- /dev/null
+++ b/packages/twenty-server/src/engine/core-modules/billing/constants/billing-entitlement-state-lock-options.constant.ts
@@ -0,0 +1,19 @@
+/* @license Enterprise */
+
+import { type CacheLockOptions } from 'src/engine/core-modules/cache-lock/cache-lock.service';
+
+// The default 5.5s expiry is shorter than this transition can run: revoking
+// RLS recomputes the workspace's role and predicate caches, which is unbounded
+// in the workspace's size. A lock that expires mid-transition hands the key to
+// a second sync and restores the races this lock exists to remove.
+//
+// maxRetries * ms must stay above ttl. withLock attempts at 0, ms, 2*ms and so
+// on, so a waiter whose budget only equals the expiry makes its last attempt
+// just before the key frees and throws instead of taking it, which is exactly
+// the case that matters: a holder that died leaves the key held for the full
+// expiry. The extra retries buy a waiter attempts on the far side of it.
+export const BILLING_ENTITLEMENT_STATE_LOCK_OPTIONS = {
+ ttl: 60_000,
+ ms: 500,
+ maxRetries: 130,
+} satisfies CacheLockOptions;
diff --git a/packages/twenty-server/src/engine/core-modules/billing/utils/build-billing-entitlement-state-lock-key.util.ts b/packages/twenty-server/src/engine/core-modules/billing/utils/build-billing-entitlement-state-lock-key.util.ts
new file mode 100644
index 00000000000..d1fdb918cfe
--- /dev/null
+++ b/packages/twenty-server/src/engine/core-modules/billing/utils/build-billing-entitlement-state-lock-key.util.ts
@@ -0,0 +1,5 @@
+/* @license Enterprise */
+
+export const buildBillingEntitlementStateLockKey = (
+ workspaceId: string,
+): string => `billing-entitlement-state:${workspaceId}`;