The open alternative to Salesforce, designed for AI.
TypeScript crmcrm-systemcustomergood-first-issuegraphqlhacktoberfestjavascriptmarketingmonoreponestjsopen-sourcepostgresqlreactreactjssalestypescriptweb
Serialize a workspace's entitlement transition (#25660)
Félix Malfait6e4b3041e56b
diff --git a/packages/twenty-server/src/engine/core-modules/billing-webhook/billing-webhook.module.ts b/packages/twenty-server/src/engine/core-modules/billing-webhook/billing-webhook.module.tsindex 206e985ceec..5e5c75b8014 100644--- a/packages/twenty-server/src/engine/core-modules/billing-webhook/billing-webhook.module.ts+++ b/packages/twenty-server/src/engine/core-modules/billing-webhook/billing-webhook.module.ts@@ -13,6 +13,7 @@ import { BillingWebhookProductService } from 'src/engine/core-modules/billing-weimport { BillingWebhookSubscriptionScheduleService } from 'src/engine/core-modules/billing-webhook/services/billing-webhook-subscription-schedule.service';import { BillingWebhookSubscriptionService } from 'src/engine/core-modules/billing-webhook/services/billing-webhook-subscription.service';import { BillingModule } from 'src/engine/core-modules/billing/billing.module';+import { CacheLockModule } from 'src/engine/core-modules/cache-lock/cache-lock.module';import { UsageLimitModule } from 'src/engine/core-modules/usage-limit/usage-limit.module';import { WorkspaceIteratorModule } from 'src/database/commands/command-runners/workspace-iterator.module';import { BillingCustomerEntity } from 'src/engine/core-modules/billing/entities/billing-customer.entity';@@ -44,6 +45,7 @@ import { WorkspaceCacheModule } from 'src/engine/workspace-cache/workspace-cacheWorkspaceModule,BillingModule,UsageLimitModule,+ CacheLockModule,WorkspaceIteratorModule,WorkspaceCacheModule,TypeOrmModule.forFeature([diff --git a/packages/twenty-server/src/engine/core-modules/billing-webhook/services/__tests__/billing-entitlement-sync.service.spec.ts b/packages/twenty-server/src/engine/core-modules/billing-webhook/services/__tests__/billing-entitlement-sync.service.spec.tsnew file mode 100644index 00000000000..5917cb3db8f--- /dev/null+++ b/packages/twenty-server/src/engine/core-modules/billing-webhook/services/__tests__/billing-entitlement-sync.service.spec.ts@@ -0,0 +1,233 @@+/* @license Enterprise */++import { Test, type TestingModule } from '@nestjs/testing';++import { BillingEntitlementSyncService } from 'src/engine/core-modules/billing-webhook/services/billing-entitlement-sync.service';+import { BillingEntitlementEntity } from 'src/engine/core-modules/billing/entities/billing-entitlement.entity';+import { BillingEntitlementKey } from 'src/engine/core-modules/billing/enums/billing-entitlement-key.enum';+import { CacheLockService } from 'src/engine/core-modules/cache-lock/cache-lock.service';+import { UsageLimitQuotaService } from 'src/engine/core-modules/usage-limit/services/usage-limit-quota.service';+import { RowLevelPermissionPredicateGroupService } from 'src/engine/metadata-modules/row-level-permission-predicate/services/row-level-permission-predicate-group.service';+import { getWorkspaceScopedRepositoryToken } from 'src/engine/twenty-orm/workspace-scoped-repository/get-workspace-scoped-repository-token.util';++const WORKSPACE_ID = '20202020-1c25-4d02-bf25-6aeccf7ea419';+const STRIPE_CUSTOMER_ID = 'cus_test';+const RLS_LOOKUP_KEY = 'RLS';++describe('BillingEntitlementSyncService', () => {+ let service: BillingEntitlementSyncService;++ const billingEntitlementRepository = {+ find: jest.fn(),+ upsert: jest.fn(),+ };++ const rowLevelPermissionPredicateGroupService = {+ deleteAllRowLevelPermissionPredicateGroups: jest.fn(),+ };++ const usageLimitQuotaService = {+ dropIntraWorkspaceLimitCounters: jest.fn(),+ };++ // A real single-holder lock rather than a pass-through, so a test that runs+ // two syncs concurrently exercises the serialization instead of asserting+ // that a stub was called. Queued rather than spin-waiting so ordering is+ // deterministic.+ const heldLockKeys = new Set<string>();+ const lockQueueByKey = new Map<string, Promise<unknown>>();+ const cacheLockService = {+ withLock: jest.fn(<TResult>(fn: () => Promise<TResult>, key: string) => {+ const runWhenFree = (lockQueueByKey.get(key) ?? Promise.resolve()).then(+ async () => {+ heldLockKeys.add(key);+ try {+ return await fn();+ } finally {+ heldLockKeys.delete(key);+ }+ },+ );++ lockQueueByKey.set(+ key,+ runWhenFree.catch(() => undefined),+ );++ return runWhenFree;+ }),+ };++ const givenStoredEntitlements = (+ entitlements: { key: BillingEntitlementKey; value: boolean }[],+ ) => billingEntitlementRepository.find.mockResolvedValue(entitlements);++ beforeEach(async () => {+ jest.clearAllMocks();+ heldLockKeys.clear();+ lockQueueByKey.clear();+ billingEntitlementRepository.upsert.mockResolvedValue(undefined);+ usageLimitQuotaService.dropIntraWorkspaceLimitCounters.mockResolvedValue(+ undefined,+ );+ rowLevelPermissionPredicateGroupService.deleteAllRowLevelPermissionPredicateGroups.mockResolvedValue(+ undefined,+ );++ const module: TestingModule = await Test.createTestingModule({+ providers: [+ BillingEntitlementSyncService,+ {+ provide: getWorkspaceScopedRepositoryToken(BillingEntitlementEntity),+ useValue: billingEntitlementRepository,+ },+ {+ provide: RowLevelPermissionPredicateGroupService,+ useValue: rowLevelPermissionPredicateGroupService,+ },+ {+ provide: UsageLimitQuotaService,+ useValue: usageLimitQuotaService,+ },+ {+ provide: CacheLockService,+ useValue: cacheLockService,+ },+ ],+ }).compile();++ service = module.get<BillingEntitlementSyncService>(+ BillingEntitlementSyncService,+ );+ });++ const syncEntitlements = (activeLookupKeys: string[]) =>+ service.syncEntitlements({+ workspaceId: WORKSPACE_ID,+ stripeCustomerId: STRIPE_CUSTOMER_ID,+ activeLookupKeys,+ });++ it('drops usage-limit counters once when two syncs observe the same grant', async () => {+ givenStoredEntitlements([+ { key: BillingEntitlementKey.USAGE_LIMIT, value: false },+ ]);++ // Once the first sync commits, the stored rows read as granted, so the+ // second sync sees no transition. Without the lock both would read the+ // pre-commit state and both would drop, and the later drop would discard+ // usage already recorded under enforcement.+ billingEntitlementRepository.upsert.mockImplementation(async () => {+ givenStoredEntitlements([+ { key: BillingEntitlementKey.USAGE_LIMIT, value: true },+ ]);+ });++ await Promise.all([+ syncEntitlements([BillingEntitlementKey.USAGE_LIMIT]),+ syncEntitlements([BillingEntitlementKey.USAGE_LIMIT]),+ ]);++ expect(+ usageLimitQuotaService.dropIntraWorkspaceLimitCounters,+ ).toHaveBeenCalledTimes(1);+ });++ it('never deletes predicates after a concurrent grant has committed', async () => {+ let isRlsGrantedInStore = true;++ billingEntitlementRepository.find.mockImplementation(async () => [+ { key: BillingEntitlementKey.RLS, value: isRlsGrantedInStore },+ ]);++ billingEntitlementRepository.upsert.mockImplementation(+ async (+ _workspaceId: string,+ entitlements: { key: BillingEntitlementKey; value: boolean }[],+ ) => {+ isRlsGrantedInStore =+ entitlements.find(+ (entitlement) => entitlement.key === BillingEntitlementKey.RLS,+ )?.value === true;+ },+ );++ // Unserialized, both syncs read the same granted state, then each await+ // hands over: the revoke's delete lands after the grant has committed and+ // strips the predicates of a workspace whose feature is back on.+ await Promise.all([+ syncEntitlements([]),+ syncEntitlements([RLS_LOOKUP_KEY]),+ ]);++ const grantCommitOrder =+ billingEntitlementRepository.upsert.mock.calls.flatMap((call, index) =>+ call[1].some(+ (entitlement: { key: BillingEntitlementKey; value: boolean }) =>+ entitlement.key === BillingEntitlementKey.RLS && entitlement.value,+ )+ ? [+ billingEntitlementRepository.upsert.mock.invocationCallOrder[+ index+ ],+ ]+ : [],+ );++ const deleteOrders =+ rowLevelPermissionPredicateGroupService+ .deleteAllRowLevelPermissionPredicateGroups.mock.invocationCallOrder;++ expect(isRlsGrantedInStore).toBe(true);+ expect(grantCommitOrder).toHaveLength(1);+ expect(+ deleteOrders.filter((order) => order > grantCommitOrder[0]),+ ).toHaveLength(0);+ });++ it('locks on a key scoped to the workspace', async () => {+ givenStoredEntitlements([]);++ const otherWorkspaceId = '20202020-1c25-4d02-bf25-6aeccf7ea420';+ const heldKeysDuringOtherWorkspaceUpsert: string[] = [];++ billingEntitlementRepository.upsert.mockImplementation(async () => {+ heldKeysDuringOtherWorkspaceUpsert.push(...heldLockKeys);+ });++ await Promise.all([+ syncEntitlements([]),+ service.syncEntitlements({+ workspaceId: otherWorkspaceId,+ stripeCustomerId: STRIPE_CUSTOMER_ID,+ activeLookupKeys: [],+ }),+ ]);++ expect(cacheLockService.withLock.mock.calls.map((call) => call[1])).toEqual(+ [+ `billing-entitlement-state:${WORKSPACE_ID}`,+ `billing-entitlement-state:${otherWorkspaceId}`,+ ],+ );++ // Two workspaces hold their own keys at once. A constant key would+ // serialize them and never show both held together.+ expect(new Set(heldKeysDuringOtherWorkspaceUpsert).size).toBe(2);+ });++ it('holds the lock for the whole transition', async () => {+ givenStoredEntitlements([]);++ let lockHeldDuringUpsert = false;++ billingEntitlementRepository.upsert.mockImplementation(async () => {+ lockHeldDuringUpsert = heldLockKeys.size === 1;+ });++ await syncEntitlements([]);++ expect(lockHeldDuringUpsert).toBe(true);+ expect(heldLockKeys.size).toBe(0);+ });+});diff --git a/packages/twenty-server/src/engine/core-modules/billing-webhook/services/billing-entitlement-sync.service.ts b/packages/twenty-server/src/engine/core-modules/billing-webhook/services/billing-entitlement-sync.service.tsindex a37abc825df..92da414ee37 100644--- a/packages/twenty-server/src/engine/core-modules/billing-webhook/services/billing-entitlement-sync.service.ts+++ b/packages/twenty-server/src/engine/core-modules/billing-webhook/services/billing-entitlement-sync.service.ts@@ -2,14 +2,25 @@import { Injectable } from '@nestjs/common';+import { CacheLockService } from 'src/engine/core-modules/cache-lock/cache-lock.service';import { BillingEntitlementEntity } from 'src/engine/core-modules/billing/entities/billing-entitlement.entity';import { BillingEntitlementKey } from 'src/engine/core-modules/billing/enums/billing-entitlement-key.enum';+import { BILLING_ENTITLEMENT_STATE_LOCK_OPTIONS } from 'src/engine/core-modules/billing/constants/billing-entitlement-state-lock-options.constant';+import { buildBillingEntitlementStateLockKey } from 'src/engine/core-modules/billing/utils/build-billing-entitlement-state-lock-key.util';import { buildBillingEntitlementsFromLookupKeys } from 'src/engine/core-modules/billing/utils/build-billing-entitlements-from-lookup-keys.util';import { UsageLimitQuotaService } from 'src/engine/core-modules/usage-limit/services/usage-limit-quota.service';import { RowLevelPermissionPredicateGroupService } from 'src/engine/metadata-modules/row-level-permission-predicate/services/row-level-permission-predicate-group.service';import { InjectWorkspaceScopedRepository } from 'src/engine/twenty-orm/workspace-scoped-repository/inject-workspace-scoped-repository.decorator';import { WorkspaceScopedRepository } from 'src/engine/twenty-orm/workspace-scoped-repository/workspace-scoped-repository';+type EntitlementTransitionArgs = {+ workspaceId: string;+ stripeCustomerId: string;+ activeLookupKeys: string[];+};++type SyncedEntitlement = { key: BillingEntitlementKey; value: boolean };+// Shared by the Stripe webhook and the reconciliation command so a repaired// entitlement carries the same consequences as one that arrived on time.// Transitions are read from the stored rows rather than from Stripe's@@ -21,17 +32,35 @@ export class BillingEntitlementSyncService {private readonly billingEntitlementRepository: WorkspaceScopedRepository<BillingEntitlementEntity>,private readonly rowLevelPermissionPredicateGroupService: RowLevelPermissionPredicateGroupService,private readonly usageLimitQuotaService: UsageLimitQuotaService,+ private readonly cacheLockService: CacheLockService,) {}async syncEntitlements({workspaceId,stripeCustomerId,activeLookupKeys,- }: {- workspaceId: string;- stripeCustomerId: string;- activeLookupKeys: string[];- }): Promise<{ key: BillingEntitlementKey; value: boolean }[]> {+ }: EntitlementTransitionArgs): Promise<SyncedEntitlement[]> {+ // The whole transition is one unit. Reading the stored rows, acting on the+ // difference and committing it are three steps, and this service is the+ // only writer of those rows, so serializing here is what makes the+ // difference a transition rather than a guess about one.+ return await this.cacheLockService.withLock(+ () =>+ this.applyEntitlementTransition({+ workspaceId,+ stripeCustomerId,+ activeLookupKeys,+ }),+ buildBillingEntitlementStateLockKey(workspaceId),+ BILLING_ENTITLEMENT_STATE_LOCK_OPTIONS,+ );+ }++ private async applyEntitlementTransition({+ workspaceId,+ stripeCustomerId,+ activeLookupKeys,+ }: EntitlementTransitionArgs): Promise<SyncedEntitlement[]> {const storedEntitlements =await this.billingEntitlementRepository.find(workspaceId);@@ -78,27 +107,12 @@ export class BillingEntitlementSyncService {// the predicate cache and never the entitlement, so committing the revoke// first is the direction that fails closed: a failure here leaves rows// filtered by predicates that outlived the feature, not unfiltered.- // Asked on every pass rather than on the revoke transition, so a failure- // is retried once the stored row already reads as revoked. Re-read rather- // than trust the snapshot this sync computed: a concurrent grant that- // committed after our upsert would otherwise have its predicates deleted- // here, which is the one direction that leaves the feature on with nothing- // to filter by.+ // Asked on every pass rather than on the revoke transition, so a cleanup+ // that failed after the revoke committed is retried by the next sync.if (!isGranted(BillingEntitlementKey.RLS)) {- const storedEntitlementsAfterUpsert =- await this.billingEntitlementRepository.find(workspaceId);-- const isStillRevoked = !storedEntitlementsAfterUpsert.some(- (entitlement) =>- entitlement.key === BillingEntitlementKey.RLS &&- entitlement.value === true,+ await this.rowLevelPermissionPredicateGroupService.deleteAllRowLevelPermissionPredicateGroups(+ workspaceId,);-- if (isStillRevoked) {- await this.rowLevelPermissionPredicateGroupService.deleteAllRowLevelPermissionPredicateGroups(- workspaceId,- );- }}return billingEntitlements.map(({ key, value }) => ({ key, value }));diff --git a/packages/twenty-server/src/engine/core-modules/billing/constants/__tests__/billing-entitlement-state-lock-options.constant.spec.ts b/packages/twenty-server/src/engine/core-modules/billing/constants/__tests__/billing-entitlement-state-lock-options.constant.spec.tsnew file mode 100644index 00000000000..a834b1da2d5--- /dev/null+++ b/packages/twenty-server/src/engine/core-modules/billing/constants/__tests__/billing-entitlement-state-lock-options.constant.spec.ts@@ -0,0 +1,16 @@+/* @license Enterprise */++import { BILLING_ENTITLEMENT_STATE_LOCK_OPTIONS } from 'src/engine/core-modules/billing/constants/billing-entitlement-state-lock-options.constant';++describe('BILLING_ENTITLEMENT_STATE_LOCK_OPTIONS', () => {+ it('waits past the expiry so a waiter can take a key its holder abandoned', () => {+ const { ttl, ms, maxRetries } = BILLING_ENTITLEMENT_STATE_LOCK_OPTIONS;++ // withLock attempts at 0, ms, 2*ms ... (maxRetries - 1) * ms, so a budget+ // that only equals the expiry makes its last attempt just before the key+ // frees. Tuning either number without the other reintroduces that.+ const lastAttemptAt = (maxRetries - 1) * ms;++ expect(lastAttemptAt).toBeGreaterThan(ttl);+ });+});diff --git a/packages/twenty-server/src/engine/core-modules/billing/constants/billing-entitlement-state-lock-options.constant.ts b/packages/twenty-server/src/engine/core-modules/billing/constants/billing-entitlement-state-lock-options.constant.tsnew file mode 100644index 00000000000..b3a6822222d--- /dev/null+++ b/packages/twenty-server/src/engine/core-modules/billing/constants/billing-entitlement-state-lock-options.constant.ts@@ -0,0 +1,19 @@+/* @license Enterprise */++import { type CacheLockOptions } from 'src/engine/core-modules/cache-lock/cache-lock.service';++// The default 5.5s expiry is shorter than this transition can run: revoking+// RLS recomputes the workspace's role and predicate caches, which is unbounded+// in the workspace's size. A lock that expires mid-transition hands the key to+// a second sync and restores the races this lock exists to remove.+//+// maxRetries * ms must stay above ttl. withLock attempts at 0, ms, 2*ms and so+// on, so a waiter whose budget only equals the expiry makes its last attempt+// just before the key frees and throws instead of taking it, which is exactly+// the case that matters: a holder that died leaves the key held for the full+// expiry. The extra retries buy a waiter attempts on the far side of it.+export const BILLING_ENTITLEMENT_STATE_LOCK_OPTIONS = {+ ttl: 60_000,+ ms: 500,+ maxRetries: 130,+} satisfies CacheLockOptions;diff --git a/packages/twenty-server/src/engine/core-modules/billing/utils/build-billing-entitlement-state-lock-key.util.ts b/packages/twenty-server/src/engine/core-modules/billing/utils/build-billing-entitlement-state-lock-key.util.tsnew file mode 100644index 00000000000..d1fdb918cfe--- /dev/null+++ b/packages/twenty-server/src/engine/core-modules/billing/utils/build-billing-entitlement-state-lock-key.util.ts@@ -0,0 +1,5 @@+/* @license Enterprise */++export const buildBillingEntitlementStateLockKey = (+ workspaceId: string,+): string => `billing-entitlement-state:${workspaceId}`;