The open alternative to Salesforce, designed for AI.
TypeScript crmcrm-systemcustomergood-first-issuegraphqlhacktoberfestjavascriptmarketingmonoreponestjsopen-sourcepostgresqlreactreactjssalestypescriptweb
fix(server): ORM v2 rollout safety - rollback-proof transactions and a working flag rollback (#24299)
Thomas Trompettef773ffcef0fc
diff --git a/packages/twenty-server/src/engine/core-modules/feature-flag/services/feature-flag.service.ts b/packages/twenty-server/src/engine/core-modules/feature-flag/services/feature-flag.service.tsindex 4269ed46a94..c0a333f95ca 100644--- a/packages/twenty-server/src/engine/core-modules/feature-flag/services/feature-flag.service.ts+++ b/packages/twenty-server/src/engine/core-modules/feature-flag/services/feature-flag.service.ts@@ -10,6 +10,7 @@ import {FeatureFlagException,FeatureFlagExceptionCode,} from 'src/engine/core-modules/feature-flag/feature-flag.exception';+import { getFeatureFlagCacheKeysToInvalidate } from 'src/engine/core-modules/feature-flag/utils/get-feature-flag-cache-keys-to-invalidate.util';import { featureFlagValidator } from 'src/engine/core-modules/feature-flag/validates/feature-flag.validate';import { publicFeatureFlagValidator } from 'src/engine/core-modules/feature-flag/validates/is-public-feature-flag.validate';import { InjectWorkspaceScopedRepository } from 'src/engine/twenty-orm/workspace-scoped-repository/inject-workspace-scoped-repository.decorator';@@ -72,9 +73,10 @@ export class FeatureFlagService {},);- await this.workspaceCacheService.invalidateAndRecompute(workspaceId, [- 'featureFlagsMap',- ]);+ await this.workspaceCacheService.invalidateAndRecompute(+ workspaceId,+ getFeatureFlagCacheKeysToInvalidate(keys),+ );}}@@ -107,15 +109,25 @@ export class FeatureFlagService {);}+ const existingFeatureFlag = await this.featureFlagRepository.findOne(+ workspaceId,+ { where: { key: featureFlag } },+ );++ if (existingFeatureFlag?.value === value) {+ return existingFeatureFlag;+ }+const result = await this.featureFlagRepository.upsertAndReturnOne(workspaceId,{ key: featureFlag, value },['workspaceId', 'key'],);- await this.workspaceCacheService.invalidateAndRecompute(workspaceId, [- 'featureFlagsMap',- ]);+ await this.workspaceCacheService.invalidateAndRecompute(+ workspaceId,+ getFeatureFlagCacheKeysToInvalidate([featureFlag]),+ );return result;}diff --git a/packages/twenty-server/src/engine/core-modules/feature-flag/utils/__tests__/get-feature-flag-cache-keys-to-invalidate.util.spec.ts b/packages/twenty-server/src/engine/core-modules/feature-flag/utils/__tests__/get-feature-flag-cache-keys-to-invalidate.util.spec.tsnew file mode 100644index 00000000000..a2493484734--- /dev/null+++ b/packages/twenty-server/src/engine/core-modules/feature-flag/utils/__tests__/get-feature-flag-cache-keys-to-invalidate.util.spec.ts@@ -0,0 +1,36 @@+import { FeatureFlagKey } from 'twenty-shared/types';++import { getFeatureFlagCacheKeysToInvalidate } from 'src/engine/core-modules/feature-flag/utils/get-feature-flag-cache-keys-to-invalidate.util';++describe('getFeatureFlagCacheKeysToInvalidate', () => {+ it('should invalidate ORMEntityMetadatas when the ORM v2 flag changes', () => {+ expect(+ getFeatureFlagCacheKeysToInvalidate([+ FeatureFlagKey.IS_ORM_V2_READ_PATH_ENABLED,+ ]),+ ).toEqual(['featureFlagsMap', 'ORMEntityMetadatas']);+ });++ it('should invalidate ORMEntityMetadatas when the ORM v2 flag changes among others', () => {+ expect(+ getFeatureFlagCacheKeysToInvalidate([+ FeatureFlagKey.IS_JSON_FILTER_ENABLED,+ FeatureFlagKey.IS_ORM_V2_READ_PATH_ENABLED,+ ]),+ ).toEqual(['featureFlagsMap', 'ORMEntityMetadatas']);+ });++ it('should only invalidate featureFlagsMap for unrelated flags', () => {+ expect(+ getFeatureFlagCacheKeysToInvalidate([+ FeatureFlagKey.IS_JSON_FILTER_ENABLED,+ ]),+ ).toEqual(['featureFlagsMap']);+ });++ it('should only invalidate featureFlagsMap for an empty change set', () => {+ expect(getFeatureFlagCacheKeysToInvalidate([])).toEqual([+ 'featureFlagsMap',+ ]);+ });+});diff --git a/packages/twenty-server/src/engine/core-modules/feature-flag/utils/get-feature-flag-cache-keys-to-invalidate.util.ts b/packages/twenty-server/src/engine/core-modules/feature-flag/utils/get-feature-flag-cache-keys-to-invalidate.util.tsnew file mode 100644index 00000000000..e7ce9fab853--- /dev/null+++ b/packages/twenty-server/src/engine/core-modules/feature-flag/utils/get-feature-flag-cache-keys-to-invalidate.util.ts@@ -0,0 +1,15 @@+import { FeatureFlagKey } from 'twenty-shared/types';++import { type WorkspaceCacheKeyName } from 'src/engine/workspace-cache/types/workspace-cache-key.type';++export const getFeatureFlagCacheKeysToInvalidate = (+ changedFeatureFlagKeys: FeatureFlagKey[],+): WorkspaceCacheKeyName[] => {+ if (+ changedFeatureFlagKeys.includes(FeatureFlagKey.IS_ORM_V2_READ_PATH_ENABLED)+ ) {+ return ['featureFlagsMap', 'ORMEntityMetadatas'];+ }++ return ['featureFlagsMap'];+};diff --git a/packages/twenty-server/src/engine/twenty-orm-v2/datasource/utils/__tests__/run-in-rollback-safe-transaction.util.spec.ts b/packages/twenty-server/src/engine/twenty-orm-v2/datasource/utils/__tests__/run-in-rollback-safe-transaction.util.spec.tsnew file mode 100644index 00000000000..70cc1739e2d--- /dev/null+++ b/packages/twenty-server/src/engine/twenty-orm-v2/datasource/utils/__tests__/run-in-rollback-safe-transaction.util.spec.ts@@ -0,0 +1,128 @@+import { type Pool, type PoolClient } from 'pg';++import { runInRollbackSafeTransaction } from 'src/engine/twenty-orm-v2/datasource/utils/run-in-rollback-safe-transaction.util';++const buildClient = () => ({+ query: jest.fn().mockResolvedValue({ rows: [] }),+ release: jest.fn(),+});++const buildPool = (client: ReturnType<typeof buildClient>): Pool =>+ ({+ connect: jest.fn().mockResolvedValue(client as unknown as PoolClient),+ }) as unknown as Pool;++describe('runInRollbackSafeTransaction', () => {+ it('should wrap the work in BEGIN/COMMIT and release the client reusable', async () => {+ const client = buildClient();++ const result = await runInRollbackSafeTransaction({+ pool: buildPool(client),+ work: async () => 'done',+ });++ expect(result).toBe('done');+ expect(client.query).toHaveBeenNthCalledWith(1, 'BEGIN');+ expect(client.query).toHaveBeenNthCalledWith(2, 'COMMIT');+ expect(client.release).toHaveBeenCalledWith(false);+ });++ it('should roll back and rethrow the work error', async () => {+ const client = buildClient();+ const workError = new Error('Query read timeout');++ await expect(+ runInRollbackSafeTransaction({+ pool: buildPool(client),+ work: async () => {+ throw workError;+ },+ }),+ ).rejects.toBe(workError);++ expect(client.query).toHaveBeenLastCalledWith('ROLLBACK');+ expect(client.release).toHaveBeenCalledWith(false);+ });++ it('should rethrow the original error, not the rollback error, when ROLLBACK also fails', async () => {+ const client = buildClient();++ client.query.mockImplementation((statement: string) =>+ statement === 'ROLLBACK'+ ? Promise.reject(new Error('Query read timeout'))+ : Promise.resolve({ rows: [] }),+ );+ const workError = new Error('original failure');++ await expect(+ runInRollbackSafeTransaction({+ pool: buildPool(client),+ work: async () => {+ throw workError;+ },+ }),+ ).rejects.toBe(workError);+ });++ it('should destroy the connection instead of pooling it when ROLLBACK fails', async () => {+ const client = buildClient();++ client.query.mockImplementation((statement: string) =>+ statement === 'ROLLBACK'+ ? Promise.reject(new Error('Query read timeout'))+ : Promise.resolve({ rows: [] }),+ );++ await expect(+ runInRollbackSafeTransaction({+ pool: buildPool(client),+ work: async () => {+ throw new Error('original failure');+ },+ }),+ ).rejects.toThrow('original failure');++ expect(client.release).toHaveBeenCalledWith(true);+ });++ it('should rethrow a BEGIN failure and release the client reusable when ROLLBACK succeeds', async () => {+ const client = buildClient();+ const beginError = new Error('BEGIN failed');++ client.query.mockImplementation((statement: string) =>+ statement === 'BEGIN'+ ? Promise.reject(beginError)+ : Promise.resolve({ rows: [] }),+ );+ const work = jest.fn();++ await expect(+ runInRollbackSafeTransaction({ pool: buildPool(client), work }),+ ).rejects.toBe(beginError);++ expect(work).not.toHaveBeenCalled();+ expect(client.query).toHaveBeenLastCalledWith('ROLLBACK');+ expect(client.release).toHaveBeenCalledWith(false);+ });++ it('should roll back when COMMIT fails and release the client reusable', async () => {+ const client = buildClient();+ const commitError = new Error('COMMIT failed');++ client.query.mockImplementation((statement: string) =>+ statement === 'COMMIT'+ ? Promise.reject(commitError)+ : Promise.resolve({ rows: [] }),+ );++ await expect(+ runInRollbackSafeTransaction({+ pool: buildPool(client),+ work: async () => 'done',+ }),+ ).rejects.toBe(commitError);++ expect(client.query).toHaveBeenLastCalledWith('ROLLBACK');+ expect(client.release).toHaveBeenCalledWith(false);+ });+});diff --git a/packages/twenty-server/src/engine/twenty-orm-v2/datasource/utils/run-in-rollback-safe-transaction.util.ts b/packages/twenty-server/src/engine/twenty-orm-v2/datasource/utils/run-in-rollback-safe-transaction.util.tsnew file mode 100644index 00000000000..3025257ebc6--- /dev/null+++ b/packages/twenty-server/src/engine/twenty-orm-v2/datasource/utils/run-in-rollback-safe-transaction.util.ts@@ -0,0 +1,44 @@+import { Logger } from '@nestjs/common';++import { type Pool, type PoolClient } from 'pg';++const logger = new Logger('runInRollbackSafeTransaction');++export const runInRollbackSafeTransaction = async <T>({+ pool,+ work,+}: {+ pool: Pool;+ work: (client: PoolClient) => Promise<T>;+}): Promise<T> => {+ const client = await pool.connect();+ let shouldDestroyConnection = false;++ try {+ await client.query('BEGIN');++ const result = await work(client);++ await client.query('COMMIT');++ return result;+ } catch (error) {+ try {+ await client.query('ROLLBACK');+ } catch (rollbackError) {+ shouldDestroyConnection = true;++ logger.warn(+ `Destroying connection after failed transaction ROLLBACK: ${+ rollbackError instanceof Error+ ? rollbackError.message+ : String(rollbackError)+ }`,+ );+ }++ throw error;+ } finally {+ client.release(shouldDestroyConnection);+ }+};diff --git a/packages/twenty-server/src/engine/twenty-orm-v2/datasource/workspace-data-source-v2.ts b/packages/twenty-server/src/engine/twenty-orm-v2/datasource/workspace-data-source-v2.tsindex 5794c3b8630..9613ba2510f 100644--- a/packages/twenty-server/src/engine/twenty-orm-v2/datasource/workspace-data-source-v2.ts+++ b/packages/twenty-server/src/engine/twenty-orm-v2/datasource/workspace-data-source-v2.ts@@ -15,6 +15,7 @@ import {TwentyOrmV2Exception,TwentyOrmV2ExceptionCode,} from 'src/engine/twenty-orm-v2/exceptions/twenty-orm-v2.exception';+import { runInRollbackSafeTransaction } from 'src/engine/twenty-orm-v2/datasource/utils/run-in-rollback-safe-transaction.util';import { WorkspaceRepositoryV2 } from 'src/engine/twenty-orm-v2/repository/workspace-repository-v2';import { type WorkspaceTableShape } from 'src/engine/twenty-orm-v2/table-shape/types/workspace-table-shape.type';import { buildWorkspaceTableShape } from 'src/engine/twenty-orm-v2/table-shape/utils/build-workspace-table-shape.util';@@ -91,23 +92,10 @@ export class WorkspaceDataSourceV2 {private async runInClientTransaction<T>(work: (executor: QueryExecutorV2) => Promise<T>,): Promise<T> {- const client = await this.pool.connect();-- try {- await client.query('BEGIN');-- const result = await work(new ClientQueryExecutor({ client }));-- await client.query('COMMIT');-- return result;- } catch (error) {- await client.query('ROLLBACK');-- throw error;- } finally {- client.release();- }+ return runInRollbackSafeTransaction({+ pool: this.pool,+ work: (client) => work(new ClientQueryExecutor({ client })),+ });}private buildRepository({diff --git a/packages/twenty-server/src/engine/twenty-orm/global-workspace-datasource/global-workspace-datasource.module.ts b/packages/twenty-server/src/engine/twenty-orm/global-workspace-datasource/global-workspace-datasource.module.tsindex 687c11cd29c..77ce2658b79 100644--- a/packages/twenty-server/src/engine/twenty-orm/global-workspace-datasource/global-workspace-datasource.module.ts+++ b/packages/twenty-server/src/engine/twenty-orm/global-workspace-datasource/global-workspace-datasource.module.ts@@ -3,6 +3,7 @@ import { TypeOrmModule } from '@nestjs/typeorm';import { TypeORMModule } from 'src/database/typeorm/typeorm.module';import { ApplicationEntity } from 'src/engine/core-modules/application/application.entity';+import { FeatureFlagEntity } from 'src/engine/core-modules/feature-flag/feature-flag.entity';import { TwentyConfigModule } from 'src/engine/core-modules/twenty-config/twenty-config.module';import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';@@ -16,6 +17,7 @@ import { GlobalWorkspaceDataSourceService } from 'src/engine/twenty-orm/global-wimport { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';import { WorkspaceORMEntityMetadatasCacheService } from 'src/engine/twenty-orm/global-workspace-datasource/workspace-orm-entity-metadatas-cache.service';import { TwentyORMModule } from 'src/engine/twenty-orm/twenty-orm.module';+import { provideWorkspaceScopedRepository } from 'src/engine/twenty-orm/workspace-scoped-repository/provide-workspace-scoped-repository';import { TwentyORMV2Module } from 'src/engine/twenty-orm-v2/twenty-orm-v2.module';import { WorkspaceCacheStorageModule } from 'src/engine/workspace-cache-storage/workspace-cache-storage.module';import { WorkspaceCacheModule } from 'src/engine/workspace-cache/workspace-cache.module';@@ -30,6 +32,7 @@ import { WorkspaceEventEmitterModule } from 'src/engine/workspace-event-emitter/ObjectMetadataEntity,FieldMetadataEntity,ApplicationEntity,+ FeatureFlagEntity,]),WorkspaceCacheStorageModule,WorkspaceManyOrAllFlatEntityMapsCacheModule,@@ -47,6 +50,7 @@ import { WorkspaceEventEmitterModule } from 'src/engine/workspace-event-emitter/EntitySchemaColumnFactory,EntitySchemaRelationFactory,WorkspaceORMEntityMetadatasCacheService,+ provideWorkspaceScopedRepository(FeatureFlagEntity),],exports: [GlobalWorkspaceDataSourceService, GlobalWorkspaceOrmManager],})diff --git a/packages/twenty-server/src/engine/twenty-orm/global-workspace-datasource/workspace-orm-entity-metadatas-cache.service.ts b/packages/twenty-server/src/engine/twenty-orm/global-workspace-datasource/workspace-orm-entity-metadatas-cache.service.tsindex e2df10435db..ec0fdfa56ee 100644--- a/packages/twenty-server/src/engine/twenty-orm/global-workspace-datasource/workspace-orm-entity-metadatas-cache.service.ts+++ b/packages/twenty-server/src/engine/twenty-orm/global-workspace-datasource/workspace-orm-entity-metadatas-cache.service.ts@@ -11,13 +11,15 @@ import { EntityMetadataBuilder } from 'typeorm/metadata-builder/EntityMetadataBuimport { WorkspaceCacheProvider } from 'src/engine/workspace-cache/interfaces/workspace-cache-provider.service';import { ApplicationEntity } from 'src/engine/core-modules/application/application.entity';+import { FeatureFlagEntity } from 'src/engine/core-modules/feature-flag/feature-flag.entity';import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';import { EntitySchemaFactory } from 'src/engine/twenty-orm/factories/entity-schema.factory';import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';import { buildEntitySchemaMetadataMaps } from 'src/engine/twenty-orm/global-workspace-datasource/types/entity-schema-metadata.type';+import { InjectWorkspaceScopedRepository } from 'src/engine/twenty-orm/workspace-scoped-repository/inject-workspace-scoped-repository.decorator';+import { WorkspaceScopedRepository } from 'src/engine/twenty-orm/workspace-scoped-repository/workspace-scoped-repository';import { WorkspaceCache } from 'src/engine/workspace-cache/decorators/workspace-cache.decorator';-import { WorkspaceCacheService } from 'src/engine/workspace-cache/services/workspace-cache.service';@Injectable()@WorkspaceCache('ORMEntityMetadatas', {@@ -34,20 +36,21 @@ export class WorkspaceORMEntityMetadatasCacheService extends WorkspaceCacheProviprivate readonly fieldMetadataRepository: Repository<FieldMetadataEntity>,@InjectRepository(ApplicationEntity)private readonly applicationRepository: Repository<ApplicationEntity>,+ @InjectWorkspaceScopedRepository(FeatureFlagEntity)+ private readonly featureFlagRepository: WorkspaceScopedRepository<FeatureFlagEntity>,private readonly entitySchemaFactory: EntitySchemaFactory,private readonly globalWorkspaceOrmManager: GlobalWorkspaceOrmManager,- private readonly workspaceCacheService: WorkspaceCacheService,) {super();}async computeForCache(workspaceId: string): Promise<EntityMetadata[]> {- const { featureFlagsMap } = await this.workspaceCacheService.getOrRecompute(+ const ormV2FeatureFlag = await this.featureFlagRepository.findOne(workspaceId,- ['featureFlagsMap'],+ { where: { key: FeatureFlagKey.IS_ORM_V2_READ_PATH_ENABLED } },);- if (featureFlagsMap[FeatureFlagKey.IS_ORM_V2_READ_PATH_ENABLED]) {+ if (ormV2FeatureFlag?.value) {return [];}