The open alternative to Salesforce, designed for AI.
TypeScript crmcrm-systemcustomergood-first-issuegraphqlhacktoberfestjavascriptmarketingmonoreponestjsopen-sourcepostgresqlreactreactjssalestypescriptweb
perf(server): scope workspace migration cache invalidation to mutated maps (#29167)
Etienne34eb0885d774
diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-migration/workspace-migration-runner/services/workspace-migration-runner.service.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-migration/workspace-migration-runner/services/workspace-migration-runner.service.tsindex 2988da739f3..ce9dee9f223 100644--- a/packages/twenty-server/src/engine/workspace-manager/workspace-migration/workspace-migration-runner/services/workspace-migration-runner.service.ts+++ b/packages/twenty-server/src/engine/workspace-manager/workspace-migration/workspace-migration-runner/services/workspace-migration-runner.service.ts@@ -32,6 +32,8 @@ import { WorkspaceMigrationRunnerActionHandlerRegistryService } from 'src/engineimport { DeferredWorkspaceMigrationActionRunnerService } from 'src/engine/workspace-manager/workspace-migration/workspace-migration-runner/services/deferred-workspace-migration-action-runner.service';import { type DeferredWorkspaceMigrationAction } from 'src/engine/workspace-manager/workspace-migration/workspace-migration-runner/types/deferred-workspace-migration-action.type';import { type MetadataEvent } from 'src/engine/workspace-manager/workspace-migration/workspace-migration-runner/types/metadata-event.type';+import { getDerivedCacheTriggers } from 'src/engine/workspace-manager/workspace-migration/workspace-migration-runner/utils/get-derived-cache-triggers.util';+import { getFlatEntityMapsKeysToInvalidateForWorkspaceMigration } from 'src/engine/workspace-manager/workspace-migration/workspace-migration-runner/utils/get-flat-entity-maps-keys-to-invalidate-for-workspace-migration.util';import { getMetadataNamesToLoadForWorkspaceMigration } from 'src/engine/workspace-manager/workspace-migration/workspace-migration-runner/utils/get-metadata-names-to-load-for-workspace-migration.util';import { buildPreallocatedIdByUniversalIdentifierFromActions } from 'src/engine/workspace-manager/workspace-migration/workspace-migration-runner/utils/build-preallocated-id-by-universal-identifier-from-actions.util';@@ -61,34 +63,27 @@ export class WorkspaceMigrationRunnerService {),),hasMorphRelationFieldChanged = false,+ hasRolesPermissionsChanged = getDerivedCacheTriggers(allFlatEntityMapsKeys)+ .hasRolesPermissionsChanged,+ hasWorkflowVersionsChanged = getDerivedCacheTriggers(allFlatEntityMapsKeys)+ .hasWorkflowVersionsChanged,+ hasApplicationVariablesChanged = getDerivedCacheTriggers(+ allFlatEntityMapsKeys,+ ).hasApplicationVariablesChanged,}: {allFlatEntityMapsKeys: (keyof AllFlatEntityMaps)[];workspaceId: string;hasSchemaMetadataChanged?: boolean;hasMorphRelationFieldChanged?: boolean;+ hasRolesPermissionsChanged?: boolean;+ hasWorkflowVersionsChanged?: boolean;+ hasApplicationVariablesChanged?: boolean;}): Promise<void> {this.logger.perfTime('Runner',`Cache invalidation ${allFlatEntityMapsKeys.join()}`,);- const hasRolesPermissionsChanged = allFlatEntityMapsKeys.some(- (flatEntityMapsKey) =>- flatEntityMapsKey === 'flatRoleMaps' ||- flatEntityMapsKey === 'flatRoleTargetMaps' ||- flatEntityMapsKey === 'flatObjectPermissionMaps' ||- flatEntityMapsKey === 'flatFieldPermissionMaps' ||- flatEntityMapsKey === 'flatRolePermissionFlagMaps',- );-- const hasWorkflowVersionsChanged = allFlatEntityMapsKeys.includes(- 'flatWorkflowVersionMaps',- );-- const hasApplicationVariablesChanged = allFlatEntityMapsKeys.includes(- 'flatApplicationVariableMaps',- );-const cacheKeyNamesToInvalidate: WorkspaceCacheKeyName[] = [...new Set([...withDerivedFieldMetadataMaps(allFlatEntityMapsKeys),@@ -286,6 +281,15 @@ export class WorkspaceMigrationRunnerService {getMetadataFlatEntityMapsKey,);+ const flatEntityMapsKeysToInvalidate =+ getFlatEntityMapsKeysToInvalidateForWorkspaceMigration(actions);++ const {+ hasRolesPermissionsChanged,+ hasWorkflowVersionsChanged,+ hasApplicationVariablesChanged,+ } = getDerivedCacheTriggers(allFlatEntityMapsKeys);+const cachedAllFlatEntityMaps =await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps<typeof allFlatEntityMapsKeys@@ -516,10 +520,13 @@ export class WorkspaceMigrationRunnerService {try {await this.invalidateCache({- allFlatEntityMapsKeys,+ allFlatEntityMapsKeys: flatEntityMapsKeysToInvalidate,workspaceId,hasSchemaMetadataChanged,hasMorphRelationFieldChanged,+ hasRolesPermissionsChanged,+ hasWorkflowVersionsChanged,+ hasApplicationVariablesChanged,});} catch (cacheError) {this.logger.error(@@ -547,10 +554,13 @@ export class WorkspaceMigrationRunnerService {try {await this.invalidateCache({- allFlatEntityMapsKeys,+ allFlatEntityMapsKeys: flatEntityMapsKeysToInvalidate,workspaceId,hasSchemaMetadataChanged,hasMorphRelationFieldChanged,+ hasRolesPermissionsChanged,+ hasWorkflowVersionsChanged,+ hasApplicationVariablesChanged,});this.recordRunPhaseMetric({@@ -575,7 +585,7 @@ export class WorkspaceMigrationRunnerService {performance.now() - postCommitInvalidateStart;this.logger.perf(- `[install-perf] Runner post-commit invalidateCache took ${postCommitInvalidateMs.toFixed(1)}ms for ${allFlatEntityMapsKeys.length} flat-maps keys`,+ `[install-perf] Runner post-commit invalidateCache took ${postCommitInvalidateMs.toFixed(1)}ms for ${flatEntityMapsKeysToInvalidate.length} flat-maps keys`,'Runner',);diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-migration/workspace-migration-runner/utils/__tests__/get-flat-entity-maps-keys-to-invalidate-for-workspace-migration.util.spec.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-migration/workspace-migration-runner/utils/__tests__/get-flat-entity-maps-keys-to-invalidate-for-workspace-migration.util.spec.tsnew file mode 100644index 00000000000..61684cc249a--- /dev/null+++ b/packages/twenty-server/src/engine/workspace-manager/workspace-migration/workspace-migration-runner/utils/__tests__/get-flat-entity-maps-keys-to-invalidate-for-workspace-migration.util.spec.ts@@ -0,0 +1,176 @@+import { type AllUniversalWorkspaceMigrationAction } from 'src/engine/workspace-manager/workspace-migration/workspace-migration-builder/types/workspace-migration-action-common.type';+import { getFlatEntityMapsKeysToInvalidateForWorkspaceMigration } from 'src/engine/workspace-manager/workspace-migration/workspace-migration-runner/utils/get-flat-entity-maps-keys-to-invalidate-for-workspace-migration.util';++const buildAction = (+ action: Partial<AllUniversalWorkspaceMigrationAction> &+ Pick<AllUniversalWorkspaceMigrationAction, 'type' | 'metadataName'>,+): AllUniversalWorkspaceMigrationAction =>+ action as AllUniversalWorkspaceMigrationAction;++describe('getFlatEntityMapsKeysToInvalidateForWorkspaceMigration', () => {+ it('invalidates only the view field map for a viewField reorder', () => {+ const result = getFlatEntityMapsKeysToInvalidateForWorkspaceMigration([+ buildAction({+ type: 'update',+ metadataName: 'viewField',+ universalIdentifier: 'view-field-1',+ update: { position: 2 },+ }),+ ]);++ expect(result).toEqual(['flatViewFieldMaps']);+ });++ it('invalidates the view field group map when a viewField update moves it between groups', () => {+ const result = getFlatEntityMapsKeysToInvalidateForWorkspaceMigration([+ buildAction({+ type: 'update',+ metadataName: 'viewField',+ universalIdentifier: 'view-field-1',+ update: {+ position: 2,+ viewFieldGroupUniversalIdentifier: 'view-field-group-1',+ },+ }),+ ]);++ expect(result.sort()).toEqual(+ ['flatViewFieldGroupMaps', 'flatViewFieldMaps'].sort(),+ );+ expect(result).not.toContain('flatObjectMetadataMaps');+ expect(result).not.toContain('flatFieldMetadataMaps');+ });++ it('invalidates the related view-side maps but not object metadata when a viewField is created', () => {+ const result = getFlatEntityMapsKeysToInvalidateForWorkspaceMigration([+ buildAction({+ type: 'create',+ metadataName: 'viewField',+ }),+ ]);++ expect(result.sort()).toEqual(+ [+ 'flatFieldMetadataMaps',+ 'flatViewFieldGroupMaps',+ 'flatViewFieldMaps',+ 'flatViewMaps',+ ].sort(),+ );+ expect(result).not.toContain('flatObjectMetadataMaps');+ });++ it('invalidates the related view-side maps but not object metadata when a viewField is deleted', () => {+ const result = getFlatEntityMapsKeysToInvalidateForWorkspaceMigration([+ buildAction({+ type: 'delete',+ metadataName: 'viewField',+ universalIdentifier: 'view-field-1',+ }),+ ]);++ expect(result.sort()).toEqual(+ [+ 'flatFieldMetadataMaps',+ 'flatViewFieldGroupMaps',+ 'flatViewFieldMaps',+ 'flatViewMaps',+ ].sort(),+ );+ expect(result).not.toContain('flatObjectMetadataMaps');+ });++ it('invalidates only the field metadata map for a non-relation field update', () => {+ const result = getFlatEntityMapsKeysToInvalidateForWorkspaceMigration([+ buildAction({+ type: 'update',+ metadataName: 'fieldMetadata',+ universalIdentifier: 'field-1',+ update: { label: 'Renamed' },+ }),+ ]);++ expect(result).toEqual(['flatFieldMetadataMaps']);+ });++ it('invalidates field, object and permission maps when an object is created (bundled fields and default permissions)', () => {+ const result = getFlatEntityMapsKeysToInvalidateForWorkspaceMigration([+ buildAction({+ type: 'create',+ metadataName: 'objectMetadata',+ }),+ ]);++ expect(result).toContain('flatObjectMetadataMaps');+ expect(result).toContain('flatFieldMetadataMaps');+ expect(result).toContain('flatObjectPermissionMaps');+ expect(result).toContain('flatFieldPermissionMaps');+ });++ it('invalidates object and field metadata maps on an index update (id re-key and isUnique derivation)', () => {+ const result = getFlatEntityMapsKeysToInvalidateForWorkspaceMigration([+ buildAction({+ type: 'update',+ metadataName: 'index',+ universalIdentifier: 'index-1',+ update: {},+ }),+ ]);++ expect(result.sort()).toEqual(+ [+ 'flatFieldMetadataMaps',+ 'flatIndexMaps',+ 'flatObjectMetadataMaps',+ ].sort(),+ );+ });++ it('invalidates the field metadata map when a unique index is created (isUnique derives from index rows)', () => {+ const result = getFlatEntityMapsKeysToInvalidateForWorkspaceMigration([+ buildAction({+ type: 'create',+ metadataName: 'index',+ }),+ ]);++ expect(result).toContain('flatFieldMetadataMaps');+ expect(result).toContain('flatObjectMetadataMaps');+ expect(result).toContain('flatIndexMaps');+ });++ it('invalidates the cascade-exposed child maps when a view is deleted', () => {+ const result = getFlatEntityMapsKeysToInvalidateForWorkspaceMigration([+ buildAction({+ type: 'delete',+ metadataName: 'view',+ universalIdentifier: 'view-1',+ }),+ ]);++ expect(result).toContain('flatViewMaps');+ expect(result).toContain('flatViewFieldMaps');+ expect(result).toContain('flatViewFieldGroupMaps');+ expect(result).toContain('flatViewSortMaps');+ expect(result).toContain('flatViewFilterMaps');+ });++ it('deduplicates keys across several actions', () => {+ const result = getFlatEntityMapsKeysToInvalidateForWorkspaceMigration([+ buildAction({+ type: 'update',+ metadataName: 'viewField',+ universalIdentifier: 'view-field-1',+ update: { position: 1 },+ }),+ buildAction({+ type: 'update',+ metadataName: 'viewField',+ universalIdentifier: 'view-field-2',+ update: { position: 2 },+ }),+ ]);++ expect(result).toEqual(['flatViewFieldMaps']);+ });+});diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-migration/workspace-migration-runner/utils/get-derived-cache-triggers.util.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-migration/workspace-migration-runner/utils/get-derived-cache-triggers.util.tsnew file mode 100644index 00000000000..be986a8a3a7--- /dev/null+++ b/packages/twenty-server/src/engine/workspace-manager/workspace-migration/workspace-migration-runner/utils/get-derived-cache-triggers.util.ts@@ -0,0 +1,26 @@+import { type AllFlatEntityMaps } from 'src/engine/metadata-modules/flat-entity/types/all-flat-entity-maps.type';++export type DerivedCacheTriggers = {+ hasRolesPermissionsChanged: boolean;+ hasWorkflowVersionsChanged: boolean;+ hasApplicationVariablesChanged: boolean;+};++export const getDerivedCacheTriggers = (+ flatEntityMapsKeys: (keyof AllFlatEntityMaps)[],+): DerivedCacheTriggers => ({+ hasRolesPermissionsChanged: flatEntityMapsKeys.some(+ (flatEntityMapsKey) =>+ flatEntityMapsKey === 'flatRoleMaps' ||+ flatEntityMapsKey === 'flatRoleTargetMaps' ||+ flatEntityMapsKey === 'flatObjectPermissionMaps' ||+ flatEntityMapsKey === 'flatFieldPermissionMaps' ||+ flatEntityMapsKey === 'flatRolePermissionFlagMaps',+ ),+ hasWorkflowVersionsChanged: flatEntityMapsKeys.includes(+ 'flatWorkflowVersionMaps',+ ),+ hasApplicationVariablesChanged: flatEntityMapsKeys.includes(+ 'flatApplicationVariableMaps',+ ),+});diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-migration/workspace-migration-runner/utils/get-flat-entity-maps-keys-to-invalidate-for-workspace-migration.util.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-migration/workspace-migration-runner/utils/get-flat-entity-maps-keys-to-invalidate-for-workspace-migration.util.tsnew file mode 100644index 00000000000..db7e84a5ce9--- /dev/null+++ b/packages/twenty-server/src/engine/workspace-manager/workspace-migration/workspace-migration-runner/utils/get-flat-entity-maps-keys-to-invalidate-for-workspace-migration.util.ts@@ -0,0 +1,68 @@+import { type AllMetadataName } from 'twenty-shared/metadata';+import { isDefined } from 'twenty-shared/utils';++import { ALL_MANY_TO_ONE_METADATA_RELATIONS } from 'src/engine/metadata-modules/flat-entity/constant/all-many-to-one-metadata-relations.constant';+import { type AllFlatEntityMaps } from 'src/engine/metadata-modules/flat-entity/types/all-flat-entity-maps.type';+import { getMetadataFlatEntityMapsKey } from 'src/engine/metadata-modules/flat-entity/utils/get-metadata-flat-entity-maps-key.util';+import { getMetadataRelatedMetadataNames } from 'src/engine/metadata-modules/flat-entity/utils/get-metadata-related-metadata-names.util';+import { type AllUniversalWorkspaceMigrationAction } from 'src/engine/workspace-manager/workspace-migration/workspace-migration-builder/types/workspace-migration-action-common.type';++type ManyToOneRelation = {+ metadataName: AllMetadataName;+ foreignKey: string;+ universalForeignKey: string | null;+};++const EXTRA_AFFECTED_METADATA_NAMES: Partial<+ Record<AllMetadataName, AllMetadataName[]>+> = {+ index: ['objectMetadata', 'fieldMetadata'],+};++const getUpdateAffectedMetadataNames = ({+ metadataName,+ changedProperties,+}: {+ metadataName: AllMetadataName;+ changedProperties: Set<string>;+}): AllMetadataName[] => {+ const manyToOneRelations = ALL_MANY_TO_ONE_METADATA_RELATIONS[+ metadataName+ ] as Record<string, ManyToOneRelation | null>;++ return Object.values(manyToOneRelations)+ .filter(+ (relation): relation is ManyToOneRelation =>+ isDefined(relation) &&+ (changedProperties.has(relation.foreignKey) ||+ (isDefined(relation.universalForeignKey) &&+ changedProperties.has(relation.universalForeignKey))),+ )+ .map((relation) => relation.metadataName);+};++export const getFlatEntityMapsKeysToInvalidateForWorkspaceMigration = (+ actions: AllUniversalWorkspaceMigrationAction[],+): (keyof AllFlatEntityMaps)[] => {+ const metadataNamesToInvalidate = new Set<AllMetadataName>();++ for (const action of actions) {+ metadataNamesToInvalidate.add(action.metadataName);++ const affectedMetadataNames = [+ ...(action.type === 'update'+ ? getUpdateAffectedMetadataNames({+ metadataName: action.metadataName,+ changedProperties: new Set(Object.keys(action.update)),+ })+ : getMetadataRelatedMetadataNames(action.metadataName)),+ ...(EXTRA_AFFECTED_METADATA_NAMES[action.metadataName] ?? []),+ ];++ for (const metadataName of affectedMetadataNames) {+ metadataNamesToInvalidate.add(metadataName);+ }+ }++ return [...metadataNamesToInvalidate].map(getMetadataFlatEntityMapsKey);+};