Skip to content

The open alternative to Salesforce, designed for AI.

perf(server): scope workspace migration cache invalidation to mutated maps (#29167)

Etienne34eb0885d774
diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-migration/workspace-migration-runner/services/workspace-migration-runner.service.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-migration/workspace-migration-runner/services/workspace-migration-runner.service.ts
index 2988da739f3..ce9dee9f223 100644
--- a/packages/twenty-server/src/engine/workspace-manager/workspace-migration/workspace-migration-runner/services/workspace-migration-runner.service.ts
+++ b/packages/twenty-server/src/engine/workspace-manager/workspace-migration/workspace-migration-runner/services/workspace-migration-runner.service.ts
@@ -32,6 +32,8 @@ import { WorkspaceMigrationRunnerActionHandlerRegistryService } from 'src/engine
import { DeferredWorkspaceMigrationActionRunnerService } from 'src/engine/workspace-manager/workspace-migration/workspace-migration-runner/services/deferred-workspace-migration-action-runner.service';
import { type DeferredWorkspaceMigrationAction } from 'src/engine/workspace-manager/workspace-migration/workspace-migration-runner/types/deferred-workspace-migration-action.type';
import { type MetadataEvent } from 'src/engine/workspace-manager/workspace-migration/workspace-migration-runner/types/metadata-event.type';
+import { getDerivedCacheTriggers } from 'src/engine/workspace-manager/workspace-migration/workspace-migration-runner/utils/get-derived-cache-triggers.util';
+import { getFlatEntityMapsKeysToInvalidateForWorkspaceMigration } from 'src/engine/workspace-manager/workspace-migration/workspace-migration-runner/utils/get-flat-entity-maps-keys-to-invalidate-for-workspace-migration.util';
import { getMetadataNamesToLoadForWorkspaceMigration } from 'src/engine/workspace-manager/workspace-migration/workspace-migration-runner/utils/get-metadata-names-to-load-for-workspace-migration.util';
import { buildPreallocatedIdByUniversalIdentifierFromActions } from 'src/engine/workspace-manager/workspace-migration/workspace-migration-runner/utils/build-preallocated-id-by-universal-identifier-from-actions.util';
@@ -61,34 +63,27 @@ export class WorkspaceMigrationRunnerService {
),
),
hasMorphRelationFieldChanged = false,
+ hasRolesPermissionsChanged = getDerivedCacheTriggers(allFlatEntityMapsKeys)
+ .hasRolesPermissionsChanged,
+ hasWorkflowVersionsChanged = getDerivedCacheTriggers(allFlatEntityMapsKeys)
+ .hasWorkflowVersionsChanged,
+ hasApplicationVariablesChanged = getDerivedCacheTriggers(
+ allFlatEntityMapsKeys,
+ ).hasApplicationVariablesChanged,
}: {
allFlatEntityMapsKeys: (keyof AllFlatEntityMaps)[];
workspaceId: string;
hasSchemaMetadataChanged?: boolean;
hasMorphRelationFieldChanged?: boolean;
+ hasRolesPermissionsChanged?: boolean;
+ hasWorkflowVersionsChanged?: boolean;
+ hasApplicationVariablesChanged?: boolean;
}): Promise<void> {
this.logger.perfTime(
'Runner',
`Cache invalidation ${allFlatEntityMapsKeys.join()}`,
);
- const hasRolesPermissionsChanged = allFlatEntityMapsKeys.some(
- (flatEntityMapsKey) =>
- flatEntityMapsKey === 'flatRoleMaps' ||
- flatEntityMapsKey === 'flatRoleTargetMaps' ||
- flatEntityMapsKey === 'flatObjectPermissionMaps' ||
- flatEntityMapsKey === 'flatFieldPermissionMaps' ||
- flatEntityMapsKey === 'flatRolePermissionFlagMaps',
- );
-
- const hasWorkflowVersionsChanged = allFlatEntityMapsKeys.includes(
- 'flatWorkflowVersionMaps',
- );
-
- const hasApplicationVariablesChanged = allFlatEntityMapsKeys.includes(
- 'flatApplicationVariableMaps',
- );
-
const cacheKeyNamesToInvalidate: WorkspaceCacheKeyName[] = [
...new Set([
...withDerivedFieldMetadataMaps(allFlatEntityMapsKeys),
@@ -286,6 +281,15 @@ export class WorkspaceMigrationRunnerService {
getMetadataFlatEntityMapsKey,
);
+ const flatEntityMapsKeysToInvalidate =
+ getFlatEntityMapsKeysToInvalidateForWorkspaceMigration(actions);
+
+ const {
+ hasRolesPermissionsChanged,
+ hasWorkflowVersionsChanged,
+ hasApplicationVariablesChanged,
+ } = getDerivedCacheTriggers(allFlatEntityMapsKeys);
+
const cachedAllFlatEntityMaps =
await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps<
typeof allFlatEntityMapsKeys
@@ -516,10 +520,13 @@ export class WorkspaceMigrationRunnerService {
try {
await this.invalidateCache({
- allFlatEntityMapsKeys,
+ allFlatEntityMapsKeys: flatEntityMapsKeysToInvalidate,
workspaceId,
hasSchemaMetadataChanged,
hasMorphRelationFieldChanged,
+ hasRolesPermissionsChanged,
+ hasWorkflowVersionsChanged,
+ hasApplicationVariablesChanged,
});
} catch (cacheError) {
this.logger.error(
@@ -547,10 +554,13 @@ export class WorkspaceMigrationRunnerService {
try {
await this.invalidateCache({
- allFlatEntityMapsKeys,
+ allFlatEntityMapsKeys: flatEntityMapsKeysToInvalidate,
workspaceId,
hasSchemaMetadataChanged,
hasMorphRelationFieldChanged,
+ hasRolesPermissionsChanged,
+ hasWorkflowVersionsChanged,
+ hasApplicationVariablesChanged,
});
this.recordRunPhaseMetric({
@@ -575,7 +585,7 @@ export class WorkspaceMigrationRunnerService {
performance.now() - postCommitInvalidateStart;
this.logger.perf(
- `[install-perf] Runner post-commit invalidateCache took ${postCommitInvalidateMs.toFixed(1)}ms for ${allFlatEntityMapsKeys.length} flat-maps keys`,
+ `[install-perf] Runner post-commit invalidateCache took ${postCommitInvalidateMs.toFixed(1)}ms for ${flatEntityMapsKeysToInvalidate.length} flat-maps keys`,
'Runner',
);
diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-migration/workspace-migration-runner/utils/__tests__/get-flat-entity-maps-keys-to-invalidate-for-workspace-migration.util.spec.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-migration/workspace-migration-runner/utils/__tests__/get-flat-entity-maps-keys-to-invalidate-for-workspace-migration.util.spec.ts
new file mode 100644
index 00000000000..61684cc249a
--- /dev/null
+++ b/packages/twenty-server/src/engine/workspace-manager/workspace-migration/workspace-migration-runner/utils/__tests__/get-flat-entity-maps-keys-to-invalidate-for-workspace-migration.util.spec.ts
@@ -0,0 +1,176 @@
+import { type AllUniversalWorkspaceMigrationAction } from 'src/engine/workspace-manager/workspace-migration/workspace-migration-builder/types/workspace-migration-action-common.type';
+import { getFlatEntityMapsKeysToInvalidateForWorkspaceMigration } from 'src/engine/workspace-manager/workspace-migration/workspace-migration-runner/utils/get-flat-entity-maps-keys-to-invalidate-for-workspace-migration.util';
+
+const buildAction = (
+ action: Partial<AllUniversalWorkspaceMigrationAction> &
+ Pick<AllUniversalWorkspaceMigrationAction, 'type' | 'metadataName'>,
+): AllUniversalWorkspaceMigrationAction =>
+ action as AllUniversalWorkspaceMigrationAction;
+
+describe('getFlatEntityMapsKeysToInvalidateForWorkspaceMigration', () => {
+ it('invalidates only the view field map for a viewField reorder', () => {
+ const result = getFlatEntityMapsKeysToInvalidateForWorkspaceMigration([
+ buildAction({
+ type: 'update',
+ metadataName: 'viewField',
+ universalIdentifier: 'view-field-1',
+ update: { position: 2 },
+ }),
+ ]);
+
+ expect(result).toEqual(['flatViewFieldMaps']);
+ });
+
+ it('invalidates the view field group map when a viewField update moves it between groups', () => {
+ const result = getFlatEntityMapsKeysToInvalidateForWorkspaceMigration([
+ buildAction({
+ type: 'update',
+ metadataName: 'viewField',
+ universalIdentifier: 'view-field-1',
+ update: {
+ position: 2,
+ viewFieldGroupUniversalIdentifier: 'view-field-group-1',
+ },
+ }),
+ ]);
+
+ expect(result.sort()).toEqual(
+ ['flatViewFieldGroupMaps', 'flatViewFieldMaps'].sort(),
+ );
+ expect(result).not.toContain('flatObjectMetadataMaps');
+ expect(result).not.toContain('flatFieldMetadataMaps');
+ });
+
+ it('invalidates the related view-side maps but not object metadata when a viewField is created', () => {
+ const result = getFlatEntityMapsKeysToInvalidateForWorkspaceMigration([
+ buildAction({
+ type: 'create',
+ metadataName: 'viewField',
+ }),
+ ]);
+
+ expect(result.sort()).toEqual(
+ [
+ 'flatFieldMetadataMaps',
+ 'flatViewFieldGroupMaps',
+ 'flatViewFieldMaps',
+ 'flatViewMaps',
+ ].sort(),
+ );
+ expect(result).not.toContain('flatObjectMetadataMaps');
+ });
+
+ it('invalidates the related view-side maps but not object metadata when a viewField is deleted', () => {
+ const result = getFlatEntityMapsKeysToInvalidateForWorkspaceMigration([
+ buildAction({
+ type: 'delete',
+ metadataName: 'viewField',
+ universalIdentifier: 'view-field-1',
+ }),
+ ]);
+
+ expect(result.sort()).toEqual(
+ [
+ 'flatFieldMetadataMaps',
+ 'flatViewFieldGroupMaps',
+ 'flatViewFieldMaps',
+ 'flatViewMaps',
+ ].sort(),
+ );
+ expect(result).not.toContain('flatObjectMetadataMaps');
+ });
+
+ it('invalidates only the field metadata map for a non-relation field update', () => {
+ const result = getFlatEntityMapsKeysToInvalidateForWorkspaceMigration([
+ buildAction({
+ type: 'update',
+ metadataName: 'fieldMetadata',
+ universalIdentifier: 'field-1',
+ update: { label: 'Renamed' },
+ }),
+ ]);
+
+ expect(result).toEqual(['flatFieldMetadataMaps']);
+ });
+
+ it('invalidates field, object and permission maps when an object is created (bundled fields and default permissions)', () => {
+ const result = getFlatEntityMapsKeysToInvalidateForWorkspaceMigration([
+ buildAction({
+ type: 'create',
+ metadataName: 'objectMetadata',
+ }),
+ ]);
+
+ expect(result).toContain('flatObjectMetadataMaps');
+ expect(result).toContain('flatFieldMetadataMaps');
+ expect(result).toContain('flatObjectPermissionMaps');
+ expect(result).toContain('flatFieldPermissionMaps');
+ });
+
+ it('invalidates object and field metadata maps on an index update (id re-key and isUnique derivation)', () => {
+ const result = getFlatEntityMapsKeysToInvalidateForWorkspaceMigration([
+ buildAction({
+ type: 'update',
+ metadataName: 'index',
+ universalIdentifier: 'index-1',
+ update: {},
+ }),
+ ]);
+
+ expect(result.sort()).toEqual(
+ [
+ 'flatFieldMetadataMaps',
+ 'flatIndexMaps',
+ 'flatObjectMetadataMaps',
+ ].sort(),
+ );
+ });
+
+ it('invalidates the field metadata map when a unique index is created (isUnique derives from index rows)', () => {
+ const result = getFlatEntityMapsKeysToInvalidateForWorkspaceMigration([
+ buildAction({
+ type: 'create',
+ metadataName: 'index',
+ }),
+ ]);
+
+ expect(result).toContain('flatFieldMetadataMaps');
+ expect(result).toContain('flatObjectMetadataMaps');
+ expect(result).toContain('flatIndexMaps');
+ });
+
+ it('invalidates the cascade-exposed child maps when a view is deleted', () => {
+ const result = getFlatEntityMapsKeysToInvalidateForWorkspaceMigration([
+ buildAction({
+ type: 'delete',
+ metadataName: 'view',
+ universalIdentifier: 'view-1',
+ }),
+ ]);
+
+ expect(result).toContain('flatViewMaps');
+ expect(result).toContain('flatViewFieldMaps');
+ expect(result).toContain('flatViewFieldGroupMaps');
+ expect(result).toContain('flatViewSortMaps');
+ expect(result).toContain('flatViewFilterMaps');
+ });
+
+ it('deduplicates keys across several actions', () => {
+ const result = getFlatEntityMapsKeysToInvalidateForWorkspaceMigration([
+ buildAction({
+ type: 'update',
+ metadataName: 'viewField',
+ universalIdentifier: 'view-field-1',
+ update: { position: 1 },
+ }),
+ buildAction({
+ type: 'update',
+ metadataName: 'viewField',
+ universalIdentifier: 'view-field-2',
+ update: { position: 2 },
+ }),
+ ]);
+
+ expect(result).toEqual(['flatViewFieldMaps']);
+ });
+});
diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-migration/workspace-migration-runner/utils/get-derived-cache-triggers.util.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-migration/workspace-migration-runner/utils/get-derived-cache-triggers.util.ts
new file mode 100644
index 00000000000..be986a8a3a7
--- /dev/null
+++ b/packages/twenty-server/src/engine/workspace-manager/workspace-migration/workspace-migration-runner/utils/get-derived-cache-triggers.util.ts
@@ -0,0 +1,26 @@
+import { type AllFlatEntityMaps } from 'src/engine/metadata-modules/flat-entity/types/all-flat-entity-maps.type';
+
+export type DerivedCacheTriggers = {
+ hasRolesPermissionsChanged: boolean;
+ hasWorkflowVersionsChanged: boolean;
+ hasApplicationVariablesChanged: boolean;
+};
+
+export const getDerivedCacheTriggers = (
+ flatEntityMapsKeys: (keyof AllFlatEntityMaps)[],
+): DerivedCacheTriggers => ({
+ hasRolesPermissionsChanged: flatEntityMapsKeys.some(
+ (flatEntityMapsKey) =>
+ flatEntityMapsKey === 'flatRoleMaps' ||
+ flatEntityMapsKey === 'flatRoleTargetMaps' ||
+ flatEntityMapsKey === 'flatObjectPermissionMaps' ||
+ flatEntityMapsKey === 'flatFieldPermissionMaps' ||
+ flatEntityMapsKey === 'flatRolePermissionFlagMaps',
+ ),
+ hasWorkflowVersionsChanged: flatEntityMapsKeys.includes(
+ 'flatWorkflowVersionMaps',
+ ),
+ hasApplicationVariablesChanged: flatEntityMapsKeys.includes(
+ 'flatApplicationVariableMaps',
+ ),
+});
diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-migration/workspace-migration-runner/utils/get-flat-entity-maps-keys-to-invalidate-for-workspace-migration.util.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-migration/workspace-migration-runner/utils/get-flat-entity-maps-keys-to-invalidate-for-workspace-migration.util.ts
new file mode 100644
index 00000000000..db7e84a5ce9
--- /dev/null
+++ b/packages/twenty-server/src/engine/workspace-manager/workspace-migration/workspace-migration-runner/utils/get-flat-entity-maps-keys-to-invalidate-for-workspace-migration.util.ts
@@ -0,0 +1,68 @@
+import { type AllMetadataName } from 'twenty-shared/metadata';
+import { isDefined } from 'twenty-shared/utils';
+
+import { ALL_MANY_TO_ONE_METADATA_RELATIONS } from 'src/engine/metadata-modules/flat-entity/constant/all-many-to-one-metadata-relations.constant';
+import { type AllFlatEntityMaps } from 'src/engine/metadata-modules/flat-entity/types/all-flat-entity-maps.type';
+import { getMetadataFlatEntityMapsKey } from 'src/engine/metadata-modules/flat-entity/utils/get-metadata-flat-entity-maps-key.util';
+import { getMetadataRelatedMetadataNames } from 'src/engine/metadata-modules/flat-entity/utils/get-metadata-related-metadata-names.util';
+import { type AllUniversalWorkspaceMigrationAction } from 'src/engine/workspace-manager/workspace-migration/workspace-migration-builder/types/workspace-migration-action-common.type';
+
+type ManyToOneRelation = {
+ metadataName: AllMetadataName;
+ foreignKey: string;
+ universalForeignKey: string | null;
+};
+
+const EXTRA_AFFECTED_METADATA_NAMES: Partial<
+ Record<AllMetadataName, AllMetadataName[]>
+> = {
+ index: ['objectMetadata', 'fieldMetadata'],
+};
+
+const getUpdateAffectedMetadataNames = ({
+ metadataName,
+ changedProperties,
+}: {
+ metadataName: AllMetadataName;
+ changedProperties: Set<string>;
+}): AllMetadataName[] => {
+ const manyToOneRelations = ALL_MANY_TO_ONE_METADATA_RELATIONS[
+ metadataName
+ ] as Record<string, ManyToOneRelation | null>;
+
+ return Object.values(manyToOneRelations)
+ .filter(
+ (relation): relation is ManyToOneRelation =>
+ isDefined(relation) &&
+ (changedProperties.has(relation.foreignKey) ||
+ (isDefined(relation.universalForeignKey) &&
+ changedProperties.has(relation.universalForeignKey))),
+ )
+ .map((relation) => relation.metadataName);
+};
+
+export const getFlatEntityMapsKeysToInvalidateForWorkspaceMigration = (
+ actions: AllUniversalWorkspaceMigrationAction[],
+): (keyof AllFlatEntityMaps)[] => {
+ const metadataNamesToInvalidate = new Set<AllMetadataName>();
+
+ for (const action of actions) {
+ metadataNamesToInvalidate.add(action.metadataName);
+
+ const affectedMetadataNames = [
+ ...(action.type === 'update'
+ ? getUpdateAffectedMetadataNames({
+ metadataName: action.metadataName,
+ changedProperties: new Set(Object.keys(action.update)),
+ })
+ : getMetadataRelatedMetadataNames(action.metadataName)),
+ ...(EXTRA_AFFECTED_METADATA_NAMES[action.metadataName] ?? []),
+ ];
+
+ for (const metadataName of affectedMetadataNames) {
+ metadataNamesToInvalidate.add(metadataName);
+ }
+ }
+
+ return [...metadataNamesToInvalidate].map(getMetadataFlatEntityMapsKey);
+};