The open alternative to Salesforce, designed for AI.
TypeScript crmcrm-systemcustomergood-first-issuegraphqlhacktoberfestjavascriptmarketingmonoreponestjsopen-sourcepostgresqlreactreactjssalestypescriptweb
Cache metadata lookups during ORM result formatting (#23593)
Weikobd65dbd47a09
diff --git a/packages/twenty-server/src/engine/twenty-orm/utils/__tests__/format-result.util.spec.ts b/packages/twenty-server/src/engine/twenty-orm/utils/__tests__/format-result.util.spec.tsnew file mode 100644index 00000000000..a40eef18a18--- /dev/null+++ b/packages/twenty-server/src/engine/twenty-orm/utils/__tests__/format-result.util.spec.ts@@ -0,0 +1,139 @@+import { FieldMetadataType, RelationType } from 'twenty-shared/types';++import { getFlatFieldsFromFlatObjectMetadata } from 'src/engine/api/graphql/workspace-schema-builder/utils/get-flat-fields-for-flat-object-metadata.util';+import { createEmptyFlatEntityMaps } from 'src/engine/metadata-modules/flat-entity/constant/create-empty-flat-entity-maps.constant';+import { type FlatEntityMaps } from 'src/engine/metadata-modules/flat-entity/types/flat-entity-maps.type';+import { addFlatEntityToFlatEntityMapsOrThrow } from 'src/engine/metadata-modules/flat-entity/utils/add-flat-entity-to-flat-entity-maps-or-throw.util';+import { getFlatFieldMetadataMock } from 'src/engine/metadata-modules/flat-field-metadata/__mocks__/get-flat-field-metadata.mock';+import { type FlatFieldMetadata } from 'src/engine/metadata-modules/flat-field-metadata/types/flat-field-metadata.type';+import { getFlatObjectMetadataMock } from 'src/engine/metadata-modules/flat-object-metadata/__mocks__/get-flat-object-metadata.mock';+import { type FlatObjectMetadata } from 'src/engine/metadata-modules/flat-object-metadata/types/flat-object-metadata.type';+import { formatResult } from 'src/engine/twenty-orm/utils/format-result.util';++jest.mock(+ 'src/engine/api/graphql/workspace-schema-builder/utils/get-flat-fields-for-flat-object-metadata.util',+ () => {+ const actual = jest.requireActual(+ 'src/engine/api/graphql/workspace-schema-builder/utils/get-flat-fields-for-flat-object-metadata.util',+ );++ return {+ ...actual,+ getFlatFieldsFromFlatObjectMetadata: jest.fn(+ actual.getFlatFieldsFromFlatObjectMetadata,+ ),+ };+ },+);++const buildFlatEntityMaps = <+ TFlatEntity extends FlatFieldMetadata | FlatObjectMetadata,+>(+ flatEntities: TFlatEntity[],+): FlatEntityMaps<TFlatEntity> =>+ flatEntities.reduce(+ (maps, flatEntity) =>+ addFlatEntityToFlatEntityMapsOrThrow({+ flatEntity,+ flatEntityMaps: maps,+ }),+ createEmptyFlatEntityMaps() as FlatEntityMaps<TFlatEntity>,+ );++describe('formatResult', () => {+ afterEach(() => {+ jest.clearAllMocks();+ });++ it('reuses derived field metadata per object type within one invocation', () => {+ const companyObjectMetadataId = '20202020-3c25-4d02-bf25-6aeccf7ea419';+ const personObjectMetadataId = '20202020-4c25-4d02-bf25-6aeccf7ea419';+ const companyNameFieldMetadata = getFlatFieldMetadataMock({+ id: '20202020-5c25-4d02-bf25-6aeccf7ea419',+ universalIdentifier: '20202020-6c25-4d02-bf25-6aeccf7ea419',+ objectMetadataId: companyObjectMetadataId,+ objectMetadataUniversalIdentifier: '20202020-7c25-4d02-bf25-6aeccf7ea419',+ name: 'name',+ type: FieldMetadataType.TEXT,+ });+ const companyPeopleFieldMetadata = getFlatFieldMetadataMock({+ id: '20202020-8c25-4d02-bf25-6aeccf7ea419',+ universalIdentifier: '20202020-9c25-4d02-bf25-6aeccf7ea419',+ objectMetadataId: companyObjectMetadataId,+ objectMetadataUniversalIdentifier:+ companyNameFieldMetadata.objectMetadataUniversalIdentifier,+ name: 'people',+ type: FieldMetadataType.RELATION,+ relationTargetObjectMetadataId: personObjectMetadataId,+ settings: {+ relationType: RelationType.ONE_TO_MANY,+ },+ });+ const personNameFieldMetadata = getFlatFieldMetadataMock({+ id: '20202020-ac25-4d02-bf25-6aeccf7ea419',+ universalIdentifier: '20202020-bc25-4d02-bf25-6aeccf7ea419',+ objectMetadataId: personObjectMetadataId,+ objectMetadataUniversalIdentifier: '20202020-cc25-4d02-bf25-6aeccf7ea419',+ name: 'name',+ type: FieldMetadataType.TEXT,+ });+ const companyObjectMetadata = getFlatObjectMetadataMock({+ id: companyObjectMetadataId,+ universalIdentifier:+ companyNameFieldMetadata.objectMetadataUniversalIdentifier,+ fieldIds: [companyNameFieldMetadata.id, companyPeopleFieldMetadata.id],+ });+ const personObjectMetadata = getFlatObjectMetadataMock({+ id: personObjectMetadataId,+ universalIdentifier:+ personNameFieldMetadata.objectMetadataUniversalIdentifier,+ fieldIds: [personNameFieldMetadata.id],+ });+ const flatObjectMetadataMaps = buildFlatEntityMaps([+ companyObjectMetadata,+ personObjectMetadata,+ ]);+ const flatFieldMetadataMaps = buildFlatEntityMaps([+ companyNameFieldMetadata,+ companyPeopleFieldMetadata,+ personNameFieldMetadata,+ ]);+ const records = [+ {+ name: 'Acme',+ people: [{ name: 'Alice' }, { name: 'Bob' }],+ },+ {+ name: 'Globex',+ people: [{ name: 'Carol' }],+ },+ ];++ const formattedRecords = formatResult(+ records,+ companyObjectMetadata,+ flatObjectMetadataMaps,+ flatFieldMetadataMaps,+ );++ expect(formattedRecords).toEqual(records);++ const firstInvocationMetadataLookupCount = jest.mocked(+ getFlatFieldsFromFlatObjectMetadata,+ ).mock.calls.length;++ expect(firstInvocationMetadataLookupCount).toBeGreaterThan(0);+ expect(firstInvocationMetadataLookupCount).toBeLessThanOrEqual(4);++ formatResult(+ records,+ companyObjectMetadata,+ flatObjectMetadataMaps,+ flatFieldMetadataMaps,+ );++ expect(+ jest.mocked(getFlatFieldsFromFlatObjectMetadata).mock.calls.length,+ ).toBe(firstInvocationMetadataLookupCount * 2);+ });+});diff --git a/packages/twenty-server/src/engine/twenty-orm/utils/format-result.util.ts b/packages/twenty-server/src/engine/twenty-orm/utils/format-result.util.tsindex 68716ebe83b..d020758b68b 100644--- a/packages/twenty-server/src/engine/twenty-orm/utils/format-result.util.ts+++ b/packages/twenty-server/src/engine/twenty-orm/utils/format-result.util.ts@@ -15,6 +15,7 @@ import {} from 'src/engine/api/common/common-args-processors/data-arg-processor/constants/null-equivalent-values.constant';import { getFlatFieldsFromFlatObjectMetadata } from 'src/engine/api/graphql/workspace-schema-builder/utils/get-flat-fields-for-flat-object-metadata.util';import { computeCompositeColumnName } from 'src/engine/metadata-modules/field-metadata/utils/compute-column-name.util';+import { isCompositeFieldMetadataType } from 'src/engine/metadata-modules/field-metadata/utils/is-composite-field-metadata-type.util';import { type FlatEntityMaps } from 'src/engine/metadata-modules/flat-entity/types/flat-entity-maps.type';import { findFlatEntityByIdInFlatEntityMaps } from 'src/engine/metadata-modules/flat-entity/utils/find-flat-entity-by-id-in-flat-entity-maps.util';import { type FlatFieldMetadata } from 'src/engine/metadata-modules/flat-field-metadata/types/flat-field-metadata.type';@@ -27,6 +28,26 @@ import { formatCompositeFieldValue } from 'src/engine/twenty-orm/utils/format-coimport { getCompositeFieldMetadataCollection } from 'src/engine/twenty-orm/utils/get-composite-field-metadata-collection';import { isFieldMetadataEntityOfType } from 'src/engine/utils/is-field-metadata-of-type.util';+type CompositeFieldMetadataWithRequiredProperties = {+ fieldMetadata: FlatFieldMetadata;+ requiredPropertyNames: string[];+};++type FormatResultObjectCache = {+ fieldMaps: FieldMapsForObject;+ compositeFieldMetadataMap: ReturnType<+ typeof getCompositeFieldMetadataMapFromCollection+ >;+ compositeFieldMetadataWithRequiredProperties: CompositeFieldMetadataWithRequiredProperties[];+ dateTimeFieldMetadataItems: FlatFieldMetadata[];+};++type FormatResultCache = {+ flatObjectMetadataMaps: FlatEntityMaps<FlatObjectMetadata>;+ flatFieldMetadataMaps: FlatEntityMaps<FlatFieldMetadata>;+ objectCacheByObjectMetadataId: Map<string, FormatResultObjectCache>;+};+export function formatResult<T>(// oxlint-disable-next-line typescript/no-explicit-anydata: any,@@ -34,6 +55,25 @@ export function formatResult<T>(flatObjectMetadataMaps: FlatEntityMaps<FlatObjectMetadata>,flatFieldMetadataMaps: FlatEntityMaps<FlatFieldMetadata>,fieldMapsForObject?: FieldMapsForObject,+): T {+ return formatResultRecursively(+ data,+ flatObjectMetadata,+ {+ flatObjectMetadataMaps,+ flatFieldMetadataMaps,+ objectCacheByObjectMetadataId: new Map(),+ },+ fieldMapsForObject,+ );+}++function formatResultRecursively<T>(+ // oxlint-disable-next-line typescript/no-explicit-any+ data: any,+ flatObjectMetadata: FlatObjectMetadata | undefined,+ cache: FormatResultCache,+ fieldMapsForObject?: FieldMapsForObject,): T {if (!isDefined(data)) {return data;@@ -42,11 +82,10 @@ export function formatResult<T>(if (!isPlainObject(data)) {if (Array.isArray(data)) {return data.map((item) =>- formatResult(+ formatResultRecursively(item,flatObjectMetadata,- flatObjectMetadataMaps,- flatFieldMetadataMaps,+ cache,fieldMapsForObject,),) as T;@@ -59,24 +98,19 @@ export function formatResult<T>(throw new Error('Object metadata is missing');}- const fieldMaps =- fieldMapsForObject ??- buildFieldMapsFromFlatObjectMetadata(- flatFieldMetadataMaps,- flatObjectMetadata,- );-- const { fieldIdByName, fieldIdByJoinColumnName } = fieldMaps;-- const compositeFieldMetadataMap = getCompositeFieldMetadataMap(+ const objectCache = getOrCreateFormatResultObjectCache({+ cache,flatObjectMetadata,- flatFieldMetadataMaps,- );+ fieldMapsForObject,+ });++ const { fieldIdByName, fieldIdByJoinColumnName } = objectCache.fieldMaps;const newData: object = {};for (const [key, value] of Object.entries(data)) {- const compositePropertyArgs = compositeFieldMetadataMap.get(key);+ const compositePropertyArgs =+ objectCache.compositeFieldMetadataMap.get(key);const fieldMetadataId =fieldIdByName[key] ||@@ -85,7 +119,7 @@ export function formatResult<T>(const fieldMetadata = findFlatEntityByIdInFlatEntityMaps({flatEntityId: fieldMetadataId,- flatEntityMaps: flatFieldMetadataMaps,+ flatEntityMaps: cache.flatFieldMetadataMaps,});if (!isDefined(fieldMetadata)) {@@ -105,7 +139,7 @@ export function formatResult<T>(const targetObjectMetadata = findFlatEntityByIdInFlatEntityMaps({flatEntityId: fieldMetadata.relationTargetObjectMetadataId,- flatEntityMaps: flatObjectMetadataMaps,+ flatEntityMaps: cache.flatObjectMetadataMaps,});if (!targetObjectMetadata) {@@ -115,11 +149,10 @@ export function formatResult<T>(}// @ts-expect-error legacy noImplicitAny- newData[key] = formatResult(+ newData[key] = formatResultRecursively(value,targetObjectMetadata,- flatObjectMetadataMaps,- flatFieldMetadataMaps,+ cache,);continue;}@@ -160,34 +193,10 @@ export function formatResult<T>(// After assembling composite fields, handle those with missing required subfieldshandleEmptyCompositeFields(newData,- flatObjectMetadata,- flatFieldMetadataMaps,+ objectCache.compositeFieldMetadataWithRequiredProperties,);- const fieldMetadataItemsOfTypeDateOnly = getFlatFieldsFromFlatObjectMetadata(- flatObjectMetadata,- flatFieldMetadataMaps,- ).filter((field) => field.type === FieldMetadataType.DATE);-- for (const dateField of fieldMetadataItemsOfTypeDateOnly) {- // @ts-expect-error legacy noImplicitAny- const rawUpdatedDate = newData[dateField.name] as string | null | undefined;-- if (!isDefined(rawUpdatedDate)) {- continue;- }-- // @ts-expect-error legacy noImplicitAny- newData[dateField.name] = rawUpdatedDate;- }-- const fieldMetadataItemsOfTypeDateTimeOnly =- getFlatFieldsFromFlatObjectMetadata(- flatObjectMetadata,- flatFieldMetadataMaps,- ).filter((field) => field.type === FieldMetadataType.DATE_TIME);-- for (const dateTimeField of fieldMetadataItemsOfTypeDateTimeOnly) {+ for (const dateTimeField of objectCache.dateTimeFieldMetadataItems) {// @ts-expect-error legacy noImplicitAnyconst rawUpdatedDateTime = newData[dateTimeField.name] as| string@@ -219,6 +228,63 @@ export function formatResult<T>(return newData as T;}+function getOrCreateFormatResultObjectCache({+ cache,+ flatObjectMetadata,+ fieldMapsForObject,+}: {+ cache: FormatResultCache;+ flatObjectMetadata: FlatObjectMetadata;+ fieldMapsForObject?: FieldMapsForObject;+}): FormatResultObjectCache {+ const cachedObjectCache = cache.objectCacheByObjectMetadataId.get(+ flatObjectMetadata.id,+ );++ if (isDefined(cachedObjectCache)) {+ return cachedObjectCache;+ }++ const flatFieldMetadataItems = getFlatFieldsFromFlatObjectMetadata(+ flatObjectMetadata,+ cache.flatFieldMetadataMaps,+ );+ const compositeFieldMetadataCollection = flatFieldMetadataItems.filter(+ (fieldMetadata) => isCompositeFieldMetadataType(fieldMetadata.type),+ );++ const objectCache = {+ fieldMaps:+ fieldMapsForObject ??+ buildFieldMapsFromFlatObjectMetadata(+ cache.flatFieldMetadataMaps,+ flatObjectMetadata,+ ),+ compositeFieldMetadataMap: getCompositeFieldMetadataMapFromCollection(+ compositeFieldMetadataCollection,+ ),+ compositeFieldMetadataWithRequiredProperties:+ compositeFieldMetadataCollection.map((fieldMetadata) => {+ const compositeType = compositeTypeDefinitions.get(fieldMetadata.type);++ return {+ fieldMetadata,+ requiredPropertyNames:+ compositeType?.properties+ .filter((property) => property.isRequired)+ .map((property) => property.name) ?? [],+ };+ }),+ dateTimeFieldMetadataItems: flatFieldMetadataItems.filter(+ (fieldMetadata) => fieldMetadata.type === FieldMetadataType.DATE_TIME,+ ),+ } satisfies FormatResultObjectCache;++ cache.objectCacheByObjectMetadataId.set(flatObjectMetadata.id, objectCache);++ return objectCache;+}+export function getCompositeFieldMetadataMap(flatObjectMetadata: FlatObjectMetadata,flatFieldMetadataMaps: FlatEntityMaps<FlatFieldMetadata>,@@ -228,6 +294,14 @@ export function getCompositeFieldMetadataMap(flatFieldMetadataMaps,);+ return getCompositeFieldMetadataMapFromCollection(+ compositeFieldMetadataCollection,+ );+}++function getCompositeFieldMetadataMapFromCollection(+ compositeFieldMetadataCollection: FlatFieldMetadata[],+) {return new Map(compositeFieldMetadataCollection.flatMap((fieldMetadata) => {const compositeType = compositeTypeDefinitions.get(fieldMetadata.type);@@ -304,42 +378,28 @@ function transformCompositeFieldNullValue(function handleEmptyCompositeFields(// oxlint-disable-next-line typescript/no-explicit-anydata: Record<string, any>,- flatObjectMetadata: FlatObjectMetadata,- flatFieldMetadataMaps: FlatEntityMaps<FlatFieldMetadata>,+ compositeFieldMetadataWithRequiredProperties: CompositeFieldMetadataWithRequiredProperties[],) {- const compositeFieldMetadataCollection = getCompositeFieldMetadataCollection(- flatObjectMetadata,- flatFieldMetadataMaps,- );-- for (const fieldMetadata of compositeFieldMetadataCollection) {+ for (const {+ fieldMetadata,+ requiredPropertyNames,+ } of compositeFieldMetadataWithRequiredProperties) {const fieldValue = data[fieldMetadata.name];if (!isDefined(fieldValue) || !isPlainObject(fieldValue)) {continue;}- const compositeType = compositeTypeDefinitions.get(fieldMetadata.type);-- if (!compositeType) {- continue;- }-// oxlint-disable-next-line typescript/no-explicit-anyconst typedFieldValue = fieldValue as Record<string, any>;- // Check if all required properties are null/undefined- const requiredProperties = compositeType.properties.filter(- (prop) => prop.isRequired,- );-- const allRequiredPropertiesAreNull = requiredProperties.every(- (prop) =>- !isDefined(typedFieldValue[prop.name]) ||- isNull(typedFieldValue[prop.name]),+ const allRequiredPropertiesAreNull = requiredPropertyNames.every(+ (propertyName) =>+ !isDefined(typedFieldValue[propertyName]) ||+ isNull(typedFieldValue[propertyName]),);- if (allRequiredPropertiesAreNull && requiredProperties.length > 0) {+ if (allRequiredPropertiesAreNull && requiredPropertyNames.length > 0) {if (fieldMetadata.isNullable) {// Field is nullable, set to nulldata[fieldMetadata.name] = null;