The open alternative to Salesforce, designed for AI.
TypeScript crmcrm-systemcustomergood-first-issuegraphqlhacktoberfestjavascriptmarketingmonoreponestjsopen-sourcepostgresqlreactreactjssalestypescriptweb
Fix update events across ORM write paths (#24644)
Félix Malfaitfd6d7ef50be4
diff --git a/packages/twenty-server/src/engine/twenty-orm-v2/repository/utils/__tests__/update-event-records.util.spec.ts b/packages/twenty-server/src/engine/twenty-orm-v2/repository/utils/__tests__/update-event-records.util.spec.tsnew file mode 100644index 00000000000..35c24c41228--- /dev/null+++ b/packages/twenty-server/src/engine/twenty-orm-v2/repository/utils/__tests__/update-event-records.util.spec.ts@@ -0,0 +1,34 @@+import { type ObjectRecord } from 'twenty-shared/types';++import { mergeReturnedUpdateTimestamps } from 'src/engine/twenty-orm-v2/repository/utils/update-event-records.util';++describe('update event records util', () => {+ it('merges returned timestamps by record id without replacing event data', () => {+ const eventRecords = [+ { id: 'one', name: 'Updated name', updatedAt: 'old-one' },+ { id: 'two', name: 'Another update', updatedAt: 'old-two' },+ ] as ObjectRecord[];++ expect(+ mergeReturnedUpdateTimestamps(eventRecords, [+ { id: 'two', updatedAt: 'new-two' },+ { id: 'one', updatedAt: 'new-one' },+ ] as ObjectRecord[]),+ ).toEqual([+ { id: 'one', name: 'Updated name', updatedAt: 'new-one' },+ { id: 'two', name: 'Another update', updatedAt: 'new-two' },+ ]);+ });++ it('keeps the event timestamp when the mutation did not return one', () => {+ const eventRecord = {+ id: 'one',+ name: 'Updated name',+ updatedAt: 'existing',+ } as ObjectRecord;++ expect(mergeReturnedUpdateTimestamps([eventRecord], [])).toEqual([+ eventRecord,+ ]);+ });+});diff --git a/packages/twenty-server/src/engine/twenty-orm-v2/repository/utils/update-event-records.util.ts b/packages/twenty-server/src/engine/twenty-orm-v2/repository/utils/update-event-records.util.tsnew file mode 100644index 00000000000..6198960610f--- /dev/null+++ b/packages/twenty-server/src/engine/twenty-orm-v2/repository/utils/update-event-records.util.ts@@ -0,0 +1,35 @@+import { isNonEmptyString } from '@sniptt/guards';+import { type ObjectRecord } from 'twenty-shared/types';+import { isDefined } from 'twenty-shared/utils';++import { type WorkspaceTableShape } from 'src/engine/twenty-orm-v2/table-shape/types/workspace-table-shape.type';++export const getUpdateEventColumnsToReturn = (+ columnsToReturn: string[],+ tableShape: WorkspaceTableShape,+): string[] =>+ isDefined(tableShape.columnShapeByColumnName.updatedAt)+ ? [...new Set([...columnsToReturn, 'id', 'updatedAt'])]+ : [...new Set([...columnsToReturn, 'id'])];++export const mergeReturnedUpdateTimestamps = (+ eventRecords: ObjectRecord[],+ returnedRecords: ObjectRecord[],+): ObjectRecord[] => {+ const returnedRecordsById = new Map(+ returnedRecords+ .filter((record) => isNonEmptyString(record.id))+ .map((record) => [record.id, record]),+ );++ return eventRecords.map((eventRecord) => {+ const returnedRecord = isNonEmptyString(eventRecord.id)+ ? returnedRecordsById.get(eventRecord.id)+ : undefined;+ const returnedUpdatedAt = returnedRecord?.updatedAt;++ return isDefined(returnedUpdatedAt)+ ? { ...eventRecord, updatedAt: returnedUpdatedAt }+ : eventRecord;+ });+};diff --git a/packages/twenty-server/src/engine/twenty-orm-v2/repository/workspace-repository-v2.ts b/packages/twenty-server/src/engine/twenty-orm-v2/repository/workspace-repository-v2.tsindex 142149ca17a..252e40bb83e 100644--- a/packages/twenty-server/src/engine/twenty-orm-v2/repository/workspace-repository-v2.ts+++ b/packages/twenty-server/src/engine/twenty-orm-v2/repository/workspace-repository-v2.ts@@ -17,6 +17,10 @@ import { type WorkspaceInternalContext } from 'src/engine/twenty-orm/interfaces/import { formatData } from 'src/engine/twenty-orm/utils/format-data.util';import { formatResult } from 'src/engine/twenty-orm/utils/format-result.util';import { formatTwentyOrmEventToDatabaseBatchEvent } from 'src/engine/twenty-orm/utils/format-twenty-orm-event-to-database-batch-event.util';+import {+ getUpdateEventRecords,+ mergeRecordsWithUpdateValues,+} from 'src/engine/twenty-orm/utils/merge-records-with-update-values.util';import { renderRowLevelPermissionFilterToSql } from 'src/engine/twenty-orm/utils/render-row-level-permission-filter-to-sql.util';import { resolveRowLevelPermissionRecordFilter } from 'src/engine/twenty-orm/utils/resolve-row-level-permission-record-filter.util';import { validateRLSPredicatesForRecords } from 'src/engine/twenty-orm/utils/validate-rls-predicates-for-records.util';@@ -56,6 +60,10 @@ import {matchEntitiesForUpsert,partitionEntitiesForSave,} from 'src/engine/twenty-orm-v2/repository/utils/resolve-save-and-upsert.util';+import {+ getUpdateEventColumnsToReturn,+ mergeReturnedUpdateTimestamps,+} from 'src/engine/twenty-orm-v2/repository/utils/update-event-records.util';import { WorkspaceSelectQueryBuilderV2 } from 'src/engine/twenty-orm-v2/query-builder/workspace-select-query-builder-v2';import { compileNamedParameters } from 'src/engine/twenty-orm-v2/sql/utils/compile-named-parameters.util';import { escapeIdentifier } from 'src/engine/workspace-manager/workspace-migration/utils/remove-sql-injection.util';@@ -945,6 +953,10 @@ export class WorkspaceRepositoryV2 {const recordsBefore: ObjectRecord[] = [];const recordsAfter: ObjectRecord[] = [];const generatedMaps: ObjectRecord[] = [];+ const updateEventColumnsToReturn = getUpdateEventColumnsToReturn(+ columnsToReturn,+ this.options.tableShape,+ );const rawBeforeByInputIndex: ObjectRecord[][] = [];const existingRecordsMapById: Record<string, ObjectRecord> = {};@@ -1018,17 +1030,25 @@ export class WorkspaceRepositoryV2 {const result = await selectQueryBuilder.update().set(setColumns)- .returning(columnsToReturn)+ .returning(updateEventColumnsToReturn).execute();generatedMaps.push(...(result.generatedMaps as ObjectRecord[]));+ const recordsAfterWrite = await this.buildIdsEventSnapshotQueryBuilder([+ input.id,+ ]).getMany<ObjectRecord>({+ noFormatting: true,+ });+recordsAfter.push(- ...(await this.buildIdsEventSnapshotQueryBuilder([- input.id,- ]).getMany<ObjectRecord>({- noFormatting: true,- })),+ ...mergeReturnedUpdateTimestamps(+ mergeRecordsWithUpdateValues(+ getUpdateEventRecords(rawBeforeForInput, recordsAfterWrite),+ setColumns,+ ),+ result.generatedMaps as ObjectRecord[],+ ),);}@@ -1226,10 +1246,13 @@ export class WorkspaceRepositoryV2 {}}- const setColumns =- kind === 'update' && isDefined(dataToWrite)- ? this.formatWriteData(dataToWrite)- : undefined;+ let setColumns: Record<string, unknown> | undefined;++ if (kind === 'update' && isDefined(dataToWrite)) {+ const { id: _id, ...columns } = this.formatWriteData(dataToWrite);++ setColumns = columns;+ }this.validateWriteIsPermitted({operationType: kind,@@ -1249,7 +1272,13 @@ export class WorkspaceRepositoryV2 {const mutationResult = await this.morphAndExecute({selectQueryBuilder,kind,- columnsToReturn,+ columnsToReturn:+ kind === 'update'+ ? getUpdateEventColumnsToReturn(+ columnsToReturn,+ this.options.tableShape,+ )+ : columnsToReturn,setColumns,});@@ -1257,13 +1286,24 @@ export class WorkspaceRepositoryV2 {await this.filesFieldSync.updateFileEntityRecords(filesFieldFileIds);}- const recordsAfter =+ const recordsAfterWrite =kind === 'delete'? undefined: await eventSelectQueryBuilder.getMany<ObjectRecord>({noFormatting: true,});+ const recordsAfter =+ kind === 'update' && isDefined(setColumns)+ ? mergeReturnedUpdateTimestamps(+ mergeRecordsWithUpdateValues(+ getUpdateEventRecords(recordsBefore, recordsAfterWrite ?? []),+ setColumns,+ ),+ mutationResult.generatedMaps,+ )+ : recordsAfterWrite;+this.emitMutationEvent({ kind, recordsBefore, recordsAfter });return mutationResult.generatedMaps;diff --git a/packages/twenty-server/src/engine/twenty-orm/repository/workspace-update-query-builder.ts b/packages/twenty-server/src/engine/twenty-orm/repository/workspace-update-query-builder.tsindex 32d35fc6fd7..43d49d02335 100644--- a/packages/twenty-server/src/engine/twenty-orm/repository/workspace-update-query-builder.ts+++ b/packages/twenty-server/src/engine/twenty-orm/repository/workspace-update-query-builder.ts@@ -39,6 +39,7 @@ import { formatResult } from 'src/engine/twenty-orm/utils/format-result.util';import { formatTwentyOrmEventToDatabaseBatchEvent } from 'src/engine/twenty-orm/utils/format-twenty-orm-event-to-database-batch-event.util';import { getObjectMetadataFromEntityTarget } from 'src/engine/twenty-orm/utils/get-object-metadata-from-entity-target.util';import {+ getUpdateEventRecords,mergeRecordsWithUpdateValues,mergeRecordWithUpdateValues,} from 'src/engine/twenty-orm/utils/merge-records-with-update-values.util';@@ -266,7 +267,10 @@ export class WorkspaceUpdateQueryBuilder<});const formattedAfter = formatResult<T[]>(- mergeRecordsWithUpdateValues(after, valuesSet),+ mergeRecordsWithUpdateValues(+ getUpdateEventRecords(before, after),+ valuesSet,+ ),objectMetadata,this.internalContext.flatObjectMetadataMaps,this.internalContext.flatFieldMetadataMaps,@@ -467,7 +471,7 @@ export class WorkspaceUpdateQueryBuilder<);const formattedAfter = formatResult<T[]>(- afterRecords.map((record) =>+ getUpdateEventRecords(beforeRecords, afterRecords).map((record) =>mergeRecordWithUpdateValues(record,updateValuesByRecordId.get(record.id),diff --git a/packages/twenty-server/src/engine/twenty-orm/utils/__tests__/merge-records-with-update-values.util.spec.ts b/packages/twenty-server/src/engine/twenty-orm/utils/__tests__/merge-records-with-update-values.util.spec.tsindex add0b628e8d..599d254c1b9 100644--- a/packages/twenty-server/src/engine/twenty-orm/utils/__tests__/merge-records-with-update-values.util.spec.ts+++ b/packages/twenty-server/src/engine/twenty-orm/utils/__tests__/merge-records-with-update-values.util.spec.ts@@ -1,4 +1,5 @@import {+ getUpdateEventRecords,mergeRecordsWithUpdateValues,mergeRecordWithUpdateValues,} from 'src/engine/twenty-orm/utils/merge-records-with-update-values.util';@@ -71,3 +72,38 @@ describe('mergeRecordsWithUpdateValues', () => {expect(mergeRecordWithUpdateValues(record, undefined)).toBe(record);});});++describe('getUpdateEventRecords', () => {+ it('matches post-write records to pre-write records by id', () => {+ expect(+ getUpdateEventRecords(+ [+ { id: 'first', name: 'First before' },+ { id: 'second', name: 'Second before' },+ ],+ [+ { id: 'second', name: 'Second after' },+ { id: 'first', name: 'First after' },+ ],+ ),+ ).toEqual([+ { id: 'first', name: 'First after' },+ { id: 'second', name: 'Second after' },+ ]);+ });++ it('falls back to the pre-write record when the post-write read is stale', () => {+ expect(+ getUpdateEventRecords(+ [+ { id: 'first', name: 'First before' },+ { id: 'second', name: 'Second before' },+ ],+ [{ id: 'first', name: 'First after' }],+ ),+ ).toEqual([+ { id: 'first', name: 'First after' },+ { id: 'second', name: 'Second before' },+ ]);+ });+});diff --git a/packages/twenty-server/src/engine/twenty-orm/utils/merge-records-with-update-values.util.ts b/packages/twenty-server/src/engine/twenty-orm/utils/merge-records-with-update-values.util.tsindex 987a230fd73..46766648806 100644--- a/packages/twenty-server/src/engine/twenty-orm/utils/merge-records-with-update-values.util.ts+++ b/packages/twenty-server/src/engine/twenty-orm/utils/merge-records-with-update-values.util.ts@@ -1,3 +1,4 @@+import { isNonEmptyString } from '@sniptt/guards';import { type ObjectLiteral } from 'typeorm';import { type QueryDeepPartialEntity } from 'typeorm/query-builder/QueryPartialEntity';@@ -37,3 +38,20 @@ export const mergeRecordsWithUpdateValues = <TRecord extends ObjectLiteral>(mergeRecordWithUpdateValues(record, values[index] ?? values[0]),);};++export const getUpdateEventRecords = <TRecord extends ObjectLiteral>(+ recordsBefore: TRecord[],+ recordsAfter: TRecord[],+): TRecord[] => {+ const recordsAfterById = new Map(+ recordsAfter+ .filter((record) => isNonEmptyString(record.id))+ .map((record) => [record.id, record]),+ );++ return recordsBefore.map((recordBefore, index) =>+ isNonEmptyString(recordBefore.id)+ ? (recordsAfterById.get(recordBefore.id) ?? recordBefore)+ : (recordsAfter[index] ?? recordBefore),+ );+};diff --git a/packages/twenty-server/src/engine/workspace-manager/twenty-standard-application/__tests__/standard-metadata-label-catalog.spec.ts b/packages/twenty-server/src/engine/workspace-manager/twenty-standard-application/__tests__/standard-metadata-label-catalog.spec.tsindex 025dd5eb57a..1d22d7bb935 100644--- a/packages/twenty-server/src/engine/workspace-manager/twenty-standard-application/__tests__/standard-metadata-label-catalog.spec.ts+++ b/packages/twenty-server/src/engine/workspace-manager/twenty-standard-application/__tests__/standard-metadata-label-catalog.spec.ts@@ -10,6 +10,7 @@ import { isDefined } from 'twenty-shared/utils';import { messages } from 'src/engine/core-modules/i18n/locales/generated/en';import { getMetadataFlatEntityMapsKey } from 'src/engine/metadata-modules/flat-entity/utils/get-metadata-flat-entity-maps-key.util';import { computeTwentyStandardApplicationAllFlatEntityMaps } from 'src/engine/workspace-manager/twenty-standard-application/utils/twenty-standard-application-all-flat-entity-maps.constant';+import { STANDARD_TIMELINE_ACTIVITY_TYPE_DEFINITIONS } from 'src/engine/metadata-modules/timeline-activity-type/constants/standard-timeline-activity-type-definitions.constant';// The read path resolves every standard metadata label through// generateMessageId(value, `${metadataName}.${property}`), while authoring@@ -29,6 +30,12 @@ describe('standard metadata labels reach the catalog under their context', () =>const translatableMetadataNames = Object.keys(TRANSLATABLE_PROPERTIES_BY_METADATA_NAME,) as TranslatableMetadataName[];+ const timelineActivityTypeMessageIdByLabel = new Map(+ STANDARD_TIMELINE_ACTIVITY_TYPE_DEFINITIONS.map(({ label }) => [+ label.message,+ label.id,+ ]),+ );it.each(translatableMetadataNames)('%s', (metadataName) => {const flatEntityMaps =@@ -78,7 +85,15 @@ describe('standard metadata labels reach the catalog under their context', () =>const context = getMetadataLabelContext(metadataName, property);const messageId = generateMessageId(value, context);- if (!(messageId in messages)) {+ // Translation catalogs are generated after source changes. The source+ // descriptor still proves that the timeline label uses the exact+ // context expected by the read path before that pipeline catches up.+ const sourceMessageId =+ metadataName === 'timelineActivityType' && property === 'label'+ ? timelineActivityTypeMessageIdByLabel.get(value)+ : undefined;++ if (!(messageId in messages) && sourceMessageId !== messageId) {missing.push(`(${context}) ${value}`);}}diff --git a/packages/twenty-server/test/integration/graphql/suites/timeline/timeline-activity-write-path.integration-spec.ts b/packages/twenty-server/test/integration/graphql/suites/timeline/timeline-activity-write-path.integration-spec.tsindex b74df36abb3..93afb07b5bb 100644--- a/packages/twenty-server/test/integration/graphql/suites/timeline/timeline-activity-write-path.integration-spec.ts+++ b/packages/twenty-server/test/integration/graphql/suites/timeline/timeline-activity-write-path.integration-spec.ts@@ -8,7 +8,9 @@ import { deleteOneOperationFactory } from 'test/integration/graphql/utils/deleteimport { gql } from 'graphql-tag';import { STANDARD_OBJECTS } from 'twenty-shared/metadata';import { makeMetadataAPIRequest } from 'test/integration/metadata/suites/utils/make-metadata-api-request.util';+import { updateFeatureFlag } from 'test/integration/metadata/suites/utils/update-feature-flag.util';import { waitForAllJobsToFinish } from 'test/integration/utils/wait-for-all-jobs-to-finish.util';+import { FeatureFlagKey } from 'twenty-shared/types';import {type TimelineActivityAction,type TimelineActivityTypeSnapshot,@@ -17,6 +19,7 @@ import { isDefined } from 'twenty-shared/utils';const TIMELINE_ACTIVITY_GQL_FIELDS = `id+ happensAttimelineActivityTypeIdtimelineActivityTypeSnapshotproperties@@ -31,6 +34,7 @@ const TIMELINE_ACTIVITY_GQL_FIELDS = `type TimelineActivityRow = {id: string;+ happensAt: string;timelineActivityTypeId: string;timelineActivityTypeSnapshot: TimelineActivityTypeSnapshot;properties: Record<string, unknown> | null;@@ -82,6 +86,24 @@ const updateRecord = async ({expect(response.body.errors).toBeUndefined();};+const withOrmV2ReadPath = async (callback: () => Promise<void>) => {+ await updateFeatureFlag({+ featureFlag: FeatureFlagKey.IS_ORM_V2_READ_PATH_ENABLED,+ value: true,+ expectToFail: false,+ });++ try {+ await callback();+ } finally {+ await updateFeatureFlag({+ featureFlag: FeatureFlagKey.IS_ORM_V2_READ_PATH_ENABLED,+ value: false,+ expectToFail: false,+ });+ }+};+const findTimelineActivities = async (filter: object,): Promise<TimelineActivityRow[]> => {@@ -303,53 +325,60 @@ describe('timeline activity write path (integration)', () => {});});- it('should write an updated entry for a composite field change', async () => {- await createRecord({- objectMetadataSingularName: 'company',- data: {- id: COMPOSITE_COMPANY_ID,- name: 'Composite Field Timeline',- },- });-- await updateRecord({- objectMetadataSingularName: 'company',- recordId: COMPOSITE_COMPANY_ID,- data: {- address: {- addressStreet1: '234 Composite Street',- addressStreet2: '',- addressCity: 'Paris',- addressState: '',- addressCountry: '',- addressPostcode: '',- addressLat: null,- addressLng: null,+ it('should write an updated entry for an ORM v2 composite field change', async () => {+ await withOrmV2ReadPath(async () => {+ await createRecord({+ objectMetadataSingularName: 'company',+ data: {+ id: COMPOSITE_COMPANY_ID,+ name: 'Composite Field Timeline',},- },- });+ });- const timelineActivities = await findTimelineActivities({- targetCompanyId: { eq: COMPOSITE_COMPANY_ID },- timelineActivityTypeId: {- eq: timelineActivityTypeIdForOrThrow('updated'),- },- });+ const updateStartedAt = Date.now();- expect(timelineActivities).toHaveLength(1);- expect(timelineActivities[0].properties).toMatchObject({- diff: {- address: {- before: {- addressStreet1: '',- addressCity: '',- },- after: {+ await updateRecord({+ objectMetadataSingularName: 'company',+ recordId: COMPOSITE_COMPANY_ID,+ data: {+ address: {addressStreet1: '234 Composite Street',+ addressStreet2: '',addressCity: 'Paris',+ addressState: '',+ addressCountry: '',+ addressPostcode: '',+ addressLat: null,+ addressLng: null,},},- },+ });++ const timelineActivities = await findTimelineActivities({+ targetCompanyId: { eq: COMPOSITE_COMPANY_ID },+ timelineActivityTypeId: {+ eq: timelineActivityTypeIdForOrThrow('updated'),+ },+ });++ expect(timelineActivities).toHaveLength(1);+ expect(+ new Date(timelineActivities[0].happensAt).getTime(),+ ).toBeGreaterThanOrEqual(updateStartedAt);+ expect(timelineActivities[0].properties).toMatchObject({+ diff: {+ address: {+ before: {+ addressStreet1: '',+ addressCity: '',+ },+ after: {+ addressStreet1: '234 Composite Street',+ addressCity: 'Paris',+ },+ },+ },+ });});});