The open alternative to Salesforce, designed for AI.
TypeScript crmcrm-systemcustomergood-first-issuegraphqlhacktoberfestjavascriptmarketingmonoreponestjsopen-sourcepostgresqlreactreactjssalestypescriptweb
Fix duplicate key errors during message import (#24154)
Thomas Trompette2ad47430505f
diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-message-folder-association.service.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-message-folder-association.service.tsindex a26988b63be..27b4839b10c 100644--- a/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-message-folder-association.service.ts+++ b/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-message-folder-association.service.ts@@ -6,11 +6,8 @@ import { type WorkspaceEntityManager } from 'src/engine/twenty-orm/entity-manageimport { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';import { buildSystemAuthContext } from 'src/engine/twenty-orm/utils/build-system-auth-context.util';import { type MessageChannelMessageAssociationMessageFolderWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-channel-message-association-message-folder.workspace-entity';--export type MessageChannelMessageAssociationFolderAssociation = {- messageChannelMessageAssociationId: string;- messageFolderIds: string[];-};+import { type MessageChannelMessageAssociationFolderAssociation } from 'src/modules/messaging/message-import-manager/types/message-channel-message-association-folder-association.type';+import { buildMessageFolderAssociationsToInsert } from 'src/modules/messaging/message-import-manager/utils/build-message-folder-associations-to-insert.util';@Injectable()export class MessagingMessageFolderAssociationService {@@ -23,7 +20,15 @@ export class MessagingMessageFolderAssociationService {workspaceId: string,transactionManager?: WorkspaceEntityManager,): Promise<void> {- if (associations.length === 0) {+ const associationIds = [+ ...new Set(+ associations+ .filter((association) => association.messageFolderIds.length > 0)+ .map((association) => association.messageChannelMessageAssociationId),+ ),+ ];++ if (associationIds.length === 0) {return;}@@ -37,24 +42,6 @@ export class MessagingMessageFolderAssociationService {'messageChannelMessageAssociationMessageFolder',);- const records = associations.flatMap((association) =>- association.messageFolderIds.map((folderId) => ({- messageChannelMessageAssociationId:- association.messageChannelMessageAssociationId,- messageFolderId: folderId,- })),- );-- if (records.length === 0) {- return;- }-- const associationIds = [- ...new Set(- records.map((record) => record.messageChannelMessageAssociationId),- ),- ];-const existingRecords = await repository.find({where: {@@ -64,19 +51,10 @@ export class MessagingMessageFolderAssociationService {transactionManager,);- const existingKeys = new Set(- existingRecords.map(- (record) =>- `${record.messageChannelMessageAssociationId}:${record.messageFolderId}`,- ),- );-- const recordsToInsert = records.filter(- (record) =>- !existingKeys.has(- `${record.messageChannelMessageAssociationId}:${record.messageFolderId}`,- ),- );+ const recordsToInsert = buildMessageFolderAssociationsToInsert({+ associations,+ existingRecords,+ });if (recordsToInsert.length > 0) {await repository.insert(recordsToInsert, transactionManager);diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-message.service.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-message.service.tsindex 6de3c0cd26b..74d02cfb457 100644--- a/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-message.service.ts+++ b/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-message.service.ts@@ -137,6 +137,13 @@ export class MessagingMessageService {messageAccumulatorMap,);+ const associationToCreateByMessageId = new Map<+ string,+ NonNullable<+ MessageAccumulator['messageChannelMessageAssociationToCreate']+ >+ >();+for (const message of messages) {const messageAccumulator = messageAccumulatorMap.get(message.externalId,@@ -183,7 +190,9 @@ export class MessagingMessageService {messageAccumulator.existingMessageChannelMessageAssociationInDB,)) {- messageAccumulator.messageChannelMessageAssociationToCreate = {+ const associationToCreate = associationToCreateByMessageId.get(+ newOrExistingMessageId,+ ) ?? {id: v4(),messageChannelId,messageId: newOrExistingMessageId,@@ -191,6 +200,14 @@ export class MessagingMessageService {messageThreadExternalId: message.messageThreadExternalId,direction: message.direction,};++ associationToCreateByMessageId.set(+ newOrExistingMessageId,+ associationToCreate,+ );++ messageAccumulator.messageChannelMessageAssociationToCreate =+ associationToCreate;}messageAccumulatorMap.set(message.externalId, messageAccumulator);@@ -258,13 +275,8 @@ export class MessagingMessageService {await messageRepository.insert(messagesToCreate, transactionManager);const messageChannelMessageAssociationsToCreate = Array.from(- messageAccumulatorMap.values(),- )- .map(- (accumulator) =>- accumulator.messageChannelMessageAssociationToCreate,- )- .filter(isDefined);+ associationToCreateByMessageId.values(),+ );await messageChannelMessageAssociationRepository.insert(messageChannelMessageAssociationsToCreate,diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-save-messages-and-enqueue-contact-creation.service.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-save-messages-and-enqueue-contact-creation.service.tsindex 8cec749d5c0..b5386189cfd 100644--- a/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-save-messages-and-enqueue-contact-creation.service.ts+++ b/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-save-messages-and-enqueue-contact-creation.service.ts@@ -23,11 +23,9 @@ import {type Participant,type ParticipantWithMessageId,} from 'src/modules/messaging/message-import-manager/drivers/gmail/types/gmail-message.type';-import {- type MessageChannelMessageAssociationFolderAssociation,- MessagingMessageFolderAssociationService,-} from 'src/modules/messaging/message-import-manager/services/messaging-message-folder-association.service';+import { MessagingMessageFolderAssociationService } from 'src/modules/messaging/message-import-manager/services/messaging-message-folder-association.service';import { MessagingMessageService } from 'src/modules/messaging/message-import-manager/services/messaging-message.service';+import { type MessageChannelMessageAssociationFolderAssociation } from 'src/modules/messaging/message-import-manager/types/message-channel-message-association-folder-association.type';import { type MessageWithParticipants } from 'src/modules/messaging/message-import-manager/types/message';import { isGroupEmail } from 'src/modules/messaging/message-import-manager/utils/is-group-email';import { MessagingMessageParticipantService } from 'src/modules/messaging/message-participant-manager/services/messaging-message-participant.service';diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/types/message-channel-message-association-folder-association.type.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/types/message-channel-message-association-folder-association.type.tsnew file mode 100644index 00000000000..ad42aba26ee--- /dev/null+++ b/packages/twenty-server/src/modules/messaging/message-import-manager/types/message-channel-message-association-folder-association.type.ts@@ -0,0 +1,4 @@+export type MessageChannelMessageAssociationFolderAssociation = {+ messageChannelMessageAssociationId: string;+ messageFolderIds: string[];+};diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/utils/__tests__/build-message-folder-associations-to-insert.util.spec.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/utils/__tests__/build-message-folder-associations-to-insert.util.spec.tsnew file mode 100644index 00000000000..59529a2a309--- /dev/null+++ b/packages/twenty-server/src/modules/messaging/message-import-manager/utils/__tests__/build-message-folder-associations-to-insert.util.spec.ts@@ -0,0 +1,133 @@+import { buildMessageFolderAssociationsToInsert } from 'src/modules/messaging/message-import-manager/utils/build-message-folder-associations-to-insert.util';++describe('buildMessageFolderAssociationsToInsert', () => {+ it('should build one record per association and folder', () => {+ expect(+ buildMessageFolderAssociationsToInsert({+ associations: [+ {+ messageChannelMessageAssociationId: 'association-1',+ messageFolderIds: ['folder-inbox', 'folder-archive'],+ },+ ],+ existingRecords: [],+ }),+ ).toEqual([+ {+ messageChannelMessageAssociationId: 'association-1',+ messageFolderId: 'folder-inbox',+ },+ {+ messageChannelMessageAssociationId: 'association-1',+ messageFolderId: 'folder-archive',+ },+ ]);+ });++ it('should keep a single record when two associations share an association id and a folder', () => {+ expect(+ buildMessageFolderAssociationsToInsert({+ associations: [+ {+ messageChannelMessageAssociationId: 'association-1',+ messageFolderIds: ['folder-inbox'],+ },+ {+ messageChannelMessageAssociationId: 'association-1',+ messageFolderIds: ['folder-inbox'],+ },+ ],+ existingRecords: [],+ }),+ ).toEqual([+ {+ messageChannelMessageAssociationId: 'association-1',+ messageFolderId: 'folder-inbox',+ },+ ]);+ });++ it('should keep a single record when one association repeats a folder id', () => {+ expect(+ buildMessageFolderAssociationsToInsert({+ associations: [+ {+ messageChannelMessageAssociationId: 'association-1',+ messageFolderIds: ['folder-inbox', 'folder-inbox'],+ },+ ],+ existingRecords: [],+ }),+ ).toEqual([+ {+ messageChannelMessageAssociationId: 'association-1',+ messageFolderId: 'folder-inbox',+ },+ ]);+ });++ it('should skip records already existing', () => {+ expect(+ buildMessageFolderAssociationsToInsert({+ associations: [+ {+ messageChannelMessageAssociationId: 'association-1',+ messageFolderIds: ['folder-inbox', 'folder-archive'],+ },+ ],+ existingRecords: [+ {+ messageChannelMessageAssociationId: 'association-1',+ messageFolderId: 'folder-inbox',+ },+ ],+ }),+ ).toEqual([+ {+ messageChannelMessageAssociationId: 'association-1',+ messageFolderId: 'folder-archive',+ },+ ]);+ });++ it('should keep records of different associations sharing a folder', () => {+ expect(+ buildMessageFolderAssociationsToInsert({+ associations: [+ {+ messageChannelMessageAssociationId: 'association-1',+ messageFolderIds: ['folder-inbox'],+ },+ {+ messageChannelMessageAssociationId: 'association-2',+ messageFolderIds: ['folder-inbox'],+ },+ ],+ existingRecords: [],+ }),+ ).toEqual([+ {+ messageChannelMessageAssociationId: 'association-1',+ messageFolderId: 'folder-inbox',+ },+ {+ messageChannelMessageAssociationId: 'association-2',+ messageFolderId: 'folder-inbox',+ },+ ]);+ });++ it('should return an empty array when no association has folders', () => {+ expect(+ buildMessageFolderAssociationsToInsert({+ associations: [+ {+ messageChannelMessageAssociationId: 'association-1',+ messageFolderIds: [],+ },+ ],+ existingRecords: [],+ }),+ ).toEqual([]);+ });+});diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/utils/build-message-folder-associations-to-insert.util.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/utils/build-message-folder-associations-to-insert.util.tsnew file mode 100644index 00000000000..10e3fc9729c--- /dev/null+++ b/packages/twenty-server/src/modules/messaging/message-import-manager/utils/build-message-folder-associations-to-insert.util.ts@@ -0,0 +1,44 @@+import { type MessageChannelMessageAssociationFolderAssociation } from 'src/modules/messaging/message-import-manager/types/message-channel-message-association-folder-association.type';++export type MessageFolderAssociationRecord = {+ messageChannelMessageAssociationId: string;+ messageFolderId: string;+};++const buildKey = ({+ messageChannelMessageAssociationId,+ messageFolderId,+}: MessageFolderAssociationRecord) =>+ `${messageChannelMessageAssociationId}:${messageFolderId}`;++export const buildMessageFolderAssociationsToInsert = ({+ associations,+ existingRecords,+}: {+ associations: MessageChannelMessageAssociationFolderAssociation[];+ existingRecords: MessageFolderAssociationRecord[];+}): MessageFolderAssociationRecord[] => {+ const keysToSkip = new Set(existingRecords.map(buildKey));++ const recordsToInsert: MessageFolderAssociationRecord[] = [];++ for (const association of associations) {+ for (const messageFolderId of association.messageFolderIds) {+ const record = {+ messageChannelMessageAssociationId:+ association.messageChannelMessageAssociationId,+ messageFolderId,+ };+ const key = buildKey(record);++ if (keysToSkip.has(key)) {+ continue;+ }++ keysToSkip.add(key);+ recordsToInsert.push(record);+ }+ }++ return recordsToInsert;+};