Skip to content

The open alternative to Salesforce, designed for AI.

Fix duplicate key errors during message import (#24154)

Thomas Trompette2ad47430505f
diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-message-folder-association.service.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-message-folder-association.service.ts
index a26988b63be..27b4839b10c 100644
--- a/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-message-folder-association.service.ts
+++ b/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-message-folder-association.service.ts
@@ -6,11 +6,8 @@ import { type WorkspaceEntityManager } from 'src/engine/twenty-orm/entity-manage
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { buildSystemAuthContext } from 'src/engine/twenty-orm/utils/build-system-auth-context.util';
import { type MessageChannelMessageAssociationMessageFolderWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-channel-message-association-message-folder.workspace-entity';
-
-export type MessageChannelMessageAssociationFolderAssociation = {
- messageChannelMessageAssociationId: string;
- messageFolderIds: string[];
-};
+import { type MessageChannelMessageAssociationFolderAssociation } from 'src/modules/messaging/message-import-manager/types/message-channel-message-association-folder-association.type';
+import { buildMessageFolderAssociationsToInsert } from 'src/modules/messaging/message-import-manager/utils/build-message-folder-associations-to-insert.util';
@Injectable()
export class MessagingMessageFolderAssociationService {
@@ -23,7 +20,15 @@ export class MessagingMessageFolderAssociationService {
workspaceId: string,
transactionManager?: WorkspaceEntityManager,
): Promise<void> {
- if (associations.length === 0) {
+ const associationIds = [
+ ...new Set(
+ associations
+ .filter((association) => association.messageFolderIds.length > 0)
+ .map((association) => association.messageChannelMessageAssociationId),
+ ),
+ ];
+
+ if (associationIds.length === 0) {
return;
}
@@ -37,24 +42,6 @@ export class MessagingMessageFolderAssociationService {
'messageChannelMessageAssociationMessageFolder',
);
- const records = associations.flatMap((association) =>
- association.messageFolderIds.map((folderId) => ({
- messageChannelMessageAssociationId:
- association.messageChannelMessageAssociationId,
- messageFolderId: folderId,
- })),
- );
-
- if (records.length === 0) {
- return;
- }
-
- const associationIds = [
- ...new Set(
- records.map((record) => record.messageChannelMessageAssociationId),
- ),
- ];
-
const existingRecords = await repository.find(
{
where: {
@@ -64,19 +51,10 @@ export class MessagingMessageFolderAssociationService {
transactionManager,
);
- const existingKeys = new Set(
- existingRecords.map(
- (record) =>
- `${record.messageChannelMessageAssociationId}:${record.messageFolderId}`,
- ),
- );
-
- const recordsToInsert = records.filter(
- (record) =>
- !existingKeys.has(
- `${record.messageChannelMessageAssociationId}:${record.messageFolderId}`,
- ),
- );
+ const recordsToInsert = buildMessageFolderAssociationsToInsert({
+ associations,
+ existingRecords,
+ });
if (recordsToInsert.length > 0) {
await repository.insert(recordsToInsert, transactionManager);
diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-message.service.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-message.service.ts
index 6de3c0cd26b..74d02cfb457 100644
--- a/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-message.service.ts
+++ b/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-message.service.ts
@@ -137,6 +137,13 @@ export class MessagingMessageService {
messageAccumulatorMap,
);
+ const associationToCreateByMessageId = new Map<
+ string,
+ NonNullable<
+ MessageAccumulator['messageChannelMessageAssociationToCreate']
+ >
+ >();
+
for (const message of messages) {
const messageAccumulator = messageAccumulatorMap.get(
message.externalId,
@@ -183,7 +190,9 @@ export class MessagingMessageService {
messageAccumulator.existingMessageChannelMessageAssociationInDB,
)
) {
- messageAccumulator.messageChannelMessageAssociationToCreate = {
+ const associationToCreate = associationToCreateByMessageId.get(
+ newOrExistingMessageId,
+ ) ?? {
id: v4(),
messageChannelId,
messageId: newOrExistingMessageId,
@@ -191,6 +200,14 @@ export class MessagingMessageService {
messageThreadExternalId: message.messageThreadExternalId,
direction: message.direction,
};
+
+ associationToCreateByMessageId.set(
+ newOrExistingMessageId,
+ associationToCreate,
+ );
+
+ messageAccumulator.messageChannelMessageAssociationToCreate =
+ associationToCreate;
}
messageAccumulatorMap.set(message.externalId, messageAccumulator);
@@ -258,13 +275,8 @@ export class MessagingMessageService {
await messageRepository.insert(messagesToCreate, transactionManager);
const messageChannelMessageAssociationsToCreate = Array.from(
- messageAccumulatorMap.values(),
- )
- .map(
- (accumulator) =>
- accumulator.messageChannelMessageAssociationToCreate,
- )
- .filter(isDefined);
+ associationToCreateByMessageId.values(),
+ );
await messageChannelMessageAssociationRepository.insert(
messageChannelMessageAssociationsToCreate,
diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-save-messages-and-enqueue-contact-creation.service.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-save-messages-and-enqueue-contact-creation.service.ts
index 8cec749d5c0..b5386189cfd 100644
--- a/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-save-messages-and-enqueue-contact-creation.service.ts
+++ b/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-save-messages-and-enqueue-contact-creation.service.ts
@@ -23,11 +23,9 @@ import {
type Participant,
type ParticipantWithMessageId,
} from 'src/modules/messaging/message-import-manager/drivers/gmail/types/gmail-message.type';
-import {
- type MessageChannelMessageAssociationFolderAssociation,
- MessagingMessageFolderAssociationService,
-} from 'src/modules/messaging/message-import-manager/services/messaging-message-folder-association.service';
+import { MessagingMessageFolderAssociationService } from 'src/modules/messaging/message-import-manager/services/messaging-message-folder-association.service';
import { MessagingMessageService } from 'src/modules/messaging/message-import-manager/services/messaging-message.service';
+import { type MessageChannelMessageAssociationFolderAssociation } from 'src/modules/messaging/message-import-manager/types/message-channel-message-association-folder-association.type';
import { type MessageWithParticipants } from 'src/modules/messaging/message-import-manager/types/message';
import { isGroupEmail } from 'src/modules/messaging/message-import-manager/utils/is-group-email';
import { MessagingMessageParticipantService } from 'src/modules/messaging/message-participant-manager/services/messaging-message-participant.service';
diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/types/message-channel-message-association-folder-association.type.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/types/message-channel-message-association-folder-association.type.ts
new file mode 100644
index 00000000000..ad42aba26ee
--- /dev/null
+++ b/packages/twenty-server/src/modules/messaging/message-import-manager/types/message-channel-message-association-folder-association.type.ts
@@ -0,0 +1,4 @@
+export type MessageChannelMessageAssociationFolderAssociation = {
+ messageChannelMessageAssociationId: string;
+ messageFolderIds: string[];
+};
diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/utils/__tests__/build-message-folder-associations-to-insert.util.spec.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/utils/__tests__/build-message-folder-associations-to-insert.util.spec.ts
new file mode 100644
index 00000000000..59529a2a309
--- /dev/null
+++ b/packages/twenty-server/src/modules/messaging/message-import-manager/utils/__tests__/build-message-folder-associations-to-insert.util.spec.ts
@@ -0,0 +1,133 @@
+import { buildMessageFolderAssociationsToInsert } from 'src/modules/messaging/message-import-manager/utils/build-message-folder-associations-to-insert.util';
+
+describe('buildMessageFolderAssociationsToInsert', () => {
+ it('should build one record per association and folder', () => {
+ expect(
+ buildMessageFolderAssociationsToInsert({
+ associations: [
+ {
+ messageChannelMessageAssociationId: 'association-1',
+ messageFolderIds: ['folder-inbox', 'folder-archive'],
+ },
+ ],
+ existingRecords: [],
+ }),
+ ).toEqual([
+ {
+ messageChannelMessageAssociationId: 'association-1',
+ messageFolderId: 'folder-inbox',
+ },
+ {
+ messageChannelMessageAssociationId: 'association-1',
+ messageFolderId: 'folder-archive',
+ },
+ ]);
+ });
+
+ it('should keep a single record when two associations share an association id and a folder', () => {
+ expect(
+ buildMessageFolderAssociationsToInsert({
+ associations: [
+ {
+ messageChannelMessageAssociationId: 'association-1',
+ messageFolderIds: ['folder-inbox'],
+ },
+ {
+ messageChannelMessageAssociationId: 'association-1',
+ messageFolderIds: ['folder-inbox'],
+ },
+ ],
+ existingRecords: [],
+ }),
+ ).toEqual([
+ {
+ messageChannelMessageAssociationId: 'association-1',
+ messageFolderId: 'folder-inbox',
+ },
+ ]);
+ });
+
+ it('should keep a single record when one association repeats a folder id', () => {
+ expect(
+ buildMessageFolderAssociationsToInsert({
+ associations: [
+ {
+ messageChannelMessageAssociationId: 'association-1',
+ messageFolderIds: ['folder-inbox', 'folder-inbox'],
+ },
+ ],
+ existingRecords: [],
+ }),
+ ).toEqual([
+ {
+ messageChannelMessageAssociationId: 'association-1',
+ messageFolderId: 'folder-inbox',
+ },
+ ]);
+ });
+
+ it('should skip records already existing', () => {
+ expect(
+ buildMessageFolderAssociationsToInsert({
+ associations: [
+ {
+ messageChannelMessageAssociationId: 'association-1',
+ messageFolderIds: ['folder-inbox', 'folder-archive'],
+ },
+ ],
+ existingRecords: [
+ {
+ messageChannelMessageAssociationId: 'association-1',
+ messageFolderId: 'folder-inbox',
+ },
+ ],
+ }),
+ ).toEqual([
+ {
+ messageChannelMessageAssociationId: 'association-1',
+ messageFolderId: 'folder-archive',
+ },
+ ]);
+ });
+
+ it('should keep records of different associations sharing a folder', () => {
+ expect(
+ buildMessageFolderAssociationsToInsert({
+ associations: [
+ {
+ messageChannelMessageAssociationId: 'association-1',
+ messageFolderIds: ['folder-inbox'],
+ },
+ {
+ messageChannelMessageAssociationId: 'association-2',
+ messageFolderIds: ['folder-inbox'],
+ },
+ ],
+ existingRecords: [],
+ }),
+ ).toEqual([
+ {
+ messageChannelMessageAssociationId: 'association-1',
+ messageFolderId: 'folder-inbox',
+ },
+ {
+ messageChannelMessageAssociationId: 'association-2',
+ messageFolderId: 'folder-inbox',
+ },
+ ]);
+ });
+
+ it('should return an empty array when no association has folders', () => {
+ expect(
+ buildMessageFolderAssociationsToInsert({
+ associations: [
+ {
+ messageChannelMessageAssociationId: 'association-1',
+ messageFolderIds: [],
+ },
+ ],
+ existingRecords: [],
+ }),
+ ).toEqual([]);
+ });
+});
diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/utils/build-message-folder-associations-to-insert.util.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/utils/build-message-folder-associations-to-insert.util.ts
new file mode 100644
index 00000000000..10e3fc9729c
--- /dev/null
+++ b/packages/twenty-server/src/modules/messaging/message-import-manager/utils/build-message-folder-associations-to-insert.util.ts
@@ -0,0 +1,44 @@
+import { type MessageChannelMessageAssociationFolderAssociation } from 'src/modules/messaging/message-import-manager/types/message-channel-message-association-folder-association.type';
+
+export type MessageFolderAssociationRecord = {
+ messageChannelMessageAssociationId: string;
+ messageFolderId: string;
+};
+
+const buildKey = ({
+ messageChannelMessageAssociationId,
+ messageFolderId,
+}: MessageFolderAssociationRecord) =>
+ `${messageChannelMessageAssociationId}:${messageFolderId}`;
+
+export const buildMessageFolderAssociationsToInsert = ({
+ associations,
+ existingRecords,
+}: {
+ associations: MessageChannelMessageAssociationFolderAssociation[];
+ existingRecords: MessageFolderAssociationRecord[];
+}): MessageFolderAssociationRecord[] => {
+ const keysToSkip = new Set(existingRecords.map(buildKey));
+
+ const recordsToInsert: MessageFolderAssociationRecord[] = [];
+
+ for (const association of associations) {
+ for (const messageFolderId of association.messageFolderIds) {
+ const record = {
+ messageChannelMessageAssociationId:
+ association.messageChannelMessageAssociationId,
+ messageFolderId,
+ };
+ const key = buildKey(record);
+
+ if (keysToSkip.has(key)) {
+ continue;
+ }
+
+ keysToSkip.add(key);
+ recordsToInsert.push(record);
+ }
+ }
+
+ return recordsToInsert;
+};