The open alternative to Salesforce, designed for AI.
TypeScript crmcrm-systemcustomergood-first-issuegraphqlhacktoberfestjavascriptmarketingmonoreponestjsopen-sourcepostgresqlreactreactjssalestypescriptweb
fix(messaging): stop group and bulk emails from creating contacts (#23137)
neo7730489d39f2b9f
diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/parse-and-format-gmail-message.util.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/parse-and-format-gmail-message.util.tsindex ff45f68336c..fec60205f9d 100644--- a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/parse-and-format-gmail-message.util.ts+++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/parse-and-format-gmail-message.util.ts@@ -31,6 +31,7 @@ export const parseAndFormatGmailMessage = (attachments,deliveredTo,labelIds,+ messageHeaders,} = parseGmailMessage(message);const isDraft = (labelIds ?? []).includes('DRAFT');@@ -86,5 +87,6 @@ export const parseAndFormatGmailMessage = (messageFolderExternalIds: labelIds,labelIds,isDraft,+ messageHeaders,};};diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/parse-gmail-message.util.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/parse-gmail-message.util.tsindex 486ec9dd6a3..029cb1e7984 100644--- a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/parse-gmail-message.util.ts+++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/parse-gmail-message.util.ts@@ -1,6 +1,7 @@import assert from 'assert';import { type gmail_v1 } from 'googleapis';+import { isDefined } from 'twenty-shared/utils';import { getAttachmentData } from 'src/modules/messaging/message-import-manager/drivers/gmail/utils/get-attachment-data.util';import { getBodyData } from 'src/modules/messaging/message-import-manager/drivers/gmail/utils/get-body-data.util';@@ -22,6 +23,10 @@ export const parseGmailMessage = (message: gmail_v1.Schema$Message) => {const historyId = message.historyId;const internalDate = message.internalDate;const labelIds = message.labelIds ?? [];+ const messageHeaders = (message.payload?.headers ?? []).flatMap(+ ({ name, value }) =>+ isDefined(name) && isDefined(value) ? [{ name, value }] : [],+ );assert(id, 'ID is missing');assert(historyId, 'History-ID is missing');@@ -54,5 +59,6 @@ export const parseGmailMessage = (message: gmail_v1.Schema$Message) => {isHtml,attachments,labelIds,+ messageHeaders,};};diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/imap/services/imap-get-messages.service.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/imap/services/imap-get-messages.service.tsindex c8138e2b59d..6720745c12f 100644--- a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/imap/services/imap-get-messages.service.ts+++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/imap/services/imap-get-messages.service.ts@@ -187,6 +187,10 @@ export class ImapGetMessagesService {participants: extractParticipantsFromParsedEmail(parsed),messageFolderExternalIds: [folderExternalId],isDraft: flags?.has('\\Draft') ?? false,+ messageHeaders: parsed.headers.map(({ key, value }) => ({+ name: key,+ value,+ })),};}}diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/inbound-email/services/inbound-email-parser.service.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/inbound-email/services/inbound-email-parser.service.tsindex 02e48987018..2e2b2a8f8e1 100644--- a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/inbound-email/services/inbound-email-parser.service.ts+++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/inbound-email/services/inbound-email-parser.service.ts@@ -36,6 +36,10 @@ export class InboundEmailParserService {attachments: [],participants: extractParticipantsFromParsedEmail(parsedEmail),isDraft: false,+ messageHeaders: parsedEmail.headers.map(({ key, value }) => ({+ name: key,+ value,+ })),};}}diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-fetch-by-batch.service.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-fetch-by-batch.service.tsindex 475e95b33eb..77929e65340 100644--- a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-fetch-by-batch.service.ts+++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-fetch-by-batch.service.ts@@ -17,6 +17,23 @@ export class MicrosoftFetchByBatchService {messageIdsByBatch: string[][];batchResponses: MicrosoftGraphBatchResponse[];}> {+ const selectedFields = [+ 'id',+ 'subject',+ 'body',+ 'receivedDateTime',+ 'internetMessageId',+ 'internetMessageHeaders',+ 'conversationId',+ 'parentFolderId',+ 'isDraft',+ 'from',+ 'replyTo',+ 'toRecipients',+ 'ccRecipients',+ 'bccRecipients',+ ].join(',');+const batchLimit = 20;const batchResponses: MicrosoftGraphBatchResponse[] = [];const messageIdsByBatch: string[][] = [];@@ -33,7 +50,7 @@ export class MicrosoftFetchByBatchService {const batchRequests = batchMessageIds.map((messageId, index) => ({id: (index + 1).toString(),method: 'GET',- url: `/me/messages/${messageId}`,+ url: `/me/messages/${messageId}?$select=${selectedFields}`,headers: {'Content-Type': 'application/json',Prefer: 'outlook.body-content-type="text"',diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-messages.interface.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-messages.interface.tsindex 4c4abaa9ffc..6215ccdf04a 100644--- a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-messages.interface.ts+++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-messages.interface.ts@@ -19,6 +19,7 @@ export interface MicrosoftGraphBatchResponse {sentDateTime?: string;hasAttachments?: boolean;internetMessageId?: string;+ internetMessageHeaders?: { name: string; value: string }[];subject?: string;bodyPreview?: string;importance?: string;diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-messages.service.spec.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-messages.service.spec.tsindex 426ccaccefe..4182136620d 100644--- a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-messages.service.spec.ts+++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-messages.service.spec.ts@@ -104,6 +104,7 @@ describe('Microsoft get messages service', () => {messageFolderExternalIds: responseExample1.body.parentFolderId? [responseExample1.body.parentFolderId]: [],+ messageHeaders: [],});const responseExample2 =@@ -151,6 +152,7 @@ describe('Microsoft get messages service', () => {messageFolderExternalIds: responseExample2.body.parentFolderId? [responseExample2.body.parentFolderId]: [],+ messageHeaders: [],});});@@ -195,6 +197,7 @@ describe('Microsoft get messages service', () => {messageFolderExternalIds: responseExample.body.parentFolderId? [responseExample.body.parentFolderId]: [],+ messageHeaders: [],});});});diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-messages.service.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-messages.service.tsindex ff67a241e06..d694def53e6 100644--- a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-messages.service.ts+++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-messages.service.ts@@ -164,6 +164,7 @@ export class MicrosoftGetMessagesService {? [response.parentFolderId]: [],isDraft: response.isDraft ?? false,+ messageHeaders: response.internetMessageHeaders ?? [],};});diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-save-messages-and-enqueue-contact-creation.service.spec.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-save-messages-and-enqueue-contact-creation.service.spec.tsindex 71bf3a6f6d3..3b4f9294bdf 100644--- a/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-save-messages-and-enqueue-contact-creation.service.spec.ts+++ b/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-save-messages-and-enqueue-contact-creation.service.spec.ts@@ -305,6 +305,54 @@ describe('MessagingSaveMessagesAndEnqueueContactCreationService', () => {},);});+ it('should not create contacts for non-sender group email participants when excludeGroupEmails is enabled', async () => {+ await service.saveMessagesAndEnqueueContactCreation(+ [+ {+ ...mockMessages[1],+ participants: [+ {+ role: MessageParticipantRole.FROM,+ handle: '[email protected]',+ displayName: 'Tim',+ },+ {+ role: MessageParticipantRole.REPLY_TO,+ handle: '[email protected]',+ displayName: 'Instagram',+ },+ {+ role: MessageParticipantRole.CC,+ handle: '[email protected]',+ displayName: 'Support',+ },+ ],+ },+ ],+ mockMessageChannel,+ mockConnectedAccount,+ workspaceId,+ );++ expect(messageQueueService.add).toHaveBeenCalledWith(+ CreateCompanyAndContactJob.name,+ {+ workspaceId,+ connectedAccount: mockConnectedAccount,+ source: FieldActorSource.EMAIL,+ contactsToCreate: [+ {+ handle: '[email protected]',+ displayName: 'Tim',+ role: MessageParticipantRole.FROM,+ shouldCreateContact: true,+ messageId: 'db-message-id-2',+ },+ ],+ },+ );+ });+it('should not create contact if the participant is the connected account', async () => {const mockMessagesWithConnectedAccount = [{diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-save-messages-and-enqueue-contact-creation.service.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-save-messages-and-enqueue-contact-creation.service.tsindex cd0e4bd8767..2eb120ee567 100644--- a/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-save-messages-and-enqueue-contact-creation.service.ts+++ b/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-save-messages-and-enqueue-contact-creation.service.ts@@ -29,6 +29,7 @@ import {} from 'src/modules/messaging/message-import-manager/services/messaging-message-folder-association.service';import { MessagingMessageService } from 'src/modules/messaging/message-import-manager/services/messaging-message.service';import { type MessageWithParticipants } from 'src/modules/messaging/message-import-manager/types/message';+import { isGroupEmail } from 'src/modules/messaging/message-import-manager/utils/is-group-email';import { MessagingMessageParticipantService } from 'src/modules/messaging/message-participant-manager/services/messaging-message-participant.service';import { isWorkEmail } from 'src/utils/is-work-email';@@ -103,6 +104,10 @@ export class MessagingSaveMessagesAndEnqueueContactCreationService {messageChannel.excludeNonProfessionalEmails &&!isWorkEmail(participant.handle);+ const isExcludedByGroupEmails =+ messageChannel.excludeGroupEmails &&+ isGroupEmail(participant.handle);+// Drafts are outgoing, so don't turn recipients of an// unsent email into CRM contacts.const shouldCreateContact =@@ -110,6 +115,7 @@ export class MessagingSaveMessagesAndEnqueueContactCreationService {!!participant.handle &&!isParticipantConnectedAccount &&!isExcludedByNonProfessionalEmails &&+ !isExcludedByGroupEmails &&(messageChannel.contactAutoCreationPolicy ===MessageChannelContactAutoCreationPolicy.SENT_AND_RECEIVED ||(messageChannel.contactAutoCreationPolicy ===diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/types/message.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/types/message.tsindex 044c1606b1a..d20232b6254 100644--- a/packages/twenty-server/src/modules/messaging/message-import-manager/types/message.ts+++ b/packages/twenty-server/src/modules/messaging/message-import-manager/types/message.ts@@ -26,6 +26,12 @@ export type Message = Omit<messageFolderIds?: string[];messageFolderExternalIds?: string[];labelIds?: string[];+ messageHeaders?: MessageHeader[];+};++export type MessageHeader = {+ name: string;+ value: string;};export type MessageAttachment = {diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/utils/__tests__/filter-emails.util.spec.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/utils/__tests__/filter-emails.util.spec.tsindex bb7c941e1b7..dc5fa58aeac 100644--- a/packages/twenty-server/src/modules/messaging/message-import-manager/utils/__tests__/filter-emails.util.spec.ts+++ b/packages/twenty-server/src/modules/messaging/message-import-manager/utils/__tests__/filter-emails.util.spec.ts@@ -139,6 +139,38 @@ describe('filterEmails', () => {expect(result[0].externalId).toBe('regular-message');});+ it('should filter out bulk mail whose sender does not look like a group address', () => {+ const primaryHandle = '[email protected]';+ const messages: MessageWithParticipants[] = [+ {+ externalId: 'newsletter-message',+ subject: 'Your weekly recap',+ receivedAt: new Date('2025-01-09T09:54:37.000Z'),+ text: 'Recap',+ headerMessageId: '<[email protected]>',+ messageThreadExternalId: 'thread-1',+ direction: MessageDirection.INCOMING,+ participants: [+ {+ role: MessageParticipantRole.FROM,+ handle: '[email protected]',+ displayName: 'Instagram',+ },+ ],+ attachments: [],+ isDraft: false,+ messageHeaders: [+ { name: 'List-Unsubscribe', value: '<https://instagram.com/unsub>' },+ ],+ },+ ];++ expect(filterEmails(primaryHandle, [], messages, [])).toEqual([]);+ expect(filterEmails(primaryHandle, [], messages, [], false)).toEqual(+ messages,+ );+ });+it('should not filter out group emails when excludeGroupEmails is false', () => {const primaryHandle = '[email protected]';const messages: MessageWithParticipants[] = [diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/utils/__tests__/is-bulk-mail.util.spec.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/utils/__tests__/is-bulk-mail.util.spec.tsnew file mode 100644index 00000000000..2c11c4927c0--- /dev/null+++ b/packages/twenty-server/src/modules/messaging/message-import-manager/utils/__tests__/is-bulk-mail.util.spec.ts@@ -0,0 +1,50 @@+import { isBulkMail } from 'src/modules/messaging/message-import-manager/utils/is-bulk-mail.util';++describe('isBulkMail', () => {+ it('should detect a List-Unsubscribe header regardless of casing', () => {+ expect(+ isBulkMail([+ { name: 'List-Unsubscribe', value: '<https://example.com/unsub>' },+ ]),+ ).toBe(true);++ expect(+ isBulkMail([+ { name: 'list-unsubscribe', value: '<mailto:[email protected]>' },+ ]),+ ).toBe(true);+ });++ it('should detect a List-Id header', () => {+ expect(+ isBulkMail([{ name: 'List-Id', value: '<newsletter.example.com>' }]),+ ).toBe(true);+ });++ it('should detect bulk precedence values only', () => {+ expect(isBulkMail([{ name: 'Precedence', value: 'bulk' }])).toBe(true);+ expect(isBulkMail([{ name: 'Precedence', value: ' List ' }])).toBe(true);+ expect(isBulkMail([{ name: 'Precedence', value: 'urgent' }])).toBe(false);+ });++ it('should treat Auto-Submitted as bulk unless it is "no"', () => {+ expect(+ isBulkMail([{ name: 'Auto-Submitted', value: 'auto-generated' }]),+ ).toBe(true);+ expect(isBulkMail([{ name: 'Auto-Submitted', value: 'no' }])).toBe(false);+ });++ it('should not flag a regular message', () => {+ expect(+ isBulkMail([+ { name: 'From', value: '[email protected]' },+ { name: 'Subject', value: 'Lunch' },+ { name: 'List-Unsubscribe', value: '' },+ ]),+ ).toBe(false);+ });++ it('should not flag a message without headers', () => {+ expect(isBulkMail([])).toBe(false);+ });+});diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/utils/filter-emails.util.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/utils/filter-emails.util.tsindex 5ab28dba91f..c6bae229bd7 100644--- a/packages/twenty-server/src/modules/messaging/message-import-manager/utils/filter-emails.util.ts+++ b/packages/twenty-server/src/modules/messaging/message-import-manager/utils/filter-emails.util.ts@@ -5,6 +5,7 @@ import { type MessageWithParticipants } from 'src/modules/messaging/message-impoimport { filterOutBlocklistedMessages } from 'src/modules/messaging/message-import-manager/utils/filter-out-blocklisted-messages.util';import { filterOutIcsAttachments } from 'src/modules/messaging/message-import-manager/utils/filter-out-ics-attachments.util';import { filterOutInternals } from 'src/modules/messaging/message-import-manager/utils/filter-out-internals.util';+import { isBulkMail } from 'src/modules/messaging/message-import-manager/utils/is-bulk-mail.util';import { isGroupEmail } from 'src/modules/messaging/message-import-manager/utils/is-group-email';import { isMessageSenderMatchingHandles } from 'src/modules/messaging/message-import-manager/utils/is-message-sender-matching-handles.util';import { isWorkEmail } from 'src/utils/is-work-email';@@ -45,6 +46,10 @@ export const filterEmails = (return true;}+ if (isBulkMail(message.messageHeaders ?? [])) {+ return false;+ }+const senderHandle = message.participants?.find((participant) => participant.role === MessageParticipantRole.FROM,)?.handle;diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/utils/is-bulk-mail.util.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/utils/is-bulk-mail.util.tsnew file mode 100644index 00000000000..e11545a96d4--- /dev/null+++ b/packages/twenty-server/src/modules/messaging/message-import-manager/utils/is-bulk-mail.util.ts@@ -0,0 +1,26 @@+import { isNonEmptyString } from '@sniptt/guards';++import { type MessageHeader } from 'src/modules/messaging/message-import-manager/types/message';++const BULK_LIST_HEADER_NAMES = ['list-unsubscribe', 'list-id'];+const BULK_PRECEDENCE_VALUES = ['bulk', 'list', 'junk'];++export const isBulkMail = (headers: MessageHeader[]): boolean =>+ headers.some(({ name, value }) => {+ const headerName = name.toLowerCase();+ const headerValue = value.trim().toLowerCase();++ if (BULK_LIST_HEADER_NAMES.includes(headerName)) {+ return isNonEmptyString(headerValue);+ }++ if (headerName === 'precedence') {+ return BULK_PRECEDENCE_VALUES.includes(headerValue);+ }++ if (headerName === 'auto-submitted') {+ return isNonEmptyString(headerValue) && headerValue !== 'no';+ }++ return false;+ });