The open alternative to Salesforce, designed for AI.
TypeScript crmcrm-systemcustomergood-first-issuegraphqlhacktoberfestjavascriptmarketingmonoreponestjsopen-sourcepostgresqlreactreactjssalestypescriptweb
fix(messaging): fix reply-quotation stripping that emptied email bodies (#21118)
neo773437eed0862f4
diff --git a/packages/twenty-server/jest.config.mjs b/packages/twenty-server/jest.config.mjsindex 1a39609770b..a845969ef33 100644--- a/packages/twenty-server/jest.config.mjs+++ b/packages/twenty-server/jest.config.mjs@@ -17,7 +17,7 @@ const jestConfig = {testEnvironment: 'node',setupFilesAfterEnv: ['./setupTests.ts'],transformIgnorePatterns: [- '/node_modules/(?!(file-type|@file-type|strtok3|token-types|@borewit|@tokenizer|uint8array-extras|read-next-line|digest-fetch|md5|js-sha256|js-sha512|base-64|charenc|crypt)/)',+ '/node_modules/(?!(file-type|@file-type|strtok3|token-types|@borewit|@tokenizer|uint8array-extras|read-next-line|digest-fetch|md5|js-sha256|js-sha512|base-64|charenc|crypt|email-reply-parser)/)',],testRegex: '.*\\.spec\\.ts$',transform: {diff --git a/packages/twenty-server/package.json b/packages/twenty-server/package.jsonindex 15bac9a4ca4..ebdf40bf807 100644--- a/packages/twenty-server/package.json+++ b/packages/twenty-server/package.json@@ -97,6 +97,7 @@"digest-fetch": "^3.1.1","dompurify": "3.3.3","dotenv": "16.4.5",+ "email-reply-parser": "^2.3.5","express": "4.22.1","express-session": "^1.18.2","file-type": "^21.3.1",diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/__tests__/parse-and-format-gmail-message.util.spec.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/__tests__/parse-and-format-gmail-message.util.spec.tsindex c3c3cbfb57f..f269acd9722 100644--- a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/__tests__/parse-and-format-gmail-message.util.spec.ts+++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/__tests__/parse-and-format-gmail-message.util.spec.ts@@ -99,6 +99,39 @@ describe('parseAndFormatGmailMessage', () => {expect(result?.direction).toBe(MessageDirection.OUTGOING);});+ it('should keep the body of an entirely-quoted forwarded message instead of emptying it', () => {+ // Regression: planer stripped the whole forwarded body, persisting text=''.+ const forwardedBody =+ '> quoted line one\n> quoted line two\n> quoted line three';++ const result = parseAndFormatGmailMessage(+ buildMessage(+ [+ { name: 'From', value: '[email protected]' },+ { name: 'To', value: '[email protected]' },+ { name: 'Message-ID', value: '<[email protected]>' },+ ],+ {+ payload: {+ headers: [+ { name: 'From', value: '[email protected]' },+ { name: 'To', value: '[email protected]' },+ { name: 'Message-ID', value: '<[email protected]>' },+ ],+ mimeType: 'text/plain',+ body: {+ data: Buffer.from(forwardedBody).toString('base64'),+ size: forwardedBody.length,+ },+ },+ },+ ),+ connectedAccount,+ );++ expect(result?.text).toBe(forwardedBody);+ });+it('should return null when required headers (`From`, `Message-ID`) are missing', () => {const result = parseAndFormatGmailMessage(buildMessage([{ name: 'To', value: '[email protected]' }]),diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/parse-and-format-gmail-message.util.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/parse-and-format-gmail-message.util.tsindex f9619bbeac6..367cab14ed2 100644--- a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/parse-and-format-gmail-message.util.ts+++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/parse-and-format-gmail-message.util.ts@@ -1,5 +1,4 @@import { type gmail_v1 as gmailV1 } from 'googleapis';-import planer from 'planer';import { MessageParticipantRole } from 'twenty-shared/types';import { isNonEmptyString } from '@sniptt/guards';import { isDefined, isNonEmptyArray } from 'twenty-shared/utils';@@ -8,8 +7,8 @@ import { type ConnectedAccountEntity } from 'src/engine/metadata-modules/connectimport { computeMessageDirection } from 'src/modules/messaging/message-import-manager/drivers/gmail/utils/compute-message-direction.util';import { parseGmailMessage } from 'src/modules/messaging/message-import-manager/drivers/gmail/utils/parse-gmail-message.util';import { type MessageWithParticipants } from 'src/modules/messaging/message-import-manager/types/message';+import { extractMessageBodyText } from 'src/modules/messaging/message-import-manager/utils/extract-message-body-text.util';import { formatAddressObjectAsParticipants } from 'src/modules/messaging/message-import-manager/utils/format-address-object-as-participants.util';-import { sanitizeString } from 'src/modules/messaging/message-import-manager/utils/sanitize-string.util';export const parseAndFormatGmailMessage = (message: gmailV1.Schema$Message,@@ -25,7 +24,8 @@ export const parseAndFormatGmailMessage = (cc,bcc,headerMessageId,- text,+ body,+ isHtml,attachments,deliveredTo,labelIds,@@ -59,10 +59,6 @@ export const parseAndFormatGmailMessage = (return null;}- const textWithoutReplyQuotations = text- ? planer.extractFrom(text, 'text/plain')- : '';-return {externalId: id,headerMessageId,@@ -71,7 +67,7 @@ export const parseAndFormatGmailMessage = (receivedAt: new Date(parseInt(internalDate)),direction: computeMessageDirection(from.address || '', connectedAccount),participants,- text: sanitizeString(textWithoutReplyQuotations),+ text: extractMessageBodyText(isHtml ? { html: body } : { text: body }),attachments,messageFolderExternalIds: labelIds,labelIds,diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/parse-gmail-message.util.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/parse-gmail-message.util.tsindex 1a2f61f5be3..e8f178f7c50 100644--- a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/parse-gmail-message.util.ts+++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/parse-gmail-message.util.ts@@ -5,7 +5,6 @@ import { type gmail_v1 } from 'googleapis';import { getAttachmentData } from 'src/modules/messaging/message-import-manager/drivers/gmail/utils/get-attachment-data.util';import { getBodyData } from 'src/modules/messaging/message-import-manager/drivers/gmail/utils/get-body-data.util';import { getPropertyFromHeaders } from 'src/modules/messaging/message-import-manager/drivers/gmail/utils/get-property-from-headers.util';-import { createHtmlToTextConverter } from 'src/modules/messaging/message-import-manager/utils/create-html-to-text-converter.util';import { safeParseEmailAddressAddress } from 'src/modules/messaging/message-import-manager/utils/safe-parse-email-address-address.util';import { safeParseEmailAddresses } from 'src/modules/messaging/message-import-manager/utils/safe-parse-email-addresses.util';@@ -31,10 +30,7 @@ export const parseGmailMessage = (message: gmail_v1.Schema$Message) => {const decodedBody = bodyResult? Buffer.from(bodyResult.data, 'base64').toString(): '';-- const text = bodyResult?.isHtml- ? createHtmlToTextConverter()(decodedBody)- : decodedBody;+ const isHtml = bodyResult?.isHtml ?? false;const attachments = getAttachmentData(message);@@ -52,7 +48,8 @@ export const parseGmailMessage = (message: gmail_v1.Schema$Message) => {to: rawTo ? safeParseEmailAddresses(rawTo) : [],cc: rawCc ? safeParseEmailAddresses(rawCc) : [],bcc: rawBcc ? safeParseEmailAddresses(rawBcc) : [],- text,+ body: decodedBody,+ isHtml,attachments,labelIds,};diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/imap/messaging-imap-driver.module.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/imap/messaging-imap-driver.module.tsindex 935434996bf..5c31e37d845 100644--- a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/imap/messaging-imap-driver.module.ts+++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/imap/messaging-imap-driver.module.ts@@ -19,7 +19,6 @@ import { ImapGetMessageListService } from 'src/modules/messaging/message-import-import { ImapGetMessagesService } from 'src/modules/messaging/message-import-manager/drivers/imap/services/imap-get-messages.service';import { ImapMessageListFetchErrorHandler } from 'src/modules/messaging/message-import-manager/drivers/imap/services/imap-message-list-fetch-error-handler.service';import { ImapMessageParserService } from 'src/modules/messaging/message-import-manager/drivers/imap/services/imap-message-parser.service';-import { ImapMessageTextExtractorService } from 'src/modules/messaging/message-import-manager/drivers/imap/services/imap-message-text-extractor.service';import { ImapMessagesImportErrorHandler } from 'src/modules/messaging/message-import-manager/drivers/imap/services/imap-messages-import-error-handler.service';import { ImapSyncService } from 'src/modules/messaging/message-import-manager/drivers/imap/services/imap-sync.service';import { MessageParticipantManagerModule } from 'src/modules/messaging/message-participant-manager/message-participant-manager.module';@@ -47,7 +46,6 @@ import { MessageParticipantManagerModule } from 'src/modules/messaging/message-pImapMessageParserService,ImapFindDraftsFolderService,ImapFindSentFolderService,- ImapMessageTextExtractorService,],exports: [ImapGetMessagesService,diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/imap/services/imap-get-messages.service.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/imap/services/imap-get-messages.service.tsindex 5ad8f8a9107..78e0fb555cc 100644--- a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/imap/services/imap-get-messages.service.ts+++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/imap/services/imap-get-messages.service.ts@@ -7,11 +7,11 @@ import { type ConnectedAccountEntity } from 'src/engine/metadata-modules/connectimport { computeMessageDirection } from 'src/modules/messaging/message-import-manager/drivers/gmail/utils/compute-message-direction.util';import { ImapClientProvider } from 'src/modules/messaging/message-import-manager/drivers/imap/providers/imap-client.provider';import { ImapMessageParserService } from 'src/modules/messaging/message-import-manager/drivers/imap/services/imap-message-parser.service';-import { ImapMessageTextExtractorService } from 'src/modules/messaging/message-import-manager/drivers/imap/services/imap-message-text-extractor.service';import { ImapMessagesImportErrorHandler } from 'src/modules/messaging/message-import-manager/drivers/imap/services/imap-messages-import-error-handler.service';import { parseMessageId } from 'src/modules/messaging/message-import-manager/drivers/imap/utils/parse-message-id.util';import { type MessageWithParticipants } from 'src/modules/messaging/message-import-manager/types/message';import { extractAddressesFromParsedEmail } from 'src/modules/messaging/message-import-manager/utils/extract-addresses-from-parsed-email.util';+import { extractMessageBodyText } from 'src/modules/messaging/message-import-manager/utils/extract-message-body-text.util';import { extractParticipantsFromParsedEmail } from 'src/modules/messaging/message-import-manager/utils/extract-participants-from-parsed-email.util';import { extractThreadIdFromParsedEmail } from 'src/modules/messaging/message-import-manager/utils/extract-thread-id-from-parsed-email.util';import { sanitizeString } from 'src/modules/messaging/message-import-manager/utils/sanitize-string.util';@@ -28,7 +28,6 @@ export class ImapGetMessagesService {constructor(private readonly imapClientProvider: ImapClientProvider,private readonly messageParser: ImapMessageParserService,- private readonly textExtractor: ImapMessageTextExtractorService,private readonly errorHandler: ImapMessagesImportErrorHandler,) {}@@ -167,9 +166,10 @@ export class ImapGetMessagesService {const fromAddresses = extractAddressesFromParsedEmail(parsed.from);const senderAddress = fromAddresses[0]?.address ?? '';- const text = sanitizeString(- this.textExtractor.extractTextWithoutReplyQuotations(parsed),- );+ const text = extractMessageBodyText({+ text: parsed.text,+ html: parsed.html,+ });return {externalId: `${folderPath}:${uid}`,diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/imap/services/imap-message-text-extractor.service.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/imap/services/imap-message-text-extractor.service.tsdeleted file mode 100644index 399eeda2e50..00000000000--- a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/imap/services/imap-message-text-extractor.service.ts+++ /dev/null@@ -1,26 +0,0 @@-import { Injectable } from '@nestjs/common';--import * as planer from 'planer';-import { safeDecodeURIComponent } from 'twenty-shared/utils';-import { type Email as ParsedEmail } from 'postal-mime';--import { createHtmlToTextConverter } from 'src/modules/messaging/message-import-manager/utils/create-html-to-text-converter.util';--@Injectable()-export class ImapMessageTextExtractorService {- private readonly convertHtmlToText = createHtmlToTextConverter();-- extractTextWithoutReplyQuotations(parsed: ParsedEmail): string {- if (parsed.text) {- const extractedText = planer.extractFrom(parsed.text, 'text/plain');-- return safeDecodeURIComponent(extractedText);- }-- if (parsed.html) {- return safeDecodeURIComponent(this.convertHtmlToText(parsed.html));- }-- return '';- }-}diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-messages.service.spec.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-messages.service.spec.tsindex 3f38d021284..d33add5c7b7 100644--- a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-messages.service.spec.ts+++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-messages.service.spec.ts@@ -152,7 +152,7 @@ describe('Microsoft get messages service', () => {});});- it('Should set empty text for html responses', () => {+ it('Should convert html responses to text', () => {const batchResponses: MicrosoftGraphBatchResponse[] =microsoftGraphBatchWithHtmlMessagesResponse;const connectedAccount = {@@ -176,7 +176,7 @@ describe('Microsoft get messages service', () => {externalId: responseExample.body.id,subject: responseExample.body.subject,receivedAt: new Date(responseExample.body.receivedDateTime),- text: '',+ text: 'test 4',headerMessageId: responseExample.body.internetMessageId,messageThreadExternalId: responseExample.body.conversationId,direction: 'OUTGOING',diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-messages.service.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-messages.service.tsindex 430af933a14..111c72a6f03 100644--- a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-messages.service.ts+++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-messages.service.ts@@ -10,6 +10,7 @@ import { computeMessageDirection } from 'src/modules/messaging/message-import-maimport { MicrosoftImportDriverException } from 'src/modules/messaging/message-import-manager/drivers/microsoft/exceptions/microsoft-import-driver.exception';import { type MicrosoftGraphBatchResponse } from 'src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-messages.interface';import { type MessageWithParticipants } from 'src/modules/messaging/message-import-manager/types/message';+import { extractMessageBodyText } from 'src/modules/messaging/message-import-manager/utils/extract-message-body-text.util';import { formatAddressObjectAsParticipants } from 'src/modules/messaging/message-import-manager/utils/format-address-object-as-participants.util';import { safeParseEmailAddress } from 'src/modules/messaging/message-import-manager/utils/safe-parse-email-address.util';@@ -130,12 +131,17 @@ export class MicrosoftGetMessagesService {: []),];+ const text = extractMessageBodyText(+ response.body?.contentType === 'text'+ ? { text: response.body?.content }+ : { html: response.body?.content },+ );+return {externalId: response.id,subject: response.subject || '',receivedAt: new Date(response.receivedDateTime),- text:- response.body?.contentType === 'text' ? response.body?.content : '',+ text,headerMessageId: response.internetMessageId,messageThreadExternalId: response.conversationId,direction: response.fromdiff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/utils/__tests__/create-html-to-text-converter.util.spec.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/utils/__tests__/create-html-to-text-converter.util.spec.tsindex 24f4106601f..fb63ee09fe4 100644--- a/packages/twenty-server/src/modules/messaging/message-import-manager/utils/__tests__/create-html-to-text-converter.util.spec.ts+++ b/packages/twenty-server/src/modules/messaging/message-import-manager/utils/__tests__/create-html-to-text-converter.util.spec.ts@@ -22,6 +22,15 @@ describe('createHtmlToTextConverter', () => {expect(convertHtmlToText('')).toBe('');});+ it('should keep the body when planer strips an entirely-quoted message to empty', () => {+ // Regression: forwarded/fully-quoted html was stripped to empty by planer.+ const result = convertHtmlToText(+ '<div class="gmail_quote"><p>Only quoted content here</p></div>',+ );++ expect(result).toBe('Only quoted content here');+ });+it('should sanitize malicious HTML', () => {const result = convertHtmlToText('<p>Hello</p><script>alert("xss")</script>',diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/imap/utils/__tests__/extract-message-text.util.spec.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/utils/__tests__/extract-message-body-text.util.spec.tssimilarity index 88%rename from packages/twenty-server/src/modules/messaging/message-import-manager/drivers/imap/utils/__tests__/extract-message-text.util.spec.tsrename to packages/twenty-server/src/modules/messaging/message-import-manager/utils/__tests__/extract-message-body-text.util.spec.tsindex e6c17cd580c..ba695c5896e 100644--- a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/imap/utils/__tests__/extract-message-text.util.spec.ts+++ b/packages/twenty-server/src/modules/messaging/message-import-manager/utils/__tests__/extract-message-body-text.util.spec.ts@@ -1,14 +1,8 @@import { type Email as ParsedMail } from 'postal-mime';-import { ImapMessageTextExtractorService } from 'src/modules/messaging/message-import-manager/drivers/imap/services/imap-message-text-extractor.service';--describe('ImapMessageTextExtractorService', () => {- let service: ImapMessageTextExtractorService;-- beforeEach(() => {- service = new ImapMessageTextExtractorService();- });+import { extractMessageBodyText } from 'src/modules/messaging/message-import-manager/utils/extract-message-body-text.util';+describe('extractMessageBodyText', () => {it('should extract text from plain text emails with lot of reply quotations', () => {const parsed: ParsedMail = {text: `Hi John,@@ -101,21 +95,24 @@ Developer SupportheaderLines: [],};- const result = service.extractTextWithoutReplyQuotations(parsed);+ const result = extractMessageBodyText({+ text: parsed.text,+ html: parsed.html,+ });expect(result).toBe(`Hi John,-Thank you for contacting Developer Support, this is Erica again. I hope you are having a good day.+Thank you for contacting Developer Support, this is Erica again. I hope you are having a good day.-I understand that you are unable to contact finance. Despite your account being expired, you should still be able to contact our finance team.+I understand that you are unable to contact finance. Despite your account being expired, you should still be able to contact our finance team.-Follow the link below the link for contacting our finance team.+Follow the link below the link for contacting our finance team.https://idmsa.apple.com/IDMSWebAuth/signin.html?path=/contact/finance/Best Regards,-Erica+EricaDeveloper Support`);});@@ -136,7 +133,10 @@ Developer Support`);headerLines: [],};- const result = service.extractTextWithoutReplyQuotations(parsed);+ const result = extractMessageBodyText({+ text: parsed.text,+ html: parsed.html,+ });expect(result).toBe('just a follow up');});@@ -158,7 +158,10 @@ Developer Support`);headerLines: [],};- const result = service.extractTextWithoutReplyQuotations(parsed);+ const result = extractMessageBodyText({+ text: parsed.text,+ html: parsed.html,+ });expect(result).toBe('just a follow up');});@@ -170,7 +173,10 @@ Developer Support`);headerLines: [],};- const result = service.extractTextWithoutReplyQuotations(parsed);+ const result = extractMessageBodyText({+ text: parsed.text,+ html: parsed.html,+ });expect(result).toBe('');});@@ -298,10 +304,13 @@ Developer Support`);</style></head><body><div id="inbox-html-wrapper"><div id="isPasted" fr-original-style="" style="user-select: inherit; scrollbar-color: var(--scrollbar-active-color) #0000; box-sizing: border-box;">Hi Sarah,</div><div fr-original-style="" style="user-select: inherit; scrollbar-color: var(--scrollbar-active-color) #0000; box-sizing: border-box;"><br fr-original-style="" style="user-select: inherit; scrollbar-color: var(--scrollbar-active-color) #0000; box-sizing: border-box;"></div><div fr-original-style="" style="user-select: inherit; scrollbar-color: var(--scrollbar-active-color) #0000; box-sizing: border-box;">I wanted to quickly follow up regarding the Q3 marketing campaign results. </div><div fr-original-style="" style="user-select: inherit; scrollbar-color: var(--scrollbar-active-color) #0000; box-sizing: border-box;">We’ve seen a 14% increase in engagement compared to last quarter, but conversions are still slightly below target. </div><div fr-original-style="" style="user-select: inherit; scrollbar-color: var(--scrollbar-active-color) #0000; box-sizing: border-box;"><br fr-original-style="" style="user-select: inherit; scrollbar-color: var(--scrollbar-active-color) #0000; box-sizing: border-box;"></div><div fr-original-style="" style="user-select: inherit; scrollbar-color: var(--scrollbar-active-color) #0000; box-sizing: border-box;">Let’s schedule a short call early next week to discuss adjustments before the Q4 push. </div><div fr-original-style="" style="user-select: inherit; scrollbar-color: var(--scrollbar-active-color) #0000; box-sizing: border-box;">Would Monday 10 AM work for you?</div><div fr-original-style="" style="user-select: inherit; scrollbar-color: var(--scrollbar-active-color) #0000; box-sizing: border-box;"><br fr-original-style="" style="user-select: inherit; scrollbar-color: var(--scrollbar-active-color) #0000; box-sizing: border-box;"></div><div fr-original-style="" style="user-select: inherit; scrollbar-color: var(--scrollbar-active-color) #0000; box-sizing: border-box;">Best regards, </div><div fr-original-style="" style="user-select: inherit; scrollbar-color: var(--scrollbar-active-color) #0000; box-sizing: border-box;">John</div><div fr-original-style="" style="user-select: inherit; scrollbar-color: var(--scrollbar-active-color) #0000; box-sizing: border-box;"><br fr-original-style="" style="user-select: inherit; scrollbar-color: var(--scrollbar-active-color) #0000; box-sizing: border-box;"></div><img class="flm-open" width="0" height="0" style="border: 0px; width: 0px; height: 0px; max-width: 100vw;" data-open-tracking-src="{{track-read-receipt}}"></div></body></html>`,};- const result = service.extractTextWithoutReplyQuotations(parsed);+ const result = extractMessageBodyText({+ text: parsed.text,+ html: parsed.html,+ });expect(result).toEqual(- `Hi Sarah,\n\nI wanted to quickly follow up regarding the Q3 marketing campaign results. \nWe’ve seen a 14% increase in engagement compared to last quarter, but conversions are still slightly below target. \n\nLet’s schedule a short call early next week to discuss adjustments before the Q4 push. \nWould Monday 10 AM work for you?\n\nBest regards, \nJohn`,+ `Hi Sarah,\n\nI wanted to quickly follow up regarding the Q3 marketing campaign results.\nWe’ve seen a 14% increase in engagement compared to last quarter, but conversions are still slightly below target.\n\nLet’s schedule a short call early next week to discuss adjustments before the Q4 push.\nWould Monday 10 AM work for you?\n\nBest regards,\nJohn`,);});@@ -314,8 +323,29 @@ Developer Support`);headerLines: [],};- const result = service.extractTextWithoutReplyQuotations(parsed);+ const result = extractMessageBodyText({+ text: parsed.text,+ html: parsed.html,+ });expect(result).toBe('Plain text content');});++ it('should preserve percent sequences instead of URI-decoding the body', () => {+ const parsed: ParsedMail = {+ text: 'See https://example.com/path%2Fto%2Ffile and a 100%20 budget cut',+ attachments: [],+ headers: [],+ headerLines: [],+ };++ const result = extractMessageBodyText({+ text: parsed.text,+ html: parsed.html,+ });++ expect(result).toBe(+ 'See https://example.com/path%2Fto%2Ffile and a 100%20 budget cut',+ );+ });});diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/utils/__tests__/extract-text-without-reply-quotations.util.spec.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/utils/__tests__/extract-text-without-reply-quotations.util.spec.tsnew file mode 100644index 00000000000..abc8aae9bc8--- /dev/null+++ b/packages/twenty-server/src/modules/messaging/message-import-manager/utils/__tests__/extract-text-without-reply-quotations.util.spec.ts@@ -0,0 +1,32 @@+import { extractTextWithoutReplyQuotations } from 'src/modules/messaging/message-import-manager/utils/extract-text-without-reply-quotations.util';++describe('extractTextWithoutReplyQuotations', () => {+ it('should keep the new reply and drop the quoted history', () => {+ const result = extractTextWithoutReplyQuotations(+ 'New reply here.\n\nOn Mon, someone wrote:\n> old line',+ );++ expect(result).toContain('New reply here.');+ expect(result).not.toContain('old line');+ });++ it('should drop a nested "On <date>, <name> wrote:" thread without quote markers', () => {+ const result = extractTextWithoutReplyQuotations(+ 'Latest answer.\n\nOn 24 Jun 2024, at 15:14, Omar M <[email protected]> wrote:\nprevious question\nsecond previous line',+ );++ expect(result).toContain('Latest answer.');+ expect(result).not.toContain('previous question');+ });++ it('should keep the full body when the message is entirely quoted (forward) and would otherwise be emptied', () => {+ // Regression: forwarded emails are entirely quotation-like, so the parser+ // returned empty and the message body was lost.+ const forwardedBody =+ '> quoted line one\n> quoted line two\n> quoted line three';++ expect(extractTextWithoutReplyQuotations(forwardedBody)).toBe(+ forwardedBody,+ );+ });+});diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/utils/__tests__/normalize-message-text.util.spec.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/utils/__tests__/normalize-message-text.util.spec.tsnew file mode 100644index 00000000000..c8d65b24183--- /dev/null+++ b/packages/twenty-server/src/modules/messaging/message-import-manager/utils/__tests__/normalize-message-text.util.spec.ts@@ -0,0 +1,29 @@+import { normalizeMessageText } from 'src/modules/messaging/message-import-manager/utils/normalize-message-text.util';++describe('normalizeMessageText', () => {+ it('should convert CRLF and bare CR to LF', () => {+ expect(normalizeMessageText('line one\r\nline two\rline three')).toBe(+ 'line one\nline two\nline three',+ );+ });++ it('should replace non-breaking spaces with regular spaces', () => {+ expect(normalizeMessageText('Hello\u00A0world')).toBe('Hello world');+ });++ it('should strip trailing whitespace on each line', () => {+ expect(normalizeMessageText('hello \nworld\t')).toBe('hello\nworld');+ });++ it('should collapse runs of three or more blank lines to one blank line', () => {+ expect(normalizeMessageText('top\n\n\n\n\nbottom')).toBe('top\n\nbottom');+ });++ it('should trim leading and trailing whitespace overall', () => {+ expect(normalizeMessageText('\r\n\r\n\r\nHello\r\n\r\n')).toBe('Hello');+ });++ it('should leave already-clean text unchanged', () => {+ expect(normalizeMessageText('Hello\n\nworld')).toBe('Hello\n\nworld');+ });+});diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/utils/create-html-to-text-converter.util.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/utils/create-html-to-text-converter.util.tsindex e795c61bedd..b3f9667cfeb 100644--- a/packages/twenty-server/src/modules/messaging/message-import-manager/utils/create-html-to-text-converter.util.ts+++ b/packages/twenty-server/src/modules/messaging/message-import-manager/utils/create-html-to-text-converter.util.ts@@ -1,7 +1,15 @@import createDOMPurify from 'dompurify';-import { convert } from 'html-to-text';+import { convert, HtmlToTextOptions } from 'html-to-text';import { JSDOM } from 'jsdom';import * as planer from 'planer';+import { isNonEmptyString } from '@sniptt/guards';++import { normalizeMessageText } from 'src/modules/messaging/message-import-manager/utils/normalize-message-text.util';++const CONVERT_OPTIONS = {+ wordwrap: false,+ preserveNewlines: true,+} satisfies HtmlToTextOptions;export const createHtmlToTextConverter = (): ((html: string) => string) => {const jsdom = new JSDOM('');@@ -15,13 +23,12 @@ export const createHtmlToTextConverter = (): ((html: string) => string) => {jsdom.window.document,);- const text = convert(cleanedHtml, {- wordwrap: false,- preserveNewlines: true,- }).trim();-- const output = text.replace(/\u00A0/g, ' ').replace(/\n{3,}/g, '\n\n');+ const text = normalizeMessageText(convert(cleanedHtml, CONVERT_OPTIONS));- return output;+ // planer can strip an entirely-quoted (e.g. forwarded) body to nothing;+ // fall back to the un-stripped sanitized html so the body is not lost.+ return isNonEmptyString(text)+ ? text+ : normalizeMessageText(convert(sanitizedHtml, CONVERT_OPTIONS));};};diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/utils/extract-message-body-text.util.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/utils/extract-message-body-text.util.tsnew file mode 100644index 00000000000..86966618811--- /dev/null+++ b/packages/twenty-server/src/modules/messaging/message-import-manager/utils/extract-message-body-text.util.ts@@ -0,0 +1,26 @@+import { isNonEmptyString } from '@sniptt/guards';++import { createHtmlToTextConverter } from 'src/modules/messaging/message-import-manager/utils/create-html-to-text-converter.util';+import { extractTextWithoutReplyQuotations } from 'src/modules/messaging/message-import-manager/utils/extract-text-without-reply-quotations.util';+import { normalizeMessageText } from 'src/modules/messaging/message-import-manager/utils/normalize-message-text.util';+import { sanitizeString } from 'src/modules/messaging/message-import-manager/utils/sanitize-string.util';++export const extractMessageBodyText = ({+ text,+ html,+}: {+ text?: string | null;+ html?: string | null;+}): string => {+ const candidate = isNonEmptyString(text)+ ? text+ : isNonEmptyString(html)+ ? createHtmlToTextConverter()(html)+ : '';++ const textWithoutReplyQuotations =+ extractTextWithoutReplyQuotations(candidate);+ const sanitizedText = sanitizeString(textWithoutReplyQuotations);++ return normalizeMessageText(sanitizedText);+};diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/utils/extract-text-without-reply-quotations.util.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/utils/extract-text-without-reply-quotations.util.tsnew file mode 100644index 00000000000..d8d65833804--- /dev/null+++ b/packages/twenty-server/src/modules/messaging/message-import-manager/utils/extract-text-without-reply-quotations.util.ts@@ -0,0 +1,15 @@+import { isNonEmptyString } from '@sniptt/guards';+import EmailReplyParser from 'email-reply-parser';++export const extractTextWithoutReplyQuotations = (text: string): string => {+ const textWithoutQuotations = new EmailReplyParser()+ .read(text)+ .getFragments()+ .filter((fragment) => !fragment.isQuoted())+ .map((fragment) => fragment.getContent())+ .join('\n');++ return isNonEmptyString(textWithoutQuotations.trim())+ ? textWithoutQuotations+ : text;+};diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/utils/normalize-message-text.util.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/utils/normalize-message-text.util.tsnew file mode 100644index 00000000000..d31b2c66cf1--- /dev/null+++ b/packages/twenty-server/src/modules/messaging/message-import-manager/utils/normalize-message-text.util.ts@@ -0,0 +1,7 @@+export const normalizeMessageText = (text: string): string =>+ text+ .replace(/\r\n?/g, '\n')+ .replace(/ /g, ' ')+ .replace(/[^\S\n]+$/gm, '')+ .replace(/\n{3,}/g, '\n\n')+ .trim();diff --git a/yarn.lock b/yarn.lockindex df80dc0f110..bf4bb63f036 100644--- a/yarn.lock+++ b/yarn.lock@@ -33291,6 +33291,18 @@ __metadata:languageName: nodelinkType: hard+"email-reply-parser@npm:^2.3.5":+ version: 2.3.5+ resolution: "email-reply-parser@npm:2.3.5"+ peerDependencies:+ re2: 1.22.1+ peerDependenciesMeta:+ re2:+ optional: true+ checksum: 10c0/1ab770b3d90cfb917dc347001db92b01916b5ee47ca7029eae39c6588458cd93c5ebe26c354c47b1052f5b7d6966e196eb893d729bf7b7c92ce546b9e6df9430+ languageName: node+ linkType: hard+"emittery@npm:^0.13.1":version: 0.13.1resolution: "emittery@npm:0.13.1"@@ -55863,6 +55875,7 @@ __metadata:digest-fetch: "npm:^3.1.1"dompurify: "npm:3.3.3"dotenv: "npm:16.4.5"+ email-reply-parser: "npm:^2.3.5"express: "npm:4.22.1"express-session: "npm:^1.18.2"file-type: "npm:^21.3.1"