The open alternative to Salesforce, designed for AI.
TypeScript crmcrm-systemcustomergood-first-issuegraphqlhacktoberfestjavascriptmarketingmonoreponestjsopen-sourcepostgresqlreactreactjssalestypescriptweb
Abstract inbound email retrieval behind message sources (#24201)
Félix Malfait783545acb415
diff --git a/packages/twenty-server/src/modules/messaging-webhooks/handlers/inbound-mail-handler.service.ts b/packages/twenty-server/src/modules/messaging-webhooks/handlers/inbound-mail-handler.service.tsindex 9ded40df3ee..12a64ca000b 100644--- a/packages/twenty-server/src/modules/messaging-webhooks/handlers/inbound-mail-handler.service.ts+++ b/packages/twenty-server/src/modules/messaging-webhooks/handlers/inbound-mail-handler.service.ts@@ -41,7 +41,8 @@ export class InboundMailHandlerService {await this.messageQueueService.add<MessagingInboundEmailImportJobData>(MessagingInboundEmailImportJob.name,{- s3Key: notification.message.reference,+ source: notification.message.source,+ reference: notification.message.reference,envelopeRecipients: notification.recipients,},{ id: notification.dedupeKey },diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/inbound-email/messaging-inbound-email-driver.module.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/inbound-email/messaging-inbound-email-driver.module.tsindex c76f61c64b1..da69082817b 100644--- a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/inbound-email/messaging-inbound-email-driver.module.ts+++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/inbound-email/messaging-inbound-email-driver.module.ts@@ -8,6 +8,8 @@ import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/worksimport { InboundEmailS3ClientProvider } from 'src/modules/messaging/message-import-manager/drivers/inbound-email/providers/inbound-email-s3-client.provider';import { InboundEmailParserService } from 'src/modules/messaging/message-import-manager/drivers/inbound-email/services/inbound-email-parser.service';import { InboundEmailStorageService } from 'src/modules/messaging/message-import-manager/drivers/inbound-email/services/inbound-email-storage.service';+import { InboundEmailMessageSourceResolverService } from 'src/modules/messaging/message-import-manager/drivers/inbound-email/sources/inbound-email-message-source-resolver.service';+import { SesS3InboundEmailMessageSourceService } from 'src/modules/messaging/message-import-manager/drivers/inbound-email/sources/ses-s3-inbound-email-message-source.service';@Module({imports: [@@ -19,11 +21,15 @@ import { InboundEmailStorageService } from 'src/modules/messaging/message-importInboundEmailS3ClientProvider,InboundEmailStorageService,InboundEmailParserService,+ SesS3InboundEmailMessageSourceService,+ InboundEmailMessageSourceResolverService,],exports: [InboundEmailS3ClientProvider,InboundEmailStorageService,InboundEmailParserService,+ SesS3InboundEmailMessageSourceService,+ InboundEmailMessageSourceResolverService,],})export class MessagingInboundEmailDriverModule {}diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/inbound-email/providers/inbound-email-s3-client.provider.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/inbound-email/providers/inbound-email-s3-client.provider.tsindex eccbd93b458..71b24971980 100644--- a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/inbound-email/providers/inbound-email-s3-client.provider.ts+++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/inbound-email/providers/inbound-email-s3-client.provider.ts@@ -3,7 +3,6 @@ import { Injectable } from '@nestjs/common';import { S3Client, type S3ClientConfig } from '@aws-sdk/client-s3';import { isNonEmptyString } from '@sniptt/guards';-import { StorageDriverType } from 'src/engine/core-modules/file-storage/interfaces/file-storage.interface';import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';import { buildAwsRequestHandlerOptions } from 'src/utils/aws-request-handler.util';@@ -13,13 +12,6 @@ export class InboundEmailS3ClientProvider {constructor(private readonly twentyConfigService: TwentyConfigService) {}- isConfigured(): boolean {- const storageType = this.twentyConfigService.get('STORAGE_TYPE');- const domain = this.twentyConfigService.get('INBOUND_EMAIL_DOMAIN');-- return storageType === StorageDriverType.S_3 && isNonEmptyString(domain);- }-getBucket(): string {const bucket = this.twentyConfigService.get('STORAGE_S3_NAME');@@ -32,18 +24,6 @@ export class InboundEmailS3ClientProvider {return bucket;}- getDomain(): string {- const domain = this.twentyConfigService.get('INBOUND_EMAIL_DOMAIN');-- if (!isNonEmptyString(domain)) {- throw new Error(- 'INBOUND_EMAIL_DOMAIN is not configured; email group is disabled.',- );- }-- return domain;- }-getClient(): S3Client {if (this.s3Client) {return this.s3Client;diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/inbound-email/services/inbound-email-import.service.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/inbound-email/services/inbound-email-import.service.tsindex 715cf8d0aa4..cfc85f85ef3 100644--- a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/inbound-email/services/inbound-email-import.service.ts+++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/inbound-email/services/inbound-email-import.service.ts@@ -1,22 +1,23 @@import { Injectable, Logger } from '@nestjs/common';import { InjectRepository } from '@nestjs/typeorm';+import { isNonEmptyString } from '@sniptt/guards';import { MessageChannelType } from 'twenty-shared/types';+import { isDefined } from 'twenty-shared/utils';import { Repository } from 'typeorm';+import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';import { ConnectedAccountEntity } from 'src/engine/metadata-modules/connected-account/entities/connected-account.entity';import { MessageChannelEntity } from 'src/engine/metadata-modules/message-channel/entities/message-channel.entity';import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';import { buildSystemAuthContext } from 'src/engine/twenty-orm/utils/build-system-auth-context.util';-import { InboundEmailS3ClientProvider } from 'src/modules/messaging/message-import-manager/drivers/inbound-email/providers/inbound-email-s3-client.provider';-import { InboundEmailParserService } from 'src/modules/messaging/message-import-manager/drivers/inbound-email/services/inbound-email-parser.service';-import { InboundEmailStorageService } from 'src/modules/messaging/message-import-manager/drivers/inbound-email/services/inbound-email-storage.service';+import { InboundEmailMessageSourceResolverService } from 'src/modules/messaging/message-import-manager/drivers/inbound-email/sources/inbound-email-message-source-resolver.service';import { type InboundEmailImportOutcome } from 'src/modules/messaging/message-import-manager/drivers/inbound-email/types/inbound-email-import-outcome.type';+import { type InboundEmailMessageReference } from 'src/modules/messaging/message-import-manager/drivers/inbound-email/types/inbound-email-message-reference.type';import { MessagingSaveMessagesAndEnqueueContactCreationService } from 'src/modules/messaging/message-import-manager/services/messaging-save-messages-and-enqueue-contact-creation.service';-import { isDefined } from 'twenty-shared/utils';type ImportInboundMessageParams = {- s3Key: string;+ messageReference: InboundEmailMessageReference;envelopeRecipients: string[];};@@ -25,9 +26,8 @@ export class InboundEmailImportService {private readonly logger = new Logger(InboundEmailImportService.name);constructor(- private readonly inboundEmailS3ClientProvider: InboundEmailS3ClientProvider,- private readonly inboundEmailStorageService: InboundEmailStorageService,- private readonly inboundEmailParserService: InboundEmailParserService,+ private readonly twentyConfigService: TwentyConfigService,+ private readonly inboundEmailMessageSourceResolverService: InboundEmailMessageSourceResolverService,private readonly globalWorkspaceOrmManager: GlobalWorkspaceOrmManager,private readonly messagingSaveMessagesAndEnqueueContactCreationService: MessagingSaveMessagesAndEnqueueContactCreationService,@InjectRepository(MessageChannelEntity)@@ -39,17 +39,32 @@ export class InboundEmailImportService {async importInboundMessage(params: ImportInboundMessageParams,): Promise<InboundEmailImportOutcome> {- const { s3Key, envelopeRecipients } = params;+ const { messageReference, envelopeRecipients } = params;++ const inboundEmailDomain = this.twentyConfigService.get(+ 'INBOUND_EMAIL_DOMAIN',+ );++ if (!isNonEmptyString(inboundEmailDomain)) {+ this.logger.warn(+ `Skipping inbound email import for ${messageReference.reference}: email group is not configured.`,+ );++ return { kind: 'unconfigured' };+ }++ const messageSource = this.inboundEmailMessageSourceResolverService.resolve(+ messageReference.source,+ );- if (!this.inboundEmailS3ClientProvider.isConfigured()) {+ if (!messageSource.isConfigured()) {this.logger.warn(- `Skipping inbound email import for ${s3Key}: email group is not configured.`,+ `Skipping inbound email import for ${messageReference.reference}: message source ${messageReference.source} is not configured.`,);return { kind: 'unconfigured' };}- const inboundEmailDomain = this.inboundEmailS3ClientProvider.getDomain();const recipient = this.matchInboundRecipient(envelopeRecipients,inboundEmailDomain,@@ -57,7 +72,7 @@ export class InboundEmailImportService {if (!isDefined(recipient)) {this.logger.warn(- `No recipient at ${inboundEmailDomain} in SNS payload for ${s3Key}`,+ `No recipient at ${inboundEmailDomain} in inbound notification for ${messageReference.reference}`,);return { kind: 'unmatched', recipient: null };@@ -69,17 +84,14 @@ export class InboundEmailImportService {if (!isDefined(messageChannel)) {this.logger.warn(- `No email group channel matches recipient ${recipient} (key ${s3Key})`,+ `No email group channel matches recipient ${recipient} (reference ${messageReference.reference})`,);return { kind: 'unmatched', recipient };}- const rawMessage =- await this.inboundEmailStorageService.getRawMessage(s3Key);- const parsedInboundMessage = await this.inboundEmailParserService.parse(- rawMessage,- s3Key,+ const message = await messageSource.fetchMessage(+ messageReference.reference,);const { workspaceId } = messageChannel;@@ -97,7 +109,7 @@ export class InboundEmailImportService {await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => {await this.messagingSaveMessagesAndEnqueueContactCreationService.saveMessagesAndEnqueueContactCreation(- [parsedInboundMessage.message],+ [message],messageChannel,connectedAccount,workspaceId,@@ -107,7 +119,7 @@ export class InboundEmailImportService {{ lite: true },);- await this.inboundEmailStorageService.deleteRawMessage(s3Key);+ await messageSource.cleanup(messageReference.reference);return {kind: 'imported',diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/inbound-email/services/inbound-email-parser.service.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/inbound-email/services/inbound-email-parser.service.tsindex 2e2b2a8f8e1..c54cfbe84ca 100644--- a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/inbound-email/services/inbound-email-parser.service.ts+++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/inbound-email/services/inbound-email-parser.service.ts@@ -13,22 +13,22 @@ import { sanitizeString } from 'src/modules/messaging/message-import-manager/utiexport class InboundEmailParserService {async parse(rawMessage: Buffer,- s3Key: string,+ reference: string,): Promise<ParsedInboundMessage> {const parsedEmail = await PostalMime.parse(rawMessage);- const message = this.buildMessage(parsedEmail, s3Key);+ const message = this.buildMessage(parsedEmail, reference);return { parsed: parsedEmail, message };}private buildMessage(parsedEmail: ParsedEmail,- s3Key: string,+ reference: string,): MessageWithParticipants {return {- externalId: `inbound-email:${s3Key}`,+ externalId: `inbound-email:${reference}`,messageThreadExternalId: extractThreadIdFromParsedEmail(parsedEmail),- headerMessageId: parsedEmail.messageId?.trim() || `inbound-${s3Key}`,+ headerMessageId: parsedEmail.messageId?.trim() || `inbound-${reference}`,subject: sanitizeString(parsedEmail.subject || ''),text: sanitizeString(parsedEmail.text || ''),receivedAt: parsedEmail.date ? new Date(parsedEmail.date) : new Date(),diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/inbound-email/sources/inbound-email-message-source-resolver.service.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/inbound-email/sources/inbound-email-message-source-resolver.service.tsnew file mode 100644index 00000000000..7fed32fcd39--- /dev/null+++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/inbound-email/sources/inbound-email-message-source-resolver.service.ts@@ -0,0 +1,24 @@+import { Injectable } from '@nestjs/common';++import { type InboundEmailMessageSourceInterface } from 'src/modules/messaging/message-import-manager/drivers/inbound-email/sources/inbound-email-message-source.interface';+import { SesS3InboundEmailMessageSourceService } from 'src/modules/messaging/message-import-manager/drivers/inbound-email/sources/ses-s3-inbound-email-message-source.service';+import { INBOUND_EMAIL_MESSAGE_SOURCE } from 'src/modules/messaging/message-import-manager/drivers/inbound-email/constants/inbound-email-message-source.constant';+import { type InboundEmailMessageSource } from 'src/modules/messaging/message-import-manager/drivers/inbound-email/types/inbound-email-message-source.type';++@Injectable()+export class InboundEmailMessageSourceResolverService {+ constructor(+ private readonly sesS3InboundEmailMessageSourceService: SesS3InboundEmailMessageSourceService,+ ) {}++ resolve(+ source: InboundEmailMessageSource,+ ): InboundEmailMessageSourceInterface {+ switch (source) {+ case INBOUND_EMAIL_MESSAGE_SOURCE.SES_S3:+ return this.sesS3InboundEmailMessageSourceService;+ default:+ throw new Error(`Unsupported inbound email message source: ${source}`);+ }+ }+}diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/inbound-email/sources/inbound-email-message-source.interface.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/inbound-email/sources/inbound-email-message-source.interface.tsnew file mode 100644index 00000000000..70318291ee5--- /dev/null+++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/inbound-email/sources/inbound-email-message-source.interface.ts@@ -0,0 +1,7 @@+import { type MessageWithParticipants } from 'src/modules/messaging/message-import-manager/types/message';++export type InboundEmailMessageSourceInterface = {+ isConfigured(): boolean;+ fetchMessage(reference: string): Promise<MessageWithParticipants>;+ cleanup(reference: string): Promise<void>;+};diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/inbound-email/sources/ses-s3-inbound-email-message-source.service.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/inbound-email/sources/ses-s3-inbound-email-message-source.service.tsnew file mode 100644index 00000000000..7975d84a3bb--- /dev/null+++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/inbound-email/sources/ses-s3-inbound-email-message-source.service.ts@@ -0,0 +1,41 @@+import { Injectable } from '@nestjs/common';++import { StorageDriverType } from 'src/engine/core-modules/file-storage/interfaces/file-storage.interface';+import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';+import { InboundEmailParserService } from 'src/modules/messaging/message-import-manager/drivers/inbound-email/services/inbound-email-parser.service';+import { InboundEmailStorageService } from 'src/modules/messaging/message-import-manager/drivers/inbound-email/services/inbound-email-storage.service';+import { type InboundEmailMessageSourceInterface } from 'src/modules/messaging/message-import-manager/drivers/inbound-email/sources/inbound-email-message-source.interface';+import { type MessageWithParticipants } from 'src/modules/messaging/message-import-manager/types/message';++// SES receipt rules park the raw MIME message in S3; the reference is the S3+// object key relayed through the inbound webhook notification.+@Injectable()+export class SesS3InboundEmailMessageSourceService implements InboundEmailMessageSourceInterface {+ constructor(+ private readonly twentyConfigService: TwentyConfigService,+ private readonly inboundEmailStorageService: InboundEmailStorageService,+ private readonly inboundEmailParserService: InboundEmailParserService,+ ) {}++ isConfigured(): boolean {+ return (+ this.twentyConfigService.get('STORAGE_TYPE') === StorageDriverType.S_3+ );+ }++ async fetchMessage(reference: string): Promise<MessageWithParticipants> {+ const rawMessage =+ await this.inboundEmailStorageService.getRawMessage(reference);++ const { message } = await this.inboundEmailParserService.parse(+ rawMessage,+ reference,+ );++ return message;+ }++ async cleanup(reference: string): Promise<void> {+ await this.inboundEmailStorageService.deleteRawMessage(reference);+ }+}diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/inbound-email/utils/__tests__/resolve-inbound-email-message-reference.util.spec.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/inbound-email/utils/__tests__/resolve-inbound-email-message-reference.util.spec.tsnew file mode 100644index 00000000000..8ca09d61f64--- /dev/null+++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/inbound-email/utils/__tests__/resolve-inbound-email-message-reference.util.spec.ts@@ -0,0 +1,22 @@+import { resolveInboundEmailMessageReference } from 'src/modules/messaging/message-import-manager/drivers/inbound-email/utils/resolve-inbound-email-message-reference.util';++describe('resolveInboundEmailMessageReference', () => {+ it('should pass through source and reference payloads', () => {+ expect(+ resolveInboundEmailMessageReference({+ source: 'SES_S3',+ reference: 'raw/object-key',+ envelopeRecipients: ['[email protected]'],+ }),+ ).toEqual({ source: 'SES_S3', reference: 'raw/object-key' });+ });++ it('should normalize legacy s3Key payloads to the SES_S3 source', () => {+ expect(+ resolveInboundEmailMessageReference({+ s3Key: 'raw/legacy-key',+ envelopeRecipients: ['[email protected]'],+ }),+ ).toEqual({ source: 'SES_S3', reference: 'raw/legacy-key' });+ });+});diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/inbound-email/utils/resolve-inbound-email-message-reference.util.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/inbound-email/utils/resolve-inbound-email-message-reference.util.tsnew file mode 100644index 00000000000..9f5ce60df9d--- /dev/null+++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/inbound-email/utils/resolve-inbound-email-message-reference.util.ts@@ -0,0 +1,16 @@+import { INBOUND_EMAIL_MESSAGE_SOURCE } from 'src/modules/messaging/message-import-manager/drivers/inbound-email/constants/inbound-email-message-source.constant';+import { type InboundEmailMessageReference } from 'src/modules/messaging/message-import-manager/drivers/inbound-email/types/inbound-email-message-reference.type';+import { type MessagingInboundEmailImportJobData } from 'src/modules/messaging/message-import-manager/jobs/messaging-inbound-email-import.job';++export const resolveInboundEmailMessageReference = (+ data: MessagingInboundEmailImportJobData,+): InboundEmailMessageReference => {+ if ('s3Key' in data) {+ return {+ source: INBOUND_EMAIL_MESSAGE_SOURCE.SES_S3,+ reference: data.s3Key,+ };+ }++ return { source: data.source, reference: data.reference };+};diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/jobs/messaging-inbound-email-import.job.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/jobs/messaging-inbound-email-import.job.tsindex f36b8a35981..5d22f14c211 100644--- a/packages/twenty-server/src/modules/messaging/message-import-manager/jobs/messaging-inbound-email-import.job.ts+++ b/packages/twenty-server/src/modules/messaging/message-import-manager/jobs/messaging-inbound-email-import.job.ts@@ -4,11 +4,16 @@ import { Process } from 'src/engine/core-modules/message-queue/decorators/procesimport { Processor } from 'src/engine/core-modules/message-queue/decorators/processor.decorator';import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';import { InboundEmailImportService } from 'src/modules/messaging/message-import-manager/drivers/inbound-email/services/inbound-email-import.service';+import { type InboundEmailMessageSource } from 'src/modules/messaging/message-import-manager/drivers/inbound-email/types/inbound-email-message-source.type';+import { resolveInboundEmailMessageReference } from 'src/modules/messaging/message-import-manager/drivers/inbound-email/utils/resolve-inbound-email-message-reference.util';export type MessagingInboundEmailImportJobData = {- s3Key: string;envelopeRecipients: string[];-};+} & (+ | { source: InboundEmailMessageSource; reference: string }+ // legacy payload shape from jobs enqueued before the source abstraction+ | { s3Key: string }+);@Processor({queueName: MessageQueue.messagingQueue,@@ -23,15 +28,15 @@ export class MessagingInboundEmailImportJob {@Process(MessagingInboundEmailImportJob.name)async handle(data: MessagingInboundEmailImportJobData): Promise<void> {- const { s3Key, envelopeRecipients } = data;+ const messageReference = resolveInboundEmailMessageReference(data);const outcome = await this.inboundEmailImportService.importInboundMessage({- s3Key,- envelopeRecipients,+ messageReference,+ envelopeRecipients: data.envelopeRecipients,});this.logger.log(- `Inbound email import outcome for ${s3Key}: ${outcome.kind}`,+ `Inbound email import outcome for ${messageReference.reference}: ${outcome.kind}`,);}}