Skip to content

The open alternative to Salesforce, designed for AI.

Abstract inbound email retrieval behind message sources (#24201)

Félix Malfait783545acb415
diff --git a/packages/twenty-server/src/modules/messaging-webhooks/handlers/inbound-mail-handler.service.ts b/packages/twenty-server/src/modules/messaging-webhooks/handlers/inbound-mail-handler.service.ts
index 9ded40df3ee..12a64ca000b 100644
--- a/packages/twenty-server/src/modules/messaging-webhooks/handlers/inbound-mail-handler.service.ts
+++ b/packages/twenty-server/src/modules/messaging-webhooks/handlers/inbound-mail-handler.service.ts
@@ -41,7 +41,8 @@ export class InboundMailHandlerService {
await this.messageQueueService.add<MessagingInboundEmailImportJobData>(
MessagingInboundEmailImportJob.name,
{
- s3Key: notification.message.reference,
+ source: notification.message.source,
+ reference: notification.message.reference,
envelopeRecipients: notification.recipients,
},
{ id: notification.dedupeKey },
diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/inbound-email/messaging-inbound-email-driver.module.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/inbound-email/messaging-inbound-email-driver.module.ts
index c76f61c64b1..da69082817b 100644
--- a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/inbound-email/messaging-inbound-email-driver.module.ts
+++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/inbound-email/messaging-inbound-email-driver.module.ts
@@ -8,6 +8,8 @@ import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/works
import { InboundEmailS3ClientProvider } from 'src/modules/messaging/message-import-manager/drivers/inbound-email/providers/inbound-email-s3-client.provider';
import { InboundEmailParserService } from 'src/modules/messaging/message-import-manager/drivers/inbound-email/services/inbound-email-parser.service';
import { InboundEmailStorageService } from 'src/modules/messaging/message-import-manager/drivers/inbound-email/services/inbound-email-storage.service';
+import { InboundEmailMessageSourceResolverService } from 'src/modules/messaging/message-import-manager/drivers/inbound-email/sources/inbound-email-message-source-resolver.service';
+import { SesS3InboundEmailMessageSourceService } from 'src/modules/messaging/message-import-manager/drivers/inbound-email/sources/ses-s3-inbound-email-message-source.service';
@Module({
imports: [
@@ -19,11 +21,15 @@ import { InboundEmailStorageService } from 'src/modules/messaging/message-import
InboundEmailS3ClientProvider,
InboundEmailStorageService,
InboundEmailParserService,
+ SesS3InboundEmailMessageSourceService,
+ InboundEmailMessageSourceResolverService,
],
exports: [
InboundEmailS3ClientProvider,
InboundEmailStorageService,
InboundEmailParserService,
+ SesS3InboundEmailMessageSourceService,
+ InboundEmailMessageSourceResolverService,
],
})
export class MessagingInboundEmailDriverModule {}
diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/inbound-email/providers/inbound-email-s3-client.provider.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/inbound-email/providers/inbound-email-s3-client.provider.ts
index eccbd93b458..71b24971980 100644
--- a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/inbound-email/providers/inbound-email-s3-client.provider.ts
+++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/inbound-email/providers/inbound-email-s3-client.provider.ts
@@ -3,7 +3,6 @@ import { Injectable } from '@nestjs/common';
import { S3Client, type S3ClientConfig } from '@aws-sdk/client-s3';
import { isNonEmptyString } from '@sniptt/guards';
-import { StorageDriverType } from 'src/engine/core-modules/file-storage/interfaces/file-storage.interface';
import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';
import { buildAwsRequestHandlerOptions } from 'src/utils/aws-request-handler.util';
@@ -13,13 +12,6 @@ export class InboundEmailS3ClientProvider {
constructor(private readonly twentyConfigService: TwentyConfigService) {}
- isConfigured(): boolean {
- const storageType = this.twentyConfigService.get('STORAGE_TYPE');
- const domain = this.twentyConfigService.get('INBOUND_EMAIL_DOMAIN');
-
- return storageType === StorageDriverType.S_3 && isNonEmptyString(domain);
- }
-
getBucket(): string {
const bucket = this.twentyConfigService.get('STORAGE_S3_NAME');
@@ -32,18 +24,6 @@ export class InboundEmailS3ClientProvider {
return bucket;
}
- getDomain(): string {
- const domain = this.twentyConfigService.get('INBOUND_EMAIL_DOMAIN');
-
- if (!isNonEmptyString(domain)) {
- throw new Error(
- 'INBOUND_EMAIL_DOMAIN is not configured; email group is disabled.',
- );
- }
-
- return domain;
- }
-
getClient(): S3Client {
if (this.s3Client) {
return this.s3Client;
diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/inbound-email/services/inbound-email-import.service.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/inbound-email/services/inbound-email-import.service.ts
index 715cf8d0aa4..cfc85f85ef3 100644
--- a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/inbound-email/services/inbound-email-import.service.ts
+++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/inbound-email/services/inbound-email-import.service.ts
@@ -1,22 +1,23 @@
import { Injectable, Logger } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
+import { isNonEmptyString } from '@sniptt/guards';
import { MessageChannelType } from 'twenty-shared/types';
+import { isDefined } from 'twenty-shared/utils';
import { Repository } from 'typeorm';
+import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';
import { ConnectedAccountEntity } from 'src/engine/metadata-modules/connected-account/entities/connected-account.entity';
import { MessageChannelEntity } from 'src/engine/metadata-modules/message-channel/entities/message-channel.entity';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { buildSystemAuthContext } from 'src/engine/twenty-orm/utils/build-system-auth-context.util';
-import { InboundEmailS3ClientProvider } from 'src/modules/messaging/message-import-manager/drivers/inbound-email/providers/inbound-email-s3-client.provider';
-import { InboundEmailParserService } from 'src/modules/messaging/message-import-manager/drivers/inbound-email/services/inbound-email-parser.service';
-import { InboundEmailStorageService } from 'src/modules/messaging/message-import-manager/drivers/inbound-email/services/inbound-email-storage.service';
+import { InboundEmailMessageSourceResolverService } from 'src/modules/messaging/message-import-manager/drivers/inbound-email/sources/inbound-email-message-source-resolver.service';
import { type InboundEmailImportOutcome } from 'src/modules/messaging/message-import-manager/drivers/inbound-email/types/inbound-email-import-outcome.type';
+import { type InboundEmailMessageReference } from 'src/modules/messaging/message-import-manager/drivers/inbound-email/types/inbound-email-message-reference.type';
import { MessagingSaveMessagesAndEnqueueContactCreationService } from 'src/modules/messaging/message-import-manager/services/messaging-save-messages-and-enqueue-contact-creation.service';
-import { isDefined } from 'twenty-shared/utils';
type ImportInboundMessageParams = {
- s3Key: string;
+ messageReference: InboundEmailMessageReference;
envelopeRecipients: string[];
};
@@ -25,9 +26,8 @@ export class InboundEmailImportService {
private readonly logger = new Logger(InboundEmailImportService.name);
constructor(
- private readonly inboundEmailS3ClientProvider: InboundEmailS3ClientProvider,
- private readonly inboundEmailStorageService: InboundEmailStorageService,
- private readonly inboundEmailParserService: InboundEmailParserService,
+ private readonly twentyConfigService: TwentyConfigService,
+ private readonly inboundEmailMessageSourceResolverService: InboundEmailMessageSourceResolverService,
private readonly globalWorkspaceOrmManager: GlobalWorkspaceOrmManager,
private readonly messagingSaveMessagesAndEnqueueContactCreationService: MessagingSaveMessagesAndEnqueueContactCreationService,
@InjectRepository(MessageChannelEntity)
@@ -39,17 +39,32 @@ export class InboundEmailImportService {
async importInboundMessage(
params: ImportInboundMessageParams,
): Promise<InboundEmailImportOutcome> {
- const { s3Key, envelopeRecipients } = params;
+ const { messageReference, envelopeRecipients } = params;
+
+ const inboundEmailDomain = this.twentyConfigService.get(
+ 'INBOUND_EMAIL_DOMAIN',
+ );
+
+ if (!isNonEmptyString(inboundEmailDomain)) {
+ this.logger.warn(
+ `Skipping inbound email import for ${messageReference.reference}: email group is not configured.`,
+ );
+
+ return { kind: 'unconfigured' };
+ }
+
+ const messageSource = this.inboundEmailMessageSourceResolverService.resolve(
+ messageReference.source,
+ );
- if (!this.inboundEmailS3ClientProvider.isConfigured()) {
+ if (!messageSource.isConfigured()) {
this.logger.warn(
- `Skipping inbound email import for ${s3Key}: email group is not configured.`,
+ `Skipping inbound email import for ${messageReference.reference}: message source ${messageReference.source} is not configured.`,
);
return { kind: 'unconfigured' };
}
- const inboundEmailDomain = this.inboundEmailS3ClientProvider.getDomain();
const recipient = this.matchInboundRecipient(
envelopeRecipients,
inboundEmailDomain,
@@ -57,7 +72,7 @@ export class InboundEmailImportService {
if (!isDefined(recipient)) {
this.logger.warn(
- `No recipient at ${inboundEmailDomain} in SNS payload for ${s3Key}`,
+ `No recipient at ${inboundEmailDomain} in inbound notification for ${messageReference.reference}`,
);
return { kind: 'unmatched', recipient: null };
@@ -69,17 +84,14 @@ export class InboundEmailImportService {
if (!isDefined(messageChannel)) {
this.logger.warn(
- `No email group channel matches recipient ${recipient} (key ${s3Key})`,
+ `No email group channel matches recipient ${recipient} (reference ${messageReference.reference})`,
);
return { kind: 'unmatched', recipient };
}
- const rawMessage =
- await this.inboundEmailStorageService.getRawMessage(s3Key);
- const parsedInboundMessage = await this.inboundEmailParserService.parse(
- rawMessage,
- s3Key,
+ const message = await messageSource.fetchMessage(
+ messageReference.reference,
);
const { workspaceId } = messageChannel;
@@ -97,7 +109,7 @@ export class InboundEmailImportService {
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
async () => {
await this.messagingSaveMessagesAndEnqueueContactCreationService.saveMessagesAndEnqueueContactCreation(
- [parsedInboundMessage.message],
+ [message],
messageChannel,
connectedAccount,
workspaceId,
@@ -107,7 +119,7 @@ export class InboundEmailImportService {
{ lite: true },
);
- await this.inboundEmailStorageService.deleteRawMessage(s3Key);
+ await messageSource.cleanup(messageReference.reference);
return {
kind: 'imported',
diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/inbound-email/services/inbound-email-parser.service.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/inbound-email/services/inbound-email-parser.service.ts
index 2e2b2a8f8e1..c54cfbe84ca 100644
--- a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/inbound-email/services/inbound-email-parser.service.ts
+++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/inbound-email/services/inbound-email-parser.service.ts
@@ -13,22 +13,22 @@ import { sanitizeString } from 'src/modules/messaging/message-import-manager/uti
export class InboundEmailParserService {
async parse(
rawMessage: Buffer,
- s3Key: string,
+ reference: string,
): Promise<ParsedInboundMessage> {
const parsedEmail = await PostalMime.parse(rawMessage);
- const message = this.buildMessage(parsedEmail, s3Key);
+ const message = this.buildMessage(parsedEmail, reference);
return { parsed: parsedEmail, message };
}
private buildMessage(
parsedEmail: ParsedEmail,
- s3Key: string,
+ reference: string,
): MessageWithParticipants {
return {
- externalId: `inbound-email:${s3Key}`,
+ externalId: `inbound-email:${reference}`,
messageThreadExternalId: extractThreadIdFromParsedEmail(parsedEmail),
- headerMessageId: parsedEmail.messageId?.trim() || `inbound-${s3Key}`,
+ headerMessageId: parsedEmail.messageId?.trim() || `inbound-${reference}`,
subject: sanitizeString(parsedEmail.subject || ''),
text: sanitizeString(parsedEmail.text || ''),
receivedAt: parsedEmail.date ? new Date(parsedEmail.date) : new Date(),
diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/inbound-email/sources/inbound-email-message-source-resolver.service.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/inbound-email/sources/inbound-email-message-source-resolver.service.ts
new file mode 100644
index 00000000000..7fed32fcd39
--- /dev/null
+++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/inbound-email/sources/inbound-email-message-source-resolver.service.ts
@@ -0,0 +1,24 @@
+import { Injectable } from '@nestjs/common';
+
+import { type InboundEmailMessageSourceInterface } from 'src/modules/messaging/message-import-manager/drivers/inbound-email/sources/inbound-email-message-source.interface';
+import { SesS3InboundEmailMessageSourceService } from 'src/modules/messaging/message-import-manager/drivers/inbound-email/sources/ses-s3-inbound-email-message-source.service';
+import { INBOUND_EMAIL_MESSAGE_SOURCE } from 'src/modules/messaging/message-import-manager/drivers/inbound-email/constants/inbound-email-message-source.constant';
+import { type InboundEmailMessageSource } from 'src/modules/messaging/message-import-manager/drivers/inbound-email/types/inbound-email-message-source.type';
+
+@Injectable()
+export class InboundEmailMessageSourceResolverService {
+ constructor(
+ private readonly sesS3InboundEmailMessageSourceService: SesS3InboundEmailMessageSourceService,
+ ) {}
+
+ resolve(
+ source: InboundEmailMessageSource,
+ ): InboundEmailMessageSourceInterface {
+ switch (source) {
+ case INBOUND_EMAIL_MESSAGE_SOURCE.SES_S3:
+ return this.sesS3InboundEmailMessageSourceService;
+ default:
+ throw new Error(`Unsupported inbound email message source: ${source}`);
+ }
+ }
+}
diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/inbound-email/sources/inbound-email-message-source.interface.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/inbound-email/sources/inbound-email-message-source.interface.ts
new file mode 100644
index 00000000000..70318291ee5
--- /dev/null
+++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/inbound-email/sources/inbound-email-message-source.interface.ts
@@ -0,0 +1,7 @@
+import { type MessageWithParticipants } from 'src/modules/messaging/message-import-manager/types/message';
+
+export type InboundEmailMessageSourceInterface = {
+ isConfigured(): boolean;
+ fetchMessage(reference: string): Promise<MessageWithParticipants>;
+ cleanup(reference: string): Promise<void>;
+};
diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/inbound-email/sources/ses-s3-inbound-email-message-source.service.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/inbound-email/sources/ses-s3-inbound-email-message-source.service.ts
new file mode 100644
index 00000000000..7975d84a3bb
--- /dev/null
+++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/inbound-email/sources/ses-s3-inbound-email-message-source.service.ts
@@ -0,0 +1,41 @@
+import { Injectable } from '@nestjs/common';
+
+import { StorageDriverType } from 'src/engine/core-modules/file-storage/interfaces/file-storage.interface';
+import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';
+import { InboundEmailParserService } from 'src/modules/messaging/message-import-manager/drivers/inbound-email/services/inbound-email-parser.service';
+import { InboundEmailStorageService } from 'src/modules/messaging/message-import-manager/drivers/inbound-email/services/inbound-email-storage.service';
+import { type InboundEmailMessageSourceInterface } from 'src/modules/messaging/message-import-manager/drivers/inbound-email/sources/inbound-email-message-source.interface';
+import { type MessageWithParticipants } from 'src/modules/messaging/message-import-manager/types/message';
+
+// SES receipt rules park the raw MIME message in S3; the reference is the S3
+// object key relayed through the inbound webhook notification.
+@Injectable()
+export class SesS3InboundEmailMessageSourceService implements InboundEmailMessageSourceInterface {
+ constructor(
+ private readonly twentyConfigService: TwentyConfigService,
+ private readonly inboundEmailStorageService: InboundEmailStorageService,
+ private readonly inboundEmailParserService: InboundEmailParserService,
+ ) {}
+
+ isConfigured(): boolean {
+ return (
+ this.twentyConfigService.get('STORAGE_TYPE') === StorageDriverType.S_3
+ );
+ }
+
+ async fetchMessage(reference: string): Promise<MessageWithParticipants> {
+ const rawMessage =
+ await this.inboundEmailStorageService.getRawMessage(reference);
+
+ const { message } = await this.inboundEmailParserService.parse(
+ rawMessage,
+ reference,
+ );
+
+ return message;
+ }
+
+ async cleanup(reference: string): Promise<void> {
+ await this.inboundEmailStorageService.deleteRawMessage(reference);
+ }
+}
diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/inbound-email/utils/__tests__/resolve-inbound-email-message-reference.util.spec.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/inbound-email/utils/__tests__/resolve-inbound-email-message-reference.util.spec.ts
new file mode 100644
index 00000000000..8ca09d61f64
--- /dev/null
+++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/inbound-email/utils/__tests__/resolve-inbound-email-message-reference.util.spec.ts
@@ -0,0 +1,22 @@
+import { resolveInboundEmailMessageReference } from 'src/modules/messaging/message-import-manager/drivers/inbound-email/utils/resolve-inbound-email-message-reference.util';
+
+describe('resolveInboundEmailMessageReference', () => {
+ it('should pass through source and reference payloads', () => {
+ expect(
+ resolveInboundEmailMessageReference({
+ source: 'SES_S3',
+ reference: 'raw/object-key',
+ envelopeRecipients: ['[email protected]'],
+ }),
+ ).toEqual({ source: 'SES_S3', reference: 'raw/object-key' });
+ });
+
+ it('should normalize legacy s3Key payloads to the SES_S3 source', () => {
+ expect(
+ resolveInboundEmailMessageReference({
+ s3Key: 'raw/legacy-key',
+ envelopeRecipients: ['[email protected]'],
+ }),
+ ).toEqual({ source: 'SES_S3', reference: 'raw/legacy-key' });
+ });
+});
diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/inbound-email/utils/resolve-inbound-email-message-reference.util.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/inbound-email/utils/resolve-inbound-email-message-reference.util.ts
new file mode 100644
index 00000000000..9f5ce60df9d
--- /dev/null
+++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/inbound-email/utils/resolve-inbound-email-message-reference.util.ts
@@ -0,0 +1,16 @@
+import { INBOUND_EMAIL_MESSAGE_SOURCE } from 'src/modules/messaging/message-import-manager/drivers/inbound-email/constants/inbound-email-message-source.constant';
+import { type InboundEmailMessageReference } from 'src/modules/messaging/message-import-manager/drivers/inbound-email/types/inbound-email-message-reference.type';
+import { type MessagingInboundEmailImportJobData } from 'src/modules/messaging/message-import-manager/jobs/messaging-inbound-email-import.job';
+
+export const resolveInboundEmailMessageReference = (
+ data: MessagingInboundEmailImportJobData,
+): InboundEmailMessageReference => {
+ if ('s3Key' in data) {
+ return {
+ source: INBOUND_EMAIL_MESSAGE_SOURCE.SES_S3,
+ reference: data.s3Key,
+ };
+ }
+
+ return { source: data.source, reference: data.reference };
+};
diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/jobs/messaging-inbound-email-import.job.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/jobs/messaging-inbound-email-import.job.ts
index f36b8a35981..5d22f14c211 100644
--- a/packages/twenty-server/src/modules/messaging/message-import-manager/jobs/messaging-inbound-email-import.job.ts
+++ b/packages/twenty-server/src/modules/messaging/message-import-manager/jobs/messaging-inbound-email-import.job.ts
@@ -4,11 +4,16 @@ import { Process } from 'src/engine/core-modules/message-queue/decorators/proces
import { Processor } from 'src/engine/core-modules/message-queue/decorators/processor.decorator';
import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
import { InboundEmailImportService } from 'src/modules/messaging/message-import-manager/drivers/inbound-email/services/inbound-email-import.service';
+import { type InboundEmailMessageSource } from 'src/modules/messaging/message-import-manager/drivers/inbound-email/types/inbound-email-message-source.type';
+import { resolveInboundEmailMessageReference } from 'src/modules/messaging/message-import-manager/drivers/inbound-email/utils/resolve-inbound-email-message-reference.util';
export type MessagingInboundEmailImportJobData = {
- s3Key: string;
envelopeRecipients: string[];
-};
+} & (
+ | { source: InboundEmailMessageSource; reference: string }
+ // legacy payload shape from jobs enqueued before the source abstraction
+ | { s3Key: string }
+);
@Processor({
queueName: MessageQueue.messagingQueue,
@@ -23,15 +28,15 @@ export class MessagingInboundEmailImportJob {
@Process(MessagingInboundEmailImportJob.name)
async handle(data: MessagingInboundEmailImportJobData): Promise<void> {
- const { s3Key, envelopeRecipients } = data;
+ const messageReference = resolveInboundEmailMessageReference(data);
const outcome = await this.inboundEmailImportService.importInboundMessage({
- s3Key,
- envelopeRecipients,
+ messageReference,
+ envelopeRecipients: data.envelopeRecipients,
});
this.logger.log(
- `Inbound email import outcome for ${s3Key}: ${outcome.kind}`,
+ `Inbound email import outcome for ${messageReference.reference}: ${outcome.kind}`,
);
}
}