Skip to content

The open alternative to Salesforce, designed for AI.

introduce MESSAGING_MESSAGES_GET_BATCH_SIZE as config variable (#19455)

neo773fda3eabb5f3e
diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/config-variables.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/config-variables.ts
index 346517abbb7..cc878ca9945 100644
--- a/packages/twenty-server/src/engine/core-modules/twenty-config/config-variables.ts
+++ b/packages/twenty-server/src/engine/core-modules/twenty-config/config-variables.ts
@@ -245,6 +245,16 @@ export class ConfigVariables {
})
MESSAGING_PROVIDER_MICROSOFT_ENABLED = false;
+ @ConfigVariablesMetadata({
+ group: ConfigVariablesGroup.ADVANCED_SETTINGS,
+ description:
+ 'Number of messages fetched per batch during message import, adjust incase of rate limiting caused by Gmail, Outlook or IMAP',
+ type: ConfigVariableType.NUMBER,
+ })
+ @CastToPositiveNumber()
+ @IsOptional()
+ MESSAGING_MESSAGES_GET_BATCH_SIZE = 400;
+
@ConfigVariablesMetadata({
group: ConfigVariablesGroup.MICROSOFT_AUTH,
description: 'Enable or disable the Microsoft Calendar integration',
diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/constants/messaging-gmail-users-messages-get-batch-size.constant.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/constants/messaging-gmail-users-messages-get-batch-size.constant.ts
deleted file mode 100644
index a8a4f4abfc0..00000000000
--- a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/constants/messaging-gmail-users-messages-get-batch-size.constant.ts
+++ /dev/null
@@ -1 +0,0 @@
-export const MESSAGING_GMAIL_USERS_MESSAGES_GET_BATCH_SIZE = 400;
diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/services/__tests__/messaging-messages-import.service.spec.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/services/__tests__/messaging-messages-import.service.spec.ts
index 02ab4e1889f..3d9fca638ec 100644
--- a/packages/twenty-server/src/modules/messaging/message-import-manager/services/__tests__/messaging-messages-import.service.spec.ts
+++ b/packages/twenty-server/src/modules/messaging/message-import-manager/services/__tests__/messaging-messages-import.service.spec.ts
@@ -15,7 +15,7 @@ import { EmailAliasManagerService } from 'src/modules/connected-account/email-al
import { ConnectedAccountRefreshTokensService } from 'src/modules/connected-account/refresh-tokens-manager/services/connected-account-refresh-tokens.service';
import { type ConnectedAccountEntity } from 'src/engine/metadata-modules/connected-account/entities/connected-account.entity';
import { MessageChannelSyncStatusService } from 'src/modules/messaging/common/services/message-channel-sync-status.service';
-import { MESSAGING_GMAIL_USERS_MESSAGES_GET_BATCH_SIZE } from 'src/modules/messaging/message-import-manager/drivers/gmail/constants/messaging-gmail-users-messages-get-batch-size.constant';
+import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';
import { MessagingAccountAuthenticationService } from 'src/modules/messaging/message-import-manager/services/messaging-account-authentication.service';
import { MessagingGetMessagesService } from 'src/modules/messaging/message-import-manager/services/messaging-get-messages.service';
import { MessageImportExceptionHandlerService } from 'src/modules/messaging/message-import-manager/services/messaging-import-exception-handler.service';
@@ -183,6 +183,12 @@ describe('MessagingMessagesImportService', () => {
findOne: jest.fn().mockResolvedValue({ userId: 'user-id' }),
},
},
+ {
+ provide: TwentyConfigService,
+ useValue: {
+ get: jest.fn().mockReturnValue(400),
+ },
+ },
];
const module: TestingModule = await Test.createTestingModule({
providers: [
@@ -281,7 +287,7 @@ describe('MessagingMessagesImportService', () => {
it('should process message batch import of more than MESSAGING_GMAIL_USERS_MESSAGES_GET_BATCH_SIZE successfully', async () => {
const arrayMessagesBig = Array.from(
- { length: MESSAGING_GMAIL_USERS_MESSAGES_GET_BATCH_SIZE + 1 },
+ { length: 401 },
(_, index) => `message-id-${index + 1}`,
);
diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-messages-import.service.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-messages-import.service.ts
index 281ecb33a6f..acd4f8d21c9 100644
--- a/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-messages-import.service.ts
+++ b/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-messages-import.service.ts
@@ -22,7 +22,7 @@ import {
MessageImportDriverException,
MessageImportDriverExceptionCode,
} from 'src/modules/messaging/message-import-manager/drivers/exceptions/message-import-driver.exception';
-import { MESSAGING_GMAIL_USERS_MESSAGES_GET_BATCH_SIZE } from 'src/modules/messaging/message-import-manager/drivers/gmail/constants/messaging-gmail-users-messages-get-batch-size.constant';
+import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';
import { MessagingAccountAuthenticationService } from 'src/modules/messaging/message-import-manager/services/messaging-account-authentication.service';
import { MessagingGetMessagesService } from 'src/modules/messaging/message-import-manager/services/messaging-get-messages.service';
import {
@@ -55,6 +55,7 @@ export class MessagingMessagesImportService {
private readonly messagingAccountAuthenticationService: MessagingAccountAuthenticationService,
@InjectRepository(UserWorkspaceEntity)
private readonly userWorkspaceRepository: Repository<UserWorkspaceEntity>,
+ private readonly twentyConfigService: TwentyConfigService,
) {}
async processMessageBatchImport(
@@ -64,6 +65,10 @@ export class MessagingMessagesImportService {
) {
let messageIdsToFetch: string[] = [];
+ const messagesGetBatchSize = this.twentyConfigService.get(
+ 'MESSAGING_MESSAGES_GET_BATCH_SIZE',
+ );
+
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => {
@@ -112,7 +117,7 @@ export class MessagingMessagesImportService {
messageIdsToFetch = await this.cacheStorage.setPop(
`messages-to-import:${workspaceId}:${messageChannel.id}`,
- MESSAGING_GMAIL_USERS_MESSAGES_GET_BATCH_SIZE,
+ messagesGetBatchSize,
);
if (!messageIdsToFetch?.length) {
@@ -210,10 +215,7 @@ export class MessagingMessagesImportService {
);
}
- if (
- messageIdsToFetch.length <
- MESSAGING_GMAIL_USERS_MESSAGES_GET_BATCH_SIZE
- ) {
+ if (messageIdsToFetch.length < messagesGetBatchSize) {
await this.messageChannelSyncStatusService.markAsCompletedAndMarkAsMessagesListFetchPending(
[messageChannel.id],
workspaceId,