The open alternative to Salesforce, designed for AI.
TypeScript crmcrm-systemcustomergood-first-issuegraphqlhacktoberfestjavascriptmarketingmonoreponestjsopen-sourcepostgresqlreactreactjssalestypescriptweb
introduce MESSAGING_MESSAGES_GET_BATCH_SIZE as config variable (#19455)
neo773fda3eabb5f3e
diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/config-variables.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/config-variables.tsindex 346517abbb7..cc878ca9945 100644--- a/packages/twenty-server/src/engine/core-modules/twenty-config/config-variables.ts+++ b/packages/twenty-server/src/engine/core-modules/twenty-config/config-variables.ts@@ -245,6 +245,16 @@ export class ConfigVariables {})MESSAGING_PROVIDER_MICROSOFT_ENABLED = false;+ @ConfigVariablesMetadata({+ group: ConfigVariablesGroup.ADVANCED_SETTINGS,+ description:+ 'Number of messages fetched per batch during message import, adjust incase of rate limiting caused by Gmail, Outlook or IMAP',+ type: ConfigVariableType.NUMBER,+ })+ @CastToPositiveNumber()+ @IsOptional()+ MESSAGING_MESSAGES_GET_BATCH_SIZE = 400;+@ConfigVariablesMetadata({group: ConfigVariablesGroup.MICROSOFT_AUTH,description: 'Enable or disable the Microsoft Calendar integration',diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/constants/messaging-gmail-users-messages-get-batch-size.constant.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/constants/messaging-gmail-users-messages-get-batch-size.constant.tsdeleted file mode 100644index a8a4f4abfc0..00000000000--- a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/constants/messaging-gmail-users-messages-get-batch-size.constant.ts+++ /dev/null@@ -1 +0,0 @@-export const MESSAGING_GMAIL_USERS_MESSAGES_GET_BATCH_SIZE = 400;diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/services/__tests__/messaging-messages-import.service.spec.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/services/__tests__/messaging-messages-import.service.spec.tsindex 02ab4e1889f..3d9fca638ec 100644--- a/packages/twenty-server/src/modules/messaging/message-import-manager/services/__tests__/messaging-messages-import.service.spec.ts+++ b/packages/twenty-server/src/modules/messaging/message-import-manager/services/__tests__/messaging-messages-import.service.spec.ts@@ -15,7 +15,7 @@ import { EmailAliasManagerService } from 'src/modules/connected-account/email-alimport { ConnectedAccountRefreshTokensService } from 'src/modules/connected-account/refresh-tokens-manager/services/connected-account-refresh-tokens.service';import { type ConnectedAccountEntity } from 'src/engine/metadata-modules/connected-account/entities/connected-account.entity';import { MessageChannelSyncStatusService } from 'src/modules/messaging/common/services/message-channel-sync-status.service';-import { MESSAGING_GMAIL_USERS_MESSAGES_GET_BATCH_SIZE } from 'src/modules/messaging/message-import-manager/drivers/gmail/constants/messaging-gmail-users-messages-get-batch-size.constant';+import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';import { MessagingAccountAuthenticationService } from 'src/modules/messaging/message-import-manager/services/messaging-account-authentication.service';import { MessagingGetMessagesService } from 'src/modules/messaging/message-import-manager/services/messaging-get-messages.service';import { MessageImportExceptionHandlerService } from 'src/modules/messaging/message-import-manager/services/messaging-import-exception-handler.service';@@ -183,6 +183,12 @@ describe('MessagingMessagesImportService', () => {findOne: jest.fn().mockResolvedValue({ userId: 'user-id' }),},},+ {+ provide: TwentyConfigService,+ useValue: {+ get: jest.fn().mockReturnValue(400),+ },+ },];const module: TestingModule = await Test.createTestingModule({providers: [@@ -281,7 +287,7 @@ describe('MessagingMessagesImportService', () => {it('should process message batch import of more than MESSAGING_GMAIL_USERS_MESSAGES_GET_BATCH_SIZE successfully', async () => {const arrayMessagesBig = Array.from(- { length: MESSAGING_GMAIL_USERS_MESSAGES_GET_BATCH_SIZE + 1 },+ { length: 401 },(_, index) => `message-id-${index + 1}`,);diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-messages-import.service.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-messages-import.service.tsindex 281ecb33a6f..acd4f8d21c9 100644--- a/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-messages-import.service.ts+++ b/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-messages-import.service.ts@@ -22,7 +22,7 @@ import {MessageImportDriverException,MessageImportDriverExceptionCode,} from 'src/modules/messaging/message-import-manager/drivers/exceptions/message-import-driver.exception';-import { MESSAGING_GMAIL_USERS_MESSAGES_GET_BATCH_SIZE } from 'src/modules/messaging/message-import-manager/drivers/gmail/constants/messaging-gmail-users-messages-get-batch-size.constant';+import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';import { MessagingAccountAuthenticationService } from 'src/modules/messaging/message-import-manager/services/messaging-account-authentication.service';import { MessagingGetMessagesService } from 'src/modules/messaging/message-import-manager/services/messaging-get-messages.service';import {@@ -55,6 +55,7 @@ export class MessagingMessagesImportService {private readonly messagingAccountAuthenticationService: MessagingAccountAuthenticationService,@InjectRepository(UserWorkspaceEntity)private readonly userWorkspaceRepository: Repository<UserWorkspaceEntity>,+ private readonly twentyConfigService: TwentyConfigService,) {}async processMessageBatchImport(@@ -64,6 +65,10 @@ export class MessagingMessagesImportService {) {let messageIdsToFetch: string[] = [];+ const messagesGetBatchSize = this.twentyConfigService.get(+ 'MESSAGING_MESSAGES_GET_BATCH_SIZE',+ );+const authContext = buildSystemAuthContext(workspaceId);await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => {@@ -112,7 +117,7 @@ export class MessagingMessagesImportService {messageIdsToFetch = await this.cacheStorage.setPop(`messages-to-import:${workspaceId}:${messageChannel.id}`,- MESSAGING_GMAIL_USERS_MESSAGES_GET_BATCH_SIZE,+ messagesGetBatchSize,);if (!messageIdsToFetch?.length) {@@ -210,10 +215,7 @@ export class MessagingMessagesImportService {);}- if (- messageIdsToFetch.length <- MESSAGING_GMAIL_USERS_MESSAGES_GET_BATCH_SIZE- ) {+ if (messageIdsToFetch.length < messagesGetBatchSize) {await this.messageChannelSyncStatusService.markAsCompletedAndMarkAsMessagesListFetchPending([messageChannel.id],workspaceId,