Skip to content

The open alternative to Salesforce, designed for AI.

Give AWS SDK clients an explicit request timeout (#23857)

Charles Bochetf5a42cdbae92
diff --git a/packages/twenty-server/src/engine/core-modules/emailing-domain/drivers/aws-ses/providers/aws-ses-client.provider.ts b/packages/twenty-server/src/engine/core-modules/emailing-domain/drivers/aws-ses/providers/aws-ses-client.provider.ts
index 232e1126667..ad27eb1872e 100644
--- a/packages/twenty-server/src/engine/core-modules/emailing-domain/drivers/aws-ses/providers/aws-ses-client.provider.ts
+++ b/packages/twenty-server/src/engine/core-modules/emailing-domain/drivers/aws-ses/providers/aws-ses-client.provider.ts
@@ -6,6 +6,7 @@ import {
} from '@aws-sdk/client-sesv2';
import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';
+import { buildAwsRequestHandlerOptions } from 'src/utils/aws-request-handler.util';
@Injectable()
export class AwsSesClientProvider {
@@ -17,6 +18,7 @@ export class AwsSesClientProvider {
if (!this.sesClient) {
const config: SESClientConfig = {
region: this.twentyConfigService.get('AWS_SES_REGION'),
+ requestHandler: buildAwsRequestHandlerOptions(),
};
const accessKeyId = this.twentyConfigService.get('AWS_SES_ACCESS_KEY_ID');
diff --git a/packages/twenty-server/src/engine/core-modules/file-storage/constants/s3-client-timeouts.constant.ts b/packages/twenty-server/src/engine/core-modules/file-storage/constants/s3-client-timeouts.constant.ts
new file mode 100644
index 00000000000..d269f9a9200
--- /dev/null
+++ b/packages/twenty-server/src/engine/core-modules/file-storage/constants/s3-client-timeouts.constant.ts
@@ -0,0 +1,3 @@
+export const FILE_STORAGE_S3_REQUEST_TIMEOUT_MS = 5 * 60 * 1000;
+export const FILE_STORAGE_S3_CONNECTION_TIMEOUT_MS = 30 * 1000;
+export const FILE_STORAGE_S3_MAX_SOCKETS = 200;
diff --git a/packages/twenty-server/src/engine/core-modules/file-storage/drivers/s3.driver.ts b/packages/twenty-server/src/engine/core-modules/file-storage/drivers/s3.driver.ts
index 42e0ba5543d..1ced1877cc4 100644
--- a/packages/twenty-server/src/engine/core-modules/file-storage/drivers/s3.driver.ts
+++ b/packages/twenty-server/src/engine/core-modules/file-storage/drivers/s3.driver.ts
@@ -24,11 +24,17 @@ import { Upload } from '@aws-sdk/lib-storage';
import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
import { isDefined } from 'twenty-shared/utils';
+import {
+ FILE_STORAGE_S3_CONNECTION_TIMEOUT_MS,
+ FILE_STORAGE_S3_MAX_SOCKETS,
+ FILE_STORAGE_S3_REQUEST_TIMEOUT_MS,
+} from 'src/engine/core-modules/file-storage/constants/s3-client-timeouts.constant';
import { type StorageDriver } from 'src/engine/core-modules/file-storage/drivers/interfaces/storage-driver.interface';
import {
FileStorageException,
FileStorageExceptionCode,
} from 'src/engine/core-modules/file-storage/interfaces/file-storage-exception';
+import { buildAwsRequestHandlerOptions } from 'src/utils/aws-request-handler.util';
export interface S3DriverOptions extends S3ClientConfig {
bucketName: string;
@@ -58,12 +64,23 @@ export class S3Driver implements StorageDriver {
return;
}
- this.s3Client = new S3({ ...s3Options, region, endpoint });
+ const requestHandler = buildAwsRequestHandlerOptions({
+ requestTimeoutMs: FILE_STORAGE_S3_REQUEST_TIMEOUT_MS,
+ connectionTimeoutMs: FILE_STORAGE_S3_CONNECTION_TIMEOUT_MS,
+ maxSockets: FILE_STORAGE_S3_MAX_SOCKETS,
+ });
+
+ this.s3Client = new S3({ ...s3Options, region, endpoint, requestHandler });
this.bucketName = bucketName;
if (presignEnabled) {
this.presignClient = presignEndpoint
- ? new S3({ ...s3Options, region, endpoint: presignEndpoint })
+ ? new S3({
+ ...s3Options,
+ region,
+ endpoint: presignEndpoint,
+ requestHandler,
+ })
: this.s3Client;
}
}
diff --git a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/constants/lambda-driver.constant.ts b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/constants/lambda-driver.constant.ts
index bfc51baa26a..db962e44024 100644
--- a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/constants/lambda-driver.constant.ts
+++ b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/constants/lambda-driver.constant.ts
@@ -16,6 +16,11 @@ export const EXECUTOR_LAMBDA_MEMORY_MB = 512;
export const EXECUTOR_LAMBDA_TIMEOUT_SECONDS = 900;
export const LAMBDA_EPHEMERAL_STORAGE_MB = 4096;
+export const LAMBDA_CLIENT_REQUEST_TIMEOUT_MS =
+ (EXECUTOR_LAMBDA_TIMEOUT_SECONDS + 60) * 1000;
+export const LAMBDA_CLIENT_MAX_SOCKETS = 500;
+export const LAMBDA_CLIENT_CONNECTION_TIMEOUT_MS = 60_000;
+
export const COMMON_LAYER_NAME_PREFIX = 'twenty-common-layer';
export const YARN_INSTALL_FUNCTION_NAME_PREFIX = 'twenty-yarn-install';
export const BUILDER_FUNCTION_NAME_PREFIX = 'twenty-builder';
diff --git a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/services/lambda-aws-client.service.ts b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/services/lambda-aws-client.service.ts
index f95f23247f1..7d49ae371bd 100644
--- a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/services/lambda-aws-client.service.ts
+++ b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/services/lambda-aws-client.service.ts
@@ -11,14 +11,20 @@ import { isDefined } from 'twenty-shared/utils';
import {
CREDENTIALS_DURATION_IN_SECONDS,
+ LAMBDA_CLIENT_CONNECTION_TIMEOUT_MS,
LAMBDA_CLIENT_MAX_ATTEMPTS,
+ LAMBDA_CLIENT_MAX_SOCKETS,
+ LAMBDA_CLIENT_REQUEST_TIMEOUT_MS,
LAMBDA_CLIENT_RETRY_MODE,
UPDATE_FUNCTION_DURATION_TIMEOUT_IN_SECONDS,
} from 'src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/constants/lambda-driver.constant';
import { type LambdaDriverOptions } from 'src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/types/lambda-driver.type';
+import { buildAwsRequestHandlerOptions } from 'src/utils/aws-request-handler.util';
export class LambdaAwsClientService {
private lambdaClient: Lambda | undefined;
+ private s3Client: S3Client | undefined;
+ private stsClient: STSClient | undefined;
private assumeRoleCredentials:
| { accessKeyId: string; secretAccessKey: string; sessionToken: string }
| undefined;
@@ -39,22 +45,40 @@ export class LambdaAwsClientService {
}),
maxAttempts: LAMBDA_CLIENT_MAX_ATTEMPTS,
retryMode: LAMBDA_CLIENT_RETRY_MODE,
+ requestHandler: buildAwsRequestHandlerOptions({
+ requestTimeoutMs: LAMBDA_CLIENT_REQUEST_TIMEOUT_MS,
+ connectionTimeoutMs: LAMBDA_CLIENT_CONNECTION_TIMEOUT_MS,
+ maxSockets: LAMBDA_CLIENT_MAX_SOCKETS,
+ }),
});
}
return this.lambdaClient;
}
+ private async getS3Client(): Promise<S3Client> {
+ if (
+ !isDefined(this.s3Client) ||
+ (isDefined(this.options.subhostingRole) &&
+ this.areAssumeRoleCredentialsExpired())
+ ) {
+ this.s3Client = new S3Client({
+ region: this.options.layerBucketRegion,
+ credentials: isDefined(this.options.subhostingRole)
+ ? await this.getAssumeRoleCredentials()
+ : this.options.credentials,
+ requestHandler: buildAwsRequestHandlerOptions(),
+ });
+ }
+
+ return this.s3Client;
+ }
+
async generatePresignedUploadUrl(
s3Key: string,
expiresIn: number = 300,
): Promise<string> {
- const s3Client = new S3Client({
- region: this.options.layerBucketRegion,
- credentials: isDefined(this.options.subhostingRole)
- ? await this.getAssumeRoleCredentials()
- : this.options.credentials,
- });
+ const s3Client = await this.getS3Client();
const putCommand = new PutObjectCommand({
Bucket: this.options.layerBucket,
@@ -107,7 +131,12 @@ export class LambdaAwsClientService {
}
private async refreshAssumeRoleCredentials() {
- const stsClient = new STSClient({ region: this.options.region });
+ this.stsClient ??= new STSClient({
+ region: this.options.region,
+ requestHandler: buildAwsRequestHandlerOptions(),
+ });
+
+ const stsClient = this.stsClient;
const assumeRoleCommand = new AssumeRoleCommand({
RoleArn: this.options.subhostingRole,
@@ -137,6 +166,7 @@ export class LambdaAwsClientService {
);
this.lambdaClient = undefined;
+ this.s3Client = undefined;
}
private async getAssumeRoleCredentials() {
diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/inbound-email/providers/inbound-email-s3-client.provider.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/inbound-email/providers/inbound-email-s3-client.provider.ts
index 2ea87cd54d6..eccbd93b458 100644
--- a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/inbound-email/providers/inbound-email-s3-client.provider.ts
+++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/inbound-email/providers/inbound-email-s3-client.provider.ts
@@ -5,6 +5,7 @@ import { isNonEmptyString } from '@sniptt/guards';
import { StorageDriverType } from 'src/engine/core-modules/file-storage/interfaces/file-storage.interface';
import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';
+import { buildAwsRequestHandlerOptions } from 'src/utils/aws-request-handler.util';
@Injectable()
export class InboundEmailS3ClientProvider {
@@ -54,7 +55,10 @@ export class InboundEmailS3ClientProvider {
throw new Error('STORAGE_S3_REGION must be set to use email group.');
}
- const config: S3ClientConfig = { region };
+ const config: S3ClientConfig = {
+ region,
+ requestHandler: buildAwsRequestHandlerOptions(),
+ };
const endpoint = this.twentyConfigService.get('STORAGE_S3_ENDPOINT');
diff --git a/packages/twenty-server/src/utils/__tests__/aws-request-handler.util.spec.ts b/packages/twenty-server/src/utils/__tests__/aws-request-handler.util.spec.ts
new file mode 100644
index 00000000000..da30e45bf15
--- /dev/null
+++ b/packages/twenty-server/src/utils/__tests__/aws-request-handler.util.spec.ts
@@ -0,0 +1,47 @@
+import {
+ AWS_DEFAULT_CONNECTION_TIMEOUT_MS,
+ AWS_DEFAULT_MAX_SOCKETS,
+ AWS_DEFAULT_REQUEST_TIMEOUT_MS,
+ buildAwsRequestHandlerOptions,
+} from 'src/utils/aws-request-handler.util';
+
+describe('buildAwsRequestHandlerOptions', () => {
+ it('always sets a non-zero request timeout', () => {
+ expect(buildAwsRequestHandlerOptions().requestTimeout).toBeGreaterThan(0);
+ });
+
+ it('always throws on request timeout rather than only warning', () => {
+ expect(buildAwsRequestHandlerOptions().throwOnRequestTimeout).toBe(true);
+ });
+
+ it('always sets a non-zero connection timeout', () => {
+ expect(buildAwsRequestHandlerOptions().connectionTimeout).toBeGreaterThan(
+ 0,
+ );
+ });
+
+ it('applies the defaults when called with no arguments', () => {
+ expect(buildAwsRequestHandlerOptions()).toEqual({
+ connectionTimeout: AWS_DEFAULT_CONNECTION_TIMEOUT_MS,
+ requestTimeout: AWS_DEFAULT_REQUEST_TIMEOUT_MS,
+ throwOnRequestTimeout: true,
+ httpsAgent: { maxSockets: AWS_DEFAULT_MAX_SOCKETS },
+ });
+ });
+
+ it('allows callers with long-running requests to raise the timeouts', () => {
+ const options = buildAwsRequestHandlerOptions({
+ requestTimeoutMs: 960_000,
+ connectionTimeoutMs: 60_000,
+ });
+
+ expect(options.requestTimeout).toBe(960_000);
+ expect(options.connectionTimeout).toBe(60_000);
+ });
+
+ it('allows the socket pool size to be raised above the SDK default', () => {
+ expect(
+ buildAwsRequestHandlerOptions({ maxSockets: 500 }).httpsAgent.maxSockets,
+ ).toBe(500);
+ });
+});
diff --git a/packages/twenty-server/src/utils/aws-request-handler.util.ts b/packages/twenty-server/src/utils/aws-request-handler.util.ts
new file mode 100644
index 00000000000..c24bdb789b7
--- /dev/null
+++ b/packages/twenty-server/src/utils/aws-request-handler.util.ts
@@ -0,0 +1,20 @@
+export const AWS_DEFAULT_CONNECTION_TIMEOUT_MS = 10_000;
+export const AWS_DEFAULT_REQUEST_TIMEOUT_MS = 30_000;
+export const AWS_DEFAULT_MAX_SOCKETS = 100;
+
+type AwsRequestHandlerOptions = {
+ requestTimeoutMs?: number;
+ connectionTimeoutMs?: number;
+ maxSockets?: number;
+};
+
+export const buildAwsRequestHandlerOptions = ({
+ requestTimeoutMs = AWS_DEFAULT_REQUEST_TIMEOUT_MS,
+ connectionTimeoutMs = AWS_DEFAULT_CONNECTION_TIMEOUT_MS,
+ maxSockets = AWS_DEFAULT_MAX_SOCKETS,
+}: AwsRequestHandlerOptions = {}) => ({
+ connectionTimeout: connectionTimeoutMs,
+ requestTimeout: requestTimeoutMs,
+ throwOnRequestTimeout: true,
+ httpsAgent: { maxSockets },
+});