Skip to content

The open alternative to Salesforce, designed for AI.

Fix lambda timeout diagnostics (#21002)

Thomas Trompette6e2cf4cbc0fc
diff --git a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda.driver.ts b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda.driver.ts
index 570fb6a0d4b..7e291edcb84 100644
--- a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda.driver.ts
+++ b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda.driver.ts
@@ -57,6 +57,12 @@ import {
} from 'src/engine/metadata-modules/logic-function/logic-function.exception';
import { type FlatLogicFunction } from 'src/engine/metadata-modules/logic-function/types/flat-logic-function.type';
+enum LambdaExecutionPhase {
+ BUILD = 'build',
+ FETCH_CODE = 'fetch-code',
+ INVOKE = 'invoke',
+}
+
const UPDATE_FUNCTION_DURATION_TIMEOUT_IN_SECONDS = 60;
const CREDENTIALS_DURATION_IN_SECONDS = 60 * 60; // 1h
const YARN_INSTALL_LAMBDA_TIMEOUT_SECONDS = 300;
@@ -982,8 +988,11 @@ export class LambdaDriver implements LogicFunctionDriver {
}> {
const lambdaExecutor = await this.getLambdaExecutor(flatLogicFunction);
+ const isActive = lambdaExecutor?.Configuration?.State === 'Active';
+
const canSkip =
isDefined(lambdaExecutor) &&
+ isActive &&
!flatApplication.isSdkLayerStale &&
this.hasExpectedLayers({
lambdaExecutor,
@@ -1156,41 +1165,49 @@ export class LambdaDriver implements LogicFunctionDriver {
env,
timeoutMs = 900_000,
}: LogicFunctionExecuteParams): Promise<LogicFunctionExecuteResult> {
- const buildStart = Date.now();
+ let currentPhase: LambdaExecutionPhase = LambdaExecutionPhase.BUILD;
+ let buildExecutorMs = 0;
+ let getBuiltCodeMs = 0;
- await this.buildLambdaExecutor({
- flatLogicFunction,
- flatApplication,
- applicationUniversalIdentifier,
- });
- const buildExecutorMs = Date.now() - buildStart;
+ try {
+ const buildStart = Date.now();
- const startTime = Date.now();
+ await this.buildLambdaExecutor({
+ flatLogicFunction,
+ flatApplication,
+ applicationUniversalIdentifier,
+ });
+ buildExecutorMs = Date.now() - buildStart;
- const compiledCode = await this.logicFunctionResourceService.getBuiltCode({
- workspaceId: flatLogicFunction.workspaceId,
- applicationUniversalIdentifier,
- builtHandlerPath: flatLogicFunction.builtHandlerPath,
- });
- const getBuiltCodeMs = Date.now() - startTime;
+ currentPhase = LambdaExecutionPhase.FETCH_CODE;
+ const fetchStart = Date.now();
- const executorPayload: LambdaDriverExecutorPayload = {
- params: payload,
- code: compiledCode,
- env: env ?? {},
- handlerName: flatLogicFunction.handlerName,
- };
+ const compiledCode = await this.logicFunctionResourceService.getBuiltCode(
+ {
+ workspaceId: flatLogicFunction.workspaceId,
+ applicationUniversalIdentifier,
+ builtHandlerPath: flatLogicFunction.builtHandlerPath,
+ },
+ );
+ getBuiltCodeMs = Date.now() - fetchStart;
- const payloadString = JSON.stringify(executorPayload);
- const params: InvokeCommandInput = {
- FunctionName: flatLogicFunction.id,
- Payload: payloadString,
- LogType: LogType.Tail,
- };
+ currentPhase = LambdaExecutionPhase.INVOKE;
- const command = new InvokeCommand(params);
+ const executorPayload: LambdaDriverExecutorPayload = {
+ params: payload,
+ code: compiledCode,
+ env: env ?? {},
+ handlerName: flatLogicFunction.handlerName,
+ };
- try {
+ const payloadString = JSON.stringify(executorPayload);
+ const params: InvokeCommandInput = {
+ FunctionName: flatLogicFunction.id,
+ Payload: payloadString,
+ LogType: LogType.Tail,
+ };
+
+ const command = new InvokeCommand(params);
const lambdaClient = await this.getLambdaClient();
const invokeStart = Date.now();
@@ -1211,7 +1228,7 @@ export class LambdaDriver implements LogicFunctionDriver {
coldStart,
} = this.parseLambdaLogResult(result.LogResult);
- const duration = Date.now() - startTime;
+ const duration = Date.now() - fetchStart;
this.logger.log(
`[lambda-timing] fnId=${flatLogicFunction.id} totalMs=${Date.now() - buildStart} buildExecutorMs=${buildExecutorMs} getBuiltCodeMs=${getBuiltCodeMs} payloadBytes=${Buffer.byteLength(payloadString, 'utf8')} invokeSendMs=${invokeSendMs} reportDurationMs=${reportDurationMs ?? 'n/a'} billedMs=${billedDurationMs ?? 'n/a'} initDurationMs=${initDurationMs ?? 'n/a'} coldStart=${coldStart}`,
@@ -1234,8 +1251,10 @@ export class LambdaDriver implements LogicFunctionDriver {
status: LogicFunctionExecutionStatus.SUCCESS,
};
} catch (error) {
+ const phaseTiming = `phase=${currentPhase} buildMs=${buildExecutorMs} fetchCodeMs=${getBuiltCodeMs}`;
+
this.logger.error(
- `Lambda invocation failed for function ${flatLogicFunction.id}: ${error instanceof Error ? error.message : String(error)}`,
+ `Lambda invocation failed for function ${flatLogicFunction.id} [${phaseTiming}]: ${error instanceof Error ? error.message : String(error)}`,
error instanceof Error ? error.stack : undefined,
);
@@ -1247,8 +1266,13 @@ export class LambdaDriver implements LogicFunctionDriver {
}
if (error instanceof Error && error.name === 'TimeoutError') {
+ const executor = await this.getLambdaExecutor(flatLogicFunction).catch(
+ () => undefined,
+ );
+ const functionState = executor?.Configuration?.State ?? 'unknown';
+
throw new LogicFunctionException(
- `Lambda invocation timed out for function '${flatLogicFunction.id}': ${error.message}`,
+ `Lambda timed out for function '${flatLogicFunction.id}' during ${currentPhase} (functionState=${functionState}, ${phaseTiming})`,
LogicFunctionExceptionCode.LOGIC_FUNCTION_EXECUTION_TIMEOUT,
);
}
@@ -1258,7 +1282,7 @@ export class LambdaDriver implements LogicFunctionDriver {
}
throw new LogicFunctionException(
- `Lambda invocation failed for function '${flatLogicFunction.id}': ${error instanceof Error ? error.message : 'Unknown error'}`,
+ `Lambda invocation failed for function '${flatLogicFunction.id}' during ${currentPhase}: ${error instanceof Error ? error.message : 'Unknown error'}`,
LogicFunctionExceptionCode.LOGIC_FUNCTION_EXECUTION_FAILED,
);
}