Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(evals): classify reasonless runtime exits as timeouts (#17238)

Tobias Wochinger7d5bdcca4127
diff --git a/packages/shared/src/server/evals/awsLambdaCodeEvalDispatcher.test.ts b/packages/shared/src/server/evals/awsLambdaCodeEvalDispatcher.test.ts
index f72d8f8e25..bb79c2b8d0 100644
--- a/packages/shared/src/server/evals/awsLambdaCodeEvalDispatcher.test.ts
+++ b/packages/shared/src/server/evals/awsLambdaCodeEvalDispatcher.test.ts
@@ -194,6 +194,39 @@ describe("AwsLambdaCodeEvalDispatcher observability", () => {
});
});
+ it("classifies Lambda runtime exits without a reason as timeouts", async () => {
+ const dispatcher = new AwsLambdaCodeEvalDispatcher({
+ lambdaClient: {
+ send: vi.fn().mockResolvedValue({
+ StatusCode: 200,
+ FunctionError: "Unhandled",
+ Payload: Buffer.from(
+ JSON.stringify({
+ errorType: "Runtime.ExitError",
+ errorMessage:
+ "RequestId: 35e2f5d1-5f0b-459e-ba5a-c6c1a9475806 Error: Runtime exited without providing a reason",
+ }),
+ ),
+ $metadata: {
+ requestId: "request-timeout",
+ httpStatusCode: 200,
+ attempts: 1,
+ totalRetryDelay: 0,
+ },
+ }),
+ } as any,
+ });
+
+ await expect(dispatcher.dispatch(baseInput)).rejects.toMatchObject({
+ code: "TIMEOUT",
+ retryable: false,
+ });
+ expectSpanAttributes({
+ "langfuse.code_eval.error.code": "TIMEOUT",
+ "langfuse.code_eval.error.retryable": false,
+ });
+ });
+
it.each([
{
errorType: "Runtime.OutOfMemory",
diff --git a/packages/shared/src/server/evals/awsLambdaCodeEvalDispatcher.ts b/packages/shared/src/server/evals/awsLambdaCodeEvalDispatcher.ts
index 99c21d3869..3dbf626737 100644
--- a/packages/shared/src/server/evals/awsLambdaCodeEvalDispatcher.ts
+++ b/packages/shared/src/server/evals/awsLambdaCodeEvalDispatcher.ts
@@ -41,7 +41,6 @@ type CodeEvalDispatcherErrorClassification = {
// this is only consulted if a future runner surfaces one of them via the
// user-code-error envelope.
const RETRYABLE_ERROR_CODES = new Set<CodeEvalDispatcherErrorCode>([
- CodeEvalDispatcherErrorCodes.TIMEOUT,
CodeEvalDispatcherErrorCodes.LAMBDA_CONCURRENCY_LIMIT,
CodeEvalDispatcherErrorCodes.LAMBDA_INVOCATION_ERROR,
]);
@@ -422,11 +421,14 @@ function classifyLambdaFunctionError(params: {
if (
errorType === "Function.TimedOut" ||
errorType === "Sandbox.Timedout" ||
- (errorMessage && isTimeoutErrorMessage(errorMessage))
+ (errorMessage && isTimeoutErrorMessage(errorMessage)) ||
+ (errorType === "Runtime.ExitError" &&
+ errorMessage !== null &&
+ /runtime exited without providing a reason/i.test(errorMessage))
) {
return new CodeEvalDispatcherError(
composedMessage || "Lambda task timed out",
- { code: CodeEvalDispatcherErrorCodes.TIMEOUT, retryable: true },
+ { code: CodeEvalDispatcherErrorCodes.TIMEOUT, retryable: false },
);
}
diff --git a/packages/shared/src/server/evals/localCodeEvalDispatcher.ts b/packages/shared/src/server/evals/localCodeEvalDispatcher.ts
index b6995d0f7c..1525e1197e 100644
--- a/packages/shared/src/server/evals/localCodeEvalDispatcher.ts
+++ b/packages/shared/src/server/evals/localCodeEvalDispatcher.ts
@@ -99,7 +99,7 @@ if (typeof evaluate !== "function") {
? CodeEvalDispatcherErrorCodes.TIMEOUT
: CodeEvalDispatcherErrorCodes.USER_CODE_ERROR,
cause: error,
- retryable: message.includes(SCRIPT_TIMEOUT_MESSAGE),
+ retryable: false,
});
} finally {
if (timeoutId) clearTimeout(timeoutId);
diff --git a/worker/src/features/evaluation/codeBased/awsLambdaCodeEvalDispatcher.test.ts b/worker/src/features/evaluation/codeBased/awsLambdaCodeEvalDispatcher.test.ts
index 91c2a956f4..dbfd0c55b4 100644
--- a/worker/src/features/evaluation/codeBased/awsLambdaCodeEvalDispatcher.test.ts
+++ b/worker/src/features/evaluation/codeBased/awsLambdaCodeEvalDispatcher.test.ts
@@ -196,7 +196,7 @@ describe("AwsLambdaCodeEvalDispatcher", () => {
} satisfies Partial<CodeEvalDispatcherError>);
});
- it("classifies timeouts via errorType Function.TimedOut as retryable", async () => {
+ it("classifies timeouts via errorType Function.TimedOut as non-retryable", async () => {
// Envelope shape captured empirically from Floci probes.
const send = vi.fn().mockResolvedValue({
FunctionError: "Unhandled",
@@ -214,11 +214,11 @@ describe("AwsLambdaCodeEvalDispatcher", () => {
await expect(dispatcher.dispatch(baseInput)).rejects.toMatchObject({
code: "TIMEOUT",
message: "Function.TimedOut: Task timed out after 30 seconds",
- retryable: true,
+ retryable: false,
} satisfies Partial<CodeEvalDispatcherError>);
});
- it("classifies timeouts via errorType Sandbox.Timedout as retryable", async () => {
+ it("classifies timeouts via errorType Sandbox.Timedout as non-retryable", async () => {
const send = vi.fn().mockResolvedValue({
FunctionError: "Unhandled",
Payload: Buffer.from(
@@ -235,11 +235,11 @@ describe("AwsLambdaCodeEvalDispatcher", () => {
await expect(dispatcher.dispatch(baseInput)).rejects.toMatchObject({
code: "TIMEOUT",
message: "Sandbox.Timedout: Task timed out after 30 seconds",
- retryable: true,
+ retryable: false,
} satisfies Partial<CodeEvalDispatcherError>);
});
- it("classifies timeouts via errorMessage substring as retryable", async () => {
+ it("classifies timeouts via errorMessage substring as non-retryable", async () => {
// Defensive fallback for runtimes that don't populate errorType.
const send = vi.fn().mockResolvedValue({
FunctionError: "Unhandled",
@@ -251,7 +251,7 @@ describe("AwsLambdaCodeEvalDispatcher", () => {
await expect(dispatcher.dispatch(baseInput)).rejects.toMatchObject({
code: "TIMEOUT",
- retryable: true,
+ retryable: false,
} satisfies Partial<CodeEvalDispatcherError>);
});
diff --git a/worker/src/features/evaluation/codeBased/executeCodeBasedEvaluation.test.ts b/worker/src/features/evaluation/codeBased/executeCodeBasedEvaluation.test.ts
index a323607060..ad42ab3a7a 100644
--- a/worker/src/features/evaluation/codeBased/executeCodeBasedEvaluation.test.ts
+++ b/worker/src/features/evaluation/codeBased/executeCodeBasedEvaluation.test.ts
@@ -613,7 +613,7 @@ describe("executeCodeBasedEvaluation", () => {
"Function.TimedOut: Task timed out after 2 seconds";
const error = new CodeEvalDispatcherError(rawTimeoutMessage, {
code: CodeEvalDispatcherErrorCodes.TIMEOUT,
- retryable: true,
+ retryable: false,
});
mocks.dispatcher.dispatch.mockRejectedValue(error);
@@ -645,7 +645,7 @@ describe("executeCodeBasedEvaluation", () => {
await expect(promise).rejects.toThrow(CodeEvalExecutionError);
await expect(promise).rejects.toMatchObject({
code: CodeEvalDispatcherErrorCodes.TIMEOUT,
- retryable: true,
+ retryable: false,
});
await expect(promise).rejects.toThrow("Evaluator timed out.");
diff --git a/worker/src/features/evaluation/codeBased/localCodeEvalDispatcher.test.ts b/worker/src/features/evaluation/codeBased/localCodeEvalDispatcher.test.ts
index 183883a298..3da2b935ac 100644
--- a/worker/src/features/evaluation/codeBased/localCodeEvalDispatcher.test.ts
+++ b/worker/src/features/evaluation/codeBased/localCodeEvalDispatcher.test.ts
@@ -193,7 +193,7 @@ describe("LocalCodeEvalDispatcher", () => {
}),
).rejects.toMatchObject({
code: "TIMEOUT",
- retryable: true,
+ retryable: false,
} satisfies Partial<CodeEvalDispatcherError>);
});
@@ -221,7 +221,7 @@ describe("LocalCodeEvalDispatcher", () => {
expect(result).toBeInstanceOf(CodeEvalDispatcherError);
expect(result).toMatchObject({
code: "TIMEOUT",
- retryable: true,
+ retryable: false,
} satisfies Partial<CodeEvalDispatcherError>);
});
diff --git a/worker/src/queues/__tests__/codeEvalExecutionQueueProcessor.test.ts b/worker/src/queues/__tests__/codeEvalExecutionQueueProcessor.test.ts
index 6c59fef07c..4222dd034c 100644
--- a/worker/src/queues/__tests__/codeEvalExecutionQueueProcessor.test.ts
+++ b/worker/src/queues/__tests__/codeEvalExecutionQueueProcessor.test.ts
@@ -324,21 +324,19 @@ describe("codeEvalExecutionQueueProcessor", () => {
);
});
- it("should preserve retryable code eval timeout messages on the final retry attempt", async () => {
+ it("should treat code eval timeouts as terminal without retrying", async () => {
const timeoutMessage =
"Evaluator timed out. Code-based evaluators must complete within the configured runtime limit. Long executions can be caused by network calls, which are forbidden and may never complete. Remove network calls, optimize your evaluator code, and try again. See https://langfuse.com/docs/evaluation/evaluation-methods/code-evaluators for details.";
const error = new CodeEvalExecutionError({
code: CodeEvalDispatcherErrorCodes.TIMEOUT,
message: timeoutMessage,
- retryable: true,
+ retryable: false,
});
(processObservationEval as Mock).mockRejectedValue(error);
await expect(
- codeEvalExecutionQueueProcessor(
- createMockJob({ attemptsMade: 9, opts: { attempts: 10 } }),
- ),
- ).rejects.toThrow(error);
+ codeEvalExecutionQueueProcessor(createMockJob()),
+ ).resolves.toBeUndefined();
expect(prisma.jobExecution.update).toHaveBeenCalledWith({
where: {
@@ -352,7 +350,7 @@ describe("codeEvalExecutionQueueProcessor", () => {
executionTraceId: "test-trace-id",
},
});
- expect(traceException).toHaveBeenCalledWith(error);
+ expect(traceException).not.toHaveBeenCalled();
expect(recordIncrement).toHaveBeenCalledWith(
"langfuse.evaluation.execution.terminal",
1,