Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(evals): classify code evaluator OOM as customer error (#16897)

Tobias Wochingere23a98f400c5
diff --git a/packages/shared/AGENTS.md b/packages/shared/AGENTS.md
index e1ee201e1f..adc165e5de 100644
--- a/packages/shared/AGENTS.md
+++ b/packages/shared/AGENTS.md
@@ -23,6 +23,7 @@
- Repository layer: `src/server/repositories/*`
- Queue payload schemas: `src/server/queues.ts`
- Queue helpers: `src/server/redis/*`
+- Code evaluator dispatcher/error contract: `src/server/evals/codeEvalDispatcherTypes.ts`. Keep provider mappings, user-visible messages, and worker terminal-outcome classification aligned when adding an error code.
- Dashboard/monitor query feature (data model + server-only builder/executor): `src/features/query/*`
- Query-builder AST (server half, WIP): `src/server/query-ast/*` — golden-SQL
recording/diff harness that captures the current SQL at the
diff --git a/packages/shared/src/server/evals/awsLambdaCodeEvalDispatcher.test.ts b/packages/shared/src/server/evals/awsLambdaCodeEvalDispatcher.test.ts
index c905a5c2c0..f72d8f8e25 100644
--- a/packages/shared/src/server/evals/awsLambdaCodeEvalDispatcher.test.ts
+++ b/packages/shared/src/server/evals/awsLambdaCodeEvalDispatcher.test.ts
@@ -14,6 +14,7 @@ const mocks = vi.hoisted(() => {
instrumentAsync: vi.fn(async (_ctx, callback) => callback(span)),
recordDistribution: vi.fn(),
traceException: vi.fn(),
+ loggerWarn: vi.fn(),
};
});
@@ -25,7 +26,7 @@ vi.mock("../instrumentation", () => ({
vi.mock("../logger", () => ({
logger: {
- warn: vi.fn(),
+ warn: mocks.loggerWarn,
},
}));
@@ -193,6 +194,46 @@ describe("AwsLambdaCodeEvalDispatcher observability", () => {
});
});
+ it.each([
+ {
+ errorType: "Runtime.OutOfMemory",
+ errorMessage: "Runtime exited with error: signal: killed",
+ },
+ {
+ errorType: "Runtime.ExitError",
+ errorMessage: "Runtime exited with error: signal: killed",
+ },
+ ])(
+ "classifies Lambda $errorType failures as evaluator memory errors",
+ async ({ errorType, errorMessage }) => {
+ const dispatcher = new AwsLambdaCodeEvalDispatcher({
+ lambdaClient: {
+ send: vi.fn().mockResolvedValue({
+ StatusCode: 200,
+ FunctionError: "Unhandled",
+ Payload: Buffer.from(JSON.stringify({ errorMessage, errorType })),
+ $metadata: {
+ requestId: "request-oom",
+ httpStatusCode: 200,
+ attempts: 1,
+ totalRetryDelay: 0,
+ },
+ }),
+ } as any,
+ });
+
+ await expect(dispatcher.dispatch(baseInput)).rejects.toMatchObject({
+ code: "OUT_OF_MEMORY",
+ retryable: false,
+ });
+ expectSpanAttributes({
+ "langfuse.code_eval.error.code": "OUT_OF_MEMORY",
+ "langfuse.code_eval.error.retryable": false,
+ });
+ expect(mocks.loggerWarn).not.toHaveBeenCalled();
+ },
+ );
+
it("records the original Lambda FunctionError before throwing the derived user-facing error", async () => {
const dispatcher = new AwsLambdaCodeEvalDispatcher({
lambdaClient: {
diff --git a/packages/shared/src/server/evals/awsLambdaCodeEvalDispatcher.ts b/packages/shared/src/server/evals/awsLambdaCodeEvalDispatcher.ts
index 89ae98f49d..99c21d3869 100644
--- a/packages/shared/src/server/evals/awsLambdaCodeEvalDispatcher.ts
+++ b/packages/shared/src/server/evals/awsLambdaCodeEvalDispatcher.ts
@@ -54,6 +54,7 @@ const USER_ERROR_CODES = new Set<CodeEvalDispatcherErrorCode>([
CodeEvalDispatcherErrorCodes.PAYLOAD_TOO_LARGE,
CodeEvalDispatcherErrorCodes.RESULT_TOO_LARGE,
CodeEvalDispatcherErrorCodes.SOURCE_TOO_LARGE,
+ CodeEvalDispatcherErrorCodes.OUT_OF_MEMORY,
CodeEvalDispatcherErrorCodes.USER_CODE_ERROR,
]);
@@ -429,7 +430,20 @@ function classifyLambdaFunctionError(params: {
);
}
- // Abnormal runtime exit (OOM kill, segfault, process.exit, SIGKILL).
+ const isOutOfMemoryError =
+ errorType === "Runtime.OutOfMemory" ||
+ (errorType === "Runtime.ExitError" &&
+ errorMessage !== null &&
+ /signal:\s*killed/i.test(errorMessage));
+
+ if (isOutOfMemoryError) {
+ return new CodeEvalDispatcherError(
+ composedMessage || "Evaluator exceeded the available memory",
+ { code: CodeEvalDispatcherErrorCodes.OUT_OF_MEMORY, retryable: false },
+ );
+ }
+
+ // Other abnormal runtime exits (segfault, process.exit, SIGKILL).
// Retrying never recovers from these.
if (errorType === "Runtime.ExitError") {
return new CodeEvalDispatcherError(
diff --git a/packages/shared/src/server/evals/codeEvalDispatcherTypes.ts b/packages/shared/src/server/evals/codeEvalDispatcherTypes.ts
index 8581bd6e1e..dd78163596 100644
--- a/packages/shared/src/server/evals/codeEvalDispatcherTypes.ts
+++ b/packages/shared/src/server/evals/codeEvalDispatcherTypes.ts
@@ -148,6 +148,7 @@ export const CodeEvalDispatcherErrorCode = z.enum([
"RESULT_TOO_LARGE",
"SOURCE_TOO_LARGE",
"TIMEOUT",
+ "OUT_OF_MEMORY",
"UNSUPPORTED_RUNTIME",
"USER_CODE_ERROR",
"LAMBDA_CONCURRENCY_LIMIT",
diff --git a/packages/shared/src/server/evals/codeEvalExecution.ts b/packages/shared/src/server/evals/codeEvalExecution.ts
index b49ab1daf3..233016f940 100644
--- a/packages/shared/src/server/evals/codeEvalExecution.ts
+++ b/packages/shared/src/server/evals/codeEvalExecution.ts
@@ -45,6 +45,9 @@ const USER_VISIBLE_CODE_EVAL_ERROR_MESSAGE_BY_CODE: Partial<
[CodeEvalDispatcherErrorCodes.TIMEOUT]: withCodeEvalDocs(
"Evaluator timed out. Code-based evaluators must complete within the configured runtime limit. Long executions can be caused by network calls, which are forbidden and may never complete. Remove network calls, optimize your evaluator code, and try again.",
),
+ [CodeEvalDispatcherErrorCodes.OUT_OF_MEMORY]: withCodeEvalDocs(
+ "Evaluator exceeded the available memory limit. Reduce memory usage in your evaluator code to stay within the limit, then try again.",
+ ),
[CodeEvalDispatcherErrorCodes.SOURCE_TOO_LARGE]: withCodeEvalDocs(
`Evaluator source code is too large. Code-based evaluator source code is limited to ${formatCodeEvalByteLimit(CODE_EVAL_SOURCE_MAX_BYTES)}. Shorten the evaluator code and try again.`,
),
diff --git a/worker/AGENTS.md b/worker/AGENTS.md
index f20e1e98e0..ba34ad7bb1 100644
--- a/worker/AGENTS.md
+++ b/worker/AGENTS.md
@@ -16,6 +16,7 @@
- Worker registration/lifecycle: `src/queues/workerManager.ts`
- Queue processors: `src/queues/*`
- Feature processors: `src/features/*`
+- Evaluation terminal-outcome classification: `src/features/evaluation/evalExecutionMetrics.ts`. Keep it aligned with shared code evaluator dispatcher error codes and user-visible error mapping.
- Service layer: `src/services/*`
- Tests: `src/__tests__/*`, `src/queues/__tests__/*`
diff --git a/worker/src/features/evaluation/codeBased/awsLambdaCodeEvalDispatcher.test.ts b/worker/src/features/evaluation/codeBased/awsLambdaCodeEvalDispatcher.test.ts
index fa0d8ab0a8..91c2a956f4 100644
--- a/worker/src/features/evaluation/codeBased/awsLambdaCodeEvalDispatcher.test.ts
+++ b/worker/src/features/evaluation/codeBased/awsLambdaCodeEvalDispatcher.test.ts
@@ -255,9 +255,8 @@ describe("AwsLambdaCodeEvalDispatcher", () => {
} satisfies Partial<CodeEvalDispatcherError>);
});
- it("classifies Runtime.ExitError as non-retryable LAMBDA_CONFIGURATION_ERROR", async () => {
- // Documented Lambda RIC errorType for abnormal runtime termination
- // (OOM kill, segfault, process.exit). Retrying never recovers.
+ it("classifies Runtime.ExitError with signal killed as non-retryable OUT_OF_MEMORY", async () => {
+ // Lambda reports some OOM kills as Runtime.ExitError with signal: killed.
const send = vi.fn().mockResolvedValue({
FunctionError: "Unhandled",
Payload: Buffer.from(
@@ -273,7 +272,7 @@ describe("AwsLambdaCodeEvalDispatcher", () => {
});
await expect(dispatcher.dispatch(baseInput)).rejects.toMatchObject({
- code: "LAMBDA_CONFIGURATION_ERROR",
+ code: "OUT_OF_MEMORY",
retryable: false,
} satisfies Partial<CodeEvalDispatcherError>);
});
diff --git a/worker/src/features/evaluation/codeBased/executeCodeBasedEvaluation.test.ts b/worker/src/features/evaluation/codeBased/executeCodeBasedEvaluation.test.ts
index 163b6723dc..c8bc9a772c 100644
--- a/worker/src/features/evaluation/codeBased/executeCodeBasedEvaluation.test.ts
+++ b/worker/src/features/evaluation/codeBased/executeCodeBasedEvaluation.test.ts
@@ -641,6 +641,56 @@ describe("executeCodeBasedEvaluation", () => {
expect(trace).not.toContain(rawTimeoutMessage);
});
+ it("shows a user-visible error when evaluator code exceeds available memory", async () => {
+ const error = new CodeEvalDispatcherError(
+ "Runtime.OutOfMemory: Runtime exited with error: signal: killed",
+ {
+ code: CodeEvalDispatcherErrorCodes.OUT_OF_MEMORY,
+ retryable: false,
+ },
+ );
+ mocks.dispatcher.dispatch.mockRejectedValue(error);
+
+ const promise = executeCodeBasedEvaluation({
+ projectId: "project-1",
+ organizationId: "org-1",
+ job: {
+ id: "job-1",
+ jobConfigurationId: "config-1",
+ jobInputTraceId: "trace-1",
+ jobInputObservationId: "obs-1",
+ jobInputDatasetItemId: null,
+ } as any,
+ config: { id: "config-1", scoreName: "default-score" } as any,
+ template: {
+ id: "template-1",
+ name: "Code evaluator",
+ type: EvalTemplateType.CODE,
+ version: 1,
+ sourceCode: "function evaluate() {}",
+ sourceCodeLanguage: EvalTemplateSourceCodeLanguage.TYPESCRIPT,
+ prompt: null,
+ outputDefinition: null,
+ } as any,
+ extractedVariables: [{ var: "input", value: "prompt" }],
+ executionMetadata: { job_execution_id: "job-1" },
+ });
+
+ await expect(promise).rejects.toMatchObject({
+ code: "OUT_OF_MEMORY",
+ retryable: false,
+ });
+ await expect(promise).rejects.toThrow(
+ "Evaluator exceeded the available memory limit. Reduce memory usage in your evaluator code to stay within the limit, then try again.",
+ );
+
+ const trace = JSON.stringify(mocks.writeInternalTrace.mock.calls[0]?.[0]);
+ expect(trace).toContain(
+ "Evaluator exceeded the available memory limit. Reduce memory usage in your evaluator code to stay within the limit, then try again.",
+ );
+ expect(trace).not.toContain("Runtime.OutOfMemory");
+ });
+
it("masks internal dispatcher errors in the internal trace", async () => {
const error = new CodeEvalDispatcherError(
"Failed to invoke code eval Lambda code-based-eval-executor-node: ResourceNotFoundException: Function not found",
diff --git a/worker/src/features/evaluation/evalExecutionMetrics.test.ts b/worker/src/features/evaluation/evalExecutionMetrics.test.ts
index 3ca04f3083..54a99bb704 100644
--- a/worker/src/features/evaluation/evalExecutionMetrics.test.ts
+++ b/worker/src/features/evaluation/evalExecutionMetrics.test.ts
@@ -107,6 +107,10 @@ describe("evaluation execution metrics", () => {
code: CodeEvalDispatcherErrorCodes.TIMEOUT,
expected: "customer_error",
},
+ {
+ code: CodeEvalDispatcherErrorCodes.OUT_OF_MEMORY,
+ expected: "customer_error",
+ },
{
code: CodeEvalDispatcherErrorCodes.LAMBDA_CONCURRENCY_LIMIT,
expected: "platform_error",
diff --git a/worker/src/features/evaluation/evalExecutionMetrics.ts b/worker/src/features/evaluation/evalExecutionMetrics.ts
index 0332258473..f3401c1d52 100644
--- a/worker/src/features/evaluation/evalExecutionMetrics.ts
+++ b/worker/src/features/evaluation/evalExecutionMetrics.ts
@@ -25,6 +25,7 @@ const CODE_EVAL_CUSTOMER_ERROR_CODES: ReadonlySet<string> = new Set([
CodeEvalDispatcherErrorCodes.RESULT_TOO_LARGE,
CodeEvalDispatcherErrorCodes.SOURCE_TOO_LARGE,
CodeEvalDispatcherErrorCodes.TIMEOUT,
+ CodeEvalDispatcherErrorCodes.OUT_OF_MEMORY,
CodeEvalDispatcherErrorCodes.USER_CODE_ERROR,
]);