🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
import { describe, expect, it, vi } from "vitest";
import {
AwsLambdaCodeEvalDispatcher,
CodeEvalDispatcherError,
type DispatchInput,
} from "@langfuse/shared/src/server";
const baseInput: DispatchInput = {
scope: {
organizationId: "org-1",
projectId: "project-1",
evaluatorId: "evaluator-1",
},
runtime: { language: "TYPESCRIPT" },
execution: { jobExecutionId: "job-1" },
code: { source: "function evaluate() {}" },
payload: {
observation: {
input: null,
output: null,
metadata: null,
toolCalls: [],
},
},
};
describe("AwsLambdaCodeEvalDispatcher", () => {
it("returns successful Lambda scores", async () => {
const send = vi.fn().mockResolvedValue({
Payload: Buffer.from(
JSON.stringify({
scores: [
{
name: "lambda-score",
value: 1,
dataType: "NUMERIC",
metadata: { source: "lambda-fixture" },
},
],
}),
),
});
const dispatcher = new AwsLambdaCodeEvalDispatcher({
lambdaClient: { send } as any,
});
await expect(dispatcher.dispatch(baseInput)).resolves.toEqual({
scores: [
{
name: "lambda-score",
value: 1,
dataType: "NUMERIC",
metadata: { source: "lambda-fixture" },
},
],
});
});
it("propagates TenantId derived from scope.organizationId:projectId", async () => {
const send = vi.fn().mockResolvedValue({
Payload: Buffer.from(
JSON.stringify({
scores: [{ name: "score", value: 1, dataType: "NUMERIC" }],
}),
),
});
const dispatcher = new AwsLambdaCodeEvalDispatcher({
lambdaClient: { send } as any,
});
await dispatcher.dispatch(baseInput);
expect(send.mock.calls[0][0].input.TenantId).toBe("org-1:project-1");
});
it("sends the nested evaluation context payload to Lambda", async () => {
const send = vi.fn().mockResolvedValue({
Payload: Buffer.from(
JSON.stringify({
scores: [{ name: "score", value: 1, dataType: "NUMERIC" }],
}),
),
});
const dispatcher = new AwsLambdaCodeEvalDispatcher({
lambdaClient: { send } as any,
});
const input: DispatchInput = {
...baseInput,
payload: {
observation: {
input: { question: "2+2" },
output: "4",
metadata: { source: "test" },
toolCalls: [],
},
experiment: {
itemExpectedOutput: "4",
itemMetadata: { difficulty: "easy" },
},
},
};
await dispatcher.dispatch(input);
const lambdaPayload = JSON.parse(
Buffer.from(send.mock.calls[0][0].input.Payload).toString("utf8"),
);
expect(lambdaPayload.payload).toEqual(input.payload);
});
it("prefers a valid dispatch result over an injected error envelope", async () => {
// Defense in depth against runners that try to smuggle an `error`
// envelope alongside valid scores to force a retry or mask the
// outcome with a masked LAMBDA_* code.
const send = vi.fn().mockResolvedValue({
Payload: Buffer.from(
JSON.stringify({
scores: [{ name: "score", value: 1, dataType: "NUMERIC" }],
error: {
code: "LAMBDA_CONCURRENCY_LIMIT",
message: "attacker-controlled",
},
}),
),
});
const dispatcher = new AwsLambdaCodeEvalDispatcher({
lambdaClient: { send } as any,
});
await expect(dispatcher.dispatch(baseInput)).resolves.toEqual({
scores: [{ name: "score", value: 1, dataType: "NUMERIC" }],
});
});
it("uses configured function names per runtime", async () => {
const send = vi.fn().mockResolvedValue({
Payload: Buffer.from(
JSON.stringify({
scores: [{ name: "score", value: 1, dataType: "NUMERIC" }],
}),
),
});
const dispatcher = new AwsLambdaCodeEvalDispatcher({
lambdaClient: { send } as any,
functionNameByLanguage: {
PYTHON: "custom-python-fn",
TYPESCRIPT: "custom-node-fn",
},
});
await dispatcher.dispatch(baseInput);
expect(send.mock.calls[0][0].input.FunctionName).toBe("custom-node-fn");
await dispatcher.dispatch({
...baseInput,
runtime: { language: "PYTHON" },
});
expect(send.mock.calls[1][0].input.FunctionName).toBe("custom-python-fn");
});
it("classifies typed user-code runner errors as non-retryable", async () => {
const send = vi.fn().mockResolvedValue({
FunctionError: "Handled",
Payload: Buffer.from(
JSON.stringify({
error: { code: "USER_CODE_ERROR", message: "ReferenceError: x" },
}),
),
});
const dispatcher = new AwsLambdaCodeEvalDispatcher({
lambdaClient: { send } as any,
});
await expect(dispatcher.dispatch(baseInput)).rejects.toMatchObject({
code: "USER_CODE_ERROR",
message: "ReferenceError: x",
retryable: false,
} satisfies Partial<CodeEvalDispatcherError>);
});
it("classifies successful runner error envelopes", async () => {
const send = vi.fn().mockResolvedValue({
Payload: Buffer.from(
JSON.stringify({
error: { code: "USER_CODE_ERROR", message: "ReferenceError: x" },
}),
),
});
const dispatcher = new AwsLambdaCodeEvalDispatcher({
lambdaClient: { send } as any,
});
await expect(dispatcher.dispatch(baseInput)).rejects.toMatchObject({
code: "USER_CODE_ERROR",
message: "ReferenceError: x",
retryable: false,
} satisfies Partial<CodeEvalDispatcherError>);
});
it("classifies timeouts via errorType Function.TimedOut as non-retryable", async () => {
// Envelope shape captured empirically from Floci probes.
const send = vi.fn().mockResolvedValue({
FunctionError: "Unhandled",
Payload: Buffer.from(
JSON.stringify({
errorMessage: "Task timed out after 30 seconds",
errorType: "Function.TimedOut",
}),
),
});
const dispatcher = new AwsLambdaCodeEvalDispatcher({
lambdaClient: { send } as any,
});
await expect(dispatcher.dispatch(baseInput)).rejects.toMatchObject({
code: "TIMEOUT",
message: "Function.TimedOut: Task timed out after 30 seconds",
retryable: false,
} satisfies Partial<CodeEvalDispatcherError>);
});
it("classifies timeouts via errorType Sandbox.Timedout as non-retryable", async () => {
const send = vi.fn().mockResolvedValue({
FunctionError: "Unhandled",
Payload: Buffer.from(
JSON.stringify({
errorMessage: "Task timed out after 30 seconds",
errorType: "Sandbox.Timedout",
}),
),
});
const dispatcher = new AwsLambdaCodeEvalDispatcher({
lambdaClient: { send } as any,
});
await expect(dispatcher.dispatch(baseInput)).rejects.toMatchObject({
code: "TIMEOUT",
message: "Sandbox.Timedout: Task timed out after 30 seconds",
retryable: false,
} satisfies Partial<CodeEvalDispatcherError>);
});
it("classifies timeouts via errorMessage substring as non-retryable", async () => {
// Defensive fallback for runtimes that don't populate errorType.
const send = vi.fn().mockResolvedValue({
FunctionError: "Unhandled",
Payload: Buffer.from(JSON.stringify({ errorMessage: "Task timed out" })),
});
const dispatcher = new AwsLambdaCodeEvalDispatcher({
lambdaClient: { send } as any,
});
await expect(dispatcher.dispatch(baseInput)).rejects.toMatchObject({
code: "TIMEOUT",
retryable: false,
} satisfies Partial<CodeEvalDispatcherError>);
});
it("classifies Runtime.ExitError with signal killed as non-retryable OUT_OF_MEMORY", async () => {
// Lambda reports some OOM kills as Runtime.ExitError with signal: killed.
const send = vi.fn().mockResolvedValue({
FunctionError: "Unhandled",
Payload: Buffer.from(
JSON.stringify({
errorMessage:
"RequestId: abc Error: Runtime exited with error: signal: killed",
errorType: "Runtime.ExitError",
}),
),
});
const dispatcher = new AwsLambdaCodeEvalDispatcher({
lambdaClient: { send } as any,
});
await expect(dispatcher.dispatch(baseInput)).rejects.toMatchObject({
code: "OUT_OF_MEMORY",
retryable: false,
} satisfies Partial<CodeEvalDispatcherError>);
});
it("classifies FunctionError=Handled without runner envelope as USER_CODE_ERROR", async () => {
// Managed runtime caught a user-code exception that escaped our
// handler's try/catch (e.g. async unhandled rejection). The envelope
// carries the language-specific exception type+message.
const send = vi.fn().mockResolvedValue({
FunctionError: "Handled",
Payload: Buffer.from(
JSON.stringify({
errorMessage: "x is not defined",
errorType: "ReferenceError",
}),
),
});
const dispatcher = new AwsLambdaCodeEvalDispatcher({
lambdaClient: { send } as any,
});
await expect(dispatcher.dispatch(baseInput)).rejects.toMatchObject({
code: "USER_CODE_ERROR",
message: "ReferenceError: x is not defined",
retryable: false,
} satisfies Partial<CodeEvalDispatcherError>);
});
it("classifies concurrency throttling as retryable", async () => {
const error = new Error("Rate exceeded");
error.name = "TooManyRequestsException";
const dispatcher = new AwsLambdaCodeEvalDispatcher({
lambdaClient: { send: vi.fn().mockRejectedValue(error) } as any,
});
await expect(dispatcher.dispatch(baseInput)).rejects.toMatchObject({
code: "LAMBDA_CONCURRENCY_LIMIT",
retryable: true,
} satisfies Partial<CodeEvalDispatcherError>);
});
it("classifies Node network timeout error codes as retryable invocation errors", async () => {
const error = Object.assign(new Error("connect ETIMEDOUT"), {
code: "ETIMEDOUT",
});
const dispatcher = new AwsLambdaCodeEvalDispatcher({
lambdaClient: { send: vi.fn().mockRejectedValue(error) } as any,
});
await expect(dispatcher.dispatch(baseInput)).rejects.toMatchObject({
code: "LAMBDA_INVOCATION_ERROR",
retryable: true,
} satisfies Partial<CodeEvalDispatcherError>);
});
it("classifies Lambda configuration errors as non-retryable", async () => {
const error = new Error("Function not found");
error.name = "ResourceNotFoundException";
const dispatcher = new AwsLambdaCodeEvalDispatcher({
lambdaClient: { send: vi.fn().mockRejectedValue(error) } as any,
});
await expect(dispatcher.dispatch(baseInput)).rejects.toMatchObject({
code: "LAMBDA_CONFIGURATION_ERROR",
retryable: false,
} satisfies Partial<CodeEvalDispatcherError>);
});
it("classifies AWS-side payload-too-large rejection as PAYLOAD_TOO_LARGE", async () => {
// If a request slips past our local `assertDispatchInputWithinLimits`
// (5.5 MB) and AWS still rejects it at the 6 MB sync-invocation limit,
// the failure should surface under the same `PAYLOAD_TOO_LARGE` code as
// the local pre-check — same cause, same fix, same non-retryability.
const error = new Error("Payload exceeds maximum allowed size");
error.name = "RequestTooLargeException";
const dispatcher = new AwsLambdaCodeEvalDispatcher({
lambdaClient: { send: vi.fn().mockRejectedValue(error) } as any,
});
await expect(dispatcher.dispatch(baseInput)).rejects.toMatchObject({
code: "PAYLOAD_TOO_LARGE",
retryable: false,
} satisfies Partial<CodeEvalDispatcherError>);
});
it("includes a small invalid Lambda result as structured error data", async () => {
const send = vi.fn().mockResolvedValue({
Payload: Buffer.from(JSON.stringify({ score: 1 })),
});
const dispatcher = new AwsLambdaCodeEvalDispatcher({
lambdaClient: { send } as any,
});
await expect(dispatcher.dispatch(baseInput)).rejects.toMatchObject({
code: "INVALID_RESULT",
retryable: false,
returnedResult: { score: 1 },
} satisfies Partial<CodeEvalDispatcherError>);
});
it("does not include a returned result for invalid JSON Lambda responses", async () => {
const send = vi.fn().mockResolvedValue({
Payload: Buffer.from("not-json"),
});
const dispatcher = new AwsLambdaCodeEvalDispatcher({
lambdaClient: { send } as any,
});
await expect(dispatcher.dispatch(baseInput)).rejects.toMatchObject({
code: "INVALID_RESULT",
retryable: false,
returnedResult: undefined,
} satisfies Partial<CodeEvalDispatcherError>);
});
it("rejects Lambda responses that exceed the wire byte-size limit", async () => {
// Build a response payload larger than the 256 KiB result cap. The
// guard inspects `response.Payload.byteLength` before any JSON parsing,
// so it should fail fast as a non-retryable `RESULT_TOO_LARGE` without
// ever allocating the parsed object.
const oversizedBody = JSON.stringify({
scores: [
{
name: "oversized-score",
value: "a".repeat(512 * 1024),
dataType: "CATEGORICAL",
},
],
});
const send = vi.fn().mockResolvedValue({
Payload: Buffer.from(oversizedBody),
});
const dispatcher = new AwsLambdaCodeEvalDispatcher({
lambdaClient: { send } as any,
});
await expect(dispatcher.dispatch(baseInput)).rejects.toMatchObject({
code: "RESULT_TOO_LARGE",
retryable: false,
} satisfies Partial<CodeEvalDispatcherError>);
});
});