🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
import {
InvokeCommand,
LambdaClient,
type InvokeCommandInput,
} from "@aws-sdk/client-lambda";
import { SpanKind, type AttributeValue, type Span } from "@opentelemetry/api";
import { z } from "zod";
import {
instrumentAsync,
recordDistribution,
traceException,
} from "../instrumentation";
import { logger } from "../logger";
import {
assertDispatchInputWithinLimits,
assertDispatchResultWithinByteLimit,
CodeEvalDispatcherError,
CodeEvalDispatcherErrorCode,
CodeEvalDispatcherErrorCodes,
parseDispatchResult,
type CodeEvalDispatcher,
type CodeEvalRuntimeLanguage,
type DispatchInput,
type DispatchResult,
} from "./codeEvalDispatcherTypes";
export const DEFAULT_LAMBDA_FUNCTION_BY_LANGUAGE = {
PYTHON: "code-based-eval-executor-python",
TYPESCRIPT: "code-based-eval-executor-node",
} satisfies Record<CodeEvalRuntimeLanguage, string>;
// Synchronous Lambda invokes hold the HTTP request open through cold starts and user code execution.
const LAMBDA_INVOKE_REQUEST_TIMEOUT_MS = 10_000;
type CodeEvalDispatcherErrorClassification = {
code: CodeEvalDispatcherErrorCode;
retryable?: boolean;
};
// Defensive: today our runners only emit non-retryable user-code codes, so
// this is only consulted if a future runner surfaces one of them via the
// user-code-error envelope.
const RETRYABLE_ERROR_CODES = new Set<CodeEvalDispatcherErrorCode>([
CodeEvalDispatcherErrorCodes.LAMBDA_CONCURRENCY_LIMIT,
CodeEvalDispatcherErrorCodes.LAMBDA_INVOCATION_ERROR,
]);
// Skipped from warn-level logging so routine user failures don't drown out
// infrastructure issues in Datadog.
const USER_ERROR_CODES = new Set<CodeEvalDispatcherErrorCode>([
CodeEvalDispatcherErrorCodes.INVALID_RESULT,
CodeEvalDispatcherErrorCodes.INVALID_SOURCE,
CodeEvalDispatcherErrorCodes.PAYLOAD_TOO_LARGE,
CodeEvalDispatcherErrorCodes.RESULT_TOO_LARGE,
CodeEvalDispatcherErrorCodes.SOURCE_TOO_LARGE,
CodeEvalDispatcherErrorCodes.OUT_OF_MEMORY,
CodeEvalDispatcherErrorCodes.USER_CODE_ERROR,
]);
const AWS_ERROR_CLASSIFICATION_BY_NAME: Record<
string,
CodeEvalDispatcherErrorClassification
> = {
TooManyRequestsException: {
code: CodeEvalDispatcherErrorCodes.LAMBDA_CONCURRENCY_LIMIT,
retryable: true,
},
AccessDeniedException: {
code: CodeEvalDispatcherErrorCodes.LAMBDA_CONFIGURATION_ERROR,
},
InvalidParameterValueException: {
code: CodeEvalDispatcherErrorCodes.LAMBDA_CONFIGURATION_ERROR,
},
// Unified with the local pre-check so "payload too large" failures share
// a single code regardless of which layer caught them.
RequestTooLargeException: {
code: CodeEvalDispatcherErrorCodes.PAYLOAD_TOO_LARGE,
},
ResourceNotFoundException: {
code: CodeEvalDispatcherErrorCodes.LAMBDA_CONFIGURATION_ERROR,
},
ResourceConflictException: {
code: CodeEvalDispatcherErrorCodes.LAMBDA_INVOCATION_ERROR,
retryable: true,
},
ServiceException: {
code: CodeEvalDispatcherErrorCodes.LAMBDA_INVOCATION_ERROR,
retryable: true,
},
};
const AWS_ERROR_CLASSIFICATION_BY_CODE: Record<
string,
CodeEvalDispatcherErrorClassification
> = {
ECONNRESET: {
code: CodeEvalDispatcherErrorCodes.LAMBDA_INVOCATION_ERROR,
retryable: true,
},
ETIMEDOUT: {
code: CodeEvalDispatcherErrorCodes.LAMBDA_INVOCATION_ERROR,
retryable: true,
},
};
// `.strict()` ensures a runner that returns a valid dispatch result alongside
// an `error` field (e.g. `{ scores: [...], error: { code, message } }`) cannot
// match here — defense in depth for the user-code envelope.
const UserCodeErrorFieldsSchema = z
.object({
code: CodeEvalDispatcherErrorCode,
message: z.string(),
})
.strict();
const UserCodeErrorSchema = z.union([
z
.object({ error: UserCodeErrorFieldsSchema })
.strict()
.transform(({ error }) => error),
UserCodeErrorFieldsSchema,
]);
type UserCodeError = z.infer<typeof UserCodeErrorSchema>;
// Process-singleton client so the AWS SDK's Keep-Alive connection pool is
// reused across all dispatcher instances. In practice the process sees one
// endpoint for its lifetime (real AWS or the Floci dev endpoint); the
// endpoint-mismatch guard exists so a future caller that passes a different
// endpoint without injecting a client doesn't silently get a stale one.
let sharedLambdaClient: LambdaClient | undefined;
let sharedLambdaClientEndpoint: string | undefined;
function getSharedLambdaClient(endpoint?: string): LambdaClient {
if (sharedLambdaClient && sharedLambdaClientEndpoint === endpoint) {
return sharedLambdaClient;
}
sharedLambdaClient = new LambdaClient({
...(endpoint ? { endpoint } : {}),
requestHandler: {
requestTimeout: LAMBDA_INVOKE_REQUEST_TIMEOUT_MS,
throwOnRequestTimeout: true,
},
});
sharedLambdaClientEndpoint = endpoint;
return sharedLambdaClient;
}
export class AwsLambdaCodeEvalDispatcher implements CodeEvalDispatcher {
public readonly name = "aws-lambda";
private readonly lambdaClient: LambdaClient;
private readonly functionNameByLanguage: Record<
CodeEvalRuntimeLanguage,
string
>;
constructor(params?: {
lambdaClient?: LambdaClient;
endpoint?: string;
functionNameByLanguage?: Partial<Record<CodeEvalRuntimeLanguage, string>>;
}) {
this.lambdaClient =
params?.lambdaClient ?? getSharedLambdaClient(params?.endpoint);
this.functionNameByLanguage = {
...DEFAULT_LAMBDA_FUNCTION_BY_LANGUAGE,
...params?.functionNameByLanguage,
};
}
async dispatch(input: DispatchInput): Promise<DispatchResult> {
return instrumentAsync(
{
name: "code-eval.dispatch.aws-lambda",
spanKind: SpanKind.CLIENT,
traceScope: "code-eval-dispatcher",
startNewTrace: true,
},
async (span) => {
// Dispatch spans are ingestion-sampled, so fleet-slowness monitors
// (per-project p95 quorum) run on this unsampled distribution instead.
// Recorded on failures too: timeouts are the slow runs that matter most.
const startTime = Date.now();
try {
return await this.dispatchWithTracing(input, span);
} finally {
recordDistribution(
"langfuse.code_eval.dispatch_duration",
Date.now() - startTime,
{
project_id: input.scope.projectId,
language: input.runtime.language,
unit: "milliseconds",
},
);
}
},
);
}
private async dispatchWithTracing(
input: DispatchInput,
span: Span,
): Promise<DispatchResult> {
const functionName = this.functionNameByLanguage[input.runtime.language];
span.setAttributes({
"eval.dispatcher.name": this.name,
"eval.job_execution.id": input.execution.jobExecutionId,
"eval.runner.language": input.runtime.language,
"eval.template.id": input.scope.evaluatorId,
"langfuse.code_eval.lambda.function_name": functionName,
"langfuse.org.id": input.scope.organizationId,
"langfuse.project.id": input.scope.projectId,
});
try {
const serializedPayload = assertDispatchInputWithinLimits(input);
span.setAttribute(
"langfuse.code_eval.payload.bytes",
Buffer.byteLength(serializedPayload, "utf8"),
);
const commandInput: InvokeCommandInput = {
FunctionName: functionName,
InvocationType: "RequestResponse",
Payload: Buffer.from(serializedPayload),
TenantId: `${input.scope.organizationId}:${input.scope.projectId}`,
};
const response = await this.lambdaClient.send(
new InvokeCommand(commandInput),
);
setSpanAttributes(span, {
...getAwsMetadataSpanAttributes(response.$metadata),
"langfuse.code_eval.lambda.status_code": response.StatusCode,
"langfuse.code_eval.lambda.executed_version": response.ExecutedVersion,
"langfuse.code_eval.lambda.function_error": response.FunctionError,
});
// Lambda function errors come back as HTTP 200 with `FunctionError`
// set; the SDK does NOT throw for these. SDK service exceptions
// (throttling, auth, etc.) hit the outer catch instead.
// https://docs.aws.amazon.com/lambda/latest/dg/invocation-errors.html
if (response.FunctionError) {
const dispatcherError = classifyLambdaFunctionError({
functionName,
functionError: response.FunctionError,
payload: response.Payload,
});
traceLambdaFunctionError({
span,
functionName,
functionError: response.FunctionError,
payload: response.Payload,
});
throw dispatcherError;
}
if (!response.Payload) {
throw new CodeEvalDispatcherError(
`Code eval Lambda ${functionName} returned an empty response`,
{
code: CodeEvalDispatcherErrorCodes.LAMBDA_INVOCATION_ERROR,
retryable: true,
},
);
}
span.setAttribute(
"langfuse.code_eval.lambda.response_payload.bytes",
response.Payload.byteLength,
);
const result = parseLambdaResponsePayload({
functionName,
payload: response.Payload,
});
span.setAttribute("eval.score.count", result.scores.length);
return result;
} catch (error) {
if (error instanceof CodeEvalDispatcherError) {
setDispatcherErrorSpanAttributes(span, error);
logDispatcherError({ functionName, error });
throw error;
}
traceException(error, span, "code_eval.aws_lambda.original_error");
const errorRecord = isRecord(error) ? error : null;
setSpanAttributes(span, {
...getAwsMetadataSpanAttributes(errorRecord?.$metadata),
"langfuse.code_eval.aws_error.name":
error instanceof Error ? error.name : undefined,
"langfuse.code_eval.aws_error.code":
typeof errorRecord?.code === "string" ? errorRecord.code : undefined,
});
const awsError = classifyAwsLambdaError(error);
const dispatcherError = new CodeEvalDispatcherError(
`Failed to invoke code eval Lambda ${functionName}: ${error instanceof Error ? error.message : String(error)}`,
{ ...awsError, cause: error },
);
setDispatcherErrorSpanAttributes(span, dispatcherError);
logDispatcherError({ functionName, error: dispatcherError });
throw dispatcherError;
}
}
}
// Warn on non-user errors only, so infrastructure issues surface in Datadog
// without drowning the channel in routine user-template failures.
function logDispatcherError(params: {
functionName: string;
error: CodeEvalDispatcherError;
}): void {
if (USER_ERROR_CODES.has(params.error.code)) return;
logger.warn(
`Code eval Lambda ${params.functionName} dispatcher failed: ${params.error.message}`,
{
code: params.error.code,
retryable: params.error.retryable,
},
);
}
function setDispatcherErrorSpanAttributes(
span: Span,
error: CodeEvalDispatcherError,
): void {
span.setAttributes({
"langfuse.code_eval.error.code": error.code,
"langfuse.code_eval.error.retryable": error.retryable,
});
}
function getAwsMetadataSpanAttributes(
metadata: unknown,
): Record<string, AttributeValue | undefined> {
if (!isRecord(metadata)) return {};
return {
"aws.request_id":
typeof metadata.requestId === "string" ? metadata.requestId : undefined,
"aws.http_status_code":
typeof metadata.httpStatusCode === "number"
? metadata.httpStatusCode
: undefined,
"aws.sdk.attempts":
typeof metadata.attempts === "number" ? metadata.attempts : undefined,
"aws.sdk.total_retry_delay_ms":
typeof metadata.totalRetryDelay === "number"
? metadata.totalRetryDelay
: undefined,
};
}
function setSpanAttributes(
span: Span,
attributes: Record<string, AttributeValue | undefined>,
): void {
const definedAttributes = Object.fromEntries(
Object.entries(attributes).filter(([, value]) => value !== undefined),
) as Record<string, AttributeValue>;
if (Object.keys(definedAttributes).length > 0) {
span.setAttributes(definedAttributes);
}
}
function traceLambdaFunctionError(params: {
span: Span;
functionName: string;
functionError: string;
payload: Uint8Array | undefined;
}): void {
const errorPayload = parseLambdaErrorPayload(params.payload);
const record = isRecord(errorPayload) ? errorPayload : null;
const error = new Error(
typeof record?.errorMessage === "string"
? record.errorMessage
: `Code eval Lambda ${params.functionName} returned FunctionError=${params.functionError}`,
);
error.name =
typeof record?.errorType === "string"
? record.errorType
: "LambdaFunctionError";
if (Array.isArray(record?.stackTrace)) {
error.stack = record.stackTrace
.filter((line) => typeof line === "string")
.join("\n");
}
traceException(error, params.span, "code_eval.aws_lambda.function_error");
}
function classifyLambdaFunctionError(params: {
functionName: string;
functionError: string;
payload: Uint8Array | undefined;
}): CodeEvalDispatcherError {
const errorPayload = parseLambdaErrorPayload(params.payload);
// Our runners emit `{ error: { code, message } }` on user-code failures.
const userCodeError = parseUserCodeError(errorPayload);
if (userCodeError) {
return createCodeEvalDispatcherErrorFromUserCodeError(userCodeError);
}
// AWS-runtime error envelope: `{ errorMessage, errorType, stackTrace? }`.
const record = isRecord(errorPayload) ? errorPayload : null;
const errorType =
typeof record?.errorType === "string" ? record.errorType : null;
const errorMessage =
typeof record?.errorMessage === "string" ? record.errorMessage : null;
const composedMessage =
errorType && errorMessage
? `${errorType}: ${errorMessage}`
: (errorMessage ?? errorType ?? "");
if (
errorType === "Function.TimedOut" ||
errorType === "Sandbox.Timedout" ||
(errorMessage && isTimeoutErrorMessage(errorMessage)) ||
(errorType === "Runtime.ExitError" &&
errorMessage !== null &&
/runtime exited without providing a reason/i.test(errorMessage))
) {
return new CodeEvalDispatcherError(
composedMessage || "Lambda task timed out",
{ code: CodeEvalDispatcherErrorCodes.TIMEOUT, retryable: false },
);
}
const isOutOfMemoryError =
errorType === "Runtime.OutOfMemory" ||
(errorType === "Runtime.ExitError" &&
errorMessage !== null &&
/signal:\s*killed/i.test(errorMessage));
if (isOutOfMemoryError) {
return new CodeEvalDispatcherError(
composedMessage || "Evaluator exceeded the available memory",
{ code: CodeEvalDispatcherErrorCodes.OUT_OF_MEMORY, retryable: false },
);
}
// Other abnormal runtime exits (segfault, process.exit, SIGKILL).
// Retrying never recovers from these.
if (errorType === "Runtime.ExitError") {
return new CodeEvalDispatcherError(
composedMessage || "Lambda runtime exited abnormally",
{
code: CodeEvalDispatcherErrorCodes.LAMBDA_CONFIGURATION_ERROR,
retryable: false,
},
);
}
// The managed runtime caught a user-code exception and populated
// errorType + errorMessage with the language-specific class and message.
if (params.functionError === "Handled") {
return new CodeEvalDispatcherError(
composedMessage || "Evaluator code threw an uncaught exception",
{ code: CodeEvalDispatcherErrorCodes.USER_CODE_ERROR, retryable: false },
);
}
// Unknown failure — default retryable so a flake doesn't burn the job.
return new CodeEvalDispatcherError(
composedMessage ||
`Code eval Lambda ${params.functionName} failed with ${params.functionError}`,
{
code: CodeEvalDispatcherErrorCodes.LAMBDA_INVOCATION_ERROR,
retryable: true,
},
);
}
function createCodeEvalDispatcherErrorFromUserCodeError(
userCodeError: UserCodeError,
): CodeEvalDispatcherError {
return new CodeEvalDispatcherError(userCodeError.message, {
code: userCodeError.code,
retryable: RETRYABLE_ERROR_CODES.has(userCodeError.code),
});
}
function classifyAwsLambdaError(
error: unknown,
): CodeEvalDispatcherErrorClassification {
if (!(error instanceof Error)) {
return {
code: CodeEvalDispatcherErrorCodes.LAMBDA_INVOCATION_ERROR,
retryable: true,
};
}
const errorCode = (error as { code?: unknown }).code;
return (
AWS_ERROR_CLASSIFICATION_BY_NAME[error.name] ??
(typeof errorCode === "string"
? AWS_ERROR_CLASSIFICATION_BY_CODE[errorCode]
: undefined) ?? {
code: CodeEvalDispatcherErrorCodes.LAMBDA_INVOCATION_ERROR,
retryable: true,
}
);
}
function parseLambdaResponsePayload(params: {
functionName: string;
payload: Uint8Array;
}): DispatchResult {
assertDispatchResultWithinByteLimit(params.payload.byteLength);
const responseBody = Buffer.from(params.payload).toString("utf8");
let parsed: unknown;
try {
parsed = JSON.parse(responseBody) as unknown;
} catch (error) {
throw new CodeEvalDispatcherError(
`Code eval Lambda ${params.functionName} returned invalid JSON`,
{ code: CodeEvalDispatcherErrorCodes.INVALID_RESULT, cause: error },
);
}
// Try the success shape first so a runner can't smuggle an `error`
// envelope alongside valid scores to force a retry or mask the outcome.
try {
return parseDispatchResult(parsed);
} catch (dispatchError) {
const userCodeError = parseUserCodeError(parsed);
if (userCodeError) {
throw createCodeEvalDispatcherErrorFromUserCodeError(userCodeError);
}
throw dispatchError;
}
}
function parseUserCodeError(payload: unknown): UserCodeError | null {
return UserCodeErrorSchema.safeParse(payload).data ?? null;
}
function parseLambdaErrorPayload(payload: Uint8Array | undefined): unknown {
if (!payload) return null;
try {
return JSON.parse(Buffer.from(payload).toString("utf8")) as unknown;
} catch {
return null;
}
}
function isTimeoutErrorMessage(message: string): boolean {
const normalized = message.toLowerCase();
return (
normalized.includes("task timed out") || normalized.includes("timeout")
);
}
function isRecord(value: unknown): value is Record<string, unknown> {
return typeof value === "object" && value !== null && !Array.isArray(value);
}