Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(evals): classify invalid model output as customer error (#16900)

Tobias Wochinger69cd809a7d58
diff --git a/worker/src/features/evaluation/evalExecutionMetrics.test.ts b/worker/src/features/evaluation/evalExecutionMetrics.test.ts
index 54a99bb704..ba5bd48617 100644
--- a/worker/src/features/evaluation/evalExecutionMetrics.test.ts
+++ b/worker/src/features/evaluation/evalExecutionMetrics.test.ts
@@ -91,6 +91,18 @@ describe("evaluation execution metrics", () => {
} satisfies EvaluatorLlmErrorClassification,
expected: "upstream_error",
},
+ ...["AI_NoObjectGeneratedError", "AI_NoOutputGeneratedError"].map(
+ (name) => ({
+ classification: {
+ kind: "ai-sdk" as const,
+ message: "model output did not match the required format",
+ isRetryable: false,
+ error: Object.assign(new Error("invalid model output"), { name }),
+ blockReason: null,
+ } satisfies EvaluatorLlmErrorClassification,
+ expected: "customer_error",
+ }),
+ ),
])(
"classifies LLM terminal errors as $expected",
({ classification, expected }) => {
diff --git a/worker/src/features/evaluation/evalExecutionMetrics.ts b/worker/src/features/evaluation/evalExecutionMetrics.ts
index f3401c1d52..242e15a9e6 100644
--- a/worker/src/features/evaluation/evalExecutionMetrics.ts
+++ b/worker/src/features/evaluation/evalExecutionMetrics.ts
@@ -18,6 +18,11 @@ export type EvalExecutionTerminalOutcome =
| "customer_error"
| "cancelled";
+const LLM_EVAL_CUSTOMER_OUTPUT_ERROR_NAMES: ReadonlySet<string> = new Set([
+ "AI_NoObjectGeneratedError",
+ "AI_NoOutputGeneratedError",
+]);
+
const CODE_EVAL_CUSTOMER_ERROR_CODES: ReadonlySet<string> = new Set([
CodeEvalDispatcherErrorCodes.INVALID_RESULT,
CodeEvalDispatcherErrorCodes.INVALID_SOURCE,
@@ -64,7 +69,9 @@ export function getLlmEvalTerminalErrorOutcome(
if (
classification.blockReason !== null ||
classification.kind === "evaluator-policy" ||
- classification.kind === "validation"
+ classification.kind === "validation" ||
+ (classification.kind === "ai-sdk" &&
+ isLlmEvalCustomerOutputError(classification.error))
) {
return "customer_error";
}
@@ -95,6 +102,36 @@ export function getCodeEvalTerminalErrorOutcome(
: "platform_error";
}
+function isLlmEvalCustomerOutputError(error: unknown): boolean {
+ const visited = new Set<unknown>();
+ const pending = [error];
+
+ while (pending.length > 0) {
+ const current = pending.pop();
+ if (
+ current === null ||
+ current === undefined ||
+ typeof current !== "object" ||
+ visited.has(current)
+ ) {
+ continue;
+ }
+
+ visited.add(current);
+ const value = current as Record<string, unknown>;
+ if (
+ typeof value.name === "string" &&
+ LLM_EVAL_CUSTOMER_OUTPUT_ERROR_NAMES.has(value.name)
+ ) {
+ return true;
+ }
+
+ pending.push(value.cause, value.lastError);
+ }
+
+ return false;
+}
+
function getEvaluatorTypeTag(
executionType: ObservationEvalExecutionType,
): "llm_as_judge" | "code_as_judge" {