🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(evals): classify invalid model output as customer error (#16900)
Tobias Wochinger69cd809a7d58
diff --git a/worker/src/features/evaluation/evalExecutionMetrics.test.ts b/worker/src/features/evaluation/evalExecutionMetrics.test.tsindex 54a99bb704..ba5bd48617 100644--- a/worker/src/features/evaluation/evalExecutionMetrics.test.ts+++ b/worker/src/features/evaluation/evalExecutionMetrics.test.ts@@ -91,6 +91,18 @@ describe("evaluation execution metrics", () => {} satisfies EvaluatorLlmErrorClassification,expected: "upstream_error",},+ ...["AI_NoObjectGeneratedError", "AI_NoOutputGeneratedError"].map(+ (name) => ({+ classification: {+ kind: "ai-sdk" as const,+ message: "model output did not match the required format",+ isRetryable: false,+ error: Object.assign(new Error("invalid model output"), { name }),+ blockReason: null,+ } satisfies EvaluatorLlmErrorClassification,+ expected: "customer_error",+ }),+ ),])("classifies LLM terminal errors as $expected",({ classification, expected }) => {diff --git a/worker/src/features/evaluation/evalExecutionMetrics.ts b/worker/src/features/evaluation/evalExecutionMetrics.tsindex f3401c1d52..242e15a9e6 100644--- a/worker/src/features/evaluation/evalExecutionMetrics.ts+++ b/worker/src/features/evaluation/evalExecutionMetrics.ts@@ -18,6 +18,11 @@ export type EvalExecutionTerminalOutcome =| "customer_error"| "cancelled";+const LLM_EVAL_CUSTOMER_OUTPUT_ERROR_NAMES: ReadonlySet<string> = new Set([+ "AI_NoObjectGeneratedError",+ "AI_NoOutputGeneratedError",+]);+const CODE_EVAL_CUSTOMER_ERROR_CODES: ReadonlySet<string> = new Set([CodeEvalDispatcherErrorCodes.INVALID_RESULT,CodeEvalDispatcherErrorCodes.INVALID_SOURCE,@@ -64,7 +69,9 @@ export function getLlmEvalTerminalErrorOutcome(if (classification.blockReason !== null ||classification.kind === "evaluator-policy" ||- classification.kind === "validation"+ classification.kind === "validation" ||+ (classification.kind === "ai-sdk" &&+ isLlmEvalCustomerOutputError(classification.error))) {return "customer_error";}@@ -95,6 +102,36 @@ export function getCodeEvalTerminalErrorOutcome(: "platform_error";}+function isLlmEvalCustomerOutputError(error: unknown): boolean {+ const visited = new Set<unknown>();+ const pending = [error];++ while (pending.length > 0) {+ const current = pending.pop();+ if (+ current === null ||+ current === undefined ||+ typeof current !== "object" ||+ visited.has(current)+ ) {+ continue;+ }++ visited.add(current);+ const value = current as Record<string, unknown>;+ if (+ typeof value.name === "string" &&+ LLM_EVAL_CUSTOMER_OUTPUT_ERROR_NAMES.has(value.name)+ ) {+ return true;+ }++ pending.push(value.cause, value.lastError);+ }++ return false;+}+function getEvaluatorTypeTag(executionType: ObservationEvalExecutionType,): "llm_as_judge" | "code_as_judge" {