🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(evals): guide structured evaluator output (#16887)
Tobias Wochinger31598e760fd1
diff --git a/worker/src/features/evaluation/__tests__/evalExecutionDeps.test.ts b/worker/src/features/evaluation/__tests__/evalExecutionDeps.test.tsindex 2112e6f509..4f0c05e759 100644--- a/worker/src/features/evaluation/__tests__/evalExecutionDeps.test.ts+++ b/worker/src/features/evaluation/__tests__/evalExecutionDeps.test.ts@@ -146,6 +146,8 @@ describe("createProductionEvalExecutionDeps", () => {schema: z.ZodType;};expect(z.toJSONSchema(modelOutput.schema)).toMatchObject({+ description:+ 'Return only top-level "score" and "scoreExplanation". Put other requested fields inside "scoreExplanation".',properties: {scoreExplanation: { type: "string" },score: { type: "number" },@@ -219,10 +221,14 @@ describe("createProductionEvalExecutionDeps", () => {? Buffer.from(value).toString("base64"): value,);- const modelFacingSchema = z.object({- scoreExplanation: structuredOutputSchema.shape.reasoning,- score: structuredOutputSchema.shape.score,- });+ const modelFacingSchema = z+ .object({+ scoreExplanation: structuredOutputSchema.shape.reasoning,+ score: structuredOutputSchema.shape.score,+ })+ .describe(+ 'Return only top-level "score" and "scoreExplanation". Put other requested fields inside "scoreExplanation".',+ );const expectedBytes =Buffer.byteLength(serializedProviderMessages, "utf8") +Buffer.byteLength(diff --git a/worker/src/features/evaluation/evalExecutionDeps.ts b/worker/src/features/evaluation/evalExecutionDeps.tsindex 46cca6ba09..89fdc76d77 100644--- a/worker/src/features/evaluation/evalExecutionDeps.ts+++ b/worker/src/features/evaluation/evalExecutionDeps.ts@@ -25,6 +25,9 @@ type StructuredOutputSchema = z.ZodObject<{score: z.ZodType;}>;+const MODEL_FACING_OUTPUT_SCHEMA_DESCRIPTION =+ 'Return only top-level "score" and "scoreExplanation". Put other requested fields inside "scoreExplanation".';+/*** Result of fetching model configuration.*/@@ -246,12 +249,14 @@ export function createProductionEvalExecutionDeps(): EvalExecutionDeps {adapter,});- // Keep the evaluator contract unchanged, but avoid exposing the- // overloaded `reasoning` field name to the model.- const modelFacingStructuredOutputSchema = z.object({- scoreExplanation: params.structuredOutputSchema.shape.reasoning,- score: params.structuredOutputSchema.shape.score,- });+ // Keep the evaluator contract unchanged while the model-facing schema+ // resolves custom output instructions into the supported fields.+ const modelFacingStructuredOutputSchema = z+ .object({+ scoreExplanation: params.structuredOutputSchema.shape.reasoning,+ score: params.structuredOutputSchema.shape.score,+ })+ .describe(MODEL_FACING_OUTPUT_SCHEMA_DESCRIPTION);// llmaj egress: provider-bound messages (including base64-expanded inline// media) plus schema, uncompressed.