Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(evals): guide structured evaluator output (#16887)

Tobias Wochinger31598e760fd1
diff --git a/worker/src/features/evaluation/__tests__/evalExecutionDeps.test.ts b/worker/src/features/evaluation/__tests__/evalExecutionDeps.test.ts
index 2112e6f509..4f0c05e759 100644
--- a/worker/src/features/evaluation/__tests__/evalExecutionDeps.test.ts
+++ b/worker/src/features/evaluation/__tests__/evalExecutionDeps.test.ts
@@ -146,6 +146,8 @@ describe("createProductionEvalExecutionDeps", () => {
schema: z.ZodType;
};
expect(z.toJSONSchema(modelOutput.schema)).toMatchObject({
+ description:
+ 'Return only top-level "score" and "scoreExplanation". Put other requested fields inside "scoreExplanation".',
properties: {
scoreExplanation: { type: "string" },
score: { type: "number" },
@@ -219,10 +221,14 @@ describe("createProductionEvalExecutionDeps", () => {
? Buffer.from(value).toString("base64")
: value,
);
- const modelFacingSchema = z.object({
- scoreExplanation: structuredOutputSchema.shape.reasoning,
- score: structuredOutputSchema.shape.score,
- });
+ const modelFacingSchema = z
+ .object({
+ scoreExplanation: structuredOutputSchema.shape.reasoning,
+ score: structuredOutputSchema.shape.score,
+ })
+ .describe(
+ 'Return only top-level "score" and "scoreExplanation". Put other requested fields inside "scoreExplanation".',
+ );
const expectedBytes =
Buffer.byteLength(serializedProviderMessages, "utf8") +
Buffer.byteLength(
diff --git a/worker/src/features/evaluation/evalExecutionDeps.ts b/worker/src/features/evaluation/evalExecutionDeps.ts
index 46cca6ba09..89fdc76d77 100644
--- a/worker/src/features/evaluation/evalExecutionDeps.ts
+++ b/worker/src/features/evaluation/evalExecutionDeps.ts
@@ -25,6 +25,9 @@ type StructuredOutputSchema = z.ZodObject<{
score: z.ZodType;
}>;
+const MODEL_FACING_OUTPUT_SCHEMA_DESCRIPTION =
+ 'Return only top-level "score" and "scoreExplanation". Put other requested fields inside "scoreExplanation".';
+
/**
* Result of fetching model configuration.
*/
@@ -246,12 +249,14 @@ export function createProductionEvalExecutionDeps(): EvalExecutionDeps {
adapter,
});
- // Keep the evaluator contract unchanged, but avoid exposing the
- // overloaded `reasoning` field name to the model.
- const modelFacingStructuredOutputSchema = z.object({
- scoreExplanation: params.structuredOutputSchema.shape.reasoning,
- score: params.structuredOutputSchema.shape.score,
- });
+ // Keep the evaluator contract unchanged while the model-facing schema
+ // resolves custom output instructions into the supported fields.
+ const modelFacingStructuredOutputSchema = z
+ .object({
+ scoreExplanation: params.structuredOutputSchema.shape.reasoning,
+ score: params.structuredOutputSchema.shape.score,
+ })
+ .describe(MODEL_FACING_OUTPUT_SCHEMA_DESCRIPTION);
// llmaj egress: provider-bound messages (including base64-expanded inline
// media) plus schema, uncompressed.