The Open Framework for Building Agents
import { defineEval } from "eve/evals";
import { z } from "zod";
const StructuredOutput = z.object({ count: z.number().int(), title: z.string() });
/**
* Core session-route runtime behavior: structured output turns.
*
* The model answers outputSchema turns with schema-conforming structured
* data; the next plain turn must not leak another `result.completed`.
*/
export default defineEval({
tags: ["real-model"],
description: "Session runtime smoke: output schema.",
async test(t) {
const structured = await t.send("Summarize this turn as structured output.", {
outputSchema: {
properties: { count: { type: "integer" }, title: { type: "string" } },
required: ["title", "count"],
type: "object",
},
});
const session = structured.session;
structured.expectOk();
const plain = await session.send("Reply normally without structured output.");
plain.expectOk();
plain.notEvent("result.completed");
t.succeeded();
// Real models choose their own field values; assert schema conformance
// rather than an exact payload.
structured.outputMatches(StructuredOutput);
},
});