Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

1.1 KB
Raw Blame
import { defineEval } from "eve/evals";
import { z } from "zod";

const StructuredOutput = z.object({ count: z.number().int(), title: z.string() });

/**
 * Core session-route runtime behavior: structured output turns.
 *
 * The model answers outputSchema turns with schema-conforming structured
 * data; the next plain turn must not leak another `result.completed`.
 */
export default defineEval({
  tags: ["real-model"],
  description: "Session runtime smoke: output schema.",

  async test(t) {
    const structured = await t.send("Summarize this turn as structured output.", {
      outputSchema: {
        properties: { count: { type: "integer" }, title: { type: "string" } },
        required: ["title", "count"],
        type: "object",
      },
    });
    const session = structured.session;
    structured.expectOk();

    const plain = await session.send("Reply normally without structured output.");
    plain.expectOk();
    plain.notEvent("result.completed");

    t.succeeded();
    // Real models choose their own field values; assert schema conformance
    // rather than an exact payload.
    structured.outputMatches(StructuredOutput);
  },
});