The Open Framework for Building Agents
feat(eve): expose eval session transcripts (#2672)
Casey Gowriea3b23c0357ce
diff --git a/.changeset/add-eval-session-transcripts.md b/.changeset/add-eval-session-transcripts.mdnew file mode 100644index 000000000..d7b4cf860--- /dev/null+++ b/.changeset/add-eval-session-transcripts.md@@ -0,0 +1,5 @@+---+"eve": patch+---++Eval contexts and sessions now expose a formatted `transcript` of observed user and assistant messages, which can be passed directly to an LLM judge for multi-turn conversation grading.diff --git a/docs/evals/cases.mdx b/docs/evals/cases.mdxindex 237dcaed1..e8e9d9e70 100644--- a/docs/evals/cases.mdx+++ b/docs/evals/cases.mdx@@ -88,11 +88,17 @@ export default defineEval({async test(t) {const first = await t.send("My favorite word is marigold.");- const second = await t.send("Thanks for remembering.");+ const second = await t.send("What is my favorite word?");await t.require(second.sessionId, equals(first.sessionId));t.succeeded();- second.messageIncludes("Thanks for remembering.");+ second.messageIncludes("marigold");++ t.judge.autoevals+ .closedQA("The assistant remembers the user's favorite word across turns", {+ on: t.transcript,+ })+ .atLeast(0.8);},});```@@ -109,6 +115,9 @@ export default defineEval({- `t.respond(responses, options?)` answers specific pending input requests and sends them as the next turn.- `t.respondAll(optionId)` answers every pending input request with the same option and sends the responses as the next turn.- `t.reply` is the last assistant message (or `null`); `t.sessionId` is the current session id; `t.events` is the full typed event stream captured so far.+- `t.transcript` formats the primary session's observed user and assistant messages in turn order. Pass it as a judge's `on` value to grade the complete conversation. An independent session returned by `t.newSession()` exposes its own `session.transcript`.++The transcript uses `User:` and `Assistant:` labels separated by blank lines. It excludes reasoning, tool calls, tool results, and messages from other sessions. It updates after each turn settles, so read it after `await t.send(...)`, `await t.respond(...)`, or `await live.result()`.Each `send` (and `respond`/`respondAll`) resolves to an immutable turn with `.message`, `.data`, `.events`, `.inputRequests`, `.toolCalls`, `.sessionId`, `.status`, and `.expectOk()`. Use `.sessionId` to relate turns or attach follow-up work to the session that produced a specific turn. `expectOk()` throws only when the turn ended failed; a session left open for a next message is the normal end state of a successful turn.diff --git a/docs/evals/judge.mdx b/docs/evals/judge.mdxindex 733153e9f..b8233d66a 100644--- a/docs/evals/judge.mdx+++ b/docs/evals/judge.mdx@@ -48,6 +48,21 @@ const draft = await t.send("Draft the welcome email.");t.judge.autoevals.closedQA("professional tone", { on: draft.message }).atLeast(0.6);```+For a multi-turn eval, pass `t.transcript` to grade the primary session's complete observed conversation instead of only its final reply:++```ts+await t.send("My favorite word is marigold. Remember it.");+await t.send("What is my favorite word?");++t.judge.autoevals+ .closedQA("The assistant remembers the user's favorite word across turns", {+ on: t.transcript,+ })+ .atLeast(0.8);+```++`t.transcript` contains the session's user and assistant messages in turn order. It excludes reasoning, tool calls, and tool results. See [Multi-turn evals](./cases#multi-turn-evals) for the transcript format and independent sessions.+## Soft scoring and thresholdsJudge assertions are soft, so the threshold rides on the assertion handle. There is no separate thresholds map:diff --git a/docs/evals/overview.mdx b/docs/evals/overview.mdxindex 2e3f6687a..0f601be0b 100644--- a/docs/evals/overview.mdx+++ b/docs/evals/overview.mdx@@ -100,7 +100,7 @@ model: mockModel({`t` is both the driver and the assertion surface. There are no separate `input`, `run`, `checks`, or `scores` fields. You write ordinary control flow, sending turns and asserting inline.-- **Drive** the agent: `t.send(...)`, `t.start(...)`, `t.cancel()`, `t.respond(...)`, `t.respondAll(...)`, `t.sendFile(...)`, `t.requireInputRequest(...)`, `t.newSession()`. Live turns returned by `start()` can wait for typed mid-turn events before cancellation or settlement. Read what came back with `t.reply` (the last assistant message), `t.sessionId`, and `t.events`. See [Cases](./cases).+- **Drive** the agent: `t.send(...)`, `t.start(...)`, `t.cancel()`, `t.respond(...)`, `t.respondAll(...)`, `t.sendFile(...)`, `t.requireInputRequest(...)`, `t.newSession()`. Live turns returned by `start()` can wait for typed mid-turn events before cancellation or settlement. Read what came back with `t.reply` (the last assistant message), `t.transcript` (the primary session's user and assistant messages), `t.sessionId`, and `t.events`. See [Cases](./cases).- **Assert** with three surfaces, covered next.## Three assertion surfaces@@ -109,7 +109,7 @@ Each surface matches a genuinely different kind of judgment:- **Scoped methods** read the final whole run on `t`, snapshot one independent session when invoked there, or inspect one immutable `EveEvalTurn`. See [Assertions](./assertions).- **`t.check(value, assertion)`** grades an explicit value with a deterministic builder from `eve/evals/expect`, such as `t.check(t.reply, includes("sunny"))`. Grade `t.reply`, an intermediate draft, parsed JSON, or anything else. See [Assertions](./assertions).-- **`t.judge.autoevals.*`** is the LLM-as-judge surface, like `t.judge.autoevals.closedQA("cites a source")`. It grades `t.reply` by default and uses the configured judge model, never the agent under test. See [Judge](./judge).+- **`t.judge.autoevals.*`** is the LLM-as-judge surface, like `t.judge.autoevals.closedQA("cites a source")`. It grades `t.reply` by default; pass `{ on: t.transcript }` to grade a multi-turn conversation. The judge uses the configured judge model, never the agent under test. See [Judge](./judge).## Gate vs softdiff --git a/e2e/fixtures/agent-basic-runtime/evals/runtime/multi-turn-session.eval.ts b/e2e/fixtures/agent-basic-runtime/evals/runtime/multi-turn-session.eval.tsindex 696133678..99f61fe9b 100644--- a/e2e/fixtures/agent-basic-runtime/evals/runtime/multi-turn-session.eval.ts+++ b/e2e/fixtures/agent-basic-runtime/evals/runtime/multi-turn-session.eval.ts@@ -1,5 +1,5 @@import { defineEval } from "eve/evals";-import { equals } from "eve/evals/expect";+import { equals, includes } from "eve/evals/expect";/*** Core session-route runtime behavior: multi-turn session continuity.@@ -19,5 +19,7 @@ export default defineEval({t.succeeded();t.messageIncludes(/marigold/i);+ t.check(t.transcript, includes("User:\nMy favorite word is marigold. Remember it."));+ t.check(t.transcript, includes("Assistant:\n"));},});diff --git a/packages/eve/src/evals/context.ts b/packages/eve/src/evals/context.tsindex 39d47fae8..2c5dcb08e 100644--- a/packages/eve/src/evals/context.ts+++ b/packages/eve/src/evals/context.ts@@ -44,6 +44,9 @@ export function createEvalContext(deps: {get events() {return primary().events;},+ get transcript() {+ return primary().transcript;+ },get pendingInputRequests() {return primary().pendingInputRequests;},diff --git a/packages/eve/src/evals/runner/execute-task.test.ts b/packages/eve/src/evals/runner/execute-task.test.tsindex ab4c02b3a..ad7201d10 100644--- a/packages/eve/src/evals/runner/execute-task.test.ts+++ b/packages/eve/src/evals/runner/execute-task.test.ts@@ -177,6 +177,53 @@ describe("executeTask", () => {]);});+ it("exposes the primary session transcript after each turn", async () => {+ const server = createScriptedServer([+ {+ sessionId: "session_1",+ events: [+ turnStarted("turn_1"),+ messageReceived("Remember marigold.", "turn_1"),+ messageCompleted("I will remember marigold.", "turn_1"),+ turnCompleted("turn_1"),+ sessionWaiting(),+ ],+ },+ {+ sessionId: "session_1",+ events: [+ turnStarted("turn_2"),+ messageReceived("What word did I ask you to remember?", "turn_2"),+ messageCompleted("marigold", "turn_2"),+ turnCompleted("turn_2"),+ sessionCompleted(),+ ],+ },+ ]);+ vi.spyOn(globalThis, "fetch").mockImplementation(server.fetch);++ await executeTask({+ client: new Client({ host: target.url }),+ target,+ evaluation: createTestEval(async (t) => {+ await t.send("Remember marigold.");+ expect(t.transcript).toBe(+ "User:\nRemember marigold.\n\nAssistant:\nI will remember marigold.",+ );++ await t.send("What word did I ask you to remember?");+ expect(t.transcript).toBe(+ [+ "User:\nRemember marigold.",+ "Assistant:\nI will remember marigold.",+ "User:\nWhat word did I ask you to remember?",+ "Assistant:\nmarigold",+ ].join("\n\n"),+ );+ }, "transcript"),+ });+ });+it("sends a single turn for input evals", async () => {const server = createScriptedServer([{@@ -218,6 +265,7 @@ describe("executeTask", () => {sessionId: "secondary",events: [turnStarted("turn_2"),+ messageReceived("secondary", "turn_2"),messageCompleted("secondary done", "turn_2"),actionsRequested("turn_2", "get_weather"),turnCompleted("turn_2"),@@ -227,18 +275,22 @@ describe("executeTask", () => {]);vi.spyOn(globalThis, "fetch").mockImplementation(server.fetch);+ let secondaryTranscript: string | undefined;const { result } = await executeTask({client: new Client({ host: target.url }),target,evaluation: createTestEval(async (t) => {await t.send("primary");- await t.newSession().send("secondary");+ const secondary = t.newSession();+ await secondary.send("secondary");+ secondaryTranscript = secondary.transcript;}, "multi-session"),});expect(result.sessionId).toBe("primary");expect(result.sessions?.map((session) => session.sessionId)).toEqual(["primary", "secondary"]);- expect(result.events).toHaveLength(9);+ expect(result.events).toHaveLength(10);+ expect(secondaryTranscript).toBe("User:\nsecondary\n\nAssistant:\nsecondary done");expect(result.derived.toolCalls.map((call) => call.sessionId)).toEqual(["secondary"]);});@@ -759,6 +811,13 @@ function turnStarted(return { data: { sequence: 0, trace, turnId }, type: "turn.started" };}+function messageReceived(message: string, turnId: string): UnstampedMessageStreamEvent {+ return {+ data: { message, parts: [{ text: message, type: "text" }], sequence: 1, turnId },+ type: "message.received",+ };+}+function turnCompleted(turnId: string): UnstampedMessageStreamEvent {return { data: { sequence: 3, turnId }, type: "turn.completed" };}diff --git a/packages/eve/src/evals/session-content.ts b/packages/eve/src/evals/session-content.tsnew file mode 100644index 000000000..d80c8cb54--- /dev/null+++ b/packages/eve/src/evals/session-content.ts@@ -0,0 +1,33 @@+import { extname } from "node:path";++import type { MessageStreamEvent } from "#protocol/message.js";++/** Formats the user and assistant messages observed in one eval session. */+export function formatEvalTranscript(events: readonly MessageStreamEvent[]): string {+ const messages: string[] = [];+ for (const event of events) {+ if (event.type === "message.received") {+ messages.push(`User:\n${event.data.message}`);+ } else if (event.type === "message.completed" && event.data.message !== null) {+ messages.push(`Assistant:\n${event.data.message}`);+ }+ }+ return messages.join("\n\n");+}++/** Infers the media type used when an eval attaches a local file. */+export function inferMediaType(filePath: string): string {+ switch (extname(filePath).toLowerCase()) {+ case ".gif":+ return "image/gif";+ case ".jpg":+ case ".jpeg":+ return "image/jpeg";+ case ".png":+ return "image/png";+ case ".webp":+ return "image/webp";+ default:+ return "application/octet-stream";+ }+}diff --git a/packages/eve/src/evals/session.ts b/packages/eve/src/evals/session.tsindex dd7acb8dc..cff1e05e4 100644--- a/packages/eve/src/evals/session.ts+++ b/packages/eve/src/evals/session.ts@@ -1,5 +1,5 @@import { readFile } from "node:fs/promises";-import { basename, extname } from "node:path";+import { basename } from "node:path";import { createTextWithFileContent } from "#client/file-parts.js";import type { Client } from "#client/client.js";@@ -21,6 +21,7 @@ import { summarizeTurnEvents } from "#client/session-utils.js";import { extractCompletedResult } from "#client/output-schema.js";import type { InputRequest, InputResponse } from "#shared/input.js";import { deriveRunFacts } from "#evals/runner/derive-run-facts.js";+import { formatEvalTranscript, inferMediaType } from "#evals/session-content.js";import { AssertionCollector } from "#evals/assertions/collector.js";import { createOutputAssertions, createScopedAssertions } from "#evals/assertions/scoped.js";import { EvalRequirementFailed } from "#evals/control-flow.js";@@ -115,6 +116,10 @@ export class EvalSessionDriver implements EveEvalSession {return this.#events;}+ get transcript(): string {+ return formatEvalTranscript(this.#events);+ }+get lastTurn(): EveEvalTurn | undefined {return this.#lastTurn;}@@ -682,19 +687,3 @@ function assertRequestHasOption(request: InputRequest, optionId: string): void {throw new Error(`Input request "${request.requestId}" does not offer option "${optionId}".`);}}--function inferMediaType(filePath: string): string {- switch (extname(filePath).toLowerCase()) {- case ".gif":- return "image/gif";- case ".jpg":- case ".jpeg":- return "image/jpeg";- case ".png":- return "image/png";- case ".webp":- return "image/webp";- default:- return "application/octet-stream";- }-}diff --git a/packages/eve/src/evals/types.ts b/packages/eve/src/evals/types.tsindex f3518e8fc..c850aea51 100644--- a/packages/eve/src/evals/types.ts+++ b/packages/eve/src/evals/types.ts@@ -291,6 +291,11 @@ export interface EveEvalLiveTurn {export interface EveEvalSessionDriver {/** All events observed on this session so far. */readonly events: readonly MessageStreamEvent[];+ /**+ * User and assistant messages observed on this session in turn order. Pass+ * this to a judge's `on` option to grade the complete conversation.+ */+ readonly transcript: string;/** Input requests left pending by the last parked turn. */readonly pendingInputRequests: readonly InputRequest[];/** Serializable cursor for resuming this session. */