The Open Framework for Building Agents
fix(evals): take each session's latest usage by time in derived.usage (#5242)
prha448804d44121
diff --git a/.changeset/eval-usage-latest-capture.md b/.changeset/eval-usage-latest-capture.mdnew file mode 100644index 000000000..300ca91b0--- /dev/null+++ b/.changeset/eval-usage-latest-capture.md@@ -0,0 +1,5 @@+---+"eve": patch+---++An eval's `derived.usage` now takes each session's latest usage by time. An eval that captured a session again with an earlier snapshot reported that snapshot's lower total. Its `costUsd` is now absent when a counted session reported no cost, instead of summing only the sessions that did.diff --git a/docs/evals/reporters.mdx b/docs/evals/reporters.mdxindex 7e068f7f3..1e6bc284e 100644--- a/docs/evals/reporters.mdx+++ b/docs/evals/reporters.mdx@@ -103,7 +103,7 @@ An eval can create several sessions, and a long session can produce several trac`result.result.derived` holds the facts the runner derived from the eval's sessions, such as tool calls and `models`: the distinct ids of the models the steps started with, in session order. Each entry in `result.result.sessions` holds the same facts for one session, in first-use order, so a reporter can tell the primary session's models from those of the other sessions the eval created or attached. A delegated subagent runs in its own session, so its models appear only when the eval attaches that session with `t.target.attachSession`. Compaction and `auto` routing calls are not included.-`derived.usage` is the token usage and cost from each session's latest `session.waiting`, `turn.waiting`, `session.failed`, or `session.completed`, including what its delegated agents spent; on a turn, it is the session's total so far. The eval's `usage` counts each captured session once, by its latest usage, and leaves out sessions another captured session opened, because that session already counts their spend. It is absent when a counted session reported no usage. It leaves out compaction calls, the turns an eval sends directly to a child session it attached, and delegated spend counted after a session's last such event.+`derived.usage` is the token usage and cost from each session's latest `session.waiting`, `turn.waiting`, `session.failed`, or `session.completed`, including what its delegated agents spent; on a turn, it is the session's total so far. The eval's `usage` counts each captured session once, by its latest usage in time (an eval can capture a session again with an earlier snapshot), and leaves out sessions another captured session opened, because that session already counts their spend. It is absent when a counted session reported no usage, and its `costUsd` is absent when a counted session reported no cost, so a partial sum never reads as the eval's cost. Compaction summary calls are included. It leaves out the turns an eval sends directly to a child session it attached, and delegated spend counted after a session's last such event.Reporter callbacks for an eval stay ordered—eval start, traced session starts, then completion—even when several evals run concurrently. Reach for a custom reporter only when a destination isn't covered. The per-run artifacts under `.eve/evals/` retain the trace contexts alongside the result for ad-hoc inspection.diff --git a/packages/eve/src/evals/runner/derive-run-facts.ts b/packages/eve/src/evals/runner/derive-run-facts.tsindex 876b3ba65..e6ef9475d 100644--- a/packages/eve/src/evals/runner/derive-run-facts.ts+++ b/packages/eve/src/evals/runner/derive-run-facts.ts@@ -76,7 +76,6 @@ export function deriveRunFacts(let messageCount = 0;let reasoningBlockCount = 0;const models = new Set<string>();- let usage: TokenUsage | undefined;let failureCode: string | undefined;const ensureToolCall = (callId: string, name: string, input: JsonObject): MutableToolCall => {@@ -178,19 +177,7 @@ export function deriveRunFacts(break;}- case "session.waiting":- case "turn.waiting": {- usage = event.data.usage;- break;- }-- case "session.completed": {- usage = event.data?.usage;- break;- }-case "session.failed": {- usage = event.data.usage;failureCode = event.data.code;break;}@@ -214,11 +201,36 @@ export function deriveRunFacts(messageCount,reasoningBlockCount,models: [...models],- usage,+ usage: latestUsage(events)?.usage,failureCode,};}+/**+ * The last event in `events` (in stream order) that carries the session's+ * running usage total (`session.waiting`, `turn.waiting`, `session.failed`, or+ * `session.completed`), with its time, so an eval that captured a session more+ * than once can keep the latest total by time rather than by capture order.+ * `usage` is undefined when that event reported none.+ */+export function latestUsage(+ events: readonly MessageStreamEvent[],+): { readonly usage: TokenUsage | undefined; readonly at: number } | undefined {+ for (let i = events.length - 1; i >= 0; i--) {+ const event = events[i];+ if (event === undefined) continue;+ switch (event.type) {+ case "session.waiting":+ case "turn.waiting":+ case "session.failed":+ return { usage: event.data.usage, at: Date.parse(event.meta.at) };+ case "session.completed":+ return { usage: event.data?.usage, at: Date.parse(event.meta.at) };+ }+ }+ return undefined;+}+/*** Every call to an agent task, with its session from the task's* `agent.started` once that session opened.diff --git a/packages/eve/src/evals/runner/execute-task.test.ts b/packages/eve/src/evals/runner/execute-task.test.tsindex 79bca9246..b0868a542 100644--- a/packages/eve/src/evals/runner/execute-task.test.ts+++ b/packages/eve/src/evals/runner/execute-task.test.ts@@ -607,7 +607,42 @@ describe("executeTask", () => {expect(result.derived.usage).toEqual(usage(30, 1));});- it("sums cost over the counted sessions that reported one", async () => {+ it("keeps a recaptured session's latest usage by time, not by capture order", async () => {+ // The second capture is an earlier snapshot of the same session: fewer events, so its+ // `session.waiting` is stamped before the first capture's.+ const server = createScriptedServer([], {+ streams: [+ {+ sessionId: "scheduled",+ events: [+ turnStarted("turn_1"),+ messageCompleted("first", "turn_1"),+ turnStarted("turn_2"),+ messageCompleted("second", "turn_2"),+ sessionWaiting(usage(30, 1, 0.3)),+ ],+ },+ {+ sessionId: "scheduled",+ events: [turnStarted("turn_1"), sessionWaiting(usage(10, 1, 0.1))],+ },+ ],+ });+ vi.spyOn(globalThis, "fetch").mockImplementation(server.fetch);++ const { result } = await executeTask({+ client: new Client({ host: target.url }),+ target,+ evaluation: createTestEval(async (t) => {+ await t.target.attachSession("scheduled");+ await t.target.attachSession("scheduled");+ }, "stale-recapture-usage"),+ });++ expect(result.derived.usage).toEqual(usage(30, 1, 0.3));+ });++ it("reports tokens but no cost when a counted session reported no cost", async () => {const server = createScriptedServer([{sessionId: "priced",@@ -637,7 +672,9 @@ describe("executeTask", () => {}, "partial-cost"),});- expect(result.derived.usage?.costUsd).toBe(0.5);+ // 0.5 would read as the eval's cost while one session's spend is unknown.+ expect(result.derived.usage).toEqual({ ...usage(12, 2), costUsd: undefined });+ expect(result.derived.usage && "costUsd" in result.derived.usage).toBe(false);});it("reports no usage when a counted session reported none", async () => {diff --git a/packages/eve/src/evals/runner/execute-task.ts b/packages/eve/src/evals/runner/execute-task.tsindex 2b2d9a04b..2bee9e1fd 100644--- a/packages/eve/src/evals/runner/execute-task.ts+++ b/packages/eve/src/evals/runner/execute-task.ts@@ -13,7 +13,7 @@ import type {EveEvalTraceContext,EveEvalTurn,} from "#evals/types.js";-import { createEmptyDerivedFacts } from "#evals/runner/derive-run-facts.js";+import { createEmptyDerivedFacts, latestUsage } from "#evals/runner/derive-run-facts.js";import { EvalSessionManager } from "#evals/session-manager.js";import type { EvalSessionStartedEvent } from "#evals/session.js";import { createEvalContext } from "#evals/context.js";@@ -184,7 +184,11 @@ function combineDerivedFacts(sessions: readonly EveEvalSessionResult[]): EveEval/*** The eval's usage: each captured session's latest usage, counted once however often the eval* captured it, less the sessions another captured session opened, whose spend that session already- * counts. No usage when a counted session reported none.+ * counts. No usage when a counted session reported none, and no cost when a counted session reported+ * none, so a partial sum never reads as the eval's cost.+ *+ * "Latest" is by time across captures, not capture order: an eval can capture a session again with+ * an earlier snapshot (fewer events) after a fuller capture.*/function evalUsage(sessions: readonly EveEvalSessionResult[]): TokenUsage | undefined {const opened = new Set(@@ -194,12 +198,24 @@ function evalUsage(sessions: readonly EveEvalSessionResult[]): TokenUsage | unde),),);- const latestById = new Map(sessions.map((session) => [session.sessionId, session.derived.usage]));- const counted = [...latestById].flatMap(([id, usage]) =>- id !== undefined && opened.has(id) ? [] : [usage],+ const latestById = new Map<string | undefined, ReturnType<typeof latestUsage>>();+ for (const session of sessions) {+ const latest = latestUsage(session.events);+ const prior = latestById.get(session.sessionId);+ // A capture with no usage event never replaces one that has one; on equal times the later+ // capture wins.+ if (prior === undefined || (latest !== undefined && latest.at >= prior.at))+ latestById.set(session.sessionId, latest ?? prior);+ }+ const counted = [...latestById].flatMap(([id, latest]) =>+ id !== undefined && opened.has(id) ? [] : [latest?.usage],);+ if (counted.length === 0) return undefined;if (!counted.every((usage): usage is TokenUsage => usage !== undefined)) return undefined;- return counted.reduce(addTokenUsage);+ const total = counted.reduce(addTokenUsage);+ if (counted.every((usage) => usage.costUsd !== undefined)) return total;+ const { costUsd: _partial, ...withoutCost } = total;+ return withoutCost;}function selectPrimarySessionId(sessions: readonly EveEvalSessionResult[]): string | undefined {diff --git a/packages/eve/src/evals/types.ts b/packages/eve/src/evals/types.tsindex d2e570617..0346341e7 100644--- a/packages/eve/src/evals/types.ts+++ b/packages/eve/src/evals/types.ts@@ -116,8 +116,9 @@ export interface EveEvalDerivedFacts {* Token usage from the latest `session.waiting`, `turn.waiting`, `session.failed`, or* `session.completed`: the* session's own model calls plus what the agents it delegated to spent, so on a turn it is the- * session's total so far. For an eval, each captured session counts once, by its latest usage,- * except sessions another captured session opened. Absent when a counted session reported none.+ * session's total so far. For an eval, each captured session counts once, by its latest usage in+ * time across captures, except sessions another captured session opened. Absent when a counted+ * session reported none; `costUsd` is absent when a counted session reported no cost.*/readonly usage?: TokenUsage;readonly failureCode?: string;