Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

ci(eval-benchmark): add derived metrics in eval benchmark records (#5236)

prha8aedd027d736
diff --git a/package.json b/package.json
index ebbad9aa3..cc415b072 100644
--- a/package.json
+++ b/package.json
@@ -26,7 +26,7 @@
"test": "pnpm test:unit && pnpm test:integration",
"test:e2e": "turbo run test:e2e --concurrency=1",
"test:webchat": "pnpm --filter eve run test:webchat",
- "test:eval-benchmark": "node --test scripts/eval-benchmark/*.test.mjs",
+ "test:eval-benchmark": "node --test scripts/eval-metrics/*.test.mjs scripts/eval-benchmark/*.test.mjs",
"test:framework-fixtures": "pnpm --filter eve run test:framework-fixtures",
"test:integration": "turbo run test:integration --filter=eve && turbo run test:integration --filter=\"!eve\"",
"test:scenario": "turbo run test:scenario",
diff --git a/scripts/eval-benchmark/records.mjs b/scripts/eval-benchmark/records.mjs
index 12a27d5c6..41d278dca 100644
--- a/scripts/eval-benchmark/records.mjs
+++ b/scripts/eval-benchmark/records.mjs
@@ -9,8 +9,16 @@
// (github_run_id, run_attempt, fixture, eval_id, model_id, world, attempt).
// Results are comparable across runs when (eval_id, fixture_digest, model_id,
// model_release, world) match. Planned legs that uploaded nothing, and expected
-// evals with no result, become `missing` records so coverage stays honest.
-// Richer measurements can be derived later from the published raw trees.
+// evals with no result, become `missing` records (outcome "gap") so coverage
+// stays honest.
+//
+// Each attempt also carries the standard metric set (latency, attribution,
+// cost, tokens, counts, reliability) derived from its eval detail artifact by
+// scripts/eval-metrics/standard.mjs, and an outcome from
+// scripts/eval-metrics/gaps.mjs that separates infrastructure gaps (provider
+// outages, timeouts before any model step) from real failures. A metric that
+// can't be derived is `unavailable` or `not-applicable` with a reason, never
+// zero.
//
// A live leg's eval whose model steps all ran on another model (fixtures script
// some delegation evals with eve's mock models) says nothing about the leg's
@@ -21,6 +29,13 @@ import { existsSync, readdirSync, readFileSync, writeFileSync } from "node:fs";
import { join } from "node:path";
import { fileURLToPath } from "node:url";
+import { classifyOutcome } from "../eval-metrics/gaps.mjs";
+import {
+ METRICS_VERSION,
+ deriveAttemptMetrics,
+ listEvalArtifacts,
+} from "../eval-metrics/standard.mjs";
+
export const RECORD_SCHEMA_VERSION = 1;
export const LEG_ARTIFACT_PREFIX = "benchmark-leg-";
@@ -105,6 +120,14 @@ export function buildRecords({ plan, legs, githubRunId, runAttempt }) {
const records = attempts.map(({ leg, meta, evalId, artifact, missingReason, uploaded }) => {
const startedAt = artifact?.startedAt ?? null;
const completedAt = artifact?.completedAt ?? null;
+ const { outcome, gap_reason } =
+ artifact === undefined
+ ? { outcome: "gap", gap_reason: missingReason }
+ : classifyOutcome(artifact);
+ const { metrics_version, metrics, failures } =
+ artifact === undefined
+ ? { metrics_version: METRICS_VERSION, metrics: {}, failures: [] }
+ : deriveAttemptMetrics(artifact);
return {
schema_version: RECORD_SCHEMA_VERSION,
github_run_id: String(githubRunId),
@@ -129,6 +152,11 @@ export function buildRecords({ plan, legs, githubRunId, runAttempt }) {
passed,
})),
error: artifact?.error ?? null,
+ outcome,
+ gap_reason: gap_reason ?? null,
+ metrics_version,
+ metrics,
+ failures,
started_at: startedAt,
completed_at: completedAt,
duration_ms:
@@ -150,16 +178,6 @@ function ranOnModel(artifact, modelId) {
return models.length === 0 || models.includes(modelId);
}
-/** Find every eval detail artifact under one `.eve/evals/<timestamp>` tree. */
-export function listEvalArtifacts(runDirectory) {
- const evalsDirectory = join(runDirectory, "evals");
- if (!existsSync(evalsDirectory)) return [];
- return readdirSync(evalsDirectory, { withFileTypes: true, recursive: true })
- .filter((entry) => entry.isFile() && entry.name.endsWith(".json"))
- .map((entry) => join(entry.parentPath, entry.name))
- .sort();
-}
-
if (process.argv[1] === fileURLToPath(import.meta.url)) {
const [planPath, legsDir, outPath] = process.argv.slice(2);
if (outPath === undefined) {
diff --git a/scripts/eval-benchmark/records.test.mjs b/scripts/eval-benchmark/records.test.mjs
index 31c725c62..dc1edfb5f 100644
--- a/scripts/eval-benchmark/records.test.mjs
+++ b/scripts/eval-benchmark/records.test.mjs
@@ -1,9 +1,10 @@
import test from "node:test";
import assert from "node:assert/strict";
-import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { copyFileSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { publishRun } from "./publish.mjs";
+import { METRICS_VERSION, STANDARD_METRICS } from "../eval-metrics/standard.mjs";
import { buildRecords, readLegs } from "./records.mjs";
const SHA = "a".repeat(40);
@@ -115,6 +116,49 @@ test("builds records and turns missing legs and evals into missing records", (t)
]);
assert.equal(records[2].model_release, null);
assert.equal(records[2].duration_ms, null);
+ assert.equal(completed.outcome, "completed");
+ assert.deepEqual(completed.metrics["latency.turn_ms"], {
+ status: "unavailable",
+ reason: "missing-session-capture",
+ });
+});
+
+test("attaches metrics to captured attempts and marks missing attempts as gaps", (t) => {
+ const legsDir = legsDirectory(t);
+ // A real captured artifact, with session events, in place of the minimal one.
+ copyFileSync(
+ new URL("../eval-metrics/fixtures/agent-tools-ends-turn-function.json", import.meta.url),
+ join(
+ legsDir,
+ "benchmark-leg-live__agent-tools__alpha__local__1",
+ "run",
+ "evals",
+ "static-tools",
+ "ends-turn-function.json",
+ ),
+ );
+ const [captured, missingEval, missingJob] = build(legsDir);
+ assert.equal(captured.outcome, "completed");
+ assert.equal(captured.gap_reason, null);
+ assert.equal(captured.metrics_version, METRICS_VERSION);
+ assert.deepEqual(Object.keys(captured.metrics).sort(), Object.keys(STANDARD_METRICS).sort());
+ assert.deepEqual(captured.metrics["tokens.input"], { status: "measured", value: 1117 });
+ // The fixture's model reports no cost: unavailable with a reason, never zero.
+ assert.deepEqual(captured.metrics["cost.total_usd"], {
+ status: "unavailable",
+ reason: "missing-rolled-up-cost",
+ });
+ assert.deepEqual(captured.failures, []);
+
+ for (const [record, reason] of [
+ [missingEval, "eval-missing"],
+ [missingJob, "job-missing"],
+ ]) {
+ assert.equal(record.outcome, "gap");
+ assert.equal(record.gap_reason, reason);
+ assert.deepEqual(record.metrics, {});
+ assert.deepEqual(record.failures, []);
+ }
});
test("excludes live-leg evals whose model steps all ran on another model", (t) => {
diff --git a/scripts/eval-metrics/fixtures/agent-subagents-continuation-local.json b/scripts/eval-metrics/fixtures/agent-subagents-continuation-local.json
new file mode 100644
index 000000000..33cddbba2
--- /dev/null
+++ b/scripts/eval-metrics/fixtures/agent-subagents-continuation-local.json
@@ -0,0 +1,798 @@
+{
+ "id": "agent-continuation-local",
+ "verdict": "passed",
+ "startedAt": "2026-10-05T23:37:05.534Z",
+ "completedAt": "2026-10-05T23:37:08.653Z",
+ "result": {
+ "derived": {
+ "usage": {
+ "cacheReadTokens": 0,
+ "cacheWriteTokens": 0,
+ "inputTokens": 6006,
+ "outputTokens": 178
+ }
+ },
+ "sessionId": "wrun_01M476V5RJPATF4WMMJD3PAX09",
+ "runtimeIdentity": {
+ "eveVersion": "0.71.2"
+ },
+ "sessions": [
+ {
+ "sessionId": "wrun_01M476V5RJPATF4WMMJD3PAX09",
+ "events": [
+ {
+ "type": "turn.started",
+ "meta": {
+ "id": "evt_01M476V6228BYTWHV7MKWND209",
+ "at": "2026-10-05T23:37:05.858Z"
+ },
+ "data": {
+ "turnId": "turn_0"
+ }
+ },
+ {
+ "type": "step.started",
+ "meta": {
+ "id": "evt_01M476V6233AEN19P97Z73SW8Y",
+ "at": "2026-10-05T23:37:05.859Z"
+ },
+ "data": {
+ "turnId": "turn_0",
+ "stepIndex": 0
+ }
+ },
+ {
+ "type": "actions.requested",
+ "meta": {
+ "id": "evt_01M476V62C7GS0XB8YSMBRD14H",
+ "at": "2026-10-05T23:37:05.868Z"
+ },
+ "data": {
+ "actions": [
+ {
+ "callId": "notebook-remember",
+ "kind": "tool-call"
+ }
+ ]
+ }
+ },
+ {
+ "type": "step.completed",
+ "meta": {
+ "id": "evt_01M476V62FPZP28T46PNM7S065",
+ "at": "2026-10-05T23:37:05.871Z"
+ },
+ "data": {
+ "turnId": "turn_0",
+ "stepIndex": 0,
+ "usage": {
+ "inputTokens": 332,
+ "outputTokens": 27,
+ "cacheReadTokens": 0,
+ "cacheWriteTokens": 0
+ },
+ "finishReason": "tool-calls"
+ }
+ },
+ {
+ "type": "task.started",
+ "meta": {
+ "id": "evt_01M476V63J65W3D9RW1NNBY17K",
+ "at": "2026-10-05T23:37:05.906Z"
+ },
+ "data": {
+ "taskId": "notebook-keeper-eqtna6",
+ "callId": "notebook-remember"
+ }
+ },
+ {
+ "type": "action.result",
+ "meta": {
+ "id": "evt_01M476V641X0H8BK1KC4JV92AZ",
+ "at": "2026-10-05T23:37:05.921Z"
+ },
+ "data": {
+ "result": {
+ "callId": "notebook-remember"
+ },
+ "status": "completed"
+ }
+ },
+ {
+ "type": "step.started",
+ "meta": {
+ "id": "evt_01M476V642AZR2058TEEM9X5EE",
+ "at": "2026-10-05T23:37:05.922Z"
+ },
+ "data": {
+ "turnId": "turn_0",
+ "stepIndex": 1
+ }
+ },
+ {
+ "type": "actions.requested",
+ "meta": {
+ "id": "evt_01M476V645XV19K4368X5VK1B6",
+ "at": "2026-10-05T23:37:05.925Z"
+ },
+ "data": {
+ "actions": [
+ {
+ "callId": "notebook-remember-wait",
+ "kind": "tool-call"
+ }
+ ]
+ }
+ },
+ {
+ "type": "step.completed",
+ "meta": {
+ "id": "evt_01M476V646ZCXJ0YY0CE8DSFT3",
+ "at": "2026-10-05T23:37:05.926Z"
+ },
+ "data": {
+ "turnId": "turn_0",
+ "stepIndex": 1,
+ "usage": {
+ "inputTokens": 402,
+ "outputTokens": 1,
+ "cacheReadTokens": 0,
+ "cacheWriteTokens": 0
+ },
+ "finishReason": "tool-calls"
+ }
+ },
+ {
+ "type": "agent.started",
+ "meta": {
+ "id": "evt_01M476V6898SRG54ZSBFDFWJZH",
+ "at": "2026-10-05T23:37:06.057Z"
+ },
+ "data": {
+ "sessionId": "wrun_01M476V658DK2JBTYGB4XWHDZ0"
+ }
+ },
+ {
+ "type": "task.settled",
+ "meta": {
+ "id": "evt_01M476V6A2Z1SG7X142798JF05",
+ "at": "2026-10-05T23:37:06.114Z"
+ },
+ "data": {
+ "taskId": "notebook-keeper-eqtna6",
+ "callId": "notebook-remember"
+ }
+ },
+ {
+ "type": "action.result",
+ "meta": {
+ "id": "evt_01M476V6AF87ZGQAP6CT77QW5G",
+ "at": "2026-10-05T23:37:06.127Z"
+ },
+ "data": {
+ "result": {
+ "callId": "notebook-remember-wait"
+ },
+ "status": "completed"
+ }
+ },
+ {
+ "type": "step.started",
+ "meta": {
+ "id": "evt_01M476V6AF87ZGQAP6CT77QW5H",
+ "at": "2026-10-05T23:37:06.127Z"
+ },
+ "data": {
+ "turnId": "turn_0",
+ "stepIndex": 2
+ }
+ },
+ {
+ "type": "message.appended",
+ "meta": {
+ "id": "evt_01M476V6AJFFZPR5418NZFPRTZ",
+ "at": "2026-10-05T23:37:06.130Z"
+ },
+ "data": {
+ "turnId": "turn_0",
+ "stepIndex": 2
+ }
+ },
+ {
+ "type": "step.completed",
+ "meta": {
+ "id": "evt_01M476V6AJFFZPR5418NZFPRV1",
+ "at": "2026-10-05T23:37:06.130Z"
+ },
+ "data": {
+ "turnId": "turn_0",
+ "stepIndex": 2,
+ "usage": {
+ "inputTokens": 468,
+ "outputTokens": 8,
+ "cacheReadTokens": 0,
+ "cacheWriteTokens": 0
+ },
+ "finishReason": "stop"
+ }
+ },
+ {
+ "type": "turn.completed",
+ "meta": {
+ "id": "evt_01M476V6AN6BYGDG7F56X57112",
+ "at": "2026-10-05T23:37:06.133Z"
+ },
+ "data": {
+ "turnId": "turn_0"
+ }
+ },
+ {
+ "type": "turn.started",
+ "meta": {
+ "id": "evt_01M476V6C9MRXR1GD56EV73ZQF",
+ "at": "2026-10-05T23:37:06.185Z"
+ },
+ "data": {
+ "turnId": "turn_1"
+ }
+ },
+ {
+ "type": "step.started",
+ "meta": {
+ "id": "evt_01M476V6C9MRXR1GD56EV73ZQH",
+ "at": "2026-10-05T23:37:06.185Z"
+ },
+ "data": {
+ "turnId": "turn_1",
+ "stepIndex": 0
+ }
+ },
+ {
+ "type": "actions.requested",
+ "meta": {
+ "id": "evt_01M476V6CC7A5R3Q13GNTS1WV3",
+ "at": "2026-10-05T23:37:06.188Z"
+ },
+ "data": {
+ "actions": [
+ {
+ "callId": "notebook-review",
+ "kind": "tool-call"
+ }
+ ]
+ }
+ },
+ {
+ "type": "step.completed",
+ "meta": {
+ "id": "evt_01M476V6CC7A5R3Q13GNTS1WV4",
+ "at": "2026-10-05T23:37:06.188Z"
+ },
+ "data": {
+ "turnId": "turn_1",
+ "stepIndex": 0,
+ "usage": {
+ "inputTokens": 496,
+ "outputTokens": 32,
+ "cacheReadTokens": 0,
+ "cacheWriteTokens": 0
+ },
+ "finishReason": "tool-calls"
+ }
+ },
+ {
+ "type": "task.started",
+ "meta": {
+ "id": "evt_01M476V6D7X4DPT801AKTD751Q",
+ "at": "2026-10-05T23:37:06.215Z"
+ },
+ "data": {
+ "taskId": "notebook-keeper-eqtna6",
+ "callId": "notebook-review"
+ }
+ },
+ {
+ "type": "action.result",
+ "meta": {
+ "id": "evt_01M476V6DRHF4FYDKZ1NVPXHE3",
+ "at": "2026-10-05T23:37:06.232Z"
+ },
+ "data": {
+ "result": {
+ "callId": "notebook-review"
+ },
+ "status": "completed"
+ }
+ },
+ {
+ "type": "step.started",
+ "meta": {
+ "id": "evt_01M476V6DSJ0SD80XPPPRQZZBH",
+ "at": "2026-10-05T23:37:06.233Z"
+ },
+ "data": {
+ "turnId": "turn_1",
+ "stepIndex": 1
+ }
+ },
+ {
+ "type": "actions.requested",
+ "meta": {
+ "id": "evt_01M476V6DV5APK6YK4ARJ56BAJ",
+ "at": "2026-10-05T23:37:06.235Z"
+ },
+ "data": {
+ "actions": [
+ {
+ "callId": "notebook-review-wait",
+ "kind": "tool-call"
+ }
+ ]
+ }
+ },
+ {
+ "type": "step.completed",
+ "meta": {
+ "id": "evt_01M476V6DV5APK6YK4ARJ56BAK",
+ "at": "2026-10-05T23:37:06.235Z"
+ },
+ "data": {
+ "turnId": "turn_1",
+ "stepIndex": 1,
+ "usage": {
+ "inputTokens": 543,
+ "outputTokens": 5,
+ "cacheReadTokens": 0,
+ "cacheWriteTokens": 0
+ },
+ "finishReason": "tool-calls"
+ }
+ },
+ {
+ "type": "action.result",
+ "meta": {
+ "id": "evt_01M476V8E667SBF93W9HCSJ4VD",
+ "at": "2026-10-05T23:37:08.294Z"
+ },
+ "data": {
+ "result": {
+ "callId": "notebook-review-wait"
+ },
+ "status": "completed"
+ }
+ },
+ {
+ "type": "step.started",
+ "meta": {
+ "id": "evt_01M476V8E7S6HBJVCYS5FSH3SW",
+ "at": "2026-10-05T23:37:08.295Z"
+ },
+ "data": {
+ "turnId": "turn_1",
+ "stepIndex": 2
+ }
+ },
+ {
+ "type": "actions.requested",
+ "meta": {
+ "id": "evt_01M476V8EAWVYYB5H2Q699KMG6",
+ "at": "2026-10-05T23:37:08.298Z"
+ },
+ "data": {
+ "actions": [
+ {
+ "callId": "notebook-cancel",
+ "kind": "tool-call"
+ }
+ ]
+ }
+ },
+ {
+ "type": "step.completed",
+ "meta": {
+ "id": "evt_01M476V8EBTW72T7YGSZTNMA8T",
+ "at": "2026-10-05T23:37:08.299Z"
+ },
+ "data": {
+ "turnId": "turn_1",
+ "stepIndex": 2,
+ "usage": {
+ "inputTokens": 576,
+ "outputTokens": 9,
+ "cacheReadTokens": 0,
+ "cacheWriteTokens": 0
+ },
+ "finishReason": "tool-calls"
+ }
+ },
+ {
+ "type": "task.settled",
+ "meta": {
+ "id": "evt_01M476V8F8Z1FX7TZSZ7ZHDJRX",
+ "at": "2026-10-05T23:37:08.328Z"
+ },
+ "data": {
+ "taskId": "notebook-keeper-eqtna6",
+ "callId": "notebook-review"
+ }
+ },
+ {
+ "type": "action.result",
+ "meta": {
+ "id": "evt_01M476V8FTN14FSP3KYHVCYTRZ",
+ "at": "2026-10-05T23:37:08.346Z"
+ },
+ "data": {
+ "result": {
+ "callId": "notebook-cancel"
+ },
+ "status": "completed"
+ }
+ },
+ {
+ "type": "step.started",
+ "meta": {
+ "id": "evt_01M476V8FTN14FSP3KYHVCYTS0",
+ "at": "2026-10-05T23:37:08.346Z"
+ },
+ "data": {
+ "turnId": "turn_1",
+ "stepIndex": 3
+ }
+ },
+ {
+ "type": "message.appended",
+ "meta": {
+ "id": "evt_01M476V8FXZHQQ4Y9YX9JK4MDH",
+ "at": "2026-10-05T23:37:08.349Z"
+ },
+ "data": {
+ "turnId": "turn_1",
+ "stepIndex": 3
+ }
+ },
+ {
+ "type": "step.completed",
+ "meta": {
+ "id": "evt_01M476V8FXZHQQ4Y9YX9JK4MDK",
+ "at": "2026-10-05T23:37:08.349Z"
+ },
+ "data": {
+ "turnId": "turn_1",
+ "stepIndex": 3,
+ "usage": {
+ "inputTokens": 639,
+ "outputTokens": 42,
+ "cacheReadTokens": 0,
+ "cacheWriteTokens": 0
+ },
+ "finishReason": "stop"
+ }
+ },
+ {
+ "type": "turn.completed",
+ "meta": {
+ "id": "evt_01M476V8G10T9NCS0BYG14C6P5",
+ "at": "2026-10-05T23:37:08.353Z"
+ },
+ "data": {
+ "turnId": "turn_1"
+ }
+ },
+ {
+ "type": "turn.started",
+ "meta": {
+ "id": "evt_01M476V8K9NZ8YZ6QQKNAH92TD",
+ "at": "2026-10-05T23:37:08.457Z"
+ },
+ "data": {
+ "turnId": "turn_2"
+ }
+ },
+ {
+ "type": "step.started",
+ "meta": {
+ "id": "evt_01M476V8KA3WS00AS63TKXAKKC",
+ "at": "2026-10-05T23:37:08.458Z"
+ },
+ "data": {
+ "turnId": "turn_2",
+ "stepIndex": 0
+ }
+ },
+ {
+ "type": "actions.requested",
+ "meta": {
+ "id": "evt_01M476V8KCQ4RPNX1HD3Y1C7N8",
+ "at": "2026-10-05T23:37:08.460Z"
+ },
+ "data": {
+ "actions": [
+ {
+ "callId": "notebook-recall",
+ "kind": "tool-call"
+ }
+ ]
+ }
+ },
+ {
+ "type": "step.completed",
+ "meta": {
+ "id": "evt_01M476V8KCQ4RPNX1HD3Y1C7N9",
+ "at": "2026-10-05T23:37:08.460Z"
+ },
+ "data": {
+ "turnId": "turn_2",
+ "stepIndex": 0,
+ "usage": {
+ "inputTokens": 695,
+ "outputTokens": 29,
+ "cacheReadTokens": 0,
+ "cacheWriteTokens": 0
+ },
+ "finishReason": "tool-calls"
+ }
+ },
+ {
+ "type": "task.started",
+ "meta": {
+ "id": "evt_01M476V8M6Z762FW3TK0BPFPDC",
+ "at": "2026-10-05T23:37:08.486Z"
+ },
+ "data": {
+ "taskId": "notebook-keeper-eqtna6",
+ "callId": "notebook-recall"
+ }
+ },
+ {
+ "type": "action.result",
+ "meta": {
+ "id": "evt_01M476V8MNP1SETGP49BVH3SB6",
+ "at": "2026-10-05T23:37:08.501Z"
+ },
+ "data": {
+ "result": {
+ "callId": "notebook-recall"
+ },
+ "status": "completed"
+ }
+ },
+ {
+ "type": "step.started",
+ "meta": {
+ "id": "evt_01M476V8MPWG53PDJJX23ZGY1K",
+ "at": "2026-10-05T23:37:08.502Z"
+ },
+ "data": {
+ "turnId": "turn_2",
+ "stepIndex": 1
+ }
+ },
+ {
+ "type": "actions.requested",
+ "meta": {
+ "id": "evt_01M476V8MRFEP96D0MWRXSBG3S",
+ "at": "2026-10-05T23:37:08.504Z"
+ },
+ "data": {
+ "actions": [
+ {
+ "callId": "notebook-recall-wait",
+ "kind": "tool-call"
+ }
+ ]
+ }
+ },
+ {
+ "type": "step.completed",
+ "meta": {
+ "id": "evt_01M476V8MRFEP96D0MWRXSBG3T",
+ "at": "2026-10-05T23:37:08.504Z"
+ },
+ "data": {
+ "turnId": "turn_2",
+ "stepIndex": 1,
+ "usage": {
+ "inputTokens": 742,
+ "outputTokens": 1,
+ "cacheReadTokens": 0,
+ "cacheWriteTokens": 0
+ },
+ "finishReason": "tool-calls"
+ }
+ },
+ {
+ "type": "task.settled",
+ "meta": {
+ "id": "evt_01M476V8RPXJGVVE5F24WJ79AF",
+ "at": "2026-10-05T23:37:08.630Z"
+ },
+ "data": {
+ "taskId": "notebook-keeper-eqtna6",
+ "callId": "notebook-recall"
+ }
+ },
+ {
+ "type": "action.result",
+ "meta": {
+ "id": "evt_01M476V8S10A6Z1Y50ERZCD2A9",
+ "at": "2026-10-05T23:37:08.641Z"
+ },
+ "data": {
+ "result": {
+ "callId": "notebook-recall-wait"
+ },
+ "status": "completed"
+ }
+ },
+ {
+ "type": "step.started",
+ "meta": {
+ "id": "evt_01M476V8S2RMQRM83HM2NQ6N86",
+ "at": "2026-10-05T23:37:08.642Z"
+ },
+ "data": {
+ "turnId": "turn_2",
+ "stepIndex": 2
+ }
+ },
+ {
+ "type": "message.appended",
+ "meta": {
+ "id": "evt_01M476V8S3ZK1JPMHQEJ5WW503",
+ "at": "2026-10-05T23:37:08.643Z"
+ },
+ "data": {
+ "turnId": "turn_2",
+ "stepIndex": 2
+ }
+ },
+ {
+ "type": "step.completed",
+ "meta": {
+ "id": "evt_01M476V8S40C4W83AANGJFAY23",
+ "at": "2026-10-05T23:37:08.644Z"
+ },
+ "data": {
+ "turnId": "turn_2",
+ "stepIndex": 2,
+ "usage": {
+ "inputTokens": 813,
+ "outputTokens": 12,
+ "cacheReadTokens": 0,
+ "cacheWriteTokens": 0
+ },
+ "finishReason": "stop"
+ }
+ },
+ {
+ "type": "turn.completed",
+ "meta": {
+ "id": "evt_01M476V8S8M483PXMJW7TCF6A5",
+ "at": "2026-10-05T23:37:08.648Z"
+ },
+ "data": {
+ "turnId": "turn_2"
+ }
+ }
+ ]
+ },
+ {
+ "sessionId": "wrun_01M476V658DK2JBTYGB4XWHDZ0",
+ "events": [
+ {
+ "type": "turn.started",
+ "meta": {
+ "id": "evt_01M476V66SMTJ5E0FTNQ3KTBAF",
+ "at": "2026-10-05T23:37:06.009Z"
+ },
+ "data": {
+ "turnId": "turn_0"
+ }
+ },
+ {
+ "type": "step.started",
+ "meta": {
+ "id": "evt_01M476V66SMTJ5E0FTNQ3KTBAH",
+ "at": "2026-10-05T23:37:06.009Z"
+ },
+ "data": {
+ "turnId": "turn_0",
+ "stepIndex": 0
+ }
+ },
+ {
+ "type": "message.appended",
+ "meta": {
+ "id": "evt_01M476V66VDRJ4S2TSRXESVFGN",
+ "at": "2026-10-05T23:37:06.011Z"
+ },
+ "data": {
+ "turnId": "turn_0",
+ "stepIndex": 0
+ }
+ },
+ {
+ "type": "step.completed",
+ "meta": {
+ "id": "evt_01M476V66VDRJ4S2TSRXESVFGQ",
+ "at": "2026-10-05T23:37:06.011Z"
+ },
+ "data": {
+ "turnId": "turn_0",
+ "stepIndex": 0,
+ "usage": {
+ "inputTokens": 130,
+ "outputTokens": 4,
+ "cacheReadTokens": 0,
+ "cacheWriteTokens": 0
+ },
+ "finishReason": "stop"
+ }
+ },
+ {
+ "type": "turn.completed",
+ "meta": {
+ "id": "evt_01M476V671EP9FBHHQBZNAD0CT",
+ "at": "2026-10-05T23:37:06.017Z"
+ },
+ "data": {
+ "turnId": "turn_0"
+ }
+ }
+ ]
+ },
+ {
+ "sessionId": "wrun_01M476V658DK2JBTYGB4XWHDZ0",
+ "events": [
+ {
+ "type": "turn.started",
+ "meta": {
+ "id": "evt_01M476V6F9EBD55ZG238NQ1ZJW",
+ "at": "2026-10-05T23:37:06.281Z"
+ },
+ "data": {
+ "turnId": "turn_1"
+ }
+ },
+ {
+ "type": "step.started",
+ "meta": {
+ "id": "evt_01M476V6F9EBD55ZG238NQ1ZJY",
+ "at": "2026-10-05T23:37:06.281Z"
+ },
+ "data": {
+ "turnId": "turn_1",
+ "stepIndex": 0
+ }
+ },
+ {
+ "type": "actions.requested",
+ "meta": {
+ "id": "evt_01M476V6FBET7W3ABD5M66T5ZH",
+ "at": "2026-10-05T23:37:06.283Z"
+ },
+ "data": {
+ "actions": [
+ {
+ "callId": "notebook-review-2",
+ "kind": "tool-call"
+ }
+ ]
+ }
+ },
+ {
+ "type": "turn.cancelled",
+ "meta": {
+ "id": "evt_01M476V8HQR8KWGGH1VA5BHRGS",
+ "at": "2026-10-05T23:37:08.407Z"
+ },
+ "data": {
+ "turnId": "turn_1"
+ }
+ }
+ ]
+ }
+ ]
+ }
+}
diff --git a/scripts/eval-metrics/fixtures/agent-tools-ends-turn-function.json b/scripts/eval-metrics/fixtures/agent-tools-ends-turn-function.json
new file mode 100644
index 000000000..8b555bf15
--- /dev/null
+++ b/scripts/eval-metrics/fixtures/agent-tools-ends-turn-function.json
@@ -0,0 +1,221 @@
+{
+ "id": "static-tools/ends-turn-function",
+ "verdict": "passed",
+ "startedAt": "2026-10-05T23:35:48.752Z",
+ "completedAt": "2026-10-05T23:35:49.802Z",
+ "result": {
+ "derived": {
+ "usage": {
+ "cacheReadTokens": 0,
+ "cacheWriteTokens": 0,
+ "inputTokens": 1117,
+ "outputTokens": 29
+ }
+ },
+ "sessionId": "wrun_01M476RTSJ569Y4TN0BBRB2ED0",
+ "runtimeIdentity": {
+ "eveVersion": "0.71.2"
+ },
+ "sessions": [
+ {
+ "sessionId": "wrun_01M476RTSJ569Y4TN0BBRB2ED0",
+ "events": [
+ {
+ "type": "turn.started",
+ "meta": {
+ "id": "evt_01M476RV3PFF8YGR8F46VHK6NW",
+ "at": "2026-10-05T23:35:49.110Z"
+ },
+ "data": {
+ "turnId": "turn_0"
+ }
+ },
+ {
+ "type": "step.started",
+ "meta": {
+ "id": "evt_01M476RV3RZCCW92ZXEJWBW8WM",
+ "at": "2026-10-05T23:35:49.112Z"
+ },
+ "data": {
+ "turnId": "turn_0",
+ "stepIndex": 0
+ }
+ },
+ {
+ "type": "actions.requested",
+ "meta": {
+ "id": "evt_01M476RV41G16Z7FSF37GAG635",
+ "at": "2026-10-05T23:35:49.121Z"
+ },
+ "data": {
+ "actions": [
+ {
+ "callId": "mock-tool-call-1-0-1",
+ "kind": "tool-call"
+ }
+ ]
+ }
+ },
+ {
+ "type": "action.result",
+ "meta": {
+ "id": "evt_01M476RV4325FKVY9H3VTMBSEW",
+ "at": "2026-10-05T23:35:49.123Z"
+ },
+ "data": {
+ "result": {
+ "callId": "mock-tool-call-1-0-1"
+ },
+ "status": "completed"
+ }
+ },
+ {
+ "type": "step.completed",
+ "meta": {
+ "id": "evt_01M476RV45DWXA28BBVBFY5HQF",
+ "at": "2026-10-05T23:35:49.125Z"
+ },
+ "data": {
+ "turnId": "turn_0",
+ "stepIndex": 0,
+ "usage": {
+ "inputTokens": 344,
+ "outputTokens": 4,
+ "cacheReadTokens": 0,
+ "cacheWriteTokens": 0
+ },
+ "finishReason": "tool-calls"
+ }
+ },
+ {
+ "type": "turn.completed",
+ "meta": {
+ "id": "evt_01M476RV4NAD1MEWDF4H2CBQWB",
+ "at": "2026-10-05T23:35:49.141Z"
+ },
+ "data": {
+ "turnId": "turn_0"
+ }
+ },
+ {
+ "type": "turn.started",
+ "meta": {
+ "id": "evt_01M476RVPD601TWZ4WEK2RYKCG",
+ "at": "2026-10-05T23:35:49.709Z"
+ },
+ "data": {
+ "turnId": "turn_1"
+ }
+ },
+ {
+ "type": "step.started",
+ "meta": {
+ "id": "evt_01M476RVPD601TWZ4WEK2RYKCJ",
+ "at": "2026-10-05T23:35:49.709Z"
+ },
+ "data": {
+ "turnId": "turn_1",
+ "stepIndex": 0
+ }
+ },
+ {
+ "type": "actions.requested",
+ "meta": {
+ "id": "evt_01M476RVPFWRV23DNSPJERPZRX",
+ "at": "2026-10-05T23:35:49.711Z"
+ },
+ "data": {
+ "actions": [
+ {
+ "callId": "mock-tool-call-2-1-1",
+ "kind": "tool-call"
+ }
+ ]
+ }
+ },
+ {
+ "type": "action.result",
+ "meta": {
+ "id": "evt_01M476RVPGVTS0HK0BT3DHYRNB",
+ "at": "2026-10-05T23:35:49.712Z"
+ },
+ "data": {
+ "result": {
+ "callId": "mock-tool-call-2-1-1"
+ },
+ "status": "completed"
+ }
+ },
+ {
+ "type": "step.completed",
+ "meta": {
+ "id": "evt_01M476RVPGVTS0HK0BT3DHYRNC",
+ "at": "2026-10-05T23:35:49.712Z"
+ },
+ "data": {
+ "turnId": "turn_1",
+ "stepIndex": 0,
+ "usage": {
+ "inputTokens": 382,
+ "outputTokens": 5,
+ "cacheReadTokens": 0,
+ "cacheWriteTokens": 0
+ },
+ "finishReason": "tool-calls"
+ }
+ },
+ {
+ "type": "step.started",
+ "meta": {
+ "id": "evt_01M476RVRMAYSTN1F5WH1BDHGM",
+ "at": "2026-10-05T23:35:49.780Z"
+ },
+ "data": {
+ "turnId": "turn_1",
+ "stepIndex": 1
+ }
+ },
+ {
+ "type": "message.appended",
+ "meta": {
+ "id": "evt_01M476RVRRACG4V1029E45SH2D",
+ "at": "2026-10-05T23:35:49.784Z"
+ },
+ "data": {
+ "turnId": "turn_1",
+ "stepIndex": 1
+ }
+ },
+ {
+ "type": "step.completed",
+ "meta": {
+ "id": "evt_01M476RVRRACG4V1029E45SH2H",
+ "at": "2026-10-05T23:35:49.784Z"
+ },
+ "data": {
+ "turnId": "turn_1",
+ "stepIndex": 1,
+ "usage": {
+ "inputTokens": 391,
+ "outputTokens": 20,
+ "cacheReadTokens": 0,
+ "cacheWriteTokens": 0
+ },
+ "finishReason": "stop"
+ }
+ },
+ {
+ "type": "turn.completed",
+ "meta": {
+ "id": "evt_01M476RVS25WY95VK1961T35DH",
+ "at": "2026-10-05T23:35:49.794Z"
+ },
+ "data": {
+ "turnId": "turn_1"
+ }
+ }
+ ]
+ }
+ ]
+ }
+}
diff --git a/scripts/eval-metrics/gaps.mjs b/scripts/eval-metrics/gaps.mjs
new file mode 100644
index 000000000..dd67ec0a1
--- /dev/null
+++ b/scripts/eval-metrics/gaps.mjs
@@ -0,0 +1,70 @@
+// Outcome classification for one eval attempt. A gap is an attempt that says
+// nothing about the model or eve: it counts toward coverage but is excluded
+// from pass rates and from latency and cost medians.
+
+/**
+ * Gap reasons, in classification priority order:
+ *
+ * - `job-missing`: The publish job found no artifact for a planned leg.
+ * - `eval-missing`: A planned eval has no result in its leg's artifacts.
+ * - `provider-unavailable`: The model provider or network failed: HTTP 429/5xx, or a network-level error.
+ * - `stream-write-failed`: eve could not write its durable event stream.
+ * - `timeout-before-first-step`: The eval timed out before any model step completed.
+ * - `error-before-first-step`: The eval errored before any model step completed.
+ */
+/** @typedef {"completed" | "timed_out" | "skipped" | "parked" | "gap"} Outcome */
+
+const FAILURE_EVENTS = new Set(["step.failed", "turn.failed", "session.failed"]);
+const NETWORK_FAILURE =
+ /\b(UND_ERR_(?:BODY|CONNECT|HEADERS)_TIMEOUT|UND_ERR_SOCKET|ECONNRESET|ECONNREFUSED|ETIMEDOUT|EAI_AGAIN|ENOTFOUND)\b/;
+const TIMEOUT_ERROR = /aborted due to timeout|TimeoutError/i;
+
+/**
+ * @param {object} evalArtifact parsed eval detail JSON
+ * @returns {{ outcome: Outcome, gap_reason?: string }}
+ */
+export function classifyOutcome(evalArtifact) {
+ if (evalArtifact?.verdict === "skipped") return { outcome: "skipped" };
+
+ const events = (evalArtifact?.result?.sessions ?? []).flatMap((session) => session.events ?? []);
+ const failures = events.filter((event) => FAILURE_EVENTS.has(event.type));
+ if (failures.some(isProviderFailure)) return gap("provider-unavailable");
+ if (failures.some((event) => event.data?.code === "WORKFLOW_STREAM_WRITE_FAILED"))
+ return gap("stream-write-failed");
+
+ const completedSteps = events.filter((event) => event.type === "step.completed").length;
+ const error = typeof evalArtifact?.error === "string" ? evalArtifact.error : undefined;
+ const timedOut = error !== undefined && TIMEOUT_ERROR.test(error);
+ // An eval that never completed a model step is a gap only when nothing in its
+ // sessions failed for a real reason. Infrastructure failures returned above,
+ // so any failure left (a first step rejected for its request, such as a
+ // context too long, or a failed session) is the eval's own failure, even
+ // though the eval then records an error.
+ const realFailure = failures.length > 0;
+ if (completedSteps === 0 && timedOut && !realFailure) return gap("timeout-before-first-step");
+ if (completedSteps === 0 && error !== undefined && !realFailure)
+ return gap("error-before-first-step");
+ if (timedOut) return { outcome: "timed_out" };
+ // The runner's own definition (eve's derive-run-facts), so the eval report
+ // and the benchmark record agree on what parked means.
+ if (evalArtifact?.result?.derived?.parked === true) return { outcome: "parked" };
+ return { outcome: "completed" };
+}
+
+function gap(reason) {
+ return { outcome: "gap", gap_reason: reason };
+}
+
+/**
+ * Provider-side and network failures are gaps. Other model-call 4xx responses
+ * (context too long, invalid request) and schema failures are real failures.
+ */
+export function isProviderFailure(event) {
+ const { code, details } = event.data ?? {};
+ if (code !== "MODEL_CALL_FAILED") return false;
+ for (const status of [details?.statusCode, details?.upstreamStatusCode]) {
+ if (typeof status === "number" && (status === 429 || status >= 500)) return true;
+ }
+ if (details?.semanticErrorId === "network-request-failed") return true;
+ return NETWORK_FAILURE.test(JSON.stringify(details ?? {}) + (event.data?.message ?? ""));
+}
diff --git a/scripts/eval-metrics/intervals.test.mjs b/scripts/eval-metrics/intervals.test.mjs
new file mode 100644
index 000000000..951732154
--- /dev/null
+++ b/scripts/eval-metrics/intervals.test.mjs
@@ -0,0 +1,100 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+import {
+ coveredLengthBetween,
+ firstDeltaInStep,
+ pairIntervals,
+ unionIntervals,
+} from "./measurements/intervals.mjs";
+
+const event = (type, id, second, data = {}) => ({
+ type,
+ meta: { id, at: `2026-01-01T00:00:${String(second).padStart(2, "0")}Z` },
+ data,
+});
+const actionKeys = {
+ startKeys: (e) => e.data.actions?.map((action) => action.callId),
+ endKeys: (e) => [e.data.result?.callId],
+ subject: "action",
+};
+const requested = (id, second, ...callIds) =>
+ event("actions.requested", id, second, { actions: callIds.map((callId) => ({ callId })) });
+const result = (id, second, callId) => event("action.result", id, second, { result: { callId } });
+
+test("pairs batched starts with out-of-order ends by key", () => {
+ const paired = pairIntervals(
+ [requested("batch", 1, "a", "b")],
+ [result("b-end", 5, "b"), result("a-end", 3, "a")],
+ actionKeys,
+ );
+ assert.equal(paired.status, "ready");
+ assert.deepEqual(
+ paired.intervals.map(({ key, startAt, endAt }) => [key, endAt - startAt]),
+ [
+ ["a", 2000],
+ ["b", 4000],
+ ],
+ );
+});
+
+test("partial, ambiguous, and malformed pairings are unavailable", () => {
+ for (const [starts, ends, reason] of [
+ [[requested("r", 1, "a")], [], "action-incomplete"],
+ [[], [result("e", 2, "a")], "action-start-missing"],
+ [[requested("r", 1, "a"), requested("again", 2, "a")], [], "ambiguous-action"],
+ [[requested("r", 1, "a")], [result("e1", 2, "a"), result("e2", 3, "a")], "ambiguous-action"],
+ [[requested("r", 1)], [], "missing-action-identity"],
+ [[requested("r", 3, "a")], [result("e", 1, "a")], "negative-duration"],
+ [[requested("r", "xx", "a")], [result("e", 1, "a")], "missing-timestamp"],
+ ])
+ assert.deepEqual(pairIntervals(starts, ends, actionKeys), { status: "unavailable", reason });
+});
+
+test("repeated starts can resolve to the earliest announcement", () => {
+ const keys = { ...actionKeys, repeatedStarts: "earliest" };
+ const paired = pairIntervals(
+ [requested("late", 4, "a"), requested("early", 2, "a")],
+ [result("e", 6, "a")],
+ keys,
+ );
+ assert.equal(paired.intervals[0].start.meta.id, "early");
+});
+
+test("unions overlapping intervals and measures coverage within and outside bounds", () => {
+ const spans = [
+ { startAt: 0, endAt: 4 },
+ { startAt: 2, endAt: 6 },
+ { startAt: 6, endAt: 7 },
+ { startAt: 10, endAt: 12 },
+ ];
+ assert.deepEqual(unionIntervals(spans), [
+ { startAt: 0, endAt: 7 },
+ { startAt: 10, endAt: 12 },
+ ]);
+ assert.equal(coveredLengthBetween(spans), 9);
+ assert.equal(coveredLengthBetween(spans, { within: [{ startAt: 3, endAt: 11 }] }), 5);
+ assert.equal(
+ coveredLengthBetween(spans, {
+ within: [{ startAt: 3, endAt: 11 }],
+ excluding: [{ startAt: 5, endAt: 10.5 }],
+ }),
+ 2.5,
+ );
+ assert.equal(coveredLengthBetween([]), 0);
+});
+
+test("finds the first text or reasoning delta of each step, ignoring other steps", () => {
+ const step = (type, id, second, stepIndex, turnId = "t") =>
+ event(type, id, second, { turnId, stepIndex });
+ const steps = firstDeltaInStep([
+ step("step.started", "s0", 1, 0),
+ step("message.appended", "m-late", 4, 0),
+ step("reasoning.appended", "r-early", 2, 0),
+ step("step.started", "s1", 5, 1),
+ step("action.input.appended", "tool-delta", 6, 1),
+ step("message.appended", "other-turn", 3, 0, "other"),
+ ]);
+ assert.equal(steps.get("t/0").firstDelta.meta.id, "r-early");
+ assert.equal(steps.get("t/1").firstDelta, undefined);
+ assert.equal(steps.has("other/0"), false);
+});
diff --git a/scripts/eval-metrics/measurements.test.mjs b/scripts/eval-metrics/measurements.test.mjs
new file mode 100644
index 000000000..5b6b41687
--- /dev/null
+++ b/scripts/eval-metrics/measurements.test.mjs
@@ -0,0 +1,34 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+import { captureSessions } from "./measurements/events.mjs";
+
+const event = (type, id, second, data = {}) => ({
+ type,
+ meta: { id, at: `2026-01-01T00:00:${String(second).padStart(2, "0")}Z` },
+ data,
+});
+
+test("deduplicates repeated captures without losing later events", () => {
+ const start = event("turn.started", "start", 0, { turnId: "turn" });
+ const later = event("turn.completed", "completed", 1, { turnId: "turn" });
+ const capture = captureSessions([
+ { sessionId: "session", events: [start, structuredClone(start)] },
+ { sessionId: "session", events: [structuredClone(start), later] },
+ ]);
+ assert.deepEqual(capture.bySession.get("session"), [start, later]);
+});
+
+test("rejects malformed or ambiguous capture identities", () => {
+ const start = event("turn.started", "start", 0);
+ const session = { sessionId: "session", events: [start] };
+ for (const [sessions, message] of [
+ [[{}], /Unsupported session capture/],
+ [[{ ...session, events: [{ type: "turn.started" }] }], /Unsupported event capture/],
+ [
+ [session, { ...session, events: [{ ...start, data: { changed: true } }] }],
+ /Conflicting event identity/,
+ ],
+ [[session, { ...session, sessionId: "other" }], /Ambiguous session evidence/],
+ ])
+ assert.throws(() => captureSessions(sessions), message);
+});
diff --git a/scripts/eval-metrics/measurements/events.mjs b/scripts/eval-metrics/measurements/events.mjs
new file mode 100644
index 000000000..7ec1be18e
--- /dev/null
+++ b/scripts/eval-metrics/measurements/events.mjs
@@ -0,0 +1,61 @@
+/**
+ * @typedef {{ status: "measured", value: number }
+ * | { status: "unavailable" | "not-applicable", reason: string }} Measurement
+ */
+
+/** @param {string} reason @returns {{ status: "unavailable", reason: string }} */
+export function unavailable(reason) {
+ return { status: "unavailable", reason };
+}
+
+/** @param {string} reason @returns {{ status: "not-applicable", reason: string }} */
+export function notApplicable(reason) {
+ return { status: "not-applicable", reason };
+}
+
+/** Merge repeated session captures and validate event identities before measuring. */
+export function captureSessions(sessions) {
+ if (!Array.isArray(sessions)) return undefined;
+
+ const bySession = new Map();
+ const eventSessions = new Map();
+ const identities = new Map();
+ for (const session of sessions) {
+ if (typeof session?.sessionId !== "string" || !Array.isArray(session.events))
+ throw new Error("Unsupported session capture shape.");
+ const events = bySession.get(session.sessionId) ?? [];
+ for (const event of session.events) {
+ if (!event || typeof event.type !== "string" || typeof event.meta?.id !== "string")
+ throw new Error("Unsupported event capture shape or missing event identity.");
+ const id = event.meta.id;
+ const serialized = JSON.stringify(event);
+ const earlier = identities.get(id);
+ if (earlier !== undefined) {
+ if (earlier !== serialized) throw new Error(`Conflicting event identity: ${id}`);
+ if (eventSessions.get(id) !== session.sessionId)
+ throw new Error(`Ambiguous session evidence for event identity: ${id}`);
+ continue;
+ }
+ identities.set(id, serialized);
+ eventSessions.set(id, session.sessionId);
+ events.push(event);
+ }
+ bySession.set(session.sessionId, events);
+ }
+
+ return { bySession, events: [...bySession.values()].flat() };
+}
+
+/** Match an event type and exact data fields, preserving capture order. */
+export function selectEvents(events, type, data = {}) {
+ return events.filter(
+ (event) =>
+ event.type === type &&
+ Object.entries(data).every(([key, value]) => event.data?.[key] === value),
+ );
+}
+
+export function timestamp(event) {
+ const value = Date.parse(event?.meta?.at ?? "");
+ return Number.isFinite(value) ? value : undefined;
+}
diff --git a/scripts/eval-metrics/measurements/intervals.mjs b/scripts/eval-metrics/measurements/intervals.mjs
new file mode 100644
index 000000000..6ab30842b
--- /dev/null
+++ b/scripts/eval-metrics/measurements/intervals.mjs
@@ -0,0 +1,150 @@
+import { selectEvents, timestamp, unavailable } from "./events.mjs";
+
+/**
+ * @typedef {{ key: string, start: object, end: object, startAt: number, endAt: number }} Interval
+ * @typedef {{ status: "ready", intervals: Interval[] } | { status: "unavailable", reason: string }} Intervals
+ */
+
+/**
+ * Pair start and end events by key, independent of capture order. One event
+ * may open or close several keys (an `actions.requested` batch, an
+ * `input.resolved` batch). Every start needs exactly one end and vice versa;
+ * `repeatedStarts: "earliest"` tolerates re-announced starts such as repeated
+ * approval candidates for one request.
+ *
+ * @param {object[]} starts
+ * @param {object[]} ends
+ * @param {{
+ * startKeys: (event: object) => unknown[],
+ * endKeys: (event: object) => unknown[],
+ * subject: string,
+ * repeatedStarts?: "ambiguous" | "earliest",
+ * closeOpen?: (start: object) => object | undefined,
+ * }} options `closeOpen` may supply an end for a start that has none, such as
+ * the terminal event of a cancelled turn.
+ * @returns {Intervals}
+ */
+export function pairIntervals(
+ starts,
+ ends,
+ { startKeys, endKeys, subject, repeatedStarts, closeOpen },
+) {
+ const opened = indexByKey(starts, startKeys, repeatedStarts === "earliest");
+ if (opened === undefined) return unavailable(`missing-${subject}-identity`);
+ if (opened === "ambiguous") return unavailable(`ambiguous-${subject}`);
+ const closed = indexByKey(ends, endKeys, false);
+ if (closed === undefined) return unavailable(`missing-${subject}-identity`);
+ if (closed === "ambiguous") return unavailable(`ambiguous-${subject}`);
+
+ const intervals = [];
+ for (const [key, start] of opened) {
+ const end = closed.get(key) ?? closeOpen?.(start);
+ if (end === undefined) return unavailable(`${subject}-incomplete`);
+ const startAt = timestamp(start);
+ const endAt = timestamp(end);
+ if (startAt === undefined || endAt === undefined) return unavailable("missing-timestamp");
+ if (endAt < startAt) return unavailable("negative-duration");
+ intervals.push({ key, start, end, startAt, endAt });
+ }
+ if ([...closed.keys()].some((key) => !opened.has(key)))
+ return unavailable(`${subject}-start-missing`);
+ return { status: "ready", intervals };
+}
+
+function indexByKey(events, keysOf, keepEarliest) {
+ const byKey = new Map();
+ for (const event of events) {
+ const keys = keysOf(event);
+ if (!Array.isArray(keys) || keys.length === 0) return undefined;
+ for (const key of keys) {
+ if (typeof key !== "string" || key.length === 0) return undefined;
+ const earlier = byKey.get(key);
+ if (earlier === undefined) byKey.set(key, event);
+ else if (!keepEarliest) return "ambiguous";
+ else if ((timestamp(event) ?? Infinity) < (timestamp(earlier) ?? Infinity))
+ byKey.set(key, event);
+ }
+ }
+ return byKey;
+}
+
+/**
+ * Merge overlapping or touching intervals into a sorted disjoint list.
+ *
+ * @param {readonly { startAt: number, endAt: number }[]} intervals
+ * @returns {{ startAt: number, endAt: number }[]}
+ */
+export function unionIntervals(intervals) {
+ const sorted = [...intervals].sort((a, b) => a.startAt - b.startAt);
+ const merged = [];
+ for (const { startAt, endAt } of sorted) {
+ const last = merged.at(-1);
+ if (last !== undefined && startAt <= last.endAt) last.endAt = Math.max(last.endAt, endAt);
+ else merged.push({ startAt, endAt });
+ }
+ return merged;
+}
+
+/**
+ * Covered length of `intervals` that lies inside `within` and outside `excluding`.
+ *
+ * @param {readonly { startAt: number, endAt: number }[]} intervals
+ * @param {{ within?: readonly { startAt: number, endAt: number }[], excluding?: readonly { startAt: number, endAt: number }[] }} [bounds]
+ */
+export function coveredLengthBetween(intervals, { within, excluding = [] } = {}) {
+ let covered = unionIntervals(intervals);
+ if (within !== undefined) covered = intersect(covered, unionIntervals(within));
+ const removed = intersect(covered, unionIntervals(excluding));
+ return lengthOf(covered) - lengthOf(removed);
+}
+
+function intersect(left, right) {
+ const result = [];
+ let i = 0;
+ let j = 0;
+ while (i < left.length && j < right.length) {
+ const startAt = Math.max(left[i].startAt, right[j].startAt);
+ const endAt = Math.min(left[i].endAt, right[j].endAt);
+ if (startAt < endAt) result.push({ startAt, endAt });
+ if (left[i].endAt < right[j].endAt) i += 1;
+ else j += 1;
+ }
+ return result;
+}
+
+function lengthOf(disjoint) {
+ return disjoint.reduce((total, { startAt, endAt }) => total + endAt - startAt, 0);
+}
+
+/**
+ * For each started model step, find the first streamed text or reasoning
+ * delta of that step. Steps that stream no delta (e.g. tool-call only) map to
+ * `undefined`; callers decide whether that is not-applicable.
+ *
+ * @param {object[]} events one session's events
+ * @returns {Map<string, { start: object, firstDelta: object | undefined }>} keyed by `turnId/stepIndex`
+ */
+export function firstDeltaInStep(events) {
+ const steps = new Map();
+ for (const start of selectEvents(events, "step.started")) {
+ const key = stepKey(start);
+ if (key !== undefined) steps.set(key, { start, firstDelta: undefined });
+ }
+ for (const event of events) {
+ if (event.type !== "message.appended" && event.type !== "reasoning.appended") continue;
+ const step = steps.get(stepKey(event));
+ if (step === undefined) continue;
+ const at = timestamp(event);
+ if (at === undefined) continue;
+ if (step.firstDelta === undefined || at < timestamp(step.firstDelta)) step.firstDelta = event;
+ }
+ return steps;
+}
+
+/** Identity of one model step within a session. */
+export function stepKey(event) {
+ const { turnId, stepIndex } = event.data ?? {};
+ return typeof turnId === "string" && Number.isInteger(stepIndex)
+ ? `${turnId}/${stepIndex}`
+ : undefined;
+}
diff --git a/scripts/eval-metrics/standard.mjs b/scripts/eval-metrics/standard.mjs
new file mode 100644
index 000000000..f8c93e20d
--- /dev/null
+++ b/scripts/eval-metrics/standard.mjs
@@ -0,0 +1,578 @@
+// Standard per-attempt metric set derived from one eval detail artifact
+// (`.eve/evals/<ts>/evals/<id>.json`). STANDARD_METRICS below lists every
+// metric with its unit and better direction.
+//
+// Scope: latency, attribution, step-level cost, and efficiency counts read the
+// eval's root sessions (sessions the eval opened, not delegated children).
+// Reliability counts read every captured session, because a failure anywhere
+// matters.
+//
+// Cost and tokens: `cost.total_usd` and `tokens.*` (but `tokens.peak_context`)
+// are the runner's `result.derived.usage`, the same totals the eval reporters
+// see: each captured session's latest usage, including delegated work and
+// compaction summaries, less sessions another captured session opened.
+// `cost.primary_usd` sums the root sessions' model steps; `cost.subagent_usd`
+// is the difference, and is unavailable when a root session compacted, since
+// compaction cost is in the total but in no step.
+//
+// Attribution partitions root turn time with a fixed precedence, so the parts
+// add up to the turn total: human wait, then task (delegated work, including
+// time a `task_wait` call blocks on it), tool (any other requested action),
+// compaction, model step, and the remainder is framework time. In-process
+// tools run inside their model step, so precedence keeps model ms to model work.
+import { existsSync, readdirSync, readFileSync } from "node:fs";
+import { join } from "node:path";
+import { fileURLToPath } from "node:url";
+
+import {
+ captureSessions,
+ notApplicable,
+ selectEvents,
+ timestamp,
+ unavailable,
+} from "./measurements/events.mjs";
+import {
+ coveredLengthBetween,
+ firstDeltaInStep,
+ pairIntervals,
+ stepKey,
+} from "./measurements/intervals.mjs";
+
+/** Bump whenever a definition below changes meaning. */
+export const METRICS_VERSION = 1;
+
+/** Every metric in the standard set, with its unit and better direction. */
+export const STANDARD_METRICS = Object.freeze({
+ "latency.eval_wall_ms": { unit: "ms", direction: "lower" },
+ "latency.turn_ms": { unit: "ms", direction: "lower" },
+ "latency.turn_ms_mean": { unit: "ms", direction: "lower" },
+ "latency.ttft_ms_p50": { unit: "ms", direction: "lower" },
+ "latency.output_tokens_per_s": { unit: "tokens/s", direction: "higher" },
+ "attribution.model_ms": { unit: "ms", direction: "lower" },
+ "attribution.tool_ms": { unit: "ms", direction: "lower" },
+ "attribution.task_ms": { unit: "ms", direction: "lower" },
+ "attribution.human_wait_ms": { unit: "ms", direction: "neutral" },
+ "attribution.compaction_ms": { unit: "ms", direction: "lower" },
+ "attribution.framework_ms": { unit: "ms", direction: "lower" },
+ "cost.total_usd": { unit: "usd", direction: "lower" },
+ "cost.primary_usd": { unit: "usd", direction: "lower" },
+ "cost.subagent_usd": { unit: "usd", direction: "lower" },
+ "tokens.input": { unit: "tokens", direction: "lower" },
+ "tokens.output": { unit: "tokens", direction: "lower" },
+ "tokens.cache_read": { unit: "tokens", direction: "neutral" },
+ "tokens.cache_write": { unit: "tokens", direction: "neutral" },
+ "tokens.cache_hit_ratio": { unit: "ratio", direction: "higher" },
+ "tokens.peak_context": { unit: "tokens", direction: "lower" },
+ "counts.turns": { unit: "count", direction: "neutral" },
+ "counts.steps": { unit: "count", direction: "lower" },
+ "counts.steps_per_turn": { unit: "ratio", direction: "lower" },
+ "counts.tool_calls": { unit: "count", direction: "neutral" },
+ "counts.tool_calls_per_step": { unit: "ratio", direction: "neutral" },
+ "counts.subagent_fan_out": { unit: "count", direction: "neutral" },
+ "counts.subagent_depth": { unit: "count", direction: "neutral" },
+ "counts.compactions": { unit: "count", direction: "lower" },
+ "counts.context_clears": { unit: "count", direction: "neutral" },
+ "reliability.step_failures": { unit: "count", direction: "lower" },
+ "reliability.turn_failures": { unit: "count", direction: "lower" },
+ "reliability.session_failures": { unit: "count", direction: "lower" },
+ "reliability.tool_errors": { unit: "count", direction: "lower" },
+ "reliability.tool_error_rate": { unit: "ratio", direction: "lower" },
+ "reliability.truncations": { unit: "count", direction: "lower" },
+});
+
+const TURN_TERMINALS = ["turn.completed", "turn.failed", "turn.cancelled"];
+const FAILURE_EVENTS = ["step.failed", "turn.failed", "session.failed"];
+const TOOL_ACTION_KINDS = new Set(["tool-call", "workflow-tool-call"]);
+
+/**
+ * @param {object} evalArtifact parsed eval detail JSON
+ * @returns {{
+ * metrics_version: number,
+ * metrics: Record<string, import("./measurements/events.mjs").Measurement>,
+ * failures: { type: string, code: string, count: number }[],
+ * }}
+ */
+export function deriveAttemptMetrics(evalArtifact) {
+ const sessions = evalArtifact?.result?.sessions;
+ let capture;
+ try {
+ capture = captureSessions(
+ Array.isArray(sessions)
+ ? sessions.filter((session) => typeof session?.sessionId === "string")
+ : undefined,
+ );
+ } catch {
+ return everyMetric(unavailable("invalid-session-capture"));
+ }
+ if (capture === undefined) return everyMetric(unavailable("missing-session-capture"));
+ if (capture.bySession.size === 0) return everyMetric(notApplicable("no-sessions"));
+
+ const rootIds = rootSessionIds(capture);
+ if (rootIds.length === 0) return everyMetric(unavailable("missing-root-session"));
+ const roots = rootIds.map((sessionId) => ({
+ events: capture.bySession.get(sessionId),
+ timeline: sessionTimeline(capture, sessionId),
+ }));
+ const allEvents = capture.events;
+
+ const metrics = {
+ ...latencyAndAttribution(evalArtifact, roots),
+ ...streaming(roots),
+ ...costAndTokens(evalArtifact, roots),
+ ...efficiency(capture, roots, rootIds),
+ ...reliability(allEvents),
+ };
+ return { metrics_version: METRICS_VERSION, metrics, failures: failureCodes(allEvents) };
+}
+
+function everyMetric(measurement) {
+ return {
+ metrics_version: METRICS_VERSION,
+ metrics: Object.fromEntries(Object.keys(STANDARD_METRICS).map((key) => [key, measurement])),
+ failures: [],
+ };
+}
+
+function measured(value) {
+ return { status: "measured", value };
+}
+
+/** Sessions the eval opened itself: neither linked by `agent.started` nor invoked as a subagent. */
+function rootSessionIds(capture) {
+ const children = new Set(
+ selectEvents(capture.events, "agent.started").map((event) => event.data?.sessionId),
+ );
+ for (const [sessionId, events] of capture.bySession) {
+ if (
+ selectEvents(events, "session.started").some((e) => e.data?.invocation?.kind === "subagent")
+ )
+ children.add(sessionId);
+ }
+ return [...capture.bySession.keys()].filter((sessionId) => !children.has(sessionId));
+}
+
+/** Pair every interval family of one session, clipping nothing yet. */
+function sessionTimeline(capture, sessionId) {
+ const events = capture.bySession.get(sessionId);
+ const turnStarts = selectEvents(events, "turn.started");
+ const turnKey = (e) => [e.data?.turnId];
+ // A root session with no turns has no turn time to attribute; without this,
+ // every latency and attribution part would read as a measured zero.
+ const turns =
+ turnStarts.length === 0
+ ? unavailable("missing-turn")
+ : pairIntervals(
+ turnStarts,
+ TURN_TERMINALS.flatMap((type) => selectEvents(events, type)),
+ { startKeys: turnKey, endKeys: turnKey, subject: "turn" },
+ );
+ const terminals = new Map(
+ turns.status === "ready" ? turns.intervals.map((turn) => [turn.key, turn.end]) : [],
+ );
+ // Work a turn abandons (a cancelled turn, a step cut off by a session
+ // limit) ends with that turn.
+ const closeOpen = (start) => terminals.get(start.data?.turnId);
+
+ const steps = pairIntervals(
+ selectEvents(events, "step.started"),
+ [...selectEvents(events, "step.completed"), ...selectEvents(events, "step.failed")],
+ { startKeys: (e) => [stepKey(e)], endKeys: (e) => [stepKey(e)], subject: "step", closeOpen },
+ );
+ const actions = pairIntervals(
+ selectEvents(events, "actions.requested"),
+ selectEvents(events, "action.result"),
+ {
+ startKeys: (e) =>
+ Array.isArray(e.data?.actions) ? e.data.actions.map((a) => a?.callId) : [],
+ endKeys: (e) => [e.data?.result?.callId],
+ subject: "action",
+ repeatedStarts: "earliest",
+ closeOpen,
+ },
+ );
+ const taskKey = (e) => [taskIdentity(e)];
+ const tasks = pairIntervals(
+ selectEvents(events, "task.started"),
+ selectEvents(events, "task.settled"),
+ { startKeys: taskKey, endKeys: taskKey, subject: "task", closeOpen },
+ );
+ const compactions = pairIntervals(
+ selectEvents(events, "compaction.requested"),
+ selectEvents(events, "compaction.completed"),
+ {
+ startKeys: (e) => [stepKey(e)],
+ endKeys: (e) => [stepKey(e)],
+ subject: "compaction",
+ closeOpen,
+ },
+ );
+ return {
+ events,
+ turns,
+ steps,
+ actions,
+ tasks,
+ compactions,
+ human: humanWait(events, closeOpen),
+ };
+}
+
+function taskIdentity(event) {
+ const { taskId, callId } = event.data ?? {};
+ return typeof taskId === "string" && typeof callId === "string"
+ ? `${taskId}/${callId}`
+ : undefined;
+}
+
+/** Input requests, responder-bound approvals, and sign-ins, from request to resolution. */
+function humanWait(events, closeOpen) {
+ const requestIds = (list) => (Array.isArray(list) ? list.map((entry) => entry?.requestId) : []);
+ const settledResolutions = (event) =>
+ (event.data?.resolutions ?? []).filter((resolution) => resolution?.outcome !== "invalid");
+ const input = pairIntervals(
+ selectEvents(events, "input.requested"),
+ selectEvents(events, "input.resolved").filter((e) => settledResolutions(e).length > 0),
+ {
+ startKeys: (e) => requestIds(e.data?.requests),
+ endKeys: (e) => requestIds(settledResolutions(e)),
+ subject: "input",
+ closeOpen,
+ },
+ );
+
+ // `approval.settled` also closes approvals delivered only through
+ // `input.requested`; only responder-bound candidates open an interval here.
+ const candidates = selectEvents(events, "approval.candidate", { outcome: "pending" });
+ const candidateIds = new Set(candidates.map((e) => e.data?.requestId));
+ const approval = pairIntervals(
+ candidates,
+ selectEvents(events, "approval.settled").filter((e) => candidateIds.has(e.data?.requestId)),
+ {
+ startKeys: (e) => [e.data?.requestId],
+ endKeys: (e) => [e.data?.requestId],
+ subject: "approval",
+ repeatedStarts: "earliest",
+ closeOpen,
+ },
+ );
+
+ // `attemptId` is not present on every `authorization.required`, so key by
+ // the turn and connection name, which both events always carry.
+ const authorizationKey = (e) =>
+ typeof e.data?.turnId === "string" && typeof e.data?.name === "string"
+ ? [`${e.data.turnId}/${e.data.name}`]
+ : [];
+ const authorization = pairIntervals(
+ selectEvents(events, "authorization.required"),
+ selectEvents(events, "authorization.completed"),
+ {
+ startKeys: authorizationKey,
+ endKeys: authorizationKey,
+ subject: "authorization",
+ repeatedStarts: "earliest",
+ closeOpen,
+ },
+ );
+
+ for (const family of [input, approval, authorization])
+ if (family.status !== "ready") return family;
+ return {
+ status: "ready",
+ intervals: [...input.intervals, ...approval.intervals, ...authorization.intervals],
+ };
+}
+
+function latencyAndAttribution(evalArtifact, roots) {
+ const parts = {
+ turn: [],
+ human: [],
+ tool: [],
+ task: [],
+ compaction: [],
+ model: [],
+ framework: [],
+ };
+ let failure;
+ let turnCount = 0;
+ const humanIntervals = [];
+ for (const { timeline } of roots) {
+ const families = ["turns", "human", "actions", "tasks", "compactions", "steps"];
+ const broken = families.map((name) => timeline[name]).find((f) => f.status !== "ready");
+ if (broken !== undefined) {
+ failure ??= broken;
+ continue;
+ }
+ const T = timeline.turns.intervals;
+ const H = timeline.human.intervals;
+ const A = timeline.actions.intervals;
+ const K = timeline.tasks.intervals;
+ const C = timeline.compactions.intervals;
+ const M = timeline.steps.intervals;
+ const total = coveredLengthBetween(T);
+ const human = coveredLengthBetween(H, { within: T });
+ parts.turn.push(total - human);
+ parts.human.push(human);
+ parts.task.push(coveredLengthBetween(K, { within: T, excluding: H }));
+ parts.tool.push(coveredLengthBetween(A, { within: T, excluding: [...H, ...K] }));
+ parts.compaction.push(coveredLengthBetween(C, { within: T, excluding: [...H, ...A, ...K] }));
+ parts.model.push(coveredLengthBetween(M, { within: T, excluding: [...H, ...A, ...K, ...C] }));
+ parts.framework.push(
+ total - coveredLengthBetween([...H, ...A, ...K, ...C, ...M], { within: T }),
+ );
+ turnCount += T.length;
+ humanIntervals.push(...H);
+ }
+
+ const sum = (values) => (failure ? failure : measured(values.reduce((a, b) => a + b, 0)));
+ return {
+ "latency.eval_wall_ms": failure ?? evalWall(evalArtifact, humanIntervals),
+ "latency.turn_ms": sum(parts.turn),
+ "latency.turn_ms_mean":
+ failure ??
+ (turnCount === 0
+ ? notApplicable("no-turns")
+ : measured(parts.turn.reduce((a, b) => a + b, 0) / turnCount)),
+ "attribution.model_ms": sum(parts.model),
+ "attribution.tool_ms": sum(parts.tool),
+ "attribution.task_ms": sum(parts.task),
+ "attribution.human_wait_ms": sum(parts.human),
+ "attribution.compaction_ms": sum(parts.compaction),
+ "attribution.framework_ms": sum(parts.framework),
+ };
+}
+
+function evalWall(evalArtifact, humanIntervals) {
+ const startAt = Date.parse(evalArtifact?.startedAt ?? "");
+ const endAt = Date.parse(evalArtifact?.completedAt ?? "");
+ if (!Number.isFinite(startAt) || !Number.isFinite(endAt))
+ return unavailable("missing-eval-timing");
+ if (endAt < startAt) return unavailable("negative-duration");
+ const human = coveredLengthBetween(humanIntervals, { within: [{ startAt, endAt }] });
+ return measured(endAt - startAt - human);
+}
+
+function streaming(roots) {
+ const ttfts = [];
+ let tokens = 0;
+ let streamMs = 0;
+ let missingTokens = false;
+ for (const { events } of roots) {
+ const completions = new Map(
+ selectEvents(events, "step.completed").map((event) => [stepKey(event), event]),
+ );
+ for (const [key, { start, firstDelta }] of firstDeltaInStep(events)) {
+ const startAt = timestamp(start);
+ if (firstDelta === undefined || startAt === undefined) continue;
+ const deltaAt = timestamp(firstDelta);
+ ttfts.push(deltaAt - startAt);
+ const completed = completions.get(key);
+ if (completed === undefined) continue;
+ const outputTokens = completed.data?.usage?.outputTokens;
+ if (typeof outputTokens !== "number") missingTokens = true;
+ else tokens += outputTokens;
+ streamMs += (timestamp(completed) ?? deltaAt) - deltaAt;
+ }
+ }
+ if (ttfts.length === 0) {
+ const none = notApplicable("no-streamed-steps");
+ return { "latency.ttft_ms_p50": none, "latency.output_tokens_per_s": none };
+ }
+ return {
+ "latency.ttft_ms_p50": measured(median(ttfts)),
+ "latency.output_tokens_per_s": missingTokens
+ ? unavailable("missing-output-tokens")
+ : streamMs <= 0
+ ? unavailable("zero-stream-duration")
+ : measured((tokens / streamMs) * 1000),
+ };
+}
+
+function median(values) {
+ const sorted = [...values].sort((a, b) => a - b);
+ const middle = Math.floor(sorted.length / 2);
+ return sorted.length % 2 === 1 ? sorted[middle] : (sorted[middle - 1] + sorted[middle]) / 2;
+}
+
+function costAndTokens(evalArtifact, roots) {
+ const steps = roots.flatMap(({ events }) => selectEvents(events, "step.completed"));
+ // Absent when a counted session reported no usage; `costUsd` is absent when
+ // one reported no cost (eve's evalUsage).
+ const usage = evalArtifact?.result?.derived?.usage;
+
+ let primary;
+ if (steps.length === 0) primary = notApplicable("no-model-steps");
+ else if (steps.some((step) => !isAmount(step.data?.usage?.costUsd)))
+ primary = unavailable("missing-step-cost");
+ else primary = measured(steps.reduce((total, step) => total + step.data.usage.costUsd, 0));
+
+ const rolled = (field, reason) =>
+ isAmount(usage?.[field]) ? measured(usage[field]) : unavailable(reason);
+ const total = rolled("costUsd", "missing-rolled-up-cost");
+
+ // Compaction summary calls add their cost to the session's rolled-up usage,
+ // but no step event reports it, so total − steps would count it as
+ // subagent spend. Until compaction events carry usage, the split is unknown.
+ const compacted = roots.some(
+ ({ events }) => selectEvents(events, "compaction.completed").length > 0,
+ );
+ let subagent;
+ if (total.status !== "measured") subagent = total;
+ else if (compacted) subagent = unavailable("compaction-cost-unattributed");
+ else if (primary.status === "not-applicable") subagent = total;
+ else if (primary.status !== "measured") subagent = primary;
+ else {
+ const difference = total.value - primary.value;
+ // Float sums of identical costs can differ in the last bits.
+ subagent =
+ difference < -1e-9
+ ? unavailable("inconsistent-cost-rollup")
+ : measured(Math.max(0, difference));
+ }
+
+ const input = rolled("inputTokens", "missing-rolled-up-usage");
+ const cacheRead = rolled("cacheReadTokens", "missing-rolled-up-usage");
+ let cacheHitRatio;
+ if (input.status !== "measured") cacheHitRatio = input;
+ else if (cacheRead.status !== "measured") cacheHitRatio = cacheRead;
+ else if (input.value === 0) cacheHitRatio = notApplicable("no-input-tokens");
+ else cacheHitRatio = measured(cacheRead.value / input.value);
+
+ let peak;
+ if (steps.length === 0) peak = notApplicable("no-model-steps");
+ else if (steps.some((step) => !isAmount(step.data?.usage?.inputTokens)))
+ peak = unavailable("missing-step-input-tokens");
+ else peak = measured(Math.max(...steps.map((step) => step.data.usage.inputTokens)));
+
+ return {
+ "cost.total_usd": total,
+ "cost.primary_usd": primary,
+ "cost.subagent_usd": subagent,
+ "tokens.input": input,
+ "tokens.output": rolled("outputTokens", "missing-rolled-up-usage"),
+ "tokens.cache_read": cacheRead,
+ "tokens.cache_write": rolled("cacheWriteTokens", "missing-rolled-up-usage"),
+ "tokens.cache_hit_ratio": cacheHitRatio,
+ "tokens.peak_context": peak,
+ };
+}
+
+function isAmount(value) {
+ return typeof value === "number" && Number.isFinite(value) && value >= 0;
+}
+
+function efficiency(capture, roots, rootIds) {
+ const rootEvents = roots.flatMap(({ events }) => events);
+ const turns = selectEvents(rootEvents, "turn.started").length;
+ const steps = selectEvents(rootEvents, "step.started").length;
+ const toolCalls = new Set();
+ for (const [index, { events }] of roots.entries()) {
+ for (const event of selectEvents(events, "actions.requested")) {
+ for (const action of event.data?.actions ?? []) {
+ if (TOOL_ACTION_KINDS.has(action?.kind))
+ toolCalls.add(`${rootIds[index]}\0${action.callId}`);
+ }
+ }
+ }
+ const ratio = (numerator, denominator, reason) =>
+ denominator === 0 ? notApplicable(reason) : measured(numerator / denominator);
+ const delegated = new Set(
+ selectEvents(rootEvents, "agent.started").map((event) => event.data?.sessionId),
+ );
+ return {
+ "counts.turns": measured(turns),
+ "counts.steps": measured(steps),
+ "counts.steps_per_turn": ratio(steps, turns, "no-turns"),
+ "counts.tool_calls": measured(toolCalls.size),
+ "counts.tool_calls_per_step": ratio(toolCalls.size, steps, "no-model-steps"),
+ "counts.subagent_fan_out": measured(delegated.size),
+ "counts.subagent_depth": measured(
+ Math.max(...rootIds.map((id) => delegationDepth(capture, id))),
+ ),
+ "counts.compactions": measured(selectEvents(rootEvents, "compaction.completed").length),
+ "counts.context_clears": measured(selectEvents(rootEvents, "context.cleared").length),
+ };
+}
+
+/** Longest `agent.started` chain below a session; uncaptured children count as leaves. */
+function delegationDepth(capture, sessionId, visiting = new Set()) {
+ if (visiting.has(sessionId)) return 0;
+ const events = capture.bySession.get(sessionId) ?? [];
+ const children = new Set(selectEvents(events, "agent.started").map((e) => e.data?.sessionId));
+ let depth = 0;
+ visiting.add(sessionId);
+ for (const child of children)
+ depth = Math.max(depth, 1 + delegationDepth(capture, child, visiting));
+ visiting.delete(sessionId);
+ return depth;
+}
+
+function reliability(events) {
+ const results = selectEvents(events, "action.result");
+ const toolErrors = results.filter((event) => event.data?.status === "failed").length;
+ return {
+ "reliability.step_failures": measured(selectEvents(events, "step.failed").length),
+ "reliability.turn_failures": measured(selectEvents(events, "turn.failed").length),
+ "reliability.session_failures": measured(selectEvents(events, "session.failed").length),
+ "reliability.tool_errors": measured(toolErrors),
+ "reliability.tool_error_rate":
+ results.length === 0
+ ? notApplicable("no-action-results")
+ : measured(toolErrors / results.length),
+ "reliability.truncations": measured(
+ selectEvents(events, "step.completed", { finishReason: "length" }).length,
+ ),
+ };
+}
+
+function failureCodes(events) {
+ const counts = new Map();
+ for (const event of events) {
+ if (!FAILURE_EVENTS.includes(event.type)) continue;
+ const key = `${event.type}\0${typeof event.data?.code === "string" ? event.data.code : "UNKNOWN"}`;
+ counts.set(key, (counts.get(key) ?? 0) + 1);
+ }
+ return [...counts]
+ .map(([key, count]) => {
+ const [type, code] = key.split("\0");
+ return { type, code, count };
+ })
+ .sort((a, b) => a.type.localeCompare(b.type) || a.code.localeCompare(b.code));
+}
+
+/**
+ * Every eval detail artifact under one `.eve/evals/<timestamp>` tree, sorted.
+ * The record builder and this file's CLI share it, so they agree on which
+ * artifacts exist.
+ */
+export function listEvalArtifacts(runDirectory) {
+ const evalsDirectory = join(runDirectory, "evals");
+ if (!existsSync(evalsDirectory)) return [];
+ return readdirSync(evalsDirectory, { withFileTypes: true, recursive: true })
+ .filter((entry) => entry.isFile() && entry.name.endsWith(".json"))
+ .map((entry) => join(entry.parentPath, entry.name))
+ .sort();
+}
+
+// `node scripts/eval-metrics/standard.mjs <.eve/evals/<timestamp>>` prints each
+// eval's outcome and metrics, then per-metric coverage across the run.
+if (process.argv[1] === fileURLToPath(import.meta.url)) {
+ const runDirectory = process.argv[2];
+ if (runDirectory === undefined) {
+ console.error("Usage: node scripts/eval-metrics/standard.mjs <.eve/evals/<timestamp>>");
+ process.exit(1);
+ }
+ const { classifyOutcome } = await import("./gaps.mjs");
+ const coverage = new Map(Object.keys(STANDARD_METRICS).map((key) => [key, 0]));
+ const artifacts = listEvalArtifacts(runDirectory);
+ for (const path of artifacts) {
+ const artifact = JSON.parse(readFileSync(path, "utf8"));
+ const { metrics } = deriveAttemptMetrics(artifact);
+ console.log(`${artifact.id}: ${JSON.stringify(classifyOutcome(artifact))}`);
+ for (const [key, m] of Object.entries(metrics)) {
+ if (m.status === "measured") coverage.set(key, coverage.get(key) + 1);
+ console.log(` ${key} ${m.status === "measured" ? m.value : `${m.status} (${m.reason})`}`);
+ }
+ }
+ console.log(`\nCoverage over ${artifacts.length} evals:`);
+ for (const [key, count] of coverage) console.log(` ${key} ${count}/${artifacts.length}`);
+}
diff --git a/scripts/eval-metrics/standard.test.mjs b/scripts/eval-metrics/standard.test.mjs
new file mode 100644
index 000000000..d8e298993
--- /dev/null
+++ b/scripts/eval-metrics/standard.test.mjs
@@ -0,0 +1,390 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+import { readFileSync } from "node:fs";
+import { classifyOutcome } from "./gaps.mjs";
+import { METRICS_VERSION, STANDARD_METRICS, deriveAttemptMetrics } from "./standard.mjs";
+
+// Fixtures are trimmed `eve eval` detail artifacts captured from mock-model
+// runs of e2e/fixtures/agent-tools and agent-subagents (mock models report
+// no cost, so cost cases inject it below).
+const fixture = (name) =>
+ JSON.parse(readFileSync(new URL(`./fixtures/${name}.json`, import.meta.url), "utf8"));
+const values = (metrics) =>
+ Object.fromEntries(
+ Object.entries(metrics).map(([key, m]) => [key, m.status === "measured" ? m.value : m.status]),
+ );
+const at = (ms) => new Date(Date.UTC(2026, 0, 1) + ms).toISOString();
+const event = (type, ms, data = {}) => ({
+ type,
+ meta: { id: `${type}@${ms}:${JSON.stringify(data)}`, at: at(ms) },
+ data,
+});
+
+const synthetic = (events, extra = {}) => ({
+ id: "synthetic",
+ verdict: "passed",
+ startedAt: at(0),
+ completedAt: at(1000),
+ result: {
+ sessionId: "s",
+ status: "completed",
+ sessions: [{ sessionId: "s", primary: true, events }],
+ },
+ ...extra,
+});
+const turn = (start, end, inner, terminal = "turn.completed") => [
+ event("turn.started", start, { turnId: "t" }),
+ ...inner,
+ event(terminal, end, { turnId: "t" }),
+];
+const compactedArtifact = () => {
+ const artifact = synthetic([
+ ...turn(0, 100, [
+ event("step.started", 10, { turnId: "t", stepIndex: 0 }),
+ event("step.completed", 90, {
+ turnId: "t",
+ stepIndex: 0,
+ usage: { inputTokens: 10, outputTokens: 2 },
+ }),
+ ]),
+ event("compaction.requested", 150, { turnId: "t", stepIndex: 1 }),
+ event("compaction.completed", 200, { turnId: "t", stepIndex: 1 }),
+ ]);
+ artifact.result.derived = { usage: { inputTokens: 10, outputTokens: 2 } };
+ return artifact;
+};
+
+function assertPartition(metrics) {
+ const v = values(metrics);
+ const parts =
+ v["attribution.model_ms"] +
+ v["attribution.tool_ms"] +
+ v["attribution.task_ms"] +
+ v["attribution.compaction_ms"] +
+ v["attribution.framework_ms"];
+ assert.equal(parts, v["latency.turn_ms"], "attribution must add up to non-human turn time");
+}
+
+test("derives representative metrics from a captured tool eval", () => {
+ const { metrics_version, metrics, failures } = deriveAttemptMetrics(
+ fixture("agent-tools-ends-turn-function"),
+ );
+ assert.equal(metrics_version, METRICS_VERSION);
+ assert.deepEqual(Object.keys(metrics).sort(), Object.keys(STANDARD_METRICS).sort());
+ assert.deepEqual(failures, []);
+ const v = values(metrics);
+ assert.equal(v["latency.eval_wall_ms"], 1050);
+ assert.equal(v["latency.turn_ms"], 116);
+ assert.equal(v["latency.ttft_ms_p50"], 4);
+ assert.equal(v["attribution.model_ms"], 17);
+ assert.equal(v["attribution.tool_ms"], 3);
+ assert.equal(v["attribution.framework_ms"], 96);
+ assert.equal(v["tokens.input"], 1117);
+ assert.equal(v["tokens.peak_context"], 391);
+ assert.equal(v["counts.turns"], 2);
+ assert.equal(v["counts.steps"], 3);
+ assert.equal(v["counts.tool_calls"], 2);
+ // Mock models report no cost: missing cost is unavailable, never zero.
+ assert.equal(metrics["cost.primary_usd"].reason, "missing-step-cost");
+ assert.equal(metrics["latency.output_tokens_per_s"].reason, "zero-stream-duration");
+ assertPartition(metrics);
+});
+
+test("attributes delegated work to tasks across repeated captures of one session", () => {
+ const { metrics } = deriveAttemptMetrics(fixture("agent-subagents-continuation-local"));
+ const v = values(metrics);
+ assert.equal(v["attribution.task_ms"], 2465);
+ assert.equal(v["attribution.tool_ms"], 133);
+ assert.equal(v["counts.turns"], 3, "child sessions are not root turns");
+ assert.equal(v["counts.subagent_fan_out"], 1);
+ assert.equal(v["counts.subagent_depth"], 1);
+ // Rolled-up usage includes the child's spend.
+ assert.equal(v["tokens.input"], 6006);
+ assertPartition(metrics);
+});
+
+test("excludes sign-in wait from latency and counts it as human wait", () => {
+ const artifact = synthetic(
+ turn(0, 1000, [
+ event("authorization.required", 100, { turnId: "t", name: "account" }),
+ event("authorization.completed", 274, { turnId: "t", name: "account" }),
+ ]),
+ );
+ artifact.completedAt = at(1072);
+ const { metrics } = deriveAttemptMetrics(artifact);
+ const v = values(metrics);
+ assert.equal(v["attribution.human_wait_ms"], 174);
+ assert.equal(v["latency.eval_wall_ms"], 1072 - 174);
+ assertPartition(metrics);
+});
+
+test("compaction outside a turn does not count toward turn time", () => {
+ const { metrics } = deriveAttemptMetrics(compactedArtifact());
+ assert.equal(values(metrics)["counts.compactions"], 1);
+ assert.equal(values(metrics)["attribution.compaction_ms"], 0);
+ assertPartition(metrics);
+});
+
+function withCosts(artifact, { stepCost, rolledUpCost }) {
+ const copy = structuredClone(artifact);
+ // The runner's eval total (result.derived.usage) carries the rolled-up cost.
+ copy.result.derived.usage.costUsd = rolledUpCost;
+ for (const session of copy.result.sessions) {
+ for (const e of session.events) {
+ if (session.sessionId !== copy.result.sessionId) continue;
+ if (e.type === "step.completed") e.data.usage.costUsd = stepCost;
+ if (["session.waiting", "turn.waiting"].includes(e.type)) e.data.usage.costUsd = rolledUpCost;
+ }
+ }
+ return copy;
+}
+
+test("splits cost into primary steps and rolled-up subagent spend", () => {
+ const artifact = withCosts(fixture("agent-subagents-continuation-local"), {
+ stepCost: 0.25,
+ rolledUpCost: 4,
+ });
+ const v = values(deriveAttemptMetrics(artifact).metrics);
+ assert.equal(v["cost.primary_usd"], 2.5, "ten primary steps");
+ assert.equal(v["cost.total_usd"], 4);
+ assert.equal(v["cost.subagent_usd"], 1.5);
+});
+
+test("cost and tokens are the runner's eval usage, and unavailable without it", () => {
+ const artifact = fixture("agent-tools-ends-turn-function");
+ artifact.result.derived.usage = {
+ cacheReadTokens: 40,
+ cacheWriteTokens: 5,
+ inputTokens: 200,
+ outputTokens: 12,
+ costUsd: 0.75,
+ };
+ const v = values(deriveAttemptMetrics(artifact).metrics);
+ assert.equal(v["cost.total_usd"], 0.75);
+ assert.equal(v["tokens.input"], 200);
+ assert.equal(v["tokens.output"], 12);
+ assert.equal(v["tokens.cache_read"], 40);
+ assert.equal(v["tokens.cache_write"], 5);
+ assert.equal(v["tokens.cache_hit_ratio"], 0.2);
+ // Peak context is per step, from the events, not the eval total.
+ assert.equal(v["tokens.peak_context"], 391);
+
+ // A counted session reported no cost: the runner leaves costUsd out.
+ delete artifact.result.derived.usage.costUsd;
+ const noCost = deriveAttemptMetrics(artifact).metrics;
+ assert.deepEqual(noCost["cost.total_usd"], {
+ status: "unavailable",
+ reason: "missing-rolled-up-cost",
+ });
+ assert.equal(noCost["tokens.input"].status, "measured");
+
+ // A counted session reported no usage at all: the runner leaves usage out.
+ delete artifact.result.derived.usage;
+ const none = deriveAttemptMetrics(artifact).metrics;
+ for (const key of ["cost.total_usd", "tokens.input", "tokens.output", "tokens.cache_hit_ratio"])
+ assert.equal(none[key].status, "unavailable", key);
+ assert.equal(none["tokens.peak_context"].status, "measured");
+});
+
+test("compaction cost stays in the total but leaves the subagent split unknown", () => {
+ // Compaction summary calls are paid and roll up into session usage, but no
+ // step reports them: total − steps would misreport them as subagent spend.
+ const artifact = withCosts(compactedArtifact(), {
+ stepCost: 0.25,
+ rolledUpCost: 2,
+ });
+ const { metrics } = deriveAttemptMetrics(artifact);
+ assert.equal(values(metrics)["cost.total_usd"], 2);
+ assert.equal(metrics["cost.primary_usd"].status, "measured");
+ assert.deepEqual(metrics["cost.subagent_usd"], {
+ status: "unavailable",
+ reason: "compaction-cost-unattributed",
+ });
+});
+
+test("a single step without cost makes cost unavailable, and inconsistent rollups are flagged", () => {
+ const artifact = withCosts(fixture("agent-tools-ends-turn-function"), {
+ stepCost: 1,
+ rolledUpCost: 3,
+ });
+ assert.equal(values(deriveAttemptMetrics(artifact).metrics)["cost.subagent_usd"], 0);
+
+ const inconsistent = withCosts(artifact, { stepCost: 1, rolledUpCost: 2 });
+ assert.equal(
+ deriveAttemptMetrics(inconsistent).metrics["cost.subagent_usd"].reason,
+ "inconsistent-cost-rollup",
+ );
+
+ const step = artifact.result.sessions[0].events.find((e) => e.type === "step.completed");
+ delete step.data.usage.costUsd;
+ const { metrics } = deriveAttemptMetrics(artifact);
+ assert.equal(metrics["cost.primary_usd"].reason, "missing-step-cost");
+ assert.equal(metrics["cost.subagent_usd"].reason, "missing-step-cost");
+ assert.equal(metrics["cost.total_usd"].status, "measured");
+});
+
+test("overlapping parallel tools count once and streaming metrics use first deltas", () => {
+ const step = { turnId: "t", stepIndex: 0 };
+ const events = turn(0, 1000, [
+ event("step.started", 100, step),
+ event("reasoning.appended", 150, step),
+ event("message.appended", 300, step),
+ event("actions.requested", 400, {
+ ...step,
+ actions: [
+ { kind: "tool-call", callId: "a" },
+ { kind: "tool-call", callId: "b" },
+ ],
+ }),
+ event("action.result", 700, { ...step, result: { callId: "b" }, status: "failed" }),
+ event("action.result", 600, { ...step, result: { callId: "a" }, status: "completed" }),
+ event("step.completed", 750, { ...step, usage: { outputTokens: 60, inputTokens: 900 } }),
+ ]);
+ const v = values(deriveAttemptMetrics(synthetic(events)).metrics);
+ assert.equal(v["attribution.tool_ms"], 300);
+ assert.equal(v["attribution.model_ms"], 350);
+ assert.equal(v["attribution.framework_ms"], 350);
+ assert.equal(v["latency.ttft_ms_p50"], 50);
+ assert.equal(v["latency.output_tokens_per_s"], 100);
+ assert.equal(v["reliability.tool_error_rate"], 0.5);
+ assert.equal(v["counts.tool_calls_per_step"], 2);
+});
+
+test("partial captures make attribution unavailable instead of guessing", () => {
+ const openTurn = [
+ event("turn.started", 0, { turnId: "t" }),
+ event("step.started", 10, { turnId: "t", stepIndex: 0 }),
+ ];
+ assert.equal(
+ deriveAttemptMetrics(synthetic(openTurn)).metrics["attribution.framework_ms"].reason,
+ "turn-incomplete",
+ );
+ const orphanResult = turn(0, 100, [event("action.result", 50, { result: { callId: "x" } })]);
+ assert.equal(
+ deriveAttemptMetrics(synthetic(orphanResult)).metrics["attribution.tool_ms"].reason,
+ "action-start-missing",
+ );
+ const openInput = turn(0, 100, [
+ event("input.requested", 50, { turnId: "unknown", requests: [{ requestId: "r" }] }),
+ ]);
+ assert.equal(
+ deriveAttemptMetrics(synthetic(openInput)).metrics["attribution.human_wait_ms"].reason,
+ "input-incomplete",
+ );
+ const noCapture = { ...synthetic([]), result: { status: "completed" } };
+ assert.equal(
+ deriveAttemptMetrics(noCapture).metrics["latency.turn_ms"].reason,
+ "missing-session-capture",
+ );
+ const conflicting = synthetic([event("turn.started", 0, { turnId: "t" })]);
+ conflicting.result.sessions.push({
+ sessionId: "s",
+ events: [{ ...conflicting.result.sessions[0].events[0], data: { turnId: "other" } }],
+ });
+ assert.equal(
+ deriveAttemptMetrics(conflicting).metrics["counts.turns"].reason,
+ "invalid-session-capture",
+ );
+});
+
+test("a cancelled turn closes work it abandoned", () => {
+ const events = turn(
+ 0,
+ 500,
+ [
+ event("step.started", 10, { turnId: "t", stepIndex: 0 }),
+ event("actions.requested", 100, {
+ turnId: "t",
+ stepIndex: 0,
+ actions: [{ kind: "tool-call", callId: "a" }],
+ }),
+ ],
+ "turn.cancelled",
+ );
+ const v = values(deriveAttemptMetrics(synthetic(events)).metrics);
+ assert.equal(v["attribution.tool_ms"], 400);
+ assert.equal(v["attribution.model_ms"], 90);
+});
+
+test("counts failures by event and code", () => {
+ const events = turn(
+ 0,
+ 100,
+ [
+ event("step.started", 10, { turnId: "t", stepIndex: 0 }),
+ event("step.failed", 20, { turnId: "t", stepIndex: 0, code: "MODEL_CALL_FAILED" }),
+ event("session.failed", 99, { code: "MODEL_CALL_FAILED" }),
+ ],
+ "turn.failed",
+ );
+ events.at(-1).data.code = "MODEL_CALL_FAILED";
+ const { failures, metrics } = deriveAttemptMetrics(synthetic(events));
+ assert.deepEqual(failures, [
+ { type: "session.failed", code: "MODEL_CALL_FAILED", count: 1 },
+ { type: "step.failed", code: "MODEL_CALL_FAILED", count: 1 },
+ { type: "turn.failed", code: "MODEL_CALL_FAILED", count: 1 },
+ ]);
+ assert.equal(values(metrics)["reliability.step_failures"], 1);
+});
+
+test("classifies outcomes and every gap reason", () => {
+ const failed = (code, details, message = "failed") =>
+ synthetic(
+ turn(0, 100, [
+ event("step.failed", 50, { turnId: "t", stepIndex: 0, code, details, message }),
+ ]),
+ );
+ const completedStep = event("step.completed", 50, { turnId: "t", stepIndex: 0 });
+
+ assert.deepEqual(classifyOutcome(fixture("agent-tools-ends-turn-function")), {
+ outcome: "completed",
+ });
+ assert.deepEqual(classifyOutcome({ ...synthetic([]), verdict: "skipped" }), {
+ outcome: "skipped",
+ });
+ for (const details of [
+ { statusCode: 429 },
+ { upstreamStatusCode: 503 },
+ { semanticErrorId: "network-request-failed" },
+ { detail: "TypeError: fetch failed [cause]: UND_ERR_HEADERS_TIMEOUT" },
+ ])
+ assert.deepEqual(classifyOutcome(failed("MODEL_CALL_FAILED", details)), {
+ outcome: "gap",
+ gap_reason: "provider-unavailable",
+ });
+ for (const details of [{ statusCode: 400 }, { statusCode: 413 }])
+ assert.equal(classifyOutcome(failed("MODEL_CALL_FAILED", details)).outcome, "completed");
+ assert.equal(classifyOutcome(failed("OUTPUT_SCHEMA_NOT_FULFILLED", {})).outcome, "completed");
+ assert.equal(
+ classifyOutcome(failed("WORKFLOW_STREAM_WRITE_FAILED", {})).gap_reason,
+ "stream-write-failed",
+ );
+
+ const timeout = "The operation was aborted due to timeout";
+ assert.equal(
+ classifyOutcome(synthetic([], { error: timeout })).gap_reason,
+ "timeout-before-first-step",
+ );
+ assert.equal(
+ classifyOutcome(synthetic([], { error: "connect ECONNREFUSED" })).gap_reason,
+ "error-before-first-step",
+ );
+ assert.deepEqual(classifyOutcome(synthetic([completedStep], { error: timeout })), {
+ outcome: "timed_out",
+ });
+ // A first step that fails for a real reason is the eval's failure, not a gap,
+ // even though the eval then records an error (e.g. from expectOk()).
+ for (const details of [{ statusCode: 400 }, { statusCode: 413 }]) {
+ const rejected = { ...failed("MODEL_CALL_FAILED", details), error: "expectOk: session failed" };
+ assert.deepEqual(classifyOutcome(rejected), { outcome: "completed" });
+ assert.deepEqual(classifyOutcome({ ...rejected, error: timeout }), { outcome: "timed_out" });
+ }
+ const sessionFailed = synthetic([event("session.failed", 40, { code: "SESSION_FAILED" })], {
+ error: "expectOk: session failed",
+ });
+ assert.deepEqual(classifyOutcome(sessionFailed), { outcome: "completed" });
+ // Parked is the runner's derived fact (eve's derive-run-facts), not re-derived here.
+ const parked = synthetic([completedStep]);
+ parked.result.derived = { parked: true };
+ assert.deepEqual(classifyOutcome(parked), { outcome: "parked" });
+});