The Open Framework for Building Agents
ci(eval-benchmark): add derived metrics in eval benchmark records (#5236)
prha8aedd027d736
diff --git a/package.json b/package.jsonindex ebbad9aa3..cc415b072 100644--- a/package.json+++ b/package.json@@ -26,7 +26,7 @@"test": "pnpm test:unit && pnpm test:integration","test:e2e": "turbo run test:e2e --concurrency=1","test:webchat": "pnpm --filter eve run test:webchat",- "test:eval-benchmark": "node --test scripts/eval-benchmark/*.test.mjs",+ "test:eval-benchmark": "node --test scripts/eval-metrics/*.test.mjs scripts/eval-benchmark/*.test.mjs","test:framework-fixtures": "pnpm --filter eve run test:framework-fixtures","test:integration": "turbo run test:integration --filter=eve && turbo run test:integration --filter=\"!eve\"","test:scenario": "turbo run test:scenario",diff --git a/scripts/eval-benchmark/records.mjs b/scripts/eval-benchmark/records.mjsindex 12a27d5c6..41d278dca 100644--- a/scripts/eval-benchmark/records.mjs+++ b/scripts/eval-benchmark/records.mjs@@ -9,8 +9,16 @@// (github_run_id, run_attempt, fixture, eval_id, model_id, world, attempt).// Results are comparable across runs when (eval_id, fixture_digest, model_id,// model_release, world) match. Planned legs that uploaded nothing, and expected-// evals with no result, become `missing` records so coverage stays honest.-// Richer measurements can be derived later from the published raw trees.+// evals with no result, become `missing` records (outcome "gap") so coverage+// stays honest.+//+// Each attempt also carries the standard metric set (latency, attribution,+// cost, tokens, counts, reliability) derived from its eval detail artifact by+// scripts/eval-metrics/standard.mjs, and an outcome from+// scripts/eval-metrics/gaps.mjs that separates infrastructure gaps (provider+// outages, timeouts before any model step) from real failures. A metric that+// can't be derived is `unavailable` or `not-applicable` with a reason, never+// zero.//// A live leg's eval whose model steps all ran on another model (fixtures script// some delegation evals with eve's mock models) says nothing about the leg's@@ -21,6 +29,13 @@ import { existsSync, readdirSync, readFileSync, writeFileSync } from "node:fs";import { join } from "node:path";import { fileURLToPath } from "node:url";+import { classifyOutcome } from "../eval-metrics/gaps.mjs";+import {+ METRICS_VERSION,+ deriveAttemptMetrics,+ listEvalArtifacts,+} from "../eval-metrics/standard.mjs";+export const RECORD_SCHEMA_VERSION = 1;export const LEG_ARTIFACT_PREFIX = "benchmark-leg-";@@ -105,6 +120,14 @@ export function buildRecords({ plan, legs, githubRunId, runAttempt }) {const records = attempts.map(({ leg, meta, evalId, artifact, missingReason, uploaded }) => {const startedAt = artifact?.startedAt ?? null;const completedAt = artifact?.completedAt ?? null;+ const { outcome, gap_reason } =+ artifact === undefined+ ? { outcome: "gap", gap_reason: missingReason }+ : classifyOutcome(artifact);+ const { metrics_version, metrics, failures } =+ artifact === undefined+ ? { metrics_version: METRICS_VERSION, metrics: {}, failures: [] }+ : deriveAttemptMetrics(artifact);return {schema_version: RECORD_SCHEMA_VERSION,github_run_id: String(githubRunId),@@ -129,6 +152,11 @@ export function buildRecords({ plan, legs, githubRunId, runAttempt }) {passed,})),error: artifact?.error ?? null,+ outcome,+ gap_reason: gap_reason ?? null,+ metrics_version,+ metrics,+ failures,started_at: startedAt,completed_at: completedAt,duration_ms:@@ -150,16 +178,6 @@ function ranOnModel(artifact, modelId) {return models.length === 0 || models.includes(modelId);}-/** Find every eval detail artifact under one `.eve/evals/<timestamp>` tree. */-export function listEvalArtifacts(runDirectory) {- const evalsDirectory = join(runDirectory, "evals");- if (!existsSync(evalsDirectory)) return [];- return readdirSync(evalsDirectory, { withFileTypes: true, recursive: true })- .filter((entry) => entry.isFile() && entry.name.endsWith(".json"))- .map((entry) => join(entry.parentPath, entry.name))- .sort();-}-if (process.argv[1] === fileURLToPath(import.meta.url)) {const [planPath, legsDir, outPath] = process.argv.slice(2);if (outPath === undefined) {diff --git a/scripts/eval-benchmark/records.test.mjs b/scripts/eval-benchmark/records.test.mjsindex 31c725c62..dc1edfb5f 100644--- a/scripts/eval-benchmark/records.test.mjs+++ b/scripts/eval-benchmark/records.test.mjs@@ -1,9 +1,10 @@import test from "node:test";import assert from "node:assert/strict";-import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";+import { copyFileSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";import { tmpdir } from "node:os";import { join } from "node:path";import { publishRun } from "./publish.mjs";+import { METRICS_VERSION, STANDARD_METRICS } from "../eval-metrics/standard.mjs";import { buildRecords, readLegs } from "./records.mjs";const SHA = "a".repeat(40);@@ -115,6 +116,49 @@ test("builds records and turns missing legs and evals into missing records", (t)]);assert.equal(records[2].model_release, null);assert.equal(records[2].duration_ms, null);+ assert.equal(completed.outcome, "completed");+ assert.deepEqual(completed.metrics["latency.turn_ms"], {+ status: "unavailable",+ reason: "missing-session-capture",+ });+});++test("attaches metrics to captured attempts and marks missing attempts as gaps", (t) => {+ const legsDir = legsDirectory(t);+ // A real captured artifact, with session events, in place of the minimal one.+ copyFileSync(+ new URL("../eval-metrics/fixtures/agent-tools-ends-turn-function.json", import.meta.url),+ join(+ legsDir,+ "benchmark-leg-live__agent-tools__alpha__local__1",+ "run",+ "evals",+ "static-tools",+ "ends-turn-function.json",+ ),+ );+ const [captured, missingEval, missingJob] = build(legsDir);+ assert.equal(captured.outcome, "completed");+ assert.equal(captured.gap_reason, null);+ assert.equal(captured.metrics_version, METRICS_VERSION);+ assert.deepEqual(Object.keys(captured.metrics).sort(), Object.keys(STANDARD_METRICS).sort());+ assert.deepEqual(captured.metrics["tokens.input"], { status: "measured", value: 1117 });+ // The fixture's model reports no cost: unavailable with a reason, never zero.+ assert.deepEqual(captured.metrics["cost.total_usd"], {+ status: "unavailable",+ reason: "missing-rolled-up-cost",+ });+ assert.deepEqual(captured.failures, []);++ for (const [record, reason] of [+ [missingEval, "eval-missing"],+ [missingJob, "job-missing"],+ ]) {+ assert.equal(record.outcome, "gap");+ assert.equal(record.gap_reason, reason);+ assert.deepEqual(record.metrics, {});+ assert.deepEqual(record.failures, []);+ }});test("excludes live-leg evals whose model steps all ran on another model", (t) => {diff --git a/scripts/eval-metrics/fixtures/agent-subagents-continuation-local.json b/scripts/eval-metrics/fixtures/agent-subagents-continuation-local.jsonnew file mode 100644index 000000000..33cddbba2--- /dev/null+++ b/scripts/eval-metrics/fixtures/agent-subagents-continuation-local.json@@ -0,0 +1,798 @@+{+ "id": "agent-continuation-local",+ "verdict": "passed",+ "startedAt": "2026-10-05T23:37:05.534Z",+ "completedAt": "2026-10-05T23:37:08.653Z",+ "result": {+ "derived": {+ "usage": {+ "cacheReadTokens": 0,+ "cacheWriteTokens": 0,+ "inputTokens": 6006,+ "outputTokens": 178+ }+ },+ "sessionId": "wrun_01M476V5RJPATF4WMMJD3PAX09",+ "runtimeIdentity": {+ "eveVersion": "0.71.2"+ },+ "sessions": [+ {+ "sessionId": "wrun_01M476V5RJPATF4WMMJD3PAX09",+ "events": [+ {+ "type": "turn.started",+ "meta": {+ "id": "evt_01M476V6228BYTWHV7MKWND209",+ "at": "2026-10-05T23:37:05.858Z"+ },+ "data": {+ "turnId": "turn_0"+ }+ },+ {+ "type": "step.started",+ "meta": {+ "id": "evt_01M476V6233AEN19P97Z73SW8Y",+ "at": "2026-10-05T23:37:05.859Z"+ },+ "data": {+ "turnId": "turn_0",+ "stepIndex": 0+ }+ },+ {+ "type": "actions.requested",+ "meta": {+ "id": "evt_01M476V62C7GS0XB8YSMBRD14H",+ "at": "2026-10-05T23:37:05.868Z"+ },+ "data": {+ "actions": [+ {+ "callId": "notebook-remember",+ "kind": "tool-call"+ }+ ]+ }+ },+ {+ "type": "step.completed",+ "meta": {+ "id": "evt_01M476V62FPZP28T46PNM7S065",+ "at": "2026-10-05T23:37:05.871Z"+ },+ "data": {+ "turnId": "turn_0",+ "stepIndex": 0,+ "usage": {+ "inputTokens": 332,+ "outputTokens": 27,+ "cacheReadTokens": 0,+ "cacheWriteTokens": 0+ },+ "finishReason": "tool-calls"+ }+ },+ {+ "type": "task.started",+ "meta": {+ "id": "evt_01M476V63J65W3D9RW1NNBY17K",+ "at": "2026-10-05T23:37:05.906Z"+ },+ "data": {+ "taskId": "notebook-keeper-eqtna6",+ "callId": "notebook-remember"+ }+ },+ {+ "type": "action.result",+ "meta": {+ "id": "evt_01M476V641X0H8BK1KC4JV92AZ",+ "at": "2026-10-05T23:37:05.921Z"+ },+ "data": {+ "result": {+ "callId": "notebook-remember"+ },+ "status": "completed"+ }+ },+ {+ "type": "step.started",+ "meta": {+ "id": "evt_01M476V642AZR2058TEEM9X5EE",+ "at": "2026-10-05T23:37:05.922Z"+ },+ "data": {+ "turnId": "turn_0",+ "stepIndex": 1+ }+ },+ {+ "type": "actions.requested",+ "meta": {+ "id": "evt_01M476V645XV19K4368X5VK1B6",+ "at": "2026-10-05T23:37:05.925Z"+ },+ "data": {+ "actions": [+ {+ "callId": "notebook-remember-wait",+ "kind": "tool-call"+ }+ ]+ }+ },+ {+ "type": "step.completed",+ "meta": {+ "id": "evt_01M476V646ZCXJ0YY0CE8DSFT3",+ "at": "2026-10-05T23:37:05.926Z"+ },+ "data": {+ "turnId": "turn_0",+ "stepIndex": 1,+ "usage": {+ "inputTokens": 402,+ "outputTokens": 1,+ "cacheReadTokens": 0,+ "cacheWriteTokens": 0+ },+ "finishReason": "tool-calls"+ }+ },+ {+ "type": "agent.started",+ "meta": {+ "id": "evt_01M476V6898SRG54ZSBFDFWJZH",+ "at": "2026-10-05T23:37:06.057Z"+ },+ "data": {+ "sessionId": "wrun_01M476V658DK2JBTYGB4XWHDZ0"+ }+ },+ {+ "type": "task.settled",+ "meta": {+ "id": "evt_01M476V6A2Z1SG7X142798JF05",+ "at": "2026-10-05T23:37:06.114Z"+ },+ "data": {+ "taskId": "notebook-keeper-eqtna6",+ "callId": "notebook-remember"+ }+ },+ {+ "type": "action.result",+ "meta": {+ "id": "evt_01M476V6AF87ZGQAP6CT77QW5G",+ "at": "2026-10-05T23:37:06.127Z"+ },+ "data": {+ "result": {+ "callId": "notebook-remember-wait"+ },+ "status": "completed"+ }+ },+ {+ "type": "step.started",+ "meta": {+ "id": "evt_01M476V6AF87ZGQAP6CT77QW5H",+ "at": "2026-10-05T23:37:06.127Z"+ },+ "data": {+ "turnId": "turn_0",+ "stepIndex": 2+ }+ },+ {+ "type": "message.appended",+ "meta": {+ "id": "evt_01M476V6AJFFZPR5418NZFPRTZ",+ "at": "2026-10-05T23:37:06.130Z"+ },+ "data": {+ "turnId": "turn_0",+ "stepIndex": 2+ }+ },+ {+ "type": "step.completed",+ "meta": {+ "id": "evt_01M476V6AJFFZPR5418NZFPRV1",+ "at": "2026-10-05T23:37:06.130Z"+ },+ "data": {+ "turnId": "turn_0",+ "stepIndex": 2,+ "usage": {+ "inputTokens": 468,+ "outputTokens": 8,+ "cacheReadTokens": 0,+ "cacheWriteTokens": 0+ },+ "finishReason": "stop"+ }+ },+ {+ "type": "turn.completed",+ "meta": {+ "id": "evt_01M476V6AN6BYGDG7F56X57112",+ "at": "2026-10-05T23:37:06.133Z"+ },+ "data": {+ "turnId": "turn_0"+ }+ },+ {+ "type": "turn.started",+ "meta": {+ "id": "evt_01M476V6C9MRXR1GD56EV73ZQF",+ "at": "2026-10-05T23:37:06.185Z"+ },+ "data": {+ "turnId": "turn_1"+ }+ },+ {+ "type": "step.started",+ "meta": {+ "id": "evt_01M476V6C9MRXR1GD56EV73ZQH",+ "at": "2026-10-05T23:37:06.185Z"+ },+ "data": {+ "turnId": "turn_1",+ "stepIndex": 0+ }+ },+ {+ "type": "actions.requested",+ "meta": {+ "id": "evt_01M476V6CC7A5R3Q13GNTS1WV3",+ "at": "2026-10-05T23:37:06.188Z"+ },+ "data": {+ "actions": [+ {+ "callId": "notebook-review",+ "kind": "tool-call"+ }+ ]+ }+ },+ {+ "type": "step.completed",+ "meta": {+ "id": "evt_01M476V6CC7A5R3Q13GNTS1WV4",+ "at": "2026-10-05T23:37:06.188Z"+ },+ "data": {+ "turnId": "turn_1",+ "stepIndex": 0,+ "usage": {+ "inputTokens": 496,+ "outputTokens": 32,+ "cacheReadTokens": 0,+ "cacheWriteTokens": 0+ },+ "finishReason": "tool-calls"+ }+ },+ {+ "type": "task.started",+ "meta": {+ "id": "evt_01M476V6D7X4DPT801AKTD751Q",+ "at": "2026-10-05T23:37:06.215Z"+ },+ "data": {+ "taskId": "notebook-keeper-eqtna6",+ "callId": "notebook-review"+ }+ },+ {+ "type": "action.result",+ "meta": {+ "id": "evt_01M476V6DRHF4FYDKZ1NVPXHE3",+ "at": "2026-10-05T23:37:06.232Z"+ },+ "data": {+ "result": {+ "callId": "notebook-review"+ },+ "status": "completed"+ }+ },+ {+ "type": "step.started",+ "meta": {+ "id": "evt_01M476V6DSJ0SD80XPPPRQZZBH",+ "at": "2026-10-05T23:37:06.233Z"+ },+ "data": {+ "turnId": "turn_1",+ "stepIndex": 1+ }+ },+ {+ "type": "actions.requested",+ "meta": {+ "id": "evt_01M476V6DV5APK6YK4ARJ56BAJ",+ "at": "2026-10-05T23:37:06.235Z"+ },+ "data": {+ "actions": [+ {+ "callId": "notebook-review-wait",+ "kind": "tool-call"+ }+ ]+ }+ },+ {+ "type": "step.completed",+ "meta": {+ "id": "evt_01M476V6DV5APK6YK4ARJ56BAK",+ "at": "2026-10-05T23:37:06.235Z"+ },+ "data": {+ "turnId": "turn_1",+ "stepIndex": 1,+ "usage": {+ "inputTokens": 543,+ "outputTokens": 5,+ "cacheReadTokens": 0,+ "cacheWriteTokens": 0+ },+ "finishReason": "tool-calls"+ }+ },+ {+ "type": "action.result",+ "meta": {+ "id": "evt_01M476V8E667SBF93W9HCSJ4VD",+ "at": "2026-10-05T23:37:08.294Z"+ },+ "data": {+ "result": {+ "callId": "notebook-review-wait"+ },+ "status": "completed"+ }+ },+ {+ "type": "step.started",+ "meta": {+ "id": "evt_01M476V8E7S6HBJVCYS5FSH3SW",+ "at": "2026-10-05T23:37:08.295Z"+ },+ "data": {+ "turnId": "turn_1",+ "stepIndex": 2+ }+ },+ {+ "type": "actions.requested",+ "meta": {+ "id": "evt_01M476V8EAWVYYB5H2Q699KMG6",+ "at": "2026-10-05T23:37:08.298Z"+ },+ "data": {+ "actions": [+ {+ "callId": "notebook-cancel",+ "kind": "tool-call"+ }+ ]+ }+ },+ {+ "type": "step.completed",+ "meta": {+ "id": "evt_01M476V8EBTW72T7YGSZTNMA8T",+ "at": "2026-10-05T23:37:08.299Z"+ },+ "data": {+ "turnId": "turn_1",+ "stepIndex": 2,+ "usage": {+ "inputTokens": 576,+ "outputTokens": 9,+ "cacheReadTokens": 0,+ "cacheWriteTokens": 0+ },+ "finishReason": "tool-calls"+ }+ },+ {+ "type": "task.settled",+ "meta": {+ "id": "evt_01M476V8F8Z1FX7TZSZ7ZHDJRX",+ "at": "2026-10-05T23:37:08.328Z"+ },+ "data": {+ "taskId": "notebook-keeper-eqtna6",+ "callId": "notebook-review"+ }+ },+ {+ "type": "action.result",+ "meta": {+ "id": "evt_01M476V8FTN14FSP3KYHVCYTRZ",+ "at": "2026-10-05T23:37:08.346Z"+ },+ "data": {+ "result": {+ "callId": "notebook-cancel"+ },+ "status": "completed"+ }+ },+ {+ "type": "step.started",+ "meta": {+ "id": "evt_01M476V8FTN14FSP3KYHVCYTS0",+ "at": "2026-10-05T23:37:08.346Z"+ },+ "data": {+ "turnId": "turn_1",+ "stepIndex": 3+ }+ },+ {+ "type": "message.appended",+ "meta": {+ "id": "evt_01M476V8FXZHQQ4Y9YX9JK4MDH",+ "at": "2026-10-05T23:37:08.349Z"+ },+ "data": {+ "turnId": "turn_1",+ "stepIndex": 3+ }+ },+ {+ "type": "step.completed",+ "meta": {+ "id": "evt_01M476V8FXZHQQ4Y9YX9JK4MDK",+ "at": "2026-10-05T23:37:08.349Z"+ },+ "data": {+ "turnId": "turn_1",+ "stepIndex": 3,+ "usage": {+ "inputTokens": 639,+ "outputTokens": 42,+ "cacheReadTokens": 0,+ "cacheWriteTokens": 0+ },+ "finishReason": "stop"+ }+ },+ {+ "type": "turn.completed",+ "meta": {+ "id": "evt_01M476V8G10T9NCS0BYG14C6P5",+ "at": "2026-10-05T23:37:08.353Z"+ },+ "data": {+ "turnId": "turn_1"+ }+ },+ {+ "type": "turn.started",+ "meta": {+ "id": "evt_01M476V8K9NZ8YZ6QQKNAH92TD",+ "at": "2026-10-05T23:37:08.457Z"+ },+ "data": {+ "turnId": "turn_2"+ }+ },+ {+ "type": "step.started",+ "meta": {+ "id": "evt_01M476V8KA3WS00AS63TKXAKKC",+ "at": "2026-10-05T23:37:08.458Z"+ },+ "data": {+ "turnId": "turn_2",+ "stepIndex": 0+ }+ },+ {+ "type": "actions.requested",+ "meta": {+ "id": "evt_01M476V8KCQ4RPNX1HD3Y1C7N8",+ "at": "2026-10-05T23:37:08.460Z"+ },+ "data": {+ "actions": [+ {+ "callId": "notebook-recall",+ "kind": "tool-call"+ }+ ]+ }+ },+ {+ "type": "step.completed",+ "meta": {+ "id": "evt_01M476V8KCQ4RPNX1HD3Y1C7N9",+ "at": "2026-10-05T23:37:08.460Z"+ },+ "data": {+ "turnId": "turn_2",+ "stepIndex": 0,+ "usage": {+ "inputTokens": 695,+ "outputTokens": 29,+ "cacheReadTokens": 0,+ "cacheWriteTokens": 0+ },+ "finishReason": "tool-calls"+ }+ },+ {+ "type": "task.started",+ "meta": {+ "id": "evt_01M476V8M6Z762FW3TK0BPFPDC",+ "at": "2026-10-05T23:37:08.486Z"+ },+ "data": {+ "taskId": "notebook-keeper-eqtna6",+ "callId": "notebook-recall"+ }+ },+ {+ "type": "action.result",+ "meta": {+ "id": "evt_01M476V8MNP1SETGP49BVH3SB6",+ "at": "2026-10-05T23:37:08.501Z"+ },+ "data": {+ "result": {+ "callId": "notebook-recall"+ },+ "status": "completed"+ }+ },+ {+ "type": "step.started",+ "meta": {+ "id": "evt_01M476V8MPWG53PDJJX23ZGY1K",+ "at": "2026-10-05T23:37:08.502Z"+ },+ "data": {+ "turnId": "turn_2",+ "stepIndex": 1+ }+ },+ {+ "type": "actions.requested",+ "meta": {+ "id": "evt_01M476V8MRFEP96D0MWRXSBG3S",+ "at": "2026-10-05T23:37:08.504Z"+ },+ "data": {+ "actions": [+ {+ "callId": "notebook-recall-wait",+ "kind": "tool-call"+ }+ ]+ }+ },+ {+ "type": "step.completed",+ "meta": {+ "id": "evt_01M476V8MRFEP96D0MWRXSBG3T",+ "at": "2026-10-05T23:37:08.504Z"+ },+ "data": {+ "turnId": "turn_2",+ "stepIndex": 1,+ "usage": {+ "inputTokens": 742,+ "outputTokens": 1,+ "cacheReadTokens": 0,+ "cacheWriteTokens": 0+ },+ "finishReason": "tool-calls"+ }+ },+ {+ "type": "task.settled",+ "meta": {+ "id": "evt_01M476V8RPXJGVVE5F24WJ79AF",+ "at": "2026-10-05T23:37:08.630Z"+ },+ "data": {+ "taskId": "notebook-keeper-eqtna6",+ "callId": "notebook-recall"+ }+ },+ {+ "type": "action.result",+ "meta": {+ "id": "evt_01M476V8S10A6Z1Y50ERZCD2A9",+ "at": "2026-10-05T23:37:08.641Z"+ },+ "data": {+ "result": {+ "callId": "notebook-recall-wait"+ },+ "status": "completed"+ }+ },+ {+ "type": "step.started",+ "meta": {+ "id": "evt_01M476V8S2RMQRM83HM2NQ6N86",+ "at": "2026-10-05T23:37:08.642Z"+ },+ "data": {+ "turnId": "turn_2",+ "stepIndex": 2+ }+ },+ {+ "type": "message.appended",+ "meta": {+ "id": "evt_01M476V8S3ZK1JPMHQEJ5WW503",+ "at": "2026-10-05T23:37:08.643Z"+ },+ "data": {+ "turnId": "turn_2",+ "stepIndex": 2+ }+ },+ {+ "type": "step.completed",+ "meta": {+ "id": "evt_01M476V8S40C4W83AANGJFAY23",+ "at": "2026-10-05T23:37:08.644Z"+ },+ "data": {+ "turnId": "turn_2",+ "stepIndex": 2,+ "usage": {+ "inputTokens": 813,+ "outputTokens": 12,+ "cacheReadTokens": 0,+ "cacheWriteTokens": 0+ },+ "finishReason": "stop"+ }+ },+ {+ "type": "turn.completed",+ "meta": {+ "id": "evt_01M476V8S8M483PXMJW7TCF6A5",+ "at": "2026-10-05T23:37:08.648Z"+ },+ "data": {+ "turnId": "turn_2"+ }+ }+ ]+ },+ {+ "sessionId": "wrun_01M476V658DK2JBTYGB4XWHDZ0",+ "events": [+ {+ "type": "turn.started",+ "meta": {+ "id": "evt_01M476V66SMTJ5E0FTNQ3KTBAF",+ "at": "2026-10-05T23:37:06.009Z"+ },+ "data": {+ "turnId": "turn_0"+ }+ },+ {+ "type": "step.started",+ "meta": {+ "id": "evt_01M476V66SMTJ5E0FTNQ3KTBAH",+ "at": "2026-10-05T23:37:06.009Z"+ },+ "data": {+ "turnId": "turn_0",+ "stepIndex": 0+ }+ },+ {+ "type": "message.appended",+ "meta": {+ "id": "evt_01M476V66VDRJ4S2TSRXESVFGN",+ "at": "2026-10-05T23:37:06.011Z"+ },+ "data": {+ "turnId": "turn_0",+ "stepIndex": 0+ }+ },+ {+ "type": "step.completed",+ "meta": {+ "id": "evt_01M476V66VDRJ4S2TSRXESVFGQ",+ "at": "2026-10-05T23:37:06.011Z"+ },+ "data": {+ "turnId": "turn_0",+ "stepIndex": 0,+ "usage": {+ "inputTokens": 130,+ "outputTokens": 4,+ "cacheReadTokens": 0,+ "cacheWriteTokens": 0+ },+ "finishReason": "stop"+ }+ },+ {+ "type": "turn.completed",+ "meta": {+ "id": "evt_01M476V671EP9FBHHQBZNAD0CT",+ "at": "2026-10-05T23:37:06.017Z"+ },+ "data": {+ "turnId": "turn_0"+ }+ }+ ]+ },+ {+ "sessionId": "wrun_01M476V658DK2JBTYGB4XWHDZ0",+ "events": [+ {+ "type": "turn.started",+ "meta": {+ "id": "evt_01M476V6F9EBD55ZG238NQ1ZJW",+ "at": "2026-10-05T23:37:06.281Z"+ },+ "data": {+ "turnId": "turn_1"+ }+ },+ {+ "type": "step.started",+ "meta": {+ "id": "evt_01M476V6F9EBD55ZG238NQ1ZJY",+ "at": "2026-10-05T23:37:06.281Z"+ },+ "data": {+ "turnId": "turn_1",+ "stepIndex": 0+ }+ },+ {+ "type": "actions.requested",+ "meta": {+ "id": "evt_01M476V6FBET7W3ABD5M66T5ZH",+ "at": "2026-10-05T23:37:06.283Z"+ },+ "data": {+ "actions": [+ {+ "callId": "notebook-review-2",+ "kind": "tool-call"+ }+ ]+ }+ },+ {+ "type": "turn.cancelled",+ "meta": {+ "id": "evt_01M476V8HQR8KWGGH1VA5BHRGS",+ "at": "2026-10-05T23:37:08.407Z"+ },+ "data": {+ "turnId": "turn_1"+ }+ }+ ]+ }+ ]+ }+}diff --git a/scripts/eval-metrics/fixtures/agent-tools-ends-turn-function.json b/scripts/eval-metrics/fixtures/agent-tools-ends-turn-function.jsonnew file mode 100644index 000000000..8b555bf15--- /dev/null+++ b/scripts/eval-metrics/fixtures/agent-tools-ends-turn-function.json@@ -0,0 +1,221 @@+{+ "id": "static-tools/ends-turn-function",+ "verdict": "passed",+ "startedAt": "2026-10-05T23:35:48.752Z",+ "completedAt": "2026-10-05T23:35:49.802Z",+ "result": {+ "derived": {+ "usage": {+ "cacheReadTokens": 0,+ "cacheWriteTokens": 0,+ "inputTokens": 1117,+ "outputTokens": 29+ }+ },+ "sessionId": "wrun_01M476RTSJ569Y4TN0BBRB2ED0",+ "runtimeIdentity": {+ "eveVersion": "0.71.2"+ },+ "sessions": [+ {+ "sessionId": "wrun_01M476RTSJ569Y4TN0BBRB2ED0",+ "events": [+ {+ "type": "turn.started",+ "meta": {+ "id": "evt_01M476RV3PFF8YGR8F46VHK6NW",+ "at": "2026-10-05T23:35:49.110Z"+ },+ "data": {+ "turnId": "turn_0"+ }+ },+ {+ "type": "step.started",+ "meta": {+ "id": "evt_01M476RV3RZCCW92ZXEJWBW8WM",+ "at": "2026-10-05T23:35:49.112Z"+ },+ "data": {+ "turnId": "turn_0",+ "stepIndex": 0+ }+ },+ {+ "type": "actions.requested",+ "meta": {+ "id": "evt_01M476RV41G16Z7FSF37GAG635",+ "at": "2026-10-05T23:35:49.121Z"+ },+ "data": {+ "actions": [+ {+ "callId": "mock-tool-call-1-0-1",+ "kind": "tool-call"+ }+ ]+ }+ },+ {+ "type": "action.result",+ "meta": {+ "id": "evt_01M476RV4325FKVY9H3VTMBSEW",+ "at": "2026-10-05T23:35:49.123Z"+ },+ "data": {+ "result": {+ "callId": "mock-tool-call-1-0-1"+ },+ "status": "completed"+ }+ },+ {+ "type": "step.completed",+ "meta": {+ "id": "evt_01M476RV45DWXA28BBVBFY5HQF",+ "at": "2026-10-05T23:35:49.125Z"+ },+ "data": {+ "turnId": "turn_0",+ "stepIndex": 0,+ "usage": {+ "inputTokens": 344,+ "outputTokens": 4,+ "cacheReadTokens": 0,+ "cacheWriteTokens": 0+ },+ "finishReason": "tool-calls"+ }+ },+ {+ "type": "turn.completed",+ "meta": {+ "id": "evt_01M476RV4NAD1MEWDF4H2CBQWB",+ "at": "2026-10-05T23:35:49.141Z"+ },+ "data": {+ "turnId": "turn_0"+ }+ },+ {+ "type": "turn.started",+ "meta": {+ "id": "evt_01M476RVPD601TWZ4WEK2RYKCG",+ "at": "2026-10-05T23:35:49.709Z"+ },+ "data": {+ "turnId": "turn_1"+ }+ },+ {+ "type": "step.started",+ "meta": {+ "id": "evt_01M476RVPD601TWZ4WEK2RYKCJ",+ "at": "2026-10-05T23:35:49.709Z"+ },+ "data": {+ "turnId": "turn_1",+ "stepIndex": 0+ }+ },+ {+ "type": "actions.requested",+ "meta": {+ "id": "evt_01M476RVPFWRV23DNSPJERPZRX",+ "at": "2026-10-05T23:35:49.711Z"+ },+ "data": {+ "actions": [+ {+ "callId": "mock-tool-call-2-1-1",+ "kind": "tool-call"+ }+ ]+ }+ },+ {+ "type": "action.result",+ "meta": {+ "id": "evt_01M476RVPGVTS0HK0BT3DHYRNB",+ "at": "2026-10-05T23:35:49.712Z"+ },+ "data": {+ "result": {+ "callId": "mock-tool-call-2-1-1"+ },+ "status": "completed"+ }+ },+ {+ "type": "step.completed",+ "meta": {+ "id": "evt_01M476RVPGVTS0HK0BT3DHYRNC",+ "at": "2026-10-05T23:35:49.712Z"+ },+ "data": {+ "turnId": "turn_1",+ "stepIndex": 0,+ "usage": {+ "inputTokens": 382,+ "outputTokens": 5,+ "cacheReadTokens": 0,+ "cacheWriteTokens": 0+ },+ "finishReason": "tool-calls"+ }+ },+ {+ "type": "step.started",+ "meta": {+ "id": "evt_01M476RVRMAYSTN1F5WH1BDHGM",+ "at": "2026-10-05T23:35:49.780Z"+ },+ "data": {+ "turnId": "turn_1",+ "stepIndex": 1+ }+ },+ {+ "type": "message.appended",+ "meta": {+ "id": "evt_01M476RVRRACG4V1029E45SH2D",+ "at": "2026-10-05T23:35:49.784Z"+ },+ "data": {+ "turnId": "turn_1",+ "stepIndex": 1+ }+ },+ {+ "type": "step.completed",+ "meta": {+ "id": "evt_01M476RVRRACG4V1029E45SH2H",+ "at": "2026-10-05T23:35:49.784Z"+ },+ "data": {+ "turnId": "turn_1",+ "stepIndex": 1,+ "usage": {+ "inputTokens": 391,+ "outputTokens": 20,+ "cacheReadTokens": 0,+ "cacheWriteTokens": 0+ },+ "finishReason": "stop"+ }+ },+ {+ "type": "turn.completed",+ "meta": {+ "id": "evt_01M476RVS25WY95VK1961T35DH",+ "at": "2026-10-05T23:35:49.794Z"+ },+ "data": {+ "turnId": "turn_1"+ }+ }+ ]+ }+ ]+ }+}diff --git a/scripts/eval-metrics/gaps.mjs b/scripts/eval-metrics/gaps.mjsnew file mode 100644index 000000000..dd67ec0a1--- /dev/null+++ b/scripts/eval-metrics/gaps.mjs@@ -0,0 +1,70 @@+// Outcome classification for one eval attempt. A gap is an attempt that says+// nothing about the model or eve: it counts toward coverage but is excluded+// from pass rates and from latency and cost medians.++/**+ * Gap reasons, in classification priority order:+ *+ * - `job-missing`: The publish job found no artifact for a planned leg.+ * - `eval-missing`: A planned eval has no result in its leg's artifacts.+ * - `provider-unavailable`: The model provider or network failed: HTTP 429/5xx, or a network-level error.+ * - `stream-write-failed`: eve could not write its durable event stream.+ * - `timeout-before-first-step`: The eval timed out before any model step completed.+ * - `error-before-first-step`: The eval errored before any model step completed.+ */+/** @typedef {"completed" | "timed_out" | "skipped" | "parked" | "gap"} Outcome */++const FAILURE_EVENTS = new Set(["step.failed", "turn.failed", "session.failed"]);+const NETWORK_FAILURE =+ /\b(UND_ERR_(?:BODY|CONNECT|HEADERS)_TIMEOUT|UND_ERR_SOCKET|ECONNRESET|ECONNREFUSED|ETIMEDOUT|EAI_AGAIN|ENOTFOUND)\b/;+const TIMEOUT_ERROR = /aborted due to timeout|TimeoutError/i;++/**+ * @param {object} evalArtifact parsed eval detail JSON+ * @returns {{ outcome: Outcome, gap_reason?: string }}+ */+export function classifyOutcome(evalArtifact) {+ if (evalArtifact?.verdict === "skipped") return { outcome: "skipped" };++ const events = (evalArtifact?.result?.sessions ?? []).flatMap((session) => session.events ?? []);+ const failures = events.filter((event) => FAILURE_EVENTS.has(event.type));+ if (failures.some(isProviderFailure)) return gap("provider-unavailable");+ if (failures.some((event) => event.data?.code === "WORKFLOW_STREAM_WRITE_FAILED"))+ return gap("stream-write-failed");++ const completedSteps = events.filter((event) => event.type === "step.completed").length;+ const error = typeof evalArtifact?.error === "string" ? evalArtifact.error : undefined;+ const timedOut = error !== undefined && TIMEOUT_ERROR.test(error);+ // An eval that never completed a model step is a gap only when nothing in its+ // sessions failed for a real reason. Infrastructure failures returned above,+ // so any failure left (a first step rejected for its request, such as a+ // context too long, or a failed session) is the eval's own failure, even+ // though the eval then records an error.+ const realFailure = failures.length > 0;+ if (completedSteps === 0 && timedOut && !realFailure) return gap("timeout-before-first-step");+ if (completedSteps === 0 && error !== undefined && !realFailure)+ return gap("error-before-first-step");+ if (timedOut) return { outcome: "timed_out" };+ // The runner's own definition (eve's derive-run-facts), so the eval report+ // and the benchmark record agree on what parked means.+ if (evalArtifact?.result?.derived?.parked === true) return { outcome: "parked" };+ return { outcome: "completed" };+}++function gap(reason) {+ return { outcome: "gap", gap_reason: reason };+}++/**+ * Provider-side and network failures are gaps. Other model-call 4xx responses+ * (context too long, invalid request) and schema failures are real failures.+ */+export function isProviderFailure(event) {+ const { code, details } = event.data ?? {};+ if (code !== "MODEL_CALL_FAILED") return false;+ for (const status of [details?.statusCode, details?.upstreamStatusCode]) {+ if (typeof status === "number" && (status === 429 || status >= 500)) return true;+ }+ if (details?.semanticErrorId === "network-request-failed") return true;+ return NETWORK_FAILURE.test(JSON.stringify(details ?? {}) + (event.data?.message ?? ""));+}diff --git a/scripts/eval-metrics/intervals.test.mjs b/scripts/eval-metrics/intervals.test.mjsnew file mode 100644index 000000000..951732154--- /dev/null+++ b/scripts/eval-metrics/intervals.test.mjs@@ -0,0 +1,100 @@+import test from "node:test";+import assert from "node:assert/strict";+import {+ coveredLengthBetween,+ firstDeltaInStep,+ pairIntervals,+ unionIntervals,+} from "./measurements/intervals.mjs";++const event = (type, id, second, data = {}) => ({+ type,+ meta: { id, at: `2026-01-01T00:00:${String(second).padStart(2, "0")}Z` },+ data,+});+const actionKeys = {+ startKeys: (e) => e.data.actions?.map((action) => action.callId),+ endKeys: (e) => [e.data.result?.callId],+ subject: "action",+};+const requested = (id, second, ...callIds) =>+ event("actions.requested", id, second, { actions: callIds.map((callId) => ({ callId })) });+const result = (id, second, callId) => event("action.result", id, second, { result: { callId } });++test("pairs batched starts with out-of-order ends by key", () => {+ const paired = pairIntervals(+ [requested("batch", 1, "a", "b")],+ [result("b-end", 5, "b"), result("a-end", 3, "a")],+ actionKeys,+ );+ assert.equal(paired.status, "ready");+ assert.deepEqual(+ paired.intervals.map(({ key, startAt, endAt }) => [key, endAt - startAt]),+ [+ ["a", 2000],+ ["b", 4000],+ ],+ );+});++test("partial, ambiguous, and malformed pairings are unavailable", () => {+ for (const [starts, ends, reason] of [+ [[requested("r", 1, "a")], [], "action-incomplete"],+ [[], [result("e", 2, "a")], "action-start-missing"],+ [[requested("r", 1, "a"), requested("again", 2, "a")], [], "ambiguous-action"],+ [[requested("r", 1, "a")], [result("e1", 2, "a"), result("e2", 3, "a")], "ambiguous-action"],+ [[requested("r", 1)], [], "missing-action-identity"],+ [[requested("r", 3, "a")], [result("e", 1, "a")], "negative-duration"],+ [[requested("r", "xx", "a")], [result("e", 1, "a")], "missing-timestamp"],+ ])+ assert.deepEqual(pairIntervals(starts, ends, actionKeys), { status: "unavailable", reason });+});++test("repeated starts can resolve to the earliest announcement", () => {+ const keys = { ...actionKeys, repeatedStarts: "earliest" };+ const paired = pairIntervals(+ [requested("late", 4, "a"), requested("early", 2, "a")],+ [result("e", 6, "a")],+ keys,+ );+ assert.equal(paired.intervals[0].start.meta.id, "early");+});++test("unions overlapping intervals and measures coverage within and outside bounds", () => {+ const spans = [+ { startAt: 0, endAt: 4 },+ { startAt: 2, endAt: 6 },+ { startAt: 6, endAt: 7 },+ { startAt: 10, endAt: 12 },+ ];+ assert.deepEqual(unionIntervals(spans), [+ { startAt: 0, endAt: 7 },+ { startAt: 10, endAt: 12 },+ ]);+ assert.equal(coveredLengthBetween(spans), 9);+ assert.equal(coveredLengthBetween(spans, { within: [{ startAt: 3, endAt: 11 }] }), 5);+ assert.equal(+ coveredLengthBetween(spans, {+ within: [{ startAt: 3, endAt: 11 }],+ excluding: [{ startAt: 5, endAt: 10.5 }],+ }),+ 2.5,+ );+ assert.equal(coveredLengthBetween([]), 0);+});++test("finds the first text or reasoning delta of each step, ignoring other steps", () => {+ const step = (type, id, second, stepIndex, turnId = "t") =>+ event(type, id, second, { turnId, stepIndex });+ const steps = firstDeltaInStep([+ step("step.started", "s0", 1, 0),+ step("message.appended", "m-late", 4, 0),+ step("reasoning.appended", "r-early", 2, 0),+ step("step.started", "s1", 5, 1),+ step("action.input.appended", "tool-delta", 6, 1),+ step("message.appended", "other-turn", 3, 0, "other"),+ ]);+ assert.equal(steps.get("t/0").firstDelta.meta.id, "r-early");+ assert.equal(steps.get("t/1").firstDelta, undefined);+ assert.equal(steps.has("other/0"), false);+});diff --git a/scripts/eval-metrics/measurements.test.mjs b/scripts/eval-metrics/measurements.test.mjsnew file mode 100644index 000000000..5b6b41687--- /dev/null+++ b/scripts/eval-metrics/measurements.test.mjs@@ -0,0 +1,34 @@+import test from "node:test";+import assert from "node:assert/strict";+import { captureSessions } from "./measurements/events.mjs";++const event = (type, id, second, data = {}) => ({+ type,+ meta: { id, at: `2026-01-01T00:00:${String(second).padStart(2, "0")}Z` },+ data,+});++test("deduplicates repeated captures without losing later events", () => {+ const start = event("turn.started", "start", 0, { turnId: "turn" });+ const later = event("turn.completed", "completed", 1, { turnId: "turn" });+ const capture = captureSessions([+ { sessionId: "session", events: [start, structuredClone(start)] },+ { sessionId: "session", events: [structuredClone(start), later] },+ ]);+ assert.deepEqual(capture.bySession.get("session"), [start, later]);+});++test("rejects malformed or ambiguous capture identities", () => {+ const start = event("turn.started", "start", 0);+ const session = { sessionId: "session", events: [start] };+ for (const [sessions, message] of [+ [[{}], /Unsupported session capture/],+ [[{ ...session, events: [{ type: "turn.started" }] }], /Unsupported event capture/],+ [+ [session, { ...session, events: [{ ...start, data: { changed: true } }] }],+ /Conflicting event identity/,+ ],+ [[session, { ...session, sessionId: "other" }], /Ambiguous session evidence/],+ ])+ assert.throws(() => captureSessions(sessions), message);+});diff --git a/scripts/eval-metrics/measurements/events.mjs b/scripts/eval-metrics/measurements/events.mjsnew file mode 100644index 000000000..7ec1be18e--- /dev/null+++ b/scripts/eval-metrics/measurements/events.mjs@@ -0,0 +1,61 @@+/**+ * @typedef {{ status: "measured", value: number }+ * | { status: "unavailable" | "not-applicable", reason: string }} Measurement+ */++/** @param {string} reason @returns {{ status: "unavailable", reason: string }} */+export function unavailable(reason) {+ return { status: "unavailable", reason };+}++/** @param {string} reason @returns {{ status: "not-applicable", reason: string }} */+export function notApplicable(reason) {+ return { status: "not-applicable", reason };+}++/** Merge repeated session captures and validate event identities before measuring. */+export function captureSessions(sessions) {+ if (!Array.isArray(sessions)) return undefined;++ const bySession = new Map();+ const eventSessions = new Map();+ const identities = new Map();+ for (const session of sessions) {+ if (typeof session?.sessionId !== "string" || !Array.isArray(session.events))+ throw new Error("Unsupported session capture shape.");+ const events = bySession.get(session.sessionId) ?? [];+ for (const event of session.events) {+ if (!event || typeof event.type !== "string" || typeof event.meta?.id !== "string")+ throw new Error("Unsupported event capture shape or missing event identity.");+ const id = event.meta.id;+ const serialized = JSON.stringify(event);+ const earlier = identities.get(id);+ if (earlier !== undefined) {+ if (earlier !== serialized) throw new Error(`Conflicting event identity: ${id}`);+ if (eventSessions.get(id) !== session.sessionId)+ throw new Error(`Ambiguous session evidence for event identity: ${id}`);+ continue;+ }+ identities.set(id, serialized);+ eventSessions.set(id, session.sessionId);+ events.push(event);+ }+ bySession.set(session.sessionId, events);+ }++ return { bySession, events: [...bySession.values()].flat() };+}++/** Match an event type and exact data fields, preserving capture order. */+export function selectEvents(events, type, data = {}) {+ return events.filter(+ (event) =>+ event.type === type &&+ Object.entries(data).every(([key, value]) => event.data?.[key] === value),+ );+}++export function timestamp(event) {+ const value = Date.parse(event?.meta?.at ?? "");+ return Number.isFinite(value) ? value : undefined;+}diff --git a/scripts/eval-metrics/measurements/intervals.mjs b/scripts/eval-metrics/measurements/intervals.mjsnew file mode 100644index 000000000..6ab30842b--- /dev/null+++ b/scripts/eval-metrics/measurements/intervals.mjs@@ -0,0 +1,150 @@+import { selectEvents, timestamp, unavailable } from "./events.mjs";++/**+ * @typedef {{ key: string, start: object, end: object, startAt: number, endAt: number }} Interval+ * @typedef {{ status: "ready", intervals: Interval[] } | { status: "unavailable", reason: string }} Intervals+ */++/**+ * Pair start and end events by key, independent of capture order. One event+ * may open or close several keys (an `actions.requested` batch, an+ * `input.resolved` batch). Every start needs exactly one end and vice versa;+ * `repeatedStarts: "earliest"` tolerates re-announced starts such as repeated+ * approval candidates for one request.+ *+ * @param {object[]} starts+ * @param {object[]} ends+ * @param {{+ * startKeys: (event: object) => unknown[],+ * endKeys: (event: object) => unknown[],+ * subject: string,+ * repeatedStarts?: "ambiguous" | "earliest",+ * closeOpen?: (start: object) => object | undefined,+ * }} options `closeOpen` may supply an end for a start that has none, such as+ * the terminal event of a cancelled turn.+ * @returns {Intervals}+ */+export function pairIntervals(+ starts,+ ends,+ { startKeys, endKeys, subject, repeatedStarts, closeOpen },+) {+ const opened = indexByKey(starts, startKeys, repeatedStarts === "earliest");+ if (opened === undefined) return unavailable(`missing-${subject}-identity`);+ if (opened === "ambiguous") return unavailable(`ambiguous-${subject}`);+ const closed = indexByKey(ends, endKeys, false);+ if (closed === undefined) return unavailable(`missing-${subject}-identity`);+ if (closed === "ambiguous") return unavailable(`ambiguous-${subject}`);++ const intervals = [];+ for (const [key, start] of opened) {+ const end = closed.get(key) ?? closeOpen?.(start);+ if (end === undefined) return unavailable(`${subject}-incomplete`);+ const startAt = timestamp(start);+ const endAt = timestamp(end);+ if (startAt === undefined || endAt === undefined) return unavailable("missing-timestamp");+ if (endAt < startAt) return unavailable("negative-duration");+ intervals.push({ key, start, end, startAt, endAt });+ }+ if ([...closed.keys()].some((key) => !opened.has(key)))+ return unavailable(`${subject}-start-missing`);+ return { status: "ready", intervals };+}++function indexByKey(events, keysOf, keepEarliest) {+ const byKey = new Map();+ for (const event of events) {+ const keys = keysOf(event);+ if (!Array.isArray(keys) || keys.length === 0) return undefined;+ for (const key of keys) {+ if (typeof key !== "string" || key.length === 0) return undefined;+ const earlier = byKey.get(key);+ if (earlier === undefined) byKey.set(key, event);+ else if (!keepEarliest) return "ambiguous";+ else if ((timestamp(event) ?? Infinity) < (timestamp(earlier) ?? Infinity))+ byKey.set(key, event);+ }+ }+ return byKey;+}++/**+ * Merge overlapping or touching intervals into a sorted disjoint list.+ *+ * @param {readonly { startAt: number, endAt: number }[]} intervals+ * @returns {{ startAt: number, endAt: number }[]}+ */+export function unionIntervals(intervals) {+ const sorted = [...intervals].sort((a, b) => a.startAt - b.startAt);+ const merged = [];+ for (const { startAt, endAt } of sorted) {+ const last = merged.at(-1);+ if (last !== undefined && startAt <= last.endAt) last.endAt = Math.max(last.endAt, endAt);+ else merged.push({ startAt, endAt });+ }+ return merged;+}++/**+ * Covered length of `intervals` that lies inside `within` and outside `excluding`.+ *+ * @param {readonly { startAt: number, endAt: number }[]} intervals+ * @param {{ within?: readonly { startAt: number, endAt: number }[], excluding?: readonly { startAt: number, endAt: number }[] }} [bounds]+ */+export function coveredLengthBetween(intervals, { within, excluding = [] } = {}) {+ let covered = unionIntervals(intervals);+ if (within !== undefined) covered = intersect(covered, unionIntervals(within));+ const removed = intersect(covered, unionIntervals(excluding));+ return lengthOf(covered) - lengthOf(removed);+}++function intersect(left, right) {+ const result = [];+ let i = 0;+ let j = 0;+ while (i < left.length && j < right.length) {+ const startAt = Math.max(left[i].startAt, right[j].startAt);+ const endAt = Math.min(left[i].endAt, right[j].endAt);+ if (startAt < endAt) result.push({ startAt, endAt });+ if (left[i].endAt < right[j].endAt) i += 1;+ else j += 1;+ }+ return result;+}++function lengthOf(disjoint) {+ return disjoint.reduce((total, { startAt, endAt }) => total + endAt - startAt, 0);+}++/**+ * For each started model step, find the first streamed text or reasoning+ * delta of that step. Steps that stream no delta (e.g. tool-call only) map to+ * `undefined`; callers decide whether that is not-applicable.+ *+ * @param {object[]} events one session's events+ * @returns {Map<string, { start: object, firstDelta: object | undefined }>} keyed by `turnId/stepIndex`+ */+export function firstDeltaInStep(events) {+ const steps = new Map();+ for (const start of selectEvents(events, "step.started")) {+ const key = stepKey(start);+ if (key !== undefined) steps.set(key, { start, firstDelta: undefined });+ }+ for (const event of events) {+ if (event.type !== "message.appended" && event.type !== "reasoning.appended") continue;+ const step = steps.get(stepKey(event));+ if (step === undefined) continue;+ const at = timestamp(event);+ if (at === undefined) continue;+ if (step.firstDelta === undefined || at < timestamp(step.firstDelta)) step.firstDelta = event;+ }+ return steps;+}++/** Identity of one model step within a session. */+export function stepKey(event) {+ const { turnId, stepIndex } = event.data ?? {};+ return typeof turnId === "string" && Number.isInteger(stepIndex)+ ? `${turnId}/${stepIndex}`+ : undefined;+}diff --git a/scripts/eval-metrics/standard.mjs b/scripts/eval-metrics/standard.mjsnew file mode 100644index 000000000..f8c93e20d--- /dev/null+++ b/scripts/eval-metrics/standard.mjs@@ -0,0 +1,578 @@+// Standard per-attempt metric set derived from one eval detail artifact+// (`.eve/evals/<ts>/evals/<id>.json`). STANDARD_METRICS below lists every+// metric with its unit and better direction.+//+// Scope: latency, attribution, step-level cost, and efficiency counts read the+// eval's root sessions (sessions the eval opened, not delegated children).+// Reliability counts read every captured session, because a failure anywhere+// matters.+//+// Cost and tokens: `cost.total_usd` and `tokens.*` (but `tokens.peak_context`)+// are the runner's `result.derived.usage`, the same totals the eval reporters+// see: each captured session's latest usage, including delegated work and+// compaction summaries, less sessions another captured session opened.+// `cost.primary_usd` sums the root sessions' model steps; `cost.subagent_usd`+// is the difference, and is unavailable when a root session compacted, since+// compaction cost is in the total but in no step.+//+// Attribution partitions root turn time with a fixed precedence, so the parts+// add up to the turn total: human wait, then task (delegated work, including+// time a `task_wait` call blocks on it), tool (any other requested action),+// compaction, model step, and the remainder is framework time. In-process+// tools run inside their model step, so precedence keeps model ms to model work.+import { existsSync, readdirSync, readFileSync } from "node:fs";+import { join } from "node:path";+import { fileURLToPath } from "node:url";++import {+ captureSessions,+ notApplicable,+ selectEvents,+ timestamp,+ unavailable,+} from "./measurements/events.mjs";+import {+ coveredLengthBetween,+ firstDeltaInStep,+ pairIntervals,+ stepKey,+} from "./measurements/intervals.mjs";++/** Bump whenever a definition below changes meaning. */+export const METRICS_VERSION = 1;++/** Every metric in the standard set, with its unit and better direction. */+export const STANDARD_METRICS = Object.freeze({+ "latency.eval_wall_ms": { unit: "ms", direction: "lower" },+ "latency.turn_ms": { unit: "ms", direction: "lower" },+ "latency.turn_ms_mean": { unit: "ms", direction: "lower" },+ "latency.ttft_ms_p50": { unit: "ms", direction: "lower" },+ "latency.output_tokens_per_s": { unit: "tokens/s", direction: "higher" },+ "attribution.model_ms": { unit: "ms", direction: "lower" },+ "attribution.tool_ms": { unit: "ms", direction: "lower" },+ "attribution.task_ms": { unit: "ms", direction: "lower" },+ "attribution.human_wait_ms": { unit: "ms", direction: "neutral" },+ "attribution.compaction_ms": { unit: "ms", direction: "lower" },+ "attribution.framework_ms": { unit: "ms", direction: "lower" },+ "cost.total_usd": { unit: "usd", direction: "lower" },+ "cost.primary_usd": { unit: "usd", direction: "lower" },+ "cost.subagent_usd": { unit: "usd", direction: "lower" },+ "tokens.input": { unit: "tokens", direction: "lower" },+ "tokens.output": { unit: "tokens", direction: "lower" },+ "tokens.cache_read": { unit: "tokens", direction: "neutral" },+ "tokens.cache_write": { unit: "tokens", direction: "neutral" },+ "tokens.cache_hit_ratio": { unit: "ratio", direction: "higher" },+ "tokens.peak_context": { unit: "tokens", direction: "lower" },+ "counts.turns": { unit: "count", direction: "neutral" },+ "counts.steps": { unit: "count", direction: "lower" },+ "counts.steps_per_turn": { unit: "ratio", direction: "lower" },+ "counts.tool_calls": { unit: "count", direction: "neutral" },+ "counts.tool_calls_per_step": { unit: "ratio", direction: "neutral" },+ "counts.subagent_fan_out": { unit: "count", direction: "neutral" },+ "counts.subagent_depth": { unit: "count", direction: "neutral" },+ "counts.compactions": { unit: "count", direction: "lower" },+ "counts.context_clears": { unit: "count", direction: "neutral" },+ "reliability.step_failures": { unit: "count", direction: "lower" },+ "reliability.turn_failures": { unit: "count", direction: "lower" },+ "reliability.session_failures": { unit: "count", direction: "lower" },+ "reliability.tool_errors": { unit: "count", direction: "lower" },+ "reliability.tool_error_rate": { unit: "ratio", direction: "lower" },+ "reliability.truncations": { unit: "count", direction: "lower" },+});++const TURN_TERMINALS = ["turn.completed", "turn.failed", "turn.cancelled"];+const FAILURE_EVENTS = ["step.failed", "turn.failed", "session.failed"];+const TOOL_ACTION_KINDS = new Set(["tool-call", "workflow-tool-call"]);++/**+ * @param {object} evalArtifact parsed eval detail JSON+ * @returns {{+ * metrics_version: number,+ * metrics: Record<string, import("./measurements/events.mjs").Measurement>,+ * failures: { type: string, code: string, count: number }[],+ * }}+ */+export function deriveAttemptMetrics(evalArtifact) {+ const sessions = evalArtifact?.result?.sessions;+ let capture;+ try {+ capture = captureSessions(+ Array.isArray(sessions)+ ? sessions.filter((session) => typeof session?.sessionId === "string")+ : undefined,+ );+ } catch {+ return everyMetric(unavailable("invalid-session-capture"));+ }+ if (capture === undefined) return everyMetric(unavailable("missing-session-capture"));+ if (capture.bySession.size === 0) return everyMetric(notApplicable("no-sessions"));++ const rootIds = rootSessionIds(capture);+ if (rootIds.length === 0) return everyMetric(unavailable("missing-root-session"));+ const roots = rootIds.map((sessionId) => ({+ events: capture.bySession.get(sessionId),+ timeline: sessionTimeline(capture, sessionId),+ }));+ const allEvents = capture.events;++ const metrics = {+ ...latencyAndAttribution(evalArtifact, roots),+ ...streaming(roots),+ ...costAndTokens(evalArtifact, roots),+ ...efficiency(capture, roots, rootIds),+ ...reliability(allEvents),+ };+ return { metrics_version: METRICS_VERSION, metrics, failures: failureCodes(allEvents) };+}++function everyMetric(measurement) {+ return {+ metrics_version: METRICS_VERSION,+ metrics: Object.fromEntries(Object.keys(STANDARD_METRICS).map((key) => [key, measurement])),+ failures: [],+ };+}++function measured(value) {+ return { status: "measured", value };+}++/** Sessions the eval opened itself: neither linked by `agent.started` nor invoked as a subagent. */+function rootSessionIds(capture) {+ const children = new Set(+ selectEvents(capture.events, "agent.started").map((event) => event.data?.sessionId),+ );+ for (const [sessionId, events] of capture.bySession) {+ if (+ selectEvents(events, "session.started").some((e) => e.data?.invocation?.kind === "subagent")+ )+ children.add(sessionId);+ }+ return [...capture.bySession.keys()].filter((sessionId) => !children.has(sessionId));+}++/** Pair every interval family of one session, clipping nothing yet. */+function sessionTimeline(capture, sessionId) {+ const events = capture.bySession.get(sessionId);+ const turnStarts = selectEvents(events, "turn.started");+ const turnKey = (e) => [e.data?.turnId];+ // A root session with no turns has no turn time to attribute; without this,+ // every latency and attribution part would read as a measured zero.+ const turns =+ turnStarts.length === 0+ ? unavailable("missing-turn")+ : pairIntervals(+ turnStarts,+ TURN_TERMINALS.flatMap((type) => selectEvents(events, type)),+ { startKeys: turnKey, endKeys: turnKey, subject: "turn" },+ );+ const terminals = new Map(+ turns.status === "ready" ? turns.intervals.map((turn) => [turn.key, turn.end]) : [],+ );+ // Work a turn abandons (a cancelled turn, a step cut off by a session+ // limit) ends with that turn.+ const closeOpen = (start) => terminals.get(start.data?.turnId);++ const steps = pairIntervals(+ selectEvents(events, "step.started"),+ [...selectEvents(events, "step.completed"), ...selectEvents(events, "step.failed")],+ { startKeys: (e) => [stepKey(e)], endKeys: (e) => [stepKey(e)], subject: "step", closeOpen },+ );+ const actions = pairIntervals(+ selectEvents(events, "actions.requested"),+ selectEvents(events, "action.result"),+ {+ startKeys: (e) =>+ Array.isArray(e.data?.actions) ? e.data.actions.map((a) => a?.callId) : [],+ endKeys: (e) => [e.data?.result?.callId],+ subject: "action",+ repeatedStarts: "earliest",+ closeOpen,+ },+ );+ const taskKey = (e) => [taskIdentity(e)];+ const tasks = pairIntervals(+ selectEvents(events, "task.started"),+ selectEvents(events, "task.settled"),+ { startKeys: taskKey, endKeys: taskKey, subject: "task", closeOpen },+ );+ const compactions = pairIntervals(+ selectEvents(events, "compaction.requested"),+ selectEvents(events, "compaction.completed"),+ {+ startKeys: (e) => [stepKey(e)],+ endKeys: (e) => [stepKey(e)],+ subject: "compaction",+ closeOpen,+ },+ );+ return {+ events,+ turns,+ steps,+ actions,+ tasks,+ compactions,+ human: humanWait(events, closeOpen),+ };+}++function taskIdentity(event) {+ const { taskId, callId } = event.data ?? {};+ return typeof taskId === "string" && typeof callId === "string"+ ? `${taskId}/${callId}`+ : undefined;+}++/** Input requests, responder-bound approvals, and sign-ins, from request to resolution. */+function humanWait(events, closeOpen) {+ const requestIds = (list) => (Array.isArray(list) ? list.map((entry) => entry?.requestId) : []);+ const settledResolutions = (event) =>+ (event.data?.resolutions ?? []).filter((resolution) => resolution?.outcome !== "invalid");+ const input = pairIntervals(+ selectEvents(events, "input.requested"),+ selectEvents(events, "input.resolved").filter((e) => settledResolutions(e).length > 0),+ {+ startKeys: (e) => requestIds(e.data?.requests),+ endKeys: (e) => requestIds(settledResolutions(e)),+ subject: "input",+ closeOpen,+ },+ );++ // `approval.settled` also closes approvals delivered only through+ // `input.requested`; only responder-bound candidates open an interval here.+ const candidates = selectEvents(events, "approval.candidate", { outcome: "pending" });+ const candidateIds = new Set(candidates.map((e) => e.data?.requestId));+ const approval = pairIntervals(+ candidates,+ selectEvents(events, "approval.settled").filter((e) => candidateIds.has(e.data?.requestId)),+ {+ startKeys: (e) => [e.data?.requestId],+ endKeys: (e) => [e.data?.requestId],+ subject: "approval",+ repeatedStarts: "earliest",+ closeOpen,+ },+ );++ // `attemptId` is not present on every `authorization.required`, so key by+ // the turn and connection name, which both events always carry.+ const authorizationKey = (e) =>+ typeof e.data?.turnId === "string" && typeof e.data?.name === "string"+ ? [`${e.data.turnId}/${e.data.name}`]+ : [];+ const authorization = pairIntervals(+ selectEvents(events, "authorization.required"),+ selectEvents(events, "authorization.completed"),+ {+ startKeys: authorizationKey,+ endKeys: authorizationKey,+ subject: "authorization",+ repeatedStarts: "earliest",+ closeOpen,+ },+ );++ for (const family of [input, approval, authorization])+ if (family.status !== "ready") return family;+ return {+ status: "ready",+ intervals: [...input.intervals, ...approval.intervals, ...authorization.intervals],+ };+}++function latencyAndAttribution(evalArtifact, roots) {+ const parts = {+ turn: [],+ human: [],+ tool: [],+ task: [],+ compaction: [],+ model: [],+ framework: [],+ };+ let failure;+ let turnCount = 0;+ const humanIntervals = [];+ for (const { timeline } of roots) {+ const families = ["turns", "human", "actions", "tasks", "compactions", "steps"];+ const broken = families.map((name) => timeline[name]).find((f) => f.status !== "ready");+ if (broken !== undefined) {+ failure ??= broken;+ continue;+ }+ const T = timeline.turns.intervals;+ const H = timeline.human.intervals;+ const A = timeline.actions.intervals;+ const K = timeline.tasks.intervals;+ const C = timeline.compactions.intervals;+ const M = timeline.steps.intervals;+ const total = coveredLengthBetween(T);+ const human = coveredLengthBetween(H, { within: T });+ parts.turn.push(total - human);+ parts.human.push(human);+ parts.task.push(coveredLengthBetween(K, { within: T, excluding: H }));+ parts.tool.push(coveredLengthBetween(A, { within: T, excluding: [...H, ...K] }));+ parts.compaction.push(coveredLengthBetween(C, { within: T, excluding: [...H, ...A, ...K] }));+ parts.model.push(coveredLengthBetween(M, { within: T, excluding: [...H, ...A, ...K, ...C] }));+ parts.framework.push(+ total - coveredLengthBetween([...H, ...A, ...K, ...C, ...M], { within: T }),+ );+ turnCount += T.length;+ humanIntervals.push(...H);+ }++ const sum = (values) => (failure ? failure : measured(values.reduce((a, b) => a + b, 0)));+ return {+ "latency.eval_wall_ms": failure ?? evalWall(evalArtifact, humanIntervals),+ "latency.turn_ms": sum(parts.turn),+ "latency.turn_ms_mean":+ failure ??+ (turnCount === 0+ ? notApplicable("no-turns")+ : measured(parts.turn.reduce((a, b) => a + b, 0) / turnCount)),+ "attribution.model_ms": sum(parts.model),+ "attribution.tool_ms": sum(parts.tool),+ "attribution.task_ms": sum(parts.task),+ "attribution.human_wait_ms": sum(parts.human),+ "attribution.compaction_ms": sum(parts.compaction),+ "attribution.framework_ms": sum(parts.framework),+ };+}++function evalWall(evalArtifact, humanIntervals) {+ const startAt = Date.parse(evalArtifact?.startedAt ?? "");+ const endAt = Date.parse(evalArtifact?.completedAt ?? "");+ if (!Number.isFinite(startAt) || !Number.isFinite(endAt))+ return unavailable("missing-eval-timing");+ if (endAt < startAt) return unavailable("negative-duration");+ const human = coveredLengthBetween(humanIntervals, { within: [{ startAt, endAt }] });+ return measured(endAt - startAt - human);+}++function streaming(roots) {+ const ttfts = [];+ let tokens = 0;+ let streamMs = 0;+ let missingTokens = false;+ for (const { events } of roots) {+ const completions = new Map(+ selectEvents(events, "step.completed").map((event) => [stepKey(event), event]),+ );+ for (const [key, { start, firstDelta }] of firstDeltaInStep(events)) {+ const startAt = timestamp(start);+ if (firstDelta === undefined || startAt === undefined) continue;+ const deltaAt = timestamp(firstDelta);+ ttfts.push(deltaAt - startAt);+ const completed = completions.get(key);+ if (completed === undefined) continue;+ const outputTokens = completed.data?.usage?.outputTokens;+ if (typeof outputTokens !== "number") missingTokens = true;+ else tokens += outputTokens;+ streamMs += (timestamp(completed) ?? deltaAt) - deltaAt;+ }+ }+ if (ttfts.length === 0) {+ const none = notApplicable("no-streamed-steps");+ return { "latency.ttft_ms_p50": none, "latency.output_tokens_per_s": none };+ }+ return {+ "latency.ttft_ms_p50": measured(median(ttfts)),+ "latency.output_tokens_per_s": missingTokens+ ? unavailable("missing-output-tokens")+ : streamMs <= 0+ ? unavailable("zero-stream-duration")+ : measured((tokens / streamMs) * 1000),+ };+}++function median(values) {+ const sorted = [...values].sort((a, b) => a - b);+ const middle = Math.floor(sorted.length / 2);+ return sorted.length % 2 === 1 ? sorted[middle] : (sorted[middle - 1] + sorted[middle]) / 2;+}++function costAndTokens(evalArtifact, roots) {+ const steps = roots.flatMap(({ events }) => selectEvents(events, "step.completed"));+ // Absent when a counted session reported no usage; `costUsd` is absent when+ // one reported no cost (eve's evalUsage).+ const usage = evalArtifact?.result?.derived?.usage;++ let primary;+ if (steps.length === 0) primary = notApplicable("no-model-steps");+ else if (steps.some((step) => !isAmount(step.data?.usage?.costUsd)))+ primary = unavailable("missing-step-cost");+ else primary = measured(steps.reduce((total, step) => total + step.data.usage.costUsd, 0));++ const rolled = (field, reason) =>+ isAmount(usage?.[field]) ? measured(usage[field]) : unavailable(reason);+ const total = rolled("costUsd", "missing-rolled-up-cost");++ // Compaction summary calls add their cost to the session's rolled-up usage,+ // but no step event reports it, so total − steps would count it as+ // subagent spend. Until compaction events carry usage, the split is unknown.+ const compacted = roots.some(+ ({ events }) => selectEvents(events, "compaction.completed").length > 0,+ );+ let subagent;+ if (total.status !== "measured") subagent = total;+ else if (compacted) subagent = unavailable("compaction-cost-unattributed");+ else if (primary.status === "not-applicable") subagent = total;+ else if (primary.status !== "measured") subagent = primary;+ else {+ const difference = total.value - primary.value;+ // Float sums of identical costs can differ in the last bits.+ subagent =+ difference < -1e-9+ ? unavailable("inconsistent-cost-rollup")+ : measured(Math.max(0, difference));+ }++ const input = rolled("inputTokens", "missing-rolled-up-usage");+ const cacheRead = rolled("cacheReadTokens", "missing-rolled-up-usage");+ let cacheHitRatio;+ if (input.status !== "measured") cacheHitRatio = input;+ else if (cacheRead.status !== "measured") cacheHitRatio = cacheRead;+ else if (input.value === 0) cacheHitRatio = notApplicable("no-input-tokens");+ else cacheHitRatio = measured(cacheRead.value / input.value);++ let peak;+ if (steps.length === 0) peak = notApplicable("no-model-steps");+ else if (steps.some((step) => !isAmount(step.data?.usage?.inputTokens)))+ peak = unavailable("missing-step-input-tokens");+ else peak = measured(Math.max(...steps.map((step) => step.data.usage.inputTokens)));++ return {+ "cost.total_usd": total,+ "cost.primary_usd": primary,+ "cost.subagent_usd": subagent,+ "tokens.input": input,+ "tokens.output": rolled("outputTokens", "missing-rolled-up-usage"),+ "tokens.cache_read": cacheRead,+ "tokens.cache_write": rolled("cacheWriteTokens", "missing-rolled-up-usage"),+ "tokens.cache_hit_ratio": cacheHitRatio,+ "tokens.peak_context": peak,+ };+}++function isAmount(value) {+ return typeof value === "number" && Number.isFinite(value) && value >= 0;+}++function efficiency(capture, roots, rootIds) {+ const rootEvents = roots.flatMap(({ events }) => events);+ const turns = selectEvents(rootEvents, "turn.started").length;+ const steps = selectEvents(rootEvents, "step.started").length;+ const toolCalls = new Set();+ for (const [index, { events }] of roots.entries()) {+ for (const event of selectEvents(events, "actions.requested")) {+ for (const action of event.data?.actions ?? []) {+ if (TOOL_ACTION_KINDS.has(action?.kind))+ toolCalls.add(`${rootIds[index]}\0${action.callId}`);+ }+ }+ }+ const ratio = (numerator, denominator, reason) =>+ denominator === 0 ? notApplicable(reason) : measured(numerator / denominator);+ const delegated = new Set(+ selectEvents(rootEvents, "agent.started").map((event) => event.data?.sessionId),+ );+ return {+ "counts.turns": measured(turns),+ "counts.steps": measured(steps),+ "counts.steps_per_turn": ratio(steps, turns, "no-turns"),+ "counts.tool_calls": measured(toolCalls.size),+ "counts.tool_calls_per_step": ratio(toolCalls.size, steps, "no-model-steps"),+ "counts.subagent_fan_out": measured(delegated.size),+ "counts.subagent_depth": measured(+ Math.max(...rootIds.map((id) => delegationDepth(capture, id))),+ ),+ "counts.compactions": measured(selectEvents(rootEvents, "compaction.completed").length),+ "counts.context_clears": measured(selectEvents(rootEvents, "context.cleared").length),+ };+}++/** Longest `agent.started` chain below a session; uncaptured children count as leaves. */+function delegationDepth(capture, sessionId, visiting = new Set()) {+ if (visiting.has(sessionId)) return 0;+ const events = capture.bySession.get(sessionId) ?? [];+ const children = new Set(selectEvents(events, "agent.started").map((e) => e.data?.sessionId));+ let depth = 0;+ visiting.add(sessionId);+ for (const child of children)+ depth = Math.max(depth, 1 + delegationDepth(capture, child, visiting));+ visiting.delete(sessionId);+ return depth;+}++function reliability(events) {+ const results = selectEvents(events, "action.result");+ const toolErrors = results.filter((event) => event.data?.status === "failed").length;+ return {+ "reliability.step_failures": measured(selectEvents(events, "step.failed").length),+ "reliability.turn_failures": measured(selectEvents(events, "turn.failed").length),+ "reliability.session_failures": measured(selectEvents(events, "session.failed").length),+ "reliability.tool_errors": measured(toolErrors),+ "reliability.tool_error_rate":+ results.length === 0+ ? notApplicable("no-action-results")+ : measured(toolErrors / results.length),+ "reliability.truncations": measured(+ selectEvents(events, "step.completed", { finishReason: "length" }).length,+ ),+ };+}++function failureCodes(events) {+ const counts = new Map();+ for (const event of events) {+ if (!FAILURE_EVENTS.includes(event.type)) continue;+ const key = `${event.type}\0${typeof event.data?.code === "string" ? event.data.code : "UNKNOWN"}`;+ counts.set(key, (counts.get(key) ?? 0) + 1);+ }+ return [...counts]+ .map(([key, count]) => {+ const [type, code] = key.split("\0");+ return { type, code, count };+ })+ .sort((a, b) => a.type.localeCompare(b.type) || a.code.localeCompare(b.code));+}++/**+ * Every eval detail artifact under one `.eve/evals/<timestamp>` tree, sorted.+ * The record builder and this file's CLI share it, so they agree on which+ * artifacts exist.+ */+export function listEvalArtifacts(runDirectory) {+ const evalsDirectory = join(runDirectory, "evals");+ if (!existsSync(evalsDirectory)) return [];+ return readdirSync(evalsDirectory, { withFileTypes: true, recursive: true })+ .filter((entry) => entry.isFile() && entry.name.endsWith(".json"))+ .map((entry) => join(entry.parentPath, entry.name))+ .sort();+}++// `node scripts/eval-metrics/standard.mjs <.eve/evals/<timestamp>>` prints each+// eval's outcome and metrics, then per-metric coverage across the run.+if (process.argv[1] === fileURLToPath(import.meta.url)) {+ const runDirectory = process.argv[2];+ if (runDirectory === undefined) {+ console.error("Usage: node scripts/eval-metrics/standard.mjs <.eve/evals/<timestamp>>");+ process.exit(1);+ }+ const { classifyOutcome } = await import("./gaps.mjs");+ const coverage = new Map(Object.keys(STANDARD_METRICS).map((key) => [key, 0]));+ const artifacts = listEvalArtifacts(runDirectory);+ for (const path of artifacts) {+ const artifact = JSON.parse(readFileSync(path, "utf8"));+ const { metrics } = deriveAttemptMetrics(artifact);+ console.log(`${artifact.id}: ${JSON.stringify(classifyOutcome(artifact))}`);+ for (const [key, m] of Object.entries(metrics)) {+ if (m.status === "measured") coverage.set(key, coverage.get(key) + 1);+ console.log(` ${key} ${m.status === "measured" ? m.value : `${m.status} (${m.reason})`}`);+ }+ }+ console.log(`\nCoverage over ${artifacts.length} evals:`);+ for (const [key, count] of coverage) console.log(` ${key} ${count}/${artifacts.length}`);+}diff --git a/scripts/eval-metrics/standard.test.mjs b/scripts/eval-metrics/standard.test.mjsnew file mode 100644index 000000000..d8e298993--- /dev/null+++ b/scripts/eval-metrics/standard.test.mjs@@ -0,0 +1,390 @@+import test from "node:test";+import assert from "node:assert/strict";+import { readFileSync } from "node:fs";+import { classifyOutcome } from "./gaps.mjs";+import { METRICS_VERSION, STANDARD_METRICS, deriveAttemptMetrics } from "./standard.mjs";++// Fixtures are trimmed `eve eval` detail artifacts captured from mock-model+// runs of e2e/fixtures/agent-tools and agent-subagents (mock models report+// no cost, so cost cases inject it below).+const fixture = (name) =>+ JSON.parse(readFileSync(new URL(`./fixtures/${name}.json`, import.meta.url), "utf8"));+const values = (metrics) =>+ Object.fromEntries(+ Object.entries(metrics).map(([key, m]) => [key, m.status === "measured" ? m.value : m.status]),+ );+const at = (ms) => new Date(Date.UTC(2026, 0, 1) + ms).toISOString();+const event = (type, ms, data = {}) => ({+ type,+ meta: { id: `${type}@${ms}:${JSON.stringify(data)}`, at: at(ms) },+ data,+});++const synthetic = (events, extra = {}) => ({+ id: "synthetic",+ verdict: "passed",+ startedAt: at(0),+ completedAt: at(1000),+ result: {+ sessionId: "s",+ status: "completed",+ sessions: [{ sessionId: "s", primary: true, events }],+ },+ ...extra,+});+const turn = (start, end, inner, terminal = "turn.completed") => [+ event("turn.started", start, { turnId: "t" }),+ ...inner,+ event(terminal, end, { turnId: "t" }),+];+const compactedArtifact = () => {+ const artifact = synthetic([+ ...turn(0, 100, [+ event("step.started", 10, { turnId: "t", stepIndex: 0 }),+ event("step.completed", 90, {+ turnId: "t",+ stepIndex: 0,+ usage: { inputTokens: 10, outputTokens: 2 },+ }),+ ]),+ event("compaction.requested", 150, { turnId: "t", stepIndex: 1 }),+ event("compaction.completed", 200, { turnId: "t", stepIndex: 1 }),+ ]);+ artifact.result.derived = { usage: { inputTokens: 10, outputTokens: 2 } };+ return artifact;+};++function assertPartition(metrics) {+ const v = values(metrics);+ const parts =+ v["attribution.model_ms"] ++ v["attribution.tool_ms"] ++ v["attribution.task_ms"] ++ v["attribution.compaction_ms"] ++ v["attribution.framework_ms"];+ assert.equal(parts, v["latency.turn_ms"], "attribution must add up to non-human turn time");+}++test("derives representative metrics from a captured tool eval", () => {+ const { metrics_version, metrics, failures } = deriveAttemptMetrics(+ fixture("agent-tools-ends-turn-function"),+ );+ assert.equal(metrics_version, METRICS_VERSION);+ assert.deepEqual(Object.keys(metrics).sort(), Object.keys(STANDARD_METRICS).sort());+ assert.deepEqual(failures, []);+ const v = values(metrics);+ assert.equal(v["latency.eval_wall_ms"], 1050);+ assert.equal(v["latency.turn_ms"], 116);+ assert.equal(v["latency.ttft_ms_p50"], 4);+ assert.equal(v["attribution.model_ms"], 17);+ assert.equal(v["attribution.tool_ms"], 3);+ assert.equal(v["attribution.framework_ms"], 96);+ assert.equal(v["tokens.input"], 1117);+ assert.equal(v["tokens.peak_context"], 391);+ assert.equal(v["counts.turns"], 2);+ assert.equal(v["counts.steps"], 3);+ assert.equal(v["counts.tool_calls"], 2);+ // Mock models report no cost: missing cost is unavailable, never zero.+ assert.equal(metrics["cost.primary_usd"].reason, "missing-step-cost");+ assert.equal(metrics["latency.output_tokens_per_s"].reason, "zero-stream-duration");+ assertPartition(metrics);+});++test("attributes delegated work to tasks across repeated captures of one session", () => {+ const { metrics } = deriveAttemptMetrics(fixture("agent-subagents-continuation-local"));+ const v = values(metrics);+ assert.equal(v["attribution.task_ms"], 2465);+ assert.equal(v["attribution.tool_ms"], 133);+ assert.equal(v["counts.turns"], 3, "child sessions are not root turns");+ assert.equal(v["counts.subagent_fan_out"], 1);+ assert.equal(v["counts.subagent_depth"], 1);+ // Rolled-up usage includes the child's spend.+ assert.equal(v["tokens.input"], 6006);+ assertPartition(metrics);+});++test("excludes sign-in wait from latency and counts it as human wait", () => {+ const artifact = synthetic(+ turn(0, 1000, [+ event("authorization.required", 100, { turnId: "t", name: "account" }),+ event("authorization.completed", 274, { turnId: "t", name: "account" }),+ ]),+ );+ artifact.completedAt = at(1072);+ const { metrics } = deriveAttemptMetrics(artifact);+ const v = values(metrics);+ assert.equal(v["attribution.human_wait_ms"], 174);+ assert.equal(v["latency.eval_wall_ms"], 1072 - 174);+ assertPartition(metrics);+});++test("compaction outside a turn does not count toward turn time", () => {+ const { metrics } = deriveAttemptMetrics(compactedArtifact());+ assert.equal(values(metrics)["counts.compactions"], 1);+ assert.equal(values(metrics)["attribution.compaction_ms"], 0);+ assertPartition(metrics);+});++function withCosts(artifact, { stepCost, rolledUpCost }) {+ const copy = structuredClone(artifact);+ // The runner's eval total (result.derived.usage) carries the rolled-up cost.+ copy.result.derived.usage.costUsd = rolledUpCost;+ for (const session of copy.result.sessions) {+ for (const e of session.events) {+ if (session.sessionId !== copy.result.sessionId) continue;+ if (e.type === "step.completed") e.data.usage.costUsd = stepCost;+ if (["session.waiting", "turn.waiting"].includes(e.type)) e.data.usage.costUsd = rolledUpCost;+ }+ }+ return copy;+}++test("splits cost into primary steps and rolled-up subagent spend", () => {+ const artifact = withCosts(fixture("agent-subagents-continuation-local"), {+ stepCost: 0.25,+ rolledUpCost: 4,+ });+ const v = values(deriveAttemptMetrics(artifact).metrics);+ assert.equal(v["cost.primary_usd"], 2.5, "ten primary steps");+ assert.equal(v["cost.total_usd"], 4);+ assert.equal(v["cost.subagent_usd"], 1.5);+});++test("cost and tokens are the runner's eval usage, and unavailable without it", () => {+ const artifact = fixture("agent-tools-ends-turn-function");+ artifact.result.derived.usage = {+ cacheReadTokens: 40,+ cacheWriteTokens: 5,+ inputTokens: 200,+ outputTokens: 12,+ costUsd: 0.75,+ };+ const v = values(deriveAttemptMetrics(artifact).metrics);+ assert.equal(v["cost.total_usd"], 0.75);+ assert.equal(v["tokens.input"], 200);+ assert.equal(v["tokens.output"], 12);+ assert.equal(v["tokens.cache_read"], 40);+ assert.equal(v["tokens.cache_write"], 5);+ assert.equal(v["tokens.cache_hit_ratio"], 0.2);+ // Peak context is per step, from the events, not the eval total.+ assert.equal(v["tokens.peak_context"], 391);++ // A counted session reported no cost: the runner leaves costUsd out.+ delete artifact.result.derived.usage.costUsd;+ const noCost = deriveAttemptMetrics(artifact).metrics;+ assert.deepEqual(noCost["cost.total_usd"], {+ status: "unavailable",+ reason: "missing-rolled-up-cost",+ });+ assert.equal(noCost["tokens.input"].status, "measured");++ // A counted session reported no usage at all: the runner leaves usage out.+ delete artifact.result.derived.usage;+ const none = deriveAttemptMetrics(artifact).metrics;+ for (const key of ["cost.total_usd", "tokens.input", "tokens.output", "tokens.cache_hit_ratio"])+ assert.equal(none[key].status, "unavailable", key);+ assert.equal(none["tokens.peak_context"].status, "measured");+});++test("compaction cost stays in the total but leaves the subagent split unknown", () => {+ // Compaction summary calls are paid and roll up into session usage, but no+ // step reports them: total − steps would misreport them as subagent spend.+ const artifact = withCosts(compactedArtifact(), {+ stepCost: 0.25,+ rolledUpCost: 2,+ });+ const { metrics } = deriveAttemptMetrics(artifact);+ assert.equal(values(metrics)["cost.total_usd"], 2);+ assert.equal(metrics["cost.primary_usd"].status, "measured");+ assert.deepEqual(metrics["cost.subagent_usd"], {+ status: "unavailable",+ reason: "compaction-cost-unattributed",+ });+});++test("a single step without cost makes cost unavailable, and inconsistent rollups are flagged", () => {+ const artifact = withCosts(fixture("agent-tools-ends-turn-function"), {+ stepCost: 1,+ rolledUpCost: 3,+ });+ assert.equal(values(deriveAttemptMetrics(artifact).metrics)["cost.subagent_usd"], 0);++ const inconsistent = withCosts(artifact, { stepCost: 1, rolledUpCost: 2 });+ assert.equal(+ deriveAttemptMetrics(inconsistent).metrics["cost.subagent_usd"].reason,+ "inconsistent-cost-rollup",+ );++ const step = artifact.result.sessions[0].events.find((e) => e.type === "step.completed");+ delete step.data.usage.costUsd;+ const { metrics } = deriveAttemptMetrics(artifact);+ assert.equal(metrics["cost.primary_usd"].reason, "missing-step-cost");+ assert.equal(metrics["cost.subagent_usd"].reason, "missing-step-cost");+ assert.equal(metrics["cost.total_usd"].status, "measured");+});++test("overlapping parallel tools count once and streaming metrics use first deltas", () => {+ const step = { turnId: "t", stepIndex: 0 };+ const events = turn(0, 1000, [+ event("step.started", 100, step),+ event("reasoning.appended", 150, step),+ event("message.appended", 300, step),+ event("actions.requested", 400, {+ ...step,+ actions: [+ { kind: "tool-call", callId: "a" },+ { kind: "tool-call", callId: "b" },+ ],+ }),+ event("action.result", 700, { ...step, result: { callId: "b" }, status: "failed" }),+ event("action.result", 600, { ...step, result: { callId: "a" }, status: "completed" }),+ event("step.completed", 750, { ...step, usage: { outputTokens: 60, inputTokens: 900 } }),+ ]);+ const v = values(deriveAttemptMetrics(synthetic(events)).metrics);+ assert.equal(v["attribution.tool_ms"], 300);+ assert.equal(v["attribution.model_ms"], 350);+ assert.equal(v["attribution.framework_ms"], 350);+ assert.equal(v["latency.ttft_ms_p50"], 50);+ assert.equal(v["latency.output_tokens_per_s"], 100);+ assert.equal(v["reliability.tool_error_rate"], 0.5);+ assert.equal(v["counts.tool_calls_per_step"], 2);+});++test("partial captures make attribution unavailable instead of guessing", () => {+ const openTurn = [+ event("turn.started", 0, { turnId: "t" }),+ event("step.started", 10, { turnId: "t", stepIndex: 0 }),+ ];+ assert.equal(+ deriveAttemptMetrics(synthetic(openTurn)).metrics["attribution.framework_ms"].reason,+ "turn-incomplete",+ );+ const orphanResult = turn(0, 100, [event("action.result", 50, { result: { callId: "x" } })]);+ assert.equal(+ deriveAttemptMetrics(synthetic(orphanResult)).metrics["attribution.tool_ms"].reason,+ "action-start-missing",+ );+ const openInput = turn(0, 100, [+ event("input.requested", 50, { turnId: "unknown", requests: [{ requestId: "r" }] }),+ ]);+ assert.equal(+ deriveAttemptMetrics(synthetic(openInput)).metrics["attribution.human_wait_ms"].reason,+ "input-incomplete",+ );+ const noCapture = { ...synthetic([]), result: { status: "completed" } };+ assert.equal(+ deriveAttemptMetrics(noCapture).metrics["latency.turn_ms"].reason,+ "missing-session-capture",+ );+ const conflicting = synthetic([event("turn.started", 0, { turnId: "t" })]);+ conflicting.result.sessions.push({+ sessionId: "s",+ events: [{ ...conflicting.result.sessions[0].events[0], data: { turnId: "other" } }],+ });+ assert.equal(+ deriveAttemptMetrics(conflicting).metrics["counts.turns"].reason,+ "invalid-session-capture",+ );+});++test("a cancelled turn closes work it abandoned", () => {+ const events = turn(+ 0,+ 500,+ [+ event("step.started", 10, { turnId: "t", stepIndex: 0 }),+ event("actions.requested", 100, {+ turnId: "t",+ stepIndex: 0,+ actions: [{ kind: "tool-call", callId: "a" }],+ }),+ ],+ "turn.cancelled",+ );+ const v = values(deriveAttemptMetrics(synthetic(events)).metrics);+ assert.equal(v["attribution.tool_ms"], 400);+ assert.equal(v["attribution.model_ms"], 90);+});++test("counts failures by event and code", () => {+ const events = turn(+ 0,+ 100,+ [+ event("step.started", 10, { turnId: "t", stepIndex: 0 }),+ event("step.failed", 20, { turnId: "t", stepIndex: 0, code: "MODEL_CALL_FAILED" }),+ event("session.failed", 99, { code: "MODEL_CALL_FAILED" }),+ ],+ "turn.failed",+ );+ events.at(-1).data.code = "MODEL_CALL_FAILED";+ const { failures, metrics } = deriveAttemptMetrics(synthetic(events));+ assert.deepEqual(failures, [+ { type: "session.failed", code: "MODEL_CALL_FAILED", count: 1 },+ { type: "step.failed", code: "MODEL_CALL_FAILED", count: 1 },+ { type: "turn.failed", code: "MODEL_CALL_FAILED", count: 1 },+ ]);+ assert.equal(values(metrics)["reliability.step_failures"], 1);+});++test("classifies outcomes and every gap reason", () => {+ const failed = (code, details, message = "failed") =>+ synthetic(+ turn(0, 100, [+ event("step.failed", 50, { turnId: "t", stepIndex: 0, code, details, message }),+ ]),+ );+ const completedStep = event("step.completed", 50, { turnId: "t", stepIndex: 0 });++ assert.deepEqual(classifyOutcome(fixture("agent-tools-ends-turn-function")), {+ outcome: "completed",+ });+ assert.deepEqual(classifyOutcome({ ...synthetic([]), verdict: "skipped" }), {+ outcome: "skipped",+ });+ for (const details of [+ { statusCode: 429 },+ { upstreamStatusCode: 503 },+ { semanticErrorId: "network-request-failed" },+ { detail: "TypeError: fetch failed [cause]: UND_ERR_HEADERS_TIMEOUT" },+ ])+ assert.deepEqual(classifyOutcome(failed("MODEL_CALL_FAILED", details)), {+ outcome: "gap",+ gap_reason: "provider-unavailable",+ });+ for (const details of [{ statusCode: 400 }, { statusCode: 413 }])+ assert.equal(classifyOutcome(failed("MODEL_CALL_FAILED", details)).outcome, "completed");+ assert.equal(classifyOutcome(failed("OUTPUT_SCHEMA_NOT_FULFILLED", {})).outcome, "completed");+ assert.equal(+ classifyOutcome(failed("WORKFLOW_STREAM_WRITE_FAILED", {})).gap_reason,+ "stream-write-failed",+ );++ const timeout = "The operation was aborted due to timeout";+ assert.equal(+ classifyOutcome(synthetic([], { error: timeout })).gap_reason,+ "timeout-before-first-step",+ );+ assert.equal(+ classifyOutcome(synthetic([], { error: "connect ECONNREFUSED" })).gap_reason,+ "error-before-first-step",+ );+ assert.deepEqual(classifyOutcome(synthetic([completedStep], { error: timeout })), {+ outcome: "timed_out",+ });+ // A first step that fails for a real reason is the eval's failure, not a gap,+ // even though the eval then records an error (e.g. from expectOk()).+ for (const details of [{ statusCode: 400 }, { statusCode: 413 }]) {+ const rejected = { ...failed("MODEL_CALL_FAILED", details), error: "expectOk: session failed" };+ assert.deepEqual(classifyOutcome(rejected), { outcome: "completed" });+ assert.deepEqual(classifyOutcome({ ...rejected, error: timeout }), { outcome: "timed_out" });+ }+ const sessionFailed = synthetic([event("session.failed", 40, { code: "SESSION_FAILED" })], {+ error: "expectOk: session failed",+ });+ assert.deepEqual(classifyOutcome(sessionFailed), { outcome: "completed" });+ // Parked is the runner's derived fact (eve's derive-run-facts), not re-derived here.+ const parked = synthetic([completedStep]);+ parked.result.derived = { parked: true };+ assert.deepEqual(classifyOutcome(parked), { outcome: "parked" });+});