Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

23.1 KB
Raw Blame
// Standard per-attempt metric set derived from one eval detail artifact
// (`.eve/evals/<ts>/evals/<id>.json`). STANDARD_METRICS below lists every
// metric with its unit and better direction.
//
// Scope: latency, attribution, step-level cost, and efficiency counts read the
// eval's root sessions (sessions the eval opened, not delegated children).
// Reliability counts read every captured session, because a failure anywhere
// matters.
//
// Cost and tokens: `cost.total_usd` and `tokens.*` (but `tokens.peak_context`)
// are the runner's `result.derived.usage`, the same totals the eval reporters
// see: each captured session's latest usage, including delegated work and
// compaction summaries, less sessions another captured session opened.
// `cost.primary_usd` sums the root sessions' model steps; `cost.subagent_usd`
// is the difference, and is unavailable when a root session compacted, since
// compaction cost is in the total but in no step.
//
// Attribution partitions root turn time with a fixed precedence, so the parts
// add up to the turn total: human wait, then task (delegated work, including
// time a `task_wait` call blocks on it), tool (any other requested action),
// compaction, model step, and the remainder is framework time. In-process
// tools run inside their model step, so precedence keeps model ms to model work.
import { existsSync, readdirSync, readFileSync } from "node:fs";
import { join } from "node:path";
import { fileURLToPath } from "node:url";

import {
  captureSessions,
  notApplicable,
  selectEvents,
  timestamp,
  unavailable,
} from "./measurements/events.mjs";
import {
  coveredLengthBetween,
  firstDeltaInStep,
  pairIntervals,
  stepKey,
} from "./measurements/intervals.mjs";

/** Bump whenever a definition below changes meaning. */
export const METRICS_VERSION = 1;

/** Every metric in the standard set, with its unit and better direction. */
export const STANDARD_METRICS = Object.freeze({
  "latency.eval_wall_ms": { unit: "ms", direction: "lower" },
  "latency.turn_ms": { unit: "ms", direction: "lower" },
  "latency.turn_ms_mean": { unit: "ms", direction: "lower" },
  "latency.ttft_ms_p50": { unit: "ms", direction: "lower" },
  "latency.output_tokens_per_s": { unit: "tokens/s", direction: "higher" },
  "attribution.model_ms": { unit: "ms", direction: "lower" },
  "attribution.tool_ms": { unit: "ms", direction: "lower" },
  "attribution.task_ms": { unit: "ms", direction: "lower" },
  "attribution.human_wait_ms": { unit: "ms", direction: "neutral" },
  "attribution.compaction_ms": { unit: "ms", direction: "lower" },
  "attribution.framework_ms": { unit: "ms", direction: "lower" },
  "cost.total_usd": { unit: "usd", direction: "lower" },
  "cost.primary_usd": { unit: "usd", direction: "lower" },
  "cost.subagent_usd": { unit: "usd", direction: "lower" },
  "tokens.input": { unit: "tokens", direction: "lower" },
  "tokens.output": { unit: "tokens", direction: "lower" },
  "tokens.cache_read": { unit: "tokens", direction: "neutral" },
  "tokens.cache_write": { unit: "tokens", direction: "neutral" },
  "tokens.cache_hit_ratio": { unit: "ratio", direction: "higher" },
  "tokens.peak_context": { unit: "tokens", direction: "lower" },
  "counts.turns": { unit: "count", direction: "neutral" },
  "counts.steps": { unit: "count", direction: "lower" },
  "counts.steps_per_turn": { unit: "ratio", direction: "lower" },
  "counts.tool_calls": { unit: "count", direction: "neutral" },
  "counts.tool_calls_per_step": { unit: "ratio", direction: "neutral" },
  "counts.subagent_fan_out": { unit: "count", direction: "neutral" },
  "counts.subagent_depth": { unit: "count", direction: "neutral" },
  "counts.compactions": { unit: "count", direction: "lower" },
  "counts.context_clears": { unit: "count", direction: "neutral" },
  "reliability.step_failures": { unit: "count", direction: "lower" },
  "reliability.turn_failures": { unit: "count", direction: "lower" },
  "reliability.session_failures": { unit: "count", direction: "lower" },
  "reliability.tool_errors": { unit: "count", direction: "lower" },
  "reliability.tool_error_rate": { unit: "ratio", direction: "lower" },
  "reliability.truncations": { unit: "count", direction: "lower" },
});

const TURN_TERMINALS = ["turn.completed", "turn.failed", "turn.cancelled"];
const FAILURE_EVENTS = ["step.failed", "turn.failed", "session.failed"];
const TOOL_ACTION_KINDS = new Set(["tool-call", "workflow-tool-call"]);

/**
 * @param {object} evalArtifact parsed eval detail JSON
 * @returns {{
 *   metrics_version: number,
 *   metrics: Record<string, import("./measurements/events.mjs").Measurement>,
 *   failures: { type: string, code: string, count: number }[],
 * }}
 */
export function deriveAttemptMetrics(evalArtifact) {
  const sessions = evalArtifact?.result?.sessions;
  let capture;
  try {
    capture = captureSessions(
      Array.isArray(sessions)
        ? sessions.filter((session) => typeof session?.sessionId === "string")
        : undefined,
    );
  } catch {
    return everyMetric(unavailable("invalid-session-capture"));
  }
  if (capture === undefined) return everyMetric(unavailable("missing-session-capture"));
  if (capture.bySession.size === 0) return everyMetric(notApplicable("no-sessions"));

  const rootIds = rootSessionIds(capture);
  if (rootIds.length === 0) return everyMetric(unavailable("missing-root-session"));
  const roots = rootIds.map((sessionId) => ({
    events: capture.bySession.get(sessionId),
    timeline: sessionTimeline(capture, sessionId),
  }));
  const allEvents = capture.events;

  const metrics = {
    ...latencyAndAttribution(evalArtifact, roots),
    ...streaming(roots),
    ...costAndTokens(evalArtifact, roots),
    ...efficiency(capture, roots, rootIds),
    ...reliability(allEvents),
  };
  return { metrics_version: METRICS_VERSION, metrics, failures: failureCodes(allEvents) };
}

function everyMetric(measurement) {
  return {
    metrics_version: METRICS_VERSION,
    metrics: Object.fromEntries(Object.keys(STANDARD_METRICS).map((key) => [key, measurement])),
    failures: [],
  };
}

function measured(value) {
  return { status: "measured", value };
}

/** Sessions the eval opened itself: neither linked by `agent.started` nor invoked as a subagent. */
function rootSessionIds(capture) {
  const children = new Set(
    selectEvents(capture.events, "agent.started").map((event) => event.data?.sessionId),
  );
  for (const [sessionId, events] of capture.bySession) {
    if (
      selectEvents(events, "session.started").some((e) => e.data?.invocation?.kind === "subagent")
    )
      children.add(sessionId);
  }
  return [...capture.bySession.keys()].filter((sessionId) => !children.has(sessionId));
}

/** Pair every interval family of one session, clipping nothing yet. */
function sessionTimeline(capture, sessionId) {
  const events = capture.bySession.get(sessionId);
  const turnStarts = selectEvents(events, "turn.started");
  const turnKey = (e) => [e.data?.turnId];
  // A root session with no turns has no turn time to attribute; without this,
  // every latency and attribution part would read as a measured zero.
  const turns =
    turnStarts.length === 0
      ? unavailable("missing-turn")
      : pairIntervals(
          turnStarts,
          TURN_TERMINALS.flatMap((type) => selectEvents(events, type)),
          { startKeys: turnKey, endKeys: turnKey, subject: "turn" },
        );
  const terminals = new Map(
    turns.status === "ready" ? turns.intervals.map((turn) => [turn.key, turn.end]) : [],
  );
  // Work a turn abandons (a cancelled turn, a step cut off by a session
  // limit) ends with that turn.
  const closeOpen = (start) => terminals.get(start.data?.turnId);

  const steps = pairIntervals(
    selectEvents(events, "step.started"),
    [...selectEvents(events, "step.completed"), ...selectEvents(events, "step.failed")],
    { startKeys: (e) => [stepKey(e)], endKeys: (e) => [stepKey(e)], subject: "step", closeOpen },
  );
  const actions = pairIntervals(
    selectEvents(events, "actions.requested"),
    selectEvents(events, "action.result"),
    {
      startKeys: (e) =>
        Array.isArray(e.data?.actions) ? e.data.actions.map((a) => a?.callId) : [],
      endKeys: (e) => [e.data?.result?.callId],
      subject: "action",
      repeatedStarts: "earliest",
      closeOpen,
    },
  );
  const taskKey = (e) => [taskIdentity(e)];
  const tasks = pairIntervals(
    selectEvents(events, "task.started"),
    selectEvents(events, "task.settled"),
    { startKeys: taskKey, endKeys: taskKey, subject: "task", closeOpen },
  );
  const compactions = pairIntervals(
    selectEvents(events, "compaction.requested"),
    selectEvents(events, "compaction.completed"),
    {
      startKeys: (e) => [stepKey(e)],
      endKeys: (e) => [stepKey(e)],
      subject: "compaction",
      closeOpen,
    },
  );
  return {
    events,
    turns,
    steps,
    actions,
    tasks,
    compactions,
    human: humanWait(events, closeOpen),
  };
}

function taskIdentity(event) {
  const { taskId, callId } = event.data ?? {};
  return typeof taskId === "string" && typeof callId === "string"
    ? `${taskId}/${callId}`
    : undefined;
}

/** Input requests, responder-bound approvals, and sign-ins, from request to resolution. */
function humanWait(events, closeOpen) {
  const requestIds = (list) => (Array.isArray(list) ? list.map((entry) => entry?.requestId) : []);
  const settledResolutions = (event) =>
    (event.data?.resolutions ?? []).filter((resolution) => resolution?.outcome !== "invalid");
  const input = pairIntervals(
    selectEvents(events, "input.requested"),
    selectEvents(events, "input.resolved").filter((e) => settledResolutions(e).length > 0),
    {
      startKeys: (e) => requestIds(e.data?.requests),
      endKeys: (e) => requestIds(settledResolutions(e)),
      subject: "input",
      closeOpen,
    },
  );

  // `approval.settled` also closes approvals delivered only through
  // `input.requested`; only responder-bound candidates open an interval here.
  const candidates = selectEvents(events, "approval.candidate", { outcome: "pending" });
  const candidateIds = new Set(candidates.map((e) => e.data?.requestId));
  const approval = pairIntervals(
    candidates,
    selectEvents(events, "approval.settled").filter((e) => candidateIds.has(e.data?.requestId)),
    {
      startKeys: (e) => [e.data?.requestId],
      endKeys: (e) => [e.data?.requestId],
      subject: "approval",
      repeatedStarts: "earliest",
      closeOpen,
    },
  );

  // `attemptId` is not present on every `authorization.required`, so key by
  // the turn and connection name, which both events always carry.
  const authorizationKey = (e) =>
    typeof e.data?.turnId === "string" && typeof e.data?.name === "string"
      ? [`${e.data.turnId}/${e.data.name}`]
      : [];
  const authorization = pairIntervals(
    selectEvents(events, "authorization.required"),
    selectEvents(events, "authorization.completed"),
    {
      startKeys: authorizationKey,
      endKeys: authorizationKey,
      subject: "authorization",
      repeatedStarts: "earliest",
      closeOpen,
    },
  );

  for (const family of [input, approval, authorization])
    if (family.status !== "ready") return family;
  return {
    status: "ready",
    intervals: [...input.intervals, ...approval.intervals, ...authorization.intervals],
  };
}

function latencyAndAttribution(evalArtifact, roots) {
  const parts = {
    turn: [],
    human: [],
    tool: [],
    task: [],
    compaction: [],
    model: [],
    framework: [],
  };
  let failure;
  let turnCount = 0;
  const humanIntervals = [];
  for (const { timeline } of roots) {
    const families = ["turns", "human", "actions", "tasks", "compactions", "steps"];
    const broken = families.map((name) => timeline[name]).find((f) => f.status !== "ready");
    if (broken !== undefined) {
      failure ??= broken;
      continue;
    }
    const T = timeline.turns.intervals;
    const H = timeline.human.intervals;
    const A = timeline.actions.intervals;
    const K = timeline.tasks.intervals;
    const C = timeline.compactions.intervals;
    const M = timeline.steps.intervals;
    const total = coveredLengthBetween(T);
    const human = coveredLengthBetween(H, { within: T });
    parts.turn.push(total - human);
    parts.human.push(human);
    parts.task.push(coveredLengthBetween(K, { within: T, excluding: H }));
    parts.tool.push(coveredLengthBetween(A, { within: T, excluding: [...H, ...K] }));
    parts.compaction.push(coveredLengthBetween(C, { within: T, excluding: [...H, ...A, ...K] }));
    parts.model.push(coveredLengthBetween(M, { within: T, excluding: [...H, ...A, ...K, ...C] }));
    parts.framework.push(
      total - coveredLengthBetween([...H, ...A, ...K, ...C, ...M], { within: T }),
    );
    turnCount += T.length;
    humanIntervals.push(...H);
  }

  const sum = (values) => (failure ? failure : measured(values.reduce((a, b) => a + b, 0)));
  return {
    "latency.eval_wall_ms": failure ?? evalWall(evalArtifact, humanIntervals),
    "latency.turn_ms": sum(parts.turn),
    "latency.turn_ms_mean":
      failure ??
      (turnCount === 0
        ? notApplicable("no-turns")
        : measured(parts.turn.reduce((a, b) => a + b, 0) / turnCount)),
    "attribution.model_ms": sum(parts.model),
    "attribution.tool_ms": sum(parts.tool),
    "attribution.task_ms": sum(parts.task),
    "attribution.human_wait_ms": sum(parts.human),
    "attribution.compaction_ms": sum(parts.compaction),
    "attribution.framework_ms": sum(parts.framework),
  };
}

function evalWall(evalArtifact, humanIntervals) {
  const startAt = Date.parse(evalArtifact?.startedAt ?? "");
  const endAt = Date.parse(evalArtifact?.completedAt ?? "");
  if (!Number.isFinite(startAt) || !Number.isFinite(endAt))
    return unavailable("missing-eval-timing");
  if (endAt < startAt) return unavailable("negative-duration");
  const human = coveredLengthBetween(humanIntervals, { within: [{ startAt, endAt }] });
  return measured(endAt - startAt - human);
}

function streaming(roots) {
  const ttfts = [];
  let tokens = 0;
  let streamMs = 0;
  let missingTokens = false;
  for (const { events } of roots) {
    const completions = new Map(
      selectEvents(events, "step.completed").map((event) => [stepKey(event), event]),
    );
    for (const [key, { start, firstDelta }] of firstDeltaInStep(events)) {
      const startAt = timestamp(start);
      if (firstDelta === undefined || startAt === undefined) continue;
      const deltaAt = timestamp(firstDelta);
      ttfts.push(deltaAt - startAt);
      const completed = completions.get(key);
      if (completed === undefined) continue;
      const outputTokens = completed.data?.usage?.outputTokens;
      if (typeof outputTokens !== "number") missingTokens = true;
      else tokens += outputTokens;
      streamMs += (timestamp(completed) ?? deltaAt) - deltaAt;
    }
  }
  if (ttfts.length === 0) {
    const none = notApplicable("no-streamed-steps");
    return { "latency.ttft_ms_p50": none, "latency.output_tokens_per_s": none };
  }
  return {
    "latency.ttft_ms_p50": measured(median(ttfts)),
    "latency.output_tokens_per_s": missingTokens
      ? unavailable("missing-output-tokens")
      : streamMs <= 0
        ? unavailable("zero-stream-duration")
        : measured((tokens / streamMs) * 1000),
  };
}

function median(values) {
  const sorted = [...values].sort((a, b) => a - b);
  const middle = Math.floor(sorted.length / 2);
  return sorted.length % 2 === 1 ? sorted[middle] : (sorted[middle - 1] + sorted[middle]) / 2;
}

function costAndTokens(evalArtifact, roots) {
  const steps = roots.flatMap(({ events }) => selectEvents(events, "step.completed"));
  // Absent when a counted session reported no usage; `costUsd` is absent when
  // one reported no cost (eve's evalUsage).
  const usage = evalArtifact?.result?.derived?.usage;

  let primary;
  if (steps.length === 0) primary = notApplicable("no-model-steps");
  else if (steps.some((step) => !isAmount(step.data?.usage?.costUsd)))
    primary = unavailable("missing-step-cost");
  else primary = measured(steps.reduce((total, step) => total + step.data.usage.costUsd, 0));

  const rolled = (field, reason) =>
    isAmount(usage?.[field]) ? measured(usage[field]) : unavailable(reason);
  const total = rolled("costUsd", "missing-rolled-up-cost");

  // Compaction summary calls add their cost to the session's rolled-up usage,
  // but no step event reports it, so total − steps would count it as
  // subagent spend. Until compaction events carry usage, the split is unknown.
  const compacted = roots.some(
    ({ events }) => selectEvents(events, "compaction.completed").length > 0,
  );
  let subagent;
  if (total.status !== "measured") subagent = total;
  else if (compacted) subagent = unavailable("compaction-cost-unattributed");
  else if (primary.status === "not-applicable") subagent = total;
  else if (primary.status !== "measured") subagent = primary;
  else {
    const difference = total.value - primary.value;
    // Float sums of identical costs can differ in the last bits.
    subagent =
      difference < -1e-9
        ? unavailable("inconsistent-cost-rollup")
        : measured(Math.max(0, difference));
  }

  const input = rolled("inputTokens", "missing-rolled-up-usage");
  const cacheRead = rolled("cacheReadTokens", "missing-rolled-up-usage");
  let cacheHitRatio;
  if (input.status !== "measured") cacheHitRatio = input;
  else if (cacheRead.status !== "measured") cacheHitRatio = cacheRead;
  else if (input.value === 0) cacheHitRatio = notApplicable("no-input-tokens");
  else cacheHitRatio = measured(cacheRead.value / input.value);

  let peak;
  if (steps.length === 0) peak = notApplicable("no-model-steps");
  else if (steps.some((step) => !isAmount(step.data?.usage?.inputTokens)))
    peak = unavailable("missing-step-input-tokens");
  else peak = measured(Math.max(...steps.map((step) => step.data.usage.inputTokens)));

  return {
    "cost.total_usd": total,
    "cost.primary_usd": primary,
    "cost.subagent_usd": subagent,
    "tokens.input": input,
    "tokens.output": rolled("outputTokens", "missing-rolled-up-usage"),
    "tokens.cache_read": cacheRead,
    "tokens.cache_write": rolled("cacheWriteTokens", "missing-rolled-up-usage"),
    "tokens.cache_hit_ratio": cacheHitRatio,
    "tokens.peak_context": peak,
  };
}

function isAmount(value) {
  return typeof value === "number" && Number.isFinite(value) && value >= 0;
}

function efficiency(capture, roots, rootIds) {
  const rootEvents = roots.flatMap(({ events }) => events);
  const turns = selectEvents(rootEvents, "turn.started").length;
  const steps = selectEvents(rootEvents, "step.started").length;
  const toolCalls = new Set();
  for (const [index, { events }] of roots.entries()) {
    for (const event of selectEvents(events, "actions.requested")) {
      for (const action of event.data?.actions ?? []) {
        if (TOOL_ACTION_KINDS.has(action?.kind))
          toolCalls.add(`${rootIds[index]}\0${action.callId}`);
      }
    }
  }
  const ratio = (numerator, denominator, reason) =>
    denominator === 0 ? notApplicable(reason) : measured(numerator / denominator);
  const delegated = new Set(
    selectEvents(rootEvents, "agent.started").map((event) => event.data?.sessionId),
  );
  return {
    "counts.turns": measured(turns),
    "counts.steps": measured(steps),
    "counts.steps_per_turn": ratio(steps, turns, "no-turns"),
    "counts.tool_calls": measured(toolCalls.size),
    "counts.tool_calls_per_step": ratio(toolCalls.size, steps, "no-model-steps"),
    "counts.subagent_fan_out": measured(delegated.size),
    "counts.subagent_depth": measured(
      Math.max(...rootIds.map((id) => delegationDepth(capture, id))),
    ),
    "counts.compactions": measured(selectEvents(rootEvents, "compaction.completed").length),
    "counts.context_clears": measured(selectEvents(rootEvents, "context.cleared").length),
  };
}

/** Longest `agent.started` chain below a session; uncaptured children count as leaves. */
function delegationDepth(capture, sessionId, visiting = new Set()) {
  if (visiting.has(sessionId)) return 0;
  const events = capture.bySession.get(sessionId) ?? [];
  const children = new Set(selectEvents(events, "agent.started").map((e) => e.data?.sessionId));
  let depth = 0;
  visiting.add(sessionId);
  for (const child of children)
    depth = Math.max(depth, 1 + delegationDepth(capture, child, visiting));
  visiting.delete(sessionId);
  return depth;
}

function reliability(events) {
  const results = selectEvents(events, "action.result");
  const toolErrors = results.filter((event) => event.data?.status === "failed").length;
  return {
    "reliability.step_failures": measured(selectEvents(events, "step.failed").length),
    "reliability.turn_failures": measured(selectEvents(events, "turn.failed").length),
    "reliability.session_failures": measured(selectEvents(events, "session.failed").length),
    "reliability.tool_errors": measured(toolErrors),
    "reliability.tool_error_rate":
      results.length === 0
        ? notApplicable("no-action-results")
        : measured(toolErrors / results.length),
    "reliability.truncations": measured(
      selectEvents(events, "step.completed", { finishReason: "length" }).length,
    ),
  };
}

function failureCodes(events) {
  const counts = new Map();
  for (const event of events) {
    if (!FAILURE_EVENTS.includes(event.type)) continue;
    const key = `${event.type}\0${typeof event.data?.code === "string" ? event.data.code : "UNKNOWN"}`;
    counts.set(key, (counts.get(key) ?? 0) + 1);
  }
  return [...counts]
    .map(([key, count]) => {
      const [type, code] = key.split("\0");
      return { type, code, count };
    })
    .sort((a, b) => a.type.localeCompare(b.type) || a.code.localeCompare(b.code));
}

/**
 * Every eval detail artifact under one `.eve/evals/<timestamp>` tree, sorted.
 * The record builder and this file's CLI share it, so they agree on which
 * artifacts exist.
 */
export function listEvalArtifacts(runDirectory) {
  const evalsDirectory = join(runDirectory, "evals");
  if (!existsSync(evalsDirectory)) return [];
  return readdirSync(evalsDirectory, { withFileTypes: true, recursive: true })
    .filter((entry) => entry.isFile() && entry.name.endsWith(".json"))
    .map((entry) => join(entry.parentPath, entry.name))
    .sort();
}

// `node scripts/eval-metrics/standard.mjs <.eve/evals/<timestamp>>` prints each
// eval's outcome and metrics, then per-metric coverage across the run.
if (process.argv[1] === fileURLToPath(import.meta.url)) {
  const runDirectory = process.argv[2];
  if (runDirectory === undefined) {
    console.error("Usage: node scripts/eval-metrics/standard.mjs <.eve/evals/<timestamp>>");
    process.exit(1);
  }
  const { classifyOutcome } = await import("./gaps.mjs");
  const coverage = new Map(Object.keys(STANDARD_METRICS).map((key) => [key, 0]));
  const artifacts = listEvalArtifacts(runDirectory);
  for (const path of artifacts) {
    const artifact = JSON.parse(readFileSync(path, "utf8"));
    const { metrics } = deriveAttemptMetrics(artifact);
    console.log(`${artifact.id}: ${JSON.stringify(classifyOutcome(artifact))}`);
    for (const [key, m] of Object.entries(metrics)) {
      if (m.status === "measured") coverage.set(key, coverage.get(key) + 1);
      console.log(`  ${key} ${m.status === "measured" ? m.value : `${m.status} (${m.reason})`}`);
    }
  }
  console.log(`\nCoverage over ${artifacts.length} evals:`);
  for (const [key, count] of coverage) console.log(`  ${key} ${count}/${artifacts.length}`);
}