Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

15.1 KB
Raw Blame
import test from "node:test";
import assert from "node:assert/strict";
import { readFileSync } from "node:fs";
import { classifyOutcome } from "./gaps.mjs";
import { METRICS_VERSION, STANDARD_METRICS, deriveAttemptMetrics } from "./standard.mjs";

// Fixtures are trimmed `eve eval` detail artifacts captured from mock-model
// runs of e2e/fixtures/agent-tools and agent-subagents (mock models report
// no cost, so cost cases inject it below).
const fixture = (name) =>
  JSON.parse(readFileSync(new URL(`./fixtures/${name}.json`, import.meta.url), "utf8"));
const values = (metrics) =>
  Object.fromEntries(
    Object.entries(metrics).map(([key, m]) => [key, m.status === "measured" ? m.value : m.status]),
  );
const at = (ms) => new Date(Date.UTC(2026, 0, 1) + ms).toISOString();
const event = (type, ms, data = {}) => ({
  type,
  meta: { id: `${type}@${ms}:${JSON.stringify(data)}`, at: at(ms) },
  data,
});

const synthetic = (events, extra = {}) => ({
  id: "synthetic",
  verdict: "passed",
  startedAt: at(0),
  completedAt: at(1000),
  result: {
    sessionId: "s",
    status: "completed",
    sessions: [{ sessionId: "s", primary: true, events }],
  },
  ...extra,
});
const turn = (start, end, inner, terminal = "turn.completed") => [
  event("turn.started", start, { turnId: "t" }),
  ...inner,
  event(terminal, end, { turnId: "t" }),
];
const compactedArtifact = () => {
  const artifact = synthetic([
    ...turn(0, 100, [
      event("step.started", 10, { turnId: "t", stepIndex: 0 }),
      event("step.completed", 90, {
        turnId: "t",
        stepIndex: 0,
        usage: { inputTokens: 10, outputTokens: 2 },
      }),
    ]),
    event("compaction.requested", 150, { turnId: "t", stepIndex: 1 }),
    event("compaction.completed", 200, { turnId: "t", stepIndex: 1 }),
  ]);
  artifact.result.derived = { usage: { inputTokens: 10, outputTokens: 2 } };
  return artifact;
};

function assertPartition(metrics) {
  const v = values(metrics);
  const parts =
    v["attribution.model_ms"] +
    v["attribution.tool_ms"] +
    v["attribution.task_ms"] +
    v["attribution.compaction_ms"] +
    v["attribution.framework_ms"];
  assert.equal(parts, v["latency.turn_ms"], "attribution must add up to non-human turn time");
}

test("derives representative metrics from a captured tool eval", () => {
  const { metrics_version, metrics, failures } = deriveAttemptMetrics(
    fixture("agent-tools-ends-turn-function"),
  );
  assert.equal(metrics_version, METRICS_VERSION);
  assert.deepEqual(Object.keys(metrics).sort(), Object.keys(STANDARD_METRICS).sort());
  assert.deepEqual(failures, []);
  const v = values(metrics);
  assert.equal(v["latency.eval_wall_ms"], 1050);
  assert.equal(v["latency.turn_ms"], 116);
  assert.equal(v["latency.ttft_ms_p50"], 4);
  assert.equal(v["attribution.model_ms"], 17);
  assert.equal(v["attribution.tool_ms"], 3);
  assert.equal(v["attribution.framework_ms"], 96);
  assert.equal(v["tokens.input"], 1117);
  assert.equal(v["tokens.peak_context"], 391);
  assert.equal(v["counts.turns"], 2);
  assert.equal(v["counts.steps"], 3);
  assert.equal(v["counts.tool_calls"], 2);
  // Mock models report no cost: missing cost is unavailable, never zero.
  assert.equal(metrics["cost.primary_usd"].reason, "missing-step-cost");
  assert.equal(metrics["latency.output_tokens_per_s"].reason, "zero-stream-duration");
  assertPartition(metrics);
});

test("attributes delegated work to tasks across repeated captures of one session", () => {
  const { metrics } = deriveAttemptMetrics(fixture("agent-subagents-continuation-local"));
  const v = values(metrics);
  assert.equal(v["attribution.task_ms"], 2465);
  assert.equal(v["attribution.tool_ms"], 133);
  assert.equal(v["counts.turns"], 3, "child sessions are not root turns");
  assert.equal(v["counts.subagent_fan_out"], 1);
  assert.equal(v["counts.subagent_depth"], 1);
  // Rolled-up usage includes the child's spend.
  assert.equal(v["tokens.input"], 6006);
  assertPartition(metrics);
});

test("excludes sign-in wait from latency and counts it as human wait", () => {
  const artifact = synthetic(
    turn(0, 1000, [
      event("authorization.required", 100, { turnId: "t", name: "account" }),
      event("authorization.completed", 274, { turnId: "t", name: "account" }),
    ]),
  );
  artifact.completedAt = at(1072);
  const { metrics } = deriveAttemptMetrics(artifact);
  const v = values(metrics);
  assert.equal(v["attribution.human_wait_ms"], 174);
  assert.equal(v["latency.eval_wall_ms"], 1072 - 174);
  assertPartition(metrics);
});

test("compaction outside a turn does not count toward turn time", () => {
  const { metrics } = deriveAttemptMetrics(compactedArtifact());
  assert.equal(values(metrics)["counts.compactions"], 1);
  assert.equal(values(metrics)["attribution.compaction_ms"], 0);
  assertPartition(metrics);
});

function withCosts(artifact, { stepCost, rolledUpCost }) {
  const copy = structuredClone(artifact);
  // The runner's eval total (result.derived.usage) carries the rolled-up cost.
  copy.result.derived.usage.costUsd = rolledUpCost;
  for (const session of copy.result.sessions) {
    for (const e of session.events) {
      if (session.sessionId !== copy.result.sessionId) continue;
      if (e.type === "step.completed") e.data.usage.costUsd = stepCost;
      if (["session.waiting", "turn.waiting"].includes(e.type)) e.data.usage.costUsd = rolledUpCost;
    }
  }
  return copy;
}

test("splits cost into primary steps and rolled-up subagent spend", () => {
  const artifact = withCosts(fixture("agent-subagents-continuation-local"), {
    stepCost: 0.25,
    rolledUpCost: 4,
  });
  const v = values(deriveAttemptMetrics(artifact).metrics);
  assert.equal(v["cost.primary_usd"], 2.5, "ten primary steps");
  assert.equal(v["cost.total_usd"], 4);
  assert.equal(v["cost.subagent_usd"], 1.5);
});

test("cost and tokens are the runner's eval usage, and unavailable without it", () => {
  const artifact = fixture("agent-tools-ends-turn-function");
  artifact.result.derived.usage = {
    cacheReadTokens: 40,
    cacheWriteTokens: 5,
    inputTokens: 200,
    outputTokens: 12,
    costUsd: 0.75,
  };
  const v = values(deriveAttemptMetrics(artifact).metrics);
  assert.equal(v["cost.total_usd"], 0.75);
  assert.equal(v["tokens.input"], 200);
  assert.equal(v["tokens.output"], 12);
  assert.equal(v["tokens.cache_read"], 40);
  assert.equal(v["tokens.cache_write"], 5);
  assert.equal(v["tokens.cache_hit_ratio"], 0.2);
  // Peak context is per step, from the events, not the eval total.
  assert.equal(v["tokens.peak_context"], 391);

  // A counted session reported no cost: the runner leaves costUsd out.
  delete artifact.result.derived.usage.costUsd;
  const noCost = deriveAttemptMetrics(artifact).metrics;
  assert.deepEqual(noCost["cost.total_usd"], {
    status: "unavailable",
    reason: "missing-rolled-up-cost",
  });
  assert.equal(noCost["tokens.input"].status, "measured");

  // A counted session reported no usage at all: the runner leaves usage out.
  delete artifact.result.derived.usage;
  const none = deriveAttemptMetrics(artifact).metrics;
  for (const key of ["cost.total_usd", "tokens.input", "tokens.output", "tokens.cache_hit_ratio"])
    assert.equal(none[key].status, "unavailable", key);
  assert.equal(none["tokens.peak_context"].status, "measured");
});

test("compaction cost stays in the total but leaves the subagent split unknown", () => {
  // Compaction summary calls are paid and roll up into session usage, but no
  // step reports them: total − steps would misreport them as subagent spend.
  const artifact = withCosts(compactedArtifact(), {
    stepCost: 0.25,
    rolledUpCost: 2,
  });
  const { metrics } = deriveAttemptMetrics(artifact);
  assert.equal(values(metrics)["cost.total_usd"], 2);
  assert.equal(metrics["cost.primary_usd"].status, "measured");
  assert.deepEqual(metrics["cost.subagent_usd"], {
    status: "unavailable",
    reason: "compaction-cost-unattributed",
  });
});

test("a single step without cost makes cost unavailable, and inconsistent rollups are flagged", () => {
  const artifact = withCosts(fixture("agent-tools-ends-turn-function"), {
    stepCost: 1,
    rolledUpCost: 3,
  });
  assert.equal(values(deriveAttemptMetrics(artifact).metrics)["cost.subagent_usd"], 0);

  const inconsistent = withCosts(artifact, { stepCost: 1, rolledUpCost: 2 });
  assert.equal(
    deriveAttemptMetrics(inconsistent).metrics["cost.subagent_usd"].reason,
    "inconsistent-cost-rollup",
  );

  const step = artifact.result.sessions[0].events.find((e) => e.type === "step.completed");
  delete step.data.usage.costUsd;
  const { metrics } = deriveAttemptMetrics(artifact);
  assert.equal(metrics["cost.primary_usd"].reason, "missing-step-cost");
  assert.equal(metrics["cost.subagent_usd"].reason, "missing-step-cost");
  assert.equal(metrics["cost.total_usd"].status, "measured");
});

test("overlapping parallel tools count once and streaming metrics use first deltas", () => {
  const step = { turnId: "t", stepIndex: 0 };
  const events = turn(0, 1000, [
    event("step.started", 100, step),
    event("reasoning.appended", 150, step),
    event("message.appended", 300, step),
    event("actions.requested", 400, {
      ...step,
      actions: [
        { kind: "tool-call", callId: "a" },
        { kind: "tool-call", callId: "b" },
      ],
    }),
    event("action.result", 700, { ...step, result: { callId: "b" }, status: "failed" }),
    event("action.result", 600, { ...step, result: { callId: "a" }, status: "completed" }),
    event("step.completed", 750, { ...step, usage: { outputTokens: 60, inputTokens: 900 } }),
  ]);
  const v = values(deriveAttemptMetrics(synthetic(events)).metrics);
  assert.equal(v["attribution.tool_ms"], 300);
  assert.equal(v["attribution.model_ms"], 350);
  assert.equal(v["attribution.framework_ms"], 350);
  assert.equal(v["latency.ttft_ms_p50"], 50);
  assert.equal(v["latency.output_tokens_per_s"], 100);
  assert.equal(v["reliability.tool_error_rate"], 0.5);
  assert.equal(v["counts.tool_calls_per_step"], 2);
});

test("partial captures make attribution unavailable instead of guessing", () => {
  const openTurn = [
    event("turn.started", 0, { turnId: "t" }),
    event("step.started", 10, { turnId: "t", stepIndex: 0 }),
  ];
  assert.equal(
    deriveAttemptMetrics(synthetic(openTurn)).metrics["attribution.framework_ms"].reason,
    "turn-incomplete",
  );
  const orphanResult = turn(0, 100, [event("action.result", 50, { result: { callId: "x" } })]);
  assert.equal(
    deriveAttemptMetrics(synthetic(orphanResult)).metrics["attribution.tool_ms"].reason,
    "action-start-missing",
  );
  const openInput = turn(0, 100, [
    event("input.requested", 50, { turnId: "unknown", requests: [{ requestId: "r" }] }),
  ]);
  assert.equal(
    deriveAttemptMetrics(synthetic(openInput)).metrics["attribution.human_wait_ms"].reason,
    "input-incomplete",
  );
  const noCapture = { ...synthetic([]), result: { status: "completed" } };
  assert.equal(
    deriveAttemptMetrics(noCapture).metrics["latency.turn_ms"].reason,
    "missing-session-capture",
  );
  const conflicting = synthetic([event("turn.started", 0, { turnId: "t" })]);
  conflicting.result.sessions.push({
    sessionId: "s",
    events: [{ ...conflicting.result.sessions[0].events[0], data: { turnId: "other" } }],
  });
  assert.equal(
    deriveAttemptMetrics(conflicting).metrics["counts.turns"].reason,
    "invalid-session-capture",
  );
});

test("a cancelled turn closes work it abandoned", () => {
  const events = turn(
    0,
    500,
    [
      event("step.started", 10, { turnId: "t", stepIndex: 0 }),
      event("actions.requested", 100, {
        turnId: "t",
        stepIndex: 0,
        actions: [{ kind: "tool-call", callId: "a" }],
      }),
    ],
    "turn.cancelled",
  );
  const v = values(deriveAttemptMetrics(synthetic(events)).metrics);
  assert.equal(v["attribution.tool_ms"], 400);
  assert.equal(v["attribution.model_ms"], 90);
});

test("counts failures by event and code", () => {
  const events = turn(
    0,
    100,
    [
      event("step.started", 10, { turnId: "t", stepIndex: 0 }),
      event("step.failed", 20, { turnId: "t", stepIndex: 0, code: "MODEL_CALL_FAILED" }),
      event("session.failed", 99, { code: "MODEL_CALL_FAILED" }),
    ],
    "turn.failed",
  );
  events.at(-1).data.code = "MODEL_CALL_FAILED";
  const { failures, metrics } = deriveAttemptMetrics(synthetic(events));
  assert.deepEqual(failures, [
    { type: "session.failed", code: "MODEL_CALL_FAILED", count: 1 },
    { type: "step.failed", code: "MODEL_CALL_FAILED", count: 1 },
    { type: "turn.failed", code: "MODEL_CALL_FAILED", count: 1 },
  ]);
  assert.equal(values(metrics)["reliability.step_failures"], 1);
});

test("classifies outcomes and every gap reason", () => {
  const failed = (code, details, message = "failed") =>
    synthetic(
      turn(0, 100, [
        event("step.failed", 50, { turnId: "t", stepIndex: 0, code, details, message }),
      ]),
    );
  const completedStep = event("step.completed", 50, { turnId: "t", stepIndex: 0 });

  assert.deepEqual(classifyOutcome(fixture("agent-tools-ends-turn-function")), {
    outcome: "completed",
  });
  assert.deepEqual(classifyOutcome({ ...synthetic([]), verdict: "skipped" }), {
    outcome: "skipped",
  });
  for (const details of [
    { statusCode: 429 },
    { upstreamStatusCode: 503 },
    { semanticErrorId: "network-request-failed" },
    { detail: "TypeError: fetch failed [cause]: UND_ERR_HEADERS_TIMEOUT" },
  ])
    assert.deepEqual(classifyOutcome(failed("MODEL_CALL_FAILED", details)), {
      outcome: "gap",
      gap_reason: "provider-unavailable",
    });
  for (const details of [{ statusCode: 400 }, { statusCode: 413 }])
    assert.equal(classifyOutcome(failed("MODEL_CALL_FAILED", details)).outcome, "completed");
  assert.equal(classifyOutcome(failed("OUTPUT_SCHEMA_NOT_FULFILLED", {})).outcome, "completed");
  assert.equal(
    classifyOutcome(failed("WORKFLOW_STREAM_WRITE_FAILED", {})).gap_reason,
    "stream-write-failed",
  );

  const timeout = "The operation was aborted due to timeout";
  assert.equal(
    classifyOutcome(synthetic([], { error: timeout })).gap_reason,
    "timeout-before-first-step",
  );
  assert.equal(
    classifyOutcome(synthetic([], { error: "connect ECONNREFUSED" })).gap_reason,
    "error-before-first-step",
  );
  assert.deepEqual(classifyOutcome(synthetic([completedStep], { error: timeout })), {
    outcome: "timed_out",
  });
  // A first step that fails for a real reason is the eval's failure, not a gap,
  // even though the eval then records an error (e.g. from expectOk()).
  for (const details of [{ statusCode: 400 }, { statusCode: 413 }]) {
    const rejected = { ...failed("MODEL_CALL_FAILED", details), error: "expectOk: session failed" };
    assert.deepEqual(classifyOutcome(rejected), { outcome: "completed" });
    assert.deepEqual(classifyOutcome({ ...rejected, error: timeout }), { outcome: "timed_out" });
  }
  const sessionFailed = synthetic([event("session.failed", 40, { code: "SESSION_FAILED" })], {
    error: "expectOk: session failed",
  });
  assert.deepEqual(classifyOutcome(sessionFailed), { outcome: "completed" });
  // Parked is the runner's derived fact (eve's derive-run-facts), not re-derived here.
  const parked = synthetic([completedStep]);
  parked.result.derived = { parked: true };
  assert.deepEqual(classifyOutcome(parked), { outcome: "parked" });
});