The Open Framework for Building Agents
docs(benchmarks): run benchmarks and retain superseded models (#2724)
Colton Padden451be3fb2d05
diff --git a/apps/benchmarks/lib/benchmark-config.test.mjs b/apps/benchmarks/lib/benchmark-config.test.mjsindex 04275db2c..e6d68c63b 100644--- a/apps/benchmarks/lib/benchmark-config.test.mjs+++ b/apps/benchmarks/lib/benchmark-config.test.mjs@@ -12,7 +12,6 @@ test("publishes only compatibility-validated models", () => {assert.deepEqual(publishedBenchmarkModels.map((benchmark) => benchmark.id),[- "claude-sonnet-4-6","kimi-k3","claude-fable-5","grok-4-6",@@ -41,5 +40,7 @@ test("allows candidate probes and rejects unknown models", () => {assert.equal(findBenchmarkModel("gpt-5-6-sol").support, "supported");assert.equal(findBenchmarkModel("gpt-5-6-terra").support, "supported");assert.equal(findBenchmarkModel("grok-4-6").support, "supported");+ assert.equal(findBenchmarkModel("claude-sonnet-4-6").support, "superseded");+ assert.equal(findBenchmarkModel("claude-opus-5").support, "candidate");assert.throws(() => findBenchmarkModel("unknown"), /Unknown model/u);});diff --git a/apps/benchmarks/lib/benchmark-config.ts b/apps/benchmarks/lib/benchmark-config.tsindex 4928309b7..3abb3cfbc 100644--- a/apps/benchmarks/lib/benchmark-config.ts+++ b/apps/benchmarks/lib/benchmark-config.ts@@ -2,7 +2,7 @@ export const authoringTreatments = ["baseline", "guided"] as const;export type AuthoringTreatment = (typeof authoringTreatments)[number];-export type AuthoringBenchmarkSupport = "supported" | "candidate";+export type AuthoringBenchmarkSupport = "supported" | "candidate" | "superseded";export interface AuthoringBenchmarkModel {readonly id: string;@@ -18,7 +18,7 @@ export const benchmarkModels = [model: "claude-sonnet-4-6",displayName: "Claude Sonnet 4.6",harness: "OpenCode",- support: "supported",+ support: "superseded",},{id: "kimi-k3",diff --git a/apps/benchmarks/scripts/export-results.mjs b/apps/benchmarks/scripts/export-results.mjsindex 5dc49b682..0d1c71e5b 100644--- a/apps/benchmarks/scripts/export-results.mjs+++ b/apps/benchmarks/scripts/export-results.mjs@@ -16,6 +16,7 @@ import {} from "./cost.mjs";import {authoringTreatments,+ benchmarkModels,findPublishedBenchmarkModel,publishedBenchmark,publishedBenchmarkModels,@@ -55,6 +56,7 @@ const experimentIds = benchmarks.flatMap((benchmark) =>authoringTreatments.map((treatment) => publishedExperimentId(benchmark, treatment)),);const stale = staleCells(experimentIds);+const previousResults = readPreviousResults();const results = [];for (const benchmark of benchmarks) {@@ -104,6 +106,7 @@ const output = {})),),results,+ ...(previousResults.length === 0 ? {} : { previouslyMeasured: previousResults }),};const destination =@@ -112,6 +115,40 @@ mkdirSync(dirname(destination), { recursive: true });writeFileSync(destination, `${JSON.stringify(output, null, 2)}\n`);console.log(`Exported ${results.length} benchmark cells to ${destination}`);+function readPreviousResults() {+ const destination =+ values.output === undefined ? outputPath : resolve(process.cwd(), values.output);+ if (!existsSync(destination)) return [];++ const previous = JSON.parse(readFileSync(destination, "utf8"));+ const supersededExperimentIds = new Set(+ benchmarkModels+ .filter((benchmark) => benchmark.support === "superseded")+ .flatMap((benchmark) =>+ authoringTreatments.map((treatment) => publishedExperimentId(benchmark, treatment)),+ ),+ );+ const supersededExperiments = (previous.experiments ?? []).filter((experiment) =>+ supersededExperimentIds.has(experiment.id),+ );+ const supersededResults = (previous.results ?? []).filter((result) =>+ supersededExperimentIds.has(result.experimentId),+ );+ const retained = (previous.previouslyMeasured ?? []).filter(+ (measurement) => Array.isArray(measurement.experiments) && Array.isArray(measurement.results),+ );+ if (supersededExperiments.length === 0) return retained;++ return [+ ...retained,+ {+ suite: previous.suite,+ experiments: supersededExperiments,+ results: supersededResults,+ },+ ];+}+function selectedBenchmarks(value) {if (value === undefined) return publishedBenchmarkModels;return value.split(",").map((model) => findPublishedBenchmarkModel(model));diff --git a/apps/benchmarks/scripts/export-results.test.mjs b/apps/benchmarks/scripts/export-results.test.mjsindex 7e9804acf..0d19c8aaa 100644--- a/apps/benchmarks/scripts/export-results.test.mjs+++ b/apps/benchmarks/scripts/export-results.test.mjs@@ -40,11 +40,10 @@ test("exports missing cells without publishing private artifacts", () => {assert.equal(output.schemaVersion, 1);assert.equal(output.suite.caseCount, 7);assert.match(output.suite.caseFingerprint, /^[0-9a-f]{64}$/u);- assert.equal(output.experiments.length, 18);+ assert.equal(output.experiments.length, 16);assert.deepEqual([...new Set(output.experiments.map((experiment) => experiment.modelDisplayName))],[- "Claude Sonnet 4.6","Kimi K3","Claude Fable 5","Grok 4.6",@@ -55,7 +54,7 @@ test("exports missing cells without publishing private artifacts", () => {"Gemini 3.1 Pro Preview",],);- assert.equal(output.results.length, 126);+ assert.equal(output.results.length, 112);assert.ok(output.results.every((result) => result.status === "missing"));for (const privateField of ["transcript", "commands", "worldEvents", "files"]) {assert.equal(raw.includes(`"${privateField}"`), false);@@ -67,6 +66,76 @@ test("exports missing cells without publishing private artifacts", () => {}});+test("retains superseded experiments separately from the current matrix", () => {+ const directory = mkdtempSync(join(tmpdir(), "eve-benchmark-export-previous-"));+ const outputPath = join(directory, "results.json");+ const resultsPath = new URL("../results", import.meta.url).pathname;+ const savedResultsPath = join(directory, "saved-results");+ try {+ writeFileSync(+ outputPath,+ JSON.stringify({+ schemaVersion: 1,+ generatedAt: "2026-08-01T00:00:00.000Z",+ suite: {+ eveRevision: "b".repeat(40),+ caseFingerprint: "c".repeat(64),+ caseCount: 1,+ runsPerCell: 1,+ },+ experiments: [+ {+ id: "claude-sonnet-4-6-opencode--baseline",+ groupId: "claude-sonnet-4-6-opencode",+ model: "claude-sonnet-4-6",+ modelDisplayName: "Claude Sonnet 4.6",+ harness: "OpenCode",+ treatment: "baseline",+ },+ ],+ results: [+ {+ experimentId: "claude-sonnet-4-6-opencode--baseline",+ caseId: "author-001-weather-tool",+ status: "current",+ passedRuns: 1,+ validRuns: 1,+ meanDurationMs: 1000,+ measuredAt: "2026-08-01T00:00:00.000Z",+ },+ ],+ }),+ );+ if (existsSync(resultsPath)) renameSync(resultsPath, savedResultsPath);+ mkdirSync(resultsPath, { recursive: true });+ execFileSync(+ process.execPath,+ [+ new URL("./export-results.mjs", import.meta.url).pathname,+ "--revision",+ "a".repeat(40),+ "--output",+ outputPath,+ ],+ { cwd: appRoot, stdio: "pipe" },+ );+ const output = JSON.parse(readFileSync(outputPath, "utf8"));+ assert.equal(output.previouslyMeasured.length, 1);+ assert.equal(+ output.previouslyMeasured[0].experiments[0].id,+ "claude-sonnet-4-6-opencode--baseline",+ );+ assert.equal(+ output.previouslyMeasured[0].results[0].experimentId,+ "claude-sonnet-4-6-opencode--baseline",+ );+ } finally {+ rmSync(resultsPath, { recursive: true, force: true });+ if (existsSync(savedResultsPath)) renameSync(savedResultsPath, resultsPath);+ rmSync(directory, { recursive: true, force: true });+ }+});+test("exports mean cost, token consumption, and tool invocations", () => {const directory = mkdtempSync(join(tmpdir(), "eve-benchmark-export-cost-"));const outputPath = join(directory, "results.json");@@ -76,7 +145,7 @@ test("exports mean cost, token consumption, and tool invocations", () => {if (existsSync(resultsPath)) renameSync(resultsPath, savedResultsPath);const runPath = join(resultsPath,- "claude-sonnet-4-6-opencode--baseline",+ "kimi-k3-opencode--baseline","run","author-001-weather-tool","run-1",@@ -113,7 +182,7 @@ test("exports mean cost, token consumption, and tool invocations", () => {"--revision","a".repeat(40),"--models",- "claude-sonnet-4-6",+ "kimi-k3","--output",outputPath,],@@ -122,7 +191,7 @@ test("exports mean cost, token consumption, and tool invocations", () => {const output = JSON.parse(readFileSync(outputPath, "utf8"));const result = output.results.find((entry) =>- entry.experimentId === "claude-sonnet-4-6-opencode--baseline" &&+ entry.experimentId === "kimi-k3-opencode--baseline" &&entry.caseId === "author-001-weather-tool",);assert.equal(result.meanEstimatedListCostUsd, 4.5);diff --git a/apps/docs/app/[lang]/benchmarks/page.tsx b/apps/docs/app/[lang]/benchmarks/page.tsxindex 013f3ae59..48d50d447 100644--- a/apps/docs/app/[lang]/benchmarks/page.tsx+++ b/apps/docs/app/[lang]/benchmarks/page.tsx@@ -26,6 +26,9 @@ export const metadata: Metadata = {const ModelBenchmarksPage = () => {const rows = benchmarkRows(benchmarkResults);+ const previouslyMeasured = (benchmarkResults.previouslyMeasured ?? []).flatMap((measurement) =>+ benchmarkRows({ ...benchmarkResults, ...measurement }),+ );const publishedAt = formatDate(benchmarkResults.generatedAt);const revision = benchmarkResults.suite.eveRevision;@@ -35,7 +38,8 @@ const ModelBenchmarksPage = () => {<h1 className="text-gray-1000 text-heading-48 sm:text-heading-64">Model Benchmarks</h1><p className="mt-5 max-w-3xl text-gray-900 text-lg">Performance of coding agents creating and changing eve projects, measured by deterministic- checks against the files, commands, and simulated external interactions each run produces.+ checks against the files, commands, and simulated external interactions each run produces+ <sup>1</sup>.</p><div className="mt-6 flex flex-wrap items-center justify-center gap-x-3 gap-y-2 text-gray-800 text-sm"><span>@@ -50,25 +54,30 @@ const ModelBenchmarksPage = () => {</div></section>- <ResultsTable rows={rows} />+ <section aria-labelledby="current-results">+ <h2 className="sr-only" id="current-results">+ Current results+ </h2>+ <ResultsTable rows={rows} />+ </section>- <div className="mt-4 space-y-2 px-4 text-gray-700 text-sm leading-6">- <p>- * eve guidance includes the <code>AGENTS.md</code> and agent-specific aliases generated by{" "}- <code>eve init</code>. The column shows the success rate when agents have access to this- guidance.- </p>- <p>- N/A marks a case that a model has not run. It counts against that model's success- rate so every model is scored against the same case set. When a case changes, eve retains- a model's most recent result until that model reruns it.- </p>- <p>- Avg List Cost is the mean estimated cost per evaluation from each run's token usage- at the provider's public list price. It is a relative comparison, not a Gateway bill:- routing, cache discounts, and provider pricing can differ.- </p>- </div>+ {previouslyMeasured.length > 0 ? (+ <section className="mt-8" aria-labelledby="retired-models">+ <h2 className="text-heading-24 text-gray-1000" id="retired-models">+ Retired Models+ </h2>+ <div className="mt-3">+ <ResultsTable rows={previouslyMeasured} showMeasurementDate />+ </div>+ </section>+ ) : null}++ <p className="mt-8 text-gray-700 text-sm leading-6">+ <sup>1</sup> Each model completes the same deterministic eve authoring tasks three times+ with and without the guidance files generated by <code>eve init</code>; success rates+ include every scheduled run, while duration, tokens, and tool calls average valid runs. Cost+ estimates apply provider public list prices to recorded token usage.+ </p></main>);};diff --git a/apps/docs/app/[lang]/benchmarks/results-table.tsx b/apps/docs/app/[lang]/benchmarks/results-table.tsxindex bf484d65c..9dfc82185 100644--- a/apps/docs/app/[lang]/benchmarks/results-table.tsx+++ b/apps/docs/app/[lang]/benchmarks/results-table.tsx@@ -20,7 +20,13 @@ interface SortState {direction: SortDirection;}-export const ResultsTable = ({ rows }: { rows: BenchmarkRow[] }) => {+export const ResultsTable = ({+ rows,+ showMeasurementDate = false,+}: {+ rows: BenchmarkRow[];+ showMeasurementDate?: boolean;+}) => {const [expanded, setExpanded] = useState<ReadonlySet<string>>(new Set());const [sort, setSort] = useState<SortState>({key: "guidedSuccessRate",@@ -76,7 +82,7 @@ export const ResultsTable = ({ rows }: { rows: BenchmarkRow[] }) => {onSort={() => toggleSort("averageDurationMs")}sortDirection={sort.key === "averageDurationMs" ? sort.direction : undefined}>- Avg Duration+ Duration</HeaderCell><HeaderCellalign="right"@@ -85,7 +91,7 @@ export const ResultsTable = ({ rows }: { rows: BenchmarkRow[] }) => {sort.key === "averageEstimatedListCostUsd" ? sort.direction : undefined}>- Avg List Cost+ Cost</HeaderCell><HeaderCellalign="right"@@ -99,7 +105,7 @@ export const ResultsTable = ({ rows }: { rows: BenchmarkRow[] }) => {onSort={() => toggleSort("guidedSuccessRate")}sortDirection={sort.key === "guidedSuccessRate" ? sort.direction : undefined}>- Success Rate with AGENTS.md *+ Success Rate with AGENTS.md</HeaderCell></tr></thead>@@ -123,6 +129,11 @@ export const ResultsTable = ({ rows }: { rows: BenchmarkRow[] }) => {/></span>{row.modelDisplayName}+ {showMeasurementDate && row.latestMeasuredAt !== null ? (+ <span className="font-normal text-gray-700 text-sm">+ {formatDate(row.latestMeasuredAt)}+ </span>+ ) : null}</button></Cell><Cell>{row.harness}</Cell>@@ -133,15 +144,15 @@ export const ResultsTable = ({ rows }: { rows: BenchmarkRow[] }) => {</tr>{isExpanded ? (<tr className="border-t border-gray-400">- <td className="p-4" colSpan={6}>- <div className="overflow-hidden rounded-lg border border-gray-400">- <table className="w-full border-collapse text-left">- <thead className="bg-gray-100 text-gray-800">+ <td className="px-4 pb-4" colSpan={6}>+ <div>+ <table className="w-full border-collapse text-left text-sm">+ <thead className="text-gray-800"><tr><HeaderCell>Evaluation</HeaderCell>- <HeaderCell align="right">Avg Duration</HeaderCell>- <HeaderCell align="right">Avg Tokens</HeaderCell>- <HeaderCell align="right">Avg Tool Calls</HeaderCell>+ <HeaderCell align="right">Duration</HeaderCell>+ <HeaderCell align="right">Tokens</HeaderCell>+ <HeaderCell align="right">Tool Calls</HeaderCell><HeaderCell align="right">Success Rate</HeaderCell><HeaderCell align="right">Success Rate with AGENTS.md</HeaderCell></tr>@@ -254,6 +265,13 @@ function compareNullableNumbers(return direction === "ascending" ? left - right : right - left;}+function formatDate(value: string): string {+ return new Intl.DateTimeFormat("en", {+ dateStyle: "medium",+ timeZone: "UTC",+ }).format(new Date(value));+}+function formatDuration(value: number | null): React.ReactNode {if (value === null) return <NotAvailable />;return `${(value / 1000).toFixed(1)}s`;diff --git a/apps/docs/lib/evals/benchmark-results.json b/apps/docs/lib/evals/benchmark-results.jsonindex 532612363..312ba679a 100644--- a/apps/docs/lib/evals/benchmark-results.json+++ b/apps/docs/lib/evals/benchmark-results.json@@ -1,29 +1,13 @@{"schemaVersion": 1,- "generatedAt": "2026-08-19T22:10:07.841Z",+ "generatedAt": "2026-08-28T20:19:42.291Z","suite": {- "eveRevision": "54a028bab589a60fc84a310304a5cf080bc7f990",+ "eveRevision": "05b12e3beb662979143397b9e691a2d7b4417a02","caseFingerprint": "786490be9d902701f57d08b3d112ca385d49a72e5591fac741511e77d67989d2","caseCount": 7,"runsPerCell": 3},"experiments": [- {- "id": "claude-sonnet-4-6-opencode--baseline",- "groupId": "claude-sonnet-4-6-opencode",- "model": "claude-sonnet-4-6",- "modelDisplayName": "Claude Sonnet 4.6",- "harness": "OpenCode",- "treatment": "baseline"- },- {- "id": "claude-sonnet-4-6-opencode--guided",- "groupId": "claude-sonnet-4-6-opencode",- "model": "claude-sonnet-4-6",- "modelDisplayName": "Claude Sonnet 4.6",- "harness": "OpenCode",- "treatment": "guided"- },{"id": "kimi-k3-opencode--baseline","groupId": "kimi-k3-opencode",@@ -154,185 +138,17 @@}],"results": [- {- "experimentId": "claude-sonnet-4-6-opencode--baseline",- "caseId": "author-001-weather-tool",- "status": "current",- "passedRuns": 0,- "validRuns": 3,- "meanDurationMs": 199451,- "measuredAt": "2026-08-19T21:35:21.709Z",- "meanEstimatedListCostUsd": 0.052948,- "meanTokenConsumption": 10626.666666666666,- "meanToolInvocationCount": 11.666666666666666- },- {- "experimentId": "claude-sonnet-4-6-opencode--baseline",- "caseId": "author-002-new-project",- "status": "current",- "passedRuns": 0,- "validRuns": 3,- "meanDurationMs": 143527,- "measuredAt": "2026-08-19T21:35:21.715Z",- "meanEstimatedListCostUsd": 0.04594499999999999,- "meanTokenConsumption": 10429.666666666666,- "meanToolInvocationCount": 10.666666666666666- },- {- "experimentId": "claude-sonnet-4-6-opencode--baseline",- "caseId": "author-003-openapi-connection",- "status": "current",- "passedRuns": 1,- "validRuns": 3,- "meanDurationMs": 158661,- "measuredAt": "2026-08-19T21:35:21.721Z",- "meanEstimatedListCostUsd": 0.11106500000000001,- "meanTokenConsumption": 27383,- "meanToolInvocationCount": 23.666666666666668- },- {- "experimentId": "claude-sonnet-4-6-opencode--baseline",- "caseId": "author-004-packaged-skill",- "status": "current",- "passedRuns": 2,- "validRuns": 3,- "meanDurationMs": 382777,- "measuredAt": "2026-08-17T22:03:22.192Z",- "meanEstimatedListCostUsd": 0.108879,- "meanTokenConsumption": 22653,- "meanToolInvocationCount": 16- },- {- "experimentId": "claude-sonnet-4-6-opencode--baseline",- "caseId": "author-005-conditional-approval",- "status": "current",- "passedRuns": 0,- "validRuns": 3,- "meanDurationMs": 225419,- "measuredAt": "2026-08-19T21:35:21.730Z",- "meanEstimatedListCostUsd": 0.10689799999999999,- "meanTokenConsumption": 22528.666666666668,- "meanToolInvocationCount": 24- },- {- "experimentId": "claude-sonnet-4-6-opencode--baseline",- "caseId": "author-006-custom-channel",- "status": "current",- "passedRuns": 1,- "validRuns": 3,- "meanDurationMs": 714111,- "measuredAt": "2026-08-17T22:19:44.996Z",- "meanEstimatedListCostUsd": 0.056847,- "meanTokenConsumption": 12625,- "meanToolInvocationCount": 10- },- {- "experimentId": "claude-sonnet-4-6-opencode--baseline",- "caseId": "author-007-digest-schedule",- "status": "current",- "passedRuns": 0,- "validRuns": 3,- "meanDurationMs": 232328,- "measuredAt": "2026-08-19T21:35:21.737Z",- "meanEstimatedListCostUsd": 0.09773,- "meanTokenConsumption": 16300.666666666666,- "meanToolInvocationCount": 33.333333333333336- },- {- "experimentId": "claude-sonnet-4-6-opencode--guided",- "caseId": "author-001-weather-tool",- "status": "current",- "passedRuns": 0,- "validRuns": 3,- "meanDurationMs": 186603,- "measuredAt": "2026-08-19T21:35:25.673Z",- "meanEstimatedListCostUsd": 0.07002399999999999,- "meanTokenConsumption": 13736,- "meanToolInvocationCount": 17.333333333333332- },- {- "experimentId": "claude-sonnet-4-6-opencode--guided",- "caseId": "author-002-new-project",- "status": "current",- "passedRuns": 2,- "validRuns": 3,- "meanDurationMs": 198555,- "measuredAt": "2026-08-19T21:35:25.695Z",- "meanEstimatedListCostUsd": 0.104669,- "meanTokenConsumption": 27117.666666666668,- "meanToolInvocationCount": 17.333333333333332- },- {- "experimentId": "claude-sonnet-4-6-opencode--guided",- "caseId": "author-003-openapi-connection",- "status": "current",- "passedRuns": 3,- "validRuns": 3,- "meanDurationMs": 132154,- "measuredAt": "2026-08-19T21:35:25.701Z",- "meanEstimatedListCostUsd": 0.045839,- "meanTokenConsumption": 11199.666666666666,- "meanToolInvocationCount": 10.333333333333334- },- {- "experimentId": "claude-sonnet-4-6-opencode--guided",- "caseId": "author-004-packaged-skill",- "status": "current",- "passedRuns": 1,- "validRuns": 3,- "meanDurationMs": 127395,- "measuredAt": "2026-08-19T21:04:16.127Z",- "meanEstimatedListCostUsd": 0.082587,- "meanTokenConsumption": 16813,- "meanToolInvocationCount": 13- },- {- "experimentId": "claude-sonnet-4-6-opencode--guided",- "caseId": "author-005-conditional-approval",- "status": "current",- "passedRuns": 0,- "validRuns": 3,- "meanDurationMs": 170150,- "measuredAt": "2026-08-19T21:35:25.711Z",- "meanEstimatedListCostUsd": 0.031358,- "meanTokenConsumption": 5668.666666666667,- "meanToolInvocationCount": 7.666666666666667- },- {- "experimentId": "claude-sonnet-4-6-opencode--guided",- "caseId": "author-006-custom-channel",- "status": "current",- "passedRuns": 2,- "validRuns": 3,- "meanDurationMs": 130916,- "measuredAt": "2026-08-19T21:04:16.135Z",- "meanEstimatedListCostUsd": 0.0346035,- "meanTokenConsumption": 6458.5,- "meanToolInvocationCount": 11- },- {- "experimentId": "claude-sonnet-4-6-opencode--guided",- "caseId": "author-007-digest-schedule",- "status": "current",- "passedRuns": 0,- "validRuns": 3,- "meanDurationMs": 260678,- "measuredAt": "2026-08-19T21:35:25.720Z",- "meanEstimatedListCostUsd": 0.09039199999999999,- "meanTokenConsumption": 13797.333333333334,- "meanToolInvocationCount": 29- },{"experimentId": "kimi-k3-opencode--baseline","caseId": "author-001-weather-tool","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 138696,- "measuredAt": "2026-08-19T21:40:48.303Z",- "meanEstimatedListCostUsd": 0.210969,- "meanTokenConsumption": 49695,- "meanToolInvocationCount": 22.333333333333332+ "meanDurationMs": 124258,+ "measuredAt": "2026-08-28T19:22:06.000Z",+ "meanEstimatedListCostUsd": 0.14751699999999998,+ "meanTokenConsumption": 31831,+ "meanToolInvocationCount": 25},{"experimentId": "kimi-k3-opencode--baseline",@@ -340,23 +156,23 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 92789,- "measuredAt": "2026-08-19T21:40:48.309Z",- "meanEstimatedListCostUsd": 0.105078,- "meanTokenConsumption": 27707.333333333332,- "meanToolInvocationCount": 11.333333333333334+ "meanDurationMs": 89304,+ "measuredAt": "2026-08-28T19:25:16.000Z",+ "meanEstimatedListCostUsd": 0.085075,+ "meanTokenConsumption": 21470.333333333332,+ "meanToolInvocationCount": 12},{"experimentId": "kimi-k3-opencode--baseline","caseId": "author-003-openapi-connection","status": "current",- "passedRuns": 3,+ "passedRuns": 0,"validRuns": 3,- "meanDurationMs": 137104,- "measuredAt": "2026-08-19T21:40:48.313Z",- "meanEstimatedListCostUsd": 0.21580199999999997,- "meanTokenConsumption": 56562,- "meanToolInvocationCount": 24+ "meanDurationMs": 454935,+ "measuredAt": "2026-08-28T19:22:06.000Z",+ "meanEstimatedListCostUsd": 0.3622910000000001,+ "meanTokenConsumption": 56591.666666666664,+ "meanToolInvocationCount": 28},{"experimentId": "kimi-k3-opencode--baseline",@@ -364,11 +180,11 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 124071,- "measuredAt": "2026-08-19T21:40:48.318Z",- "meanEstimatedListCostUsd": 0.136688,- "meanTokenConsumption": 29278.666666666668,- "meanToolInvocationCount": 22+ "meanDurationMs": 101585,+ "measuredAt": "2026-08-28T19:22:06.000Z",+ "meanEstimatedListCostUsd": 0.09421099999999999,+ "meanTokenConsumption": 19086.333333333332,+ "meanToolInvocationCount": 15.666666666666666},{"experimentId": "kimi-k3-opencode--baseline",@@ -376,11 +192,11 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 74874,- "measuredAt": "2026-08-19T21:40:48.321Z",- "meanEstimatedListCostUsd": 0.09797499999999999,- "meanTokenConsumption": 25718.333333333332,- "meanToolInvocationCount": 17+ "meanDurationMs": 98137,+ "measuredAt": "2026-08-28T19:22:06.000Z",+ "meanEstimatedListCostUsd": 0.097954,+ "meanTokenConsumption": 23523.333333333332,+ "meanToolInvocationCount": 20},{"experimentId": "kimi-k3-opencode--baseline",@@ -388,11 +204,11 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 101672,- "measuredAt": "2026-08-19T21:40:48.324Z",- "meanEstimatedListCostUsd": 0.17105,- "meanTokenConsumption": 47146,- "meanToolInvocationCount": 22.333333333333332+ "meanDurationMs": 78637,+ "measuredAt": "2026-08-28T19:22:06.000Z",+ "meanEstimatedListCostUsd": 0.107612,+ "meanTokenConsumption": 26372,+ "meanToolInvocationCount": 19.333333333333332},{"experimentId": "kimi-k3-opencode--baseline",@@ -400,11 +216,11 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 118063,- "measuredAt": "2026-08-19T21:40:48.328Z",- "meanEstimatedListCostUsd": 0.144896,- "meanTokenConsumption": 33914.666666666664,- "meanToolInvocationCount": 18+ "meanDurationMs": 117347,+ "measuredAt": "2026-08-28T19:22:06.000Z",+ "meanEstimatedListCostUsd": 0.103991,+ "meanTokenConsumption": 21791.666666666668,+ "meanToolInvocationCount": 16.333333333333332},{"experimentId": "kimi-k3-opencode--guided",@@ -412,11 +228,11 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 126658,- "measuredAt": "2026-08-19T21:41:29.650Z",- "meanEstimatedListCostUsd": 0.141837,- "meanTokenConsumption": 34811,- "meanToolInvocationCount": 12.666666666666666+ "meanDurationMs": 73445,+ "measuredAt": "2026-08-28T19:20:34.000Z",+ "meanEstimatedListCostUsd": 0.08992099999999999,+ "meanTokenConsumption": 20789.666666666668,+ "meanToolInvocationCount": 12.333333333333334},{"experimentId": "kimi-k3-opencode--guided",@@ -424,11 +240,11 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 100389,- "measuredAt": "2026-08-19T21:41:29.664Z",- "meanEstimatedListCostUsd": 0.089512,- "meanTokenConsumption": 23058.666666666668,- "meanToolInvocationCount": 12.666666666666666+ "meanDurationMs": 90748,+ "measuredAt": "2026-08-28T19:25:31.000Z",+ "meanEstimatedListCostUsd": 0.081476,+ "meanTokenConsumption": 20988,+ "meanToolInvocationCount": 12},{"experimentId": "kimi-k3-opencode--guided",@@ -436,11 +252,11 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 77889,- "measuredAt": "2026-08-19T21:41:29.676Z",- "meanEstimatedListCostUsd": 0.099338,- "meanTokenConsumption": 24498,- "meanToolInvocationCount": 14+ "meanDurationMs": 71243,+ "measuredAt": "2026-08-28T19:25:31.000Z",+ "meanEstimatedListCostUsd": 0.081825,+ "meanTokenConsumption": 19215,+ "meanToolInvocationCount": 13.333333333333334},{"experimentId": "kimi-k3-opencode--guided",@@ -448,11 +264,11 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 62832,- "measuredAt": "2026-08-19T21:41:29.684Z",- "meanEstimatedListCostUsd": 0.065712,- "meanTokenConsumption": 14302.666666666666,- "meanToolInvocationCount": 10+ "meanDurationMs": 126758,+ "measuredAt": "2026-08-28T19:20:34.000Z",+ "meanEstimatedListCostUsd": 0.08192899999999999,+ "meanTokenConsumption": 16876.333333333332,+ "meanToolInvocationCount": 15},{"experimentId": "kimi-k3-opencode--guided",@@ -460,11 +276,11 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 66484,- "measuredAt": "2026-08-19T21:41:29.691Z",- "meanEstimatedListCostUsd": 0.073148,- "meanTokenConsumption": 18416,- "meanToolInvocationCount": 12.333333333333334+ "meanDurationMs": 95028,+ "measuredAt": "2026-08-28T19:20:34.000Z",+ "meanEstimatedListCostUsd": 0.117859,+ "meanTokenConsumption": 29467.666666666668,+ "meanToolInvocationCount": 16.333333333333332},{"experimentId": "kimi-k3-opencode--guided",@@ -472,10 +288,10 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 81700,- "measuredAt": "2026-08-19T21:41:29.696Z",- "meanEstimatedListCostUsd": 0.11507,- "meanTokenConsumption": 32112.666666666668,+ "meanDurationMs": 49272,+ "measuredAt": "2026-08-28T19:20:34.000Z",+ "meanEstimatedListCostUsd": 0.0695,+ "meanTokenConsumption": 17840,"meanToolInvocationCount": 10.333333333333334},{@@ -484,11 +300,11 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 115791,- "measuredAt": "2026-08-19T21:41:29.701Z",- "meanEstimatedListCostUsd": 0.080707,- "meanTokenConsumption": 19710.333333333332,- "meanToolInvocationCount": 11.666666666666666+ "meanDurationMs": 77381,+ "measuredAt": "2026-08-28T19:20:34.000Z",+ "meanEstimatedListCostUsd": 0.061534,+ "meanTokenConsumption": 14636.666666666666,+ "meanToolInvocationCount": 12.333333333333334},{"experimentId": "claude-fable-5-opencode--baseline",@@ -496,11 +312,11 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 146227,- "measuredAt": "2026-08-19T21:45:40.647Z",- "meanEstimatedListCostUsd": 0.26486333333333334,- "meanTokenConsumption": 15385,- "meanToolInvocationCount": 11.666666666666666+ "meanDurationMs": 72217,+ "measuredAt": "2026-08-28T19:18:44.000Z",+ "meanEstimatedListCostUsd": 0.30839,+ "meanTokenConsumption": 19988.333333333332,+ "meanToolInvocationCount": 12},{"experimentId": "claude-fable-5-opencode--baseline",@@ -508,11 +324,11 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 145107,- "measuredAt": "2026-08-19T21:45:40.656Z",- "meanEstimatedListCostUsd": 0.1758166666666667,- "meanTokenConsumption": 12067,- "meanToolInvocationCount": 10.333333333333334+ "meanDurationMs": 93899,+ "measuredAt": "2026-08-28T19:25:21.000Z",+ "meanEstimatedListCostUsd": 0.22944333333333336,+ "meanTokenConsumption": 17757.666666666668,+ "meanToolInvocationCount": 9},{"experimentId": "claude-fable-5-opencode--baseline",@@ -520,11 +336,11 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 158009,- "measuredAt": "2026-08-19T21:45:40.664Z",- "meanEstimatedListCostUsd": 0.2196233333333333,- "meanTokenConsumption": 11679.666666666666,- "meanToolInvocationCount": 19.666666666666668+ "meanDurationMs": 80987,+ "measuredAt": "2026-08-28T19:25:21.000Z",+ "meanEstimatedListCostUsd": 0.3146933333333333,+ "meanTokenConsumption": 23109.333333333332,+ "meanToolInvocationCount": 16.666666666666668},{"experimentId": "claude-fable-5-opencode--baseline",@@ -532,11 +348,11 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 83086,- "measuredAt": "2026-08-19T21:45:40.670Z",- "meanEstimatedListCostUsd": 0.18186666666666665,- "meanTokenConsumption": 10064,- "meanToolInvocationCount": 11+ "meanDurationMs": 62158,+ "measuredAt": "2026-08-28T19:18:44.000Z",+ "meanEstimatedListCostUsd": 0.22673666666666667,+ "meanTokenConsumption": 14869.666666666666,+ "meanToolInvocationCount": 10.333333333333334},{"experimentId": "claude-fable-5-opencode--baseline",@@ -544,11 +360,11 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 109979,- "measuredAt": "2026-08-19T21:45:40.674Z",- "meanEstimatedListCostUsd": 0.20606333333333335,- "meanTokenConsumption": 14737,- "meanToolInvocationCount": 12.333333333333334+ "meanDurationMs": 55663,+ "measuredAt": "2026-08-28T19:18:44.000Z",+ "meanEstimatedListCostUsd": 0.30195666666666665,+ "meanTokenConsumption": 24389,+ "meanToolInvocationCount": 11.666666666666666},{"experimentId": "claude-fable-5-opencode--baseline",@@ -556,10 +372,10 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 103442,- "measuredAt": "2026-08-19T21:45:40.679Z",- "meanEstimatedListCostUsd": 0.15650666666666668,- "meanTokenConsumption": 11056,+ "meanDurationMs": 53720,+ "measuredAt": "2026-08-28T19:18:44.000Z",+ "meanEstimatedListCostUsd": 0.2800366666666667,+ "meanTokenConsumption": 23315.666666666668,"meanToolInvocationCount": 10.333333333333334},{@@ -568,11 +384,11 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 82465,- "measuredAt": "2026-08-19T21:45:40.684Z",- "meanEstimatedListCostUsd": 0.1743066666666667,- "meanTokenConsumption": 11940,- "meanToolInvocationCount": 10.333333333333334+ "meanDurationMs": 53286,+ "measuredAt": "2026-08-28T19:18:44.000Z",+ "meanEstimatedListCostUsd": 0.2505233333333334,+ "meanTokenConsumption": 19769.666666666668,+ "meanToolInvocationCount": 11},{"experimentId": "claude-fable-5-opencode--guided",@@ -580,11 +396,11 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 84096,- "measuredAt": "2026-08-19T21:43:56.532Z",- "meanEstimatedListCostUsd": 0.26636,- "meanTokenConsumption": 18337.333333333332,- "meanToolInvocationCount": 6.666666666666667+ "meanDurationMs": 53166,+ "measuredAt": "2026-08-28T19:18:27.000Z",+ "meanEstimatedListCostUsd": 0.3223233333333333,+ "meanTokenConsumption": 23616.333333333332,+ "meanToolInvocationCount": 7.333333333333333},{"experimentId": "claude-fable-5-opencode--guided",@@ -592,23 +408,23 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 114080,- "measuredAt": "2026-08-19T21:43:56.538Z",- "meanEstimatedListCostUsd": 0.20138,- "meanTokenConsumption": 14823.333333333334,- "meanToolInvocationCount": 9.333333333333334+ "meanDurationMs": 85836,+ "measuredAt": "2026-08-28T19:25:03.000Z",+ "meanEstimatedListCostUsd": 0.24718666666666667,+ "meanTokenConsumption": 19625.333333333332,+ "meanToolInvocationCount": 8.333333333333334},{"experimentId": "claude-fable-5-opencode--guided","caseId": "author-003-openapi-connection","status": "current",- "passedRuns": 3,+ "passedRuns": 0,"validRuns": 3,- "meanDurationMs": 86233,- "measuredAt": "2026-08-19T21:43:56.542Z",- "meanEstimatedListCostUsd": 0.15039666666666668,- "meanTokenConsumption": 10385,- "meanToolInvocationCount": 8+ "meanDurationMs": 277471,+ "measuredAt": "2026-08-28T19:18:27.000Z",+ "meanEstimatedListCostUsd": 0.4114,+ "meanTokenConsumption": 19754.666666666668,+ "meanToolInvocationCount": 7.666666666666667},{"experimentId": "claude-fable-5-opencode--guided",@@ -616,11 +432,11 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 94717,- "measuredAt": "2026-08-19T21:43:56.546Z",- "meanEstimatedListCostUsd": 0.24288333333333334,- "meanTokenConsumption": 15823,- "meanToolInvocationCount": 9.333333333333334+ "meanDurationMs": 70438,+ "measuredAt": "2026-08-28T19:18:27.000Z",+ "meanEstimatedListCostUsd": 0.25587,+ "meanTokenConsumption": 16792.333333333332,+ "meanToolInvocationCount": 11.333333333333334},{"experimentId": "claude-fable-5-opencode--guided",@@ -628,11 +444,11 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 72597,- "measuredAt": "2026-08-19T21:43:56.549Z",- "meanEstimatedListCostUsd": 0.14973333333333336,- "meanTokenConsumption": 10974.666666666666,- "meanToolInvocationCount": 6.333333333333333+ "meanDurationMs": 41451,+ "measuredAt": "2026-08-28T19:18:27.000Z",+ "meanEstimatedListCostUsd": 0.29036,+ "meanTokenConsumption": 24874.666666666668,+ "meanToolInvocationCount": 7},{"experimentId": "claude-fable-5-opencode--guided",@@ -640,11 +456,11 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 80685,- "measuredAt": "2026-08-19T21:43:56.553Z",- "meanEstimatedListCostUsd": 0.22277333333333335,- "meanTokenConsumption": 18986.666666666668,- "meanToolInvocationCount": 6.666666666666667+ "meanDurationMs": 39958,+ "measuredAt": "2026-08-28T19:18:27.000Z",+ "meanEstimatedListCostUsd": 0.21587,+ "meanTokenConsumption": 18063,+ "meanToolInvocationCount": 8},{"experimentId": "claude-fable-5-opencode--guided",@@ -652,23 +468,23 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 78720,- "measuredAt": "2026-08-19T21:43:56.556Z",- "meanEstimatedListCostUsd": 0.12009666666666667,- "meanTokenConsumption": 8643,- "meanToolInvocationCount": 6.666666666666667+ "meanDurationMs": 36170,+ "measuredAt": "2026-08-28T19:18:27.000Z",+ "meanEstimatedListCostUsd": 0.16552,+ "meanTokenConsumption": 13610.666666666666,+ "meanToolInvocationCount": 6.333333333333333},{"experimentId": "grok-4-6-opencode--baseline","caseId": "author-001-weather-tool","status": "current",- "passedRuns": 1,+ "passedRuns": 3,"validRuns": 3,- "meanDurationMs": 77369,- "measuredAt": "2026-08-19T21:03:51.002Z",- "meanEstimatedListCostUsd": 0.115442,- "meanTokenConsumption": 54837,- "meanToolInvocationCount": 18+ "meanDurationMs": 38613,+ "measuredAt": "2026-08-28T19:22:01.000Z",+ "meanEstimatedListCostUsd": 0.10434266666666665,+ "meanTokenConsumption": 49566,+ "meanToolInvocationCount": 19},{"experimentId": "grok-4-6-opencode--baseline",@@ -676,47 +492,47 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 153371,- "measuredAt": "2026-08-19T21:03:51.007Z",- "meanEstimatedListCostUsd": 0.07613733333333333,- "meanTokenConsumption": 35449.333333333336,- "meanToolInvocationCount": 17+ "meanDurationMs": 83985,+ "measuredAt": "2026-08-28T19:25:12.000Z",+ "meanEstimatedListCostUsd": 0.07232866666666667,+ "meanTokenConsumption": 32827.666666666664,+ "meanToolInvocationCount": 16},{"experimentId": "grok-4-6-opencode--baseline","caseId": "author-003-openapi-connection","status": "current",- "passedRuns": 3,+ "passedRuns": 0,"validRuns": 3,- "meanDurationMs": 183575,- "measuredAt": "2026-08-19T21:03:51.019Z",- "meanEstimatedListCostUsd": 0.20139533333333334,- "meanTokenConsumption": 96650.33333333333,- "meanToolInvocationCount": 27.666666666666668+ "meanDurationMs": 521954,+ "measuredAt": "2026-08-28T19:22:01.000Z",+ "meanEstimatedListCostUsd": 1.0741786666666668,+ "meanTokenConsumption": 503422.6666666667,+ "meanToolInvocationCount": 76.66666666666667},{"experimentId": "grok-4-6-opencode--baseline","caseId": "author-004-packaged-skill","status": "current",- "passedRuns": 1,+ "passedRuns": 3,"validRuns": 3,- "meanDurationMs": 139390,- "measuredAt": "2026-08-19T20:13:40.145Z",- "meanEstimatedListCostUsd": 0.107158,- "meanTokenConsumption": 50371,- "meanToolInvocationCount": 23+ "meanDurationMs": 41894,+ "measuredAt": "2026-08-28T19:22:01.000Z",+ "meanEstimatedListCostUsd": 0.065152,+ "meanTokenConsumption": 29506,+ "meanToolInvocationCount": 23.333333333333332},{"experimentId": "grok-4-6-opencode--baseline","caseId": "author-005-conditional-approval","status": "current",- "passedRuns": 1,+ "passedRuns": 3,"validRuns": 3,- "meanDurationMs": 144404,- "measuredAt": "2026-08-19T20:13:40.148Z",- "meanEstimatedListCostUsd": 0.088416,- "meanTokenConsumption": 39046,- "meanToolInvocationCount": 26+ "meanDurationMs": 50151,+ "measuredAt": "2026-08-28T19:22:01.000Z",+ "meanEstimatedListCostUsd": 0.141536,+ "meanTokenConsumption": 66629.33333333333,+ "meanToolInvocationCount": 22},{"experimentId": "grok-4-6-opencode--baseline",@@ -724,11 +540,11 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 61386,- "measuredAt": "2026-08-19T22:10:06.843Z",- "meanEstimatedListCostUsd": 0.08925733333333334,- "meanTokenConsumption": 42053.333333333336,- "meanToolInvocationCount": 18.666666666666668+ "meanDurationMs": 37062,+ "measuredAt": "2026-08-28T19:22:01.000Z",+ "meanEstimatedListCostUsd": 0.07319533333333333,+ "meanTokenConsumption": 33801.666666666664,+ "meanToolInvocationCount": 17.666666666666668},{"experimentId": "grok-4-6-opencode--baseline",@@ -736,23 +552,23 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 57619,- "measuredAt": "2026-08-19T20:30:55.587Z",- "meanEstimatedListCostUsd": 0.10759599999999998,- "meanTokenConsumption": 50025.333333333336,- "meanToolInvocationCount": 24.333333333333332+ "meanDurationMs": 38280,+ "measuredAt": "2026-08-28T19:22:01.000Z",+ "meanEstimatedListCostUsd": 0.045748000000000004,+ "meanTokenConsumption": 20122.666666666668,+ "meanToolInvocationCount": 18},{"experimentId": "grok-4-6-opencode--guided","caseId": "author-001-weather-tool","status": "current",- "passedRuns": 2,+ "passedRuns": 3,"validRuns": 3,- "meanDurationMs": 139801,- "measuredAt": "2026-08-19T21:03:56.151Z",- "meanEstimatedListCostUsd": 0.07778299999999999,- "meanTokenConsumption": 36365.5,- "meanToolInvocationCount": 13+ "meanDurationMs": 31700,+ "measuredAt": "2026-08-28T19:22:02.000Z",+ "meanEstimatedListCostUsd": 0.049695333333333334,+ "meanTokenConsumption": 22963.666666666668,+ "meanToolInvocationCount": 9.666666666666666},{"experimentId": "grok-4-6-opencode--guided",@@ -760,11 +576,11 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 144413,- "measuredAt": "2026-08-19T21:03:56.161Z",- "meanEstimatedListCostUsd": 0.07078,- "meanTokenConsumption": 32605.333333333332,- "meanToolInvocationCount": 17.333333333333332+ "meanDurationMs": 83621,+ "measuredAt": "2026-08-28T19:25:07.000Z",+ "meanEstimatedListCostUsd": 0.082842,+ "meanTokenConsumption": 38261,+ "meanToolInvocationCount": 16.333333333333332},{"experimentId": "grok-4-6-opencode--guided",@@ -772,35 +588,35 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 159516,- "measuredAt": "2026-08-19T21:03:56.168Z",- "meanEstimatedListCostUsd": 0.10099799999999999,- "meanTokenConsumption": 49024.333333333336,- "meanToolInvocationCount": 10.666666666666666+ "meanDurationMs": 34833,+ "measuredAt": "2026-08-28T19:25:07.000Z",+ "meanEstimatedListCostUsd": 0.055174,+ "meanTokenConsumption": 25952.333333333332,+ "meanToolInvocationCount": 10},{"experimentId": "grok-4-6-opencode--guided","caseId": "author-004-packaged-skill","status": "current",- "passedRuns": 2,+ "passedRuns": 3,"validRuns": 3,- "meanDurationMs": 143386,- "measuredAt": "2026-08-19T21:03:56.173Z",- "meanEstimatedListCostUsd": 0.067512,- "meanTokenConsumption": 31751,- "meanToolInvocationCount": 8.5+ "meanDurationMs": 30135,+ "measuredAt": "2026-08-28T19:22:02.000Z",+ "meanEstimatedListCostUsd": 0.035581999999999996,+ "meanTokenConsumption": 16043.666666666666,+ "meanToolInvocationCount": 8.666666666666666},{"experimentId": "grok-4-6-opencode--guided","caseId": "author-005-conditional-approval","status": "current",- "passedRuns": 1,+ "passedRuns": 3,"validRuns": 3,- "meanDurationMs": 75491,- "measuredAt": "2026-08-19T21:03:56.177Z",- "meanEstimatedListCostUsd": 0.04133,- "meanTokenConsumption": 18681,- "meanToolInvocationCount": 14+ "meanDurationMs": 34026,+ "measuredAt": "2026-08-28T19:22:02.000Z",+ "meanEstimatedListCostUsd": 0.038532666666666666,+ "meanTokenConsumption": 17059.666666666668,+ "meanToolInvocationCount": 11.666666666666666},{"experimentId": "grok-4-6-opencode--guided",@@ -808,10 +624,10 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 30928,- "measuredAt": "2026-08-19T20:23:28.755Z",- "meanEstimatedListCostUsd": 0.026432,- "meanTokenConsumption": 11709.333333333334,+ "meanDurationMs": 32109,+ "measuredAt": "2026-08-28T19:22:02.000Z",+ "meanEstimatedListCostUsd": 0.033709333333333334,+ "meanTokenConsumption": 14610,"meanToolInvocationCount": 9.666666666666666},{@@ -820,10 +636,10 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 33100,- "measuredAt": "2026-08-19T20:23:28.761Z",- "meanEstimatedListCostUsd": 0.03239266666666667,- "meanTokenConsumption": 14937,+ "meanDurationMs": 25583,+ "measuredAt": "2026-08-28T19:22:02.000Z",+ "meanEstimatedListCostUsd": 0.023458666666666666,+ "meanTokenConsumption": 10328,"meanToolInvocationCount": 7.333333333333333},{@@ -832,10 +648,10 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 70992,- "measuredAt": "2026-08-19T21:47:37.853Z",- "meanEstimatedListCostUsd": 0.10240083333333333,- "meanTokenConsumption": 27690.333333333332,+ "meanDurationMs": 62123,+ "measuredAt": "2026-08-28T19:16:59.000Z",+ "meanEstimatedListCostUsd": 0.13379416666666666,+ "meanTokenConsumption": 40526,"meanToolInvocationCount": 27.666666666666668},{@@ -844,11 +660,11 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 85989,- "measuredAt": "2026-08-19T21:47:37.868Z",- "meanEstimatedListCostUsd": 0.054655,- "meanTokenConsumption": 16612,- "meanToolInvocationCount": 15+ "meanDurationMs": 86110,+ "measuredAt": "2026-08-28T19:25:11.000Z",+ "meanEstimatedListCostUsd": 0.05052583333333333,+ "meanTokenConsumption": 14690.333333333334,+ "meanToolInvocationCount": 15.666666666666666},{"experimentId": "gpt-5-6-sol-opencode--baseline",@@ -856,22 +672,22 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 70955,- "measuredAt": "2026-08-19T21:47:37.878Z",- "meanEstimatedListCostUsd": 0.13146500000000003,- "meanTokenConsumption": 43889.333333333336,- "meanToolInvocationCount": 28.333333333333332+ "meanDurationMs": 62925,+ "measuredAt": "2026-08-28T19:25:11.000Z",+ "meanEstimatedListCostUsd": 0.09841583333333333,+ "meanTokenConsumption": 31109.666666666668,+ "meanToolInvocationCount": 27.333333333333332},{"experimentId": "gpt-5-6-sol-opencode--baseline","caseId": "author-004-packaged-skill","status": "current",- "passedRuns": 3,+ "passedRuns": 2,"validRuns": 3,- "meanDurationMs": 68796,- "measuredAt": "2026-08-19T21:47:37.886Z",- "meanEstimatedListCostUsd": 0.07487333333333333,- "meanTokenConsumption": 16249.333333333334,+ "meanDurationMs": 60423,+ "measuredAt": "2026-08-28T19:16:59.000Z",+ "meanEstimatedListCostUsd": 0.04697083333333333,+ "meanTokenConsumption": 7770,"meanToolInvocationCount": 17.666666666666668},{@@ -880,11 +696,11 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 50730,- "measuredAt": "2026-08-19T21:47:37.891Z",- "meanEstimatedListCostUsd": 0.07232916666666667,- "meanTokenConsumption": 21083.333333333332,- "meanToolInvocationCount": 21.666666666666668+ "meanDurationMs": 63881,+ "measuredAt": "2026-08-28T19:16:59.000Z",+ "meanEstimatedListCostUsd": 0.10394666666666667,+ "meanTokenConsumption": 32730.333333333332,+ "meanToolInvocationCount": 29},{"experimentId": "gpt-5-6-sol-opencode--baseline",@@ -892,23 +708,23 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 66339,- "measuredAt": "2026-08-19T21:47:37.896Z",- "meanEstimatedListCostUsd": 0.09810583333333334,- "meanTokenConsumption": 31694,- "meanToolInvocationCount": 25+ "meanDurationMs": 47385,+ "measuredAt": "2026-08-28T19:16:59.000Z",+ "meanEstimatedListCostUsd": 0.07596750000000001,+ "meanTokenConsumption": 23072,+ "meanToolInvocationCount": 24.333333333333332},{"experimentId": "gpt-5-6-sol-opencode--baseline","caseId": "author-007-digest-schedule","status": "current",- "passedRuns": 2,+ "passedRuns": 3,"validRuns": 3,- "meanDurationMs": 43646,- "measuredAt": "2026-08-19T21:47:37.900Z",- "meanEstimatedListCostUsd": 0.06730583333333333,- "meanTokenConsumption": 21627.333333333332,- "meanToolInvocationCount": 20.666666666666668+ "meanDurationMs": 44948,+ "measuredAt": "2026-08-28T19:16:59.000Z",+ "meanEstimatedListCostUsd": 0.0608525,+ "meanTokenConsumption": 18521,+ "meanToolInvocationCount": 22},{"experimentId": "gpt-5-6-sol-opencode--guided",@@ -916,11 +732,11 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 49812,- "measuredAt": "2026-08-19T21:47:41.834Z",- "meanEstimatedListCostUsd": 0.05889416666666667,- "meanTokenConsumption": 13082.666666666666,- "meanToolInvocationCount": 13.333333333333334+ "meanDurationMs": 43515,+ "measuredAt": "2026-08-28T19:16:11.000Z",+ "meanEstimatedListCostUsd": 0.06586666666666667,+ "meanTokenConsumption": 17868.333333333332,+ "meanToolInvocationCount": 11},{"experimentId": "gpt-5-6-sol-opencode--guided",@@ -928,11 +744,11 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 85597,- "measuredAt": "2026-08-19T21:47:41.847Z",- "meanEstimatedListCostUsd": 0.0342875,- "meanTokenConsumption": 8640,- "meanToolInvocationCount": 14.333333333333334+ "meanDurationMs": 80901,+ "measuredAt": "2026-08-28T19:25:01.000Z",+ "meanEstimatedListCostUsd": 0.05137250000000001,+ "meanTokenConsumption": 15322.333333333334,+ "meanToolInvocationCount": 15},{"experimentId": "gpt-5-6-sol-opencode--guided",@@ -940,11 +756,11 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 43078,- "measuredAt": "2026-08-19T21:47:41.856Z",- "meanEstimatedListCostUsd": 0.04016083333333333,- "meanTokenConsumption": 11582.666666666666,- "meanToolInvocationCount": 12+ "meanDurationMs": 42021,+ "measuredAt": "2026-08-28T19:25:01.000Z",+ "meanEstimatedListCostUsd": 0.04389166666666666,+ "meanTokenConsumption": 13418.333333333334,+ "meanToolInvocationCount": 9},{"experimentId": "gpt-5-6-sol-opencode--guided",@@ -952,11 +768,11 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 57028,- "measuredAt": "2026-08-19T21:47:41.862Z",- "meanEstimatedListCostUsd": 0.04584583333333333,- "meanTokenConsumption": 9393.333333333334,- "meanToolInvocationCount": 7.333333333333333+ "meanDurationMs": 45659,+ "measuredAt": "2026-08-28T19:16:11.000Z",+ "meanEstimatedListCostUsd": 0.04002,+ "meanTokenConsumption": 7084.666666666667,+ "meanToolInvocationCount": 6.666666666666667},{"experimentId": "gpt-5-6-sol-opencode--guided",@@ -964,10 +780,10 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 46860,- "measuredAt": "2026-08-19T21:47:41.867Z",- "meanEstimatedListCostUsd": 0.0345875,- "meanTokenConsumption": 9476.666666666666,+ "meanDurationMs": 34718,+ "measuredAt": "2026-08-28T19:16:11.000Z",+ "meanEstimatedListCostUsd": 0.04937916666666666,+ "meanTokenConsumption": 15085,"meanToolInvocationCount": 10.333333333333334},{@@ -976,11 +792,11 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 43671,- "measuredAt": "2026-08-19T21:47:41.871Z",- "meanEstimatedListCostUsd": 0.03670166666666667,- "meanTokenConsumption": 11112.333333333334,- "meanToolInvocationCount": 9.333333333333334+ "meanDurationMs": 36827,+ "measuredAt": "2026-08-28T19:16:11.000Z",+ "meanEstimatedListCostUsd": 0.0360025,+ "meanTokenConsumption": 9957.666666666666,+ "meanToolInvocationCount": 12},{"experimentId": "gpt-5-6-sol-opencode--guided",@@ -988,47 +804,47 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 39656,- "measuredAt": "2026-08-19T21:47:41.875Z",- "meanEstimatedListCostUsd": 0.036234166666666665,- "meanTokenConsumption": 10917,- "meanToolInvocationCount": 9.333333333333334+ "meanDurationMs": 28258,+ "measuredAt": "2026-08-28T19:16:11.000Z",+ "meanEstimatedListCostUsd": 0.042734999999999995,+ "meanTokenConsumption": 14049,+ "meanToolInvocationCount": 8.333333333333334},{"experimentId": "gpt-5-6-terra-opencode--baseline","caseId": "author-001-weather-tool","status": "current",- "passedRuns": 3,+ "passedRuns": 2,"validRuns": 3,- "meanDurationMs": 51999,- "measuredAt": "2026-08-19T21:49:40.887Z",- "meanEstimatedListCostUsd": 0.078974,- "meanTokenConsumption": 27382,- "meanToolInvocationCount": 28.666666666666668+ "meanDurationMs": 42935,+ "measuredAt": "2026-08-28T19:18:05.000Z",+ "meanEstimatedListCostUsd": 0.062688,+ "meanTokenConsumption": 22812.333333333332,+ "meanToolInvocationCount": 19},{"experimentId": "gpt-5-6-terra-opencode--baseline","caseId": "author-002-new-project","status": "current",- "passedRuns": 3,+ "passedRuns": 0,"validRuns": 3,- "meanDurationMs": 79557,- "measuredAt": "2026-08-19T21:49:40.898Z",- "meanEstimatedListCostUsd": 0.028153333333333336,- "meanTokenConsumption": 8605,- "meanToolInvocationCount": 15.666666666666666+ "meanDurationMs": 262688,+ "measuredAt": "2026-08-28T19:18:05.000Z",+ "meanEstimatedListCostUsd": 0.06566466666666666,+ "meanTokenConsumption": 10689,+ "meanToolInvocationCount": 21},{"experimentId": "gpt-5-6-terra-opencode--baseline","caseId": "author-003-openapi-connection","status": "current",- "passedRuns": 1,+ "passedRuns": 0,"validRuns": 3,- "meanDurationMs": 39576,- "measuredAt": "2026-08-19T21:49:40.904Z",- "meanEstimatedListCostUsd": 0.04701266666666667,- "meanTokenConsumption": 17094.666666666668,- "meanToolInvocationCount": 22.333333333333332+ "meanDurationMs": 119239,+ "measuredAt": "2026-08-28T19:18:05.000Z",+ "meanEstimatedListCostUsd": 0.05318866666666666,+ "meanTokenConsumption": 17214.333333333332,+ "meanToolInvocationCount": 12},{"experimentId": "gpt-5-6-terra-opencode--baseline",@@ -1036,47 +852,47 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 47507,- "measuredAt": "2026-08-19T21:49:40.910Z",- "meanEstimatedListCostUsd": 0.03413466666666667,- "meanTokenConsumption": 7104,- "meanToolInvocationCount": 14+ "meanDurationMs": 49408,+ "measuredAt": "2026-08-28T19:18:05.000Z",+ "meanEstimatedListCostUsd": 0.04473866666666667,+ "meanTokenConsumption": 11359.333333333334,+ "meanToolInvocationCount": 15},{"experimentId": "gpt-5-6-terra-opencode--baseline","caseId": "author-005-conditional-approval","status": "current",- "passedRuns": 3,+ "passedRuns": 2,"validRuns": 3,- "meanDurationMs": 56880,- "measuredAt": "2026-08-19T21:49:40.918Z",- "meanEstimatedListCostUsd": 0.081758,- "meanTokenConsumption": 32617.333333333332,- "meanToolInvocationCount": 27+ "meanDurationMs": 51631,+ "measuredAt": "2026-08-28T19:18:05.000Z",+ "meanEstimatedListCostUsd": 0.08744866666666666,+ "meanTokenConsumption": 34586,+ "meanToolInvocationCount": 27.333333333333332},{"experimentId": "gpt-5-6-terra-opencode--baseline","caseId": "author-006-custom-channel","status": "current",- "passedRuns": 3,+ "passedRuns": 2,"validRuns": 3,- "meanDurationMs": 46407,- "measuredAt": "2026-08-19T21:49:40.923Z",- "meanEstimatedListCostUsd": 0.09911933333333334,- "meanTokenConsumption": 42253,- "meanToolInvocationCount": 23+ "meanDurationMs": 34464,+ "measuredAt": "2026-08-28T19:18:05.000Z",+ "meanEstimatedListCostUsd": 0.054481999999999996,+ "meanTokenConsumption": 21539.333333333332,+ "meanToolInvocationCount": 19.666666666666668},{"experimentId": "gpt-5-6-terra-opencode--baseline","caseId": "author-007-digest-schedule","status": "current",- "passedRuns": 3,+ "passedRuns": 2,"validRuns": 3,- "meanDurationMs": 42246,- "measuredAt": "2026-08-19T21:49:40.927Z",- "meanEstimatedListCostUsd": 0.057112,- "meanTokenConsumption": 22742.666666666668,- "meanToolInvocationCount": 22+ "meanDurationMs": 30799,+ "measuredAt": "2026-08-28T19:18:05.000Z",+ "meanEstimatedListCostUsd": 0.03727,+ "meanTokenConsumption": 14183.333333333334,+ "meanToolInvocationCount": 14.333333333333334},{"experimentId": "gpt-5-6-terra-opencode--guided",@@ -1084,11 +900,11 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 41608,- "measuredAt": "2026-08-19T21:49:28.518Z",- "meanEstimatedListCostUsd": 0.06261066666666666,- "meanTokenConsumption": 21113.666666666668,- "meanToolInvocationCount": 17+ "meanDurationMs": 42543,+ "measuredAt": "2026-08-28T19:16:16.000Z",+ "meanEstimatedListCostUsd": 0.053156666666666665,+ "meanTokenConsumption": 16486.666666666668,+ "meanToolInvocationCount": 16},{"experimentId": "gpt-5-6-terra-opencode--guided",@@ -1096,11 +912,11 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 74695,- "measuredAt": "2026-08-19T21:49:28.526Z",- "meanEstimatedListCostUsd": 0.049492666666666664,- "meanTokenConsumption": 19446.333333333332,- "meanToolInvocationCount": 15.666666666666666+ "meanDurationMs": 80660,+ "measuredAt": "2026-08-28T19:25:15.000Z",+ "meanEstimatedListCostUsd": 0.04396933333333333,+ "meanTokenConsumption": 16811.333333333332,+ "meanToolInvocationCount": 14},{"experimentId": "gpt-5-6-terra-opencode--guided",@@ -1108,11 +924,11 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 40796,- "measuredAt": "2026-08-19T21:49:28.533Z",- "meanEstimatedListCostUsd": 0.042858,- "meanTokenConsumption": 17250.666666666668,- "meanToolInvocationCount": 11.333333333333334+ "meanDurationMs": 44816,+ "measuredAt": "2026-08-28T19:25:15.000Z",+ "meanEstimatedListCostUsd": 0.040738,+ "meanTokenConsumption": 14277.333333333334,+ "meanToolInvocationCount": 18},{"experimentId": "gpt-5-6-terra-opencode--guided",@@ -1120,23 +936,23 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 44243,- "measuredAt": "2026-08-19T21:49:28.538Z",- "meanEstimatedListCostUsd": 0.04276266666666667,- "meanTokenConsumption": 11531.333333333334,- "meanToolInvocationCount": 14+ "meanDurationMs": 44710,+ "measuredAt": "2026-08-28T19:16:16.000Z",+ "meanEstimatedListCostUsd": 0.044548,+ "meanTokenConsumption": 13047.333333333334,+ "meanToolInvocationCount": 13.333333333333334},{"experimentId": "gpt-5-6-terra-opencode--guided","caseId": "author-005-conditional-approval","status": "current",- "passedRuns": 2,+ "passedRuns": 3,"validRuns": 3,- "meanDurationMs": 33729,- "measuredAt": "2026-08-19T21:49:28.542Z",- "meanEstimatedListCostUsd": 0.03496066666666667,- "meanTokenConsumption": 12010.333333333334,- "meanToolInvocationCount": 14.666666666666666+ "meanDurationMs": 36268,+ "measuredAt": "2026-08-28T19:16:16.000Z",+ "meanEstimatedListCostUsd": 0.046585999999999995,+ "meanTokenConsumption": 17268,+ "meanToolInvocationCount": 16},{"experimentId": "gpt-5-6-terra-opencode--guided",@@ -1144,11 +960,11 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 37089,- "measuredAt": "2026-08-19T21:49:28.546Z",- "meanEstimatedListCostUsd": 0.02781866666666667,- "meanTokenConsumption": 9269.333333333334,- "meanToolInvocationCount": 13+ "meanDurationMs": 34596,+ "measuredAt": "2026-08-28T19:16:16.000Z",+ "meanEstimatedListCostUsd": 0.04468933333333333,+ "meanTokenConsumption": 16843,+ "meanToolInvocationCount": 15.333333333333334},{"experimentId": "gpt-5-6-terra-opencode--guided",@@ -1156,11 +972,11 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 36703,- "measuredAt": "2026-08-19T21:49:28.550Z",- "meanEstimatedListCostUsd": 0.022504,- "meanTokenConsumption": 7360.333333333333,- "meanToolInvocationCount": 11.333333333333334+ "meanDurationMs": 27983,+ "measuredAt": "2026-08-28T19:16:16.000Z",+ "meanEstimatedListCostUsd": 0.022269333333333335,+ "meanTokenConsumption": 7259.666666666667,+ "meanToolInvocationCount": 11.666666666666666},{"experimentId": "claude-sonnet-5-opencode--baseline",@@ -1168,11 +984,11 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 59225,- "measuredAt": "2026-08-19T21:58:17.895Z",- "meanEstimatedListCostUsd": 0.08490400000000002,- "meanTokenConsumption": 33980,- "meanToolInvocationCount": 13+ "meanDurationMs": 84504,+ "measuredAt": "2026-08-28T19:20:33.000Z",+ "meanEstimatedListCostUsd": 0.087304,+ "meanTokenConsumption": 30901.333333333332,+ "meanToolInvocationCount": 17.666666666666668},{"experimentId": "claude-sonnet-5-opencode--baseline",@@ -1180,11 +996,11 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 90978,- "measuredAt": "2026-08-19T21:58:17.902Z",- "meanEstimatedListCostUsd": 0.041563333333333334,- "meanTokenConsumption": 13628.333333333334,- "meanToolInvocationCount": 12+ "meanDurationMs": 85287,+ "measuredAt": "2026-08-28T19:27:58.000Z",+ "meanEstimatedListCostUsd": 0.047190666666666665,+ "meanTokenConsumption": 16207.333333333334,+ "meanToolInvocationCount": 12.333333333333334},{"experimentId": "claude-sonnet-5-opencode--baseline",@@ -1192,11 +1008,11 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 195665,- "measuredAt": "2026-08-19T21:58:17.912Z",- "meanEstimatedListCostUsd": 0.1760786666666667,- "meanTokenConsumption": 50079.333333333336,- "meanToolInvocationCount": 39.666666666666664+ "meanDurationMs": 178438,+ "measuredAt": "2026-08-28T19:27:58.000Z",+ "meanEstimatedListCostUsd": 0.23833933333333335,+ "meanTokenConsumption": 86543,+ "meanToolInvocationCount": 37},{"experimentId": "claude-sonnet-5-opencode--baseline",@@ -1204,11 +1020,11 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 108879,- "measuredAt": "2026-08-19T21:58:17.920Z",- "meanEstimatedListCostUsd": 0.14531933333333333,- "meanTokenConsumption": 52455.666666666664,- "meanToolInvocationCount": 20.666666666666668+ "meanDurationMs": 69785,+ "measuredAt": "2026-08-28T19:20:33.000Z",+ "meanEstimatedListCostUsd": 0.05839,+ "meanTokenConsumption": 16595,+ "meanToolInvocationCount": 14},{"experimentId": "claude-sonnet-5-opencode--baseline",@@ -1216,11 +1032,11 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 95353,- "measuredAt": "2026-08-19T21:58:17.926Z",- "meanEstimatedListCostUsd": 0.11524333333333335,- "meanTokenConsumption": 43313.666666666664,- "meanToolInvocationCount": 22.666666666666668+ "meanDurationMs": 69671,+ "measuredAt": "2026-08-28T19:20:33.000Z",+ "meanEstimatedListCostUsd": 0.08692,+ "meanTokenConsumption": 33568,+ "meanToolInvocationCount": 15.666666666666666},{"experimentId": "claude-sonnet-5-opencode--baseline",@@ -1228,23 +1044,23 @@"status": "current","passedRuns": 1,"validRuns": 3,- "meanDurationMs": 291562,- "measuredAt": "2026-08-19T21:58:17.931Z",- "meanEstimatedListCostUsd": 0.045122666666666665,- "meanTokenConsumption": 17692,- "meanToolInvocationCount": 10+ "meanDurationMs": 352409,+ "measuredAt": "2026-08-28T19:20:33.000Z",+ "meanEstimatedListCostUsd": 0.017415333333333335,+ "meanTokenConsumption": 5797,+ "meanToolInvocationCount": 6},{"experimentId": "claude-sonnet-5-opencode--baseline","caseId": "author-007-digest-schedule","status": "current",- "passedRuns": 2,+ "passedRuns": 3,"validRuns": 3,- "meanDurationMs": 116395,- "measuredAt": "2026-08-19T21:58:17.936Z",- "meanEstimatedListCostUsd": 0.10074666666666666,- "meanTokenConsumption": 33780,- "meanToolInvocationCount": 19.333333333333332+ "meanDurationMs": 69696,+ "measuredAt": "2026-08-28T19:20:33.000Z",+ "meanEstimatedListCostUsd": 0.06399133333333333,+ "meanTokenConsumption": 21950.333333333332,+ "meanToolInvocationCount": 14},{"experimentId": "claude-sonnet-5-opencode--guided",@@ -1252,11 +1068,11 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 105884,- "measuredAt": "2026-08-19T21:52:41.883Z",- "meanEstimatedListCostUsd": 0.09806266666666667,- "meanTokenConsumption": 30726,- "meanToolInvocationCount": 14.666666666666666+ "meanDurationMs": 72487,+ "measuredAt": "2026-08-28T19:19:54.000Z",+ "meanEstimatedListCostUsd": 0.06199066666666666,+ "meanTokenConsumption": 20955.333333333332,+ "meanToolInvocationCount": 12.333333333333334},{"experimentId": "claude-sonnet-5-opencode--guided",@@ -1264,11 +1080,11 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 106358,- "measuredAt": "2026-08-19T21:52:41.896Z",- "meanEstimatedListCostUsd": 0.06011466666666667,- "meanTokenConsumption": 20532,- "meanToolInvocationCount": 14.666666666666666+ "meanDurationMs": 97671,+ "measuredAt": "2026-08-28T19:25:30.000Z",+ "meanEstimatedListCostUsd": 0.037938000000000006,+ "meanTokenConsumption": 11189,+ "meanToolInvocationCount": 13.666666666666666},{"experimentId": "claude-sonnet-5-opencode--guided",@@ -1276,11 +1092,11 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 57823,- "measuredAt": "2026-08-19T21:52:41.905Z",- "meanEstimatedListCostUsd": 0.047889999999999995,- "meanTokenConsumption": 17711.666666666668,- "meanToolInvocationCount": 11.333333333333334+ "meanDurationMs": 53872,+ "measuredAt": "2026-08-28T19:25:30.000Z",+ "meanEstimatedListCostUsd": 0.06860866666666667,+ "meanTokenConsumption": 27953.666666666668,+ "meanToolInvocationCount": 12},{"experimentId": "claude-sonnet-5-opencode--guided",@@ -1288,11 +1104,11 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 50374,- "measuredAt": "2026-08-19T21:52:41.911Z",- "meanEstimatedListCostUsd": 0.04749,- "meanTokenConsumption": 16147.666666666666,- "meanToolInvocationCount": 8+ "meanDurationMs": 54347,+ "measuredAt": "2026-08-28T19:19:54.000Z",+ "meanEstimatedListCostUsd": 0.05166133333333334,+ "meanTokenConsumption": 17134.666666666668,+ "meanToolInvocationCount": 10.333333333333334},{"experimentId": "claude-sonnet-5-opencode--guided",@@ -1300,11 +1116,11 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 44694,- "measuredAt": "2026-08-19T21:52:41.917Z",- "meanEstimatedListCostUsd": 0.05203,- "meanTokenConsumption": 20456.333333333332,- "meanToolInvocationCount": 9.666666666666666+ "meanDurationMs": 57140,+ "measuredAt": "2026-08-28T19:19:54.000Z",+ "meanEstimatedListCostUsd": 0.04402533333333333,+ "meanTokenConsumption": 15630,+ "meanToolInvocationCount": 12},{"experimentId": "claude-sonnet-5-opencode--guided",@@ -1312,11 +1128,11 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 45902,- "measuredAt": "2026-08-19T21:52:41.922Z",- "meanEstimatedListCostUsd": 0.05530466666666667,- "meanTokenConsumption": 22887,- "meanToolInvocationCount": 10+ "meanDurationMs": 46268,+ "measuredAt": "2026-08-28T19:19:54.000Z",+ "meanEstimatedListCostUsd": 0.026169333333333333,+ "meanTokenConsumption": 8867.333333333334,+ "meanToolInvocationCount": 8},{"experimentId": "claude-sonnet-5-opencode--guided",@@ -1324,11 +1140,11 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 37072,- "measuredAt": "2026-08-19T21:52:41.927Z",- "meanEstimatedListCostUsd": 0.017157333333333333,- "meanTokenConsumption": 6138.666666666667,- "meanToolInvocationCount": 5.333333333333333+ "meanDurationMs": 27822,+ "measuredAt": "2026-08-28T19:19:54.000Z",+ "meanEstimatedListCostUsd": 0.01684333333333333,+ "meanTokenConsumption": 5760.333333333333,+ "meanToolInvocationCount": 6},{"experimentId": "glm-5-2-opencode--baseline",@@ -1336,11 +1152,11 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 45432,- "measuredAt": "2026-08-19T22:00:25.730Z",- "meanEstimatedListCostUsd": 0.018507533333333333,- "meanTokenConsumption": 11404.666666666666,- "meanToolInvocationCount": 17.333333333333332+ "meanDurationMs": 103530,+ "measuredAt": "2026-08-28T19:22:09.000Z",+ "meanEstimatedListCostUsd": 0.07017806666666666,+ "meanTokenConsumption": 37804.333333333336,+ "meanToolInvocationCount": 25},{"experimentId": "glm-5-2-opencode--baseline",@@ -1348,11 +1164,11 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 80706,- "measuredAt": "2026-08-19T22:00:25.746Z",- "meanEstimatedListCostUsd": 0.011617466666666668,- "meanTokenConsumption": 6995.333333333333,- "meanToolInvocationCount": 11.333333333333334+ "meanDurationMs": 100847,+ "measuredAt": "2026-08-28T19:26:40.000Z",+ "meanEstimatedListCostUsd": 0.03248553333333333,+ "meanTokenConsumption": 19034.666666666668,+ "meanToolInvocationCount": 18.333333333333332},{"experimentId": "glm-5-2-opencode--baseline",@@ -1360,11 +1176,11 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 60232,- "measuredAt": "2026-08-19T22:00:25.755Z",- "meanEstimatedListCostUsd": 0.027101133333333333,- "meanTokenConsumption": 15908.666666666666,- "meanToolInvocationCount": 28+ "meanDurationMs": 134009,+ "measuredAt": "2026-08-28T19:26:40.000Z",+ "meanEstimatedListCostUsd": 0.09391753333333334,+ "meanTokenConsumption": 49094.666666666664,+ "meanToolInvocationCount": 40.333333333333336},{"experimentId": "glm-5-2-opencode--baseline",@@ -1372,23 +1188,23 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 49435,- "measuredAt": "2026-08-19T22:00:25.762Z",- "meanEstimatedListCostUsd": 0.018872933333333335,- "meanTokenConsumption": 8945.666666666666,- "meanToolInvocationCount": 13.666666666666666+ "meanDurationMs": 214119,+ "measuredAt": "2026-08-28T19:22:09.000Z",+ "meanEstimatedListCostUsd": 0.07759993333333333,+ "meanTokenConsumption": 41325.666666666664,+ "meanToolInvocationCount": 36.333333333333336},{"experimentId": "glm-5-2-opencode--baseline","caseId": "author-005-conditional-approval","status": "current",- "passedRuns": 2,+ "passedRuns": 3,"validRuns": 3,- "meanDurationMs": 60175,- "measuredAt": "2026-08-19T22:00:25.768Z",- "meanEstimatedListCostUsd": 0.02182706666666667,- "meanTokenConsumption": 13249.333333333334,- "meanToolInvocationCount": 20.333333333333332+ "meanDurationMs": 58527,+ "measuredAt": "2026-08-28T19:22:09.000Z",+ "meanEstimatedListCostUsd": 0.07383253333333333,+ "meanTokenConsumption": 48329.666666666664,+ "meanToolInvocationCount": 25.333333333333332},{"experimentId": "glm-5-2-opencode--baseline",@@ -1396,23 +1212,23 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 69165,- "measuredAt": "2026-08-19T22:00:25.773Z",- "meanEstimatedListCostUsd": 0.04950993333333333,- "meanTokenConsumption": 30240.666666666668,+ "meanDurationMs": 75109,+ "measuredAt": "2026-08-28T19:22:09.000Z",+ "meanEstimatedListCostUsd": 0.07836139999999998,+ "meanTokenConsumption": 48506,"meanToolInvocationCount": 27.666666666666668},{"experimentId": "glm-5-2-opencode--baseline","caseId": "author-007-digest-schedule","status": "current",- "passedRuns": 1,+ "passedRuns": 3,"validRuns": 3,- "meanDurationMs": 85415,- "measuredAt": "2026-08-19T22:00:25.778Z",- "meanEstimatedListCostUsd": 0.0311514,- "meanTokenConsumption": 16861,- "meanToolInvocationCount": 15.333333333333334+ "meanDurationMs": 93384,+ "measuredAt": "2026-08-28T19:22:09.000Z",+ "meanEstimatedListCostUsd": 0.04282053333333333,+ "meanTokenConsumption": 24309.666666666668,+ "meanToolInvocationCount": 19},{"experimentId": "glm-5-2-opencode--guided",@@ -1420,11 +1236,11 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 44063,- "measuredAt": "2026-08-19T22:00:30.879Z",- "meanEstimatedListCostUsd": 0.009911666666666666,- "meanTokenConsumption": 5828.333333333333,- "meanToolInvocationCount": 9.333333333333334+ "meanDurationMs": 68607,+ "measuredAt": "2026-08-28T19:22:01.000Z",+ "meanEstimatedListCostUsd": 0.04188693333333334,+ "meanTokenConsumption": 22360.666666666668,+ "meanToolInvocationCount": 16.333333333333332},{"experimentId": "glm-5-2-opencode--guided",@@ -1432,23 +1248,23 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 81227,- "measuredAt": "2026-08-19T22:00:30.891Z",- "meanEstimatedListCostUsd": 0.010340466666666666,- "meanTokenConsumption": 5905.333333333333,- "meanToolInvocationCount": 10.666666666666666+ "meanDurationMs": 90517,+ "measuredAt": "2026-08-28T19:25:29.000Z",+ "meanEstimatedListCostUsd": 0.02843253333333333,+ "meanTokenConsumption": 16824.666666666668,+ "meanToolInvocationCount": 14.333333333333334},{"experimentId": "glm-5-2-opencode--guided","caseId": "author-003-openapi-connection","status": "current",- "passedRuns": 3,+ "passedRuns": 0,"validRuns": 3,- "meanDurationMs": 42824,- "measuredAt": "2026-08-19T22:00:30.905Z",- "meanEstimatedListCostUsd": 0.0144878,- "meanTokenConsumption": 9217,- "meanToolInvocationCount": 10.666666666666666+ "meanDurationMs": 524221,+ "measuredAt": "2026-08-28T19:22:01.000Z",+ "meanEstimatedListCostUsd": 0.1800297333333333,+ "meanTokenConsumption": 78267.66666666667,+ "meanToolInvocationCount": 24.666666666666668},{"experimentId": "glm-5-2-opencode--guided",@@ -1456,23 +1272,23 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 46385,- "measuredAt": "2026-08-19T22:00:30.912Z",- "meanEstimatedListCostUsd": 0.011119466666666666,- "meanTokenConsumption": 5550.333333333333,- "meanToolInvocationCount": 8.666666666666666+ "meanDurationMs": 68821,+ "measuredAt": "2026-08-28T19:22:01.000Z",+ "meanEstimatedListCostUsd": 0.031183066666666665,+ "meanTokenConsumption": 16504.333333333332,+ "meanToolInvocationCount": 15},{"experimentId": "glm-5-2-opencode--guided","caseId": "author-005-conditional-approval","status": "current",- "passedRuns": 3,+ "passedRuns": 2,"validRuns": 3,- "meanDurationMs": 55905,- "measuredAt": "2026-08-19T22:00:30.917Z",- "meanEstimatedListCostUsd": 0.014120933333333334,- "meanTokenConsumption": 8725.666666666666,- "meanToolInvocationCount": 10+ "meanDurationMs": 80061,+ "measuredAt": "2026-08-28T19:22:01.000Z",+ "meanEstimatedListCostUsd": 0.040431333333333326,+ "meanTokenConsumption": 26306.666666666668,+ "meanToolInvocationCount": 14.333333333333334},{"experimentId": "glm-5-2-opencode--guided",@@ -1480,11 +1296,11 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 43974,- "measuredAt": "2026-08-19T22:00:30.922Z",- "meanEstimatedListCostUsd": 0.015401466666666667,- "meanTokenConsumption": 9535.333333333334,- "meanToolInvocationCount": 10.666666666666666+ "meanDurationMs": 56079,+ "measuredAt": "2026-08-28T19:22:01.000Z",+ "meanEstimatedListCostUsd": 0.03310953333333334,+ "meanTokenConsumption": 19814.666666666668,+ "meanToolInvocationCount": 11},{"experimentId": "glm-5-2-opencode--guided",@@ -1492,95 +1308,95 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 35151,- "measuredAt": "2026-08-19T22:00:30.925Z",- "meanEstimatedListCostUsd": 0.005594666666666667,- "meanTokenConsumption": 3248.3333333333335,- "meanToolInvocationCount": 7+ "meanDurationMs": 74552,+ "measuredAt": "2026-08-28T19:22:01.000Z",+ "meanEstimatedListCostUsd": 0.029523933333333335,+ "meanTokenConsumption": 16295.666666666666,+ "meanToolInvocationCount": 12.333333333333334},{"experimentId": "gemini-3-1-pro-preview-opencode--baseline","caseId": "author-001-weather-tool","status": "current",- "passedRuns": 1,+ "passedRuns": 0,"validRuns": 3,- "meanDurationMs": 67954,- "measuredAt": "2026-08-19T22:06:54.450Z",- "meanEstimatedListCostUsd": 0.17139933333333335,- "meanTokenConsumption": 78096.33333333333,- "meanToolInvocationCount": 18+ "meanDurationMs": 54876,+ "measuredAt": "2026-08-28T19:20:40.000Z",+ "meanEstimatedListCostUsd": 0.11818266666666667,+ "meanTokenConsumption": 45096.333333333336,+ "meanToolInvocationCount": 9.666666666666666},{"experimentId": "gemini-3-1-pro-preview-opencode--baseline","caseId": "author-002-new-project","status": "current",- "passedRuns": 2,+ "passedRuns": 3,"validRuns": 3,- "meanDurationMs": 176726,- "measuredAt": "2026-08-19T22:06:54.459Z",- "meanEstimatedListCostUsd": 0.12588133333333332,- "meanTokenConsumption": 57477.333333333336,- "meanToolInvocationCount": 19+ "meanDurationMs": 82415,+ "measuredAt": "2026-08-28T19:26:24.000Z",+ "meanEstimatedListCostUsd": 0.11070399999999998,+ "meanTokenConsumption": 51112,+ "meanToolInvocationCount": 11.666666666666666},{"experimentId": "gemini-3-1-pro-preview-opencode--baseline","caseId": "author-003-openapi-connection","status": "current",- "passedRuns": 2,+ "passedRuns": 1,"validRuns": 3,- "meanDurationMs": 105009,- "measuredAt": "2026-08-19T22:06:54.468Z",- "meanEstimatedListCostUsd": 0.20446399999999998,- "meanTokenConsumption": 93695.33333333333,- "meanToolInvocationCount": 22+ "meanDurationMs": 102903,+ "measuredAt": "2026-08-28T19:26:24.000Z",+ "meanEstimatedListCostUsd": 0.43870533333333334,+ "meanTokenConsumption": 211094.33333333334,+ "meanToolInvocationCount": 28},{"experimentId": "gemini-3-1-pro-preview-opencode--baseline","caseId": "author-004-packaged-skill","status": "current",- "passedRuns": 1,+ "passedRuns": 2,"validRuns": 3,- "meanDurationMs": 49418,- "measuredAt": "2026-08-19T22:06:54.477Z",- "meanEstimatedListCostUsd": 0.04231066666666666,- "meanTokenConsumption": 13160.333333333334,- "meanToolInvocationCount": 5.666666666666667+ "meanDurationMs": 33942,+ "measuredAt": "2026-08-28T19:20:40.000Z",+ "meanEstimatedListCostUsd": 0.07116533333333334,+ "meanTokenConsumption": 26049.333333333332,+ "meanToolInvocationCount": 5.333333333333333},{"experimentId": "gemini-3-1-pro-preview-opencode--baseline","caseId": "author-005-conditional-approval","status": "current",- "passedRuns": 0,+ "passedRuns": 2,"validRuns": 3,- "meanDurationMs": 100093,- "measuredAt": "2026-08-19T22:06:54.483Z",- "meanEstimatedListCostUsd": 0.27252733333333334,- "meanTokenConsumption": 126355.33333333333,- "meanToolInvocationCount": 24.666666666666668+ "meanDurationMs": 227315,+ "measuredAt": "2026-08-28T19:20:40.000Z",+ "meanEstimatedListCostUsd": 0.5367073333333333,+ "meanTokenConsumption": 251920.33333333334,+ "meanToolInvocationCount": 42.666666666666664},{"experimentId": "gemini-3-1-pro-preview-opencode--baseline","caseId": "author-006-custom-channel","status": "current",- "passedRuns": 2,+ "passedRuns": 1,"validRuns": 3,- "meanDurationMs": 184235,- "measuredAt": "2026-08-19T22:06:54.488Z",- "meanEstimatedListCostUsd": 0.5857926666666667,- "meanTokenConsumption": 278326.3333333333,- "meanToolInvocationCount": 48.333333333333336+ "meanDurationMs": 133303,+ "measuredAt": "2026-08-28T19:20:40.000Z",+ "meanEstimatedListCostUsd": 0.467548,+ "meanTokenConsumption": 225615.66666666666,+ "meanToolInvocationCount": 30.666666666666668},{"experimentId": "gemini-3-1-pro-preview-opencode--baseline","caseId": "author-007-digest-schedule","status": "current",- "passedRuns": 2,+ "passedRuns": 3,"validRuns": 3,- "meanDurationMs": 149748,- "measuredAt": "2026-08-19T22:06:54.495Z",- "meanEstimatedListCostUsd": 0.12809666666666666,- "meanTokenConsumption": 57671.666666666664,- "meanToolInvocationCount": 16.333333333333332+ "meanDurationMs": 61813,+ "measuredAt": "2026-08-28T19:20:40.000Z",+ "meanEstimatedListCostUsd": 0.212552,+ "meanTokenConsumption": 101059.33333333333,+ "meanToolInvocationCount": 15},{"experimentId": "gemini-3-1-pro-preview-opencode--guided",@@ -1588,35 +1404,35 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 38449,- "measuredAt": "2026-08-19T22:03:45.194Z",- "meanEstimatedListCostUsd": 0.068454,- "meanTokenConsumption": 31198.666666666668,- "meanToolInvocationCount": 8+ "meanDurationMs": 37112,+ "measuredAt": "2026-08-28T19:17:42.000Z",+ "meanEstimatedListCostUsd": 0.08827333333333333,+ "meanTokenConsumption": 40508.333333333336,+ "meanToolInvocationCount": 7.666666666666667},{"experimentId": "gemini-3-1-pro-preview-opencode--guided","caseId": "author-002-new-project","status": "current",- "passedRuns": 3,+ "passedRuns": 0,"validRuns": 3,- "meanDurationMs": 105467,- "measuredAt": "2026-08-19T22:03:45.206Z",- "meanEstimatedListCostUsd": 0.07058666666666667,- "meanTokenConsumption": 32808.333333333336,- "meanToolInvocationCount": 10+ "meanDurationMs": 260182,+ "measuredAt": "2026-08-28T19:17:42.000Z",+ "meanEstimatedListCostUsd": 0.237908,+ "meanTokenConsumption": 109484,+ "meanToolInvocationCount": 28.333333333333332},{"experimentId": "gemini-3-1-pro-preview-opencode--guided","caseId": "author-003-openapi-connection","status": "current",- "passedRuns": 3,+ "passedRuns": 0,"validRuns": 3,- "meanDurationMs": 54859,- "measuredAt": "2026-08-19T22:03:45.214Z",- "meanEstimatedListCostUsd": 0.10357533333333334,- "meanTokenConsumption": 48067.666666666664,- "meanToolInvocationCount": 10.666666666666666+ "meanDurationMs": 201322,+ "measuredAt": "2026-08-28T19:17:42.000Z",+ "meanEstimatedListCostUsd": 0.3107266666666667,+ "meanTokenConsumption": 132131.66666666666,+ "meanToolInvocationCount": 34.333333333333336},{"experimentId": "gemini-3-1-pro-preview-opencode--guided",@@ -1624,11 +1440,11 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 91269,- "measuredAt": "2026-08-19T22:03:45.221Z",- "meanEstimatedListCostUsd": 0.05254333333333333,- "meanTokenConsumption": 21473.333333333332,- "meanToolInvocationCount": 7.666666666666667+ "meanDurationMs": 41440,+ "measuredAt": "2026-08-28T19:17:42.000Z",+ "meanEstimatedListCostUsd": 0.08369866666666666,+ "meanTokenConsumption": 34357.666666666664,+ "meanToolInvocationCount": 8.666666666666666},{"experimentId": "gemini-3-1-pro-preview-opencode--guided",@@ -1636,11 +1452,11 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 65793,- "measuredAt": "2026-08-19T22:03:45.226Z",- "meanEstimatedListCostUsd": 0.07310599999999999,- "meanTokenConsumption": 33233,- "meanToolInvocationCount": 8+ "meanDurationMs": 33301,+ "measuredAt": "2026-08-28T19:17:42.000Z",+ "meanEstimatedListCostUsd": 0.09258866666666667,+ "meanTokenConsumption": 42437.666666666664,+ "meanToolInvocationCount": 7.333333333333333},{"experimentId": "gemini-3-1-pro-preview-opencode--guided",@@ -1648,11 +1464,11 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 56233,- "measuredAt": "2026-08-19T22:03:45.232Z",- "meanEstimatedListCostUsd": 0.08904266666666667,- "meanTokenConsumption": 41173,- "meanToolInvocationCount": 7+ "meanDurationMs": 34493,+ "measuredAt": "2026-08-28T19:17:42.000Z",+ "meanEstimatedListCostUsd": 0.09662666666666668,+ "meanTokenConsumption": 45305,+ "meanToolInvocationCount": 7.333333333333333},{"experimentId": "gemini-3-1-pro-preview-opencode--guided",@@ -1660,11 +1476,209 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 79041,- "measuredAt": "2026-08-19T22:03:45.237Z",- "meanEstimatedListCostUsd": 0.08614333333333335,- "meanTokenConsumption": 40725,- "meanToolInvocationCount": 7.333333333333333+ "meanDurationMs": 33257,+ "measuredAt": "2026-08-28T19:17:42.000Z",+ "meanEstimatedListCostUsd": 0.059873333333333334,+ "meanTokenConsumption": 27681.666666666668,+ "meanToolInvocationCount": 4.666666666666667+ }+ ],+ "previouslyMeasured": [+ {+ "suite": {+ "eveRevision": "54a028bab589a60fc84a310304a5cf080bc7f990",+ "caseFingerprint": "786490be9d902701f57d08b3d112ca385d49a72e5591fac741511e77d67989d2",+ "caseCount": 7,+ "runsPerCell": 3+ },+ "experiments": [+ {+ "id": "claude-sonnet-4-6-opencode--baseline",+ "groupId": "claude-sonnet-4-6-opencode",+ "model": "claude-sonnet-4-6",+ "modelDisplayName": "Claude Sonnet 4.6",+ "harness": "OpenCode",+ "treatment": "baseline"+ },+ {+ "id": "claude-sonnet-4-6-opencode--guided",+ "groupId": "claude-sonnet-4-6-opencode",+ "model": "claude-sonnet-4-6",+ "modelDisplayName": "Claude Sonnet 4.6",+ "harness": "OpenCode",+ "treatment": "guided"+ }+ ],+ "results": [+ {+ "experimentId": "claude-sonnet-4-6-opencode--baseline",+ "caseId": "author-001-weather-tool",+ "status": "current",+ "passedRuns": 0,+ "validRuns": 3,+ "meanDurationMs": 199451,+ "measuredAt": "2026-08-19T21:35:21.709Z",+ "meanEstimatedListCostUsd": 0.052948,+ "meanTokenConsumption": 10626.666666666666,+ "meanToolInvocationCount": 11.666666666666666+ },+ {+ "experimentId": "claude-sonnet-4-6-opencode--baseline",+ "caseId": "author-002-new-project",+ "status": "current",+ "passedRuns": 0,+ "validRuns": 3,+ "meanDurationMs": 143527,+ "measuredAt": "2026-08-19T21:35:21.715Z",+ "meanEstimatedListCostUsd": 0.04594499999999999,+ "meanTokenConsumption": 10429.666666666666,+ "meanToolInvocationCount": 10.666666666666666+ },+ {+ "experimentId": "claude-sonnet-4-6-opencode--baseline",+ "caseId": "author-003-openapi-connection",+ "status": "current",+ "passedRuns": 1,+ "validRuns": 3,+ "meanDurationMs": 158661,+ "measuredAt": "2026-08-19T21:35:21.721Z",+ "meanEstimatedListCostUsd": 0.11106500000000001,+ "meanTokenConsumption": 27383,+ "meanToolInvocationCount": 23.666666666666668+ },+ {+ "experimentId": "claude-sonnet-4-6-opencode--baseline",+ "caseId": "author-004-packaged-skill",+ "status": "current",+ "passedRuns": 2,+ "validRuns": 3,+ "meanDurationMs": 382777,+ "measuredAt": "2026-08-17T22:03:22.192Z",+ "meanEstimatedListCostUsd": 0.108879,+ "meanTokenConsumption": 22653,+ "meanToolInvocationCount": 16+ },+ {+ "experimentId": "claude-sonnet-4-6-opencode--baseline",+ "caseId": "author-005-conditional-approval",+ "status": "current",+ "passedRuns": 0,+ "validRuns": 3,+ "meanDurationMs": 225419,+ "measuredAt": "2026-08-19T21:35:21.730Z",+ "meanEstimatedListCostUsd": 0.10689799999999999,+ "meanTokenConsumption": 22528.666666666668,+ "meanToolInvocationCount": 24+ },+ {+ "experimentId": "claude-sonnet-4-6-opencode--baseline",+ "caseId": "author-006-custom-channel",+ "status": "current",+ "passedRuns": 1,+ "validRuns": 3,+ "meanDurationMs": 714111,+ "measuredAt": "2026-08-17T22:19:44.996Z",+ "meanEstimatedListCostUsd": 0.056847,+ "meanTokenConsumption": 12625,+ "meanToolInvocationCount": 10+ },+ {+ "experimentId": "claude-sonnet-4-6-opencode--baseline",+ "caseId": "author-007-digest-schedule",+ "status": "current",+ "passedRuns": 0,+ "validRuns": 3,+ "meanDurationMs": 232328,+ "measuredAt": "2026-08-19T21:35:21.737Z",+ "meanEstimatedListCostUsd": 0.09773,+ "meanTokenConsumption": 16300.666666666666,+ "meanToolInvocationCount": 33.333333333333336+ },+ {+ "experimentId": "claude-sonnet-4-6-opencode--guided",+ "caseId": "author-001-weather-tool",+ "status": "current",+ "passedRuns": 0,+ "validRuns": 3,+ "meanDurationMs": 186603,+ "measuredAt": "2026-08-19T21:35:25.673Z",+ "meanEstimatedListCostUsd": 0.07002399999999999,+ "meanTokenConsumption": 13736,+ "meanToolInvocationCount": 17.333333333333332+ },+ {+ "experimentId": "claude-sonnet-4-6-opencode--guided",+ "caseId": "author-002-new-project",+ "status": "current",+ "passedRuns": 2,+ "validRuns": 3,+ "meanDurationMs": 198555,+ "measuredAt": "2026-08-19T21:35:25.695Z",+ "meanEstimatedListCostUsd": 0.104669,+ "meanTokenConsumption": 27117.666666666668,+ "meanToolInvocationCount": 17.333333333333332+ },+ {+ "experimentId": "claude-sonnet-4-6-opencode--guided",+ "caseId": "author-003-openapi-connection",+ "status": "current",+ "passedRuns": 3,+ "validRuns": 3,+ "meanDurationMs": 132154,+ "measuredAt": "2026-08-19T21:35:25.701Z",+ "meanEstimatedListCostUsd": 0.045839,+ "meanTokenConsumption": 11199.666666666666,+ "meanToolInvocationCount": 10.333333333333334+ },+ {+ "experimentId": "claude-sonnet-4-6-opencode--guided",+ "caseId": "author-004-packaged-skill",+ "status": "current",+ "passedRuns": 1,+ "validRuns": 3,+ "meanDurationMs": 127395,+ "measuredAt": "2026-08-19T21:04:16.127Z",+ "meanEstimatedListCostUsd": 0.082587,+ "meanTokenConsumption": 16813,+ "meanToolInvocationCount": 13+ },+ {+ "experimentId": "claude-sonnet-4-6-opencode--guided",+ "caseId": "author-005-conditional-approval",+ "status": "current",+ "passedRuns": 0,+ "validRuns": 3,+ "meanDurationMs": 170150,+ "measuredAt": "2026-08-19T21:35:25.711Z",+ "meanEstimatedListCostUsd": 0.031358,+ "meanTokenConsumption": 5668.666666666667,+ "meanToolInvocationCount": 7.666666666666667+ },+ {+ "experimentId": "claude-sonnet-4-6-opencode--guided",+ "caseId": "author-006-custom-channel",+ "status": "current",+ "passedRuns": 2,+ "validRuns": 3,+ "meanDurationMs": 130916,+ "measuredAt": "2026-08-19T21:04:16.135Z",+ "meanEstimatedListCostUsd": 0.0346035,+ "meanTokenConsumption": 6458.5,+ "meanToolInvocationCount": 11+ },+ {+ "experimentId": "claude-sonnet-4-6-opencode--guided",+ "caseId": "author-007-digest-schedule",+ "status": "current",+ "passedRuns": 0,+ "validRuns": 3,+ "meanDurationMs": 260678,+ "measuredAt": "2026-08-19T21:35:25.720Z",+ "meanEstimatedListCostUsd": 0.09039199999999999,+ "meanTokenConsumption": 13797.333333333334,+ "meanToolInvocationCount": 29+ }+ ]}]}diff --git a/apps/docs/lib/evals/results.ts b/apps/docs/lib/evals/results.tsindex 3a9a483ac..0d06d7cf5 100644--- a/apps/docs/lib/evals/results.ts+++ b/apps/docs/lib/evals/results.ts@@ -25,17 +25,26 @@ export interface BenchmarkResult {measuredAt?: string;}+export interface BenchmarkSuite {+ eveRevision: string | null;+ caseFingerprint: string | null;+ caseCount: number;+ runsPerCell: number;+}++export interface PreviouslyMeasuredBenchmarkResults {+ suite: BenchmarkSuite;+ experiments: BenchmarkExperiment[];+ results: BenchmarkResult[];+}+export interface PublishedBenchmarkResults {schemaVersion: 1;generatedAt: string | null;- suite: {- eveRevision: string | null;- caseFingerprint: string | null;- caseCount: number;- runsPerCell: number;- };+ suite: BenchmarkSuite;experiments: BenchmarkExperiment[];results: BenchmarkResult[];+ previouslyMeasured?: PreviouslyMeasuredBenchmarkResults[];}export interface BenchmarkCaseRow {