Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

docs(benchmarks): run benchmarks and retain superseded models (#2724)

Colton Padden451be3fb2d05
diff --git a/apps/benchmarks/lib/benchmark-config.test.mjs b/apps/benchmarks/lib/benchmark-config.test.mjs
index 04275db2c..e6d68c63b 100644
--- a/apps/benchmarks/lib/benchmark-config.test.mjs
+++ b/apps/benchmarks/lib/benchmark-config.test.mjs
@@ -12,7 +12,6 @@ test("publishes only compatibility-validated models", () => {
assert.deepEqual(
publishedBenchmarkModels.map((benchmark) => benchmark.id),
[
- "claude-sonnet-4-6",
"kimi-k3",
"claude-fable-5",
"grok-4-6",
@@ -41,5 +40,7 @@ test("allows candidate probes and rejects unknown models", () => {
assert.equal(findBenchmarkModel("gpt-5-6-sol").support, "supported");
assert.equal(findBenchmarkModel("gpt-5-6-terra").support, "supported");
assert.equal(findBenchmarkModel("grok-4-6").support, "supported");
+ assert.equal(findBenchmarkModel("claude-sonnet-4-6").support, "superseded");
+ assert.equal(findBenchmarkModel("claude-opus-5").support, "candidate");
assert.throws(() => findBenchmarkModel("unknown"), /Unknown model/u);
});
diff --git a/apps/benchmarks/lib/benchmark-config.ts b/apps/benchmarks/lib/benchmark-config.ts
index 4928309b7..3abb3cfbc 100644
--- a/apps/benchmarks/lib/benchmark-config.ts
+++ b/apps/benchmarks/lib/benchmark-config.ts
@@ -2,7 +2,7 @@ export const authoringTreatments = ["baseline", "guided"] as const;
export type AuthoringTreatment = (typeof authoringTreatments)[number];
-export type AuthoringBenchmarkSupport = "supported" | "candidate";
+export type AuthoringBenchmarkSupport = "supported" | "candidate" | "superseded";
export interface AuthoringBenchmarkModel {
readonly id: string;
@@ -18,7 +18,7 @@ export const benchmarkModels = [
model: "claude-sonnet-4-6",
displayName: "Claude Sonnet 4.6",
harness: "OpenCode",
- support: "supported",
+ support: "superseded",
},
{
id: "kimi-k3",
diff --git a/apps/benchmarks/scripts/export-results.mjs b/apps/benchmarks/scripts/export-results.mjs
index 5dc49b682..0d1c71e5b 100644
--- a/apps/benchmarks/scripts/export-results.mjs
+++ b/apps/benchmarks/scripts/export-results.mjs
@@ -16,6 +16,7 @@ import {
} from "./cost.mjs";
import {
authoringTreatments,
+ benchmarkModels,
findPublishedBenchmarkModel,
publishedBenchmark,
publishedBenchmarkModels,
@@ -55,6 +56,7 @@ const experimentIds = benchmarks.flatMap((benchmark) =>
authoringTreatments.map((treatment) => publishedExperimentId(benchmark, treatment)),
);
const stale = staleCells(experimentIds);
+const previousResults = readPreviousResults();
const results = [];
for (const benchmark of benchmarks) {
@@ -104,6 +106,7 @@ const output = {
})),
),
results,
+ ...(previousResults.length === 0 ? {} : { previouslyMeasured: previousResults }),
};
const destination =
@@ -112,6 +115,40 @@ mkdirSync(dirname(destination), { recursive: true });
writeFileSync(destination, `${JSON.stringify(output, null, 2)}\n`);
console.log(`Exported ${results.length} benchmark cells to ${destination}`);
+function readPreviousResults() {
+ const destination =
+ values.output === undefined ? outputPath : resolve(process.cwd(), values.output);
+ if (!existsSync(destination)) return [];
+
+ const previous = JSON.parse(readFileSync(destination, "utf8"));
+ const supersededExperimentIds = new Set(
+ benchmarkModels
+ .filter((benchmark) => benchmark.support === "superseded")
+ .flatMap((benchmark) =>
+ authoringTreatments.map((treatment) => publishedExperimentId(benchmark, treatment)),
+ ),
+ );
+ const supersededExperiments = (previous.experiments ?? []).filter((experiment) =>
+ supersededExperimentIds.has(experiment.id),
+ );
+ const supersededResults = (previous.results ?? []).filter((result) =>
+ supersededExperimentIds.has(result.experimentId),
+ );
+ const retained = (previous.previouslyMeasured ?? []).filter(
+ (measurement) => Array.isArray(measurement.experiments) && Array.isArray(measurement.results),
+ );
+ if (supersededExperiments.length === 0) return retained;
+
+ return [
+ ...retained,
+ {
+ suite: previous.suite,
+ experiments: supersededExperiments,
+ results: supersededResults,
+ },
+ ];
+}
+
function selectedBenchmarks(value) {
if (value === undefined) return publishedBenchmarkModels;
return value.split(",").map((model) => findPublishedBenchmarkModel(model));
diff --git a/apps/benchmarks/scripts/export-results.test.mjs b/apps/benchmarks/scripts/export-results.test.mjs
index 7e9804acf..0d19c8aaa 100644
--- a/apps/benchmarks/scripts/export-results.test.mjs
+++ b/apps/benchmarks/scripts/export-results.test.mjs
@@ -40,11 +40,10 @@ test("exports missing cells without publishing private artifacts", () => {
assert.equal(output.schemaVersion, 1);
assert.equal(output.suite.caseCount, 7);
assert.match(output.suite.caseFingerprint, /^[0-9a-f]{64}$/u);
- assert.equal(output.experiments.length, 18);
+ assert.equal(output.experiments.length, 16);
assert.deepEqual(
[...new Set(output.experiments.map((experiment) => experiment.modelDisplayName))],
[
- "Claude Sonnet 4.6",
"Kimi K3",
"Claude Fable 5",
"Grok 4.6",
@@ -55,7 +54,7 @@ test("exports missing cells without publishing private artifacts", () => {
"Gemini 3.1 Pro Preview",
],
);
- assert.equal(output.results.length, 126);
+ assert.equal(output.results.length, 112);
assert.ok(output.results.every((result) => result.status === "missing"));
for (const privateField of ["transcript", "commands", "worldEvents", "files"]) {
assert.equal(raw.includes(`"${privateField}"`), false);
@@ -67,6 +66,76 @@ test("exports missing cells without publishing private artifacts", () => {
}
});
+test("retains superseded experiments separately from the current matrix", () => {
+ const directory = mkdtempSync(join(tmpdir(), "eve-benchmark-export-previous-"));
+ const outputPath = join(directory, "results.json");
+ const resultsPath = new URL("../results", import.meta.url).pathname;
+ const savedResultsPath = join(directory, "saved-results");
+ try {
+ writeFileSync(
+ outputPath,
+ JSON.stringify({
+ schemaVersion: 1,
+ generatedAt: "2026-08-01T00:00:00.000Z",
+ suite: {
+ eveRevision: "b".repeat(40),
+ caseFingerprint: "c".repeat(64),
+ caseCount: 1,
+ runsPerCell: 1,
+ },
+ experiments: [
+ {
+ id: "claude-sonnet-4-6-opencode--baseline",
+ groupId: "claude-sonnet-4-6-opencode",
+ model: "claude-sonnet-4-6",
+ modelDisplayName: "Claude Sonnet 4.6",
+ harness: "OpenCode",
+ treatment: "baseline",
+ },
+ ],
+ results: [
+ {
+ experimentId: "claude-sonnet-4-6-opencode--baseline",
+ caseId: "author-001-weather-tool",
+ status: "current",
+ passedRuns: 1,
+ validRuns: 1,
+ meanDurationMs: 1000,
+ measuredAt: "2026-08-01T00:00:00.000Z",
+ },
+ ],
+ }),
+ );
+ if (existsSync(resultsPath)) renameSync(resultsPath, savedResultsPath);
+ mkdirSync(resultsPath, { recursive: true });
+ execFileSync(
+ process.execPath,
+ [
+ new URL("./export-results.mjs", import.meta.url).pathname,
+ "--revision",
+ "a".repeat(40),
+ "--output",
+ outputPath,
+ ],
+ { cwd: appRoot, stdio: "pipe" },
+ );
+ const output = JSON.parse(readFileSync(outputPath, "utf8"));
+ assert.equal(output.previouslyMeasured.length, 1);
+ assert.equal(
+ output.previouslyMeasured[0].experiments[0].id,
+ "claude-sonnet-4-6-opencode--baseline",
+ );
+ assert.equal(
+ output.previouslyMeasured[0].results[0].experimentId,
+ "claude-sonnet-4-6-opencode--baseline",
+ );
+ } finally {
+ rmSync(resultsPath, { recursive: true, force: true });
+ if (existsSync(savedResultsPath)) renameSync(savedResultsPath, resultsPath);
+ rmSync(directory, { recursive: true, force: true });
+ }
+});
+
test("exports mean cost, token consumption, and tool invocations", () => {
const directory = mkdtempSync(join(tmpdir(), "eve-benchmark-export-cost-"));
const outputPath = join(directory, "results.json");
@@ -76,7 +145,7 @@ test("exports mean cost, token consumption, and tool invocations", () => {
if (existsSync(resultsPath)) renameSync(resultsPath, savedResultsPath);
const runPath = join(
resultsPath,
- "claude-sonnet-4-6-opencode--baseline",
+ "kimi-k3-opencode--baseline",
"run",
"author-001-weather-tool",
"run-1",
@@ -113,7 +182,7 @@ test("exports mean cost, token consumption, and tool invocations", () => {
"--revision",
"a".repeat(40),
"--models",
- "claude-sonnet-4-6",
+ "kimi-k3",
"--output",
outputPath,
],
@@ -122,7 +191,7 @@ test("exports mean cost, token consumption, and tool invocations", () => {
const output = JSON.parse(readFileSync(outputPath, "utf8"));
const result = output.results.find(
(entry) =>
- entry.experimentId === "claude-sonnet-4-6-opencode--baseline" &&
+ entry.experimentId === "kimi-k3-opencode--baseline" &&
entry.caseId === "author-001-weather-tool",
);
assert.equal(result.meanEstimatedListCostUsd, 4.5);
diff --git a/apps/docs/app/[lang]/benchmarks/page.tsx b/apps/docs/app/[lang]/benchmarks/page.tsx
index 013f3ae59..48d50d447 100644
--- a/apps/docs/app/[lang]/benchmarks/page.tsx
+++ b/apps/docs/app/[lang]/benchmarks/page.tsx
@@ -26,6 +26,9 @@ export const metadata: Metadata = {
const ModelBenchmarksPage = () => {
const rows = benchmarkRows(benchmarkResults);
+ const previouslyMeasured = (benchmarkResults.previouslyMeasured ?? []).flatMap((measurement) =>
+ benchmarkRows({ ...benchmarkResults, ...measurement }),
+ );
const publishedAt = formatDate(benchmarkResults.generatedAt);
const revision = benchmarkResults.suite.eveRevision;
@@ -35,7 +38,8 @@ const ModelBenchmarksPage = () => {
<h1 className="text-gray-1000 text-heading-48 sm:text-heading-64">Model Benchmarks</h1>
<p className="mt-5 max-w-3xl text-gray-900 text-lg">
Performance of coding agents creating and changing eve projects, measured by deterministic
- checks against the files, commands, and simulated external interactions each run produces.
+ checks against the files, commands, and simulated external interactions each run produces
+ <sup>1</sup>.
</p>
<div className="mt-6 flex flex-wrap items-center justify-center gap-x-3 gap-y-2 text-gray-800 text-sm">
<span>
@@ -50,25 +54,30 @@ const ModelBenchmarksPage = () => {
</div>
</section>
- <ResultsTable rows={rows} />
+ <section aria-labelledby="current-results">
+ <h2 className="sr-only" id="current-results">
+ Current results
+ </h2>
+ <ResultsTable rows={rows} />
+ </section>
- <div className="mt-4 space-y-2 px-4 text-gray-700 text-sm leading-6">
- <p>
- * eve guidance includes the <code>AGENTS.md</code> and agent-specific aliases generated by{" "}
- <code>eve init</code>. The column shows the success rate when agents have access to this
- guidance.
- </p>
- <p>
- N/A marks a case that a model has not run. It counts against that model&apos;s success
- rate so every model is scored against the same case set. When a case changes, eve retains
- a model&apos;s most recent result until that model reruns it.
- </p>
- <p>
- Avg List Cost is the mean estimated cost per evaluation from each run&apos;s token usage
- at the provider&apos;s public list price. It is a relative comparison, not a Gateway bill:
- routing, cache discounts, and provider pricing can differ.
- </p>
- </div>
+ {previouslyMeasured.length > 0 ? (
+ <section className="mt-8" aria-labelledby="retired-models">
+ <h2 className="text-heading-24 text-gray-1000" id="retired-models">
+ Retired Models
+ </h2>
+ <div className="mt-3">
+ <ResultsTable rows={previouslyMeasured} showMeasurementDate />
+ </div>
+ </section>
+ ) : null}
+
+ <p className="mt-8 text-gray-700 text-sm leading-6">
+ <sup>1</sup> Each model completes the same deterministic eve authoring tasks three times
+ with and without the guidance files generated by <code>eve init</code>; success rates
+ include every scheduled run, while duration, tokens, and tool calls average valid runs. Cost
+ estimates apply provider public list prices to recorded token usage.
+ </p>
</main>
);
};
diff --git a/apps/docs/app/[lang]/benchmarks/results-table.tsx b/apps/docs/app/[lang]/benchmarks/results-table.tsx
index bf484d65c..9dfc82185 100644
--- a/apps/docs/app/[lang]/benchmarks/results-table.tsx
+++ b/apps/docs/app/[lang]/benchmarks/results-table.tsx
@@ -20,7 +20,13 @@ interface SortState {
direction: SortDirection;
}
-export const ResultsTable = ({ rows }: { rows: BenchmarkRow[] }) => {
+export const ResultsTable = ({
+ rows,
+ showMeasurementDate = false,
+}: {
+ rows: BenchmarkRow[];
+ showMeasurementDate?: boolean;
+}) => {
const [expanded, setExpanded] = useState<ReadonlySet<string>>(new Set());
const [sort, setSort] = useState<SortState>({
key: "guidedSuccessRate",
@@ -76,7 +82,7 @@ export const ResultsTable = ({ rows }: { rows: BenchmarkRow[] }) => {
onSort={() => toggleSort("averageDurationMs")}
sortDirection={sort.key === "averageDurationMs" ? sort.direction : undefined}
>
- Avg Duration
+ Duration
</HeaderCell>
<HeaderCell
align="right"
@@ -85,7 +91,7 @@ export const ResultsTable = ({ rows }: { rows: BenchmarkRow[] }) => {
sort.key === "averageEstimatedListCostUsd" ? sort.direction : undefined
}
>
- Avg List Cost
+ Cost
</HeaderCell>
<HeaderCell
align="right"
@@ -99,7 +105,7 @@ export const ResultsTable = ({ rows }: { rows: BenchmarkRow[] }) => {
onSort={() => toggleSort("guidedSuccessRate")}
sortDirection={sort.key === "guidedSuccessRate" ? sort.direction : undefined}
>
- Success Rate with AGENTS.md *
+ Success Rate with AGENTS.md
</HeaderCell>
</tr>
</thead>
@@ -123,6 +129,11 @@ export const ResultsTable = ({ rows }: { rows: BenchmarkRow[] }) => {
/>
</span>
{row.modelDisplayName}
+ {showMeasurementDate && row.latestMeasuredAt !== null ? (
+ <span className="font-normal text-gray-700 text-sm">
+ {formatDate(row.latestMeasuredAt)}
+ </span>
+ ) : null}
</button>
</Cell>
<Cell>{row.harness}</Cell>
@@ -133,15 +144,15 @@ export const ResultsTable = ({ rows }: { rows: BenchmarkRow[] }) => {
</tr>
{isExpanded ? (
<tr className="border-t border-gray-400">
- <td className="p-4" colSpan={6}>
- <div className="overflow-hidden rounded-lg border border-gray-400">
- <table className="w-full border-collapse text-left">
- <thead className="bg-gray-100 text-gray-800">
+ <td className="px-4 pb-4" colSpan={6}>
+ <div>
+ <table className="w-full border-collapse text-left text-sm">
+ <thead className="text-gray-800">
<tr>
<HeaderCell>Evaluation</HeaderCell>
- <HeaderCell align="right">Avg Duration</HeaderCell>
- <HeaderCell align="right">Avg Tokens</HeaderCell>
- <HeaderCell align="right">Avg Tool Calls</HeaderCell>
+ <HeaderCell align="right">Duration</HeaderCell>
+ <HeaderCell align="right">Tokens</HeaderCell>
+ <HeaderCell align="right">Tool Calls</HeaderCell>
<HeaderCell align="right">Success Rate</HeaderCell>
<HeaderCell align="right">Success Rate with AGENTS.md</HeaderCell>
</tr>
@@ -254,6 +265,13 @@ function compareNullableNumbers(
return direction === "ascending" ? left - right : right - left;
}
+function formatDate(value: string): string {
+ return new Intl.DateTimeFormat("en", {
+ dateStyle: "medium",
+ timeZone: "UTC",
+ }).format(new Date(value));
+}
+
function formatDuration(value: number | null): React.ReactNode {
if (value === null) return <NotAvailable />;
return `${(value / 1000).toFixed(1)}s`;
diff --git a/apps/docs/lib/evals/benchmark-results.json b/apps/docs/lib/evals/benchmark-results.json
index 532612363..312ba679a 100644
--- a/apps/docs/lib/evals/benchmark-results.json
+++ b/apps/docs/lib/evals/benchmark-results.json
@@ -1,29 +1,13 @@
{
"schemaVersion": 1,
- "generatedAt": "2026-08-19T22:10:07.841Z",
+ "generatedAt": "2026-08-28T20:19:42.291Z",
"suite": {
- "eveRevision": "54a028bab589a60fc84a310304a5cf080bc7f990",
+ "eveRevision": "05b12e3beb662979143397b9e691a2d7b4417a02",
"caseFingerprint": "786490be9d902701f57d08b3d112ca385d49a72e5591fac741511e77d67989d2",
"caseCount": 7,
"runsPerCell": 3
},
"experiments": [
- {
- "id": "claude-sonnet-4-6-opencode--baseline",
- "groupId": "claude-sonnet-4-6-opencode",
- "model": "claude-sonnet-4-6",
- "modelDisplayName": "Claude Sonnet 4.6",
- "harness": "OpenCode",
- "treatment": "baseline"
- },
- {
- "id": "claude-sonnet-4-6-opencode--guided",
- "groupId": "claude-sonnet-4-6-opencode",
- "model": "claude-sonnet-4-6",
- "modelDisplayName": "Claude Sonnet 4.6",
- "harness": "OpenCode",
- "treatment": "guided"
- },
{
"id": "kimi-k3-opencode--baseline",
"groupId": "kimi-k3-opencode",
@@ -154,185 +138,17 @@
}
],
"results": [
- {
- "experimentId": "claude-sonnet-4-6-opencode--baseline",
- "caseId": "author-001-weather-tool",
- "status": "current",
- "passedRuns": 0,
- "validRuns": 3,
- "meanDurationMs": 199451,
- "measuredAt": "2026-08-19T21:35:21.709Z",
- "meanEstimatedListCostUsd": 0.052948,
- "meanTokenConsumption": 10626.666666666666,
- "meanToolInvocationCount": 11.666666666666666
- },
- {
- "experimentId": "claude-sonnet-4-6-opencode--baseline",
- "caseId": "author-002-new-project",
- "status": "current",
- "passedRuns": 0,
- "validRuns": 3,
- "meanDurationMs": 143527,
- "measuredAt": "2026-08-19T21:35:21.715Z",
- "meanEstimatedListCostUsd": 0.04594499999999999,
- "meanTokenConsumption": 10429.666666666666,
- "meanToolInvocationCount": 10.666666666666666
- },
- {
- "experimentId": "claude-sonnet-4-6-opencode--baseline",
- "caseId": "author-003-openapi-connection",
- "status": "current",
- "passedRuns": 1,
- "validRuns": 3,
- "meanDurationMs": 158661,
- "measuredAt": "2026-08-19T21:35:21.721Z",
- "meanEstimatedListCostUsd": 0.11106500000000001,
- "meanTokenConsumption": 27383,
- "meanToolInvocationCount": 23.666666666666668
- },
- {
- "experimentId": "claude-sonnet-4-6-opencode--baseline",
- "caseId": "author-004-packaged-skill",
- "status": "current",
- "passedRuns": 2,
- "validRuns": 3,
- "meanDurationMs": 382777,
- "measuredAt": "2026-08-17T22:03:22.192Z",
- "meanEstimatedListCostUsd": 0.108879,
- "meanTokenConsumption": 22653,
- "meanToolInvocationCount": 16
- },
- {
- "experimentId": "claude-sonnet-4-6-opencode--baseline",
- "caseId": "author-005-conditional-approval",
- "status": "current",
- "passedRuns": 0,
- "validRuns": 3,
- "meanDurationMs": 225419,
- "measuredAt": "2026-08-19T21:35:21.730Z",
- "meanEstimatedListCostUsd": 0.10689799999999999,
- "meanTokenConsumption": 22528.666666666668,
- "meanToolInvocationCount": 24
- },
- {
- "experimentId": "claude-sonnet-4-6-opencode--baseline",
- "caseId": "author-006-custom-channel",
- "status": "current",
- "passedRuns": 1,
- "validRuns": 3,
- "meanDurationMs": 714111,
- "measuredAt": "2026-08-17T22:19:44.996Z",
- "meanEstimatedListCostUsd": 0.056847,
- "meanTokenConsumption": 12625,
- "meanToolInvocationCount": 10
- },
- {
- "experimentId": "claude-sonnet-4-6-opencode--baseline",
- "caseId": "author-007-digest-schedule",
- "status": "current",
- "passedRuns": 0,
- "validRuns": 3,
- "meanDurationMs": 232328,
- "measuredAt": "2026-08-19T21:35:21.737Z",
- "meanEstimatedListCostUsd": 0.09773,
- "meanTokenConsumption": 16300.666666666666,
- "meanToolInvocationCount": 33.333333333333336
- },
- {
- "experimentId": "claude-sonnet-4-6-opencode--guided",
- "caseId": "author-001-weather-tool",
- "status": "current",
- "passedRuns": 0,
- "validRuns": 3,
- "meanDurationMs": 186603,
- "measuredAt": "2026-08-19T21:35:25.673Z",
- "meanEstimatedListCostUsd": 0.07002399999999999,
- "meanTokenConsumption": 13736,
- "meanToolInvocationCount": 17.333333333333332
- },
- {
- "experimentId": "claude-sonnet-4-6-opencode--guided",
- "caseId": "author-002-new-project",
- "status": "current",
- "passedRuns": 2,
- "validRuns": 3,
- "meanDurationMs": 198555,
- "measuredAt": "2026-08-19T21:35:25.695Z",
- "meanEstimatedListCostUsd": 0.104669,
- "meanTokenConsumption": 27117.666666666668,
- "meanToolInvocationCount": 17.333333333333332
- },
- {
- "experimentId": "claude-sonnet-4-6-opencode--guided",
- "caseId": "author-003-openapi-connection",
- "status": "current",
- "passedRuns": 3,
- "validRuns": 3,
- "meanDurationMs": 132154,
- "measuredAt": "2026-08-19T21:35:25.701Z",
- "meanEstimatedListCostUsd": 0.045839,
- "meanTokenConsumption": 11199.666666666666,
- "meanToolInvocationCount": 10.333333333333334
- },
- {
- "experimentId": "claude-sonnet-4-6-opencode--guided",
- "caseId": "author-004-packaged-skill",
- "status": "current",
- "passedRuns": 1,
- "validRuns": 3,
- "meanDurationMs": 127395,
- "measuredAt": "2026-08-19T21:04:16.127Z",
- "meanEstimatedListCostUsd": 0.082587,
- "meanTokenConsumption": 16813,
- "meanToolInvocationCount": 13
- },
- {
- "experimentId": "claude-sonnet-4-6-opencode--guided",
- "caseId": "author-005-conditional-approval",
- "status": "current",
- "passedRuns": 0,
- "validRuns": 3,
- "meanDurationMs": 170150,
- "measuredAt": "2026-08-19T21:35:25.711Z",
- "meanEstimatedListCostUsd": 0.031358,
- "meanTokenConsumption": 5668.666666666667,
- "meanToolInvocationCount": 7.666666666666667
- },
- {
- "experimentId": "claude-sonnet-4-6-opencode--guided",
- "caseId": "author-006-custom-channel",
- "status": "current",
- "passedRuns": 2,
- "validRuns": 3,
- "meanDurationMs": 130916,
- "measuredAt": "2026-08-19T21:04:16.135Z",
- "meanEstimatedListCostUsd": 0.0346035,
- "meanTokenConsumption": 6458.5,
- "meanToolInvocationCount": 11
- },
- {
- "experimentId": "claude-sonnet-4-6-opencode--guided",
- "caseId": "author-007-digest-schedule",
- "status": "current",
- "passedRuns": 0,
- "validRuns": 3,
- "meanDurationMs": 260678,
- "measuredAt": "2026-08-19T21:35:25.720Z",
- "meanEstimatedListCostUsd": 0.09039199999999999,
- "meanTokenConsumption": 13797.333333333334,
- "meanToolInvocationCount": 29
- },
{
"experimentId": "kimi-k3-opencode--baseline",
"caseId": "author-001-weather-tool",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 138696,
- "measuredAt": "2026-08-19T21:40:48.303Z",
- "meanEstimatedListCostUsd": 0.210969,
- "meanTokenConsumption": 49695,
- "meanToolInvocationCount": 22.333333333333332
+ "meanDurationMs": 124258,
+ "measuredAt": "2026-08-28T19:22:06.000Z",
+ "meanEstimatedListCostUsd": 0.14751699999999998,
+ "meanTokenConsumption": 31831,
+ "meanToolInvocationCount": 25
},
{
"experimentId": "kimi-k3-opencode--baseline",
@@ -340,23 +156,23 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 92789,
- "measuredAt": "2026-08-19T21:40:48.309Z",
- "meanEstimatedListCostUsd": 0.105078,
- "meanTokenConsumption": 27707.333333333332,
- "meanToolInvocationCount": 11.333333333333334
+ "meanDurationMs": 89304,
+ "measuredAt": "2026-08-28T19:25:16.000Z",
+ "meanEstimatedListCostUsd": 0.085075,
+ "meanTokenConsumption": 21470.333333333332,
+ "meanToolInvocationCount": 12
},
{
"experimentId": "kimi-k3-opencode--baseline",
"caseId": "author-003-openapi-connection",
"status": "current",
- "passedRuns": 3,
+ "passedRuns": 0,
"validRuns": 3,
- "meanDurationMs": 137104,
- "measuredAt": "2026-08-19T21:40:48.313Z",
- "meanEstimatedListCostUsd": 0.21580199999999997,
- "meanTokenConsumption": 56562,
- "meanToolInvocationCount": 24
+ "meanDurationMs": 454935,
+ "measuredAt": "2026-08-28T19:22:06.000Z",
+ "meanEstimatedListCostUsd": 0.3622910000000001,
+ "meanTokenConsumption": 56591.666666666664,
+ "meanToolInvocationCount": 28
},
{
"experimentId": "kimi-k3-opencode--baseline",
@@ -364,11 +180,11 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 124071,
- "measuredAt": "2026-08-19T21:40:48.318Z",
- "meanEstimatedListCostUsd": 0.136688,
- "meanTokenConsumption": 29278.666666666668,
- "meanToolInvocationCount": 22
+ "meanDurationMs": 101585,
+ "measuredAt": "2026-08-28T19:22:06.000Z",
+ "meanEstimatedListCostUsd": 0.09421099999999999,
+ "meanTokenConsumption": 19086.333333333332,
+ "meanToolInvocationCount": 15.666666666666666
},
{
"experimentId": "kimi-k3-opencode--baseline",
@@ -376,11 +192,11 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 74874,
- "measuredAt": "2026-08-19T21:40:48.321Z",
- "meanEstimatedListCostUsd": 0.09797499999999999,
- "meanTokenConsumption": 25718.333333333332,
- "meanToolInvocationCount": 17
+ "meanDurationMs": 98137,
+ "measuredAt": "2026-08-28T19:22:06.000Z",
+ "meanEstimatedListCostUsd": 0.097954,
+ "meanTokenConsumption": 23523.333333333332,
+ "meanToolInvocationCount": 20
},
{
"experimentId": "kimi-k3-opencode--baseline",
@@ -388,11 +204,11 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 101672,
- "measuredAt": "2026-08-19T21:40:48.324Z",
- "meanEstimatedListCostUsd": 0.17105,
- "meanTokenConsumption": 47146,
- "meanToolInvocationCount": 22.333333333333332
+ "meanDurationMs": 78637,
+ "measuredAt": "2026-08-28T19:22:06.000Z",
+ "meanEstimatedListCostUsd": 0.107612,
+ "meanTokenConsumption": 26372,
+ "meanToolInvocationCount": 19.333333333333332
},
{
"experimentId": "kimi-k3-opencode--baseline",
@@ -400,11 +216,11 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 118063,
- "measuredAt": "2026-08-19T21:40:48.328Z",
- "meanEstimatedListCostUsd": 0.144896,
- "meanTokenConsumption": 33914.666666666664,
- "meanToolInvocationCount": 18
+ "meanDurationMs": 117347,
+ "measuredAt": "2026-08-28T19:22:06.000Z",
+ "meanEstimatedListCostUsd": 0.103991,
+ "meanTokenConsumption": 21791.666666666668,
+ "meanToolInvocationCount": 16.333333333333332
},
{
"experimentId": "kimi-k3-opencode--guided",
@@ -412,11 +228,11 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 126658,
- "measuredAt": "2026-08-19T21:41:29.650Z",
- "meanEstimatedListCostUsd": 0.141837,
- "meanTokenConsumption": 34811,
- "meanToolInvocationCount": 12.666666666666666
+ "meanDurationMs": 73445,
+ "measuredAt": "2026-08-28T19:20:34.000Z",
+ "meanEstimatedListCostUsd": 0.08992099999999999,
+ "meanTokenConsumption": 20789.666666666668,
+ "meanToolInvocationCount": 12.333333333333334
},
{
"experimentId": "kimi-k3-opencode--guided",
@@ -424,11 +240,11 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 100389,
- "measuredAt": "2026-08-19T21:41:29.664Z",
- "meanEstimatedListCostUsd": 0.089512,
- "meanTokenConsumption": 23058.666666666668,
- "meanToolInvocationCount": 12.666666666666666
+ "meanDurationMs": 90748,
+ "measuredAt": "2026-08-28T19:25:31.000Z",
+ "meanEstimatedListCostUsd": 0.081476,
+ "meanTokenConsumption": 20988,
+ "meanToolInvocationCount": 12
},
{
"experimentId": "kimi-k3-opencode--guided",
@@ -436,11 +252,11 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 77889,
- "measuredAt": "2026-08-19T21:41:29.676Z",
- "meanEstimatedListCostUsd": 0.099338,
- "meanTokenConsumption": 24498,
- "meanToolInvocationCount": 14
+ "meanDurationMs": 71243,
+ "measuredAt": "2026-08-28T19:25:31.000Z",
+ "meanEstimatedListCostUsd": 0.081825,
+ "meanTokenConsumption": 19215,
+ "meanToolInvocationCount": 13.333333333333334
},
{
"experimentId": "kimi-k3-opencode--guided",
@@ -448,11 +264,11 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 62832,
- "measuredAt": "2026-08-19T21:41:29.684Z",
- "meanEstimatedListCostUsd": 0.065712,
- "meanTokenConsumption": 14302.666666666666,
- "meanToolInvocationCount": 10
+ "meanDurationMs": 126758,
+ "measuredAt": "2026-08-28T19:20:34.000Z",
+ "meanEstimatedListCostUsd": 0.08192899999999999,
+ "meanTokenConsumption": 16876.333333333332,
+ "meanToolInvocationCount": 15
},
{
"experimentId": "kimi-k3-opencode--guided",
@@ -460,11 +276,11 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 66484,
- "measuredAt": "2026-08-19T21:41:29.691Z",
- "meanEstimatedListCostUsd": 0.073148,
- "meanTokenConsumption": 18416,
- "meanToolInvocationCount": 12.333333333333334
+ "meanDurationMs": 95028,
+ "measuredAt": "2026-08-28T19:20:34.000Z",
+ "meanEstimatedListCostUsd": 0.117859,
+ "meanTokenConsumption": 29467.666666666668,
+ "meanToolInvocationCount": 16.333333333333332
},
{
"experimentId": "kimi-k3-opencode--guided",
@@ -472,10 +288,10 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 81700,
- "measuredAt": "2026-08-19T21:41:29.696Z",
- "meanEstimatedListCostUsd": 0.11507,
- "meanTokenConsumption": 32112.666666666668,
+ "meanDurationMs": 49272,
+ "measuredAt": "2026-08-28T19:20:34.000Z",
+ "meanEstimatedListCostUsd": 0.0695,
+ "meanTokenConsumption": 17840,
"meanToolInvocationCount": 10.333333333333334
},
{
@@ -484,11 +300,11 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 115791,
- "measuredAt": "2026-08-19T21:41:29.701Z",
- "meanEstimatedListCostUsd": 0.080707,
- "meanTokenConsumption": 19710.333333333332,
- "meanToolInvocationCount": 11.666666666666666
+ "meanDurationMs": 77381,
+ "measuredAt": "2026-08-28T19:20:34.000Z",
+ "meanEstimatedListCostUsd": 0.061534,
+ "meanTokenConsumption": 14636.666666666666,
+ "meanToolInvocationCount": 12.333333333333334
},
{
"experimentId": "claude-fable-5-opencode--baseline",
@@ -496,11 +312,11 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 146227,
- "measuredAt": "2026-08-19T21:45:40.647Z",
- "meanEstimatedListCostUsd": 0.26486333333333334,
- "meanTokenConsumption": 15385,
- "meanToolInvocationCount": 11.666666666666666
+ "meanDurationMs": 72217,
+ "measuredAt": "2026-08-28T19:18:44.000Z",
+ "meanEstimatedListCostUsd": 0.30839,
+ "meanTokenConsumption": 19988.333333333332,
+ "meanToolInvocationCount": 12
},
{
"experimentId": "claude-fable-5-opencode--baseline",
@@ -508,11 +324,11 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 145107,
- "measuredAt": "2026-08-19T21:45:40.656Z",
- "meanEstimatedListCostUsd": 0.1758166666666667,
- "meanTokenConsumption": 12067,
- "meanToolInvocationCount": 10.333333333333334
+ "meanDurationMs": 93899,
+ "measuredAt": "2026-08-28T19:25:21.000Z",
+ "meanEstimatedListCostUsd": 0.22944333333333336,
+ "meanTokenConsumption": 17757.666666666668,
+ "meanToolInvocationCount": 9
},
{
"experimentId": "claude-fable-5-opencode--baseline",
@@ -520,11 +336,11 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 158009,
- "measuredAt": "2026-08-19T21:45:40.664Z",
- "meanEstimatedListCostUsd": 0.2196233333333333,
- "meanTokenConsumption": 11679.666666666666,
- "meanToolInvocationCount": 19.666666666666668
+ "meanDurationMs": 80987,
+ "measuredAt": "2026-08-28T19:25:21.000Z",
+ "meanEstimatedListCostUsd": 0.3146933333333333,
+ "meanTokenConsumption": 23109.333333333332,
+ "meanToolInvocationCount": 16.666666666666668
},
{
"experimentId": "claude-fable-5-opencode--baseline",
@@ -532,11 +348,11 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 83086,
- "measuredAt": "2026-08-19T21:45:40.670Z",
- "meanEstimatedListCostUsd": 0.18186666666666665,
- "meanTokenConsumption": 10064,
- "meanToolInvocationCount": 11
+ "meanDurationMs": 62158,
+ "measuredAt": "2026-08-28T19:18:44.000Z",
+ "meanEstimatedListCostUsd": 0.22673666666666667,
+ "meanTokenConsumption": 14869.666666666666,
+ "meanToolInvocationCount": 10.333333333333334
},
{
"experimentId": "claude-fable-5-opencode--baseline",
@@ -544,11 +360,11 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 109979,
- "measuredAt": "2026-08-19T21:45:40.674Z",
- "meanEstimatedListCostUsd": 0.20606333333333335,
- "meanTokenConsumption": 14737,
- "meanToolInvocationCount": 12.333333333333334
+ "meanDurationMs": 55663,
+ "measuredAt": "2026-08-28T19:18:44.000Z",
+ "meanEstimatedListCostUsd": 0.30195666666666665,
+ "meanTokenConsumption": 24389,
+ "meanToolInvocationCount": 11.666666666666666
},
{
"experimentId": "claude-fable-5-opencode--baseline",
@@ -556,10 +372,10 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 103442,
- "measuredAt": "2026-08-19T21:45:40.679Z",
- "meanEstimatedListCostUsd": 0.15650666666666668,
- "meanTokenConsumption": 11056,
+ "meanDurationMs": 53720,
+ "measuredAt": "2026-08-28T19:18:44.000Z",
+ "meanEstimatedListCostUsd": 0.2800366666666667,
+ "meanTokenConsumption": 23315.666666666668,
"meanToolInvocationCount": 10.333333333333334
},
{
@@ -568,11 +384,11 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 82465,
- "measuredAt": "2026-08-19T21:45:40.684Z",
- "meanEstimatedListCostUsd": 0.1743066666666667,
- "meanTokenConsumption": 11940,
- "meanToolInvocationCount": 10.333333333333334
+ "meanDurationMs": 53286,
+ "measuredAt": "2026-08-28T19:18:44.000Z",
+ "meanEstimatedListCostUsd": 0.2505233333333334,
+ "meanTokenConsumption": 19769.666666666668,
+ "meanToolInvocationCount": 11
},
{
"experimentId": "claude-fable-5-opencode--guided",
@@ -580,11 +396,11 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 84096,
- "measuredAt": "2026-08-19T21:43:56.532Z",
- "meanEstimatedListCostUsd": 0.26636,
- "meanTokenConsumption": 18337.333333333332,
- "meanToolInvocationCount": 6.666666666666667
+ "meanDurationMs": 53166,
+ "measuredAt": "2026-08-28T19:18:27.000Z",
+ "meanEstimatedListCostUsd": 0.3223233333333333,
+ "meanTokenConsumption": 23616.333333333332,
+ "meanToolInvocationCount": 7.333333333333333
},
{
"experimentId": "claude-fable-5-opencode--guided",
@@ -592,23 +408,23 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 114080,
- "measuredAt": "2026-08-19T21:43:56.538Z",
- "meanEstimatedListCostUsd": 0.20138,
- "meanTokenConsumption": 14823.333333333334,
- "meanToolInvocationCount": 9.333333333333334
+ "meanDurationMs": 85836,
+ "measuredAt": "2026-08-28T19:25:03.000Z",
+ "meanEstimatedListCostUsd": 0.24718666666666667,
+ "meanTokenConsumption": 19625.333333333332,
+ "meanToolInvocationCount": 8.333333333333334
},
{
"experimentId": "claude-fable-5-opencode--guided",
"caseId": "author-003-openapi-connection",
"status": "current",
- "passedRuns": 3,
+ "passedRuns": 0,
"validRuns": 3,
- "meanDurationMs": 86233,
- "measuredAt": "2026-08-19T21:43:56.542Z",
- "meanEstimatedListCostUsd": 0.15039666666666668,
- "meanTokenConsumption": 10385,
- "meanToolInvocationCount": 8
+ "meanDurationMs": 277471,
+ "measuredAt": "2026-08-28T19:18:27.000Z",
+ "meanEstimatedListCostUsd": 0.4114,
+ "meanTokenConsumption": 19754.666666666668,
+ "meanToolInvocationCount": 7.666666666666667
},
{
"experimentId": "claude-fable-5-opencode--guided",
@@ -616,11 +432,11 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 94717,
- "measuredAt": "2026-08-19T21:43:56.546Z",
- "meanEstimatedListCostUsd": 0.24288333333333334,
- "meanTokenConsumption": 15823,
- "meanToolInvocationCount": 9.333333333333334
+ "meanDurationMs": 70438,
+ "measuredAt": "2026-08-28T19:18:27.000Z",
+ "meanEstimatedListCostUsd": 0.25587,
+ "meanTokenConsumption": 16792.333333333332,
+ "meanToolInvocationCount": 11.333333333333334
},
{
"experimentId": "claude-fable-5-opencode--guided",
@@ -628,11 +444,11 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 72597,
- "measuredAt": "2026-08-19T21:43:56.549Z",
- "meanEstimatedListCostUsd": 0.14973333333333336,
- "meanTokenConsumption": 10974.666666666666,
- "meanToolInvocationCount": 6.333333333333333
+ "meanDurationMs": 41451,
+ "measuredAt": "2026-08-28T19:18:27.000Z",
+ "meanEstimatedListCostUsd": 0.29036,
+ "meanTokenConsumption": 24874.666666666668,
+ "meanToolInvocationCount": 7
},
{
"experimentId": "claude-fable-5-opencode--guided",
@@ -640,11 +456,11 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 80685,
- "measuredAt": "2026-08-19T21:43:56.553Z",
- "meanEstimatedListCostUsd": 0.22277333333333335,
- "meanTokenConsumption": 18986.666666666668,
- "meanToolInvocationCount": 6.666666666666667
+ "meanDurationMs": 39958,
+ "measuredAt": "2026-08-28T19:18:27.000Z",
+ "meanEstimatedListCostUsd": 0.21587,
+ "meanTokenConsumption": 18063,
+ "meanToolInvocationCount": 8
},
{
"experimentId": "claude-fable-5-opencode--guided",
@@ -652,23 +468,23 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 78720,
- "measuredAt": "2026-08-19T21:43:56.556Z",
- "meanEstimatedListCostUsd": 0.12009666666666667,
- "meanTokenConsumption": 8643,
- "meanToolInvocationCount": 6.666666666666667
+ "meanDurationMs": 36170,
+ "measuredAt": "2026-08-28T19:18:27.000Z",
+ "meanEstimatedListCostUsd": 0.16552,
+ "meanTokenConsumption": 13610.666666666666,
+ "meanToolInvocationCount": 6.333333333333333
},
{
"experimentId": "grok-4-6-opencode--baseline",
"caseId": "author-001-weather-tool",
"status": "current",
- "passedRuns": 1,
+ "passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 77369,
- "measuredAt": "2026-08-19T21:03:51.002Z",
- "meanEstimatedListCostUsd": 0.115442,
- "meanTokenConsumption": 54837,
- "meanToolInvocationCount": 18
+ "meanDurationMs": 38613,
+ "measuredAt": "2026-08-28T19:22:01.000Z",
+ "meanEstimatedListCostUsd": 0.10434266666666665,
+ "meanTokenConsumption": 49566,
+ "meanToolInvocationCount": 19
},
{
"experimentId": "grok-4-6-opencode--baseline",
@@ -676,47 +492,47 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 153371,
- "measuredAt": "2026-08-19T21:03:51.007Z",
- "meanEstimatedListCostUsd": 0.07613733333333333,
- "meanTokenConsumption": 35449.333333333336,
- "meanToolInvocationCount": 17
+ "meanDurationMs": 83985,
+ "measuredAt": "2026-08-28T19:25:12.000Z",
+ "meanEstimatedListCostUsd": 0.07232866666666667,
+ "meanTokenConsumption": 32827.666666666664,
+ "meanToolInvocationCount": 16
},
{
"experimentId": "grok-4-6-opencode--baseline",
"caseId": "author-003-openapi-connection",
"status": "current",
- "passedRuns": 3,
+ "passedRuns": 0,
"validRuns": 3,
- "meanDurationMs": 183575,
- "measuredAt": "2026-08-19T21:03:51.019Z",
- "meanEstimatedListCostUsd": 0.20139533333333334,
- "meanTokenConsumption": 96650.33333333333,
- "meanToolInvocationCount": 27.666666666666668
+ "meanDurationMs": 521954,
+ "measuredAt": "2026-08-28T19:22:01.000Z",
+ "meanEstimatedListCostUsd": 1.0741786666666668,
+ "meanTokenConsumption": 503422.6666666667,
+ "meanToolInvocationCount": 76.66666666666667
},
{
"experimentId": "grok-4-6-opencode--baseline",
"caseId": "author-004-packaged-skill",
"status": "current",
- "passedRuns": 1,
+ "passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 139390,
- "measuredAt": "2026-08-19T20:13:40.145Z",
- "meanEstimatedListCostUsd": 0.107158,
- "meanTokenConsumption": 50371,
- "meanToolInvocationCount": 23
+ "meanDurationMs": 41894,
+ "measuredAt": "2026-08-28T19:22:01.000Z",
+ "meanEstimatedListCostUsd": 0.065152,
+ "meanTokenConsumption": 29506,
+ "meanToolInvocationCount": 23.333333333333332
},
{
"experimentId": "grok-4-6-opencode--baseline",
"caseId": "author-005-conditional-approval",
"status": "current",
- "passedRuns": 1,
+ "passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 144404,
- "measuredAt": "2026-08-19T20:13:40.148Z",
- "meanEstimatedListCostUsd": 0.088416,
- "meanTokenConsumption": 39046,
- "meanToolInvocationCount": 26
+ "meanDurationMs": 50151,
+ "measuredAt": "2026-08-28T19:22:01.000Z",
+ "meanEstimatedListCostUsd": 0.141536,
+ "meanTokenConsumption": 66629.33333333333,
+ "meanToolInvocationCount": 22
},
{
"experimentId": "grok-4-6-opencode--baseline",
@@ -724,11 +540,11 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 61386,
- "measuredAt": "2026-08-19T22:10:06.843Z",
- "meanEstimatedListCostUsd": 0.08925733333333334,
- "meanTokenConsumption": 42053.333333333336,
- "meanToolInvocationCount": 18.666666666666668
+ "meanDurationMs": 37062,
+ "measuredAt": "2026-08-28T19:22:01.000Z",
+ "meanEstimatedListCostUsd": 0.07319533333333333,
+ "meanTokenConsumption": 33801.666666666664,
+ "meanToolInvocationCount": 17.666666666666668
},
{
"experimentId": "grok-4-6-opencode--baseline",
@@ -736,23 +552,23 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 57619,
- "measuredAt": "2026-08-19T20:30:55.587Z",
- "meanEstimatedListCostUsd": 0.10759599999999998,
- "meanTokenConsumption": 50025.333333333336,
- "meanToolInvocationCount": 24.333333333333332
+ "meanDurationMs": 38280,
+ "measuredAt": "2026-08-28T19:22:01.000Z",
+ "meanEstimatedListCostUsd": 0.045748000000000004,
+ "meanTokenConsumption": 20122.666666666668,
+ "meanToolInvocationCount": 18
},
{
"experimentId": "grok-4-6-opencode--guided",
"caseId": "author-001-weather-tool",
"status": "current",
- "passedRuns": 2,
+ "passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 139801,
- "measuredAt": "2026-08-19T21:03:56.151Z",
- "meanEstimatedListCostUsd": 0.07778299999999999,
- "meanTokenConsumption": 36365.5,
- "meanToolInvocationCount": 13
+ "meanDurationMs": 31700,
+ "measuredAt": "2026-08-28T19:22:02.000Z",
+ "meanEstimatedListCostUsd": 0.049695333333333334,
+ "meanTokenConsumption": 22963.666666666668,
+ "meanToolInvocationCount": 9.666666666666666
},
{
"experimentId": "grok-4-6-opencode--guided",
@@ -760,11 +576,11 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 144413,
- "measuredAt": "2026-08-19T21:03:56.161Z",
- "meanEstimatedListCostUsd": 0.07078,
- "meanTokenConsumption": 32605.333333333332,
- "meanToolInvocationCount": 17.333333333333332
+ "meanDurationMs": 83621,
+ "measuredAt": "2026-08-28T19:25:07.000Z",
+ "meanEstimatedListCostUsd": 0.082842,
+ "meanTokenConsumption": 38261,
+ "meanToolInvocationCount": 16.333333333333332
},
{
"experimentId": "grok-4-6-opencode--guided",
@@ -772,35 +588,35 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 159516,
- "measuredAt": "2026-08-19T21:03:56.168Z",
- "meanEstimatedListCostUsd": 0.10099799999999999,
- "meanTokenConsumption": 49024.333333333336,
- "meanToolInvocationCount": 10.666666666666666
+ "meanDurationMs": 34833,
+ "measuredAt": "2026-08-28T19:25:07.000Z",
+ "meanEstimatedListCostUsd": 0.055174,
+ "meanTokenConsumption": 25952.333333333332,
+ "meanToolInvocationCount": 10
},
{
"experimentId": "grok-4-6-opencode--guided",
"caseId": "author-004-packaged-skill",
"status": "current",
- "passedRuns": 2,
+ "passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 143386,
- "measuredAt": "2026-08-19T21:03:56.173Z",
- "meanEstimatedListCostUsd": 0.067512,
- "meanTokenConsumption": 31751,
- "meanToolInvocationCount": 8.5
+ "meanDurationMs": 30135,
+ "measuredAt": "2026-08-28T19:22:02.000Z",
+ "meanEstimatedListCostUsd": 0.035581999999999996,
+ "meanTokenConsumption": 16043.666666666666,
+ "meanToolInvocationCount": 8.666666666666666
},
{
"experimentId": "grok-4-6-opencode--guided",
"caseId": "author-005-conditional-approval",
"status": "current",
- "passedRuns": 1,
+ "passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 75491,
- "measuredAt": "2026-08-19T21:03:56.177Z",
- "meanEstimatedListCostUsd": 0.04133,
- "meanTokenConsumption": 18681,
- "meanToolInvocationCount": 14
+ "meanDurationMs": 34026,
+ "measuredAt": "2026-08-28T19:22:02.000Z",
+ "meanEstimatedListCostUsd": 0.038532666666666666,
+ "meanTokenConsumption": 17059.666666666668,
+ "meanToolInvocationCount": 11.666666666666666
},
{
"experimentId": "grok-4-6-opencode--guided",
@@ -808,10 +624,10 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 30928,
- "measuredAt": "2026-08-19T20:23:28.755Z",
- "meanEstimatedListCostUsd": 0.026432,
- "meanTokenConsumption": 11709.333333333334,
+ "meanDurationMs": 32109,
+ "measuredAt": "2026-08-28T19:22:02.000Z",
+ "meanEstimatedListCostUsd": 0.033709333333333334,
+ "meanTokenConsumption": 14610,
"meanToolInvocationCount": 9.666666666666666
},
{
@@ -820,10 +636,10 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 33100,
- "measuredAt": "2026-08-19T20:23:28.761Z",
- "meanEstimatedListCostUsd": 0.03239266666666667,
- "meanTokenConsumption": 14937,
+ "meanDurationMs": 25583,
+ "measuredAt": "2026-08-28T19:22:02.000Z",
+ "meanEstimatedListCostUsd": 0.023458666666666666,
+ "meanTokenConsumption": 10328,
"meanToolInvocationCount": 7.333333333333333
},
{
@@ -832,10 +648,10 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 70992,
- "measuredAt": "2026-08-19T21:47:37.853Z",
- "meanEstimatedListCostUsd": 0.10240083333333333,
- "meanTokenConsumption": 27690.333333333332,
+ "meanDurationMs": 62123,
+ "measuredAt": "2026-08-28T19:16:59.000Z",
+ "meanEstimatedListCostUsd": 0.13379416666666666,
+ "meanTokenConsumption": 40526,
"meanToolInvocationCount": 27.666666666666668
},
{
@@ -844,11 +660,11 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 85989,
- "measuredAt": "2026-08-19T21:47:37.868Z",
- "meanEstimatedListCostUsd": 0.054655,
- "meanTokenConsumption": 16612,
- "meanToolInvocationCount": 15
+ "meanDurationMs": 86110,
+ "measuredAt": "2026-08-28T19:25:11.000Z",
+ "meanEstimatedListCostUsd": 0.05052583333333333,
+ "meanTokenConsumption": 14690.333333333334,
+ "meanToolInvocationCount": 15.666666666666666
},
{
"experimentId": "gpt-5-6-sol-opencode--baseline",
@@ -856,22 +672,22 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 70955,
- "measuredAt": "2026-08-19T21:47:37.878Z",
- "meanEstimatedListCostUsd": 0.13146500000000003,
- "meanTokenConsumption": 43889.333333333336,
- "meanToolInvocationCount": 28.333333333333332
+ "meanDurationMs": 62925,
+ "measuredAt": "2026-08-28T19:25:11.000Z",
+ "meanEstimatedListCostUsd": 0.09841583333333333,
+ "meanTokenConsumption": 31109.666666666668,
+ "meanToolInvocationCount": 27.333333333333332
},
{
"experimentId": "gpt-5-6-sol-opencode--baseline",
"caseId": "author-004-packaged-skill",
"status": "current",
- "passedRuns": 3,
+ "passedRuns": 2,
"validRuns": 3,
- "meanDurationMs": 68796,
- "measuredAt": "2026-08-19T21:47:37.886Z",
- "meanEstimatedListCostUsd": 0.07487333333333333,
- "meanTokenConsumption": 16249.333333333334,
+ "meanDurationMs": 60423,
+ "measuredAt": "2026-08-28T19:16:59.000Z",
+ "meanEstimatedListCostUsd": 0.04697083333333333,
+ "meanTokenConsumption": 7770,
"meanToolInvocationCount": 17.666666666666668
},
{
@@ -880,11 +696,11 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 50730,
- "measuredAt": "2026-08-19T21:47:37.891Z",
- "meanEstimatedListCostUsd": 0.07232916666666667,
- "meanTokenConsumption": 21083.333333333332,
- "meanToolInvocationCount": 21.666666666666668
+ "meanDurationMs": 63881,
+ "measuredAt": "2026-08-28T19:16:59.000Z",
+ "meanEstimatedListCostUsd": 0.10394666666666667,
+ "meanTokenConsumption": 32730.333333333332,
+ "meanToolInvocationCount": 29
},
{
"experimentId": "gpt-5-6-sol-opencode--baseline",
@@ -892,23 +708,23 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 66339,
- "measuredAt": "2026-08-19T21:47:37.896Z",
- "meanEstimatedListCostUsd": 0.09810583333333334,
- "meanTokenConsumption": 31694,
- "meanToolInvocationCount": 25
+ "meanDurationMs": 47385,
+ "measuredAt": "2026-08-28T19:16:59.000Z",
+ "meanEstimatedListCostUsd": 0.07596750000000001,
+ "meanTokenConsumption": 23072,
+ "meanToolInvocationCount": 24.333333333333332
},
{
"experimentId": "gpt-5-6-sol-opencode--baseline",
"caseId": "author-007-digest-schedule",
"status": "current",
- "passedRuns": 2,
+ "passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 43646,
- "measuredAt": "2026-08-19T21:47:37.900Z",
- "meanEstimatedListCostUsd": 0.06730583333333333,
- "meanTokenConsumption": 21627.333333333332,
- "meanToolInvocationCount": 20.666666666666668
+ "meanDurationMs": 44948,
+ "measuredAt": "2026-08-28T19:16:59.000Z",
+ "meanEstimatedListCostUsd": 0.0608525,
+ "meanTokenConsumption": 18521,
+ "meanToolInvocationCount": 22
},
{
"experimentId": "gpt-5-6-sol-opencode--guided",
@@ -916,11 +732,11 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 49812,
- "measuredAt": "2026-08-19T21:47:41.834Z",
- "meanEstimatedListCostUsd": 0.05889416666666667,
- "meanTokenConsumption": 13082.666666666666,
- "meanToolInvocationCount": 13.333333333333334
+ "meanDurationMs": 43515,
+ "measuredAt": "2026-08-28T19:16:11.000Z",
+ "meanEstimatedListCostUsd": 0.06586666666666667,
+ "meanTokenConsumption": 17868.333333333332,
+ "meanToolInvocationCount": 11
},
{
"experimentId": "gpt-5-6-sol-opencode--guided",
@@ -928,11 +744,11 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 85597,
- "measuredAt": "2026-08-19T21:47:41.847Z",
- "meanEstimatedListCostUsd": 0.0342875,
- "meanTokenConsumption": 8640,
- "meanToolInvocationCount": 14.333333333333334
+ "meanDurationMs": 80901,
+ "measuredAt": "2026-08-28T19:25:01.000Z",
+ "meanEstimatedListCostUsd": 0.05137250000000001,
+ "meanTokenConsumption": 15322.333333333334,
+ "meanToolInvocationCount": 15
},
{
"experimentId": "gpt-5-6-sol-opencode--guided",
@@ -940,11 +756,11 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 43078,
- "measuredAt": "2026-08-19T21:47:41.856Z",
- "meanEstimatedListCostUsd": 0.04016083333333333,
- "meanTokenConsumption": 11582.666666666666,
- "meanToolInvocationCount": 12
+ "meanDurationMs": 42021,
+ "measuredAt": "2026-08-28T19:25:01.000Z",
+ "meanEstimatedListCostUsd": 0.04389166666666666,
+ "meanTokenConsumption": 13418.333333333334,
+ "meanToolInvocationCount": 9
},
{
"experimentId": "gpt-5-6-sol-opencode--guided",
@@ -952,11 +768,11 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 57028,
- "measuredAt": "2026-08-19T21:47:41.862Z",
- "meanEstimatedListCostUsd": 0.04584583333333333,
- "meanTokenConsumption": 9393.333333333334,
- "meanToolInvocationCount": 7.333333333333333
+ "meanDurationMs": 45659,
+ "measuredAt": "2026-08-28T19:16:11.000Z",
+ "meanEstimatedListCostUsd": 0.04002,
+ "meanTokenConsumption": 7084.666666666667,
+ "meanToolInvocationCount": 6.666666666666667
},
{
"experimentId": "gpt-5-6-sol-opencode--guided",
@@ -964,10 +780,10 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 46860,
- "measuredAt": "2026-08-19T21:47:41.867Z",
- "meanEstimatedListCostUsd": 0.0345875,
- "meanTokenConsumption": 9476.666666666666,
+ "meanDurationMs": 34718,
+ "measuredAt": "2026-08-28T19:16:11.000Z",
+ "meanEstimatedListCostUsd": 0.04937916666666666,
+ "meanTokenConsumption": 15085,
"meanToolInvocationCount": 10.333333333333334
},
{
@@ -976,11 +792,11 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 43671,
- "measuredAt": "2026-08-19T21:47:41.871Z",
- "meanEstimatedListCostUsd": 0.03670166666666667,
- "meanTokenConsumption": 11112.333333333334,
- "meanToolInvocationCount": 9.333333333333334
+ "meanDurationMs": 36827,
+ "measuredAt": "2026-08-28T19:16:11.000Z",
+ "meanEstimatedListCostUsd": 0.0360025,
+ "meanTokenConsumption": 9957.666666666666,
+ "meanToolInvocationCount": 12
},
{
"experimentId": "gpt-5-6-sol-opencode--guided",
@@ -988,47 +804,47 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 39656,
- "measuredAt": "2026-08-19T21:47:41.875Z",
- "meanEstimatedListCostUsd": 0.036234166666666665,
- "meanTokenConsumption": 10917,
- "meanToolInvocationCount": 9.333333333333334
+ "meanDurationMs": 28258,
+ "measuredAt": "2026-08-28T19:16:11.000Z",
+ "meanEstimatedListCostUsd": 0.042734999999999995,
+ "meanTokenConsumption": 14049,
+ "meanToolInvocationCount": 8.333333333333334
},
{
"experimentId": "gpt-5-6-terra-opencode--baseline",
"caseId": "author-001-weather-tool",
"status": "current",
- "passedRuns": 3,
+ "passedRuns": 2,
"validRuns": 3,
- "meanDurationMs": 51999,
- "measuredAt": "2026-08-19T21:49:40.887Z",
- "meanEstimatedListCostUsd": 0.078974,
- "meanTokenConsumption": 27382,
- "meanToolInvocationCount": 28.666666666666668
+ "meanDurationMs": 42935,
+ "measuredAt": "2026-08-28T19:18:05.000Z",
+ "meanEstimatedListCostUsd": 0.062688,
+ "meanTokenConsumption": 22812.333333333332,
+ "meanToolInvocationCount": 19
},
{
"experimentId": "gpt-5-6-terra-opencode--baseline",
"caseId": "author-002-new-project",
"status": "current",
- "passedRuns": 3,
+ "passedRuns": 0,
"validRuns": 3,
- "meanDurationMs": 79557,
- "measuredAt": "2026-08-19T21:49:40.898Z",
- "meanEstimatedListCostUsd": 0.028153333333333336,
- "meanTokenConsumption": 8605,
- "meanToolInvocationCount": 15.666666666666666
+ "meanDurationMs": 262688,
+ "measuredAt": "2026-08-28T19:18:05.000Z",
+ "meanEstimatedListCostUsd": 0.06566466666666666,
+ "meanTokenConsumption": 10689,
+ "meanToolInvocationCount": 21
},
{
"experimentId": "gpt-5-6-terra-opencode--baseline",
"caseId": "author-003-openapi-connection",
"status": "current",
- "passedRuns": 1,
+ "passedRuns": 0,
"validRuns": 3,
- "meanDurationMs": 39576,
- "measuredAt": "2026-08-19T21:49:40.904Z",
- "meanEstimatedListCostUsd": 0.04701266666666667,
- "meanTokenConsumption": 17094.666666666668,
- "meanToolInvocationCount": 22.333333333333332
+ "meanDurationMs": 119239,
+ "measuredAt": "2026-08-28T19:18:05.000Z",
+ "meanEstimatedListCostUsd": 0.05318866666666666,
+ "meanTokenConsumption": 17214.333333333332,
+ "meanToolInvocationCount": 12
},
{
"experimentId": "gpt-5-6-terra-opencode--baseline",
@@ -1036,47 +852,47 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 47507,
- "measuredAt": "2026-08-19T21:49:40.910Z",
- "meanEstimatedListCostUsd": 0.03413466666666667,
- "meanTokenConsumption": 7104,
- "meanToolInvocationCount": 14
+ "meanDurationMs": 49408,
+ "measuredAt": "2026-08-28T19:18:05.000Z",
+ "meanEstimatedListCostUsd": 0.04473866666666667,
+ "meanTokenConsumption": 11359.333333333334,
+ "meanToolInvocationCount": 15
},
{
"experimentId": "gpt-5-6-terra-opencode--baseline",
"caseId": "author-005-conditional-approval",
"status": "current",
- "passedRuns": 3,
+ "passedRuns": 2,
"validRuns": 3,
- "meanDurationMs": 56880,
- "measuredAt": "2026-08-19T21:49:40.918Z",
- "meanEstimatedListCostUsd": 0.081758,
- "meanTokenConsumption": 32617.333333333332,
- "meanToolInvocationCount": 27
+ "meanDurationMs": 51631,
+ "measuredAt": "2026-08-28T19:18:05.000Z",
+ "meanEstimatedListCostUsd": 0.08744866666666666,
+ "meanTokenConsumption": 34586,
+ "meanToolInvocationCount": 27.333333333333332
},
{
"experimentId": "gpt-5-6-terra-opencode--baseline",
"caseId": "author-006-custom-channel",
"status": "current",
- "passedRuns": 3,
+ "passedRuns": 2,
"validRuns": 3,
- "meanDurationMs": 46407,
- "measuredAt": "2026-08-19T21:49:40.923Z",
- "meanEstimatedListCostUsd": 0.09911933333333334,
- "meanTokenConsumption": 42253,
- "meanToolInvocationCount": 23
+ "meanDurationMs": 34464,
+ "measuredAt": "2026-08-28T19:18:05.000Z",
+ "meanEstimatedListCostUsd": 0.054481999999999996,
+ "meanTokenConsumption": 21539.333333333332,
+ "meanToolInvocationCount": 19.666666666666668
},
{
"experimentId": "gpt-5-6-terra-opencode--baseline",
"caseId": "author-007-digest-schedule",
"status": "current",
- "passedRuns": 3,
+ "passedRuns": 2,
"validRuns": 3,
- "meanDurationMs": 42246,
- "measuredAt": "2026-08-19T21:49:40.927Z",
- "meanEstimatedListCostUsd": 0.057112,
- "meanTokenConsumption": 22742.666666666668,
- "meanToolInvocationCount": 22
+ "meanDurationMs": 30799,
+ "measuredAt": "2026-08-28T19:18:05.000Z",
+ "meanEstimatedListCostUsd": 0.03727,
+ "meanTokenConsumption": 14183.333333333334,
+ "meanToolInvocationCount": 14.333333333333334
},
{
"experimentId": "gpt-5-6-terra-opencode--guided",
@@ -1084,11 +900,11 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 41608,
- "measuredAt": "2026-08-19T21:49:28.518Z",
- "meanEstimatedListCostUsd": 0.06261066666666666,
- "meanTokenConsumption": 21113.666666666668,
- "meanToolInvocationCount": 17
+ "meanDurationMs": 42543,
+ "measuredAt": "2026-08-28T19:16:16.000Z",
+ "meanEstimatedListCostUsd": 0.053156666666666665,
+ "meanTokenConsumption": 16486.666666666668,
+ "meanToolInvocationCount": 16
},
{
"experimentId": "gpt-5-6-terra-opencode--guided",
@@ -1096,11 +912,11 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 74695,
- "measuredAt": "2026-08-19T21:49:28.526Z",
- "meanEstimatedListCostUsd": 0.049492666666666664,
- "meanTokenConsumption": 19446.333333333332,
- "meanToolInvocationCount": 15.666666666666666
+ "meanDurationMs": 80660,
+ "measuredAt": "2026-08-28T19:25:15.000Z",
+ "meanEstimatedListCostUsd": 0.04396933333333333,
+ "meanTokenConsumption": 16811.333333333332,
+ "meanToolInvocationCount": 14
},
{
"experimentId": "gpt-5-6-terra-opencode--guided",
@@ -1108,11 +924,11 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 40796,
- "measuredAt": "2026-08-19T21:49:28.533Z",
- "meanEstimatedListCostUsd": 0.042858,
- "meanTokenConsumption": 17250.666666666668,
- "meanToolInvocationCount": 11.333333333333334
+ "meanDurationMs": 44816,
+ "measuredAt": "2026-08-28T19:25:15.000Z",
+ "meanEstimatedListCostUsd": 0.040738,
+ "meanTokenConsumption": 14277.333333333334,
+ "meanToolInvocationCount": 18
},
{
"experimentId": "gpt-5-6-terra-opencode--guided",
@@ -1120,23 +936,23 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 44243,
- "measuredAt": "2026-08-19T21:49:28.538Z",
- "meanEstimatedListCostUsd": 0.04276266666666667,
- "meanTokenConsumption": 11531.333333333334,
- "meanToolInvocationCount": 14
+ "meanDurationMs": 44710,
+ "measuredAt": "2026-08-28T19:16:16.000Z",
+ "meanEstimatedListCostUsd": 0.044548,
+ "meanTokenConsumption": 13047.333333333334,
+ "meanToolInvocationCount": 13.333333333333334
},
{
"experimentId": "gpt-5-6-terra-opencode--guided",
"caseId": "author-005-conditional-approval",
"status": "current",
- "passedRuns": 2,
+ "passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 33729,
- "measuredAt": "2026-08-19T21:49:28.542Z",
- "meanEstimatedListCostUsd": 0.03496066666666667,
- "meanTokenConsumption": 12010.333333333334,
- "meanToolInvocationCount": 14.666666666666666
+ "meanDurationMs": 36268,
+ "measuredAt": "2026-08-28T19:16:16.000Z",
+ "meanEstimatedListCostUsd": 0.046585999999999995,
+ "meanTokenConsumption": 17268,
+ "meanToolInvocationCount": 16
},
{
"experimentId": "gpt-5-6-terra-opencode--guided",
@@ -1144,11 +960,11 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 37089,
- "measuredAt": "2026-08-19T21:49:28.546Z",
- "meanEstimatedListCostUsd": 0.02781866666666667,
- "meanTokenConsumption": 9269.333333333334,
- "meanToolInvocationCount": 13
+ "meanDurationMs": 34596,
+ "measuredAt": "2026-08-28T19:16:16.000Z",
+ "meanEstimatedListCostUsd": 0.04468933333333333,
+ "meanTokenConsumption": 16843,
+ "meanToolInvocationCount": 15.333333333333334
},
{
"experimentId": "gpt-5-6-terra-opencode--guided",
@@ -1156,11 +972,11 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 36703,
- "measuredAt": "2026-08-19T21:49:28.550Z",
- "meanEstimatedListCostUsd": 0.022504,
- "meanTokenConsumption": 7360.333333333333,
- "meanToolInvocationCount": 11.333333333333334
+ "meanDurationMs": 27983,
+ "measuredAt": "2026-08-28T19:16:16.000Z",
+ "meanEstimatedListCostUsd": 0.022269333333333335,
+ "meanTokenConsumption": 7259.666666666667,
+ "meanToolInvocationCount": 11.666666666666666
},
{
"experimentId": "claude-sonnet-5-opencode--baseline",
@@ -1168,11 +984,11 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 59225,
- "measuredAt": "2026-08-19T21:58:17.895Z",
- "meanEstimatedListCostUsd": 0.08490400000000002,
- "meanTokenConsumption": 33980,
- "meanToolInvocationCount": 13
+ "meanDurationMs": 84504,
+ "measuredAt": "2026-08-28T19:20:33.000Z",
+ "meanEstimatedListCostUsd": 0.087304,
+ "meanTokenConsumption": 30901.333333333332,
+ "meanToolInvocationCount": 17.666666666666668
},
{
"experimentId": "claude-sonnet-5-opencode--baseline",
@@ -1180,11 +996,11 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 90978,
- "measuredAt": "2026-08-19T21:58:17.902Z",
- "meanEstimatedListCostUsd": 0.041563333333333334,
- "meanTokenConsumption": 13628.333333333334,
- "meanToolInvocationCount": 12
+ "meanDurationMs": 85287,
+ "measuredAt": "2026-08-28T19:27:58.000Z",
+ "meanEstimatedListCostUsd": 0.047190666666666665,
+ "meanTokenConsumption": 16207.333333333334,
+ "meanToolInvocationCount": 12.333333333333334
},
{
"experimentId": "claude-sonnet-5-opencode--baseline",
@@ -1192,11 +1008,11 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 195665,
- "measuredAt": "2026-08-19T21:58:17.912Z",
- "meanEstimatedListCostUsd": 0.1760786666666667,
- "meanTokenConsumption": 50079.333333333336,
- "meanToolInvocationCount": 39.666666666666664
+ "meanDurationMs": 178438,
+ "measuredAt": "2026-08-28T19:27:58.000Z",
+ "meanEstimatedListCostUsd": 0.23833933333333335,
+ "meanTokenConsumption": 86543,
+ "meanToolInvocationCount": 37
},
{
"experimentId": "claude-sonnet-5-opencode--baseline",
@@ -1204,11 +1020,11 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 108879,
- "measuredAt": "2026-08-19T21:58:17.920Z",
- "meanEstimatedListCostUsd": 0.14531933333333333,
- "meanTokenConsumption": 52455.666666666664,
- "meanToolInvocationCount": 20.666666666666668
+ "meanDurationMs": 69785,
+ "measuredAt": "2026-08-28T19:20:33.000Z",
+ "meanEstimatedListCostUsd": 0.05839,
+ "meanTokenConsumption": 16595,
+ "meanToolInvocationCount": 14
},
{
"experimentId": "claude-sonnet-5-opencode--baseline",
@@ -1216,11 +1032,11 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 95353,
- "measuredAt": "2026-08-19T21:58:17.926Z",
- "meanEstimatedListCostUsd": 0.11524333333333335,
- "meanTokenConsumption": 43313.666666666664,
- "meanToolInvocationCount": 22.666666666666668
+ "meanDurationMs": 69671,
+ "measuredAt": "2026-08-28T19:20:33.000Z",
+ "meanEstimatedListCostUsd": 0.08692,
+ "meanTokenConsumption": 33568,
+ "meanToolInvocationCount": 15.666666666666666
},
{
"experimentId": "claude-sonnet-5-opencode--baseline",
@@ -1228,23 +1044,23 @@
"status": "current",
"passedRuns": 1,
"validRuns": 3,
- "meanDurationMs": 291562,
- "measuredAt": "2026-08-19T21:58:17.931Z",
- "meanEstimatedListCostUsd": 0.045122666666666665,
- "meanTokenConsumption": 17692,
- "meanToolInvocationCount": 10
+ "meanDurationMs": 352409,
+ "measuredAt": "2026-08-28T19:20:33.000Z",
+ "meanEstimatedListCostUsd": 0.017415333333333335,
+ "meanTokenConsumption": 5797,
+ "meanToolInvocationCount": 6
},
{
"experimentId": "claude-sonnet-5-opencode--baseline",
"caseId": "author-007-digest-schedule",
"status": "current",
- "passedRuns": 2,
+ "passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 116395,
- "measuredAt": "2026-08-19T21:58:17.936Z",
- "meanEstimatedListCostUsd": 0.10074666666666666,
- "meanTokenConsumption": 33780,
- "meanToolInvocationCount": 19.333333333333332
+ "meanDurationMs": 69696,
+ "measuredAt": "2026-08-28T19:20:33.000Z",
+ "meanEstimatedListCostUsd": 0.06399133333333333,
+ "meanTokenConsumption": 21950.333333333332,
+ "meanToolInvocationCount": 14
},
{
"experimentId": "claude-sonnet-5-opencode--guided",
@@ -1252,11 +1068,11 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 105884,
- "measuredAt": "2026-08-19T21:52:41.883Z",
- "meanEstimatedListCostUsd": 0.09806266666666667,
- "meanTokenConsumption": 30726,
- "meanToolInvocationCount": 14.666666666666666
+ "meanDurationMs": 72487,
+ "measuredAt": "2026-08-28T19:19:54.000Z",
+ "meanEstimatedListCostUsd": 0.06199066666666666,
+ "meanTokenConsumption": 20955.333333333332,
+ "meanToolInvocationCount": 12.333333333333334
},
{
"experimentId": "claude-sonnet-5-opencode--guided",
@@ -1264,11 +1080,11 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 106358,
- "measuredAt": "2026-08-19T21:52:41.896Z",
- "meanEstimatedListCostUsd": 0.06011466666666667,
- "meanTokenConsumption": 20532,
- "meanToolInvocationCount": 14.666666666666666
+ "meanDurationMs": 97671,
+ "measuredAt": "2026-08-28T19:25:30.000Z",
+ "meanEstimatedListCostUsd": 0.037938000000000006,
+ "meanTokenConsumption": 11189,
+ "meanToolInvocationCount": 13.666666666666666
},
{
"experimentId": "claude-sonnet-5-opencode--guided",
@@ -1276,11 +1092,11 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 57823,
- "measuredAt": "2026-08-19T21:52:41.905Z",
- "meanEstimatedListCostUsd": 0.047889999999999995,
- "meanTokenConsumption": 17711.666666666668,
- "meanToolInvocationCount": 11.333333333333334
+ "meanDurationMs": 53872,
+ "measuredAt": "2026-08-28T19:25:30.000Z",
+ "meanEstimatedListCostUsd": 0.06860866666666667,
+ "meanTokenConsumption": 27953.666666666668,
+ "meanToolInvocationCount": 12
},
{
"experimentId": "claude-sonnet-5-opencode--guided",
@@ -1288,11 +1104,11 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 50374,
- "measuredAt": "2026-08-19T21:52:41.911Z",
- "meanEstimatedListCostUsd": 0.04749,
- "meanTokenConsumption": 16147.666666666666,
- "meanToolInvocationCount": 8
+ "meanDurationMs": 54347,
+ "measuredAt": "2026-08-28T19:19:54.000Z",
+ "meanEstimatedListCostUsd": 0.05166133333333334,
+ "meanTokenConsumption": 17134.666666666668,
+ "meanToolInvocationCount": 10.333333333333334
},
{
"experimentId": "claude-sonnet-5-opencode--guided",
@@ -1300,11 +1116,11 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 44694,
- "measuredAt": "2026-08-19T21:52:41.917Z",
- "meanEstimatedListCostUsd": 0.05203,
- "meanTokenConsumption": 20456.333333333332,
- "meanToolInvocationCount": 9.666666666666666
+ "meanDurationMs": 57140,
+ "measuredAt": "2026-08-28T19:19:54.000Z",
+ "meanEstimatedListCostUsd": 0.04402533333333333,
+ "meanTokenConsumption": 15630,
+ "meanToolInvocationCount": 12
},
{
"experimentId": "claude-sonnet-5-opencode--guided",
@@ -1312,11 +1128,11 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 45902,
- "measuredAt": "2026-08-19T21:52:41.922Z",
- "meanEstimatedListCostUsd": 0.05530466666666667,
- "meanTokenConsumption": 22887,
- "meanToolInvocationCount": 10
+ "meanDurationMs": 46268,
+ "measuredAt": "2026-08-28T19:19:54.000Z",
+ "meanEstimatedListCostUsd": 0.026169333333333333,
+ "meanTokenConsumption": 8867.333333333334,
+ "meanToolInvocationCount": 8
},
{
"experimentId": "claude-sonnet-5-opencode--guided",
@@ -1324,11 +1140,11 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 37072,
- "measuredAt": "2026-08-19T21:52:41.927Z",
- "meanEstimatedListCostUsd": 0.017157333333333333,
- "meanTokenConsumption": 6138.666666666667,
- "meanToolInvocationCount": 5.333333333333333
+ "meanDurationMs": 27822,
+ "measuredAt": "2026-08-28T19:19:54.000Z",
+ "meanEstimatedListCostUsd": 0.01684333333333333,
+ "meanTokenConsumption": 5760.333333333333,
+ "meanToolInvocationCount": 6
},
{
"experimentId": "glm-5-2-opencode--baseline",
@@ -1336,11 +1152,11 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 45432,
- "measuredAt": "2026-08-19T22:00:25.730Z",
- "meanEstimatedListCostUsd": 0.018507533333333333,
- "meanTokenConsumption": 11404.666666666666,
- "meanToolInvocationCount": 17.333333333333332
+ "meanDurationMs": 103530,
+ "measuredAt": "2026-08-28T19:22:09.000Z",
+ "meanEstimatedListCostUsd": 0.07017806666666666,
+ "meanTokenConsumption": 37804.333333333336,
+ "meanToolInvocationCount": 25
},
{
"experimentId": "glm-5-2-opencode--baseline",
@@ -1348,11 +1164,11 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 80706,
- "measuredAt": "2026-08-19T22:00:25.746Z",
- "meanEstimatedListCostUsd": 0.011617466666666668,
- "meanTokenConsumption": 6995.333333333333,
- "meanToolInvocationCount": 11.333333333333334
+ "meanDurationMs": 100847,
+ "measuredAt": "2026-08-28T19:26:40.000Z",
+ "meanEstimatedListCostUsd": 0.03248553333333333,
+ "meanTokenConsumption": 19034.666666666668,
+ "meanToolInvocationCount": 18.333333333333332
},
{
"experimentId": "glm-5-2-opencode--baseline",
@@ -1360,11 +1176,11 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 60232,
- "measuredAt": "2026-08-19T22:00:25.755Z",
- "meanEstimatedListCostUsd": 0.027101133333333333,
- "meanTokenConsumption": 15908.666666666666,
- "meanToolInvocationCount": 28
+ "meanDurationMs": 134009,
+ "measuredAt": "2026-08-28T19:26:40.000Z",
+ "meanEstimatedListCostUsd": 0.09391753333333334,
+ "meanTokenConsumption": 49094.666666666664,
+ "meanToolInvocationCount": 40.333333333333336
},
{
"experimentId": "glm-5-2-opencode--baseline",
@@ -1372,23 +1188,23 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 49435,
- "measuredAt": "2026-08-19T22:00:25.762Z",
- "meanEstimatedListCostUsd": 0.018872933333333335,
- "meanTokenConsumption": 8945.666666666666,
- "meanToolInvocationCount": 13.666666666666666
+ "meanDurationMs": 214119,
+ "measuredAt": "2026-08-28T19:22:09.000Z",
+ "meanEstimatedListCostUsd": 0.07759993333333333,
+ "meanTokenConsumption": 41325.666666666664,
+ "meanToolInvocationCount": 36.333333333333336
},
{
"experimentId": "glm-5-2-opencode--baseline",
"caseId": "author-005-conditional-approval",
"status": "current",
- "passedRuns": 2,
+ "passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 60175,
- "measuredAt": "2026-08-19T22:00:25.768Z",
- "meanEstimatedListCostUsd": 0.02182706666666667,
- "meanTokenConsumption": 13249.333333333334,
- "meanToolInvocationCount": 20.333333333333332
+ "meanDurationMs": 58527,
+ "measuredAt": "2026-08-28T19:22:09.000Z",
+ "meanEstimatedListCostUsd": 0.07383253333333333,
+ "meanTokenConsumption": 48329.666666666664,
+ "meanToolInvocationCount": 25.333333333333332
},
{
"experimentId": "glm-5-2-opencode--baseline",
@@ -1396,23 +1212,23 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 69165,
- "measuredAt": "2026-08-19T22:00:25.773Z",
- "meanEstimatedListCostUsd": 0.04950993333333333,
- "meanTokenConsumption": 30240.666666666668,
+ "meanDurationMs": 75109,
+ "measuredAt": "2026-08-28T19:22:09.000Z",
+ "meanEstimatedListCostUsd": 0.07836139999999998,
+ "meanTokenConsumption": 48506,
"meanToolInvocationCount": 27.666666666666668
},
{
"experimentId": "glm-5-2-opencode--baseline",
"caseId": "author-007-digest-schedule",
"status": "current",
- "passedRuns": 1,
+ "passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 85415,
- "measuredAt": "2026-08-19T22:00:25.778Z",
- "meanEstimatedListCostUsd": 0.0311514,
- "meanTokenConsumption": 16861,
- "meanToolInvocationCount": 15.333333333333334
+ "meanDurationMs": 93384,
+ "measuredAt": "2026-08-28T19:22:09.000Z",
+ "meanEstimatedListCostUsd": 0.04282053333333333,
+ "meanTokenConsumption": 24309.666666666668,
+ "meanToolInvocationCount": 19
},
{
"experimentId": "glm-5-2-opencode--guided",
@@ -1420,11 +1236,11 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 44063,
- "measuredAt": "2026-08-19T22:00:30.879Z",
- "meanEstimatedListCostUsd": 0.009911666666666666,
- "meanTokenConsumption": 5828.333333333333,
- "meanToolInvocationCount": 9.333333333333334
+ "meanDurationMs": 68607,
+ "measuredAt": "2026-08-28T19:22:01.000Z",
+ "meanEstimatedListCostUsd": 0.04188693333333334,
+ "meanTokenConsumption": 22360.666666666668,
+ "meanToolInvocationCount": 16.333333333333332
},
{
"experimentId": "glm-5-2-opencode--guided",
@@ -1432,23 +1248,23 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 81227,
- "measuredAt": "2026-08-19T22:00:30.891Z",
- "meanEstimatedListCostUsd": 0.010340466666666666,
- "meanTokenConsumption": 5905.333333333333,
- "meanToolInvocationCount": 10.666666666666666
+ "meanDurationMs": 90517,
+ "measuredAt": "2026-08-28T19:25:29.000Z",
+ "meanEstimatedListCostUsd": 0.02843253333333333,
+ "meanTokenConsumption": 16824.666666666668,
+ "meanToolInvocationCount": 14.333333333333334
},
{
"experimentId": "glm-5-2-opencode--guided",
"caseId": "author-003-openapi-connection",
"status": "current",
- "passedRuns": 3,
+ "passedRuns": 0,
"validRuns": 3,
- "meanDurationMs": 42824,
- "measuredAt": "2026-08-19T22:00:30.905Z",
- "meanEstimatedListCostUsd": 0.0144878,
- "meanTokenConsumption": 9217,
- "meanToolInvocationCount": 10.666666666666666
+ "meanDurationMs": 524221,
+ "measuredAt": "2026-08-28T19:22:01.000Z",
+ "meanEstimatedListCostUsd": 0.1800297333333333,
+ "meanTokenConsumption": 78267.66666666667,
+ "meanToolInvocationCount": 24.666666666666668
},
{
"experimentId": "glm-5-2-opencode--guided",
@@ -1456,23 +1272,23 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 46385,
- "measuredAt": "2026-08-19T22:00:30.912Z",
- "meanEstimatedListCostUsd": 0.011119466666666666,
- "meanTokenConsumption": 5550.333333333333,
- "meanToolInvocationCount": 8.666666666666666
+ "meanDurationMs": 68821,
+ "measuredAt": "2026-08-28T19:22:01.000Z",
+ "meanEstimatedListCostUsd": 0.031183066666666665,
+ "meanTokenConsumption": 16504.333333333332,
+ "meanToolInvocationCount": 15
},
{
"experimentId": "glm-5-2-opencode--guided",
"caseId": "author-005-conditional-approval",
"status": "current",
- "passedRuns": 3,
+ "passedRuns": 2,
"validRuns": 3,
- "meanDurationMs": 55905,
- "measuredAt": "2026-08-19T22:00:30.917Z",
- "meanEstimatedListCostUsd": 0.014120933333333334,
- "meanTokenConsumption": 8725.666666666666,
- "meanToolInvocationCount": 10
+ "meanDurationMs": 80061,
+ "measuredAt": "2026-08-28T19:22:01.000Z",
+ "meanEstimatedListCostUsd": 0.040431333333333326,
+ "meanTokenConsumption": 26306.666666666668,
+ "meanToolInvocationCount": 14.333333333333334
},
{
"experimentId": "glm-5-2-opencode--guided",
@@ -1480,11 +1296,11 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 43974,
- "measuredAt": "2026-08-19T22:00:30.922Z",
- "meanEstimatedListCostUsd": 0.015401466666666667,
- "meanTokenConsumption": 9535.333333333334,
- "meanToolInvocationCount": 10.666666666666666
+ "meanDurationMs": 56079,
+ "measuredAt": "2026-08-28T19:22:01.000Z",
+ "meanEstimatedListCostUsd": 0.03310953333333334,
+ "meanTokenConsumption": 19814.666666666668,
+ "meanToolInvocationCount": 11
},
{
"experimentId": "glm-5-2-opencode--guided",
@@ -1492,95 +1308,95 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 35151,
- "measuredAt": "2026-08-19T22:00:30.925Z",
- "meanEstimatedListCostUsd": 0.005594666666666667,
- "meanTokenConsumption": 3248.3333333333335,
- "meanToolInvocationCount": 7
+ "meanDurationMs": 74552,
+ "measuredAt": "2026-08-28T19:22:01.000Z",
+ "meanEstimatedListCostUsd": 0.029523933333333335,
+ "meanTokenConsumption": 16295.666666666666,
+ "meanToolInvocationCount": 12.333333333333334
},
{
"experimentId": "gemini-3-1-pro-preview-opencode--baseline",
"caseId": "author-001-weather-tool",
"status": "current",
- "passedRuns": 1,
+ "passedRuns": 0,
"validRuns": 3,
- "meanDurationMs": 67954,
- "measuredAt": "2026-08-19T22:06:54.450Z",
- "meanEstimatedListCostUsd": 0.17139933333333335,
- "meanTokenConsumption": 78096.33333333333,
- "meanToolInvocationCount": 18
+ "meanDurationMs": 54876,
+ "measuredAt": "2026-08-28T19:20:40.000Z",
+ "meanEstimatedListCostUsd": 0.11818266666666667,
+ "meanTokenConsumption": 45096.333333333336,
+ "meanToolInvocationCount": 9.666666666666666
},
{
"experimentId": "gemini-3-1-pro-preview-opencode--baseline",
"caseId": "author-002-new-project",
"status": "current",
- "passedRuns": 2,
+ "passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 176726,
- "measuredAt": "2026-08-19T22:06:54.459Z",
- "meanEstimatedListCostUsd": 0.12588133333333332,
- "meanTokenConsumption": 57477.333333333336,
- "meanToolInvocationCount": 19
+ "meanDurationMs": 82415,
+ "measuredAt": "2026-08-28T19:26:24.000Z",
+ "meanEstimatedListCostUsd": 0.11070399999999998,
+ "meanTokenConsumption": 51112,
+ "meanToolInvocationCount": 11.666666666666666
},
{
"experimentId": "gemini-3-1-pro-preview-opencode--baseline",
"caseId": "author-003-openapi-connection",
"status": "current",
- "passedRuns": 2,
+ "passedRuns": 1,
"validRuns": 3,
- "meanDurationMs": 105009,
- "measuredAt": "2026-08-19T22:06:54.468Z",
- "meanEstimatedListCostUsd": 0.20446399999999998,
- "meanTokenConsumption": 93695.33333333333,
- "meanToolInvocationCount": 22
+ "meanDurationMs": 102903,
+ "measuredAt": "2026-08-28T19:26:24.000Z",
+ "meanEstimatedListCostUsd": 0.43870533333333334,
+ "meanTokenConsumption": 211094.33333333334,
+ "meanToolInvocationCount": 28
},
{
"experimentId": "gemini-3-1-pro-preview-opencode--baseline",
"caseId": "author-004-packaged-skill",
"status": "current",
- "passedRuns": 1,
+ "passedRuns": 2,
"validRuns": 3,
- "meanDurationMs": 49418,
- "measuredAt": "2026-08-19T22:06:54.477Z",
- "meanEstimatedListCostUsd": 0.04231066666666666,
- "meanTokenConsumption": 13160.333333333334,
- "meanToolInvocationCount": 5.666666666666667
+ "meanDurationMs": 33942,
+ "measuredAt": "2026-08-28T19:20:40.000Z",
+ "meanEstimatedListCostUsd": 0.07116533333333334,
+ "meanTokenConsumption": 26049.333333333332,
+ "meanToolInvocationCount": 5.333333333333333
},
{
"experimentId": "gemini-3-1-pro-preview-opencode--baseline",
"caseId": "author-005-conditional-approval",
"status": "current",
- "passedRuns": 0,
+ "passedRuns": 2,
"validRuns": 3,
- "meanDurationMs": 100093,
- "measuredAt": "2026-08-19T22:06:54.483Z",
- "meanEstimatedListCostUsd": 0.27252733333333334,
- "meanTokenConsumption": 126355.33333333333,
- "meanToolInvocationCount": 24.666666666666668
+ "meanDurationMs": 227315,
+ "measuredAt": "2026-08-28T19:20:40.000Z",
+ "meanEstimatedListCostUsd": 0.5367073333333333,
+ "meanTokenConsumption": 251920.33333333334,
+ "meanToolInvocationCount": 42.666666666666664
},
{
"experimentId": "gemini-3-1-pro-preview-opencode--baseline",
"caseId": "author-006-custom-channel",
"status": "current",
- "passedRuns": 2,
+ "passedRuns": 1,
"validRuns": 3,
- "meanDurationMs": 184235,
- "measuredAt": "2026-08-19T22:06:54.488Z",
- "meanEstimatedListCostUsd": 0.5857926666666667,
- "meanTokenConsumption": 278326.3333333333,
- "meanToolInvocationCount": 48.333333333333336
+ "meanDurationMs": 133303,
+ "measuredAt": "2026-08-28T19:20:40.000Z",
+ "meanEstimatedListCostUsd": 0.467548,
+ "meanTokenConsumption": 225615.66666666666,
+ "meanToolInvocationCount": 30.666666666666668
},
{
"experimentId": "gemini-3-1-pro-preview-opencode--baseline",
"caseId": "author-007-digest-schedule",
"status": "current",
- "passedRuns": 2,
+ "passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 149748,
- "measuredAt": "2026-08-19T22:06:54.495Z",
- "meanEstimatedListCostUsd": 0.12809666666666666,
- "meanTokenConsumption": 57671.666666666664,
- "meanToolInvocationCount": 16.333333333333332
+ "meanDurationMs": 61813,
+ "measuredAt": "2026-08-28T19:20:40.000Z",
+ "meanEstimatedListCostUsd": 0.212552,
+ "meanTokenConsumption": 101059.33333333333,
+ "meanToolInvocationCount": 15
},
{
"experimentId": "gemini-3-1-pro-preview-opencode--guided",
@@ -1588,35 +1404,35 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 38449,
- "measuredAt": "2026-08-19T22:03:45.194Z",
- "meanEstimatedListCostUsd": 0.068454,
- "meanTokenConsumption": 31198.666666666668,
- "meanToolInvocationCount": 8
+ "meanDurationMs": 37112,
+ "measuredAt": "2026-08-28T19:17:42.000Z",
+ "meanEstimatedListCostUsd": 0.08827333333333333,
+ "meanTokenConsumption": 40508.333333333336,
+ "meanToolInvocationCount": 7.666666666666667
},
{
"experimentId": "gemini-3-1-pro-preview-opencode--guided",
"caseId": "author-002-new-project",
"status": "current",
- "passedRuns": 3,
+ "passedRuns": 0,
"validRuns": 3,
- "meanDurationMs": 105467,
- "measuredAt": "2026-08-19T22:03:45.206Z",
- "meanEstimatedListCostUsd": 0.07058666666666667,
- "meanTokenConsumption": 32808.333333333336,
- "meanToolInvocationCount": 10
+ "meanDurationMs": 260182,
+ "measuredAt": "2026-08-28T19:17:42.000Z",
+ "meanEstimatedListCostUsd": 0.237908,
+ "meanTokenConsumption": 109484,
+ "meanToolInvocationCount": 28.333333333333332
},
{
"experimentId": "gemini-3-1-pro-preview-opencode--guided",
"caseId": "author-003-openapi-connection",
"status": "current",
- "passedRuns": 3,
+ "passedRuns": 0,
"validRuns": 3,
- "meanDurationMs": 54859,
- "measuredAt": "2026-08-19T22:03:45.214Z",
- "meanEstimatedListCostUsd": 0.10357533333333334,
- "meanTokenConsumption": 48067.666666666664,
- "meanToolInvocationCount": 10.666666666666666
+ "meanDurationMs": 201322,
+ "measuredAt": "2026-08-28T19:17:42.000Z",
+ "meanEstimatedListCostUsd": 0.3107266666666667,
+ "meanTokenConsumption": 132131.66666666666,
+ "meanToolInvocationCount": 34.333333333333336
},
{
"experimentId": "gemini-3-1-pro-preview-opencode--guided",
@@ -1624,11 +1440,11 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 91269,
- "measuredAt": "2026-08-19T22:03:45.221Z",
- "meanEstimatedListCostUsd": 0.05254333333333333,
- "meanTokenConsumption": 21473.333333333332,
- "meanToolInvocationCount": 7.666666666666667
+ "meanDurationMs": 41440,
+ "measuredAt": "2026-08-28T19:17:42.000Z",
+ "meanEstimatedListCostUsd": 0.08369866666666666,
+ "meanTokenConsumption": 34357.666666666664,
+ "meanToolInvocationCount": 8.666666666666666
},
{
"experimentId": "gemini-3-1-pro-preview-opencode--guided",
@@ -1636,11 +1452,11 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 65793,
- "measuredAt": "2026-08-19T22:03:45.226Z",
- "meanEstimatedListCostUsd": 0.07310599999999999,
- "meanTokenConsumption": 33233,
- "meanToolInvocationCount": 8
+ "meanDurationMs": 33301,
+ "measuredAt": "2026-08-28T19:17:42.000Z",
+ "meanEstimatedListCostUsd": 0.09258866666666667,
+ "meanTokenConsumption": 42437.666666666664,
+ "meanToolInvocationCount": 7.333333333333333
},
{
"experimentId": "gemini-3-1-pro-preview-opencode--guided",
@@ -1648,11 +1464,11 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 56233,
- "measuredAt": "2026-08-19T22:03:45.232Z",
- "meanEstimatedListCostUsd": 0.08904266666666667,
- "meanTokenConsumption": 41173,
- "meanToolInvocationCount": 7
+ "meanDurationMs": 34493,
+ "measuredAt": "2026-08-28T19:17:42.000Z",
+ "meanEstimatedListCostUsd": 0.09662666666666668,
+ "meanTokenConsumption": 45305,
+ "meanToolInvocationCount": 7.333333333333333
},
{
"experimentId": "gemini-3-1-pro-preview-opencode--guided",
@@ -1660,11 +1476,209 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 79041,
- "measuredAt": "2026-08-19T22:03:45.237Z",
- "meanEstimatedListCostUsd": 0.08614333333333335,
- "meanTokenConsumption": 40725,
- "meanToolInvocationCount": 7.333333333333333
+ "meanDurationMs": 33257,
+ "measuredAt": "2026-08-28T19:17:42.000Z",
+ "meanEstimatedListCostUsd": 0.059873333333333334,
+ "meanTokenConsumption": 27681.666666666668,
+ "meanToolInvocationCount": 4.666666666666667
+ }
+ ],
+ "previouslyMeasured": [
+ {
+ "suite": {
+ "eveRevision": "54a028bab589a60fc84a310304a5cf080bc7f990",
+ "caseFingerprint": "786490be9d902701f57d08b3d112ca385d49a72e5591fac741511e77d67989d2",
+ "caseCount": 7,
+ "runsPerCell": 3
+ },
+ "experiments": [
+ {
+ "id": "claude-sonnet-4-6-opencode--baseline",
+ "groupId": "claude-sonnet-4-6-opencode",
+ "model": "claude-sonnet-4-6",
+ "modelDisplayName": "Claude Sonnet 4.6",
+ "harness": "OpenCode",
+ "treatment": "baseline"
+ },
+ {
+ "id": "claude-sonnet-4-6-opencode--guided",
+ "groupId": "claude-sonnet-4-6-opencode",
+ "model": "claude-sonnet-4-6",
+ "modelDisplayName": "Claude Sonnet 4.6",
+ "harness": "OpenCode",
+ "treatment": "guided"
+ }
+ ],
+ "results": [
+ {
+ "experimentId": "claude-sonnet-4-6-opencode--baseline",
+ "caseId": "author-001-weather-tool",
+ "status": "current",
+ "passedRuns": 0,
+ "validRuns": 3,
+ "meanDurationMs": 199451,
+ "measuredAt": "2026-08-19T21:35:21.709Z",
+ "meanEstimatedListCostUsd": 0.052948,
+ "meanTokenConsumption": 10626.666666666666,
+ "meanToolInvocationCount": 11.666666666666666
+ },
+ {
+ "experimentId": "claude-sonnet-4-6-opencode--baseline",
+ "caseId": "author-002-new-project",
+ "status": "current",
+ "passedRuns": 0,
+ "validRuns": 3,
+ "meanDurationMs": 143527,
+ "measuredAt": "2026-08-19T21:35:21.715Z",
+ "meanEstimatedListCostUsd": 0.04594499999999999,
+ "meanTokenConsumption": 10429.666666666666,
+ "meanToolInvocationCount": 10.666666666666666
+ },
+ {
+ "experimentId": "claude-sonnet-4-6-opencode--baseline",
+ "caseId": "author-003-openapi-connection",
+ "status": "current",
+ "passedRuns": 1,
+ "validRuns": 3,
+ "meanDurationMs": 158661,
+ "measuredAt": "2026-08-19T21:35:21.721Z",
+ "meanEstimatedListCostUsd": 0.11106500000000001,
+ "meanTokenConsumption": 27383,
+ "meanToolInvocationCount": 23.666666666666668
+ },
+ {
+ "experimentId": "claude-sonnet-4-6-opencode--baseline",
+ "caseId": "author-004-packaged-skill",
+ "status": "current",
+ "passedRuns": 2,
+ "validRuns": 3,
+ "meanDurationMs": 382777,
+ "measuredAt": "2026-08-17T22:03:22.192Z",
+ "meanEstimatedListCostUsd": 0.108879,
+ "meanTokenConsumption": 22653,
+ "meanToolInvocationCount": 16
+ },
+ {
+ "experimentId": "claude-sonnet-4-6-opencode--baseline",
+ "caseId": "author-005-conditional-approval",
+ "status": "current",
+ "passedRuns": 0,
+ "validRuns": 3,
+ "meanDurationMs": 225419,
+ "measuredAt": "2026-08-19T21:35:21.730Z",
+ "meanEstimatedListCostUsd": 0.10689799999999999,
+ "meanTokenConsumption": 22528.666666666668,
+ "meanToolInvocationCount": 24
+ },
+ {
+ "experimentId": "claude-sonnet-4-6-opencode--baseline",
+ "caseId": "author-006-custom-channel",
+ "status": "current",
+ "passedRuns": 1,
+ "validRuns": 3,
+ "meanDurationMs": 714111,
+ "measuredAt": "2026-08-17T22:19:44.996Z",
+ "meanEstimatedListCostUsd": 0.056847,
+ "meanTokenConsumption": 12625,
+ "meanToolInvocationCount": 10
+ },
+ {
+ "experimentId": "claude-sonnet-4-6-opencode--baseline",
+ "caseId": "author-007-digest-schedule",
+ "status": "current",
+ "passedRuns": 0,
+ "validRuns": 3,
+ "meanDurationMs": 232328,
+ "measuredAt": "2026-08-19T21:35:21.737Z",
+ "meanEstimatedListCostUsd": 0.09773,
+ "meanTokenConsumption": 16300.666666666666,
+ "meanToolInvocationCount": 33.333333333333336
+ },
+ {
+ "experimentId": "claude-sonnet-4-6-opencode--guided",
+ "caseId": "author-001-weather-tool",
+ "status": "current",
+ "passedRuns": 0,
+ "validRuns": 3,
+ "meanDurationMs": 186603,
+ "measuredAt": "2026-08-19T21:35:25.673Z",
+ "meanEstimatedListCostUsd": 0.07002399999999999,
+ "meanTokenConsumption": 13736,
+ "meanToolInvocationCount": 17.333333333333332
+ },
+ {
+ "experimentId": "claude-sonnet-4-6-opencode--guided",
+ "caseId": "author-002-new-project",
+ "status": "current",
+ "passedRuns": 2,
+ "validRuns": 3,
+ "meanDurationMs": 198555,
+ "measuredAt": "2026-08-19T21:35:25.695Z",
+ "meanEstimatedListCostUsd": 0.104669,
+ "meanTokenConsumption": 27117.666666666668,
+ "meanToolInvocationCount": 17.333333333333332
+ },
+ {
+ "experimentId": "claude-sonnet-4-6-opencode--guided",
+ "caseId": "author-003-openapi-connection",
+ "status": "current",
+ "passedRuns": 3,
+ "validRuns": 3,
+ "meanDurationMs": 132154,
+ "measuredAt": "2026-08-19T21:35:25.701Z",
+ "meanEstimatedListCostUsd": 0.045839,
+ "meanTokenConsumption": 11199.666666666666,
+ "meanToolInvocationCount": 10.333333333333334
+ },
+ {
+ "experimentId": "claude-sonnet-4-6-opencode--guided",
+ "caseId": "author-004-packaged-skill",
+ "status": "current",
+ "passedRuns": 1,
+ "validRuns": 3,
+ "meanDurationMs": 127395,
+ "measuredAt": "2026-08-19T21:04:16.127Z",
+ "meanEstimatedListCostUsd": 0.082587,
+ "meanTokenConsumption": 16813,
+ "meanToolInvocationCount": 13
+ },
+ {
+ "experimentId": "claude-sonnet-4-6-opencode--guided",
+ "caseId": "author-005-conditional-approval",
+ "status": "current",
+ "passedRuns": 0,
+ "validRuns": 3,
+ "meanDurationMs": 170150,
+ "measuredAt": "2026-08-19T21:35:25.711Z",
+ "meanEstimatedListCostUsd": 0.031358,
+ "meanTokenConsumption": 5668.666666666667,
+ "meanToolInvocationCount": 7.666666666666667
+ },
+ {
+ "experimentId": "claude-sonnet-4-6-opencode--guided",
+ "caseId": "author-006-custom-channel",
+ "status": "current",
+ "passedRuns": 2,
+ "validRuns": 3,
+ "meanDurationMs": 130916,
+ "measuredAt": "2026-08-19T21:04:16.135Z",
+ "meanEstimatedListCostUsd": 0.0346035,
+ "meanTokenConsumption": 6458.5,
+ "meanToolInvocationCount": 11
+ },
+ {
+ "experimentId": "claude-sonnet-4-6-opencode--guided",
+ "caseId": "author-007-digest-schedule",
+ "status": "current",
+ "passedRuns": 0,
+ "validRuns": 3,
+ "meanDurationMs": 260678,
+ "measuredAt": "2026-08-19T21:35:25.720Z",
+ "meanEstimatedListCostUsd": 0.09039199999999999,
+ "meanTokenConsumption": 13797.333333333334,
+ "meanToolInvocationCount": 29
+ }
+ ]
}
]
}
diff --git a/apps/docs/lib/evals/results.ts b/apps/docs/lib/evals/results.ts
index 3a9a483ac..0d06d7cf5 100644
--- a/apps/docs/lib/evals/results.ts
+++ b/apps/docs/lib/evals/results.ts
@@ -25,17 +25,26 @@ export interface BenchmarkResult {
measuredAt?: string;
}
+export interface BenchmarkSuite {
+ eveRevision: string | null;
+ caseFingerprint: string | null;
+ caseCount: number;
+ runsPerCell: number;
+}
+
+export interface PreviouslyMeasuredBenchmarkResults {
+ suite: BenchmarkSuite;
+ experiments: BenchmarkExperiment[];
+ results: BenchmarkResult[];
+}
+
export interface PublishedBenchmarkResults {
schemaVersion: 1;
generatedAt: string | null;
- suite: {
- eveRevision: string | null;
- caseFingerprint: string | null;
- caseCount: number;
- runsPerCell: number;
- };
+ suite: BenchmarkSuite;
experiments: BenchmarkExperiment[];
results: BenchmarkResult[];
+ previouslyMeasured?: PreviouslyMeasuredBenchmarkResults[];
}
export interface BenchmarkCaseRow {