Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

Benchmark Claude Fable 5.1 and GLM 5.3 (#2884)

Colton Padden577d78b5d902
diff --git a/apps/benchmarks/lib/benchmark-config.test.mjs b/apps/benchmarks/lib/benchmark-config.test.mjs
index 7d9ff1b0a..253dc44da 100644
--- a/apps/benchmarks/lib/benchmark-config.test.mjs
+++ b/apps/benchmarks/lib/benchmark-config.test.mjs
@@ -14,12 +14,12 @@ test("publishes only compatibility-validated models", () => {
publishedBenchmarkModels.map((benchmark) => benchmark.id),
[
"kimi-k3",
- "claude-fable-5",
+ "claude-fable-5-1",
"grok-4-6",
"gpt-5-6-sol",
"gpt-5-6-terra",
"claude-sonnet-5",
- "glm-5-2",
+ "glm-5-3",
"claude-opus-5",
"gemini-3-1-pro-preview",
],
@@ -35,8 +35,8 @@ test("publishes only compatibility-validated models", () => {
]);
assert.equal(findPublishedBenchmarkModel("grok-4-6").model, "xai/grok-4.6");
assert.equal(findPublishedBenchmarkModel("claude-sonnet-5").model, "anthropic/claude-sonnet-5");
- assert.equal(findPublishedBenchmarkModel("glm-5-2").model, "zai/glm-5.2");
- assert.equal(findPublishedBenchmarkModel("claude-fable-5").harness, "Claude Code");
+ assert.equal(findPublishedBenchmarkModel("glm-5-3").model, "zai/glm-5.3");
+ assert.equal(findPublishedBenchmarkModel("claude-fable-5-1").harness, "Claude Code");
assert.equal(findPublishedBenchmarkModel("claude-sonnet-5").harness, "Claude Code");
assert.equal(findPublishedBenchmarkModel("claude-opus-5").harness, "Claude Code");
assert.equal(findPublishedBenchmarkModel("kimi-k3").harness, "OpenCode");
@@ -50,6 +50,8 @@ test("allows candidate probes and rejects unknown models", () => {
assert.equal(findBenchmarkModel("gpt-5-6-terra").support, "supported");
assert.equal(findBenchmarkModel("grok-4-6").support, "supported");
assert.equal(findBenchmarkModel("claude-sonnet-4-6").support, "superseded");
+ assert.equal(findBenchmarkModel("claude-fable-5").support, "superseded");
+ assert.equal(findBenchmarkModel("glm-5-2").support, "superseded");
assert.equal(findBenchmarkModel("claude-opus-5").support, "supported");
assert.throws(() => findBenchmarkModel("unknown"), /Unknown model/u);
});
diff --git a/apps/benchmarks/lib/benchmark-config.ts b/apps/benchmarks/lib/benchmark-config.ts
index 54c083f06..c3ff526c3 100644
--- a/apps/benchmarks/lib/benchmark-config.ts
+++ b/apps/benchmarks/lib/benchmark-config.ts
@@ -32,6 +32,13 @@ export const benchmarkModels = [
model: "anthropic/claude-fable-5",
displayName: "Claude Fable 5",
harness: "Claude Code",
+ support: "superseded",
+ },
+ {
+ id: "claude-fable-5-1",
+ model: "anthropic/claude-fable-5.1",
+ displayName: "Claude Fable 5.1",
+ harness: "Claude Code",
support: "supported",
},
{
@@ -67,6 +74,13 @@ export const benchmarkModels = [
model: "zai/glm-5.2",
displayName: "GLM 5.2",
harness: "OpenCode",
+ support: "superseded",
+ },
+ {
+ id: "glm-5-3",
+ model: "zai/glm-5.3",
+ displayName: "GLM 5.3",
+ harness: "OpenCode",
support: "supported",
},
{
diff --git a/apps/benchmarks/scripts/cost.mjs b/apps/benchmarks/scripts/cost.mjs
index f00b00fde..f89ce73ce 100644
--- a/apps/benchmarks/scripts/cost.mjs
+++ b/apps/benchmarks/scripts/cost.mjs
@@ -25,6 +25,14 @@ export const modelPricing = {
cacheRead: 1,
cacheWrite: 12.5,
},
+ "anthropic/claude-fable-5.1": {
+ effectiveDate: "2026-09-02",
+ source: "https://ai-gateway.vercel.sh/v1/models",
+ input: 10,
+ output: 50,
+ cacheRead: 0.25,
+ cacheWrite: 12.5,
+ },
"xai/grok-4.6": {
effectiveDate: "2026-08-17",
source: "https://docs.x.ai/developers/models",
@@ -73,6 +81,14 @@ export const modelPricing = {
cacheRead: 0.26,
cacheWrite: 0,
},
+ "zai/glm-5.3": {
+ effectiveDate: "2026-09-02",
+ source: "https://ai-gateway.vercel.sh/v1/models",
+ input: 1.4,
+ output: 4.4,
+ cacheRead: 0.14,
+ cacheWrite: 0,
+ },
"google/gemini-3.1-pro-preview": {
effectiveDate: "2026-08-17",
source: "https://ai.google.dev/gemini-api/docs/pricing",
diff --git a/apps/benchmarks/scripts/export-results.test.mjs b/apps/benchmarks/scripts/export-results.test.mjs
index e96c56d98..30123d0fd 100644
--- a/apps/benchmarks/scripts/export-results.test.mjs
+++ b/apps/benchmarks/scripts/export-results.test.mjs
@@ -45,12 +45,12 @@ test("exports missing cells without publishing private artifacts", () => {
[...new Set(output.experiments.map((experiment) => experiment.modelDisplayName))],
[
"Kimi K3",
- "Claude Fable 5",
+ "Claude Fable 5.1",
"Grok 4.6",
"GPT-5.6 Sol",
"GPT-5.6 Terra",
"Claude Sonnet 5",
- "GLM 5.2",
+ "GLM 5.3",
"Claude Opus 5",
"Gemini 3.1 Pro Preview",
],
diff --git a/apps/docs/lib/evals/benchmark-results.json b/apps/docs/lib/evals/benchmark-results.json
index 70285c06a..f8c2c0854 100644
--- a/apps/docs/lib/evals/benchmark-results.json
+++ b/apps/docs/lib/evals/benchmark-results.json
@@ -1,9 +1,9 @@
{
"schemaVersion": 1,
- "generatedAt": "2026-08-31T20:02:25.881Z",
+ "generatedAt": "2026-09-02T14:38:34.637Z",
"suite": {
- "eveRevision": "450681ad42ce561ba53b687fd903681d7337359e",
- "caseFingerprint": "d30d6ad26abc32b54fde75c0e632d16186ed32b018e30c59baef624c0f370de9",
+ "eveRevision": "78fa9046b8ad377b7fdca2c6d18cd3c10afcfc77",
+ "caseFingerprint": "165cb912c2f1459eb03fafdf13813b6be4ac78eb35429ed0d9e21cfb6b9adb58",
"caseCount": 7,
"runsPerCell": 3
},
@@ -25,18 +25,18 @@
"treatment": "guided"
},
{
- "id": "claude-fable-5-claude-code--baseline",
- "groupId": "claude-fable-5-claude-code",
- "model": "anthropic/claude-fable-5",
- "modelDisplayName": "Claude Fable 5",
+ "id": "claude-fable-5-1-claude-code--baseline",
+ "groupId": "claude-fable-5-1-claude-code",
+ "model": "anthropic/claude-fable-5.1",
+ "modelDisplayName": "Claude Fable 5.1",
"harness": "Claude Code",
"treatment": "baseline"
},
{
- "id": "claude-fable-5-claude-code--guided",
- "groupId": "claude-fable-5-claude-code",
- "model": "anthropic/claude-fable-5",
- "modelDisplayName": "Claude Fable 5",
+ "id": "claude-fable-5-1-claude-code--guided",
+ "groupId": "claude-fable-5-1-claude-code",
+ "model": "anthropic/claude-fable-5.1",
+ "modelDisplayName": "Claude Fable 5.1",
"harness": "Claude Code",
"treatment": "guided"
},
@@ -105,18 +105,18 @@
"treatment": "guided"
},
{
- "id": "glm-5-2-opencode--baseline",
- "groupId": "glm-5-2-opencode",
- "model": "zai/glm-5.2",
- "modelDisplayName": "GLM 5.2",
+ "id": "glm-5-3-opencode--baseline",
+ "groupId": "glm-5-3-opencode",
+ "model": "zai/glm-5.3",
+ "modelDisplayName": "GLM 5.3",
"harness": "OpenCode",
"treatment": "baseline"
},
{
- "id": "glm-5-2-opencode--guided",
- "groupId": "glm-5-2-opencode",
- "model": "zai/glm-5.2",
- "modelDisplayName": "GLM 5.2",
+ "id": "glm-5-3-opencode--guided",
+ "groupId": "glm-5-3-opencode",
+ "model": "zai/glm-5.3",
+ "modelDisplayName": "GLM 5.3",
"harness": "OpenCode",
"treatment": "guided"
},
@@ -323,172 +323,172 @@
"meanToolInvocationCount": 12.333333333333334
},
{
- "experimentId": "claude-fable-5-claude-code--baseline",
+ "experimentId": "claude-fable-5-1-claude-code--baseline",
"caseId": "author-001-weather-tool",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 127045,
- "measuredAt": "2026-08-28T22:06:23.308Z",
- "meanEstimatedListCostUsd": 6.18823,
- "meanToolInvocationCount": 17.666666666666668,
- "meanTokenConsumption": 598539
+ "meanDurationMs": 186262,
+ "measuredAt": "2026-09-02T14:19:29.095Z",
+ "meanEstimatedListCostUsd": 1.291232,
+ "meanTokenConsumption": 8972,
+ "meanToolInvocationCount": 11.666666666666666
},
{
- "experimentId": "claude-fable-5-claude-code--baseline",
+ "experimentId": "claude-fable-5-1-claude-code--baseline",
"caseId": "author-002-new-project",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 169780,
- "measuredAt": "2026-08-28T22:06:23.319Z",
- "meanEstimatedListCostUsd": 3.83675,
- "meanToolInvocationCount": 8.666666666666666,
- "meanTokenConsumption": 375135
+ "meanDurationMs": 148272,
+ "measuredAt": "2026-09-02T14:19:29.109Z",
+ "meanEstimatedListCostUsd": 0.7348391666666666,
+ "meanTokenConsumption": 3895.3333333333335,
+ "meanToolInvocationCount": 9
},
{
- "experimentId": "claude-fable-5-claude-code--baseline",
+ "experimentId": "claude-fable-5-1-claude-code--baseline",
"caseId": "author-003-openapi-connection",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 258551,
- "measuredAt": "2026-08-28T22:06:23.327Z",
- "meanEstimatedListCostUsd": 5.43717,
- "meanToolInvocationCount": 12.666666666666666,
- "meanTokenConsumption": 528873
+ "meanDurationMs": 183777,
+ "measuredAt": "2026-09-02T14:19:29.120Z",
+ "meanEstimatedListCostUsd": 1.31527725,
+ "meanTokenConsumption": 8248,
+ "meanToolInvocationCount": 16.666666666666668
},
{
- "experimentId": "claude-fable-5-claude-code--baseline",
+ "experimentId": "claude-fable-5-1-claude-code--baseline",
"caseId": "author-004-packaged-skill",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 89365,
- "measuredAt": "2026-08-28T22:06:23.334Z",
- "meanEstimatedListCostUsd": 3.97663,
- "meanToolInvocationCount": 11.666666666666666,
- "meanTokenConsumption": 383479
+ "meanDurationMs": 180513,
+ "measuredAt": "2026-09-02T14:19:29.129Z",
+ "meanEstimatedListCostUsd": 1.0393497499999997,
+ "meanTokenConsumption": 9160.333333333334,
+ "meanToolInvocationCount": 9.666666666666666
},
{
- "experimentId": "claude-fable-5-claude-code--baseline",
+ "experimentId": "claude-fable-5-1-claude-code--baseline",
"caseId": "author-005-conditional-approval",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 123612,
- "measuredAt": "2026-08-28T22:06:23.339Z",
- "meanEstimatedListCostUsd": 5.19217,
- "meanToolInvocationCount": 17.666666666666668,
- "meanTokenConsumption": 507153
+ "meanDurationMs": 181059,
+ "measuredAt": "2026-09-02T14:19:29.137Z",
+ "meanEstimatedListCostUsd": 1.3291795000000002,
+ "meanTokenConsumption": 7962.333333333333,
+ "meanToolInvocationCount": 16.666666666666668
},
{
- "experimentId": "claude-fable-5-claude-code--baseline",
+ "experimentId": "claude-fable-5-1-claude-code--baseline",
"caseId": "author-006-custom-channel",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 104483,
- "measuredAt": "2026-08-28T22:06:23.346Z",
- "meanEstimatedListCostUsd": 7.76357,
- "meanToolInvocationCount": 15.666666666666666,
- "meanTokenConsumption": 761257
+ "meanDurationMs": 149380,
+ "measuredAt": "2026-09-02T14:19:29.147Z",
+ "meanEstimatedListCostUsd": 1.0626429999999998,
+ "meanTokenConsumption": 4479.333333333333,
+ "meanToolInvocationCount": 10.666666666666666
},
{
- "experimentId": "claude-fable-5-claude-code--baseline",
+ "experimentId": "claude-fable-5-1-claude-code--baseline",
"caseId": "author-007-digest-schedule",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 156463,
- "measuredAt": "2026-08-28T22:06:23.352Z",
- "meanEstimatedListCostUsd": 3.47642,
- "meanToolInvocationCount": 23.666666666666668,
- "meanTokenConsumption": 339290
+ "meanDurationMs": 151473,
+ "measuredAt": "2026-09-02T14:19:29.154Z",
+ "meanEstimatedListCostUsd": 0.9185504999999999,
+ "meanTokenConsumption": 4827.666666666667,
+ "meanToolInvocationCount": 11.666666666666666
},
{
- "experimentId": "claude-fable-5-claude-code--guided",
+ "experimentId": "claude-fable-5-1-claude-code--guided",
"caseId": "author-001-weather-tool",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 80508,
- "measuredAt": "2026-08-28T22:36:19.362Z",
- "meanEstimatedListCostUsd": 3.87527,
- "meanToolInvocationCount": 7.666666666666667,
- "meanTokenConsumption": 375115
+ "meanDurationMs": 142627,
+ "measuredAt": "2026-09-02T14:18:53.362Z",
+ "meanEstimatedListCostUsd": 0.9972783333333334,
+ "meanTokenConsumption": 6633,
+ "meanToolInvocationCount": 6
},
{
- "experimentId": "claude-fable-5-claude-code--guided",
+ "experimentId": "claude-fable-5-1-claude-code--guided",
"caseId": "author-002-new-project",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 80206,
- "measuredAt": "2026-08-28T22:36:19.376Z",
- "meanEstimatedListCostUsd": 3.40311,
- "meanToolInvocationCount": 9,
- "meanTokenConsumption": 332259
+ "meanDurationMs": 155541,
+ "measuredAt": "2026-09-02T14:18:53.381Z",
+ "meanEstimatedListCostUsd": 1.6608135,
+ "meanTokenConsumption": 4615,
+ "meanToolInvocationCount": 9.666666666666666
},
{
- "experimentId": "claude-fable-5-claude-code--guided",
+ "experimentId": "claude-fable-5-1-claude-code--guided",
"caseId": "author-003-openapi-connection",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 71462,
- "measuredAt": "2026-08-28T22:36:19.386Z",
- "meanEstimatedListCostUsd": 3.58212,
- "meanToolInvocationCount": 8.666666666666666,
- "meanTokenConsumption": 350688
+ "meanDurationMs": 132220,
+ "measuredAt": "2026-09-02T14:18:53.396Z",
+ "meanEstimatedListCostUsd": 0.8656001666666667,
+ "meanTokenConsumption": 3575,
+ "meanToolInvocationCount": 7.333333333333333
},
{
- "experimentId": "claude-fable-5-claude-code--guided",
+ "experimentId": "claude-fable-5-1-claude-code--guided",
"caseId": "author-004-packaged-skill",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 87034,
- "measuredAt": "2026-08-28T22:36:19.391Z",
- "meanEstimatedListCostUsd": 4.10959,
- "meanToolInvocationCount": 7.666666666666667,
- "meanTokenConsumption": 397447
+ "meanDurationMs": 161398,
+ "measuredAt": "2026-09-02T14:18:53.406Z",
+ "meanEstimatedListCostUsd": 0.7871509166666666,
+ "meanTokenConsumption": 5333.333333333333,
+ "meanToolInvocationCount": 8
},
{
- "experimentId": "claude-fable-5-claude-code--guided",
+ "experimentId": "claude-fable-5-1-claude-code--guided",
"caseId": "author-005-conditional-approval",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 59604,
- "measuredAt": "2026-08-28T22:36:19.396Z",
- "meanEstimatedListCostUsd": 3.38543,
- "meanToolInvocationCount": 7,
- "meanTokenConsumption": 331919
+ "meanDurationMs": 143233,
+ "measuredAt": "2026-09-02T14:18:53.417Z",
+ "meanEstimatedListCostUsd": 0.8737515833333332,
+ "meanTokenConsumption": 3909.3333333333335,
+ "meanToolInvocationCount": 7
},
{
- "experimentId": "claude-fable-5-claude-code--guided",
+ "experimentId": "claude-fable-5-1-claude-code--guided",
"caseId": "author-006-custom-channel",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 64829,
- "measuredAt": "2026-08-28T22:36:19.402Z",
- "meanEstimatedListCostUsd": 3.81826,
- "meanToolInvocationCount": 7,
- "meanTokenConsumption": 374610
+ "meanDurationMs": 115430,
+ "measuredAt": "2026-09-02T14:18:53.429Z",
+ "meanEstimatedListCostUsd": 0.63728775,
+ "meanTokenConsumption": 1828.6666666666667,
+ "meanToolInvocationCount": 5.666666666666667
},
{
- "experimentId": "claude-fable-5-claude-code--guided",
+ "experimentId": "claude-fable-5-1-claude-code--guided",
"caseId": "author-007-digest-schedule",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 71055,
- "measuredAt": "2026-08-28T22:36:19.407Z",
- "meanEstimatedListCostUsd": 3.07521,
- "meanToolInvocationCount": 7,
- "meanTokenConsumption": 299897
+ "meanDurationMs": 130462,
+ "measuredAt": "2026-09-02T14:18:53.440Z",
+ "meanEstimatedListCostUsd": 0.6112984166666667,
+ "meanTokenConsumption": 2535.3333333333335,
+ "meanToolInvocationCount": 7.333333333333333
},
{
"experimentId": "grok-4-6-opencode--baseline",
@@ -1000,11 +1000,11 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 105859,
- "measuredAt": "2026-08-28T22:44:03.383Z",
- "meanEstimatedListCostUsd": 1.735734,
- "meanToolInvocationCount": 18,
- "meanTokenConsumption": 851675
+ "meanDurationMs": 119150,
+ "measuredAt": "2026-09-02T14:38:33.830Z",
+ "meanEstimatedListCostUsd": 0.4004288333333334,
+ "meanTokenConsumption": 4827,
+ "meanToolInvocationCount": 11.333333333333334
},
{
"experimentId": "claude-sonnet-5-claude-code--baseline",
@@ -1012,11 +1012,11 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 73537,
- "measuredAt": "2026-08-28T22:44:03.395Z",
- "meanEstimatedListCostUsd": 1.07418,
- "meanToolInvocationCount": 11,
- "meanTokenConsumption": 530522
+ "meanDurationMs": 117102,
+ "measuredAt": "2026-09-02T14:38:33.846Z",
+ "meanEstimatedListCostUsd": 0.38767966666666664,
+ "meanTokenConsumption": 3335.6666666666665,
+ "meanToolInvocationCount": 8.333333333333334
},
{
"experimentId": "claude-sonnet-5-claude-code--baseline",
@@ -1024,11 +1024,11 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 78931,
- "measuredAt": "2026-08-28T22:44:03.405Z",
- "meanEstimatedListCostUsd": 1.59483,
- "meanToolInvocationCount": 18,
- "meanTokenConsumption": 789511
+ "meanDurationMs": 146914,
+ "measuredAt": "2026-09-02T14:38:33.859Z",
+ "meanEstimatedListCostUsd": 0.6134467,
+ "meanTokenConsumption": 7727,
+ "meanToolInvocationCount": 20.333333333333332
},
{
"experimentId": "claude-sonnet-5-claude-code--baseline",
@@ -1036,11 +1036,11 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 71745,
- "measuredAt": "2026-08-28T22:44:03.411Z",
- "meanEstimatedListCostUsd": 1.021194,
- "meanToolInvocationCount": 11,
- "meanTokenConsumption": 499669
+ "meanDurationMs": 128723,
+ "measuredAt": "2026-09-02T14:38:33.872Z",
+ "meanEstimatedListCostUsd": 0.2731853333333334,
+ "meanTokenConsumption": 4274,
+ "meanToolInvocationCount": 10.333333333333334
},
{
"experimentId": "claude-sonnet-5-claude-code--baseline",
@@ -1048,11 +1048,11 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 93642,
- "measuredAt": "2026-08-28T22:44:03.416Z",
- "meanEstimatedListCostUsd": 1.82048,
- "meanToolInvocationCount": 17.666666666666668,
- "meanTokenConsumption": 901060
+ "meanDurationMs": 112957,
+ "measuredAt": "2026-09-02T14:38:33.882Z",
+ "meanEstimatedListCostUsd": 0.3310536,
+ "meanTokenConsumption": 3567.3333333333335,
+ "meanToolInvocationCount": 12.666666666666666
},
{
"experimentId": "claude-sonnet-5-claude-code--baseline",
@@ -1060,11 +1060,11 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 91912,
- "measuredAt": "2026-08-28T22:44:03.422Z",
- "meanEstimatedListCostUsd": 1.172788,
- "meanToolInvocationCount": 17.333333333333332,
- "meanTokenConsumption": 578826
+ "meanDurationMs": 206252,
+ "measuredAt": "2026-09-02T14:38:33.890Z",
+ "meanEstimatedListCostUsd": 0.3613244666666667,
+ "meanTokenConsumption": 5164.666666666667,
+ "meanToolInvocationCount": 9.666666666666666
},
{
"experimentId": "claude-sonnet-5-claude-code--baseline",
@@ -1072,11 +1072,11 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 84747,
- "measuredAt": "2026-08-28T22:44:03.428Z",
- "meanEstimatedListCostUsd": 1.011918,
- "meanToolInvocationCount": 13.333333333333334,
- "meanTokenConsumption": 498895
+ "meanDurationMs": 104010,
+ "measuredAt": "2026-09-02T14:38:33.898Z",
+ "meanEstimatedListCostUsd": 0.22264466666666669,
+ "meanTokenConsumption": 1994.3333333333333,
+ "meanToolInvocationCount": 7.333333333333333
},
{
"experimentId": "claude-sonnet-5-claude-code--guided",
@@ -1084,11 +1084,11 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 69779,
- "measuredAt": "2026-08-28T23:16:01.692Z",
- "meanEstimatedListCostUsd": 1.629858,
- "meanToolInvocationCount": 12,
- "meanTokenConsumption": 801201
+ "meanDurationMs": 130089,
+ "measuredAt": "2026-09-02T14:34:31.320Z",
+ "meanEstimatedListCostUsd": 0.40971483333333336,
+ "meanTokenConsumption": 4909,
+ "meanToolInvocationCount": 10.666666666666666
},
{
"experimentId": "claude-sonnet-5-claude-code--guided",
@@ -1096,11 +1096,11 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 56970,
- "measuredAt": "2026-08-28T23:16:01.703Z",
- "meanEstimatedListCostUsd": 0.965568,
- "meanToolInvocationCount": 7.666666666666667,
- "meanTokenConsumption": 476704
+ "meanDurationMs": 119473,
+ "measuredAt": "2026-09-02T14:34:31.340Z",
+ "meanEstimatedListCostUsd": 0.2871293,
+ "meanTokenConsumption": 3610.6666666666665,
+ "meanToolInvocationCount": 10.333333333333334
},
{
"experimentId": "claude-sonnet-5-claude-code--guided",
@@ -1108,11 +1108,11 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 42652,
- "measuredAt": "2026-08-28T23:16:01.711Z",
- "meanEstimatedListCostUsd": 0.82867,
- "meanToolInvocationCount": 8.333333333333334,
- "meanTokenConsumption": 410267
+ "meanDurationMs": 112748,
+ "measuredAt": "2026-09-02T14:34:31.354Z",
+ "meanEstimatedListCostUsd": 0.3381717,
+ "meanTokenConsumption": 2513,
+ "meanToolInvocationCount": 8.333333333333334
},
{
"experimentId": "claude-sonnet-5-claude-code--guided",
@@ -1120,11 +1120,11 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 45265,
- "measuredAt": "2026-08-28T23:16:01.717Z",
- "meanEstimatedListCostUsd": 0.67903,
- "meanToolInvocationCount": 5,
- "meanTokenConsumption": 332683
+ "meanDurationMs": 115311,
+ "measuredAt": "2026-09-02T14:34:31.365Z",
+ "meanEstimatedListCostUsd": 0.26473703333333337,
+ "meanTokenConsumption": 7318.666666666667,
+ "meanToolInvocationCount": 5
},
{
"experimentId": "claude-sonnet-5-claude-code--guided",
@@ -1132,11 +1132,11 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 39690,
- "measuredAt": "2026-08-28T23:16:01.722Z",
- "meanEstimatedListCostUsd": 0.972688,
- "meanToolInvocationCount": 7,
- "meanTokenConsumption": 480972
+ "meanDurationMs": 117023,
+ "measuredAt": "2026-09-02T14:34:31.375Z",
+ "meanEstimatedListCostUsd": 0.34835236666666664,
+ "meanTokenConsumption": 2814,
+ "meanToolInvocationCount": 9
},
{
"experimentId": "claude-sonnet-5-claude-code--guided",
@@ -1144,11 +1144,11 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 38868,
- "measuredAt": "2026-08-28T23:16:01.727Z",
- "meanEstimatedListCostUsd": 1.219158,
- "meanToolInvocationCount": 7.333333333333333,
- "meanTokenConsumption": 604483
+ "meanDurationMs": 110065,
+ "measuredAt": "2026-09-02T14:34:31.383Z",
+ "meanEstimatedListCostUsd": 0.2831804,
+ "meanTokenConsumption": 1897.6666666666667,
+ "meanToolInvocationCount": 6
},
{
"experimentId": "claude-sonnet-5-claude-code--guided",
@@ -1156,179 +1156,179 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 68924,
- "measuredAt": "2026-08-28T23:16:01.731Z",
- "meanEstimatedListCostUsd": 1.053378,
- "meanToolInvocationCount": 15,
- "meanTokenConsumption": 521965
+ "meanDurationMs": 113169,
+ "measuredAt": "2026-09-02T14:34:31.393Z",
+ "meanEstimatedListCostUsd": 0.31014963333333334,
+ "meanTokenConsumption": 2200,
+ "meanToolInvocationCount": 9
},
{
- "experimentId": "glm-5-2-opencode--baseline",
+ "experimentId": "glm-5-3-opencode--baseline",
"caseId": "author-001-weather-tool",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 103530,
- "measuredAt": "2026-08-28T19:22:09.000Z",
- "meanEstimatedListCostUsd": 0.07017806666666666,
- "meanTokenConsumption": 37804.333333333336,
- "meanToolInvocationCount": 25
+ "meanDurationMs": 229776,
+ "measuredAt": "2026-09-02T14:24:14.610Z",
+ "meanEstimatedListCostUsd": 0.13380119999999998,
+ "meanTokenConsumption": 39736,
+ "meanToolInvocationCount": 27.666666666666668
},
{
- "experimentId": "glm-5-2-opencode--baseline",
+ "experimentId": "glm-5-3-opencode--baseline",
"caseId": "author-002-new-project",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 100847,
- "measuredAt": "2026-08-28T19:26:40.000Z",
- "meanEstimatedListCostUsd": 0.03248553333333333,
- "meanTokenConsumption": 19034.666666666668,
- "meanToolInvocationCount": 18.333333333333332
+ "meanDurationMs": 253800,
+ "measuredAt": "2026-09-02T14:24:14.640Z",
+ "meanEstimatedListCostUsd": 0.04476281333333334,
+ "meanTokenConsumption": 19797.666666666668,
+ "meanToolInvocationCount": 13.333333333333334
},
{
- "experimentId": "glm-5-2-opencode--baseline",
+ "experimentId": "glm-5-3-opencode--baseline",
"caseId": "author-003-openapi-connection",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 134009,
- "measuredAt": "2026-08-28T19:26:40.000Z",
- "meanEstimatedListCostUsd": 0.09391753333333334,
- "meanTokenConsumption": 49094.666666666664,
- "meanToolInvocationCount": 40.333333333333336
+ "meanDurationMs": 275839,
+ "measuredAt": "2026-09-02T14:24:14.653Z",
+ "meanEstimatedListCostUsd": 0.21305283333333336,
+ "meanTokenConsumption": 42271,
+ "meanToolInvocationCount": 43.666666666666664
},
{
- "experimentId": "glm-5-2-opencode--baseline",
+ "experimentId": "glm-5-3-opencode--baseline",
"caseId": "author-004-packaged-skill",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 214119,
- "measuredAt": "2026-08-28T19:22:09.000Z",
- "meanEstimatedListCostUsd": 0.07759993333333333,
- "meanTokenConsumption": 41325.666666666664,
- "meanToolInvocationCount": 36.333333333333336
+ "meanDurationMs": 237736,
+ "measuredAt": "2026-09-02T14:24:14.669Z",
+ "meanEstimatedListCostUsd": 0.06231241333333334,
+ "meanTokenConsumption": 18199.666666666668,
+ "meanToolInvocationCount": 18.333333333333332
},
{
- "experimentId": "glm-5-2-opencode--baseline",
+ "experimentId": "glm-5-3-opencode--baseline",
"caseId": "author-005-conditional-approval",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 58527,
- "measuredAt": "2026-08-28T19:22:09.000Z",
- "meanEstimatedListCostUsd": 0.07383253333333333,
- "meanTokenConsumption": 48329.666666666664,
- "meanToolInvocationCount": 25.333333333333332
+ "meanDurationMs": 267634,
+ "measuredAt": "2026-09-02T14:24:14.679Z",
+ "meanEstimatedListCostUsd": 0.15534122666666667,
+ "meanTokenConsumption": 43595,
+ "meanToolInvocationCount": 35.333333333333336
},
{
- "experimentId": "glm-5-2-opencode--baseline",
+ "experimentId": "glm-5-3-opencode--baseline",
"caseId": "author-006-custom-channel",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 75109,
- "measuredAt": "2026-08-28T19:22:09.000Z",
- "meanEstimatedListCostUsd": 0.07836139999999998,
- "meanTokenConsumption": 48506,
- "meanToolInvocationCount": 27.666666666666668
+ "meanDurationMs": 173067,
+ "measuredAt": "2026-09-02T14:24:14.691Z",
+ "meanEstimatedListCostUsd": 0.15079565333333333,
+ "meanTokenConsumption": 55561.666666666664,
+ "meanToolInvocationCount": 29
},
{
- "experimentId": "glm-5-2-opencode--baseline",
+ "experimentId": "glm-5-3-opencode--baseline",
"caseId": "author-007-digest-schedule",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 93384,
- "measuredAt": "2026-08-28T19:22:09.000Z",
- "meanEstimatedListCostUsd": 0.04282053333333333,
- "meanTokenConsumption": 24309.666666666668,
- "meanToolInvocationCount": 19
+ "meanDurationMs": 150379,
+ "measuredAt": "2026-09-02T14:24:14.703Z",
+ "meanEstimatedListCostUsd": 0.061922139999999994,
+ "meanTokenConsumption": 17250,
+ "meanToolInvocationCount": 23.333333333333332
},
{
- "experimentId": "glm-5-2-opencode--guided",
+ "experimentId": "glm-5-3-opencode--guided",
"caseId": "author-001-weather-tool",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 68607,
- "measuredAt": "2026-08-28T19:22:01.000Z",
- "meanEstimatedListCostUsd": 0.04188693333333334,
- "meanTokenConsumption": 22360.666666666668,
- "meanToolInvocationCount": 16.333333333333332
+ "meanDurationMs": 192393,
+ "measuredAt": "2026-09-02T14:24:09.125Z",
+ "meanEstimatedListCostUsd": 0.05878674,
+ "meanTokenConsumption": 23335,
+ "meanToolInvocationCount": 10
},
{
- "experimentId": "glm-5-2-opencode--guided",
+ "experimentId": "glm-5-3-opencode--guided",
"caseId": "author-002-new-project",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 90517,
- "measuredAt": "2026-08-28T19:25:29.000Z",
- "meanEstimatedListCostUsd": 0.02843253333333333,
- "meanTokenConsumption": 16824.666666666668,
- "meanToolInvocationCount": 14.333333333333334
+ "meanDurationMs": 314441,
+ "measuredAt": "2026-09-02T14:24:09.147Z",
+ "meanEstimatedListCostUsd": 0.04504718666666666,
+ "meanTokenConsumption": 18359.333333333332,
+ "meanToolInvocationCount": 14
},
{
- "experimentId": "glm-5-2-opencode--guided",
+ "experimentId": "glm-5-3-opencode--guided",
"caseId": "author-003-openapi-connection",
"status": "current",
- "passedRuns": 0,
+ "passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 524221,
- "measuredAt": "2026-08-28T19:22:01.000Z",
- "meanEstimatedListCostUsd": 0.1800297333333333,
- "meanTokenConsumption": 78267.66666666667,
- "meanToolInvocationCount": 24.666666666666668
+ "meanDurationMs": 141942,
+ "measuredAt": "2026-09-02T14:24:09.165Z",
+ "meanEstimatedListCostUsd": 0.062038473333333344,
+ "meanTokenConsumption": 22144.666666666668,
+ "meanToolInvocationCount": 15
},
{
- "experimentId": "glm-5-2-opencode--guided",
+ "experimentId": "glm-5-3-opencode--guided",
"caseId": "author-004-packaged-skill",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 68821,
- "measuredAt": "2026-08-28T19:22:01.000Z",
- "meanEstimatedListCostUsd": 0.031183066666666665,
- "meanTokenConsumption": 16504.333333333332,
- "meanToolInvocationCount": 15
+ "meanDurationMs": 140163,
+ "measuredAt": "2026-09-02T14:24:09.179Z",
+ "meanEstimatedListCostUsd": 0.04043716,
+ "meanTokenConsumption": 14312.666666666666,
+ "meanToolInvocationCount": 10.666666666666666
},
{
- "experimentId": "glm-5-2-opencode--guided",
+ "experimentId": "glm-5-3-opencode--guided",
"caseId": "author-005-conditional-approval",
"status": "current",
- "passedRuns": 2,
+ "passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 80061,
- "measuredAt": "2026-08-28T19:22:01.000Z",
- "meanEstimatedListCostUsd": 0.040431333333333326,
- "meanTokenConsumption": 26306.666666666668,
- "meanToolInvocationCount": 14.333333333333334
+ "meanDurationMs": 129477,
+ "measuredAt": "2026-09-02T14:24:09.190Z",
+ "meanEstimatedListCostUsd": 0.04672530666666667,
+ "meanTokenConsumption": 21094.666666666668,
+ "meanToolInvocationCount": 8.666666666666666
},
{
- "experimentId": "glm-5-2-opencode--guided",
+ "experimentId": "glm-5-3-opencode--guided",
"caseId": "author-006-custom-channel",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 56079,
- "measuredAt": "2026-08-28T19:22:01.000Z",
- "meanEstimatedListCostUsd": 0.03310953333333334,
- "meanTokenConsumption": 19814.666666666668,
- "meanToolInvocationCount": 11
+ "meanDurationMs": 117801,
+ "measuredAt": "2026-09-02T14:24:09.200Z",
+ "meanEstimatedListCostUsd": 0.036290826666666665,
+ "meanTokenConsumption": 13584.333333333334,
+ "meanToolInvocationCount": 10.666666666666666
},
{
- "experimentId": "glm-5-2-opencode--guided",
+ "experimentId": "glm-5-3-opencode--guided",
"caseId": "author-007-digest-schedule",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 74552,
- "measuredAt": "2026-08-28T19:22:01.000Z",
- "meanEstimatedListCostUsd": 0.029523933333333335,
- "meanTokenConsumption": 16295.666666666666,
- "meanToolInvocationCount": 12.333333333333334
+ "meanDurationMs": 117068,
+ "measuredAt": "2026-09-02T14:24:09.209Z",
+ "meanEstimatedListCostUsd": 0.024231453333333333,
+ "meanTokenConsumption": 7749.333333333333,
+ "meanToolInvocationCount": 8.333333333333334
},
{
"experimentId": "claude-opus-5-claude-code--baseline",
@@ -1336,11 +1336,11 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 135534,
- "measuredAt": "2026-08-29T00:05:25.109Z",
- "meanEstimatedListCostUsd": 4.933595,
- "meanToolInvocationCount": 22.666666666666668,
- "meanTokenConsumption": 947443
+ "meanDurationMs": 155202,
+ "measuredAt": "2026-09-02T14:35:36.609Z",
+ "meanEstimatedListCostUsd": 1.1599598333333334,
+ "meanTokenConsumption": 10714.666666666666,
+ "meanToolInvocationCount": 17.333333333333332
},
{
"experimentId": "claude-opus-5-claude-code--baseline",
@@ -1348,11 +1348,11 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 92582,
- "measuredAt": "2026-08-29T00:05:25.121Z",
- "meanEstimatedListCostUsd": 2.50535,
- "meanToolInvocationCount": 13.333333333333334,
- "meanTokenConsumption": 488330
+ "meanDurationMs": 124976,
+ "measuredAt": "2026-09-02T14:35:36.628Z",
+ "meanEstimatedListCostUsd": 0.8593296666666667,
+ "meanTokenConsumption": 3845.3333333333335,
+ "meanToolInvocationCount": 10.333333333333334
},
{
"experimentId": "claude-opus-5-claude-code--baseline",
@@ -1360,11 +1360,11 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 191566,
- "measuredAt": "2026-08-29T00:05:25.129Z",
- "meanEstimatedListCostUsd": 14.008915,
- "meanToolInvocationCount": 36.333333333333336,
- "meanTokenConsumption": 2751467
+ "meanDurationMs": 160123,
+ "measuredAt": "2026-09-02T14:35:36.643Z",
+ "meanEstimatedListCostUsd": 1.3671014166666666,
+ "meanTokenConsumption": 7793.333333333333,
+ "meanToolInvocationCount": 19
},
{
"experimentId": "claude-opus-5-claude-code--baseline",
@@ -1372,11 +1372,11 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 125522,
- "measuredAt": "2026-08-29T00:05:25.138Z",
- "meanEstimatedListCostUsd": 3.74517,
- "meanToolInvocationCount": 16,
- "meanTokenConsumption": 722970
+ "meanDurationMs": 181217,
+ "measuredAt": "2026-09-02T14:35:36.656Z",
+ "meanEstimatedListCostUsd": 1.1116483333333333,
+ "meanTokenConsumption": 16371.666666666666,
+ "meanToolInvocationCount": 12.666666666666666
},
{
"experimentId": "claude-opus-5-claude-code--baseline",
@@ -1384,11 +1384,11 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 135687,
- "measuredAt": "2026-08-29T00:05:25.146Z",
- "meanEstimatedListCostUsd": 4.87658,
- "meanToolInvocationCount": 26.666666666666668,
- "meanTokenConsumption": 952132
+ "meanDurationMs": 153333,
+ "measuredAt": "2026-09-02T14:35:36.667Z",
+ "meanEstimatedListCostUsd": 1.03157675,
+ "meanTokenConsumption": 6217.333333333333,
+ "meanToolInvocationCount": 17.333333333333332
},
{
"experimentId": "claude-opus-5-claude-code--baseline",
@@ -1396,11 +1396,11 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 202636,
- "measuredAt": "2026-08-29T00:05:25.152Z",
- "meanToolInvocationCount": 26.333333333333332,
- "meanTokenConsumption": 1822465,
- "meanEstimatedListCostUsd": 9.304045
+ "meanDurationMs": 136155,
+ "measuredAt": "2026-09-02T14:35:36.676Z",
+ "meanEstimatedListCostUsd": 0.9316169166666667,
+ "meanTokenConsumption": 4836.666666666667,
+ "meanToolInvocationCount": 16.666666666666668
},
{
"experimentId": "claude-opus-5-claude-code--baseline",
@@ -1408,11 +1408,11 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 163050,
- "measuredAt": "2026-08-29T00:05:25.160Z",
- "meanEstimatedListCostUsd": 5.362835,
- "meanToolInvocationCount": 26,
- "meanTokenConsumption": 1044543
+ "meanDurationMs": 120633,
+ "measuredAt": "2026-09-02T14:35:36.685Z",
+ "meanEstimatedListCostUsd": 0.5067333333333334,
+ "meanTokenConsumption": 3232.6666666666665,
+ "meanToolInvocationCount": 9.333333333333334
},
{
"experimentId": "claude-opus-5-claude-code--guided",
@@ -1420,11 +1420,11 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 106811,
- "measuredAt": "2026-08-29T00:13:01.331Z",
- "meanEstimatedListCostUsd": 11.438505,
- "meanToolInvocationCount": 14.666666666666666,
- "meanTokenConsumption": 2234333
+ "meanDurationMs": 147978,
+ "measuredAt": "2026-09-02T14:34:53.900Z",
+ "meanEstimatedListCostUsd": 1.0020195,
+ "meanTokenConsumption": 11004.333333333334,
+ "meanToolInvocationCount": 11
},
{
"experimentId": "claude-opus-5-claude-code--guided",
@@ -1432,11 +1432,11 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 91787,
- "measuredAt": "2026-08-29T00:13:01.344Z",
- "meanEstimatedListCostUsd": 2.4951,
- "meanToolInvocationCount": 16,
- "meanTokenConsumption": 486252
+ "meanDurationMs": 138714,
+ "measuredAt": "2026-09-02T14:34:53.929Z",
+ "meanEstimatedListCostUsd": 1.0472271666666666,
+ "meanTokenConsumption": 8633.666666666666,
+ "meanToolInvocationCount": 14.333333333333334
},
{
"experimentId": "claude-opus-5-claude-code--guided",
@@ -1444,11 +1444,11 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 114352,
- "measuredAt": "2026-08-29T00:13:01.356Z",
- "meanEstimatedListCostUsd": 2.90723,
- "meanToolInvocationCount": 20.666666666666668,
- "meanTokenConsumption": 568682
+ "meanDurationMs": 137520,
+ "measuredAt": "2026-09-02T14:34:53.944Z",
+ "meanEstimatedListCostUsd": 0.7869943333333334,
+ "meanTokenConsumption": 5193.666666666667,
+ "meanToolInvocationCount": 14
},
{
"experimentId": "claude-opus-5-claude-code--guided",
@@ -1456,11 +1456,11 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 77606,
- "measuredAt": "2026-08-29T00:13:01.363Z",
- "meanEstimatedListCostUsd": 2.328845,
- "meanToolInvocationCount": 10,
- "meanTokenConsumption": 446685
+ "meanDurationMs": 153316,
+ "measuredAt": "2026-09-02T14:34:53.956Z",
+ "meanEstimatedListCostUsd": 0.6677339166666667,
+ "meanTokenConsumption": 10291.666666666666,
+ "meanToolInvocationCount": 7.666666666666667
},
{
"experimentId": "claude-opus-5-claude-code--guided",
@@ -1468,11 +1468,11 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 104699,
- "measuredAt": "2026-08-29T00:13:01.370Z",
- "meanEstimatedListCostUsd": 4.55151,
- "meanToolInvocationCount": 19,
- "meanTokenConsumption": 888618
+ "meanDurationMs": 116393,
+ "measuredAt": "2026-09-02T14:34:53.964Z",
+ "meanEstimatedListCostUsd": 0.5910709166666667,
+ "meanTokenConsumption": 2916.6666666666665,
+ "meanToolInvocationCount": 8.333333333333334
},
{
"experimentId": "claude-opus-5-claude-code--guided",
@@ -1480,11 +1480,11 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 162364,
- "measuredAt": "2026-08-29T00:13:01.377Z",
- "meanEstimatedListCostUsd": 6.511585,
- "meanToolInvocationCount": 16.333333333333332,
- "meanTokenConsumption": 1266737
+ "meanDurationMs": 107570,
+ "measuredAt": "2026-09-02T14:34:53.973Z",
+ "meanEstimatedListCostUsd": 0.4942074166666666,
+ "meanTokenConsumption": 2636,
+ "meanToolInvocationCount": 6.666666666666667
},
{
"experimentId": "claude-opus-5-claude-code--guided",
@@ -1492,11 +1492,11 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 89702,
- "measuredAt": "2026-08-29T00:13:01.384Z",
- "meanEstimatedListCostUsd": 3.44083,
- "meanToolInvocationCount": 16.333333333333332,
- "meanTokenConsumption": 674858
+ "meanDurationMs": 114311,
+ "measuredAt": "2026-09-02T14:34:53.981Z",
+ "meanEstimatedListCostUsd": 0.46051358333333337,
+ "meanTokenConsumption": 2592.6666666666665,
+ "meanToolInvocationCount": 7.666666666666667
},
{
"experimentId": "gemini-3-1-pro-preview-opencode--baseline",
@@ -1863,6 +1863,386 @@
"meanToolInvocationCount": 29
}
]
+ },
+ {
+ "suite": {
+ "eveRevision": "450681ad42ce561ba53b687fd903681d7337359e",
+ "caseFingerprint": "d30d6ad26abc32b54fde75c0e632d16186ed32b018e30c59baef624c0f370de9",
+ "caseCount": 7,
+ "runsPerCell": 3
+ },
+ "experiments": [
+ {
+ "id": "claude-fable-5-claude-code--baseline",
+ "groupId": "claude-fable-5-claude-code",
+ "model": "anthropic/claude-fable-5",
+ "modelDisplayName": "Claude Fable 5",
+ "harness": "Claude Code",
+ "treatment": "baseline"
+ },
+ {
+ "id": "claude-fable-5-claude-code--guided",
+ "groupId": "claude-fable-5-claude-code",
+ "model": "anthropic/claude-fable-5",
+ "modelDisplayName": "Claude Fable 5",
+ "harness": "Claude Code",
+ "treatment": "guided"
+ },
+ {
+ "id": "glm-5-2-opencode--baseline",
+ "groupId": "glm-5-2-opencode",
+ "model": "zai/glm-5.2",
+ "modelDisplayName": "GLM 5.2",
+ "harness": "OpenCode",
+ "treatment": "baseline"
+ },
+ {
+ "id": "glm-5-2-opencode--guided",
+ "groupId": "glm-5-2-opencode",
+ "model": "zai/glm-5.2",
+ "modelDisplayName": "GLM 5.2",
+ "harness": "OpenCode",
+ "treatment": "guided"
+ }
+ ],
+ "results": [
+ {
+ "experimentId": "claude-fable-5-claude-code--baseline",
+ "caseId": "author-001-weather-tool",
+ "status": "current",
+ "passedRuns": 3,
+ "validRuns": 3,
+ "meanDurationMs": 127045,
+ "measuredAt": "2026-08-28T22:06:23.308Z",
+ "meanEstimatedListCostUsd": 6.18823,
+ "meanToolInvocationCount": 17.666666666666668,
+ "meanTokenConsumption": 598539
+ },
+ {
+ "experimentId": "claude-fable-5-claude-code--baseline",
+ "caseId": "author-002-new-project",
+ "status": "current",
+ "passedRuns": 3,
+ "validRuns": 3,
+ "meanDurationMs": 169780,
+ "measuredAt": "2026-08-28T22:06:23.319Z",
+ "meanEstimatedListCostUsd": 3.83675,
+ "meanToolInvocationCount": 8.666666666666666,
+ "meanTokenConsumption": 375135
+ },
+ {
+ "experimentId": "claude-fable-5-claude-code--baseline",
+ "caseId": "author-003-openapi-connection",
+ "status": "current",
+ "passedRuns": 3,
+ "validRuns": 3,
+ "meanDurationMs": 258551,
+ "measuredAt": "2026-08-28T22:06:23.327Z",
+ "meanEstimatedListCostUsd": 5.43717,
+ "meanToolInvocationCount": 12.666666666666666,
+ "meanTokenConsumption": 528873
+ },
+ {
+ "experimentId": "claude-fable-5-claude-code--baseline",
+ "caseId": "author-004-packaged-skill",
+ "status": "current",
+ "passedRuns": 3,
+ "validRuns": 3,
+ "meanDurationMs": 89365,
+ "measuredAt": "2026-08-28T22:06:23.334Z",
+ "meanEstimatedListCostUsd": 3.97663,
+ "meanToolInvocationCount": 11.666666666666666,
+ "meanTokenConsumption": 383479
+ },
+ {
+ "experimentId": "claude-fable-5-claude-code--baseline",
+ "caseId": "author-005-conditional-approval",
+ "status": "current",
+ "passedRuns": 3,
+ "validRuns": 3,
+ "meanDurationMs": 123612,
+ "measuredAt": "2026-08-28T22:06:23.339Z",
+ "meanEstimatedListCostUsd": 5.19217,
+ "meanToolInvocationCount": 17.666666666666668,
+ "meanTokenConsumption": 507153
+ },
+ {
+ "experimentId": "claude-fable-5-claude-code--baseline",
+ "caseId": "author-006-custom-channel",
+ "status": "current",
+ "passedRuns": 3,
+ "validRuns": 3,
+ "meanDurationMs": 104483,
+ "measuredAt": "2026-08-28T22:06:23.346Z",
+ "meanEstimatedListCostUsd": 7.76357,
+ "meanToolInvocationCount": 15.666666666666666,
+ "meanTokenConsumption": 761257
+ },
+ {
+ "experimentId": "claude-fable-5-claude-code--baseline",
+ "caseId": "author-007-digest-schedule",
+ "status": "current",
+ "passedRuns": 3,
+ "validRuns": 3,
+ "meanDurationMs": 156463,
+ "measuredAt": "2026-08-28T22:06:23.352Z",
+ "meanEstimatedListCostUsd": 3.47642,
+ "meanToolInvocationCount": 23.666666666666668,
+ "meanTokenConsumption": 339290
+ },
+ {
+ "experimentId": "claude-fable-5-claude-code--guided",
+ "caseId": "author-001-weather-tool",
+ "status": "current",
+ "passedRuns": 3,
+ "validRuns": 3,
+ "meanDurationMs": 80508,
+ "measuredAt": "2026-08-28T22:36:19.362Z",
+ "meanEstimatedListCostUsd": 3.87527,
+ "meanToolInvocationCount": 7.666666666666667,
+ "meanTokenConsumption": 375115
+ },
+ {
+ "experimentId": "claude-fable-5-claude-code--guided",
+ "caseId": "author-002-new-project",
+ "status": "current",
+ "passedRuns": 3,
+ "validRuns": 3,
+ "meanDurationMs": 80206,
+ "measuredAt": "2026-08-28T22:36:19.376Z",
+ "meanEstimatedListCostUsd": 3.40311,
+ "meanToolInvocationCount": 9,
+ "meanTokenConsumption": 332259
+ },
+ {
+ "experimentId": "claude-fable-5-claude-code--guided",
+ "caseId": "author-003-openapi-connection",
+ "status": "current",
+ "passedRuns": 3,
+ "validRuns": 3,
+ "meanDurationMs": 71462,
+ "measuredAt": "2026-08-28T22:36:19.386Z",
+ "meanEstimatedListCostUsd": 3.58212,
+ "meanToolInvocationCount": 8.666666666666666,
+ "meanTokenConsumption": 350688
+ },
+ {
+ "experimentId": "claude-fable-5-claude-code--guided",
+ "caseId": "author-004-packaged-skill",
+ "status": "current",
+ "passedRuns": 3,
+ "validRuns": 3,
+ "meanDurationMs": 87034,
+ "measuredAt": "2026-08-28T22:36:19.391Z",
+ "meanEstimatedListCostUsd": 4.10959,
+ "meanToolInvocationCount": 7.666666666666667,
+ "meanTokenConsumption": 397447
+ },
+ {
+ "experimentId": "claude-fable-5-claude-code--guided",
+ "caseId": "author-005-conditional-approval",
+ "status": "current",
+ "passedRuns": 3,
+ "validRuns": 3,
+ "meanDurationMs": 59604,
+ "measuredAt": "2026-08-28T22:36:19.396Z",
+ "meanEstimatedListCostUsd": 3.38543,
+ "meanToolInvocationCount": 7,
+ "meanTokenConsumption": 331919
+ },
+ {
+ "experimentId": "claude-fable-5-claude-code--guided",
+ "caseId": "author-006-custom-channel",
+ "status": "current",
+ "passedRuns": 3,
+ "validRuns": 3,
+ "meanDurationMs": 64829,
+ "measuredAt": "2026-08-28T22:36:19.402Z",
+ "meanEstimatedListCostUsd": 3.81826,
+ "meanToolInvocationCount": 7,
+ "meanTokenConsumption": 374610
+ },
+ {
+ "experimentId": "claude-fable-5-claude-code--guided",
+ "caseId": "author-007-digest-schedule",
+ "status": "current",
+ "passedRuns": 3,
+ "validRuns": 3,
+ "meanDurationMs": 71055,
+ "measuredAt": "2026-08-28T22:36:19.407Z",
+ "meanEstimatedListCostUsd": 3.07521,
+ "meanToolInvocationCount": 7,
+ "meanTokenConsumption": 299897
+ },
+ {
+ "experimentId": "glm-5-2-opencode--baseline",
+ "caseId": "author-001-weather-tool",
+ "status": "current",
+ "passedRuns": 3,
+ "validRuns": 3,
+ "meanDurationMs": 103530,
+ "measuredAt": "2026-08-28T19:22:09.000Z",
+ "meanEstimatedListCostUsd": 0.07017806666666666,
+ "meanTokenConsumption": 37804.333333333336,
+ "meanToolInvocationCount": 25
+ },
+ {
+ "experimentId": "glm-5-2-opencode--baseline",
+ "caseId": "author-002-new-project",
+ "status": "current",
+ "passedRuns": 3,
+ "validRuns": 3,
+ "meanDurationMs": 100847,
+ "measuredAt": "2026-08-28T19:26:40.000Z",
+ "meanEstimatedListCostUsd": 0.03248553333333333,
+ "meanTokenConsumption": 19034.666666666668,
+ "meanToolInvocationCount": 18.333333333333332
+ },
+ {
+ "experimentId": "glm-5-2-opencode--baseline",
+ "caseId": "author-003-openapi-connection",
+ "status": "current",
+ "passedRuns": 3,
+ "validRuns": 3,
+ "meanDurationMs": 134009,
+ "measuredAt": "2026-08-28T19:26:40.000Z",
+ "meanEstimatedListCostUsd": 0.09391753333333334,
+ "meanTokenConsumption": 49094.666666666664,
+ "meanToolInvocationCount": 40.333333333333336
+ },
+ {
+ "experimentId": "glm-5-2-opencode--baseline",
+ "caseId": "author-004-packaged-skill",
+ "status": "current",
+ "passedRuns": 3,
+ "validRuns": 3,
+ "meanDurationMs": 214119,
+ "measuredAt": "2026-08-28T19:22:09.000Z",
+ "meanEstimatedListCostUsd": 0.07759993333333333,
+ "meanTokenConsumption": 41325.666666666664,
+ "meanToolInvocationCount": 36.333333333333336
+ },
+ {
+ "experimentId": "glm-5-2-opencode--baseline",
+ "caseId": "author-005-conditional-approval",
+ "status": "current",
+ "passedRuns": 3,
+ "validRuns": 3,
+ "meanDurationMs": 58527,
+ "measuredAt": "2026-08-28T19:22:09.000Z",
+ "meanEstimatedListCostUsd": 0.07383253333333333,
+ "meanTokenConsumption": 48329.666666666664,
+ "meanToolInvocationCount": 25.333333333333332
+ },
+ {
+ "experimentId": "glm-5-2-opencode--baseline",
+ "caseId": "author-006-custom-channel",
+ "status": "current",
+ "passedRuns": 3,
+ "validRuns": 3,
+ "meanDurationMs": 75109,
+ "measuredAt": "2026-08-28T19:22:09.000Z",
+ "meanEstimatedListCostUsd": 0.07836139999999998,
+ "meanTokenConsumption": 48506,
+ "meanToolInvocationCount": 27.666666666666668
+ },
+ {
+ "experimentId": "glm-5-2-opencode--baseline",
+ "caseId": "author-007-digest-schedule",
+ "status": "current",
+ "passedRuns": 3,
+ "validRuns": 3,
+ "meanDurationMs": 93384,
+ "measuredAt": "2026-08-28T19:22:09.000Z",
+ "meanEstimatedListCostUsd": 0.04282053333333333,
+ "meanTokenConsumption": 24309.666666666668,
+ "meanToolInvocationCount": 19
+ },
+ {
+ "experimentId": "glm-5-2-opencode--guided",
+ "caseId": "author-001-weather-tool",
+ "status": "current",
+ "passedRuns": 3,
+ "validRuns": 3,
+ "meanDurationMs": 68607,
+ "measuredAt": "2026-08-28T19:22:01.000Z",
+ "meanEstimatedListCostUsd": 0.04188693333333334,
+ "meanTokenConsumption": 22360.666666666668,
+ "meanToolInvocationCount": 16.333333333333332
+ },
+ {
+ "experimentId": "glm-5-2-opencode--guided",
+ "caseId": "author-002-new-project",
+ "status": "current",
+ "passedRuns": 3,
+ "validRuns": 3,
+ "meanDurationMs": 90517,
+ "measuredAt": "2026-08-28T19:25:29.000Z",
+ "meanEstimatedListCostUsd": 0.02843253333333333,
+ "meanTokenConsumption": 16824.666666666668,
+ "meanToolInvocationCount": 14.333333333333334
+ },
+ {
+ "experimentId": "glm-5-2-opencode--guided",
+ "caseId": "author-003-openapi-connection",
+ "status": "current",
+ "passedRuns": 0,
+ "validRuns": 3,
+ "meanDurationMs": 524221,
+ "measuredAt": "2026-08-28T19:22:01.000Z",
+ "meanEstimatedListCostUsd": 0.1800297333333333,
+ "meanTokenConsumption": 78267.66666666667,
+ "meanToolInvocationCount": 24.666666666666668
+ },
+ {
+ "experimentId": "glm-5-2-opencode--guided",
+ "caseId": "author-004-packaged-skill",
+ "status": "current",
+ "passedRuns": 3,
+ "validRuns": 3,
+ "meanDurationMs": 68821,
+ "measuredAt": "2026-08-28T19:22:01.000Z",
+ "meanEstimatedListCostUsd": 0.031183066666666665,
+ "meanTokenConsumption": 16504.333333333332,
+ "meanToolInvocationCount": 15
+ },
+ {
+ "experimentId": "glm-5-2-opencode--guided",
+ "caseId": "author-005-conditional-approval",
+ "status": "current",
+ "passedRuns": 2,
+ "validRuns": 3,
+ "meanDurationMs": 80061,
+ "measuredAt": "2026-08-28T19:22:01.000Z",
+ "meanEstimatedListCostUsd": 0.040431333333333326,
+ "meanTokenConsumption": 26306.666666666668,
+ "meanToolInvocationCount": 14.333333333333334
+ },
+ {
+ "experimentId": "glm-5-2-opencode--guided",
+ "caseId": "author-006-custom-channel",
+ "status": "current",
+ "passedRuns": 3,
+ "validRuns": 3,
+ "meanDurationMs": 56079,
+ "measuredAt": "2026-08-28T19:22:01.000Z",
+ "meanEstimatedListCostUsd": 0.03310953333333334,
+ "meanTokenConsumption": 19814.666666666668,
+ "meanToolInvocationCount": 11
+ },
+ {
+ "experimentId": "glm-5-2-opencode--guided",
+ "caseId": "author-007-digest-schedule",
+ "status": "current",
+ "passedRuns": 3,
+ "validRuns": 3,
+ "meanDurationMs": 74552,
+ "measuredAt": "2026-08-28T19:22:01.000Z",
+ "meanEstimatedListCostUsd": 0.029523933333333335,
+ "meanTokenConsumption": 16295.666666666666,
+ "meanToolInvocationCount": 12.333333333333334
+ }
+ ]
}
]
}