The Open Framework for Building Agents
Benchmark Claude Fable 5.1 and GLM 5.3 (#2884)
Colton Padden577d78b5d902
diff --git a/apps/benchmarks/lib/benchmark-config.test.mjs b/apps/benchmarks/lib/benchmark-config.test.mjsindex 7d9ff1b0a..253dc44da 100644--- a/apps/benchmarks/lib/benchmark-config.test.mjs+++ b/apps/benchmarks/lib/benchmark-config.test.mjs@@ -14,12 +14,12 @@ test("publishes only compatibility-validated models", () => {publishedBenchmarkModels.map((benchmark) => benchmark.id),["kimi-k3",- "claude-fable-5",+ "claude-fable-5-1","grok-4-6","gpt-5-6-sol","gpt-5-6-terra","claude-sonnet-5",- "glm-5-2",+ "glm-5-3","claude-opus-5","gemini-3-1-pro-preview",],@@ -35,8 +35,8 @@ test("publishes only compatibility-validated models", () => {]);assert.equal(findPublishedBenchmarkModel("grok-4-6").model, "xai/grok-4.6");assert.equal(findPublishedBenchmarkModel("claude-sonnet-5").model, "anthropic/claude-sonnet-5");- assert.equal(findPublishedBenchmarkModel("glm-5-2").model, "zai/glm-5.2");- assert.equal(findPublishedBenchmarkModel("claude-fable-5").harness, "Claude Code");+ assert.equal(findPublishedBenchmarkModel("glm-5-3").model, "zai/glm-5.3");+ assert.equal(findPublishedBenchmarkModel("claude-fable-5-1").harness, "Claude Code");assert.equal(findPublishedBenchmarkModel("claude-sonnet-5").harness, "Claude Code");assert.equal(findPublishedBenchmarkModel("claude-opus-5").harness, "Claude Code");assert.equal(findPublishedBenchmarkModel("kimi-k3").harness, "OpenCode");@@ -50,6 +50,8 @@ test("allows candidate probes and rejects unknown models", () => {assert.equal(findBenchmarkModel("gpt-5-6-terra").support, "supported");assert.equal(findBenchmarkModel("grok-4-6").support, "supported");assert.equal(findBenchmarkModel("claude-sonnet-4-6").support, "superseded");+ assert.equal(findBenchmarkModel("claude-fable-5").support, "superseded");+ assert.equal(findBenchmarkModel("glm-5-2").support, "superseded");assert.equal(findBenchmarkModel("claude-opus-5").support, "supported");assert.throws(() => findBenchmarkModel("unknown"), /Unknown model/u);});diff --git a/apps/benchmarks/lib/benchmark-config.ts b/apps/benchmarks/lib/benchmark-config.tsindex 54c083f06..c3ff526c3 100644--- a/apps/benchmarks/lib/benchmark-config.ts+++ b/apps/benchmarks/lib/benchmark-config.ts@@ -32,6 +32,13 @@ export const benchmarkModels = [model: "anthropic/claude-fable-5",displayName: "Claude Fable 5",harness: "Claude Code",+ support: "superseded",+ },+ {+ id: "claude-fable-5-1",+ model: "anthropic/claude-fable-5.1",+ displayName: "Claude Fable 5.1",+ harness: "Claude Code",support: "supported",},{@@ -67,6 +74,13 @@ export const benchmarkModels = [model: "zai/glm-5.2",displayName: "GLM 5.2",harness: "OpenCode",+ support: "superseded",+ },+ {+ id: "glm-5-3",+ model: "zai/glm-5.3",+ displayName: "GLM 5.3",+ harness: "OpenCode",support: "supported",},{diff --git a/apps/benchmarks/scripts/cost.mjs b/apps/benchmarks/scripts/cost.mjsindex f00b00fde..f89ce73ce 100644--- a/apps/benchmarks/scripts/cost.mjs+++ b/apps/benchmarks/scripts/cost.mjs@@ -25,6 +25,14 @@ export const modelPricing = {cacheRead: 1,cacheWrite: 12.5,},+ "anthropic/claude-fable-5.1": {+ effectiveDate: "2026-09-02",+ source: "https://ai-gateway.vercel.sh/v1/models",+ input: 10,+ output: 50,+ cacheRead: 0.25,+ cacheWrite: 12.5,+ },"xai/grok-4.6": {effectiveDate: "2026-08-17",source: "https://docs.x.ai/developers/models",@@ -73,6 +81,14 @@ export const modelPricing = {cacheRead: 0.26,cacheWrite: 0,},+ "zai/glm-5.3": {+ effectiveDate: "2026-09-02",+ source: "https://ai-gateway.vercel.sh/v1/models",+ input: 1.4,+ output: 4.4,+ cacheRead: 0.14,+ cacheWrite: 0,+ },"google/gemini-3.1-pro-preview": {effectiveDate: "2026-08-17",source: "https://ai.google.dev/gemini-api/docs/pricing",diff --git a/apps/benchmarks/scripts/export-results.test.mjs b/apps/benchmarks/scripts/export-results.test.mjsindex e96c56d98..30123d0fd 100644--- a/apps/benchmarks/scripts/export-results.test.mjs+++ b/apps/benchmarks/scripts/export-results.test.mjs@@ -45,12 +45,12 @@ test("exports missing cells without publishing private artifacts", () => {[...new Set(output.experiments.map((experiment) => experiment.modelDisplayName))],["Kimi K3",- "Claude Fable 5",+ "Claude Fable 5.1","Grok 4.6","GPT-5.6 Sol","GPT-5.6 Terra","Claude Sonnet 5",- "GLM 5.2",+ "GLM 5.3","Claude Opus 5","Gemini 3.1 Pro Preview",],diff --git a/apps/docs/lib/evals/benchmark-results.json b/apps/docs/lib/evals/benchmark-results.jsonindex 70285c06a..f8c2c0854 100644--- a/apps/docs/lib/evals/benchmark-results.json+++ b/apps/docs/lib/evals/benchmark-results.json@@ -1,9 +1,9 @@{"schemaVersion": 1,- "generatedAt": "2026-08-31T20:02:25.881Z",+ "generatedAt": "2026-09-02T14:38:34.637Z","suite": {- "eveRevision": "450681ad42ce561ba53b687fd903681d7337359e",- "caseFingerprint": "d30d6ad26abc32b54fde75c0e632d16186ed32b018e30c59baef624c0f370de9",+ "eveRevision": "78fa9046b8ad377b7fdca2c6d18cd3c10afcfc77",+ "caseFingerprint": "165cb912c2f1459eb03fafdf13813b6be4ac78eb35429ed0d9e21cfb6b9adb58","caseCount": 7,"runsPerCell": 3},@@ -25,18 +25,18 @@"treatment": "guided"},{- "id": "claude-fable-5-claude-code--baseline",- "groupId": "claude-fable-5-claude-code",- "model": "anthropic/claude-fable-5",- "modelDisplayName": "Claude Fable 5",+ "id": "claude-fable-5-1-claude-code--baseline",+ "groupId": "claude-fable-5-1-claude-code",+ "model": "anthropic/claude-fable-5.1",+ "modelDisplayName": "Claude Fable 5.1","harness": "Claude Code","treatment": "baseline"},{- "id": "claude-fable-5-claude-code--guided",- "groupId": "claude-fable-5-claude-code",- "model": "anthropic/claude-fable-5",- "modelDisplayName": "Claude Fable 5",+ "id": "claude-fable-5-1-claude-code--guided",+ "groupId": "claude-fable-5-1-claude-code",+ "model": "anthropic/claude-fable-5.1",+ "modelDisplayName": "Claude Fable 5.1","harness": "Claude Code","treatment": "guided"},@@ -105,18 +105,18 @@"treatment": "guided"},{- "id": "glm-5-2-opencode--baseline",- "groupId": "glm-5-2-opencode",- "model": "zai/glm-5.2",- "modelDisplayName": "GLM 5.2",+ "id": "glm-5-3-opencode--baseline",+ "groupId": "glm-5-3-opencode",+ "model": "zai/glm-5.3",+ "modelDisplayName": "GLM 5.3","harness": "OpenCode","treatment": "baseline"},{- "id": "glm-5-2-opencode--guided",- "groupId": "glm-5-2-opencode",- "model": "zai/glm-5.2",- "modelDisplayName": "GLM 5.2",+ "id": "glm-5-3-opencode--guided",+ "groupId": "glm-5-3-opencode",+ "model": "zai/glm-5.3",+ "modelDisplayName": "GLM 5.3","harness": "OpenCode","treatment": "guided"},@@ -323,172 +323,172 @@"meanToolInvocationCount": 12.333333333333334},{- "experimentId": "claude-fable-5-claude-code--baseline",+ "experimentId": "claude-fable-5-1-claude-code--baseline","caseId": "author-001-weather-tool","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 127045,- "measuredAt": "2026-08-28T22:06:23.308Z",- "meanEstimatedListCostUsd": 6.18823,- "meanToolInvocationCount": 17.666666666666668,- "meanTokenConsumption": 598539+ "meanDurationMs": 186262,+ "measuredAt": "2026-09-02T14:19:29.095Z",+ "meanEstimatedListCostUsd": 1.291232,+ "meanTokenConsumption": 8972,+ "meanToolInvocationCount": 11.666666666666666},{- "experimentId": "claude-fable-5-claude-code--baseline",+ "experimentId": "claude-fable-5-1-claude-code--baseline","caseId": "author-002-new-project","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 169780,- "measuredAt": "2026-08-28T22:06:23.319Z",- "meanEstimatedListCostUsd": 3.83675,- "meanToolInvocationCount": 8.666666666666666,- "meanTokenConsumption": 375135+ "meanDurationMs": 148272,+ "measuredAt": "2026-09-02T14:19:29.109Z",+ "meanEstimatedListCostUsd": 0.7348391666666666,+ "meanTokenConsumption": 3895.3333333333335,+ "meanToolInvocationCount": 9},{- "experimentId": "claude-fable-5-claude-code--baseline",+ "experimentId": "claude-fable-5-1-claude-code--baseline","caseId": "author-003-openapi-connection","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 258551,- "measuredAt": "2026-08-28T22:06:23.327Z",- "meanEstimatedListCostUsd": 5.43717,- "meanToolInvocationCount": 12.666666666666666,- "meanTokenConsumption": 528873+ "meanDurationMs": 183777,+ "measuredAt": "2026-09-02T14:19:29.120Z",+ "meanEstimatedListCostUsd": 1.31527725,+ "meanTokenConsumption": 8248,+ "meanToolInvocationCount": 16.666666666666668},{- "experimentId": "claude-fable-5-claude-code--baseline",+ "experimentId": "claude-fable-5-1-claude-code--baseline","caseId": "author-004-packaged-skill","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 89365,- "measuredAt": "2026-08-28T22:06:23.334Z",- "meanEstimatedListCostUsd": 3.97663,- "meanToolInvocationCount": 11.666666666666666,- "meanTokenConsumption": 383479+ "meanDurationMs": 180513,+ "measuredAt": "2026-09-02T14:19:29.129Z",+ "meanEstimatedListCostUsd": 1.0393497499999997,+ "meanTokenConsumption": 9160.333333333334,+ "meanToolInvocationCount": 9.666666666666666},{- "experimentId": "claude-fable-5-claude-code--baseline",+ "experimentId": "claude-fable-5-1-claude-code--baseline","caseId": "author-005-conditional-approval","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 123612,- "measuredAt": "2026-08-28T22:06:23.339Z",- "meanEstimatedListCostUsd": 5.19217,- "meanToolInvocationCount": 17.666666666666668,- "meanTokenConsumption": 507153+ "meanDurationMs": 181059,+ "measuredAt": "2026-09-02T14:19:29.137Z",+ "meanEstimatedListCostUsd": 1.3291795000000002,+ "meanTokenConsumption": 7962.333333333333,+ "meanToolInvocationCount": 16.666666666666668},{- "experimentId": "claude-fable-5-claude-code--baseline",+ "experimentId": "claude-fable-5-1-claude-code--baseline","caseId": "author-006-custom-channel","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 104483,- "measuredAt": "2026-08-28T22:06:23.346Z",- "meanEstimatedListCostUsd": 7.76357,- "meanToolInvocationCount": 15.666666666666666,- "meanTokenConsumption": 761257+ "meanDurationMs": 149380,+ "measuredAt": "2026-09-02T14:19:29.147Z",+ "meanEstimatedListCostUsd": 1.0626429999999998,+ "meanTokenConsumption": 4479.333333333333,+ "meanToolInvocationCount": 10.666666666666666},{- "experimentId": "claude-fable-5-claude-code--baseline",+ "experimentId": "claude-fable-5-1-claude-code--baseline","caseId": "author-007-digest-schedule","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 156463,- "measuredAt": "2026-08-28T22:06:23.352Z",- "meanEstimatedListCostUsd": 3.47642,- "meanToolInvocationCount": 23.666666666666668,- "meanTokenConsumption": 339290+ "meanDurationMs": 151473,+ "measuredAt": "2026-09-02T14:19:29.154Z",+ "meanEstimatedListCostUsd": 0.9185504999999999,+ "meanTokenConsumption": 4827.666666666667,+ "meanToolInvocationCount": 11.666666666666666},{- "experimentId": "claude-fable-5-claude-code--guided",+ "experimentId": "claude-fable-5-1-claude-code--guided","caseId": "author-001-weather-tool","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 80508,- "measuredAt": "2026-08-28T22:36:19.362Z",- "meanEstimatedListCostUsd": 3.87527,- "meanToolInvocationCount": 7.666666666666667,- "meanTokenConsumption": 375115+ "meanDurationMs": 142627,+ "measuredAt": "2026-09-02T14:18:53.362Z",+ "meanEstimatedListCostUsd": 0.9972783333333334,+ "meanTokenConsumption": 6633,+ "meanToolInvocationCount": 6},{- "experimentId": "claude-fable-5-claude-code--guided",+ "experimentId": "claude-fable-5-1-claude-code--guided","caseId": "author-002-new-project","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 80206,- "measuredAt": "2026-08-28T22:36:19.376Z",- "meanEstimatedListCostUsd": 3.40311,- "meanToolInvocationCount": 9,- "meanTokenConsumption": 332259+ "meanDurationMs": 155541,+ "measuredAt": "2026-09-02T14:18:53.381Z",+ "meanEstimatedListCostUsd": 1.6608135,+ "meanTokenConsumption": 4615,+ "meanToolInvocationCount": 9.666666666666666},{- "experimentId": "claude-fable-5-claude-code--guided",+ "experimentId": "claude-fable-5-1-claude-code--guided","caseId": "author-003-openapi-connection","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 71462,- "measuredAt": "2026-08-28T22:36:19.386Z",- "meanEstimatedListCostUsd": 3.58212,- "meanToolInvocationCount": 8.666666666666666,- "meanTokenConsumption": 350688+ "meanDurationMs": 132220,+ "measuredAt": "2026-09-02T14:18:53.396Z",+ "meanEstimatedListCostUsd": 0.8656001666666667,+ "meanTokenConsumption": 3575,+ "meanToolInvocationCount": 7.333333333333333},{- "experimentId": "claude-fable-5-claude-code--guided",+ "experimentId": "claude-fable-5-1-claude-code--guided","caseId": "author-004-packaged-skill","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 87034,- "measuredAt": "2026-08-28T22:36:19.391Z",- "meanEstimatedListCostUsd": 4.10959,- "meanToolInvocationCount": 7.666666666666667,- "meanTokenConsumption": 397447+ "meanDurationMs": 161398,+ "measuredAt": "2026-09-02T14:18:53.406Z",+ "meanEstimatedListCostUsd": 0.7871509166666666,+ "meanTokenConsumption": 5333.333333333333,+ "meanToolInvocationCount": 8},{- "experimentId": "claude-fable-5-claude-code--guided",+ "experimentId": "claude-fable-5-1-claude-code--guided","caseId": "author-005-conditional-approval","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 59604,- "measuredAt": "2026-08-28T22:36:19.396Z",- "meanEstimatedListCostUsd": 3.38543,- "meanToolInvocationCount": 7,- "meanTokenConsumption": 331919+ "meanDurationMs": 143233,+ "measuredAt": "2026-09-02T14:18:53.417Z",+ "meanEstimatedListCostUsd": 0.8737515833333332,+ "meanTokenConsumption": 3909.3333333333335,+ "meanToolInvocationCount": 7},{- "experimentId": "claude-fable-5-claude-code--guided",+ "experimentId": "claude-fable-5-1-claude-code--guided","caseId": "author-006-custom-channel","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 64829,- "measuredAt": "2026-08-28T22:36:19.402Z",- "meanEstimatedListCostUsd": 3.81826,- "meanToolInvocationCount": 7,- "meanTokenConsumption": 374610+ "meanDurationMs": 115430,+ "measuredAt": "2026-09-02T14:18:53.429Z",+ "meanEstimatedListCostUsd": 0.63728775,+ "meanTokenConsumption": 1828.6666666666667,+ "meanToolInvocationCount": 5.666666666666667},{- "experimentId": "claude-fable-5-claude-code--guided",+ "experimentId": "claude-fable-5-1-claude-code--guided","caseId": "author-007-digest-schedule","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 71055,- "measuredAt": "2026-08-28T22:36:19.407Z",- "meanEstimatedListCostUsd": 3.07521,- "meanToolInvocationCount": 7,- "meanTokenConsumption": 299897+ "meanDurationMs": 130462,+ "measuredAt": "2026-09-02T14:18:53.440Z",+ "meanEstimatedListCostUsd": 0.6112984166666667,+ "meanTokenConsumption": 2535.3333333333335,+ "meanToolInvocationCount": 7.333333333333333},{"experimentId": "grok-4-6-opencode--baseline",@@ -1000,11 +1000,11 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 105859,- "measuredAt": "2026-08-28T22:44:03.383Z",- "meanEstimatedListCostUsd": 1.735734,- "meanToolInvocationCount": 18,- "meanTokenConsumption": 851675+ "meanDurationMs": 119150,+ "measuredAt": "2026-09-02T14:38:33.830Z",+ "meanEstimatedListCostUsd": 0.4004288333333334,+ "meanTokenConsumption": 4827,+ "meanToolInvocationCount": 11.333333333333334},{"experimentId": "claude-sonnet-5-claude-code--baseline",@@ -1012,11 +1012,11 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 73537,- "measuredAt": "2026-08-28T22:44:03.395Z",- "meanEstimatedListCostUsd": 1.07418,- "meanToolInvocationCount": 11,- "meanTokenConsumption": 530522+ "meanDurationMs": 117102,+ "measuredAt": "2026-09-02T14:38:33.846Z",+ "meanEstimatedListCostUsd": 0.38767966666666664,+ "meanTokenConsumption": 3335.6666666666665,+ "meanToolInvocationCount": 8.333333333333334},{"experimentId": "claude-sonnet-5-claude-code--baseline",@@ -1024,11 +1024,11 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 78931,- "measuredAt": "2026-08-28T22:44:03.405Z",- "meanEstimatedListCostUsd": 1.59483,- "meanToolInvocationCount": 18,- "meanTokenConsumption": 789511+ "meanDurationMs": 146914,+ "measuredAt": "2026-09-02T14:38:33.859Z",+ "meanEstimatedListCostUsd": 0.6134467,+ "meanTokenConsumption": 7727,+ "meanToolInvocationCount": 20.333333333333332},{"experimentId": "claude-sonnet-5-claude-code--baseline",@@ -1036,11 +1036,11 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 71745,- "measuredAt": "2026-08-28T22:44:03.411Z",- "meanEstimatedListCostUsd": 1.021194,- "meanToolInvocationCount": 11,- "meanTokenConsumption": 499669+ "meanDurationMs": 128723,+ "measuredAt": "2026-09-02T14:38:33.872Z",+ "meanEstimatedListCostUsd": 0.2731853333333334,+ "meanTokenConsumption": 4274,+ "meanToolInvocationCount": 10.333333333333334},{"experimentId": "claude-sonnet-5-claude-code--baseline",@@ -1048,11 +1048,11 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 93642,- "measuredAt": "2026-08-28T22:44:03.416Z",- "meanEstimatedListCostUsd": 1.82048,- "meanToolInvocationCount": 17.666666666666668,- "meanTokenConsumption": 901060+ "meanDurationMs": 112957,+ "measuredAt": "2026-09-02T14:38:33.882Z",+ "meanEstimatedListCostUsd": 0.3310536,+ "meanTokenConsumption": 3567.3333333333335,+ "meanToolInvocationCount": 12.666666666666666},{"experimentId": "claude-sonnet-5-claude-code--baseline",@@ -1060,11 +1060,11 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 91912,- "measuredAt": "2026-08-28T22:44:03.422Z",- "meanEstimatedListCostUsd": 1.172788,- "meanToolInvocationCount": 17.333333333333332,- "meanTokenConsumption": 578826+ "meanDurationMs": 206252,+ "measuredAt": "2026-09-02T14:38:33.890Z",+ "meanEstimatedListCostUsd": 0.3613244666666667,+ "meanTokenConsumption": 5164.666666666667,+ "meanToolInvocationCount": 9.666666666666666},{"experimentId": "claude-sonnet-5-claude-code--baseline",@@ -1072,11 +1072,11 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 84747,- "measuredAt": "2026-08-28T22:44:03.428Z",- "meanEstimatedListCostUsd": 1.011918,- "meanToolInvocationCount": 13.333333333333334,- "meanTokenConsumption": 498895+ "meanDurationMs": 104010,+ "measuredAt": "2026-09-02T14:38:33.898Z",+ "meanEstimatedListCostUsd": 0.22264466666666669,+ "meanTokenConsumption": 1994.3333333333333,+ "meanToolInvocationCount": 7.333333333333333},{"experimentId": "claude-sonnet-5-claude-code--guided",@@ -1084,11 +1084,11 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 69779,- "measuredAt": "2026-08-28T23:16:01.692Z",- "meanEstimatedListCostUsd": 1.629858,- "meanToolInvocationCount": 12,- "meanTokenConsumption": 801201+ "meanDurationMs": 130089,+ "measuredAt": "2026-09-02T14:34:31.320Z",+ "meanEstimatedListCostUsd": 0.40971483333333336,+ "meanTokenConsumption": 4909,+ "meanToolInvocationCount": 10.666666666666666},{"experimentId": "claude-sonnet-5-claude-code--guided",@@ -1096,11 +1096,11 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 56970,- "measuredAt": "2026-08-28T23:16:01.703Z",- "meanEstimatedListCostUsd": 0.965568,- "meanToolInvocationCount": 7.666666666666667,- "meanTokenConsumption": 476704+ "meanDurationMs": 119473,+ "measuredAt": "2026-09-02T14:34:31.340Z",+ "meanEstimatedListCostUsd": 0.2871293,+ "meanTokenConsumption": 3610.6666666666665,+ "meanToolInvocationCount": 10.333333333333334},{"experimentId": "claude-sonnet-5-claude-code--guided",@@ -1108,11 +1108,11 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 42652,- "measuredAt": "2026-08-28T23:16:01.711Z",- "meanEstimatedListCostUsd": 0.82867,- "meanToolInvocationCount": 8.333333333333334,- "meanTokenConsumption": 410267+ "meanDurationMs": 112748,+ "measuredAt": "2026-09-02T14:34:31.354Z",+ "meanEstimatedListCostUsd": 0.3381717,+ "meanTokenConsumption": 2513,+ "meanToolInvocationCount": 8.333333333333334},{"experimentId": "claude-sonnet-5-claude-code--guided",@@ -1120,11 +1120,11 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 45265,- "measuredAt": "2026-08-28T23:16:01.717Z",- "meanEstimatedListCostUsd": 0.67903,- "meanToolInvocationCount": 5,- "meanTokenConsumption": 332683+ "meanDurationMs": 115311,+ "measuredAt": "2026-09-02T14:34:31.365Z",+ "meanEstimatedListCostUsd": 0.26473703333333337,+ "meanTokenConsumption": 7318.666666666667,+ "meanToolInvocationCount": 5},{"experimentId": "claude-sonnet-5-claude-code--guided",@@ -1132,11 +1132,11 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 39690,- "measuredAt": "2026-08-28T23:16:01.722Z",- "meanEstimatedListCostUsd": 0.972688,- "meanToolInvocationCount": 7,- "meanTokenConsumption": 480972+ "meanDurationMs": 117023,+ "measuredAt": "2026-09-02T14:34:31.375Z",+ "meanEstimatedListCostUsd": 0.34835236666666664,+ "meanTokenConsumption": 2814,+ "meanToolInvocationCount": 9},{"experimentId": "claude-sonnet-5-claude-code--guided",@@ -1144,11 +1144,11 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 38868,- "measuredAt": "2026-08-28T23:16:01.727Z",- "meanEstimatedListCostUsd": 1.219158,- "meanToolInvocationCount": 7.333333333333333,- "meanTokenConsumption": 604483+ "meanDurationMs": 110065,+ "measuredAt": "2026-09-02T14:34:31.383Z",+ "meanEstimatedListCostUsd": 0.2831804,+ "meanTokenConsumption": 1897.6666666666667,+ "meanToolInvocationCount": 6},{"experimentId": "claude-sonnet-5-claude-code--guided",@@ -1156,179 +1156,179 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 68924,- "measuredAt": "2026-08-28T23:16:01.731Z",- "meanEstimatedListCostUsd": 1.053378,- "meanToolInvocationCount": 15,- "meanTokenConsumption": 521965+ "meanDurationMs": 113169,+ "measuredAt": "2026-09-02T14:34:31.393Z",+ "meanEstimatedListCostUsd": 0.31014963333333334,+ "meanTokenConsumption": 2200,+ "meanToolInvocationCount": 9},{- "experimentId": "glm-5-2-opencode--baseline",+ "experimentId": "glm-5-3-opencode--baseline","caseId": "author-001-weather-tool","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 103530,- "measuredAt": "2026-08-28T19:22:09.000Z",- "meanEstimatedListCostUsd": 0.07017806666666666,- "meanTokenConsumption": 37804.333333333336,- "meanToolInvocationCount": 25+ "meanDurationMs": 229776,+ "measuredAt": "2026-09-02T14:24:14.610Z",+ "meanEstimatedListCostUsd": 0.13380119999999998,+ "meanTokenConsumption": 39736,+ "meanToolInvocationCount": 27.666666666666668},{- "experimentId": "glm-5-2-opencode--baseline",+ "experimentId": "glm-5-3-opencode--baseline","caseId": "author-002-new-project","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 100847,- "measuredAt": "2026-08-28T19:26:40.000Z",- "meanEstimatedListCostUsd": 0.03248553333333333,- "meanTokenConsumption": 19034.666666666668,- "meanToolInvocationCount": 18.333333333333332+ "meanDurationMs": 253800,+ "measuredAt": "2026-09-02T14:24:14.640Z",+ "meanEstimatedListCostUsd": 0.04476281333333334,+ "meanTokenConsumption": 19797.666666666668,+ "meanToolInvocationCount": 13.333333333333334},{- "experimentId": "glm-5-2-opencode--baseline",+ "experimentId": "glm-5-3-opencode--baseline","caseId": "author-003-openapi-connection","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 134009,- "measuredAt": "2026-08-28T19:26:40.000Z",- "meanEstimatedListCostUsd": 0.09391753333333334,- "meanTokenConsumption": 49094.666666666664,- "meanToolInvocationCount": 40.333333333333336+ "meanDurationMs": 275839,+ "measuredAt": "2026-09-02T14:24:14.653Z",+ "meanEstimatedListCostUsd": 0.21305283333333336,+ "meanTokenConsumption": 42271,+ "meanToolInvocationCount": 43.666666666666664},{- "experimentId": "glm-5-2-opencode--baseline",+ "experimentId": "glm-5-3-opencode--baseline","caseId": "author-004-packaged-skill","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 214119,- "measuredAt": "2026-08-28T19:22:09.000Z",- "meanEstimatedListCostUsd": 0.07759993333333333,- "meanTokenConsumption": 41325.666666666664,- "meanToolInvocationCount": 36.333333333333336+ "meanDurationMs": 237736,+ "measuredAt": "2026-09-02T14:24:14.669Z",+ "meanEstimatedListCostUsd": 0.06231241333333334,+ "meanTokenConsumption": 18199.666666666668,+ "meanToolInvocationCount": 18.333333333333332},{- "experimentId": "glm-5-2-opencode--baseline",+ "experimentId": "glm-5-3-opencode--baseline","caseId": "author-005-conditional-approval","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 58527,- "measuredAt": "2026-08-28T19:22:09.000Z",- "meanEstimatedListCostUsd": 0.07383253333333333,- "meanTokenConsumption": 48329.666666666664,- "meanToolInvocationCount": 25.333333333333332+ "meanDurationMs": 267634,+ "measuredAt": "2026-09-02T14:24:14.679Z",+ "meanEstimatedListCostUsd": 0.15534122666666667,+ "meanTokenConsumption": 43595,+ "meanToolInvocationCount": 35.333333333333336},{- "experimentId": "glm-5-2-opencode--baseline",+ "experimentId": "glm-5-3-opencode--baseline","caseId": "author-006-custom-channel","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 75109,- "measuredAt": "2026-08-28T19:22:09.000Z",- "meanEstimatedListCostUsd": 0.07836139999999998,- "meanTokenConsumption": 48506,- "meanToolInvocationCount": 27.666666666666668+ "meanDurationMs": 173067,+ "measuredAt": "2026-09-02T14:24:14.691Z",+ "meanEstimatedListCostUsd": 0.15079565333333333,+ "meanTokenConsumption": 55561.666666666664,+ "meanToolInvocationCount": 29},{- "experimentId": "glm-5-2-opencode--baseline",+ "experimentId": "glm-5-3-opencode--baseline","caseId": "author-007-digest-schedule","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 93384,- "measuredAt": "2026-08-28T19:22:09.000Z",- "meanEstimatedListCostUsd": 0.04282053333333333,- "meanTokenConsumption": 24309.666666666668,- "meanToolInvocationCount": 19+ "meanDurationMs": 150379,+ "measuredAt": "2026-09-02T14:24:14.703Z",+ "meanEstimatedListCostUsd": 0.061922139999999994,+ "meanTokenConsumption": 17250,+ "meanToolInvocationCount": 23.333333333333332},{- "experimentId": "glm-5-2-opencode--guided",+ "experimentId": "glm-5-3-opencode--guided","caseId": "author-001-weather-tool","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 68607,- "measuredAt": "2026-08-28T19:22:01.000Z",- "meanEstimatedListCostUsd": 0.04188693333333334,- "meanTokenConsumption": 22360.666666666668,- "meanToolInvocationCount": 16.333333333333332+ "meanDurationMs": 192393,+ "measuredAt": "2026-09-02T14:24:09.125Z",+ "meanEstimatedListCostUsd": 0.05878674,+ "meanTokenConsumption": 23335,+ "meanToolInvocationCount": 10},{- "experimentId": "glm-5-2-opencode--guided",+ "experimentId": "glm-5-3-opencode--guided","caseId": "author-002-new-project","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 90517,- "measuredAt": "2026-08-28T19:25:29.000Z",- "meanEstimatedListCostUsd": 0.02843253333333333,- "meanTokenConsumption": 16824.666666666668,- "meanToolInvocationCount": 14.333333333333334+ "meanDurationMs": 314441,+ "measuredAt": "2026-09-02T14:24:09.147Z",+ "meanEstimatedListCostUsd": 0.04504718666666666,+ "meanTokenConsumption": 18359.333333333332,+ "meanToolInvocationCount": 14},{- "experimentId": "glm-5-2-opencode--guided",+ "experimentId": "glm-5-3-opencode--guided","caseId": "author-003-openapi-connection","status": "current",- "passedRuns": 0,+ "passedRuns": 3,"validRuns": 3,- "meanDurationMs": 524221,- "measuredAt": "2026-08-28T19:22:01.000Z",- "meanEstimatedListCostUsd": 0.1800297333333333,- "meanTokenConsumption": 78267.66666666667,- "meanToolInvocationCount": 24.666666666666668+ "meanDurationMs": 141942,+ "measuredAt": "2026-09-02T14:24:09.165Z",+ "meanEstimatedListCostUsd": 0.062038473333333344,+ "meanTokenConsumption": 22144.666666666668,+ "meanToolInvocationCount": 15},{- "experimentId": "glm-5-2-opencode--guided",+ "experimentId": "glm-5-3-opencode--guided","caseId": "author-004-packaged-skill","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 68821,- "measuredAt": "2026-08-28T19:22:01.000Z",- "meanEstimatedListCostUsd": 0.031183066666666665,- "meanTokenConsumption": 16504.333333333332,- "meanToolInvocationCount": 15+ "meanDurationMs": 140163,+ "measuredAt": "2026-09-02T14:24:09.179Z",+ "meanEstimatedListCostUsd": 0.04043716,+ "meanTokenConsumption": 14312.666666666666,+ "meanToolInvocationCount": 10.666666666666666},{- "experimentId": "glm-5-2-opencode--guided",+ "experimentId": "glm-5-3-opencode--guided","caseId": "author-005-conditional-approval","status": "current",- "passedRuns": 2,+ "passedRuns": 3,"validRuns": 3,- "meanDurationMs": 80061,- "measuredAt": "2026-08-28T19:22:01.000Z",- "meanEstimatedListCostUsd": 0.040431333333333326,- "meanTokenConsumption": 26306.666666666668,- "meanToolInvocationCount": 14.333333333333334+ "meanDurationMs": 129477,+ "measuredAt": "2026-09-02T14:24:09.190Z",+ "meanEstimatedListCostUsd": 0.04672530666666667,+ "meanTokenConsumption": 21094.666666666668,+ "meanToolInvocationCount": 8.666666666666666},{- "experimentId": "glm-5-2-opencode--guided",+ "experimentId": "glm-5-3-opencode--guided","caseId": "author-006-custom-channel","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 56079,- "measuredAt": "2026-08-28T19:22:01.000Z",- "meanEstimatedListCostUsd": 0.03310953333333334,- "meanTokenConsumption": 19814.666666666668,- "meanToolInvocationCount": 11+ "meanDurationMs": 117801,+ "measuredAt": "2026-09-02T14:24:09.200Z",+ "meanEstimatedListCostUsd": 0.036290826666666665,+ "meanTokenConsumption": 13584.333333333334,+ "meanToolInvocationCount": 10.666666666666666},{- "experimentId": "glm-5-2-opencode--guided",+ "experimentId": "glm-5-3-opencode--guided","caseId": "author-007-digest-schedule","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 74552,- "measuredAt": "2026-08-28T19:22:01.000Z",- "meanEstimatedListCostUsd": 0.029523933333333335,- "meanTokenConsumption": 16295.666666666666,- "meanToolInvocationCount": 12.333333333333334+ "meanDurationMs": 117068,+ "measuredAt": "2026-09-02T14:24:09.209Z",+ "meanEstimatedListCostUsd": 0.024231453333333333,+ "meanTokenConsumption": 7749.333333333333,+ "meanToolInvocationCount": 8.333333333333334},{"experimentId": "claude-opus-5-claude-code--baseline",@@ -1336,11 +1336,11 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 135534,- "measuredAt": "2026-08-29T00:05:25.109Z",- "meanEstimatedListCostUsd": 4.933595,- "meanToolInvocationCount": 22.666666666666668,- "meanTokenConsumption": 947443+ "meanDurationMs": 155202,+ "measuredAt": "2026-09-02T14:35:36.609Z",+ "meanEstimatedListCostUsd": 1.1599598333333334,+ "meanTokenConsumption": 10714.666666666666,+ "meanToolInvocationCount": 17.333333333333332},{"experimentId": "claude-opus-5-claude-code--baseline",@@ -1348,11 +1348,11 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 92582,- "measuredAt": "2026-08-29T00:05:25.121Z",- "meanEstimatedListCostUsd": 2.50535,- "meanToolInvocationCount": 13.333333333333334,- "meanTokenConsumption": 488330+ "meanDurationMs": 124976,+ "measuredAt": "2026-09-02T14:35:36.628Z",+ "meanEstimatedListCostUsd": 0.8593296666666667,+ "meanTokenConsumption": 3845.3333333333335,+ "meanToolInvocationCount": 10.333333333333334},{"experimentId": "claude-opus-5-claude-code--baseline",@@ -1360,11 +1360,11 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 191566,- "measuredAt": "2026-08-29T00:05:25.129Z",- "meanEstimatedListCostUsd": 14.008915,- "meanToolInvocationCount": 36.333333333333336,- "meanTokenConsumption": 2751467+ "meanDurationMs": 160123,+ "measuredAt": "2026-09-02T14:35:36.643Z",+ "meanEstimatedListCostUsd": 1.3671014166666666,+ "meanTokenConsumption": 7793.333333333333,+ "meanToolInvocationCount": 19},{"experimentId": "claude-opus-5-claude-code--baseline",@@ -1372,11 +1372,11 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 125522,- "measuredAt": "2026-08-29T00:05:25.138Z",- "meanEstimatedListCostUsd": 3.74517,- "meanToolInvocationCount": 16,- "meanTokenConsumption": 722970+ "meanDurationMs": 181217,+ "measuredAt": "2026-09-02T14:35:36.656Z",+ "meanEstimatedListCostUsd": 1.1116483333333333,+ "meanTokenConsumption": 16371.666666666666,+ "meanToolInvocationCount": 12.666666666666666},{"experimentId": "claude-opus-5-claude-code--baseline",@@ -1384,11 +1384,11 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 135687,- "measuredAt": "2026-08-29T00:05:25.146Z",- "meanEstimatedListCostUsd": 4.87658,- "meanToolInvocationCount": 26.666666666666668,- "meanTokenConsumption": 952132+ "meanDurationMs": 153333,+ "measuredAt": "2026-09-02T14:35:36.667Z",+ "meanEstimatedListCostUsd": 1.03157675,+ "meanTokenConsumption": 6217.333333333333,+ "meanToolInvocationCount": 17.333333333333332},{"experimentId": "claude-opus-5-claude-code--baseline",@@ -1396,11 +1396,11 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 202636,- "measuredAt": "2026-08-29T00:05:25.152Z",- "meanToolInvocationCount": 26.333333333333332,- "meanTokenConsumption": 1822465,- "meanEstimatedListCostUsd": 9.304045+ "meanDurationMs": 136155,+ "measuredAt": "2026-09-02T14:35:36.676Z",+ "meanEstimatedListCostUsd": 0.9316169166666667,+ "meanTokenConsumption": 4836.666666666667,+ "meanToolInvocationCount": 16.666666666666668},{"experimentId": "claude-opus-5-claude-code--baseline",@@ -1408,11 +1408,11 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 163050,- "measuredAt": "2026-08-29T00:05:25.160Z",- "meanEstimatedListCostUsd": 5.362835,- "meanToolInvocationCount": 26,- "meanTokenConsumption": 1044543+ "meanDurationMs": 120633,+ "measuredAt": "2026-09-02T14:35:36.685Z",+ "meanEstimatedListCostUsd": 0.5067333333333334,+ "meanTokenConsumption": 3232.6666666666665,+ "meanToolInvocationCount": 9.333333333333334},{"experimentId": "claude-opus-5-claude-code--guided",@@ -1420,11 +1420,11 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 106811,- "measuredAt": "2026-08-29T00:13:01.331Z",- "meanEstimatedListCostUsd": 11.438505,- "meanToolInvocationCount": 14.666666666666666,- "meanTokenConsumption": 2234333+ "meanDurationMs": 147978,+ "measuredAt": "2026-09-02T14:34:53.900Z",+ "meanEstimatedListCostUsd": 1.0020195,+ "meanTokenConsumption": 11004.333333333334,+ "meanToolInvocationCount": 11},{"experimentId": "claude-opus-5-claude-code--guided",@@ -1432,11 +1432,11 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 91787,- "measuredAt": "2026-08-29T00:13:01.344Z",- "meanEstimatedListCostUsd": 2.4951,- "meanToolInvocationCount": 16,- "meanTokenConsumption": 486252+ "meanDurationMs": 138714,+ "measuredAt": "2026-09-02T14:34:53.929Z",+ "meanEstimatedListCostUsd": 1.0472271666666666,+ "meanTokenConsumption": 8633.666666666666,+ "meanToolInvocationCount": 14.333333333333334},{"experimentId": "claude-opus-5-claude-code--guided",@@ -1444,11 +1444,11 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 114352,- "measuredAt": "2026-08-29T00:13:01.356Z",- "meanEstimatedListCostUsd": 2.90723,- "meanToolInvocationCount": 20.666666666666668,- "meanTokenConsumption": 568682+ "meanDurationMs": 137520,+ "measuredAt": "2026-09-02T14:34:53.944Z",+ "meanEstimatedListCostUsd": 0.7869943333333334,+ "meanTokenConsumption": 5193.666666666667,+ "meanToolInvocationCount": 14},{"experimentId": "claude-opus-5-claude-code--guided",@@ -1456,11 +1456,11 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 77606,- "measuredAt": "2026-08-29T00:13:01.363Z",- "meanEstimatedListCostUsd": 2.328845,- "meanToolInvocationCount": 10,- "meanTokenConsumption": 446685+ "meanDurationMs": 153316,+ "measuredAt": "2026-09-02T14:34:53.956Z",+ "meanEstimatedListCostUsd": 0.6677339166666667,+ "meanTokenConsumption": 10291.666666666666,+ "meanToolInvocationCount": 7.666666666666667},{"experimentId": "claude-opus-5-claude-code--guided",@@ -1468,11 +1468,11 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 104699,- "measuredAt": "2026-08-29T00:13:01.370Z",- "meanEstimatedListCostUsd": 4.55151,- "meanToolInvocationCount": 19,- "meanTokenConsumption": 888618+ "meanDurationMs": 116393,+ "measuredAt": "2026-09-02T14:34:53.964Z",+ "meanEstimatedListCostUsd": 0.5910709166666667,+ "meanTokenConsumption": 2916.6666666666665,+ "meanToolInvocationCount": 8.333333333333334},{"experimentId": "claude-opus-5-claude-code--guided",@@ -1480,11 +1480,11 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 162364,- "measuredAt": "2026-08-29T00:13:01.377Z",- "meanEstimatedListCostUsd": 6.511585,- "meanToolInvocationCount": 16.333333333333332,- "meanTokenConsumption": 1266737+ "meanDurationMs": 107570,+ "measuredAt": "2026-09-02T14:34:53.973Z",+ "meanEstimatedListCostUsd": 0.4942074166666666,+ "meanTokenConsumption": 2636,+ "meanToolInvocationCount": 6.666666666666667},{"experimentId": "claude-opus-5-claude-code--guided",@@ -1492,11 +1492,11 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 89702,- "measuredAt": "2026-08-29T00:13:01.384Z",- "meanEstimatedListCostUsd": 3.44083,- "meanToolInvocationCount": 16.333333333333332,- "meanTokenConsumption": 674858+ "meanDurationMs": 114311,+ "measuredAt": "2026-09-02T14:34:53.981Z",+ "meanEstimatedListCostUsd": 0.46051358333333337,+ "meanTokenConsumption": 2592.6666666666665,+ "meanToolInvocationCount": 7.666666666666667},{"experimentId": "gemini-3-1-pro-preview-opencode--baseline",@@ -1863,6 +1863,386 @@"meanToolInvocationCount": 29}]+ },+ {+ "suite": {+ "eveRevision": "450681ad42ce561ba53b687fd903681d7337359e",+ "caseFingerprint": "d30d6ad26abc32b54fde75c0e632d16186ed32b018e30c59baef624c0f370de9",+ "caseCount": 7,+ "runsPerCell": 3+ },+ "experiments": [+ {+ "id": "claude-fable-5-claude-code--baseline",+ "groupId": "claude-fable-5-claude-code",+ "model": "anthropic/claude-fable-5",+ "modelDisplayName": "Claude Fable 5",+ "harness": "Claude Code",+ "treatment": "baseline"+ },+ {+ "id": "claude-fable-5-claude-code--guided",+ "groupId": "claude-fable-5-claude-code",+ "model": "anthropic/claude-fable-5",+ "modelDisplayName": "Claude Fable 5",+ "harness": "Claude Code",+ "treatment": "guided"+ },+ {+ "id": "glm-5-2-opencode--baseline",+ "groupId": "glm-5-2-opencode",+ "model": "zai/glm-5.2",+ "modelDisplayName": "GLM 5.2",+ "harness": "OpenCode",+ "treatment": "baseline"+ },+ {+ "id": "glm-5-2-opencode--guided",+ "groupId": "glm-5-2-opencode",+ "model": "zai/glm-5.2",+ "modelDisplayName": "GLM 5.2",+ "harness": "OpenCode",+ "treatment": "guided"+ }+ ],+ "results": [+ {+ "experimentId": "claude-fable-5-claude-code--baseline",+ "caseId": "author-001-weather-tool",+ "status": "current",+ "passedRuns": 3,+ "validRuns": 3,+ "meanDurationMs": 127045,+ "measuredAt": "2026-08-28T22:06:23.308Z",+ "meanEstimatedListCostUsd": 6.18823,+ "meanToolInvocationCount": 17.666666666666668,+ "meanTokenConsumption": 598539+ },+ {+ "experimentId": "claude-fable-5-claude-code--baseline",+ "caseId": "author-002-new-project",+ "status": "current",+ "passedRuns": 3,+ "validRuns": 3,+ "meanDurationMs": 169780,+ "measuredAt": "2026-08-28T22:06:23.319Z",+ "meanEstimatedListCostUsd": 3.83675,+ "meanToolInvocationCount": 8.666666666666666,+ "meanTokenConsumption": 375135+ },+ {+ "experimentId": "claude-fable-5-claude-code--baseline",+ "caseId": "author-003-openapi-connection",+ "status": "current",+ "passedRuns": 3,+ "validRuns": 3,+ "meanDurationMs": 258551,+ "measuredAt": "2026-08-28T22:06:23.327Z",+ "meanEstimatedListCostUsd": 5.43717,+ "meanToolInvocationCount": 12.666666666666666,+ "meanTokenConsumption": 528873+ },+ {+ "experimentId": "claude-fable-5-claude-code--baseline",+ "caseId": "author-004-packaged-skill",+ "status": "current",+ "passedRuns": 3,+ "validRuns": 3,+ "meanDurationMs": 89365,+ "measuredAt": "2026-08-28T22:06:23.334Z",+ "meanEstimatedListCostUsd": 3.97663,+ "meanToolInvocationCount": 11.666666666666666,+ "meanTokenConsumption": 383479+ },+ {+ "experimentId": "claude-fable-5-claude-code--baseline",+ "caseId": "author-005-conditional-approval",+ "status": "current",+ "passedRuns": 3,+ "validRuns": 3,+ "meanDurationMs": 123612,+ "measuredAt": "2026-08-28T22:06:23.339Z",+ "meanEstimatedListCostUsd": 5.19217,+ "meanToolInvocationCount": 17.666666666666668,+ "meanTokenConsumption": 507153+ },+ {+ "experimentId": "claude-fable-5-claude-code--baseline",+ "caseId": "author-006-custom-channel",+ "status": "current",+ "passedRuns": 3,+ "validRuns": 3,+ "meanDurationMs": 104483,+ "measuredAt": "2026-08-28T22:06:23.346Z",+ "meanEstimatedListCostUsd": 7.76357,+ "meanToolInvocationCount": 15.666666666666666,+ "meanTokenConsumption": 761257+ },+ {+ "experimentId": "claude-fable-5-claude-code--baseline",+ "caseId": "author-007-digest-schedule",+ "status": "current",+ "passedRuns": 3,+ "validRuns": 3,+ "meanDurationMs": 156463,+ "measuredAt": "2026-08-28T22:06:23.352Z",+ "meanEstimatedListCostUsd": 3.47642,+ "meanToolInvocationCount": 23.666666666666668,+ "meanTokenConsumption": 339290+ },+ {+ "experimentId": "claude-fable-5-claude-code--guided",+ "caseId": "author-001-weather-tool",+ "status": "current",+ "passedRuns": 3,+ "validRuns": 3,+ "meanDurationMs": 80508,+ "measuredAt": "2026-08-28T22:36:19.362Z",+ "meanEstimatedListCostUsd": 3.87527,+ "meanToolInvocationCount": 7.666666666666667,+ "meanTokenConsumption": 375115+ },+ {+ "experimentId": "claude-fable-5-claude-code--guided",+ "caseId": "author-002-new-project",+ "status": "current",+ "passedRuns": 3,+ "validRuns": 3,+ "meanDurationMs": 80206,+ "measuredAt": "2026-08-28T22:36:19.376Z",+ "meanEstimatedListCostUsd": 3.40311,+ "meanToolInvocationCount": 9,+ "meanTokenConsumption": 332259+ },+ {+ "experimentId": "claude-fable-5-claude-code--guided",+ "caseId": "author-003-openapi-connection",+ "status": "current",+ "passedRuns": 3,+ "validRuns": 3,+ "meanDurationMs": 71462,+ "measuredAt": "2026-08-28T22:36:19.386Z",+ "meanEstimatedListCostUsd": 3.58212,+ "meanToolInvocationCount": 8.666666666666666,+ "meanTokenConsumption": 350688+ },+ {+ "experimentId": "claude-fable-5-claude-code--guided",+ "caseId": "author-004-packaged-skill",+ "status": "current",+ "passedRuns": 3,+ "validRuns": 3,+ "meanDurationMs": 87034,+ "measuredAt": "2026-08-28T22:36:19.391Z",+ "meanEstimatedListCostUsd": 4.10959,+ "meanToolInvocationCount": 7.666666666666667,+ "meanTokenConsumption": 397447+ },+ {+ "experimentId": "claude-fable-5-claude-code--guided",+ "caseId": "author-005-conditional-approval",+ "status": "current",+ "passedRuns": 3,+ "validRuns": 3,+ "meanDurationMs": 59604,+ "measuredAt": "2026-08-28T22:36:19.396Z",+ "meanEstimatedListCostUsd": 3.38543,+ "meanToolInvocationCount": 7,+ "meanTokenConsumption": 331919+ },+ {+ "experimentId": "claude-fable-5-claude-code--guided",+ "caseId": "author-006-custom-channel",+ "status": "current",+ "passedRuns": 3,+ "validRuns": 3,+ "meanDurationMs": 64829,+ "measuredAt": "2026-08-28T22:36:19.402Z",+ "meanEstimatedListCostUsd": 3.81826,+ "meanToolInvocationCount": 7,+ "meanTokenConsumption": 374610+ },+ {+ "experimentId": "claude-fable-5-claude-code--guided",+ "caseId": "author-007-digest-schedule",+ "status": "current",+ "passedRuns": 3,+ "validRuns": 3,+ "meanDurationMs": 71055,+ "measuredAt": "2026-08-28T22:36:19.407Z",+ "meanEstimatedListCostUsd": 3.07521,+ "meanToolInvocationCount": 7,+ "meanTokenConsumption": 299897+ },+ {+ "experimentId": "glm-5-2-opencode--baseline",+ "caseId": "author-001-weather-tool",+ "status": "current",+ "passedRuns": 3,+ "validRuns": 3,+ "meanDurationMs": 103530,+ "measuredAt": "2026-08-28T19:22:09.000Z",+ "meanEstimatedListCostUsd": 0.07017806666666666,+ "meanTokenConsumption": 37804.333333333336,+ "meanToolInvocationCount": 25+ },+ {+ "experimentId": "glm-5-2-opencode--baseline",+ "caseId": "author-002-new-project",+ "status": "current",+ "passedRuns": 3,+ "validRuns": 3,+ "meanDurationMs": 100847,+ "measuredAt": "2026-08-28T19:26:40.000Z",+ "meanEstimatedListCostUsd": 0.03248553333333333,+ "meanTokenConsumption": 19034.666666666668,+ "meanToolInvocationCount": 18.333333333333332+ },+ {+ "experimentId": "glm-5-2-opencode--baseline",+ "caseId": "author-003-openapi-connection",+ "status": "current",+ "passedRuns": 3,+ "validRuns": 3,+ "meanDurationMs": 134009,+ "measuredAt": "2026-08-28T19:26:40.000Z",+ "meanEstimatedListCostUsd": 0.09391753333333334,+ "meanTokenConsumption": 49094.666666666664,+ "meanToolInvocationCount": 40.333333333333336+ },+ {+ "experimentId": "glm-5-2-opencode--baseline",+ "caseId": "author-004-packaged-skill",+ "status": "current",+ "passedRuns": 3,+ "validRuns": 3,+ "meanDurationMs": 214119,+ "measuredAt": "2026-08-28T19:22:09.000Z",+ "meanEstimatedListCostUsd": 0.07759993333333333,+ "meanTokenConsumption": 41325.666666666664,+ "meanToolInvocationCount": 36.333333333333336+ },+ {+ "experimentId": "glm-5-2-opencode--baseline",+ "caseId": "author-005-conditional-approval",+ "status": "current",+ "passedRuns": 3,+ "validRuns": 3,+ "meanDurationMs": 58527,+ "measuredAt": "2026-08-28T19:22:09.000Z",+ "meanEstimatedListCostUsd": 0.07383253333333333,+ "meanTokenConsumption": 48329.666666666664,+ "meanToolInvocationCount": 25.333333333333332+ },+ {+ "experimentId": "glm-5-2-opencode--baseline",+ "caseId": "author-006-custom-channel",+ "status": "current",+ "passedRuns": 3,+ "validRuns": 3,+ "meanDurationMs": 75109,+ "measuredAt": "2026-08-28T19:22:09.000Z",+ "meanEstimatedListCostUsd": 0.07836139999999998,+ "meanTokenConsumption": 48506,+ "meanToolInvocationCount": 27.666666666666668+ },+ {+ "experimentId": "glm-5-2-opencode--baseline",+ "caseId": "author-007-digest-schedule",+ "status": "current",+ "passedRuns": 3,+ "validRuns": 3,+ "meanDurationMs": 93384,+ "measuredAt": "2026-08-28T19:22:09.000Z",+ "meanEstimatedListCostUsd": 0.04282053333333333,+ "meanTokenConsumption": 24309.666666666668,+ "meanToolInvocationCount": 19+ },+ {+ "experimentId": "glm-5-2-opencode--guided",+ "caseId": "author-001-weather-tool",+ "status": "current",+ "passedRuns": 3,+ "validRuns": 3,+ "meanDurationMs": 68607,+ "measuredAt": "2026-08-28T19:22:01.000Z",+ "meanEstimatedListCostUsd": 0.04188693333333334,+ "meanTokenConsumption": 22360.666666666668,+ "meanToolInvocationCount": 16.333333333333332+ },+ {+ "experimentId": "glm-5-2-opencode--guided",+ "caseId": "author-002-new-project",+ "status": "current",+ "passedRuns": 3,+ "validRuns": 3,+ "meanDurationMs": 90517,+ "measuredAt": "2026-08-28T19:25:29.000Z",+ "meanEstimatedListCostUsd": 0.02843253333333333,+ "meanTokenConsumption": 16824.666666666668,+ "meanToolInvocationCount": 14.333333333333334+ },+ {+ "experimentId": "glm-5-2-opencode--guided",+ "caseId": "author-003-openapi-connection",+ "status": "current",+ "passedRuns": 0,+ "validRuns": 3,+ "meanDurationMs": 524221,+ "measuredAt": "2026-08-28T19:22:01.000Z",+ "meanEstimatedListCostUsd": 0.1800297333333333,+ "meanTokenConsumption": 78267.66666666667,+ "meanToolInvocationCount": 24.666666666666668+ },+ {+ "experimentId": "glm-5-2-opencode--guided",+ "caseId": "author-004-packaged-skill",+ "status": "current",+ "passedRuns": 3,+ "validRuns": 3,+ "meanDurationMs": 68821,+ "measuredAt": "2026-08-28T19:22:01.000Z",+ "meanEstimatedListCostUsd": 0.031183066666666665,+ "meanTokenConsumption": 16504.333333333332,+ "meanToolInvocationCount": 15+ },+ {+ "experimentId": "glm-5-2-opencode--guided",+ "caseId": "author-005-conditional-approval",+ "status": "current",+ "passedRuns": 2,+ "validRuns": 3,+ "meanDurationMs": 80061,+ "measuredAt": "2026-08-28T19:22:01.000Z",+ "meanEstimatedListCostUsd": 0.040431333333333326,+ "meanTokenConsumption": 26306.666666666668,+ "meanToolInvocationCount": 14.333333333333334+ },+ {+ "experimentId": "glm-5-2-opencode--guided",+ "caseId": "author-006-custom-channel",+ "status": "current",+ "passedRuns": 3,+ "validRuns": 3,+ "meanDurationMs": 56079,+ "measuredAt": "2026-08-28T19:22:01.000Z",+ "meanEstimatedListCostUsd": 0.03310953333333334,+ "meanTokenConsumption": 19814.666666666668,+ "meanToolInvocationCount": 11+ },+ {+ "experimentId": "glm-5-2-opencode--guided",+ "caseId": "author-007-digest-schedule",+ "status": "current",+ "passedRuns": 3,+ "validRuns": 3,+ "meanDurationMs": 74552,+ "measuredAt": "2026-08-28T19:22:01.000Z",+ "meanEstimatedListCostUsd": 0.029523933333333335,+ "meanTokenConsumption": 16295.666666666666,+ "meanToolInvocationCount": 12.333333333333334+ }+ ]}]}