The Open Framework for Building Agents
{
"schemaVersion": 1,
"generatedAt": "2026-10-08T14:33:57.335Z",
"suite": {
"eveRevision": "279e2b6a81198f6008004f596474cf05140ca7bc",
"caseFingerprint": "336fa6e2645a717a4bf4d906c3ae1b9f4ce0839bfb7bc9065068490dd696d42e",
"caseCount": 7,
"runsPerCell": 3
},
"experiments": [
{
"id": "kimi-k3-opencode--baseline",
"groupId": "kimi-k3-opencode",
"model": "moonshotai/kimi-k3",
"modelDisplayName": "Kimi K3",
"harness": "OpenCode",
"treatment": "baseline"
},
{
"id": "kimi-k3-opencode--guided",
"groupId": "kimi-k3-opencode",
"model": "moonshotai/kimi-k3",
"modelDisplayName": "Kimi K3",
"harness": "OpenCode",
"treatment": "guided"
},
{
"id": "claude-fable-5-1-claude-code--baseline",
"groupId": "claude-fable-5-1-claude-code",
"model": "anthropic/claude-fable-5.1",
"modelDisplayName": "Claude Fable 5.1 (high)",
"harness": "Claude Code",
"treatment": "baseline"
},
{
"id": "claude-fable-5-1-claude-code--guided",
"groupId": "claude-fable-5-1-claude-code",
"model": "anthropic/claude-fable-5.1",
"modelDisplayName": "Claude Fable 5.1 (high)",
"harness": "Claude Code",
"treatment": "guided"
},
{
"id": "gpt-5-6-terra-codex--baseline",
"groupId": "gpt-5-6-terra-codex",
"model": "openai/gpt-5.6-terra",
"modelDisplayName": "GPT-5.6 Terra",
"harness": "Codex",
"treatment": "baseline"
},
{
"id": "gpt-5-6-terra-codex--guided",
"groupId": "gpt-5-6-terra-codex",
"model": "openai/gpt-5.6-terra",
"modelDisplayName": "GPT-5.6 Terra",
"harness": "Codex",
"treatment": "guided"
},
{
"id": "gpt-6-astra-high-codex--baseline",
"groupId": "gpt-6-astra-high-codex",
"model": "openai/gpt-6-astra",
"modelDisplayName": "GPT-6 Astra (high)",
"harness": "Codex",
"treatment": "baseline"
},
{
"id": "gpt-6-astra-high-codex--guided",
"groupId": "gpt-6-astra-high-codex",
"model": "openai/gpt-6-astra",
"modelDisplayName": "GPT-6 Astra (high)",
"harness": "Codex",
"treatment": "guided"
},
{
"id": "claude-sonnet-5-claude-code--baseline",
"groupId": "claude-sonnet-5-claude-code",
"model": "anthropic/claude-sonnet-5",
"modelDisplayName": "Claude Sonnet 5",
"harness": "Claude Code",
"treatment": "baseline"
},
{
"id": "claude-sonnet-5-claude-code--guided",
"groupId": "claude-sonnet-5-claude-code",
"model": "anthropic/claude-sonnet-5",
"modelDisplayName": "Claude Sonnet 5",
"harness": "Claude Code",
"treatment": "guided"
},
{
"id": "glm-5-3-opencode--baseline",
"groupId": "glm-5-3-opencode",
"model": "zai/glm-5.3",
"modelDisplayName": "GLM 5.3",
"harness": "OpenCode",
"treatment": "baseline"
},
{
"id": "glm-5-3-opencode--guided",
"groupId": "glm-5-3-opencode",
"model": "zai/glm-5.3",
"modelDisplayName": "GLM 5.3",
"harness": "OpenCode",
"treatment": "guided"
},
{
"id": "gemini-3-1-pro-preview-opencode--baseline",
"groupId": "gemini-3-1-pro-preview-opencode",
"model": "google/gemini-3.1-pro-preview",
"modelDisplayName": "Gemini 3.1 Pro Preview",
"harness": "OpenCode",
"treatment": "baseline"
},
{
"id": "gemini-3-1-pro-preview-opencode--guided",
"groupId": "gemini-3-1-pro-preview-opencode",
"model": "google/gemini-3.1-pro-preview",
"modelDisplayName": "Gemini 3.1 Pro Preview",
"harness": "OpenCode",
"treatment": "guided"
},
{
"id": "gemini-3-8-flash-opencode--baseline",
"groupId": "gemini-3-8-flash-opencode",
"model": "google/gemini-3.8-flash",
"modelDisplayName": "Gemini 3.8 Flash",
"harness": "OpenCode",
"treatment": "baseline"
},
{
"id": "gemini-3-8-flash-opencode--guided",
"groupId": "gemini-3-8-flash-opencode",
"model": "google/gemini-3.8-flash",
"modelDisplayName": "Gemini 3.8 Flash",
"harness": "OpenCode",
"treatment": "guided"
},
{
"id": "minimax-m3-opencode--baseline",
"groupId": "minimax-m3-opencode",
"model": "minimax/minimax-m3",
"modelDisplayName": "MiniMax M3",
"harness": "OpenCode",
"treatment": "baseline"
},
{
"id": "minimax-m3-opencode--guided",
"groupId": "minimax-m3-opencode",
"model": "minimax/minimax-m3",
"modelDisplayName": "MiniMax M3",
"harness": "OpenCode",
"treatment": "guided"
},
{
"id": "kimi-k2-7-code-opencode--baseline",
"groupId": "kimi-k2-7-code-opencode",
"model": "moonshotai/kimi-k2.7-code",
"modelDisplayName": "Kimi K2.7 Code",
"harness": "OpenCode",
"treatment": "baseline"
},
{
"id": "kimi-k2-7-code-opencode--guided",
"groupId": "kimi-k2-7-code-opencode",
"model": "moonshotai/kimi-k2.7-code",
"modelDisplayName": "Kimi K2.7 Code",
"harness": "OpenCode",
"treatment": "guided"
},
{
"id": "claude-opus-5-5-high-claude-code--baseline",
"groupId": "claude-opus-5-5-high-claude-code",
"model": "anthropic/claude-opus-5.5",
"modelDisplayName": "Claude Opus 5.5 (high)",
"harness": "Claude Code",
"treatment": "baseline"
},
{
"id": "claude-opus-5-5-high-claude-code--guided",
"groupId": "claude-opus-5-5-high-claude-code",
"model": "anthropic/claude-opus-5.5",
"modelDisplayName": "Claude Opus 5.5 (high)",
"harness": "Claude Code",
"treatment": "guided"
},
{
"id": "gpt-6-sol-high-codex--baseline",
"groupId": "gpt-6-sol-high-codex",
"model": "openai/gpt-6-sol",
"modelDisplayName": "GPT-6 Sol (high)",
"harness": "Codex",
"treatment": "baseline"
},
{
"id": "gpt-6-sol-high-codex--guided",
"groupId": "gpt-6-sol-high-codex",
"model": "openai/gpt-6-sol",
"modelDisplayName": "GPT-6 Sol (high)",
"harness": "Codex",
"treatment": "guided"
},
{
"id": "gpt-6-luna-high-codex--baseline",
"groupId": "gpt-6-luna-high-codex",
"model": "openai/gpt-6-luna",
"modelDisplayName": "GPT-6 Luna (high)",
"harness": "Codex",
"treatment": "baseline"
},
{
"id": "gpt-6-luna-high-codex--guided",
"groupId": "gpt-6-luna-high-codex",
"model": "openai/gpt-6-luna",
"modelDisplayName": "GPT-6 Luna (high)",
"harness": "Codex",
"treatment": "guided"
},
{
"id": "grok-4-7-opencode--baseline",
"groupId": "grok-4-7-opencode",
"model": "spacexai/grok-4.7",
"modelDisplayName": "Grok 4.7",
"harness": "OpenCode",
"treatment": "baseline"
},
{
"id": "grok-4-7-opencode--guided",
"groupId": "grok-4-7-opencode",
"model": "spacexai/grok-4.7",
"modelDisplayName": "Grok 4.7",
"harness": "OpenCode",
"treatment": "guided"
}
],
"results": [
{
"experimentId": "kimi-k3-opencode--baseline",
"caseId": "author-001-weather-tool",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 182488,
"measuredAt": "2026-10-07T17:53:49.672Z",
"meanEstimatedListCostUsd": 0.3365506,
"meanTokenConsumption": 39811,
"meanToolInvocationCount": 27.666666666666668
},
{
"experimentId": "kimi-k3-opencode--baseline",
"caseId": "author-002-new-project",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 118940,
"measuredAt": "2026-10-07T17:53:49.687Z",
"meanEstimatedListCostUsd": 0.0926576,
"meanTokenConsumption": 15709.333333333334,
"meanToolInvocationCount": 11
},
{
"experimentId": "kimi-k3-opencode--baseline",
"caseId": "author-003-openapi-connection",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 173791,
"measuredAt": "2026-10-07T17:53:49.695Z",
"meanEstimatedListCostUsd": 0.2977262,
"meanTokenConsumption": 39151.666666666664,
"meanToolInvocationCount": 27.333333333333332
},
{
"experimentId": "kimi-k3-opencode--baseline",
"caseId": "author-004-packaged-skill",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 154644,
"measuredAt": "2026-10-07T17:53:49.707Z",
"meanEstimatedListCostUsd": 0.142479,
"meanTokenConsumption": 18281,
"meanToolInvocationCount": 19.333333333333332
},
{
"experimentId": "kimi-k3-opencode--baseline",
"caseId": "author-005-conditional-approval",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 182203,
"measuredAt": "2026-10-07T17:53:49.717Z",
"meanEstimatedListCostUsd": 0.3224834,
"meanTokenConsumption": 40129.666666666664,
"meanToolInvocationCount": 28
},
{
"experimentId": "kimi-k3-opencode--baseline",
"caseId": "author-006-custom-channel",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 154439,
"measuredAt": "2026-10-07T17:53:49.730Z",
"meanEstimatedListCostUsd": 0.2093862,
"meanTokenConsumption": 26541,
"meanToolInvocationCount": 23.666666666666668
},
{
"experimentId": "kimi-k3-opencode--baseline",
"caseId": "author-007-digest-schedule",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 132808,
"measuredAt": "2026-10-07T17:53:49.747Z",
"meanEstimatedListCostUsd": 0.1187692,
"meanTokenConsumption": 17728.666666666668,
"meanToolInvocationCount": 18
},
{
"experimentId": "kimi-k3-opencode--guided",
"caseId": "author-001-weather-tool",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 178412,
"measuredAt": "2026-10-07T18:06:24.212Z",
"meanEstimatedListCostUsd": 0.1784928,
"meanTokenConsumption": 25840,
"meanToolInvocationCount": 16.666666666666668
},
{
"experimentId": "kimi-k3-opencode--guided",
"caseId": "author-002-new-project",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 219061,
"measuredAt": "2026-10-07T18:06:24.226Z",
"meanEstimatedListCostUsd": 0.1086592,
"meanTokenConsumption": 19816,
"meanToolInvocationCount": 12.666666666666666
},
{
"experimentId": "kimi-k3-opencode--guided",
"caseId": "author-003-openapi-connection",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 156025,
"measuredAt": "2026-10-07T18:06:24.238Z",
"meanEstimatedListCostUsd": 0.12732,
"meanTokenConsumption": 26533.333333333332,
"meanToolInvocationCount": 10.333333333333334
},
{
"experimentId": "kimi-k3-opencode--guided",
"caseId": "author-004-packaged-skill",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 145832,
"measuredAt": "2026-10-07T18:06:24.248Z",
"meanEstimatedListCostUsd": 0.11396500000000002,
"meanTokenConsumption": 19580.333333333332,
"meanToolInvocationCount": 10.333333333333334
},
{
"experimentId": "kimi-k3-opencode--guided",
"caseId": "author-005-conditional-approval",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 148846,
"measuredAt": "2026-10-07T18:06:24.257Z",
"meanEstimatedListCostUsd": 0.16298139999999997,
"meanTokenConsumption": 30186.333333333332,
"meanToolInvocationCount": 12.666666666666666
},
{
"experimentId": "kimi-k3-opencode--guided",
"caseId": "author-006-custom-channel",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 157626,
"measuredAt": "2026-10-07T18:06:24.270Z",
"meanEstimatedListCostUsd": 0.1153282,
"meanTokenConsumption": 17712.333333333332,
"meanToolInvocationCount": 11
},
{
"experimentId": "kimi-k3-opencode--guided",
"caseId": "author-007-digest-schedule",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 142137,
"measuredAt": "2026-10-07T18:06:24.286Z",
"meanEstimatedListCostUsd": 0.08892719999999998,
"meanTokenConsumption": 15018.666666666666,
"meanToolInvocationCount": 10
},
{
"experimentId": "claude-fable-5-1-claude-code--baseline",
"caseId": "author-001-weather-tool",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 299284,
"measuredAt": "2026-10-07T18:34:17.712Z",
"meanEstimatedListCostUsd": 1.7602164166666665,
"meanTokenConsumption": 14277,
"meanToolInvocationCount": 15
},
{
"experimentId": "claude-fable-5-1-claude-code--baseline",
"caseId": "author-002-new-project",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 220684,
"measuredAt": "2026-10-07T18:34:17.721Z",
"meanEstimatedListCostUsd": 0.8802499166666667,
"meanTokenConsumption": 6197.333333333333,
"meanToolInvocationCount": 9.666666666666666
},
{
"experimentId": "claude-fable-5-1-claude-code--baseline",
"caseId": "author-003-openapi-connection",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 315667,
"measuredAt": "2026-10-07T18:34:17.732Z",
"meanEstimatedListCostUsd": 1.7283125833333335,
"meanTokenConsumption": 12128.333333333334,
"meanToolInvocationCount": 23
},
{
"experimentId": "claude-fable-5-1-claude-code--baseline",
"caseId": "author-004-packaged-skill",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 237115,
"measuredAt": "2026-10-07T18:34:17.746Z",
"meanEstimatedListCostUsd": 1.02154625,
"meanTokenConsumption": 8290,
"meanToolInvocationCount": 9.333333333333334
},
{
"experimentId": "claude-fable-5-1-claude-code--baseline",
"caseId": "author-005-conditional-approval",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 282249,
"measuredAt": "2026-10-07T18:34:17.763Z",
"meanEstimatedListCostUsd": 1.7565878333333333,
"meanTokenConsumption": 9842.666666666666,
"meanToolInvocationCount": 19.666666666666668
},
{
"experimentId": "claude-fable-5-1-claude-code--baseline",
"caseId": "author-006-custom-channel",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 230125,
"measuredAt": "2026-10-07T18:34:17.779Z",
"meanEstimatedListCostUsd": 1.1730868333333333,
"meanTokenConsumption": 4567.666666666667,
"meanToolInvocationCount": 12
},
{
"experimentId": "claude-fable-5-1-claude-code--baseline",
"caseId": "author-007-digest-schedule",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 180733,
"measuredAt": "2026-10-07T18:34:17.794Z",
"meanEstimatedListCostUsd": 0.8628102499999999,
"meanTokenConsumption": 4052.3333333333335,
"meanToolInvocationCount": 10
},
{
"experimentId": "claude-fable-5-1-claude-code--guided",
"caseId": "author-001-weather-tool",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 216964,
"measuredAt": "2026-10-07T18:38:17.304Z",
"meanEstimatedListCostUsd": 1.1694255,
"meanTokenConsumption": 8664.333333333334,
"meanToolInvocationCount": 9.333333333333334
},
{
"experimentId": "claude-fable-5-1-claude-code--guided",
"caseId": "author-002-new-project",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 177448,
"measuredAt": "2026-10-07T18:38:17.319Z",
"meanEstimatedListCostUsd": 0.6489536666666667,
"meanTokenConsumption": 4501,
"meanToolInvocationCount": 7.333333333333333
},
{
"experimentId": "claude-fable-5-1-claude-code--guided",
"caseId": "author-003-openapi-connection",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 166095,
"measuredAt": "2026-10-07T18:38:17.334Z",
"meanEstimatedListCostUsd": 0.6243346666666666,
"meanTokenConsumption": 2917.3333333333335,
"meanToolInvocationCount": 7.333333333333333
},
{
"experimentId": "claude-fable-5-1-claude-code--guided",
"caseId": "author-004-packaged-skill",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 232620,
"measuredAt": "2026-10-07T18:38:17.345Z",
"meanEstimatedListCostUsd": 0.7477252499999999,
"meanTokenConsumption": 6484.666666666667,
"meanToolInvocationCount": 9
},
{
"experimentId": "claude-fable-5-1-claude-code--guided",
"caseId": "author-005-conditional-approval",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 170755,
"measuredAt": "2026-10-07T18:38:17.357Z",
"meanEstimatedListCostUsd": 0.8186226666666667,
"meanTokenConsumption": 4187.333333333333,
"meanToolInvocationCount": 7.333333333333333
},
{
"experimentId": "claude-fable-5-1-claude-code--guided",
"caseId": "author-006-custom-channel",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 154662,
"measuredAt": "2026-10-07T18:38:17.370Z",
"meanEstimatedListCostUsd": 0.6756555833333332,
"meanTokenConsumption": 2156,
"meanToolInvocationCount": 6
},
{
"experimentId": "claude-fable-5-1-claude-code--guided",
"caseId": "author-007-digest-schedule",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 176359,
"measuredAt": "2026-10-07T18:38:17.381Z",
"meanEstimatedListCostUsd": 0.5568065833333334,
"meanTokenConsumption": 2716.3333333333335,
"meanToolInvocationCount": 7.333333333333333
},
{
"experimentId": "gpt-5-6-terra-codex--baseline",
"caseId": "author-001-weather-tool",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 122298,
"measuredAt": "2026-10-07T21:58:48.506Z",
"meanEstimatedListCostUsd": 0.22163763333333333,
"meanTokenConsumption": 37612.666666666664,
"meanToolInvocationCount": 7.333333333333333
},
{
"experimentId": "gpt-5-6-terra-codex--baseline",
"caseId": "author-002-new-project",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 138040,
"measuredAt": "2026-10-07T21:58:48.529Z",
"meanEstimatedListCostUsd": 0.18387216666666664,
"meanTokenConsumption": 26238.333333333332,
"meanToolInvocationCount": 11.333333333333334
},
{
"experimentId": "gpt-5-6-terra-codex--baseline",
"caseId": "author-003-openapi-connection",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 117932,
"measuredAt": "2026-10-07T21:58:48.540Z",
"meanEstimatedListCostUsd": 0.17790966666666666,
"meanTokenConsumption": 30419,
"meanToolInvocationCount": 7.333333333333333
},
{
"experimentId": "gpt-5-6-terra-codex--baseline",
"caseId": "author-004-packaged-skill",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 137504,
"measuredAt": "2026-10-07T21:58:48.551Z",
"meanEstimatedListCostUsd": 0.18458839999999999,
"meanTokenConsumption": 30275,
"meanToolInvocationCount": 6.666666666666667
},
{
"experimentId": "gpt-5-6-terra-codex--baseline",
"caseId": "author-005-conditional-approval",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 127649,
"measuredAt": "2026-10-07T21:58:48.568Z",
"meanEstimatedListCostUsd": 0.19955179999999997,
"meanTokenConsumption": 33755.666666666664,
"meanToolInvocationCount": 6.666666666666667
},
{
"experimentId": "gpt-5-6-terra-codex--baseline",
"caseId": "author-006-custom-channel",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 116757,
"measuredAt": "2026-10-07T21:58:48.584Z",
"meanEstimatedListCostUsd": 0.17057733333333333,
"meanTokenConsumption": 29653,
"meanToolInvocationCount": 6.333333333333333
},
{
"experimentId": "gpt-5-6-terra-codex--baseline",
"caseId": "author-007-digest-schedule",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 112507,
"measuredAt": "2026-10-07T21:58:48.600Z",
"meanEstimatedListCostUsd": 0.14675936666666667,
"meanTokenConsumption": 25050.666666666668,
"meanToolInvocationCount": 7
},
{
"experimentId": "gpt-5-6-terra-codex--guided",
"caseId": "author-001-weather-tool",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 119359,
"measuredAt": "2026-10-07T22:01:49.428Z",
"meanEstimatedListCostUsd": 0.1435413,
"meanTokenConsumption": 24455,
"meanToolInvocationCount": 4.666666666666667
},
{
"experimentId": "gpt-5-6-terra-codex--guided",
"caseId": "author-002-new-project",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 161276,
"measuredAt": "2026-10-07T22:01:49.437Z",
"meanEstimatedListCostUsd": 0.21618046666666665,
"meanTokenConsumption": 29838.333333333332,
"meanToolInvocationCount": 12.333333333333334
},
{
"experimentId": "gpt-5-6-terra-codex--guided",
"caseId": "author-003-openapi-connection",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 111613,
"measuredAt": "2026-10-07T22:01:49.449Z",
"meanEstimatedListCostUsd": 0.12379853333333335,
"meanTokenConsumption": 22123,
"meanToolInvocationCount": 4.333333333333333
},
{
"experimentId": "gpt-5-6-terra-codex--guided",
"caseId": "author-004-packaged-skill",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 130435,
"measuredAt": "2026-10-07T22:01:49.456Z",
"meanEstimatedListCostUsd": 0.11441699999999999,
"meanTokenConsumption": 19053.333333333332,
"meanToolInvocationCount": 3.6666666666666665
},
{
"experimentId": "gpt-5-6-terra-codex--guided",
"caseId": "author-005-conditional-approval",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 122684,
"measuredAt": "2026-10-07T22:01:49.463Z",
"meanEstimatedListCostUsd": 0.15166520000000003,
"meanTokenConsumption": 27042.333333333332,
"meanToolInvocationCount": 4.333333333333333
},
{
"experimentId": "gpt-5-6-terra-codex--guided",
"caseId": "author-006-custom-channel",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 133774,
"measuredAt": "2026-10-07T22:01:49.471Z",
"meanEstimatedListCostUsd": 0.11311536666666666,
"meanTokenConsumption": 20847.666666666668,
"meanToolInvocationCount": 3.3333333333333335
},
{
"experimentId": "gpt-5-6-terra-codex--guided",
"caseId": "author-007-digest-schedule",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 104430,
"measuredAt": "2026-10-07T22:01:49.483Z",
"meanEstimatedListCostUsd": 0.09763046666666665,
"meanTokenConsumption": 18167.666666666668,
"meanToolInvocationCount": 3.3333333333333335
},
{
"experimentId": "gpt-6-astra-high-codex--baseline",
"caseId": "author-001-weather-tool",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 237825,
"measuredAt": "2026-10-07T17:36:21.398Z",
"meanEstimatedListCostUsd": 1.2291793333333334,
"meanTokenConsumption": 40990.333333333336,
"meanToolInvocationCount": 14
},
{
"experimentId": "gpt-6-astra-high-codex--baseline",
"caseId": "author-002-new-project",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 134301,
"measuredAt": "2026-10-07T17:36:21.424Z",
"meanEstimatedListCostUsd": 0.746785,
"meanTokenConsumption": 23511,
"meanToolInvocationCount": 10.333333333333334
},
{
"experimentId": "gpt-6-astra-high-codex--baseline",
"caseId": "author-003-openapi-connection",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 141746,
"measuredAt": "2026-10-07T17:36:21.437Z",
"meanEstimatedListCostUsd": 0.8998284999999999,
"meanTokenConsumption": 32579.666666666668,
"meanToolInvocationCount": 10.666666666666666
},
{
"experimentId": "gpt-6-astra-high-codex--baseline",
"caseId": "author-004-packaged-skill",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 217226,
"measuredAt": "2026-10-07T17:36:21.447Z",
"meanEstimatedListCostUsd": 0.9555386666666666,
"meanTokenConsumption": 31480.666666666668,
"meanToolInvocationCount": 13.333333333333334
},
{
"experimentId": "gpt-6-astra-high-codex--baseline",
"caseId": "author-005-conditional-approval",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 170948,
"measuredAt": "2026-10-07T17:36:21.456Z",
"meanEstimatedListCostUsd": 1.1034936666666666,
"meanTokenConsumption": 37715.333333333336,
"meanToolInvocationCount": 12.666666666666666
},
{
"experimentId": "gpt-6-astra-high-codex--baseline",
"caseId": "author-006-custom-channel",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 156521,
"measuredAt": "2026-10-07T17:36:21.466Z",
"meanEstimatedListCostUsd": 1.3502271666666665,
"meanTokenConsumption": 46148.666666666664,
"meanToolInvocationCount": 13
},
{
"experimentId": "gpt-6-astra-high-codex--baseline",
"caseId": "author-007-digest-schedule",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 140369,
"measuredAt": "2026-10-07T17:36:21.479Z",
"meanEstimatedListCostUsd": 0.7009501666666668,
"meanTokenConsumption": 23989.333333333332,
"meanToolInvocationCount": 9.666666666666666
},
{
"experimentId": "gpt-6-astra-high-codex--guided",
"caseId": "author-001-weather-tool",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 167803,
"measuredAt": "2026-10-07T17:40:30.710Z",
"meanEstimatedListCostUsd": 0.8353351666666667,
"meanTokenConsumption": 29147.333333333332,
"meanToolInvocationCount": 7.333333333333333
},
{
"experimentId": "gpt-6-astra-high-codex--guided",
"caseId": "author-002-new-project",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 128241,
"measuredAt": "2026-10-07T17:40:30.723Z",
"meanEstimatedListCostUsd": 0.8843616666666666,
"meanTokenConsumption": 27507.333333333332,
"meanToolInvocationCount": 13.666666666666666
},
{
"experimentId": "gpt-6-astra-high-codex--guided",
"caseId": "author-003-openapi-connection",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 143101,
"measuredAt": "2026-10-07T17:40:30.733Z",
"meanEstimatedListCostUsd": 0.6892401666666667,
"meanTokenConsumption": 25532.666666666668,
"meanToolInvocationCount": 8.333333333333334
},
{
"experimentId": "gpt-6-astra-high-codex--guided",
"caseId": "author-004-packaged-skill",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 196512,
"measuredAt": "2026-10-07T17:40:30.742Z",
"meanEstimatedListCostUsd": 0.7988335000000001,
"meanTokenConsumption": 26211.666666666668,
"meanToolInvocationCount": 8.666666666666666
},
{
"experimentId": "gpt-6-astra-high-codex--guided",
"caseId": "author-005-conditional-approval",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 128279,
"measuredAt": "2026-10-07T17:40:30.750Z",
"meanEstimatedListCostUsd": 0.6823355000000001,
"meanTokenConsumption": 25595.333333333332,
"meanToolInvocationCount": 5.333333333333333
},
{
"experimentId": "gpt-6-astra-high-codex--guided",
"caseId": "author-006-custom-channel",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 143013,
"measuredAt": "2026-10-07T17:40:30.757Z",
"meanEstimatedListCostUsd": 0.5633055,
"meanTokenConsumption": 21594.666666666668,
"meanToolInvocationCount": 5
},
{
"experimentId": "gpt-6-astra-high-codex--guided",
"caseId": "author-007-digest-schedule",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 133217,
"measuredAt": "2026-10-07T17:40:30.763Z",
"meanEstimatedListCostUsd": 0.5985591666666666,
"meanTokenConsumption": 21658.333333333332,
"meanToolInvocationCount": 6.666666666666667
},
{
"experimentId": "claude-sonnet-5-claude-code--baseline",
"caseId": "author-001-weather-tool",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 127080,
"measuredAt": "2026-10-07T17:14:32.284Z",
"meanEstimatedListCostUsd": 0.33771066666666666,
"meanTokenConsumption": 5721.333333333333,
"meanToolInvocationCount": 10.666666666666666
},
{
"experimentId": "claude-sonnet-5-claude-code--baseline",
"caseId": "author-002-new-project",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 122433,
"measuredAt": "2026-10-07T17:14:32.309Z",
"meanEstimatedListCostUsd": 0.3149904,
"meanTokenConsumption": 4382.333333333333,
"meanToolInvocationCount": 9.333333333333334
},
{
"experimentId": "claude-sonnet-5-claude-code--baseline",
"caseId": "author-003-openapi-connection",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 121003,
"measuredAt": "2026-10-07T17:14:32.328Z",
"meanEstimatedListCostUsd": 0.4386330666666666,
"meanTokenConsumption": 5224,
"meanToolInvocationCount": 16
},
{
"experimentId": "claude-sonnet-5-claude-code--baseline",
"caseId": "author-004-packaged-skill",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 135343,
"measuredAt": "2026-10-07T17:14:32.344Z",
"meanEstimatedListCostUsd": 0.32376240000000006,
"meanTokenConsumption": 5496,
"meanToolInvocationCount": 11.666666666666666
},
{
"experimentId": "claude-sonnet-5-claude-code--baseline",
"caseId": "author-005-conditional-approval",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 112919,
"measuredAt": "2026-10-07T17:14:32.359Z",
"meanEstimatedListCostUsd": 0.33600040000000003,
"meanTokenConsumption": 3654,
"meanToolInvocationCount": 10.333333333333334
},
{
"experimentId": "claude-sonnet-5-claude-code--baseline",
"caseId": "author-006-custom-channel",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 115056,
"measuredAt": "2026-10-07T17:14:32.373Z",
"meanEstimatedListCostUsd": 0.32271563333333336,
"meanTokenConsumption": 4199,
"meanToolInvocationCount": 11.666666666666666
},
{
"experimentId": "claude-sonnet-5-claude-code--baseline",
"caseId": "author-007-digest-schedule",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 112597,
"measuredAt": "2026-10-07T17:14:32.383Z",
"meanEstimatedListCostUsd": 0.20493166666666665,
"meanTokenConsumption": 2047.6666666666667,
"meanToolInvocationCount": 7.666666666666667
},
{
"experimentId": "claude-sonnet-5-claude-code--guided",
"caseId": "author-001-weather-tool",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 122291,
"measuredAt": "2026-10-07T17:19:07.735Z",
"meanEstimatedListCostUsd": 0.3211207,
"meanTokenConsumption": 3416.6666666666665,
"meanToolInvocationCount": 9
},
{
"experimentId": "claude-sonnet-5-claude-code--guided",
"caseId": "author-002-new-project",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 115898,
"measuredAt": "2026-10-07T17:19:07.746Z",
"meanEstimatedListCostUsd": 0.2441989,
"meanTokenConsumption": 3790.3333333333335,
"meanToolInvocationCount": 8.333333333333334
},
{
"experimentId": "claude-sonnet-5-claude-code--guided",
"caseId": "author-003-openapi-connection",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 123110,
"measuredAt": "2026-10-07T17:19:07.756Z",
"meanEstimatedListCostUsd": 0.27517036666666667,
"meanTokenConsumption": 2186.3333333333335,
"meanToolInvocationCount": 7.666666666666667
},
{
"experimentId": "claude-sonnet-5-claude-code--guided",
"caseId": "author-004-packaged-skill",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 127050,
"measuredAt": "2026-10-07T17:19:07.766Z",
"meanEstimatedListCostUsd": 0.1290148,
"meanTokenConsumption": 2172.6666666666665,
"meanToolInvocationCount": 5
},
{
"experimentId": "claude-sonnet-5-claude-code--guided",
"caseId": "author-005-conditional-approval",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 132250,
"measuredAt": "2026-10-07T17:19:07.780Z",
"meanEstimatedListCostUsd": 0.350098,
"meanTokenConsumption": 3087,
"meanToolInvocationCount": 8.666666666666666
},
{
"experimentId": "claude-sonnet-5-claude-code--guided",
"caseId": "author-006-custom-channel",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 140975,
"measuredAt": "2026-10-07T17:19:07.793Z",
"meanEstimatedListCostUsd": 0.29428096666666664,
"meanTokenConsumption": 2261.6666666666665,
"meanToolInvocationCount": 8
},
{
"experimentId": "claude-sonnet-5-claude-code--guided",
"caseId": "author-007-digest-schedule",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 105861,
"measuredAt": "2026-10-07T17:19:07.804Z",
"meanEstimatedListCostUsd": 0.14527139999999997,
"meanTokenConsumption": 1264.3333333333333,
"meanToolInvocationCount": 4.333333333333333
},
{
"experimentId": "glm-5-3-opencode--baseline",
"caseId": "author-001-weather-tool",
"status": "current",
"passedRuns": 2,
"validRuns": 3,
"meanDurationMs": 610963,
"measuredAt": "2026-10-07T21:26:04.429Z",
"meanEstimatedListCostUsd": 0.15013914,
"meanTokenConsumption": 36303.5,
"meanToolInvocationCount": 29
},
{
"experimentId": "glm-5-3-opencode--baseline",
"caseId": "author-002-new-project",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 304196,
"measuredAt": "2026-10-07T21:26:04.443Z",
"meanEstimatedListCostUsd": 0.07239437333333333,
"meanTokenConsumption": 17362,
"meanToolInvocationCount": 18.333333333333332
},
{
"experimentId": "glm-5-3-opencode--baseline",
"caseId": "author-003-openapi-connection",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 352602,
"measuredAt": "2026-10-07T21:26:04.457Z",
"meanEstimatedListCostUsd": 0.15343737333333332,
"meanTokenConsumption": 32562.666666666668,
"meanToolInvocationCount": 34.666666666666664
},
{
"experimentId": "glm-5-3-opencode--baseline",
"caseId": "author-004-packaged-skill",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 372568,
"measuredAt": "2026-10-07T21:26:04.479Z",
"meanEstimatedListCostUsd": 0.0860574,
"meanTokenConsumption": 20747,
"meanToolInvocationCount": 24.333333333333332
},
{
"experimentId": "glm-5-3-opencode--baseline",
"caseId": "author-005-conditional-approval",
"status": "current",
"passedRuns": 2,
"validRuns": 3,
"meanDurationMs": 409549,
"measuredAt": "2026-10-07T21:26:04.497Z",
"meanEstimatedListCostUsd": 0.12993986666666668,
"meanTokenConsumption": 30566,
"meanToolInvocationCount": 29.333333333333332
},
{
"experimentId": "glm-5-3-opencode--baseline",
"caseId": "author-006-custom-channel",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 378657,
"measuredAt": "2026-10-07T21:26:04.520Z",
"meanEstimatedListCostUsd": 0.15328993333333335,
"meanTokenConsumption": 34802,
"meanToolInvocationCount": 32
},
{
"experimentId": "glm-5-3-opencode--baseline",
"caseId": "author-007-digest-schedule",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 282814,
"measuredAt": "2026-10-07T21:26:04.542Z",
"meanEstimatedListCostUsd": 0.06492005333333334,
"meanTokenConsumption": 14953,
"meanToolInvocationCount": 26.333333333333332
},
{
"experimentId": "glm-5-3-opencode--guided",
"caseId": "author-001-weather-tool",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 368173,
"measuredAt": "2026-10-07T21:33:02.073Z",
"meanEstimatedListCostUsd": 0.11302922666666666,
"meanTokenConsumption": 25322.333333333332,
"meanToolInvocationCount": 19.666666666666668
},
{
"experimentId": "glm-5-3-opencode--guided",
"caseId": "author-002-new-project",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 225257,
"measuredAt": "2026-10-07T21:33:02.095Z",
"meanEstimatedListCostUsd": 0.051035119999999996,
"meanTokenConsumption": 14036.666666666666,
"meanToolInvocationCount": 17.333333333333332
},
{
"experimentId": "glm-5-3-opencode--guided",
"caseId": "author-003-openapi-connection",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 187845,
"measuredAt": "2026-10-07T21:33:02.118Z",
"meanEstimatedListCostUsd": 0.038902653333333335,
"meanTokenConsumption": 12674,
"meanToolInvocationCount": 10.666666666666666
},
{
"experimentId": "glm-5-3-opencode--guided",
"caseId": "author-004-packaged-skill",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 240967,
"measuredAt": "2026-10-07T21:33:02.139Z",
"meanEstimatedListCostUsd": 0.04021544,
"meanTokenConsumption": 9470.666666666666,
"meanToolInvocationCount": 11.666666666666666
},
{
"experimentId": "glm-5-3-opencode--guided",
"caseId": "author-005-conditional-approval",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 286860,
"measuredAt": "2026-10-07T21:33:02.157Z",
"meanEstimatedListCostUsd": 0.09585413333333333,
"meanTokenConsumption": 23787.666666666668,
"meanToolInvocationCount": 19
},
{
"experimentId": "glm-5-3-opencode--guided",
"caseId": "author-006-custom-channel",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 251041,
"measuredAt": "2026-10-07T21:33:02.182Z",
"meanEstimatedListCostUsd": 0.07182817333333334,
"meanTokenConsumption": 19015,
"meanToolInvocationCount": 17
},
{
"experimentId": "glm-5-3-opencode--guided",
"caseId": "author-007-digest-schedule",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 203204,
"measuredAt": "2026-10-07T21:33:02.208Z",
"meanEstimatedListCostUsd": 0.035970919999999997,
"meanTokenConsumption": 8741.666666666666,
"meanToolInvocationCount": 11.333333333333334
},
{
"experimentId": "gemini-3-1-pro-preview-opencode--baseline",
"caseId": "author-001-weather-tool",
"status": "current",
"passedRuns": 2,
"validRuns": 3,
"meanDurationMs": 247589,
"measuredAt": "2026-10-07T21:48:35.018Z",
"meanEstimatedListCostUsd": 0.5625234,
"meanTokenConsumption": 221445.33333333334,
"meanToolInvocationCount": 20.333333333333332
},
{
"experimentId": "gemini-3-1-pro-preview-opencode--baseline",
"caseId": "author-002-new-project",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 125133,
"measuredAt": "2026-10-07T21:48:35.035Z",
"meanEstimatedListCostUsd": 0.0943808,
"meanTokenConsumption": 37827.666666666664,
"meanToolInvocationCount": 7.333333333333333
},
{
"experimentId": "gemini-3-1-pro-preview-opencode--baseline",
"caseId": "author-003-openapi-connection",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 296490,
"measuredAt": "2026-10-07T21:48:35.048Z",
"meanEstimatedListCostUsd": 0.5817098666666666,
"meanTokenConsumption": 204829.33333333334,
"meanToolInvocationCount": 29.333333333333332
},
{
"experimentId": "gemini-3-1-pro-preview-opencode--baseline",
"caseId": "author-004-packaged-skill",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 144194,
"measuredAt": "2026-10-07T21:48:35.070Z",
"meanEstimatedListCostUsd": 0.0816354,
"meanTokenConsumption": 30500.666666666668,
"meanToolInvocationCount": 5
},
{
"experimentId": "gemini-3-1-pro-preview-opencode--baseline",
"caseId": "author-005-conditional-approval",
"status": "current",
"passedRuns": 1,
"validRuns": 3,
"meanDurationMs": 220270,
"measuredAt": "2026-10-07T21:48:35.083Z",
"meanEstimatedListCostUsd": 0.35649686666666663,
"meanTokenConsumption": 114200.66666666667,
"meanToolInvocationCount": 17.666666666666668
},
{
"experimentId": "gemini-3-1-pro-preview-opencode--baseline",
"caseId": "author-006-custom-channel",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 232728,
"measuredAt": "2026-10-07T21:48:35.102Z",
"meanEstimatedListCostUsd": 0.29122646666666663,
"meanTokenConsumption": 110766.33333333333,
"meanToolInvocationCount": 16
},
{
"experimentId": "gemini-3-1-pro-preview-opencode--baseline",
"caseId": "author-007-digest-schedule",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 152122,
"measuredAt": "2026-10-07T21:48:35.121Z",
"meanEstimatedListCostUsd": 0.14784206666666666,
"meanTokenConsumption": 58074,
"meanToolInvocationCount": 11.666666666666666
},
{
"experimentId": "gemini-3-1-pro-preview-opencode--guided",
"caseId": "author-001-weather-tool",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 141789,
"measuredAt": "2026-10-07T21:52:05.892Z",
"meanEstimatedListCostUsd": 0.153628,
"meanTokenConsumption": 54885.333333333336,
"meanToolInvocationCount": 10
},
{
"experimentId": "gemini-3-1-pro-preview-opencode--guided",
"caseId": "author-002-new-project",
"status": "current",
"passedRuns": 2,
"validRuns": 3,
"meanDurationMs": 145109,
"measuredAt": "2026-10-07T21:52:05.906Z",
"meanEstimatedListCostUsd": 0.12079206666666666,
"meanTokenConsumption": 46447.333333333336,
"meanToolInvocationCount": 10.333333333333334
},
{
"experimentId": "gemini-3-1-pro-preview-opencode--guided",
"caseId": "author-003-openapi-connection",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 145639,
"measuredAt": "2026-10-07T21:52:05.916Z",
"meanEstimatedListCostUsd": 0.14859993333333335,
"meanTokenConsumption": 51636,
"meanToolInvocationCount": 9.666666666666666
},
{
"experimentId": "gemini-3-1-pro-preview-opencode--guided",
"caseId": "author-004-packaged-skill",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 141402,
"measuredAt": "2026-10-07T21:52:05.927Z",
"meanEstimatedListCostUsd": 0.09085193333333332,
"meanTokenConsumption": 32130.333333333332,
"meanToolInvocationCount": 5.666666666666667
},
{
"experimentId": "gemini-3-1-pro-preview-opencode--guided",
"caseId": "author-005-conditional-approval",
"status": "current",
"passedRuns": 2,
"validRuns": 3,
"meanDurationMs": 183922,
"measuredAt": "2026-10-07T21:52:05.937Z",
"meanEstimatedListCostUsd": 0.18213406666666668,
"meanTokenConsumption": 63521.333333333336,
"meanToolInvocationCount": 11.333333333333334
},
{
"experimentId": "gemini-3-1-pro-preview-opencode--guided",
"caseId": "author-006-custom-channel",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 145893,
"measuredAt": "2026-10-07T21:52:05.951Z",
"meanEstimatedListCostUsd": 0.12927106666666668,
"meanTokenConsumption": 43747.333333333336,
"meanToolInvocationCount": 8.666666666666666
},
{
"experimentId": "gemini-3-1-pro-preview-opencode--guided",
"caseId": "author-007-digest-schedule",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 145629,
"measuredAt": "2026-10-07T21:52:05.964Z",
"meanEstimatedListCostUsd": 0.09688193333333334,
"meanTokenConsumption": 38146.333333333336,
"meanToolInvocationCount": 7
},
{
"experimentId": "gemini-3-8-flash-opencode--baseline",
"caseId": "author-001-weather-tool",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 480431,
"measuredAt": "2026-10-07T20:32:06.638Z",
"meanEstimatedListCostUsd": 1.2837646249999999,
"meanTokenConsumption": 600828.3333333334,
"meanToolInvocationCount": 109
},
{
"experimentId": "gemini-3-8-flash-opencode--baseline",
"caseId": "author-002-new-project",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 191968,
"measuredAt": "2026-10-07T20:32:06.669Z",
"meanEstimatedListCostUsd": 0.12271715,
"meanTokenConsumption": 117868.33333333333,
"meanToolInvocationCount": 23
},
{
"experimentId": "gemini-3-8-flash-opencode--baseline",
"caseId": "author-003-openapi-connection",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 475757,
"measuredAt": "2026-10-07T20:32:06.679Z",
"meanEstimatedListCostUsd": 0.7462383749999999,
"meanTokenConsumption": 458394,
"meanToolInvocationCount": 85
},
{
"experimentId": "gemini-3-8-flash-opencode--baseline",
"caseId": "author-004-packaged-skill",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 264501,
"measuredAt": "2026-10-07T20:32:06.705Z",
"meanEstimatedListCostUsd": 0.326150925,
"meanTokenConsumption": 212912.33333333334,
"meanToolInvocationCount": 42.333333333333336
},
{
"experimentId": "gemini-3-8-flash-opencode--baseline",
"caseId": "author-005-conditional-approval",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 439824,
"measuredAt": "2026-10-07T20:32:06.731Z",
"meanEstimatedListCostUsd": 0.5367316999999999,
"meanTokenConsumption": 309792.6666666667,
"meanToolInvocationCount": 65.33333333333333
},
{
"experimentId": "gemini-3-8-flash-opencode--baseline",
"caseId": "author-006-custom-channel",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 343512,
"measuredAt": "2026-10-07T20:32:06.761Z",
"meanEstimatedListCostUsd": 0.6466183249999999,
"meanTokenConsumption": 334050.6666666667,
"meanToolInvocationCount": 64
},
{
"experimentId": "gemini-3-8-flash-opencode--baseline",
"caseId": "author-007-digest-schedule",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 291390,
"measuredAt": "2026-10-07T20:32:06.785Z",
"meanEstimatedListCostUsd": 0.38674920000000007,
"meanTokenConsumption": 351992.3333333333,
"meanToolInvocationCount": 43.333333333333336
},
{
"experimentId": "gemini-3-8-flash-opencode--guided",
"caseId": "author-001-weather-tool",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 327592,
"measuredAt": "2026-10-07T20:39:20.348Z",
"meanEstimatedListCostUsd": 0.51476285,
"meanTokenConsumption": 328742,
"meanToolInvocationCount": 55.666666666666664
},
{
"experimentId": "gemini-3-8-flash-opencode--guided",
"caseId": "author-002-new-project",
"status": "current",
"passedRuns": 1,
"validRuns": 3,
"meanDurationMs": 226209,
"measuredAt": "2026-10-07T20:39:20.384Z",
"meanEstimatedListCostUsd": 0.1381991,
"meanTokenConsumption": 117559.66666666667,
"meanToolInvocationCount": 24.666666666666668
},
{
"experimentId": "gemini-3-8-flash-opencode--guided",
"caseId": "author-003-openapi-connection",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 306740,
"measuredAt": "2026-10-07T20:39:20.395Z",
"meanEstimatedListCostUsd": 0.4550721749999999,
"meanTokenConsumption": 308868.6666666667,
"meanToolInvocationCount": 57
},
{
"experimentId": "gemini-3-8-flash-opencode--guided",
"caseId": "author-004-packaged-skill",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 267083,
"measuredAt": "2026-10-07T20:39:20.424Z",
"meanEstimatedListCostUsd": 0.21935792499999998,
"meanTokenConsumption": 151813,
"meanToolInvocationCount": 33.666666666666664
},
{
"experimentId": "gemini-3-8-flash-opencode--guided",
"caseId": "author-005-conditional-approval",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 248208,
"measuredAt": "2026-10-07T20:39:20.449Z",
"meanEstimatedListCostUsd": 0.30564869999999994,
"meanTokenConsumption": 204861.33333333334,
"meanToolInvocationCount": 42
},
{
"experimentId": "gemini-3-8-flash-opencode--guided",
"caseId": "author-006-custom-channel",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 178534,
"measuredAt": "2026-10-07T20:39:20.489Z",
"meanEstimatedListCostUsd": 0.18658105,
"meanTokenConsumption": 156321,
"meanToolInvocationCount": 29
},
{
"experimentId": "gemini-3-8-flash-opencode--guided",
"caseId": "author-007-digest-schedule",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 190950,
"measuredAt": "2026-10-07T20:39:20.518Z",
"meanEstimatedListCostUsd": 0.17541602499999998,
"meanTokenConsumption": 133149,
"meanToolInvocationCount": 27.666666666666668
},
{
"experimentId": "minimax-m3-opencode--baseline",
"caseId": "author-001-weather-tool",
"status": "current",
"passedRuns": 2,
"validRuns": 3,
"meanDurationMs": 137827,
"measuredAt": "2026-10-07T19:57:38.058Z",
"meanEstimatedListCostUsd": 0.03919382,
"meanTokenConsumption": 40222,
"meanToolInvocationCount": 29.333333333333332
},
{
"experimentId": "minimax-m3-opencode--baseline",
"caseId": "author-002-new-project",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 142346,
"measuredAt": "2026-10-07T19:57:38.073Z",
"meanEstimatedListCostUsd": 0.02446828,
"meanTokenConsumption": 24358,
"meanToolInvocationCount": 17.333333333333332
},
{
"experimentId": "minimax-m3-opencode--baseline",
"caseId": "author-003-openapi-connection",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 145218,
"measuredAt": "2026-10-07T19:57:38.085Z",
"meanEstimatedListCostUsd": 0.04590585999999999,
"meanTokenConsumption": 53114.666666666664,
"meanToolInvocationCount": 27.666666666666668
},
{
"experimentId": "minimax-m3-opencode--baseline",
"caseId": "author-004-packaged-skill",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 154028,
"measuredAt": "2026-10-07T19:57:38.099Z",
"meanEstimatedListCostUsd": 0.03372036,
"meanTokenConsumption": 35014.333333333336,
"meanToolInvocationCount": 23
},
{
"experimentId": "minimax-m3-opencode--baseline",
"caseId": "author-005-conditional-approval",
"status": "current",
"passedRuns": 2,
"validRuns": 3,
"meanDurationMs": 159346,
"measuredAt": "2026-10-07T19:57:38.112Z",
"meanEstimatedListCostUsd": 0.037709999999999994,
"meanTokenConsumption": 38013.333333333336,
"meanToolInvocationCount": 27.666666666666668
},
{
"experimentId": "minimax-m3-opencode--baseline",
"caseId": "author-006-custom-channel",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 196082,
"measuredAt": "2026-10-07T19:57:38.126Z",
"meanEstimatedListCostUsd": 0.09671368000000001,
"meanTokenConsumption": 77110,
"meanToolInvocationCount": 47
},
{
"experimentId": "minimax-m3-opencode--baseline",
"caseId": "author-007-digest-schedule",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 139910,
"measuredAt": "2026-10-07T19:57:38.143Z",
"meanEstimatedListCostUsd": 0.017940519999999998,
"meanTokenConsumption": 18308.333333333332,
"meanToolInvocationCount": 19
},
{
"experimentId": "minimax-m3-opencode--guided",
"caseId": "author-001-weather-tool",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 120197,
"measuredAt": "2026-10-07T20:00:50.683Z",
"meanEstimatedListCostUsd": 0.018265399999999998,
"meanTokenConsumption": 29428.666666666668,
"meanToolInvocationCount": 13.666666666666666
},
{
"experimentId": "minimax-m3-opencode--guided",
"caseId": "author-002-new-project",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 122122,
"measuredAt": "2026-10-07T20:00:50.696Z",
"meanEstimatedListCostUsd": 0.01290936,
"meanTokenConsumption": 14803,
"meanToolInvocationCount": 12
},
{
"experimentId": "minimax-m3-opencode--guided",
"caseId": "author-003-openapi-connection",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 127857,
"measuredAt": "2026-10-07T20:00:50.706Z",
"meanEstimatedListCostUsd": 0.024050899999999997,
"meanTokenConsumption": 32894,
"meanToolInvocationCount": 15
},
{
"experimentId": "minimax-m3-opencode--guided",
"caseId": "author-004-packaged-skill",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 148211,
"measuredAt": "2026-10-07T20:00:50.716Z",
"meanEstimatedListCostUsd": 0.010811939999999999,
"meanTokenConsumption": 9126.333333333334,
"meanToolInvocationCount": 10
},
{
"experimentId": "minimax-m3-opencode--guided",
"caseId": "author-005-conditional-approval",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 148887,
"measuredAt": "2026-10-07T20:00:50.725Z",
"meanEstimatedListCostUsd": 0.0288123,
"meanTokenConsumption": 39894,
"meanToolInvocationCount": 19
},
{
"experimentId": "minimax-m3-opencode--guided",
"caseId": "author-006-custom-channel",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 119587,
"measuredAt": "2026-10-07T20:00:50.736Z",
"meanEstimatedListCostUsd": 0.01628684,
"meanTokenConsumption": 22547.333333333332,
"meanToolInvocationCount": 9.666666666666666
},
{
"experimentId": "minimax-m3-opencode--guided",
"caseId": "author-007-digest-schedule",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 116089,
"measuredAt": "2026-10-07T20:00:50.745Z",
"meanEstimatedListCostUsd": 0.01088158,
"meanTokenConsumption": 13971.666666666666,
"meanToolInvocationCount": 10
},
{
"experimentId": "kimi-k2-7-code-opencode--baseline",
"caseId": "author-001-weather-tool",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 217800,
"measuredAt": "2026-10-07T19:45:29.923Z",
"meanEstimatedListCostUsd": 0.14899567666666666,
"meanTokenConsumption": 34828.666666666664,
"meanToolInvocationCount": 32
},
{
"experimentId": "kimi-k2-7-code-opencode--baseline",
"caseId": "author-002-new-project",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 174725,
"measuredAt": "2026-10-07T19:45:29.947Z",
"meanEstimatedListCostUsd": 0.06421423000000001,
"meanTokenConsumption": 19656.666666666668,
"meanToolInvocationCount": 22.666666666666668
},
{
"experimentId": "kimi-k2-7-code-opencode--baseline",
"caseId": "author-003-openapi-connection",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 172017,
"measuredAt": "2026-10-07T19:45:29.963Z",
"meanEstimatedListCostUsd": 0.12660882333333334,
"meanTokenConsumption": 35982.666666666664,
"meanToolInvocationCount": 25.333333333333332
},
{
"experimentId": "kimi-k2-7-code-opencode--baseline",
"caseId": "author-004-packaged-skill",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 176932,
"measuredAt": "2026-10-07T19:45:29.978Z",
"meanEstimatedListCostUsd": 0.09377714666666666,
"meanTokenConsumption": 27106.333333333332,
"meanToolInvocationCount": 25.666666666666668
},
{
"experimentId": "kimi-k2-7-code-opencode--baseline",
"caseId": "author-005-conditional-approval",
"status": "current",
"passedRuns": 2,
"validRuns": 3,
"meanDurationMs": 189532,
"measuredAt": "2026-10-07T19:45:29.991Z",
"meanEstimatedListCostUsd": 0.13552957000000002,
"meanTokenConsumption": 37226,
"meanToolInvocationCount": 25.333333333333332
},
{
"experimentId": "kimi-k2-7-code-opencode--baseline",
"caseId": "author-006-custom-channel",
"status": "current",
"passedRuns": 2,
"validRuns": 3,
"meanDurationMs": 248066,
"measuredAt": "2026-10-07T19:45:30.008Z",
"meanEstimatedListCostUsd": 0.13744147333333334,
"meanTokenConsumption": 38285,
"meanToolInvocationCount": 28.666666666666668
},
{
"experimentId": "kimi-k2-7-code-opencode--baseline",
"caseId": "author-007-digest-schedule",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 187907,
"measuredAt": "2026-10-07T19:45:30.021Z",
"meanEstimatedListCostUsd": 0.06076566333333333,
"meanTokenConsumption": 19027.666666666668,
"meanToolInvocationCount": 25.666666666666668
},
{
"experimentId": "kimi-k2-7-code-opencode--guided",
"caseId": "author-001-weather-tool",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 181961,
"measuredAt": "2026-10-07T19:49:07.568Z",
"meanEstimatedListCostUsd": 0.08877174666666666,
"meanTokenConsumption": 26044.333333333332,
"meanToolInvocationCount": 20
},
{
"experimentId": "kimi-k2-7-code-opencode--guided",
"caseId": "author-002-new-project",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 139537,
"measuredAt": "2026-10-07T19:49:07.579Z",
"meanEstimatedListCostUsd": 0.04228683333333333,
"meanTokenConsumption": 14718,
"meanToolInvocationCount": 17.333333333333332
},
{
"experimentId": "kimi-k2-7-code-opencode--guided",
"caseId": "author-003-openapi-connection",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 185254,
"measuredAt": "2026-10-07T19:49:07.588Z",
"meanEstimatedListCostUsd": 0.04005971333333333,
"meanTokenConsumption": 14683.333333333334,
"meanToolInvocationCount": 12.666666666666666
},
{
"experimentId": "kimi-k2-7-code-opencode--guided",
"caseId": "author-004-packaged-skill",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 153696,
"measuredAt": "2026-10-07T19:49:07.597Z",
"meanEstimatedListCostUsd": 0.03293041666666666,
"meanTokenConsumption": 10946,
"meanToolInvocationCount": 11
},
{
"experimentId": "kimi-k2-7-code-opencode--guided",
"caseId": "author-005-conditional-approval",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 159597,
"measuredAt": "2026-10-07T19:49:07.606Z",
"meanEstimatedListCostUsd": 0.059909359999999995,
"meanTokenConsumption": 21384.666666666668,
"meanToolInvocationCount": 15.333333333333334
},
{
"experimentId": "kimi-k2-7-code-opencode--guided",
"caseId": "author-006-custom-channel",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 142256,
"measuredAt": "2026-10-07T19:49:07.619Z",
"meanEstimatedListCostUsd": 0.040052736666666665,
"meanTokenConsumption": 15506,
"meanToolInvocationCount": 12.333333333333334
},
{
"experimentId": "kimi-k2-7-code-opencode--guided",
"caseId": "author-007-digest-schedule",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 139483,
"measuredAt": "2026-10-07T19:49:07.629Z",
"meanEstimatedListCostUsd": 0.04300942,
"meanTokenConsumption": 13041.333333333334,
"meanToolInvocationCount": 12.333333333333334
},
{
"experimentId": "claude-opus-5-5-high-claude-code--baseline",
"caseId": "author-001-weather-tool",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 148854,
"measuredAt": "2026-10-07T18:15:47.177Z",
"meanEstimatedListCostUsd": 0.5982536666666667,
"meanTokenConsumption": 8172.666666666667,
"meanToolInvocationCount": 11.666666666666666
},
{
"experimentId": "claude-opus-5-5-high-claude-code--baseline",
"caseId": "author-002-new-project",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 131272,
"measuredAt": "2026-10-07T18:15:47.196Z",
"meanEstimatedListCostUsd": 0.22304533333333335,
"meanTokenConsumption": 2829.3333333333335,
"meanToolInvocationCount": 8
},
{
"experimentId": "claude-opus-5-5-high-claude-code--baseline",
"caseId": "author-003-openapi-connection",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 169954,
"measuredAt": "2026-10-07T18:15:47.207Z",
"meanEstimatedListCostUsd": 0.33688120000000005,
"meanTokenConsumption": 4193.333333333333,
"meanToolInvocationCount": 11.666666666666666
},
{
"experimentId": "claude-opus-5-5-high-claude-code--baseline",
"caseId": "author-004-packaged-skill",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 195780,
"measuredAt": "2026-10-07T18:15:47.217Z",
"meanEstimatedListCostUsd": 0.44434179999999995,
"meanTokenConsumption": 9915,
"meanToolInvocationCount": 10.666666666666666
},
{
"experimentId": "claude-opus-5-5-high-claude-code--baseline",
"caseId": "author-005-conditional-approval",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 158936,
"measuredAt": "2026-10-07T18:15:47.228Z",
"meanEstimatedListCostUsd": 0.5887976,
"meanTokenConsumption": 7503.666666666667,
"meanToolInvocationCount": 12.333333333333334
},
{
"experimentId": "claude-opus-5-5-high-claude-code--baseline",
"caseId": "author-006-custom-channel",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 137260,
"measuredAt": "2026-10-07T18:15:47.239Z",
"meanEstimatedListCostUsd": 0.3630187333333333,
"meanTokenConsumption": 3004,
"meanToolInvocationCount": 7.666666666666667
},
{
"experimentId": "claude-opus-5-5-high-claude-code--baseline",
"caseId": "author-007-digest-schedule",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 135953,
"measuredAt": "2026-10-07T18:15:47.248Z",
"meanEstimatedListCostUsd": 0.24009426666666667,
"meanTokenConsumption": 2781.6666666666665,
"meanToolInvocationCount": 7.666666666666667
},
{
"experimentId": "claude-opus-5-5-high-claude-code--guided",
"caseId": "author-001-weather-tool",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 152041,
"measuredAt": "2026-10-07T18:19:58.247Z",
"meanEstimatedListCostUsd": 0.37882766666666673,
"meanTokenConsumption": 6229.333333333333,
"meanToolInvocationCount": 6.666666666666667
},
{
"experimentId": "claude-opus-5-5-high-claude-code--guided",
"caseId": "author-002-new-project",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 179646,
"measuredAt": "2026-10-07T18:19:58.259Z",
"meanEstimatedListCostUsd": 0.2818064,
"meanTokenConsumption": 3093.3333333333335,
"meanToolInvocationCount": 9
},
{
"experimentId": "claude-opus-5-5-high-claude-code--guided",
"caseId": "author-003-openapi-connection",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 132152,
"measuredAt": "2026-10-07T18:19:58.271Z",
"meanEstimatedListCostUsd": 0.26926126666666667,
"meanTokenConsumption": 2762.3333333333335,
"meanToolInvocationCount": 5.666666666666667
},
{
"experimentId": "claude-opus-5-5-high-claude-code--guided",
"caseId": "author-004-packaged-skill",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 168608,
"measuredAt": "2026-10-07T18:19:58.280Z",
"meanEstimatedListCostUsd": 0.2669548,
"meanTokenConsumption": 5441,
"meanToolInvocationCount": 8
},
{
"experimentId": "claude-opus-5-5-high-claude-code--guided",
"caseId": "author-005-conditional-approval",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 152304,
"measuredAt": "2026-10-07T18:19:58.292Z",
"meanEstimatedListCostUsd": 0.2563342,
"meanTokenConsumption": 2932.3333333333335,
"meanToolInvocationCount": 7
},
{
"experimentId": "claude-opus-5-5-high-claude-code--guided",
"caseId": "author-006-custom-channel",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 142903,
"measuredAt": "2026-10-07T18:19:58.304Z",
"meanEstimatedListCostUsd": 0.20084280000000002,
"meanTokenConsumption": 2039,
"meanToolInvocationCount": 5
},
{
"experimentId": "claude-opus-5-5-high-claude-code--guided",
"caseId": "author-007-digest-schedule",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 132872,
"measuredAt": "2026-10-07T18:19:58.316Z",
"meanEstimatedListCostUsd": 0.14850560000000002,
"meanTokenConsumption": 1478,
"meanToolInvocationCount": 5.666666666666667
},
{
"experimentId": "gpt-6-sol-high-codex--baseline",
"caseId": "author-001-weather-tool",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 235603,
"measuredAt": "2026-10-07T18:49:55.511Z",
"meanEstimatedListCostUsd": 0.40047706666666666,
"meanTokenConsumption": 56350,
"meanToolInvocationCount": 25.666666666666668
},
{
"experimentId": "gpt-6-sol-high-codex--baseline",
"caseId": "author-002-new-project",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 155467,
"measuredAt": "2026-10-07T18:49:55.529Z",
"meanEstimatedListCostUsd": 0.22136066666666665,
"meanTokenConsumption": 31200.666666666668,
"meanToolInvocationCount": 19.666666666666668
},
{
"experimentId": "gpt-6-sol-high-codex--baseline",
"caseId": "author-003-openapi-connection",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 140201,
"measuredAt": "2026-10-07T18:49:55.541Z",
"meanEstimatedListCostUsd": 0.24634376666666669,
"meanTokenConsumption": 42107.333333333336,
"meanToolInvocationCount": 18
},
{
"experimentId": "gpt-6-sol-high-codex--baseline",
"caseId": "author-004-packaged-skill",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 147782,
"measuredAt": "2026-10-07T18:49:55.561Z",
"meanEstimatedListCostUsd": 0.23484953333333333,
"meanTokenConsumption": 37910,
"meanToolInvocationCount": 18.666666666666668
},
{
"experimentId": "gpt-6-sol-high-codex--baseline",
"caseId": "author-005-conditional-approval",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 184292,
"measuredAt": "2026-10-07T18:49:55.570Z",
"meanEstimatedListCostUsd": 0.3206735333333333,
"meanTokenConsumption": 49776,
"meanToolInvocationCount": 22.333333333333332
},
{
"experimentId": "gpt-6-sol-high-codex--baseline",
"caseId": "author-006-custom-channel",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 141113,
"measuredAt": "2026-10-07T18:49:55.583Z",
"meanEstimatedListCostUsd": 0.24225123333333332,
"meanTokenConsumption": 41215.666666666664,
"meanToolInvocationCount": 17.333333333333332
},
{
"experimentId": "gpt-6-sol-high-codex--baseline",
"caseId": "author-007-digest-schedule",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 143502,
"measuredAt": "2026-10-07T18:49:55.595Z",
"meanEstimatedListCostUsd": 0.1879093666666667,
"meanTokenConsumption": 32017.666666666668,
"meanToolInvocationCount": 13.666666666666666
},
{
"experimentId": "gpt-6-sol-high-codex--guided",
"caseId": "author-001-weather-tool",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 162399,
"measuredAt": "2026-10-07T18:53:33.041Z",
"meanEstimatedListCostUsd": 0.22911669999999998,
"meanTokenConsumption": 36498.333333333336,
"meanToolInvocationCount": 12.333333333333334
},
{
"experimentId": "gpt-6-sol-high-codex--guided",
"caseId": "author-002-new-project",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 154722,
"measuredAt": "2026-10-07T18:53:33.054Z",
"meanEstimatedListCostUsd": 0.2336195,
"meanTokenConsumption": 35601.666666666664,
"meanToolInvocationCount": 17.666666666666668
},
{
"experimentId": "gpt-6-sol-high-codex--guided",
"caseId": "author-003-openapi-connection",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 163348,
"measuredAt": "2026-10-07T18:53:33.066Z",
"meanEstimatedListCostUsd": 0.1844876666666667,
"meanTokenConsumption": 29518,
"meanToolInvocationCount": 13.666666666666666
},
{
"experimentId": "gpt-6-sol-high-codex--guided",
"caseId": "author-004-packaged-skill",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 123914,
"measuredAt": "2026-10-07T18:53:33.074Z",
"meanEstimatedListCostUsd": 0.12451166666666667,
"meanTokenConsumption": 21815.666666666668,
"meanToolInvocationCount": 5.333333333333333
},
{
"experimentId": "gpt-6-sol-high-codex--guided",
"caseId": "author-005-conditional-approval",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 168992,
"measuredAt": "2026-10-07T18:53:33.083Z",
"meanEstimatedListCostUsd": 0.21053246666666667,
"meanTokenConsumption": 32490.333333333332,
"meanToolInvocationCount": 13.333333333333334
},
{
"experimentId": "gpt-6-sol-high-codex--guided",
"caseId": "author-006-custom-channel",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 145077,
"measuredAt": "2026-10-07T18:53:33.091Z",
"meanEstimatedListCostUsd": 0.14791393333333333,
"meanTokenConsumption": 25818,
"meanToolInvocationCount": 8
},
{
"experimentId": "gpt-6-sol-high-codex--guided",
"caseId": "author-007-digest-schedule",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 136865,
"measuredAt": "2026-10-07T18:53:33.100Z",
"meanEstimatedListCostUsd": 0.13206753333333335,
"meanTokenConsumption": 21706.333333333332,
"meanToolInvocationCount": 9.666666666666666
},
{
"experimentId": "gpt-6-luna-high-codex--baseline",
"caseId": "author-001-weather-tool",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 172981,
"measuredAt": "2026-10-07T19:02:42.860Z",
"meanEstimatedListCostUsd": 0.014657728333333335,
"meanTokenConsumption": 46713.666666666664,
"meanToolInvocationCount": 11.333333333333334
},
{
"experimentId": "gpt-6-luna-high-codex--baseline",
"caseId": "author-002-new-project",
"status": "current",
"passedRuns": 2,
"validRuns": 3,
"meanDurationMs": 208392,
"measuredAt": "2026-10-07T19:02:42.873Z",
"meanEstimatedListCostUsd": 0.02044859666666667,
"meanTokenConsumption": 53849,
"meanToolInvocationCount": 15
},
{
"experimentId": "gpt-6-luna-high-codex--baseline",
"caseId": "author-003-openapi-connection",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 122976,
"measuredAt": "2026-10-07T19:02:42.882Z",
"meanEstimatedListCostUsd": 0.008839191666666668,
"meanTokenConsumption": 29309,
"meanToolInvocationCount": 15.333333333333334
},
{
"experimentId": "gpt-6-luna-high-codex--baseline",
"caseId": "author-004-packaged-skill",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 131981,
"measuredAt": "2026-10-07T19:02:42.890Z",
"meanEstimatedListCostUsd": 0.00776646,
"meanTokenConsumption": 25521.333333333332,
"meanToolInvocationCount": 9
},
{
"experimentId": "gpt-6-luna-high-codex--baseline",
"caseId": "author-005-conditional-approval",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 153769,
"measuredAt": "2026-10-07T19:02:42.898Z",
"meanEstimatedListCostUsd": 0.012869688333333332,
"meanTokenConsumption": 40411.333333333336,
"meanToolInvocationCount": 10.333333333333334
},
{
"experimentId": "gpt-6-luna-high-codex--baseline",
"caseId": "author-006-custom-channel",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 137271,
"measuredAt": "2026-10-07T19:02:42.909Z",
"meanEstimatedListCostUsd": 0.013865816666666668,
"meanTokenConsumption": 46150,
"meanToolInvocationCount": 9.666666666666666
},
{
"experimentId": "gpt-6-luna-high-codex--baseline",
"caseId": "author-007-digest-schedule",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 129008,
"measuredAt": "2026-10-07T19:02:42.919Z",
"meanEstimatedListCostUsd": 0.007547296666666667,
"meanTokenConsumption": 24802.666666666668,
"meanToolInvocationCount": 11
},
{
"experimentId": "gpt-6-luna-high-codex--guided",
"caseId": "author-001-weather-tool",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 140987,
"measuredAt": "2026-10-07T19:09:00.404Z",
"meanEstimatedListCostUsd": 0.011568716666666668,
"meanTokenConsumption": 35932.666666666664,
"meanToolInvocationCount": 10.666666666666666
},
{
"experimentId": "gpt-6-luna-high-codex--guided",
"caseId": "author-002-new-project",
"status": "current",
"passedRuns": 2,
"validRuns": 3,
"meanDurationMs": 243860,
"measuredAt": "2026-10-07T19:09:00.418Z",
"meanEstimatedListCostUsd": 0.025036305,
"meanTokenConsumption": 60119.333333333336,
"meanToolInvocationCount": 19.666666666666668
},
{
"experimentId": "gpt-6-luna-high-codex--guided",
"caseId": "author-003-openapi-connection",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 116960,
"measuredAt": "2026-10-07T19:09:00.454Z",
"meanEstimatedListCostUsd": 0.007297571666666666,
"meanTokenConsumption": 23631,
"meanToolInvocationCount": 6.333333333333333
},
{
"experimentId": "gpt-6-luna-high-codex--guided",
"caseId": "author-004-packaged-skill",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 148482,
"measuredAt": "2026-10-07T19:09:00.463Z",
"meanEstimatedListCostUsd": 0.007017896666666666,
"meanTokenConsumption": 23253.666666666668,
"meanToolInvocationCount": 6.666666666666667
},
{
"experimentId": "gpt-6-luna-high-codex--guided",
"caseId": "author-005-conditional-approval",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 164463,
"measuredAt": "2026-10-07T19:09:00.470Z",
"meanEstimatedListCostUsd": 0.00887198,
"meanTokenConsumption": 28326.333333333332,
"meanToolInvocationCount": 8
},
{
"experimentId": "gpt-6-luna-high-codex--guided",
"caseId": "author-006-custom-channel",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 117293,
"measuredAt": "2026-10-07T19:09:00.477Z",
"meanEstimatedListCostUsd": 0.006835386666666668,
"meanTokenConsumption": 22433,
"meanToolInvocationCount": 8
},
{
"experimentId": "gpt-6-luna-high-codex--guided",
"caseId": "author-007-digest-schedule",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 138527,
"measuredAt": "2026-10-07T19:09:00.484Z",
"meanEstimatedListCostUsd": 0.008701025000000001,
"meanTokenConsumption": 26987.333333333332,
"meanToolInvocationCount": 7.333333333333333
},
{
"experimentId": "grok-4-7-opencode--baseline",
"caseId": "author-001-weather-tool",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 136051,
"measuredAt": "2026-10-07T19:28:24.953Z",
"meanEstimatedListCostUsd": 0.17287466666666665,
"meanTokenConsumption": 52152,
"meanToolInvocationCount": 18.333333333333332
},
{
"experimentId": "grok-4-7-opencode--baseline",
"caseId": "author-002-new-project",
"status": "current",
"passedRuns": 2,
"validRuns": 3,
"meanDurationMs": 190658,
"measuredAt": "2026-10-07T19:28:24.962Z",
"meanEstimatedListCostUsd": 0.21032133333333333,
"meanTokenConsumption": 42335.333333333336,
"meanToolInvocationCount": 26
},
{
"experimentId": "grok-4-7-opencode--baseline",
"caseId": "author-003-openapi-connection",
"status": "current",
"passedRuns": 2,
"validRuns": 3,
"meanDurationMs": 383235,
"measuredAt": "2026-10-07T19:28:24.975Z",
"meanEstimatedListCostUsd": 0.143828,
"meanTokenConsumption": 43541,
"meanToolInvocationCount": 17
},
{
"experimentId": "grok-4-7-opencode--baseline",
"caseId": "author-004-packaged-skill",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 111116,
"measuredAt": "2026-10-07T19:28:24.981Z",
"meanEstimatedListCostUsd": 0.13283466666666666,
"meanTokenConsumption": 34128,
"meanToolInvocationCount": 17.666666666666668
},
{
"experimentId": "grok-4-7-opencode--baseline",
"caseId": "author-005-conditional-approval",
"status": "current",
"passedRuns": 2,
"validRuns": 3,
"meanDurationMs": 383889,
"measuredAt": "2026-10-07T19:28:24.992Z",
"meanEstimatedListCostUsd": 0.226556,
"meanTokenConsumption": 65023,
"meanToolInvocationCount": 23.5
},
{
"experimentId": "grok-4-7-opencode--baseline",
"caseId": "author-006-custom-channel",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 116196,
"measuredAt": "2026-10-07T19:28:25.000Z",
"meanEstimatedListCostUsd": 0.13987,
"meanTokenConsumption": 25978.333333333332,
"meanToolInvocationCount": 18
},
{
"experimentId": "grok-4-7-opencode--baseline",
"caseId": "author-007-digest-schedule",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 115736,
"measuredAt": "2026-10-07T19:28:25.013Z",
"meanEstimatedListCostUsd": 0.09889266666666667,
"meanTokenConsumption": 27181,
"meanToolInvocationCount": 15.333333333333334
},
{
"experimentId": "grok-4-7-opencode--guided",
"caseId": "author-001-weather-tool",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 131291,
"measuredAt": "2026-10-07T19:33:57.511Z",
"meanEstimatedListCostUsd": 0.13136866666666666,
"meanTokenConsumption": 29745,
"meanToolInvocationCount": 11.333333333333334
},
{
"experimentId": "grok-4-7-opencode--guided",
"caseId": "author-002-new-project",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 137195,
"measuredAt": "2026-10-07T19:33:57.518Z",
"meanEstimatedListCostUsd": 0.222338,
"meanTokenConsumption": 61214.333333333336,
"meanToolInvocationCount": 23.333333333333332
},
{
"experimentId": "grok-4-7-opencode--guided",
"caseId": "author-003-openapi-connection",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 110807,
"measuredAt": "2026-10-07T19:33:57.527Z",
"meanEstimatedListCostUsd": 0.14055266666666666,
"meanTokenConsumption": 35289,
"meanToolInvocationCount": 12
},
{
"experimentId": "grok-4-7-opencode--guided",
"caseId": "author-004-packaged-skill",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 134888,
"measuredAt": "2026-10-07T19:33:57.533Z",
"meanEstimatedListCostUsd": 0.05279999999999999,
"meanTokenConsumption": 10001.333333333334,
"meanToolInvocationCount": 5.666666666666667
},
{
"experimentId": "grok-4-7-opencode--guided",
"caseId": "author-005-conditional-approval",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 108481,
"measuredAt": "2026-10-07T19:33:57.539Z",
"meanEstimatedListCostUsd": 0.10046466666666666,
"meanTokenConsumption": 18673.666666666668,
"meanToolInvocationCount": 9.666666666666666
},
{
"experimentId": "grok-4-7-opencode--guided",
"caseId": "author-006-custom-channel",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 108256,
"measuredAt": "2026-10-07T19:33:57.548Z",
"meanEstimatedListCostUsd": 0.09657466666666666,
"meanTokenConsumption": 15122,
"meanToolInvocationCount": 7
},
{
"experimentId": "grok-4-7-opencode--guided",
"caseId": "author-007-digest-schedule",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 115721,
"measuredAt": "2026-10-07T19:33:57.559Z",
"meanEstimatedListCostUsd": 0.07453466666666668,
"meanTokenConsumption": 23851.333333333332,
"meanToolInvocationCount": 5
}
],
"previouslyMeasured": [
{
"suite": {
"eveRevision": "54a028bab589a60fc84a310304a5cf080bc7f990",
"caseFingerprint": "786490be9d902701f57d08b3d112ca385d49a72e5591fac741511e77d67989d2",
"caseCount": 7,
"runsPerCell": 3
},
"experiments": [
{
"id": "claude-sonnet-4-6-opencode--baseline",
"groupId": "claude-sonnet-4-6-opencode",
"model": "claude-sonnet-4-6",
"modelDisplayName": "Claude Sonnet 4.6",
"harness": "OpenCode",
"treatment": "baseline"
},
{
"id": "claude-sonnet-4-6-opencode--guided",
"groupId": "claude-sonnet-4-6-opencode",
"model": "claude-sonnet-4-6",
"modelDisplayName": "Claude Sonnet 4.6",
"harness": "OpenCode",
"treatment": "guided"
}
],
"results": [
{
"experimentId": "claude-sonnet-4-6-opencode--baseline",
"caseId": "author-001-weather-tool",
"status": "current",
"passedRuns": 0,
"validRuns": 3,
"meanDurationMs": 199451,
"measuredAt": "2026-08-19T21:35:21.709Z",
"meanEstimatedListCostUsd": 0.052948,
"meanTokenConsumption": 10626.666666666666,
"meanToolInvocationCount": 11.666666666666666
},
{
"experimentId": "claude-sonnet-4-6-opencode--baseline",
"caseId": "author-002-new-project",
"status": "current",
"passedRuns": 0,
"validRuns": 3,
"meanDurationMs": 143527,
"measuredAt": "2026-08-19T21:35:21.715Z",
"meanEstimatedListCostUsd": 0.04594499999999999,
"meanTokenConsumption": 10429.666666666666,
"meanToolInvocationCount": 10.666666666666666
},
{
"experimentId": "claude-sonnet-4-6-opencode--baseline",
"caseId": "author-003-openapi-connection",
"status": "current",
"passedRuns": 1,
"validRuns": 3,
"meanDurationMs": 158661,
"measuredAt": "2026-08-19T21:35:21.721Z",
"meanEstimatedListCostUsd": 0.11106500000000001,
"meanTokenConsumption": 27383,
"meanToolInvocationCount": 23.666666666666668
},
{
"experimentId": "claude-sonnet-4-6-opencode--baseline",
"caseId": "author-004-packaged-skill",
"status": "current",
"passedRuns": 2,
"validRuns": 3,
"meanDurationMs": 382777,
"measuredAt": "2026-08-17T22:03:22.192Z",
"meanEstimatedListCostUsd": 0.108879,
"meanTokenConsumption": 22653,
"meanToolInvocationCount": 16
},
{
"experimentId": "claude-sonnet-4-6-opencode--baseline",
"caseId": "author-005-conditional-approval",
"status": "current",
"passedRuns": 0,
"validRuns": 3,
"meanDurationMs": 225419,
"measuredAt": "2026-08-19T21:35:21.730Z",
"meanEstimatedListCostUsd": 0.10689799999999999,
"meanTokenConsumption": 22528.666666666668,
"meanToolInvocationCount": 24
},
{
"experimentId": "claude-sonnet-4-6-opencode--baseline",
"caseId": "author-006-custom-channel",
"status": "current",
"passedRuns": 1,
"validRuns": 3,
"meanDurationMs": 714111,
"measuredAt": "2026-08-17T22:19:44.996Z",
"meanEstimatedListCostUsd": 0.056847,
"meanTokenConsumption": 12625,
"meanToolInvocationCount": 10
},
{
"experimentId": "claude-sonnet-4-6-opencode--baseline",
"caseId": "author-007-digest-schedule",
"status": "current",
"passedRuns": 0,
"validRuns": 3,
"meanDurationMs": 232328,
"measuredAt": "2026-08-19T21:35:21.737Z",
"meanEstimatedListCostUsd": 0.09773,
"meanTokenConsumption": 16300.666666666666,
"meanToolInvocationCount": 33.333333333333336
},
{
"experimentId": "claude-sonnet-4-6-opencode--guided",
"caseId": "author-001-weather-tool",
"status": "current",
"passedRuns": 0,
"validRuns": 3,
"meanDurationMs": 186603,
"measuredAt": "2026-08-19T21:35:25.673Z",
"meanEstimatedListCostUsd": 0.07002399999999999,
"meanTokenConsumption": 13736,
"meanToolInvocationCount": 17.333333333333332
},
{
"experimentId": "claude-sonnet-4-6-opencode--guided",
"caseId": "author-002-new-project",
"status": "current",
"passedRuns": 2,
"validRuns": 3,
"meanDurationMs": 198555,
"measuredAt": "2026-08-19T21:35:25.695Z",
"meanEstimatedListCostUsd": 0.104669,
"meanTokenConsumption": 27117.666666666668,
"meanToolInvocationCount": 17.333333333333332
},
{
"experimentId": "claude-sonnet-4-6-opencode--guided",
"caseId": "author-003-openapi-connection",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 132154,
"measuredAt": "2026-08-19T21:35:25.701Z",
"meanEstimatedListCostUsd": 0.045839,
"meanTokenConsumption": 11199.666666666666,
"meanToolInvocationCount": 10.333333333333334
},
{
"experimentId": "claude-sonnet-4-6-opencode--guided",
"caseId": "author-004-packaged-skill",
"status": "current",
"passedRuns": 1,
"validRuns": 3,
"meanDurationMs": 127395,
"measuredAt": "2026-08-19T21:04:16.127Z",
"meanEstimatedListCostUsd": 0.082587,
"meanTokenConsumption": 16813,
"meanToolInvocationCount": 13
},
{
"experimentId": "claude-sonnet-4-6-opencode--guided",
"caseId": "author-005-conditional-approval",
"status": "current",
"passedRuns": 0,
"validRuns": 3,
"meanDurationMs": 170150,
"measuredAt": "2026-08-19T21:35:25.711Z",
"meanEstimatedListCostUsd": 0.031358,
"meanTokenConsumption": 5668.666666666667,
"meanToolInvocationCount": 7.666666666666667
},
{
"experimentId": "claude-sonnet-4-6-opencode--guided",
"caseId": "author-006-custom-channel",
"status": "current",
"passedRuns": 2,
"validRuns": 3,
"meanDurationMs": 130916,
"measuredAt": "2026-08-19T21:04:16.135Z",
"meanEstimatedListCostUsd": 0.0346035,
"meanTokenConsumption": 6458.5,
"meanToolInvocationCount": 11
},
{
"experimentId": "claude-sonnet-4-6-opencode--guided",
"caseId": "author-007-digest-schedule",
"status": "current",
"passedRuns": 0,
"validRuns": 3,
"meanDurationMs": 260678,
"measuredAt": "2026-08-19T21:35:25.720Z",
"meanEstimatedListCostUsd": 0.09039199999999999,
"meanTokenConsumption": 13797.333333333334,
"meanToolInvocationCount": 29
}
]
},
{
"suite": {
"eveRevision": "450681ad42ce561ba53b687fd903681d7337359e",
"caseFingerprint": "d30d6ad26abc32b54fde75c0e632d16186ed32b018e30c59baef624c0f370de9",
"caseCount": 7,
"runsPerCell": 3
},
"experiments": [
{
"id": "claude-fable-5-claude-code--baseline",
"groupId": "claude-fable-5-claude-code",
"model": "anthropic/claude-fable-5",
"modelDisplayName": "Claude Fable 5",
"harness": "Claude Code",
"treatment": "baseline"
},
{
"id": "claude-fable-5-claude-code--guided",
"groupId": "claude-fable-5-claude-code",
"model": "anthropic/claude-fable-5",
"modelDisplayName": "Claude Fable 5",
"harness": "Claude Code",
"treatment": "guided"
}
],
"results": [
{
"experimentId": "claude-fable-5-claude-code--baseline",
"caseId": "author-001-weather-tool",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 127045,
"measuredAt": "2026-08-28T22:06:23.308Z",
"meanEstimatedListCostUsd": 6.18823,
"meanToolInvocationCount": 17.666666666666668,
"meanTokenConsumption": 598539
},
{
"experimentId": "claude-fable-5-claude-code--baseline",
"caseId": "author-002-new-project",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 169780,
"measuredAt": "2026-08-28T22:06:23.319Z",
"meanEstimatedListCostUsd": 3.83675,
"meanToolInvocationCount": 8.666666666666666,
"meanTokenConsumption": 375135
},
{
"experimentId": "claude-fable-5-claude-code--baseline",
"caseId": "author-003-openapi-connection",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 258551,
"measuredAt": "2026-08-28T22:06:23.327Z",
"meanEstimatedListCostUsd": 5.43717,
"meanToolInvocationCount": 12.666666666666666,
"meanTokenConsumption": 528873
},
{
"experimentId": "claude-fable-5-claude-code--baseline",
"caseId": "author-004-packaged-skill",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 89365,
"measuredAt": "2026-08-28T22:06:23.334Z",
"meanEstimatedListCostUsd": 3.97663,
"meanToolInvocationCount": 11.666666666666666,
"meanTokenConsumption": 383479
},
{
"experimentId": "claude-fable-5-claude-code--baseline",
"caseId": "author-005-conditional-approval",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 123612,
"measuredAt": "2026-08-28T22:06:23.339Z",
"meanEstimatedListCostUsd": 5.19217,
"meanToolInvocationCount": 17.666666666666668,
"meanTokenConsumption": 507153
},
{
"experimentId": "claude-fable-5-claude-code--baseline",
"caseId": "author-006-custom-channel",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 104483,
"measuredAt": "2026-08-28T22:06:23.346Z",
"meanEstimatedListCostUsd": 7.76357,
"meanToolInvocationCount": 15.666666666666666,
"meanTokenConsumption": 761257
},
{
"experimentId": "claude-fable-5-claude-code--baseline",
"caseId": "author-007-digest-schedule",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 156463,
"measuredAt": "2026-08-28T22:06:23.352Z",
"meanEstimatedListCostUsd": 3.47642,
"meanToolInvocationCount": 23.666666666666668,
"meanTokenConsumption": 339290
},
{
"experimentId": "claude-fable-5-claude-code--guided",
"caseId": "author-001-weather-tool",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 80508,
"measuredAt": "2026-08-28T22:36:19.362Z",
"meanEstimatedListCostUsd": 3.87527,
"meanToolInvocationCount": 7.666666666666667,
"meanTokenConsumption": 375115
},
{
"experimentId": "claude-fable-5-claude-code--guided",
"caseId": "author-002-new-project",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 80206,
"measuredAt": "2026-08-28T22:36:19.376Z",
"meanEstimatedListCostUsd": 3.40311,
"meanToolInvocationCount": 9,
"meanTokenConsumption": 332259
},
{
"experimentId": "claude-fable-5-claude-code--guided",
"caseId": "author-003-openapi-connection",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 71462,
"measuredAt": "2026-08-28T22:36:19.386Z",
"meanEstimatedListCostUsd": 3.58212,
"meanToolInvocationCount": 8.666666666666666,
"meanTokenConsumption": 350688
},
{
"experimentId": "claude-fable-5-claude-code--guided",
"caseId": "author-004-packaged-skill",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 87034,
"measuredAt": "2026-08-28T22:36:19.391Z",
"meanEstimatedListCostUsd": 4.10959,
"meanToolInvocationCount": 7.666666666666667,
"meanTokenConsumption": 397447
},
{
"experimentId": "claude-fable-5-claude-code--guided",
"caseId": "author-005-conditional-approval",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 59604,
"measuredAt": "2026-08-28T22:36:19.396Z",
"meanEstimatedListCostUsd": 3.38543,
"meanToolInvocationCount": 7,
"meanTokenConsumption": 331919
},
{
"experimentId": "claude-fable-5-claude-code--guided",
"caseId": "author-006-custom-channel",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 64829,
"measuredAt": "2026-08-28T22:36:19.402Z",
"meanEstimatedListCostUsd": 3.81826,
"meanToolInvocationCount": 7,
"meanTokenConsumption": 374610
},
{
"experimentId": "claude-fable-5-claude-code--guided",
"caseId": "author-007-digest-schedule",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 71055,
"measuredAt": "2026-08-28T22:36:19.407Z",
"meanEstimatedListCostUsd": 3.07521,
"meanToolInvocationCount": 7,
"meanTokenConsumption": 299897
}
]
},
{
"suite": {
"eveRevision": "6ce61cd85ecf4621bcefd363a3e8b9445b7e6e91",
"caseFingerprint": "7d990b7e2c429ee4a2cf4f758d15f408f61d0e5b1ef9782ba69a33e435cdb169",
"caseCount": 7,
"runsPerCell": 3
},
"experiments": [
{
"id": "grok-4-6-opencode--baseline",
"groupId": "grok-4-6-opencode",
"model": "xai/grok-4.6",
"modelDisplayName": "Grok 4.6",
"harness": "OpenCode",
"treatment": "baseline"
},
{
"id": "grok-4-6-opencode--guided",
"groupId": "grok-4-6-opencode",
"model": "xai/grok-4.6",
"modelDisplayName": "Grok 4.6",
"harness": "OpenCode",
"treatment": "guided"
},
{
"id": "gpt-5-6-sol-codex--baseline",
"groupId": "gpt-5-6-sol-codex",
"model": "openai/gpt-5.6-sol",
"modelDisplayName": "GPT-5.6 Sol",
"harness": "Codex",
"treatment": "baseline"
},
{
"id": "gpt-5-6-sol-codex--guided",
"groupId": "gpt-5-6-sol-codex",
"model": "openai/gpt-5.6-sol",
"modelDisplayName": "GPT-5.6 Sol",
"harness": "Codex",
"treatment": "guided"
},
{
"id": "claude-opus-5-claude-code--baseline",
"groupId": "claude-opus-5-claude-code",
"model": "anthropic/claude-opus-5",
"modelDisplayName": "Claude Opus 5",
"harness": "Claude Code",
"treatment": "baseline"
},
{
"id": "claude-opus-5-claude-code--guided",
"groupId": "claude-opus-5-claude-code",
"model": "anthropic/claude-opus-5",
"modelDisplayName": "Claude Opus 5",
"harness": "Claude Code",
"treatment": "guided"
}
],
"results": [
{
"experimentId": "grok-4-6-opencode--baseline",
"caseId": "author-001-weather-tool",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 38613,
"measuredAt": "2026-08-28T19:22:01.000Z",
"meanEstimatedListCostUsd": 0.10434266666666665,
"meanTokenConsumption": 49566,
"meanToolInvocationCount": 19
},
{
"experimentId": "grok-4-6-opencode--baseline",
"caseId": "author-002-new-project",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 83985,
"measuredAt": "2026-08-28T19:25:12.000Z",
"meanEstimatedListCostUsd": 0.07232866666666667,
"meanTokenConsumption": 32827.666666666664,
"meanToolInvocationCount": 16
},
{
"experimentId": "grok-4-6-opencode--baseline",
"caseId": "author-003-openapi-connection",
"status": "current",
"passedRuns": 0,
"validRuns": 3,
"meanDurationMs": 521954,
"measuredAt": "2026-08-28T19:22:01.000Z",
"meanEstimatedListCostUsd": 1.0741786666666668,
"meanTokenConsumption": 503422.6666666667,
"meanToolInvocationCount": 76.66666666666667
},
{
"experimentId": "grok-4-6-opencode--baseline",
"caseId": "author-004-packaged-skill",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 41894,
"measuredAt": "2026-08-28T19:22:01.000Z",
"meanEstimatedListCostUsd": 0.065152,
"meanTokenConsumption": 29506,
"meanToolInvocationCount": 23.333333333333332
},
{
"experimentId": "grok-4-6-opencode--baseline",
"caseId": "author-005-conditional-approval",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 50151,
"measuredAt": "2026-08-28T19:22:01.000Z",
"meanEstimatedListCostUsd": 0.141536,
"meanTokenConsumption": 66629.33333333333,
"meanToolInvocationCount": 22
},
{
"experimentId": "grok-4-6-opencode--baseline",
"caseId": "author-006-custom-channel",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 37062,
"measuredAt": "2026-08-28T19:22:01.000Z",
"meanEstimatedListCostUsd": 0.07319533333333333,
"meanTokenConsumption": 33801.666666666664,
"meanToolInvocationCount": 17.666666666666668
},
{
"experimentId": "grok-4-6-opencode--baseline",
"caseId": "author-007-digest-schedule",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 38280,
"measuredAt": "2026-08-28T19:22:01.000Z",
"meanEstimatedListCostUsd": 0.045748000000000004,
"meanTokenConsumption": 20122.666666666668,
"meanToolInvocationCount": 18
},
{
"experimentId": "grok-4-6-opencode--guided",
"caseId": "author-001-weather-tool",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 31700,
"measuredAt": "2026-08-28T19:22:02.000Z",
"meanEstimatedListCostUsd": 0.049695333333333334,
"meanTokenConsumption": 22963.666666666668,
"meanToolInvocationCount": 9.666666666666666
},
{
"experimentId": "grok-4-6-opencode--guided",
"caseId": "author-002-new-project",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 83621,
"measuredAt": "2026-08-28T19:25:07.000Z",
"meanEstimatedListCostUsd": 0.082842,
"meanTokenConsumption": 38261,
"meanToolInvocationCount": 16.333333333333332
},
{
"experimentId": "grok-4-6-opencode--guided",
"caseId": "author-003-openapi-connection",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 34833,
"measuredAt": "2026-08-28T19:25:07.000Z",
"meanEstimatedListCostUsd": 0.055174,
"meanTokenConsumption": 25952.333333333332,
"meanToolInvocationCount": 10
},
{
"experimentId": "grok-4-6-opencode--guided",
"caseId": "author-004-packaged-skill",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 30135,
"measuredAt": "2026-08-28T19:22:02.000Z",
"meanEstimatedListCostUsd": 0.035581999999999996,
"meanTokenConsumption": 16043.666666666666,
"meanToolInvocationCount": 8.666666666666666
},
{
"experimentId": "grok-4-6-opencode--guided",
"caseId": "author-005-conditional-approval",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 34026,
"measuredAt": "2026-08-28T19:22:02.000Z",
"meanEstimatedListCostUsd": 0.038532666666666666,
"meanTokenConsumption": 17059.666666666668,
"meanToolInvocationCount": 11.666666666666666
},
{
"experimentId": "grok-4-6-opencode--guided",
"caseId": "author-006-custom-channel",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 32109,
"measuredAt": "2026-08-28T19:22:02.000Z",
"meanEstimatedListCostUsd": 0.033709333333333334,
"meanTokenConsumption": 14610,
"meanToolInvocationCount": 9.666666666666666
},
{
"experimentId": "grok-4-6-opencode--guided",
"caseId": "author-007-digest-schedule",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 25583,
"measuredAt": "2026-08-28T19:22:02.000Z",
"meanEstimatedListCostUsd": 0.023458666666666666,
"meanTokenConsumption": 10328,
"meanToolInvocationCount": 7.333333333333333
},
{
"experimentId": "gpt-5-6-sol-codex--baseline",
"caseId": "author-001-weather-tool",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 175042,
"measuredAt": "2026-08-31T19:20:09.055Z",
"meanEstimatedListCostUsd": 0.42096675,
"meanTokenConsumption": 49663.666666666664,
"meanToolInvocationCount": 13.666666666666666
},
{
"experimentId": "gpt-5-6-sol-codex--baseline",
"caseId": "author-002-new-project",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 122470,
"measuredAt": "2026-08-31T19:20:09.085Z",
"meanEstimatedListCostUsd": 0.23273858333333333,
"meanTokenConsumption": 30848.666666666668,
"meanToolInvocationCount": 10.333333333333334
},
{
"experimentId": "gpt-5-6-sol-codex--baseline",
"caseId": "author-003-openapi-connection",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 132036,
"measuredAt": "2026-08-31T19:20:09.104Z",
"meanEstimatedListCostUsd": 0.367498625,
"meanTokenConsumption": 48982.666666666664,
"meanToolInvocationCount": 9.333333333333334
},
{
"experimentId": "gpt-5-6-sol-codex--baseline",
"caseId": "author-004-packaged-skill",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 137447,
"measuredAt": "2026-08-31T19:20:09.128Z",
"meanEstimatedListCostUsd": 0.26755258333333337,
"meanTokenConsumption": 36330.666666666664,
"meanToolInvocationCount": 6
},
{
"experimentId": "gpt-5-6-sol-codex--baseline",
"caseId": "author-005-conditional-approval",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 121857,
"measuredAt": "2026-08-31T19:20:09.143Z",
"meanEstimatedListCostUsd": 0.2530605833333333,
"meanTokenConsumption": 32384,
"meanToolInvocationCount": 10
},
{
"experimentId": "gpt-5-6-sol-codex--baseline",
"caseId": "author-006-custom-channel",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 115653,
"measuredAt": "2026-08-31T19:20:09.156Z",
"meanEstimatedListCostUsd": 0.251469,
"meanTokenConsumption": 33854.666666666664,
"meanToolInvocationCount": 8
},
{
"experimentId": "gpt-5-6-sol-codex--baseline",
"caseId": "author-007-digest-schedule",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 121172,
"measuredAt": "2026-08-31T19:20:09.170Z",
"meanEstimatedListCostUsd": 0.284567375,
"meanTokenConsumption": 37330.666666666664,
"meanToolInvocationCount": 11
},
{
"experimentId": "gpt-5-6-sol-codex--guided",
"caseId": "author-001-weather-tool",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 115694,
"measuredAt": "2026-08-31T19:18:53.704Z",
"meanEstimatedListCostUsd": 0.18167699999999998,
"meanTokenConsumption": 24500,
"meanToolInvocationCount": 4.666666666666667
},
{
"experimentId": "gpt-5-6-sol-codex--guided",
"caseId": "author-002-new-project",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 132614,
"measuredAt": "2026-08-31T19:18:53.727Z",
"meanEstimatedListCostUsd": 0.22479154166666668,
"meanTokenConsumption": 28792.333333333332,
"meanToolInvocationCount": 9.333333333333334
},
{
"experimentId": "gpt-5-6-sol-codex--guided",
"caseId": "author-003-openapi-connection",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 101648,
"measuredAt": "2026-08-31T19:18:53.741Z",
"meanEstimatedListCostUsd": 0.14007875,
"meanTokenConsumption": 19209.666666666668,
"meanToolInvocationCount": 5.333333333333333
},
{
"experimentId": "gpt-5-6-sol-codex--guided",
"caseId": "author-004-packaged-skill",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 129791,
"measuredAt": "2026-08-31T19:18:53.751Z",
"meanEstimatedListCostUsd": 0.19388683333333334,
"meanTokenConsumption": 24916,
"meanToolInvocationCount": 6
},
{
"experimentId": "gpt-5-6-sol-codex--guided",
"caseId": "author-005-conditional-approval",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 108054,
"measuredAt": "2026-08-31T19:18:53.759Z",
"meanEstimatedListCostUsd": 0.15307379166666665,
"meanTokenConsumption": 20874.333333333332,
"meanToolInvocationCount": 4.666666666666667
},
{
"experimentId": "gpt-5-6-sol-codex--guided",
"caseId": "author-006-custom-channel",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 100410,
"measuredAt": "2026-08-31T19:18:53.769Z",
"meanEstimatedListCostUsd": 0.108055625,
"meanTokenConsumption": 15663.666666666666,
"meanToolInvocationCount": 3
},
{
"experimentId": "gpt-5-6-sol-codex--guided",
"caseId": "author-007-digest-schedule",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 103849,
"measuredAt": "2026-08-31T19:18:53.779Z",
"meanEstimatedListCostUsd": 0.11547295833333333,
"meanTokenConsumption": 16687.333333333332,
"meanToolInvocationCount": 3.6666666666666665
},
{
"experimentId": "claude-opus-5-claude-code--baseline",
"caseId": "author-001-weather-tool",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 155202,
"measuredAt": "2026-09-02T14:35:36.609Z",
"meanEstimatedListCostUsd": 1.1599598333333334,
"meanTokenConsumption": 10714.666666666666,
"meanToolInvocationCount": 17.333333333333332
},
{
"experimentId": "claude-opus-5-claude-code--baseline",
"caseId": "author-002-new-project",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 124976,
"measuredAt": "2026-09-02T14:35:36.628Z",
"meanEstimatedListCostUsd": 0.8593296666666667,
"meanTokenConsumption": 3845.3333333333335,
"meanToolInvocationCount": 10.333333333333334
},
{
"experimentId": "claude-opus-5-claude-code--baseline",
"caseId": "author-003-openapi-connection",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 160123,
"measuredAt": "2026-09-02T14:35:36.643Z",
"meanEstimatedListCostUsd": 1.3671014166666666,
"meanTokenConsumption": 7793.333333333333,
"meanToolInvocationCount": 19
},
{
"experimentId": "claude-opus-5-claude-code--baseline",
"caseId": "author-004-packaged-skill",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 181217,
"measuredAt": "2026-09-02T14:35:36.656Z",
"meanEstimatedListCostUsd": 1.1116483333333333,
"meanTokenConsumption": 16371.666666666666,
"meanToolInvocationCount": 12.666666666666666
},
{
"experimentId": "claude-opus-5-claude-code--baseline",
"caseId": "author-005-conditional-approval",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 153333,
"measuredAt": "2026-09-02T14:35:36.667Z",
"meanEstimatedListCostUsd": 1.03157675,
"meanTokenConsumption": 6217.333333333333,
"meanToolInvocationCount": 17.333333333333332
},
{
"experimentId": "claude-opus-5-claude-code--baseline",
"caseId": "author-006-custom-channel",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 136155,
"measuredAt": "2026-09-02T14:35:36.676Z",
"meanEstimatedListCostUsd": 0.9316169166666667,
"meanTokenConsumption": 4836.666666666667,
"meanToolInvocationCount": 16.666666666666668
},
{
"experimentId": "claude-opus-5-claude-code--baseline",
"caseId": "author-007-digest-schedule",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 120633,
"measuredAt": "2026-09-02T14:35:36.685Z",
"meanEstimatedListCostUsd": 0.5067333333333334,
"meanTokenConsumption": 3232.6666666666665,
"meanToolInvocationCount": 9.333333333333334
},
{
"experimentId": "claude-opus-5-claude-code--guided",
"caseId": "author-001-weather-tool",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 147978,
"measuredAt": "2026-09-02T14:34:53.900Z",
"meanEstimatedListCostUsd": 1.0020195,
"meanTokenConsumption": 11004.333333333334,
"meanToolInvocationCount": 11
},
{
"experimentId": "claude-opus-5-claude-code--guided",
"caseId": "author-002-new-project",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 138714,
"measuredAt": "2026-09-02T14:34:53.929Z",
"meanEstimatedListCostUsd": 1.0472271666666666,
"meanTokenConsumption": 8633.666666666666,
"meanToolInvocationCount": 14.333333333333334
},
{
"experimentId": "claude-opus-5-claude-code--guided",
"caseId": "author-003-openapi-connection",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 137520,
"measuredAt": "2026-09-02T14:34:53.944Z",
"meanEstimatedListCostUsd": 0.7869943333333334,
"meanTokenConsumption": 5193.666666666667,
"meanToolInvocationCount": 14
},
{
"experimentId": "claude-opus-5-claude-code--guided",
"caseId": "author-004-packaged-skill",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 153316,
"measuredAt": "2026-09-02T14:34:53.956Z",
"meanEstimatedListCostUsd": 0.6677339166666667,
"meanTokenConsumption": 10291.666666666666,
"meanToolInvocationCount": 7.666666666666667
},
{
"experimentId": "claude-opus-5-claude-code--guided",
"caseId": "author-005-conditional-approval",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 116393,
"measuredAt": "2026-09-02T14:34:53.964Z",
"meanEstimatedListCostUsd": 0.5910709166666667,
"meanTokenConsumption": 2916.6666666666665,
"meanToolInvocationCount": 8.333333333333334
},
{
"experimentId": "claude-opus-5-claude-code--guided",
"caseId": "author-006-custom-channel",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 107570,
"measuredAt": "2026-09-02T14:34:53.973Z",
"meanEstimatedListCostUsd": 0.4942074166666666,
"meanTokenConsumption": 2636,
"meanToolInvocationCount": 6.666666666666667
},
{
"experimentId": "claude-opus-5-claude-code--guided",
"caseId": "author-007-digest-schedule",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 114311,
"measuredAt": "2026-09-02T14:34:53.981Z",
"meanEstimatedListCostUsd": 0.46051358333333337,
"meanTokenConsumption": 2592.6666666666665,
"meanToolInvocationCount": 7.666666666666667
}
]
},
{
"suite": {
"eveRevision": "279e2b6a81198f6008004f596474cf05140ca7bc",
"caseFingerprint": "336fa6e2645a717a4bf4d906c3ae1b9f4ce0839bfb7bc9065068490dd696d42e",
"caseCount": 7,
"runsPerCell": 3
},
"experiments": [
{
"id": "glm-5-2-opencode--baseline",
"groupId": "glm-5-2-opencode",
"model": "zai/glm-5.2",
"modelDisplayName": "GLM 5.2",
"harness": "OpenCode",
"treatment": "baseline"
},
{
"id": "glm-5-2-opencode--guided",
"groupId": "glm-5-2-opencode",
"model": "zai/glm-5.2",
"modelDisplayName": "GLM 5.2",
"harness": "OpenCode",
"treatment": "guided"
}
],
"results": [
{
"experimentId": "glm-5-2-opencode--baseline",
"caseId": "author-001-weather-tool",
"status": "current",
"passedRuns": 0,
"validRuns": 3,
"meanDurationMs": 183955,
"measuredAt": "2026-10-07T20:53:18.454Z",
"meanEstimatedListCostUsd": 0.08985164,
"meanTokenConsumption": 33383.666666666664,
"meanToolInvocationCount": 23.666666666666668
},
{
"experimentId": "glm-5-2-opencode--baseline",
"caseId": "author-002-new-project",
"status": "current",
"passedRuns": 2,
"validRuns": 3,
"meanDurationMs": 146900,
"measuredAt": "2026-10-07T20:53:18.471Z",
"meanEstimatedListCostUsd": 0.06131117333333334,
"meanTokenConsumption": 26307.666666666668,
"meanToolInvocationCount": 16.666666666666668
},
{
"experimentId": "glm-5-2-opencode--baseline",
"caseId": "author-003-openapi-connection",
"status": "current",
"passedRuns": 2,
"validRuns": 3,
"meanDurationMs": 263531,
"measuredAt": "2026-10-07T20:53:18.486Z",
"meanEstimatedListCostUsd": 0.24344989333333333,
"meanTokenConsumption": 59370.666666666664,
"meanToolInvocationCount": 41.333333333333336
},
{
"experimentId": "glm-5-2-opencode--baseline",
"caseId": "author-004-packaged-skill",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 215844,
"measuredAt": "2026-10-07T20:53:18.509Z",
"meanEstimatedListCostUsd": 0.10365373333333334,
"meanTokenConsumption": 34354,
"meanToolInvocationCount": 27
},
{
"experimentId": "glm-5-2-opencode--baseline",
"caseId": "author-005-conditional-approval",
"status": "current",
"passedRuns": 1,
"validRuns": 3,
"meanDurationMs": 203192,
"measuredAt": "2026-10-07T20:53:18.528Z",
"meanEstimatedListCostUsd": 0.19839237333333334,
"meanTokenConsumption": 75866.66666666667,
"meanToolInvocationCount": 32.666666666666664
},
{
"experimentId": "glm-5-2-opencode--baseline",
"caseId": "author-006-custom-channel",
"status": "current",
"passedRuns": 2,
"validRuns": 3,
"meanDurationMs": 240346,
"measuredAt": "2026-10-07T20:53:18.555Z",
"meanEstimatedListCostUsd": 0.3499838533333333,
"meanTokenConsumption": 96427,
"meanToolInvocationCount": 45.333333333333336
},
{
"experimentId": "glm-5-2-opencode--baseline",
"caseId": "author-007-digest-schedule",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 155457,
"measuredAt": "2026-10-07T20:53:18.586Z",
"meanEstimatedListCostUsd": 0.07361798666666665,
"meanTokenConsumption": 34515.333333333336,
"meanToolInvocationCount": 18.666666666666668
},
{
"experimentId": "glm-5-2-opencode--guided",
"caseId": "author-001-weather-tool",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 211084,
"measuredAt": "2026-10-07T20:58:05.582Z",
"meanEstimatedListCostUsd": 0.12096889333333334,
"meanTokenConsumption": 30802.666666666668,
"meanToolInvocationCount": 21.666666666666668
},
{
"experimentId": "glm-5-2-opencode--guided",
"caseId": "author-002-new-project",
"status": "current",
"passedRuns": 1,
"validRuns": 3,
"meanDurationMs": 146105,
"measuredAt": "2026-10-07T20:58:05.598Z",
"meanEstimatedListCostUsd": 0.045339653333333334,
"meanTokenConsumption": 14588.666666666666,
"meanToolInvocationCount": 16.333333333333332
},
{
"experimentId": "glm-5-2-opencode--guided",
"caseId": "author-003-openapi-connection",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 159353,
"measuredAt": "2026-10-07T20:58:05.607Z",
"meanEstimatedListCostUsd": 0.07640521333333335,
"meanTokenConsumption": 20298.666666666668,
"meanToolInvocationCount": 19.333333333333332
},
{
"experimentId": "glm-5-2-opencode--guided",
"caseId": "author-004-packaged-skill",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 190722,
"measuredAt": "2026-10-07T20:58:05.620Z",
"meanEstimatedListCostUsd": 0.06202376000000001,
"meanTokenConsumption": 15365.666666666666,
"meanToolInvocationCount": 17.666666666666668
},
{
"experimentId": "glm-5-2-opencode--guided",
"caseId": "author-005-conditional-approval",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 155636,
"measuredAt": "2026-10-07T20:58:05.635Z",
"meanEstimatedListCostUsd": 0.049076973333333336,
"meanTokenConsumption": 18203,
"meanToolInvocationCount": 14.666666666666666
},
{
"experimentId": "glm-5-2-opencode--guided",
"caseId": "author-006-custom-channel",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 168719,
"measuredAt": "2026-10-07T20:58:05.650Z",
"meanEstimatedListCostUsd": 0.07198956000000001,
"meanTokenConsumption": 19972.666666666668,
"meanToolInvocationCount": 19
},
{
"experimentId": "glm-5-2-opencode--guided",
"caseId": "author-007-digest-schedule",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
"meanDurationMs": 147630,
"measuredAt": "2026-10-07T20:58:05.665Z",
"meanEstimatedListCostUsd": 0.04161758666666667,
"meanTokenConsumption": 10529.333333333334,
"meanToolInvocationCount": 13.333333333333334
}
]
}
]
}