The Open Framework for Building Agents
chore(docs): update benchmarks Opus 5.5, GPT-6, Grok 4.7 (#4514)
Colton Paddenb6c8cd65c090
diff --git a/apps/benchmarks/README.md b/apps/benchmarks/README.mdindex 2c2d6c89f..740866515 100644--- a/apps/benchmarks/README.md+++ b/apps/benchmarks/README.md@@ -32,6 +32,8 @@ are not supported by the native runner.The runner uses agent-eval's native Gateway harnesses: OpenCode for other providers, Claude Codefor Anthropic models, and Codex for OpenAI models. Each attempt starts an isolated Vercel Sandbox,then scaffolds the selected immutable canary with `npx` before the coding agent starts.+The starting workspace includes pinned `[email protected]` for the local default sandbox provider.+Scaffolded workspaces must pass typecheck and build before any model inference starts.Local runs use the `guided` treatment by default, which keeps the `AGENTS.md` and aliases generatedby `eve init`. Pass `--treatment baseline` to remove those files before the coding agent starts.@@ -51,7 +53,7 @@ working tree and defaults to `origin/main`:pnpm benchmark:publish --drypnpm benchmark:publishpnpm benchmark:publish --revision <commit>-pnpm benchmark:publish --models kimi-k3,gpt-5-6-sol+pnpm benchmark:publish --models kimi-k3,gpt-6-sol-high```Pass `--allow-dirty` only for a local, noncanonical run. It bypasses the clean-tree check, so itsdiff --git a/apps/benchmarks/lib/authoring-case.ts b/apps/benchmarks/lib/authoring-case.tsindex 44e29bbc3..af0a4ead4 100644--- a/apps/benchmarks/lib/authoring-case.ts+++ b/apps/benchmarks/lib/authoring-case.ts@@ -39,12 +39,12 @@ export interface AuthoringCase {}export const emptyProject: AuthoringStartingPoint = {- id: "empty-v2",+ id: "empty-v3",workspace: "empty",};export const simpleProject: AuthoringStartingPoint = {- id: "simple-v2",+ id: "simple-v3",workspace: "scaffolded",};diff --git a/apps/benchmarks/lib/benchmark-config.test.mjs b/apps/benchmarks/lib/benchmark-config.test.mjsindex 304a2aa95..ff5641898 100644--- a/apps/benchmarks/lib/benchmark-config.test.mjs+++ b/apps/benchmarks/lib/benchmark-config.test.mjs@@ -1,69 +1,32 @@import assert from "node:assert/strict";import { test } from "node:test";-import {- findBenchmarkModel,- findPublishedBenchmarkModel,- harnessId,- publishedBenchmark,- publishedBenchmarkModels,-} from "./benchmark-config.ts";+import { findBenchmarkModel, findPublishedBenchmarkModel, harnessId } from "./benchmark-config.ts";-test("publishes only compatibility-validated models", () => {- assert.deepEqual(- publishedBenchmarkModels.map((benchmark) => benchmark.id),- [- "kimi-k3",- "claude-fable-5-1",- "grok-4-6",- "gpt-5-6-sol",- "gpt-5-6-terra",- "gpt-6-astra-high",- "claude-sonnet-5",- "glm-5-3",- "claude-opus-5",- "gemini-3-1-pro-preview",- "gemini-3-8-flash",- "minimax-m3",- "kimi-k2-7-code",- ],- );- assert.deepEqual(publishedBenchmark.caseIds, [- "author-001-weather-tool",- "author-002-new-project",- "author-003-openapi-connection",- "author-004-packaged-skill",- "author-005-conditional-approval",- "author-006-custom-channel",- "author-007-digest-schedule",- ]);- assert.equal(findPublishedBenchmarkModel("grok-4-6").model, "xai/grok-4.6");- assert.equal(findPublishedBenchmarkModel("claude-sonnet-5").model, "anthropic/claude-sonnet-5");+test("selects qualified Gateway IDs and explicit reasoning settings for publication", () => {+ assert.equal(findPublishedBenchmarkModel("grok-4-7").model, "spacexai/grok-4.7");assert.equal(findPublishedBenchmarkModel("glm-5-3").model, "zai/glm-5.3");- assert.equal(- findPublishedBenchmarkModel("gpt-6-astra-high").agentModel,- "openai/gpt-6-astra?reasoningEffort=high",- );- assert.equal(findPublishedBenchmarkModel("claude-fable-5-1").harness, "Claude Code");- assert.equal(findPublishedBenchmarkModel("claude-sonnet-5").harness, "Claude Code");- assert.equal(findPublishedBenchmarkModel("claude-opus-5").harness, "Claude Code");- assert.equal(findPublishedBenchmarkModel("kimi-k3").harness, "OpenCode");- assert.equal(findPublishedBenchmarkModel("gpt-5-6-sol").harness, "Codex");- assert.equal(findPublishedBenchmarkModel("gpt-5-6-terra").harness, "Codex");- assert.equal(harnessId("Codex"), "codex");+ for (const [id, model] of [+ ["gpt-6-sol-high", "openai/gpt-6-sol"],+ ["gpt-6-luna-high", "openai/gpt-6-luna"],+ ["gpt-6-astra-high", "openai/gpt-6-astra"],+ ]) {+ const benchmark = findPublishedBenchmarkModel(id);+ assert.equal(benchmark.agentModel, `${model}?reasoningEffort=high`);+ assert.equal(harnessId(benchmark.harness), "codex");+ }+ for (const id of ["claude-opus-5-5-high", "claude-fable-5-1"]) {+ const benchmark = findPublishedBenchmarkModel(id);+ assert.equal(benchmark.agentOptions.effort, "high");+ assert.equal(harnessId(benchmark.harness), "claude-code");+ }});-test("allows candidate probes and rejects unknown models", () => {- assert.equal(findBenchmarkModel("gpt-5-6-sol").support, "supported");- assert.equal(findBenchmarkModel("gpt-5-6-terra").support, "supported");- assert.equal(findBenchmarkModel("gpt-6-astra-high").support, "supported");- assert.equal(findBenchmarkModel("gemini-3-8-flash").support, "supported");- assert.equal(findBenchmarkModel("minimax-m3").support, "supported");- assert.equal(findBenchmarkModel("kimi-k2-7-code").support, "supported");- assert.equal(findBenchmarkModel("grok-4-6").support, "supported");- assert.equal(findBenchmarkModel("claude-sonnet-4-6").support, "superseded");- assert.equal(findBenchmarkModel("claude-fable-5").support, "superseded");+test("allows local retired probes without admitting them to publication", () => {+ assert.equal(findBenchmarkModel("grok-4-6").support, "superseded");assert.equal(findBenchmarkModel("glm-5-2").support, "superseded");- assert.equal(findBenchmarkModel("claude-opus-5").support, "supported");+ for (const id of ["grok-4-6", "glm-5-2"]) {+ assert.throws(() => findPublishedBenchmarkModel(id), /not in the supported publication set/u);+ }assert.throws(() => findBenchmarkModel("unknown"), /Unknown model/u);});diff --git a/apps/benchmarks/lib/benchmark-config.ts b/apps/benchmarks/lib/benchmark-config.tsindex 4238fa85f..73b101fc9 100644--- a/apps/benchmarks/lib/benchmark-config.ts+++ b/apps/benchmarks/lib/benchmark-config.ts@@ -11,6 +11,7 @@ export interface AuthoringBenchmarkModel {readonly agentModel?: string;readonly displayName: string;readonly harness: "OpenCode" | "Claude Code" | "Codex";+ readonly agentOptions?: Readonly<Record<string, unknown>>;readonly support: AuthoringBenchmarkSupport;}@@ -39,8 +40,9 @@ export const benchmarkModels = [{id: "claude-fable-5-1",model: "anthropic/claude-fable-5.1",- displayName: "Claude Fable 5.1",+ displayName: "Claude Fable 5.1 (high)",harness: "Claude Code",+ agentOptions: { effort: "high" },support: "supported",},{@@ -48,14 +50,14 @@ export const benchmarkModels = [model: "xai/grok-4.6",displayName: "Grok 4.6",harness: "OpenCode",- support: "supported",+ support: "superseded",},{id: "gpt-5-6-sol",model: "openai/gpt-5.6-sol",displayName: "GPT-5.6 Sol",harness: "Codex",- support: "supported",+ support: "superseded",},{id: "gpt-5-6-terra",@@ -98,7 +100,7 @@ export const benchmarkModels = [model: "anthropic/claude-opus-5",displayName: "Claude Opus 5",harness: "Claude Code",- support: "supported",+ support: "superseded",},{id: "gemini-3-1-pro-preview",@@ -128,6 +130,57 @@ export const benchmarkModels = [harness: "OpenCode",support: "supported",},+ {+ id: "claude-opus-5-5-high",+ model: "anthropic/claude-opus-5.5",+ displayName: "Claude Opus 5.5 (high)",+ harness: "Claude Code",+ agentOptions: { effort: "high" },+ support: "supported",+ },+ {+ id: "gpt-6-sol-high",+ model: "openai/gpt-6-sol",+ agentModel: "openai/gpt-6-sol?reasoningEffort=high",+ displayName: "GPT-6 Sol (high)",+ harness: "Codex",+ support: "supported",+ },+ {+ id: "gpt-6-luna-high",+ model: "openai/gpt-6-luna",+ agentModel: "openai/gpt-6-luna?reasoningEffort=high",+ displayName: "GPT-6 Luna (high)",+ harness: "Codex",+ support: "supported",+ },+ {+ id: "grok-4-7",+ model: "spacexai/grok-4.7",+ displayName: "Grok 4.7",+ harness: "OpenCode",+ agentOptions: {+ binaryUrl:+ "https://ymdea60kblwwhidh.public.blob.vercel-storage.com/opencode-linux-x64-RCtfS54uaTwa5i9C3bpzguLy3jEBP6",+ extraProviders: {+ vercel: {+ models: {+ "spacexai/grok-4.7": {+ name: "Grok 4.7",+ reasoning: true,+ tool_call: true,+ temperature: true,+ attachment: true,+ modalities: { input: ["text", "image"], output: ["text"] },+ limit: { context: 500000, output: 131072 },+ cost: { input: 0, output: 0, cache_read: 0 },+ },+ },+ },+ },+ },+ support: "supported",+ },] as const satisfies ReadonlyArray<AuthoringBenchmarkModel>;export const publishedBenchmarkModels = benchmarkModels.filter(diff --git a/apps/benchmarks/lib/experiment.test.mjs b/apps/benchmarks/lib/experiment.test.mjsindex eea192eee..76092ef2a 100644--- a/apps/benchmarks/lib/experiment.test.mjs+++ b/apps/benchmarks/lib/experiment.test.mjs@@ -56,7 +56,7 @@ test("passes an explicit native model override to Codex", () => {assert.equal(config.model, "openai/gpt-6-astra?reasoningEffort=high");});-test("strips the Gateway provider prefix for native Claude Code", () => {+test("strips the Gateway prefix and preserves explicit Claude Code effort", () => {const config = authoringExperiment({...common,benchmark: {@@ -64,10 +64,13 @@ test("strips the Gateway provider prefix for native Claude Code", () => {model: "anthropic/claude-sonnet-5",displayName: "Claude Sonnet 5",harness: "Claude Code",+ agentOptions: { effort: "high" },support: "supported",},});assert.equal(config.agent, "vercel-ai-gateway/claude-code");assert.equal(config.model, "claude-sonnet-5");+ assert.equal(config.agentOptions.effort, "high");+ assert.equal(config.agentOptions.verbose, false);});diff --git a/apps/benchmarks/lib/experiment.ts b/apps/benchmarks/lib/experiment.tsindex b12b016a5..9ab37717f 100644--- a/apps/benchmarks/lib/experiment.ts+++ b/apps/benchmarks/lib/experiment.ts@@ -36,7 +36,14 @@ export function authoringExperiment(options: {revision: options.revision,treatment: options.treatment,}),- agentOptions: { verbose: options.verbose ?? false },+ agentOptions: {+ cliPackage:+ options.benchmark.harness === "Claude Code"+ ? "@anthropic-ai/[email protected]"+ : undefined,+ ...options.benchmark.agentOptions,+ verbose: options.verbose ?? false,+ },};}diff --git a/apps/benchmarks/lib/native-authoring-setup.test.mjs b/apps/benchmarks/lib/native-authoring-setup.test.mjsindex f0ce0a6ad..3ad90526a 100644--- a/apps/benchmarks/lib/native-authoring-setup.test.mjs+++ b/apps/benchmarks/lib/native-authoring-setup.test.mjs@@ -44,9 +44,37 @@ test("bootstraps the selected source before a native agent starts", async () =>commands[2].args[1],/AI_AGENT=claude EVE_INIT_PACKAGE_SPEC=.* eve init \. --model openai\/gpt-5\.5/u,);- assert.match(commands[3].args[1], /mkdir -p agent\/lib/u);- assert.match(commands[4].args[1], /rm -f AGENTS\.md CLAUDE\.md GEMINI\.md/u);- assert.match(commands[5].args[1], /git add \. && git commit --amend --no-edit --quiet/u);+ assert.match(commands[3].args[1], /npm install --save-dev --save-exact just-bash@3\.1\.0/u);+ assert.match(commands[4].args[1], /mkdir -p agent\/lib/u);+ assert.match(commands[5].args[1], /rm -f AGENTS\.md CLAUDE\.md GEMINI\.md/u);+ assert.match(commands[6].args[1], /npm run typecheck && npm run build/u);+ assert.match(commands[7].args[1], /git add \. && git commit --amend --no-edit --quiet/u);assert.deepEqual(Object.keys(writes[1]), ["/workspace/agent/lib/inventory-openapi.ts"]);assert.match(writes[1]["/workspace/agent/lib/inventory-openapi.ts"], /getStock/u);});++test("rejects a broken starting workspace before handing it to the model", async () => {+ const setup = createNativeAuthoringSetup({+ packageSpec: "https://pkg.eve.dev/commit/eve.tgz",+ revision: "commit",+ treatment: "guided",+ });+ await assert.rejects(+ setup({+ async readFile() {+ return JSON.stringify({ startingPoint: "scaffolded", revision: "commit", setupIds: [] });+ },+ async writeFiles() {},+ async runCommand(_command, args) {+ const failed = args[1].includes("npm run typecheck && npm run build");+ return {+ stdout: "",+ stderr: failed ? "missing optional package" : "",+ exitCode: failed ? 1 : 0,+ };+ },+ getWorkingDirectory: () => "/workspace",+ }),+ /starting workspace validation failed.*missing optional package/su,+ );+});diff --git a/apps/benchmarks/lib/native-authoring-setup.ts b/apps/benchmarks/lib/native-authoring-setup.tsindex 8d9475824..95f69dd94 100644--- a/apps/benchmarks/lib/native-authoring-setup.ts+++ b/apps/benchmarks/lib/native-authoring-setup.ts@@ -55,9 +55,20 @@ export function createNativeAuthoringSetup(options: {});}+ // DefaultSandbox selects just-bash inside the Linux eval container, where Docker+ // and microsandbox are unavailable. Supply its optional peer before the model runs.+ await run(+ sandbox,+ "npm install --save-dev --save-exact [email protected]",+ "local sandbox dependency",+ );+const context = setupContext(sandbox, bootstrap.projectDirectory);for (const setup of setupsFor(bootstrap.setupIds)) await setup.onSession?.(context);if (options.treatment === "baseline") await run(sandbox, "rm -f AGENTS.md CLAUDE.md GEMINI.md");+ if (bootstrap.startingPoint === "scaffolded") {+ await run(sandbox, "npm run typecheck && npm run build", "starting workspace validation");+ }await run(sandbox, "git add . && git commit --amend --no-edit --quiet");};}diff --git a/apps/benchmarks/scripts/cost.mjs b/apps/benchmarks/scripts/cost.mjsindex 0e144f591..784917c4f 100644--- a/apps/benchmarks/scripts/cost.mjs+++ b/apps/benchmarks/scripts/cost.mjs@@ -33,6 +33,38 @@ export const modelPricing = {cacheRead: 0.25,cacheWrite: 12.5,},+ "anthropic/claude-opus-5.5": {+ effectiveDate: "2026-10-07",+ source: "https://ai-gateway.vercel.sh/v1/models",+ input: 4,+ output: 20,+ cacheRead: 0.2,+ cacheWrite: 5,+ },+ "openai/gpt-6-sol": {+ effectiveDate: "2026-10-07",+ source: "https://ai-gateway.vercel.sh/v1/models",+ input: 2,+ output: 10,+ cacheRead: 0.2,+ cacheWrite: 2.5,+ },+ "openai/gpt-6-luna": {+ effectiveDate: "2026-10-07",+ source: "https://ai-gateway.vercel.sh/v1/models",+ input: 0.1,+ output: 0.5,+ cacheRead: 0.01,+ cacheWrite: 0.125,+ },+ "spacexai/grok-4.7": {+ effectiveDate: "2026-10-07",+ source: "https://ai-gateway.vercel.sh/v1/models",+ input: 2,+ output: 6,+ cacheRead: 0.5,+ cacheWrite: 0,+ },"xai/grok-4.6": {effectiveDate: "2026-08-17",source: "https://docs.x.ai/developers/models",@@ -81,12 +113,13 @@ export const modelPricing = {cacheRead: 0.2,cacheWrite: 2.5,},+ // Use the Gateway US endpoint's list rates, not the catalog's cheaper global route."zai/glm-5.2": {- effectiveDate: "2026-08-17",- source: "https://docs.z.ai/guides/overview/pricing",+ effectiveDate: "2026-10-07",+ source: "https://ai-gateway.vercel.sh/v1/models",input: 1.4,output: 4.4,- cacheRead: 0.26,+ cacheRead: 0.14,cacheWrite: 0,},"zai/glm-5.3": {@@ -122,11 +155,11 @@ export const modelPricing = {cacheWrite: 0,},"moonshotai/kimi-k2.7-code": {- effectiveDate: "2026-09-10",+ effectiveDate: "2026-10-07",source: "https://ai-gateway.vercel.sh/v1/models",input: 0.95,output: 4,- cacheRead: 0.16,+ cacheRead: 0.19,cacheWrite: 0,},};diff --git a/apps/benchmarks/scripts/export-results.test.mjs b/apps/benchmarks/scripts/export-results.test.mjsindex f7e66ef4e..e25ea7e65 100644--- a/apps/benchmarks/scripts/export-results.test.mjs+++ b/apps/benchmarks/scripts/export-results.test.mjs@@ -40,26 +40,8 @@ test("exports missing cells without publishing private artifacts", () => {assert.equal(output.schemaVersion, 1);assert.equal(output.suite.caseCount, 7);assert.match(output.suite.caseFingerprint, /^[0-9a-f]{64}$/u);- assert.equal(output.experiments.length, 26);- assert.deepEqual(- [...new Set(output.experiments.map((experiment) => experiment.modelDisplayName))],- [- "Kimi K3",- "Claude Fable 5.1",- "Grok 4.6",- "GPT-5.6 Sol",- "GPT-5.6 Terra",- "GPT-6 Astra (high)",- "Claude Sonnet 5",- "GLM 5.3",- "Claude Opus 5",- "Gemini 3.1 Pro Preview",- "Gemini 3.8 Flash",- "MiniMax M3",- "Kimi K2.7 Code",- ],- );- assert.equal(output.results.length, 182);+ assert.equal(output.experiments.length, 28);+ assert.equal(output.results.length, 196);assert.ok(output.results.every((result) => result.status === "missing"));for (const privateField of ["transcript", "commands", "worldEvents", "files"]) {assert.equal(raw.includes(`"${privateField}"`), false);diff --git a/apps/docs/lib/evals/benchmark-results.json b/apps/docs/lib/evals/benchmark-results.jsonindex ac205506e..50108a437 100644--- a/apps/docs/lib/evals/benchmark-results.json+++ b/apps/docs/lib/evals/benchmark-results.json@@ -1,9 +1,9 @@{"schemaVersion": 1,- "generatedAt": "2026-09-15T16:54:39.105Z",+ "generatedAt": "2026-10-08T14:33:57.335Z","suite": {- "eveRevision": "6ce61cd85ecf4621bcefd363a3e8b9445b7e6e91",- "caseFingerprint": "7d990b7e2c429ee4a2cf4f758d15f408f61d0e5b1ef9782ba69a33e435cdb169",+ "eveRevision": "279e2b6a81198f6008004f596474cf05140ca7bc",+ "caseFingerprint": "336fa6e2645a717a4bf4d906c3ae1b9f4ce0839bfb7bc9065068490dd696d42e","caseCount": 7,"runsPerCell": 3},@@ -28,7 +28,7 @@"id": "claude-fable-5-1-claude-code--baseline","groupId": "claude-fable-5-1-claude-code","model": "anthropic/claude-fable-5.1",- "modelDisplayName": "Claude Fable 5.1",+ "modelDisplayName": "Claude Fable 5.1 (high)","harness": "Claude Code","treatment": "baseline"},@@ -36,42 +36,10 @@"id": "claude-fable-5-1-claude-code--guided","groupId": "claude-fable-5-1-claude-code","model": "anthropic/claude-fable-5.1",- "modelDisplayName": "Claude Fable 5.1",+ "modelDisplayName": "Claude Fable 5.1 (high)","harness": "Claude Code","treatment": "guided"},- {- "id": "grok-4-6-opencode--baseline",- "groupId": "grok-4-6-opencode",- "model": "xai/grok-4.6",- "modelDisplayName": "Grok 4.6",- "harness": "OpenCode",- "treatment": "baseline"- },- {- "id": "grok-4-6-opencode--guided",- "groupId": "grok-4-6-opencode",- "model": "xai/grok-4.6",- "modelDisplayName": "Grok 4.6",- "harness": "OpenCode",- "treatment": "guided"- },- {- "id": "gpt-5-6-sol-codex--baseline",- "groupId": "gpt-5-6-sol-codex",- "model": "openai/gpt-5.6-sol",- "modelDisplayName": "GPT-5.6 Sol",- "harness": "Codex",- "treatment": "baseline"- },- {- "id": "gpt-5-6-sol-codex--guided",- "groupId": "gpt-5-6-sol-codex",- "model": "openai/gpt-5.6-sol",- "modelDisplayName": "GPT-5.6 Sol",- "harness": "Codex",- "treatment": "guided"- },{"id": "gpt-5-6-terra-codex--baseline","groupId": "gpt-5-6-terra-codex",@@ -136,22 +104,6 @@"harness": "OpenCode","treatment": "guided"},- {- "id": "claude-opus-5-claude-code--baseline",- "groupId": "claude-opus-5-claude-code",- "model": "anthropic/claude-opus-5",- "modelDisplayName": "Claude Opus 5",- "harness": "Claude Code",- "treatment": "baseline"- },- {- "id": "claude-opus-5-claude-code--guided",- "groupId": "claude-opus-5-claude-code",- "model": "anthropic/claude-opus-5",- "modelDisplayName": "Claude Opus 5",- "harness": "Claude Code",- "treatment": "guided"- },{"id": "gemini-3-1-pro-preview-opencode--baseline","groupId": "gemini-3-1-pro-preview-opencode",@@ -215,6 +167,70 @@"modelDisplayName": "Kimi K2.7 Code","harness": "OpenCode","treatment": "guided"+ },+ {+ "id": "claude-opus-5-5-high-claude-code--baseline",+ "groupId": "claude-opus-5-5-high-claude-code",+ "model": "anthropic/claude-opus-5.5",+ "modelDisplayName": "Claude Opus 5.5 (high)",+ "harness": "Claude Code",+ "treatment": "baseline"+ },+ {+ "id": "claude-opus-5-5-high-claude-code--guided",+ "groupId": "claude-opus-5-5-high-claude-code",+ "model": "anthropic/claude-opus-5.5",+ "modelDisplayName": "Claude Opus 5.5 (high)",+ "harness": "Claude Code",+ "treatment": "guided"+ },+ {+ "id": "gpt-6-sol-high-codex--baseline",+ "groupId": "gpt-6-sol-high-codex",+ "model": "openai/gpt-6-sol",+ "modelDisplayName": "GPT-6 Sol (high)",+ "harness": "Codex",+ "treatment": "baseline"+ },+ {+ "id": "gpt-6-sol-high-codex--guided",+ "groupId": "gpt-6-sol-high-codex",+ "model": "openai/gpt-6-sol",+ "modelDisplayName": "GPT-6 Sol (high)",+ "harness": "Codex",+ "treatment": "guided"+ },+ {+ "id": "gpt-6-luna-high-codex--baseline",+ "groupId": "gpt-6-luna-high-codex",+ "model": "openai/gpt-6-luna",+ "modelDisplayName": "GPT-6 Luna (high)",+ "harness": "Codex",+ "treatment": "baseline"+ },+ {+ "id": "gpt-6-luna-high-codex--guided",+ "groupId": "gpt-6-luna-high-codex",+ "model": "openai/gpt-6-luna",+ "modelDisplayName": "GPT-6 Luna (high)",+ "harness": "Codex",+ "treatment": "guided"+ },+ {+ "id": "grok-4-7-opencode--baseline",+ "groupId": "grok-4-7-opencode",+ "model": "spacexai/grok-4.7",+ "modelDisplayName": "Grok 4.7",+ "harness": "OpenCode",+ "treatment": "baseline"+ },+ {+ "id": "grok-4-7-opencode--guided",+ "groupId": "grok-4-7-opencode",+ "model": "spacexai/grok-4.7",+ "modelDisplayName": "Grok 4.7",+ "harness": "OpenCode",+ "treatment": "guided"}],"results": [@@ -224,11 +240,11 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 124258,- "measuredAt": "2026-08-28T19:22:06.000Z",- "meanEstimatedListCostUsd": 0.14751699999999998,- "meanTokenConsumption": 31831,- "meanToolInvocationCount": 25+ "meanDurationMs": 182488,+ "measuredAt": "2026-10-07T17:53:49.672Z",+ "meanEstimatedListCostUsd": 0.3365506,+ "meanTokenConsumption": 39811,+ "meanToolInvocationCount": 27.666666666666668},{"experimentId": "kimi-k3-opencode--baseline",@@ -236,23 +252,23 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 89304,- "measuredAt": "2026-08-28T19:25:16.000Z",- "meanEstimatedListCostUsd": 0.085075,- "meanTokenConsumption": 21470.333333333332,- "meanToolInvocationCount": 12+ "meanDurationMs": 118940,+ "measuredAt": "2026-10-07T17:53:49.687Z",+ "meanEstimatedListCostUsd": 0.0926576,+ "meanTokenConsumption": 15709.333333333334,+ "meanToolInvocationCount": 11},{"experimentId": "kimi-k3-opencode--baseline","caseId": "author-003-openapi-connection","status": "current",- "passedRuns": 0,+ "passedRuns": 3,"validRuns": 3,- "meanDurationMs": 454935,- "measuredAt": "2026-08-28T19:22:06.000Z",- "meanEstimatedListCostUsd": 0.3622910000000001,- "meanTokenConsumption": 56591.666666666664,- "meanToolInvocationCount": 28+ "meanDurationMs": 173791,+ "measuredAt": "2026-10-07T17:53:49.695Z",+ "meanEstimatedListCostUsd": 0.2977262,+ "meanTokenConsumption": 39151.666666666664,+ "meanToolInvocationCount": 27.333333333333332},{"experimentId": "kimi-k3-opencode--baseline",@@ -260,11 +276,11 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 101585,- "measuredAt": "2026-08-28T19:22:06.000Z",- "meanEstimatedListCostUsd": 0.09421099999999999,- "meanTokenConsumption": 19086.333333333332,- "meanToolInvocationCount": 15.666666666666666+ "meanDurationMs": 154644,+ "measuredAt": "2026-10-07T17:53:49.707Z",+ "meanEstimatedListCostUsd": 0.142479,+ "meanTokenConsumption": 18281,+ "meanToolInvocationCount": 19.333333333333332},{"experimentId": "kimi-k3-opencode--baseline",@@ -272,11 +288,11 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 98137,- "measuredAt": "2026-08-28T19:22:06.000Z",- "meanEstimatedListCostUsd": 0.097954,- "meanTokenConsumption": 23523.333333333332,- "meanToolInvocationCount": 20+ "meanDurationMs": 182203,+ "measuredAt": "2026-10-07T17:53:49.717Z",+ "meanEstimatedListCostUsd": 0.3224834,+ "meanTokenConsumption": 40129.666666666664,+ "meanToolInvocationCount": 28},{"experimentId": "kimi-k3-opencode--baseline",@@ -284,11 +300,11 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 78637,- "measuredAt": "2026-08-28T19:22:06.000Z",- "meanEstimatedListCostUsd": 0.107612,- "meanTokenConsumption": 26372,- "meanToolInvocationCount": 19.333333333333332+ "meanDurationMs": 154439,+ "measuredAt": "2026-10-07T17:53:49.730Z",+ "meanEstimatedListCostUsd": 0.2093862,+ "meanTokenConsumption": 26541,+ "meanToolInvocationCount": 23.666666666666668},{"experimentId": "kimi-k3-opencode--baseline",@@ -296,11 +312,11 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 117347,- "measuredAt": "2026-08-28T19:22:06.000Z",- "meanEstimatedListCostUsd": 0.103991,- "meanTokenConsumption": 21791.666666666668,- "meanToolInvocationCount": 16.333333333333332+ "meanDurationMs": 132808,+ "measuredAt": "2026-10-07T17:53:49.747Z",+ "meanEstimatedListCostUsd": 0.1187692,+ "meanTokenConsumption": 17728.666666666668,+ "meanToolInvocationCount": 18},{"experimentId": "kimi-k3-opencode--guided",@@ -308,11 +324,11 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 73445,- "measuredAt": "2026-08-28T19:20:34.000Z",- "meanEstimatedListCostUsd": 0.08992099999999999,- "meanTokenConsumption": 20789.666666666668,- "meanToolInvocationCount": 12.333333333333334+ "meanDurationMs": 178412,+ "measuredAt": "2026-10-07T18:06:24.212Z",+ "meanEstimatedListCostUsd": 0.1784928,+ "meanTokenConsumption": 25840,+ "meanToolInvocationCount": 16.666666666666668},{"experimentId": "kimi-k3-opencode--guided",@@ -320,11 +336,11 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 90748,- "measuredAt": "2026-08-28T19:25:31.000Z",- "meanEstimatedListCostUsd": 0.081476,- "meanTokenConsumption": 20988,- "meanToolInvocationCount": 12+ "meanDurationMs": 219061,+ "measuredAt": "2026-10-07T18:06:24.226Z",+ "meanEstimatedListCostUsd": 0.1086592,+ "meanTokenConsumption": 19816,+ "meanToolInvocationCount": 12.666666666666666},{"experimentId": "kimi-k3-opencode--guided",@@ -332,11 +348,11 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 71243,- "measuredAt": "2026-08-28T19:25:31.000Z",- "meanEstimatedListCostUsd": 0.081825,- "meanTokenConsumption": 19215,- "meanToolInvocationCount": 13.333333333333334+ "meanDurationMs": 156025,+ "measuredAt": "2026-10-07T18:06:24.238Z",+ "meanEstimatedListCostUsd": 0.12732,+ "meanTokenConsumption": 26533.333333333332,+ "meanToolInvocationCount": 10.333333333333334},{"experimentId": "kimi-k3-opencode--guided",@@ -344,11 +360,11 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 126758,- "measuredAt": "2026-08-28T19:20:34.000Z",- "meanEstimatedListCostUsd": 0.08192899999999999,- "meanTokenConsumption": 16876.333333333332,- "meanToolInvocationCount": 15+ "meanDurationMs": 145832,+ "measuredAt": "2026-10-07T18:06:24.248Z",+ "meanEstimatedListCostUsd": 0.11396500000000002,+ "meanTokenConsumption": 19580.333333333332,+ "meanToolInvocationCount": 10.333333333333334},{"experimentId": "kimi-k3-opencode--guided",@@ -356,11 +372,11 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 95028,- "measuredAt": "2026-08-28T19:20:34.000Z",- "meanEstimatedListCostUsd": 0.117859,- "meanTokenConsumption": 29467.666666666668,- "meanToolInvocationCount": 16.333333333333332+ "meanDurationMs": 148846,+ "measuredAt": "2026-10-07T18:06:24.257Z",+ "meanEstimatedListCostUsd": 0.16298139999999997,+ "meanTokenConsumption": 30186.333333333332,+ "meanToolInvocationCount": 12.666666666666666},{"experimentId": "kimi-k3-opencode--guided",@@ -368,11 +384,11 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 49272,- "measuredAt": "2026-08-28T19:20:34.000Z",- "meanEstimatedListCostUsd": 0.0695,- "meanTokenConsumption": 17840,- "meanToolInvocationCount": 10.333333333333334+ "meanDurationMs": 157626,+ "measuredAt": "2026-10-07T18:06:24.270Z",+ "meanEstimatedListCostUsd": 0.1153282,+ "meanTokenConsumption": 17712.333333333332,+ "meanToolInvocationCount": 11},{"experimentId": "kimi-k3-opencode--guided",@@ -380,11 +396,11 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 77381,- "measuredAt": "2026-08-28T19:20:34.000Z",- "meanEstimatedListCostUsd": 0.061534,- "meanTokenConsumption": 14636.666666666666,- "meanToolInvocationCount": 12.333333333333334+ "meanDurationMs": 142137,+ "measuredAt": "2026-10-07T18:06:24.286Z",+ "meanEstimatedListCostUsd": 0.08892719999999998,+ "meanTokenConsumption": 15018.666666666666,+ "meanToolInvocationCount": 10},{"experimentId": "claude-fable-5-1-claude-code--baseline",@@ -392,11 +408,11 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 186262,- "measuredAt": "2026-09-02T14:19:29.095Z",- "meanEstimatedListCostUsd": 1.291232,- "meanTokenConsumption": 8972,- "meanToolInvocationCount": 11.666666666666666+ "meanDurationMs": 299284,+ "measuredAt": "2026-10-07T18:34:17.712Z",+ "meanEstimatedListCostUsd": 1.7602164166666665,+ "meanTokenConsumption": 14277,+ "meanToolInvocationCount": 15},{"experimentId": "claude-fable-5-1-claude-code--baseline",@@ -404,11 +420,11 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 148272,- "measuredAt": "2026-09-02T14:19:29.109Z",- "meanEstimatedListCostUsd": 0.7348391666666666,- "meanTokenConsumption": 3895.3333333333335,- "meanToolInvocationCount": 9+ "meanDurationMs": 220684,+ "measuredAt": "2026-10-07T18:34:17.721Z",+ "meanEstimatedListCostUsd": 0.8802499166666667,+ "meanTokenConsumption": 6197.333333333333,+ "meanToolInvocationCount": 9.666666666666666},{"experimentId": "claude-fable-5-1-claude-code--baseline",@@ -416,11 +432,11 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 183777,- "measuredAt": "2026-09-02T14:19:29.120Z",- "meanEstimatedListCostUsd": 1.31527725,- "meanTokenConsumption": 8248,- "meanToolInvocationCount": 16.666666666666668+ "meanDurationMs": 315667,+ "measuredAt": "2026-10-07T18:34:17.732Z",+ "meanEstimatedListCostUsd": 1.7283125833333335,+ "meanTokenConsumption": 12128.333333333334,+ "meanToolInvocationCount": 23},{"experimentId": "claude-fable-5-1-claude-code--baseline",@@ -428,11 +444,11 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 180513,- "measuredAt": "2026-09-02T14:19:29.129Z",- "meanEstimatedListCostUsd": 1.0393497499999997,- "meanTokenConsumption": 9160.333333333334,- "meanToolInvocationCount": 9.666666666666666+ "meanDurationMs": 237115,+ "measuredAt": "2026-10-07T18:34:17.746Z",+ "meanEstimatedListCostUsd": 1.02154625,+ "meanTokenConsumption": 8290,+ "meanToolInvocationCount": 9.333333333333334},{"experimentId": "claude-fable-5-1-claude-code--baseline",@@ -440,11 +456,11 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 181059,- "measuredAt": "2026-09-02T14:19:29.137Z",- "meanEstimatedListCostUsd": 1.3291795000000002,- "meanTokenConsumption": 7962.333333333333,- "meanToolInvocationCount": 16.666666666666668+ "meanDurationMs": 282249,+ "measuredAt": "2026-10-07T18:34:17.763Z",+ "meanEstimatedListCostUsd": 1.7565878333333333,+ "meanTokenConsumption": 9842.666666666666,+ "meanToolInvocationCount": 19.666666666666668},{"experimentId": "claude-fable-5-1-claude-code--baseline",@@ -452,11 +468,11 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 149380,- "measuredAt": "2026-09-02T14:19:29.147Z",- "meanEstimatedListCostUsd": 1.0626429999999998,- "meanTokenConsumption": 4479.333333333333,- "meanToolInvocationCount": 10.666666666666666+ "meanDurationMs": 230125,+ "measuredAt": "2026-10-07T18:34:17.779Z",+ "meanEstimatedListCostUsd": 1.1730868333333333,+ "meanTokenConsumption": 4567.666666666667,+ "meanToolInvocationCount": 12},{"experimentId": "claude-fable-5-1-claude-code--baseline",@@ -464,11 +480,11 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 151473,- "measuredAt": "2026-09-02T14:19:29.154Z",- "meanEstimatedListCostUsd": 0.9185504999999999,- "meanTokenConsumption": 4827.666666666667,- "meanToolInvocationCount": 11.666666666666666+ "meanDurationMs": 180733,+ "measuredAt": "2026-10-07T18:34:17.794Z",+ "meanEstimatedListCostUsd": 0.8628102499999999,+ "meanTokenConsumption": 4052.3333333333335,+ "meanToolInvocationCount": 10},{"experimentId": "claude-fable-5-1-claude-code--guided",@@ -476,11 +492,11 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 142627,- "measuredAt": "2026-09-02T14:18:53.362Z",- "meanEstimatedListCostUsd": 0.9972783333333334,- "meanTokenConsumption": 6633,- "meanToolInvocationCount": 6+ "meanDurationMs": 216964,+ "measuredAt": "2026-10-07T18:38:17.304Z",+ "meanEstimatedListCostUsd": 1.1694255,+ "meanTokenConsumption": 8664.333333333334,+ "meanToolInvocationCount": 9.333333333333334},{"experimentId": "claude-fable-5-1-claude-code--guided",@@ -488,11 +504,11 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 155541,- "measuredAt": "2026-09-02T14:18:53.381Z",- "meanEstimatedListCostUsd": 1.6608135,- "meanTokenConsumption": 4615,- "meanToolInvocationCount": 9.666666666666666+ "meanDurationMs": 177448,+ "measuredAt": "2026-10-07T18:38:17.319Z",+ "meanEstimatedListCostUsd": 0.6489536666666667,+ "meanTokenConsumption": 4501,+ "meanToolInvocationCount": 7.333333333333333},{"experimentId": "claude-fable-5-1-claude-code--guided",@@ -500,10 +516,10 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 132220,- "measuredAt": "2026-09-02T14:18:53.396Z",- "meanEstimatedListCostUsd": 0.8656001666666667,- "meanTokenConsumption": 3575,+ "meanDurationMs": 166095,+ "measuredAt": "2026-10-07T18:38:17.334Z",+ "meanEstimatedListCostUsd": 0.6243346666666666,+ "meanTokenConsumption": 2917.3333333333335,"meanToolInvocationCount": 7.333333333333333},{@@ -512,11 +528,11 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 161398,- "measuredAt": "2026-09-02T14:18:53.406Z",- "meanEstimatedListCostUsd": 0.7871509166666666,- "meanTokenConsumption": 5333.333333333333,- "meanToolInvocationCount": 8+ "meanDurationMs": 232620,+ "measuredAt": "2026-10-07T18:38:17.345Z",+ "meanEstimatedListCostUsd": 0.7477252499999999,+ "meanTokenConsumption": 6484.666666666667,+ "meanToolInvocationCount": 9},{"experimentId": "claude-fable-5-1-claude-code--guided",@@ -524,11 +540,11 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 143233,- "measuredAt": "2026-09-02T14:18:53.417Z",- "meanEstimatedListCostUsd": 0.8737515833333332,- "meanTokenConsumption": 3909.3333333333335,- "meanToolInvocationCount": 7+ "meanDurationMs": 170755,+ "measuredAt": "2026-10-07T18:38:17.357Z",+ "meanEstimatedListCostUsd": 0.8186226666666667,+ "meanTokenConsumption": 4187.333333333333,+ "meanToolInvocationCount": 7.333333333333333},{"experimentId": "claude-fable-5-1-claude-code--guided",@@ -536,11 +552,11 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 115430,- "measuredAt": "2026-09-02T14:18:53.429Z",- "meanEstimatedListCostUsd": 0.63728775,- "meanTokenConsumption": 1828.6666666666667,- "meanToolInvocationCount": 5.666666666666667+ "meanDurationMs": 154662,+ "measuredAt": "2026-10-07T18:38:17.370Z",+ "meanEstimatedListCostUsd": 0.6756555833333332,+ "meanTokenConsumption": 2156,+ "meanToolInvocationCount": 6},{"experimentId": "claude-fable-5-1-claude-code--guided",@@ -548,1859 +564,2027 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 130462,- "measuredAt": "2026-09-02T14:18:53.440Z",- "meanEstimatedListCostUsd": 0.6112984166666667,- "meanTokenConsumption": 2535.3333333333335,+ "meanDurationMs": 176359,+ "measuredAt": "2026-10-07T18:38:17.381Z",+ "meanEstimatedListCostUsd": 0.5568065833333334,+ "meanTokenConsumption": 2716.3333333333335,"meanToolInvocationCount": 7.333333333333333},{- "experimentId": "grok-4-6-opencode--baseline",+ "experimentId": "gpt-5-6-terra-codex--baseline","caseId": "author-001-weather-tool","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 38613,- "measuredAt": "2026-08-28T19:22:01.000Z",- "meanEstimatedListCostUsd": 0.10434266666666665,- "meanTokenConsumption": 49566,- "meanToolInvocationCount": 19+ "meanDurationMs": 122298,+ "measuredAt": "2026-10-07T21:58:48.506Z",+ "meanEstimatedListCostUsd": 0.22163763333333333,+ "meanTokenConsumption": 37612.666666666664,+ "meanToolInvocationCount": 7.333333333333333},{- "experimentId": "grok-4-6-opencode--baseline",+ "experimentId": "gpt-5-6-terra-codex--baseline","caseId": "author-002-new-project","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 83985,- "measuredAt": "2026-08-28T19:25:12.000Z",- "meanEstimatedListCostUsd": 0.07232866666666667,- "meanTokenConsumption": 32827.666666666664,- "meanToolInvocationCount": 16+ "meanDurationMs": 138040,+ "measuredAt": "2026-10-07T21:58:48.529Z",+ "meanEstimatedListCostUsd": 0.18387216666666664,+ "meanTokenConsumption": 26238.333333333332,+ "meanToolInvocationCount": 11.333333333333334},{- "experimentId": "grok-4-6-opencode--baseline",+ "experimentId": "gpt-5-6-terra-codex--baseline","caseId": "author-003-openapi-connection","status": "current",- "passedRuns": 0,+ "passedRuns": 3,"validRuns": 3,- "meanDurationMs": 521954,- "measuredAt": "2026-08-28T19:22:01.000Z",- "meanEstimatedListCostUsd": 1.0741786666666668,- "meanTokenConsumption": 503422.6666666667,- "meanToolInvocationCount": 76.66666666666667+ "meanDurationMs": 117932,+ "measuredAt": "2026-10-07T21:58:48.540Z",+ "meanEstimatedListCostUsd": 0.17790966666666666,+ "meanTokenConsumption": 30419,+ "meanToolInvocationCount": 7.333333333333333},{- "experimentId": "grok-4-6-opencode--baseline",+ "experimentId": "gpt-5-6-terra-codex--baseline","caseId": "author-004-packaged-skill","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 41894,- "measuredAt": "2026-08-28T19:22:01.000Z",- "meanEstimatedListCostUsd": 0.065152,- "meanTokenConsumption": 29506,- "meanToolInvocationCount": 23.333333333333332+ "meanDurationMs": 137504,+ "measuredAt": "2026-10-07T21:58:48.551Z",+ "meanEstimatedListCostUsd": 0.18458839999999999,+ "meanTokenConsumption": 30275,+ "meanToolInvocationCount": 6.666666666666667},{- "experimentId": "grok-4-6-opencode--baseline",+ "experimentId": "gpt-5-6-terra-codex--baseline","caseId": "author-005-conditional-approval","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 50151,- "measuredAt": "2026-08-28T19:22:01.000Z",- "meanEstimatedListCostUsd": 0.141536,- "meanTokenConsumption": 66629.33333333333,- "meanToolInvocationCount": 22+ "meanDurationMs": 127649,+ "measuredAt": "2026-10-07T21:58:48.568Z",+ "meanEstimatedListCostUsd": 0.19955179999999997,+ "meanTokenConsumption": 33755.666666666664,+ "meanToolInvocationCount": 6.666666666666667},{- "experimentId": "grok-4-6-opencode--baseline",+ "experimentId": "gpt-5-6-terra-codex--baseline","caseId": "author-006-custom-channel","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 37062,- "measuredAt": "2026-08-28T19:22:01.000Z",- "meanEstimatedListCostUsd": 0.07319533333333333,- "meanTokenConsumption": 33801.666666666664,- "meanToolInvocationCount": 17.666666666666668+ "meanDurationMs": 116757,+ "measuredAt": "2026-10-07T21:58:48.584Z",+ "meanEstimatedListCostUsd": 0.17057733333333333,+ "meanTokenConsumption": 29653,+ "meanToolInvocationCount": 6.333333333333333},{- "experimentId": "grok-4-6-opencode--baseline",+ "experimentId": "gpt-5-6-terra-codex--baseline","caseId": "author-007-digest-schedule","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 38280,- "measuredAt": "2026-08-28T19:22:01.000Z",- "meanEstimatedListCostUsd": 0.045748000000000004,- "meanTokenConsumption": 20122.666666666668,- "meanToolInvocationCount": 18+ "meanDurationMs": 112507,+ "measuredAt": "2026-10-07T21:58:48.600Z",+ "meanEstimatedListCostUsd": 0.14675936666666667,+ "meanTokenConsumption": 25050.666666666668,+ "meanToolInvocationCount": 7},{- "experimentId": "grok-4-6-opencode--guided",+ "experimentId": "gpt-5-6-terra-codex--guided","caseId": "author-001-weather-tool","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 31700,- "measuredAt": "2026-08-28T19:22:02.000Z",- "meanEstimatedListCostUsd": 0.049695333333333334,- "meanTokenConsumption": 22963.666666666668,- "meanToolInvocationCount": 9.666666666666666+ "meanDurationMs": 119359,+ "measuredAt": "2026-10-07T22:01:49.428Z",+ "meanEstimatedListCostUsd": 0.1435413,+ "meanTokenConsumption": 24455,+ "meanToolInvocationCount": 4.666666666666667},{- "experimentId": "grok-4-6-opencode--guided",+ "experimentId": "gpt-5-6-terra-codex--guided","caseId": "author-002-new-project","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 83621,- "measuredAt": "2026-08-28T19:25:07.000Z",- "meanEstimatedListCostUsd": 0.082842,- "meanTokenConsumption": 38261,- "meanToolInvocationCount": 16.333333333333332+ "meanDurationMs": 161276,+ "measuredAt": "2026-10-07T22:01:49.437Z",+ "meanEstimatedListCostUsd": 0.21618046666666665,+ "meanTokenConsumption": 29838.333333333332,+ "meanToolInvocationCount": 12.333333333333334},{- "experimentId": "grok-4-6-opencode--guided",+ "experimentId": "gpt-5-6-terra-codex--guided","caseId": "author-003-openapi-connection","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 34833,- "measuredAt": "2026-08-28T19:25:07.000Z",- "meanEstimatedListCostUsd": 0.055174,- "meanTokenConsumption": 25952.333333333332,- "meanToolInvocationCount": 10+ "meanDurationMs": 111613,+ "measuredAt": "2026-10-07T22:01:49.449Z",+ "meanEstimatedListCostUsd": 0.12379853333333335,+ "meanTokenConsumption": 22123,+ "meanToolInvocationCount": 4.333333333333333},{- "experimentId": "grok-4-6-opencode--guided",+ "experimentId": "gpt-5-6-terra-codex--guided","caseId": "author-004-packaged-skill","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 30135,- "measuredAt": "2026-08-28T19:22:02.000Z",- "meanEstimatedListCostUsd": 0.035581999999999996,- "meanTokenConsumption": 16043.666666666666,- "meanToolInvocationCount": 8.666666666666666+ "meanDurationMs": 130435,+ "measuredAt": "2026-10-07T22:01:49.456Z",+ "meanEstimatedListCostUsd": 0.11441699999999999,+ "meanTokenConsumption": 19053.333333333332,+ "meanToolInvocationCount": 3.6666666666666665},{- "experimentId": "grok-4-6-opencode--guided",+ "experimentId": "gpt-5-6-terra-codex--guided","caseId": "author-005-conditional-approval","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 34026,- "measuredAt": "2026-08-28T19:22:02.000Z",- "meanEstimatedListCostUsd": 0.038532666666666666,- "meanTokenConsumption": 17059.666666666668,- "meanToolInvocationCount": 11.666666666666666+ "meanDurationMs": 122684,+ "measuredAt": "2026-10-07T22:01:49.463Z",+ "meanEstimatedListCostUsd": 0.15166520000000003,+ "meanTokenConsumption": 27042.333333333332,+ "meanToolInvocationCount": 4.333333333333333},{- "experimentId": "grok-4-6-opencode--guided",+ "experimentId": "gpt-5-6-terra-codex--guided","caseId": "author-006-custom-channel","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 32109,- "measuredAt": "2026-08-28T19:22:02.000Z",- "meanEstimatedListCostUsd": 0.033709333333333334,- "meanTokenConsumption": 14610,- "meanToolInvocationCount": 9.666666666666666+ "meanDurationMs": 133774,+ "measuredAt": "2026-10-07T22:01:49.471Z",+ "meanEstimatedListCostUsd": 0.11311536666666666,+ "meanTokenConsumption": 20847.666666666668,+ "meanToolInvocationCount": 3.3333333333333335},{- "experimentId": "grok-4-6-opencode--guided",+ "experimentId": "gpt-5-6-terra-codex--guided","caseId": "author-007-digest-schedule","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 25583,- "measuredAt": "2026-08-28T19:22:02.000Z",- "meanEstimatedListCostUsd": 0.023458666666666666,- "meanTokenConsumption": 10328,- "meanToolInvocationCount": 7.333333333333333+ "meanDurationMs": 104430,+ "measuredAt": "2026-10-07T22:01:49.483Z",+ "meanEstimatedListCostUsd": 0.09763046666666665,+ "meanTokenConsumption": 18167.666666666668,+ "meanToolInvocationCount": 3.3333333333333335},{- "experimentId": "gpt-5-6-sol-codex--baseline",+ "experimentId": "gpt-6-astra-high-codex--baseline","caseId": "author-001-weather-tool","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 175042,- "measuredAt": "2026-08-31T19:20:09.055Z",- "meanEstimatedListCostUsd": 0.42096675,- "meanTokenConsumption": 49663.666666666664,- "meanToolInvocationCount": 13.666666666666666+ "meanDurationMs": 237825,+ "measuredAt": "2026-10-07T17:36:21.398Z",+ "meanEstimatedListCostUsd": 1.2291793333333334,+ "meanTokenConsumption": 40990.333333333336,+ "meanToolInvocationCount": 14},{- "experimentId": "gpt-5-6-sol-codex--baseline",+ "experimentId": "gpt-6-astra-high-codex--baseline","caseId": "author-002-new-project","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 122470,- "measuredAt": "2026-08-31T19:20:09.085Z",- "meanEstimatedListCostUsd": 0.23273858333333333,- "meanTokenConsumption": 30848.666666666668,+ "meanDurationMs": 134301,+ "measuredAt": "2026-10-07T17:36:21.424Z",+ "meanEstimatedListCostUsd": 0.746785,+ "meanTokenConsumption": 23511,"meanToolInvocationCount": 10.333333333333334},{- "experimentId": "gpt-5-6-sol-codex--baseline",+ "experimentId": "gpt-6-astra-high-codex--baseline","caseId": "author-003-openapi-connection","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 132036,- "measuredAt": "2026-08-31T19:20:09.104Z",- "meanEstimatedListCostUsd": 0.367498625,- "meanTokenConsumption": 48982.666666666664,- "meanToolInvocationCount": 9.333333333333334+ "meanDurationMs": 141746,+ "measuredAt": "2026-10-07T17:36:21.437Z",+ "meanEstimatedListCostUsd": 0.8998284999999999,+ "meanTokenConsumption": 32579.666666666668,+ "meanToolInvocationCount": 10.666666666666666},{- "experimentId": "gpt-5-6-sol-codex--baseline",+ "experimentId": "gpt-6-astra-high-codex--baseline","caseId": "author-004-packaged-skill","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 137447,- "measuredAt": "2026-08-31T19:20:09.128Z",- "meanEstimatedListCostUsd": 0.26755258333333337,- "meanTokenConsumption": 36330.666666666664,- "meanToolInvocationCount": 6+ "meanDurationMs": 217226,+ "measuredAt": "2026-10-07T17:36:21.447Z",+ "meanEstimatedListCostUsd": 0.9555386666666666,+ "meanTokenConsumption": 31480.666666666668,+ "meanToolInvocationCount": 13.333333333333334},{- "experimentId": "gpt-5-6-sol-codex--baseline",+ "experimentId": "gpt-6-astra-high-codex--baseline","caseId": "author-005-conditional-approval","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 121857,- "measuredAt": "2026-08-31T19:20:09.143Z",- "meanEstimatedListCostUsd": 0.2530605833333333,- "meanTokenConsumption": 32384,- "meanToolInvocationCount": 10+ "meanDurationMs": 170948,+ "measuredAt": "2026-10-07T17:36:21.456Z",+ "meanEstimatedListCostUsd": 1.1034936666666666,+ "meanTokenConsumption": 37715.333333333336,+ "meanToolInvocationCount": 12.666666666666666},{- "experimentId": "gpt-5-6-sol-codex--baseline",+ "experimentId": "gpt-6-astra-high-codex--baseline","caseId": "author-006-custom-channel","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 115653,- "measuredAt": "2026-08-31T19:20:09.156Z",- "meanEstimatedListCostUsd": 0.251469,- "meanTokenConsumption": 33854.666666666664,- "meanToolInvocationCount": 8+ "meanDurationMs": 156521,+ "measuredAt": "2026-10-07T17:36:21.466Z",+ "meanEstimatedListCostUsd": 1.3502271666666665,+ "meanTokenConsumption": 46148.666666666664,+ "meanToolInvocationCount": 13},{- "experimentId": "gpt-5-6-sol-codex--baseline",+ "experimentId": "gpt-6-astra-high-codex--baseline","caseId": "author-007-digest-schedule","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 121172,- "measuredAt": "2026-08-31T19:20:09.170Z",- "meanEstimatedListCostUsd": 0.284567375,- "meanTokenConsumption": 37330.666666666664,- "meanToolInvocationCount": 11+ "meanDurationMs": 140369,+ "measuredAt": "2026-10-07T17:36:21.479Z",+ "meanEstimatedListCostUsd": 0.7009501666666668,+ "meanTokenConsumption": 23989.333333333332,+ "meanToolInvocationCount": 9.666666666666666},{- "experimentId": "gpt-5-6-sol-codex--guided",+ "experimentId": "gpt-6-astra-high-codex--guided","caseId": "author-001-weather-tool","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 115694,- "measuredAt": "2026-08-31T19:18:53.704Z",- "meanEstimatedListCostUsd": 0.18167699999999998,- "meanTokenConsumption": 24500,- "meanToolInvocationCount": 4.666666666666667+ "meanDurationMs": 167803,+ "measuredAt": "2026-10-07T17:40:30.710Z",+ "meanEstimatedListCostUsd": 0.8353351666666667,+ "meanTokenConsumption": 29147.333333333332,+ "meanToolInvocationCount": 7.333333333333333},{- "experimentId": "gpt-5-6-sol-codex--guided",+ "experimentId": "gpt-6-astra-high-codex--guided","caseId": "author-002-new-project","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 132614,- "measuredAt": "2026-08-31T19:18:53.727Z",- "meanEstimatedListCostUsd": 0.22479154166666668,- "meanTokenConsumption": 28792.333333333332,- "meanToolInvocationCount": 9.333333333333334+ "meanDurationMs": 128241,+ "measuredAt": "2026-10-07T17:40:30.723Z",+ "meanEstimatedListCostUsd": 0.8843616666666666,+ "meanTokenConsumption": 27507.333333333332,+ "meanToolInvocationCount": 13.666666666666666},{- "experimentId": "gpt-5-6-sol-codex--guided",+ "experimentId": "gpt-6-astra-high-codex--guided","caseId": "author-003-openapi-connection","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 101648,- "measuredAt": "2026-08-31T19:18:53.741Z",- "meanEstimatedListCostUsd": 0.14007875,- "meanTokenConsumption": 19209.666666666668,- "meanToolInvocationCount": 5.333333333333333+ "meanDurationMs": 143101,+ "measuredAt": "2026-10-07T17:40:30.733Z",+ "meanEstimatedListCostUsd": 0.6892401666666667,+ "meanTokenConsumption": 25532.666666666668,+ "meanToolInvocationCount": 8.333333333333334},{- "experimentId": "gpt-5-6-sol-codex--guided",+ "experimentId": "gpt-6-astra-high-codex--guided","caseId": "author-004-packaged-skill","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 129791,- "measuredAt": "2026-08-31T19:18:53.751Z",- "meanEstimatedListCostUsd": 0.19388683333333334,- "meanTokenConsumption": 24916,- "meanToolInvocationCount": 6+ "meanDurationMs": 196512,+ "measuredAt": "2026-10-07T17:40:30.742Z",+ "meanEstimatedListCostUsd": 0.7988335000000001,+ "meanTokenConsumption": 26211.666666666668,+ "meanToolInvocationCount": 8.666666666666666},{- "experimentId": "gpt-5-6-sol-codex--guided",+ "experimentId": "gpt-6-astra-high-codex--guided","caseId": "author-005-conditional-approval","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 108054,- "measuredAt": "2026-08-31T19:18:53.759Z",- "meanEstimatedListCostUsd": 0.15307379166666665,- "meanTokenConsumption": 20874.333333333332,- "meanToolInvocationCount": 4.666666666666667+ "meanDurationMs": 128279,+ "measuredAt": "2026-10-07T17:40:30.750Z",+ "meanEstimatedListCostUsd": 0.6823355000000001,+ "meanTokenConsumption": 25595.333333333332,+ "meanToolInvocationCount": 5.333333333333333},{- "experimentId": "gpt-5-6-sol-codex--guided",+ "experimentId": "gpt-6-astra-high-codex--guided","caseId": "author-006-custom-channel","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 100410,- "measuredAt": "2026-08-31T19:18:53.769Z",- "meanEstimatedListCostUsd": 0.108055625,- "meanTokenConsumption": 15663.666666666666,- "meanToolInvocationCount": 3+ "meanDurationMs": 143013,+ "measuredAt": "2026-10-07T17:40:30.757Z",+ "meanEstimatedListCostUsd": 0.5633055,+ "meanTokenConsumption": 21594.666666666668,+ "meanToolInvocationCount": 5},{- "experimentId": "gpt-5-6-sol-codex--guided",+ "experimentId": "gpt-6-astra-high-codex--guided","caseId": "author-007-digest-schedule","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 103849,- "measuredAt": "2026-08-31T19:18:53.779Z",- "meanEstimatedListCostUsd": 0.11547295833333333,- "meanTokenConsumption": 16687.333333333332,- "meanToolInvocationCount": 3.6666666666666665+ "meanDurationMs": 133217,+ "measuredAt": "2026-10-07T17:40:30.763Z",+ "meanEstimatedListCostUsd": 0.5985591666666666,+ "meanTokenConsumption": 21658.333333333332,+ "meanToolInvocationCount": 6.666666666666667},{- "experimentId": "gpt-5-6-terra-codex--baseline",+ "experimentId": "claude-sonnet-5-claude-code--baseline","caseId": "author-001-weather-tool","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 119086,- "measuredAt": "2026-08-31T20:01:56.276Z",- "meanEstimatedListCostUsd": 0.2552948666666666,- "meanTokenConsumption": 44396,- "meanToolInvocationCount": 6+ "meanDurationMs": 127080,+ "measuredAt": "2026-10-07T17:14:32.284Z",+ "meanEstimatedListCostUsd": 0.33771066666666666,+ "meanTokenConsumption": 5721.333333333333,+ "meanToolInvocationCount": 10.666666666666666},{- "experimentId": "gpt-5-6-terra-codex--baseline",+ "experimentId": "claude-sonnet-5-claude-code--baseline","caseId": "author-002-new-project","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 117037,- "measuredAt": "2026-08-31T20:01:56.312Z",- "meanEstimatedListCostUsd": 0.1636081,- "meanTokenConsumption": 26205.333333333332,- "meanToolInvocationCount": 8.666666666666666+ "meanDurationMs": 122433,+ "measuredAt": "2026-10-07T17:14:32.309Z",+ "meanEstimatedListCostUsd": 0.3149904,+ "meanTokenConsumption": 4382.333333333333,+ "meanToolInvocationCount": 9.333333333333334},{- "experimentId": "gpt-5-6-terra-codex--baseline",+ "experimentId": "claude-sonnet-5-claude-code--baseline","caseId": "author-003-openapi-connection","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 115379,- "measuredAt": "2026-08-31T20:01:56.326Z",- "meanEstimatedListCostUsd": 0.18920926666666668,- "meanTokenConsumption": 33257.666666666664,- "meanToolInvocationCount": 6.333333333333333+ "meanDurationMs": 121003,+ "measuredAt": "2026-10-07T17:14:32.328Z",+ "meanEstimatedListCostUsd": 0.4386330666666666,+ "meanTokenConsumption": 5224,+ "meanToolInvocationCount": 16},{- "experimentId": "gpt-5-6-terra-codex--baseline",+ "experimentId": "claude-sonnet-5-claude-code--baseline","caseId": "author-004-packaged-skill","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 119436,- "measuredAt": "2026-08-31T20:01:56.338Z",- "meanEstimatedListCostUsd": 0.16241646666666668,- "meanTokenConsumption": 27236.666666666668,- "meanToolInvocationCount": 5.666666666666667+ "meanDurationMs": 135343,+ "measuredAt": "2026-10-07T17:14:32.344Z",+ "meanEstimatedListCostUsd": 0.32376240000000006,+ "meanTokenConsumption": 5496,+ "meanToolInvocationCount": 11.666666666666666},{- "experimentId": "gpt-5-6-terra-codex--baseline",+ "experimentId": "claude-sonnet-5-claude-code--baseline","caseId": "author-005-conditional-approval","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 105622,- "measuredAt": "2026-08-31T20:01:56.348Z",- "meanEstimatedListCostUsd": 0.1821714666666667,- "meanTokenConsumption": 32177,- "meanToolInvocationCount": 6.333333333333333+ "meanDurationMs": 112919,+ "measuredAt": "2026-10-07T17:14:32.359Z",+ "meanEstimatedListCostUsd": 0.33600040000000003,+ "meanTokenConsumption": 3654,+ "meanToolInvocationCount": 10.333333333333334},{- "experimentId": "gpt-5-6-terra-codex--baseline",+ "experimentId": "claude-sonnet-5-claude-code--baseline","caseId": "author-006-custom-channel","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 108983,- "measuredAt": "2026-08-31T20:01:56.359Z",- "meanEstimatedListCostUsd": 0.2155289,- "meanTokenConsumption": 37516,- "meanToolInvocationCount": 7+ "meanDurationMs": 115056,+ "measuredAt": "2026-10-07T17:14:32.373Z",+ "meanEstimatedListCostUsd": 0.32271563333333336,+ "meanTokenConsumption": 4199,+ "meanToolInvocationCount": 11.666666666666666},{- "experimentId": "gpt-5-6-terra-codex--baseline",+ "experimentId": "claude-sonnet-5-claude-code--baseline","caseId": "author-007-digest-schedule","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 103918,- "measuredAt": "2026-08-31T20:01:56.372Z",- "meanEstimatedListCostUsd": 0.12170393333333335,- "meanTokenConsumption": 21681.666666666668,- "meanToolInvocationCount": 5.333333333333333+ "meanDurationMs": 112597,+ "measuredAt": "2026-10-07T17:14:32.383Z",+ "meanEstimatedListCostUsd": 0.20493166666666665,+ "meanTokenConsumption": 2047.6666666666667,+ "meanToolInvocationCount": 7.666666666666667},{- "experimentId": "gpt-5-6-terra-codex--guided",+ "experimentId": "claude-sonnet-5-claude-code--guided","caseId": "author-001-weather-tool","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 113388,- "measuredAt": "2026-08-31T20:02:25.229Z",- "meanEstimatedListCostUsd": 0.15296396666666667,- "meanTokenConsumption": 26379.666666666668,- "meanToolInvocationCount": 5+ "meanDurationMs": 122291,+ "measuredAt": "2026-10-07T17:19:07.735Z",+ "meanEstimatedListCostUsd": 0.3211207,+ "meanTokenConsumption": 3416.6666666666665,+ "meanToolInvocationCount": 9},{- "experimentId": "gpt-5-6-terra-codex--guided",+ "experimentId": "claude-sonnet-5-claude-code--guided","caseId": "author-002-new-project","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 128859,- "measuredAt": "2026-08-31T20:02:25.256Z",- "meanEstimatedListCostUsd": 0.1434823,- "meanTokenConsumption": 22726.666666666668,- "meanToolInvocationCount": 10.333333333333334+ "meanDurationMs": 115898,+ "measuredAt": "2026-10-07T17:19:07.746Z",+ "meanEstimatedListCostUsd": 0.2441989,+ "meanTokenConsumption": 3790.3333333333335,+ "meanToolInvocationCount": 8.333333333333334},{- "experimentId": "gpt-5-6-terra-codex--guided",+ "experimentId": "claude-sonnet-5-claude-code--guided","caseId": "author-003-openapi-connection","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 101879,- "measuredAt": "2026-08-31T20:02:25.271Z",- "meanEstimatedListCostUsd": 0.1165826,- "meanTokenConsumption": 20464.333333333332,- "meanToolInvocationCount": 4.666666666666667+ "meanDurationMs": 123110,+ "measuredAt": "2026-10-07T17:19:07.756Z",+ "meanEstimatedListCostUsd": 0.27517036666666667,+ "meanTokenConsumption": 2186.3333333333335,+ "meanToolInvocationCount": 7.666666666666667},{- "experimentId": "gpt-5-6-terra-codex--guided",+ "experimentId": "claude-sonnet-5-claude-code--guided","caseId": "author-004-packaged-skill","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 112598,- "measuredAt": "2026-08-31T20:02:25.279Z",- "meanEstimatedListCostUsd": 0.11589093333333333,- "meanTokenConsumption": 19442.333333333332,- "meanToolInvocationCount": 3.6666666666666665+ "meanDurationMs": 127050,+ "measuredAt": "2026-10-07T17:19:07.766Z",+ "meanEstimatedListCostUsd": 0.1290148,+ "meanTokenConsumption": 2172.6666666666665,+ "meanToolInvocationCount": 5},{- "experimentId": "gpt-5-6-terra-codex--guided",+ "experimentId": "claude-sonnet-5-claude-code--guided","caseId": "author-005-conditional-approval","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 102311,- "measuredAt": "2026-08-31T20:02:25.288Z",- "meanEstimatedListCostUsd": 0.1219281,- "meanTokenConsumption": 22099.666666666668,- "meanToolInvocationCount": 4+ "meanDurationMs": 132250,+ "measuredAt": "2026-10-07T17:19:07.780Z",+ "meanEstimatedListCostUsd": 0.350098,+ "meanTokenConsumption": 3087,+ "meanToolInvocationCount": 8.666666666666666},{- "experimentId": "gpt-5-6-terra-codex--guided",+ "experimentId": "claude-sonnet-5-claude-code--guided","caseId": "author-006-custom-channel","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 100329,- "measuredAt": "2026-08-31T20:02:25.295Z",- "meanEstimatedListCostUsd": 0.10267943333333335,- "meanTokenConsumption": 18267.333333333332,- "meanToolInvocationCount": 4+ "meanDurationMs": 140975,+ "measuredAt": "2026-10-07T17:19:07.793Z",+ "meanEstimatedListCostUsd": 0.29428096666666664,+ "meanTokenConsumption": 2261.6666666666665,+ "meanToolInvocationCount": 8},{- "experimentId": "gpt-5-6-terra-codex--guided",+ "experimentId": "claude-sonnet-5-claude-code--guided","caseId": "author-007-digest-schedule","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 96962,- "measuredAt": "2026-08-31T20:02:25.301Z",- "meanEstimatedListCostUsd": 0.08585559999999999,- "meanTokenConsumption": 15490.333333333334,- "meanToolInvocationCount": 3.6666666666666665+ "meanDurationMs": 105861,+ "measuredAt": "2026-10-07T17:19:07.804Z",+ "meanEstimatedListCostUsd": 0.14527139999999997,+ "meanTokenConsumption": 1264.3333333333333,+ "meanToolInvocationCount": 4.333333333333333},{- "experimentId": "gpt-6-astra-high-codex--baseline",+ "experimentId": "glm-5-3-opencode--baseline","caseId": "author-001-weather-tool","status": "current",- "passedRuns": 3,+ "passedRuns": 2,"validRuns": 3,- "meanDurationMs": 165398,- "measuredAt": "2026-09-15T16:38:38.989Z",- "meanEstimatedListCostUsd": 1.164401,- "meanTokenConsumption": 38281.666666666664,- "meanToolInvocationCount": 14+ "meanDurationMs": 610963,+ "measuredAt": "2026-10-07T21:26:04.429Z",+ "meanEstimatedListCostUsd": 0.15013914,+ "meanTokenConsumption": 36303.5,+ "meanToolInvocationCount": 29},{- "experimentId": "gpt-6-astra-high-codex--baseline",+ "experimentId": "glm-5-3-opencode--baseline","caseId": "author-002-new-project","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 114673,- "measuredAt": "2026-09-15T16:38:39.004Z",- "meanEstimatedListCostUsd": 0.77442,- "meanTokenConsumption": 24544.666666666668,- "meanToolInvocationCount": 12.666666666666666+ "meanDurationMs": 304196,+ "measuredAt": "2026-10-07T21:26:04.443Z",+ "meanEstimatedListCostUsd": 0.07239437333333333,+ "meanTokenConsumption": 17362,+ "meanToolInvocationCount": 18.333333333333332},{- "experimentId": "gpt-6-astra-high-codex--baseline",+ "experimentId": "glm-5-3-opencode--baseline","caseId": "author-003-openapi-connection","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 139045,- "measuredAt": "2026-09-15T16:38:39.012Z",- "meanEstimatedListCostUsd": 0.9837176666666666,- "meanTokenConsumption": 34029,- "meanToolInvocationCount": 11.666666666666666+ "meanDurationMs": 352602,+ "measuredAt": "2026-10-07T21:26:04.457Z",+ "meanEstimatedListCostUsd": 0.15343737333333332,+ "meanTokenConsumption": 32562.666666666668,+ "meanToolInvocationCount": 34.666666666666664},{- "experimentId": "gpt-6-astra-high-codex--baseline",+ "experimentId": "glm-5-3-opencode--baseline","caseId": "author-004-packaged-skill","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 163959,- "measuredAt": "2026-09-15T16:38:39.020Z",- "meanEstimatedListCostUsd": 1.0705643333333335,- "meanTokenConsumption": 33997.333333333336,- "meanToolInvocationCount": 16+ "meanDurationMs": 372568,+ "measuredAt": "2026-10-07T21:26:04.479Z",+ "meanEstimatedListCostUsd": 0.0860574,+ "meanTokenConsumption": 20747,+ "meanToolInvocationCount": 24.333333333333332},{- "experimentId": "gpt-6-astra-high-codex--baseline",+ "experimentId": "glm-5-3-opencode--baseline","caseId": "author-005-conditional-approval","status": "current",- "passedRuns": 3,+ "passedRuns": 2,"validRuns": 3,- "meanDurationMs": 140774,- "measuredAt": "2026-09-15T16:38:39.030Z",- "meanEstimatedListCostUsd": 1.1568405000000002,- "meanTokenConsumption": 38287.333333333336,- "meanToolInvocationCount": 11.666666666666666+ "meanDurationMs": 409549,+ "measuredAt": "2026-10-07T21:26:04.497Z",+ "meanEstimatedListCostUsd": 0.12993986666666668,+ "meanTokenConsumption": 30566,+ "meanToolInvocationCount": 29.333333333333332},{- "experimentId": "gpt-6-astra-high-codex--baseline",+ "experimentId": "glm-5-3-opencode--baseline","caseId": "author-006-custom-channel","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 124395,- "measuredAt": "2026-09-15T16:38:39.041Z",- "meanEstimatedListCostUsd": 0.9883605000000001,- "meanTokenConsumption": 35251.666666666664,- "meanToolInvocationCount": 11.333333333333334+ "meanDurationMs": 378657,+ "measuredAt": "2026-10-07T21:26:04.520Z",+ "meanEstimatedListCostUsd": 0.15328993333333335,+ "meanTokenConsumption": 34802,+ "meanToolInvocationCount": 32},{- "experimentId": "gpt-6-astra-high-codex--baseline",+ "experimentId": "glm-5-3-opencode--baseline","caseId": "author-007-digest-schedule","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 122178,- "measuredAt": "2026-09-15T16:38:39.053Z",- "meanEstimatedListCostUsd": 0.7496145,- "meanTokenConsumption": 26059.333333333332,- "meanToolInvocationCount": 9.333333333333334+ "meanDurationMs": 282814,+ "measuredAt": "2026-10-07T21:26:04.542Z",+ "meanEstimatedListCostUsd": 0.06492005333333334,+ "meanTokenConsumption": 14953,+ "meanToolInvocationCount": 26.333333333333332},{- "experimentId": "gpt-6-astra-high-codex--guided",+ "experimentId": "glm-5-3-opencode--guided","caseId": "author-001-weather-tool","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 134344,- "measuredAt": "2026-09-15T16:38:40.721Z",- "meanEstimatedListCostUsd": 0.7615181666666667,- "meanTokenConsumption": 27210.666666666668,- "meanToolInvocationCount": 6.666666666666667+ "meanDurationMs": 368173,+ "measuredAt": "2026-10-07T21:33:02.073Z",+ "meanEstimatedListCostUsd": 0.11302922666666666,+ "meanTokenConsumption": 25322.333333333332,+ "meanToolInvocationCount": 19.666666666666668},{- "experimentId": "gpt-6-astra-high-codex--guided",+ "experimentId": "glm-5-3-opencode--guided","caseId": "author-002-new-project","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 118677,- "measuredAt": "2026-09-15T16:38:40.727Z",- "meanEstimatedListCostUsd": 0.6784536666666666,- "meanTokenConsumption": 21192,- "meanToolInvocationCount": 11.666666666666666+ "meanDurationMs": 225257,+ "measuredAt": "2026-10-07T21:33:02.095Z",+ "meanEstimatedListCostUsd": 0.051035119999999996,+ "meanTokenConsumption": 14036.666666666666,+ "meanToolInvocationCount": 17.333333333333332},{- "experimentId": "gpt-6-astra-high-codex--guided",+ "experimentId": "glm-5-3-opencode--guided","caseId": "author-003-openapi-connection","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 113782,- "measuredAt": "2026-09-15T16:38:40.734Z",- "meanEstimatedListCostUsd": 0.6524503333333334,- "meanTokenConsumption": 24654.333333333332,- "meanToolInvocationCount": 7.333333333333333+ "meanDurationMs": 187845,+ "measuredAt": "2026-10-07T21:33:02.118Z",+ "meanEstimatedListCostUsd": 0.038902653333333335,+ "meanTokenConsumption": 12674,+ "meanToolInvocationCount": 10.666666666666666},{- "experimentId": "gpt-6-astra-high-codex--guided",+ "experimentId": "glm-5-3-opencode--guided","caseId": "author-004-packaged-skill","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 161943,- "measuredAt": "2026-09-15T16:38:40.740Z",- "meanEstimatedListCostUsd": 0.8340828333333333,- "meanTokenConsumption": 26331.333333333332,- "meanToolInvocationCount": 10.333333333333334+ "meanDurationMs": 240967,+ "measuredAt": "2026-10-07T21:33:02.139Z",+ "meanEstimatedListCostUsd": 0.04021544,+ "meanTokenConsumption": 9470.666666666666,+ "meanToolInvocationCount": 11.666666666666666},{- "experimentId": "gpt-6-astra-high-codex--guided",+ "experimentId": "glm-5-3-opencode--guided","caseId": "author-005-conditional-approval","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 120364,- "measuredAt": "2026-09-15T16:38:40.749Z",- "meanEstimatedListCostUsd": 0.674382,- "meanTokenConsumption": 24265.333333333332,- "meanToolInvocationCount": 5.666666666666667+ "meanDurationMs": 286860,+ "measuredAt": "2026-10-07T21:33:02.157Z",+ "meanEstimatedListCostUsd": 0.09585413333333333,+ "meanTokenConsumption": 23787.666666666668,+ "meanToolInvocationCount": 19},{- "experimentId": "gpt-6-astra-high-codex--guided",+ "experimentId": "glm-5-3-opencode--guided","caseId": "author-006-custom-channel","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 104841,- "measuredAt": "2026-09-15T16:38:40.756Z",- "meanEstimatedListCostUsd": 0.5142598333333334,- "meanTokenConsumption": 20021.333333333332,- "meanToolInvocationCount": 5+ "meanDurationMs": 251041,+ "measuredAt": "2026-10-07T21:33:02.182Z",+ "meanEstimatedListCostUsd": 0.07182817333333334,+ "meanTokenConsumption": 19015,+ "meanToolInvocationCount": 17},{- "experimentId": "gpt-6-astra-high-codex--guided",+ "experimentId": "glm-5-3-opencode--guided","caseId": "author-007-digest-schedule","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 119293,- "measuredAt": "2026-09-15T16:38:40.763Z",- "meanEstimatedListCostUsd": 0.5865495,- "meanTokenConsumption": 21058.333333333332,- "meanToolInvocationCount": 7+ "meanDurationMs": 203204,+ "measuredAt": "2026-10-07T21:33:02.208Z",+ "meanEstimatedListCostUsd": 0.035970919999999997,+ "meanTokenConsumption": 8741.666666666666,+ "meanToolInvocationCount": 11.333333333333334},{- "experimentId": "claude-sonnet-5-claude-code--baseline",+ "experimentId": "gemini-3-1-pro-preview-opencode--baseline","caseId": "author-001-weather-tool","status": "current",- "passedRuns": 3,+ "passedRuns": 2,"validRuns": 3,- "meanDurationMs": 119150,- "measuredAt": "2026-09-02T14:38:33.830Z",- "meanEstimatedListCostUsd": 0.4004288333333334,- "meanTokenConsumption": 4827,- "meanToolInvocationCount": 11.333333333333334+ "meanDurationMs": 247589,+ "measuredAt": "2026-10-07T21:48:35.018Z",+ "meanEstimatedListCostUsd": 0.5625234,+ "meanTokenConsumption": 221445.33333333334,+ "meanToolInvocationCount": 20.333333333333332},{- "experimentId": "claude-sonnet-5-claude-code--baseline",+ "experimentId": "gemini-3-1-pro-preview-opencode--baseline","caseId": "author-002-new-project","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 117102,- "measuredAt": "2026-09-02T14:38:33.846Z",- "meanEstimatedListCostUsd": 0.38767966666666664,- "meanTokenConsumption": 3335.6666666666665,- "meanToolInvocationCount": 8.333333333333334+ "meanDurationMs": 125133,+ "measuredAt": "2026-10-07T21:48:35.035Z",+ "meanEstimatedListCostUsd": 0.0943808,+ "meanTokenConsumption": 37827.666666666664,+ "meanToolInvocationCount": 7.333333333333333},{- "experimentId": "claude-sonnet-5-claude-code--baseline",+ "experimentId": "gemini-3-1-pro-preview-opencode--baseline","caseId": "author-003-openapi-connection","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 146914,- "measuredAt": "2026-09-02T14:38:33.859Z",- "meanEstimatedListCostUsd": 0.6134467,- "meanTokenConsumption": 7727,- "meanToolInvocationCount": 20.333333333333332+ "meanDurationMs": 296490,+ "measuredAt": "2026-10-07T21:48:35.048Z",+ "meanEstimatedListCostUsd": 0.5817098666666666,+ "meanTokenConsumption": 204829.33333333334,+ "meanToolInvocationCount": 29.333333333333332},{- "experimentId": "claude-sonnet-5-claude-code--baseline",+ "experimentId": "gemini-3-1-pro-preview-opencode--baseline","caseId": "author-004-packaged-skill","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 128723,- "measuredAt": "2026-09-02T14:38:33.872Z",- "meanEstimatedListCostUsd": 0.2731853333333334,- "meanTokenConsumption": 4274,- "meanToolInvocationCount": 10.333333333333334+ "meanDurationMs": 144194,+ "measuredAt": "2026-10-07T21:48:35.070Z",+ "meanEstimatedListCostUsd": 0.0816354,+ "meanTokenConsumption": 30500.666666666668,+ "meanToolInvocationCount": 5},{- "experimentId": "claude-sonnet-5-claude-code--baseline",+ "experimentId": "gemini-3-1-pro-preview-opencode--baseline","caseId": "author-005-conditional-approval","status": "current",- "passedRuns": 3,+ "passedRuns": 1,"validRuns": 3,- "meanDurationMs": 112957,- "measuredAt": "2026-09-02T14:38:33.882Z",- "meanEstimatedListCostUsd": 0.3310536,- "meanTokenConsumption": 3567.3333333333335,- "meanToolInvocationCount": 12.666666666666666+ "meanDurationMs": 220270,+ "measuredAt": "2026-10-07T21:48:35.083Z",+ "meanEstimatedListCostUsd": 0.35649686666666663,+ "meanTokenConsumption": 114200.66666666667,+ "meanToolInvocationCount": 17.666666666666668},{- "experimentId": "claude-sonnet-5-claude-code--baseline",+ "experimentId": "gemini-3-1-pro-preview-opencode--baseline","caseId": "author-006-custom-channel","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 206252,- "measuredAt": "2026-09-02T14:38:33.890Z",- "meanEstimatedListCostUsd": 0.3613244666666667,- "meanTokenConsumption": 5164.666666666667,- "meanToolInvocationCount": 9.666666666666666+ "meanDurationMs": 232728,+ "measuredAt": "2026-10-07T21:48:35.102Z",+ "meanEstimatedListCostUsd": 0.29122646666666663,+ "meanTokenConsumption": 110766.33333333333,+ "meanToolInvocationCount": 16},{- "experimentId": "claude-sonnet-5-claude-code--baseline",+ "experimentId": "gemini-3-1-pro-preview-opencode--baseline","caseId": "author-007-digest-schedule","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 104010,- "measuredAt": "2026-09-02T14:38:33.898Z",- "meanEstimatedListCostUsd": 0.22264466666666669,- "meanTokenConsumption": 1994.3333333333333,- "meanToolInvocationCount": 7.333333333333333+ "meanDurationMs": 152122,+ "measuredAt": "2026-10-07T21:48:35.121Z",+ "meanEstimatedListCostUsd": 0.14784206666666666,+ "meanTokenConsumption": 58074,+ "meanToolInvocationCount": 11.666666666666666},{- "experimentId": "claude-sonnet-5-claude-code--guided",+ "experimentId": "gemini-3-1-pro-preview-opencode--guided","caseId": "author-001-weather-tool","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 130089,- "measuredAt": "2026-09-02T14:34:31.320Z",- "meanEstimatedListCostUsd": 0.40971483333333336,- "meanTokenConsumption": 4909,- "meanToolInvocationCount": 10.666666666666666+ "meanDurationMs": 141789,+ "measuredAt": "2026-10-07T21:52:05.892Z",+ "meanEstimatedListCostUsd": 0.153628,+ "meanTokenConsumption": 54885.333333333336,+ "meanToolInvocationCount": 10},{- "experimentId": "claude-sonnet-5-claude-code--guided",+ "experimentId": "gemini-3-1-pro-preview-opencode--guided","caseId": "author-002-new-project","status": "current",- "passedRuns": 3,+ "passedRuns": 2,"validRuns": 3,- "meanDurationMs": 119473,- "measuredAt": "2026-09-02T14:34:31.340Z",- "meanEstimatedListCostUsd": 0.2871293,- "meanTokenConsumption": 3610.6666666666665,+ "meanDurationMs": 145109,+ "measuredAt": "2026-10-07T21:52:05.906Z",+ "meanEstimatedListCostUsd": 0.12079206666666666,+ "meanTokenConsumption": 46447.333333333336,"meanToolInvocationCount": 10.333333333333334},{- "experimentId": "claude-sonnet-5-claude-code--guided",+ "experimentId": "gemini-3-1-pro-preview-opencode--guided","caseId": "author-003-openapi-connection","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 112748,- "measuredAt": "2026-09-02T14:34:31.354Z",- "meanEstimatedListCostUsd": 0.3381717,- "meanTokenConsumption": 2513,- "meanToolInvocationCount": 8.333333333333334+ "meanDurationMs": 145639,+ "measuredAt": "2026-10-07T21:52:05.916Z",+ "meanEstimatedListCostUsd": 0.14859993333333335,+ "meanTokenConsumption": 51636,+ "meanToolInvocationCount": 9.666666666666666},{- "experimentId": "claude-sonnet-5-claude-code--guided",+ "experimentId": "gemini-3-1-pro-preview-opencode--guided","caseId": "author-004-packaged-skill","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 115311,- "measuredAt": "2026-09-02T14:34:31.365Z",- "meanEstimatedListCostUsd": 0.26473703333333337,- "meanTokenConsumption": 7318.666666666667,- "meanToolInvocationCount": 5+ "meanDurationMs": 141402,+ "measuredAt": "2026-10-07T21:52:05.927Z",+ "meanEstimatedListCostUsd": 0.09085193333333332,+ "meanTokenConsumption": 32130.333333333332,+ "meanToolInvocationCount": 5.666666666666667},{- "experimentId": "claude-sonnet-5-claude-code--guided",+ "experimentId": "gemini-3-1-pro-preview-opencode--guided","caseId": "author-005-conditional-approval","status": "current",- "passedRuns": 3,+ "passedRuns": 2,"validRuns": 3,- "meanDurationMs": 117023,- "measuredAt": "2026-09-02T14:34:31.375Z",- "meanEstimatedListCostUsd": 0.34835236666666664,- "meanTokenConsumption": 2814,- "meanToolInvocationCount": 9+ "meanDurationMs": 183922,+ "measuredAt": "2026-10-07T21:52:05.937Z",+ "meanEstimatedListCostUsd": 0.18213406666666668,+ "meanTokenConsumption": 63521.333333333336,+ "meanToolInvocationCount": 11.333333333333334},{- "experimentId": "claude-sonnet-5-claude-code--guided",+ "experimentId": "gemini-3-1-pro-preview-opencode--guided","caseId": "author-006-custom-channel","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 110065,- "measuredAt": "2026-09-02T14:34:31.383Z",- "meanEstimatedListCostUsd": 0.2831804,- "meanTokenConsumption": 1897.6666666666667,- "meanToolInvocationCount": 6+ "meanDurationMs": 145893,+ "measuredAt": "2026-10-07T21:52:05.951Z",+ "meanEstimatedListCostUsd": 0.12927106666666668,+ "meanTokenConsumption": 43747.333333333336,+ "meanToolInvocationCount": 8.666666666666666},{- "experimentId": "claude-sonnet-5-claude-code--guided",+ "experimentId": "gemini-3-1-pro-preview-opencode--guided","caseId": "author-007-digest-schedule","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 113169,- "measuredAt": "2026-09-02T14:34:31.393Z",- "meanEstimatedListCostUsd": 0.31014963333333334,- "meanTokenConsumption": 2200,- "meanToolInvocationCount": 9+ "meanDurationMs": 145629,+ "measuredAt": "2026-10-07T21:52:05.964Z",+ "meanEstimatedListCostUsd": 0.09688193333333334,+ "meanTokenConsumption": 38146.333333333336,+ "meanToolInvocationCount": 7},{- "experimentId": "glm-5-3-opencode--baseline",+ "experimentId": "gemini-3-8-flash-opencode--baseline","caseId": "author-001-weather-tool","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 229776,- "measuredAt": "2026-09-02T14:24:14.610Z",- "meanEstimatedListCostUsd": 0.13380119999999998,- "meanTokenConsumption": 39736,- "meanToolInvocationCount": 27.666666666666668+ "meanDurationMs": 480431,+ "measuredAt": "2026-10-07T20:32:06.638Z",+ "meanEstimatedListCostUsd": 1.2837646249999999,+ "meanTokenConsumption": 600828.3333333334,+ "meanToolInvocationCount": 109},{- "experimentId": "glm-5-3-opencode--baseline",+ "experimentId": "gemini-3-8-flash-opencode--baseline","caseId": "author-002-new-project","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 253800,- "measuredAt": "2026-09-02T14:24:14.640Z",- "meanEstimatedListCostUsd": 0.04476281333333334,- "meanTokenConsumption": 19797.666666666668,- "meanToolInvocationCount": 13.333333333333334+ "meanDurationMs": 191968,+ "measuredAt": "2026-10-07T20:32:06.669Z",+ "meanEstimatedListCostUsd": 0.12271715,+ "meanTokenConsumption": 117868.33333333333,+ "meanToolInvocationCount": 23},{- "experimentId": "glm-5-3-opencode--baseline",+ "experimentId": "gemini-3-8-flash-opencode--baseline","caseId": "author-003-openapi-connection","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 275839,- "measuredAt": "2026-09-02T14:24:14.653Z",- "meanEstimatedListCostUsd": 0.21305283333333336,- "meanTokenConsumption": 42271,- "meanToolInvocationCount": 43.666666666666664+ "meanDurationMs": 475757,+ "measuredAt": "2026-10-07T20:32:06.679Z",+ "meanEstimatedListCostUsd": 0.7462383749999999,+ "meanTokenConsumption": 458394,+ "meanToolInvocationCount": 85},{- "experimentId": "glm-5-3-opencode--baseline",+ "experimentId": "gemini-3-8-flash-opencode--baseline","caseId": "author-004-packaged-skill","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 237736,- "measuredAt": "2026-09-02T14:24:14.669Z",- "meanEstimatedListCostUsd": 0.06231241333333334,- "meanTokenConsumption": 18199.666666666668,- "meanToolInvocationCount": 18.333333333333332+ "meanDurationMs": 264501,+ "measuredAt": "2026-10-07T20:32:06.705Z",+ "meanEstimatedListCostUsd": 0.326150925,+ "meanTokenConsumption": 212912.33333333334,+ "meanToolInvocationCount": 42.333333333333336},{- "experimentId": "glm-5-3-opencode--baseline",+ "experimentId": "gemini-3-8-flash-opencode--baseline","caseId": "author-005-conditional-approval","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 267634,- "measuredAt": "2026-09-02T14:24:14.679Z",- "meanEstimatedListCostUsd": 0.15534122666666667,- "meanTokenConsumption": 43595,- "meanToolInvocationCount": 35.333333333333336+ "meanDurationMs": 439824,+ "measuredAt": "2026-10-07T20:32:06.731Z",+ "meanEstimatedListCostUsd": 0.5367316999999999,+ "meanTokenConsumption": 309792.6666666667,+ "meanToolInvocationCount": 65.33333333333333},{- "experimentId": "glm-5-3-opencode--baseline",+ "experimentId": "gemini-3-8-flash-opencode--baseline","caseId": "author-006-custom-channel","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 173067,- "measuredAt": "2026-09-02T14:24:14.691Z",- "meanEstimatedListCostUsd": 0.15079565333333333,- "meanTokenConsumption": 55561.666666666664,- "meanToolInvocationCount": 29+ "meanDurationMs": 343512,+ "measuredAt": "2026-10-07T20:32:06.761Z",+ "meanEstimatedListCostUsd": 0.6466183249999999,+ "meanTokenConsumption": 334050.6666666667,+ "meanToolInvocationCount": 64},{- "experimentId": "glm-5-3-opencode--baseline",+ "experimentId": "gemini-3-8-flash-opencode--baseline","caseId": "author-007-digest-schedule","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 150379,- "measuredAt": "2026-09-02T14:24:14.703Z",- "meanEstimatedListCostUsd": 0.061922139999999994,- "meanTokenConsumption": 17250,- "meanToolInvocationCount": 23.333333333333332+ "meanDurationMs": 291390,+ "measuredAt": "2026-10-07T20:32:06.785Z",+ "meanEstimatedListCostUsd": 0.38674920000000007,+ "meanTokenConsumption": 351992.3333333333,+ "meanToolInvocationCount": 43.333333333333336},{- "experimentId": "glm-5-3-opencode--guided",+ "experimentId": "gemini-3-8-flash-opencode--guided","caseId": "author-001-weather-tool","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 192393,- "measuredAt": "2026-09-02T14:24:09.125Z",- "meanEstimatedListCostUsd": 0.05878674,- "meanTokenConsumption": 23335,- "meanToolInvocationCount": 10+ "meanDurationMs": 327592,+ "measuredAt": "2026-10-07T20:39:20.348Z",+ "meanEstimatedListCostUsd": 0.51476285,+ "meanTokenConsumption": 328742,+ "meanToolInvocationCount": 55.666666666666664},{- "experimentId": "glm-5-3-opencode--guided",+ "experimentId": "gemini-3-8-flash-opencode--guided","caseId": "author-002-new-project","status": "current",- "passedRuns": 3,+ "passedRuns": 1,"validRuns": 3,- "meanDurationMs": 314441,- "measuredAt": "2026-09-02T14:24:09.147Z",- "meanEstimatedListCostUsd": 0.04504718666666666,- "meanTokenConsumption": 18359.333333333332,- "meanToolInvocationCount": 14+ "meanDurationMs": 226209,+ "measuredAt": "2026-10-07T20:39:20.384Z",+ "meanEstimatedListCostUsd": 0.1381991,+ "meanTokenConsumption": 117559.66666666667,+ "meanToolInvocationCount": 24.666666666666668},{- "experimentId": "glm-5-3-opencode--guided",+ "experimentId": "gemini-3-8-flash-opencode--guided","caseId": "author-003-openapi-connection","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 141942,- "measuredAt": "2026-09-02T14:24:09.165Z",- "meanEstimatedListCostUsd": 0.062038473333333344,- "meanTokenConsumption": 22144.666666666668,- "meanToolInvocationCount": 15+ "meanDurationMs": 306740,+ "measuredAt": "2026-10-07T20:39:20.395Z",+ "meanEstimatedListCostUsd": 0.4550721749999999,+ "meanTokenConsumption": 308868.6666666667,+ "meanToolInvocationCount": 57},{- "experimentId": "glm-5-3-opencode--guided",+ "experimentId": "gemini-3-8-flash-opencode--guided","caseId": "author-004-packaged-skill","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 140163,- "measuredAt": "2026-09-02T14:24:09.179Z",- "meanEstimatedListCostUsd": 0.04043716,- "meanTokenConsumption": 14312.666666666666,- "meanToolInvocationCount": 10.666666666666666+ "meanDurationMs": 267083,+ "measuredAt": "2026-10-07T20:39:20.424Z",+ "meanEstimatedListCostUsd": 0.21935792499999998,+ "meanTokenConsumption": 151813,+ "meanToolInvocationCount": 33.666666666666664},{- "experimentId": "glm-5-3-opencode--guided",+ "experimentId": "gemini-3-8-flash-opencode--guided","caseId": "author-005-conditional-approval","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 129477,- "measuredAt": "2026-09-02T14:24:09.190Z",- "meanEstimatedListCostUsd": 0.04672530666666667,- "meanTokenConsumption": 21094.666666666668,- "meanToolInvocationCount": 8.666666666666666+ "meanDurationMs": 248208,+ "measuredAt": "2026-10-07T20:39:20.449Z",+ "meanEstimatedListCostUsd": 0.30564869999999994,+ "meanTokenConsumption": 204861.33333333334,+ "meanToolInvocationCount": 42},{- "experimentId": "glm-5-3-opencode--guided",+ "experimentId": "gemini-3-8-flash-opencode--guided","caseId": "author-006-custom-channel","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 117801,- "measuredAt": "2026-09-02T14:24:09.200Z",- "meanEstimatedListCostUsd": 0.036290826666666665,- "meanTokenConsumption": 13584.333333333334,- "meanToolInvocationCount": 10.666666666666666+ "meanDurationMs": 178534,+ "measuredAt": "2026-10-07T20:39:20.489Z",+ "meanEstimatedListCostUsd": 0.18658105,+ "meanTokenConsumption": 156321,+ "meanToolInvocationCount": 29},{- "experimentId": "glm-5-3-opencode--guided",+ "experimentId": "gemini-3-8-flash-opencode--guided","caseId": "author-007-digest-schedule","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 117068,- "measuredAt": "2026-09-02T14:24:09.209Z",- "meanEstimatedListCostUsd": 0.024231453333333333,- "meanTokenConsumption": 7749.333333333333,- "meanToolInvocationCount": 8.333333333333334+ "meanDurationMs": 190950,+ "measuredAt": "2026-10-07T20:39:20.518Z",+ "meanEstimatedListCostUsd": 0.17541602499999998,+ "meanTokenConsumption": 133149,+ "meanToolInvocationCount": 27.666666666666668},{- "experimentId": "claude-opus-5-claude-code--baseline",+ "experimentId": "minimax-m3-opencode--baseline","caseId": "author-001-weather-tool","status": "current",- "passedRuns": 3,+ "passedRuns": 2,"validRuns": 3,- "meanDurationMs": 155202,- "measuredAt": "2026-09-02T14:35:36.609Z",- "meanEstimatedListCostUsd": 1.1599598333333334,- "meanTokenConsumption": 10714.666666666666,- "meanToolInvocationCount": 17.333333333333332+ "meanDurationMs": 137827,+ "measuredAt": "2026-10-07T19:57:38.058Z",+ "meanEstimatedListCostUsd": 0.03919382,+ "meanTokenConsumption": 40222,+ "meanToolInvocationCount": 29.333333333333332},{- "experimentId": "claude-opus-5-claude-code--baseline",+ "experimentId": "minimax-m3-opencode--baseline","caseId": "author-002-new-project","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 124976,- "measuredAt": "2026-09-02T14:35:36.628Z",- "meanEstimatedListCostUsd": 0.8593296666666667,- "meanTokenConsumption": 3845.3333333333335,- "meanToolInvocationCount": 10.333333333333334+ "meanDurationMs": 142346,+ "measuredAt": "2026-10-07T19:57:38.073Z",+ "meanEstimatedListCostUsd": 0.02446828,+ "meanTokenConsumption": 24358,+ "meanToolInvocationCount": 17.333333333333332},{- "experimentId": "claude-opus-5-claude-code--baseline",+ "experimentId": "minimax-m3-opencode--baseline","caseId": "author-003-openapi-connection","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 160123,- "measuredAt": "2026-09-02T14:35:36.643Z",- "meanEstimatedListCostUsd": 1.3671014166666666,- "meanTokenConsumption": 7793.333333333333,- "meanToolInvocationCount": 19+ "meanDurationMs": 145218,+ "measuredAt": "2026-10-07T19:57:38.085Z",+ "meanEstimatedListCostUsd": 0.04590585999999999,+ "meanTokenConsumption": 53114.666666666664,+ "meanToolInvocationCount": 27.666666666666668},{- "experimentId": "claude-opus-5-claude-code--baseline",+ "experimentId": "minimax-m3-opencode--baseline","caseId": "author-004-packaged-skill","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 181217,- "measuredAt": "2026-09-02T14:35:36.656Z",- "meanEstimatedListCostUsd": 1.1116483333333333,- "meanTokenConsumption": 16371.666666666666,- "meanToolInvocationCount": 12.666666666666666+ "meanDurationMs": 154028,+ "measuredAt": "2026-10-07T19:57:38.099Z",+ "meanEstimatedListCostUsd": 0.03372036,+ "meanTokenConsumption": 35014.333333333336,+ "meanToolInvocationCount": 23},{- "experimentId": "claude-opus-5-claude-code--baseline",+ "experimentId": "minimax-m3-opencode--baseline","caseId": "author-005-conditional-approval","status": "current",- "passedRuns": 3,+ "passedRuns": 2,"validRuns": 3,- "meanDurationMs": 153333,- "measuredAt": "2026-09-02T14:35:36.667Z",- "meanEstimatedListCostUsd": 1.03157675,- "meanTokenConsumption": 6217.333333333333,- "meanToolInvocationCount": 17.333333333333332+ "meanDurationMs": 159346,+ "measuredAt": "2026-10-07T19:57:38.112Z",+ "meanEstimatedListCostUsd": 0.037709999999999994,+ "meanTokenConsumption": 38013.333333333336,+ "meanToolInvocationCount": 27.666666666666668},{- "experimentId": "claude-opus-5-claude-code--baseline",+ "experimentId": "minimax-m3-opencode--baseline","caseId": "author-006-custom-channel","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 136155,- "measuredAt": "2026-09-02T14:35:36.676Z",- "meanEstimatedListCostUsd": 0.9316169166666667,- "meanTokenConsumption": 4836.666666666667,- "meanToolInvocationCount": 16.666666666666668+ "meanDurationMs": 196082,+ "measuredAt": "2026-10-07T19:57:38.126Z",+ "meanEstimatedListCostUsd": 0.09671368000000001,+ "meanTokenConsumption": 77110,+ "meanToolInvocationCount": 47},{- "experimentId": "claude-opus-5-claude-code--baseline",+ "experimentId": "minimax-m3-opencode--baseline","caseId": "author-007-digest-schedule","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 120633,- "measuredAt": "2026-09-02T14:35:36.685Z",- "meanEstimatedListCostUsd": 0.5067333333333334,- "meanTokenConsumption": 3232.6666666666665,- "meanToolInvocationCount": 9.333333333333334+ "meanDurationMs": 139910,+ "measuredAt": "2026-10-07T19:57:38.143Z",+ "meanEstimatedListCostUsd": 0.017940519999999998,+ "meanTokenConsumption": 18308.333333333332,+ "meanToolInvocationCount": 19},{- "experimentId": "claude-opus-5-claude-code--guided",+ "experimentId": "minimax-m3-opencode--guided","caseId": "author-001-weather-tool","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 147978,- "measuredAt": "2026-09-02T14:34:53.900Z",- "meanEstimatedListCostUsd": 1.0020195,- "meanTokenConsumption": 11004.333333333334,- "meanToolInvocationCount": 11+ "meanDurationMs": 120197,+ "measuredAt": "2026-10-07T20:00:50.683Z",+ "meanEstimatedListCostUsd": 0.018265399999999998,+ "meanTokenConsumption": 29428.666666666668,+ "meanToolInvocationCount": 13.666666666666666},{- "experimentId": "claude-opus-5-claude-code--guided",+ "experimentId": "minimax-m3-opencode--guided","caseId": "author-002-new-project","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 138714,- "measuredAt": "2026-09-02T14:34:53.929Z",- "meanEstimatedListCostUsd": 1.0472271666666666,- "meanTokenConsumption": 8633.666666666666,- "meanToolInvocationCount": 14.333333333333334+ "meanDurationMs": 122122,+ "measuredAt": "2026-10-07T20:00:50.696Z",+ "meanEstimatedListCostUsd": 0.01290936,+ "meanTokenConsumption": 14803,+ "meanToolInvocationCount": 12},{- "experimentId": "claude-opus-5-claude-code--guided",+ "experimentId": "minimax-m3-opencode--guided","caseId": "author-003-openapi-connection","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 137520,- "measuredAt": "2026-09-02T14:34:53.944Z",- "meanEstimatedListCostUsd": 0.7869943333333334,- "meanTokenConsumption": 5193.666666666667,- "meanToolInvocationCount": 14+ "meanDurationMs": 127857,+ "measuredAt": "2026-10-07T20:00:50.706Z",+ "meanEstimatedListCostUsd": 0.024050899999999997,+ "meanTokenConsumption": 32894,+ "meanToolInvocationCount": 15},{- "experimentId": "claude-opus-5-claude-code--guided",+ "experimentId": "minimax-m3-opencode--guided","caseId": "author-004-packaged-skill","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 153316,- "measuredAt": "2026-09-02T14:34:53.956Z",- "meanEstimatedListCostUsd": 0.6677339166666667,- "meanTokenConsumption": 10291.666666666666,- "meanToolInvocationCount": 7.666666666666667+ "meanDurationMs": 148211,+ "measuredAt": "2026-10-07T20:00:50.716Z",+ "meanEstimatedListCostUsd": 0.010811939999999999,+ "meanTokenConsumption": 9126.333333333334,+ "meanToolInvocationCount": 10},{- "experimentId": "claude-opus-5-claude-code--guided",+ "experimentId": "minimax-m3-opencode--guided","caseId": "author-005-conditional-approval","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 116393,- "measuredAt": "2026-09-02T14:34:53.964Z",- "meanEstimatedListCostUsd": 0.5910709166666667,- "meanTokenConsumption": 2916.6666666666665,- "meanToolInvocationCount": 8.333333333333334+ "meanDurationMs": 148887,+ "measuredAt": "2026-10-07T20:00:50.725Z",+ "meanEstimatedListCostUsd": 0.0288123,+ "meanTokenConsumption": 39894,+ "meanToolInvocationCount": 19},{- "experimentId": "claude-opus-5-claude-code--guided",+ "experimentId": "minimax-m3-opencode--guided","caseId": "author-006-custom-channel","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 107570,- "measuredAt": "2026-09-02T14:34:53.973Z",- "meanEstimatedListCostUsd": 0.4942074166666666,- "meanTokenConsumption": 2636,- "meanToolInvocationCount": 6.666666666666667+ "meanDurationMs": 119587,+ "measuredAt": "2026-10-07T20:00:50.736Z",+ "meanEstimatedListCostUsd": 0.01628684,+ "meanTokenConsumption": 22547.333333333332,+ "meanToolInvocationCount": 9.666666666666666},{- "experimentId": "claude-opus-5-claude-code--guided",+ "experimentId": "minimax-m3-opencode--guided","caseId": "author-007-digest-schedule","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 114311,- "measuredAt": "2026-09-02T14:34:53.981Z",- "meanEstimatedListCostUsd": 0.46051358333333337,- "meanTokenConsumption": 2592.6666666666665,- "meanToolInvocationCount": 7.666666666666667+ "meanDurationMs": 116089,+ "measuredAt": "2026-10-07T20:00:50.745Z",+ "meanEstimatedListCostUsd": 0.01088158,+ "meanTokenConsumption": 13971.666666666666,+ "meanToolInvocationCount": 10},{- "experimentId": "gemini-3-1-pro-preview-opencode--baseline",+ "experimentId": "kimi-k2-7-code-opencode--baseline","caseId": "author-001-weather-tool","status": "current",- "passedRuns": 0,+ "passedRuns": 3,"validRuns": 3,- "meanDurationMs": 54876,- "measuredAt": "2026-08-28T19:20:40.000Z",- "meanEstimatedListCostUsd": 0.11818266666666667,- "meanTokenConsumption": 45096.333333333336,- "meanToolInvocationCount": 9.666666666666666+ "meanDurationMs": 217800,+ "measuredAt": "2026-10-07T19:45:29.923Z",+ "meanEstimatedListCostUsd": 0.14899567666666666,+ "meanTokenConsumption": 34828.666666666664,+ "meanToolInvocationCount": 32},{- "experimentId": "gemini-3-1-pro-preview-opencode--baseline",+ "experimentId": "kimi-k2-7-code-opencode--baseline","caseId": "author-002-new-project","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 82415,- "measuredAt": "2026-08-28T19:26:24.000Z",- "meanEstimatedListCostUsd": 0.11070399999999998,- "meanTokenConsumption": 51112,- "meanToolInvocationCount": 11.666666666666666+ "meanDurationMs": 174725,+ "measuredAt": "2026-10-07T19:45:29.947Z",+ "meanEstimatedListCostUsd": 0.06421423000000001,+ "meanTokenConsumption": 19656.666666666668,+ "meanToolInvocationCount": 22.666666666666668},{- "experimentId": "gemini-3-1-pro-preview-opencode--baseline",+ "experimentId": "kimi-k2-7-code-opencode--baseline","caseId": "author-003-openapi-connection","status": "current",- "passedRuns": 1,+ "passedRuns": 3,"validRuns": 3,- "meanDurationMs": 102903,- "measuredAt": "2026-08-28T19:26:24.000Z",- "meanEstimatedListCostUsd": 0.43870533333333334,- "meanTokenConsumption": 211094.33333333334,- "meanToolInvocationCount": 28+ "meanDurationMs": 172017,+ "measuredAt": "2026-10-07T19:45:29.963Z",+ "meanEstimatedListCostUsd": 0.12660882333333334,+ "meanTokenConsumption": 35982.666666666664,+ "meanToolInvocationCount": 25.333333333333332},{- "experimentId": "gemini-3-1-pro-preview-opencode--baseline",+ "experimentId": "kimi-k2-7-code-opencode--baseline","caseId": "author-004-packaged-skill","status": "current",- "passedRuns": 2,+ "passedRuns": 3,"validRuns": 3,- "meanDurationMs": 33942,- "measuredAt": "2026-08-28T19:20:40.000Z",- "meanEstimatedListCostUsd": 0.07116533333333334,- "meanTokenConsumption": 26049.333333333332,- "meanToolInvocationCount": 5.333333333333333+ "meanDurationMs": 176932,+ "measuredAt": "2026-10-07T19:45:29.978Z",+ "meanEstimatedListCostUsd": 0.09377714666666666,+ "meanTokenConsumption": 27106.333333333332,+ "meanToolInvocationCount": 25.666666666666668},{- "experimentId": "gemini-3-1-pro-preview-opencode--baseline",+ "experimentId": "kimi-k2-7-code-opencode--baseline","caseId": "author-005-conditional-approval","status": "current","passedRuns": 2,"validRuns": 3,- "meanDurationMs": 227315,- "measuredAt": "2026-08-28T19:20:40.000Z",- "meanEstimatedListCostUsd": 0.5367073333333333,- "meanTokenConsumption": 251920.33333333334,- "meanToolInvocationCount": 42.666666666666664+ "meanDurationMs": 189532,+ "measuredAt": "2026-10-07T19:45:29.991Z",+ "meanEstimatedListCostUsd": 0.13552957000000002,+ "meanTokenConsumption": 37226,+ "meanToolInvocationCount": 25.333333333333332},{- "experimentId": "gemini-3-1-pro-preview-opencode--baseline",+ "experimentId": "kimi-k2-7-code-opencode--baseline","caseId": "author-006-custom-channel","status": "current",- "passedRuns": 1,+ "passedRuns": 2,"validRuns": 3,- "meanDurationMs": 133303,- "measuredAt": "2026-08-28T19:20:40.000Z",- "meanEstimatedListCostUsd": 0.467548,- "meanTokenConsumption": 225615.66666666666,- "meanToolInvocationCount": 30.666666666666668+ "meanDurationMs": 248066,+ "measuredAt": "2026-10-07T19:45:30.008Z",+ "meanEstimatedListCostUsd": 0.13744147333333334,+ "meanTokenConsumption": 38285,+ "meanToolInvocationCount": 28.666666666666668},{- "experimentId": "gemini-3-1-pro-preview-opencode--baseline",+ "experimentId": "kimi-k2-7-code-opencode--baseline","caseId": "author-007-digest-schedule","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 61813,- "measuredAt": "2026-08-28T19:20:40.000Z",- "meanEstimatedListCostUsd": 0.212552,- "meanTokenConsumption": 101059.33333333333,- "meanToolInvocationCount": 15+ "meanDurationMs": 187907,+ "measuredAt": "2026-10-07T19:45:30.021Z",+ "meanEstimatedListCostUsd": 0.06076566333333333,+ "meanTokenConsumption": 19027.666666666668,+ "meanToolInvocationCount": 25.666666666666668},{- "experimentId": "gemini-3-1-pro-preview-opencode--guided",+ "experimentId": "kimi-k2-7-code-opencode--guided","caseId": "author-001-weather-tool","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 37112,- "measuredAt": "2026-08-28T19:17:42.000Z",- "meanEstimatedListCostUsd": 0.08827333333333333,- "meanTokenConsumption": 40508.333333333336,- "meanToolInvocationCount": 7.666666666666667+ "meanDurationMs": 181961,+ "measuredAt": "2026-10-07T19:49:07.568Z",+ "meanEstimatedListCostUsd": 0.08877174666666666,+ "meanTokenConsumption": 26044.333333333332,+ "meanToolInvocationCount": 20},{- "experimentId": "gemini-3-1-pro-preview-opencode--guided",+ "experimentId": "kimi-k2-7-code-opencode--guided","caseId": "author-002-new-project","status": "current",- "passedRuns": 0,+ "passedRuns": 3,"validRuns": 3,- "meanDurationMs": 260182,- "measuredAt": "2026-08-28T19:17:42.000Z",- "meanEstimatedListCostUsd": 0.237908,- "meanTokenConsumption": 109484,- "meanToolInvocationCount": 28.333333333333332+ "meanDurationMs": 139537,+ "measuredAt": "2026-10-07T19:49:07.579Z",+ "meanEstimatedListCostUsd": 0.04228683333333333,+ "meanTokenConsumption": 14718,+ "meanToolInvocationCount": 17.333333333333332},{- "experimentId": "gemini-3-1-pro-preview-opencode--guided",+ "experimentId": "kimi-k2-7-code-opencode--guided","caseId": "author-003-openapi-connection","status": "current",- "passedRuns": 0,+ "passedRuns": 3,"validRuns": 3,- "meanDurationMs": 201322,- "measuredAt": "2026-08-28T19:17:42.000Z",- "meanEstimatedListCostUsd": 0.3107266666666667,- "meanTokenConsumption": 132131.66666666666,- "meanToolInvocationCount": 34.333333333333336+ "meanDurationMs": 185254,+ "measuredAt": "2026-10-07T19:49:07.588Z",+ "meanEstimatedListCostUsd": 0.04005971333333333,+ "meanTokenConsumption": 14683.333333333334,+ "meanToolInvocationCount": 12.666666666666666},{- "experimentId": "gemini-3-1-pro-preview-opencode--guided",+ "experimentId": "kimi-k2-7-code-opencode--guided","caseId": "author-004-packaged-skill","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 41440,- "measuredAt": "2026-08-28T19:17:42.000Z",- "meanEstimatedListCostUsd": 0.08369866666666666,- "meanTokenConsumption": 34357.666666666664,- "meanToolInvocationCount": 8.666666666666666+ "meanDurationMs": 153696,+ "measuredAt": "2026-10-07T19:49:07.597Z",+ "meanEstimatedListCostUsd": 0.03293041666666666,+ "meanTokenConsumption": 10946,+ "meanToolInvocationCount": 11},{- "experimentId": "gemini-3-1-pro-preview-opencode--guided",+ "experimentId": "kimi-k2-7-code-opencode--guided","caseId": "author-005-conditional-approval","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 33301,- "measuredAt": "2026-08-28T19:17:42.000Z",- "meanEstimatedListCostUsd": 0.09258866666666667,- "meanTokenConsumption": 42437.666666666664,- "meanToolInvocationCount": 7.333333333333333+ "meanDurationMs": 159597,+ "measuredAt": "2026-10-07T19:49:07.606Z",+ "meanEstimatedListCostUsd": 0.059909359999999995,+ "meanTokenConsumption": 21384.666666666668,+ "meanToolInvocationCount": 15.333333333333334},{- "experimentId": "gemini-3-1-pro-preview-opencode--guided",+ "experimentId": "kimi-k2-7-code-opencode--guided","caseId": "author-006-custom-channel","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 34493,- "measuredAt": "2026-08-28T19:17:42.000Z",- "meanEstimatedListCostUsd": 0.09662666666666668,- "meanTokenConsumption": 45305,- "meanToolInvocationCount": 7.333333333333333+ "meanDurationMs": 142256,+ "measuredAt": "2026-10-07T19:49:07.619Z",+ "meanEstimatedListCostUsd": 0.040052736666666665,+ "meanTokenConsumption": 15506,+ "meanToolInvocationCount": 12.333333333333334},{- "experimentId": "gemini-3-1-pro-preview-opencode--guided",+ "experimentId": "kimi-k2-7-code-opencode--guided","caseId": "author-007-digest-schedule","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 33257,- "measuredAt": "2026-08-28T19:17:42.000Z",- "meanEstimatedListCostUsd": 0.059873333333333334,- "meanTokenConsumption": 27681.666666666668,- "meanToolInvocationCount": 4.666666666666667+ "meanDurationMs": 139483,+ "measuredAt": "2026-10-07T19:49:07.629Z",+ "meanEstimatedListCostUsd": 0.04300942,+ "meanTokenConsumption": 13041.333333333334,+ "meanToolInvocationCount": 12.333333333333334},{- "experimentId": "gemini-3-8-flash-opencode--baseline",+ "experimentId": "claude-opus-5-5-high-claude-code--baseline","caseId": "author-001-weather-tool","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 368613,- "measuredAt": "2026-09-15T16:50:43.236Z",- "meanEstimatedListCostUsd": 0.398698,- "meanTokenConsumption": 256875.66666666666,- "meanToolInvocationCount": 53.333333333333336+ "meanDurationMs": 148854,+ "measuredAt": "2026-10-07T18:15:47.177Z",+ "meanEstimatedListCostUsd": 0.5982536666666667,+ "meanTokenConsumption": 8172.666666666667,+ "meanToolInvocationCount": 11.666666666666666},{- "experimentId": "gemini-3-8-flash-opencode--baseline",+ "experimentId": "claude-opus-5-5-high-claude-code--baseline","caseId": "author-002-new-project","status": "current",- "passedRuns": 2,+ "passedRuns": 3,"validRuns": 3,- "meanDurationMs": 181428,- "measuredAt": "2026-09-15T16:50:43.274Z",- "meanEstimatedListCostUsd": 0.07038494999999999,- "meanTokenConsumption": 75876.33333333333,- "meanToolInvocationCount": 15.666666666666666+ "meanDurationMs": 131272,+ "measuredAt": "2026-10-07T18:15:47.196Z",+ "meanEstimatedListCostUsd": 0.22304533333333335,+ "meanTokenConsumption": 2829.3333333333335,+ "meanToolInvocationCount": 8},{- "experimentId": "gemini-3-8-flash-opencode--baseline",+ "experimentId": "claude-opus-5-5-high-claude-code--baseline","caseId": "author-003-openapi-connection","status": "current",- "passedRuns": 1,+ "passedRuns": 3,"validRuns": 3,- "meanDurationMs": 730583,- "measuredAt": "2026-09-15T16:50:43.288Z",- "meanEstimatedListCostUsd": 0.35561909999999997,- "meanTokenConsumption": 235963,- "meanToolInvocationCount": 56+ "meanDurationMs": 169954,+ "measuredAt": "2026-10-07T18:15:47.207Z",+ "meanEstimatedListCostUsd": 0.33688120000000005,+ "meanTokenConsumption": 4193.333333333333,+ "meanToolInvocationCount": 11.666666666666666},{- "experimentId": "gemini-3-8-flash-opencode--baseline",+ "experimentId": "claude-opus-5-5-high-claude-code--baseline","caseId": "author-004-packaged-skill","status": "current",- "passedRuns": 2,+ "passedRuns": 3,"validRuns": 3,- "meanDurationMs": 273870,- "measuredAt": "2026-09-15T16:50:43.297Z",- "meanEstimatedListCostUsd": 0.252016075,- "meanTokenConsumption": 189061,- "meanToolInvocationCount": 40.333333333333336+ "meanDurationMs": 195780,+ "measuredAt": "2026-10-07T18:15:47.217Z",+ "meanEstimatedListCostUsd": 0.44434179999999995,+ "meanTokenConsumption": 9915,+ "meanToolInvocationCount": 10.666666666666666},{- "experimentId": "gemini-3-8-flash-opencode--baseline",+ "experimentId": "claude-opus-5-5-high-claude-code--baseline","caseId": "author-005-conditional-approval","status": "current",- "passedRuns": 2,+ "passedRuns": 3,"validRuns": 3,- "meanDurationMs": 208514,- "measuredAt": "2026-09-15T16:50:43.312Z",- "meanEstimatedListCostUsd": 0.3936613499999999,- "meanTokenConsumption": 330781.3333333333,- "meanToolInvocationCount": 45+ "meanDurationMs": 158936,+ "measuredAt": "2026-10-07T18:15:47.228Z",+ "meanEstimatedListCostUsd": 0.5887976,+ "meanTokenConsumption": 7503.666666666667,+ "meanToolInvocationCount": 12.333333333333334},{- "experimentId": "gemini-3-8-flash-opencode--baseline",+ "experimentId": "claude-opus-5-5-high-claude-code--baseline","caseId": "author-006-custom-channel","status": "current",- "passedRuns": 2,+ "passedRuns": 3,"validRuns": 3,- "meanDurationMs": 463841,- "measuredAt": "2026-09-15T16:50:43.333Z",- "meanEstimatedListCostUsd": 0.349989725,- "meanTokenConsumption": 239936.33333333334,- "meanToolInvocationCount": 41.333333333333336+ "meanDurationMs": 137260,+ "measuredAt": "2026-10-07T18:15:47.239Z",+ "meanEstimatedListCostUsd": 0.3630187333333333,+ "meanTokenConsumption": 3004,+ "meanToolInvocationCount": 7.666666666666667},{- "experimentId": "gemini-3-8-flash-opencode--baseline",+ "experimentId": "claude-opus-5-5-high-claude-code--baseline","caseId": "author-007-digest-schedule","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 307880,- "measuredAt": "2026-09-15T16:50:43.349Z",- "meanEstimatedListCostUsd": 0.8133175250000001,- "meanTokenConsumption": 852160.3333333334,- "meanToolInvocationCount": 54.666666666666664+ "meanDurationMs": 135953,+ "measuredAt": "2026-10-07T18:15:47.248Z",+ "meanEstimatedListCostUsd": 0.24009426666666667,+ "meanTokenConsumption": 2781.6666666666665,+ "meanToolInvocationCount": 7.666666666666667},{- "experimentId": "gemini-3-8-flash-opencode--guided",+ "experimentId": "claude-opus-5-5-high-claude-code--guided","caseId": "author-001-weather-tool","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 226490,- "measuredAt": "2026-09-15T16:48:47.804Z",- "meanEstimatedListCostUsd": 0.216276975,- "meanTokenConsumption": 176030.66666666666,- "meanToolInvocationCount": 35.333333333333336+ "meanDurationMs": 152041,+ "measuredAt": "2026-10-07T18:19:58.247Z",+ "meanEstimatedListCostUsd": 0.37882766666666673,+ "meanTokenConsumption": 6229.333333333333,+ "meanToolInvocationCount": 6.666666666666667},{- "experimentId": "gemini-3-8-flash-opencode--guided",+ "experimentId": "claude-opus-5-5-high-claude-code--guided","caseId": "author-002-new-project","status": "current",- "passedRuns": 1,+ "passedRuns": 3,"validRuns": 3,- "meanDurationMs": 212565,- "measuredAt": "2026-09-15T16:48:47.822Z",- "meanEstimatedListCostUsd": 0.056899675000000004,- "meanTokenConsumption": 58487.333333333336,- "meanToolInvocationCount": 14.333333333333334+ "meanDurationMs": 179646,+ "measuredAt": "2026-10-07T18:19:58.259Z",+ "meanEstimatedListCostUsd": 0.2818064,+ "meanTokenConsumption": 3093.3333333333335,+ "meanToolInvocationCount": 9},{- "experimentId": "gemini-3-8-flash-opencode--guided",+ "experimentId": "claude-opus-5-5-high-claude-code--guided","caseId": "author-003-openapi-connection","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 292952,- "measuredAt": "2026-09-15T16:48:47.831Z",- "meanEstimatedListCostUsd": 0.3152235,- "meanTokenConsumption": 263335.3333333333,- "meanToolInvocationCount": 40.333333333333336+ "meanDurationMs": 132152,+ "measuredAt": "2026-10-07T18:19:58.271Z",+ "meanEstimatedListCostUsd": 0.26926126666666667,+ "meanTokenConsumption": 2762.3333333333335,+ "meanToolInvocationCount": 5.666666666666667},{- "experimentId": "gemini-3-8-flash-opencode--guided",+ "experimentId": "claude-opus-5-5-high-claude-code--guided","caseId": "author-004-packaged-skill","status": "current",- "passedRuns": 2,+ "passedRuns": 3,"validRuns": 3,- "meanDurationMs": 129278,- "measuredAt": "2026-09-15T16:48:47.847Z",- "meanEstimatedListCostUsd": 0.050644475,- "meanTokenConsumption": 56305.666666666664,- "meanToolInvocationCount": 9.333333333333334+ "meanDurationMs": 168608,+ "measuredAt": "2026-10-07T18:19:58.280Z",+ "meanEstimatedListCostUsd": 0.2669548,+ "meanTokenConsumption": 5441,+ "meanToolInvocationCount": 8},{- "experimentId": "gemini-3-8-flash-opencode--guided",+ "experimentId": "claude-opus-5-5-high-claude-code--guided","caseId": "author-005-conditional-approval","status": "current",- "passedRuns": 1,+ "passedRuns": 3,"validRuns": 3,- "meanDurationMs": 648386,- "measuredAt": "2026-09-15T16:48:47.859Z",- "meanEstimatedListCostUsd": 0.612731225,- "meanTokenConsumption": 449588,- "meanToolInvocationCount": 57.333333333333336+ "meanDurationMs": 152304,+ "measuredAt": "2026-10-07T18:19:58.292Z",+ "meanEstimatedListCostUsd": 0.2563342,+ "meanTokenConsumption": 2932.3333333333335,+ "meanToolInvocationCount": 7},{- "experimentId": "gemini-3-8-flash-opencode--guided",+ "experimentId": "claude-opus-5-5-high-claude-code--guided","caseId": "author-006-custom-channel","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 225166,- "measuredAt": "2026-09-15T16:48:47.880Z",- "meanEstimatedListCostUsd": 0.265912825,- "meanTokenConsumption": 260007.33333333334,- "meanToolInvocationCount": 29+ "meanDurationMs": 142903,+ "measuredAt": "2026-10-07T18:19:58.304Z",+ "meanEstimatedListCostUsd": 0.20084280000000002,+ "meanTokenConsumption": 2039,+ "meanToolInvocationCount": 5},{- "experimentId": "gemini-3-8-flash-opencode--guided",+ "experimentId": "claude-opus-5-5-high-claude-code--guided","caseId": "author-007-digest-schedule","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 133720,- "measuredAt": "2026-09-15T16:48:47.896Z",- "meanEstimatedListCostUsd": 0.06414665000000001,- "meanTokenConsumption": 59535.666666666664,- "meanToolInvocationCount": 12.666666666666666+ "meanDurationMs": 132872,+ "measuredAt": "2026-10-07T18:19:58.316Z",+ "meanEstimatedListCostUsd": 0.14850560000000002,+ "meanTokenConsumption": 1478,+ "meanToolInvocationCount": 5.666666666666667},{- "experimentId": "minimax-m3-opencode--baseline",+ "experimentId": "gpt-6-sol-high-codex--baseline","caseId": "author-001-weather-tool","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 139687,- "measuredAt": "2026-09-15T16:42:26.677Z",- "meanEstimatedListCostUsd": 0.028696399999999997,- "meanTokenConsumption": 36094,- "meanToolInvocationCount": 23.333333333333332+ "meanDurationMs": 235603,+ "measuredAt": "2026-10-07T18:49:55.511Z",+ "meanEstimatedListCostUsd": 0.40047706666666666,+ "meanTokenConsumption": 56350,+ "meanToolInvocationCount": 25.666666666666668},{- "experimentId": "minimax-m3-opencode--baseline",+ "experimentId": "gpt-6-sol-high-codex--baseline","caseId": "author-002-new-project","status": "current",- "passedRuns": 1,+ "passedRuns": 3,"validRuns": 3,- "meanDurationMs": 336656,- "measuredAt": "2026-09-15T16:42:26.709Z",- "meanEstimatedListCostUsd": 0.015437459999999998,- "meanTokenConsumption": 19649,- "meanToolInvocationCount": 15+ "meanDurationMs": 155467,+ "measuredAt": "2026-10-07T18:49:55.529Z",+ "meanEstimatedListCostUsd": 0.22136066666666665,+ "meanTokenConsumption": 31200.666666666668,+ "meanToolInvocationCount": 19.666666666666668},{- "experimentId": "minimax-m3-opencode--baseline",+ "experimentId": "gpt-6-sol-high-codex--baseline","caseId": "author-003-openapi-connection","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 149683,- "measuredAt": "2026-09-15T16:42:26.715Z",- "meanEstimatedListCostUsd": 0.0360468,- "meanTokenConsumption": 44776.666666666664,- "meanToolInvocationCount": 26.333333333333332+ "meanDurationMs": 140201,+ "measuredAt": "2026-10-07T18:49:55.541Z",+ "meanEstimatedListCostUsd": 0.24634376666666669,+ "meanTokenConsumption": 42107.333333333336,+ "meanToolInvocationCount": 18},{- "experimentId": "minimax-m3-opencode--baseline",+ "experimentId": "gpt-6-sol-high-codex--baseline","caseId": "author-004-packaged-skill","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 169255,- "measuredAt": "2026-09-15T16:42:26.734Z",- "meanEstimatedListCostUsd": 0.022754239999999995,- "meanTokenConsumption": 24665.333333333332,- "meanToolInvocationCount": 17.333333333333332+ "meanDurationMs": 147782,+ "measuredAt": "2026-10-07T18:49:55.561Z",+ "meanEstimatedListCostUsd": 0.23484953333333333,+ "meanTokenConsumption": 37910,+ "meanToolInvocationCount": 18.666666666666668},{- "experimentId": "minimax-m3-opencode--baseline",+ "experimentId": "gpt-6-sol-high-codex--baseline","caseId": "author-005-conditional-approval","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 152482,- "measuredAt": "2026-09-15T16:42:26.748Z",- "meanEstimatedListCostUsd": 0.037158,- "meanTokenConsumption": 47089.666666666664,- "meanToolInvocationCount": 24.666666666666668+ "meanDurationMs": 184292,+ "measuredAt": "2026-10-07T18:49:55.570Z",+ "meanEstimatedListCostUsd": 0.3206735333333333,+ "meanTokenConsumption": 49776,+ "meanToolInvocationCount": 22.333333333333332},{- "experimentId": "minimax-m3-opencode--baseline",+ "experimentId": "gpt-6-sol-high-codex--baseline","caseId": "author-006-custom-channel","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 175652,- "measuredAt": "2026-09-15T16:42:26.764Z",- "meanEstimatedListCostUsd": 0.10778759999999998,- "meanTokenConsumption": 91708.33333333333,- "meanToolInvocationCount": 45.333333333333336+ "meanDurationMs": 141113,+ "measuredAt": "2026-10-07T18:49:55.583Z",+ "meanEstimatedListCostUsd": 0.24225123333333332,+ "meanTokenConsumption": 41215.666666666664,+ "meanToolInvocationCount": 17.333333333333332},{- "experimentId": "minimax-m3-opencode--baseline",+ "experimentId": "gpt-6-sol-high-codex--baseline","caseId": "author-007-digest-schedule","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 142420,- "measuredAt": "2026-09-15T16:42:26.784Z",- "meanEstimatedListCostUsd": 0.018700320000000003,- "meanTokenConsumption": 22837.333333333332,- "meanToolInvocationCount": 17.333333333333332+ "meanDurationMs": 143502,+ "measuredAt": "2026-10-07T18:49:55.595Z",+ "meanEstimatedListCostUsd": 0.1879093666666667,+ "meanTokenConsumption": 32017.666666666668,+ "meanToolInvocationCount": 13.666666666666666},{- "experimentId": "minimax-m3-opencode--guided",+ "experimentId": "gpt-6-sol-high-codex--guided","caseId": "author-001-weather-tool","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 133642,- "measuredAt": "2026-09-15T16:41:09.940Z",- "meanEstimatedListCostUsd": 0.011224940000000001,- "meanTokenConsumption": 19059.333333333332,- "meanToolInvocationCount": 9.666666666666666+ "meanDurationMs": 162399,+ "measuredAt": "2026-10-07T18:53:33.041Z",+ "meanEstimatedListCostUsd": 0.22911669999999998,+ "meanTokenConsumption": 36498.333333333336,+ "meanToolInvocationCount": 12.333333333333334},{- "experimentId": "minimax-m3-opencode--guided",+ "experimentId": "gpt-6-sol-high-codex--guided","caseId": "author-002-new-project","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 213766,- "measuredAt": "2026-09-15T16:54:38.085Z",- "meanEstimatedListCostUsd": 0.01131386,- "meanTokenConsumption": 11259.666666666666,- "meanToolInvocationCount": 11.666666666666666+ "meanDurationMs": 154722,+ "measuredAt": "2026-10-07T18:53:33.054Z",+ "meanEstimatedListCostUsd": 0.2336195,+ "meanTokenConsumption": 35601.666666666664,+ "meanToolInvocationCount": 17.666666666666668},{- "experimentId": "minimax-m3-opencode--guided",+ "experimentId": "gpt-6-sol-high-codex--guided","caseId": "author-003-openapi-connection","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 120271,- "measuredAt": "2026-09-15T16:41:09.951Z",- "meanEstimatedListCostUsd": 0.02107842,- "meanTokenConsumption": 32130.333333333332,- "meanToolInvocationCount": 14.333333333333334+ "meanDurationMs": 163348,+ "measuredAt": "2026-10-07T18:53:33.066Z",+ "meanEstimatedListCostUsd": 0.1844876666666667,+ "meanTokenConsumption": 29518,+ "meanToolInvocationCount": 13.666666666666666},{- "experimentId": "minimax-m3-opencode--guided",+ "experimentId": "gpt-6-sol-high-codex--guided","caseId": "author-004-packaged-skill","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 135700,- "measuredAt": "2026-09-15T16:41:09.959Z",- "meanEstimatedListCostUsd": 0.01193822,- "meanTokenConsumption": 15964,- "meanToolInvocationCount": 9+ "meanDurationMs": 123914,+ "measuredAt": "2026-10-07T18:53:33.074Z",+ "meanEstimatedListCostUsd": 0.12451166666666667,+ "meanTokenConsumption": 21815.666666666668,+ "meanToolInvocationCount": 5.333333333333333},{- "experimentId": "minimax-m3-opencode--guided",+ "experimentId": "gpt-6-sol-high-codex--guided","caseId": "author-005-conditional-approval","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 115474,- "measuredAt": "2026-09-15T16:41:09.966Z",- "meanEstimatedListCostUsd": 0.01810478,- "meanTokenConsumption": 33213.333333333336,- "meanToolInvocationCount": 11.333333333333334+ "meanDurationMs": 168992,+ "measuredAt": "2026-10-07T18:53:33.083Z",+ "meanEstimatedListCostUsd": 0.21053246666666667,+ "meanTokenConsumption": 32490.333333333332,+ "meanToolInvocationCount": 13.333333333333334},{- "experimentId": "minimax-m3-opencode--guided",+ "experimentId": "gpt-6-sol-high-codex--guided","caseId": "author-006-custom-channel","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 113667,- "measuredAt": "2026-09-15T16:41:09.974Z",- "meanEstimatedListCostUsd": 0.012573259999999998,- "meanTokenConsumption": 17879.666666666668,- "meanToolInvocationCount": 13+ "meanDurationMs": 145077,+ "measuredAt": "2026-10-07T18:53:33.091Z",+ "meanEstimatedListCostUsd": 0.14791393333333333,+ "meanTokenConsumption": 25818,+ "meanToolInvocationCount": 8},{- "experimentId": "minimax-m3-opencode--guided",+ "experimentId": "gpt-6-sol-high-codex--guided","caseId": "author-007-digest-schedule","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 123417,- "measuredAt": "2026-09-15T16:41:09.982Z",- "meanEstimatedListCostUsd": 0.0110418,- "meanTokenConsumption": 14192.333333333334,- "meanToolInvocationCount": 10.666666666666666+ "meanDurationMs": 136865,+ "measuredAt": "2026-10-07T18:53:33.100Z",+ "meanEstimatedListCostUsd": 0.13206753333333335,+ "meanTokenConsumption": 21706.333333333332,+ "meanToolInvocationCount": 9.666666666666666},{- "experimentId": "kimi-k2-7-code-opencode--baseline",+ "experimentId": "gpt-6-luna-high-codex--baseline","caseId": "author-001-weather-tool","status": "current",- "passedRuns": 2,+ "passedRuns": 3,"validRuns": 3,- "meanDurationMs": 208245,- "measuredAt": "2026-09-15T16:48:32.061Z",- "meanEstimatedListCostUsd": 0.14676820333333332,- "meanTokenConsumption": 56908.666666666664,- "meanToolInvocationCount": 34+ "meanDurationMs": 172981,+ "measuredAt": "2026-10-07T19:02:42.860Z",+ "meanEstimatedListCostUsd": 0.014657728333333335,+ "meanTokenConsumption": 46713.666666666664,+ "meanToolInvocationCount": 11.333333333333334},{- "experimentId": "kimi-k2-7-code-opencode--baseline",+ "experimentId": "gpt-6-luna-high-codex--baseline","caseId": "author-002-new-project","status": "current",+ "passedRuns": 2,+ "validRuns": 3,+ "meanDurationMs": 208392,+ "measuredAt": "2026-10-07T19:02:42.873Z",+ "meanEstimatedListCostUsd": 0.02044859666666667,+ "meanTokenConsumption": 53849,+ "meanToolInvocationCount": 15+ },+ {+ "experimentId": "gpt-6-luna-high-codex--baseline",+ "caseId": "author-003-openapi-connection",+ "status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 429898,- "measuredAt": "2026-09-15T16:48:32.072Z",- "meanEstimatedListCostUsd": 0.04066295,- "meanTokenConsumption": 22378.666666666668,- "meanToolInvocationCount": 20.333333333333332+ "meanDurationMs": 122976,+ "measuredAt": "2026-10-07T19:02:42.882Z",+ "meanEstimatedListCostUsd": 0.008839191666666668,+ "meanTokenConsumption": 29309,+ "meanToolInvocationCount": 15.333333333333334},{- "experimentId": "kimi-k2-7-code-opencode--baseline",+ "experimentId": "gpt-6-luna-high-codex--baseline",+ "caseId": "author-004-packaged-skill",+ "status": "current",+ "passedRuns": 3,+ "validRuns": 3,+ "meanDurationMs": 131981,+ "measuredAt": "2026-10-07T19:02:42.890Z",+ "meanEstimatedListCostUsd": 0.00776646,+ "meanTokenConsumption": 25521.333333333332,+ "meanToolInvocationCount": 9+ },+ {+ "experimentId": "gpt-6-luna-high-codex--baseline",+ "caseId": "author-005-conditional-approval",+ "status": "current",+ "passedRuns": 3,+ "validRuns": 3,+ "meanDurationMs": 153769,+ "measuredAt": "2026-10-07T19:02:42.898Z",+ "meanEstimatedListCostUsd": 0.012869688333333332,+ "meanTokenConsumption": 40411.333333333336,+ "meanToolInvocationCount": 10.333333333333334+ },+ {+ "experimentId": "gpt-6-luna-high-codex--baseline",+ "caseId": "author-006-custom-channel",+ "status": "current",+ "passedRuns": 3,+ "validRuns": 3,+ "meanDurationMs": 137271,+ "measuredAt": "2026-10-07T19:02:42.909Z",+ "meanEstimatedListCostUsd": 0.013865816666666668,+ "meanTokenConsumption": 46150,+ "meanToolInvocationCount": 9.666666666666666+ },+ {+ "experimentId": "gpt-6-luna-high-codex--baseline",+ "caseId": "author-007-digest-schedule",+ "status": "current",+ "passedRuns": 3,+ "validRuns": 3,+ "meanDurationMs": 129008,+ "measuredAt": "2026-10-07T19:02:42.919Z",+ "meanEstimatedListCostUsd": 0.007547296666666667,+ "meanTokenConsumption": 24802.666666666668,+ "meanToolInvocationCount": 11+ },+ {+ "experimentId": "gpt-6-luna-high-codex--guided",+ "caseId": "author-001-weather-tool",+ "status": "current",+ "passedRuns": 3,+ "validRuns": 3,+ "meanDurationMs": 140987,+ "measuredAt": "2026-10-07T19:09:00.404Z",+ "meanEstimatedListCostUsd": 0.011568716666666668,+ "meanTokenConsumption": 35932.666666666664,+ "meanToolInvocationCount": 10.666666666666666+ },+ {+ "experimentId": "gpt-6-luna-high-codex--guided",+ "caseId": "author-002-new-project",+ "status": "current",+ "passedRuns": 2,+ "validRuns": 3,+ "meanDurationMs": 243860,+ "measuredAt": "2026-10-07T19:09:00.418Z",+ "meanEstimatedListCostUsd": 0.025036305,+ "meanTokenConsumption": 60119.333333333336,+ "meanToolInvocationCount": 19.666666666666668+ },+ {+ "experimentId": "gpt-6-luna-high-codex--guided","caseId": "author-003-openapi-connection","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 181692,- "measuredAt": "2026-09-15T16:48:32.080Z",- "meanEstimatedListCostUsd": 0.13857528666666666,- "meanTokenConsumption": 46478.333333333336,- "meanToolInvocationCount": 29.333333333333332+ "meanDurationMs": 116960,+ "measuredAt": "2026-10-07T19:09:00.454Z",+ "meanEstimatedListCostUsd": 0.007297571666666666,+ "meanTokenConsumption": 23631,+ "meanToolInvocationCount": 6.333333333333333},{- "experimentId": "kimi-k2-7-code-opencode--baseline",+ "experimentId": "gpt-6-luna-high-codex--guided","caseId": "author-004-packaged-skill","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 173418,- "measuredAt": "2026-09-15T16:48:32.092Z",- "meanEstimatedListCostUsd": 0.05450496666666666,- "meanTokenConsumption": 26974.333333333332,- "meanToolInvocationCount": 18.666666666666668+ "meanDurationMs": 148482,+ "measuredAt": "2026-10-07T19:09:00.463Z",+ "meanEstimatedListCostUsd": 0.007017896666666666,+ "meanTokenConsumption": 23253.666666666668,+ "meanToolInvocationCount": 6.666666666666667},{- "experimentId": "kimi-k2-7-code-opencode--baseline",+ "experimentId": "gpt-6-luna-high-codex--guided","caseId": "author-005-conditional-approval","status": "current",+ "passedRuns": 3,+ "validRuns": 3,+ "meanDurationMs": 164463,+ "measuredAt": "2026-10-07T19:09:00.470Z",+ "meanEstimatedListCostUsd": 0.00887198,+ "meanTokenConsumption": 28326.333333333332,+ "meanToolInvocationCount": 8+ },+ {+ "experimentId": "gpt-6-luna-high-codex--guided",+ "caseId": "author-006-custom-channel",+ "status": "current",+ "passedRuns": 3,+ "validRuns": 3,+ "meanDurationMs": 117293,+ "measuredAt": "2026-10-07T19:09:00.477Z",+ "meanEstimatedListCostUsd": 0.006835386666666668,+ "meanTokenConsumption": 22433,+ "meanToolInvocationCount": 8+ },+ {+ "experimentId": "gpt-6-luna-high-codex--guided",+ "caseId": "author-007-digest-schedule",+ "status": "current",+ "passedRuns": 3,+ "validRuns": 3,+ "meanDurationMs": 138527,+ "measuredAt": "2026-10-07T19:09:00.484Z",+ "meanEstimatedListCostUsd": 0.008701025000000001,+ "meanTokenConsumption": 26987.333333333332,+ "meanToolInvocationCount": 7.333333333333333+ },+ {+ "experimentId": "grok-4-7-opencode--baseline",+ "caseId": "author-001-weather-tool",+ "status": "current",+ "passedRuns": 3,+ "validRuns": 3,+ "meanDurationMs": 136051,+ "measuredAt": "2026-10-07T19:28:24.953Z",+ "meanEstimatedListCostUsd": 0.17287466666666665,+ "meanTokenConsumption": 52152,+ "meanToolInvocationCount": 18.333333333333332+ },+ {+ "experimentId": "grok-4-7-opencode--baseline",+ "caseId": "author-002-new-project",+ "status": "current","passedRuns": 2,"validRuns": 3,- "meanDurationMs": 204741,- "measuredAt": "2026-09-15T16:48:32.101Z",- "meanEstimatedListCostUsd": 0.09485771,- "meanTokenConsumption": 38354,- "meanToolInvocationCount": 32+ "meanDurationMs": 190658,+ "measuredAt": "2026-10-07T19:28:24.962Z",+ "meanEstimatedListCostUsd": 0.21032133333333333,+ "meanTokenConsumption": 42335.333333333336,+ "meanToolInvocationCount": 26},{- "experimentId": "kimi-k2-7-code-opencode--baseline",+ "experimentId": "grok-4-7-opencode--baseline",+ "caseId": "author-003-openapi-connection",+ "status": "current",+ "passedRuns": 2,+ "validRuns": 3,+ "meanDurationMs": 383235,+ "measuredAt": "2026-10-07T19:28:24.975Z",+ "meanEstimatedListCostUsd": 0.143828,+ "meanTokenConsumption": 43541,+ "meanToolInvocationCount": 17+ },+ {+ "experimentId": "grok-4-7-opencode--baseline",+ "caseId": "author-004-packaged-skill",+ "status": "current",+ "passedRuns": 3,+ "validRuns": 3,+ "meanDurationMs": 111116,+ "measuredAt": "2026-10-07T19:28:24.981Z",+ "meanEstimatedListCostUsd": 0.13283466666666666,+ "meanTokenConsumption": 34128,+ "meanToolInvocationCount": 17.666666666666668+ },+ {+ "experimentId": "grok-4-7-opencode--baseline",+ "caseId": "author-005-conditional-approval",+ "status": "current",+ "passedRuns": 2,+ "validRuns": 3,+ "meanDurationMs": 383889,+ "measuredAt": "2026-10-07T19:28:24.992Z",+ "meanEstimatedListCostUsd": 0.226556,+ "meanTokenConsumption": 65023,+ "meanToolInvocationCount": 23.5+ },+ {+ "experimentId": "grok-4-7-opencode--baseline","caseId": "author-006-custom-channel","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 197645,- "measuredAt": "2026-09-15T16:48:32.111Z",- "meanEstimatedListCostUsd": 0.14388091,- "meanTokenConsumption": 51045.333333333336,- "meanToolInvocationCount": 34.333333333333336+ "meanDurationMs": 116196,+ "measuredAt": "2026-10-07T19:28:25.000Z",+ "meanEstimatedListCostUsd": 0.13987,+ "meanTokenConsumption": 25978.333333333332,+ "meanToolInvocationCount": 18},{- "experimentId": "kimi-k2-7-code-opencode--baseline",+ "experimentId": "grok-4-7-opencode--baseline","caseId": "author-007-digest-schedule","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 157203,- "measuredAt": "2026-09-15T16:48:32.128Z",- "meanEstimatedListCostUsd": 0.05187631333333333,- "meanTokenConsumption": 19794.666666666668,- "meanToolInvocationCount": 20.666666666666668+ "meanDurationMs": 115736,+ "measuredAt": "2026-10-07T19:28:25.013Z",+ "meanEstimatedListCostUsd": 0.09889266666666667,+ "meanTokenConsumption": 27181,+ "meanToolInvocationCount": 15.333333333333334},{- "experimentId": "kimi-k2-7-code-opencode--guided",+ "experimentId": "grok-4-7-opencode--guided","caseId": "author-001-weather-tool","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 166225,- "measuredAt": "2026-09-15T16:42:45.614Z",- "meanEstimatedListCostUsd": 0.05287293333333334,- "meanTokenConsumption": 20028.666666666668,- "meanToolInvocationCount": 16.666666666666668+ "meanDurationMs": 131291,+ "measuredAt": "2026-10-07T19:33:57.511Z",+ "meanEstimatedListCostUsd": 0.13136866666666666,+ "meanTokenConsumption": 29745,+ "meanToolInvocationCount": 11.333333333333334},{- "experimentId": "kimi-k2-7-code-opencode--guided",+ "experimentId": "grok-4-7-opencode--guided","caseId": "author-002-new-project","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 293340,- "measuredAt": "2026-09-15T16:42:45.647Z",- "meanEstimatedListCostUsd": 0.04787821666666667,- "meanTokenConsumption": 21184.333333333332,- "meanToolInvocationCount": 22.666666666666668+ "meanDurationMs": 137195,+ "measuredAt": "2026-10-07T19:33:57.518Z",+ "meanEstimatedListCostUsd": 0.222338,+ "meanTokenConsumption": 61214.333333333336,+ "meanToolInvocationCount": 23.333333333333332},{- "experimentId": "kimi-k2-7-code-opencode--guided",+ "experimentId": "grok-4-7-opencode--guided","caseId": "author-003-openapi-connection","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 166937,- "measuredAt": "2026-09-15T16:42:45.661Z",- "meanEstimatedListCostUsd": 0.05114830333333333,- "meanTokenConsumption": 20667.666666666668,- "meanToolInvocationCount": 17.666666666666668+ "meanDurationMs": 110807,+ "measuredAt": "2026-10-07T19:33:57.527Z",+ "meanEstimatedListCostUsd": 0.14055266666666666,+ "meanTokenConsumption": 35289,+ "meanToolInvocationCount": 12},{- "experimentId": "kimi-k2-7-code-opencode--guided",+ "experimentId": "grok-4-7-opencode--guided","caseId": "author-004-packaged-skill","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 137327,- "measuredAt": "2026-09-15T16:42:45.672Z",- "meanEstimatedListCostUsd": 0.03893913333333334,- "meanTokenConsumption": 20582,- "meanToolInvocationCount": 12+ "meanDurationMs": 134888,+ "measuredAt": "2026-10-07T19:33:57.533Z",+ "meanEstimatedListCostUsd": 0.05279999999999999,+ "meanTokenConsumption": 10001.333333333334,+ "meanToolInvocationCount": 5.666666666666667},{- "experimentId": "kimi-k2-7-code-opencode--guided",+ "experimentId": "grok-4-7-opencode--guided","caseId": "author-005-conditional-approval","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 155897,- "measuredAt": "2026-09-15T16:42:45.681Z",- "meanEstimatedListCostUsd": 0.042350406666666666,- "meanTokenConsumption": 22769,- "meanToolInvocationCount": 13.666666666666666+ "meanDurationMs": 108481,+ "measuredAt": "2026-10-07T19:33:57.539Z",+ "meanEstimatedListCostUsd": 0.10046466666666666,+ "meanTokenConsumption": 18673.666666666668,+ "meanToolInvocationCount": 9.666666666666666},{- "experimentId": "kimi-k2-7-code-opencode--guided",+ "experimentId": "grok-4-7-opencode--guided","caseId": "author-006-custom-channel","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 143732,- "measuredAt": "2026-09-15T16:42:45.690Z",- "meanEstimatedListCostUsd": 0.05902574,- "meanTokenConsumption": 32431.666666666668,- "meanToolInvocationCount": 15.666666666666666+ "meanDurationMs": 108256,+ "measuredAt": "2026-10-07T19:33:57.548Z",+ "meanEstimatedListCostUsd": 0.09657466666666666,+ "meanTokenConsumption": 15122,+ "meanToolInvocationCount": 7},{- "experimentId": "kimi-k2-7-code-opencode--guided",+ "experimentId": "grok-4-7-opencode--guided","caseId": "author-007-digest-schedule","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 171310,- "measuredAt": "2026-09-15T16:42:45.699Z",- "meanEstimatedListCostUsd": 0.047509576666666664,- "meanTokenConsumption": 20610.666666666668,- "meanToolInvocationCount": 17+ "meanDurationMs": 115721,+ "measuredAt": "2026-10-07T19:33:57.559Z",+ "meanEstimatedListCostUsd": 0.07453466666666668,+ "meanTokenConsumption": 23851.333333333332,+ "meanToolInvocationCount": 5}],"previouslyMeasured": [@@ -2623,22 +2807,6 @@"modelDisplayName": "Claude Fable 5","harness": "Claude Code","treatment": "guided"- },- {- "id": "glm-5-2-opencode--baseline",- "groupId": "glm-5-2-opencode",- "model": "zai/glm-5.2",- "modelDisplayName": "GLM 5.2",- "harness": "OpenCode",- "treatment": "baseline"- },- {- "id": "glm-5-2-opencode--guided",- "groupId": "glm-5-2-opencode",- "model": "zai/glm-5.2",- "modelDisplayName": "GLM 5.2",- "harness": "OpenCode",- "treatment": "guided"}],"results": [@@ -2809,42 +2977,634 @@"meanEstimatedListCostUsd": 3.07521,"meanToolInvocationCount": 7,"meanTokenConsumption": 299897+ }+ ]+ },+ {+ "suite": {+ "eveRevision": "6ce61cd85ecf4621bcefd363a3e8b9445b7e6e91",+ "caseFingerprint": "7d990b7e2c429ee4a2cf4f758d15f408f61d0e5b1ef9782ba69a33e435cdb169",+ "caseCount": 7,+ "runsPerCell": 3+ },+ "experiments": [+ {+ "id": "grok-4-6-opencode--baseline",+ "groupId": "grok-4-6-opencode",+ "model": "xai/grok-4.6",+ "modelDisplayName": "Grok 4.6",+ "harness": "OpenCode",+ "treatment": "baseline"},{- "experimentId": "glm-5-2-opencode--baseline",+ "id": "grok-4-6-opencode--guided",+ "groupId": "grok-4-6-opencode",+ "model": "xai/grok-4.6",+ "modelDisplayName": "Grok 4.6",+ "harness": "OpenCode",+ "treatment": "guided"+ },+ {+ "id": "gpt-5-6-sol-codex--baseline",+ "groupId": "gpt-5-6-sol-codex",+ "model": "openai/gpt-5.6-sol",+ "modelDisplayName": "GPT-5.6 Sol",+ "harness": "Codex",+ "treatment": "baseline"+ },+ {+ "id": "gpt-5-6-sol-codex--guided",+ "groupId": "gpt-5-6-sol-codex",+ "model": "openai/gpt-5.6-sol",+ "modelDisplayName": "GPT-5.6 Sol",+ "harness": "Codex",+ "treatment": "guided"+ },+ {+ "id": "claude-opus-5-claude-code--baseline",+ "groupId": "claude-opus-5-claude-code",+ "model": "anthropic/claude-opus-5",+ "modelDisplayName": "Claude Opus 5",+ "harness": "Claude Code",+ "treatment": "baseline"+ },+ {+ "id": "claude-opus-5-claude-code--guided",+ "groupId": "claude-opus-5-claude-code",+ "model": "anthropic/claude-opus-5",+ "modelDisplayName": "Claude Opus 5",+ "harness": "Claude Code",+ "treatment": "guided"+ }+ ],+ "results": [+ {+ "experimentId": "grok-4-6-opencode--baseline","caseId": "author-001-weather-tool","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 103530,- "measuredAt": "2026-08-28T19:22:09.000Z",- "meanEstimatedListCostUsd": 0.07017806666666666,- "meanTokenConsumption": 37804.333333333336,- "meanToolInvocationCount": 25+ "meanDurationMs": 38613,+ "measuredAt": "2026-08-28T19:22:01.000Z",+ "meanEstimatedListCostUsd": 0.10434266666666665,+ "meanTokenConsumption": 49566,+ "meanToolInvocationCount": 19},{- "experimentId": "glm-5-2-opencode--baseline",+ "experimentId": "grok-4-6-opencode--baseline","caseId": "author-002-new-project","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 100847,- "measuredAt": "2026-08-28T19:26:40.000Z",- "meanEstimatedListCostUsd": 0.03248553333333333,- "meanTokenConsumption": 19034.666666666668,- "meanToolInvocationCount": 18.333333333333332+ "meanDurationMs": 83985,+ "measuredAt": "2026-08-28T19:25:12.000Z",+ "meanEstimatedListCostUsd": 0.07232866666666667,+ "meanTokenConsumption": 32827.666666666664,+ "meanToolInvocationCount": 16},{- "experimentId": "glm-5-2-opencode--baseline",+ "experimentId": "grok-4-6-opencode--baseline","caseId": "author-003-openapi-connection","status": "current",- "passedRuns": 3,+ "passedRuns": 0,"validRuns": 3,- "meanDurationMs": 134009,- "measuredAt": "2026-08-28T19:26:40.000Z",- "meanEstimatedListCostUsd": 0.09391753333333334,- "meanTokenConsumption": 49094.666666666664,- "meanToolInvocationCount": 40.333333333333336+ "meanDurationMs": 521954,+ "measuredAt": "2026-08-28T19:22:01.000Z",+ "meanEstimatedListCostUsd": 1.0741786666666668,+ "meanTokenConsumption": 503422.6666666667,+ "meanToolInvocationCount": 76.66666666666667+ },+ {+ "experimentId": "grok-4-6-opencode--baseline",+ "caseId": "author-004-packaged-skill",+ "status": "current",+ "passedRuns": 3,+ "validRuns": 3,+ "meanDurationMs": 41894,+ "measuredAt": "2026-08-28T19:22:01.000Z",+ "meanEstimatedListCostUsd": 0.065152,+ "meanTokenConsumption": 29506,+ "meanToolInvocationCount": 23.333333333333332+ },+ {+ "experimentId": "grok-4-6-opencode--baseline",+ "caseId": "author-005-conditional-approval",+ "status": "current",+ "passedRuns": 3,+ "validRuns": 3,+ "meanDurationMs": 50151,+ "measuredAt": "2026-08-28T19:22:01.000Z",+ "meanEstimatedListCostUsd": 0.141536,+ "meanTokenConsumption": 66629.33333333333,+ "meanToolInvocationCount": 22+ },+ {+ "experimentId": "grok-4-6-opencode--baseline",+ "caseId": "author-006-custom-channel",+ "status": "current",+ "passedRuns": 3,+ "validRuns": 3,+ "meanDurationMs": 37062,+ "measuredAt": "2026-08-28T19:22:01.000Z",+ "meanEstimatedListCostUsd": 0.07319533333333333,+ "meanTokenConsumption": 33801.666666666664,+ "meanToolInvocationCount": 17.666666666666668+ },+ {+ "experimentId": "grok-4-6-opencode--baseline",+ "caseId": "author-007-digest-schedule",+ "status": "current",+ "passedRuns": 3,+ "validRuns": 3,+ "meanDurationMs": 38280,+ "measuredAt": "2026-08-28T19:22:01.000Z",+ "meanEstimatedListCostUsd": 0.045748000000000004,+ "meanTokenConsumption": 20122.666666666668,+ "meanToolInvocationCount": 18+ },+ {+ "experimentId": "grok-4-6-opencode--guided",+ "caseId": "author-001-weather-tool",+ "status": "current",+ "passedRuns": 3,+ "validRuns": 3,+ "meanDurationMs": 31700,+ "measuredAt": "2026-08-28T19:22:02.000Z",+ "meanEstimatedListCostUsd": 0.049695333333333334,+ "meanTokenConsumption": 22963.666666666668,+ "meanToolInvocationCount": 9.666666666666666+ },+ {+ "experimentId": "grok-4-6-opencode--guided",+ "caseId": "author-002-new-project",+ "status": "current",+ "passedRuns": 3,+ "validRuns": 3,+ "meanDurationMs": 83621,+ "measuredAt": "2026-08-28T19:25:07.000Z",+ "meanEstimatedListCostUsd": 0.082842,+ "meanTokenConsumption": 38261,+ "meanToolInvocationCount": 16.333333333333332+ },+ {+ "experimentId": "grok-4-6-opencode--guided",+ "caseId": "author-003-openapi-connection",+ "status": "current",+ "passedRuns": 3,+ "validRuns": 3,+ "meanDurationMs": 34833,+ "measuredAt": "2026-08-28T19:25:07.000Z",+ "meanEstimatedListCostUsd": 0.055174,+ "meanTokenConsumption": 25952.333333333332,+ "meanToolInvocationCount": 10+ },+ {+ "experimentId": "grok-4-6-opencode--guided",+ "caseId": "author-004-packaged-skill",+ "status": "current",+ "passedRuns": 3,+ "validRuns": 3,+ "meanDurationMs": 30135,+ "measuredAt": "2026-08-28T19:22:02.000Z",+ "meanEstimatedListCostUsd": 0.035581999999999996,+ "meanTokenConsumption": 16043.666666666666,+ "meanToolInvocationCount": 8.666666666666666+ },+ {+ "experimentId": "grok-4-6-opencode--guided",+ "caseId": "author-005-conditional-approval",+ "status": "current",+ "passedRuns": 3,+ "validRuns": 3,+ "meanDurationMs": 34026,+ "measuredAt": "2026-08-28T19:22:02.000Z",+ "meanEstimatedListCostUsd": 0.038532666666666666,+ "meanTokenConsumption": 17059.666666666668,+ "meanToolInvocationCount": 11.666666666666666+ },+ {+ "experimentId": "grok-4-6-opencode--guided",+ "caseId": "author-006-custom-channel",+ "status": "current",+ "passedRuns": 3,+ "validRuns": 3,+ "meanDurationMs": 32109,+ "measuredAt": "2026-08-28T19:22:02.000Z",+ "meanEstimatedListCostUsd": 0.033709333333333334,+ "meanTokenConsumption": 14610,+ "meanToolInvocationCount": 9.666666666666666+ },+ {+ "experimentId": "grok-4-6-opencode--guided",+ "caseId": "author-007-digest-schedule",+ "status": "current",+ "passedRuns": 3,+ "validRuns": 3,+ "meanDurationMs": 25583,+ "measuredAt": "2026-08-28T19:22:02.000Z",+ "meanEstimatedListCostUsd": 0.023458666666666666,+ "meanTokenConsumption": 10328,+ "meanToolInvocationCount": 7.333333333333333+ },+ {+ "experimentId": "gpt-5-6-sol-codex--baseline",+ "caseId": "author-001-weather-tool",+ "status": "current",+ "passedRuns": 3,+ "validRuns": 3,+ "meanDurationMs": 175042,+ "measuredAt": "2026-08-31T19:20:09.055Z",+ "meanEstimatedListCostUsd": 0.42096675,+ "meanTokenConsumption": 49663.666666666664,+ "meanToolInvocationCount": 13.666666666666666+ },+ {+ "experimentId": "gpt-5-6-sol-codex--baseline",+ "caseId": "author-002-new-project",+ "status": "current",+ "passedRuns": 3,+ "validRuns": 3,+ "meanDurationMs": 122470,+ "measuredAt": "2026-08-31T19:20:09.085Z",+ "meanEstimatedListCostUsd": 0.23273858333333333,+ "meanTokenConsumption": 30848.666666666668,+ "meanToolInvocationCount": 10.333333333333334+ },+ {+ "experimentId": "gpt-5-6-sol-codex--baseline",+ "caseId": "author-003-openapi-connection",+ "status": "current",+ "passedRuns": 3,+ "validRuns": 3,+ "meanDurationMs": 132036,+ "measuredAt": "2026-08-31T19:20:09.104Z",+ "meanEstimatedListCostUsd": 0.367498625,+ "meanTokenConsumption": 48982.666666666664,+ "meanToolInvocationCount": 9.333333333333334+ },+ {+ "experimentId": "gpt-5-6-sol-codex--baseline",+ "caseId": "author-004-packaged-skill",+ "status": "current",+ "passedRuns": 3,+ "validRuns": 3,+ "meanDurationMs": 137447,+ "measuredAt": "2026-08-31T19:20:09.128Z",+ "meanEstimatedListCostUsd": 0.26755258333333337,+ "meanTokenConsumption": 36330.666666666664,+ "meanToolInvocationCount": 6+ },+ {+ "experimentId": "gpt-5-6-sol-codex--baseline",+ "caseId": "author-005-conditional-approval",+ "status": "current",+ "passedRuns": 3,+ "validRuns": 3,+ "meanDurationMs": 121857,+ "measuredAt": "2026-08-31T19:20:09.143Z",+ "meanEstimatedListCostUsd": 0.2530605833333333,+ "meanTokenConsumption": 32384,+ "meanToolInvocationCount": 10+ },+ {+ "experimentId": "gpt-5-6-sol-codex--baseline",+ "caseId": "author-006-custom-channel",+ "status": "current",+ "passedRuns": 3,+ "validRuns": 3,+ "meanDurationMs": 115653,+ "measuredAt": "2026-08-31T19:20:09.156Z",+ "meanEstimatedListCostUsd": 0.251469,+ "meanTokenConsumption": 33854.666666666664,+ "meanToolInvocationCount": 8+ },+ {+ "experimentId": "gpt-5-6-sol-codex--baseline",+ "caseId": "author-007-digest-schedule",+ "status": "current",+ "passedRuns": 3,+ "validRuns": 3,+ "meanDurationMs": 121172,+ "measuredAt": "2026-08-31T19:20:09.170Z",+ "meanEstimatedListCostUsd": 0.284567375,+ "meanTokenConsumption": 37330.666666666664,+ "meanToolInvocationCount": 11+ },+ {+ "experimentId": "gpt-5-6-sol-codex--guided",+ "caseId": "author-001-weather-tool",+ "status": "current",+ "passedRuns": 3,+ "validRuns": 3,+ "meanDurationMs": 115694,+ "measuredAt": "2026-08-31T19:18:53.704Z",+ "meanEstimatedListCostUsd": 0.18167699999999998,+ "meanTokenConsumption": 24500,+ "meanToolInvocationCount": 4.666666666666667+ },+ {+ "experimentId": "gpt-5-6-sol-codex--guided",+ "caseId": "author-002-new-project",+ "status": "current",+ "passedRuns": 3,+ "validRuns": 3,+ "meanDurationMs": 132614,+ "measuredAt": "2026-08-31T19:18:53.727Z",+ "meanEstimatedListCostUsd": 0.22479154166666668,+ "meanTokenConsumption": 28792.333333333332,+ "meanToolInvocationCount": 9.333333333333334+ },+ {+ "experimentId": "gpt-5-6-sol-codex--guided",+ "caseId": "author-003-openapi-connection",+ "status": "current",+ "passedRuns": 3,+ "validRuns": 3,+ "meanDurationMs": 101648,+ "measuredAt": "2026-08-31T19:18:53.741Z",+ "meanEstimatedListCostUsd": 0.14007875,+ "meanTokenConsumption": 19209.666666666668,+ "meanToolInvocationCount": 5.333333333333333+ },+ {+ "experimentId": "gpt-5-6-sol-codex--guided",+ "caseId": "author-004-packaged-skill",+ "status": "current",+ "passedRuns": 3,+ "validRuns": 3,+ "meanDurationMs": 129791,+ "measuredAt": "2026-08-31T19:18:53.751Z",+ "meanEstimatedListCostUsd": 0.19388683333333334,+ "meanTokenConsumption": 24916,+ "meanToolInvocationCount": 6+ },+ {+ "experimentId": "gpt-5-6-sol-codex--guided",+ "caseId": "author-005-conditional-approval",+ "status": "current",+ "passedRuns": 3,+ "validRuns": 3,+ "meanDurationMs": 108054,+ "measuredAt": "2026-08-31T19:18:53.759Z",+ "meanEstimatedListCostUsd": 0.15307379166666665,+ "meanTokenConsumption": 20874.333333333332,+ "meanToolInvocationCount": 4.666666666666667+ },+ {+ "experimentId": "gpt-5-6-sol-codex--guided",+ "caseId": "author-006-custom-channel",+ "status": "current",+ "passedRuns": 3,+ "validRuns": 3,+ "meanDurationMs": 100410,+ "measuredAt": "2026-08-31T19:18:53.769Z",+ "meanEstimatedListCostUsd": 0.108055625,+ "meanTokenConsumption": 15663.666666666666,+ "meanToolInvocationCount": 3+ },+ {+ "experimentId": "gpt-5-6-sol-codex--guided",+ "caseId": "author-007-digest-schedule",+ "status": "current",+ "passedRuns": 3,+ "validRuns": 3,+ "meanDurationMs": 103849,+ "measuredAt": "2026-08-31T19:18:53.779Z",+ "meanEstimatedListCostUsd": 0.11547295833333333,+ "meanTokenConsumption": 16687.333333333332,+ "meanToolInvocationCount": 3.6666666666666665+ },+ {+ "experimentId": "claude-opus-5-claude-code--baseline",+ "caseId": "author-001-weather-tool",+ "status": "current",+ "passedRuns": 3,+ "validRuns": 3,+ "meanDurationMs": 155202,+ "measuredAt": "2026-09-02T14:35:36.609Z",+ "meanEstimatedListCostUsd": 1.1599598333333334,+ "meanTokenConsumption": 10714.666666666666,+ "meanToolInvocationCount": 17.333333333333332+ },+ {+ "experimentId": "claude-opus-5-claude-code--baseline",+ "caseId": "author-002-new-project",+ "status": "current",+ "passedRuns": 3,+ "validRuns": 3,+ "meanDurationMs": 124976,+ "measuredAt": "2026-09-02T14:35:36.628Z",+ "meanEstimatedListCostUsd": 0.8593296666666667,+ "meanTokenConsumption": 3845.3333333333335,+ "meanToolInvocationCount": 10.333333333333334+ },+ {+ "experimentId": "claude-opus-5-claude-code--baseline",+ "caseId": "author-003-openapi-connection",+ "status": "current",+ "passedRuns": 3,+ "validRuns": 3,+ "meanDurationMs": 160123,+ "measuredAt": "2026-09-02T14:35:36.643Z",+ "meanEstimatedListCostUsd": 1.3671014166666666,+ "meanTokenConsumption": 7793.333333333333,+ "meanToolInvocationCount": 19+ },+ {+ "experimentId": "claude-opus-5-claude-code--baseline",+ "caseId": "author-004-packaged-skill",+ "status": "current",+ "passedRuns": 3,+ "validRuns": 3,+ "meanDurationMs": 181217,+ "measuredAt": "2026-09-02T14:35:36.656Z",+ "meanEstimatedListCostUsd": 1.1116483333333333,+ "meanTokenConsumption": 16371.666666666666,+ "meanToolInvocationCount": 12.666666666666666+ },+ {+ "experimentId": "claude-opus-5-claude-code--baseline",+ "caseId": "author-005-conditional-approval",+ "status": "current",+ "passedRuns": 3,+ "validRuns": 3,+ "meanDurationMs": 153333,+ "measuredAt": "2026-09-02T14:35:36.667Z",+ "meanEstimatedListCostUsd": 1.03157675,+ "meanTokenConsumption": 6217.333333333333,+ "meanToolInvocationCount": 17.333333333333332+ },+ {+ "experimentId": "claude-opus-5-claude-code--baseline",+ "caseId": "author-006-custom-channel",+ "status": "current",+ "passedRuns": 3,+ "validRuns": 3,+ "meanDurationMs": 136155,+ "measuredAt": "2026-09-02T14:35:36.676Z",+ "meanEstimatedListCostUsd": 0.9316169166666667,+ "meanTokenConsumption": 4836.666666666667,+ "meanToolInvocationCount": 16.666666666666668+ },+ {+ "experimentId": "claude-opus-5-claude-code--baseline",+ "caseId": "author-007-digest-schedule",+ "status": "current",+ "passedRuns": 3,+ "validRuns": 3,+ "meanDurationMs": 120633,+ "measuredAt": "2026-09-02T14:35:36.685Z",+ "meanEstimatedListCostUsd": 0.5067333333333334,+ "meanTokenConsumption": 3232.6666666666665,+ "meanToolInvocationCount": 9.333333333333334+ },+ {+ "experimentId": "claude-opus-5-claude-code--guided",+ "caseId": "author-001-weather-tool",+ "status": "current",+ "passedRuns": 3,+ "validRuns": 3,+ "meanDurationMs": 147978,+ "measuredAt": "2026-09-02T14:34:53.900Z",+ "meanEstimatedListCostUsd": 1.0020195,+ "meanTokenConsumption": 11004.333333333334,+ "meanToolInvocationCount": 11+ },+ {+ "experimentId": "claude-opus-5-claude-code--guided",+ "caseId": "author-002-new-project",+ "status": "current",+ "passedRuns": 3,+ "validRuns": 3,+ "meanDurationMs": 138714,+ "measuredAt": "2026-09-02T14:34:53.929Z",+ "meanEstimatedListCostUsd": 1.0472271666666666,+ "meanTokenConsumption": 8633.666666666666,+ "meanToolInvocationCount": 14.333333333333334+ },+ {+ "experimentId": "claude-opus-5-claude-code--guided",+ "caseId": "author-003-openapi-connection",+ "status": "current",+ "passedRuns": 3,+ "validRuns": 3,+ "meanDurationMs": 137520,+ "measuredAt": "2026-09-02T14:34:53.944Z",+ "meanEstimatedListCostUsd": 0.7869943333333334,+ "meanTokenConsumption": 5193.666666666667,+ "meanToolInvocationCount": 14+ },+ {+ "experimentId": "claude-opus-5-claude-code--guided",+ "caseId": "author-004-packaged-skill",+ "status": "current",+ "passedRuns": 3,+ "validRuns": 3,+ "meanDurationMs": 153316,+ "measuredAt": "2026-09-02T14:34:53.956Z",+ "meanEstimatedListCostUsd": 0.6677339166666667,+ "meanTokenConsumption": 10291.666666666666,+ "meanToolInvocationCount": 7.666666666666667+ },+ {+ "experimentId": "claude-opus-5-claude-code--guided",+ "caseId": "author-005-conditional-approval",+ "status": "current",+ "passedRuns": 3,+ "validRuns": 3,+ "meanDurationMs": 116393,+ "measuredAt": "2026-09-02T14:34:53.964Z",+ "meanEstimatedListCostUsd": 0.5910709166666667,+ "meanTokenConsumption": 2916.6666666666665,+ "meanToolInvocationCount": 8.333333333333334+ },+ {+ "experimentId": "claude-opus-5-claude-code--guided",+ "caseId": "author-006-custom-channel",+ "status": "current",+ "passedRuns": 3,+ "validRuns": 3,+ "meanDurationMs": 107570,+ "measuredAt": "2026-09-02T14:34:53.973Z",+ "meanEstimatedListCostUsd": 0.4942074166666666,+ "meanTokenConsumption": 2636,+ "meanToolInvocationCount": 6.666666666666667+ },+ {+ "experimentId": "claude-opus-5-claude-code--guided",+ "caseId": "author-007-digest-schedule",+ "status": "current",+ "passedRuns": 3,+ "validRuns": 3,+ "meanDurationMs": 114311,+ "measuredAt": "2026-09-02T14:34:53.981Z",+ "meanEstimatedListCostUsd": 0.46051358333333337,+ "meanTokenConsumption": 2592.6666666666665,+ "meanToolInvocationCount": 7.666666666666667+ }+ ]+ },+ {+ "suite": {+ "eveRevision": "279e2b6a81198f6008004f596474cf05140ca7bc",+ "caseFingerprint": "336fa6e2645a717a4bf4d906c3ae1b9f4ce0839bfb7bc9065068490dd696d42e",+ "caseCount": 7,+ "runsPerCell": 3+ },+ "experiments": [+ {+ "id": "glm-5-2-opencode--baseline",+ "groupId": "glm-5-2-opencode",+ "model": "zai/glm-5.2",+ "modelDisplayName": "GLM 5.2",+ "harness": "OpenCode",+ "treatment": "baseline"+ },+ {+ "id": "glm-5-2-opencode--guided",+ "groupId": "glm-5-2-opencode",+ "model": "zai/glm-5.2",+ "modelDisplayName": "GLM 5.2",+ "harness": "OpenCode",+ "treatment": "guided"+ }+ ],+ "results": [+ {+ "experimentId": "glm-5-2-opencode--baseline",+ "caseId": "author-001-weather-tool",+ "status": "current",+ "passedRuns": 0,+ "validRuns": 3,+ "meanDurationMs": 183955,+ "measuredAt": "2026-10-07T20:53:18.454Z",+ "meanEstimatedListCostUsd": 0.08985164,+ "meanTokenConsumption": 33383.666666666664,+ "meanToolInvocationCount": 23.666666666666668+ },+ {+ "experimentId": "glm-5-2-opencode--baseline",+ "caseId": "author-002-new-project",+ "status": "current",+ "passedRuns": 2,+ "validRuns": 3,+ "meanDurationMs": 146900,+ "measuredAt": "2026-10-07T20:53:18.471Z",+ "meanEstimatedListCostUsd": 0.06131117333333334,+ "meanTokenConsumption": 26307.666666666668,+ "meanToolInvocationCount": 16.666666666666668+ },+ {+ "experimentId": "glm-5-2-opencode--baseline",+ "caseId": "author-003-openapi-connection",+ "status": "current",+ "passedRuns": 2,+ "validRuns": 3,+ "meanDurationMs": 263531,+ "measuredAt": "2026-10-07T20:53:18.486Z",+ "meanEstimatedListCostUsd": 0.24344989333333333,+ "meanTokenConsumption": 59370.666666666664,+ "meanToolInvocationCount": 41.333333333333336},{"experimentId": "glm-5-2-opencode--baseline",@@ -2852,35 +3612,35 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 214119,- "measuredAt": "2026-08-28T19:22:09.000Z",- "meanEstimatedListCostUsd": 0.07759993333333333,- "meanTokenConsumption": 41325.666666666664,- "meanToolInvocationCount": 36.333333333333336+ "meanDurationMs": 215844,+ "measuredAt": "2026-10-07T20:53:18.509Z",+ "meanEstimatedListCostUsd": 0.10365373333333334,+ "meanTokenConsumption": 34354,+ "meanToolInvocationCount": 27},{"experimentId": "glm-5-2-opencode--baseline","caseId": "author-005-conditional-approval","status": "current",- "passedRuns": 3,+ "passedRuns": 1,"validRuns": 3,- "meanDurationMs": 58527,- "measuredAt": "2026-08-28T19:22:09.000Z",- "meanEstimatedListCostUsd": 0.07383253333333333,- "meanTokenConsumption": 48329.666666666664,- "meanToolInvocationCount": 25.333333333333332+ "meanDurationMs": 203192,+ "measuredAt": "2026-10-07T20:53:18.528Z",+ "meanEstimatedListCostUsd": 0.19839237333333334,+ "meanTokenConsumption": 75866.66666666667,+ "meanToolInvocationCount": 32.666666666666664},{"experimentId": "glm-5-2-opencode--baseline","caseId": "author-006-custom-channel","status": "current",- "passedRuns": 3,+ "passedRuns": 2,"validRuns": 3,- "meanDurationMs": 75109,- "measuredAt": "2026-08-28T19:22:09.000Z",- "meanEstimatedListCostUsd": 0.07836139999999998,- "meanTokenConsumption": 48506,- "meanToolInvocationCount": 27.666666666666668+ "meanDurationMs": 240346,+ "measuredAt": "2026-10-07T20:53:18.555Z",+ "meanEstimatedListCostUsd": 0.3499838533333333,+ "meanTokenConsumption": 96427,+ "meanToolInvocationCount": 45.333333333333336},{"experimentId": "glm-5-2-opencode--baseline",@@ -2888,11 +3648,11 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 93384,- "measuredAt": "2026-08-28T19:22:09.000Z",- "meanEstimatedListCostUsd": 0.04282053333333333,- "meanTokenConsumption": 24309.666666666668,- "meanToolInvocationCount": 19+ "meanDurationMs": 155457,+ "measuredAt": "2026-10-07T20:53:18.586Z",+ "meanEstimatedListCostUsd": 0.07361798666666665,+ "meanTokenConsumption": 34515.333333333336,+ "meanToolInvocationCount": 18.666666666666668},{"experimentId": "glm-5-2-opencode--guided",@@ -2900,35 +3660,35 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 68607,- "measuredAt": "2026-08-28T19:22:01.000Z",- "meanEstimatedListCostUsd": 0.04188693333333334,- "meanTokenConsumption": 22360.666666666668,- "meanToolInvocationCount": 16.333333333333332+ "meanDurationMs": 211084,+ "measuredAt": "2026-10-07T20:58:05.582Z",+ "meanEstimatedListCostUsd": 0.12096889333333334,+ "meanTokenConsumption": 30802.666666666668,+ "meanToolInvocationCount": 21.666666666666668},{"experimentId": "glm-5-2-opencode--guided","caseId": "author-002-new-project","status": "current",- "passedRuns": 3,+ "passedRuns": 1,"validRuns": 3,- "meanDurationMs": 90517,- "measuredAt": "2026-08-28T19:25:29.000Z",- "meanEstimatedListCostUsd": 0.02843253333333333,- "meanTokenConsumption": 16824.666666666668,- "meanToolInvocationCount": 14.333333333333334+ "meanDurationMs": 146105,+ "measuredAt": "2026-10-07T20:58:05.598Z",+ "meanEstimatedListCostUsd": 0.045339653333333334,+ "meanTokenConsumption": 14588.666666666666,+ "meanToolInvocationCount": 16.333333333333332},{"experimentId": "glm-5-2-opencode--guided","caseId": "author-003-openapi-connection","status": "current",- "passedRuns": 0,+ "passedRuns": 3,"validRuns": 3,- "meanDurationMs": 524221,- "measuredAt": "2026-08-28T19:22:01.000Z",- "meanEstimatedListCostUsd": 0.1800297333333333,- "meanTokenConsumption": 78267.66666666667,- "meanToolInvocationCount": 24.666666666666668+ "meanDurationMs": 159353,+ "measuredAt": "2026-10-07T20:58:05.607Z",+ "meanEstimatedListCostUsd": 0.07640521333333335,+ "meanTokenConsumption": 20298.666666666668,+ "meanToolInvocationCount": 19.333333333333332},{"experimentId": "glm-5-2-opencode--guided",@@ -2936,23 +3696,23 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 68821,- "measuredAt": "2026-08-28T19:22:01.000Z",- "meanEstimatedListCostUsd": 0.031183066666666665,- "meanTokenConsumption": 16504.333333333332,- "meanToolInvocationCount": 15+ "meanDurationMs": 190722,+ "measuredAt": "2026-10-07T20:58:05.620Z",+ "meanEstimatedListCostUsd": 0.06202376000000001,+ "meanTokenConsumption": 15365.666666666666,+ "meanToolInvocationCount": 17.666666666666668},{"experimentId": "glm-5-2-opencode--guided","caseId": "author-005-conditional-approval","status": "current",- "passedRuns": 2,+ "passedRuns": 3,"validRuns": 3,- "meanDurationMs": 80061,- "measuredAt": "2026-08-28T19:22:01.000Z",- "meanEstimatedListCostUsd": 0.040431333333333326,- "meanTokenConsumption": 26306.666666666668,- "meanToolInvocationCount": 14.333333333333334+ "meanDurationMs": 155636,+ "measuredAt": "2026-10-07T20:58:05.635Z",+ "meanEstimatedListCostUsd": 0.049076973333333336,+ "meanTokenConsumption": 18203,+ "meanToolInvocationCount": 14.666666666666666},{"experimentId": "glm-5-2-opencode--guided",@@ -2960,11 +3720,11 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 56079,- "measuredAt": "2026-08-28T19:22:01.000Z",- "meanEstimatedListCostUsd": 0.03310953333333334,- "meanTokenConsumption": 19814.666666666668,- "meanToolInvocationCount": 11+ "meanDurationMs": 168719,+ "measuredAt": "2026-10-07T20:58:05.650Z",+ "meanEstimatedListCostUsd": 0.07198956000000001,+ "meanTokenConsumption": 19972.666666666668,+ "meanToolInvocationCount": 19},{"experimentId": "glm-5-2-opencode--guided",@@ -2972,11 +3732,11 @@"status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 74552,- "measuredAt": "2026-08-28T19:22:01.000Z",- "meanEstimatedListCostUsd": 0.029523933333333335,- "meanTokenConsumption": 16295.666666666666,- "meanToolInvocationCount": 12.333333333333334+ "meanDurationMs": 147630,+ "measuredAt": "2026-10-07T20:58:05.665Z",+ "meanEstimatedListCostUsd": 0.04161758666666667,+ "meanTokenConsumption": 10529.333333333334,+ "meanToolInvocationCount": 13.333333333333334}]}