Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

chore(docs): update benchmarks Opus 5.5, GPT-6, Grok 4.7 (#4514)

Colton Paddenb6c8cd65c090
diff --git a/apps/benchmarks/README.md b/apps/benchmarks/README.md
index 2c2d6c89f..740866515 100644
--- a/apps/benchmarks/README.md
+++ b/apps/benchmarks/README.md
@@ -32,6 +32,8 @@ are not supported by the native runner.
The runner uses agent-eval's native Gateway harnesses: OpenCode for other providers, Claude Code
for Anthropic models, and Codex for OpenAI models. Each attempt starts an isolated Vercel Sandbox,
then scaffolds the selected immutable canary with `npx` before the coding agent starts.
+The starting workspace includes pinned `[email protected]` for the local default sandbox provider.
+Scaffolded workspaces must pass typecheck and build before any model inference starts.
Local runs use the `guided` treatment by default, which keeps the `AGENTS.md` and aliases generated
by `eve init`. Pass `--treatment baseline` to remove those files before the coding agent starts.
@@ -51,7 +53,7 @@ working tree and defaults to `origin/main`:
pnpm benchmark:publish --dry
pnpm benchmark:publish
pnpm benchmark:publish --revision <commit>
-pnpm benchmark:publish --models kimi-k3,gpt-5-6-sol
+pnpm benchmark:publish --models kimi-k3,gpt-6-sol-high
```
Pass `--allow-dirty` only for a local, noncanonical run. It bypasses the clean-tree check, so its
diff --git a/apps/benchmarks/lib/authoring-case.ts b/apps/benchmarks/lib/authoring-case.ts
index 44e29bbc3..af0a4ead4 100644
--- a/apps/benchmarks/lib/authoring-case.ts
+++ b/apps/benchmarks/lib/authoring-case.ts
@@ -39,12 +39,12 @@ export interface AuthoringCase {
}
export const emptyProject: AuthoringStartingPoint = {
- id: "empty-v2",
+ id: "empty-v3",
workspace: "empty",
};
export const simpleProject: AuthoringStartingPoint = {
- id: "simple-v2",
+ id: "simple-v3",
workspace: "scaffolded",
};
diff --git a/apps/benchmarks/lib/benchmark-config.test.mjs b/apps/benchmarks/lib/benchmark-config.test.mjs
index 304a2aa95..ff5641898 100644
--- a/apps/benchmarks/lib/benchmark-config.test.mjs
+++ b/apps/benchmarks/lib/benchmark-config.test.mjs
@@ -1,69 +1,32 @@
import assert from "node:assert/strict";
import { test } from "node:test";
-import {
- findBenchmarkModel,
- findPublishedBenchmarkModel,
- harnessId,
- publishedBenchmark,
- publishedBenchmarkModels,
-} from "./benchmark-config.ts";
+import { findBenchmarkModel, findPublishedBenchmarkModel, harnessId } from "./benchmark-config.ts";
-test("publishes only compatibility-validated models", () => {
- assert.deepEqual(
- publishedBenchmarkModels.map((benchmark) => benchmark.id),
- [
- "kimi-k3",
- "claude-fable-5-1",
- "grok-4-6",
- "gpt-5-6-sol",
- "gpt-5-6-terra",
- "gpt-6-astra-high",
- "claude-sonnet-5",
- "glm-5-3",
- "claude-opus-5",
- "gemini-3-1-pro-preview",
- "gemini-3-8-flash",
- "minimax-m3",
- "kimi-k2-7-code",
- ],
- );
- assert.deepEqual(publishedBenchmark.caseIds, [
- "author-001-weather-tool",
- "author-002-new-project",
- "author-003-openapi-connection",
- "author-004-packaged-skill",
- "author-005-conditional-approval",
- "author-006-custom-channel",
- "author-007-digest-schedule",
- ]);
- assert.equal(findPublishedBenchmarkModel("grok-4-6").model, "xai/grok-4.6");
- assert.equal(findPublishedBenchmarkModel("claude-sonnet-5").model, "anthropic/claude-sonnet-5");
+test("selects qualified Gateway IDs and explicit reasoning settings for publication", () => {
+ assert.equal(findPublishedBenchmarkModel("grok-4-7").model, "spacexai/grok-4.7");
assert.equal(findPublishedBenchmarkModel("glm-5-3").model, "zai/glm-5.3");
- assert.equal(
- findPublishedBenchmarkModel("gpt-6-astra-high").agentModel,
- "openai/gpt-6-astra?reasoningEffort=high",
- );
- assert.equal(findPublishedBenchmarkModel("claude-fable-5-1").harness, "Claude Code");
- assert.equal(findPublishedBenchmarkModel("claude-sonnet-5").harness, "Claude Code");
- assert.equal(findPublishedBenchmarkModel("claude-opus-5").harness, "Claude Code");
- assert.equal(findPublishedBenchmarkModel("kimi-k3").harness, "OpenCode");
- assert.equal(findPublishedBenchmarkModel("gpt-5-6-sol").harness, "Codex");
- assert.equal(findPublishedBenchmarkModel("gpt-5-6-terra").harness, "Codex");
- assert.equal(harnessId("Codex"), "codex");
+ for (const [id, model] of [
+ ["gpt-6-sol-high", "openai/gpt-6-sol"],
+ ["gpt-6-luna-high", "openai/gpt-6-luna"],
+ ["gpt-6-astra-high", "openai/gpt-6-astra"],
+ ]) {
+ const benchmark = findPublishedBenchmarkModel(id);
+ assert.equal(benchmark.agentModel, `${model}?reasoningEffort=high`);
+ assert.equal(harnessId(benchmark.harness), "codex");
+ }
+ for (const id of ["claude-opus-5-5-high", "claude-fable-5-1"]) {
+ const benchmark = findPublishedBenchmarkModel(id);
+ assert.equal(benchmark.agentOptions.effort, "high");
+ assert.equal(harnessId(benchmark.harness), "claude-code");
+ }
});
-test("allows candidate probes and rejects unknown models", () => {
- assert.equal(findBenchmarkModel("gpt-5-6-sol").support, "supported");
- assert.equal(findBenchmarkModel("gpt-5-6-terra").support, "supported");
- assert.equal(findBenchmarkModel("gpt-6-astra-high").support, "supported");
- assert.equal(findBenchmarkModel("gemini-3-8-flash").support, "supported");
- assert.equal(findBenchmarkModel("minimax-m3").support, "supported");
- assert.equal(findBenchmarkModel("kimi-k2-7-code").support, "supported");
- assert.equal(findBenchmarkModel("grok-4-6").support, "supported");
- assert.equal(findBenchmarkModel("claude-sonnet-4-6").support, "superseded");
- assert.equal(findBenchmarkModel("claude-fable-5").support, "superseded");
+test("allows local retired probes without admitting them to publication", () => {
+ assert.equal(findBenchmarkModel("grok-4-6").support, "superseded");
assert.equal(findBenchmarkModel("glm-5-2").support, "superseded");
- assert.equal(findBenchmarkModel("claude-opus-5").support, "supported");
+ for (const id of ["grok-4-6", "glm-5-2"]) {
+ assert.throws(() => findPublishedBenchmarkModel(id), /not in the supported publication set/u);
+ }
assert.throws(() => findBenchmarkModel("unknown"), /Unknown model/u);
});
diff --git a/apps/benchmarks/lib/benchmark-config.ts b/apps/benchmarks/lib/benchmark-config.ts
index 4238fa85f..73b101fc9 100644
--- a/apps/benchmarks/lib/benchmark-config.ts
+++ b/apps/benchmarks/lib/benchmark-config.ts
@@ -11,6 +11,7 @@ export interface AuthoringBenchmarkModel {
readonly agentModel?: string;
readonly displayName: string;
readonly harness: "OpenCode" | "Claude Code" | "Codex";
+ readonly agentOptions?: Readonly<Record<string, unknown>>;
readonly support: AuthoringBenchmarkSupport;
}
@@ -39,8 +40,9 @@ export const benchmarkModels = [
{
id: "claude-fable-5-1",
model: "anthropic/claude-fable-5.1",
- displayName: "Claude Fable 5.1",
+ displayName: "Claude Fable 5.1 (high)",
harness: "Claude Code",
+ agentOptions: { effort: "high" },
support: "supported",
},
{
@@ -48,14 +50,14 @@ export const benchmarkModels = [
model: "xai/grok-4.6",
displayName: "Grok 4.6",
harness: "OpenCode",
- support: "supported",
+ support: "superseded",
},
{
id: "gpt-5-6-sol",
model: "openai/gpt-5.6-sol",
displayName: "GPT-5.6 Sol",
harness: "Codex",
- support: "supported",
+ support: "superseded",
},
{
id: "gpt-5-6-terra",
@@ -98,7 +100,7 @@ export const benchmarkModels = [
model: "anthropic/claude-opus-5",
displayName: "Claude Opus 5",
harness: "Claude Code",
- support: "supported",
+ support: "superseded",
},
{
id: "gemini-3-1-pro-preview",
@@ -128,6 +130,57 @@ export const benchmarkModels = [
harness: "OpenCode",
support: "supported",
},
+ {
+ id: "claude-opus-5-5-high",
+ model: "anthropic/claude-opus-5.5",
+ displayName: "Claude Opus 5.5 (high)",
+ harness: "Claude Code",
+ agentOptions: { effort: "high" },
+ support: "supported",
+ },
+ {
+ id: "gpt-6-sol-high",
+ model: "openai/gpt-6-sol",
+ agentModel: "openai/gpt-6-sol?reasoningEffort=high",
+ displayName: "GPT-6 Sol (high)",
+ harness: "Codex",
+ support: "supported",
+ },
+ {
+ id: "gpt-6-luna-high",
+ model: "openai/gpt-6-luna",
+ agentModel: "openai/gpt-6-luna?reasoningEffort=high",
+ displayName: "GPT-6 Luna (high)",
+ harness: "Codex",
+ support: "supported",
+ },
+ {
+ id: "grok-4-7",
+ model: "spacexai/grok-4.7",
+ displayName: "Grok 4.7",
+ harness: "OpenCode",
+ agentOptions: {
+ binaryUrl:
+ "https://ymdea60kblwwhidh.public.blob.vercel-storage.com/opencode-linux-x64-RCtfS54uaTwa5i9C3bpzguLy3jEBP6",
+ extraProviders: {
+ vercel: {
+ models: {
+ "spacexai/grok-4.7": {
+ name: "Grok 4.7",
+ reasoning: true,
+ tool_call: true,
+ temperature: true,
+ attachment: true,
+ modalities: { input: ["text", "image"], output: ["text"] },
+ limit: { context: 500000, output: 131072 },
+ cost: { input: 0, output: 0, cache_read: 0 },
+ },
+ },
+ },
+ },
+ },
+ support: "supported",
+ },
] as const satisfies ReadonlyArray<AuthoringBenchmarkModel>;
export const publishedBenchmarkModels = benchmarkModels.filter(
diff --git a/apps/benchmarks/lib/experiment.test.mjs b/apps/benchmarks/lib/experiment.test.mjs
index eea192eee..76092ef2a 100644
--- a/apps/benchmarks/lib/experiment.test.mjs
+++ b/apps/benchmarks/lib/experiment.test.mjs
@@ -56,7 +56,7 @@ test("passes an explicit native model override to Codex", () => {
assert.equal(config.model, "openai/gpt-6-astra?reasoningEffort=high");
});
-test("strips the Gateway provider prefix for native Claude Code", () => {
+test("strips the Gateway prefix and preserves explicit Claude Code effort", () => {
const config = authoringExperiment({
...common,
benchmark: {
@@ -64,10 +64,13 @@ test("strips the Gateway provider prefix for native Claude Code", () => {
model: "anthropic/claude-sonnet-5",
displayName: "Claude Sonnet 5",
harness: "Claude Code",
+ agentOptions: { effort: "high" },
support: "supported",
},
});
assert.equal(config.agent, "vercel-ai-gateway/claude-code");
assert.equal(config.model, "claude-sonnet-5");
+ assert.equal(config.agentOptions.effort, "high");
+ assert.equal(config.agentOptions.verbose, false);
});
diff --git a/apps/benchmarks/lib/experiment.ts b/apps/benchmarks/lib/experiment.ts
index b12b016a5..9ab37717f 100644
--- a/apps/benchmarks/lib/experiment.ts
+++ b/apps/benchmarks/lib/experiment.ts
@@ -36,7 +36,14 @@ export function authoringExperiment(options: {
revision: options.revision,
treatment: options.treatment,
}),
- agentOptions: { verbose: options.verbose ?? false },
+ agentOptions: {
+ cliPackage:
+ options.benchmark.harness === "Claude Code"
+ ? "@anthropic-ai/[email protected]"
+ : undefined,
+ ...options.benchmark.agentOptions,
+ verbose: options.verbose ?? false,
+ },
};
}
diff --git a/apps/benchmarks/lib/native-authoring-setup.test.mjs b/apps/benchmarks/lib/native-authoring-setup.test.mjs
index f0ce0a6ad..3ad90526a 100644
--- a/apps/benchmarks/lib/native-authoring-setup.test.mjs
+++ b/apps/benchmarks/lib/native-authoring-setup.test.mjs
@@ -44,9 +44,37 @@ test("bootstraps the selected source before a native agent starts", async () =>
commands[2].args[1],
/AI_AGENT=claude EVE_INIT_PACKAGE_SPEC=.* eve init \. --model openai\/gpt-5\.5/u,
);
- assert.match(commands[3].args[1], /mkdir -p agent\/lib/u);
- assert.match(commands[4].args[1], /rm -f AGENTS\.md CLAUDE\.md GEMINI\.md/u);
- assert.match(commands[5].args[1], /git add \. && git commit --amend --no-edit --quiet/u);
+ assert.match(commands[3].args[1], /npm install --save-dev --save-exact just-bash@3\.1\.0/u);
+ assert.match(commands[4].args[1], /mkdir -p agent\/lib/u);
+ assert.match(commands[5].args[1], /rm -f AGENTS\.md CLAUDE\.md GEMINI\.md/u);
+ assert.match(commands[6].args[1], /npm run typecheck && npm run build/u);
+ assert.match(commands[7].args[1], /git add \. && git commit --amend --no-edit --quiet/u);
assert.deepEqual(Object.keys(writes[1]), ["/workspace/agent/lib/inventory-openapi.ts"]);
assert.match(writes[1]["/workspace/agent/lib/inventory-openapi.ts"], /getStock/u);
});
+
+test("rejects a broken starting workspace before handing it to the model", async () => {
+ const setup = createNativeAuthoringSetup({
+ packageSpec: "https://pkg.eve.dev/commit/eve.tgz",
+ revision: "commit",
+ treatment: "guided",
+ });
+ await assert.rejects(
+ setup({
+ async readFile() {
+ return JSON.stringify({ startingPoint: "scaffolded", revision: "commit", setupIds: [] });
+ },
+ async writeFiles() {},
+ async runCommand(_command, args) {
+ const failed = args[1].includes("npm run typecheck && npm run build");
+ return {
+ stdout: "",
+ stderr: failed ? "missing optional package" : "",
+ exitCode: failed ? 1 : 0,
+ };
+ },
+ getWorkingDirectory: () => "/workspace",
+ }),
+ /starting workspace validation failed.*missing optional package/su,
+ );
+});
diff --git a/apps/benchmarks/lib/native-authoring-setup.ts b/apps/benchmarks/lib/native-authoring-setup.ts
index 8d9475824..95f69dd94 100644
--- a/apps/benchmarks/lib/native-authoring-setup.ts
+++ b/apps/benchmarks/lib/native-authoring-setup.ts
@@ -55,9 +55,20 @@ export function createNativeAuthoringSetup(options: {
});
}
+ // DefaultSandbox selects just-bash inside the Linux eval container, where Docker
+ // and microsandbox are unavailable. Supply its optional peer before the model runs.
+ await run(
+ sandbox,
+ "npm install --save-dev --save-exact [email protected]",
+ "local sandbox dependency",
+ );
+
const context = setupContext(sandbox, bootstrap.projectDirectory);
for (const setup of setupsFor(bootstrap.setupIds)) await setup.onSession?.(context);
if (options.treatment === "baseline") await run(sandbox, "rm -f AGENTS.md CLAUDE.md GEMINI.md");
+ if (bootstrap.startingPoint === "scaffolded") {
+ await run(sandbox, "npm run typecheck && npm run build", "starting workspace validation");
+ }
await run(sandbox, "git add . && git commit --amend --no-edit --quiet");
};
}
diff --git a/apps/benchmarks/scripts/cost.mjs b/apps/benchmarks/scripts/cost.mjs
index 0e144f591..784917c4f 100644
--- a/apps/benchmarks/scripts/cost.mjs
+++ b/apps/benchmarks/scripts/cost.mjs
@@ -33,6 +33,38 @@ export const modelPricing = {
cacheRead: 0.25,
cacheWrite: 12.5,
},
+ "anthropic/claude-opus-5.5": {
+ effectiveDate: "2026-10-07",
+ source: "https://ai-gateway.vercel.sh/v1/models",
+ input: 4,
+ output: 20,
+ cacheRead: 0.2,
+ cacheWrite: 5,
+ },
+ "openai/gpt-6-sol": {
+ effectiveDate: "2026-10-07",
+ source: "https://ai-gateway.vercel.sh/v1/models",
+ input: 2,
+ output: 10,
+ cacheRead: 0.2,
+ cacheWrite: 2.5,
+ },
+ "openai/gpt-6-luna": {
+ effectiveDate: "2026-10-07",
+ source: "https://ai-gateway.vercel.sh/v1/models",
+ input: 0.1,
+ output: 0.5,
+ cacheRead: 0.01,
+ cacheWrite: 0.125,
+ },
+ "spacexai/grok-4.7": {
+ effectiveDate: "2026-10-07",
+ source: "https://ai-gateway.vercel.sh/v1/models",
+ input: 2,
+ output: 6,
+ cacheRead: 0.5,
+ cacheWrite: 0,
+ },
"xai/grok-4.6": {
effectiveDate: "2026-08-17",
source: "https://docs.x.ai/developers/models",
@@ -81,12 +113,13 @@ export const modelPricing = {
cacheRead: 0.2,
cacheWrite: 2.5,
},
+ // Use the Gateway US endpoint's list rates, not the catalog's cheaper global route.
"zai/glm-5.2": {
- effectiveDate: "2026-08-17",
- source: "https://docs.z.ai/guides/overview/pricing",
+ effectiveDate: "2026-10-07",
+ source: "https://ai-gateway.vercel.sh/v1/models",
input: 1.4,
output: 4.4,
- cacheRead: 0.26,
+ cacheRead: 0.14,
cacheWrite: 0,
},
"zai/glm-5.3": {
@@ -122,11 +155,11 @@ export const modelPricing = {
cacheWrite: 0,
},
"moonshotai/kimi-k2.7-code": {
- effectiveDate: "2026-09-10",
+ effectiveDate: "2026-10-07",
source: "https://ai-gateway.vercel.sh/v1/models",
input: 0.95,
output: 4,
- cacheRead: 0.16,
+ cacheRead: 0.19,
cacheWrite: 0,
},
};
diff --git a/apps/benchmarks/scripts/export-results.test.mjs b/apps/benchmarks/scripts/export-results.test.mjs
index f7e66ef4e..e25ea7e65 100644
--- a/apps/benchmarks/scripts/export-results.test.mjs
+++ b/apps/benchmarks/scripts/export-results.test.mjs
@@ -40,26 +40,8 @@ test("exports missing cells without publishing private artifacts", () => {
assert.equal(output.schemaVersion, 1);
assert.equal(output.suite.caseCount, 7);
assert.match(output.suite.caseFingerprint, /^[0-9a-f]{64}$/u);
- assert.equal(output.experiments.length, 26);
- assert.deepEqual(
- [...new Set(output.experiments.map((experiment) => experiment.modelDisplayName))],
- [
- "Kimi K3",
- "Claude Fable 5.1",
- "Grok 4.6",
- "GPT-5.6 Sol",
- "GPT-5.6 Terra",
- "GPT-6 Astra (high)",
- "Claude Sonnet 5",
- "GLM 5.3",
- "Claude Opus 5",
- "Gemini 3.1 Pro Preview",
- "Gemini 3.8 Flash",
- "MiniMax M3",
- "Kimi K2.7 Code",
- ],
- );
- assert.equal(output.results.length, 182);
+ assert.equal(output.experiments.length, 28);
+ assert.equal(output.results.length, 196);
assert.ok(output.results.every((result) => result.status === "missing"));
for (const privateField of ["transcript", "commands", "worldEvents", "files"]) {
assert.equal(raw.includes(`"${privateField}"`), false);
diff --git a/apps/docs/lib/evals/benchmark-results.json b/apps/docs/lib/evals/benchmark-results.json
index ac205506e..50108a437 100644
--- a/apps/docs/lib/evals/benchmark-results.json
+++ b/apps/docs/lib/evals/benchmark-results.json
@@ -1,9 +1,9 @@
{
"schemaVersion": 1,
- "generatedAt": "2026-09-15T16:54:39.105Z",
+ "generatedAt": "2026-10-08T14:33:57.335Z",
"suite": {
- "eveRevision": "6ce61cd85ecf4621bcefd363a3e8b9445b7e6e91",
- "caseFingerprint": "7d990b7e2c429ee4a2cf4f758d15f408f61d0e5b1ef9782ba69a33e435cdb169",
+ "eveRevision": "279e2b6a81198f6008004f596474cf05140ca7bc",
+ "caseFingerprint": "336fa6e2645a717a4bf4d906c3ae1b9f4ce0839bfb7bc9065068490dd696d42e",
"caseCount": 7,
"runsPerCell": 3
},
@@ -28,7 +28,7 @@
"id": "claude-fable-5-1-claude-code--baseline",
"groupId": "claude-fable-5-1-claude-code",
"model": "anthropic/claude-fable-5.1",
- "modelDisplayName": "Claude Fable 5.1",
+ "modelDisplayName": "Claude Fable 5.1 (high)",
"harness": "Claude Code",
"treatment": "baseline"
},
@@ -36,42 +36,10 @@
"id": "claude-fable-5-1-claude-code--guided",
"groupId": "claude-fable-5-1-claude-code",
"model": "anthropic/claude-fable-5.1",
- "modelDisplayName": "Claude Fable 5.1",
+ "modelDisplayName": "Claude Fable 5.1 (high)",
"harness": "Claude Code",
"treatment": "guided"
},
- {
- "id": "grok-4-6-opencode--baseline",
- "groupId": "grok-4-6-opencode",
- "model": "xai/grok-4.6",
- "modelDisplayName": "Grok 4.6",
- "harness": "OpenCode",
- "treatment": "baseline"
- },
- {
- "id": "grok-4-6-opencode--guided",
- "groupId": "grok-4-6-opencode",
- "model": "xai/grok-4.6",
- "modelDisplayName": "Grok 4.6",
- "harness": "OpenCode",
- "treatment": "guided"
- },
- {
- "id": "gpt-5-6-sol-codex--baseline",
- "groupId": "gpt-5-6-sol-codex",
- "model": "openai/gpt-5.6-sol",
- "modelDisplayName": "GPT-5.6 Sol",
- "harness": "Codex",
- "treatment": "baseline"
- },
- {
- "id": "gpt-5-6-sol-codex--guided",
- "groupId": "gpt-5-6-sol-codex",
- "model": "openai/gpt-5.6-sol",
- "modelDisplayName": "GPT-5.6 Sol",
- "harness": "Codex",
- "treatment": "guided"
- },
{
"id": "gpt-5-6-terra-codex--baseline",
"groupId": "gpt-5-6-terra-codex",
@@ -136,22 +104,6 @@
"harness": "OpenCode",
"treatment": "guided"
},
- {
- "id": "claude-opus-5-claude-code--baseline",
- "groupId": "claude-opus-5-claude-code",
- "model": "anthropic/claude-opus-5",
- "modelDisplayName": "Claude Opus 5",
- "harness": "Claude Code",
- "treatment": "baseline"
- },
- {
- "id": "claude-opus-5-claude-code--guided",
- "groupId": "claude-opus-5-claude-code",
- "model": "anthropic/claude-opus-5",
- "modelDisplayName": "Claude Opus 5",
- "harness": "Claude Code",
- "treatment": "guided"
- },
{
"id": "gemini-3-1-pro-preview-opencode--baseline",
"groupId": "gemini-3-1-pro-preview-opencode",
@@ -215,6 +167,70 @@
"modelDisplayName": "Kimi K2.7 Code",
"harness": "OpenCode",
"treatment": "guided"
+ },
+ {
+ "id": "claude-opus-5-5-high-claude-code--baseline",
+ "groupId": "claude-opus-5-5-high-claude-code",
+ "model": "anthropic/claude-opus-5.5",
+ "modelDisplayName": "Claude Opus 5.5 (high)",
+ "harness": "Claude Code",
+ "treatment": "baseline"
+ },
+ {
+ "id": "claude-opus-5-5-high-claude-code--guided",
+ "groupId": "claude-opus-5-5-high-claude-code",
+ "model": "anthropic/claude-opus-5.5",
+ "modelDisplayName": "Claude Opus 5.5 (high)",
+ "harness": "Claude Code",
+ "treatment": "guided"
+ },
+ {
+ "id": "gpt-6-sol-high-codex--baseline",
+ "groupId": "gpt-6-sol-high-codex",
+ "model": "openai/gpt-6-sol",
+ "modelDisplayName": "GPT-6 Sol (high)",
+ "harness": "Codex",
+ "treatment": "baseline"
+ },
+ {
+ "id": "gpt-6-sol-high-codex--guided",
+ "groupId": "gpt-6-sol-high-codex",
+ "model": "openai/gpt-6-sol",
+ "modelDisplayName": "GPT-6 Sol (high)",
+ "harness": "Codex",
+ "treatment": "guided"
+ },
+ {
+ "id": "gpt-6-luna-high-codex--baseline",
+ "groupId": "gpt-6-luna-high-codex",
+ "model": "openai/gpt-6-luna",
+ "modelDisplayName": "GPT-6 Luna (high)",
+ "harness": "Codex",
+ "treatment": "baseline"
+ },
+ {
+ "id": "gpt-6-luna-high-codex--guided",
+ "groupId": "gpt-6-luna-high-codex",
+ "model": "openai/gpt-6-luna",
+ "modelDisplayName": "GPT-6 Luna (high)",
+ "harness": "Codex",
+ "treatment": "guided"
+ },
+ {
+ "id": "grok-4-7-opencode--baseline",
+ "groupId": "grok-4-7-opencode",
+ "model": "spacexai/grok-4.7",
+ "modelDisplayName": "Grok 4.7",
+ "harness": "OpenCode",
+ "treatment": "baseline"
+ },
+ {
+ "id": "grok-4-7-opencode--guided",
+ "groupId": "grok-4-7-opencode",
+ "model": "spacexai/grok-4.7",
+ "modelDisplayName": "Grok 4.7",
+ "harness": "OpenCode",
+ "treatment": "guided"
}
],
"results": [
@@ -224,11 +240,11 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 124258,
- "measuredAt": "2026-08-28T19:22:06.000Z",
- "meanEstimatedListCostUsd": 0.14751699999999998,
- "meanTokenConsumption": 31831,
- "meanToolInvocationCount": 25
+ "meanDurationMs": 182488,
+ "measuredAt": "2026-10-07T17:53:49.672Z",
+ "meanEstimatedListCostUsd": 0.3365506,
+ "meanTokenConsumption": 39811,
+ "meanToolInvocationCount": 27.666666666666668
},
{
"experimentId": "kimi-k3-opencode--baseline",
@@ -236,23 +252,23 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 89304,
- "measuredAt": "2026-08-28T19:25:16.000Z",
- "meanEstimatedListCostUsd": 0.085075,
- "meanTokenConsumption": 21470.333333333332,
- "meanToolInvocationCount": 12
+ "meanDurationMs": 118940,
+ "measuredAt": "2026-10-07T17:53:49.687Z",
+ "meanEstimatedListCostUsd": 0.0926576,
+ "meanTokenConsumption": 15709.333333333334,
+ "meanToolInvocationCount": 11
},
{
"experimentId": "kimi-k3-opencode--baseline",
"caseId": "author-003-openapi-connection",
"status": "current",
- "passedRuns": 0,
+ "passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 454935,
- "measuredAt": "2026-08-28T19:22:06.000Z",
- "meanEstimatedListCostUsd": 0.3622910000000001,
- "meanTokenConsumption": 56591.666666666664,
- "meanToolInvocationCount": 28
+ "meanDurationMs": 173791,
+ "measuredAt": "2026-10-07T17:53:49.695Z",
+ "meanEstimatedListCostUsd": 0.2977262,
+ "meanTokenConsumption": 39151.666666666664,
+ "meanToolInvocationCount": 27.333333333333332
},
{
"experimentId": "kimi-k3-opencode--baseline",
@@ -260,11 +276,11 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 101585,
- "measuredAt": "2026-08-28T19:22:06.000Z",
- "meanEstimatedListCostUsd": 0.09421099999999999,
- "meanTokenConsumption": 19086.333333333332,
- "meanToolInvocationCount": 15.666666666666666
+ "meanDurationMs": 154644,
+ "measuredAt": "2026-10-07T17:53:49.707Z",
+ "meanEstimatedListCostUsd": 0.142479,
+ "meanTokenConsumption": 18281,
+ "meanToolInvocationCount": 19.333333333333332
},
{
"experimentId": "kimi-k3-opencode--baseline",
@@ -272,11 +288,11 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 98137,
- "measuredAt": "2026-08-28T19:22:06.000Z",
- "meanEstimatedListCostUsd": 0.097954,
- "meanTokenConsumption": 23523.333333333332,
- "meanToolInvocationCount": 20
+ "meanDurationMs": 182203,
+ "measuredAt": "2026-10-07T17:53:49.717Z",
+ "meanEstimatedListCostUsd": 0.3224834,
+ "meanTokenConsumption": 40129.666666666664,
+ "meanToolInvocationCount": 28
},
{
"experimentId": "kimi-k3-opencode--baseline",
@@ -284,11 +300,11 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 78637,
- "measuredAt": "2026-08-28T19:22:06.000Z",
- "meanEstimatedListCostUsd": 0.107612,
- "meanTokenConsumption": 26372,
- "meanToolInvocationCount": 19.333333333333332
+ "meanDurationMs": 154439,
+ "measuredAt": "2026-10-07T17:53:49.730Z",
+ "meanEstimatedListCostUsd": 0.2093862,
+ "meanTokenConsumption": 26541,
+ "meanToolInvocationCount": 23.666666666666668
},
{
"experimentId": "kimi-k3-opencode--baseline",
@@ -296,11 +312,11 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 117347,
- "measuredAt": "2026-08-28T19:22:06.000Z",
- "meanEstimatedListCostUsd": 0.103991,
- "meanTokenConsumption": 21791.666666666668,
- "meanToolInvocationCount": 16.333333333333332
+ "meanDurationMs": 132808,
+ "measuredAt": "2026-10-07T17:53:49.747Z",
+ "meanEstimatedListCostUsd": 0.1187692,
+ "meanTokenConsumption": 17728.666666666668,
+ "meanToolInvocationCount": 18
},
{
"experimentId": "kimi-k3-opencode--guided",
@@ -308,11 +324,11 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 73445,
- "measuredAt": "2026-08-28T19:20:34.000Z",
- "meanEstimatedListCostUsd": 0.08992099999999999,
- "meanTokenConsumption": 20789.666666666668,
- "meanToolInvocationCount": 12.333333333333334
+ "meanDurationMs": 178412,
+ "measuredAt": "2026-10-07T18:06:24.212Z",
+ "meanEstimatedListCostUsd": 0.1784928,
+ "meanTokenConsumption": 25840,
+ "meanToolInvocationCount": 16.666666666666668
},
{
"experimentId": "kimi-k3-opencode--guided",
@@ -320,11 +336,11 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 90748,
- "measuredAt": "2026-08-28T19:25:31.000Z",
- "meanEstimatedListCostUsd": 0.081476,
- "meanTokenConsumption": 20988,
- "meanToolInvocationCount": 12
+ "meanDurationMs": 219061,
+ "measuredAt": "2026-10-07T18:06:24.226Z",
+ "meanEstimatedListCostUsd": 0.1086592,
+ "meanTokenConsumption": 19816,
+ "meanToolInvocationCount": 12.666666666666666
},
{
"experimentId": "kimi-k3-opencode--guided",
@@ -332,11 +348,11 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 71243,
- "measuredAt": "2026-08-28T19:25:31.000Z",
- "meanEstimatedListCostUsd": 0.081825,
- "meanTokenConsumption": 19215,
- "meanToolInvocationCount": 13.333333333333334
+ "meanDurationMs": 156025,
+ "measuredAt": "2026-10-07T18:06:24.238Z",
+ "meanEstimatedListCostUsd": 0.12732,
+ "meanTokenConsumption": 26533.333333333332,
+ "meanToolInvocationCount": 10.333333333333334
},
{
"experimentId": "kimi-k3-opencode--guided",
@@ -344,11 +360,11 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 126758,
- "measuredAt": "2026-08-28T19:20:34.000Z",
- "meanEstimatedListCostUsd": 0.08192899999999999,
- "meanTokenConsumption": 16876.333333333332,
- "meanToolInvocationCount": 15
+ "meanDurationMs": 145832,
+ "measuredAt": "2026-10-07T18:06:24.248Z",
+ "meanEstimatedListCostUsd": 0.11396500000000002,
+ "meanTokenConsumption": 19580.333333333332,
+ "meanToolInvocationCount": 10.333333333333334
},
{
"experimentId": "kimi-k3-opencode--guided",
@@ -356,11 +372,11 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 95028,
- "measuredAt": "2026-08-28T19:20:34.000Z",
- "meanEstimatedListCostUsd": 0.117859,
- "meanTokenConsumption": 29467.666666666668,
- "meanToolInvocationCount": 16.333333333333332
+ "meanDurationMs": 148846,
+ "measuredAt": "2026-10-07T18:06:24.257Z",
+ "meanEstimatedListCostUsd": 0.16298139999999997,
+ "meanTokenConsumption": 30186.333333333332,
+ "meanToolInvocationCount": 12.666666666666666
},
{
"experimentId": "kimi-k3-opencode--guided",
@@ -368,11 +384,11 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 49272,
- "measuredAt": "2026-08-28T19:20:34.000Z",
- "meanEstimatedListCostUsd": 0.0695,
- "meanTokenConsumption": 17840,
- "meanToolInvocationCount": 10.333333333333334
+ "meanDurationMs": 157626,
+ "measuredAt": "2026-10-07T18:06:24.270Z",
+ "meanEstimatedListCostUsd": 0.1153282,
+ "meanTokenConsumption": 17712.333333333332,
+ "meanToolInvocationCount": 11
},
{
"experimentId": "kimi-k3-opencode--guided",
@@ -380,11 +396,11 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 77381,
- "measuredAt": "2026-08-28T19:20:34.000Z",
- "meanEstimatedListCostUsd": 0.061534,
- "meanTokenConsumption": 14636.666666666666,
- "meanToolInvocationCount": 12.333333333333334
+ "meanDurationMs": 142137,
+ "measuredAt": "2026-10-07T18:06:24.286Z",
+ "meanEstimatedListCostUsd": 0.08892719999999998,
+ "meanTokenConsumption": 15018.666666666666,
+ "meanToolInvocationCount": 10
},
{
"experimentId": "claude-fable-5-1-claude-code--baseline",
@@ -392,11 +408,11 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 186262,
- "measuredAt": "2026-09-02T14:19:29.095Z",
- "meanEstimatedListCostUsd": 1.291232,
- "meanTokenConsumption": 8972,
- "meanToolInvocationCount": 11.666666666666666
+ "meanDurationMs": 299284,
+ "measuredAt": "2026-10-07T18:34:17.712Z",
+ "meanEstimatedListCostUsd": 1.7602164166666665,
+ "meanTokenConsumption": 14277,
+ "meanToolInvocationCount": 15
},
{
"experimentId": "claude-fable-5-1-claude-code--baseline",
@@ -404,11 +420,11 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 148272,
- "measuredAt": "2026-09-02T14:19:29.109Z",
- "meanEstimatedListCostUsd": 0.7348391666666666,
- "meanTokenConsumption": 3895.3333333333335,
- "meanToolInvocationCount": 9
+ "meanDurationMs": 220684,
+ "measuredAt": "2026-10-07T18:34:17.721Z",
+ "meanEstimatedListCostUsd": 0.8802499166666667,
+ "meanTokenConsumption": 6197.333333333333,
+ "meanToolInvocationCount": 9.666666666666666
},
{
"experimentId": "claude-fable-5-1-claude-code--baseline",
@@ -416,11 +432,11 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 183777,
- "measuredAt": "2026-09-02T14:19:29.120Z",
- "meanEstimatedListCostUsd": 1.31527725,
- "meanTokenConsumption": 8248,
- "meanToolInvocationCount": 16.666666666666668
+ "meanDurationMs": 315667,
+ "measuredAt": "2026-10-07T18:34:17.732Z",
+ "meanEstimatedListCostUsd": 1.7283125833333335,
+ "meanTokenConsumption": 12128.333333333334,
+ "meanToolInvocationCount": 23
},
{
"experimentId": "claude-fable-5-1-claude-code--baseline",
@@ -428,11 +444,11 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 180513,
- "measuredAt": "2026-09-02T14:19:29.129Z",
- "meanEstimatedListCostUsd": 1.0393497499999997,
- "meanTokenConsumption": 9160.333333333334,
- "meanToolInvocationCount": 9.666666666666666
+ "meanDurationMs": 237115,
+ "measuredAt": "2026-10-07T18:34:17.746Z",
+ "meanEstimatedListCostUsd": 1.02154625,
+ "meanTokenConsumption": 8290,
+ "meanToolInvocationCount": 9.333333333333334
},
{
"experimentId": "claude-fable-5-1-claude-code--baseline",
@@ -440,11 +456,11 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 181059,
- "measuredAt": "2026-09-02T14:19:29.137Z",
- "meanEstimatedListCostUsd": 1.3291795000000002,
- "meanTokenConsumption": 7962.333333333333,
- "meanToolInvocationCount": 16.666666666666668
+ "meanDurationMs": 282249,
+ "measuredAt": "2026-10-07T18:34:17.763Z",
+ "meanEstimatedListCostUsd": 1.7565878333333333,
+ "meanTokenConsumption": 9842.666666666666,
+ "meanToolInvocationCount": 19.666666666666668
},
{
"experimentId": "claude-fable-5-1-claude-code--baseline",
@@ -452,11 +468,11 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 149380,
- "measuredAt": "2026-09-02T14:19:29.147Z",
- "meanEstimatedListCostUsd": 1.0626429999999998,
- "meanTokenConsumption": 4479.333333333333,
- "meanToolInvocationCount": 10.666666666666666
+ "meanDurationMs": 230125,
+ "measuredAt": "2026-10-07T18:34:17.779Z",
+ "meanEstimatedListCostUsd": 1.1730868333333333,
+ "meanTokenConsumption": 4567.666666666667,
+ "meanToolInvocationCount": 12
},
{
"experimentId": "claude-fable-5-1-claude-code--baseline",
@@ -464,11 +480,11 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 151473,
- "measuredAt": "2026-09-02T14:19:29.154Z",
- "meanEstimatedListCostUsd": 0.9185504999999999,
- "meanTokenConsumption": 4827.666666666667,
- "meanToolInvocationCount": 11.666666666666666
+ "meanDurationMs": 180733,
+ "measuredAt": "2026-10-07T18:34:17.794Z",
+ "meanEstimatedListCostUsd": 0.8628102499999999,
+ "meanTokenConsumption": 4052.3333333333335,
+ "meanToolInvocationCount": 10
},
{
"experimentId": "claude-fable-5-1-claude-code--guided",
@@ -476,11 +492,11 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 142627,
- "measuredAt": "2026-09-02T14:18:53.362Z",
- "meanEstimatedListCostUsd": 0.9972783333333334,
- "meanTokenConsumption": 6633,
- "meanToolInvocationCount": 6
+ "meanDurationMs": 216964,
+ "measuredAt": "2026-10-07T18:38:17.304Z",
+ "meanEstimatedListCostUsd": 1.1694255,
+ "meanTokenConsumption": 8664.333333333334,
+ "meanToolInvocationCount": 9.333333333333334
},
{
"experimentId": "claude-fable-5-1-claude-code--guided",
@@ -488,11 +504,11 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 155541,
- "measuredAt": "2026-09-02T14:18:53.381Z",
- "meanEstimatedListCostUsd": 1.6608135,
- "meanTokenConsumption": 4615,
- "meanToolInvocationCount": 9.666666666666666
+ "meanDurationMs": 177448,
+ "measuredAt": "2026-10-07T18:38:17.319Z",
+ "meanEstimatedListCostUsd": 0.6489536666666667,
+ "meanTokenConsumption": 4501,
+ "meanToolInvocationCount": 7.333333333333333
},
{
"experimentId": "claude-fable-5-1-claude-code--guided",
@@ -500,10 +516,10 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 132220,
- "measuredAt": "2026-09-02T14:18:53.396Z",
- "meanEstimatedListCostUsd": 0.8656001666666667,
- "meanTokenConsumption": 3575,
+ "meanDurationMs": 166095,
+ "measuredAt": "2026-10-07T18:38:17.334Z",
+ "meanEstimatedListCostUsd": 0.6243346666666666,
+ "meanTokenConsumption": 2917.3333333333335,
"meanToolInvocationCount": 7.333333333333333
},
{
@@ -512,11 +528,11 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 161398,
- "measuredAt": "2026-09-02T14:18:53.406Z",
- "meanEstimatedListCostUsd": 0.7871509166666666,
- "meanTokenConsumption": 5333.333333333333,
- "meanToolInvocationCount": 8
+ "meanDurationMs": 232620,
+ "measuredAt": "2026-10-07T18:38:17.345Z",
+ "meanEstimatedListCostUsd": 0.7477252499999999,
+ "meanTokenConsumption": 6484.666666666667,
+ "meanToolInvocationCount": 9
},
{
"experimentId": "claude-fable-5-1-claude-code--guided",
@@ -524,11 +540,11 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 143233,
- "measuredAt": "2026-09-02T14:18:53.417Z",
- "meanEstimatedListCostUsd": 0.8737515833333332,
- "meanTokenConsumption": 3909.3333333333335,
- "meanToolInvocationCount": 7
+ "meanDurationMs": 170755,
+ "measuredAt": "2026-10-07T18:38:17.357Z",
+ "meanEstimatedListCostUsd": 0.8186226666666667,
+ "meanTokenConsumption": 4187.333333333333,
+ "meanToolInvocationCount": 7.333333333333333
},
{
"experimentId": "claude-fable-5-1-claude-code--guided",
@@ -536,11 +552,11 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 115430,
- "measuredAt": "2026-09-02T14:18:53.429Z",
- "meanEstimatedListCostUsd": 0.63728775,
- "meanTokenConsumption": 1828.6666666666667,
- "meanToolInvocationCount": 5.666666666666667
+ "meanDurationMs": 154662,
+ "measuredAt": "2026-10-07T18:38:17.370Z",
+ "meanEstimatedListCostUsd": 0.6756555833333332,
+ "meanTokenConsumption": 2156,
+ "meanToolInvocationCount": 6
},
{
"experimentId": "claude-fable-5-1-claude-code--guided",
@@ -548,1859 +564,2027 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 130462,
- "measuredAt": "2026-09-02T14:18:53.440Z",
- "meanEstimatedListCostUsd": 0.6112984166666667,
- "meanTokenConsumption": 2535.3333333333335,
+ "meanDurationMs": 176359,
+ "measuredAt": "2026-10-07T18:38:17.381Z",
+ "meanEstimatedListCostUsd": 0.5568065833333334,
+ "meanTokenConsumption": 2716.3333333333335,
"meanToolInvocationCount": 7.333333333333333
},
{
- "experimentId": "grok-4-6-opencode--baseline",
+ "experimentId": "gpt-5-6-terra-codex--baseline",
"caseId": "author-001-weather-tool",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 38613,
- "measuredAt": "2026-08-28T19:22:01.000Z",
- "meanEstimatedListCostUsd": 0.10434266666666665,
- "meanTokenConsumption": 49566,
- "meanToolInvocationCount": 19
+ "meanDurationMs": 122298,
+ "measuredAt": "2026-10-07T21:58:48.506Z",
+ "meanEstimatedListCostUsd": 0.22163763333333333,
+ "meanTokenConsumption": 37612.666666666664,
+ "meanToolInvocationCount": 7.333333333333333
},
{
- "experimentId": "grok-4-6-opencode--baseline",
+ "experimentId": "gpt-5-6-terra-codex--baseline",
"caseId": "author-002-new-project",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 83985,
- "measuredAt": "2026-08-28T19:25:12.000Z",
- "meanEstimatedListCostUsd": 0.07232866666666667,
- "meanTokenConsumption": 32827.666666666664,
- "meanToolInvocationCount": 16
+ "meanDurationMs": 138040,
+ "measuredAt": "2026-10-07T21:58:48.529Z",
+ "meanEstimatedListCostUsd": 0.18387216666666664,
+ "meanTokenConsumption": 26238.333333333332,
+ "meanToolInvocationCount": 11.333333333333334
},
{
- "experimentId": "grok-4-6-opencode--baseline",
+ "experimentId": "gpt-5-6-terra-codex--baseline",
"caseId": "author-003-openapi-connection",
"status": "current",
- "passedRuns": 0,
+ "passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 521954,
- "measuredAt": "2026-08-28T19:22:01.000Z",
- "meanEstimatedListCostUsd": 1.0741786666666668,
- "meanTokenConsumption": 503422.6666666667,
- "meanToolInvocationCount": 76.66666666666667
+ "meanDurationMs": 117932,
+ "measuredAt": "2026-10-07T21:58:48.540Z",
+ "meanEstimatedListCostUsd": 0.17790966666666666,
+ "meanTokenConsumption": 30419,
+ "meanToolInvocationCount": 7.333333333333333
},
{
- "experimentId": "grok-4-6-opencode--baseline",
+ "experimentId": "gpt-5-6-terra-codex--baseline",
"caseId": "author-004-packaged-skill",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 41894,
- "measuredAt": "2026-08-28T19:22:01.000Z",
- "meanEstimatedListCostUsd": 0.065152,
- "meanTokenConsumption": 29506,
- "meanToolInvocationCount": 23.333333333333332
+ "meanDurationMs": 137504,
+ "measuredAt": "2026-10-07T21:58:48.551Z",
+ "meanEstimatedListCostUsd": 0.18458839999999999,
+ "meanTokenConsumption": 30275,
+ "meanToolInvocationCount": 6.666666666666667
},
{
- "experimentId": "grok-4-6-opencode--baseline",
+ "experimentId": "gpt-5-6-terra-codex--baseline",
"caseId": "author-005-conditional-approval",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 50151,
- "measuredAt": "2026-08-28T19:22:01.000Z",
- "meanEstimatedListCostUsd": 0.141536,
- "meanTokenConsumption": 66629.33333333333,
- "meanToolInvocationCount": 22
+ "meanDurationMs": 127649,
+ "measuredAt": "2026-10-07T21:58:48.568Z",
+ "meanEstimatedListCostUsd": 0.19955179999999997,
+ "meanTokenConsumption": 33755.666666666664,
+ "meanToolInvocationCount": 6.666666666666667
},
{
- "experimentId": "grok-4-6-opencode--baseline",
+ "experimentId": "gpt-5-6-terra-codex--baseline",
"caseId": "author-006-custom-channel",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 37062,
- "measuredAt": "2026-08-28T19:22:01.000Z",
- "meanEstimatedListCostUsd": 0.07319533333333333,
- "meanTokenConsumption": 33801.666666666664,
- "meanToolInvocationCount": 17.666666666666668
+ "meanDurationMs": 116757,
+ "measuredAt": "2026-10-07T21:58:48.584Z",
+ "meanEstimatedListCostUsd": 0.17057733333333333,
+ "meanTokenConsumption": 29653,
+ "meanToolInvocationCount": 6.333333333333333
},
{
- "experimentId": "grok-4-6-opencode--baseline",
+ "experimentId": "gpt-5-6-terra-codex--baseline",
"caseId": "author-007-digest-schedule",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 38280,
- "measuredAt": "2026-08-28T19:22:01.000Z",
- "meanEstimatedListCostUsd": 0.045748000000000004,
- "meanTokenConsumption": 20122.666666666668,
- "meanToolInvocationCount": 18
+ "meanDurationMs": 112507,
+ "measuredAt": "2026-10-07T21:58:48.600Z",
+ "meanEstimatedListCostUsd": 0.14675936666666667,
+ "meanTokenConsumption": 25050.666666666668,
+ "meanToolInvocationCount": 7
},
{
- "experimentId": "grok-4-6-opencode--guided",
+ "experimentId": "gpt-5-6-terra-codex--guided",
"caseId": "author-001-weather-tool",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 31700,
- "measuredAt": "2026-08-28T19:22:02.000Z",
- "meanEstimatedListCostUsd": 0.049695333333333334,
- "meanTokenConsumption": 22963.666666666668,
- "meanToolInvocationCount": 9.666666666666666
+ "meanDurationMs": 119359,
+ "measuredAt": "2026-10-07T22:01:49.428Z",
+ "meanEstimatedListCostUsd": 0.1435413,
+ "meanTokenConsumption": 24455,
+ "meanToolInvocationCount": 4.666666666666667
},
{
- "experimentId": "grok-4-6-opencode--guided",
+ "experimentId": "gpt-5-6-terra-codex--guided",
"caseId": "author-002-new-project",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 83621,
- "measuredAt": "2026-08-28T19:25:07.000Z",
- "meanEstimatedListCostUsd": 0.082842,
- "meanTokenConsumption": 38261,
- "meanToolInvocationCount": 16.333333333333332
+ "meanDurationMs": 161276,
+ "measuredAt": "2026-10-07T22:01:49.437Z",
+ "meanEstimatedListCostUsd": 0.21618046666666665,
+ "meanTokenConsumption": 29838.333333333332,
+ "meanToolInvocationCount": 12.333333333333334
},
{
- "experimentId": "grok-4-6-opencode--guided",
+ "experimentId": "gpt-5-6-terra-codex--guided",
"caseId": "author-003-openapi-connection",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 34833,
- "measuredAt": "2026-08-28T19:25:07.000Z",
- "meanEstimatedListCostUsd": 0.055174,
- "meanTokenConsumption": 25952.333333333332,
- "meanToolInvocationCount": 10
+ "meanDurationMs": 111613,
+ "measuredAt": "2026-10-07T22:01:49.449Z",
+ "meanEstimatedListCostUsd": 0.12379853333333335,
+ "meanTokenConsumption": 22123,
+ "meanToolInvocationCount": 4.333333333333333
},
{
- "experimentId": "grok-4-6-opencode--guided",
+ "experimentId": "gpt-5-6-terra-codex--guided",
"caseId": "author-004-packaged-skill",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 30135,
- "measuredAt": "2026-08-28T19:22:02.000Z",
- "meanEstimatedListCostUsd": 0.035581999999999996,
- "meanTokenConsumption": 16043.666666666666,
- "meanToolInvocationCount": 8.666666666666666
+ "meanDurationMs": 130435,
+ "measuredAt": "2026-10-07T22:01:49.456Z",
+ "meanEstimatedListCostUsd": 0.11441699999999999,
+ "meanTokenConsumption": 19053.333333333332,
+ "meanToolInvocationCount": 3.6666666666666665
},
{
- "experimentId": "grok-4-6-opencode--guided",
+ "experimentId": "gpt-5-6-terra-codex--guided",
"caseId": "author-005-conditional-approval",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 34026,
- "measuredAt": "2026-08-28T19:22:02.000Z",
- "meanEstimatedListCostUsd": 0.038532666666666666,
- "meanTokenConsumption": 17059.666666666668,
- "meanToolInvocationCount": 11.666666666666666
+ "meanDurationMs": 122684,
+ "measuredAt": "2026-10-07T22:01:49.463Z",
+ "meanEstimatedListCostUsd": 0.15166520000000003,
+ "meanTokenConsumption": 27042.333333333332,
+ "meanToolInvocationCount": 4.333333333333333
},
{
- "experimentId": "grok-4-6-opencode--guided",
+ "experimentId": "gpt-5-6-terra-codex--guided",
"caseId": "author-006-custom-channel",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 32109,
- "measuredAt": "2026-08-28T19:22:02.000Z",
- "meanEstimatedListCostUsd": 0.033709333333333334,
- "meanTokenConsumption": 14610,
- "meanToolInvocationCount": 9.666666666666666
+ "meanDurationMs": 133774,
+ "measuredAt": "2026-10-07T22:01:49.471Z",
+ "meanEstimatedListCostUsd": 0.11311536666666666,
+ "meanTokenConsumption": 20847.666666666668,
+ "meanToolInvocationCount": 3.3333333333333335
},
{
- "experimentId": "grok-4-6-opencode--guided",
+ "experimentId": "gpt-5-6-terra-codex--guided",
"caseId": "author-007-digest-schedule",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 25583,
- "measuredAt": "2026-08-28T19:22:02.000Z",
- "meanEstimatedListCostUsd": 0.023458666666666666,
- "meanTokenConsumption": 10328,
- "meanToolInvocationCount": 7.333333333333333
+ "meanDurationMs": 104430,
+ "measuredAt": "2026-10-07T22:01:49.483Z",
+ "meanEstimatedListCostUsd": 0.09763046666666665,
+ "meanTokenConsumption": 18167.666666666668,
+ "meanToolInvocationCount": 3.3333333333333335
},
{
- "experimentId": "gpt-5-6-sol-codex--baseline",
+ "experimentId": "gpt-6-astra-high-codex--baseline",
"caseId": "author-001-weather-tool",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 175042,
- "measuredAt": "2026-08-31T19:20:09.055Z",
- "meanEstimatedListCostUsd": 0.42096675,
- "meanTokenConsumption": 49663.666666666664,
- "meanToolInvocationCount": 13.666666666666666
+ "meanDurationMs": 237825,
+ "measuredAt": "2026-10-07T17:36:21.398Z",
+ "meanEstimatedListCostUsd": 1.2291793333333334,
+ "meanTokenConsumption": 40990.333333333336,
+ "meanToolInvocationCount": 14
},
{
- "experimentId": "gpt-5-6-sol-codex--baseline",
+ "experimentId": "gpt-6-astra-high-codex--baseline",
"caseId": "author-002-new-project",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 122470,
- "measuredAt": "2026-08-31T19:20:09.085Z",
- "meanEstimatedListCostUsd": 0.23273858333333333,
- "meanTokenConsumption": 30848.666666666668,
+ "meanDurationMs": 134301,
+ "measuredAt": "2026-10-07T17:36:21.424Z",
+ "meanEstimatedListCostUsd": 0.746785,
+ "meanTokenConsumption": 23511,
"meanToolInvocationCount": 10.333333333333334
},
{
- "experimentId": "gpt-5-6-sol-codex--baseline",
+ "experimentId": "gpt-6-astra-high-codex--baseline",
"caseId": "author-003-openapi-connection",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 132036,
- "measuredAt": "2026-08-31T19:20:09.104Z",
- "meanEstimatedListCostUsd": 0.367498625,
- "meanTokenConsumption": 48982.666666666664,
- "meanToolInvocationCount": 9.333333333333334
+ "meanDurationMs": 141746,
+ "measuredAt": "2026-10-07T17:36:21.437Z",
+ "meanEstimatedListCostUsd": 0.8998284999999999,
+ "meanTokenConsumption": 32579.666666666668,
+ "meanToolInvocationCount": 10.666666666666666
},
{
- "experimentId": "gpt-5-6-sol-codex--baseline",
+ "experimentId": "gpt-6-astra-high-codex--baseline",
"caseId": "author-004-packaged-skill",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 137447,
- "measuredAt": "2026-08-31T19:20:09.128Z",
- "meanEstimatedListCostUsd": 0.26755258333333337,
- "meanTokenConsumption": 36330.666666666664,
- "meanToolInvocationCount": 6
+ "meanDurationMs": 217226,
+ "measuredAt": "2026-10-07T17:36:21.447Z",
+ "meanEstimatedListCostUsd": 0.9555386666666666,
+ "meanTokenConsumption": 31480.666666666668,
+ "meanToolInvocationCount": 13.333333333333334
},
{
- "experimentId": "gpt-5-6-sol-codex--baseline",
+ "experimentId": "gpt-6-astra-high-codex--baseline",
"caseId": "author-005-conditional-approval",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 121857,
- "measuredAt": "2026-08-31T19:20:09.143Z",
- "meanEstimatedListCostUsd": 0.2530605833333333,
- "meanTokenConsumption": 32384,
- "meanToolInvocationCount": 10
+ "meanDurationMs": 170948,
+ "measuredAt": "2026-10-07T17:36:21.456Z",
+ "meanEstimatedListCostUsd": 1.1034936666666666,
+ "meanTokenConsumption": 37715.333333333336,
+ "meanToolInvocationCount": 12.666666666666666
},
{
- "experimentId": "gpt-5-6-sol-codex--baseline",
+ "experimentId": "gpt-6-astra-high-codex--baseline",
"caseId": "author-006-custom-channel",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 115653,
- "measuredAt": "2026-08-31T19:20:09.156Z",
- "meanEstimatedListCostUsd": 0.251469,
- "meanTokenConsumption": 33854.666666666664,
- "meanToolInvocationCount": 8
+ "meanDurationMs": 156521,
+ "measuredAt": "2026-10-07T17:36:21.466Z",
+ "meanEstimatedListCostUsd": 1.3502271666666665,
+ "meanTokenConsumption": 46148.666666666664,
+ "meanToolInvocationCount": 13
},
{
- "experimentId": "gpt-5-6-sol-codex--baseline",
+ "experimentId": "gpt-6-astra-high-codex--baseline",
"caseId": "author-007-digest-schedule",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 121172,
- "measuredAt": "2026-08-31T19:20:09.170Z",
- "meanEstimatedListCostUsd": 0.284567375,
- "meanTokenConsumption": 37330.666666666664,
- "meanToolInvocationCount": 11
+ "meanDurationMs": 140369,
+ "measuredAt": "2026-10-07T17:36:21.479Z",
+ "meanEstimatedListCostUsd": 0.7009501666666668,
+ "meanTokenConsumption": 23989.333333333332,
+ "meanToolInvocationCount": 9.666666666666666
},
{
- "experimentId": "gpt-5-6-sol-codex--guided",
+ "experimentId": "gpt-6-astra-high-codex--guided",
"caseId": "author-001-weather-tool",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 115694,
- "measuredAt": "2026-08-31T19:18:53.704Z",
- "meanEstimatedListCostUsd": 0.18167699999999998,
- "meanTokenConsumption": 24500,
- "meanToolInvocationCount": 4.666666666666667
+ "meanDurationMs": 167803,
+ "measuredAt": "2026-10-07T17:40:30.710Z",
+ "meanEstimatedListCostUsd": 0.8353351666666667,
+ "meanTokenConsumption": 29147.333333333332,
+ "meanToolInvocationCount": 7.333333333333333
},
{
- "experimentId": "gpt-5-6-sol-codex--guided",
+ "experimentId": "gpt-6-astra-high-codex--guided",
"caseId": "author-002-new-project",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 132614,
- "measuredAt": "2026-08-31T19:18:53.727Z",
- "meanEstimatedListCostUsd": 0.22479154166666668,
- "meanTokenConsumption": 28792.333333333332,
- "meanToolInvocationCount": 9.333333333333334
+ "meanDurationMs": 128241,
+ "measuredAt": "2026-10-07T17:40:30.723Z",
+ "meanEstimatedListCostUsd": 0.8843616666666666,
+ "meanTokenConsumption": 27507.333333333332,
+ "meanToolInvocationCount": 13.666666666666666
},
{
- "experimentId": "gpt-5-6-sol-codex--guided",
+ "experimentId": "gpt-6-astra-high-codex--guided",
"caseId": "author-003-openapi-connection",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 101648,
- "measuredAt": "2026-08-31T19:18:53.741Z",
- "meanEstimatedListCostUsd": 0.14007875,
- "meanTokenConsumption": 19209.666666666668,
- "meanToolInvocationCount": 5.333333333333333
+ "meanDurationMs": 143101,
+ "measuredAt": "2026-10-07T17:40:30.733Z",
+ "meanEstimatedListCostUsd": 0.6892401666666667,
+ "meanTokenConsumption": 25532.666666666668,
+ "meanToolInvocationCount": 8.333333333333334
},
{
- "experimentId": "gpt-5-6-sol-codex--guided",
+ "experimentId": "gpt-6-astra-high-codex--guided",
"caseId": "author-004-packaged-skill",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 129791,
- "measuredAt": "2026-08-31T19:18:53.751Z",
- "meanEstimatedListCostUsd": 0.19388683333333334,
- "meanTokenConsumption": 24916,
- "meanToolInvocationCount": 6
+ "meanDurationMs": 196512,
+ "measuredAt": "2026-10-07T17:40:30.742Z",
+ "meanEstimatedListCostUsd": 0.7988335000000001,
+ "meanTokenConsumption": 26211.666666666668,
+ "meanToolInvocationCount": 8.666666666666666
},
{
- "experimentId": "gpt-5-6-sol-codex--guided",
+ "experimentId": "gpt-6-astra-high-codex--guided",
"caseId": "author-005-conditional-approval",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 108054,
- "measuredAt": "2026-08-31T19:18:53.759Z",
- "meanEstimatedListCostUsd": 0.15307379166666665,
- "meanTokenConsumption": 20874.333333333332,
- "meanToolInvocationCount": 4.666666666666667
+ "meanDurationMs": 128279,
+ "measuredAt": "2026-10-07T17:40:30.750Z",
+ "meanEstimatedListCostUsd": 0.6823355000000001,
+ "meanTokenConsumption": 25595.333333333332,
+ "meanToolInvocationCount": 5.333333333333333
},
{
- "experimentId": "gpt-5-6-sol-codex--guided",
+ "experimentId": "gpt-6-astra-high-codex--guided",
"caseId": "author-006-custom-channel",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 100410,
- "measuredAt": "2026-08-31T19:18:53.769Z",
- "meanEstimatedListCostUsd": 0.108055625,
- "meanTokenConsumption": 15663.666666666666,
- "meanToolInvocationCount": 3
+ "meanDurationMs": 143013,
+ "measuredAt": "2026-10-07T17:40:30.757Z",
+ "meanEstimatedListCostUsd": 0.5633055,
+ "meanTokenConsumption": 21594.666666666668,
+ "meanToolInvocationCount": 5
},
{
- "experimentId": "gpt-5-6-sol-codex--guided",
+ "experimentId": "gpt-6-astra-high-codex--guided",
"caseId": "author-007-digest-schedule",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 103849,
- "measuredAt": "2026-08-31T19:18:53.779Z",
- "meanEstimatedListCostUsd": 0.11547295833333333,
- "meanTokenConsumption": 16687.333333333332,
- "meanToolInvocationCount": 3.6666666666666665
+ "meanDurationMs": 133217,
+ "measuredAt": "2026-10-07T17:40:30.763Z",
+ "meanEstimatedListCostUsd": 0.5985591666666666,
+ "meanTokenConsumption": 21658.333333333332,
+ "meanToolInvocationCount": 6.666666666666667
},
{
- "experimentId": "gpt-5-6-terra-codex--baseline",
+ "experimentId": "claude-sonnet-5-claude-code--baseline",
"caseId": "author-001-weather-tool",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 119086,
- "measuredAt": "2026-08-31T20:01:56.276Z",
- "meanEstimatedListCostUsd": 0.2552948666666666,
- "meanTokenConsumption": 44396,
- "meanToolInvocationCount": 6
+ "meanDurationMs": 127080,
+ "measuredAt": "2026-10-07T17:14:32.284Z",
+ "meanEstimatedListCostUsd": 0.33771066666666666,
+ "meanTokenConsumption": 5721.333333333333,
+ "meanToolInvocationCount": 10.666666666666666
},
{
- "experimentId": "gpt-5-6-terra-codex--baseline",
+ "experimentId": "claude-sonnet-5-claude-code--baseline",
"caseId": "author-002-new-project",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 117037,
- "measuredAt": "2026-08-31T20:01:56.312Z",
- "meanEstimatedListCostUsd": 0.1636081,
- "meanTokenConsumption": 26205.333333333332,
- "meanToolInvocationCount": 8.666666666666666
+ "meanDurationMs": 122433,
+ "measuredAt": "2026-10-07T17:14:32.309Z",
+ "meanEstimatedListCostUsd": 0.3149904,
+ "meanTokenConsumption": 4382.333333333333,
+ "meanToolInvocationCount": 9.333333333333334
},
{
- "experimentId": "gpt-5-6-terra-codex--baseline",
+ "experimentId": "claude-sonnet-5-claude-code--baseline",
"caseId": "author-003-openapi-connection",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 115379,
- "measuredAt": "2026-08-31T20:01:56.326Z",
- "meanEstimatedListCostUsd": 0.18920926666666668,
- "meanTokenConsumption": 33257.666666666664,
- "meanToolInvocationCount": 6.333333333333333
+ "meanDurationMs": 121003,
+ "measuredAt": "2026-10-07T17:14:32.328Z",
+ "meanEstimatedListCostUsd": 0.4386330666666666,
+ "meanTokenConsumption": 5224,
+ "meanToolInvocationCount": 16
},
{
- "experimentId": "gpt-5-6-terra-codex--baseline",
+ "experimentId": "claude-sonnet-5-claude-code--baseline",
"caseId": "author-004-packaged-skill",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 119436,
- "measuredAt": "2026-08-31T20:01:56.338Z",
- "meanEstimatedListCostUsd": 0.16241646666666668,
- "meanTokenConsumption": 27236.666666666668,
- "meanToolInvocationCount": 5.666666666666667
+ "meanDurationMs": 135343,
+ "measuredAt": "2026-10-07T17:14:32.344Z",
+ "meanEstimatedListCostUsd": 0.32376240000000006,
+ "meanTokenConsumption": 5496,
+ "meanToolInvocationCount": 11.666666666666666
},
{
- "experimentId": "gpt-5-6-terra-codex--baseline",
+ "experimentId": "claude-sonnet-5-claude-code--baseline",
"caseId": "author-005-conditional-approval",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 105622,
- "measuredAt": "2026-08-31T20:01:56.348Z",
- "meanEstimatedListCostUsd": 0.1821714666666667,
- "meanTokenConsumption": 32177,
- "meanToolInvocationCount": 6.333333333333333
+ "meanDurationMs": 112919,
+ "measuredAt": "2026-10-07T17:14:32.359Z",
+ "meanEstimatedListCostUsd": 0.33600040000000003,
+ "meanTokenConsumption": 3654,
+ "meanToolInvocationCount": 10.333333333333334
},
{
- "experimentId": "gpt-5-6-terra-codex--baseline",
+ "experimentId": "claude-sonnet-5-claude-code--baseline",
"caseId": "author-006-custom-channel",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 108983,
- "measuredAt": "2026-08-31T20:01:56.359Z",
- "meanEstimatedListCostUsd": 0.2155289,
- "meanTokenConsumption": 37516,
- "meanToolInvocationCount": 7
+ "meanDurationMs": 115056,
+ "measuredAt": "2026-10-07T17:14:32.373Z",
+ "meanEstimatedListCostUsd": 0.32271563333333336,
+ "meanTokenConsumption": 4199,
+ "meanToolInvocationCount": 11.666666666666666
},
{
- "experimentId": "gpt-5-6-terra-codex--baseline",
+ "experimentId": "claude-sonnet-5-claude-code--baseline",
"caseId": "author-007-digest-schedule",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 103918,
- "measuredAt": "2026-08-31T20:01:56.372Z",
- "meanEstimatedListCostUsd": 0.12170393333333335,
- "meanTokenConsumption": 21681.666666666668,
- "meanToolInvocationCount": 5.333333333333333
+ "meanDurationMs": 112597,
+ "measuredAt": "2026-10-07T17:14:32.383Z",
+ "meanEstimatedListCostUsd": 0.20493166666666665,
+ "meanTokenConsumption": 2047.6666666666667,
+ "meanToolInvocationCount": 7.666666666666667
},
{
- "experimentId": "gpt-5-6-terra-codex--guided",
+ "experimentId": "claude-sonnet-5-claude-code--guided",
"caseId": "author-001-weather-tool",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 113388,
- "measuredAt": "2026-08-31T20:02:25.229Z",
- "meanEstimatedListCostUsd": 0.15296396666666667,
- "meanTokenConsumption": 26379.666666666668,
- "meanToolInvocationCount": 5
+ "meanDurationMs": 122291,
+ "measuredAt": "2026-10-07T17:19:07.735Z",
+ "meanEstimatedListCostUsd": 0.3211207,
+ "meanTokenConsumption": 3416.6666666666665,
+ "meanToolInvocationCount": 9
},
{
- "experimentId": "gpt-5-6-terra-codex--guided",
+ "experimentId": "claude-sonnet-5-claude-code--guided",
"caseId": "author-002-new-project",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 128859,
- "measuredAt": "2026-08-31T20:02:25.256Z",
- "meanEstimatedListCostUsd": 0.1434823,
- "meanTokenConsumption": 22726.666666666668,
- "meanToolInvocationCount": 10.333333333333334
+ "meanDurationMs": 115898,
+ "measuredAt": "2026-10-07T17:19:07.746Z",
+ "meanEstimatedListCostUsd": 0.2441989,
+ "meanTokenConsumption": 3790.3333333333335,
+ "meanToolInvocationCount": 8.333333333333334
},
{
- "experimentId": "gpt-5-6-terra-codex--guided",
+ "experimentId": "claude-sonnet-5-claude-code--guided",
"caseId": "author-003-openapi-connection",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 101879,
- "measuredAt": "2026-08-31T20:02:25.271Z",
- "meanEstimatedListCostUsd": 0.1165826,
- "meanTokenConsumption": 20464.333333333332,
- "meanToolInvocationCount": 4.666666666666667
+ "meanDurationMs": 123110,
+ "measuredAt": "2026-10-07T17:19:07.756Z",
+ "meanEstimatedListCostUsd": 0.27517036666666667,
+ "meanTokenConsumption": 2186.3333333333335,
+ "meanToolInvocationCount": 7.666666666666667
},
{
- "experimentId": "gpt-5-6-terra-codex--guided",
+ "experimentId": "claude-sonnet-5-claude-code--guided",
"caseId": "author-004-packaged-skill",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 112598,
- "measuredAt": "2026-08-31T20:02:25.279Z",
- "meanEstimatedListCostUsd": 0.11589093333333333,
- "meanTokenConsumption": 19442.333333333332,
- "meanToolInvocationCount": 3.6666666666666665
+ "meanDurationMs": 127050,
+ "measuredAt": "2026-10-07T17:19:07.766Z",
+ "meanEstimatedListCostUsd": 0.1290148,
+ "meanTokenConsumption": 2172.6666666666665,
+ "meanToolInvocationCount": 5
},
{
- "experimentId": "gpt-5-6-terra-codex--guided",
+ "experimentId": "claude-sonnet-5-claude-code--guided",
"caseId": "author-005-conditional-approval",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 102311,
- "measuredAt": "2026-08-31T20:02:25.288Z",
- "meanEstimatedListCostUsd": 0.1219281,
- "meanTokenConsumption": 22099.666666666668,
- "meanToolInvocationCount": 4
+ "meanDurationMs": 132250,
+ "measuredAt": "2026-10-07T17:19:07.780Z",
+ "meanEstimatedListCostUsd": 0.350098,
+ "meanTokenConsumption": 3087,
+ "meanToolInvocationCount": 8.666666666666666
},
{
- "experimentId": "gpt-5-6-terra-codex--guided",
+ "experimentId": "claude-sonnet-5-claude-code--guided",
"caseId": "author-006-custom-channel",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 100329,
- "measuredAt": "2026-08-31T20:02:25.295Z",
- "meanEstimatedListCostUsd": 0.10267943333333335,
- "meanTokenConsumption": 18267.333333333332,
- "meanToolInvocationCount": 4
+ "meanDurationMs": 140975,
+ "measuredAt": "2026-10-07T17:19:07.793Z",
+ "meanEstimatedListCostUsd": 0.29428096666666664,
+ "meanTokenConsumption": 2261.6666666666665,
+ "meanToolInvocationCount": 8
},
{
- "experimentId": "gpt-5-6-terra-codex--guided",
+ "experimentId": "claude-sonnet-5-claude-code--guided",
"caseId": "author-007-digest-schedule",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 96962,
- "measuredAt": "2026-08-31T20:02:25.301Z",
- "meanEstimatedListCostUsd": 0.08585559999999999,
- "meanTokenConsumption": 15490.333333333334,
- "meanToolInvocationCount": 3.6666666666666665
+ "meanDurationMs": 105861,
+ "measuredAt": "2026-10-07T17:19:07.804Z",
+ "meanEstimatedListCostUsd": 0.14527139999999997,
+ "meanTokenConsumption": 1264.3333333333333,
+ "meanToolInvocationCount": 4.333333333333333
},
{
- "experimentId": "gpt-6-astra-high-codex--baseline",
+ "experimentId": "glm-5-3-opencode--baseline",
"caseId": "author-001-weather-tool",
"status": "current",
- "passedRuns": 3,
+ "passedRuns": 2,
"validRuns": 3,
- "meanDurationMs": 165398,
- "measuredAt": "2026-09-15T16:38:38.989Z",
- "meanEstimatedListCostUsd": 1.164401,
- "meanTokenConsumption": 38281.666666666664,
- "meanToolInvocationCount": 14
+ "meanDurationMs": 610963,
+ "measuredAt": "2026-10-07T21:26:04.429Z",
+ "meanEstimatedListCostUsd": 0.15013914,
+ "meanTokenConsumption": 36303.5,
+ "meanToolInvocationCount": 29
},
{
- "experimentId": "gpt-6-astra-high-codex--baseline",
+ "experimentId": "glm-5-3-opencode--baseline",
"caseId": "author-002-new-project",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 114673,
- "measuredAt": "2026-09-15T16:38:39.004Z",
- "meanEstimatedListCostUsd": 0.77442,
- "meanTokenConsumption": 24544.666666666668,
- "meanToolInvocationCount": 12.666666666666666
+ "meanDurationMs": 304196,
+ "measuredAt": "2026-10-07T21:26:04.443Z",
+ "meanEstimatedListCostUsd": 0.07239437333333333,
+ "meanTokenConsumption": 17362,
+ "meanToolInvocationCount": 18.333333333333332
},
{
- "experimentId": "gpt-6-astra-high-codex--baseline",
+ "experimentId": "glm-5-3-opencode--baseline",
"caseId": "author-003-openapi-connection",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 139045,
- "measuredAt": "2026-09-15T16:38:39.012Z",
- "meanEstimatedListCostUsd": 0.9837176666666666,
- "meanTokenConsumption": 34029,
- "meanToolInvocationCount": 11.666666666666666
+ "meanDurationMs": 352602,
+ "measuredAt": "2026-10-07T21:26:04.457Z",
+ "meanEstimatedListCostUsd": 0.15343737333333332,
+ "meanTokenConsumption": 32562.666666666668,
+ "meanToolInvocationCount": 34.666666666666664
},
{
- "experimentId": "gpt-6-astra-high-codex--baseline",
+ "experimentId": "glm-5-3-opencode--baseline",
"caseId": "author-004-packaged-skill",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 163959,
- "measuredAt": "2026-09-15T16:38:39.020Z",
- "meanEstimatedListCostUsd": 1.0705643333333335,
- "meanTokenConsumption": 33997.333333333336,
- "meanToolInvocationCount": 16
+ "meanDurationMs": 372568,
+ "measuredAt": "2026-10-07T21:26:04.479Z",
+ "meanEstimatedListCostUsd": 0.0860574,
+ "meanTokenConsumption": 20747,
+ "meanToolInvocationCount": 24.333333333333332
},
{
- "experimentId": "gpt-6-astra-high-codex--baseline",
+ "experimentId": "glm-5-3-opencode--baseline",
"caseId": "author-005-conditional-approval",
"status": "current",
- "passedRuns": 3,
+ "passedRuns": 2,
"validRuns": 3,
- "meanDurationMs": 140774,
- "measuredAt": "2026-09-15T16:38:39.030Z",
- "meanEstimatedListCostUsd": 1.1568405000000002,
- "meanTokenConsumption": 38287.333333333336,
- "meanToolInvocationCount": 11.666666666666666
+ "meanDurationMs": 409549,
+ "measuredAt": "2026-10-07T21:26:04.497Z",
+ "meanEstimatedListCostUsd": 0.12993986666666668,
+ "meanTokenConsumption": 30566,
+ "meanToolInvocationCount": 29.333333333333332
},
{
- "experimentId": "gpt-6-astra-high-codex--baseline",
+ "experimentId": "glm-5-3-opencode--baseline",
"caseId": "author-006-custom-channel",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 124395,
- "measuredAt": "2026-09-15T16:38:39.041Z",
- "meanEstimatedListCostUsd": 0.9883605000000001,
- "meanTokenConsumption": 35251.666666666664,
- "meanToolInvocationCount": 11.333333333333334
+ "meanDurationMs": 378657,
+ "measuredAt": "2026-10-07T21:26:04.520Z",
+ "meanEstimatedListCostUsd": 0.15328993333333335,
+ "meanTokenConsumption": 34802,
+ "meanToolInvocationCount": 32
},
{
- "experimentId": "gpt-6-astra-high-codex--baseline",
+ "experimentId": "glm-5-3-opencode--baseline",
"caseId": "author-007-digest-schedule",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 122178,
- "measuredAt": "2026-09-15T16:38:39.053Z",
- "meanEstimatedListCostUsd": 0.7496145,
- "meanTokenConsumption": 26059.333333333332,
- "meanToolInvocationCount": 9.333333333333334
+ "meanDurationMs": 282814,
+ "measuredAt": "2026-10-07T21:26:04.542Z",
+ "meanEstimatedListCostUsd": 0.06492005333333334,
+ "meanTokenConsumption": 14953,
+ "meanToolInvocationCount": 26.333333333333332
},
{
- "experimentId": "gpt-6-astra-high-codex--guided",
+ "experimentId": "glm-5-3-opencode--guided",
"caseId": "author-001-weather-tool",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 134344,
- "measuredAt": "2026-09-15T16:38:40.721Z",
- "meanEstimatedListCostUsd": 0.7615181666666667,
- "meanTokenConsumption": 27210.666666666668,
- "meanToolInvocationCount": 6.666666666666667
+ "meanDurationMs": 368173,
+ "measuredAt": "2026-10-07T21:33:02.073Z",
+ "meanEstimatedListCostUsd": 0.11302922666666666,
+ "meanTokenConsumption": 25322.333333333332,
+ "meanToolInvocationCount": 19.666666666666668
},
{
- "experimentId": "gpt-6-astra-high-codex--guided",
+ "experimentId": "glm-5-3-opencode--guided",
"caseId": "author-002-new-project",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 118677,
- "measuredAt": "2026-09-15T16:38:40.727Z",
- "meanEstimatedListCostUsd": 0.6784536666666666,
- "meanTokenConsumption": 21192,
- "meanToolInvocationCount": 11.666666666666666
+ "meanDurationMs": 225257,
+ "measuredAt": "2026-10-07T21:33:02.095Z",
+ "meanEstimatedListCostUsd": 0.051035119999999996,
+ "meanTokenConsumption": 14036.666666666666,
+ "meanToolInvocationCount": 17.333333333333332
},
{
- "experimentId": "gpt-6-astra-high-codex--guided",
+ "experimentId": "glm-5-3-opencode--guided",
"caseId": "author-003-openapi-connection",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 113782,
- "measuredAt": "2026-09-15T16:38:40.734Z",
- "meanEstimatedListCostUsd": 0.6524503333333334,
- "meanTokenConsumption": 24654.333333333332,
- "meanToolInvocationCount": 7.333333333333333
+ "meanDurationMs": 187845,
+ "measuredAt": "2026-10-07T21:33:02.118Z",
+ "meanEstimatedListCostUsd": 0.038902653333333335,
+ "meanTokenConsumption": 12674,
+ "meanToolInvocationCount": 10.666666666666666
},
{
- "experimentId": "gpt-6-astra-high-codex--guided",
+ "experimentId": "glm-5-3-opencode--guided",
"caseId": "author-004-packaged-skill",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 161943,
- "measuredAt": "2026-09-15T16:38:40.740Z",
- "meanEstimatedListCostUsd": 0.8340828333333333,
- "meanTokenConsumption": 26331.333333333332,
- "meanToolInvocationCount": 10.333333333333334
+ "meanDurationMs": 240967,
+ "measuredAt": "2026-10-07T21:33:02.139Z",
+ "meanEstimatedListCostUsd": 0.04021544,
+ "meanTokenConsumption": 9470.666666666666,
+ "meanToolInvocationCount": 11.666666666666666
},
{
- "experimentId": "gpt-6-astra-high-codex--guided",
+ "experimentId": "glm-5-3-opencode--guided",
"caseId": "author-005-conditional-approval",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 120364,
- "measuredAt": "2026-09-15T16:38:40.749Z",
- "meanEstimatedListCostUsd": 0.674382,
- "meanTokenConsumption": 24265.333333333332,
- "meanToolInvocationCount": 5.666666666666667
+ "meanDurationMs": 286860,
+ "measuredAt": "2026-10-07T21:33:02.157Z",
+ "meanEstimatedListCostUsd": 0.09585413333333333,
+ "meanTokenConsumption": 23787.666666666668,
+ "meanToolInvocationCount": 19
},
{
- "experimentId": "gpt-6-astra-high-codex--guided",
+ "experimentId": "glm-5-3-opencode--guided",
"caseId": "author-006-custom-channel",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 104841,
- "measuredAt": "2026-09-15T16:38:40.756Z",
- "meanEstimatedListCostUsd": 0.5142598333333334,
- "meanTokenConsumption": 20021.333333333332,
- "meanToolInvocationCount": 5
+ "meanDurationMs": 251041,
+ "measuredAt": "2026-10-07T21:33:02.182Z",
+ "meanEstimatedListCostUsd": 0.07182817333333334,
+ "meanTokenConsumption": 19015,
+ "meanToolInvocationCount": 17
},
{
- "experimentId": "gpt-6-astra-high-codex--guided",
+ "experimentId": "glm-5-3-opencode--guided",
"caseId": "author-007-digest-schedule",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 119293,
- "measuredAt": "2026-09-15T16:38:40.763Z",
- "meanEstimatedListCostUsd": 0.5865495,
- "meanTokenConsumption": 21058.333333333332,
- "meanToolInvocationCount": 7
+ "meanDurationMs": 203204,
+ "measuredAt": "2026-10-07T21:33:02.208Z",
+ "meanEstimatedListCostUsd": 0.035970919999999997,
+ "meanTokenConsumption": 8741.666666666666,
+ "meanToolInvocationCount": 11.333333333333334
},
{
- "experimentId": "claude-sonnet-5-claude-code--baseline",
+ "experimentId": "gemini-3-1-pro-preview-opencode--baseline",
"caseId": "author-001-weather-tool",
"status": "current",
- "passedRuns": 3,
+ "passedRuns": 2,
"validRuns": 3,
- "meanDurationMs": 119150,
- "measuredAt": "2026-09-02T14:38:33.830Z",
- "meanEstimatedListCostUsd": 0.4004288333333334,
- "meanTokenConsumption": 4827,
- "meanToolInvocationCount": 11.333333333333334
+ "meanDurationMs": 247589,
+ "measuredAt": "2026-10-07T21:48:35.018Z",
+ "meanEstimatedListCostUsd": 0.5625234,
+ "meanTokenConsumption": 221445.33333333334,
+ "meanToolInvocationCount": 20.333333333333332
},
{
- "experimentId": "claude-sonnet-5-claude-code--baseline",
+ "experimentId": "gemini-3-1-pro-preview-opencode--baseline",
"caseId": "author-002-new-project",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 117102,
- "measuredAt": "2026-09-02T14:38:33.846Z",
- "meanEstimatedListCostUsd": 0.38767966666666664,
- "meanTokenConsumption": 3335.6666666666665,
- "meanToolInvocationCount": 8.333333333333334
+ "meanDurationMs": 125133,
+ "measuredAt": "2026-10-07T21:48:35.035Z",
+ "meanEstimatedListCostUsd": 0.0943808,
+ "meanTokenConsumption": 37827.666666666664,
+ "meanToolInvocationCount": 7.333333333333333
},
{
- "experimentId": "claude-sonnet-5-claude-code--baseline",
+ "experimentId": "gemini-3-1-pro-preview-opencode--baseline",
"caseId": "author-003-openapi-connection",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 146914,
- "measuredAt": "2026-09-02T14:38:33.859Z",
- "meanEstimatedListCostUsd": 0.6134467,
- "meanTokenConsumption": 7727,
- "meanToolInvocationCount": 20.333333333333332
+ "meanDurationMs": 296490,
+ "measuredAt": "2026-10-07T21:48:35.048Z",
+ "meanEstimatedListCostUsd": 0.5817098666666666,
+ "meanTokenConsumption": 204829.33333333334,
+ "meanToolInvocationCount": 29.333333333333332
},
{
- "experimentId": "claude-sonnet-5-claude-code--baseline",
+ "experimentId": "gemini-3-1-pro-preview-opencode--baseline",
"caseId": "author-004-packaged-skill",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 128723,
- "measuredAt": "2026-09-02T14:38:33.872Z",
- "meanEstimatedListCostUsd": 0.2731853333333334,
- "meanTokenConsumption": 4274,
- "meanToolInvocationCount": 10.333333333333334
+ "meanDurationMs": 144194,
+ "measuredAt": "2026-10-07T21:48:35.070Z",
+ "meanEstimatedListCostUsd": 0.0816354,
+ "meanTokenConsumption": 30500.666666666668,
+ "meanToolInvocationCount": 5
},
{
- "experimentId": "claude-sonnet-5-claude-code--baseline",
+ "experimentId": "gemini-3-1-pro-preview-opencode--baseline",
"caseId": "author-005-conditional-approval",
"status": "current",
- "passedRuns": 3,
+ "passedRuns": 1,
"validRuns": 3,
- "meanDurationMs": 112957,
- "measuredAt": "2026-09-02T14:38:33.882Z",
- "meanEstimatedListCostUsd": 0.3310536,
- "meanTokenConsumption": 3567.3333333333335,
- "meanToolInvocationCount": 12.666666666666666
+ "meanDurationMs": 220270,
+ "measuredAt": "2026-10-07T21:48:35.083Z",
+ "meanEstimatedListCostUsd": 0.35649686666666663,
+ "meanTokenConsumption": 114200.66666666667,
+ "meanToolInvocationCount": 17.666666666666668
},
{
- "experimentId": "claude-sonnet-5-claude-code--baseline",
+ "experimentId": "gemini-3-1-pro-preview-opencode--baseline",
"caseId": "author-006-custom-channel",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 206252,
- "measuredAt": "2026-09-02T14:38:33.890Z",
- "meanEstimatedListCostUsd": 0.3613244666666667,
- "meanTokenConsumption": 5164.666666666667,
- "meanToolInvocationCount": 9.666666666666666
+ "meanDurationMs": 232728,
+ "measuredAt": "2026-10-07T21:48:35.102Z",
+ "meanEstimatedListCostUsd": 0.29122646666666663,
+ "meanTokenConsumption": 110766.33333333333,
+ "meanToolInvocationCount": 16
},
{
- "experimentId": "claude-sonnet-5-claude-code--baseline",
+ "experimentId": "gemini-3-1-pro-preview-opencode--baseline",
"caseId": "author-007-digest-schedule",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 104010,
- "measuredAt": "2026-09-02T14:38:33.898Z",
- "meanEstimatedListCostUsd": 0.22264466666666669,
- "meanTokenConsumption": 1994.3333333333333,
- "meanToolInvocationCount": 7.333333333333333
+ "meanDurationMs": 152122,
+ "measuredAt": "2026-10-07T21:48:35.121Z",
+ "meanEstimatedListCostUsd": 0.14784206666666666,
+ "meanTokenConsumption": 58074,
+ "meanToolInvocationCount": 11.666666666666666
},
{
- "experimentId": "claude-sonnet-5-claude-code--guided",
+ "experimentId": "gemini-3-1-pro-preview-opencode--guided",
"caseId": "author-001-weather-tool",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 130089,
- "measuredAt": "2026-09-02T14:34:31.320Z",
- "meanEstimatedListCostUsd": 0.40971483333333336,
- "meanTokenConsumption": 4909,
- "meanToolInvocationCount": 10.666666666666666
+ "meanDurationMs": 141789,
+ "measuredAt": "2026-10-07T21:52:05.892Z",
+ "meanEstimatedListCostUsd": 0.153628,
+ "meanTokenConsumption": 54885.333333333336,
+ "meanToolInvocationCount": 10
},
{
- "experimentId": "claude-sonnet-5-claude-code--guided",
+ "experimentId": "gemini-3-1-pro-preview-opencode--guided",
"caseId": "author-002-new-project",
"status": "current",
- "passedRuns": 3,
+ "passedRuns": 2,
"validRuns": 3,
- "meanDurationMs": 119473,
- "measuredAt": "2026-09-02T14:34:31.340Z",
- "meanEstimatedListCostUsd": 0.2871293,
- "meanTokenConsumption": 3610.6666666666665,
+ "meanDurationMs": 145109,
+ "measuredAt": "2026-10-07T21:52:05.906Z",
+ "meanEstimatedListCostUsd": 0.12079206666666666,
+ "meanTokenConsumption": 46447.333333333336,
"meanToolInvocationCount": 10.333333333333334
},
{
- "experimentId": "claude-sonnet-5-claude-code--guided",
+ "experimentId": "gemini-3-1-pro-preview-opencode--guided",
"caseId": "author-003-openapi-connection",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 112748,
- "measuredAt": "2026-09-02T14:34:31.354Z",
- "meanEstimatedListCostUsd": 0.3381717,
- "meanTokenConsumption": 2513,
- "meanToolInvocationCount": 8.333333333333334
+ "meanDurationMs": 145639,
+ "measuredAt": "2026-10-07T21:52:05.916Z",
+ "meanEstimatedListCostUsd": 0.14859993333333335,
+ "meanTokenConsumption": 51636,
+ "meanToolInvocationCount": 9.666666666666666
},
{
- "experimentId": "claude-sonnet-5-claude-code--guided",
+ "experimentId": "gemini-3-1-pro-preview-opencode--guided",
"caseId": "author-004-packaged-skill",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 115311,
- "measuredAt": "2026-09-02T14:34:31.365Z",
- "meanEstimatedListCostUsd": 0.26473703333333337,
- "meanTokenConsumption": 7318.666666666667,
- "meanToolInvocationCount": 5
+ "meanDurationMs": 141402,
+ "measuredAt": "2026-10-07T21:52:05.927Z",
+ "meanEstimatedListCostUsd": 0.09085193333333332,
+ "meanTokenConsumption": 32130.333333333332,
+ "meanToolInvocationCount": 5.666666666666667
},
{
- "experimentId": "claude-sonnet-5-claude-code--guided",
+ "experimentId": "gemini-3-1-pro-preview-opencode--guided",
"caseId": "author-005-conditional-approval",
"status": "current",
- "passedRuns": 3,
+ "passedRuns": 2,
"validRuns": 3,
- "meanDurationMs": 117023,
- "measuredAt": "2026-09-02T14:34:31.375Z",
- "meanEstimatedListCostUsd": 0.34835236666666664,
- "meanTokenConsumption": 2814,
- "meanToolInvocationCount": 9
+ "meanDurationMs": 183922,
+ "measuredAt": "2026-10-07T21:52:05.937Z",
+ "meanEstimatedListCostUsd": 0.18213406666666668,
+ "meanTokenConsumption": 63521.333333333336,
+ "meanToolInvocationCount": 11.333333333333334
},
{
- "experimentId": "claude-sonnet-5-claude-code--guided",
+ "experimentId": "gemini-3-1-pro-preview-opencode--guided",
"caseId": "author-006-custom-channel",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 110065,
- "measuredAt": "2026-09-02T14:34:31.383Z",
- "meanEstimatedListCostUsd": 0.2831804,
- "meanTokenConsumption": 1897.6666666666667,
- "meanToolInvocationCount": 6
+ "meanDurationMs": 145893,
+ "measuredAt": "2026-10-07T21:52:05.951Z",
+ "meanEstimatedListCostUsd": 0.12927106666666668,
+ "meanTokenConsumption": 43747.333333333336,
+ "meanToolInvocationCount": 8.666666666666666
},
{
- "experimentId": "claude-sonnet-5-claude-code--guided",
+ "experimentId": "gemini-3-1-pro-preview-opencode--guided",
"caseId": "author-007-digest-schedule",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 113169,
- "measuredAt": "2026-09-02T14:34:31.393Z",
- "meanEstimatedListCostUsd": 0.31014963333333334,
- "meanTokenConsumption": 2200,
- "meanToolInvocationCount": 9
+ "meanDurationMs": 145629,
+ "measuredAt": "2026-10-07T21:52:05.964Z",
+ "meanEstimatedListCostUsd": 0.09688193333333334,
+ "meanTokenConsumption": 38146.333333333336,
+ "meanToolInvocationCount": 7
},
{
- "experimentId": "glm-5-3-opencode--baseline",
+ "experimentId": "gemini-3-8-flash-opencode--baseline",
"caseId": "author-001-weather-tool",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 229776,
- "measuredAt": "2026-09-02T14:24:14.610Z",
- "meanEstimatedListCostUsd": 0.13380119999999998,
- "meanTokenConsumption": 39736,
- "meanToolInvocationCount": 27.666666666666668
+ "meanDurationMs": 480431,
+ "measuredAt": "2026-10-07T20:32:06.638Z",
+ "meanEstimatedListCostUsd": 1.2837646249999999,
+ "meanTokenConsumption": 600828.3333333334,
+ "meanToolInvocationCount": 109
},
{
- "experimentId": "glm-5-3-opencode--baseline",
+ "experimentId": "gemini-3-8-flash-opencode--baseline",
"caseId": "author-002-new-project",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 253800,
- "measuredAt": "2026-09-02T14:24:14.640Z",
- "meanEstimatedListCostUsd": 0.04476281333333334,
- "meanTokenConsumption": 19797.666666666668,
- "meanToolInvocationCount": 13.333333333333334
+ "meanDurationMs": 191968,
+ "measuredAt": "2026-10-07T20:32:06.669Z",
+ "meanEstimatedListCostUsd": 0.12271715,
+ "meanTokenConsumption": 117868.33333333333,
+ "meanToolInvocationCount": 23
},
{
- "experimentId": "glm-5-3-opencode--baseline",
+ "experimentId": "gemini-3-8-flash-opencode--baseline",
"caseId": "author-003-openapi-connection",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 275839,
- "measuredAt": "2026-09-02T14:24:14.653Z",
- "meanEstimatedListCostUsd": 0.21305283333333336,
- "meanTokenConsumption": 42271,
- "meanToolInvocationCount": 43.666666666666664
+ "meanDurationMs": 475757,
+ "measuredAt": "2026-10-07T20:32:06.679Z",
+ "meanEstimatedListCostUsd": 0.7462383749999999,
+ "meanTokenConsumption": 458394,
+ "meanToolInvocationCount": 85
},
{
- "experimentId": "glm-5-3-opencode--baseline",
+ "experimentId": "gemini-3-8-flash-opencode--baseline",
"caseId": "author-004-packaged-skill",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 237736,
- "measuredAt": "2026-09-02T14:24:14.669Z",
- "meanEstimatedListCostUsd": 0.06231241333333334,
- "meanTokenConsumption": 18199.666666666668,
- "meanToolInvocationCount": 18.333333333333332
+ "meanDurationMs": 264501,
+ "measuredAt": "2026-10-07T20:32:06.705Z",
+ "meanEstimatedListCostUsd": 0.326150925,
+ "meanTokenConsumption": 212912.33333333334,
+ "meanToolInvocationCount": 42.333333333333336
},
{
- "experimentId": "glm-5-3-opencode--baseline",
+ "experimentId": "gemini-3-8-flash-opencode--baseline",
"caseId": "author-005-conditional-approval",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 267634,
- "measuredAt": "2026-09-02T14:24:14.679Z",
- "meanEstimatedListCostUsd": 0.15534122666666667,
- "meanTokenConsumption": 43595,
- "meanToolInvocationCount": 35.333333333333336
+ "meanDurationMs": 439824,
+ "measuredAt": "2026-10-07T20:32:06.731Z",
+ "meanEstimatedListCostUsd": 0.5367316999999999,
+ "meanTokenConsumption": 309792.6666666667,
+ "meanToolInvocationCount": 65.33333333333333
},
{
- "experimentId": "glm-5-3-opencode--baseline",
+ "experimentId": "gemini-3-8-flash-opencode--baseline",
"caseId": "author-006-custom-channel",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 173067,
- "measuredAt": "2026-09-02T14:24:14.691Z",
- "meanEstimatedListCostUsd": 0.15079565333333333,
- "meanTokenConsumption": 55561.666666666664,
- "meanToolInvocationCount": 29
+ "meanDurationMs": 343512,
+ "measuredAt": "2026-10-07T20:32:06.761Z",
+ "meanEstimatedListCostUsd": 0.6466183249999999,
+ "meanTokenConsumption": 334050.6666666667,
+ "meanToolInvocationCount": 64
},
{
- "experimentId": "glm-5-3-opencode--baseline",
+ "experimentId": "gemini-3-8-flash-opencode--baseline",
"caseId": "author-007-digest-schedule",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 150379,
- "measuredAt": "2026-09-02T14:24:14.703Z",
- "meanEstimatedListCostUsd": 0.061922139999999994,
- "meanTokenConsumption": 17250,
- "meanToolInvocationCount": 23.333333333333332
+ "meanDurationMs": 291390,
+ "measuredAt": "2026-10-07T20:32:06.785Z",
+ "meanEstimatedListCostUsd": 0.38674920000000007,
+ "meanTokenConsumption": 351992.3333333333,
+ "meanToolInvocationCount": 43.333333333333336
},
{
- "experimentId": "glm-5-3-opencode--guided",
+ "experimentId": "gemini-3-8-flash-opencode--guided",
"caseId": "author-001-weather-tool",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 192393,
- "measuredAt": "2026-09-02T14:24:09.125Z",
- "meanEstimatedListCostUsd": 0.05878674,
- "meanTokenConsumption": 23335,
- "meanToolInvocationCount": 10
+ "meanDurationMs": 327592,
+ "measuredAt": "2026-10-07T20:39:20.348Z",
+ "meanEstimatedListCostUsd": 0.51476285,
+ "meanTokenConsumption": 328742,
+ "meanToolInvocationCount": 55.666666666666664
},
{
- "experimentId": "glm-5-3-opencode--guided",
+ "experimentId": "gemini-3-8-flash-opencode--guided",
"caseId": "author-002-new-project",
"status": "current",
- "passedRuns": 3,
+ "passedRuns": 1,
"validRuns": 3,
- "meanDurationMs": 314441,
- "measuredAt": "2026-09-02T14:24:09.147Z",
- "meanEstimatedListCostUsd": 0.04504718666666666,
- "meanTokenConsumption": 18359.333333333332,
- "meanToolInvocationCount": 14
+ "meanDurationMs": 226209,
+ "measuredAt": "2026-10-07T20:39:20.384Z",
+ "meanEstimatedListCostUsd": 0.1381991,
+ "meanTokenConsumption": 117559.66666666667,
+ "meanToolInvocationCount": 24.666666666666668
},
{
- "experimentId": "glm-5-3-opencode--guided",
+ "experimentId": "gemini-3-8-flash-opencode--guided",
"caseId": "author-003-openapi-connection",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 141942,
- "measuredAt": "2026-09-02T14:24:09.165Z",
- "meanEstimatedListCostUsd": 0.062038473333333344,
- "meanTokenConsumption": 22144.666666666668,
- "meanToolInvocationCount": 15
+ "meanDurationMs": 306740,
+ "measuredAt": "2026-10-07T20:39:20.395Z",
+ "meanEstimatedListCostUsd": 0.4550721749999999,
+ "meanTokenConsumption": 308868.6666666667,
+ "meanToolInvocationCount": 57
},
{
- "experimentId": "glm-5-3-opencode--guided",
+ "experimentId": "gemini-3-8-flash-opencode--guided",
"caseId": "author-004-packaged-skill",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 140163,
- "measuredAt": "2026-09-02T14:24:09.179Z",
- "meanEstimatedListCostUsd": 0.04043716,
- "meanTokenConsumption": 14312.666666666666,
- "meanToolInvocationCount": 10.666666666666666
+ "meanDurationMs": 267083,
+ "measuredAt": "2026-10-07T20:39:20.424Z",
+ "meanEstimatedListCostUsd": 0.21935792499999998,
+ "meanTokenConsumption": 151813,
+ "meanToolInvocationCount": 33.666666666666664
},
{
- "experimentId": "glm-5-3-opencode--guided",
+ "experimentId": "gemini-3-8-flash-opencode--guided",
"caseId": "author-005-conditional-approval",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 129477,
- "measuredAt": "2026-09-02T14:24:09.190Z",
- "meanEstimatedListCostUsd": 0.04672530666666667,
- "meanTokenConsumption": 21094.666666666668,
- "meanToolInvocationCount": 8.666666666666666
+ "meanDurationMs": 248208,
+ "measuredAt": "2026-10-07T20:39:20.449Z",
+ "meanEstimatedListCostUsd": 0.30564869999999994,
+ "meanTokenConsumption": 204861.33333333334,
+ "meanToolInvocationCount": 42
},
{
- "experimentId": "glm-5-3-opencode--guided",
+ "experimentId": "gemini-3-8-flash-opencode--guided",
"caseId": "author-006-custom-channel",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 117801,
- "measuredAt": "2026-09-02T14:24:09.200Z",
- "meanEstimatedListCostUsd": 0.036290826666666665,
- "meanTokenConsumption": 13584.333333333334,
- "meanToolInvocationCount": 10.666666666666666
+ "meanDurationMs": 178534,
+ "measuredAt": "2026-10-07T20:39:20.489Z",
+ "meanEstimatedListCostUsd": 0.18658105,
+ "meanTokenConsumption": 156321,
+ "meanToolInvocationCount": 29
},
{
- "experimentId": "glm-5-3-opencode--guided",
+ "experimentId": "gemini-3-8-flash-opencode--guided",
"caseId": "author-007-digest-schedule",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 117068,
- "measuredAt": "2026-09-02T14:24:09.209Z",
- "meanEstimatedListCostUsd": 0.024231453333333333,
- "meanTokenConsumption": 7749.333333333333,
- "meanToolInvocationCount": 8.333333333333334
+ "meanDurationMs": 190950,
+ "measuredAt": "2026-10-07T20:39:20.518Z",
+ "meanEstimatedListCostUsd": 0.17541602499999998,
+ "meanTokenConsumption": 133149,
+ "meanToolInvocationCount": 27.666666666666668
},
{
- "experimentId": "claude-opus-5-claude-code--baseline",
+ "experimentId": "minimax-m3-opencode--baseline",
"caseId": "author-001-weather-tool",
"status": "current",
- "passedRuns": 3,
+ "passedRuns": 2,
"validRuns": 3,
- "meanDurationMs": 155202,
- "measuredAt": "2026-09-02T14:35:36.609Z",
- "meanEstimatedListCostUsd": 1.1599598333333334,
- "meanTokenConsumption": 10714.666666666666,
- "meanToolInvocationCount": 17.333333333333332
+ "meanDurationMs": 137827,
+ "measuredAt": "2026-10-07T19:57:38.058Z",
+ "meanEstimatedListCostUsd": 0.03919382,
+ "meanTokenConsumption": 40222,
+ "meanToolInvocationCount": 29.333333333333332
},
{
- "experimentId": "claude-opus-5-claude-code--baseline",
+ "experimentId": "minimax-m3-opencode--baseline",
"caseId": "author-002-new-project",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 124976,
- "measuredAt": "2026-09-02T14:35:36.628Z",
- "meanEstimatedListCostUsd": 0.8593296666666667,
- "meanTokenConsumption": 3845.3333333333335,
- "meanToolInvocationCount": 10.333333333333334
+ "meanDurationMs": 142346,
+ "measuredAt": "2026-10-07T19:57:38.073Z",
+ "meanEstimatedListCostUsd": 0.02446828,
+ "meanTokenConsumption": 24358,
+ "meanToolInvocationCount": 17.333333333333332
},
{
- "experimentId": "claude-opus-5-claude-code--baseline",
+ "experimentId": "minimax-m3-opencode--baseline",
"caseId": "author-003-openapi-connection",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 160123,
- "measuredAt": "2026-09-02T14:35:36.643Z",
- "meanEstimatedListCostUsd": 1.3671014166666666,
- "meanTokenConsumption": 7793.333333333333,
- "meanToolInvocationCount": 19
+ "meanDurationMs": 145218,
+ "measuredAt": "2026-10-07T19:57:38.085Z",
+ "meanEstimatedListCostUsd": 0.04590585999999999,
+ "meanTokenConsumption": 53114.666666666664,
+ "meanToolInvocationCount": 27.666666666666668
},
{
- "experimentId": "claude-opus-5-claude-code--baseline",
+ "experimentId": "minimax-m3-opencode--baseline",
"caseId": "author-004-packaged-skill",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 181217,
- "measuredAt": "2026-09-02T14:35:36.656Z",
- "meanEstimatedListCostUsd": 1.1116483333333333,
- "meanTokenConsumption": 16371.666666666666,
- "meanToolInvocationCount": 12.666666666666666
+ "meanDurationMs": 154028,
+ "measuredAt": "2026-10-07T19:57:38.099Z",
+ "meanEstimatedListCostUsd": 0.03372036,
+ "meanTokenConsumption": 35014.333333333336,
+ "meanToolInvocationCount": 23
},
{
- "experimentId": "claude-opus-5-claude-code--baseline",
+ "experimentId": "minimax-m3-opencode--baseline",
"caseId": "author-005-conditional-approval",
"status": "current",
- "passedRuns": 3,
+ "passedRuns": 2,
"validRuns": 3,
- "meanDurationMs": 153333,
- "measuredAt": "2026-09-02T14:35:36.667Z",
- "meanEstimatedListCostUsd": 1.03157675,
- "meanTokenConsumption": 6217.333333333333,
- "meanToolInvocationCount": 17.333333333333332
+ "meanDurationMs": 159346,
+ "measuredAt": "2026-10-07T19:57:38.112Z",
+ "meanEstimatedListCostUsd": 0.037709999999999994,
+ "meanTokenConsumption": 38013.333333333336,
+ "meanToolInvocationCount": 27.666666666666668
},
{
- "experimentId": "claude-opus-5-claude-code--baseline",
+ "experimentId": "minimax-m3-opencode--baseline",
"caseId": "author-006-custom-channel",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 136155,
- "measuredAt": "2026-09-02T14:35:36.676Z",
- "meanEstimatedListCostUsd": 0.9316169166666667,
- "meanTokenConsumption": 4836.666666666667,
- "meanToolInvocationCount": 16.666666666666668
+ "meanDurationMs": 196082,
+ "measuredAt": "2026-10-07T19:57:38.126Z",
+ "meanEstimatedListCostUsd": 0.09671368000000001,
+ "meanTokenConsumption": 77110,
+ "meanToolInvocationCount": 47
},
{
- "experimentId": "claude-opus-5-claude-code--baseline",
+ "experimentId": "minimax-m3-opencode--baseline",
"caseId": "author-007-digest-schedule",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 120633,
- "measuredAt": "2026-09-02T14:35:36.685Z",
- "meanEstimatedListCostUsd": 0.5067333333333334,
- "meanTokenConsumption": 3232.6666666666665,
- "meanToolInvocationCount": 9.333333333333334
+ "meanDurationMs": 139910,
+ "measuredAt": "2026-10-07T19:57:38.143Z",
+ "meanEstimatedListCostUsd": 0.017940519999999998,
+ "meanTokenConsumption": 18308.333333333332,
+ "meanToolInvocationCount": 19
},
{
- "experimentId": "claude-opus-5-claude-code--guided",
+ "experimentId": "minimax-m3-opencode--guided",
"caseId": "author-001-weather-tool",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 147978,
- "measuredAt": "2026-09-02T14:34:53.900Z",
- "meanEstimatedListCostUsd": 1.0020195,
- "meanTokenConsumption": 11004.333333333334,
- "meanToolInvocationCount": 11
+ "meanDurationMs": 120197,
+ "measuredAt": "2026-10-07T20:00:50.683Z",
+ "meanEstimatedListCostUsd": 0.018265399999999998,
+ "meanTokenConsumption": 29428.666666666668,
+ "meanToolInvocationCount": 13.666666666666666
},
{
- "experimentId": "claude-opus-5-claude-code--guided",
+ "experimentId": "minimax-m3-opencode--guided",
"caseId": "author-002-new-project",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 138714,
- "measuredAt": "2026-09-02T14:34:53.929Z",
- "meanEstimatedListCostUsd": 1.0472271666666666,
- "meanTokenConsumption": 8633.666666666666,
- "meanToolInvocationCount": 14.333333333333334
+ "meanDurationMs": 122122,
+ "measuredAt": "2026-10-07T20:00:50.696Z",
+ "meanEstimatedListCostUsd": 0.01290936,
+ "meanTokenConsumption": 14803,
+ "meanToolInvocationCount": 12
},
{
- "experimentId": "claude-opus-5-claude-code--guided",
+ "experimentId": "minimax-m3-opencode--guided",
"caseId": "author-003-openapi-connection",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 137520,
- "measuredAt": "2026-09-02T14:34:53.944Z",
- "meanEstimatedListCostUsd": 0.7869943333333334,
- "meanTokenConsumption": 5193.666666666667,
- "meanToolInvocationCount": 14
+ "meanDurationMs": 127857,
+ "measuredAt": "2026-10-07T20:00:50.706Z",
+ "meanEstimatedListCostUsd": 0.024050899999999997,
+ "meanTokenConsumption": 32894,
+ "meanToolInvocationCount": 15
},
{
- "experimentId": "claude-opus-5-claude-code--guided",
+ "experimentId": "minimax-m3-opencode--guided",
"caseId": "author-004-packaged-skill",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 153316,
- "measuredAt": "2026-09-02T14:34:53.956Z",
- "meanEstimatedListCostUsd": 0.6677339166666667,
- "meanTokenConsumption": 10291.666666666666,
- "meanToolInvocationCount": 7.666666666666667
+ "meanDurationMs": 148211,
+ "measuredAt": "2026-10-07T20:00:50.716Z",
+ "meanEstimatedListCostUsd": 0.010811939999999999,
+ "meanTokenConsumption": 9126.333333333334,
+ "meanToolInvocationCount": 10
},
{
- "experimentId": "claude-opus-5-claude-code--guided",
+ "experimentId": "minimax-m3-opencode--guided",
"caseId": "author-005-conditional-approval",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 116393,
- "measuredAt": "2026-09-02T14:34:53.964Z",
- "meanEstimatedListCostUsd": 0.5910709166666667,
- "meanTokenConsumption": 2916.6666666666665,
- "meanToolInvocationCount": 8.333333333333334
+ "meanDurationMs": 148887,
+ "measuredAt": "2026-10-07T20:00:50.725Z",
+ "meanEstimatedListCostUsd": 0.0288123,
+ "meanTokenConsumption": 39894,
+ "meanToolInvocationCount": 19
},
{
- "experimentId": "claude-opus-5-claude-code--guided",
+ "experimentId": "minimax-m3-opencode--guided",
"caseId": "author-006-custom-channel",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 107570,
- "measuredAt": "2026-09-02T14:34:53.973Z",
- "meanEstimatedListCostUsd": 0.4942074166666666,
- "meanTokenConsumption": 2636,
- "meanToolInvocationCount": 6.666666666666667
+ "meanDurationMs": 119587,
+ "measuredAt": "2026-10-07T20:00:50.736Z",
+ "meanEstimatedListCostUsd": 0.01628684,
+ "meanTokenConsumption": 22547.333333333332,
+ "meanToolInvocationCount": 9.666666666666666
},
{
- "experimentId": "claude-opus-5-claude-code--guided",
+ "experimentId": "minimax-m3-opencode--guided",
"caseId": "author-007-digest-schedule",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 114311,
- "measuredAt": "2026-09-02T14:34:53.981Z",
- "meanEstimatedListCostUsd": 0.46051358333333337,
- "meanTokenConsumption": 2592.6666666666665,
- "meanToolInvocationCount": 7.666666666666667
+ "meanDurationMs": 116089,
+ "measuredAt": "2026-10-07T20:00:50.745Z",
+ "meanEstimatedListCostUsd": 0.01088158,
+ "meanTokenConsumption": 13971.666666666666,
+ "meanToolInvocationCount": 10
},
{
- "experimentId": "gemini-3-1-pro-preview-opencode--baseline",
+ "experimentId": "kimi-k2-7-code-opencode--baseline",
"caseId": "author-001-weather-tool",
"status": "current",
- "passedRuns": 0,
+ "passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 54876,
- "measuredAt": "2026-08-28T19:20:40.000Z",
- "meanEstimatedListCostUsd": 0.11818266666666667,
- "meanTokenConsumption": 45096.333333333336,
- "meanToolInvocationCount": 9.666666666666666
+ "meanDurationMs": 217800,
+ "measuredAt": "2026-10-07T19:45:29.923Z",
+ "meanEstimatedListCostUsd": 0.14899567666666666,
+ "meanTokenConsumption": 34828.666666666664,
+ "meanToolInvocationCount": 32
},
{
- "experimentId": "gemini-3-1-pro-preview-opencode--baseline",
+ "experimentId": "kimi-k2-7-code-opencode--baseline",
"caseId": "author-002-new-project",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 82415,
- "measuredAt": "2026-08-28T19:26:24.000Z",
- "meanEstimatedListCostUsd": 0.11070399999999998,
- "meanTokenConsumption": 51112,
- "meanToolInvocationCount": 11.666666666666666
+ "meanDurationMs": 174725,
+ "measuredAt": "2026-10-07T19:45:29.947Z",
+ "meanEstimatedListCostUsd": 0.06421423000000001,
+ "meanTokenConsumption": 19656.666666666668,
+ "meanToolInvocationCount": 22.666666666666668
},
{
- "experimentId": "gemini-3-1-pro-preview-opencode--baseline",
+ "experimentId": "kimi-k2-7-code-opencode--baseline",
"caseId": "author-003-openapi-connection",
"status": "current",
- "passedRuns": 1,
+ "passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 102903,
- "measuredAt": "2026-08-28T19:26:24.000Z",
- "meanEstimatedListCostUsd": 0.43870533333333334,
- "meanTokenConsumption": 211094.33333333334,
- "meanToolInvocationCount": 28
+ "meanDurationMs": 172017,
+ "measuredAt": "2026-10-07T19:45:29.963Z",
+ "meanEstimatedListCostUsd": 0.12660882333333334,
+ "meanTokenConsumption": 35982.666666666664,
+ "meanToolInvocationCount": 25.333333333333332
},
{
- "experimentId": "gemini-3-1-pro-preview-opencode--baseline",
+ "experimentId": "kimi-k2-7-code-opencode--baseline",
"caseId": "author-004-packaged-skill",
"status": "current",
- "passedRuns": 2,
+ "passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 33942,
- "measuredAt": "2026-08-28T19:20:40.000Z",
- "meanEstimatedListCostUsd": 0.07116533333333334,
- "meanTokenConsumption": 26049.333333333332,
- "meanToolInvocationCount": 5.333333333333333
+ "meanDurationMs": 176932,
+ "measuredAt": "2026-10-07T19:45:29.978Z",
+ "meanEstimatedListCostUsd": 0.09377714666666666,
+ "meanTokenConsumption": 27106.333333333332,
+ "meanToolInvocationCount": 25.666666666666668
},
{
- "experimentId": "gemini-3-1-pro-preview-opencode--baseline",
+ "experimentId": "kimi-k2-7-code-opencode--baseline",
"caseId": "author-005-conditional-approval",
"status": "current",
"passedRuns": 2,
"validRuns": 3,
- "meanDurationMs": 227315,
- "measuredAt": "2026-08-28T19:20:40.000Z",
- "meanEstimatedListCostUsd": 0.5367073333333333,
- "meanTokenConsumption": 251920.33333333334,
- "meanToolInvocationCount": 42.666666666666664
+ "meanDurationMs": 189532,
+ "measuredAt": "2026-10-07T19:45:29.991Z",
+ "meanEstimatedListCostUsd": 0.13552957000000002,
+ "meanTokenConsumption": 37226,
+ "meanToolInvocationCount": 25.333333333333332
},
{
- "experimentId": "gemini-3-1-pro-preview-opencode--baseline",
+ "experimentId": "kimi-k2-7-code-opencode--baseline",
"caseId": "author-006-custom-channel",
"status": "current",
- "passedRuns": 1,
+ "passedRuns": 2,
"validRuns": 3,
- "meanDurationMs": 133303,
- "measuredAt": "2026-08-28T19:20:40.000Z",
- "meanEstimatedListCostUsd": 0.467548,
- "meanTokenConsumption": 225615.66666666666,
- "meanToolInvocationCount": 30.666666666666668
+ "meanDurationMs": 248066,
+ "measuredAt": "2026-10-07T19:45:30.008Z",
+ "meanEstimatedListCostUsd": 0.13744147333333334,
+ "meanTokenConsumption": 38285,
+ "meanToolInvocationCount": 28.666666666666668
},
{
- "experimentId": "gemini-3-1-pro-preview-opencode--baseline",
+ "experimentId": "kimi-k2-7-code-opencode--baseline",
"caseId": "author-007-digest-schedule",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 61813,
- "measuredAt": "2026-08-28T19:20:40.000Z",
- "meanEstimatedListCostUsd": 0.212552,
- "meanTokenConsumption": 101059.33333333333,
- "meanToolInvocationCount": 15
+ "meanDurationMs": 187907,
+ "measuredAt": "2026-10-07T19:45:30.021Z",
+ "meanEstimatedListCostUsd": 0.06076566333333333,
+ "meanTokenConsumption": 19027.666666666668,
+ "meanToolInvocationCount": 25.666666666666668
},
{
- "experimentId": "gemini-3-1-pro-preview-opencode--guided",
+ "experimentId": "kimi-k2-7-code-opencode--guided",
"caseId": "author-001-weather-tool",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 37112,
- "measuredAt": "2026-08-28T19:17:42.000Z",
- "meanEstimatedListCostUsd": 0.08827333333333333,
- "meanTokenConsumption": 40508.333333333336,
- "meanToolInvocationCount": 7.666666666666667
+ "meanDurationMs": 181961,
+ "measuredAt": "2026-10-07T19:49:07.568Z",
+ "meanEstimatedListCostUsd": 0.08877174666666666,
+ "meanTokenConsumption": 26044.333333333332,
+ "meanToolInvocationCount": 20
},
{
- "experimentId": "gemini-3-1-pro-preview-opencode--guided",
+ "experimentId": "kimi-k2-7-code-opencode--guided",
"caseId": "author-002-new-project",
"status": "current",
- "passedRuns": 0,
+ "passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 260182,
- "measuredAt": "2026-08-28T19:17:42.000Z",
- "meanEstimatedListCostUsd": 0.237908,
- "meanTokenConsumption": 109484,
- "meanToolInvocationCount": 28.333333333333332
+ "meanDurationMs": 139537,
+ "measuredAt": "2026-10-07T19:49:07.579Z",
+ "meanEstimatedListCostUsd": 0.04228683333333333,
+ "meanTokenConsumption": 14718,
+ "meanToolInvocationCount": 17.333333333333332
},
{
- "experimentId": "gemini-3-1-pro-preview-opencode--guided",
+ "experimentId": "kimi-k2-7-code-opencode--guided",
"caseId": "author-003-openapi-connection",
"status": "current",
- "passedRuns": 0,
+ "passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 201322,
- "measuredAt": "2026-08-28T19:17:42.000Z",
- "meanEstimatedListCostUsd": 0.3107266666666667,
- "meanTokenConsumption": 132131.66666666666,
- "meanToolInvocationCount": 34.333333333333336
+ "meanDurationMs": 185254,
+ "measuredAt": "2026-10-07T19:49:07.588Z",
+ "meanEstimatedListCostUsd": 0.04005971333333333,
+ "meanTokenConsumption": 14683.333333333334,
+ "meanToolInvocationCount": 12.666666666666666
},
{
- "experimentId": "gemini-3-1-pro-preview-opencode--guided",
+ "experimentId": "kimi-k2-7-code-opencode--guided",
"caseId": "author-004-packaged-skill",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 41440,
- "measuredAt": "2026-08-28T19:17:42.000Z",
- "meanEstimatedListCostUsd": 0.08369866666666666,
- "meanTokenConsumption": 34357.666666666664,
- "meanToolInvocationCount": 8.666666666666666
+ "meanDurationMs": 153696,
+ "measuredAt": "2026-10-07T19:49:07.597Z",
+ "meanEstimatedListCostUsd": 0.03293041666666666,
+ "meanTokenConsumption": 10946,
+ "meanToolInvocationCount": 11
},
{
- "experimentId": "gemini-3-1-pro-preview-opencode--guided",
+ "experimentId": "kimi-k2-7-code-opencode--guided",
"caseId": "author-005-conditional-approval",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 33301,
- "measuredAt": "2026-08-28T19:17:42.000Z",
- "meanEstimatedListCostUsd": 0.09258866666666667,
- "meanTokenConsumption": 42437.666666666664,
- "meanToolInvocationCount": 7.333333333333333
+ "meanDurationMs": 159597,
+ "measuredAt": "2026-10-07T19:49:07.606Z",
+ "meanEstimatedListCostUsd": 0.059909359999999995,
+ "meanTokenConsumption": 21384.666666666668,
+ "meanToolInvocationCount": 15.333333333333334
},
{
- "experimentId": "gemini-3-1-pro-preview-opencode--guided",
+ "experimentId": "kimi-k2-7-code-opencode--guided",
"caseId": "author-006-custom-channel",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 34493,
- "measuredAt": "2026-08-28T19:17:42.000Z",
- "meanEstimatedListCostUsd": 0.09662666666666668,
- "meanTokenConsumption": 45305,
- "meanToolInvocationCount": 7.333333333333333
+ "meanDurationMs": 142256,
+ "measuredAt": "2026-10-07T19:49:07.619Z",
+ "meanEstimatedListCostUsd": 0.040052736666666665,
+ "meanTokenConsumption": 15506,
+ "meanToolInvocationCount": 12.333333333333334
},
{
- "experimentId": "gemini-3-1-pro-preview-opencode--guided",
+ "experimentId": "kimi-k2-7-code-opencode--guided",
"caseId": "author-007-digest-schedule",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 33257,
- "measuredAt": "2026-08-28T19:17:42.000Z",
- "meanEstimatedListCostUsd": 0.059873333333333334,
- "meanTokenConsumption": 27681.666666666668,
- "meanToolInvocationCount": 4.666666666666667
+ "meanDurationMs": 139483,
+ "measuredAt": "2026-10-07T19:49:07.629Z",
+ "meanEstimatedListCostUsd": 0.04300942,
+ "meanTokenConsumption": 13041.333333333334,
+ "meanToolInvocationCount": 12.333333333333334
},
{
- "experimentId": "gemini-3-8-flash-opencode--baseline",
+ "experimentId": "claude-opus-5-5-high-claude-code--baseline",
"caseId": "author-001-weather-tool",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 368613,
- "measuredAt": "2026-09-15T16:50:43.236Z",
- "meanEstimatedListCostUsd": 0.398698,
- "meanTokenConsumption": 256875.66666666666,
- "meanToolInvocationCount": 53.333333333333336
+ "meanDurationMs": 148854,
+ "measuredAt": "2026-10-07T18:15:47.177Z",
+ "meanEstimatedListCostUsd": 0.5982536666666667,
+ "meanTokenConsumption": 8172.666666666667,
+ "meanToolInvocationCount": 11.666666666666666
},
{
- "experimentId": "gemini-3-8-flash-opencode--baseline",
+ "experimentId": "claude-opus-5-5-high-claude-code--baseline",
"caseId": "author-002-new-project",
"status": "current",
- "passedRuns": 2,
+ "passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 181428,
- "measuredAt": "2026-09-15T16:50:43.274Z",
- "meanEstimatedListCostUsd": 0.07038494999999999,
- "meanTokenConsumption": 75876.33333333333,
- "meanToolInvocationCount": 15.666666666666666
+ "meanDurationMs": 131272,
+ "measuredAt": "2026-10-07T18:15:47.196Z",
+ "meanEstimatedListCostUsd": 0.22304533333333335,
+ "meanTokenConsumption": 2829.3333333333335,
+ "meanToolInvocationCount": 8
},
{
- "experimentId": "gemini-3-8-flash-opencode--baseline",
+ "experimentId": "claude-opus-5-5-high-claude-code--baseline",
"caseId": "author-003-openapi-connection",
"status": "current",
- "passedRuns": 1,
+ "passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 730583,
- "measuredAt": "2026-09-15T16:50:43.288Z",
- "meanEstimatedListCostUsd": 0.35561909999999997,
- "meanTokenConsumption": 235963,
- "meanToolInvocationCount": 56
+ "meanDurationMs": 169954,
+ "measuredAt": "2026-10-07T18:15:47.207Z",
+ "meanEstimatedListCostUsd": 0.33688120000000005,
+ "meanTokenConsumption": 4193.333333333333,
+ "meanToolInvocationCount": 11.666666666666666
},
{
- "experimentId": "gemini-3-8-flash-opencode--baseline",
+ "experimentId": "claude-opus-5-5-high-claude-code--baseline",
"caseId": "author-004-packaged-skill",
"status": "current",
- "passedRuns": 2,
+ "passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 273870,
- "measuredAt": "2026-09-15T16:50:43.297Z",
- "meanEstimatedListCostUsd": 0.252016075,
- "meanTokenConsumption": 189061,
- "meanToolInvocationCount": 40.333333333333336
+ "meanDurationMs": 195780,
+ "measuredAt": "2026-10-07T18:15:47.217Z",
+ "meanEstimatedListCostUsd": 0.44434179999999995,
+ "meanTokenConsumption": 9915,
+ "meanToolInvocationCount": 10.666666666666666
},
{
- "experimentId": "gemini-3-8-flash-opencode--baseline",
+ "experimentId": "claude-opus-5-5-high-claude-code--baseline",
"caseId": "author-005-conditional-approval",
"status": "current",
- "passedRuns": 2,
+ "passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 208514,
- "measuredAt": "2026-09-15T16:50:43.312Z",
- "meanEstimatedListCostUsd": 0.3936613499999999,
- "meanTokenConsumption": 330781.3333333333,
- "meanToolInvocationCount": 45
+ "meanDurationMs": 158936,
+ "measuredAt": "2026-10-07T18:15:47.228Z",
+ "meanEstimatedListCostUsd": 0.5887976,
+ "meanTokenConsumption": 7503.666666666667,
+ "meanToolInvocationCount": 12.333333333333334
},
{
- "experimentId": "gemini-3-8-flash-opencode--baseline",
+ "experimentId": "claude-opus-5-5-high-claude-code--baseline",
"caseId": "author-006-custom-channel",
"status": "current",
- "passedRuns": 2,
+ "passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 463841,
- "measuredAt": "2026-09-15T16:50:43.333Z",
- "meanEstimatedListCostUsd": 0.349989725,
- "meanTokenConsumption": 239936.33333333334,
- "meanToolInvocationCount": 41.333333333333336
+ "meanDurationMs": 137260,
+ "measuredAt": "2026-10-07T18:15:47.239Z",
+ "meanEstimatedListCostUsd": 0.3630187333333333,
+ "meanTokenConsumption": 3004,
+ "meanToolInvocationCount": 7.666666666666667
},
{
- "experimentId": "gemini-3-8-flash-opencode--baseline",
+ "experimentId": "claude-opus-5-5-high-claude-code--baseline",
"caseId": "author-007-digest-schedule",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 307880,
- "measuredAt": "2026-09-15T16:50:43.349Z",
- "meanEstimatedListCostUsd": 0.8133175250000001,
- "meanTokenConsumption": 852160.3333333334,
- "meanToolInvocationCount": 54.666666666666664
+ "meanDurationMs": 135953,
+ "measuredAt": "2026-10-07T18:15:47.248Z",
+ "meanEstimatedListCostUsd": 0.24009426666666667,
+ "meanTokenConsumption": 2781.6666666666665,
+ "meanToolInvocationCount": 7.666666666666667
},
{
- "experimentId": "gemini-3-8-flash-opencode--guided",
+ "experimentId": "claude-opus-5-5-high-claude-code--guided",
"caseId": "author-001-weather-tool",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 226490,
- "measuredAt": "2026-09-15T16:48:47.804Z",
- "meanEstimatedListCostUsd": 0.216276975,
- "meanTokenConsumption": 176030.66666666666,
- "meanToolInvocationCount": 35.333333333333336
+ "meanDurationMs": 152041,
+ "measuredAt": "2026-10-07T18:19:58.247Z",
+ "meanEstimatedListCostUsd": 0.37882766666666673,
+ "meanTokenConsumption": 6229.333333333333,
+ "meanToolInvocationCount": 6.666666666666667
},
{
- "experimentId": "gemini-3-8-flash-opencode--guided",
+ "experimentId": "claude-opus-5-5-high-claude-code--guided",
"caseId": "author-002-new-project",
"status": "current",
- "passedRuns": 1,
+ "passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 212565,
- "measuredAt": "2026-09-15T16:48:47.822Z",
- "meanEstimatedListCostUsd": 0.056899675000000004,
- "meanTokenConsumption": 58487.333333333336,
- "meanToolInvocationCount": 14.333333333333334
+ "meanDurationMs": 179646,
+ "measuredAt": "2026-10-07T18:19:58.259Z",
+ "meanEstimatedListCostUsd": 0.2818064,
+ "meanTokenConsumption": 3093.3333333333335,
+ "meanToolInvocationCount": 9
},
{
- "experimentId": "gemini-3-8-flash-opencode--guided",
+ "experimentId": "claude-opus-5-5-high-claude-code--guided",
"caseId": "author-003-openapi-connection",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 292952,
- "measuredAt": "2026-09-15T16:48:47.831Z",
- "meanEstimatedListCostUsd": 0.3152235,
- "meanTokenConsumption": 263335.3333333333,
- "meanToolInvocationCount": 40.333333333333336
+ "meanDurationMs": 132152,
+ "measuredAt": "2026-10-07T18:19:58.271Z",
+ "meanEstimatedListCostUsd": 0.26926126666666667,
+ "meanTokenConsumption": 2762.3333333333335,
+ "meanToolInvocationCount": 5.666666666666667
},
{
- "experimentId": "gemini-3-8-flash-opencode--guided",
+ "experimentId": "claude-opus-5-5-high-claude-code--guided",
"caseId": "author-004-packaged-skill",
"status": "current",
- "passedRuns": 2,
+ "passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 129278,
- "measuredAt": "2026-09-15T16:48:47.847Z",
- "meanEstimatedListCostUsd": 0.050644475,
- "meanTokenConsumption": 56305.666666666664,
- "meanToolInvocationCount": 9.333333333333334
+ "meanDurationMs": 168608,
+ "measuredAt": "2026-10-07T18:19:58.280Z",
+ "meanEstimatedListCostUsd": 0.2669548,
+ "meanTokenConsumption": 5441,
+ "meanToolInvocationCount": 8
},
{
- "experimentId": "gemini-3-8-flash-opencode--guided",
+ "experimentId": "claude-opus-5-5-high-claude-code--guided",
"caseId": "author-005-conditional-approval",
"status": "current",
- "passedRuns": 1,
+ "passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 648386,
- "measuredAt": "2026-09-15T16:48:47.859Z",
- "meanEstimatedListCostUsd": 0.612731225,
- "meanTokenConsumption": 449588,
- "meanToolInvocationCount": 57.333333333333336
+ "meanDurationMs": 152304,
+ "measuredAt": "2026-10-07T18:19:58.292Z",
+ "meanEstimatedListCostUsd": 0.2563342,
+ "meanTokenConsumption": 2932.3333333333335,
+ "meanToolInvocationCount": 7
},
{
- "experimentId": "gemini-3-8-flash-opencode--guided",
+ "experimentId": "claude-opus-5-5-high-claude-code--guided",
"caseId": "author-006-custom-channel",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 225166,
- "measuredAt": "2026-09-15T16:48:47.880Z",
- "meanEstimatedListCostUsd": 0.265912825,
- "meanTokenConsumption": 260007.33333333334,
- "meanToolInvocationCount": 29
+ "meanDurationMs": 142903,
+ "measuredAt": "2026-10-07T18:19:58.304Z",
+ "meanEstimatedListCostUsd": 0.20084280000000002,
+ "meanTokenConsumption": 2039,
+ "meanToolInvocationCount": 5
},
{
- "experimentId": "gemini-3-8-flash-opencode--guided",
+ "experimentId": "claude-opus-5-5-high-claude-code--guided",
"caseId": "author-007-digest-schedule",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 133720,
- "measuredAt": "2026-09-15T16:48:47.896Z",
- "meanEstimatedListCostUsd": 0.06414665000000001,
- "meanTokenConsumption": 59535.666666666664,
- "meanToolInvocationCount": 12.666666666666666
+ "meanDurationMs": 132872,
+ "measuredAt": "2026-10-07T18:19:58.316Z",
+ "meanEstimatedListCostUsd": 0.14850560000000002,
+ "meanTokenConsumption": 1478,
+ "meanToolInvocationCount": 5.666666666666667
},
{
- "experimentId": "minimax-m3-opencode--baseline",
+ "experimentId": "gpt-6-sol-high-codex--baseline",
"caseId": "author-001-weather-tool",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 139687,
- "measuredAt": "2026-09-15T16:42:26.677Z",
- "meanEstimatedListCostUsd": 0.028696399999999997,
- "meanTokenConsumption": 36094,
- "meanToolInvocationCount": 23.333333333333332
+ "meanDurationMs": 235603,
+ "measuredAt": "2026-10-07T18:49:55.511Z",
+ "meanEstimatedListCostUsd": 0.40047706666666666,
+ "meanTokenConsumption": 56350,
+ "meanToolInvocationCount": 25.666666666666668
},
{
- "experimentId": "minimax-m3-opencode--baseline",
+ "experimentId": "gpt-6-sol-high-codex--baseline",
"caseId": "author-002-new-project",
"status": "current",
- "passedRuns": 1,
+ "passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 336656,
- "measuredAt": "2026-09-15T16:42:26.709Z",
- "meanEstimatedListCostUsd": 0.015437459999999998,
- "meanTokenConsumption": 19649,
- "meanToolInvocationCount": 15
+ "meanDurationMs": 155467,
+ "measuredAt": "2026-10-07T18:49:55.529Z",
+ "meanEstimatedListCostUsd": 0.22136066666666665,
+ "meanTokenConsumption": 31200.666666666668,
+ "meanToolInvocationCount": 19.666666666666668
},
{
- "experimentId": "minimax-m3-opencode--baseline",
+ "experimentId": "gpt-6-sol-high-codex--baseline",
"caseId": "author-003-openapi-connection",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 149683,
- "measuredAt": "2026-09-15T16:42:26.715Z",
- "meanEstimatedListCostUsd": 0.0360468,
- "meanTokenConsumption": 44776.666666666664,
- "meanToolInvocationCount": 26.333333333333332
+ "meanDurationMs": 140201,
+ "measuredAt": "2026-10-07T18:49:55.541Z",
+ "meanEstimatedListCostUsd": 0.24634376666666669,
+ "meanTokenConsumption": 42107.333333333336,
+ "meanToolInvocationCount": 18
},
{
- "experimentId": "minimax-m3-opencode--baseline",
+ "experimentId": "gpt-6-sol-high-codex--baseline",
"caseId": "author-004-packaged-skill",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 169255,
- "measuredAt": "2026-09-15T16:42:26.734Z",
- "meanEstimatedListCostUsd": 0.022754239999999995,
- "meanTokenConsumption": 24665.333333333332,
- "meanToolInvocationCount": 17.333333333333332
+ "meanDurationMs": 147782,
+ "measuredAt": "2026-10-07T18:49:55.561Z",
+ "meanEstimatedListCostUsd": 0.23484953333333333,
+ "meanTokenConsumption": 37910,
+ "meanToolInvocationCount": 18.666666666666668
},
{
- "experimentId": "minimax-m3-opencode--baseline",
+ "experimentId": "gpt-6-sol-high-codex--baseline",
"caseId": "author-005-conditional-approval",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 152482,
- "measuredAt": "2026-09-15T16:42:26.748Z",
- "meanEstimatedListCostUsd": 0.037158,
- "meanTokenConsumption": 47089.666666666664,
- "meanToolInvocationCount": 24.666666666666668
+ "meanDurationMs": 184292,
+ "measuredAt": "2026-10-07T18:49:55.570Z",
+ "meanEstimatedListCostUsd": 0.3206735333333333,
+ "meanTokenConsumption": 49776,
+ "meanToolInvocationCount": 22.333333333333332
},
{
- "experimentId": "minimax-m3-opencode--baseline",
+ "experimentId": "gpt-6-sol-high-codex--baseline",
"caseId": "author-006-custom-channel",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 175652,
- "measuredAt": "2026-09-15T16:42:26.764Z",
- "meanEstimatedListCostUsd": 0.10778759999999998,
- "meanTokenConsumption": 91708.33333333333,
- "meanToolInvocationCount": 45.333333333333336
+ "meanDurationMs": 141113,
+ "measuredAt": "2026-10-07T18:49:55.583Z",
+ "meanEstimatedListCostUsd": 0.24225123333333332,
+ "meanTokenConsumption": 41215.666666666664,
+ "meanToolInvocationCount": 17.333333333333332
},
{
- "experimentId": "minimax-m3-opencode--baseline",
+ "experimentId": "gpt-6-sol-high-codex--baseline",
"caseId": "author-007-digest-schedule",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 142420,
- "measuredAt": "2026-09-15T16:42:26.784Z",
- "meanEstimatedListCostUsd": 0.018700320000000003,
- "meanTokenConsumption": 22837.333333333332,
- "meanToolInvocationCount": 17.333333333333332
+ "meanDurationMs": 143502,
+ "measuredAt": "2026-10-07T18:49:55.595Z",
+ "meanEstimatedListCostUsd": 0.1879093666666667,
+ "meanTokenConsumption": 32017.666666666668,
+ "meanToolInvocationCount": 13.666666666666666
},
{
- "experimentId": "minimax-m3-opencode--guided",
+ "experimentId": "gpt-6-sol-high-codex--guided",
"caseId": "author-001-weather-tool",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 133642,
- "measuredAt": "2026-09-15T16:41:09.940Z",
- "meanEstimatedListCostUsd": 0.011224940000000001,
- "meanTokenConsumption": 19059.333333333332,
- "meanToolInvocationCount": 9.666666666666666
+ "meanDurationMs": 162399,
+ "measuredAt": "2026-10-07T18:53:33.041Z",
+ "meanEstimatedListCostUsd": 0.22911669999999998,
+ "meanTokenConsumption": 36498.333333333336,
+ "meanToolInvocationCount": 12.333333333333334
},
{
- "experimentId": "minimax-m3-opencode--guided",
+ "experimentId": "gpt-6-sol-high-codex--guided",
"caseId": "author-002-new-project",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 213766,
- "measuredAt": "2026-09-15T16:54:38.085Z",
- "meanEstimatedListCostUsd": 0.01131386,
- "meanTokenConsumption": 11259.666666666666,
- "meanToolInvocationCount": 11.666666666666666
+ "meanDurationMs": 154722,
+ "measuredAt": "2026-10-07T18:53:33.054Z",
+ "meanEstimatedListCostUsd": 0.2336195,
+ "meanTokenConsumption": 35601.666666666664,
+ "meanToolInvocationCount": 17.666666666666668
},
{
- "experimentId": "minimax-m3-opencode--guided",
+ "experimentId": "gpt-6-sol-high-codex--guided",
"caseId": "author-003-openapi-connection",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 120271,
- "measuredAt": "2026-09-15T16:41:09.951Z",
- "meanEstimatedListCostUsd": 0.02107842,
- "meanTokenConsumption": 32130.333333333332,
- "meanToolInvocationCount": 14.333333333333334
+ "meanDurationMs": 163348,
+ "measuredAt": "2026-10-07T18:53:33.066Z",
+ "meanEstimatedListCostUsd": 0.1844876666666667,
+ "meanTokenConsumption": 29518,
+ "meanToolInvocationCount": 13.666666666666666
},
{
- "experimentId": "minimax-m3-opencode--guided",
+ "experimentId": "gpt-6-sol-high-codex--guided",
"caseId": "author-004-packaged-skill",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 135700,
- "measuredAt": "2026-09-15T16:41:09.959Z",
- "meanEstimatedListCostUsd": 0.01193822,
- "meanTokenConsumption": 15964,
- "meanToolInvocationCount": 9
+ "meanDurationMs": 123914,
+ "measuredAt": "2026-10-07T18:53:33.074Z",
+ "meanEstimatedListCostUsd": 0.12451166666666667,
+ "meanTokenConsumption": 21815.666666666668,
+ "meanToolInvocationCount": 5.333333333333333
},
{
- "experimentId": "minimax-m3-opencode--guided",
+ "experimentId": "gpt-6-sol-high-codex--guided",
"caseId": "author-005-conditional-approval",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 115474,
- "measuredAt": "2026-09-15T16:41:09.966Z",
- "meanEstimatedListCostUsd": 0.01810478,
- "meanTokenConsumption": 33213.333333333336,
- "meanToolInvocationCount": 11.333333333333334
+ "meanDurationMs": 168992,
+ "measuredAt": "2026-10-07T18:53:33.083Z",
+ "meanEstimatedListCostUsd": 0.21053246666666667,
+ "meanTokenConsumption": 32490.333333333332,
+ "meanToolInvocationCount": 13.333333333333334
},
{
- "experimentId": "minimax-m3-opencode--guided",
+ "experimentId": "gpt-6-sol-high-codex--guided",
"caseId": "author-006-custom-channel",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 113667,
- "measuredAt": "2026-09-15T16:41:09.974Z",
- "meanEstimatedListCostUsd": 0.012573259999999998,
- "meanTokenConsumption": 17879.666666666668,
- "meanToolInvocationCount": 13
+ "meanDurationMs": 145077,
+ "measuredAt": "2026-10-07T18:53:33.091Z",
+ "meanEstimatedListCostUsd": 0.14791393333333333,
+ "meanTokenConsumption": 25818,
+ "meanToolInvocationCount": 8
},
{
- "experimentId": "minimax-m3-opencode--guided",
+ "experimentId": "gpt-6-sol-high-codex--guided",
"caseId": "author-007-digest-schedule",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 123417,
- "measuredAt": "2026-09-15T16:41:09.982Z",
- "meanEstimatedListCostUsd": 0.0110418,
- "meanTokenConsumption": 14192.333333333334,
- "meanToolInvocationCount": 10.666666666666666
+ "meanDurationMs": 136865,
+ "measuredAt": "2026-10-07T18:53:33.100Z",
+ "meanEstimatedListCostUsd": 0.13206753333333335,
+ "meanTokenConsumption": 21706.333333333332,
+ "meanToolInvocationCount": 9.666666666666666
},
{
- "experimentId": "kimi-k2-7-code-opencode--baseline",
+ "experimentId": "gpt-6-luna-high-codex--baseline",
"caseId": "author-001-weather-tool",
"status": "current",
- "passedRuns": 2,
+ "passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 208245,
- "measuredAt": "2026-09-15T16:48:32.061Z",
- "meanEstimatedListCostUsd": 0.14676820333333332,
- "meanTokenConsumption": 56908.666666666664,
- "meanToolInvocationCount": 34
+ "meanDurationMs": 172981,
+ "measuredAt": "2026-10-07T19:02:42.860Z",
+ "meanEstimatedListCostUsd": 0.014657728333333335,
+ "meanTokenConsumption": 46713.666666666664,
+ "meanToolInvocationCount": 11.333333333333334
},
{
- "experimentId": "kimi-k2-7-code-opencode--baseline",
+ "experimentId": "gpt-6-luna-high-codex--baseline",
"caseId": "author-002-new-project",
"status": "current",
+ "passedRuns": 2,
+ "validRuns": 3,
+ "meanDurationMs": 208392,
+ "measuredAt": "2026-10-07T19:02:42.873Z",
+ "meanEstimatedListCostUsd": 0.02044859666666667,
+ "meanTokenConsumption": 53849,
+ "meanToolInvocationCount": 15
+ },
+ {
+ "experimentId": "gpt-6-luna-high-codex--baseline",
+ "caseId": "author-003-openapi-connection",
+ "status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 429898,
- "measuredAt": "2026-09-15T16:48:32.072Z",
- "meanEstimatedListCostUsd": 0.04066295,
- "meanTokenConsumption": 22378.666666666668,
- "meanToolInvocationCount": 20.333333333333332
+ "meanDurationMs": 122976,
+ "measuredAt": "2026-10-07T19:02:42.882Z",
+ "meanEstimatedListCostUsd": 0.008839191666666668,
+ "meanTokenConsumption": 29309,
+ "meanToolInvocationCount": 15.333333333333334
},
{
- "experimentId": "kimi-k2-7-code-opencode--baseline",
+ "experimentId": "gpt-6-luna-high-codex--baseline",
+ "caseId": "author-004-packaged-skill",
+ "status": "current",
+ "passedRuns": 3,
+ "validRuns": 3,
+ "meanDurationMs": 131981,
+ "measuredAt": "2026-10-07T19:02:42.890Z",
+ "meanEstimatedListCostUsd": 0.00776646,
+ "meanTokenConsumption": 25521.333333333332,
+ "meanToolInvocationCount": 9
+ },
+ {
+ "experimentId": "gpt-6-luna-high-codex--baseline",
+ "caseId": "author-005-conditional-approval",
+ "status": "current",
+ "passedRuns": 3,
+ "validRuns": 3,
+ "meanDurationMs": 153769,
+ "measuredAt": "2026-10-07T19:02:42.898Z",
+ "meanEstimatedListCostUsd": 0.012869688333333332,
+ "meanTokenConsumption": 40411.333333333336,
+ "meanToolInvocationCount": 10.333333333333334
+ },
+ {
+ "experimentId": "gpt-6-luna-high-codex--baseline",
+ "caseId": "author-006-custom-channel",
+ "status": "current",
+ "passedRuns": 3,
+ "validRuns": 3,
+ "meanDurationMs": 137271,
+ "measuredAt": "2026-10-07T19:02:42.909Z",
+ "meanEstimatedListCostUsd": 0.013865816666666668,
+ "meanTokenConsumption": 46150,
+ "meanToolInvocationCount": 9.666666666666666
+ },
+ {
+ "experimentId": "gpt-6-luna-high-codex--baseline",
+ "caseId": "author-007-digest-schedule",
+ "status": "current",
+ "passedRuns": 3,
+ "validRuns": 3,
+ "meanDurationMs": 129008,
+ "measuredAt": "2026-10-07T19:02:42.919Z",
+ "meanEstimatedListCostUsd": 0.007547296666666667,
+ "meanTokenConsumption": 24802.666666666668,
+ "meanToolInvocationCount": 11
+ },
+ {
+ "experimentId": "gpt-6-luna-high-codex--guided",
+ "caseId": "author-001-weather-tool",
+ "status": "current",
+ "passedRuns": 3,
+ "validRuns": 3,
+ "meanDurationMs": 140987,
+ "measuredAt": "2026-10-07T19:09:00.404Z",
+ "meanEstimatedListCostUsd": 0.011568716666666668,
+ "meanTokenConsumption": 35932.666666666664,
+ "meanToolInvocationCount": 10.666666666666666
+ },
+ {
+ "experimentId": "gpt-6-luna-high-codex--guided",
+ "caseId": "author-002-new-project",
+ "status": "current",
+ "passedRuns": 2,
+ "validRuns": 3,
+ "meanDurationMs": 243860,
+ "measuredAt": "2026-10-07T19:09:00.418Z",
+ "meanEstimatedListCostUsd": 0.025036305,
+ "meanTokenConsumption": 60119.333333333336,
+ "meanToolInvocationCount": 19.666666666666668
+ },
+ {
+ "experimentId": "gpt-6-luna-high-codex--guided",
"caseId": "author-003-openapi-connection",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 181692,
- "measuredAt": "2026-09-15T16:48:32.080Z",
- "meanEstimatedListCostUsd": 0.13857528666666666,
- "meanTokenConsumption": 46478.333333333336,
- "meanToolInvocationCount": 29.333333333333332
+ "meanDurationMs": 116960,
+ "measuredAt": "2026-10-07T19:09:00.454Z",
+ "meanEstimatedListCostUsd": 0.007297571666666666,
+ "meanTokenConsumption": 23631,
+ "meanToolInvocationCount": 6.333333333333333
},
{
- "experimentId": "kimi-k2-7-code-opencode--baseline",
+ "experimentId": "gpt-6-luna-high-codex--guided",
"caseId": "author-004-packaged-skill",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 173418,
- "measuredAt": "2026-09-15T16:48:32.092Z",
- "meanEstimatedListCostUsd": 0.05450496666666666,
- "meanTokenConsumption": 26974.333333333332,
- "meanToolInvocationCount": 18.666666666666668
+ "meanDurationMs": 148482,
+ "measuredAt": "2026-10-07T19:09:00.463Z",
+ "meanEstimatedListCostUsd": 0.007017896666666666,
+ "meanTokenConsumption": 23253.666666666668,
+ "meanToolInvocationCount": 6.666666666666667
},
{
- "experimentId": "kimi-k2-7-code-opencode--baseline",
+ "experimentId": "gpt-6-luna-high-codex--guided",
"caseId": "author-005-conditional-approval",
"status": "current",
+ "passedRuns": 3,
+ "validRuns": 3,
+ "meanDurationMs": 164463,
+ "measuredAt": "2026-10-07T19:09:00.470Z",
+ "meanEstimatedListCostUsd": 0.00887198,
+ "meanTokenConsumption": 28326.333333333332,
+ "meanToolInvocationCount": 8
+ },
+ {
+ "experimentId": "gpt-6-luna-high-codex--guided",
+ "caseId": "author-006-custom-channel",
+ "status": "current",
+ "passedRuns": 3,
+ "validRuns": 3,
+ "meanDurationMs": 117293,
+ "measuredAt": "2026-10-07T19:09:00.477Z",
+ "meanEstimatedListCostUsd": 0.006835386666666668,
+ "meanTokenConsumption": 22433,
+ "meanToolInvocationCount": 8
+ },
+ {
+ "experimentId": "gpt-6-luna-high-codex--guided",
+ "caseId": "author-007-digest-schedule",
+ "status": "current",
+ "passedRuns": 3,
+ "validRuns": 3,
+ "meanDurationMs": 138527,
+ "measuredAt": "2026-10-07T19:09:00.484Z",
+ "meanEstimatedListCostUsd": 0.008701025000000001,
+ "meanTokenConsumption": 26987.333333333332,
+ "meanToolInvocationCount": 7.333333333333333
+ },
+ {
+ "experimentId": "grok-4-7-opencode--baseline",
+ "caseId": "author-001-weather-tool",
+ "status": "current",
+ "passedRuns": 3,
+ "validRuns": 3,
+ "meanDurationMs": 136051,
+ "measuredAt": "2026-10-07T19:28:24.953Z",
+ "meanEstimatedListCostUsd": 0.17287466666666665,
+ "meanTokenConsumption": 52152,
+ "meanToolInvocationCount": 18.333333333333332
+ },
+ {
+ "experimentId": "grok-4-7-opencode--baseline",
+ "caseId": "author-002-new-project",
+ "status": "current",
"passedRuns": 2,
"validRuns": 3,
- "meanDurationMs": 204741,
- "measuredAt": "2026-09-15T16:48:32.101Z",
- "meanEstimatedListCostUsd": 0.09485771,
- "meanTokenConsumption": 38354,
- "meanToolInvocationCount": 32
+ "meanDurationMs": 190658,
+ "measuredAt": "2026-10-07T19:28:24.962Z",
+ "meanEstimatedListCostUsd": 0.21032133333333333,
+ "meanTokenConsumption": 42335.333333333336,
+ "meanToolInvocationCount": 26
},
{
- "experimentId": "kimi-k2-7-code-opencode--baseline",
+ "experimentId": "grok-4-7-opencode--baseline",
+ "caseId": "author-003-openapi-connection",
+ "status": "current",
+ "passedRuns": 2,
+ "validRuns": 3,
+ "meanDurationMs": 383235,
+ "measuredAt": "2026-10-07T19:28:24.975Z",
+ "meanEstimatedListCostUsd": 0.143828,
+ "meanTokenConsumption": 43541,
+ "meanToolInvocationCount": 17
+ },
+ {
+ "experimentId": "grok-4-7-opencode--baseline",
+ "caseId": "author-004-packaged-skill",
+ "status": "current",
+ "passedRuns": 3,
+ "validRuns": 3,
+ "meanDurationMs": 111116,
+ "measuredAt": "2026-10-07T19:28:24.981Z",
+ "meanEstimatedListCostUsd": 0.13283466666666666,
+ "meanTokenConsumption": 34128,
+ "meanToolInvocationCount": 17.666666666666668
+ },
+ {
+ "experimentId": "grok-4-7-opencode--baseline",
+ "caseId": "author-005-conditional-approval",
+ "status": "current",
+ "passedRuns": 2,
+ "validRuns": 3,
+ "meanDurationMs": 383889,
+ "measuredAt": "2026-10-07T19:28:24.992Z",
+ "meanEstimatedListCostUsd": 0.226556,
+ "meanTokenConsumption": 65023,
+ "meanToolInvocationCount": 23.5
+ },
+ {
+ "experimentId": "grok-4-7-opencode--baseline",
"caseId": "author-006-custom-channel",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 197645,
- "measuredAt": "2026-09-15T16:48:32.111Z",
- "meanEstimatedListCostUsd": 0.14388091,
- "meanTokenConsumption": 51045.333333333336,
- "meanToolInvocationCount": 34.333333333333336
+ "meanDurationMs": 116196,
+ "measuredAt": "2026-10-07T19:28:25.000Z",
+ "meanEstimatedListCostUsd": 0.13987,
+ "meanTokenConsumption": 25978.333333333332,
+ "meanToolInvocationCount": 18
},
{
- "experimentId": "kimi-k2-7-code-opencode--baseline",
+ "experimentId": "grok-4-7-opencode--baseline",
"caseId": "author-007-digest-schedule",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 157203,
- "measuredAt": "2026-09-15T16:48:32.128Z",
- "meanEstimatedListCostUsd": 0.05187631333333333,
- "meanTokenConsumption": 19794.666666666668,
- "meanToolInvocationCount": 20.666666666666668
+ "meanDurationMs": 115736,
+ "measuredAt": "2026-10-07T19:28:25.013Z",
+ "meanEstimatedListCostUsd": 0.09889266666666667,
+ "meanTokenConsumption": 27181,
+ "meanToolInvocationCount": 15.333333333333334
},
{
- "experimentId": "kimi-k2-7-code-opencode--guided",
+ "experimentId": "grok-4-7-opencode--guided",
"caseId": "author-001-weather-tool",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 166225,
- "measuredAt": "2026-09-15T16:42:45.614Z",
- "meanEstimatedListCostUsd": 0.05287293333333334,
- "meanTokenConsumption": 20028.666666666668,
- "meanToolInvocationCount": 16.666666666666668
+ "meanDurationMs": 131291,
+ "measuredAt": "2026-10-07T19:33:57.511Z",
+ "meanEstimatedListCostUsd": 0.13136866666666666,
+ "meanTokenConsumption": 29745,
+ "meanToolInvocationCount": 11.333333333333334
},
{
- "experimentId": "kimi-k2-7-code-opencode--guided",
+ "experimentId": "grok-4-7-opencode--guided",
"caseId": "author-002-new-project",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 293340,
- "measuredAt": "2026-09-15T16:42:45.647Z",
- "meanEstimatedListCostUsd": 0.04787821666666667,
- "meanTokenConsumption": 21184.333333333332,
- "meanToolInvocationCount": 22.666666666666668
+ "meanDurationMs": 137195,
+ "measuredAt": "2026-10-07T19:33:57.518Z",
+ "meanEstimatedListCostUsd": 0.222338,
+ "meanTokenConsumption": 61214.333333333336,
+ "meanToolInvocationCount": 23.333333333333332
},
{
- "experimentId": "kimi-k2-7-code-opencode--guided",
+ "experimentId": "grok-4-7-opencode--guided",
"caseId": "author-003-openapi-connection",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 166937,
- "measuredAt": "2026-09-15T16:42:45.661Z",
- "meanEstimatedListCostUsd": 0.05114830333333333,
- "meanTokenConsumption": 20667.666666666668,
- "meanToolInvocationCount": 17.666666666666668
+ "meanDurationMs": 110807,
+ "measuredAt": "2026-10-07T19:33:57.527Z",
+ "meanEstimatedListCostUsd": 0.14055266666666666,
+ "meanTokenConsumption": 35289,
+ "meanToolInvocationCount": 12
},
{
- "experimentId": "kimi-k2-7-code-opencode--guided",
+ "experimentId": "grok-4-7-opencode--guided",
"caseId": "author-004-packaged-skill",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 137327,
- "measuredAt": "2026-09-15T16:42:45.672Z",
- "meanEstimatedListCostUsd": 0.03893913333333334,
- "meanTokenConsumption": 20582,
- "meanToolInvocationCount": 12
+ "meanDurationMs": 134888,
+ "measuredAt": "2026-10-07T19:33:57.533Z",
+ "meanEstimatedListCostUsd": 0.05279999999999999,
+ "meanTokenConsumption": 10001.333333333334,
+ "meanToolInvocationCount": 5.666666666666667
},
{
- "experimentId": "kimi-k2-7-code-opencode--guided",
+ "experimentId": "grok-4-7-opencode--guided",
"caseId": "author-005-conditional-approval",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 155897,
- "measuredAt": "2026-09-15T16:42:45.681Z",
- "meanEstimatedListCostUsd": 0.042350406666666666,
- "meanTokenConsumption": 22769,
- "meanToolInvocationCount": 13.666666666666666
+ "meanDurationMs": 108481,
+ "measuredAt": "2026-10-07T19:33:57.539Z",
+ "meanEstimatedListCostUsd": 0.10046466666666666,
+ "meanTokenConsumption": 18673.666666666668,
+ "meanToolInvocationCount": 9.666666666666666
},
{
- "experimentId": "kimi-k2-7-code-opencode--guided",
+ "experimentId": "grok-4-7-opencode--guided",
"caseId": "author-006-custom-channel",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 143732,
- "measuredAt": "2026-09-15T16:42:45.690Z",
- "meanEstimatedListCostUsd": 0.05902574,
- "meanTokenConsumption": 32431.666666666668,
- "meanToolInvocationCount": 15.666666666666666
+ "meanDurationMs": 108256,
+ "measuredAt": "2026-10-07T19:33:57.548Z",
+ "meanEstimatedListCostUsd": 0.09657466666666666,
+ "meanTokenConsumption": 15122,
+ "meanToolInvocationCount": 7
},
{
- "experimentId": "kimi-k2-7-code-opencode--guided",
+ "experimentId": "grok-4-7-opencode--guided",
"caseId": "author-007-digest-schedule",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 171310,
- "measuredAt": "2026-09-15T16:42:45.699Z",
- "meanEstimatedListCostUsd": 0.047509576666666664,
- "meanTokenConsumption": 20610.666666666668,
- "meanToolInvocationCount": 17
+ "meanDurationMs": 115721,
+ "measuredAt": "2026-10-07T19:33:57.559Z",
+ "meanEstimatedListCostUsd": 0.07453466666666668,
+ "meanTokenConsumption": 23851.333333333332,
+ "meanToolInvocationCount": 5
}
],
"previouslyMeasured": [
@@ -2623,22 +2807,6 @@
"modelDisplayName": "Claude Fable 5",
"harness": "Claude Code",
"treatment": "guided"
- },
- {
- "id": "glm-5-2-opencode--baseline",
- "groupId": "glm-5-2-opencode",
- "model": "zai/glm-5.2",
- "modelDisplayName": "GLM 5.2",
- "harness": "OpenCode",
- "treatment": "baseline"
- },
- {
- "id": "glm-5-2-opencode--guided",
- "groupId": "glm-5-2-opencode",
- "model": "zai/glm-5.2",
- "modelDisplayName": "GLM 5.2",
- "harness": "OpenCode",
- "treatment": "guided"
}
],
"results": [
@@ -2809,42 +2977,634 @@
"meanEstimatedListCostUsd": 3.07521,
"meanToolInvocationCount": 7,
"meanTokenConsumption": 299897
+ }
+ ]
+ },
+ {
+ "suite": {
+ "eveRevision": "6ce61cd85ecf4621bcefd363a3e8b9445b7e6e91",
+ "caseFingerprint": "7d990b7e2c429ee4a2cf4f758d15f408f61d0e5b1ef9782ba69a33e435cdb169",
+ "caseCount": 7,
+ "runsPerCell": 3
+ },
+ "experiments": [
+ {
+ "id": "grok-4-6-opencode--baseline",
+ "groupId": "grok-4-6-opencode",
+ "model": "xai/grok-4.6",
+ "modelDisplayName": "Grok 4.6",
+ "harness": "OpenCode",
+ "treatment": "baseline"
},
{
- "experimentId": "glm-5-2-opencode--baseline",
+ "id": "grok-4-6-opencode--guided",
+ "groupId": "grok-4-6-opencode",
+ "model": "xai/grok-4.6",
+ "modelDisplayName": "Grok 4.6",
+ "harness": "OpenCode",
+ "treatment": "guided"
+ },
+ {
+ "id": "gpt-5-6-sol-codex--baseline",
+ "groupId": "gpt-5-6-sol-codex",
+ "model": "openai/gpt-5.6-sol",
+ "modelDisplayName": "GPT-5.6 Sol",
+ "harness": "Codex",
+ "treatment": "baseline"
+ },
+ {
+ "id": "gpt-5-6-sol-codex--guided",
+ "groupId": "gpt-5-6-sol-codex",
+ "model": "openai/gpt-5.6-sol",
+ "modelDisplayName": "GPT-5.6 Sol",
+ "harness": "Codex",
+ "treatment": "guided"
+ },
+ {
+ "id": "claude-opus-5-claude-code--baseline",
+ "groupId": "claude-opus-5-claude-code",
+ "model": "anthropic/claude-opus-5",
+ "modelDisplayName": "Claude Opus 5",
+ "harness": "Claude Code",
+ "treatment": "baseline"
+ },
+ {
+ "id": "claude-opus-5-claude-code--guided",
+ "groupId": "claude-opus-5-claude-code",
+ "model": "anthropic/claude-opus-5",
+ "modelDisplayName": "Claude Opus 5",
+ "harness": "Claude Code",
+ "treatment": "guided"
+ }
+ ],
+ "results": [
+ {
+ "experimentId": "grok-4-6-opencode--baseline",
"caseId": "author-001-weather-tool",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 103530,
- "measuredAt": "2026-08-28T19:22:09.000Z",
- "meanEstimatedListCostUsd": 0.07017806666666666,
- "meanTokenConsumption": 37804.333333333336,
- "meanToolInvocationCount": 25
+ "meanDurationMs": 38613,
+ "measuredAt": "2026-08-28T19:22:01.000Z",
+ "meanEstimatedListCostUsd": 0.10434266666666665,
+ "meanTokenConsumption": 49566,
+ "meanToolInvocationCount": 19
},
{
- "experimentId": "glm-5-2-opencode--baseline",
+ "experimentId": "grok-4-6-opencode--baseline",
"caseId": "author-002-new-project",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 100847,
- "measuredAt": "2026-08-28T19:26:40.000Z",
- "meanEstimatedListCostUsd": 0.03248553333333333,
- "meanTokenConsumption": 19034.666666666668,
- "meanToolInvocationCount": 18.333333333333332
+ "meanDurationMs": 83985,
+ "measuredAt": "2026-08-28T19:25:12.000Z",
+ "meanEstimatedListCostUsd": 0.07232866666666667,
+ "meanTokenConsumption": 32827.666666666664,
+ "meanToolInvocationCount": 16
},
{
- "experimentId": "glm-5-2-opencode--baseline",
+ "experimentId": "grok-4-6-opencode--baseline",
"caseId": "author-003-openapi-connection",
"status": "current",
- "passedRuns": 3,
+ "passedRuns": 0,
"validRuns": 3,
- "meanDurationMs": 134009,
- "measuredAt": "2026-08-28T19:26:40.000Z",
- "meanEstimatedListCostUsd": 0.09391753333333334,
- "meanTokenConsumption": 49094.666666666664,
- "meanToolInvocationCount": 40.333333333333336
+ "meanDurationMs": 521954,
+ "measuredAt": "2026-08-28T19:22:01.000Z",
+ "meanEstimatedListCostUsd": 1.0741786666666668,
+ "meanTokenConsumption": 503422.6666666667,
+ "meanToolInvocationCount": 76.66666666666667
+ },
+ {
+ "experimentId": "grok-4-6-opencode--baseline",
+ "caseId": "author-004-packaged-skill",
+ "status": "current",
+ "passedRuns": 3,
+ "validRuns": 3,
+ "meanDurationMs": 41894,
+ "measuredAt": "2026-08-28T19:22:01.000Z",
+ "meanEstimatedListCostUsd": 0.065152,
+ "meanTokenConsumption": 29506,
+ "meanToolInvocationCount": 23.333333333333332
+ },
+ {
+ "experimentId": "grok-4-6-opencode--baseline",
+ "caseId": "author-005-conditional-approval",
+ "status": "current",
+ "passedRuns": 3,
+ "validRuns": 3,
+ "meanDurationMs": 50151,
+ "measuredAt": "2026-08-28T19:22:01.000Z",
+ "meanEstimatedListCostUsd": 0.141536,
+ "meanTokenConsumption": 66629.33333333333,
+ "meanToolInvocationCount": 22
+ },
+ {
+ "experimentId": "grok-4-6-opencode--baseline",
+ "caseId": "author-006-custom-channel",
+ "status": "current",
+ "passedRuns": 3,
+ "validRuns": 3,
+ "meanDurationMs": 37062,
+ "measuredAt": "2026-08-28T19:22:01.000Z",
+ "meanEstimatedListCostUsd": 0.07319533333333333,
+ "meanTokenConsumption": 33801.666666666664,
+ "meanToolInvocationCount": 17.666666666666668
+ },
+ {
+ "experimentId": "grok-4-6-opencode--baseline",
+ "caseId": "author-007-digest-schedule",
+ "status": "current",
+ "passedRuns": 3,
+ "validRuns": 3,
+ "meanDurationMs": 38280,
+ "measuredAt": "2026-08-28T19:22:01.000Z",
+ "meanEstimatedListCostUsd": 0.045748000000000004,
+ "meanTokenConsumption": 20122.666666666668,
+ "meanToolInvocationCount": 18
+ },
+ {
+ "experimentId": "grok-4-6-opencode--guided",
+ "caseId": "author-001-weather-tool",
+ "status": "current",
+ "passedRuns": 3,
+ "validRuns": 3,
+ "meanDurationMs": 31700,
+ "measuredAt": "2026-08-28T19:22:02.000Z",
+ "meanEstimatedListCostUsd": 0.049695333333333334,
+ "meanTokenConsumption": 22963.666666666668,
+ "meanToolInvocationCount": 9.666666666666666
+ },
+ {
+ "experimentId": "grok-4-6-opencode--guided",
+ "caseId": "author-002-new-project",
+ "status": "current",
+ "passedRuns": 3,
+ "validRuns": 3,
+ "meanDurationMs": 83621,
+ "measuredAt": "2026-08-28T19:25:07.000Z",
+ "meanEstimatedListCostUsd": 0.082842,
+ "meanTokenConsumption": 38261,
+ "meanToolInvocationCount": 16.333333333333332
+ },
+ {
+ "experimentId": "grok-4-6-opencode--guided",
+ "caseId": "author-003-openapi-connection",
+ "status": "current",
+ "passedRuns": 3,
+ "validRuns": 3,
+ "meanDurationMs": 34833,
+ "measuredAt": "2026-08-28T19:25:07.000Z",
+ "meanEstimatedListCostUsd": 0.055174,
+ "meanTokenConsumption": 25952.333333333332,
+ "meanToolInvocationCount": 10
+ },
+ {
+ "experimentId": "grok-4-6-opencode--guided",
+ "caseId": "author-004-packaged-skill",
+ "status": "current",
+ "passedRuns": 3,
+ "validRuns": 3,
+ "meanDurationMs": 30135,
+ "measuredAt": "2026-08-28T19:22:02.000Z",
+ "meanEstimatedListCostUsd": 0.035581999999999996,
+ "meanTokenConsumption": 16043.666666666666,
+ "meanToolInvocationCount": 8.666666666666666
+ },
+ {
+ "experimentId": "grok-4-6-opencode--guided",
+ "caseId": "author-005-conditional-approval",
+ "status": "current",
+ "passedRuns": 3,
+ "validRuns": 3,
+ "meanDurationMs": 34026,
+ "measuredAt": "2026-08-28T19:22:02.000Z",
+ "meanEstimatedListCostUsd": 0.038532666666666666,
+ "meanTokenConsumption": 17059.666666666668,
+ "meanToolInvocationCount": 11.666666666666666
+ },
+ {
+ "experimentId": "grok-4-6-opencode--guided",
+ "caseId": "author-006-custom-channel",
+ "status": "current",
+ "passedRuns": 3,
+ "validRuns": 3,
+ "meanDurationMs": 32109,
+ "measuredAt": "2026-08-28T19:22:02.000Z",
+ "meanEstimatedListCostUsd": 0.033709333333333334,
+ "meanTokenConsumption": 14610,
+ "meanToolInvocationCount": 9.666666666666666
+ },
+ {
+ "experimentId": "grok-4-6-opencode--guided",
+ "caseId": "author-007-digest-schedule",
+ "status": "current",
+ "passedRuns": 3,
+ "validRuns": 3,
+ "meanDurationMs": 25583,
+ "measuredAt": "2026-08-28T19:22:02.000Z",
+ "meanEstimatedListCostUsd": 0.023458666666666666,
+ "meanTokenConsumption": 10328,
+ "meanToolInvocationCount": 7.333333333333333
+ },
+ {
+ "experimentId": "gpt-5-6-sol-codex--baseline",
+ "caseId": "author-001-weather-tool",
+ "status": "current",
+ "passedRuns": 3,
+ "validRuns": 3,
+ "meanDurationMs": 175042,
+ "measuredAt": "2026-08-31T19:20:09.055Z",
+ "meanEstimatedListCostUsd": 0.42096675,
+ "meanTokenConsumption": 49663.666666666664,
+ "meanToolInvocationCount": 13.666666666666666
+ },
+ {
+ "experimentId": "gpt-5-6-sol-codex--baseline",
+ "caseId": "author-002-new-project",
+ "status": "current",
+ "passedRuns": 3,
+ "validRuns": 3,
+ "meanDurationMs": 122470,
+ "measuredAt": "2026-08-31T19:20:09.085Z",
+ "meanEstimatedListCostUsd": 0.23273858333333333,
+ "meanTokenConsumption": 30848.666666666668,
+ "meanToolInvocationCount": 10.333333333333334
+ },
+ {
+ "experimentId": "gpt-5-6-sol-codex--baseline",
+ "caseId": "author-003-openapi-connection",
+ "status": "current",
+ "passedRuns": 3,
+ "validRuns": 3,
+ "meanDurationMs": 132036,
+ "measuredAt": "2026-08-31T19:20:09.104Z",
+ "meanEstimatedListCostUsd": 0.367498625,
+ "meanTokenConsumption": 48982.666666666664,
+ "meanToolInvocationCount": 9.333333333333334
+ },
+ {
+ "experimentId": "gpt-5-6-sol-codex--baseline",
+ "caseId": "author-004-packaged-skill",
+ "status": "current",
+ "passedRuns": 3,
+ "validRuns": 3,
+ "meanDurationMs": 137447,
+ "measuredAt": "2026-08-31T19:20:09.128Z",
+ "meanEstimatedListCostUsd": 0.26755258333333337,
+ "meanTokenConsumption": 36330.666666666664,
+ "meanToolInvocationCount": 6
+ },
+ {
+ "experimentId": "gpt-5-6-sol-codex--baseline",
+ "caseId": "author-005-conditional-approval",
+ "status": "current",
+ "passedRuns": 3,
+ "validRuns": 3,
+ "meanDurationMs": 121857,
+ "measuredAt": "2026-08-31T19:20:09.143Z",
+ "meanEstimatedListCostUsd": 0.2530605833333333,
+ "meanTokenConsumption": 32384,
+ "meanToolInvocationCount": 10
+ },
+ {
+ "experimentId": "gpt-5-6-sol-codex--baseline",
+ "caseId": "author-006-custom-channel",
+ "status": "current",
+ "passedRuns": 3,
+ "validRuns": 3,
+ "meanDurationMs": 115653,
+ "measuredAt": "2026-08-31T19:20:09.156Z",
+ "meanEstimatedListCostUsd": 0.251469,
+ "meanTokenConsumption": 33854.666666666664,
+ "meanToolInvocationCount": 8
+ },
+ {
+ "experimentId": "gpt-5-6-sol-codex--baseline",
+ "caseId": "author-007-digest-schedule",
+ "status": "current",
+ "passedRuns": 3,
+ "validRuns": 3,
+ "meanDurationMs": 121172,
+ "measuredAt": "2026-08-31T19:20:09.170Z",
+ "meanEstimatedListCostUsd": 0.284567375,
+ "meanTokenConsumption": 37330.666666666664,
+ "meanToolInvocationCount": 11
+ },
+ {
+ "experimentId": "gpt-5-6-sol-codex--guided",
+ "caseId": "author-001-weather-tool",
+ "status": "current",
+ "passedRuns": 3,
+ "validRuns": 3,
+ "meanDurationMs": 115694,
+ "measuredAt": "2026-08-31T19:18:53.704Z",
+ "meanEstimatedListCostUsd": 0.18167699999999998,
+ "meanTokenConsumption": 24500,
+ "meanToolInvocationCount": 4.666666666666667
+ },
+ {
+ "experimentId": "gpt-5-6-sol-codex--guided",
+ "caseId": "author-002-new-project",
+ "status": "current",
+ "passedRuns": 3,
+ "validRuns": 3,
+ "meanDurationMs": 132614,
+ "measuredAt": "2026-08-31T19:18:53.727Z",
+ "meanEstimatedListCostUsd": 0.22479154166666668,
+ "meanTokenConsumption": 28792.333333333332,
+ "meanToolInvocationCount": 9.333333333333334
+ },
+ {
+ "experimentId": "gpt-5-6-sol-codex--guided",
+ "caseId": "author-003-openapi-connection",
+ "status": "current",
+ "passedRuns": 3,
+ "validRuns": 3,
+ "meanDurationMs": 101648,
+ "measuredAt": "2026-08-31T19:18:53.741Z",
+ "meanEstimatedListCostUsd": 0.14007875,
+ "meanTokenConsumption": 19209.666666666668,
+ "meanToolInvocationCount": 5.333333333333333
+ },
+ {
+ "experimentId": "gpt-5-6-sol-codex--guided",
+ "caseId": "author-004-packaged-skill",
+ "status": "current",
+ "passedRuns": 3,
+ "validRuns": 3,
+ "meanDurationMs": 129791,
+ "measuredAt": "2026-08-31T19:18:53.751Z",
+ "meanEstimatedListCostUsd": 0.19388683333333334,
+ "meanTokenConsumption": 24916,
+ "meanToolInvocationCount": 6
+ },
+ {
+ "experimentId": "gpt-5-6-sol-codex--guided",
+ "caseId": "author-005-conditional-approval",
+ "status": "current",
+ "passedRuns": 3,
+ "validRuns": 3,
+ "meanDurationMs": 108054,
+ "measuredAt": "2026-08-31T19:18:53.759Z",
+ "meanEstimatedListCostUsd": 0.15307379166666665,
+ "meanTokenConsumption": 20874.333333333332,
+ "meanToolInvocationCount": 4.666666666666667
+ },
+ {
+ "experimentId": "gpt-5-6-sol-codex--guided",
+ "caseId": "author-006-custom-channel",
+ "status": "current",
+ "passedRuns": 3,
+ "validRuns": 3,
+ "meanDurationMs": 100410,
+ "measuredAt": "2026-08-31T19:18:53.769Z",
+ "meanEstimatedListCostUsd": 0.108055625,
+ "meanTokenConsumption": 15663.666666666666,
+ "meanToolInvocationCount": 3
+ },
+ {
+ "experimentId": "gpt-5-6-sol-codex--guided",
+ "caseId": "author-007-digest-schedule",
+ "status": "current",
+ "passedRuns": 3,
+ "validRuns": 3,
+ "meanDurationMs": 103849,
+ "measuredAt": "2026-08-31T19:18:53.779Z",
+ "meanEstimatedListCostUsd": 0.11547295833333333,
+ "meanTokenConsumption": 16687.333333333332,
+ "meanToolInvocationCount": 3.6666666666666665
+ },
+ {
+ "experimentId": "claude-opus-5-claude-code--baseline",
+ "caseId": "author-001-weather-tool",
+ "status": "current",
+ "passedRuns": 3,
+ "validRuns": 3,
+ "meanDurationMs": 155202,
+ "measuredAt": "2026-09-02T14:35:36.609Z",
+ "meanEstimatedListCostUsd": 1.1599598333333334,
+ "meanTokenConsumption": 10714.666666666666,
+ "meanToolInvocationCount": 17.333333333333332
+ },
+ {
+ "experimentId": "claude-opus-5-claude-code--baseline",
+ "caseId": "author-002-new-project",
+ "status": "current",
+ "passedRuns": 3,
+ "validRuns": 3,
+ "meanDurationMs": 124976,
+ "measuredAt": "2026-09-02T14:35:36.628Z",
+ "meanEstimatedListCostUsd": 0.8593296666666667,
+ "meanTokenConsumption": 3845.3333333333335,
+ "meanToolInvocationCount": 10.333333333333334
+ },
+ {
+ "experimentId": "claude-opus-5-claude-code--baseline",
+ "caseId": "author-003-openapi-connection",
+ "status": "current",
+ "passedRuns": 3,
+ "validRuns": 3,
+ "meanDurationMs": 160123,
+ "measuredAt": "2026-09-02T14:35:36.643Z",
+ "meanEstimatedListCostUsd": 1.3671014166666666,
+ "meanTokenConsumption": 7793.333333333333,
+ "meanToolInvocationCount": 19
+ },
+ {
+ "experimentId": "claude-opus-5-claude-code--baseline",
+ "caseId": "author-004-packaged-skill",
+ "status": "current",
+ "passedRuns": 3,
+ "validRuns": 3,
+ "meanDurationMs": 181217,
+ "measuredAt": "2026-09-02T14:35:36.656Z",
+ "meanEstimatedListCostUsd": 1.1116483333333333,
+ "meanTokenConsumption": 16371.666666666666,
+ "meanToolInvocationCount": 12.666666666666666
+ },
+ {
+ "experimentId": "claude-opus-5-claude-code--baseline",
+ "caseId": "author-005-conditional-approval",
+ "status": "current",
+ "passedRuns": 3,
+ "validRuns": 3,
+ "meanDurationMs": 153333,
+ "measuredAt": "2026-09-02T14:35:36.667Z",
+ "meanEstimatedListCostUsd": 1.03157675,
+ "meanTokenConsumption": 6217.333333333333,
+ "meanToolInvocationCount": 17.333333333333332
+ },
+ {
+ "experimentId": "claude-opus-5-claude-code--baseline",
+ "caseId": "author-006-custom-channel",
+ "status": "current",
+ "passedRuns": 3,
+ "validRuns": 3,
+ "meanDurationMs": 136155,
+ "measuredAt": "2026-09-02T14:35:36.676Z",
+ "meanEstimatedListCostUsd": 0.9316169166666667,
+ "meanTokenConsumption": 4836.666666666667,
+ "meanToolInvocationCount": 16.666666666666668
+ },
+ {
+ "experimentId": "claude-opus-5-claude-code--baseline",
+ "caseId": "author-007-digest-schedule",
+ "status": "current",
+ "passedRuns": 3,
+ "validRuns": 3,
+ "meanDurationMs": 120633,
+ "measuredAt": "2026-09-02T14:35:36.685Z",
+ "meanEstimatedListCostUsd": 0.5067333333333334,
+ "meanTokenConsumption": 3232.6666666666665,
+ "meanToolInvocationCount": 9.333333333333334
+ },
+ {
+ "experimentId": "claude-opus-5-claude-code--guided",
+ "caseId": "author-001-weather-tool",
+ "status": "current",
+ "passedRuns": 3,
+ "validRuns": 3,
+ "meanDurationMs": 147978,
+ "measuredAt": "2026-09-02T14:34:53.900Z",
+ "meanEstimatedListCostUsd": 1.0020195,
+ "meanTokenConsumption": 11004.333333333334,
+ "meanToolInvocationCount": 11
+ },
+ {
+ "experimentId": "claude-opus-5-claude-code--guided",
+ "caseId": "author-002-new-project",
+ "status": "current",
+ "passedRuns": 3,
+ "validRuns": 3,
+ "meanDurationMs": 138714,
+ "measuredAt": "2026-09-02T14:34:53.929Z",
+ "meanEstimatedListCostUsd": 1.0472271666666666,
+ "meanTokenConsumption": 8633.666666666666,
+ "meanToolInvocationCount": 14.333333333333334
+ },
+ {
+ "experimentId": "claude-opus-5-claude-code--guided",
+ "caseId": "author-003-openapi-connection",
+ "status": "current",
+ "passedRuns": 3,
+ "validRuns": 3,
+ "meanDurationMs": 137520,
+ "measuredAt": "2026-09-02T14:34:53.944Z",
+ "meanEstimatedListCostUsd": 0.7869943333333334,
+ "meanTokenConsumption": 5193.666666666667,
+ "meanToolInvocationCount": 14
+ },
+ {
+ "experimentId": "claude-opus-5-claude-code--guided",
+ "caseId": "author-004-packaged-skill",
+ "status": "current",
+ "passedRuns": 3,
+ "validRuns": 3,
+ "meanDurationMs": 153316,
+ "measuredAt": "2026-09-02T14:34:53.956Z",
+ "meanEstimatedListCostUsd": 0.6677339166666667,
+ "meanTokenConsumption": 10291.666666666666,
+ "meanToolInvocationCount": 7.666666666666667
+ },
+ {
+ "experimentId": "claude-opus-5-claude-code--guided",
+ "caseId": "author-005-conditional-approval",
+ "status": "current",
+ "passedRuns": 3,
+ "validRuns": 3,
+ "meanDurationMs": 116393,
+ "measuredAt": "2026-09-02T14:34:53.964Z",
+ "meanEstimatedListCostUsd": 0.5910709166666667,
+ "meanTokenConsumption": 2916.6666666666665,
+ "meanToolInvocationCount": 8.333333333333334
+ },
+ {
+ "experimentId": "claude-opus-5-claude-code--guided",
+ "caseId": "author-006-custom-channel",
+ "status": "current",
+ "passedRuns": 3,
+ "validRuns": 3,
+ "meanDurationMs": 107570,
+ "measuredAt": "2026-09-02T14:34:53.973Z",
+ "meanEstimatedListCostUsd": 0.4942074166666666,
+ "meanTokenConsumption": 2636,
+ "meanToolInvocationCount": 6.666666666666667
+ },
+ {
+ "experimentId": "claude-opus-5-claude-code--guided",
+ "caseId": "author-007-digest-schedule",
+ "status": "current",
+ "passedRuns": 3,
+ "validRuns": 3,
+ "meanDurationMs": 114311,
+ "measuredAt": "2026-09-02T14:34:53.981Z",
+ "meanEstimatedListCostUsd": 0.46051358333333337,
+ "meanTokenConsumption": 2592.6666666666665,
+ "meanToolInvocationCount": 7.666666666666667
+ }
+ ]
+ },
+ {
+ "suite": {
+ "eveRevision": "279e2b6a81198f6008004f596474cf05140ca7bc",
+ "caseFingerprint": "336fa6e2645a717a4bf4d906c3ae1b9f4ce0839bfb7bc9065068490dd696d42e",
+ "caseCount": 7,
+ "runsPerCell": 3
+ },
+ "experiments": [
+ {
+ "id": "glm-5-2-opencode--baseline",
+ "groupId": "glm-5-2-opencode",
+ "model": "zai/glm-5.2",
+ "modelDisplayName": "GLM 5.2",
+ "harness": "OpenCode",
+ "treatment": "baseline"
+ },
+ {
+ "id": "glm-5-2-opencode--guided",
+ "groupId": "glm-5-2-opencode",
+ "model": "zai/glm-5.2",
+ "modelDisplayName": "GLM 5.2",
+ "harness": "OpenCode",
+ "treatment": "guided"
+ }
+ ],
+ "results": [
+ {
+ "experimentId": "glm-5-2-opencode--baseline",
+ "caseId": "author-001-weather-tool",
+ "status": "current",
+ "passedRuns": 0,
+ "validRuns": 3,
+ "meanDurationMs": 183955,
+ "measuredAt": "2026-10-07T20:53:18.454Z",
+ "meanEstimatedListCostUsd": 0.08985164,
+ "meanTokenConsumption": 33383.666666666664,
+ "meanToolInvocationCount": 23.666666666666668
+ },
+ {
+ "experimentId": "glm-5-2-opencode--baseline",
+ "caseId": "author-002-new-project",
+ "status": "current",
+ "passedRuns": 2,
+ "validRuns": 3,
+ "meanDurationMs": 146900,
+ "measuredAt": "2026-10-07T20:53:18.471Z",
+ "meanEstimatedListCostUsd": 0.06131117333333334,
+ "meanTokenConsumption": 26307.666666666668,
+ "meanToolInvocationCount": 16.666666666666668
+ },
+ {
+ "experimentId": "glm-5-2-opencode--baseline",
+ "caseId": "author-003-openapi-connection",
+ "status": "current",
+ "passedRuns": 2,
+ "validRuns": 3,
+ "meanDurationMs": 263531,
+ "measuredAt": "2026-10-07T20:53:18.486Z",
+ "meanEstimatedListCostUsd": 0.24344989333333333,
+ "meanTokenConsumption": 59370.666666666664,
+ "meanToolInvocationCount": 41.333333333333336
},
{
"experimentId": "glm-5-2-opencode--baseline",
@@ -2852,35 +3612,35 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 214119,
- "measuredAt": "2026-08-28T19:22:09.000Z",
- "meanEstimatedListCostUsd": 0.07759993333333333,
- "meanTokenConsumption": 41325.666666666664,
- "meanToolInvocationCount": 36.333333333333336
+ "meanDurationMs": 215844,
+ "measuredAt": "2026-10-07T20:53:18.509Z",
+ "meanEstimatedListCostUsd": 0.10365373333333334,
+ "meanTokenConsumption": 34354,
+ "meanToolInvocationCount": 27
},
{
"experimentId": "glm-5-2-opencode--baseline",
"caseId": "author-005-conditional-approval",
"status": "current",
- "passedRuns": 3,
+ "passedRuns": 1,
"validRuns": 3,
- "meanDurationMs": 58527,
- "measuredAt": "2026-08-28T19:22:09.000Z",
- "meanEstimatedListCostUsd": 0.07383253333333333,
- "meanTokenConsumption": 48329.666666666664,
- "meanToolInvocationCount": 25.333333333333332
+ "meanDurationMs": 203192,
+ "measuredAt": "2026-10-07T20:53:18.528Z",
+ "meanEstimatedListCostUsd": 0.19839237333333334,
+ "meanTokenConsumption": 75866.66666666667,
+ "meanToolInvocationCount": 32.666666666666664
},
{
"experimentId": "glm-5-2-opencode--baseline",
"caseId": "author-006-custom-channel",
"status": "current",
- "passedRuns": 3,
+ "passedRuns": 2,
"validRuns": 3,
- "meanDurationMs": 75109,
- "measuredAt": "2026-08-28T19:22:09.000Z",
- "meanEstimatedListCostUsd": 0.07836139999999998,
- "meanTokenConsumption": 48506,
- "meanToolInvocationCount": 27.666666666666668
+ "meanDurationMs": 240346,
+ "measuredAt": "2026-10-07T20:53:18.555Z",
+ "meanEstimatedListCostUsd": 0.3499838533333333,
+ "meanTokenConsumption": 96427,
+ "meanToolInvocationCount": 45.333333333333336
},
{
"experimentId": "glm-5-2-opencode--baseline",
@@ -2888,11 +3648,11 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 93384,
- "measuredAt": "2026-08-28T19:22:09.000Z",
- "meanEstimatedListCostUsd": 0.04282053333333333,
- "meanTokenConsumption": 24309.666666666668,
- "meanToolInvocationCount": 19
+ "meanDurationMs": 155457,
+ "measuredAt": "2026-10-07T20:53:18.586Z",
+ "meanEstimatedListCostUsd": 0.07361798666666665,
+ "meanTokenConsumption": 34515.333333333336,
+ "meanToolInvocationCount": 18.666666666666668
},
{
"experimentId": "glm-5-2-opencode--guided",
@@ -2900,35 +3660,35 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 68607,
- "measuredAt": "2026-08-28T19:22:01.000Z",
- "meanEstimatedListCostUsd": 0.04188693333333334,
- "meanTokenConsumption": 22360.666666666668,
- "meanToolInvocationCount": 16.333333333333332
+ "meanDurationMs": 211084,
+ "measuredAt": "2026-10-07T20:58:05.582Z",
+ "meanEstimatedListCostUsd": 0.12096889333333334,
+ "meanTokenConsumption": 30802.666666666668,
+ "meanToolInvocationCount": 21.666666666666668
},
{
"experimentId": "glm-5-2-opencode--guided",
"caseId": "author-002-new-project",
"status": "current",
- "passedRuns": 3,
+ "passedRuns": 1,
"validRuns": 3,
- "meanDurationMs": 90517,
- "measuredAt": "2026-08-28T19:25:29.000Z",
- "meanEstimatedListCostUsd": 0.02843253333333333,
- "meanTokenConsumption": 16824.666666666668,
- "meanToolInvocationCount": 14.333333333333334
+ "meanDurationMs": 146105,
+ "measuredAt": "2026-10-07T20:58:05.598Z",
+ "meanEstimatedListCostUsd": 0.045339653333333334,
+ "meanTokenConsumption": 14588.666666666666,
+ "meanToolInvocationCount": 16.333333333333332
},
{
"experimentId": "glm-5-2-opencode--guided",
"caseId": "author-003-openapi-connection",
"status": "current",
- "passedRuns": 0,
+ "passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 524221,
- "measuredAt": "2026-08-28T19:22:01.000Z",
- "meanEstimatedListCostUsd": 0.1800297333333333,
- "meanTokenConsumption": 78267.66666666667,
- "meanToolInvocationCount": 24.666666666666668
+ "meanDurationMs": 159353,
+ "measuredAt": "2026-10-07T20:58:05.607Z",
+ "meanEstimatedListCostUsd": 0.07640521333333335,
+ "meanTokenConsumption": 20298.666666666668,
+ "meanToolInvocationCount": 19.333333333333332
},
{
"experimentId": "glm-5-2-opencode--guided",
@@ -2936,23 +3696,23 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 68821,
- "measuredAt": "2026-08-28T19:22:01.000Z",
- "meanEstimatedListCostUsd": 0.031183066666666665,
- "meanTokenConsumption": 16504.333333333332,
- "meanToolInvocationCount": 15
+ "meanDurationMs": 190722,
+ "measuredAt": "2026-10-07T20:58:05.620Z",
+ "meanEstimatedListCostUsd": 0.06202376000000001,
+ "meanTokenConsumption": 15365.666666666666,
+ "meanToolInvocationCount": 17.666666666666668
},
{
"experimentId": "glm-5-2-opencode--guided",
"caseId": "author-005-conditional-approval",
"status": "current",
- "passedRuns": 2,
+ "passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 80061,
- "measuredAt": "2026-08-28T19:22:01.000Z",
- "meanEstimatedListCostUsd": 0.040431333333333326,
- "meanTokenConsumption": 26306.666666666668,
- "meanToolInvocationCount": 14.333333333333334
+ "meanDurationMs": 155636,
+ "measuredAt": "2026-10-07T20:58:05.635Z",
+ "meanEstimatedListCostUsd": 0.049076973333333336,
+ "meanTokenConsumption": 18203,
+ "meanToolInvocationCount": 14.666666666666666
},
{
"experimentId": "glm-5-2-opencode--guided",
@@ -2960,11 +3720,11 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 56079,
- "measuredAt": "2026-08-28T19:22:01.000Z",
- "meanEstimatedListCostUsd": 0.03310953333333334,
- "meanTokenConsumption": 19814.666666666668,
- "meanToolInvocationCount": 11
+ "meanDurationMs": 168719,
+ "measuredAt": "2026-10-07T20:58:05.650Z",
+ "meanEstimatedListCostUsd": 0.07198956000000001,
+ "meanTokenConsumption": 19972.666666666668,
+ "meanToolInvocationCount": 19
},
{
"experimentId": "glm-5-2-opencode--guided",
@@ -2972,11 +3732,11 @@
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 74552,
- "measuredAt": "2026-08-28T19:22:01.000Z",
- "meanEstimatedListCostUsd": 0.029523933333333335,
- "meanTokenConsumption": 16295.666666666666,
- "meanToolInvocationCount": 12.333333333333334
+ "meanDurationMs": 147630,
+ "measuredAt": "2026-10-07T20:58:05.665Z",
+ "meanEstimatedListCostUsd": 0.04161758666666667,
+ "meanTokenConsumption": 10529.333333333334,
+ "meanToolInvocationCount": 13.333333333333334
}
]
}