graff — a fast agentic coding harness in Zig: multi-provider, MCP, workflows, DGM evolution loop, TS/Python SDKs
Python
{
"model": "one gateway model (same id in every arm)",
"tasks": 21,
"reps": 3,
"arms": {
"graff": "graff (ADR 0231 build)",
"claude-code": "Claude Code",
"opencode2": "OpenCode 2",
"pi": "pi"
},
"summary": {
"all": {
"graff": {
"runs": 63,
"passed": 63,
"wall_s_per_task": 8.7,
"cached_input_share": 0.953,
"output_tokens_per_task": 573,
"model_calls_per_task": 3.5
},
"claude-code": {
"runs": 63,
"passed": 63,
"wall_s_per_task": 13.2,
"cached_input_share": 0.599,
"output_tokens_per_task": 926,
"model_calls_per_task": 3.5
},
"opencode2": {
"runs": 63,
"passed": 63,
"wall_s_per_task": 14.1,
"cached_input_share": 0.86,
"output_tokens_per_task": 978,
"model_calls_per_task": 4.3
},
"pi": {
"runs": 33,
"passed": 33,
"wall_s_per_task": 6.0,
"cached_input_share": 0.765,
"output_tokens_per_task": 262,
"model_calls_per_task": 2.8
}
},
"core": {
"graff": {
"runs": 33,
"passed": 33,
"wall_s_per_task": 7.0,
"cached_input_share": 0.944,
"output_tokens_per_task": 293,
"model_calls_per_task": 2.8
},
"claude-code": {
"runs": 33,
"passed": 33,
"wall_s_per_task": 6.0,
"cached_input_share": 0.888,
"output_tokens_per_task": 272,
"model_calls_per_task": 2.6
},
"opencode2": {
"runs": 33,
"passed": 33,
"wall_s_per_task": 6.8,
"cached_input_share": 0.868,
"output_tokens_per_task": 363,
"model_calls_per_task": 3.2
},
"pi": {
"runs": 33,
"passed": 33,
"wall_s_per_task": 6.0,
"cached_input_share": 0.765,
"output_tokens_per_task": 262,
"model_calls_per_task": 2.8
}
},
"mcp": {
"graff": {
"runs": 30,
"passed": 30,
"wall_s_per_task": 10.6,
"cached_input_share": 0.96,
"output_tokens_per_task": 882,
"model_calls_per_task": 4.2
},
"claude-code": {
"runs": 30,
"passed": 30,
"wall_s_per_task": 21.2,
"cached_input_share": 0.493,
"output_tokens_per_task": 1645,
"model_calls_per_task": 4.5
},
"opencode2": {
"runs": 30,
"passed": 30,
"wall_s_per_task": 22.1,
"cached_input_share": 0.856,
"output_tokens_per_task": 1654,
"model_calls_per_task": 5.5
}
}
},
"per_task": {
"csv-sum": {
"suite": "core",
"graff": {
"passed": 3,
"runs": 3,
"wall_s": 8.4,
"model_calls": 3
},
"claude-code": {
"passed": 3,
"runs": 3,
"wall_s": 6.9,
"model_calls": 3
},
"opencode2": {
"passed": 3,
"runs": 3,
"wall_s": 5.9,
"model_calls": 3
},
"pi": {
"passed": 3,
"runs": 3,
"wall_s": 4.7,
"model_calls": 3
}
},
"dead-code": {
"suite": "core",
"graff": {
"passed": 3,
"runs": 3,
"wall_s": 5.3,
"model_calls": 3
},
"claude-code": {
"passed": 3,
"runs": 3,
"wall_s": 6.0,
"model_calls": 3
},
"opencode2": {
"passed": 3,
"runs": 3,
"wall_s": 6.8,
"model_calls": 4
},
"pi": {
"passed": 3,
"runs": 3,
"wall_s": 7.8,
"model_calls": 3
}
},
"exact-reply": {
"suite": "core",
"graff": {
"passed": 3,
"runs": 3,
"wall_s": 6.2,
"model_calls": 1
},
"claude-code": {
"passed": 3,
"runs": 3,
"wall_s": 1.7,
"model_calls": 1
},
"opencode2": {
"passed": 3,
"runs": 3,
"wall_s": 1.7,
"model_calls": 1
},
"pi": {
"passed": 3,
"runs": 3,
"wall_s": 1.9,
"model_calls": 1
}
},
"file-ops": {
"suite": "core",
"graff": {
"passed": 3,
"runs": 3,
"wall_s": 6.8,
"model_calls": 2
},
"claude-code": {
"passed": 3,
"runs": 3,
"wall_s": 6.0,
"model_calls": 2.7
},
"opencode2": {
"passed": 3,
"runs": 3,
"wall_s": 5.8,
"model_calls": 3
},
"pi": {
"passed": 3,
"runs": 3,
"wall_s": 3.5,
"model_calls": 2
}
},
"fix-fib": {
"suite": "core",
"graff": {
"passed": 3,
"runs": 3,
"wall_s": 8.7,
"model_calls": 4
},
"claude-code": {
"passed": 3,
"runs": 3,
"wall_s": 6.4,
"model_calls": 3
},
"opencode2": {
"passed": 3,
"runs": 3,
"wall_s": 7.9,
"model_calls": 4
},
"pi": {
"passed": 3,
"runs": 3,
"wall_s": 12.0,
"model_calls": 4
}
},
"git-ops": {
"suite": "core",
"graff": {
"passed": 3,
"runs": 3,
"wall_s": 4.1,
"model_calls": 2
},
"claude-code": {
"passed": 3,
"runs": 3,
"wall_s": 4.6,
"model_calls": 2
},
"opencode2": {
"passed": 3,
"runs": 3,
"wall_s": 6.3,
"model_calls": 3
},
"pi": {
"passed": 3,
"runs": 3,
"wall_s": 7.1,
"model_calls": 2.3
}
},
"json-transform": {
"suite": "core",
"graff": {
"passed": 3,
"runs": 3,
"wall_s": 5.8,
"model_calls": 3
},
"claude-code": {
"passed": 3,
"runs": 3,
"wall_s": 6.7,
"model_calls": 3
},
"opencode2": {
"passed": 3,
"runs": 3,
"wall_s": 6.3,
"model_calls": 3
},
"pi": {
"passed": 3,
"runs": 3,
"wall_s": 5.0,
"model_calls": 3
}
},
"linear-issues": {
"suite": "mcp",
"graff": {
"passed": 3,
"runs": 3,
"wall_s": 8.8,
"model_calls": 3.3
},
"claude-code": {
"passed": 3,
"runs": 3,
"wall_s": 18.6,
"model_calls": 3.7
},
"opencode2": {
"passed": 3,
"runs": 3,
"wall_s": 22.9,
"model_calls": 5.7
}
},
"linear-nohint": {
"suite": "mcp",
"graff": {
"passed": 3,
"runs": 3,
"wall_s": 11.1,
"model_calls": 4
},
"claude-code": {
"passed": 3,
"runs": 3,
"wall_s": 20.1,
"model_calls": 4
},
"opencode2": {
"passed": 3,
"runs": 3,
"wall_s": 18.7,
"model_calls": 5.7
}
},
"linear-nohint-warm": {
"suite": "mcp",
"graff": {
"passed": 3,
"runs": 3,
"wall_s": 10.2,
"model_calls": 3.3
},
"claude-code": {
"passed": 3,
"runs": 3,
"wall_s": 19.3,
"model_calls": 4
},
"opencode2": {
"passed": 3,
"runs": 3,
"wall_s": 23.0,
"model_calls": 6
}
},
"linear-quiet": {
"suite": "mcp",
"graff": {
"passed": 3,
"runs": 3,
"wall_s": 9.2,
"model_calls": 4
},
"claude-code": {
"passed": 3,
"runs": 3,
"wall_s": 28.0,
"model_calls": 5
},
"opencode2": {
"passed": 3,
"runs": 3,
"wall_s": 21.2,
"model_calls": 5.7
}
},
"linear-quiet-warm": {
"suite": "mcp",
"graff": {
"passed": 3,
"runs": 3,
"wall_s": 9.8,
"model_calls": 4
},
"claude-code": {
"passed": 3,
"runs": 3,
"wall_s": 19.3,
"model_calls": 4.3
},
"opencode2": {
"passed": 3,
"runs": 3,
"wall_s": 21.4,
"model_calls": 5.3
}
},
"linear-reduce": {
"suite": "mcp",
"graff": {
"passed": 3,
"runs": 3,
"wall_s": 11.5,
"model_calls": 4
},
"claude-code": {
"passed": 3,
"runs": 3,
"wall_s": 21.7,
"model_calls": 4
},
"opencode2": {
"passed": 3,
"runs": 3,
"wall_s": 22.8,
"model_calls": 6
}
},
"linear-reduce-warm": {
"suite": "mcp",
"graff": {
"passed": 3,
"runs": 3,
"wall_s": 10.1,
"model_calls": 4
},
"claude-code": {
"passed": 3,
"runs": 3,
"wall_s": 23.5,
"model_calls": 4.7
},
"opencode2": {
"passed": 3,
"runs": 3,
"wall_s": 19.2,
"model_calls": 5
}
},
"linear-sidecar": {
"suite": "mcp",
"graff": {
"passed": 3,
"runs": 3,
"wall_s": 11.2,
"model_calls": 4
},
"claude-code": {
"passed": 3,
"runs": 3,
"wall_s": 21.5,
"model_calls": 4
},
"opencode2": {
"passed": 3,
"runs": 3,
"wall_s": 22.3,
"model_calls": 6
}
},
"linear-split": {
"suite": "mcp",
"graff": {
"passed": 3,
"runs": 3,
"wall_s": 15.6,
"model_calls": 7.3
},
"claude-code": {
"passed": 3,
"runs": 3,
"wall_s": 20.3,
"model_calls": 7
},
"opencode2": {
"passed": 3,
"runs": 3,
"wall_s": 26.5,
"model_calls": 3
}
},
"linear-warm": {
"suite": "mcp",
"graff": {
"passed": 3,
"runs": 3,
"wall_s": 8.8,
"model_calls": 4
},
"claude-code": {
"passed": 3,
"runs": 3,
"wall_s": 19.8,
"model_calls": 4
},
"opencode2": {
"passed": 3,
"runs": 3,
"wall_s": 23.4,
"model_calls": 6.3
}
},
"recall-noise": {
"suite": "core",
"graff": {
"passed": 3,
"runs": 3,
"wall_s": 3.5,
"model_calls": 2
},
"claude-code": {
"passed": 3,
"runs": 3,
"wall_s": 3.7,
"model_calls": 2
},
"opencode2": {
"passed": 3,
"runs": 3,
"wall_s": 5.7,
"model_calls": 3
},
"pi": {
"passed": 3,
"runs": 3,
"wall_s": 3.5,
"model_calls": 2
}
},
"refactor-rename": {
"suite": "core",
"graff": {
"passed": 3,
"runs": 3,
"wall_s": 10.7,
"model_calls": 5
},
"claude-code": {
"passed": 3,
"runs": 3,
"wall_s": 6.5,
"model_calls": 3
},
"opencode2": {
"passed": 3,
"runs": 3,
"wall_s": 10.6,
"model_calls": 4
},
"pi": {
"passed": 3,
"runs": 3,
"wall_s": 7.9,
"model_calls": 4
}
},
"regex-count": {
"suite": "core",
"graff": {
"passed": 3,
"runs": 3,
"wall_s": 10.5,
"model_calls": 3
},
"claude-code": {
"passed": 3,
"runs": 3,
"wall_s": 7.8,
"model_calls": 3.3
},
"opencode2": {
"passed": 3,
"runs": 3,
"wall_s": 8.7,
"model_calls": 3.3
},
"pi": {
"passed": 3,
"runs": 3,
"wall_s": 6.6,
"model_calls": 3.3
}
},
"write-tests": {
"suite": "core",
"graff": {
"passed": 3,
"runs": 3,
"wall_s": 7.0,
"model_calls": 3
},
"claude-code": {
"passed": 3,
"runs": 3,
"wall_s": 9.3,
"model_calls": 3
},
"opencode2": {
"passed": 3,
"runs": 3,
"wall_s": 9.2,
"model_calls": 4
},
"pi": {
"passed": 3,
"runs": 3,
"wall_s": 6.4,
"model_calls": 3
}
}
},
"runs": [
{
"arm": "claude-code",
"task": "csv-sum",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 6.73,
"model_calls": 3,
"input_tokens": 67670,
"cached_input_tokens": 60999,
"output_tokens": 230
},
{
"arm": "claude-code",
"task": "csv-sum",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 7.09,
"model_calls": 3,
"input_tokens": 67671,
"cached_input_tokens": 60999,
"output_tokens": 236
},
{
"arm": "claude-code",
"task": "csv-sum",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 7.02,
"model_calls": 3,
"input_tokens": 67663,
"cached_input_tokens": 60999,
"output_tokens": 227
},
{
"arm": "claude-code",
"task": "dead-code",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 7.32,
"model_calls": 3,
"input_tokens": 67857,
"cached_input_tokens": 60993,
"output_tokens": 176
},
{
"arm": "claude-code",
"task": "dead-code",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 5.27,
"model_calls": 3,
"input_tokens": 67851,
"cached_input_tokens": 60993,
"output_tokens": 170
},
{
"arm": "claude-code",
"task": "dead-code",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 5.43,
"model_calls": 3,
"input_tokens": 67980,
"cached_input_tokens": 60993,
"output_tokens": 213
},
{
"arm": "claude-code",
"task": "exact-reply",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 1.79,
"model_calls": 1,
"input_tokens": 22408,
"cached_input_tokens": 17105,
"output_tokens": 4
},
{
"arm": "claude-code",
"task": "exact-reply",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 1.81,
"model_calls": 1,
"input_tokens": 22409,
"cached_input_tokens": 17105,
"output_tokens": 4
},
{
"arm": "claude-code",
"task": "exact-reply",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 1.62,
"model_calls": 1,
"input_tokens": 22408,
"cached_input_tokens": 17105,
"output_tokens": 4
},
{
"arm": "claude-code",
"task": "file-ops",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 7.44,
"model_calls": 3,
"input_tokens": 67824,
"cached_input_tokens": 60993,
"output_tokens": 278
},
{
"arm": "claude-code",
"task": "file-ops",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 6.18,
"model_calls": 3,
"input_tokens": 67818,
"cached_input_tokens": 60993,
"output_tokens": 277
},
{
"arm": "claude-code",
"task": "file-ops",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 4.45,
"model_calls": 2,
"input_tokens": 45024,
"cached_input_tokens": 39038,
"output_tokens": 227
},
{
"arm": "claude-code",
"task": "fix-fib",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 6.29,
"model_calls": 3,
"input_tokens": 68239,
"cached_input_tokens": 60999,
"output_tokens": 371
},
{
"arm": "claude-code",
"task": "fix-fib",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 6.64,
"model_calls": 3,
"input_tokens": 68239,
"cached_input_tokens": 60999,
"output_tokens": 372
},
{
"arm": "claude-code",
"task": "fix-fib",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 6.35,
"model_calls": 3,
"input_tokens": 68233,
"cached_input_tokens": 60999,
"output_tokens": 371
},
{
"arm": "claude-code",
"task": "git-ops",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 4.69,
"model_calls": 2,
"input_tokens": 45117,
"cached_input_tokens": 39038,
"output_tokens": 226
},
{
"arm": "claude-code",
"task": "git-ops",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 4.95,
"model_calls": 2,
"input_tokens": 45098,
"cached_input_tokens": 39038,
"output_tokens": 276
},
{
"arm": "claude-code",
"task": "git-ops",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 4.16,
"model_calls": 2,
"input_tokens": 45100,
"cached_input_tokens": 39038,
"output_tokens": 239
},
{
"arm": "claude-code",
"task": "json-transform",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 6.31,
"model_calls": 3,
"input_tokens": 67944,
"cached_input_tokens": 60993,
"output_tokens": 339
},
{
"arm": "claude-code",
"task": "json-transform",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 6.74,
"model_calls": 3,
"input_tokens": 67953,
"cached_input_tokens": 60993,
"output_tokens": 318
},
{
"arm": "claude-code",
"task": "json-transform",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 6.94,
"model_calls": 3,
"input_tokens": 67944,
"cached_input_tokens": 60993,
"output_tokens": 325
},
{
"arm": "claude-code",
"task": "linear-issues",
"suite": "mcp",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 16.82,
"model_calls": 4,
"input_tokens": 169851,
"cached_input_tokens": 83758,
"output_tokens": 1556
},
{
"arm": "claude-code",
"task": "linear-issues",
"suite": "mcp",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 16.87,
"model_calls": 3,
"input_tokens": 144910,
"cached_input_tokens": 61584,
"output_tokens": 1366
},
{
"arm": "claude-code",
"task": "linear-issues",
"suite": "mcp",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 22.25,
"model_calls": 4,
"input_tokens": 170065,
"cached_input_tokens": 83758,
"output_tokens": 1678
},
{
"arm": "claude-code",
"task": "linear-nohint",
"suite": "mcp",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 20.68,
"model_calls": 4,
"input_tokens": 169849,
"cached_input_tokens": 83767,
"output_tokens": 1674
},
{
"arm": "claude-code",
"task": "linear-nohint",
"suite": "mcp",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 18.8,
"model_calls": 4,
"input_tokens": 169408,
"cached_input_tokens": 83767,
"output_tokens": 1382
},
{
"arm": "claude-code",
"task": "linear-nohint",
"suite": "mcp",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 20.79,
"model_calls": 4,
"input_tokens": 205410,
"cached_input_tokens": 83767,
"output_tokens": 1417
},
{
"arm": "claude-code",
"task": "linear-nohint-warm",
"suite": "mcp",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 18.48,
"model_calls": 4,
"input_tokens": 169460,
"cached_input_tokens": 83794,
"output_tokens": 1503
},
{
"arm": "claude-code",
"task": "linear-nohint-warm",
"suite": "mcp",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 22.62,
"model_calls": 4,
"input_tokens": 206396,
"cached_input_tokens": 83794,
"output_tokens": 1824
},
{
"arm": "claude-code",
"task": "linear-nohint-warm",
"suite": "mcp",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 16.79,
"model_calls": 4,
"input_tokens": 205770,
"cached_input_tokens": 83794,
"output_tokens": 1473
},
{
"arm": "claude-code",
"task": "linear-quiet",
"suite": "mcp",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 21.49,
"model_calls": 4,
"input_tokens": 169392,
"cached_input_tokens": 83767,
"output_tokens": 1526
},
{
"arm": "claude-code",
"task": "linear-quiet",
"suite": "mcp",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 30.45,
"model_calls": 6,
"input_tokens": 293424,
"cached_input_tokens": 128184,
"output_tokens": 2361
},
{
"arm": "claude-code",
"task": "linear-quiet",
"suite": "mcp",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 32.17,
"model_calls": 5,
"input_tokens": 231177,
"cached_input_tokens": 105965,
"output_tokens": 1863
},
{
"arm": "claude-code",
"task": "linear-quiet-warm",
"suite": "mcp",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 24.64,
"model_calls": 5,
"input_tokens": 231129,
"cached_input_tokens": 106001,
"output_tokens": 1933
},
{
"arm": "claude-code",
"task": "linear-quiet-warm",
"suite": "mcp",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 15.69,
"model_calls": 4,
"input_tokens": 169413,
"cached_input_tokens": 83794,
"output_tokens": 1402
},
{
"arm": "claude-code",
"task": "linear-quiet-warm",
"suite": "mcp",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 17.51,
"model_calls": 4,
"input_tokens": 169464,
"cached_input_tokens": 83794,
"output_tokens": 1492
},
{
"arm": "claude-code",
"task": "linear-reduce",
"suite": "mcp",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 17.47,
"model_calls": 4,
"input_tokens": 170042,
"cached_input_tokens": 83767,
"output_tokens": 1456
},
{
"arm": "claude-code",
"task": "linear-reduce",
"suite": "mcp",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 24.0,
"model_calls": 4,
"input_tokens": 170234,
"cached_input_tokens": 83767,
"output_tokens": 1748
},
{
"arm": "claude-code",
"task": "linear-reduce",
"suite": "mcp",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 23.49,
"model_calls": 4,
"input_tokens": 169913,
"cached_input_tokens": 83767,
"output_tokens": 1555
},
{
"arm": "claude-code",
"task": "linear-reduce-warm",
"suite": "mcp",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 24.94,
"model_calls": 5,
"input_tokens": 232235,
"cached_input_tokens": 106001,
"output_tokens": 2258
},
{
"arm": "claude-code",
"task": "linear-reduce-warm",
"suite": "mcp",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 23.32,
"model_calls": 5,
"input_tokens": 231251,
"cached_input_tokens": 106001,
"output_tokens": 1915
},
{
"arm": "claude-code",
"task": "linear-reduce-warm",
"suite": "mcp",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 22.24,
"model_calls": 4,
"input_tokens": 170289,
"cached_input_tokens": 83794,
"output_tokens": 1792
},
{
"arm": "claude-code",
"task": "linear-sidecar",
"suite": "mcp",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 25.81,
"model_calls": 4,
"input_tokens": 170071,
"cached_input_tokens": 83776,
"output_tokens": 1629
},
{
"arm": "claude-code",
"task": "linear-sidecar",
"suite": "mcp",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 17.52,
"model_calls": 4,
"input_tokens": 169813,
"cached_input_tokens": 83776,
"output_tokens": 1443
},
{
"arm": "claude-code",
"task": "linear-sidecar",
"suite": "mcp",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 21.23,
"model_calls": 4,
"input_tokens": 170174,
"cached_input_tokens": 83776,
"output_tokens": 1679
},
{
"arm": "claude-code",
"task": "linear-split",
"suite": "mcp",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 20.53,
"model_calls": 6,
"input_tokens": 100734,
"cached_input_tokens": 83749,
"output_tokens": 1528
},
{
"arm": "claude-code",
"task": "linear-split",
"suite": "mcp",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 18.18,
"model_calls": 9,
"input_tokens": 131230,
"cached_input_tokens": 107157,
"output_tokens": 1627
},
{
"arm": "claude-code",
"task": "linear-split",
"suite": "mcp",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 22.33,
"model_calls": 6,
"input_tokens": 100546,
"cached_input_tokens": 83749,
"output_tokens": 1494
},
{
"arm": "claude-code",
"task": "linear-warm",
"suite": "mcp",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 18.95,
"model_calls": 4,
"input_tokens": 169963,
"cached_input_tokens": 83758,
"output_tokens": 1534
},
{
"arm": "claude-code",
"task": "linear-warm",
"suite": "mcp",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 18.16,
"model_calls": 4,
"input_tokens": 169806,
"cached_input_tokens": 83758,
"output_tokens": 1394
},
{
"arm": "claude-code",
"task": "linear-warm",
"suite": "mcp",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 22.18,
"model_calls": 4,
"input_tokens": 170101,
"cached_input_tokens": 83758,
"output_tokens": 1847
},
{
"arm": "claude-code",
"task": "recall-noise",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 3.76,
"model_calls": 2,
"input_tokens": 45330,
"cached_input_tokens": 39041,
"output_tokens": 104
},
{
"arm": "claude-code",
"task": "recall-noise",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 3.15,
"model_calls": 2,
"input_tokens": 45330,
"cached_input_tokens": 39041,
"output_tokens": 104
},
{
"arm": "claude-code",
"task": "recall-noise",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 4.25,
"model_calls": 2,
"input_tokens": 45328,
"cached_input_tokens": 39041,
"output_tokens": 104
},
{
"arm": "claude-code",
"task": "refactor-rename",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 6.14,
"model_calls": 3,
"input_tokens": 67833,
"cached_input_tokens": 61005,
"output_tokens": 254
},
{
"arm": "claude-code",
"task": "refactor-rename",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 6.98,
"model_calls": 3,
"input_tokens": 67827,
"cached_input_tokens": 61005,
"output_tokens": 248
},
{
"arm": "claude-code",
"task": "refactor-rename",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 6.3,
"model_calls": 3,
"input_tokens": 67827,
"cached_input_tokens": 61005,
"output_tokens": 251
},
{
"arm": "claude-code",
"task": "regex-count",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 7.11,
"model_calls": 3,
"input_tokens": 67999,
"cached_input_tokens": 60993,
"output_tokens": 399
},
{
"arm": "claude-code",
"task": "regex-count",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 10.21,
"model_calls": 4,
"input_tokens": 90726,
"cached_input_tokens": 82969,
"output_tokens": 396
},
{
"arm": "claude-code",
"task": "regex-count",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 6.19,
"model_calls": 3,
"input_tokens": 67999,
"cached_input_tokens": 60993,
"output_tokens": 358
},
{
"arm": "claude-code",
"task": "write-tests",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 8.28,
"model_calls": 3,
"input_tokens": 68157,
"cached_input_tokens": 60993,
"output_tokens": 636
},
{
"arm": "claude-code",
"task": "write-tests",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 7.78,
"model_calls": 3,
"input_tokens": 68166,
"cached_input_tokens": 60993,
"output_tokens": 618
},
{
"arm": "claude-code",
"task": "write-tests",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 11.88,
"model_calls": 3,
"input_tokens": 68160,
"cached_input_tokens": 60993,
"output_tokens": 637
},
{
"arm": "graff",
"task": "csv-sum",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 9.05,
"model_calls": 3,
"input_tokens": 27862,
"cached_input_tokens": 22546,
"output_tokens": 393
},
{
"arm": "graff",
"task": "csv-sum",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 7.82,
"model_calls": 3,
"input_tokens": 27839,
"cached_input_tokens": 27756,
"output_tokens": 303
},
{
"arm": "graff",
"task": "csv-sum",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 8.27,
"model_calls": 3,
"input_tokens": 27927,
"cached_input_tokens": 27756,
"output_tokens": 341
},
{
"arm": "graff",
"task": "dead-code",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 5.31,
"model_calls": 3,
"input_tokens": 28218,
"cached_input_tokens": 22632,
"output_tokens": 162
},
{
"arm": "graff",
"task": "dead-code",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 4.86,
"model_calls": 3,
"input_tokens": 28228,
"cached_input_tokens": 27748,
"output_tokens": 165
},
{
"arm": "graff",
"task": "dead-code",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 5.6,
"model_calls": 3,
"input_tokens": 28218,
"cached_input_tokens": 28210,
"output_tokens": 160
},
{
"arm": "graff",
"task": "exact-reply",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 12.27,
"model_calls": 1,
"input_tokens": 9121,
"cached_input_tokens": 9117,
"output_tokens": 51
},
{
"arm": "graff",
"task": "exact-reply",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 1.74,
"model_calls": 1,
"input_tokens": 9121,
"cached_input_tokens": 9117,
"output_tokens": 51
},
{
"arm": "graff",
"task": "exact-reply",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 4.73,
"model_calls": 1,
"input_tokens": 9121,
"cached_input_tokens": 9117,
"output_tokens": 51
},
{
"arm": "graff",
"task": "file-ops",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 3.5,
"model_calls": 2,
"input_tokens": 18395,
"cached_input_tokens": 18186,
"output_tokens": 162
},
{
"arm": "graff",
"task": "file-ops",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 3.2,
"model_calls": 2,
"input_tokens": 18395,
"cached_input_tokens": 18389,
"output_tokens": 145
},
{
"arm": "graff",
"task": "file-ops",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 13.55,
"model_calls": 2,
"input_tokens": 18395,
"cached_input_tokens": 18389,
"output_tokens": 148
},
{
"arm": "graff",
"task": "fix-fib",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 8.33,
"model_calls": 4,
"input_tokens": 38157,
"cached_input_tokens": 32356,
"output_tokens": 386
},
{
"arm": "graff",
"task": "fix-fib",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 7.79,
"model_calls": 4,
"input_tokens": 38157,
"cached_input_tokens": 37477,
"output_tokens": 388
},
{
"arm": "graff",
"task": "fix-fib",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 9.91,
"model_calls": 4,
"input_tokens": 38157,
"cached_input_tokens": 37477,
"output_tokens": 388
},
{
"arm": "graff",
"task": "git-ops",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 4.47,
"model_calls": 2,
"input_tokens": 18597,
"cached_input_tokens": 13279,
"output_tokens": 311
},
{
"arm": "graff",
"task": "git-ops",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 4.13,
"model_calls": 2,
"input_tokens": 18594,
"cached_input_tokens": 18398,
"output_tokens": 266
},
{
"arm": "graff",
"task": "git-ops",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 3.71,
"model_calls": 2,
"input_tokens": 18590,
"cached_input_tokens": 18398,
"output_tokens": 240
},
{
"arm": "graff",
"task": "json-transform",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 6.04,
"model_calls": 3,
"input_tokens": 27943,
"cached_input_tokens": 27935,
"output_tokens": 240
},
{
"arm": "graff",
"task": "json-transform",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 5.62,
"model_calls": 3,
"input_tokens": 27943,
"cached_input_tokens": 27935,
"output_tokens": 273
},
{
"arm": "graff",
"task": "json-transform",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 5.63,
"model_calls": 3,
"input_tokens": 27943,
"cached_input_tokens": 27935,
"output_tokens": 252
},
{
"arm": "graff",
"task": "linear-issues",
"suite": "mcp",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 8.63,
"model_calls": 3,
"input_tokens": 30825,
"cached_input_tokens": 29264,
"output_tokens": 500
},
{
"arm": "graff",
"task": "linear-issues",
"suite": "mcp",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 10.04,
"model_calls": 4,
"input_tokens": 41510,
"cached_input_tokens": 40531,
"output_tokens": 723
},
{
"arm": "graff",
"task": "linear-issues",
"suite": "mcp",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 7.73,
"model_calls": 3,
"input_tokens": 30744,
"cached_input_tokens": 30536,
"output_tokens": 637
},
{
"arm": "graff",
"task": "linear-nohint",
"suite": "mcp",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 10.56,
"model_calls": 4,
"input_tokens": 41221,
"cached_input_tokens": 39503,
"output_tokens": 856
},
{
"arm": "graff",
"task": "linear-nohint",
"suite": "mcp",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 11.37,
"model_calls": 4,
"input_tokens": 41244,
"cached_input_tokens": 40545,
"output_tokens": 868
},
{
"arm": "graff",
"task": "linear-nohint",
"suite": "mcp",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 11.3,
"model_calls": 4,
"input_tokens": 41221,
"cached_input_tokens": 40925,
"output_tokens": 846
},
{
"arm": "graff",
"task": "linear-nohint-warm",
"suite": "mcp",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 11.75,
"model_calls": 4,
"input_tokens": 42475,
"cached_input_tokens": 40513,
"output_tokens": 798
},
{
"arm": "graff",
"task": "linear-nohint-warm",
"suite": "mcp",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 10.19,
"model_calls": 3,
"input_tokens": 32072,
"cached_input_tokens": 30981,
"output_tokens": 959
},
{
"arm": "graff",
"task": "linear-nohint-warm",
"suite": "mcp",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 8.52,
"model_calls": 3,
"input_tokens": 31797,
"cached_input_tokens": 31503,
"output_tokens": 683
},
{
"arm": "graff",
"task": "linear-quiet",
"suite": "mcp",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 11.71,
"model_calls": 4,
"input_tokens": 41002,
"cached_input_tokens": 40808,
"output_tokens": 613
},
{
"arm": "graff",
"task": "linear-quiet",
"suite": "mcp",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 7.96,
"model_calls": 4,
"input_tokens": 41013,
"cached_input_tokens": 40808,
"output_tokens": 587
},
{
"arm": "graff",
"task": "linear-quiet",
"suite": "mcp",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 7.84,
"model_calls": 4,
"input_tokens": 41002,
"cached_input_tokens": 40299,
"output_tokens": 514
},
{
"arm": "graff",
"task": "linear-quiet-warm",
"suite": "mcp",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 9.61,
"model_calls": 4,
"input_tokens": 43108,
"cached_input_tokens": 40900,
"output_tokens": 744
},
{
"arm": "graff",
"task": "linear-quiet-warm",
"suite": "mcp",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 10.05,
"model_calls": 4,
"input_tokens": 43108,
"cached_input_tokens": 43098,
"output_tokens": 683
},
{
"arm": "graff",
"task": "linear-quiet-warm",
"suite": "mcp",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 9.86,
"model_calls": 4,
"input_tokens": 43108,
"cached_input_tokens": 43098,
"output_tokens": 645
},
{
"arm": "graff",
"task": "linear-reduce",
"suite": "mcp",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 10.76,
"model_calls": 4,
"input_tokens": 41266,
"cached_input_tokens": 39561,
"output_tokens": 765
},
{
"arm": "graff",
"task": "linear-reduce",
"suite": "mcp",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 11.79,
"model_calls": 4,
"input_tokens": 41308,
"cached_input_tokens": 40370,
"output_tokens": 869
},
{
"arm": "graff",
"task": "linear-reduce",
"suite": "mcp",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 11.84,
"model_calls": 4,
"input_tokens": 41229,
"cached_input_tokens": 40477,
"output_tokens": 701
},
{
"arm": "graff",
"task": "linear-reduce-warm",
"suite": "mcp",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 9.32,
"model_calls": 4,
"input_tokens": 42800,
"cached_input_tokens": 40737,
"output_tokens": 729
},
{
"arm": "graff",
"task": "linear-reduce-warm",
"suite": "mcp",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 12.21,
"model_calls": 4,
"input_tokens": 42819,
"cached_input_tokens": 41903,
"output_tokens": 751
},
{
"arm": "graff",
"task": "linear-reduce-warm",
"suite": "mcp",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 8.7,
"model_calls": 4,
"input_tokens": 42818,
"cached_input_tokens": 42613,
"output_tokens": 588
},
{
"arm": "graff",
"task": "linear-sidecar",
"suite": "mcp",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 14.15,
"model_calls": 5,
"input_tokens": 63150,
"cached_input_tokens": 58966,
"output_tokens": 1302
},
{
"arm": "graff",
"task": "linear-sidecar",
"suite": "mcp",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 10.74,
"model_calls": 4,
"input_tokens": 49882,
"cached_input_tokens": 48906,
"output_tokens": 1173
},
{
"arm": "graff",
"task": "linear-sidecar",
"suite": "mcp",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 8.66,
"model_calls": 3,
"input_tokens": 37282,
"cached_input_tokens": 36266,
"output_tokens": 833
},
{
"arm": "graff",
"task": "linear-split",
"suite": "mcp",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 15.27,
"model_calls": 7,
"input_tokens": 63761,
"cached_input_tokens": 46131,
"output_tokens": 1874
},
{
"arm": "graff",
"task": "linear-split",
"suite": "mcp",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 15.49,
"model_calls": 7,
"input_tokens": 63887,
"cached_input_tokens": 60554,
"output_tokens": 2014
},
{
"arm": "graff",
"task": "linear-split",
"suite": "mcp",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 15.96,
"model_calls": 8,
"input_tokens": 77784,
"cached_input_tokens": 73972,
"output_tokens": 2184
},
{
"arm": "graff",
"task": "linear-warm",
"suite": "mcp",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 9.36,
"model_calls": 4,
"input_tokens": 43475,
"cached_input_tokens": 41215,
"output_tokens": 741
},
{
"arm": "graff",
"task": "linear-warm",
"suite": "mcp",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 8.46,
"model_calls": 4,
"input_tokens": 43085,
"cached_input_tokens": 42111,
"output_tokens": 590
},
{
"arm": "graff",
"task": "linear-warm",
"suite": "mcp",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 8.55,
"model_calls": 4,
"input_tokens": 43098,
"cached_input_tokens": 42654,
"output_tokens": 684
},
{
"arm": "graff",
"task": "recall-noise",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 3.74,
"model_calls": 2,
"input_tokens": 18747,
"cached_input_tokens": 13253,
"output_tokens": 110
},
{
"arm": "graff",
"task": "recall-noise",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 3.27,
"model_calls": 2,
"input_tokens": 18747,
"cached_input_tokens": 18741,
"output_tokens": 110
},
{
"arm": "graff",
"task": "recall-noise",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 3.4,
"model_calls": 2,
"input_tokens": 18747,
"cached_input_tokens": 18741,
"output_tokens": 110
},
{
"arm": "graff",
"task": "refactor-rename",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 10.42,
"model_calls": 5,
"input_tokens": 48198,
"cached_input_tokens": 42190,
"output_tokens": 665
},
{
"arm": "graff",
"task": "refactor-rename",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 10.7,
"model_calls": 5,
"input_tokens": 48174,
"cached_input_tokens": 47635,
"output_tokens": 614
},
{
"arm": "graff",
"task": "refactor-rename",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 11.1,
"model_calls": 5,
"input_tokens": 48194,
"cached_input_tokens": 47316,
"output_tokens": 656
},
{
"arm": "graff",
"task": "regex-count",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 12.27,
"model_calls": 3,
"input_tokens": 28160,
"cached_input_tokens": 22709,
"output_tokens": 423
},
{
"arm": "graff",
"task": "regex-count",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 7.78,
"model_calls": 3,
"input_tokens": 28085,
"cached_input_tokens": 27811,
"output_tokens": 327
},
{
"arm": "graff",
"task": "regex-count",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 11.54,
"model_calls": 3,
"input_tokens": 28146,
"cached_input_tokens": 27829,
"output_tokens": 435
},
{
"arm": "graff",
"task": "write-tests",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 7.09,
"model_calls": 3,
"input_tokens": 28080,
"cached_input_tokens": 22556,
"output_tokens": 440
},
{
"arm": "graff",
"task": "write-tests",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 6.75,
"model_calls": 3,
"input_tokens": 28080,
"cached_input_tokens": 28072,
"output_tokens": 482
},
{
"arm": "graff",
"task": "write-tests",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 7.03,
"model_calls": 3,
"input_tokens": 28080,
"cached_input_tokens": 28072,
"output_tokens": 415
},
{
"arm": "opencode2",
"task": "csv-sum",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 6.36,
"model_calls": 3,
"input_tokens": 23865,
"cached_input_tokens": 20630,
"output_tokens": 282
},
{
"arm": "opencode2",
"task": "csv-sum",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 5.79,
"model_calls": 3,
"input_tokens": 23865,
"cached_input_tokens": 20671,
"output_tokens": 321
},
{
"arm": "opencode2",
"task": "csv-sum",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 5.48,
"model_calls": 3,
"input_tokens": 23865,
"cached_input_tokens": 20671,
"output_tokens": 324
},
{
"arm": "opencode2",
"task": "dead-code",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 6.29,
"model_calls": 4,
"input_tokens": 32305,
"cached_input_tokens": 28619,
"output_tokens": 289
},
{
"arm": "opencode2",
"task": "dead-code",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 7.09,
"model_calls": 4,
"input_tokens": 32305,
"cached_input_tokens": 28671,
"output_tokens": 289
},
{
"arm": "opencode2",
"task": "dead-code",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 7.03,
"model_calls": 4,
"input_tokens": 32301,
"cached_input_tokens": 28669,
"output_tokens": 287
},
{
"arm": "opencode2",
"task": "exact-reply",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 1.87,
"model_calls": 1,
"input_tokens": 7484,
"cached_input_tokens": 4644,
"output_tokens": 4
},
{
"arm": "opencode2",
"task": "exact-reply",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 1.57,
"model_calls": 1,
"input_tokens": 7484,
"cached_input_tokens": 4644,
"output_tokens": 4
},
{
"arm": "opencode2",
"task": "exact-reply",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 1.79,
"model_calls": 1,
"input_tokens": 7484,
"cached_input_tokens": 4644,
"output_tokens": 4
},
{
"arm": "opencode2",
"task": "file-ops",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 5.24,
"model_calls": 3,
"input_tokens": 23744,
"cached_input_tokens": 20568,
"output_tokens": 286
},
{
"arm": "opencode2",
"task": "file-ops",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 6.35,
"model_calls": 3,
"input_tokens": 23744,
"cached_input_tokens": 20614,
"output_tokens": 276
},
{
"arm": "opencode2",
"task": "file-ops",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 5.7,
"model_calls": 3,
"input_tokens": 23744,
"cached_input_tokens": 20614,
"output_tokens": 311
},
{
"arm": "opencode2",
"task": "fix-fib",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 8.52,
"model_calls": 4,
"input_tokens": 32294,
"cached_input_tokens": 28798,
"output_tokens": 355
},
{
"arm": "opencode2",
"task": "fix-fib",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 8.35,
"model_calls": 4,
"input_tokens": 32294,
"cached_input_tokens": 28857,
"output_tokens": 344
},
{
"arm": "opencode2",
"task": "fix-fib",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 6.76,
"model_calls": 4,
"input_tokens": 32294,
"cached_input_tokens": 28857,
"output_tokens": 358
},
{
"arm": "opencode2",
"task": "git-ops",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 6.21,
"model_calls": 3,
"input_tokens": 24080,
"cached_input_tokens": 20623,
"output_tokens": 357
},
{
"arm": "opencode2",
"task": "git-ops",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 5.95,
"model_calls": 3,
"input_tokens": 24075,
"cached_input_tokens": 20691,
"output_tokens": 340
},
{
"arm": "opencode2",
"task": "git-ops",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 6.76,
"model_calls": 3,
"input_tokens": 24084,
"cached_input_tokens": 20691,
"output_tokens": 332
},
{
"arm": "opencode2",
"task": "json-transform",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 6.34,
"model_calls": 3,
"input_tokens": 23966,
"cached_input_tokens": 20665,
"output_tokens": 383
},
{
"arm": "opencode2",
"task": "json-transform",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 5.81,
"model_calls": 3,
"input_tokens": 23966,
"cached_input_tokens": 20719,
"output_tokens": 344
},
{
"arm": "opencode2",
"task": "json-transform",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 6.66,
"model_calls": 3,
"input_tokens": 23966,
"cached_input_tokens": 20719,
"output_tokens": 384
},
{
"arm": "opencode2",
"task": "linear-issues",
"suite": "mcp",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 23.4,
"model_calls": 6,
"input_tokens": 65814,
"cached_input_tokens": 57075,
"output_tokens": 1818
},
{
"arm": "opencode2",
"task": "linear-issues",
"suite": "mcp",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 18.99,
"model_calls": 6,
"input_tokens": 65767,
"cached_input_tokens": 57326,
"output_tokens": 1745
},
{
"arm": "opencode2",
"task": "linear-issues",
"suite": "mcp",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 26.43,
"model_calls": 5,
"input_tokens": 53853,
"cached_input_tokens": 45358,
"output_tokens": 1783
},
{
"arm": "opencode2",
"task": "linear-nohint",
"suite": "mcp",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 19.09,
"model_calls": 5,
"input_tokens": 57856,
"cached_input_tokens": 46884,
"output_tokens": 1634
},
{
"arm": "opencode2",
"task": "linear-nohint",
"suite": "mcp",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 19.06,
"model_calls": 6,
"input_tokens": 65561,
"cached_input_tokens": 56903,
"output_tokens": 1742
},
{
"arm": "opencode2",
"task": "linear-nohint",
"suite": "mcp",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 18.06,
"model_calls": 6,
"input_tokens": 68449,
"cached_input_tokens": 58710,
"output_tokens": 1726
},
{
"arm": "opencode2",
"task": "linear-nohint-warm",
"suite": "mcp",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 30.95,
"model_calls": 6,
"input_tokens": 66382,
"cached_input_tokens": 57487,
"output_tokens": 1624
},
{
"arm": "opencode2",
"task": "linear-nohint-warm",
"suite": "mcp",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 19.19,
"model_calls": 6,
"input_tokens": 65088,
"cached_input_tokens": 56446,
"output_tokens": 1658
},
{
"arm": "opencode2",
"task": "linear-nohint-warm",
"suite": "mcp",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 18.98,
"model_calls": 6,
"input_tokens": 66345,
"cached_input_tokens": 57604,
"output_tokens": 1644
},
{
"arm": "opencode2",
"task": "linear-quiet",
"suite": "mcp",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 16.61,
"model_calls": 5,
"input_tokens": 45917,
"cached_input_tokens": 40113,
"output_tokens": 1443
},
{
"arm": "opencode2",
"task": "linear-quiet",
"suite": "mcp",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 22.39,
"model_calls": 6,
"input_tokens": 58273,
"cached_input_tokens": 51693,
"output_tokens": 1867
},
{
"arm": "opencode2",
"task": "linear-quiet",
"suite": "mcp",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 24.64,
"model_calls": 6,
"input_tokens": 57350,
"cached_input_tokens": 50964,
"output_tokens": 1849
},
{
"arm": "opencode2",
"task": "linear-quiet-warm",
"suite": "mcp",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 20.65,
"model_calls": 5,
"input_tokens": 47358,
"cached_input_tokens": 41341,
"output_tokens": 1761
},
{
"arm": "opencode2",
"task": "linear-quiet-warm",
"suite": "mcp",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 20.65,
"model_calls": 5,
"input_tokens": 47471,
"cached_input_tokens": 41447,
"output_tokens": 1974
},
{
"arm": "opencode2",
"task": "linear-quiet-warm",
"suite": "mcp",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 22.93,
"model_calls": 6,
"input_tokens": 58086,
"cached_input_tokens": 51508,
"output_tokens": 1783
},
{
"arm": "opencode2",
"task": "linear-reduce",
"suite": "mcp",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 20.34,
"model_calls": 6,
"input_tokens": 56123,
"cached_input_tokens": 49789,
"output_tokens": 1798
},
{
"arm": "opencode2",
"task": "linear-reduce",
"suite": "mcp",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 22.47,
"model_calls": 6,
"input_tokens": 59512,
"cached_input_tokens": 52199,
"output_tokens": 1901
},
{
"arm": "opencode2",
"task": "linear-reduce",
"suite": "mcp",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 25.54,
"model_calls": 6,
"input_tokens": 57388,
"cached_input_tokens": 50882,
"output_tokens": 1990
},
{
"arm": "opencode2",
"task": "linear-reduce-warm",
"suite": "mcp",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 22.16,
"model_calls": 6,
"input_tokens": 57297,
"cached_input_tokens": 51036,
"output_tokens": 1853
},
{
"arm": "opencode2",
"task": "linear-reduce-warm",
"suite": "mcp",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 16.99,
"model_calls": 4,
"input_tokens": 36639,
"cached_input_tokens": 31328,
"output_tokens": 1681
},
{
"arm": "opencode2",
"task": "linear-reduce-warm",
"suite": "mcp",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 18.47,
"model_calls": 5,
"input_tokens": 47319,
"cached_input_tokens": 41186,
"output_tokens": 1897
},
{
"arm": "opencode2",
"task": "linear-sidecar",
"suite": "mcp",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 22.03,
"model_calls": 6,
"input_tokens": 67258,
"cached_input_tokens": 57985,
"output_tokens": 1889
},
{
"arm": "opencode2",
"task": "linear-sidecar",
"suite": "mcp",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 22.89,
"model_calls": 6,
"input_tokens": 67495,
"cached_input_tokens": 58046,
"output_tokens": 1789
},
{
"arm": "opencode2",
"task": "linear-sidecar",
"suite": "mcp",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 21.86,
"model_calls": 6,
"input_tokens": 66217,
"cached_input_tokens": 57489,
"output_tokens": 1838
},
{
"arm": "opencode2",
"task": "linear-split",
"suite": "mcp",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 25.72,
"model_calls": 3,
"input_tokens": 21547,
"cached_input_tokens": 13793,
"output_tokens": 594
},
{
"arm": "opencode2",
"task": "linear-split",
"suite": "mcp",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 23.44,
"model_calls": 3,
"input_tokens": 20988,
"cached_input_tokens": 16802,
"output_tokens": 406
},
{
"arm": "opencode2",
"task": "linear-split",
"suite": "mcp",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 30.32,
"model_calls": 3,
"input_tokens": 32210,
"cached_input_tokens": 22190,
"output_tokens": 666
},
{
"arm": "opencode2",
"task": "linear-warm",
"suite": "mcp",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 21.66,
"model_calls": 6,
"input_tokens": 69118,
"cached_input_tokens": 59552,
"output_tokens": 1774
},
{
"arm": "opencode2",
"task": "linear-warm",
"suite": "mcp",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 20.66,
"model_calls": 6,
"input_tokens": 66240,
"cached_input_tokens": 57633,
"output_tokens": 1691
},
{
"arm": "opencode2",
"task": "linear-warm",
"suite": "mcp",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 27.87,
"model_calls": 7,
"input_tokens": 271673,
"cached_input_tokens": 224144,
"output_tokens": 1799
},
{
"arm": "opencode2",
"task": "recall-noise",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 5.36,
"model_calls": 3,
"input_tokens": 24029,
"cached_input_tokens": 20531,
"output_tokens": 167
},
{
"arm": "opencode2",
"task": "recall-noise",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 5.66,
"model_calls": 3,
"input_tokens": 24029,
"cached_input_tokens": 20567,
"output_tokens": 167
},
{
"arm": "opencode2",
"task": "recall-noise",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 5.97,
"model_calls": 3,
"input_tokens": 24029,
"cached_input_tokens": 20567,
"output_tokens": 167
},
{
"arm": "opencode2",
"task": "refactor-rename",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 7.86,
"model_calls": 4,
"input_tokens": 32910,
"cached_input_tokens": 29006,
"output_tokens": 692
},
{
"arm": "opencode2",
"task": "refactor-rename",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 14.67,
"model_calls": 4,
"input_tokens": 32878,
"cached_input_tokens": 29060,
"output_tokens": 679
},
{
"arm": "opencode2",
"task": "refactor-rename",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 9.22,
"model_calls": 4,
"input_tokens": 32966,
"cached_input_tokens": 29104,
"output_tokens": 716
},
{
"arm": "opencode2",
"task": "regex-count",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 8.44,
"model_calls": 3,
"input_tokens": 24257,
"cached_input_tokens": 20810,
"output_tokens": 519
},
{
"arm": "opencode2",
"task": "regex-count",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 9.48,
"model_calls": 4,
"input_tokens": 32509,
"cached_input_tokens": 28973,
"output_tokens": 539
},
{
"arm": "opencode2",
"task": "regex-count",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 8.06,
"model_calls": 3,
"input_tokens": 24324,
"cached_input_tokens": 20935,
"output_tokens": 454
},
{
"arm": "opencode2",
"task": "write-tests",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 8.76,
"model_calls": 4,
"input_tokens": 32026,
"cached_input_tokens": 28378,
"output_tokens": 710
},
{
"arm": "opencode2",
"task": "write-tests",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 9.63,
"model_calls": 4,
"input_tokens": 31983,
"cached_input_tokens": 28440,
"output_tokens": 628
},
{
"arm": "opencode2",
"task": "write-tests",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 9.35,
"model_calls": 4,
"input_tokens": 32019,
"cached_input_tokens": 28438,
"output_tokens": 671
},
{
"arm": "pi",
"task": "csv-sum",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 4.57,
"model_calls": 3,
"input_tokens": 7198,
"cached_input_tokens": 5896,
"output_tokens": 224
},
{
"arm": "pi",
"task": "csv-sum",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 4.53,
"model_calls": 3,
"input_tokens": 7223,
"cached_input_tokens": 5896,
"output_tokens": 230
},
{
"arm": "pi",
"task": "csv-sum",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 4.99,
"model_calls": 3,
"input_tokens": 7223,
"cached_input_tokens": 5896,
"output_tokens": 204
},
{
"arm": "pi",
"task": "dead-code",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 4.56,
"model_calls": 3,
"input_tokens": 7558,
"cached_input_tokens": 5984,
"output_tokens": 231
},
{
"arm": "pi",
"task": "dead-code",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 6.37,
"model_calls": 3,
"input_tokens": 7536,
"cached_input_tokens": 5973,
"output_tokens": 220
},
{
"arm": "pi",
"task": "dead-code",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 12.41,
"model_calls": 3,
"input_tokens": 7572,
"cached_input_tokens": 5991,
"output_tokens": 238
},
{
"arm": "pi",
"task": "exact-reply",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 2.13,
"model_calls": 1,
"input_tokens": 2271,
"cached_input_tokens": 0,
"output_tokens": 4
},
{
"arm": "pi",
"task": "exact-reply",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 1.73,
"model_calls": 1,
"input_tokens": 2271,
"cached_input_tokens": 0,
"output_tokens": 4
},
{
"arm": "pi",
"task": "exact-reply",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 1.73,
"model_calls": 1,
"input_tokens": 2271,
"cached_input_tokens": 0,
"output_tokens": 4
},
{
"arm": "pi",
"task": "file-ops",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 3.34,
"model_calls": 2,
"input_tokens": 4746,
"cached_input_tokens": 3489,
"output_tokens": 168
},
{
"arm": "pi",
"task": "file-ops",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 3.71,
"model_calls": 2,
"input_tokens": 4746,
"cached_input_tokens": 3489,
"output_tokens": 164
},
{
"arm": "pi",
"task": "file-ops",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 3.37,
"model_calls": 2,
"input_tokens": 4746,
"cached_input_tokens": 3489,
"output_tokens": 168
},
{
"arm": "pi",
"task": "fix-fib",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 17.83,
"model_calls": 4,
"input_tokens": 10457,
"cached_input_tokens": 8722,
"output_tokens": 432
},
{
"arm": "pi",
"task": "fix-fib",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 8.32,
"model_calls": 4,
"input_tokens": 10449,
"cached_input_tokens": 7532,
"output_tokens": 417
},
{
"arm": "pi",
"task": "fix-fib",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 9.91,
"model_calls": 4,
"input_tokens": 10434,
"cached_input_tokens": 7532,
"output_tokens": 464
},
{
"arm": "pi",
"task": "git-ops",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 4.9,
"model_calls": 2,
"input_tokens": 5016,
"cached_input_tokens": 3507,
"output_tokens": 286
},
{
"arm": "pi",
"task": "git-ops",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 11.01,
"model_calls": 3,
"input_tokens": 7451,
"cached_input_tokens": 5897,
"output_tokens": 330
},
{
"arm": "pi",
"task": "git-ops",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 5.27,
"model_calls": 2,
"input_tokens": 5009,
"cached_input_tokens": 3507,
"output_tokens": 274
},
{
"arm": "pi",
"task": "json-transform",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 4.74,
"model_calls": 3,
"input_tokens": 7341,
"cached_input_tokens": 5927,
"output_tokens": 281
},
{
"arm": "pi",
"task": "json-transform",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 5.47,
"model_calls": 3,
"input_tokens": 7298,
"cached_input_tokens": 5927,
"output_tokens": 238
},
{
"arm": "pi",
"task": "json-transform",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 4.7,
"model_calls": 3,
"input_tokens": 7357,
"cached_input_tokens": 5934,
"output_tokens": 240
},
{
"arm": "pi",
"task": "recall-noise",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 3.6,
"model_calls": 2,
"input_tokens": 4979,
"cached_input_tokens": 3480,
"output_tokens": 61
},
{
"arm": "pi",
"task": "recall-noise",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 3.68,
"model_calls": 2,
"input_tokens": 4979,
"cached_input_tokens": 3480,
"output_tokens": 61
},
{
"arm": "pi",
"task": "recall-noise",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 3.19,
"model_calls": 2,
"input_tokens": 4979,
"cached_input_tokens": 3480,
"output_tokens": 61
},
{
"arm": "pi",
"task": "refactor-rename",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 7.65,
"model_calls": 4,
"input_tokens": 10136,
"cached_input_tokens": 8537,
"output_tokens": 538
},
{
"arm": "pi",
"task": "refactor-rename",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 7.25,
"model_calls": 4,
"input_tokens": 10226,
"cached_input_tokens": 8601,
"output_tokens": 384
},
{
"arm": "pi",
"task": "refactor-rename",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 8.75,
"model_calls": 4,
"input_tokens": 10230,
"cached_input_tokens": 8579,
"output_tokens": 541
},
{
"arm": "pi",
"task": "regex-count",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 6.21,
"model_calls": 3,
"input_tokens": 7399,
"cached_input_tokens": 6012,
"output_tokens": 257
},
{
"arm": "pi",
"task": "regex-count",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 7.97,
"model_calls": 4,
"input_tokens": 10428,
"cached_input_tokens": 8756,
"output_tokens": 518
},
{
"arm": "pi",
"task": "regex-count",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 5.51,
"model_calls": 3,
"input_tokens": 7389,
"cached_input_tokens": 5993,
"output_tokens": 316
},
{
"arm": "pi",
"task": "write-tests",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 5.82,
"model_calls": 3,
"input_tokens": 7324,
"cached_input_tokens": 5903,
"output_tokens": 386
},
{
"arm": "pi",
"task": "write-tests",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 6.09,
"model_calls": 3,
"input_tokens": 7324,
"cached_input_tokens": 5903,
"output_tokens": 357
},
{
"arm": "pi",
"task": "write-tests",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 7.22,
"model_calls": 3,
"input_tokens": 7324,
"cached_input_tokens": 5903,
"output_tokens": 350
}
]
}