graff — a fast agentic coding harness in Zig: multi-provider, MCP, workflows, DGM evolution loop, TS/Python SDKs
Python
{
"model": "gpt-6.1-sol",
"tasks": 21,
"reps": 3,
"arms": {
"graff": "graff (ADR 0231 build)",
"codex-app-server": "Codex app server",
"pi": "pi"
},
"summary": {
"all": {
"graff": {
"runs": 63,
"passed": 63,
"wall_s_per_task": 22.2,
"list_usd_per_task": 0.0131,
"cached_input_share": 0.904,
"output_tokens_per_task": 348,
"model_calls_per_task": 3.1
},
"codex-app-server": {
"runs": 63,
"passed": 63,
"wall_s_per_task": 28.6,
"list_usd_per_task": 0.0426,
"cached_input_share": 0.855,
"output_tokens_per_task": 409,
"model_calls_per_task": 4.4
},
"pi": {
"runs": 33,
"passed": 33,
"wall_s_per_task": 17.0,
"list_usd_per_task": 0.0067,
"cached_input_share": 0.334,
"output_tokens_per_task": 107,
"model_calls_per_task": 3.2
}
},
"core": {
"graff": {
"runs": 33,
"passed": 33,
"wall_s_per_task": 16.0,
"list_usd_per_task": 0.0093,
"cached_input_share": 0.923,
"output_tokens_per_task": 164,
"model_calls_per_task": 3.0
},
"codex-app-server": {
"runs": 33,
"passed": 33,
"wall_s_per_task": 20.4,
"list_usd_per_task": 0.0156,
"cached_input_share": 0.896,
"output_tokens_per_task": 209,
"model_calls_per_task": 3.1
},
"pi": {
"runs": 33,
"passed": 33,
"wall_s_per_task": 17.0,
"list_usd_per_task": 0.0067,
"cached_input_share": 0.334,
"output_tokens_per_task": 107,
"model_calls_per_task": 3.2
}
},
"mcp": {
"graff": {
"runs": 30,
"passed": 30,
"wall_s_per_task": 29.0,
"list_usd_per_task": 0.0173,
"cached_input_share": 0.887,
"output_tokens_per_task": 550,
"model_calls_per_task": 3.3
},
"codex-app-server": {
"runs": 30,
"passed": 30,
"wall_s_per_task": 37.8,
"list_usd_per_task": 0.0722,
"cached_input_share": 0.839,
"output_tokens_per_task": 628,
"model_calls_per_task": 5.8
}
}
},
"per_task": {
"csv-sum": {
"suite": "core",
"graff": {
"passed": 3,
"runs": 3,
"wall_s": 16.0,
"list_usd": 0.0116,
"model_calls": 3.3
},
"codex-app-server": {
"passed": 3,
"runs": 3,
"wall_s": 21.5,
"list_usd": 0.018,
"model_calls": 3.3
},
"pi": {
"passed": 3,
"runs": 3,
"wall_s": 13.8,
"list_usd": 0.0051,
"model_calls": 3
}
},
"dead-code": {
"suite": "core",
"graff": {
"passed": 3,
"runs": 3,
"wall_s": 10.6,
"list_usd": 0.0059,
"model_calls": 2
},
"codex-app-server": {
"passed": 3,
"runs": 3,
"wall_s": 21.0,
"list_usd": 0.0171,
"model_calls": 4
},
"pi": {
"passed": 3,
"runs": 3,
"wall_s": 18.2,
"list_usd": 0.0068,
"model_calls": 4
}
},
"exact-reply": {
"suite": "core",
"graff": {
"passed": 3,
"runs": 3,
"wall_s": 5.0,
"list_usd": 0.0014,
"model_calls": 1
},
"codex-app-server": {
"passed": 3,
"runs": 3,
"wall_s": 8.3,
"list_usd": 0.0052,
"model_calls": 1
},
"pi": {
"passed": 3,
"runs": 3,
"wall_s": 8.5,
"list_usd": 0.0023,
"model_calls": 1
}
},
"file-ops": {
"suite": "core",
"graff": {
"passed": 3,
"runs": 3,
"wall_s": 14.5,
"list_usd": 0.0107,
"model_calls": 3
},
"codex-app-server": {
"passed": 3,
"runs": 3,
"wall_s": 13.1,
"list_usd": 0.0116,
"model_calls": 2
},
"pi": {
"passed": 3,
"runs": 3,
"wall_s": 23.0,
"list_usd": 0.0059,
"model_calls": 3
}
},
"fix-fib": {
"suite": "core",
"graff": {
"passed": 3,
"runs": 3,
"wall_s": 21.4,
"list_usd": 0.0143,
"model_calls": 4
},
"codex-app-server": {
"passed": 3,
"runs": 3,
"wall_s": 23.5,
"list_usd": 0.0202,
"model_calls": 4
},
"pi": {
"passed": 3,
"runs": 3,
"wall_s": 22.3,
"list_usd": 0.0103,
"model_calls": 5
}
},
"git-ops": {
"suite": "core",
"graff": {
"passed": 3,
"runs": 3,
"wall_s": 21.5,
"list_usd": 0.0115,
"model_calls": 3
},
"codex-app-server": {
"passed": 3,
"runs": 3,
"wall_s": 20.4,
"list_usd": 0.0159,
"model_calls": 3
},
"pi": {
"passed": 3,
"runs": 3,
"wall_s": 15.5,
"list_usd": 0.0059,
"model_calls": 3
}
},
"json-transform": {
"suite": "core",
"graff": {
"passed": 3,
"runs": 3,
"wall_s": 16.8,
"list_usd": 0.0089,
"model_calls": 3
},
"codex-app-server": {
"passed": 3,
"runs": 3,
"wall_s": 30.6,
"list_usd": 0.0169,
"model_calls": 4
},
"pi": {
"passed": 3,
"runs": 3,
"wall_s": 15.2,
"list_usd": 0.0063,
"model_calls": 3
}
},
"linear-issues": {
"suite": "mcp",
"graff": {
"passed": 3,
"runs": 3,
"wall_s": 22.8,
"list_usd": 0.0175,
"model_calls": 2
},
"codex-app-server": {
"passed": 3,
"runs": 3,
"wall_s": 26.8,
"list_usd": 0.0436,
"model_calls": 4
}
},
"linear-nohint": {
"suite": "mcp",
"graff": {
"passed": 3,
"runs": 3,
"wall_s": 30.7,
"list_usd": 0.0196,
"model_calls": 4
},
"codex-app-server": {
"passed": 3,
"runs": 3,
"wall_s": 28.5,
"list_usd": 0.062,
"model_calls": 4.3
}
},
"linear-nohint-warm": {
"suite": "mcp",
"graff": {
"passed": 3,
"runs": 3,
"wall_s": 27.8,
"list_usd": 0.0173,
"model_calls": 3
},
"codex-app-server": {
"passed": 3,
"runs": 3,
"wall_s": 34.2,
"list_usd": 0.0702,
"model_calls": 5
}
},
"linear-quiet": {
"suite": "mcp",
"graff": {
"passed": 3,
"runs": 3,
"wall_s": 22.0,
"list_usd": 0.0107,
"model_calls": 2
},
"codex-app-server": {
"passed": 3,
"runs": 3,
"wall_s": 36.6,
"list_usd": 0.0389,
"model_calls": 4.3
}
},
"linear-quiet-warm": {
"suite": "mcp",
"graff": {
"passed": 3,
"runs": 3,
"wall_s": 24.0,
"list_usd": 0.0132,
"model_calls": 2.3
},
"codex-app-server": {
"passed": 3,
"runs": 3,
"wall_s": 33.4,
"list_usd": 0.0456,
"model_calls": 4
}
},
"linear-reduce": {
"suite": "mcp",
"graff": {
"passed": 3,
"runs": 3,
"wall_s": 22.5,
"list_usd": 0.0108,
"model_calls": 2
},
"codex-app-server": {
"passed": 3,
"runs": 3,
"wall_s": 39.2,
"list_usd": 0.0386,
"model_calls": 5.3
}
},
"linear-reduce-warm": {
"suite": "mcp",
"graff": {
"passed": 3,
"runs": 3,
"wall_s": 26.6,
"list_usd": 0.0145,
"model_calls": 3
},
"codex-app-server": {
"passed": 3,
"runs": 3,
"wall_s": 37.7,
"list_usd": 0.0483,
"model_calls": 5.3
}
},
"linear-sidecar": {
"suite": "mcp",
"graff": {
"passed": 3,
"runs": 3,
"wall_s": 38.3,
"list_usd": 0.0252,
"model_calls": 4.3
},
"codex-app-server": {
"passed": 3,
"runs": 3,
"wall_s": 35.4,
"list_usd": 0.05,
"model_calls": 4.7
}
},
"linear-split": {
"suite": "mcp",
"graff": {
"passed": 3,
"runs": 3,
"wall_s": 52.5,
"list_usd": 0.0322,
"model_calls": 8
},
"codex-app-server": {
"passed": 3,
"runs": 3,
"wall_s": 66.4,
"list_usd": 0.2772,
"model_calls": 17
}
},
"linear-warm": {
"suite": "mcp",
"graff": {
"passed": 3,
"runs": 3,
"wall_s": 23.0,
"list_usd": 0.012,
"model_calls": 2
},
"codex-app-server": {
"passed": 3,
"runs": 3,
"wall_s": 39.4,
"list_usd": 0.0477,
"model_calls": 4.3
}
},
"recall-noise": {
"suite": "core",
"graff": {
"passed": 3,
"runs": 3,
"wall_s": 9.9,
"list_usd": 0.0081,
"model_calls": 2
},
"codex-app-server": {
"passed": 3,
"runs": 3,
"wall_s": 10.5,
"list_usd": 0.0092,
"model_calls": 2
},
"pi": {
"passed": 3,
"runs": 3,
"wall_s": 10.3,
"list_usd": 0.0053,
"model_calls": 2
}
},
"refactor-rename": {
"suite": "core",
"graff": {
"passed": 3,
"runs": 3,
"wall_s": 23.0,
"list_usd": 0.0141,
"model_calls": 4
},
"codex-app-server": {
"passed": 3,
"runs": 3,
"wall_s": 29.0,
"list_usd": 0.0224,
"model_calls": 4
},
"pi": {
"passed": 3,
"runs": 3,
"wall_s": 23.4,
"list_usd": 0.0132,
"model_calls": 5
}
},
"regex-count": {
"suite": "core",
"graff": {
"passed": 3,
"runs": 3,
"wall_s": 17.2,
"list_usd": 0.0071,
"model_calls": 3.3
},
"codex-app-server": {
"passed": 3,
"runs": 3,
"wall_s": 18.0,
"list_usd": 0.0158,
"model_calls": 3
},
"pi": {
"passed": 3,
"runs": 3,
"wall_s": 13.3,
"list_usd": 0.0033,
"model_calls": 2
}
},
"write-tests": {
"suite": "core",
"graff": {
"passed": 3,
"runs": 3,
"wall_s": 20.4,
"list_usd": 0.0087,
"model_calls": 4
},
"codex-app-server": {
"passed": 3,
"runs": 3,
"wall_s": 28.1,
"list_usd": 0.019,
"model_calls": 4
},
"pi": {
"passed": 3,
"runs": 3,
"wall_s": 23.8,
"list_usd": 0.0093,
"model_calls": 4.7
}
}
},
"runs": [
{
"arm": "codex-app-server",
"task": "csv-sum",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 21.1,
"model_calls": 3,
"input_tokens": 43494,
"cached_input_tokens": 36864,
"output_tokens": 265,
"list_usd": 0.019596
},
{
"arm": "codex-app-server",
"task": "csv-sum",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 22.6,
"model_calls": 3,
"input_tokens": 43342,
"cached_input_tokens": 38784,
"output_tokens": 326,
"list_usd": 0.016254
},
{
"arm": "codex-app-server",
"task": "csv-sum",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 20.91,
"model_calls": 4,
"input_tokens": 57931,
"cached_input_tokens": 52992,
"output_tokens": 288,
"list_usd": 0.018057
},
{
"arm": "codex-app-server",
"task": "dead-code",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 22.78,
"model_calls": 4,
"input_tokens": 57946,
"cached_input_tokens": 52864,
"output_tokens": 150,
"list_usd": 0.01695
},
{
"arm": "codex-app-server",
"task": "dead-code",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 21.11,
"model_calls": 4,
"input_tokens": 58070,
"cached_input_tokens": 52864,
"output_tokens": 150,
"list_usd": 0.017198
},
{
"arm": "codex-app-server",
"task": "dead-code",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 19.15,
"model_calls": 4,
"input_tokens": 58043,
"cached_input_tokens": 52864,
"output_tokens": 165,
"list_usd": 0.017294
},
{
"arm": "codex-app-server",
"task": "exact-reply",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 6.37,
"model_calls": 1,
"input_tokens": 14234,
"cached_input_tokens": 12288,
"output_tokens": 5,
"list_usd": 0.005171
},
{
"arm": "codex-app-server",
"task": "exact-reply",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 9.23,
"model_calls": 1,
"input_tokens": 14235,
"cached_input_tokens": 12288,
"output_tokens": 5,
"list_usd": 0.005173
},
{
"arm": "codex-app-server",
"task": "exact-reply",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 9.37,
"model_calls": 1,
"input_tokens": 14234,
"cached_input_tokens": 12288,
"output_tokens": 5,
"list_usd": 0.005171
},
{
"arm": "codex-app-server",
"task": "file-ops",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 11.53,
"model_calls": 2,
"input_tokens": 28646,
"cached_input_tokens": 24576,
"output_tokens": 97,
"list_usd": 0.011568
},
{
"arm": "codex-app-server",
"task": "file-ops",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 15.1,
"model_calls": 2,
"input_tokens": 28646,
"cached_input_tokens": 24576,
"output_tokens": 99,
"list_usd": 0.011588
},
{
"arm": "codex-app-server",
"task": "file-ops",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 12.7,
"model_calls": 2,
"input_tokens": 28646,
"cached_input_tokens": 24576,
"output_tokens": 101,
"list_usd": 0.011608
},
{
"arm": "codex-app-server",
"task": "fix-fib",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 22.94,
"model_calls": 4,
"input_tokens": 58587,
"cached_input_tokens": 53120,
"output_tokens": 284,
"list_usd": 0.019086
},
{
"arm": "codex-app-server",
"task": "fix-fib",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 23.54,
"model_calls": 4,
"input_tokens": 58579,
"cached_input_tokens": 53120,
"output_tokens": 285,
"list_usd": 0.01908
},
{
"arm": "codex-app-server",
"task": "fix-fib",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 24.12,
"model_calls": 4,
"input_tokens": 58566,
"cached_input_tokens": 51328,
"output_tokens": 279,
"list_usd": 0.022399
},
{
"arm": "codex-app-server",
"task": "git-ops",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 22.45,
"model_calls": 3,
"input_tokens": 43610,
"cached_input_tokens": 38656,
"output_tokens": 218,
"list_usd": 0.015954
},
{
"arm": "codex-app-server",
"task": "git-ops",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 17.47,
"model_calls": 3,
"input_tokens": 43653,
"cached_input_tokens": 38656,
"output_tokens": 215,
"list_usd": 0.01601
},
{
"arm": "codex-app-server",
"task": "git-ops",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 21.17,
"model_calls": 3,
"input_tokens": 43532,
"cached_input_tokens": 38656,
"output_tokens": 221,
"list_usd": 0.015828
},
{
"arm": "codex-app-server",
"task": "json-transform",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 28.19,
"model_calls": 4,
"input_tokens": 58004,
"cached_input_tokens": 52992,
"output_tokens": 252,
"list_usd": 0.017843
},
{
"arm": "codex-app-server",
"task": "json-transform",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 40.09,
"model_calls": 4,
"input_tokens": 58013,
"cached_input_tokens": 54912,
"output_tokens": 287,
"list_usd": 0.014563
},
{
"arm": "codex-app-server",
"task": "json-transform",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 23.55,
"model_calls": 4,
"input_tokens": 58008,
"cached_input_tokens": 52864,
"output_tokens": 274,
"list_usd": 0.018314
},
{
"arm": "codex-app-server",
"task": "linear-issues",
"suite": "mcp",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 26.65,
"model_calls": 4,
"input_tokens": 78780,
"cached_input_tokens": 66176,
"output_tokens": 428,
"list_usd": 0.036106
},
{
"arm": "codex-app-server",
"task": "linear-issues",
"suite": "mcp",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 28.72,
"model_calls": 4,
"input_tokens": 78019,
"cached_input_tokens": 60416,
"output_tokens": 448,
"list_usd": 0.045728
},
{
"arm": "codex-app-server",
"task": "linear-issues",
"suite": "mcp",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 25.05,
"model_calls": 4,
"input_tokens": 91803,
"cached_input_tokens": 72960,
"output_tokens": 402,
"list_usd": 0.049002
},
{
"arm": "codex-app-server",
"task": "linear-nohint",
"suite": "mcp",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 27.89,
"model_calls": 4,
"input_tokens": 104953,
"cached_input_tokens": 81664,
"output_tokens": 462,
"list_usd": 0.059364
},
{
"arm": "codex-app-server",
"task": "linear-nohint",
"suite": "mcp",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 31.32,
"model_calls": 5,
"input_tokens": 138111,
"cached_input_tokens": 116480,
"output_tokens": 456,
"list_usd": 0.05947
},
{
"arm": "codex-app-server",
"task": "linear-nohint",
"suite": "mcp",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 26.24,
"model_calls": 4,
"input_tokens": 105177,
"cached_input_tokens": 77696,
"output_tokens": 442,
"list_usd": 0.067152
},
{
"arm": "codex-app-server",
"task": "linear-nohint-warm",
"suite": "mcp",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 30.78,
"model_calls": 4,
"input_tokens": 104995,
"cached_input_tokens": 83584,
"output_tokens": 471,
"list_usd": 0.05589
},
{
"arm": "codex-app-server",
"task": "linear-nohint-warm",
"suite": "mcp",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 38.38,
"model_calls": 6,
"input_tokens": 167525,
"cached_input_tokens": 130304,
"output_tokens": 612,
"list_usd": 0.093592
},
{
"arm": "codex-app-server",
"task": "linear-nohint-warm",
"suite": "mcp",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 33.44,
"model_calls": 5,
"input_tokens": 134923,
"cached_input_tokens": 112256,
"output_tokens": 467,
"list_usd": 0.06123
},
{
"arm": "codex-app-server",
"task": "linear-quiet",
"suite": "mcp",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 45.11,
"model_calls": 5,
"input_tokens": 107148,
"cached_input_tokens": 92800,
"output_tokens": 671,
"list_usd": 0.044686
},
{
"arm": "codex-app-server",
"task": "linear-quiet",
"suite": "mcp",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 29.19,
"model_calls": 4,
"input_tokens": 63325,
"cached_input_tokens": 52736,
"output_tokens": 494,
"list_usd": 0.031392
},
{
"arm": "codex-app-server",
"task": "linear-quiet",
"suite": "mcp",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 35.47,
"model_calls": 4,
"input_tokens": 81625,
"cached_input_tokens": 67584,
"output_tokens": 575,
"list_usd": 0.04059
},
{
"arm": "codex-app-server",
"task": "linear-quiet-warm",
"suite": "mcp",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 27.33,
"model_calls": 4,
"input_tokens": 63408,
"cached_input_tokens": 52992,
"output_tokens": 466,
"list_usd": 0.030791
},
{
"arm": "codex-app-server",
"task": "linear-quiet-warm",
"suite": "mcp",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 35.83,
"model_calls": 4,
"input_tokens": 81586,
"cached_input_tokens": 54656,
"output_tokens": 547,
"list_usd": 0.064796
},
{
"arm": "codex-app-server",
"task": "linear-quiet-warm",
"suite": "mcp",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 37.12,
"model_calls": 4,
"input_tokens": 81885,
"cached_input_tokens": 67712,
"output_tokens": 612,
"list_usd": 0.041237
},
{
"arm": "codex-app-server",
"task": "linear-reduce",
"suite": "mcp",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 30.24,
"model_calls": 4,
"input_tokens": 63323,
"cached_input_tokens": 54656,
"output_tokens": 519,
"list_usd": 0.02799
},
{
"arm": "codex-app-server",
"task": "linear-reduce",
"suite": "mcp",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 48.78,
"model_calls": 6,
"input_tokens": 132981,
"cached_input_tokens": 114944,
"output_tokens": 674,
"list_usd": 0.054308
},
{
"arm": "codex-app-server",
"task": "linear-reduce",
"suite": "mcp",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 38.67,
"model_calls": 6,
"input_tokens": 96523,
"cached_input_tokens": 87424,
"output_tokens": 655,
"list_usd": 0.03349
},
{
"arm": "codex-app-server",
"task": "linear-reduce-warm",
"suite": "mcp",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 45.0,
"model_calls": 6,
"input_tokens": 138736,
"cached_input_tokens": 112384,
"output_tokens": 658,
"list_usd": 0.070522
},
{
"arm": "codex-app-server",
"task": "linear-reduce-warm",
"suite": "mcp",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 41.32,
"model_calls": 6,
"input_tokens": 132919,
"cached_input_tokens": 118400,
"output_tokens": 649,
"list_usd": 0.047368
},
{
"arm": "codex-app-server",
"task": "linear-reduce-warm",
"suite": "mcp",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 26.82,
"model_calls": 4,
"input_tokens": 63275,
"cached_input_tokens": 54656,
"output_tokens": 436,
"list_usd": 0.027064
},
{
"arm": "codex-app-server",
"task": "linear-sidecar",
"suite": "mcp",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 41.51,
"model_calls": 5,
"input_tokens": 139088,
"cached_input_tokens": 115200,
"output_tokens": 585,
"list_usd": 0.065146
},
{
"arm": "codex-app-server",
"task": "linear-sidecar",
"suite": "mcp",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 36.42,
"model_calls": 5,
"input_tokens": 96819,
"cached_input_tokens": 79232,
"output_tokens": 559,
"list_usd": 0.048687
},
{
"arm": "codex-app-server",
"task": "linear-sidecar",
"suite": "mcp",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 28.14,
"model_calls": 4,
"input_tokens": 78076,
"cached_input_tokens": 65664,
"output_tokens": 474,
"list_usd": 0.03613
},
{
"arm": "codex-app-server",
"task": "linear-split",
"suite": "mcp",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 68.37,
"model_calls": 17,
"input_tokens": 376049,
"cached_input_tokens": 341632,
"output_tokens": 1479,
"list_usd": 0.228179
},
{
"arm": "codex-app-server",
"task": "linear-split",
"suite": "mcp",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 67.48,
"model_calls": 16,
"input_tokens": 353717,
"cached_input_tokens": 295936,
"output_tokens": 1460,
"list_usd": 0.312211
},
{
"arm": "codex-app-server",
"task": "linear-split",
"suite": "mcp",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 63.22,
"model_calls": 18,
"input_tokens": 440631,
"cached_input_tokens": 392832,
"output_tokens": 1423,
"list_usd": 0.291107
},
{
"arm": "codex-app-server",
"task": "linear-warm",
"suite": "mcp",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 46.12,
"model_calls": 5,
"input_tokens": 105284,
"cached_input_tokens": 87936,
"output_tokens": 580,
"list_usd": 0.04929
},
{
"arm": "codex-app-server",
"task": "linear-warm",
"suite": "mcp",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 33.35,
"model_calls": 4,
"input_tokens": 89282,
"cached_input_tokens": 73600,
"output_tokens": 569,
"list_usd": 0.044414
},
{
"arm": "codex-app-server",
"task": "linear-warm",
"suite": "mcp",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 38.58,
"model_calls": 4,
"input_tokens": 80241,
"cached_input_tokens": 61312,
"output_tokens": 553,
"list_usd": 0.049519
},
{
"arm": "codex-app-server",
"task": "recall-noise",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 10.44,
"model_calls": 2,
"input_tokens": 28812,
"cached_input_tokens": 26368,
"output_tokens": 52,
"list_usd": 0.008045
},
{
"arm": "codex-app-server",
"task": "recall-noise",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 9.8,
"model_calls": 2,
"input_tokens": 28807,
"cached_input_tokens": 26368,
"output_tokens": 51,
"list_usd": 0.008025
},
{
"arm": "codex-app-server",
"task": "recall-noise",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 11.18,
"model_calls": 2,
"input_tokens": 28817,
"cached_input_tokens": 24576,
"output_tokens": 57,
"list_usd": 0.01151
},
{
"arm": "codex-app-server",
"task": "refactor-rename",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 26.91,
"model_calls": 4,
"input_tokens": 58485,
"cached_input_tokens": 52992,
"output_tokens": 429,
"list_usd": 0.020575
},
{
"arm": "codex-app-server",
"task": "refactor-rename",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 32.06,
"model_calls": 4,
"input_tokens": 58307,
"cached_input_tokens": 51328,
"output_tokens": 332,
"list_usd": 0.022411
},
{
"arm": "codex-app-server",
"task": "refactor-rename",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 28.13,
"model_calls": 4,
"input_tokens": 58435,
"cached_input_tokens": 50944,
"output_tokens": 420,
"list_usd": 0.024276
},
{
"arm": "codex-app-server",
"task": "regex-count",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 18.3,
"model_calls": 3,
"input_tokens": 43226,
"cached_input_tokens": 36864,
"output_tokens": 175,
"list_usd": 0.01816
},
{
"arm": "codex-app-server",
"task": "regex-count",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 17.35,
"model_calls": 3,
"input_tokens": 43195,
"cached_input_tokens": 38784,
"output_tokens": 170,
"list_usd": 0.0144
},
{
"arm": "codex-app-server",
"task": "regex-count",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 18.26,
"model_calls": 3,
"input_tokens": 43262,
"cached_input_tokens": 38656,
"output_tokens": 190,
"list_usd": 0.014978
},
{
"arm": "codex-app-server",
"task": "write-tests",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 31.21,
"model_calls": 4,
"input_tokens": 58018,
"cached_input_tokens": 52864,
"output_tokens": 361,
"list_usd": 0.019204
},
{
"arm": "codex-app-server",
"task": "write-tests",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 24.56,
"model_calls": 4,
"input_tokens": 58006,
"cached_input_tokens": 52992,
"output_tokens": 356,
"list_usd": 0.018887
},
{
"arm": "codex-app-server",
"task": "write-tests",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 28.53,
"model_calls": 4,
"input_tokens": 57998,
"cached_input_tokens": 52864,
"output_tokens": 349,
"list_usd": 0.019044
},
{
"arm": "graff",
"task": "csv-sum",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 13.5,
"model_calls": 3,
"input_tokens": 31280,
"cached_input_tokens": 26624,
"output_tokens": 107,
"list_usd": 0.013044
},
{
"arm": "graff",
"task": "csv-sum",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 14.83,
"model_calls": 3,
"input_tokens": 31243,
"cached_input_tokens": 26624,
"output_tokens": 101,
"list_usd": 0.01291
},
{
"arm": "graff",
"task": "csv-sum",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 19.63,
"model_calls": 4,
"input_tokens": 41912,
"cached_input_tokens": 40448,
"output_tokens": 186,
"list_usd": 0.008833
},
{
"arm": "graff",
"task": "dead-code",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 9.57,
"model_calls": 2,
"input_tokens": 20881,
"cached_input_tokens": 20352,
"output_tokens": 56,
"list_usd": 0.003653
},
{
"arm": "graff",
"task": "dead-code",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 9.3,
"model_calls": 2,
"input_tokens": 20881,
"cached_input_tokens": 16512,
"output_tokens": 56,
"list_usd": 0.010949
},
{
"arm": "graff",
"task": "dead-code",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 12.97,
"model_calls": 2,
"input_tokens": 20881,
"cached_input_tokens": 20608,
"output_tokens": 56,
"list_usd": 0.003167
},
{
"arm": "graff",
"task": "exact-reply",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 4.66,
"model_calls": 1,
"input_tokens": 10306,
"cached_input_tokens": 10112,
"output_tokens": 5,
"list_usd": 0.001449
},
{
"arm": "graff",
"task": "exact-reply",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 3.89,
"model_calls": 1,
"input_tokens": 10306,
"cached_input_tokens": 10112,
"output_tokens": 5,
"list_usd": 0.001449
},
{
"arm": "graff",
"task": "exact-reply",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 6.55,
"model_calls": 1,
"input_tokens": 10306,
"cached_input_tokens": 10112,
"output_tokens": 5,
"list_usd": 0.001449
},
{
"arm": "graff",
"task": "file-ops",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 14.98,
"model_calls": 3,
"input_tokens": 31156,
"cached_input_tokens": 26624,
"output_tokens": 125,
"list_usd": 0.012976
},
{
"arm": "graff",
"task": "file-ops",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 15.58,
"model_calls": 3,
"input_tokens": 31192,
"cached_input_tokens": 30336,
"output_tokens": 135,
"list_usd": 0.006096
},
{
"arm": "graff",
"task": "file-ops",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 13.01,
"model_calls": 3,
"input_tokens": 31158,
"cached_input_tokens": 26624,
"output_tokens": 131,
"list_usd": 0.01304
},
{
"arm": "graff",
"task": "fix-fib",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 22.95,
"model_calls": 4,
"input_tokens": 42668,
"cached_input_tokens": 41344,
"output_tokens": 260,
"list_usd": 0.009382
},
{
"arm": "graff",
"task": "fix-fib",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 20.02,
"model_calls": 4,
"input_tokens": 42674,
"cached_input_tokens": 41344,
"output_tokens": 272,
"list_usd": 0.009514
},
{
"arm": "graff",
"task": "fix-fib",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 21.31,
"model_calls": 4,
"input_tokens": 42407,
"cached_input_tokens": 33280,
"output_tokens": 239,
"list_usd": 0.023972
},
{
"arm": "graff",
"task": "git-ops",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 21.0,
"model_calls": 3,
"input_tokens": 31815,
"cached_input_tokens": 30464,
"output_tokens": 326,
"list_usd": 0.009008
},
{
"arm": "graff",
"task": "git-ops",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 21.12,
"model_calls": 3,
"input_tokens": 31791,
"cached_input_tokens": 30464,
"output_tokens": 307,
"list_usd": 0.00877
},
{
"arm": "graff",
"task": "git-ops",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 22.3,
"model_calls": 3,
"input_tokens": 31852,
"cached_input_tokens": 26624,
"output_tokens": 367,
"list_usd": 0.016788
},
{
"arm": "graff",
"task": "json-transform",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 18.13,
"model_calls": 3,
"input_tokens": 31395,
"cached_input_tokens": 30592,
"output_tokens": 175,
"list_usd": 0.006415
},
{
"arm": "graff",
"task": "json-transform",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 16.53,
"model_calls": 3,
"input_tokens": 31400,
"cached_input_tokens": 30592,
"output_tokens": 180,
"list_usd": 0.006475
},
{
"arm": "graff",
"task": "json-transform",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 15.79,
"model_calls": 3,
"input_tokens": 31401,
"cached_input_tokens": 26752,
"output_tokens": 181,
"list_usd": 0.013783
},
{
"arm": "graff",
"task": "linear-issues",
"suite": "mcp",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 23.02,
"model_calls": 2,
"input_tokens": 22866,
"cached_input_tokens": 20224,
"output_tokens": 422,
"list_usd": 0.011526
},
{
"arm": "graff",
"task": "linear-issues",
"suite": "mcp",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 23.65,
"model_calls": 2,
"input_tokens": 22836,
"cached_input_tokens": 20736,
"output_tokens": 434,
"list_usd": 0.010614
},
{
"arm": "graff",
"task": "linear-issues",
"suite": "mcp",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 21.87,
"model_calls": 2,
"input_tokens": 22789,
"cached_input_tokens": 10112,
"output_tokens": 386,
"list_usd": 0.030225
},
{
"arm": "graff",
"task": "linear-nohint",
"suite": "mcp",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 20.44,
"model_calls": 2,
"input_tokens": 22777,
"cached_input_tokens": 20224,
"output_tokens": 399,
"list_usd": 0.011118
},
{
"arm": "graff",
"task": "linear-nohint",
"suite": "mcp",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 37.99,
"model_calls": 5,
"input_tokens": 58041,
"cached_input_tokens": 52352,
"output_tokens": 561,
"list_usd": 0.022223
},
{
"arm": "graff",
"task": "linear-nohint",
"suite": "mcp",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 33.81,
"model_calls": 5,
"input_tokens": 58107,
"cached_input_tokens": 50560,
"output_tokens": 525,
"list_usd": 0.0254
},
{
"arm": "graff",
"task": "linear-nohint-warm",
"suite": "mcp",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 34.92,
"model_calls": 4,
"input_tokens": 47810,
"cached_input_tokens": 40448,
"output_tokens": 611,
"list_usd": 0.024879
},
{
"arm": "graff",
"task": "linear-nohint-warm",
"suite": "mcp",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 26.5,
"model_calls": 3,
"input_tokens": 35402,
"cached_input_tokens": 31104,
"output_tokens": 451,
"list_usd": 0.016216
},
{
"arm": "graff",
"task": "linear-nohint-warm",
"suite": "mcp",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 21.9,
"model_calls": 2,
"input_tokens": 23300,
"cached_input_tokens": 20992,
"output_tokens": 420,
"list_usd": 0.010915
},
{
"arm": "graff",
"task": "linear-quiet",
"suite": "mcp",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 22.5,
"model_calls": 2,
"input_tokens": 22933,
"cached_input_tokens": 20736,
"output_tokens": 421,
"list_usd": 0.010678
},
{
"arm": "graff",
"task": "linear-quiet",
"suite": "mcp",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 23.89,
"model_calls": 2,
"input_tokens": 22916,
"cached_input_tokens": 20736,
"output_tokens": 409,
"list_usd": 0.010524
},
{
"arm": "graff",
"task": "linear-quiet",
"suite": "mcp",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 19.72,
"model_calls": 2,
"input_tokens": 22815,
"cached_input_tokens": 20224,
"output_tokens": 357,
"list_usd": 0.010774
},
{
"arm": "graff",
"task": "linear-quiet-warm",
"suite": "mcp",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 21.03,
"model_calls": 2,
"input_tokens": 23413,
"cached_input_tokens": 20224,
"output_tokens": 385,
"list_usd": 0.01225
},
{
"arm": "graff",
"task": "linear-quiet-warm",
"suite": "mcp",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 21.77,
"model_calls": 2,
"input_tokens": 23480,
"cached_input_tokens": 20992,
"output_tokens": 442,
"list_usd": 0.011495
},
{
"arm": "graff",
"task": "linear-quiet-warm",
"suite": "mcp",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 29.15,
"model_calls": 3,
"input_tokens": 35478,
"cached_input_tokens": 31872,
"output_tokens": 555,
"list_usd": 0.015949
},
{
"arm": "graff",
"task": "linear-reduce",
"suite": "mcp",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 21.84,
"model_calls": 2,
"input_tokens": 22900,
"cached_input_tokens": 20736,
"output_tokens": 425,
"list_usd": 0.010652
},
{
"arm": "graff",
"task": "linear-reduce",
"suite": "mcp",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 25.31,
"model_calls": 2,
"input_tokens": 23013,
"cached_input_tokens": 20736,
"output_tokens": 487,
"list_usd": 0.011498
},
{
"arm": "graff",
"task": "linear-reduce",
"suite": "mcp",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 20.31,
"model_calls": 2,
"input_tokens": 22917,
"cached_input_tokens": 20736,
"output_tokens": 395,
"list_usd": 0.010386
},
{
"arm": "graff",
"task": "linear-reduce-warm",
"suite": "mcp",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 23.4,
"model_calls": 3,
"input_tokens": 35190,
"cached_input_tokens": 31104,
"output_tokens": 442,
"list_usd": 0.015702
},
{
"arm": "graff",
"task": "linear-reduce-warm",
"suite": "mcp",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 31.58,
"model_calls": 3,
"input_tokens": 35247,
"cached_input_tokens": 31872,
"output_tokens": 491,
"list_usd": 0.014847
},
{
"arm": "graff",
"task": "linear-reduce-warm",
"suite": "mcp",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 24.86,
"model_calls": 3,
"input_tokens": 35204,
"cached_input_tokens": 32640,
"output_tokens": 455,
"list_usd": 0.012942
},
{
"arm": "graff",
"task": "linear-sidecar",
"suite": "mcp",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 36.66,
"model_calls": 4,
"input_tokens": 50303,
"cached_input_tokens": 42112,
"output_tokens": 690,
"list_usd": 0.027493
},
{
"arm": "graff",
"task": "linear-sidecar",
"suite": "mcp",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 41.79,
"model_calls": 5,
"input_tokens": 65453,
"cached_input_tokens": 59392,
"output_tokens": 853,
"list_usd": 0.026591
},
{
"arm": "graff",
"task": "linear-sidecar",
"suite": "mcp",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 36.38,
"model_calls": 4,
"input_tokens": 51164,
"cached_input_tokens": 45440,
"output_tokens": 553,
"list_usd": 0.021522
},
{
"arm": "graff",
"task": "linear-split",
"suite": "mcp",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 46.43,
"model_calls": 8,
"input_tokens": 83470,
"cached_input_tokens": 77312,
"output_tokens": 1175,
"list_usd": 0.031797
},
{
"arm": "graff",
"task": "linear-split",
"suite": "mcp",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 63.62,
"model_calls": 8,
"input_tokens": 83506,
"cached_input_tokens": 76672,
"output_tokens": 1257,
"list_usd": 0.033905
},
{
"arm": "graff",
"task": "linear-split",
"suite": "mcp",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 47.37,
"model_calls": 8,
"input_tokens": 83664,
"cached_input_tokens": 78336,
"output_tokens": 1255,
"list_usd": 0.03104
},
{
"arm": "graff",
"task": "linear-warm",
"suite": "mcp",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 22.62,
"model_calls": 2,
"input_tokens": 23364,
"cached_input_tokens": 20224,
"output_tokens": 418,
"list_usd": 0.012482
},
{
"arm": "graff",
"task": "linear-warm",
"suite": "mcp",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 21.14,
"model_calls": 2,
"input_tokens": 23372,
"cached_input_tokens": 20224,
"output_tokens": 421,
"list_usd": 0.012528
},
{
"arm": "graff",
"task": "linear-warm",
"suite": "mcp",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 25.22,
"model_calls": 2,
"input_tokens": 23334,
"cached_input_tokens": 20992,
"output_tokens": 415,
"list_usd": 0.010933
},
{
"arm": "graff",
"task": "recall-noise",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 10.96,
"model_calls": 2,
"input_tokens": 20929,
"cached_input_tokens": 20480,
"output_tokens": 29,
"list_usd": 0.003236
},
{
"arm": "graff",
"task": "recall-noise",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 8.48,
"model_calls": 2,
"input_tokens": 20929,
"cached_input_tokens": 16768,
"output_tokens": 29,
"list_usd": 0.010289
},
{
"arm": "graff",
"task": "recall-noise",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 10.2,
"model_calls": 2,
"input_tokens": 20929,
"cached_input_tokens": 16512,
"output_tokens": 29,
"list_usd": 0.010775
},
{
"arm": "graff",
"task": "refactor-rename",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 25.3,
"model_calls": 4,
"input_tokens": 42255,
"cached_input_tokens": 34688,
"output_tokens": 280,
"list_usd": 0.021403
},
{
"arm": "graff",
"task": "refactor-rename",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 22.34,
"model_calls": 4,
"input_tokens": 42253,
"cached_input_tokens": 40448,
"output_tokens": 280,
"list_usd": 0.010455
},
{
"arm": "graff",
"task": "refactor-rename",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 21.46,
"model_calls": 4,
"input_tokens": 42240,
"cached_input_tokens": 40448,
"output_tokens": 275,
"list_usd": 0.010379
},
{
"arm": "graff",
"task": "regex-count",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 15.54,
"model_calls": 3,
"input_tokens": 31356,
"cached_input_tokens": 30336,
"output_tokens": 153,
"list_usd": 0.006604
},
{
"arm": "graff",
"task": "regex-count",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 20.09,
"model_calls": 4,
"input_tokens": 41931,
"cached_input_tokens": 40704,
"output_tokens": 179,
"list_usd": 0.008314
},
{
"arm": "graff",
"task": "regex-count",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 15.89,
"model_calls": 3,
"input_tokens": 31358,
"cached_input_tokens": 30464,
"output_tokens": 154,
"list_usd": 0.006374
},
{
"arm": "graff",
"task": "write-tests",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 21.6,
"model_calls": 4,
"input_tokens": 42032,
"cached_input_tokens": 40960,
"output_tokens": 255,
"list_usd": 0.00879
},
{
"arm": "graff",
"task": "write-tests",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 19.43,
"model_calls": 4,
"input_tokens": 41997,
"cached_input_tokens": 40832,
"output_tokens": 236,
"list_usd": 0.008773
},
{
"arm": "graff",
"task": "write-tests",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 20.05,
"model_calls": 4,
"input_tokens": 41989,
"cached_input_tokens": 40960,
"output_tokens": 233,
"list_usd": 0.008484
},
{
"arm": "pi",
"task": "csv-sum",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 15.0,
"model_calls": 3,
"input_tokens": 3573,
"cached_input_tokens": 1024,
"output_tokens": 58,
"list_usd": 0.00578
},
{
"arm": "pi",
"task": "csv-sum",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 13.29,
"model_calls": 3,
"input_tokens": 3573,
"cached_input_tokens": 2048,
"output_tokens": 58,
"list_usd": 0.003835
},
{
"arm": "pi",
"task": "csv-sum",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 13.23,
"model_calls": 3,
"input_tokens": 3573,
"cached_input_tokens": 1024,
"output_tokens": 58,
"list_usd": 0.00578
},
{
"arm": "pi",
"task": "dead-code",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 18.01,
"model_calls": 4,
"input_tokens": 4977,
"cached_input_tokens": 3200,
"output_tokens": 73,
"list_usd": 0.004604
},
{
"arm": "pi",
"task": "dead-code",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 16.44,
"model_calls": 4,
"input_tokens": 4965,
"cached_input_tokens": 2048,
"output_tokens": 69,
"list_usd": 0.006729
},
{
"arm": "pi",
"task": "dead-code",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 20.18,
"model_calls": 4,
"input_tokens": 5082,
"cached_input_tokens": 1024,
"output_tokens": 93,
"list_usd": 0.009148
},
{
"arm": "pi",
"task": "exact-reply",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 7.38,
"model_calls": 1,
"input_tokens": 1115,
"cached_input_tokens": 0,
"output_tokens": 5,
"list_usd": 0.00228
},
{
"arm": "pi",
"task": "exact-reply",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 6.55,
"model_calls": 1,
"input_tokens": 1115,
"cached_input_tokens": 0,
"output_tokens": 5,
"list_usd": 0.00228
},
{
"arm": "pi",
"task": "exact-reply",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 11.46,
"model_calls": 1,
"input_tokens": 1115,
"cached_input_tokens": 0,
"output_tokens": 5,
"list_usd": 0.00228
},
{
"arm": "pi",
"task": "file-ops",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 25.16,
"model_calls": 3,
"input_tokens": 3540,
"cached_input_tokens": 2048,
"output_tokens": 78,
"list_usd": 0.003969
},
{
"arm": "pi",
"task": "file-ops",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 27.22,
"model_calls": 3,
"input_tokens": 3540,
"cached_input_tokens": 0,
"output_tokens": 78,
"list_usd": 0.00786
},
{
"arm": "pi",
"task": "file-ops",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 16.71,
"model_calls": 3,
"input_tokens": 3540,
"cached_input_tokens": 1024,
"output_tokens": 78,
"list_usd": 0.005914
},
{
"arm": "pi",
"task": "fix-fib",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 22.03,
"model_calls": 5,
"input_tokens": 6612,
"cached_input_tokens": 2176,
"output_tokens": 149,
"list_usd": 0.01058
},
{
"arm": "pi",
"task": "fix-fib",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 22.07,
"model_calls": 5,
"input_tokens": 6612,
"cached_input_tokens": 2304,
"output_tokens": 149,
"list_usd": 0.010336
},
{
"arm": "pi",
"task": "fix-fib",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 22.87,
"model_calls": 5,
"input_tokens": 6612,
"cached_input_tokens": 2432,
"output_tokens": 149,
"list_usd": 0.010093
},
{
"arm": "pi",
"task": "git-ops",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 14.13,
"model_calls": 3,
"input_tokens": 3956,
"cached_input_tokens": 2176,
"output_tokens": 137,
"list_usd": 0.005148
},
{
"arm": "pi",
"task": "git-ops",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 15.88,
"model_calls": 3,
"input_tokens": 3971,
"cached_input_tokens": 2176,
"output_tokens": 143,
"list_usd": 0.005238
},
{
"arm": "pi",
"task": "git-ops",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 16.54,
"model_calls": 3,
"input_tokens": 3957,
"cached_input_tokens": 1024,
"output_tokens": 142,
"list_usd": 0.007388
},
{
"arm": "pi",
"task": "json-transform",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 15.03,
"model_calls": 3,
"input_tokens": 3663,
"cached_input_tokens": 1024,
"output_tokens": 94,
"list_usd": 0.00632
},
{
"arm": "pi",
"task": "json-transform",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 15.51,
"model_calls": 3,
"input_tokens": 3663,
"cached_input_tokens": 2048,
"output_tokens": 94,
"list_usd": 0.004375
},
{
"arm": "pi",
"task": "json-transform",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 15.01,
"model_calls": 3,
"input_tokens": 3663,
"cached_input_tokens": 0,
"output_tokens": 95,
"list_usd": 0.008276
},
{
"arm": "pi",
"task": "recall-noise",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 9.56,
"model_calls": 2,
"input_tokens": 2499,
"cached_input_tokens": 0,
"output_tokens": 28,
"list_usd": 0.005278
},
{
"arm": "pi",
"task": "recall-noise",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 9.73,
"model_calls": 2,
"input_tokens": 2499,
"cached_input_tokens": 0,
"output_tokens": 28,
"list_usd": 0.005278
},
{
"arm": "pi",
"task": "recall-noise",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 11.65,
"model_calls": 2,
"input_tokens": 2499,
"cached_input_tokens": 0,
"output_tokens": 28,
"list_usd": 0.005278
},
{
"arm": "pi",
"task": "refactor-rename",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 23.26,
"model_calls": 5,
"input_tokens": 7583,
"cached_input_tokens": 2688,
"output_tokens": 289,
"list_usd": 0.012949
},
{
"arm": "pi",
"task": "refactor-rename",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 24.13,
"model_calls": 5,
"input_tokens": 6947,
"cached_input_tokens": 1152,
"output_tokens": 281,
"list_usd": 0.014515
},
{
"arm": "pi",
"task": "refactor-rename",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 22.81,
"model_calls": 5,
"input_tokens": 7008,
"cached_input_tokens": 2432,
"output_tokens": 279,
"list_usd": 0.012185
},
{
"arm": "pi",
"task": "regex-count",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 11.5,
"model_calls": 2,
"input_tokens": 2361,
"cached_input_tokens": 1024,
"output_tokens": 55,
"list_usd": 0.003326
},
{
"arm": "pi",
"task": "regex-count",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 12.91,
"model_calls": 2,
"input_tokens": 2357,
"cached_input_tokens": 1024,
"output_tokens": 51,
"list_usd": 0.003278
},
{
"arm": "pi",
"task": "regex-count",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 15.61,
"model_calls": 2,
"input_tokens": 2357,
"cached_input_tokens": 1024,
"output_tokens": 51,
"list_usd": 0.003278
},
{
"arm": "pi",
"task": "write-tests",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 29.83,
"model_calls": 5,
"input_tokens": 6414,
"cached_input_tokens": 2048,
"output_tokens": 266,
"list_usd": 0.011597
},
{
"arm": "pi",
"task": "write-tests",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 22.82,
"model_calls": 5,
"input_tokens": 6278,
"cached_input_tokens": 3200,
"output_tokens": 196,
"list_usd": 0.008436
},
{
"arm": "pi",
"task": "write-tests",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 18.68,
"model_calls": 4,
"input_tokens": 5003,
"cached_input_tokens": 2048,
"output_tokens": 179,
"list_usd": 0.007905
}
]
}