graff — a fast agentic coding harness in Zig: multi-provider, MCP, workflows, DGM evolution loop, TS/Python SDKs
Python
{
"model": "one gateway model (same id in every arm)",
"tasks": 21,
"reps": 3,
"arms": {
"graff": "graff (ADR 0231 build)",
"opencode2": "OpenCode 2",
"pi": "pi"
},
"summary": {
"all": {
"graff": {
"runs": 63,
"passed": 63,
"wall_s_per_task": 9.1,
"cached_input_share": 0.945,
"output_tokens_per_task": 582,
"model_calls_per_task": 3.6
},
"opencode2": {
"runs": 63,
"passed": 62,
"wall_s_per_task": 14.4,
"cached_input_share": 0.862,
"output_tokens_per_task": 980,
"model_calls_per_task": 4.2
},
"pi": {
"runs": 33,
"passed": 33,
"wall_s_per_task": 5.4,
"cached_input_share": 0.785,
"output_tokens_per_task": 258,
"model_calls_per_task": 2.7
}
},
"core": {
"graff": {
"runs": 33,
"passed": 33,
"wall_s_per_task": 7.3,
"cached_input_share": 0.937,
"output_tokens_per_task": 277,
"model_calls_per_task": 2.8
},
"opencode2": {
"runs": 33,
"passed": 33,
"wall_s_per_task": 6.9,
"cached_input_share": 0.868,
"output_tokens_per_task": 366,
"model_calls_per_task": 3.2
},
"pi": {
"runs": 33,
"passed": 33,
"wall_s_per_task": 5.4,
"cached_input_share": 0.785,
"output_tokens_per_task": 258,
"model_calls_per_task": 2.7
}
},
"mcp": {
"graff": {
"runs": 30,
"passed": 30,
"wall_s_per_task": 11.0,
"cached_input_share": 0.95,
"output_tokens_per_task": 917,
"model_calls_per_task": 4.4
},
"opencode2": {
"runs": 30,
"passed": 29,
"wall_s_per_task": 22.6,
"cached_input_share": 0.858,
"output_tokens_per_task": 1655,
"model_calls_per_task": 5.3
}
}
},
"per_task": {
"csv-sum": {
"suite": "core",
"graff": {
"passed": 3,
"runs": 3,
"wall_s": 9.5,
"model_calls": 3.3
},
"opencode2": {
"passed": 3,
"runs": 3,
"wall_s": 7.8,
"model_calls": 3
},
"pi": {
"passed": 3,
"runs": 3,
"wall_s": 5.0,
"model_calls": 3
}
},
"dead-code": {
"suite": "core",
"graff": {
"passed": 3,
"runs": 3,
"wall_s": 6.9,
"model_calls": 3
},
"opencode2": {
"passed": 3,
"runs": 3,
"wall_s": 6.8,
"model_calls": 4
},
"pi": {
"passed": 3,
"runs": 3,
"wall_s": 6.0,
"model_calls": 3
}
},
"exact-reply": {
"suite": "core",
"graff": {
"passed": 3,
"runs": 3,
"wall_s": 1.8,
"model_calls": 1
},
"opencode2": {
"passed": 3,
"runs": 3,
"wall_s": 2.1,
"model_calls": 1
},
"pi": {
"passed": 3,
"runs": 3,
"wall_s": 1.7,
"model_calls": 1
}
},
"file-ops": {
"suite": "core",
"graff": {
"passed": 3,
"runs": 3,
"wall_s": 4.2,
"model_calls": 2
},
"opencode2": {
"passed": 3,
"runs": 3,
"wall_s": 5.9,
"model_calls": 3
},
"pi": {
"passed": 3,
"runs": 3,
"wall_s": 4.6,
"model_calls": 2
}
},
"fix-fib": {
"suite": "core",
"graff": {
"passed": 3,
"runs": 3,
"wall_s": 8.1,
"model_calls": 4
},
"opencode2": {
"passed": 3,
"runs": 3,
"wall_s": 7.6,
"model_calls": 4
},
"pi": {
"passed": 3,
"runs": 3,
"wall_s": 7.6,
"model_calls": 4
}
},
"git-ops": {
"suite": "core",
"graff": {
"passed": 3,
"runs": 3,
"wall_s": 6.1,
"model_calls": 2
},
"opencode2": {
"passed": 3,
"runs": 3,
"wall_s": 6.2,
"model_calls": 3
},
"pi": {
"passed": 3,
"runs": 3,
"wall_s": 4.6,
"model_calls": 2
}
},
"json-transform": {
"suite": "core",
"graff": {
"passed": 3,
"runs": 3,
"wall_s": 5.5,
"model_calls": 3
},
"opencode2": {
"passed": 3,
"runs": 3,
"wall_s": 6.0,
"model_calls": 3
},
"pi": {
"passed": 3,
"runs": 3,
"wall_s": 5.6,
"model_calls": 3
}
},
"linear-issues": {
"suite": "mcp",
"graff": {
"passed": 3,
"runs": 3,
"wall_s": 7.5,
"model_calls": 3
},
"opencode2": {
"passed": 3,
"runs": 3,
"wall_s": 30.4,
"model_calls": 6
}
},
"linear-nohint": {
"suite": "mcp",
"graff": {
"passed": 3,
"runs": 3,
"wall_s": 10.0,
"model_calls": 4
},
"opencode2": {
"passed": 3,
"runs": 3,
"wall_s": 20.2,
"model_calls": 5.3
}
},
"linear-nohint-warm": {
"suite": "mcp",
"graff": {
"passed": 3,
"runs": 3,
"wall_s": 9.9,
"model_calls": 3.3
},
"opencode2": {
"passed": 3,
"runs": 3,
"wall_s": 20.4,
"model_calls": 5.7
}
},
"linear-quiet": {
"suite": "mcp",
"graff": {
"passed": 3,
"runs": 3,
"wall_s": 11.3,
"model_calls": 4
},
"opencode2": {
"passed": 3,
"runs": 3,
"wall_s": 20.2,
"model_calls": 5.7
}
},
"linear-quiet-warm": {
"suite": "mcp",
"graff": {
"passed": 3,
"runs": 3,
"wall_s": 15.7,
"model_calls": 5
},
"opencode2": {
"passed": 3,
"runs": 3,
"wall_s": 21.5,
"model_calls": 5
}
},
"linear-reduce": {
"suite": "mcp",
"graff": {
"passed": 3,
"runs": 3,
"wall_s": 9.5,
"model_calls": 4
},
"opencode2": {
"passed": 3,
"runs": 3,
"wall_s": 20.9,
"model_calls": 5.7
}
},
"linear-reduce-warm": {
"suite": "mcp",
"graff": {
"passed": 3,
"runs": 3,
"wall_s": 8.8,
"model_calls": 4
},
"opencode2": {
"passed": 3,
"runs": 3,
"wall_s": 19.2,
"model_calls": 5.7
}
},
"linear-sidecar": {
"suite": "mcp",
"graff": {
"passed": 3,
"runs": 3,
"wall_s": 8.9,
"model_calls": 3.7
},
"opencode2": {
"passed": 3,
"runs": 3,
"wall_s": 22.5,
"model_calls": 6
}
},
"linear-split": {
"suite": "mcp",
"graff": {
"passed": 3,
"runs": 3,
"wall_s": 18.9,
"model_calls": 9
},
"opencode2": {
"passed": 2,
"runs": 3,
"wall_s": 30.0,
"model_calls": 2.7
}
},
"linear-warm": {
"suite": "mcp",
"graff": {
"passed": 3,
"runs": 3,
"wall_s": 9.1,
"model_calls": 3.7
},
"opencode2": {
"passed": 3,
"runs": 3,
"wall_s": 20.6,
"model_calls": 5.7
}
},
"recall-noise": {
"suite": "core",
"graff": {
"passed": 3,
"runs": 3,
"wall_s": 7.8,
"model_calls": 2
},
"opencode2": {
"passed": 3,
"runs": 3,
"wall_s": 7.6,
"model_calls": 3
},
"pi": {
"passed": 3,
"runs": 3,
"wall_s": 4.4,
"model_calls": 2
}
},
"refactor-rename": {
"suite": "core",
"graff": {
"passed": 3,
"runs": 3,
"wall_s": 10.3,
"model_calls": 5
},
"opencode2": {
"passed": 3,
"runs": 3,
"wall_s": 8.6,
"model_calls": 4
},
"pi": {
"passed": 3,
"runs": 3,
"wall_s": 7.8,
"model_calls": 4
}
},
"regex-count": {
"suite": "core",
"graff": {
"passed": 3,
"runs": 3,
"wall_s": 14.1,
"model_calls": 3
},
"opencode2": {
"passed": 3,
"runs": 3,
"wall_s": 8.6,
"model_calls": 3.3
},
"pi": {
"passed": 3,
"runs": 3,
"wall_s": 6.0,
"model_calls": 3
}
},
"write-tests": {
"suite": "core",
"graff": {
"passed": 3,
"runs": 3,
"wall_s": 6.6,
"model_calls": 3
},
"opencode2": {
"passed": 3,
"runs": 3,
"wall_s": 8.4,
"model_calls": 4
},
"pi": {
"passed": 3,
"runs": 3,
"wall_s": 6.1,
"model_calls": 3
}
}
},
"runs": [
{
"arm": "graff",
"task": "csv-sum",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 7.9,
"model_calls": 3,
"input_tokens": 27883,
"cached_input_tokens": 22546,
"output_tokens": 311
},
{
"arm": "graff",
"task": "csv-sum",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 8.23,
"model_calls": 3,
"input_tokens": 27912,
"cached_input_tokens": 27756,
"output_tokens": 355
},
{
"arm": "graff",
"task": "csv-sum",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 12.24,
"model_calls": 4,
"input_tokens": 37348,
"cached_input_tokens": 37159,
"output_tokens": 471
},
{
"arm": "graff",
"task": "dead-code",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 6.92,
"model_calls": 3,
"input_tokens": 28218,
"cached_input_tokens": 22632,
"output_tokens": 160
},
{
"arm": "graff",
"task": "dead-code",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 4.44,
"model_calls": 3,
"input_tokens": 28218,
"cached_input_tokens": 28210,
"output_tokens": 160
},
{
"arm": "graff",
"task": "dead-code",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 9.36,
"model_calls": 3,
"input_tokens": 28228,
"cached_input_tokens": 27748,
"output_tokens": 165
},
{
"arm": "graff",
"task": "exact-reply",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 1.89,
"model_calls": 1,
"input_tokens": 9121,
"cached_input_tokens": 9117,
"output_tokens": 51
},
{
"arm": "graff",
"task": "exact-reply",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 1.75,
"model_calls": 1,
"input_tokens": 9121,
"cached_input_tokens": 9117,
"output_tokens": 51
},
{
"arm": "graff",
"task": "exact-reply",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 1.76,
"model_calls": 1,
"input_tokens": 9121,
"cached_input_tokens": 9117,
"output_tokens": 51
},
{
"arm": "graff",
"task": "file-ops",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 4.93,
"model_calls": 2,
"input_tokens": 18395,
"cached_input_tokens": 18186,
"output_tokens": 143
},
{
"arm": "graff",
"task": "file-ops",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 4.22,
"model_calls": 2,
"input_tokens": 18395,
"cached_input_tokens": 18389,
"output_tokens": 141
},
{
"arm": "graff",
"task": "file-ops",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 3.39,
"model_calls": 2,
"input_tokens": 18395,
"cached_input_tokens": 18389,
"output_tokens": 158
},
{
"arm": "graff",
"task": "fix-fib",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 7.41,
"model_calls": 4,
"input_tokens": 38412,
"cached_input_tokens": 32526,
"output_tokens": 394
},
{
"arm": "graff",
"task": "fix-fib",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 8.11,
"model_calls": 4,
"input_tokens": null,
"cached_input_tokens": null,
"output_tokens": null
},
{
"arm": "graff",
"task": "fix-fib",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 8.65,
"model_calls": 4,
"input_tokens": 38412,
"cached_input_tokens": 37647,
"output_tokens": 407
},
{
"arm": "graff",
"task": "git-ops",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 4.24,
"model_calls": 2,
"input_tokens": 18596,
"cached_input_tokens": 13279,
"output_tokens": 258
},
{
"arm": "graff",
"task": "git-ops",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 4.27,
"model_calls": 2,
"input_tokens": 18590,
"cached_input_tokens": 18398,
"output_tokens": 240
},
{
"arm": "graff",
"task": "git-ops",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 9.77,
"model_calls": 2,
"input_tokens": 18595,
"cached_input_tokens": 18398,
"output_tokens": 264
},
{
"arm": "graff",
"task": "json-transform",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 5.44,
"model_calls": 3,
"input_tokens": 27943,
"cached_input_tokens": 22578,
"output_tokens": 254
},
{
"arm": "graff",
"task": "json-transform",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 5.77,
"model_calls": 3,
"input_tokens": 27943,
"cached_input_tokens": 27935,
"output_tokens": 261
},
{
"arm": "graff",
"task": "json-transform",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 5.41,
"model_calls": 3,
"input_tokens": 27943,
"cached_input_tokens": 27935,
"output_tokens": 256
},
{
"arm": "graff",
"task": "linear-issues",
"suite": "mcp",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 7.63,
"model_calls": 3,
"input_tokens": 30744,
"cached_input_tokens": 20170,
"output_tokens": 636
},
{
"arm": "graff",
"task": "linear-issues",
"suite": "mcp",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 7.05,
"model_calls": 3,
"input_tokens": 30882,
"cached_input_tokens": 30536,
"output_tokens": 611
},
{
"arm": "graff",
"task": "linear-issues",
"suite": "mcp",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 7.81,
"model_calls": 3,
"input_tokens": 30934,
"cached_input_tokens": 30536,
"output_tokens": 647
},
{
"arm": "graff",
"task": "linear-nohint",
"suite": "mcp",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 9.38,
"model_calls": 4,
"input_tokens": 41204,
"cached_input_tokens": 39485,
"output_tokens": 907
},
{
"arm": "graff",
"task": "linear-nohint",
"suite": "mcp",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 10.07,
"model_calls": 4,
"input_tokens": 41193,
"cached_input_tokens": 40898,
"output_tokens": 851
},
{
"arm": "graff",
"task": "linear-nohint",
"suite": "mcp",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 10.6,
"model_calls": 4,
"input_tokens": 41231,
"cached_input_tokens": 40166,
"output_tokens": 853
},
{
"arm": "graff",
"task": "linear-nohint-warm",
"suite": "mcp",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 12.69,
"model_calls": 4,
"input_tokens": 42833,
"cached_input_tokens": 40824,
"output_tokens": 676
},
{
"arm": "graff",
"task": "linear-nohint-warm",
"suite": "mcp",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 8.45,
"model_calls": 3,
"input_tokens": 32072,
"cached_input_tokens": 31503,
"output_tokens": 993
},
{
"arm": "graff",
"task": "linear-nohint-warm",
"suite": "mcp",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 8.64,
"model_calls": 3,
"input_tokens": 32072,
"cached_input_tokens": 32064,
"output_tokens": 922
},
{
"arm": "graff",
"task": "linear-quiet",
"suite": "mcp",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 7.98,
"model_calls": 4,
"input_tokens": 41013,
"cached_input_tokens": 39412,
"output_tokens": 594
},
{
"arm": "graff",
"task": "linear-quiet",
"suite": "mcp",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 17.67,
"model_calls": 4,
"input_tokens": 41013,
"cached_input_tokens": 40808,
"output_tokens": 617
},
{
"arm": "graff",
"task": "linear-quiet",
"suite": "mcp",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 8.12,
"model_calls": 4,
"input_tokens": 41013,
"cached_input_tokens": 41003,
"output_tokens": 580
},
{
"arm": "graff",
"task": "linear-quiet-warm",
"suite": "mcp",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 27.89,
"model_calls": 7,
"input_tokens": 78474,
"cached_input_tokens": 75338,
"output_tokens": 1305
},
{
"arm": "graff",
"task": "linear-quiet-warm",
"suite": "mcp",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 10.06,
"model_calls": 4,
"input_tokens": 43098,
"cached_input_tokens": 42796,
"output_tokens": 653
},
{
"arm": "graff",
"task": "linear-quiet-warm",
"suite": "mcp",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 9.08,
"model_calls": 4,
"input_tokens": 43040,
"cached_input_tokens": 42395,
"output_tokens": 679
},
{
"arm": "graff",
"task": "linear-reduce",
"suite": "mcp",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 11.39,
"model_calls": 4,
"input_tokens": 41229,
"cached_input_tokens": 39545,
"output_tokens": 774
},
{
"arm": "graff",
"task": "linear-reduce",
"suite": "mcp",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 8.94,
"model_calls": 4,
"input_tokens": 41245,
"cached_input_tokens": 40485,
"output_tokens": 616
},
{
"arm": "graff",
"task": "linear-reduce",
"suite": "mcp",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 8.18,
"model_calls": 4,
"input_tokens": 41252,
"cached_input_tokens": 40488,
"output_tokens": 596
},
{
"arm": "graff",
"task": "linear-reduce-warm",
"suite": "mcp",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 9.74,
"model_calls": 4,
"input_tokens": 42823,
"cached_input_tokens": 40749,
"output_tokens": 639
},
{
"arm": "graff",
"task": "linear-reduce-warm",
"suite": "mcp",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 8.15,
"model_calls": 4,
"input_tokens": 42815,
"cached_input_tokens": 41901,
"output_tokens": 548
},
{
"arm": "graff",
"task": "linear-reduce-warm",
"suite": "mcp",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 8.54,
"model_calls": 4,
"input_tokens": 42815,
"cached_input_tokens": 42805,
"output_tokens": 622
},
{
"arm": "graff",
"task": "linear-sidecar",
"suite": "mcp",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 9.51,
"model_calls": 4,
"input_tokens": 49605,
"cached_input_tokens": 45782,
"output_tokens": 912
},
{
"arm": "graff",
"task": "linear-sidecar",
"suite": "mcp",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 8.25,
"model_calls": 3,
"input_tokens": 37300,
"cached_input_tokens": 36266,
"output_tokens": 896
},
{
"arm": "graff",
"task": "linear-sidecar",
"suite": "mcp",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 8.99,
"model_calls": 4,
"input_tokens": 49603,
"cached_input_tokens": 48530,
"output_tokens": 875
},
{
"arm": "graff",
"task": "linear-split",
"suite": "mcp",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 16.16,
"model_calls": 8,
"input_tokens": 77647,
"cached_input_tokens": 59430,
"output_tokens": 2062
},
{
"arm": "graff",
"task": "linear-split",
"suite": "mcp",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 17.19,
"model_calls": 8,
"input_tokens": 77930,
"cached_input_tokens": 74039,
"output_tokens": 2168
},
{
"arm": "graff",
"task": "linear-split",
"suite": "mcp",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 23.48,
"model_calls": 11,
"input_tokens": 104516,
"cached_input_tokens": 95787,
"output_tokens": 2901
},
{
"arm": "graff",
"task": "linear-warm",
"suite": "mcp",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 9.33,
"model_calls": 4,
"input_tokens": 43066,
"cached_input_tokens": 40995,
"output_tokens": 698
},
{
"arm": "graff",
"task": "linear-warm",
"suite": "mcp",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 9.63,
"model_calls": 4,
"input_tokens": 43097,
"cached_input_tokens": 42654,
"output_tokens": 719
},
{
"arm": "graff",
"task": "linear-warm",
"suite": "mcp",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 8.39,
"model_calls": 3,
"input_tokens": 32244,
"cached_input_tokens": 31674,
"output_tokens": 953
},
{
"arm": "graff",
"task": "recall-noise",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 16.2,
"model_calls": 2,
"input_tokens": 18747,
"cached_input_tokens": 13253,
"output_tokens": 63
},
{
"arm": "graff",
"task": "recall-noise",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 3.56,
"model_calls": 2,
"input_tokens": 18747,
"cached_input_tokens": 18741,
"output_tokens": 110
},
{
"arm": "graff",
"task": "recall-noise",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 3.58,
"model_calls": 2,
"input_tokens": 18747,
"cached_input_tokens": 18741,
"output_tokens": 110
},
{
"arm": "graff",
"task": "refactor-rename",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 9.61,
"model_calls": 5,
"input_tokens": 47904,
"cached_input_tokens": 42043,
"output_tokens": 555
},
{
"arm": "graff",
"task": "refactor-rename",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 11.28,
"model_calls": 5,
"input_tokens": 48174,
"cached_input_tokens": 47635,
"output_tokens": 639
},
{
"arm": "graff",
"task": "refactor-rename",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 9.99,
"model_calls": 5,
"input_tokens": 47904,
"cached_input_tokens": 47892,
"output_tokens": 525
},
{
"arm": "graff",
"task": "regex-count",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 20.91,
"model_calls": 3,
"input_tokens": 28150,
"cached_input_tokens": 22704,
"output_tokens": 505
},
{
"arm": "graff",
"task": "regex-count",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 10.03,
"model_calls": 3,
"input_tokens": 28165,
"cached_input_tokens": 27851,
"output_tokens": 365
},
{
"arm": "graff",
"task": "regex-count",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 11.32,
"model_calls": 3,
"input_tokens": 28061,
"cached_input_tokens": 27784,
"output_tokens": 404
},
{
"arm": "graff",
"task": "write-tests",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 6.49,
"model_calls": 3,
"input_tokens": 28080,
"cached_input_tokens": 22556,
"output_tokens": 446
},
{
"arm": "graff",
"task": "write-tests",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 7.22,
"model_calls": 3,
"input_tokens": 28080,
"cached_input_tokens": 28072,
"output_tokens": 420
},
{
"arm": "graff",
"task": "write-tests",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 6.13,
"model_calls": 3,
"input_tokens": 28080,
"cached_input_tokens": 28072,
"output_tokens": 444
},
{
"arm": "opencode2",
"task": "csv-sum",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 10.68,
"model_calls": 3,
"input_tokens": 23865,
"cached_input_tokens": 20630,
"output_tokens": 311
},
{
"arm": "opencode2",
"task": "csv-sum",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 6.96,
"model_calls": 3,
"input_tokens": 23865,
"cached_input_tokens": 20671,
"output_tokens": 319
},
{
"arm": "opencode2",
"task": "csv-sum",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 5.9,
"model_calls": 3,
"input_tokens": 23865,
"cached_input_tokens": 20671,
"output_tokens": 304
},
{
"arm": "opencode2",
"task": "dead-code",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 6.91,
"model_calls": 4,
"input_tokens": 32597,
"cached_input_tokens": 28619,
"output_tokens": 345
},
{
"arm": "opencode2",
"task": "dead-code",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 7.53,
"model_calls": 4,
"input_tokens": 32305,
"cached_input_tokens": 28671,
"output_tokens": 289
},
{
"arm": "opencode2",
"task": "dead-code",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 5.97,
"model_calls": 4,
"input_tokens": 32305,
"cached_input_tokens": 28671,
"output_tokens": 289
},
{
"arm": "opencode2",
"task": "exact-reply",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 2.49,
"model_calls": 1,
"input_tokens": 8280,
"cached_input_tokens": 5436,
"output_tokens": 13
},
{
"arm": "opencode2",
"task": "exact-reply",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 1.75,
"model_calls": 1,
"input_tokens": 7484,
"cached_input_tokens": 4644,
"output_tokens": 4
},
{
"arm": "opencode2",
"task": "exact-reply",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 1.96,
"model_calls": 1,
"input_tokens": 7484,
"cached_input_tokens": 4644,
"output_tokens": 4
},
{
"arm": "opencode2",
"task": "file-ops",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 6.73,
"model_calls": 3,
"input_tokens": 23691,
"cached_input_tokens": 20568,
"output_tokens": 255
},
{
"arm": "opencode2",
"task": "file-ops",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 5.77,
"model_calls": 3,
"input_tokens": 23744,
"cached_input_tokens": 20614,
"output_tokens": 282
},
{
"arm": "opencode2",
"task": "file-ops",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 5.21,
"model_calls": 3,
"input_tokens": 23857,
"cached_input_tokens": 20614,
"output_tokens": 295
},
{
"arm": "opencode2",
"task": "fix-fib",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 8.21,
"model_calls": 4,
"input_tokens": 32294,
"cached_input_tokens": 28798,
"output_tokens": 345
},
{
"arm": "opencode2",
"task": "fix-fib",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 6.74,
"model_calls": 4,
"input_tokens": 32294,
"cached_input_tokens": 28857,
"output_tokens": 358
},
{
"arm": "opencode2",
"task": "fix-fib",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 7.86,
"model_calls": 4,
"input_tokens": 32294,
"cached_input_tokens": 28857,
"output_tokens": 358
},
{
"arm": "opencode2",
"task": "git-ops",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 6.65,
"model_calls": 3,
"input_tokens": 23234,
"cached_input_tokens": 19849,
"output_tokens": 319
},
{
"arm": "opencode2",
"task": "git-ops",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 6.05,
"model_calls": 3,
"input_tokens": 24079,
"cached_input_tokens": 20691,
"output_tokens": 353
},
{
"arm": "opencode2",
"task": "git-ops",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 5.79,
"model_calls": 3,
"input_tokens": 24083,
"cached_input_tokens": 20687,
"output_tokens": 330
},
{
"arm": "opencode2",
"task": "json-transform",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 5.75,
"model_calls": 3,
"input_tokens": 23966,
"cached_input_tokens": 20665,
"output_tokens": 378
},
{
"arm": "opencode2",
"task": "json-transform",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 6.12,
"model_calls": 3,
"input_tokens": 23966,
"cached_input_tokens": 20719,
"output_tokens": 349
},
{
"arm": "opencode2",
"task": "json-transform",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 6.07,
"model_calls": 3,
"input_tokens": 23966,
"cached_input_tokens": 20719,
"output_tokens": 391
},
{
"arm": "opencode2",
"task": "linear-issues",
"suite": "mcp",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 39.11,
"model_calls": 6,
"input_tokens": 64184,
"cached_input_tokens": 55752,
"output_tokens": 1632
},
{
"arm": "opencode2",
"task": "linear-issues",
"suite": "mcp",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 27.91,
"model_calls": 6,
"input_tokens": 66506,
"cached_input_tokens": 57714,
"output_tokens": 1849
},
{
"arm": "opencode2",
"task": "linear-issues",
"suite": "mcp",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 24.19,
"model_calls": 6,
"input_tokens": 67866,
"cached_input_tokens": 58508,
"output_tokens": 1966
},
{
"arm": "opencode2",
"task": "linear-nohint",
"suite": "mcp",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 19.28,
"model_calls": 5,
"input_tokens": 55723,
"cached_input_tokens": 46086,
"output_tokens": 1773
},
{
"arm": "opencode2",
"task": "linear-nohint",
"suite": "mcp",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 20.57,
"model_calls": 6,
"input_tokens": 67456,
"cached_input_tokens": 58096,
"output_tokens": 1759
},
{
"arm": "opencode2",
"task": "linear-nohint",
"suite": "mcp",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 20.79,
"model_calls": 5,
"input_tokens": 55608,
"cached_input_tokens": 46175,
"output_tokens": 1693
},
{
"arm": "opencode2",
"task": "linear-nohint-warm",
"suite": "mcp",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 25.38,
"model_calls": 6,
"input_tokens": 67190,
"cached_input_tokens": 57961,
"output_tokens": 1782
},
{
"arm": "opencode2",
"task": "linear-nohint-warm",
"suite": "mcp",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 16.84,
"model_calls": 5,
"input_tokens": 53445,
"cached_input_tokens": 45015,
"output_tokens": 1547
},
{
"arm": "opencode2",
"task": "linear-nohint-warm",
"suite": "mcp",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 18.92,
"model_calls": 6,
"input_tokens": 66914,
"cached_input_tokens": 57976,
"output_tokens": 1719
},
{
"arm": "opencode2",
"task": "linear-quiet",
"suite": "mcp",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 22.99,
"model_calls": 6,
"input_tokens": 58611,
"cached_input_tokens": 51656,
"output_tokens": 1652
},
{
"arm": "opencode2",
"task": "linear-quiet",
"suite": "mcp",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 17.81,
"model_calls": 5,
"input_tokens": 47567,
"cached_input_tokens": 41313,
"output_tokens": 1653
},
{
"arm": "opencode2",
"task": "linear-quiet",
"suite": "mcp",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 19.95,
"model_calls": 6,
"input_tokens": 58287,
"cached_input_tokens": 51569,
"output_tokens": 1867
},
{
"arm": "opencode2",
"task": "linear-quiet-warm",
"suite": "mcp",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 15.55,
"model_calls": 4,
"input_tokens": 36901,
"cached_input_tokens": 31489,
"output_tokens": 1737
},
{
"arm": "opencode2",
"task": "linear-quiet-warm",
"suite": "mcp",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 27.03,
"model_calls": 5,
"input_tokens": 54234,
"cached_input_tokens": 44605,
"output_tokens": 1786
},
{
"arm": "opencode2",
"task": "linear-quiet-warm",
"suite": "mcp",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 22.01,
"model_calls": 6,
"input_tokens": 59377,
"cached_input_tokens": 52389,
"output_tokens": 1848
},
{
"arm": "opencode2",
"task": "linear-reduce",
"suite": "mcp",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 20.74,
"model_calls": 6,
"input_tokens": 60410,
"cached_input_tokens": 52832,
"output_tokens": 1888
},
{
"arm": "opencode2",
"task": "linear-reduce",
"suite": "mcp",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 20.63,
"model_calls": 6,
"input_tokens": 66525,
"cached_input_tokens": 57121,
"output_tokens": 1891
},
{
"arm": "opencode2",
"task": "linear-reduce",
"suite": "mcp",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 21.28,
"model_calls": 5,
"input_tokens": 48244,
"cached_input_tokens": 41797,
"output_tokens": 2266
},
{
"arm": "opencode2",
"task": "linear-reduce-warm",
"suite": "mcp",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 16.98,
"model_calls": 5,
"input_tokens": 47381,
"cached_input_tokens": 40951,
"output_tokens": 1725
},
{
"arm": "opencode2",
"task": "linear-reduce-warm",
"suite": "mcp",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 18.83,
"model_calls": 6,
"input_tokens": 57363,
"cached_input_tokens": 50959,
"output_tokens": 1764
},
{
"arm": "opencode2",
"task": "linear-reduce-warm",
"suite": "mcp",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 21.87,
"model_calls": 6,
"input_tokens": 58147,
"cached_input_tokens": 51586,
"output_tokens": 1819
},
{
"arm": "opencode2",
"task": "linear-sidecar",
"suite": "mcp",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 21.1,
"model_calls": 6,
"input_tokens": 65921,
"cached_input_tokens": 57064,
"output_tokens": 1761
},
{
"arm": "opencode2",
"task": "linear-sidecar",
"suite": "mcp",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 21.24,
"model_calls": 6,
"input_tokens": 65529,
"cached_input_tokens": 57447,
"output_tokens": 1833
},
{
"arm": "opencode2",
"task": "linear-sidecar",
"suite": "mcp",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 25.05,
"model_calls": 6,
"input_tokens": 65650,
"cached_input_tokens": 57132,
"output_tokens": 1712
},
{
"arm": "opencode2",
"task": "linear-split",
"suite": "mcp",
"rep": 1,
"pass": false,
"timed_out": false,
"wall_s": 42.55,
"model_calls": 3,
"input_tokens": 13805,
"cached_input_tokens": 6430,
"output_tokens": 291
},
{
"arm": "opencode2",
"task": "linear-split",
"suite": "mcp",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 24.9,
"model_calls": 3,
"input_tokens": 21079,
"cached_input_tokens": 16530,
"output_tokens": 583
},
{
"arm": "opencode2",
"task": "linear-split",
"suite": "mcp",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 22.6,
"model_calls": 2,
"input_tokens": 13327,
"cached_input_tokens": 9691,
"output_tokens": 355
},
{
"arm": "opencode2",
"task": "linear-warm",
"suite": "mcp",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 22.22,
"model_calls": 6,
"input_tokens": 67825,
"cached_input_tokens": 58324,
"output_tokens": 1803
},
{
"arm": "opencode2",
"task": "linear-warm",
"suite": "mcp",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 22.32,
"model_calls": 6,
"input_tokens": 66147,
"cached_input_tokens": 57536,
"output_tokens": 1984
},
{
"arm": "opencode2",
"task": "linear-warm",
"suite": "mcp",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 17.27,
"model_calls": 5,
"input_tokens": 53869,
"cached_input_tokens": 45387,
"output_tokens": 1714
},
{
"arm": "opencode2",
"task": "recall-noise",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 11.66,
"model_calls": 3,
"input_tokens": 24029,
"cached_input_tokens": 20531,
"output_tokens": 167
},
{
"arm": "opencode2",
"task": "recall-noise",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 5.56,
"model_calls": 3,
"input_tokens": 24029,
"cached_input_tokens": 20567,
"output_tokens": 167
},
{
"arm": "opencode2",
"task": "recall-noise",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 5.48,
"model_calls": 3,
"input_tokens": 23211,
"cached_input_tokens": 19753,
"output_tokens": 153
},
{
"arm": "opencode2",
"task": "refactor-rename",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 8.12,
"model_calls": 4,
"input_tokens": 32966,
"cached_input_tokens": 29034,
"output_tokens": 730
},
{
"arm": "opencode2",
"task": "refactor-rename",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 9.01,
"model_calls": 4,
"input_tokens": 32969,
"cached_input_tokens": 29076,
"output_tokens": 801
},
{
"arm": "opencode2",
"task": "refactor-rename",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 8.68,
"model_calls": 4,
"input_tokens": 32998,
"cached_input_tokens": 29120,
"output_tokens": 737
},
{
"arm": "opencode2",
"task": "regex-count",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 9.25,
"model_calls": 3,
"input_tokens": 24358,
"cached_input_tokens": 20847,
"output_tokens": 496
},
{
"arm": "opencode2",
"task": "regex-count",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 8.45,
"model_calls": 3,
"input_tokens": 24207,
"cached_input_tokens": 20897,
"output_tokens": 401
},
{
"arm": "opencode2",
"task": "regex-count",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 7.99,
"model_calls": 4,
"input_tokens": 32621,
"cached_input_tokens": 29005,
"output_tokens": 619
},
{
"arm": "opencode2",
"task": "write-tests",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 8.5,
"model_calls": 4,
"input_tokens": 32008,
"cached_input_tokens": 28378,
"output_tokens": 655
},
{
"arm": "opencode2",
"task": "write-tests",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 7.73,
"model_calls": 4,
"input_tokens": 31962,
"cached_input_tokens": 28438,
"output_tokens": 605
},
{
"arm": "opencode2",
"task": "write-tests",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 9.01,
"model_calls": 4,
"input_tokens": 32008,
"cached_input_tokens": 28438,
"output_tokens": 640
},
{
"arm": "pi",
"task": "csv-sum",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 4.9,
"model_calls": 3,
"input_tokens": 7244,
"cached_input_tokens": 5896,
"output_tokens": 262
},
{
"arm": "pi",
"task": "csv-sum",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 4.53,
"model_calls": 3,
"input_tokens": 7223,
"cached_input_tokens": 5896,
"output_tokens": 215
},
{
"arm": "pi",
"task": "csv-sum",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 5.54,
"model_calls": 3,
"input_tokens": 7223,
"cached_input_tokens": 5896,
"output_tokens": 235
},
{
"arm": "pi",
"task": "dead-code",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 6.25,
"model_calls": 3,
"input_tokens": 7488,
"cached_input_tokens": 5970,
"output_tokens": 175
},
{
"arm": "pi",
"task": "dead-code",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 6.43,
"model_calls": 3,
"input_tokens": 8379,
"cached_input_tokens": 5999,
"output_tokens": 291
},
{
"arm": "pi",
"task": "dead-code",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 5.19,
"model_calls": 3,
"input_tokens": 7572,
"cached_input_tokens": 5991,
"output_tokens": 238
},
{
"arm": "pi",
"task": "exact-reply",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 1.77,
"model_calls": 1,
"input_tokens": 2271,
"cached_input_tokens": 1190,
"output_tokens": 4
},
{
"arm": "pi",
"task": "exact-reply",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 1.64,
"model_calls": 1,
"input_tokens": 2271,
"cached_input_tokens": 1190,
"output_tokens": 4
},
{
"arm": "pi",
"task": "exact-reply",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 1.82,
"model_calls": 1,
"input_tokens": 2271,
"cached_input_tokens": 1190,
"output_tokens": 4
},
{
"arm": "pi",
"task": "file-ops",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 6.38,
"model_calls": 2,
"input_tokens": 4746,
"cached_input_tokens": 3489,
"output_tokens": 163
},
{
"arm": "pi",
"task": "file-ops",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 3.95,
"model_calls": 2,
"input_tokens": 4746,
"cached_input_tokens": 3489,
"output_tokens": 169
},
{
"arm": "pi",
"task": "file-ops",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 3.57,
"model_calls": 2,
"input_tokens": 4746,
"cached_input_tokens": 3489,
"output_tokens": 168
},
{
"arm": "pi",
"task": "fix-fib",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 7.84,
"model_calls": 4,
"input_tokens": 10457,
"cached_input_tokens": 8722,
"output_tokens": 434
},
{
"arm": "pi",
"task": "fix-fib",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 7.13,
"model_calls": 4,
"input_tokens": 10704,
"cached_input_tokens": 8892,
"output_tokens": 417
},
{
"arm": "pi",
"task": "fix-fib",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 7.78,
"model_calls": 4,
"input_tokens": 10710,
"cached_input_tokens": 8892,
"output_tokens": 428
},
{
"arm": "pi",
"task": "git-ops",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 4.73,
"model_calls": 2,
"input_tokens": 5009,
"cached_input_tokens": 3507,
"output_tokens": 289
},
{
"arm": "pi",
"task": "git-ops",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 4.53,
"model_calls": 2,
"input_tokens": 5009,
"cached_input_tokens": 3507,
"output_tokens": 257
},
{
"arm": "pi",
"task": "git-ops",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 4.51,
"model_calls": 2,
"input_tokens": 5009,
"cached_input_tokens": 3507,
"output_tokens": 254
},
{
"arm": "pi",
"task": "json-transform",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 5.42,
"model_calls": 3,
"input_tokens": 7314,
"cached_input_tokens": 5927,
"output_tokens": 205
},
{
"arm": "pi",
"task": "json-transform",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 5.51,
"model_calls": 3,
"input_tokens": 7298,
"cached_input_tokens": 5927,
"output_tokens": 249
},
{
"arm": "pi",
"task": "json-transform",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 5.76,
"model_calls": 3,
"input_tokens": 7341,
"cached_input_tokens": 5927,
"output_tokens": 261
},
{
"arm": "pi",
"task": "recall-noise",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 7.43,
"model_calls": 2,
"input_tokens": 4979,
"cached_input_tokens": 3480,
"output_tokens": 61
},
{
"arm": "pi",
"task": "recall-noise",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 2.89,
"model_calls": 2,
"input_tokens": 4979,
"cached_input_tokens": 3480,
"output_tokens": 61
},
{
"arm": "pi",
"task": "recall-noise",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 2.91,
"model_calls": 2,
"input_tokens": 4979,
"cached_input_tokens": 3480,
"output_tokens": 61
},
{
"arm": "pi",
"task": "refactor-rename",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 7.98,
"model_calls": 4,
"input_tokens": 10202,
"cached_input_tokens": 8555,
"output_tokens": 535
},
{
"arm": "pi",
"task": "refactor-rename",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 7.92,
"model_calls": 4,
"input_tokens": 10185,
"cached_input_tokens": 8555,
"output_tokens": 501
},
{
"arm": "pi",
"task": "refactor-rename",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 7.41,
"model_calls": 4,
"input_tokens": 10202,
"cached_input_tokens": 8589,
"output_tokens": 420
},
{
"arm": "pi",
"task": "regex-count",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 6.3,
"model_calls": 3,
"input_tokens": 7414,
"cached_input_tokens": 6012,
"output_tokens": 343
},
{
"arm": "pi",
"task": "regex-count",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 5.5,
"model_calls": 3,
"input_tokens": 7433,
"cached_input_tokens": 6029,
"output_tokens": 311
},
{
"arm": "pi",
"task": "regex-count",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 6.24,
"model_calls": 3,
"input_tokens": 7429,
"cached_input_tokens": 6027,
"output_tokens": 349
},
{
"arm": "pi",
"task": "write-tests",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 6.24,
"model_calls": 3,
"input_tokens": 7324,
"cached_input_tokens": 5903,
"output_tokens": 428
},
{
"arm": "pi",
"task": "write-tests",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 5.32,
"model_calls": 3,
"input_tokens": 7324,
"cached_input_tokens": 5903,
"output_tokens": 324
},
{
"arm": "pi",
"task": "write-tests",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 6.68,
"model_calls": 3,
"input_tokens": 7324,
"cached_input_tokens": 5903,
"output_tokens": 401
}
]
}