Skip to content

graff — a fast agentic coding harness in Zig: multi-provider, MCP, workflows, DGM evolution loop, TS/Python SDKs

Python
46.4 KB
Raw Blame
{
 "model": "one gateway model (same id in every arm)",
 "tasks": 21,
 "reps": 3,
 "arms": {
  "graff": "graff (ADR 0231 build)",
  "opencode2": "OpenCode 2",
  "pi": "pi"
 },
 "summary": {
  "all": {
   "graff": {
    "runs": 63,
    "passed": 63,
    "wall_s_per_task": 9.1,
    "cached_input_share": 0.945,
    "output_tokens_per_task": 582,
    "model_calls_per_task": 3.6
   },
   "opencode2": {
    "runs": 63,
    "passed": 62,
    "wall_s_per_task": 14.4,
    "cached_input_share": 0.862,
    "output_tokens_per_task": 980,
    "model_calls_per_task": 4.2
   },
   "pi": {
    "runs": 33,
    "passed": 33,
    "wall_s_per_task": 5.4,
    "cached_input_share": 0.785,
    "output_tokens_per_task": 258,
    "model_calls_per_task": 2.7
   }
  },
  "core": {
   "graff": {
    "runs": 33,
    "passed": 33,
    "wall_s_per_task": 7.3,
    "cached_input_share": 0.937,
    "output_tokens_per_task": 277,
    "model_calls_per_task": 2.8
   },
   "opencode2": {
    "runs": 33,
    "passed": 33,
    "wall_s_per_task": 6.9,
    "cached_input_share": 0.868,
    "output_tokens_per_task": 366,
    "model_calls_per_task": 3.2
   },
   "pi": {
    "runs": 33,
    "passed": 33,
    "wall_s_per_task": 5.4,
    "cached_input_share": 0.785,
    "output_tokens_per_task": 258,
    "model_calls_per_task": 2.7
   }
  },
  "mcp": {
   "graff": {
    "runs": 30,
    "passed": 30,
    "wall_s_per_task": 11.0,
    "cached_input_share": 0.95,
    "output_tokens_per_task": 917,
    "model_calls_per_task": 4.4
   },
   "opencode2": {
    "runs": 30,
    "passed": 29,
    "wall_s_per_task": 22.6,
    "cached_input_share": 0.858,
    "output_tokens_per_task": 1655,
    "model_calls_per_task": 5.3
   }
  }
 },
 "per_task": {
  "csv-sum": {
   "suite": "core",
   "graff": {
    "passed": 3,
    "runs": 3,
    "wall_s": 9.5,
    "model_calls": 3.3
   },
   "opencode2": {
    "passed": 3,
    "runs": 3,
    "wall_s": 7.8,
    "model_calls": 3
   },
   "pi": {
    "passed": 3,
    "runs": 3,
    "wall_s": 5.0,
    "model_calls": 3
   }
  },
  "dead-code": {
   "suite": "core",
   "graff": {
    "passed": 3,
    "runs": 3,
    "wall_s": 6.9,
    "model_calls": 3
   },
   "opencode2": {
    "passed": 3,
    "runs": 3,
    "wall_s": 6.8,
    "model_calls": 4
   },
   "pi": {
    "passed": 3,
    "runs": 3,
    "wall_s": 6.0,
    "model_calls": 3
   }
  },
  "exact-reply": {
   "suite": "core",
   "graff": {
    "passed": 3,
    "runs": 3,
    "wall_s": 1.8,
    "model_calls": 1
   },
   "opencode2": {
    "passed": 3,
    "runs": 3,
    "wall_s": 2.1,
    "model_calls": 1
   },
   "pi": {
    "passed": 3,
    "runs": 3,
    "wall_s": 1.7,
    "model_calls": 1
   }
  },
  "file-ops": {
   "suite": "core",
   "graff": {
    "passed": 3,
    "runs": 3,
    "wall_s": 4.2,
    "model_calls": 2
   },
   "opencode2": {
    "passed": 3,
    "runs": 3,
    "wall_s": 5.9,
    "model_calls": 3
   },
   "pi": {
    "passed": 3,
    "runs": 3,
    "wall_s": 4.6,
    "model_calls": 2
   }
  },
  "fix-fib": {
   "suite": "core",
   "graff": {
    "passed": 3,
    "runs": 3,
    "wall_s": 8.1,
    "model_calls": 4
   },
   "opencode2": {
    "passed": 3,
    "runs": 3,
    "wall_s": 7.6,
    "model_calls": 4
   },
   "pi": {
    "passed": 3,
    "runs": 3,
    "wall_s": 7.6,
    "model_calls": 4
   }
  },
  "git-ops": {
   "suite": "core",
   "graff": {
    "passed": 3,
    "runs": 3,
    "wall_s": 6.1,
    "model_calls": 2
   },
   "opencode2": {
    "passed": 3,
    "runs": 3,
    "wall_s": 6.2,
    "model_calls": 3
   },
   "pi": {
    "passed": 3,
    "runs": 3,
    "wall_s": 4.6,
    "model_calls": 2
   }
  },
  "json-transform": {
   "suite": "core",
   "graff": {
    "passed": 3,
    "runs": 3,
    "wall_s": 5.5,
    "model_calls": 3
   },
   "opencode2": {
    "passed": 3,
    "runs": 3,
    "wall_s": 6.0,
    "model_calls": 3
   },
   "pi": {
    "passed": 3,
    "runs": 3,
    "wall_s": 5.6,
    "model_calls": 3
   }
  },
  "linear-issues": {
   "suite": "mcp",
   "graff": {
    "passed": 3,
    "runs": 3,
    "wall_s": 7.5,
    "model_calls": 3
   },
   "opencode2": {
    "passed": 3,
    "runs": 3,
    "wall_s": 30.4,
    "model_calls": 6
   }
  },
  "linear-nohint": {
   "suite": "mcp",
   "graff": {
    "passed": 3,
    "runs": 3,
    "wall_s": 10.0,
    "model_calls": 4
   },
   "opencode2": {
    "passed": 3,
    "runs": 3,
    "wall_s": 20.2,
    "model_calls": 5.3
   }
  },
  "linear-nohint-warm": {
   "suite": "mcp",
   "graff": {
    "passed": 3,
    "runs": 3,
    "wall_s": 9.9,
    "model_calls": 3.3
   },
   "opencode2": {
    "passed": 3,
    "runs": 3,
    "wall_s": 20.4,
    "model_calls": 5.7
   }
  },
  "linear-quiet": {
   "suite": "mcp",
   "graff": {
    "passed": 3,
    "runs": 3,
    "wall_s": 11.3,
    "model_calls": 4
   },
   "opencode2": {
    "passed": 3,
    "runs": 3,
    "wall_s": 20.2,
    "model_calls": 5.7
   }
  },
  "linear-quiet-warm": {
   "suite": "mcp",
   "graff": {
    "passed": 3,
    "runs": 3,
    "wall_s": 15.7,
    "model_calls": 5
   },
   "opencode2": {
    "passed": 3,
    "runs": 3,
    "wall_s": 21.5,
    "model_calls": 5
   }
  },
  "linear-reduce": {
   "suite": "mcp",
   "graff": {
    "passed": 3,
    "runs": 3,
    "wall_s": 9.5,
    "model_calls": 4
   },
   "opencode2": {
    "passed": 3,
    "runs": 3,
    "wall_s": 20.9,
    "model_calls": 5.7
   }
  },
  "linear-reduce-warm": {
   "suite": "mcp",
   "graff": {
    "passed": 3,
    "runs": 3,
    "wall_s": 8.8,
    "model_calls": 4
   },
   "opencode2": {
    "passed": 3,
    "runs": 3,
    "wall_s": 19.2,
    "model_calls": 5.7
   }
  },
  "linear-sidecar": {
   "suite": "mcp",
   "graff": {
    "passed": 3,
    "runs": 3,
    "wall_s": 8.9,
    "model_calls": 3.7
   },
   "opencode2": {
    "passed": 3,
    "runs": 3,
    "wall_s": 22.5,
    "model_calls": 6
   }
  },
  "linear-split": {
   "suite": "mcp",
   "graff": {
    "passed": 3,
    "runs": 3,
    "wall_s": 18.9,
    "model_calls": 9
   },
   "opencode2": {
    "passed": 2,
    "runs": 3,
    "wall_s": 30.0,
    "model_calls": 2.7
   }
  },
  "linear-warm": {
   "suite": "mcp",
   "graff": {
    "passed": 3,
    "runs": 3,
    "wall_s": 9.1,
    "model_calls": 3.7
   },
   "opencode2": {
    "passed": 3,
    "runs": 3,
    "wall_s": 20.6,
    "model_calls": 5.7
   }
  },
  "recall-noise": {
   "suite": "core",
   "graff": {
    "passed": 3,
    "runs": 3,
    "wall_s": 7.8,
    "model_calls": 2
   },
   "opencode2": {
    "passed": 3,
    "runs": 3,
    "wall_s": 7.6,
    "model_calls": 3
   },
   "pi": {
    "passed": 3,
    "runs": 3,
    "wall_s": 4.4,
    "model_calls": 2
   }
  },
  "refactor-rename": {
   "suite": "core",
   "graff": {
    "passed": 3,
    "runs": 3,
    "wall_s": 10.3,
    "model_calls": 5
   },
   "opencode2": {
    "passed": 3,
    "runs": 3,
    "wall_s": 8.6,
    "model_calls": 4
   },
   "pi": {
    "passed": 3,
    "runs": 3,
    "wall_s": 7.8,
    "model_calls": 4
   }
  },
  "regex-count": {
   "suite": "core",
   "graff": {
    "passed": 3,
    "runs": 3,
    "wall_s": 14.1,
    "model_calls": 3
   },
   "opencode2": {
    "passed": 3,
    "runs": 3,
    "wall_s": 8.6,
    "model_calls": 3.3
   },
   "pi": {
    "passed": 3,
    "runs": 3,
    "wall_s": 6.0,
    "model_calls": 3
   }
  },
  "write-tests": {
   "suite": "core",
   "graff": {
    "passed": 3,
    "runs": 3,
    "wall_s": 6.6,
    "model_calls": 3
   },
   "opencode2": {
    "passed": 3,
    "runs": 3,
    "wall_s": 8.4,
    "model_calls": 4
   },
   "pi": {
    "passed": 3,
    "runs": 3,
    "wall_s": 6.1,
    "model_calls": 3
   }
  }
 },
 "runs": [
  {
   "arm": "graff",
   "task": "csv-sum",
   "suite": "core",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 7.9,
   "model_calls": 3,
   "input_tokens": 27883,
   "cached_input_tokens": 22546,
   "output_tokens": 311
  },
  {
   "arm": "graff",
   "task": "csv-sum",
   "suite": "core",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 8.23,
   "model_calls": 3,
   "input_tokens": 27912,
   "cached_input_tokens": 27756,
   "output_tokens": 355
  },
  {
   "arm": "graff",
   "task": "csv-sum",
   "suite": "core",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 12.24,
   "model_calls": 4,
   "input_tokens": 37348,
   "cached_input_tokens": 37159,
   "output_tokens": 471
  },
  {
   "arm": "graff",
   "task": "dead-code",
   "suite": "core",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 6.92,
   "model_calls": 3,
   "input_tokens": 28218,
   "cached_input_tokens": 22632,
   "output_tokens": 160
  },
  {
   "arm": "graff",
   "task": "dead-code",
   "suite": "core",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 4.44,
   "model_calls": 3,
   "input_tokens": 28218,
   "cached_input_tokens": 28210,
   "output_tokens": 160
  },
  {
   "arm": "graff",
   "task": "dead-code",
   "suite": "core",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 9.36,
   "model_calls": 3,
   "input_tokens": 28228,
   "cached_input_tokens": 27748,
   "output_tokens": 165
  },
  {
   "arm": "graff",
   "task": "exact-reply",
   "suite": "core",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 1.89,
   "model_calls": 1,
   "input_tokens": 9121,
   "cached_input_tokens": 9117,
   "output_tokens": 51
  },
  {
   "arm": "graff",
   "task": "exact-reply",
   "suite": "core",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 1.75,
   "model_calls": 1,
   "input_tokens": 9121,
   "cached_input_tokens": 9117,
   "output_tokens": 51
  },
  {
   "arm": "graff",
   "task": "exact-reply",
   "suite": "core",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 1.76,
   "model_calls": 1,
   "input_tokens": 9121,
   "cached_input_tokens": 9117,
   "output_tokens": 51
  },
  {
   "arm": "graff",
   "task": "file-ops",
   "suite": "core",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 4.93,
   "model_calls": 2,
   "input_tokens": 18395,
   "cached_input_tokens": 18186,
   "output_tokens": 143
  },
  {
   "arm": "graff",
   "task": "file-ops",
   "suite": "core",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 4.22,
   "model_calls": 2,
   "input_tokens": 18395,
   "cached_input_tokens": 18389,
   "output_tokens": 141
  },
  {
   "arm": "graff",
   "task": "file-ops",
   "suite": "core",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 3.39,
   "model_calls": 2,
   "input_tokens": 18395,
   "cached_input_tokens": 18389,
   "output_tokens": 158
  },
  {
   "arm": "graff",
   "task": "fix-fib",
   "suite": "core",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 7.41,
   "model_calls": 4,
   "input_tokens": 38412,
   "cached_input_tokens": 32526,
   "output_tokens": 394
  },
  {
   "arm": "graff",
   "task": "fix-fib",
   "suite": "core",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 8.11,
   "model_calls": 4,
   "input_tokens": null,
   "cached_input_tokens": null,
   "output_tokens": null
  },
  {
   "arm": "graff",
   "task": "fix-fib",
   "suite": "core",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 8.65,
   "model_calls": 4,
   "input_tokens": 38412,
   "cached_input_tokens": 37647,
   "output_tokens": 407
  },
  {
   "arm": "graff",
   "task": "git-ops",
   "suite": "core",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 4.24,
   "model_calls": 2,
   "input_tokens": 18596,
   "cached_input_tokens": 13279,
   "output_tokens": 258
  },
  {
   "arm": "graff",
   "task": "git-ops",
   "suite": "core",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 4.27,
   "model_calls": 2,
   "input_tokens": 18590,
   "cached_input_tokens": 18398,
   "output_tokens": 240
  },
  {
   "arm": "graff",
   "task": "git-ops",
   "suite": "core",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 9.77,
   "model_calls": 2,
   "input_tokens": 18595,
   "cached_input_tokens": 18398,
   "output_tokens": 264
  },
  {
   "arm": "graff",
   "task": "json-transform",
   "suite": "core",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 5.44,
   "model_calls": 3,
   "input_tokens": 27943,
   "cached_input_tokens": 22578,
   "output_tokens": 254
  },
  {
   "arm": "graff",
   "task": "json-transform",
   "suite": "core",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 5.77,
   "model_calls": 3,
   "input_tokens": 27943,
   "cached_input_tokens": 27935,
   "output_tokens": 261
  },
  {
   "arm": "graff",
   "task": "json-transform",
   "suite": "core",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 5.41,
   "model_calls": 3,
   "input_tokens": 27943,
   "cached_input_tokens": 27935,
   "output_tokens": 256
  },
  {
   "arm": "graff",
   "task": "linear-issues",
   "suite": "mcp",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 7.63,
   "model_calls": 3,
   "input_tokens": 30744,
   "cached_input_tokens": 20170,
   "output_tokens": 636
  },
  {
   "arm": "graff",
   "task": "linear-issues",
   "suite": "mcp",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 7.05,
   "model_calls": 3,
   "input_tokens": 30882,
   "cached_input_tokens": 30536,
   "output_tokens": 611
  },
  {
   "arm": "graff",
   "task": "linear-issues",
   "suite": "mcp",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 7.81,
   "model_calls": 3,
   "input_tokens": 30934,
   "cached_input_tokens": 30536,
   "output_tokens": 647
  },
  {
   "arm": "graff",
   "task": "linear-nohint",
   "suite": "mcp",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 9.38,
   "model_calls": 4,
   "input_tokens": 41204,
   "cached_input_tokens": 39485,
   "output_tokens": 907
  },
  {
   "arm": "graff",
   "task": "linear-nohint",
   "suite": "mcp",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 10.07,
   "model_calls": 4,
   "input_tokens": 41193,
   "cached_input_tokens": 40898,
   "output_tokens": 851
  },
  {
   "arm": "graff",
   "task": "linear-nohint",
   "suite": "mcp",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 10.6,
   "model_calls": 4,
   "input_tokens": 41231,
   "cached_input_tokens": 40166,
   "output_tokens": 853
  },
  {
   "arm": "graff",
   "task": "linear-nohint-warm",
   "suite": "mcp",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 12.69,
   "model_calls": 4,
   "input_tokens": 42833,
   "cached_input_tokens": 40824,
   "output_tokens": 676
  },
  {
   "arm": "graff",
   "task": "linear-nohint-warm",
   "suite": "mcp",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 8.45,
   "model_calls": 3,
   "input_tokens": 32072,
   "cached_input_tokens": 31503,
   "output_tokens": 993
  },
  {
   "arm": "graff",
   "task": "linear-nohint-warm",
   "suite": "mcp",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 8.64,
   "model_calls": 3,
   "input_tokens": 32072,
   "cached_input_tokens": 32064,
   "output_tokens": 922
  },
  {
   "arm": "graff",
   "task": "linear-quiet",
   "suite": "mcp",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 7.98,
   "model_calls": 4,
   "input_tokens": 41013,
   "cached_input_tokens": 39412,
   "output_tokens": 594
  },
  {
   "arm": "graff",
   "task": "linear-quiet",
   "suite": "mcp",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 17.67,
   "model_calls": 4,
   "input_tokens": 41013,
   "cached_input_tokens": 40808,
   "output_tokens": 617
  },
  {
   "arm": "graff",
   "task": "linear-quiet",
   "suite": "mcp",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 8.12,
   "model_calls": 4,
   "input_tokens": 41013,
   "cached_input_tokens": 41003,
   "output_tokens": 580
  },
  {
   "arm": "graff",
   "task": "linear-quiet-warm",
   "suite": "mcp",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 27.89,
   "model_calls": 7,
   "input_tokens": 78474,
   "cached_input_tokens": 75338,
   "output_tokens": 1305
  },
  {
   "arm": "graff",
   "task": "linear-quiet-warm",
   "suite": "mcp",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 10.06,
   "model_calls": 4,
   "input_tokens": 43098,
   "cached_input_tokens": 42796,
   "output_tokens": 653
  },
  {
   "arm": "graff",
   "task": "linear-quiet-warm",
   "suite": "mcp",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 9.08,
   "model_calls": 4,
   "input_tokens": 43040,
   "cached_input_tokens": 42395,
   "output_tokens": 679
  },
  {
   "arm": "graff",
   "task": "linear-reduce",
   "suite": "mcp",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 11.39,
   "model_calls": 4,
   "input_tokens": 41229,
   "cached_input_tokens": 39545,
   "output_tokens": 774
  },
  {
   "arm": "graff",
   "task": "linear-reduce",
   "suite": "mcp",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 8.94,
   "model_calls": 4,
   "input_tokens": 41245,
   "cached_input_tokens": 40485,
   "output_tokens": 616
  },
  {
   "arm": "graff",
   "task": "linear-reduce",
   "suite": "mcp",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 8.18,
   "model_calls": 4,
   "input_tokens": 41252,
   "cached_input_tokens": 40488,
   "output_tokens": 596
  },
  {
   "arm": "graff",
   "task": "linear-reduce-warm",
   "suite": "mcp",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 9.74,
   "model_calls": 4,
   "input_tokens": 42823,
   "cached_input_tokens": 40749,
   "output_tokens": 639
  },
  {
   "arm": "graff",
   "task": "linear-reduce-warm",
   "suite": "mcp",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 8.15,
   "model_calls": 4,
   "input_tokens": 42815,
   "cached_input_tokens": 41901,
   "output_tokens": 548
  },
  {
   "arm": "graff",
   "task": "linear-reduce-warm",
   "suite": "mcp",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 8.54,
   "model_calls": 4,
   "input_tokens": 42815,
   "cached_input_tokens": 42805,
   "output_tokens": 622
  },
  {
   "arm": "graff",
   "task": "linear-sidecar",
   "suite": "mcp",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 9.51,
   "model_calls": 4,
   "input_tokens": 49605,
   "cached_input_tokens": 45782,
   "output_tokens": 912
  },
  {
   "arm": "graff",
   "task": "linear-sidecar",
   "suite": "mcp",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 8.25,
   "model_calls": 3,
   "input_tokens": 37300,
   "cached_input_tokens": 36266,
   "output_tokens": 896
  },
  {
   "arm": "graff",
   "task": "linear-sidecar",
   "suite": "mcp",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 8.99,
   "model_calls": 4,
   "input_tokens": 49603,
   "cached_input_tokens": 48530,
   "output_tokens": 875
  },
  {
   "arm": "graff",
   "task": "linear-split",
   "suite": "mcp",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 16.16,
   "model_calls": 8,
   "input_tokens": 77647,
   "cached_input_tokens": 59430,
   "output_tokens": 2062
  },
  {
   "arm": "graff",
   "task": "linear-split",
   "suite": "mcp",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 17.19,
   "model_calls": 8,
   "input_tokens": 77930,
   "cached_input_tokens": 74039,
   "output_tokens": 2168
  },
  {
   "arm": "graff",
   "task": "linear-split",
   "suite": "mcp",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 23.48,
   "model_calls": 11,
   "input_tokens": 104516,
   "cached_input_tokens": 95787,
   "output_tokens": 2901
  },
  {
   "arm": "graff",
   "task": "linear-warm",
   "suite": "mcp",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 9.33,
   "model_calls": 4,
   "input_tokens": 43066,
   "cached_input_tokens": 40995,
   "output_tokens": 698
  },
  {
   "arm": "graff",
   "task": "linear-warm",
   "suite": "mcp",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 9.63,
   "model_calls": 4,
   "input_tokens": 43097,
   "cached_input_tokens": 42654,
   "output_tokens": 719
  },
  {
   "arm": "graff",
   "task": "linear-warm",
   "suite": "mcp",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 8.39,
   "model_calls": 3,
   "input_tokens": 32244,
   "cached_input_tokens": 31674,
   "output_tokens": 953
  },
  {
   "arm": "graff",
   "task": "recall-noise",
   "suite": "core",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 16.2,
   "model_calls": 2,
   "input_tokens": 18747,
   "cached_input_tokens": 13253,
   "output_tokens": 63
  },
  {
   "arm": "graff",
   "task": "recall-noise",
   "suite": "core",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 3.56,
   "model_calls": 2,
   "input_tokens": 18747,
   "cached_input_tokens": 18741,
   "output_tokens": 110
  },
  {
   "arm": "graff",
   "task": "recall-noise",
   "suite": "core",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 3.58,
   "model_calls": 2,
   "input_tokens": 18747,
   "cached_input_tokens": 18741,
   "output_tokens": 110
  },
  {
   "arm": "graff",
   "task": "refactor-rename",
   "suite": "core",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 9.61,
   "model_calls": 5,
   "input_tokens": 47904,
   "cached_input_tokens": 42043,
   "output_tokens": 555
  },
  {
   "arm": "graff",
   "task": "refactor-rename",
   "suite": "core",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 11.28,
   "model_calls": 5,
   "input_tokens": 48174,
   "cached_input_tokens": 47635,
   "output_tokens": 639
  },
  {
   "arm": "graff",
   "task": "refactor-rename",
   "suite": "core",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 9.99,
   "model_calls": 5,
   "input_tokens": 47904,
   "cached_input_tokens": 47892,
   "output_tokens": 525
  },
  {
   "arm": "graff",
   "task": "regex-count",
   "suite": "core",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 20.91,
   "model_calls": 3,
   "input_tokens": 28150,
   "cached_input_tokens": 22704,
   "output_tokens": 505
  },
  {
   "arm": "graff",
   "task": "regex-count",
   "suite": "core",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 10.03,
   "model_calls": 3,
   "input_tokens": 28165,
   "cached_input_tokens": 27851,
   "output_tokens": 365
  },
  {
   "arm": "graff",
   "task": "regex-count",
   "suite": "core",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 11.32,
   "model_calls": 3,
   "input_tokens": 28061,
   "cached_input_tokens": 27784,
   "output_tokens": 404
  },
  {
   "arm": "graff",
   "task": "write-tests",
   "suite": "core",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 6.49,
   "model_calls": 3,
   "input_tokens": 28080,
   "cached_input_tokens": 22556,
   "output_tokens": 446
  },
  {
   "arm": "graff",
   "task": "write-tests",
   "suite": "core",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 7.22,
   "model_calls": 3,
   "input_tokens": 28080,
   "cached_input_tokens": 28072,
   "output_tokens": 420
  },
  {
   "arm": "graff",
   "task": "write-tests",
   "suite": "core",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 6.13,
   "model_calls": 3,
   "input_tokens": 28080,
   "cached_input_tokens": 28072,
   "output_tokens": 444
  },
  {
   "arm": "opencode2",
   "task": "csv-sum",
   "suite": "core",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 10.68,
   "model_calls": 3,
   "input_tokens": 23865,
   "cached_input_tokens": 20630,
   "output_tokens": 311
  },
  {
   "arm": "opencode2",
   "task": "csv-sum",
   "suite": "core",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 6.96,
   "model_calls": 3,
   "input_tokens": 23865,
   "cached_input_tokens": 20671,
   "output_tokens": 319
  },
  {
   "arm": "opencode2",
   "task": "csv-sum",
   "suite": "core",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 5.9,
   "model_calls": 3,
   "input_tokens": 23865,
   "cached_input_tokens": 20671,
   "output_tokens": 304
  },
  {
   "arm": "opencode2",
   "task": "dead-code",
   "suite": "core",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 6.91,
   "model_calls": 4,
   "input_tokens": 32597,
   "cached_input_tokens": 28619,
   "output_tokens": 345
  },
  {
   "arm": "opencode2",
   "task": "dead-code",
   "suite": "core",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 7.53,
   "model_calls": 4,
   "input_tokens": 32305,
   "cached_input_tokens": 28671,
   "output_tokens": 289
  },
  {
   "arm": "opencode2",
   "task": "dead-code",
   "suite": "core",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 5.97,
   "model_calls": 4,
   "input_tokens": 32305,
   "cached_input_tokens": 28671,
   "output_tokens": 289
  },
  {
   "arm": "opencode2",
   "task": "exact-reply",
   "suite": "core",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 2.49,
   "model_calls": 1,
   "input_tokens": 8280,
   "cached_input_tokens": 5436,
   "output_tokens": 13
  },
  {
   "arm": "opencode2",
   "task": "exact-reply",
   "suite": "core",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 1.75,
   "model_calls": 1,
   "input_tokens": 7484,
   "cached_input_tokens": 4644,
   "output_tokens": 4
  },
  {
   "arm": "opencode2",
   "task": "exact-reply",
   "suite": "core",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 1.96,
   "model_calls": 1,
   "input_tokens": 7484,
   "cached_input_tokens": 4644,
   "output_tokens": 4
  },
  {
   "arm": "opencode2",
   "task": "file-ops",
   "suite": "core",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 6.73,
   "model_calls": 3,
   "input_tokens": 23691,
   "cached_input_tokens": 20568,
   "output_tokens": 255
  },
  {
   "arm": "opencode2",
   "task": "file-ops",
   "suite": "core",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 5.77,
   "model_calls": 3,
   "input_tokens": 23744,
   "cached_input_tokens": 20614,
   "output_tokens": 282
  },
  {
   "arm": "opencode2",
   "task": "file-ops",
   "suite": "core",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 5.21,
   "model_calls": 3,
   "input_tokens": 23857,
   "cached_input_tokens": 20614,
   "output_tokens": 295
  },
  {
   "arm": "opencode2",
   "task": "fix-fib",
   "suite": "core",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 8.21,
   "model_calls": 4,
   "input_tokens": 32294,
   "cached_input_tokens": 28798,
   "output_tokens": 345
  },
  {
   "arm": "opencode2",
   "task": "fix-fib",
   "suite": "core",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 6.74,
   "model_calls": 4,
   "input_tokens": 32294,
   "cached_input_tokens": 28857,
   "output_tokens": 358
  },
  {
   "arm": "opencode2",
   "task": "fix-fib",
   "suite": "core",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 7.86,
   "model_calls": 4,
   "input_tokens": 32294,
   "cached_input_tokens": 28857,
   "output_tokens": 358
  },
  {
   "arm": "opencode2",
   "task": "git-ops",
   "suite": "core",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 6.65,
   "model_calls": 3,
   "input_tokens": 23234,
   "cached_input_tokens": 19849,
   "output_tokens": 319
  },
  {
   "arm": "opencode2",
   "task": "git-ops",
   "suite": "core",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 6.05,
   "model_calls": 3,
   "input_tokens": 24079,
   "cached_input_tokens": 20691,
   "output_tokens": 353
  },
  {
   "arm": "opencode2",
   "task": "git-ops",
   "suite": "core",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 5.79,
   "model_calls": 3,
   "input_tokens": 24083,
   "cached_input_tokens": 20687,
   "output_tokens": 330
  },
  {
   "arm": "opencode2",
   "task": "json-transform",
   "suite": "core",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 5.75,
   "model_calls": 3,
   "input_tokens": 23966,
   "cached_input_tokens": 20665,
   "output_tokens": 378
  },
  {
   "arm": "opencode2",
   "task": "json-transform",
   "suite": "core",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 6.12,
   "model_calls": 3,
   "input_tokens": 23966,
   "cached_input_tokens": 20719,
   "output_tokens": 349
  },
  {
   "arm": "opencode2",
   "task": "json-transform",
   "suite": "core",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 6.07,
   "model_calls": 3,
   "input_tokens": 23966,
   "cached_input_tokens": 20719,
   "output_tokens": 391
  },
  {
   "arm": "opencode2",
   "task": "linear-issues",
   "suite": "mcp",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 39.11,
   "model_calls": 6,
   "input_tokens": 64184,
   "cached_input_tokens": 55752,
   "output_tokens": 1632
  },
  {
   "arm": "opencode2",
   "task": "linear-issues",
   "suite": "mcp",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 27.91,
   "model_calls": 6,
   "input_tokens": 66506,
   "cached_input_tokens": 57714,
   "output_tokens": 1849
  },
  {
   "arm": "opencode2",
   "task": "linear-issues",
   "suite": "mcp",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 24.19,
   "model_calls": 6,
   "input_tokens": 67866,
   "cached_input_tokens": 58508,
   "output_tokens": 1966
  },
  {
   "arm": "opencode2",
   "task": "linear-nohint",
   "suite": "mcp",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 19.28,
   "model_calls": 5,
   "input_tokens": 55723,
   "cached_input_tokens": 46086,
   "output_tokens": 1773
  },
  {
   "arm": "opencode2",
   "task": "linear-nohint",
   "suite": "mcp",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 20.57,
   "model_calls": 6,
   "input_tokens": 67456,
   "cached_input_tokens": 58096,
   "output_tokens": 1759
  },
  {
   "arm": "opencode2",
   "task": "linear-nohint",
   "suite": "mcp",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 20.79,
   "model_calls": 5,
   "input_tokens": 55608,
   "cached_input_tokens": 46175,
   "output_tokens": 1693
  },
  {
   "arm": "opencode2",
   "task": "linear-nohint-warm",
   "suite": "mcp",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 25.38,
   "model_calls": 6,
   "input_tokens": 67190,
   "cached_input_tokens": 57961,
   "output_tokens": 1782
  },
  {
   "arm": "opencode2",
   "task": "linear-nohint-warm",
   "suite": "mcp",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 16.84,
   "model_calls": 5,
   "input_tokens": 53445,
   "cached_input_tokens": 45015,
   "output_tokens": 1547
  },
  {
   "arm": "opencode2",
   "task": "linear-nohint-warm",
   "suite": "mcp",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 18.92,
   "model_calls": 6,
   "input_tokens": 66914,
   "cached_input_tokens": 57976,
   "output_tokens": 1719
  },
  {
   "arm": "opencode2",
   "task": "linear-quiet",
   "suite": "mcp",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 22.99,
   "model_calls": 6,
   "input_tokens": 58611,
   "cached_input_tokens": 51656,
   "output_tokens": 1652
  },
  {
   "arm": "opencode2",
   "task": "linear-quiet",
   "suite": "mcp",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 17.81,
   "model_calls": 5,
   "input_tokens": 47567,
   "cached_input_tokens": 41313,
   "output_tokens": 1653
  },
  {
   "arm": "opencode2",
   "task": "linear-quiet",
   "suite": "mcp",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 19.95,
   "model_calls": 6,
   "input_tokens": 58287,
   "cached_input_tokens": 51569,
   "output_tokens": 1867
  },
  {
   "arm": "opencode2",
   "task": "linear-quiet-warm",
   "suite": "mcp",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 15.55,
   "model_calls": 4,
   "input_tokens": 36901,
   "cached_input_tokens": 31489,
   "output_tokens": 1737
  },
  {
   "arm": "opencode2",
   "task": "linear-quiet-warm",
   "suite": "mcp",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 27.03,
   "model_calls": 5,
   "input_tokens": 54234,
   "cached_input_tokens": 44605,
   "output_tokens": 1786
  },
  {
   "arm": "opencode2",
   "task": "linear-quiet-warm",
   "suite": "mcp",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 22.01,
   "model_calls": 6,
   "input_tokens": 59377,
   "cached_input_tokens": 52389,
   "output_tokens": 1848
  },
  {
   "arm": "opencode2",
   "task": "linear-reduce",
   "suite": "mcp",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 20.74,
   "model_calls": 6,
   "input_tokens": 60410,
   "cached_input_tokens": 52832,
   "output_tokens": 1888
  },
  {
   "arm": "opencode2",
   "task": "linear-reduce",
   "suite": "mcp",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 20.63,
   "model_calls": 6,
   "input_tokens": 66525,
   "cached_input_tokens": 57121,
   "output_tokens": 1891
  },
  {
   "arm": "opencode2",
   "task": "linear-reduce",
   "suite": "mcp",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 21.28,
   "model_calls": 5,
   "input_tokens": 48244,
   "cached_input_tokens": 41797,
   "output_tokens": 2266
  },
  {
   "arm": "opencode2",
   "task": "linear-reduce-warm",
   "suite": "mcp",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 16.98,
   "model_calls": 5,
   "input_tokens": 47381,
   "cached_input_tokens": 40951,
   "output_tokens": 1725
  },
  {
   "arm": "opencode2",
   "task": "linear-reduce-warm",
   "suite": "mcp",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 18.83,
   "model_calls": 6,
   "input_tokens": 57363,
   "cached_input_tokens": 50959,
   "output_tokens": 1764
  },
  {
   "arm": "opencode2",
   "task": "linear-reduce-warm",
   "suite": "mcp",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 21.87,
   "model_calls": 6,
   "input_tokens": 58147,
   "cached_input_tokens": 51586,
   "output_tokens": 1819
  },
  {
   "arm": "opencode2",
   "task": "linear-sidecar",
   "suite": "mcp",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 21.1,
   "model_calls": 6,
   "input_tokens": 65921,
   "cached_input_tokens": 57064,
   "output_tokens": 1761
  },
  {
   "arm": "opencode2",
   "task": "linear-sidecar",
   "suite": "mcp",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 21.24,
   "model_calls": 6,
   "input_tokens": 65529,
   "cached_input_tokens": 57447,
   "output_tokens": 1833
  },
  {
   "arm": "opencode2",
   "task": "linear-sidecar",
   "suite": "mcp",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 25.05,
   "model_calls": 6,
   "input_tokens": 65650,
   "cached_input_tokens": 57132,
   "output_tokens": 1712
  },
  {
   "arm": "opencode2",
   "task": "linear-split",
   "suite": "mcp",
   "rep": 1,
   "pass": false,
   "timed_out": false,
   "wall_s": 42.55,
   "model_calls": 3,
   "input_tokens": 13805,
   "cached_input_tokens": 6430,
   "output_tokens": 291
  },
  {
   "arm": "opencode2",
   "task": "linear-split",
   "suite": "mcp",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 24.9,
   "model_calls": 3,
   "input_tokens": 21079,
   "cached_input_tokens": 16530,
   "output_tokens": 583
  },
  {
   "arm": "opencode2",
   "task": "linear-split",
   "suite": "mcp",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 22.6,
   "model_calls": 2,
   "input_tokens": 13327,
   "cached_input_tokens": 9691,
   "output_tokens": 355
  },
  {
   "arm": "opencode2",
   "task": "linear-warm",
   "suite": "mcp",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 22.22,
   "model_calls": 6,
   "input_tokens": 67825,
   "cached_input_tokens": 58324,
   "output_tokens": 1803
  },
  {
   "arm": "opencode2",
   "task": "linear-warm",
   "suite": "mcp",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 22.32,
   "model_calls": 6,
   "input_tokens": 66147,
   "cached_input_tokens": 57536,
   "output_tokens": 1984
  },
  {
   "arm": "opencode2",
   "task": "linear-warm",
   "suite": "mcp",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 17.27,
   "model_calls": 5,
   "input_tokens": 53869,
   "cached_input_tokens": 45387,
   "output_tokens": 1714
  },
  {
   "arm": "opencode2",
   "task": "recall-noise",
   "suite": "core",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 11.66,
   "model_calls": 3,
   "input_tokens": 24029,
   "cached_input_tokens": 20531,
   "output_tokens": 167
  },
  {
   "arm": "opencode2",
   "task": "recall-noise",
   "suite": "core",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 5.56,
   "model_calls": 3,
   "input_tokens": 24029,
   "cached_input_tokens": 20567,
   "output_tokens": 167
  },
  {
   "arm": "opencode2",
   "task": "recall-noise",
   "suite": "core",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 5.48,
   "model_calls": 3,
   "input_tokens": 23211,
   "cached_input_tokens": 19753,
   "output_tokens": 153
  },
  {
   "arm": "opencode2",
   "task": "refactor-rename",
   "suite": "core",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 8.12,
   "model_calls": 4,
   "input_tokens": 32966,
   "cached_input_tokens": 29034,
   "output_tokens": 730
  },
  {
   "arm": "opencode2",
   "task": "refactor-rename",
   "suite": "core",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 9.01,
   "model_calls": 4,
   "input_tokens": 32969,
   "cached_input_tokens": 29076,
   "output_tokens": 801
  },
  {
   "arm": "opencode2",
   "task": "refactor-rename",
   "suite": "core",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 8.68,
   "model_calls": 4,
   "input_tokens": 32998,
   "cached_input_tokens": 29120,
   "output_tokens": 737
  },
  {
   "arm": "opencode2",
   "task": "regex-count",
   "suite": "core",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 9.25,
   "model_calls": 3,
   "input_tokens": 24358,
   "cached_input_tokens": 20847,
   "output_tokens": 496
  },
  {
   "arm": "opencode2",
   "task": "regex-count",
   "suite": "core",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 8.45,
   "model_calls": 3,
   "input_tokens": 24207,
   "cached_input_tokens": 20897,
   "output_tokens": 401
  },
  {
   "arm": "opencode2",
   "task": "regex-count",
   "suite": "core",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 7.99,
   "model_calls": 4,
   "input_tokens": 32621,
   "cached_input_tokens": 29005,
   "output_tokens": 619
  },
  {
   "arm": "opencode2",
   "task": "write-tests",
   "suite": "core",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 8.5,
   "model_calls": 4,
   "input_tokens": 32008,
   "cached_input_tokens": 28378,
   "output_tokens": 655
  },
  {
   "arm": "opencode2",
   "task": "write-tests",
   "suite": "core",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 7.73,
   "model_calls": 4,
   "input_tokens": 31962,
   "cached_input_tokens": 28438,
   "output_tokens": 605
  },
  {
   "arm": "opencode2",
   "task": "write-tests",
   "suite": "core",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 9.01,
   "model_calls": 4,
   "input_tokens": 32008,
   "cached_input_tokens": 28438,
   "output_tokens": 640
  },
  {
   "arm": "pi",
   "task": "csv-sum",
   "suite": "core",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 4.9,
   "model_calls": 3,
   "input_tokens": 7244,
   "cached_input_tokens": 5896,
   "output_tokens": 262
  },
  {
   "arm": "pi",
   "task": "csv-sum",
   "suite": "core",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 4.53,
   "model_calls": 3,
   "input_tokens": 7223,
   "cached_input_tokens": 5896,
   "output_tokens": 215
  },
  {
   "arm": "pi",
   "task": "csv-sum",
   "suite": "core",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 5.54,
   "model_calls": 3,
   "input_tokens": 7223,
   "cached_input_tokens": 5896,
   "output_tokens": 235
  },
  {
   "arm": "pi",
   "task": "dead-code",
   "suite": "core",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 6.25,
   "model_calls": 3,
   "input_tokens": 7488,
   "cached_input_tokens": 5970,
   "output_tokens": 175
  },
  {
   "arm": "pi",
   "task": "dead-code",
   "suite": "core",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 6.43,
   "model_calls": 3,
   "input_tokens": 8379,
   "cached_input_tokens": 5999,
   "output_tokens": 291
  },
  {
   "arm": "pi",
   "task": "dead-code",
   "suite": "core",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 5.19,
   "model_calls": 3,
   "input_tokens": 7572,
   "cached_input_tokens": 5991,
   "output_tokens": 238
  },
  {
   "arm": "pi",
   "task": "exact-reply",
   "suite": "core",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 1.77,
   "model_calls": 1,
   "input_tokens": 2271,
   "cached_input_tokens": 1190,
   "output_tokens": 4
  },
  {
   "arm": "pi",
   "task": "exact-reply",
   "suite": "core",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 1.64,
   "model_calls": 1,
   "input_tokens": 2271,
   "cached_input_tokens": 1190,
   "output_tokens": 4
  },
  {
   "arm": "pi",
   "task": "exact-reply",
   "suite": "core",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 1.82,
   "model_calls": 1,
   "input_tokens": 2271,
   "cached_input_tokens": 1190,
   "output_tokens": 4
  },
  {
   "arm": "pi",
   "task": "file-ops",
   "suite": "core",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 6.38,
   "model_calls": 2,
   "input_tokens": 4746,
   "cached_input_tokens": 3489,
   "output_tokens": 163
  },
  {
   "arm": "pi",
   "task": "file-ops",
   "suite": "core",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 3.95,
   "model_calls": 2,
   "input_tokens": 4746,
   "cached_input_tokens": 3489,
   "output_tokens": 169
  },
  {
   "arm": "pi",
   "task": "file-ops",
   "suite": "core",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 3.57,
   "model_calls": 2,
   "input_tokens": 4746,
   "cached_input_tokens": 3489,
   "output_tokens": 168
  },
  {
   "arm": "pi",
   "task": "fix-fib",
   "suite": "core",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 7.84,
   "model_calls": 4,
   "input_tokens": 10457,
   "cached_input_tokens": 8722,
   "output_tokens": 434
  },
  {
   "arm": "pi",
   "task": "fix-fib",
   "suite": "core",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 7.13,
   "model_calls": 4,
   "input_tokens": 10704,
   "cached_input_tokens": 8892,
   "output_tokens": 417
  },
  {
   "arm": "pi",
   "task": "fix-fib",
   "suite": "core",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 7.78,
   "model_calls": 4,
   "input_tokens": 10710,
   "cached_input_tokens": 8892,
   "output_tokens": 428
  },
  {
   "arm": "pi",
   "task": "git-ops",
   "suite": "core",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 4.73,
   "model_calls": 2,
   "input_tokens": 5009,
   "cached_input_tokens": 3507,
   "output_tokens": 289
  },
  {
   "arm": "pi",
   "task": "git-ops",
   "suite": "core",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 4.53,
   "model_calls": 2,
   "input_tokens": 5009,
   "cached_input_tokens": 3507,
   "output_tokens": 257
  },
  {
   "arm": "pi",
   "task": "git-ops",
   "suite": "core",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 4.51,
   "model_calls": 2,
   "input_tokens": 5009,
   "cached_input_tokens": 3507,
   "output_tokens": 254
  },
  {
   "arm": "pi",
   "task": "json-transform",
   "suite": "core",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 5.42,
   "model_calls": 3,
   "input_tokens": 7314,
   "cached_input_tokens": 5927,
   "output_tokens": 205
  },
  {
   "arm": "pi",
   "task": "json-transform",
   "suite": "core",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 5.51,
   "model_calls": 3,
   "input_tokens": 7298,
   "cached_input_tokens": 5927,
   "output_tokens": 249
  },
  {
   "arm": "pi",
   "task": "json-transform",
   "suite": "core",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 5.76,
   "model_calls": 3,
   "input_tokens": 7341,
   "cached_input_tokens": 5927,
   "output_tokens": 261
  },
  {
   "arm": "pi",
   "task": "recall-noise",
   "suite": "core",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 7.43,
   "model_calls": 2,
   "input_tokens": 4979,
   "cached_input_tokens": 3480,
   "output_tokens": 61
  },
  {
   "arm": "pi",
   "task": "recall-noise",
   "suite": "core",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 2.89,
   "model_calls": 2,
   "input_tokens": 4979,
   "cached_input_tokens": 3480,
   "output_tokens": 61
  },
  {
   "arm": "pi",
   "task": "recall-noise",
   "suite": "core",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 2.91,
   "model_calls": 2,
   "input_tokens": 4979,
   "cached_input_tokens": 3480,
   "output_tokens": 61
  },
  {
   "arm": "pi",
   "task": "refactor-rename",
   "suite": "core",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 7.98,
   "model_calls": 4,
   "input_tokens": 10202,
   "cached_input_tokens": 8555,
   "output_tokens": 535
  },
  {
   "arm": "pi",
   "task": "refactor-rename",
   "suite": "core",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 7.92,
   "model_calls": 4,
   "input_tokens": 10185,
   "cached_input_tokens": 8555,
   "output_tokens": 501
  },
  {
   "arm": "pi",
   "task": "refactor-rename",
   "suite": "core",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 7.41,
   "model_calls": 4,
   "input_tokens": 10202,
   "cached_input_tokens": 8589,
   "output_tokens": 420
  },
  {
   "arm": "pi",
   "task": "regex-count",
   "suite": "core",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 6.3,
   "model_calls": 3,
   "input_tokens": 7414,
   "cached_input_tokens": 6012,
   "output_tokens": 343
  },
  {
   "arm": "pi",
   "task": "regex-count",
   "suite": "core",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 5.5,
   "model_calls": 3,
   "input_tokens": 7433,
   "cached_input_tokens": 6029,
   "output_tokens": 311
  },
  {
   "arm": "pi",
   "task": "regex-count",
   "suite": "core",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 6.24,
   "model_calls": 3,
   "input_tokens": 7429,
   "cached_input_tokens": 6027,
   "output_tokens": 349
  },
  {
   "arm": "pi",
   "task": "write-tests",
   "suite": "core",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 6.24,
   "model_calls": 3,
   "input_tokens": 7324,
   "cached_input_tokens": 5903,
   "output_tokens": 428
  },
  {
   "arm": "pi",
   "task": "write-tests",
   "suite": "core",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 5.32,
   "model_calls": 3,
   "input_tokens": 7324,
   "cached_input_tokens": 5903,
   "output_tokens": 324
  },
  {
   "arm": "pi",
   "task": "write-tests",
   "suite": "core",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 6.68,
   "model_calls": 3,
   "input_tokens": 7324,
   "cached_input_tokens": 5903,
   "output_tokens": 401
  }
 ]
}