Skip to content

graff — a fast agentic coding harness in Zig: multi-provider, MCP, workflows, DGM evolution loop, TS/Python SDKs

Python
52.4 KB
Raw Blame
{
 "model": "gpt-6.1-sol",
 "tasks": 21,
 "reps": 3,
 "arms": {
  "graff": "graff (ADR 0231 build)",
  "codex-app-server": "Codex app server",
  "pi": "pi"
 },
 "summary": {
  "all": {
   "graff": {
    "runs": 63,
    "passed": 63,
    "wall_s_per_task": 22.2,
    "list_usd_per_task": 0.0131,
    "cached_input_share": 0.904,
    "output_tokens_per_task": 348,
    "model_calls_per_task": 3.1
   },
   "codex-app-server": {
    "runs": 63,
    "passed": 63,
    "wall_s_per_task": 28.6,
    "list_usd_per_task": 0.0426,
    "cached_input_share": 0.855,
    "output_tokens_per_task": 409,
    "model_calls_per_task": 4.4
   },
   "pi": {
    "runs": 33,
    "passed": 33,
    "wall_s_per_task": 17.0,
    "list_usd_per_task": 0.0067,
    "cached_input_share": 0.334,
    "output_tokens_per_task": 107,
    "model_calls_per_task": 3.2
   }
  },
  "core": {
   "graff": {
    "runs": 33,
    "passed": 33,
    "wall_s_per_task": 16.0,
    "list_usd_per_task": 0.0093,
    "cached_input_share": 0.923,
    "output_tokens_per_task": 164,
    "model_calls_per_task": 3.0
   },
   "codex-app-server": {
    "runs": 33,
    "passed": 33,
    "wall_s_per_task": 20.4,
    "list_usd_per_task": 0.0156,
    "cached_input_share": 0.896,
    "output_tokens_per_task": 209,
    "model_calls_per_task": 3.1
   },
   "pi": {
    "runs": 33,
    "passed": 33,
    "wall_s_per_task": 17.0,
    "list_usd_per_task": 0.0067,
    "cached_input_share": 0.334,
    "output_tokens_per_task": 107,
    "model_calls_per_task": 3.2
   }
  },
  "mcp": {
   "graff": {
    "runs": 30,
    "passed": 30,
    "wall_s_per_task": 29.0,
    "list_usd_per_task": 0.0173,
    "cached_input_share": 0.887,
    "output_tokens_per_task": 550,
    "model_calls_per_task": 3.3
   },
   "codex-app-server": {
    "runs": 30,
    "passed": 30,
    "wall_s_per_task": 37.8,
    "list_usd_per_task": 0.0722,
    "cached_input_share": 0.839,
    "output_tokens_per_task": 628,
    "model_calls_per_task": 5.8
   }
  }
 },
 "per_task": {
  "csv-sum": {
   "suite": "core",
   "graff": {
    "passed": 3,
    "runs": 3,
    "wall_s": 16.0,
    "list_usd": 0.0116,
    "model_calls": 3.3
   },
   "codex-app-server": {
    "passed": 3,
    "runs": 3,
    "wall_s": 21.5,
    "list_usd": 0.018,
    "model_calls": 3.3
   },
   "pi": {
    "passed": 3,
    "runs": 3,
    "wall_s": 13.8,
    "list_usd": 0.0051,
    "model_calls": 3
   }
  },
  "dead-code": {
   "suite": "core",
   "graff": {
    "passed": 3,
    "runs": 3,
    "wall_s": 10.6,
    "list_usd": 0.0059,
    "model_calls": 2
   },
   "codex-app-server": {
    "passed": 3,
    "runs": 3,
    "wall_s": 21.0,
    "list_usd": 0.0171,
    "model_calls": 4
   },
   "pi": {
    "passed": 3,
    "runs": 3,
    "wall_s": 18.2,
    "list_usd": 0.0068,
    "model_calls": 4
   }
  },
  "exact-reply": {
   "suite": "core",
   "graff": {
    "passed": 3,
    "runs": 3,
    "wall_s": 5.0,
    "list_usd": 0.0014,
    "model_calls": 1
   },
   "codex-app-server": {
    "passed": 3,
    "runs": 3,
    "wall_s": 8.3,
    "list_usd": 0.0052,
    "model_calls": 1
   },
   "pi": {
    "passed": 3,
    "runs": 3,
    "wall_s": 8.5,
    "list_usd": 0.0023,
    "model_calls": 1
   }
  },
  "file-ops": {
   "suite": "core",
   "graff": {
    "passed": 3,
    "runs": 3,
    "wall_s": 14.5,
    "list_usd": 0.0107,
    "model_calls": 3
   },
   "codex-app-server": {
    "passed": 3,
    "runs": 3,
    "wall_s": 13.1,
    "list_usd": 0.0116,
    "model_calls": 2
   },
   "pi": {
    "passed": 3,
    "runs": 3,
    "wall_s": 23.0,
    "list_usd": 0.0059,
    "model_calls": 3
   }
  },
  "fix-fib": {
   "suite": "core",
   "graff": {
    "passed": 3,
    "runs": 3,
    "wall_s": 21.4,
    "list_usd": 0.0143,
    "model_calls": 4
   },
   "codex-app-server": {
    "passed": 3,
    "runs": 3,
    "wall_s": 23.5,
    "list_usd": 0.0202,
    "model_calls": 4
   },
   "pi": {
    "passed": 3,
    "runs": 3,
    "wall_s": 22.3,
    "list_usd": 0.0103,
    "model_calls": 5
   }
  },
  "git-ops": {
   "suite": "core",
   "graff": {
    "passed": 3,
    "runs": 3,
    "wall_s": 21.5,
    "list_usd": 0.0115,
    "model_calls": 3
   },
   "codex-app-server": {
    "passed": 3,
    "runs": 3,
    "wall_s": 20.4,
    "list_usd": 0.0159,
    "model_calls": 3
   },
   "pi": {
    "passed": 3,
    "runs": 3,
    "wall_s": 15.5,
    "list_usd": 0.0059,
    "model_calls": 3
   }
  },
  "json-transform": {
   "suite": "core",
   "graff": {
    "passed": 3,
    "runs": 3,
    "wall_s": 16.8,
    "list_usd": 0.0089,
    "model_calls": 3
   },
   "codex-app-server": {
    "passed": 3,
    "runs": 3,
    "wall_s": 30.6,
    "list_usd": 0.0169,
    "model_calls": 4
   },
   "pi": {
    "passed": 3,
    "runs": 3,
    "wall_s": 15.2,
    "list_usd": 0.0063,
    "model_calls": 3
   }
  },
  "linear-issues": {
   "suite": "mcp",
   "graff": {
    "passed": 3,
    "runs": 3,
    "wall_s": 22.8,
    "list_usd": 0.0175,
    "model_calls": 2
   },
   "codex-app-server": {
    "passed": 3,
    "runs": 3,
    "wall_s": 26.8,
    "list_usd": 0.0436,
    "model_calls": 4
   }
  },
  "linear-nohint": {
   "suite": "mcp",
   "graff": {
    "passed": 3,
    "runs": 3,
    "wall_s": 30.7,
    "list_usd": 0.0196,
    "model_calls": 4
   },
   "codex-app-server": {
    "passed": 3,
    "runs": 3,
    "wall_s": 28.5,
    "list_usd": 0.062,
    "model_calls": 4.3
   }
  },
  "linear-nohint-warm": {
   "suite": "mcp",
   "graff": {
    "passed": 3,
    "runs": 3,
    "wall_s": 27.8,
    "list_usd": 0.0173,
    "model_calls": 3
   },
   "codex-app-server": {
    "passed": 3,
    "runs": 3,
    "wall_s": 34.2,
    "list_usd": 0.0702,
    "model_calls": 5
   }
  },
  "linear-quiet": {
   "suite": "mcp",
   "graff": {
    "passed": 3,
    "runs": 3,
    "wall_s": 22.0,
    "list_usd": 0.0107,
    "model_calls": 2
   },
   "codex-app-server": {
    "passed": 3,
    "runs": 3,
    "wall_s": 36.6,
    "list_usd": 0.0389,
    "model_calls": 4.3
   }
  },
  "linear-quiet-warm": {
   "suite": "mcp",
   "graff": {
    "passed": 3,
    "runs": 3,
    "wall_s": 24.0,
    "list_usd": 0.0132,
    "model_calls": 2.3
   },
   "codex-app-server": {
    "passed": 3,
    "runs": 3,
    "wall_s": 33.4,
    "list_usd": 0.0456,
    "model_calls": 4
   }
  },
  "linear-reduce": {
   "suite": "mcp",
   "graff": {
    "passed": 3,
    "runs": 3,
    "wall_s": 22.5,
    "list_usd": 0.0108,
    "model_calls": 2
   },
   "codex-app-server": {
    "passed": 3,
    "runs": 3,
    "wall_s": 39.2,
    "list_usd": 0.0386,
    "model_calls": 5.3
   }
  },
  "linear-reduce-warm": {
   "suite": "mcp",
   "graff": {
    "passed": 3,
    "runs": 3,
    "wall_s": 26.6,
    "list_usd": 0.0145,
    "model_calls": 3
   },
   "codex-app-server": {
    "passed": 3,
    "runs": 3,
    "wall_s": 37.7,
    "list_usd": 0.0483,
    "model_calls": 5.3
   }
  },
  "linear-sidecar": {
   "suite": "mcp",
   "graff": {
    "passed": 3,
    "runs": 3,
    "wall_s": 38.3,
    "list_usd": 0.0252,
    "model_calls": 4.3
   },
   "codex-app-server": {
    "passed": 3,
    "runs": 3,
    "wall_s": 35.4,
    "list_usd": 0.05,
    "model_calls": 4.7
   }
  },
  "linear-split": {
   "suite": "mcp",
   "graff": {
    "passed": 3,
    "runs": 3,
    "wall_s": 52.5,
    "list_usd": 0.0322,
    "model_calls": 8
   },
   "codex-app-server": {
    "passed": 3,
    "runs": 3,
    "wall_s": 66.4,
    "list_usd": 0.2772,
    "model_calls": 17
   }
  },
  "linear-warm": {
   "suite": "mcp",
   "graff": {
    "passed": 3,
    "runs": 3,
    "wall_s": 23.0,
    "list_usd": 0.012,
    "model_calls": 2
   },
   "codex-app-server": {
    "passed": 3,
    "runs": 3,
    "wall_s": 39.4,
    "list_usd": 0.0477,
    "model_calls": 4.3
   }
  },
  "recall-noise": {
   "suite": "core",
   "graff": {
    "passed": 3,
    "runs": 3,
    "wall_s": 9.9,
    "list_usd": 0.0081,
    "model_calls": 2
   },
   "codex-app-server": {
    "passed": 3,
    "runs": 3,
    "wall_s": 10.5,
    "list_usd": 0.0092,
    "model_calls": 2
   },
   "pi": {
    "passed": 3,
    "runs": 3,
    "wall_s": 10.3,
    "list_usd": 0.0053,
    "model_calls": 2
   }
  },
  "refactor-rename": {
   "suite": "core",
   "graff": {
    "passed": 3,
    "runs": 3,
    "wall_s": 23.0,
    "list_usd": 0.0141,
    "model_calls": 4
   },
   "codex-app-server": {
    "passed": 3,
    "runs": 3,
    "wall_s": 29.0,
    "list_usd": 0.0224,
    "model_calls": 4
   },
   "pi": {
    "passed": 3,
    "runs": 3,
    "wall_s": 23.4,
    "list_usd": 0.0132,
    "model_calls": 5
   }
  },
  "regex-count": {
   "suite": "core",
   "graff": {
    "passed": 3,
    "runs": 3,
    "wall_s": 17.2,
    "list_usd": 0.0071,
    "model_calls": 3.3
   },
   "codex-app-server": {
    "passed": 3,
    "runs": 3,
    "wall_s": 18.0,
    "list_usd": 0.0158,
    "model_calls": 3
   },
   "pi": {
    "passed": 3,
    "runs": 3,
    "wall_s": 13.3,
    "list_usd": 0.0033,
    "model_calls": 2
   }
  },
  "write-tests": {
   "suite": "core",
   "graff": {
    "passed": 3,
    "runs": 3,
    "wall_s": 20.4,
    "list_usd": 0.0087,
    "model_calls": 4
   },
   "codex-app-server": {
    "passed": 3,
    "runs": 3,
    "wall_s": 28.1,
    "list_usd": 0.019,
    "model_calls": 4
   },
   "pi": {
    "passed": 3,
    "runs": 3,
    "wall_s": 23.8,
    "list_usd": 0.0093,
    "model_calls": 4.7
   }
  }
 },
 "runs": [
  {
   "arm": "codex-app-server",
   "task": "csv-sum",
   "suite": "core",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 21.1,
   "model_calls": 3,
   "input_tokens": 43494,
   "cached_input_tokens": 36864,
   "output_tokens": 265,
   "list_usd": 0.019596
  },
  {
   "arm": "codex-app-server",
   "task": "csv-sum",
   "suite": "core",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 22.6,
   "model_calls": 3,
   "input_tokens": 43342,
   "cached_input_tokens": 38784,
   "output_tokens": 326,
   "list_usd": 0.016254
  },
  {
   "arm": "codex-app-server",
   "task": "csv-sum",
   "suite": "core",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 20.91,
   "model_calls": 4,
   "input_tokens": 57931,
   "cached_input_tokens": 52992,
   "output_tokens": 288,
   "list_usd": 0.018057
  },
  {
   "arm": "codex-app-server",
   "task": "dead-code",
   "suite": "core",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 22.78,
   "model_calls": 4,
   "input_tokens": 57946,
   "cached_input_tokens": 52864,
   "output_tokens": 150,
   "list_usd": 0.01695
  },
  {
   "arm": "codex-app-server",
   "task": "dead-code",
   "suite": "core",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 21.11,
   "model_calls": 4,
   "input_tokens": 58070,
   "cached_input_tokens": 52864,
   "output_tokens": 150,
   "list_usd": 0.017198
  },
  {
   "arm": "codex-app-server",
   "task": "dead-code",
   "suite": "core",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 19.15,
   "model_calls": 4,
   "input_tokens": 58043,
   "cached_input_tokens": 52864,
   "output_tokens": 165,
   "list_usd": 0.017294
  },
  {
   "arm": "codex-app-server",
   "task": "exact-reply",
   "suite": "core",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 6.37,
   "model_calls": 1,
   "input_tokens": 14234,
   "cached_input_tokens": 12288,
   "output_tokens": 5,
   "list_usd": 0.005171
  },
  {
   "arm": "codex-app-server",
   "task": "exact-reply",
   "suite": "core",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 9.23,
   "model_calls": 1,
   "input_tokens": 14235,
   "cached_input_tokens": 12288,
   "output_tokens": 5,
   "list_usd": 0.005173
  },
  {
   "arm": "codex-app-server",
   "task": "exact-reply",
   "suite": "core",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 9.37,
   "model_calls": 1,
   "input_tokens": 14234,
   "cached_input_tokens": 12288,
   "output_tokens": 5,
   "list_usd": 0.005171
  },
  {
   "arm": "codex-app-server",
   "task": "file-ops",
   "suite": "core",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 11.53,
   "model_calls": 2,
   "input_tokens": 28646,
   "cached_input_tokens": 24576,
   "output_tokens": 97,
   "list_usd": 0.011568
  },
  {
   "arm": "codex-app-server",
   "task": "file-ops",
   "suite": "core",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 15.1,
   "model_calls": 2,
   "input_tokens": 28646,
   "cached_input_tokens": 24576,
   "output_tokens": 99,
   "list_usd": 0.011588
  },
  {
   "arm": "codex-app-server",
   "task": "file-ops",
   "suite": "core",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 12.7,
   "model_calls": 2,
   "input_tokens": 28646,
   "cached_input_tokens": 24576,
   "output_tokens": 101,
   "list_usd": 0.011608
  },
  {
   "arm": "codex-app-server",
   "task": "fix-fib",
   "suite": "core",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 22.94,
   "model_calls": 4,
   "input_tokens": 58587,
   "cached_input_tokens": 53120,
   "output_tokens": 284,
   "list_usd": 0.019086
  },
  {
   "arm": "codex-app-server",
   "task": "fix-fib",
   "suite": "core",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 23.54,
   "model_calls": 4,
   "input_tokens": 58579,
   "cached_input_tokens": 53120,
   "output_tokens": 285,
   "list_usd": 0.01908
  },
  {
   "arm": "codex-app-server",
   "task": "fix-fib",
   "suite": "core",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 24.12,
   "model_calls": 4,
   "input_tokens": 58566,
   "cached_input_tokens": 51328,
   "output_tokens": 279,
   "list_usd": 0.022399
  },
  {
   "arm": "codex-app-server",
   "task": "git-ops",
   "suite": "core",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 22.45,
   "model_calls": 3,
   "input_tokens": 43610,
   "cached_input_tokens": 38656,
   "output_tokens": 218,
   "list_usd": 0.015954
  },
  {
   "arm": "codex-app-server",
   "task": "git-ops",
   "suite": "core",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 17.47,
   "model_calls": 3,
   "input_tokens": 43653,
   "cached_input_tokens": 38656,
   "output_tokens": 215,
   "list_usd": 0.01601
  },
  {
   "arm": "codex-app-server",
   "task": "git-ops",
   "suite": "core",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 21.17,
   "model_calls": 3,
   "input_tokens": 43532,
   "cached_input_tokens": 38656,
   "output_tokens": 221,
   "list_usd": 0.015828
  },
  {
   "arm": "codex-app-server",
   "task": "json-transform",
   "suite": "core",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 28.19,
   "model_calls": 4,
   "input_tokens": 58004,
   "cached_input_tokens": 52992,
   "output_tokens": 252,
   "list_usd": 0.017843
  },
  {
   "arm": "codex-app-server",
   "task": "json-transform",
   "suite": "core",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 40.09,
   "model_calls": 4,
   "input_tokens": 58013,
   "cached_input_tokens": 54912,
   "output_tokens": 287,
   "list_usd": 0.014563
  },
  {
   "arm": "codex-app-server",
   "task": "json-transform",
   "suite": "core",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 23.55,
   "model_calls": 4,
   "input_tokens": 58008,
   "cached_input_tokens": 52864,
   "output_tokens": 274,
   "list_usd": 0.018314
  },
  {
   "arm": "codex-app-server",
   "task": "linear-issues",
   "suite": "mcp",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 26.65,
   "model_calls": 4,
   "input_tokens": 78780,
   "cached_input_tokens": 66176,
   "output_tokens": 428,
   "list_usd": 0.036106
  },
  {
   "arm": "codex-app-server",
   "task": "linear-issues",
   "suite": "mcp",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 28.72,
   "model_calls": 4,
   "input_tokens": 78019,
   "cached_input_tokens": 60416,
   "output_tokens": 448,
   "list_usd": 0.045728
  },
  {
   "arm": "codex-app-server",
   "task": "linear-issues",
   "suite": "mcp",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 25.05,
   "model_calls": 4,
   "input_tokens": 91803,
   "cached_input_tokens": 72960,
   "output_tokens": 402,
   "list_usd": 0.049002
  },
  {
   "arm": "codex-app-server",
   "task": "linear-nohint",
   "suite": "mcp",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 27.89,
   "model_calls": 4,
   "input_tokens": 104953,
   "cached_input_tokens": 81664,
   "output_tokens": 462,
   "list_usd": 0.059364
  },
  {
   "arm": "codex-app-server",
   "task": "linear-nohint",
   "suite": "mcp",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 31.32,
   "model_calls": 5,
   "input_tokens": 138111,
   "cached_input_tokens": 116480,
   "output_tokens": 456,
   "list_usd": 0.05947
  },
  {
   "arm": "codex-app-server",
   "task": "linear-nohint",
   "suite": "mcp",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 26.24,
   "model_calls": 4,
   "input_tokens": 105177,
   "cached_input_tokens": 77696,
   "output_tokens": 442,
   "list_usd": 0.067152
  },
  {
   "arm": "codex-app-server",
   "task": "linear-nohint-warm",
   "suite": "mcp",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 30.78,
   "model_calls": 4,
   "input_tokens": 104995,
   "cached_input_tokens": 83584,
   "output_tokens": 471,
   "list_usd": 0.05589
  },
  {
   "arm": "codex-app-server",
   "task": "linear-nohint-warm",
   "suite": "mcp",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 38.38,
   "model_calls": 6,
   "input_tokens": 167525,
   "cached_input_tokens": 130304,
   "output_tokens": 612,
   "list_usd": 0.093592
  },
  {
   "arm": "codex-app-server",
   "task": "linear-nohint-warm",
   "suite": "mcp",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 33.44,
   "model_calls": 5,
   "input_tokens": 134923,
   "cached_input_tokens": 112256,
   "output_tokens": 467,
   "list_usd": 0.06123
  },
  {
   "arm": "codex-app-server",
   "task": "linear-quiet",
   "suite": "mcp",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 45.11,
   "model_calls": 5,
   "input_tokens": 107148,
   "cached_input_tokens": 92800,
   "output_tokens": 671,
   "list_usd": 0.044686
  },
  {
   "arm": "codex-app-server",
   "task": "linear-quiet",
   "suite": "mcp",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 29.19,
   "model_calls": 4,
   "input_tokens": 63325,
   "cached_input_tokens": 52736,
   "output_tokens": 494,
   "list_usd": 0.031392
  },
  {
   "arm": "codex-app-server",
   "task": "linear-quiet",
   "suite": "mcp",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 35.47,
   "model_calls": 4,
   "input_tokens": 81625,
   "cached_input_tokens": 67584,
   "output_tokens": 575,
   "list_usd": 0.04059
  },
  {
   "arm": "codex-app-server",
   "task": "linear-quiet-warm",
   "suite": "mcp",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 27.33,
   "model_calls": 4,
   "input_tokens": 63408,
   "cached_input_tokens": 52992,
   "output_tokens": 466,
   "list_usd": 0.030791
  },
  {
   "arm": "codex-app-server",
   "task": "linear-quiet-warm",
   "suite": "mcp",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 35.83,
   "model_calls": 4,
   "input_tokens": 81586,
   "cached_input_tokens": 54656,
   "output_tokens": 547,
   "list_usd": 0.064796
  },
  {
   "arm": "codex-app-server",
   "task": "linear-quiet-warm",
   "suite": "mcp",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 37.12,
   "model_calls": 4,
   "input_tokens": 81885,
   "cached_input_tokens": 67712,
   "output_tokens": 612,
   "list_usd": 0.041237
  },
  {
   "arm": "codex-app-server",
   "task": "linear-reduce",
   "suite": "mcp",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 30.24,
   "model_calls": 4,
   "input_tokens": 63323,
   "cached_input_tokens": 54656,
   "output_tokens": 519,
   "list_usd": 0.02799
  },
  {
   "arm": "codex-app-server",
   "task": "linear-reduce",
   "suite": "mcp",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 48.78,
   "model_calls": 6,
   "input_tokens": 132981,
   "cached_input_tokens": 114944,
   "output_tokens": 674,
   "list_usd": 0.054308
  },
  {
   "arm": "codex-app-server",
   "task": "linear-reduce",
   "suite": "mcp",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 38.67,
   "model_calls": 6,
   "input_tokens": 96523,
   "cached_input_tokens": 87424,
   "output_tokens": 655,
   "list_usd": 0.03349
  },
  {
   "arm": "codex-app-server",
   "task": "linear-reduce-warm",
   "suite": "mcp",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 45.0,
   "model_calls": 6,
   "input_tokens": 138736,
   "cached_input_tokens": 112384,
   "output_tokens": 658,
   "list_usd": 0.070522
  },
  {
   "arm": "codex-app-server",
   "task": "linear-reduce-warm",
   "suite": "mcp",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 41.32,
   "model_calls": 6,
   "input_tokens": 132919,
   "cached_input_tokens": 118400,
   "output_tokens": 649,
   "list_usd": 0.047368
  },
  {
   "arm": "codex-app-server",
   "task": "linear-reduce-warm",
   "suite": "mcp",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 26.82,
   "model_calls": 4,
   "input_tokens": 63275,
   "cached_input_tokens": 54656,
   "output_tokens": 436,
   "list_usd": 0.027064
  },
  {
   "arm": "codex-app-server",
   "task": "linear-sidecar",
   "suite": "mcp",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 41.51,
   "model_calls": 5,
   "input_tokens": 139088,
   "cached_input_tokens": 115200,
   "output_tokens": 585,
   "list_usd": 0.065146
  },
  {
   "arm": "codex-app-server",
   "task": "linear-sidecar",
   "suite": "mcp",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 36.42,
   "model_calls": 5,
   "input_tokens": 96819,
   "cached_input_tokens": 79232,
   "output_tokens": 559,
   "list_usd": 0.048687
  },
  {
   "arm": "codex-app-server",
   "task": "linear-sidecar",
   "suite": "mcp",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 28.14,
   "model_calls": 4,
   "input_tokens": 78076,
   "cached_input_tokens": 65664,
   "output_tokens": 474,
   "list_usd": 0.03613
  },
  {
   "arm": "codex-app-server",
   "task": "linear-split",
   "suite": "mcp",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 68.37,
   "model_calls": 17,
   "input_tokens": 376049,
   "cached_input_tokens": 341632,
   "output_tokens": 1479,
   "list_usd": 0.228179
  },
  {
   "arm": "codex-app-server",
   "task": "linear-split",
   "suite": "mcp",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 67.48,
   "model_calls": 16,
   "input_tokens": 353717,
   "cached_input_tokens": 295936,
   "output_tokens": 1460,
   "list_usd": 0.312211
  },
  {
   "arm": "codex-app-server",
   "task": "linear-split",
   "suite": "mcp",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 63.22,
   "model_calls": 18,
   "input_tokens": 440631,
   "cached_input_tokens": 392832,
   "output_tokens": 1423,
   "list_usd": 0.291107
  },
  {
   "arm": "codex-app-server",
   "task": "linear-warm",
   "suite": "mcp",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 46.12,
   "model_calls": 5,
   "input_tokens": 105284,
   "cached_input_tokens": 87936,
   "output_tokens": 580,
   "list_usd": 0.04929
  },
  {
   "arm": "codex-app-server",
   "task": "linear-warm",
   "suite": "mcp",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 33.35,
   "model_calls": 4,
   "input_tokens": 89282,
   "cached_input_tokens": 73600,
   "output_tokens": 569,
   "list_usd": 0.044414
  },
  {
   "arm": "codex-app-server",
   "task": "linear-warm",
   "suite": "mcp",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 38.58,
   "model_calls": 4,
   "input_tokens": 80241,
   "cached_input_tokens": 61312,
   "output_tokens": 553,
   "list_usd": 0.049519
  },
  {
   "arm": "codex-app-server",
   "task": "recall-noise",
   "suite": "core",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 10.44,
   "model_calls": 2,
   "input_tokens": 28812,
   "cached_input_tokens": 26368,
   "output_tokens": 52,
   "list_usd": 0.008045
  },
  {
   "arm": "codex-app-server",
   "task": "recall-noise",
   "suite": "core",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 9.8,
   "model_calls": 2,
   "input_tokens": 28807,
   "cached_input_tokens": 26368,
   "output_tokens": 51,
   "list_usd": 0.008025
  },
  {
   "arm": "codex-app-server",
   "task": "recall-noise",
   "suite": "core",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 11.18,
   "model_calls": 2,
   "input_tokens": 28817,
   "cached_input_tokens": 24576,
   "output_tokens": 57,
   "list_usd": 0.01151
  },
  {
   "arm": "codex-app-server",
   "task": "refactor-rename",
   "suite": "core",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 26.91,
   "model_calls": 4,
   "input_tokens": 58485,
   "cached_input_tokens": 52992,
   "output_tokens": 429,
   "list_usd": 0.020575
  },
  {
   "arm": "codex-app-server",
   "task": "refactor-rename",
   "suite": "core",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 32.06,
   "model_calls": 4,
   "input_tokens": 58307,
   "cached_input_tokens": 51328,
   "output_tokens": 332,
   "list_usd": 0.022411
  },
  {
   "arm": "codex-app-server",
   "task": "refactor-rename",
   "suite": "core",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 28.13,
   "model_calls": 4,
   "input_tokens": 58435,
   "cached_input_tokens": 50944,
   "output_tokens": 420,
   "list_usd": 0.024276
  },
  {
   "arm": "codex-app-server",
   "task": "regex-count",
   "suite": "core",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 18.3,
   "model_calls": 3,
   "input_tokens": 43226,
   "cached_input_tokens": 36864,
   "output_tokens": 175,
   "list_usd": 0.01816
  },
  {
   "arm": "codex-app-server",
   "task": "regex-count",
   "suite": "core",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 17.35,
   "model_calls": 3,
   "input_tokens": 43195,
   "cached_input_tokens": 38784,
   "output_tokens": 170,
   "list_usd": 0.0144
  },
  {
   "arm": "codex-app-server",
   "task": "regex-count",
   "suite": "core",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 18.26,
   "model_calls": 3,
   "input_tokens": 43262,
   "cached_input_tokens": 38656,
   "output_tokens": 190,
   "list_usd": 0.014978
  },
  {
   "arm": "codex-app-server",
   "task": "write-tests",
   "suite": "core",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 31.21,
   "model_calls": 4,
   "input_tokens": 58018,
   "cached_input_tokens": 52864,
   "output_tokens": 361,
   "list_usd": 0.019204
  },
  {
   "arm": "codex-app-server",
   "task": "write-tests",
   "suite": "core",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 24.56,
   "model_calls": 4,
   "input_tokens": 58006,
   "cached_input_tokens": 52992,
   "output_tokens": 356,
   "list_usd": 0.018887
  },
  {
   "arm": "codex-app-server",
   "task": "write-tests",
   "suite": "core",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 28.53,
   "model_calls": 4,
   "input_tokens": 57998,
   "cached_input_tokens": 52864,
   "output_tokens": 349,
   "list_usd": 0.019044
  },
  {
   "arm": "graff",
   "task": "csv-sum",
   "suite": "core",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 13.5,
   "model_calls": 3,
   "input_tokens": 31280,
   "cached_input_tokens": 26624,
   "output_tokens": 107,
   "list_usd": 0.013044
  },
  {
   "arm": "graff",
   "task": "csv-sum",
   "suite": "core",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 14.83,
   "model_calls": 3,
   "input_tokens": 31243,
   "cached_input_tokens": 26624,
   "output_tokens": 101,
   "list_usd": 0.01291
  },
  {
   "arm": "graff",
   "task": "csv-sum",
   "suite": "core",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 19.63,
   "model_calls": 4,
   "input_tokens": 41912,
   "cached_input_tokens": 40448,
   "output_tokens": 186,
   "list_usd": 0.008833
  },
  {
   "arm": "graff",
   "task": "dead-code",
   "suite": "core",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 9.57,
   "model_calls": 2,
   "input_tokens": 20881,
   "cached_input_tokens": 20352,
   "output_tokens": 56,
   "list_usd": 0.003653
  },
  {
   "arm": "graff",
   "task": "dead-code",
   "suite": "core",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 9.3,
   "model_calls": 2,
   "input_tokens": 20881,
   "cached_input_tokens": 16512,
   "output_tokens": 56,
   "list_usd": 0.010949
  },
  {
   "arm": "graff",
   "task": "dead-code",
   "suite": "core",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 12.97,
   "model_calls": 2,
   "input_tokens": 20881,
   "cached_input_tokens": 20608,
   "output_tokens": 56,
   "list_usd": 0.003167
  },
  {
   "arm": "graff",
   "task": "exact-reply",
   "suite": "core",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 4.66,
   "model_calls": 1,
   "input_tokens": 10306,
   "cached_input_tokens": 10112,
   "output_tokens": 5,
   "list_usd": 0.001449
  },
  {
   "arm": "graff",
   "task": "exact-reply",
   "suite": "core",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 3.89,
   "model_calls": 1,
   "input_tokens": 10306,
   "cached_input_tokens": 10112,
   "output_tokens": 5,
   "list_usd": 0.001449
  },
  {
   "arm": "graff",
   "task": "exact-reply",
   "suite": "core",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 6.55,
   "model_calls": 1,
   "input_tokens": 10306,
   "cached_input_tokens": 10112,
   "output_tokens": 5,
   "list_usd": 0.001449
  },
  {
   "arm": "graff",
   "task": "file-ops",
   "suite": "core",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 14.98,
   "model_calls": 3,
   "input_tokens": 31156,
   "cached_input_tokens": 26624,
   "output_tokens": 125,
   "list_usd": 0.012976
  },
  {
   "arm": "graff",
   "task": "file-ops",
   "suite": "core",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 15.58,
   "model_calls": 3,
   "input_tokens": 31192,
   "cached_input_tokens": 30336,
   "output_tokens": 135,
   "list_usd": 0.006096
  },
  {
   "arm": "graff",
   "task": "file-ops",
   "suite": "core",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 13.01,
   "model_calls": 3,
   "input_tokens": 31158,
   "cached_input_tokens": 26624,
   "output_tokens": 131,
   "list_usd": 0.01304
  },
  {
   "arm": "graff",
   "task": "fix-fib",
   "suite": "core",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 22.95,
   "model_calls": 4,
   "input_tokens": 42668,
   "cached_input_tokens": 41344,
   "output_tokens": 260,
   "list_usd": 0.009382
  },
  {
   "arm": "graff",
   "task": "fix-fib",
   "suite": "core",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 20.02,
   "model_calls": 4,
   "input_tokens": 42674,
   "cached_input_tokens": 41344,
   "output_tokens": 272,
   "list_usd": 0.009514
  },
  {
   "arm": "graff",
   "task": "fix-fib",
   "suite": "core",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 21.31,
   "model_calls": 4,
   "input_tokens": 42407,
   "cached_input_tokens": 33280,
   "output_tokens": 239,
   "list_usd": 0.023972
  },
  {
   "arm": "graff",
   "task": "git-ops",
   "suite": "core",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 21.0,
   "model_calls": 3,
   "input_tokens": 31815,
   "cached_input_tokens": 30464,
   "output_tokens": 326,
   "list_usd": 0.009008
  },
  {
   "arm": "graff",
   "task": "git-ops",
   "suite": "core",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 21.12,
   "model_calls": 3,
   "input_tokens": 31791,
   "cached_input_tokens": 30464,
   "output_tokens": 307,
   "list_usd": 0.00877
  },
  {
   "arm": "graff",
   "task": "git-ops",
   "suite": "core",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 22.3,
   "model_calls": 3,
   "input_tokens": 31852,
   "cached_input_tokens": 26624,
   "output_tokens": 367,
   "list_usd": 0.016788
  },
  {
   "arm": "graff",
   "task": "json-transform",
   "suite": "core",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 18.13,
   "model_calls": 3,
   "input_tokens": 31395,
   "cached_input_tokens": 30592,
   "output_tokens": 175,
   "list_usd": 0.006415
  },
  {
   "arm": "graff",
   "task": "json-transform",
   "suite": "core",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 16.53,
   "model_calls": 3,
   "input_tokens": 31400,
   "cached_input_tokens": 30592,
   "output_tokens": 180,
   "list_usd": 0.006475
  },
  {
   "arm": "graff",
   "task": "json-transform",
   "suite": "core",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 15.79,
   "model_calls": 3,
   "input_tokens": 31401,
   "cached_input_tokens": 26752,
   "output_tokens": 181,
   "list_usd": 0.013783
  },
  {
   "arm": "graff",
   "task": "linear-issues",
   "suite": "mcp",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 23.02,
   "model_calls": 2,
   "input_tokens": 22866,
   "cached_input_tokens": 20224,
   "output_tokens": 422,
   "list_usd": 0.011526
  },
  {
   "arm": "graff",
   "task": "linear-issues",
   "suite": "mcp",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 23.65,
   "model_calls": 2,
   "input_tokens": 22836,
   "cached_input_tokens": 20736,
   "output_tokens": 434,
   "list_usd": 0.010614
  },
  {
   "arm": "graff",
   "task": "linear-issues",
   "suite": "mcp",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 21.87,
   "model_calls": 2,
   "input_tokens": 22789,
   "cached_input_tokens": 10112,
   "output_tokens": 386,
   "list_usd": 0.030225
  },
  {
   "arm": "graff",
   "task": "linear-nohint",
   "suite": "mcp",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 20.44,
   "model_calls": 2,
   "input_tokens": 22777,
   "cached_input_tokens": 20224,
   "output_tokens": 399,
   "list_usd": 0.011118
  },
  {
   "arm": "graff",
   "task": "linear-nohint",
   "suite": "mcp",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 37.99,
   "model_calls": 5,
   "input_tokens": 58041,
   "cached_input_tokens": 52352,
   "output_tokens": 561,
   "list_usd": 0.022223
  },
  {
   "arm": "graff",
   "task": "linear-nohint",
   "suite": "mcp",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 33.81,
   "model_calls": 5,
   "input_tokens": 58107,
   "cached_input_tokens": 50560,
   "output_tokens": 525,
   "list_usd": 0.0254
  },
  {
   "arm": "graff",
   "task": "linear-nohint-warm",
   "suite": "mcp",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 34.92,
   "model_calls": 4,
   "input_tokens": 47810,
   "cached_input_tokens": 40448,
   "output_tokens": 611,
   "list_usd": 0.024879
  },
  {
   "arm": "graff",
   "task": "linear-nohint-warm",
   "suite": "mcp",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 26.5,
   "model_calls": 3,
   "input_tokens": 35402,
   "cached_input_tokens": 31104,
   "output_tokens": 451,
   "list_usd": 0.016216
  },
  {
   "arm": "graff",
   "task": "linear-nohint-warm",
   "suite": "mcp",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 21.9,
   "model_calls": 2,
   "input_tokens": 23300,
   "cached_input_tokens": 20992,
   "output_tokens": 420,
   "list_usd": 0.010915
  },
  {
   "arm": "graff",
   "task": "linear-quiet",
   "suite": "mcp",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 22.5,
   "model_calls": 2,
   "input_tokens": 22933,
   "cached_input_tokens": 20736,
   "output_tokens": 421,
   "list_usd": 0.010678
  },
  {
   "arm": "graff",
   "task": "linear-quiet",
   "suite": "mcp",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 23.89,
   "model_calls": 2,
   "input_tokens": 22916,
   "cached_input_tokens": 20736,
   "output_tokens": 409,
   "list_usd": 0.010524
  },
  {
   "arm": "graff",
   "task": "linear-quiet",
   "suite": "mcp",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 19.72,
   "model_calls": 2,
   "input_tokens": 22815,
   "cached_input_tokens": 20224,
   "output_tokens": 357,
   "list_usd": 0.010774
  },
  {
   "arm": "graff",
   "task": "linear-quiet-warm",
   "suite": "mcp",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 21.03,
   "model_calls": 2,
   "input_tokens": 23413,
   "cached_input_tokens": 20224,
   "output_tokens": 385,
   "list_usd": 0.01225
  },
  {
   "arm": "graff",
   "task": "linear-quiet-warm",
   "suite": "mcp",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 21.77,
   "model_calls": 2,
   "input_tokens": 23480,
   "cached_input_tokens": 20992,
   "output_tokens": 442,
   "list_usd": 0.011495
  },
  {
   "arm": "graff",
   "task": "linear-quiet-warm",
   "suite": "mcp",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 29.15,
   "model_calls": 3,
   "input_tokens": 35478,
   "cached_input_tokens": 31872,
   "output_tokens": 555,
   "list_usd": 0.015949
  },
  {
   "arm": "graff",
   "task": "linear-reduce",
   "suite": "mcp",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 21.84,
   "model_calls": 2,
   "input_tokens": 22900,
   "cached_input_tokens": 20736,
   "output_tokens": 425,
   "list_usd": 0.010652
  },
  {
   "arm": "graff",
   "task": "linear-reduce",
   "suite": "mcp",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 25.31,
   "model_calls": 2,
   "input_tokens": 23013,
   "cached_input_tokens": 20736,
   "output_tokens": 487,
   "list_usd": 0.011498
  },
  {
   "arm": "graff",
   "task": "linear-reduce",
   "suite": "mcp",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 20.31,
   "model_calls": 2,
   "input_tokens": 22917,
   "cached_input_tokens": 20736,
   "output_tokens": 395,
   "list_usd": 0.010386
  },
  {
   "arm": "graff",
   "task": "linear-reduce-warm",
   "suite": "mcp",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 23.4,
   "model_calls": 3,
   "input_tokens": 35190,
   "cached_input_tokens": 31104,
   "output_tokens": 442,
   "list_usd": 0.015702
  },
  {
   "arm": "graff",
   "task": "linear-reduce-warm",
   "suite": "mcp",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 31.58,
   "model_calls": 3,
   "input_tokens": 35247,
   "cached_input_tokens": 31872,
   "output_tokens": 491,
   "list_usd": 0.014847
  },
  {
   "arm": "graff",
   "task": "linear-reduce-warm",
   "suite": "mcp",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 24.86,
   "model_calls": 3,
   "input_tokens": 35204,
   "cached_input_tokens": 32640,
   "output_tokens": 455,
   "list_usd": 0.012942
  },
  {
   "arm": "graff",
   "task": "linear-sidecar",
   "suite": "mcp",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 36.66,
   "model_calls": 4,
   "input_tokens": 50303,
   "cached_input_tokens": 42112,
   "output_tokens": 690,
   "list_usd": 0.027493
  },
  {
   "arm": "graff",
   "task": "linear-sidecar",
   "suite": "mcp",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 41.79,
   "model_calls": 5,
   "input_tokens": 65453,
   "cached_input_tokens": 59392,
   "output_tokens": 853,
   "list_usd": 0.026591
  },
  {
   "arm": "graff",
   "task": "linear-sidecar",
   "suite": "mcp",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 36.38,
   "model_calls": 4,
   "input_tokens": 51164,
   "cached_input_tokens": 45440,
   "output_tokens": 553,
   "list_usd": 0.021522
  },
  {
   "arm": "graff",
   "task": "linear-split",
   "suite": "mcp",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 46.43,
   "model_calls": 8,
   "input_tokens": 83470,
   "cached_input_tokens": 77312,
   "output_tokens": 1175,
   "list_usd": 0.031797
  },
  {
   "arm": "graff",
   "task": "linear-split",
   "suite": "mcp",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 63.62,
   "model_calls": 8,
   "input_tokens": 83506,
   "cached_input_tokens": 76672,
   "output_tokens": 1257,
   "list_usd": 0.033905
  },
  {
   "arm": "graff",
   "task": "linear-split",
   "suite": "mcp",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 47.37,
   "model_calls": 8,
   "input_tokens": 83664,
   "cached_input_tokens": 78336,
   "output_tokens": 1255,
   "list_usd": 0.03104
  },
  {
   "arm": "graff",
   "task": "linear-warm",
   "suite": "mcp",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 22.62,
   "model_calls": 2,
   "input_tokens": 23364,
   "cached_input_tokens": 20224,
   "output_tokens": 418,
   "list_usd": 0.012482
  },
  {
   "arm": "graff",
   "task": "linear-warm",
   "suite": "mcp",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 21.14,
   "model_calls": 2,
   "input_tokens": 23372,
   "cached_input_tokens": 20224,
   "output_tokens": 421,
   "list_usd": 0.012528
  },
  {
   "arm": "graff",
   "task": "linear-warm",
   "suite": "mcp",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 25.22,
   "model_calls": 2,
   "input_tokens": 23334,
   "cached_input_tokens": 20992,
   "output_tokens": 415,
   "list_usd": 0.010933
  },
  {
   "arm": "graff",
   "task": "recall-noise",
   "suite": "core",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 10.96,
   "model_calls": 2,
   "input_tokens": 20929,
   "cached_input_tokens": 20480,
   "output_tokens": 29,
   "list_usd": 0.003236
  },
  {
   "arm": "graff",
   "task": "recall-noise",
   "suite": "core",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 8.48,
   "model_calls": 2,
   "input_tokens": 20929,
   "cached_input_tokens": 16768,
   "output_tokens": 29,
   "list_usd": 0.010289
  },
  {
   "arm": "graff",
   "task": "recall-noise",
   "suite": "core",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 10.2,
   "model_calls": 2,
   "input_tokens": 20929,
   "cached_input_tokens": 16512,
   "output_tokens": 29,
   "list_usd": 0.010775
  },
  {
   "arm": "graff",
   "task": "refactor-rename",
   "suite": "core",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 25.3,
   "model_calls": 4,
   "input_tokens": 42255,
   "cached_input_tokens": 34688,
   "output_tokens": 280,
   "list_usd": 0.021403
  },
  {
   "arm": "graff",
   "task": "refactor-rename",
   "suite": "core",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 22.34,
   "model_calls": 4,
   "input_tokens": 42253,
   "cached_input_tokens": 40448,
   "output_tokens": 280,
   "list_usd": 0.010455
  },
  {
   "arm": "graff",
   "task": "refactor-rename",
   "suite": "core",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 21.46,
   "model_calls": 4,
   "input_tokens": 42240,
   "cached_input_tokens": 40448,
   "output_tokens": 275,
   "list_usd": 0.010379
  },
  {
   "arm": "graff",
   "task": "regex-count",
   "suite": "core",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 15.54,
   "model_calls": 3,
   "input_tokens": 31356,
   "cached_input_tokens": 30336,
   "output_tokens": 153,
   "list_usd": 0.006604
  },
  {
   "arm": "graff",
   "task": "regex-count",
   "suite": "core",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 20.09,
   "model_calls": 4,
   "input_tokens": 41931,
   "cached_input_tokens": 40704,
   "output_tokens": 179,
   "list_usd": 0.008314
  },
  {
   "arm": "graff",
   "task": "regex-count",
   "suite": "core",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 15.89,
   "model_calls": 3,
   "input_tokens": 31358,
   "cached_input_tokens": 30464,
   "output_tokens": 154,
   "list_usd": 0.006374
  },
  {
   "arm": "graff",
   "task": "write-tests",
   "suite": "core",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 21.6,
   "model_calls": 4,
   "input_tokens": 42032,
   "cached_input_tokens": 40960,
   "output_tokens": 255,
   "list_usd": 0.00879
  },
  {
   "arm": "graff",
   "task": "write-tests",
   "suite": "core",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 19.43,
   "model_calls": 4,
   "input_tokens": 41997,
   "cached_input_tokens": 40832,
   "output_tokens": 236,
   "list_usd": 0.008773
  },
  {
   "arm": "graff",
   "task": "write-tests",
   "suite": "core",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 20.05,
   "model_calls": 4,
   "input_tokens": 41989,
   "cached_input_tokens": 40960,
   "output_tokens": 233,
   "list_usd": 0.008484
  },
  {
   "arm": "pi",
   "task": "csv-sum",
   "suite": "core",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 15.0,
   "model_calls": 3,
   "input_tokens": 3573,
   "cached_input_tokens": 1024,
   "output_tokens": 58,
   "list_usd": 0.00578
  },
  {
   "arm": "pi",
   "task": "csv-sum",
   "suite": "core",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 13.29,
   "model_calls": 3,
   "input_tokens": 3573,
   "cached_input_tokens": 2048,
   "output_tokens": 58,
   "list_usd": 0.003835
  },
  {
   "arm": "pi",
   "task": "csv-sum",
   "suite": "core",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 13.23,
   "model_calls": 3,
   "input_tokens": 3573,
   "cached_input_tokens": 1024,
   "output_tokens": 58,
   "list_usd": 0.00578
  },
  {
   "arm": "pi",
   "task": "dead-code",
   "suite": "core",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 18.01,
   "model_calls": 4,
   "input_tokens": 4977,
   "cached_input_tokens": 3200,
   "output_tokens": 73,
   "list_usd": 0.004604
  },
  {
   "arm": "pi",
   "task": "dead-code",
   "suite": "core",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 16.44,
   "model_calls": 4,
   "input_tokens": 4965,
   "cached_input_tokens": 2048,
   "output_tokens": 69,
   "list_usd": 0.006729
  },
  {
   "arm": "pi",
   "task": "dead-code",
   "suite": "core",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 20.18,
   "model_calls": 4,
   "input_tokens": 5082,
   "cached_input_tokens": 1024,
   "output_tokens": 93,
   "list_usd": 0.009148
  },
  {
   "arm": "pi",
   "task": "exact-reply",
   "suite": "core",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 7.38,
   "model_calls": 1,
   "input_tokens": 1115,
   "cached_input_tokens": 0,
   "output_tokens": 5,
   "list_usd": 0.00228
  },
  {
   "arm": "pi",
   "task": "exact-reply",
   "suite": "core",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 6.55,
   "model_calls": 1,
   "input_tokens": 1115,
   "cached_input_tokens": 0,
   "output_tokens": 5,
   "list_usd": 0.00228
  },
  {
   "arm": "pi",
   "task": "exact-reply",
   "suite": "core",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 11.46,
   "model_calls": 1,
   "input_tokens": 1115,
   "cached_input_tokens": 0,
   "output_tokens": 5,
   "list_usd": 0.00228
  },
  {
   "arm": "pi",
   "task": "file-ops",
   "suite": "core",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 25.16,
   "model_calls": 3,
   "input_tokens": 3540,
   "cached_input_tokens": 2048,
   "output_tokens": 78,
   "list_usd": 0.003969
  },
  {
   "arm": "pi",
   "task": "file-ops",
   "suite": "core",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 27.22,
   "model_calls": 3,
   "input_tokens": 3540,
   "cached_input_tokens": 0,
   "output_tokens": 78,
   "list_usd": 0.00786
  },
  {
   "arm": "pi",
   "task": "file-ops",
   "suite": "core",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 16.71,
   "model_calls": 3,
   "input_tokens": 3540,
   "cached_input_tokens": 1024,
   "output_tokens": 78,
   "list_usd": 0.005914
  },
  {
   "arm": "pi",
   "task": "fix-fib",
   "suite": "core",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 22.03,
   "model_calls": 5,
   "input_tokens": 6612,
   "cached_input_tokens": 2176,
   "output_tokens": 149,
   "list_usd": 0.01058
  },
  {
   "arm": "pi",
   "task": "fix-fib",
   "suite": "core",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 22.07,
   "model_calls": 5,
   "input_tokens": 6612,
   "cached_input_tokens": 2304,
   "output_tokens": 149,
   "list_usd": 0.010336
  },
  {
   "arm": "pi",
   "task": "fix-fib",
   "suite": "core",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 22.87,
   "model_calls": 5,
   "input_tokens": 6612,
   "cached_input_tokens": 2432,
   "output_tokens": 149,
   "list_usd": 0.010093
  },
  {
   "arm": "pi",
   "task": "git-ops",
   "suite": "core",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 14.13,
   "model_calls": 3,
   "input_tokens": 3956,
   "cached_input_tokens": 2176,
   "output_tokens": 137,
   "list_usd": 0.005148
  },
  {
   "arm": "pi",
   "task": "git-ops",
   "suite": "core",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 15.88,
   "model_calls": 3,
   "input_tokens": 3971,
   "cached_input_tokens": 2176,
   "output_tokens": 143,
   "list_usd": 0.005238
  },
  {
   "arm": "pi",
   "task": "git-ops",
   "suite": "core",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 16.54,
   "model_calls": 3,
   "input_tokens": 3957,
   "cached_input_tokens": 1024,
   "output_tokens": 142,
   "list_usd": 0.007388
  },
  {
   "arm": "pi",
   "task": "json-transform",
   "suite": "core",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 15.03,
   "model_calls": 3,
   "input_tokens": 3663,
   "cached_input_tokens": 1024,
   "output_tokens": 94,
   "list_usd": 0.00632
  },
  {
   "arm": "pi",
   "task": "json-transform",
   "suite": "core",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 15.51,
   "model_calls": 3,
   "input_tokens": 3663,
   "cached_input_tokens": 2048,
   "output_tokens": 94,
   "list_usd": 0.004375
  },
  {
   "arm": "pi",
   "task": "json-transform",
   "suite": "core",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 15.01,
   "model_calls": 3,
   "input_tokens": 3663,
   "cached_input_tokens": 0,
   "output_tokens": 95,
   "list_usd": 0.008276
  },
  {
   "arm": "pi",
   "task": "recall-noise",
   "suite": "core",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 9.56,
   "model_calls": 2,
   "input_tokens": 2499,
   "cached_input_tokens": 0,
   "output_tokens": 28,
   "list_usd": 0.005278
  },
  {
   "arm": "pi",
   "task": "recall-noise",
   "suite": "core",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 9.73,
   "model_calls": 2,
   "input_tokens": 2499,
   "cached_input_tokens": 0,
   "output_tokens": 28,
   "list_usd": 0.005278
  },
  {
   "arm": "pi",
   "task": "recall-noise",
   "suite": "core",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 11.65,
   "model_calls": 2,
   "input_tokens": 2499,
   "cached_input_tokens": 0,
   "output_tokens": 28,
   "list_usd": 0.005278
  },
  {
   "arm": "pi",
   "task": "refactor-rename",
   "suite": "core",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 23.26,
   "model_calls": 5,
   "input_tokens": 7583,
   "cached_input_tokens": 2688,
   "output_tokens": 289,
   "list_usd": 0.012949
  },
  {
   "arm": "pi",
   "task": "refactor-rename",
   "suite": "core",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 24.13,
   "model_calls": 5,
   "input_tokens": 6947,
   "cached_input_tokens": 1152,
   "output_tokens": 281,
   "list_usd": 0.014515
  },
  {
   "arm": "pi",
   "task": "refactor-rename",
   "suite": "core",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 22.81,
   "model_calls": 5,
   "input_tokens": 7008,
   "cached_input_tokens": 2432,
   "output_tokens": 279,
   "list_usd": 0.012185
  },
  {
   "arm": "pi",
   "task": "regex-count",
   "suite": "core",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 11.5,
   "model_calls": 2,
   "input_tokens": 2361,
   "cached_input_tokens": 1024,
   "output_tokens": 55,
   "list_usd": 0.003326
  },
  {
   "arm": "pi",
   "task": "regex-count",
   "suite": "core",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 12.91,
   "model_calls": 2,
   "input_tokens": 2357,
   "cached_input_tokens": 1024,
   "output_tokens": 51,
   "list_usd": 0.003278
  },
  {
   "arm": "pi",
   "task": "regex-count",
   "suite": "core",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 15.61,
   "model_calls": 2,
   "input_tokens": 2357,
   "cached_input_tokens": 1024,
   "output_tokens": 51,
   "list_usd": 0.003278
  },
  {
   "arm": "pi",
   "task": "write-tests",
   "suite": "core",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 29.83,
   "model_calls": 5,
   "input_tokens": 6414,
   "cached_input_tokens": 2048,
   "output_tokens": 266,
   "list_usd": 0.011597
  },
  {
   "arm": "pi",
   "task": "write-tests",
   "suite": "core",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 22.82,
   "model_calls": 5,
   "input_tokens": 6278,
   "cached_input_tokens": 3200,
   "output_tokens": 196,
   "list_usd": 0.008436
  },
  {
   "arm": "pi",
   "task": "write-tests",
   "suite": "core",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 18.68,
   "model_calls": 4,
   "input_tokens": 5003,
   "cached_input_tokens": 2048,
   "output_tokens": 179,
   "list_usd": 0.007905
  }
 ]
}