Skip to content

graff — a fast agentic coding harness in Zig: multi-provider, MCP, workflows, DGM evolution loop, TS/Python SDKs

Python
45.1 KB
Raw Blame
{
 "model": {
  "graff-v302-12": "grok-4.7 (public xAI API)",
  "grok-cli": "grok-4.7, served as grok-4.7-build on the Grok CLI route"
 },
 "tasks": 21,
 "reps": 3,
 "time_limit_s": 240,
 "arms": {
  "graff-v302-12": "graff v0.0.302.12 (release)",
  "grok-cli": "Grok CLI 1.0.41"
 },
 "list_rates_per_1m": {
  "input": 2.0,
  "cached_input": 0.5,
  "output": 6.0
 },
 "summary": {
  "all": {
   "graff-v302-12": {
    "runs": 63,
    "passed": 63,
    "timed_out": 0,
    "runs_without_usage": 0,
    "wall_s_per_task": 15.5,
    "list_usd_per_task": 0.0391,
    "prompt_tokens_per_task": 29696,
    "cached_input_share": 0.567,
    "output_tokens_per_task": 823,
    "model_calls_per_task": 4.3
   },
   "grok-cli": {
    "runs": 63,
    "passed": 54,
    "timed_out": 13,
    "runs_without_usage": 13,
    "wall_s_per_task": 106.9,
    "list_usd_per_task": 0.3306,
    "prompt_tokens_per_task": 466465,
    "cached_input_share": 0.895,
    "output_tokens_per_task": 4007,
    "model_calls_per_task": 10.4,
    "grok_reported_usd_per_task": 0.0981
   }
  },
  "core": {
   "graff-v302-12": {
    "runs": 33,
    "passed": 33,
    "timed_out": 0,
    "runs_without_usage": 0,
    "wall_s_per_task": 6.6,
    "list_usd_per_task": 0.0215,
    "prompt_tokens_per_task": 19262,
    "cached_input_share": 0.645,
    "output_tokens_per_task": 263,
    "model_calls_per_task": 3.1
   },
   "grok-cli": {
    "runs": 33,
    "passed": 33,
    "timed_out": 0,
    "runs_without_usage": 0,
    "wall_s_per_task": 14.8,
    "list_usd_per_task": 0.0578,
    "prompt_tokens_per_task": 67046,
    "cached_input_share": 0.794,
    "output_tokens_per_task": 580,
    "model_calls_per_task": 3.6,
    "grok_reported_usd_per_task": 0.0196
   }
  },
  "mcp": {
   "graff-v302-12": {
    "runs": 30,
    "passed": 30,
    "timed_out": 0,
    "runs_without_usage": 0,
    "wall_s_per_task": 25.3,
    "list_usd_per_task": 0.0584,
    "prompt_tokens_per_task": 41174,
    "cached_input_share": 0.527,
    "output_tokens_per_task": 1439,
    "model_calls_per_task": 5.7
   },
   "grok-cli": {
    "runs": 30,
    "passed": 21,
    "timed_out": 13,
    "runs_without_usage": 13,
    "wall_s_per_task": 208.4,
    "list_usd_per_task": 0.8602,
    "prompt_tokens_per_task": 1241807,
    "cached_input_share": 0.906,
    "output_tokens_per_task": 10659,
    "model_calls_per_task": 23.5,
    "grok_reported_usd_per_task": 0.2503
   }
  }
 },
 "per_task": {
  "csv-sum": {
   "suite": "core",
   "graff-v302-12": {
    "passed": 3,
    "runs": 3,
    "wall_s": 6.2,
    "model_calls": 3.7,
    "list_usd": 0.0187
   },
   "grok-cli": {
    "passed": 3,
    "runs": 3,
    "wall_s": 18.5,
    "model_calls": 5,
    "list_usd": 0.0673
   }
  },
  "dead-code": {
   "suite": "core",
   "graff-v302-12": {
    "passed": 3,
    "runs": 3,
    "wall_s": 5.0,
    "model_calls": 3,
    "list_usd": 0.021
   },
   "grok-cli": {
    "passed": 3,
    "runs": 3,
    "wall_s": 12.3,
    "model_calls": 3,
    "list_usd": 0.047
   }
  },
  "exact-reply": {
   "suite": "core",
   "graff-v302-12": {
    "passed": 3,
    "runs": 3,
    "wall_s": 2.0,
    "model_calls": 1,
    "list_usd": 0.0081
   },
   "grok-cli": {
    "passed": 3,
    "runs": 3,
    "wall_s": 5.0,
    "model_calls": 1,
    "list_usd": 0.0273
   }
  },
  "file-ops": {
   "suite": "core",
   "graff-v302-12": {
    "passed": 3,
    "runs": 3,
    "wall_s": 5.2,
    "model_calls": 2,
    "list_usd": 0.0131
   },
   "grok-cli": {
    "passed": 3,
    "runs": 3,
    "wall_s": 7.1,
    "model_calls": 2,
    "list_usd": 0.0294
   }
  },
  "fix-fib": {
   "suite": "core",
   "graff-v302-12": {
    "passed": 3,
    "runs": 3,
    "wall_s": 9.6,
    "model_calls": 4,
    "list_usd": 0.0294
   },
   "grok-cli": {
    "passed": 3,
    "runs": 3,
    "wall_s": 17.0,
    "model_calls": 4,
    "list_usd": 0.0716
   }
  },
  "git-ops": {
   "suite": "core",
   "graff-v302-12": {
    "passed": 3,
    "runs": 3,
    "wall_s": 8.5,
    "model_calls": 3,
    "list_usd": 0.0303
   },
   "grok-cli": {
    "passed": 3,
    "runs": 3,
    "wall_s": 16.7,
    "model_calls": 3,
    "list_usd": 0.0528
   }
  },
  "json-transform": {
   "suite": "core",
   "graff-v302-12": {
    "passed": 3,
    "runs": 3,
    "wall_s": 5.8,
    "model_calls": 3,
    "list_usd": 0.0184
   },
   "grok-cli": {
    "passed": 3,
    "runs": 3,
    "wall_s": 16.4,
    "model_calls": 4,
    "list_usd": 0.0621
   }
  },
  "linear-issues": {
   "suite": "mcp",
   "graff-v302-12": {
    "passed": 3,
    "runs": 3,
    "wall_s": 19.7,
    "model_calls": 4,
    "list_usd": 0.0494
   },
   "grok-cli": {
    "passed": 3,
    "runs": 3,
    "wall_s": 173.0,
    "model_calls": 16.5,
    "list_usd": 0.5809
   }
  },
  "linear-nohint": {
   "suite": "mcp",
   "graff-v302-12": {
    "passed": 3,
    "runs": 3,
    "wall_s": 24.5,
    "model_calls": 6,
    "list_usd": 0.0676
   },
   "grok-cli": {
    "passed": 2,
    "runs": 3,
    "wall_s": 214.5,
    "model_calls": 28.5,
    "list_usd": 1.0344
   }
  },
  "linear-nohint-warm": {
   "suite": "mcp",
   "graff-v302-12": {
    "passed": 3,
    "runs": 3,
    "wall_s": 20.0,
    "model_calls": 5.7,
    "list_usd": 0.048
   },
   "grok-cli": {
    "passed": 3,
    "runs": 3,
    "wall_s": 189.9,
    "model_calls": 27,
    "list_usd": 1.0498
   }
  },
  "linear-quiet": {
   "suite": "mcp",
   "graff-v302-12": {
    "passed": 3,
    "runs": 3,
    "wall_s": 21.3,
    "model_calls": 4.3,
    "list_usd": 0.0424
   },
   "grok-cli": {
    "passed": 2,
    "runs": 3,
    "wall_s": 208.1,
    "model_calls": 24,
    "list_usd": 0.933
   }
  },
  "linear-quiet-warm": {
   "suite": "mcp",
   "graff-v302-12": {
    "passed": 3,
    "runs": 3,
    "wall_s": 23.1,
    "model_calls": 4,
    "list_usd": 0.0388
   },
   "grok-cli": {
    "passed": 2,
    "runs": 3,
    "wall_s": 203.9,
    "model_calls": 22.5,
    "list_usd": 0.7481
   }
  },
  "linear-reduce": {
   "suite": "mcp",
   "graff-v302-12": {
    "passed": 3,
    "runs": 3,
    "wall_s": 21.3,
    "model_calls": 4.7,
    "list_usd": 0.0527
   },
   "grok-cli": {
    "passed": 3,
    "runs": 3,
    "wall_s": 214.5,
    "model_calls": 18,
    "list_usd": 0.5054
   }
  },
  "linear-reduce-warm": {
   "suite": "mcp",
   "graff-v302-12": {
    "passed": 3,
    "runs": 3,
    "wall_s": 14.9,
    "model_calls": 4.3,
    "list_usd": 0.0469
   },
   "grok-cli": {
    "passed": 2,
    "runs": 3,
    "wall_s": 210.9,
    "model_calls": 18,
    "list_usd": 0.6274
   }
  },
  "linear-sidecar": {
   "suite": "mcp",
   "graff-v302-12": {
    "passed": 3,
    "runs": 3,
    "wall_s": 46.7,
    "model_calls": 9,
    "list_usd": 0.0878
   },
   "grok-cli": {
    "passed": 2,
    "runs": 3,
    "wall_s": 233.3,
    "model_calls": 29,
    "list_usd": 1.0836
   }
  },
  "linear-split": {
   "suite": "mcp",
   "graff-v302-12": {
    "passed": 3,
    "runs": 3,
    "wall_s": 42.5,
    "model_calls": 11.3,
    "list_usd": 0.1089
   },
   "grok-cli": {
    "passed": 0,
    "runs": 3,
    "wall_s": 240.5,
    "model_calls": 0,
    "list_usd": 0
   }
  },
  "linear-warm": {
   "suite": "mcp",
   "graff-v302-12": {
    "passed": 3,
    "runs": 3,
    "wall_s": 18.7,
    "model_calls": 4,
    "list_usd": 0.0416
   },
   "grok-cli": {
    "passed": 2,
    "runs": 3,
    "wall_s": 194.9,
    "model_calls": 20.5,
    "list_usd": 0.791
   }
  },
  "recall-noise": {
   "suite": "core",
   "graff-v302-12": {
    "passed": 3,
    "runs": 3,
    "wall_s": 4.0,
    "model_calls": 2,
    "list_usd": 0.0171
   },
   "grok-cli": {
    "passed": 3,
    "runs": 3,
    "wall_s": 7.3,
    "model_calls": 2.3,
    "list_usd": 0.0439
   }
  },
  "refactor-rename": {
   "suite": "core",
   "graff-v302-12": {
    "passed": 3,
    "runs": 3,
    "wall_s": 8.4,
    "model_calls": 5,
    "list_usd": 0.0316
   },
   "grok-cli": {
    "passed": 3,
    "runs": 3,
    "wall_s": 18.8,
    "model_calls": 5,
    "list_usd": 0.0706
   }
  },
  "regex-count": {
   "suite": "core",
   "graff-v302-12": {
    "passed": 3,
    "runs": 3,
    "wall_s": 10.8,
    "model_calls": 4,
    "list_usd": 0.0268
   },
   "grok-cli": {
    "passed": 3,
    "runs": 3,
    "wall_s": 26.4,
    "model_calls": 5.3,
    "list_usd": 0.0765
   }
  },
  "write-tests": {
   "suite": "core",
   "graff-v302-12": {
    "passed": 3,
    "runs": 3,
    "wall_s": 6.6,
    "model_calls": 3,
    "list_usd": 0.0218
   },
   "grok-cli": {
    "passed": 3,
    "runs": 3,
    "wall_s": 16.8,
    "model_calls": 5,
    "list_usd": 0.087
   }
  }
 },
 "runs": [
  {
   "arm": "graff-v302-12",
   "task": "csv-sum",
   "suite": "core",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 6.87,
   "model_calls": 4,
   "input_tokens": 25430,
   "cached_input_tokens": 19840,
   "output_tokens": 272,
   "list_usd": 0.022732
  },
  {
   "arm": "graff-v302-12",
   "task": "csv-sum",
   "suite": "core",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 6.62,
   "model_calls": 4,
   "input_tokens": 25344,
   "cached_input_tokens": 19840,
   "output_tokens": 226,
   "list_usd": 0.022284
  },
  {
   "arm": "graff-v302-12",
   "task": "csv-sum",
   "suite": "core",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 5.19,
   "model_calls": 3,
   "input_tokens": 18617,
   "cached_input_tokens": 18176,
   "output_tokens": 181,
   "list_usd": 0.011056
  },
  {
   "arm": "grok-cli",
   "task": "csv-sum",
   "suite": "core",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 15.46,
   "model_calls": 5,
   "input_tokens": 92148,
   "cached_input_tokens": 85120,
   "output_tokens": 532,
   "list_usd": 0.059808,
   "grok_reported_usd": 0.020335
  },
  {
   "arm": "grok-cli",
   "task": "csv-sum",
   "suite": "core",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 19.59,
   "model_calls": 5,
   "input_tokens": 92602,
   "cached_input_tokens": 85120,
   "output_tokens": 926,
   "list_usd": 0.06308,
   "grok_reported_usd": 0.021447
  },
  {
   "arm": "grok-cli",
   "task": "csv-sum",
   "suite": "core",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 20.4,
   "model_calls": 5,
   "input_tokens": 92665,
   "cached_input_tokens": 74496,
   "output_tokens": 910,
   "list_usd": 0.079046,
   "grok_reported_usd": 0.026876
  },
  {
   "arm": "graff-v302-12",
   "task": "dead-code",
   "suite": "core",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 4.08,
   "model_calls": 3,
   "input_tokens": 18681,
   "cached_input_tokens": 18176,
   "output_tokens": 102,
   "list_usd": 0.01071
  },
  {
   "arm": "graff-v302-12",
   "task": "dead-code",
   "suite": "core",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 5.22,
   "model_calls": 3,
   "input_tokens": 18621,
   "cached_input_tokens": 7808,
   "output_tokens": 206,
   "list_usd": 0.026766
  },
  {
   "arm": "graff-v302-12",
   "task": "dead-code",
   "suite": "core",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 5.77,
   "model_calls": 3,
   "input_tokens": 18613,
   "cached_input_tokens": 8320,
   "output_tokens": 110,
   "list_usd": 0.025406
  },
  {
   "arm": "grok-cli",
   "task": "dead-code",
   "suite": "core",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 16.77,
   "model_calls": 3,
   "input_tokens": 55529,
   "cached_input_tokens": 48512,
   "output_tokens": 380,
   "list_usd": 0.04057,
   "grok_reported_usd": 0.013794
  },
  {
   "arm": "grok-cli",
   "task": "dead-code",
   "suite": "core",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 10.26,
   "model_calls": 3,
   "input_tokens": 55263,
   "cached_input_tokens": 48512,
   "output_tokens": 316,
   "list_usd": 0.039654,
   "grok_reported_usd": 0.013482
  },
  {
   "arm": "grok-cli",
   "task": "dead-code",
   "suite": "core",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 10.01,
   "model_calls": 3,
   "input_tokens": 55538,
   "cached_input_tokens": 35072,
   "output_tokens": 374,
   "list_usd": 0.060712,
   "grok_reported_usd": 0.020642
  },
  {
   "arm": "graff-v302-12",
   "task": "exact-reply",
   "suite": "core",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 1.43,
   "model_calls": 1,
   "input_tokens": 6055,
   "cached_input_tokens": 1152,
   "output_tokens": 32,
   "list_usd": 0.010574
  },
  {
   "arm": "graff-v302-12",
   "task": "exact-reply",
   "suite": "core",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 2.78,
   "model_calls": 1,
   "input_tokens": 6055,
   "cached_input_tokens": 6016,
   "output_tokens": 32,
   "list_usd": 0.003278
  },
  {
   "arm": "graff-v302-12",
   "task": "exact-reply",
   "suite": "core",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 1.7,
   "model_calls": 1,
   "input_tokens": 6055,
   "cached_input_tokens": 1152,
   "output_tokens": 26,
   "list_usd": 0.010538
  },
  {
   "arm": "grok-cli",
   "task": "exact-reply",
   "suite": "core",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 4.7,
   "model_calls": 1,
   "input_tokens": 18126,
   "cached_input_tokens": 4608,
   "output_tokens": 30,
   "list_usd": 0.02952,
   "grok_reported_usd": 0.010037
  },
  {
   "arm": "grok-cli",
   "task": "exact-reply",
   "suite": "core",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 4.72,
   "model_calls": 1,
   "input_tokens": 18125,
   "cached_input_tokens": 1664,
   "output_tokens": 29,
   "list_usd": 0.033928,
   "grok_reported_usd": 0.011536
  },
  {
   "arm": "grok-cli",
   "task": "exact-reply",
   "suite": "core",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 5.43,
   "model_calls": 1,
   "input_tokens": 18126,
   "cached_input_tokens": 12032,
   "output_tokens": 30,
   "list_usd": 0.018384,
   "grok_reported_usd": 0.006251
  },
  {
   "arm": "graff-v302-12",
   "task": "file-ops",
   "suite": "core",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 5.32,
   "model_calls": 2,
   "input_tokens": 12272,
   "cached_input_tokens": 2304,
   "output_tokens": 241,
   "list_usd": 0.022534
  },
  {
   "arm": "graff-v302-12",
   "task": "file-ops",
   "suite": "core",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 4.73,
   "model_calls": 2,
   "input_tokens": 12251,
   "cached_input_tokens": 12032,
   "output_tokens": 280,
   "list_usd": 0.008134
  },
  {
   "arm": "graff-v302-12",
   "task": "file-ops",
   "suite": "core",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 5.57,
   "model_calls": 2,
   "input_tokens": 12261,
   "cached_input_tokens": 12032,
   "output_tokens": 339,
   "list_usd": 0.008508
  },
  {
   "arm": "grok-cli",
   "task": "file-ops",
   "suite": "core",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 7.14,
   "model_calls": 2,
   "input_tokens": 36599,
   "cached_input_tokens": 30080,
   "output_tokens": 213,
   "list_usd": 0.029356,
   "grok_reported_usd": 0.009981
  },
  {
   "arm": "grok-cli",
   "task": "file-ops",
   "suite": "core",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 7.63,
   "model_calls": 2,
   "input_tokens": 36458,
   "cached_input_tokens": 30080,
   "output_tokens": 279,
   "list_usd": 0.02947,
   "grok_reported_usd": 0.01002
  },
  {
   "arm": "grok-cli",
   "task": "file-ops",
   "suite": "core",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 6.63,
   "model_calls": 2,
   "input_tokens": 36468,
   "cached_input_tokens": 30080,
   "output_tokens": 254,
   "list_usd": 0.02934,
   "grok_reported_usd": 0.009976
  },
  {
   "arm": "graff-v302-12",
   "task": "fix-fib",
   "suite": "core",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 7.6,
   "model_calls": 4,
   "input_tokens": 25459,
   "cached_input_tokens": 14976,
   "output_tokens": 266,
   "list_usd": 0.03005
  },
  {
   "arm": "graff-v302-12",
   "task": "fix-fib",
   "suite": "core",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 11.83,
   "model_calls": 4,
   "input_tokens": 26049,
   "cached_input_tokens": 9728,
   "output_tokens": 585,
   "list_usd": 0.041016
  },
  {
   "arm": "graff-v302-12",
   "task": "fix-fib",
   "suite": "core",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 9.43,
   "model_calls": 4,
   "input_tokens": 25880,
   "cached_input_tokens": 24960,
   "output_tokens": 483,
   "list_usd": 0.017218
  },
  {
   "arm": "grok-cli",
   "task": "fix-fib",
   "suite": "core",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 17.61,
   "model_calls": 4,
   "input_tokens": 74856,
   "cached_input_tokens": 45056,
   "output_tokens": 778,
   "list_usd": 0.086796,
   "grok_reported_usd": 0.029511
  },
  {
   "arm": "grok-cli",
   "task": "fix-fib",
   "suite": "core",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 15.6,
   "model_calls": 4,
   "input_tokens": 74808,
   "cached_input_tokens": 54016,
   "output_tokens": 740,
   "list_usd": 0.073032,
   "grok_reported_usd": 0.024831
  },
  {
   "arm": "grok-cli",
   "task": "fix-fib",
   "suite": "core",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 17.75,
   "model_calls": 4,
   "input_tokens": 75374,
   "cached_input_tokens": 67840,
   "output_tokens": 982,
   "list_usd": 0.05488,
   "grok_reported_usd": 0.018659
  },
  {
   "arm": "graff-v302-12",
   "task": "git-ops",
   "suite": "core",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 6.83,
   "model_calls": 3,
   "input_tokens": 19044,
   "cached_input_tokens": 7808,
   "output_tokens": 289,
   "list_usd": 0.02811
  },
  {
   "arm": "graff-v302-12",
   "task": "git-ops",
   "suite": "core",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 8.47,
   "model_calls": 3,
   "input_tokens": 19053,
   "cached_input_tokens": 8320,
   "output_tokens": 334,
   "list_usd": 0.02763
  },
  {
   "arm": "graff-v302-12",
   "task": "git-ops",
   "suite": "core",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 10.2,
   "model_calls": 3,
   "input_tokens": 19363,
   "cached_input_tokens": 3456,
   "output_tokens": 257,
   "list_usd": 0.035084
  },
  {
   "arm": "grok-cli",
   "task": "git-ops",
   "suite": "core",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 16.17,
   "model_calls": 3,
   "input_tokens": 55655,
   "cached_input_tokens": 42496,
   "output_tokens": 770,
   "list_usd": 0.052186,
   "grok_reported_usd": 0.017743
  },
  {
   "arm": "grok-cli",
   "task": "git-ops",
   "suite": "core",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 13.88,
   "model_calls": 3,
   "input_tokens": 55929,
   "cached_input_tokens": 48640,
   "output_tokens": 587,
   "list_usd": 0.04242,
   "grok_reported_usd": 0.014423
  },
  {
   "arm": "grok-cli",
   "task": "git-ops",
   "suite": "core",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 19.98,
   "model_calls": 3,
   "input_tokens": 56324,
   "cached_input_tokens": 36608,
   "output_tokens": 1026,
   "list_usd": 0.063892,
   "grok_reported_usd": 0.021723
  },
  {
   "arm": "graff-v302-12",
   "task": "json-transform",
   "suite": "core",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 5.23,
   "model_calls": 3,
   "input_tokens": 18583,
   "cached_input_tokens": 8320,
   "output_tokens": 187,
   "list_usd": 0.025808
  },
  {
   "arm": "graff-v302-12",
   "task": "json-transform",
   "suite": "core",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 4.33,
   "model_calls": 3,
   "input_tokens": 18513,
   "cached_input_tokens": 18176,
   "output_tokens": 154,
   "list_usd": 0.010686
  },
  {
   "arm": "graff-v302-12",
   "task": "json-transform",
   "suite": "core",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 7.97,
   "model_calls": 3,
   "input_tokens": 18575,
   "cached_input_tokens": 13312,
   "output_tokens": 244,
   "list_usd": 0.018646
  },
  {
   "arm": "grok-cli",
   "task": "json-transform",
   "suite": "core",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 23.08,
   "model_calls": 5,
   "input_tokens": 93868,
   "cached_input_tokens": 86272,
   "output_tokens": 1074,
   "list_usd": 0.064772,
   "grok_reported_usd": 0.022022
  },
  {
   "arm": "grok-cli",
   "task": "json-transform",
   "suite": "core",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 14.76,
   "model_calls": 4,
   "input_tokens": 73895,
   "cached_input_tokens": 47104,
   "output_tokens": 619,
   "list_usd": 0.080848,
   "grok_reported_usd": 0.027488
  },
  {
   "arm": "grok-cli",
   "task": "json-transform",
   "suite": "core",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 11.51,
   "model_calls": 3,
   "input_tokens": 55144,
   "cached_input_tokens": 48384,
   "output_tokens": 498,
   "list_usd": 0.0407,
   "grok_reported_usd": 0.013838
  },
  {
   "arm": "graff-v302-12",
   "task": "linear-issues",
   "suite": "mcp",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 23.65,
   "model_calls": 4,
   "input_tokens": 29665,
   "cached_input_tokens": 11392,
   "output_tokens": 1952,
   "list_usd": 0.053954
  },
  {
   "arm": "graff-v302-12",
   "task": "linear-issues",
   "suite": "mcp",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 15.2,
   "model_calls": 4,
   "input_tokens": 27981,
   "cached_input_tokens": 14976,
   "output_tokens": 1087,
   "list_usd": 0.04002
  },
  {
   "arm": "graff-v302-12",
   "task": "linear-issues",
   "suite": "mcp",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 20.29,
   "model_calls": 4,
   "input_tokens": 28490,
   "cached_input_tokens": 7552,
   "output_tokens": 1441,
   "list_usd": 0.054298
  },
  {
   "arm": "grok-cli",
   "task": "linear-issues",
   "suite": "mcp",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 103.43,
   "model_calls": 12,
   "input_tokens": 482817,
   "cached_input_tokens": 362368,
   "output_tokens": 7106,
   "list_usd": 0.464718,
   "grok_reported_usd": 0.158004
  },
  {
   "arm": "grok-cli",
   "task": "linear-issues",
   "suite": "mcp",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 175.26,
   "model_calls": 21,
   "input_tokens": 949340,
   "cached_input_tokens": 841344,
   "output_tokens": 10073,
   "list_usd": 0.697102,
   "grok_reported_usd": 0.237015
  },
  {
   "arm": "grok-cli",
   "task": "linear-issues",
   "suite": "mcp",
   "rep": 3,
   "pass": true,
   "timed_out": true,
   "wall_s": 240.29,
   "model_calls": null,
   "input_tokens": null,
   "cached_input_tokens": null,
   "output_tokens": null,
   "list_usd": null,
   "grok_reported_usd": null
  },
  {
   "arm": "graff-v302-12",
   "task": "linear-nohint",
   "suite": "mcp",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 21.25,
   "model_calls": 4,
   "input_tokens": 28486,
   "cached_input_tokens": 19200,
   "output_tokens": 1482,
   "list_usd": 0.037064
  },
  {
   "arm": "graff-v302-12",
   "task": "linear-nohint",
   "suite": "mcp",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 24.78,
   "model_calls": 4,
   "input_tokens": 27843,
   "cached_input_tokens": 9984,
   "output_tokens": 679,
   "list_usd": 0.044784
  },
  {
   "arm": "graff-v302-12",
   "task": "linear-nohint",
   "suite": "mcp",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 27.48,
   "model_calls": 10,
   "input_tokens": 78496,
   "cached_input_tokens": 29312,
   "output_tokens": 1323,
   "list_usd": 0.120962
  },
  {
   "arm": "grok-cli",
   "task": "linear-nohint",
   "suite": "mcp",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 213.72,
   "model_calls": 25,
   "input_tokens": 1408433,
   "cached_input_tokens": 1233408,
   "output_tokens": 13385,
   "list_usd": 1.047064,
   "grok_reported_usd": 0.356002
  },
  {
   "arm": "grok-cli",
   "task": "linear-nohint",
   "suite": "mcp",
   "rep": 2,
   "pass": false,
   "timed_out": true,
   "wall_s": 240.16,
   "model_calls": null,
   "input_tokens": null,
   "cached_input_tokens": null,
   "output_tokens": null,
   "list_usd": null,
   "grok_reported_usd": null
  },
  {
   "arm": "grok-cli",
   "task": "linear-nohint",
   "suite": "mcp",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 189.64,
   "model_calls": 32,
   "input_tokens": 1551071,
   "cached_input_tokens": 1428096,
   "output_tokens": 10284,
   "list_usd": 1.021702,
   "grok_reported_usd": 0.347379
  },
  {
   "arm": "graff-v302-12",
   "task": "linear-nohint-warm",
   "suite": "mcp",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 20.88,
   "model_calls": 7,
   "input_tokens": 53157,
   "cached_input_tokens": 39808,
   "output_tokens": 1368,
   "list_usd": 0.05481
  },
  {
   "arm": "graff-v302-12",
   "task": "linear-nohint-warm",
   "suite": "mcp",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 12.53,
   "model_calls": 5,
   "input_tokens": 35971,
   "cached_input_tokens": 23424,
   "output_tokens": 659,
   "list_usd": 0.04076
  },
  {
   "arm": "graff-v302-12",
   "task": "linear-nohint-warm",
   "suite": "mcp",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 26.64,
   "model_calls": 5,
   "input_tokens": 36533,
   "cached_input_tokens": 20352,
   "output_tokens": 999,
   "list_usd": 0.048532
  },
  {
   "arm": "grok-cli",
   "task": "linear-nohint-warm",
   "suite": "mcp",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 169.92,
   "model_calls": 25,
   "input_tokens": 1300683,
   "cached_input_tokens": 1123456,
   "output_tokens": 10414,
   "list_usd": 0.978666,
   "grok_reported_usd": 0.332746
  },
  {
   "arm": "grok-cli",
   "task": "linear-nohint-warm",
   "suite": "mcp",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 213.3,
   "model_calls": 32,
   "input_tokens": 1820740,
   "cached_input_tokens": 1674752,
   "output_tokens": 10705,
   "list_usd": 1.193582,
   "grok_reported_usd": 0.405818
  },
  {
   "arm": "grok-cli",
   "task": "linear-nohint-warm",
   "suite": "mcp",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 186.58,
   "model_calls": 24,
   "input_tokens": 1548710,
   "cached_input_tokens": 1456128,
   "output_tokens": 10677,
   "list_usd": 0.97729,
   "grok_reported_usd": 0.332279
  },
  {
   "arm": "graff-v302-12",
   "task": "linear-quiet",
   "suite": "mcp",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 36.39,
   "model_calls": 5,
   "input_tokens": 39325,
   "cached_input_tokens": 24576,
   "output_tokens": 2686,
   "list_usd": 0.057902
  },
  {
   "arm": "graff-v302-12",
   "task": "linear-quiet",
   "suite": "mcp",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 15.7,
   "model_calls": 4,
   "input_tokens": 27753,
   "cached_input_tokens": 13056,
   "output_tokens": 1010,
   "list_usd": 0.041982
  },
  {
   "arm": "graff-v302-12",
   "task": "linear-quiet",
   "suite": "mcp",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 11.75,
   "model_calls": 4,
   "input_tokens": 27195,
   "cached_input_tokens": 20480,
   "output_tokens": 610,
   "list_usd": 0.02733
  },
  {
   "arm": "grok-cli",
   "task": "linear-quiet",
   "suite": "mcp",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 175.49,
   "model_calls": 21,
   "input_tokens": 1055864,
   "cached_input_tokens": 945408,
   "output_tokens": 10229,
   "list_usd": 0.75499,
   "grok_reported_usd": 0.256697
  },
  {
   "arm": "grok-cli",
   "task": "linear-quiet",
   "suite": "mcp",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 208.65,
   "model_calls": 27,
   "input_tokens": 1609676,
   "cached_input_tokens": 1456896,
   "output_tokens": 12836,
   "list_usd": 1.111024,
   "grok_reported_usd": 0.377748
  },
  {
   "arm": "grok-cli",
   "task": "linear-quiet",
   "suite": "mcp",
   "rep": 3,
   "pass": false,
   "timed_out": true,
   "wall_s": 240.26,
   "model_calls": null,
   "input_tokens": null,
   "cached_input_tokens": null,
   "output_tokens": null,
   "list_usd": null,
   "grok_reported_usd": null
  },
  {
   "arm": "graff-v302-12",
   "task": "linear-quiet-warm",
   "suite": "mcp",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 14.07,
   "model_calls": 4,
   "input_tokens": 28290,
   "cached_input_tokens": 18944,
   "output_tokens": 922,
   "list_usd": 0.033696
  },
  {
   "arm": "graff-v302-12",
   "task": "linear-quiet-warm",
   "suite": "mcp",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 29.45,
   "model_calls": 4,
   "input_tokens": 30738,
   "cached_input_tokens": 27264,
   "output_tokens": 2076,
   "list_usd": 0.033036
  },
  {
   "arm": "graff-v302-12",
   "task": "linear-quiet-warm",
   "suite": "mcp",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 25.89,
   "model_calls": 4,
   "input_tokens": 30195,
   "cached_input_tokens": 14720,
   "output_tokens": 1913,
   "list_usd": 0.049788
  },
  {
   "arm": "grok-cli",
   "task": "linear-quiet-warm",
   "suite": "mcp",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 149.0,
   "model_calls": 14,
   "input_tokens": 586404,
   "cached_input_tokens": 515200,
   "output_tokens": 9160,
   "list_usd": 0.454968,
   "grok_reported_usd": 0.154689
  },
  {
   "arm": "grok-cli",
   "task": "linear-quiet-warm",
   "suite": "mcp",
   "rep": 2,
   "pass": false,
   "timed_out": true,
   "wall_s": 240.41,
   "model_calls": null,
   "input_tokens": null,
   "cached_input_tokens": null,
   "output_tokens": null,
   "list_usd": null,
   "grok_reported_usd": null
  },
  {
   "arm": "grok-cli",
   "task": "linear-quiet-warm",
   "suite": "mcp",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 222.33,
   "model_calls": 31,
   "input_tokens": 1712511,
   "cached_input_tokens": 1638528,
   "output_tokens": 12329,
   "list_usd": 1.041204,
   "grok_reported_usd": 0.354009
  },
  {
   "arm": "graff-v302-12",
   "task": "linear-reduce",
   "suite": "mcp",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 9.6,
   "model_calls": 4,
   "input_tokens": 27229,
   "cached_input_tokens": 18048,
   "output_tokens": 585,
   "list_usd": 0.030896
  },
  {
   "arm": "graff-v302-12",
   "task": "linear-reduce",
   "suite": "mcp",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 24.44,
   "model_calls": 6,
   "input_tokens": 42016,
   "cached_input_tokens": 12288,
   "output_tokens": 1452,
   "list_usd": 0.074312
  },
  {
   "arm": "graff-v302-12",
   "task": "linear-reduce",
   "suite": "mcp",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 29.79,
   "model_calls": 4,
   "input_tokens": 28643,
   "cached_input_tokens": 8960,
   "output_tokens": 1502,
   "list_usd": 0.052858
  },
  {
   "arm": "grok-cli",
   "task": "linear-reduce",
   "suite": "mcp",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 162.9,
   "model_calls": 18,
   "input_tokens": 743688,
   "cached_input_tokens": 697472,
   "output_tokens": 10703,
   "list_usd": 0.505386,
   "grok_reported_usd": 0.171831
  },
  {
   "arm": "grok-cli",
   "task": "linear-reduce",
   "suite": "mcp",
   "rep": 2,
   "pass": true,
   "timed_out": true,
   "wall_s": 240.09,
   "model_calls": null,
   "input_tokens": null,
   "cached_input_tokens": null,
   "output_tokens": null,
   "list_usd": null,
   "grok_reported_usd": null
  },
  {
   "arm": "grok-cli",
   "task": "linear-reduce",
   "suite": "mcp",
   "rep": 3,
   "pass": true,
   "timed_out": true,
   "wall_s": 240.36,
   "model_calls": null,
   "input_tokens": null,
   "cached_input_tokens": null,
   "output_tokens": null,
   "list_usd": null,
   "grok_reported_usd": null
  },
  {
   "arm": "graff-v302-12",
   "task": "linear-reduce-warm",
   "suite": "mcp",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 15.68,
   "model_calls": 5,
   "input_tokens": 35763,
   "cached_input_tokens": 14848,
   "output_tokens": 927,
   "list_usd": 0.054816
  },
  {
   "arm": "graff-v302-12",
   "task": "linear-reduce-warm",
   "suite": "mcp",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 19.93,
   "model_calls": 4,
   "input_tokens": 29420,
   "cached_input_tokens": 13056,
   "output_tokens": 1400,
   "list_usd": 0.047656
  },
  {
   "arm": "graff-v302-12",
   "task": "linear-reduce-warm",
   "suite": "mcp",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 9.22,
   "model_calls": 4,
   "input_tokens": 28176,
   "cached_input_tokens": 14720,
   "output_tokens": 658,
   "list_usd": 0.03822
  },
  {
   "arm": "grok-cli",
   "task": "linear-reduce-warm",
   "suite": "mcp",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 151.94,
   "model_calls": 18,
   "input_tokens": 839586,
   "cached_input_tokens": 738944,
   "output_tokens": 9438,
   "list_usd": 0.627384,
   "grok_reported_usd": 0.213311
  },
  {
   "arm": "grok-cli",
   "task": "linear-reduce-warm",
   "suite": "mcp",
   "rep": 2,
   "pass": false,
   "timed_out": true,
   "wall_s": 240.5,
   "model_calls": null,
   "input_tokens": null,
   "cached_input_tokens": null,
   "output_tokens": null,
   "list_usd": null,
   "grok_reported_usd": null
  },
  {
   "arm": "grok-cli",
   "task": "linear-reduce-warm",
   "suite": "mcp",
   "rep": 3,
   "pass": true,
   "timed_out": true,
   "wall_s": 240.19,
   "model_calls": null,
   "input_tokens": null,
   "cached_input_tokens": null,
   "output_tokens": null,
   "list_usd": null,
   "grok_reported_usd": null
  },
  {
   "arm": "graff-v302-12",
   "task": "linear-sidecar",
   "suite": "mcp",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 90.94,
   "model_calls": 11,
   "input_tokens": 78174,
   "cached_input_tokens": 48384,
   "output_tokens": 2329,
   "list_usd": 0.097746
  },
  {
   "arm": "graff-v302-12",
   "task": "linear-sidecar",
   "suite": "mcp",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 25.12,
   "model_calls": 9,
   "input_tokens": 63155,
   "cached_input_tokens": 27648,
   "output_tokens": 1462,
   "list_usd": 0.09361
  },
  {
   "arm": "graff-v302-12",
   "task": "linear-sidecar",
   "suite": "mcp",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 23.94,
   "model_calls": 7,
   "input_tokens": 49237,
   "cached_input_tokens": 22656,
   "output_tokens": 1265,
   "list_usd": 0.07208
  },
  {
   "arm": "grok-cli",
   "task": "linear-sidecar",
   "suite": "mcp",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 227.95,
   "model_calls": 28,
   "input_tokens": 1173801,
   "cached_input_tokens": 1011840,
   "output_tokens": 12284,
   "list_usd": 0.903546,
   "grok_reported_usd": 0.009746
  },
  {
   "arm": "grok-cli",
   "task": "linear-sidecar",
   "suite": "mcp",
   "rep": 2,
   "pass": false,
   "timed_out": true,
   "wall_s": 240.46,
   "model_calls": null,
   "input_tokens": null,
   "cached_input_tokens": null,
   "output_tokens": null,
   "list_usd": null,
   "grok_reported_usd": null
  },
  {
   "arm": "grok-cli",
   "task": "linear-sidecar",
   "suite": "mcp",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 231.62,
   "model_calls": 30,
   "input_tokens": 2012313,
   "cached_input_tokens": 1892224,
   "output_tokens": 12893,
   "list_usd": 1.263648,
   "grok_reported_usd": 0.009541
  },
  {
   "arm": "graff-v302-12",
   "task": "linear-split",
   "suite": "mcp",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 40.8,
   "model_calls": 13,
   "input_tokens": 89608,
   "cached_input_tokens": 50816,
   "output_tokens": 2727,
   "list_usd": 0.119354
  },
  {
   "arm": "graff-v302-12",
   "task": "linear-split",
   "suite": "mcp",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 49.21,
   "model_calls": 8,
   "input_tokens": 57236,
   "cached_input_tokens": 23040,
   "output_tokens": 2549,
   "list_usd": 0.095206
  },
  {
   "arm": "graff-v302-12",
   "task": "linear-split",
   "suite": "mcp",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 37.53,
   "model_calls": 13,
   "input_tokens": 90059,
   "cached_input_tokens": 56320,
   "output_tokens": 2750,
   "list_usd": 0.112138
  },
  {
   "arm": "grok-cli",
   "task": "linear-split",
   "suite": "mcp",
   "rep": 1,
   "pass": false,
   "timed_out": true,
   "wall_s": 240.58,
   "model_calls": null,
   "input_tokens": null,
   "cached_input_tokens": null,
   "output_tokens": null,
   "list_usd": null,
   "grok_reported_usd": null
  },
  {
   "arm": "grok-cli",
   "task": "linear-split",
   "suite": "mcp",
   "rep": 2,
   "pass": false,
   "timed_out": true,
   "wall_s": 240.49,
   "model_calls": null,
   "input_tokens": null,
   "cached_input_tokens": null,
   "output_tokens": null,
   "list_usd": null,
   "grok_reported_usd": null
  },
  {
   "arm": "grok-cli",
   "task": "linear-split",
   "suite": "mcp",
   "rep": 3,
   "pass": false,
   "timed_out": true,
   "wall_s": 240.44,
   "model_calls": null,
   "input_tokens": null,
   "cached_input_tokens": null,
   "output_tokens": null,
   "list_usd": null,
   "grok_reported_usd": null
  },
  {
   "arm": "graff-v302-12",
   "task": "linear-warm",
   "suite": "mcp",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 24.97,
   "model_calls": 4,
   "input_tokens": 28948,
   "cached_input_tokens": 4608,
   "output_tokens": 1393,
   "list_usd": 0.059342
  },
  {
   "arm": "graff-v302-12",
   "task": "linear-warm",
   "suite": "mcp",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 11.25,
   "model_calls": 4,
   "input_tokens": 27939,
   "cached_input_tokens": 19968,
   "output_tokens": 617,
   "list_usd": 0.029628
  },
  {
   "arm": "graff-v302-12",
   "task": "linear-warm",
   "suite": "mcp",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 19.88,
   "model_calls": 4,
   "input_tokens": 29492,
   "cached_input_tokens": 20736,
   "output_tokens": 1345,
   "list_usd": 0.03595
  },
  {
   "arm": "grok-cli",
   "task": "linear-warm",
   "suite": "mcp",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 178.9,
   "model_calls": 19,
   "input_tokens": 1031519,
   "cached_input_tokens": 905984,
   "output_tokens": 9122,
   "list_usd": 0.758794,
   "grok_reported_usd": 0.25799
  },
  {
   "arm": "grok-cli",
   "task": "linear-warm",
   "suite": "mcp",
   "rep": 2,
   "pass": false,
   "timed_out": true,
   "wall_s": 240.28,
   "model_calls": null,
   "input_tokens": null,
   "cached_input_tokens": null,
   "output_tokens": null,
   "list_usd": null,
   "grok_reported_usd": null
  },
  {
   "arm": "grok-cli",
   "task": "linear-warm",
   "suite": "mcp",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 165.4,
   "model_calls": 22,
   "input_tokens": 1283560,
   "cached_input_tokens": 1200896,
   "output_tokens": 9559,
   "list_usd": 0.82313,
   "grok_reported_usd": 0.279864
  },
  {
   "arm": "graff-v302-12",
   "task": "recall-noise",
   "suite": "core",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 4.51,
   "model_calls": 2,
   "input_tokens": 12390,
   "cached_input_tokens": 7168,
   "output_tokens": 105,
   "list_usd": 0.014658
  },
  {
   "arm": "graff-v302-12",
   "task": "recall-noise",
   "suite": "core",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 3.92,
   "model_calls": 2,
   "input_tokens": 12390,
   "cached_input_tokens": 7168,
   "output_tokens": 117,
   "list_usd": 0.01473
  },
  {
   "arm": "graff-v302-12",
   "task": "recall-noise",
   "suite": "core",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 3.66,
   "model_calls": 2,
   "input_tokens": 12399,
   "cached_input_tokens": 2304,
   "output_tokens": 107,
   "list_usd": 0.021984
  },
  {
   "arm": "grok-cli",
   "task": "recall-noise",
   "suite": "core",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 9.02,
   "model_calls": 3,
   "input_tokens": 54881,
   "cached_input_tokens": 34816,
   "output_tokens": 174,
   "list_usd": 0.058582,
   "grok_reported_usd": 0.019918
  },
  {
   "arm": "grok-cli",
   "task": "recall-noise",
   "suite": "core",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 6.01,
   "model_calls": 2,
   "input_tokens": 36589,
   "cached_input_tokens": 30080,
   "output_tokens": 131,
   "list_usd": 0.028844,
   "grok_reported_usd": 0.009807
  },
  {
   "arm": "grok-cli",
   "task": "recall-noise",
   "suite": "core",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 6.73,
   "model_calls": 2,
   "input_tokens": 36569,
   "cached_input_tokens": 19712,
   "output_tokens": 123,
   "list_usd": 0.044308,
   "grok_reported_usd": 0.015065
  },
  {
   "arm": "graff-v302-12",
   "task": "refactor-rename",
   "suite": "core",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 7.42,
   "model_calls": 4,
   "input_tokens": 25559,
   "cached_input_tokens": 14976,
   "output_tokens": 294,
   "list_usd": 0.030418
  },
  {
   "arm": "graff-v302-12",
   "task": "refactor-rename",
   "suite": "core",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 7.49,
   "model_calls": 4,
   "input_tokens": 25551,
   "cached_input_tokens": 19584,
   "output_tokens": 307,
   "list_usd": 0.023568
  },
  {
   "arm": "graff-v302-12",
   "task": "refactor-rename",
   "suite": "core",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 10.22,
   "model_calls": 7,
   "input_tokens": 45672,
   "cached_input_tokens": 34816,
   "output_tokens": 307,
   "list_usd": 0.040962
  },
  {
   "arm": "grok-cli",
   "task": "refactor-rename",
   "suite": "core",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 14.83,
   "model_calls": 5,
   "input_tokens": 92924,
   "cached_input_tokens": 66048,
   "output_tokens": 526,
   "list_usd": 0.089932,
   "grok_reported_usd": 0.030577
  },
  {
   "arm": "grok-cli",
   "task": "refactor-rename",
   "suite": "core",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 19.18,
   "model_calls": 5,
   "input_tokens": 93469,
   "cached_input_tokens": 86016,
   "output_tokens": 533,
   "list_usd": 0.061112,
   "grok_reported_usd": 0.020778
  },
  {
   "arm": "grok-cli",
   "task": "refactor-rename",
   "suite": "core",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 22.34,
   "model_calls": 5,
   "input_tokens": 92953,
   "cached_input_tokens": 85760,
   "output_tokens": 596,
   "list_usd": 0.060842,
   "grok_reported_usd": 0.020686
  },
  {
   "arm": "graff-v302-12",
   "task": "regex-count",
   "suite": "core",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 12.08,
   "model_calls": 4,
   "input_tokens": 24937,
   "cached_input_tokens": 19584,
   "output_tokens": 683,
   "list_usd": 0.024596
  },
  {
   "arm": "graff-v302-12",
   "task": "regex-count",
   "suite": "core",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 10.44,
   "model_calls": 4,
   "input_tokens": 25043,
   "cached_input_tokens": 13824,
   "output_tokens": 404,
   "list_usd": 0.031774
  },
  {
   "arm": "graff-v302-12",
   "task": "regex-count",
   "suite": "core",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 9.77,
   "model_calls": 4,
   "input_tokens": 24959,
   "cached_input_tokens": 19456,
   "output_tokens": 539,
   "list_usd": 0.023968
  },
  {
   "arm": "grok-cli",
   "task": "regex-count",
   "suite": "core",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 29.86,
   "model_calls": 6,
   "input_tokens": 113525,
   "cached_input_tokens": 99200,
   "output_tokens": 1468,
   "list_usd": 0.087058,
   "grok_reported_usd": 0.0296
  },
  {
   "arm": "grok-cli",
   "task": "regex-count",
   "suite": "core",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 33.87,
   "model_calls": 6,
   "input_tokens": 115681,
   "cached_input_tokens": 107136,
   "output_tokens": 1777,
   "list_usd": 0.08132,
   "grok_reported_usd": 0.027649
  },
  {
   "arm": "grok-cli",
   "task": "regex-count",
   "suite": "core",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 15.44,
   "model_calls": 4,
   "input_tokens": 74378,
   "cached_input_tokens": 61184,
   "output_tokens": 698,
   "list_usd": 0.061168,
   "grok_reported_usd": 0.020797
  },
  {
   "arm": "graff-v302-12",
   "task": "write-tests",
   "suite": "core",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 5.86,
   "model_calls": 3,
   "input_tokens": 18684,
   "cached_input_tokens": 13312,
   "output_tokens": 305,
   "list_usd": 0.01923
  },
  {
   "arm": "graff-v302-12",
   "task": "write-tests",
   "suite": "core",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 8.11,
   "model_calls": 3,
   "input_tokens": 18684,
   "cached_input_tokens": 8320,
   "output_tokens": 362,
   "list_usd": 0.02706
  },
  {
   "arm": "graff-v302-12",
   "task": "write-tests",
   "suite": "core",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 5.85,
   "model_calls": 3,
   "input_tokens": 18610,
   "cached_input_tokens": 13312,
   "output_tokens": 304,
   "list_usd": 0.019076
  },
  {
   "arm": "grok-cli",
   "task": "write-tests",
   "suite": "core",
   "rep": 1,
   "pass": true,
   "timed_out": false,
   "wall_s": 16.68,
   "model_calls": 5,
   "input_tokens": 92336,
   "cached_input_tokens": 85248,
   "output_tokens": 569,
   "list_usd": 0.060214,
   "grok_reported_usd": 0.020473
  },
  {
   "arm": "grok-cli",
   "task": "write-tests",
   "suite": "core",
   "rep": 2,
   "pass": true,
   "timed_out": false,
   "wall_s": 15.69,
   "model_calls": 5,
   "input_tokens": 92880,
   "cached_input_tokens": 49408,
   "output_tokens": 591,
   "list_usd": 0.115194,
   "grok_reported_usd": 0.039166
  },
  {
   "arm": "grok-cli",
   "task": "write-tests",
   "suite": "core",
   "rep": 3,
   "pass": true,
   "timed_out": false,
   "wall_s": 18.1,
   "model_calls": 5,
   "input_tokens": 92832,
   "cached_input_tokens": 69248,
   "output_tokens": 612,
   "list_usd": 0.085464,
   "grok_reported_usd": 0.029058
  }
 ]
}