graff — a fast agentic coding harness in Zig: multi-provider, MCP, workflows, DGM evolution loop, TS/Python SDKs
Python
{
"model": {
"graff-v302-12": "grok-4.7 (public xAI API)",
"grok-cli": "grok-4.7, served as grok-4.7-build on the Grok CLI route"
},
"tasks": 21,
"reps": 3,
"time_limit_s": 240,
"arms": {
"graff-v302-12": "graff v0.0.302.12 (release)",
"grok-cli": "Grok CLI 1.0.41"
},
"list_rates_per_1m": {
"input": 2.0,
"cached_input": 0.5,
"output": 6.0
},
"summary": {
"all": {
"graff-v302-12": {
"runs": 63,
"passed": 63,
"timed_out": 0,
"runs_without_usage": 0,
"wall_s_per_task": 15.5,
"list_usd_per_task": 0.0391,
"prompt_tokens_per_task": 29696,
"cached_input_share": 0.567,
"output_tokens_per_task": 823,
"model_calls_per_task": 4.3
},
"grok-cli": {
"runs": 63,
"passed": 54,
"timed_out": 13,
"runs_without_usage": 13,
"wall_s_per_task": 106.9,
"list_usd_per_task": 0.3306,
"prompt_tokens_per_task": 466465,
"cached_input_share": 0.895,
"output_tokens_per_task": 4007,
"model_calls_per_task": 10.4,
"grok_reported_usd_per_task": 0.0981
}
},
"core": {
"graff-v302-12": {
"runs": 33,
"passed": 33,
"timed_out": 0,
"runs_without_usage": 0,
"wall_s_per_task": 6.6,
"list_usd_per_task": 0.0215,
"prompt_tokens_per_task": 19262,
"cached_input_share": 0.645,
"output_tokens_per_task": 263,
"model_calls_per_task": 3.1
},
"grok-cli": {
"runs": 33,
"passed": 33,
"timed_out": 0,
"runs_without_usage": 0,
"wall_s_per_task": 14.8,
"list_usd_per_task": 0.0578,
"prompt_tokens_per_task": 67046,
"cached_input_share": 0.794,
"output_tokens_per_task": 580,
"model_calls_per_task": 3.6,
"grok_reported_usd_per_task": 0.0196
}
},
"mcp": {
"graff-v302-12": {
"runs": 30,
"passed": 30,
"timed_out": 0,
"runs_without_usage": 0,
"wall_s_per_task": 25.3,
"list_usd_per_task": 0.0584,
"prompt_tokens_per_task": 41174,
"cached_input_share": 0.527,
"output_tokens_per_task": 1439,
"model_calls_per_task": 5.7
},
"grok-cli": {
"runs": 30,
"passed": 21,
"timed_out": 13,
"runs_without_usage": 13,
"wall_s_per_task": 208.4,
"list_usd_per_task": 0.8602,
"prompt_tokens_per_task": 1241807,
"cached_input_share": 0.906,
"output_tokens_per_task": 10659,
"model_calls_per_task": 23.5,
"grok_reported_usd_per_task": 0.2503
}
}
},
"per_task": {
"csv-sum": {
"suite": "core",
"graff-v302-12": {
"passed": 3,
"runs": 3,
"wall_s": 6.2,
"model_calls": 3.7,
"list_usd": 0.0187
},
"grok-cli": {
"passed": 3,
"runs": 3,
"wall_s": 18.5,
"model_calls": 5,
"list_usd": 0.0673
}
},
"dead-code": {
"suite": "core",
"graff-v302-12": {
"passed": 3,
"runs": 3,
"wall_s": 5.0,
"model_calls": 3,
"list_usd": 0.021
},
"grok-cli": {
"passed": 3,
"runs": 3,
"wall_s": 12.3,
"model_calls": 3,
"list_usd": 0.047
}
},
"exact-reply": {
"suite": "core",
"graff-v302-12": {
"passed": 3,
"runs": 3,
"wall_s": 2.0,
"model_calls": 1,
"list_usd": 0.0081
},
"grok-cli": {
"passed": 3,
"runs": 3,
"wall_s": 5.0,
"model_calls": 1,
"list_usd": 0.0273
}
},
"file-ops": {
"suite": "core",
"graff-v302-12": {
"passed": 3,
"runs": 3,
"wall_s": 5.2,
"model_calls": 2,
"list_usd": 0.0131
},
"grok-cli": {
"passed": 3,
"runs": 3,
"wall_s": 7.1,
"model_calls": 2,
"list_usd": 0.0294
}
},
"fix-fib": {
"suite": "core",
"graff-v302-12": {
"passed": 3,
"runs": 3,
"wall_s": 9.6,
"model_calls": 4,
"list_usd": 0.0294
},
"grok-cli": {
"passed": 3,
"runs": 3,
"wall_s": 17.0,
"model_calls": 4,
"list_usd": 0.0716
}
},
"git-ops": {
"suite": "core",
"graff-v302-12": {
"passed": 3,
"runs": 3,
"wall_s": 8.5,
"model_calls": 3,
"list_usd": 0.0303
},
"grok-cli": {
"passed": 3,
"runs": 3,
"wall_s": 16.7,
"model_calls": 3,
"list_usd": 0.0528
}
},
"json-transform": {
"suite": "core",
"graff-v302-12": {
"passed": 3,
"runs": 3,
"wall_s": 5.8,
"model_calls": 3,
"list_usd": 0.0184
},
"grok-cli": {
"passed": 3,
"runs": 3,
"wall_s": 16.4,
"model_calls": 4,
"list_usd": 0.0621
}
},
"linear-issues": {
"suite": "mcp",
"graff-v302-12": {
"passed": 3,
"runs": 3,
"wall_s": 19.7,
"model_calls": 4,
"list_usd": 0.0494
},
"grok-cli": {
"passed": 3,
"runs": 3,
"wall_s": 173.0,
"model_calls": 16.5,
"list_usd": 0.5809
}
},
"linear-nohint": {
"suite": "mcp",
"graff-v302-12": {
"passed": 3,
"runs": 3,
"wall_s": 24.5,
"model_calls": 6,
"list_usd": 0.0676
},
"grok-cli": {
"passed": 2,
"runs": 3,
"wall_s": 214.5,
"model_calls": 28.5,
"list_usd": 1.0344
}
},
"linear-nohint-warm": {
"suite": "mcp",
"graff-v302-12": {
"passed": 3,
"runs": 3,
"wall_s": 20.0,
"model_calls": 5.7,
"list_usd": 0.048
},
"grok-cli": {
"passed": 3,
"runs": 3,
"wall_s": 189.9,
"model_calls": 27,
"list_usd": 1.0498
}
},
"linear-quiet": {
"suite": "mcp",
"graff-v302-12": {
"passed": 3,
"runs": 3,
"wall_s": 21.3,
"model_calls": 4.3,
"list_usd": 0.0424
},
"grok-cli": {
"passed": 2,
"runs": 3,
"wall_s": 208.1,
"model_calls": 24,
"list_usd": 0.933
}
},
"linear-quiet-warm": {
"suite": "mcp",
"graff-v302-12": {
"passed": 3,
"runs": 3,
"wall_s": 23.1,
"model_calls": 4,
"list_usd": 0.0388
},
"grok-cli": {
"passed": 2,
"runs": 3,
"wall_s": 203.9,
"model_calls": 22.5,
"list_usd": 0.7481
}
},
"linear-reduce": {
"suite": "mcp",
"graff-v302-12": {
"passed": 3,
"runs": 3,
"wall_s": 21.3,
"model_calls": 4.7,
"list_usd": 0.0527
},
"grok-cli": {
"passed": 3,
"runs": 3,
"wall_s": 214.5,
"model_calls": 18,
"list_usd": 0.5054
}
},
"linear-reduce-warm": {
"suite": "mcp",
"graff-v302-12": {
"passed": 3,
"runs": 3,
"wall_s": 14.9,
"model_calls": 4.3,
"list_usd": 0.0469
},
"grok-cli": {
"passed": 2,
"runs": 3,
"wall_s": 210.9,
"model_calls": 18,
"list_usd": 0.6274
}
},
"linear-sidecar": {
"suite": "mcp",
"graff-v302-12": {
"passed": 3,
"runs": 3,
"wall_s": 46.7,
"model_calls": 9,
"list_usd": 0.0878
},
"grok-cli": {
"passed": 2,
"runs": 3,
"wall_s": 233.3,
"model_calls": 29,
"list_usd": 1.0836
}
},
"linear-split": {
"suite": "mcp",
"graff-v302-12": {
"passed": 3,
"runs": 3,
"wall_s": 42.5,
"model_calls": 11.3,
"list_usd": 0.1089
},
"grok-cli": {
"passed": 0,
"runs": 3,
"wall_s": 240.5,
"model_calls": 0,
"list_usd": 0
}
},
"linear-warm": {
"suite": "mcp",
"graff-v302-12": {
"passed": 3,
"runs": 3,
"wall_s": 18.7,
"model_calls": 4,
"list_usd": 0.0416
},
"grok-cli": {
"passed": 2,
"runs": 3,
"wall_s": 194.9,
"model_calls": 20.5,
"list_usd": 0.791
}
},
"recall-noise": {
"suite": "core",
"graff-v302-12": {
"passed": 3,
"runs": 3,
"wall_s": 4.0,
"model_calls": 2,
"list_usd": 0.0171
},
"grok-cli": {
"passed": 3,
"runs": 3,
"wall_s": 7.3,
"model_calls": 2.3,
"list_usd": 0.0439
}
},
"refactor-rename": {
"suite": "core",
"graff-v302-12": {
"passed": 3,
"runs": 3,
"wall_s": 8.4,
"model_calls": 5,
"list_usd": 0.0316
},
"grok-cli": {
"passed": 3,
"runs": 3,
"wall_s": 18.8,
"model_calls": 5,
"list_usd": 0.0706
}
},
"regex-count": {
"suite": "core",
"graff-v302-12": {
"passed": 3,
"runs": 3,
"wall_s": 10.8,
"model_calls": 4,
"list_usd": 0.0268
},
"grok-cli": {
"passed": 3,
"runs": 3,
"wall_s": 26.4,
"model_calls": 5.3,
"list_usd": 0.0765
}
},
"write-tests": {
"suite": "core",
"graff-v302-12": {
"passed": 3,
"runs": 3,
"wall_s": 6.6,
"model_calls": 3,
"list_usd": 0.0218
},
"grok-cli": {
"passed": 3,
"runs": 3,
"wall_s": 16.8,
"model_calls": 5,
"list_usd": 0.087
}
}
},
"runs": [
{
"arm": "graff-v302-12",
"task": "csv-sum",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 6.87,
"model_calls": 4,
"input_tokens": 25430,
"cached_input_tokens": 19840,
"output_tokens": 272,
"list_usd": 0.022732
},
{
"arm": "graff-v302-12",
"task": "csv-sum",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 6.62,
"model_calls": 4,
"input_tokens": 25344,
"cached_input_tokens": 19840,
"output_tokens": 226,
"list_usd": 0.022284
},
{
"arm": "graff-v302-12",
"task": "csv-sum",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 5.19,
"model_calls": 3,
"input_tokens": 18617,
"cached_input_tokens": 18176,
"output_tokens": 181,
"list_usd": 0.011056
},
{
"arm": "grok-cli",
"task": "csv-sum",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 15.46,
"model_calls": 5,
"input_tokens": 92148,
"cached_input_tokens": 85120,
"output_tokens": 532,
"list_usd": 0.059808,
"grok_reported_usd": 0.020335
},
{
"arm": "grok-cli",
"task": "csv-sum",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 19.59,
"model_calls": 5,
"input_tokens": 92602,
"cached_input_tokens": 85120,
"output_tokens": 926,
"list_usd": 0.06308,
"grok_reported_usd": 0.021447
},
{
"arm": "grok-cli",
"task": "csv-sum",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 20.4,
"model_calls": 5,
"input_tokens": 92665,
"cached_input_tokens": 74496,
"output_tokens": 910,
"list_usd": 0.079046,
"grok_reported_usd": 0.026876
},
{
"arm": "graff-v302-12",
"task": "dead-code",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 4.08,
"model_calls": 3,
"input_tokens": 18681,
"cached_input_tokens": 18176,
"output_tokens": 102,
"list_usd": 0.01071
},
{
"arm": "graff-v302-12",
"task": "dead-code",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 5.22,
"model_calls": 3,
"input_tokens": 18621,
"cached_input_tokens": 7808,
"output_tokens": 206,
"list_usd": 0.026766
},
{
"arm": "graff-v302-12",
"task": "dead-code",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 5.77,
"model_calls": 3,
"input_tokens": 18613,
"cached_input_tokens": 8320,
"output_tokens": 110,
"list_usd": 0.025406
},
{
"arm": "grok-cli",
"task": "dead-code",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 16.77,
"model_calls": 3,
"input_tokens": 55529,
"cached_input_tokens": 48512,
"output_tokens": 380,
"list_usd": 0.04057,
"grok_reported_usd": 0.013794
},
{
"arm": "grok-cli",
"task": "dead-code",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 10.26,
"model_calls": 3,
"input_tokens": 55263,
"cached_input_tokens": 48512,
"output_tokens": 316,
"list_usd": 0.039654,
"grok_reported_usd": 0.013482
},
{
"arm": "grok-cli",
"task": "dead-code",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 10.01,
"model_calls": 3,
"input_tokens": 55538,
"cached_input_tokens": 35072,
"output_tokens": 374,
"list_usd": 0.060712,
"grok_reported_usd": 0.020642
},
{
"arm": "graff-v302-12",
"task": "exact-reply",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 1.43,
"model_calls": 1,
"input_tokens": 6055,
"cached_input_tokens": 1152,
"output_tokens": 32,
"list_usd": 0.010574
},
{
"arm": "graff-v302-12",
"task": "exact-reply",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 2.78,
"model_calls": 1,
"input_tokens": 6055,
"cached_input_tokens": 6016,
"output_tokens": 32,
"list_usd": 0.003278
},
{
"arm": "graff-v302-12",
"task": "exact-reply",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 1.7,
"model_calls": 1,
"input_tokens": 6055,
"cached_input_tokens": 1152,
"output_tokens": 26,
"list_usd": 0.010538
},
{
"arm": "grok-cli",
"task": "exact-reply",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 4.7,
"model_calls": 1,
"input_tokens": 18126,
"cached_input_tokens": 4608,
"output_tokens": 30,
"list_usd": 0.02952,
"grok_reported_usd": 0.010037
},
{
"arm": "grok-cli",
"task": "exact-reply",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 4.72,
"model_calls": 1,
"input_tokens": 18125,
"cached_input_tokens": 1664,
"output_tokens": 29,
"list_usd": 0.033928,
"grok_reported_usd": 0.011536
},
{
"arm": "grok-cli",
"task": "exact-reply",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 5.43,
"model_calls": 1,
"input_tokens": 18126,
"cached_input_tokens": 12032,
"output_tokens": 30,
"list_usd": 0.018384,
"grok_reported_usd": 0.006251
},
{
"arm": "graff-v302-12",
"task": "file-ops",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 5.32,
"model_calls": 2,
"input_tokens": 12272,
"cached_input_tokens": 2304,
"output_tokens": 241,
"list_usd": 0.022534
},
{
"arm": "graff-v302-12",
"task": "file-ops",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 4.73,
"model_calls": 2,
"input_tokens": 12251,
"cached_input_tokens": 12032,
"output_tokens": 280,
"list_usd": 0.008134
},
{
"arm": "graff-v302-12",
"task": "file-ops",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 5.57,
"model_calls": 2,
"input_tokens": 12261,
"cached_input_tokens": 12032,
"output_tokens": 339,
"list_usd": 0.008508
},
{
"arm": "grok-cli",
"task": "file-ops",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 7.14,
"model_calls": 2,
"input_tokens": 36599,
"cached_input_tokens": 30080,
"output_tokens": 213,
"list_usd": 0.029356,
"grok_reported_usd": 0.009981
},
{
"arm": "grok-cli",
"task": "file-ops",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 7.63,
"model_calls": 2,
"input_tokens": 36458,
"cached_input_tokens": 30080,
"output_tokens": 279,
"list_usd": 0.02947,
"grok_reported_usd": 0.01002
},
{
"arm": "grok-cli",
"task": "file-ops",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 6.63,
"model_calls": 2,
"input_tokens": 36468,
"cached_input_tokens": 30080,
"output_tokens": 254,
"list_usd": 0.02934,
"grok_reported_usd": 0.009976
},
{
"arm": "graff-v302-12",
"task": "fix-fib",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 7.6,
"model_calls": 4,
"input_tokens": 25459,
"cached_input_tokens": 14976,
"output_tokens": 266,
"list_usd": 0.03005
},
{
"arm": "graff-v302-12",
"task": "fix-fib",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 11.83,
"model_calls": 4,
"input_tokens": 26049,
"cached_input_tokens": 9728,
"output_tokens": 585,
"list_usd": 0.041016
},
{
"arm": "graff-v302-12",
"task": "fix-fib",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 9.43,
"model_calls": 4,
"input_tokens": 25880,
"cached_input_tokens": 24960,
"output_tokens": 483,
"list_usd": 0.017218
},
{
"arm": "grok-cli",
"task": "fix-fib",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 17.61,
"model_calls": 4,
"input_tokens": 74856,
"cached_input_tokens": 45056,
"output_tokens": 778,
"list_usd": 0.086796,
"grok_reported_usd": 0.029511
},
{
"arm": "grok-cli",
"task": "fix-fib",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 15.6,
"model_calls": 4,
"input_tokens": 74808,
"cached_input_tokens": 54016,
"output_tokens": 740,
"list_usd": 0.073032,
"grok_reported_usd": 0.024831
},
{
"arm": "grok-cli",
"task": "fix-fib",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 17.75,
"model_calls": 4,
"input_tokens": 75374,
"cached_input_tokens": 67840,
"output_tokens": 982,
"list_usd": 0.05488,
"grok_reported_usd": 0.018659
},
{
"arm": "graff-v302-12",
"task": "git-ops",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 6.83,
"model_calls": 3,
"input_tokens": 19044,
"cached_input_tokens": 7808,
"output_tokens": 289,
"list_usd": 0.02811
},
{
"arm": "graff-v302-12",
"task": "git-ops",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 8.47,
"model_calls": 3,
"input_tokens": 19053,
"cached_input_tokens": 8320,
"output_tokens": 334,
"list_usd": 0.02763
},
{
"arm": "graff-v302-12",
"task": "git-ops",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 10.2,
"model_calls": 3,
"input_tokens": 19363,
"cached_input_tokens": 3456,
"output_tokens": 257,
"list_usd": 0.035084
},
{
"arm": "grok-cli",
"task": "git-ops",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 16.17,
"model_calls": 3,
"input_tokens": 55655,
"cached_input_tokens": 42496,
"output_tokens": 770,
"list_usd": 0.052186,
"grok_reported_usd": 0.017743
},
{
"arm": "grok-cli",
"task": "git-ops",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 13.88,
"model_calls": 3,
"input_tokens": 55929,
"cached_input_tokens": 48640,
"output_tokens": 587,
"list_usd": 0.04242,
"grok_reported_usd": 0.014423
},
{
"arm": "grok-cli",
"task": "git-ops",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 19.98,
"model_calls": 3,
"input_tokens": 56324,
"cached_input_tokens": 36608,
"output_tokens": 1026,
"list_usd": 0.063892,
"grok_reported_usd": 0.021723
},
{
"arm": "graff-v302-12",
"task": "json-transform",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 5.23,
"model_calls": 3,
"input_tokens": 18583,
"cached_input_tokens": 8320,
"output_tokens": 187,
"list_usd": 0.025808
},
{
"arm": "graff-v302-12",
"task": "json-transform",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 4.33,
"model_calls": 3,
"input_tokens": 18513,
"cached_input_tokens": 18176,
"output_tokens": 154,
"list_usd": 0.010686
},
{
"arm": "graff-v302-12",
"task": "json-transform",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 7.97,
"model_calls": 3,
"input_tokens": 18575,
"cached_input_tokens": 13312,
"output_tokens": 244,
"list_usd": 0.018646
},
{
"arm": "grok-cli",
"task": "json-transform",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 23.08,
"model_calls": 5,
"input_tokens": 93868,
"cached_input_tokens": 86272,
"output_tokens": 1074,
"list_usd": 0.064772,
"grok_reported_usd": 0.022022
},
{
"arm": "grok-cli",
"task": "json-transform",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 14.76,
"model_calls": 4,
"input_tokens": 73895,
"cached_input_tokens": 47104,
"output_tokens": 619,
"list_usd": 0.080848,
"grok_reported_usd": 0.027488
},
{
"arm": "grok-cli",
"task": "json-transform",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 11.51,
"model_calls": 3,
"input_tokens": 55144,
"cached_input_tokens": 48384,
"output_tokens": 498,
"list_usd": 0.0407,
"grok_reported_usd": 0.013838
},
{
"arm": "graff-v302-12",
"task": "linear-issues",
"suite": "mcp",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 23.65,
"model_calls": 4,
"input_tokens": 29665,
"cached_input_tokens": 11392,
"output_tokens": 1952,
"list_usd": 0.053954
},
{
"arm": "graff-v302-12",
"task": "linear-issues",
"suite": "mcp",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 15.2,
"model_calls": 4,
"input_tokens": 27981,
"cached_input_tokens": 14976,
"output_tokens": 1087,
"list_usd": 0.04002
},
{
"arm": "graff-v302-12",
"task": "linear-issues",
"suite": "mcp",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 20.29,
"model_calls": 4,
"input_tokens": 28490,
"cached_input_tokens": 7552,
"output_tokens": 1441,
"list_usd": 0.054298
},
{
"arm": "grok-cli",
"task": "linear-issues",
"suite": "mcp",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 103.43,
"model_calls": 12,
"input_tokens": 482817,
"cached_input_tokens": 362368,
"output_tokens": 7106,
"list_usd": 0.464718,
"grok_reported_usd": 0.158004
},
{
"arm": "grok-cli",
"task": "linear-issues",
"suite": "mcp",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 175.26,
"model_calls": 21,
"input_tokens": 949340,
"cached_input_tokens": 841344,
"output_tokens": 10073,
"list_usd": 0.697102,
"grok_reported_usd": 0.237015
},
{
"arm": "grok-cli",
"task": "linear-issues",
"suite": "mcp",
"rep": 3,
"pass": true,
"timed_out": true,
"wall_s": 240.29,
"model_calls": null,
"input_tokens": null,
"cached_input_tokens": null,
"output_tokens": null,
"list_usd": null,
"grok_reported_usd": null
},
{
"arm": "graff-v302-12",
"task": "linear-nohint",
"suite": "mcp",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 21.25,
"model_calls": 4,
"input_tokens": 28486,
"cached_input_tokens": 19200,
"output_tokens": 1482,
"list_usd": 0.037064
},
{
"arm": "graff-v302-12",
"task": "linear-nohint",
"suite": "mcp",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 24.78,
"model_calls": 4,
"input_tokens": 27843,
"cached_input_tokens": 9984,
"output_tokens": 679,
"list_usd": 0.044784
},
{
"arm": "graff-v302-12",
"task": "linear-nohint",
"suite": "mcp",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 27.48,
"model_calls": 10,
"input_tokens": 78496,
"cached_input_tokens": 29312,
"output_tokens": 1323,
"list_usd": 0.120962
},
{
"arm": "grok-cli",
"task": "linear-nohint",
"suite": "mcp",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 213.72,
"model_calls": 25,
"input_tokens": 1408433,
"cached_input_tokens": 1233408,
"output_tokens": 13385,
"list_usd": 1.047064,
"grok_reported_usd": 0.356002
},
{
"arm": "grok-cli",
"task": "linear-nohint",
"suite": "mcp",
"rep": 2,
"pass": false,
"timed_out": true,
"wall_s": 240.16,
"model_calls": null,
"input_tokens": null,
"cached_input_tokens": null,
"output_tokens": null,
"list_usd": null,
"grok_reported_usd": null
},
{
"arm": "grok-cli",
"task": "linear-nohint",
"suite": "mcp",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 189.64,
"model_calls": 32,
"input_tokens": 1551071,
"cached_input_tokens": 1428096,
"output_tokens": 10284,
"list_usd": 1.021702,
"grok_reported_usd": 0.347379
},
{
"arm": "graff-v302-12",
"task": "linear-nohint-warm",
"suite": "mcp",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 20.88,
"model_calls": 7,
"input_tokens": 53157,
"cached_input_tokens": 39808,
"output_tokens": 1368,
"list_usd": 0.05481
},
{
"arm": "graff-v302-12",
"task": "linear-nohint-warm",
"suite": "mcp",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 12.53,
"model_calls": 5,
"input_tokens": 35971,
"cached_input_tokens": 23424,
"output_tokens": 659,
"list_usd": 0.04076
},
{
"arm": "graff-v302-12",
"task": "linear-nohint-warm",
"suite": "mcp",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 26.64,
"model_calls": 5,
"input_tokens": 36533,
"cached_input_tokens": 20352,
"output_tokens": 999,
"list_usd": 0.048532
},
{
"arm": "grok-cli",
"task": "linear-nohint-warm",
"suite": "mcp",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 169.92,
"model_calls": 25,
"input_tokens": 1300683,
"cached_input_tokens": 1123456,
"output_tokens": 10414,
"list_usd": 0.978666,
"grok_reported_usd": 0.332746
},
{
"arm": "grok-cli",
"task": "linear-nohint-warm",
"suite": "mcp",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 213.3,
"model_calls": 32,
"input_tokens": 1820740,
"cached_input_tokens": 1674752,
"output_tokens": 10705,
"list_usd": 1.193582,
"grok_reported_usd": 0.405818
},
{
"arm": "grok-cli",
"task": "linear-nohint-warm",
"suite": "mcp",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 186.58,
"model_calls": 24,
"input_tokens": 1548710,
"cached_input_tokens": 1456128,
"output_tokens": 10677,
"list_usd": 0.97729,
"grok_reported_usd": 0.332279
},
{
"arm": "graff-v302-12",
"task": "linear-quiet",
"suite": "mcp",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 36.39,
"model_calls": 5,
"input_tokens": 39325,
"cached_input_tokens": 24576,
"output_tokens": 2686,
"list_usd": 0.057902
},
{
"arm": "graff-v302-12",
"task": "linear-quiet",
"suite": "mcp",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 15.7,
"model_calls": 4,
"input_tokens": 27753,
"cached_input_tokens": 13056,
"output_tokens": 1010,
"list_usd": 0.041982
},
{
"arm": "graff-v302-12",
"task": "linear-quiet",
"suite": "mcp",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 11.75,
"model_calls": 4,
"input_tokens": 27195,
"cached_input_tokens": 20480,
"output_tokens": 610,
"list_usd": 0.02733
},
{
"arm": "grok-cli",
"task": "linear-quiet",
"suite": "mcp",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 175.49,
"model_calls": 21,
"input_tokens": 1055864,
"cached_input_tokens": 945408,
"output_tokens": 10229,
"list_usd": 0.75499,
"grok_reported_usd": 0.256697
},
{
"arm": "grok-cli",
"task": "linear-quiet",
"suite": "mcp",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 208.65,
"model_calls": 27,
"input_tokens": 1609676,
"cached_input_tokens": 1456896,
"output_tokens": 12836,
"list_usd": 1.111024,
"grok_reported_usd": 0.377748
},
{
"arm": "grok-cli",
"task": "linear-quiet",
"suite": "mcp",
"rep": 3,
"pass": false,
"timed_out": true,
"wall_s": 240.26,
"model_calls": null,
"input_tokens": null,
"cached_input_tokens": null,
"output_tokens": null,
"list_usd": null,
"grok_reported_usd": null
},
{
"arm": "graff-v302-12",
"task": "linear-quiet-warm",
"suite": "mcp",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 14.07,
"model_calls": 4,
"input_tokens": 28290,
"cached_input_tokens": 18944,
"output_tokens": 922,
"list_usd": 0.033696
},
{
"arm": "graff-v302-12",
"task": "linear-quiet-warm",
"suite": "mcp",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 29.45,
"model_calls": 4,
"input_tokens": 30738,
"cached_input_tokens": 27264,
"output_tokens": 2076,
"list_usd": 0.033036
},
{
"arm": "graff-v302-12",
"task": "linear-quiet-warm",
"suite": "mcp",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 25.89,
"model_calls": 4,
"input_tokens": 30195,
"cached_input_tokens": 14720,
"output_tokens": 1913,
"list_usd": 0.049788
},
{
"arm": "grok-cli",
"task": "linear-quiet-warm",
"suite": "mcp",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 149.0,
"model_calls": 14,
"input_tokens": 586404,
"cached_input_tokens": 515200,
"output_tokens": 9160,
"list_usd": 0.454968,
"grok_reported_usd": 0.154689
},
{
"arm": "grok-cli",
"task": "linear-quiet-warm",
"suite": "mcp",
"rep": 2,
"pass": false,
"timed_out": true,
"wall_s": 240.41,
"model_calls": null,
"input_tokens": null,
"cached_input_tokens": null,
"output_tokens": null,
"list_usd": null,
"grok_reported_usd": null
},
{
"arm": "grok-cli",
"task": "linear-quiet-warm",
"suite": "mcp",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 222.33,
"model_calls": 31,
"input_tokens": 1712511,
"cached_input_tokens": 1638528,
"output_tokens": 12329,
"list_usd": 1.041204,
"grok_reported_usd": 0.354009
},
{
"arm": "graff-v302-12",
"task": "linear-reduce",
"suite": "mcp",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 9.6,
"model_calls": 4,
"input_tokens": 27229,
"cached_input_tokens": 18048,
"output_tokens": 585,
"list_usd": 0.030896
},
{
"arm": "graff-v302-12",
"task": "linear-reduce",
"suite": "mcp",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 24.44,
"model_calls": 6,
"input_tokens": 42016,
"cached_input_tokens": 12288,
"output_tokens": 1452,
"list_usd": 0.074312
},
{
"arm": "graff-v302-12",
"task": "linear-reduce",
"suite": "mcp",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 29.79,
"model_calls": 4,
"input_tokens": 28643,
"cached_input_tokens": 8960,
"output_tokens": 1502,
"list_usd": 0.052858
},
{
"arm": "grok-cli",
"task": "linear-reduce",
"suite": "mcp",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 162.9,
"model_calls": 18,
"input_tokens": 743688,
"cached_input_tokens": 697472,
"output_tokens": 10703,
"list_usd": 0.505386,
"grok_reported_usd": 0.171831
},
{
"arm": "grok-cli",
"task": "linear-reduce",
"suite": "mcp",
"rep": 2,
"pass": true,
"timed_out": true,
"wall_s": 240.09,
"model_calls": null,
"input_tokens": null,
"cached_input_tokens": null,
"output_tokens": null,
"list_usd": null,
"grok_reported_usd": null
},
{
"arm": "grok-cli",
"task": "linear-reduce",
"suite": "mcp",
"rep": 3,
"pass": true,
"timed_out": true,
"wall_s": 240.36,
"model_calls": null,
"input_tokens": null,
"cached_input_tokens": null,
"output_tokens": null,
"list_usd": null,
"grok_reported_usd": null
},
{
"arm": "graff-v302-12",
"task": "linear-reduce-warm",
"suite": "mcp",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 15.68,
"model_calls": 5,
"input_tokens": 35763,
"cached_input_tokens": 14848,
"output_tokens": 927,
"list_usd": 0.054816
},
{
"arm": "graff-v302-12",
"task": "linear-reduce-warm",
"suite": "mcp",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 19.93,
"model_calls": 4,
"input_tokens": 29420,
"cached_input_tokens": 13056,
"output_tokens": 1400,
"list_usd": 0.047656
},
{
"arm": "graff-v302-12",
"task": "linear-reduce-warm",
"suite": "mcp",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 9.22,
"model_calls": 4,
"input_tokens": 28176,
"cached_input_tokens": 14720,
"output_tokens": 658,
"list_usd": 0.03822
},
{
"arm": "grok-cli",
"task": "linear-reduce-warm",
"suite": "mcp",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 151.94,
"model_calls": 18,
"input_tokens": 839586,
"cached_input_tokens": 738944,
"output_tokens": 9438,
"list_usd": 0.627384,
"grok_reported_usd": 0.213311
},
{
"arm": "grok-cli",
"task": "linear-reduce-warm",
"suite": "mcp",
"rep": 2,
"pass": false,
"timed_out": true,
"wall_s": 240.5,
"model_calls": null,
"input_tokens": null,
"cached_input_tokens": null,
"output_tokens": null,
"list_usd": null,
"grok_reported_usd": null
},
{
"arm": "grok-cli",
"task": "linear-reduce-warm",
"suite": "mcp",
"rep": 3,
"pass": true,
"timed_out": true,
"wall_s": 240.19,
"model_calls": null,
"input_tokens": null,
"cached_input_tokens": null,
"output_tokens": null,
"list_usd": null,
"grok_reported_usd": null
},
{
"arm": "graff-v302-12",
"task": "linear-sidecar",
"suite": "mcp",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 90.94,
"model_calls": 11,
"input_tokens": 78174,
"cached_input_tokens": 48384,
"output_tokens": 2329,
"list_usd": 0.097746
},
{
"arm": "graff-v302-12",
"task": "linear-sidecar",
"suite": "mcp",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 25.12,
"model_calls": 9,
"input_tokens": 63155,
"cached_input_tokens": 27648,
"output_tokens": 1462,
"list_usd": 0.09361
},
{
"arm": "graff-v302-12",
"task": "linear-sidecar",
"suite": "mcp",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 23.94,
"model_calls": 7,
"input_tokens": 49237,
"cached_input_tokens": 22656,
"output_tokens": 1265,
"list_usd": 0.07208
},
{
"arm": "grok-cli",
"task": "linear-sidecar",
"suite": "mcp",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 227.95,
"model_calls": 28,
"input_tokens": 1173801,
"cached_input_tokens": 1011840,
"output_tokens": 12284,
"list_usd": 0.903546,
"grok_reported_usd": 0.009746
},
{
"arm": "grok-cli",
"task": "linear-sidecar",
"suite": "mcp",
"rep": 2,
"pass": false,
"timed_out": true,
"wall_s": 240.46,
"model_calls": null,
"input_tokens": null,
"cached_input_tokens": null,
"output_tokens": null,
"list_usd": null,
"grok_reported_usd": null
},
{
"arm": "grok-cli",
"task": "linear-sidecar",
"suite": "mcp",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 231.62,
"model_calls": 30,
"input_tokens": 2012313,
"cached_input_tokens": 1892224,
"output_tokens": 12893,
"list_usd": 1.263648,
"grok_reported_usd": 0.009541
},
{
"arm": "graff-v302-12",
"task": "linear-split",
"suite": "mcp",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 40.8,
"model_calls": 13,
"input_tokens": 89608,
"cached_input_tokens": 50816,
"output_tokens": 2727,
"list_usd": 0.119354
},
{
"arm": "graff-v302-12",
"task": "linear-split",
"suite": "mcp",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 49.21,
"model_calls": 8,
"input_tokens": 57236,
"cached_input_tokens": 23040,
"output_tokens": 2549,
"list_usd": 0.095206
},
{
"arm": "graff-v302-12",
"task": "linear-split",
"suite": "mcp",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 37.53,
"model_calls": 13,
"input_tokens": 90059,
"cached_input_tokens": 56320,
"output_tokens": 2750,
"list_usd": 0.112138
},
{
"arm": "grok-cli",
"task": "linear-split",
"suite": "mcp",
"rep": 1,
"pass": false,
"timed_out": true,
"wall_s": 240.58,
"model_calls": null,
"input_tokens": null,
"cached_input_tokens": null,
"output_tokens": null,
"list_usd": null,
"grok_reported_usd": null
},
{
"arm": "grok-cli",
"task": "linear-split",
"suite": "mcp",
"rep": 2,
"pass": false,
"timed_out": true,
"wall_s": 240.49,
"model_calls": null,
"input_tokens": null,
"cached_input_tokens": null,
"output_tokens": null,
"list_usd": null,
"grok_reported_usd": null
},
{
"arm": "grok-cli",
"task": "linear-split",
"suite": "mcp",
"rep": 3,
"pass": false,
"timed_out": true,
"wall_s": 240.44,
"model_calls": null,
"input_tokens": null,
"cached_input_tokens": null,
"output_tokens": null,
"list_usd": null,
"grok_reported_usd": null
},
{
"arm": "graff-v302-12",
"task": "linear-warm",
"suite": "mcp",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 24.97,
"model_calls": 4,
"input_tokens": 28948,
"cached_input_tokens": 4608,
"output_tokens": 1393,
"list_usd": 0.059342
},
{
"arm": "graff-v302-12",
"task": "linear-warm",
"suite": "mcp",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 11.25,
"model_calls": 4,
"input_tokens": 27939,
"cached_input_tokens": 19968,
"output_tokens": 617,
"list_usd": 0.029628
},
{
"arm": "graff-v302-12",
"task": "linear-warm",
"suite": "mcp",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 19.88,
"model_calls": 4,
"input_tokens": 29492,
"cached_input_tokens": 20736,
"output_tokens": 1345,
"list_usd": 0.03595
},
{
"arm": "grok-cli",
"task": "linear-warm",
"suite": "mcp",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 178.9,
"model_calls": 19,
"input_tokens": 1031519,
"cached_input_tokens": 905984,
"output_tokens": 9122,
"list_usd": 0.758794,
"grok_reported_usd": 0.25799
},
{
"arm": "grok-cli",
"task": "linear-warm",
"suite": "mcp",
"rep": 2,
"pass": false,
"timed_out": true,
"wall_s": 240.28,
"model_calls": null,
"input_tokens": null,
"cached_input_tokens": null,
"output_tokens": null,
"list_usd": null,
"grok_reported_usd": null
},
{
"arm": "grok-cli",
"task": "linear-warm",
"suite": "mcp",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 165.4,
"model_calls": 22,
"input_tokens": 1283560,
"cached_input_tokens": 1200896,
"output_tokens": 9559,
"list_usd": 0.82313,
"grok_reported_usd": 0.279864
},
{
"arm": "graff-v302-12",
"task": "recall-noise",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 4.51,
"model_calls": 2,
"input_tokens": 12390,
"cached_input_tokens": 7168,
"output_tokens": 105,
"list_usd": 0.014658
},
{
"arm": "graff-v302-12",
"task": "recall-noise",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 3.92,
"model_calls": 2,
"input_tokens": 12390,
"cached_input_tokens": 7168,
"output_tokens": 117,
"list_usd": 0.01473
},
{
"arm": "graff-v302-12",
"task": "recall-noise",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 3.66,
"model_calls": 2,
"input_tokens": 12399,
"cached_input_tokens": 2304,
"output_tokens": 107,
"list_usd": 0.021984
},
{
"arm": "grok-cli",
"task": "recall-noise",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 9.02,
"model_calls": 3,
"input_tokens": 54881,
"cached_input_tokens": 34816,
"output_tokens": 174,
"list_usd": 0.058582,
"grok_reported_usd": 0.019918
},
{
"arm": "grok-cli",
"task": "recall-noise",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 6.01,
"model_calls": 2,
"input_tokens": 36589,
"cached_input_tokens": 30080,
"output_tokens": 131,
"list_usd": 0.028844,
"grok_reported_usd": 0.009807
},
{
"arm": "grok-cli",
"task": "recall-noise",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 6.73,
"model_calls": 2,
"input_tokens": 36569,
"cached_input_tokens": 19712,
"output_tokens": 123,
"list_usd": 0.044308,
"grok_reported_usd": 0.015065
},
{
"arm": "graff-v302-12",
"task": "refactor-rename",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 7.42,
"model_calls": 4,
"input_tokens": 25559,
"cached_input_tokens": 14976,
"output_tokens": 294,
"list_usd": 0.030418
},
{
"arm": "graff-v302-12",
"task": "refactor-rename",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 7.49,
"model_calls": 4,
"input_tokens": 25551,
"cached_input_tokens": 19584,
"output_tokens": 307,
"list_usd": 0.023568
},
{
"arm": "graff-v302-12",
"task": "refactor-rename",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 10.22,
"model_calls": 7,
"input_tokens": 45672,
"cached_input_tokens": 34816,
"output_tokens": 307,
"list_usd": 0.040962
},
{
"arm": "grok-cli",
"task": "refactor-rename",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 14.83,
"model_calls": 5,
"input_tokens": 92924,
"cached_input_tokens": 66048,
"output_tokens": 526,
"list_usd": 0.089932,
"grok_reported_usd": 0.030577
},
{
"arm": "grok-cli",
"task": "refactor-rename",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 19.18,
"model_calls": 5,
"input_tokens": 93469,
"cached_input_tokens": 86016,
"output_tokens": 533,
"list_usd": 0.061112,
"grok_reported_usd": 0.020778
},
{
"arm": "grok-cli",
"task": "refactor-rename",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 22.34,
"model_calls": 5,
"input_tokens": 92953,
"cached_input_tokens": 85760,
"output_tokens": 596,
"list_usd": 0.060842,
"grok_reported_usd": 0.020686
},
{
"arm": "graff-v302-12",
"task": "regex-count",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 12.08,
"model_calls": 4,
"input_tokens": 24937,
"cached_input_tokens": 19584,
"output_tokens": 683,
"list_usd": 0.024596
},
{
"arm": "graff-v302-12",
"task": "regex-count",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 10.44,
"model_calls": 4,
"input_tokens": 25043,
"cached_input_tokens": 13824,
"output_tokens": 404,
"list_usd": 0.031774
},
{
"arm": "graff-v302-12",
"task": "regex-count",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 9.77,
"model_calls": 4,
"input_tokens": 24959,
"cached_input_tokens": 19456,
"output_tokens": 539,
"list_usd": 0.023968
},
{
"arm": "grok-cli",
"task": "regex-count",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 29.86,
"model_calls": 6,
"input_tokens": 113525,
"cached_input_tokens": 99200,
"output_tokens": 1468,
"list_usd": 0.087058,
"grok_reported_usd": 0.0296
},
{
"arm": "grok-cli",
"task": "regex-count",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 33.87,
"model_calls": 6,
"input_tokens": 115681,
"cached_input_tokens": 107136,
"output_tokens": 1777,
"list_usd": 0.08132,
"grok_reported_usd": 0.027649
},
{
"arm": "grok-cli",
"task": "regex-count",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 15.44,
"model_calls": 4,
"input_tokens": 74378,
"cached_input_tokens": 61184,
"output_tokens": 698,
"list_usd": 0.061168,
"grok_reported_usd": 0.020797
},
{
"arm": "graff-v302-12",
"task": "write-tests",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 5.86,
"model_calls": 3,
"input_tokens": 18684,
"cached_input_tokens": 13312,
"output_tokens": 305,
"list_usd": 0.01923
},
{
"arm": "graff-v302-12",
"task": "write-tests",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 8.11,
"model_calls": 3,
"input_tokens": 18684,
"cached_input_tokens": 8320,
"output_tokens": 362,
"list_usd": 0.02706
},
{
"arm": "graff-v302-12",
"task": "write-tests",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 5.85,
"model_calls": 3,
"input_tokens": 18610,
"cached_input_tokens": 13312,
"output_tokens": 304,
"list_usd": 0.019076
},
{
"arm": "grok-cli",
"task": "write-tests",
"suite": "core",
"rep": 1,
"pass": true,
"timed_out": false,
"wall_s": 16.68,
"model_calls": 5,
"input_tokens": 92336,
"cached_input_tokens": 85248,
"output_tokens": 569,
"list_usd": 0.060214,
"grok_reported_usd": 0.020473
},
{
"arm": "grok-cli",
"task": "write-tests",
"suite": "core",
"rep": 2,
"pass": true,
"timed_out": false,
"wall_s": 15.69,
"model_calls": 5,
"input_tokens": 92880,
"cached_input_tokens": 49408,
"output_tokens": 591,
"list_usd": 0.115194,
"grok_reported_usd": 0.039166
},
{
"arm": "grok-cli",
"task": "write-tests",
"suite": "core",
"rep": 3,
"pass": true,
"timed_out": false,
"wall_s": 18.1,
"model_calls": 5,
"input_tokens": 92832,
"cached_input_tokens": 69248,
"output_tokens": 612,
"list_usd": 0.085464,
"grok_reported_usd": 0.029058
}
]
}