Skip to content

graff — a fast agentic coding harness in Zig: multi-provider, MCP, workflows, DGM evolution loop, TS/Python SDKs

Python
11.5 KB
Raw Blame
{
  "title": "Live evals \u00b7 12 gated PRs",
  "date": "2026-09-09",
  "model": "grok-4.6",
  "seat": "SuperGrok",
  "n": 3,
  "score": {
    "task_pass": ">=2/3",
    "cost": "passing reps of passing tasks",
    "band": "$2 / $0.50 cached / $6 out per 1M (official low band)",
    "failed_usd": 0,
    "cash_usd": 0
  },
  "caveats": [
    "Only graff-195 is G1\u2013G6 certified; the other 11 are published live tasks.",
    "Stored JSONL list$ high-bands the rep sum; headline $ is the official per-request low band.",
    "A check-green with no tokens does not count (exo turbo-ws-duplex / turbo-asgi died in <1s).",
    "Wall is a hang detector. Do not score Graff first-token (boot \u203a)."
  ],
  "tasks": [
    "graff-195",
    "graff-726",
    "graff-727",
    "graff-servers",
    "graff-interrupt",
    "graff-acp-pixels",
    "graff-gemini-ix",
    "codedb-arch",
    "codedb-hybrid",
    "turbo-ws-load",
    "turbo-ws-duplex",
    "turbo-asgi"
  ],
  "harnesses": {
    "graff-dev": {
      "tasks": {
        "graff-195": {
          "n": 3,
          "wins": 3,
          "pass": true,
          "cost": 1.3459,
          "timeouts": 0
        },
        "graff-726": {
          "n": 3,
          "wins": 3,
          "pass": true,
          "cost": 1.8705,
          "timeouts": 0
        },
        "graff-727": {
          "n": 3,
          "wins": 2,
          "pass": true,
          "cost": 0.9148,
          "timeouts": 0
        },
        "graff-servers": {
          "n": 3,
          "wins": 3,
          "pass": true,
          "cost": 1.2976,
          "timeouts": 0
        },
        "graff-interrupt": {
          "n": 3,
          "wins": 3,
          "pass": true,
          "cost": 2.3299,
          "timeouts": 0
        },
        "graff-acp-pixels": {
          "n": 3,
          "wins": 3,
          "pass": true,
          "cost": 1.2514,
          "timeouts": 0
        },
        "graff-gemini-ix": {
          "n": 3,
          "wins": 2,
          "pass": true,
          "cost": 0.6365,
          "timeouts": 1
        },
        "codedb-arch": {
          "n": 3,
          "wins": 3,
          "pass": true,
          "cost": 2.0764,
          "timeouts": 0
        },
        "codedb-hybrid": {
          "n": 3,
          "wins": 3,
          "pass": true,
          "cost": 2.7353,
          "timeouts": 0
        },
        "turbo-ws-load": {
          "n": 3,
          "wins": 3,
          "pass": true,
          "cost": 2.301,
          "timeouts": 0
        },
        "turbo-ws-duplex": {
          "n": 3,
          "wins": 3,
          "pass": true,
          "cost": 2.7231,
          "timeouts": 0
        },
        "turbo-asgi": {
          "n": 3,
          "wins": 3,
          "pass": true,
          "cost": 2.0022,
          "timeouts": 0
        }
      },
      "honest_usd": 21.48,
      "pass_reps": 34,
      "n_reps": 36,
      "pass_tasks": 12,
      "n_tasks": 12,
      "mean_wall_s": 264,
      "timeout_ok": 1,
      "timeout_fail": 0,
      "label": "graff",
      "sources": [
        "results/run-20260908-173944.jsonl",
        "results/run-20260908-182322.jsonl"
      ]
    },
    "grok": {
      "tasks": {
        "graff-195": {
          "n": 3,
          "wins": 2,
          "pass": true,
          "cost": 1.9414,
          "timeouts": 1
        },
        "graff-726": {
          "n": 3,
          "wins": 3,
          "pass": true,
          "cost": 1.9048,
          "timeouts": 0
        },
        "graff-727": {
          "n": 3,
          "wins": 1,
          "pass": false,
          "cost": 0.0,
          "timeouts": 1
        },
        "graff-servers": {
          "n": 3,
          "wins": 3,
          "pass": true,
          "cost": 2.3201,
          "timeouts": 0
        },
        "graff-interrupt": {
          "n": 3,
          "wins": 3,
          "pass": true,
          "cost": 2.2116,
          "timeouts": 0
        },
        "graff-acp-pixels": {
          "n": 3,
          "wins": 2,
          "pass": true,
          "cost": 2.0025,
          "timeouts": 1
        },
        "graff-gemini-ix": {
          "n": 3,
          "wins": 3,
          "pass": true,
          "cost": 1.7837,
          "timeouts": 0
        },
        "codedb-arch": {
          "n": 3,
          "wins": 3,
          "pass": true,
          "cost": 5.092,
          "timeouts": 0
        },
        "codedb-hybrid": {
          "n": 3,
          "wins": 3,
          "pass": true,
          "cost": 4.0847,
          "timeouts": 0
        },
        "turbo-ws-load": {
          "n": 3,
          "wins": 3,
          "pass": true,
          "cost": 2.7395,
          "timeouts": 0
        },
        "turbo-ws-duplex": {
          "n": 3,
          "wins": 3,
          "pass": true,
          "cost": 3.1379,
          "timeouts": 0
        },
        "turbo-asgi": {
          "n": 3,
          "wins": 3,
          "pass": true,
          "cost": 6.4704,
          "timeouts": 0
        }
      },
      "honest_usd": 33.69,
      "pass_reps": 32,
      "n_reps": 36,
      "pass_tasks": 11,
      "n_tasks": 12,
      "mean_wall_s": 362,
      "timeout_ok": 1,
      "timeout_fail": 2,
      "label": "grok",
      "sources": [
        "results/run-20260908-173944.jsonl",
        "results/run-20260909-011118.jsonl"
      ]
    },
    "opencode": {
      "tasks": {
        "graff-195": {
          "n": 3,
          "wins": 3,
          "pass": true,
          "cost": 2.3473,
          "timeouts": 0
        },
        "graff-726": {
          "n": 3,
          "wins": 3,
          "pass": true,
          "cost": 1.4321,
          "timeouts": 0
        },
        "graff-727": {
          "n": 3,
          "wins": 3,
          "pass": true,
          "cost": 0.9391,
          "timeouts": 1
        },
        "graff-servers": {
          "n": 3,
          "wins": 3,
          "pass": true,
          "cost": 2.0728,
          "timeouts": 1
        },
        "graff-interrupt": {
          "n": 3,
          "wins": 3,
          "pass": true,
          "cost": 1.0457,
          "timeouts": 1
        },
        "graff-acp-pixels": {
          "n": 3,
          "wins": 3,
          "pass": true,
          "cost": 1.3471,
          "timeouts": 1
        },
        "graff-gemini-ix": {
          "n": 3,
          "wins": 3,
          "pass": true,
          "cost": 1.3579,
          "timeouts": 1
        },
        "codedb-arch": {
          "n": 3,
          "wins": 3,
          "pass": true,
          "cost": 3.0088,
          "timeouts": 0
        },
        "codedb-hybrid": {
          "n": 3,
          "wins": 3,
          "pass": true,
          "cost": 3.8869,
          "timeouts": 0
        },
        "turbo-ws-load": {
          "n": 3,
          "wins": 3,
          "pass": true,
          "cost": 2.4725,
          "timeouts": 0
        },
        "turbo-ws-duplex": {
          "n": 3,
          "wins": 3,
          "pass": true,
          "cost": 2.5499,
          "timeouts": 0
        },
        "turbo-asgi": {
          "n": 3,
          "wins": 3,
          "pass": true,
          "cost": 3.2466,
          "timeouts": 0
        }
      },
      "honest_usd": 25.71,
      "pass_reps": 36,
      "n_reps": 36,
      "pass_tasks": 12,
      "n_tasks": 12,
      "mean_wall_s": 309,
      "timeout_ok": 5,
      "timeout_fail": 0,
      "label": "OpenCode",
      "sources": [
        "results/run-20260909-042501.jsonl"
      ]
    },
    "pi-xai": {
      "tasks": {
        "graff-195": {
          "n": 3,
          "wins": 3,
          "pass": true,
          "cost": 1.351,
          "timeouts": 0
        },
        "graff-726": {
          "n": 3,
          "wins": 3,
          "pass": true,
          "cost": 0.784,
          "timeouts": 0
        },
        "graff-727": {
          "n": 3,
          "wins": 3,
          "pass": true,
          "cost": 0.7027,
          "timeouts": 0
        },
        "graff-servers": {
          "n": 3,
          "wins": 3,
          "pass": true,
          "cost": 1.1425,
          "timeouts": 0
        },
        "graff-interrupt": {
          "n": 3,
          "wins": 3,
          "pass": true,
          "cost": 0.9211,
          "timeouts": 0
        },
        "graff-acp-pixels": {
          "n": 3,
          "wins": 3,
          "pass": true,
          "cost": 1.1851,
          "timeouts": 0
        },
        "graff-gemini-ix": {
          "n": 3,
          "wins": 3,
          "pass": true,
          "cost": 0.9448,
          "timeouts": 0
        },
        "codedb-arch": {
          "n": 3,
          "wins": 2,
          "pass": true,
          "cost": 1.7248,
          "timeouts": 2
        },
        "codedb-hybrid": {
          "n": 3,
          "wins": 3,
          "pass": true,
          "cost": 3.3579,
          "timeouts": 3
        },
        "turbo-ws-load": {
          "n": 3,
          "wins": 3,
          "pass": true,
          "cost": 1.9917,
          "timeouts": 1
        },
        "turbo-ws-duplex": {
          "n": 3,
          "wins": 3,
          "pass": true,
          "cost": 2.5246,
          "timeouts": 0
        },
        "turbo-asgi": {
          "n": 3,
          "wins": 3,
          "pass": true,
          "cost": 1.8364,
          "timeouts": 0
        }
      },
      "honest_usd": 18.47,
      "pass_reps": 35,
      "n_reps": 36,
      "pass_tasks": 12,
      "n_tasks": 12,
      "mean_wall_s": 334,
      "timeout_ok": 5,
      "timeout_fail": 1,
      "label": "Pi",
      "sources": [
        "results/run-20260909-095017.jsonl"
      ]
    },
    "exo": {
      "tasks": {
        "graff-195": {
          "n": 3,
          "wins": 3,
          "pass": true,
          "cost": 1.9886,
          "timeouts": 0
        },
        "graff-726": {
          "n": 3,
          "wins": 3,
          "pass": true,
          "cost": 1.5625,
          "timeouts": 0
        },
        "graff-727": {
          "n": 3,
          "wins": 3,
          "pass": true,
          "cost": 2.4545,
          "timeouts": 0
        },
        "graff-servers": {
          "n": 3,
          "wins": 2,
          "pass": true,
          "cost": 1.0468,
          "timeouts": 1
        },
        "graff-interrupt": {
          "n": 3,
          "wins": 3,
          "pass": true,
          "cost": 1.3081,
          "timeouts": 0
        },
        "graff-acp-pixels": {
          "n": 3,
          "wins": 2,
          "pass": true,
          "cost": 1.4308,
          "timeouts": 1
        },
        "graff-gemini-ix": {
          "n": 3,
          "wins": 1,
          "pass": false,
          "cost": 0.0,
          "timeouts": 2
        },
        "codedb-arch": {
          "n": 3,
          "wins": 2,
          "pass": true,
          "cost": 1.39,
          "timeouts": 1
        },
        "codedb-hybrid": {
          "n": 3,
          "wins": 3,
          "pass": true,
          "cost": 3.483,
          "timeouts": 0
        },
        "turbo-ws-load": {
          "n": 3,
          "wins": 2,
          "pass": true,
          "cost": 1.3484,
          "timeouts": 0
        },
        "turbo-ws-duplex": {
          "n": 3,
          "wins": 0,
          "pass": false,
          "cost": 0.0,
          "timeouts": 0
        },
        "turbo-asgi": {
          "n": 3,
          "wins": 0,
          "pass": false,
          "cost": 0.0,
          "timeouts": 0
        }
      },
      "honest_usd": 16.01,
      "pass_reps": 24,
      "n_reps": 36,
      "pass_tasks": 9,
      "n_tasks": 12,
      "mean_wall_s": 281,
      "timeout_ok": 3,
      "timeout_fail": 2,
      "label": "exo",
      "sources": [
        "results/run-20260909-095016.jsonl"
      ]
    }
  }
}