graff — a fast agentic coding harness in Zig: multi-provider, MCP, workflows, DGM evolution loop, TS/Python SDKs
Python
evals: --interleave orders runs task by task across harnesses
justrach54fcfdb933f5
diff --git a/graff-evals/harnesses.json b/graff-evals/harnesses.jsonindex 5be226e9..c8813255 100644--- a/graff-evals/harnesses.json+++ b/graff-evals/harnesses.json@@ -376,6 +376,56 @@"default_model": "gpt-6.1-sol","note": "hill-climb arm vs the Codex app server; GRAFF_HC names the switches"},+ "graff-hc-r4-effort": {+ "cmd": ["{repo}/zig-out/bin/graff", "repl", "--model", "{model}", "--yolo"],+ "stdin": "{prompt}\n",+ "answer": "stdout",+ "usage": "graff-stderr",+ "env": {"GRAFF_POST_DEADLINE_SECS": "90", "GRAFF_HC": "user-key,fold-more,effort-low"},+ "model_prefix": "codex",+ "default_model": "gpt-6.1-sol",+ "note": "hill-climb arm vs the Codex app server; GRAFF_HC names the switches"+ },+ "graff-hc-r4-verb": {+ "cmd": ["{repo}/zig-out/bin/graff", "repl", "--model", "{model}", "--yolo"],+ "stdin": "{prompt}\n",+ "answer": "stdout",+ "usage": "graff-stderr",+ "env": {"GRAFF_POST_DEADLINE_SECS": "90", "GRAFF_HC": "user-key,fold-more,verbosity-low"},+ "model_prefix": "codex",+ "default_model": "gpt-6.1-sol",+ "note": "hill-climb arm vs the Codex app server; GRAFF_HC names the switches"+ },+ "graff-hc-r4-both": {+ "cmd": ["{repo}/zig-out/bin/graff", "repl", "--model", "{model}", "--yolo"],+ "stdin": "{prompt}\n",+ "answer": "stdout",+ "usage": "graff-stderr",+ "env": {"GRAFF_POST_DEADLINE_SECS": "90", "GRAFF_HC": "user-key,fold-more,effort-low,verbosity-low"},+ "model_prefix": "codex",+ "default_model": "gpt-6.1-sol",+ "note": "hill-climb arm vs the Codex app server; GRAFF_HC names the switches"+ },+ "graff-hc-r5-slimnote": {+ "cmd": ["{repo}/zig-out/bin/graff", "repl", "--model", "{model}", "--yolo"],+ "stdin": "{prompt}\n",+ "answer": "stdout",+ "usage": "graff-stderr",+ "env": {"GRAFF_POST_DEADLINE_SECS": "90", "GRAFF_HC": "user-key,fold-more,slim-note"},+ "model_prefix": "codex",+ "default_model": "gpt-6.1-sol",+ "note": "hill-climb arm vs the Codex app server; GRAFF_HC names the switches"+ },+ "graff-hc-r5-ids": {+ "cmd": ["{repo}/zig-out/bin/graff", "repl", "--model", "{model}", "--yolo"],+ "stdin": "{prompt}\n",+ "answer": "stdout",+ "usage": "graff-stderr",+ "env": {"GRAFF_POST_DEADLINE_SECS": "90", "GRAFF_HC": "user-key,fold-more,codex-ids"},+ "model_prefix": "codex",+ "default_model": "gpt-6.1-sol",+ "note": "hill-climb arm vs the Codex app server; GRAFF_HC names the switches"+ },"codex-app-server": {"cmd": ["python3", "{repo}/graff-evals/codex_app_server.py", "{model}", "{prompt}"],"answer": "stdout",diff --git a/graff-evals/run.py b/graff-evals/run.pyindex a4d771b1..3bad96b6 100755--- a/graff-evals/run.py+++ b/graff-evals/run.py@@ -527,6 +527,8 @@ def main():ap.add_argument("--suite", default="all",help="core, rlm, swe, mcp, inhouse, live, comma-mix, or all (core+rlm+swe; mcp/inhouse/live are opt-in)")ap.add_argument("--reps", type=int, default=1)+ ap.add_argument("--interleave", action="store_true",+ help="order runs task by task across harnesses, so A/B arms see the same backend conditions")ap.add_argument("--jobs", "-j", type=int, default=1,help="parallel task×harness runs (default 1; each has its own sandbox)")ap.add_argument("--interactive", action="store_true", help="pick a task+harness, watch it live")@@ -573,6 +575,10 @@ def main():continuefor rep in range(1, args.reps + 1):work.append((hname, harness, task, model, rep))+ if args.interleave:+ arm_at = {h: i for i, h in enumerate(args.harness.split(","))}+ task_at = {tid: i for i, tid in enumerate(picked)}+ work.sort(key=lambda w: (w[4], task_at[w[2]["id"]], arm_at[w[0]]))jobs = max(1, args.jobs)print(f"{len(work)} runs · {jobs} worker{'s' if jobs != 1 else ''} · {out_path}", flush=True)records = []