Skip to content

graff — a fast agentic coding harness in Zig: multi-provider, MCP, workflows, DGM evolution loop, TS/Python SDKs

Python

evals: --interleave orders runs task by task across harnesses

justrach54fcfdb933f5
diff --git a/graff-evals/harnesses.json b/graff-evals/harnesses.json
index 5be226e9..c8813255 100644
--- a/graff-evals/harnesses.json
+++ b/graff-evals/harnesses.json
@@ -376,6 +376,56 @@
"default_model": "gpt-6.1-sol",
"note": "hill-climb arm vs the Codex app server; GRAFF_HC names the switches"
},
+ "graff-hc-r4-effort": {
+ "cmd": ["{repo}/zig-out/bin/graff", "repl", "--model", "{model}", "--yolo"],
+ "stdin": "{prompt}\n",
+ "answer": "stdout",
+ "usage": "graff-stderr",
+ "env": {"GRAFF_POST_DEADLINE_SECS": "90", "GRAFF_HC": "user-key,fold-more,effort-low"},
+ "model_prefix": "codex",
+ "default_model": "gpt-6.1-sol",
+ "note": "hill-climb arm vs the Codex app server; GRAFF_HC names the switches"
+ },
+ "graff-hc-r4-verb": {
+ "cmd": ["{repo}/zig-out/bin/graff", "repl", "--model", "{model}", "--yolo"],
+ "stdin": "{prompt}\n",
+ "answer": "stdout",
+ "usage": "graff-stderr",
+ "env": {"GRAFF_POST_DEADLINE_SECS": "90", "GRAFF_HC": "user-key,fold-more,verbosity-low"},
+ "model_prefix": "codex",
+ "default_model": "gpt-6.1-sol",
+ "note": "hill-climb arm vs the Codex app server; GRAFF_HC names the switches"
+ },
+ "graff-hc-r4-both": {
+ "cmd": ["{repo}/zig-out/bin/graff", "repl", "--model", "{model}", "--yolo"],
+ "stdin": "{prompt}\n",
+ "answer": "stdout",
+ "usage": "graff-stderr",
+ "env": {"GRAFF_POST_DEADLINE_SECS": "90", "GRAFF_HC": "user-key,fold-more,effort-low,verbosity-low"},
+ "model_prefix": "codex",
+ "default_model": "gpt-6.1-sol",
+ "note": "hill-climb arm vs the Codex app server; GRAFF_HC names the switches"
+ },
+ "graff-hc-r5-slimnote": {
+ "cmd": ["{repo}/zig-out/bin/graff", "repl", "--model", "{model}", "--yolo"],
+ "stdin": "{prompt}\n",
+ "answer": "stdout",
+ "usage": "graff-stderr",
+ "env": {"GRAFF_POST_DEADLINE_SECS": "90", "GRAFF_HC": "user-key,fold-more,slim-note"},
+ "model_prefix": "codex",
+ "default_model": "gpt-6.1-sol",
+ "note": "hill-climb arm vs the Codex app server; GRAFF_HC names the switches"
+ },
+ "graff-hc-r5-ids": {
+ "cmd": ["{repo}/zig-out/bin/graff", "repl", "--model", "{model}", "--yolo"],
+ "stdin": "{prompt}\n",
+ "answer": "stdout",
+ "usage": "graff-stderr",
+ "env": {"GRAFF_POST_DEADLINE_SECS": "90", "GRAFF_HC": "user-key,fold-more,codex-ids"},
+ "model_prefix": "codex",
+ "default_model": "gpt-6.1-sol",
+ "note": "hill-climb arm vs the Codex app server; GRAFF_HC names the switches"
+ },
"codex-app-server": {
"cmd": ["python3", "{repo}/graff-evals/codex_app_server.py", "{model}", "{prompt}"],
"answer": "stdout",
diff --git a/graff-evals/run.py b/graff-evals/run.py
index a4d771b1..3bad96b6 100755
--- a/graff-evals/run.py
+++ b/graff-evals/run.py
@@ -527,6 +527,8 @@ def main():
ap.add_argument("--suite", default="all",
help="core, rlm, swe, mcp, inhouse, live, comma-mix, or all (core+rlm+swe; mcp/inhouse/live are opt-in)")
ap.add_argument("--reps", type=int, default=1)
+ ap.add_argument("--interleave", action="store_true",
+ help="order runs task by task across harnesses, so A/B arms see the same backend conditions")
ap.add_argument("--jobs", "-j", type=int, default=1,
help="parallel task×harness runs (default 1; each has its own sandbox)")
ap.add_argument("--interactive", action="store_true", help="pick a task+harness, watch it live")
@@ -573,6 +575,10 @@ def main():
continue
for rep in range(1, args.reps + 1):
work.append((hname, harness, task, model, rep))
+ if args.interleave:
+ arm_at = {h: i for i, h in enumerate(args.harness.split(","))}
+ task_at = {tid: i for i, tid in enumerate(picked)}
+ work.sort(key=lambda w: (w[4], task_at[w[2]["id"]], arm_at[w[0]]))
jobs = max(1, args.jobs)
print(f"{len(work)} runs · {jobs} worker{'s' if jobs != 1 else ''} · {out_path}", flush=True)
records = []