Skip to content

graff — a fast agentic coding harness in Zig: multi-provider, MCP, workflows, DGM evolution loop, TS/Python SDKs

Python

evals: drop the hill-climb arms; they need an experiment build

justrach102f0544c151
diff --git a/graff-evals/harnesses.json b/graff-evals/harnesses.json
index c8813255..6890452c 100644
--- a/graff-evals/harnesses.json
+++ b/graff-evals/harnesses.json
@@ -316,116 +316,6 @@
"default_model": "gpt-6.1-sol",
"note": "scripted graff repl over the Codex WebSocket: the transport the Codex app server uses"
},
- "graff-hc-champ": {
- "cmd": ["{repo}/zig-out/bin/graff", "repl", "--model", "{model}", "--yolo"],
- "stdin": "{prompt}\n",
- "answer": "stdout",
- "usage": "graff-stderr",
- "env": {"GRAFF_POST_DEADLINE_SECS": "90"},
- "model_prefix": "codex",
- "default_model": "gpt-6.1-sol",
- "note": "hill-climb arm vs the Codex app server; GRAFF_HC names the switches"
- },
- "graff-hc-habits": {
- "cmd": ["{repo}/zig-out/bin/graff", "repl", "--model", "{model}", "--yolo"],
- "stdin": "{prompt}\n",
- "answer": "stdout",
- "usage": "graff-stderr",
- "env": {"GRAFF_POST_DEADLINE_SECS": "90", "GRAFF_HC": "issue-soft,headsup-short,new-file-no-read"},
- "model_prefix": "codex",
- "default_model": "gpt-6.1-sol",
- "note": "hill-climb arm vs the Codex app server; GRAFF_HC names the switches"
- },
- "graff-hc-fold": {
- "cmd": ["{repo}/zig-out/bin/graff", "repl", "--model", "{model}", "--yolo"],
- "stdin": "{prompt}\n",
- "answer": "stdout",
- "usage": "graff-stderr",
- "env": {"GRAFF_POST_DEADLINE_SECS": "90", "GRAFF_HC": "fold-more"},
- "model_prefix": "codex",
- "default_model": "gpt-6.1-sol",
- "note": "hill-climb arm vs the Codex app server; GRAFF_HC names the switches"
- },
- "graff-hc-userkey": {
- "cmd": ["{repo}/zig-out/bin/graff", "repl", "--model", "{model}", "--yolo"],
- "stdin": "{prompt}\n",
- "answer": "stdout",
- "usage": "graff-stderr",
- "env": {"GRAFF_POST_DEADLINE_SECS": "90", "GRAFF_HC": "user-key"},
- "model_prefix": "codex",
- "default_model": "gpt-6.1-sol",
- "note": "hill-climb arm vs the Codex app server; GRAFF_HC names the switches"
- },
- "graff-hc-userkey-habits": {
- "cmd": ["{repo}/zig-out/bin/graff", "repl", "--model", "{model}", "--yolo"],
- "stdin": "{prompt}\n",
- "answer": "stdout",
- "usage": "graff-stderr",
- "env": {"GRAFF_POST_DEADLINE_SECS": "90", "GRAFF_HC": "user-key,issue-soft,headsup-short,new-file-no-read"},
- "model_prefix": "codex",
- "default_model": "gpt-6.1-sol",
- "note": "hill-climb arm vs the Codex app server; GRAFF_HC names the switches"
- },
- "graff-hc-userkey-fold": {
- "cmd": ["{repo}/zig-out/bin/graff", "repl", "--model", "{model}", "--yolo"],
- "stdin": "{prompt}\n",
- "answer": "stdout",
- "usage": "graff-stderr",
- "env": {"GRAFF_POST_DEADLINE_SECS": "90", "GRAFF_HC": "user-key,fold-more"},
- "model_prefix": "codex",
- "default_model": "gpt-6.1-sol",
- "note": "hill-climb arm vs the Codex app server; GRAFF_HC names the switches"
- },
- "graff-hc-r4-effort": {
- "cmd": ["{repo}/zig-out/bin/graff", "repl", "--model", "{model}", "--yolo"],
- "stdin": "{prompt}\n",
- "answer": "stdout",
- "usage": "graff-stderr",
- "env": {"GRAFF_POST_DEADLINE_SECS": "90", "GRAFF_HC": "user-key,fold-more,effort-low"},
- "model_prefix": "codex",
- "default_model": "gpt-6.1-sol",
- "note": "hill-climb arm vs the Codex app server; GRAFF_HC names the switches"
- },
- "graff-hc-r4-verb": {
- "cmd": ["{repo}/zig-out/bin/graff", "repl", "--model", "{model}", "--yolo"],
- "stdin": "{prompt}\n",
- "answer": "stdout",
- "usage": "graff-stderr",
- "env": {"GRAFF_POST_DEADLINE_SECS": "90", "GRAFF_HC": "user-key,fold-more,verbosity-low"},
- "model_prefix": "codex",
- "default_model": "gpt-6.1-sol",
- "note": "hill-climb arm vs the Codex app server; GRAFF_HC names the switches"
- },
- "graff-hc-r4-both": {
- "cmd": ["{repo}/zig-out/bin/graff", "repl", "--model", "{model}", "--yolo"],
- "stdin": "{prompt}\n",
- "answer": "stdout",
- "usage": "graff-stderr",
- "env": {"GRAFF_POST_DEADLINE_SECS": "90", "GRAFF_HC": "user-key,fold-more,effort-low,verbosity-low"},
- "model_prefix": "codex",
- "default_model": "gpt-6.1-sol",
- "note": "hill-climb arm vs the Codex app server; GRAFF_HC names the switches"
- },
- "graff-hc-r5-slimnote": {
- "cmd": ["{repo}/zig-out/bin/graff", "repl", "--model", "{model}", "--yolo"],
- "stdin": "{prompt}\n",
- "answer": "stdout",
- "usage": "graff-stderr",
- "env": {"GRAFF_POST_DEADLINE_SECS": "90", "GRAFF_HC": "user-key,fold-more,slim-note"},
- "model_prefix": "codex",
- "default_model": "gpt-6.1-sol",
- "note": "hill-climb arm vs the Codex app server; GRAFF_HC names the switches"
- },
- "graff-hc-r5-ids": {
- "cmd": ["{repo}/zig-out/bin/graff", "repl", "--model", "{model}", "--yolo"],
- "stdin": "{prompt}\n",
- "answer": "stdout",
- "usage": "graff-stderr",
- "env": {"GRAFF_POST_DEADLINE_SECS": "90", "GRAFF_HC": "user-key,fold-more,codex-ids"},
- "model_prefix": "codex",
- "default_model": "gpt-6.1-sol",
- "note": "hill-climb arm vs the Codex app server; GRAFF_HC names the switches"
- },
"codex-app-server": {
"cmd": ["python3", "{repo}/graff-evals/codex_app_server.py", "{model}", "{prompt}"],
"answer": "stdout",