Skip to content

graff — a fast agentic coding harness in Zig: multi-provider, MCP, workflows, DGM evolution loop, TS/Python SDKs

Python

evals: per-call trace for the Codex app-server arm, GPT-6 list prices, hill-climb arms

justrachad136afd3d6d
diff --git a/graff-evals/codex_app_server.py b/graff-evals/codex_app_server.py
index 93760760..a53e32cc 100755
--- a/graff-evals/codex_app_server.py
+++ b/graff-evals/codex_app_server.py
@@ -33,6 +33,38 @@ import time
from datetime import datetime, timezone
+def trace(ev: dict) -> None:
+ """EVAL_TRACE_DIR: one JSONL per sandbox of per-call usage and item sizes
+ (what each output token went to), for output and cache analysis."""
+ root = os.environ.get("EVAL_TRACE_DIR")
+ if not root:
+ return
+ os.makedirs(root, exist_ok=True)
+ with open(os.path.join(root, os.path.basename(os.getcwd()) + ".jsonl"), "a") as fh:
+ fh.write(json.dumps(ev) + "\n")
+
+
+def item_sizes(item: dict) -> dict:
+ kind = item.get("type")
+ out = {"ev": "item", "type": kind}
+ if kind == "agentMessage":
+ text = item.get("text") or ""
+ out.update(chars=len(text), phase=item.get("phase"), head=text[:160])
+ elif kind == "reasoning":
+ out.update(summary_chars=sum(len(x) if isinstance(x, str) else len(json.dumps(x)) for x in (item.get("summary") or [])))
+ elif kind == "commandExecution":
+ out.update(chars=len(item.get("command") or ""), result_chars=len(item.get("aggregatedOutput") or ""), head=(item.get("command") or "")[:160])
+ elif kind == "fileChange":
+ out.update(chars=sum(len(json.dumps(c)) for c in (item.get("changes") or [])))
+ elif kind in ("mcpToolCall", "dynamicToolCall"):
+ out.update(tool=item.get("tool"), chars=len(json.dumps(item.get("arguments"))), result_chars=len(json.dumps(item.get("result") or item.get("contentItems"))))
+ elif kind == "collabAgentToolCall":
+ out.update(tool=item.get("tool"), chars=len(item.get("prompt") or ""))
+ elif kind == "plan":
+ out.update(chars=len(item.get("text") or ""))
+ return out
+
+
def codex_home(model: str, sandbox: str) -> str:
home = tempfile.mkdtemp(prefix="eval-codex-home-")
source = os.environ.get("CODEX_HOME") or os.path.expanduser("~/.codex")
@@ -156,10 +188,12 @@ def run_turn(srv: AppServer, thread_id: str, text: str, deadline: float, steer:
state["after_item"] = time.monotonic()
elif method == "item/completed":
item = params.get("item") or {}
+ trace(item_sizes(item))
if item.get("type") == "agentMessage":
state["answer"] = item.get("text") or state["answer"]
elif method == "thread/tokenUsage/updated":
state["usage"] = (params.get("tokenUsage") or {}).get("total")
+ trace({"ev": "usage", "last": (params.get("tokenUsage") or {}).get("last")})
state["calls"] = state.get("calls", 0) + 1
if os.environ.get("EVAL_DEBUG"):
print(f"[usage-event] {json.dumps(params.get('tokenUsage'))}", file=sys.stderr)
diff --git a/graff-evals/harnesses.json b/graff-evals/harnesses.json
index 6890452c..5be226e9 100644
--- a/graff-evals/harnesses.json
+++ b/graff-evals/harnesses.json
@@ -316,6 +316,66 @@
"default_model": "gpt-6.1-sol",
"note": "scripted graff repl over the Codex WebSocket: the transport the Codex app server uses"
},
+ "graff-hc-champ": {
+ "cmd": ["{repo}/zig-out/bin/graff", "repl", "--model", "{model}", "--yolo"],
+ "stdin": "{prompt}\n",
+ "answer": "stdout",
+ "usage": "graff-stderr",
+ "env": {"GRAFF_POST_DEADLINE_SECS": "90"},
+ "model_prefix": "codex",
+ "default_model": "gpt-6.1-sol",
+ "note": "hill-climb arm vs the Codex app server; GRAFF_HC names the switches"
+ },
+ "graff-hc-habits": {
+ "cmd": ["{repo}/zig-out/bin/graff", "repl", "--model", "{model}", "--yolo"],
+ "stdin": "{prompt}\n",
+ "answer": "stdout",
+ "usage": "graff-stderr",
+ "env": {"GRAFF_POST_DEADLINE_SECS": "90", "GRAFF_HC": "issue-soft,headsup-short,new-file-no-read"},
+ "model_prefix": "codex",
+ "default_model": "gpt-6.1-sol",
+ "note": "hill-climb arm vs the Codex app server; GRAFF_HC names the switches"
+ },
+ "graff-hc-fold": {
+ "cmd": ["{repo}/zig-out/bin/graff", "repl", "--model", "{model}", "--yolo"],
+ "stdin": "{prompt}\n",
+ "answer": "stdout",
+ "usage": "graff-stderr",
+ "env": {"GRAFF_POST_DEADLINE_SECS": "90", "GRAFF_HC": "fold-more"},
+ "model_prefix": "codex",
+ "default_model": "gpt-6.1-sol",
+ "note": "hill-climb arm vs the Codex app server; GRAFF_HC names the switches"
+ },
+ "graff-hc-userkey": {
+ "cmd": ["{repo}/zig-out/bin/graff", "repl", "--model", "{model}", "--yolo"],
+ "stdin": "{prompt}\n",
+ "answer": "stdout",
+ "usage": "graff-stderr",
+ "env": {"GRAFF_POST_DEADLINE_SECS": "90", "GRAFF_HC": "user-key"},
+ "model_prefix": "codex",
+ "default_model": "gpt-6.1-sol",
+ "note": "hill-climb arm vs the Codex app server; GRAFF_HC names the switches"
+ },
+ "graff-hc-userkey-habits": {
+ "cmd": ["{repo}/zig-out/bin/graff", "repl", "--model", "{model}", "--yolo"],
+ "stdin": "{prompt}\n",
+ "answer": "stdout",
+ "usage": "graff-stderr",
+ "env": {"GRAFF_POST_DEADLINE_SECS": "90", "GRAFF_HC": "user-key,issue-soft,headsup-short,new-file-no-read"},
+ "model_prefix": "codex",
+ "default_model": "gpt-6.1-sol",
+ "note": "hill-climb arm vs the Codex app server; GRAFF_HC names the switches"
+ },
+ "graff-hc-userkey-fold": {
+ "cmd": ["{repo}/zig-out/bin/graff", "repl", "--model", "{model}", "--yolo"],
+ "stdin": "{prompt}\n",
+ "answer": "stdout",
+ "usage": "graff-stderr",
+ "env": {"GRAFF_POST_DEADLINE_SECS": "90", "GRAFF_HC": "user-key,fold-more"},
+ "model_prefix": "codex",
+ "default_model": "gpt-6.1-sol",
+ "note": "hill-climb arm vs the Codex app server; GRAFF_HC names the switches"
+ },
"codex-app-server": {
"cmd": ["python3", "{repo}/graff-evals/codex_app_server.py", "{model}", "{prompt}"],
"answer": "stdout",
diff --git a/graff-evals/list_price.py b/graff-evals/list_price.py
index e8eebc32..e0ab3cf2 100644
--- a/graff-evals/list_price.py
+++ b/graff-evals/list_price.py
@@ -66,6 +66,9 @@ RATES = {
"high_cache": 0.075,
"high_out": 3.75,
},
+ # graff src/pricing_table.zig (OpenAI GPT-6 list prices per 1M; Codex route)
+ "gpt-6-sol": {"high_at": 272_000, "in": 2.0, "cache": 0.2, "out": 10.0, "high_in": 4.0, "high_cache": 0.4, "high_out": 15.0},
+ "gpt-6.1-sol": {"high_at": 272_000, "in": 2.0, "cache": 0.1, "out": 10.0, "high_in": 4.0, "high_cache": 0.2, "high_out": 15.0},
# platform.kimi.ai/docs/pricing/chat-k3 — cache-hit / cache-miss / output
"k3": {
"high_at": 0,