graff — a fast agentic coding harness in Zig: multi-provider, MCP, workflows, DGM evolution loop, TS/Python SDKs
Python
evals: per-call trace for the Codex app-server arm, GPT-6 list prices, hill-climb arms
justrachad136afd3d6d
diff --git a/graff-evals/codex_app_server.py b/graff-evals/codex_app_server.pyindex 93760760..a53e32cc 100755--- a/graff-evals/codex_app_server.py+++ b/graff-evals/codex_app_server.py@@ -33,6 +33,38 @@ import timefrom datetime import datetime, timezone+def trace(ev: dict) -> None:+ """EVAL_TRACE_DIR: one JSONL per sandbox of per-call usage and item sizes+ (what each output token went to), for output and cache analysis."""+ root = os.environ.get("EVAL_TRACE_DIR")+ if not root:+ return+ os.makedirs(root, exist_ok=True)+ with open(os.path.join(root, os.path.basename(os.getcwd()) + ".jsonl"), "a") as fh:+ fh.write(json.dumps(ev) + "\n")+++def item_sizes(item: dict) -> dict:+ kind = item.get("type")+ out = {"ev": "item", "type": kind}+ if kind == "agentMessage":+ text = item.get("text") or ""+ out.update(chars=len(text), phase=item.get("phase"), head=text[:160])+ elif kind == "reasoning":+ out.update(summary_chars=sum(len(x) if isinstance(x, str) else len(json.dumps(x)) for x in (item.get("summary") or [])))+ elif kind == "commandExecution":+ out.update(chars=len(item.get("command") or ""), result_chars=len(item.get("aggregatedOutput") or ""), head=(item.get("command") or "")[:160])+ elif kind == "fileChange":+ out.update(chars=sum(len(json.dumps(c)) for c in (item.get("changes") or [])))+ elif kind in ("mcpToolCall", "dynamicToolCall"):+ out.update(tool=item.get("tool"), chars=len(json.dumps(item.get("arguments"))), result_chars=len(json.dumps(item.get("result") or item.get("contentItems"))))+ elif kind == "collabAgentToolCall":+ out.update(tool=item.get("tool"), chars=len(item.get("prompt") or ""))+ elif kind == "plan":+ out.update(chars=len(item.get("text") or ""))+ return out++def codex_home(model: str, sandbox: str) -> str:home = tempfile.mkdtemp(prefix="eval-codex-home-")source = os.environ.get("CODEX_HOME") or os.path.expanduser("~/.codex")@@ -156,10 +188,12 @@ def run_turn(srv: AppServer, thread_id: str, text: str, deadline: float, steer:state["after_item"] = time.monotonic()elif method == "item/completed":item = params.get("item") or {}+ trace(item_sizes(item))if item.get("type") == "agentMessage":state["answer"] = item.get("text") or state["answer"]elif method == "thread/tokenUsage/updated":state["usage"] = (params.get("tokenUsage") or {}).get("total")+ trace({"ev": "usage", "last": (params.get("tokenUsage") or {}).get("last")})state["calls"] = state.get("calls", 0) + 1if os.environ.get("EVAL_DEBUG"):print(f"[usage-event] {json.dumps(params.get('tokenUsage'))}", file=sys.stderr)diff --git a/graff-evals/harnesses.json b/graff-evals/harnesses.jsonindex 6890452c..5be226e9 100644--- a/graff-evals/harnesses.json+++ b/graff-evals/harnesses.json@@ -316,6 +316,66 @@"default_model": "gpt-6.1-sol","note": "scripted graff repl over the Codex WebSocket: the transport the Codex app server uses"},+ "graff-hc-champ": {+ "cmd": ["{repo}/zig-out/bin/graff", "repl", "--model", "{model}", "--yolo"],+ "stdin": "{prompt}\n",+ "answer": "stdout",+ "usage": "graff-stderr",+ "env": {"GRAFF_POST_DEADLINE_SECS": "90"},+ "model_prefix": "codex",+ "default_model": "gpt-6.1-sol",+ "note": "hill-climb arm vs the Codex app server; GRAFF_HC names the switches"+ },+ "graff-hc-habits": {+ "cmd": ["{repo}/zig-out/bin/graff", "repl", "--model", "{model}", "--yolo"],+ "stdin": "{prompt}\n",+ "answer": "stdout",+ "usage": "graff-stderr",+ "env": {"GRAFF_POST_DEADLINE_SECS": "90", "GRAFF_HC": "issue-soft,headsup-short,new-file-no-read"},+ "model_prefix": "codex",+ "default_model": "gpt-6.1-sol",+ "note": "hill-climb arm vs the Codex app server; GRAFF_HC names the switches"+ },+ "graff-hc-fold": {+ "cmd": ["{repo}/zig-out/bin/graff", "repl", "--model", "{model}", "--yolo"],+ "stdin": "{prompt}\n",+ "answer": "stdout",+ "usage": "graff-stderr",+ "env": {"GRAFF_POST_DEADLINE_SECS": "90", "GRAFF_HC": "fold-more"},+ "model_prefix": "codex",+ "default_model": "gpt-6.1-sol",+ "note": "hill-climb arm vs the Codex app server; GRAFF_HC names the switches"+ },+ "graff-hc-userkey": {+ "cmd": ["{repo}/zig-out/bin/graff", "repl", "--model", "{model}", "--yolo"],+ "stdin": "{prompt}\n",+ "answer": "stdout",+ "usage": "graff-stderr",+ "env": {"GRAFF_POST_DEADLINE_SECS": "90", "GRAFF_HC": "user-key"},+ "model_prefix": "codex",+ "default_model": "gpt-6.1-sol",+ "note": "hill-climb arm vs the Codex app server; GRAFF_HC names the switches"+ },+ "graff-hc-userkey-habits": {+ "cmd": ["{repo}/zig-out/bin/graff", "repl", "--model", "{model}", "--yolo"],+ "stdin": "{prompt}\n",+ "answer": "stdout",+ "usage": "graff-stderr",+ "env": {"GRAFF_POST_DEADLINE_SECS": "90", "GRAFF_HC": "user-key,issue-soft,headsup-short,new-file-no-read"},+ "model_prefix": "codex",+ "default_model": "gpt-6.1-sol",+ "note": "hill-climb arm vs the Codex app server; GRAFF_HC names the switches"+ },+ "graff-hc-userkey-fold": {+ "cmd": ["{repo}/zig-out/bin/graff", "repl", "--model", "{model}", "--yolo"],+ "stdin": "{prompt}\n",+ "answer": "stdout",+ "usage": "graff-stderr",+ "env": {"GRAFF_POST_DEADLINE_SECS": "90", "GRAFF_HC": "user-key,fold-more"},+ "model_prefix": "codex",+ "default_model": "gpt-6.1-sol",+ "note": "hill-climb arm vs the Codex app server; GRAFF_HC names the switches"+ },"codex-app-server": {"cmd": ["python3", "{repo}/graff-evals/codex_app_server.py", "{model}", "{prompt}"],"answer": "stdout",diff --git a/graff-evals/list_price.py b/graff-evals/list_price.pyindex e8eebc32..e0ab3cf2 100644--- a/graff-evals/list_price.py+++ b/graff-evals/list_price.py@@ -66,6 +66,9 @@ RATES = {"high_cache": 0.075,"high_out": 3.75,},+ # graff src/pricing_table.zig (OpenAI GPT-6 list prices per 1M; Codex route)+ "gpt-6-sol": {"high_at": 272_000, "in": 2.0, "cache": 0.2, "out": 10.0, "high_in": 4.0, "high_cache": 0.4, "high_out": 15.0},+ "gpt-6.1-sol": {"high_at": 272_000, "in": 2.0, "cache": 0.1, "out": 10.0, "high_in": 4.0, "high_cache": 0.2, "high_out": 15.0},# platform.kimi.ai/docs/pricing/chat-k3 — cache-hit / cache-miss / output"k3": {"high_at": 0,