graff — a fast agentic coding harness in Zig: multi-provider, MCP, workflows, DGM evolution loop, TS/Python SDKs
Python
evals: drive the Codex app server as a harness
justrach7ed0d7667185
diff --git a/graff-evals/README.md b/graff-evals/README.mdindex abea61c4..b9a66cb4 100644--- a/graff-evals/README.md+++ b/graff-evals/README.md@@ -48,6 +48,11 @@ CODEGRAFF_API_KEY=cg_sk_… ./run.py --suite swe --harness graff-dev,opencode-co./run.py --suite live --harness graff-dev --reps 3 -j 1# same SuperGrok seat, other harnesses (OpenCode / Pi / exo local-process):./run.py --suite live --harness opencode,pi-xai,exo --reps 3 -j 1+# ChatGPT login, 1:1 with the Codex app server (needs `codex`; the driver copies the+# login without its refresh token). Keep --output-root outside any repo so neither+# harness reads repo docs; the mcp suite then needs scripts/linear_fixture_mcp.py+# copied next to that root (the setup looks for it in a parent directory).+./run.py --suite core,mcp --harness graff-codex-repl,codex-app-server --model gpt-6.1-sol -j 6```## Run itdiff --git a/graff-evals/codex_app_server.py b/graff-evals/codex_app_server.pynew file mode 100755index 00000000..93760760--- /dev/null+++ b/graff-evals/codex_app_server.py@@ -0,0 +1,251 @@+#!/usr/bin/env python3+"""Drive `codex app-server` (JSON-RPC over stdio) for one graff-evals task.++usage: codex_app_server.py <model> <prompt> (cwd = the task sandbox)+ codex_app_server.py --probe <model> (model list + rate limits)++Codex runs from a throwaway CODEX_HOME outside the sandbox: an access-only+copy of the ChatGPT login (no refresh token, so an eval can never rotate the+real one), a config.toml with the model and effort, and the sandbox's+`.mcp.json` servers registered as codex `mcp_servers`. Project docs are off+(`project_doc_max_bytes = 0`), as graff sees none in an isolated sandbox.++The final agent message goes to stdout (the runner's `answer: stdout`). Token+usage from `thread/tokenUsage/updated` goes to stderr as graff's `[usage]`+footer, so the runner's graff-stderr parser scores both harnesses alike.++Steering (graff-evals steering scenarios): EVAL_STEER=<text> sends+`turn/steer` once the first agent text or reasoning arrives+(EVAL_STEER_AFTER=text|start); EVAL_FOLLOWUP=<text> runs one more turn after+the first completes. Neither is set for ordinary tasks.+"""+from __future__ import annotations++import json+import os+import queue+import shutil+import subprocess+import sys+import tempfile+import threading+import time+from datetime import datetime, timezone+++def codex_home(model: str, sandbox: str) -> str:+ home = tempfile.mkdtemp(prefix="eval-codex-home-")+ source = os.environ.get("CODEX_HOME") or os.path.expanduser("~/.codex")+ real = json.load(open(os.path.join(source, "auth.json")))+ tokens = real.get("tokens") or {}+ access_only = {+ "OPENAI_API_KEY": None,+ "tokens": {+ "id_token": tokens.get("id_token"),+ "access_token": tokens.get("access_token"),+ "refresh_token": "",+ "account_id": tokens.get("account_id"),+ },+ "last_refresh": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%fZ"),+ }+ with open(os.path.join(home, "auth.json"), "w") as fh:+ json.dump(access_only, fh)+ os.chmod(os.path.join(home, "auth.json"), 0o600)+ lines = [+ f'model = "{model}"',+ f'model_reasoning_effort = "{os.environ.get("EVAL_EFFORT", "medium")}"',+ 'approval_policy = "never"',+ 'sandbox_mode = "danger-full-access"',+ "project_doc_max_bytes = 0",+ "check_for_update_on_startup = false",+ ]+ mcp = os.path.join(sandbox, ".mcp.json")+ if os.path.exists(mcp):+ servers = (json.load(open(mcp)).get("mcpServers") or {})+ for name, spec in servers.items():+ args = [a if os.path.isabs(a) or not os.path.exists(os.path.join(sandbox, a)) else os.path.join(sandbox, a)+ for a in spec.get("args", [])]+ lines += ["", f"[mcp_servers.{name}]", f"command = {json.dumps(spec['command'])}", f"args = {json.dumps(args)}"]+ if spec.get("env"):+ lines.append("env = { " + ", ".join(f"{k} = {json.dumps(v)}" for k, v in spec["env"].items()) + " }")+ with open(os.path.join(home, "config.toml"), "w") as fh:+ fh.write("\n".join(lines) + "\n")+ return home+++class AppServer:+ def __init__(self, home: str, cwd: str):+ env = dict(os.environ, CODEX_HOME=home)+ env.pop("OPENAI_API_KEY", None)+ self.p = subprocess.Popen(["codex", "app-server"], cwd=cwd, env=env, stdin=subprocess.PIPE,+ stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True, bufsize=1)+ self.inbox: queue.Queue = queue.Queue()+ self.next_id = 0+ threading.Thread(target=self._pump, daemon=True).start()++ def _pump(self):+ for line in self.p.stdout:+ line = line.strip()+ if not line:+ continue+ try:+ self.inbox.put(json.loads(line))+ except ValueError:+ pass+ self.inbox.put(None)++ def send(self, obj: dict):+ self.p.stdin.write(json.dumps(obj) + "\n")+ self.p.stdin.flush()++ def request(self, method: str, params: dict | None, on_event=None, timeout: float = 60.0):+ self.next_id += 1+ rid = self.next_id+ self.send({"id": rid, "method": method, "params": params or {}})+ end = time.monotonic() + timeout+ while time.monotonic() < end:+ msg = self.next(end - time.monotonic())+ if msg is None:+ break+ if msg.get("id") == rid and ("result" in msg or "error" in msg):+ if "error" in msg:+ raise RuntimeError(f"{method}: {json.dumps(msg['error'])[:300]}")+ return msg["result"]+ if on_event:+ on_event(msg)+ raise TimeoutError(method)++ def next(self, timeout: float):+ try:+ return self.inbox.get(timeout=max(timeout, 0.01))+ except queue.Empty:+ return None++ def answer_server_request(self, msg: dict) -> bool:+ """Approvals should never come with approval_policy=never; accept if they do."""+ if "id" in msg and "method" in msg and "result" not in msg:+ self.send({"id": msg["id"], "result": {"decision": "accept"}})+ return True+ return False++ def close(self):+ try:+ self.p.stdin.close()+ self.p.wait(timeout=5)+ except Exception:+ self.p.kill()+++def text_input(text: str) -> list[dict]:+ return [{"type": "text", "text": text, "text_elements": []}]+++def run_turn(srv: AppServer, thread_id: str, text: str, deadline: float, steer: str = "", steer_after: str = "text"):+ """One turn; returns (final agent text, turn status, steer outcome)."""+ state = {"turn": None, "done": None, "answer": "", "steered": "", "usage": None, "armed": 0.0, "t0": time.monotonic()}+ delay = float(os.environ.get("EVAL_STEER_DELAY_S", "0"))++ def on_event(msg):+ if srv.answer_server_request(msg):+ return+ method = msg.get("method")+ params = msg.get("params") or {}+ if method == "turn/started":+ state["turn"] = (params.get("turn") or {}).get("id") or state["turn"]+ elif method == "item/started" and (params.get("item") or {}).get("type") == "agentMessage" and state.get("steer_at") and not state.get("after_item"):+ state["after_item"] = time.monotonic()+ elif method == "item/completed":+ item = params.get("item") or {}+ if item.get("type") == "agentMessage":+ state["answer"] = item.get("text") or state["answer"]+ elif method == "thread/tokenUsage/updated":+ state["usage"] = (params.get("tokenUsage") or {}).get("total")+ state["calls"] = state.get("calls", 0) + 1+ if os.environ.get("EVAL_DEBUG"):+ print(f"[usage-event] {json.dumps(params.get('tokenUsage'))}", file=sys.stderr)+ elif method == "turn/completed":+ state["done"] = params.get("turn") or {}+ if steer and not state["steered"] and state["turn"]:+ trigger = method in ("item/agentMessage/delta", "item/reasoning/summaryTextDelta") if steer_after == "text" else method == "turn/started"+ if trigger and not state["armed"]:+ state["armed"] = time.monotonic()+ if state["armed"] and time.monotonic() - state["armed"] >= delay:+ state["steered"] = "sent"+ state["steer_at"] = time.monotonic()+ try:+ srv.request("turn/steer", {"threadId": thread_id, "input": text_input(steer), "expectedTurnId": state["turn"]},+ on_event=on_event, timeout=30)+ state["steered"] = "accepted"+ except Exception as e: # noqa: BLE001+ state["steered"] = f"refused: {e}"[:200]++ result = srv.request("turn/start", {"threadId": thread_id, "input": text_input(text)}, on_event=on_event, timeout=60)+ state["turn"] = state["turn"] or (result.get("turn") or {}).get("id")+ while state["done"] is None and time.monotonic() < deadline:+ msg = srv.next(deadline - time.monotonic())+ if msg is None:+ break+ on_event(msg)+ status = (state["done"] or {}).get("status", "timeout")+ if state.get("steer_at"):+ after = state.get("after_item")+ print(f"[codex-app-server] steer at {state['steer_at'] - state['t0']:.1f}s; next agent message "+ f"{'+' + format(after - state['steer_at'], '.1f') + 's' if after else 'never'}; turn done "+ f"+{time.monotonic() - state['steer_at']:.1f}s", file=sys.stderr)+ if os.environ.get("EVAL_PRINT_TURNS"):+ print(f"--- turn answer ---\n{state['answer']}", file=sys.stderr)+ return state["answer"], status, state["steered"], state["usage"], state.get("calls", 0)+++def footer(usage: dict | None, calls: int) -> str:+ """graff's `[usage]` shape; codex's inputTokens, like graff's `in`, includes cached reads."""+ u = usage or {}+ return (f"[usage] {calls} api call(s) · {u.get('inputTokens', 0)} in ({u.get('cachedInputTokens', 0)} cached, "+ f"{u.get('cacheWriteInputTokens', 0)} cache writes) + {u.get('outputTokens', 0)} out tokens")+++def main():+ if len(sys.argv) >= 3 and sys.argv[1] == "--probe":+ model, sandbox = sys.argv[2], os.getcwd()+ home = codex_home(model, sandbox)+ srv = AppServer(home, sandbox)+ try:+ srv.request("initialize", {"clientInfo": {"name": "graff-evals", "title": "graff-evals", "version": "0.1"}, "capabilities": None})+ srv.send({"method": "initialized"})+ models = srv.request("model/list", {})+ names = [m.get("model") or m.get("id") for m in (models.get("data") or models.get("models") or [])]+ print("models:", [n for n in names if n and ("gpt-6" in n or "gpt-5.6" in n)])+ limits = srv.request("account/rateLimits/read", {})+ print("rate limits:", json.dumps(limits)[:600])+ finally:+ srv.close()+ shutil.rmtree(home, ignore_errors=True)+ return 0+ model, prompt = sys.argv[1], sys.argv[2]+ sandbox = os.getcwd()+ deadline = time.monotonic() + float(os.environ.get("EVAL_TIMEOUT_S", "900"))+ home = codex_home(model, sandbox)+ srv = AppServer(home, sandbox)+ try:+ srv.request("initialize", {"clientInfo": {"name": "graff-evals", "title": "graff-evals", "version": "0.1"}, "capabilities": None})+ srv.send({"method": "initialized"})+ thread = srv.request("thread/start", {"model": model, "cwd": sandbox, "approvalPolicy": "never",+ "sandbox": "danger-full-access", "ephemeral": True}, timeout=60)+ thread_id = (thread.get("thread") or {}).get("id")+ answer, status, steered, usage, calls = run_turn(srv, thread_id, prompt, deadline, os.environ.get("EVAL_STEER", ""),+ os.environ.get("EVAL_STEER_AFTER", "text"))+ turns = 1+ if os.environ.get("EVAL_FOLLOWUP") and status == "completed":+ answer, status, _, usage2, calls2 = run_turn(srv, thread_id, os.environ["EVAL_FOLLOWUP"], deadline)+ usage, calls, turns = usage2 or usage, calls + calls2, 2+ print(answer)+ print(f"[codex-app-server] status={status} steer={steered or '-'} turns={turns}", file=sys.stderr)+ print(footer(usage, calls), file=sys.stderr)+ return 0 if status == "completed" else 1+ finally:+ srv.close()+ shutil.rmtree(home, ignore_errors=True)+++if __name__ == "__main__":+ sys.exit(main())diff --git a/graff-evals/harnesses.json b/graff-evals/harnesses.jsonindex ae906c0e..6890452c 100644--- a/graff-evals/harnesses.json+++ b/graff-evals/harnesses.json@@ -298,5 +298,28 @@"env": {"GRAFF_POST_DEADLINE_SECS": "90"},"default_model": "gpt-6-sol","note": "GPT-6 prompting guide candidate (g6-plain-prose); judged by hillclimb.py round on held-out tasks before any prompt change ships (ADR 0214)"+ },+ "graff-codex": {+ "cmd": ["{repo}/zig-out/bin/graff", "-p", "{prompt}", "--model", "{model}", "--yolo", "--no-lean"],+ "answer": "stdout",+ "usage": "graff-stderr",+ "model_prefix": "codex",+ "default_model": "gpt-6.1-sol"+ },+ "graff-codex-repl": {+ "cmd": ["{repo}/zig-out/bin/graff", "repl", "--model", "{model}", "--yolo"],+ "stdin": "{prompt}\n",+ "answer": "stdout",+ "usage": "graff-stderr",+ "env": {"GRAFF_POST_DEADLINE_SECS": "90"},+ "model_prefix": "codex",+ "default_model": "gpt-6.1-sol",+ "note": "scripted graff repl over the Codex WebSocket: the transport the Codex app server uses"+ },+ "codex-app-server": {+ "cmd": ["python3", "{repo}/graff-evals/codex_app_server.py", "{model}", "{prompt}"],+ "answer": "stdout",+ "usage": "graff-stderr",+ "default_model": "gpt-6.1-sol"}}