Skip to content

graff — a fast agentic coding harness in Zig: multi-provider, MCP, workflows, DGM evolution loop, TS/Python SDKs

Python
14.4 KB
Raw Blame
#!/usr/bin/env python3
"""Drive `codex app-server` (JSON-RPC over stdio) for one graff-evals task.

usage: codex_app_server.py <model> <prompt>            (cwd = the task sandbox)
       codex_app_server.py --probe <model>             (model list + rate limits)

Codex runs from a throwaway CODEX_HOME outside the sandbox: an access-only
copy of the ChatGPT login (no refresh token, so an eval can never rotate the
real one), a config.toml with the model (and EVAL_EFFORT, if set), and the sandbox's
`.mcp.json` servers registered as codex `mcp_servers`. Project docs are off
(`project_doc_max_bytes = 0`), as graff sees none in an isolated sandbox.

The final agent message goes to stdout (the runner's `answer: stdout`). Token
usage from `thread/tokenUsage/updated` goes to stderr as graff's `[usage]`
footer, so the runner's graff-stderr parser scores both harnesses alike.

Steering (graff-evals steering scenarios): EVAL_STEER=<text> sends
`turn/steer` once the first agent text or reasoning arrives
(EVAL_STEER_AFTER=text|start); EVAL_FOLLOWUP=<text> runs one more turn after
the first completes. Neither is set for ordinary tasks.
"""
from __future__ import annotations

import json
import os
import queue
import shutil
import subprocess
import sys
import tempfile
import threading
import time
from datetime import datetime, timezone


T0 = time.monotonic()


def trace(ev: dict) -> None:
    """EVAL_TRACE_DIR: one JSONL per sandbox of per-call usage and item sizes
    (what each output token went to), for output and cache analysis."""
    root = os.environ.get("EVAL_TRACE_DIR")
    if not root:
        return
    os.makedirs(root, exist_ok=True)
    ev = dict(ev, t=round((time.monotonic() - T0) * 1000))  # ms since the driver started
    with open(os.path.join(root, os.path.basename(os.getcwd()) + ".jsonl"), "a") as fh:
        fh.write(json.dumps(ev) + "\n")


def item_sizes(item: dict) -> dict:
    kind = item.get("type")
    out = {"ev": "item", "type": kind}
    if kind == "agentMessage":
        text = item.get("text") or ""
        out.update(chars=len(text), phase=item.get("phase"), head=text[:160])
    elif kind == "reasoning":
        out.update(summary_chars=sum(len(x) if isinstance(x, str) else len(json.dumps(x)) for x in (item.get("summary") or [])))
    elif kind == "commandExecution":
        out.update(chars=len(item.get("command") or ""), result_chars=len(item.get("aggregatedOutput") or ""), head=(item.get("command") or "")[:160])
    elif kind == "fileChange":
        out.update(chars=sum(len(json.dumps(c)) for c in (item.get("changes") or [])))
    elif kind in ("mcpToolCall", "dynamicToolCall"):
        out.update(tool=item.get("tool"), chars=len(json.dumps(item.get("arguments"))), result_chars=len(json.dumps(item.get("result") or item.get("contentItems"))))
    elif kind == "collabAgentToolCall":
        out.update(tool=item.get("tool"), chars=len(item.get("prompt") or ""))
    elif kind == "plan":
        out.update(chars=len(item.get("text") or ""))
    return out


def codex_home(model: str, sandbox: str) -> str:
    home = tempfile.mkdtemp(prefix="eval-codex-home-")
    source = os.environ.get("CODEX_HOME") or os.path.expanduser("~/.codex")
    real = json.load(open(os.path.join(source, "auth.json")))
    tokens = real.get("tokens") or {}
    access_only = {
        "OPENAI_API_KEY": None,
        "tokens": {
            "id_token": tokens.get("id_token"),
            "access_token": tokens.get("access_token"),
            "refresh_token": "",
            "account_id": tokens.get("account_id"),
        },
        "last_refresh": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%fZ"),
    }
    with open(os.path.join(home, "auth.json"), "w") as fh:
        json.dump(access_only, fh)
    os.chmod(os.path.join(home, "auth.json"), 0o600)
    lines = [f'model = "{model}"']
    # Unset, the app server runs at its model's catalog default effort, as
    # graff does (ADR 0226). EVAL_EFFORT pins a level.
    if os.environ.get("EVAL_EFFORT"):
        lines.append(f'model_reasoning_effort = "{os.environ["EVAL_EFFORT"]}"')
    lines += [
        'approval_policy = "never"',
        'sandbox_mode = "danger-full-access"',
        "project_doc_max_bytes = 0",
        "check_for_update_on_startup = false",
    ]
    mcp = os.path.join(sandbox, ".mcp.json")
    if os.path.exists(mcp):
        servers = (json.load(open(mcp)).get("mcpServers") or {})
        for name, spec in servers.items():
            args = [a if os.path.isabs(a) or not os.path.exists(os.path.join(sandbox, a)) else os.path.join(sandbox, a)
                    for a in spec.get("args", [])]
            lines += ["", f"[mcp_servers.{name}]", f"command = {json.dumps(spec['command'])}", f"args = {json.dumps(args)}"]
            if spec.get("env"):
                lines.append("env = { " + ", ".join(f"{k} = {json.dumps(v)}" for k, v in spec["env"].items()) + " }")
    with open(os.path.join(home, "config.toml"), "w") as fh:
        fh.write("\n".join(lines) + "\n")
    return home


class AppServer:
    def __init__(self, home: str, cwd: str):
        env = dict(os.environ, CODEX_HOME=home)
        env.pop("OPENAI_API_KEY", None)
        self.p = subprocess.Popen(["codex", "app-server"], cwd=cwd, env=env, stdin=subprocess.PIPE,
                                  stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True, bufsize=1)
        self.inbox: queue.Queue = queue.Queue()
        self.next_id = 0
        threading.Thread(target=self._pump, daemon=True).start()

    def _pump(self):
        for line in self.p.stdout:
            line = line.strip()
            if not line:
                continue
            try:
                self.inbox.put(json.loads(line))
            except ValueError:
                pass
        self.inbox.put(None)

    def send(self, obj: dict):
        self.p.stdin.write(json.dumps(obj) + "\n")
        self.p.stdin.flush()

    def request(self, method: str, params: dict | None, on_event=None, timeout: float = 60.0):
        self.next_id += 1
        rid = self.next_id
        self.send({"id": rid, "method": method, "params": params or {}})
        end = time.monotonic() + timeout
        while time.monotonic() < end:
            msg = self.next(end - time.monotonic())
            if msg is None:
                break
            if msg.get("id") == rid and ("result" in msg or "error" in msg):
                if "error" in msg:
                    raise RuntimeError(f"{method}: {json.dumps(msg['error'])[:300]}")
                return msg["result"]
            if on_event:
                on_event(msg)
        raise TimeoutError(method)

    def next(self, timeout: float):
        try:
            return self.inbox.get(timeout=max(timeout, 0.01))
        except queue.Empty:
            return None

    def answer_server_request(self, msg: dict) -> bool:
        """Approvals should never come with approval_policy=never; accept if they do."""
        if "id" in msg and "method" in msg and "result" not in msg:
            self.send({"id": msg["id"], "result": {"decision": "accept"}})
            return True
        return False

    def close(self):
        try:
            self.p.stdin.close()
            self.p.wait(timeout=5)
        except Exception:
            self.p.kill()


def text_input(text: str) -> list[dict]:
    return [{"type": "text", "text": text, "text_elements": []}]


def summed(totals: dict) -> dict:
    """Every thread's cumulative usage added up: sub-agent threads bill too."""
    out: dict = {}
    for usage in totals.values():
        for key, value in (usage or {}).items():
            if isinstance(value, (int, float)):
                out[key] = out.get(key, 0) + value
    return out


def run_turn(srv: AppServer, thread_id: str, text: str, deadline: float, steer: str = "", steer_after: str = "text",
             totals: dict | None = None):
    """One turn; returns (final agent text, turn status, steer outcome).

    Sub-agents run on their own threads over the same connection. Only the
    root thread's events start, answer or end the turn; every thread's usage
    counts (`totals` maps thread id to its cumulative usage, across turns).
    """
    totals = {} if totals is None else totals
    state = {"turn": None, "done": None, "answer": "", "steered": "", "usage": None, "armed": 0.0, "t0": time.monotonic()}
    delay = float(os.environ.get("EVAL_STEER_DELAY_S", "0"))

    def on_event(msg):
        if srv.answer_server_request(msg):
            return
        method = msg.get("method")
        params = msg.get("params") or {}
        tid = params.get("threadId")
        root = tid is None or tid == thread_id
        if method == "turn/started" and root:
            state["turn"] = (params.get("turn") or {}).get("id") or state["turn"]
        elif method == "item/started" and root and (params.get("item") or {}).get("type") == "agentMessage" and state.get("steer_at") and not state.get("after_item"):
            state["after_item"] = time.monotonic()
        elif method == "item/completed":
            item = params.get("item") or {}
            trace(dict(item_sizes(item), thread="root" if root else "child", tid=(tid or "")[-8:]))
            if root and item.get("type") == "agentMessage":
                state["answer"] = item.get("text") or state["answer"]
        elif method == "thread/tokenUsage/updated":
            totals[tid or thread_id] = (params.get("tokenUsage") or {}).get("total")
            state["usage"] = summed(totals)
            trace({"ev": "usage", "thread": "root" if root else "child", "tid": (tid or "")[-8:], "last": (params.get("tokenUsage") or {}).get("last")})
            state["calls"] = state.get("calls", 0) + 1
            if os.environ.get("EVAL_DEBUG"):
                print(f"[usage-event] {json.dumps(params.get('tokenUsage'))}", file=sys.stderr)
        elif method == "turn/completed" and root:
            state["done"] = params.get("turn") or {}
        if steer and not state["steered"] and state["turn"]:
            trigger = method in ("item/agentMessage/delta", "item/reasoning/summaryTextDelta") if steer_after == "text" else method == "turn/started"
            if trigger and not state["armed"]:
                state["armed"] = time.monotonic()
            if state["armed"] and time.monotonic() - state["armed"] >= delay:
                state["steered"] = "sent"
                state["steer_at"] = time.monotonic()
                try:
                    srv.request("turn/steer", {"threadId": thread_id, "input": text_input(steer), "expectedTurnId": state["turn"]},
                                on_event=on_event, timeout=30)
                    state["steered"] = "accepted"
                except Exception as e:  # noqa: BLE001
                    state["steered"] = f"refused: {e}"[:200]

    result = srv.request("turn/start", {"threadId": thread_id, "input": text_input(text)}, on_event=on_event, timeout=60)
    state["turn"] = state["turn"] or (result.get("turn") or {}).get("id")
    while state["done"] is None and time.monotonic() < deadline:
        msg = srv.next(deadline - time.monotonic())
        if msg is None:
            break
        on_event(msg)
    status = (state["done"] or {}).get("status", "timeout")
    if state.get("steer_at"):
        after = state.get("after_item")
        print(f"[codex-app-server] steer at {state['steer_at'] - state['t0']:.1f}s; next agent message "
              f"{'+' + format(after - state['steer_at'], '.1f') + 's' if after else 'never'}; turn done "
              f"+{time.monotonic() - state['steer_at']:.1f}s", file=sys.stderr)
    if os.environ.get("EVAL_PRINT_TURNS"):
        print(f"--- turn answer ---\n{state['answer']}", file=sys.stderr)
    return state["answer"], status, state["steered"], state["usage"], state.get("calls", 0)


def footer(usage: dict | None, calls: int) -> str:
    """graff's `[usage]` shape; codex's inputTokens, like graff's `in`, includes cached reads."""
    u = usage or {}
    return (f"[usage] {calls} api call(s) · {u.get('inputTokens', 0)} in ({u.get('cachedInputTokens', 0)} cached, "
            f"{u.get('cacheWriteInputTokens', 0)} cache writes) + {u.get('outputTokens', 0)} out tokens")


def main():
    if len(sys.argv) >= 3 and sys.argv[1] == "--probe":
        model, sandbox = sys.argv[2], os.getcwd()
        home = codex_home(model, sandbox)
        srv = AppServer(home, sandbox)
        try:
            srv.request("initialize", {"clientInfo": {"name": "graff-evals", "title": "graff-evals", "version": "0.1"}, "capabilities": None})
            srv.send({"method": "initialized"})
            models = srv.request("model/list", {})
            names = [m.get("model") or m.get("id") for m in (models.get("data") or models.get("models") or [])]
            print("models:", [n for n in names if n and ("gpt-6" in n or "gpt-5.6" in n)])
            limits = srv.request("account/rateLimits/read", {})
            print("rate limits:", json.dumps(limits)[:600])
        finally:
            srv.close()
            shutil.rmtree(home, ignore_errors=True)
        return 0
    model, prompt = sys.argv[1], sys.argv[2]
    sandbox = os.getcwd()
    deadline = time.monotonic() + float(os.environ.get("EVAL_TIMEOUT_S", "900"))
    home = codex_home(model, sandbox)
    srv = AppServer(home, sandbox)
    try:
        srv.request("initialize", {"clientInfo": {"name": "graff-evals", "title": "graff-evals", "version": "0.1"}, "capabilities": None})
        srv.send({"method": "initialized"})
        thread = srv.request("thread/start", {"model": model, "cwd": sandbox, "approvalPolicy": "never",
                                              "sandbox": "danger-full-access", "ephemeral": True}, timeout=60)
        thread_id = (thread.get("thread") or {}).get("id")
        totals: dict = {}
        answer, status, steered, usage, calls = run_turn(srv, thread_id, prompt, deadline, os.environ.get("EVAL_STEER", ""),
                                                         os.environ.get("EVAL_STEER_AFTER", "text"), totals)
        turns = 1
        if os.environ.get("EVAL_FOLLOWUP") and status == "completed":
            answer, status, _, usage2, calls2 = run_turn(srv, thread_id, os.environ["EVAL_FOLLOWUP"], deadline, totals=totals)
            usage, calls, turns = usage2 or usage, calls + calls2, 2
        print(answer)
        print(f"[codex-app-server] status={status} steer={steered or '-'} turns={turns}", file=sys.stderr)
        print(footer(usage, calls), file=sys.stderr)
        return 0 if status == "completed" else 1
    finally:
        srv.close()
        shutil.rmtree(home, ignore_errors=True)


if __name__ == "__main__":
    sys.exit(main())