graff — a fast agentic coding harness in Zig: multi-provider, MCP, workflows, DGM evolution loop, TS/Python SDKs
Python
graff-evals: the app-server driver follows the root thread
justrach98c0ff9c49c1
diff --git a/graff-evals/codex_app_server.py b/graff-evals/codex_app_server.pyindex a53e32cc..e67c1e6a 100755--- a/graff-evals/codex_app_server.py+++ b/graff-evals/codex_app_server.py@@ -172,8 +172,25 @@ def text_input(text: str) -> list[dict]:return [{"type": "text", "text": text, "text_elements": []}]-def run_turn(srv: AppServer, thread_id: str, text: str, deadline: float, steer: str = "", steer_after: str = "text"):- """One turn; returns (final agent text, turn status, steer outcome)."""+def summed(totals: dict) -> dict:+ """Every thread's cumulative usage added up: sub-agent threads bill too."""+ out: dict = {}+ for usage in totals.values():+ for key, value in (usage or {}).items():+ if isinstance(value, (int, float)):+ out[key] = out.get(key, 0) + value+ return out+++def run_turn(srv: AppServer, thread_id: str, text: str, deadline: float, steer: str = "", steer_after: str = "text",+ totals: dict | None = None):+ """One turn; returns (final agent text, turn status, steer outcome).++ Sub-agents run on their own threads over the same connection. Only the+ root thread's events start, answer or end the turn; every thread's usage+ counts (`totals` maps thread id to its cumulative usage, across turns).+ """+ totals = {} if totals is None else totalsstate = {"turn": None, "done": None, "answer": "", "steered": "", "usage": None, "armed": 0.0, "t0": time.monotonic()}delay = float(os.environ.get("EVAL_STEER_DELAY_S", "0"))@@ -182,22 +199,25 @@ def run_turn(srv: AppServer, thread_id: str, text: str, deadline: float, steer:returnmethod = msg.get("method")params = msg.get("params") or {}- if method == "turn/started":+ tid = params.get("threadId")+ root = tid is None or tid == thread_id+ if method == "turn/started" and root:state["turn"] = (params.get("turn") or {}).get("id") or state["turn"]- elif method == "item/started" and (params.get("item") or {}).get("type") == "agentMessage" and state.get("steer_at") and not state.get("after_item"):+ elif method == "item/started" and root and (params.get("item") or {}).get("type") == "agentMessage" and state.get("steer_at") and not state.get("after_item"):state["after_item"] = time.monotonic()elif method == "item/completed":item = params.get("item") or {}- trace(item_sizes(item))- if item.get("type") == "agentMessage":+ trace(dict(item_sizes(item), thread="root" if root else "child"))+ if root and item.get("type") == "agentMessage":state["answer"] = item.get("text") or state["answer"]elif method == "thread/tokenUsage/updated":- state["usage"] = (params.get("tokenUsage") or {}).get("total")- trace({"ev": "usage", "last": (params.get("tokenUsage") or {}).get("last")})+ totals[tid or thread_id] = (params.get("tokenUsage") or {}).get("total")+ state["usage"] = summed(totals)+ trace({"ev": "usage", "thread": "root" if root else "child", "last": (params.get("tokenUsage") or {}).get("last")})state["calls"] = state.get("calls", 0) + 1if os.environ.get("EVAL_DEBUG"):print(f"[usage-event] {json.dumps(params.get('tokenUsage'))}", file=sys.stderr)- elif method == "turn/completed":+ elif method == "turn/completed" and root:state["done"] = params.get("turn") or {}if steer and not state["steered"] and state["turn"]:trigger = method in ("item/agentMessage/delta", "item/reasoning/summaryTextDelta") if steer_after == "text" else method == "turn/started"@@ -266,11 +286,12 @@ def main():thread = srv.request("thread/start", {"model": model, "cwd": sandbox, "approvalPolicy": "never","sandbox": "danger-full-access", "ephemeral": True}, timeout=60)thread_id = (thread.get("thread") or {}).get("id")+ totals: dict = {}answer, status, steered, usage, calls = run_turn(srv, thread_id, prompt, deadline, os.environ.get("EVAL_STEER", ""),- os.environ.get("EVAL_STEER_AFTER", "text"))+ os.environ.get("EVAL_STEER_AFTER", "text"), totals)turns = 1if os.environ.get("EVAL_FOLLOWUP") and status == "completed":- answer, status, _, usage2, calls2 = run_turn(srv, thread_id, os.environ["EVAL_FOLLOWUP"], deadline)+ answer, status, _, usage2, calls2 = run_turn(srv, thread_id, os.environ["EVAL_FOLLOWUP"], deadline, totals=totals)usage, calls, turns = usage2 or usage, calls + calls2, 2print(answer)print(f"[codex-app-server] status={status} steer={steered or '-'} turns={turns}", file=sys.stderr)diff --git a/graff-evals/test_measurement.py b/graff-evals/test_measurement.pyindex d465ea7f..f564cc50 100644--- a/graff-evals/test_measurement.py+++ b/graff-evals/test_measurement.py@@ -6,9 +6,11 @@ import osfrom pathlib import Pathimport sysimport tempfile+import timeimport unittestfrom unittest.mock import patch+import codex_app_serverimport list_priceimport measurementimport request_capture@@ -442,5 +444,42 @@ class MeasurementTests(unittest.TestCase):self.assertEqual((Path(sandboxes) / '.eval-stdout.txt').stat().st_mode & 0o777, 0o600)+class CodexAppServerDriverTests(unittest.TestCase):+ def test_sub_agent_threads_neither_answer_nor_end_the_root_turn(self):+ root, child = 'thread-root', 'thread-child'++ def usage(tid, total):+ return {'method': 'thread/tokenUsage/updated', 'params': {'threadId': tid, 'tokenUsage': {'total': total}}}++ events = [+ {'method': 'turn/started', 'params': {'threadId': root, 'turn': {'id': 'turn-root'}}},+ {'method': 'turn/started', 'params': {'threadId': child, 'turn': {'id': 'turn-child'}}},+ usage(root, {'inputTokens': 100, 'cachedInputTokens': 60, 'outputTokens': 10}),+ usage(child, {'inputTokens': 40, 'cachedInputTokens': 0, 'outputTokens': 5}),+ {'method': 'item/completed', 'params': {'threadId': child, 'item': {'type': 'agentMessage', 'text': 'one half'}}},+ {'method': 'turn/completed', 'params': {'threadId': child, 'turn': {'id': 'turn-child', 'status': 'completed'}}},+ usage(root, {'inputTokens': 250, 'cachedInputTokens': 200, 'outputTokens': 30}),+ {'method': 'item/completed', 'params': {'threadId': root, 'item': {'type': 'agentMessage', 'text': 'wrote report.json'}}},+ {'method': 'turn/completed', 'params': {'threadId': root, 'turn': {'id': 'turn-root', 'status': 'completed'}}},+ ]++ class FakeServer:+ def answer_server_request(self, msg):+ return False++ def request(self, method, params, on_event=None, timeout=60.0):+ return {'turn': {'id': 'turn-root'}}++ def next(self, timeout):+ return events.pop(0) if events else None++ answer, status, _, usage_total, calls = codex_app_server.run_turn(FakeServer(), root, 'go', time.monotonic() + 5)+ self.assertEqual(answer, 'wrote report.json')+ self.assertEqual(status, 'completed')+ # The child's usage counts; the root's later total replaces its earlier one.+ self.assertEqual(usage_total, {'inputTokens': 290, 'cachedInputTokens': 200, 'outputTokens': 35})+ self.assertEqual(calls, 3)++if __name__ == '__main__':unittest.main()