Skip to content

graff — a fast agentic coding harness in Zig: multi-provider, MCP, workflows, DGM evolution loop, TS/Python SDKs

Python

graff-evals: the app-server driver follows the root thread

justrach98c0ff9c49c1
diff --git a/graff-evals/codex_app_server.py b/graff-evals/codex_app_server.py
index a53e32cc..e67c1e6a 100755
--- a/graff-evals/codex_app_server.py
+++ b/graff-evals/codex_app_server.py
@@ -172,8 +172,25 @@ def text_input(text: str) -> list[dict]:
return [{"type": "text", "text": text, "text_elements": []}]
-def run_turn(srv: AppServer, thread_id: str, text: str, deadline: float, steer: str = "", steer_after: str = "text"):
- """One turn; returns (final agent text, turn status, steer outcome)."""
+def summed(totals: dict) -> dict:
+ """Every thread's cumulative usage added up: sub-agent threads bill too."""
+ out: dict = {}
+ for usage in totals.values():
+ for key, value in (usage or {}).items():
+ if isinstance(value, (int, float)):
+ out[key] = out.get(key, 0) + value
+ return out
+
+
+def run_turn(srv: AppServer, thread_id: str, text: str, deadline: float, steer: str = "", steer_after: str = "text",
+ totals: dict | None = None):
+ """One turn; returns (final agent text, turn status, steer outcome).
+
+ Sub-agents run on their own threads over the same connection. Only the
+ root thread's events start, answer or end the turn; every thread's usage
+ counts (`totals` maps thread id to its cumulative usage, across turns).
+ """
+ totals = {} if totals is None else totals
state = {"turn": None, "done": None, "answer": "", "steered": "", "usage": None, "armed": 0.0, "t0": time.monotonic()}
delay = float(os.environ.get("EVAL_STEER_DELAY_S", "0"))
@@ -182,22 +199,25 @@ def run_turn(srv: AppServer, thread_id: str, text: str, deadline: float, steer:
return
method = msg.get("method")
params = msg.get("params") or {}
- if method == "turn/started":
+ tid = params.get("threadId")
+ root = tid is None or tid == thread_id
+ if method == "turn/started" and root:
state["turn"] = (params.get("turn") or {}).get("id") or state["turn"]
- elif method == "item/started" and (params.get("item") or {}).get("type") == "agentMessage" and state.get("steer_at") and not state.get("after_item"):
+ elif method == "item/started" and root and (params.get("item") or {}).get("type") == "agentMessage" and state.get("steer_at") and not state.get("after_item"):
state["after_item"] = time.monotonic()
elif method == "item/completed":
item = params.get("item") or {}
- trace(item_sizes(item))
- if item.get("type") == "agentMessage":
+ trace(dict(item_sizes(item), thread="root" if root else "child"))
+ if root and item.get("type") == "agentMessage":
state["answer"] = item.get("text") or state["answer"]
elif method == "thread/tokenUsage/updated":
- state["usage"] = (params.get("tokenUsage") or {}).get("total")
- trace({"ev": "usage", "last": (params.get("tokenUsage") or {}).get("last")})
+ totals[tid or thread_id] = (params.get("tokenUsage") or {}).get("total")
+ state["usage"] = summed(totals)
+ trace({"ev": "usage", "thread": "root" if root else "child", "last": (params.get("tokenUsage") or {}).get("last")})
state["calls"] = state.get("calls", 0) + 1
if os.environ.get("EVAL_DEBUG"):
print(f"[usage-event] {json.dumps(params.get('tokenUsage'))}", file=sys.stderr)
- elif method == "turn/completed":
+ elif method == "turn/completed" and root:
state["done"] = params.get("turn") or {}
if steer and not state["steered"] and state["turn"]:
trigger = method in ("item/agentMessage/delta", "item/reasoning/summaryTextDelta") if steer_after == "text" else method == "turn/started"
@@ -266,11 +286,12 @@ def main():
thread = srv.request("thread/start", {"model": model, "cwd": sandbox, "approvalPolicy": "never",
"sandbox": "danger-full-access", "ephemeral": True}, timeout=60)
thread_id = (thread.get("thread") or {}).get("id")
+ totals: dict = {}
answer, status, steered, usage, calls = run_turn(srv, thread_id, prompt, deadline, os.environ.get("EVAL_STEER", ""),
- os.environ.get("EVAL_STEER_AFTER", "text"))
+ os.environ.get("EVAL_STEER_AFTER", "text"), totals)
turns = 1
if os.environ.get("EVAL_FOLLOWUP") and status == "completed":
- answer, status, _, usage2, calls2 = run_turn(srv, thread_id, os.environ["EVAL_FOLLOWUP"], deadline)
+ answer, status, _, usage2, calls2 = run_turn(srv, thread_id, os.environ["EVAL_FOLLOWUP"], deadline, totals=totals)
usage, calls, turns = usage2 or usage, calls + calls2, 2
print(answer)
print(f"[codex-app-server] status={status} steer={steered or '-'} turns={turns}", file=sys.stderr)
diff --git a/graff-evals/test_measurement.py b/graff-evals/test_measurement.py
index d465ea7f..f564cc50 100644
--- a/graff-evals/test_measurement.py
+++ b/graff-evals/test_measurement.py
@@ -6,9 +6,11 @@ import os
from pathlib import Path
import sys
import tempfile
+import time
import unittest
from unittest.mock import patch
+import codex_app_server
import list_price
import measurement
import request_capture
@@ -442,5 +444,42 @@ class MeasurementTests(unittest.TestCase):
self.assertEqual((Path(sandboxes) / '.eval-stdout.txt').stat().st_mode & 0o777, 0o600)
+class CodexAppServerDriverTests(unittest.TestCase):
+ def test_sub_agent_threads_neither_answer_nor_end_the_root_turn(self):
+ root, child = 'thread-root', 'thread-child'
+
+ def usage(tid, total):
+ return {'method': 'thread/tokenUsage/updated', 'params': {'threadId': tid, 'tokenUsage': {'total': total}}}
+
+ events = [
+ {'method': 'turn/started', 'params': {'threadId': root, 'turn': {'id': 'turn-root'}}},
+ {'method': 'turn/started', 'params': {'threadId': child, 'turn': {'id': 'turn-child'}}},
+ usage(root, {'inputTokens': 100, 'cachedInputTokens': 60, 'outputTokens': 10}),
+ usage(child, {'inputTokens': 40, 'cachedInputTokens': 0, 'outputTokens': 5}),
+ {'method': 'item/completed', 'params': {'threadId': child, 'item': {'type': 'agentMessage', 'text': 'one half'}}},
+ {'method': 'turn/completed', 'params': {'threadId': child, 'turn': {'id': 'turn-child', 'status': 'completed'}}},
+ usage(root, {'inputTokens': 250, 'cachedInputTokens': 200, 'outputTokens': 30}),
+ {'method': 'item/completed', 'params': {'threadId': root, 'item': {'type': 'agentMessage', 'text': 'wrote report.json'}}},
+ {'method': 'turn/completed', 'params': {'threadId': root, 'turn': {'id': 'turn-root', 'status': 'completed'}}},
+ ]
+
+ class FakeServer:
+ def answer_server_request(self, msg):
+ return False
+
+ def request(self, method, params, on_event=None, timeout=60.0):
+ return {'turn': {'id': 'turn-root'}}
+
+ def next(self, timeout):
+ return events.pop(0) if events else None
+
+ answer, status, _, usage_total, calls = codex_app_server.run_turn(FakeServer(), root, 'go', time.monotonic() + 5)
+ self.assertEqual(answer, 'wrote report.json')
+ self.assertEqual(status, 'completed')
+ # The child's usage counts; the root's later total replaces its earlier one.
+ self.assertEqual(usage_total, {'inputTokens': 290, 'cachedInputTokens': 200, 'outputTokens': 35})
+ self.assertEqual(calls, 3)
+
+
if __name__ == '__main__':
unittest.main()