graff — a fast agentic coding harness in Zig: multi-provider, MCP, workflows, DGM evolution loop, TS/Python SDKs
Python
evals: agy and graff-on-Codegraff harnesses for Gemini 4 Argon comparisons (#1661)
Rachb2fb13f41802
diff --git a/graff-evals/graff_json.py b/graff-evals/graff_json.pynew file mode 100644index 00000000..c2a37afc--- /dev/null+++ b/graff-evals/graff_json.py@@ -0,0 +1,40 @@+#!/usr/bin/env python3+"""Run graff --json for one prompt at a chosen effort, for run.py harnesses.++Prints the final answer on stdout and the cumulative usage footer on stderr in+the `[usage] …` shape measurement.graff_usage reads. GRAFF_BIN picks the binary.+"""+import argparse+import json+import os+import subprocess+import sys++ap = argparse.ArgumentParser()+ap.add_argument("--effort", default="default")+ap.add_argument("--model", required=True)+ap.add_argument("prompt")+args = ap.parse_args()++lines = [] if args.effort == "default" else [json.dumps({"type": "set_effort", "level": args.effort})]+lines.append(json.dumps({"type": "user", "text": args.prompt}))+proc = subprocess.run([os.environ.get("GRAFF_BIN", "graff"), "--json", "--model", args.model, "--yolo"],+ input="\n".join(lines) + "\n", capture_output=True, text=True)+turn, errors = None, []+for line in proc.stdout.splitlines():+ try:+ ev = json.loads(line)+ except json.JSONDecodeError:+ continue+ if ev.get("type") == "turn":+ turn = ev+ elif ev.get("type") == "error":+ errors.append(ev.get("message", ""))+if turn is None:+ sys.stderr.write(proc.stderr[-2000:] + "\n" + "\n".join(errors) + "\n")+ sys.exit(proc.returncode or 1)+print(turn.get("text", ""))+cost = turn.get("cost_usd")+sys.stderr.write(f"[usage] {turn.get('api_calls', 0)} api call(s) · {turn.get('input_tokens', 0)} in "+ f"({turn.get('cache_read_tokens', 0)} cached) + {turn.get('output_tokens', 0)} out tokens"+ + (f" · ${cost:.6f}" if cost is not None else "") + "\n")diff --git a/graff-evals/harnesses.json b/graff-evals/harnesses.jsonindex 3e3fbc9a..ac3f186a 100644--- a/graff-evals/harnesses.json+++ b/graff-evals/harnesses.json@@ -395,5 +395,43 @@"usage": "graff-stderr","default_model": "gpt-6-1-sol-medium","note": "Devin CLI over ACP, the way the Harness desktop app drives it (devin acp, tools auto-approved). Token usage is summed from the per-call usage_update notifications and priced at Devin's list price. Needs `devin auth login`; DEVIN_BIN overrides the binary."+ },+ "agy-high": {+ "cmd": ["agy", "-p", "{prompt}", "--output-format", "json", "--dangerously-skip-permissions", "--model", "{model}"],+ "answer": "agy-json",+ "usage": "agy-json",+ "capabilities": [],+ "model_map": {"gemini-4-argon": "gemini-4-argon-high"},+ "default_model": "gemini-4-argon",+ "note": "Antigravity CLI print mode on its own sign-in, high reasoning. input_tokens there excludes the cached part; calls are not reported."+ },+ "agy-low": {+ "cmd": ["agy", "-p", "{prompt}", "--output-format", "json", "--dangerously-skip-permissions", "--model", "{model}"],+ "answer": "agy-json",+ "usage": "agy-json",+ "capabilities": [],+ "model_map": {"gemini-4-argon": "gemini-4-argon-low"},+ "default_model": "gemini-4-argon",+ "note": "Antigravity CLI print mode on its own sign-in, low reasoning. input_tokens there excludes the cached part; calls are not reported."+ },+ "graff-gateway-high": {+ "cmd": ["python3", "{repo}/graff-evals/graff_json.py", "--effort", "high", "--model", "{model}", "{prompt}"],+ "answer": "stdout",+ "usage": "graff-stderr",+ "capabilities": [],+ "env": {"GRAFF_NO_TELEMETRY": "1", "GRAFF_BIN": "{repo}/zig-out/bin/graff"},+ "model_prefix": "codegraff",+ "default_model": "gemini-4-argon",+ "note": "graff --json on the Codegraff gateway, set_effort high before the prompt (-p has no effort flag)"+ },+ "graff-gateway-default": {+ "cmd": ["python3", "{repo}/graff-evals/graff_json.py", "--effort", "default", "--model", "{model}", "{prompt}"],+ "answer": "stdout",+ "usage": "graff-stderr",+ "capabilities": [],+ "env": {"GRAFF_NO_TELEMETRY": "1", "GRAFF_BIN": "{repo}/zig-out/bin/graff"},+ "model_prefix": "codegraff",+ "default_model": "gemini-4-argon",+ "note": "graff --json on the Codegraff gateway, set_effort default before the prompt (-p has no effort flag)"}}diff --git a/graff-evals/list_price.py b/graff-evals/list_price.pyindex 7f4cd81e..e445d079 100644--- a/graff-evals/list_price.py+++ b/graff-evals/list_price.py@@ -67,6 +67,8 @@ RATES = {"high_cache": 0.40,"high_out": 4.0,},+ # Gemini 4 Argon list per 1M+ "gemini-4-argon": {"high_at": 0, "in": 2.0, "cache": 0.10, "out": 10.0, "high_in": 2.0, "high_cache": 0.10, "high_out": 10.0},"gemini-3.8-flash": {"high_at": 0,"in": 0.75,@@ -130,6 +132,8 @@ def rates_for(model: str) -> dict | None:return RATES["muse-spark-1.2-contributor"]if model.startswith("muse-spark-1.2") or leaf.startswith("muse-spark-1.2"):return RATES["muse-spark-1.2"]+ if model.startswith("gemini-4-argon") or leaf.startswith("gemini-4-argon"):+ return RATES["gemini-4-argon"]if model.startswith("gemini") or leaf.startswith("gemini"):return RATES["gemini-3.8-flash"]if model in ("k3", "kimi-k3") or leaf in ("k3", "kimi-k3") or leaf.startswith("kimi-k3"):diff --git a/graff-evals/run.py b/graff-evals/run.pyindex 888e111d..0d4b8a0f 100755--- a/graff-evals/run.py+++ b/graff-evals/run.py@@ -183,6 +183,25 @@ def _parse_gemini_json(stdout):return answer, usage+def _parse_agy_json(stdout):+ """Antigravity CLI `-p --output-format json`: one object; input_tokens excludes the+ cache_read part and output_tokens includes thinking. num_turns counts conversation+ turns, not model calls, so calls stays unknown."""+ answer, usage = stdout.strip(), {}+ for line in reversed(stdout.strip().splitlines()):+ try:+ data = json.loads(line)+ except json.JSONDecodeError:+ continue+ u = data.get("usage") or {}+ answer = (data.get("response") or "").strip()+ cached = int(u.get("cache_read_tokens") or 0)+ usage = {"in": int(u.get("input_tokens") or 0) + cached, "cached": cached,+ "out": int(u.get("output_tokens") or 0), "reasoning": int(u.get("thinking_tokens") or 0)}+ break+ return answer, usage++def parse_answer_and_usage(harness, stdout, stderr, sandbox=""):answer, usage = stdout.strip(), {}if harness["answer"] == "grok-stream":@@ -239,6 +258,8 @@ def parse_answer_and_usage(harness, stdout, stderr, sandbox=""):answer, usage = _parse_opencode_json(stdout)if harness["answer"] == "gemini-json":answer, usage = _parse_gemini_json(stdout)+ if harness["answer"] == "agy-json":+ answer, usage = _parse_agy_json(stdout)if harness["answer"] == "muse" or harness.get("usage") == "muse":answer = stdout.strip()usage = {}