Skip to content

graff — a fast agentic coding harness in Zig: multi-provider, MCP, workflows, DGM evolution loop, TS/Python SDKs

Python

evals: agy and graff-on-Codegraff harnesses for Gemini 4 Argon comparisons (#1661)

Rachb2fb13f41802
diff --git a/graff-evals/graff_json.py b/graff-evals/graff_json.py
new file mode 100644
index 00000000..c2a37afc
--- /dev/null
+++ b/graff-evals/graff_json.py
@@ -0,0 +1,40 @@
+#!/usr/bin/env python3
+"""Run graff --json for one prompt at a chosen effort, for run.py harnesses.
+
+Prints the final answer on stdout and the cumulative usage footer on stderr in
+the `[usage] …` shape measurement.graff_usage reads. GRAFF_BIN picks the binary.
+"""
+import argparse
+import json
+import os
+import subprocess
+import sys
+
+ap = argparse.ArgumentParser()
+ap.add_argument("--effort", default="default")
+ap.add_argument("--model", required=True)
+ap.add_argument("prompt")
+args = ap.parse_args()
+
+lines = [] if args.effort == "default" else [json.dumps({"type": "set_effort", "level": args.effort})]
+lines.append(json.dumps({"type": "user", "text": args.prompt}))
+proc = subprocess.run([os.environ.get("GRAFF_BIN", "graff"), "--json", "--model", args.model, "--yolo"],
+ input="\n".join(lines) + "\n", capture_output=True, text=True)
+turn, errors = None, []
+for line in proc.stdout.splitlines():
+ try:
+ ev = json.loads(line)
+ except json.JSONDecodeError:
+ continue
+ if ev.get("type") == "turn":
+ turn = ev
+ elif ev.get("type") == "error":
+ errors.append(ev.get("message", ""))
+if turn is None:
+ sys.stderr.write(proc.stderr[-2000:] + "\n" + "\n".join(errors) + "\n")
+ sys.exit(proc.returncode or 1)
+print(turn.get("text", ""))
+cost = turn.get("cost_usd")
+sys.stderr.write(f"[usage] {turn.get('api_calls', 0)} api call(s) · {turn.get('input_tokens', 0)} in "
+ f"({turn.get('cache_read_tokens', 0)} cached) + {turn.get('output_tokens', 0)} out tokens"
+ + (f" · ${cost:.6f}" if cost is not None else "") + "\n")
diff --git a/graff-evals/harnesses.json b/graff-evals/harnesses.json
index 3e3fbc9a..ac3f186a 100644
--- a/graff-evals/harnesses.json
+++ b/graff-evals/harnesses.json
@@ -395,5 +395,43 @@
"usage": "graff-stderr",
"default_model": "gpt-6-1-sol-medium",
"note": "Devin CLI over ACP, the way the Harness desktop app drives it (devin acp, tools auto-approved). Token usage is summed from the per-call usage_update notifications and priced at Devin's list price. Needs `devin auth login`; DEVIN_BIN overrides the binary."
+ },
+ "agy-high": {
+ "cmd": ["agy", "-p", "{prompt}", "--output-format", "json", "--dangerously-skip-permissions", "--model", "{model}"],
+ "answer": "agy-json",
+ "usage": "agy-json",
+ "capabilities": [],
+ "model_map": {"gemini-4-argon": "gemini-4-argon-high"},
+ "default_model": "gemini-4-argon",
+ "note": "Antigravity CLI print mode on its own sign-in, high reasoning. input_tokens there excludes the cached part; calls are not reported."
+ },
+ "agy-low": {
+ "cmd": ["agy", "-p", "{prompt}", "--output-format", "json", "--dangerously-skip-permissions", "--model", "{model}"],
+ "answer": "agy-json",
+ "usage": "agy-json",
+ "capabilities": [],
+ "model_map": {"gemini-4-argon": "gemini-4-argon-low"},
+ "default_model": "gemini-4-argon",
+ "note": "Antigravity CLI print mode on its own sign-in, low reasoning. input_tokens there excludes the cached part; calls are not reported."
+ },
+ "graff-gateway-high": {
+ "cmd": ["python3", "{repo}/graff-evals/graff_json.py", "--effort", "high", "--model", "{model}", "{prompt}"],
+ "answer": "stdout",
+ "usage": "graff-stderr",
+ "capabilities": [],
+ "env": {"GRAFF_NO_TELEMETRY": "1", "GRAFF_BIN": "{repo}/zig-out/bin/graff"},
+ "model_prefix": "codegraff",
+ "default_model": "gemini-4-argon",
+ "note": "graff --json on the Codegraff gateway, set_effort high before the prompt (-p has no effort flag)"
+ },
+ "graff-gateway-default": {
+ "cmd": ["python3", "{repo}/graff-evals/graff_json.py", "--effort", "default", "--model", "{model}", "{prompt}"],
+ "answer": "stdout",
+ "usage": "graff-stderr",
+ "capabilities": [],
+ "env": {"GRAFF_NO_TELEMETRY": "1", "GRAFF_BIN": "{repo}/zig-out/bin/graff"},
+ "model_prefix": "codegraff",
+ "default_model": "gemini-4-argon",
+ "note": "graff --json on the Codegraff gateway, set_effort default before the prompt (-p has no effort flag)"
}
}
diff --git a/graff-evals/list_price.py b/graff-evals/list_price.py
index 7f4cd81e..e445d079 100644
--- a/graff-evals/list_price.py
+++ b/graff-evals/list_price.py
@@ -67,6 +67,8 @@ RATES = {
"high_cache": 0.40,
"high_out": 4.0,
},
+ # Gemini 4 Argon list per 1M
+ "gemini-4-argon": {"high_at": 0, "in": 2.0, "cache": 0.10, "out": 10.0, "high_in": 2.0, "high_cache": 0.10, "high_out": 10.0},
"gemini-3.8-flash": {
"high_at": 0,
"in": 0.75,
@@ -130,6 +132,8 @@ def rates_for(model: str) -> dict | None:
return RATES["muse-spark-1.2-contributor"]
if model.startswith("muse-spark-1.2") or leaf.startswith("muse-spark-1.2"):
return RATES["muse-spark-1.2"]
+ if model.startswith("gemini-4-argon") or leaf.startswith("gemini-4-argon"):
+ return RATES["gemini-4-argon"]
if model.startswith("gemini") or leaf.startswith("gemini"):
return RATES["gemini-3.8-flash"]
if model in ("k3", "kimi-k3") or leaf in ("k3", "kimi-k3") or leaf.startswith("kimi-k3"):
diff --git a/graff-evals/run.py b/graff-evals/run.py
index 888e111d..0d4b8a0f 100755
--- a/graff-evals/run.py
+++ b/graff-evals/run.py
@@ -183,6 +183,25 @@ def _parse_gemini_json(stdout):
return answer, usage
+def _parse_agy_json(stdout):
+ """Antigravity CLI `-p --output-format json`: one object; input_tokens excludes the
+ cache_read part and output_tokens includes thinking. num_turns counts conversation
+ turns, not model calls, so calls stays unknown."""
+ answer, usage = stdout.strip(), {}
+ for line in reversed(stdout.strip().splitlines()):
+ try:
+ data = json.loads(line)
+ except json.JSONDecodeError:
+ continue
+ u = data.get("usage") or {}
+ answer = (data.get("response") or "").strip()
+ cached = int(u.get("cache_read_tokens") or 0)
+ usage = {"in": int(u.get("input_tokens") or 0) + cached, "cached": cached,
+ "out": int(u.get("output_tokens") or 0), "reasoning": int(u.get("thinking_tokens") or 0)}
+ break
+ return answer, usage
+
+
def parse_answer_and_usage(harness, stdout, stderr, sandbox=""):
answer, usage = stdout.strip(), {}
if harness["answer"] == "grok-stream":
@@ -239,6 +258,8 @@ def parse_answer_and_usage(harness, stdout, stderr, sandbox=""):
answer, usage = _parse_opencode_json(stdout)
if harness["answer"] == "gemini-json":
answer, usage = _parse_gemini_json(stdout)
+ if harness["answer"] == "agy-json":
+ answer, usage = _parse_agy_json(stdout)
if harness["answer"] == "muse" or harness.get("usage") == "muse":
answer = stdout.strip()
usage = {}