Skip to content

graff — a fast agentic coding harness in Zig: multi-provider, MCP, workflows, DGM evolution loop, TS/Python SDKs

Python

graff-evals: a Devin arm driven over ACP

justrach692d7afc659d
diff --git a/graff-evals/devin_acp_run.py b/graff-evals/devin_acp_run.py
new file mode 100644
index 00000000..ec62556b
--- /dev/null
+++ b/graff-evals/devin_acp_run.py
@@ -0,0 +1,136 @@
+#!/usr/bin/env python3
+"""Eval arm: one prompt to Devin over ACP, as Harness drives it.
+
+usage: devin_acp_run.py <model> <prompt> (cwd = the task sandbox)
+
+Spawns `devin ... acp`, opens a session in the cwd, sends the prompt,
+auto-approves tool permissions, prints the answer to stdout and graff's
+`[usage]` footer to stderr so run.py's graff-stderr parser scores it like
+the other arms. DEVIN_ACP_DEBUG=1 also dumps the raw prompt response.
+"""
+import json, os, subprocess, sys, threading, time
+
+# Devin list prices per 1M tokens, from `devin models list` (2026-10-04).
+PRICES = {"gpt-6-1-sol": (2.0, 0.1, 10.0), "gpt-6-sol": (2.0, 0.2, 10.0)}
+
+
+def price_for(model):
+ for prefix, p in PRICES.items():
+ if model.startswith(prefix):
+ return p
+ return None
+
+
+def main():
+ model, prompt = sys.argv[1], sys.argv[2]
+ cwd = os.getcwd()
+ devin = os.environ.get("DEVIN_BIN", os.path.expanduser("~/.local/bin/devin"))
+ cmd = [devin, "--model", model, "--permission-mode", "dangerous", "--respect-workspace-trust=false", "acp"]
+ proc = subprocess.Popen(cmd, cwd=cwd, stdin=subprocess.PIPE, stdout=subprocess.PIPE,
+ stderr=subprocess.DEVNULL, text=True, bufsize=1)
+ pending, lock = {}, threading.Lock()
+ answer, tool_calls, next_id = [], [0], [0]
+ calls, last_key = [], [None] # per-model-call usage from usage_update._meta
+
+ def send(obj):
+ proc.stdin.write(json.dumps(obj) + "\n")
+ proc.stdin.flush()
+
+ def request(method, params):
+ with lock:
+ next_id[0] += 1
+ rid = next_id[0]
+ ev = threading.Event()
+ pending[rid] = {"ev": ev, "res": None}
+ send({"jsonrpc": "2.0", "id": rid, "method": method, "params": params})
+ return rid, ev
+
+ def reader():
+ for line in proc.stdout:
+ line = line.strip()
+ if not line:
+ continue
+ try:
+ msg = json.loads(line)
+ except ValueError:
+ continue
+ if "id" in msg and ("result" in msg or "error" in msg) and "method" not in msg:
+ slot = pending.get(msg["id"])
+ if slot:
+ slot["res"] = msg
+ slot["ev"].set()
+ elif msg.get("method") == "session/request_permission":
+ opts = (msg.get("params") or {}).get("options") or []
+ pick = next((o for o in opts if o.get("kind") == "allow_always"), None) or \
+ next((o for o in opts if str(o.get("kind", "")).startswith("allow")), None) or (opts[0] if opts else None)
+ outcome = {"outcome": "selected", "optionId": pick["optionId"]} if pick else {"outcome": "cancelled"}
+ send({"jsonrpc": "2.0", "id": msg["id"], "result": {"outcome": outcome}})
+ elif msg.get("method") == "session/update":
+ upd = (msg.get("params") or {}).get("update") or {}
+ kind = upd.get("sessionUpdate")
+ if kind == "agent_message_chunk":
+ content = upd.get("content") or {}
+ if content.get("type") == "text":
+ answer.append(content.get("text", ""))
+ elif kind == "tool_call":
+ tool_calls[0] += 1
+ elif kind == "usage_update":
+ meta = upd.get("_meta") or {}
+ key = tuple(int(meta.get(f"cognition.ai/{k}", 0) or 0) for k in
+ ("inputTokens", "outputTokens", "cachedReadTokens", "cachedWriteTokens"))
+ sub = meta.get("cognition.ai/subagent_context") or {}
+ # Devin echoes each root call once more tagged parentAgentId=root.
+ if not (sub.get("parentAgentId") == "root" and key == last_key[0]) and any(key):
+ calls.append(key)
+ last_key[0] = key
+ if os.environ.get("DEVIN_ACP_DEBUG") and kind not in ("agent_message_chunk", "agent_thought_chunk"):
+ print(f"[devin-acp] update {kind}: {json.dumps(upd)[:300]}", file=sys.stderr)
+ elif "id" in msg and "method" in msg:
+ # Any other client request (fs/terminal): we advertised none.
+ send({"jsonrpc": "2.0", "id": msg["id"], "error": {"code": -32601, "message": "not supported"}})
+
+ threading.Thread(target=reader, daemon=True).start()
+ t0 = time.monotonic()
+ rid, ev = request("initialize", {"protocolVersion": 1, "clientCapabilities": {
+ "fs": {"readTextFile": False, "writeTextFile": False}, "terminal": False}})
+ ev.wait(60)
+ rid, ev = request("session/new", {"cwd": cwd, "mcpServers": []})
+ ev.wait(120)
+ res = pending[rid]["res"] or {}
+ sid = (res.get("result") or {}).get("sessionId")
+ if not sid:
+ print(f"[devin-acp] session/new failed: {json.dumps(res)[:400]}", file=sys.stderr)
+ proc.kill()
+ sys.exit(2)
+ rid, ev = request("session/prompt", {"sessionId": sid, "prompt": [{"type": "text", "text": prompt}]})
+ ev.wait(float(os.environ.get("DEVIN_ACP_TIMEOUT", "1500")))
+ res = pending[rid]["res"] or {}
+ result = res.get("result") or {}
+ if os.environ.get("DEVIN_ACP_DEBUG"):
+ print(f"[devin-acp] prompt response: {json.dumps(res)[:2000]}", file=sys.stderr)
+ proc.stdin.close()
+ try:
+ proc.wait(10)
+ except subprocess.TimeoutExpired:
+ proc.kill()
+
+ text = "".join(answer).strip()
+ print(text)
+ usage = bool(calls)
+ inp, out, cached, writes = (sum(c[i] for c in calls) for i in range(4))
+ price = price_for(model)
+ tail = ""
+ if usage and price:
+ uncached = max(inp - cached, 0)
+ cost = (uncached * price[0] + cached * price[1] + out * price[2]) / 1e6
+ tail = f" · ${cost:.8f}"
+ elif not usage:
+ tail = " · totals incomplete: 1 call(s) missing usage (tokens and cost unknown)"
+ print(f"[usage] {len(calls)} api call(s) · {inp} in ({cached} cached, {writes} cache writes) + {out} out tokens{tail}",
+ file=sys.stderr)
+ print(f"[devin-acp] stop={result.get('stopReason')} tools={tool_calls[0]} wall={time.monotonic() - t0:.1f}s",
+ file=sys.stderr)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/graff-evals/harnesses.json b/graff-evals/harnesses.json
index 7a1fb6ba..8f69f4b8 100644
--- a/graff-evals/harnesses.json
+++ b/graff-evals/harnesses.json
@@ -380,5 +380,12 @@
"graff-gateway-repl-norlm": {"cmd": ["{repo}/zig-out/bin/graff", "repl", "--model", "{model}", "--yolo"], "stdin": "\u001b[200~{prompt}\u001b[201~\n", "answer": "stdout", "usage": "graff-stderr", "env": {"GRAFF_POST_DEADLINE_SECS": "90", "GRAFF_NO_TELEMETRY": "1", "GRAFF_RLM": "0"}, "model_prefix": "codegraff", "note": "graff-gateway-repl with rlm off (GRAFF_RLM=0: the structured-only catalog), the counterfactual for code-mode comparisons; pass --model"},
"pi1": {"cmd": ["python3", "{repo}/graff-evals/pi1_run.py", "{model}", "{prompt}"], "answer": "stdout", "usage": "graff-stderr", "model_prefix": "codegraff", "env": {"PI1_CODEMODE": "0"}, "note": "Pi 1.0 (@earendil-works/pi-coding-agent 1.x) headless with its default tools; MCP tools are reached through codemode, Pi's default MCP exposure. PI1_MODELS / PI1_AUTH copy a models.json / auth.json into the run's private ~/.pi/agent, PI1_THINKING passes --thinking (match the other arm's effort), PI1_BIN picks Pi 1.x; pass --model provider/model for a provider other than codegraff"},
"pi1-codemode": {"cmd": ["python3", "{repo}/graff-evals/pi1_run.py", "{model}", "{prompt}"], "answer": "stdout", "usage": "graff-stderr", "model_prefix": "codegraff", "env": {"PI1_CODEMODE": "1"}, "note": "Pi 1.0 with codemode added to the default tools (settings defaultTools +codemode, not --tools, which drops MCP tools). PI1_MODELS / PI1_AUTH copy a models.json / auth.json into the run's private ~/.pi/agent, PI1_THINKING passes --thinking (match the other arm's effort), PI1_BIN picks Pi 1.x; pass --model provider/model for a provider other than codegraff"},
- "pi1-direct": {"cmd": ["python3", "{repo}/graff-evals/pi1_run.py", "{model}", "{prompt}"], "answer": "stdout", "usage": "graff-stderr", "model_prefix": "codegraff", "env": {"PI1_CODEMODE": "0", "PI1_MCP_EXPOSURE": "direct"}, "note": "Pi 1.0 default tools with every MCP server at exposure direct: no codemode at all, the counterfactual for codemode on MCP tasks. PI1_MODELS / PI1_AUTH copy a models.json / auth.json into the run's private ~/.pi/agent, PI1_THINKING passes --thinking (match the other arm's effort), PI1_BIN picks Pi 1.x; pass --model provider/model for a provider other than codegraff"}
+ "pi1-direct": {"cmd": ["python3", "{repo}/graff-evals/pi1_run.py", "{model}", "{prompt}"], "answer": "stdout", "usage": "graff-stderr", "model_prefix": "codegraff", "env": {"PI1_CODEMODE": "0", "PI1_MCP_EXPOSURE": "direct"}, "note": "Pi 1.0 default tools with every MCP server at exposure direct: no codemode at all, the counterfactual for codemode on MCP tasks. PI1_MODELS / PI1_AUTH copy a models.json / auth.json into the run's private ~/.pi/agent, PI1_THINKING passes --thinking (match the other arm's effort), PI1_BIN picks Pi 1.x; pass --model provider/model for a provider other than codegraff"},
+ "devin-acp": {
+ "cmd": ["python3", "{repo}/graff-evals/devin_acp_run.py", "{model}", "{prompt}"],
+ "answer": "stdout",
+ "usage": "graff-stderr",
+ "default_model": "gpt-6-1-sol-medium",
+ "note": "Devin CLI over ACP, the way the Harness desktop app drives it (devin acp, tools auto-approved). Token usage is summed from the per-call usage_update notifications and priced at Devin's list price. Needs `devin auth login`; DEVIN_BIN overrides the binary."
+ }
}