graff — a fast agentic coding harness in Zig: multi-provider, MCP, workflows, DGM evolution loop, TS/Python SDKs
Python
{
"graff": {
"cmd": ["graff", "-p", "{prompt}", "--model", "{model}", "--yolo"],
"answer": "stdout",
"usage": "graff-stderr",
"capabilities": ["output-schema"],
"schema_args": ["--output-schema", "{schema}"],
"env": {"GRAFF_POST_DEADLINE_SECS": "90"},
"default_model": "grok-4.6"
},
"graff-full": {
"cmd": ["graff", "-p", "{prompt}", "--model", "{model}", "--yolo", "--no-lean"],
"answer": "stdout",
"usage": "graff-stderr",
"capabilities": ["output-schema"],
"schema_args": ["--output-schema", "{schema}"],
"default_model": "grok-4.6"
},
"graff-dev": {
"cmd": ["{repo}/zig-out/bin/graff", "-p", "{prompt}", "--model", "{model}", "--yolo"],
"answer": "stdout",
"usage": "graff-stderr",
"capabilities": ["output-schema"],
"schema_args": ["--output-schema", "{schema}"],
"env": {"GRAFF_POST_DEADLINE_SECS": "90"},
"default_model": "grok-4.6"
},
"graff-dev-repl": {
"cmd": ["{repo}/zig-out/bin/graff", "repl", "--model", "{model}", "--yolo"],
"stdin": "{prompt}\n",
"answer": "stdout",
"usage": "graff-stderr",
"capabilities": [],
"env": {"GRAFF_POST_DEADLINE_SECS": "90"},
"default_model": "grok-4.6",
"note": "scripted `graff repl` (non-TTY). Same learn-auto path as -p. TTY `graff repl` is the TUI."
},
"graff-dev-noxsearch": {
"cmd": ["{repo}/zig-out/bin/graff", "-p", "{prompt}", "--model", "{model}", "--yolo"],
"answer": "stdout",
"usage": "graff-stderr",
"capabilities": ["output-schema"],
"schema_args": ["--output-schema", "{schema}"],
"env": {"GRAFF_XAI_X_SEARCH": "0", "GRAFF_POST_DEADLINE_SECS": "90"},
"default_model": "grok-4.6",
"note": "hillclimb candidate: ADR 0031 opt-out. Hosted x_search off; catalog unchanged."
},
"graff-dev-rlm": {
"cmd": ["{repo}/zig-out/bin/graff", "-p", "{prompt}", "--model", "{model}", "--yolo", "--rlm"],
"answer": "stdout",
"usage": "graff-stderr",
"capabilities": ["output-schema"],
"schema_args": ["--output-schema", "{schema}"],
"env": {"GRAFF_POST_DEADLINE_SECS": "90"},
"default_model": "grok-4.6",
"note": "explicit --rlm (same as graff-dev after RLM became the default)"
},
"graff-dev-nolean": {
"cmd": ["{repo}/zig-out/bin/graff", "-p", "{prompt}", "--model", "{model}", "--yolo", "--no-lean"],
"answer": "stdout",
"usage": "graff-stderr",
"capabilities": ["output-schema"],
"schema_args": ["--output-schema", "{schema}"],
"env": {"GRAFF_POST_DEADLINE_SECS": "90"},
"default_model": "grok-4.6",
"note": "full harness for the MCP bench: --no-lean (complete catalog). Lean is a different prefix and is not comparable."
},
"graff-dev-rlm-struct": {
"cmd": ["{repo}/zig-out/bin/graff", "-p", "{prompt}", "--model", "{model}", "--yolo", "--no-lean"],
"answer": "stdout",
"usage": "graff-stderr",
"capabilities": ["output-schema"],
"schema_args": ["--output-schema", "{schema}"],
"env": {"GRAFF_POST_DEADLINE_SECS": "90", "GRAFF_RLM_MCP": "0"},
"default_model": "grok-4.6",
"note": "default rlm but MCP stays structured-only (today's gap / variant B)"
},
"graff-dev-old-nolean": {
"cmd": ["{repo}/zig-out/bin/graff", "-p", "{prompt}", "--model", "{model}", "--yolo", "--old", "--no-lean"],
"answer": "stdout",
"usage": "graff-stderr",
"capabilities": ["output-schema"],
"schema_args": ["--output-schema", "{schema}"],
"env": {"GRAFF_POST_DEADLINE_SECS": "90"},
"default_model": "grok-4.6",
"note": "--old structured MCP only + --no-lean so load_tool_schemas is visible"
},
"graff-dev-old": {
"cmd": ["{repo}/zig-out/bin/graff", "-p", "{prompt}", "--model", "{model}", "--yolo", "--old"],
"answer": "stdout",
"usage": "graff-stderr",
"capabilities": ["output-schema"],
"schema_args": ["--output-schema", "{schema}"],
"env": {"GRAFF_POST_DEADLINE_SECS": "90"},
"default_model": "grok-4.6",
"note": "pre-rlm structured-only catalog (--old)"
},
"grok": {
"cmd": ["grok", "-p", "{prompt}", "-m", "{model}", "--always-approve", "--output-format", "streaming-messages-json"],
"answer": "grok-stream",
"usage": "grok-stream",
"capabilities": [],
"default_model": "grok-4.6"
},
"claude": {
"cmd": ["claude", "-p", "{prompt}", "--model", "{model}", "--dangerously-skip-permissions"],
"answer": "stdout",
"usage": null,
"capabilities": [],
"default_model": "claude-sonnet-5"
},
"dsh": {
"cmd": ["dsh", "--profile", "headless", "--patch", "{repo}/graff-evals/dsh-luna.yml", "{prompt}"],
"answer": "stdout",
"usage": null,
"capabilities": [],
"env": {"DSH_PERMISSION_MODE": "danger-full-access"},
"default_model": "gpt-5.6-luna",
"note": "no --model flag; the model is pinned by the dsh-luna.yml patch layer"
},
"pi": {
"cmd": ["pi", "-p", "--mode", "json", "--provider", "openai", "--model", "{model}", "--no-session", "{prompt}"],
"answer": "pi-json",
"usage": "pi-json",
"capabilities": [],
"default_model": "gpt-5.6-luna"
},
"dsh-deepseek": {
"cmd": ["dsh", "--profile", "headless", "{prompt}"],
"answer": "stdout",
"usage": null,
"capabilities": [],
"env": {"DSH_PERMISSION_MODE": "danger-full-access"},
"default_model": "deepseek-v4-flash",
"note": "dsh's native default route (deepseek-official/deepseek-v4-flash); no patch layer"
},
"pi-deepseek": {
"cmd": ["pi", "-p", "--mode", "json", "--provider", "deepseek", "--model", "{model}", "--no-session", "{prompt}"],
"answer": "pi-json",
"usage": "pi-json",
"capabilities": [],
"default_model": "deepseek-v4-flash"
},
"pi-codegraff": {
"cmd": ["pi", "-p", "--mode", "json", "--provider", "codegraff", "--model", "{model}", "--no-session", "{prompt}"],
"answer": "pi-json",
"usage": "pi-json",
"capabilities": [],
"default_model": "gemini-3.7-flash",
"note": "uses ~/.pi/agent/models.json provider codegraff → gateway.codegraff.com; Gemini tool loops need the codegraff-gemini-echo extension"
},
"opencode-codegraff": {
"cmd": ["{repo}/graff-evals/opencode-codegraff.sh", "-m", "codegraff/{model}", "--dir", "{sandbox}", "{prompt}"],
"answer": "opencode-json",
"usage": "opencode-json",
"capabilities": [],
"default_model": "deepseek-v4-flash",
"note": "OpenCode on the Codegraff gateway (same CODEGRAFF_API_KEY seat as graff-dev / pi-codegraff). Wrapper needs `opencode` on PATH (~/.opencode/bin)."
},
"pi-xai": {
"cmd": ["{repo}/graff-evals/pi-xai.sh", "-p", "--mode", "json", "--provider", "xai", "--model", "{model}", "--no-session", "{prompt}"],
"answer": "pi-json",
"usage": "pi-json",
"capabilities": [],
"default_model": "grok-4.6",
"note": "same SuperGrok seat as graff-dev. Wrapper reads graff-oauth.json into XAI_API_KEY. `pi` must be on PATH."
},
"opencode": {
"cmd": ["{repo}/graff-evals/opencode-xai.sh", "--dir", "{sandbox}", "-m", "{model}", "{prompt}"],
"answer": "opencode-json",
"usage": "opencode-json",
"capabilities": [],
"model_prefix": "xai",
"default_model": "xai/grok-4.6",
"note": "OpenCode 1.18 headless on SuperGrok. Wrapper maps graff-oauth.json → XAI_API_KEY and injects X-XAI-Token-Auth via opencode-xai.json. --dir is the eval sandbox. --pure skips plugins."
},
"exo": {
"cmd": ["{repo}/graff-evals/exo-xai.sh", "--model", "{model}", "{prompt}"],
"answer": "stdout",
"usage": "graff-stderr",
"capabilities": [],
"default_model": "grok-4.6",
"note": "exoharness/exo (https://github.com/exoharness/exo) on SuperGrok via local-process (no Docker). Wrapper refreshes graff-oauth, injects X-XAI-Token-Auth through xai-header-proxy.py, mounts the eval sandbox at /home/exo/workspace. Needs `exo` on PATH or EXO_BIN / /tmp/exo/target/release/exo."
},
"opencode-zen": {
"cmd": ["opencode", "run", "--format", "json", "--auto", "--pure", "--dir", "{sandbox}", "-m", "{model}", "{prompt}"],
"answer": "opencode-json",
"usage": "opencode-json",
"capabilities": [],
"default_model": "opencode/big-pickle",
"note": "OpenCode native Zen free model — NOT grok-4.6. Plot on the mixed-model chart only."
},
"dsh-grok": {
"cmd": ["dsh", "--profile", "headless", "--patch", "{repo}/graff-evals/dsh-grok.yml", "{prompt}"],
"answer": "stdout",
"usage": null,
"capabilities": [],
"env": {"DSH_PERMISSION_MODE": "danger-full-access"},
"default_model": "grok-4.6",
"note": "Same-model attempt. dsh 0.1.1-rc.2 pi-ai catalog has no grok-4.6 (only grok-4.5 / 4.3 / grok-build-0.1). Expect UNKNOWN_MODEL. Use dsh-xai for a live dsh point."
},
"dsh-xai": {
"cmd": ["dsh", "--profile", "headless", "--patch", "{repo}/graff-evals/dsh-xai.yml", "{prompt}"],
"answer": "stdout",
"usage": null,
"capabilities": [],
"env": {"DSH_PERMISSION_MODE": "danger-full-access"},
"default_model": "grok-4.5",
"note": "deepseek-harness on grok-4.5 (nearest xAI id in its catalog). NOT grok-4.6. Mixed-model chart only. Needs Node >=22.15 (zstd). SuperGrok OAuth → XAI_API_KEY via attach-dsh-xai-oauth.py (local; see dsh-local-oauth.md). No usage footer."
},
"gemini-cli": {
"cmd": ["{repo}/graff-evals/gemini-cli.sh", "-p", "{prompt}", "-m", "{model}", "-y", "--skip-trust", "-o", "json", "--allowed-mcp-server-names", "none"],
"answer": "gemini-json",
"usage": "gemini-json",
"capabilities": [],
"default_model": "gemini-3.5-flash",
"note": "Official Google Gemini CLI (@google/gemini-cli) driven with GEMINI_API_KEY"
},
"pi-dev": {
"cmd": ["pi", "-p", "--mode", "json", "--provider", "codegraff", "--model", "{model}", "--no-session", "{prompt}"],
"answer": "pi-json",
"usage": "pi-json",
"capabilities": [],
"default_model": "gemini-3.8-flash",
"note": "Pi coding agent (pi.dev) on Codegraff gateway"
},
"graff-dev-completion": {
"cmd": [
"{repo}/zig-out/bin/graff",
"-p",
"{prompt}",
"--model",
"{model}",
"--yolo",
"--no-lean",
"--no-telemetry",
"--no-subagent-tier"
],
"answer": "stdout",
"usage": "graff-stderr",
"capabilities": [],
"env": {
"GRAFF_FLEET": "off",
"GRAFF_LEARN_AUTO": "off",
"GRAFF_NO_TELEMETRY": "1",
"GRAFF_MCP_CONFIG": "{repo}/graff-evals/live/graff-621/empty-mcp.json"
},
"default_model": "grok-4.6"
},
"fx-completion": {
"cmd": ["{repo}/graff-evals/fx-completion.py", "{model}", "{prompt}"],
"answer": "stdout",
"usage": null,
"capabilities": [],
"env": {"FX_MAX_AGENT_STEPS": "0"},
"default_model": "grok-4.6"
},
"graff-dev-g6-followthrough": {
"cmd": ["{repo}/zig-out/bin/graff", "-p", "{prompt}", "--model", "{model}", "--yolo", "--append-system-prompt",
"Infer the user's intent and task scope from the instructions and prior conversation. Bias towards action and carry the intended task to completion. When the user asks for new work or a fix, persist until the goal is complete, including read-only actions, isolated worktrees and draft PRs, unless a step is clearly destructive or irreversible. Treat \"can you\", \"I want to\" and \"help me\" as instructions to do the work, not to describe it. Before asking a clarifying question, finish the work that is already authorized so the user approves a concrete, reviewable result."],
"answer": "stdout",
"usage": "graff-stderr",
"capabilities": ["output-schema"],
"schema_args": ["--output-schema", "{schema}"],
"env": {"GRAFF_POST_DEADLINE_SECS": "90"},
"default_model": "gpt-6-sol",
"note": "GPT-6 prompting guide candidate (g6-followthrough); judged by hillclimb.py round on held-out tasks before any prompt change ships (ADR 0214)"
},
"graff-dev-g6-skill-precedence": {
"cmd": ["{repo}/zig-out/bin/graff", "-p", "{prompt}", "--model", "{model}", "--yolo", "--append-system-prompt",
"The user's instructions take precedence over guidelines in a skill or instruction file. If a skill causes you to pause, ask for confirmation, or leave work unfinished, name the file, quote the instruction, and say how it applies."],
"answer": "stdout",
"usage": "graff-stderr",
"capabilities": ["output-schema"],
"schema_args": ["--output-schema", "{schema}"],
"env": {"GRAFF_POST_DEADLINE_SECS": "90"},
"default_model": "gpt-6-sol",
"note": "GPT-6 prompting guide candidate (g6-skill-precedence); judged by hillclimb.py round on held-out tasks before any prompt change ships (ADR 0214)"
},
"graff-dev-g6-test-calibration": {
"cmd": ["{repo}/zig-out/bin/graff", "-p", "{prompt}", "--model", "{model}", "--yolo", "--append-system-prompt",
"Do not write tests for reversible, low-impact changes that mirror the implementation. Run the tests and checks the change needs; broaden or repeat them only when new changes, failures or open concerns justify it."],
"answer": "stdout",
"usage": "graff-stderr",
"capabilities": ["output-schema"],
"schema_args": ["--output-schema", "{schema}"],
"env": {"GRAFF_POST_DEADLINE_SECS": "90"},
"default_model": "gpt-6-sol",
"note": "GPT-6 prompting guide candidate (g6-test-calibration); judged by hillclimb.py round on held-out tasks before any prompt change ships (ADR 0214)"
},
"graff-dev-g6-plain-prose": {
"cmd": ["{repo}/zig-out/bin/graff", "-p", "{prompt}", "--model", "{model}", "--yolo", "--append-system-prompt",
"Default to clear, concise paragraphs, each developing one idea. Use lists only for parallel, sequential or comparable information. State the main point early, in plain language with precise verbs, and avoid stock phrases and concluding summaries."],
"answer": "stdout",
"usage": "graff-stderr",
"capabilities": ["output-schema"],
"schema_args": ["--output-schema", "{schema}"],
"env": {"GRAFF_POST_DEADLINE_SECS": "90"},
"default_model": "gpt-6-sol",
"note": "GPT-6 prompting guide candidate (g6-plain-prose); judged by hillclimb.py round on held-out tasks before any prompt change ships (ADR 0214)"
},
"pi-chatgpt-new": {
"cmd": ["npx", "-y", "@earendil-works/[email protected]", "-p", "--mode", "json", "--provider", "openai", "--model", "{model}", "--no-session", "{prompt}"],
"answer": "pi-json",
"env": {},
"default_model": "gpt-6.1-sol"
},
"codex-chatgpt-new": {
"cmd": ["codex", "exec", "--skip-git-repo-check", "--dangerously-bypass-approvals-and-sandbox", "-m", "{model}", "-c", "model_provider=\"openai_token_sharing\"", "-c", "model_providers.openai_token_sharing.name=\"OpenAI Token Sharing\"", "-c", "model_providers.openai_token_sharing.base_url=\"https://api.openai.com/v1\"", "-c", "model_providers.openai_token_sharing.model_catalog_url=\"https://api.openai.com/v1/models\"", "-c", "features.api_key_model_discovery=true", "-c", "model_providers.openai_token_sharing.env_key=\"ACCESS_TOKEN\"", "-c", "model_providers.openai_token_sharing.wire_api=\"responses\"", "-c", "model_providers.openai_token_sharing.requires_openai_auth=false", "-c", "model_providers.openai_token_sharing.supports_websockets=false", "{prompt}"],
"answer": "stdout",
"env": {"CODEX_HOME": "{sandbox}/.codex-home"},
"default_model": "gpt-6.1-sol"
},
"graff-codex": {
"cmd": ["{repo}/zig-out/bin/graff", "-p", "{prompt}", "--model", "{model}", "--yolo", "--no-lean"],
"answer": "stdout",
"usage": "graff-stderr",
"model_prefix": "codex",
"default_model": "gpt-6.1-sol"
},
"graff-codex-repl": {
"cmd": ["{repo}/zig-out/bin/graff", "repl", "--model", "{model}", "--yolo"],
"stdin": "{prompt}\n",
"answer": "stdout",
"usage": "graff-stderr",
"env": {"GRAFF_POST_DEADLINE_SECS": "90"},
"model_prefix": "codex",
"default_model": "gpt-6.1-sol",
"note": "scripted graff repl over the Codex WebSocket: the transport the Codex app server uses"
},
"codex-app-server": {
"cmd": ["python3", "{repo}/graff-evals/codex_app_server.py", "{model}", "{prompt}"],
"answer": "stdout",
"usage": "graff-stderr",
"default_model": "gpt-6.1-sol"
},
"graff-repl": {
"cmd": ["{repo}/zig-out/bin/graff", "repl", "--model", "{model}", "--yolo"],
"stdin": "\u001b[200~{prompt}\u001b[201~\n",
"answer": "stdout",
"usage": "graff-stderr",
"env": {"GRAFF_POST_DEADLINE_SECS": "90", "GRAFF_NO_TELEMETRY": "1"},
"model_prefix": "codex",
"default_model": "gpt-6.1-sol",
"note": "scripted graff repl over the Codex WebSocket. The prompt rides one bracketed-paste frame, so a multi-line task is one turn as in the other harnesses (needs the piped paste of ADR 0231)."
},
"graff-gateway-repl": {
"cmd": ["{repo}/zig-out/bin/graff", "repl", "--model", "{model}", "--yolo"],
"stdin": "\u001b[200~{prompt}\u001b[201~\n",
"answer": "stdout",
"usage": "graff-stderr",
"env": {"GRAFF_POST_DEADLINE_SECS": "90", "GRAFF_NO_TELEMETRY": "1"},
"model_prefix": "codegraff",
"note": "scripted graff repl on the Codegraff gateway (CODEGRAFF_API_KEY); pass --model"
},
"graff-anthropic-repl": {
"cmd": ["{repo}/zig-out/bin/graff", "repl", "--model", "{model}", "--yolo"],
"stdin": "\u001b[200~{prompt}\u001b[201~\n",
"answer": "stdout",
"usage": "graff-stderr",
"env": {"GRAFF_POST_DEADLINE_SECS": "90", "GRAFF_NO_TELEMETRY": "1"},
"note": "scripted graff repl on the Anthropic API (ANTHROPIC_API_KEY); pass --provider anthropic and a claude-* --model"
},
"pi-codex": {
"cmd": ["pi", "-p", "--mode", "json", "--provider", "openai-codex", "--model", "{model}", "--thinking", "low", "--no-session", "{prompt}"],
"answer": "pi-json",
"usage": "pi-json",
"capabilities": [],
"default_model": "gpt-6.1-sol",
"note": "pi on the ChatGPT plan (provider openai-codex, signed in through pi; declare gpt-6.1-sol in ~/.pi/agent/models.json until pi lists it). Low effort, the catalog default graff and the Codex app server use. No MCP client: core suite only."
},
"opencode2": {
"cmd": ["python3", "{repo}/graff-evals/opencode2_run.py", "{model}", "{prompt}"],
"answer": "stdout",
"usage": "graff-stderr",
"model_prefix": "codegraff",
"note": "OpenCode 2 (@opencode/cli 2.x; OPENCODE2_BIN when `opencode` on PATH is 1.x) with a codegraff provider in its config; pass --model. The wrapper maps .mcp.json into opencode.json and reads usage from OpenCode's session database."
},
"claude-code": {
"cmd": ["python3", "{repo}/graff-evals/claude_code_run.py", "{model}", "{prompt}"],
"answer": "stdout",
"usage": "graff-stderr",
"env": {"DISABLE_TELEMETRY": "1", "DISABLE_AUTOUPDATER": "1", "DISABLE_ERROR_REPORTING": "1", "CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC": "1"},
"note": "Claude Code (`claude -p`, stream-json) with the task's .mcp.json as its only MCP config. Set ANTHROPIC_BASE_URL and ANTHROPIC_AUTH_TOKEN for the endpoint; a chat-completions-only endpoint needs an Anthropic-format proxy in front (evals/graff-vs-claude-code). Pass --model."
},
"graff-deepseek": {"cmd": ["{repo}/zig-out/bin/graff", "repl", "--model", "{model}", "--yolo"], "stdin": "\u001b[200~{prompt}\u001b[201~\n", "answer": "stdout", "usage": "graff-stderr", "env": {"GRAFF_POST_DEADLINE_SECS": "90", "GRAFF_NO_TELEMETRY": "1"}, "model_prefix": "deepseek", "default_model": "deepseek-v4-pro", "note": "scripted graff repl on DeepSeek's own API (DEEPSEEK_API_KEY)"},
"graff-mlx-repl": {"cmd": ["{repo}/zig-out/bin/graff", "repl", "--model", "{model}", "--yolo"], "stdin": "\u001b[200~{prompt}\u001b[201~\n", "answer": "stdout", "usage": "graff-stderr", "env": {"GRAFF_POST_DEADLINE_SECS": "300", "GRAFF_NO_TELEMETRY": "1"}, "model_prefix": "mlx", "default_model": "flint", "note": "scripted graff repl on a local MLX server at 127.0.0.1:8080 (MLX_API_KEY; for Flint, `rec serve --port 8080` and MLX_API_KEY=$(rec auth token)). The server keeps one prompt cache, so run -j 1; the longer post deadline covers a cold local prefill"},
"deepseek-harness": {"cmd": ["python3", "{repo}/graff-evals/dsh_run.py", "{model}", "{prompt}"], "answer": "stdout", "usage": "graff-stderr", "env": {"DSH_PERMISSION_MODE": "danger-full-access"}, "default_model": "deepseek-v4-pro", "note": "deepseek-harness (@deepseek-ai/dsh, DSH_BIN when dsh is not on PATH) headless on DeepSeek's own API; the model and the task's MCP servers go in a patch layer"},
"graff-gateway-repl-norlm": {"cmd": ["{repo}/zig-out/bin/graff", "repl", "--model", "{model}", "--yolo"], "stdin": "\u001b[200~{prompt}\u001b[201~\n", "answer": "stdout", "usage": "graff-stderr", "env": {"GRAFF_POST_DEADLINE_SECS": "90", "GRAFF_NO_TELEMETRY": "1", "GRAFF_RLM": "0"}, "model_prefix": "codegraff", "note": "graff-gateway-repl with rlm off (GRAFF_RLM=0: the structured-only catalog), the counterfactual for code-mode comparisons; pass --model"},
"pi1": {"cmd": ["python3", "{repo}/graff-evals/pi1_run.py", "{model}", "{prompt}"], "answer": "stdout", "usage": "graff-stderr", "model_prefix": "codegraff", "env": {"PI1_CODEMODE": "0"}, "note": "Pi 1.0 (@earendil-works/pi-coding-agent 1.x) headless with its default tools; MCP tools are reached through codemode, Pi's default MCP exposure. PI1_MODELS / PI1_AUTH copy a models.json / auth.json into the run's private ~/.pi/agent, PI1_THINKING passes --thinking (match the other arm's effort), PI1_BIN picks Pi 1.x; pass --model provider/model for a provider other than codegraff"},
"pi1-codemode": {"cmd": ["python3", "{repo}/graff-evals/pi1_run.py", "{model}", "{prompt}"], "answer": "stdout", "usage": "graff-stderr", "model_prefix": "codegraff", "env": {"PI1_CODEMODE": "1"}, "note": "Pi 1.0 with codemode added to the default tools (settings defaultTools +codemode, not --tools, which drops MCP tools). PI1_MODELS / PI1_AUTH copy a models.json / auth.json into the run's private ~/.pi/agent, PI1_THINKING passes --thinking (match the other arm's effort), PI1_BIN picks Pi 1.x; pass --model provider/model for a provider other than codegraff"},
"pi1-direct": {"cmd": ["python3", "{repo}/graff-evals/pi1_run.py", "{model}", "{prompt}"], "answer": "stdout", "usage": "graff-stderr", "model_prefix": "codegraff", "env": {"PI1_CODEMODE": "0", "PI1_MCP_EXPOSURE": "direct"}, "note": "Pi 1.0 default tools with every MCP server at exposure direct: no codemode at all, the counterfactual for codemode on MCP tasks. PI1_MODELS / PI1_AUTH copy a models.json / auth.json into the run's private ~/.pi/agent, PI1_THINKING passes --thinking (match the other arm's effort), PI1_BIN picks Pi 1.x; pass --model provider/model for a provider other than codegraff"},
"devin-acp": {
"cmd": ["python3", "{repo}/graff-evals/devin_acp_run.py", "{model}", "{prompt}"],
"answer": "stdout",
"usage": "graff-stderr",
"default_model": "gpt-6-1-sol-medium",
"note": "Devin CLI over ACP, the way the Harness desktop app drives it (devin acp, tools auto-approved). Token usage is summed from the per-call usage_update notifications and priced at Devin's list price. Needs `devin auth login`; DEVIN_BIN overrides the binary."
},
"agy-high": {
"cmd": ["agy", "-p", "{prompt}", "--output-format", "json", "--dangerously-skip-permissions", "--model", "{model}"],
"answer": "agy-json",
"usage": "agy-json",
"capabilities": [],
"model_map": {"gemini-4-argon": "gemini-4-argon-high"},
"default_model": "gemini-4-argon",
"note": "Antigravity CLI print mode on its own sign-in, high reasoning. input_tokens there excludes the cached part; calls are not reported."
},
"agy-low": {
"cmd": ["agy", "-p", "{prompt}", "--output-format", "json", "--dangerously-skip-permissions", "--model", "{model}"],
"answer": "agy-json",
"usage": "agy-json",
"capabilities": [],
"model_map": {"gemini-4-argon": "gemini-4-argon-low"},
"default_model": "gemini-4-argon",
"note": "Antigravity CLI print mode on its own sign-in, low reasoning. input_tokens there excludes the cached part; calls are not reported."
},
"graff-gateway-high": {
"cmd": ["python3", "{repo}/graff-evals/graff_json.py", "--effort", "high", "--model", "{model}", "{prompt}"],
"answer": "stdout",
"usage": "graff-stderr",
"capabilities": [],
"env": {"GRAFF_NO_TELEMETRY": "1", "GRAFF_BIN": "{repo}/zig-out/bin/graff"},
"model_prefix": "codegraff",
"default_model": "gemini-4-argon",
"note": "graff --json on the Codegraff gateway, set_effort high before the prompt (-p has no effort flag)"
},
"graff-gateway-default": {
"cmd": ["python3", "{repo}/graff-evals/graff_json.py", "--effort", "default", "--model", "{model}", "{prompt}"],
"answer": "stdout",
"usage": "graff-stderr",
"capabilities": [],
"env": {"GRAFF_NO_TELEMETRY": "1", "GRAFF_BIN": "{repo}/zig-out/bin/graff"},
"model_prefix": "codegraff",
"default_model": "gemini-4-argon",
"note": "graff --json on the Codegraff gateway, set_effort default before the prompt (-p has no effort flag)"
},
"graff-dev-claude-agent": {
"cmd": ["{repo}/zig-out/bin/graff", "-p", "{prompt}", "--model", "claude-agent/{model}", "--yolo"],
"answer": "stdout",
"usage": "graff-stderr",
"capabilities": [],
"env": {"GRAFF_CLAUDE_AGENT": "1", "GRAFF_MCP_CONFIG": "{repo}/graff-evals/no-global-mcp.json", "GRAFF_POST_DEADLINE_SECS": "90"},
"default_model": "sonnet",
"note": "graff on the Claude Agent provider (ADR 0309): the user's own `claude` login runs the model; graff's prompt, tools and loop. GRAFF_MCP_CONFIG names a file that does not exist: no global MCP servers, while a task's .mcp.json still loads (an existing empty file would switch MCP off entirely). Pass a Claude Code model alias (sonnet, opus, haiku)."
},
"graff-dev-claude-agent-ccprompt": {
"cmd": ["{repo}/zig-out/bin/graff", "-p", "{prompt}", "--model", "claude-agent/{model}", "--yolo"],
"answer": "stdout",
"usage": "graff-stderr",
"capabilities": [],
"env": {"GRAFF_CLAUDE_AGENT": "1", "GRAFF_CLAUDE_AGENT_PROMPT": "claude", "GRAFF_MCP_CONFIG": "{repo}/graff-evals/no-global-mcp.json", "GRAFF_POST_DEADLINE_SECS": "90"},
"default_model": "sonnet",
"note": "graff-dev-claude-agent keeping Claude Code's system prompt with graff's appended."
},
"claude-code-clean": {
"cmd": ["python3", "{repo}/graff-evals/claude_code_run.py", "{model}", "{prompt}"],
"answer": "stdout",
"usage": "graff-stderr",
"env": {"CLAUDE_CODE_RUN_ISOLATED": "1", "CLAUDE_CODE_DISABLE_CLAUDE_MDS": "1", "DISABLE_TELEMETRY": "1", "DISABLE_AUTOUPDATER": "1", "DISABLE_ERROR_REPORTING": "1"},
"default_model": "sonnet",
"note": "`claude -p` on its own login with no user settings, hooks, plugins, CLAUDE.md or global MCP: the isolation graff-dev-claude-agent runs with. Launch the runner without an enclosing Claude Code session's CLAUDE* variables."
}
}