graff — a fast agentic coding harness in Zig: multi-provider, MCP, workflows, DGM evolution loop, TS/Python SDKs
Python
#!/usr/bin/env python3
"""Tier 2 of the internal eval set: model-backed harness behavior.
Tier 1 (scripts/eval-tier1.sh) proves the code compiles, the tests run, and the
generated artifacts match. It cannot tell you whether a real run *behaves*: that
a goal produces a checklist and can finish it, that an empty todo_write leaves
the list alone, that compaction does not lose completed work. Those need a model
in the loop, which is why this tier is not in the pre-push hook.
The cases are data - evals/harness_behavior.jsonl - and each one carries the
regression it guards. A case scripts the model's replies, so the default run is
offline and free: scripts/eval/mock_model.py replays the script on --port and
records what the harness sent back. The default port uses lmstudio; other ports
use a workspace-local OpenAI-compatible router. Point --provider/--model at a
real provider to run the same cases against one.
python3 scripts/eval-tier2.py every case, scripted model
python3 scripts/eval-tier2.py --only todo-empty-write-noop
python3 scripts/eval-tier2.py --list
python3 scripts/eval-tier2.py --dump goal-checklist-completes
A case can send structured setup controls with `controls_before_prompt`, such
as selecting a local vision model before its first user message.
`prompt_options` supplies explicit user-request fields; `github_fixture` installs
a local-only GitHub CLI with scripted head/check evidence.
Assertions a case can make:
{"event": {...}} some emitted event matches this subset
{"no_event": {...}} none does
{"events_at_least": {"match": {...}, "count": N}}
{"events_at_most": {"match": {...}, "count": N}}
{"request_contains": {"index": -1, "text": "..."}} what the harness sent
{"request_lacks": {"index": -1, "text": "..."}}
{"request_count": N} exact captured wire count (scripted only)
{"stdout_equals": "..."} exact raw stdout, including newlines
{"final_text_contains": "..."}
{"file_equals": {"path": "relative/file", "content": "exact bytes"}}
{"stderr_contains": "..."} what the harness printed to stderr
A case that deliberately DIES declares the exit code it wants with
{"expected_exit": N} (default 0). Without it the runner auto-failed every
nonzero exit, which made the harness's own fatal paths untestable - the
loudest of them being run_budget.exhaustedFatal (#368), the last line a
one-shot run prints when the model-call pool runs dry. A timeout is never
an expected exit and always fails.
A case can set `verify_command` to an evaluator-owned argv executed after the
agent exits. A failed or timed-out verifier fails the case independently of prose.
A case may also carry {"workspace_seed": {"relative/path": "content", ...}}:
files written into the fresh workspace BEFORE graff boots, in scripted and
--provider modes alike - for state the harness reads at startup, like the
trajectory archive bench_priors folds. Under --provider the ScriptedModel
never runs, so request_contains/request_lacks are skipped and scoring rests
on the event and final-text asserts.
`claude_transcripts` seeds per-project Claude JSONL files and runs the explicit
import command before startup, for conversation import/resume regressions.
"""
from __future__ import annotations
import argparse
import json
import os
import pathlib
import shutil
import subprocess
import sys
import tempfile
import time
from typing import Any
REPO = pathlib.Path(__file__).resolve().parents[1]
CASES = REPO / "evals" / "harness_behavior.jsonl"
sys.path.insert(0, str(REPO / "scripts" / "eval"))
from mock_model import ScriptedModel # noqa: E402
from github_fixture import prepare as prepare_github
def load_cases() -> list[dict[str, Any]]:
cases = []
for number, line in enumerate(CASES.read_text(encoding="utf-8").splitlines(), 1):
line = line.strip()
if not line or line.startswith("//"):
continue
try:
cases.append(json.loads(line))
except ValueError as exc:
sys.exit(f"eval-tier2: {CASES}:{number} is not valid JSON: {exc}")
return cases
def subset_matches(event: dict[str, Any], wanted: dict[str, Any]) -> bool:
for key, value in wanted.items():
if key not in event:
return False
if isinstance(value, dict):
if not isinstance(event[key], dict) or not subset_matches(event[key], value):
return False
elif event[key] != value:
return False
return True
class Run:
"""One harness run: the events it emitted and the requests it sent."""
def __init__(self, events: list[dict[str, Any]], requests: list[dict[str, Any]],
stderr: str, exit_code: int | None, files=None, verification=None, stdout="") -> None:
self.events = events
self.requests = requests
self.stderr = stderr
self.stdout = stdout
self.exit_code = exit_code
self.files = files or {}
self.verification = verification
def final_text(self) -> str:
for event in reversed(self.events):
if event.get("type") == "turn":
return str(event.get("text", ""))
return ""
def request_text(self, index: int) -> str | None:
if not self.requests:
return None
try:
return json.dumps(self.requests[index])
except IndexError:
return None
def execute(case: dict[str, Any], graff: str, port: int,
provider: str | None, model: str | None, evidence_dir: pathlib.Path | None = None) -> Run:
scripted = ScriptedModel(case.get("script", []))
if case.get("feedback_scenario"):
if provider:
raise ValueError("feedback lifecycle fixtures require the scripted model")
from subagent_feedback_model import FeedbackModel
scripted = FeedbackModel(case["feedback_scenario"], load_tools=case.get("feedback_load_tools", False))
if "routing_reverse" in case:
if provider:
raise ValueError("routing isolation fixtures require the scripted model")
from subagent_routing_model import RoutingModel
scripted = RoutingModel(reverse=case["routing_reverse"])
bound = 0 if provider else scripted.start(port)
peer = None
try:
with tempfile.TemporaryDirectory(prefix="graff-tier2-") as workspace:
env = {
key: value for key, value in os.environ.items()
if not key.endswith("_API_KEY")
}
env.update({
"HOME": workspace,
"LMSTUDIO_API_KEY": "local",
"GRAFF_NO_TELEMETRY": "1",
"GRAFF_FLEET": "off",
"GRAFF_NO_SMOLIFY": "1",
"NO_COLOR": "1",
})
# Per-case environment. A case that needs the harness to reach a
# state it will not reach on its own - a tiny context window so a
# SUBAGENT compacts, say - sets it here rather than in the shared
# block, so one case cannot quietly change the others.
env.update({k: str(v) for k, v in case.get("env", {}).items()})
# Pre-boot world: state the harness reads at startup (the trajectory
# archive folds in bench_priors before any scripted step could write it).
for rel, content in case.get("workspace_seed", {}).items():
dest = pathlib.Path(workspace) / rel
dest.parent.mkdir(parents=True, exist_ok=True)
dest.write_text(content, encoding="utf-8")
selected_model = model or "lmstudio"
if not provider and bound != 1234:
# lmstudio has a fixed endpoint, not an LMSTUDIO_URL override.
# Use the existing workspace router so chat AND catalog calls
# stay on the bound loopback port (including --port 0).
router = pathlib.Path(workspace) / ".graff" / ".config.router"
router.parent.mkdir(parents=True, exist_ok=True)
router.write_text(json.dumps({
"id": "tier2", "base_url": f"http://127.0.0.1:{bound}/v1",
"env_key": "TIER2_API_KEY", "default_model": selected_model,
}), encoding="utf-8")
env["TIER2_API_KEY"] = "local"
selected_model = f"tier2/{selected_model}"
if case.get("claude_transcripts"):
canonical = str(pathlib.Path(workspace).resolve())
slug = "".join(c if c.isascii() and c.isalnum() else "-" for c in canonical)
project = pathlib.Path(workspace) / ".claude" / "projects" / slug
project.mkdir(parents=True)
for name, content in case["claude_transcripts"].items():
(project / name).write_text(content, encoding="utf-8")
subprocess.run([graff, "mcp", "import"], cwd=workspace, env=env,
check=True, capture_output=True, text=True, timeout=30)
# Real linked-worktree topology for ownership/checkpoint cases.
if case.get("nested_worktree"):
for command in [
["git", "init", "-q"],
["git", "-c", "user.name=Fixture", "-c", "[email protected]", "commit", "--allow-empty", "-qm", "fixture"],
["git", "worktree", "add", "--detach", "nested"],
]:
subprocess.run(command, cwd=workspace, env=env, check=True, capture_output=True)
if "github_fixture" in case:
prepare_github(pathlib.Path(workspace), env, case["github_fixture"])
# A genuinely live second graff in the same workspace, for #469
# presence cases: the checkpoint only exists while a co-resident
# session does, and a real peer announces (and probes) better than
# any fixture — a forged record can never carry a live start-id.
if case.get("live_peer"):
peer = subprocess.Popen(
[graff, "--json", "--yolo", "--model", selected_model],
cwd=str(pathlib.Path(workspace) / "nested") if case.get("peer_in_nested_worktree") else workspace, env=env, stdin=subprocess.PIPE,
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, text=True)
live_dir = pathlib.Path(workspace) / ".graff" / "live"
deadline = time.time() + 30
while time.time() < deadline:
if live_dir.is_dir() and any(live_dir.glob("*.json")):
break
if peer.poll() is not None:
break
time.sleep(0.1)
if peer.poll() is not None or not live_dir.is_dir() or not any(live_dir.glob("*.json")):
raise RuntimeError("Live-peer fixture did not announce before the deadline")
argv = [graff, "--json", "--yolo", "--model", selected_model]
if provider:
argv += ["--subagent-provider", provider]
argv += list(case.get("args", []))
stdin_lines = [json.dumps(control) for control in case.get("controls_before_prompt", [])]
stdin_lines.append(json.dumps({"type": "user", "text": case["prompt"], **case.get("prompt_options", {})}))
for control in case.get("controls_after_turn", []):
stdin_lines.append(json.dumps(control))
if case.get("second_prompt"):
stdin_lines.append(json.dumps({"type": "user", "text": case["second_prompt"]}))
try:
done = subprocess.run(
argv, cwd=workspace, env=env, text=True, capture_output=True,
input="\n".join(stdin_lines) + "\n",
timeout=case.get("timeout_s", 60),
)
stdout, stderr, code = done.stdout, done.stderr, done.returncode
except subprocess.TimeoutExpired as exc:
stdout = exc.stdout if isinstance(exc.stdout, str) else (exc.stdout or b"").decode("utf-8", "ignore")
stderr = exc.stderr if isinstance(exc.stderr, str) else (exc.stderr or b"").decode("utf-8", "ignore")
code = None
events = []
for line in stdout.splitlines():
line = line.strip()
if line.startswith("{"):
try:
events.append(json.loads(line))
except ValueError:
pass
files = {}
for claim in case.get("assert", []):
if "file_equals" not in claim:
continue
rel = claim["file_equals"]["path"]
path = pathlib.Path(workspace) / rel
try:
path.resolve().relative_to(pathlib.Path(workspace).resolve())
if path.stat().st_size > 1024 * 1024:
raise ValueError("asserted file exceeds 1 MiB")
files[rel] = path.read_text(encoding="utf-8")
except (OSError, ValueError):
files[rel] = None
verification = None
if case.get("verify_command"):
# Evaluator-owned command, executed after the agent stops. Its
# assertions are not a file the agent can rewrite to pass.
try:
checked = subprocess.run(case["verify_command"], cwd=workspace,
env=env, capture_output=True, text=True,
timeout=15)
verification = dict(exit_code=checked.returncode,
stdout=checked.stdout, stderr=checked.stderr)
except subprocess.TimeoutExpired:
verification = dict(exit_code=None, stdout="", stderr="verifier timed out")
if evidence_dir is not None:
destination = evidence_dir / case["id"]
destination.mkdir(parents=True, exist_ok=True, mode=0o700)
(destination / "result.json").write_text(json.dumps(dict(
events=events, requests=list(scripted.requests), files=files,
verification=verification, exit_code=code, stdout=stdout), indent=2))
(destination / "stderr.log").write_text(stderr)
for folder in ("traces", "trajectories", "sessions"):
source = pathlib.Path(workspace) / ".graff" / folder
if source.exists():
shutil.copytree(source, destination / folder, dirs_exist_ok=True)
return Run(events, list(scripted.requests), stderr, code, files, verification, stdout)
finally:
if peer is not None:
peer.kill()
if bound:
scripted.stop()
# Let the socket clear before the next case reuses the port.
time.sleep(0.05)
def evaluate(case: dict[str, Any], run: Run, live: bool = False) -> list[str]:
failures = []
if case.get("verify_command"):
if run.verification is None or run.verification.get("exit_code") != 0:
failures.append(f"independent workspace verification failed: {run.verification!r}")
for claim in case.get("assert", []):
(kind, spec), = claim.items()
if kind == "file_equals":
if run.files.get(spec["path"]) != spec["content"]:
failures.append(f"workspace file {spec['path']!r} does not match the required outcome")
elif kind == "event":
if not any(subset_matches(e, spec) for e in run.events):
failures.append(f"no event matched {json.dumps(spec)}")
elif kind == "no_event":
if any(subset_matches(e, spec) for e in run.events):
failures.append(f"an event matched {json.dumps(spec)} and none should have")
elif kind in ("events_at_least", "events_at_most"):
seen = sum(1 for e in run.events if subset_matches(e, spec["match"]))
want = spec["count"]
if kind == "events_at_least" and seen < want:
failures.append(
f"{seen} event(s) matched {json.dumps(spec['match'])}, wanted at least {want}")
if kind == "events_at_most" and seen > want:
failures.append(
f"{seen} event(s) matched {json.dumps(spec['match'])}, wanted at most {want}")
elif kind == "request_tool_calls_at_most":
if live:
continue
try:
messages = run.requests[spec["index"]]["messages"]
count = sum(call.get("function", {}).get("name") == spec["name"]
for message in messages for call in message.get("tool_calls", []))
if count > spec["count"]:
failures.append(f"request[{spec['index']}] already contains {count} {spec['name']} calls")
except (IndexError, KeyError):
failures.append("missing message history for tool-order assertion")
elif kind in ("request_contains", "request_lacks"):
if live:
continue # a live provider records no requests; scripted-only surface
body = run.request_text(spec.get("index", -1))
if body is None:
failures.append(f"the harness sent no request at index {spec.get('index', -1)}")
continue
present = spec["text"] in body
if kind == "request_contains" and not present:
failures.append(
f"request[{spec.get('index', -1)}] never mentioned {spec['text']!r}")
if kind == "request_lacks" and present:
failures.append(
f"request[{spec.get('index', -1)}] mentioned {spec['text']!r} and should not have")
elif kind == "request_count":
if not live and len(run.requests) != spec:
failures.append(f"captured {len(run.requests)} requests, wanted exactly {spec}")
elif kind == "stdout_equals":
if run.stdout != spec:
failures.append(f"stdout was {run.stdout!r}, wanted exactly {spec!r}")
elif kind == "final_text_contains":
if spec not in run.final_text():
failures.append(f"the final answer was {run.final_text()!r}, wanted {spec!r}")
elif kind == "stderr_contains":
# The harness's own last words. stdout carries the event stream, so
# anything the harness says on the way out - a fatal line, a phase
# SKIPPED note, a CLI refusal - is only visible here.
if spec not in run.stderr:
failures.append(f"stderr never mentioned {spec!r}")
else:
failures.append(f"unknown assertion {kind!r}")
return failures
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument("--graff", default=str(REPO / "zig-out" / "bin" / "graff"))
parser.add_argument("--only", action="append", default=[], help="case id (repeatable)")
parser.add_argument("--list", action="store_true")
parser.add_argument("--dump", help="run one case and print its events and requests")
parser.add_argument("--port", type=int, default=1234, help="scripted-model port (0 selects a free port)")
parser.add_argument("--evidence-dir", type=pathlib.Path, help="retain private events, requests, outcomes and harness traces")
parser.add_argument("--provider", help="run against a real provider instead of the script")
parser.add_argument("--model", help="model name for --provider")
args = parser.parse_args()
cases = load_cases()
if args.list:
for case in cases:
print(f"{case['id']}\n {case['why']}")
return
wanted = set(args.only) | ({args.dump} if args.dump else set())
if wanted:
known = {case["id"] for case in cases}
missing = wanted - known
if missing:
sys.exit(f"eval-tier2: no case named {', '.join(sorted(missing))} (try --list)")
cases = [case for case in cases if case["id"] in wanted]
if not pathlib.Path(args.graff).exists():
sys.exit(f"eval-tier2: {args.graff} does not exist - run `zig build` first")
if args.provider:
skipped = [case["id"] for case in cases if case.get("scripted_only")]
cases = [case for case in cases if not case.get("scripted_only")]
if skipped:
print("scripted-only lifecycle fixtures skipped: " + ", ".join(skipped))
if not cases:
sys.exit("eval-tier2: selected cases require the local scripted model; omit --provider")
if args.provider:
print(f"tier 2: {len(cases)} case(s) against {args.provider}"
f"/{args.model or 'default'} - this spends real calls"
" (request_* asserts skipped: no scripted request log)\n")
else:
print(f"tier 2: {len(cases)} case(s) against the scripted model"
f" on 127.0.0.1:{args.port} - offline, no spend\n")
failed = []
for case in cases:
run = execute(case, args.graff, args.port, args.provider, args.model, args.evidence_dir)
if args.dump:
print(json.dumps({"events": run.events, "requests": run.requests,
"exit_code": run.exit_code, "stderr": run.stderr[-2000:],
"files": run.files, "verification": run.verification, "stdout": run.stdout},
indent=2))
return
problems = evaluate(case, run, live=bool(args.provider))
# A harness that crashed, hung, or died instantly must FAIL the case on
# its own, not merely annotate someone else's failure. Without this a
# case whose assertions are all upper bounds passes against a binary
# that never ran at all. The default is still 0; a case that means to
# exercise a fatal path names its code with "expected_exit", which is
# the only way the harness's own deliberate deaths become testable.
# A timeout is never expected - it is the absence of a decision.
want = case.get("expected_exit", 0)
if run.exit_code is None:
problems.append("graff did not exit (timed out)")
elif run.exit_code != want:
problems.append(f"graff exited {run.exit_code}, wanted {want}")
if problems:
failed.append(case["id"])
print(f" FAIL {case['id']}")
print(f" guards: {case['why']}")
for problem in problems:
print(f" - {problem}")
if run.exit_code not in (want, None):
print(f" graff exited {run.exit_code} (wanted {want}); stderr tail:")
print(" " + run.stderr.strip().splitlines()[-1] if run.stderr.strip() else "")
else:
print(f" pass {case['id']}")
print()
if failed:
print(f"tier 2 red - {len(failed)} of {len(cases)} case(s) failed: {', '.join(failed)}")
print("rerun one with: python3 scripts/eval-tier2.py --only <id>")
print("see everything it did: python3 scripts/eval-tier2.py --dump <id>")
sys.exit(1)
print(f"tier 2 green - {len(cases)} case(s)")
if __name__ == "__main__":
main()