Skip to content

graff — a fast agentic coding harness in Zig: multi-provider, MCP, workflows, DGM evolution loop, TS/Python SDKs

Python
23.2 KB
Raw Blame
#!/usr/bin/env python3
"""Tier 2 of the internal eval set: model-backed harness behavior.

Tier 1 (scripts/eval-tier1.sh) proves the code compiles, the tests run, and the
generated artifacts match. It cannot tell you whether a real run *behaves*: that
a goal produces a checklist and can finish it, that an empty todo_write leaves
the list alone, that compaction does not lose completed work. Those need a model
in the loop, which is why this tier is not in the pre-push hook.

The cases are data - evals/harness_behavior.jsonl - and each one carries the
regression it guards. A case scripts the model's replies, so the default run is
offline and free: scripts/eval/mock_model.py replays the script on --port and
records what the harness sent back. The default port uses lmstudio; other ports
use a workspace-local OpenAI-compatible router. Point --provider/--model at a
real provider to run the same cases against one.

  python3 scripts/eval-tier2.py                       every case, scripted model
  python3 scripts/eval-tier2.py --only todo-empty-write-noop
  python3 scripts/eval-tier2.py --list
  python3 scripts/eval-tier2.py --dump goal-checklist-completes

A case can send structured setup controls with `controls_before_prompt`, such
as selecting a local vision model before its first user message.
`prompt_options` supplies explicit user-request fields; `github_fixture` installs
a local-only GitHub CLI with scripted head/check evidence.

Assertions a case can make:

  {"event": {...}}                        some emitted event matches this subset
  {"no_event": {...}}                     none does
  {"events_at_least": {"match": {...}, "count": N}}
  {"events_at_most":  {"match": {...}, "count": N}}
  {"request_contains": {"index": -1, "text": "..."}}   what the harness sent
  {"request_lacks":    {"index": -1, "text": "..."}}
  {"request_count": N}                     exact captured wire count (scripted only)
  {"stdout_equals": "..."}                 exact raw stdout, including newlines
  {"final_text_contains": "..."}
  {"file_equals": {"path": "relative/file", "content": "exact bytes"}}
  {"stderr_contains": "..."}                what the harness printed to stderr

A case that deliberately DIES declares the exit code it wants with
{"expected_exit": N} (default 0). Without it the runner auto-failed every
nonzero exit, which made the harness's own fatal paths untestable - the
loudest of them being run_budget.exhaustedFatal (#368), the last line a
one-shot run prints when the model-call pool runs dry. A timeout is never
an expected exit and always fails.

A case can set `verify_command` to an evaluator-owned argv executed after the
agent exits. A failed or timed-out verifier fails the case independently of prose.

A case may also carry {"workspace_seed": {"relative/path": "content", ...}}:
files written into the fresh workspace BEFORE graff boots, in scripted and
--provider modes alike - for state the harness reads at startup, like the
trajectory archive bench_priors folds. Under --provider the ScriptedModel
never runs, so request_contains/request_lacks are skipped and scoring rests
on the event and final-text asserts.
`claude_transcripts` seeds per-project Claude JSONL files and runs the explicit
import command before startup, for conversation import/resume regressions.
"""

from __future__ import annotations

import argparse
import json
import os
import pathlib
import shutil
import subprocess
import sys
import tempfile
import time
from typing import Any

REPO = pathlib.Path(__file__).resolve().parents[1]
CASES = REPO / "evals" / "harness_behavior.jsonl"
sys.path.insert(0, str(REPO / "scripts" / "eval"))
from mock_model import ScriptedModel  # noqa: E402
from github_fixture import prepare as prepare_github


def load_cases() -> list[dict[str, Any]]:
    cases = []
    for number, line in enumerate(CASES.read_text(encoding="utf-8").splitlines(), 1):
        line = line.strip()
        if not line or line.startswith("//"):
            continue
        try:
            cases.append(json.loads(line))
        except ValueError as exc:
            sys.exit(f"eval-tier2: {CASES}:{number} is not valid JSON: {exc}")
    return cases


def subset_matches(event: dict[str, Any], wanted: dict[str, Any]) -> bool:
    for key, value in wanted.items():
        if key not in event:
            return False
        if isinstance(value, dict):
            if not isinstance(event[key], dict) or not subset_matches(event[key], value):
                return False
        elif event[key] != value:
            return False
    return True


class Run:
    """One harness run: the events it emitted and the requests it sent."""

    def __init__(self, events: list[dict[str, Any]], requests: list[dict[str, Any]],
                 stderr: str, exit_code: int | None, files=None, verification=None, stdout="") -> None:
        self.events = events
        self.requests = requests
        self.stderr = stderr
        self.stdout = stdout
        self.exit_code = exit_code
        self.files = files or {}
        self.verification = verification

    def final_text(self) -> str:
        for event in reversed(self.events):
            if event.get("type") == "turn":
                return str(event.get("text", ""))
        return ""

    def request_text(self, index: int) -> str | None:
        if not self.requests:
            return None
        try:
            return json.dumps(self.requests[index])
        except IndexError:
            return None


def execute(case: dict[str, Any], graff: str, port: int,
            provider: str | None, model: str | None, evidence_dir: pathlib.Path | None = None) -> Run:
    scripted = ScriptedModel(case.get("script", []))
    if case.get("feedback_scenario"):
        if provider:
            raise ValueError("feedback lifecycle fixtures require the scripted model")
        from subagent_feedback_model import FeedbackModel
        scripted = FeedbackModel(case["feedback_scenario"], load_tools=case.get("feedback_load_tools", False))
    if "routing_reverse" in case:
        if provider:
            raise ValueError("routing isolation fixtures require the scripted model")
        from subagent_routing_model import RoutingModel
        scripted = RoutingModel(reverse=case["routing_reverse"])
    bound = 0 if provider else scripted.start(port)
    peer = None
    try:
        with tempfile.TemporaryDirectory(prefix="graff-tier2-") as workspace:
            env = {
                key: value for key, value in os.environ.items()
                if not key.endswith("_API_KEY")
            }
            env.update({
                "HOME": workspace,
                "LMSTUDIO_API_KEY": "local",
                "GRAFF_NO_TELEMETRY": "1",
                "GRAFF_FLEET": "off",
                "GRAFF_NO_SMOLIFY": "1",
                "NO_COLOR": "1",
            })
            # Per-case environment. A case that needs the harness to reach a
            # state it will not reach on its own - a tiny context window so a
            # SUBAGENT compacts, say - sets it here rather than in the shared
            # block, so one case cannot quietly change the others.
            env.update({k: str(v) for k, v in case.get("env", {}).items()})
            # Pre-boot world: state the harness reads at startup (the trajectory
            # archive folds in bench_priors before any scripted step could write it).
            for rel, content in case.get("workspace_seed", {}).items():
                dest = pathlib.Path(workspace) / rel
                dest.parent.mkdir(parents=True, exist_ok=True)
                dest.write_text(content, encoding="utf-8")
            selected_model = model or "lmstudio"
            if not provider and bound != 1234:
                # lmstudio has a fixed endpoint, not an LMSTUDIO_URL override.
                # Use the existing workspace router so chat AND catalog calls
                # stay on the bound loopback port (including --port 0).
                router = pathlib.Path(workspace) / ".graff" / ".config.router"
                router.parent.mkdir(parents=True, exist_ok=True)
                router.write_text(json.dumps({
                    "id": "tier2", "base_url": f"http://127.0.0.1:{bound}/v1",
                    "env_key": "TIER2_API_KEY", "default_model": selected_model,
                }), encoding="utf-8")
                env["TIER2_API_KEY"] = "local"
                selected_model = f"tier2/{selected_model}"
            if case.get("claude_transcripts"):
                canonical = str(pathlib.Path(workspace).resolve())
                slug = "".join(c if c.isascii() and c.isalnum() else "-" for c in canonical)
                project = pathlib.Path(workspace) / ".claude" / "projects" / slug
                project.mkdir(parents=True)
                for name, content in case["claude_transcripts"].items():
                    (project / name).write_text(content, encoding="utf-8")
                subprocess.run([graff, "mcp", "import"], cwd=workspace, env=env,
                               check=True, capture_output=True, text=True, timeout=30)
            # Real linked-worktree topology for ownership/checkpoint cases.
            if case.get("nested_worktree"):
                for command in [
                    ["git", "init", "-q"],
                    ["git", "-c", "user.name=Fixture", "-c", "[email protected]", "commit", "--allow-empty", "-qm", "fixture"],
                    ["git", "worktree", "add", "--detach", "nested"],
                ]:
                    subprocess.run(command, cwd=workspace, env=env, check=True, capture_output=True)
            if "github_fixture" in case:
                prepare_github(pathlib.Path(workspace), env, case["github_fixture"])
            # A genuinely live second graff in the same workspace, for #469
            # presence cases: the checkpoint only exists while a co-resident
            # session does, and a real peer announces (and probes) better than
            # any fixture — a forged record can never carry a live start-id.
            if case.get("live_peer"):
                peer = subprocess.Popen(
                    [graff, "--json", "--yolo", "--model", selected_model],
                    cwd=str(pathlib.Path(workspace) / "nested") if case.get("peer_in_nested_worktree") else workspace, env=env, stdin=subprocess.PIPE,
                    stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, text=True)
                live_dir = pathlib.Path(workspace) / ".graff" / "live"
                deadline = time.time() + 30
                while time.time() < deadline:
                    if live_dir.is_dir() and any(live_dir.glob("*.json")):
                        break
                    if peer.poll() is not None:
                        break
                    time.sleep(0.1)
                if peer.poll() is not None or not live_dir.is_dir() or not any(live_dir.glob("*.json")):
                    raise RuntimeError("Live-peer fixture did not announce before the deadline")
            argv = [graff, "--json", "--yolo", "--model", selected_model]
            if provider:
                argv += ["--subagent-provider", provider]
            argv += list(case.get("args", []))

            stdin_lines = [json.dumps(control) for control in case.get("controls_before_prompt", [])]
            stdin_lines.append(json.dumps({"type": "user", "text": case["prompt"], **case.get("prompt_options", {})}))
            for control in case.get("controls_after_turn", []):
                stdin_lines.append(json.dumps(control))
            if case.get("second_prompt"):
                stdin_lines.append(json.dumps({"type": "user", "text": case["second_prompt"]}))

            try:
                done = subprocess.run(
                    argv, cwd=workspace, env=env, text=True, capture_output=True,
                    input="\n".join(stdin_lines) + "\n",
                    timeout=case.get("timeout_s", 60),
                )
                stdout, stderr, code = done.stdout, done.stderr, done.returncode
            except subprocess.TimeoutExpired as exc:
                stdout = exc.stdout if isinstance(exc.stdout, str) else (exc.stdout or b"").decode("utf-8", "ignore")
                stderr = exc.stderr if isinstance(exc.stderr, str) else (exc.stderr or b"").decode("utf-8", "ignore")
                code = None

            events = []
            for line in stdout.splitlines():
                line = line.strip()
                if line.startswith("{"):
                    try:
                        events.append(json.loads(line))
                    except ValueError:
                        pass
            files = {}
            for claim in case.get("assert", []):
                if "file_equals" not in claim:
                    continue
                rel = claim["file_equals"]["path"]
                path = pathlib.Path(workspace) / rel
                try:
                    path.resolve().relative_to(pathlib.Path(workspace).resolve())
                    if path.stat().st_size > 1024 * 1024:
                        raise ValueError("asserted file exceeds 1 MiB")
                    files[rel] = path.read_text(encoding="utf-8")
                except (OSError, ValueError):
                    files[rel] = None
            verification = None
            if case.get("verify_command"):
                # Evaluator-owned command, executed after the agent stops. Its
                # assertions are not a file the agent can rewrite to pass.
                try:
                    checked = subprocess.run(case["verify_command"], cwd=workspace,
                                             env=env, capture_output=True, text=True,
                                             timeout=15)
                    verification = dict(exit_code=checked.returncode,
                                        stdout=checked.stdout, stderr=checked.stderr)
                except subprocess.TimeoutExpired:
                    verification = dict(exit_code=None, stdout="", stderr="verifier timed out")
            if evidence_dir is not None:
                destination = evidence_dir / case["id"]
                destination.mkdir(parents=True, exist_ok=True, mode=0o700)
                (destination / "result.json").write_text(json.dumps(dict(
                    events=events, requests=list(scripted.requests), files=files,
                    verification=verification, exit_code=code, stdout=stdout), indent=2))
                (destination / "stderr.log").write_text(stderr)
                for folder in ("traces", "trajectories", "sessions"):
                    source = pathlib.Path(workspace) / ".graff" / folder
                    if source.exists():
                        shutil.copytree(source, destination / folder, dirs_exist_ok=True)
            return Run(events, list(scripted.requests), stderr, code, files, verification, stdout)
    finally:
        if peer is not None:
            peer.kill()
        if bound:
            scripted.stop()
            # Let the socket clear before the next case reuses the port.
            time.sleep(0.05)


def evaluate(case: dict[str, Any], run: Run, live: bool = False) -> list[str]:
    failures = []
    if case.get("verify_command"):
        if run.verification is None or run.verification.get("exit_code") != 0:
            failures.append(f"independent workspace verification failed: {run.verification!r}")
    for claim in case.get("assert", []):
        (kind, spec), = claim.items()
        if kind == "file_equals":
            if run.files.get(spec["path"]) != spec["content"]:
                failures.append(f"workspace file {spec['path']!r} does not match the required outcome")
        elif kind == "event":
            if not any(subset_matches(e, spec) for e in run.events):
                failures.append(f"no event matched {json.dumps(spec)}")
        elif kind == "no_event":
            if any(subset_matches(e, spec) for e in run.events):
                failures.append(f"an event matched {json.dumps(spec)} and none should have")
        elif kind in ("events_at_least", "events_at_most"):
            seen = sum(1 for e in run.events if subset_matches(e, spec["match"]))
            want = spec["count"]
            if kind == "events_at_least" and seen < want:
                failures.append(
                    f"{seen} event(s) matched {json.dumps(spec['match'])}, wanted at least {want}")
            if kind == "events_at_most" and seen > want:
                failures.append(
                    f"{seen} event(s) matched {json.dumps(spec['match'])}, wanted at most {want}")
        elif kind == "request_tool_calls_at_most":
            if live:
                continue
            try:
                messages = run.requests[spec["index"]]["messages"]
                count = sum(call.get("function", {}).get("name") == spec["name"]
                            for message in messages for call in message.get("tool_calls", []))
                if count > spec["count"]:
                    failures.append(f"request[{spec['index']}] already contains {count} {spec['name']} calls")
            except (IndexError, KeyError):
                failures.append("missing message history for tool-order assertion")
        elif kind in ("request_contains", "request_lacks"):
            if live:
                continue  # a live provider records no requests; scripted-only surface
            body = run.request_text(spec.get("index", -1))
            if body is None:
                failures.append(f"the harness sent no request at index {spec.get('index', -1)}")
                continue
            present = spec["text"] in body
            if kind == "request_contains" and not present:
                failures.append(
                    f"request[{spec.get('index', -1)}] never mentioned {spec['text']!r}")
            if kind == "request_lacks" and present:
                failures.append(
                    f"request[{spec.get('index', -1)}] mentioned {spec['text']!r} and should not have")
        elif kind == "request_count":
            if not live and len(run.requests) != spec:
                failures.append(f"captured {len(run.requests)} requests, wanted exactly {spec}")
        elif kind == "stdout_equals":
            if run.stdout != spec:
                failures.append(f"stdout was {run.stdout!r}, wanted exactly {spec!r}")
        elif kind == "final_text_contains":
            if spec not in run.final_text():
                failures.append(f"the final answer was {run.final_text()!r}, wanted {spec!r}")
        elif kind == "stderr_contains":
            # The harness's own last words. stdout carries the event stream, so
            # anything the harness says on the way out - a fatal line, a phase
            # SKIPPED note, a CLI refusal - is only visible here.
            if spec not in run.stderr:
                failures.append(f"stderr never mentioned {spec!r}")
        else:
            failures.append(f"unknown assertion {kind!r}")
    return failures


def main() -> None:
    parser = argparse.ArgumentParser(description=__doc__,
                                     formatter_class=argparse.RawDescriptionHelpFormatter)
    parser.add_argument("--graff", default=str(REPO / "zig-out" / "bin" / "graff"))
    parser.add_argument("--only", action="append", default=[], help="case id (repeatable)")
    parser.add_argument("--list", action="store_true")
    parser.add_argument("--dump", help="run one case and print its events and requests")
    parser.add_argument("--port", type=int, default=1234, help="scripted-model port (0 selects a free port)")
    parser.add_argument("--evidence-dir", type=pathlib.Path, help="retain private events, requests, outcomes and harness traces")
    parser.add_argument("--provider", help="run against a real provider instead of the script")
    parser.add_argument("--model", help="model name for --provider")
    args = parser.parse_args()

    cases = load_cases()
    if args.list:
        for case in cases:
            print(f"{case['id']}\n    {case['why']}")
        return

    wanted = set(args.only) | ({args.dump} if args.dump else set())
    if wanted:
        known = {case["id"] for case in cases}
        missing = wanted - known
        if missing:
            sys.exit(f"eval-tier2: no case named {', '.join(sorted(missing))} (try --list)")
        cases = [case for case in cases if case["id"] in wanted]

    if not pathlib.Path(args.graff).exists():
        sys.exit(f"eval-tier2: {args.graff} does not exist - run `zig build` first")

    if args.provider:
        skipped = [case["id"] for case in cases if case.get("scripted_only")]
        cases = [case for case in cases if not case.get("scripted_only")]
        if skipped:
            print("scripted-only lifecycle fixtures skipped: " + ", ".join(skipped))
        if not cases:
            sys.exit("eval-tier2: selected cases require the local scripted model; omit --provider")

    if args.provider:
        print(f"tier 2: {len(cases)} case(s) against {args.provider}"
              f"/{args.model or 'default'} - this spends real calls"
              " (request_* asserts skipped: no scripted request log)\n")
    else:
        print(f"tier 2: {len(cases)} case(s) against the scripted model"
              f" on 127.0.0.1:{args.port} - offline, no spend\n")

    failed = []
    for case in cases:
        run = execute(case, args.graff, args.port, args.provider, args.model, args.evidence_dir)
        if args.dump:
            print(json.dumps({"events": run.events, "requests": run.requests,
                              "exit_code": run.exit_code, "stderr": run.stderr[-2000:],
                              "files": run.files, "verification": run.verification, "stdout": run.stdout},
                             indent=2))
            return
        problems = evaluate(case, run, live=bool(args.provider))
        # A harness that crashed, hung, or died instantly must FAIL the case on
        # its own, not merely annotate someone else's failure. Without this a
        # case whose assertions are all upper bounds passes against a binary
        # that never ran at all. The default is still 0; a case that means to
        # exercise a fatal path names its code with "expected_exit", which is
        # the only way the harness's own deliberate deaths become testable.
        # A timeout is never expected - it is the absence of a decision.
        want = case.get("expected_exit", 0)
        if run.exit_code is None:
            problems.append("graff did not exit (timed out)")
        elif run.exit_code != want:
            problems.append(f"graff exited {run.exit_code}, wanted {want}")
        if problems:
            failed.append(case["id"])
            print(f"  FAIL  {case['id']}")
            print(f"        guards: {case['why']}")
            for problem in problems:
                print(f"        - {problem}")
            if run.exit_code not in (want, None):
                print(f"        graff exited {run.exit_code} (wanted {want}); stderr tail:")
                print("        " + run.stderr.strip().splitlines()[-1] if run.stderr.strip() else "")
        else:
            print(f"  pass  {case['id']}")

    print()
    if failed:
        print(f"tier 2 red - {len(failed)} of {len(cases)} case(s) failed: {', '.join(failed)}")
        print("rerun one with: python3 scripts/eval-tier2.py --only <id>")
        print("see everything it did: python3 scripts/eval-tier2.py --dump <id>")
        sys.exit(1)
    print(f"tier 2 green - {len(cases)} case(s)")


if __name__ == "__main__":
    main()