Skip to content

graff — a fast agentic coding harness in Zig: multi-provider, MCP, workflows, DGM evolution loop, TS/Python SDKs

Python
10.9 KB
Raw Blame
"""Materialize a live-eval sandbox: sparse package checkout, no SPEC.md.

Refuses to start if the public check is already green on the parent tree
(G1 / G6). Historical July 2026 parents do not compile on Zig 0.17; those
tasks use a reconstructed parent (current package minus the fix) so the
gates are real. See artifacts/graff-evals-live/LIVE.md.
"""
from __future__ import annotations

import json
import os
import re
import shutil
import subprocess
import sys

EVALS = os.path.dirname(os.path.abspath(__file__))
REPO = os.path.dirname(EVALS)

# Never copy these into a live sandbox (spoilers + eval machinery).
SKIP_DIRS = {
    ".git", ".sandboxes", "graff-evals", "artifacts", "docs", "apps",
    "node_modules", "zig-out", "zig-cache", ".zig-cache", "__pycache__",
    "results", "evals",
}

# Enough of the package to `zig build test`. Not the function, not graff-evals.
DEFAULT_SPARSE = (
    "build.zig", "build.zig.zon", "src", "vendor", "TUI",
    "spec", "examples", "assets", "scripts",
    "gui/public/favicon.png",
)


def load_catalog():
    with open(os.path.join(EVALS, "live", "catalog.json")) as f:
        return json.load(f)


def manifest(task_id):
    cat = load_catalog()
    mined_path = os.path.join(EVALS, "live", "mined.json")  # ADR 0214, machine-managed
    mined = json.load(open(mined_path)) if os.path.exists(mined_path) else {}
    for t in (cat["published"] + cat.get("reserve", []) + cat.get("dropped", [])
              + mined.get("mined", []) + mined.get("candidates", [])):
        if t["id"] == task_id:
            return t
    raise SystemExit(f"unknown live task: {task_id}")


def _copy_sparse(src_root, dest, paths):
    for rel in paths:
        src = os.path.join(src_root, rel)
        dst = os.path.join(dest, rel)
        if not os.path.exists(src):
            continue
        os.makedirs(os.path.dirname(dst) or dest, exist_ok=True)
        if os.path.isdir(src):
            shutil.copytree(src, dst, dirs_exist_ok=True, ignore=shutil.ignore_patterns(
                "__pycache__", ".DS_Store", "*.pyc", "zig-cache", ".zig-cache"))
        else:
            shutil.copy2(src, dst)


def copy_local_package(dest, paths=None):
    _copy_sparse(REPO, dest, paths or DEFAULT_SPARSE)


def run_public(task, sandbox, timeout=None):
    script = os.path.join(EVALS, "live", task["id"], "check_public.sh")
    env = dict(os.environ, TASK_ROOT=EVALS)
    return subprocess.run(["/bin/sh", script], cwd=sandbox, capture_output=True,
                          text=True, timeout=timeout or task.get("check_timeout_s", 180),
                          env=env)


def run_hidden(task, sandbox, timeout=None):
    hidden = task.get("hidden")
    if not hidden:
        return subprocess.CompletedProcess([], 0, "", "")
    path = hidden if os.path.isabs(hidden) else os.path.join(EVALS, hidden)
    env = dict(os.environ, TASK_ROOT=EVALS)
    return subprocess.run(["/bin/sh", path], cwd=sandbox, capture_output=True,
                          text=True, timeout=timeout or task.get("check_timeout_s", 180),
                          env=env)


def _drop_build_cache(sandbox):
    for stale in (".zig-cache", "zig-out"):
        p = os.path.join(sandbox, stale)
        if os.path.exists(p):
            shutil.rmtree(p)


def apply_parent(task, sandbox):
    breaker = os.path.join(EVALS, "live", task["id"], "break_parent.py")
    if os.path.exists(breaker):
        subprocess.check_call([sys.executable, breaker, sandbox])
        _drop_build_cache(sandbox)


def apply_grease(task, sandbox):
    grease = os.path.join(EVALS, "live", task["id"], "grease.py")
    if not os.path.exists(grease):
        raise SystemExit(f"{task['id']}: missing grease.py (G3)")
    subprocess.check_call([sys.executable, grease, sandbox])
    _drop_build_cache(sandbox)


def spoilers_in(sandbox):
    """G5: SPEC.md or builder-brief text must not land in the exam tree."""
    hits = []
    skip = {".eval-answer.txt"}
    needles = ("SPEC.md",)
    text_needles = (
        "LIVE.md",
        "check_public.sh",
        "graff-evals/hidden",
        "this is the hidden case",
    )
    for root, dirs, files in os.walk(sandbox):
        dirs[:] = [d for d in dirs if d not in SKIP_DIRS and d != ".git"]
        for name in files:
            if name in skip or name.endswith((".png", ".o", ".a")):
                continue
            rel = os.path.relpath(os.path.join(root, name), sandbox)
            if name in needles or name == "SPEC.md":
                hits.append(rel)
                continue
            path = os.path.join(root, name)
            try:
                body = open(path, errors="replace").read(200_000)
            except OSError:
                continue
            for n in text_needles:
                if n in body:
                    hits.append(f"{rel}:{n}")
    return hits


def clone_historical_parent(task, dest):
    repo = task["repo"]
    sha = task["historical_parent"]
    cache_root = os.environ.get("GRAFF_LIVE_CACHE", "/tmp/graff-live-repos")
    cache = os.path.join(cache_root, repo.replace("/", "_"))
    url = f"https://github.com/{repo}.git"
    os.makedirs(cache_root, exist_ok=True)
    if not os.path.isdir(os.path.join(cache, ".git")):
        subprocess.check_call(["git", "clone", "--filter=blob:none", url, cache], timeout=180)
    subprocess.check_call(["git", "-C", cache, "fetch", "--filter=blob:none", "origin", sha], timeout=120)
    archive = subprocess.Popen(["git", "-C", cache, "archive", sha], stdout=subprocess.PIPE)
    subprocess.check_call(["tar", "-x", "-C", dest], stdin=archive.stdout)
    if archive.wait() != 0:
        raise SystemExit(f"git archive {sha} failed")
    pin = os.path.join(EVALS, "live", task["id"], "pin_public.py")
    if os.path.exists(pin):
        subprocess.check_call([sys.executable, pin, dest])


def archive_local(rev, dest, paths=None):
    """A historical tree straight from this checkout's git history (ADR 0214:
    mined tasks). Recent parents build on the pinned toolchain as they are."""
    os.makedirs(dest, exist_ok=True)
    # The repository's own instructions come along: a real graff session reads
    # them (for example, that a new test file must be registered to run).
    wanted = list(paths or DEFAULT_SPARSE) + ["AGENTS.md", "CLAUDE.md"]
    # Whatever that revision's build.zig embeds or builds from (for example a
    # stylesheet under apps/ before the desktop app left) must come along.
    build = subprocess.run(["git", "-C", REPO, "show", f"{rev}:build.zig"],
                           capture_output=True, text=True).stdout
    wanted += [p for p in re.findall(r'b\.path\("([^"]+)"\)', build) if p not in wanted]
    have = [p for p in wanted
            if subprocess.run(["git", "-C", REPO, "cat-file", "-e", f"{rev}:{p}"],
                              capture_output=True).returncode == 0]
    arch = subprocess.Popen(["git", "-C", REPO, "archive", rev, *have], stdout=subprocess.PIPE)
    subprocess.check_call(["tar", "-x", "-C", dest], stdin=arch.stdout)
    if arch.wait() != 0:
        raise SystemExit(f"git archive {rev} failed")
    # Fetched dependencies live in the untracked zig-pkg/ cache. Copy it (it
    # is small) so a sandbox build does not refetch every package, and an
    # agent working in the sandbox cannot write into this checkout's copy.
    pkg = os.path.join(REPO, "zig-pkg")
    if os.path.isdir(pkg) and not os.path.exists(os.path.join(dest, "zig-pkg")):
        shutil.copytree(pkg, os.path.join(dest, "zig-pkg"), symlinks=True)


def test_block(text, name):
    """The full `test "name" { … }` block, braces matched outside strings,
    char literals, `//` comments and `\\\\` multiline string lines."""
    head = f'test "{name}"'
    start = text.find(head)
    if start < 0:
        return None
    # Include the doc comments / blank line directly above? No: the block only.
    i = text.find("{", start + len(head))
    if i < 0:
        return None
    depth, n = 0, len(text)
    while i < n:
        c = text[i]
        if c == "/" and text.startswith("//", i):
            j = text.find("\n", i)
            i = n if j < 0 else j
            continue
        if c == "\\" and text.startswith("\\\\", i):
            j = text.find("\n", i)
            i = n if j < 0 else j
            continue
        if c in "\"'":
            j = i + 1
            while j < n and text[j] != c:
                j += 2 if text[j] == "\\" else 1
            i = j + 1
            continue
        if c == "{":
            depth += 1
        elif c == "}":
            depth -= 1
            if depth == 0:
                return text[start:i + 1] + "\n"
        i += 1
    return None


def named_case(task_id):
    """(file, test name) a mined task appended and told the model not to
    edit, or None for any other task."""
    try:
        task = manifest(task_id)
    except SystemExit:
        return None
    if task.get("source") != "local-git" or not task.get("public_file") or not task.get("public_filter"):
        return None
    return task["public_file"], task["public_filter"]


def append_case(sandbox, rel, inc, name):
    """Append a test block unless a test with that name is already there."""
    path = os.path.join(sandbox, rel)
    text = open(path).read() if os.path.exists(path) else ""
    if f'test "{name}"' in text:
        return
    with open(path, "a") as f:
        f.write(open(inc).read())


def setup(task_id, sandbox, allow_green=False):
    task = manifest(task_id)
    os.makedirs(sandbox, exist_ok=True)
    # Drop leftover exam files from a reused sandbox.
    for name in os.listdir(sandbox):
        if name.startswith("."):
            continue
        path = os.path.join(sandbox, name)
        if os.path.isdir(path):
            shutil.rmtree(path)
        else:
            os.remove(path)
    if task.get("source") == "clone":
        clone_historical_parent(task, sandbox)
    elif task.get("source") == "local-git":
        archive_local(task["historical_parent"], sandbox, task.get("sparse"))
        append_case(sandbox, task["public_file"],
                    os.path.join(EVALS, "live", task["id"], "public_case.inc"), task["public_filter"])
    else:
        copy_local_package(sandbox, task.get("sparse") or DEFAULT_SPARSE)
        apply_parent(task, sandbox)
    if spoilers_in(sandbox):
        raise SystemExit(f"G5: spoiler files in sandbox: {spoilers_in(sandbox)}")
    public = run_public(task, sandbox)
    if public.returncode == 0 and not allow_green:
        sys.stderr.write(
            f"setup_live: refuse {task_id}: parent is already green (G1/G6)\n")
        sys.stderr.write(public.stdout[-400:] + public.stderr[-400:])
        return 2
    return 0


def main(argv=None):
    argv = list(sys.argv[1:] if argv is None else argv)
    allow = False
    if "--allow-green" in argv:
        allow = True
        argv.remove("--allow-green")
    if len(argv) < 1:
        raise SystemExit("usage: live_setup.py <task-id> [sandbox] [--allow-green]")
    task_id = argv[0]
    sandbox = argv[1] if len(argv) > 1 else os.getcwd()
    raise SystemExit(setup(task_id, sandbox, allow_green=allow))


if __name__ == "__main__":
    main()