Skip to content

graff — a fast agentic coding harness in Zig: multi-provider, MCP, workflows, DGM evolution loop, TS/Python SDKs

Python
14.3 KB
Raw Blame
#!/usr/bin/env bash
# Tier 1 of the internal eval set: every check here is deterministic, offline,
# and free. No provider calls, no network, no model in the loop. This is what
# the pre-push hook runs, so it has to stay honest and it has to stay fast:
# a warm fmt/reach/sdk subset is seconds; a post-src run rebuilds zig and
# the engine integration probes (minutes).
#
# Tier 2 is the model-backed behavioral eval set (scripts/eval-tier2.py). It is
# deliberately NOT in the hook: it spends turns and it is slower.
#
#   scripts/eval-tier1.sh              run every check
#   scripts/eval-tier1.sh --list       show the check names
#   scripts/eval-tier1.sh --only sdk   run one check
#   scripts/eval-tier1.sh --only fmt,reach   run a comma-separated list (#641)
#
set -uo pipefail

repo_root=$(git -C "$(dirname "${BASH_SOURCE[0]}")/.." rev-parse --show-toplevel)
cd "$repo_root"

# git exports GIT_DIR (and friends) to hooks. Anything this script spawns —
# the test suite's git-fixture tests, graff itself in the sdk check — inherits
# them, and a child's `git init`/`git commit` in its OWN tmpdir then operates
# on THIS repo instead: `git init` re-marks the checkout bare (core.bare=true)
# and `git commit` lands a stray "fixture" commit on the current branch, with
# the tmpdir as the work tree. Scrub the hook environment so child git
# processes discover their repo from their cwd like they expect.
unset GIT_DIR GIT_WORK_TREE GIT_INDEX_FILE GIT_COMMON_DIR GIT_OBJECT_DIRECTORY GIT_PREFIX

CHECKS=(fmt lines spec reach build shell tests terminal invariants sdk)

usage() {
  cat <<'EOF'
usage: scripts/eval-tier1.sh [--only <check[,check...]>] [--list]

checks, in order:
  fmt         zig fmt --check src build.zig
  lines       scripts/check-zig-lines.sh (600-line ceiling, AGENTS.md)
  spec        spec/conformance.py (kernel properties + fixture staleness)
  reach       every file that declares tests is reachable from the test root
  build       zig build
  shell       durable shell IDs and stale-handle isolation after a crash
  tests       zig build test, and the suite count never shrinks
  terminal    PTY cleanup, line-REPL prompt reflow, and live thinking controls
  invariants  the named goal/loop/todo tests actually ran, not just compiled
  sdk         the committed SDKs match `graff --schema`
EOF
}

onlys=()
while (($#)); do
  case "$1" in
    --list) printf '%s\n' "${CHECKS[@]}"; exit 0 ;;
    --only)
      IFS=',' read -ra _parts <<< "${2:-}"
      onlys+=("${_parts[@]}")
      shift 2 || true
      ;;
    --only=*)
      IFS=',' read -ra _parts <<< "${1#--only=}"
      onlys+=("${_parts[@]}")
      shift
      ;;
    -h|--help) usage; exit 0 ;;
    *) printf 'eval-tier1: unknown argument %s\n\n' "$1" >&2; usage >&2; exit 2 ;;
  esac
done

if ((${#onlys[@]} > 0)); then
  for only in "${onlys[@]}"; do
    [[ -n "$only" ]] || continue
    found=0
    for name in "${CHECKS[@]}"; do [[ $name == "$only" ]] && found=1; done
    if ((!found)); then
      printf 'eval-tier1: no check named %s (try --list)\n' "$only" >&2
      exit 2
    fi
  done
fi

wanted() {
  if ((${#onlys[@]} == 0)); then return 0; fi
  local c=$1
  for only in "${onlys[@]}"; do [[ $only == "$c" ]] && return 0; done
  return 1
}

failed=()
warned=()
started=$SECONDS

# scripts/eval/tier1_invariants.py `count` exits with this when the suite has run
# far enough ahead of test_count_baseline that the floor guards nothing. Green,
# but say so out loud - that ratchet stalled 350 tests behind for months (#439).
RATCHET_STALLED=3

announce() { printf '\n\033[1m==> %s\033[0m  %s\n' "$1" "$2"; }
record_fail() {
  failed+=("$1")
  printf '\n  \033[31mFAILED\033[0m %s\n  rerun just this check: scripts/eval-tier1.sh --only %s\n' "$1" "$1"
}
record_warn() {
  warned+=("$1")
  printf '  \033[33mWARN\033[0m %s: %s\n' "$1" "$2"
}

# --- fmt ---------------------------------------------------------------
if wanted fmt; then
  announce fmt "zig fmt --check src build.zig"
  if zig fmt --check src build.zig; then
    printf '  formatting is clean\n'
  else
    printf '  the files above are unformatted.\n    fix: zig fmt src build.zig\n'
    record_fail fmt
  fi
fi

# --- lines -------------------------------------------------------------
if wanted lines; then
  announce lines "600-line ceiling on hand-written Zig (AGENTS.md)"
  if bash scripts/check-zig-lines.sh; then :; else
    printf '    fix: split the file above into focused sibling modules.\n'
    record_fail lines
  fi
fi

# --- spec --------------------------------------------------------------
if wanted spec; then
  announce spec "spec/conformance.py — kernel properties + fixture staleness"
  # The Zig leg of the corpus already gates via `tests` (the conformance tests
  # embed the fixtures); this closes the triangle's other side pre-push: the
  # executable model's properties, and that the committed fixtures match it.
  if python3 spec/conformance.py; then :; else
    printf '    a kernel drifted: ratchet the model (lean-proofs/ + spec/ref/) or the impl,\n'
    printf '    then regenerate fixtures: python3 spec/conformance.py --export\n'
    record_fail spec
  fi
fi

# --- reach -------------------------------------------------------------
if wanted reach; then
  announce reach "every declared test is actually COMPILED IN, not just imported"
  # tier1_invariants.py static modelled reachability as a textual @import graph,
  # which over-approximates what Zig analyses: it printed "every test file is
  # reachable" and exited 0 while 37 tests in 12 files were silently skipped.
  # test_reachability.py diffs declared names against the compiled binary
  # instead, so it cannot be fooled by an import Zig never follows.
  if python3 scripts/eval/test_reachability.py; then :; else
    record_fail reach
  fi
fi

# --- build -------------------------------------------------------------
build_ok=1
if wanted build; then
  announce build "zig build"
  if zig build; then
    printf '  zig-out/bin/graff is current\n'
  else
    build_ok=0
    record_fail build
  fi
fi

# The suite count and the SDK gate both need a working compile; skip rather
# than report a second, derived failure.
skip_dependent() {
  printf '\n  \033[33mskipped\033[0m %s (zig build failed above)\n' "$1"
}

# --- shell -------------------------------------------------------------
if wanted shell; then
  if ((!build_ok)); then
    skip_dependent shell
  else
    announce shell "durable reservations and stale handles cannot target new jobs"
    if (
      probe_dir=$(mktemp -d) || exit 1
      trap 'rm -rf "$probe_dir"' EXIT
      if [[ $(uname -s) == Darwin && -z ${SDKROOT:-} ]]; then
        SDKROOT=$(xcrun --sdk macosx --show-sdk-path) || exit 1
        export SDKROOT
      fi
      python3 scripts/eval/process_guard.py --timeout 180 zig build-exe src/shell_identity_probe.zig -lc -femit-bin="$probe_dir/identity" &&
      python3 scripts/eval/process_guard.py --timeout 60 python3 scripts/test-shell-identity.py "$probe_dir/identity" &&
      python3 scripts/eval/process_guard.py --timeout 90 python3 scripts/test-shell-restart.py zig-out/bin/graff &&
      python3 scripts/eval/process_guard.py --timeout 90 python3 scripts/test-acp-released-fixes.py zig-out/bin/graff &&
      python3 scripts/eval/process_guard.py --timeout 120 python3 scripts/test-acp-permissions.py zig-out/bin/graff &&
      python3 scripts/eval/process_guard.py --timeout 90 python3 scripts/test-acp-session-load.py zig-out/bin/graff &&
      python3 scripts/eval/process_guard.py --timeout 120 python3 scripts/test-acp-reload-compaction-count.py zig-out/bin/graff &&
      python3 scripts/eval/process_guard.py --timeout 90 python3 scripts/test-acp-session-fork.py zig-out/bin/graff &&
      python3 scripts/eval/process_guard.py --timeout 90 python3 scripts/test-acp-resume-identity.py zig-out/bin/graff &&
      python3 scripts/eval/process_guard.py --timeout 120 python3 scripts/test-acp-workspace.py zig-out/bin/graff &&
      python3 scripts/eval/process_guard.py --timeout 90 python3 scripts/test-acp-images.py zig-out/bin/graff
    ); then :; else
      record_fail shell
    fi
  fi
fi

# --- tests -------------------------------------------------------------
suite_count() {
  # `--summary all` prints "P/T tests passed (S skipped)" when the run step
  # actually ran. T (passed + skipped) is the suite size — using P alone
  # falsely reports a shrink whenever anything is skipped (#794). A fully
  # cached `zig build test` on zig 0.17 prints only "Build Summary: 4/4
  # steps succeeded", so this comes back empty and the caller falls back
  # to a scan of the artifact (#439).
  printf '%s\n' "$1" | python3 scripts/eval/tier1_test_binary.py summary
}

# The count off the compiled artifact instead of off the build summary: a test
# binary run with no arguments prints "All N tests passed." every time, cached
# build or not, and running it re-proves the suite besides. Pass the same
# -Dtest-filter values the build used so the right artifact is picked out of
# .zig-cache/o (see scripts/eval/tier1_test_binary.py).
artifact_count() {
  python3 scripts/eval/tier1_test_binary.py count "$@"
}

if wanted tests; then
  if ((!build_ok)); then
    skip_dependent tests
  else
    announce tests "zig build test, and the suite count never shrinks"
    if ! python3 scripts/eval/test_tier1_count.py; then
      printf '    fix: skipped tests must count toward the suite ratchet (#794)\n'
      record_fail tests
    else
    # One full run. 600s killed a green suite on a warm tree before the
    # binary finished; process_guard's cap is 1800s.
    out=$(python3 scripts/eval/process_guard.py --timeout 1800 zig build test --summary all 2>&1)
    status=$?
    if ((status != 0)); then
      printf '%s\n' "$out"
    else
      printf '%s\n' "$out" | tail -4
    fi
    if ((status != 0)); then
      printf '    fix: the failing test names are above; rerun one with\n'
      printf '         zig build unit -Dtest-filter="<part of the name>"\n'
      record_fail tests
    else
      count=$(suite_count "$out")
      if [[ -z "$count" ]]; then
        printf '  cached build; scanning the compiled artifact for the suite count\n'
        count=$(python3 scripts/eval/tier1_test_binary.py scan)
      fi
      if [[ -z "$count" ]]; then
        printf '  FAIL could not read the test count from the build summary or the test artifact\n'
        record_fail tests
      else
        python3 scripts/eval/tier1_invariants.py count --observed "$count"
        case $? in
          0) : ;;
          "$RATCHET_STALLED") record_warn tests "test_count_baseline is stale (see above)" ;;
          *) record_fail tests ;;
        esac
      fi
    fi
    fi
  fi
fi

# --- terminal ----------------------------------------------------------
if wanted terminal; then
  if ((!build_ok)); then
    skip_dependent terminal
  else
    announce terminal "PTY cleanup, line-REPL prompt reflow, and live thinking controls"
    if python3 scripts/eval/test_pty_cleanup.py &&
      python3 scripts/eval/process_guard.py --timeout 90 python3 scripts/test-repl-prompt-reflow.py zig-out/bin/graff &&
      python3 scripts/eval/process_guard.py --timeout 90 python3 scripts/test-repl-thinking.py zig-out/bin/graff; then :; else
      record_fail terminal
    fi
  fi
fi

# --- invariants --------------------------------------------------------
if wanted invariants; then
  if ((!build_ok)); then
    skip_dependent invariants
  else
    announce invariants "the named goal/loop/todo tests are compiled into the suite"
    # #641: the unfiltered `tests` artifact already ran those names. Scanning
    # it avoids two filtered `zig build test` recompiles (~1–3 min cold).
    inv_ok=1
    if ! python3 scripts/eval/tier1_test_binary.py resolve >/dev/null 2>&1; then
      printf '  no test artifact yet; compiling the unfiltered suite once\n'
      if ! python3 scripts/eval/process_guard.py --timeout 1800 zig build test --summary all >/dev/null; then
        inv_ok=0
        record_fail invariants
      fi
    fi
    if ((inv_ok)); then
      if python3 scripts/eval/tier1_invariants.py invariants --scan; then :; else
        printf '    the required set (scripts/eval/tier1-manifest.json):\n'
        python3 scripts/eval/tier1_invariants.py list
        record_fail invariants
      fi
    fi
  fi
fi

# --- sdk ---------------------------------------------------------------
if wanted sdk; then
  if ((!build_ok)); then
    skip_dependent sdk
  else
    announce sdk "the committed SDKs match \`graff --schema\`"
    # The generator rewrites sdk/ IN PLACE, so running it over uncommitted work
    # there destroys it silently. Refuse instead: the check cannot tell a stale
    # commit from an edit in progress, and only one of those is safe to clobber.
    if [ -n "$(git status --porcelain -- sdk/)" ]; then
      printf '  sdk/ has uncommitted changes; skipping rather than overwriting them:\n'
      git status --porcelain -- sdk/ | sed 's/^/    /'
      printf '  commit or stash them, then rerun: scripts/eval-tier1.sh --only sdk\n'
      record_fail sdk
    # Generate against a PRISTINE HOME. graff --schema folds in the live model
    # catalogs cached under $HOME, so a machine that has talked to a provider
    # emits models CI has never heard of: the committed SDK then matched this
    # laptop and failed CI with a drift the local check called clean.
    elif sdk_home=$(mktemp -d) && HOME="$sdk_home" python3 sdk/generate.py --harness ./zig-out/bin/graff >/dev/null; then
      rm -rf "$sdk_home"
      if git diff --exit-code --stat -- sdk/; then
        printf '  sdk/ is in sync with the schema\n'
      else
        printf '\n  The generator just rewrote the files above, so the committed SDKs were\n'
        printf '  stale. They are regenerated in your working tree right now - review and\n'
        printf '  commit them:\n'
        printf '    git add sdk/ && git commit -m "chore(sdk): regenerate from the schema"\n'
        record_fail sdk
      fi
    else
      printf '  sdk/generate.py failed\n'
      record_fail sdk
    fi
  fi
fi

# --- verdict -----------------------------------------------------------
elapsed=$((SECONDS - started))
if ((${#warned[@]} > 0)); then
  printf '\n\033[33m%d warning(s)\033[0m: %s\n' "${#warned[@]}" "${warned[*]}"
fi
if ((${#failed[@]} == 0)); then
  printf '\n\033[32mtier 1 green\033[0m in %ss\n' "$elapsed"
  exit 0
fi
printf '\n\033[31mtier 1 red\033[0m in %ss - %d check(s) failed: %s\n' \
  "$elapsed" "${#failed[@]}" "${failed[*]}"
printf 'rerun one with: scripts/eval-tier1.sh --only <check>\n'
exit 1