graff — a fast agentic coding harness in Zig: multi-provider, MCP, workflows, DGM evolution loop, TS/Python SDKs
Python
Add a matched release-cache benchmark
justrach7f6ac9d963a9
diff --git a/.github/workflows/release-cache-benchmark.yml b/.github/workflows/release-cache-benchmark.ymlnew file mode 100644index 00000000..74b93b7d--- /dev/null+++ b/.github/workflows/release-cache-benchmark.yml@@ -0,0 +1,117 @@+name: release cache benchmark++# Branch-only experiment: no tags, releases, signing, or production cache keys.+on:+ push:+ branches: [perf/nanocompile-ci-measure]++permissions:+ contents: read++jobs:+ cold:+ name: cold (${{ matrix.mode }}, ${{ matrix.target }})+ runs-on: ubuntu-24.04+ timeout-minutes: 30+ strategy:+ fail-fast: false+ matrix:+ mode: [plain, nano]+ target: [aarch64-macos, x86_64-macos, x86_64-linux, aarch64-linux, x86_64-windows, aarch64-windows]+ steps:+ - uses: actions/checkout@v7++ # Do not use the normal setup action: it restores previous build caches.+ - name: Install pinned Zig without build caches+ run: python3 scripts/install-zig-ci.py++ - name: Checkout pinned nanocompile+ if: matrix.mode == 'nano'+ uses: actions/checkout@v7+ with:+ repository: justrach/nanocompile+ ref: dec24ac64b2c810cca247f850ed1784830362df5+ path: .nanocompile-source+ persist-credentials: false++ - name: Build pinned nanocompile+ if: matrix.mode == 'nano'+ working-directory: .nanocompile-source+ run: |+ zig build -Doptimize=ReleaseFast --prefix "$GITHUB_WORKSPACE/.nanocompile-tools"+ echo "NANOCOMPILE_EXE=$GITHUB_WORKSPACE/.nanocompile-tools/bin/nanocompile" >> "$GITHUB_ENV"++ - name: Cold build+ run: python3 scripts/benchmark-release-cache.py --target '${{ matrix.target }}' --mode '${{ matrix.mode }}' --phase cold++ - name: Save measured build caches+ uses: actions/cache/save@v6+ with:+ path: |+ .zig-cache+ .zig-global-cache+ .nanocompile-cache+ .benchmark-state+ key: release-benchmark-v1-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.mode }}-${{ matrix.target }}++ - name: Save timing results+ uses: actions/upload-artifact@v7+ with:+ name: cold-${{ matrix.mode }}-${{ matrix.target }}+ path: .benchmark-results/*.json+ if-no-files-found: error+ retention-days: 7++ warm:+ name: warm (${{ matrix.mode }}, ${{ matrix.target }})+ needs: cold+ runs-on: ubuntu-24.04+ timeout-minutes: 30+ strategy:+ fail-fast: false+ matrix:+ mode: [plain, nano]+ target: [aarch64-macos, x86_64-macos, x86_64-linux, aarch64-linux, x86_64-windows, aarch64-windows]+ steps:+ - uses: actions/checkout@v7++ - name: Install pinned Zig without build caches+ run: python3 scripts/install-zig-ci.py++ - name: Checkout pinned nanocompile+ if: matrix.mode == 'nano'+ uses: actions/checkout@v7+ with:+ repository: justrach/nanocompile+ ref: dec24ac64b2c810cca247f850ed1784830362df5+ path: .nanocompile-source+ persist-credentials: false++ - name: Build pinned nanocompile+ if: matrix.mode == 'nano'+ working-directory: .nanocompile-source+ run: |+ zig build -Doptimize=ReleaseFast --prefix "$GITHUB_WORKSPACE/.nanocompile-tools"+ echo "NANOCOMPILE_EXE=$GITHUB_WORKSPACE/.nanocompile-tools/bin/nanocompile" >> "$GITHUB_ENV"++ - name: Restore measured build caches+ uses: actions/cache/restore@v6+ with:+ path: |+ .zig-cache+ .zig-global-cache+ .nanocompile-cache+ .benchmark-state+ key: release-benchmark-v1-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.mode }}-${{ matrix.target }}+ fail-on-cache-miss: true++ - name: Warm rebuild or declared-task restore+ run: python3 scripts/benchmark-release-cache.py --target '${{ matrix.target }}' --mode '${{ matrix.mode }}' --phase warm++ - name: Save timing results+ uses: actions/upload-artifact@v7+ with:+ name: warm-${{ matrix.mode }}-${{ matrix.target }}+ path: .benchmark-results/*.json+ if-no-files-found: error+ retention-days: 7diff --git a/scripts/benchmark-release-cache.py b/scripts/benchmark-release-cache.pynew file mode 100644index 00000000..8588f8c0--- /dev/null+++ b/scripts/benchmark-release-cache.py@@ -0,0 +1,82 @@+#!/usr/bin/env python3+"""Compare plain Zig and declared-task caching without publishing a release."""+import argparse+import hashlib+import importlib.util+import json+import os+from pathlib import Path+import shutil+import subprocess+import time+++ROOT = Path(__file__).resolve().parents[1]+spec = importlib.util.spec_from_file_location("release_build", ROOT / "scripts/build-release.py")+release = importlib.util.module_from_spec(spec)+spec.loader.exec_module(release)+++def timed(metrics, name, operation):+ start = time.perf_counter()+ result = operation()+ metrics[name] = round(time.perf_counter() - start, 6)+ return result+++def main():+ parser = argparse.ArgumentParser(description=__doc__)+ parser.add_argument("--target", required=True, choices=release.TARGETS)+ parser.add_argument("--mode", required=True, choices=("plain", "nano"))+ parser.add_argument("--phase", required=True, choices=("cold", "warm"))+ args = parser.parse_args()+ start = time.perf_counter()+ metrics = {"target": args.target, "mode": args.mode, "phase": args.phase}+ zig = Path(shutil.which("zig")).resolve()+ cache = ROOT / ".nanocompile-cache"+ env = release.environment(cache)++ def run(command):+ subprocess.run(command, cwd=ROOT, env=env, check=True)++ # Fetch is reported separately, but also included in total invocation time.+ timed(metrics, "fetch_seconds", lambda: run([str(zig), "build", "--fetch=all"]))+ contract, output = timed(metrics, "contract_seconds", lambda: release.contract(+ args.target, "0.302.43", zig))+ state_path = ROOT / ".benchmark-state" / (args.mode + "-" + args.target + ".json")+ expected = None+ if args.phase == "warm":+ expected = json.loads(state_path.read_text())["sha256"]+ if output.exists():+ raise RuntimeError("Benchmark requires an absent output before each invocation")+ command = contract["tasks"][0]["command"]+ events = cache / "events"+ offset = events.stat().st_size if events.exists() else 0+ if args.mode == "nano":+ spec_path = ROOT / ".zig-cache" / ("benchmark-" + args.target + ".json")+ spec_path.parent.mkdir(parents=True, exist_ok=True)+ spec_path.write_text(json.dumps(contract, sort_keys=True) + "\n")+ command = [os.environ["NANOCOMPILE_EXE"], "task", str(spec_path)]+ timed(metrics, "task_seconds", lambda: run(command))+ if args.mode == "nano":+ with events.open("rb") as source:+ source.seek(offset)+ hit = b"task_hit\n" in source.read()+ metrics["task_hit"] = hit+ if hit != (args.phase == "warm"):+ raise RuntimeError("Expected a cold miss or a warm declared-task hit")+ digest = hashlib.sha256(output.read_bytes()).hexdigest()+ if expected is not None and digest != expected:+ raise RuntimeError("Warm artifact differs from the cold artifact")+ state_path.parent.mkdir(parents=True, exist_ok=True)+ state_path.write_text(json.dumps({"sha256": digest}) + "\n")+ metrics["artifact_sha256"] = digest+ metrics["invocation_seconds"] = round(time.perf_counter() - start, 6)+ result = ROOT / ".benchmark-results" / (args.phase + "-" + args.mode + "-" + args.target + ".json")+ result.parent.mkdir(parents=True, exist_ok=True)+ result.write_text(json.dumps(metrics, indent=2) + "\n")+ print(json.dumps(metrics, sort_keys=True), flush=True)+++if __name__ == "__main__":+ main()