Skip to content

graff — a fast agentic coding harness in Zig: multi-provider, MCP, workflows, DGM evolution loop, TS/Python SDKs

Python

Add a matched release-cache benchmark

justrach7f6ac9d963a9
diff --git a/.github/workflows/release-cache-benchmark.yml b/.github/workflows/release-cache-benchmark.yml
new file mode 100644
index 00000000..74b93b7d
--- /dev/null
+++ b/.github/workflows/release-cache-benchmark.yml
@@ -0,0 +1,117 @@
+name: release cache benchmark
+
+# Branch-only experiment: no tags, releases, signing, or production cache keys.
+on:
+ push:
+ branches: [perf/nanocompile-ci-measure]
+
+permissions:
+ contents: read
+
+jobs:
+ cold:
+ name: cold (${{ matrix.mode }}, ${{ matrix.target }})
+ runs-on: ubuntu-24.04
+ timeout-minutes: 30
+ strategy:
+ fail-fast: false
+ matrix:
+ mode: [plain, nano]
+ target: [aarch64-macos, x86_64-macos, x86_64-linux, aarch64-linux, x86_64-windows, aarch64-windows]
+ steps:
+ - uses: actions/checkout@v7
+
+ # Do not use the normal setup action: it restores previous build caches.
+ - name: Install pinned Zig without build caches
+ run: python3 scripts/install-zig-ci.py
+
+ - name: Checkout pinned nanocompile
+ if: matrix.mode == 'nano'
+ uses: actions/checkout@v7
+ with:
+ repository: justrach/nanocompile
+ ref: dec24ac64b2c810cca247f850ed1784830362df5
+ path: .nanocompile-source
+ persist-credentials: false
+
+ - name: Build pinned nanocompile
+ if: matrix.mode == 'nano'
+ working-directory: .nanocompile-source
+ run: |
+ zig build -Doptimize=ReleaseFast --prefix "$GITHUB_WORKSPACE/.nanocompile-tools"
+ echo "NANOCOMPILE_EXE=$GITHUB_WORKSPACE/.nanocompile-tools/bin/nanocompile" >> "$GITHUB_ENV"
+
+ - name: Cold build
+ run: python3 scripts/benchmark-release-cache.py --target '${{ matrix.target }}' --mode '${{ matrix.mode }}' --phase cold
+
+ - name: Save measured build caches
+ uses: actions/cache/save@v6
+ with:
+ path: |
+ .zig-cache
+ .zig-global-cache
+ .nanocompile-cache
+ .benchmark-state
+ key: release-benchmark-v1-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.mode }}-${{ matrix.target }}
+
+ - name: Save timing results
+ uses: actions/upload-artifact@v7
+ with:
+ name: cold-${{ matrix.mode }}-${{ matrix.target }}
+ path: .benchmark-results/*.json
+ if-no-files-found: error
+ retention-days: 7
+
+ warm:
+ name: warm (${{ matrix.mode }}, ${{ matrix.target }})
+ needs: cold
+ runs-on: ubuntu-24.04
+ timeout-minutes: 30
+ strategy:
+ fail-fast: false
+ matrix:
+ mode: [plain, nano]
+ target: [aarch64-macos, x86_64-macos, x86_64-linux, aarch64-linux, x86_64-windows, aarch64-windows]
+ steps:
+ - uses: actions/checkout@v7
+
+ - name: Install pinned Zig without build caches
+ run: python3 scripts/install-zig-ci.py
+
+ - name: Checkout pinned nanocompile
+ if: matrix.mode == 'nano'
+ uses: actions/checkout@v7
+ with:
+ repository: justrach/nanocompile
+ ref: dec24ac64b2c810cca247f850ed1784830362df5
+ path: .nanocompile-source
+ persist-credentials: false
+
+ - name: Build pinned nanocompile
+ if: matrix.mode == 'nano'
+ working-directory: .nanocompile-source
+ run: |
+ zig build -Doptimize=ReleaseFast --prefix "$GITHUB_WORKSPACE/.nanocompile-tools"
+ echo "NANOCOMPILE_EXE=$GITHUB_WORKSPACE/.nanocompile-tools/bin/nanocompile" >> "$GITHUB_ENV"
+
+ - name: Restore measured build caches
+ uses: actions/cache/restore@v6
+ with:
+ path: |
+ .zig-cache
+ .zig-global-cache
+ .nanocompile-cache
+ .benchmark-state
+ key: release-benchmark-v1-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.mode }}-${{ matrix.target }}
+ fail-on-cache-miss: true
+
+ - name: Warm rebuild or declared-task restore
+ run: python3 scripts/benchmark-release-cache.py --target '${{ matrix.target }}' --mode '${{ matrix.mode }}' --phase warm
+
+ - name: Save timing results
+ uses: actions/upload-artifact@v7
+ with:
+ name: warm-${{ matrix.mode }}-${{ matrix.target }}
+ path: .benchmark-results/*.json
+ if-no-files-found: error
+ retention-days: 7
diff --git a/scripts/benchmark-release-cache.py b/scripts/benchmark-release-cache.py
new file mode 100644
index 00000000..8588f8c0
--- /dev/null
+++ b/scripts/benchmark-release-cache.py
@@ -0,0 +1,82 @@
+#!/usr/bin/env python3
+"""Compare plain Zig and declared-task caching without publishing a release."""
+import argparse
+import hashlib
+import importlib.util
+import json
+import os
+from pathlib import Path
+import shutil
+import subprocess
+import time
+
+
+ROOT = Path(__file__).resolve().parents[1]
+spec = importlib.util.spec_from_file_location("release_build", ROOT / "scripts/build-release.py")
+release = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(release)
+
+
+def timed(metrics, name, operation):
+ start = time.perf_counter()
+ result = operation()
+ metrics[name] = round(time.perf_counter() - start, 6)
+ return result
+
+
+def main():
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("--target", required=True, choices=release.TARGETS)
+ parser.add_argument("--mode", required=True, choices=("plain", "nano"))
+ parser.add_argument("--phase", required=True, choices=("cold", "warm"))
+ args = parser.parse_args()
+ start = time.perf_counter()
+ metrics = {"target": args.target, "mode": args.mode, "phase": args.phase}
+ zig = Path(shutil.which("zig")).resolve()
+ cache = ROOT / ".nanocompile-cache"
+ env = release.environment(cache)
+
+ def run(command):
+ subprocess.run(command, cwd=ROOT, env=env, check=True)
+
+ # Fetch is reported separately, but also included in total invocation time.
+ timed(metrics, "fetch_seconds", lambda: run([str(zig), "build", "--fetch=all"]))
+ contract, output = timed(metrics, "contract_seconds", lambda: release.contract(
+ args.target, "0.302.43", zig))
+ state_path = ROOT / ".benchmark-state" / (args.mode + "-" + args.target + ".json")
+ expected = None
+ if args.phase == "warm":
+ expected = json.loads(state_path.read_text())["sha256"]
+ if output.exists():
+ raise RuntimeError("Benchmark requires an absent output before each invocation")
+ command = contract["tasks"][0]["command"]
+ events = cache / "events"
+ offset = events.stat().st_size if events.exists() else 0
+ if args.mode == "nano":
+ spec_path = ROOT / ".zig-cache" / ("benchmark-" + args.target + ".json")
+ spec_path.parent.mkdir(parents=True, exist_ok=True)
+ spec_path.write_text(json.dumps(contract, sort_keys=True) + "\n")
+ command = [os.environ["NANOCOMPILE_EXE"], "task", str(spec_path)]
+ timed(metrics, "task_seconds", lambda: run(command))
+ if args.mode == "nano":
+ with events.open("rb") as source:
+ source.seek(offset)
+ hit = b"task_hit\n" in source.read()
+ metrics["task_hit"] = hit
+ if hit != (args.phase == "warm"):
+ raise RuntimeError("Expected a cold miss or a warm declared-task hit")
+ digest = hashlib.sha256(output.read_bytes()).hexdigest()
+ if expected is not None and digest != expected:
+ raise RuntimeError("Warm artifact differs from the cold artifact")
+ state_path.parent.mkdir(parents=True, exist_ok=True)
+ state_path.write_text(json.dumps({"sha256": digest}) + "\n")
+ metrics["artifact_sha256"] = digest
+ metrics["invocation_seconds"] = round(time.perf_counter() - start, 6)
+ result = ROOT / ".benchmark-results" / (args.phase + "-" + args.mode + "-" + args.target + ".json")
+ result.parent.mkdir(parents=True, exist_ok=True)
+ result.write_text(json.dumps(metrics, indent=2) + "\n")
+ print(json.dumps(metrics, sort_keys=True), flush=True)
+
+
+if __name__ == "__main__":
+ main()