Skip to content

graff — a fast agentic coding harness in Zig: multi-provider, MCP, workflows, DGM evolution loop, TS/Python SDKs

Python

Evals: rounds can measure their own noise; mined sandboxes carry AGENTS.md

justrach09778ce35df4
diff --git a/graff-evals/README.md b/graff-evals/README.md
index 1686011b..1f4d245c 100644
--- a/graff-evals/README.md
+++ b/graff-evals/README.md
@@ -115,6 +115,7 @@ not tuned against.
./hillclimb.py split --suite mined # fixed train/test split; never redrawn between rounds
./hillclimb.py noise --suite mined --harness graff-dev --model gpt-6-sol --reps 3
./hillclimb.py round --suite mined --champion graff-dev --candidate graff-dev-g6-followthrough --model gpt-6-sol
+./hillclimb.py round --suite mined --candidate graff-dev-g6-followthrough --model gpt-6-sol --reps 2 --noise-from-round # no separate baseline
```
A round keeps the candidate only when its train score beats the measured
diff --git a/graff-evals/hillclimb.py b/graff-evals/hillclimb.py
index 5b7f5986..62c2fe74 100755
--- a/graff-evals/hillclimb.py
+++ b/graff-evals/hillclimb.py
@@ -389,18 +389,29 @@ def cmd_round(args) -> None:
noise_file = os.path.join(LOG_DIR, f"noise-{args.suite}-{args.champion}-{args.model}.json")
min_effect = args.min_effect
reps = args.reps
- if min_effect is None:
+ if min_effect is None and not args.noise_from_round:
if not os.path.exists(noise_file):
- raise SystemExit(f"no noise measurement ({noise_file}); run hillclimb.py noise first or pass --min-effect")
+ raise SystemExit(f"no noise measurement ({noise_file}); run hillclimb.py noise first, "
+ "pass --min-effect, or use --noise-from-round")
with open(noise_file) as f:
noise = json.load(f)
min_effect = noise["sets"]["train"][args.goal]["band"]
reps = reps or noise["reps"]
- reps = reps or 3
+ reps = reps or (2 if args.noise_from_round else 3)
+ if args.noise_from_round and reps < 2:
+ raise SystemExit("--noise-from-round needs at least 2 repetitions")
path = run_eval(f"{args.champion},{args.candidate}", args.model, args.suite,
split["train"] + split["test"], reps, args.jobs)
rows = load_jsonl(path)
pick = lambda h, s: [r for r in rows if r.get("harness") == h and r.get("task") in split[s]]
+ if min_effect is None:
+ # The champion's own repetitions in this round give the noise band.
+ noise = noise_summary(rows, args.champion, split, reps)
+ min_effect = noise["sets"]["train"][args.goal]["band"]
+ print(f"noise from this round's champion repetitions: {args.goal} band {min_effect:.4f}")
+ for name in ("train", "test"):
+ print(f"\n{name}")
+ print(table(bucket(pick(args.champion, name) + pick(args.candidate, name))))
scores = {h: {s: metric(pick(h, s), args.goal) for s in ("train", "test")} for h in (args.champion, args.candidate)}
keep, why = judge(scores[args.champion]["train"], scores[args.candidate]["train"],
scores[args.champion]["test"], scores[args.candidate]["test"], min_effect, args.goal)
@@ -480,6 +491,8 @@ def main() -> None:
p.add_argument("--goal", choices=sorted(GOALS), default="pass")
p.add_argument("--reps", type=int, default=None)
p.add_argument("--min-effect", type=float, default=None)
+ p.add_argument("--noise-from-round", action="store_true",
+ help="take the noise band from the champion's repetitions in this round (no separate baseline)")
p.add_argument("--jobs", "-j", type=int, default=1)
p.set_defaults(fn=cmd_round)
diff --git a/graff-evals/live_setup.py b/graff-evals/live_setup.py
index 4f410354..bcdf2fba 100644
--- a/graff-evals/live_setup.py
+++ b/graff-evals/live_setup.py
@@ -161,7 +161,9 @@ def archive_local(rev, dest, paths=None):
"""A historical tree straight from this checkout's git history (ADR 0214:
mined tasks). Recent parents build on the pinned toolchain as they are."""
os.makedirs(dest, exist_ok=True)
- wanted = list(paths or DEFAULT_SPARSE)
+ # The repository's own instructions come along: a real graff session reads
+ # them (for example, that a new test file must be registered to run).
+ wanted = list(paths or DEFAULT_SPARSE) + ["AGENTS.md", "CLAUDE.md"]
# Whatever that revision's build.zig embeds or builds from (for example a
# stylesheet under apps/ before the desktop app left) must come along.
build = subprocess.run(["git", "-C", REPO, "show", f"{rev}:build.zig"],