Skip to content

graff — a fast agentic coding harness in Zig: multi-provider, MCP, workflows, DGM evolution loop, TS/Python SDKs

Python
24.5 KB
Raw Blame
#!/usr/bin/env python3
"""Autoresearch-style hillclimb for graff vs grok-build.

Propose a harness change, run the same tasks, keep only a measured win
on wall / first-token latency / tool calls / tokens / list-price USD.
Never keep grok-build's heap or a 4-tool catalog (ADR 0024).

    ./hillclimb.py self-test
    ./hillclimb.py score results/run-….jsonl
    ./hillclimb.py decide --champion results/a.jsonl --candidate results/b.jsonl
    ./hillclimb.py iterate --suite core --task exact-reply,fix-fib,file-ops

Held-out loop (ADR 0214): fix a train/test split once, measure noise from
repeated baseline runs, then judge one candidate per round. A candidate is
kept only when its train score beats the noise band AND its test score
improves; train up with test flat is overfitting and is reverted. Only train
failures are ever printed; the test set is scores only.

    ./hillclimb.py split --suite mined [--test-frac 0.33]
    ./hillclimb.py noise --suite mined --harness graff-dev --model gpt-6-sol --reps 3
    ./hillclimb.py round --suite mined --champion graff-dev --candidate graff-dev-x --model gpt-6-sol
"""
from __future__ import annotations

import argparse, hashlib, json, math, os, statistics, subprocess, sys, time

from list_price import attach, self_test as price_self_test

ROOT = os.path.dirname(os.path.abspath(__file__))
LOG_DIR = os.path.join(ROOT, "hillclimb")
CANDIDATES_PATH = os.path.join(LOG_DIR, "candidates.json")
LOG_PATH = os.path.join(LOG_DIR, "log.jsonl")
OURS_DEFAULT = "graff-dev"
THEIRS_DEFAULT = "grok"
AXES = ("wall_s", "first_out_s", "tok_calls", "list_tokens", "list_usd")
# Refuse a "win" that is just grok-build's 165M process (ADR 0024).
HEAP_OURS_KB = 20 * 1024
HEAP_THEIRS_KB = 80 * 1024


def load_jsonl(path: str) -> list[dict]:
    rows = []
    with open(path) as f:
        for line in f:
            line = line.strip()
            if not line:
                continue
            rec = json.loads(line)
            attach(rec)
            rows.append(rec)
    return rows


def bucket(records: list[dict]) -> dict[str, dict]:
    by: dict[str, dict] = {}
    for r in records:
        b = by.setdefault(r.get("harness") or "?", {
            "n": 0, "ok": 0, "wall_s": 0.0, "first_out_s": 0.0, "first_n": 0,
            "tok_calls": 0, "list_tokens": 0, "list_ordinary": 0, "list_cached": 0,
            "list_out": 0, "list_usd": 0.0, "usd_n": 0, "rss_peak_kb": 0, "tasks": [],
        })
        b["n"] += 1
        b["ok"] += bool(r.get("outcome_ok"))
        b["wall_s"] += r.get("wall_s") or 0
        if r.get("first_out_s") is not None:
            b["first_out_s"] += r["first_out_s"]
            b["first_n"] += 1
        b["tok_calls"] += r.get("tok_calls") or 0
        b["list_tokens"] += r.get("list_tokens") or 0
        b["list_ordinary"] += r.get("list_ordinary") or 0
        b["list_cached"] += r.get("list_cached") or 0
        b["list_out"] += r.get("list_out") or 0
        usd = r.get("list_usd")
        if usd is None:
            usd = r.get("tok_cost_usd")
        if usd is not None:
            b["list_usd"] += usd
            b["usd_n"] += 1
        b["rss_peak_kb"] = max(b["rss_peak_kb"], r.get("rss_peak_kb") or 0)
        b["tasks"].append(r.get("task"))
    for b in by.values():
        b["first_out_s"] = (b["first_out_s"] / b["first_n"]) if b["first_n"] else 0.0
        b["list_usd"] = round(b["list_usd"], 6)
        b["pass"] = f"{b['ok']}/{b['n']}"
    return by


def table(by: dict[str, dict]) -> str:
    hdr = f"{'harness':<22} {'pass':>7} {'wall':>8} {'first':>7} {'calls':>6} {'tokens':>9} {'in':>8} {'cached':>8} {'out':>7} {'list$':>9} {'rss':>8}"
    lines = [hdr]
    for h, b in by.items():
        rss = f"{b['rss_peak_kb'] / 1024:.1f}M" if b["rss_peak_kb"] else "—"
        lines.append(
            f"{h:<22} {b['pass']:>7} {b['wall_s']:>7.1f}s {b['first_out_s']:>6.1f}s "
            f"{b['tok_calls']:>6} {b['list_tokens']:>9} {b['list_ordinary']:>8} "
            f"{b['list_cached']:>8} {b['list_out']:>7} ${b['list_usd']:<8.4f} {rss:>8}"
        )
    return "\n".join(lines)


def _delta(cand: float, champ: float) -> tuple[float, str]:
    if champ == 0 and cand == 0:
        return 0.0, "wash"
    if champ == 0:
        return 1.0, "up"
    rel = (cand - champ) / champ
    if abs(rel) < 0.02:
        return rel, "wash"
    return rel, ("win" if rel < 0 else "loss")


def decide(champ: dict, cand: dict, theirs: dict | None = None, candidate_id: str = "") -> dict:
    """Keep only if pass holds and a majority of axes improve. Cite numbers."""
    reasons = []
    if cand["ok"] < champ["ok"]:
        return {"keep": False, "why": "pass rate dropped", "axes": {}, "champ": champ, "cand": cand}
    if cand["rss_peak_kb"] >= HEAP_THEIRS_KB and champ["rss_peak_kb"] <= HEAP_OURS_KB:
        return {"keep": False, "why": "heap steal (ADR 0024: refuse grok-build RSS)", "axes": {}, "champ": champ, "cand": cand}
    if "4-tool" in (candidate_id or "") or "four-tool" in (candidate_id or ""):
        return {"keep": False, "why": "4-tool catalog is forbidden (ADR 0024)", "axes": {}, "champ": champ, "cand": cand}

    axes = {}
    wins = 0
    for name in AXES:
        rel, verdict = _delta(cand[name], champ[name])
        axes[name] = {"champ": champ[name], "cand": cand[name], "rel": round(rel, 4), "verdict": verdict}
        if verdict == "win":
            wins += 1
        reasons.append(f"{name}: {champ[name]} → {cand[name]} ({verdict}, {rel:+.1%})")

    # Majority of the five named axes, and list-price USD must not get worse
    # unless we also gained a pass.
    keep = cand["ok"] >= champ["ok"] and wins >= 3 and axes["list_usd"]["verdict"] != "loss"
    if axes["list_usd"]["verdict"] == "loss" and cand["ok"] > champ["ok"]:
        keep = wins >= 3
    why = ("keep: " if keep else "drop: ") + "; ".join(reasons)
    out = {"keep": keep, "why": why, "axes": axes, "wins": wins, "champ": {
        k: champ[k] for k in ("pass", "wall_s", "first_out_s", "tok_calls", "list_tokens", "list_usd", "rss_peak_kb")
    }, "cand": {k: cand[k] for k in ("pass", "wall_s", "first_out_s", "tok_calls", "list_tokens", "list_usd", "rss_peak_kb")}}
    if theirs:
        out["theirs"] = {k: theirs[k] for k in ("pass", "wall_s", "first_out_s", "tok_calls", "list_tokens", "list_usd", "rss_peak_kb")}
    return out


def append_log(entry: dict) -> None:
    os.makedirs(LOG_DIR, exist_ok=True)
    entry = dict(entry)
    entry["ts"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
    with open(LOG_PATH, "a") as f:
        f.write(json.dumps(entry, ensure_ascii=False) + "\n")


def run_eval(harnesses: str, model: str, suite: str, tasks: list[str] | None, reps: int, jobs: int) -> str:
    # run.py writes each run to its own directory (measurement.isolated_paths);
    # name it here so this loop reads exactly the run it started.
    out_root = os.path.join(ROOT, "results", "hc-" + time.strftime("%Y%m%d-%H%M%S") + f"-{os.getpid()}")
    cmd = [sys.executable, os.path.join(ROOT, "run.py"),
           "--harness", harnesses, "--model", model, "--suite", suite,
           "--reps", str(reps), "-j", str(jobs), "--output-root", out_root]
    if tasks:
        for t in tasks:
            cmd += ["--task", t]
    print("+", " ".join(cmd), flush=True)
    proc = subprocess.run(cmd, cwd=ROOT)
    if proc.returncode != 0:
        raise SystemExit(f"run.py exited {proc.returncode}")
    path = os.path.join(out_root, "results.jsonl")
    if not os.path.exists(path):
        raise SystemExit(f"run.py produced no {path}")
    return path


def cmd_score(args) -> None:
    rows = load_jsonl(args.jsonl)
    by = bucket(rows)
    print(table(by))
    if args.json:
        print(json.dumps(by, indent=2))


def cmd_decide(args) -> None:
    champ_src = load_jsonl(args.champion)
    cand_src = champ_src if args.same_file else load_jsonl(args.candidate)
    cand_name = args.cand_harness or args.ours
    champ_rows = [r for r in champ_src if r.get("harness") == args.ours]
    cand_rows = [r for r in cand_src if r.get("harness") == cand_name]
    theirs_rows = [r for r in cand_src if r.get("harness") == args.theirs]
    if not theirs_rows:
        theirs_rows = [r for r in champ_src if r.get("harness") == args.theirs]
    if not champ_rows or not cand_rows:
        raise SystemExit(f"need rows for {args.ours} and {cand_name}")
    by_c, by_k = bucket(champ_rows), bucket(cand_rows)
    champ, cand = by_c[args.ours], by_k[cand_name]
    theirs = bucket(theirs_rows)[args.theirs] if theirs_rows else None
    print("champion\n" + table(by_c))
    print("\ncandidate\n" + table(by_k))
    if theirs:
        print("\ntheirs\n" + table({args.theirs: theirs}))
    d = decide(champ, cand, theirs, candidate_id=args.candidate_id)
    print("\n" + ("KEEP" if d["keep"] else "DROP"))
    print(d["why"])
    append_log({"kind": "decide", "candidate_id": args.candidate_id, **{k: d[k] for k in d if k != "axes"}, "axes": d["axes"]})
    if args.json:
        print(json.dumps(d, indent=2))


def cmd_iterate(args) -> None:
    with open(CANDIDATES_PATH) as f:
        cands = json.load(f)["candidates"]
    if args.only:
        want = {s.strip() for s in args.only.split(",") if s.strip()}
        cands = [c for c in cands if c["id"] in want]
    else:
        cands = [c for c in cands if not c.get("kept")]
    tasks = [t.strip() for t in (args.task or "").split(",") if t.strip()] or None
    harnesses = [args.ours]
    if args.theirs:
        harnesses.append(args.theirs)
    harnesses += [c["harness"] for c in cands]
    path = run_eval(",".join(harnesses), args.model, args.suite, tasks, args.reps, args.jobs)
    rows = load_jsonl(path)
    by = bucket(rows)
    print("\n" + table(by))
    blocked = []
    if args.theirs and args.theirs not in by:
        blocked.append(f"{args.theirs} produced no rows")
    elif args.theirs:
        theirs_err = [r for r in rows if r.get("harness") == args.theirs and r.get("error")]
        if theirs_err:
            blocked.append(theirs_err[0].get("error") or "theirs error")
    champ = by.get(args.ours)
    if not champ:
        raise SystemExit(f"champion {args.ours} missing from {path}")
    for c in cands:
        cand = by.get(c["harness"])
        if not cand:
            print(f"DROP {c['id']}: no rows")
            append_log({"kind": "iterate", "candidate_id": c["id"], "keep": False, "why": "no rows", "results": path})
            continue
        d = decide(champ, cand, by.get(args.theirs), candidate_id=c["id"])
        print(f"\n{c['id']}: {'KEEP' if d['keep'] else 'DROP'}")
        print(d["why"])
        append_log({"kind": "iterate", "candidate_id": c["id"], "results": path, "blocked": blocked, **d})
    if blocked:
        print("\nblocked-honest:", "; ".join(blocked))


# ── held-out loop (ADR 0214) ─────────────────────────────────────────────

TASKS_DIR = os.path.join(ROOT, "tasks")
GOALS = {"pass": +1, "usd": -1, "wall": -1, "tokens": -1}  # +1: higher is better
STALL_ROUNDS = 3
HEADROOM = 0.95


def suite_tasks(suite: str) -> list[str]:
    ids = []
    for name in sorted(os.listdir(TASKS_DIR)):
        if name.endswith(".json"):
            with open(os.path.join(TASKS_DIR, name)) as f:
                t = json.load(f)
            if t.get("suite") == suite:
                ids.append(t["id"])
    return ids


def make_split(ids: list[str], test_frac: float, seed: str) -> dict:
    if len(ids) < 3:
        raise SystemExit(f"need at least 3 tasks to hold some out, have {len(ids)}")
    order = sorted(ids, key=lambda i: hashlib.sha256(f"{seed}:{i}".encode()).hexdigest())
    n_test = min(len(ids) - 1, max(1, round(len(ids) * test_frac)))
    return {"seed": seed, "test": sorted(order[:n_test]), "train": sorted(order[n_test:])}


def split_path(suite: str) -> str:
    return os.path.join(LOG_DIR, f"split-{suite}.json")


def load_split(suite: str) -> dict:
    path = split_path(suite)
    if not os.path.exists(path):
        raise SystemExit(f"no split for {suite}; run: hillclimb.py split --suite {suite}")
    with open(path) as f:
        return json.load(f)


def metric(rows: list[dict], goal: str) -> float:
    if not rows:
        return 0.0
    if goal == "pass":
        return sum(bool(r.get("outcome_ok")) for r in rows) / len(rows)
    key = {"usd": "list_usd", "wall": "wall_s", "tokens": "list_tokens"}[goal]
    vals = [r.get(key) or 0 for r in rows]
    return sum(vals) / len(vals)


def rep_scores(rows: list[dict], harness: str, tasks: list[str], goal: str) -> list[float]:
    """One score per repetition: the goal metric over that rep's tasks."""
    by_rep: dict[int, list[dict]] = {}
    for r in rows:
        if r.get("harness") == harness and r.get("task") in tasks:
            by_rep.setdefault(int(r.get("rep") or 0), []).append(r)
    return [metric(v, goal) for _, v in sorted(by_rep.items())]


def band(sd: float, reps: int) -> float:
    """Two standard errors of a difference between two `reps`-rep means."""
    return 2 * sd * math.sqrt(2 / max(1, reps))


def noise_summary(rows: list[dict], harness: str, split: dict, reps: int) -> dict:
    out = {"harness": harness, "reps": reps, "sets": {}}
    for name in ("train", "test"):
        entry = {}
        for goal in GOALS:
            scores = rep_scores(rows, harness, split[name], goal)
            sd = statistics.pstdev(scores) if len(scores) > 1 else 0.0
            entry[goal] = {"mean": round(statistics.mean(scores), 6) if scores else 0.0,
                           "rep_sd": round(sd, 6), "band": round(band(sd, reps), 6)}
        flaky = sorted({r["task"] for r in rows if r.get("harness") == harness and r.get("task") in split[name]
                        and len({bool(x.get("outcome_ok")) for x in rows
                                 if x.get("harness") == harness and x.get("task") == r["task"]}) > 1})
        entry["flaky_tasks"] = flaky if name == "train" else len(flaky)  # never name test tasks
        out["sets"][name] = entry
    out["headroom_warning"] = out["sets"]["train"]["pass"]["mean"] >= HEADROOM
    return out


def judge(champ_train: float, cand_train: float, champ_test: float, cand_test: float,
          min_effect: float, goal: str = "pass") -> tuple[bool, str]:
    sign = GOALS[goal]
    d_train = (cand_train - champ_train) * sign
    d_test = (cand_test - champ_test) * sign
    if d_train < -min_effect or d_test < 0:
        return False, f"regressed (train {d_train:+.3f}, test {d_test:+.3f})"
    if d_train > min_effect and d_test > 0:
        return True, f"kept: train {d_train:+.3f} beats the noise band {min_effect:.3f} and test improved {d_test:+.3f}"
    if d_train > min_effect:
        return False, f"overfit: train {d_train:+.3f} improved but test is flat ({d_test:+.3f})"
    return False, f"within noise: train {d_train:+.3f} vs band {min_effect:.3f}"


def stalled(suite: str) -> int:
    """Consecutive non-kept rounds for this suite, most recent first."""
    if not os.path.exists(LOG_PATH):
        return 0
    n = 0
    with open(LOG_PATH) as f:
        rounds = [json.loads(l) for l in f if l.strip()]
    for e in reversed([r for r in rounds if r.get("kind") == "round" and r.get("suite") == suite]):
        if e.get("keep"):
            break
        n += 1
    return n


def cmd_split(args) -> None:
    path = split_path(args.suite)
    if os.path.exists(path) and not args.force:
        raise SystemExit(f"{path} exists; the test set must stay fixed across rounds (--force to redraw)")
    split = make_split(suite_tasks(args.suite), args.test_frac, args.seed)
    split["suite"] = args.suite
    os.makedirs(LOG_DIR, exist_ok=True)
    with open(path, "w") as f:
        json.dump(split, f, indent=1)
        f.write("\n")
    print(f"{args.suite}: {len(split['train'])} train, {len(split['test'])} held out -> {path}")


def cmd_noise(args) -> None:
    split = load_split(args.suite)
    path = run_eval(args.harness, args.model, args.suite, split["train"] + split["test"], args.reps, args.jobs)
    summary = noise_summary(load_jsonl(path), args.harness, split, args.reps)
    summary.update(suite=args.suite, model=args.model, results=path)
    out = os.path.join(LOG_DIR, f"noise-{args.suite}-{args.harness}-{args.model}.json")
    with open(out, "w") as f:
        json.dump(summary, f, indent=1)
        f.write("\n")
    tr = summary["sets"]["train"]
    print(f"train pass {tr['pass']['mean']:.3f} (rep sd {tr['pass']['rep_sd']:.3f}, band {tr['pass']['band']:.3f});"
          f" flaky: {', '.join(tr['flaky_tasks']) or 'none'}")
    if summary["headroom_warning"]:
        print(f"WARNING: baseline is at or above {HEADROOM:.0%} on train; improvements will be hard to see")
    print(f"-> {out}")


def cmd_round(args) -> None:
    split = load_split(args.suite)
    noise_file = os.path.join(LOG_DIR, f"noise-{args.suite}-{args.champion}-{args.model}.json")
    min_effect = args.min_effect
    reps = args.reps
    if min_effect is None and not args.noise_from_round:
        if not os.path.exists(noise_file):
            raise SystemExit(f"no noise measurement ({noise_file}); run hillclimb.py noise first, "
                             "pass --min-effect, or use --noise-from-round")
        with open(noise_file) as f:
            noise = json.load(f)
        min_effect = noise["sets"]["train"][args.goal]["band"]
        reps = reps or noise["reps"]
    reps = reps or (2 if args.noise_from_round else 3)
    if args.noise_from_round and reps < 2:
        raise SystemExit("--noise-from-round needs at least 2 repetitions")
    path = run_eval(f"{args.champion},{args.candidate}", args.model, args.suite,
                    split["train"] + split["test"], reps, args.jobs)
    rows = load_jsonl(path)
    pick = lambda h, s: [r for r in rows if r.get("harness") == h and r.get("task") in split[s]]
    if min_effect is None:
        # The champion's own repetitions in this round give the noise band.
        noise = noise_summary(rows, args.champion, split, reps)
        min_effect = noise["sets"]["train"][args.goal]["band"]
        print(f"noise from this round's champion repetitions: {args.goal} band {min_effect:.4f}")
    for name in ("train", "test"):
        print(f"\n{name}")
        print(table(bucket(pick(args.champion, name) + pick(args.candidate, name))))
    scores = {h: {s: metric(pick(h, s), args.goal) for s in ("train", "test")} for h in (args.champion, args.candidate)}
    keep, why = judge(scores[args.champion]["train"], scores[args.candidate]["train"],
                      scores[args.champion]["test"], scores[args.candidate]["test"], min_effect, args.goal)
    guard = metric(pick(args.candidate, "train"), "pass") < metric(pick(args.champion, "train"), "pass") - min_effect
    if keep and args.goal != "pass" and guard:
        keep, why = False, "pass rate dropped while chasing " + args.goal
    print(f"{args.goal}: champion train {scores[args.champion]['train']:.4f} test {scores[args.champion]['test']:.4f}"
          f" | candidate train {scores[args.candidate]['train']:.4f} test {scores[args.candidate]['test']:.4f}")
    print(("KEEP " if keep else "REVERT ") + why)
    append_log({"kind": "round", "suite": args.suite, "goal": args.goal, "candidate_id": args.candidate_id or args.candidate,
                "champion": args.champion, "candidate": args.candidate, "model": args.model, "reps": reps,
                "min_effect": min_effect, "scores": scores, "keep": keep, "why": why, "results": path})
    n = stalled(args.suite)
    if n >= STALL_ROUNDS:
        print(f"\nstalled: {n} rounds without a kept change. Analyze what still fails on TRAIN by root cause:")
        for r in pick(args.champion, "train"):
            if not r.get("outcome_ok"):
                print(f"  {r['task']} rep {r.get('rep')}: {(r.get('check_note') or r.get('error') or '')[:160]}")
        print("If no single fix could beat the noise band, add repetitions or tasks instead of more rounds.")


def main() -> None:
    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    sub = ap.add_subparsers(dest="cmd", required=True)

    p = sub.add_parser("self-test")
    p.set_defaults(fn=lambda _: (price_self_test(), _decide_self_test(), _heldout_self_test(), print("hillclimb self-test ok")))

    p = sub.add_parser("score")
    p.add_argument("jsonl")
    p.add_argument("--json", action="store_true")
    p.set_defaults(fn=cmd_score)

    p = sub.add_parser("decide")
    p.add_argument("--champion", required=True)
    p.add_argument("--candidate", default=None)
    p.add_argument("--same-file", action="store_true")
    p.add_argument("--ours", default=OURS_DEFAULT)
    p.add_argument("--cand-harness", default=None)
    p.add_argument("--theirs", default=THEIRS_DEFAULT)
    p.add_argument("--candidate-id", default="")
    p.add_argument("--json", action="store_true")
    p.set_defaults(fn=cmd_decide)

    p = sub.add_parser("iterate")
    p.add_argument("--ours", default=OURS_DEFAULT)
    p.add_argument("--theirs", default=THEIRS_DEFAULT)
    p.add_argument("--model", default="grok-4.6")
    p.add_argument("--suite", default="core")
    p.add_argument("--task", default="")
    p.add_argument("--reps", type=int, default=1)
    p.add_argument("--jobs", "-j", type=int, default=1)
    p.add_argument("--only", default="")
    p.set_defaults(fn=cmd_iterate)

    p = sub.add_parser("split")
    p.add_argument("--suite", required=True)
    p.add_argument("--test-frac", type=float, default=0.33)
    p.add_argument("--seed", default="graff")
    p.add_argument("--force", action="store_true")
    p.set_defaults(fn=cmd_split)

    p = sub.add_parser("noise")
    p.add_argument("--suite", required=True)
    p.add_argument("--harness", default=OURS_DEFAULT)
    p.add_argument("--model", default="grok-4.6")
    p.add_argument("--reps", type=int, default=3)
    p.add_argument("--jobs", "-j", type=int, default=1)
    p.set_defaults(fn=cmd_noise)

    p = sub.add_parser("round")
    p.add_argument("--suite", required=True)
    p.add_argument("--champion", default=OURS_DEFAULT)
    p.add_argument("--candidate", required=True)
    p.add_argument("--candidate-id", default="")
    p.add_argument("--model", default="grok-4.6")
    p.add_argument("--goal", choices=sorted(GOALS), default="pass")
    p.add_argument("--reps", type=int, default=None)
    p.add_argument("--min-effect", type=float, default=None)
    p.add_argument("--noise-from-round", action="store_true",
                   help="take the noise band from the champion's repetitions in this round (no separate baseline)")
    p.add_argument("--jobs", "-j", type=int, default=1)
    p.set_defaults(fn=cmd_round)

    args = ap.parse_args()
    if args.cmd == "decide" and not args.candidate and not args.same_file:
        ap.error("decide needs --candidate or --same-file")
    if args.cmd == "decide" and args.same_file:
        args.candidate = args.champion
    args.fn(args)


def _decide_self_test() -> None:
    champ = {"ok": 3, "n": 3, "pass": "3/3", "wall_s": 30.0, "first_out_s": 2.0,
             "tok_calls": 10, "list_tokens": 20_000, "list_usd": 0.05, "rss_peak_kb": 9000}
    better = dict(champ, wall_s=20.0, tok_calls=6, list_tokens=12_000, list_usd=0.03, first_out_s=1.5)
    d = decide(champ, better)
    assert d["keep"], d
    worse = dict(champ, ok=2, wall_s=10.0, list_usd=0.01)
    d = decide(champ, worse)
    assert not d["keep"]
    heap = dict(better, rss_peak_kb=170_000)
    d = decide(champ, heap)
    assert not d["keep"]
    catalog = decide(champ, better, candidate_id="four-tool-copy")
    assert not catalog["keep"]



def _heldout_self_test() -> None:
    ids = [f"t{i}" for i in range(12)]
    a, b = make_split(ids, 0.33, "graff"), make_split(list(reversed(ids)), 0.33, "graff")
    assert a == b, "split must not depend on listing order"
    assert not set(a["train"]) & set(a["test"]) and sorted(a["train"] + a["test"]) == sorted(ids)
    assert len(a["test"]) == 4
    assert make_split(ids, 0.33, "other") != a
    rows = [{"harness": "h", "task": t, "rep": rep, "outcome_ok": not (t == "t0" and rep == 1)}
            for rep in range(3) for t in ids]
    split = {"train": [t for t in ids if t != "t11"], "test": ["t11"]}
    n = noise_summary(rows, "h", split, 3)
    assert n["sets"]["train"]["pass"]["rep_sd"] > 0 and n["sets"]["test"]["pass"]["rep_sd"] == 0
    assert n["sets"]["train"]["flaky_tasks"] == ["t0"] and n["headroom_warning"]
    keep, why = judge(0.60, 0.80, 0.50, 0.60, 0.05)
    assert keep, why
    keep, why = judge(0.60, 0.80, 0.50, 0.50, 0.05)
    assert not keep and why.startswith("overfit"), why
    keep, why = judge(0.60, 0.62, 0.50, 0.60, 0.05)
    assert not keep and why.startswith("within noise"), why
    keep, why = judge(0.60, 0.70, 0.50, 0.40, 0.05)
    assert not keep and why.startswith("regressed"), why
    keep, why = judge(0.050, 0.030, 0.050, 0.040, 0.005, goal="usd")
    assert keep, why


if __name__ == "__main__":
    main()