Official Codebase of 3DCodeVerse
Python
flywheel: the refine exporter folds into the package; the format conversion moves to toolkits
ziyaof67a02d89078
diff --git a/harness/bench/refine_pairs.py b/harness/bench/refine_pairs.pydeleted file mode 100644index 278a6120..00000000--- a/harness/bench/refine_pairs.py+++ /dev/null@@ -1,585 +0,0 @@-"""Refine rounds as training pairs: what the round was TOLD, what it CHANGED, what it SCORED.--``codeverse/flywheel/pairs.py`` exports whole code trees per round and never looks at-what asked for the change, so a refine round arrives as two 30 KB files with no cause.-This exports the transition instead — one row per refine round::-- (previous round's judge issues + improvement plan + gate ERROR findings- + the instruction lines the agent sessions were handed) -> src/ diff -> score delta--Both sides come from the round's OWN recorded commit, never from ``HEAD``: only 54 of the-239 recorded runs have HEAD at their last round, 152 end on a "restore best round rNN"-commit, and looking a commit up by message is ambiguous (``r01 refine`` also matches-``r01 refine: generated``).-- python bench/refine_pairs.py bench/out --out refine_pairs.jsonl --summary- python bench/refine_pairs.py bench/out/aa_articulated --summary- python bench/refine_pairs.py bench/out/aa_articulated/arms/control/cells/art_med_tool_chest/\\- harness_gemini-cli_gemini-3.7-flash/run --summary--``root`` is a directory OF batteries (``bench/out``), one battery, a runs directory or a-single run — every layout the flywheel scanner knows, and a single run reports the same-identity a whole-tree scan gives it. A run reachable through more than one path (54 of-the 239) is exported once, labelled with the battery it PHYSICALLY lives in: ``bench/out``'s-aliases are cross-battery symlinks (``compare_v4_calm/cells/*`` points into ``compare_v4``-and ``compare_v4_harness_calm``), so labelling by the first path scanned credited 54 of the-205 rows to a battery they never ran in. Every run and round that cannot be exported is-counted under a named reason; the scan report is printed whether or not rows were written.--Two instruction fields, because they are not the same list. ``refine_tasks`` is-``RoundRecord.instructions`` — every task ``build_refine_instructions`` compiled, before-grouping. ``instruction_lines`` is what the agents actually read: the per-group output of-``orchestrator.compact_instructions``, folded by target and capped at-``policy.max_instructions_per_task``, recovered from the round's own prompt files. They-differ for 167 of the corpus's 205 transitions.-"""--from __future__ import annotations--import argparse-import itertools-import json-import re-import statistics-import sys-from collections import Counter-from collections.abc import Iterator, Sequence-from dataclasses import dataclass, field-from pathlib import Path-from typing import Any--sys.path.insert(0, str(Path(__file__).resolve().parents[1]))--from codeverse.contracts.run import RUN_PATH_NOISE, RoundRecord, RunId, RunRecord # noqa: E402-from codeverse.flywheel import _git # noqa: E402-from codeverse.flywheel.quality import prompt_hash # noqa: E402-from codeverse.flywheel.record import ( # noqa: E402- BATTERY_MARKERS,- RUN_SEARCH_DEPTH,- FoundRun,- RecordError,- effective_judgment,- is_run_dir,- iter_runs,- load_record,- run_id_for,-)-from codeverse.workspace import Workspace # noqa: E402--SCHEMA_VERSION = 1--#: round kinds that are a refine transition — a round handed instructions derived from-#: the previous round's verdict, and the only three the recorded corpus holds.-#: ``baseline`` has no predecessor, and ``texture`` / ``asset:*`` rounds are driven by-#: the texture pass rather than by a judgment.-REFINE_KINDS: tuple[str, ...] = ("refine", "rewrite", "detail")--#: cap on the inlined diff. The corpus's largest transition diff is ~62 KB, so nothing-#: recorded today is truncated; the cap only stops a pathological row from eating a-#: dataset, and every row records it next to the pre-cap size.-MAX_DIFF_BYTES = 200_000--#: |Δscore| below which a round counts as ``unchanged``. Same 0.05 the flywheel's-#: preference pairs already use as "a real difference" (``flywheel/pairs.build_pairs``).-OUTCOME_THRESHOLD = 0.05--DROP_REASONS: tuple[str, ...] = (- "not_a_battery",- "empty_battery",- "unreadable_record",- "duplicate_run",- "missing_predecessor",- "missing_commit",- "missing_prev_judgment",- "no_instructions",- "git_read_failed",- "empty_diff",-)---# --------------------------------------------------------------------------- instructions-_FILES_RE = re.compile(r"\s*\(files:\s*(?P<files>[^)]*)\)\s*$")-_HEAD_RE = re.compile(r"^\[(?P<source>[^/\]]+)/(?P<kind>[^\]]+)\]\s*(?P<rest>.*)$", re.S)-_TARGET_RE = re.compile(r"[A-Za-z0-9_.:+-]{1,64}")-_BULLET_SPLIT_RE = re.compile(r";\s+(?=\([a-z]\)\s)")-# the optional instance override is ``[Leg_1]``; the ``/`` keeps it from swallowing the-# ``[source/kind]`` head that follows it-_BULLET_RE = re.compile(r"^\((?P<letter>[a-z])\)\s*(?:\[(?P<target>[^\]/]+)\]\s*)?(?P<rest>.*)$", re.S)---#: heading every refine template renders ``compact_instructions``' output under-#: (``prompts/tracks/refine_object.j2``, ``refine_graphics.j2``, ``scene_refine.j2``)-_CHANGES_HEADING = "## Changes to make"-#: what follows the numbered list in those templates-_CHANGES_END = ("## ", "Targets:", "Files in scope:", "EDIT ONLY THESE FILES")-_NUMBERED_RE = re.compile(r"^\s*\d+\.\s+(?P<line>.*)$")---def _task(source: str, kind: str, target: str, instruction: str, files: list[str]) -> dict[str, Any]:- return {"source": source, "kind": kind, "target": target.strip(),- "instruction": instruction.strip(), "files": files}---def changes_block(prompt: str) -> list[str]:- """The instruction lines of one rendered refine prompt.-- The templates number the lines (``{{ loop.index }}. {{ t }}``), so the numbering is- stripped back off; a line wrapped over several lines of markdown is rejoined onto the- entry it belongs to. A prompt without the heading (a rebuild, a texture pass, a- baseline) yields nothing.- """- out: list[str] = []- started = False- for ln in prompt.splitlines():- if not started:- started = ln.startswith(_CHANGES_HEADING)- continue- if ln.startswith(_CHANGES_END):- break- if (m := _NUMBERED_RE.match(ln)) is not None:- out.append(m.group("line").strip())- elif ln.strip() and out:- out[-1] += " " + ln.strip()- return out---def sent_instructions(ws: Workspace, round_index: int) -> list[dict[str, Any]]:- """What each of the round's agent sessions was actually handed, per session.-- A refine round fans out over file-disjoint groups, and each group's session gets its- OWN ``compact_instructions`` output — folded by target, capped at- ``policy.max_instructions_per_task``. ``RoundRecord.instructions`` keeps none of- that: it is the flat pre-grouping task list. The prompt files the sessions were- started with (``trajectories/<label>_rNN/prompt.md``) do, so they are the only record- of the real brief; 195 of the corpus's 205 transitions still have theirs on disk.- A ``<label>.a<n>`` retry re-sends the same brief, so identical blocks are kept once.- """- traj = ws.root / "trajectories"- suffix = f"_r{round_index:02d}"- dirs = sorted(traj.glob(f"*{suffix}")) if traj.is_dir() else []- out: list[dict[str, Any]] = []- seen: set[tuple[str, ...]] = set()- # sorted by LABEL, not by directory name: `.` sorts before `_`, so sorting the names- # puts `refine.a2_r01` in front of the `refine_r01` whose retry it is, and the retry- # would then be the session the brief is credited to- for label in sorted(d.name[: -len(suffix)] for d in dirs):- try:- text = (traj / f"{label}{suffix}" / "prompt.md").read_text(errors="replace")- except OSError:- continue- lines = changes_block(text)- if not lines or tuple(lines) in seen:- continue- seen.add(tuple(lines))- out.append({"session": label, "lines": lines})- return out---def parse_instruction(line: str) -> list[dict[str, Any]]:- """The task(s) encoded in one recorded instruction line.-- ``orchestrator.compact_instructions`` emits three shapes and the corpus holds all- three: a single ``RefineTask.line()`` (``[source/kind] target: text (files: a, b)``),- the grouped form (``target: (a) [source/kind] text; (b) …``) and — detail rounds — a- bare sentence. A line that matches none of them still yields one task carrying its- whole text, so no instruction is silently lost; the target prefix is only honoured- when it is a single bare token, or a sentence's own colon would be read as one.- """- raw = line.strip()- files: list[str] = []- if (m := _FILES_RE.search(raw)) is not None:- files = [f.strip() for f in m.group("files").split(",") if f.strip()]- raw = raw[: m.start()].rstrip()- if (head := _HEAD_RE.match(raw)) is not None:- target, sep, text = head.group("rest").partition(": ")- if not sep:- target, text = "overall", head.group("rest")- return [_task(head.group("source"), head.group("kind"), target, text, files)]- target, sep, rest = raw.partition(": ")- if not sep or _TARGET_RE.fullmatch(target) is None:- return [_task("", "", "", raw, files)]- bullets = _BULLET_SPLIT_RE.split(rest)- if _BULLET_RE.match(bullets[0]) is None:- return [_task("", "", target, rest, files)]- out: list[dict[str, Any]] = []- for b in bullets:- bm = _BULLET_RE.match(b)- if bm is None:- out.append(_task("", "", target, b, files))- continue- who, inner = bm.group("target") or target, bm.group("rest")- if (hm := _HEAD_RE.match(inner)) is not None:- out.append(_task(hm.group("source"), hm.group("kind"), who, hm.group("rest"), files))- else:- out.append(_task("", "", who, inner, files))- return out---# --------------------------------------------------------------------------- rows-def outcome_label(delta: float | None, *, threshold: float = OUTCOME_THRESHOLD) -> str:- """``improved`` / ``regressed`` / ``unchanged`` by ``threshold``; ``unscored`` when- either side has no usable verdict (a degraded judge outage is *not* a 0.0)."""- if delta is None:- return "unscored"- if delta > threshold:- return "improved"- if delta < -threshold:- return "regressed"- return "unchanged"---def _gate_errors(rnd: RoundRecord) -> list[dict[str, Any]]:- """The ERROR findings of a round's gates — the half of the refine brief the judge- did not write. ``fix_hint`` is what ``build_refine_instructions`` copies verbatim."""- return [{"gate": g.gate, "target": f.target or "", "message": f.message,- "fix_hint": f.fix_hint, "data": f.data}- for g in rnd.gates for f in g.errors]---def _side(rnd: RoundRecord, files: list[str]) -> dict[str, Any]:- j = effective_judgment(rnd)- return {- "round": rnd.index,- "kind": rnd.kind,- "commit": rnd.commit,- "score": j.overall if j else None,- "passed": j.passed if j else None,- "build_ok": None if rnd.build is None else rnd.build.ok,- "gate_errors": sum(len(g.errors) for g in rnd.gates),- "files": files,- }---def _diff_lines(text: str) -> tuple[int, int]:- """(added, removed) body lines of a unified diff — the ``+++``/``---`` headers are- file markers, not content, and would otherwise count once per changed file."""- added = removed = 0- for ln in text.splitlines():- if ln.startswith("+") and not ln.startswith("+++"):- added += 1- elif ln.startswith("-") and not ln.startswith("---"):- removed += 1- return added, removed---def transition_row(- ws: Workspace, rec: RunRecord, run_id: RunId, prev: RoundRecord, cur: RoundRecord,- *, threshold: float = OUTCOME_THRESHOLD, max_diff_bytes: int = MAX_DIFF_BYTES,- files_cache: dict[str, list[str]] | None = None,-) -> dict[str, Any]:- """One training row for the transition ``prev`` → ``cur``. Raises- :class:`~codeverse.flywheel._git.GitReadError` when either commit is unreadable."""- cache = files_cache if files_cache is not None else {}- for c in (prev.commit, cur.commit):- if c not in cache:- cache[c] = _git.list_files_at(ws, c)- diff, diff_bytes, truncated = _git.diff_between(ws, prev.commit, cur.commit, max_bytes=max_diff_bytes)- added, removed = _diff_lines(diff)- pj, cj = effective_judgment(prev), effective_judgment(cur)- delta = None if pj is None or cj is None else round(cj.overall - pj.overall, 4)- sent = sent_instructions(ws, cur.index)- lines = [ln for s in sent for ln in s["lines"]] or list(cur.instructions)- return {- "schema_version": SCHEMA_VERSION,- "battery": run_id.battery,- "run": run_id.slug,- "run_dir": str(ws.root),- "cell": run_id.cell,- "arm": run_id.arm,- "prompt_id": rec.spec.id,- "prompt": rec.spec.prompt,- "prompt_hash": prompt_hash(rec.spec.prompt),- "track": rec.spec.track.value,- "language": rec.spec.language.value,- "generator": rec.spec.backends.generator,- "agent_backend": cur.agent_backend,- "round": cur.index,- "round_kind": cur.kind,- "issues": [i.model_dump(mode="json") for i in pj.issues] if pj else [],- "improvement_plan": [i.model_dump(mode="json") for i in pj.improvement_plan] if pj else [],- "gate_findings": _gate_errors(prev),- # what was COMPILED (RoundRecord.instructions) vs what was SENT (the per-group- # compacted prompts). Neither is the other: see the module docstring.- "refine_tasks": list(cur.instructions),- "sent_instructions": sent,- "instruction_source": "prompt" if sent else "record",- "instruction_lines": lines,- "instruction_tasks": [t for line in lines for t in parse_instruction(line)],- "diff": diff,- "diff_bytes": diff_bytes,- "diff_added": added,- "diff_removed": removed,- "diff_truncated": truncated,- "diff_max_bytes": max_diff_bytes,- "changed_files": _git.changed_files_between(ws, prev.commit, cur.commit),- "before": _side(prev, cache[prev.commit]),- "after": _side(cur, cache[cur.commit]),- "score_before": pj.overall if pj else None,- "score_after": cj.overall if cj else None,- "score_delta": delta,- "outcome": outcome_label(delta, threshold=threshold),- "outcome_threshold": threshold,- }---# --------------------------------------------------------------------------- scan-@dataclass-class Report:- """What one export saw: totals, named drops, and a digest per written row."""-- roots: list[str] = field(default_factory=list)- skipped: list[str] = field(default_factory=list)- runs: int = 0- kinds: Counter[str] = field(default_factory=Counter)- drops: Counter[str] = field(default_factory=Counter)- aliases: Counter[str] = field(default_factory=Counter)- stats: list[dict[str, Any]] = field(default_factory=list)-- @property- def rows(self) -> int:- return len(self.stats)-- @property- def transitions(self) -> int:- """Refine-kind rounds seen — rows plus every per-transition drop."""- return sum(self.kinds[k] for k in REFINE_KINDS)---def scan_roots(root: Path) -> tuple[list[Path], list[Path]]:- """``(scan roots, skipped children)`` for ``root``.-- One scan root per battery, so ``RunId.battery`` stays a battery name: ``bench/out``- holds runs 5 levels down, and scanning it whole would label every run ``out`` and- collapse the per-battery summary. A child that is neither a run nor a battery is- RETURNED as skipped rather than dropped in silence — ``bench/out`` holds three of- them (report-only merge directories with no cells of their own).- """- if is_run_dir(root) or any((root / m).is_dir() for m in BATTERY_MARKERS):- return [root], []- kids = sorted(p for p in root.iterdir() if p.is_dir())- if any(is_run_dir(k) for k in kids):- return [root], []- roots = [k for k in kids if is_run_dir(k) or any((k / m).is_dir() for m in BATTERY_MARKERS)]- if not roots:- return [root], []- return roots, [k for k in kids if k not in roots]---def battery_root_for(run_dir: Path) -> Path:- """The root that gives ``run_dir`` its battery label when no scan root claims it.-- The OUTERMOST ancestor (within :data:`RUN_SEARCH_DEPTH`) whose path down to the run- begins with a battery marker — ``bench/out/<battery>/arms/<arm>/cells/…/run`` reports- ``<battery>``, the same label a whole-``bench/out`` scan mints for it, so pointing the- exporter at one run and finding it in a full scan produce the same identity. A layout- with no marker at all (``<dir>/<slug>``) falls back to the run's parent.- """- root = run_dir.parent- for p in run_dir.parents[:RUN_SEARCH_DEPTH]:- parts = run_dir.relative_to(p).parts- if parts[0] in RUN_PATH_NOISE:- root = p- return root---def physical_run_id(resolved: Path, roots: Sequence[Path], found: RunId) -> RunId:- """``found`` re-minted against the battery the run PHYSICALLY lives in.-- ``found`` names the root the scan REACHED the run through, and in ``bench/out`` that- is not always where it ran: 54 runs are reachable through a symlink in another- battery's cells. The deepest scan root containing the resolved path wins; a run- linked in from outside the scanned tree keeps its own :func:`battery_root_for`.- """- owners = [r for r in roots if resolved.is_relative_to(r)]- if owners:- return run_id_for(max(owners, key=lambda r: len(r.parts)), resolved)- return run_id_for(battery_root_for(resolved), resolved)---def _found_runs(scan: Path, report: Report) -> Iterator[FoundRun]:- """Every run under one scan root — or ``scan`` itself when it IS a run directory.-- ``iter_runs`` searches strictly BELOW its argument, so a lone run directory (the- documented single-run input) yields nothing there and the export silently reported- zero rows. It is read here instead, against the battery root its path sits in.- """- if is_run_dir(scan):- ws = Workspace(scan)- try:- rec = load_record(ws)- except RecordError:- report.drops["unreadable_record"] += 1- return- yield FoundRun(ws, rec, run_id_for(battery_root_for(ws.root), ws.root))- return- found = iter_runs(scan, on_error=lambda _p, _e: report.drops.update(["unreadable_record"]))- try: # iter_runs is a generator: it raises for an empty battery on the first step- found = itertools.chain([next(found)], found)- except FileNotFoundError:- report.drops["empty_battery"] += 1- return- except StopIteration:- return- yield from found---def iter_rows(- root: Path, report: Report, *, threshold: float = OUTCOME_THRESHOLD,- max_diff_bytes: int = MAX_DIFF_BYTES,-) -> Iterator[dict[str, Any]]:- """Yield one row per exportable refine transition under ``root``, filling ``report``.-- Every run and every round is accounted for: a run is exported once per resolved path- (``bench/out``'s cross-battery symlinks make runs reachable twice), it is labelled- with the battery it physically lives in, and a child directory or transition that- cannot be exported increments a named reason instead of vanishing.- """- roots, skipped = scan_roots(root)- report.skipped = [str(p) for p in skipped]- if skipped: # a Counter records a zero once touched, and "dropped: none" must stay true- report.drops["not_a_battery"] += len(skipped)- batteries = [r.resolve() for r in roots if not is_run_dir(r)]- seen: set[Path] = set()- for scan in roots:- report.roots.append(str(scan))- for ws, rec, found_id in _found_runs(scan, report):- resolved = ws.root # Workspace resolves its root on construction- if resolved in seen:- report.drops["duplicate_run"] += 1- continue- seen.add(resolved)- run_id = physical_run_id(resolved, batteries, found_id)- if run_id.battery != found_id.battery:- report.aliases[f"{found_id.battery} -> {run_id.battery}"] += 1- report.runs += 1- yield from _run_rows(ws, rec, run_id, report, threshold=threshold, max_diff_bytes=max_diff_bytes)---def _run_rows(- ws: Workspace, rec: RunRecord, run_id: RunId, report: Report, *, threshold: float, max_diff_bytes: int-) -> Iterator[dict[str, Any]]:- by_index = {r.index: r for r in rec.rounds}- files_cache: dict[str, list[str]] = {}- for cur in rec.rounds:- report.kinds[cur.kind] += 1- if cur.kind not in REFINE_KINDS:- continue- prev = by_index.get(cur.index - 1)- if prev is None:- report.drops["missing_predecessor"] += 1- continue- if not prev.commit or not cur.commit:- report.drops["missing_commit"] += 1- continue- if effective_judgment(prev) is None:- report.drops["missing_prev_judgment"] += 1- continue- # the RECORD's task list decides exportability: a round with no compiled task was- # never given a brief, while a missing prompt file only costs the row its- # per-session breakdown (transition_row then falls back to this same list)- if not cur.instructions:- report.drops["no_instructions"] += 1- continue- try:- row = transition_row(ws, rec, run_id, prev, cur, threshold=threshold,- max_diff_bytes=max_diff_bytes, files_cache=files_cache)- except _git.GitReadError:- report.drops["git_read_failed"] += 1- continue- if not row["diff"].strip():- report.drops["empty_diff"] += 1- continue- report.stats.append({"battery": row["battery"], "track": row["track"], "outcome": row["outcome"],- "diff_bytes": row["diff_bytes"], "diff_added": row["diff_added"],- "diff_removed": row["diff_removed"]})- yield row---def export(- root: Path, out: Path | None, *, threshold: float = OUTCOME_THRESHOLD,- max_diff_bytes: int = MAX_DIFF_BYTES,-) -> Report:- """Write the rows under ``root`` as JSONL to ``out`` (``None`` = count only)."""- report = Report()- rows = iter_rows(root, report, threshold=threshold, max_diff_bytes=max_diff_bytes)- if out is None:- for _ in rows:- pass- return report- out.parent.mkdir(parents=True, exist_ok=True)- tmp = out.with_suffix(out.suffix + ".tmp")- with tmp.open("w") as fh:- for row in rows:- fh.write(json.dumps(row, ensure_ascii=False) + "\n")- tmp.replace(out)- return report---# --------------------------------------------------------------------------- report-def _table(title: str, counts: Counter[str], *, by_count: bool = True) -> list[str]:- if not counts:- return []- items = counts.most_common() if by_count else sorted(counts.items())- width = max(len(k) for k, _ in items)- return [f"{title}:"] + [f" {k:<{width}} {n:>5}" for k, n in items]---def scan_report(report: Report) -> str:- """Totals + every drop reason. Printed always: a truncated export must never look- like a small corpus."""- by_kind = ", ".join(f"{n} {k}" for k, n in sorted(report.kinds.items())) or "none"- lines = [f"{report.rows} row(s) from {report.runs} run(s) in {len(report.roots)} root(s)",- f"rounds seen: {sum(report.kinds.values())} "- f"({report.transitions} refine-kind; {by_kind})"]- lines += _table("dropped", report.drops, by_count=False) or ["dropped: none"]- if report.skipped:- lines += ["skipped (neither a run nor a battery):"] + [f" {p}" for p in report.skipped]- # not a drop: the run WAS exported, under the battery it physically lives in- lines += _table("reached through another battery's symlink", report.aliases, by_count=False)- return "\n".join(lines)---def summary_report(report: Report, *, threshold: float = OUTCOME_THRESHOLD) -> str:- """Counts per battery / track / outcome, and how big the diffs are."""- if not report.stats:- return "no rows to summarise"- lines = _table("rows per battery", Counter(s["battery"] for s in report.stats))- lines += [""] + _table("rows per track", Counter(s["track"] for s in report.stats))- lines += [""] + _table(f"rows per outcome (|delta| > {threshold})",- Counter(s["outcome"] for s in report.stats))- sizes = sorted(s["diff_bytes"] for s in report.stats)- churn = sorted(s["diff_added"] + s["diff_removed"] for s in report.stats)- lines += ["", f"diff size: median {int(statistics.median(sizes))} bytes "- f"(min {sizes[0]}, max {sizes[-1]}); "- f"median {int(statistics.median(churn))} changed lines"]- return "\n".join(lines)---def main(argv: Sequence[str] | None = None) -> int:- ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)- ap.add_argument("root", type=Path, help="directory of batteries, a battery, a runs dir or one run")- ap.add_argument("--out", type=Path, help="write the rows here as JSONL")- ap.add_argument("--summary", action="store_true", help="also print per-battery/track/outcome counts")- ap.add_argument("--threshold", type=float, default=OUTCOME_THRESHOLD,- help=f"|delta| above which a round improved/regressed (default {OUTCOME_THRESHOLD})")- ap.add_argument("--max-diff-bytes", type=int, default=MAX_DIFF_BYTES,- help=f"cap on the inlined diff, recorded on every row (default {MAX_DIFF_BYTES})")- args = ap.parse_args(argv)-- if not args.root.is_dir():- ap.error(f"{args.root} is not a directory")- if args.out is None and not args.summary:- ap.error("nothing to do: pass --out to write rows, --summary to print counts, or both")-- report = export(args.root, args.out, threshold=args.threshold, max_diff_bytes=args.max_diff_bytes)- if args.out is not None:- print(f"wrote {args.out}")- print(scan_report(report))- if args.summary:- print()- print(summary_report(report, threshold=args.threshold))- return 0 if report.rows else 1---if __name__ == "__main__":- raise SystemExit(main())diff --git a/harness/bench/refine_sft.py b/harness/bench/refine_sft.pydeleted file mode 100644index 921b5082..00000000--- a/harness/bench/refine_sft.py+++ /dev/null@@ -1,184 +0,0 @@-"""Refine rounds as SFT samples: the brief the round was given, answered with the code it produced.--``bench/refine_pairs.py`` exports the transition as data (issues, findings, instructions,-diff, delta). This turns the rows that IMPROVED into the message shape the finetune-pipeline consumes (``{"id", "messages": [system, user, assistant]}`` — see-``toolkits/llamafactory/to_llamafactory.py``), so the corpus teaches the one thing the-harness produces and a one-shot dataset cannot: given a judged, gated object and a list of-what is wrong with it, write the corrected files.-- python bench/refine_sft.py bench/out --out refine_sft.jsonl --summary--The assistant turn is the FULL text of every file the round changed, in the same-``=== FILE: path ===`` envelope ``tracks/generation`` parses, because that is the answer-shape a trained model has to produce. Rows are dropped, and counted, when the answer-would not fit ``--max-answer-chars`` or a side cannot be read back out of the run's repo.-"""--from __future__ import annotations--import argparse-import json-import sys-from collections import Counter-from pathlib import Path-from statistics import median--sys.path.insert(0, str(Path(__file__).resolve().parents[1]))--from bench.refine_pairs import Report, iter_rows # noqa: E402-from codeverse.contracts.common import Language # noqa: E402-from codeverse.flywheel._git import GitReadError, read_tree_at # noqa: E402-from codeverse.tracks.prompting import language_system_prompt # noqa: E402-from codeverse.workspace import Workspace # noqa: E402--#: a sample whose answer is longer than this is dropped rather than truncated: half a-#: file teaches a model to write half a file-MAX_ANSWER_CHARS = 120_000-#: outcomes worth training on. A regressed round is a real transition and stays in-#: refine_pairs.jsonl (it is evidence about the loop), but it is not an answer to copy.-DEFAULT_OUTCOMES = ("improved",)-SCHEMA_VERSION = 1---def envelope(files: dict[str, str]) -> str:- """The ``=== FILE: path === … === END FILE ===`` blocks ``tracks.generation.parse_multifile``- reads — the same envelope a single-shot answer must produce, terminator included."""- return "\n".join(f"=== FILE: {p} ===\n{files[p].rstrip()}\n=== END FILE ===" for p in sorted(files))---def _bullets(items: list[dict], *keys: str, limit: int = 12) -> list[str]:- out = []- for it in items[:limit]:- parts = [str(it.get(k, "")).strip() for k in keys]- line = " — ".join(p for p in parts if p)- if line:- out.append(f"- {line}")- return out---def user_turn(row: dict, before: dict[str, str]) -> str:- """The brief: what was asked for, what was wrong, and the code as it stood."""- lines = [f"Refine this {row['track'].replace('_', ' ')} written in {row['language']}.", "",- "## Request", row["prompt"].strip(), ""]- if row.get("gate_findings"):- lines += ["## Gate errors (measured by the harness — fix these first)"]- lines += _bullets(row["gate_findings"], "target", "message", "fix_hint")- lines += [""]- if row.get("issues"):- lines += ["## What the judge saw"]- lines += _bullets(row["issues"], "target", "severity", "detail")- lines += [""]- if row.get("instruction_lines"):- lines += ["## Changes to make", *(f"{i}. {ln}" for i, ln in enumerate(row["instruction_lines"], 1)), ""]- lines += ["## Current files", envelope(before), "",- "Return every file you change in full, in the same `=== FILE: path ===` envelope."]- return "\n".join(lines)---def sample(row: dict, *, max_answer_chars: int = MAX_ANSWER_CHARS) -> dict | None:- """One SFT row, or ``None`` with the reason recorded on ``row['_drop']``."""- changed = [f for f in row.get("changed_files") or [] if f.startswith("src/")]- if not changed:- row["_drop"] = "no_src_change"- return None- ws = Workspace(Path(row["run_dir"]))- try:- before_tree = read_tree_at(ws, row["before"]["commit"])- after_tree = read_tree_at(ws, row["after"]["commit"])- except (GitReadError, OSError) as e:- row["_drop"] = f"git_read_failed: {type(e).__name__}"- return None-- def _text(tree: dict[str, bytes], paths: list[str]) -> dict[str, str] | None:- out = {}- for p in paths:- raw = tree.get(p)- if raw is None:- return None- try:- out[p] = raw.decode("utf-8")- except UnicodeDecodeError:- return None- return out-- after = _text(after_tree, changed)- if after is None:- row["_drop"] = "answer_unreadable"- return None- # the brief carries the files the round STARTED from: whatever it changed, plus the- # rest of src/ it had to stay consistent with- before = _text(before_tree, sorted(p for p in before_tree if p.startswith("src/"))) or {}- answer = envelope(after)- if len(answer) > max_answer_chars:- row["_drop"] = "answer_too_long"- return None- try:- system = language_system_prompt(Language(row["language"]), tools=False)- except ValueError:- row["_drop"] = "unknown_language"- return None- return {- "id": f"{row['battery']}/{row['run']}/r{row['round']:02d}",- "schema_version": SCHEMA_VERSION,- "track": row["track"], "language": row["language"], "prompt_id": row["prompt_id"],- "score_before": row["score_before"], "score_after": row["score_after"],- "score_delta": row["score_delta"], "changed_files": changed,- "messages": [{"role": "system", "content": system},- {"role": "user", "content": user_turn(row, before)},- {"role": "assistant", "content": answer}],- }---def build(root: Path, *, outcomes: tuple[str, ...] = DEFAULT_OUTCOMES,- max_answer_chars: int = MAX_ANSWER_CHARS) -> tuple[list[dict], Report, Counter]:- report = Report()- drops: Counter = Counter()- out: list[dict] = []- for row in iter_rows(root, report):- if row["outcome"] not in outcomes:- drops[f"outcome:{row['outcome']}"] += 1- continue- s = sample(row, max_answer_chars=max_answer_chars)- if s is None:- drops[row.get("_drop", "unknown")] += 1- continue- out.append(s)- return out, report, drops---def main(argv: list[str] | None = None) -> int:- ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])- ap.add_argument("root", type=Path, help="a corpus of batteries, one battery, a runs dir or one run")- ap.add_argument("--out", type=Path, default=None, help="write the samples here (JSONL)")- ap.add_argument("--outcomes", default=",".join(DEFAULT_OUTCOMES),- help=f"comma list of refine_pairs outcomes to keep (default {DEFAULT_OUTCOMES[0]})")- ap.add_argument("--max-answer-chars", type=int, default=MAX_ANSWER_CHARS)- ap.add_argument("--summary", action="store_true", help="print the counts")- ns = ap.parse_args(argv)- outcomes = tuple(x for x in ns.outcomes.split(",") if x)- rows, report, drops = build(ns.root, outcomes=outcomes, max_answer_chars=ns.max_answer_chars)- if ns.out:- ns.out.parent.mkdir(parents=True, exist_ok=True)- ns.out.write_text("".join(json.dumps(r) + "\n" for r in rows))- print(f"wrote {ns.out}")- print(f"{len(rows)} sample(s) from {report.runs} run(s)")- if ns.summary:- print(f"rounds seen: {sum(report.kinds.values())} (" +- ", ".join(f"{k} {v}" for k, v in sorted(report.kinds.items())) + ")")- if report.drops:- print("dropped by the scan: " + ", ".join(f"{k} {v}" for k, v in sorted(report.drops.items())))- if drops:- print("not exported: " + ", ".join(f"{k} {v}" for k, v in sorted(drops.items())))- if rows:- ans = [len(r["messages"][2]["content"]) for r in rows]- usr = [len(r["messages"][1]["content"]) for r in rows]- print(f"answer chars: median {int(median(ans))}, max {max(ans)}; "- f"brief chars: median {int(median(usr))}, max {max(usr)}")- for field in ("track", "language"):- print(f"by {field}: " + ", ".join(f"{k} {v}" for k, v in sorted(Counter(r[field] for r in rows).items())))- return 0---if __name__ == "__main__":- raise SystemExit(main())diff --git a/harness/codeverse/cli/flywheel_cmd.py b/harness/codeverse/cli/flywheel_cmd.pyindex a51e2c66..078085b4 100644--- a/harness/codeverse/cli/flywheel_cmd.py+++ b/harness/codeverse/cli/flywheel_cmd.py@@ -77,6 +77,20 @@ def pairs_cmd(ok(f"{n} pairs → {out_jsonl}")+@flywheel_app.command("refine")+def refine_cmd(+ runs_dir: Annotated[Path, typer.Argument()],+ out_jsonl: Annotated[Path, typer.Argument()],+ with_code: Annotated[bool, typer.Option("--with-code", help="inline the changed files, so a "+ "format converter needs no access to the runs")] = False,+) -> None:+ """Refine rounds as transitions (brief → diff → score delta) → JSONL."""+ from codeverse.flywheel.refine import build_refine++ n, drops = build_refine(runs_dir, out_jsonl, with_code=with_code)+ ok(f"{n} transitions → {out_jsonl}" + (f"; not exported: {dict(drops)}" if drops else ""))++@flywheel_app.command("caption")def caption_cmd(target: Annotated[str, typer.Argument(help="run slug/dir, or a runs root with --all")],diff --git a/harness/codeverse/flywheel/pairs.py b/harness/codeverse/flywheel/pairs.pyindex c60ecf70..2d537f60 100644--- a/harness/codeverse/flywheel/pairs.py+++ b/harness/codeverse/flywheel/pairs.py@@ -79,6 +79,12 @@ def _base(ws: Workspace, rec: RunRecord, kind: str, *, slug: str | None = None)}+#: the judge Δ at which two rounds are a preference pair — and, in+#: :mod:`codeverse.flywheel.refine`, the Δ at which a transition counts as improved. One+#: constant, so a Δ that is a preference pair here is never "unchanged" there.+MIN_PREFERENCE_DELTA = 0.05++def preference_pairs(ws: Workspace, rec: RunRecord, *, min_delta: float,slug: str | None = None) -> list[dict[str, Any]]:cache: dict[str, dict[str, Any]] = {}diff --git a/harness/codeverse/flywheel/refine.py b/harness/codeverse/flywheel/refine.pynew file mode 100644index 00000000..0c1bfc36--- /dev/null+++ b/harness/codeverse/flywheel/refine.py@@ -0,0 +1,204 @@+"""Refine rounds as transitions: what the round was told, what it changed, what it scored.++:mod:`codeverse.flywheel.pairs` exports whole code trees per round and never looks at what+asked for the change, so a refine round arrives as two 30 kB files with no cause. This+exports the transition instead — one row per refine round::++ (the previous round's judge issues + improvement plan + gate ERRORs with their+ fix hints + the refine tasks compiled for the round) -> src/ diff -> score delta++Both sides come from the round's OWN recorded commit, never ``HEAD``: only 54 of 239+recorded runs have HEAD at their last round, and 152 end on a "restore best round rNN"+commit. ``instructions`` is ``RoundRecord.instructions``, the task list+``build_refine_instructions`` compiled — not the per-group prompt text the sessions were+handed, which is a rendering of it.++The rows are data, not a training format: ``toolkits/llamafactory/build_refine_sft.py``+turns them into the message shape the finetune pipeline consumes. ``--with-code`` inlines+the changed files so that conversion needs no access to the runs.+"""++from __future__ import annotations++from collections import Counter+from collections.abc import Iterator+from pathlib import Path+from typing import Any++from pydantic import BaseModel, Field++from codeverse.contracts.artifacts import Severity+from codeverse.contracts.run import RoundRecord, RunRecord+from codeverse.flywheel._git import GitReadError, changed_files_between, diff_between, read_tree_at+from codeverse.flywheel.pairs import MIN_PREFERENCE_DELTA+from codeverse.flywheel.record import effective_judgment, effective_score, iter_runs+from codeverse.workspace import Workspace++#: rounds that answer a judgment. A texture or asset round is driven by its own pass, and+#: a repair round by a build error the judge never saw.+REFINE_KINDS: tuple[str, ...] = ("refine", "rewrite", "detail")+#: a diff longer than this is truncated, with the untruncated size kept on the row+MAX_DIFF_BYTES = 200_000+++class RefineTransition(BaseModel):+ """One refine round: its brief, its diff, its score delta."""++ battery: str+ run: str+ run_dir: str+ prompt_id: str+ prompt: str+ track: str+ language: str+ generator: str+ round: int+ round_kind: str+ issues: list[dict[str, Any]] = Field(description="the PREVIOUS round's judge issues")+ improvement_plan: list[dict[str, Any]]+ gate_errors: list[dict[str, Any]] = Field(description="the previous round's gate ERRORs, with fix hints")+ instructions: list[str] = Field(description="RoundRecord.instructions — the tasks compiled for this round")+ before_commit: str+ after_commit: str+ changed_files: list[str]+ diff: str+ diff_bytes: int = Field(description="the UNTRUNCATED size, so a capped row still records what it was")+ diff_truncated: bool+ score_before: float | None+ score_after: float | None+ score_delta: float | None+ outcome: str = Field(description="improved | unchanged | regressed | unscored")+ before_files: dict[str, str] | None = None+ after_files: dict[str, str] | None = None+++def _issues(rnd: RoundRecord) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:+ j = effective_judgment(rnd)+ if j is None:+ return [], []+ return ([i.model_dump(mode="json") for i in j.issues],+ [i.model_dump(mode="json") for i in j.improvement_plan])+++def _gate_errors(rnd: RoundRecord) -> list[dict[str, Any]]:+ return [{"gate": g.gate, "target": f.target, "message": f.message, "fix_hint": f.fix_hint}+ for g in rnd.gates for f in g.findings if f.severity == Severity.ERROR]+++def outcome_of(delta: float | None, threshold: float) -> str:+ """The label. One rule, shared with ``pairs.MIN_PREFERENCE_DELTA``, so a Δ that is a+ preference pair there is never "unchanged" here."""+ if delta is None:+ return "unscored"+ if delta >= threshold:+ return "improved"+ return "regressed" if delta <= -threshold else "unchanged"+++def _transition(ws: Workspace, rec: RunRecord, prev: RoundRecord, cur: RoundRecord, *, battery: str, run: str,+ threshold: float, max_diff_bytes: int, with_code: bool) -> RefineTransition:+ diff, total, truncated = diff_between(ws, prev.commit, cur.commit, max_bytes=max_diff_bytes)+ changed = changed_files_between(ws, prev.commit, cur.commit)+ before_s, after_s = effective_score(prev), effective_score(cur)+ delta = None if before_s is None or after_s is None else round(after_s - before_s, 4)+ issues, plan = _issues(prev)+ files_before = files_after = None+ if with_code:+ def _text(commit: str, keep: list[str] | None) -> dict[str, str]:+ out = {}+ for path, raw in read_tree_at(ws, commit).items():+ if keep is not None and path not in keep:+ continue+ try:+ out[path] = raw.decode("utf-8")+ except UnicodeDecodeError:+ continue # a binary file under src/: the diff already says it changed+ return out+ files_before, files_after = _text(prev.commit, None), _text(cur.commit, changed)+ return RefineTransition(+ battery=battery, run=run, run_dir=str(ws.root), prompt_id=rec.spec.id, prompt=rec.spec.prompt,+ track=rec.spec.track.value, language=rec.spec.language.value, generator=rec.spec.backends.generator,+ round=cur.index, round_kind=cur.kind, issues=issues, improvement_plan=plan,+ gate_errors=_gate_errors(prev), instructions=list(cur.instructions),+ before_commit=prev.commit, after_commit=cur.commit, changed_files=changed,+ diff=diff, diff_bytes=total, diff_truncated=truncated,+ score_before=before_s, score_after=after_s, score_delta=delta,+ outcome=outcome_of(delta, threshold), before_files=files_before, after_files=files_after)+++#: path segments that are run LAYOUT, not a battery name (compare_backends and ab_plan+#: bury their runs this deep)+_LAYOUT_DIRS = frozenset({"cells", "runs", "arms"})+++def _identity(root: Path, found: Any) -> tuple[str, str]:+ """``(battery, run)`` for a run, taken from where it PHYSICALLY lives.++ ``bench/out``'s batteries symlink each other's cells (54 of 239 runs are reachable+ twice), so labelling by the path a scan happened to reach first credits a run to a+ battery it never ran in."""+ try:+ rel = found.ws.root.resolve().relative_to(root.resolve()).as_posix()+ except ValueError:+ return found.run_id.battery, found.run_id.rel+ head, _, rest = rel.partition("/")+ if not rest or head in _LAYOUT_DIRS:+ return found.run_id.battery, rel+ return head, rest+++def transitions(runs_dir: Path | str, *, threshold: float = MIN_PREFERENCE_DELTA,+ max_diff_bytes: int = MAX_DIFF_BYTES, with_code: bool = False,+ drops: Counter[str] | None = None) -> Iterator[RefineTransition]:+ """Every exportable refine transition under ``runs_dir``; every one that is not is counted.++ A run reachable through more than one path (54 of 239 under ``bench/out``, where the+ batteries symlink each other's cells) is exported once, under the battery it physically+ lives in."""+ drops = drops if drops is not None else Counter()+ root = Path(runs_dir)+ seen: set[Path] = set()+ for found in iter_runs(root, on_error=lambda p, e: drops.update([f"unreadable_record: {type(e).__name__}"])):+ physical = found.ws.root.resolve()+ if physical in seen:+ drops["duplicate_run"] += 1+ continue+ seen.add(physical)+ rounds = {r.index: r for r in found.record.rounds}+ for cur in found.record.rounds:+ if cur.kind not in REFINE_KINDS:+ continue+ prev = rounds.get(cur.index - 1)+ if prev is None:+ drops["no_predecessor"] += 1+ elif not (prev.commit and cur.commit):+ drops["no_commit"] += 1+ elif not prev.build.ok:+ drops["predecessor_build_failed"] += 1+ elif effective_judgment(prev) is None:+ drops["predecessor_unjudged"] += 1+ else:+ try:+ battery, run = _identity(root, found)+ yield _transition(found.ws, found.record, prev, cur, battery=battery, run=run,+ threshold=threshold, max_diff_bytes=max_diff_bytes, with_code=with_code)+ except GitReadError as e:+ drops[f"git_read_failed: {str(e)[:60]}"] += 1+++def build_refine(runs_dir: Path | str, out_jsonl: Path | str, **kw: Any) -> tuple[int, Counter[str]]:+ """Write the transitions as JSONL; return ``(rows, drops)``."""+ drops: Counter[str] = Counter()+ out = Path(out_jsonl)+ out.parent.mkdir(parents=True, exist_ok=True)+ n = 0+ tmp = out.with_suffix(out.suffix + ".part")+ with tmp.open("w") as fh:+ for row in transitions(runs_dir, drops=drops, **kw):+ fh.write(row.model_dump_json(exclude_none=True) + "\n")+ n += 1+ tmp.replace(out)+ return n, drops+++__all__ = ["MAX_DIFF_BYTES", "REFINE_KINDS", "RefineTransition", "build_refine", "outcome_of", "transitions"]diff --git a/harness/tests/flywheel_cli/test_refine.py b/harness/tests/flywheel_cli/test_refine.pynew file mode 100644index 00000000..f13bca9c--- /dev/null+++ b/harness/tests/flywheel_cli/test_refine.py@@ -0,0 +1,105 @@+"""Refine transitions (codeverse/flywheel/refine.py)."""++from __future__ import annotations++import json+from pathlib import Path++import pytest++from codeverse.contracts.artifacts import GateFinding, GateReport, Severity+from codeverse.flywheel.pairs import MIN_PREFERENCE_DELTA+from codeverse.flywheel.record import load_record+from codeverse.flywheel.refine import build_refine, outcome_of, transitions+from tests.flywheel_cli.conftest import make_fake_run+++def _corpus(root: Path, *, scores=(0.30, 0.80), battery: str = "batt", slug: str = "chair"):+ ws, _ = make_fake_run(root / battery / "runs", slug, scores=scores)+ rec = load_record(ws)+ rec.rounds[0].gates = [GateReport(gate="connectivity", passed=False, findings=[GateFinding(+ gate="connectivity", severity=Severity.ERROR, target="Leg", message="part 'Leg' is floating",+ fix_hint="move Leg down 12 mm"), GateFinding(+ gate="connectivity", severity=Severity.WARN, target="Seat", message="thin", fix_hint="")])]+ ws.write_json(ws.record_path, rec)+ return ws+++def _rows(root: Path, **kw):+ from collections import Counter++ drops: Counter[str] = Counter()+ return list(transitions(root, drops=drops, **kw)), drops+++def test_a_refine_round_carries_its_brief_its_diff_and_its_delta(tmp_path):+ _corpus(tmp_path)+ rows, drops = _rows(tmp_path)+ assert len(rows) == 1 and not drops+ t = rows[0]+ assert t.round == 1 and t.round_kind == "refine" and t.battery == "batt"+ assert t.score_before == 0.30 and t.score_after == 0.80 and t.score_delta == 0.50 and t.outcome == "improved"+ assert t.instructions == ["thicken the legs"]+ assert [g["message"] for g in t.gate_errors] == ["part 'Leg' is floating"] # ERRORs only+ assert t.gate_errors[0]["fix_hint"] == "move Leg down 12 mm"+ assert any("thicken the legs" in i["instruction"] for i in t.improvement_plan)+ assert t.changed_files == ["src/model.py", "src/parts/leg.py"]+ assert "size=2.0" in t.diff and t.diff_bytes == len(t.diff) and not t.diff_truncated+ assert t.before_commit and t.after_commit and t.before_commit != t.after_commit+ assert t.before_files is None and t.after_files is None # --with-code is opt-in+++def test_with_code_inlines_the_answer_and_the_tree_it_started_from(tmp_path):+ _corpus(tmp_path)+ rows, _ = _rows(tmp_path, with_code=True)+ t = rows[0]+ assert set(t.after_files) == {"src/model.py", "src/parts/leg.py"} # what changed+ assert "size=2.0" in t.after_files["src/model.py"]+ assert "src/model.py" in t.before_files and "size=1.0" in t.before_files["src/model.py"]+ assert "src/parts/leg.py" not in t.before_files # it did not exist yet++[email protected]("delta,label", [(0.5, "improved"), (0.05, "improved"), (0.04, "unchanged"),+ (0.0, "unchanged"), (-0.05, "regressed"), (None, "unscored")])+def test_the_outcome_rule_is_the_one_pairs_uses(delta, label):+ assert outcome_of(delta, MIN_PREFERENCE_DELTA) == label+++def test_a_run_reached_through_a_symlink_is_exported_once_under_its_physical_battery(tmp_path):+ _corpus(tmp_path, battery="zz_physical")+ alias = tmp_path / "aa_alias" / "runs"+ alias.mkdir(parents=True)+ (alias / "chair").symlink_to(tmp_path / "zz_physical" / "runs" / "chair")+ rows, drops = _rows(tmp_path)+ assert len(rows) == 1 and drops == {"duplicate_run": 1}+ assert rows[0].battery == "zz_physical", "labelled by the battery it was reached through first"++[email protected]("break_it,reason", [+ ("kind", None), ("commit", "no_commit"), ("build", "predecessor_build_failed"),+ ("judgment", "predecessor_unjudged"),+])+def test_every_unexportable_round_is_counted_under_a_named_reason(tmp_path, break_it, reason):+ ws = _corpus(tmp_path)+ rec = load_record(ws)+ if break_it == "kind":+ rec.rounds[1].kind = "texture"+ elif break_it == "commit":+ rec.rounds[0].commit = ""+ elif break_it == "build":+ rec.rounds[0].build.ok = False+ else:+ rec.rounds[0].judgment = None+ ws.write_json(ws.record_path, rec)+ rows, drops = _rows(tmp_path)+ assert rows == []+ assert drops == ({} if reason is None else {reason: 1}), f"{break_it}: {dict(drops)}"+++def test_build_refine_writes_jsonl_atomically(tmp_path):+ _corpus(tmp_path)+ out = tmp_path / "nested" / "refine.jsonl"+ n, drops = build_refine(tmp_path, out)+ assert n == 1 and not drops and not list(out.parent.glob("*.part"))+ row = json.loads(out.read_text().splitlines()[0])+ assert row["outcome"] == "improved" and "before_files" not in row # exclude_none keeps rows smalldiff --git a/harness/tests/flywheel_cli/test_refine_pairs.py b/harness/tests/flywheel_cli/test_refine_pairs.pydeleted file mode 100644index b776d539..00000000--- a/harness/tests/flywheel_cli/test_refine_pairs.py+++ /dev/null@@ -1,548 +0,0 @@-"""``bench/refine_pairs.py``: refine transitions exported as (brief -> diff -> delta) rows.--Every case builds a corpus in the real on-disk layout — ``<root>/<battery>/runs/<slug>``,-a git repo with two commits and two round records — because the exporter's whole job is-to survive that layout: the recorded round shas rather than ``HEAD`` (a finished run ends-on a "restore best round rNN" commit), one row per resolved run path labelled with the-battery it physically lives in (``bench/out``'s cell symlinks point ACROSS batteries), the-brief the agent sessions were really handed rather than the pre-grouping task list, and a-named reason for every run and transition it cannot export.-"""--from __future__ import annotations--import json-from collections.abc import Callable-from pathlib import Path--import pytest--from bench.refine_pairs import (- DROP_REASONS,- Report,- changes_block,- export,- iter_rows,- main,- outcome_label,- parse_instruction,- scan_roots,- transition_row,-)-from codeverse.contracts.artifacts import BuildResult, GateFinding, GateReport, Judgment, Severity-from codeverse.contracts.run import RoundRecord, RunRecord-from codeverse.flywheel.record import load_record, run_id_for-from codeverse.judges.rubrics import is_degraded-from codeverse.workspace import Workspace-from tests.flywheel_cli.conftest import make_fake_run---def _corpus(root: Path, *, battery: str = "batt", slug: str = "chair") -> Workspace:- """One battery holding one run: two judged rounds, ``src/`` committed twice."""- ws, _ = make_fake_run(root / battery / "runs", slug)- return ws---def _degraded() -> Judgment:- """What ``judges.rubrics.degraded_judgment`` writes when the judge call fails: a- 0.0 verdict flagged both ways. ``is_degraded`` below keeps this honest."""- return Judgment(rubric="static_object_v1", judge_backend="gemini:gemini-3.1-pro-preview",- scores={"fidelity": 0.0}, overall=0.0, passed=False,- summary="judge_error: 503 from the judge", n_samples=0,- raw=json.dumps({"status": "degraded", "error": "503"}))---def _prompt(ws: Workspace, session: str, round_index: int, lines: list[str]) -> None:- """One session's prompt file, in the shape ``agents.cli_common.begin_session`` writes- and ``prompts/tracks/refine_object.j2`` renders."""- d = ws.root / "trajectories" / f"{session}_r{round_index:02d}"- d.mkdir(parents=True, exist_ok=True)- numbered = "\n".join(f"{i}. {ln}" for i, ln in enumerate(lines, 1))- (d / "prompt.md").write_text(- f"<!-- system -->\nYou are a Blender author\n\n<!-- prompt -->\n"- f"# Refine `Chair` — round {round_index}\n\nThe object already builds.\n\n"- f"## Changes to make (in priority order; `[gate/...]` items are hard failures — fix them first)\n\n"- f"{numbered}\n\nTargets: Leg\n\n## What the judge saw\n1. not an instruction\n")---def _patch(ws: Workspace, edit: Callable[[RunRecord], None]) -> RunRecord:- rec = load_record(ws)- edit(rec)- ws.write_json(ws.record_path, rec)- return rec---def _rows(root: Path, **kw) -> tuple[list[dict], Report]:- report = Report()- return list(iter_rows(root, report, **kw)), report---# --------------------------------------------------------------------------- pure functions-def test_parse_instruction_refine_task_line():- tasks = parse_instruction("[gate/gate:connectivity] Leg_0: weld the leg to the seat (files: src/model.py, src/parts/leg.py)")- assert tasks == [{"source": "gate", "kind": "gate:connectivity", "target": "Leg_0",- "instruction": "weld the leg to the seat",- "files": ["src/model.py", "src/parts/leg.py"]}]---def test_parse_instruction_grouped_bullets():- """``compact_instructions`` folds ``Leg_0``/``Leg_1`` into one ``Leg:`` line; each- bullet is its own task and may name the instance it came from."""- tasks = parse_instruction(- "Leg: (a) [gate/gate:contract] too thin; (b) [Leg_1] [judge/geometry] add a fillet (files: src/model.py)")- assert [(t["source"], t["kind"], t["target"], t["instruction"]) for t in tasks] == [- ("gate", "gate:contract", "Leg", "too thin"),- ("judge", "geometry", "Leg_1", "add a fillet"),- ]- assert all(t["files"] == ["src/model.py"] for t in tasks)---def test_parse_instruction_bare_sentence_keeps_its_colon():- """Detail rounds send prose. A sentence's own colon is not a target: only a single- bare token before ``: `` is, or the whole instruction would be cut in half."""- prose = "Add the panel lines where the shells meet (lids, trays): 1-2 mm wide insets."- assert parse_instruction(prose) == [- {"source": "", "kind": "", "target": "", "instruction": prose, "files": []}]- assert parse_instruction("window_frame: Slope the exterior sill.") == [- {"source": "", "kind": "", "target": "window_frame", "instruction": "Slope the exterior sill.", "files": []}]---def test_outcome_label():- assert outcome_label(0.2) == "improved"- assert outcome_label(-0.2) == "regressed"- assert outcome_label(0.01) == "unchanged"- assert outcome_label(0.2, threshold=0.5) == "unchanged"- assert outcome_label(None) == "unscored"---def test_scan_roots_picks_batteries_not_the_parent(tmp_path):- root = tmp_path / "out"- _corpus(root, battery="batt_a")- _corpus(root, battery="batt_b")- (root / "merged_report").mkdir() # neither a run nor a battery: reported, not silent- assert scan_roots(root) == ([root / "batt_a", root / "batt_b"], [root / "merged_report"])- assert scan_roots(root / "batt_a") == ([root / "batt_a"], [])---def test_changes_block_reads_the_numbered_list_and_stops_at_the_next_section():- """The refine templates number ``compact_instructions``' lines and follow them with- ``Targets:``; a numbered line in a LATER section is not an instruction."""- prompt = ("# Refine `Chair` — round 1\n\n"- "## Changes to make (in priority order)\n\n"- "1. Leg: (a) [judge/geometry] thicken\n\n2. [judge/material] Seat: oak,\n quarter-sawn\n\n"- "Targets: Leg, Seat\n\n## What the judge saw\n1. legs too thin\n")- assert changes_block(prompt) == ["Leg: (a) [judge/geometry] thicken",- "[judge/material] Seat: oak, quarter-sawn"]- assert changes_block("# Rebuild `Chair`\n\n1. fix the syntax error\n") == []---# --------------------------------------------------------------------------- rows-def test_row_carries_brief_diff_and_delta(tmp_path):- root = tmp_path / "out"- ws = _corpus(root)- rows, report = _rows(root)- (row,) = rows- assert (row["battery"], row["run"], row["round"], row["round_kind"]) == ("batt", "chair", 1, "refine")- assert row["prompt_id"] == "chair" and row["prompt"] == "a wooden dining chair"- assert (row["track"], row["language"]) == ("static_object", "blender")- # the brief is the PREVIOUS round's verdict, the change is this round's- assert [i["target"] for i in row["issues"]] == ["Leg"]- assert [i["instruction"] for i in row["improvement_plan"]] == ["thicken the legs", "add a back rail"]- assert row["refine_tasks"] == ["thicken the legs"]- assert row["before"]["files"] == ["src/model.py"]- assert row["after"]["files"] == ["src/model.py", "src/parts/leg.py"]- assert row["changed_files"] == ["src/model.py", "src/parts/leg.py"]- assert "--- a/src/model.py" in row["diff"] and "+++ b/src/parts/leg.py" in row["diff"]- assert row["diff_bytes"] == len(row["diff"].encode()) and not row["diff_truncated"]- assert row["diff_added"] > 0 and row["diff_removed"] > 0- assert (row["score_before"], row["score_after"], row["score_delta"]) == (0.55, 0.8, 0.25)- assert row["before"]["build_ok"] is True and row["after"]["build_ok"] is True- assert row["outcome"] == "improved"- assert report.rows == 1 and not report.drops- assert report.kinds == {"baseline": 1, "refine": 1}- assert ws.root.name == "chair"---def test_outcome_threshold_is_recorded_on_the_row(tmp_path):- root = tmp_path / "out"- _corpus(root)- (row,), _ = _rows(root, threshold=0.5)- assert row["outcome"] == "unchanged" and row["outcome_threshold"] == 0.5---def test_gate_error_findings_carry_fix_hints(tmp_path):- """The refine brief is half judge, half gate: ``build_refine_instructions`` copies a- finding's ``fix_hint`` verbatim, and only ERROR findings become tasks."""- root = tmp_path / "out"- ws = _corpus(root)- _patch(ws, lambda rec: rec.rounds[0].gates.append(GateReport(gate="connectivity", passed=False, findings=[- GateFinding(gate="connectivity", severity=Severity.ERROR, target="Leg_0",- message="Leg_0 is not connected", fix_hint="move Leg_0 up by 3 mm"),- GateFinding(gate="connectivity", severity=Severity.WARN, target="Seat", message="1.2 mm overlap")])))- (row,), _ = _rows(root)- assert row["gate_findings"] == [{"gate": "connectivity", "target": "Leg_0",- "message": "Leg_0 is not connected",- "fix_hint": "move Leg_0 up by 3 mm", "data": {}}]- assert row["before"]["gate_errors"] == 1---def test_diff_is_capped_and_says_so(tmp_path):- """The cap must actually cut the text: a row that carries the WHOLE diff and merely- says "truncated" is the failure the cap exists to prevent."""- root = tmp_path / "out"- _corpus(root)- (row,), _ = _rows(root, max_diff_bytes=120)- assert row["diff_truncated"] and row["diff_max_bytes"] == 120- assert row["diff_bytes"] > 120 and "[truncated" in row["diff"]- head, marker, _rest = row["diff"].partition("\n... [truncated ")- assert marker and len(head.encode()) == 120, "the kept head is exactly the cap"- assert f"{row['diff_bytes'] - 120} bytes" in row["diff"], "and it names what it dropped"- (whole,), _ = _rows(root) # the same diff uncapped: same prefix, more of it- assert whole["diff"].startswith(head) and len(whole["diff"]) > len(head)- assert not whole["diff_truncated"] and whole["diff_bytes"] == row["diff_bytes"]---def test_diff_lines_count_content_not_file_headers(tmp_path):- """``+++``/``---`` are one pair per changed FILE. Counting them inflates every row's- churn by the number of files the round touched — the label a preference model reads."""- root = tmp_path / "out"- _corpus(root)- (row,), _ = _rows(root)- # round 1 rewrites 3 of the 4 lines of src/model.py and adds the 1-line src/parts/leg.py- assert (row["diff_added"], row["diff_removed"]) == (4, 3)- naive_added = sum(1 for ln in row["diff"].splitlines() if ln.startswith("+"))- naive_removed = sum(1 for ln in row["diff"].splitlines() if ln.startswith("-"))- assert (naive_added, naive_removed) == (6, 5), "two file header pairs sit in that diff"---def test_changed_files_are_the_ones_this_round_touched(tmp_path):- """Per-file attribution is the round's OWN diff, not the file list at its commit: a- round that edits one of two files must not be recorded as having rewritten both."""- root = tmp_path / "out"- ws = _corpus(root)- (ws.src / "parts" / "leg.py").write_text("LEG = 0.06\n")- commit = ws.commit("round 2")- _patch(ws, lambda rec: rec.rounds.append(RoundRecord(- index=2, kind="refine", commit=commit, instructions=["thicken the legs again"],- build=BuildResult(ok=True, language="blender"), judgment=rec.rounds[1].judgment)))- rows, _ = _rows(root)- second = rows[-1]- assert second["round"] == 2- assert second["changed_files"] == ["src/parts/leg.py"]- assert second["after"]["files"] == ["src/model.py", "src/parts/leg.py"], "both exist, one changed"- assert "src/model.py" not in second["diff"]---def test_only_refine_kind_rounds_are_transitions(tmp_path):- """A texture pass edits ``src/`` and has everything a row needs — a judged- predecessor, a commit, instructions. It is still not a refine transition: nothing- derived it from a verdict, so it is a round SEEN and a row not written, with no drop- to explain (the drop reasons are for transitions that should have been exportable)."""- root = tmp_path / "out"- ws = _corpus(root)- (ws.src / "model.py").write_text("# textured\nimport bpy\n")- commit = ws.commit("texture pass")- _patch(ws, lambda rec: rec.rounds.append(RoundRecord(- index=2, kind="texture", commit=commit, instructions=["bake a quarter-sawn oak albedo"],- build=BuildResult(ok=True, language="blender"))))- rows, report = _rows(root)- assert [r["round_kind"] for r in rows] == ["refine"]- assert report.kinds == {"baseline": 1, "refine": 1, "texture": 1}- assert report.transitions == 1 and not report.drops---def test_a_degraded_verdict_is_unscored_not_a_zero(tmp_path):- """A judge outage writes a 0.0 ``judge_error:`` Judgment. Read as a score it turns a- fine round into the corpus's worst regression (0.0 - 0.55), and the sign of that row- is exactly what the pairs are for."""- assert is_degraded(_degraded()), "the fixture must stay the real degraded shape"- root = tmp_path / "out"- ws = _corpus(root)- _patch(ws, lambda rec: setattr(rec.rounds[1], "judgment", _degraded()))- (row,), report = _rows(root)- assert row["score_before"] == 0.55- assert row["score_after"] is None and row["score_delta"] is None- assert row["outcome"] == "unscored"- assert row["after"]["score"] is None and row["after"]["passed"] is None- assert not report.drops---def test_a_degraded_previous_verdict_drops_the_transition(tmp_path):- """The brief is the PREVIOUS round's verdict. A judge outage wrote no verdict, so- there is no brief — and a 0.0 one would make every issue list empty on purpose."""- root = tmp_path / "out"- ws = _corpus(root)- _patch(ws, lambda rec: setattr(rec.rounds[0], "judgment", _degraded()))- rows, report = _rows(root)- assert rows == [] and dict(report.drops) == {"missing_prev_judgment": 1}---# --------------------------------------------------------------------------- instructions-def test_instruction_lines_are_the_ones_the_sessions_were_handed(tmp_path):- """``RoundRecord.instructions`` is the COMPILED task list; each file-disjoint group's- agent session is handed its own ``compact_instructions`` output, folded by target and- capped. The two differ for 167 of the corpus's 205 transitions, so the row carries- both and says which one ``instruction_tasks`` was parsed from."""- root = tmp_path / "out"- ws = _corpus(root)- compiled = ["[judge/geometry] Leg_0: thicken it (files: src/parts/leg.py)",- "[judge/geometry] Leg_1: thicken it (files: src/parts/leg.py)",- "[judge/material] Seat: quarter-sawn oak (files: src/model.py)"]- _patch(ws, lambda rec: setattr(rec.rounds[1], "instructions", compiled))- leg = "Leg: (a) [judge/geometry] thicken it; (b) [Leg_1] [judge/geometry] thicken it (files: src/parts/leg.py)"- seat = "[judge/material] Seat: quarter-sawn oak (files: src/model.py)"- _prompt(ws, "refine_leg", 1, [leg])- _prompt(ws, "refine_seat", 1, [seat])-- (row,), _ = _rows(root)- assert row["instruction_source"] == "prompt"- assert [s["session"] for s in row["sent_instructions"]] == ["refine_leg", "refine_seat"]- assert row["instruction_lines"] == [leg, seat]- assert row["refine_tasks"] == compiled- assert row["instruction_lines"] != row["refine_tasks"]- assert [t["target"] for t in row["instruction_tasks"]] == ["Leg", "Leg_1", "Seat"]---def test_a_retry_session_is_not_a_second_brief(tmp_path):- """``<label>.a2`` is the SAME brief re-sent after a silent bail (``_session_label``);- counting it twice would double every task in the row."""- root = tmp_path / "out"- ws = _corpus(root)- line = "Leg: (a) [judge/geometry] thicken it (files: src/parts/leg.py)"- _prompt(ws, "refine", 1, [line])- _prompt(ws, "refine.a2", 1, [line])- (row,), _ = _rows(root)- assert row["instruction_lines"] == [line]- assert [s["session"] for s in row["sent_instructions"]] == ["refine"]---def test_instruction_lines_fall_back_to_the_record_when_no_prompt_survives(tmp_path):- """10 of the corpus's 205 transitions have no prompt file left. The compiled task- list is then all there is, and the row says so rather than shipping an empty brief."""- root = tmp_path / "out"- _corpus(root)- (row,), _ = _rows(root)- assert row["sent_instructions"] == [] and row["instruction_source"] == "record"- assert row["instruction_lines"] == row["refine_tasks"] == ["thicken the legs"]---# --------------------------------------------------------------------------- identity-def test_a_cross_battery_alias_is_labelled_with_the_physical_battery(tmp_path):- """``bench/out``'s aliases point ACROSS batteries (``compare_v4_calm/cells/*`` is a- symlink into ``compare_v4`` and ``compare_v4_harness_calm``) and the alias sorts- FIRST, so labelling by the root that reached a run first credits it to a battery it- never ran in. The label is the battery the run physically lives in; the alias is- reported, not lost."""- root = tmp_path / "out"- ws = _corpus(root, battery="zz_physical", slug="chair")- alias = root / "aa_alias" / "runs"- alias.mkdir(parents=True)- (alias / "chair").symlink_to(ws.root, target_is_directory=True)-- rows, report = _rows(root)- assert report.roots == [str(root / "aa_alias"), str(root / "zz_physical")]- assert [r["battery"] for r in rows] == ["zz_physical"]- assert rows[0]["run_dir"] == str(ws.root)- assert dict(report.drops) == {"duplicate_run": 1}- assert dict(report.aliases) == {"aa_alias -> zz_physical": 1}---def test_a_single_run_directory_exports_with_the_identity_a_full_scan_gives_it(tmp_path):- """The documented one-run input. ``iter_runs`` searches strictly BELOW its argument,- so the run itself was invisible and the export reported zero rows as a success."""- root = tmp_path / "out"- ws = _corpus(root, battery="batt", slug="chair")- whole, _ = _rows(root)- alone, report = _rows(ws.root)- assert len(alone) == 1 and report.runs == 1 and not report.drops- assert (alone[0]["battery"], alone[0]["run"]) == ("batt", "chair")- assert alone[0] == whole[0]---def test_a_single_nested_run_keeps_the_whole_battery_path(tmp_path):- """The same for the deepest layout, ``<battery>/arms/<arm>/cells/<cell>/<slug>``,- where the battery is the OUTERMOST marker on the path: stopping at the first one up- (``<arm>``, which owns a ``cells/``) labels the run with its arm instead."""- root = tmp_path / "out"- ws, _ = make_fake_run(root / "batt" / "arms" / "control" / "cells" / "chair_cell", "harness_arm")- whole, _ = _rows(root)- (row,), report = _rows(ws.root)- assert (row["battery"], row["arm"], row["cell"]) == ("batt", "control", "chair_cell")- assert row["run"] == "control__chair_cell__harness_arm"- assert report.roots == [str(ws.root)]- assert row == whole[0]---def test_main_accepts_one_run_directory(tmp_path, capsys):- root = tmp_path / "out"- ws = _corpus(root)- assert main([str(ws.root), "--summary"]) == 0- assert "1 row(s) from 1 run(s) in 1 root(s)" in capsys.readouterr().out---def test_diff_uses_the_recorded_shas_not_head(tmp_path):- """152 of the 239 recorded runs end on a "restore best round rNN" commit and only 54- have HEAD at their last round, so the recorded sha is the only usable handle."""- root = tmp_path / "out"- ws = _corpus(root)- rec = load_record(ws)- (ws.src / "model.py").write_text("# restored baseline\n")- ws.commit("restore best round r00")- (row,), _ = _rows(root)- assert row["after"]["commit"] == rec.rounds[1].commit- assert "restored baseline" not in row["diff"]---def test_nested_battery_layout_keeps_battery_cell_and_arm(tmp_path):- root = tmp_path / "out"- make_fake_run(root / "cmp" / "cells" / "chair_cell", "harness_arm")- (row,), report = _rows(root)- assert (row["battery"], row["cell"], row["arm"]) == ("cmp", "chair_cell", "harness_arm")- assert row["run"] == "chair_cell__harness_arm"- assert report.roots == [str(root / "cmp")]---def test_transition_row_is_callable_on_one_run(tmp_path):- """The row builder is usable without the scanner (one run, two rounds in hand)."""- ws, rec = make_fake_run(tmp_path / "runs")- run_id = run_id_for(tmp_path / "runs", ws.root)- row = transition_row(ws, rec, run_id, rec.rounds[0], rec.rounds[1])- assert row["run"] == ws.root.name and row["score_delta"] == 0.25---# --------------------------------------------------------------------------- drops-def _drop_not_a_battery(root: Path) -> None:- _corpus(root)- (root / "merged_report").mkdir() # bench/out holds three of these---def _drop_empty_battery(root: Path) -> None:- (root / "batt" / "cells").mkdir(parents=True)---def _drop_unreadable_record(root: Path) -> None:- _corpus(root)- broken = root / "batt" / "runs" / "broken"- broken.mkdir()- (broken / "record.json").write_text("{not json")---def _drop_duplicate_run(root: Path) -> None:- ws = _corpus(root)- (root / "batt" / "runs" / "chair_link").symlink_to(ws.root, target_is_directory=True)---def _drop_missing_predecessor(root: Path) -> None:- _patch(_corpus(root), lambda rec: setattr(rec.rounds[1], "index", 5))---def _drop_missing_commit(root: Path) -> None:- _patch(_corpus(root), lambda rec: setattr(rec.rounds[1], "commit", ""))---def _drop_missing_prev_judgment(root: Path) -> None:- _patch(_corpus(root), lambda rec: setattr(rec.rounds[0], "judgment", None))---def _drop_no_instructions(root: Path) -> None:- _patch(_corpus(root), lambda rec: setattr(rec.rounds[1], "instructions", []))---def _drop_git_read_failed(root: Path) -> None:- _patch(_corpus(root), lambda rec: setattr(rec.rounds[1], "commit", "0" * 40))---def _drop_empty_diff(root: Path) -> None:- _patch(_corpus(root), lambda rec: setattr(rec.rounds[1], "commit", rec.rounds[0].commit))---#: one corpus per reason; the parametrisation below fails loudly if a reason has none-DROP_CORPUS: dict[str, Callable[[Path], None]] = {- "not_a_battery": _drop_not_a_battery,- "empty_battery": _drop_empty_battery,- "unreadable_record": _drop_unreadable_record,- "duplicate_run": _drop_duplicate_run,- "missing_predecessor": _drop_missing_predecessor,- "missing_commit": _drop_missing_commit,- "missing_prev_judgment": _drop_missing_prev_judgment,- "no_instructions": _drop_no_instructions,- "git_read_failed": _drop_git_read_failed,- "empty_diff": _drop_empty_diff,-}-#: the scan-level drops leave the run itself exportable-KEEPS_ROW = {"not_a_battery", "unreadable_record", "duplicate_run"}--[email protected]("reason", DROP_REASONS)-def test_every_drop_reason_is_counted(tmp_path, reason):- root = tmp_path / "out"- DROP_CORPUS[reason](root)- rows, report = _rows(root)- assert dict(report.drops) == {reason: 1}- assert len(rows) == report.rows == (1 if reason in KEEPS_ROW else 0)---def test_a_skipped_child_is_named_in_the_report(tmp_path, capsys):- """Counting it is half the job: "every drop is counted" has to say WHICH directory- was passed over, or a mistyped battery name reads as an empty corpus."""- from bench.refine_pairs import scan_report-- root = tmp_path / "out"- _drop_not_a_battery(root)- _, report = _rows(root)- assert report.skipped == [str(root / "merged_report")]- assert str(root / "merged_report") in scan_report(report)- assert "not_a_battery" in scan_report(report)---def test_drop_corpus_covers_every_declared_reason():- assert set(DROP_CORPUS) == set(DROP_REASONS)---# --------------------------------------------------------------------------- cli-def test_main_writes_jsonl_and_prints_the_summary(tmp_path, capsys):- root = tmp_path / "out"- _corpus(root, battery="batt_a", slug="chair")- _corpus(root, battery="batt_b", slug="stool")- out = tmp_path / "pairs" / "refine.jsonl"-- assert main([str(root), "--out", str(out), "--summary"]) == 0- printed = capsys.readouterr().out- rows = [json.loads(line) for line in out.read_text().splitlines()]-- assert len(rows) == 2 and {r["battery"] for r in rows} == {"batt_a", "batt_b"}- assert "2 row(s) from 2 run(s) in 2 root(s)" in printed- assert "dropped: none" in printed- assert "rows per battery" in printed and "batt_a" in printed- assert "rows per track" in printed and "static_object" in printed- assert "rows per outcome" in printed and "improved" in printed- assert "diff size: median" in printed---def test_main_reports_drops_without_writing(tmp_path, capsys):- root = tmp_path / "out"- _drop_no_instructions(root)- assert main([str(root), "--summary"]) == 1- printed = capsys.readouterr().out- assert "0 row(s) from 1 run(s)" in printed- assert "no_instructions" in printed- assert "no rows to summarise" in printed---def test_main_needs_somewhere_to_put_the_result(tmp_path):- root = tmp_path / "out"- _corpus(root)- with pytest.raises(SystemExit):- main([str(root)])- with pytest.raises(SystemExit):- main([str(tmp_path / "nope"), "--summary"])---def test_export_replaces_the_target_atomically(tmp_path):- root = tmp_path / "out"- _corpus(root)- out = tmp_path / "refine.jsonl"- out.write_text("stale\n")- report = export(root, out)- assert report.rows == 1 and json.loads(out.read_text())["run"] == "chair"- assert not list(tmp_path.glob("*.tmp"))diff --git a/harness/tests/flywheel_cli/test_refine_sft.py b/harness/tests/flywheel_cli/test_refine_sft.pydeleted file mode 100644index 79f3f8a2..00000000--- a/harness/tests/flywheel_cli/test_refine_sft.py+++ /dev/null@@ -1,104 +0,0 @@-"""Refine rounds as SFT samples (bench/refine_sft.py)."""--from __future__ import annotations--import json-from pathlib import Path--import pytest--from bench.refine_sft import DEFAULT_OUTCOMES, build, envelope, main, sample, user_turn-from codeverse.contracts.artifacts import GateFinding, GateReport, Severity-from codeverse.flywheel.record import load_record-from codeverse.tracks.generation import parse_multifile-from tests.flywheel_cli.conftest import make_fake_run---def _corpus(root: Path, *, scores: tuple[float, float] = (0.30, 0.80)) -> Path:- ws, _ = make_fake_run(root / "batt" / "runs", "chair", scores=scores)- rec = load_record(ws)- rec.rounds[0].gates = [GateReport(gate="connectivity", passed=False, findings=[GateFinding(- gate="connectivity", severity=Severity.ERROR, target="Leg",- message="part 'Leg' is floating", fix_hint="move Leg down 12 mm so it meets Seat")])]- ws.write_json(ws.record_path, rec)- return root---def test_an_improved_round_becomes_a_three_message_sample(tmp_path):- rows, report, drops = build(_corpus(tmp_path))- assert len(rows) == 1 and report.runs == 1 and not drops- s = rows[0]- assert [m["role"] for m in s["messages"]] == ["system", "user", "assistant"]- assert s["score_before"] == 0.30 and s["score_after"] == 0.80- assert s["changed_files"] == ["src/model.py", "src/parts/leg.py"]- assert s["id"].endswith("/r01") and s["track"] == "static_object" and s["language"] == "blender"- # the answer is the file the round produced, and it parses back through the envelope- files = parse_multifile(s["messages"][2]["content"])- assert set(files) == {"src/model.py", "src/parts/leg.py"} and "size=2.0" in files["src/model.py"]- assert "bpy" in s["messages"][0]["content"].lower()---def test_the_brief_carries_what_the_round_was_told_and_the_code_it_started_from(tmp_path):- rows, _, _ = build(_corpus(tmp_path))- user = rows[0]["messages"][1]["content"]- assert "a wooden dining chair" in user- assert "part 'Leg' is floating" in user and "move Leg down 12 mm" in user # gate finding + fix hint- assert "thicken the legs" in user # the judge's plan / instruction- assert "=== FILE: src/model.py ===" in user and "size=1.0" in user # the BEFORE state- assert "size=2.0" not in user # never the answer- assert "in full" in user.rsplit("\n", 1)[-1]---def test_only_improved_rounds_are_exported_and_the_rest_are_counted(tmp_path):- rows, _, drops = build(_corpus(tmp_path, scores=(0.80, 0.80))) # unchanged- assert rows == [] and drops == {"outcome:unchanged": 1}- rows, _, drops = build(_corpus(tmp_path / "b", scores=(0.80, 0.30)))- assert rows == [] and drops == {"outcome:regressed": 1}- rows, _, _ = build(_corpus(tmp_path / "c", scores=(0.80, 0.30)), outcomes=("regressed",))- assert len(rows) == 1 and DEFAULT_OUTCOMES == ("improved",)---def test_an_answer_over_the_cap_is_dropped_not_truncated(tmp_path):- rows, _, drops = build(_corpus(tmp_path), max_answer_chars=50)- assert rows == [] and drops == {"answer_too_long": 1}---def test_a_row_whose_commit_is_gone_is_counted_not_raised(tmp_path):- from bench.refine_pairs import Report, iter_rows-- row = next(iter(iter_rows(_corpus(tmp_path), Report())))- row["after"]["commit"] = "0" * 40- assert sample(row) is None and row["_drop"].startswith("git_read_failed")---def test_envelope_and_user_turn_are_pure(tmp_path):- assert envelope({"b.py": "B", "a.py": "A"}) == (- "=== FILE: a.py ===\nA\n=== END FILE ===\n=== FILE: b.py ===\nB\n=== END FILE ===")- assert parse_multifile(envelope({"a.py": "A"})) == {"a.py": "A"}- row = {"track": "static_object", "language": "blender", "prompt": "a stool",- "gate_findings": [], "issues": [], "instruction_lines": []}- text = user_turn(row, {"src/x.py": "code"})- assert "## Gate errors" not in text and "## What the judge saw" not in text- assert "## Current files" in text and "a stool" in text---def test_main_writes_jsonl_and_reports(tmp_path, capsys):- out = tmp_path / "sft.jsonl"- assert main([str(_corpus(tmp_path)), "--out", str(out), "--summary"]) == 0- printed = capsys.readouterr().out- assert "1 sample(s) from 1 run(s)" in printed and "answer chars: median" in printed- assert "by language: blender 1" in printed- rows = [json.loads(x) for x in out.read_text().splitlines()]- assert len(rows) == 1 and rows[0]["schema_version"] == 1--[email protected]("lang,entry", [("urdf_blender", "src/model.py")])-def test_an_articulated_run_keeps_only_its_changed_src_files(tmp_path, lang, entry):- from codeverse.contracts.common import Language-- ws, _ = make_fake_run(tmp_path / "batt" / "runs", "cab", language=Language(lang))- rows, _, _ = build(tmp_path)- assert len(rows) == 1- assert rows[0]["changed_files"] == [entry, "src/parts/leg.py"] and rows[0]["language"] == lang- assert set(parse_multifile(rows[0]["messages"][2]["content"])) == {entry, "src/parts/leg.py"}- assert "robot.urdf" in rows[0]["messages"][1]["content"] # the brief still shows the whole src/