Skip to content

Official Codebase of 3DCodeVerse

Python

flywheel: the refine exporter folds into the package; the format conversion moves to toolkits

ziyaof67a02d89078
diff --git a/harness/bench/refine_pairs.py b/harness/bench/refine_pairs.py
deleted file mode 100644
index 278a6120..00000000
--- a/harness/bench/refine_pairs.py
+++ /dev/null
@@ -1,585 +0,0 @@
-"""Refine rounds as training pairs: what the round was TOLD, what it CHANGED, what it SCORED.
-
-``codeverse/flywheel/pairs.py`` exports whole code trees per round and never looks at
-what asked for the change, so a refine round arrives as two 30 KB files with no cause.
-This exports the transition instead — one row per refine round::
-
- (previous round's judge issues + improvement plan + gate ERROR findings
- + the instruction lines the agent sessions were handed) -> src/ diff -> score delta
-
-Both sides come from the round's OWN recorded commit, never from ``HEAD``: only 54 of the
-239 recorded runs have HEAD at their last round, 152 end on a "restore best round rNN"
-commit, and looking a commit up by message is ambiguous (``r01 refine`` also matches
-``r01 refine: generated``).
-
- python bench/refine_pairs.py bench/out --out refine_pairs.jsonl --summary
- python bench/refine_pairs.py bench/out/aa_articulated --summary
- python bench/refine_pairs.py bench/out/aa_articulated/arms/control/cells/art_med_tool_chest/\\
- harness_gemini-cli_gemini-3.7-flash/run --summary
-
-``root`` is a directory OF batteries (``bench/out``), one battery, a runs directory or a
-single run — every layout the flywheel scanner knows, and a single run reports the same
-identity a whole-tree scan gives it. A run reachable through more than one path (54 of
-the 239) is exported once, labelled with the battery it PHYSICALLY lives in: ``bench/out``'s
-aliases are cross-battery symlinks (``compare_v4_calm/cells/*`` points into ``compare_v4``
-and ``compare_v4_harness_calm``), so labelling by the first path scanned credited 54 of the
-205 rows to a battery they never ran in. Every run and round that cannot be exported is
-counted under a named reason; the scan report is printed whether or not rows were written.
-
-Two instruction fields, because they are not the same list. ``refine_tasks`` is
-``RoundRecord.instructions`` — every task ``build_refine_instructions`` compiled, before
-grouping. ``instruction_lines`` is what the agents actually read: the per-group output of
-``orchestrator.compact_instructions``, folded by target and capped at
-``policy.max_instructions_per_task``, recovered from the round's own prompt files. They
-differ for 167 of the corpus's 205 transitions.
-"""
-
-from __future__ import annotations
-
-import argparse
-import itertools
-import json
-import re
-import statistics
-import sys
-from collections import Counter
-from collections.abc import Iterator, Sequence
-from dataclasses import dataclass, field
-from pathlib import Path
-from typing import Any
-
-sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
-
-from codeverse.contracts.run import RUN_PATH_NOISE, RoundRecord, RunId, RunRecord # noqa: E402
-from codeverse.flywheel import _git # noqa: E402
-from codeverse.flywheel.quality import prompt_hash # noqa: E402
-from codeverse.flywheel.record import ( # noqa: E402
- BATTERY_MARKERS,
- RUN_SEARCH_DEPTH,
- FoundRun,
- RecordError,
- effective_judgment,
- is_run_dir,
- iter_runs,
- load_record,
- run_id_for,
-)
-from codeverse.workspace import Workspace # noqa: E402
-
-SCHEMA_VERSION = 1
-
-#: round kinds that are a refine transition — a round handed instructions derived from
-#: the previous round's verdict, and the only three the recorded corpus holds.
-#: ``baseline`` has no predecessor, and ``texture`` / ``asset:*`` rounds are driven by
-#: the texture pass rather than by a judgment.
-REFINE_KINDS: tuple[str, ...] = ("refine", "rewrite", "detail")
-
-#: cap on the inlined diff. The corpus's largest transition diff is ~62 KB, so nothing
-#: recorded today is truncated; the cap only stops a pathological row from eating a
-#: dataset, and every row records it next to the pre-cap size.
-MAX_DIFF_BYTES = 200_000
-
-#: |Δscore| below which a round counts as ``unchanged``. Same 0.05 the flywheel's
-#: preference pairs already use as "a real difference" (``flywheel/pairs.build_pairs``).
-OUTCOME_THRESHOLD = 0.05
-
-DROP_REASONS: tuple[str, ...] = (
- "not_a_battery",
- "empty_battery",
- "unreadable_record",
- "duplicate_run",
- "missing_predecessor",
- "missing_commit",
- "missing_prev_judgment",
- "no_instructions",
- "git_read_failed",
- "empty_diff",
-)
-
-
-# --------------------------------------------------------------------------- instructions
-_FILES_RE = re.compile(r"\s*\(files:\s*(?P<files>[^)]*)\)\s*$")
-_HEAD_RE = re.compile(r"^\[(?P<source>[^/\]]+)/(?P<kind>[^\]]+)\]\s*(?P<rest>.*)$", re.S)
-_TARGET_RE = re.compile(r"[A-Za-z0-9_.:+-]{1,64}")
-_BULLET_SPLIT_RE = re.compile(r";\s+(?=\([a-z]\)\s)")
-# the optional instance override is ``[Leg_1]``; the ``/`` keeps it from swallowing the
-# ``[source/kind]`` head that follows it
-_BULLET_RE = re.compile(r"^\((?P<letter>[a-z])\)\s*(?:\[(?P<target>[^\]/]+)\]\s*)?(?P<rest>.*)$", re.S)
-
-
-#: heading every refine template renders ``compact_instructions``' output under
-#: (``prompts/tracks/refine_object.j2``, ``refine_graphics.j2``, ``scene_refine.j2``)
-_CHANGES_HEADING = "## Changes to make"
-#: what follows the numbered list in those templates
-_CHANGES_END = ("## ", "Targets:", "Files in scope:", "EDIT ONLY THESE FILES")
-_NUMBERED_RE = re.compile(r"^\s*\d+\.\s+(?P<line>.*)$")
-
-
-def _task(source: str, kind: str, target: str, instruction: str, files: list[str]) -> dict[str, Any]:
- return {"source": source, "kind": kind, "target": target.strip(),
- "instruction": instruction.strip(), "files": files}
-
-
-def changes_block(prompt: str) -> list[str]:
- """The instruction lines of one rendered refine prompt.
-
- The templates number the lines (``{{ loop.index }}. {{ t }}``), so the numbering is
- stripped back off; a line wrapped over several lines of markdown is rejoined onto the
- entry it belongs to. A prompt without the heading (a rebuild, a texture pass, a
- baseline) yields nothing.
- """
- out: list[str] = []
- started = False
- for ln in prompt.splitlines():
- if not started:
- started = ln.startswith(_CHANGES_HEADING)
- continue
- if ln.startswith(_CHANGES_END):
- break
- if (m := _NUMBERED_RE.match(ln)) is not None:
- out.append(m.group("line").strip())
- elif ln.strip() and out:
- out[-1] += " " + ln.strip()
- return out
-
-
-def sent_instructions(ws: Workspace, round_index: int) -> list[dict[str, Any]]:
- """What each of the round's agent sessions was actually handed, per session.
-
- A refine round fans out over file-disjoint groups, and each group's session gets its
- OWN ``compact_instructions`` output — folded by target, capped at
- ``policy.max_instructions_per_task``. ``RoundRecord.instructions`` keeps none of
- that: it is the flat pre-grouping task list. The prompt files the sessions were
- started with (``trajectories/<label>_rNN/prompt.md``) do, so they are the only record
- of the real brief; 195 of the corpus's 205 transitions still have theirs on disk.
- A ``<label>.a<n>`` retry re-sends the same brief, so identical blocks are kept once.
- """
- traj = ws.root / "trajectories"
- suffix = f"_r{round_index:02d}"
- dirs = sorted(traj.glob(f"*{suffix}")) if traj.is_dir() else []
- out: list[dict[str, Any]] = []
- seen: set[tuple[str, ...]] = set()
- # sorted by LABEL, not by directory name: `.` sorts before `_`, so sorting the names
- # puts `refine.a2_r01` in front of the `refine_r01` whose retry it is, and the retry
- # would then be the session the brief is credited to
- for label in sorted(d.name[: -len(suffix)] for d in dirs):
- try:
- text = (traj / f"{label}{suffix}" / "prompt.md").read_text(errors="replace")
- except OSError:
- continue
- lines = changes_block(text)
- if not lines or tuple(lines) in seen:
- continue
- seen.add(tuple(lines))
- out.append({"session": label, "lines": lines})
- return out
-
-
-def parse_instruction(line: str) -> list[dict[str, Any]]:
- """The task(s) encoded in one recorded instruction line.
-
- ``orchestrator.compact_instructions`` emits three shapes and the corpus holds all
- three: a single ``RefineTask.line()`` (``[source/kind] target: text (files: a, b)``),
- the grouped form (``target: (a) [source/kind] text; (b) …``) and — detail rounds — a
- bare sentence. A line that matches none of them still yields one task carrying its
- whole text, so no instruction is silently lost; the target prefix is only honoured
- when it is a single bare token, or a sentence's own colon would be read as one.
- """
- raw = line.strip()
- files: list[str] = []
- if (m := _FILES_RE.search(raw)) is not None:
- files = [f.strip() for f in m.group("files").split(",") if f.strip()]
- raw = raw[: m.start()].rstrip()
- if (head := _HEAD_RE.match(raw)) is not None:
- target, sep, text = head.group("rest").partition(": ")
- if not sep:
- target, text = "overall", head.group("rest")
- return [_task(head.group("source"), head.group("kind"), target, text, files)]
- target, sep, rest = raw.partition(": ")
- if not sep or _TARGET_RE.fullmatch(target) is None:
- return [_task("", "", "", raw, files)]
- bullets = _BULLET_SPLIT_RE.split(rest)
- if _BULLET_RE.match(bullets[0]) is None:
- return [_task("", "", target, rest, files)]
- out: list[dict[str, Any]] = []
- for b in bullets:
- bm = _BULLET_RE.match(b)
- if bm is None:
- out.append(_task("", "", target, b, files))
- continue
- who, inner = bm.group("target") or target, bm.group("rest")
- if (hm := _HEAD_RE.match(inner)) is not None:
- out.append(_task(hm.group("source"), hm.group("kind"), who, hm.group("rest"), files))
- else:
- out.append(_task("", "", who, inner, files))
- return out
-
-
-# --------------------------------------------------------------------------- rows
-def outcome_label(delta: float | None, *, threshold: float = OUTCOME_THRESHOLD) -> str:
- """``improved`` / ``regressed`` / ``unchanged`` by ``threshold``; ``unscored`` when
- either side has no usable verdict (a degraded judge outage is *not* a 0.0)."""
- if delta is None:
- return "unscored"
- if delta > threshold:
- return "improved"
- if delta < -threshold:
- return "regressed"
- return "unchanged"
-
-
-def _gate_errors(rnd: RoundRecord) -> list[dict[str, Any]]:
- """The ERROR findings of a round's gates — the half of the refine brief the judge
- did not write. ``fix_hint`` is what ``build_refine_instructions`` copies verbatim."""
- return [{"gate": g.gate, "target": f.target or "", "message": f.message,
- "fix_hint": f.fix_hint, "data": f.data}
- for g in rnd.gates for f in g.errors]
-
-
-def _side(rnd: RoundRecord, files: list[str]) -> dict[str, Any]:
- j = effective_judgment(rnd)
- return {
- "round": rnd.index,
- "kind": rnd.kind,
- "commit": rnd.commit,
- "score": j.overall if j else None,
- "passed": j.passed if j else None,
- "build_ok": None if rnd.build is None else rnd.build.ok,
- "gate_errors": sum(len(g.errors) for g in rnd.gates),
- "files": files,
- }
-
-
-def _diff_lines(text: str) -> tuple[int, int]:
- """(added, removed) body lines of a unified diff — the ``+++``/``---`` headers are
- file markers, not content, and would otherwise count once per changed file."""
- added = removed = 0
- for ln in text.splitlines():
- if ln.startswith("+") and not ln.startswith("+++"):
- added += 1
- elif ln.startswith("-") and not ln.startswith("---"):
- removed += 1
- return added, removed
-
-
-def transition_row(
- ws: Workspace, rec: RunRecord, run_id: RunId, prev: RoundRecord, cur: RoundRecord,
- *, threshold: float = OUTCOME_THRESHOLD, max_diff_bytes: int = MAX_DIFF_BYTES,
- files_cache: dict[str, list[str]] | None = None,
-) -> dict[str, Any]:
- """One training row for the transition ``prev`` → ``cur``. Raises
- :class:`~codeverse.flywheel._git.GitReadError` when either commit is unreadable."""
- cache = files_cache if files_cache is not None else {}
- for c in (prev.commit, cur.commit):
- if c not in cache:
- cache[c] = _git.list_files_at(ws, c)
- diff, diff_bytes, truncated = _git.diff_between(ws, prev.commit, cur.commit, max_bytes=max_diff_bytes)
- added, removed = _diff_lines(diff)
- pj, cj = effective_judgment(prev), effective_judgment(cur)
- delta = None if pj is None or cj is None else round(cj.overall - pj.overall, 4)
- sent = sent_instructions(ws, cur.index)
- lines = [ln for s in sent for ln in s["lines"]] or list(cur.instructions)
- return {
- "schema_version": SCHEMA_VERSION,
- "battery": run_id.battery,
- "run": run_id.slug,
- "run_dir": str(ws.root),
- "cell": run_id.cell,
- "arm": run_id.arm,
- "prompt_id": rec.spec.id,
- "prompt": rec.spec.prompt,
- "prompt_hash": prompt_hash(rec.spec.prompt),
- "track": rec.spec.track.value,
- "language": rec.spec.language.value,
- "generator": rec.spec.backends.generator,
- "agent_backend": cur.agent_backend,
- "round": cur.index,
- "round_kind": cur.kind,
- "issues": [i.model_dump(mode="json") for i in pj.issues] if pj else [],
- "improvement_plan": [i.model_dump(mode="json") for i in pj.improvement_plan] if pj else [],
- "gate_findings": _gate_errors(prev),
- # what was COMPILED (RoundRecord.instructions) vs what was SENT (the per-group
- # compacted prompts). Neither is the other: see the module docstring.
- "refine_tasks": list(cur.instructions),
- "sent_instructions": sent,
- "instruction_source": "prompt" if sent else "record",
- "instruction_lines": lines,
- "instruction_tasks": [t for line in lines for t in parse_instruction(line)],
- "diff": diff,
- "diff_bytes": diff_bytes,
- "diff_added": added,
- "diff_removed": removed,
- "diff_truncated": truncated,
- "diff_max_bytes": max_diff_bytes,
- "changed_files": _git.changed_files_between(ws, prev.commit, cur.commit),
- "before": _side(prev, cache[prev.commit]),
- "after": _side(cur, cache[cur.commit]),
- "score_before": pj.overall if pj else None,
- "score_after": cj.overall if cj else None,
- "score_delta": delta,
- "outcome": outcome_label(delta, threshold=threshold),
- "outcome_threshold": threshold,
- }
-
-
-# --------------------------------------------------------------------------- scan
-@dataclass
-class Report:
- """What one export saw: totals, named drops, and a digest per written row."""
-
- roots: list[str] = field(default_factory=list)
- skipped: list[str] = field(default_factory=list)
- runs: int = 0
- kinds: Counter[str] = field(default_factory=Counter)
- drops: Counter[str] = field(default_factory=Counter)
- aliases: Counter[str] = field(default_factory=Counter)
- stats: list[dict[str, Any]] = field(default_factory=list)
-
- @property
- def rows(self) -> int:
- return len(self.stats)
-
- @property
- def transitions(self) -> int:
- """Refine-kind rounds seen — rows plus every per-transition drop."""
- return sum(self.kinds[k] for k in REFINE_KINDS)
-
-
-def scan_roots(root: Path) -> tuple[list[Path], list[Path]]:
- """``(scan roots, skipped children)`` for ``root``.
-
- One scan root per battery, so ``RunId.battery`` stays a battery name: ``bench/out``
- holds runs 5 levels down, and scanning it whole would label every run ``out`` and
- collapse the per-battery summary. A child that is neither a run nor a battery is
- RETURNED as skipped rather than dropped in silence — ``bench/out`` holds three of
- them (report-only merge directories with no cells of their own).
- """
- if is_run_dir(root) or any((root / m).is_dir() for m in BATTERY_MARKERS):
- return [root], []
- kids = sorted(p for p in root.iterdir() if p.is_dir())
- if any(is_run_dir(k) for k in kids):
- return [root], []
- roots = [k for k in kids if is_run_dir(k) or any((k / m).is_dir() for m in BATTERY_MARKERS)]
- if not roots:
- return [root], []
- return roots, [k for k in kids if k not in roots]
-
-
-def battery_root_for(run_dir: Path) -> Path:
- """The root that gives ``run_dir`` its battery label when no scan root claims it.
-
- The OUTERMOST ancestor (within :data:`RUN_SEARCH_DEPTH`) whose path down to the run
- begins with a battery marker — ``bench/out/<battery>/arms/<arm>/cells/…/run`` reports
- ``<battery>``, the same label a whole-``bench/out`` scan mints for it, so pointing the
- exporter at one run and finding it in a full scan produce the same identity. A layout
- with no marker at all (``<dir>/<slug>``) falls back to the run's parent.
- """
- root = run_dir.parent
- for p in run_dir.parents[:RUN_SEARCH_DEPTH]:
- parts = run_dir.relative_to(p).parts
- if parts[0] in RUN_PATH_NOISE:
- root = p
- return root
-
-
-def physical_run_id(resolved: Path, roots: Sequence[Path], found: RunId) -> RunId:
- """``found`` re-minted against the battery the run PHYSICALLY lives in.
-
- ``found`` names the root the scan REACHED the run through, and in ``bench/out`` that
- is not always where it ran: 54 runs are reachable through a symlink in another
- battery's cells. The deepest scan root containing the resolved path wins; a run
- linked in from outside the scanned tree keeps its own :func:`battery_root_for`.
- """
- owners = [r for r in roots if resolved.is_relative_to(r)]
- if owners:
- return run_id_for(max(owners, key=lambda r: len(r.parts)), resolved)
- return run_id_for(battery_root_for(resolved), resolved)
-
-
-def _found_runs(scan: Path, report: Report) -> Iterator[FoundRun]:
- """Every run under one scan root — or ``scan`` itself when it IS a run directory.
-
- ``iter_runs`` searches strictly BELOW its argument, so a lone run directory (the
- documented single-run input) yields nothing there and the export silently reported
- zero rows. It is read here instead, against the battery root its path sits in.
- """
- if is_run_dir(scan):
- ws = Workspace(scan)
- try:
- rec = load_record(ws)
- except RecordError:
- report.drops["unreadable_record"] += 1
- return
- yield FoundRun(ws, rec, run_id_for(battery_root_for(ws.root), ws.root))
- return
- found = iter_runs(scan, on_error=lambda _p, _e: report.drops.update(["unreadable_record"]))
- try: # iter_runs is a generator: it raises for an empty battery on the first step
- found = itertools.chain([next(found)], found)
- except FileNotFoundError:
- report.drops["empty_battery"] += 1
- return
- except StopIteration:
- return
- yield from found
-
-
-def iter_rows(
- root: Path, report: Report, *, threshold: float = OUTCOME_THRESHOLD,
- max_diff_bytes: int = MAX_DIFF_BYTES,
-) -> Iterator[dict[str, Any]]:
- """Yield one row per exportable refine transition under ``root``, filling ``report``.
-
- Every run and every round is accounted for: a run is exported once per resolved path
- (``bench/out``'s cross-battery symlinks make runs reachable twice), it is labelled
- with the battery it physically lives in, and a child directory or transition that
- cannot be exported increments a named reason instead of vanishing.
- """
- roots, skipped = scan_roots(root)
- report.skipped = [str(p) for p in skipped]
- if skipped: # a Counter records a zero once touched, and "dropped: none" must stay true
- report.drops["not_a_battery"] += len(skipped)
- batteries = [r.resolve() for r in roots if not is_run_dir(r)]
- seen: set[Path] = set()
- for scan in roots:
- report.roots.append(str(scan))
- for ws, rec, found_id in _found_runs(scan, report):
- resolved = ws.root # Workspace resolves its root on construction
- if resolved in seen:
- report.drops["duplicate_run"] += 1
- continue
- seen.add(resolved)
- run_id = physical_run_id(resolved, batteries, found_id)
- if run_id.battery != found_id.battery:
- report.aliases[f"{found_id.battery} -> {run_id.battery}"] += 1
- report.runs += 1
- yield from _run_rows(ws, rec, run_id, report, threshold=threshold, max_diff_bytes=max_diff_bytes)
-
-
-def _run_rows(
- ws: Workspace, rec: RunRecord, run_id: RunId, report: Report, *, threshold: float, max_diff_bytes: int
-) -> Iterator[dict[str, Any]]:
- by_index = {r.index: r for r in rec.rounds}
- files_cache: dict[str, list[str]] = {}
- for cur in rec.rounds:
- report.kinds[cur.kind] += 1
- if cur.kind not in REFINE_KINDS:
- continue
- prev = by_index.get(cur.index - 1)
- if prev is None:
- report.drops["missing_predecessor"] += 1
- continue
- if not prev.commit or not cur.commit:
- report.drops["missing_commit"] += 1
- continue
- if effective_judgment(prev) is None:
- report.drops["missing_prev_judgment"] += 1
- continue
- # the RECORD's task list decides exportability: a round with no compiled task was
- # never given a brief, while a missing prompt file only costs the row its
- # per-session breakdown (transition_row then falls back to this same list)
- if not cur.instructions:
- report.drops["no_instructions"] += 1
- continue
- try:
- row = transition_row(ws, rec, run_id, prev, cur, threshold=threshold,
- max_diff_bytes=max_diff_bytes, files_cache=files_cache)
- except _git.GitReadError:
- report.drops["git_read_failed"] += 1
- continue
- if not row["diff"].strip():
- report.drops["empty_diff"] += 1
- continue
- report.stats.append({"battery": row["battery"], "track": row["track"], "outcome": row["outcome"],
- "diff_bytes": row["diff_bytes"], "diff_added": row["diff_added"],
- "diff_removed": row["diff_removed"]})
- yield row
-
-
-def export(
- root: Path, out: Path | None, *, threshold: float = OUTCOME_THRESHOLD,
- max_diff_bytes: int = MAX_DIFF_BYTES,
-) -> Report:
- """Write the rows under ``root`` as JSONL to ``out`` (``None`` = count only)."""
- report = Report()
- rows = iter_rows(root, report, threshold=threshold, max_diff_bytes=max_diff_bytes)
- if out is None:
- for _ in rows:
- pass
- return report
- out.parent.mkdir(parents=True, exist_ok=True)
- tmp = out.with_suffix(out.suffix + ".tmp")
- with tmp.open("w") as fh:
- for row in rows:
- fh.write(json.dumps(row, ensure_ascii=False) + "\n")
- tmp.replace(out)
- return report
-
-
-# --------------------------------------------------------------------------- report
-def _table(title: str, counts: Counter[str], *, by_count: bool = True) -> list[str]:
- if not counts:
- return []
- items = counts.most_common() if by_count else sorted(counts.items())
- width = max(len(k) for k, _ in items)
- return [f"{title}:"] + [f" {k:<{width}} {n:>5}" for k, n in items]
-
-
-def scan_report(report: Report) -> str:
- """Totals + every drop reason. Printed always: a truncated export must never look
- like a small corpus."""
- by_kind = ", ".join(f"{n} {k}" for k, n in sorted(report.kinds.items())) or "none"
- lines = [f"{report.rows} row(s) from {report.runs} run(s) in {len(report.roots)} root(s)",
- f"rounds seen: {sum(report.kinds.values())} "
- f"({report.transitions} refine-kind; {by_kind})"]
- lines += _table("dropped", report.drops, by_count=False) or ["dropped: none"]
- if report.skipped:
- lines += ["skipped (neither a run nor a battery):"] + [f" {p}" for p in report.skipped]
- # not a drop: the run WAS exported, under the battery it physically lives in
- lines += _table("reached through another battery's symlink", report.aliases, by_count=False)
- return "\n".join(lines)
-
-
-def summary_report(report: Report, *, threshold: float = OUTCOME_THRESHOLD) -> str:
- """Counts per battery / track / outcome, and how big the diffs are."""
- if not report.stats:
- return "no rows to summarise"
- lines = _table("rows per battery", Counter(s["battery"] for s in report.stats))
- lines += [""] + _table("rows per track", Counter(s["track"] for s in report.stats))
- lines += [""] + _table(f"rows per outcome (|delta| > {threshold})",
- Counter(s["outcome"] for s in report.stats))
- sizes = sorted(s["diff_bytes"] for s in report.stats)
- churn = sorted(s["diff_added"] + s["diff_removed"] for s in report.stats)
- lines += ["", f"diff size: median {int(statistics.median(sizes))} bytes "
- f"(min {sizes[0]}, max {sizes[-1]}); "
- f"median {int(statistics.median(churn))} changed lines"]
- return "\n".join(lines)
-
-
-def main(argv: Sequence[str] | None = None) -> int:
- ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
- ap.add_argument("root", type=Path, help="directory of batteries, a battery, a runs dir or one run")
- ap.add_argument("--out", type=Path, help="write the rows here as JSONL")
- ap.add_argument("--summary", action="store_true", help="also print per-battery/track/outcome counts")
- ap.add_argument("--threshold", type=float, default=OUTCOME_THRESHOLD,
- help=f"|delta| above which a round improved/regressed (default {OUTCOME_THRESHOLD})")
- ap.add_argument("--max-diff-bytes", type=int, default=MAX_DIFF_BYTES,
- help=f"cap on the inlined diff, recorded on every row (default {MAX_DIFF_BYTES})")
- args = ap.parse_args(argv)
-
- if not args.root.is_dir():
- ap.error(f"{args.root} is not a directory")
- if args.out is None and not args.summary:
- ap.error("nothing to do: pass --out to write rows, --summary to print counts, or both")
-
- report = export(args.root, args.out, threshold=args.threshold, max_diff_bytes=args.max_diff_bytes)
- if args.out is not None:
- print(f"wrote {args.out}")
- print(scan_report(report))
- if args.summary:
- print()
- print(summary_report(report, threshold=args.threshold))
- return 0 if report.rows else 1
-
-
-if __name__ == "__main__":
- raise SystemExit(main())
diff --git a/harness/bench/refine_sft.py b/harness/bench/refine_sft.py
deleted file mode 100644
index 921b5082..00000000
--- a/harness/bench/refine_sft.py
+++ /dev/null
@@ -1,184 +0,0 @@
-"""Refine rounds as SFT samples: the brief the round was given, answered with the code it produced.
-
-``bench/refine_pairs.py`` exports the transition as data (issues, findings, instructions,
-diff, delta). This turns the rows that IMPROVED into the message shape the finetune
-pipeline consumes (``{"id", "messages": [system, user, assistant]}`` — see
-``toolkits/llamafactory/to_llamafactory.py``), so the corpus teaches the one thing the
-harness produces and a one-shot dataset cannot: given a judged, gated object and a list of
-what is wrong with it, write the corrected files.
-
- python bench/refine_sft.py bench/out --out refine_sft.jsonl --summary
-
-The assistant turn is the FULL text of every file the round changed, in the same
-``=== FILE: path ===`` envelope ``tracks/generation`` parses, because that is the answer
-shape a trained model has to produce. Rows are dropped, and counted, when the answer
-would not fit ``--max-answer-chars`` or a side cannot be read back out of the run's repo.
-"""
-
-from __future__ import annotations
-
-import argparse
-import json
-import sys
-from collections import Counter
-from pathlib import Path
-from statistics import median
-
-sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
-
-from bench.refine_pairs import Report, iter_rows # noqa: E402
-from codeverse.contracts.common import Language # noqa: E402
-from codeverse.flywheel._git import GitReadError, read_tree_at # noqa: E402
-from codeverse.tracks.prompting import language_system_prompt # noqa: E402
-from codeverse.workspace import Workspace # noqa: E402
-
-#: a sample whose answer is longer than this is dropped rather than truncated: half a
-#: file teaches a model to write half a file
-MAX_ANSWER_CHARS = 120_000
-#: outcomes worth training on. A regressed round is a real transition and stays in
-#: refine_pairs.jsonl (it is evidence about the loop), but it is not an answer to copy.
-DEFAULT_OUTCOMES = ("improved",)
-SCHEMA_VERSION = 1
-
-
-def envelope(files: dict[str, str]) -> str:
- """The ``=== FILE: path === … === END FILE ===`` blocks ``tracks.generation.parse_multifile``
- reads — the same envelope a single-shot answer must produce, terminator included."""
- return "\n".join(f"=== FILE: {p} ===\n{files[p].rstrip()}\n=== END FILE ===" for p in sorted(files))
-
-
-def _bullets(items: list[dict], *keys: str, limit: int = 12) -> list[str]:
- out = []
- for it in items[:limit]:
- parts = [str(it.get(k, "")).strip() for k in keys]
- line = " — ".join(p for p in parts if p)
- if line:
- out.append(f"- {line}")
- return out
-
-
-def user_turn(row: dict, before: dict[str, str]) -> str:
- """The brief: what was asked for, what was wrong, and the code as it stood."""
- lines = [f"Refine this {row['track'].replace('_', ' ')} written in {row['language']}.", "",
- "## Request", row["prompt"].strip(), ""]
- if row.get("gate_findings"):
- lines += ["## Gate errors (measured by the harness — fix these first)"]
- lines += _bullets(row["gate_findings"], "target", "message", "fix_hint")
- lines += [""]
- if row.get("issues"):
- lines += ["## What the judge saw"]
- lines += _bullets(row["issues"], "target", "severity", "detail")
- lines += [""]
- if row.get("instruction_lines"):
- lines += ["## Changes to make", *(f"{i}. {ln}" for i, ln in enumerate(row["instruction_lines"], 1)), ""]
- lines += ["## Current files", envelope(before), "",
- "Return every file you change in full, in the same `=== FILE: path ===` envelope."]
- return "\n".join(lines)
-
-
-def sample(row: dict, *, max_answer_chars: int = MAX_ANSWER_CHARS) -> dict | None:
- """One SFT row, or ``None`` with the reason recorded on ``row['_drop']``."""
- changed = [f for f in row.get("changed_files") or [] if f.startswith("src/")]
- if not changed:
- row["_drop"] = "no_src_change"
- return None
- ws = Workspace(Path(row["run_dir"]))
- try:
- before_tree = read_tree_at(ws, row["before"]["commit"])
- after_tree = read_tree_at(ws, row["after"]["commit"])
- except (GitReadError, OSError) as e:
- row["_drop"] = f"git_read_failed: {type(e).__name__}"
- return None
-
- def _text(tree: dict[str, bytes], paths: list[str]) -> dict[str, str] | None:
- out = {}
- for p in paths:
- raw = tree.get(p)
- if raw is None:
- return None
- try:
- out[p] = raw.decode("utf-8")
- except UnicodeDecodeError:
- return None
- return out
-
- after = _text(after_tree, changed)
- if after is None:
- row["_drop"] = "answer_unreadable"
- return None
- # the brief carries the files the round STARTED from: whatever it changed, plus the
- # rest of src/ it had to stay consistent with
- before = _text(before_tree, sorted(p for p in before_tree if p.startswith("src/"))) or {}
- answer = envelope(after)
- if len(answer) > max_answer_chars:
- row["_drop"] = "answer_too_long"
- return None
- try:
- system = language_system_prompt(Language(row["language"]), tools=False)
- except ValueError:
- row["_drop"] = "unknown_language"
- return None
- return {
- "id": f"{row['battery']}/{row['run']}/r{row['round']:02d}",
- "schema_version": SCHEMA_VERSION,
- "track": row["track"], "language": row["language"], "prompt_id": row["prompt_id"],
- "score_before": row["score_before"], "score_after": row["score_after"],
- "score_delta": row["score_delta"], "changed_files": changed,
- "messages": [{"role": "system", "content": system},
- {"role": "user", "content": user_turn(row, before)},
- {"role": "assistant", "content": answer}],
- }
-
-
-def build(root: Path, *, outcomes: tuple[str, ...] = DEFAULT_OUTCOMES,
- max_answer_chars: int = MAX_ANSWER_CHARS) -> tuple[list[dict], Report, Counter]:
- report = Report()
- drops: Counter = Counter()
- out: list[dict] = []
- for row in iter_rows(root, report):
- if row["outcome"] not in outcomes:
- drops[f"outcome:{row['outcome']}"] += 1
- continue
- s = sample(row, max_answer_chars=max_answer_chars)
- if s is None:
- drops[row.get("_drop", "unknown")] += 1
- continue
- out.append(s)
- return out, report, drops
-
-
-def main(argv: list[str] | None = None) -> int:
- ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
- ap.add_argument("root", type=Path, help="a corpus of batteries, one battery, a runs dir or one run")
- ap.add_argument("--out", type=Path, default=None, help="write the samples here (JSONL)")
- ap.add_argument("--outcomes", default=",".join(DEFAULT_OUTCOMES),
- help=f"comma list of refine_pairs outcomes to keep (default {DEFAULT_OUTCOMES[0]})")
- ap.add_argument("--max-answer-chars", type=int, default=MAX_ANSWER_CHARS)
- ap.add_argument("--summary", action="store_true", help="print the counts")
- ns = ap.parse_args(argv)
- outcomes = tuple(x for x in ns.outcomes.split(",") if x)
- rows, report, drops = build(ns.root, outcomes=outcomes, max_answer_chars=ns.max_answer_chars)
- if ns.out:
- ns.out.parent.mkdir(parents=True, exist_ok=True)
- ns.out.write_text("".join(json.dumps(r) + "\n" for r in rows))
- print(f"wrote {ns.out}")
- print(f"{len(rows)} sample(s) from {report.runs} run(s)")
- if ns.summary:
- print(f"rounds seen: {sum(report.kinds.values())} (" +
- ", ".join(f"{k} {v}" for k, v in sorted(report.kinds.items())) + ")")
- if report.drops:
- print("dropped by the scan: " + ", ".join(f"{k} {v}" for k, v in sorted(report.drops.items())))
- if drops:
- print("not exported: " + ", ".join(f"{k} {v}" for k, v in sorted(drops.items())))
- if rows:
- ans = [len(r["messages"][2]["content"]) for r in rows]
- usr = [len(r["messages"][1]["content"]) for r in rows]
- print(f"answer chars: median {int(median(ans))}, max {max(ans)}; "
- f"brief chars: median {int(median(usr))}, max {max(usr)}")
- for field in ("track", "language"):
- print(f"by {field}: " + ", ".join(f"{k} {v}" for k, v in sorted(Counter(r[field] for r in rows).items())))
- return 0
-
-
-if __name__ == "__main__":
- raise SystemExit(main())
diff --git a/harness/codeverse/cli/flywheel_cmd.py b/harness/codeverse/cli/flywheel_cmd.py
index a51e2c66..078085b4 100644
--- a/harness/codeverse/cli/flywheel_cmd.py
+++ b/harness/codeverse/cli/flywheel_cmd.py
@@ -77,6 +77,20 @@ def pairs_cmd(
ok(f"{n} pairs → {out_jsonl}")
+@flywheel_app.command("refine")
+def refine_cmd(
+ runs_dir: Annotated[Path, typer.Argument()],
+ out_jsonl: Annotated[Path, typer.Argument()],
+ with_code: Annotated[bool, typer.Option("--with-code", help="inline the changed files, so a "
+ "format converter needs no access to the runs")] = False,
+) -> None:
+ """Refine rounds as transitions (brief → diff → score delta) → JSONL."""
+ from codeverse.flywheel.refine import build_refine
+
+ n, drops = build_refine(runs_dir, out_jsonl, with_code=with_code)
+ ok(f"{n} transitions → {out_jsonl}" + (f"; not exported: {dict(drops)}" if drops else ""))
+
+
@flywheel_app.command("caption")
def caption_cmd(
target: Annotated[str, typer.Argument(help="run slug/dir, or a runs root with --all")],
diff --git a/harness/codeverse/flywheel/pairs.py b/harness/codeverse/flywheel/pairs.py
index c60ecf70..2d537f60 100644
--- a/harness/codeverse/flywheel/pairs.py
+++ b/harness/codeverse/flywheel/pairs.py
@@ -79,6 +79,12 @@ def _base(ws: Workspace, rec: RunRecord, kind: str, *, slug: str | None = None)
}
+#: the judge Δ at which two rounds are a preference pair — and, in
+#: :mod:`codeverse.flywheel.refine`, the Δ at which a transition counts as improved. One
+#: constant, so a Δ that is a preference pair here is never "unchanged" there.
+MIN_PREFERENCE_DELTA = 0.05
+
+
def preference_pairs(ws: Workspace, rec: RunRecord, *, min_delta: float,
slug: str | None = None) -> list[dict[str, Any]]:
cache: dict[str, dict[str, Any]] = {}
diff --git a/harness/codeverse/flywheel/refine.py b/harness/codeverse/flywheel/refine.py
new file mode 100644
index 00000000..0c1bfc36
--- /dev/null
+++ b/harness/codeverse/flywheel/refine.py
@@ -0,0 +1,204 @@
+"""Refine rounds as transitions: what the round was told, what it changed, what it scored.
+
+:mod:`codeverse.flywheel.pairs` exports whole code trees per round and never looks at what
+asked for the change, so a refine round arrives as two 30 kB files with no cause. This
+exports the transition instead — one row per refine round::
+
+ (the previous round's judge issues + improvement plan + gate ERRORs with their
+ fix hints + the refine tasks compiled for the round) -> src/ diff -> score delta
+
+Both sides come from the round's OWN recorded commit, never ``HEAD``: only 54 of 239
+recorded runs have HEAD at their last round, and 152 end on a "restore best round rNN"
+commit. ``instructions`` is ``RoundRecord.instructions``, the task list
+``build_refine_instructions`` compiled — not the per-group prompt text the sessions were
+handed, which is a rendering of it.
+
+The rows are data, not a training format: ``toolkits/llamafactory/build_refine_sft.py``
+turns them into the message shape the finetune pipeline consumes. ``--with-code`` inlines
+the changed files so that conversion needs no access to the runs.
+"""
+
+from __future__ import annotations
+
+from collections import Counter
+from collections.abc import Iterator
+from pathlib import Path
+from typing import Any
+
+from pydantic import BaseModel, Field
+
+from codeverse.contracts.artifacts import Severity
+from codeverse.contracts.run import RoundRecord, RunRecord
+from codeverse.flywheel._git import GitReadError, changed_files_between, diff_between, read_tree_at
+from codeverse.flywheel.pairs import MIN_PREFERENCE_DELTA
+from codeverse.flywheel.record import effective_judgment, effective_score, iter_runs
+from codeverse.workspace import Workspace
+
+#: rounds that answer a judgment. A texture or asset round is driven by its own pass, and
+#: a repair round by a build error the judge never saw.
+REFINE_KINDS: tuple[str, ...] = ("refine", "rewrite", "detail")
+#: a diff longer than this is truncated, with the untruncated size kept on the row
+MAX_DIFF_BYTES = 200_000
+
+
+class RefineTransition(BaseModel):
+ """One refine round: its brief, its diff, its score delta."""
+
+ battery: str
+ run: str
+ run_dir: str
+ prompt_id: str
+ prompt: str
+ track: str
+ language: str
+ generator: str
+ round: int
+ round_kind: str
+ issues: list[dict[str, Any]] = Field(description="the PREVIOUS round's judge issues")
+ improvement_plan: list[dict[str, Any]]
+ gate_errors: list[dict[str, Any]] = Field(description="the previous round's gate ERRORs, with fix hints")
+ instructions: list[str] = Field(description="RoundRecord.instructions — the tasks compiled for this round")
+ before_commit: str
+ after_commit: str
+ changed_files: list[str]
+ diff: str
+ diff_bytes: int = Field(description="the UNTRUNCATED size, so a capped row still records what it was")
+ diff_truncated: bool
+ score_before: float | None
+ score_after: float | None
+ score_delta: float | None
+ outcome: str = Field(description="improved | unchanged | regressed | unscored")
+ before_files: dict[str, str] | None = None
+ after_files: dict[str, str] | None = None
+
+
+def _issues(rnd: RoundRecord) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
+ j = effective_judgment(rnd)
+ if j is None:
+ return [], []
+ return ([i.model_dump(mode="json") for i in j.issues],
+ [i.model_dump(mode="json") for i in j.improvement_plan])
+
+
+def _gate_errors(rnd: RoundRecord) -> list[dict[str, Any]]:
+ return [{"gate": g.gate, "target": f.target, "message": f.message, "fix_hint": f.fix_hint}
+ for g in rnd.gates for f in g.findings if f.severity == Severity.ERROR]
+
+
+def outcome_of(delta: float | None, threshold: float) -> str:
+ """The label. One rule, shared with ``pairs.MIN_PREFERENCE_DELTA``, so a Δ that is a
+ preference pair there is never "unchanged" here."""
+ if delta is None:
+ return "unscored"
+ if delta >= threshold:
+ return "improved"
+ return "regressed" if delta <= -threshold else "unchanged"
+
+
+def _transition(ws: Workspace, rec: RunRecord, prev: RoundRecord, cur: RoundRecord, *, battery: str, run: str,
+ threshold: float, max_diff_bytes: int, with_code: bool) -> RefineTransition:
+ diff, total, truncated = diff_between(ws, prev.commit, cur.commit, max_bytes=max_diff_bytes)
+ changed = changed_files_between(ws, prev.commit, cur.commit)
+ before_s, after_s = effective_score(prev), effective_score(cur)
+ delta = None if before_s is None or after_s is None else round(after_s - before_s, 4)
+ issues, plan = _issues(prev)
+ files_before = files_after = None
+ if with_code:
+ def _text(commit: str, keep: list[str] | None) -> dict[str, str]:
+ out = {}
+ for path, raw in read_tree_at(ws, commit).items():
+ if keep is not None and path not in keep:
+ continue
+ try:
+ out[path] = raw.decode("utf-8")
+ except UnicodeDecodeError:
+ continue # a binary file under src/: the diff already says it changed
+ return out
+ files_before, files_after = _text(prev.commit, None), _text(cur.commit, changed)
+ return RefineTransition(
+ battery=battery, run=run, run_dir=str(ws.root), prompt_id=rec.spec.id, prompt=rec.spec.prompt,
+ track=rec.spec.track.value, language=rec.spec.language.value, generator=rec.spec.backends.generator,
+ round=cur.index, round_kind=cur.kind, issues=issues, improvement_plan=plan,
+ gate_errors=_gate_errors(prev), instructions=list(cur.instructions),
+ before_commit=prev.commit, after_commit=cur.commit, changed_files=changed,
+ diff=diff, diff_bytes=total, diff_truncated=truncated,
+ score_before=before_s, score_after=after_s, score_delta=delta,
+ outcome=outcome_of(delta, threshold), before_files=files_before, after_files=files_after)
+
+
+#: path segments that are run LAYOUT, not a battery name (compare_backends and ab_plan
+#: bury their runs this deep)
+_LAYOUT_DIRS = frozenset({"cells", "runs", "arms"})
+
+
+def _identity(root: Path, found: Any) -> tuple[str, str]:
+ """``(battery, run)`` for a run, taken from where it PHYSICALLY lives.
+
+ ``bench/out``'s batteries symlink each other's cells (54 of 239 runs are reachable
+ twice), so labelling by the path a scan happened to reach first credits a run to a
+ battery it never ran in."""
+ try:
+ rel = found.ws.root.resolve().relative_to(root.resolve()).as_posix()
+ except ValueError:
+ return found.run_id.battery, found.run_id.rel
+ head, _, rest = rel.partition("/")
+ if not rest or head in _LAYOUT_DIRS:
+ return found.run_id.battery, rel
+ return head, rest
+
+
+def transitions(runs_dir: Path | str, *, threshold: float = MIN_PREFERENCE_DELTA,
+ max_diff_bytes: int = MAX_DIFF_BYTES, with_code: bool = False,
+ drops: Counter[str] | None = None) -> Iterator[RefineTransition]:
+ """Every exportable refine transition under ``runs_dir``; every one that is not is counted.
+
+ A run reachable through more than one path (54 of 239 under ``bench/out``, where the
+ batteries symlink each other's cells) is exported once, under the battery it physically
+ lives in."""
+ drops = drops if drops is not None else Counter()
+ root = Path(runs_dir)
+ seen: set[Path] = set()
+ for found in iter_runs(root, on_error=lambda p, e: drops.update([f"unreadable_record: {type(e).__name__}"])):
+ physical = found.ws.root.resolve()
+ if physical in seen:
+ drops["duplicate_run"] += 1
+ continue
+ seen.add(physical)
+ rounds = {r.index: r for r in found.record.rounds}
+ for cur in found.record.rounds:
+ if cur.kind not in REFINE_KINDS:
+ continue
+ prev = rounds.get(cur.index - 1)
+ if prev is None:
+ drops["no_predecessor"] += 1
+ elif not (prev.commit and cur.commit):
+ drops["no_commit"] += 1
+ elif not prev.build.ok:
+ drops["predecessor_build_failed"] += 1
+ elif effective_judgment(prev) is None:
+ drops["predecessor_unjudged"] += 1
+ else:
+ try:
+ battery, run = _identity(root, found)
+ yield _transition(found.ws, found.record, prev, cur, battery=battery, run=run,
+ threshold=threshold, max_diff_bytes=max_diff_bytes, with_code=with_code)
+ except GitReadError as e:
+ drops[f"git_read_failed: {str(e)[:60]}"] += 1
+
+
+def build_refine(runs_dir: Path | str, out_jsonl: Path | str, **kw: Any) -> tuple[int, Counter[str]]:
+ """Write the transitions as JSONL; return ``(rows, drops)``."""
+ drops: Counter[str] = Counter()
+ out = Path(out_jsonl)
+ out.parent.mkdir(parents=True, exist_ok=True)
+ n = 0
+ tmp = out.with_suffix(out.suffix + ".part")
+ with tmp.open("w") as fh:
+ for row in transitions(runs_dir, drops=drops, **kw):
+ fh.write(row.model_dump_json(exclude_none=True) + "\n")
+ n += 1
+ tmp.replace(out)
+ return n, drops
+
+
+__all__ = ["MAX_DIFF_BYTES", "REFINE_KINDS", "RefineTransition", "build_refine", "outcome_of", "transitions"]
diff --git a/harness/tests/flywheel_cli/test_refine.py b/harness/tests/flywheel_cli/test_refine.py
new file mode 100644
index 00000000..f13bca9c
--- /dev/null
+++ b/harness/tests/flywheel_cli/test_refine.py
@@ -0,0 +1,105 @@
+"""Refine transitions (codeverse/flywheel/refine.py)."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import pytest
+
+from codeverse.contracts.artifacts import GateFinding, GateReport, Severity
+from codeverse.flywheel.pairs import MIN_PREFERENCE_DELTA
+from codeverse.flywheel.record import load_record
+from codeverse.flywheel.refine import build_refine, outcome_of, transitions
+from tests.flywheel_cli.conftest import make_fake_run
+
+
+def _corpus(root: Path, *, scores=(0.30, 0.80), battery: str = "batt", slug: str = "chair"):
+ ws, _ = make_fake_run(root / battery / "runs", slug, scores=scores)
+ rec = load_record(ws)
+ rec.rounds[0].gates = [GateReport(gate="connectivity", passed=False, findings=[GateFinding(
+ gate="connectivity", severity=Severity.ERROR, target="Leg", message="part 'Leg' is floating",
+ fix_hint="move Leg down 12 mm"), GateFinding(
+ gate="connectivity", severity=Severity.WARN, target="Seat", message="thin", fix_hint="")])]
+ ws.write_json(ws.record_path, rec)
+ return ws
+
+
+def _rows(root: Path, **kw):
+ from collections import Counter
+
+ drops: Counter[str] = Counter()
+ return list(transitions(root, drops=drops, **kw)), drops
+
+
+def test_a_refine_round_carries_its_brief_its_diff_and_its_delta(tmp_path):
+ _corpus(tmp_path)
+ rows, drops = _rows(tmp_path)
+ assert len(rows) == 1 and not drops
+ t = rows[0]
+ assert t.round == 1 and t.round_kind == "refine" and t.battery == "batt"
+ assert t.score_before == 0.30 and t.score_after == 0.80 and t.score_delta == 0.50 and t.outcome == "improved"
+ assert t.instructions == ["thicken the legs"]
+ assert [g["message"] for g in t.gate_errors] == ["part 'Leg' is floating"] # ERRORs only
+ assert t.gate_errors[0]["fix_hint"] == "move Leg down 12 mm"
+ assert any("thicken the legs" in i["instruction"] for i in t.improvement_plan)
+ assert t.changed_files == ["src/model.py", "src/parts/leg.py"]
+ assert "size=2.0" in t.diff and t.diff_bytes == len(t.diff) and not t.diff_truncated
+ assert t.before_commit and t.after_commit and t.before_commit != t.after_commit
+ assert t.before_files is None and t.after_files is None # --with-code is opt-in
+
+
+def test_with_code_inlines_the_answer_and_the_tree_it_started_from(tmp_path):
+ _corpus(tmp_path)
+ rows, _ = _rows(tmp_path, with_code=True)
+ t = rows[0]
+ assert set(t.after_files) == {"src/model.py", "src/parts/leg.py"} # what changed
+ assert "size=2.0" in t.after_files["src/model.py"]
+ assert "src/model.py" in t.before_files and "size=1.0" in t.before_files["src/model.py"]
+ assert "src/parts/leg.py" not in t.before_files # it did not exist yet
+
+
[email protected]("delta,label", [(0.5, "improved"), (0.05, "improved"), (0.04, "unchanged"),
+ (0.0, "unchanged"), (-0.05, "regressed"), (None, "unscored")])
+def test_the_outcome_rule_is_the_one_pairs_uses(delta, label):
+ assert outcome_of(delta, MIN_PREFERENCE_DELTA) == label
+
+
+def test_a_run_reached_through_a_symlink_is_exported_once_under_its_physical_battery(tmp_path):
+ _corpus(tmp_path, battery="zz_physical")
+ alias = tmp_path / "aa_alias" / "runs"
+ alias.mkdir(parents=True)
+ (alias / "chair").symlink_to(tmp_path / "zz_physical" / "runs" / "chair")
+ rows, drops = _rows(tmp_path)
+ assert len(rows) == 1 and drops == {"duplicate_run": 1}
+ assert rows[0].battery == "zz_physical", "labelled by the battery it was reached through first"
+
+
[email protected]("break_it,reason", [
+ ("kind", None), ("commit", "no_commit"), ("build", "predecessor_build_failed"),
+ ("judgment", "predecessor_unjudged"),
+])
+def test_every_unexportable_round_is_counted_under_a_named_reason(tmp_path, break_it, reason):
+ ws = _corpus(tmp_path)
+ rec = load_record(ws)
+ if break_it == "kind":
+ rec.rounds[1].kind = "texture"
+ elif break_it == "commit":
+ rec.rounds[0].commit = ""
+ elif break_it == "build":
+ rec.rounds[0].build.ok = False
+ else:
+ rec.rounds[0].judgment = None
+ ws.write_json(ws.record_path, rec)
+ rows, drops = _rows(tmp_path)
+ assert rows == []
+ assert drops == ({} if reason is None else {reason: 1}), f"{break_it}: {dict(drops)}"
+
+
+def test_build_refine_writes_jsonl_atomically(tmp_path):
+ _corpus(tmp_path)
+ out = tmp_path / "nested" / "refine.jsonl"
+ n, drops = build_refine(tmp_path, out)
+ assert n == 1 and not drops and not list(out.parent.glob("*.part"))
+ row = json.loads(out.read_text().splitlines()[0])
+ assert row["outcome"] == "improved" and "before_files" not in row # exclude_none keeps rows small
diff --git a/harness/tests/flywheel_cli/test_refine_pairs.py b/harness/tests/flywheel_cli/test_refine_pairs.py
deleted file mode 100644
index b776d539..00000000
--- a/harness/tests/flywheel_cli/test_refine_pairs.py
+++ /dev/null
@@ -1,548 +0,0 @@
-"""``bench/refine_pairs.py``: refine transitions exported as (brief -> diff -> delta) rows.
-
-Every case builds a corpus in the real on-disk layout — ``<root>/<battery>/runs/<slug>``,
-a git repo with two commits and two round records — because the exporter's whole job is
-to survive that layout: the recorded round shas rather than ``HEAD`` (a finished run ends
-on a "restore best round rNN" commit), one row per resolved run path labelled with the
-battery it physically lives in (``bench/out``'s cell symlinks point ACROSS batteries), the
-brief the agent sessions were really handed rather than the pre-grouping task list, and a
-named reason for every run and transition it cannot export.
-"""
-
-from __future__ import annotations
-
-import json
-from collections.abc import Callable
-from pathlib import Path
-
-import pytest
-
-from bench.refine_pairs import (
- DROP_REASONS,
- Report,
- changes_block,
- export,
- iter_rows,
- main,
- outcome_label,
- parse_instruction,
- scan_roots,
- transition_row,
-)
-from codeverse.contracts.artifacts import BuildResult, GateFinding, GateReport, Judgment, Severity
-from codeverse.contracts.run import RoundRecord, RunRecord
-from codeverse.flywheel.record import load_record, run_id_for
-from codeverse.judges.rubrics import is_degraded
-from codeverse.workspace import Workspace
-from tests.flywheel_cli.conftest import make_fake_run
-
-
-def _corpus(root: Path, *, battery: str = "batt", slug: str = "chair") -> Workspace:
- """One battery holding one run: two judged rounds, ``src/`` committed twice."""
- ws, _ = make_fake_run(root / battery / "runs", slug)
- return ws
-
-
-def _degraded() -> Judgment:
- """What ``judges.rubrics.degraded_judgment`` writes when the judge call fails: a
- 0.0 verdict flagged both ways. ``is_degraded`` below keeps this honest."""
- return Judgment(rubric="static_object_v1", judge_backend="gemini:gemini-3.1-pro-preview",
- scores={"fidelity": 0.0}, overall=0.0, passed=False,
- summary="judge_error: 503 from the judge", n_samples=0,
- raw=json.dumps({"status": "degraded", "error": "503"}))
-
-
-def _prompt(ws: Workspace, session: str, round_index: int, lines: list[str]) -> None:
- """One session's prompt file, in the shape ``agents.cli_common.begin_session`` writes
- and ``prompts/tracks/refine_object.j2`` renders."""
- d = ws.root / "trajectories" / f"{session}_r{round_index:02d}"
- d.mkdir(parents=True, exist_ok=True)
- numbered = "\n".join(f"{i}. {ln}" for i, ln in enumerate(lines, 1))
- (d / "prompt.md").write_text(
- f"<!-- system -->\nYou are a Blender author\n\n<!-- prompt -->\n"
- f"# Refine `Chair` — round {round_index}\n\nThe object already builds.\n\n"
- f"## Changes to make (in priority order; `[gate/...]` items are hard failures — fix them first)\n\n"
- f"{numbered}\n\nTargets: Leg\n\n## What the judge saw\n1. not an instruction\n")
-
-
-def _patch(ws: Workspace, edit: Callable[[RunRecord], None]) -> RunRecord:
- rec = load_record(ws)
- edit(rec)
- ws.write_json(ws.record_path, rec)
- return rec
-
-
-def _rows(root: Path, **kw) -> tuple[list[dict], Report]:
- report = Report()
- return list(iter_rows(root, report, **kw)), report
-
-
-# --------------------------------------------------------------------------- pure functions
-def test_parse_instruction_refine_task_line():
- tasks = parse_instruction("[gate/gate:connectivity] Leg_0: weld the leg to the seat (files: src/model.py, src/parts/leg.py)")
- assert tasks == [{"source": "gate", "kind": "gate:connectivity", "target": "Leg_0",
- "instruction": "weld the leg to the seat",
- "files": ["src/model.py", "src/parts/leg.py"]}]
-
-
-def test_parse_instruction_grouped_bullets():
- """``compact_instructions`` folds ``Leg_0``/``Leg_1`` into one ``Leg:`` line; each
- bullet is its own task and may name the instance it came from."""
- tasks = parse_instruction(
- "Leg: (a) [gate/gate:contract] too thin; (b) [Leg_1] [judge/geometry] add a fillet (files: src/model.py)")
- assert [(t["source"], t["kind"], t["target"], t["instruction"]) for t in tasks] == [
- ("gate", "gate:contract", "Leg", "too thin"),
- ("judge", "geometry", "Leg_1", "add a fillet"),
- ]
- assert all(t["files"] == ["src/model.py"] for t in tasks)
-
-
-def test_parse_instruction_bare_sentence_keeps_its_colon():
- """Detail rounds send prose. A sentence's own colon is not a target: only a single
- bare token before ``: `` is, or the whole instruction would be cut in half."""
- prose = "Add the panel lines where the shells meet (lids, trays): 1-2 mm wide insets."
- assert parse_instruction(prose) == [
- {"source": "", "kind": "", "target": "", "instruction": prose, "files": []}]
- assert parse_instruction("window_frame: Slope the exterior sill.") == [
- {"source": "", "kind": "", "target": "window_frame", "instruction": "Slope the exterior sill.", "files": []}]
-
-
-def test_outcome_label():
- assert outcome_label(0.2) == "improved"
- assert outcome_label(-0.2) == "regressed"
- assert outcome_label(0.01) == "unchanged"
- assert outcome_label(0.2, threshold=0.5) == "unchanged"
- assert outcome_label(None) == "unscored"
-
-
-def test_scan_roots_picks_batteries_not_the_parent(tmp_path):
- root = tmp_path / "out"
- _corpus(root, battery="batt_a")
- _corpus(root, battery="batt_b")
- (root / "merged_report").mkdir() # neither a run nor a battery: reported, not silent
- assert scan_roots(root) == ([root / "batt_a", root / "batt_b"], [root / "merged_report"])
- assert scan_roots(root / "batt_a") == ([root / "batt_a"], [])
-
-
-def test_changes_block_reads_the_numbered_list_and_stops_at_the_next_section():
- """The refine templates number ``compact_instructions``' lines and follow them with
- ``Targets:``; a numbered line in a LATER section is not an instruction."""
- prompt = ("# Refine `Chair` — round 1\n\n"
- "## Changes to make (in priority order)\n\n"
- "1. Leg: (a) [judge/geometry] thicken\n\n2. [judge/material] Seat: oak,\n quarter-sawn\n\n"
- "Targets: Leg, Seat\n\n## What the judge saw\n1. legs too thin\n")
- assert changes_block(prompt) == ["Leg: (a) [judge/geometry] thicken",
- "[judge/material] Seat: oak, quarter-sawn"]
- assert changes_block("# Rebuild `Chair`\n\n1. fix the syntax error\n") == []
-
-
-# --------------------------------------------------------------------------- rows
-def test_row_carries_brief_diff_and_delta(tmp_path):
- root = tmp_path / "out"
- ws = _corpus(root)
- rows, report = _rows(root)
- (row,) = rows
- assert (row["battery"], row["run"], row["round"], row["round_kind"]) == ("batt", "chair", 1, "refine")
- assert row["prompt_id"] == "chair" and row["prompt"] == "a wooden dining chair"
- assert (row["track"], row["language"]) == ("static_object", "blender")
- # the brief is the PREVIOUS round's verdict, the change is this round's
- assert [i["target"] for i in row["issues"]] == ["Leg"]
- assert [i["instruction"] for i in row["improvement_plan"]] == ["thicken the legs", "add a back rail"]
- assert row["refine_tasks"] == ["thicken the legs"]
- assert row["before"]["files"] == ["src/model.py"]
- assert row["after"]["files"] == ["src/model.py", "src/parts/leg.py"]
- assert row["changed_files"] == ["src/model.py", "src/parts/leg.py"]
- assert "--- a/src/model.py" in row["diff"] and "+++ b/src/parts/leg.py" in row["diff"]
- assert row["diff_bytes"] == len(row["diff"].encode()) and not row["diff_truncated"]
- assert row["diff_added"] > 0 and row["diff_removed"] > 0
- assert (row["score_before"], row["score_after"], row["score_delta"]) == (0.55, 0.8, 0.25)
- assert row["before"]["build_ok"] is True and row["after"]["build_ok"] is True
- assert row["outcome"] == "improved"
- assert report.rows == 1 and not report.drops
- assert report.kinds == {"baseline": 1, "refine": 1}
- assert ws.root.name == "chair"
-
-
-def test_outcome_threshold_is_recorded_on_the_row(tmp_path):
- root = tmp_path / "out"
- _corpus(root)
- (row,), _ = _rows(root, threshold=0.5)
- assert row["outcome"] == "unchanged" and row["outcome_threshold"] == 0.5
-
-
-def test_gate_error_findings_carry_fix_hints(tmp_path):
- """The refine brief is half judge, half gate: ``build_refine_instructions`` copies a
- finding's ``fix_hint`` verbatim, and only ERROR findings become tasks."""
- root = tmp_path / "out"
- ws = _corpus(root)
- _patch(ws, lambda rec: rec.rounds[0].gates.append(GateReport(gate="connectivity", passed=False, findings=[
- GateFinding(gate="connectivity", severity=Severity.ERROR, target="Leg_0",
- message="Leg_0 is not connected", fix_hint="move Leg_0 up by 3 mm"),
- GateFinding(gate="connectivity", severity=Severity.WARN, target="Seat", message="1.2 mm overlap")])))
- (row,), _ = _rows(root)
- assert row["gate_findings"] == [{"gate": "connectivity", "target": "Leg_0",
- "message": "Leg_0 is not connected",
- "fix_hint": "move Leg_0 up by 3 mm", "data": {}}]
- assert row["before"]["gate_errors"] == 1
-
-
-def test_diff_is_capped_and_says_so(tmp_path):
- """The cap must actually cut the text: a row that carries the WHOLE diff and merely
- says "truncated" is the failure the cap exists to prevent."""
- root = tmp_path / "out"
- _corpus(root)
- (row,), _ = _rows(root, max_diff_bytes=120)
- assert row["diff_truncated"] and row["diff_max_bytes"] == 120
- assert row["diff_bytes"] > 120 and "[truncated" in row["diff"]
- head, marker, _rest = row["diff"].partition("\n... [truncated ")
- assert marker and len(head.encode()) == 120, "the kept head is exactly the cap"
- assert f"{row['diff_bytes'] - 120} bytes" in row["diff"], "and it names what it dropped"
- (whole,), _ = _rows(root) # the same diff uncapped: same prefix, more of it
- assert whole["diff"].startswith(head) and len(whole["diff"]) > len(head)
- assert not whole["diff_truncated"] and whole["diff_bytes"] == row["diff_bytes"]
-
-
-def test_diff_lines_count_content_not_file_headers(tmp_path):
- """``+++``/``---`` are one pair per changed FILE. Counting them inflates every row's
- churn by the number of files the round touched — the label a preference model reads."""
- root = tmp_path / "out"
- _corpus(root)
- (row,), _ = _rows(root)
- # round 1 rewrites 3 of the 4 lines of src/model.py and adds the 1-line src/parts/leg.py
- assert (row["diff_added"], row["diff_removed"]) == (4, 3)
- naive_added = sum(1 for ln in row["diff"].splitlines() if ln.startswith("+"))
- naive_removed = sum(1 for ln in row["diff"].splitlines() if ln.startswith("-"))
- assert (naive_added, naive_removed) == (6, 5), "two file header pairs sit in that diff"
-
-
-def test_changed_files_are_the_ones_this_round_touched(tmp_path):
- """Per-file attribution is the round's OWN diff, not the file list at its commit: a
- round that edits one of two files must not be recorded as having rewritten both."""
- root = tmp_path / "out"
- ws = _corpus(root)
- (ws.src / "parts" / "leg.py").write_text("LEG = 0.06\n")
- commit = ws.commit("round 2")
- _patch(ws, lambda rec: rec.rounds.append(RoundRecord(
- index=2, kind="refine", commit=commit, instructions=["thicken the legs again"],
- build=BuildResult(ok=True, language="blender"), judgment=rec.rounds[1].judgment)))
- rows, _ = _rows(root)
- second = rows[-1]
- assert second["round"] == 2
- assert second["changed_files"] == ["src/parts/leg.py"]
- assert second["after"]["files"] == ["src/model.py", "src/parts/leg.py"], "both exist, one changed"
- assert "src/model.py" not in second["diff"]
-
-
-def test_only_refine_kind_rounds_are_transitions(tmp_path):
- """A texture pass edits ``src/`` and has everything a row needs — a judged
- predecessor, a commit, instructions. It is still not a refine transition: nothing
- derived it from a verdict, so it is a round SEEN and a row not written, with no drop
- to explain (the drop reasons are for transitions that should have been exportable)."""
- root = tmp_path / "out"
- ws = _corpus(root)
- (ws.src / "model.py").write_text("# textured\nimport bpy\n")
- commit = ws.commit("texture pass")
- _patch(ws, lambda rec: rec.rounds.append(RoundRecord(
- index=2, kind="texture", commit=commit, instructions=["bake a quarter-sawn oak albedo"],
- build=BuildResult(ok=True, language="blender"))))
- rows, report = _rows(root)
- assert [r["round_kind"] for r in rows] == ["refine"]
- assert report.kinds == {"baseline": 1, "refine": 1, "texture": 1}
- assert report.transitions == 1 and not report.drops
-
-
-def test_a_degraded_verdict_is_unscored_not_a_zero(tmp_path):
- """A judge outage writes a 0.0 ``judge_error:`` Judgment. Read as a score it turns a
- fine round into the corpus's worst regression (0.0 - 0.55), and the sign of that row
- is exactly what the pairs are for."""
- assert is_degraded(_degraded()), "the fixture must stay the real degraded shape"
- root = tmp_path / "out"
- ws = _corpus(root)
- _patch(ws, lambda rec: setattr(rec.rounds[1], "judgment", _degraded()))
- (row,), report = _rows(root)
- assert row["score_before"] == 0.55
- assert row["score_after"] is None and row["score_delta"] is None
- assert row["outcome"] == "unscored"
- assert row["after"]["score"] is None and row["after"]["passed"] is None
- assert not report.drops
-
-
-def test_a_degraded_previous_verdict_drops_the_transition(tmp_path):
- """The brief is the PREVIOUS round's verdict. A judge outage wrote no verdict, so
- there is no brief — and a 0.0 one would make every issue list empty on purpose."""
- root = tmp_path / "out"
- ws = _corpus(root)
- _patch(ws, lambda rec: setattr(rec.rounds[0], "judgment", _degraded()))
- rows, report = _rows(root)
- assert rows == [] and dict(report.drops) == {"missing_prev_judgment": 1}
-
-
-# --------------------------------------------------------------------------- instructions
-def test_instruction_lines_are_the_ones_the_sessions_were_handed(tmp_path):
- """``RoundRecord.instructions`` is the COMPILED task list; each file-disjoint group's
- agent session is handed its own ``compact_instructions`` output, folded by target and
- capped. The two differ for 167 of the corpus's 205 transitions, so the row carries
- both and says which one ``instruction_tasks`` was parsed from."""
- root = tmp_path / "out"
- ws = _corpus(root)
- compiled = ["[judge/geometry] Leg_0: thicken it (files: src/parts/leg.py)",
- "[judge/geometry] Leg_1: thicken it (files: src/parts/leg.py)",
- "[judge/material] Seat: quarter-sawn oak (files: src/model.py)"]
- _patch(ws, lambda rec: setattr(rec.rounds[1], "instructions", compiled))
- leg = "Leg: (a) [judge/geometry] thicken it; (b) [Leg_1] [judge/geometry] thicken it (files: src/parts/leg.py)"
- seat = "[judge/material] Seat: quarter-sawn oak (files: src/model.py)"
- _prompt(ws, "refine_leg", 1, [leg])
- _prompt(ws, "refine_seat", 1, [seat])
-
- (row,), _ = _rows(root)
- assert row["instruction_source"] == "prompt"
- assert [s["session"] for s in row["sent_instructions"]] == ["refine_leg", "refine_seat"]
- assert row["instruction_lines"] == [leg, seat]
- assert row["refine_tasks"] == compiled
- assert row["instruction_lines"] != row["refine_tasks"]
- assert [t["target"] for t in row["instruction_tasks"]] == ["Leg", "Leg_1", "Seat"]
-
-
-def test_a_retry_session_is_not_a_second_brief(tmp_path):
- """``<label>.a2`` is the SAME brief re-sent after a silent bail (``_session_label``);
- counting it twice would double every task in the row."""
- root = tmp_path / "out"
- ws = _corpus(root)
- line = "Leg: (a) [judge/geometry] thicken it (files: src/parts/leg.py)"
- _prompt(ws, "refine", 1, [line])
- _prompt(ws, "refine.a2", 1, [line])
- (row,), _ = _rows(root)
- assert row["instruction_lines"] == [line]
- assert [s["session"] for s in row["sent_instructions"]] == ["refine"]
-
-
-def test_instruction_lines_fall_back_to_the_record_when_no_prompt_survives(tmp_path):
- """10 of the corpus's 205 transitions have no prompt file left. The compiled task
- list is then all there is, and the row says so rather than shipping an empty brief."""
- root = tmp_path / "out"
- _corpus(root)
- (row,), _ = _rows(root)
- assert row["sent_instructions"] == [] and row["instruction_source"] == "record"
- assert row["instruction_lines"] == row["refine_tasks"] == ["thicken the legs"]
-
-
-# --------------------------------------------------------------------------- identity
-def test_a_cross_battery_alias_is_labelled_with_the_physical_battery(tmp_path):
- """``bench/out``'s aliases point ACROSS batteries (``compare_v4_calm/cells/*`` is a
- symlink into ``compare_v4`` and ``compare_v4_harness_calm``) and the alias sorts
- FIRST, so labelling by the root that reached a run first credits it to a battery it
- never ran in. The label is the battery the run physically lives in; the alias is
- reported, not lost."""
- root = tmp_path / "out"
- ws = _corpus(root, battery="zz_physical", slug="chair")
- alias = root / "aa_alias" / "runs"
- alias.mkdir(parents=True)
- (alias / "chair").symlink_to(ws.root, target_is_directory=True)
-
- rows, report = _rows(root)
- assert report.roots == [str(root / "aa_alias"), str(root / "zz_physical")]
- assert [r["battery"] for r in rows] == ["zz_physical"]
- assert rows[0]["run_dir"] == str(ws.root)
- assert dict(report.drops) == {"duplicate_run": 1}
- assert dict(report.aliases) == {"aa_alias -> zz_physical": 1}
-
-
-def test_a_single_run_directory_exports_with_the_identity_a_full_scan_gives_it(tmp_path):
- """The documented one-run input. ``iter_runs`` searches strictly BELOW its argument,
- so the run itself was invisible and the export reported zero rows as a success."""
- root = tmp_path / "out"
- ws = _corpus(root, battery="batt", slug="chair")
- whole, _ = _rows(root)
- alone, report = _rows(ws.root)
- assert len(alone) == 1 and report.runs == 1 and not report.drops
- assert (alone[0]["battery"], alone[0]["run"]) == ("batt", "chair")
- assert alone[0] == whole[0]
-
-
-def test_a_single_nested_run_keeps_the_whole_battery_path(tmp_path):
- """The same for the deepest layout, ``<battery>/arms/<arm>/cells/<cell>/<slug>``,
- where the battery is the OUTERMOST marker on the path: stopping at the first one up
- (``<arm>``, which owns a ``cells/``) labels the run with its arm instead."""
- root = tmp_path / "out"
- ws, _ = make_fake_run(root / "batt" / "arms" / "control" / "cells" / "chair_cell", "harness_arm")
- whole, _ = _rows(root)
- (row,), report = _rows(ws.root)
- assert (row["battery"], row["arm"], row["cell"]) == ("batt", "control", "chair_cell")
- assert row["run"] == "control__chair_cell__harness_arm"
- assert report.roots == [str(ws.root)]
- assert row == whole[0]
-
-
-def test_main_accepts_one_run_directory(tmp_path, capsys):
- root = tmp_path / "out"
- ws = _corpus(root)
- assert main([str(ws.root), "--summary"]) == 0
- assert "1 row(s) from 1 run(s) in 1 root(s)" in capsys.readouterr().out
-
-
-def test_diff_uses_the_recorded_shas_not_head(tmp_path):
- """152 of the 239 recorded runs end on a "restore best round rNN" commit and only 54
- have HEAD at their last round, so the recorded sha is the only usable handle."""
- root = tmp_path / "out"
- ws = _corpus(root)
- rec = load_record(ws)
- (ws.src / "model.py").write_text("# restored baseline\n")
- ws.commit("restore best round r00")
- (row,), _ = _rows(root)
- assert row["after"]["commit"] == rec.rounds[1].commit
- assert "restored baseline" not in row["diff"]
-
-
-def test_nested_battery_layout_keeps_battery_cell_and_arm(tmp_path):
- root = tmp_path / "out"
- make_fake_run(root / "cmp" / "cells" / "chair_cell", "harness_arm")
- (row,), report = _rows(root)
- assert (row["battery"], row["cell"], row["arm"]) == ("cmp", "chair_cell", "harness_arm")
- assert row["run"] == "chair_cell__harness_arm"
- assert report.roots == [str(root / "cmp")]
-
-
-def test_transition_row_is_callable_on_one_run(tmp_path):
- """The row builder is usable without the scanner (one run, two rounds in hand)."""
- ws, rec = make_fake_run(tmp_path / "runs")
- run_id = run_id_for(tmp_path / "runs", ws.root)
- row = transition_row(ws, rec, run_id, rec.rounds[0], rec.rounds[1])
- assert row["run"] == ws.root.name and row["score_delta"] == 0.25
-
-
-# --------------------------------------------------------------------------- drops
-def _drop_not_a_battery(root: Path) -> None:
- _corpus(root)
- (root / "merged_report").mkdir() # bench/out holds three of these
-
-
-def _drop_empty_battery(root: Path) -> None:
- (root / "batt" / "cells").mkdir(parents=True)
-
-
-def _drop_unreadable_record(root: Path) -> None:
- _corpus(root)
- broken = root / "batt" / "runs" / "broken"
- broken.mkdir()
- (broken / "record.json").write_text("{not json")
-
-
-def _drop_duplicate_run(root: Path) -> None:
- ws = _corpus(root)
- (root / "batt" / "runs" / "chair_link").symlink_to(ws.root, target_is_directory=True)
-
-
-def _drop_missing_predecessor(root: Path) -> None:
- _patch(_corpus(root), lambda rec: setattr(rec.rounds[1], "index", 5))
-
-
-def _drop_missing_commit(root: Path) -> None:
- _patch(_corpus(root), lambda rec: setattr(rec.rounds[1], "commit", ""))
-
-
-def _drop_missing_prev_judgment(root: Path) -> None:
- _patch(_corpus(root), lambda rec: setattr(rec.rounds[0], "judgment", None))
-
-
-def _drop_no_instructions(root: Path) -> None:
- _patch(_corpus(root), lambda rec: setattr(rec.rounds[1], "instructions", []))
-
-
-def _drop_git_read_failed(root: Path) -> None:
- _patch(_corpus(root), lambda rec: setattr(rec.rounds[1], "commit", "0" * 40))
-
-
-def _drop_empty_diff(root: Path) -> None:
- _patch(_corpus(root), lambda rec: setattr(rec.rounds[1], "commit", rec.rounds[0].commit))
-
-
-#: one corpus per reason; the parametrisation below fails loudly if a reason has none
-DROP_CORPUS: dict[str, Callable[[Path], None]] = {
- "not_a_battery": _drop_not_a_battery,
- "empty_battery": _drop_empty_battery,
- "unreadable_record": _drop_unreadable_record,
- "duplicate_run": _drop_duplicate_run,
- "missing_predecessor": _drop_missing_predecessor,
- "missing_commit": _drop_missing_commit,
- "missing_prev_judgment": _drop_missing_prev_judgment,
- "no_instructions": _drop_no_instructions,
- "git_read_failed": _drop_git_read_failed,
- "empty_diff": _drop_empty_diff,
-}
-#: the scan-level drops leave the run itself exportable
-KEEPS_ROW = {"not_a_battery", "unreadable_record", "duplicate_run"}
-
-
[email protected]("reason", DROP_REASONS)
-def test_every_drop_reason_is_counted(tmp_path, reason):
- root = tmp_path / "out"
- DROP_CORPUS[reason](root)
- rows, report = _rows(root)
- assert dict(report.drops) == {reason: 1}
- assert len(rows) == report.rows == (1 if reason in KEEPS_ROW else 0)
-
-
-def test_a_skipped_child_is_named_in_the_report(tmp_path, capsys):
- """Counting it is half the job: "every drop is counted" has to say WHICH directory
- was passed over, or a mistyped battery name reads as an empty corpus."""
- from bench.refine_pairs import scan_report
-
- root = tmp_path / "out"
- _drop_not_a_battery(root)
- _, report = _rows(root)
- assert report.skipped == [str(root / "merged_report")]
- assert str(root / "merged_report") in scan_report(report)
- assert "not_a_battery" in scan_report(report)
-
-
-def test_drop_corpus_covers_every_declared_reason():
- assert set(DROP_CORPUS) == set(DROP_REASONS)
-
-
-# --------------------------------------------------------------------------- cli
-def test_main_writes_jsonl_and_prints_the_summary(tmp_path, capsys):
- root = tmp_path / "out"
- _corpus(root, battery="batt_a", slug="chair")
- _corpus(root, battery="batt_b", slug="stool")
- out = tmp_path / "pairs" / "refine.jsonl"
-
- assert main([str(root), "--out", str(out), "--summary"]) == 0
- printed = capsys.readouterr().out
- rows = [json.loads(line) for line in out.read_text().splitlines()]
-
- assert len(rows) == 2 and {r["battery"] for r in rows} == {"batt_a", "batt_b"}
- assert "2 row(s) from 2 run(s) in 2 root(s)" in printed
- assert "dropped: none" in printed
- assert "rows per battery" in printed and "batt_a" in printed
- assert "rows per track" in printed and "static_object" in printed
- assert "rows per outcome" in printed and "improved" in printed
- assert "diff size: median" in printed
-
-
-def test_main_reports_drops_without_writing(tmp_path, capsys):
- root = tmp_path / "out"
- _drop_no_instructions(root)
- assert main([str(root), "--summary"]) == 1
- printed = capsys.readouterr().out
- assert "0 row(s) from 1 run(s)" in printed
- assert "no_instructions" in printed
- assert "no rows to summarise" in printed
-
-
-def test_main_needs_somewhere_to_put_the_result(tmp_path):
- root = tmp_path / "out"
- _corpus(root)
- with pytest.raises(SystemExit):
- main([str(root)])
- with pytest.raises(SystemExit):
- main([str(tmp_path / "nope"), "--summary"])
-
-
-def test_export_replaces_the_target_atomically(tmp_path):
- root = tmp_path / "out"
- _corpus(root)
- out = tmp_path / "refine.jsonl"
- out.write_text("stale\n")
- report = export(root, out)
- assert report.rows == 1 and json.loads(out.read_text())["run"] == "chair"
- assert not list(tmp_path.glob("*.tmp"))
diff --git a/harness/tests/flywheel_cli/test_refine_sft.py b/harness/tests/flywheel_cli/test_refine_sft.py
deleted file mode 100644
index 79f3f8a2..00000000
--- a/harness/tests/flywheel_cli/test_refine_sft.py
+++ /dev/null
@@ -1,104 +0,0 @@
-"""Refine rounds as SFT samples (bench/refine_sft.py)."""
-
-from __future__ import annotations
-
-import json
-from pathlib import Path
-
-import pytest
-
-from bench.refine_sft import DEFAULT_OUTCOMES, build, envelope, main, sample, user_turn
-from codeverse.contracts.artifacts import GateFinding, GateReport, Severity
-from codeverse.flywheel.record import load_record
-from codeverse.tracks.generation import parse_multifile
-from tests.flywheel_cli.conftest import make_fake_run
-
-
-def _corpus(root: Path, *, scores: tuple[float, float] = (0.30, 0.80)) -> Path:
- ws, _ = make_fake_run(root / "batt" / "runs", "chair", scores=scores)
- rec = load_record(ws)
- rec.rounds[0].gates = [GateReport(gate="connectivity", passed=False, findings=[GateFinding(
- gate="connectivity", severity=Severity.ERROR, target="Leg",
- message="part 'Leg' is floating", fix_hint="move Leg down 12 mm so it meets Seat")])]
- ws.write_json(ws.record_path, rec)
- return root
-
-
-def test_an_improved_round_becomes_a_three_message_sample(tmp_path):
- rows, report, drops = build(_corpus(tmp_path))
- assert len(rows) == 1 and report.runs == 1 and not drops
- s = rows[0]
- assert [m["role"] for m in s["messages"]] == ["system", "user", "assistant"]
- assert s["score_before"] == 0.30 and s["score_after"] == 0.80
- assert s["changed_files"] == ["src/model.py", "src/parts/leg.py"]
- assert s["id"].endswith("/r01") and s["track"] == "static_object" and s["language"] == "blender"
- # the answer is the file the round produced, and it parses back through the envelope
- files = parse_multifile(s["messages"][2]["content"])
- assert set(files) == {"src/model.py", "src/parts/leg.py"} and "size=2.0" in files["src/model.py"]
- assert "bpy" in s["messages"][0]["content"].lower()
-
-
-def test_the_brief_carries_what_the_round_was_told_and_the_code_it_started_from(tmp_path):
- rows, _, _ = build(_corpus(tmp_path))
- user = rows[0]["messages"][1]["content"]
- assert "a wooden dining chair" in user
- assert "part 'Leg' is floating" in user and "move Leg down 12 mm" in user # gate finding + fix hint
- assert "thicken the legs" in user # the judge's plan / instruction
- assert "=== FILE: src/model.py ===" in user and "size=1.0" in user # the BEFORE state
- assert "size=2.0" not in user # never the answer
- assert "in full" in user.rsplit("\n", 1)[-1]
-
-
-def test_only_improved_rounds_are_exported_and_the_rest_are_counted(tmp_path):
- rows, _, drops = build(_corpus(tmp_path, scores=(0.80, 0.80))) # unchanged
- assert rows == [] and drops == {"outcome:unchanged": 1}
- rows, _, drops = build(_corpus(tmp_path / "b", scores=(0.80, 0.30)))
- assert rows == [] and drops == {"outcome:regressed": 1}
- rows, _, _ = build(_corpus(tmp_path / "c", scores=(0.80, 0.30)), outcomes=("regressed",))
- assert len(rows) == 1 and DEFAULT_OUTCOMES == ("improved",)
-
-
-def test_an_answer_over_the_cap_is_dropped_not_truncated(tmp_path):
- rows, _, drops = build(_corpus(tmp_path), max_answer_chars=50)
- assert rows == [] and drops == {"answer_too_long": 1}
-
-
-def test_a_row_whose_commit_is_gone_is_counted_not_raised(tmp_path):
- from bench.refine_pairs import Report, iter_rows
-
- row = next(iter(iter_rows(_corpus(tmp_path), Report())))
- row["after"]["commit"] = "0" * 40
- assert sample(row) is None and row["_drop"].startswith("git_read_failed")
-
-
-def test_envelope_and_user_turn_are_pure(tmp_path):
- assert envelope({"b.py": "B", "a.py": "A"}) == (
- "=== FILE: a.py ===\nA\n=== END FILE ===\n=== FILE: b.py ===\nB\n=== END FILE ===")
- assert parse_multifile(envelope({"a.py": "A"})) == {"a.py": "A"}
- row = {"track": "static_object", "language": "blender", "prompt": "a stool",
- "gate_findings": [], "issues": [], "instruction_lines": []}
- text = user_turn(row, {"src/x.py": "code"})
- assert "## Gate errors" not in text and "## What the judge saw" not in text
- assert "## Current files" in text and "a stool" in text
-
-
-def test_main_writes_jsonl_and_reports(tmp_path, capsys):
- out = tmp_path / "sft.jsonl"
- assert main([str(_corpus(tmp_path)), "--out", str(out), "--summary"]) == 0
- printed = capsys.readouterr().out
- assert "1 sample(s) from 1 run(s)" in printed and "answer chars: median" in printed
- assert "by language: blender 1" in printed
- rows = [json.loads(x) for x in out.read_text().splitlines()]
- assert len(rows) == 1 and rows[0]["schema_version"] == 1
-
-
[email protected]("lang,entry", [("urdf_blender", "src/model.py")])
-def test_an_articulated_run_keeps_only_its_changed_src_files(tmp_path, lang, entry):
- from codeverse.contracts.common import Language
-
- ws, _ = make_fake_run(tmp_path / "batt" / "runs", "cab", language=Language(lang))
- rows, _, _ = build(tmp_path)
- assert len(rows) == 1
- assert rows[0]["changed_files"] == [entry, "src/parts/leg.py"] and rows[0]["language"] == lang
- assert set(parse_multifile(rows[0]["messages"][2]["content"])) == {entry, "src/parts/leg.py"}
- assert "robot.urdf" in rows[0]["messages"][1]["content"] # the brief still shows the whole src/