Skip to content

Official Codebase of 3DCodeVerse

Python

toolkits: the refine SFT converter belongs in the tree it is named from

ziyaoc9e9707bcdfe
diff --git a/harness/docs/PAPER_WRITING.md b/harness/docs/PAPER_WRITING.md
index ddf77e7c..8afd3929 100644
--- a/harness/docs/PAPER_WRITING.md
+++ b/harness/docs/PAPER_WRITING.md
@@ -296,15 +296,18 @@ that moved the bill.
round's judge issues and improvement plan, the gate ERROR findings with their fix hints, the
instruction lines the sessions were actually handed, the unified `src/` diff between the two
recorded commits, and the score delta with an improved/unchanged/regressed label. On the
-recorded corpus: **205 rows from 239 runs**, coverage 1.00 (drops only at run level: 54
-cross-battery symlink aliases, 3 non-batteries, 2 empty), median diff 26 kB / 313 changed
-lines, **87 improved / 73 unchanged / 44 regressed**. Both sides come from the round's own
-recorded commit — only 54 of 239 runs have `HEAD` at their last round.
+recorded corpus, re-exported through the shipped CLI on 2026-09-03 (`bench/out`, 21
+batteries): **254 rows from 87 runs across 13 batteries**, the only drops being 54
+cross-battery symlink aliases (`duplicate_run`), median diff 26.2 kB over 2 changed files,
+**112 improved / 81 unchanged / 59 regressed / 2 unscored**, median gain on an improved
+round +0.197, 119 rows carrying gate ERRORs. Both sides come from the round's own recorded
+commit, not from `HEAD`. (The same export read 205 rows on 2026-09-02; the corpus grows
+with every battery, so the selector is the date.)
`toolkits/llamafactory/build_refine_sft.py` turns the improved transitions into the message
-shape the finetune pipeline consumes: **87 samples** (71 articulated, 16 static; median answer 26.6 kB, median
-brief 28.6 kB), answered with the files the round produced in the `=== FILE: path ===`
-envelope. This is the one thing the harness produces that a one-shot corpus cannot: *given
+shape the finetune pipeline consumes: **112 samples** (92 articulated, 20 static; median
+answer 25.4 kB, median brief 28.2 kB), answered with the files the round produced in the
+`=== FILE: path ===` envelope. This is the one thing the harness produces that a one-shot corpus cannot: *given
a judged, gated object and a list of what is wrong with it, write the corrected files.*
The pre-existing `flywheel/pairs.py` saw 57 % fewer transitions and carried neither the
@@ -380,7 +383,7 @@ instructions nor a diff.
| base64 amplification | 261 k tokens vs 516 for one 275 kB sheet | measured on a recorded blob + vendor bundle | arithmetic + matching recorded prompt jumps |
| token growth in turns | input(t) = 12 364 + 1 822·t; total ∝ n^1.60; dollars ∝ n^1.19–1.34 | 1 063 in-process sessions, 30 401 turns | the in-process agent, since gemini-cli emits no per-turn usage |
| coupled mechanisms declared | 8 of 14 prompts; poses drive 1–2 joints (was 5–6) | `wave2_lean` URDFs and round records | — |
-| refine corpus | 205 transitions, 87 SFT samples | `3dcv flywheel refine`, `toolkits/llamafactory/build_refine_sft.py` over `bench/out` | corpus grows with every battery |
+| refine corpus | 254 transitions, 112 SFT samples (2026-09-03) | `3dcv flywheel refine`, `toolkits/llamafactory/build_refine_sft.py` over `bench/out` | corpus grows with every battery |
## 9.2 Open questions
diff --git a/toolkits/llamafactory/build_refine_sft.py b/toolkits/llamafactory/build_refine_sft.py
new file mode 100644
index 00000000..c6a2bdee
--- /dev/null
+++ b/toolkits/llamafactory/build_refine_sft.py
@@ -0,0 +1,130 @@
+"""Harness refine transitions -> messages-jsonl for the SFT pipeline.
+
+The harness exports what only it knows (`3dcv flywheel refine --with-code`): for every
+refine round, the previous round's judge issues and gate errors, the tasks compiled for
+the round, the src/ diff between the two recorded commits, the score delta, and the files
+on both sides. This turns the rounds that IMPROVED into the three-message shape
+`to_llamafactory.py` consumes, which is the one supervision signal a one-shot corpus
+cannot carry: given a judged, gated object and a list of what is wrong with it, write the
+corrected files.
+
+ 3dcv flywheel refine bench/out refine.jsonl --with-code # in the harness
+ python build_refine_sft.py refine.jsonl --out sft/refine.jsonl --summary
+ python to_llamafactory.py --data_dir sft --name 3dcv_refine
+
+The answer is every file the round changed, in the `=== FILE: path ===` envelope the
+harness's own parser reads, so a trained model's output can be fed straight back into it.
+"""
+
+import argparse
+import json
+import statistics
+from collections import Counter
+from pathlib import Path
+
+#: an answer longer than this is dropped rather than truncated: half a file teaches a
+#: model to write half a file
+MAX_ANSWER_CHARS = 120_000
+#: outcomes worth training on. A regressed round is evidence about the loop, not an
+#: answer to copy; it stays in the harness export.
+KEEP_OUTCOMES = ("improved",)
+SYSTEM = ("You are an expert {language} author. You are given a 3D artifact that already builds, the "
+ "problems a judge and a set of deterministic geometric gates found in it, and the code as it "
+ "stands. Return corrected files.")
+
+
+def envelope(files):
+ """`=== FILE: path === ... === END FILE ===`, the harness's multi-file answer format."""
+ return "\n".join(f"=== FILE: {p} ===\n{files[p].rstrip()}\n=== END FILE ===" for p in sorted(files))
+
+
+def bullets(items, keys, limit=12):
+ out = []
+ for it in items[:limit]:
+ line = " — ".join(str(it.get(k, "")).strip() for k in keys if str(it.get(k, "")).strip())
+ if line:
+ out.append(f"- {line}")
+ return out
+
+
+def user_turn(row):
+ """The brief: the request, what was wrong, what to change, and the code it started from."""
+ lines = [f"Refine this {row['track'].replace('_', ' ')} written in {row['language']}.", "",
+ "## Request", row["prompt"].strip(), ""]
+ if row.get("gate_errors"):
+ lines += ["## Gate errors (measured by the harness — fix these first)",
+ *bullets(row["gate_errors"], ("target", "message", "fix_hint")), ""]
+ if row.get("issues"):
+ lines += ["## What the judge saw", *bullets(row["issues"], ("target", "severity", "detail")), ""]
+ if row.get("instructions"):
+ lines += ["## Changes to make", *(f"{i}. {ln}" for i, ln in enumerate(row["instructions"], 1)), ""]
+ lines += ["## Current files", envelope(row["before_files"]), "",
+ "Return every file you change in full, in the same `=== FILE: path ===` envelope."]
+ return "\n".join(lines)
+
+
+def sample(row, max_answer_chars=MAX_ANSWER_CHARS):
+ """One SFT row, or `(None, reason)`."""
+ after = row.get("after_files") or {}
+ if not after:
+ return None, "no_code" # exported without --with-code, or nothing readable changed
+ if not row.get("before_files"):
+ return None, "no_before_code"
+ answer = envelope(after)
+ if len(answer) > max_answer_chars:
+ return None, "answer_too_long"
+ return {"id": f"{row['battery']}/{row['run']}/r{row['round']:02d}",
+ "track": row["track"], "language": row["language"], "prompt_id": row["prompt_id"],
+ "score_before": row["score_before"], "score_after": row["score_after"],
+ "score_delta": row["score_delta"], "changed_files": row["changed_files"],
+ "messages": [{"role": "system", "content": SYSTEM.format(language=row["language"])},
+ {"role": "user", "content": user_turn(row)},
+ {"role": "assistant", "content": answer}]}, ""
+
+
+def main(argv=None):
+ ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
+ ap.add_argument("transitions", type=Path, help="JSONL from `3dcv flywheel refine --with-code`")
+ ap.add_argument("--out", type=Path, default=None)
+ ap.add_argument("--outcomes", default=",".join(KEEP_OUTCOMES))
+ ap.add_argument("--max-answer-chars", type=int, default=MAX_ANSWER_CHARS)
+ ap.add_argument("--summary", action="store_true")
+ ns = ap.parse_args(argv)
+
+ keep = {x for x in ns.outcomes.split(",") if x}
+ rows, drops = [], Counter()
+ for line in ns.transitions.read_text().splitlines():
+ if not line.strip():
+ continue
+ row = json.loads(line)
+ if row["outcome"] not in keep:
+ drops[f"outcome:{row['outcome']}"] += 1
+ continue
+ s, reason = sample(row, ns.max_answer_chars)
+ if s is None:
+ drops[reason] += 1
+ else:
+ rows.append(s)
+ if ns.out:
+ ns.out.parent.mkdir(parents=True, exist_ok=True)
+ tmp = ns.out.with_suffix(ns.out.suffix + ".part")
+ tmp.write_text("".join(json.dumps(r) + "\n" for r in rows))
+ tmp.replace(ns.out)
+ print(f"wrote {ns.out}")
+ print(f"{len(rows)} sample(s) from {ns.transitions}")
+ if ns.summary:
+ if drops:
+ print("not exported: " + ", ".join(f"{k} {v}" for k, v in sorted(drops.items())))
+ if rows:
+ ans = [len(r["messages"][2]["content"]) for r in rows]
+ brief = [len(r["messages"][1]["content"]) for r in rows]
+ print(f"answer chars: median {int(statistics.median(ans))}, max {max(ans)}; "
+ f"brief chars: median {int(statistics.median(brief))}, max {max(brief)}")
+ for field in ("track", "language"):
+ print(f"by {field}: " + ", ".join(f"{k} {v}" for k, v in
+ sorted(Counter(r[field] for r in rows).items())))
+ return 0 if rows else 1
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())