Official Codebase of 3DCodeVerse
Python
flywheel: the dataset ships exactly what export selected
gaoypeng83cd8a26a229
diff --git a/harness/codeverse/cli/flywheel_cmd.py b/harness/codeverse/cli/flywheel_cmd.pyindex 7f9222c9..12b4789e 100644--- a/harness/codeverse/cli/flywheel_cmd.py+++ b/harness/codeverse/cli/flywheel_cmd.py@@ -24,9 +24,10 @@ def export_cmd(tar_prefix: Annotated[str, typer.Option("--tar-prefix", help="repo-root-relative prefix for the tar column")] = "",include_unbuilt: Annotated[bool, typer.Option("--include-unbuilt", help="also export runs whose best round never built")] = False,captions_dir: Annotated[Path | None, typer.Option("--captions-dir", help="side-car captions written by `caption --out`")] = None,- drop_duplicates: Annotated[bool, typer.Option("--drop-duplicates", help="leave exact duplicates (code fingerprint + prompt) out of the index")] = False,+ drop_duplicates: Annotated[bool, typer.Option("--drop-duplicates", help="leave normalised duplicates (normalised-code fingerprint + prompt) out of the index, manifest and tars (recorded under the manifest's dropped)")] = False,) -> None:- """Export runs → sample folders + metadata.jsonl/.parquet (+ optional plain tars)."""+ """Export runs → sample folders + dataset_manifest.json + metadata.jsonl/.parquet+ (+ optional plain tars, packed FROM the manifest)."""from codeverse.flywheel.export import export_samplesif pack:@@ -45,7 +46,8 @@ def export_cmd(tiers = " ".join(f"{t}:{rep.tiers.get(t, 0)}" for t in "ABCD")console.print(kv_table("export", {"runs": rep.n_runs, "exported": rep.n_exported, "indexed": rep.n_indexed,"duplicates": rep.n_duplicates, "tiers": tiers,- "skipped": len(rep.skipped), "jsonl": rep.jsonl,+ "skipped": len(rep.skipped), "manifest": rep.manifest,+ "jsonl": rep.jsonl,"parquet": rep.parquet or "(skipped: no pyarrow)"}))for note in rep.notes:warn(escape(note))@@ -154,7 +156,9 @@ def dedupe_cmd()items = []- for meta_path in sorted(dataset_dir.rglob("meta.json")):+ # exactly track/language/key deep — a src/meta.json in a sample's LLM-written+ # code tree must not become a phantom sample+ for meta_path in sorted(dataset_dir.glob("*/*/*/meta.json")):sdir = meta_path.parentmeta = json.loads(meta_path.read_text())files = {p.relative_to(sdir).as_posix(): p.read_bytes() for p in (sdir / "src").rglob("*") if p.is_file()}diff --git a/harness/codeverse/flywheel/dedupe.py b/harness/codeverse/flywheel/dedupe.pyindex 306dd81d..898463eb 100644--- a/harness/codeverse/flywheel/dedupe.py+++ b/harness/codeverse/flywheel/dedupe.py@@ -1,6 +1,7 @@"""Near-duplicate detection: code fingerprints + coarse mesh fingerprints.* ``code_fingerprint(files)`` sha256 of whitespace/comment-normalised code.+* ``code_sha256(files)`` sha256 of the raw bytes (the exact-content hash).* ``mesh_fingerprint(glb)`` bbox extents (cm), triangle bucket and a 16³ voxeloccupancy set computed with trimesh — compare with Jaccard.* ``near_duplicates(items)`` union-find groups: identical code fingerprint OR@@ -49,6 +50,21 @@ def code_fingerprint(files: Mapping[str, str | bytes]) -> str:return h.hexdigest()+def code_sha256(files: Mapping[str, str | bytes]) -> str:+ """Order-independent sha256 over the RAW ``(path, bytes)`` pairs — the+ exact-content counterpart of :func:`code_fingerprint` (no normalisation,+ no lossy decode)."""+ h = hashlib.sha256()+ for path in sorted(files):+ data = files[path]+ raw = data.encode("utf-8") if isinstance(data, str) else data+ h.update(path.encode())+ h.update(b"\0")+ h.update(raw)+ h.update(b"\0")+ return h.hexdigest()++class MeshFingerprint(BaseModel):extents_cm: tuple[int, int, int]tri_count: int@@ -107,7 +123,7 @@ class DedupeItem(BaseModel):def near_duplicates(items: Iterable[DedupeItem], *, mesh_threshold: float = 0.9) -> list[list[str]]:- """Groups (size ≥ 2) of ids that are exact-code duplicates or near-identical meshes."""+ """Groups (size ≥ 2) of ids that are normalised-code duplicates or near-identical meshes."""items = list(items)parent = {it.id: it.id for it in items}diff --git a/harness/codeverse/flywheel/export.py b/harness/codeverse/flywheel/export.pyindex 3765c8ec..2022239b 100644--- a/harness/codeverse/flywheel/export.py+++ b/harness/codeverse/flywheel/export.py@@ -1,9 +1,11 @@"""Export run records → dataset sample folders + ``metadata.parquet``.``export_samples(runs_dir, out_dir)`` writes one sample folder per run (see-``sample.py`` for the layout) and then rebuilds the text index from *every*-``meta.json`` under ``out_dir`` (so repeated exports into the same folder stay-consistent):+``sample.py`` for the layout), records every file it wrote — sha256 each — in+``<out>/dataset_manifest.json`` (``flywheel/manifest.py``) and builds the text+index FROM those manifest entries, never from a directory rescan (so repeated+exports into the same folder stay consistent and nothing the selection did not+choose can leak into the index):* ``<out>/metadata.parquet`` — STORAGE_RULES §4 columns (``id, key, name,captions{detailed,instruction,factory}, meta_json, code, tar, byte_start,@@ -14,6 +16,9 @@ consistent):dedupe pass (``""`` = canonical row).* ``<out>/metadata.jsonl`` — same rows, one JSON object per line.* ``<out>/duplicates.json`` — the duplicate groups found by the last export.+* ``<out>/dataset_manifest.json`` — the generation manifest: the filters, one+ hashed entry per exported sample, and every dropped run with its reason+ (``duplicate_of:<id>`` for de-duplicated rows). ``pack.py`` consumes ONLY this.Captions come from ``record.extra["captions"]``, else ``<ws>/captions.json``,else ``<captions_dir>/<slug>.json`` (side-car written by ``3dcv flywheel caption --out``).@@ -21,17 +26,26 @@ else ``<captions_dir>/<slug>.json`` (side-car written by ``3dcv flywheel captionfrom __future__ import annotations+import hashlibimport jsonimport loggingimport shutilfrom pathlib import Path-from typing import Any+from typing import Any, NamedTuplefrom pydantic import BaseModel, Fieldfrom codeverse.contracts.common import ENTRY_FILE, Languagefrom codeverse.contracts.run import RunId, RunRecordfrom codeverse.flywheel import sample as S+from codeverse.flywheel.dedupe import code_sha256+from codeverse.flywheel.manifest import (+ DatasetGenerationManifest,+ DroppedRun,+ ManifestEntry,+ ManifestFilters,+ write_manifest,+)from codeverse.flywheel.quality import DuplicateGroup, mark_duplicatesfrom codeverse.flywheel.record import FoundRun, iter_runsfrom codeverse.workspace import Workspace@@ -55,6 +69,7 @@ class ExportReport(BaseModel):tiers: dict[str, int] = Field(default_factory=dict, description="quality tier → count (indexed rows)")parquet: str = ""jsonl: str = ""+ manifest: str = ""notes: list[str] = Field(default_factory=list,description="non-fatal degradations, e.g. parquet skipped for a missing extra")@@ -89,11 +104,29 @@ def _caption_texts(caps: dict[str, Any]) -> dict[str, str]:return {k: str(caps.get(k, "") or "") for k in CAPTION_KEYS} if caps.get("detailed") else {}+class ExportedSample(NamedTuple):+ """What :func:`export_one` wrote: the sample dir plus the hashes the manifest needs."""++ dest: Path+ file_hashes: dict[str, str]+ code_sha256: str+ meta: S.SampleMeta+++def _sha256_file(path: Path) -> str:+ h = hashlib.sha256()+ with path.open("rb") as fh:+ for chunk in iter(lambda: fh.read(1 << 20), b""):+ h.update(chunk)+ return h.hexdigest()++def export_one(ws: Workspace, record: RunRecord, out_dir: Path, *, overwrite: bool = True,captions_dir: Path | None = None, run_id: RunId | None = None-) -> Path:- """Write the sample folder for one run; returns the sample dir.+) -> ExportedSample:+ """Write the sample folder for one run; returns the dir plus a sha256 per file+ written (every byte in the folder passes through here, so it is hashed here).``run_id`` carries the collision-safe identity minted at discovery time; withoutone the key falls back to the directory basename (flat layouts only)."""@@ -124,7 +157,9 @@ def export_one(tmp = dest / "meta.json.tmp"tmp.write_text(meta.model_dump_json(indent=2))tmp.replace(dest / "meta.json")- return dest+ hashes = {rel: _sha256_file(dest / rel) for rel in all_files}+ return ExportedSample(dest=dest, file_hashes=hashes,+ code_sha256=code_sha256(files), meta=meta)def export_samples(@@ -145,9 +180,11 @@ def export_samples(API stability); the best round is chosen by ``record.best_round`` or thehighest judged score. Runs whose best round never built are skipped unless``include_unbuilt`` (failed runs are still useful for repair pairs, not as- dataset samples). Exact duplicates (same code fingerprint + prompt) are- marked in the index (``duplicate_of``) and, with ``drop_duplicates``, left- out of it (their folders stay on disk).+ dataset samples). Normalised duplicates (same+ ``dedupe.code_fingerprint`` — a hash of whitespace/comment-normalised code —+ plus the same prompt) are marked in the index (``duplicate_of``) and, with+ ``drop_duplicates``, left out of it AND of the manifest entries, recorded+ under ``manifest.dropped`` as ``duplicate_of:<id>`` (folders stay on disk)."""if not best_round:raise ValueError("export_samples: only best_round=True is supported")@@ -172,40 +209,68 @@ def export_samples(f"duplicate sample id {rel!r}: {prev} and {ws.root} would export onto each other")dest_owner[rel] = ws.root+ entries: list[ManifestEntry] = []+ dropped: list[DroppedRun] = []++ def _skip(ws_root: Path, rid: RunId, reason: str) -> None:+ rep.skipped[str(ws_root)] = reason+ dropped.append(DroppedRun(run=rid, reason=reason))+for ws, rec, rid in found:rep.n_runs += 1rnd = S.best_round_record(rec)j = S.effective_judgment(rnd) if rnd is not None else Nonescore = j.overall if j is not None else Noneif rnd is None:- rep.skipped[str(ws.root)] = "no rounds"+ _skip(ws.root, rid, "no rounds")continueif not include_unbuilt and not (rnd.build is not None and rnd.build.ok):- rep.skipped[str(ws.root)] = "best round did not build"+ _skip(ws.root, rid, "best round did not build")continueif only_passed and not (j is not None and j.passed):- rep.skipped[str(ws.root)] = "not passed"+ _skip(ws.root, rid, "not passed")continueif min_score is not None and (score is None or score < min_score):- rep.skipped[str(ws.root)] = f"score {score} < min_score {min_score}"+ _skip(ws.root, rid, f"score {score} < min_score {min_score}")continuetry:- dest = export_one(ws, rec, out, overwrite=overwrite,- captions_dir=Path(captions_dir) if captions_dir else None, run_id=rid)+ exported = export_one(ws, rec, out, overwrite=overwrite,+ captions_dir=Path(captions_dir) if captions_dir else None, run_id=rid)except S.SampleError as e:- rep.skipped[str(ws.root)] = str(e)+ _skip(ws.root, rid, str(e))continueexcept Exception as e: # git errors etc. — report, keep going- rep.skipped[str(ws.root)] = f"{type(e).__name__}: {e}"+ _skip(ws.root, rid, f"{type(e).__name__}: {e}")continuerep.n_exported += 1- rep.exported.append(str(dest.relative_to(out)))- rows = collect_rows(out)+ rel_dir = exported.dest.relative_to(out).as_posix()+ rep.exported.append(rel_dir)+ entries.append(ManifestEntry(+ run=rid, sample_id=exported.meta.id, sample_rel_dir=rel_dir,+ files=exported.file_hashes, code_fingerprint=exported.meta.code_fingerprint,+ code_sha256=exported.code_sha256))+ # rows come from the manifest's own selection, never a rescan — a rescan is how+ # dropped duplicates used to ship and how a planted src/meta.json became a row+ rows = [row_for_sample(out / e.sample_rel_dir) for e in entries]rep.duplicates = mark_duplicates(rows)rep.n_duplicates = sum(len(g.duplicates) for g in rep.duplicates)(out / DUPLICATES_NAME).write_text(json.dumps([g.model_dump() for g in rep.duplicates], indent=2))if drop_duplicates:+ dup_of = {str(r["id"]): str(r["duplicate_of"]) for r in rows if r["duplicate_of"]}rows = [r for r in rows if not r["duplicate_of"]]+ kept: list[ManifestEntry] = []+ for e in entries:+ canonical = dup_of.get(e.sample_id)+ if canonical:+ dropped.append(DroppedRun(run=e.run, reason=f"duplicate_of:{canonical}"))+ else:+ kept.append(e)+ entries = kept+ rep.manifest = str(write_manifest(DatasetGenerationManifest(+ filters=ManifestFilters(min_score=min_score, only_passed=only_passed,+ include_unbuilt=include_unbuilt, drop_duplicates=drop_duplicates,+ best_round=best_round, captions_dir=str(captions_dir or "")),+ entries=entries, dropped=dropped), out))rep.n_indexed = len(rows)for r in rows:rep.tiers[r["quality_tier"]] = rep.tiers.get(r["quality_tier"], 0) + 1@@ -266,8 +331,12 @@ def row_for_sample(sample_dir: Path) -> dict[str, Any]:def collect_rows(out_dir: Path) -> list[dict[str, Any]]:+ """Recovery/debug rescan of a dataset folder — export/pack build their rows from+ the manifest, never from this. Exactly three levels deep (track/language/key,+ the shape ``sample_rel_dir`` has), so an LLM-written ``src/meta.json`` inside a+ sample's code tree can never inject a phantom row."""rows = []- for meta in sorted(out_dir.rglob("meta.json")):+ for meta in sorted(out_dir.glob("*/*/*/meta.json")):try:rows.append(row_for_sample(meta.parent))except (OSError, ValueError, KeyError) as e:diff --git a/harness/codeverse/flywheel/manifest.py b/harness/codeverse/flywheel/manifest.pynew file mode 100644index 00000000..20a6b01d--- /dev/null+++ b/harness/codeverse/flywheel/manifest.py@@ -0,0 +1,90 @@+"""The dataset generation manifest: what export selected, dropped, and wrote.++``export_samples`` writes ``<out>/dataset_manifest.json`` from its OWN selection+loop — every sample folder it wrote, with a sha256 per file — and ``pack_samples``+consumes ONLY this manifest. No directory rescan can resurrect a duplicate that+``--drop-duplicates`` removed, and a planted ``src/meta.json`` inside a sample's+LLM-written code tree can never become a phantom row. Nothing is dropped+silently: every run the selection refused (filters, export errors, de-dup) is+recorded under ``dropped`` with its reason.+"""++from __future__ import annotations++import json+from datetime import UTC, datetime+from pathlib import Path++from pydantic import BaseModel, Field++from codeverse.contracts.run import RunId++MANIFEST_NAME = "dataset_manifest.json"+++class ManifestError(RuntimeError):+ """The manifest is missing or unreadable."""+++class ManifestFilters(BaseModel):+ """The selection knobs the export ran with (provenance for the dataset)."""++ min_score: float | None = None+ only_passed: bool = False+ include_unbuilt: bool = False+ drop_duplicates: bool = False+ best_round: bool = True+ captions_dir: str = ""+++class ManifestEntry(BaseModel):+ """One exported sample folder, hashed file by file at write time."""++ run: RunId+ sample_id: str+ sample_rel_dir: str = Field(description="sample folder relative to the dataset root (posix)")+ files: dict[str, str] = Field(+ description="sample-relative file path → sha256 of the bytes the export wrote")+ code_fingerprint: str = Field(+ default="", description="sha256 of the NORMALISED src tree (dedupe.code_fingerprint)")+ code_sha256: str = Field(+ default="", description="sha256 of the RAW src tree bytes (dedupe.code_sha256 — the exact hash)")+++class DroppedRun(BaseModel):+ """A run the selection refused — nothing is dropped silently."""++ run: RunId+ reason: str = Field(description='filter reason, export error, or "duplicate_of:<sample id>"')+++class DatasetGenerationManifest(BaseModel):+ generated_at: datetime | None = Field(+ default=None, description="stamped at write time when not passed in")+ filters: ManifestFilters = Field(default_factory=ManifestFilters)+ entries: list[ManifestEntry] = Field(default_factory=list)+ dropped: list[DroppedRun] = Field(default_factory=list)+++def write_manifest(manifest: DatasetGenerationManifest, out_dir: Path | str) -> Path:+ """Atomic write (tmp + replace) of ``<out_dir>/dataset_manifest.json``."""+ if manifest.generated_at is None:+ manifest = manifest.model_copy(update={"generated_at": datetime.now(UTC)})+ path = Path(out_dir) / MANIFEST_NAME+ tmp = path.with_suffix(".json.tmp")+ tmp.write_text(manifest.model_dump_json(indent=2))+ tmp.replace(path)+ return path+++def load_manifest(out_dir: Path | str) -> DatasetGenerationManifest:+ """Read ``<out_dir>/dataset_manifest.json``; missing or invalid is a hard error."""+ path = Path(out_dir) / MANIFEST_NAME+ if not path.is_file():+ raise ManifestError(+ f"{path} not found — run export first (3dcv flywheel export writes it): "+ f"pack no longer rescans directories")+ try:+ return DatasetGenerationManifest.model_validate_json(path.read_text())+ except (ValueError, json.JSONDecodeError) as e:+ raise ManifestError(f"{path} is not a valid dataset manifest: {e}") from ediff --git a/harness/codeverse/flywheel/pack.py b/harness/codeverse/flywheel/pack.pyindex ac6fc851..3a9ef3ff 100644--- a/harness/codeverse/flywheel/pack.py+++ b/harness/codeverse/flywheel/pack.py@@ -1,14 +1,23 @@-"""Pack exported sample folders into plain ``samples-NNN.tar`` files and fill+"""Pack the manifest's sample folders into plain ``samples-NNN.tar`` files and fillthe ``tar / byte_start / byte_len / n_files`` locator columns (STORAGE_RULES §3-4).+``pack_samples`` consumes ONLY ``<out>/dataset_manifest.json`` (written by+``export_samples``) — no directory rescan, so a duplicate dropped at export can+never ship and nothing outside the manifest reaches an archive. Every file is+sha256-verified against the manifest as it is added (fail fast, naming the file);+each archive is written as ``samples-NNN.tar.tmp`` and moved into place only+after the WHOLE pack loop succeeded, so a failed re-pack leaves the previous+archives and index untouched.+Each sample's files are written consecutively so ``byte_start..+byte_len`` is avalid sub-tar; ``verify_locators`` round-trips every row."""from __future__ import annotations+import hashlibimport io-import json+import osimport tarfilefrom pathlib import Pathfrom typing import Any@@ -18,13 +27,16 @@ from pydantic import BaseModel, Fieldfrom codeverse.flywheel.export import (JSONL_NAME,PARQUET_NAME,- collect_rows,+ row_for_sample,write_jsonl,write_parquet,)+from codeverse.flywheel.manifest import ManifestEntry, ManifestError, load_manifestfrom codeverse.flywheel.quality import mark_duplicatesMAX_TAR_BYTES = int(2.5 * 1024**3)+#: per-member allowance for tar headers/padding in the rollover size estimate+TAR_HEADER_SLACK = 1024class PackError(RuntimeError):@@ -37,53 +49,98 @@ class PackReport(BaseModel):parquet: str = ""-def _sample_dir_for(out_dir: Path, meta: dict[str, Any]) -> Path:- return out_dir / meta["track"] / meta["language"] / meta["key"]+def _entry_sizes(sdir: Path, entry: ManifestEntry, rels: list[str]) -> int:+ size = 0+ for rel in rels:+ try:+ size += (sdir / rel).stat().st_size+ except OSError as e:+ raise PackError(f"{entry.sample_id}: missing file {rel} ({e}) — re-run export") from None+ return size + TAR_HEADER_SLACK * (len(rels) + 1)def pack_samples(out_dir: Path | str, *, tar_prefix: str = "", max_tar_bytes: int = MAX_TAR_BYTES) -> PackReport:- """Write ``samples-NNN.tar`` under ``out_dir`` and rewrite the index with locators.+ """Write ``samples-NNN.tar`` under ``out_dir`` from the dataset manifest and+ rewrite the index (rows rebuilt from the manifest's sample dirs, locators filled).``tar_prefix`` is prepended to the tar file name in the ``tar`` column so itcan be repo-root-relative (e.g. ``"3dcodeverse/static_object/"``)."""out = Path(out_dir)- rows = collect_rows(out)- if not rows:- raise PackError(f"no samples under {out}")+ try:+ manifest = load_manifest(out)+ except ManifestError as e:+ raise PackError(str(e)) from None+ if not manifest.entries:+ raise PackError(f"the manifest under {out} lists no samples (all filtered or dropped)")+ rows: list[dict[str, Any]] = []+ for entry in manifest.entries:+ sdir = out / entry.sample_rel_dir+ try:+ rows.append(row_for_sample(sdir))+ except (OSError, ValueError, KeyError) as e:+ raise PackError(f"{entry.sample_id}: unreadable sample dir {sdir} ({e}) — re-run export") from emark_duplicates(rows)rep = PackReport()tar_idx = 0tar: tarfile.TarFile | None = None- tar_path: Path | None = None+ tmp_to_final: list[tuple[Path, Path]] = []+ final_name = ""def _open_new() -> None:- nonlocal tar, tar_path, tar_idx+ nonlocal tar, tar_idx, final_nameif tar is not None:tar.close()- tar_path = out / f"samples-{tar_idx:03d}.tar"- tar = tarfile.open(tar_path, mode="w", format=tarfile.PAX_FORMAT) # noqa: SIM115 - closed in _open_new/after loop- rep.tars.append(tar_path.name)+ final = out / f"samples-{tar_idx:03d}.tar"+ tmp = out / f"samples-{tar_idx:03d}.tar.tmp"+ tar = tarfile.open(tmp, mode="w", format=tarfile.PAX_FORMAT) # noqa: SIM115 - closed in _open_new/finally+ tmp_to_final.append((tmp, final))+ final_name = final.name+ rep.tars.append(final.name)tar_idx += 1- _open_new()- assert tar is not None and tar_path is not None- for row in rows:- meta = json.loads(row["meta_json"])- sdir = _sample_dir_for(out, meta)- files = sorted(p for p in sdir.rglob("*") if p.is_file())- size = sum(p.stat().st_size for p in files) + 1024 * (len(files) + 1)- if tar.offset > 0 and tar.offset + size > max_tar_bytes:- _open_new()- start = tar.offset- for p in files:- tar.add(p, arcname=f"{meta['key']}/{p.relative_to(sdir).as_posix()}", recursive=False)- row["tar"] = f"{tar_prefix}{tar_path.name}"- row["byte_start"] = start- row["byte_len"] = tar.offset - start- row["n_files"] = len(files)- rep.n_samples += 1- tar.close()+ done = False+ try:+ _open_new()+ assert tar is not None+ for entry, row in zip(manifest.entries, rows, strict=True):+ sdir = out / entry.sample_rel_dir+ rels = sorted(entry.files)+ if tar.offset > 0 and tar.offset + _entry_sizes(sdir, entry, rels) > max_tar_bytes:+ _open_new()+ start = tar.offset+ for rel in rels:+ p = sdir / rel+ try:+ data = p.read_bytes()+ except OSError as e:+ raise PackError(f"{entry.sample_id}: missing file {rel} ({e}) — re-run export") from None+ digest = hashlib.sha256(data).hexdigest()+ if digest != entry.files[rel]:+ raise PackError(+ f"{entry.sample_id}: {rel} changed since export (sha256 {digest} != "+ f"manifest {entry.files[rel]}) — re-run export")+ info = tarfile.TarInfo(name=f"{row['key']}/{rel}")+ info.size = len(data)+ info.mtime = int(p.stat().st_mtime)+ info.mode = 0o644+ tar.addfile(info, io.BytesIO(data))+ row["tar"] = f"{tar_prefix}{final_name}"+ row["byte_start"] = start+ row["byte_len"] = tar.offset - start+ row["n_files"] = len(rels)+ rep.n_samples += 1+ tar.close()+ tar = None+ done = True+ finally:+ if not done: # leave the previous archives and index exactly as they were+ if tar is not None:+ tar.close()+ for tmp, _final in tmp_to_final:+ tmp.unlink(missing_ok=True)+ for tmp, final in tmp_to_final:+ os.replace(tmp, final)rep.parquet = str(write_parquet(rows, out / PARQUET_NAME))write_jsonl(rows, out / JSONL_NAME)return repdiff --git a/harness/codeverse/flywheel/quality.py b/harness/codeverse/flywheel/quality.pyindex d7fe8f4c..25844d19 100644--- a/harness/codeverse/flywheel/quality.py+++ b/harness/codeverse/flywheel/quality.py@@ -1,4 +1,4 @@-"""Dataset hygiene: quality tiers + exact-duplicate detection by (code fingerprint, prompt).+"""Dataset hygiene: quality tiers + normalised-duplicate detection by (code fingerprint, prompt).* ``quality_tier(passed, gate_errors, score)`` → ``A | B | C | D``::@@ -61,8 +61,11 @@ def _rank(row: dict[str, Any]) -> tuple:def find_duplicates(rows: Iterable[dict[str, Any]]) -> list[DuplicateGroup]:- """Group rows by exact (code_fingerprint, prompt_hash); only groups with ≥ 2 rows are returned.+ """Group rows by identical (code_fingerprint, prompt_hash); only groups with ≥ 2 rows are returned.+ The fingerprint is of NORMALISED code (``dedupe.normalise_code`` strips comments+ and whitespace), so these are normalised duplicates, not raw-byte-exact ones —+ the raw hash lives in the manifest as ``code_sha256``.Rows without a fingerprint are never grouped. The rows are not modified;use ``mark_duplicates`` to stamp ``duplicate_of``."""diff --git a/harness/codeverse/gallery/compare.py b/harness/codeverse/gallery/compare.pyindex 3e193174..3917ae47 100644--- a/harness/codeverse/gallery/compare.py+++ b/harness/codeverse/gallery/compare.py@@ -151,6 +151,18 @@ def render_compare(entries: list[RunEntry], urls: UrlMaker, *, note: str = "") -return page_shell("compare — 3dcv gallery", body, extra_css=COMPARE_CSS)+#: a leading one of these makes a spreadsheet treat the cell as a formula (CSV injection)+_FORMULA_PREFIXES = ("=", "+", "-", "@")+++def csv_safe(value: object) -> object:+ """Neutralise spreadsheet formula injection: a string cell starting with+ ``= + - @`` (an LLM-written prompt or title can) is prefixed with ``'``."""+ if isinstance(value, str) and value.startswith(_FORMULA_PREFIXES):+ return "'" + value+ return value++CSV_COLUMNS = ("battery", "slug", "title", "verdict", "tier", "track", "language", "generator","judge", "score", "baseline_score", "delta", "gate_errors", "cost_usd", "minutes","rounds", "best_round", "status", "state", "path", "prompt")@@ -164,12 +176,12 @@ def export_csv(entries: list[RunEntry]) -> str:for e in entries:delta = (e.score - e.baseline_scoreif e.score is not None and e.baseline_score is not None else None)- writer.writerow([+ writer.writerow([csv_safe(v) for v in [e.battery, e.slug, e.title, e.verdict, e.tier, e.track, e.language, e.generator,e.judge, "" if e.score is None else f"{e.score:.4f}","" if e.baseline_score is None else f"{e.baseline_score:.4f}","" if delta is None else f"{delta:+.4f}", e.gate_errors, f"{e.cost_usd:.4f}","" if e.minutes is None else f"{e.minutes:.2f}", e.rounds,"" if e.best_round is None else e.best_round, e.status, e.state, e.path, e.prompt,- ])+ ]])return buf.getvalue()diff --git a/harness/codeverse/gallery/server.py b/harness/codeverse/gallery/server.pyindex 11f4990c..13f26395 100644--- a/harness/codeverse/gallery/server.py+++ b/harness/codeverse/gallery/server.py@@ -43,6 +43,11 @@ DEFAULT_HOST = "127.0.0.1"DEFAULT_PORT = 8765LOOPBACK = {"127.0.0.1", "::1", "localhost", "0:0:0:0:0:0:0:1"}STREAM_CHUNK = 256 * 1024+#: /file/ content types a browser would EXECUTE: served with ``CSP: sandbox`` so an+#: LLM-authored HTML/SVG artifact never runs same-origin with the gallery. An+#: ``<img src=….svg>`` embed keeps rendering — image loads create no document, so+#: the sandbox directive never applies to them.+SANDBOXED_TYPES = frozenset({"text/html", "image/svg+xml"})class GalleryError(RuntimeError):@@ -222,7 +227,11 @@ class GalleryApp:return Response.html(code_page.render_dir(entry, self.urls, rel))if not target.is_file():raise FileNotFoundError(f"{entry.slug}/{rel}")- return Response(content_type=content_type(target), path=target)+ ctype = content_type(target)+ headers: dict[str, str] = {}+ if ctype.partition(";")[0].strip() in SANDBOXED_TYPES:+ headers["Content-Security-Policy"] = "sandbox"+ return Response(content_type=ctype, path=target, headers=headers)def _detail(self, entry: RunEntry) -> Response:from codeverse.flywheel.record import RecordError, load_recorddiff --git a/harness/codeverse/gallery/urls.py b/harness/codeverse/gallery/urls.pyindex 0c5a6887..c8a21d6b 100644--- a/harness/codeverse/gallery/urls.py+++ b/harness/codeverse/gallery/urls.py@@ -118,6 +118,8 @@ CONTENT_TYPES: dict[str, str] = {".frag": "text/plain; charset=utf-8",".vert": "text/plain; charset=utf-8",".glsl": "text/plain; charset=utf-8",+ ".html": "text/html; charset=utf-8",+ ".htm": "text/html; charset=utf-8",".md": "text/plain; charset=utf-8",".txt": "text/plain; charset=utf-8",".log": "text/plain; charset=utf-8",diff --git a/harness/tests/flywheel_cli/test_manifest_pack.py b/harness/tests/flywheel_cli/test_manifest_pack.pynew file mode 100644index 00000000..90e50320--- /dev/null+++ b/harness/tests/flywheel_cli/test_manifest_pack.py@@ -0,0 +1,154 @@+"""The dataset generation manifest + the manifest-driven pack.++Export used to filter duplicates only in memory while ``pack_samples`` re-scanned+the directory tree — so ``--drop-duplicates --pack`` shipped every dropped+duplicate, a re-pack truncated the previous valid archive at open, and a planted+``src/meta.json`` (sample folders carry the run's LLM-written src/** tree) could+inject a phantom row. The manifest is now the single source of truth: pack tars+ONLY manifest entries, verifies every byte against the recorded sha256, and+replaces archives atomically after the whole loop succeeded.+"""++from __future__ import annotations++import hashlib+import json+import re+import tarfile+from pathlib import Path++import pytest++from codeverse.flywheel.export import collect_rows, export_samples+from codeverse.flywheel.manifest import (+ MANIFEST_NAME,+ ManifestError,+ load_manifest,+ write_manifest,+)+from codeverse.flywheel.pack import PackError, pack_samples, verify_locators++CHAIR_ID = "3dcodeverse/static_object/blender/wooden_chair_ab12cd34"+++def _tar_member_names(out: Path) -> list[str]:+ names: list[str] = []+ for tp in sorted(out.glob("samples-*.tar")):+ with tarfile.open(tp) as tf:+ names += [m.name for m in tf.getmembers()]+ return names+++# --------------------------------------------------------------------------- manifest+def test_export_writes_a_hashed_manifest(runs_dir: Path, tmp_path: Path):+ out = tmp_path / "ds"+ rep = export_samples(runs_dir, out)+ assert rep.manifest == str(out / MANIFEST_NAME)+ m = load_manifest(out)+ assert m.generated_at is not None+ assert m.filters.drop_duplicates is False and m.filters.min_score is None+ assert len(m.entries) == 3 and m.dropped == []+ e = next(x for x in m.entries if x.sample_id == CHAIR_ID)+ assert e.run.battery == "runs" and e.run.rel == "wooden_chair_ab12cd34"+ assert e.sample_rel_dir == "static_object/blender/wooden_chair_ab12cd34"+ assert {"meta.json", "captions.json", "code.py", "src/model.py"} <= set(e.files)+ for rel, digest in e.files.items(): # every byte the export wrote is hashed+ assert hashlib.sha256((out / e.sample_rel_dir / rel).read_bytes()).hexdigest() == digest+ assert len(e.code_sha256) == 64 and len(e.code_fingerprint) == 64+ assert e.code_sha256 != e.code_fingerprint # raw-exact vs normalised+++def test_filtered_runs_are_recorded_as_dropped(runs_dir: Path, tmp_path: Path):+ export_samples(runs_dir, tmp_path / "ds", min_score=0.75)+ m = load_manifest(tmp_path / "ds")+ assert len(m.entries) == 1 and len(m.dropped) == 2+ assert m.filters.min_score == 0.75+ assert all("min_score" in d.reason for d in m.dropped)+ assert {d.run.rel for d in m.dropped} == {"wooden_chair_codex", "lamp_three"}+++# --------------------------------------------------------------------------- drop-duplicates → pack+def test_drop_duplicates_then_pack_ships_no_duplicate(runs_dir: Path, tmp_path: Path):+ out = tmp_path / "ds"+ # the codex run shares prompt + best-round code with the chair → normalised duplicate+ rep = export_samples(runs_dir, out, drop_duplicates=True)+ assert rep.n_exported == 3 and rep.n_indexed == 2 and rep.n_duplicates == 1+ m = load_manifest(out)+ assert {e.run.rel for e in m.entries} == {"wooden_chair_ab12cd34", "lamp_three"}+ dup = [d for d in m.dropped if d.reason.startswith("duplicate_of:")]+ assert len(dup) == 1 and dup[0].run.rel == "wooden_chair_codex"+ assert dup[0].reason == f"duplicate_of:{CHAIR_ID}"+ # the duplicate FOLDER stays on disk by design — it must still never ship+ assert (out / "static_object" / "blender" / "wooden_chair_codex").is_dir()+ prep = pack_samples(out)+ assert prep.n_samples == 2 and verify_locators(out) == 2+ names = _tar_member_names(out)+ assert names and not any(n.startswith("wooden_chair_codex/") for n in names)+ rows = [json.loads(ln) for ln in (out / "metadata.jsonl").read_text().splitlines()]+ assert {r["key"] for r in rows} == {"wooden_chair_ab12cd34", "lamp_three"}+ assert all(r["tar"] for r in rows)+++# --------------------------------------------------------------------------- pack is manifest-only+def test_pack_without_a_manifest_is_a_hard_error(tmp_path: Path):+ out = tmp_path / "ds"+ (out / "static_object" / "blender" / "x").mkdir(parents=True) # samples alone don't count+ with pytest.raises(PackError, match="no longer rescans"):+ pack_samples(out)+ with pytest.raises(ManifestError, match="dataset_manifest.json"):+ load_manifest(out)+++def test_repack_failure_leaves_the_old_tar_and_index_intact(runs_dir: Path, tmp_path: Path):+ out = tmp_path / "ds"+ export_samples(runs_dir, out)+ pack_samples(out)+ tar_path = out / "samples-000.tar"+ before = {p.name: p.read_bytes() for p in (tar_path, out / "metadata.jsonl", out / "metadata.parquet")}+ # corrupt the recorded hash of a file in the LAST entry: the mismatch fires+ # mid-pack, after earlier samples were already added to the new tmp archive+ m = load_manifest(out)+ victim = m.entries[-1]+ rel = sorted(victim.files)[0]+ victim.files[rel] = "0" * 64+ write_manifest(m, out)+ with pytest.raises(PackError, match=re.escape(rel)):+ pack_samples(out)+ assert tar_path.read_bytes() == before["samples-000.tar"] # old archive untouched+ assert (out / "metadata.jsonl").read_bytes() == before["metadata.jsonl"]+ assert (out / "metadata.parquet").read_bytes() == before["metadata.parquet"]+ assert not list(out.glob("*.tmp")) # the half-written tmp tar is cleaned up+ assert verify_locators(out) == 3 # the previous pack still round-trips+++def test_a_tampered_sample_file_fails_fast_and_names_it(runs_dir: Path, tmp_path: Path):+ out = tmp_path / "ds"+ export_samples(runs_dir, out)+ (out / "static_object" / "blender" / "wooden_chair_ab12cd34" / "code.py").write_text("tampered")+ with pytest.raises(PackError, match="code.py"):+ pack_samples(out)+ assert not list(out.glob("samples-*.tar")) and not list(out.glob("*.tmp"))+++def test_repack_over_an_existing_archive_succeeds_atomically(runs_dir: Path, tmp_path: Path):+ out = tmp_path / "ds"+ export_samples(runs_dir, out)+ assert pack_samples(out).n_samples == 3+ prep2 = pack_samples(out) # re-pack over the existing archive (used to truncate it at open)+ assert prep2.n_samples == 3 and verify_locators(out) == 3+ assert not list(out.glob("*.tmp"))+++# --------------------------------------------------------------------------- phantom rows+def test_planted_src_meta_json_is_not_a_row_and_not_packed(runs_dir: Path, tmp_path: Path):+ out = tmp_path / "ds"+ export_samples(runs_dir, out)+ sdir = out / "static_object" / "blender" / "wooden_chair_ab12cd34"+ (sdir / "src" / "meta.json").write_text(json.dumps({"id": "phantom", "key": "phantom"}))+ rows = collect_rows(out) # the recovery/debug rescan is exactly 3 levels deep+ assert len(rows) == 3 and all(r["id"] != "phantom" for r in rows)+ pack_samples(out) # and pack never looks at the directory tree at all+ assert verify_locators(out) == 3+ assert "wooden_chair_ab12cd34/src/meta.json" not in _tar_member_names(out)+ keys = {json.loads(ln)["key"] for ln in (out / "metadata.jsonl").read_text().splitlines()}+ assert keys == {"wooden_chair_ab12cd34", "wooden_chair_codex", "lamp_three"}diff --git a/harness/tests/gallery/test_hardening.py b/harness/tests/gallery/test_hardening.pynew file mode 100644index 00000000..9a63875d--- /dev/null+++ b/harness/tests/gallery/test_hardening.py@@ -0,0 +1,58 @@+"""Serving LLM-authored HTML/SVG must be inert (CSP sandbox, honest content type),+and CSV cells must never reach a spreadsheet as formulas."""++from __future__ import annotations++import csv+import io+from pathlib import Path++from codeverse.gallery.compare import CSV_COLUMNS, csv_safe, export_csv+from codeverse.gallery.index import build_index+from codeverse.gallery.server import SANDBOXED_TYPES, GalleryApp+from codeverse.gallery.urls import content_type+from tests.flywheel_cli.conftest import make_fake_run+++# --------------------------------------------------------------------------- content types + CSP+def test_html_is_typed_html_not_guessed():+ assert content_type("a.html") == "text/html; charset=utf-8"+ assert content_type("a.htm") == "text/html; charset=utf-8"+ assert content_type("a.svg") == "image/svg+xml"+ assert {"text/html", "image/svg+xml"} == SANDBOXED_TYPES+++def test_llm_html_and_svg_file_responses_are_sandboxed(gallery_tree: dict[str, Path]):+ app = GalleryApp([gallery_tree["runs"]])+ run = gallery_tree["runs"] / "wooden_chair_ab12cd34"+ (run / "artifacts" / "page.html").write_text("<script>fetch('/api/index')</script>")+ (run / "artifacts" / "figure.svg").write_text(+ "<svg xmlns='http://www.w3.org/2000/svg'><script>1</script></svg>")+ html = app.route("/file/runs/wooden_chair_ab12cd34/artifacts/page.html")+ assert html.status == 200 and html.content_type == "text/html; charset=utf-8"+ assert html.headers["Content-Security-Policy"] == "sandbox"+ svg = app.route("/file/runs/wooden_chair_ab12cd34/artifacts/figure.svg")+ assert svg.status == 200 and svg.content_type == "image/svg+xml"+ assert svg.headers["Content-Security-Policy"] == "sandbox"+ # images and code stay untouched — <img src=…> previews keep rendering+ png = app.route("/file/runs/wooden_chair_ab12cd34/artifacts/renders/r01/sheet.png")+ assert png.status == 200 and "Content-Security-Policy" not in png.headers+ src = app.route("/file/runs/wooden_chair_ab12cd34/src/model.py")+ assert src.status == 200 and "Content-Security-Policy" not in src.headers+++# --------------------------------------------------------------------------- csv injection+def test_csv_safe_neutralises_formula_prefixes():+ assert csv_safe("=HYPERLINK(1)") == "'=HYPERLINK(1)"+ assert csv_safe("+1") == "'+1" and csv_safe("-1") == "'-1" and csv_safe("@cmd") == "'@cmd"+ assert csv_safe("a chair") == "a chair"+ assert csv_safe(3) == 3 and csv_safe(None) is None+++def test_a_formula_prompt_round_trips_csv_escaped(tmp_path: Path):+ runs = tmp_path / "runs"+ make_fake_run(runs, "formula_prompt", prompt="=2+5|cmd", scores=(0.5, 0.9))+ text = export_csv(build_index([runs]).entries())+ rows = list(csv.reader(io.StringIO(text)))+ assert rows[0] == list(CSV_COLUMNS) and len(rows) == 2+ assert rows[1][CSV_COLUMNS.index("prompt")] == "'=2+5|cmd"