Skip to content

Official Codebase of 3DCodeVerse

Python

flywheel: the dataset ships exactly what export selected

gaoypeng83cd8a26a229
diff --git a/harness/codeverse/cli/flywheel_cmd.py b/harness/codeverse/cli/flywheel_cmd.py
index 7f9222c9..12b4789e 100644
--- a/harness/codeverse/cli/flywheel_cmd.py
+++ b/harness/codeverse/cli/flywheel_cmd.py
@@ -24,9 +24,10 @@ def export_cmd(
tar_prefix: Annotated[str, typer.Option("--tar-prefix", help="repo-root-relative prefix for the tar column")] = "",
include_unbuilt: Annotated[bool, typer.Option("--include-unbuilt", help="also export runs whose best round never built")] = False,
captions_dir: Annotated[Path | None, typer.Option("--captions-dir", help="side-car captions written by `caption --out`")] = None,
- drop_duplicates: Annotated[bool, typer.Option("--drop-duplicates", help="leave exact duplicates (code fingerprint + prompt) out of the index")] = False,
+ drop_duplicates: Annotated[bool, typer.Option("--drop-duplicates", help="leave normalised duplicates (normalised-code fingerprint + prompt) out of the index, manifest and tars (recorded under the manifest's dropped)")] = False,
) -> None:
- """Export runs → sample folders + metadata.jsonl/.parquet (+ optional plain tars)."""
+ """Export runs → sample folders + dataset_manifest.json + metadata.jsonl/.parquet
+ (+ optional plain tars, packed FROM the manifest)."""
from codeverse.flywheel.export import export_samples
if pack:
@@ -45,7 +46,8 @@ def export_cmd(
tiers = " ".join(f"{t}:{rep.tiers.get(t, 0)}" for t in "ABCD")
console.print(kv_table("export", {"runs": rep.n_runs, "exported": rep.n_exported, "indexed": rep.n_indexed,
"duplicates": rep.n_duplicates, "tiers": tiers,
- "skipped": len(rep.skipped), "jsonl": rep.jsonl,
+ "skipped": len(rep.skipped), "manifest": rep.manifest,
+ "jsonl": rep.jsonl,
"parquet": rep.parquet or "(skipped: no pyarrow)"}))
for note in rep.notes:
warn(escape(note))
@@ -154,7 +156,9 @@ def dedupe_cmd(
)
items = []
- for meta_path in sorted(dataset_dir.rglob("meta.json")):
+ # exactly track/language/key deep — a src/meta.json in a sample's LLM-written
+ # code tree must not become a phantom sample
+ for meta_path in sorted(dataset_dir.glob("*/*/*/meta.json")):
sdir = meta_path.parent
meta = json.loads(meta_path.read_text())
files = {p.relative_to(sdir).as_posix(): p.read_bytes() for p in (sdir / "src").rglob("*") if p.is_file()}
diff --git a/harness/codeverse/flywheel/dedupe.py b/harness/codeverse/flywheel/dedupe.py
index 306dd81d..898463eb 100644
--- a/harness/codeverse/flywheel/dedupe.py
+++ b/harness/codeverse/flywheel/dedupe.py
@@ -1,6 +1,7 @@
"""Near-duplicate detection: code fingerprints + coarse mesh fingerprints.
* ``code_fingerprint(files)`` sha256 of whitespace/comment-normalised code.
+* ``code_sha256(files)`` sha256 of the raw bytes (the exact-content hash).
* ``mesh_fingerprint(glb)`` bbox extents (cm), triangle bucket and a 16³ voxel
occupancy set computed with trimesh — compare with Jaccard.
* ``near_duplicates(items)`` union-find groups: identical code fingerprint OR
@@ -49,6 +50,21 @@ def code_fingerprint(files: Mapping[str, str | bytes]) -> str:
return h.hexdigest()
+def code_sha256(files: Mapping[str, str | bytes]) -> str:
+ """Order-independent sha256 over the RAW ``(path, bytes)`` pairs — the
+ exact-content counterpart of :func:`code_fingerprint` (no normalisation,
+ no lossy decode)."""
+ h = hashlib.sha256()
+ for path in sorted(files):
+ data = files[path]
+ raw = data.encode("utf-8") if isinstance(data, str) else data
+ h.update(path.encode())
+ h.update(b"\0")
+ h.update(raw)
+ h.update(b"\0")
+ return h.hexdigest()
+
+
class MeshFingerprint(BaseModel):
extents_cm: tuple[int, int, int]
tri_count: int
@@ -107,7 +123,7 @@ class DedupeItem(BaseModel):
def near_duplicates(items: Iterable[DedupeItem], *, mesh_threshold: float = 0.9) -> list[list[str]]:
- """Groups (size ≥ 2) of ids that are exact-code duplicates or near-identical meshes."""
+ """Groups (size ≥ 2) of ids that are normalised-code duplicates or near-identical meshes."""
items = list(items)
parent = {it.id: it.id for it in items}
diff --git a/harness/codeverse/flywheel/export.py b/harness/codeverse/flywheel/export.py
index 3765c8ec..2022239b 100644
--- a/harness/codeverse/flywheel/export.py
+++ b/harness/codeverse/flywheel/export.py
@@ -1,9 +1,11 @@
"""Export run records → dataset sample folders + ``metadata.parquet``.
``export_samples(runs_dir, out_dir)`` writes one sample folder per run (see
-``sample.py`` for the layout) and then rebuilds the text index from *every*
-``meta.json`` under ``out_dir`` (so repeated exports into the same folder stay
-consistent):
+``sample.py`` for the layout), records every file it wrote — sha256 each — in
+``<out>/dataset_manifest.json`` (``flywheel/manifest.py``) and builds the text
+index FROM those manifest entries, never from a directory rescan (so repeated
+exports into the same folder stay consistent and nothing the selection did not
+choose can leak into the index):
* ``<out>/metadata.parquet`` — STORAGE_RULES §4 columns (``id, key, name,
captions{detailed,instruction,factory}, meta_json, code, tar, byte_start,
@@ -14,6 +16,9 @@ consistent):
dedupe pass (``""`` = canonical row).
* ``<out>/metadata.jsonl`` — same rows, one JSON object per line.
* ``<out>/duplicates.json`` — the duplicate groups found by the last export.
+* ``<out>/dataset_manifest.json`` — the generation manifest: the filters, one
+ hashed entry per exported sample, and every dropped run with its reason
+ (``duplicate_of:<id>`` for de-duplicated rows). ``pack.py`` consumes ONLY this.
Captions come from ``record.extra["captions"]``, else ``<ws>/captions.json``,
else ``<captions_dir>/<slug>.json`` (side-car written by ``3dcv flywheel caption --out``).
@@ -21,17 +26,26 @@ else ``<captions_dir>/<slug>.json`` (side-car written by ``3dcv flywheel caption
from __future__ import annotations
+import hashlib
import json
import logging
import shutil
from pathlib import Path
-from typing import Any
+from typing import Any, NamedTuple
from pydantic import BaseModel, Field
from codeverse.contracts.common import ENTRY_FILE, Language
from codeverse.contracts.run import RunId, RunRecord
from codeverse.flywheel import sample as S
+from codeverse.flywheel.dedupe import code_sha256
+from codeverse.flywheel.manifest import (
+ DatasetGenerationManifest,
+ DroppedRun,
+ ManifestEntry,
+ ManifestFilters,
+ write_manifest,
+)
from codeverse.flywheel.quality import DuplicateGroup, mark_duplicates
from codeverse.flywheel.record import FoundRun, iter_runs
from codeverse.workspace import Workspace
@@ -55,6 +69,7 @@ class ExportReport(BaseModel):
tiers: dict[str, int] = Field(default_factory=dict, description="quality tier → count (indexed rows)")
parquet: str = ""
jsonl: str = ""
+ manifest: str = ""
notes: list[str] = Field(default_factory=list,
description="non-fatal degradations, e.g. parquet skipped for a missing extra")
@@ -89,11 +104,29 @@ def _caption_texts(caps: dict[str, Any]) -> dict[str, str]:
return {k: str(caps.get(k, "") or "") for k in CAPTION_KEYS} if caps.get("detailed") else {}
+class ExportedSample(NamedTuple):
+ """What :func:`export_one` wrote: the sample dir plus the hashes the manifest needs."""
+
+ dest: Path
+ file_hashes: dict[str, str]
+ code_sha256: str
+ meta: S.SampleMeta
+
+
+def _sha256_file(path: Path) -> str:
+ h = hashlib.sha256()
+ with path.open("rb") as fh:
+ for chunk in iter(lambda: fh.read(1 << 20), b""):
+ h.update(chunk)
+ return h.hexdigest()
+
+
def export_one(
ws: Workspace, record: RunRecord, out_dir: Path, *, overwrite: bool = True,
captions_dir: Path | None = None, run_id: RunId | None = None
-) -> Path:
- """Write the sample folder for one run; returns the sample dir.
+) -> ExportedSample:
+ """Write the sample folder for one run; returns the dir plus a sha256 per file
+ written (every byte in the folder passes through here, so it is hashed here).
``run_id`` carries the collision-safe identity minted at discovery time; without
one the key falls back to the directory basename (flat layouts only)."""
@@ -124,7 +157,9 @@ def export_one(
tmp = dest / "meta.json.tmp"
tmp.write_text(meta.model_dump_json(indent=2))
tmp.replace(dest / "meta.json")
- return dest
+ hashes = {rel: _sha256_file(dest / rel) for rel in all_files}
+ return ExportedSample(dest=dest, file_hashes=hashes,
+ code_sha256=code_sha256(files), meta=meta)
def export_samples(
@@ -145,9 +180,11 @@ def export_samples(
API stability); the best round is chosen by ``record.best_round`` or the
highest judged score. Runs whose best round never built are skipped unless
``include_unbuilt`` (failed runs are still useful for repair pairs, not as
- dataset samples). Exact duplicates (same code fingerprint + prompt) are
- marked in the index (``duplicate_of``) and, with ``drop_duplicates``, left
- out of it (their folders stay on disk).
+ dataset samples). Normalised duplicates (same
+ ``dedupe.code_fingerprint`` — a hash of whitespace/comment-normalised code —
+ plus the same prompt) are marked in the index (``duplicate_of``) and, with
+ ``drop_duplicates``, left out of it AND of the manifest entries, recorded
+ under ``manifest.dropped`` as ``duplicate_of:<id>`` (folders stay on disk).
"""
if not best_round:
raise ValueError("export_samples: only best_round=True is supported")
@@ -172,40 +209,68 @@ def export_samples(
f"duplicate sample id {rel!r}: {prev} and {ws.root} would export onto each other")
dest_owner[rel] = ws.root
+ entries: list[ManifestEntry] = []
+ dropped: list[DroppedRun] = []
+
+ def _skip(ws_root: Path, rid: RunId, reason: str) -> None:
+ rep.skipped[str(ws_root)] = reason
+ dropped.append(DroppedRun(run=rid, reason=reason))
+
for ws, rec, rid in found:
rep.n_runs += 1
rnd = S.best_round_record(rec)
j = S.effective_judgment(rnd) if rnd is not None else None
score = j.overall if j is not None else None
if rnd is None:
- rep.skipped[str(ws.root)] = "no rounds"
+ _skip(ws.root, rid, "no rounds")
continue
if not include_unbuilt and not (rnd.build is not None and rnd.build.ok):
- rep.skipped[str(ws.root)] = "best round did not build"
+ _skip(ws.root, rid, "best round did not build")
continue
if only_passed and not (j is not None and j.passed):
- rep.skipped[str(ws.root)] = "not passed"
+ _skip(ws.root, rid, "not passed")
continue
if min_score is not None and (score is None or score < min_score):
- rep.skipped[str(ws.root)] = f"score {score} < min_score {min_score}"
+ _skip(ws.root, rid, f"score {score} < min_score {min_score}")
continue
try:
- dest = export_one(ws, rec, out, overwrite=overwrite,
- captions_dir=Path(captions_dir) if captions_dir else None, run_id=rid)
+ exported = export_one(ws, rec, out, overwrite=overwrite,
+ captions_dir=Path(captions_dir) if captions_dir else None, run_id=rid)
except S.SampleError as e:
- rep.skipped[str(ws.root)] = str(e)
+ _skip(ws.root, rid, str(e))
continue
except Exception as e: # git errors etc. — report, keep going
- rep.skipped[str(ws.root)] = f"{type(e).__name__}: {e}"
+ _skip(ws.root, rid, f"{type(e).__name__}: {e}")
continue
rep.n_exported += 1
- rep.exported.append(str(dest.relative_to(out)))
- rows = collect_rows(out)
+ rel_dir = exported.dest.relative_to(out).as_posix()
+ rep.exported.append(rel_dir)
+ entries.append(ManifestEntry(
+ run=rid, sample_id=exported.meta.id, sample_rel_dir=rel_dir,
+ files=exported.file_hashes, code_fingerprint=exported.meta.code_fingerprint,
+ code_sha256=exported.code_sha256))
+ # rows come from the manifest's own selection, never a rescan — a rescan is how
+ # dropped duplicates used to ship and how a planted src/meta.json became a row
+ rows = [row_for_sample(out / e.sample_rel_dir) for e in entries]
rep.duplicates = mark_duplicates(rows)
rep.n_duplicates = sum(len(g.duplicates) for g in rep.duplicates)
(out / DUPLICATES_NAME).write_text(json.dumps([g.model_dump() for g in rep.duplicates], indent=2))
if drop_duplicates:
+ dup_of = {str(r["id"]): str(r["duplicate_of"]) for r in rows if r["duplicate_of"]}
rows = [r for r in rows if not r["duplicate_of"]]
+ kept: list[ManifestEntry] = []
+ for e in entries:
+ canonical = dup_of.get(e.sample_id)
+ if canonical:
+ dropped.append(DroppedRun(run=e.run, reason=f"duplicate_of:{canonical}"))
+ else:
+ kept.append(e)
+ entries = kept
+ rep.manifest = str(write_manifest(DatasetGenerationManifest(
+ filters=ManifestFilters(min_score=min_score, only_passed=only_passed,
+ include_unbuilt=include_unbuilt, drop_duplicates=drop_duplicates,
+ best_round=best_round, captions_dir=str(captions_dir or "")),
+ entries=entries, dropped=dropped), out))
rep.n_indexed = len(rows)
for r in rows:
rep.tiers[r["quality_tier"]] = rep.tiers.get(r["quality_tier"], 0) + 1
@@ -266,8 +331,12 @@ def row_for_sample(sample_dir: Path) -> dict[str, Any]:
def collect_rows(out_dir: Path) -> list[dict[str, Any]]:
+ """Recovery/debug rescan of a dataset folder — export/pack build their rows from
+ the manifest, never from this. Exactly three levels deep (track/language/key,
+ the shape ``sample_rel_dir`` has), so an LLM-written ``src/meta.json`` inside a
+ sample's code tree can never inject a phantom row."""
rows = []
- for meta in sorted(out_dir.rglob("meta.json")):
+ for meta in sorted(out_dir.glob("*/*/*/meta.json")):
try:
rows.append(row_for_sample(meta.parent))
except (OSError, ValueError, KeyError) as e:
diff --git a/harness/codeverse/flywheel/manifest.py b/harness/codeverse/flywheel/manifest.py
new file mode 100644
index 00000000..20a6b01d
--- /dev/null
+++ b/harness/codeverse/flywheel/manifest.py
@@ -0,0 +1,90 @@
+"""The dataset generation manifest: what export selected, dropped, and wrote.
+
+``export_samples`` writes ``<out>/dataset_manifest.json`` from its OWN selection
+loop — every sample folder it wrote, with a sha256 per file — and ``pack_samples``
+consumes ONLY this manifest. No directory rescan can resurrect a duplicate that
+``--drop-duplicates`` removed, and a planted ``src/meta.json`` inside a sample's
+LLM-written code tree can never become a phantom row. Nothing is dropped
+silently: every run the selection refused (filters, export errors, de-dup) is
+recorded under ``dropped`` with its reason.
+"""
+
+from __future__ import annotations
+
+import json
+from datetime import UTC, datetime
+from pathlib import Path
+
+from pydantic import BaseModel, Field
+
+from codeverse.contracts.run import RunId
+
+MANIFEST_NAME = "dataset_manifest.json"
+
+
+class ManifestError(RuntimeError):
+ """The manifest is missing or unreadable."""
+
+
+class ManifestFilters(BaseModel):
+ """The selection knobs the export ran with (provenance for the dataset)."""
+
+ min_score: float | None = None
+ only_passed: bool = False
+ include_unbuilt: bool = False
+ drop_duplicates: bool = False
+ best_round: bool = True
+ captions_dir: str = ""
+
+
+class ManifestEntry(BaseModel):
+ """One exported sample folder, hashed file by file at write time."""
+
+ run: RunId
+ sample_id: str
+ sample_rel_dir: str = Field(description="sample folder relative to the dataset root (posix)")
+ files: dict[str, str] = Field(
+ description="sample-relative file path → sha256 of the bytes the export wrote")
+ code_fingerprint: str = Field(
+ default="", description="sha256 of the NORMALISED src tree (dedupe.code_fingerprint)")
+ code_sha256: str = Field(
+ default="", description="sha256 of the RAW src tree bytes (dedupe.code_sha256 — the exact hash)")
+
+
+class DroppedRun(BaseModel):
+ """A run the selection refused — nothing is dropped silently."""
+
+ run: RunId
+ reason: str = Field(description='filter reason, export error, or "duplicate_of:<sample id>"')
+
+
+class DatasetGenerationManifest(BaseModel):
+ generated_at: datetime | None = Field(
+ default=None, description="stamped at write time when not passed in")
+ filters: ManifestFilters = Field(default_factory=ManifestFilters)
+ entries: list[ManifestEntry] = Field(default_factory=list)
+ dropped: list[DroppedRun] = Field(default_factory=list)
+
+
+def write_manifest(manifest: DatasetGenerationManifest, out_dir: Path | str) -> Path:
+ """Atomic write (tmp + replace) of ``<out_dir>/dataset_manifest.json``."""
+ if manifest.generated_at is None:
+ manifest = manifest.model_copy(update={"generated_at": datetime.now(UTC)})
+ path = Path(out_dir) / MANIFEST_NAME
+ tmp = path.with_suffix(".json.tmp")
+ tmp.write_text(manifest.model_dump_json(indent=2))
+ tmp.replace(path)
+ return path
+
+
+def load_manifest(out_dir: Path | str) -> DatasetGenerationManifest:
+ """Read ``<out_dir>/dataset_manifest.json``; missing or invalid is a hard error."""
+ path = Path(out_dir) / MANIFEST_NAME
+ if not path.is_file():
+ raise ManifestError(
+ f"{path} not found — run export first (3dcv flywheel export writes it): "
+ f"pack no longer rescans directories")
+ try:
+ return DatasetGenerationManifest.model_validate_json(path.read_text())
+ except (ValueError, json.JSONDecodeError) as e:
+ raise ManifestError(f"{path} is not a valid dataset manifest: {e}") from e
diff --git a/harness/codeverse/flywheel/pack.py b/harness/codeverse/flywheel/pack.py
index ac6fc851..3a9ef3ff 100644
--- a/harness/codeverse/flywheel/pack.py
+++ b/harness/codeverse/flywheel/pack.py
@@ -1,14 +1,23 @@
-"""Pack exported sample folders into plain ``samples-NNN.tar`` files and fill
+"""Pack the manifest's sample folders into plain ``samples-NNN.tar`` files and fill
the ``tar / byte_start / byte_len / n_files`` locator columns (STORAGE_RULES §3-4).
+``pack_samples`` consumes ONLY ``<out>/dataset_manifest.json`` (written by
+``export_samples``) — no directory rescan, so a duplicate dropped at export can
+never ship and nothing outside the manifest reaches an archive. Every file is
+sha256-verified against the manifest as it is added (fail fast, naming the file);
+each archive is written as ``samples-NNN.tar.tmp`` and moved into place only
+after the WHOLE pack loop succeeded, so a failed re-pack leaves the previous
+archives and index untouched.
+
Each sample's files are written consecutively so ``byte_start..+byte_len`` is a
valid sub-tar; ``verify_locators`` round-trips every row.
"""
from __future__ import annotations
+import hashlib
import io
-import json
+import os
import tarfile
from pathlib import Path
from typing import Any
@@ -18,13 +27,16 @@ from pydantic import BaseModel, Field
from codeverse.flywheel.export import (
JSONL_NAME,
PARQUET_NAME,
- collect_rows,
+ row_for_sample,
write_jsonl,
write_parquet,
)
+from codeverse.flywheel.manifest import ManifestEntry, ManifestError, load_manifest
from codeverse.flywheel.quality import mark_duplicates
MAX_TAR_BYTES = int(2.5 * 1024**3)
+#: per-member allowance for tar headers/padding in the rollover size estimate
+TAR_HEADER_SLACK = 1024
class PackError(RuntimeError):
@@ -37,53 +49,98 @@ class PackReport(BaseModel):
parquet: str = ""
-def _sample_dir_for(out_dir: Path, meta: dict[str, Any]) -> Path:
- return out_dir / meta["track"] / meta["language"] / meta["key"]
+def _entry_sizes(sdir: Path, entry: ManifestEntry, rels: list[str]) -> int:
+ size = 0
+ for rel in rels:
+ try:
+ size += (sdir / rel).stat().st_size
+ except OSError as e:
+ raise PackError(f"{entry.sample_id}: missing file {rel} ({e}) — re-run export") from None
+ return size + TAR_HEADER_SLACK * (len(rels) + 1)
def pack_samples(out_dir: Path | str, *, tar_prefix: str = "", max_tar_bytes: int = MAX_TAR_BYTES) -> PackReport:
- """Write ``samples-NNN.tar`` under ``out_dir`` and rewrite the index with locators.
+ """Write ``samples-NNN.tar`` under ``out_dir`` from the dataset manifest and
+ rewrite the index (rows rebuilt from the manifest's sample dirs, locators filled).
``tar_prefix`` is prepended to the tar file name in the ``tar`` column so it
can be repo-root-relative (e.g. ``"3dcodeverse/static_object/"``).
"""
out = Path(out_dir)
- rows = collect_rows(out)
- if not rows:
- raise PackError(f"no samples under {out}")
+ try:
+ manifest = load_manifest(out)
+ except ManifestError as e:
+ raise PackError(str(e)) from None
+ if not manifest.entries:
+ raise PackError(f"the manifest under {out} lists no samples (all filtered or dropped)")
+ rows: list[dict[str, Any]] = []
+ for entry in manifest.entries:
+ sdir = out / entry.sample_rel_dir
+ try:
+ rows.append(row_for_sample(sdir))
+ except (OSError, ValueError, KeyError) as e:
+ raise PackError(f"{entry.sample_id}: unreadable sample dir {sdir} ({e}) — re-run export") from e
mark_duplicates(rows)
rep = PackReport()
tar_idx = 0
tar: tarfile.TarFile | None = None
- tar_path: Path | None = None
+ tmp_to_final: list[tuple[Path, Path]] = []
+ final_name = ""
def _open_new() -> None:
- nonlocal tar, tar_path, tar_idx
+ nonlocal tar, tar_idx, final_name
if tar is not None:
tar.close()
- tar_path = out / f"samples-{tar_idx:03d}.tar"
- tar = tarfile.open(tar_path, mode="w", format=tarfile.PAX_FORMAT) # noqa: SIM115 - closed in _open_new/after loop
- rep.tars.append(tar_path.name)
+ final = out / f"samples-{tar_idx:03d}.tar"
+ tmp = out / f"samples-{tar_idx:03d}.tar.tmp"
+ tar = tarfile.open(tmp, mode="w", format=tarfile.PAX_FORMAT) # noqa: SIM115 - closed in _open_new/finally
+ tmp_to_final.append((tmp, final))
+ final_name = final.name
+ rep.tars.append(final.name)
tar_idx += 1
- _open_new()
- assert tar is not None and tar_path is not None
- for row in rows:
- meta = json.loads(row["meta_json"])
- sdir = _sample_dir_for(out, meta)
- files = sorted(p for p in sdir.rglob("*") if p.is_file())
- size = sum(p.stat().st_size for p in files) + 1024 * (len(files) + 1)
- if tar.offset > 0 and tar.offset + size > max_tar_bytes:
- _open_new()
- start = tar.offset
- for p in files:
- tar.add(p, arcname=f"{meta['key']}/{p.relative_to(sdir).as_posix()}", recursive=False)
- row["tar"] = f"{tar_prefix}{tar_path.name}"
- row["byte_start"] = start
- row["byte_len"] = tar.offset - start
- row["n_files"] = len(files)
- rep.n_samples += 1
- tar.close()
+ done = False
+ try:
+ _open_new()
+ assert tar is not None
+ for entry, row in zip(manifest.entries, rows, strict=True):
+ sdir = out / entry.sample_rel_dir
+ rels = sorted(entry.files)
+ if tar.offset > 0 and tar.offset + _entry_sizes(sdir, entry, rels) > max_tar_bytes:
+ _open_new()
+ start = tar.offset
+ for rel in rels:
+ p = sdir / rel
+ try:
+ data = p.read_bytes()
+ except OSError as e:
+ raise PackError(f"{entry.sample_id}: missing file {rel} ({e}) — re-run export") from None
+ digest = hashlib.sha256(data).hexdigest()
+ if digest != entry.files[rel]:
+ raise PackError(
+ f"{entry.sample_id}: {rel} changed since export (sha256 {digest} != "
+ f"manifest {entry.files[rel]}) — re-run export")
+ info = tarfile.TarInfo(name=f"{row['key']}/{rel}")
+ info.size = len(data)
+ info.mtime = int(p.stat().st_mtime)
+ info.mode = 0o644
+ tar.addfile(info, io.BytesIO(data))
+ row["tar"] = f"{tar_prefix}{final_name}"
+ row["byte_start"] = start
+ row["byte_len"] = tar.offset - start
+ row["n_files"] = len(rels)
+ rep.n_samples += 1
+ tar.close()
+ tar = None
+ done = True
+ finally:
+ if not done: # leave the previous archives and index exactly as they were
+ if tar is not None:
+ tar.close()
+ for tmp, _final in tmp_to_final:
+ tmp.unlink(missing_ok=True)
+ for tmp, final in tmp_to_final:
+ os.replace(tmp, final)
rep.parquet = str(write_parquet(rows, out / PARQUET_NAME))
write_jsonl(rows, out / JSONL_NAME)
return rep
diff --git a/harness/codeverse/flywheel/quality.py b/harness/codeverse/flywheel/quality.py
index d7fe8f4c..25844d19 100644
--- a/harness/codeverse/flywheel/quality.py
+++ b/harness/codeverse/flywheel/quality.py
@@ -1,4 +1,4 @@
-"""Dataset hygiene: quality tiers + exact-duplicate detection by (code fingerprint, prompt).
+"""Dataset hygiene: quality tiers + normalised-duplicate detection by (code fingerprint, prompt).
* ``quality_tier(passed, gate_errors, score)`` → ``A | B | C | D``::
@@ -61,8 +61,11 @@ def _rank(row: dict[str, Any]) -> tuple:
def find_duplicates(rows: Iterable[dict[str, Any]]) -> list[DuplicateGroup]:
- """Group rows by exact (code_fingerprint, prompt_hash); only groups with ≥ 2 rows are returned.
+ """Group rows by identical (code_fingerprint, prompt_hash); only groups with ≥ 2 rows are returned.
+ The fingerprint is of NORMALISED code (``dedupe.normalise_code`` strips comments
+ and whitespace), so these are normalised duplicates, not raw-byte-exact ones —
+ the raw hash lives in the manifest as ``code_sha256``.
Rows without a fingerprint are never grouped. The rows are not modified;
use ``mark_duplicates`` to stamp ``duplicate_of``.
"""
diff --git a/harness/codeverse/gallery/compare.py b/harness/codeverse/gallery/compare.py
index 3e193174..3917ae47 100644
--- a/harness/codeverse/gallery/compare.py
+++ b/harness/codeverse/gallery/compare.py
@@ -151,6 +151,18 @@ def render_compare(entries: list[RunEntry], urls: UrlMaker, *, note: str = "") -
return page_shell("compare — 3dcv gallery", body, extra_css=COMPARE_CSS)
+#: a leading one of these makes a spreadsheet treat the cell as a formula (CSV injection)
+_FORMULA_PREFIXES = ("=", "+", "-", "@")
+
+
+def csv_safe(value: object) -> object:
+ """Neutralise spreadsheet formula injection: a string cell starting with
+ ``= + - @`` (an LLM-written prompt or title can) is prefixed with ``'``."""
+ if isinstance(value, str) and value.startswith(_FORMULA_PREFIXES):
+ return "'" + value
+ return value
+
+
CSV_COLUMNS = ("battery", "slug", "title", "verdict", "tier", "track", "language", "generator",
"judge", "score", "baseline_score", "delta", "gate_errors", "cost_usd", "minutes",
"rounds", "best_round", "status", "state", "path", "prompt")
@@ -164,12 +176,12 @@ def export_csv(entries: list[RunEntry]) -> str:
for e in entries:
delta = (e.score - e.baseline_score
if e.score is not None and e.baseline_score is not None else None)
- writer.writerow([
+ writer.writerow([csv_safe(v) for v in [
e.battery, e.slug, e.title, e.verdict, e.tier, e.track, e.language, e.generator,
e.judge, "" if e.score is None else f"{e.score:.4f}",
"" if e.baseline_score is None else f"{e.baseline_score:.4f}",
"" if delta is None else f"{delta:+.4f}", e.gate_errors, f"{e.cost_usd:.4f}",
"" if e.minutes is None else f"{e.minutes:.2f}", e.rounds,
"" if e.best_round is None else e.best_round, e.status, e.state, e.path, e.prompt,
- ])
+ ]])
return buf.getvalue()
diff --git a/harness/codeverse/gallery/server.py b/harness/codeverse/gallery/server.py
index 11f4990c..13f26395 100644
--- a/harness/codeverse/gallery/server.py
+++ b/harness/codeverse/gallery/server.py
@@ -43,6 +43,11 @@ DEFAULT_HOST = "127.0.0.1"
DEFAULT_PORT = 8765
LOOPBACK = {"127.0.0.1", "::1", "localhost", "0:0:0:0:0:0:0:1"}
STREAM_CHUNK = 256 * 1024
+#: /file/ content types a browser would EXECUTE: served with ``CSP: sandbox`` so an
+#: LLM-authored HTML/SVG artifact never runs same-origin with the gallery. An
+#: ``<img src=….svg>`` embed keeps rendering — image loads create no document, so
+#: the sandbox directive never applies to them.
+SANDBOXED_TYPES = frozenset({"text/html", "image/svg+xml"})
class GalleryError(RuntimeError):
@@ -222,7 +227,11 @@ class GalleryApp:
return Response.html(code_page.render_dir(entry, self.urls, rel))
if not target.is_file():
raise FileNotFoundError(f"{entry.slug}/{rel}")
- return Response(content_type=content_type(target), path=target)
+ ctype = content_type(target)
+ headers: dict[str, str] = {}
+ if ctype.partition(";")[0].strip() in SANDBOXED_TYPES:
+ headers["Content-Security-Policy"] = "sandbox"
+ return Response(content_type=ctype, path=target, headers=headers)
def _detail(self, entry: RunEntry) -> Response:
from codeverse.flywheel.record import RecordError, load_record
diff --git a/harness/codeverse/gallery/urls.py b/harness/codeverse/gallery/urls.py
index 0c5a6887..c8a21d6b 100644
--- a/harness/codeverse/gallery/urls.py
+++ b/harness/codeverse/gallery/urls.py
@@ -118,6 +118,8 @@ CONTENT_TYPES: dict[str, str] = {
".frag": "text/plain; charset=utf-8",
".vert": "text/plain; charset=utf-8",
".glsl": "text/plain; charset=utf-8",
+ ".html": "text/html; charset=utf-8",
+ ".htm": "text/html; charset=utf-8",
".md": "text/plain; charset=utf-8",
".txt": "text/plain; charset=utf-8",
".log": "text/plain; charset=utf-8",
diff --git a/harness/tests/flywheel_cli/test_manifest_pack.py b/harness/tests/flywheel_cli/test_manifest_pack.py
new file mode 100644
index 00000000..90e50320
--- /dev/null
+++ b/harness/tests/flywheel_cli/test_manifest_pack.py
@@ -0,0 +1,154 @@
+"""The dataset generation manifest + the manifest-driven pack.
+
+Export used to filter duplicates only in memory while ``pack_samples`` re-scanned
+the directory tree — so ``--drop-duplicates --pack`` shipped every dropped
+duplicate, a re-pack truncated the previous valid archive at open, and a planted
+``src/meta.json`` (sample folders carry the run's LLM-written src/** tree) could
+inject a phantom row. The manifest is now the single source of truth: pack tars
+ONLY manifest entries, verifies every byte against the recorded sha256, and
+replaces archives atomically after the whole loop succeeded.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import re
+import tarfile
+from pathlib import Path
+
+import pytest
+
+from codeverse.flywheel.export import collect_rows, export_samples
+from codeverse.flywheel.manifest import (
+ MANIFEST_NAME,
+ ManifestError,
+ load_manifest,
+ write_manifest,
+)
+from codeverse.flywheel.pack import PackError, pack_samples, verify_locators
+
+CHAIR_ID = "3dcodeverse/static_object/blender/wooden_chair_ab12cd34"
+
+
+def _tar_member_names(out: Path) -> list[str]:
+ names: list[str] = []
+ for tp in sorted(out.glob("samples-*.tar")):
+ with tarfile.open(tp) as tf:
+ names += [m.name for m in tf.getmembers()]
+ return names
+
+
+# --------------------------------------------------------------------------- manifest
+def test_export_writes_a_hashed_manifest(runs_dir: Path, tmp_path: Path):
+ out = tmp_path / "ds"
+ rep = export_samples(runs_dir, out)
+ assert rep.manifest == str(out / MANIFEST_NAME)
+ m = load_manifest(out)
+ assert m.generated_at is not None
+ assert m.filters.drop_duplicates is False and m.filters.min_score is None
+ assert len(m.entries) == 3 and m.dropped == []
+ e = next(x for x in m.entries if x.sample_id == CHAIR_ID)
+ assert e.run.battery == "runs" and e.run.rel == "wooden_chair_ab12cd34"
+ assert e.sample_rel_dir == "static_object/blender/wooden_chair_ab12cd34"
+ assert {"meta.json", "captions.json", "code.py", "src/model.py"} <= set(e.files)
+ for rel, digest in e.files.items(): # every byte the export wrote is hashed
+ assert hashlib.sha256((out / e.sample_rel_dir / rel).read_bytes()).hexdigest() == digest
+ assert len(e.code_sha256) == 64 and len(e.code_fingerprint) == 64
+ assert e.code_sha256 != e.code_fingerprint # raw-exact vs normalised
+
+
+def test_filtered_runs_are_recorded_as_dropped(runs_dir: Path, tmp_path: Path):
+ export_samples(runs_dir, tmp_path / "ds", min_score=0.75)
+ m = load_manifest(tmp_path / "ds")
+ assert len(m.entries) == 1 and len(m.dropped) == 2
+ assert m.filters.min_score == 0.75
+ assert all("min_score" in d.reason for d in m.dropped)
+ assert {d.run.rel for d in m.dropped} == {"wooden_chair_codex", "lamp_three"}
+
+
+# --------------------------------------------------------------------------- drop-duplicates → pack
+def test_drop_duplicates_then_pack_ships_no_duplicate(runs_dir: Path, tmp_path: Path):
+ out = tmp_path / "ds"
+ # the codex run shares prompt + best-round code with the chair → normalised duplicate
+ rep = export_samples(runs_dir, out, drop_duplicates=True)
+ assert rep.n_exported == 3 and rep.n_indexed == 2 and rep.n_duplicates == 1
+ m = load_manifest(out)
+ assert {e.run.rel for e in m.entries} == {"wooden_chair_ab12cd34", "lamp_three"}
+ dup = [d for d in m.dropped if d.reason.startswith("duplicate_of:")]
+ assert len(dup) == 1 and dup[0].run.rel == "wooden_chair_codex"
+ assert dup[0].reason == f"duplicate_of:{CHAIR_ID}"
+ # the duplicate FOLDER stays on disk by design — it must still never ship
+ assert (out / "static_object" / "blender" / "wooden_chair_codex").is_dir()
+ prep = pack_samples(out)
+ assert prep.n_samples == 2 and verify_locators(out) == 2
+ names = _tar_member_names(out)
+ assert names and not any(n.startswith("wooden_chair_codex/") for n in names)
+ rows = [json.loads(ln) for ln in (out / "metadata.jsonl").read_text().splitlines()]
+ assert {r["key"] for r in rows} == {"wooden_chair_ab12cd34", "lamp_three"}
+ assert all(r["tar"] for r in rows)
+
+
+# --------------------------------------------------------------------------- pack is manifest-only
+def test_pack_without_a_manifest_is_a_hard_error(tmp_path: Path):
+ out = tmp_path / "ds"
+ (out / "static_object" / "blender" / "x").mkdir(parents=True) # samples alone don't count
+ with pytest.raises(PackError, match="no longer rescans"):
+ pack_samples(out)
+ with pytest.raises(ManifestError, match="dataset_manifest.json"):
+ load_manifest(out)
+
+
+def test_repack_failure_leaves_the_old_tar_and_index_intact(runs_dir: Path, tmp_path: Path):
+ out = tmp_path / "ds"
+ export_samples(runs_dir, out)
+ pack_samples(out)
+ tar_path = out / "samples-000.tar"
+ before = {p.name: p.read_bytes() for p in (tar_path, out / "metadata.jsonl", out / "metadata.parquet")}
+ # corrupt the recorded hash of a file in the LAST entry: the mismatch fires
+ # mid-pack, after earlier samples were already added to the new tmp archive
+ m = load_manifest(out)
+ victim = m.entries[-1]
+ rel = sorted(victim.files)[0]
+ victim.files[rel] = "0" * 64
+ write_manifest(m, out)
+ with pytest.raises(PackError, match=re.escape(rel)):
+ pack_samples(out)
+ assert tar_path.read_bytes() == before["samples-000.tar"] # old archive untouched
+ assert (out / "metadata.jsonl").read_bytes() == before["metadata.jsonl"]
+ assert (out / "metadata.parquet").read_bytes() == before["metadata.parquet"]
+ assert not list(out.glob("*.tmp")) # the half-written tmp tar is cleaned up
+ assert verify_locators(out) == 3 # the previous pack still round-trips
+
+
+def test_a_tampered_sample_file_fails_fast_and_names_it(runs_dir: Path, tmp_path: Path):
+ out = tmp_path / "ds"
+ export_samples(runs_dir, out)
+ (out / "static_object" / "blender" / "wooden_chair_ab12cd34" / "code.py").write_text("tampered")
+ with pytest.raises(PackError, match="code.py"):
+ pack_samples(out)
+ assert not list(out.glob("samples-*.tar")) and not list(out.glob("*.tmp"))
+
+
+def test_repack_over_an_existing_archive_succeeds_atomically(runs_dir: Path, tmp_path: Path):
+ out = tmp_path / "ds"
+ export_samples(runs_dir, out)
+ assert pack_samples(out).n_samples == 3
+ prep2 = pack_samples(out) # re-pack over the existing archive (used to truncate it at open)
+ assert prep2.n_samples == 3 and verify_locators(out) == 3
+ assert not list(out.glob("*.tmp"))
+
+
+# --------------------------------------------------------------------------- phantom rows
+def test_planted_src_meta_json_is_not_a_row_and_not_packed(runs_dir: Path, tmp_path: Path):
+ out = tmp_path / "ds"
+ export_samples(runs_dir, out)
+ sdir = out / "static_object" / "blender" / "wooden_chair_ab12cd34"
+ (sdir / "src" / "meta.json").write_text(json.dumps({"id": "phantom", "key": "phantom"}))
+ rows = collect_rows(out) # the recovery/debug rescan is exactly 3 levels deep
+ assert len(rows) == 3 and all(r["id"] != "phantom" for r in rows)
+ pack_samples(out) # and pack never looks at the directory tree at all
+ assert verify_locators(out) == 3
+ assert "wooden_chair_ab12cd34/src/meta.json" not in _tar_member_names(out)
+ keys = {json.loads(ln)["key"] for ln in (out / "metadata.jsonl").read_text().splitlines()}
+ assert keys == {"wooden_chair_ab12cd34", "wooden_chair_codex", "lamp_three"}
diff --git a/harness/tests/gallery/test_hardening.py b/harness/tests/gallery/test_hardening.py
new file mode 100644
index 00000000..9a63875d
--- /dev/null
+++ b/harness/tests/gallery/test_hardening.py
@@ -0,0 +1,58 @@
+"""Serving LLM-authored HTML/SVG must be inert (CSP sandbox, honest content type),
+and CSV cells must never reach a spreadsheet as formulas."""
+
+from __future__ import annotations
+
+import csv
+import io
+from pathlib import Path
+
+from codeverse.gallery.compare import CSV_COLUMNS, csv_safe, export_csv
+from codeverse.gallery.index import build_index
+from codeverse.gallery.server import SANDBOXED_TYPES, GalleryApp
+from codeverse.gallery.urls import content_type
+from tests.flywheel_cli.conftest import make_fake_run
+
+
+# --------------------------------------------------------------------------- content types + CSP
+def test_html_is_typed_html_not_guessed():
+ assert content_type("a.html") == "text/html; charset=utf-8"
+ assert content_type("a.htm") == "text/html; charset=utf-8"
+ assert content_type("a.svg") == "image/svg+xml"
+ assert {"text/html", "image/svg+xml"} == SANDBOXED_TYPES
+
+
+def test_llm_html_and_svg_file_responses_are_sandboxed(gallery_tree: dict[str, Path]):
+ app = GalleryApp([gallery_tree["runs"]])
+ run = gallery_tree["runs"] / "wooden_chair_ab12cd34"
+ (run / "artifacts" / "page.html").write_text("<script>fetch('/api/index')</script>")
+ (run / "artifacts" / "figure.svg").write_text(
+ "<svg xmlns='http://www.w3.org/2000/svg'><script>1</script></svg>")
+ html = app.route("/file/runs/wooden_chair_ab12cd34/artifacts/page.html")
+ assert html.status == 200 and html.content_type == "text/html; charset=utf-8"
+ assert html.headers["Content-Security-Policy"] == "sandbox"
+ svg = app.route("/file/runs/wooden_chair_ab12cd34/artifacts/figure.svg")
+ assert svg.status == 200 and svg.content_type == "image/svg+xml"
+ assert svg.headers["Content-Security-Policy"] == "sandbox"
+ # images and code stay untouched — <img src=…> previews keep rendering
+ png = app.route("/file/runs/wooden_chair_ab12cd34/artifacts/renders/r01/sheet.png")
+ assert png.status == 200 and "Content-Security-Policy" not in png.headers
+ src = app.route("/file/runs/wooden_chair_ab12cd34/src/model.py")
+ assert src.status == 200 and "Content-Security-Policy" not in src.headers
+
+
+# --------------------------------------------------------------------------- csv injection
+def test_csv_safe_neutralises_formula_prefixes():
+ assert csv_safe("=HYPERLINK(1)") == "'=HYPERLINK(1)"
+ assert csv_safe("+1") == "'+1" and csv_safe("-1") == "'-1" and csv_safe("@cmd") == "'@cmd"
+ assert csv_safe("a chair") == "a chair"
+ assert csv_safe(3) == 3 and csv_safe(None) is None
+
+
+def test_a_formula_prompt_round_trips_csv_escaped(tmp_path: Path):
+ runs = tmp_path / "runs"
+ make_fake_run(runs, "formula_prompt", prompt="=2+5|cmd", scores=(0.5, 0.9))
+ text = export_csv(build_index([runs]).entries())
+ rows = list(csv.reader(io.StringIO(text)))
+ assert rows[0] == list(CSV_COLUMNS) and len(rows) == 2
+ assert rows[1][CSV_COLUMNS.index("prompt")] == "'=2+5|cmd"