Official Codebase of 3DCodeVerse
Python
toolkits: the pair builder, the GLSL verifier for the 44 unflattened shards, and a converter that no longer drops data
gaoypengf9f0be69bdfb
diff --git a/toolkits/llamafactory/README.md b/toolkits/llamafactory/README.mdindex a86d5be5..a023ea45 100644--- a/toolkits/llamafactory/README.md+++ b/toolkits/llamafactory/README.md@@ -36,3 +36,13 @@ rows also cannot be packed: set `packing: false` and `neat_packing: false`.Python 3.12 with `pandas`, `pyarrow`, `transformers` (tokenizer only), `playwright` (+ `playwright install chromium`),and `huggingface_hub` for the upload paths; `glslangValidator` on PATH (conda-forge `glslang`); Blender / OpenSCADas system binaries. Paths to the binaries are constants at the top of each script — change them there.+| `build_pairs_3dcodebench.py` | the three pair types for one source subdirectory — text→code, image→code, image+text→code — with one caption per sample rather than one row per caption variant. Renders are read by **scanning each tar**, because the byte offsets in `metadata.parquet` cannot be trusted (for `3dcodebench/instances_geo` only 577 of 1,953 point at a real tar header; the archives were repacked after the metadata was written, and following them silently cost 70% of the image pairs) |+| `verify_unflattened_glsl.py` | compile-verifies the 44 Shadertoy shards `flatten_multifile.py` never reached — their metadata carries no `shader_json`, so there was no renderpass structure to merge a Common tab from. Writes the same `compiles` / `repair` columns the flattened shards have, so one filter covers the whole GLSL corpus |+| `runners/` | the per-source drivers that call the above over every subdirectory |++### A third rule these tools now encode++3. **A conversion must never decide what a benchmark forbids.** `convert_subdirs.py` used to drop any sample+ overlapping the 212 3DCodeBench factories, which silently emptied both `3dcodebench/instances_*` folders —+ 100% of their rows were overlaps. The overlap is now *counted* in `qc.json` and the rows are kept, so the+ split is a decision the training run makes, not one the converter already made.diff --git a/toolkits/llamafactory/build_pairs_3dcodebench.py b/toolkits/llamafactory/build_pairs_3dcodebench.pynew file mode 100644index 00000000..42847bc7--- /dev/null+++ b/toolkits/llamafactory/build_pairs_3dcodebench.py@@ -0,0 +1,297 @@+"""Build all three pair types for a 3DCodeVerse subdirectory: text->code, image->code, image+text->code.++The corpus so far only carries text->code, and its row count is inflated ~3x because every caption variant is a+separate row -- an augmentation three independent runs showed does not help. Here each SAMPLE appears once per+pair type with one caption, and the renders come from the sample's own slice of the tar (metadata carries+`tar`/`byte_start`/`byte_len`, so a ranged read fetches one sample without downloading a 2 GB shard).++ python scripts/build_pairs_3dcodebench.py --subdir 3dcodebench/factories_geo --out DIR [--views 4] [--workers 8]+ [--limit N] [--local_tars DIR]++Writes <out>/<name>_{text,img,imgtext}/train.parquet + dataset_info.json + qc.json.+"""+import argparse+import collections+import io+import json+import os+import tarfile+import time++import pandas as pd+import pyarrow.parquet as pq+import requests++META = "/wekafs/ict/hx_624/data/3dcodeverse_meta"+REPO = "ilabai/3dcodeverse"+SYS = {+ "blender": "You are an expert in procedural 3D modeling with Blender Python (bpy). Write a complete, standalone Blender 5 Python script that builds the requested object from scratch (clear the default scene first). Output ONLY the code in one ```python block.",+ "glsl": "You are an expert GLSL shader programmer. Write a complete Shadertoy-style fragment shader. Output ONLY the code in one ```glsl block.",+}+FENCE_RE = __import__('re').compile(r'^```[a-zA-Z]*\n|\n?```$')+FENCE = {"blender": "python", "glsl": "glsl"}+ASK_TEXT = {"blender": "Write the Blender Python code that builds this object: ",+ "glsl": "Write the shader described here: "}+ASK_IMG = {"blender": "These are reference renders of one object. Write the Blender Python code that builds it.",+ "glsl": "These are frames rendered by one shader. Write the shader that produces them."}+ASK_BOTH = {"blender": "These are reference renders of one object. Write the Blender Python code that builds it.\n\nThe object: ",+ "glsl": "These are frames rendered by one shader. Write the shader that produces them.\n\nThe shader: "}+IMPERATIVE = ("write", "create", "implement", "make", "build", "generate", "design", "produce")+++def caption_of(caps):+ """the best human description, reduced to a noun phrase because the ASK_* strings already supply the verb"""+ for k in ("user_instruction", "instruction", "detailed", "brief", "factory"):+ v = (caps.get(k) or "").strip()+ if not v or len(v) < 15:+ continue+ low = v.lower()+ if low.startswith(IMPERATIVE):+ for sep in (" that builds ", " that renders ", " that reproduces ", " of ", " for "):+ if sep in low:+ return v[low.index(sep) + len(sep):].strip()[:1200]+ continue+ return v[:1200]+ return None+++def fetch_sample(row, token, local_tars=None):+ """-> list of render bytes for one sample, via a local tar if we have it, else a ranged read"""+ tar_rel, start, length = row["tar"], int(row["byte_start"]), int(row["byte_len"])+ if local_tars:+ p = os.path.join(local_tars, os.path.basename(tar_rel))+ if os.path.exists(p):+ with open(p, "rb") as f:+ f.seek(start)+ blob = f.read(length)+ return read_renders(blob)+ url = f"https://huggingface.co/datasets/{REPO}/resolve/main/{tar_rel}"+ hdr = {"Authorization": f"Bearer {token}", "Range": f"bytes={start}-{start + length - 1}"}+ # under a dozen concurrent readers the Hub throttles and times out; returning [] on the first failure looks+ # exactly like "this sample has no renders", which silently cost 70% of the image pairs on the first run+ for attempt in range(4):+ try:+ r = requests.get(url, headers=hdr, timeout=300)+ if r.status_code in (200, 206):+ return read_renders(r.content)+ last = f"HTTP {r.status_code}"+ except Exception as e:+ last = type(e).__name__+ time.sleep(2 ** attempt)+ raise FetchError(last)+++class FetchError(Exception):+ """the sample could not be read at all — distinct from a sample that genuinely carries no renders"""+++def read_renders(blob):+ """A byte slice out of the middle of a tar has no end-of-archive marker, so tarfile treats it as truncated —+ and it raises in getmembers(), not in open(), which is why guarding only the open() let one bad sample kill a+ whole run. Append the two zero blocks tar expects, and scan defensively."""+ blob = blob + b"\0" * 1024+ try:+ tf = tarfile.open(fileobj=io.BytesIO(blob))+ except Exception:+ return []+ out = []+ try:+ members = sorted(tf.getmembers(), key=lambda m: m.name)+ except Exception:+ return []+ for m in members:+ if m.isfile() and "/renders/" in m.name and m.name.endswith(".png"):+ try:+ f = tf.extractfile(m)+ if f:+ out.append((os.path.basename(m.name), f.read()))+ except Exception:+ pass+ return out++++def renders_by_scanning(subdir, sample_ids, views, token, local_tars=None, cache_root="/wekafs/ict/hx_624/data/pair_tars"):+ """Index every sample's renders by walking each tar once.++ The byte offsets in metadata.parquet cannot be trusted: for 3dcodebench/instances_geo only 577 of 1,953 point+ at a real tar header (checked by reading 512 bytes at each offset and looking for the ustar magic), because+ the archives were repacked after the metadata was written. Sequential scanning ignores the offsets entirely,+ and for a whole-subdirectory conversion it is also cheaper -- one pass per tar instead of one request per+ sample.+ """+ import glob as _glob+ from huggingface_hub import hf_hub_download+ want = {sid.split("/")[-1] for sid in sample_ids}+ found = {}+ tars = sorted({r for r in _tars_of(subdir)})+ for i, tar_rel in enumerate(tars, 1):+ if local_tars:+ path = os.path.join(local_tars, os.path.basename(tar_rel))+ if not os.path.exists(path):+ path = None+ else:+ path = None+ if path is None:+ path = hf_hub_download(REPO, tar_rel, repo_type="dataset", token=token, cache_dir=cache_root)+ n_before = len(found)+ try:+ with tarfile.open(path) as tf:+ for m in tf:+ if not (m.isfile() and "/renders/" in m.name and m.name.endswith(".png")):+ continue+ key = m.name.split("/")[0]+ if key not in want:+ continue+ lst = found.setdefault(key, [])+ if len(lst) >= views:+ continue+ f = tf.extractfile(m)+ if f:+ lst.append((os.path.basename(m.name), f.read()))+ except Exception as e:+ print(f"[pairs] tar {os.path.basename(tar_rel)} unreadable: {type(e).__name__}", flush=True)+ print(f"[pairs] scanned {i}/{len(tars)} {os.path.basename(tar_rel)}: "+ f"+{len(found)-n_before} samples ({len(found):,}/{len(want):,})", flush=True)+ return found+++def _has_verify_cols(path):+ import pyarrow.parquet as _pq+ try:+ return {"compiles", "repair"} <= set(_pq.ParquetFile(path).schema_arrow.names)+ except Exception:+ return False+++def _tars_of(subdir):+ from huggingface_hub import HfApi+ api = HfApi(token=os.environ.get("HF_TOKEN", ""))+ return [f for f in api.list_repo_files(REPO, repo_type="dataset")+ if f.startswith(f"{subdir}/") and f.endswith(".tar")]+++def main():+ ap = argparse.ArgumentParser()+ ap.add_argument("--subdir", required=True)+ ap.add_argument("--out", required=True)+ ap.add_argument("--views", type=int, default=4)+ ap.add_argument("--workers", type=int, default=8)+ ap.add_argument("--limit", type=int, default=None)+ ap.add_argument("--local_tars", default=None)+ ap.add_argument("--img_root", default="/wekafs/ict/hx_624/data/pair_images")+ ap.add_argument("--dialect", default="blender")+ # the compile verification lives in the *_llamafactory parquets; carrying it into the pair datasets is the+ # difference between "here is some GLSL" and "here is GLSL that a compiler accepted"+ ap.add_argument("--verified_from", default=None, help="dir with <name>_llamafactory/train.parquet holding compiles/repair")+ a = ap.parse_args()+ token = os.environ.get("HF_TOKEN", "")+ rows = pq.read_table(f"{META}/{a.subdir}/metadata.parquet").to_pylist()+ if a.limit:+ rows = rows[: a.limit]+ print(f"[pairs] {a.subdir}: {len(rows):,} source samples", flush=True)++ from concurrent.futures import ThreadPoolExecutor+ name = a.subdir.replace("/", "__")+ img_dir = os.path.join(a.img_root, a.subdir)+ os.makedirs(img_dir, exist_ok=True)+ qc = collections.Counter()++ verified = {}+ if a.verified_from:+ # the verification lives in two places depending on the shard: the 68 that could be flattened carry it in+ # <sub>_llamafactory_flat, the 44 that could not are in the separately verified tree+ base = os.path.basename(a.subdir)+ cands = []+ for root in a.verified_from.split(","):+ cands += [os.path.join(root, f"{base}_llamafactory_flat", "train.parquet"),+ os.path.join(root, f"{base}_llamafactory", "train.parquet")]+ vp = next((c for c in cands if os.path.exists(c) and _has_verify_cols(c)), None)+ if vp:+ vdf = pd.read_parquet(vp, columns=["id", "compiles", "repair", "conversations"])+ for t in vdf.itertuples():+ # one row per caption variant; the code is identical across them, so first wins+ verified.setdefault(str(t.id), (bool(t.compiles), t.repair, t.conversations[1]["value"]))+ print(f"[pairs] verification joined for {len(verified):,} ids from {vp}", flush=True)+ else:+ print(f"[pairs] WARNING no parquet with compiles/repair for {base} — pairs carry no compiles column", flush=True)+ ids = [str(r["id"]) for r in rows]+ render_index = renders_by_scanning(a.subdir, ids, a.views, token, a.local_tars)++ def one(r):+ code = r.get("code") or ""+ if len(code.strip()) < 40:+ qc["short_code"] += 1+ return None+ caps = r.get("captions")+ caps = dict(caps) if isinstance(caps, dict) else (json.loads(caps) if isinstance(caps, str) else {})+ cap = caption_of(caps)+ renders = render_index.get(str(r["id"]).split("/")[-1], [])[: a.views]+ paths = []+ for fname, blob in renders:+ d = os.path.join(img_dir, str(r["id"]).replace("/", "_"))+ os.makedirs(d, exist_ok=True)+ p = os.path.join(d, fname)+ if not os.path.exists(p):+ open(p, "wb").write(blob)+ paths.append(p)+ if not paths:+ qc["no_renders"] += 1+ if not cap:+ qc["no_caption"] += 1+ v = verified.get(str(r["id"]))+ if v:+ # prefer the verified (possibly repaired) program over the raw metadata code+ code = FENCE_RE.sub("", v[2].strip()) or code+ return {"id": str(r["id"]), "code": code, "caption": cap, "images": paths,+ "compiles": (v[0] if v else None), "repair": (v[1] if v else None)}++ with ThreadPoolExecutor(a.workers) as ex:+ got = [x for x in ex.map(one, rows) if x]+ print(f"[pairs] usable samples: {len(got):,} | qc={dict(qc)}", flush=True)+ if qc["fetch_failed"]:+ print(f"[pairs] WARNING {qc['fetch_failed']:,} samples could not be fetched after 4 attempts — their image "+ f"pairs are MISSING, not absent by nature. Re-run to fill them in.", flush=True)++ d, fence = a.dialect, FENCE.get(a.dialect, "python")+ out = {"text": [], "img": [], "imgtext": []}+ for g in got:+ gpt = {"from": "gpt", "value": f"```{fence}\n{g['code'].strip()}\n```"}+ base = {"id": g["id"], "system": SYS[d]}+ if g["compiles"] is not None:+ base["compiles"] = g["compiles"]; base["repair"] = g["repair"]+ if g["caption"]:+ out["text"].append(dict(base, conversations=[{"from": "human", "value": ASK_TEXT[d] + g["caption"]}, gpt]))+ if g["images"]:+ ph = "<image>" * len(g["images"])+ out["img"].append(dict(base, images=list(g["images"]),+ conversations=[{"from": "human", "value": ph + "\n" + ASK_IMG[d]}, gpt]))+ if g["caption"]:+ out["imgtext"].append(dict(base, images=list(g["images"]),+ conversations=[{"from": "human", "value": ph + "\n" + ASK_BOTH[d] + g["caption"]}, gpt]))+ info = {}+ for kind, rowsk in out.items():+ if not rowsk:+ print(f"[pairs] {kind}: 0 rows — not written", flush=True)+ continue+ folder = os.path.join(a.out, f"{os.path.basename(a.subdir)}_llamafactory_{kind}")+ os.makedirs(folder, exist_ok=True)+ pd.DataFrame(rowsk).to_parquet(os.path.join(folder, "train.parquet"), index=False)+ cols = {"messages": "conversations", "system": "system"}+ if kind != "text":+ cols["images"] = "images"+ entry = {"file_name": "train.parquet", "formatting": "sharegpt", "columns": cols}+ json.dump({f"3dcv_{name}_{kind}": entry}, open(os.path.join(folder, "dataset_info.json"), "w"), indent=2)+ json.dump({"subdir": a.subdir, "pair_type": kind, "rows": len(rowsk),+ "source_samples": len(rows), "usable": len(got), **{k: v for k, v in qc.items()}},+ open(os.path.join(folder, "qc.json"), "w"), indent=2)+ info[kind] = len(rowsk)+ print(f"[pairs] {kind:8s} {len(rowsk):>7,} rows -> {folder}", flush=True)+ # the invariant LLaMA-Factory enforces on multimodal rows+ bad = sum(1 for k in ("img", "imgtext") for r in out[k]+ if r["conversations"][0]["value"].count("<image>") != len(r["images"]))+ print(f"[pairs] placeholder/image mismatches: {bad} (must be 0)", flush=True)+++if __name__ == "__main__":+ main()diff --git a/toolkits/llamafactory/convert_subdirs.py b/toolkits/llamafactory/convert_subdirs.pyindex 739f04e8..c63f1a7a 100644--- a/toolkits/llamafactory/convert_subdirs.py+++ b/toolkits/llamafactory/convert_subdirs.py@@ -193,8 +193,10 @@ def convert_one(args):code = MEDIA.sub(lambda m: f"<{m.group(1)} >", code)norm_name = r["name"].replace("Factory", "").replace("_geo", "").replace("_tex", "")if r["name"] in bench_factories or r["name"].rsplit("_", 1)[0] in bench_factories or norm_name in bench_factories:- qc["benchmark_overlap"] += 1 # flagged, and excluded below- continue+ # counted, never dropped: which samples a benchmark forbids is a decision for whoever defines the+ # benchmark, and a converter that quietly removes 100% of a subdirectory (as this once did for both+ # 3dcodebench/instances_*) is the wrong place to make it+ qc["benchmark_overlap"] += 1caps = normalize_caps(r["caps"])n_before = len(out)for kind in ("instruction", "detailed", "factory"):@@ -222,7 +224,7 @@ def convert_one(args):open(os.path.join(d, "dataset_info.json"), "w"), indent=2)tok_lens.sort()stat = {"subdir": sub, "dialect": dialect, "source_samples": len(rows), "exported_pairs": len(out),- "unique_samples_exported": len(seen_code) - qc["benchmark_overlap"],+ "unique_samples_exported": len(seen_code),"images_per_sample_median": (sorted(img_counts)[len(img_counts) // 2] if img_counts else 0),"tokens_p50": tok_lens[len(tok_lens) // 2] if tok_lens else 0,"tokens_p90": tok_lens[int(len(tok_lens) * 0.9)] if tok_lens else 0,@@ -248,7 +250,8 @@ Generated by `finetune/scripts/convert_subdirs.py` from `{sub}/metadata.parquet`{issues}`syntax_bad` rows are still exported (a shader can be valid without balanced braces in our crude check); every-other counter means the sample was dropped. `benchmark_overlap` = the sample belongs to one of the 212+other counter means the sample was dropped, EXCEPT `benchmark_overlap`, which is counted and kept: the+sample belongs to one of the 2123DCodeBench evaluation factories and is excluded to avoid test leakage.## Use@@ -266,10 +269,14 @@ def main():ap.add_argument("--out", default="/wekafs/ict/hx_624/hf_subdirs")ap.add_argument("--workers", type=int, default=16)ap.add_argument("--limit_subdirs", type=int, default=None)+ ap.add_argument("--only", default=None, help="comma-separated subdirs to convert")a = ap.parse_args()bench = frozenset(os.path.basename(p).replace("_seed0", "") for p in glob.glob(f"{BENCH_DIR}/*_seed0"))subs = ["/".join(p.split("/")[-3:-1]) for p in sorted(glob.glob(f"{META}/*/*/metadata.parquet"))]subs += ["blender_distill", "threejs_distill", "animation2code"]+ if a.only:+ want = {x.strip() for x in a.only.split(",") if x.strip()}+ subs = [x for x in subs if x in want]if a.limit_subdirs:subs = subs[: a.limit_subdirs]print(f"[conv] {len(subs)} subdirs -> {a.out}", flush=True)diff --git a/toolkits/llamafactory/runners/run_3dcb_pairs.sh b/toolkits/llamafactory/runners/run_3dcb_pairs.shnew file mode 100755index 00000000..44ff4e69--- /dev/null+++ b/toolkits/llamafactory/runners/run_3dcb_pairs.sh@@ -0,0 +1,32 @@+#!/bin/bash+# Build text->code, image->code and image+text->code for every 3DCodeBench subdirectory.+# Renders come from a sequential scan of each tar, NOT from metadata's byte offsets: for instances_geo only 577+# of 1,953 offsets point at a real tar header (the archives were repacked after the metadata was written), and+# trusting them silently cost 70% of the image pairs.+set -uo pipefail; cd /wekafs/ict/hx_624/llm-ft+source /wekafs/ict/hx_624/anaconda3/etc/profile.d/conda.sh; conda activate llmft+OUT=/wekafs/ict/hx_624/hf_pairs+for SUB in instances_tex instances_geo_full instances_tex_full; do+ # "the folder exists" is not the same as "it was built correctly": instances_tex was skipped by that test+ # while holding the 578-of-1,951 image pairs the broken byte offsets produced. Require the img rows to match+ # the source count before treating a subdir as done.+ Q="$OUT/${SUB}_llamafactory_img/qc.json"+ if [ -f "$Q" ] && python3 -c "import json,sys; q=json.load(open('$Q')); sys.exit(0 if q['rows'] >= 0.95*q['source_samples'] else 1)"; then+ echo "[3dcb] $SUB already built with full render coverage — skipping"; continue+ fi+ M=/wekafs/ict/hx_624/data/3dcodeverse_meta/3dcodebench/$SUB/metadata.parquet+ if [ ! -f "$M" ]; then+ echo "[3dcb] fetching metadata for $SUB"+ python - "$SUB" <<'PY'+import os, sys, shutil+from huggingface_hub import hf_hub_download+sub = sys.argv[1]+p = hf_hub_download("ilabai/3dcodeverse", f"3dcodebench/{sub}/metadata.parquet", repo_type="dataset", token=os.environ["HF_TOKEN"])+d = f"/wekafs/ict/hx_624/data/3dcodeverse_meta/3dcodebench/{sub}"+os.makedirs(d, exist_ok=True); shutil.copy(p, f"{d}/metadata.parquet")+PY+ fi+ echo "[3dcb] $(date) building $SUB"+ python scripts/build_pairs_3dcodebench.py --subdir 3dcodebench/$SUB --out $OUT --workers 16 --views 4 2>&1 | tail -6+done+echo "[3dcb] ALL DONE"diff --git a/toolkits/llamafactory/runners/run_shadertoy_pairs.sh b/toolkits/llamafactory/runners/run_shadertoy_pairs.shnew file mode 100755index 00000000..b24896b9--- /dev/null+++ b/toolkits/llamafactory/runners/run_shadertoy_pairs.sh@@ -0,0 +1,25 @@+#!/bin/bash+# All three pair types for every shadertoy shard. Renders come from scanning each shard's tars (the byte offsets+# in metadata are not reliable), and the code comes from whichever verified parquet exists for that shard --+# *_llamafactory_flat for the 68 that could be flattened, the separately verified tree for the other 44 -- so the+# pairs carry a compiles column instead of being unverified GLSL.+set -uo pipefail; cd /wekafs/ict/hx_624/llm-ft+source /wekafs/ict/hx_624/anaconda3/etc/profile.d/conda.sh; conda activate llmft+OUT=/wekafs/ict/hx_624/hf_pairs_shadertoy+VER="/wekafs/ict/hx_624/hf_subdirs/shadertoy,/wekafs/ict/hx_624/hf_subdirs_verified/shadertoy"+PAR=${PAR:-4}+mkdir -p $OUT+build () {+ local SUB=$1+ local Q="$OUT/${SUB}_llamafactory_img/qc.json"+ if [ -f "$Q" ]; then echo "[st] $SUB already built — skipping"; return; fi+ python scripts/build_pairs_3dcodebench.py --subdir shadertoy/$SUB --dialect glsl \+ --out $OUT --workers 12 --views 4 --verified_from "$VER" > logs/pairs_st_$SUB.log 2>&1+ local line=$(grep -E "^\[pairs\] (text|img|imgtext)" logs/pairs_st_$SUB.log | tr '\n' ' ' | tr -s ' ')+ echo "[st] $(date +%H:%M) $SUB ${line:-FAILED: $(grep -iE 'error|Traceback' logs/pairs_st_$SUB.log | tail -1 | cut -c1-90)}"+}+export -f build; export OUT VER+SHARDS=$(ls -d /wekafs/ict/hx_624/data/3dcodeverse_meta/shadertoy/*/ | xargs -n1 basename)+echo "[st] $(echo "$SHARDS" | wc -l) shards, $PAR at a time"+echo "$SHARDS" | xargs -P $PAR -I{} bash -c 'build "$@"' _ {}+echo "[st] ALL DONE"diff --git a/toolkits/llamafactory/verify_unflattened_glsl.py b/toolkits/llamafactory/verify_unflattened_glsl.pynew file mode 100644index 00000000..5bb8dc33--- /dev/null+++ b/toolkits/llamafactory/verify_unflattened_glsl.py@@ -0,0 +1,108 @@+"""Compile-verify the 44 shadertoy shards that the flattening pass never reached.++Those shards' metadata carries no `shader_json`, so there was no renderpass structure to merge a Common tab+from and `flatten_multifile.py` skipped them — exactly 44 of 112, with zero mismatches. The consequence is that+73,281 shaders are verified (99.04% compile) while 46,680 have never been near a compiler. This runs the same+checker and the same minimal repairs over those, and writes the `compiles` / `repair` columns the flattened+folders already carry, so a training mix can filter the whole corpus by one rule instead of two.++usage: python scripts/verify_unflattened_glsl.py [--workers 40] [--limit_shards N] [--out DIR]+"""+import argparse+import collections+import glob+import json+import os+import re+import sys++import pandas as pd++sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))+from qc_repair import glsl_compile, repair_glsl++SRC = "/wekafs/ict/hx_624/hf_subdirs/shadertoy"+FENCE = re.compile(r"^```[a-zA-Z]*\n|\n?```$")+++def strip_fence(v):+ return FENCE.sub("", v.strip())+++def verify_one(code):+ """-> (compiles, repair_label, code_to_keep). Never returns a program that does not compile as 'repaired'."""+ ok, _ = glsl_compile(code)+ if ok:+ return True, "original", code+ try:+ fixed, recipe = repair_glsl(code)+ except Exception:+ return False, "unfixable", code+ if fixed and fixed != code:+ ok2, _ = glsl_compile(fixed)+ if ok2:+ return True, f"repaired:{recipe or 'other'}", fixed+ return False, "unfixable", code+++def do_shard(args):+ sub, out_root = args+ p = f"{SRC}/{sub}_llamafactory/train.parquet"+ df = pd.read_parquet(p)+ # one shader appears once per caption variant; compile each distinct program once+ by_code = {}+ codes = [strip_fence(c[1]["value"]) for c in df["conversations"]]+ for c in set(codes):+ by_code[c] = verify_one(c)+ okc, howc, conv = [], [], []+ for row, c in zip(df.itertuples(), codes):+ ok, how, keep = by_code[c]+ okc.append(ok); howc.append(how)+ turns = [dict(t) for t in row.conversations]+ turns[1] = dict(turns[1], value=f"```glsl\n{keep}\n```")+ conv.append(turns)+ df["conversations"] = conv+ df["compiles"] = okc+ df["repair"] = howc+ d = os.path.join(out_root, f"{sub}_llamafactory")+ os.makedirs(d, exist_ok=True)+ df.to_parquet(os.path.join(d, "train.parquet"), index=False)+ for extra in ("dataset_info.json", "qc.json", "README.md"):+ s = f"{SRC}/{sub}_llamafactory/{extra}"+ if os.path.exists(s):+ open(os.path.join(d, extra), "w").write(open(s).read())+ st = {"shard": sub, "rows": len(df), "unique_programs": len(by_code),+ "compiles": int(sum(okc)), "repaired": int(sum(1 for h in howc if h.startswith("repaired"))),+ "unfixable": int(sum(1 for h in howc if h == "unfixable"))}+ json.dump(st, open(os.path.join(d, "verify.json"), "w"), indent=2)+ return st+++def main():+ ap = argparse.ArgumentParser()+ ap.add_argument("--workers", type=int, default=40)+ ap.add_argument("--limit_shards", type=int, default=None)+ ap.add_argument("--out", default="/wekafs/ict/hx_624/hf_subdirs_verified/shadertoy")+ a = ap.parse_args()+ flat = {os.path.basename(p)[: -len("_llamafactory_flat")] for p in glob.glob(f"{SRC}/*_llamafactory_flat")}+ allp = {os.path.basename(p)[: -len("_llamafactory")] for p in glob.glob(f"{SRC}/*_llamafactory")}+ todo = sorted(allp - flat)+ if a.limit_shards:+ todo = todo[: a.limit_shards]+ os.makedirs(a.out, exist_ok=True)+ print(f"[verify] {len(todo)} shards never verified -> {a.out}", flush=True)+ from concurrent.futures import ProcessPoolExecutor+ tot = collections.Counter()+ with ProcessPoolExecutor(a.workers) as ex:+ for st in ex.map(do_shard, [(s, a.out) for s in todo]):+ for k in ("rows", "unique_programs", "compiles", "repaired", "unfixable"):+ tot[k] += st[k]+ print(f"[verify] {st['shard']:4s} rows={st['rows']:>6,} uniq={st['unique_programs']:>5,} "+ f"ok={st['compiles']:>6,} repaired={st['repaired']:>4,} unfixable={st['unfixable']:>4,}", flush=True)+ rate = tot["compiles"] / max(tot["rows"], 1) * 100+ print(f"[verify] TOTAL rows={tot['rows']:,} unique={tot['unique_programs']:,} "+ f"compiles={tot['compiles']:,} ({rate:.2f}%) repaired={tot['repaired']:,} unfixable={tot['unfixable']:,}", flush=True)+++if __name__ == "__main__":+ main()