Skip to content

Official Codebase of 3DCodeVerse

Python

toolkits: the pair builder, the GLSL verifier for the 44 unflattened shards, and a converter that no longer drops data

gaoypengf9f0be69bdfb
diff --git a/toolkits/llamafactory/README.md b/toolkits/llamafactory/README.md
index a86d5be5..a023ea45 100644
--- a/toolkits/llamafactory/README.md
+++ b/toolkits/llamafactory/README.md
@@ -36,3 +36,13 @@ rows also cannot be packed: set `packing: false` and `neat_packing: false`.
Python 3.12 with `pandas`, `pyarrow`, `transformers` (tokenizer only), `playwright` (+ `playwright install chromium`),
and `huggingface_hub` for the upload paths; `glslangValidator` on PATH (conda-forge `glslang`); Blender / OpenSCAD
as system binaries. Paths to the binaries are constants at the top of each script — change them there.
+| `build_pairs_3dcodebench.py` | the three pair types for one source subdirectory — text→code, image→code, image+text→code — with one caption per sample rather than one row per caption variant. Renders are read by **scanning each tar**, because the byte offsets in `metadata.parquet` cannot be trusted (for `3dcodebench/instances_geo` only 577 of 1,953 point at a real tar header; the archives were repacked after the metadata was written, and following them silently cost 70% of the image pairs) |
+| `verify_unflattened_glsl.py` | compile-verifies the 44 Shadertoy shards `flatten_multifile.py` never reached — their metadata carries no `shader_json`, so there was no renderpass structure to merge a Common tab from. Writes the same `compiles` / `repair` columns the flattened shards have, so one filter covers the whole GLSL corpus |
+| `runners/` | the per-source drivers that call the above over every subdirectory |
+
+### A third rule these tools now encode
+
+3. **A conversion must never decide what a benchmark forbids.** `convert_subdirs.py` used to drop any sample
+ overlapping the 212 3DCodeBench factories, which silently emptied both `3dcodebench/instances_*` folders —
+ 100% of their rows were overlaps. The overlap is now *counted* in `qc.json` and the rows are kept, so the
+ split is a decision the training run makes, not one the converter already made.
diff --git a/toolkits/llamafactory/build_pairs_3dcodebench.py b/toolkits/llamafactory/build_pairs_3dcodebench.py
new file mode 100644
index 00000000..42847bc7
--- /dev/null
+++ b/toolkits/llamafactory/build_pairs_3dcodebench.py
@@ -0,0 +1,297 @@
+"""Build all three pair types for a 3DCodeVerse subdirectory: text->code, image->code, image+text->code.
+
+The corpus so far only carries text->code, and its row count is inflated ~3x because every caption variant is a
+separate row -- an augmentation three independent runs showed does not help. Here each SAMPLE appears once per
+pair type with one caption, and the renders come from the sample's own slice of the tar (metadata carries
+`tar`/`byte_start`/`byte_len`, so a ranged read fetches one sample without downloading a 2 GB shard).
+
+ python scripts/build_pairs_3dcodebench.py --subdir 3dcodebench/factories_geo --out DIR [--views 4] [--workers 8]
+ [--limit N] [--local_tars DIR]
+
+Writes <out>/<name>_{text,img,imgtext}/train.parquet + dataset_info.json + qc.json.
+"""
+import argparse
+import collections
+import io
+import json
+import os
+import tarfile
+import time
+
+import pandas as pd
+import pyarrow.parquet as pq
+import requests
+
+META = "/wekafs/ict/hx_624/data/3dcodeverse_meta"
+REPO = "ilabai/3dcodeverse"
+SYS = {
+ "blender": "You are an expert in procedural 3D modeling with Blender Python (bpy). Write a complete, standalone Blender 5 Python script that builds the requested object from scratch (clear the default scene first). Output ONLY the code in one ```python block.",
+ "glsl": "You are an expert GLSL shader programmer. Write a complete Shadertoy-style fragment shader. Output ONLY the code in one ```glsl block.",
+}
+FENCE_RE = __import__('re').compile(r'^```[a-zA-Z]*\n|\n?```$')
+FENCE = {"blender": "python", "glsl": "glsl"}
+ASK_TEXT = {"blender": "Write the Blender Python code that builds this object: ",
+ "glsl": "Write the shader described here: "}
+ASK_IMG = {"blender": "These are reference renders of one object. Write the Blender Python code that builds it.",
+ "glsl": "These are frames rendered by one shader. Write the shader that produces them."}
+ASK_BOTH = {"blender": "These are reference renders of one object. Write the Blender Python code that builds it.\n\nThe object: ",
+ "glsl": "These are frames rendered by one shader. Write the shader that produces them.\n\nThe shader: "}
+IMPERATIVE = ("write", "create", "implement", "make", "build", "generate", "design", "produce")
+
+
+def caption_of(caps):
+ """the best human description, reduced to a noun phrase because the ASK_* strings already supply the verb"""
+ for k in ("user_instruction", "instruction", "detailed", "brief", "factory"):
+ v = (caps.get(k) or "").strip()
+ if not v or len(v) < 15:
+ continue
+ low = v.lower()
+ if low.startswith(IMPERATIVE):
+ for sep in (" that builds ", " that renders ", " that reproduces ", " of ", " for "):
+ if sep in low:
+ return v[low.index(sep) + len(sep):].strip()[:1200]
+ continue
+ return v[:1200]
+ return None
+
+
+def fetch_sample(row, token, local_tars=None):
+ """-> list of render bytes for one sample, via a local tar if we have it, else a ranged read"""
+ tar_rel, start, length = row["tar"], int(row["byte_start"]), int(row["byte_len"])
+ if local_tars:
+ p = os.path.join(local_tars, os.path.basename(tar_rel))
+ if os.path.exists(p):
+ with open(p, "rb") as f:
+ f.seek(start)
+ blob = f.read(length)
+ return read_renders(blob)
+ url = f"https://huggingface.co/datasets/{REPO}/resolve/main/{tar_rel}"
+ hdr = {"Authorization": f"Bearer {token}", "Range": f"bytes={start}-{start + length - 1}"}
+ # under a dozen concurrent readers the Hub throttles and times out; returning [] on the first failure looks
+ # exactly like "this sample has no renders", which silently cost 70% of the image pairs on the first run
+ for attempt in range(4):
+ try:
+ r = requests.get(url, headers=hdr, timeout=300)
+ if r.status_code in (200, 206):
+ return read_renders(r.content)
+ last = f"HTTP {r.status_code}"
+ except Exception as e:
+ last = type(e).__name__
+ time.sleep(2 ** attempt)
+ raise FetchError(last)
+
+
+class FetchError(Exception):
+ """the sample could not be read at all — distinct from a sample that genuinely carries no renders"""
+
+
+def read_renders(blob):
+ """A byte slice out of the middle of a tar has no end-of-archive marker, so tarfile treats it as truncated —
+ and it raises in getmembers(), not in open(), which is why guarding only the open() let one bad sample kill a
+ whole run. Append the two zero blocks tar expects, and scan defensively."""
+ blob = blob + b"\0" * 1024
+ try:
+ tf = tarfile.open(fileobj=io.BytesIO(blob))
+ except Exception:
+ return []
+ out = []
+ try:
+ members = sorted(tf.getmembers(), key=lambda m: m.name)
+ except Exception:
+ return []
+ for m in members:
+ if m.isfile() and "/renders/" in m.name and m.name.endswith(".png"):
+ try:
+ f = tf.extractfile(m)
+ if f:
+ out.append((os.path.basename(m.name), f.read()))
+ except Exception:
+ pass
+ return out
+
+
+
+def renders_by_scanning(subdir, sample_ids, views, token, local_tars=None, cache_root="/wekafs/ict/hx_624/data/pair_tars"):
+ """Index every sample's renders by walking each tar once.
+
+ The byte offsets in metadata.parquet cannot be trusted: for 3dcodebench/instances_geo only 577 of 1,953 point
+ at a real tar header (checked by reading 512 bytes at each offset and looking for the ustar magic), because
+ the archives were repacked after the metadata was written. Sequential scanning ignores the offsets entirely,
+ and for a whole-subdirectory conversion it is also cheaper -- one pass per tar instead of one request per
+ sample.
+ """
+ import glob as _glob
+ from huggingface_hub import hf_hub_download
+ want = {sid.split("/")[-1] for sid in sample_ids}
+ found = {}
+ tars = sorted({r for r in _tars_of(subdir)})
+ for i, tar_rel in enumerate(tars, 1):
+ if local_tars:
+ path = os.path.join(local_tars, os.path.basename(tar_rel))
+ if not os.path.exists(path):
+ path = None
+ else:
+ path = None
+ if path is None:
+ path = hf_hub_download(REPO, tar_rel, repo_type="dataset", token=token, cache_dir=cache_root)
+ n_before = len(found)
+ try:
+ with tarfile.open(path) as tf:
+ for m in tf:
+ if not (m.isfile() and "/renders/" in m.name and m.name.endswith(".png")):
+ continue
+ key = m.name.split("/")[0]
+ if key not in want:
+ continue
+ lst = found.setdefault(key, [])
+ if len(lst) >= views:
+ continue
+ f = tf.extractfile(m)
+ if f:
+ lst.append((os.path.basename(m.name), f.read()))
+ except Exception as e:
+ print(f"[pairs] tar {os.path.basename(tar_rel)} unreadable: {type(e).__name__}", flush=True)
+ print(f"[pairs] scanned {i}/{len(tars)} {os.path.basename(tar_rel)}: "
+ f"+{len(found)-n_before} samples ({len(found):,}/{len(want):,})", flush=True)
+ return found
+
+
+def _has_verify_cols(path):
+ import pyarrow.parquet as _pq
+ try:
+ return {"compiles", "repair"} <= set(_pq.ParquetFile(path).schema_arrow.names)
+ except Exception:
+ return False
+
+
+def _tars_of(subdir):
+ from huggingface_hub import HfApi
+ api = HfApi(token=os.environ.get("HF_TOKEN", ""))
+ return [f for f in api.list_repo_files(REPO, repo_type="dataset")
+ if f.startswith(f"{subdir}/") and f.endswith(".tar")]
+
+
+def main():
+ ap = argparse.ArgumentParser()
+ ap.add_argument("--subdir", required=True)
+ ap.add_argument("--out", required=True)
+ ap.add_argument("--views", type=int, default=4)
+ ap.add_argument("--workers", type=int, default=8)
+ ap.add_argument("--limit", type=int, default=None)
+ ap.add_argument("--local_tars", default=None)
+ ap.add_argument("--img_root", default="/wekafs/ict/hx_624/data/pair_images")
+ ap.add_argument("--dialect", default="blender")
+ # the compile verification lives in the *_llamafactory parquets; carrying it into the pair datasets is the
+ # difference between "here is some GLSL" and "here is GLSL that a compiler accepted"
+ ap.add_argument("--verified_from", default=None, help="dir with <name>_llamafactory/train.parquet holding compiles/repair")
+ a = ap.parse_args()
+ token = os.environ.get("HF_TOKEN", "")
+ rows = pq.read_table(f"{META}/{a.subdir}/metadata.parquet").to_pylist()
+ if a.limit:
+ rows = rows[: a.limit]
+ print(f"[pairs] {a.subdir}: {len(rows):,} source samples", flush=True)
+
+ from concurrent.futures import ThreadPoolExecutor
+ name = a.subdir.replace("/", "__")
+ img_dir = os.path.join(a.img_root, a.subdir)
+ os.makedirs(img_dir, exist_ok=True)
+ qc = collections.Counter()
+
+ verified = {}
+ if a.verified_from:
+ # the verification lives in two places depending on the shard: the 68 that could be flattened carry it in
+ # <sub>_llamafactory_flat, the 44 that could not are in the separately verified tree
+ base = os.path.basename(a.subdir)
+ cands = []
+ for root in a.verified_from.split(","):
+ cands += [os.path.join(root, f"{base}_llamafactory_flat", "train.parquet"),
+ os.path.join(root, f"{base}_llamafactory", "train.parquet")]
+ vp = next((c for c in cands if os.path.exists(c) and _has_verify_cols(c)), None)
+ if vp:
+ vdf = pd.read_parquet(vp, columns=["id", "compiles", "repair", "conversations"])
+ for t in vdf.itertuples():
+ # one row per caption variant; the code is identical across them, so first wins
+ verified.setdefault(str(t.id), (bool(t.compiles), t.repair, t.conversations[1]["value"]))
+ print(f"[pairs] verification joined for {len(verified):,} ids from {vp}", flush=True)
+ else:
+ print(f"[pairs] WARNING no parquet with compiles/repair for {base} — pairs carry no compiles column", flush=True)
+ ids = [str(r["id"]) for r in rows]
+ render_index = renders_by_scanning(a.subdir, ids, a.views, token, a.local_tars)
+
+ def one(r):
+ code = r.get("code") or ""
+ if len(code.strip()) < 40:
+ qc["short_code"] += 1
+ return None
+ caps = r.get("captions")
+ caps = dict(caps) if isinstance(caps, dict) else (json.loads(caps) if isinstance(caps, str) else {})
+ cap = caption_of(caps)
+ renders = render_index.get(str(r["id"]).split("/")[-1], [])[: a.views]
+ paths = []
+ for fname, blob in renders:
+ d = os.path.join(img_dir, str(r["id"]).replace("/", "_"))
+ os.makedirs(d, exist_ok=True)
+ p = os.path.join(d, fname)
+ if not os.path.exists(p):
+ open(p, "wb").write(blob)
+ paths.append(p)
+ if not paths:
+ qc["no_renders"] += 1
+ if not cap:
+ qc["no_caption"] += 1
+ v = verified.get(str(r["id"]))
+ if v:
+ # prefer the verified (possibly repaired) program over the raw metadata code
+ code = FENCE_RE.sub("", v[2].strip()) or code
+ return {"id": str(r["id"]), "code": code, "caption": cap, "images": paths,
+ "compiles": (v[0] if v else None), "repair": (v[1] if v else None)}
+
+ with ThreadPoolExecutor(a.workers) as ex:
+ got = [x for x in ex.map(one, rows) if x]
+ print(f"[pairs] usable samples: {len(got):,} | qc={dict(qc)}", flush=True)
+ if qc["fetch_failed"]:
+ print(f"[pairs] WARNING {qc['fetch_failed']:,} samples could not be fetched after 4 attempts — their image "
+ f"pairs are MISSING, not absent by nature. Re-run to fill them in.", flush=True)
+
+ d, fence = a.dialect, FENCE.get(a.dialect, "python")
+ out = {"text": [], "img": [], "imgtext": []}
+ for g in got:
+ gpt = {"from": "gpt", "value": f"```{fence}\n{g['code'].strip()}\n```"}
+ base = {"id": g["id"], "system": SYS[d]}
+ if g["compiles"] is not None:
+ base["compiles"] = g["compiles"]; base["repair"] = g["repair"]
+ if g["caption"]:
+ out["text"].append(dict(base, conversations=[{"from": "human", "value": ASK_TEXT[d] + g["caption"]}, gpt]))
+ if g["images"]:
+ ph = "<image>" * len(g["images"])
+ out["img"].append(dict(base, images=list(g["images"]),
+ conversations=[{"from": "human", "value": ph + "\n" + ASK_IMG[d]}, gpt]))
+ if g["caption"]:
+ out["imgtext"].append(dict(base, images=list(g["images"]),
+ conversations=[{"from": "human", "value": ph + "\n" + ASK_BOTH[d] + g["caption"]}, gpt]))
+ info = {}
+ for kind, rowsk in out.items():
+ if not rowsk:
+ print(f"[pairs] {kind}: 0 rows — not written", flush=True)
+ continue
+ folder = os.path.join(a.out, f"{os.path.basename(a.subdir)}_llamafactory_{kind}")
+ os.makedirs(folder, exist_ok=True)
+ pd.DataFrame(rowsk).to_parquet(os.path.join(folder, "train.parquet"), index=False)
+ cols = {"messages": "conversations", "system": "system"}
+ if kind != "text":
+ cols["images"] = "images"
+ entry = {"file_name": "train.parquet", "formatting": "sharegpt", "columns": cols}
+ json.dump({f"3dcv_{name}_{kind}": entry}, open(os.path.join(folder, "dataset_info.json"), "w"), indent=2)
+ json.dump({"subdir": a.subdir, "pair_type": kind, "rows": len(rowsk),
+ "source_samples": len(rows), "usable": len(got), **{k: v for k, v in qc.items()}},
+ open(os.path.join(folder, "qc.json"), "w"), indent=2)
+ info[kind] = len(rowsk)
+ print(f"[pairs] {kind:8s} {len(rowsk):>7,} rows -> {folder}", flush=True)
+ # the invariant LLaMA-Factory enforces on multimodal rows
+ bad = sum(1 for k in ("img", "imgtext") for r in out[k]
+ if r["conversations"][0]["value"].count("<image>") != len(r["images"]))
+ print(f"[pairs] placeholder/image mismatches: {bad} (must be 0)", flush=True)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/toolkits/llamafactory/convert_subdirs.py b/toolkits/llamafactory/convert_subdirs.py
index 739f04e8..c63f1a7a 100644
--- a/toolkits/llamafactory/convert_subdirs.py
+++ b/toolkits/llamafactory/convert_subdirs.py
@@ -193,8 +193,10 @@ def convert_one(args):
code = MEDIA.sub(lambda m: f"<{m.group(1)} >", code)
norm_name = r["name"].replace("Factory", "").replace("_geo", "").replace("_tex", "")
if r["name"] in bench_factories or r["name"].rsplit("_", 1)[0] in bench_factories or norm_name in bench_factories:
- qc["benchmark_overlap"] += 1 # flagged, and excluded below
- continue
+ # counted, never dropped: which samples a benchmark forbids is a decision for whoever defines the
+ # benchmark, and a converter that quietly removes 100% of a subdirectory (as this once did for both
+ # 3dcodebench/instances_*) is the wrong place to make it
+ qc["benchmark_overlap"] += 1
caps = normalize_caps(r["caps"])
n_before = len(out)
for kind in ("instruction", "detailed", "factory"):
@@ -222,7 +224,7 @@ def convert_one(args):
open(os.path.join(d, "dataset_info.json"), "w"), indent=2)
tok_lens.sort()
stat = {"subdir": sub, "dialect": dialect, "source_samples": len(rows), "exported_pairs": len(out),
- "unique_samples_exported": len(seen_code) - qc["benchmark_overlap"],
+ "unique_samples_exported": len(seen_code),
"images_per_sample_median": (sorted(img_counts)[len(img_counts) // 2] if img_counts else 0),
"tokens_p50": tok_lens[len(tok_lens) // 2] if tok_lens else 0,
"tokens_p90": tok_lens[int(len(tok_lens) * 0.9)] if tok_lens else 0,
@@ -248,7 +250,8 @@ Generated by `finetune/scripts/convert_subdirs.py` from `{sub}/metadata.parquet`
{issues}
`syntax_bad` rows are still exported (a shader can be valid without balanced braces in our crude check); every
-other counter means the sample was dropped. `benchmark_overlap` = the sample belongs to one of the 212
+other counter means the sample was dropped, EXCEPT `benchmark_overlap`, which is counted and kept: the
+sample belongs to one of the 212
3DCodeBench evaluation factories and is excluded to avoid test leakage.
## Use
@@ -266,10 +269,14 @@ def main():
ap.add_argument("--out", default="/wekafs/ict/hx_624/hf_subdirs")
ap.add_argument("--workers", type=int, default=16)
ap.add_argument("--limit_subdirs", type=int, default=None)
+ ap.add_argument("--only", default=None, help="comma-separated subdirs to convert")
a = ap.parse_args()
bench = frozenset(os.path.basename(p).replace("_seed0", "") for p in glob.glob(f"{BENCH_DIR}/*_seed0"))
subs = ["/".join(p.split("/")[-3:-1]) for p in sorted(glob.glob(f"{META}/*/*/metadata.parquet"))]
subs += ["blender_distill", "threejs_distill", "animation2code"]
+ if a.only:
+ want = {x.strip() for x in a.only.split(",") if x.strip()}
+ subs = [x for x in subs if x in want]
if a.limit_subdirs:
subs = subs[: a.limit_subdirs]
print(f"[conv] {len(subs)} subdirs -> {a.out}", flush=True)
diff --git a/toolkits/llamafactory/runners/run_3dcb_pairs.sh b/toolkits/llamafactory/runners/run_3dcb_pairs.sh
new file mode 100755
index 00000000..44ff4e69
--- /dev/null
+++ b/toolkits/llamafactory/runners/run_3dcb_pairs.sh
@@ -0,0 +1,32 @@
+#!/bin/bash
+# Build text->code, image->code and image+text->code for every 3DCodeBench subdirectory.
+# Renders come from a sequential scan of each tar, NOT from metadata's byte offsets: for instances_geo only 577
+# of 1,953 offsets point at a real tar header (the archives were repacked after the metadata was written), and
+# trusting them silently cost 70% of the image pairs.
+set -uo pipefail; cd /wekafs/ict/hx_624/llm-ft
+source /wekafs/ict/hx_624/anaconda3/etc/profile.d/conda.sh; conda activate llmft
+OUT=/wekafs/ict/hx_624/hf_pairs
+for SUB in instances_tex instances_geo_full instances_tex_full; do
+ # "the folder exists" is not the same as "it was built correctly": instances_tex was skipped by that test
+ # while holding the 578-of-1,951 image pairs the broken byte offsets produced. Require the img rows to match
+ # the source count before treating a subdir as done.
+ Q="$OUT/${SUB}_llamafactory_img/qc.json"
+ if [ -f "$Q" ] && python3 -c "import json,sys; q=json.load(open('$Q')); sys.exit(0 if q['rows'] >= 0.95*q['source_samples'] else 1)"; then
+ echo "[3dcb] $SUB already built with full render coverage — skipping"; continue
+ fi
+ M=/wekafs/ict/hx_624/data/3dcodeverse_meta/3dcodebench/$SUB/metadata.parquet
+ if [ ! -f "$M" ]; then
+ echo "[3dcb] fetching metadata for $SUB"
+ python - "$SUB" <<'PY'
+import os, sys, shutil
+from huggingface_hub import hf_hub_download
+sub = sys.argv[1]
+p = hf_hub_download("ilabai/3dcodeverse", f"3dcodebench/{sub}/metadata.parquet", repo_type="dataset", token=os.environ["HF_TOKEN"])
+d = f"/wekafs/ict/hx_624/data/3dcodeverse_meta/3dcodebench/{sub}"
+os.makedirs(d, exist_ok=True); shutil.copy(p, f"{d}/metadata.parquet")
+PY
+ fi
+ echo "[3dcb] $(date) building $SUB"
+ python scripts/build_pairs_3dcodebench.py --subdir 3dcodebench/$SUB --out $OUT --workers 16 --views 4 2>&1 | tail -6
+done
+echo "[3dcb] ALL DONE"
diff --git a/toolkits/llamafactory/runners/run_shadertoy_pairs.sh b/toolkits/llamafactory/runners/run_shadertoy_pairs.sh
new file mode 100755
index 00000000..b24896b9
--- /dev/null
+++ b/toolkits/llamafactory/runners/run_shadertoy_pairs.sh
@@ -0,0 +1,25 @@
+#!/bin/bash
+# All three pair types for every shadertoy shard. Renders come from scanning each shard's tars (the byte offsets
+# in metadata are not reliable), and the code comes from whichever verified parquet exists for that shard --
+# *_llamafactory_flat for the 68 that could be flattened, the separately verified tree for the other 44 -- so the
+# pairs carry a compiles column instead of being unverified GLSL.
+set -uo pipefail; cd /wekafs/ict/hx_624/llm-ft
+source /wekafs/ict/hx_624/anaconda3/etc/profile.d/conda.sh; conda activate llmft
+OUT=/wekafs/ict/hx_624/hf_pairs_shadertoy
+VER="/wekafs/ict/hx_624/hf_subdirs/shadertoy,/wekafs/ict/hx_624/hf_subdirs_verified/shadertoy"
+PAR=${PAR:-4}
+mkdir -p $OUT
+build () {
+ local SUB=$1
+ local Q="$OUT/${SUB}_llamafactory_img/qc.json"
+ if [ -f "$Q" ]; then echo "[st] $SUB already built — skipping"; return; fi
+ python scripts/build_pairs_3dcodebench.py --subdir shadertoy/$SUB --dialect glsl \
+ --out $OUT --workers 12 --views 4 --verified_from "$VER" > logs/pairs_st_$SUB.log 2>&1
+ local line=$(grep -E "^\[pairs\] (text|img|imgtext)" logs/pairs_st_$SUB.log | tr '\n' ' ' | tr -s ' ')
+ echo "[st] $(date +%H:%M) $SUB ${line:-FAILED: $(grep -iE 'error|Traceback' logs/pairs_st_$SUB.log | tail -1 | cut -c1-90)}"
+}
+export -f build; export OUT VER
+SHARDS=$(ls -d /wekafs/ict/hx_624/data/3dcodeverse_meta/shadertoy/*/ | xargs -n1 basename)
+echo "[st] $(echo "$SHARDS" | wc -l) shards, $PAR at a time"
+echo "$SHARDS" | xargs -P $PAR -I{} bash -c 'build "$@"' _ {}
+echo "[st] ALL DONE"
diff --git a/toolkits/llamafactory/verify_unflattened_glsl.py b/toolkits/llamafactory/verify_unflattened_glsl.py
new file mode 100644
index 00000000..5bb8dc33
--- /dev/null
+++ b/toolkits/llamafactory/verify_unflattened_glsl.py
@@ -0,0 +1,108 @@
+"""Compile-verify the 44 shadertoy shards that the flattening pass never reached.
+
+Those shards' metadata carries no `shader_json`, so there was no renderpass structure to merge a Common tab
+from and `flatten_multifile.py` skipped them — exactly 44 of 112, with zero mismatches. The consequence is that
+73,281 shaders are verified (99.04% compile) while 46,680 have never been near a compiler. This runs the same
+checker and the same minimal repairs over those, and writes the `compiles` / `repair` columns the flattened
+folders already carry, so a training mix can filter the whole corpus by one rule instead of two.
+
+usage: python scripts/verify_unflattened_glsl.py [--workers 40] [--limit_shards N] [--out DIR]
+"""
+import argparse
+import collections
+import glob
+import json
+import os
+import re
+import sys
+
+import pandas as pd
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from qc_repair import glsl_compile, repair_glsl
+
+SRC = "/wekafs/ict/hx_624/hf_subdirs/shadertoy"
+FENCE = re.compile(r"^```[a-zA-Z]*\n|\n?```$")
+
+
+def strip_fence(v):
+ return FENCE.sub("", v.strip())
+
+
+def verify_one(code):
+ """-> (compiles, repair_label, code_to_keep). Never returns a program that does not compile as 'repaired'."""
+ ok, _ = glsl_compile(code)
+ if ok:
+ return True, "original", code
+ try:
+ fixed, recipe = repair_glsl(code)
+ except Exception:
+ return False, "unfixable", code
+ if fixed and fixed != code:
+ ok2, _ = glsl_compile(fixed)
+ if ok2:
+ return True, f"repaired:{recipe or 'other'}", fixed
+ return False, "unfixable", code
+
+
+def do_shard(args):
+ sub, out_root = args
+ p = f"{SRC}/{sub}_llamafactory/train.parquet"
+ df = pd.read_parquet(p)
+ # one shader appears once per caption variant; compile each distinct program once
+ by_code = {}
+ codes = [strip_fence(c[1]["value"]) for c in df["conversations"]]
+ for c in set(codes):
+ by_code[c] = verify_one(c)
+ okc, howc, conv = [], [], []
+ for row, c in zip(df.itertuples(), codes):
+ ok, how, keep = by_code[c]
+ okc.append(ok); howc.append(how)
+ turns = [dict(t) for t in row.conversations]
+ turns[1] = dict(turns[1], value=f"```glsl\n{keep}\n```")
+ conv.append(turns)
+ df["conversations"] = conv
+ df["compiles"] = okc
+ df["repair"] = howc
+ d = os.path.join(out_root, f"{sub}_llamafactory")
+ os.makedirs(d, exist_ok=True)
+ df.to_parquet(os.path.join(d, "train.parquet"), index=False)
+ for extra in ("dataset_info.json", "qc.json", "README.md"):
+ s = f"{SRC}/{sub}_llamafactory/{extra}"
+ if os.path.exists(s):
+ open(os.path.join(d, extra), "w").write(open(s).read())
+ st = {"shard": sub, "rows": len(df), "unique_programs": len(by_code),
+ "compiles": int(sum(okc)), "repaired": int(sum(1 for h in howc if h.startswith("repaired"))),
+ "unfixable": int(sum(1 for h in howc if h == "unfixable"))}
+ json.dump(st, open(os.path.join(d, "verify.json"), "w"), indent=2)
+ return st
+
+
+def main():
+ ap = argparse.ArgumentParser()
+ ap.add_argument("--workers", type=int, default=40)
+ ap.add_argument("--limit_shards", type=int, default=None)
+ ap.add_argument("--out", default="/wekafs/ict/hx_624/hf_subdirs_verified/shadertoy")
+ a = ap.parse_args()
+ flat = {os.path.basename(p)[: -len("_llamafactory_flat")] for p in glob.glob(f"{SRC}/*_llamafactory_flat")}
+ allp = {os.path.basename(p)[: -len("_llamafactory")] for p in glob.glob(f"{SRC}/*_llamafactory")}
+ todo = sorted(allp - flat)
+ if a.limit_shards:
+ todo = todo[: a.limit_shards]
+ os.makedirs(a.out, exist_ok=True)
+ print(f"[verify] {len(todo)} shards never verified -> {a.out}", flush=True)
+ from concurrent.futures import ProcessPoolExecutor
+ tot = collections.Counter()
+ with ProcessPoolExecutor(a.workers) as ex:
+ for st in ex.map(do_shard, [(s, a.out) for s in todo]):
+ for k in ("rows", "unique_programs", "compiles", "repaired", "unfixable"):
+ tot[k] += st[k]
+ print(f"[verify] {st['shard']:4s} rows={st['rows']:>6,} uniq={st['unique_programs']:>5,} "
+ f"ok={st['compiles']:>6,} repaired={st['repaired']:>4,} unfixable={st['unfixable']:>4,}", flush=True)
+ rate = tot["compiles"] / max(tot["rows"], 1) * 100
+ print(f"[verify] TOTAL rows={tot['rows']:,} unique={tot['unique_programs']:,} "
+ f"compiles={tot['compiles']:,} ({rate:.2f}%) repaired={tot['repaired']:,} unfixable={tot['unfixable']:,}", flush=True)
+
+
+if __name__ == "__main__":
+ main()