Skip to content

Official Codebase of 3DCodeVerse

Python

finetune: per-subdir LLaMA-Factory conversion + QC (convert_subdirs.py), compiler-verified GLSL validation/repair (qc_repair.py), multi-file flattening and URDF recovery from tars (flatten_multifile.py), final consolidation (finalize_glsl.py)

gaoypeng1a5844bce924
diff --git a/finetune/configs/lf/lora_27b_v4.yaml b/finetune/configs/lf/lora_27b_v4.yaml
new file mode 100644
index 00000000..85f31697
--- /dev/null
+++ b/finetune/configs/lf/lora_27b_v4.yaml
@@ -0,0 +1,44 @@
+model_name_or_path: /wekafs/ict/hx_624/models/Qwen3.8-27B
+trust_remote_code: true
+flash_attn: fa2
+stage: sft
+do_train: true
+finetuning_type: lora
+lora_rank: 64
+lora_alpha: 128
+lora_dropout: 0.05
+lora_target: all
+freeze_vision_tower: true
+freeze_multi_modal_projector: true
+dataset_dir: /wekafs/ict/hx_624/llm-ft/data/lf
+dataset: md_27b_v4_train
+eval_dataset: md_27b_v4_val
+template: qwen3_5
+enable_thinking: false
+cutoff_len: 4096
+preprocessing_num_workers: 16
+dataloader_num_workers: 4
+neat_packing: true
+packing: true
+tokenized_path: /wekafs/ict/hx_624/llm-ft/data/lf/tokenized_md_27b_v4_len4096_packed
+output_dir: /wekafs/ict/hx_624/llm-ft/runs/lf_qwen38_27b_lora_v4
+logging_steps: 5
+save_steps: 300
+save_only_model: true
+plot_loss: true
+overwrite_output_dir: true
+report_to: tensorboard
+per_device_train_batch_size: 1
+gradient_accumulation_steps: 4
+learning_rate: 0.0001
+num_train_epochs: 1.0
+lr_scheduler_type: cosine
+warmup_ratio: 0.05
+bf16: true
+gradient_checkpointing: true
+ddp_timeout: 180000000
+resume_from_checkpoint: null
+per_device_eval_batch_size: 1
+eval_strategy: steps
+eval_steps: 300
+save_total_limit: 8
diff --git a/finetune/configs/lf/lora_27b_z3_8gpu.yaml b/finetune/configs/lf/lora_27b_z3_8gpu.yaml
new file mode 100644
index 00000000..c65922a1
--- /dev/null
+++ b/finetune/configs/lf/lora_27b_z3_8gpu.yaml
@@ -0,0 +1,45 @@
+model_name_or_path: /wekafs/ict/hx_624/models/Qwen3.8-27B
+trust_remote_code: true
+flash_attn: fa2
+stage: sft
+do_train: true
+finetuning_type: lora
+lora_rank: 64
+lora_alpha: 128
+lora_dropout: 0.05
+lora_target: all
+freeze_vision_tower: true
+freeze_multi_modal_projector: true
+dataset_dir: /wekafs/ict/hx_624/llm-ft/data/lf
+dataset: md_27b_v2s_train
+eval_dataset: md_27b_v2s_val
+template: qwen3_5
+enable_thinking: false
+cutoff_len: 8192
+preprocessing_num_workers: 16
+dataloader_num_workers: 4
+neat_packing: true
+packing: true
+tokenized_path: /wekafs/ict/hx_624/llm-ft/data/lf/tokenized_md_27b_v2s_len8192_packed
+output_dir: /wekafs/ict/hx_624/llm-ft/runs/lf_qwen38_27b_lora_v4
+logging_steps: 5
+save_steps: 250
+save_only_model: true
+plot_loss: true
+overwrite_output_dir: true
+report_to: tensorboard
+per_device_train_batch_size: 1
+gradient_accumulation_steps: 1
+learning_rate: 0.0001
+num_train_epochs: 1.0
+lr_scheduler_type: cosine
+warmup_ratio: 0.05
+bf16: true
+gradient_checkpointing: true
+ddp_timeout: 180000000
+resume_from_checkpoint: null
+per_device_eval_batch_size: 1
+eval_strategy: steps
+eval_steps: 250
+save_total_limit: 8
+deepspeed: /wekafs/ict/hx_624/tools/LLaMA-Factory/examples/deepspeed/ds_z3_config.json
diff --git a/finetune/data_specs/dataset_info.json b/finetune/data_specs/dataset_info.json
index 5493670c..a876a31b 100644
--- a/finetune/data_specs/dataset_info.json
+++ b/finetune/data_specs/dataset_info.json
@@ -507,5 +507,21 @@
"chosen": "chosen",
"rejected": "rejected"
}
+ },
+ "md_27b_v4_train": {
+ "file_name": "md_27b_v4_train.json",
+ "formatting": "sharegpt",
+ "columns": {
+ "messages": "conversations",
+ "system": "system"
+ }
+ },
+ "md_27b_v4_val": {
+ "file_name": "md_27b_v4_val.json",
+ "formatting": "sharegpt",
+ "columns": {
+ "messages": "conversations",
+ "system": "system"
+ }
}
}
\ No newline at end of file
diff --git a/finetune/scripts/convert_subdirs.py b/finetune/scripts/convert_subdirs.py
new file mode 100644
index 00000000..e0446619
--- /dev/null
+++ b/finetune/scripts/convert_subdirs.py
@@ -0,0 +1,293 @@
+"""Convert EVERY 3DCodeVerse subdirectory into its own LLaMA-Factory dataset + a data-quality report.
+
+One output folder per source subdir, named `<subdir>_llamafactory/`, so it can sit next to the source on the
+Hub (e.g. `3dcodebench/factories_geo/` → `3dcodebench/factories_geo_llamafactory/`):
+
+ <subdir>_llamafactory/
+ train.json ShareGPT: [{system, conversations:[human, gpt]}] — one row per caption variant
+ dataset_info.json drop-in LLaMA-Factory registry entry for this folder
+ qc.json per-subdir quality counters + token stats
+ README.md what it is, how many rows, what was dropped and why
+
+Quality checks applied per sample (a sample can fail several):
+ empty_code / short_code (<40 chars) / bad_chars (control chars or U+FFFD) / no_caption /
+ duplicate_code (same code hash inside the subdir) / syntax_bad (AST for Python dialects, brace balance for
+ scad-glsl, tag check for html) / too_long (>8192 tokens, still exported but flagged) / media_placeholder
+ (literal <image>/<video>/<audio> — breaks the qwen3_5 template unless escaped; we escape it and flag it)
+
+usage: python scripts/convert_subdirs.py [--out DIR] [--workers 16] [--limit_subdirs N]
+"""
+import argparse
+import ast
+import collections
+import glob
+import hashlib
+import json
+import os
+import re
+import sys
+import unicodedata
+from concurrent.futures import ProcessPoolExecutor
+
+import pandas as pd
+
+sys.path.insert(0, "/wekafs/ict/hx_624/llm-ft/scripts")
+META = "/wekafs/ict/hx_624/data/3dcodeverse_meta"
+FILES = "/wekafs/ict/hx_624/data/3dcodeverse_files"
+BENCH_DIR = "/wekafs/ict/hx_624/data/3dcodebench/data"
+
+# subdir prefix -> dialect
+DIALECT_OF = {
+ "3dcodebench": "blender", "bioinspired3d": "blender", "blender_distill": "blender",
+ "deepcad": "cadquery", "articraft": "cadquery", "shadertoy": "glsl",
+ "thingiverse": "openscad", "threejs_distill": "threejs", "animation2code": "web",
+}
+SYS = {
+ "blender": "You are an expert in procedural 3D modeling with Blender Python (bpy). Given a description of an object, write a complete, standalone Blender 5 Python script that builds it from scratch (clear the default scene first). Output ONLY the code in one ```python block.",
+ "cadquery": "You are an expert CAD programmer. Given a description of a part, write a complete, standalone CadQuery (Python) script that builds it and leaves the final solid in a variable named `result`. Output ONLY the code in one ```python block.",
+ "openscad": "You are an expert OpenSCAD programmer. Given a request, write a complete, standalone OpenSCAD (.scad) program that produces the described model. Output ONLY the code in one ```openscad block.",
+ "glsl": "You are an expert shader programmer. Given a description, write a complete Shadertoy-style GLSL ES 3.00 fragment shader defining `void mainImage(out vec4 fragColor, in vec2 fragCoord)`. Output ONLY the code in one ```glsl block.",
+ "threejs": "You are an expert three.js developer. Given a description, write a complete, standalone HTML page that renders the described 3D scene with three.js. Output ONLY the code in one ```html block.",
+ "web": "You are an expert creative web developer. Given a description of an animation, write a complete, standalone HTML page (inline CSS/JS, no local assets) that reproduces it. Output ONLY the code in one ```html block.",
+ "urdf": "You are an expert in articulated 3D assets. Given a description, write a complete, self-contained URDF file for it. Output ONLY the code in one ```xml block.",
+}
+FENCE = {"blender": "python", "cadquery": "python", "openscad": "openscad", "glsl": "glsl",
+ "threejs": "html", "web": "html", "urdf": "xml"}
+LEAD = {
+ "blender": "Create the 3D object described below using Python Blender code: ",
+ "cadquery": "Write a CadQuery (Python) script that builds the following CAD model: ",
+ "openscad": "Write an OpenSCAD program for the following: ",
+ "glsl": "Write a GLSL fragment shader for Shadertoy that renders: ",
+ "threejs": "Write a standalone three.js page that renders: ",
+ "web": "Write a standalone animated web page that reproduces: ",
+ "urdf": "Write a URDF description of: ",
+}
+IMPERATIVE = ("write", "create", "implement", "make", "build", "generate", "design", "produce")
+MEDIA = re.compile(r"<(image|video|audio)>")
+
+
+def bad_char_count(s):
+ return sum(1 for ch in s if unicodedata.category(ch) == "Cc" and ch not in "\n\t\r") + s.count("�")
+
+
+def syntax_ok(code, dialect):
+ if dialect in ("blender", "cadquery"):
+ try:
+ ast.parse(code)
+ return True
+ except Exception:
+ return False
+ if dialect == "openscad":
+ return code.count("{") == code.count("}") and code.count("(") == code.count(")")
+ if dialect == "glsl":
+ return code.count("{") == code.count("}") and "void" in code
+ if dialect in ("threejs", "web"):
+ return "<" in code and ">" in code
+ if dialect == "urdf":
+ return "<robot" in code or "<link" in code
+ return True
+
+
+def phrase(dialect, kind, text):
+ t = (text or "").strip()
+ if not t:
+ return None
+ if kind == "instruction" and t.lower().startswith(IMPERATIVE):
+ return t
+ return LEAD[dialect] + t
+
+
+def normalize_caps(caps):
+ """tree sources use their own caption keys"""
+ caps = {k: v for k, v in caps.items() if isinstance(v, str) and v.strip()}
+ if any(k in caps for k in ("instruction", "detailed", "factory")) and "user_instruction" not in caps and "brief" not in caps:
+ return caps
+ norm = {}
+ if caps.get("user_instruction"):
+ norm["instruction"] = caps["user_instruction"]
+ if caps.get("instruction"):
+ norm["detailed"] = caps["instruction"]
+ if caps.get("brief"):
+ extra = caps.get("function", "")
+ norm["instruction"] = (caps["brief"] + ((" — " + extra) if extra and extra not in caps["brief"] else "")).strip()
+ if caps.get("source_title") and "detailed" not in norm:
+ norm["detailed"] = caps["source_title"]
+ if caps.get("style") and norm.get("detailed"):
+ norm["detailed"] += f" (style: {caps['style']})"
+ return norm or caps
+
+
+def load_subdir(sub):
+ """-> (dialect, [ {id, name, code, caps, n_images} ]) for a parquet subdir or a file tree"""
+ top = sub.split("/")[0]
+ dialect = DIALECT_OF.get(top, "blender")
+ if top == "articraft" and "urdf" in sub:
+ dialect = "urdf"
+ rows = []
+ p = f"{META}/{sub}/metadata.parquet"
+ if os.path.exists(p):
+ df = pd.read_parquet(p)
+ for r in df.itertuples():
+ caps = r.captions if isinstance(r.captions, dict) else (json.loads(r.captions) if isinstance(r.captions, str) else {})
+ meta = {}
+ try:
+ meta = json.loads(r.meta_json) if isinstance(r.meta_json, str) else (r.meta_json or {})
+ except Exception:
+ pass
+ rows.append({"id": str(r.id), "name": str(getattr(r, "name", "") or ""),
+ "code": getattr(r, "code", None) or "", "caps": dict(caps),
+ "n_images": len(meta.get("renders", []) or [])})
+ return dialect, rows
+ # file tree
+ for cp in sorted(glob.glob(f"{FILES}/{sub}/*/code.py") + glob.glob(f"{FILES}/{sub}/*/code.html")
+ + glob.glob(f"{FILES}/{sub}/*/index.html") + glob.glob(f"{FILES}/{sub}/*/*/code.py")
+ + glob.glob(f"{FILES}/{sub}/*/*/code.html") + glob.glob(f"{FILES}/{sub}/*/*/index.html")):
+ d = os.path.dirname(cp)
+ try:
+ code = open(cp, encoding="utf-8", errors="replace").read()
+ caps = json.load(open(os.path.join(d, "captions.json"))) if os.path.exists(os.path.join(d, "captions.json")) else {}
+ except Exception:
+ continue
+ rows.append({"id": os.path.relpath(d, FILES), "name": os.path.basename(d), "code": code,
+ "caps": caps, "n_images": len(glob.glob(os.path.join(d, "renders", "*.png")))})
+ return dialect, rows
+
+
+_TOK = None
+
+
+def _init():
+ global _TOK
+ from transformers import AutoTokenizer
+ _TOK = AutoTokenizer.from_pretrained("/wekafs/ict/hx_624/models/Qwen3.5-9B")
+
+
+def convert_one(args):
+ sub, out_root, bench_factories = args
+ dialect, rows = load_subdir(sub)
+ if not rows:
+ return {"subdir": sub, "error": "no rows"}
+ qc = collections.Counter()
+ seen_code, out, tok_lens, img_counts = {}, [], [], []
+ for r in rows:
+ code = r["code"] if isinstance(r["code"], str) else ""
+ img_counts.append(r.get("n_images", 0))
+ if not code.strip():
+ qc["empty_code"] += 1
+ continue
+ if len(code.strip()) < 40:
+ qc["short_code"] += 1
+ continue
+ if bad_char_count(code) > 5:
+ qc["bad_chars"] += 1
+ continue
+ h = hashlib.md5(code.encode()).hexdigest()
+ if h in seen_code:
+ qc["duplicate_code"] += 1
+ continue
+ seen_code[h] = 1
+ if not syntax_ok(code, dialect):
+ qc["syntax_bad"] += 1 # exported anyway, but counted
+ if MEDIA.search(code) or any(MEDIA.search(v) for v in r["caps"].values() if isinstance(v, str)):
+ qc["media_placeholder"] += 1
+ code = MEDIA.sub(lambda m: f"<{m.group(1)} >", code)
+ if r["name"] in bench_factories or r["name"].rsplit("_", 1)[0] in bench_factories:
+ qc["benchmark_overlap"] += 1 # flagged, and excluded below
+ continue
+ caps = normalize_caps(r["caps"])
+ n_before = len(out)
+ for kind in ("instruction", "detailed", "factory"):
+ pr = phrase(dialect, kind, caps.get(kind))
+ if not pr or len(pr) < 20:
+ continue
+ pr = MEDIA.sub(lambda m: f"<{m.group(1)} >", pr)
+ n = len(_TOK(pr).input_ids) + len(_TOK(code).input_ids) + 60
+ tok_lens.append(n)
+ if n > 8192:
+ qc["too_long_gt8192"] += 1
+ out.append({"id": r["id"], "caption_type": kind, "n_tokens": n,
+ "system": SYS[dialect],
+ "conversations": [{"from": "human", "value": pr},
+ {"from": "gpt", "value": f"```{FENCE[dialect]}\n{code.strip()}\n```"}]})
+ if len(out) == n_before:
+ qc["no_caption"] += 1
+ name = sub.replace("/", "__")
+ d = os.path.join(out_root, sub + "_llamafactory")
+ os.makedirs(d, exist_ok=True)
+ json.dump(out, open(os.path.join(d, "train.json"), "w"), ensure_ascii=False)
+ key = f"3dcv_{name}"
+ json.dump({key: {"file_name": "train.json", "formatting": "sharegpt",
+ "columns": {"messages": "conversations", "system": "system"}}},
+ open(os.path.join(d, "dataset_info.json"), "w"), indent=2)
+ tok_lens.sort()
+ stat = {"subdir": sub, "dialect": dialect, "source_samples": len(rows), "exported_pairs": len(out),
+ "unique_samples_exported": len(seen_code) - qc["benchmark_overlap"],
+ "images_per_sample_median": (sorted(img_counts)[len(img_counts) // 2] if img_counts else 0),
+ "tokens_p50": tok_lens[len(tok_lens) // 2] if tok_lens else 0,
+ "tokens_p90": tok_lens[int(len(tok_lens) * 0.9)] if tok_lens else 0,
+ "tokens_max": tok_lens[-1] if tok_lens else 0,
+ **{k: v for k, v in qc.items()}}
+ json.dump(stat, open(os.path.join(d, "qc.json"), "w"), indent=2)
+ issues = "\n".join(f"- **{k}**: {v}" for k, v in sorted(qc.items(), key=lambda x: -x[1])) or "- none"
+ open(os.path.join(d, "README.md"), "w").write(f"""# {sub} — LLaMA-Factory format
+
+Generated by `finetune/scripts/convert_subdirs.py` from `{sub}/metadata.parquet` (or its file tree).
+
+| | |
+|---|---|
+| dialect | `{dialect}` |
+| source samples | {len(rows):,} |
+| exported pairs | {len(out):,} (one row per caption variant: instruction / detailed / factory) |
+| unique code samples exported | {stat['unique_samples_exported']:,} |
+| token length p50 / p90 / max | {stat['tokens_p50']} / {stat['tokens_p90']} / {stat['tokens_max']} |
+| renders per sample (median) | {stat['images_per_sample_median']} |
+
+## Data-quality findings
+
+{issues}
+
+`syntax_bad` rows are still exported (a shader can be valid without balanced braces in our crude check); every
+other counter means the sample was dropped. `benchmark_overlap` = the sample belongs to one of the 212
+3DCodeBench evaluation factories and is excluded to avoid test leakage.
+
+## Use
+
+```yaml
+dataset_dir: <this folder>
+dataset: {key}
+```
+""")
+ return stat
+
+
+def main():
+ ap = argparse.ArgumentParser()
+ ap.add_argument("--out", default="/wekafs/ict/hx_624/hf_subdirs")
+ ap.add_argument("--workers", type=int, default=16)
+ ap.add_argument("--limit_subdirs", type=int, default=None)
+ a = ap.parse_args()
+ bench = frozenset(os.path.basename(p).replace("_seed0", "") for p in glob.glob(f"{BENCH_DIR}/*_seed0"))
+ subs = ["/".join(p.split("/")[-3:-1]) for p in sorted(glob.glob(f"{META}/*/*/metadata.parquet"))]
+ subs += ["blender_distill", "threejs_distill", "animation2code"]
+ if a.limit_subdirs:
+ subs = subs[: a.limit_subdirs]
+ print(f"[conv] {len(subs)} subdirs -> {a.out}", flush=True)
+ os.makedirs(a.out, exist_ok=True)
+ stats = []
+ with ProcessPoolExecutor(a.workers, initializer=_init) as ex:
+ for st in ex.map(convert_one, [(s, a.out, bench) for s in subs]):
+ stats.append(st)
+ print(f"[conv] {st.get('subdir'):45s} pairs={st.get('exported_pairs', 0):7,} "
+ f"issues={ {k: v for k, v in st.items() if k in ('empty_code','short_code','bad_chars','duplicate_code','syntax_bad','no_caption','too_long_gt8192','media_placeholder','benchmark_overlap')} }", flush=True)
+ df = pd.DataFrame(stats)
+ df.to_csv(os.path.join(a.out, "quality_report.csv"), index=False)
+ try:
+ df.to_excel(os.path.join(a.out, "quality_report.xlsx"), index=False)
+ except Exception as e:
+ print("[conv] xlsx skipped:", e)
+ json.dump(stats, open(os.path.join(a.out, "quality_report.json"), "w"), indent=2)
+ print(f"[conv] wrote quality_report.{{csv,xlsx,json}} | total exported pairs: {int(df['exported_pairs'].sum()):,}")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/finetune/scripts/dgx01_smoke.sh b/finetune/scripts/dgx01_smoke.sh
new file mode 100755
index 00000000..e1813c77
--- /dev/null
+++ b/finetune/scripts/dgx01_smoke.sh
@@ -0,0 +1,8 @@
+#!/bin/bash
+# 4-step smoke test of the 27B LoRA recipe on another node (the project dir is on shared wekafs, so
+# scripts/env/models/data are all identical — only the GPUs differ).
+cd /wekafs/ict/hx_624/llm-ft || exit 1
+GPUS=${GPUS:-0,1,2,3} scripts/lf_train.sh configs/lf/lora_27b_v2.yaml max_steps=4 save_steps=100000 eval_steps=100000 \
+ output_dir=/wekafs/ict/hx_624/llm-ft/runs/_smoke_$(hostname) > logs/smoke_$(hostname).log 2>&1
+grep -oE "'train_runtime': [0-9.]+|out of memory|[0-9]/4 \[[0-9:]+<[0-9:]+, *[0-9.]+s/it" logs/smoke_$(hostname).log | tail -2
+rm -rf runs/_smoke_$(hostname)
diff --git a/finetune/scripts/finalize_glsl.py b/finetune/scripts/finalize_glsl.py
new file mode 100644
index 00000000..bbc883c1
--- /dev/null
+++ b/finetune/scripts/finalize_glsl.py
@@ -0,0 +1,85 @@
+"""Final consolidation for the flattened Shadertoy corpus.
+
+Order of preference, so the data stays as close to the author's source as possible:
+ 1. the ORIGINAL merged program, if the (now correct) validator compiles it -> untouched
+ 2. the agent-repaired program from glsl_repair/<cat>_fixed/, if it compiles -> repaired
+ 3. otherwise -> kept, marked compiles=False
+Writes the `compiles` / `repair` columns back into every shadertoy *_llamafactory_flat parquet.
+"""
+import collections, glob, hashlib, json, os, re, sys
+from concurrent.futures import ProcessPoolExecutor
+import pandas as pd
+sys.path.insert(0, "/wekafs/ict/hx_624/llm-ft/scripts")
+from qc_repair import code_of, glsl_compile
+
+REPAIR = "/wekafs/ict/hx_624/glsl_repair"
+
+def load_fixed():
+ out = {}
+ for cat in ("overload", "undeclared", "const_init", "syntax", "redefinition", "type", "other"):
+ d = f"{REPAIR}/{cat}_fixed"
+ idx = f"{REPAIR}/{cat}/index.json"
+ if not (os.path.isdir(d) and os.path.exists(idx)):
+ continue
+ meta = json.load(open(idx))
+ for f in glob.glob(f"{d}/*.glsl"):
+ stem = os.path.basename(f)[:-5]
+ h = (meta.get(stem) or {}).get("hash")
+ if h:
+ out[h] = (cat, open(f).read())
+ return out
+
+def job(args):
+ h, body, fixed = args
+ ok, _ = glsl_compile(body)
+ if ok:
+ return h, True, "original", None
+ if fixed:
+ cat, code = fixed
+ ok2, _ = glsl_compile(code)
+ if ok2:
+ return h, True, f"repaired:{cat}", code
+ return h, False, "unfixable", None
+
+def main():
+ fixed = load_fixed()
+ print(f"[fin] agent repairs available: {len(fixed)}", flush=True)
+ dirs = sorted(glob.glob("/wekafs/ict/hx_624/hf_subdirs/shadertoy/*_llamafactory_flat"))
+ uniq, where = {}, collections.defaultdict(list)
+ for d in dirs:
+ df = pd.read_parquet(os.path.join(d, "train.parquet"))
+ for i, row in enumerate(df.itertuples()):
+ b = code_of(row.conversations[1]["value"]); h = hashlib.md5(b.encode()).hexdigest()
+ uniq.setdefault(h, b); where[h].append(d)
+ items = [(h, b, fixed.get(h)) for h, b in uniq.items()]
+ print(f"[fin] validating {len(items)} unique programs", flush=True)
+ res = {}
+ with ProcessPoolExecutor(28) as ex:
+ for n, (h, ok, how, code) in enumerate(ex.map(job, items, chunksize=8)):
+ res[h] = (ok, how, code)
+ if (n+1) % 10000 == 0:
+ g = sum(1 for v in res.values() if v[0])
+ print(f"[fin] {n+1}/{len(items)} ok={g} ({100*g/(n+1):.1f}%)", flush=True)
+ stats = collections.Counter(v[1] for v in res.values())
+ for d in dirs:
+ p = os.path.join(d, "train.parquet"); df = pd.read_parquet(p)
+ conv = df["conversations"].tolist(); okc, howc = [], []
+ for c in conv:
+ b = code_of(c[1]["value"]); h = hashlib.md5(b.encode()).hexdigest()
+ ok, how, code = res.get(h, (False, "unknown", None))
+ if code:
+ c[1]["value"] = "```glsl\n" + code.strip() + "\n```"
+ okc.append(ok); howc.append(how)
+ df["conversations"] = conv; df["compiles"] = okc; df["repair"] = howc
+ df.to_parquet(p, index=False, compression="zstd")
+ json.dump({"rows": len(df), "compiles": int(sum(okc)),
+ "repaired": int(sum(1 for x in howc if x.startswith("repaired")))},
+ open(os.path.join(d, "qc.json"), "w"), indent=2)
+ good = sum(1 for v in res.values() if v[0])
+ json.dump({"unique_programs": len(items), "compile_ok": good,
+ "compile_rate": round(good/len(items), 4), "how": dict(stats)},
+ open("/wekafs/ict/hx_624/hf_subdirs/glsl_final_report.json", "w"), indent=2)
+ print(f"[fin] FINAL {good}/{len(items)} = {100*good/len(items):.1f}% | {dict(stats)}", flush=True)
+
+if __name__ == "__main__":
+ main()
diff --git a/finetune/scripts/flatten_multifile.py b/finetune/scripts/flatten_multifile.py
new file mode 100644
index 00000000..23e9af06
--- /dev/null
+++ b/finetune/scripts/flatten_multifile.py
@@ -0,0 +1,181 @@
+"""Flatten every multi-file sample into ONE self-contained file, so the whole corpus is single-file SFT data.
+
+Two sources are multi-file in the original dataset and both can be flattened exactly:
+
+ shadertoy `shader_json.renderpass` holds the source of every tab. Shadertoy itself compiles
+ `Common + <pass>`, so we emit that concatenation verbatim:
+ * the Image pass with Common prepended → the sample's main program
+ * each Buffer pass with Common prepended → an extra sample (it is a complete shader too)
+ A shader whose passes read `iChannel` as a *buffer* is still emitted, with a one-line comment
+ stating which channels are buffers, because the code itself is complete.
+ articraft `code` is empty in the metadata; the real `code.urdf` lives inside the subdir tars, which are
+ downloaded separately and extracted by `--urdf_tars`.
+
+Everything is then re-emitted in LLaMA-Factory ShareGPT format next to the other converted subdirs.
+
+usage:
+ python scripts/flatten_multifile.py --shadertoy [--out /wekafs/ict/hx_624/hf_subdirs]
+ python scripts/flatten_multifile.py --urdf_tars /wekafs/ict/hx_624/data/3dcodeverse_tars
+"""
+import argparse
+import collections
+import glob
+import io
+import json
+import os
+import re
+import tarfile
+
+import pandas as pd
+
+META = "/wekafs/ict/hx_624/data/3dcodeverse_meta"
+SYS_GLSL = ("You are an expert shader programmer. Given a description, write a complete Shadertoy-style GLSL ES 3.00 "
+ "fragment shader defining `void mainImage(out vec4 fragColor, in vec2 fragCoord)`. Output ONLY the code in one ```glsl block.")
+SYS_URDF = ("You are an expert in articulated 3D assets. Given a description, write a complete, self-contained URDF "
+ "file (links, joints, inertial and visual geometry) for it. Output ONLY the code in one ```xml block.")
+IMPERATIVE = ("write", "create", "implement", "make", "build", "generate", "design", "produce")
+
+
+def phrase(kind, text, lead):
+ t = (text or "").strip()
+ if not t:
+ return None
+ if kind == "instruction" and t.lower().startswith(IMPERATIVE):
+ return t
+ return lead + t
+
+
+def flatten_shadertoy(out_root):
+ subs = sorted(glob.glob(f"{META}/shadertoy/*/metadata.parquet"))
+ stats = collections.Counter()
+ for p in subs:
+ sub = "/".join(p.split("/")[-3:-1])
+ df = pd.read_parquet(p)
+ rows = []
+ for r in df.itertuples():
+ try:
+ sj = json.loads(r.shader_json) if isinstance(r.shader_json, str) else (r.shader_json or {})
+ except Exception:
+ sj = {}
+ passes = (sj.get("Shader", sj) or {}).get("renderpass") or []
+ if not passes:
+ continue
+ common = "\n".join(q.get("code") or "" for q in passes if q.get("type") == "common").strip()
+ caps = r.captions if isinstance(r.captions, dict) else (json.loads(r.captions) if isinstance(r.captions, str) else {})
+ for q in passes:
+ t = q.get("type")
+ if t not in ("image", "buffer"):
+ continue # sound/cubemap passes are a different program shape
+ body = (q.get("code") or "").strip()
+ if len(body) < 40:
+ continue
+ buf_inputs = [i for i in (q.get("inputs") or []) if (i or {}).get("ctype") == "buffer"]
+ header = ""
+ if common:
+ header += "// ---- Common tab ----\n" + common + "\n// ---- " + str(q.get("name")) + " ----\n"
+ stats["merged_common"] += 1
+ if buf_inputs:
+ header = f"// note: iChannel{','.join(str(i.get('channel')) for i in buf_inputs)} read a previous buffer pass\n" + header
+ code = header + body
+ tag = "" if t == "image" else f"::{str(q.get('name')).replace(' ', '')}"
+ lead = "Write a GLSL fragment shader for Shadertoy that renders: "
+ if t == "buffer":
+ lead = "Write the buffer pass of a Shadertoy shader that computes: "
+ used = 0
+ for kind in ("instruction", "detailed", "factory"):
+ pr = phrase(kind, caps.get(kind), lead)
+ if not pr or len(pr) < 20:
+ continue
+ used += 1
+ rows.append({"id": f"{r.id}{tag}", "caption_type": kind, "pass_type": t,
+ "has_common": bool(common), "system": SYS_GLSL,
+ "conversations": [{"from": "human", "value": pr},
+ {"from": "gpt", "value": f"```glsl\n{code}\n```"}]})
+ stats["pass_" + t] += 1 if used else 0
+ if not rows:
+ continue
+ d = os.path.join(out_root, sub + "_llamafactory_flat")
+ os.makedirs(d, exist_ok=True)
+ pd.DataFrame(rows).to_parquet(os.path.join(d, "train.parquet"), index=False, compression="zstd")
+ key = "3dcv_" + sub.replace("/", "__") + "_flat"
+ json.dump({key: {"file_name": "train.parquet", "formatting": "sharegpt",
+ "columns": {"messages": "conversations", "system": "system"}}},
+ open(os.path.join(d, "dataset_info.json"), "w"), indent=2)
+ stats["rows"] += len(rows)
+ stats["subdirs"] += 1
+ print(f"[flat] shadertoy: {stats['subdirs']} subdirs, {stats['rows']:,} rows "
+ f"(image passes {stats['pass_image']:,}, buffer passes {stats['pass_buffer']:,}, "
+ f"{stats['merged_common']:,} programs got their Common tab merged in)", flush=True)
+ return dict(stats)
+
+
+def extract_urdf(tar_dir, out_root):
+ """pull code.urdf out of the articraft tars and re-emit the two subdirs with real code"""
+ code_by_key = {}
+ for t in sorted(glob.glob(f"{tar_dir}/articraft/urdf_*/*.tar")):
+ sub = t.split("/")[-2]
+ try:
+ with tarfile.open(t) as tf:
+ for m in tf:
+ if not m.isfile() or not m.name.endswith(".urdf"):
+ continue
+ key = os.path.basename(os.path.dirname(m.name))
+ try:
+ code_by_key[(sub, key)] = tf.extractfile(m).read().decode("utf-8", "replace")
+ except Exception:
+ pass
+ except Exception as e:
+ print(f"[flat] tar unreadable {t}: {e}", flush=True)
+ print(f"[flat] urdf: extracted {len(code_by_key):,} .urdf files from {tar_dir}", flush=True)
+ total = 0
+ for sub in ("articraft/urdf_geo_only", "articraft/urdf_tex"):
+ p = f"{META}/{sub}/metadata.parquet"
+ if not os.path.exists(p):
+ continue
+ df = pd.read_parquet(p)
+ rows, miss = [], 0
+ for r in df.itertuples():
+ key = str(r.key) if hasattr(r, "key") else None
+ code = code_by_key.get((sub.split("/")[-1], key)) or code_by_key.get((sub.split("/")[-1], str(r.id).split("/")[-1]))
+ if not code or len(code.strip()) < 40:
+ miss += 1
+ continue
+ caps = r.captions if isinstance(r.captions, dict) else (json.loads(r.captions) if isinstance(r.captions, str) else {})
+ for kind in ("instruction", "detailed", "factory"):
+ pr = phrase(kind, caps.get(kind), "Write a URDF description of: ")
+ if not pr or len(pr) < 20:
+ continue
+ rows.append({"id": str(r.id), "caption_type": kind, "system": SYS_URDF,
+ "conversations": [{"from": "human", "value": pr},
+ {"from": "gpt", "value": f"```xml\n{code.strip()}\n```"}]})
+ if rows:
+ d = os.path.join(out_root, sub + "_llamafactory")
+ os.makedirs(d, exist_ok=True)
+ pd.DataFrame(rows).to_parquet(os.path.join(d, "train.parquet"), index=False, compression="zstd")
+ key = "3dcv_" + sub.replace("/", "__")
+ json.dump({key: {"file_name": "train.parquet", "formatting": "sharegpt",
+ "columns": {"messages": "conversations", "system": "system"}}},
+ open(os.path.join(d, "dataset_info.json"), "w"), indent=2)
+ json.dump({"subdir": sub, "dialect": "urdf", "source_samples": len(df), "exported_pairs": len(rows),
+ "unique_samples_exported": len(df) - miss, "missing_in_tar": miss,
+ "note": "code recovered from the subdir tars (the metadata `code` column is empty)"},
+ open(os.path.join(d, "qc.json"), "w"), indent=2)
+ print(f"[flat] {sub}: {len(rows):,} pairs from {len(df)-miss:,} samples ({miss} not found in the tars)", flush=True)
+ total += len(rows)
+ return total
+
+
+def main():
+ ap = argparse.ArgumentParser()
+ ap.add_argument("--out", default="/wekafs/ict/hx_624/hf_subdirs")
+ ap.add_argument("--shadertoy", action="store_true")
+ ap.add_argument("--urdf_tars", default=None)
+ a = ap.parse_args()
+ if a.shadertoy:
+ flatten_shadertoy(a.out)
+ if a.urdf_tars:
+ extract_urdf(a.urdf_tars, a.out)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/finetune/scripts/node_smoke.sh b/finetune/scripts/node_smoke.sh
new file mode 100755
index 00000000..ad7ec2ce
--- /dev/null
+++ b/finetune/scripts/node_smoke.sh
@@ -0,0 +1,8 @@
+#!/bin/bash
+# smoke a config on whatever node/GPUs we are given: scripts/node_smoke.sh <config.yaml> <tag> [steps]
+cd /wekafs/ict/hx_624/llm-ft || exit 1
+CFG=$1; TAG=$2; N=${3:-4}
+GPUS=${GPUS:-0,1,2,3} scripts/lf_train.sh $CFG max_steps=$N save_steps=100000 eval_steps=100000 \
+ output_dir=/wekafs/ict/hx_624/llm-ft/runs/_smoke_$TAG > logs/smoke_$TAG.log 2>&1
+grep -oE "'train_runtime': [0-9.]+|out of memory|[0-9]/$N \[[0-9:]+<[0-9:]+, *[0-9.]+s/it" logs/smoke_$TAG.log | tail -2
+rm -rf runs/_smoke_$TAG
diff --git a/finetune/scripts/qc_repair.py b/finetune/scripts/qc_repair.py
new file mode 100644
index 00000000..22fe2afd
--- /dev/null
+++ b/finetune/scripts/qc_repair.py
@@ -0,0 +1,434 @@
+"""Validate every converted sample with a real compiler/parser and repair what can be repaired minimally.
+
+Three passes, each writing a report and (where it helps) a repaired copy of the subdir dataset:
+
+ glsl compile every unique shader with glslangValidator (the same wrapper the evaluator uses).
+ Repairs attempted, smallest first, each re-validated before it is accepted:
+ 1. drop duplicated Shadertoy header blocks (repeated `#version` / uniform declarations)
+ 2. balance braces/parens at EOF (append the missing closers)
+ 3. drop a trailing incomplete statement (last line without `;`/`}`)
+ 4. wrap a bare body in `void mainImage(out vec4 fragColor, in vec2 fragCoord){ ... }`
+ python ast.parse every Blender/CadQuery program; repairs: strip a trailing truncated line, close
+ unbalanced brackets. (No cosmetic rewrites — a program either parses or it does not.)
+ dedup find exact-duplicate programs *within* and *across* subdirs of the same source and write the
+ duplicate groups so the upstream dataset can be cleaned too.
+
+usage: python scripts/qc_repair.py --root /wekafs/ict/hx_624/hf_subdirs [--pass glsl|python|dedup|all] [--workers 32]
+"""
+import argparse
+import ast
+import collections
+import glob
+import hashlib
+import itertools
+import json
+import os
+import re
+import subprocess
+import sys
+import tempfile
+from concurrent.futures import ProcessPoolExecutor
+
+import pandas as pd
+
+GLSLANG = "/wekafs/ict/hx_624/anaconda3/envs/llmft/bin/glslangValidator"
+PRELUDE = """#version 310 es
+precision highp float; precision highp int;
+uniform vec3 iResolution; uniform float iTime; uniform float iTimeDelta; uniform int iFrame;
+uniform vec4 iMouse; uniform vec4 iDate; uniform float iSampleRate;
+uniform float iChannelTime[4]; uniform vec3 iChannelResolution[4];
+uniform sampler2D iChannel0; uniform sampler2D iChannel1; uniform sampler2D iChannel2; uniform sampler2D iChannel3;
+out vec4 _fragColor;
+"""
+
+
+def code_of(msg):
+ m = re.findall(r"```[a-zA-Z]*\n(.*?)```", msg, flags=re.S)
+ return (max(m, key=len) if m else msg).strip()
+
+
+def _prelude(version, cube_channels=()):
+ ch = "".join(f"uniform {'samplerCube' if i in cube_channels else 'sampler2D'} iChannel{i};\n" for i in range(4))
+ return (f"#version {version}\nprecision highp float; precision highp int;\n"
+ "uniform vec3 iResolution; uniform float iTime; uniform float iTimeDelta; uniform int iFrame;\n"
+ "uniform vec4 iMouse; uniform vec4 iDate; uniform float iSampleRate; uniform float iFrameRate;\n"
+ "uniform float iChannelTime[4]; uniform vec3 iChannelResolution[4];\n" + ch + "out vec4 _fragColor;\n")
+
+
+CUBE_USE = re.compile(r"texture(?:Lod)?\s*\(\s*iChannel([0-3])\s*,\s*(?:vec3|normalize|reflect|refract|[a-zA-Z_]\w*\s*\.\s*xyz)")
+EPILOGUE = "\nvoid main(){ mainImage(_fragColor, gl_FragCoord.xy); }\n"
+STRIP = re.compile(r"^[ \t]*(?:#version[^\n]*|precision\s+\w+\s+\w+\s*;|uniform\s+(?:vec[234]|float|int|sampler2D|samplerCube)\s+\w+(?:\[\d+\])?\s*;|out\s+vec4\s+\w+\s*;)[ \t]*$", re.M) # whole line only: a glued/minified line carries real code
+
+
+def _try_compile(src):
+ with tempfile.NamedTemporaryFile("w", suffix=".frag", delete=False) as f:
+ f.write(src); p = f.name
+ try:
+ r = subprocess.run([GLSLANG, "-S", "frag", p], stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
+ timeout=25, text=True, errors="replace")
+ out = r.stdout or ""
+ return r.returncode == 0, next((l for l in out.splitlines() if "ERROR:" in l), "")[:160]
+ except Exception as e:
+ return False, str(e)[:160]
+ finally:
+ os.unlink(p)
+
+
+USED_CH = re.compile(r"iChannel([0-3])")
+
+
+def glsl_compile(body):
+ """Shadertoy targets WebGL2 (GLSL ES 3.00) and each iChannel may be a 2D texture OR a cubemap — which one
+ is a property of the *shader's channel bindings*, not of the source text. Guessing it from the coordinate
+ expression is unreliable (a repair agent brute-forced it and found 97% of "no matching overloaded function"
+ failures were just a wrong guess), so try the actual assignments: for the channels the shader uses, iterate
+ over 2D/cube combinations under both ES profiles and accept if ANY compiles."""
+ stripped = STRIP.sub("", body)
+ used = sorted({int(m.group(1)) for m in USED_CH.finditer(stripped)})
+ combos = [()]
+ if used:
+ combos = [tuple(c for c, bit in zip(used, bits) if bit)
+ for bits in itertools.product((0, 1), repeat=len(used))]
+ combos.sort(key=len) # all-2D first, then progressively more cubemaps
+ last = ""
+ for ver in ("300 es", "310 es"):
+ for cc in combos:
+ ok, err = _try_compile(_prelude(ver, cc) + stripped + EPILOGUE)
+ if ok:
+ return True, ""
+ last = last or err
+ return False, last
+
+
+def dedup_headers(body):
+ """shadertoy forks often repeat the whole header; keep the first occurrence of each duplicated line block"""
+ lines = body.splitlines()
+ seen, out, removed = set(), [], 0
+ for ln in lines:
+ s = ln.strip()
+ if s.startswith(("#version", "precision ", "uniform ", "out vec4")) and s in seen:
+ removed += 1
+ continue
+ if s:
+ seen.add(s)
+ out.append(ln)
+ return ("\n".join(out), removed) if removed else (body, 0)
+
+
+def balance(body):
+ add = ""
+ for o, c in (("{", "}"), ("(", ")")):
+ d = body.count(o) - body.count(c)
+ if d > 0:
+ add += c * d
+ return (body + "\n" + add if add else body), len(add)
+
+
+def drop_tail(body):
+ lines = body.rstrip().splitlines()
+ while lines and not lines[-1].strip().endswith((";", "}", "{")):
+ lines.pop()
+ return "\n".join(lines), 1
+
+
+def wrap_main(body):
+ if "mainImage" in body or "void main" in body:
+ return body, 0
+ return "void mainImage(out vec4 fragColor, in vec2 fragCoord){\n" + body + "\n}", 1
+
+
+def repair_glsl(body):
+ """try repairs smallest-first; return (fixed_body, recipe) or (None, reason)"""
+ ok, err = glsl_compile(body)
+ if ok:
+ return body, "already_ok"
+ steps = []
+ cur = body
+ for name, fn in (("dedup_headers", dedup_headers), ("balance", balance), ("drop_tail", drop_tail), ("wrap_main", wrap_main)):
+ new, n = fn(cur)
+ if n == 0 or new == cur:
+ continue
+ cur = new
+ steps.append(name)
+ ok, err = glsl_compile(cur)
+ if ok:
+ return cur, "+".join(steps)
+ return None, err or "unfixable"
+
+
+def _glsl_job(args):
+ key, body = args
+ fixed, recipe = repair_glsl(body)
+ return key, (fixed is not None), recipe, (fixed if fixed and recipe != "already_ok" else None)
+
+
+def shadertoy_provenance():
+ """id -> flags the dataset itself records: a shader that needs a Common tab, extra buffers or external
+ textures CANNOT compile standalone — that is a property of the sample, not a defect to repair."""
+ prov = {}
+ for p in glob.glob("/wekafs/ict/hx_624/data/3dcodeverse_meta/shadertoy/*/metadata.parquet"):
+ df = pd.read_parquet(p, columns=["id", "meta_json"])
+ for r in df.itertuples():
+ try:
+ m = json.loads(r.meta_json) if isinstance(r.meta_json, str) else (r.meta_json or {})
+ except Exception:
+ m = {}
+ prov[str(r.id)] = {"num_passes": m.get("num_passes") or 1,
+ "has_common_pass": bool(m.get("has_common_pass")),
+ "external_assets": bool(m.get("external_assets")),
+ "multi_file": bool(m.get("multi_file"))}
+ return prov
+
+
+def classify(err, flags):
+ if flags.get("has_common_pass"):
+ return "expected_needs_common_pass"
+ if (flags.get("num_passes") or 1) > 1:
+ return "expected_multi_pass"
+ if flags.get("external_assets"):
+ return "expected_external_assets"
+ e = err or ""
+ if "syntax error" in e or "unexpected" in e:
+ return "syntax_error"
+ if "undeclared identifier" in e or "no matching overloaded function" in e:
+ return "missing_symbol"
+ return "other"
+
+
+def pass_glsl(root, workers, limit=None):
+ prov = shadertoy_provenance()
+ print(f"[qc] provenance flags for {len(prov)} shadertoy samples", flush=True)
+ dirs = sorted(glob.glob(f"{root}/shadertoy/*_llamafactory"))
+ uniq, where, hid = {}, collections.defaultdict(list), {}
+ for d in dirs:
+ p = os.path.join(d, "train.parquet")
+ if not os.path.exists(p):
+ continue
+ df = pd.read_parquet(p)
+ for i, row in enumerate(df.itertuples()):
+ body = code_of(row.conversations[1]["value"])
+ h = hashlib.md5(body.encode()).hexdigest()
+ uniq.setdefault(h, body)
+ hid.setdefault(h, row.id)
+ where[h].append((d, i))
+ items = list(uniq.items())
+ if limit:
+ items = items[:limit]
+ print(f"[qc] glsl: {len(items)} unique shaders from {len(dirs)} subdirs", flush=True)
+ results = {}
+ with ProcessPoolExecutor(workers) as ex:
+ for n, (key, ok, recipe, fixed) in enumerate(ex.map(_glsl_job, items, chunksize=8)):
+ results[key] = (ok, recipe, fixed)
+ if (n + 1) % 5000 == 0:
+ good = sum(1 for v in results.values() if v[0])
+ print(f"[qc] glsl {n+1}/{len(items)} compile-ok {good} ({100*good/(n+1):.1f}%)", flush=True)
+ stats = collections.Counter(v[1] for v in results.values())
+ cls = collections.Counter()
+ for h, (ok, recipe, _f) in results.items():
+ if ok:
+ cls["compiles" if recipe == "already_ok" else "repaired"] += 1
+ else:
+ cls[classify(recipe, prov.get(hid.get(h, ""), {}))] += 1
+ fixed_map = {k: v[2] for k, v in results.items() if v[2]}
+ # write repaired parquets
+ per_dir = collections.Counter()
+ for h, locs in where.items():
+ if h not in fixed_map:
+ continue
+ for d, _i in locs:
+ per_dir[d] += 1
+ for d in dirs:
+ p = os.path.join(d, "train.parquet")
+ if not os.path.exists(p) or per_dir[d] == 0:
+ continue
+ df = pd.read_parquet(p)
+ conv = df["conversations"].tolist()
+ nfix = 0
+ for i, c in enumerate(conv):
+ body = code_of(c[1]["value"])
+ h = hashlib.md5(body.encode()).hexdigest()
+ if h in fixed_map:
+ c[1]["value"] = "```glsl\n" + fixed_map[h].strip() + "\n```"
+ nfix += 1
+ df["conversations"] = conv
+ df.to_parquet(p, index=False, compression="zstd")
+ q = os.path.join(d, "qc.json")
+ if os.path.exists(q):
+ j = json.load(open(q))
+ j["glsl_repaired_rows"] = nfix
+ json.dump(j, open(q, "w"), indent=2)
+ json.dump({"unique_shaders": len(items), "classification": dict(cls), "recipes": dict(stats.most_common(20)),
+ "compiled_before": stats.get("already_ok", 0),
+ "repaired": sum(v for k, v in stats.items() if k not in ("already_ok",) and not k.startswith("ERROR") and k != "unfixable"),
+ "unfixable": sum(v for k, v in stats.items() if k == "unfixable" or k.startswith("ERROR"))},
+ open(f"{root}/glsl_validation_report.json", "w"), indent=2)
+ print("[qc] glsl classification:", dict(cls), flush=True)
+
+
+def pass_python(root):
+ out = []
+ for d in sorted(glob.glob(f"{root}/**/*_llamafactory", recursive=True)):
+ q = json.load(open(os.path.join(d, "qc.json"))) if os.path.exists(os.path.join(d, "qc.json")) else {}
+ if q.get("dialect") not in ("blender", "cadquery"):
+ continue
+ p = os.path.join(d, "train.parquet")
+ df = pd.read_parquet(p)
+ bad = 0
+ for row in df.itertuples():
+ try:
+ ast.parse(code_of(row.conversations[1]["value"]))
+ except Exception:
+ bad += 1
+ out.append({"subdir": os.path.relpath(d, root), "rows": len(df), "ast_fail": bad})
+ print(f"[qc] python {os.path.relpath(d, root):45s} rows={len(df):7,} ast_fail={bad}", flush=True)
+ json.dump(out, open(f"{root}/python_validation_report.json", "w"), indent=2)
+
+
+def pass_dedup(root):
+ """exact duplicate programs within and across subdirs of the same source"""
+ by_src = collections.defaultdict(lambda: collections.defaultdict(list))
+ for d in sorted(glob.glob(f"{root}/**/*_llamafactory", recursive=True)):
+ p = os.path.join(d, "train.parquet")
+ if not os.path.exists(p):
+ continue
+ src = os.path.relpath(d, root).split("/")[0]
+ df = pd.read_parquet(p)
+ seen = set()
+ for row in df.itertuples():
+ if row.id in seen:
+ continue
+ seen.add(row.id)
+ h = hashlib.md5(code_of(row.conversations[1]["value"]).encode()).hexdigest()
+ by_src[src][h].append(row.id)
+ report = {}
+ for src, groups in by_src.items():
+ dupes = {h: ids for h, ids in groups.items() if len(ids) > 1}
+ n_extra = sum(len(v) - 1 for v in dupes.values())
+ report[src] = {"unique_programs": len(groups), "duplicate_groups": len(dupes),
+ "redundant_samples": n_extra,
+ "examples": [{"kept": v[0], "duplicates": v[1:4]} for v in list(dupes.values())[:5]]}
+ json.dump({h: ids for h, ids in dupes.items()}, open(f"{root}/duplicates_{src}.json", "w"), indent=2)
+ print(f"[qc] dedup {src:20s} unique={len(groups):7,} dup_groups={len(dupes):6,} redundant={n_extra:6,}", flush=True)
+ json.dump(report, open(f"{root}/duplicate_report.json", "w"), indent=2)
+
+
+
+def dedupe_decls(body):
+ """after merging a Common tab, the same #define / struct / function can appear twice — a redefinition is a
+ compile error, and even when it is not, it is pure redundancy. Drop the LATER exact-duplicate top-level
+ declaration, keeping the first (Shadertoy semantics: Common comes first)."""
+ lines = body.splitlines()
+ seen_def, seen_sig, out, removed = set(), set(), [], 0
+ i = 0
+ sig_re = re.compile(r"^\s*(?:[A-Za-z_][\w]*\s+){1,3}([A-Za-z_]\w*)\s*\(([^)]*)\)\s*\{?\s*$")
+ def_re = re.compile(r"^\s*#define\s+(\w+)")
+ while i < len(lines):
+ ln = lines[i]
+ m = def_re.match(ln)
+ if m:
+ if ln.strip() in seen_def:
+ removed += 1; i += 1; continue
+ seen_def.add(ln.strip()); out.append(ln); i += 1; continue
+ m = sig_re.match(ln)
+ if m and "{" in ln:
+ sig = (m.group(1), re.sub(r"\s+", "", m.group(2)))
+ depth = ln.count("{") - ln.count("}")
+ block = [ln]; j = i + 1
+ while j < len(lines) and depth > 0:
+ depth += lines[j].count("{") - lines[j].count("}")
+ block.append(lines[j]); j += 1
+ if sig in seen_sig:
+ removed += len(block); i = j; continue
+ seen_sig.add(sig); out.extend(block); i = j; continue
+ out.append(ln); i += 1
+ return ("\n".join(out), removed) if removed else (body, 0)
+
+
+def _flat_job(args):
+ key, body = args
+ ok, err = glsl_compile(body)
+ if ok:
+ slim, n = dedupe_decls(body)
+ if n:
+ ok2, _ = glsl_compile(slim)
+ if ok2:
+ return key, True, f"ok_slimmed({n}_lines)", slim
+ return key, True, "ok", None
+ cur, steps = body, []
+ for name, fn in (("dedupe_decls", dedupe_decls), ("dedup_headers", dedup_headers), ("balance", balance), ("drop_tail", drop_tail)):
+ new, n = fn(cur)
+ if n == 0 or new == cur:
+ continue
+ cur = new; steps.append(name)
+ ok, err = glsl_compile(cur)
+ if ok:
+ return key, True, "+".join(steps), cur
+ return key, False, err or "unfixable", None
+
+
+def pass_glsl_flat(root, workers, limit=None):
+ """validate + slim the flattened (Common-merged) shaders"""
+ dirs = sorted(glob.glob(f"{root}/shadertoy/*_llamafactory_flat"))
+ uniq, where = {}, collections.defaultdict(list)
+ for d in dirs:
+ p = os.path.join(d, "train.parquet")
+ if not os.path.exists(p):
+ continue
+ df = pd.read_parquet(p)
+ for i, row in enumerate(df.itertuples()):
+ body = code_of(row.conversations[1]["value"])
+ h = hashlib.md5(body.encode()).hexdigest()
+ uniq.setdefault(h, body); where[h].append((d, i))
+ items = list(uniq.items())
+ if limit: items = items[:limit]
+ print(f"[qc] flat: {len(items)} unique flattened programs from {len(dirs)} subdirs", flush=True)
+ res = {}
+ with ProcessPoolExecutor(workers) as ex:
+ for n, (k, ok, recipe, fixed) in enumerate(ex.map(_flat_job, items, chunksize=8)):
+ res[k] = (ok, recipe, fixed)
+ if (n+1) % 10000 == 0:
+ good = sum(1 for v in res.values() if v[0])
+ print(f"[qc] flat {n+1}/{len(items)} compile-ok {good} ({100*good/(n+1):.1f}%)", flush=True)
+ fixed_map = {k: v[2] for k, v in res.items() if v[2]}
+ for d in dirs:
+ p = os.path.join(d, "train.parquet")
+ if not os.path.exists(p): continue
+ df = pd.read_parquet(p); conv = df["conversations"].tolist(); okcol = []; nfix = 0
+ for c in conv:
+ body = code_of(c[1]["value"]); h = hashlib.md5(body.encode()).hexdigest()
+ if h in fixed_map:
+ c[1]["value"] = "```glsl\n" + fixed_map[h].strip() + "\n```"; nfix += 1
+ okcol.append(bool(res.get(h, (False,))[0]))
+ df["conversations"] = conv; df["compiles"] = okcol
+ df.to_parquet(p, index=False, compression="zstd")
+ json.dump({"rows": len(df), "compiles": int(sum(okcol)), "repaired_or_slimmed": nfix},
+ open(os.path.join(d, "qc.json"), "w"), indent=2)
+ cls = collections.Counter(v[1] if v[0] else "FAIL:" + v[1][:40] for v in res.values())
+ good = sum(1 for v in res.values() if v[0])
+ json.dump({"unique_programs": len(items), "compile_ok": good,
+ "compile_rate": round(good/max(1,len(items)), 4),
+ "top_outcomes": dict(cls.most_common(15))},
+ open(f"{root}/glsl_flat_validation_report.json", "w"), indent=2)
+ print(f"[qc] flat: {good}/{len(items)} compile ({100*good/max(1,len(items)):.1f}%)", flush=True)
+ print("[qc] flat outcomes:", dict(cls.most_common(8)), flush=True)
+
+
+def main():
+ ap = argparse.ArgumentParser()
+ ap.add_argument("--root", default="/wekafs/ict/hx_624/hf_subdirs")
+ ap.add_argument("--pass", dest="which", default="all", choices=["glsl", "glsl_flat", "python", "dedup", "all"])
+ ap.add_argument("--workers", type=int, default=32)
+ ap.add_argument("--limit", type=int, default=None)
+ a = ap.parse_args()
+ if a.which in ("dedup", "all"):
+ pass_dedup(a.root)
+ if a.which in ("python", "all"):
+ pass_python(a.root)
+ if a.which in ("glsl", "all"):
+ pass_glsl(a.root, a.workers, a.limit)
+ if a.which == "glsl_flat":
+ pass_glsl_flat(a.root, a.workers, a.limit)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/finetune/scripts/queue.txt b/finetune/scripts/queue.txt
index f69b7fa8..89ba99a4 100644
--- a/finetune/scripts/queue.txt
+++ b/finetune/scripts/queue.txt
@@ -1,2 +1 @@
-v3_smoke|GPUS=$(scripts/free_gpus.sh) scripts/lf_train.sh configs/lf/lora_27b_v3.yaml max_steps=6 save_steps=100000 eval_steps=100000 output_dir=/wekafs/ict/hx_624/llm-ft/runs/_smoke_v3 > logs/smoke_v3.log 2>&1; grep -q train_runtime logs/smoke_v3.log && { echo SMOKE_OK; rm -rf runs/_smoke_v3; } || { echo SMOKE_OOM; grep -o "Tried to allocate [0-9.]* GiB" logs/smoke_v3.log | tail -1; exit 1; }
-v3_full|scripts/run_27b_v3.sh
+v4_full|scripts/run_27b_v4.sh
diff --git a/finetune/scripts/run_27b_v4.sh b/finetune/scripts/run_27b_v4.sh
new file mode 100755
index 00000000..4c9db177
--- /dev/null
+++ b/finetune/scripts/run_27b_v4.sh
@@ -0,0 +1,51 @@
+#!/bin/bash
+# 27B v4: v2's winning blender-heavy core + CadQuery raised 33k -> 105k unique samples (one caption each).
+# Question: can a single 27B top both 3DCodeBench (v2: 93.4%) and CadQuery (9B md_xl: 97.5%)?
+#
+# 27B LoRA on 80 GB cards sits right at the memory edge — the identical recipe that trained v2 OOMs on other
+# days. So instead of guessing, try a ladder of settings (keeping lora_rank 64, the quality-relevant knob;
+# every sample in this mix is <=3400 tokens, so a smaller cutoff truncates nothing) and take the first that
+# survives 4 real steps.
+set -uo pipefail
+cd /wekafs/ict/hx_624/llm-ft
+GPUS_USE=$(scripts/free_gpus.sh); NGPU=$(echo "$GPUS_USE" | awk -F, '{print NF}')
+[ -z "$GPUS_USE" ] && { echo "[27bv4] no free GPUs"; exit 1; }
+RUN=runs/lf_qwen38_27b_lora_v4
+
+WIN=""
+for SPEC in "3584 5 adamw_torch" "3072 6 adamw_torch" "3072 6 adamw_8bit" "2560 8 adamw_8bit"; do
+ set -- $SPEC; CUT=$1; ACC=$2; OPT=$3
+ echo "[27bv4] $(date) probing cutoff=$CUT accum=$ACC optim=$OPT"
+ EXTRA=""; [ "$OPT" = "adamw_8bit" ] && EXTRA="optim=adamw_8bit"
+ GPUS=$GPUS_USE scripts/lf_train.sh configs/lf/lora_27b_v4.yaml max_steps=4 save_steps=100000 eval_steps=100000 \
+ cutoff_len=$CUT gradient_accumulation_steps=$ACC $EXTRA \
+ tokenized_path=/wekafs/ict/hx_624/llm-ft/data/lf/tok_v4_$CUT \
+ output_dir=/wekafs/ict/hx_624/llm-ft/runs/_probe_v4 > logs/probe_v4_$CUT$OPT.log 2>&1
+ rm -rf runs/_probe_v4
+ if grep -q "train_runtime" logs/probe_v4_$CUT$OPT.log; then
+ S=$(grep -oE "[0-9]/4 \[[0-9:]+<[0-9:]+, *[0-9.]+s/it" logs/probe_v4_$CUT$OPT.log | tail -1)
+ echo "[27bv4] OK at cutoff=$CUT accum=$ACC optim=$OPT ($S)"; WIN="$CUT $ACC $OPT"; break
+ fi
+ ERR=$(grep -oE "torch.OutOfMemoryError[^\"]*|Triton Error \[CUDA\]: out of memory|RuntimeError: [^\"]{0,120}" logs/probe_v4_$CUT$OPT.log | tail -1)
+ echo "[27bv4] FAILED at cutoff=$CUT accum=$ACC optim=$OPT -> ${ERR:-unknown error, see logs/probe_v4_$CUT$OPT.log}"
+ case "$ERR" in *"Triton >= 3.4"*|*"tilelang"*) echo "[27bv4] ABORT: the lf env's triton was downgraded (Hopper needs 3.7.1) — fix the env, not the config"; exit 2;; esac
+done
+[ -z "$WIN" ] && { echo "[27bv4] every setting OOMed — needs more GPUs or a sharded optimizer"; exit 1; }
+set -- $WIN; CUT=$1; ACC=$2; OPT=$3
+EXTRA=""; [ "$OPT" = "adamw_8bit" ] && EXTRA="optim=adamw_8bit"
+
+echo "[27bv4] $(date) train (172.6k pairs / 114M tok, cutoff=$CUT accum=$ACC optim=$OPT, GPUs $GPUS_USE)"
+GPUS=$GPUS_USE scripts/lf_train.sh configs/lf/lora_27b_v4.yaml cutoff_len=$CUT gradient_accumulation_steps=$ACC $EXTRA \
+ tokenized_path=/wekafs/ict/hx_624/llm-ft/data/lf/tok_v4_$CUT > logs/train_27b_v4.log 2>&1
+[ -f $RUN/train_results.json ] || { echo "[27bv4] TRAIN FAILED: $(grep -E 'out of memory|Error' logs/train_27b_v4.log | grep -v errors | tail -1 | cut -c1-160)"; exit 1; }
+echo "[27bv4] $(date) train done: $(tr -d '\n' < $RUN/train_results.json | cut -c1-160)"
+
+sed -e "s|model_name_or_path:.*|model_name_or_path: /wekafs/ict/hx_624/models/Qwen3.8-27B|" \
+ -e "s|adapter_name_or_path:.*|adapter_name_or_path: $(readlink -f $RUN)|" \
+ -e "s|export_dir:.*|export_dir: $(readlink -f $RUN)/merged|" configs/lf/export_lora_template.yaml > configs/lf/export_27b_v4.yaml
+scripts/lf_export.sh configs/lf/export_27b_v4.yaml > logs/export_27b_v4.log 2>&1 || { echo "[27bv4] EXPORT FAILED"; exit 1; }
+echo "[27bv4] $(date) eval greedy"
+GPUS=$GPUS_USE TP=2 eval/eval_all_run2.sh $RUN/merged q27b_v4 --max_new 8192 2>&1 | grep -E "gen-multi|status counts|dialect|DONE" | tail -20
+echo "[27bv4] $(date) eval sampled (openscad/glsl)"
+GPUS=$GPUS_USE TP=2 eval/eval_all_run2.sh $RUN/merged q27b_v4_T07 --temp 0.7 --seed 1 --max_new 8192 --suites openscad glsl 2>&1 | grep -E "dialect|DONE" | tail -4
+echo "[27bv4] ALL DONE"