Official Codebase of 3DCodeVerse
Python
finetune: per-subdir LLaMA-Factory conversion + QC (convert_subdirs.py), compiler-verified GLSL validation/repair (qc_repair.py), multi-file flattening and URDF recovery from tars (flatten_multifile.py), final consolidation (finalize_glsl.py)
gaoypeng1a5844bce924
diff --git a/finetune/configs/lf/lora_27b_v4.yaml b/finetune/configs/lf/lora_27b_v4.yamlnew file mode 100644index 00000000..85f31697--- /dev/null+++ b/finetune/configs/lf/lora_27b_v4.yaml@@ -0,0 +1,44 @@+model_name_or_path: /wekafs/ict/hx_624/models/Qwen3.8-27B+trust_remote_code: true+flash_attn: fa2+stage: sft+do_train: true+finetuning_type: lora+lora_rank: 64+lora_alpha: 128+lora_dropout: 0.05+lora_target: all+freeze_vision_tower: true+freeze_multi_modal_projector: true+dataset_dir: /wekafs/ict/hx_624/llm-ft/data/lf+dataset: md_27b_v4_train+eval_dataset: md_27b_v4_val+template: qwen3_5+enable_thinking: false+cutoff_len: 4096+preprocessing_num_workers: 16+dataloader_num_workers: 4+neat_packing: true+packing: true+tokenized_path: /wekafs/ict/hx_624/llm-ft/data/lf/tokenized_md_27b_v4_len4096_packed+output_dir: /wekafs/ict/hx_624/llm-ft/runs/lf_qwen38_27b_lora_v4+logging_steps: 5+save_steps: 300+save_only_model: true+plot_loss: true+overwrite_output_dir: true+report_to: tensorboard+per_device_train_batch_size: 1+gradient_accumulation_steps: 4+learning_rate: 0.0001+num_train_epochs: 1.0+lr_scheduler_type: cosine+warmup_ratio: 0.05+bf16: true+gradient_checkpointing: true+ddp_timeout: 180000000+resume_from_checkpoint: null+per_device_eval_batch_size: 1+eval_strategy: steps+eval_steps: 300+save_total_limit: 8diff --git a/finetune/configs/lf/lora_27b_z3_8gpu.yaml b/finetune/configs/lf/lora_27b_z3_8gpu.yamlnew file mode 100644index 00000000..c65922a1--- /dev/null+++ b/finetune/configs/lf/lora_27b_z3_8gpu.yaml@@ -0,0 +1,45 @@+model_name_or_path: /wekafs/ict/hx_624/models/Qwen3.8-27B+trust_remote_code: true+flash_attn: fa2+stage: sft+do_train: true+finetuning_type: lora+lora_rank: 64+lora_alpha: 128+lora_dropout: 0.05+lora_target: all+freeze_vision_tower: true+freeze_multi_modal_projector: true+dataset_dir: /wekafs/ict/hx_624/llm-ft/data/lf+dataset: md_27b_v2s_train+eval_dataset: md_27b_v2s_val+template: qwen3_5+enable_thinking: false+cutoff_len: 8192+preprocessing_num_workers: 16+dataloader_num_workers: 4+neat_packing: true+packing: true+tokenized_path: /wekafs/ict/hx_624/llm-ft/data/lf/tokenized_md_27b_v2s_len8192_packed+output_dir: /wekafs/ict/hx_624/llm-ft/runs/lf_qwen38_27b_lora_v4+logging_steps: 5+save_steps: 250+save_only_model: true+plot_loss: true+overwrite_output_dir: true+report_to: tensorboard+per_device_train_batch_size: 1+gradient_accumulation_steps: 1+learning_rate: 0.0001+num_train_epochs: 1.0+lr_scheduler_type: cosine+warmup_ratio: 0.05+bf16: true+gradient_checkpointing: true+ddp_timeout: 180000000+resume_from_checkpoint: null+per_device_eval_batch_size: 1+eval_strategy: steps+eval_steps: 250+save_total_limit: 8+deepspeed: /wekafs/ict/hx_624/tools/LLaMA-Factory/examples/deepspeed/ds_z3_config.jsondiff --git a/finetune/data_specs/dataset_info.json b/finetune/data_specs/dataset_info.jsonindex 5493670c..a876a31b 100644--- a/finetune/data_specs/dataset_info.json+++ b/finetune/data_specs/dataset_info.json@@ -507,5 +507,21 @@"chosen": "chosen","rejected": "rejected"}+ },+ "md_27b_v4_train": {+ "file_name": "md_27b_v4_train.json",+ "formatting": "sharegpt",+ "columns": {+ "messages": "conversations",+ "system": "system"+ }+ },+ "md_27b_v4_val": {+ "file_name": "md_27b_v4_val.json",+ "formatting": "sharegpt",+ "columns": {+ "messages": "conversations",+ "system": "system"+ }}}\ No newline at end of filediff --git a/finetune/scripts/convert_subdirs.py b/finetune/scripts/convert_subdirs.pynew file mode 100644index 00000000..e0446619--- /dev/null+++ b/finetune/scripts/convert_subdirs.py@@ -0,0 +1,293 @@+"""Convert EVERY 3DCodeVerse subdirectory into its own LLaMA-Factory dataset + a data-quality report.++One output folder per source subdir, named `<subdir>_llamafactory/`, so it can sit next to the source on the+Hub (e.g. `3dcodebench/factories_geo/` → `3dcodebench/factories_geo_llamafactory/`):++ <subdir>_llamafactory/+ train.json ShareGPT: [{system, conversations:[human, gpt]}] — one row per caption variant+ dataset_info.json drop-in LLaMA-Factory registry entry for this folder+ qc.json per-subdir quality counters + token stats+ README.md what it is, how many rows, what was dropped and why++Quality checks applied per sample (a sample can fail several):+ empty_code / short_code (<40 chars) / bad_chars (control chars or U+FFFD) / no_caption /+ duplicate_code (same code hash inside the subdir) / syntax_bad (AST for Python dialects, brace balance for+ scad-glsl, tag check for html) / too_long (>8192 tokens, still exported but flagged) / media_placeholder+ (literal <image>/<video>/<audio> — breaks the qwen3_5 template unless escaped; we escape it and flag it)++usage: python scripts/convert_subdirs.py [--out DIR] [--workers 16] [--limit_subdirs N]+"""+import argparse+import ast+import collections+import glob+import hashlib+import json+import os+import re+import sys+import unicodedata+from concurrent.futures import ProcessPoolExecutor++import pandas as pd++sys.path.insert(0, "/wekafs/ict/hx_624/llm-ft/scripts")+META = "/wekafs/ict/hx_624/data/3dcodeverse_meta"+FILES = "/wekafs/ict/hx_624/data/3dcodeverse_files"+BENCH_DIR = "/wekafs/ict/hx_624/data/3dcodebench/data"++# subdir prefix -> dialect+DIALECT_OF = {+ "3dcodebench": "blender", "bioinspired3d": "blender", "blender_distill": "blender",+ "deepcad": "cadquery", "articraft": "cadquery", "shadertoy": "glsl",+ "thingiverse": "openscad", "threejs_distill": "threejs", "animation2code": "web",+}+SYS = {+ "blender": "You are an expert in procedural 3D modeling with Blender Python (bpy). Given a description of an object, write a complete, standalone Blender 5 Python script that builds it from scratch (clear the default scene first). Output ONLY the code in one ```python block.",+ "cadquery": "You are an expert CAD programmer. Given a description of a part, write a complete, standalone CadQuery (Python) script that builds it and leaves the final solid in a variable named `result`. Output ONLY the code in one ```python block.",+ "openscad": "You are an expert OpenSCAD programmer. Given a request, write a complete, standalone OpenSCAD (.scad) program that produces the described model. Output ONLY the code in one ```openscad block.",+ "glsl": "You are an expert shader programmer. Given a description, write a complete Shadertoy-style GLSL ES 3.00 fragment shader defining `void mainImage(out vec4 fragColor, in vec2 fragCoord)`. Output ONLY the code in one ```glsl block.",+ "threejs": "You are an expert three.js developer. Given a description, write a complete, standalone HTML page that renders the described 3D scene with three.js. Output ONLY the code in one ```html block.",+ "web": "You are an expert creative web developer. Given a description of an animation, write a complete, standalone HTML page (inline CSS/JS, no local assets) that reproduces it. Output ONLY the code in one ```html block.",+ "urdf": "You are an expert in articulated 3D assets. Given a description, write a complete, self-contained URDF file for it. Output ONLY the code in one ```xml block.",+}+FENCE = {"blender": "python", "cadquery": "python", "openscad": "openscad", "glsl": "glsl",+ "threejs": "html", "web": "html", "urdf": "xml"}+LEAD = {+ "blender": "Create the 3D object described below using Python Blender code: ",+ "cadquery": "Write a CadQuery (Python) script that builds the following CAD model: ",+ "openscad": "Write an OpenSCAD program for the following: ",+ "glsl": "Write a GLSL fragment shader for Shadertoy that renders: ",+ "threejs": "Write a standalone three.js page that renders: ",+ "web": "Write a standalone animated web page that reproduces: ",+ "urdf": "Write a URDF description of: ",+}+IMPERATIVE = ("write", "create", "implement", "make", "build", "generate", "design", "produce")+MEDIA = re.compile(r"<(image|video|audio)>")+++def bad_char_count(s):+ return sum(1 for ch in s if unicodedata.category(ch) == "Cc" and ch not in "\n\t\r") + s.count("�")+++def syntax_ok(code, dialect):+ if dialect in ("blender", "cadquery"):+ try:+ ast.parse(code)+ return True+ except Exception:+ return False+ if dialect == "openscad":+ return code.count("{") == code.count("}") and code.count("(") == code.count(")")+ if dialect == "glsl":+ return code.count("{") == code.count("}") and "void" in code+ if dialect in ("threejs", "web"):+ return "<" in code and ">" in code+ if dialect == "urdf":+ return "<robot" in code or "<link" in code+ return True+++def phrase(dialect, kind, text):+ t = (text or "").strip()+ if not t:+ return None+ if kind == "instruction" and t.lower().startswith(IMPERATIVE):+ return t+ return LEAD[dialect] + t+++def normalize_caps(caps):+ """tree sources use their own caption keys"""+ caps = {k: v for k, v in caps.items() if isinstance(v, str) and v.strip()}+ if any(k in caps for k in ("instruction", "detailed", "factory")) and "user_instruction" not in caps and "brief" not in caps:+ return caps+ norm = {}+ if caps.get("user_instruction"):+ norm["instruction"] = caps["user_instruction"]+ if caps.get("instruction"):+ norm["detailed"] = caps["instruction"]+ if caps.get("brief"):+ extra = caps.get("function", "")+ norm["instruction"] = (caps["brief"] + ((" — " + extra) if extra and extra not in caps["brief"] else "")).strip()+ if caps.get("source_title") and "detailed" not in norm:+ norm["detailed"] = caps["source_title"]+ if caps.get("style") and norm.get("detailed"):+ norm["detailed"] += f" (style: {caps['style']})"+ return norm or caps+++def load_subdir(sub):+ """-> (dialect, [ {id, name, code, caps, n_images} ]) for a parquet subdir or a file tree"""+ top = sub.split("/")[0]+ dialect = DIALECT_OF.get(top, "blender")+ if top == "articraft" and "urdf" in sub:+ dialect = "urdf"+ rows = []+ p = f"{META}/{sub}/metadata.parquet"+ if os.path.exists(p):+ df = pd.read_parquet(p)+ for r in df.itertuples():+ caps = r.captions if isinstance(r.captions, dict) else (json.loads(r.captions) if isinstance(r.captions, str) else {})+ meta = {}+ try:+ meta = json.loads(r.meta_json) if isinstance(r.meta_json, str) else (r.meta_json or {})+ except Exception:+ pass+ rows.append({"id": str(r.id), "name": str(getattr(r, "name", "") or ""),+ "code": getattr(r, "code", None) or "", "caps": dict(caps),+ "n_images": len(meta.get("renders", []) or [])})+ return dialect, rows+ # file tree+ for cp in sorted(glob.glob(f"{FILES}/{sub}/*/code.py") + glob.glob(f"{FILES}/{sub}/*/code.html")+ + glob.glob(f"{FILES}/{sub}/*/index.html") + glob.glob(f"{FILES}/{sub}/*/*/code.py")+ + glob.glob(f"{FILES}/{sub}/*/*/code.html") + glob.glob(f"{FILES}/{sub}/*/*/index.html")):+ d = os.path.dirname(cp)+ try:+ code = open(cp, encoding="utf-8", errors="replace").read()+ caps = json.load(open(os.path.join(d, "captions.json"))) if os.path.exists(os.path.join(d, "captions.json")) else {}+ except Exception:+ continue+ rows.append({"id": os.path.relpath(d, FILES), "name": os.path.basename(d), "code": code,+ "caps": caps, "n_images": len(glob.glob(os.path.join(d, "renders", "*.png")))})+ return dialect, rows+++_TOK = None+++def _init():+ global _TOK+ from transformers import AutoTokenizer+ _TOK = AutoTokenizer.from_pretrained("/wekafs/ict/hx_624/models/Qwen3.5-9B")+++def convert_one(args):+ sub, out_root, bench_factories = args+ dialect, rows = load_subdir(sub)+ if not rows:+ return {"subdir": sub, "error": "no rows"}+ qc = collections.Counter()+ seen_code, out, tok_lens, img_counts = {}, [], [], []+ for r in rows:+ code = r["code"] if isinstance(r["code"], str) else ""+ img_counts.append(r.get("n_images", 0))+ if not code.strip():+ qc["empty_code"] += 1+ continue+ if len(code.strip()) < 40:+ qc["short_code"] += 1+ continue+ if bad_char_count(code) > 5:+ qc["bad_chars"] += 1+ continue+ h = hashlib.md5(code.encode()).hexdigest()+ if h in seen_code:+ qc["duplicate_code"] += 1+ continue+ seen_code[h] = 1+ if not syntax_ok(code, dialect):+ qc["syntax_bad"] += 1 # exported anyway, but counted+ if MEDIA.search(code) or any(MEDIA.search(v) for v in r["caps"].values() if isinstance(v, str)):+ qc["media_placeholder"] += 1+ code = MEDIA.sub(lambda m: f"<{m.group(1)} >", code)+ if r["name"] in bench_factories or r["name"].rsplit("_", 1)[0] in bench_factories:+ qc["benchmark_overlap"] += 1 # flagged, and excluded below+ continue+ caps = normalize_caps(r["caps"])+ n_before = len(out)+ for kind in ("instruction", "detailed", "factory"):+ pr = phrase(dialect, kind, caps.get(kind))+ if not pr or len(pr) < 20:+ continue+ pr = MEDIA.sub(lambda m: f"<{m.group(1)} >", pr)+ n = len(_TOK(pr).input_ids) + len(_TOK(code).input_ids) + 60+ tok_lens.append(n)+ if n > 8192:+ qc["too_long_gt8192"] += 1+ out.append({"id": r["id"], "caption_type": kind, "n_tokens": n,+ "system": SYS[dialect],+ "conversations": [{"from": "human", "value": pr},+ {"from": "gpt", "value": f"```{FENCE[dialect]}\n{code.strip()}\n```"}]})+ if len(out) == n_before:+ qc["no_caption"] += 1+ name = sub.replace("/", "__")+ d = os.path.join(out_root, sub + "_llamafactory")+ os.makedirs(d, exist_ok=True)+ json.dump(out, open(os.path.join(d, "train.json"), "w"), ensure_ascii=False)+ key = f"3dcv_{name}"+ json.dump({key: {"file_name": "train.json", "formatting": "sharegpt",+ "columns": {"messages": "conversations", "system": "system"}}},+ open(os.path.join(d, "dataset_info.json"), "w"), indent=2)+ tok_lens.sort()+ stat = {"subdir": sub, "dialect": dialect, "source_samples": len(rows), "exported_pairs": len(out),+ "unique_samples_exported": len(seen_code) - qc["benchmark_overlap"],+ "images_per_sample_median": (sorted(img_counts)[len(img_counts) // 2] if img_counts else 0),+ "tokens_p50": tok_lens[len(tok_lens) // 2] if tok_lens else 0,+ "tokens_p90": tok_lens[int(len(tok_lens) * 0.9)] if tok_lens else 0,+ "tokens_max": tok_lens[-1] if tok_lens else 0,+ **{k: v for k, v in qc.items()}}+ json.dump(stat, open(os.path.join(d, "qc.json"), "w"), indent=2)+ issues = "\n".join(f"- **{k}**: {v}" for k, v in sorted(qc.items(), key=lambda x: -x[1])) or "- none"+ open(os.path.join(d, "README.md"), "w").write(f"""# {sub} — LLaMA-Factory format++Generated by `finetune/scripts/convert_subdirs.py` from `{sub}/metadata.parquet` (or its file tree).++| | |+|---|---|+| dialect | `{dialect}` |+| source samples | {len(rows):,} |+| exported pairs | {len(out):,} (one row per caption variant: instruction / detailed / factory) |+| unique code samples exported | {stat['unique_samples_exported']:,} |+| token length p50 / p90 / max | {stat['tokens_p50']} / {stat['tokens_p90']} / {stat['tokens_max']} |+| renders per sample (median) | {stat['images_per_sample_median']} |++## Data-quality findings++{issues}++`syntax_bad` rows are still exported (a shader can be valid without balanced braces in our crude check); every+other counter means the sample was dropped. `benchmark_overlap` = the sample belongs to one of the 212+3DCodeBench evaluation factories and is excluded to avoid test leakage.++## Use++```yaml+dataset_dir: <this folder>+dataset: {key}+```+""")+ return stat+++def main():+ ap = argparse.ArgumentParser()+ ap.add_argument("--out", default="/wekafs/ict/hx_624/hf_subdirs")+ ap.add_argument("--workers", type=int, default=16)+ ap.add_argument("--limit_subdirs", type=int, default=None)+ a = ap.parse_args()+ bench = frozenset(os.path.basename(p).replace("_seed0", "") for p in glob.glob(f"{BENCH_DIR}/*_seed0"))+ subs = ["/".join(p.split("/")[-3:-1]) for p in sorted(glob.glob(f"{META}/*/*/metadata.parquet"))]+ subs += ["blender_distill", "threejs_distill", "animation2code"]+ if a.limit_subdirs:+ subs = subs[: a.limit_subdirs]+ print(f"[conv] {len(subs)} subdirs -> {a.out}", flush=True)+ os.makedirs(a.out, exist_ok=True)+ stats = []+ with ProcessPoolExecutor(a.workers, initializer=_init) as ex:+ for st in ex.map(convert_one, [(s, a.out, bench) for s in subs]):+ stats.append(st)+ print(f"[conv] {st.get('subdir'):45s} pairs={st.get('exported_pairs', 0):7,} "+ f"issues={ {k: v for k, v in st.items() if k in ('empty_code','short_code','bad_chars','duplicate_code','syntax_bad','no_caption','too_long_gt8192','media_placeholder','benchmark_overlap')} }", flush=True)+ df = pd.DataFrame(stats)+ df.to_csv(os.path.join(a.out, "quality_report.csv"), index=False)+ try:+ df.to_excel(os.path.join(a.out, "quality_report.xlsx"), index=False)+ except Exception as e:+ print("[conv] xlsx skipped:", e)+ json.dump(stats, open(os.path.join(a.out, "quality_report.json"), "w"), indent=2)+ print(f"[conv] wrote quality_report.{{csv,xlsx,json}} | total exported pairs: {int(df['exported_pairs'].sum()):,}")+++if __name__ == "__main__":+ main()diff --git a/finetune/scripts/dgx01_smoke.sh b/finetune/scripts/dgx01_smoke.shnew file mode 100755index 00000000..e1813c77--- /dev/null+++ b/finetune/scripts/dgx01_smoke.sh@@ -0,0 +1,8 @@+#!/bin/bash+# 4-step smoke test of the 27B LoRA recipe on another node (the project dir is on shared wekafs, so+# scripts/env/models/data are all identical — only the GPUs differ).+cd /wekafs/ict/hx_624/llm-ft || exit 1+GPUS=${GPUS:-0,1,2,3} scripts/lf_train.sh configs/lf/lora_27b_v2.yaml max_steps=4 save_steps=100000 eval_steps=100000 \+ output_dir=/wekafs/ict/hx_624/llm-ft/runs/_smoke_$(hostname) > logs/smoke_$(hostname).log 2>&1+grep -oE "'train_runtime': [0-9.]+|out of memory|[0-9]/4 \[[0-9:]+<[0-9:]+, *[0-9.]+s/it" logs/smoke_$(hostname).log | tail -2+rm -rf runs/_smoke_$(hostname)diff --git a/finetune/scripts/finalize_glsl.py b/finetune/scripts/finalize_glsl.pynew file mode 100644index 00000000..bbc883c1--- /dev/null+++ b/finetune/scripts/finalize_glsl.py@@ -0,0 +1,85 @@+"""Final consolidation for the flattened Shadertoy corpus.++Order of preference, so the data stays as close to the author's source as possible:+ 1. the ORIGINAL merged program, if the (now correct) validator compiles it -> untouched+ 2. the agent-repaired program from glsl_repair/<cat>_fixed/, if it compiles -> repaired+ 3. otherwise -> kept, marked compiles=False+Writes the `compiles` / `repair` columns back into every shadertoy *_llamafactory_flat parquet.+"""+import collections, glob, hashlib, json, os, re, sys+from concurrent.futures import ProcessPoolExecutor+import pandas as pd+sys.path.insert(0, "/wekafs/ict/hx_624/llm-ft/scripts")+from qc_repair import code_of, glsl_compile++REPAIR = "/wekafs/ict/hx_624/glsl_repair"++def load_fixed():+ out = {}+ for cat in ("overload", "undeclared", "const_init", "syntax", "redefinition", "type", "other"):+ d = f"{REPAIR}/{cat}_fixed"+ idx = f"{REPAIR}/{cat}/index.json"+ if not (os.path.isdir(d) and os.path.exists(idx)):+ continue+ meta = json.load(open(idx))+ for f in glob.glob(f"{d}/*.glsl"):+ stem = os.path.basename(f)[:-5]+ h = (meta.get(stem) or {}).get("hash")+ if h:+ out[h] = (cat, open(f).read())+ return out++def job(args):+ h, body, fixed = args+ ok, _ = glsl_compile(body)+ if ok:+ return h, True, "original", None+ if fixed:+ cat, code = fixed+ ok2, _ = glsl_compile(code)+ if ok2:+ return h, True, f"repaired:{cat}", code+ return h, False, "unfixable", None++def main():+ fixed = load_fixed()+ print(f"[fin] agent repairs available: {len(fixed)}", flush=True)+ dirs = sorted(glob.glob("/wekafs/ict/hx_624/hf_subdirs/shadertoy/*_llamafactory_flat"))+ uniq, where = {}, collections.defaultdict(list)+ for d in dirs:+ df = pd.read_parquet(os.path.join(d, "train.parquet"))+ for i, row in enumerate(df.itertuples()):+ b = code_of(row.conversations[1]["value"]); h = hashlib.md5(b.encode()).hexdigest()+ uniq.setdefault(h, b); where[h].append(d)+ items = [(h, b, fixed.get(h)) for h, b in uniq.items()]+ print(f"[fin] validating {len(items)} unique programs", flush=True)+ res = {}+ with ProcessPoolExecutor(28) as ex:+ for n, (h, ok, how, code) in enumerate(ex.map(job, items, chunksize=8)):+ res[h] = (ok, how, code)+ if (n+1) % 10000 == 0:+ g = sum(1 for v in res.values() if v[0])+ print(f"[fin] {n+1}/{len(items)} ok={g} ({100*g/(n+1):.1f}%)", flush=True)+ stats = collections.Counter(v[1] for v in res.values())+ for d in dirs:+ p = os.path.join(d, "train.parquet"); df = pd.read_parquet(p)+ conv = df["conversations"].tolist(); okc, howc = [], []+ for c in conv:+ b = code_of(c[1]["value"]); h = hashlib.md5(b.encode()).hexdigest()+ ok, how, code = res.get(h, (False, "unknown", None))+ if code:+ c[1]["value"] = "```glsl\n" + code.strip() + "\n```"+ okc.append(ok); howc.append(how)+ df["conversations"] = conv; df["compiles"] = okc; df["repair"] = howc+ df.to_parquet(p, index=False, compression="zstd")+ json.dump({"rows": len(df), "compiles": int(sum(okc)),+ "repaired": int(sum(1 for x in howc if x.startswith("repaired")))},+ open(os.path.join(d, "qc.json"), "w"), indent=2)+ good = sum(1 for v in res.values() if v[0])+ json.dump({"unique_programs": len(items), "compile_ok": good,+ "compile_rate": round(good/len(items), 4), "how": dict(stats)},+ open("/wekafs/ict/hx_624/hf_subdirs/glsl_final_report.json", "w"), indent=2)+ print(f"[fin] FINAL {good}/{len(items)} = {100*good/len(items):.1f}% | {dict(stats)}", flush=True)++if __name__ == "__main__":+ main()diff --git a/finetune/scripts/flatten_multifile.py b/finetune/scripts/flatten_multifile.pynew file mode 100644index 00000000..23e9af06--- /dev/null+++ b/finetune/scripts/flatten_multifile.py@@ -0,0 +1,181 @@+"""Flatten every multi-file sample into ONE self-contained file, so the whole corpus is single-file SFT data.++Two sources are multi-file in the original dataset and both can be flattened exactly:++ shadertoy `shader_json.renderpass` holds the source of every tab. Shadertoy itself compiles+ `Common + <pass>`, so we emit that concatenation verbatim:+ * the Image pass with Common prepended → the sample's main program+ * each Buffer pass with Common prepended → an extra sample (it is a complete shader too)+ A shader whose passes read `iChannel` as a *buffer* is still emitted, with a one-line comment+ stating which channels are buffers, because the code itself is complete.+ articraft `code` is empty in the metadata; the real `code.urdf` lives inside the subdir tars, which are+ downloaded separately and extracted by `--urdf_tars`.++Everything is then re-emitted in LLaMA-Factory ShareGPT format next to the other converted subdirs.++usage:+ python scripts/flatten_multifile.py --shadertoy [--out /wekafs/ict/hx_624/hf_subdirs]+ python scripts/flatten_multifile.py --urdf_tars /wekafs/ict/hx_624/data/3dcodeverse_tars+"""+import argparse+import collections+import glob+import io+import json+import os+import re+import tarfile++import pandas as pd++META = "/wekafs/ict/hx_624/data/3dcodeverse_meta"+SYS_GLSL = ("You are an expert shader programmer. Given a description, write a complete Shadertoy-style GLSL ES 3.00 "+ "fragment shader defining `void mainImage(out vec4 fragColor, in vec2 fragCoord)`. Output ONLY the code in one ```glsl block.")+SYS_URDF = ("You are an expert in articulated 3D assets. Given a description, write a complete, self-contained URDF "+ "file (links, joints, inertial and visual geometry) for it. Output ONLY the code in one ```xml block.")+IMPERATIVE = ("write", "create", "implement", "make", "build", "generate", "design", "produce")+++def phrase(kind, text, lead):+ t = (text or "").strip()+ if not t:+ return None+ if kind == "instruction" and t.lower().startswith(IMPERATIVE):+ return t+ return lead + t+++def flatten_shadertoy(out_root):+ subs = sorted(glob.glob(f"{META}/shadertoy/*/metadata.parquet"))+ stats = collections.Counter()+ for p in subs:+ sub = "/".join(p.split("/")[-3:-1])+ df = pd.read_parquet(p)+ rows = []+ for r in df.itertuples():+ try:+ sj = json.loads(r.shader_json) if isinstance(r.shader_json, str) else (r.shader_json or {})+ except Exception:+ sj = {}+ passes = (sj.get("Shader", sj) or {}).get("renderpass") or []+ if not passes:+ continue+ common = "\n".join(q.get("code") or "" for q in passes if q.get("type") == "common").strip()+ caps = r.captions if isinstance(r.captions, dict) else (json.loads(r.captions) if isinstance(r.captions, str) else {})+ for q in passes:+ t = q.get("type")+ if t not in ("image", "buffer"):+ continue # sound/cubemap passes are a different program shape+ body = (q.get("code") or "").strip()+ if len(body) < 40:+ continue+ buf_inputs = [i for i in (q.get("inputs") or []) if (i or {}).get("ctype") == "buffer"]+ header = ""+ if common:+ header += "// ---- Common tab ----\n" + common + "\n// ---- " + str(q.get("name")) + " ----\n"+ stats["merged_common"] += 1+ if buf_inputs:+ header = f"// note: iChannel{','.join(str(i.get('channel')) for i in buf_inputs)} read a previous buffer pass\n" + header+ code = header + body+ tag = "" if t == "image" else f"::{str(q.get('name')).replace(' ', '')}"+ lead = "Write a GLSL fragment shader for Shadertoy that renders: "+ if t == "buffer":+ lead = "Write the buffer pass of a Shadertoy shader that computes: "+ used = 0+ for kind in ("instruction", "detailed", "factory"):+ pr = phrase(kind, caps.get(kind), lead)+ if not pr or len(pr) < 20:+ continue+ used += 1+ rows.append({"id": f"{r.id}{tag}", "caption_type": kind, "pass_type": t,+ "has_common": bool(common), "system": SYS_GLSL,+ "conversations": [{"from": "human", "value": pr},+ {"from": "gpt", "value": f"```glsl\n{code}\n```"}]})+ stats["pass_" + t] += 1 if used else 0+ if not rows:+ continue+ d = os.path.join(out_root, sub + "_llamafactory_flat")+ os.makedirs(d, exist_ok=True)+ pd.DataFrame(rows).to_parquet(os.path.join(d, "train.parquet"), index=False, compression="zstd")+ key = "3dcv_" + sub.replace("/", "__") + "_flat"+ json.dump({key: {"file_name": "train.parquet", "formatting": "sharegpt",+ "columns": {"messages": "conversations", "system": "system"}}},+ open(os.path.join(d, "dataset_info.json"), "w"), indent=2)+ stats["rows"] += len(rows)+ stats["subdirs"] += 1+ print(f"[flat] shadertoy: {stats['subdirs']} subdirs, {stats['rows']:,} rows "+ f"(image passes {stats['pass_image']:,}, buffer passes {stats['pass_buffer']:,}, "+ f"{stats['merged_common']:,} programs got their Common tab merged in)", flush=True)+ return dict(stats)+++def extract_urdf(tar_dir, out_root):+ """pull code.urdf out of the articraft tars and re-emit the two subdirs with real code"""+ code_by_key = {}+ for t in sorted(glob.glob(f"{tar_dir}/articraft/urdf_*/*.tar")):+ sub = t.split("/")[-2]+ try:+ with tarfile.open(t) as tf:+ for m in tf:+ if not m.isfile() or not m.name.endswith(".urdf"):+ continue+ key = os.path.basename(os.path.dirname(m.name))+ try:+ code_by_key[(sub, key)] = tf.extractfile(m).read().decode("utf-8", "replace")+ except Exception:+ pass+ except Exception as e:+ print(f"[flat] tar unreadable {t}: {e}", flush=True)+ print(f"[flat] urdf: extracted {len(code_by_key):,} .urdf files from {tar_dir}", flush=True)+ total = 0+ for sub in ("articraft/urdf_geo_only", "articraft/urdf_tex"):+ p = f"{META}/{sub}/metadata.parquet"+ if not os.path.exists(p):+ continue+ df = pd.read_parquet(p)+ rows, miss = [], 0+ for r in df.itertuples():+ key = str(r.key) if hasattr(r, "key") else None+ code = code_by_key.get((sub.split("/")[-1], key)) or code_by_key.get((sub.split("/")[-1], str(r.id).split("/")[-1]))+ if not code or len(code.strip()) < 40:+ miss += 1+ continue+ caps = r.captions if isinstance(r.captions, dict) else (json.loads(r.captions) if isinstance(r.captions, str) else {})+ for kind in ("instruction", "detailed", "factory"):+ pr = phrase(kind, caps.get(kind), "Write a URDF description of: ")+ if not pr or len(pr) < 20:+ continue+ rows.append({"id": str(r.id), "caption_type": kind, "system": SYS_URDF,+ "conversations": [{"from": "human", "value": pr},+ {"from": "gpt", "value": f"```xml\n{code.strip()}\n```"}]})+ if rows:+ d = os.path.join(out_root, sub + "_llamafactory")+ os.makedirs(d, exist_ok=True)+ pd.DataFrame(rows).to_parquet(os.path.join(d, "train.parquet"), index=False, compression="zstd")+ key = "3dcv_" + sub.replace("/", "__")+ json.dump({key: {"file_name": "train.parquet", "formatting": "sharegpt",+ "columns": {"messages": "conversations", "system": "system"}}},+ open(os.path.join(d, "dataset_info.json"), "w"), indent=2)+ json.dump({"subdir": sub, "dialect": "urdf", "source_samples": len(df), "exported_pairs": len(rows),+ "unique_samples_exported": len(df) - miss, "missing_in_tar": miss,+ "note": "code recovered from the subdir tars (the metadata `code` column is empty)"},+ open(os.path.join(d, "qc.json"), "w"), indent=2)+ print(f"[flat] {sub}: {len(rows):,} pairs from {len(df)-miss:,} samples ({miss} not found in the tars)", flush=True)+ total += len(rows)+ return total+++def main():+ ap = argparse.ArgumentParser()+ ap.add_argument("--out", default="/wekafs/ict/hx_624/hf_subdirs")+ ap.add_argument("--shadertoy", action="store_true")+ ap.add_argument("--urdf_tars", default=None)+ a = ap.parse_args()+ if a.shadertoy:+ flatten_shadertoy(a.out)+ if a.urdf_tars:+ extract_urdf(a.urdf_tars, a.out)+++if __name__ == "__main__":+ main()diff --git a/finetune/scripts/node_smoke.sh b/finetune/scripts/node_smoke.shnew file mode 100755index 00000000..ad7ec2ce--- /dev/null+++ b/finetune/scripts/node_smoke.sh@@ -0,0 +1,8 @@+#!/bin/bash+# smoke a config on whatever node/GPUs we are given: scripts/node_smoke.sh <config.yaml> <tag> [steps]+cd /wekafs/ict/hx_624/llm-ft || exit 1+CFG=$1; TAG=$2; N=${3:-4}+GPUS=${GPUS:-0,1,2,3} scripts/lf_train.sh $CFG max_steps=$N save_steps=100000 eval_steps=100000 \+ output_dir=/wekafs/ict/hx_624/llm-ft/runs/_smoke_$TAG > logs/smoke_$TAG.log 2>&1+grep -oE "'train_runtime': [0-9.]+|out of memory|[0-9]/$N \[[0-9:]+<[0-9:]+, *[0-9.]+s/it" logs/smoke_$TAG.log | tail -2+rm -rf runs/_smoke_$TAGdiff --git a/finetune/scripts/qc_repair.py b/finetune/scripts/qc_repair.pynew file mode 100644index 00000000..22fe2afd--- /dev/null+++ b/finetune/scripts/qc_repair.py@@ -0,0 +1,434 @@+"""Validate every converted sample with a real compiler/parser and repair what can be repaired minimally.++Three passes, each writing a report and (where it helps) a repaired copy of the subdir dataset:++ glsl compile every unique shader with glslangValidator (the same wrapper the evaluator uses).+ Repairs attempted, smallest first, each re-validated before it is accepted:+ 1. drop duplicated Shadertoy header blocks (repeated `#version` / uniform declarations)+ 2. balance braces/parens at EOF (append the missing closers)+ 3. drop a trailing incomplete statement (last line without `;`/`}`)+ 4. wrap a bare body in `void mainImage(out vec4 fragColor, in vec2 fragCoord){ ... }`+ python ast.parse every Blender/CadQuery program; repairs: strip a trailing truncated line, close+ unbalanced brackets. (No cosmetic rewrites — a program either parses or it does not.)+ dedup find exact-duplicate programs *within* and *across* subdirs of the same source and write the+ duplicate groups so the upstream dataset can be cleaned too.++usage: python scripts/qc_repair.py --root /wekafs/ict/hx_624/hf_subdirs [--pass glsl|python|dedup|all] [--workers 32]+"""+import argparse+import ast+import collections+import glob+import hashlib+import itertools+import json+import os+import re+import subprocess+import sys+import tempfile+from concurrent.futures import ProcessPoolExecutor++import pandas as pd++GLSLANG = "/wekafs/ict/hx_624/anaconda3/envs/llmft/bin/glslangValidator"+PRELUDE = """#version 310 es+precision highp float; precision highp int;+uniform vec3 iResolution; uniform float iTime; uniform float iTimeDelta; uniform int iFrame;+uniform vec4 iMouse; uniform vec4 iDate; uniform float iSampleRate;+uniform float iChannelTime[4]; uniform vec3 iChannelResolution[4];+uniform sampler2D iChannel0; uniform sampler2D iChannel1; uniform sampler2D iChannel2; uniform sampler2D iChannel3;+out vec4 _fragColor;+"""+++def code_of(msg):+ m = re.findall(r"```[a-zA-Z]*\n(.*?)```", msg, flags=re.S)+ return (max(m, key=len) if m else msg).strip()+++def _prelude(version, cube_channels=()):+ ch = "".join(f"uniform {'samplerCube' if i in cube_channels else 'sampler2D'} iChannel{i};\n" for i in range(4))+ return (f"#version {version}\nprecision highp float; precision highp int;\n"+ "uniform vec3 iResolution; uniform float iTime; uniform float iTimeDelta; uniform int iFrame;\n"+ "uniform vec4 iMouse; uniform vec4 iDate; uniform float iSampleRate; uniform float iFrameRate;\n"+ "uniform float iChannelTime[4]; uniform vec3 iChannelResolution[4];\n" + ch + "out vec4 _fragColor;\n")+++CUBE_USE = re.compile(r"texture(?:Lod)?\s*\(\s*iChannel([0-3])\s*,\s*(?:vec3|normalize|reflect|refract|[a-zA-Z_]\w*\s*\.\s*xyz)")+EPILOGUE = "\nvoid main(){ mainImage(_fragColor, gl_FragCoord.xy); }\n"+STRIP = re.compile(r"^[ \t]*(?:#version[^\n]*|precision\s+\w+\s+\w+\s*;|uniform\s+(?:vec[234]|float|int|sampler2D|samplerCube)\s+\w+(?:\[\d+\])?\s*;|out\s+vec4\s+\w+\s*;)[ \t]*$", re.M) # whole line only: a glued/minified line carries real code+++def _try_compile(src):+ with tempfile.NamedTemporaryFile("w", suffix=".frag", delete=False) as f:+ f.write(src); p = f.name+ try:+ r = subprocess.run([GLSLANG, "-S", "frag", p], stdout=subprocess.PIPE, stderr=subprocess.STDOUT,+ timeout=25, text=True, errors="replace")+ out = r.stdout or ""+ return r.returncode == 0, next((l for l in out.splitlines() if "ERROR:" in l), "")[:160]+ except Exception as e:+ return False, str(e)[:160]+ finally:+ os.unlink(p)+++USED_CH = re.compile(r"iChannel([0-3])")+++def glsl_compile(body):+ """Shadertoy targets WebGL2 (GLSL ES 3.00) and each iChannel may be a 2D texture OR a cubemap — which one+ is a property of the *shader's channel bindings*, not of the source text. Guessing it from the coordinate+ expression is unreliable (a repair agent brute-forced it and found 97% of "no matching overloaded function"+ failures were just a wrong guess), so try the actual assignments: for the channels the shader uses, iterate+ over 2D/cube combinations under both ES profiles and accept if ANY compiles."""+ stripped = STRIP.sub("", body)+ used = sorted({int(m.group(1)) for m in USED_CH.finditer(stripped)})+ combos = [()]+ if used:+ combos = [tuple(c for c, bit in zip(used, bits) if bit)+ for bits in itertools.product((0, 1), repeat=len(used))]+ combos.sort(key=len) # all-2D first, then progressively more cubemaps+ last = ""+ for ver in ("300 es", "310 es"):+ for cc in combos:+ ok, err = _try_compile(_prelude(ver, cc) + stripped + EPILOGUE)+ if ok:+ return True, ""+ last = last or err+ return False, last+++def dedup_headers(body):+ """shadertoy forks often repeat the whole header; keep the first occurrence of each duplicated line block"""+ lines = body.splitlines()+ seen, out, removed = set(), [], 0+ for ln in lines:+ s = ln.strip()+ if s.startswith(("#version", "precision ", "uniform ", "out vec4")) and s in seen:+ removed += 1+ continue+ if s:+ seen.add(s)+ out.append(ln)+ return ("\n".join(out), removed) if removed else (body, 0)+++def balance(body):+ add = ""+ for o, c in (("{", "}"), ("(", ")")):+ d = body.count(o) - body.count(c)+ if d > 0:+ add += c * d+ return (body + "\n" + add if add else body), len(add)+++def drop_tail(body):+ lines = body.rstrip().splitlines()+ while lines and not lines[-1].strip().endswith((";", "}", "{")):+ lines.pop()+ return "\n".join(lines), 1+++def wrap_main(body):+ if "mainImage" in body or "void main" in body:+ return body, 0+ return "void mainImage(out vec4 fragColor, in vec2 fragCoord){\n" + body + "\n}", 1+++def repair_glsl(body):+ """try repairs smallest-first; return (fixed_body, recipe) or (None, reason)"""+ ok, err = glsl_compile(body)+ if ok:+ return body, "already_ok"+ steps = []+ cur = body+ for name, fn in (("dedup_headers", dedup_headers), ("balance", balance), ("drop_tail", drop_tail), ("wrap_main", wrap_main)):+ new, n = fn(cur)+ if n == 0 or new == cur:+ continue+ cur = new+ steps.append(name)+ ok, err = glsl_compile(cur)+ if ok:+ return cur, "+".join(steps)+ return None, err or "unfixable"+++def _glsl_job(args):+ key, body = args+ fixed, recipe = repair_glsl(body)+ return key, (fixed is not None), recipe, (fixed if fixed and recipe != "already_ok" else None)+++def shadertoy_provenance():+ """id -> flags the dataset itself records: a shader that needs a Common tab, extra buffers or external+ textures CANNOT compile standalone — that is a property of the sample, not a defect to repair."""+ prov = {}+ for p in glob.glob("/wekafs/ict/hx_624/data/3dcodeverse_meta/shadertoy/*/metadata.parquet"):+ df = pd.read_parquet(p, columns=["id", "meta_json"])+ for r in df.itertuples():+ try:+ m = json.loads(r.meta_json) if isinstance(r.meta_json, str) else (r.meta_json or {})+ except Exception:+ m = {}+ prov[str(r.id)] = {"num_passes": m.get("num_passes") or 1,+ "has_common_pass": bool(m.get("has_common_pass")),+ "external_assets": bool(m.get("external_assets")),+ "multi_file": bool(m.get("multi_file"))}+ return prov+++def classify(err, flags):+ if flags.get("has_common_pass"):+ return "expected_needs_common_pass"+ if (flags.get("num_passes") or 1) > 1:+ return "expected_multi_pass"+ if flags.get("external_assets"):+ return "expected_external_assets"+ e = err or ""+ if "syntax error" in e or "unexpected" in e:+ return "syntax_error"+ if "undeclared identifier" in e or "no matching overloaded function" in e:+ return "missing_symbol"+ return "other"+++def pass_glsl(root, workers, limit=None):+ prov = shadertoy_provenance()+ print(f"[qc] provenance flags for {len(prov)} shadertoy samples", flush=True)+ dirs = sorted(glob.glob(f"{root}/shadertoy/*_llamafactory"))+ uniq, where, hid = {}, collections.defaultdict(list), {}+ for d in dirs:+ p = os.path.join(d, "train.parquet")+ if not os.path.exists(p):+ continue+ df = pd.read_parquet(p)+ for i, row in enumerate(df.itertuples()):+ body = code_of(row.conversations[1]["value"])+ h = hashlib.md5(body.encode()).hexdigest()+ uniq.setdefault(h, body)+ hid.setdefault(h, row.id)+ where[h].append((d, i))+ items = list(uniq.items())+ if limit:+ items = items[:limit]+ print(f"[qc] glsl: {len(items)} unique shaders from {len(dirs)} subdirs", flush=True)+ results = {}+ with ProcessPoolExecutor(workers) as ex:+ for n, (key, ok, recipe, fixed) in enumerate(ex.map(_glsl_job, items, chunksize=8)):+ results[key] = (ok, recipe, fixed)+ if (n + 1) % 5000 == 0:+ good = sum(1 for v in results.values() if v[0])+ print(f"[qc] glsl {n+1}/{len(items)} compile-ok {good} ({100*good/(n+1):.1f}%)", flush=True)+ stats = collections.Counter(v[1] for v in results.values())+ cls = collections.Counter()+ for h, (ok, recipe, _f) in results.items():+ if ok:+ cls["compiles" if recipe == "already_ok" else "repaired"] += 1+ else:+ cls[classify(recipe, prov.get(hid.get(h, ""), {}))] += 1+ fixed_map = {k: v[2] for k, v in results.items() if v[2]}+ # write repaired parquets+ per_dir = collections.Counter()+ for h, locs in where.items():+ if h not in fixed_map:+ continue+ for d, _i in locs:+ per_dir[d] += 1+ for d in dirs:+ p = os.path.join(d, "train.parquet")+ if not os.path.exists(p) or per_dir[d] == 0:+ continue+ df = pd.read_parquet(p)+ conv = df["conversations"].tolist()+ nfix = 0+ for i, c in enumerate(conv):+ body = code_of(c[1]["value"])+ h = hashlib.md5(body.encode()).hexdigest()+ if h in fixed_map:+ c[1]["value"] = "```glsl\n" + fixed_map[h].strip() + "\n```"+ nfix += 1+ df["conversations"] = conv+ df.to_parquet(p, index=False, compression="zstd")+ q = os.path.join(d, "qc.json")+ if os.path.exists(q):+ j = json.load(open(q))+ j["glsl_repaired_rows"] = nfix+ json.dump(j, open(q, "w"), indent=2)+ json.dump({"unique_shaders": len(items), "classification": dict(cls), "recipes": dict(stats.most_common(20)),+ "compiled_before": stats.get("already_ok", 0),+ "repaired": sum(v for k, v in stats.items() if k not in ("already_ok",) and not k.startswith("ERROR") and k != "unfixable"),+ "unfixable": sum(v for k, v in stats.items() if k == "unfixable" or k.startswith("ERROR"))},+ open(f"{root}/glsl_validation_report.json", "w"), indent=2)+ print("[qc] glsl classification:", dict(cls), flush=True)+++def pass_python(root):+ out = []+ for d in sorted(glob.glob(f"{root}/**/*_llamafactory", recursive=True)):+ q = json.load(open(os.path.join(d, "qc.json"))) if os.path.exists(os.path.join(d, "qc.json")) else {}+ if q.get("dialect") not in ("blender", "cadquery"):+ continue+ p = os.path.join(d, "train.parquet")+ df = pd.read_parquet(p)+ bad = 0+ for row in df.itertuples():+ try:+ ast.parse(code_of(row.conversations[1]["value"]))+ except Exception:+ bad += 1+ out.append({"subdir": os.path.relpath(d, root), "rows": len(df), "ast_fail": bad})+ print(f"[qc] python {os.path.relpath(d, root):45s} rows={len(df):7,} ast_fail={bad}", flush=True)+ json.dump(out, open(f"{root}/python_validation_report.json", "w"), indent=2)+++def pass_dedup(root):+ """exact duplicate programs within and across subdirs of the same source"""+ by_src = collections.defaultdict(lambda: collections.defaultdict(list))+ for d in sorted(glob.glob(f"{root}/**/*_llamafactory", recursive=True)):+ p = os.path.join(d, "train.parquet")+ if not os.path.exists(p):+ continue+ src = os.path.relpath(d, root).split("/")[0]+ df = pd.read_parquet(p)+ seen = set()+ for row in df.itertuples():+ if row.id in seen:+ continue+ seen.add(row.id)+ h = hashlib.md5(code_of(row.conversations[1]["value"]).encode()).hexdigest()+ by_src[src][h].append(row.id)+ report = {}+ for src, groups in by_src.items():+ dupes = {h: ids for h, ids in groups.items() if len(ids) > 1}+ n_extra = sum(len(v) - 1 for v in dupes.values())+ report[src] = {"unique_programs": len(groups), "duplicate_groups": len(dupes),+ "redundant_samples": n_extra,+ "examples": [{"kept": v[0], "duplicates": v[1:4]} for v in list(dupes.values())[:5]]}+ json.dump({h: ids for h, ids in dupes.items()}, open(f"{root}/duplicates_{src}.json", "w"), indent=2)+ print(f"[qc] dedup {src:20s} unique={len(groups):7,} dup_groups={len(dupes):6,} redundant={n_extra:6,}", flush=True)+ json.dump(report, open(f"{root}/duplicate_report.json", "w"), indent=2)++++def dedupe_decls(body):+ """after merging a Common tab, the same #define / struct / function can appear twice — a redefinition is a+ compile error, and even when it is not, it is pure redundancy. Drop the LATER exact-duplicate top-level+ declaration, keeping the first (Shadertoy semantics: Common comes first)."""+ lines = body.splitlines()+ seen_def, seen_sig, out, removed = set(), set(), [], 0+ i = 0+ sig_re = re.compile(r"^\s*(?:[A-Za-z_][\w]*\s+){1,3}([A-Za-z_]\w*)\s*\(([^)]*)\)\s*\{?\s*$")+ def_re = re.compile(r"^\s*#define\s+(\w+)")+ while i < len(lines):+ ln = lines[i]+ m = def_re.match(ln)+ if m:+ if ln.strip() in seen_def:+ removed += 1; i += 1; continue+ seen_def.add(ln.strip()); out.append(ln); i += 1; continue+ m = sig_re.match(ln)+ if m and "{" in ln:+ sig = (m.group(1), re.sub(r"\s+", "", m.group(2)))+ depth = ln.count("{") - ln.count("}")+ block = [ln]; j = i + 1+ while j < len(lines) and depth > 0:+ depth += lines[j].count("{") - lines[j].count("}")+ block.append(lines[j]); j += 1+ if sig in seen_sig:+ removed += len(block); i = j; continue+ seen_sig.add(sig); out.extend(block); i = j; continue+ out.append(ln); i += 1+ return ("\n".join(out), removed) if removed else (body, 0)+++def _flat_job(args):+ key, body = args+ ok, err = glsl_compile(body)+ if ok:+ slim, n = dedupe_decls(body)+ if n:+ ok2, _ = glsl_compile(slim)+ if ok2:+ return key, True, f"ok_slimmed({n}_lines)", slim+ return key, True, "ok", None+ cur, steps = body, []+ for name, fn in (("dedupe_decls", dedupe_decls), ("dedup_headers", dedup_headers), ("balance", balance), ("drop_tail", drop_tail)):+ new, n = fn(cur)+ if n == 0 or new == cur:+ continue+ cur = new; steps.append(name)+ ok, err = glsl_compile(cur)+ if ok:+ return key, True, "+".join(steps), cur+ return key, False, err or "unfixable", None+++def pass_glsl_flat(root, workers, limit=None):+ """validate + slim the flattened (Common-merged) shaders"""+ dirs = sorted(glob.glob(f"{root}/shadertoy/*_llamafactory_flat"))+ uniq, where = {}, collections.defaultdict(list)+ for d in dirs:+ p = os.path.join(d, "train.parquet")+ if not os.path.exists(p):+ continue+ df = pd.read_parquet(p)+ for i, row in enumerate(df.itertuples()):+ body = code_of(row.conversations[1]["value"])+ h = hashlib.md5(body.encode()).hexdigest()+ uniq.setdefault(h, body); where[h].append((d, i))+ items = list(uniq.items())+ if limit: items = items[:limit]+ print(f"[qc] flat: {len(items)} unique flattened programs from {len(dirs)} subdirs", flush=True)+ res = {}+ with ProcessPoolExecutor(workers) as ex:+ for n, (k, ok, recipe, fixed) in enumerate(ex.map(_flat_job, items, chunksize=8)):+ res[k] = (ok, recipe, fixed)+ if (n+1) % 10000 == 0:+ good = sum(1 for v in res.values() if v[0])+ print(f"[qc] flat {n+1}/{len(items)} compile-ok {good} ({100*good/(n+1):.1f}%)", flush=True)+ fixed_map = {k: v[2] for k, v in res.items() if v[2]}+ for d in dirs:+ p = os.path.join(d, "train.parquet")+ if not os.path.exists(p): continue+ df = pd.read_parquet(p); conv = df["conversations"].tolist(); okcol = []; nfix = 0+ for c in conv:+ body = code_of(c[1]["value"]); h = hashlib.md5(body.encode()).hexdigest()+ if h in fixed_map:+ c[1]["value"] = "```glsl\n" + fixed_map[h].strip() + "\n```"; nfix += 1+ okcol.append(bool(res.get(h, (False,))[0]))+ df["conversations"] = conv; df["compiles"] = okcol+ df.to_parquet(p, index=False, compression="zstd")+ json.dump({"rows": len(df), "compiles": int(sum(okcol)), "repaired_or_slimmed": nfix},+ open(os.path.join(d, "qc.json"), "w"), indent=2)+ cls = collections.Counter(v[1] if v[0] else "FAIL:" + v[1][:40] for v in res.values())+ good = sum(1 for v in res.values() if v[0])+ json.dump({"unique_programs": len(items), "compile_ok": good,+ "compile_rate": round(good/max(1,len(items)), 4),+ "top_outcomes": dict(cls.most_common(15))},+ open(f"{root}/glsl_flat_validation_report.json", "w"), indent=2)+ print(f"[qc] flat: {good}/{len(items)} compile ({100*good/max(1,len(items)):.1f}%)", flush=True)+ print("[qc] flat outcomes:", dict(cls.most_common(8)), flush=True)+++def main():+ ap = argparse.ArgumentParser()+ ap.add_argument("--root", default="/wekafs/ict/hx_624/hf_subdirs")+ ap.add_argument("--pass", dest="which", default="all", choices=["glsl", "glsl_flat", "python", "dedup", "all"])+ ap.add_argument("--workers", type=int, default=32)+ ap.add_argument("--limit", type=int, default=None)+ a = ap.parse_args()+ if a.which in ("dedup", "all"):+ pass_dedup(a.root)+ if a.which in ("python", "all"):+ pass_python(a.root)+ if a.which in ("glsl", "all"):+ pass_glsl(a.root, a.workers, a.limit)+ if a.which == "glsl_flat":+ pass_glsl_flat(a.root, a.workers, a.limit)+++if __name__ == "__main__":+ main()diff --git a/finetune/scripts/queue.txt b/finetune/scripts/queue.txtindex f69b7fa8..89ba99a4 100644--- a/finetune/scripts/queue.txt+++ b/finetune/scripts/queue.txt@@ -1,2 +1 @@-v3_smoke|GPUS=$(scripts/free_gpus.sh) scripts/lf_train.sh configs/lf/lora_27b_v3.yaml max_steps=6 save_steps=100000 eval_steps=100000 output_dir=/wekafs/ict/hx_624/llm-ft/runs/_smoke_v3 > logs/smoke_v3.log 2>&1; grep -q train_runtime logs/smoke_v3.log && { echo SMOKE_OK; rm -rf runs/_smoke_v3; } || { echo SMOKE_OOM; grep -o "Tried to allocate [0-9.]* GiB" logs/smoke_v3.log | tail -1; exit 1; }-v3_full|scripts/run_27b_v3.sh+v4_full|scripts/run_27b_v4.shdiff --git a/finetune/scripts/run_27b_v4.sh b/finetune/scripts/run_27b_v4.shnew file mode 100755index 00000000..4c9db177--- /dev/null+++ b/finetune/scripts/run_27b_v4.sh@@ -0,0 +1,51 @@+#!/bin/bash+# 27B v4: v2's winning blender-heavy core + CadQuery raised 33k -> 105k unique samples (one caption each).+# Question: can a single 27B top both 3DCodeBench (v2: 93.4%) and CadQuery (9B md_xl: 97.5%)?+#+# 27B LoRA on 80 GB cards sits right at the memory edge — the identical recipe that trained v2 OOMs on other+# days. So instead of guessing, try a ladder of settings (keeping lora_rank 64, the quality-relevant knob;+# every sample in this mix is <=3400 tokens, so a smaller cutoff truncates nothing) and take the first that+# survives 4 real steps.+set -uo pipefail+cd /wekafs/ict/hx_624/llm-ft+GPUS_USE=$(scripts/free_gpus.sh); NGPU=$(echo "$GPUS_USE" | awk -F, '{print NF}')+[ -z "$GPUS_USE" ] && { echo "[27bv4] no free GPUs"; exit 1; }+RUN=runs/lf_qwen38_27b_lora_v4++WIN=""+for SPEC in "3584 5 adamw_torch" "3072 6 adamw_torch" "3072 6 adamw_8bit" "2560 8 adamw_8bit"; do+ set -- $SPEC; CUT=$1; ACC=$2; OPT=$3+ echo "[27bv4] $(date) probing cutoff=$CUT accum=$ACC optim=$OPT"+ EXTRA=""; [ "$OPT" = "adamw_8bit" ] && EXTRA="optim=adamw_8bit"+ GPUS=$GPUS_USE scripts/lf_train.sh configs/lf/lora_27b_v4.yaml max_steps=4 save_steps=100000 eval_steps=100000 \+ cutoff_len=$CUT gradient_accumulation_steps=$ACC $EXTRA \+ tokenized_path=/wekafs/ict/hx_624/llm-ft/data/lf/tok_v4_$CUT \+ output_dir=/wekafs/ict/hx_624/llm-ft/runs/_probe_v4 > logs/probe_v4_$CUT$OPT.log 2>&1+ rm -rf runs/_probe_v4+ if grep -q "train_runtime" logs/probe_v4_$CUT$OPT.log; then+ S=$(grep -oE "[0-9]/4 \[[0-9:]+<[0-9:]+, *[0-9.]+s/it" logs/probe_v4_$CUT$OPT.log | tail -1)+ echo "[27bv4] OK at cutoff=$CUT accum=$ACC optim=$OPT ($S)"; WIN="$CUT $ACC $OPT"; break+ fi+ ERR=$(grep -oE "torch.OutOfMemoryError[^\"]*|Triton Error \[CUDA\]: out of memory|RuntimeError: [^\"]{0,120}" logs/probe_v4_$CUT$OPT.log | tail -1)+ echo "[27bv4] FAILED at cutoff=$CUT accum=$ACC optim=$OPT -> ${ERR:-unknown error, see logs/probe_v4_$CUT$OPT.log}"+ case "$ERR" in *"Triton >= 3.4"*|*"tilelang"*) echo "[27bv4] ABORT: the lf env's triton was downgraded (Hopper needs 3.7.1) — fix the env, not the config"; exit 2;; esac+done+[ -z "$WIN" ] && { echo "[27bv4] every setting OOMed — needs more GPUs or a sharded optimizer"; exit 1; }+set -- $WIN; CUT=$1; ACC=$2; OPT=$3+EXTRA=""; [ "$OPT" = "adamw_8bit" ] && EXTRA="optim=adamw_8bit"++echo "[27bv4] $(date) train (172.6k pairs / 114M tok, cutoff=$CUT accum=$ACC optim=$OPT, GPUs $GPUS_USE)"+GPUS=$GPUS_USE scripts/lf_train.sh configs/lf/lora_27b_v4.yaml cutoff_len=$CUT gradient_accumulation_steps=$ACC $EXTRA \+ tokenized_path=/wekafs/ict/hx_624/llm-ft/data/lf/tok_v4_$CUT > logs/train_27b_v4.log 2>&1+[ -f $RUN/train_results.json ] || { echo "[27bv4] TRAIN FAILED: $(grep -E 'out of memory|Error' logs/train_27b_v4.log | grep -v errors | tail -1 | cut -c1-160)"; exit 1; }+echo "[27bv4] $(date) train done: $(tr -d '\n' < $RUN/train_results.json | cut -c1-160)"++sed -e "s|model_name_or_path:.*|model_name_or_path: /wekafs/ict/hx_624/models/Qwen3.8-27B|" \+ -e "s|adapter_name_or_path:.*|adapter_name_or_path: $(readlink -f $RUN)|" \+ -e "s|export_dir:.*|export_dir: $(readlink -f $RUN)/merged|" configs/lf/export_lora_template.yaml > configs/lf/export_27b_v4.yaml+scripts/lf_export.sh configs/lf/export_27b_v4.yaml > logs/export_27b_v4.log 2>&1 || { echo "[27bv4] EXPORT FAILED"; exit 1; }+echo "[27bv4] $(date) eval greedy"+GPUS=$GPUS_USE TP=2 eval/eval_all_run2.sh $RUN/merged q27b_v4 --max_new 8192 2>&1 | grep -E "gen-multi|status counts|dialect|DONE" | tail -20+echo "[27bv4] $(date) eval sampled (openscad/glsl)"+GPUS=$GPUS_USE TP=2 eval/eval_all_run2.sh $RUN/merged q27b_v4_T07 --temp 0.7 --seed 1 --max_new 8192 --suites openscad glsl 2>&1 | grep -E "dialect|DONE" | tail -4+echo "[27bv4] ALL DONE"