Skip to content

Official Codebase of 3DCodeVerse

Python

finetune: 27B v2 (blender-heavy + bootstrapped data) reaches 3DCodeBench 93.4% / Blender-heldout F 0.890; caption-augmentation ablation (md_max_9b vs md_v3), scipy env fix + rescored runs, expandable_segments OOM pitfall; REPORT 11.6-11.10

gaoypengd4edd5f7fb5c
diff --git a/finetune/configs/lf/export_27b_max.yaml b/finetune/configs/lf/export_27b_max.yaml
new file mode 100644
index 00000000..4c6ed795
--- /dev/null
+++ b/finetune/configs/lf/export_27b_max.yaml
@@ -0,0 +1,12 @@
+### Merge a LoRA adapter into the base model (LLaMA-Factory): llamafactory-cli export <this file>
+model_name_or_path: /wekafs/ict/hx_624/models/Qwen3.8-27B
+adapter_name_or_path: /wekafs/ict/hx_624/llm-ft/runs/lf_qwen38_27b_lora_md_max
+template: qwen3_5
+enable_thinking: false
+trust_remote_code: true
+finetuning_type: lora
+
+export_dir: /wekafs/ict/hx_624/llm-ft/runs/lf_qwen38_27b_lora_md_max/merged
+export_size: 5
+export_device: cpu
+export_legacy_format: false
diff --git a/finetune/configs/lf/export_27b_v2.yaml b/finetune/configs/lf/export_27b_v2.yaml
new file mode 100644
index 00000000..73ae5c5d
--- /dev/null
+++ b/finetune/configs/lf/export_27b_v2.yaml
@@ -0,0 +1,12 @@
+### Merge a LoRA adapter into the base model (LLaMA-Factory): llamafactory-cli export <this file>
+model_name_or_path: /wekafs/ict/hx_624/models/Qwen3.8-27B
+adapter_name_or_path: /wekafs/ict/hx_624/llm-ft/runs/lf_qwen38_27b_lora_v2
+template: qwen3_5
+enable_thinking: false
+trust_remote_code: true
+finetuning_type: lora
+
+export_dir: /wekafs/ict/hx_624/llm-ft/runs/lf_qwen38_27b_lora_v2/merged
+export_size: 5
+export_device: cpu
+export_legacy_format: false
diff --git a/finetune/configs/lf/export_md_max_9b.yaml b/finetune/configs/lf/export_md_max_9b.yaml
new file mode 100644
index 00000000..e8581c46
--- /dev/null
+++ b/finetune/configs/lf/export_md_max_9b.yaml
@@ -0,0 +1,12 @@
+### Merge a LoRA adapter into the base model (LLaMA-Factory): llamafactory-cli export <this file>
+model_name_or_path: /wekafs/ict/hx_624/models/Qwen3.5-9B
+adapter_name_or_path: /wekafs/ict/hx_624/llm-ft/runs/lf_qwen35_9b_lora_md_max
+template: qwen3_5
+enable_thinking: false
+trust_remote_code: true
+finetuning_type: lora
+
+export_dir: /wekafs/ict/hx_624/llm-ft/runs/lf_qwen35_9b_lora_md_max/merged
+export_size: 5
+export_device: cpu
+export_legacy_format: false
diff --git a/finetune/configs/lf/export_md_v3.yaml b/finetune/configs/lf/export_md_v3.yaml
new file mode 100644
index 00000000..642197cb
--- /dev/null
+++ b/finetune/configs/lf/export_md_v3.yaml
@@ -0,0 +1,12 @@
+### Merge a LoRA adapter into the base model (LLaMA-Factory): llamafactory-cli export <this file>
+model_name_or_path: /wekafs/ict/hx_624/models/Qwen3.5-9B
+adapter_name_or_path: /wekafs/ict/hx_624/llm-ft/runs/lf_qwen35_9b_lora_md_v3
+template: qwen3_5
+enable_thinking: false
+trust_remote_code: true
+finetuning_type: lora
+
+export_dir: /wekafs/ict/hx_624/llm-ft/runs/lf_qwen35_9b_lora_md_v3/merged
+export_size: 5
+export_device: cpu
+export_legacy_format: false
diff --git a/finetune/configs/lf/lora_27b_v2.yaml b/finetune/configs/lf/lora_27b_v2.yaml
new file mode 100644
index 00000000..7a609a1b
--- /dev/null
+++ b/finetune/configs/lf/lora_27b_v2.yaml
@@ -0,0 +1,44 @@
+model_name_or_path: /wekafs/ict/hx_624/models/Qwen3.8-27B
+trust_remote_code: true
+flash_attn: fa2
+stage: sft
+do_train: true
+finetuning_type: lora
+lora_rank: 64
+lora_alpha: 128
+lora_dropout: 0.05
+lora_target: all
+freeze_vision_tower: true
+freeze_multi_modal_projector: true
+dataset_dir: /wekafs/ict/hx_624/llm-ft/data/lf
+dataset: md_27b_v2s_train
+eval_dataset: md_27b_v2s_val
+template: qwen3_5
+enable_thinking: false
+cutoff_len: 4096
+preprocessing_num_workers: 16
+dataloader_num_workers: 4
+neat_packing: true
+packing: true
+tokenized_path: /wekafs/ict/hx_624/llm-ft/data/lf/tokenized_md_27b_v2s_len4096_packed
+output_dir: /wekafs/ict/hx_624/llm-ft/runs/lf_qwen38_27b_lora_v2
+logging_steps: 5
+save_steps: 250
+save_only_model: true
+plot_loss: true
+overwrite_output_dir: true
+report_to: tensorboard
+per_device_train_batch_size: 1
+gradient_accumulation_steps: 4
+learning_rate: 0.0001
+num_train_epochs: 1.0
+lr_scheduler_type: cosine
+warmup_ratio: 0.05
+bf16: true
+gradient_checkpointing: true
+ddp_timeout: 180000000
+resume_from_checkpoint: null
+per_device_eval_batch_size: 1
+eval_strategy: steps
+eval_steps: 250
+save_total_limit: 8
diff --git a/finetune/configs/lf/lora_md_v3.yaml b/finetune/configs/lf/lora_md_v3.yaml
new file mode 100644
index 00000000..a265ff27
--- /dev/null
+++ b/finetune/configs/lf/lora_md_v3.yaml
@@ -0,0 +1,44 @@
+model_name_or_path: /wekafs/ict/hx_624/models/Qwen3.5-9B
+trust_remote_code: true
+flash_attn: fa2
+stage: sft
+do_train: true
+finetuning_type: lora
+lora_rank: 64
+lora_alpha: 128
+lora_dropout: 0.05
+lora_target: all
+freeze_vision_tower: true
+freeze_multi_modal_projector: true
+dataset_dir: /wekafs/ict/hx_624/llm-ft/data/lf
+dataset: md_v3_train
+eval_dataset: md_v3_val
+template: qwen3_5
+enable_thinking: false
+cutoff_len: 8192
+preprocessing_num_workers: 16
+dataloader_num_workers: 4
+neat_packing: true
+packing: true
+tokenized_path: /wekafs/ict/hx_624/llm-ft/data/lf/tokenized_md_v3_len8192_packed
+output_dir: /wekafs/ict/hx_624/llm-ft/runs/lf_qwen35_9b_lora_md_v3
+logging_steps: 5
+save_steps: 500
+save_only_model: true
+plot_loss: true
+overwrite_output_dir: true
+report_to: tensorboard
+per_device_train_batch_size: 1
+gradient_accumulation_steps: 8
+learning_rate: 0.0001
+num_train_epochs: 1.0
+lr_scheduler_type: cosine
+warmup_ratio: 0.05
+bf16: true
+gradient_checkpointing: true
+ddp_timeout: 180000000
+resume_from_checkpoint: null
+per_device_eval_batch_size: 1
+eval_strategy: steps
+eval_steps: 500
+save_total_limit: 10
diff --git a/finetune/docs/SETUP.md b/finetune/docs/SETUP.md
index 481f4d83..2d2b0aad 100644
--- a/finetune/docs/SETUP.md
+++ b/finetune/docs/SETUP.md
@@ -51,7 +51,7 @@ pip uninstall -y torchaudio # a mismatched torchaudio bre
## 2. External tools (called by absolute path from `eval/`)
| tool | how | path key |
|---|---|---|
-| Blender 5.0.1 (headless; executes generated bpy code, exports GLB) | `wget https://download.blender.org/release/Blender5.0/blender-5.0.1-linux-x64.tar.xz && tar xf … -C $WORKSPACE/tools` | `BLENDER` in `eval/run_bench.py`, `eval/blender_dialect_eval.py`, `scripts/*dpo*.sh` |
+| Blender 5.0.1 (headless; executes generated bpy code, exports GLB) | `wget https://download.blender.org/release/Blender5.0/blender-5.0.1-linux-x64.tar.xz && tar xf … -C $WORKSPACE/tools` — **then `blender-5.0.1-linux-x64/5.0/python/bin/python3.11 -m pip install scipy`**: 15 of the 212 3DCodeBench reference scripts import scipy, and without it any model that writes the same idiom is scored as a failure (this cost the tuned 27B 8.1 points before we caught it) | `BLENDER` in `eval/run_bench.py`, `eval/blender_dialect_eval.py`, `scripts/*dpo*.sh` |
| OpenSCAD (nightly AppImage, extracted so it runs without FUSE) | `wget …/OpenSCAD-2026.08.19-x86_64.AppImage && ./OpenSCAD-*.AppImage --appimage-extract` → `squashfs-root/AppRun` | `OPENSCAD` in `eval/dialect_runners.py` |
| glslangValidator | conda-forge `glslang` (in `llmft`) | `GLSLANG` in `eval/dialect_runners.py` |
| Chromium (Playwright) | see `llmft` above | `PLAYWRIGHT_BROWSERS_PATH` |
diff --git a/finetune/scripts/build_md_max.py b/finetune/scripts/build_md_max.py
index 78937983..6e207ec1 100644
--- a/finetune/scripts/build_md_max.py
+++ b/finetune/scripts/build_md_max.py
@@ -219,6 +219,9 @@ def main():
d["n"] += 1
d["tok"] += n
d["by_caption"][p["caption"]] = d["by_caption"].get(p["caption"], 0) + 1
+ def _san(t): return t.replace("<image>", "<image >").replace("<video>", "<video >").replace("<audio>", "<audio >")
+ for p_ in keep:
+ p_["code"] = _san(p_["code"]); p_["prompt"] = _san(p_["prompt"])
random.shuffle(keep)
os.makedirs(a.out, exist_ok=True)
diff --git a/finetune/scripts/reexec_scipy_affected.sh b/finetune/scripts/reexec_scipy_affected.sh
new file mode 100755
index 00000000..6e927054
--- /dev/null
+++ b/finetune/scripts/reexec_scipy_affected.sh
@@ -0,0 +1,15 @@
+#!/bin/bash
+# Re-execute + re-score every 3DCodeBench run that had failures caused by the missing scipy in Blender's Python.
+set -uo pipefail; cd /wekafs/ict/hx_624/llm-ft
+source /wekafs/ict/hx_624/anaconda3/etc/profile.d/conda.sh; conda activate llmft
+for RUN in qwen35_9b_lora_v7_ep4 md_big_T07_s2 qwen35_9b_full_md_bal qwen35_9b_lora_md_bal dpo_geo_md_xl_T07_s3 \
+ md_xl_T07_s2 md_xl_T07_s3 qwen35_9b_full_v2_3gpu qwen35_9b_full_v3_4gpu qwen35_9b_lora_md_glsl \
+ qwen35_9b_lora_md_xl_ck3150 qwen35_9b_lora_v6_detail; do
+ D=eval/out/$RUN; [ -d $D ] || continue
+ OLD=$(python3 -c "import json;print(json.load(open('$D/summary.json'))['exec_ok'])" 2>/dev/null || echo "?")
+ python eval/run_bench.py --gen_dir $D --workers 20 --timeout 300 > /dev/null 2>&1
+ python eval/metrics.py --gen_dir $D > /dev/null 2>&1
+ NEW=$(python3 -c "import json;s=json.load(open('$D/summary.json'));print(f\"{s['exec_ok']}/{s['n_tasks']} = {100*s['exec_ok_rate']:.1f}% [email protected](all) {s['[email protected]_mean_all(fail=0)']:.3f}\")" 2>/dev/null)
+ echo "[rescore] $RUN: was exec_ok=$OLD -> now $NEW"
+done
+echo "[rescore] DONE"
diff --git a/finetune/scripts/run_27b_v2b.sh b/finetune/scripts/run_27b_v2b.sh
new file mode 100755
index 00000000..ad5ff38f
--- /dev/null
+++ b/finetune/scripts/run_27b_v2b.sh
@@ -0,0 +1,22 @@
+#!/bin/bash
+# After the 9B md_max run: 27B v2 — Blender-heavy mix (adds the 15.7k execution-verified bootstrapped samples,
+# Blender x2) to fix the 3DCodeBench regression; then merge + eval (greedy everywhere + T=0.7 for openscad/glsl).
+set -uo pipefail; cd /wekafs/ict/hx_624/llm-ft
+# require the GPUs to stay idle for 3 consecutive checks (an eval between suites can look idle for a moment)
+IDLE=0
+while [ $IDLE -lt 3 ]; do
+ if [ "$(nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits -i 0,1,2,3 | sort -n | tail -1)" -lt 8000 ]; then IDLE=$((IDLE+1)); else IDLE=0; fi
+ sleep 30
+done
+echo "[27bv2] $(date) train (104k pairs / 77M tok, blender-heavy, DDP@4096, 4 GPUs)"
+GPUS=0,1,2,3 scripts/lf_train.sh configs/lf/lora_27b_v2.yaml > logs/train_27b_v2.log 2>&1
+RUN=runs/lf_qwen38_27b_lora_v2
+[ -f $RUN/train_results.json ] || { echo "[27bv2] TRAIN FAILED: $(grep -E 'OutOfMemory|Error' logs/train_27b_v2.log | grep -v errors | tail -1 | cut -c1-200)"; exit 1; }
+echo "[27bv2] $(date) train done: $(tr -d '\n' < $RUN/train_results.json | cut -c1-200)"
+sed -e "s|model_name_or_path:.*|model_name_or_path: /wekafs/ict/hx_624/models/Qwen3.8-27B|" -e "s|adapter_name_or_path:.*|adapter_name_or_path: $(readlink -f $RUN)|" -e "s|export_dir:.*|export_dir: $(readlink -f $RUN)/merged|" configs/lf/export_lora_template.yaml > configs/lf/export_27b_v2.yaml
+scripts/lf_export.sh configs/lf/export_27b_v2.yaml > logs/export_27b_v2.log 2>&1 || { echo "[27bv2] EXPORT FAILED"; exit 1; }
+echo "[27bv2] $(date) eval greedy (TP=2)"
+GPUS=0,1 TP=2 eval/eval_all_run2.sh $RUN/merged q27b_v2 --max_new 8192 2>&1 | grep -E "gen-multi|extract\]|status counts|dialect|DONE" | tail -30
+echo "[27bv2] $(date) eval sampled (T=0.7) for the boilerplate-heavy dialects"
+GPUS=0,1 TP=2 eval/eval_all_run2.sh $RUN/merged q27b_v2_T07 --temp 0.7 --seed 1 --max_new 8192 --suites openscad glsl 2>&1 | grep -E "gen-multi|dialect|DONE" | tail -8
+echo "[27bv2] ALL DONE"
diff --git a/finetune/scripts/run_md_v3.sh b/finetune/scripts/run_md_v3.sh
new file mode 100755
index 00000000..f6147ccf
--- /dev/null
+++ b/finetune/scripts/run_md_v3.sh
@@ -0,0 +1,16 @@
+#!/bin/bash
+# Control for the md_max regression: same sources & token budget, but 1 caption per sample + all bootstrapped Blender.
+set -uo pipefail; cd /wekafs/ict/hx_624/llm-ft
+until grep -qE "\[27bv2\] (ALL DONE|TRAIN FAILED|EXPORT FAILED)" logs/run_27b_v2.log 2>/dev/null; do sleep 120; done
+IDLE=0; while [ $IDLE -lt 3 ]; do
+ if [ "$(nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits -i 0,1,2,3 | sort -n | tail -1)" -lt 8000 ]; then IDLE=$((IDLE+1)); else IDLE=0; fi; sleep 30; done
+echo "[v3] $(date) train 9B on md_v3 (1 caption/sample + bootstrapped blender)"
+GPUS=0,1,2,3 scripts/lf_train.sh configs/lf/lora_md_v3.yaml > logs/train_md_v3.log 2>&1
+RUN=runs/lf_qwen35_9b_lora_md_v3
+[ -f $RUN/train_results.json ] || { echo "[v3] TRAIN FAILED"; exit 1; }
+echo "[v3] $(date) train done: $(tr -d '\n' < $RUN/train_results.json | cut -c1-160)"
+sed -e "s|model_name_or_path:.*|model_name_or_path: /wekafs/ict/hx_624/models/Qwen3.5-9B|" -e "s|adapter_name_or_path:.*|adapter_name_or_path: $(readlink -f $RUN)|" -e "s|export_dir:.*|export_dir: $(readlink -f $RUN)/merged|" configs/lf/export_lora_template.yaml > configs/lf/export_md_v3.yaml
+scripts/lf_export.sh configs/lf/export_md_v3.yaml > logs/export_md_v3.log 2>&1 || { echo "[v3] EXPORT FAILED"; exit 1; }
+GPUS=0 TP=1 eval/eval_all_run2.sh $RUN/merged q9b_v3 --max_new 8192 2>&1 | grep -E "gen-multi|status counts|dialect|DONE" | tail -20
+GPUS=1 TP=1 eval/eval_all_run2.sh $RUN/merged q9b_v3_T07 --temp 0.7 --seed 1 --max_new 8192 --suites openscad glsl 2>&1 | grep -E "dialect|DONE" | tail -4
+echo "[v3] ALL DONE"