Official Codebase of 3DCodeVerse
Python
27b: multimodal LoRA fits one empty H100 and nothing less; queue a per-suite noise profile
gaoypengd4dca2cfcb77
diff --git a/finetune/configs/lf/lora_27b_mm_img_text.yaml b/finetune/configs/lf/lora_27b_mm_img_text.yamlindex 4e51f1db..be3413cb 100644--- a/finetune/configs/lf/lora_27b_mm_img_text.yaml+++ b/finetune/configs/lf/lora_27b_mm_img_text.yaml@@ -21,7 +21,7 @@ preprocessing_num_workers: 16dataloader_num_workers: 4neat_packing: falsepacking: false-tokenized_path: /wekafs/ict/hx_624/llm-ft/data/lf/tok_mm1_27b_img_text_len4096_px448+tokenized_path: /wekafs/ict/hx_624/llm-ft/data/lf/tok_mm1_27b_len4096_px448boutput_dir: /wekafs/ict/hx_624/llm-ft/runs/lf_qwen38_27b_mm_img_textlogging_steps: 5save_steps: 300diff --git a/finetune/scripts/queue.txt b/finetune/scripts/queue.txtindex 1921ca59..3e1620da 100644--- a/finetune/scripts/queue.txt+++ b/finetune/scripts/queue.txt@@ -8,3 +8,5 @@ img_budget2|scripts/eval_img_budget.shimg_decoding|bash scripts/run_img_decoding.shmm1_singleview|bash scripts/run_9b_mm1.shmmmix2_eval|bash scripts/run_mmmix2_eval.sh+noise_profile|bash scripts/run_noise_profile.sh+27b_mm_eval|bash scripts/run_27b_mm_eval.shdiff --git a/finetune/scripts/run_27b_mm_eval.sh b/finetune/scripts/run_27b_mm_eval.shnew file mode 100755index 00000000..439fd285--- /dev/null+++ b/finetune/scripts/run_27b_mm_eval.sh@@ -0,0 +1,22 @@+#!/bin/bash+# Score the 27B multimodal LoRA once its 4h16m single-GPU run finishes. It needs a card with room for a 27B+# (~55 GB of weights), which on this box means a nearly empty one.+set -uo pipefail; cd /wekafs/ict/hx_624/llm-ft+RUN=runs/lf_qwen38_27b_mm_img_text+[ -f $RUN/train_results.json ] || { echo "[27bmmeval] training has not finished — nothing to score"; exit 0; }+if [ ! -d $RUN/merged ]; then+ sed -e "s|model_name_or_path:.*|model_name_or_path: /wekafs/ict/hx_624/models/Qwen3.8-27B|" -e "s|adapter_name_or_path:.*|adapter_name_or_path: $(readlink -f $RUN)|" -e "s|export_dir:.*|export_dir: $(readlink -f $RUN)/merged|" configs/lf/export_lora_template.yaml > configs/lf/export_27b_mm.yaml+ scripts/lf_export.sh configs/lf/export_27b_mm.yaml > logs/export_27b_mm.log 2>&1 || { echo "[27bmmeval] EXPORT FAILED"; exit 0; }+fi+G=""; for _ in $(seq 1 60); do G=${GPUS:-$(scripts/free_gpus.sh 70000 | cut -d, -f1)}; [ -n "$G" ] && break; sleep 60; done+[ -z "$G" ] && { echo "[27bmmeval] skipped: a 27B needs ~70 GB free and no card had it"; exit 0; }+source /wekafs/ict/hx_624/anaconda3/etc/profile.d/conda.sh; conda activate vllm+export VLLM_CACHE_ROOT=/wekafs/ict/hx_624/cache/vllm HF_HOME=/wekafs/ict/hx_624/cache/huggingface VLLM_LOGGING_LEVEL=WARNING CUDA_VISIBLE_DEVICES=$G+OUT=eval/out/q27b_mm_img_text_bench; mkdir -p $OUT+python eval/generate_vllm_img.py --model $RUN/merged --prompts eval/out/_img4text_bench_prompts.jsonl --out $OUT \+ --max_new_tokens 32768 --max_model_len 40960 2>&1 | tail -1+N=$(ls $OUT | grep -vc extract_stats.json || true)+[ "${N:-0}" -lt 5 ] && { echo "[27bmmeval] GENERATION PRODUCED NOTHING — not scoring"; exit 0; }+conda activate llmft+python eval/run_bench.py --gen_dir $OUT --workers 24 --timeout 300 2>&1 | grep "status counts" | tail -1+echo "[27bmmeval] DONE"diff --git a/finetune/scripts/run_noise_profile.sh b/finetune/scripts/run_noise_profile.shnew file mode 100755index 00000000..bfbe27ef--- /dev/null+++ b/finetune/scripts/run_noise_profile.sh@@ -0,0 +1,32 @@+#!/bin/bash+# OpenSCAD moved 40.0% -> 30.0% between two runs of the SAME model through the SAME harness, ten times the+# +-2pp floor I estimated from 3DCodeBench. So the floor is not one number: it differs per suite, and long-output+# dialects swing hardest. Run one model through all six suites three times and report the per-suite spread, so+# every future comparison can be read against the right threshold instead of a guessed one.+set -uo pipefail; cd /wekafs/ict/hx_624/llm-ft+M=runs/lf_qwen35_9b_mm_text/merged+[ -d "$M" ] || { echo "[noise] no model"; exit 0; }+for R in 1 2 3; do+ G=""; for _ in $(seq 1 60); do G=${GPUS:-$(scripts/free_gpus.sh 40000 | cut -d, -f1)}; [ -n "$G" ] && break; sleep 30; done+ [ -z "$G" ] && { echo "[noise] run $R skipped: no GPU with 40 GB free"; continue; }+ echo "[noise] $(date) repeat $R of 3 on GPU $G"+ GPUS=$G TP=1 bash eval/eval_all_run2.sh $M q9b_noise_r$R 2>&1 | tail -3+done+python - <<'PY'+import json, os, statistics as st+suites = ["bench", "blender", "cadquery", "openscad", "glsl", "threejs"]+runs = []+for r in (1, 2, 3):+ p = f"eval/out/q9b_noise_r{r}_report.json"+ if os.path.exists(p):+ runs.append({s["suite"]: s["exec_rate"] for s in json.load(open(p))["suites"]})+if len(runs) < 2:+ print("[noise] fewer than two runs completed — cannot estimate a spread"); raise SystemExit+print(f"[noise] {len(runs)} repeats of an IDENTICAL model+harness")+print(f" {'suite':10s} " + " ".join(f"run{i+1:>6}" for i in range(len(runs))) + f" {'spread':>9s}")+for s in suites:+ v = [r[s] * 100 for r in runs if s in r]+ if len(v) < 2: continue+ print(f" {s:10s} " + " ".join(f"{x:9.1f}" for x in v) + f" {max(v)-min(v):8.1f}pp")+PY+echo "[noise] DONE"