Skip to content

Official Codebase of 3DCodeVerse

Python

27b: multimodal LoRA fits one empty H100 and nothing less; queue a per-suite noise profile

gaoypengd4dca2cfcb77
diff --git a/finetune/configs/lf/lora_27b_mm_img_text.yaml b/finetune/configs/lf/lora_27b_mm_img_text.yaml
index 4e51f1db..be3413cb 100644
--- a/finetune/configs/lf/lora_27b_mm_img_text.yaml
+++ b/finetune/configs/lf/lora_27b_mm_img_text.yaml
@@ -21,7 +21,7 @@ preprocessing_num_workers: 16
dataloader_num_workers: 4
neat_packing: false
packing: false
-tokenized_path: /wekafs/ict/hx_624/llm-ft/data/lf/tok_mm1_27b_img_text_len4096_px448
+tokenized_path: /wekafs/ict/hx_624/llm-ft/data/lf/tok_mm1_27b_len4096_px448b
output_dir: /wekafs/ict/hx_624/llm-ft/runs/lf_qwen38_27b_mm_img_text
logging_steps: 5
save_steps: 300
diff --git a/finetune/scripts/queue.txt b/finetune/scripts/queue.txt
index 1921ca59..3e1620da 100644
--- a/finetune/scripts/queue.txt
+++ b/finetune/scripts/queue.txt
@@ -8,3 +8,5 @@ img_budget2|scripts/eval_img_budget.sh
img_decoding|bash scripts/run_img_decoding.sh
mm1_singleview|bash scripts/run_9b_mm1.sh
mmmix2_eval|bash scripts/run_mmmix2_eval.sh
+noise_profile|bash scripts/run_noise_profile.sh
+27b_mm_eval|bash scripts/run_27b_mm_eval.sh
diff --git a/finetune/scripts/run_27b_mm_eval.sh b/finetune/scripts/run_27b_mm_eval.sh
new file mode 100755
index 00000000..439fd285
--- /dev/null
+++ b/finetune/scripts/run_27b_mm_eval.sh
@@ -0,0 +1,22 @@
+#!/bin/bash
+# Score the 27B multimodal LoRA once its 4h16m single-GPU run finishes. It needs a card with room for a 27B
+# (~55 GB of weights), which on this box means a nearly empty one.
+set -uo pipefail; cd /wekafs/ict/hx_624/llm-ft
+RUN=runs/lf_qwen38_27b_mm_img_text
+[ -f $RUN/train_results.json ] || { echo "[27bmmeval] training has not finished — nothing to score"; exit 0; }
+if [ ! -d $RUN/merged ]; then
+ sed -e "s|model_name_or_path:.*|model_name_or_path: /wekafs/ict/hx_624/models/Qwen3.8-27B|" -e "s|adapter_name_or_path:.*|adapter_name_or_path: $(readlink -f $RUN)|" -e "s|export_dir:.*|export_dir: $(readlink -f $RUN)/merged|" configs/lf/export_lora_template.yaml > configs/lf/export_27b_mm.yaml
+ scripts/lf_export.sh configs/lf/export_27b_mm.yaml > logs/export_27b_mm.log 2>&1 || { echo "[27bmmeval] EXPORT FAILED"; exit 0; }
+fi
+G=""; for _ in $(seq 1 60); do G=${GPUS:-$(scripts/free_gpus.sh 70000 | cut -d, -f1)}; [ -n "$G" ] && break; sleep 60; done
+[ -z "$G" ] && { echo "[27bmmeval] skipped: a 27B needs ~70 GB free and no card had it"; exit 0; }
+source /wekafs/ict/hx_624/anaconda3/etc/profile.d/conda.sh; conda activate vllm
+export VLLM_CACHE_ROOT=/wekafs/ict/hx_624/cache/vllm HF_HOME=/wekafs/ict/hx_624/cache/huggingface VLLM_LOGGING_LEVEL=WARNING CUDA_VISIBLE_DEVICES=$G
+OUT=eval/out/q27b_mm_img_text_bench; mkdir -p $OUT
+python eval/generate_vllm_img.py --model $RUN/merged --prompts eval/out/_img4text_bench_prompts.jsonl --out $OUT \
+ --max_new_tokens 32768 --max_model_len 40960 2>&1 | tail -1
+N=$(ls $OUT | grep -vc extract_stats.json || true)
+[ "${N:-0}" -lt 5 ] && { echo "[27bmmeval] GENERATION PRODUCED NOTHING — not scoring"; exit 0; }
+conda activate llmft
+python eval/run_bench.py --gen_dir $OUT --workers 24 --timeout 300 2>&1 | grep "status counts" | tail -1
+echo "[27bmmeval] DONE"
diff --git a/finetune/scripts/run_noise_profile.sh b/finetune/scripts/run_noise_profile.sh
new file mode 100755
index 00000000..bfbe27ef
--- /dev/null
+++ b/finetune/scripts/run_noise_profile.sh
@@ -0,0 +1,32 @@
+#!/bin/bash
+# OpenSCAD moved 40.0% -> 30.0% between two runs of the SAME model through the SAME harness, ten times the
+# +-2pp floor I estimated from 3DCodeBench. So the floor is not one number: it differs per suite, and long-output
+# dialects swing hardest. Run one model through all six suites three times and report the per-suite spread, so
+# every future comparison can be read against the right threshold instead of a guessed one.
+set -uo pipefail; cd /wekafs/ict/hx_624/llm-ft
+M=runs/lf_qwen35_9b_mm_text/merged
+[ -d "$M" ] || { echo "[noise] no model"; exit 0; }
+for R in 1 2 3; do
+ G=""; for _ in $(seq 1 60); do G=${GPUS:-$(scripts/free_gpus.sh 40000 | cut -d, -f1)}; [ -n "$G" ] && break; sleep 30; done
+ [ -z "$G" ] && { echo "[noise] run $R skipped: no GPU with 40 GB free"; continue; }
+ echo "[noise] $(date) repeat $R of 3 on GPU $G"
+ GPUS=$G TP=1 bash eval/eval_all_run2.sh $M q9b_noise_r$R 2>&1 | tail -3
+done
+python - <<'PY'
+import json, os, statistics as st
+suites = ["bench", "blender", "cadquery", "openscad", "glsl", "threejs"]
+runs = []
+for r in (1, 2, 3):
+ p = f"eval/out/q9b_noise_r{r}_report.json"
+ if os.path.exists(p):
+ runs.append({s["suite"]: s["exec_rate"] for s in json.load(open(p))["suites"]})
+if len(runs) < 2:
+ print("[noise] fewer than two runs completed — cannot estimate a spread"); raise SystemExit
+print(f"[noise] {len(runs)} repeats of an IDENTICAL model+harness")
+print(f" {'suite':10s} " + " ".join(f"run{i+1:>6}" for i in range(len(runs))) + f" {'spread':>9s}")
+for s in suites:
+ v = [r[s] * 100 for r in runs if s in r]
+ if len(v) < 2: continue
+ print(f" {s:10s} " + " ".join(f"{x:9.1f}" for x in v) + f" {max(v)-min(v):8.1f}pp")
+PY
+echo "[noise] DONE"