Skip to content

Official Codebase of 3DCodeVerse

Python

queue: score mmmix2, test single-view training, and probe whether decoding alone fixes the image model

gaoypengdb0543cd6b42
diff --git a/finetune/configs/lf/lora_9b_mm1_img_text.yaml b/finetune/configs/lf/lora_9b_mm1_img_text.yaml
new file mode 100644
index 00000000..0b441c65
--- /dev/null
+++ b/finetune/configs/lf/lora_9b_mm1_img_text.yaml
@@ -0,0 +1,44 @@
+model_name_or_path: /wekafs/ict/hx_624/models/Qwen3.5-9B
+trust_remote_code: true
+flash_attn: fa2
+stage: sft
+do_train: true
+finetuning_type: lora
+lora_rank: 64
+lora_alpha: 128
+lora_dropout: 0.05
+lora_target: all
+freeze_vision_tower: true
+freeze_multi_modal_projector: true
+dataset_dir: /wekafs/ict/hx_624/llm-ft/data/mm1
+dataset: 3dcv_mm_img_text_train
+eval_dataset: 3dcv_mm_img_text_val
+template: qwen3_5
+enable_thinking: false
+cutoff_len: 8192
+preprocessing_num_workers: 16
+dataloader_num_workers: 4
+neat_packing: false
+packing: false
+tokenized_path: /wekafs/ict/hx_624/llm-ft/data/lf/tok_mm1_9b_img_text
+output_dir: /wekafs/ict/hx_624/llm-ft/runs/lf_qwen35_9b_mm1_img_text
+logging_steps: 5
+save_steps: 300
+save_only_model: true
+plot_loss: true
+overwrite_output_dir: true
+report_to: tensorboard
+per_device_train_batch_size: 1
+gradient_accumulation_steps: 8
+learning_rate: 0.0001
+num_train_epochs: 2.0
+lr_scheduler_type: cosine
+warmup_ratio: 0.05
+bf16: true
+gradient_checkpointing: true
+ddp_timeout: 180000000
+resume_from_checkpoint: null
+per_device_eval_batch_size: 1
+eval_strategy: steps
+eval_steps: 300
+save_total_limit: 3
diff --git a/finetune/scripts/queue.txt b/finetune/scripts/queue.txt
index ab9d6935..1921ca59 100644
--- a/finetune/scripts/queue.txt
+++ b/finetune/scripts/queue.txt
@@ -6,3 +6,5 @@ mmvis|scripts/run_9b_mmvis.sh
img_budget2|scripts/eval_img_budget.sh
27b_mm|bash scripts/run_27b_mm.sh
img_decoding|bash scripts/run_img_decoding.sh
+mm1_singleview|bash scripts/run_9b_mm1.sh
+mmmix2_eval|bash scripts/run_mmmix2_eval.sh
diff --git a/finetune/scripts/run_9b_mm1.sh b/finetune/scripts/run_9b_mm1.sh
new file mode 100755
index 00000000..5bf6e1df
--- /dev/null
+++ b/finetune/scripts/run_9b_mm1.sh
@@ -0,0 +1,32 @@
+#!/bin/bash
+# 12.8 found one reference view beats four at INFERENCE (64.6% vs 47.6%) on weights trained with four views.
+# Does it also hold when the model is TRAINED on one view? Same samples, same split, same everything but the
+# number of renders per sample -- so the comparison against mm_img_text (48.6%) is clean.
+set -uo pipefail; cd /wekafs/ict/hx_624/llm-ft
+V=mm1_img_text; RUN=runs/lf_qwen35_9b_$V
+G=""; for _ in $(seq 1 60); do G=${GPUS:-$(scripts/free_gpus.sh)}; [ -n "$G" ] && break; sleep 60; done
+[ -z "$G" ] && { echo "[$V] skipped: no GPU with room after 60 min"; exit 0; }
+echo "[$V] $(date) train on GPUs $G"
+GPUS=$G scripts/lf_train.sh configs/lf/lora_9b_$V.yaml > logs/train_9b_$V.log 2>&1
+[ -f $RUN/train_results.json ] || { echo "[$V] TRAIN FAILED: $(grep -iE 'out of memory|Error|Traceback' logs/train_9b_$V.log | grep -v errors | tail -1 | cut -c1-190)"; exit 0; }
+echo "[$V] train done: $(tr -d '\n' < $RUN/train_results.json | cut -c1-130)"
+sed -e "s|model_name_or_path:.*|model_name_or_path: /wekafs/ict/hx_624/models/Qwen3.5-9B|" -e "s|adapter_name_or_path:.*|adapter_name_or_path: $(readlink -f $RUN)|" -e "s|export_dir:.*|export_dir: $(readlink -f $RUN)/merged|" configs/lf/export_lora_template.yaml > configs/lf/export_9b_$V.yaml
+scripts/lf_export.sh configs/lf/export_9b_$V.yaml > logs/export_9b_$V.log 2>&1 || { echo "[$V] EXPORT FAILED"; exit 0; }
+G1=""; for _ in $(seq 1 40); do G1=$(scripts/free_gpus.sh 40000 | cut -d, -f1); [ -n "$G1" ] && break; sleep 30; done
+[ -z "$G1" ] && { echo "[$V] eval skipped: no GPU with 40 GB free"; exit 0; }
+source /wekafs/ict/hx_624/anaconda3/etc/profile.d/conda.sh; conda activate vllm
+export VLLM_CACHE_ROOT=/wekafs/ict/hx_624/cache/vllm HF_HOME=/wekafs/ict/hx_624/cache/huggingface VLLM_LOGGING_LEVEL=WARNING CUDA_VISIBLE_DEVICES=$G1
+OUT=eval/out/q9b_mm1_img_text_bench; mkdir -p $OUT
+# the SAME prompts and the SAME budget mm_img_text was scored with, so 12.7's cross-harness floor does not apply
+python eval/generate_vllm_img.py --model $RUN/merged --prompts eval/out/_img4text_bench_prompts.jsonl --out $OUT \
+ --max_new_tokens 32768 --max_model_len 40960 2>&1 | tail -1
+N=$(ls $OUT | grep -vc extract_stats.json || true)
+[ "${N:-0}" -lt 5 ] && { echo "[$V] GENERATION PRODUCED NOTHING — not scoring"; exit 0; }
+conda activate llmft
+python eval/run_bench.py --gen_dir $OUT --workers 24 --timeout 300 2>&1 | grep "status counts" | tail -1
+python - "$OUT" <<'PY'
+import json, sys
+r=[json.loads(l) for l in open(f"{sys.argv[1]}/gens_shard0.jsonl")]
+print(f"[mm1] unfinished={sum(1 for x in r if not x.get('finished',True))}/{len(r)} mean_tok={sum(x.get('n_new_tokens',0) for x in r)/len(r):.0f}")
+PY
+echo "[$V] DONE"
diff --git a/finetune/scripts/run_mmmix2_eval.sh b/finetune/scripts/run_mmmix2_eval.sh
new file mode 100755
index 00000000..d4ac4c9c
--- /dev/null
+++ b/finetune/scripts/run_mmmix2_eval.sh
@@ -0,0 +1,32 @@
+#!/bin/bash
+# mmmix2 mixes image+text, image-only and the full text corpus into one model. It trained but was never scored.
+# Two questions: does mixing image data cost the TEXT path anything (compare against md_9b_noleak, 84.9%), and
+# does one model handle both input types? Text suites and image paths are all run here so the numbers are
+# comparable within this one harness (12.7).
+set -uo pipefail; cd /wekafs/ict/hx_624/llm-ft
+RUN=runs/lf_qwen35_9b_mmmix2/merged
+[ -d "$RUN" ] || { echo "[mmmix2eval] no merged model"; exit 0; }
+G=""; for _ in $(seq 1 60); do G=${GPUS:-$(scripts/free_gpus.sh 40000 | cut -d, -f1)}; [ -n "$G" ] && break; sleep 30; done
+[ -z "$G" ] && { echo "[mmmix2eval] no GPU with 40 GB free after 30 min"; exit 0; }
+echo "[mmmix2eval] $(date) six text suites on GPU $G"
+GPUS=$G TP=1 bash eval/eval_all_run2.sh $RUN q9b_mmmix2_all 2>&1 | tail -20
+source /wekafs/ict/hx_624/anaconda3/etc/profile.d/conda.sh
+for SPEC in "_img4text_bench_prompts img_text" "_img4_bench_prompts img"; do
+ set -- $SPEC
+ OUT=eval/out/q9b_mmmix2_$2; mkdir -p $OUT
+ echo "[mmmix2eval] image path: $2"
+ conda activate vllm
+ export VLLM_CACHE_ROOT=/wekafs/ict/hx_624/cache/vllm HF_HOME=/wekafs/ict/hx_624/cache/huggingface VLLM_LOGGING_LEVEL=WARNING CUDA_VISIBLE_DEVICES=$G
+ python eval/generate_vllm_img.py --model $RUN --prompts eval/out/$1.jsonl --out $OUT \
+ --max_new_tokens 32768 --max_model_len 40960 2>&1 | tail -1
+ N=$(ls $OUT | grep -vc extract_stats.json || true)
+ [ "${N:-0}" -lt 5 ] && { echo "[mmmix2eval] $2 PRODUCED NOTHING — not scoring"; continue; }
+ conda activate llmft
+ python eval/run_bench.py --gen_dir $OUT --workers 24 --timeout 300 2>&1 | grep "status counts" | tail -1
+ python - "$OUT" <<'PY'
+import json, sys
+r=[json.loads(l) for l in open(f"{sys.argv[1]}/gens_shard0.jsonl")]
+print(f"[mmmix2eval] unfinished={sum(1 for x in r if not x.get('finished',True))}/{len(r)} mean_tok={sum(x.get('n_new_tokens',0) for x in r)/len(r):.0f}")
+PY
+done
+echo "[mmmix2eval] ALL DONE"