Official Codebase of 3DCodeVerse
Python
queue: score mmmix2, test single-view training, and probe whether decoding alone fixes the image model
gaoypengdb0543cd6b42
diff --git a/finetune/configs/lf/lora_9b_mm1_img_text.yaml b/finetune/configs/lf/lora_9b_mm1_img_text.yamlnew file mode 100644index 00000000..0b441c65--- /dev/null+++ b/finetune/configs/lf/lora_9b_mm1_img_text.yaml@@ -0,0 +1,44 @@+model_name_or_path: /wekafs/ict/hx_624/models/Qwen3.5-9B+trust_remote_code: true+flash_attn: fa2+stage: sft+do_train: true+finetuning_type: lora+lora_rank: 64+lora_alpha: 128+lora_dropout: 0.05+lora_target: all+freeze_vision_tower: true+freeze_multi_modal_projector: true+dataset_dir: /wekafs/ict/hx_624/llm-ft/data/mm1+dataset: 3dcv_mm_img_text_train+eval_dataset: 3dcv_mm_img_text_val+template: qwen3_5+enable_thinking: false+cutoff_len: 8192+preprocessing_num_workers: 16+dataloader_num_workers: 4+neat_packing: false+packing: false+tokenized_path: /wekafs/ict/hx_624/llm-ft/data/lf/tok_mm1_9b_img_text+output_dir: /wekafs/ict/hx_624/llm-ft/runs/lf_qwen35_9b_mm1_img_text+logging_steps: 5+save_steps: 300+save_only_model: true+plot_loss: true+overwrite_output_dir: true+report_to: tensorboard+per_device_train_batch_size: 1+gradient_accumulation_steps: 8+learning_rate: 0.0001+num_train_epochs: 2.0+lr_scheduler_type: cosine+warmup_ratio: 0.05+bf16: true+gradient_checkpointing: true+ddp_timeout: 180000000+resume_from_checkpoint: null+per_device_eval_batch_size: 1+eval_strategy: steps+eval_steps: 300+save_total_limit: 3diff --git a/finetune/scripts/queue.txt b/finetune/scripts/queue.txtindex ab9d6935..1921ca59 100644--- a/finetune/scripts/queue.txt+++ b/finetune/scripts/queue.txt@@ -6,3 +6,5 @@ mmvis|scripts/run_9b_mmvis.shimg_budget2|scripts/eval_img_budget.sh27b_mm|bash scripts/run_27b_mm.shimg_decoding|bash scripts/run_img_decoding.sh+mm1_singleview|bash scripts/run_9b_mm1.sh+mmmix2_eval|bash scripts/run_mmmix2_eval.shdiff --git a/finetune/scripts/run_9b_mm1.sh b/finetune/scripts/run_9b_mm1.shnew file mode 100755index 00000000..5bf6e1df--- /dev/null+++ b/finetune/scripts/run_9b_mm1.sh@@ -0,0 +1,32 @@+#!/bin/bash+# 12.8 found one reference view beats four at INFERENCE (64.6% vs 47.6%) on weights trained with four views.+# Does it also hold when the model is TRAINED on one view? Same samples, same split, same everything but the+# number of renders per sample -- so the comparison against mm_img_text (48.6%) is clean.+set -uo pipefail; cd /wekafs/ict/hx_624/llm-ft+V=mm1_img_text; RUN=runs/lf_qwen35_9b_$V+G=""; for _ in $(seq 1 60); do G=${GPUS:-$(scripts/free_gpus.sh)}; [ -n "$G" ] && break; sleep 60; done+[ -z "$G" ] && { echo "[$V] skipped: no GPU with room after 60 min"; exit 0; }+echo "[$V] $(date) train on GPUs $G"+GPUS=$G scripts/lf_train.sh configs/lf/lora_9b_$V.yaml > logs/train_9b_$V.log 2>&1+[ -f $RUN/train_results.json ] || { echo "[$V] TRAIN FAILED: $(grep -iE 'out of memory|Error|Traceback' logs/train_9b_$V.log | grep -v errors | tail -1 | cut -c1-190)"; exit 0; }+echo "[$V] train done: $(tr -d '\n' < $RUN/train_results.json | cut -c1-130)"+sed -e "s|model_name_or_path:.*|model_name_or_path: /wekafs/ict/hx_624/models/Qwen3.5-9B|" -e "s|adapter_name_or_path:.*|adapter_name_or_path: $(readlink -f $RUN)|" -e "s|export_dir:.*|export_dir: $(readlink -f $RUN)/merged|" configs/lf/export_lora_template.yaml > configs/lf/export_9b_$V.yaml+scripts/lf_export.sh configs/lf/export_9b_$V.yaml > logs/export_9b_$V.log 2>&1 || { echo "[$V] EXPORT FAILED"; exit 0; }+G1=""; for _ in $(seq 1 40); do G1=$(scripts/free_gpus.sh 40000 | cut -d, -f1); [ -n "$G1" ] && break; sleep 30; done+[ -z "$G1" ] && { echo "[$V] eval skipped: no GPU with 40 GB free"; exit 0; }+source /wekafs/ict/hx_624/anaconda3/etc/profile.d/conda.sh; conda activate vllm+export VLLM_CACHE_ROOT=/wekafs/ict/hx_624/cache/vllm HF_HOME=/wekafs/ict/hx_624/cache/huggingface VLLM_LOGGING_LEVEL=WARNING CUDA_VISIBLE_DEVICES=$G1+OUT=eval/out/q9b_mm1_img_text_bench; mkdir -p $OUT+# the SAME prompts and the SAME budget mm_img_text was scored with, so 12.7's cross-harness floor does not apply+python eval/generate_vllm_img.py --model $RUN/merged --prompts eval/out/_img4text_bench_prompts.jsonl --out $OUT \+ --max_new_tokens 32768 --max_model_len 40960 2>&1 | tail -1+N=$(ls $OUT | grep -vc extract_stats.json || true)+[ "${N:-0}" -lt 5 ] && { echo "[$V] GENERATION PRODUCED NOTHING — not scoring"; exit 0; }+conda activate llmft+python eval/run_bench.py --gen_dir $OUT --workers 24 --timeout 300 2>&1 | grep "status counts" | tail -1+python - "$OUT" <<'PY'+import json, sys+r=[json.loads(l) for l in open(f"{sys.argv[1]}/gens_shard0.jsonl")]+print(f"[mm1] unfinished={sum(1 for x in r if not x.get('finished',True))}/{len(r)} mean_tok={sum(x.get('n_new_tokens',0) for x in r)/len(r):.0f}")+PY+echo "[$V] DONE"diff --git a/finetune/scripts/run_mmmix2_eval.sh b/finetune/scripts/run_mmmix2_eval.shnew file mode 100755index 00000000..d4ac4c9c--- /dev/null+++ b/finetune/scripts/run_mmmix2_eval.sh@@ -0,0 +1,32 @@+#!/bin/bash+# mmmix2 mixes image+text, image-only and the full text corpus into one model. It trained but was never scored.+# Two questions: does mixing image data cost the TEXT path anything (compare against md_9b_noleak, 84.9%), and+# does one model handle both input types? Text suites and image paths are all run here so the numbers are+# comparable within this one harness (12.7).+set -uo pipefail; cd /wekafs/ict/hx_624/llm-ft+RUN=runs/lf_qwen35_9b_mmmix2/merged+[ -d "$RUN" ] || { echo "[mmmix2eval] no merged model"; exit 0; }+G=""; for _ in $(seq 1 60); do G=${GPUS:-$(scripts/free_gpus.sh 40000 | cut -d, -f1)}; [ -n "$G" ] && break; sleep 30; done+[ -z "$G" ] && { echo "[mmmix2eval] no GPU with 40 GB free after 30 min"; exit 0; }+echo "[mmmix2eval] $(date) six text suites on GPU $G"+GPUS=$G TP=1 bash eval/eval_all_run2.sh $RUN q9b_mmmix2_all 2>&1 | tail -20+source /wekafs/ict/hx_624/anaconda3/etc/profile.d/conda.sh+for SPEC in "_img4text_bench_prompts img_text" "_img4_bench_prompts img"; do+ set -- $SPEC+ OUT=eval/out/q9b_mmmix2_$2; mkdir -p $OUT+ echo "[mmmix2eval] image path: $2"+ conda activate vllm+ export VLLM_CACHE_ROOT=/wekafs/ict/hx_624/cache/vllm HF_HOME=/wekafs/ict/hx_624/cache/huggingface VLLM_LOGGING_LEVEL=WARNING CUDA_VISIBLE_DEVICES=$G+ python eval/generate_vllm_img.py --model $RUN --prompts eval/out/$1.jsonl --out $OUT \+ --max_new_tokens 32768 --max_model_len 40960 2>&1 | tail -1+ N=$(ls $OUT | grep -vc extract_stats.json || true)+ [ "${N:-0}" -lt 5 ] && { echo "[mmmix2eval] $2 PRODUCED NOTHING — not scoring"; continue; }+ conda activate llmft+ python eval/run_bench.py --gen_dir $OUT --workers 24 --timeout 300 2>&1 | grep "status counts" | tail -1+ python - "$OUT" <<'PY'+import json, sys+r=[json.loads(l) for l in open(f"{sys.argv[1]}/gens_shard0.jsonl")]+print(f"[mmmix2eval] unfinished={sum(1 for x in r if not x.get('finished',True))}/{len(r)} mean_tok={sum(x.get('n_new_tokens',0) for x in r)/len(r):.0f}")+PY+done+echo "[mmmix2eval] ALL DONE"