Official Codebase of 3DCodeVerse
Python
eval: more output budget did not buy better code, and one reference view beats four
gaoypeng091472ad692a
diff --git a/finetune/configs/lf/export_9b_mmmix2.yaml b/finetune/configs/lf/export_9b_mmmix2.yamlnew file mode 100644index 00000000..6a622e6c--- /dev/null+++ b/finetune/configs/lf/export_9b_mmmix2.yaml@@ -0,0 +1,12 @@+### Merge a LoRA adapter into the base model (LLaMA-Factory): llamafactory-cli export <this file>+model_name_or_path: /wekafs/ict/hx_624/models/Qwen3.5-9B+adapter_name_or_path: /wekafs/ict/hx_624/llm-ft/runs/lf_qwen35_9b_mmmix2+template: qwen3_5+enable_thinking: false+trust_remote_code: true+finetuning_type: lora++export_dir: /wekafs/ict/hx_624/llm-ft/runs/lf_qwen35_9b_mmmix2/merged+export_size: 5+export_device: cpu+export_legacy_format: falsediff --git a/finetune/eval/generate_vllm_img.py b/finetune/eval/generate_vllm_img.pyindex 52dd39e7..a56141ea 100644--- a/finetune/eval/generate_vllm_img.py+++ b/finetune/eval/generate_vllm_img.py@@ -34,9 +34,11 @@ def auto_gpu_mem(requested, tp=1, headroom_gib=4.0):def main():ap = argparse.ArgumentParser()ap.add_argument("--model", required=True); ap.add_argument("--prompts", required=True); ap.add_argument("--out", required=True)- ap.add_argument("--max_new_tokens", type=int, default=6144); ap.add_argument("--max_model_len", type=int, default=12288)+ ap.add_argument("--max_new_tokens", type=int, default=32768); ap.add_argument("--max_model_len", type=int, default=12288)ap.add_argument("--temperature", type=float, default=0.0); ap.add_argument("--seed", type=int, default=0)ap.add_argument("--tp", type=int, default=1); ap.add_argument("--gpu_mem", type=float, default=0.88)+ ap.add_argument("--presence_penalty", type=float, default=0.0)+ ap.add_argument("--repetition_penalty", type=float, default=1.0)ap.add_argument("--dialect", default="blender"); ap.add_argument("--limit", type=int, default=None)a = ap.parse_args()from vllm import LLM, SamplingParams@@ -47,6 +49,7 @@ def main():llm = LLM(model=a.model, dtype="bfloat16", tensor_parallel_size=a.tp, max_model_len=a.max_model_len,gpu_memory_utilization=auto_gpu_mem(a.gpu_mem, getattr(a, 'tp', 1)), limit_mm_per_prompt={"image": 4, "video": 0}, trust_remote_code=True)sp = SamplingParams(temperature=a.temperature, top_p=0.95 if a.temperature > 0 else 1.0,+ presence_penalty=a.presence_penalty, repetition_penalty=a.repetition_penalty,max_tokens=a.max_new_tokens, seed=a.seed)import base64, iodef data_url(path):diff --git a/finetune/scripts/queue.txt b/finetune/scripts/queue.txtindex fc485ad6..ab9d6935 100644--- a/finetune/scripts/queue.txt+++ b/finetune/scripts/queue.txt@@ -5,3 +5,4 @@ mmmix2|scripts/run_9b_mmmix2.shmmvis|scripts/run_9b_mmvis.shimg_budget2|scripts/eval_img_budget.sh27b_mm|bash scripts/run_27b_mm.sh+img_decoding|bash scripts/run_img_decoding.shdiff --git a/finetune/scripts/run_img_decoding.sh b/finetune/scripts/run_img_decoding.shnew file mode 100755index 00000000..eff16e55--- /dev/null+++ b/finetune/scripts/run_img_decoding.sh@@ -0,0 +1,36 @@+#!/bin/bash+# mm_img scores 36.8% while writing 15,445 tokens per task and leaving 97 of 212 unfinished. Is that a capability+# gap, or the same non-termination pathology the three.js LoRA had (where T=0.7 recovered 30% -> 50%)?+# Four decoding settings on the SAME model answer it. If a decoding change alone moves the score, the model can+# build these objects and simply does not know when to stop.+set -uo pipefail; cd /wekafs/ict/hx_624/llm-ft+source /wekafs/ict/hx_624/anaconda3/etc/profile.d/conda.sh+RUN=runs/lf_qwen35_9b_mm_img/merged+[ -d "$RUN" ] || { echo "[imgdec] no merged model"; exit 0; }+G=""; for _ in $(seq 1 60); do G=${GPUS:-$(scripts/free_gpus.sh 40000 | cut -d, -f1)}; [ -n "$G" ] && break; sleep 30; done+[ -z "$G" ] && { echo "[imgdec] no GPU with 40 GB free"; exit 0; }+run () { # name, extra flags+ local NAME=$1; shift+ local OUT=eval/out/q9b_mm_img_dec_$NAME; mkdir -p $OUT+ echo "[imgdec] $NAME: $* "+ conda activate vllm+ export VLLM_CACHE_ROOT=/wekafs/ict/hx_624/cache/vllm HF_HOME=/wekafs/ict/hx_624/cache/huggingface VLLM_LOGGING_LEVEL=WARNING CUDA_VISIBLE_DEVICES=$G+ python eval/generate_vllm_img.py --model $RUN --prompts eval/out/_img4_bench_prompts.jsonl --out $OUT \+ --max_new_tokens 32768 --max_model_len 40960 "$@" 2>&1 | tail -1+ local N=$(ls $OUT | grep -vc extract_stats.json || true)+ [ "${N:-0}" -lt 5 ] && { echo "[imgdec] $NAME PRODUCED NOTHING — not scoring"; return; }+ conda activate llmft+ python eval/run_bench.py --gen_dir $OUT --workers 24 --timeout 300 2>&1 | grep "status counts" | tail -1+ python - "$OUT" <<'PY'+import json, sys+rows = [json.loads(l) for l in open(f"{sys.argv[1]}/gens_shard0.jsonl")]+tr = sum(1 for r in rows if not r.get("finished", True))+print(f"[imgdec] unfinished={tr}/{len(rows)} mean_tok={sum(r.get('n_new_tokens',0) for r in rows)/len(rows):.0f}")+PY+}+# every setting is compared against the SAME harness, so 12.7's cross-harness noise floor does not apply here+run greedy+run temp07 --temperature 0.7 --seed 1+run presence --presence_penalty 0.5+run reppen --repetition_penalty 1.05+echo "[imgdec] ALL DONE"diff --git a/finetune/scripts/supervisor2.sh b/finetune/scripts/supervisor2.shnew file mode 100644index 00000000..ce7ff1e4--- /dev/null+++ b/finetune/scripts/supervisor2.sh@@ -0,0 +1,48 @@+#!/bin/bash+# Keeps GPUs 0-3 busy: runs the pending jobs in scripts/queue.txt one at a time, restarts nothing that succeeded,+# and reports every state change to logs/supervisor.log. One line per job: "<label>|<command>".+set -uo pipefail; cd /wekafs/ict/hx_624/llm-ft+Q=scripts/queue.txt; STATE=logs/.supervisor_done+touch $STATE+while true; do+ PENDING=""+ while IFS='|' read -r label cmd; do+ [ -z "${label:-}" ] && continue+ case "$label" in \#*) continue;; esac+ grep -qx "$label" $STATE || { PENDING="$label|$cmd"; break; }+ done < $Q+ if [ -z "$PENDING" ]; then+ echo "[sup] $(date) queue empty — idling (add a line to $Q to continue)"+ IDLE_MIN=0+ while [ -z "$PENDING" ]; do+ sleep 300; IDLE_MIN=$((IDLE_MIN+5))+ [ $((IDLE_MIN % 30)) -eq 0 ] && echo "[sup] WARNING $(date) idle ${IDLE_MIN}m with GPUs [$(scripts/free_gpus.sh)] free"+ while IFS='|' read -r label cmd; do+ [ -z "${label:-}" ] && continue; case "$label" in \#*) continue;; esac+ grep -qx "$label" $STATE || { PENDING="$label|$cmd"; break; }+ done < $Q+ done+ fi+ label="${PENDING%%|*}"; cmd="${PENDING#*|}"+ # wait until no training/inference of mine is running and the GPUs are idle (3 confirmations)+ STABLE=0+ while [ $STABLE -lt 3 ]; do+ # count only REAL trainer/inference processes (python/torchrun): a plain `pgrep -f` also matches+ # monitoring shells whose own command line contains these strings, which deadlocks the queue+ MINE=0+ for pid in $(pgrep -u $USER -f "llamafactory-cli train|generate_multi.py|generate_vllm.py" 2>/dev/null); do+ case "$(ps -o comm= -p $pid 2>/dev/null)" in python*|torchrun*|pt_main*) MINE=$((MINE+1));; esac+ done+ F=$(scripts/free_gpus.sh); N=$(echo "$F" | awk -F, 'NF&&$1!=""{print NF}')+ if [ "$MINE" -eq 0 ] && [ "${N:-0}" -ge 2 ]; then STABLE=$((STABLE+1)); else STABLE=0; fi+ sleep 30+ done+ # the wait above can last hours; another process may have finished this job meanwhile, so re-confirm the+ # decision instead of acting on one made before the wait (this is how mm_multisuite ran a second time)+ if grep -qx "$label" $STATE; then echo "[sup] $(date) skipping $label — completed while waiting"; continue; fi+ echo "[sup] $(date) starting: $label"+ bash -c "$cmd" >> logs/sup_$label.log 2>&1+ RC=$?+ if [ $RC -eq 0 ]; then echo "[sup] $(date) OK: $label"; else echo "[sup] $(date) FAILED (rc=$RC): $label — see logs/sup_$label.log"; fi+ echo "$label" >> $STATE+done