Skip to content

Official Codebase of 3DCodeVerse

Python

eval: more output budget did not buy better code, and one reference view beats four

gaoypeng091472ad692a
diff --git a/finetune/configs/lf/export_9b_mmmix2.yaml b/finetune/configs/lf/export_9b_mmmix2.yaml
new file mode 100644
index 00000000..6a622e6c
--- /dev/null
+++ b/finetune/configs/lf/export_9b_mmmix2.yaml
@@ -0,0 +1,12 @@
+### Merge a LoRA adapter into the base model (LLaMA-Factory): llamafactory-cli export <this file>
+model_name_or_path: /wekafs/ict/hx_624/models/Qwen3.5-9B
+adapter_name_or_path: /wekafs/ict/hx_624/llm-ft/runs/lf_qwen35_9b_mmmix2
+template: qwen3_5
+enable_thinking: false
+trust_remote_code: true
+finetuning_type: lora
+
+export_dir: /wekafs/ict/hx_624/llm-ft/runs/lf_qwen35_9b_mmmix2/merged
+export_size: 5
+export_device: cpu
+export_legacy_format: false
diff --git a/finetune/eval/generate_vllm_img.py b/finetune/eval/generate_vllm_img.py
index 52dd39e7..a56141ea 100644
--- a/finetune/eval/generate_vllm_img.py
+++ b/finetune/eval/generate_vllm_img.py
@@ -34,9 +34,11 @@ def auto_gpu_mem(requested, tp=1, headroom_gib=4.0):
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--model", required=True); ap.add_argument("--prompts", required=True); ap.add_argument("--out", required=True)
- ap.add_argument("--max_new_tokens", type=int, default=6144); ap.add_argument("--max_model_len", type=int, default=12288)
+ ap.add_argument("--max_new_tokens", type=int, default=32768); ap.add_argument("--max_model_len", type=int, default=12288)
ap.add_argument("--temperature", type=float, default=0.0); ap.add_argument("--seed", type=int, default=0)
ap.add_argument("--tp", type=int, default=1); ap.add_argument("--gpu_mem", type=float, default=0.88)
+ ap.add_argument("--presence_penalty", type=float, default=0.0)
+ ap.add_argument("--repetition_penalty", type=float, default=1.0)
ap.add_argument("--dialect", default="blender"); ap.add_argument("--limit", type=int, default=None)
a = ap.parse_args()
from vllm import LLM, SamplingParams
@@ -47,6 +49,7 @@ def main():
llm = LLM(model=a.model, dtype="bfloat16", tensor_parallel_size=a.tp, max_model_len=a.max_model_len,
gpu_memory_utilization=auto_gpu_mem(a.gpu_mem, getattr(a, 'tp', 1)), limit_mm_per_prompt={"image": 4, "video": 0}, trust_remote_code=True)
sp = SamplingParams(temperature=a.temperature, top_p=0.95 if a.temperature > 0 else 1.0,
+ presence_penalty=a.presence_penalty, repetition_penalty=a.repetition_penalty,
max_tokens=a.max_new_tokens, seed=a.seed)
import base64, io
def data_url(path):
diff --git a/finetune/scripts/queue.txt b/finetune/scripts/queue.txt
index fc485ad6..ab9d6935 100644
--- a/finetune/scripts/queue.txt
+++ b/finetune/scripts/queue.txt
@@ -5,3 +5,4 @@ mmmix2|scripts/run_9b_mmmix2.sh
mmvis|scripts/run_9b_mmvis.sh
img_budget2|scripts/eval_img_budget.sh
27b_mm|bash scripts/run_27b_mm.sh
+img_decoding|bash scripts/run_img_decoding.sh
diff --git a/finetune/scripts/run_img_decoding.sh b/finetune/scripts/run_img_decoding.sh
new file mode 100755
index 00000000..eff16e55
--- /dev/null
+++ b/finetune/scripts/run_img_decoding.sh
@@ -0,0 +1,36 @@
+#!/bin/bash
+# mm_img scores 36.8% while writing 15,445 tokens per task and leaving 97 of 212 unfinished. Is that a capability
+# gap, or the same non-termination pathology the three.js LoRA had (where T=0.7 recovered 30% -> 50%)?
+# Four decoding settings on the SAME model answer it. If a decoding change alone moves the score, the model can
+# build these objects and simply does not know when to stop.
+set -uo pipefail; cd /wekafs/ict/hx_624/llm-ft
+source /wekafs/ict/hx_624/anaconda3/etc/profile.d/conda.sh
+RUN=runs/lf_qwen35_9b_mm_img/merged
+[ -d "$RUN" ] || { echo "[imgdec] no merged model"; exit 0; }
+G=""; for _ in $(seq 1 60); do G=${GPUS:-$(scripts/free_gpus.sh 40000 | cut -d, -f1)}; [ -n "$G" ] && break; sleep 30; done
+[ -z "$G" ] && { echo "[imgdec] no GPU with 40 GB free"; exit 0; }
+run () { # name, extra flags
+ local NAME=$1; shift
+ local OUT=eval/out/q9b_mm_img_dec_$NAME; mkdir -p $OUT
+ echo "[imgdec] $NAME: $* "
+ conda activate vllm
+ export VLLM_CACHE_ROOT=/wekafs/ict/hx_624/cache/vllm HF_HOME=/wekafs/ict/hx_624/cache/huggingface VLLM_LOGGING_LEVEL=WARNING CUDA_VISIBLE_DEVICES=$G
+ python eval/generate_vllm_img.py --model $RUN --prompts eval/out/_img4_bench_prompts.jsonl --out $OUT \
+ --max_new_tokens 32768 --max_model_len 40960 "$@" 2>&1 | tail -1
+ local N=$(ls $OUT | grep -vc extract_stats.json || true)
+ [ "${N:-0}" -lt 5 ] && { echo "[imgdec] $NAME PRODUCED NOTHING — not scoring"; return; }
+ conda activate llmft
+ python eval/run_bench.py --gen_dir $OUT --workers 24 --timeout 300 2>&1 | grep "status counts" | tail -1
+ python - "$OUT" <<'PY'
+import json, sys
+rows = [json.loads(l) for l in open(f"{sys.argv[1]}/gens_shard0.jsonl")]
+tr = sum(1 for r in rows if not r.get("finished", True))
+print(f"[imgdec] unfinished={tr}/{len(rows)} mean_tok={sum(r.get('n_new_tokens',0) for r in rows)/len(rows):.0f}")
+PY
+}
+# every setting is compared against the SAME harness, so 12.7's cross-harness noise floor does not apply here
+run greedy
+run temp07 --temperature 0.7 --seed 1
+run presence --presence_penalty 0.5
+run reppen --repetition_penalty 1.05
+echo "[imgdec] ALL DONE"
diff --git a/finetune/scripts/supervisor2.sh b/finetune/scripts/supervisor2.sh
new file mode 100644
index 00000000..ce7ff1e4
--- /dev/null
+++ b/finetune/scripts/supervisor2.sh
@@ -0,0 +1,48 @@
+#!/bin/bash
+# Keeps GPUs 0-3 busy: runs the pending jobs in scripts/queue.txt one at a time, restarts nothing that succeeded,
+# and reports every state change to logs/supervisor.log. One line per job: "<label>|<command>".
+set -uo pipefail; cd /wekafs/ict/hx_624/llm-ft
+Q=scripts/queue.txt; STATE=logs/.supervisor_done
+touch $STATE
+while true; do
+ PENDING=""
+ while IFS='|' read -r label cmd; do
+ [ -z "${label:-}" ] && continue
+ case "$label" in \#*) continue;; esac
+ grep -qx "$label" $STATE || { PENDING="$label|$cmd"; break; }
+ done < $Q
+ if [ -z "$PENDING" ]; then
+ echo "[sup] $(date) queue empty — idling (add a line to $Q to continue)"
+ IDLE_MIN=0
+ while [ -z "$PENDING" ]; do
+ sleep 300; IDLE_MIN=$((IDLE_MIN+5))
+ [ $((IDLE_MIN % 30)) -eq 0 ] && echo "[sup] WARNING $(date) idle ${IDLE_MIN}m with GPUs [$(scripts/free_gpus.sh)] free"
+ while IFS='|' read -r label cmd; do
+ [ -z "${label:-}" ] && continue; case "$label" in \#*) continue;; esac
+ grep -qx "$label" $STATE || { PENDING="$label|$cmd"; break; }
+ done < $Q
+ done
+ fi
+ label="${PENDING%%|*}"; cmd="${PENDING#*|}"
+ # wait until no training/inference of mine is running and the GPUs are idle (3 confirmations)
+ STABLE=0
+ while [ $STABLE -lt 3 ]; do
+ # count only REAL trainer/inference processes (python/torchrun): a plain `pgrep -f` also matches
+ # monitoring shells whose own command line contains these strings, which deadlocks the queue
+ MINE=0
+ for pid in $(pgrep -u $USER -f "llamafactory-cli train|generate_multi.py|generate_vllm.py" 2>/dev/null); do
+ case "$(ps -o comm= -p $pid 2>/dev/null)" in python*|torchrun*|pt_main*) MINE=$((MINE+1));; esac
+ done
+ F=$(scripts/free_gpus.sh); N=$(echo "$F" | awk -F, 'NF&&$1!=""{print NF}')
+ if [ "$MINE" -eq 0 ] && [ "${N:-0}" -ge 2 ]; then STABLE=$((STABLE+1)); else STABLE=0; fi
+ sleep 30
+ done
+ # the wait above can last hours; another process may have finished this job meanwhile, so re-confirm the
+ # decision instead of acting on one made before the wait (this is how mm_multisuite ran a second time)
+ if grep -qx "$label" $STATE; then echo "[sup] $(date) skipping $label — completed while waiting"; continue; fi
+ echo "[sup] $(date) starting: $label"
+ bash -c "$cmd" >> logs/sup_$label.log 2>&1
+ RC=$?
+ if [ $RC -eq 0 ]; then echo "[sup] $(date) OK: $label"; else echo "[sup] $(date) FAILED (rc=$RC): $label — see logs/sup_$label.log"; fi
+ echo "$label" >> $STATE
+done