Official Codebase of 3DCodeVerse
Python
eval: greedy is reproducible; the noise comes from changing the harness
gaoypeng3ab7a0e8c7fe
diff --git a/finetune/scripts/eval_img_budget.sh b/finetune/scripts/eval_img_budget.shindex 3b46944b..0345f85c 100755--- a/finetune/scripts/eval_img_budget.sh+++ b/finetune/scripts/eval_img_budget.sh@@ -4,7 +4,7 @@# budget limits the output. Probe downward from the model maximum and use the largest length that initialises;# report the truncation count, which must be 0 for the score to mean anything.set -uo pipefail; cd /wekafs/ict/hx_624/llm-ft-G=$(scripts/free_gpus.sh | cut -d, -f1)+G=${GPUS:-$(scripts/free_gpus.sh 40000 | cut -d, -f1)} # an eval needs room for a 9B, not a trainer's 60 GBsource /wekafs/ict/hx_624/anaconda3/etc/profile.d/conda.sh; conda activate vllmexport VLLM_CACHE_ROOT=/wekafs/ict/hx_624/cache/vllm HF_HOME=/wekafs/ict/hx_624/cache/huggingface VLLM_LOGGING_LEVEL=WARNING CUDA_VISIBLE_DEVICES=$Gdiff --git a/finetune/scripts/repro_probe.sh b/finetune/scripts/repro_probe.shnew file mode 100755index 00000000..e73da6a0--- /dev/null+++ b/finetune/scripts/repro_probe.sh@@ -0,0 +1,28 @@+#!/bin/bash+# 12.7 showed two DIFFERENT harnesses disagree on 29 of 211 tasks at temperature 0. The open question is whether+# greedy is reproducible when NOTHING changes: same script, same flags, same card. Three identical runs answer it.+set -uo pipefail; cd /wekafs/ict/hx_624/llm-ft+source /wekafs/ict/hx_624/anaconda3/etc/profile.d/conda.sh; conda activate vllm+export VLLM_CACHE_ROOT=/wekafs/ict/hx_624/cache/vllm HF_HOME=/wekafs/ict/hx_624/cache/huggingface VLLM_LOGGING_LEVEL=WARNING+M=runs/lf_qwen35_9b_mm_text/merged+G=""; for _ in $(seq 1 40); do G=$(scripts/free_gpus.sh 40000 | cut -d, -f1); [ -n "$G" ] && break; sleep 30; done+[ -z "$G" ] && { echo "[repro] no GPU with 40 GB free"; exit 0; }+export CUDA_VISIBLE_DEVICES=$G+for R in 1 2 3; do+ OUT=eval/out/_repro_r$R; mkdir -p $OUT+ echo "[repro] run $R on GPU $G"+ python eval/generate_vllm.py --model $M --prompts data/sft_v1/bench_prompts.jsonl --out $OUT --tp 1 \+ --no_think --dialect blender --max_new_tokens 32768 --max_model_len 40960 2>&1 | tail -1+done+conda activate llmft+python - <<'PY'+import os, glob, itertools+runs = ["eval/out/_repro_r1", "eval/out/_repro_r2", "eval/out/_repro_r3"]+tasks = sorted(set.intersection(*[{os.path.basename(p) for p in glob.glob(r + "/*") if os.path.isdir(p)} for r in runs]))+print(f"[repro] {len(tasks)} tasks common to all three runs")+for a, b in itertools.combinations(range(3), 2):+ same = sum(1 for t in tasks+ if open(f"{runs[a]}/{t}/code.py", "rb").read() == open(f"{runs[b]}/{t}/code.py", "rb").read())+ print(f"[repro] run{a+1} vs run{b+1}: byte-identical {same}/{len(tasks)}")+PY+echo "[repro] DONE"