Skip to content

Official Codebase of 3DCodeVerse

Python

finetune: md_xl/md_bal builders, geometry-feedback DPO (geo_dpo_build.sh), exploration schedulers, ntok cache, GLB renderer + contact sheets; REPORT §9–10 (md_xl scale-up, curve, geometry DPO, re-balance)

gaoypengce40be067e28
diff --git a/finetune/configs/lf/dpo_geo_md_xl.yaml b/finetune/configs/lf/dpo_geo_md_xl.yaml
new file mode 100644
index 00000000..9bd44edf
--- /dev/null
+++ b/finetune/configs/lf/dpo_geo_md_xl.yaml
@@ -0,0 +1,42 @@
+### DPO with execution-feedback preference pairs, on top of LoRA-v1 (merged)
+model_name_or_path: /wekafs/ict/hx_624/llm-ft/runs/lf_qwen35_9b_lora_md_xl/merged_geo
+trust_remote_code: true
+flash_attn: fa2
+stage: dpo
+do_train: true
+finetuning_type: lora
+lora_rank: 16
+lora_alpha: 32
+lora_dropout: 0.05
+lora_target: all
+freeze_vision_tower: true
+freeze_multi_modal_projector: true
+pref_beta: 0.1
+pref_loss: sigmoid
+dataset_dir: /wekafs/ict/hx_624/llm-ft/data/lf
+dataset: dpo_geo_pairs
+template: qwen3_5
+enable_thinking: false
+cutoff_len: 8192
+preprocessing_num_workers: 16
+dataloader_num_workers: 2
+output_dir: /wekafs/ict/hx_624/llm-ft/runs/lf_qwen35_9b_dpo_geo_md_xl
+logging_steps: 5
+save_steps: 1000
+save_only_model: true
+plot_loss: true
+overwrite_output_dir: true
+report_to: tensorboard
+per_device_train_batch_size: 1
+gradient_accumulation_steps: 8
+learning_rate: 5.0e-6
+num_train_epochs: 2.0
+lr_scheduler_type: cosine
+warmup_ratio: 0.1
+bf16: true
+gradient_checkpointing: true
+ddp_timeout: 180000000
+val_size: 0.02
+per_device_eval_batch_size: 1
+eval_strategy: steps
+eval_steps: 100
diff --git a/finetune/configs/lf/dpo_md_xl.yaml b/finetune/configs/lf/dpo_md_xl.yaml
new file mode 100644
index 00000000..7b3397ad
--- /dev/null
+++ b/finetune/configs/lf/dpo_md_xl.yaml
@@ -0,0 +1,42 @@
+### DPO with execution-feedback preference pairs, on top of LoRA-v1 (merged)
+model_name_or_path: /wekafs/ict/hx_624/llm-ft/runs/lf_qwen35_9b_lora_md_xl/merged
+trust_remote_code: true
+flash_attn: fa2
+stage: dpo
+do_train: true
+finetuning_type: lora
+lora_rank: 16
+lora_alpha: 32
+lora_dropout: 0.05
+lora_target: all
+freeze_vision_tower: true
+freeze_multi_modal_projector: true
+pref_beta: 0.1
+pref_loss: sigmoid
+dataset_dir: /wekafs/ict/hx_624/llm-ft/data/lf
+dataset: dpo_md_xl_pairs
+template: qwen3_5
+enable_thinking: false
+cutoff_len: 8192
+preprocessing_num_workers: 16
+dataloader_num_workers: 2
+output_dir: /wekafs/ict/hx_624/llm-ft/runs/lf_qwen35_9b_dpo_md_xl
+logging_steps: 5
+save_steps: 1000
+save_only_model: true
+plot_loss: true
+overwrite_output_dir: /wekafs/ict/hx_624/llm-ft/runs/lf_qwen35_9b_dpo_md_xl
+report_to: tensorboard
+per_device_train_batch_size: 1
+gradient_accumulation_steps: 8
+learning_rate: 5.0e-6
+num_train_epochs: 2.0
+lr_scheduler_type: cosine
+warmup_ratio: 0.1
+bf16: true
+gradient_checkpointing: true
+ddp_timeout: 180000000
+val_size: 0.02
+per_device_eval_batch_size: 1
+eval_strategy: steps
+eval_steps: 100
diff --git a/finetune/configs/lf/export_dpo_geo_md_xl.yaml b/finetune/configs/lf/export_dpo_geo_md_xl.yaml
new file mode 100644
index 00000000..ee105071
--- /dev/null
+++ b/finetune/configs/lf/export_dpo_geo_md_xl.yaml
@@ -0,0 +1,12 @@
+### Merge a LoRA adapter into the base model (LLaMA-Factory): llamafactory-cli export <this file>
+model_name_or_path: /wekafs/ict/hx_624/llm-ft/runs/lf_qwen35_9b_lora_md_xl/merged_geo
+adapter_name_or_path: /wekafs/ict/hx_624/llm-ft/runs/lf_qwen35_9b_dpo_geo_md_xl
+template: qwen3_5
+enable_thinking: false
+trust_remote_code: true
+finetuning_type: lora
+
+export_dir: /wekafs/ict/hx_624/llm-ft/runs/lf_qwen35_9b_dpo_geo_md_xl/merged
+export_size: 5
+export_device: cpu
+export_legacy_format: false
diff --git a/finetune/configs/lf/export_dpo_md_xl.yaml b/finetune/configs/lf/export_dpo_md_xl.yaml
new file mode 100644
index 00000000..bda73106
--- /dev/null
+++ b/finetune/configs/lf/export_dpo_md_xl.yaml
@@ -0,0 +1,12 @@
+### Merge a LoRA adapter into the base model (LLaMA-Factory): llamafactory-cli export <this file>
+model_name_or_path: /wekafs/ict/hx_624/llm-ft/runs/lf_qwen35_9b_lora_md_xl/merged
+adapter_name_or_path: /wekafs/ict/hx_624/llm-ft/runs/lf_qwen35_9b_dpo_md_xl
+template: qwen3_5
+enable_thinking: false
+trust_remote_code: true
+finetuning_type: lora
+
+export_dir: /wekafs/ict/hx_624/llm-ft/runs/lf_qwen35_9b_dpo_md_xl/merged
+export_size: 5
+export_device: cpu
+export_legacy_format: false
diff --git a/finetune/configs/lf/export_md_bal.yaml b/finetune/configs/lf/export_md_bal.yaml
new file mode 100644
index 00000000..1a9a7f8b
--- /dev/null
+++ b/finetune/configs/lf/export_md_bal.yaml
@@ -0,0 +1,12 @@
+### Merge a LoRA adapter into the base model (LLaMA-Factory): llamafactory-cli export <this file>
+model_name_or_path: /wekafs/ict/hx_624/models/Qwen3.5-9B
+adapter_name_or_path: /wekafs/ict/hx_624/llm-ft/runs/lf_qwen35_9b_lora_md_bal
+template: qwen3_5
+enable_thinking: false
+trust_remote_code: true
+finetuning_type: lora
+
+export_dir: /wekafs/ict/hx_624/llm-ft/runs/lf_qwen35_9b_lora_md_bal/merged
+export_size: 5
+export_device: cpu
+export_legacy_format: false
diff --git a/finetune/configs/lf/export_md_xl_ck1350.yaml b/finetune/configs/lf/export_md_xl_ck1350.yaml
new file mode 100644
index 00000000..147064c7
--- /dev/null
+++ b/finetune/configs/lf/export_md_xl_ck1350.yaml
@@ -0,0 +1,12 @@
+### Merge a LoRA adapter into the base model (LLaMA-Factory): llamafactory-cli export <this file>
+model_name_or_path: /wekafs/ict/hx_624/models/Qwen3.5-9B
+adapter_name_or_path: /wekafs/ict/hx_624/llm-ft/runs/lf_qwen35_9b_lora_md_xl/checkpoint-1350
+template: qwen3_5
+enable_thinking: false
+trust_remote_code: true
+finetuning_type: lora
+
+export_dir: /wekafs/ict/hx_624/llm-ft/runs/lf_qwen35_9b_lora_md_xl/checkpoint-1350/merged
+export_size: 5
+export_device: cpu
+export_legacy_format: false
diff --git a/finetune/configs/lf/export_md_xl_ck1800.yaml b/finetune/configs/lf/export_md_xl_ck1800.yaml
new file mode 100644
index 00000000..49c8bbf0
--- /dev/null
+++ b/finetune/configs/lf/export_md_xl_ck1800.yaml
@@ -0,0 +1,12 @@
+### Merge a LoRA adapter into the base model (LLaMA-Factory): llamafactory-cli export <this file>
+model_name_or_path: /wekafs/ict/hx_624/models/Qwen3.5-9B
+adapter_name_or_path: /wekafs/ict/hx_624/llm-ft/runs/lf_qwen35_9b_lora_md_xl/checkpoint-1800
+template: qwen3_5
+enable_thinking: false
+trust_remote_code: true
+finetuning_type: lora
+
+export_dir: /wekafs/ict/hx_624/llm-ft/runs/lf_qwen35_9b_lora_md_xl/checkpoint-1800/merged
+export_size: 5
+export_device: cpu
+export_legacy_format: false
diff --git a/finetune/configs/lf/export_md_xl_ck2250.yaml b/finetune/configs/lf/export_md_xl_ck2250.yaml
new file mode 100644
index 00000000..238e03cf
--- /dev/null
+++ b/finetune/configs/lf/export_md_xl_ck2250.yaml
@@ -0,0 +1,12 @@
+### Merge a LoRA adapter into the base model (LLaMA-Factory): llamafactory-cli export <this file>
+model_name_or_path: /wekafs/ict/hx_624/models/Qwen3.5-9B
+adapter_name_or_path: /wekafs/ict/hx_624/llm-ft/runs/lf_qwen35_9b_lora_md_xl/checkpoint-2250
+template: qwen3_5
+enable_thinking: false
+trust_remote_code: true
+finetuning_type: lora
+
+export_dir: /wekafs/ict/hx_624/llm-ft/runs/lf_qwen35_9b_lora_md_xl/checkpoint-2250/merged
+export_size: 5
+export_device: cpu
+export_legacy_format: false
diff --git a/finetune/configs/lf/export_md_xl_ck2700.yaml b/finetune/configs/lf/export_md_xl_ck2700.yaml
new file mode 100644
index 00000000..8d739754
--- /dev/null
+++ b/finetune/configs/lf/export_md_xl_ck2700.yaml
@@ -0,0 +1,12 @@
+### Merge a LoRA adapter into the base model (LLaMA-Factory): llamafactory-cli export <this file>
+model_name_or_path: /wekafs/ict/hx_624/models/Qwen3.5-9B
+adapter_name_or_path: /wekafs/ict/hx_624/llm-ft/runs/lf_qwen35_9b_lora_md_xl/checkpoint-2700
+template: qwen3_5
+enable_thinking: false
+trust_remote_code: true
+finetuning_type: lora
+
+export_dir: /wekafs/ict/hx_624/llm-ft/runs/lf_qwen35_9b_lora_md_xl/checkpoint-2700/merged
+export_size: 5
+export_device: cpu
+export_legacy_format: false
diff --git a/finetune/configs/lf/export_md_xl_ck3150.yaml b/finetune/configs/lf/export_md_xl_ck3150.yaml
new file mode 100644
index 00000000..838a7dc1
--- /dev/null
+++ b/finetune/configs/lf/export_md_xl_ck3150.yaml
@@ -0,0 +1,12 @@
+### Merge a LoRA adapter into the base model (LLaMA-Factory): llamafactory-cli export <this file>
+model_name_or_path: /wekafs/ict/hx_624/models/Qwen3.5-9B
+adapter_name_or_path: /wekafs/ict/hx_624/llm-ft/runs/lf_qwen35_9b_lora_md_xl/checkpoint-3150
+template: qwen3_5
+enable_thinking: false
+trust_remote_code: true
+finetuning_type: lora
+
+export_dir: /wekafs/ict/hx_624/llm-ft/runs/lf_qwen35_9b_lora_md_xl/checkpoint-3150/merged
+export_size: 5
+export_device: cpu
+export_legacy_format: false
diff --git a/finetune/configs/lf/export_md_xl_ck3600.yaml b/finetune/configs/lf/export_md_xl_ck3600.yaml
new file mode 100644
index 00000000..8009ea3b
--- /dev/null
+++ b/finetune/configs/lf/export_md_xl_ck3600.yaml
@@ -0,0 +1,12 @@
+### Merge a LoRA adapter into the base model (LLaMA-Factory): llamafactory-cli export <this file>
+model_name_or_path: /wekafs/ict/hx_624/models/Qwen3.5-9B
+adapter_name_or_path: /wekafs/ict/hx_624/llm-ft/runs/lf_qwen35_9b_lora_md_xl/checkpoint-3600
+template: qwen3_5
+enable_thinking: false
+trust_remote_code: true
+finetuning_type: lora
+
+export_dir: /wekafs/ict/hx_624/llm-ft/runs/lf_qwen35_9b_lora_md_xl/checkpoint-3600/merged
+export_size: 5
+export_device: cpu
+export_legacy_format: false
diff --git a/finetune/configs/lf/export_md_xl_ck900.yaml b/finetune/configs/lf/export_md_xl_ck900.yaml
new file mode 100644
index 00000000..21c131e3
--- /dev/null
+++ b/finetune/configs/lf/export_md_xl_ck900.yaml
@@ -0,0 +1,12 @@
+### Merge a LoRA adapter into the base model (LLaMA-Factory): llamafactory-cli export <this file>
+model_name_or_path: /wekafs/ict/hx_624/models/Qwen3.5-9B
+adapter_name_or_path: /wekafs/ict/hx_624/llm-ft/runs/lf_qwen35_9b_lora_md_xl/checkpoint-900
+template: qwen3_5
+enable_thinking: false
+trust_remote_code: true
+finetuning_type: lora
+
+export_dir: /wekafs/ict/hx_624/llm-ft/runs/lf_qwen35_9b_lora_md_xl/checkpoint-900/merged
+export_size: 5
+export_device: cpu
+export_legacy_format: false
diff --git a/finetune/data_specs/dataset_info.json b/finetune/data_specs/dataset_info.json
index d4cb7d10..2611010d 100644
--- a/finetune/data_specs/dataset_info.json
+++ b/finetune/data_specs/dataset_info.json
@@ -346,5 +346,27 @@
"messages": "conversations",
"system": "system"
}
+ },
+ "dpo_md_xl_pairs": {
+ "file_name": "dpo_md_xl_pairs.json",
+ "ranking": true,
+ "formatting": "sharegpt",
+ "columns": {
+ "messages": "conversations",
+ "system": "system",
+ "chosen": "chosen",
+ "rejected": "rejected"
+ }
+ },
+ "dpo_geo_pairs": {
+ "file_name": "dpo_geo_pairs.json",
+ "ranking": true,
+ "formatting": "sharegpt",
+ "columns": {
+ "messages": "conversations",
+ "system": "system",
+ "chosen": "chosen",
+ "rejected": "rejected"
+ }
}
}
\ No newline at end of file
diff --git a/finetune/docs/assets/md_xl_vs_geo_dpo_contact_sheet.png b/finetune/docs/assets/md_xl_vs_geo_dpo_contact_sheet.png
new file mode 100644
index 00000000..8cfe8c68
Binary files /dev/null and b/finetune/docs/assets/md_xl_vs_geo_dpo_contact_sheet.png differ
diff --git a/finetune/scripts/contact_sheet_compare.py b/finetune/scripts/contact_sheet_compare.py
new file mode 100644
index 00000000..750836b2
--- /dev/null
+++ b/finetune/scripts/contact_sheet_compare.py
@@ -0,0 +1,32 @@
+"""3-column contact sheet on the SAME tasks: model A vs model B vs GT (tasks = the selection used by contact_sheet_3dcodebench.py for model A)."""
+import json, os, sys, subprocess, random
+from concurrent.futures import ThreadPoolExecutor
+from PIL import Image, ImageDraw, ImageFont
+BL = "/wekafs/ict/hx_624/tools/blender-5.0.1-linux-x64/blender"; R = os.path.abspath("eval/render_glb.py"); GT = "/wekafs/ict/hx_624/data/3dcodebench/data"
+dirA, nameA, dirB, nameB, out_png = sys.argv[1:6]; k = 4
+mA = {r["task"]: r for r in map(json.loads, open(f"{dirA}/metrics.jsonl"))}; mB = {r["task"]: r for r in map(json.loads, open(f"{dirB}/metrics.jsonl"))}
+ok = sorted([r for r in mA.values() if r["exec"] == "OK" and r.get("[email protected]") is not None], key=lambda r: -r["[email protected]"]); random.seed(3); n = len(ok)
+sel = [("best", r) for r in ok[:k]] + [("median", r) for r in ok[n//2 - k//2: n//2 - k//2 + k]] + [("worst", r) for r in ok[-k:]] + [("random", r) for r in random.sample(ok[k:-k], k)]
+def render(glb, png):
+ if os.path.exists(png) or not os.path.exists(glb): return
+ subprocess.run([BL, "-b", "--factory-startup", "-noaudio", "--python", R, "--", "--glb", glb, "--out", png], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=300, env={**os.environ, "HOME": "/tmp"})
+wA = os.path.abspath(f"{dirA}/_renders"); wB = os.path.abspath(f"{dirB}/_renders"); os.makedirs(wA, exist_ok=True); os.makedirs(wB, exist_ok=True)
+jobs = []
+for tag, r in sel:
+ t = r["task"]; jobs += [(os.path.abspath(f"{dirA}/{t}/out.glb"), f"{wA}/{t}_gen.png"), (os.path.abspath(f"{dirB}/{t}/out.glb"), f"{wB}/{t}_gen.png"), (f"{GT}/{t}/glb/{t}.glb", f"{wA}/{t}_gt.png")]
+with ThreadPoolExecutor(8) as ex: list(ex.map(lambda j: render(*j), jobs))
+S = 180; cols = k; rows = 4
+sheet = Image.new("RGB", (cols * (3 * S + 16) + 40, rows * (S + 52) + 10), "white"); d = ImageDraw.Draw(sheet)
+try: font = ImageFont.truetype("/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf", 12); fb = ImageFont.truetype("/usr/share/fonts/truetype/dejavu/DejaVuSans-Bold.ttf", 12)
+except Exception: font = fb = ImageFont.load_default()
+for i, (tag, r) in enumerate(sel):
+ row, col = i // cols, i % cols; x0 = 40 + col * (3 * S + 16); y0 = 10 + row * (S + 52); t = r["task"]
+ for j, p in enumerate((f"{wA}/{t}_gen.png", f"{wB}/{t}_gen.png", f"{wA}/{t}_gt.png")):
+ try: im = Image.open(p).convert("RGB").resize((S, S))
+ except Exception: im = Image.new("RGB", (S, S), (230, 230, 230))
+ sheet.paste(im, (x0 + j * S, y0 + 20))
+ fa = r["[email protected]"]; fbv = mB.get(t, {}).get("[email protected]"); sb = f"{fbv:.2f}" if fbv is not None else mB.get(t, {}).get("exec", "?")
+ d.text((x0, y0 + 2), f"{t.replace('_seed0','')} [email protected] {nameA}={fa:.2f} | {nameB}={sb}", fill="black", font=fb)
+ d.text((x0, y0 + S + 24), nameA, fill=(60, 60, 60), font=font); d.text((x0 + S, y0 + S + 24), nameB, fill=(60, 60, 60), font=font); d.text((x0 + 2 * S, y0 + S + 24), "ground truth", fill=(60, 60, 60), font=font)
+ if col == 0: d.text((4, y0 + S // 2), tag, fill=(120, 0, 0), font=fb)
+sheet.save(out_png); print("wrote", out_png, sheet.size)
diff --git a/finetune/scripts/explore_sched2.sh b/finetune/scripts/explore_sched2.sh
new file mode 100755
index 00000000..055bf300
--- /dev/null
+++ b/finetune/scripts/explore_sched2.sh
@@ -0,0 +1,36 @@
+#!/bin/bash
+# revised 8h scheduler (GPUs 0,2,3 idle now, GPU 1 busy with the md_xl sweep):
+# (1) now: md_bal LoRA on GPUs 0,2,3 (train+merge+evals) (2) after sweep: geometry-feedback DPO on GPU 1 (train+export+evals)
+# (3) after both: full FT md_bal on GPUs 0,1,2,3 (ZeRO-3) + evals
+set -uo pipefail; cd /wekafs/ict/hx_624/llm-ft
+echo "[sched] $(date) md_bal LoRA on GPUs 0,2,3"
+scripts/run_md_exp.sh 0,2,3 md_bal > logs/exp_md_bal.log 2>&1 &
+( until grep -q "\[bigsweep\] DONE" logs/md_xl_ckpt_sweep.log 2>/dev/null; do sleep 60; done
+ echo "[sched] $(date) geometry-feedback DPO on GPU 1"
+ GPUS=1 scripts/lf_train.sh configs/lf/dpo_geo_md_xl.yaml gradient_accumulation_steps=8 cutoff_len=4096 > logs/train_dpo_geo_md_xl.log 2>&1
+ RUN=runs/lf_qwen35_9b_dpo_geo_md_xl
+ if [ ! -f $RUN/train_results.json ]; then echo "[sched] GEO DPO TRAIN FAILED"; grep -E "Error" logs/train_dpo_geo_md_xl.log | grep -v errors | tail -2 | cut -c1-200; echo "[sched] GEO_DONE"; exit 0; fi
+ echo "[sched] $(date) geo dpo train done: $(tr -d '\n' < $RUN/train_results.json | cut -c1-200)"
+ BASE=runs/lf_qwen35_9b_lora_md_xl/merged_geo
+ sed -e "s|model_name_or_path:.*|model_name_or_path: $(readlink -f $BASE)|" -e "s|adapter_name_or_path:.*|adapter_name_or_path: $(readlink -f $RUN)|" -e "s|export_dir:.*|export_dir: $(readlink -f $RUN)/merged|" configs/lf/export_lora_template.yaml > configs/lf/export_dpo_geo_md_xl.yaml
+ scripts/lf_export.sh configs/lf/export_dpo_geo_md_xl.yaml > logs/export_dpo_geo_md_xl.log 2>&1
+ GPUS=1 eval/run_eval_vllm.sh $RUN/merged eval/out/qwen35_9b_dpo_geo_md_xl --no_think --max_new_tokens 6144 > logs/eval_dpo_geo_md_xl.log 2>&1
+ echo "[sched] geo dpo 3DCodeBench: $(python3 -c "import json; s=json.load(open('eval/out/qwen35_9b_dpo_geo_md_xl/summary.json')); print(s['exec_ok'], round(s['[email protected]_mean_all(fail=0)'],3), 'scored', round(s['[email protected]_mean_scored'],3), '[email protected] scored', round(s['[email protected]_mean_scored'],3))")"
+ GPUS=1 eval/run_dialect_eval.sh $RUN/merged dpo_geo_md_xl blender cadquery > logs/mdeval_dpo_geo_md_xl.log 2>&1
+ for D in blender cadquery; do echo "[sched] geo dpo $D: $(cat eval/out/md_dpo_geo_md_xl_$D/summary.json | tr -d '\n ' | cut -c1-200)"; done
+ GPUS=1 eval/best_of_n.sh $RUN/merged dpo_geo_md_xl_T07 4 0.7 2>&1 | grep -E "pass@" | tail -1 | sed 's/^/[sched] geo dpo /'
+ echo "[sched] GEO_DONE" ) &
+until grep -qE "\[md md_bal\] (DONE|TRAIN FAILED|EXPORT FAILED)" logs/exp_md_bal.log 2>/dev/null; do sleep 60; done
+until grep -q "GEO_DONE" logs/explore_sched2.log 2>/dev/null; do sleep 60; done
+echo "[sched] $(date) full FT md_bal on GPUs 0,1,2,3 (ZeRO-3, accum 2)"
+GPUS=0,1,2,3 scripts/lf_train.sh configs/lf/full_md_bal.yaml gradient_accumulation_steps=2 > logs/train_full_md_bal.log 2>&1
+RUN=runs/lf_qwen35_9b_full_md_bal
+if [ -f $RUN/train_results.json ]; then
+ echo "[sched] $(date) full md_bal train done: $(tr -d '\n' < $RUN/train_results.json | cut -c1-200)"
+ GPUS=0 eval/run_eval_vllm.sh $RUN eval/out/qwen35_9b_full_md_bal --no_think --max_new_tokens 6144 > logs/eval_full_md_bal.log 2>&1 &
+ GPUS=1 eval/run_dialect_eval.sh $RUN full_md_bal > logs/mdeval_full_md_bal.log 2>&1 &
+ wait
+ echo "[sched] full md_bal 3DCodeBench: $(python3 -c "import json; s=json.load(open('eval/out/qwen35_9b_full_md_bal/summary.json')); print(s['exec_ok'], round(s['[email protected]_mean_all(fail=0)'],3))")"
+ for D in cadquery openscad glsl blender; do echo "[sched] full md_bal $D: $(cat eval/out/md_full_md_bal_$D/summary.json | tr -d '\n ' | cut -c1-160)"; done
+else echo "[sched] FULL TRAIN FAILED"; grep -E "Error" logs/train_full_md_bal.log | grep -v errors | tail -2 | cut -c1-200; fi
+echo "[sched] FULL_DONE"; echo "[sched] ALL DONE"