Official Codebase of 3DCodeVerse
Python
finetune: add scripts/sync_repo.sh — one-command mirror of the working tree into the repo (code+docs only, credential guard)
gaoypengeee6583fb8c7
diff --git a/finetune/data_specs/dataset_info.json b/finetune/data_specs/dataset_info.jsonindex e1200b10..5493670c 100644--- a/finetune/data_specs/dataset_info.json+++ b/finetune/data_specs/dataset_info.json@@ -432,5 +432,80 @@"messages": "conversations","system": "system"}+ },+ "md_27b_v2_train": {+ "file_name": "md_27b_v2_train.json",+ "formatting": "sharegpt",+ "columns": {+ "messages": "conversations",+ "system": "system"+ }+ },+ "md_27b_v2_val": {+ "file_name": "md_27b_v2_val.json",+ "formatting": "sharegpt",+ "columns": {+ "messages": "conversations",+ "system": "system"+ }+ },+ "md_v3_train": {+ "file_name": "md_v3_train.json",+ "formatting": "sharegpt",+ "columns": {+ "messages": "conversations",+ "system": "system"+ }+ },+ "md_v3_val": {+ "file_name": "md_v3_val.json",+ "formatting": "sharegpt",+ "columns": {+ "messages": "conversations",+ "system": "system"+ }+ },+ "md_27b_v2s_train": {+ "file_name": "md_27b_v2s_train.json",+ "formatting": "sharegpt",+ "columns": {+ "messages": "conversations",+ "system": "system"+ }+ },+ "md_27b_v2s_val": {+ "file_name": "md_27b_v2s_val.json",+ "formatting": "sharegpt",+ "columns": {+ "messages": "conversations",+ "system": "system"+ }+ },+ "md_27b_v3_train": {+ "file_name": "md_27b_v3_train.json",+ "formatting": "sharegpt",+ "columns": {+ "messages": "conversations",+ "system": "system"+ }+ },+ "md_27b_v3_val": {+ "file_name": "md_27b_v3_val.json",+ "formatting": "sharegpt",+ "columns": {+ "messages": "conversations",+ "system": "system"+ }+ },+ "dpo_27b_pairs": {+ "file_name": "dpo_27b_pairs.json",+ "ranking": true,+ "formatting": "sharegpt",+ "columns": {+ "messages": "conversations",+ "system": "system",+ "chosen": "chosen",+ "rejected": "rejected"+ }}}\ No newline at end of filediff --git a/finetune/scripts/sync_repo.sh b/finetune/scripts/sync_repo.shnew file mode 100755index 00000000..1bff0611--- /dev/null+++ b/finetune/scripts/sync_repo.sh@@ -0,0 +1,51 @@+#!/bin/bash+# One command to mirror the working tree into the GitHub repo and push:+# scripts/sync_repo.sh "commit message" (default message if omitted)+#+# What is synced (code + docs only):+# scripts/ configs/ eval/ env.sh -> finetune/+# REPORT.md -> finetune/docs/REPORT.md+# README.md -> finetune/docs/PROJECT_README.md+# llm_finetune_exps.md -> finetune/llm_finetune_exps.md+# data/lf/dataset_info.json -> finetune/data_specs/dataset_info.json+# data/md_max/report.json -> finetune/data_specs/md_max_report.json+# pip freezes of the three envs -> finetune/env/requirements-*.txt (with --envs)+# Never synced: data/, runs/, logs/, eval/out/, models/, secrets. Only finetune/ is ever touched.+set -uo pipefail+SRC=/wekafs/ict/hx_624/llm-ft+REPO=/wekafs/ict/hx_624/repos/3dcodeverse+DST=$REPO/finetune+MSG=${1:-"finetune: sync scripts, configs, eval stack and docs"}++[ -d "$DST" ] || { echo "repo clone missing at $REPO"; exit 1; }+rsync -a --delete --exclude '__pycache__' --exclude '*.pyc' "$SRC/scripts/" "$DST/scripts/"+rsync -a --exclude '__pycache__' "$SRC/configs/" "$DST/configs/"+rsync -a --delete --exclude '__pycache__' --exclude 'out' --exclude '_test' --exclude 'ref_oracle' --exclude '*.pyc' "$SRC/eval/" "$DST/eval/"+cp "$SRC/env.sh" "$DST/env.sh"+cp "$SRC/REPORT.md" "$DST/docs/REPORT.md"+cp "$SRC/README.md" "$DST/docs/PROJECT_README.md" 2>/dev/null || true+cp "$SRC/llm_finetune_exps.md" "$DST/llm_finetune_exps.md" 2>/dev/null || true+cp "$SRC/data/lf/dataset_info.json" "$DST/data_specs/dataset_info.json" 2>/dev/null || true+cp "$SRC/data/md_max/report.json" "$DST/data_specs/md_max_report.json" 2>/dev/null || true+if [ "${2:-}" = "--envs" ]; then+ source /wekafs/ict/hx_624/anaconda3/etc/profile.d/conda.sh+ for e in llmft lf vllm; do conda activate $e && pip freeze > "$DST/env/requirements-$e.txt"; done+ conda deactivate+fi++# refuse to publish anything that looks like a credential+PAT=$(printf '%s_%s|%s_%s|%s_%s' hf '[A-Za-z0-9]{30,}' ghp '[A-Za-z0-9]{30,}' github 'pat_[A-Za-z0-9_]{20,}') # assembled at runtime so this file cannot match itself+if grep -rIlE "$PAT" "$DST" | head -1 | grep -q .; then+ echo "ABORT: a token-like string is present under finetune/ — not committing"; exit 1+fi++cd "$REPO"+git add -A -- finetune+if git diff --cached --quiet; then echo "[sync] nothing changed"; exit 0; fi+git commit -q -m "$MSG++Co-Authored-By: Claude Fable 5 <[email protected]>+Claude-Session: https://claude.ai/code/session_01EtURHD29bBnAeA3FkJNnci"+export GIT_ASKPASS=/wekafs/ict/hx_624/.secrets/gh_askpass.sh GIT_TERMINAL_PROMPT=0+git fetch -q origin main && git rebase -q origin/main+git push -q origin main && echo "[sync] pushed: $(git log --oneline -1)"