Skip to content

Official Codebase of 3DCodeVerse

Python

finetune: add scripts/sync_repo.sh — one-command mirror of the working tree into the repo (code+docs only, credential guard)

gaoypengeee6583fb8c7
diff --git a/finetune/data_specs/dataset_info.json b/finetune/data_specs/dataset_info.json
index e1200b10..5493670c 100644
--- a/finetune/data_specs/dataset_info.json
+++ b/finetune/data_specs/dataset_info.json
@@ -432,5 +432,80 @@
"messages": "conversations",
"system": "system"
}
+ },
+ "md_27b_v2_train": {
+ "file_name": "md_27b_v2_train.json",
+ "formatting": "sharegpt",
+ "columns": {
+ "messages": "conversations",
+ "system": "system"
+ }
+ },
+ "md_27b_v2_val": {
+ "file_name": "md_27b_v2_val.json",
+ "formatting": "sharegpt",
+ "columns": {
+ "messages": "conversations",
+ "system": "system"
+ }
+ },
+ "md_v3_train": {
+ "file_name": "md_v3_train.json",
+ "formatting": "sharegpt",
+ "columns": {
+ "messages": "conversations",
+ "system": "system"
+ }
+ },
+ "md_v3_val": {
+ "file_name": "md_v3_val.json",
+ "formatting": "sharegpt",
+ "columns": {
+ "messages": "conversations",
+ "system": "system"
+ }
+ },
+ "md_27b_v2s_train": {
+ "file_name": "md_27b_v2s_train.json",
+ "formatting": "sharegpt",
+ "columns": {
+ "messages": "conversations",
+ "system": "system"
+ }
+ },
+ "md_27b_v2s_val": {
+ "file_name": "md_27b_v2s_val.json",
+ "formatting": "sharegpt",
+ "columns": {
+ "messages": "conversations",
+ "system": "system"
+ }
+ },
+ "md_27b_v3_train": {
+ "file_name": "md_27b_v3_train.json",
+ "formatting": "sharegpt",
+ "columns": {
+ "messages": "conversations",
+ "system": "system"
+ }
+ },
+ "md_27b_v3_val": {
+ "file_name": "md_27b_v3_val.json",
+ "formatting": "sharegpt",
+ "columns": {
+ "messages": "conversations",
+ "system": "system"
+ }
+ },
+ "dpo_27b_pairs": {
+ "file_name": "dpo_27b_pairs.json",
+ "ranking": true,
+ "formatting": "sharegpt",
+ "columns": {
+ "messages": "conversations",
+ "system": "system",
+ "chosen": "chosen",
+ "rejected": "rejected"
+ }
}
}
\ No newline at end of file
diff --git a/finetune/scripts/sync_repo.sh b/finetune/scripts/sync_repo.sh
new file mode 100755
index 00000000..1bff0611
--- /dev/null
+++ b/finetune/scripts/sync_repo.sh
@@ -0,0 +1,51 @@
+#!/bin/bash
+# One command to mirror the working tree into the GitHub repo and push:
+# scripts/sync_repo.sh "commit message" (default message if omitted)
+#
+# What is synced (code + docs only):
+# scripts/ configs/ eval/ env.sh -> finetune/
+# REPORT.md -> finetune/docs/REPORT.md
+# README.md -> finetune/docs/PROJECT_README.md
+# llm_finetune_exps.md -> finetune/llm_finetune_exps.md
+# data/lf/dataset_info.json -> finetune/data_specs/dataset_info.json
+# data/md_max/report.json -> finetune/data_specs/md_max_report.json
+# pip freezes of the three envs -> finetune/env/requirements-*.txt (with --envs)
+# Never synced: data/, runs/, logs/, eval/out/, models/, secrets. Only finetune/ is ever touched.
+set -uo pipefail
+SRC=/wekafs/ict/hx_624/llm-ft
+REPO=/wekafs/ict/hx_624/repos/3dcodeverse
+DST=$REPO/finetune
+MSG=${1:-"finetune: sync scripts, configs, eval stack and docs"}
+
+[ -d "$DST" ] || { echo "repo clone missing at $REPO"; exit 1; }
+rsync -a --delete --exclude '__pycache__' --exclude '*.pyc' "$SRC/scripts/" "$DST/scripts/"
+rsync -a --exclude '__pycache__' "$SRC/configs/" "$DST/configs/"
+rsync -a --delete --exclude '__pycache__' --exclude 'out' --exclude '_test' --exclude 'ref_oracle' --exclude '*.pyc' "$SRC/eval/" "$DST/eval/"
+cp "$SRC/env.sh" "$DST/env.sh"
+cp "$SRC/REPORT.md" "$DST/docs/REPORT.md"
+cp "$SRC/README.md" "$DST/docs/PROJECT_README.md" 2>/dev/null || true
+cp "$SRC/llm_finetune_exps.md" "$DST/llm_finetune_exps.md" 2>/dev/null || true
+cp "$SRC/data/lf/dataset_info.json" "$DST/data_specs/dataset_info.json" 2>/dev/null || true
+cp "$SRC/data/md_max/report.json" "$DST/data_specs/md_max_report.json" 2>/dev/null || true
+if [ "${2:-}" = "--envs" ]; then
+ source /wekafs/ict/hx_624/anaconda3/etc/profile.d/conda.sh
+ for e in llmft lf vllm; do conda activate $e && pip freeze > "$DST/env/requirements-$e.txt"; done
+ conda deactivate
+fi
+
+# refuse to publish anything that looks like a credential
+PAT=$(printf '%s_%s|%s_%s|%s_%s' hf '[A-Za-z0-9]{30,}' ghp '[A-Za-z0-9]{30,}' github 'pat_[A-Za-z0-9_]{20,}') # assembled at runtime so this file cannot match itself
+if grep -rIlE "$PAT" "$DST" | head -1 | grep -q .; then
+ echo "ABORT: a token-like string is present under finetune/ — not committing"; exit 1
+fi
+
+cd "$REPO"
+git add -A -- finetune
+if git diff --cached --quiet; then echo "[sync] nothing changed"; exit 0; fi
+git commit -q -m "$MSG
+
+Co-Authored-By: Claude Fable 5 <[email protected]>
+Claude-Session: https://claude.ai/code/session_01EtURHD29bBnAeA3FkJNnci"
+export GIT_ASKPASS=/wekafs/ict/hx_624/.secrets/gh_askpass.sh GIT_TERMINAL_PROMPT=0
+git fetch -q origin main && git rebase -q origin/main
+git push -q origin main && echo "[sync] pushed: $(git log --oneline -1)"