A straightforward method for training your LLM, from downloading data to generating text.
refactor(paths): keep data, checkpoints and logs in the repo, not /ephemeral
FareedKhan-devd62b5f221968
diff --git a/config/paths.py b/config/paths.pynew file mode 100644index 0000000..3559a8d--- /dev/null+++ b/config/paths.py@@ -0,0 +1,18 @@+"""+Default locations of everything that is too big for git.++All paths are relative to the repository root, which is where every command in the README+is run from:++ data/ datasets (HDF5 token streams and JSONL files)+ models/ checkpoints of every stage+ logs/ metrics (one JSONL file per run)++To keep them on another disk, point the paths in ``configs/*.json`` somewhere else+(``$VARS`` are expanded, e.g. ``"$SCRATCH/models/sft.pt"``), pass ``--out_ckpt`` and friends+on the command line, or simply make ``data``/``models`` symlinks.+"""++DATA_DIR = "data"+CKPT_DIR = "models"+LOG_DIR = "logs"diff --git a/config/post_training_config.py b/config/post_training_config.pyindex a0f83f0..7b655fd 100644--- a/config/post_training_config.py+++ b/config/post_training_config.py@@ -16,12 +16,7 @@ from __future__ import annotationsfrom dataclasses import dataclass, field, replace--# Shared paths (all heavy artifacts live on the 1.5TB /ephemeral disk).-EPHEMERAL = "/ephemeral"-CKPT_DIR = f"{EPHEMERAL}/ckpts"-DATA_DIR = f"{EPHEMERAL}/data"-LOG_DIR = f"{EPHEMERAL}/logs"+from config.paths import CKPT_DIR, DATA_DIR, LOG_DIR@dataclass@@ -34,7 +29,7 @@ class BaseModelConfig:n_blocks: int = 24# --- runtime ---- device: str = "cuda"+ device: str = "auto" # auto | cuda | mps | cpuamp_dtype: str | None = "bf16" # None | "bf16"; bf16 needs no GradScaler on H100seed: int = 1337compile: bool = False # torch.compile the model (big speedup, slow 1st step)@@ -47,8 +42,8 @@ class BaseModelConfig:@dataclassclass PretrainConfig(BaseModelConfig):"""Pretrain the mid base model from scratch on the Pile HDF5 (mix in task text late)."""- train_path: str = "data/train/pile_train.h5"- dev_path: str = "data/val/pile_dev.h5"+ train_path: str = f"{DATA_DIR}/pile_train.h5"+ dev_path: str = f"{DATA_DIR}/pile_dev.h5"batch_size: int = 24 # per-GPU micro-batchgrad_accum: int = 8 # effective batch = batch_size * grad_accum * worldtrain_steps: int = 200_000@@ -67,6 +62,7 @@ class PretrainConfig(BaseModelConfig):class SFTConfig(BaseModelConfig):pretrained_ckpt: str = f"{CKPT_DIR}/base_pretrained.pt"data_path: str = f"{DATA_DIR}/sft_packed.h5"+ dev_path: str = f"{DATA_DIR}/sft_dev_packed.h5"out_ckpt: str = f"{CKPT_DIR}/sft.pt"batch_size: int = 16grad_accum: int = 2@@ -85,6 +81,7 @@ class SFTConfig(BaseModelConfig):class RewardConfig(BaseModelConfig):sft_ckpt: str = f"{CKPT_DIR}/sft.pt"pref_path: str = f"{DATA_DIR}/preferences.jsonl"+ test_path: str = f"{DATA_DIR}/preferences_test.jsonl"out_ckpt: str = f"{CKPT_DIR}/reward.pt"batch_size: int = 8 # pairs per step (2x sequences through the model)epochs: int = 1@@ -101,6 +98,7 @@ class RewardConfig(BaseModelConfig):class DPOConfig(BaseModelConfig):sft_ckpt: str = f"{CKPT_DIR}/sft.pt" # init policy + frozen referencepref_path: str = f"{DATA_DIR}/preferences.jsonl"+ test_path: str = f"{DATA_DIR}/preferences_test.jsonl"out_ckpt: str = f"{CKPT_DIR}/dpo.pt"loss_type: str = "dpo" # "dpo" | "orpo" | "kto"beta: float = 0.1diff --git a/configs/base.json b/configs/base.jsonindex b9730b9..1aeadd0 100644--- a/configs/base.json+++ b/configs/base.json@@ -4,12 +4,12 @@"n_embed": 1024,"n_head": 16,"n_blocks": 24,- "device": "cuda",+ "device": "auto","amp_dtype": "bf16","seed": 1337,"compile": false,- "ckpt_dir": "/ephemeral/ckpts",- "log_dir": "/ephemeral/logs",+ "ckpt_dir": "models",+ "log_dir": "logs","use_wandb": false,"wandb_project": "train-llm-from-scratch-posttrain"}diff --git a/configs/dpo.json b/configs/dpo.jsonindex 3b31b16..d9e32a8 100644--- a/configs/dpo.json+++ b/configs/dpo.json@@ -1,7 +1,8 @@{- "sft_ckpt": "/ephemeral/ckpts/sft.pt",- "pref_path": "/ephemeral/data/preferences.jsonl",- "out_ckpt": "/ephemeral/ckpts/dpo.pt",+ "sft_ckpt": "models/sft.pt",+ "pref_path": "data/preferences.jsonl",+ "test_path": "data/preferences_test.jsonl",+ "out_ckpt": "models/dpo.pt","loss_type": "dpo","beta": 0.1,"orpo_lambda": 1.0,diff --git a/configs/grpo.json b/configs/grpo.jsonindex 6da1a66..b3691c9 100644--- a/configs/grpo.json+++ b/configs/grpo.json@@ -1,10 +1,10 @@{- "sft_ckpt": "/ephemeral/ckpts/sft.pt",- "prompt_path": "/ephemeral/data/rl_prompts_train.jsonl",- "eval_prompt_path": "/ephemeral/data/rl_prompts_test.jsonl",- "curriculum_path": "/ephemeral/data/arithmetic_prompts.jsonl",+ "sft_ckpt": "models/sft.pt",+ "prompt_path": "data/rl_prompts_train.jsonl",+ "eval_prompt_path": "data/rl_prompts_test.jsonl",+ "curriculum_path": "data/arithmetic_prompts.jsonl","curriculum_iters": 100,- "out_ckpt": "/ephemeral/ckpts/grpo.pt",+ "out_ckpt": "models/grpo.pt","iterations": 1000,"prompts_per_iter": 8,"group_size": 8,diff --git a/configs/ppo.json b/configs/ppo.jsonindex 5c45b4f..fbc9871 100644--- a/configs/ppo.json+++ b/configs/ppo.json@@ -1,9 +1,9 @@{- "sft_ckpt": "/ephemeral/ckpts/sft.pt",- "reward_ckpt": "/ephemeral/ckpts/reward.pt",- "prompt_path": "/ephemeral/data/rl_prompts_train.jsonl",- "eval_prompt_path": "/ephemeral/data/rl_prompts_test.jsonl",- "out_ckpt": "/ephemeral/ckpts/ppo.pt",+ "sft_ckpt": "models/sft.pt",+ "reward_ckpt": "models/reward.pt",+ "prompt_path": "data/rl_prompts_train.jsonl",+ "eval_prompt_path": "data/rl_prompts_test.jsonl",+ "out_ckpt": "models/ppo.pt","reward_source": "verifier","iterations": 1000,"prompts_per_iter": 32,diff --git a/configs/pretrain.json b/configs/pretrain.jsonindex 6b7f077..15a1010 100644--- a/configs/pretrain.json+++ b/configs/pretrain.json@@ -1,6 +1,6 @@{- "train_path": "/ephemeral/data/pile_train.h5",- "dev_path": "/ephemeral/data/pile_dev.h5",+ "train_path": "data/pile_train.h5",+ "dev_path": "data/pile_dev.h5","batch_size": 8,"grad_accum": 12,"train_steps": 50000,@@ -11,6 +11,6 @@"min_lr": 3e-05,"weight_decay": 0.1,"grad_clip": 1.0,- "out_ckpt": "/ephemeral/ckpts/base_pretrained.pt",+ "out_ckpt": "models/base_pretrained.pt","save_every": 1000}diff --git a/configs/reward.json b/configs/reward.jsonindex 4037732..400c200 100644--- a/configs/reward.json+++ b/configs/reward.json@@ -1,7 +1,8 @@{- "sft_ckpt": "/ephemeral/ckpts/sft.pt",- "pref_path": "/ephemeral/data/preferences.jsonl",- "out_ckpt": "/ephemeral/ckpts/reward.pt",+ "sft_ckpt": "models/sft.pt",+ "pref_path": "data/preferences.jsonl",+ "test_path": "data/preferences_test.jsonl",+ "out_ckpt": "models/reward.pt","batch_size": 8,"epochs": 1,"eval_steps": 200,diff --git a/configs/sft.json b/configs/sft.jsonindex b59a839..374a208 100644--- a/configs/sft.json+++ b/configs/sft.json@@ -1,7 +1,8 @@{- "pretrained_ckpt": "/ephemeral/ckpts/base_pretrained.pt",- "data_path": "/ephemeral/data/sft_packed.h5",- "out_ckpt": "/ephemeral/ckpts/sft.pt",+ "pretrained_ckpt": "models/base_pretrained.pt",+ "data_path": "data/sft_packed.h5",+ "dev_path": "data/sft_dev_packed.h5",+ "out_ckpt": "models/sft.pt","batch_size": 16,"grad_accum": 2,"epochs": 3,diff --git a/configs/smoke/base.json b/configs/smoke/base.jsonindex 19f90c8..8303146 100644--- a/configs/smoke/base.json+++ b/configs/smoke/base.json@@ -5,5 +5,7 @@"n_head": 4,"n_blocks": 2,"device": "cpu",- "amp_dtype": null+ "amp_dtype": null,+ "ckpt_dir": "models/smoke",+ "log_dir": "logs/smoke"}diff --git a/configs/smoke/dpo.json b/configs/smoke/dpo.jsonindex 02a571c..c65ecbb 100644--- a/configs/smoke/dpo.json+++ b/configs/smoke/dpo.json@@ -1,4 +1,6 @@{+ "sft_ckpt": "models/smoke/sft.pt",+ "out_ckpt": "models/smoke/dpo.pt","batch_size": 4,"epochs": 1,"eval_steps": 10000,diff --git a/configs/smoke/grpo.json b/configs/smoke/grpo.jsonindex 8cce3d6..60344ce 100644--- a/configs/smoke/grpo.json+++ b/configs/smoke/grpo.json@@ -1,4 +1,6 @@{+ "sft_ckpt": "models/smoke/sft.pt",+ "out_ckpt": "models/smoke/grpo.pt","iterations": 2,"prompts_per_iter": 2,"group_size": 4,diff --git a/configs/smoke/ppo.json b/configs/smoke/ppo.jsonindex 5d07eb8..dbaf024 100644--- a/configs/smoke/ppo.json+++ b/configs/smoke/ppo.json@@ -1,4 +1,7 @@{+ "sft_ckpt": "models/smoke/sft.pt",+ "reward_ckpt": "models/smoke/reward.pt",+ "out_ckpt": "models/smoke/ppo.pt","iterations": 2,"prompts_per_iter": 4,"rollout_len": 32,diff --git a/configs/smoke/pretrain.json b/configs/smoke/pretrain.jsonindex ad122b0..1a912b4 100644--- a/configs/smoke/pretrain.json+++ b/configs/smoke/pretrain.json@@ -1,10 +1,11 @@{+ "train_path": "data/pile_train.h5",+ "dev_path": "data/pile_dev.h5",+ "out_ckpt": "models/smoke/base_pretrained.pt","batch_size": 4,"grad_accum": 1,"train_steps": 20,"eval_steps": 10000,"warmup_steps": 2,- "save_every": 10000,- "train_path": "/ephemeral/data/pile_train.h5",- "dev_path": "/ephemeral/data/pile_dev.h5"+ "save_every": 10000}diff --git a/configs/smoke/reward.json b/configs/smoke/reward.jsonindex 02a571c..fd820fc 100644--- a/configs/smoke/reward.json+++ b/configs/smoke/reward.json@@ -1,4 +1,6 @@{+ "sft_ckpt": "models/smoke/sft.pt",+ "out_ckpt": "models/smoke/reward.pt","batch_size": 4,"epochs": 1,"eval_steps": 10000,diff --git a/configs/smoke/sft.json b/configs/smoke/sft.jsonindex 832fc5d..d2147af 100644--- a/configs/smoke/sft.json+++ b/configs/smoke/sft.json@@ -1,4 +1,6 @@{+ "pretrained_ckpt": "models/smoke/base_pretrained.pt",+ "out_ckpt": "models/smoke/sft.pt","max_steps": 10,"batch_size": 4,"eval_steps": 10000,diff --git a/scripts/chat.py b/scripts/chat.pyindex 1331990..9a1f66a 100644--- a/scripts/chat.py+++ b/scripts/chat.py@@ -5,11 +5,11 @@ Model dimensions are read from the checkpoint, so you only pass the path. Use thtemplate for instruction-tuned models, or --raw for base-model continuation.One-shot:- python scripts/chat.py --ckpt /ephemeral/ckpts/sft.pt --prompt "What is 13 + 29?"- python scripts/chat.py --ckpt /ephemeral/ckpts/grpo.pt --prompt "..." --greedy- python scripts/chat.py --ckpt /ephemeral/ckpts/base_pretrained.pt --raw --prompt "Once upon a time"+ python scripts/chat.py --ckpt models/sft.pt --prompt "What is 13 + 29?"+ python scripts/chat.py --ckpt models/grpo.pt --prompt "..." --greedy+ python scripts/chat.py --ckpt models/base_pretrained.pt --raw --prompt "Once upon a time"Interactive REPL (no --prompt):- python scripts/chat.py --ckpt /ephemeral/ckpts/sft.pt+ python scripts/chat.py --ckpt models/sft.pt"""from __future__ import annotationsdiff --git a/scripts/eval_post_training.py b/scripts/eval_post_training.pyindex 959c677..7af68c1 100644--- a/scripts/eval_post_training.py+++ b/scripts/eval_post_training.py@@ -3,10 +3,10 @@ Evaluate any stage checkpoint on GSM8K (greedy) and optionally dump sample generUse it to build the headline "GSM8K accuracy across stages" table:for s in base_pretrained sft dpo ppo grpo; do- python scripts/eval_post_training.py --ckpt /ephemeral/ckpts/$s.pt \- --label $s --limit 200 --append /ephemeral/logs/stage_table.jsonl+ python scripts/eval_post_training.py --ckpt models/$s.pt \+ --label $s --limit 200 --append logs/stage_table.jsonldone- python scripts/eval_post_training.py --table /ephemeral/logs/stage_table.jsonl+ python scripts/eval_post_training.py --table logs/stage_table.jsonlModel dimensions are read from the checkpoint's stored ``cfg`` so you don't have to repeatthem. Reward checkpoints (which have a reward head, not an LM head only) still load becausediff --git a/scripts/prepare_preference_data.py b/scripts/prepare_preference_data.pyindex 3bd223e..466ff8a 100644--- a/scripts/prepare_preference_data.py+++ b/scripts/prepare_preference_data.py@@ -7,8 +7,8 @@ Emits JSONL of ``{"prompt", "chosen", "rejected"}`` (train + held-out test). Thetest split is what reward-model preference accuracy is measured on.Example:- HF_HOME=/ephemeral/hf_cache python scripts/prepare_preference_data.py \- --source both --max_per_source 40000 --out_dir /ephemeral/data+ python scripts/prepare_preference_data.py \+ --source both --max_per_source 40000 --out_dir data"""from __future__ import annotations@@ -22,7 +22,6 @@ import argparseimport jsonimport os-os.environ.setdefault("HF_HOME", "/ephemeral/hf_cache")_ASSISTANT_MARKER = "\n\nAssistant:"@@ -102,7 +101,7 @@ def main():p = argparse.ArgumentParser()p.add_argument("--source", choices=["hh", "ultrafeedback", "both"], default="both")p.add_argument("--max_per_source", type=int, default=40000)- p.add_argument("--out_dir", default="/ephemeral/data")+ p.add_argument("--out_dir", default="data")args = p.parse_args()train = collect(args.source, args.max_per_source, "train")diff --git a/scripts/prepare_pretrain_data.py b/scripts/prepare_pretrain_data.pyindex 4ace471..940ff9c 100644--- a/scripts/prepare_pretrain_data.py+++ b/scripts/prepare_pretrain_data.py@@ -4,15 +4,15 @@ pretraining. Faster than the original ``data_preprocess.py`` (which resizes thedocument): here we stream-decompress, batch-tokenize with tiktoken, and write tokens tothe HDF5 in large chunks.-Writes to /ephemeral by default (the 1.5TB disk).+Writes under data/ by default (pass --out / --raw_dir to use another disk).Examples:# dev split from the Pile validation filepython scripts/prepare_pretrain_data.py --split val \- --out /ephemeral/data/pile_dev.h5+ --out data/pile_dev.h5# one training shardpython scripts/prepare_pretrain_data.py --split train --num_shards 1 \- --out /ephemeral/data/pile_train.h5+ --out data/pile_train.h5"""from __future__ import annotations@@ -122,7 +122,7 @@ def main():p = argparse.ArgumentParser()p.add_argument("--split", choices=["train", "val"], required=True)p.add_argument("--num_shards", type=int, default=1, help="train shards to use")- p.add_argument("--raw_dir", default="/ephemeral/data/pile_raw")+ p.add_argument("--raw_dir", default="data/pile_raw")p.add_argument("--out", required=True)p.add_argument("--max_tokens", type=int, default=None, help="stop after this many tokens")args = p.parse_args()diff --git a/scripts/prepare_rl_prompts.py b/scripts/prepare_rl_prompts.pyindex 99e8c4d..66746ed 100644--- a/scripts/prepare_rl_prompts.py+++ b/scripts/prepare_rl_prompts.py@@ -6,7 +6,7 @@ Build RL prompt sets ({"prompt", "gold"}) for PPO/GRPO:where even a weak model gets some non-zero reward so RL has signal to start from)Example:- HF_HOME=/ephemeral/hf_cache python scripts/prepare_rl_prompts.py --out_dir /ephemeral/data+ python scripts/prepare_rl_prompts.py --out_dir data"""from __future__ import annotations@@ -21,7 +21,6 @@ import jsonimport osimport random-os.environ.setdefault("HF_HOME", "/ephemeral/hf_cache")from src.post_training.rewards import gsm8k_gold_answer@@ -61,7 +60,7 @@ def arithmetic_prompts(n: int, max_val: int, seed: int):def main():p = argparse.ArgumentParser()- p.add_argument("--out_dir", default="/ephemeral/data")+ p.add_argument("--out_dir", default="data")p.add_argument("--train_limit", type=int, default=None)p.add_argument("--test_limit", type=int, default=500)p.add_argument("--arith_n", type=int, default=5000)diff --git a/scripts/prepare_sft_data.py b/scripts/prepare_sft_data.pyindex 59474e3..8e42fc7 100644--- a/scripts/prepare_sft_data.py+++ b/scripts/prepare_sft_data.py@@ -3,15 +3,15 @@ Build the SFT dataset from real public instruction data + GSM8K, render it throuchat template (masking prompt tokens), pack to fixed-length rows, and write a packedHDF5 (``tokens`` + ``loss_mask``) plus a held-out dev file.-Datasets (downloaded via HuggingFace ``datasets`` to /ephemeral/hf_cache):+Datasets (downloaded with the HuggingFace ``datasets`` library; set HF_HOME to move its cache):- tatsu-lab/alpaca (general instruction following)- databricks/databricks-dolly-15k (general instruction following)- openai/gsm8k (main, train) (math, reformatted into <think>/<answer> so the modellearns the reasoning format the RL verifier rewards)Example:- HF_HOME=/ephemeral/hf_cache python scripts/prepare_sft_data.py \- --context_length 1024 --out_dir /ephemeral/data+ python scripts/prepare_sft_data.py \+ --context_length 1024 --out_dir data"""from __future__ import annotations@@ -31,7 +31,6 @@ import numpy as npfrom src.post_training.chat_template import encode_chat, ANSWER_OPEN, ANSWER_CLOSE, THINK_OPEN, THINK_CLOSEfrom src.post_training.sft import pack_examples-os.environ.setdefault("HF_HOME", "/ephemeral/hf_cache")_CALC_RE = re.compile(r"<<[^>]*>>") # GSM8K calculator annotations_HASH_RE = re.compile(r"####\s*(.+)\s*$")@@ -112,7 +111,7 @@ def write_packed(examples, context_length: int, out_path: str) -> int:def main():p = argparse.ArgumentParser()p.add_argument("--context_length", type=int, default=1024)- p.add_argument("--out_dir", default="/ephemeral/data")+ p.add_argument("--out_dir", default="data")p.add_argument("--dev_frac", type=float, default=0.02)p.add_argument("--limit_per_set", type=int, default=None, help="cap examples per dataset (debug)")p.add_argument("--seed", type=int, default=42)diff --git a/scripts/train_dpo.py b/scripts/train_dpo.pyindex fc6aa0f..9d4f35d 100644--- a/scripts/train_dpo.py+++ b/scripts/train_dpo.py@@ -16,6 +16,7 @@ from pathlib import Pathsys.path.insert(0, str(Path(__file__).resolve().parents[1])) # run from the repo without installing+import osimport timeimport torch@@ -32,8 +33,6 @@ from src.post_training.utils import (amp_autocast, load_backbone_from_ckpt, make_frozen_copy, save_stage_ckpt, set_seed, unwrap,)-TEST_PATH = "/ephemeral/data/preferences_test.jsonl"-def _logps(model, ids, mask, requires_grad):return sequence_logprobs(model, ids, mask, requires_grad=requires_grad)@@ -61,8 +60,10 @@ def _compute_losses(policy, ref, batch, cfg, ctx):@torch.no_grad()def eval_implicit_acc(policy, ref, cfg, ctx, max_batches: int = 100) -> tuple[float, float]:+ if not os.path.exists(cfg.test_path):+ return float("nan"), float("nan")policy.eval()- it = get_preference_iterator(TEST_PATH, cfg.batch_size, cfg.max_len, device=ctx.device,+ it = get_preference_iterator(cfg.test_path, cfg.batch_size, cfg.max_len, device=ctx.device,rank=ctx.rank, world_size=ctx.world_size, shuffle=False, infinite=False)acc, marg, n = 0.0, 0.0, 0for batch in it:@@ -86,7 +87,7 @@ def main():policy = ddp_wrap(policy, ctx)optimizer = configure_optimizer(unwrap(policy), cfg.lr, cfg.weight_decay)- with open(cfg.pref_path) as f:+ with open(cfg.pref_path, encoding="utf-8") as f:n_rows = sum(1 for line in f if line.strip())total_steps = max(1, (n_rows // (cfg.batch_size * ctx.world_size)) * cfg.epochs)diff --git a/scripts/train_reward.py b/scripts/train_reward.pyindex a9191df..d4665a3 100644--- a/scripts/train_reward.py+++ b/scripts/train_reward.py@@ -17,6 +17,7 @@ from pathlib import Pathsys.path.insert(0, str(Path(__file__).resolve().parents[1])) # run from the repo without installing+import osimport timeimport torch@@ -31,9 +32,6 @@ from src.post_training.reward_model import RewardModelfrom src.post_training.reward_train import bradley_terry_loss, preference_accuracy, reward_marginfrom src.post_training.utils import amp_autocast, load_backbone_from_ckpt, save_stage_ckpt, set_seed, unwrap-TEST_PATH = "/ephemeral/data/preferences_test.jsonl"--def _pair_rewards(rm, batch, cfg, ctx):"""Forward chosen+rejected in one pass; return (chosen_rewards, rejected_rewards)."""B = batch["chosen_ids"].size(0)@@ -46,8 +44,10 @@ def _pair_rewards(rm, batch, cfg, ctx):@torch.no_grad()def eval_accuracy(rm, cfg, ctx, max_batches: int = 100) -> tuple[float, float]:+ if not os.path.exists(cfg.test_path):+ return float("nan"), float("nan")rm.eval()- it = get_preference_iterator(TEST_PATH, cfg.batch_size, cfg.max_len, device=ctx.device,+ it = get_preference_iterator(cfg.test_path, cfg.batch_size, cfg.max_len, device=ctx.device,rank=ctx.rank, world_size=ctx.world_size, shuffle=False, infinite=False)acc, marg, n = 0.0, 0.0, 0for batch in it:@@ -74,8 +74,7 @@ def main():rm = ddp_wrap(rm, ctx, find_unused_parameters=True)optimizer = configure_optimizer(unwrap(rm), cfg.lr, cfg.weight_decay)- import json- with open(cfg.pref_path) as f:+ with open(cfg.pref_path, encoding="utf-8") as f:n_rows = sum(1 for line in f if line.strip())total_steps = max(1, (n_rows // (cfg.batch_size * ctx.world_size)) * cfg.epochs)diff --git a/scripts/train_sft.py b/scripts/train_sft.pyindex 0649fef..5b351a6 100644--- a/scripts/train_sft.py+++ b/scripts/train_sft.py@@ -19,6 +19,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[1])) # run from the repimport contextlibimport math+import osimport timeimport torch@@ -32,11 +33,11 @@ from src.post_training.optim import configure_optimizer, cosine_lrfrom src.post_training.sft import sft_lossfrom src.post_training.utils import amp_autocast, load_backbone_from_ckpt, save_stage_ckpt, set_seed, unwrap-DEV_PATH = "/ephemeral/data/sft_dev_packed.h5"-@torch.no_grad()def eval_dev(model, cfg, ctx, dev_path: str, max_batches: int = 50) -> float:+ if not os.path.exists(dev_path):+ return float("nan")model.eval()it = get_sft_batch_iterator(dev_path, cfg.batch_size, device=ctx.device,rank=ctx.rank, world_size=ctx.world_size, shuffle=False, infinite=False)@@ -105,7 +106,7 @@ def main():logger.log(step, {"train_loss": loss.item(), "lr": lr})if step > 0 and step % cfg.eval_steps == 0:- dev = reduce_scalar(eval_dev(model, cfg, ctx, DEV_PATH), ctx)+ dev = reduce_scalar(eval_dev(model, cfg, ctx, cfg.dev_path), ctx)if ctx.is_main:print(f" [eval] step {step} | dev_loss {dev:.4f} | dev_ppl {math.exp(min(20, dev)):.2f}")if logger:@@ -119,7 +120,7 @@ def main():# Use the unwrapped model for the final eval: the other ranks have already reached# cleanup(), so calling the DDP-wrapped model here would launch a collective with no# peer and hang (NCCL timeout). The periodic eval above runs on all ranks, so it is fine.- dev = eval_dev(unwrap(model), cfg, ctx, DEV_PATH)+ dev = eval_dev(unwrap(model), cfg, ctx, cfg.dev_path)save_stage_ckpt(cfg.out_ckpt, model, optimizer, stage="sft", cfg=cfg, step=total_steps,metrics={"dev_loss": dev})print(f"Done SFT. dev_loss {dev:.4f} -> {cfg.out_ckpt}")diff --git a/src/post_training/evaluation.py b/src/post_training/evaluation.pyindex c0527d1..cf1f3e6 100644--- a/src/post_training/evaluation.py+++ b/src/post_training/evaluation.py@@ -106,8 +106,6 @@ def gsm8k_accuracy(def load_gsm8k_eval(split: str = "test", limit: int | None = 200) -> list[tuple[str, str]]:"""Load ``(question, answer_field)`` pairs from GSM8K for evaluation."""- import os- os.environ.setdefault("HF_HOME", "/ephemeral/hf_cache")from datasets import load_datasetds = load_dataset("openai/gsm8k", "main", split=split)