A straightforward method for training your LLM, from downloading data to generating text.
feat(presets): named model sizes from a laptop CPU to the original 3B
FareedKhan-devcd97a4dcf80b
diff --git a/config/config.py b/config/config.pyindex 5e69eca..095e6cc 100644--- a/config/config.py+++ b/config/config.py@@ -1,6 +1,11 @@# --- Configuration ---+#+# The plain constants used by scripts/train_transformer.py. The defaults below describe the+# original ~3B parameter model. For smaller runs, use a named preset instead of editing this+# file, for example `python scripts/train_transformer.py --preset tiny` on a laptop CPU+# (all presets are in config/presets.py).-import torch+from src.device import resolve_device# Define vocabulary size and transformer configuration (3 Billion)VOCAB_SIZE = 50304 # Number of unique tokens in the vocabulary@@ -35,8 +40,8 @@ USE_GRADIENT_CHECKPOINTING = False # recompute block activations in backward toGRAD_ACCUM_STEPS = 1 # micro-batches per optimizer step (effective batch x N)REPORT_MEMORY_BUDGET = False # print a rough VRAM budget before training (CUDA only)-# Device configuration-DEVICE = 'cuda' if torch.cuda.is_available() else 'cpu'+# Device configuration: CUDA if available, then Apple Silicon (MPS), then the CPU+DEVICE = resolve_device("auto")# Store all configurations in a dictionary for easy access and modificationdefault_config = {diff --git a/config/presets.py b/config/presets.pynew file mode 100644index 0000000..d665a7d--- /dev/null+++ b/config/presets.py@@ -0,0 +1,90 @@+"""+Named sizes for ``scripts/train_transformer.py --preset NAME``.++Each preset overrides the matching constants of ``config/config.py`` (same key names), so a+preset is just "the config you would have typed by hand". Pick one by hardware:++========= =============================== =========================================+preset data meant for+========= =============================== =========================================+tiny data/tiny (TinyStories + BPE) any laptop CPU, a few minutes+student data/tiny a laptop CPU, about half an hour+small data/tiny a fast CPU or any GPU+13m data/train, data/val (the Pile) the 13M model from the README, one GPU+77m the Pile the README's 77M base, one GPU+3b the Pile the original default of config.py (big GPU)+========= =============================== =========================================++The CPU presets train on the TinyStories data made by ``scripts/prepare_tiny_data.py``. Its+small BPE vocabulary (4096 by default) is read from the data file at startup, so the vocab+size here is only the fallback for data without that information.++Every preset trains on windows as long as the model's context (``t_context_length ==+context_length``). The original config trains on 16-token windows, which leaves most of+the position embeddings untrained; see the docs for why that matters for generation.+"""++from __future__ import annotations++from typing import Any++TINY_TRAIN = "data/tiny/train.h5"+TINY_DEV = "data/tiny/val.h5"+PILE_TRAIN = "data/train/pile_train.h5"+PILE_DEV = "data/val/pile_dev.h5"+++def _cpu(n_embed: int, n_head: int, n_blocks: int, context: int, steps: int, batch: int, lr: float, name: str) -> dict[str, Any]:+ return {+ "vocab_size": 4096,+ "context_length": context,+ "n_embed": n_embed,+ "n_head": n_head,+ "n_blocks": n_blocks,+ "train_path": TINY_TRAIN,+ "dev_path": TINY_DEV,+ "t_batch_size": batch,+ "t_context_length": context,+ "t_train_steps": steps,+ "t_eval_steps": max(100, steps // 10),+ "t_eval_iters": 20,+ "t_lr": lr,+ "t_lr_decayed": lr / 10,+ "t_lr_decay_step": int(steps * 0.8), # constant, then a 10x drop for the last 20%+ "t_out_path": f"models/{name}.pt",+ "sample_prompt": "Once upon a time",+ }+++PRESETS: dict[str, dict[str, Any]] = {+ "tiny": _cpu(n_embed=64, n_head=4, n_blocks=2, context=128, steps=1500, batch=32, lr=2e-3, name="tiny"),+ "student": _cpu(n_embed=192, n_head=6, n_blocks=4, context=256, steps=4000, batch=32, lr=1e-3, name="student"),+ "small": _cpu(n_embed=384, n_head=6, n_blocks=6, context=256, steps=8000, batch=32, lr=6e-4, name="small"),+ # The GPU sizes used in the README, on the Pile data from scripts/data_preprocess.py.+ "13m": {+ "vocab_size": 50304, "context_length": 128, "n_embed": 128, "n_head": 8, "n_blocks": 1,+ "train_path": PILE_TRAIN, "dev_path": PILE_DEV, "t_batch_size": 64, "t_context_length": 128,+ "t_train_steps": 20000, "t_eval_steps": 1000, "t_eval_iters": 100, "t_lr": 5e-4,+ "t_lr_decayed": 5e-5, "t_lr_decay_step": 16000, "t_out_path": "models/13m.pt",+ },+ "77m": {+ "vocab_size": 50304, "context_length": 512, "n_embed": 512, "n_head": 8, "n_blocks": 8,+ "train_path": PILE_TRAIN, "dev_path": PILE_DEV, "t_batch_size": 24, "t_context_length": 512,+ "t_train_steps": 20000, "t_eval_steps": 1000, "t_eval_iters": 100, "t_lr": 3e-4,+ "t_lr_decayed": 3e-5, "t_lr_decay_step": 16000, "t_out_path": "models/77m.pt",+ },+ # The original default of config/config.py (about 3B parameters), kept for reference.+ "3b": {+ "vocab_size": 50304, "context_length": 512, "n_embed": 2048, "n_head": 16, "n_blocks": 64,+ "train_path": PILE_TRAIN, "dev_path": PILE_DEV, "t_batch_size": 32, "t_context_length": 16,+ "t_train_steps": 200000, "t_eval_steps": 1000, "t_eval_iters": 250, "t_lr": 5e-4,+ "t_lr_decayed": 5e-5, "t_lr_decay_step": 50000, "t_out_path": "models/transformer_B.pt",+ },+}+++def apply_preset(config: dict[str, Any], name: str) -> dict[str, Any]:+ """Return a copy of ``config`` with preset ``name`` applied."""+ if name not in PRESETS:+ raise ValueError(f"unknown preset {name!r}; choose one of {sorted(PRESETS)}")+ return {**config, **PRESETS[name]}