A straightforward method for training your LLM, from downloading data to generating text.
{
"train_path": "data/pile_train.h5",
"dev_path": "data/pile_dev.h5",
"batch_size": 8,
"grad_accum": 12,
"train_steps": 50000,
"eval_steps": 1000,
"eval_iters": 100,
"warmup_steps": 2000,
"optimizer": "adamw",
"lr_schedule": "cosine",
"lr": 0.0003,
"min_lr": 3e-05,
"weight_decay": 0.1,
"grad_clip": 1.0,
"out_ckpt": "models/base_pretrained.pt",
"save_every": 1000
}