A straightforward method for training your LLM, from downloading data to generating text.
{
"pretrained_ckpt": "models/base_pretrained.pt",
"data_path": "data/sft_packed.h5",
"dev_path": "data/sft_dev_packed.h5",
"out_ckpt": "models/sft.pt",
"batch_size": 16,
"grad_accum": 2,
"epochs": 3,
"max_steps": -1,
"eval_steps": 200,
"warmup_steps": 100,
"lr": 1e-05,
"min_lr": 1e-06,
"weight_decay": 0.0,
"grad_clip": 1.0,
"save_every": 500,
"lora_rank": 0,
"lora_alpha": 16.0,
"lora_dropout": 0.0
}