Skip to content

A straightforward method for training your LLM, from downloading data to generating text.

399 B
Raw Blame
{
  "train_path": "data/pile_train.h5",
  "dev_path": "data/pile_dev.h5",
  "batch_size": 8,
  "grad_accum": 12,
  "train_steps": 50000,
  "eval_steps": 1000,
  "eval_iters": 100,
  "warmup_steps": 2000,
  "optimizer": "adamw",
  "lr_schedule": "cosine",
  "lr": 0.0003,
  "min_lr": 3e-05,
  "weight_decay": 0.1,
  "grad_clip": 1.0,
  "out_ckpt": "models/base_pretrained.pt",
  "save_every": 1000
}