Skip to content

A straightforward method for training your LLM, from downloading data to generating text.

439 B
Raw Blame
{
  "pretrained_ckpt": "models/base_pretrained.pt",
  "data_path": "data/sft_packed.h5",
  "dev_path": "data/sft_dev_packed.h5",
  "out_ckpt": "models/sft.pt",
  "batch_size": 16,
  "grad_accum": 2,
  "epochs": 3,
  "max_steps": -1,
  "eval_steps": 200,
  "warmup_steps": 100,
  "lr": 1e-05,
  "min_lr": 1e-06,
  "weight_decay": 0.0,
  "grad_clip": 1.0,
  "save_every": 500,
  "lora_rank": 0,
  "lora_alpha": 16.0,
  "lora_dropout": 0.0
}