Skip to content

A straightforward method for training your LLM, from downloading data to generating text.

574 B
Raw Blame
{
  "arch": "classic",
  "vocab_size": 50304,
  "context_length": 1024,
  "n_embed": 1024,
  "n_head": 16,
  "n_blocks": 24,
  "n_kv_head": null,
  "attention": "gqa",
  "kv_latent_dim": null,
  "rope_theta": 10000.0,
  "qk_norm": true,
  "attn_gate": false,
  "sliding_window": null,
  "tie_embeddings": true,
  "n_experts": 0,
  "moe_top_k": 2,
  "n_shared_experts": 0,
  "device": "auto",
  "amp_dtype": "bf16",
  "seed": 1337,
  "compile": false,
  "ckpt_dir": "models",
  "log_dir": "logs",
  "use_wandb": false,
  "wandb_project": "train-llm-from-scratch-posttrain"
}