A straightforward method for training your LLM, from downloading data to generating text.
{
"arch": "classic",
"vocab_size": 50304,
"context_length": 1024,
"n_embed": 1024,
"n_head": 16,
"n_blocks": 24,
"n_kv_head": null,
"attention": "gqa",
"kv_latent_dim": null,
"rope_theta": 10000.0,
"qk_norm": true,
"attn_gate": false,
"sliding_window": null,
"tie_embeddings": true,
"n_experts": 0,
"moe_top_k": 2,
"n_shared_experts": 0,
"device": "auto",
"amp_dtype": "bf16",
"seed": 1337,
"compile": false,
"ckpt_dir": "models",
"log_dir": "logs",
"use_wandb": false,
"wandb_project": "train-llm-from-scratch-posttrain"
}