Skip to content

A straightforward method for training your LLM, from downloading data to generating text.

fix: debugging info

Tianyi (Alex) Qiu481a64fb8677
diff --git a/PITFALLS.md b/PITFALLS.md
index 64c2507..efe394f 100644
--- a/PITFALLS.md
+++ b/PITFALLS.md
@@ -2,4 +2,5 @@
- `attn = torch.tril(attn, diagonal=0) + torch.triu(torch.ones_like(attn) * (-torch.inf), diagonal=1)`
- `k = torch.transpose(k, -1, -2)`
- `x += self.attn(self.ln_attn(x)); x += self.mlp(self.ln_mlp(x))`
-- `x = self.lm_head(self.ln_lmhead(x))`
\ No newline at end of file
+- `x = self.lm_head(self.ln_lmhead(x))`
+- Send model & all tensors to device
\ No newline at end of file
diff --git a/scripts/train_transformer.py b/scripts/train_transformer.py
index 2034417..3c0a179 100644
--- a/scripts/train_transformer.py
+++ b/scripts/train_transformer.py
@@ -70,7 +70,8 @@ optimizer = torch.optim.Adam(
config.T_LR,
)
-print(f"Parameter count: {sum(param.numel() for param in model_params if param.requires_grad)}")
+print(f"All parameter count: {sum(param.numel() for param in model_params)}")
+print(f"Trainable parameter count: {sum(param.numel() for param in model_params if param.requires_grad)}")
train_losses = []
eval_losses = []