A straightforward method for training your LLM, from downloading data to generating text.
fix: moving tensors to cuda in evaluation
Tianyi (Alex) Qiu5b22ab73e437
diff --git a/scripts/train_transformer.py b/scripts/train_transformer.pyindex 61c7db3..ce89c01 100644--- a/scripts/train_transformer.py+++ b/scripts/train_transformer.py@@ -22,9 +22,6 @@ model = Transformer(device=device,).to(device)-if device.type != 'cpu':- model = model.cuda()-criterion = torch.nn.CrossEntropyLoss()def create_dataloader(split: Literal["train", "val"]) -> Iterator[Tuple[torch.Tensor, torch.Tensor]]:@@ -60,6 +57,8 @@ def evaluate_model(model: Transformer, num_batches: int = config.T_EVAL_ITERS, sprint("Evaluation start.")for _ in tqdm(list(range(num_batches))):x, y = next(dataloader)+ x = x.to(device)+ y = y.to(device)assert len(x.shape) == 2 and x.shape[0] == y.shape[0]losses.append(