Skip to content

A straightforward method for training your LLM, from downloading data to generating text.

Improve modern GPU compatibility and document RTX 5090 validation

吴佳浩dfe8db05b1f5
diff --git a/README.md b/README.md
index c3ccd4e..2292fd5 100644
--- a/README.md
+++ b/README.md
@@ -96,6 +96,7 @@ You will need a GPU to train your model. Colab or Kaggle T4 will work for traini
| NVIDIA RTX 4070 Ti | 12 GB | Medium | ✘ | ✔ | ~1.5B |
| NVIDIA RTX 4080 | 16 GB | Medium | ✘ | ✔ | ~2B |
| NVIDIA RTX 4090 | 24 GB | Large | ✔ | ✔ | ~4B |
+| NVIDIA RTX 5090 | 32 GB* | Large | ✔* | ✔ | 13M verified; larger configs TBD* |
| NVIDIA RTX 4060 Ti | 8 GB | Small | ✘ | ✔ | ~1B |
| NVIDIA RTX 4060 | 8 GB | Small | ✘ | ✔ | ~1B |
| NVIDIA RTX 4050 | 6 GB | Small | ✘ | ✔ | ~0.75B |
@@ -112,6 +113,21 @@ You will need a GPU to train your model. Colab or Kaggle T4 will work for traini
The 13M LLM training is the training of a 13+ million-parameter model, and the 2B LLM training is the training of a 2+ billion-parameter model. The data size is categorized as small, medium, and large. The small data size is around 1 GB, the medium data size is around 5 GB, and the large data size is around 10 GB.
+* Community validation on an NVIDIA GeForce RTX 5090 has confirmed that the official README flow works for the 13M configuration using PyTorch 2.11.0+cu128 and CUDA 12.8. In one successful run, the 13M model trained to completion, wrote a checkpoint, and the generated checkpoint was successfully loaded for text generation. Practical limits for larger configurations still depend on training dtype, batch size, context length, optimizer state, and any memory-saving techniques used.
+
+### Community-tested modern GPU notes
+
+A community run on an NVIDIA GeForce RTX 5090 validated the following:
+- official dataset download and preprocessing flow completed successfully
+- the README-recommended 13M configuration trained successfully to completion
+- checkpoint saving and text generation both worked
+- PyTorch version: 2.11.0+cu128
+- CUDA version: 12.8
+- PyTorch-reported peak VRAM during 13M training stayed around 0.60 GiB allocated / 0.67 GiB reserved
+- observed training throughput during eval intervals was typically in the tens of thousands of tokens/sec on this small config
+
+The training script now prints lightweight runtime diagnostics to make these reports easier to collect. Newer PyTorch versions may also require explicit checkpoint loading compatibility handling during generation.
+
## Code Structure
The codebase is organized as follows:
diff --git a/config/config.py b/config/config.py
index ae4bf50..015b731 100644
--- a/config/config.py
+++ b/config/config.py
@@ -1,5 +1,7 @@
# --- Configuration ---
+import torch
+
# Define vocabulary size and transformer configuration (3 Billion)
VOCAB_SIZE = 50304 # Number of unique tokens in the vocabulary
CONTEXT_LENGTH = 512 # Maximum sequence length for the model
@@ -23,7 +25,7 @@ T_LR_DECAYED = 5e-5 # Learning rate after decay
T_OUT_PATH = "models/transformer_B.pt" # Path to save the trained model
# Device configuration
-DEVICE = 'cuda'
+DEVICE = 'cuda' if torch.cuda.is_available() else 'cpu'
# Store all configurations in a dictionary for easy access and modification
default_config = {
@@ -44,4 +46,4 @@ default_config = {
't_lr_decayed': T_LR_DECAYED,
't_out_path': T_OUT_PATH,
'device': DEVICE,
-}
\ No newline at end of file
+}
diff --git a/scripts/generate_text.py b/scripts/generate_text.py
index f02328e..ce7a4aa 100644
--- a/scripts/generate_text.py
+++ b/scripts/generate_text.py
@@ -4,6 +4,13 @@ import argparse
from config.config import default_config as config
from src.models.transformer import Transformer # Assuming your Transformer class is in this module
+
+def load_checkpoint(model_path: str, device: str):
+ try:
+ return torch.load(model_path, map_location=torch.device(device), weights_only=False)
+ except TypeError:
+ return torch.load(model_path, map_location=torch.device(device))
+
def generate_text(model_path: str, input_text: str, max_new_tokens: int = 100, device: str = 'cuda') -> str:
"""
Generates text using a pre-trained Transformer model.
@@ -18,7 +25,7 @@ def generate_text(model_path: str, input_text: str, max_new_tokens: int = 100, d
str: The generated text.
"""
# Load the model checkpoint
- checkpoint = torch.load(model_path, map_location=torch.device(device))
+ checkpoint = load_checkpoint(model_path, device)
# Initialize the model using the configuration from config.py
model = Transformer(
diff --git a/scripts/train_transformer.py b/scripts/train_transformer.py
index 8cbb14a..2e6f004 100644
--- a/scripts/train_transformer.py
+++ b/scripts/train_transformer.py
@@ -1,6 +1,7 @@
import torch
import torch.nn.functional as F
import os
+import time
from tqdm import tqdm
import numpy as np
from config.config import default_config as config
@@ -8,8 +9,51 @@ from src.models.transformer import Transformer
from data_loader.data_loader import get_batch_iterator
from typing import Dict
+
+def bytes_to_gib(num_bytes: int) -> float:
+ return num_bytes / (1024 ** 3)
+
+
+def get_device_report(device: str) -> str:
+ lines = [
+ f"PyTorch version: {torch.__version__}",
+ f"Configured device: {device}",
+ f"CUDA available: {torch.cuda.is_available()}",
+ f"CUDA version: {torch.version.cuda}",
+ ]
+
+ if device.startswith('cuda') and torch.cuda.is_available():
+ device_index = torch.cuda.current_device()
+ props = torch.cuda.get_device_properties(device_index)
+ total_vram_gib = bytes_to_gib(props.total_memory)
+ lines.extend([
+ f"GPU name: {torch.cuda.get_device_name(device_index)}",
+ f"GPU capability: {props.major}.{props.minor}",
+ f"Total VRAM: {total_vram_gib:.2f} GiB",
+ ])
+ else:
+ lines.append("GPU name: N/A (running without CUDA)")
+
+ return "\n".join(lines)
+
+
+def get_peak_memory_report(device: str) -> str:
+ if device.startswith('cuda') and torch.cuda.is_available():
+ peak_allocated = bytes_to_gib(torch.cuda.max_memory_allocated())
+ peak_reserved = bytes_to_gib(torch.cuda.max_memory_reserved())
+ return (
+ f"Peak VRAM allocated: {peak_allocated:.2f} GiB | "
+ f"Peak VRAM reserved: {peak_reserved:.2f} GiB"
+ )
+ return "Peak VRAM allocated: N/A | Peak VRAM reserved: N/A"
+
+
# --- Initialize the Model and Print Parameters ---
+print(get_device_report(config['device']))
+if config['device'].startswith('cuda') and torch.cuda.is_available():
+ torch.cuda.reset_peak_memory_stats()
+
model = Transformer(
n_head=config['n_head'],
n_embed=config['n_embed'],
@@ -24,61 +68,38 @@ print(f"Total number of parameters in the model: {total_params:,}")
# --- Optimizer Setup and Loss Tracking ---
-# Set up the AdamW optimizer with the specified learning rate.
optimizer = torch.optim.AdamW(model.parameters(), lr=config['t_lr'])
-
-# List to track loss values during training.
losses = []
-
-# Define a window size for averaging recent losses in the training loop.
AVG_WINDOW = 64
-# Helper function to estimate the average loss for training and development data.
+
@torch.no_grad()
def estimate_loss(steps: int) -> Dict[str, float]:
- """
- Evaluate the model on training and development datasets and calculate average loss.
-
- Args:
- steps (int): Number of steps to evaluate.
-
- Returns:
- dict: Dictionary containing average losses for 'train' and 'dev' splits.
- """
out = {}
- model.eval() # Set the model to evaluation mode.
+ model.eval()
for split in ['train', 'dev']:
- # Select the appropriate data path for the current split.
data_path = config['train_path'] if split == 'train' else config['dev_path']
-
- # Create a batch iterator for evaluation.
batch_iterator_eval = get_batch_iterator(
data_path, config['t_batch_size'], config['t_context_length'], device=config['device']
)
- # Initialize a tensor to track loss values for each evaluation step.
losses_eval = torch.zeros(steps)
for k in range(steps):
try:
- # Fetch a batch and calculate the loss.
xb, yb = next(batch_iterator_eval)
_, loss = model(xb, yb)
losses_eval[k] = loss.item()
except StopIteration:
- # Handle the case where the data iterator ends early.
print(f"Warning: Iterator for {split} ended early.")
break
- # Compute the mean loss for the current split.
out[split] = losses_eval[:k + 1].mean()
- model.train() # Restore the model to training mode.
+ model.train()
return out
-# --- Training Loop ---
-# Create a batch iterator for the training data.
batch_iterator = get_batch_iterator(
config['train_path'],
config['t_batch_size'],
@@ -86,53 +107,54 @@ batch_iterator = get_batch_iterator(
device=config['device']
)
-# Create a progress bar to monitor training progress.
+tokens_per_step = config['t_batch_size'] * config['t_context_length']
+last_eval_time = time.perf_counter()
+
pbar = tqdm(range(config['t_train_steps']))
for step in pbar:
try:
- # Fetch a batch of input and target data.
+ step_start_time = time.perf_counter()
xb, yb = next(batch_iterator)
-
- # Perform a forward pass and compute the loss.
_, loss = model(xb, yb)
- # Record the loss for tracking.
losses.append(loss.item())
pbar.set_description(f"Train loss: {np.mean(losses[-AVG_WINDOW:]):.4f}")
- # Backpropagate the loss and update the model parameters.
optimizer.zero_grad(set_to_none=True)
loss.backward()
optimizer.step()
- # Periodically evaluate the model on training and development data.
+ step_time = time.perf_counter() - step_start_time
+ tokens_per_second = tokens_per_step / step_time if step_time > 0 else float('inf')
+
if step % config['t_eval_steps'] == 0:
evaluation_losses = estimate_loss(config['t_eval_iters'])
train_loss = evaluation_losses['train']
dev_loss = evaluation_losses['dev']
- print(f"Step: {step}, Train loss: {train_loss:.4f}, Dev loss: {dev_loss:.4f}")
+ now = time.perf_counter()
+ elapsed_since_eval = now - last_eval_time
+ last_eval_time = now
+ print(
+ f"Step: {step}, Train loss: {train_loss:.4f}, Dev loss: {dev_loss:.4f}, "
+ f"Step time: {step_time:.3f}s, Throughput: {tokens_per_second:.2f} tokens/s, "
+ f"Elapsed since last eval: {elapsed_since_eval:.2f}s"
+ )
+ print(get_peak_memory_report(config['device']))
- # Decay the learning rate at the specified step.
if step == config['t_lr_decay_step']:
print('Decaying learning rate')
for g in optimizer.param_groups:
g['lr'] = config['t_lr_decayed']
except StopIteration:
- # Handle the case where the training data iterator ends early.
print("Training data iterator finished early.")
break
-# --- Save Model and Final Evaluation ---
-
-# Create the output directory if it does not exist.
os.makedirs(config['t_out_path'].split('/')[0], exist_ok=True)
-# Perform a final evaluation of the model on training and development datasets.
evaluation_losses = estimate_loss(200)
train_loss = evaluation_losses['train']
dev_loss = evaluation_losses['dev']
-# Ensure unique model save path in case the file already exists.
modified_model_out_path = config['t_out_path']
save_tries = 0
while os.path.exists(modified_model_out_path):
@@ -140,7 +162,6 @@ while os.path.exists(modified_model_out_path):
model_out_name = os.path.splitext(config['t_out_path'])[0]
modified_model_out_path = model_out_name + f"_{save_tries}" + ".pt"
-# Save the model's state dictionary, optimizer state, and training metadata.
torch.save(
{
'model_state_dict': model.state_dict(),
@@ -149,8 +170,12 @@ torch.save(
'train_loss': train_loss,
'dev_loss': dev_loss,
'steps': len(losses),
+ 'device': config['device'],
+ 'pytorch_version': torch.__version__,
+ 'cuda_version': torch.version.cuda,
},
modified_model_out_path
)
print(f"Saved model to {modified_model_out_path}")
-print(f"Finished training. Train loss: {train_loss:.4f}, Dev loss: {dev_loss:.4f}")
\ No newline at end of file
+print(get_peak_memory_report(config['device']))
+print(f"Finished training. Train loss: {train_loss:.4f}, Dev loss: {dev_loss:.4f}")